Merge branch 'id2'

# Conflicts:
#	packages/adapters/src/tmdb/index.ts
This commit is contained in:
root committed 2026-09-29 17:42:38 +08:00
commit f30600eafc
35 files changed
+13649 -92

No files matched your search

@@ -1,10 +1,11 @@
import { createHash } from "node:crypto";
import { type MetadataCandidate, TmdbError } from "@umt/adapters";
import { type MetadataCandidate, recallTmdbCandidates, TmdbError } from "@umt/adapters";
import type { EvidenceLevel, PoolCandidate } from "@umt/contracts";
import { type SourceReleaseRow, sql } from "@umt/db";
import {
type IdentityCandidate as DomainCandidate,
type IdentityResolution,
isGenericAliasName,
normalizeTitle,
PARSER_VERSION,
type ParsedRelease,
@@ -35,6 +36,7 @@ import {
resolveEntityId,
} from "./entities.ts";
import { detachFromIdentityDecisions, openGroupDecision, refreshGroupDecision } from "./identity-decisions.ts";
import { tmdbRecallInput, tmdbToCandidate } from "./tmdb-candidate.ts";
import { loadRecognizeCache, type RecognizeMeta, recognizeMetaOf, saveRecognizeCache } from "./tmdb-recognize-cache.ts";
// resolveObservation:parseRelease → 本地召回(别名 pg_trgm + 已有外部身份 + 人工锁)→ [按需] TMDB 候选 → domain.resolveIdentity。
@@ -97,26 +99,7 @@ export function evidenceVersion(parsed: ParsedRelease | null, ranked: RankedCand
return `p${PARSER_VERSION}:${h}`;
}
function yearOf(d: string | null | undefined): number | undefined {
return d && /^\d{4}/.test(d) ? Number(d.slice(0, 4)) : undefined;
}
export function tmdbToCandidate(m: MetadataCandidate): DomainCandidate {
const titles = [m.title, ...(m.originalTitle ? [m.originalTitle] : [])];
for (const t of m.titles) if (t.kind !== "alias" && !titles.includes(t.text)) titles.push(t.text);
const aliases = [...new Set([...m.aliases, ...m.titles.filter((t) => t.kind === "alias").map((t) => t.text)])];
const seasonYears = m.seasons?.map((s) => yearOf(s.airDate)).filter((y): y is number => y !== undefined);
return {
provider: "tmdb",
entityType: m.entityType,
externalId: m.externalId,
titles,
...(m.year !== null ? { year: m.year } : {}),
aliases,
...(seasonYears?.length ? { seasonYears } : {}),
...(m.externalIds?.imdb ? { linkedExternalIds: [{ provider: "imdb", id: m.externalIds.imdb }] } : {}),
};
}
export { tmdbToCandidate };
function poolCandidate(r: RankedCandidate): PoolCandidate {
const c = r.candidate;
@@ -208,15 +191,8 @@ async function tmdbCandidates(
return { remote: [], cacheHit: true, meta };
}
}
const queries = [parsed.titleCn, parsed.titleEn, parsed.title].filter((t): t is string => !!t);
const seen = new Map<string, MetadataCandidate>();
for (const q of [...new Set(queries)].slice(0, 2)) {
for (const m of await tmdb.searchAll(q)) {
const k = `${m.entityType}:${m.externalId}`;
if (!seen.has(k)) seen.set(k, m);
}
}
return { remote: [...seen.values()].slice(0, 20), cacheHit: false, meta };
const remote = await recallTmdbCandidates(tmdb, tmdbRecallInput(parsed));
return { remote, cacheHit: false, meta };
}
async function tmdbByExplicitIds(
@@ -512,6 +488,11 @@ function rankedSummary(r: RankedCandidate | undefined) {
return r ? { ...poolCandidate(r), titleMatch: r.titleMatch, yearMatch: r.yearMatch, typeMatch: r.typeMatch } : null;
}
/** 自动绑定后能否把站点标题登记为别名:标题精确命中,或别名命中且有第二种文字的名称佐证 */
export function siteAliasAllowed(best: RankedCandidate): boolean {
return best.titleMatch === "exact" || (best.titleMatch === "alias" && best.signals.nameLangs > 1);
}
async function bindAuto(
rt: SourcesRuntime,
release: SourceReleaseRow,
@@ -563,9 +544,17 @@ async function bindAuto(
.where("id", "=", entityId)
.executeTakeFirstOrThrow();
const units = await ensureUnits(trx, entityId, entity.kind, parsed);
// 站点标题作为别名:下次同名资源可直接本地召回
if (parsed.titleCn) await addAliasIfAbsent(trx, entityId, parsed.titleCn, "site_title", "site", "zh");
if (parsed.titleEn) await addAliasIfAbsent(trx, entityId, parsed.titleEn, "site_title", "site", "en");
// 站点标题作为别名:下次同名资源可直接本地召回。只在名称本身命中过(不是仅靠别名单线命中)时登记,
// 且通用词(分类词 / 单字 / 停用词)不登记,避免错误别名链式扩散(ID-2)
if (siteAliasAllowed(best)) {
for (const [t, lang] of [
[parsed.titleCn, "zh"],
[parsed.titleEn, "en"],
] as const) {
if (t && !isGenericAliasName(normalizeTitle(t)))
await addAliasIfAbsent(trx, entityId, t, "site_title", "site", lang);
}
}
const evidence = {
level,
@@ -784,6 +773,8 @@ async function toPool(
.set({
parser_version: cols.parserVersion,
parsed: jsonb(cols.parsed),
// 重新识别(标题更新 / 复查)后不再能确认:撤销之前的自动绑定(人工锁定的不走到这里)
...(release.media_entity_id ? { media_entity_id: null, content_unit_id: null } : {}),
resolution_status: toReleaseResolutionStatus(contractStatus),
resolution_evidence: jsonb({
reason,
@@ -1,3 +1,4 @@
import { createHash } from "node:crypto";
import { type SiteDetailPage, SiteError, type SitePage, type SiteRelease } from "@umt/adapters";
import type { SiteSnapshotKind } from "@umt/contracts";
import { type SiteRow, sql } from "@umt/db";
@@ -41,7 +42,14 @@ export async function upsertSiteRelease(
const parsed = parseRelease(r.title, r.subtitle ? { subtitle: r.subtitle } : {});
const c = siteReleaseToSourceRelease(r, { siteId: site.id, parsed });
const row = (
await sql<{ id: string; inserted: boolean }>`
await sql<{
id: string;
inserted: boolean;
resolution_locked: boolean;
prev_title: string | null;
prev_subtitle: string | null;
}>`
with prev as (select title, subtitle from source_release where site_id = ${site.id} and site_release_id = ${c.siteReleaseId})
insert into source_release (site_id, site_release_id, title, subtitle, category_raw, tags_raw, size_bytes, infohash_v1,
infohash_v2, promotion, promotion_expires_at, seeders, leechers, source_published_at, parsed, parser_version, raw,
detail_fetched_at)
@@ -51,6 +59,12 @@ export async function upsertSiteRelease(
${jsonb(c.parsed ?? {})}::jsonb, ${c.parserVersion ?? null}, ${jsonb(c.raw ?? {})}::jsonb, ${null})
on conflict (site_id, site_release_id) do update set
title = excluded.title, subtitle = coalesce(excluded.subtitle, source_release.subtitle),
parsed = case when source_release.title is distinct from excluded.title
or coalesce(excluded.subtitle, source_release.subtitle) is distinct from source_release.subtitle
then excluded.parsed else source_release.parsed end,
parser_version = case when source_release.title is distinct from excluded.title
or coalesce(excluded.subtitle, source_release.subtitle) is distinct from source_release.subtitle
then excluded.parser_version else source_release.parser_version end,
category_raw = coalesce(excluded.category_raw, source_release.category_raw), tags_raw = excluded.tags_raw,
size_bytes = coalesce(excluded.size_bytes, source_release.size_bytes),
infohash_v1 = coalesce(excluded.infohash_v1, source_release.infohash_v1),
@@ -60,7 +74,8 @@ export async function upsertSiteRelease(
source_published_at = coalesce(source_release.source_published_at, excluded.source_published_at),
raw = source_release.raw || excluded.raw, last_seen_at = now(),
availability = case when source_release.availability = 'expired' then 'active' else source_release.availability end
returning id, (xmax = 0) as inserted`.execute(trx)
returning id, (xmax = 0) as inserted, resolution_locked,
(select p.title from prev p) as prev_title, (select p.subtitle from prev p) as prev_subtitle`.execute(trx)
).rows[0]!;
if (row.inserted) {
await rt.ctx.outbox.emit(
@@ -69,8 +84,23 @@ export async function upsertSiteRelease(
{ sourceReleaseId: row.id, observationId: null, siteId: site.id, isNew: true },
{ idempotencyKey: `release.observed:${row.id}` },
);
} else if (
!row.resolution_locked &&
(row.prev_title !== c.title || (c.subtitle != null && row.prev_subtitle !== c.subtitle))
) {
// 标题 / 副标题更新(RSS 标题 → 列表标题等):按新标题重新解析并重新识别,不沿用旧解析(ID-2)
const digest = createHash("sha1")
.update(`${c.title}\n${c.subtitle ?? ""}`)
.digest("hex")
.slice(0, 16);
await rt.ctx.outbox.emit(
trx,
"release.observed",
{ sourceReleaseId: row.id, observationId: null, siteId: site.id, isNew: false },
{ idempotencyKey: `release.observed:${row.id}:${digest}` },
);
}
return row;
return { id: row.id, inserted: row.inserted };
}
/** 凭据失效:账号标失效 + 站点健康 failing + 待处理(不重试) */
@@ -0,0 +1,51 @@
import type { MetadataCandidate } from "@umt/adapters";
import { type IdentityCandidate, isGenericAliasName, normalizeTitle, type ParsedRelease } from "@umt/domain";
function yearOf(d: string | null | undefined): number | undefined {
return d && /^\d{4}/.test(d) ? Number(d.slice(0, 4)) : undefined;
}
/**
* TMDB 候选 → 身份候选。标题 = 主标题 / 原标题 / 其他语言的本地化标题(精确命中算 exact);
* 别名 = alternative_titles + translations(对照 MP __get_names,命中算 alias,ID-2 起译名不再算主标题)。
* 剧集带各季首播年份与季号(取了详情才有)。
*/
export function tmdbToCandidate(m: MetadataCandidate): IdentityCandidate {
const titles = [m.title, ...(m.originalTitle ? [m.originalTitle] : [])].filter(Boolean);
for (const t of m.titles)
if ((t.kind === "localized" || t.kind === "original") && !titles.includes(t.text)) titles.push(t.text);
const aliases = [
...new Set([
...m.aliases,
...m.titles.filter((t) => t.kind === "alias" || t.kind === "translation").map((t) => t.text),
]),
].filter((a) => !titles.includes(a));
const seasons = (m.seasons ?? []).filter((s) => s.seasonNumber > 0);
const seasonYears = seasons.map((s) => yearOf(s.airDate)).filter((y): y is number => y !== undefined);
return {
provider: "tmdb",
entityType: m.entityType,
externalId: m.externalId,
titles,
...(m.year !== null ? { year: m.year } : {}),
aliases,
...(seasonYears.length ? { seasonYears } : {}),
...(seasons.length ? { seasonNumbers: seasons.map((s) => s.seasonNumber) } : {}),
...(m.externalIds?.imdb ? { linkedExternalIds: [{ provider: "imdb", id: m.externalIds.imdb }] } : {}),
};
}
/**
* 召回输入(ID-2):中文名(含副标题补出的)在前、英文名在后,通用词不搜;剧集线索只搜 tv;年份用于挑选拉详情的候选。
* 对照 MP _prepare_search_names(cn_name → en_name)。
*/
export function tmdbRecallInput(parsed: ParsedRelease) {
const names = [parsed.titleCn, parsed.titleEn, parsed.title].filter(
(t): t is string => !!t && !isGenericAliasName(normalizeTitle(t)),
);
return {
names,
...(parsed.mediaTypeHint === "tv" ? { type: "tv" as const } : {}),
...(parsed.year !== undefined ? { year: parsed.year } : {}),
};
}
@@ -231,9 +231,15 @@ export const sourcesOutboxRoutes: Record<string, OutboxRouteFn> = {
},
"release.observed": (ev) => {
const p = ev.payload as P;
// 标题更新(isNew=false):按新标题重新解析并重新评估,已识别的也不短路(ID-2);触发来源仍是 observed(DEC-1 不变)
const retitled = p.isNew === false;
return {
task: "resolveObservation",
input: { sourceReleaseId: p.sourceReleaseId, observationId: p.observationId },
input: {
sourceReleaseId: p.sourceReleaseId,
observationId: p.observationId,
...(retitled ? { force: true, trigger: "observed" } : {}),
},
};
},
"feed.gap_suspected": (ev) => {