Merge branch 'id2'

# Conflicts:
#	packages/adapters/src/tmdb/index.ts
This commit is contained in:
root committed 2026-09-29 17:42:38 +08:00
commit f30600eafc
35 files changed
+13649 -92

No files matched your search

@@ -1,10 +1,11 @@
import { createHash } from "node:crypto";
import { type MetadataCandidate, TmdbError } from "@umt/adapters";
import { type MetadataCandidate, recallTmdbCandidates, TmdbError } from "@umt/adapters";
import type { EvidenceLevel, PoolCandidate } from "@umt/contracts";
import { type SourceReleaseRow, sql } from "@umt/db";
import {
type IdentityCandidate as DomainCandidate,
type IdentityResolution,
isGenericAliasName,
normalizeTitle,
PARSER_VERSION,
type ParsedRelease,
@@ -35,6 +36,7 @@ import {
resolveEntityId,
} from "./entities.ts";
import { detachFromIdentityDecisions, openGroupDecision, refreshGroupDecision } from "./identity-decisions.ts";
import { tmdbRecallInput, tmdbToCandidate } from "./tmdb-candidate.ts";
import { loadRecognizeCache, type RecognizeMeta, recognizeMetaOf, saveRecognizeCache } from "./tmdb-recognize-cache.ts";
// resolveObservation:parseRelease → 本地召回(别名 pg_trgm + 已有外部身份 + 人工锁)→ [按需] TMDB 候选 → domain.resolveIdentity。
@@ -97,26 +99,7 @@ export function evidenceVersion(parsed: ParsedRelease | null, ranked: RankedCand
return `p${PARSER_VERSION}:${h}`;
}
function yearOf(d: string | null | undefined): number | undefined {
return d && /^\d{4}/.test(d) ? Number(d.slice(0, 4)) : undefined;
}
export function tmdbToCandidate(m: MetadataCandidate): DomainCandidate {
const titles = [m.title, ...(m.originalTitle ? [m.originalTitle] : [])];
for (const t of m.titles) if (t.kind !== "alias" && !titles.includes(t.text)) titles.push(t.text);
const aliases = [...new Set([...m.aliases, ...m.titles.filter((t) => t.kind === "alias").map((t) => t.text)])];
const seasonYears = m.seasons?.map((s) => yearOf(s.airDate)).filter((y): y is number => y !== undefined);
return {
provider: "tmdb",
entityType: m.entityType,
externalId: m.externalId,
titles,
...(m.year !== null ? { year: m.year } : {}),
aliases,
...(seasonYears?.length ? { seasonYears } : {}),
...(m.externalIds?.imdb ? { linkedExternalIds: [{ provider: "imdb", id: m.externalIds.imdb }] } : {}),
};
}
export { tmdbToCandidate };
function poolCandidate(r: RankedCandidate): PoolCandidate {
const c = r.candidate;
@@ -208,15 +191,8 @@ async function tmdbCandidates(
return { remote: [], cacheHit: true, meta };
}
}
const queries = [parsed.titleCn, parsed.titleEn, parsed.title].filter((t): t is string => !!t);
const seen = new Map<string, MetadataCandidate>();
for (const q of [...new Set(queries)].slice(0, 2)) {
for (const m of await tmdb.searchAll(q)) {
const k = `${m.entityType}:${m.externalId}`;
if (!seen.has(k)) seen.set(k, m);
}
}
return { remote: [...seen.values()].slice(0, 20), cacheHit: false, meta };
const remote = await recallTmdbCandidates(tmdb, tmdbRecallInput(parsed));
return { remote, cacheHit: false, meta };
}
async function tmdbByExplicitIds(
@@ -512,6 +488,11 @@ function rankedSummary(r: RankedCandidate | undefined) {
return r ? { ...poolCandidate(r), titleMatch: r.titleMatch, yearMatch: r.yearMatch, typeMatch: r.typeMatch } : null;
}
/** 自动绑定后能否把站点标题登记为别名:标题精确命中,或别名命中且有第二种文字的名称佐证 */
export function siteAliasAllowed(best: RankedCandidate): boolean {
return best.titleMatch === "exact" || (best.titleMatch === "alias" && best.signals.nameLangs > 1);
}
async function bindAuto(
rt: SourcesRuntime,
release: SourceReleaseRow,
@@ -563,9 +544,17 @@ async function bindAuto(
.where("id", "=", entityId)
.executeTakeFirstOrThrow();
const units = await ensureUnits(trx, entityId, entity.kind, parsed);
// 站点标题作为别名:下次同名资源可直接本地召回
if (parsed.titleCn) await addAliasIfAbsent(trx, entityId, parsed.titleCn, "site_title", "site", "zh");
if (parsed.titleEn) await addAliasIfAbsent(trx, entityId, parsed.titleEn, "site_title", "site", "en");
// 站点标题作为别名:下次同名资源可直接本地召回。只在名称本身命中过(不是仅靠别名单线命中)时登记,
// 且通用词(分类词 / 单字 / 停用词)不登记,避免错误别名链式扩散(ID-2)
if (siteAliasAllowed(best)) {
for (const [t, lang] of [
[parsed.titleCn, "zh"],
[parsed.titleEn, "en"],
] as const) {
if (t && !isGenericAliasName(normalizeTitle(t)))
await addAliasIfAbsent(trx, entityId, t, "site_title", "site", lang);
}
}
const evidence = {
level,
@@ -784,6 +773,8 @@ async function toPool(
.set({
parser_version: cols.parserVersion,
parsed: jsonb(cols.parsed),
// 重新识别(标题更新 / 复查)后不再能确认:撤销之前的自动绑定(人工锁定的不走到这里)
...(release.media_entity_id ? { media_entity_id: null, content_unit_id: null } : {}),
resolution_status: toReleaseResolutionStatus(contractStatus),
resolution_evidence: jsonb({
reason,
@@ -1,3 +1,4 @@
import { createHash } from "node:crypto";
import { type SiteDetailPage, SiteError, type SitePage, type SiteRelease } from "@umt/adapters";
import type { SiteSnapshotKind } from "@umt/contracts";
import { type SiteRow, sql } from "@umt/db";
@@ -41,7 +42,14 @@ export async function upsertSiteRelease(
const parsed = parseRelease(r.title, r.subtitle ? { subtitle: r.subtitle } : {});
const c = siteReleaseToSourceRelease(r, { siteId: site.id, parsed });
const row = (
await sql<{ id: string; inserted: boolean }>`
await sql<{
id: string;
inserted: boolean;
resolution_locked: boolean;
prev_title: string | null;
prev_subtitle: string | null;
}>`
with prev as (select title, subtitle from source_release where site_id = ${site.id} and site_release_id = ${c.siteReleaseId})
insert into source_release (site_id, site_release_id, title, subtitle, category_raw, tags_raw, size_bytes, infohash_v1,
infohash_v2, promotion, promotion_expires_at, seeders, leechers, source_published_at, parsed, parser_version, raw,
detail_fetched_at)
@@ -51,6 +59,12 @@ export async function upsertSiteRelease(
${jsonb(c.parsed ?? {})}::jsonb, ${c.parserVersion ?? null}, ${jsonb(c.raw ?? {})}::jsonb, ${null})
on conflict (site_id, site_release_id) do update set
title = excluded.title, subtitle = coalesce(excluded.subtitle, source_release.subtitle),
parsed = case when source_release.title is distinct from excluded.title
or coalesce(excluded.subtitle, source_release.subtitle) is distinct from source_release.subtitle
then excluded.parsed else source_release.parsed end,
parser_version = case when source_release.title is distinct from excluded.title
or coalesce(excluded.subtitle, source_release.subtitle) is distinct from source_release.subtitle
then excluded.parser_version else source_release.parser_version end,
category_raw = coalesce(excluded.category_raw, source_release.category_raw), tags_raw = excluded.tags_raw,
size_bytes = coalesce(excluded.size_bytes, source_release.size_bytes),
infohash_v1 = coalesce(excluded.infohash_v1, source_release.infohash_v1),
@@ -60,7 +74,8 @@ export async function upsertSiteRelease(
source_published_at = coalesce(source_release.source_published_at, excluded.source_published_at),
raw = source_release.raw || excluded.raw, last_seen_at = now(),
availability = case when source_release.availability = 'expired' then 'active' else source_release.availability end
returning id, (xmax = 0) as inserted`.execute(trx)
returning id, (xmax = 0) as inserted, resolution_locked,
(select p.title from prev p) as prev_title, (select p.subtitle from prev p) as prev_subtitle`.execute(trx)
).rows[0]!;
if (row.inserted) {
await rt.ctx.outbox.emit(
@@ -69,8 +84,23 @@ export async function upsertSiteRelease(
{ sourceReleaseId: row.id, observationId: null, siteId: site.id, isNew: true },
{ idempotencyKey: `release.observed:${row.id}` },
);
} else if (
!row.resolution_locked &&
(row.prev_title !== c.title || (c.subtitle != null && row.prev_subtitle !== c.subtitle))
) {
// 标题 / 副标题更新(RSS 标题 → 列表标题等):按新标题重新解析并重新识别,不沿用旧解析(ID-2)
const digest = createHash("sha1")
.update(`${c.title}\n${c.subtitle ?? ""}`)
.digest("hex")
.slice(0, 16);
await rt.ctx.outbox.emit(
trx,
"release.observed",
{ sourceReleaseId: row.id, observationId: null, siteId: site.id, isNew: false },
{ idempotencyKey: `release.observed:${row.id}:${digest}` },
);
}
return row;
return { id: row.id, inserted: row.inserted };
}
/** 凭据失效:账号标失效 + 站点健康 failing + 待处理(不重试) */
@@ -0,0 +1,51 @@
import type { MetadataCandidate } from "@umt/adapters";
import { type IdentityCandidate, isGenericAliasName, normalizeTitle, type ParsedRelease } from "@umt/domain";
function yearOf(d: string | null | undefined): number | undefined {
return d && /^\d{4}/.test(d) ? Number(d.slice(0, 4)) : undefined;
}
/**
* TMDB 候选 → 身份候选。标题 = 主标题 / 原标题 / 其他语言的本地化标题(精确命中算 exact);
* 别名 = alternative_titles + translations(对照 MP __get_names,命中算 alias,ID-2 起译名不再算主标题)。
* 剧集带各季首播年份与季号(取了详情才有)。
*/
export function tmdbToCandidate(m: MetadataCandidate): IdentityCandidate {
const titles = [m.title, ...(m.originalTitle ? [m.originalTitle] : [])].filter(Boolean);
for (const t of m.titles)
if ((t.kind === "localized" || t.kind === "original") && !titles.includes(t.text)) titles.push(t.text);
const aliases = [
...new Set([
...m.aliases,
...m.titles.filter((t) => t.kind === "alias" || t.kind === "translation").map((t) => t.text),
]),
].filter((a) => !titles.includes(a));
const seasons = (m.seasons ?? []).filter((s) => s.seasonNumber > 0);
const seasonYears = seasons.map((s) => yearOf(s.airDate)).filter((y): y is number => y !== undefined);
return {
provider: "tmdb",
entityType: m.entityType,
externalId: m.externalId,
titles,
...(m.year !== null ? { year: m.year } : {}),
aliases,
...(seasonYears.length ? { seasonYears } : {}),
...(seasons.length ? { seasonNumbers: seasons.map((s) => s.seasonNumber) } : {}),
...(m.externalIds?.imdb ? { linkedExternalIds: [{ provider: "imdb", id: m.externalIds.imdb }] } : {}),
};
}
/**
* 召回输入(ID-2):中文名(含副标题补出的)在前、英文名在后,通用词不搜;剧集线索只搜 tv;年份用于挑选拉详情的候选。
* 对照 MP _prepare_search_names(cn_name → en_name)。
*/
export function tmdbRecallInput(parsed: ParsedRelease) {
const names = [parsed.titleCn, parsed.titleEn, parsed.title].filter(
(t): t is string => !!t && !isGenericAliasName(normalizeTitle(t)),
);
return {
names,
...(parsed.mediaTypeHint === "tv" ? { type: "tv" as const } : {}),
...(parsed.year !== undefined ? { year: parsed.year } : {}),
};
}
@@ -231,9 +231,15 @@ export const sourcesOutboxRoutes: Record<string, OutboxRouteFn> = {
},
"release.observed": (ev) => {
const p = ev.payload as P;
// 标题更新(isNew=false):按新标题重新解析并重新评估,已识别的也不短路(ID-2);触发来源仍是 observed(DEC-1 不变)
const retitled = p.isNew === false;
return {
task: "resolveObservation",
input: { sourceReleaseId: p.sourceReleaseId, observationId: p.observationId },
input: {
sourceReleaseId: p.sourceReleaseId,
observationId: p.observationId,
...(retitled ? { force: true, trigger: "observed" } : {}),
},
};
},
"feed.gap_suspected": (ev) => {
@@ -0,0 +1,238 @@
/**
* 身份匹配差分(ID-2):憨憨真实标题 → parseRelease → TMDB 候选召回(recallTmdbCandidates)→ resolveIdentity,
* 与 MoviePilot(MetaInfo + TheMovieDbModule 名称识别链,见 gen-identity-mp.py)逐条比对。
*
* 数据在 packages/domain/test/differential/identity/:
* samples.json 输入标题(用户栈只读导出)
* mp.json MP 侧期望(gen-identity-mp.py 生成)
* tmdb-fixtures.json 我们侧用到的 TMDB 响应(脱敏裁剪,测试离线回放)
* deviations.json 逐条归因(mp_defect / intentional / known_gap)
* poisoned-local.json 真机误识别:含分类词别名的本地作品(回归「电视剧」别名把 15 条资源并入一部剧)
* 重新录制:identity-differential.run.ts(走本地 TMDB 缓存代理)。
*/
import { readFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { recallTmdbCandidates, TmdbClient } from "@umt/adapters";
import { type IdentityCandidate, type IdentityResolution, parseRelease, resolveIdentity } from "@umt/domain";
import { localIdentityCandidate } from "@umt/mapping";
import { tmdbRecallInput, tmdbToCandidate } from "../../src/modules/sources-identity/service/tmdb-candidate.ts";
export const DATA_DIR = join(
dirname(fileURLToPath(import.meta.url)),
"../../../../packages/domain/test/differential/identity",
);
export interface Sample {
id: string;
work: string;
kind: string;
title: string;
subtitle: string | null;
}
export interface MpResult {
meta?: Record<string, unknown>;
names?: string[];
match: { tmdbId: number; type: "movie" | "tv"; title: string; year: number | null; by: string } | null;
error?: string;
}
export type Verdict = "consistent" | "ours_missed" | "ours_wrong" | "ours_extra";
export interface Deviation {
kind: "mp_defect" | "intentional" | "known_gap";
expect?: string | null;
note: string;
}
export interface Row {
id: string;
title: string;
parsed: { titleCn?: string; titleEn?: string; altTitles?: string[]; year?: number; season?: number };
mp: string | null;
ours: string | null;
status: IdentityResolution["status"];
reason: string;
best: string | null;
bestEvidence: string[];
verdict: Verdict;
deviation?: Deviation;
}
const readJson = <T>(f: string): T => JSON.parse(readFileSync(join(DATA_DIR, f), "utf8")) as T;
export function loadData() {
return {
samples: readJson<{ samples: Sample[] }>("samples.json").samples,
mp: readJson<{ results: Record<string, MpResult> }>("mp.json").results,
deviations: readJson<{ deviations: Record<string, Deviation> }>("deviations.json").deviations,
};
}
/** TMDB 请求的缓存键:路径 + 排序后的参数(去掉凭证) */
export function fixtureKey(url: string): string {
const u = new URL(url);
const params = [...u.searchParams.entries()].filter(([k]) => k !== "api_key").sort(([a], [b]) => a.localeCompare(b));
return `${u.pathname.replace(/^\/3/, "")}?${new URLSearchParams(params).toString()}`;
}
const pick = (o: Record<string, unknown>, keys: string[]) =>
Object.fromEntries(keys.filter((k) => o[k] !== undefined && o[k] !== null).map((k) => [k, o[k]]));
const BRIEF = [
"id",
"media_type",
"title",
"name",
"original_title",
"original_name",
"original_language",
"release_date",
"first_air_date",
"popularity",
"origin_country",
];
/** 只留识别用到的字段(标题、日期、别名、译名标题、外部 ID、季列表) */
export function trimTmdb(body: unknown): unknown {
const b = body as Record<string, unknown>;
if (Array.isArray(b.results)) {
return { results: (b.results as Record<string, unknown>[]).map((r) => pick(r, BRIEF)) };
}
const out = pick(b, BRIEF);
const alt = b.alternative_titles as Record<string, unknown[]> | undefined;
if (alt) {
const list = (alt.titles ?? alt.results ?? []) as Record<string, unknown>[];
out.alternative_titles = { [alt.titles ? "titles" : "results"]: list.map((a) => pick(a, ["title", "iso_3166_1"])) };
}
const tr = (b.translations as { translations?: Record<string, unknown>[] } | undefined)?.translations;
if (tr) {
out.translations = {
translations: tr
.map((t) => ({
iso_639_1: t.iso_639_1,
iso_3166_1: t.iso_3166_1,
data: pick((t.data ?? {}) as Record<string, unknown>, ["title", "name"]),
}))
.filter((t) => Object.keys(t.data).length),
};
}
const ext = b.external_ids as Record<string, unknown> | undefined;
if (ext) out.external_ids = pick(ext, ["imdb_id", "tvdb_id"]);
if (Array.isArray(b.seasons))
out.seasons = (b.seasons as Record<string, unknown>[]).map((s) =>
pick(s, ["season_number", "air_date", "episode_count", "name"]),
);
return out;
}
/** 离线回放:按键取录制的响应;没录到的请求直接报错(说明召回逻辑变了,需要重新录制) */
export function fixtureFetch(fixtures: Record<string, unknown>): typeof fetch {
return (async (input: string | URL | Request) => {
const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url;
const key = fixtureKey(url);
if (!(key in fixtures)) throw new Error(`TMDB 回放缺少请求 ${key}(召回逻辑变化,需重新录制)`);
const body = fixtures[key] as { __status?: number };
const status = body.__status ?? 200;
return new Response(JSON.stringify(status === 200 ? body : { status_message: "not found" }), {
status,
headers: { "content-type": "application/json" },
});
}) as typeof fetch;
}
export function tmdbFor(fetchImpl: typeof fetch, baseUrl = "http://tmdb.fixture/3"): TmdbClient {
return new TmdbClient({ apiKey: "x", baseUrl, fetch: fetchImpl, retries: 0 });
}
const keyOf = (c: { entityType: string; externalId: string } | undefined) =>
c ? `${c.entityType}:${c.externalId}` : null;
/** 单条:解析 → 召回 → 打分(local = 额外的本地候选,例如真机误识别场景) */
export async function resolveSample(tmdb: TmdbClient, s: Sample, local: IdentityCandidate[] = []) {
const parsed = parseRelease(s.title, s.subtitle ? { subtitle: s.subtitle } : {});
const remote = await recallTmdbCandidates(tmdb, tmdbRecallInput(parsed));
const resolution = resolveIdentity(parsed, [...local, ...remote.map(tmdbToCandidate)]);
return { parsed, resolution };
}
export async function runDifferential(tmdb: TmdbClient): Promise<Row[]> {
const { samples, mp, deviations } = loadData();
const rows: Row[] = [];
for (const s of samples) {
const { parsed, resolution } = await resolveSample(tmdb, s);
const m = mp[s.id]?.match ?? null;
const mpKey = m ? `${m.type}:${m.tmdbId}` : null;
const best = resolution.best?.candidate;
const ours = resolution.status === "auto" ? keyOf(best) : null;
const verdict: Verdict =
mpKey === ours ? "consistent" : !ours ? "ours_missed" : !mpKey ? "ours_extra" : "ours_wrong";
rows.push({
id: s.id,
title: s.title,
parsed: {
...(parsed.titleCn ? { titleCn: parsed.titleCn } : {}),
...(parsed.titleEn ? { titleEn: parsed.titleEn } : {}),
...(parsed.altTitles ? { altTitles: parsed.altTitles } : {}),
...(parsed.year !== undefined ? { year: parsed.year } : {}),
...(parsed.season !== undefined ? { season: parsed.season } : {}),
},
mp: mpKey,
ours,
status: resolution.status,
reason: resolution.reason,
best: best ? `${keyOf(best)} ${best.titles[0] ?? ""} ${best.year ?? ""}`.trim() : null,
bestEvidence: resolution.best ? [`level:${resolution.best.level}`, ...resolution.best.evidence] : [],
verdict,
...(deviations[s.id] ? { deviation: deviations[s.id] } : {}),
});
}
return rows;
}
/** 汇总:非故意偏离项的一致率(MP 缺陷且我们正确的计为一致) */
export function summarize(rows: Row[]) {
const scored = rows.filter((r) => r.deviation?.kind !== "intentional");
const ok = scored.filter(
(r) => r.verdict === "consistent" || (r.deviation?.kind === "mp_defect" && r.ours === (r.deviation.expect ?? null)),
);
const count = (v: Verdict) => rows.filter((r) => r.verdict === v).length;
return {
samples: rows.length,
works: new Set(loadData().samples.map((s) => s.work)).size,
consistent: count("consistent"),
oursMissed: count("ours_missed"),
oursWrong: count("ours_wrong"),
oursExtra: count("ours_extra"),
intentional: rows.length - scored.length,
mpDefect: rows.filter((r) => r.deviation?.kind === "mp_defect").length,
rate: scored.length ? ok.length / scored.length : 1,
};
}
export interface PoisonedCase {
title: string;
subtitle: string | null;
/** 正确的 TMDB 身份 type:id,或 not_local = 不得自动并入被污染的本地作品 */
expect: string;
}
/** 真机误识别回归:带「电视剧」等错误站点别名的本地作品 + 被误并入的资源,重新识别的结果 */
export async function runPoisoned(tmdb: TmdbClient) {
const d = readJson<{
entity: { id: string; kind: "series"; title: string; originalTitle: string; year: number };
aliases: string[];
identities: { provider: string; externalId: string; status: "accepted" }[];
cases: PoisonedCase[];
}>("poisoned-local.json");
const local = localIdentityCandidate(d.entity, d.aliases, d.identities)!;
const out: { c: PoisonedCase; status: string; best: string | null; local: boolean; ok: boolean }[] = [];
for (const c of d.cases) {
const s: Sample = { id: "poisoned", work: "", kind: "", title: c.title, subtitle: c.subtitle };
const { resolution } = await resolveSample(tmdb, s, [local]);
const best = resolution.best?.candidate;
const auto = resolution.status === "auto";
// 本地作品已映射 tmdb:312027:自动并入本地作品等价于识别为该 TMDB 身份
const key = !auto || !best ? null : best.provider === "local" ? "local" : keyOf(best);
const effective = key === "local" ? "tv:312027" : key;
const ok = c.expect === "not_local" ? key !== "local" : effective === c.expect;
out.push({ c, status: resolution.status, best: key, local: key === "local", ok });
}
return out;
}
@@ -0,0 +1,52 @@
/**
* 重新录制身份匹配差分(ID-2):我们侧的 TMDB 请求走本地缓存代理(scratchpad tmdb_proxy.py,上行计数封顶),
* 录下的响应裁剪后写入 tmdb-fixtures.json,逐条结果写入 ours.json 并打印汇总。
*
* TMDB_PROXY=http://127.0.0.1:18731 deno run -A apps/server/test/sources-identity/identity-differential.run.ts
*/
import { writeFileSync } from "node:fs";
import { join } from "node:path";
import {
DATA_DIR,
fixtureKey,
runDifferential,
runPoisoned,
summarize,
tmdbFor,
trimTmdb,
} from "./identity-differential.lib.ts";
const proxy = Deno.env.get("TMDB_PROXY") ?? "http://127.0.0.1:18731";
const fixtures: Record<string, unknown> = {};
const recording: typeof fetch = async (input, init) => {
const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url;
const res = await fetch(url, init);
const body = await res.clone().json();
// 录制时代理离线未命中的详情按 404 记下(回放同样走"取不到详情"分支)
fixtures[fixtureKey(url)] = res.ok ? trimTmdb(body) : { __status: res.status };
return res;
};
const tmdb = tmdbFor(recording, `${proxy}/3`);
const rows = await runDifferential(tmdb);
const poisoned = await runPoisoned(tmdb);
for (const p of poisoned) if (!p.ok) console.log("POISONED FAIL", p.c.title.slice(0, 60), p.status, p.best);
console.log("poisoned", poisoned.filter((p) => p.ok).length, "/", poisoned.length);
const sorted = Object.fromEntries(Object.entries(fixtures).sort(([a], [b]) => a.localeCompare(b)));
writeFileSync(join(DATA_DIR, "tmdb-fixtures.json"), `${JSON.stringify(sorted)}\n`);
writeFileSync(join(DATA_DIR, "ours.json"), `${JSON.stringify({ summary: summarize(rows), rows }, null, 1)}\n`);
for (const r of rows)
if (r.verdict !== "consistent")
console.log(
r.id,
r.verdict,
r.deviation?.kind ?? "",
"| mp",
r.mp,
"| ours",
r.status,
r.best,
r.bestEvidence.join(","),
"|",
r.title.slice(0, 50),
);
console.log(summarize(rows));
@@ -0,0 +1,65 @@
/**
* 身份匹配差分回归(ID-2):憨憨真实标题,MP 期望(mp.json)vs 我们(parse → 召回 → resolveIdentity),
* TMDB 响应离线回放(tmdb-fixtures.json)。说明见 identity-differential.lib.ts 与
* docs/porting/moviepilot-identity-match.md。
*/
import { readFileSync } from "node:fs";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import {
DATA_DIR,
fixtureFetch,
loadData,
runDifferential,
runPoisoned,
summarize,
tmdbFor,
} from "./identity-differential.lib.ts";
const fixtures = JSON.parse(readFileSync(join(DATA_DIR, "tmdb-fixtures.json"), "utf8")) as Record<string, unknown>;
const tmdb = () => tmdbFor(fixtureFetch(fixtures));
describe("身份匹配差分(MP recognize ↔ resolveIdentity)", async () => {
const rows = await runDifferential(tmdb());
const byId = new Map(rows.map((r) => [r.id, r]));
const { samples, deviations } = loadData();
it("样本 ≥ 100 条,覆盖「英文 [中文]」RSS、动漫、续作、通用词、短剧", () => {
expect(samples.length).toBeGreaterThanOrEqual(100);
const kinds = new Set(samples.map((s) => s.kind));
for (const k of ["rss_bracket", "title_subtitle", "short_drama"]) expect(kinds.has(k), k).toBe(true);
});
it("非故意偏离项一致率 ≥ 95%", () => {
expect(summarize(rows).rate).toBeGreaterThanOrEqual(0.95);
});
it("登记的偏离都对应存在的样本,且 mp_defect 的结果与登记一致", () => {
for (const [id, d] of Object.entries(deviations)) {
const r = byId.get(id);
expect(r, id).toBeDefined();
if (d.kind === "mp_defect") expect(r!.ours, id).toBe(d.expect ?? null);
}
});
it("未登记的样本全部与 MP 一致(新增不一致需要归因后登记)", () => {
const unexplained = rows.filter((r) => r.verdict !== "consistent" && !r.deviation).map((r) => `${r.id} ${r.title}`);
expect(unexplained).toEqual([]);
});
it("brief 三类:方括号中文名(武神主宰)、续作章节(物怪第三章)、通用词 + 副标题中文名(档案)", () => {
for (const r of rows.filter((x) => x.title.includes("Martial Master"))) expect(r.ours, r.id).toBe("tv:110181");
for (const r of rows.filter((x) => x.title.startsWith("Mononoke The Movie Chapter III")))
expect(r.ours, r.id).toBe("movie:1322562");
for (const r of rows.filter((x) => x.title.startsWith("Archives S01 2009"))) expect(r.ours, r.id).toBe("tv:201401");
});
});
describe("真机误识别回归:分类词别名污染的本地作品", async () => {
const results = await runPoisoned(tmdb());
it("被误并入《东北有个周东北》的资源重新识别到各自的作品,只有英文名时也不并入", () => {
expect(results.length).toBeGreaterThanOrEqual(15);
const bad = results.filter((r) => !r.ok).map((r) => `${r.c.title} → ${r.status} ${r.best}`);
expect(bad).toEqual([]);
});
});
@@ -1,5 +1,6 @@
import { CapabilityRegistry } from "@umt/agent";
import { sql } from "@umt/db";
import { PARSER_VERSION } from "@umt/domain";
import { createMockTmdb, type MockTmdb, TMDB_FIXTURES, type TmdbFixture } from "@umt/testkit";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { refreshMetadata } from "../../src/modules/sources-identity/service/metadata.ts";
@@ -82,7 +83,11 @@ describe.skipIf(!up)("resolveObservation / 身份与元数据", () => {
const created = await h.db.selectFrom("media_entity").selectAll().where("id", "=", oppId).executeTakeFirstOrThrow();
expect((created.metadata as Record<string, { source: string }>).overview?.source).toBe("tmdb");
const rel = await h.db.selectFrom("source_release").selectAll().where("id", "=", r.id).executeTakeFirstOrThrow();
expect(rel).toMatchObject({ resolution_status: "resolved", media_entity_id: oppId, parser_version: "1" });
expect(rel).toMatchObject({
resolution_status: "resolved",
media_entity_id: oppId,
parser_version: String(PARSER_VERSION),
});
expect((rel.parsed as { year: number }).year).toBe(2023);
const ev = (await h.outbox("release.resolved")).find((x) => x.payload.sourceReleaseId === r.id);
expect(ev?.payload).toMatchObject({ mediaEntityId: oppId, evidenceLevel: "rule_confirmed", contentUnitIds: [] });
@@ -0,0 +1,75 @@
/**
* ID-2:站点资源标题更新(RSS 带分类前缀 / 内嵌描述的标题 → 列表标题)后按新标题重新解析并重新识别,
* 不沿用旧解析;已识别的资源也不短路。
*/
import type { SiteRelease } from "@umt/adapters";
import { PARSER_VERSION } from "@umt/domain";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { upsertSiteRelease } from "../../src/modules/sources-identity/service/snapshot.ts";
import { sourcesOutboxRoutes } from "../../src/modules/sources-identity/tasks.ts";
import { createHarness, type Harness, pgAvailable, seedSite } from "./harness.ts";
const up = await pgAvailable();
const rel = (title: string, subtitle?: string): SiteRelease => ({
siteId: "x",
siteReleaseId: "9001",
title,
...(subtitle ? { subtitle } : {}),
category: { id: "402", name: "剧集" },
sizeBytes: null,
seeders: null,
leechers: null,
snatched: null,
promotion: { type: "none", downloadFactor: 1, uploadFactor: 1, expiresAt: null } as SiteRelease["promotion"],
hr: false,
uploadedAt: null,
detailUrl: "https://hhan.example/details.php?id=9001",
tags: [],
raw: {},
observedAt: new Date().toISOString(),
});
describe.skipIf(!up)("标题更新后重新解析 + 重新识别", () => {
let h: Harness;
let siteId: string;
beforeAll(async () => {
h = await createHarness("retitle");
siteId = (await seedSite(h, { baseUrl: "https://hhan.example/" })).site.id;
});
afterAll(async () => {
await h?.close();
});
it("RSS 标题 → 列表标题:parsed 按新标题重算,发 release.observed(isNew=false)并强制重新识别", async () => {
const rss =
"[电视剧]Irreplaceable S01 2026 1080p YK WEB-DL H.264 DTS5.1-HHWEB[无可替代 | 第03集 | 1080p | 类型: 剧情][510.37 MB]";
const first = await h.db.transaction().execute((trx) => upsertSiteRelease(h.rt, trx, { id: siteId }, rel(rss)));
expect(first.inserted).toBe(true);
let row = await h.db.selectFrom("source_release").selectAll().where("id", "=", first.id).executeTakeFirstOrThrow();
// 分类前缀不进片名;内嵌描述里的中文名补作中文名
expect(row.parsed).toMatchObject({ titleCn: "无可替代", titleEn: "Irreplaceable", category: "电视剧" });
const listTitle = "Irreplaceable S01 2026 1080p YK WEB-DL H.264 DTS5.1-HHWEB";
const again = await h.db
.transaction()
.execute((trx) => upsertSiteRelease(h.rt, trx, { id: siteId }, rel(listTitle, "无可替代 | 第04集 | 1080p")));
expect(again).toEqual({ id: first.id, inserted: false });
row = await h.db.selectFrom("source_release").selectAll().where("id", "=", first.id).executeTakeFirstOrThrow();
expect(row.title).toBe(listTitle);
expect(row.parser_version).toBe(String(PARSER_VERSION));
expect(row.parsed).toMatchObject({ titleCn: "无可替代", episodes: { start: 4, end: 4 } });
const events = (await h.outbox("release.observed")).filter((e) => e.payload.sourceReleaseId === first.id);
expect(events.map((e) => e.payload.isNew)).toEqual([true, false]);
const route = sourcesOutboxRoutes["release.observed"]!({ payload: events[1]!.payload } as never);
expect(route).toMatchObject({ task: "resolveObservation", input: { force: true, trigger: "observed" } });
// 同标题再次快照:不再发重新识别
await h.db
.transaction()
.execute((trx) => upsertSiteRelease(h.rt, trx, { id: siteId }, rel(listTitle, "无可替代 | 第04集 | 1080p")));
const after = (await h.outbox("release.observed")).filter((e) => e.payload.sourceReleaseId === first.id);
expect(after).toHaveLength(2);
});
});
+1
View File
@@ -10,6 +10,7 @@ const UNIT = [
"test/api-registry.test.ts",
"test/auth.test.ts",
"test/health.test.ts",
"test/sources-identity/identity-differential.test.ts",
"test/invoke-limits.test.ts",
"test/logger.test.ts",
"test/mapping-compat.test.ts",