Files
UHD_Media_Tool/apps/server/test/sources-identity/id-escalation.test.ts
T

449 lines
22 KiB
TypeScript

/**
* ID-3 识别升级链(样本驱动,零真实请求):标题识别不确定 → 抓站点详情页(观众 DET-2 脱敏样本 / 憨憨 HH-2 样本,经注入的 fetch)
* → IMDb / TMDB / 豆瓣 ID → TMDB(mock)→ 确定;每站每小时上限;RSS 入库不抓;事项不开 / 自动关闭;豆瓣条目缓存路径。
*/
import { readFileSync } from "node:fs";
import { sql } from "@umt/db";
import { normalizeTitle } from "@umt/domain";
import {
createMockTmdb,
HHANCLUB_DETAIL_PAGE_2026,
HHANCLUB_FILELIST_FRAGMENT,
type MockTmdb,
type TmdbFixture,
} from "@umt/testkit";
import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest";
import {
DOUBAN_SUBJECT_REGION,
doubanSubjectFromSnippet,
loadDoubanSubject,
saveDoubanSubject,
} from "../../src/modules/sources-identity/service/douban-subject.ts";
import {
ESCALATION_COUNTER_REGION,
escalationCount,
} from "../../src/modules/sources-identity/service/id-escalation.ts";
import { resolveRelease } from "../../src/modules/sources-identity/service/resolve.ts";
import { sharedCache } from "../../src/shared-cache.ts";
import { createHarness, type Harness, pgAvailable, seedSite } from "./harness.ts";
const pg = await pgAvailable();
const SAMPLES = new URL("../../../../packages/adapters/test/differential/inputs/detail/", import.meta.url);
const audiencesDetail = (id: string) => readFileSync(new URL(`rl-audiences-detail-${id}.html.txt`, SAMPLES), "utf8");
const fx = (
type: "movie" | "tv",
id: number,
en: string,
zh: string,
date: string,
imdb: string | null = null,
): TmdbFixture => ({
type,
id,
original_title: en,
original_language: "zh",
origin_country: ["CN"],
date,
titles: { "zh-CN": zh, "en-US": en },
popularity: 10,
alternative_titles: [],
external_ids: { imdb_id: imdb },
...(type === "tv"
? {
seasons: [
{ season_number: 1, air_date: date, names: { "zh-CN": "第 1 季", "en-US": "Season 1" }, episodes: [] },
],
}
: {}),
});
describe.skipIf(!pg)("ID-3 识别升级链:详情页 ID", () => {
let h: Harness;
let tmdb: MockTmdb;
let audiences: string;
let hhan: string;
const siteRequests: string[] = [];
const metaRequests: string[] = [];
beforeAll(async () => {
tmdb = await createMockTmdb({
fixtures: [
// 观众 727077《我不是大师》:站点英文名 A Prophet 与 TMDB 英文名对不上
fx("tv", 990002, "The Fortune Teller", "我不是大师", "2026-09-01", "tt36758489"),
// 观众 725858《Rose》:同名同年两部 → 标题识别有歧义
fx("movie", 990001, "Rose", "罗斯", "2026-03-01", "tt25399154"),
fx("movie", 990011, "Rose", "玫瑰", "2026-05-01", "tt0000011"),
// 憨憨详情样本:TMDB tv/290001(站点英文名 Example Show 搜不到)
fx("tv", 290001, "Sample Series", "示例剧集", "2026-08-01", "tt45950001"),
// 豆瓣路径:只凭豆瓣条目(标题 + 年份)匹配
fx("tv", 990021, "Lanterns of Qing", "青灯记", "2025-04-01"),
// bgm 路径:罗马音标题搜不到,bgm 条目中文名能搜到
fx("tv", 990031, "Frieren: Beyond Journey's End", "葬送的芙莉莲", "2023-09-29"),
],
});
h = await createHarness("id3", {
tmdbUrl: tmdb.url,
// 站点请求全部走样本;没有样本的地址直接报错(零真实请求)
fetch: async (input) => {
const u = new URL(typeof input === "string" ? input : input instanceof URL ? input.href : input.url);
siteRequests.push(`${u.host}${u.pathname}${u.search}`);
const id = u.searchParams.get("id") ?? "";
const html = (body: string) =>
new Response(body, { status: 200, headers: { "content-type": "text/html; charset=utf-8" } });
if (u.host === "audiences.me" && u.pathname === "/details.php") return html(audiencesDetail(id));
if (u.host === "hhanclub.net" && u.pathname === "/details.php") return html(HHANCLUB_DETAIL_PAGE_2026);
if (u.host === "hhanclub.net" && u.pathname === "/viewfilelist.php") return html(HHANCLUB_FILELIST_FRAGMENT);
throw new Error(`测试不允许的站点请求 ${u.href}`);
},
// bgm.tv:第二季条目(按站点给的 ID)与搜索(公开 BT 站兜底)
metaFetch: async (input, init) => {
const u = new URL(typeof input === "string" ? input : input instanceof URL ? input.href : input.url);
metaRequests.push(`${init?.method ?? "GET"} ${u.host}${u.pathname}`);
const json = (b: unknown) =>
new Response(JSON.stringify(b), { headers: { "content-type": "application/json" } });
if (u.host === "api.bgm.tv" && u.pathname === "/v0/subjects/515759")
return json({
id: 515759,
name: "葬送のフリーレン 第2期",
name_cn: "葬送的芙莉莲 第二季",
date: "2026-01-16",
});
if (u.host === "api.bgm.tv" && u.pathname === "/v0/search/subjects")
return json({
data: [{ id: 400602, name: "葬送のフリーレン", name_cn: "葬送的芙莉莲", date: "2023-09-29" }],
});
return new Response("", { status: 404 });
},
});
audiences = (await seedSite(h, { key: "audiences", baseUrl: "https://audiences.me/", cookie: "c_secure_uid=1" }))
.site.id;
hhan = (await seedSite(h, { key: "hhanclub", baseUrl: "https://hhanclub.net/", cookie: "c_secure_uid=2" })).site.id;
});
afterAll(async () => {
await h?.close();
await tmdb?.close();
});
beforeEach(async () => {
await setCap(10);
await sql`delete from kv_cache where region = ${ESCALATION_COUNTER_REGION}`.execute(h.db);
});
const setCap = async (cap: number) => {
await sql`insert into settings (key, value) values ('identity', ${JSON.stringify({ detailIdsPerSiteHour: cap })}::jsonb)
on conflict (key) do update set value = excluded.value`.execute(h.db);
};
const release = (siteId: string, siteReleaseId: string, title: string, raw: unknown = {}) =>
h.db
.insertInto("source_release")
.values({ site_id: siteId, site_release_id: siteReleaseId, title, raw: JSON.stringify(raw) } as never)
.returningAll()
.executeTakeFirstOrThrow();
const row = (id: string) =>
h.db.selectFrom("source_release").selectAll().where("id", "=", id).executeTakeFirstOrThrow();
const detailRequests = (id: string) =>
siteRequests.filter((r) => r.includes("/details.php") && r.includes(`id=${id}&`));
const tmdbIdOf = async (entityId: string) =>
(
await h.db
.selectFrom("external_identity")
.select("external_id")
.where("media_entity_id", "=", entityId)
.where("provider", "=", "tmdb")
.executeTakeFirstOrThrow()
).external_id;
const openDecisions = () =>
h.db
.selectFrom("decision")
.select(["id", "kind", "status"])
.where("kind", "in", ["identity_ambiguous", "identity_unresolved"])
.where("status", "in", ["open", "investigating", "awaiting_user"])
.execute();
it("观众 727077:列表只有豆瓣 ID,搜索结果标题认不出 → 抓详情取 IMDb → TMDB find → 自动确认;不登记对不上的站点名", async () => {
// 列表页只给了豆瓣 ID(样本 rl-audiences-list 里这一行没有 IMDb 徽标)
const r = await release(audiences, "727077", "A Prophet S01E16 2026 2160p iQIYI WEB-DL H265 HDR DDP5.1-ADWeb", {
_site: { doubanId: "37297001" },
});
const find0 = tmdb.callsTo(/\/3\/find\/tt36758489/).length;
const res = await resolveRelease(h.rt, { sourceReleaseId: r.id, trigger: "search" });
expect(res.outcome).toBe("auto");
expect(detailRequests("727077")).toHaveLength(1);
expect(tmdb.callsTo(/\/3\/find\/tt36758489/).length).toBe(find0 + 1);
expect(await tmdbIdOf(res.mediaEntityId!)).toBe("990002");
const after = await row(r.id);
expect(after.media_entity_id).toBe(res.mediaEntityId);
expect((after.raw as { _detail?: unknown })._detail).toMatchObject({ imdbId: "tt36758489", doubanId: "37297001" });
expect((after.raw as { _escalation?: { status: string } })._escalation?.status).toBe("fetched");
expect(after.resolution_evidence).toMatchObject({
escalation: { detail: "fetched", via: "site_id", pending: false },
});
// 站点英文名 A Prophet 与作品名对不上:只凭 ID 确认,不登记为别名(ID-2 守卫照旧)
const aliases = await h.db
.selectFrom("title_alias")
.select("alias")
.where("media_entity_id", "=", res.mediaEntityId!)
.where("source", "=", "site")
.execute();
expect(aliases.map((a) => a.alias)).not.toContain("A Prophet");
expect(await escalationCount(h.rt, audiences)).toBe(1);
// 再识别一次:详情已抓过,不再请求站点;IDs 在 raw._detail 里,第一轮就按 ID 认出
const again = await resolveRelease(h.rt, { sourceReleaseId: r.id, force: true, trigger: "manual" });
expect(["auto", "skipped"]).toContain(again.outcome);
expect(detailRequests("727077")).toHaveLength(1);
});
it("观众 725858:同名同年两部 → 有歧义的事项;手动重新识别抓详情 → IMDb 确认 → 事项自动关闭;详情页豆瓣信息块进豆瓣条目缓存", async () => {
const r = await release(audiences, "725858", "Rose 2026 1080p BluRay AVC DTS-HD MA 5.1-DIY@Audies");
// 不允许升级时(旧行为):歧义 → 开 identity_ambiguous
const first = await resolveRelease(h.rt, { sourceReleaseId: r.id, trigger: "manual", escalate: false });
expect(first.outcome).toBe("ambiguous");
expect(first.decisionId).not.toBeNull();
expect(detailRequests("725858")).toHaveLength(0);
const res = await resolveRelease(h.rt, { sourceReleaseId: r.id, force: true, trigger: "manual" });
expect(res.outcome).toBe("auto");
expect(await tmdbIdOf(res.mediaEntityId!)).toBe("990001");
expect(detailRequests("725858")).toHaveLength(1);
const d = await h.db
.selectFrom("decision")
.select(["status"])
.where("id", "=", first.decisionId!)
.executeTakeFirst();
expect(d?.status).toBe("resolved");
const pool = await h.db
.selectFrom("identity_candidate")
.select(["status", "resolved_media_entity_id"])
.where("source_release_id", "=", r.id)
.executeTakeFirstOrThrow();
expect(pool).toMatchObject({ status: "resolved", resolved_media_entity_id: res.mediaEntityId });
// 详情页信息块(◎片名 Rose / ◎译名 罗斯 / ◎年代 2026)+ 豆瓣 ID 35322132 → 豆瓣条目
const subject = await loadDoubanSubject(h.ctx, "35322132");
expect(subject).toMatchObject({ id: "35322132", title: "Rose", year: 2026, source: "site_detail" });
expect(subject?.aliases).toContain("罗斯");
});
it("憨憨详情样本:MediaInfo 里的 TMDB tv/290001 直取详情 → 自动确认", async () => {
const r = await release(hhan, "91003", "Example Show S01 2026 2160p YK WEB-DL H.265 DV HQ AAC2.0-HHWEB");
const res = await resolveRelease(h.rt, { sourceReleaseId: r.id, trigger: "search" });
expect(res.outcome).toBe("auto");
expect(await tmdbIdOf(res.mediaEntityId!)).toBe("290001");
expect(detailRequests("91003")).toHaveLength(1);
expect((await row(r.id)).resolution_evidence).toMatchObject({ escalation: { via: "site_id" } });
});
it("每站每小时上限:达到后推迟(不开事项、15 分钟内复查),不再请求站点;0 = 不抓", async () => {
await setCap(1);
const a = await release(audiences, "990101", "Nowhere Title 2026 1080p WEB-DL");
const b = await release(audiences, "990102", "Another Nowhere 2026 1080p WEB-DL");
const before = siteRequests.length;
await resolveRelease(h.rt, { sourceReleaseId: a.id, trigger: "search" });
const rb = await resolveRelease(h.rt, { sourceReleaseId: b.id, trigger: "search" });
// a 用掉名额(样本里没有这个 id 的页面,详情请求失败也算用掉;失败不阻塞,照常进池,6 小时内不再为识别重抓)
expect(siteRequests.length - before).toBe(1);
expect((await row(a.id)).resolution_evidence).toMatchObject({
escalation: { detail: "failed:fetch_failed", pending: false },
});
await resolveRelease(h.rt, { sourceReleaseId: a.id, force: true, trigger: "manual" });
expect((await row(a.id)).resolution_evidence).toMatchObject({ escalation: { detail: "skipped:recent_failure" } });
expect(siteRequests.length - before).toBe(1);
expect(rb.decisionId).toBeNull();
const pool = await h.db
.selectFrom("identity_candidate")
.select(["query", "next_review_at"])
.where("source_release_id", "=", b.id)
.executeTakeFirstOrThrow();
expect(pool.query).toMatchObject({ escalation: { detail: "deferred:cap_reached", pending: true } });
expect(pool.next_review_at!.getTime() - Date.now()).toBeLessThanOrEqual(15 * 60_000 + 5_000);
await setCap(0);
const c = await release(audiences, "990103", "Third Nowhere 2026 1080p WEB-DL");
await resolveRelease(h.rt, { sourceReleaseId: c.id, trigger: "search" });
expect((await row(c.id)).resolution_evidence).toMatchObject({ escalation: { detail: "skipped:disabled" } });
expect(siteRequests.length - before).toBe(1);
});
it("订阅相关的资源:RSS 入库时不抓详情、不开「没识别出作品」;复查时抓详情认出;旧事项自动关闭", async () => {
// 本地作品(如订阅)登记了别名「Example Show」,但没有 TMDB 身份、年份也对不上:资源与它相关(tracked_alias)却认不出
const e = await h.db
.insertInto("media_entity")
.values({ kind: "series", title: "Example Show", year: 1990 } as never)
.returning("id")
.executeTakeFirstOrThrow();
await sql`insert into title_alias (media_entity_id, alias, normalized, kind, source)
values (${e.id}, 'Example Show', ${normalizeTitle("Example Show")}, 'other', 'user')`.execute(h.db);
const r = await release(hhan, "91004", "Example Show S01E18 2027 2160p YK WEB-DL H.265 DV HQ AAC2.0-HHWEB");
// 旧行为(不升级):相关却认不出 → identity_unresolved
const old = await resolveRelease(h.rt, { sourceReleaseId: r.id, trigger: "review", force: true, escalate: false });
expect(old.decisionId).not.toBeNull();
const oldKind = (
await h.db.selectFrom("decision").select("kind").where("id", "=", old.decisionId!).executeTakeFirstOrThrow()
).kind;
// RSS 入库(observed):不抓详情,推迟到复查;不再保留事项
const obs = await resolveRelease(h.rt, { sourceReleaseId: r.id, force: true, trigger: "observed" });
expect(obs.decisionId).toBeNull();
expect(detailRequests("91004")).toHaveLength(0);
expect((await row(r.id)).resolution_evidence).toMatchObject({
escalation: { detail: "not_now:batch_ingest", pending: true },
});
const d1 = await h.db.selectFrom("decision").select("status").where("id", "=", old.decisionId!).executeTakeFirst();
expect(d1?.status).not.toBe("open");
// 复查(订阅相关):抓详情 → TMDB tv/290001(年份 2027 与 2026 差 1 年,在容差内)→ 自动确认
const rev = await resolveRelease(h.rt, { sourceReleaseId: r.id, force: true, trigger: "review" });
expect(detailRequests("91004")).toHaveLength(1);
expect(rev.outcome).toBe("auto");
expect(oldKind).toBe("identity_unresolved");
expect(await openDecisions()).toHaveLength(0);
});
it("豆瓣路径:详情只有豆瓣 ID → 缓存的豆瓣条目(搜索摘要抽取)标题 + 年份 → TMDB 匹配(ID-2 打分)", async () => {
// META-2 搜索摘要里抽到的豆瓣条目(不请求豆瓣)
const s = doubanSubjectFromSnippet(
{
url: "https://movie.douban.com/subject/36000099/",
title: "青灯记 (豆瓣)",
snippet: "导演: 某导演 主演: 甲 / 乙 类型: 剧情 / 古装 制片国家/地区: 中国大陆 年份: 2025 集数: 30",
},
new Date(),
);
expect(s).toMatchObject({ id: "36000099", title: "青灯记", year: 2025, source: "web_snippet" });
await saveDoubanSubject(h.ctx, s!);
// 详情已抓过,只给了豆瓣 ID
const r = await release(hhan, "91005", "Qing Deng Ji S01E03 2025 1080p WEB-DL H264 AAC-HHWEB", {
_detail: { doubanId: "36000099" },
});
await h.db.updateTable("source_release").set({ detail_fetched_at: new Date() }).where("id", "=", r.id).execute();
const res = await resolveRelease(h.rt, { sourceReleaseId: r.id, trigger: "manual" });
expect(res.outcome).toBe("auto");
expect(await tmdbIdOf(res.mediaEntityId!)).toBe("990021");
expect(detailRequests("91005")).toHaveLength(0);
expect((await row(r.id)).resolution_evidence).toMatchObject({
escalation: { via: "douban", douban: { id: "36000099", title: "青灯记", source: "web_snippet" } },
});
// 缓存里没有这个豆瓣条目、在线也查不到(404):只进池;有 ID 的资源不开「没识别出作品」
await sharedCache(h.ctx).delete(DOUBAN_SUBJECT_REGION, "36000098");
const r2 = await release(hhan, "91006", "Unknown Drama S01E01 2025 1080p WEB-DL", {
_detail: { doubanId: "36000098" },
});
await h.db.updateTable("source_release").set({ detail_fetched_at: new Date() }).where("id", "=", r2.id).execute();
const res2 = await resolveRelease(h.rt, { sourceReleaseId: r2.id, trigger: "manual" });
expect(res2.outcome).toBe("no_match");
expect(res2.decisionId).toBeNull();
expect(siteRequests.some((x) => x.includes("douban"))).toBe(false);
});
it("bgm 路径:AniBT 给的 bgm ID → 第二季条目去掉季名与年份 → TMDB;公开 BT 站没 ID 时按标题搜 bgm", async () => {
const pub = await h.db
.insertInto("site")
.values({
key: "anibt",
name: "AniBT",
kind: "public_bt",
adapter: "api",
base_url: "https://anibt.net/",
} as never)
.returning("id")
.executeTakeFirstOrThrow();
const r = await release(pub.id, "rel_A1", "[Nix-Raws] Sousou no Frieren S2 - 05 [1080p]", {
_site: { bangumiId: "515759" },
});
const res = await resolveRelease(h.rt, { sourceReleaseId: r.id, trigger: "manual" });
expect(res.outcome).toBe("auto");
expect(await tmdbIdOf(res.mediaEntityId!)).toBe("990031");
expect((await row(r.id)).resolution_evidence).toMatchObject({
escalation: { via: "bangumi", bangumi: { id: "515759", search: false } },
});
// 条目进缓存:再识别同 ID 不再请求
const n = metaRequests.length;
const r2 = await release(pub.id, "rel_A2", "[Nix-Raws] Sousou no Frieren S2 - 06 [1080p]", {
_site: { bangumiId: "515759" },
});
await resolveRelease(h.rt, { sourceReleaseId: r2.id, trigger: "manual" });
expect(metaRequests.length).toBe(n);
// 详情里的 bgm 条目(蜜柑番组页 / 简介链接)同样用上
const rd = await release(pub.id, "rel_A9", "[Nix-Raws] Sousou no Frieren S2 - 07 [1080p]", {
_detail: { bangumiId: "515759" },
});
await h.db.updateTable("source_release").set({ detail_fetched_at: new Date() }).where("id", "=", rd.id).execute();
const resd = await resolveRelease(h.rt, { sourceReleaseId: rd.id, trigger: "manual" });
expect(await tmdbIdOf(resd.mediaEntityId!)).toBe("990031");
const r3 = await release(pub.id, "rel_A3", "[SubsPlease] Sousou no Frieren - 03 (1080p)");
const res3 = await resolveRelease(h.rt, { sourceReleaseId: r3.id, trigger: "manual" });
expect(res3.outcome).toBe("auto");
expect(await tmdbIdOf(res3.mediaEntityId!)).toBe("990031");
expect(metaRequests).toContain("POST api.bgm.tv/v0/search/subjects");
// PT 站的资源不按标题搜 bgm
const m = metaRequests.length;
const r4 = await release(hhan, "91099", "Sousou no Frieren S01E03 1080p WEB-DL");
await resolveRelease(h.rt, { sourceReleaseId: r4.id, trigger: "manual" });
expect(metaRequests.length).toBe(m);
});
it("升级链没认出(豆瓣召回无匹配)时保留原有歧义:候选不丢,照常开有歧义的事项", async () => {
await saveDoubanSubject(h.ctx, {
id: "36000097",
title: "查无此片",
aliases: [],
// 年份对不上:按豆瓣条目召回 TMDB 拿不到候选
year: 1990,
source: "web_snippet",
ref: "x",
at: new Date().toISOString(),
});
const r = await release(audiences, "725859", "Rose 2026 2160p WEB-DL H265-ADWeb", {
_detail: { doubanId: "36000097" },
});
await h.db.updateTable("source_release").set({ detail_fetched_at: new Date() }).where("id", "=", r.id).execute();
const res = await resolveRelease(h.rt, { sourceReleaseId: r.id, trigger: "manual" });
expect(res.outcome).toBe("ambiguous");
expect(res.decisionId).not.toBeNull();
const d = await h.db.selectFrom("decision").select("kind").where("id", "=", res.decisionId!).executeTakeFirst();
expect(d?.kind).toBe("identity_ambiguous");
});
it("池条目补全前先走升级链:详情页给了 ID 就直接认出,返回 409,不做网页补全", async () => {
const r = await release(hhan, "91007", "Example Show S01 2026 2160p YK WEB-DL H.265 DV HQ AAC2.0-HHWEB");
const first = await resolveRelease(h.rt, { sourceReleaseId: r.id, trigger: "manual", escalate: false });
expect(first.outcome).toBe("no_match");
const pool = await h.db
.selectFrom("identity_candidate")
.select("id")
.where("source_release_id", "=", r.id)
.executeTakeFirstOrThrow();
const res = await h.req(`/identity-pool/${pool.id}/enrich`, { body: {} });
expect(res.status).toBe(409);
expect(((await res.json()) as { error: { message: string } }).error.message).toContain("《示例剧集》");
expect(detailRequests("91007")).toHaveLength(1);
expect((await row(r.id)).resolution_status).toBe("resolved");
});
it("站点详情来源的豆瓣条目不会被搜索摘要覆盖", async () => {
await saveDoubanSubject(h.ctx, {
id: "1",
title: "甲",
aliases: [],
year: 2020,
source: "site_detail",
ref: "x",
at: new Date().toISOString(),
});
await saveDoubanSubject(h.ctx, {
id: "1",
title: "乙",
aliases: [],
year: 2021,
source: "web_snippet",
ref: "y",
at: new Date().toISOString(),
});
expect((await loadDoubanSubject(h.ctx, "1"))?.title).toBe("甲");
});
});