From a6d655d2ce844aacdbc21b4a9a68306faf6d2c70 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 1 Oct 2026 00:15:20 +0800 Subject: [PATCH 1/4] feat(identity): trust site-provided IMDb/TMDB IDs in scoring (ID-3) MatchOptions.trustedIds: a candidate fetched by a site-given ID is at least multi_evidence even when titles differ; year/type conflicts stay candidates. Differential group site-ids-* against MP search/result.py _match_source. Co-Authored-By: Claude Opus 5.5 --- packages/domain/src/identity/match.ts | 20 ++++ .../differential/identity/gen-site-ids-mp.py | 62 ++++++++++++ .../identity/site-ids-deviations.json | 13 +++ .../differential/identity/site-ids-mp.json | 27 +++++ .../identity/site-ids-samples.json | 97 ++++++++++++++++++ packages/domain/test/identity-id3.test.ts | 98 +++++++++++++++++++ 6 files changed, 317 insertions(+) create mode 100644 packages/domain/test/differential/identity/gen-site-ids-mp.py create mode 100644 packages/domain/test/differential/identity/site-ids-deviations.json create mode 100644 packages/domain/test/differential/identity/site-ids-mp.json create mode 100644 packages/domain/test/differential/identity/site-ids-samples.json create mode 100644 packages/domain/test/identity-id3.test.ts diff --git a/packages/domain/src/identity/match.ts b/packages/domain/src/identity/match.ts index 97f9184d..25097e61 100644 --- a/packages/domain/src/identity/match.ts +++ b/packages/domain/src/identity/match.ts @@ -86,6 +86,13 @@ export interface MatchOptions { * rankCandidates / resolveIdentity 按池自动计算;单独调用 scoreCandidate 时可传入。 */ genericNames?: ReadonlySet; + /** + * ID-3:站点列表 / 详情页给出的外部 ID(IMDb / TMDB)。候选按这些 ID 取来(或已登记同一 ID)时视为同一作品的强证据: + * 标题对不上也至少 multi_evidence(可自动确认);年份冲突降为 needs_confirm(ID 可能挂错,只作候选); + * 续作序号冲突 / 类型冲突不放宽。对照 MP search/result.py `_match_source`:种子的 IMDb ID 与作品一致即判为命中,不再比标题。 + * 标题里的 `[tmdbid=…]` 不算(仍按 explicit_id 只提升一级)。 + */ + trustedIds?: readonly { provider: string; id: string }[]; } /** 池里撞名的阈值:单个拉丁词被 ≥ 3 个候选的名称包含 */ @@ -293,6 +300,19 @@ export function scoreCandidate(p: ParsedRelease, c: IdentityCandidate, opts: Mat evidence.push("type_conflict"); } if (explicit && level !== "rejected" && tm !== "none") level = raise(level); + const trusted = + opts.trustedIds?.some( + (x) => + (x.provider === c.provider && x.id === c.externalId) || + c.linkedExternalIds?.some((l) => l.provider === x.provider && l.id === x.id), + ) ?? false; + if (trusted && sequel !== "conflict" && ty !== "mismatch") { + evidence.push("site_id"); + if (ym === "mismatch") { + level = "needs_confirm"; + evidence.push("site_id_year_conflict"); + } else if (LEVEL_RANK[level] < LEVEL_RANK.multi_evidence) level = "multi_evidence"; + } return { candidate: c, level, diff --git a/packages/domain/test/differential/identity/gen-site-ids-mp.py b/packages/domain/test/differential/identity/gen-site-ids-mp.py new file mode 100644 index 00000000..bc7237c8 --- /dev/null +++ b/packages/domain/test/differential/identity/gen-site-ids-mp.py @@ -0,0 +1,62 @@ +"""身份匹配差分(ID-3 站点外部 ID 组):用 MP 上游 search/result.py `_match_source` 生成 site-ids-mp.json。 + +用法(MP 仓库根目录,用其 .venv;不访问网络): + cd /opt/moviepilot-plus/upstream/MoviePilot && PYTHONPATH=. \ + .venv/bin/python /opt/uhd_media_tool/packages/domain/test/differential/identity/gen-site-ids-mp.py + +每条样本 = 站点种子(标题 / 副标题 / 站点给出的 IMDb ID)+ 目标作品(TMDB 标题 / 年份 / 类型 / IMDb ID)。 +记录 MP `_match_source` 的判定:种子 IMDb 与作品一致时返回 "imdb"(不再比标题),否则走标题匹配。 +只记录行为输出,不复制 MP 源码。 +""" + +import json +import pathlib +import subprocess + +from app.chain.search.result import _match_source +from app.domain.context import MediaInfo, TorrentInfo +from app.domain.metainfo import MetaInfo +from app.schemas.types import MediaSource, MediaType + +HERE = pathlib.Path(__file__).parent +samples = json.loads((HERE / "site-ids-samples.json").read_text(encoding="utf-8"))["samples"] + + +def run(s: dict) -> dict: + t = s["torrent"] + w = s["work"] + torrent = TorrentInfo( + site=1, + site_name="site", + title=t["title"], + description=t.get("subtitle"), + media_source=MediaSource.IMDb if t.get("imdbId") else None, + media_id=t.get("imdbId"), + ) + meta = MetaInfo(t["title"], t.get("subtitle")) + media = MediaInfo() + media.type = MediaType.MOVIE if w["type"] == "movie" else MediaType.TV + media.title = w["title"] + media.original_title = w.get("originalTitle") or w["title"] + media.en_title = w.get("originalTitle") or w["title"] + media.year = str(w["year"]) + media.tmdb_id = w["tmdbId"] + media.imdb_id = w.get("imdbId") + media.names = [] + try: + by = _match_source(torrent, meta, media, {}) + except Exception as e: # noqa: BLE001 - 标题匹配分支需要的上下文缺失时记为未命中 + return {"match": None, "error": f"{type(e).__name__}: {e}"} + return {"match": by} + + +out = { + "generatedFrom": { + "repo": "MoviePilot", + "commit": subprocess.check_output(["git", "rev-parse", "--short", "HEAD"], text=True).strip(), + "entry": "app.chain.search.result._match_source", + }, + "results": {s["id"]: run(s) for s in samples}, +} +(HERE / "site-ids-mp.json").write_text(json.dumps(out, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") +print(json.dumps(out["results"], ensure_ascii=False, indent=1)) diff --git a/packages/domain/test/differential/identity/site-ids-deviations.json b/packages/domain/test/differential/identity/site-ids-deviations.json new file mode 100644 index 00000000..edfce1cd --- /dev/null +++ b/packages/domain/test/differential/identity/site-ids-deviations.json @@ -0,0 +1,13 @@ +{ + "note": "ID-3 站点外部 ID 组的逐条归因。MP `_match_source` 只要种子 IMDb 与作品一致就判命中;我们把站点 ID 当强证据但保留年份 / 类型硬规则。", + "deviations": { + "S-004": { + "kind": "intentional", + "note": "年份差 37 年:站点 IMDb 链接挂错(挂到同名新片)的典型。我们只作候选(needs_confirm → 待确认),不自动绑定;MP 直接按 IMDb 命中" + }, + "S-006": { + "kind": "intentional", + "note": "剧集标题(S01)配电影 IMDb:类型冲突,只作候选;MP 按 IMDb 命中" + } + } +} diff --git a/packages/domain/test/differential/identity/site-ids-mp.json b/packages/domain/test/differential/identity/site-ids-mp.json new file mode 100644 index 00000000..1006e3ee --- /dev/null +++ b/packages/domain/test/differential/identity/site-ids-mp.json @@ -0,0 +1,27 @@ +{ + "generatedFrom": { + "repo": "MoviePilot", + "commit": "e6820a8", + "entry": "app.chain.search.result._match_source" + }, + "results": { + "S-001": { + "match": "imdb" + }, + "S-002": { + "match": "imdb" + }, + "S-003": { + "match": "imdb" + }, + "S-004": { + "match": "imdb" + }, + "S-005": { + "match": "imdb" + }, + "S-006": { + "match": "imdb" + } + } +} diff --git a/packages/domain/test/differential/identity/site-ids-samples.json b/packages/domain/test/differential/identity/site-ids-samples.json new file mode 100644 index 00000000..e5e0961f --- /dev/null +++ b/packages/domain/test/differential/identity/site-ids-samples.json @@ -0,0 +1,97 @@ +{ + "note": "ID-3 站点外部 ID 组:种子标题 / 副标题取自观众(DET-2)详情样本与憨憨样本,IMDb ID 为样本页面上的链接;作品侧 TMDB id / 标题为测试夹具(不是 TMDB 真实数据)。expectOurs = 我们在 trustedIds 下的判定(auto / ambiguous / no_match)。", + "samples": [ + { + "id": "S-001", + "note": "观众 725858:标题与作品名一致,IMDb 一致", + "torrent": { + "title": "Rose 2026 1080p BluRay AVC DTS-HD MA 5.1-DIY@Audies", + "subtitle": "罗斯 / 扑朔迷离的露丝(港)【德版原盘DIY 简繁字幕】", + "imdbId": "tt25399154" + }, + "work": { + "type": "movie", + "tmdbId": 990001, + "title": "罗斯", + "originalTitle": "Rose", + "year": 2026, + "imdbId": "tt25399154" + }, + "expectOurs": "auto" + }, + { + "id": "S-002", + "note": "观众 727077:英文名(A Prophet)与作品英文名对不上,IMDb 一致", + "torrent": { + "title": "A Prophet S01E16 2026 2160p iQIYI WEB-DL H265 HDR DDP5.1-ADWeb", + "subtitle": "我不是大师 第16集 | 类型：剧情 悬疑 | 主演：李现 李一桐 吴刚 刘冠麟 果靖霖 | 长风起 / 将相 / 江相 / 我是个算命先生", + "imdbId": "tt36758489" + }, + "work": { + "type": "tv", + "tmdbId": 990002, + "title": "我不是大师", + "originalTitle": "The Fortune Teller", + "year": 2026, + "imdbId": "tt36758489" + }, + "expectOurs": "auto" + }, + { + "id": "S-003", + "note": "标题完全对不上(站点用拼音名),IMDb 一致", + "torrent": { "title": "Wu Ji 2005 1080p BluRay x264-WiKi", "imdbId": "tt0417976" }, + "work": { + "type": "movie", + "tmdbId": 990003, + "title": "The Promise", + "originalTitle": "The Promise", + "year": 2005, + "imdbId": "tt0417976" + }, + "expectOurs": "auto" + }, + { + "id": "S-004", + "note": "IMDb 一致但年份差 37 年(站点 IMDb 链接挂错的典型)", + "torrent": { "title": "Dune 1984 1080p BluRay x264-GRP", "imdbId": "tt1160419" }, + "work": { + "type": "movie", + "tmdbId": 990004, + "title": "Dune", + "originalTitle": "Dune", + "year": 2021, + "imdbId": "tt1160419" + }, + "expectOurs": "ambiguous" + }, + { + "id": "S-005", + "note": "IMDb 一致,标题对不上,年份差 1 年(容差内)", + "torrent": { "title": "Hero 2003 1080p BluRay x264-GRP", "imdbId": "tt0299977" }, + "work": { + "type": "movie", + "tmdbId": 990005, + "title": "英雄", + "originalTitle": "英雄", + "year": 2002, + "imdbId": "tt0299977" + }, + "expectOurs": "auto" + }, + { + "id": "S-006", + "note": "剧集标题(S01)配电影的 IMDb:类型冲突", + "torrent": { "title": "Sherlock S01 2010 1080p BluRay x264-GRP", "imdbId": "tt1475582" }, + "work": { + "type": "movie", + "tmdbId": 990006, + "title": "Sherlock Holmes", + "originalTitle": "Sherlock Holmes", + "year": 2009, + "imdbId": "tt1475582" + }, + "expectOurs": "ambiguous" + } + ] +} diff --git a/packages/domain/test/identity-id3.test.ts b/packages/domain/test/identity-id3.test.ts new file mode 100644 index 00000000..fc371e77 --- /dev/null +++ b/packages/domain/test/identity-id3.test.ts @@ -0,0 +1,98 @@ +/** + * ID-3:站点列表 / 详情页给出的外部 ID(trustedIds)作为强证据。 + * 差分组:packages/domain/test/differential/identity/site-ids-*.json(MP search/result.py `_match_source`, + * 由 gen-site-ids-mp.py 生成)。MP 只要种子 IMDb 与作品一致就判命中;我们年份冲突 / 类型冲突时只作候选(登记的偏离)。 + */ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; +import { type IdentityCandidate, parseRelease, resolveIdentity, scoreCandidate } from "../src/index.ts"; + +const DIR = new URL("./differential/identity/", import.meta.url); +const read = (name: string): T => JSON.parse(readFileSync(new URL(name, DIR), "utf8")) as T; + +interface Sample { + id: string; + note: string; + torrent: { title: string; subtitle?: string; imdbId?: string }; + work: { type: "movie" | "tv"; tmdbId: number; title: string; originalTitle?: string; year: number; imdbId?: string }; + expectOurs: "auto" | "ambiguous" | "no_match"; +} + +const samples = read<{ samples: Sample[] }>("site-ids-samples.json").samples; +const mp = read<{ results: Record }>("site-ids-mp.json").results; +const deviations = read<{ deviations: Record }>( + "site-ids-deviations.json", +).deviations; + +const candidateOf = (w: Sample["work"]): IdentityCandidate => ({ + provider: "tmdb", + entityType: w.type, + externalId: String(w.tmdbId), + titles: [...new Set([w.title, ...(w.originalTitle ? [w.originalTitle] : [])])], + year: w.year, + aliases: [], + ...(w.imdbId ? { linkedExternalIds: [{ provider: "imdb", id: w.imdbId }] } : {}), +}); + +const ours = (s: Sample) => { + const p = parseRelease(s.torrent.title, s.torrent.subtitle ? { subtitle: s.torrent.subtitle } : {}); + const trustedIds = s.torrent.imdbId ? [{ provider: "imdb", id: s.torrent.imdbId }] : []; + return resolveIdentity(p, [candidateOf(s.work)], { trustedIds }); +}; + +describe("差分:站点外部 ID(MP _match_source)", () => { + it("样本与 MP 输出一一对应", () => { + expect(samples.length).toBeGreaterThanOrEqual(6); + for (const s of samples) expect(mp[s.id], s.id).toBeDefined(); + }); + + for (const s of samples) { + it(`${s.id} ${s.note}`, () => { + const r = ours(s); + expect(r.status, s.id).toBe(s.expectOurs); + const mpMatched = mp[s.id]!.match !== null; + const oursMatched = r.status === "auto"; + if (mpMatched === oursMatched) expect(deviations[s.id], `${s.id} 已一致,请删除登记`).toBeUndefined(); + else expect(deviations[s.id]?.kind, `${s.id} 与 MP 不一致但没有登记`).toBe("intentional"); + }); + } +}); + +describe("trustedIds 评分", () => { + const movie: IdentityCandidate = { + provider: "tmdb", + entityType: "movie", + externalId: "872585", + titles: ["Oppenheimer", "奥本海默"], + year: 2023, + aliases: [], + linkedExternalIds: [{ provider: "imdb", id: "tt15398776" }], + }; + it("标题对不上:trusted → multi_evidence(auto);不 trusted 仍是 rejected", () => { + const p = parseRelease("Something Else 2023 1080p WEB-DL"); + expect(scoreCandidate(p, movie).level).toBe("rejected"); + const r = scoreCandidate(p, movie, { trustedIds: [{ provider: "imdb", id: "tt15398776" }] }); + expect(r.level).toBe("multi_evidence"); + expect(r.evidence).toContain("site_id"); + }); + it("TMDB ID 直接对上候选同样算", () => { + const p = parseRelease("Something Else 2023 1080p WEB-DL"); + expect(scoreCandidate(p, movie, { trustedIds: [{ provider: "tmdb", id: "872585" }] }).level).toBe("multi_evidence"); + }); + it("标题也对上时不降级(rule_confirmed 保留)", () => { + const p = parseRelease("Oppenheimer 2023 1080p WEB-DL"); + expect(scoreCandidate(p, movie, { trustedIds: [{ provider: "imdb", id: "tt15398776" }] }).level).toBe( + "rule_confirmed", + ); + }); + it("别的 ID 不影响", () => { + const p = parseRelease("Something Else 2023 1080p WEB-DL"); + expect(scoreCandidate(p, movie, { trustedIds: [{ provider: "imdb", id: "tt0000001" }] }).level).toBe("rejected"); + }); + it("年份冲突 → needs_confirm(ID 可能挂错)", () => { + const p = parseRelease("Oppenheimer 1990 1080p WEB-DL"); + const r = scoreCandidate(p, movie, { trustedIds: [{ provider: "imdb", id: "tt15398776" }] }); + expect(r.level).toBe("needs_confirm"); + expect(r.evidence).toContain("site_id_year_conflict"); + }); +}); From cba214e93eda35a59d9ac6e0b7c19ee47ab017bc Mon Sep 17 00:00:00 2001 From: root Date: Thu, 1 Oct 2026 00:15:20 +0800 Subject: [PATCH 2/4] feat(sites): extract douban IDs from list rows (audiences) (ID-3) Co-Authored-By: Claude Opus 5.5 --- .../src/sites/definitions/audiences.json | 10 +++++++ packages/adapters/src/sites/spider/map.ts | 1 + packages/adapters/src/sites/spider/spider.ts | 10 +++++++ .../test/differential/list-ids.test.ts | 30 +++++++++++++++++++ .../movieclaw-overrides/audiences.json | 12 +++++++- 5 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 packages/adapters/test/differential/list-ids.test.ts diff --git a/packages/adapters/src/sites/definitions/audiences.json b/packages/adapters/src/sites/definitions/audiences.json index b657b49e..3a8c83d8 100644 --- a/packages/adapters/src/sites/definitions/audiences.json +++ b/packages/adapters/src/sites/definitions/audiences.json @@ -152,6 +152,16 @@ } ] }, + "doubanid": { + "selector": "a[href*=\"douban.com/subject/\"]", + "attribute": "href", + "filters": [ + { + "name": "re_search", + "args": ["subject/(\\d+)", 1] + } + ] + }, "freedate": { "selector": "img[class^=\"pro_\"]", "attribute": "title", diff --git a/packages/adapters/src/sites/spider/map.ts b/packages/adapters/src/sites/spider/map.ts index 1215b9b1..0fab73cc 100644 --- a/packages/adapters/src/sites/spider/map.ts +++ b/packages/adapters/src/sites/spider/map.ts @@ -78,5 +78,6 @@ export function spiderTorrentToRelease( }; if (t.description) release.subtitle = t.description; if (t.imdbid) release.imdbId = t.imdbid; + if (t.doubanid) release.doubanId = t.doubanid; return release; } diff --git a/packages/adapters/src/sites/spider/spider.ts b/packages/adapters/src/sites/spider/spider.ts index d197da06..ece36689 100644 --- a/packages/adapters/src/sites/spider/spider.ts +++ b/packages/adapters/src/sites/spider/spider.ts @@ -67,6 +67,8 @@ export interface SpiderTorrent { seeders?: number; size?: number; imdbid?: string; + /** UMT 扩展(ID-3,MP 没有):列表行里的豆瓣 subject id,字段 `doubanid` 与 imdbid 同样的选择器写法 */ + doubanid?: string; downloadvolumefactor?: number; uploadvolumefactor?: number; pubdate?: string; @@ -476,6 +478,7 @@ export class SiteSpider { this.getSeeders(row); this.getSize(row); this.getImdbid(row); + this.getDoubanid(row); this.getVolumeFactor(row, "downloadvolumefactor"); this.getVolumeFactor(row, "uploadvolumefactor"); this.getPubdate(row); @@ -659,6 +662,13 @@ export class SiteSpider { } } + private getDoubanid(row: Sel) { + if (!("doubanid" in this.fields)) return; + const selector = this.fields.doubanid ?? {}; + const v = this.filterText(this.safeQuery(row, selector), selector.filters); + if (v != null && /^\d{4,10}$/.test(String(v))) this.info.doubanid = String(v); + } + private getImdbid(row: Sel) { if (!("imdbid" in this.fields)) return; const selector = this.fields.imdbid ?? {}; diff --git a/packages/adapters/test/differential/list-ids.test.ts b/packages/adapters/test/differential/list-ids.test.ts new file mode 100644 index 00000000..7d3b187c --- /dev/null +++ b/packages/adapters/test/differential/list-ids.test.ts @@ -0,0 +1,30 @@ +/** + * ID-3 列表页外部 ID:观众列表行右侧的评分徽标带 IMDb / 豆瓣链接(DET-2 列表样本 rl-audiences-list,脱敏)。 + * imdbid 是 MP 字段;doubanid 是 UMT 扩展字段(MP SiteSpider 不读,索引差分只比 MP 字段,不受影响)。 + */ +import { describe, expect, it } from "vitest"; +import { audiencesDefinition, SiteSpider } from "../../src/sites/index.ts"; +import { spiderTorrentToRelease } from "../../src/sites/spider/map.ts"; +import { readInput } from "./diff-util.ts"; + +describe("观众列表页:IMDb / 豆瓣 ID(真实样本)", () => { + const html = readInput("inputs/indexer/rl-audiences-list.html.txt"); + const rows = new SiteSpider(audiencesDefinition, {}).parse(html).torrents; + const byId = (id: string) => rows.find((r) => r.page_url?.includes(`id=${id}&`)); + + it("有豆瓣徽标的行都抽到豆瓣 ID,有 IMDb 徽标的抽到 IMDb", () => { + expect(byId("725858")).toMatchObject({ imdbid: "tt25399154", doubanid: "35322132" }); + expect(byId("726249")).toMatchObject({ imdbid: "tt0039689", doubanid: "1302338" }); + // 剧集行只有豆瓣(IMDb 在详情页里) + const prophet = byId("727077")!; + expect(prophet.doubanid).toBe("37297001"); + expect(prophet.imdbid).toBeUndefined(); + expect(rows.filter((r) => r.doubanid).length).toBe(10); + }); + + it("映射到 SiteRelease.doubanId / imdbId", () => { + const r = spiderTorrentToRelease(byId("727077")!, audiencesDefinition, "2026-09-30T00:00:00.000Z"); + expect(r).toMatchObject({ siteReleaseId: "727077", doubanId: "37297001" }); + expect(r?.imdbId).toBeUndefined(); + }); +}); diff --git a/scripts/mp-sync/movieclaw-overrides/audiences.json b/scripts/mp-sync/movieclaw-overrides/audiences.json index 23510f09..9dfd2012 100644 --- a/scripts/mp-sync/movieclaw-overrides/audiences.json +++ b/scripts/mp-sync/movieclaw-overrides/audiences.json @@ -1,5 +1,5 @@ { - "$comment": "W1-3b 手写观众定义(按 MP indexer schema;仅对 testkit mock 验证)。深合并到 MovieClaw v0.27.0 转换结果之上,本文件的值优先,MovieClaw 值只补空。 umt.detail.fields=null:保留 W1-3b rowhead 详情模型(已对 mock 验证),不采用 MovieClaw 默认逐字段选择器。 DET-2(2026-09-30 真实账号核对列表页 1 次 + 详情页 2 次):列表副标题取 span.torrent-subtitle-text(span.torrent-subtitle-line 里先是标签);详情 h1 促销是 font.free / font.halfdown 等 + 「剩余时间」span[title](umt.detail.promo);做种 / 下载 / 完成数在基本信息行右侧 #seeders / #leechers / a[title=完成](没有「同伴」行);种子 Hash 在 #copyTorrentHashLink[data-hash];上传者在 span.torrent-uploader-wrap(匿名为「匿名」);简介开头的站点声明 #ad_torrentdetail 去掉;文件清单不在页面上(点击「N个文件」后由 common.js 的 toggleFileList AJAX 加载,端点未核对),暂不另发请求。", + "$comment": "W1-3b 手写观众定义(按 MP indexer schema;仅对 testkit mock 验证)。深合并到 MovieClaw v0.27.0 转换结果之上,本文件的值优先,MovieClaw 值只补空。 umt.detail.fields=null:保留 W1-3b rowhead 详情模型(已对 mock 验证),不采用 MovieClaw 默认逐字段选择器。 DET-2(2026-09-30 真实账号核对列表页 1 次 + 详情页 2 次):列表副标题取 span.torrent-subtitle-text(span.torrent-subtitle-line 里先是标签);详情 h1 促销是 font.free / font.halfdown 等 + 「剩余时间」span[title](umt.detail.promo);做种 / 下载 / 完成数在基本信息行右侧 #seeders / #leechers / a[title=完成](没有「同伴」行);种子 Hash 在 #copyTorrentHashLink[data-hash];上传者在 span.torrent-uploader-wrap(匿名为「匿名」);简介开头的站点声明 #ad_torrentdetail 去掉;文件清单不在页面上(点击「N个文件」后由 common.js 的 toggleFileList AJAX 加载,端点未核对),暂不另发请求。 ID-3:列表行的豆瓣链接(评分徽标 a[href*=douban.com/subject/],DET-2 列表样本 rl-audiences-list 核对)抽成 torrents.fields.doubanid(UMT 扩展字段,MP 不读)。", "id": "audiences", "name": "观众", "domain": "https://audiences.me/", @@ -84,6 +84,16 @@ } ] }, + "doubanid": { + "selector": "a[href*=\"douban.com/subject/\"]", + "attribute": "href", + "filters": [ + { + "name": "re_search", + "args": ["subject/(\\d+)", 1] + } + ] + }, "date_elapsed": { "selector": "td:nth-child(4) > span" }, From 3d5a50b30afad7bf63de57d4bd7efc2a4f8b8cd8 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 1 Oct 2026 00:15:20 +0800 Subject: [PATCH 3/4] feat(identity): escalate ambiguous/no_match via site detail-page IDs (ID-3) Fetch the detail page (cached, rate-limited, per-site hourly cap identity.detailIdsPerSiteHour) for search results, subscription-related candidates, pool enrich and manual re-identify; resolve IMDb/TMDB via TMDB, douban via cached douban subjects (site detail info block / META-2 snippets). No identity_unresolved decisions for releases with site IDs. Co-Authored-By: Claude Opus 5.5 --- .../modules/sources-identity/routes/enrich.ts | 38 ++ .../sources-identity/service/detail-page.ts | 7 + .../service/douban-subject.ts | 112 ++++++ .../sources-identity/service/enrich.ts | 13 + .../sources-identity/service/id-escalation.ts | 155 ++++++++ .../sources-identity/service/resolve.ts | 181 ++++++++- .../sources-identity/id-escalation.test.ts | 356 ++++++++++++++++++ apps/web/app/components/settings/metadata.tsx | 8 + packages/contracts/src/settings.ts | 7 + 9 files changed, 867 insertions(+), 10 deletions(-) create mode 100644 apps/server/src/modules/sources-identity/service/douban-subject.ts create mode 100644 apps/server/src/modules/sources-identity/service/id-escalation.ts create mode 100644 apps/server/test/sources-identity/id-escalation.test.ts diff --git a/apps/server/src/modules/sources-identity/routes/enrich.ts b/apps/server/src/modules/sources-identity/routes/enrich.ts index a3e92fd2..101636e9 100644 --- a/apps/server/src/modules/sources-identity/routes/enrich.ts +++ b/apps/server/src/modules/sources-identity/routes/enrich.ts @@ -15,8 +15,10 @@ import { ApiException, errorBody } from "../../../http/errors.ts"; import type { ModuleApp } from "../../types.ts"; import { route } from "../http.ts"; import type { SourcesRuntime } from "../runtime.ts"; +import { siteExplicitIds } from "../service/detail-page.ts"; import { applyEnrichment, domainAllowed, enrichStats, hostOf } from "../service/enrich.ts"; import { fetchPublicImage, imageResponse } from "../service/image-proxy.ts"; +import { resolveRelease } from "../service/resolve.ts"; // META-1 元数据补全 API:先看建议(POST /identity-pool/{id}/enrich,经能力注册中心调用 metadata.enrich, // 授权 / 审计与 Agent 同一路径),再应用(POST /identity-pool/{id}/enrich/apply,只接受控制台管理员会话); @@ -40,6 +42,13 @@ export function enrichRoutes(app: ModuleApp, rt: () => SourcesRuntime): void { }, async ({ params, body, actor, c }) => { const ctx = rt().ctx; + // ID-3:补全(网页搜索)之前先走识别升级链——站点详情页有 IMDb / TMDB / 豆瓣 ID 就能直接认出来,不必搜网页 + const resolved = await escalateBeforeEnrich(rt(), params.id); + if (resolved) + throw new ApiException( + "conflict", + `站点详情页给出了作品 ID，已识别为《${resolved.title}》，不需要补全。刷新待识别列表查看`, + ); try { return await ctx.capabilities.invoke( "metadata.enrich", @@ -131,3 +140,32 @@ export function enrichRoutes(app: ModuleApp, rt: () => SourcesRuntime): void { return imageResponse(got, 86400 * 7); }); } + +/** + * 池条目补全前的识别升级:资源还没抓过详情页、或已有豆瓣 ID 时,按手动重新识别走一次(允许抓详情);认出来返回作品。 + * 升级失败不影响补全。 + */ +async function escalateBeforeEnrich(r: SourcesRuntime, poolEntryId: string): Promise<{ title: string } | null> { + const db = r.ctx.db; + const row = await db + .selectFrom("identity_candidate as p") + .innerJoin("source_release as s", "s.id", "p.source_release_id") + .select(["s.id", "s.detail_fetched_at", "s.raw", "s.resolution_locked", "p.status"]) + .where("p.id", "=", poolEntryId) + .executeTakeFirst(); + if (row?.status !== "pending" || row.resolution_locked) return null; + if (row.detail_fetched_at && !siteExplicitIds(row.raw).some((x) => x.provider === "douban")) return null; + try { + const res = await resolveRelease(r, { sourceReleaseId: row.id, force: true, trigger: "manual", escalate: true }); + if (res.outcome !== "auto" || !res.mediaEntityId) return null; + const e = await db + .selectFrom("media_entity") + .select(["title"]) + .where("id", "=", res.mediaEntityId) + .executeTakeFirst(); + return { title: e?.title ?? "?" }; + } catch (e) { + r.ctx.logger.warn({ poolEntryId, err: { message: (e as Error).message } }, "补全前的识别升级失败"); + return null; + } +} diff --git a/apps/server/src/modules/sources-identity/service/detail-page.ts b/apps/server/src/modules/sources-identity/service/detail-page.ts index 284d820b..257ad92d 100644 --- a/apps/server/src/modules/sources-identity/service/detail-page.ts +++ b/apps/server/src/modules/sources-identity/service/detail-page.ts @@ -4,6 +4,7 @@ import { sql } from "@umt/db"; import { sharedCache, storageSettings } from "../../../shared-cache.ts"; import type { SourcesRuntime } from "../runtime.ts"; import type { Exec } from "./common.ts"; +import { doubanSubjectFromDetail, saveDoubanSubject } from "./douban-subject.ts"; // 站点详情页扩展内容(简介 / MediaInfo / 截图 / 外部 ID / 文件 / 标签 / 上传者): // - 只在按需抓详情时得到(工作台打开详情、release.inspect 复核),放 PG kv_cache region site.detail, @@ -44,6 +45,12 @@ export async function saveDetailPage( }; await sql`update source_release set raw = raw || jsonb_build_object('_detail', ${JSON.stringify(ids)}::jsonb) where id = ${srId}`.execute(exec); + // ID-3:详情页的豆瓣信息块(◎片名 / ◎年代)连同豆瓣 ID 存成豆瓣条目,只有豆瓣 ID 时按标题 + 年份匹配 TMDB + const subject = doubanSubjectFromDetail(page, `source_release:${srId}`, rt.now()); + if (subject) + await saveDoubanSubject(rt.ctx, subject).catch((e) => + rt.ctx.logger.warn({ err: { message: (e as Error).message } }, "豆瓣条目缓存写入失败"), + ); } async function loadStored(rt: SourcesRuntime, srId: string): Promise { diff --git a/apps/server/src/modules/sources-identity/service/douban-subject.ts b/apps/server/src/modules/sources-identity/service/douban-subject.ts new file mode 100644 index 00000000..d54c82e1 --- /dev/null +++ b/apps/server/src/modules/sources-identity/service/douban-subject.ts @@ -0,0 +1,112 @@ +import { formatDetailPage } from "@umt/adapters"; +import { pgCacheFor } from "@umt/db"; +import type { ServerContext } from "../../types.ts"; + +// ID-3 豆瓣条目缓存(kv_cache region douban.subject,key = 豆瓣 subject id):只存标题 / 别名 / 年份,供「详情页只有豆瓣 ID」时 +// 按标题 + 年份匹配 TMDB(对照 MP chain/media/projection.py 豆瓣 → TMDB:取豆瓣条目的原名 / 中文名 / 英文名 + 年份做 TMDB 名称匹配)。 +// 本工具不请求豆瓣页面或接口,条目只从已经拿到的数据里抽: +// - site_detail:站点详情页的豆瓣信息块(◎片名 / ◎译名 / ◎年代,多数由 PT-Gen 从豆瓣生成),详情页同时给了豆瓣 ID 时写入; +// - web_snippet:元数据补全(META-2)网页搜索结果里 douban.com/subject/ 的搜索标题与摘要。 + +export const DOUBAN_SUBJECT_REGION = "douban.subject"; +const TTL_SEC = 30 * 86_400; +const MAX_ENTRIES = 20_000; + +export interface DoubanSubject { + id: string; + /** 豆瓣显示名(通常是中文名) */ + title: string; + /** 译名 / 又名 / 原名 */ + aliases: string[]; + year: number | null; + source: "site_detail" | "web_snippet"; + /** 出处:站点资源 id 或搜索结果地址 */ + ref: string; + at: string; +} + +const okYear = (y: number) => y >= 1900 && y <= 2100; + +function splitNames(v: string | undefined): string[] { + return (v ?? "") + .split(/\s*\/\s*/) + .map((x) => x.trim()) + .filter((x) => x.length > 0 && x.length <= 80); +} + +/** 站点详情页信息块 → 豆瓣条目(需要详情页同时给出豆瓣 ID 与「片名」) */ +export function doubanSubjectFromDetail( + page: { doubanId?: string | null; descriptionText?: string | null }, + ref: string, + now: Date, +): DoubanSubject | null { + if (!page.doubanId || !page.descriptionText) return null; + const { infoBlock } = formatDetailPage({ descriptionText: page.descriptionText, mediaInfo: null, files: [] }); + const get = (k: string) => infoBlock.find((x) => x.key === k)?.value?.trim(); + const names = splitNames(get("片名")); + const title = names[0]; + if (!title) return null; + const y = Number(/(?:19|20)\d{2}/.exec(get("年代") ?? get("年份") ?? get("上映日期") ?? "")?.[0]); + const aliases = [...new Set([...names.slice(1), ...splitNames(get("译名")), ...splitNames(get("又名"))])].filter( + (a) => a !== title, + ); + return { + id: page.doubanId, + title, + aliases: aliases.slice(0, 10), + year: okYear(y) ? y : null, + source: "site_detail", + ref, + at: now.toISOString(), + }; +} + +/** 网页搜索结果(douban.com/subject/)→ 豆瓣条目:标题「X (豆瓣)」,年份取标题里的 (2025) 或摘要「年份: 2025」 */ +export function doubanSubjectFromSnippet( + hit: { url: string; title?: string | null; snippet?: string | null }, + now: Date, +): DoubanSubject | null { + const id = /douban\.com\/(?:movie\/)?subject\/(\d{4,10})/.exec(hit.url)?.[1]; + if (!id) return null; + let t = (hit.title ?? "") + .replace(/\s*[(（]豆瓣[)）]\s*$/, "") + .replace(/\s*[-_|]\s*豆瓣.*$/, "") + .trim(); + const ty = /[(（]((?:19|20)\d{2})[)）]\s*$/.exec(t); + if (ty) t = t.slice(0, ty.index).trim(); + if (!t || t === "豆瓣" || /^豆瓣/.test(t)) return null; + const s = hit.snippet ?? ""; + const sy = /(?:年份|首播|上映日期)[:：]\s*((?:19|20)\d{2})/.exec(s)?.[1]; + const y = Number(ty?.[1] ?? sy); + const alias = /又名[:：]\s*([^\n·|]{1,120})/.exec(s)?.[1]; + return { + id, + title: t, + aliases: splitNames(alias).slice(0, 10), + year: okYear(y) ? y : null, + source: "web_snippet", + ref: hit.url, + at: now.toISOString(), + }; +} + +type Db = Pick; + +/** 写入缓存;站点详情来源优先(已有 site_detail 条目时不用搜索摘要覆盖) */ +export async function saveDoubanSubject(ctx: Db, s: DoubanSubject): Promise { + const cache = pgCacheFor(ctx.db); + if (s.source === "web_snippet") { + const prev = await cache.get(DOUBAN_SUBJECT_REGION, s.id); + if (prev?.source === "site_detail") return; + } + await cache.set(DOUBAN_SUBJECT_REGION, s.id, s, { ttl: TTL_SEC, maxsize: MAX_ENTRIES }); +} + +export async function loadDoubanSubject(ctx: Db, id: string): Promise { + try { + const v = await pgCacheFor(ctx.db).get(DOUBAN_SUBJECT_REGION, id); + return v && typeof v.title === "string" && v.title ? v : null; + } catch { + return null; + } +} diff --git a/apps/server/src/modules/sources-identity/service/enrich.ts b/apps/server/src/modules/sources-identity/service/enrich.ts index 0bc86192..2225b896 100644 --- a/apps/server/src/modules/sources-identity/service/enrich.ts +++ b/apps/server/src/modules/sources-identity/service/enrich.ts @@ -22,6 +22,7 @@ import type { ServerContext } from "../../types.ts"; import { jsonb } from "../rows.ts"; import type { SourcesRuntime } from "../runtime.ts"; import { type Actor, audit, type Exec } from "./common.ts"; +import { type DoubanSubject, doubanSubjectFromSnippet, saveDoubanSubject } from "./douban-subject.ts"; import { baseConfidence, type Candidate, @@ -490,6 +491,7 @@ export async function suggestEnrichment( .returningAll() .executeTakeFirstOrThrow(); + const doubanSubjects: DoubanSubject[] = []; try { const r = await runEnrichPipeline( deps, @@ -499,10 +501,15 @@ export async function suggestEnrichment( sites: await siteHosts(ctx), id: row.id, logger: ctx.logger, + doubanSubjects, ...(opts.signal ? { signal: opts.signal } : {}), }, usage, ); + for (const ds of doubanSubjects) + await saveDoubanSubject(ctx, ds).catch((e) => + ctx.logger.warn({ err: { message: (e as Error).message } }, "豆瓣条目缓存写入失败"), + ); if (!r.found) { const failed = await finish({ status: "failed", suggestion: r.suggestion, error: r.error }); return toOutput(failed, false, await budgetOf(), await sameTitleEntities(ctx.db, [q.title])); @@ -540,6 +547,8 @@ export interface PipelineInput { id: string; logger: Pick; signal?: AbortSignal; + /** ID-3:搜索结果里的豆瓣条目(douban.com/subject/ 的标题 / 年份)收集到这里,由调用方写豆瓣条目缓存 */ + doubanSubjects?: DoubanSubject[]; } export interface PipelineResult { @@ -598,6 +607,10 @@ export async function runEnrichPipeline( // 白名单外的网站一律按「其它网站的搜索摘要」处理(不读网页) const kind: SourceKind = listed ? sourceKindOf(host, settings.domains) : "snippet"; kindOf.set(h.url, kind); + if (input.doubanSubjects && /(^|\.)douban\.com$/.test(host)) { + const ds = doubanSubjectFromSnippet(h, new Date()); + if (ds && !input.doubanSubjects.some((x) => x.id === ds.id)) input.doubanSubjects.push(ds); + } const snippetOnly = kind === "douban"; const allowed = listed && !snippetOnly; const src: EnrichSource = { diff --git a/apps/server/src/modules/sources-identity/service/id-escalation.ts b/apps/server/src/modules/sources-identity/service/id-escalation.ts new file mode 100644 index 00000000..81f53270 --- /dev/null +++ b/apps/server/src/modules/sources-identity/service/id-escalation.ts @@ -0,0 +1,155 @@ +import { type MetadataCandidate, recallTmdbCandidates } from "@umt/adapters"; +import { type SourceReleaseRow, sql } from "@umt/db"; +import { type ParsedRelease, parseRelease } from "@umt/domain"; +import type { SourcesRuntime } from "../runtime.ts"; +import type { siteExplicitIds } from "./detail-page.ts"; +import { type DoubanSubject, loadDoubanSubject } from "./douban-subject.ts"; +import { refreshReleaseDetail } from "./snapshot.ts"; +import { tmdbRecallInput } from "./tmdb-candidate.ts"; + +// ID-3 识别升级链:标题识别结果为 ambiguous / no_match 时,站点资源有详情页且还没抓过 → 抓详情(走站点详情缓存与共享限流器, +// 失败不阻塞)→ 取 IMDb / TMDB / 豆瓣 ID → TMDB 详情 / find(豆瓣 ID 经已缓存的豆瓣条目按标题 + 年份匹配)→ 重新打分。 +// 触发范围克制:搜索结果、订阅相关的候选(命中订阅 / 本地作品别名)、池条目补全、手动重新识别;RSS 批量入库不抓, +// 留给上述时机。每站每小时上限 identity.detailIdsPerSiteHour(缺省 10,0 = 不抓),计数在 kv_cache region identity.detail_ids。 + +export const ESCALATION_COUNTER_REGION = "identity.detail_ids"; +export const DEFAULT_DETAIL_IDS_PER_SITE_HOUR = 10; +/** 抓详情失败后多久内不再为识别重抓(站点错误 / 限流) */ +const RETRY_AFTER_FAILURE_MS = 6 * 3600_000; + +export type SiteIds = ReturnType; + +/** 记在 source_release.raw._escalation:最近一次为识别抓详情的结果 */ +interface EscalationMark { + at: string; + status: "fetched" | "failed"; + error?: string; +} + +export type DetailFetchResult = + | { status: "fetched"; release: SourceReleaseRow } + | { + status: "skipped"; + reason: "disabled" | "already_fetched" | "no_detail" | "site_unavailable" | "recent_failure"; + } + | { status: "deferred"; reason: "cap_reached" } + /** 抓了但失败:不阻塞识别,照常往下走(6 小时内不再为识别重抓) */ + | { status: "failed"; reason: "fetch_failed"; error: string }; + +export async function detailIdsPerSiteHour(rt: SourcesRuntime): Promise { + const s = await rt.ctx.settings.get("identity").catch(() => null); + return s?.detailIdsPerSiteHour ?? DEFAULT_DETAIL_IDS_PER_SITE_HOUR; +} + +/** + * 每站每小时计数:原子地占一个名额;已满返回 false。kv_cache 是 UNLOGGED 表,PG 崩溃丢计数只会让这一小时多抓几个,可接受。 + */ +export async function takeEscalationSlot(rt: SourcesRuntime, siteId: string, cap: number): Promise { + if (cap <= 0) return false; + const hour = Math.floor(rt.now().getTime() / 3_600_000); + const r = await sql<{ n: number }>` + insert into kv_cache (region, key, value, expires_at, updated_at, accessed_at) + values (${ESCALATION_COUNTER_REGION}, ${`${siteId}:${hour}`}, '1'::jsonb, now() + interval '2 hours', now(), now()) + on conflict (region, key) do update set value = to_jsonb((kv_cache.value #>> '{}')::int + 1), updated_at = now() + where (kv_cache.value #>> '{}')::int < ${cap} + returning (value #>> '{}')::int as n`.execute(rt.ctx.db); + return r.rows.length > 0; +} + +/** 这一小时该站点已为识别抓了多少个详情页 */ +export async function escalationCount(rt: SourcesRuntime, siteId: string): Promise { + const hour = Math.floor(rt.now().getTime() / 3_600_000); + const r = await sql<{ n: number }>`select (value #>> '{}')::int as n from kv_cache + where region = ${ESCALATION_COUNTER_REGION} and key = ${`${siteId}:${hour}`} and expires_at > now()`.execute( + rt.ctx.db, + ); + return Number(r.rows[0]?.n ?? 0); +} + +function markOf(raw: unknown): EscalationMark | null { + const m = ((raw ?? {}) as { _escalation?: EscalationMark })._escalation; + return m && typeof m.at === "string" ? m : null; +} + +async function mark(rt: SourcesRuntime, srId: string, m: EscalationMark): Promise { + await sql`update source_release set raw = raw || jsonb_build_object('_escalation', ${JSON.stringify(m)}::jsonb) + where id = ${srId}`.execute(rt.ctx.db); +} + +/** + * 为识别抓一次详情页(只在还没抓过时)。走 refreshReleaseDetail:站点详情缓存、共享限流器、请求计数、凭据失效处理都照旧; + * 失败只记在 raw._escalation,6 小时内不再为识别重抓,不抛错。 + */ +export async function fetchDetailForIds(rt: SourcesRuntime, release: SourceReleaseRow): Promise { + if (release.detail_fetched_at) return { status: "skipped", reason: "already_fetched" }; + const prev = markOf(release.raw); + if (prev?.status === "failed" && rt.now().getTime() - Date.parse(prev.at) < RETRY_AFTER_FAILURE_MS) + return { status: "skipped", reason: "recent_failure" }; + const cap = await detailIdsPerSiteHour(rt); + if (cap <= 0) return { status: "skipped", reason: "disabled" }; + const site = await rt.ctx.db.selectFrom("site").selectAll().where("id", "=", release.site_id).executeTakeFirst(); + if (!site?.enabled || site.health_status === "failing") return { status: "skipped", reason: "site_unavailable" }; + try { + if (!rt.siteAdapter(site).capabilities.detail) return { status: "skipped", reason: "no_detail" }; + } catch { + return { status: "skipped", reason: "no_detail" }; + } + if (!(await takeEscalationSlot(rt, site.id, cap))) return { status: "deferred", reason: "cap_reached" }; + const err = await refreshReleaseDetail(rt, release.id).catch((e) => (e as Error).message); + const at = rt.now().toISOString(); + if (err) { + await mark(rt, release.id, { at, status: "failed", error: err.slice(0, 300) }); + rt.ctx.logger.info({ sourceReleaseId: release.id, err }, "识别升级时抓详情页失败"); + return { status: "failed", reason: "fetch_failed", error: err }; + } + await mark(rt, release.id, { at, status: "fetched" }); + const fresh = await rt.ctx.db + .selectFrom("source_release") + .selectAll() + .where("id", "=", release.id) + .executeTakeFirst(); + return fresh ? { status: "fetched", release: fresh } : { status: "skipped", reason: "site_unavailable" }; +} + +/** 新拿到的 ID(不在旧集合里的) */ +export function newIds(before: SiteIds, after: SiteIds): SiteIds { + return after.filter((a) => !before.some((b) => b.provider === a.provider && b.id === a.id)); +} + +/** + * 豆瓣 ID → 缓存的豆瓣条目 → 按条目标题 / 别名 + 年份召回 TMDB 候选(对照 MP 豆瓣 → TMDB 投影:原名 / 中文名 / 英文名 + 年份)。 + * 打分用的解析 = 豆瓣条目的名称与年份,季集与类型线索沿用资源标题的解析。缓存里没有这个条目 → null(不请求豆瓣)。 + */ +export async function doubanRecall( + rt: SourcesRuntime, + parsed: ParsedRelease, + doubanId: string, +): Promise<{ subject: DoubanSubject; parsed: ParsedRelease; remote: MetadataCandidate[] } | null> { + const subject = await loadDoubanSubject(rt.ctx, doubanId); + if (!subject) return null; + const tmdb = await rt.tmdb(); + if (!tmdb) return null; + const p = doubanParsed(parsed, subject); + const remote = await recallTmdbCandidates(tmdb, tmdbRecallInput(p)); + return { subject, parsed: p, remote }; +} + +const CJK = /[㐀-鿿]/; + +/** 豆瓣条目 + 资源解析 → 打分用的解析(名称 / 年份取豆瓣,季集 / 类型取资源) */ +export function doubanParsed(parsed: ParsedRelease, s: DoubanSubject): ParsedRelease { + const names = [s.title, ...s.aliases]; + const cn = names.find((n) => CJK.test(n)); + const en = names.find((n) => !CJK.test(n) && /[A-Za-z]/.test(n)); + const base = parseRelease(s.title); + return { + ...parsed, + title: cn ?? en ?? base.title ?? s.title, + ...(cn ? { titleCn: cn } : {}), + ...(en ? { titleEn: en } : {}), + altTitles: names.filter((n) => n !== cn && n !== en).slice(0, 8), + ...(s.year !== null ? { year: s.year } : {}), + explicitIds: [...parsed.explicitIds], + evidence: [...parsed.evidence, "douban_subject"], + }; +} diff --git a/apps/server/src/modules/sources-identity/service/resolve.ts b/apps/server/src/modules/sources-identity/service/resolve.ts index 0f4ea625..ec39c0f8 100644 --- a/apps/server/src/modules/sources-identity/service/resolve.ts +++ b/apps/server/src/modules/sources-identity/service/resolve.ts @@ -12,6 +12,7 @@ import { queryTitles, type RankedCandidate, resolveIdentity, + scoreCandidate, } from "@umt/domain"; import { localIdentityCandidate, @@ -34,6 +35,7 @@ import { looksLikeVariety, resolveEntityId, } from "./entities.ts"; +import { doubanRecall, fetchDetailForIds, newIds } from "./id-escalation.ts"; import { detachFromIdentityDecisions, openGroupDecision, refreshGroupDecision } from "./identity-decisions.ts"; import { tmdbRecallInput, tmdbToCandidate } from "./tmdb-candidate.ts"; import { loadRecognizeCache, type RecognizeMeta, recognizeMetaOf, saveRecognizeCache } from "./tmdb-recognize-cache.ts"; @@ -63,6 +65,11 @@ export interface ResolveInput { * manual / search 总是查 TMDB;observed / review 只在命中本地别名、新作品收录规则或设置打开时查。 */ trigger?: ResolveTrigger | undefined; + /** + * ID-3:标题识别不确定时允许抓站点详情页取 ID(识别升级链)。缺省按 trigger 判断(manual / search / 订阅相关的复查); + * 池条目补全前由调用方显式打开。 + */ + escalate?: boolean | undefined; } /** @@ -217,6 +224,108 @@ async function tmdbByExplicitIds( return found; } +/** 站点给出的 IMDb / TMDB ID 作为强证据(ID-3,domain MatchOptions.trustedIds);豆瓣 ID 不直接对应 TMDB 候选 */ +function trustedOf(ids: ReturnType) { + return { trustedIds: ids.filter((x) => x.provider !== "douban").map((x) => ({ provider: x.provider, id: x.id })) }; +} + +/** 识别升级链的证据(记在 resolution_evidence.escalation 与池条目 query.escalation) */ +export interface EscalationEvidence { + /** 详情页:fetched 本次抓到;skipped:<原因>;deferred:cap_reached(稍后复查时再抓);failed:fetch_failed;not_now:<原因>(本次不抓) */ + detail: string; + /** 最后用上的路径:site_id(详情 / 列表的 IMDb、TMDB)/ douban(豆瓣条目 → 标题 + 年份) */ + via: "site_id" | "douban" | null; + /** 抓详情后资源上的外部 ID */ + ids: string[]; + /** 本次没抓、之后还会抓(每站每小时限额已满 / RSS 入库时不抓):不开待处理,尽快复查。抓了失败的不算(不阻塞) */ + pending: boolean; + /** 豆瓣条目(豆瓣路径) */ + douban?: { id: string; title: string; year: number | null; source: string }; +} + +/** + * 什么时候允许为识别抓详情页:搜索结果、手动重新识别、调用方显式要求(池条目补全)、订阅相关的候选(命中订阅 / 本地作品别名) + * 的复查、上次因限额 / 入库时机而推迟的复查。RSS / 列表批量入库(observed)不抓,推迟到复查。 + */ +function escalationFetchPolicy( + trigger: ResolveTrigger, + lookup: LookupReason, + input: ResolveInput, + prevQuery: unknown, +): "fetch" | "defer" | "no" { + if (input.escalate === false) return "no"; + if (input.escalate || trigger === "manual" || trigger === "search") return "fetch"; + const prev = ((prevQuery ?? {}) as { escalation?: { pending?: boolean } }).escalation; + if (trigger === "review" && (lookup === "tracked_alias" || prev?.pending)) return "fetch"; + if (trigger === "observed" && lookup === "tracked_alias") return "defer"; + return "no"; +} + +interface EscalationOutcome { + evidence: EscalationEvidence; + resolution?: IdentityResolution; + remote?: MetadataCandidate[]; + parsed?: ParsedRelease; +} + +/** + * ID-3 识别升级链:抓详情(按策略)→ 新拿到的 IMDb / TMDB → TMDB 详情 / find → 打分;还不能确认且有豆瓣 ID → + * 缓存的豆瓣条目按标题 + 年份召回 TMDB → 打分。返回更好的一次结果(auto 优先),都没有就只带证据。 + */ +async function escalateIdentity( + rt: SourcesRuntime, + release: SourceReleaseRow, + parsed: ParsedRelease, + local: DomainCandidate[], + siteIds: ReturnType, + policy: "fetch" | "defer", +): Promise { + const ev: EscalationEvidence = { detail: "not_now", via: null, ids: [], pending: false }; + let raw = release.raw; + if (policy === "fetch") { + const f = await fetchDetailForIds(rt, release); + ev.detail = f.status === "fetched" ? "fetched" : `${f.status}:${f.reason}`; + if (f.status === "fetched") raw = f.release.raw; + if (f.status === "deferred") ev.pending = true; + } else if (!release.detail_fetched_at) { + ev.detail = "not_now:batch_ingest"; + ev.pending = true; + } else ev.detail = "skipped:already_fetched"; + const after = siteExplicitIds(raw); + ev.ids = after.map((x) => `${x.provider}:${x.id}`); + const tmdb = await rt.tmdb(); + if (!tmdb) return { evidence: ev }; + let best: EscalationOutcome | null = null; + // 1) 新拿到的 IMDb / TMDB(列表上已有的在第一轮已用过) + const added = newIds(siteIds, after).filter((x) => x.provider !== "douban"); + if (added.length) { + const p: ParsedRelease = { ...parsed, explicitIds: [...parsed.explicitIds] }; + for (const x of after) + if (!p.explicitIds.some((e) => e.provider === x.provider)) p.explicitIds.push({ provider: x.provider, id: x.id }); + const remote = await tmdbByExplicitIds(tmdb, after); + if (remote.length) { + const res = resolveIdentity(p, [...local, ...remote.map(tmdbToCandidate)], trustedOf(after)); + best = { evidence: ev, resolution: res, remote, parsed: p }; + ev.via = "site_id"; + if (res.status === "auto") return best; + } + } + // 2) 豆瓣 ID → 缓存的豆瓣条目(标题 + 年份)→ TMDB + const douban = after.find((x) => x.provider === "douban"); + if (douban) { + const d = await doubanRecall(rt, parsed, douban.id); + if (d) { + ev.douban = { id: d.subject.id, title: d.subject.title, year: d.subject.year, source: d.subject.source }; + const res = resolveIdentity(d.parsed, [...local, ...d.remote.map(tmdbToCandidate)]); + if (res.status === "auto" || !best || (best.resolution?.status === "no_match" && res.status !== "no_match")) { + ev.via = "douban"; + best = { evidence: ev, resolution: res, remote: d.remote, parsed: d.parsed }; + } + } + } + return best ?? { evidence: ev }; +} + async function loadRelease(rt: SourcesRuntime, input: ResolveInput) { const { db } = rt.ctx; const observationId = input.observationId ?? null; @@ -291,13 +400,14 @@ export async function resolveRelease(rt: SourcesRuntime, input: ResolveInput): P return toPool(rt, release, observationId, parsed, null, [], "unmatched", "unparseable_title", null, { tracked: false, lookup: null, + escalation: null, }); } // 召回:本地优先;TMDB 只在有理由时查(惰性识别),查询失败不等于无结果 const local = await localCandidates(ctx.db, parsed, release); const lookup = await lookupReason(rt, parsed, trigger, local.length > 0, pool?.reason ?? null); - const scope = { tracked: local.length > 0, lookup }; + const scope: ResolveScope = { tracked: local.length > 0, lookup, escalation: null }; if (!lookup) { // 不相关的资源:只进池(可检索、可手动指定),不查 TMDB、不开待处理 return toPool(rt, release, observationId, parsed, null, [], "unmatched", "no_local_match", null, scope); @@ -314,7 +424,7 @@ export async function resolveRelease(rt: SourcesRuntime, input: ResolveInput): P lookupError = `${e.kind}: ${e.message}`; } const candidates = [...local, ...remote.map(tmdbToCandidate)]; - const resolution = resolveIdentity(parsed, candidates); + let resolution = resolveIdentity(parsed, candidates, trustedOf(siteIds)); // 识别缓存回写(MP TmdbCache.update):识别到 TMDB 条目写正常条目,确认无匹配写负缓存;TMDB 出错 / 有歧义不写 if (recognize.meta && !recognize.cacheHit && !lookupError) { const top = resolution.status === "auto" ? resolution.best?.candidate : undefined; @@ -325,7 +435,29 @@ export async function resolveRelease(rt: SourcesRuntime, input: ResolveInput): P const info = hit ?? (resolution.status === "no_match" ? null : undefined); await saveRecognizeCache(rt, recognize.meta, info); } - const version = evidenceVersion(parsed, resolution.ranked); + + // ID-3 识别升级链:标题认不准时用站点详情页的 IMDb / TMDB / 豆瓣 ID 再认一次 + let escalation: EscalationEvidence | null = null; + let scored = parsed; + if (resolution.status !== "auto" && !lookupError) { + const policy = escalationFetchPolicy(trigger, lookup, input, pool?.query); + if (policy !== "no") { + try { + const up = await escalateIdentity(rt, release, parsed, local, siteIds, policy); + escalation = up.evidence; + if (up.resolution && up.remote && up.parsed) { + resolution = up.resolution; + remote = up.remote; + scored = up.parsed; + } + } catch (e) { + if (!(e instanceof TmdbError)) throw e; + lookupError = `${e.kind}: ${e.message}`; + } + } + } + const version = evidenceVersion(scored, resolution.ranked); + scope.escalation = escalation; // 已识别且证据未变:幂等短路(强制复查时) if ( @@ -366,7 +498,18 @@ export async function resolveRelease(rt: SourcesRuntime, input: ResolveInput): P } } } - return bindAuto(rt, release, observationId, parsed, cols, resolution, version, detailed ?? null, lookup); + return bindAuto( + rt, + release, + observationId, + parsed, + cols, + resolution, + version, + detailed ?? null, + lookup, + escalation, + ); } if (resolution.status === "auto" && !autoAccept) { @@ -514,9 +657,12 @@ async function bindAuto( version: string, tmdbDetail: MetadataCandidate | null, lookup: LookupReason, + escalation: EscalationEvidence | null = null, ): Promise { const { ctx } = rt; const best = resolution.best!; + // 别名登记按资源自己的标题判断(升级链靠 ID / 豆瓣名确认的,站点标题未必对得上;ID-2 / FIX-5 守卫照旧) + const aliasBest = escalation?.via ? scoreCandidate(parsed, best.candidate) : best; const level: EvidenceLevel = toContractEvidenceLevel(best.level) ?? "needs_review"; const site = ((release.raw as Record)?._site ?? {}) as { categoryName?: string }; const variety = looksLikeVariety(parsed, site.categoryName ?? release.category_raw); @@ -558,8 +704,8 @@ async function bindAuto( // 站点标题作为别名:下次同名资源可直接本地召回。只在名称本身命中过(不是仅靠别名单线命中)时登记, // 且通用词(分类词 / 单字 / 停用词)不登记,避免错误别名链式扩散(ID-2)。 // FIX-5:证据等级非最高、年份不相容、与已确认外部标题冲突、已属于别的作品的名称也不登记(alias-guard.ts) - const aliasOutcome: SiteAliasOutcome = siteAliasAllowed(best) - ? await addSiteAliases(trx, entityId, best, parsed) + const aliasOutcome: SiteAliasOutcome = siteAliasAllowed(aliasBest) + ? await addSiteAliases(trx, entityId, aliasBest, parsed) : { added: [], skipped: [] }; const evidence = { @@ -569,6 +715,7 @@ async function bindAuto( best: rankedSummary(best), candidateCount: resolution.ranked.length, lookup, + ...(escalation ? { escalation } : {}), }; await trx .updateTable("source_release") @@ -669,8 +816,13 @@ type PoolReason = "ambiguous" | "no_match" | "proposed" | "lookup_failed" | "unm interface ResolveScope { tracked: boolean; lookup: LookupReason | null; + /** ID-3 识别升级链的证据(没走升级链为 null) */ + escalation: EscalationEvidence | null; } +/** 推迟的升级(限额已满 / 入库时不抓)多久后复查 */ +const ESCALATION_RETRY_MS = 15 * 60_000; + async function toPool( rt: SourcesRuntime, release: SourceReleaseRow, @@ -701,6 +853,7 @@ async function toPool( season: parsed.season ?? null, episodes: parsed.episodes ?? null, lookup: scope.lookup, + ...(scope.escalation ? { escalation: scope.escalation } : {}), }; const contractStatus = reason === "proposed" @@ -717,7 +870,10 @@ async function toPool( .executeTakeFirst(); // 证据版本变化 = 新证据:从头计数;同一证据反复复查按退避递增,达上限停止自动复查(仍可检索) const attempts = prev && prev.evidence_version === version ? prev.attempts + 1 : 1; - const delay = reviewDelayMs(reason, attempts); + const baseDelay = reviewDelayMs(reason, attempts); + // 升级链推迟了(还能拿到 ID):尽快复查,不受退避上限影响 + const pendingEscalation = scope.escalation?.pending === true; + const delay = pendingEscalation ? Math.min(baseDelay ?? ESCALATION_RETRY_MS, ESCALATION_RETRY_MS) : baseDelay; const nextReviewAt = delay === null ? null : new Date(now.getTime() + delay); const evidence = { @@ -729,10 +885,14 @@ async function toPool( query, }; // 只收相关的:查过 TMDB 仍有歧义(有候选可选),或与本地作品相关却无结果 / 查询失败且复查已停 - const wantsDecision = - reason === "ambiguous" || reason === "proposed" + // ID-3:能从站点拿到 ID 的资源不开"没识别出作品":升级链还在推迟中(之后会抓详情)不开任何事项; + // 资源上已有 IMDb / TMDB / 豆瓣 ID 却仍无结果的,不开 identity_unresolved(只留在池里,可手动指定) + const hasSiteIds = siteExplicitIds(release.raw).length > 0 || (scope.escalation?.ids.length ?? 0) > 0; + const wantsDecision = pendingEscalation + ? false + : reason === "ambiguous" || reason === "proposed" ? candidates.length > 0 - : scope.tracked && (reason === "no_match" || (reason === "lookup_failed" && delay === null)); + : scope.tracked && !hasSiteIds && (reason === "no_match" || (reason === "lookup_failed" && delay === null)); let decisionId: string | null = null; if (wantsDecision) { const ambiguous = (reason === "ambiguous" || reason === "proposed") && candidates.length > 0; @@ -791,6 +951,7 @@ async function toPool( evidenceVersion: version, candidateCount: candidates.length, lookup: scope.lookup, + ...(scope.escalation ? { escalation: scope.escalation } : {}), }), }) .where("id", "=", release.id) diff --git a/apps/server/test/sources-identity/id-escalation.test.ts b/apps/server/test/sources-identity/id-escalation.test.ts new file mode 100644 index 00000000..23e8a712 --- /dev/null +++ b/apps/server/test/sources-identity/id-escalation.test.ts @@ -0,0 +1,356 @@ +/** + * ID-3 识别升级链(样本驱动,零真实请求):标题识别不确定 → 抓站点详情页(观众 DET-2 脱敏样本 / 憨憨 HH-2 样本,经注入的 fetch) + * → IMDb / TMDB / 豆瓣 ID → TMDB(mock)→ 确定;每站每小时上限;RSS 入库不抓;事项不开 / 自动关闭;豆瓣条目缓存路径。 + */ +import { readFileSync } from "node:fs"; +import { sql } from "@umt/db"; +import { normalizeTitle } from "@umt/domain"; +import { + createMockTmdb, + HHANCLUB_DETAIL_PAGE_2026, + HHANCLUB_FILELIST_FRAGMENT, + type MockTmdb, + type TmdbFixture, +} from "@umt/testkit"; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; +import { + DOUBAN_SUBJECT_REGION, + doubanSubjectFromSnippet, + loadDoubanSubject, + saveDoubanSubject, +} from "../../src/modules/sources-identity/service/douban-subject.ts"; +import { + ESCALATION_COUNTER_REGION, + escalationCount, +} from "../../src/modules/sources-identity/service/id-escalation.ts"; +import { resolveRelease } from "../../src/modules/sources-identity/service/resolve.ts"; +import { sharedCache } from "../../src/shared-cache.ts"; +import { createHarness, type Harness, pgAvailable, seedSite } from "./harness.ts"; + +const pg = await pgAvailable(); + +const SAMPLES = new URL("../../../../packages/adapters/test/differential/inputs/detail/", import.meta.url); +const audiencesDetail = (id: string) => readFileSync(new URL(`rl-audiences-detail-${id}.html.txt`, SAMPLES), "utf8"); + +const fx = ( + type: "movie" | "tv", + id: number, + en: string, + zh: string, + date: string, + imdb: string | null = null, +): TmdbFixture => ({ + type, + id, + original_title: en, + original_language: "zh", + origin_country: ["CN"], + date, + titles: { "zh-CN": zh, "en-US": en }, + popularity: 10, + alternative_titles: [], + external_ids: { imdb_id: imdb }, + ...(type === "tv" + ? { + seasons: [ + { season_number: 1, air_date: date, names: { "zh-CN": "第 1 季", "en-US": "Season 1" }, episodes: [] }, + ], + } + : {}), +}); + +describe.skipIf(!pg)("ID-3 识别升级链:详情页 ID", () => { + let h: Harness; + let tmdb: MockTmdb; + let audiences: string; + let hhan: string; + const siteRequests: string[] = []; + + beforeAll(async () => { + tmdb = await createMockTmdb({ + fixtures: [ + // 观众 727077《我不是大师》:站点英文名 A Prophet 与 TMDB 英文名对不上 + fx("tv", 990002, "The Fortune Teller", "我不是大师", "2026-09-01", "tt36758489"), + // 观众 725858《Rose》:同名同年两部 → 标题识别有歧义 + fx("movie", 990001, "Rose", "罗斯", "2026-03-01", "tt25399154"), + fx("movie", 990011, "Rose", "玫瑰", "2026-05-01", "tt0000011"), + // 憨憨详情样本:TMDB tv/290001(站点英文名 Example Show 搜不到) + fx("tv", 290001, "Sample Series", "示例剧集", "2026-08-01", "tt45950001"), + // 豆瓣路径:只凭豆瓣条目(标题 + 年份)匹配 + fx("tv", 990021, "Lanterns of Qing", "青灯记", "2025-04-01"), + ], + }); + h = await createHarness("id3", { + tmdbUrl: tmdb.url, + // 站点请求全部走样本;没有样本的地址直接报错(零真实请求) + fetch: async (input) => { + const u = new URL(typeof input === "string" ? input : input instanceof URL ? input.href : input.url); + siteRequests.push(`${u.host}${u.pathname}${u.search}`); + const id = u.searchParams.get("id") ?? ""; + const html = (body: string) => + new Response(body, { status: 200, headers: { "content-type": "text/html; charset=utf-8" } }); + if (u.host === "audiences.me" && u.pathname === "/details.php") return html(audiencesDetail(id)); + if (u.host === "hhanclub.net" && u.pathname === "/details.php") return html(HHANCLUB_DETAIL_PAGE_2026); + if (u.host === "hhanclub.net" && u.pathname === "/viewfilelist.php") return html(HHANCLUB_FILELIST_FRAGMENT); + throw new Error(`测试不允许的站点请求 ${u.href}`); + }, + }); + audiences = (await seedSite(h, { key: "audiences", baseUrl: "https://audiences.me/", cookie: "c_secure_uid=1" })) + .site.id; + hhan = (await seedSite(h, { key: "hhanclub", baseUrl: "https://hhanclub.net/", cookie: "c_secure_uid=2" })).site.id; + }); + afterAll(async () => { + await h?.close(); + await tmdb?.close(); + }); + beforeEach(async () => { + await setCap(10); + await sql`delete from kv_cache where region = ${ESCALATION_COUNTER_REGION}`.execute(h.db); + }); + + const setCap = async (cap: number) => { + await sql`insert into settings (key, value) values ('identity', ${JSON.stringify({ detailIdsPerSiteHour: cap })}::jsonb) + on conflict (key) do update set value = excluded.value`.execute(h.db); + }; + const release = (siteId: string, siteReleaseId: string, title: string, raw: unknown = {}) => + h.db + .insertInto("source_release") + .values({ site_id: siteId, site_release_id: siteReleaseId, title, raw: JSON.stringify(raw) } as never) + .returningAll() + .executeTakeFirstOrThrow(); + const row = (id: string) => + h.db.selectFrom("source_release").selectAll().where("id", "=", id).executeTakeFirstOrThrow(); + const detailRequests = (id: string) => + siteRequests.filter((r) => r.includes("/details.php") && r.includes(`id=${id}&`)); + const tmdbIdOf = async (entityId: string) => + ( + await h.db + .selectFrom("external_identity") + .select("external_id") + .where("media_entity_id", "=", entityId) + .where("provider", "=", "tmdb") + .executeTakeFirstOrThrow() + ).external_id; + const openDecisions = () => + h.db + .selectFrom("decision") + .select(["id", "kind", "status"]) + .where("kind", "in", ["identity_ambiguous", "identity_unresolved"]) + .where("status", "in", ["open", "investigating", "awaiting_user"]) + .execute(); + + it("观众 727077:列表只有豆瓣 ID,搜索结果标题认不出 → 抓详情取 IMDb → TMDB find → 自动确认;不登记对不上的站点名", async () => { + // 列表页只给了豆瓣 ID(样本 rl-audiences-list 里这一行没有 IMDb 徽标) + const r = await release(audiences, "727077", "A Prophet S01E16 2026 2160p iQIYI WEB-DL H265 HDR DDP5.1-ADWeb", { + _site: { doubanId: "37297001" }, + }); + const find0 = tmdb.callsTo(/\/3\/find\/tt36758489/).length; + const res = await resolveRelease(h.rt, { sourceReleaseId: r.id, trigger: "search" }); + expect(res.outcome).toBe("auto"); + expect(detailRequests("727077")).toHaveLength(1); + expect(tmdb.callsTo(/\/3\/find\/tt36758489/).length).toBe(find0 + 1); + expect(await tmdbIdOf(res.mediaEntityId!)).toBe("990002"); + const after = await row(r.id); + expect(after.media_entity_id).toBe(res.mediaEntityId); + expect((after.raw as { _detail?: unknown })._detail).toMatchObject({ imdbId: "tt36758489", doubanId: "37297001" }); + expect((after.raw as { _escalation?: { status: string } })._escalation?.status).toBe("fetched"); + expect(after.resolution_evidence).toMatchObject({ + escalation: { detail: "fetched", via: "site_id", pending: false }, + }); + // 站点英文名 A Prophet 与作品名对不上:只凭 ID 确认,不登记为别名(ID-2 守卫照旧) + const aliases = await h.db + .selectFrom("title_alias") + .select("alias") + .where("media_entity_id", "=", res.mediaEntityId!) + .where("source", "=", "site") + .execute(); + expect(aliases.map((a) => a.alias)).not.toContain("A Prophet"); + expect(await escalationCount(h.rt, audiences)).toBe(1); + + // 再识别一次:详情已抓过,不再请求站点;IDs 在 raw._detail 里,第一轮就按 ID 认出 + const again = await resolveRelease(h.rt, { sourceReleaseId: r.id, force: true, trigger: "manual" }); + expect(["auto", "skipped"]).toContain(again.outcome); + expect(detailRequests("727077")).toHaveLength(1); + }); + + it("观众 725858:同名同年两部 → 有歧义的事项;手动重新识别抓详情 → IMDb 确认 → 事项自动关闭;详情页豆瓣信息块进豆瓣条目缓存", async () => { + const r = await release(audiences, "725858", "Rose 2026 1080p BluRay AVC DTS-HD MA 5.1-DIY@Audies"); + // 不允许升级时(旧行为):歧义 → 开 identity_ambiguous + const first = await resolveRelease(h.rt, { sourceReleaseId: r.id, trigger: "manual", escalate: false }); + expect(first.outcome).toBe("ambiguous"); + expect(first.decisionId).not.toBeNull(); + expect(detailRequests("725858")).toHaveLength(0); + + const res = await resolveRelease(h.rt, { sourceReleaseId: r.id, force: true, trigger: "manual" }); + expect(res.outcome).toBe("auto"); + expect(await tmdbIdOf(res.mediaEntityId!)).toBe("990001"); + expect(detailRequests("725858")).toHaveLength(1); + const d = await h.db + .selectFrom("decision") + .select(["status"]) + .where("id", "=", first.decisionId!) + .executeTakeFirst(); + expect(d?.status).toBe("resolved"); + const pool = await h.db + .selectFrom("identity_candidate") + .select(["status", "resolved_media_entity_id"]) + .where("source_release_id", "=", r.id) + .executeTakeFirstOrThrow(); + expect(pool).toMatchObject({ status: "resolved", resolved_media_entity_id: res.mediaEntityId }); + // 详情页信息块(◎片名 Rose / ◎译名 罗斯 / ◎年代 2026)+ 豆瓣 ID 35322132 → 豆瓣条目 + const subject = await loadDoubanSubject(h.ctx, "35322132"); + expect(subject).toMatchObject({ id: "35322132", title: "Rose", year: 2026, source: "site_detail" }); + expect(subject?.aliases).toContain("罗斯"); + }); + + it("憨憨详情样本:MediaInfo 里的 TMDB tv/290001 直取详情 → 自动确认", async () => { + const r = await release(hhan, "91003", "Example Show S01 2026 2160p YK WEB-DL H.265 DV HQ AAC2.0-HHWEB"); + const res = await resolveRelease(h.rt, { sourceReleaseId: r.id, trigger: "search" }); + expect(res.outcome).toBe("auto"); + expect(await tmdbIdOf(res.mediaEntityId!)).toBe("290001"); + expect(detailRequests("91003")).toHaveLength(1); + expect((await row(r.id)).resolution_evidence).toMatchObject({ escalation: { via: "site_id" } }); + }); + + it("每站每小时上限:达到后推迟(不开事项、15 分钟内复查),不再请求站点;0 = 不抓", async () => { + await setCap(1); + const a = await release(audiences, "990101", "Nowhere Title 2026 1080p WEB-DL"); + const b = await release(audiences, "990102", "Another Nowhere 2026 1080p WEB-DL"); + const before = siteRequests.length; + await resolveRelease(h.rt, { sourceReleaseId: a.id, trigger: "search" }); + const rb = await resolveRelease(h.rt, { sourceReleaseId: b.id, trigger: "search" }); + // a 用掉名额(样本里没有这个 id 的页面,详情请求失败也算用掉;失败不阻塞,照常进池,6 小时内不再为识别重抓) + expect(siteRequests.length - before).toBe(1); + expect((await row(a.id)).resolution_evidence).toMatchObject({ + escalation: { detail: "failed:fetch_failed", pending: false }, + }); + await resolveRelease(h.rt, { sourceReleaseId: a.id, force: true, trigger: "manual" }); + expect((await row(a.id)).resolution_evidence).toMatchObject({ escalation: { detail: "skipped:recent_failure" } }); + expect(siteRequests.length - before).toBe(1); + expect(rb.decisionId).toBeNull(); + const pool = await h.db + .selectFrom("identity_candidate") + .select(["query", "next_review_at"]) + .where("source_release_id", "=", b.id) + .executeTakeFirstOrThrow(); + expect(pool.query).toMatchObject({ escalation: { detail: "deferred:cap_reached", pending: true } }); + expect(pool.next_review_at!.getTime() - Date.now()).toBeLessThanOrEqual(15 * 60_000 + 5_000); + + await setCap(0); + const c = await release(audiences, "990103", "Third Nowhere 2026 1080p WEB-DL"); + await resolveRelease(h.rt, { sourceReleaseId: c.id, trigger: "search" }); + expect((await row(c.id)).resolution_evidence).toMatchObject({ escalation: { detail: "skipped:disabled" } }); + expect(siteRequests.length - before).toBe(1); + }); + + it("订阅相关的资源:RSS 入库时不抓详情、不开「没识别出作品」;复查时抓详情认出;旧事项自动关闭", async () => { + // 本地作品(如订阅)登记了别名「Example Show」,但没有 TMDB 身份、年份也对不上:资源与它相关(tracked_alias)却认不出 + const e = await h.db + .insertInto("media_entity") + .values({ kind: "series", title: "Example Show", year: 1990 } as never) + .returning("id") + .executeTakeFirstOrThrow(); + await sql`insert into title_alias (media_entity_id, alias, normalized, kind, source) + values (${e.id}, 'Example Show', ${normalizeTitle("Example Show")}, 'other', 'user')`.execute(h.db); + const r = await release(hhan, "91004", "Example Show S01E18 2027 2160p YK WEB-DL H.265 DV HQ AAC2.0-HHWEB"); + + // 旧行为(不升级):相关却认不出 → identity_unresolved + const old = await resolveRelease(h.rt, { sourceReleaseId: r.id, trigger: "review", force: true, escalate: false }); + expect(old.decisionId).not.toBeNull(); + const oldKind = ( + await h.db.selectFrom("decision").select("kind").where("id", "=", old.decisionId!).executeTakeFirstOrThrow() + ).kind; + + // RSS 入库(observed):不抓详情,推迟到复查;不再保留事项 + const obs = await resolveRelease(h.rt, { sourceReleaseId: r.id, force: true, trigger: "observed" }); + expect(obs.decisionId).toBeNull(); + expect(detailRequests("91004")).toHaveLength(0); + expect((await row(r.id)).resolution_evidence).toMatchObject({ + escalation: { detail: "not_now:batch_ingest", pending: true }, + }); + const d1 = await h.db.selectFrom("decision").select("status").where("id", "=", old.decisionId!).executeTakeFirst(); + expect(d1?.status).not.toBe("open"); + + // 复查(订阅相关):抓详情 → TMDB tv/290001(年份 2027 与 2026 差 1 年,在容差内)→ 自动确认 + const rev = await resolveRelease(h.rt, { sourceReleaseId: r.id, force: true, trigger: "review" }); + expect(detailRequests("91004")).toHaveLength(1); + expect(rev.outcome).toBe("auto"); + expect(oldKind).toBe("identity_unresolved"); + expect(await openDecisions()).toHaveLength(0); + }); + + it("豆瓣路径:详情只有豆瓣 ID → 缓存的豆瓣条目(搜索摘要抽取)标题 + 年份 → TMDB 匹配(ID-2 打分)", async () => { + // META-2 搜索摘要里抽到的豆瓣条目(不请求豆瓣) + const s = doubanSubjectFromSnippet( + { + url: "https://movie.douban.com/subject/36000099/", + title: "青灯记 (豆瓣)", + snippet: "导演: 某导演 主演: 甲 / 乙 类型: 剧情 / 古装 制片国家/地区: 中国大陆 年份: 2025 集数: 30", + }, + new Date(), + ); + expect(s).toMatchObject({ id: "36000099", title: "青灯记", year: 2025, source: "web_snippet" }); + await saveDoubanSubject(h.ctx, s!); + // 详情已抓过,只给了豆瓣 ID + const r = await release(hhan, "91005", "Qing Deng Ji S01E03 2025 1080p WEB-DL H264 AAC-HHWEB", { + _detail: { doubanId: "36000099" }, + }); + await h.db.updateTable("source_release").set({ detail_fetched_at: new Date() }).where("id", "=", r.id).execute(); + const res = await resolveRelease(h.rt, { sourceReleaseId: r.id, trigger: "manual" }); + expect(res.outcome).toBe("auto"); + expect(await tmdbIdOf(res.mediaEntityId!)).toBe("990021"); + expect(detailRequests("91005")).toHaveLength(0); + expect((await row(r.id)).resolution_evidence).toMatchObject({ + escalation: { via: "douban", douban: { id: "36000099", title: "青灯记", source: "web_snippet" } }, + }); + + // 缓存里没有这个豆瓣条目:不请求豆瓣,只进池;有 ID 的资源不开「没识别出作品」 + await sharedCache(h.ctx).delete(DOUBAN_SUBJECT_REGION, "36000098"); + const r2 = await release(hhan, "91006", "Unknown Drama S01E01 2025 1080p WEB-DL", { + _detail: { doubanId: "36000098" }, + }); + await h.db.updateTable("source_release").set({ detail_fetched_at: new Date() }).where("id", "=", r2.id).execute(); + const res2 = await resolveRelease(h.rt, { sourceReleaseId: r2.id, trigger: "manual" }); + expect(res2.outcome).toBe("no_match"); + expect(res2.decisionId).toBeNull(); + expect(siteRequests.some((x) => x.includes("douban"))).toBe(false); + }); + + it("池条目补全前先走升级链:详情页给了 ID 就直接认出,返回 409,不做网页补全", async () => { + const r = await release(hhan, "91007", "Example Show S01 2026 2160p YK WEB-DL H.265 DV HQ AAC2.0-HHWEB"); + const first = await resolveRelease(h.rt, { sourceReleaseId: r.id, trigger: "manual", escalate: false }); + expect(first.outcome).toBe("no_match"); + const pool = await h.db + .selectFrom("identity_candidate") + .select("id") + .where("source_release_id", "=", r.id) + .executeTakeFirstOrThrow(); + const res = await h.req(`/identity-pool/${pool.id}/enrich`, { body: {} }); + expect(res.status).toBe(409); + expect(((await res.json()) as { error: { message: string } }).error.message).toContain("《示例剧集》"); + expect(detailRequests("91007")).toHaveLength(1); + expect((await row(r.id)).resolution_status).toBe("resolved"); + }); + + it("站点详情来源的豆瓣条目不会被搜索摘要覆盖", async () => { + await saveDoubanSubject(h.ctx, { + id: "1", + title: "甲", + aliases: [], + year: 2020, + source: "site_detail", + ref: "x", + at: new Date().toISOString(), + }); + await saveDoubanSubject(h.ctx, { + id: "1", + title: "乙", + aliases: [], + year: 2021, + source: "web_snippet", + ref: "y", + at: new Date().toISOString(), + }); + expect((await loadDoubanSubject(h.ctx, "1"))?.title).toBe("甲"); + }); +}); diff --git a/apps/web/app/components/settings/metadata.tsx b/apps/web/app/components/settings/metadata.tsx index 80d3d8eb..42562ce7 100644 --- a/apps/web/app/components/settings/metadata.tsx +++ b/apps/web/app/components/settings/metadata.tsx @@ -207,6 +207,12 @@ const IDENTITY_CARD: CardSpec = { hint: "每条新资源都查一次 TMDB。RSS 量大时每天可能上千次请求，站点独有的短剧和综艺多半查不到", wide: true, }, + { + name: "identity.detailIdsPerSiteHour", + label: "识别时每站每小时最多抓详情页", + kind: "number", + hint: "标题认不准时，抓站点详情页取 IMDb、TMDB 或豆瓣 ID 再识别；只在搜索、订阅相关资源、补全和手动重新识别时抓，RSS 入库不抓。0 = 不抓", + }, { name: "identity.autoEnrich", label: "TMDB 查不到时自动补全元数据", @@ -241,6 +247,7 @@ const IDENTITY_CARD: CardSpec = { "identity.enrichDailyBudget": n(s.identity.enrichDailyBudget), "identity.enrichDomains": s.identity.enrichDomains.join("\n"), "identity.enrichSnippetAllSites": s.identity.enrichSnippetAllSites ?? true, + "identity.detailIdsPerSiteHour": n(s.identity.detailIdsPerSiteHour ?? 10), }), toPatch: (v) => ({ identity: { @@ -252,6 +259,7 @@ const IDENTITY_CARD: CardSpec = { .map((d) => d.trim().toLowerCase()) .filter(Boolean), enrichSnippetAllSites: v["identity.enrichSnippetAllSites"] === true, + detailIdsPerSiteHour: Number(v["identity.detailIdsPerSiteHour"]), }, }), extra: () => , diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index a1483faf..ca4b3e48 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -496,6 +496,11 @@ export const IdentitySettings = z.object({ * 缺省开;关掉后只用白名单网站的摘要与网页。旧设置没有这个键,按开处理 */ enrichSnippetAllSites: z.boolean().optional(), + /** + * ID-3 识别升级:标题识别不确定时,每个站点每小时最多为识别抓多少个详情页(取 IMDb / TMDB / 豆瓣 ID);0 = 不抓。 + * 只在搜索结果、订阅相关的候选、池条目补全和手动重新识别时抓,RSS 批量入库不抓。旧设置没有这个键,按缺省 10 + */ + detailIdsPerSiteHour: z.number().int().min(0).max(200).optional(), }); export type IdentitySettings = z.infer; @@ -699,6 +704,7 @@ export const DEFAULT_SETTINGS: GlobalSettings = { enrichDailyBudget: 20, enrichDomains: DEFAULT_ENRICH_DOMAINS, enrichSnippetAllSites: true, + detailIdsPerSiteHour: 10, }, search: { defaultSiteMode: "default", siteIds: [] }, storage: DEFAULT_STORAGE_SETTINGS, @@ -998,6 +1004,7 @@ export const SETTINGS_SECTIONS: readonly SettingsSectionDef[] = [ "identity.enrichDailyBudget", "identity.enrichDomains", "identity.enrichSnippetAllSites", + "identity.detailIdsPerSiteHour", "cache.globalImageCache", "cache.globalImageCacheDays", ], From bca9ecb6cf1ffc9b9188137e8009962b3a0785c4 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 1 Oct 2026 00:15:20 +0800 Subject: [PATCH 4/4] docs(identity): ID-3 escalation chain, triggers and list IDs Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 10 +++++++ docs/runbook/identity.md | 62 +++++++++++++++++++++++++++++++++++++++- docs/runbook/sites.md | 5 ++-- 3 files changed, 74 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29b5f7e4..109fa23b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ ## [Unreleased] +### 新增(ID-3 用站点详情页的 ID 识别资源) +- 识别升级链:标题认不准(有歧义 / 没结果)时,资源有站点详情页且还没抓过,就抓一次详情(走详情缓存与站点限流,失败不影响识别)取 IMDb / TMDB / 豆瓣 ID,再按 ID 查 TMDB 确认作品;确认后照常回填资源、登记别名(守卫照旧)、关闭同资源的待识别项。 +- 只在搜索结果、订阅相关的资源、池条目补全前、手动重新识别时抓;RSS 批量入库不抓(订阅相关的推迟到复查)。每站每小时上限可在设置 → 元数据 → 资源识别里调(缺省 10,0 = 不抓)。 +- 站点给出的 IMDb / TMDB ID 作为强证据参与打分:标题对不上也能自动确认;年份或类型冲突时只作候选(与 MP 的差别登记在差分组 `site-ids-*`)。 +- 详情只有豆瓣 ID 时,用缓存的豆瓣条目(来自站点详情页的豆瓣信息块或元数据补全的搜索摘要,不请求豆瓣)按标题 + 年份匹配 TMDB。 +- 观众列表页的豆瓣链接抽成 `doubanId`(新增 UMT 扩展字段 `torrents.fields.doubanid`)。 + +### 变更 +- 能从站点拿到 ID 的资源不再开「没识别出作品」事项;升级链还在排队(每站每小时限额已满)时不开事项、15 分钟内复查;已开的事项在资源被识别后自动关闭。 + ## [0.3.6] - 2026-09-30 ### 新增(SEARCH-4 资源筛选对齐 MP) diff --git a/docs/runbook/identity.md b/docs/runbook/identity.md index b9c78049..416441ec 100644 --- a/docs/runbook/identity.md +++ b/docs/runbook/identity.md @@ -26,6 +26,9 @@ 3. 查了 TMDB 的,召回 TMDB 候选(`recallTmdbCandidates`:中文名 → 英文名,按类型单语言搜索,标题对不上的按年份相近度拉前几个详情比对别名 / 译名), 交给 `domain.resolveIdentity` 打分:auto 建档 / 关联并发 `release.resolved`;否则进池。打分规则与 MP 的对照见 `docs/porting/moviepilot-identity-match.md`(ID-2)。 + 站点列表 / 详情页给出的 IMDb / TMDB ID(`raw._site` / `raw._detail`)作为 `trustedIds` 参与打分(ID-3):候选按这个 ID 取来时, + 标题对不上也至少 `multi_evidence`(可自动确认);年份冲突降为 `needs_confirm`;续作序号 / 类型冲突不放宽。 + 标题认不准时走「识别升级链」(见下节)。 4. 自动绑定时把站点标题登记为作品别名,只在标题精确命中(或别名命中且另一种文字的名称也命中)时登记,分类词 / 单字 / 停用词不登记。 FIX-5 守卫(`service/alias-guard.ts`):逐个名称再查一遍,以下任一成立就不登记,原因记在 `resolution_attempt.best.aliases.skipped` (`added` 是登记了的):证据等级不是 `rule_confirmed`(`low_evidence`);资源年份与作品年份差超过 1 年且不是季年份(`year_conflict`); @@ -39,6 +42,62 @@ 搜索模块想让搜索结果直接识别,入队 `resolveObservation` 时传 `trigger: "search"`(目前搜索结果经 `release.observed` 进来, 按 observed 处理;命中订阅别名的照样会查)。 +## 识别升级链:用站点详情页的 ID(ID-3) + +用户反馈(2026-09-30):"既然能抓 PT 站详情页,详情有 TMDB、IMDb 或豆瓣,为啥还判断不了是什么资源?" 以前详情页只在用户打开时才抓, +标题识别不出就直接进待识别 / 待处理。现在标题识别结果是 `ambiguous` / `no_match` 时按下面的链再认一次 +(`service/id-escalation.ts`、`resolve.ts escalateIdentity`): + +1. **抓详情**(只在资源还没抓过详情时,`detail_fetched_at` 为空):走 `refreshReleaseDetail`——站点详情缓存、共享限流器、请求计数、 + 凭据失效处理都照旧;失败不阻塞识别,记在 `raw._escalation {at, status, error}`,6 小时内不再为识别重抓。 +2. **新拿到的 IMDb / TMDB ID** → TMDB 详情 / `find` → 按 `trustedIds` 打分(列表上已有的 ID 在第一轮就用过)。 +3. 还不能确认且有**豆瓣 ID** → `kv_cache` region `douban.subject` 里缓存的豆瓣条目(标题 / 别名 / 年份)→ 按条目名称 + 年份召回 TMDB → + ID-2 打分(对照 MP `chain/media/projection.py` 豆瓣 → TMDB:原名 / 中文名 / 英文名 + 年份)。**不请求豆瓣**;条目来源只有两个: + 站点详情页的豆瓣信息块(◎片名 / ◎译名 / ◎年代,详情页同时给了豆瓣 ID 时写入,`source = site_detail`)和元数据补全(META-2) + 网页搜索结果里 `douban.com/subject/` 的搜索标题与摘要(`source = web_snippet`,不覆盖 site_detail 的)。缓存 30 天。 +4. 确定(auto)→ 照常绑定:回填资源、建 / 关联作品、关闭同资源的池条目并从合并事项里摘除(组空即关)。站点标题登记为别名时 + 按资源**自己的标题**重新判断(靠 ID / 豆瓣名确认、标题对不上的不登记),FIX-5 守卫照旧。 + +证据记在 `source_release.resolution_evidence.escalation` 与池条目 `query.escalation`:`detail`(`fetched` / `skipped:<原因>` / +`deferred:cap_reached` / `failed:fetch_failed` / `not_now:batch_ingest`)、`via`(`site_id` / `douban` / null)、`ids`、`pending`、`douban`。 + +### 什么时候抓详情 + +| 触发 | 抓详情? | +|---|---| +| 搜索结果(`trigger: "search"`,搜索完成后逐条入队识别) | 是 | +| 手动「重新识别」、Agent `identity.resolve`(`manual`) | 是 | +| 池条目「元数据补全」之前(`POST /identity-pool/{id}/enrich`) | 是:资源没抓过详情或有豆瓣 ID 时先按手动重新识别走一次;认出来就返回 409「已识别为《X》,不需要补全」,不消耗补全预算 | +| 订阅相关的候选(命中订阅 / 本地作品别名,`tracked_alias`)的池复查(`review`) | 是 | +| RSS / 列表批量入库(`observed`) | 否。订阅相关的推迟(`not_now:batch_ingest`,15 分钟内复查时再抓);其余不抓 | +| 上次因限额推迟(`pending`)的复查 | 是 | + +每站每小时上限 `identity.detailIdsPerSiteHour`(设置 → 元数据 → 资源识别「识别时每站每小时最多抓详情页」,缺省 10,0 = 不抓), +计数在 `kv_cache` region `identity.detail_ids`(key `:<小时>`,原子自增,2 小时过期)。达到上限记 `deferred:cap_reached`、 +15 分钟内复查。这些详情请求同样计入站点请求统计(`site_request_count`)。 + +### 待处理 + +- 升级链推迟中(`pending`:每站每小时限额已满 / 入库时不抓):不开任何身份事项,之前开的从合并事项摘除。抓了但失败的不算推迟(不阻塞,照常进池 / 开事项)。 +- 资源上已有 IMDb / TMDB / 豆瓣 ID 却仍无结果:不开「没识别出作品」(`identity_unresolved`),只留在池里,可手动指定。 +- 有候选的歧义(`identity_ambiguous`)照旧开;资源被识别后(升级链 / 复查 / 手动)自动从合并事项摘除,组空即关闭。 + +### 排查 + +```sql +-- 最近为识别抓详情的资源 +select id, title, raw->'_escalation' esc, resolution_evidence->'escalation' ev from source_release + where raw ? '_escalation' order by (raw->'_escalation'->>'at') desc limit 20; +-- 这一小时各站点用了多少名额 +select key, value from kv_cache where region = 'identity.detail_ids' and expires_at > now(); +-- 缓存的豆瓣条目 +select key, value->>'title' title, value->>'year' y, value->>'source' src from kv_cache where region = 'douban.subject' limit 20; +``` + +测试:`apps/server/test/sources-identity/id-escalation.test.ts`(观众 DET-2 / 憨憨 HH-2 详情样本经注入 fetch、TMDB mock,零真实请求); +差分组 `packages/domain/test/differential/identity/site-ids-*.json`(MP `search/result.py _match_source`:种子 IMDb 与作品一致即命中; +我们年份 / 类型冲突时只作候选,登记为 intentional,见 `site-ids-deviations.json`),`packages/domain/test/identity-id3.test.ts`。 + ## 接管任务按文件名识别(QB-1) 用户手动加进 qB、贴了管理标签的任务(`download_task.origin = adopted`,没有 `source_release` / 计划)下载完成后,`verifyAndImport` @@ -63,7 +122,8 @@ |---|---| | `unmatched` | 否 | | `ambiguous` / `proposed`(查过 TMDB,有候选) | 是,`identity_ambiguous`,级别 warning | -| `no_match` / 复查已停的 `lookup_failed` | 只有本地召回有候选(与本地作品相关)时开 `identity_unresolved`,级别 warning | +| `no_match` / 复查已停的 `lookup_failed` | 只有本地召回有候选(与本地作品相关)、且资源上没有站点给的 IMDb / TMDB / 豆瓣 ID 时开 `identity_unresolved`,级别 warning | +| 识别升级链推迟中(ID-3) | 否(之后会抓详情再认) | 合并:dedupeKey `identity-group::<归一化标题>`,同来源同标题的资源并成一条。成员 = `identity_candidate.decision_id` 指向该事项且仍 `pending` 的条目。标题 1 条时写资源原标题,多条时写「来源「X」的 N 条资源待识别 / 待确认身份:<标题>」, diff --git a/docs/runbook/sites.md b/docs/runbook/sites.md index c0c48c06..801f9f53 100644 --- a/docs/runbook/sites.md +++ b/docs/runbook/sites.md @@ -112,9 +112,10 @@ 资源详情抽屉的「站点详情页」一节:简介、MediaInfo / BDInfo、截图、IMDb / 豆瓣 / TMDB 链接、文件清单、标签、上传者。 -- 只按需抓:打开抽屉时如果没有缓存,前端自动调一次 `GET /sites/{id}/releases/{rid}?refresh=true`(同一资源本次会话只自动一次),或点「复核详情」;`release.inspect` 的 `refreshDetail` 同一路径。列表快照、搜索不抓详情。 +- 只按需抓:打开抽屉时如果没有缓存,前端自动调一次 `GET /sites/{id}/releases/{rid}?refresh=true`(同一资源本次会话只自动一次),或点「复核详情」;`release.inspect` 的 `refreshDetail` 同一路径。列表快照、搜索本身不抓详情;标题识别认不准时,识别升级链(ID-3)会为取 ID 抓一次(同一路径、同一限流器,每站每小时有上限,见 `docs/runbook/identity.md`「识别升级链」)。 - 缓存:内容放 PG `kv_cache` region `site.detail`,key = `source_release.id`,TTL 取站点详情缓存期(至少 24 小时)。缓存期内 `detail_fetched_at` 新鲜且缓存还在就不请求站点;kv_cache 被清掉(过期 / 重启丢 UNLOGGED 表)后下次刷新会重新抓一次。 -- 外部 ID 另写 `source_release.raw._detail`(`imdbId / doubanId / tmdb{type,id}`),列表快照 upsert 不覆盖它。身份解析把它和列表给的 `raw._site.imdbId / doubanId` 一起当显式 ID:本地召回查已接受的外部身份;需要查 TMDB 时,有 TMDB ID 直接取详情,否则 IMDb ID 走 `/find/{imdb}?external_source=imdb_id`,命中就不再按标题搜索;TMDB 不支持豆瓣 ID,豆瓣只用于本地召回。是否查 TMDB 仍按惰性识别规则(DEC-1),外部 ID 本身不触发查询。 +- 外部 ID 另写 `source_release.raw._detail`(`imdbId / doubanId / tmdb{type,id}`),列表快照 upsert 不覆盖它。身份解析把它和列表给的 `raw._site.imdbId / doubanId` 一起当显式 ID:本地召回查已接受的外部身份;需要查 TMDB 时,有 TMDB ID 直接取详情,否则 IMDb ID 走 `/find/{imdb}?external_source=imdb_id`,命中就不再按标题搜索;TMDB 不支持豆瓣 ID:豆瓣 ID 用于本地召回,识别升级链(ID-3)里再经缓存的豆瓣条目(标题 + 年份)匹配 TMDB。是否查 TMDB 仍按惰性识别规则(DEC-1),外部 ID 本身不触发查询。 +- 列表页外部 ID(ID-3 核对):NexusPHP 定义的 `torrents.fields.imdbid`(MP 字段)全部站点都有;豆瓣是 UMT 扩展字段 `torrents.fields.doubanid`(MP 不读,索引差分只比 MP 字段),写法同 imdbid。现有列表样本里只有观众 `rl-audiences-list` 带 ID 徽标(10 行豆瓣、5 行 IMDb),已给观众补 `doubanid`(`a[href*="douban.com/subject/"]` + `re_search subject/(\d+)`,覆盖层 `scripts/mp-sync/movieclaw-overrides/audiences.json`,测试 `packages/adapters/test/differential/list-ids.test.ts`)。憨憨 / 彩虹岛的列表样本没有 IMDb / 豆瓣链接(ID 只在详情页),其它站点没有列表样本,暂不加选择器。 - 截图:响应里只有代理路径 `/api/v1/img/site-detail/{releaseId}/{index}`,服务端从缓存取原始地址,公网校验(每跳重定向复核)、不带站点凭据、按文件头认图片(不放行 SVG)、≤ 8 MB,浏览器缓存 24 小时。站内附件如果需要登录才能看会显示为裂图,不会为截图消耗站点请求。 - 憨憨详情页(HH-2 按真实页面 `probe.labels` 核对):没有 `#kdescr`,内容分节为「标签 div + 相邻值 div」:`mediainfo`、`海报`、`豆瓣`(内有 `span.font-bold` 小标签 片名 / 年代 / 类别 / 导演 …)、`截图`、`种子文件`(只有「查看列表」按钮)、`发布者`、`发布时间`。对应规则:简介没有容器时取「豆瓣 / 影片信息 / IMDb」信息块,小标签补冒号、一项一行;截图取「截图」一节的 img(`data-src` 优先,海报不算截图),没有再取简介里的图;IMDb 依次取页面链接 → 简介 / 信息块 / MediaInfo 里的 `imdb.com/title/tt…` 或 `IMDB : tt…` 行(只收 `tt` + 7~8 位,不收孤立 tt 串),TMDB 取链接或 `TMDB : tv/294990` 行,豆瓣取链接或 `douban.com/subject/…`;兜底出的 IMDb / 豆瓣也回填到 `SiteRelease.imdbId / doubanId`;发布时间没有 `span[title]` 时取「发布时间 / 上传时间」字段文本。文件清单:定义 `umt.detail.page.fileListPath`(憨憨 `viewfilelist.php?id={id}`)配置了且页面上没有内联清单时,同一次详情刷新里再请求 1 次(走同一限流器、计数),失败只是没有清单。fixture `HHANCLUB_DETAIL_PAGE_2026` / `HHANCLUB_FILELIST_FRAGMENT`:标签名称与顺序来自真实页面,嵌套按已验证的标题 / 副标题结构推定,部署后打开一条憨憨资源的详情抽屉核对一次。 - 观众详情页(DET-2 按真实页面核对,样本 `packages/adapters/test/differential/inputs/detail/rl-audiences-detail-*.html.txt` 已脱敏,回归测试 `test/differential/detail.test.ts`;列表页样本 `inputs/indexer/rl-audiences-list.html.txt` 进 MP 差分):标准 rowhead 表格,但有几处与通用 NexusPHP 不同,写在定义里。h1 的促销是 `font.free / font.halfdown …`(列表是 `img.pro_*`)加「剩余时间：」和到期 `span[title]`,用 `umt.detail.promo` 覆盖列表的促销字段;h1 的「剩余时间：」文本不进标题,标题区里的 `span[title]` 也不当发布时间(发布时间取「下载」行「发布于」的 span)。没有「同伴」行,做种 / 下载 / 完成数取基本信息行右侧 `#seeders / #leechers / a[title=完成]`(`umt.detail.fields.seeders / leechers / snatched`);种子 Hash 取 `#copyTorrentHashLink[data-hash]`(`fields.infoHash`,回填 `infoHash`)。副标题行右侧有豆瓣 / IMDb / 烂番茄 / Metacritic 评分徽标(`[title*=评分]`),解析时去掉;评分本身本工具没有字段,不收。标签是 `span.detail-tag-list > span.tags`,只取最内层。上传者在「下载」行的 `span.torrent-uploader-wrap`(匿名发布为「匿名」);另有「发布者带宽」一行,所以上传者标签改为精确匹配。简介开头的站点声明 `#ad_torrentdetail` 用 `umt.detail.page.descriptionStrip` 去掉;MediaInfo / BDInfo 是 `div.codetop`(文件名 / BDInfo)+ `div.show`(摘要表或同一份 BDInfo)+ `div.hide > div.codemain`(原文),三者都从简介里去掉;简介是 `
\r\n`,源码换行不再算一行(以前每行之间多一空行)。截图排除豆瓣海报(站点经 `dbimgs.audiences.me/?…doubanio.com…poster` 代理)与站点横幅 `/banners/`。IMDb / 豆瓣 ID 取简介里的链接;TMDB 页面上没有;单种 H&R 标记页面上没有(列表也没有),保持「未知」。文件清单不在页面上(点「N个文件」后 AJAX 加载,端点未核对),暂不另发请求。列表副标题改取 `span.torrent-subtitle-text`(`span.torrent-subtitle-line` 里先是标签,以前副标题前面会粘着「官方中字DIY…」)。