feat(identity): escalate ambiguous/no_match via site detail-page IDs (ID-3)
Fetch the detail page (cached, rate-limited, per-site hourly cap identity.detailIdsPerSiteHour) for search results, subscription-related candidates, pool enrich and manual re-identify; resolve IMDb/TMDB via TMDB, douban via cached douban subjects (site detail info block / META-2 snippets). No identity_unresolved decisions for releases with site IDs. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
cba214e93e
commit
3d5a50b30a
9 files changed
+867
-10
No files matched your search
@@ -15,8 +15,10 @@ import { ApiException, errorBody } from "../../../http/errors.ts";
|
||||
import type { ModuleApp } from "../../types.ts";
|
||||
import { route } from "../http.ts";
|
||||
import type { SourcesRuntime } from "../runtime.ts";
|
||||
import { siteExplicitIds } from "../service/detail-page.ts";
|
||||
import { applyEnrichment, domainAllowed, enrichStats, hostOf } from "../service/enrich.ts";
|
||||
import { fetchPublicImage, imageResponse } from "../service/image-proxy.ts";
|
||||
import { resolveRelease } from "../service/resolve.ts";
|
||||
|
||||
// META-1 元数据补全 API:先看建议(POST /identity-pool/{id}/enrich,经能力注册中心调用 metadata.enrich,
|
||||
// 授权 / 审计与 Agent 同一路径),再应用(POST /identity-pool/{id}/enrich/apply,只接受控制台管理员会话);
|
||||
@@ -40,6 +42,13 @@ export function enrichRoutes(app: ModuleApp, rt: () => SourcesRuntime): void {
|
||||
},
|
||||
async ({ params, body, actor, c }) => {
|
||||
const ctx = rt().ctx;
|
||||
// ID-3:补全(网页搜索)之前先走识别升级链——站点详情页有 IMDb / TMDB / 豆瓣 ID 就能直接认出来,不必搜网页
|
||||
const resolved = await escalateBeforeEnrich(rt(), params.id);
|
||||
if (resolved)
|
||||
throw new ApiException(
|
||||
"conflict",
|
||||
`站点详情页给出了作品 ID，已识别为《${resolved.title}》，不需要补全。刷新待识别列表查看`,
|
||||
);
|
||||
try {
|
||||
return await ctx.capabilities.invoke(
|
||||
"metadata.enrich",
|
||||
@@ -131,3 +140,32 @@ export function enrichRoutes(app: ModuleApp, rt: () => SourcesRuntime): void {
|
||||
return imageResponse(got, 86400 * 7);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 池条目补全前的识别升级:资源还没抓过详情页、或已有豆瓣 ID 时,按手动重新识别走一次(允许抓详情);认出来返回作品。
|
||||
* 升级失败不影响补全。
|
||||
*/
|
||||
async function escalateBeforeEnrich(r: SourcesRuntime, poolEntryId: string): Promise<{ title: string } | null> {
|
||||
const db = r.ctx.db;
|
||||
const row = await db
|
||||
.selectFrom("identity_candidate as p")
|
||||
.innerJoin("source_release as s", "s.id", "p.source_release_id")
|
||||
.select(["s.id", "s.detail_fetched_at", "s.raw", "s.resolution_locked", "p.status"])
|
||||
.where("p.id", "=", poolEntryId)
|
||||
.executeTakeFirst();
|
||||
if (row?.status !== "pending" || row.resolution_locked) return null;
|
||||
if (row.detail_fetched_at && !siteExplicitIds(row.raw).some((x) => x.provider === "douban")) return null;
|
||||
try {
|
||||
const res = await resolveRelease(r, { sourceReleaseId: row.id, force: true, trigger: "manual", escalate: true });
|
||||
if (res.outcome !== "auto" || !res.mediaEntityId) return null;
|
||||
const e = await db
|
||||
.selectFrom("media_entity")
|
||||
.select(["title"])
|
||||
.where("id", "=", res.mediaEntityId)
|
||||
.executeTakeFirst();
|
||||
return { title: e?.title ?? "?" };
|
||||
} catch (e) {
|
||||
r.ctx.logger.warn({ poolEntryId, err: { message: (e as Error).message } }, "补全前的识别升级失败");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import { sql } from "@umt/db";
|
||||
import { sharedCache, storageSettings } from "../../../shared-cache.ts";
|
||||
import type { SourcesRuntime } from "../runtime.ts";
|
||||
import type { Exec } from "./common.ts";
|
||||
import { doubanSubjectFromDetail, saveDoubanSubject } from "./douban-subject.ts";
|
||||
|
||||
// 站点详情页扩展内容(简介 / MediaInfo / 截图 / 外部 ID / 文件 / 标签 / 上传者):
|
||||
// - 只在按需抓详情时得到(工作台打开详情、release.inspect 复核),放 PG kv_cache region site.detail,
|
||||
@@ -44,6 +45,12 @@ export async function saveDetailPage(
|
||||
};
|
||||
await sql`update source_release set raw = raw || jsonb_build_object('_detail', ${JSON.stringify(ids)}::jsonb)
|
||||
where id = ${srId}`.execute(exec);
|
||||
// ID-3:详情页的豆瓣信息块(◎片名 / ◎年代)连同豆瓣 ID 存成豆瓣条目,只有豆瓣 ID 时按标题 + 年份匹配 TMDB
|
||||
const subject = doubanSubjectFromDetail(page, `source_release:${srId}`, rt.now());
|
||||
if (subject)
|
||||
await saveDoubanSubject(rt.ctx, subject).catch((e) =>
|
||||
rt.ctx.logger.warn({ err: { message: (e as Error).message } }, "豆瓣条目缓存写入失败"),
|
||||
);
|
||||
}
|
||||
|
||||
async function loadStored(rt: SourcesRuntime, srId: string): Promise<StoredPage | null> {
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
import { formatDetailPage } from "@umt/adapters";
|
||||
import { pgCacheFor } from "@umt/db";
|
||||
import type { ServerContext } from "../../types.ts";
|
||||
|
||||
// ID-3 豆瓣条目缓存(kv_cache region douban.subject,key = 豆瓣 subject id):只存标题 / 别名 / 年份,供「详情页只有豆瓣 ID」时
|
||||
// 按标题 + 年份匹配 TMDB(对照 MP chain/media/projection.py 豆瓣 → TMDB:取豆瓣条目的原名 / 中文名 / 英文名 + 年份做 TMDB 名称匹配)。
|
||||
// 本工具不请求豆瓣页面或接口,条目只从已经拿到的数据里抽:
|
||||
// - site_detail:站点详情页的豆瓣信息块(◎片名 / ◎译名 / ◎年代,多数由 PT-Gen 从豆瓣生成),详情页同时给了豆瓣 ID 时写入;
|
||||
// - web_snippet:元数据补全(META-2)网页搜索结果里 douban.com/subject/<id> 的搜索标题与摘要。
|
||||
|
||||
export const DOUBAN_SUBJECT_REGION = "douban.subject";
|
||||
const TTL_SEC = 30 * 86_400;
|
||||
const MAX_ENTRIES = 20_000;
|
||||
|
||||
export interface DoubanSubject {
|
||||
id: string;
|
||||
/** 豆瓣显示名(通常是中文名) */
|
||||
title: string;
|
||||
/** 译名 / 又名 / 原名 */
|
||||
aliases: string[];
|
||||
year: number | null;
|
||||
source: "site_detail" | "web_snippet";
|
||||
/** 出处:站点资源 id 或搜索结果地址 */
|
||||
ref: string;
|
||||
at: string;
|
||||
}
|
||||
|
||||
const okYear = (y: number) => y >= 1900 && y <= 2100;
|
||||
|
||||
function splitNames(v: string | undefined): string[] {
|
||||
return (v ?? "")
|
||||
.split(/\s*\/\s*/)
|
||||
.map((x) => x.trim())
|
||||
.filter((x) => x.length > 0 && x.length <= 80);
|
||||
}
|
||||
|
||||
/** 站点详情页信息块 → 豆瓣条目(需要详情页同时给出豆瓣 ID 与「片名」) */
|
||||
export function doubanSubjectFromDetail(
|
||||
page: { doubanId?: string | null; descriptionText?: string | null },
|
||||
ref: string,
|
||||
now: Date,
|
||||
): DoubanSubject | null {
|
||||
if (!page.doubanId || !page.descriptionText) return null;
|
||||
const { infoBlock } = formatDetailPage({ descriptionText: page.descriptionText, mediaInfo: null, files: [] });
|
||||
const get = (k: string) => infoBlock.find((x) => x.key === k)?.value?.trim();
|
||||
const names = splitNames(get("片名"));
|
||||
const title = names[0];
|
||||
if (!title) return null;
|
||||
const y = Number(/(?:19|20)\d{2}/.exec(get("年代") ?? get("年份") ?? get("上映日期") ?? "")?.[0]);
|
||||
const aliases = [...new Set([...names.slice(1), ...splitNames(get("译名")), ...splitNames(get("又名"))])].filter(
|
||||
(a) => a !== title,
|
||||
);
|
||||
return {
|
||||
id: page.doubanId,
|
||||
title,
|
||||
aliases: aliases.slice(0, 10),
|
||||
year: okYear(y) ? y : null,
|
||||
source: "site_detail",
|
||||
ref,
|
||||
at: now.toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
/** 网页搜索结果(douban.com/subject/<id>)→ 豆瓣条目:标题「X (豆瓣)」,年份取标题里的 (2025) 或摘要「年份: 2025」 */
|
||||
export function doubanSubjectFromSnippet(
|
||||
hit: { url: string; title?: string | null; snippet?: string | null },
|
||||
now: Date,
|
||||
): DoubanSubject | null {
|
||||
const id = /douban\.com\/(?:movie\/)?subject\/(\d{4,10})/.exec(hit.url)?.[1];
|
||||
if (!id) return null;
|
||||
let t = (hit.title ?? "")
|
||||
.replace(/\s*[(（]豆瓣[)）]\s*$/, "")
|
||||
.replace(/\s*[-_|]\s*豆瓣.*$/, "")
|
||||
.trim();
|
||||
const ty = /[(（]((?:19|20)\d{2})[)）]\s*$/.exec(t);
|
||||
if (ty) t = t.slice(0, ty.index).trim();
|
||||
if (!t || t === "豆瓣" || /^豆瓣/.test(t)) return null;
|
||||
const s = hit.snippet ?? "";
|
||||
const sy = /(?:年份|首播|上映日期)[:：]\s*((?:19|20)\d{2})/.exec(s)?.[1];
|
||||
const y = Number(ty?.[1] ?? sy);
|
||||
const alias = /又名[:：]\s*([^\n·|]{1,120})/.exec(s)?.[1];
|
||||
return {
|
||||
id,
|
||||
title: t,
|
||||
aliases: splitNames(alias).slice(0, 10),
|
||||
year: okYear(y) ? y : null,
|
||||
source: "web_snippet",
|
||||
ref: hit.url,
|
||||
at: now.toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
type Db = Pick<ServerContext, "db">;
|
||||
|
||||
/** 写入缓存;站点详情来源优先(已有 site_detail 条目时不用搜索摘要覆盖) */
|
||||
export async function saveDoubanSubject(ctx: Db, s: DoubanSubject): Promise<void> {
|
||||
const cache = pgCacheFor(ctx.db);
|
||||
if (s.source === "web_snippet") {
|
||||
const prev = await cache.get<DoubanSubject>(DOUBAN_SUBJECT_REGION, s.id);
|
||||
if (prev?.source === "site_detail") return;
|
||||
}
|
||||
await cache.set(DOUBAN_SUBJECT_REGION, s.id, s, { ttl: TTL_SEC, maxsize: MAX_ENTRIES });
|
||||
}
|
||||
|
||||
export async function loadDoubanSubject(ctx: Db, id: string): Promise<DoubanSubject | null> {
|
||||
try {
|
||||
const v = await pgCacheFor(ctx.db).get<DoubanSubject>(DOUBAN_SUBJECT_REGION, id);
|
||||
return v && typeof v.title === "string" && v.title ? v : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -22,6 +22,7 @@ import type { ServerContext } from "../../types.ts";
|
||||
import { jsonb } from "../rows.ts";
|
||||
import type { SourcesRuntime } from "../runtime.ts";
|
||||
import { type Actor, audit, type Exec } from "./common.ts";
|
||||
import { type DoubanSubject, doubanSubjectFromSnippet, saveDoubanSubject } from "./douban-subject.ts";
|
||||
import {
|
||||
baseConfidence,
|
||||
type Candidate,
|
||||
@@ -490,6 +491,7 @@ export async function suggestEnrichment(
|
||||
.returningAll()
|
||||
.executeTakeFirstOrThrow();
|
||||
|
||||
const doubanSubjects: DoubanSubject[] = [];
|
||||
try {
|
||||
const r = await runEnrichPipeline(
|
||||
deps,
|
||||
@@ -499,10 +501,15 @@ export async function suggestEnrichment(
|
||||
sites: await siteHosts(ctx),
|
||||
id: row.id,
|
||||
logger: ctx.logger,
|
||||
doubanSubjects,
|
||||
...(opts.signal ? { signal: opts.signal } : {}),
|
||||
},
|
||||
usage,
|
||||
);
|
||||
for (const ds of doubanSubjects)
|
||||
await saveDoubanSubject(ctx, ds).catch((e) =>
|
||||
ctx.logger.warn({ err: { message: (e as Error).message } }, "豆瓣条目缓存写入失败"),
|
||||
);
|
||||
if (!r.found) {
|
||||
const failed = await finish({ status: "failed", suggestion: r.suggestion, error: r.error });
|
||||
return toOutput(failed, false, await budgetOf(), await sameTitleEntities(ctx.db, [q.title]));
|
||||
@@ -540,6 +547,8 @@ export interface PipelineInput {
|
||||
id: string;
|
||||
logger: Pick<ServerContext["logger"], "warn">;
|
||||
signal?: AbortSignal;
|
||||
/** ID-3:搜索结果里的豆瓣条目(douban.com/subject/<id> 的标题 / 年份)收集到这里,由调用方写豆瓣条目缓存 */
|
||||
doubanSubjects?: DoubanSubject[];
|
||||
}
|
||||
|
||||
export interface PipelineResult {
|
||||
@@ -598,6 +607,10 @@ export async function runEnrichPipeline(
|
||||
// 白名单外的网站一律按「其它网站的搜索摘要」处理(不读网页)
|
||||
const kind: SourceKind = listed ? sourceKindOf(host, settings.domains) : "snippet";
|
||||
kindOf.set(h.url, kind);
|
||||
if (input.doubanSubjects && /(^|\.)douban\.com$/.test(host)) {
|
||||
const ds = doubanSubjectFromSnippet(h, new Date());
|
||||
if (ds && !input.doubanSubjects.some((x) => x.id === ds.id)) input.doubanSubjects.push(ds);
|
||||
}
|
||||
const snippetOnly = kind === "douban";
|
||||
const allowed = listed && !snippetOnly;
|
||||
const src: EnrichSource = {
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
import { type MetadataCandidate, recallTmdbCandidates } from "@umt/adapters";
|
||||
import { type SourceReleaseRow, sql } from "@umt/db";
|
||||
import { type ParsedRelease, parseRelease } from "@umt/domain";
|
||||
import type { SourcesRuntime } from "../runtime.ts";
|
||||
import type { siteExplicitIds } from "./detail-page.ts";
|
||||
import { type DoubanSubject, loadDoubanSubject } from "./douban-subject.ts";
|
||||
import { refreshReleaseDetail } from "./snapshot.ts";
|
||||
import { tmdbRecallInput } from "./tmdb-candidate.ts";
|
||||
|
||||
// ID-3 识别升级链:标题识别结果为 ambiguous / no_match 时,站点资源有详情页且还没抓过 → 抓详情(走站点详情缓存与共享限流器,
|
||||
// 失败不阻塞)→ 取 IMDb / TMDB / 豆瓣 ID → TMDB 详情 / find(豆瓣 ID 经已缓存的豆瓣条目按标题 + 年份匹配)→ 重新打分。
|
||||
// 触发范围克制:搜索结果、订阅相关的候选(命中订阅 / 本地作品别名)、池条目补全、手动重新识别;RSS 批量入库不抓,
|
||||
// 留给上述时机。每站每小时上限 identity.detailIdsPerSiteHour(缺省 10,0 = 不抓),计数在 kv_cache region identity.detail_ids。
|
||||
|
||||
export const ESCALATION_COUNTER_REGION = "identity.detail_ids";
|
||||
export const DEFAULT_DETAIL_IDS_PER_SITE_HOUR = 10;
|
||||
/** 抓详情失败后多久内不再为识别重抓(站点错误 / 限流) */
|
||||
const RETRY_AFTER_FAILURE_MS = 6 * 3600_000;
|
||||
|
||||
export type SiteIds = ReturnType<typeof siteExplicitIds>;
|
||||
|
||||
/** 记在 source_release.raw._escalation:最近一次为识别抓详情的结果 */
|
||||
interface EscalationMark {
|
||||
at: string;
|
||||
status: "fetched" | "failed";
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export type DetailFetchResult =
|
||||
| { status: "fetched"; release: SourceReleaseRow }
|
||||
| {
|
||||
status: "skipped";
|
||||
reason: "disabled" | "already_fetched" | "no_detail" | "site_unavailable" | "recent_failure";
|
||||
}
|
||||
| { status: "deferred"; reason: "cap_reached" }
|
||||
/** 抓了但失败:不阻塞识别,照常往下走(6 小时内不再为识别重抓) */
|
||||
| { status: "failed"; reason: "fetch_failed"; error: string };
|
||||
|
||||
export async function detailIdsPerSiteHour(rt: SourcesRuntime): Promise<number> {
|
||||
const s = await rt.ctx.settings.get("identity").catch(() => null);
|
||||
return s?.detailIdsPerSiteHour ?? DEFAULT_DETAIL_IDS_PER_SITE_HOUR;
|
||||
}
|
||||
|
||||
/**
|
||||
* 每站每小时计数:原子地占一个名额;已满返回 false。kv_cache 是 UNLOGGED 表,PG 崩溃丢计数只会让这一小时多抓几个,可接受。
|
||||
*/
|
||||
export async function takeEscalationSlot(rt: SourcesRuntime, siteId: string, cap: number): Promise<boolean> {
|
||||
if (cap <= 0) return false;
|
||||
const hour = Math.floor(rt.now().getTime() / 3_600_000);
|
||||
const r = await sql<{ n: number }>`
|
||||
insert into kv_cache (region, key, value, expires_at, updated_at, accessed_at)
|
||||
values (${ESCALATION_COUNTER_REGION}, ${`${siteId}:${hour}`}, '1'::jsonb, now() + interval '2 hours', now(), now())
|
||||
on conflict (region, key) do update set value = to_jsonb((kv_cache.value #>> '{}')::int + 1), updated_at = now()
|
||||
where (kv_cache.value #>> '{}')::int < ${cap}
|
||||
returning (value #>> '{}')::int as n`.execute(rt.ctx.db);
|
||||
return r.rows.length > 0;
|
||||
}
|
||||
|
||||
/** 这一小时该站点已为识别抓了多少个详情页 */
|
||||
export async function escalationCount(rt: SourcesRuntime, siteId: string): Promise<number> {
|
||||
const hour = Math.floor(rt.now().getTime() / 3_600_000);
|
||||
const r = await sql<{ n: number }>`select (value #>> '{}')::int as n from kv_cache
|
||||
where region = ${ESCALATION_COUNTER_REGION} and key = ${`${siteId}:${hour}`} and expires_at > now()`.execute(
|
||||
rt.ctx.db,
|
||||
);
|
||||
return Number(r.rows[0]?.n ?? 0);
|
||||
}
|
||||
|
||||
function markOf(raw: unknown): EscalationMark | null {
|
||||
const m = ((raw ?? {}) as { _escalation?: EscalationMark })._escalation;
|
||||
return m && typeof m.at === "string" ? m : null;
|
||||
}
|
||||
|
||||
async function mark(rt: SourcesRuntime, srId: string, m: EscalationMark): Promise<void> {
|
||||
await sql`update source_release set raw = raw || jsonb_build_object('_escalation', ${JSON.stringify(m)}::jsonb)
|
||||
where id = ${srId}`.execute(rt.ctx.db);
|
||||
}
|
||||
|
||||
/**
|
||||
* 为识别抓一次详情页(只在还没抓过时)。走 refreshReleaseDetail:站点详情缓存、共享限流器、请求计数、凭据失效处理都照旧;
|
||||
* 失败只记在 raw._escalation,6 小时内不再为识别重抓,不抛错。
|
||||
*/
|
||||
export async function fetchDetailForIds(rt: SourcesRuntime, release: SourceReleaseRow): Promise<DetailFetchResult> {
|
||||
if (release.detail_fetched_at) return { status: "skipped", reason: "already_fetched" };
|
||||
const prev = markOf(release.raw);
|
||||
if (prev?.status === "failed" && rt.now().getTime() - Date.parse(prev.at) < RETRY_AFTER_FAILURE_MS)
|
||||
return { status: "skipped", reason: "recent_failure" };
|
||||
const cap = await detailIdsPerSiteHour(rt);
|
||||
if (cap <= 0) return { status: "skipped", reason: "disabled" };
|
||||
const site = await rt.ctx.db.selectFrom("site").selectAll().where("id", "=", release.site_id).executeTakeFirst();
|
||||
if (!site?.enabled || site.health_status === "failing") return { status: "skipped", reason: "site_unavailable" };
|
||||
try {
|
||||
if (!rt.siteAdapter(site).capabilities.detail) return { status: "skipped", reason: "no_detail" };
|
||||
} catch {
|
||||
return { status: "skipped", reason: "no_detail" };
|
||||
}
|
||||
if (!(await takeEscalationSlot(rt, site.id, cap))) return { status: "deferred", reason: "cap_reached" };
|
||||
const err = await refreshReleaseDetail(rt, release.id).catch((e) => (e as Error).message);
|
||||
const at = rt.now().toISOString();
|
||||
if (err) {
|
||||
await mark(rt, release.id, { at, status: "failed", error: err.slice(0, 300) });
|
||||
rt.ctx.logger.info({ sourceReleaseId: release.id, err }, "识别升级时抓详情页失败");
|
||||
return { status: "failed", reason: "fetch_failed", error: err };
|
||||
}
|
||||
await mark(rt, release.id, { at, status: "fetched" });
|
||||
const fresh = await rt.ctx.db
|
||||
.selectFrom("source_release")
|
||||
.selectAll()
|
||||
.where("id", "=", release.id)
|
||||
.executeTakeFirst();
|
||||
return fresh ? { status: "fetched", release: fresh } : { status: "skipped", reason: "site_unavailable" };
|
||||
}
|
||||
|
||||
/** 新拿到的 ID(不在旧集合里的) */
|
||||
export function newIds(before: SiteIds, after: SiteIds): SiteIds {
|
||||
return after.filter((a) => !before.some((b) => b.provider === a.provider && b.id === a.id));
|
||||
}
|
||||
|
||||
/**
|
||||
* 豆瓣 ID → 缓存的豆瓣条目 → 按条目标题 / 别名 + 年份召回 TMDB 候选(对照 MP 豆瓣 → TMDB 投影:原名 / 中文名 / 英文名 + 年份)。
|
||||
* 打分用的解析 = 豆瓣条目的名称与年份,季集与类型线索沿用资源标题的解析。缓存里没有这个条目 → null(不请求豆瓣)。
|
||||
*/
|
||||
export async function doubanRecall(
|
||||
rt: SourcesRuntime,
|
||||
parsed: ParsedRelease,
|
||||
doubanId: string,
|
||||
): Promise<{ subject: DoubanSubject; parsed: ParsedRelease; remote: MetadataCandidate[] } | null> {
|
||||
const subject = await loadDoubanSubject(rt.ctx, doubanId);
|
||||
if (!subject) return null;
|
||||
const tmdb = await rt.tmdb();
|
||||
if (!tmdb) return null;
|
||||
const p = doubanParsed(parsed, subject);
|
||||
const remote = await recallTmdbCandidates(tmdb, tmdbRecallInput(p));
|
||||
return { subject, parsed: p, remote };
|
||||
}
|
||||
|
||||
const CJK = /[㐀-鿿]/;
|
||||
|
||||
/** 豆瓣条目 + 资源解析 → 打分用的解析(名称 / 年份取豆瓣,季集 / 类型取资源) */
|
||||
export function doubanParsed(parsed: ParsedRelease, s: DoubanSubject): ParsedRelease {
|
||||
const names = [s.title, ...s.aliases];
|
||||
const cn = names.find((n) => CJK.test(n));
|
||||
const en = names.find((n) => !CJK.test(n) && /[A-Za-z]/.test(n));
|
||||
const base = parseRelease(s.title);
|
||||
return {
|
||||
...parsed,
|
||||
title: cn ?? en ?? base.title ?? s.title,
|
||||
...(cn ? { titleCn: cn } : {}),
|
||||
...(en ? { titleEn: en } : {}),
|
||||
altTitles: names.filter((n) => n !== cn && n !== en).slice(0, 8),
|
||||
...(s.year !== null ? { year: s.year } : {}),
|
||||
explicitIds: [...parsed.explicitIds],
|
||||
evidence: [...parsed.evidence, "douban_subject"],
|
||||
};
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
queryTitles,
|
||||
type RankedCandidate,
|
||||
resolveIdentity,
|
||||
scoreCandidate,
|
||||
} from "@umt/domain";
|
||||
import {
|
||||
localIdentityCandidate,
|
||||
@@ -34,6 +35,7 @@ import {
|
||||
looksLikeVariety,
|
||||
resolveEntityId,
|
||||
} from "./entities.ts";
|
||||
import { doubanRecall, fetchDetailForIds, newIds } from "./id-escalation.ts";
|
||||
import { detachFromIdentityDecisions, openGroupDecision, refreshGroupDecision } from "./identity-decisions.ts";
|
||||
import { tmdbRecallInput, tmdbToCandidate } from "./tmdb-candidate.ts";
|
||||
import { loadRecognizeCache, type RecognizeMeta, recognizeMetaOf, saveRecognizeCache } from "./tmdb-recognize-cache.ts";
|
||||
@@ -63,6 +65,11 @@ export interface ResolveInput {
|
||||
* manual / search 总是查 TMDB;observed / review 只在命中本地别名、新作品收录规则或设置打开时查。
|
||||
*/
|
||||
trigger?: ResolveTrigger | undefined;
|
||||
/**
|
||||
* ID-3:标题识别不确定时允许抓站点详情页取 ID(识别升级链)。缺省按 trigger 判断(manual / search / 订阅相关的复查);
|
||||
* 池条目补全前由调用方显式打开。
|
||||
*/
|
||||
escalate?: boolean | undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -217,6 +224,108 @@ async function tmdbByExplicitIds(
|
||||
return found;
|
||||
}
|
||||
|
||||
/** 站点给出的 IMDb / TMDB ID 作为强证据(ID-3,domain MatchOptions.trustedIds);豆瓣 ID 不直接对应 TMDB 候选 */
|
||||
function trustedOf(ids: ReturnType<typeof siteExplicitIds>) {
|
||||
return { trustedIds: ids.filter((x) => x.provider !== "douban").map((x) => ({ provider: x.provider, id: x.id })) };
|
||||
}
|
||||
|
||||
/** 识别升级链的证据(记在 resolution_evidence.escalation 与池条目 query.escalation) */
|
||||
export interface EscalationEvidence {
|
||||
/** 详情页:fetched 本次抓到;skipped:<原因>;deferred:cap_reached(稍后复查时再抓);failed:fetch_failed;not_now:<原因>(本次不抓) */
|
||||
detail: string;
|
||||
/** 最后用上的路径:site_id(详情 / 列表的 IMDb、TMDB)/ douban(豆瓣条目 → 标题 + 年份) */
|
||||
via: "site_id" | "douban" | null;
|
||||
/** 抓详情后资源上的外部 ID */
|
||||
ids: string[];
|
||||
/** 本次没抓、之后还会抓(每站每小时限额已满 / RSS 入库时不抓):不开待处理,尽快复查。抓了失败的不算(不阻塞) */
|
||||
pending: boolean;
|
||||
/** 豆瓣条目(豆瓣路径) */
|
||||
douban?: { id: string; title: string; year: number | null; source: string };
|
||||
}
|
||||
|
||||
/**
|
||||
* 什么时候允许为识别抓详情页:搜索结果、手动重新识别、调用方显式要求(池条目补全)、订阅相关的候选(命中订阅 / 本地作品别名)
|
||||
* 的复查、上次因限额 / 入库时机而推迟的复查。RSS / 列表批量入库(observed)不抓,推迟到复查。
|
||||
*/
|
||||
function escalationFetchPolicy(
|
||||
trigger: ResolveTrigger,
|
||||
lookup: LookupReason,
|
||||
input: ResolveInput,
|
||||
prevQuery: unknown,
|
||||
): "fetch" | "defer" | "no" {
|
||||
if (input.escalate === false) return "no";
|
||||
if (input.escalate || trigger === "manual" || trigger === "search") return "fetch";
|
||||
const prev = ((prevQuery ?? {}) as { escalation?: { pending?: boolean } }).escalation;
|
||||
if (trigger === "review" && (lookup === "tracked_alias" || prev?.pending)) return "fetch";
|
||||
if (trigger === "observed" && lookup === "tracked_alias") return "defer";
|
||||
return "no";
|
||||
}
|
||||
|
||||
interface EscalationOutcome {
|
||||
evidence: EscalationEvidence;
|
||||
resolution?: IdentityResolution;
|
||||
remote?: MetadataCandidate[];
|
||||
parsed?: ParsedRelease;
|
||||
}
|
||||
|
||||
/**
|
||||
* ID-3 识别升级链:抓详情(按策略)→ 新拿到的 IMDb / TMDB → TMDB 详情 / find → 打分;还不能确认且有豆瓣 ID →
|
||||
* 缓存的豆瓣条目按标题 + 年份召回 TMDB → 打分。返回更好的一次结果(auto 优先),都没有就只带证据。
|
||||
*/
|
||||
async function escalateIdentity(
|
||||
rt: SourcesRuntime,
|
||||
release: SourceReleaseRow,
|
||||
parsed: ParsedRelease,
|
||||
local: DomainCandidate[],
|
||||
siteIds: ReturnType<typeof siteExplicitIds>,
|
||||
policy: "fetch" | "defer",
|
||||
): Promise<EscalationOutcome> {
|
||||
const ev: EscalationEvidence = { detail: "not_now", via: null, ids: [], pending: false };
|
||||
let raw = release.raw;
|
||||
if (policy === "fetch") {
|
||||
const f = await fetchDetailForIds(rt, release);
|
||||
ev.detail = f.status === "fetched" ? "fetched" : `${f.status}:${f.reason}`;
|
||||
if (f.status === "fetched") raw = f.release.raw;
|
||||
if (f.status === "deferred") ev.pending = true;
|
||||
} else if (!release.detail_fetched_at) {
|
||||
ev.detail = "not_now:batch_ingest";
|
||||
ev.pending = true;
|
||||
} else ev.detail = "skipped:already_fetched";
|
||||
const after = siteExplicitIds(raw);
|
||||
ev.ids = after.map((x) => `${x.provider}:${x.id}`);
|
||||
const tmdb = await rt.tmdb();
|
||||
if (!tmdb) return { evidence: ev };
|
||||
let best: EscalationOutcome | null = null;
|
||||
// 1) 新拿到的 IMDb / TMDB(列表上已有的在第一轮已用过)
|
||||
const added = newIds(siteIds, after).filter((x) => x.provider !== "douban");
|
||||
if (added.length) {
|
||||
const p: ParsedRelease = { ...parsed, explicitIds: [...parsed.explicitIds] };
|
||||
for (const x of after)
|
||||
if (!p.explicitIds.some((e) => e.provider === x.provider)) p.explicitIds.push({ provider: x.provider, id: x.id });
|
||||
const remote = await tmdbByExplicitIds(tmdb, after);
|
||||
if (remote.length) {
|
||||
const res = resolveIdentity(p, [...local, ...remote.map(tmdbToCandidate)], trustedOf(after));
|
||||
best = { evidence: ev, resolution: res, remote, parsed: p };
|
||||
ev.via = "site_id";
|
||||
if (res.status === "auto") return best;
|
||||
}
|
||||
}
|
||||
// 2) 豆瓣 ID → 缓存的豆瓣条目(标题 + 年份)→ TMDB
|
||||
const douban = after.find((x) => x.provider === "douban");
|
||||
if (douban) {
|
||||
const d = await doubanRecall(rt, parsed, douban.id);
|
||||
if (d) {
|
||||
ev.douban = { id: d.subject.id, title: d.subject.title, year: d.subject.year, source: d.subject.source };
|
||||
const res = resolveIdentity(d.parsed, [...local, ...d.remote.map(tmdbToCandidate)]);
|
||||
if (res.status === "auto" || !best || (best.resolution?.status === "no_match" && res.status !== "no_match")) {
|
||||
ev.via = "douban";
|
||||
best = { evidence: ev, resolution: res, remote: d.remote, parsed: d.parsed };
|
||||
}
|
||||
}
|
||||
}
|
||||
return best ?? { evidence: ev };
|
||||
}
|
||||
|
||||
async function loadRelease(rt: SourcesRuntime, input: ResolveInput) {
|
||||
const { db } = rt.ctx;
|
||||
const observationId = input.observationId ?? null;
|
||||
@@ -291,13 +400,14 @@ export async function resolveRelease(rt: SourcesRuntime, input: ResolveInput): P
|
||||
return toPool(rt, release, observationId, parsed, null, [], "unmatched", "unparseable_title", null, {
|
||||
tracked: false,
|
||||
lookup: null,
|
||||
escalation: null,
|
||||
});
|
||||
}
|
||||
|
||||
// 召回:本地优先;TMDB 只在有理由时查(惰性识别),查询失败不等于无结果
|
||||
const local = await localCandidates(ctx.db, parsed, release);
|
||||
const lookup = await lookupReason(rt, parsed, trigger, local.length > 0, pool?.reason ?? null);
|
||||
const scope = { tracked: local.length > 0, lookup };
|
||||
const scope: ResolveScope = { tracked: local.length > 0, lookup, escalation: null };
|
||||
if (!lookup) {
|
||||
// 不相关的资源:只进池(可检索、可手动指定),不查 TMDB、不开待处理
|
||||
return toPool(rt, release, observationId, parsed, null, [], "unmatched", "no_local_match", null, scope);
|
||||
@@ -314,7 +424,7 @@ export async function resolveRelease(rt: SourcesRuntime, input: ResolveInput): P
|
||||
lookupError = `${e.kind}: ${e.message}`;
|
||||
}
|
||||
const candidates = [...local, ...remote.map(tmdbToCandidate)];
|
||||
const resolution = resolveIdentity(parsed, candidates);
|
||||
let resolution = resolveIdentity(parsed, candidates, trustedOf(siteIds));
|
||||
// 识别缓存回写(MP TmdbCache.update):识别到 TMDB 条目写正常条目,确认无匹配写负缓存;TMDB 出错 / 有歧义不写
|
||||
if (recognize.meta && !recognize.cacheHit && !lookupError) {
|
||||
const top = resolution.status === "auto" ? resolution.best?.candidate : undefined;
|
||||
@@ -325,7 +435,29 @@ export async function resolveRelease(rt: SourcesRuntime, input: ResolveInput): P
|
||||
const info = hit ?? (resolution.status === "no_match" ? null : undefined);
|
||||
await saveRecognizeCache(rt, recognize.meta, info);
|
||||
}
|
||||
const version = evidenceVersion(parsed, resolution.ranked);
|
||||
|
||||
// ID-3 识别升级链:标题认不准时用站点详情页的 IMDb / TMDB / 豆瓣 ID 再认一次
|
||||
let escalation: EscalationEvidence | null = null;
|
||||
let scored = parsed;
|
||||
if (resolution.status !== "auto" && !lookupError) {
|
||||
const policy = escalationFetchPolicy(trigger, lookup, input, pool?.query);
|
||||
if (policy !== "no") {
|
||||
try {
|
||||
const up = await escalateIdentity(rt, release, parsed, local, siteIds, policy);
|
||||
escalation = up.evidence;
|
||||
if (up.resolution && up.remote && up.parsed) {
|
||||
resolution = up.resolution;
|
||||
remote = up.remote;
|
||||
scored = up.parsed;
|
||||
}
|
||||
} catch (e) {
|
||||
if (!(e instanceof TmdbError)) throw e;
|
||||
lookupError = `${e.kind}: ${e.message}`;
|
||||
}
|
||||
}
|
||||
}
|
||||
const version = evidenceVersion(scored, resolution.ranked);
|
||||
scope.escalation = escalation;
|
||||
|
||||
// 已识别且证据未变:幂等短路(强制复查时)
|
||||
if (
|
||||
@@ -366,7 +498,18 @@ export async function resolveRelease(rt: SourcesRuntime, input: ResolveInput): P
|
||||
}
|
||||
}
|
||||
}
|
||||
return bindAuto(rt, release, observationId, parsed, cols, resolution, version, detailed ?? null, lookup);
|
||||
return bindAuto(
|
||||
rt,
|
||||
release,
|
||||
observationId,
|
||||
parsed,
|
||||
cols,
|
||||
resolution,
|
||||
version,
|
||||
detailed ?? null,
|
||||
lookup,
|
||||
escalation,
|
||||
);
|
||||
}
|
||||
|
||||
if (resolution.status === "auto" && !autoAccept) {
|
||||
@@ -514,9 +657,12 @@ async function bindAuto(
|
||||
version: string,
|
||||
tmdbDetail: MetadataCandidate | null,
|
||||
lookup: LookupReason,
|
||||
escalation: EscalationEvidence | null = null,
|
||||
): Promise<ResolveResult> {
|
||||
const { ctx } = rt;
|
||||
const best = resolution.best!;
|
||||
// 别名登记按资源自己的标题判断(升级链靠 ID / 豆瓣名确认的,站点标题未必对得上;ID-2 / FIX-5 守卫照旧)
|
||||
const aliasBest = escalation?.via ? scoreCandidate(parsed, best.candidate) : best;
|
||||
const level: EvidenceLevel = toContractEvidenceLevel(best.level) ?? "needs_review";
|
||||
const site = ((release.raw as Record<string, unknown>)?._site ?? {}) as { categoryName?: string };
|
||||
const variety = looksLikeVariety(parsed, site.categoryName ?? release.category_raw);
|
||||
@@ -558,8 +704,8 @@ async function bindAuto(
|
||||
// 站点标题作为别名:下次同名资源可直接本地召回。只在名称本身命中过(不是仅靠别名单线命中)时登记,
|
||||
// 且通用词(分类词 / 单字 / 停用词)不登记,避免错误别名链式扩散(ID-2)。
|
||||
// FIX-5:证据等级非最高、年份不相容、与已确认外部标题冲突、已属于别的作品的名称也不登记(alias-guard.ts)
|
||||
const aliasOutcome: SiteAliasOutcome = siteAliasAllowed(best)
|
||||
? await addSiteAliases(trx, entityId, best, parsed)
|
||||
const aliasOutcome: SiteAliasOutcome = siteAliasAllowed(aliasBest)
|
||||
? await addSiteAliases(trx, entityId, aliasBest, parsed)
|
||||
: { added: [], skipped: [] };
|
||||
|
||||
const evidence = {
|
||||
@@ -569,6 +715,7 @@ async function bindAuto(
|
||||
best: rankedSummary(best),
|
||||
candidateCount: resolution.ranked.length,
|
||||
lookup,
|
||||
...(escalation ? { escalation } : {}),
|
||||
};
|
||||
await trx
|
||||
.updateTable("source_release")
|
||||
@@ -669,8 +816,13 @@ type PoolReason = "ambiguous" | "no_match" | "proposed" | "lookup_failed" | "unm
|
||||
interface ResolveScope {
|
||||
tracked: boolean;
|
||||
lookup: LookupReason | null;
|
||||
/** ID-3 识别升级链的证据(没走升级链为 null) */
|
||||
escalation: EscalationEvidence | null;
|
||||
}
|
||||
|
||||
/** 推迟的升级(限额已满 / 入库时不抓)多久后复查 */
|
||||
const ESCALATION_RETRY_MS = 15 * 60_000;
|
||||
|
||||
async function toPool(
|
||||
rt: SourcesRuntime,
|
||||
release: SourceReleaseRow,
|
||||
@@ -701,6 +853,7 @@ async function toPool(
|
||||
season: parsed.season ?? null,
|
||||
episodes: parsed.episodes ?? null,
|
||||
lookup: scope.lookup,
|
||||
...(scope.escalation ? { escalation: scope.escalation } : {}),
|
||||
};
|
||||
const contractStatus =
|
||||
reason === "proposed"
|
||||
@@ -717,7 +870,10 @@ async function toPool(
|
||||
.executeTakeFirst();
|
||||
// 证据版本变化 = 新证据:从头计数;同一证据反复复查按退避递增,达上限停止自动复查(仍可检索)
|
||||
const attempts = prev && prev.evidence_version === version ? prev.attempts + 1 : 1;
|
||||
const delay = reviewDelayMs(reason, attempts);
|
||||
const baseDelay = reviewDelayMs(reason, attempts);
|
||||
// 升级链推迟了(还能拿到 ID):尽快复查,不受退避上限影响
|
||||
const pendingEscalation = scope.escalation?.pending === true;
|
||||
const delay = pendingEscalation ? Math.min(baseDelay ?? ESCALATION_RETRY_MS, ESCALATION_RETRY_MS) : baseDelay;
|
||||
const nextReviewAt = delay === null ? null : new Date(now.getTime() + delay);
|
||||
|
||||
const evidence = {
|
||||
@@ -729,10 +885,14 @@ async function toPool(
|
||||
query,
|
||||
};
|
||||
// 只收相关的:查过 TMDB 仍有歧义(有候选可选),或与本地作品相关却无结果 / 查询失败且复查已停
|
||||
const wantsDecision =
|
||||
reason === "ambiguous" || reason === "proposed"
|
||||
// ID-3:能从站点拿到 ID 的资源不开"没识别出作品":升级链还在推迟中(之后会抓详情)不开任何事项;
|
||||
// 资源上已有 IMDb / TMDB / 豆瓣 ID 却仍无结果的,不开 identity_unresolved(只留在池里,可手动指定)
|
||||
const hasSiteIds = siteExplicitIds(release.raw).length > 0 || (scope.escalation?.ids.length ?? 0) > 0;
|
||||
const wantsDecision = pendingEscalation
|
||||
? false
|
||||
: reason === "ambiguous" || reason === "proposed"
|
||||
? candidates.length > 0
|
||||
: scope.tracked && (reason === "no_match" || (reason === "lookup_failed" && delay === null));
|
||||
: scope.tracked && !hasSiteIds && (reason === "no_match" || (reason === "lookup_failed" && delay === null));
|
||||
let decisionId: string | null = null;
|
||||
if (wantsDecision) {
|
||||
const ambiguous = (reason === "ambiguous" || reason === "proposed") && candidates.length > 0;
|
||||
@@ -791,6 +951,7 @@ async function toPool(
|
||||
evidenceVersion: version,
|
||||
candidateCount: candidates.length,
|
||||
lookup: scope.lookup,
|
||||
...(scope.escalation ? { escalation: scope.escalation } : {}),
|
||||
}),
|
||||
})
|
||||
.where("id", "=", release.id)
|
||||
|
||||
@@ -0,0 +1,356 @@
|
||||
/**
|
||||
* ID-3 识别升级链(样本驱动,零真实请求):标题识别不确定 → 抓站点详情页(观众 DET-2 脱敏样本 / 憨憨 HH-2 样本,经注入的 fetch)
|
||||
* → IMDb / TMDB / 豆瓣 ID → TMDB(mock)→ 确定;每站每小时上限;RSS 入库不抓;事项不开 / 自动关闭;豆瓣条目缓存路径。
|
||||
*/
|
||||
import { readFileSync } from "node:fs";
|
||||
import { sql } from "@umt/db";
|
||||
import { normalizeTitle } from "@umt/domain";
|
||||
import {
|
||||
createMockTmdb,
|
||||
HHANCLUB_DETAIL_PAGE_2026,
|
||||
HHANCLUB_FILELIST_FRAGMENT,
|
||||
type MockTmdb,
|
||||
type TmdbFixture,
|
||||
} from "@umt/testkit";
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
DOUBAN_SUBJECT_REGION,
|
||||
doubanSubjectFromSnippet,
|
||||
loadDoubanSubject,
|
||||
saveDoubanSubject,
|
||||
} from "../../src/modules/sources-identity/service/douban-subject.ts";
|
||||
import {
|
||||
ESCALATION_COUNTER_REGION,
|
||||
escalationCount,
|
||||
} from "../../src/modules/sources-identity/service/id-escalation.ts";
|
||||
import { resolveRelease } from "../../src/modules/sources-identity/service/resolve.ts";
|
||||
import { sharedCache } from "../../src/shared-cache.ts";
|
||||
import { createHarness, type Harness, pgAvailable, seedSite } from "./harness.ts";
|
||||
|
||||
const pg = await pgAvailable();
|
||||
|
||||
const SAMPLES = new URL("../../../../packages/adapters/test/differential/inputs/detail/", import.meta.url);
|
||||
const audiencesDetail = (id: string) => readFileSync(new URL(`rl-audiences-detail-${id}.html.txt`, SAMPLES), "utf8");
|
||||
|
||||
const fx = (
|
||||
type: "movie" | "tv",
|
||||
id: number,
|
||||
en: string,
|
||||
zh: string,
|
||||
date: string,
|
||||
imdb: string | null = null,
|
||||
): TmdbFixture => ({
|
||||
type,
|
||||
id,
|
||||
original_title: en,
|
||||
original_language: "zh",
|
||||
origin_country: ["CN"],
|
||||
date,
|
||||
titles: { "zh-CN": zh, "en-US": en },
|
||||
popularity: 10,
|
||||
alternative_titles: [],
|
||||
external_ids: { imdb_id: imdb },
|
||||
...(type === "tv"
|
||||
? {
|
||||
seasons: [
|
||||
{ season_number: 1, air_date: date, names: { "zh-CN": "第 1 季", "en-US": "Season 1" }, episodes: [] },
|
||||
],
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
|
||||
describe.skipIf(!pg)("ID-3 识别升级链:详情页 ID", () => {
|
||||
let h: Harness;
|
||||
let tmdb: MockTmdb;
|
||||
let audiences: string;
|
||||
let hhan: string;
|
||||
const siteRequests: string[] = [];
|
||||
|
||||
beforeAll(async () => {
|
||||
tmdb = await createMockTmdb({
|
||||
fixtures: [
|
||||
// 观众 727077《我不是大师》:站点英文名 A Prophet 与 TMDB 英文名对不上
|
||||
fx("tv", 990002, "The Fortune Teller", "我不是大师", "2026-09-01", "tt36758489"),
|
||||
// 观众 725858《Rose》:同名同年两部 → 标题识别有歧义
|
||||
fx("movie", 990001, "Rose", "罗斯", "2026-03-01", "tt25399154"),
|
||||
fx("movie", 990011, "Rose", "玫瑰", "2026-05-01", "tt0000011"),
|
||||
// 憨憨详情样本:TMDB tv/290001(站点英文名 Example Show 搜不到)
|
||||
fx("tv", 290001, "Sample Series", "示例剧集", "2026-08-01", "tt45950001"),
|
||||
// 豆瓣路径:只凭豆瓣条目(标题 + 年份)匹配
|
||||
fx("tv", 990021, "Lanterns of Qing", "青灯记", "2025-04-01"),
|
||||
],
|
||||
});
|
||||
h = await createHarness("id3", {
|
||||
tmdbUrl: tmdb.url,
|
||||
// 站点请求全部走样本;没有样本的地址直接报错(零真实请求)
|
||||
fetch: async (input) => {
|
||||
const u = new URL(typeof input === "string" ? input : input instanceof URL ? input.href : input.url);
|
||||
siteRequests.push(`${u.host}${u.pathname}${u.search}`);
|
||||
const id = u.searchParams.get("id") ?? "";
|
||||
const html = (body: string) =>
|
||||
new Response(body, { status: 200, headers: { "content-type": "text/html; charset=utf-8" } });
|
||||
if (u.host === "audiences.me" && u.pathname === "/details.php") return html(audiencesDetail(id));
|
||||
if (u.host === "hhanclub.net" && u.pathname === "/details.php") return html(HHANCLUB_DETAIL_PAGE_2026);
|
||||
if (u.host === "hhanclub.net" && u.pathname === "/viewfilelist.php") return html(HHANCLUB_FILELIST_FRAGMENT);
|
||||
throw new Error(`测试不允许的站点请求 ${u.href}`);
|
||||
},
|
||||
});
|
||||
audiences = (await seedSite(h, { key: "audiences", baseUrl: "https://audiences.me/", cookie: "c_secure_uid=1" }))
|
||||
.site.id;
|
||||
hhan = (await seedSite(h, { key: "hhanclub", baseUrl: "https://hhanclub.net/", cookie: "c_secure_uid=2" })).site.id;
|
||||
});
|
||||
afterAll(async () => {
|
||||
await h?.close();
|
||||
await tmdb?.close();
|
||||
});
|
||||
beforeEach(async () => {
|
||||
await setCap(10);
|
||||
await sql`delete from kv_cache where region = ${ESCALATION_COUNTER_REGION}`.execute(h.db);
|
||||
});
|
||||
|
||||
const setCap = async (cap: number) => {
|
||||
await sql`insert into settings (key, value) values ('identity', ${JSON.stringify({ detailIdsPerSiteHour: cap })}::jsonb)
|
||||
on conflict (key) do update set value = excluded.value`.execute(h.db);
|
||||
};
|
||||
const release = (siteId: string, siteReleaseId: string, title: string, raw: unknown = {}) =>
|
||||
h.db
|
||||
.insertInto("source_release")
|
||||
.values({ site_id: siteId, site_release_id: siteReleaseId, title, raw: JSON.stringify(raw) } as never)
|
||||
.returningAll()
|
||||
.executeTakeFirstOrThrow();
|
||||
const row = (id: string) =>
|
||||
h.db.selectFrom("source_release").selectAll().where("id", "=", id).executeTakeFirstOrThrow();
|
||||
const detailRequests = (id: string) =>
|
||||
siteRequests.filter((r) => r.includes("/details.php") && r.includes(`id=${id}&`));
|
||||
const tmdbIdOf = async (entityId: string) =>
|
||||
(
|
||||
await h.db
|
||||
.selectFrom("external_identity")
|
||||
.select("external_id")
|
||||
.where("media_entity_id", "=", entityId)
|
||||
.where("provider", "=", "tmdb")
|
||||
.executeTakeFirstOrThrow()
|
||||
).external_id;
|
||||
const openDecisions = () =>
|
||||
h.db
|
||||
.selectFrom("decision")
|
||||
.select(["id", "kind", "status"])
|
||||
.where("kind", "in", ["identity_ambiguous", "identity_unresolved"])
|
||||
.where("status", "in", ["open", "investigating", "awaiting_user"])
|
||||
.execute();
|
||||
|
||||
it("观众 727077:列表只有豆瓣 ID,搜索结果标题认不出 → 抓详情取 IMDb → TMDB find → 自动确认;不登记对不上的站点名", async () => {
|
||||
// 列表页只给了豆瓣 ID(样本 rl-audiences-list 里这一行没有 IMDb 徽标)
|
||||
const r = await release(audiences, "727077", "A Prophet S01E16 2026 2160p iQIYI WEB-DL H265 HDR DDP5.1-ADWeb", {
|
||||
_site: { doubanId: "37297001" },
|
||||
});
|
||||
const find0 = tmdb.callsTo(/\/3\/find\/tt36758489/).length;
|
||||
const res = await resolveRelease(h.rt, { sourceReleaseId: r.id, trigger: "search" });
|
||||
expect(res.outcome).toBe("auto");
|
||||
expect(detailRequests("727077")).toHaveLength(1);
|
||||
expect(tmdb.callsTo(/\/3\/find\/tt36758489/).length).toBe(find0 + 1);
|
||||
expect(await tmdbIdOf(res.mediaEntityId!)).toBe("990002");
|
||||
const after = await row(r.id);
|
||||
expect(after.media_entity_id).toBe(res.mediaEntityId);
|
||||
expect((after.raw as { _detail?: unknown })._detail).toMatchObject({ imdbId: "tt36758489", doubanId: "37297001" });
|
||||
expect((after.raw as { _escalation?: { status: string } })._escalation?.status).toBe("fetched");
|
||||
expect(after.resolution_evidence).toMatchObject({
|
||||
escalation: { detail: "fetched", via: "site_id", pending: false },
|
||||
});
|
||||
// 站点英文名 A Prophet 与作品名对不上:只凭 ID 确认,不登记为别名(ID-2 守卫照旧)
|
||||
const aliases = await h.db
|
||||
.selectFrom("title_alias")
|
||||
.select("alias")
|
||||
.where("media_entity_id", "=", res.mediaEntityId!)
|
||||
.where("source", "=", "site")
|
||||
.execute();
|
||||
expect(aliases.map((a) => a.alias)).not.toContain("A Prophet");
|
||||
expect(await escalationCount(h.rt, audiences)).toBe(1);
|
||||
|
||||
// 再识别一次:详情已抓过,不再请求站点;IDs 在 raw._detail 里,第一轮就按 ID 认出
|
||||
const again = await resolveRelease(h.rt, { sourceReleaseId: r.id, force: true, trigger: "manual" });
|
||||
expect(["auto", "skipped"]).toContain(again.outcome);
|
||||
expect(detailRequests("727077")).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("观众 725858:同名同年两部 → 有歧义的事项;手动重新识别抓详情 → IMDb 确认 → 事项自动关闭;详情页豆瓣信息块进豆瓣条目缓存", async () => {
|
||||
const r = await release(audiences, "725858", "Rose 2026 1080p BluRay AVC DTS-HD MA 5.1-DIY@Audies");
|
||||
// 不允许升级时(旧行为):歧义 → 开 identity_ambiguous
|
||||
const first = await resolveRelease(h.rt, { sourceReleaseId: r.id, trigger: "manual", escalate: false });
|
||||
expect(first.outcome).toBe("ambiguous");
|
||||
expect(first.decisionId).not.toBeNull();
|
||||
expect(detailRequests("725858")).toHaveLength(0);
|
||||
|
||||
const res = await resolveRelease(h.rt, { sourceReleaseId: r.id, force: true, trigger: "manual" });
|
||||
expect(res.outcome).toBe("auto");
|
||||
expect(await tmdbIdOf(res.mediaEntityId!)).toBe("990001");
|
||||
expect(detailRequests("725858")).toHaveLength(1);
|
||||
const d = await h.db
|
||||
.selectFrom("decision")
|
||||
.select(["status"])
|
||||
.where("id", "=", first.decisionId!)
|
||||
.executeTakeFirst();
|
||||
expect(d?.status).toBe("resolved");
|
||||
const pool = await h.db
|
||||
.selectFrom("identity_candidate")
|
||||
.select(["status", "resolved_media_entity_id"])
|
||||
.where("source_release_id", "=", r.id)
|
||||
.executeTakeFirstOrThrow();
|
||||
expect(pool).toMatchObject({ status: "resolved", resolved_media_entity_id: res.mediaEntityId });
|
||||
// 详情页信息块(◎片名 Rose / ◎译名 罗斯 / ◎年代 2026)+ 豆瓣 ID 35322132 → 豆瓣条目
|
||||
const subject = await loadDoubanSubject(h.ctx, "35322132");
|
||||
expect(subject).toMatchObject({ id: "35322132", title: "Rose", year: 2026, source: "site_detail" });
|
||||
expect(subject?.aliases).toContain("罗斯");
|
||||
});
|
||||
|
||||
it("憨憨详情样本:MediaInfo 里的 TMDB tv/290001 直取详情 → 自动确认", async () => {
|
||||
const r = await release(hhan, "91003", "Example Show S01 2026 2160p YK WEB-DL H.265 DV HQ AAC2.0-HHWEB");
|
||||
const res = await resolveRelease(h.rt, { sourceReleaseId: r.id, trigger: "search" });
|
||||
expect(res.outcome).toBe("auto");
|
||||
expect(await tmdbIdOf(res.mediaEntityId!)).toBe("290001");
|
||||
expect(detailRequests("91003")).toHaveLength(1);
|
||||
expect((await row(r.id)).resolution_evidence).toMatchObject({ escalation: { via: "site_id" } });
|
||||
});
|
||||
|
||||
it("每站每小时上限:达到后推迟(不开事项、15 分钟内复查),不再请求站点;0 = 不抓", async () => {
|
||||
await setCap(1);
|
||||
const a = await release(audiences, "990101", "Nowhere Title 2026 1080p WEB-DL");
|
||||
const b = await release(audiences, "990102", "Another Nowhere 2026 1080p WEB-DL");
|
||||
const before = siteRequests.length;
|
||||
await resolveRelease(h.rt, { sourceReleaseId: a.id, trigger: "search" });
|
||||
const rb = await resolveRelease(h.rt, { sourceReleaseId: b.id, trigger: "search" });
|
||||
// a 用掉名额(样本里没有这个 id 的页面,详情请求失败也算用掉;失败不阻塞,照常进池,6 小时内不再为识别重抓)
|
||||
expect(siteRequests.length - before).toBe(1);
|
||||
expect((await row(a.id)).resolution_evidence).toMatchObject({
|
||||
escalation: { detail: "failed:fetch_failed", pending: false },
|
||||
});
|
||||
await resolveRelease(h.rt, { sourceReleaseId: a.id, force: true, trigger: "manual" });
|
||||
expect((await row(a.id)).resolution_evidence).toMatchObject({ escalation: { detail: "skipped:recent_failure" } });
|
||||
expect(siteRequests.length - before).toBe(1);
|
||||
expect(rb.decisionId).toBeNull();
|
||||
const pool = await h.db
|
||||
.selectFrom("identity_candidate")
|
||||
.select(["query", "next_review_at"])
|
||||
.where("source_release_id", "=", b.id)
|
||||
.executeTakeFirstOrThrow();
|
||||
expect(pool.query).toMatchObject({ escalation: { detail: "deferred:cap_reached", pending: true } });
|
||||
expect(pool.next_review_at!.getTime() - Date.now()).toBeLessThanOrEqual(15 * 60_000 + 5_000);
|
||||
|
||||
await setCap(0);
|
||||
const c = await release(audiences, "990103", "Third Nowhere 2026 1080p WEB-DL");
|
||||
await resolveRelease(h.rt, { sourceReleaseId: c.id, trigger: "search" });
|
||||
expect((await row(c.id)).resolution_evidence).toMatchObject({ escalation: { detail: "skipped:disabled" } });
|
||||
expect(siteRequests.length - before).toBe(1);
|
||||
});
|
||||
|
||||
it("订阅相关的资源:RSS 入库时不抓详情、不开「没识别出作品」;复查时抓详情认出;旧事项自动关闭", async () => {
|
||||
// 本地作品(如订阅)登记了别名「Example Show」,但没有 TMDB 身份、年份也对不上:资源与它相关(tracked_alias)却认不出
|
||||
const e = await h.db
|
||||
.insertInto("media_entity")
|
||||
.values({ kind: "series", title: "Example Show", year: 1990 } as never)
|
||||
.returning("id")
|
||||
.executeTakeFirstOrThrow();
|
||||
await sql`insert into title_alias (media_entity_id, alias, normalized, kind, source)
|
||||
values (${e.id}, 'Example Show', ${normalizeTitle("Example Show")}, 'other', 'user')`.execute(h.db);
|
||||
const r = await release(hhan, "91004", "Example Show S01E18 2027 2160p YK WEB-DL H.265 DV HQ AAC2.0-HHWEB");
|
||||
|
||||
// 旧行为(不升级):相关却认不出 → identity_unresolved
|
||||
const old = await resolveRelease(h.rt, { sourceReleaseId: r.id, trigger: "review", force: true, escalate: false });
|
||||
expect(old.decisionId).not.toBeNull();
|
||||
const oldKind = (
|
||||
await h.db.selectFrom("decision").select("kind").where("id", "=", old.decisionId!).executeTakeFirstOrThrow()
|
||||
).kind;
|
||||
|
||||
// RSS 入库(observed):不抓详情,推迟到复查;不再保留事项
|
||||
const obs = await resolveRelease(h.rt, { sourceReleaseId: r.id, force: true, trigger: "observed" });
|
||||
expect(obs.decisionId).toBeNull();
|
||||
expect(detailRequests("91004")).toHaveLength(0);
|
||||
expect((await row(r.id)).resolution_evidence).toMatchObject({
|
||||
escalation: { detail: "not_now:batch_ingest", pending: true },
|
||||
});
|
||||
const d1 = await h.db.selectFrom("decision").select("status").where("id", "=", old.decisionId!).executeTakeFirst();
|
||||
expect(d1?.status).not.toBe("open");
|
||||
|
||||
// 复查(订阅相关):抓详情 → TMDB tv/290001(年份 2027 与 2026 差 1 年,在容差内)→ 自动确认
|
||||
const rev = await resolveRelease(h.rt, { sourceReleaseId: r.id, force: true, trigger: "review" });
|
||||
expect(detailRequests("91004")).toHaveLength(1);
|
||||
expect(rev.outcome).toBe("auto");
|
||||
expect(oldKind).toBe("identity_unresolved");
|
||||
expect(await openDecisions()).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("豆瓣路径:详情只有豆瓣 ID → 缓存的豆瓣条目(搜索摘要抽取)标题 + 年份 → TMDB 匹配(ID-2 打分)", async () => {
|
||||
// META-2 搜索摘要里抽到的豆瓣条目(不请求豆瓣)
|
||||
const s = doubanSubjectFromSnippet(
|
||||
{
|
||||
url: "https://movie.douban.com/subject/36000099/",
|
||||
title: "青灯记 (豆瓣)",
|
||||
snippet: "导演: 某导演 主演: 甲 / 乙 类型: 剧情 / 古装 制片国家/地区: 中国大陆 年份: 2025 集数: 30",
|
||||
},
|
||||
new Date(),
|
||||
);
|
||||
expect(s).toMatchObject({ id: "36000099", title: "青灯记", year: 2025, source: "web_snippet" });
|
||||
await saveDoubanSubject(h.ctx, s!);
|
||||
// 详情已抓过,只给了豆瓣 ID
|
||||
const r = await release(hhan, "91005", "Qing Deng Ji S01E03 2025 1080p WEB-DL H264 AAC-HHWEB", {
|
||||
_detail: { doubanId: "36000099" },
|
||||
});
|
||||
await h.db.updateTable("source_release").set({ detail_fetched_at: new Date() }).where("id", "=", r.id).execute();
|
||||
const res = await resolveRelease(h.rt, { sourceReleaseId: r.id, trigger: "manual" });
|
||||
expect(res.outcome).toBe("auto");
|
||||
expect(await tmdbIdOf(res.mediaEntityId!)).toBe("990021");
|
||||
expect(detailRequests("91005")).toHaveLength(0);
|
||||
expect((await row(r.id)).resolution_evidence).toMatchObject({
|
||||
escalation: { via: "douban", douban: { id: "36000099", title: "青灯记", source: "web_snippet" } },
|
||||
});
|
||||
|
||||
// 缓存里没有这个豆瓣条目:不请求豆瓣,只进池;有 ID 的资源不开「没识别出作品」
|
||||
await sharedCache(h.ctx).delete(DOUBAN_SUBJECT_REGION, "36000098");
|
||||
const r2 = await release(hhan, "91006", "Unknown Drama S01E01 2025 1080p WEB-DL", {
|
||||
_detail: { doubanId: "36000098" },
|
||||
});
|
||||
await h.db.updateTable("source_release").set({ detail_fetched_at: new Date() }).where("id", "=", r2.id).execute();
|
||||
const res2 = await resolveRelease(h.rt, { sourceReleaseId: r2.id, trigger: "manual" });
|
||||
expect(res2.outcome).toBe("no_match");
|
||||
expect(res2.decisionId).toBeNull();
|
||||
expect(siteRequests.some((x) => x.includes("douban"))).toBe(false);
|
||||
});
|
||||
|
||||
it("池条目补全前先走升级链:详情页给了 ID 就直接认出,返回 409,不做网页补全", async () => {
|
||||
const r = await release(hhan, "91007", "Example Show S01 2026 2160p YK WEB-DL H.265 DV HQ AAC2.0-HHWEB");
|
||||
const first = await resolveRelease(h.rt, { sourceReleaseId: r.id, trigger: "manual", escalate: false });
|
||||
expect(first.outcome).toBe("no_match");
|
||||
const pool = await h.db
|
||||
.selectFrom("identity_candidate")
|
||||
.select("id")
|
||||
.where("source_release_id", "=", r.id)
|
||||
.executeTakeFirstOrThrow();
|
||||
const res = await h.req(`/identity-pool/${pool.id}/enrich`, { body: {} });
|
||||
expect(res.status).toBe(409);
|
||||
expect(((await res.json()) as { error: { message: string } }).error.message).toContain("《示例剧集》");
|
||||
expect(detailRequests("91007")).toHaveLength(1);
|
||||
expect((await row(r.id)).resolution_status).toBe("resolved");
|
||||
});
|
||||
|
||||
it("站点详情来源的豆瓣条目不会被搜索摘要覆盖", async () => {
|
||||
await saveDoubanSubject(h.ctx, {
|
||||
id: "1",
|
||||
title: "甲",
|
||||
aliases: [],
|
||||
year: 2020,
|
||||
source: "site_detail",
|
||||
ref: "x",
|
||||
at: new Date().toISOString(),
|
||||
});
|
||||
await saveDoubanSubject(h.ctx, {
|
||||
id: "1",
|
||||
title: "乙",
|
||||
aliases: [],
|
||||
year: 2021,
|
||||
source: "web_snippet",
|
||||
ref: "y",
|
||||
at: new Date().toISOString(),
|
||||
});
|
||||
expect((await loadDoubanSubject(h.ctx, "1"))?.title).toBe("甲");
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user