UHD_Media_Tool/apps/server/test/sources-identity/enrich.test.ts

import { sql } from "@umt/db";
import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest";
import { registerMetadataCapabilities, robotsAllowedFor } from "../../src/modules/agent/capabilities-metadata.ts";
import { createCapabilityRegistry } from "../../src/modules/agent/capability-registry.ts";
import {
  AUTO_ENRICH_TASK,
  applyEnrichment,
  type ModelCompletion,
  maybeAutoEnrich,
  provisionalEntitiesForTmdbReview,
} from "../../src/modules/sources-identity/service/enrich.ts";
import { extractFromPage } from "../../src/modules/sources-identity/service/enrich-sources.ts";

import { createWebSearchModule } from "../../src/modules/websearch/index.ts";
import { createHarness, type Harness, json, pgAvailable, seedSite } from "./harness.ts";

// META-1 元数据补全:fake 模型 + mock 搜索 / 网页。建议结构与证据等级、白名单外域名(含重定向出白名单、已登记站点)不读、
// 7 天标题缓存、每日预算、应用后的临时作品与字段级元数据(web / agent,不冒充 TMDB)、同名去重、自动触发入队。

const pg = await pgAvailable();
// biome-ignore lint/suspicious/noExplicitAny: 测试断言直接读 JSON
type Any = any;

const TITLE = "霸总的替身新娘";
const BAIKE = "https://baike.example.com/item/bazong";
const VIDEO = "https://v.example.com/show/1";
const REDIR = "https://v.example.com/r/2";
const EVIL = "https://evil.example.net/x";
const SITE_PAGE = "https://pt.hhclub.example/details.php?id=1";
const POSTER = "https://img.example-cdn.com/p/1.jpg";
const OTHER = "https://baike.example.com/item/other";

const BAIKE_HTML = `<!doctype html><html><head><title>${TITLE}_百科</title>
<meta property="og:image" content="${POSTER}"></head><body><h1>${TITLE}</h1>
<p>${TITLE}，又名：替身娇妻。类型：都市、甜宠</p><p>首播时间：2024年3月1日</p><p>播出平台：红果短剧</p><p>共80集</p>
<p>主演：林小雨、陈默</p>
<p>剧情简介：落魄千金为救母亲替嫁给冷面总裁，两人在一次次误会中走近，最终揭开当年的家族阴谋。</p></body></html>`;
const VIDEO_HTML = `<!doctype html><html><head><title>${TITLE} 全80集</title></head>
<body><p>${TITLE} 全80集 2024 红果短剧独播</p></body></html>`;

let braveCalls = 0;
const pageHits: string[] = [];
const mockFetch = (async (input: string | URL | Request) => {
  const url = new URL(typeof input === "string" ? input : input instanceof URL ? input.href : input.url);
  if (url.hostname === "api.search.brave.com") {
    braveCalls++;
    return Response.json({
      web: {
        results: [
          { title: `${TITLE} - 百科`, url: BAIKE, description: `${TITLE} 共80集 红果短剧` },
          { title: `${TITLE} 种子`, url: SITE_PAGE, description: "站点资源" },
          { title: "不相关网站", url: EVIL, description: TITLE },
          { title: `${TITLE} 全集`, url: VIDEO, description: `${TITLE} 全80集` },
          { title: `${TITLE} 跳转`, url: REDIR, description: TITLE },
          { title: "另一部剧_百科", url: OTHER, description: "无关条目" },
        ],
      },
    });
  }
  pageHits.push(url.href);
  if (url.href === BAIKE) return new Response(BAIKE_HTML, { headers: { "content-type": "text/html; charset=utf-8" } });
  if (url.href === VIDEO) return new Response(VIDEO_HTML, { headers: { "content-type": "text/html; charset=utf-8" } });
  if (url.href === REDIR) return new Response(null, { status: 302, headers: { location: EVIL } });
  if (url.href === EVIL)
    return new Response("<html><title>x</title><body>x</body></html>", { headers: { "content-type": "text/html" } });
  return new Response("not found", { status: 404 });
}) as typeof fetch;
const resolveHost = async () => ["93.184.216.34"];

/** fake 模型:照网页给值,简介是改写过的(不是原文),海报给一个网页里没有的地址(应被丢弃) */
let modelMode: "json" | "none" | "garbage" = "json";
const fakeComplete = async (_system: string, user: string): Promise<ModelCompletion | null> => {
  if (modelMode === "none") return null;
  if (modelMode === "garbage") return { text: "抱歉，我无法完成", model: "fake", inputTokens: 10, outputTokens: 5 };
  const prompt = JSON.parse(user) as { pages: { url: string }[] };
  const baike = prompt.pages.find((p) => p.url === BAIKE)?.url ?? null;
  return {
    text: `\`\`\`json\n${JSON.stringify({
      title: { value: TITLE, sourceUrl: baike },
      aliases: [{ value: "替身娇妻", sourceUrl: baike }],
      year: { value: 2024, sourceUrl: baike },
      platform: { value: "红果短剧", sourceUrl: baike },
      episodeCount: { value: 80, sourceUrl: baike },
      synopsis: { value: "一个替嫁千金与总裁从误会到相爱的故事。", sourceUrl: baike },
      posterUrl: { value: "https://img.example-cdn.com/made-up.jpg", sourceUrl: baike },
      genre: [
        { value: "都市", sourceUrl: baike },
        { value: "甜宠", sourceUrl: baike },
      ],
    })}\n\`\`\``,
    model: "fake-extract",
    inputTokens: 1200,
    outputTokens: 300,
  };
};

describe.skipIf(!pg)("META-1 元数据补全", () => {
  let h: Harness;
  let siteId: string;
  let n = 0;

  const setSettings = async (key: string, value: unknown) => {
    await sql`insert into settings (key, value) values (${key}, ${JSON.stringify(value)}::jsonb)
      on conflict (key) do update set value = excluded.value`.execute(h.db);
  };
  const poolEntry = async (title: string, query: Record<string, unknown> = { title }) => {
    const r = await h.db
      .insertInto("source_release")
      .values({ site_id: siteId, site_release_id: String(700000 + ++n), title: `${title} 2024 1080p WEB-DL-HHWEB` })
      .returningAll()
      .executeTakeFirstOrThrow();
    const p = await sql<{
      id: string;
    }>`insert into identity_candidate (source_release_id, reason, query, normalized_title, evidence_version)
      values (${r.id}, 'unmatched', ${JSON.stringify(query)}::jsonb, ${title}, 'v1') returning id`.execute(h.db);
    return { poolId: p.rows[0]!.id, releaseId: r.id };
  };
  const invoke = (input: Record<string, unknown>, taskId = crypto.randomUUID()) =>
    h.ctx.capabilities.invoke("metadata.enrich", input, {
      actorId: "user:admin",
      taskId,
      scopeIds: [],
      signal: new AbortController().signal,
      source: "web",
    }) as Promise<Any>;
  const actor = { type: "user" as const, id: "admin" };

  beforeAll(async () => {
    h = await createHarness("enrich");
    siteId = (await seedSite(h, { key: "hhclub", baseUrl: "https://pt.hhclub.example/" })).site.id;
    const registry = createCapabilityRegistry({ logger: h.ctx.logger, db: h.db, settings: h.ctx.settings });
    createWebSearchModule({ fetch: mockFetch, resolveHost }).registerCapabilities?.(registry, h.ctx);
    registerMetadataCapabilities(registry, h.ctx, { complete: fakeComplete, fetch: mockFetch, resolveHost });
    (h.ctx as { capabilities: unknown }).capabilities = registry;
    const ref = await h.secrets.create({ purpose: "other", plaintext: "BSA-enrich-test", label: "websearch" });
    await setSettings("agent", { webSearch: { provider: "brave", apiKeySecretRef: ref, maxResults: 10 } });
    await setSettings("identity", {
      enrichDailyBudget: 10,
      enrichDomains: ["baike.example.com", "v.example.com", "hhclub.example"],
    });
  });
  afterAll(async () => {
    await h?.close();
  });
  beforeEach(() => {
    modelMode = "json";
  });

  it("建议:字段带出处、来源类别、置信度与证据等级;白名单外域名 / 已登记站点 / 跳转出白名单的网页不读;不写作品表", async () => {
    const { poolId } = await poolEntry(TITLE);
    const entitiesBefore = await h.db.selectFrom("media_entity").select("id").execute();
    const out = await invoke({ poolEntryId: poolId });
    expect(out).toMatchObject({ status: "suggested", cached: false, normalizedTitle: expect.any(String) });
    const f = out.suggestion.fields;
    expect(out.suggestion.extractor).toBe("model");
    expect(f.title).toMatchObject({ value: TITLE, sourceUrl: BAIKE, verbatim: true });
    // 规则抽取在同一网页得到相同值 → 规则确认
    expect(f.episodeCount).toMatchObject({ value: 80, evidenceLevel: "rule_confirmed", verbatim: true });
    expect(f.year).toMatchObject({ value: 2024, evidenceLevel: "rule_confirmed" });
    expect(f.platform).toMatchObject({ value: "红果短剧", evidenceLevel: "rule_confirmed" });
    // 模型改写的简介(置信度 0.3)不如网页原文(规则抽取)→ 取原文
    expect(f.synopsis).toMatchObject({ verbatim: true, sourceUrl: BAIKE, sourceKind: "other" });
    expect(f.synopsis.value).toMatch(/^落魄千金为救母亲/);
    // 模型自己拼的海报地址丢弃,用网页的 og:image
    expect(f.posterUrl).toMatchObject({ value: POSTER, sourceUrl: BAIKE });
    // 两个网站都写了 80 集:置信度叠加
    expect(f.episodeCount.hosts).toBe(2);
    expect(f.episodeCount.confidence).toBeGreaterThan(0.8);
    expect(f.cast.map((c: Any) => c.value)).toEqual(["林小雨", "陈默"]);
    // 各来源的尝试与命中(命中率统计用)
    const att = Object.fromEntries(out.usage.attempts.map((a: Any) => [a.kind, a]));
    expect(att.other).toMatchObject({ fetched: 2, hit: true });
    expect(att.snippet.hit).toBe(false);
    expect(f.aliases.map((a: Any) => a.value)).toEqual(["替身娇妻"]);
    expect(f.genre.map((g: Any) => g.value)).toEqual(["都市", "甜宠"]);

    const src = Object.fromEntries(out.suggestion.sources.map((s: Any) => [s.url, s]));
    expect(src[EVIL]).toMatchObject({ allowed: false, fetched: false, skipped: "not_allowed", kind: "snippet" });
    // 已登记站点的域名即使在白名单里也不读
    expect(src[SITE_PAGE]).toMatchObject({ allowed: false, fetched: false, skipped: "site" });
    expect(src[BAIKE]).toMatchObject({ fetched: true });
    expect(src[REDIR]).toMatchObject({ fetched: false, skipped: "not_allowed" });
    // 白名单内但标题对不上:不读
    expect(src[OTHER]).toMatchObject({ allowed: true, fetched: false, skipped: "irrelevant" });
    expect(pageHits).not.toContain(OTHER);
    expect(pageHits).not.toContain("https://evil.example.net/robots.txt");
    expect(pageHits).not.toContain(SITE_PAGE);
    expect(pageHits.filter((u) => u === EVIL)).toHaveLength(1); // 只有重定向那一跳,结果被丢弃
    // 读网页前先看 robots.txt(每个网站一次,缓存 1 天)
    expect(pageHits.filter((u) => u.endsWith("/robots.txt")).sort()).toEqual([
      "https://baike.example.com/robots.txt",
      "https://v.example.com/robots.txt",
    ]);
    expect(out.usage).toMatchObject({ searchCalls: 1, fetchCalls: 3, inputTokens: 1200, outputTokens: 300 });
    expect(out.budget).toEqual({ usedToday: 1, dailyLimit: 10 });
    expect(out.notice).toContain("不是 TMDB");
    // 不落作品表
    expect(await h.db.selectFrom("media_entity").select("id").execute()).toHaveLength(entitiesBefore.length);
    const row = await h.db
      .selectFrom("metadata_enrichment")
      .selectAll()
      .where("id", "=", out.enrichmentId)
      .executeTakeFirstOrThrow();
    expect(row).toMatchObject({ status: "suggested", trigger: "manual", identity_candidate_id: poolId });
  });

  it("同一标题 7 天内命中缓存:不再搜索、不计预算", async () => {
    const calls = braveCalls;
    const hits = pageHits.length;
    const out = await invoke({ title: TITLE });
    expect(out.cached).toBe(true);
    expect(out.status).toBe("suggested");
    expect(braveCalls).toBe(calls);
    expect(pageHits.length).toBe(hits);
    expect(out.budget.usedToday).toBe(1);
  });

  it("没有可用模型 / 模型输出不是 JSON:按规则抽取", async () => {
    modelMode = "none";
    const a = await invoke({ title: TITLE, fresh: true });
    expect(a.suggestion.extractor).toBe("rule");
    expect(a.suggestion.fields.episodeCount).toMatchObject({ value: 80, evidenceLevel: "rule_confirmed" });
    // 规则抽取的海报取 og:image
    expect(a.suggestion.fields.posterUrl).toMatchObject({ value: POSTER, verbatim: true });
    modelMode = "garbage";
    const b = await invoke({ title: TITLE, fresh: true });
    expect(b.suggestion.extractor).toBe("rule");
    expect(b.usage.inputTokens).toBe(10);
  });

  it("每日预算用完:拒绝新的补全(缓存仍可用)", async () => {
    const used = (await invoke({ title: TITLE })).budget.usedToday;
    await setSettings("identity", {
      enrichDailyBudget: used,
      enrichDomains: ["baike.example.com", "v.example.com"],
    });
    await expect(invoke({ title: "另一部短剧" })).rejects.toMatchObject({ code: "budget" });
    // HTTP 映射为 429
    const { poolId } = await poolEntry("第三部短剧");
    const res = await h.req(`/identity-pool/${poolId}/enrich`, { body: {} });
    expect(res.status).toBe(429);
    // 命中缓存不受预算影响
    expect((await invoke({ title: TITLE })).cached).toBe(true);
    await setSettings("identity", { enrichDailyBudget: 50, enrichDomains: ["baike.example.com", "v.example.com"] });
  });

  it("网页搜索未开启:不占预算,直接报前置条件", async () => {
    await setSettings("agent", { webSearch: { provider: "none" } });
    const before = await h.db.selectFrom("metadata_enrichment").select("id").execute();
    await expect(invoke({ title: "未开启时" })).rejects.toMatchObject({ code: "precondition" });
    expect(await h.db.selectFrom("metadata_enrichment").select("id").execute()).toHaveLength(before.length);
    const ref = await h.secrets.create({ purpose: "other", plaintext: "BSA-enrich-test", label: "websearch-2" });
    await setSettings("agent", { webSearch: { provider: "brave", apiKeySecretRef: ref, maxResults: 10 } });
  });

  it("应用:建临时作品 + 季集单元 + 字段级元数据(web / agent + 证据)+ 别名,关联待识别资源;同名去重", async () => {
    const { poolId, releaseId } = await poolEntry(TITLE);
    const s = await invoke({ poolEntryId: poolId, fresh: true });
    // 应用只接受控制台会话:API Key 被拒
    const denied = await h.req(`/identity-pool/${poolId}/enrich/apply`, { body: { enrichmentId: s.enrichmentId } });
    expect(denied.status).toBe(403);

    const r = await applyEnrichment(h.rt, poolId, { enrichmentId: s.enrichmentId, lock: false }, actor);
    expect(r).toMatchObject({ created: true, assigned: true, contentUnits: 81 });
    const e = await h.db
      .selectFrom("media_entity")
      .selectAll()
      .where("id", "=", r.mediaEntityId)
      .executeTakeFirstOrThrow();
    expect(e).toMatchObject({ status: "provisional", kind: "series", title: TITLE, year: 2024 });
    const meta = e.metadata as Any;
    expect(meta.title).toMatchObject({ value: TITLE, source: "web" });
    expect(meta.overview).toMatchObject({ source: "web" }); // 网页原文
    // 主演:没有 TMDB 人物 ID,用负数占位
    expect(meta.cast.value).toEqual([
      { id: -1, name: "林小雨", originalName: null, role: null, profilePath: null },
      { id: -2, name: "陈默", originalName: null, role: null, profilePath: null },
    ]);
    expect(meta.episodeCount).toMatchObject({ value: 80, source: "web" });
    expect(meta.genres.value).toEqual(["都市", "甜宠"]);
    const fields = await h.db.selectFrom("metadata_field").selectAll().where("media_entity_id", "=", e.id).execute();
    expect(fields.every((x) => x.source === "web" || x.source === "agent")).toBe(true);
    expect(fields.find((x) => x.field === "episodeCount")?.evidence).toMatchObject({
      sourceUrl: BAIKE,
      evidenceLevel: "rule_confirmed",
      enrichmentId: s.enrichmentId,
    });
    const ids = await h.db.selectFrom("external_identity").select("id").where("media_entity_id", "=", e.id).execute();
    expect(ids).toHaveLength(0); // 不冒充 TMDB,也不编外部身份
    const aliases = await h.db.selectFrom("title_alias").selectAll().where("media_entity_id", "=", e.id).execute();
    expect(aliases.map((a) => a.alias).sort()).toEqual([TITLE, "替身娇妻"].sort());
    expect(aliases.every((a) => a.source === "agent")).toBe(true);
    const rel = await h.db
      .selectFrom("source_release")
      .selectAll()
      .where("id", "=", releaseId)
      .executeTakeFirstOrThrow();
    expect(rel.media_entity_id).toBe(e.id);
    const pool = await h.db
      .selectFrom("identity_candidate")
      .selectAll()
      .where("id", "=", poolId)
      .executeTakeFirstOrThrow();
    expect(pool.status).toBe("resolved");
    // 作品详情接口带字段证据
    const d = await json(await h.req(`/media-entities/${e.id}/detail`));
    expect(d.body.data.metadataFields.find((x: Any) => x.field === "year").evidence.sourceUrl).toBe(BAIKE);
    // 留给身份巡检补 TMDB 映射
    expect((await provisionalEntitiesForTmdbReview(h.db)).map((x) => x.mediaEntityId)).toContain(e.id);

    // 同名去重:另一条同标题资源应用后并入同一作品
    const second = await poolEntry(TITLE);
    const s2 = await invoke({ poolEntryId: second.poolId });
    expect(s2.cached).toBe(true);
    expect(s2.existing.map((x: Any) => x.mediaEntityId)).toContain(e.id);
    const r2 = await applyEnrichment(h.rt, second.poolId, { enrichmentId: s2.enrichmentId, lock: true }, actor);
    expect(r2).toMatchObject({ created: false, mediaEntityId: e.id, assigned: true });
    const locked = await h.db
      .selectFrom("media_entity")
      .select("field_locks")
      .where("id", "=", e.id)
      .executeTakeFirstOrThrow();
    expect(locked.field_locks).toContain("episodeCount");
    const count = await h.db.selectFrom("media_entity").select("id").where("title", "=", TITLE).execute();
    expect(count).toHaveLength(1);
  });

  it("自动触发:只在订阅 / 规则命中且 TMDB 无结果、设置打开时入队一次", async () => {
    const { poolId, releaseId } = await poolEntry("自动补全短剧");
    await setSettings("identity", { autoEnrich: false, enrichDailyBudget: 50, enrichDomains: ["baike.example.com"] });
    expect(await maybeAutoEnrich(h.rt, releaseId, "no_match", "new_title_rule")).toBe(false);
    await setSettings("identity", { autoEnrich: true, enrichDailyBudget: 50, enrichDomains: ["baike.example.com"] });
    expect(await maybeAutoEnrich(h.rt, releaseId, "unmatched", null)).toBe(false);
    expect(await maybeAutoEnrich(h.rt, releaseId, "no_match", "manual")).toBe(false);
    expect(await maybeAutoEnrich(h.rt, releaseId, "no_match", "new_title_rule")).toBe(true);
    await maybeAutoEnrich(h.rt, releaseId, "no_match", "tracked_alias");
    const q = h.engine.enqueued.filter((x) => x.task === AUTO_ENRICH_TASK);
    expect(q).toHaveLength(1);
    expect(q[0]!.input).toEqual({ poolEntryId: poolId });
  });

  it("规则抽取只用标题对得上的网页", () => {
    const r = extractFromPage("别的剧", {
      url: BAIKE,
      host: "baike.example.com",
      kind: "other",
      title: `${TITLE}_百科`,
      text: `${TITLE} 共80集`,
      image: POSTER,
    });
    expect(r).toEqual([]);
  });

  it("robots.txt:不允许的网址不读(按网站缓存);5xx 当作不允许,404 当作没有限制", async () => {
    const hits: string[] = [];
    const f = (async (input: string | URL | Request) => {
      const u = new URL(typeof input === "string" ? input : input instanceof URL ? input.href : input.url);
      hits.push(u.href);
      if (u.hostname === "robots-a.example.com")
        return new Response("User-agent: *\nDisallow: /series/\n", { headers: { "content-type": "text/plain" } });
      if (u.hostname === "robots-b.example.com") return new Response("err", { status: 503 });
      return new Response("nf", { status: 404 });
    }) as typeof fetch;
    const o = { fetch: f, resolveHost };
    expect(await robotsAllowedFor(h.ctx, "https://robots-a.example.com/series/1", o)).toBe(false);
    expect(await robotsAllowedFor(h.ctx, "https://robots-a.example.com/detail?id=1", o)).toBe(true);
    expect(hits.filter((x) => x.includes("robots-a"))).toHaveLength(1);
    expect(await robotsAllowedFor(h.ctx, "https://robots-b.example.com/x", o)).toBe(false);
    expect(await robotsAllowedFor(h.ctx, "https://robots-c.example.com/x", o)).toBe(true);
  });

  it("命中率:最近 30 天按来源类别统计(GET /identity/enrich-stats)", async () => {
    const res = await json(await h.req("/identity/enrich-stats?days=30"));
    expect(res.status).toBe(200);
    const d = res.body.data;
    expect(d.days).toBe(30);
    expect(d.total).toBeGreaterThan(0);
    expect(d.tracked).toBe(d.total);
    const other = d.sources.find((x: Any) => x.kind === "other");
    expect(other.hits).toBeGreaterThan(0);
    expect(other.rate).toBeGreaterThan(0);
    expect(d.sources.find((x: Any) => x.kind === "douban")).toMatchObject({ hits: 0, rate: 0 });
  });
});