feat(identity): source-aware short-drama enrichment with snippet mode and hit rate (META-2)

- classify enrich sources: encyclopedia, platform pages (hongguo, kuaikaw, video
  platforms), douban (search snippet only) and other sites' search snippets
  (snippet mode, pages never opened); one extractor per source, fields carry
  source kind and confidence, merged by noisy-OR across hosts
- new cast field (applied as MediaCredit with placeholder ids)
- robots.txt check (cached per host 1 day) and login-wall/captcha give-up
- per-enrichment source attempts + GET /identity/enrich-stats, 30-day hit-rate
  table under settings -> metadata -> identity
- setting identity.enrichSnippetAllSites (default on); default domains add
  baike.com and kuaikaw.cn
- replay fixtures from the real run on the 4 user-library short dramas
  (sanitized Brave responses, hongguo page, robots.txt)

Co-Authored-By: Claude Opus 5.5 <[email protected]>

apps/server/test/sources-identity/enrich.test.ts

@@ -1,6 +1,6 @@
 import { sql } from "@umt/db";
 import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest";
-import { registerMetadataCapabilities } from "../../src/modules/agent/capabilities-metadata.ts";
+import { registerMetadataCapabilities, robotsAllowedFor } from "../../src/modules/agent/capabilities-metadata.ts";
 import { createCapabilityRegistry } from "../../src/modules/agent/capability-registry.ts";
 import {
   AUTO_ENRICH_TASK,
@@ -8,8 +8,9 @@ import {
   type ModelCompletion,
   maybeAutoEnrich,
   provisionalEntitiesForTmdbReview,
-  ruleExtract,
 } from "../../src/modules/sources-identity/service/enrich.ts";
+import { extractFromPage } from "../../src/modules/sources-identity/service/enrich-sources.ts";
+
 import { createWebSearchModule } from "../../src/modules/websearch/index.ts";
 import { createHarness, type Harness, json, pgAvailable, seedSite } from "./harness.ts";
 
@@ -32,6 +33,7 @@ const OTHER = "https://baike.example.com/item/other";
 const BAIKE_HTML = `<!doctype html><html><head><title>${TITLE}_百科</title>
 <meta property="og:image" content="${POSTER}"></head><body><h1>${TITLE}</h1>
 <p>${TITLE}，又名：替身娇妻。类型：都市、甜宠</p><p>首播时间：2024年3月1日</p><p>播出平台：红果短剧</p><p>共80集</p>
+<p>主演：林小雨、陈默</p>
 <p>剧情简介：落魄千金为救母亲替嫁给冷面总裁，两人在一次次误会中走近，最终揭开当年的家族阴谋。</p></body></html>`;
 const VIDEO_HTML = `<!doctype html><html><head><title>${TITLE} 全80集</title></head>
 <body><p>${TITLE} 全80集 2024 红果短剧独播</p></body></html>`;
@@ -128,7 +130,7 @@ describe.skipIf(!pg)("META-1 元数据补全", () => {
     siteId = (await seedSite(h, { key: "hhclub", baseUrl: "https://pt.hhclub.example/" })).site.id;
     const registry = createCapabilityRegistry({ logger: h.ctx.logger, db: h.db, settings: h.ctx.settings });
     createWebSearchModule({ fetch: mockFetch, resolveHost }).registerCapabilities?.(registry, h.ctx);
-    registerMetadataCapabilities(registry, h.ctx, { complete: fakeComplete });
+    registerMetadataCapabilities(registry, h.ctx, { complete: fakeComplete, fetch: mockFetch, resolveHost });
     (h.ctx as { capabilities: unknown }).capabilities = registry;
     const ref = await h.secrets.create({ purpose: "other", plaintext: "BSA-enrich-test", label: "websearch" });
     await setSettings("agent", { webSearch: { provider: "brave", apiKeySecretRef: ref, maxResults: 10 } });
@@ -144,7 +146,7 @@ describe.skipIf(!pg)("META-1 元数据补全", () => {
     modelMode = "json";
   });
 
-  it("建议:字段带出处与证据等级;白名单外域名 / 已登记站点 / 跳转出白名单的网页不用;不写作品表", async () => {
+  it("建议:字段带出处、来源类别、置信度与证据等级;白名单外域名 / 已登记站点 / 跳转出白名单的网页不读;不写作品表", async () => {
     const { poolId } = await poolEntry(TITLE);
     const entitiesBefore = await h.db.selectFrom("media_entity").select("id").execute();
     const out = await invoke({ poolEntryId: poolId });
@@ -156,15 +158,24 @@ describe.skipIf(!pg)("META-1 元数据补全", () => {
     expect(f.episodeCount).toMatchObject({ value: 80, evidenceLevel: "rule_confirmed", verbatim: true });
     expect(f.year).toMatchObject({ value: 2024, evidenceLevel: "rule_confirmed" });
     expect(f.platform).toMatchObject({ value: "红果短剧", evidenceLevel: "rule_confirmed" });
-    // 改写的简介不是原文 → 待复核、Agent 归纳
-    expect(f.synopsis).toMatchObject({ evidenceLevel: "needs_review", verbatim: false });
-    // 模型自己拼的海报地址丢弃
-    expect(f.posterUrl).toBeNull();
+    // 模型改写的简介(置信度 0.3)不如网页原文(规则抽取)→ 取原文
+    expect(f.synopsis).toMatchObject({ verbatim: true, sourceUrl: BAIKE, sourceKind: "other" });
+    expect(f.synopsis.value).toMatch(/^落魄千金为救母亲/);
+    // 模型自己拼的海报地址丢弃,用网页的 og:image
+    expect(f.posterUrl).toMatchObject({ value: POSTER, sourceUrl: BAIKE });
+    // 两个网站都写了 80 集:置信度叠加
+    expect(f.episodeCount.hosts).toBe(2);
+    expect(f.episodeCount.confidence).toBeGreaterThan(0.8);
+    expect(f.cast.map((c: Any) => c.value)).toEqual(["林小雨", "陈默"]);
+    // 各来源的尝试与命中(命中率统计用)
+    const att = Object.fromEntries(out.usage.attempts.map((a: Any) => [a.kind, a]));
+    expect(att.other).toMatchObject({ fetched: 2, hit: true });
+    expect(att.snippet.hit).toBe(false);
     expect(f.aliases.map((a: Any) => a.value)).toEqual(["替身娇妻"]);
     expect(f.genre.map((g: Any) => g.value)).toEqual(["都市", "甜宠"]);
 
     const src = Object.fromEntries(out.suggestion.sources.map((s: Any) => [s.url, s]));
-    expect(src[EVIL]).toMatchObject({ allowed: false, fetched: false, skipped: "not_allowed" });
+    expect(src[EVIL]).toMatchObject({ allowed: false, fetched: false, skipped: "not_allowed", kind: "snippet" });
     // 已登记站点的域名即使在白名单里也不读
     expect(src[SITE_PAGE]).toMatchObject({ allowed: false, fetched: false, skipped: "site" });
     expect(src[BAIKE]).toMatchObject({ fetched: true });
@@ -172,8 +183,14 @@ describe.skipIf(!pg)("META-1 元数据补全", () => {
     // 白名单内但标题对不上:不读
     expect(src[OTHER]).toMatchObject({ allowed: true, fetched: false, skipped: "irrelevant" });
     expect(pageHits).not.toContain(OTHER);
+    expect(pageHits).not.toContain("https://evil.example.net/robots.txt");
     expect(pageHits).not.toContain(SITE_PAGE);
     expect(pageHits.filter((u) => u === EVIL)).toHaveLength(1); // 只有重定向那一跳,结果被丢弃
+    // 读网页前先看 robots.txt(每个网站一次,缓存 1 天)
+    expect(pageHits.filter((u) => u.endsWith("/robots.txt")).sort()).toEqual([
+      "https://baike.example.com/robots.txt",
+      "https://v.example.com/robots.txt",
+    ]);
     expect(out.usage).toMatchObject({ searchCalls: 1, fetchCalls: 3, inputTokens: 1200, outputTokens: 300 });
     expect(out.budget).toEqual({ usedToday: 1, dailyLimit: 10 });
     expect(out.notice).toContain("不是 TMDB");
@@ -253,7 +270,12 @@ describe.skipIf(!pg)("META-1 元数据补全", () => {
     expect(e).toMatchObject({ status: "provisional", kind: "series", title: TITLE, year: 2024 });
     const meta = e.metadata as Any;
     expect(meta.title).toMatchObject({ value: TITLE, source: "web" });
-    expect(meta.overview).toMatchObject({ source: "agent" }); // 改写的简介标为 Agent 生成
+    expect(meta.overview).toMatchObject({ source: "web" }); // 网页原文
+    // 主演:没有 TMDB 人物 ID,用负数占位
+    expect(meta.cast.value).toEqual([
+      { id: -1, name: "林小雨", originalName: null, role: null, profilePath: null },
+      { id: -2, name: "陈默", originalName: null, role: null, profilePath: null },
+    ]);
     expect(meta.episodeCount).toMatchObject({ value: 80, source: "web" });
     expect(meta.genres.value).toEqual(["都市", "甜宠"]);
     const fields = await h.db.selectFrom("metadata_field").selectAll().where("media_entity_id", "=", e.id).execute();
@@ -318,10 +340,45 @@ describe.skipIf(!pg)("META-1 元数据补全", () => {
   });
 
   it("规则抽取只用标题对得上的网页", () => {
-    const r = ruleExtract({ title: "别的剧" }, [
-      { url: BAIKE, finalUrl: BAIKE, title: `${TITLE}_百科`, text: `${TITLE} 共80集`, image: POSTER },
-    ]);
-    expect(r.title).toBeNull();
-    expect(r.episodeCount).toBeNull();
+    const r = extractFromPage("别的剧", {
+      url: BAIKE,
+      host: "baike.example.com",
+      kind: "other",
+      title: `${TITLE}_百科`,
+      text: `${TITLE} 共80集`,
+      image: POSTER,
+    });
+    expect(r).toEqual([]);
+  });
+
+  it("robots.txt:不允许的网址不读(按网站缓存);5xx 当作不允许,404 当作没有限制", async () => {
+    const hits: string[] = [];
+    const f = (async (input: string | URL | Request) => {
+      const u = new URL(typeof input === "string" ? input : input instanceof URL ? input.href : input.url);
+      hits.push(u.href);
+      if (u.hostname === "robots-a.example.com")
+        return new Response("User-agent: *\nDisallow: /series/\n", { headers: { "content-type": "text/plain" } });
+      if (u.hostname === "robots-b.example.com") return new Response("err", { status: 503 });
+      return new Response("nf", { status: 404 });
+    }) as typeof fetch;
+    const o = { fetch: f, resolveHost };
+    expect(await robotsAllowedFor(h.ctx, "https://robots-a.example.com/series/1", o)).toBe(false);
+    expect(await robotsAllowedFor(h.ctx, "https://robots-a.example.com/detail?id=1", o)).toBe(true);
+    expect(hits.filter((x) => x.includes("robots-a"))).toHaveLength(1);
+    expect(await robotsAllowedFor(h.ctx, "https://robots-b.example.com/x", o)).toBe(false);
+    expect(await robotsAllowedFor(h.ctx, "https://robots-c.example.com/x", o)).toBe(true);
+  });
+
+  it("命中率:最近 30 天按来源类别统计(GET /identity/enrich-stats)", async () => {
+    const res = await json(await h.req("/identity/enrich-stats?days=30"));
+    expect(res.status).toBe(200);
+    const d = res.body.data;
+    expect(d.days).toBe(30);
+    expect(d.total).toBeGreaterThan(0);
+    expect(d.tracked).toBe(d.total);
+    const other = d.sources.find((x: Any) => x.kind === "other");
+    expect(other.hits).toBeGreaterThan(0);
+    expect(other.rate).toBeGreaterThan(0);
+    expect(d.sources.find((x: Any) => x.kind === "douban")).toMatchObject({ hits: 0, rate: 0 });
   });
 });