import { sql } from "@umt/db"; import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; import { registerMetadataCapabilities } from "../../src/modules/agent/capabilities-metadata.ts"; import { createCapabilityRegistry } from "../../src/modules/agent/capability-registry.ts"; import { AUTO_ENRICH_TASK, applyEnrichment, type ModelCompletion, maybeAutoEnrich, provisionalEntitiesForTmdbReview, ruleExtract, } from "../../src/modules/sources-identity/service/enrich.ts"; import { createWebSearchModule } from "../../src/modules/websearch/index.ts"; import { createHarness, type Harness, json, pgAvailable, seedSite } from "./harness.ts"; // META-1 元数据补全:fake 模型 + mock 搜索 / 网页。建议结构与证据等级、白名单外域名(含重定向出白名单、已登记站点)不读、 // 7 天标题缓存、每日预算、应用后的临时作品与字段级元数据(web / agent,不冒充 TMDB)、同名去重、自动触发入队。 const pg = await pgAvailable(); // biome-ignore lint/suspicious/noExplicitAny: 测试断言直接读 JSON type Any = any; const TITLE = "霸总的替身新娘"; const BAIKE = "https://baike.example.com/item/bazong"; const VIDEO = "https://v.example.com/show/1"; const REDIR = "https://v.example.com/r/2"; const EVIL = "https://evil.example.net/x"; const SITE_PAGE = "https://pt.hhclub.example/details.php?id=1"; const POSTER = "https://img.example-cdn.com/p/1.jpg"; const OTHER = "https://baike.example.com/item/other"; const BAIKE_HTML = `${TITLE}_百科

${TITLE}

${TITLE},又名:替身娇妻。类型:都市、甜宠

首播时间:2024年3月1日

播出平台:红果短剧

共80集

剧情简介:落魄千金为救母亲替嫁给冷面总裁,两人在一次次误会中走近,最终揭开当年的家族阴谋。

`; const VIDEO_HTML = `${TITLE} 全80集

${TITLE} 全80集 2024 红果短剧独播

`; let braveCalls = 0; const pageHits: string[] = []; const mockFetch = (async (input: string | URL | Request) => { const url = new URL(typeof input === "string" ? input : input instanceof URL ? input.href : input.url); if (url.hostname === "api.search.brave.com") { braveCalls++; return Response.json({ web: { results: [ { title: `${TITLE} - 百科`, url: BAIKE, description: `${TITLE} 共80集 红果短剧` }, { title: `${TITLE} 种子`, url: SITE_PAGE, description: "站点资源" }, { title: "不相关网站", url: EVIL, description: TITLE }, { title: `${TITLE} 全集`, url: VIDEO, description: `${TITLE} 全80集` }, { title: `${TITLE} 跳转`, url: REDIR, description: TITLE }, { title: "另一部剧_百科", url: OTHER, description: "无关条目" }, ], }, }); } pageHits.push(url.href); if (url.href === BAIKE) return new Response(BAIKE_HTML, { headers: { "content-type": "text/html; charset=utf-8" } }); if (url.href === VIDEO) return new Response(VIDEO_HTML, { headers: { "content-type": "text/html; charset=utf-8" } }); if (url.href === REDIR) return new Response(null, { status: 302, headers: { location: EVIL } }); if (url.href === EVIL) return new Response("xx", { headers: { "content-type": "text/html" } }); return new Response("not found", { status: 404 }); }) as typeof fetch; const resolveHost = async () => ["93.184.216.34"]; /** fake 模型:照网页给值,简介是改写过的(不是原文),海报给一个网页里没有的地址(应被丢弃) */ let modelMode: "json" | "none" | "garbage" = "json"; const fakeComplete = async (_system: string, user: string): Promise => { if (modelMode === "none") return null; if (modelMode === "garbage") return { text: "抱歉,我无法完成", model: "fake", inputTokens: 10, outputTokens: 5 }; const prompt = JSON.parse(user) as { pages: { url: string }[] }; const baike = prompt.pages.find((p) => p.url === BAIKE)?.url ?? null; return { text: `\`\`\`json\n${JSON.stringify({ title: { value: TITLE, sourceUrl: baike }, aliases: [{ value: "替身娇妻", sourceUrl: baike }], year: { value: 2024, sourceUrl: baike }, platform: { value: "红果短剧", sourceUrl: baike }, episodeCount: { value: 80, sourceUrl: baike }, synopsis: { value: "一个替嫁千金与总裁从误会到相爱的故事。", sourceUrl: baike }, posterUrl: { value: "https://img.example-cdn.com/made-up.jpg", sourceUrl: baike }, genre: [ { value: "都市", sourceUrl: baike }, { value: "甜宠", sourceUrl: baike }, ], })}\n\`\`\``, model: "fake-extract", inputTokens: 1200, outputTokens: 300, }; }; describe.skipIf(!pg)("META-1 元数据补全", () => { let h: Harness; let siteId: string; let n = 0; const setSettings = async (key: string, value: unknown) => { await sql`insert into settings (key, value) values (${key}, ${JSON.stringify(value)}::jsonb) on conflict (key) do update set value = excluded.value`.execute(h.db); }; const poolEntry = async (title: string, query: Record = { title }) => { const r = await h.db .insertInto("source_release") .values({ site_id: siteId, site_release_id: String(700000 + ++n), title: `${title} 2024 1080p WEB-DL-HHWEB` }) .returningAll() .executeTakeFirstOrThrow(); const p = await sql<{ id: string; }>`insert into identity_candidate (source_release_id, reason, query, normalized_title, evidence_version) values (${r.id}, 'unmatched', ${JSON.stringify(query)}::jsonb, ${title}, 'v1') returning id`.execute(h.db); return { poolId: p.rows[0]!.id, releaseId: r.id }; }; const invoke = (input: Record, taskId = crypto.randomUUID()) => h.ctx.capabilities.invoke("metadata.enrich", input, { actorId: "user:admin", taskId, scopeIds: [], signal: new AbortController().signal, source: "web", }) as Promise; const actor = { type: "user" as const, id: "admin" }; beforeAll(async () => { h = await createHarness("enrich"); siteId = (await seedSite(h, { key: "hhclub", baseUrl: "https://pt.hhclub.example/" })).site.id; const registry = createCapabilityRegistry({ logger: h.ctx.logger, db: h.db, settings: h.ctx.settings }); createWebSearchModule({ fetch: mockFetch, resolveHost }).registerCapabilities?.(registry, h.ctx); registerMetadataCapabilities(registry, h.ctx, { complete: fakeComplete }); (h.ctx as { capabilities: unknown }).capabilities = registry; const ref = await h.secrets.create({ purpose: "other", plaintext: "BSA-enrich-test", label: "websearch" }); await setSettings("agent", { webSearch: { provider: "brave", apiKeySecretRef: ref, maxResults: 10 } }); await setSettings("identity", { enrichDailyBudget: 10, enrichDomains: ["baike.example.com", "v.example.com", "hhclub.example"], }); }); afterAll(async () => { await h?.close(); }); beforeEach(() => { modelMode = "json"; }); it("建议:字段带出处与证据等级;白名单外域名 / 已登记站点 / 跳转出白名单的网页不用;不写作品表", async () => { const { poolId } = await poolEntry(TITLE); const entitiesBefore = await h.db.selectFrom("media_entity").select("id").execute(); const out = await invoke({ poolEntryId: poolId }); expect(out).toMatchObject({ status: "suggested", cached: false, normalizedTitle: expect.any(String) }); const f = out.suggestion.fields; expect(out.suggestion.extractor).toBe("model"); expect(f.title).toMatchObject({ value: TITLE, sourceUrl: BAIKE, verbatim: true }); // 规则抽取在同一网页得到相同值 → 规则确认 expect(f.episodeCount).toMatchObject({ value: 80, evidenceLevel: "rule_confirmed", verbatim: true }); expect(f.year).toMatchObject({ value: 2024, evidenceLevel: "rule_confirmed" }); expect(f.platform).toMatchObject({ value: "红果短剧", evidenceLevel: "rule_confirmed" }); // 改写的简介不是原文 → 待复核、Agent 归纳 expect(f.synopsis).toMatchObject({ evidenceLevel: "needs_review", verbatim: false }); // 模型自己拼的海报地址丢弃 expect(f.posterUrl).toBeNull(); expect(f.aliases.map((a: Any) => a.value)).toEqual(["替身娇妻"]); expect(f.genre.map((g: Any) => g.value)).toEqual(["都市", "甜宠"]); const src = Object.fromEntries(out.suggestion.sources.map((s: Any) => [s.url, s])); expect(src[EVIL]).toMatchObject({ allowed: false, fetched: false, skipped: "not_allowed" }); // 已登记站点的域名即使在白名单里也不读 expect(src[SITE_PAGE]).toMatchObject({ allowed: false, fetched: false, skipped: "site" }); expect(src[BAIKE]).toMatchObject({ fetched: true }); expect(src[REDIR]).toMatchObject({ fetched: false, skipped: "not_allowed" }); // 白名单内但标题对不上:不读 expect(src[OTHER]).toMatchObject({ allowed: true, fetched: false, skipped: "irrelevant" }); expect(pageHits).not.toContain(OTHER); expect(pageHits).not.toContain(SITE_PAGE); expect(pageHits.filter((u) => u === EVIL)).toHaveLength(1); // 只有重定向那一跳,结果被丢弃 expect(out.usage).toMatchObject({ searchCalls: 1, fetchCalls: 3, inputTokens: 1200, outputTokens: 300 }); expect(out.budget).toEqual({ usedToday: 1, dailyLimit: 10 }); expect(out.notice).toContain("不是 TMDB"); // 不落作品表 expect(await h.db.selectFrom("media_entity").select("id").execute()).toHaveLength(entitiesBefore.length); const row = await h.db .selectFrom("metadata_enrichment") .selectAll() .where("id", "=", out.enrichmentId) .executeTakeFirstOrThrow(); expect(row).toMatchObject({ status: "suggested", trigger: "manual", identity_candidate_id: poolId }); }); it("同一标题 7 天内命中缓存:不再搜索、不计预算", async () => { const calls = braveCalls; const hits = pageHits.length; const out = await invoke({ title: TITLE }); expect(out.cached).toBe(true); expect(out.status).toBe("suggested"); expect(braveCalls).toBe(calls); expect(pageHits.length).toBe(hits); expect(out.budget.usedToday).toBe(1); }); it("没有可用模型 / 模型输出不是 JSON:按规则抽取", async () => { modelMode = "none"; const a = await invoke({ title: TITLE, fresh: true }); expect(a.suggestion.extractor).toBe("rule"); expect(a.suggestion.fields.episodeCount).toMatchObject({ value: 80, evidenceLevel: "rule_confirmed" }); // 规则抽取的海报取 og:image expect(a.suggestion.fields.posterUrl).toMatchObject({ value: POSTER, verbatim: true }); modelMode = "garbage"; const b = await invoke({ title: TITLE, fresh: true }); expect(b.suggestion.extractor).toBe("rule"); expect(b.usage.inputTokens).toBe(10); }); it("每日预算用完:拒绝新的补全(缓存仍可用)", async () => { const used = (await invoke({ title: TITLE })).budget.usedToday; await setSettings("identity", { enrichDailyBudget: used, enrichDomains: ["baike.example.com", "v.example.com"], }); await expect(invoke({ title: "另一部短剧" })).rejects.toMatchObject({ code: "budget" }); // HTTP 映射为 429 const { poolId } = await poolEntry("第三部短剧"); const res = await h.req(`/identity-pool/${poolId}/enrich`, { body: {} }); expect(res.status).toBe(429); // 命中缓存不受预算影响 expect((await invoke({ title: TITLE })).cached).toBe(true); await setSettings("identity", { enrichDailyBudget: 50, enrichDomains: ["baike.example.com", "v.example.com"] }); }); it("网页搜索未开启:不占预算,直接报前置条件", async () => { await setSettings("agent", { webSearch: { provider: "none" } }); const before = await h.db.selectFrom("metadata_enrichment").select("id").execute(); await expect(invoke({ title: "未开启时" })).rejects.toMatchObject({ code: "precondition" }); expect(await h.db.selectFrom("metadata_enrichment").select("id").execute()).toHaveLength(before.length); const ref = await h.secrets.create({ purpose: "other", plaintext: "BSA-enrich-test", label: "websearch-2" }); await setSettings("agent", { webSearch: { provider: "brave", apiKeySecretRef: ref, maxResults: 10 } }); }); it("应用:建临时作品 + 季集单元 + 字段级元数据(web / agent + 证据)+ 别名,关联待识别资源;同名去重", async () => { const { poolId, releaseId } = await poolEntry(TITLE); const s = await invoke({ poolEntryId: poolId, fresh: true }); // 应用只接受控制台会话:API Key 被拒 const denied = await h.req(`/identity-pool/${poolId}/enrich/apply`, { body: { enrichmentId: s.enrichmentId } }); expect(denied.status).toBe(403); const r = await applyEnrichment(h.rt, poolId, { enrichmentId: s.enrichmentId, lock: false }, actor); expect(r).toMatchObject({ created: true, assigned: true, contentUnits: 81 }); const e = await h.db .selectFrom("media_entity") .selectAll() .where("id", "=", r.mediaEntityId) .executeTakeFirstOrThrow(); expect(e).toMatchObject({ status: "provisional", kind: "series", title: TITLE, year: 2024 }); const meta = e.metadata as Any; expect(meta.title).toMatchObject({ value: TITLE, source: "web" }); expect(meta.overview).toMatchObject({ source: "agent" }); // 改写的简介标为 Agent 生成 expect(meta.episodeCount).toMatchObject({ value: 80, source: "web" }); expect(meta.genres.value).toEqual(["都市", "甜宠"]); const fields = await h.db.selectFrom("metadata_field").selectAll().where("media_entity_id", "=", e.id).execute(); expect(fields.every((x) => x.source === "web" || x.source === "agent")).toBe(true); expect(fields.find((x) => x.field === "episodeCount")?.evidence).toMatchObject({ sourceUrl: BAIKE, evidenceLevel: "rule_confirmed", enrichmentId: s.enrichmentId, }); const ids = await h.db.selectFrom("external_identity").select("id").where("media_entity_id", "=", e.id).execute(); expect(ids).toHaveLength(0); // 不冒充 TMDB,也不编外部身份 const aliases = await h.db.selectFrom("title_alias").selectAll().where("media_entity_id", "=", e.id).execute(); expect(aliases.map((a) => a.alias).sort()).toEqual([TITLE, "替身娇妻"].sort()); expect(aliases.every((a) => a.source === "agent")).toBe(true); const rel = await h.db .selectFrom("source_release") .selectAll() .where("id", "=", releaseId) .executeTakeFirstOrThrow(); expect(rel.media_entity_id).toBe(e.id); const pool = await h.db .selectFrom("identity_candidate") .selectAll() .where("id", "=", poolId) .executeTakeFirstOrThrow(); expect(pool.status).toBe("resolved"); // 作品详情接口带字段证据 const d = await json(await h.req(`/media-entities/${e.id}/detail`)); expect(d.body.data.metadataFields.find((x: Any) => x.field === "year").evidence.sourceUrl).toBe(BAIKE); // 留给身份巡检补 TMDB 映射 expect((await provisionalEntitiesForTmdbReview(h.db)).map((x) => x.mediaEntityId)).toContain(e.id); // 同名去重:另一条同标题资源应用后并入同一作品 const second = await poolEntry(TITLE); const s2 = await invoke({ poolEntryId: second.poolId }); expect(s2.cached).toBe(true); expect(s2.existing.map((x: Any) => x.mediaEntityId)).toContain(e.id); const r2 = await applyEnrichment(h.rt, second.poolId, { enrichmentId: s2.enrichmentId, lock: true }, actor); expect(r2).toMatchObject({ created: false, mediaEntityId: e.id, assigned: true }); const locked = await h.db .selectFrom("media_entity") .select("field_locks") .where("id", "=", e.id) .executeTakeFirstOrThrow(); expect(locked.field_locks).toContain("episodeCount"); const count = await h.db.selectFrom("media_entity").select("id").where("title", "=", TITLE).execute(); expect(count).toHaveLength(1); }); it("自动触发:只在订阅 / 规则命中且 TMDB 无结果、设置打开时入队一次", async () => { const { poolId, releaseId } = await poolEntry("自动补全短剧"); await setSettings("identity", { autoEnrich: false, enrichDailyBudget: 50, enrichDomains: ["baike.example.com"] }); expect(await maybeAutoEnrich(h.rt, releaseId, "no_match", "new_title_rule")).toBe(false); await setSettings("identity", { autoEnrich: true, enrichDailyBudget: 50, enrichDomains: ["baike.example.com"] }); expect(await maybeAutoEnrich(h.rt, releaseId, "unmatched", null)).toBe(false); expect(await maybeAutoEnrich(h.rt, releaseId, "no_match", "manual")).toBe(false); expect(await maybeAutoEnrich(h.rt, releaseId, "no_match", "new_title_rule")).toBe(true); await maybeAutoEnrich(h.rt, releaseId, "no_match", "tracked_alias"); const q = h.engine.enqueued.filter((x) => x.task === AUTO_ENRICH_TASK); expect(q).toHaveLength(1); expect(q[0]!.input).toEqual({ poolEntryId: poolId }); }); it("规则抽取只用标题对得上的网页", () => { const r = ruleExtract({ title: "别的剧" }, [ { url: BAIKE, finalUrl: BAIKE, title: `${TITLE}_百科`, text: `${TITLE} 共80集`, image: POSTER }, ]); expect(r.title).toBeNull(); expect(r.episodeCount).toBeNull(); }); });