UHD_Media_Tool/apps/server/test/sources-identity/enrich-sources.test.ts

import { readFileSync } from "node:fs";
import { normalizeBraveResults } from "@umt/adapters";
import { DEFAULT_ENRICH_DOMAINS, type EnrichUsage } from "@umt/contracts";
import { describe, expect, it } from "vitest";
import { type EnrichDeps, runEnrichPipeline } from "../../src/modules/sources-identity/service/enrich.ts";
import {
  type Candidate,
  episodeRun,
  extractFromPage,
  extractFromSnippet,
  looksBlocked,
  mergeCandidates,
  robotsAllows,
  siteFilterDomains,
  sourceKindOf,
} from "../../src/modules/sources-identity/service/enrich-sources.ts";

// META-2 补全来源与抽取器(纯函数,不连库):来源分类、robots.txt、登录墙、各来源抽取器、按置信度合并;
// 再用真机验证时录制的 Brave 响应(test/fixtures/enrich,已脱敏、不含 Key)回放用户库里那 4 条短剧,锁住命中结果。

const FIX = new URL("../fixtures/enrich/", import.meta.url);
const read = (f: string) => readFileSync(new URL(f, FIX), "utf8");
const brave = JSON.parse(read("brave-shortdrama.json")) as { responses: { q: string; body: unknown }[] };
const hongguo = JSON.parse(read("hongguo-detail.json")) as {
  url: string;
  finalUrl: string;
  title: string;
  text: string;
  image: string | null;
};

describe("来源分类", () => {
  it("百科 / 平台 / 豆瓣 / 白名单其它 / 摘要", () => {
    const wl = DEFAULT_ENRICH_DOMAINS;
    expect(sourceKindOf("baike.baidu.com", wl)).toBe("encyclopedia");
    expect(sourceKindOf("zh.wikipedia.org", wl)).toBe("encyclopedia");
    expect(sourceKindOf("www.baike.com", wl)).toBe("encyclopedia");
    expect(sourceKindOf("hongguoduanju.com", wl)).toBe("platform");
    expect(sourceKindOf("www.kuaikaw.cn", wl)).toBe("platform");
    expect(sourceKindOf("m.douban.com", wl)).toBe("douban");
    expect(sourceKindOf("mydramalist.com", wl)).toBe("platform");
    expect(sourceKindOf("baike.example.com", ["baike.example.com"])).toBe("other");
    expect(sourceKindOf("m.qbcdj.com", wl)).toBe("snippet");
  });
  it("第二次搜索的 site: 限定:百科、豆瓣、平台在前", () => {
    expect(siteFilterDomains(DEFAULT_ENRICH_DOMAINS, 6)).toEqual([
      "baike.baidu.com",
      "wikipedia.org",
      "baike.sogou.com",
      "baike.com",
      "douban.com",
      "hongguoduanju.com",
    ]);
  });
});

describe("robots.txt 与登录墙", () => {
  const hg = read("robots-hongguo.txt");
  it("红果:/series/ 不允许,/detail 允许;百度百科对 * 全站不允许", () => {
    expect(robotsAllows(hg, "https://hongguoduanju.com/series/chunshanxuan25889048")).toBe(false);
    expect(robotsAllows(hg, "https://hongguoduanju.com/detail?series_id=7516417700743105561")).toBe(true);
    // /query/ 只对 Googlebot 不允许
    expect(robotsAllows(hg, "https://hongguoduanju.com/query/abc")).toBe(true);
    expect(robotsAllows(hg, "https://hongguoduanju.com/category?x=1&gender=1")).toBe(false);
    expect(robotsAllows(read("robots-baike.txt"), "https://baike.baidu.com/item/x")).toBe(false);
  });
  it("最长匹配优先,同长 Allow 胜;指名本程序的组优先于 *", () => {
    const txt = "User-agent: *\nDisallow: /a\nAllow: /a/b\n\nUser-agent: umt-websearch\nDisallow: /private$\n";
    expect(robotsAllows(txt, "https://x.test/a/b/c")).toBe(true);
    expect(robotsAllows(txt, "https://x.test/private")).toBe(false);
    expect(robotsAllows(txt, "https://x.test/private/ok")).toBe(true);
    expect(robotsAllows(txt, "https://x.test/a/c", "other-bot")).toBe(false);
    expect(robotsAllows("", "https://x.test/any")).toBe(true);
  });
  it("登录墙 / 验证码页", () => {
    expect(looksBlocked({ title: "安全验证", text: "请完成下方验证后继续" })).toBe(true);
    expect(looksBlocked({ title: "x", text: "请先登录" })).toBe(true);
    expect(looksBlocked({ title: hongguo.title, text: hongguo.text })).toBe(false);
  });
});

describe("抽取器", () => {
  it("红果作品页:只看本作品主体(相关短剧的「全63集」不算),选集列表得出集数,演员、简介、平台", () => {
    const c = extractFromPage("当我凝视深渊", {
      url: hongguo.url,
      host: "hongguoduanju.com",
      kind: "platform",
      title: hongguo.title,
      text: hongguo.text,
      image: hongguo.image,
    });
    const one = (f: string) => c.filter((x) => x.field === f).map((x) => x.value);
    expect(one("title")).toEqual(["当我凝视深渊"]);
    expect(one("episodeCount")).toEqual([72]);
    expect(one("cast")).toEqual(["盛以婕", "于轩鸿灏", "宋效如", "杨晓彤", "王希如", "吴大锤", "孟岳"]);
    expect(String(one("synopsis")[0])).toMatch(/^风投女王姜然/);
    expect(one("platform")).toEqual(["红果短剧"]);
    expect(c.every((x) => x.kind === "platform" && x.confidence > 0.6)).toBe(true);
    // 标题对不上的网页不抽
    expect(extractFromPage("别的剧", { ...hongguo, host: "hongguoduanju.com", kind: "platform" })).toEqual([]);
  });
  it("选集列表:从 1 开始连续的最长一段", () => {
    expect(episodeRun("播放\n1\n2\n3\n4\n热评\n1\n2")).toBe(4);
    expect(episodeRun("1\n2")).toBeNull();
  });
  it("豆瓣摘要:导演 / 主演 / 类型 / 年份", () => {
    const c = extractFromSnippet("某短剧", {
      url: "https://movie.douban.com/subject/1/",
      host: "movie.douban.com",
      kind: "douban",
      title: "某短剧 (豆瓣)",
      snippet: "导演: 朱心亦 主演: 付妤舒 / 刘念 类型: 爱情 / 短片 制片国家/地区: 中国大陆 年份: 2025 集数: 24",
    });
    const v = (f: string) => c.filter((x) => x.field === f).map((x) => x.value);
    expect(v("cast")).toEqual(["付妤舒", "刘念"]);
    expect(v("year")).toEqual([2025]);
    expect(v("episodeCount")).toEqual([24]);
    expect(v("genre")).toEqual(["爱情", "短片"]);
  });
  it("百科摘要句式:执导 / 主演 / 共 N 集 / 于某日在某平台首播 / 又名", () => {
    const c = extractFromSnippet("夫人她专治不服", {
      url: "https://baike.baidu.com/item/x",
      host: "baike.baidu.com",
      kind: "encyclopedia",
      title: "夫人她专治不服_百度百科",
      snippet:
        "《夫人她专治不服》是由周茜茜执导，王小耶、丁丁编剧，李柯以、王培延主演的中国大陆剧情爱情短剧，全剧共69集，每集时长2分钟，于2025年8月15日在爱奇艺首播，又名“哄不了”",
    });
    const v = (f: string) => c.filter((x) => x.field === f).map((x) => x.value);
    expect(v("cast")).toEqual(["李柯以", "王培延"]);
    expect(v("episodeCount")).toEqual([69]);
    expect(v("year")).toEqual([2025]);
    expect(v("platform")).toEqual(["爱奇艺"]);
    expect(v("aliases")).toEqual(["哄不了"]);
    expect(v("title")).toEqual(["夫人她专治不服"]);
  });
  it("摘要模式:结果标题对不上不用;网盘字样不当主演;拼音别名不要", () => {
    const base = { url: "https://a.test/1", host: "a.test", kind: "snippet" as const };
    expect(extractFromSnippet("京夜喜你", { ...base, title: "京夜沉沦", snippet: "京夜喜你 共41集" })).toEqual([]);
    const c = extractFromSnippet("京夜喜你", {
      ...base,
      title: "京夜喜你（41集）祁乐&吴思君 (2026)",
      snippet:
        "《京夜喜你》都市情感短剧，共41集，主演：祁乐＆吴思君，2026年热播，提供夸克网盘、百度网盘免费下载。 影片别名：jingyexini · 类型：都市日常",
    });
    const v = (f: string) => c.filter((x) => x.field === f).map((x) => x.value);
    expect(v("cast")).toEqual(["祁乐", "吴思君"]);
    expect(v("episodeCount")).toEqual([41]);
    expect(v("year")).toEqual([2026]);
    expect(v("aliases")).toEqual([]);
    // 其它网站的摘要:基础置信度 0.35(类型按 0.8 折算)
    expect(c.filter((x) => x.field !== "genre").every((x) => x.confidence === 0.35)).toBe(true);
  });
});

describe("合并", () => {
  const cand = (host: string, value: number, confidence: number, kind: Candidate["kind"] = "snippet"): Candidate => ({
    field: "episodeCount",
    value,
    sourceUrl: `https://${host}/x`,
    host,
    kind,
    confidence,
    verbatim: true,
    by: "rule",
  });
  it("不同网站给出同一值按 noisy-OR 叠加;同一网站只算一次;取叠加后最高的一组", () => {
    const m = mergeCandidates([
      cand("a.test", 30, 0.35),
      cand("b.test", 75, 0.35),
      cand("c.test", 75, 0.35),
      cand("b.test", 75, 0.35),
    ]);
    expect(m.episodeCount).toMatchObject({ value: 75, hosts: 2, confidence: 0.58, evidenceLevel: "multi_evidence" });
  });
  it("白名单网站原文里的规则抽取值 = 规则确认;低于 0.3 的丢弃", () => {
    const m = mergeCandidates([cand("hongguoduanju.com", 72, 0.8, "platform"), cand("a.test", 63, 0.35)]);
    expect(m.episodeCount).toMatchObject({ value: 72, evidenceLevel: "rule_confirmed", sourceKind: "platform" });
    expect(mergeCandidates([cand("a.test", 5, 0.2)]).episodeCount).toBeNull();
  });
});

describe("真机录制回放:用户库里的 4 条短剧(2026-09-30)", () => {
  const deps = (n: number): EnrichDeps => ({
    async search(query) {
      const r = brave.responses.find((x) => x.q === query);
      if (!r) throw new Error(`fixture 里没有这次搜索:${query}`);
      return normalizeBraveResults(r.body)
        .slice(0, n)
        .map((x) => ({ title: x.title || null, url: x.url, snippet: x.snippet || null }));
    },
    async fetch(url) {
      throw new Error(`回放不读网页:${url}`);
    },
    async complete() {
      return null;
    },
    async robotsAllowed() {
      return true;
    },
  });
  const run = async (title: string, n: number, snippetAllSites = true) => {
    const usage: EnrichUsage = { searchCalls: 0, fetchCalls: 0, inputTokens: 0, outputTokens: 0 };
    const r = await runEnrichPipeline(
      deps(n),
      {
        q: { title, year: null, hints: null },
        settings: { domains: DEFAULT_ENRICH_DOMAINS, snippetAllSites },
        sites: ["hhanclub.net", "example.com"],
        id: "fixture",
        logger: { warn: () => undefined },
      },
      usage,
    );
    return { r, usage, f: r.suggestion.fields };
  };

  it("每次 5 条结果(用户的网页搜索设置):4 条都从搜索摘要补到标题、年份,3 条有集数,白名单网站都没有结果", async () => {
    const a = await run("我被亲戚误会破产了", 5);
    expect(a.r.found).toBe(true);
    expect(a.f.title?.value).toBe("我被亲戚误会破产了");
    expect(a.f.year?.value).toBe(2026);
    expect(a.f.episodeCount).toMatchObject({ value: 75, sourceKind: "snippet", evidenceLevel: "needs_review" });
    expect(a.f.synopsis?.value).toMatch(/^周远公司上市后回乡给爷爷祝寿/);
    expect(a.f.cast.map((c) => c.value)).toEqual(["刘俊达", "刘诣文"]);

    const b = await run("重生60 我有情报系统", 5);
    expect(b.f.title?.value).toBe("重生60，我有情报系统"); // 用网页上的写法(带逗号)
    expect(b.f.year).toMatchObject({ value: 2026, evidenceLevel: "multi_evidence" });
    expect(b.f.episodeCount?.value).toBe(80);
    expect(b.f.cast.map((c) => c.value)).toEqual(["龙泽", "冯韵霏"]);
    expect(b.f.cast[0]!.hosts).toBe(3);

    const c = await run("京夜喜你", 5);
    expect(c.f.episodeCount?.value).toBe(41);
    expect(c.f.cast.map((x) => x.value)).toEqual(["祁乐", "吴思君"]);

    const d = await run("春城恋曲", 5);
    expect(d.f.year?.value).toBe(2026);
    expect(d.f.episodeCount).toBeNull();
    expect(d.f.cast.map((x) => x.value)).toEqual(["宋喃", "郑天龙"]);

    for (const x of [a, b, c, d]) {
      expect(x.usage.searchCalls).toBe(2); // 摘要不够「集数 + 年份各两个网站」,做了第二次 site: 限定搜索
      expect(x.usage.fetchCalls).toBe(0);
      const att = Object.fromEntries(x.usage.attempts!.map((t) => [t.kind, t]));
      expect(att.snippet!.hit).toBe(true);
      expect(att.encyclopedia!.hit || att.platform!.hit || att.douban!.hit).toBe(false);
      // 聚合站只用摘要,不读网页
      expect(x.r.suggestion.sources.filter((s) => s.kind === "snippet").every((s) => !s.fetched)).toBe(true);
    }
  });

  it("每次 10 条结果:证据更多;《京夜喜你》摘要够用,只搜一次", async () => {
    const c = await run("京夜喜你", 10);
    expect(c.usage.searchCalls).toBe(1);
    expect(c.f.episodeCount).toMatchObject({ value: 41, hosts: 2, evidenceLevel: "multi_evidence" });
    expect(c.f.synopsis?.value).toMatch(/^京夜喜你以旧日救赎重逢心动/);
    const d = await run("春城恋曲", 10);
    expect(d.f.cast.map((x) => x.hosts)).toEqual([4, 4]);
  });

  it("关掉「也用其它网站的搜索摘要」:4 条都抽不到字段(白名单网站只有七成字对得上的榜单 / 首页,不读)", async () => {
    for (const t of ["我被亲戚误会破产了", "重生60 我有情报系统", "京夜喜你", "春城恋曲"]) {
      const x = await run(t, 5, false);
      const f = x.f;
      expect([f.title, f.year, f.episodeCount, f.synopsis, f.platform].every((v) => v === null)).toBe(true);
      expect(f.cast).toEqual([]);
      expect(x.usage.fetchCalls).toBe(0);
    }
  });
});