Merge branch 'meta2'
Co-Authored-By: Claude Fable 5.1 <[email protected]> # Conflicts: # CHANGELOG.md
This commit is contained in:
root committed
commit
6f5371be2d
23 files changed
+2615
-323
No files matched your search
@@ -0,0 +1,272 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { normalizeBraveResults } from "@umt/adapters";
|
||||
import { DEFAULT_ENRICH_DOMAINS, type EnrichUsage } from "@umt/contracts";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { type EnrichDeps, runEnrichPipeline } from "../../src/modules/sources-identity/service/enrich.ts";
|
||||
import {
|
||||
type Candidate,
|
||||
episodeRun,
|
||||
extractFromPage,
|
||||
extractFromSnippet,
|
||||
looksBlocked,
|
||||
mergeCandidates,
|
||||
robotsAllows,
|
||||
siteFilterDomains,
|
||||
sourceKindOf,
|
||||
} from "../../src/modules/sources-identity/service/enrich-sources.ts";
|
||||
|
||||
// META-2 补全来源与抽取器(纯函数,不连库):来源分类、robots.txt、登录墙、各来源抽取器、按置信度合并;
|
||||
// 再用真机验证时录制的 Brave 响应(test/fixtures/enrich,已脱敏、不含 Key)回放用户库里那 4 条短剧,锁住命中结果。
|
||||
|
||||
const FIX = new URL("../fixtures/enrich/", import.meta.url);
|
||||
const read = (f: string) => readFileSync(new URL(f, FIX), "utf8");
|
||||
const brave = JSON.parse(read("brave-shortdrama.json")) as { responses: { q: string; body: unknown }[] };
|
||||
const hongguo = JSON.parse(read("hongguo-detail.json")) as {
|
||||
url: string;
|
||||
finalUrl: string;
|
||||
title: string;
|
||||
text: string;
|
||||
image: string | null;
|
||||
};
|
||||
|
||||
describe("来源分类", () => {
|
||||
it("百科 / 平台 / 豆瓣 / 白名单其它 / 摘要", () => {
|
||||
const wl = DEFAULT_ENRICH_DOMAINS;
|
||||
expect(sourceKindOf("baike.baidu.com", wl)).toBe("encyclopedia");
|
||||
expect(sourceKindOf("zh.wikipedia.org", wl)).toBe("encyclopedia");
|
||||
expect(sourceKindOf("www.baike.com", wl)).toBe("encyclopedia");
|
||||
expect(sourceKindOf("hongguoduanju.com", wl)).toBe("platform");
|
||||
expect(sourceKindOf("www.kuaikaw.cn", wl)).toBe("platform");
|
||||
expect(sourceKindOf("m.douban.com", wl)).toBe("douban");
|
||||
expect(sourceKindOf("mydramalist.com", wl)).toBe("platform");
|
||||
expect(sourceKindOf("baike.example.com", ["baike.example.com"])).toBe("other");
|
||||
expect(sourceKindOf("m.qbcdj.com", wl)).toBe("snippet");
|
||||
});
|
||||
it("第二次搜索的 site: 限定:百科、豆瓣、平台在前", () => {
|
||||
expect(siteFilterDomains(DEFAULT_ENRICH_DOMAINS, 6)).toEqual([
|
||||
"baike.baidu.com",
|
||||
"wikipedia.org",
|
||||
"baike.sogou.com",
|
||||
"baike.com",
|
||||
"douban.com",
|
||||
"hongguoduanju.com",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("robots.txt 与登录墙", () => {
|
||||
const hg = read("robots-hongguo.txt");
|
||||
it("红果:/series/ 不允许,/detail 允许;百度百科对 * 全站不允许", () => {
|
||||
expect(robotsAllows(hg, "https://hongguoduanju.com/series/chunshanxuan25889048")).toBe(false);
|
||||
expect(robotsAllows(hg, "https://hongguoduanju.com/detail?series_id=7516417700743105561")).toBe(true);
|
||||
// /query/ 只对 Googlebot 不允许
|
||||
expect(robotsAllows(hg, "https://hongguoduanju.com/query/abc")).toBe(true);
|
||||
expect(robotsAllows(hg, "https://hongguoduanju.com/category?x=1&gender=1")).toBe(false);
|
||||
expect(robotsAllows(read("robots-baike.txt"), "https://baike.baidu.com/item/x")).toBe(false);
|
||||
});
|
||||
it("最长匹配优先,同长 Allow 胜;指名本程序的组优先于 *", () => {
|
||||
const txt = "User-agent: *\nDisallow: /a\nAllow: /a/b\n\nUser-agent: umt-websearch\nDisallow: /private$\n";
|
||||
expect(robotsAllows(txt, "https://x.test/a/b/c")).toBe(true);
|
||||
expect(robotsAllows(txt, "https://x.test/private")).toBe(false);
|
||||
expect(robotsAllows(txt, "https://x.test/private/ok")).toBe(true);
|
||||
expect(robotsAllows(txt, "https://x.test/a/c", "other-bot")).toBe(false);
|
||||
expect(robotsAllows("", "https://x.test/any")).toBe(true);
|
||||
});
|
||||
it("登录墙 / 验证码页", () => {
|
||||
expect(looksBlocked({ title: "安全验证", text: "请完成下方验证后继续" })).toBe(true);
|
||||
expect(looksBlocked({ title: "x", text: "请先登录" })).toBe(true);
|
||||
expect(looksBlocked({ title: hongguo.title, text: hongguo.text })).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("抽取器", () => {
|
||||
it("红果作品页:只看本作品主体(相关短剧的「全63集」不算),选集列表得出集数,演员、简介、平台", () => {
|
||||
const c = extractFromPage("当我凝视深渊", {
|
||||
url: hongguo.url,
|
||||
host: "hongguoduanju.com",
|
||||
kind: "platform",
|
||||
title: hongguo.title,
|
||||
text: hongguo.text,
|
||||
image: hongguo.image,
|
||||
});
|
||||
const one = (f: string) => c.filter((x) => x.field === f).map((x) => x.value);
|
||||
expect(one("title")).toEqual(["当我凝视深渊"]);
|
||||
expect(one("episodeCount")).toEqual([72]);
|
||||
expect(one("cast")).toEqual(["盛以婕", "于轩鸿灏", "宋效如", "杨晓彤", "王希如", "吴大锤", "孟岳"]);
|
||||
expect(String(one("synopsis")[0])).toMatch(/^风投女王姜然/);
|
||||
expect(one("platform")).toEqual(["红果短剧"]);
|
||||
expect(c.every((x) => x.kind === "platform" && x.confidence > 0.6)).toBe(true);
|
||||
// 标题对不上的网页不抽
|
||||
expect(extractFromPage("别的剧", { ...hongguo, host: "hongguoduanju.com", kind: "platform" })).toEqual([]);
|
||||
});
|
||||
it("选集列表:从 1 开始连续的最长一段", () => {
|
||||
expect(episodeRun("播放\n1\n2\n3\n4\n热评\n1\n2")).toBe(4);
|
||||
expect(episodeRun("1\n2")).toBeNull();
|
||||
});
|
||||
it("豆瓣摘要:导演 / 主演 / 类型 / 年份", () => {
|
||||
const c = extractFromSnippet("某短剧", {
|
||||
url: "https://movie.douban.com/subject/1/",
|
||||
host: "movie.douban.com",
|
||||
kind: "douban",
|
||||
title: "某短剧 (豆瓣)",
|
||||
snippet: "导演: 朱心亦 主演: 付妤舒 / 刘念 类型: 爱情 / 短片 制片国家/地区: 中国大陆 年份: 2025 集数: 24",
|
||||
});
|
||||
const v = (f: string) => c.filter((x) => x.field === f).map((x) => x.value);
|
||||
expect(v("cast")).toEqual(["付妤舒", "刘念"]);
|
||||
expect(v("year")).toEqual([2025]);
|
||||
expect(v("episodeCount")).toEqual([24]);
|
||||
expect(v("genre")).toEqual(["爱情", "短片"]);
|
||||
});
|
||||
it("百科摘要句式:执导 / 主演 / 共 N 集 / 于某日在某平台首播 / 又名", () => {
|
||||
const c = extractFromSnippet("夫人她专治不服", {
|
||||
url: "https://baike.baidu.com/item/x",
|
||||
host: "baike.baidu.com",
|
||||
kind: "encyclopedia",
|
||||
title: "夫人她专治不服_百度百科",
|
||||
snippet:
|
||||
"《夫人她专治不服》是由周茜茜执导，王小耶、丁丁编剧，李柯以、王培延主演的中国大陆剧情爱情短剧，全剧共69集，每集时长2分钟，于2025年8月15日在爱奇艺首播，又名“哄不了”",
|
||||
});
|
||||
const v = (f: string) => c.filter((x) => x.field === f).map((x) => x.value);
|
||||
expect(v("cast")).toEqual(["李柯以", "王培延"]);
|
||||
expect(v("episodeCount")).toEqual([69]);
|
||||
expect(v("year")).toEqual([2025]);
|
||||
expect(v("platform")).toEqual(["爱奇艺"]);
|
||||
expect(v("aliases")).toEqual(["哄不了"]);
|
||||
expect(v("title")).toEqual(["夫人她专治不服"]);
|
||||
});
|
||||
it("摘要模式:结果标题对不上不用;网盘字样不当主演;拼音别名不要", () => {
|
||||
const base = { url: "https://a.test/1", host: "a.test", kind: "snippet" as const };
|
||||
expect(extractFromSnippet("京夜喜你", { ...base, title: "京夜沉沦", snippet: "京夜喜你 共41集" })).toEqual([]);
|
||||
const c = extractFromSnippet("京夜喜你", {
|
||||
...base,
|
||||
title: "京夜喜你（41集）祁乐&吴思君 (2026)",
|
||||
snippet:
|
||||
"《京夜喜你》都市情感短剧，共41集，主演：祁乐＆吴思君，2026年热播，提供夸克网盘、百度网盘免费下载。 影片别名：jingyexini · 类型：都市日常",
|
||||
});
|
||||
const v = (f: string) => c.filter((x) => x.field === f).map((x) => x.value);
|
||||
expect(v("cast")).toEqual(["祁乐", "吴思君"]);
|
||||
expect(v("episodeCount")).toEqual([41]);
|
||||
expect(v("year")).toEqual([2026]);
|
||||
expect(v("aliases")).toEqual([]);
|
||||
// 其它网站的摘要:基础置信度 0.35(类型按 0.8 折算)
|
||||
expect(c.filter((x) => x.field !== "genre").every((x) => x.confidence === 0.35)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("合并", () => {
|
||||
const cand = (host: string, value: number, confidence: number, kind: Candidate["kind"] = "snippet"): Candidate => ({
|
||||
field: "episodeCount",
|
||||
value,
|
||||
sourceUrl: `https://${host}/x`,
|
||||
host,
|
||||
kind,
|
||||
confidence,
|
||||
verbatim: true,
|
||||
by: "rule",
|
||||
});
|
||||
it("不同网站给出同一值按 noisy-OR 叠加;同一网站只算一次;取叠加后最高的一组", () => {
|
||||
const m = mergeCandidates([
|
||||
cand("a.test", 30, 0.35),
|
||||
cand("b.test", 75, 0.35),
|
||||
cand("c.test", 75, 0.35),
|
||||
cand("b.test", 75, 0.35),
|
||||
]);
|
||||
expect(m.episodeCount).toMatchObject({ value: 75, hosts: 2, confidence: 0.58, evidenceLevel: "multi_evidence" });
|
||||
});
|
||||
it("白名单网站原文里的规则抽取值 = 规则确认;低于 0.3 的丢弃", () => {
|
||||
const m = mergeCandidates([cand("hongguoduanju.com", 72, 0.8, "platform"), cand("a.test", 63, 0.35)]);
|
||||
expect(m.episodeCount).toMatchObject({ value: 72, evidenceLevel: "rule_confirmed", sourceKind: "platform" });
|
||||
expect(mergeCandidates([cand("a.test", 5, 0.2)]).episodeCount).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("真机录制回放:用户库里的 4 条短剧(2026-09-30)", () => {
|
||||
const deps = (n: number): EnrichDeps => ({
|
||||
async search(query) {
|
||||
const r = brave.responses.find((x) => x.q === query);
|
||||
if (!r) throw new Error(`fixture 里没有这次搜索:${query}`);
|
||||
return normalizeBraveResults(r.body)
|
||||
.slice(0, n)
|
||||
.map((x) => ({ title: x.title || null, url: x.url, snippet: x.snippet || null }));
|
||||
},
|
||||
async fetch(url) {
|
||||
throw new Error(`回放不读网页:${url}`);
|
||||
},
|
||||
async complete() {
|
||||
return null;
|
||||
},
|
||||
async robotsAllowed() {
|
||||
return true;
|
||||
},
|
||||
});
|
||||
const run = async (title: string, n: number, snippetAllSites = true) => {
|
||||
const usage: EnrichUsage = { searchCalls: 0, fetchCalls: 0, inputTokens: 0, outputTokens: 0 };
|
||||
const r = await runEnrichPipeline(
|
||||
deps(n),
|
||||
{
|
||||
q: { title, year: null, hints: null },
|
||||
settings: { domains: DEFAULT_ENRICH_DOMAINS, snippetAllSites },
|
||||
sites: ["hhanclub.net", "example.com"],
|
||||
id: "fixture",
|
||||
logger: { warn: () => undefined },
|
||||
},
|
||||
usage,
|
||||
);
|
||||
return { r, usage, f: r.suggestion.fields };
|
||||
};
|
||||
|
||||
it("每次 5 条结果(用户的网页搜索设置):4 条都从搜索摘要补到标题、年份,3 条有集数,白名单网站都没有结果", async () => {
|
||||
const a = await run("我被亲戚误会破产了", 5);
|
||||
expect(a.r.found).toBe(true);
|
||||
expect(a.f.title?.value).toBe("我被亲戚误会破产了");
|
||||
expect(a.f.year?.value).toBe(2026);
|
||||
expect(a.f.episodeCount).toMatchObject({ value: 75, sourceKind: "snippet", evidenceLevel: "needs_review" });
|
||||
expect(a.f.synopsis?.value).toMatch(/^周远公司上市后回乡给爷爷祝寿/);
|
||||
expect(a.f.cast.map((c) => c.value)).toEqual(["刘俊达", "刘诣文"]);
|
||||
|
||||
const b = await run("重生60 我有情报系统", 5);
|
||||
expect(b.f.title?.value).toBe("重生60，我有情报系统"); // 用网页上的写法(带逗号)
|
||||
expect(b.f.year).toMatchObject({ value: 2026, evidenceLevel: "multi_evidence" });
|
||||
expect(b.f.episodeCount?.value).toBe(80);
|
||||
expect(b.f.cast.map((c) => c.value)).toEqual(["龙泽", "冯韵霏"]);
|
||||
expect(b.f.cast[0]!.hosts).toBe(3);
|
||||
|
||||
const c = await run("京夜喜你", 5);
|
||||
expect(c.f.episodeCount?.value).toBe(41);
|
||||
expect(c.f.cast.map((x) => x.value)).toEqual(["祁乐", "吴思君"]);
|
||||
|
||||
const d = await run("春城恋曲", 5);
|
||||
expect(d.f.year?.value).toBe(2026);
|
||||
expect(d.f.episodeCount).toBeNull();
|
||||
expect(d.f.cast.map((x) => x.value)).toEqual(["宋喃", "郑天龙"]);
|
||||
|
||||
for (const x of [a, b, c, d]) {
|
||||
expect(x.usage.searchCalls).toBe(2); // 摘要不够「集数 + 年份各两个网站」,做了第二次 site: 限定搜索
|
||||
expect(x.usage.fetchCalls).toBe(0);
|
||||
const att = Object.fromEntries(x.usage.attempts!.map((t) => [t.kind, t]));
|
||||
expect(att.snippet!.hit).toBe(true);
|
||||
expect(att.encyclopedia!.hit || att.platform!.hit || att.douban!.hit).toBe(false);
|
||||
// 聚合站只用摘要,不读网页
|
||||
expect(x.r.suggestion.sources.filter((s) => s.kind === "snippet").every((s) => !s.fetched)).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("每次 10 条结果:证据更多;《京夜喜你》摘要够用,只搜一次", async () => {
|
||||
const c = await run("京夜喜你", 10);
|
||||
expect(c.usage.searchCalls).toBe(1);
|
||||
expect(c.f.episodeCount).toMatchObject({ value: 41, hosts: 2, evidenceLevel: "multi_evidence" });
|
||||
expect(c.f.synopsis?.value).toMatch(/^京夜喜你以旧日救赎重逢心动/);
|
||||
const d = await run("春城恋曲", 10);
|
||||
expect(d.f.cast.map((x) => x.hosts)).toEqual([4, 4]);
|
||||
});
|
||||
|
||||
it("关掉「也用其它网站的搜索摘要」:4 条都抽不到字段(白名单网站只有七成字对得上的榜单 / 首页,不读)", async () => {
|
||||
for (const t of ["我被亲戚误会破产了", "重生60 我有情报系统", "京夜喜你", "春城恋曲"]) {
|
||||
const x = await run(t, 5, false);
|
||||
const f = x.f;
|
||||
expect([f.title, f.year, f.episodeCount, f.synopsis, f.platform].every((v) => v === null)).toBe(true);
|
||||
expect(f.cast).toEqual([]);
|
||||
expect(x.usage.fetchCalls).toBe(0);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,6 @@
|
||||
import { sql } from "@umt/db";
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest";
|
||||
import { registerMetadataCapabilities } from "../../src/modules/agent/capabilities-metadata.ts";
|
||||
import { registerMetadataCapabilities, robotsAllowedFor } from "../../src/modules/agent/capabilities-metadata.ts";
|
||||
import { createCapabilityRegistry } from "../../src/modules/agent/capability-registry.ts";
|
||||
import {
|
||||
AUTO_ENRICH_TASK,
|
||||
@@ -8,8 +8,9 @@ import {
|
||||
type ModelCompletion,
|
||||
maybeAutoEnrich,
|
||||
provisionalEntitiesForTmdbReview,
|
||||
ruleExtract,
|
||||
} from "../../src/modules/sources-identity/service/enrich.ts";
|
||||
import { extractFromPage } from "../../src/modules/sources-identity/service/enrich-sources.ts";
|
||||
|
||||
import { createWebSearchModule } from "../../src/modules/websearch/index.ts";
|
||||
import { createHarness, type Harness, json, pgAvailable, seedSite } from "./harness.ts";
|
||||
|
||||
@@ -32,6 +33,7 @@ const OTHER = "https://baike.example.com/item/other";
|
||||
const BAIKE_HTML = `<!doctype html><html><head><title>${TITLE}_百科</title>
|
||||
<meta property="og:image" content="${POSTER}"></head><body><h1>${TITLE}</h1>
|
||||
<p>${TITLE}，又名：替身娇妻。类型：都市、甜宠</p><p>首播时间：2024年3月1日</p><p>播出平台：红果短剧</p><p>共80集</p>
|
||||
<p>主演：林小雨、陈默</p>
|
||||
<p>剧情简介：落魄千金为救母亲替嫁给冷面总裁，两人在一次次误会中走近，最终揭开当年的家族阴谋。</p></body></html>`;
|
||||
const VIDEO_HTML = `<!doctype html><html><head><title>${TITLE} 全80集</title></head>
|
||||
<body><p>${TITLE} 全80集 2024 红果短剧独播</p></body></html>`;
|
||||
@@ -128,7 +130,7 @@ describe.skipIf(!pg)("META-1 元数据补全", () => {
|
||||
siteId = (await seedSite(h, { key: "hhclub", baseUrl: "https://pt.hhclub.example/" })).site.id;
|
||||
const registry = createCapabilityRegistry({ logger: h.ctx.logger, db: h.db, settings: h.ctx.settings });
|
||||
createWebSearchModule({ fetch: mockFetch, resolveHost }).registerCapabilities?.(registry, h.ctx);
|
||||
registerMetadataCapabilities(registry, h.ctx, { complete: fakeComplete });
|
||||
registerMetadataCapabilities(registry, h.ctx, { complete: fakeComplete, fetch: mockFetch, resolveHost });
|
||||
(h.ctx as { capabilities: unknown }).capabilities = registry;
|
||||
const ref = await h.secrets.create({ purpose: "other", plaintext: "BSA-enrich-test", label: "websearch" });
|
||||
await setSettings("agent", { webSearch: { provider: "brave", apiKeySecretRef: ref, maxResults: 10 } });
|
||||
@@ -144,7 +146,7 @@ describe.skipIf(!pg)("META-1 元数据补全", () => {
|
||||
modelMode = "json";
|
||||
});
|
||||
|
||||
it("建议:字段带出处与证据等级;白名单外域名 / 已登记站点 / 跳转出白名单的网页不用;不写作品表", async () => {
|
||||
it("建议:字段带出处、来源类别、置信度与证据等级;白名单外域名 / 已登记站点 / 跳转出白名单的网页不读;不写作品表", async () => {
|
||||
const { poolId } = await poolEntry(TITLE);
|
||||
const entitiesBefore = await h.db.selectFrom("media_entity").select("id").execute();
|
||||
const out = await invoke({ poolEntryId: poolId });
|
||||
@@ -156,15 +158,24 @@ describe.skipIf(!pg)("META-1 元数据补全", () => {
|
||||
expect(f.episodeCount).toMatchObject({ value: 80, evidenceLevel: "rule_confirmed", verbatim: true });
|
||||
expect(f.year).toMatchObject({ value: 2024, evidenceLevel: "rule_confirmed" });
|
||||
expect(f.platform).toMatchObject({ value: "红果短剧", evidenceLevel: "rule_confirmed" });
|
||||
// 改写的简介不是原文 → 待复核、Agent 归纳
|
||||
expect(f.synopsis).toMatchObject({ evidenceLevel: "needs_review", verbatim: false });
|
||||
// 模型自己拼的海报地址丢弃
|
||||
expect(f.posterUrl).toBeNull();
|
||||
// 模型改写的简介(置信度 0.3)不如网页原文(规则抽取)→ 取原文
|
||||
expect(f.synopsis).toMatchObject({ verbatim: true, sourceUrl: BAIKE, sourceKind: "other" });
|
||||
expect(f.synopsis.value).toMatch(/^落魄千金为救母亲/);
|
||||
// 模型自己拼的海报地址丢弃,用网页的 og:image
|
||||
expect(f.posterUrl).toMatchObject({ value: POSTER, sourceUrl: BAIKE });
|
||||
// 两个网站都写了 80 集:置信度叠加
|
||||
expect(f.episodeCount.hosts).toBe(2);
|
||||
expect(f.episodeCount.confidence).toBeGreaterThan(0.8);
|
||||
expect(f.cast.map((c: Any) => c.value)).toEqual(["林小雨", "陈默"]);
|
||||
// 各来源的尝试与命中(命中率统计用)
|
||||
const att = Object.fromEntries(out.usage.attempts.map((a: Any) => [a.kind, a]));
|
||||
expect(att.other).toMatchObject({ fetched: 2, hit: true });
|
||||
expect(att.snippet.hit).toBe(false);
|
||||
expect(f.aliases.map((a: Any) => a.value)).toEqual(["替身娇妻"]);
|
||||
expect(f.genre.map((g: Any) => g.value)).toEqual(["都市", "甜宠"]);
|
||||
|
||||
const src = Object.fromEntries(out.suggestion.sources.map((s: Any) => [s.url, s]));
|
||||
expect(src[EVIL]).toMatchObject({ allowed: false, fetched: false, skipped: "not_allowed" });
|
||||
expect(src[EVIL]).toMatchObject({ allowed: false, fetched: false, skipped: "not_allowed", kind: "snippet" });
|
||||
// 已登记站点的域名即使在白名单里也不读
|
||||
expect(src[SITE_PAGE]).toMatchObject({ allowed: false, fetched: false, skipped: "site" });
|
||||
expect(src[BAIKE]).toMatchObject({ fetched: true });
|
||||
@@ -172,8 +183,14 @@ describe.skipIf(!pg)("META-1 元数据补全", () => {
|
||||
// 白名单内但标题对不上:不读
|
||||
expect(src[OTHER]).toMatchObject({ allowed: true, fetched: false, skipped: "irrelevant" });
|
||||
expect(pageHits).not.toContain(OTHER);
|
||||
expect(pageHits).not.toContain("https://evil.example.net/robots.txt");
|
||||
expect(pageHits).not.toContain(SITE_PAGE);
|
||||
expect(pageHits.filter((u) => u === EVIL)).toHaveLength(1); // 只有重定向那一跳,结果被丢弃
|
||||
// 读网页前先看 robots.txt(每个网站一次,缓存 1 天)
|
||||
expect(pageHits.filter((u) => u.endsWith("/robots.txt")).sort()).toEqual([
|
||||
"https://baike.example.com/robots.txt",
|
||||
"https://v.example.com/robots.txt",
|
||||
]);
|
||||
expect(out.usage).toMatchObject({ searchCalls: 1, fetchCalls: 3, inputTokens: 1200, outputTokens: 300 });
|
||||
expect(out.budget).toEqual({ usedToday: 1, dailyLimit: 10 });
|
||||
expect(out.notice).toContain("不是 TMDB");
|
||||
@@ -253,7 +270,12 @@ describe.skipIf(!pg)("META-1 元数据补全", () => {
|
||||
expect(e).toMatchObject({ status: "provisional", kind: "series", title: TITLE, year: 2024 });
|
||||
const meta = e.metadata as Any;
|
||||
expect(meta.title).toMatchObject({ value: TITLE, source: "web" });
|
||||
expect(meta.overview).toMatchObject({ source: "agent" }); // 改写的简介标为 Agent 生成
|
||||
expect(meta.overview).toMatchObject({ source: "web" }); // 网页原文
|
||||
// 主演:没有 TMDB 人物 ID,用负数占位
|
||||
expect(meta.cast.value).toEqual([
|
||||
{ id: -1, name: "林小雨", originalName: null, role: null, profilePath: null },
|
||||
{ id: -2, name: "陈默", originalName: null, role: null, profilePath: null },
|
||||
]);
|
||||
expect(meta.episodeCount).toMatchObject({ value: 80, source: "web" });
|
||||
expect(meta.genres.value).toEqual(["都市", "甜宠"]);
|
||||
const fields = await h.db.selectFrom("metadata_field").selectAll().where("media_entity_id", "=", e.id).execute();
|
||||
@@ -318,10 +340,45 @@ describe.skipIf(!pg)("META-1 元数据补全", () => {
|
||||
});
|
||||
|
||||
it("规则抽取只用标题对得上的网页", () => {
|
||||
const r = ruleExtract({ title: "别的剧" }, [
|
||||
{ url: BAIKE, finalUrl: BAIKE, title: `${TITLE}_百科`, text: `${TITLE} 共80集`, image: POSTER },
|
||||
]);
|
||||
expect(r.title).toBeNull();
|
||||
expect(r.episodeCount).toBeNull();
|
||||
const r = extractFromPage("别的剧", {
|
||||
url: BAIKE,
|
||||
host: "baike.example.com",
|
||||
kind: "other",
|
||||
title: `${TITLE}_百科`,
|
||||
text: `${TITLE} 共80集`,
|
||||
image: POSTER,
|
||||
});
|
||||
expect(r).toEqual([]);
|
||||
});
|
||||
|
||||
it("robots.txt:不允许的网址不读(按网站缓存);5xx 当作不允许,404 当作没有限制", async () => {
|
||||
const hits: string[] = [];
|
||||
const f = (async (input: string | URL | Request) => {
|
||||
const u = new URL(typeof input === "string" ? input : input instanceof URL ? input.href : input.url);
|
||||
hits.push(u.href);
|
||||
if (u.hostname === "robots-a.example.com")
|
||||
return new Response("User-agent: *\nDisallow: /series/\n", { headers: { "content-type": "text/plain" } });
|
||||
if (u.hostname === "robots-b.example.com") return new Response("err", { status: 503 });
|
||||
return new Response("nf", { status: 404 });
|
||||
}) as typeof fetch;
|
||||
const o = { fetch: f, resolveHost };
|
||||
expect(await robotsAllowedFor(h.ctx, "https://robots-a.example.com/series/1", o)).toBe(false);
|
||||
expect(await robotsAllowedFor(h.ctx, "https://robots-a.example.com/detail?id=1", o)).toBe(true);
|
||||
expect(hits.filter((x) => x.includes("robots-a"))).toHaveLength(1);
|
||||
expect(await robotsAllowedFor(h.ctx, "https://robots-b.example.com/x", o)).toBe(false);
|
||||
expect(await robotsAllowedFor(h.ctx, "https://robots-c.example.com/x", o)).toBe(true);
|
||||
});
|
||||
|
||||
it("命中率:最近 30 天按来源类别统计(GET /identity/enrich-stats)", async () => {
|
||||
const res = await json(await h.req("/identity/enrich-stats?days=30"));
|
||||
expect(res.status).toBe(200);
|
||||
const d = res.body.data;
|
||||
expect(d.days).toBe(30);
|
||||
expect(d.total).toBeGreaterThan(0);
|
||||
expect(d.tracked).toBe(d.total);
|
||||
const other = d.sources.find((x: Any) => x.kind === "other");
|
||||
expect(other.hits).toBeGreaterThan(0);
|
||||
expect(other.rate).toBeGreaterThan(0);
|
||||
expect(d.sources.find((x: Any) => x.kind === "douban")).toMatchObject({ hits: 0, rate: 0 });
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user