feat(identity): source-aware short-drama enrichment with snippet mode and hit rate (META-2)

- classify enrich sources: encyclopedia, platform pages (hongguo, kuaikaw, video
  platforms), douban (search snippet only) and other sites' search snippets
  (snippet mode, pages never opened); one extractor per source, fields carry
  source kind and confidence, merged by noisy-OR across hosts
- new cast field (applied as MediaCredit with placeholder ids)
- robots.txt check (cached per host 1 day) and login-wall/captcha give-up
- per-enrichment source attempts + GET /identity/enrich-stats, 30-day hit-rate
  table under settings -> metadata -> identity
- setting identity.enrichSnippetAllSites (default on); default domains add
  baike.com and kuaikaw.cn
- replay fixtures from the real run on the 4 user-library short dramas
  (sanitized Brave responses, hongguo page, robots.txt)

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed 2026-09-30 18:40:19 +08:00
1 parent 081db4651c
commit 9489e40e0c
23 files changed
+2615 -323

No files matched your search

+9
View File
@@ -2200,6 +2200,15 @@ export const Api = {
body: identity.EnrichApplyInput,
response: identity.EnrichApplyResult,
}),
// META-2 补全命中率(按来源类别)
enrichStats: ep({
method: "get",
path: "/identity/enrich-stats",
scope: "read",
summary: "元数据补全命中率(最近 N 天,按来源类别:百科、平台、豆瓣摘要、其它网站摘要)",
query: identity.EnrichStatsQuery,
response: identity.EnrichStats,
}),
},
// ── 媒体库 / 库存 / 首页 ──
+1 -1
View File
@@ -581,7 +581,7 @@ export const CAPABILITY_CATALOG: Record<KnownCapabilityId, CapabilityDescriptor>
"metadata.enrich": d(
"metadata.enrich",
"补全元数据",
"TMDB 查不到的作品(短剧、平台独播剧)按标题查网页:搜索后只读白名单网站(平台、百科),抽取标题、别名、年份、平台、集数、简介、海报、类型,每个字段带出处和证据等级。只给建议,不改作品;同一标题 7 天内直接返回上次结果",
"TMDB 查不到的作品(短剧、平台独播剧)按标题查网页:搜索后读白名单网站(平台、百科,遵守 robots.txt),豆瓣和其它网站只用搜索摘要,抽取标题、别名、年份、平台、集数、主演、简介、海报、类型,每个字段带出处、置信度和证据等级。只给建议,不改作品;同一标题 7 天内直接返回上次结果",
"propose",
"MetadataEnrich",
"organize.execute",
+69 -4
View File
@@ -285,14 +285,27 @@ export const FieldEvidence = z.object({
});
export type FieldEvidence = z.infer<typeof FieldEvidence>;
/**
* META-2 补全来源类别:encyclopedia = 百科(百度 / 搜狗 / 快懂百科、维基);platform = 平台作品页(短剧平台、视频平台);
* douban = 豆瓣(只用搜索摘要,不读页面);snippet = 其它网站的搜索摘要(只读搜索结果摘要,不读页面);other = 白名单里的其它网站
*/
export const EnrichSourceKind = z.enum(["encyclopedia", "platform", "douban", "snippet", "other"]);
export type EnrichSourceKind = z.infer<typeof EnrichSourceKind>;
const enrichValue = <T extends z.ZodType>(value: T) =>
z.object({
value,
/** 出处网页(白名单域名);模型给不出出处时为 null */
/** 出处网页(白名单域名,或摘要模式下的搜索结果链接);模型给不出出处时为 null */
sourceUrl: z.string().nullable(),
evidenceLevel: EnrichEvidenceLevel,
/** 值在出处网页正文里原样出现(否则是 Agent 归纳 / 改写的生成内容) */
/** 值在出处网页正文(或搜索摘要)里原样出现(否则是 Agent 归纳 / 改写的生成内容) */
verbatim: z.boolean(),
/** META-2:出处的来源类别(旧记录没有) */
sourceKind: EnrichSourceKind.optional(),
/** META-2:合并后的置信度 0–1(多个网站给出同一值时按 noisy-OR 叠加) */
confidence: z.number().min(0).max(1).optional(),
/** META-2:给出同一值的不同网站数 */
hosts: z.number().int().nonnegative().optional(),
});
export const EnrichFields = z.object({
@@ -304,6 +317,11 @@ export const EnrichFields = z.object({
synopsis: enrichValue(z.string().min(1).max(2000)).nullable(),
posterUrl: enrichValue(z.string().min(1).max(2048)).nullable(),
genre: z.array(enrichValue(z.string().min(1).max(40))).max(10),
/** META-2:主演(旧记录没有,按空数组) */
cast: z
.array(enrichValue(z.string().min(1).max(60)))
.max(20)
.default([]),
});
export type EnrichFields = z.infer<typeof EnrichFields>;
@@ -316,6 +334,7 @@ export const EnrichFieldName = z.enum([
"synopsis",
"posterUrl",
"genre",
"cast",
]);
export type EnrichFieldName = z.infer<typeof EnrichFieldName>;
@@ -326,9 +345,19 @@ export const EnrichSource = z.object({
domain: z.string(),
allowed: z.boolean(),
fetched: z.boolean(),
/** not_allowed = 不在白名单;site = 是已登记站点的域名;irrelevant = 标题对不上;limit = 超出读取条数;error = 读取失败 */
skipped: z.enum(["not_allowed", "site", "irrelevant", "limit", "error"]).nullable(),
/**
* not_allowed = 不在白名单(摘要模式仍可能用它的搜索摘要);site = 是已登记站点的域名;irrelevant = 标题对不上;
* limit = 超出读取条数;error = 读取失败;snippet_only = 只用搜索摘要的网站(豆瓣);robots = robots.txt 不允许;
* blocked = 页面是登录墙 / 验证码,放弃该来源
*/
skipped: z
.enum(["not_allowed", "site", "irrelevant", "limit", "error", "snippet_only", "robots", "blocked"])
.nullable(),
error: z.string().nullable(),
/** META-2 来源类别 */
kind: EnrichSourceKind.optional(),
/** META-2 这条链接的搜索摘要被用来抽字段 */
snippetUsed: z.boolean().optional(),
});
export type EnrichSource = z.infer<typeof EnrichSource>;
@@ -343,11 +372,23 @@ export const EnrichSuggestion = z.object({
});
export type EnrichSuggestion = z.infer<typeof EnrichSuggestion>;
/** META-2 一次补全里某类来源的尝试与命中:results = 标题对得上的搜索结果数;hit = 最终建议里至少一个字段取自它 */
export const EnrichSourceAttempt = z.object({
kind: EnrichSourceKind,
results: z.number().int().nonnegative(),
fetched: z.number().int().nonnegative(),
hit: z.boolean(),
fields: z.array(z.string()),
});
export type EnrichSourceAttempt = z.infer<typeof EnrichSourceAttempt>;
export const EnrichUsage = z.object({
searchCalls: z.number().int().nonnegative(),
fetchCalls: z.number().int().nonnegative(),
inputTokens: z.number().int().nonnegative(),
outputTokens: z.number().int().nonnegative(),
/** META-2 各类来源的尝试与命中(命中率统计用;旧记录没有) */
attempts: z.array(EnrichSourceAttempt).optional(),
});
export type EnrichUsage = z.infer<typeof EnrichUsage>;
@@ -373,6 +414,30 @@ export const MetadataEnrichment = EntityBase.extend({
});
export type MetadataEnrichment = z.infer<typeof MetadataEnrichment>;
/** GET /identity/enrich-stats:最近 N 天补全命中率(按来源类别) */
export const EnrichStatsQuery = z.object({ days: z.coerce.number().int().min(1).max(365).default(30) });
export const EnrichStats = z.object({
days: z.number().int(),
/** 发起过搜索的补全次数(命中 7 天缓存的不计) */
total: z.number().int().nonnegative(),
/** 其中出了建议的次数 */
suggested: z.number().int().nonnegative(),
/** 有来源明细(META-2 之后)的次数:下面各来源的命中率以它为分母 */
tracked: z.number().int().nonnegative(),
sources: z.array(
z.object({
kind: EnrichSourceKind,
/** 有标题对得上的搜索结果的次数 */
withResults: z.number().int().nonnegative(),
/** 建议里至少一个字段取自这类来源的次数 */
hits: z.number().int().nonnegative(),
/** hits / tracked(0–1);tracked = 0 时为 null */
rate: z.number().min(0).max(1).nullable(),
}),
),
});
export type EnrichStats = z.infer<typeof EnrichStats>;
/** POST /identity-pool/{id}/enrich 的请求体;fresh = 忽略归一化标题缓存重新查(计入每日预算) */
export const EnrichRequest = z.object({
hints: z.string().trim().max(200).optional(),
+10
View File
@@ -452,6 +452,9 @@ export const DEFAULT_ENRICH_DOMAINS: string[] = [
"ixigua.com",
"baike.sogou.com",
"mydramalist.com",
// META-2:快懂百科、河马剧场(短剧平台公开作品页);红果短剧的 /series/ 页 robots 不允许,只读 robots 放行的页面
"baike.com",
"kuaikaw.cn",
];
export const IdentitySettings = z.object({
@@ -475,6 +478,11 @@ export const IdentitySettings = z.object({
.regex(/^[a-z0-9.-]+\.[a-z]{2,}$/),
)
.max(100),
/**
* META-2 摘要模式:白名单外网站(多为短剧聚合站)的搜索结果摘要也用来抽字段(只读搜索引擎给的摘要,不打开这些网页)。
* 缺省开;关掉后只用白名单网站的摘要与网页。旧设置没有这个键,按开处理
*/
enrichSnippetAllSites: z.boolean().optional(),
});
export type IdentitySettings = z.infer<typeof IdentitySettings>;
@@ -668,6 +676,7 @@ export const DEFAULT_SETTINGS: GlobalSettings = {
autoEnrich: false,
enrichDailyBudget: 20,
enrichDomains: DEFAULT_ENRICH_DOMAINS,
enrichSnippetAllSites: true,
},
search: { defaultSiteMode: "default", siteIds: [] },
storage: DEFAULT_STORAGE_SETTINGS,
@@ -960,6 +969,7 @@ export const SETTINGS_SECTIONS: readonly SettingsSectionDef[] = [
"identity.autoEnrich",
"identity.enrichDailyBudget",
"identity.enrichDomains",
"identity.enrichSnippetAllSites",
"cache.globalImageCache",
"cache.globalImageCacheDays",
],