feat(identity): source-aware short-drama enrichment with snippet mode and hit rate (META-2)

- classify enrich sources: encyclopedia, platform pages (hongguo, kuaikaw, video
  platforms), douban (search snippet only) and other sites' search snippets
  (snippet mode, pages never opened); one extractor per source, fields carry
  source kind and confidence, merged by noisy-OR across hosts
- new cast field (applied as MediaCredit with placeholder ids)
- robots.txt check (cached per host 1 day) and login-wall/captcha give-up
- per-enrichment source attempts + GET /identity/enrich-stats, 30-day hit-rate
  table under settings -> metadata -> identity
- setting identity.enrichSnippetAllSites (default on); default domains add
  baike.com and kuaikaw.cn
- replay fixtures from the real run on the 4 user-library short dramas
  (sanitized Brave responses, hongguo page, robots.txt)

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed 2026-09-30 18:40:19 +08:00
1 parent 081db4651c
commit 9489e40e0c
23 files changed
+2615 -323

No files matched your search

+69 -4
View File
@@ -285,14 +285,27 @@ export const FieldEvidence = z.object({
});
export type FieldEvidence = z.infer<typeof FieldEvidence>;
/**
* META-2 补全来源类别:encyclopedia = 百科(百度 / 搜狗 / 快懂百科、维基);platform = 平台作品页(短剧平台、视频平台);
* douban = 豆瓣(只用搜索摘要,不读页面);snippet = 其它网站的搜索摘要(只读搜索结果摘要,不读页面);other = 白名单里的其它网站
*/
export const EnrichSourceKind = z.enum(["encyclopedia", "platform", "douban", "snippet", "other"]);
export type EnrichSourceKind = z.infer<typeof EnrichSourceKind>;
const enrichValue = <T extends z.ZodType>(value: T) =>
z.object({
value,
/** 出处网页(白名单域名);模型给不出出处时为 null */
/** 出处网页(白名单域名,或摘要模式下的搜索结果链接);模型给不出出处时为 null */
sourceUrl: z.string().nullable(),
evidenceLevel: EnrichEvidenceLevel,
/** 值在出处网页正文里原样出现(否则是 Agent 归纳 / 改写的生成内容) */
/** 值在出处网页正文(或搜索摘要)里原样出现(否则是 Agent 归纳 / 改写的生成内容) */
verbatim: z.boolean(),
/** META-2:出处的来源类别(旧记录没有) */
sourceKind: EnrichSourceKind.optional(),
/** META-2:合并后的置信度 0–1(多个网站给出同一值时按 noisy-OR 叠加) */
confidence: z.number().min(0).max(1).optional(),
/** META-2:给出同一值的不同网站数 */
hosts: z.number().int().nonnegative().optional(),
});
export const EnrichFields = z.object({
@@ -304,6 +317,11 @@ export const EnrichFields = z.object({
synopsis: enrichValue(z.string().min(1).max(2000)).nullable(),
posterUrl: enrichValue(z.string().min(1).max(2048)).nullable(),
genre: z.array(enrichValue(z.string().min(1).max(40))).max(10),
/** META-2:主演(旧记录没有,按空数组) */
cast: z
.array(enrichValue(z.string().min(1).max(60)))
.max(20)
.default([]),
});
export type EnrichFields = z.infer<typeof EnrichFields>;
@@ -316,6 +334,7 @@ export const EnrichFieldName = z.enum([
"synopsis",
"posterUrl",
"genre",
"cast",
]);
export type EnrichFieldName = z.infer<typeof EnrichFieldName>;
@@ -326,9 +345,19 @@ export const EnrichSource = z.object({
domain: z.string(),
allowed: z.boolean(),
fetched: z.boolean(),
/** not_allowed = 不在白名单;site = 是已登记站点的域名;irrelevant = 标题对不上;limit = 超出读取条数;error = 读取失败 */
skipped: z.enum(["not_allowed", "site", "irrelevant", "limit", "error"]).nullable(),
/**
* not_allowed = 不在白名单(摘要模式仍可能用它的搜索摘要);site = 是已登记站点的域名;irrelevant = 标题对不上;
* limit = 超出读取条数;error = 读取失败;snippet_only = 只用搜索摘要的网站(豆瓣);robots = robots.txt 不允许;
* blocked = 页面是登录墙 / 验证码,放弃该来源
*/
skipped: z
.enum(["not_allowed", "site", "irrelevant", "limit", "error", "snippet_only", "robots", "blocked"])
.nullable(),
error: z.string().nullable(),
/** META-2 来源类别 */
kind: EnrichSourceKind.optional(),
/** META-2 这条链接的搜索摘要被用来抽字段 */
snippetUsed: z.boolean().optional(),
});
export type EnrichSource = z.infer<typeof EnrichSource>;
@@ -343,11 +372,23 @@ export const EnrichSuggestion = z.object({
});
export type EnrichSuggestion = z.infer<typeof EnrichSuggestion>;
/** META-2 一次补全里某类来源的尝试与命中:results = 标题对得上的搜索结果数;hit = 最终建议里至少一个字段取自它 */
export const EnrichSourceAttempt = z.object({
kind: EnrichSourceKind,
results: z.number().int().nonnegative(),
fetched: z.number().int().nonnegative(),
hit: z.boolean(),
fields: z.array(z.string()),
});
export type EnrichSourceAttempt = z.infer<typeof EnrichSourceAttempt>;
export const EnrichUsage = z.object({
searchCalls: z.number().int().nonnegative(),
fetchCalls: z.number().int().nonnegative(),
inputTokens: z.number().int().nonnegative(),
outputTokens: z.number().int().nonnegative(),
/** META-2 各类来源的尝试与命中(命中率统计用;旧记录没有) */
attempts: z.array(EnrichSourceAttempt).optional(),
});
export type EnrichUsage = z.infer<typeof EnrichUsage>;
@@ -373,6 +414,30 @@ export const MetadataEnrichment = EntityBase.extend({
});
export type MetadataEnrichment = z.infer<typeof MetadataEnrichment>;
/** GET /identity/enrich-stats:最近 N 天补全命中率(按来源类别) */
export const EnrichStatsQuery = z.object({ days: z.coerce.number().int().min(1).max(365).default(30) });
export const EnrichStats = z.object({
days: z.number().int(),
/** 发起过搜索的补全次数(命中 7 天缓存的不计) */
total: z.number().int().nonnegative(),
/** 其中出了建议的次数 */
suggested: z.number().int().nonnegative(),
/** 有来源明细(META-2 之后)的次数:下面各来源的命中率以它为分母 */
tracked: z.number().int().nonnegative(),
sources: z.array(
z.object({
kind: EnrichSourceKind,
/** 有标题对得上的搜索结果的次数 */
withResults: z.number().int().nonnegative(),
/** 建议里至少一个字段取自这类来源的次数 */
hits: z.number().int().nonnegative(),
/** hits / tracked(0–1);tracked = 0 时为 null */
rate: z.number().min(0).max(1).nullable(),
}),
),
});
export type EnrichStats = z.infer<typeof EnrichStats>;
/** POST /identity-pool/{id}/enrich 的请求体;fresh = 忽略归一化标题缓存重新查(计入每日预算) */
export const EnrichRequest = z.object({
hints: z.string().trim().max(200).optional(),