feat(identity): source-aware short-drama enrichment with snippet mode and hit rate (META-2)
- classify enrich sources: encyclopedia, platform pages (hongguo, kuaikaw, video platforms), douban (search snippet only) and other sites' search snippets (snippet mode, pages never opened); one extractor per source, fields carry source kind and confidence, merged by noisy-OR across hosts - new cast field (applied as MediaCredit with placeholder ids) - robots.txt check (cached per host 1 day) and login-wall/captcha give-up - per-enrichment source attempts + GET /identity/enrich-stats, 30-day hit-rate table under settings -> metadata -> identity - setting identity.enrichSnippetAllSites (default on); default domains add baike.com and kuaikaw.cn - replay fixtures from the real run on the 4 user-library short dramas (sanitized Brave responses, hongguo page, robots.txt) Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
081db4651c
commit
9489e40e0c
23 files changed
+2615
-323
No files matched your search
@@ -285,14 +285,27 @@ export const FieldEvidence = z.object({
|
||||
});
|
||||
export type FieldEvidence = z.infer<typeof FieldEvidence>;
|
||||
|
||||
/**
|
||||
* META-2 补全来源类别:encyclopedia = 百科(百度 / 搜狗 / 快懂百科、维基);platform = 平台作品页(短剧平台、视频平台);
|
||||
* douban = 豆瓣(只用搜索摘要,不读页面);snippet = 其它网站的搜索摘要(只读搜索结果摘要,不读页面);other = 白名单里的其它网站
|
||||
*/
|
||||
export const EnrichSourceKind = z.enum(["encyclopedia", "platform", "douban", "snippet", "other"]);
|
||||
export type EnrichSourceKind = z.infer<typeof EnrichSourceKind>;
|
||||
|
||||
const enrichValue = <T extends z.ZodType>(value: T) =>
|
||||
z.object({
|
||||
value,
|
||||
/** 出处网页(白名单域名);模型给不出出处时为 null */
|
||||
/** 出处网页(白名单域名,或摘要模式下的搜索结果链接);模型给不出出处时为 null */
|
||||
sourceUrl: z.string().nullable(),
|
||||
evidenceLevel: EnrichEvidenceLevel,
|
||||
/** 值在出处网页正文里原样出现(否则是 Agent 归纳 / 改写的生成内容) */
|
||||
/** 值在出处网页正文(或搜索摘要)里原样出现(否则是 Agent 归纳 / 改写的生成内容) */
|
||||
verbatim: z.boolean(),
|
||||
/** META-2:出处的来源类别(旧记录没有) */
|
||||
sourceKind: EnrichSourceKind.optional(),
|
||||
/** META-2:合并后的置信度 0–1(多个网站给出同一值时按 noisy-OR 叠加) */
|
||||
confidence: z.number().min(0).max(1).optional(),
|
||||
/** META-2:给出同一值的不同网站数 */
|
||||
hosts: z.number().int().nonnegative().optional(),
|
||||
});
|
||||
|
||||
export const EnrichFields = z.object({
|
||||
@@ -304,6 +317,11 @@ export const EnrichFields = z.object({
|
||||
synopsis: enrichValue(z.string().min(1).max(2000)).nullable(),
|
||||
posterUrl: enrichValue(z.string().min(1).max(2048)).nullable(),
|
||||
genre: z.array(enrichValue(z.string().min(1).max(40))).max(10),
|
||||
/** META-2:主演(旧记录没有,按空数组) */
|
||||
cast: z
|
||||
.array(enrichValue(z.string().min(1).max(60)))
|
||||
.max(20)
|
||||
.default([]),
|
||||
});
|
||||
export type EnrichFields = z.infer<typeof EnrichFields>;
|
||||
|
||||
@@ -316,6 +334,7 @@ export const EnrichFieldName = z.enum([
|
||||
"synopsis",
|
||||
"posterUrl",
|
||||
"genre",
|
||||
"cast",
|
||||
]);
|
||||
export type EnrichFieldName = z.infer<typeof EnrichFieldName>;
|
||||
|
||||
@@ -326,9 +345,19 @@ export const EnrichSource = z.object({
|
||||
domain: z.string(),
|
||||
allowed: z.boolean(),
|
||||
fetched: z.boolean(),
|
||||
/** not_allowed = 不在白名单;site = 是已登记站点的域名;irrelevant = 标题对不上;limit = 超出读取条数;error = 读取失败 */
|
||||
skipped: z.enum(["not_allowed", "site", "irrelevant", "limit", "error"]).nullable(),
|
||||
/**
|
||||
* not_allowed = 不在白名单(摘要模式仍可能用它的搜索摘要);site = 是已登记站点的域名;irrelevant = 标题对不上;
|
||||
* limit = 超出读取条数;error = 读取失败;snippet_only = 只用搜索摘要的网站(豆瓣);robots = robots.txt 不允许;
|
||||
* blocked = 页面是登录墙 / 验证码,放弃该来源
|
||||
*/
|
||||
skipped: z
|
||||
.enum(["not_allowed", "site", "irrelevant", "limit", "error", "snippet_only", "robots", "blocked"])
|
||||
.nullable(),
|
||||
error: z.string().nullable(),
|
||||
/** META-2 来源类别 */
|
||||
kind: EnrichSourceKind.optional(),
|
||||
/** META-2 这条链接的搜索摘要被用来抽字段 */
|
||||
snippetUsed: z.boolean().optional(),
|
||||
});
|
||||
export type EnrichSource = z.infer<typeof EnrichSource>;
|
||||
|
||||
@@ -343,11 +372,23 @@ export const EnrichSuggestion = z.object({
|
||||
});
|
||||
export type EnrichSuggestion = z.infer<typeof EnrichSuggestion>;
|
||||
|
||||
/** META-2 一次补全里某类来源的尝试与命中:results = 标题对得上的搜索结果数;hit = 最终建议里至少一个字段取自它 */
|
||||
export const EnrichSourceAttempt = z.object({
|
||||
kind: EnrichSourceKind,
|
||||
results: z.number().int().nonnegative(),
|
||||
fetched: z.number().int().nonnegative(),
|
||||
hit: z.boolean(),
|
||||
fields: z.array(z.string()),
|
||||
});
|
||||
export type EnrichSourceAttempt = z.infer<typeof EnrichSourceAttempt>;
|
||||
|
||||
export const EnrichUsage = z.object({
|
||||
searchCalls: z.number().int().nonnegative(),
|
||||
fetchCalls: z.number().int().nonnegative(),
|
||||
inputTokens: z.number().int().nonnegative(),
|
||||
outputTokens: z.number().int().nonnegative(),
|
||||
/** META-2 各类来源的尝试与命中(命中率统计用;旧记录没有) */
|
||||
attempts: z.array(EnrichSourceAttempt).optional(),
|
||||
});
|
||||
export type EnrichUsage = z.infer<typeof EnrichUsage>;
|
||||
|
||||
@@ -373,6 +414,30 @@ export const MetadataEnrichment = EntityBase.extend({
|
||||
});
|
||||
export type MetadataEnrichment = z.infer<typeof MetadataEnrichment>;
|
||||
|
||||
/** GET /identity/enrich-stats:最近 N 天补全命中率(按来源类别) */
|
||||
export const EnrichStatsQuery = z.object({ days: z.coerce.number().int().min(1).max(365).default(30) });
|
||||
export const EnrichStats = z.object({
|
||||
days: z.number().int(),
|
||||
/** 发起过搜索的补全次数(命中 7 天缓存的不计) */
|
||||
total: z.number().int().nonnegative(),
|
||||
/** 其中出了建议的次数 */
|
||||
suggested: z.number().int().nonnegative(),
|
||||
/** 有来源明细(META-2 之后)的次数:下面各来源的命中率以它为分母 */
|
||||
tracked: z.number().int().nonnegative(),
|
||||
sources: z.array(
|
||||
z.object({
|
||||
kind: EnrichSourceKind,
|
||||
/** 有标题对得上的搜索结果的次数 */
|
||||
withResults: z.number().int().nonnegative(),
|
||||
/** 建议里至少一个字段取自这类来源的次数 */
|
||||
hits: z.number().int().nonnegative(),
|
||||
/** hits / tracked(0–1);tracked = 0 时为 null */
|
||||
rate: z.number().min(0).max(1).nullable(),
|
||||
}),
|
||||
),
|
||||
});
|
||||
export type EnrichStats = z.infer<typeof EnrichStats>;
|
||||
|
||||
/** POST /identity-pool/{id}/enrich 的请求体;fresh = 忽略归一化标题缓存重新查(计入每日预算) */
|
||||
export const EnrichRequest = z.object({
|
||||
hints: z.string().trim().max(200).optional(),
|
||||
|
||||
Reference in new issue
Block a user