feat(identity): web metadata enrichment for titles missing on TMDB (META-1)
- metadata.enrich (propose, low, replay safe): normalized-title cache (kv_cache
region enrich, 7 days) -> daily budget -> web.search (title + 短剧 / 电视剧 平台,
second query limited to whitelist via site:) -> web.fetch of up to 3 relevant
whitelisted pages (registered site domains never read, redirects out of the
whitelist dropped) -> upstream model extraction with rule fallback -> server-side
evidence check (rule_confirmed / multi_evidence / needs_review). Suggestion only.
- POST /identity-pool/{id}/enrich (suggest) and /enrich/apply (console session):
provisional entity or same-title merge, season/episode units, field-level
metadata with source web/agent + evidence, aliases, release assignment.
Poster proxy GET /img/enrich/{id} (og:image from whitelisted pages only).
- Auto-enrich once per pool entry when subscription/rule lookups find nothing on
TMDB (identity.autoEnrich, off by default); identity.enrichDailyBudget,
identity.enrichDomains settings; web.fetch now returns og:image.
- Pool sheet "元数据补全" section; entity detail shows evidence links.
- Migration 0049_enrichment (metadata_enrichment, metadata_field.evidence, source
web). Requires image rebuild / api + worker restart to take effect.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
18a5cf1de7
commit
f4666a4671
36 files changed
+2514
-25
No files matched your search
@@ -32,6 +32,8 @@ export interface FetchedPage {
|
||||
text: string;
|
||||
truncated: boolean;
|
||||
bytes: number;
|
||||
/** HTML 的 og:image / twitter:image(绝对 http(s) 地址,已脱敏);没有为 null */
|
||||
image: string | null;
|
||||
}
|
||||
|
||||
export const PAGE_MAX_BYTES = 512 * 1024;
|
||||
@@ -155,5 +157,26 @@ export async function fetchPage(raw: string, opts: FetchPageOptions = {}): Promi
|
||||
text: clean.text,
|
||||
truncated: body.truncated || clean.truncated,
|
||||
bytes: body.bytes.byteLength,
|
||||
image: isHtml ? ogImage(decoded, url) : null,
|
||||
};
|
||||
}
|
||||
|
||||
/** <meta property="og:image" content="…">(属性顺序不限),退而取 twitter:image;相对地址按页面地址补全 */
|
||||
export function ogImage(html: string, base: URL): string | null {
|
||||
const head = html.slice(0, 200_000);
|
||||
for (const key of ["og:image", "og:image:url", "twitter:image"]) {
|
||||
for (const m of head.matchAll(/<meta\b[^>]*>/gi)) {
|
||||
const tag = m[0];
|
||||
const name = tag.match(/\b(?:property|name)\s*=\s*["']([^"']+)["']/i)?.[1]?.toLowerCase();
|
||||
if (name !== key) continue;
|
||||
const content = tag.match(/\bcontent\s*=\s*["']([^"']+)["']/i)?.[1]?.trim();
|
||||
if (!content) continue;
|
||||
try {
|
||||
const u = new URL(content.replace(/&/g, "&"), base);
|
||||
if (u.protocol !== "http:" && u.protocol !== "https:") continue;
|
||||
return redactUrl(u.href).slice(0, 2048);
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -4,7 +4,7 @@ import * as assets from "./assets.ts";
|
||||
import * as auth from "./auth.ts";
|
||||
import * as browser from "./browser.ts";
|
||||
import * as cache from "./cache.ts";
|
||||
import { WebSearchTestInput, WebSearchTestResult } from "./capabilities-io.ts";
|
||||
import { MetadataEnrichOutput, WebSearchTestInput, WebSearchTestResult } from "./capabilities-io.ts";
|
||||
import { CapabilityDescriptor, KnownCapabilityId } from "./capability.ts";
|
||||
import { type ApiScope, CursorQuery, cursorPage, HealthStatus, Id, IdParam } from "./common.ts";
|
||||
import * as content from "./content.ts";
|
||||
@@ -2099,6 +2099,26 @@ export const Api = {
|
||||
params: IdParam,
|
||||
response: TaskAccepted,
|
||||
}),
|
||||
// META-1 元数据补全:先看建议(metadata.enrich,不改作品),再在控制台应用
|
||||
enrich: ep({
|
||||
method: "post",
|
||||
path: "/identity-pool/{id}/enrich",
|
||||
scope: "organize.execute",
|
||||
summary: "元数据补全建议(网页搜索 + 白名单网页 + 模型抽取;不改作品;同一标题 7 天内返回上次结果)",
|
||||
params: IdParam,
|
||||
body: identity.EnrichRequest,
|
||||
response: MetadataEnrichOutput,
|
||||
}),
|
||||
enrichApply: ep({
|
||||
method: "post",
|
||||
path: "/identity-pool/{id}/enrich/apply",
|
||||
scope: "admin",
|
||||
auth: "session",
|
||||
summary: "应用元数据补全建议:建本地临时作品(或并入同名作品)并关联待识别资源(只接受控制台管理员会话)",
|
||||
params: IdParam,
|
||||
body: identity.EnrichApplyInput,
|
||||
response: identity.EnrichApplyResult,
|
||||
}),
|
||||
},
|
||||
|
||||
// ── 媒体库 / 库存 / 首页 ──
|
||||
|
||||
@@ -7,7 +7,7 @@ import { CAPABILITY_CATALOG, type CapabilitySpec, type KnownCapabilityId } from
|
||||
import { Id, JsonObject, NonNegInt } from "./common.ts";
|
||||
import { MediaEntity, MediaKind } from "./content.ts";
|
||||
import { AcquisitionPlan, DownloadStatus, DownloadTask, Task, TaskStatus } from "./execution.ts";
|
||||
import { ExternalEntityType, ExternalIdentity } from "./identity.ts";
|
||||
import { EnrichmentStatus, EnrichSuggestion, EnrichUsage, ExternalEntityType, ExternalIdentity } from "./identity.ts";
|
||||
import { ServerMapping } from "./ops.ts";
|
||||
import { OrganizationAction, OrganizationRecord } from "./organization.ts";
|
||||
import {
|
||||
@@ -773,6 +773,38 @@ export const WebFetchOutput = z.object({
|
||||
text: z.string(),
|
||||
truncated: z.boolean(),
|
||||
bytes: NonNegInt,
|
||||
/** 页面 og:image(绝对地址);没有时为 null */
|
||||
image: z.string().nullable().optional(),
|
||||
});
|
||||
|
||||
// META-1 metadata.enrich:TMDB 没有的作品(短剧等)按标题查网页、抽字段,输出带证据的建议(不落作品表)
|
||||
export const MetadataEnrichInput = z
|
||||
.object({
|
||||
/** 待识别池条目 id(按它的解析标题 / 年份查) */
|
||||
poolEntryId: Id.optional(),
|
||||
title: z.string().trim().min(1).max(200).optional(),
|
||||
year: z.number().int().min(1900).max(2100).optional(),
|
||||
/** 额外检索词,如平台名、主演 */
|
||||
hints: z.string().trim().max(200).optional(),
|
||||
/** 忽略归一化标题缓存(7 天)重新查,计入每日预算 */
|
||||
fresh: z.boolean().optional(),
|
||||
})
|
||||
.refine((v) => !!v.poolEntryId !== !!v.title, { message: "poolEntryId 与 title 必须且只能给一个" });
|
||||
export const MetadataEnrichOutput = z.object({
|
||||
enrichmentId: Id,
|
||||
status: EnrichmentStatus,
|
||||
/** 命中归一化标题缓存:没有新的搜索 / 读取 / 模型调用 */
|
||||
cached: z.boolean(),
|
||||
normalizedTitle: z.string(),
|
||||
query: z.object({ title: z.string(), year: z.number().int().nullable(), hints: z.string().nullable() }),
|
||||
suggestion: EnrichSuggestion.nullable(),
|
||||
error: z.string().nullable(),
|
||||
usage: EnrichUsage,
|
||||
budget: z.object({ usedToday: NonNegInt, dailyLimit: NonNegInt }),
|
||||
/** 同名去重:标题 / 别名相同的已有作品(应用时可选择并入) */
|
||||
existing: z.array(z.object({ mediaEntityId: Id, title: z.string(), year: z.number().int().nullable() })),
|
||||
appliedMediaEntityId: Id.nullable(),
|
||||
notice: z.string(),
|
||||
});
|
||||
|
||||
/** POST /websearch/test */
|
||||
@@ -828,6 +860,7 @@ export const CAPABILITY_IO = {
|
||||
"agent.propose_options": { input: AgentProposeOptionsInput, output: AgentProposeOptionsOutput },
|
||||
"web.search": { input: WebSearchInput, output: WebSearchOutput },
|
||||
"web.fetch": { input: WebFetchInput, output: WebFetchOutput },
|
||||
"metadata.enrich": { input: MetadataEnrichInput, output: MetadataEnrichOutput },
|
||||
} as const satisfies Record<KnownCapabilityId, { input: z.ZodType; output: z.ZodType }>;
|
||||
|
||||
type CapabilityIo = typeof CAPABILITY_IO;
|
||||
@@ -913,3 +946,5 @@ export type WebSearchInput = z.infer<typeof WebSearchInput>;
|
||||
export type WebSearchOutput = z.infer<typeof WebSearchOutput>;
|
||||
export type WebFetchInput = z.infer<typeof WebFetchInput>;
|
||||
export type WebFetchOutput = z.infer<typeof WebFetchOutput>;
|
||||
export type MetadataEnrichInput = z.infer<typeof MetadataEnrichInput>;
|
||||
export type MetadataEnrichOutput = z.infer<typeof MetadataEnrichOutput>;
|
||||
@@ -89,6 +89,7 @@ export const CAPABILITY_IDS = [
|
||||
"agent.propose_options",
|
||||
"web.search",
|
||||
"web.fetch",
|
||||
"metadata.enrich",
|
||||
] as const;
|
||||
export const KnownCapabilityId = z.enum(CAPABILITY_IDS);
|
||||
export type KnownCapabilityId = z.infer<typeof KnownCapabilityId>;
|
||||
@@ -563,6 +564,19 @@ export const CAPABILITY_CATALOG: Record<KnownCapabilityId, CapabilityDescriptor>
|
||||
true,
|
||||
false,
|
||||
),
|
||||
// META-1:只产出建议(存补全记录,不写作品表);消耗网页搜索次数,受每日预算约束。应用建议走控制台 /identity-pool/{id}/enrich/apply
|
||||
"metadata.enrich": d(
|
||||
"metadata.enrich",
|
||||
"补全元数据",
|
||||
"TMDB 查不到的作品(短剧、平台独播剧)按标题查网页:搜索后只读白名单网站(平台、百科),抽取标题、别名、年份、平台、集数、简介、海报、类型,每个字段带出处和证据等级。只给建议,不改作品;同一标题 7 天内直接返回上次结果",
|
||||
"propose",
|
||||
"MetadataEnrich",
|
||||
"organize.execute",
|
||||
"low",
|
||||
true,
|
||||
false,
|
||||
"safe",
|
||||
),
|
||||
};
|
||||
|
||||
function d(
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { z } from "zod";
|
||||
import { ActorType, DateOnly, EntityBase, Id, JsonObject, SERVER_MANAGED, Timestamp } from "./common.ts";
|
||||
import { ExternalIdentity } from "./identity.ts";
|
||||
import { ExternalIdentity, FieldEvidence } from "./identity.ts";
|
||||
|
||||
// 内容域:本地作品、内容单元、别名(REQ §3.1、§5.1、§5.4、§7.5)
|
||||
// 本地 UUID 是主身份;季/集编号、外部 ID 都可修订,不是唯一身份。
|
||||
@@ -149,7 +149,8 @@ export type TitleAliasUpdate = z.infer<typeof TitleAliasUpdate>;
|
||||
|
||||
// ── W2-B:字段级元数据(REQ §7.3)与作品详情 ──
|
||||
|
||||
export const MetadataSource = z.enum(["tmdb", "user", "agent", "nfo", "site", "import"]);
|
||||
/** web = 值出自网页原文(META-1 补全);agent = Agent 归纳 / 翻译的生成内容。两者都不冒充 TMDB */
|
||||
export const MetadataSource = z.enum(["tmdb", "user", "agent", "nfo", "site", "import", "web"]);
|
||||
export type MetadataSource = z.infer<typeof MetadataSource>;
|
||||
|
||||
/** 某来源给出的一个候选字段值(刷新只写这里,不直接改最终展示值) */
|
||||
@@ -162,6 +163,8 @@ export const MetadataField = EntityBase.extend({
|
||||
source: MetadataSource,
|
||||
sourceRevision: z.string().nullable(),
|
||||
fetchedAt: Timestamp,
|
||||
/** 字段证据(META-1 补全写入:来源网页、证据等级、补全记录);TMDB 等 API 来源为 null */
|
||||
evidence: FieldEvidence.nullable().optional(),
|
||||
});
|
||||
export type MetadataField = z.infer<typeof MetadataField>;
|
||||
|
||||
@@ -190,6 +193,11 @@ export const EntityMetadataField = z.enum([
|
||||
"originCountry",
|
||||
"posterPath",
|
||||
"backdropPath",
|
||||
// META-1 补全(TMDB 没有的作品):播出平台、集数、类型、网页海报地址(经图片代理显示)
|
||||
"platform",
|
||||
"episodeCount",
|
||||
"genres",
|
||||
"posterUrl",
|
||||
]);
|
||||
export type EntityMetadataField = z.infer<typeof EntityMetadataField>;
|
||||
|
||||
|
||||
@@ -266,3 +266,142 @@ export const IdentityPoolItem = IdentityCandidate.extend({
|
||||
}),
|
||||
});
|
||||
export type IdentityPoolItem = z.infer<typeof IdentityPoolItem>;
|
||||
|
||||
// ── META-1:TMDB 缺失作品的元数据补全(REQ R03 / R10、§7.3、§7.5)──
|
||||
// 网页搜索 + 读取白名单域名的网页 → 模型(或规则)抽字段,每个字段带来源网页与证据等级;
|
||||
// 建议先存补全记录(metadata_enrichment),用户确认后才建本地临时作品并写字段级元数据(来源 web / agent,不冒充 TMDB)。
|
||||
|
||||
/** 补全字段的证据等级(服务端按网页原文核对得出,不采信模型自报) */
|
||||
export const EnrichEvidenceLevel = z.enum(["rule_confirmed", "multi_evidence", "needs_review"]);
|
||||
export type EnrichEvidenceLevel = z.infer<typeof EnrichEvidenceLevel>;
|
||||
|
||||
/** 字段证据:写在 metadata_field.evidence,作品详情"元数据来源"里显示链接 */
|
||||
export const FieldEvidence = z.object({
|
||||
sourceUrl: z.string().nullable(),
|
||||
evidenceLevel: EnrichEvidenceLevel,
|
||||
enrichmentId: Id.nullable(),
|
||||
/** model = 模型从网页正文抽取;rule = 规则从网页正文抽取 */
|
||||
extractor: z.enum(["model", "rule"]).optional(),
|
||||
});
|
||||
export type FieldEvidence = z.infer<typeof FieldEvidence>;
|
||||
|
||||
const enrichValue = <T extends z.ZodType>(value: T) =>
|
||||
z.object({
|
||||
value,
|
||||
/** 出处网页(白名单域名);模型给不出出处时为 null */
|
||||
sourceUrl: z.string().nullable(),
|
||||
evidenceLevel: EnrichEvidenceLevel,
|
||||
/** 值在出处网页正文里原样出现(否则是 Agent 归纳 / 改写的生成内容) */
|
||||
verbatim: z.boolean(),
|
||||
});
|
||||
|
||||
export const EnrichFields = z.object({
|
||||
title: enrichValue(z.string().min(1).max(200)).nullable(),
|
||||
aliases: z.array(enrichValue(z.string().min(1).max(200))).max(20),
|
||||
year: enrichValue(z.number().int().min(1900).max(2100)).nullable(),
|
||||
platform: enrichValue(z.string().min(1).max(100)).nullable(),
|
||||
episodeCount: enrichValue(z.number().int().min(1).max(2000)).nullable(),
|
||||
synopsis: enrichValue(z.string().min(1).max(2000)).nullable(),
|
||||
posterUrl: enrichValue(z.string().min(1).max(2048)).nullable(),
|
||||
genre: z.array(enrichValue(z.string().min(1).max(40))).max(10),
|
||||
});
|
||||
export type EnrichFields = z.infer<typeof EnrichFields>;
|
||||
|
||||
export const EnrichFieldName = z.enum([
|
||||
"title",
|
||||
"aliases",
|
||||
"year",
|
||||
"platform",
|
||||
"episodeCount",
|
||||
"synopsis",
|
||||
"posterUrl",
|
||||
"genre",
|
||||
]);
|
||||
export type EnrichFieldName = z.infer<typeof EnrichFieldName>;
|
||||
|
||||
/** 搜索结果里的一条链接:是否在白名单、是否读取、为什么跳过 */
|
||||
export const EnrichSource = z.object({
|
||||
url: z.string(),
|
||||
title: z.string().nullable(),
|
||||
domain: z.string(),
|
||||
allowed: z.boolean(),
|
||||
fetched: z.boolean(),
|
||||
/** not_allowed = 不在白名单;site = 是已登记站点的域名;irrelevant = 标题对不上;limit = 超出读取条数;error = 读取失败 */
|
||||
skipped: z.enum(["not_allowed", "site", "irrelevant", "limit", "error"]).nullable(),
|
||||
error: z.string().nullable(),
|
||||
});
|
||||
export type EnrichSource = z.infer<typeof EnrichSource>;
|
||||
|
||||
export const EnrichSuggestion = z.object({
|
||||
fields: EnrichFields,
|
||||
/** 建档类型推断:有集数按剧集 */
|
||||
kind: z.enum(["series", "movie"]),
|
||||
queries: z.array(z.string()),
|
||||
sources: z.array(EnrichSource),
|
||||
extractor: z.enum(["model", "rule"]),
|
||||
model: z.string().nullable(),
|
||||
});
|
||||
export type EnrichSuggestion = z.infer<typeof EnrichSuggestion>;
|
||||
|
||||
export const EnrichUsage = z.object({
|
||||
searchCalls: z.number().int().nonnegative(),
|
||||
fetchCalls: z.number().int().nonnegative(),
|
||||
inputTokens: z.number().int().nonnegative(),
|
||||
outputTokens: z.number().int().nonnegative(),
|
||||
});
|
||||
export type EnrichUsage = z.infer<typeof EnrichUsage>;
|
||||
|
||||
export const EnrichmentStatus = z.enum(["suggested", "applied", "failed"]);
|
||||
export type EnrichmentStatus = z.infer<typeof EnrichmentStatus>;
|
||||
export const EnrichmentTrigger = z.enum(["manual", "auto", "agent"]);
|
||||
export type EnrichmentTrigger = z.infer<typeof EnrichmentTrigger>;
|
||||
|
||||
/** 补全记录(metadata_enrichment 行) */
|
||||
export const MetadataEnrichment = EntityBase.extend({
|
||||
identityCandidateId: Id.nullable(),
|
||||
normalizedTitle: z.string(),
|
||||
query: JsonObject,
|
||||
status: EnrichmentStatus,
|
||||
trigger: EnrichmentTrigger,
|
||||
suggestion: JsonObject.nullable(),
|
||||
error: z.string().nullable(),
|
||||
usage: JsonObject,
|
||||
mediaEntityId: Id.nullable(),
|
||||
appliedAt: Timestamp.nullable(),
|
||||
appliedBy: z.string().nullable(),
|
||||
createdBy: z.string().nullable(),
|
||||
});
|
||||
export type MetadataEnrichment = z.infer<typeof MetadataEnrichment>;
|
||||
|
||||
/** POST /identity-pool/{id}/enrich 的请求体;fresh = 忽略归一化标题缓存重新查(计入每日预算) */
|
||||
export const EnrichRequest = z.object({
|
||||
hints: z.string().trim().max(200).optional(),
|
||||
fresh: z.boolean().optional(),
|
||||
});
|
||||
export type EnrichRequest = z.infer<typeof EnrichRequest>;
|
||||
|
||||
/** POST /identity-pool/{id}/enrich/apply */
|
||||
export const EnrichApplyInput = z.object({
|
||||
enrichmentId: Id,
|
||||
/** 只应用这些字段;缺省应用建议里有值的全部字段 */
|
||||
fields: z.array(EnrichFieldName).min(1).optional(),
|
||||
/** 应用的字段同时加锁(之后的自动来源不改写) */
|
||||
lock: z.boolean().default(false),
|
||||
/** 并入已有作品而不是新建(同名去重时由用户选) */
|
||||
mediaEntityId: Id.optional(),
|
||||
kind: z.enum(["series", "movie"]).optional(),
|
||||
});
|
||||
export type EnrichApplyInput = z.infer<typeof EnrichApplyInput>;
|
||||
|
||||
export const EnrichApplyResult = z.object({
|
||||
enrichmentId: Id,
|
||||
mediaEntityId: Id,
|
||||
/** false = 同名去重命中已有作品,字段写到它上面 */
|
||||
created: z.boolean(),
|
||||
contentUnits: z.number().int().nonnegative(),
|
||||
fields: z.array(z.string()),
|
||||
aliases: z.number().int().nonnegative(),
|
||||
/** 待识别资源已关联到该作品 */
|
||||
assigned: z.boolean(),
|
||||
});
|
||||
export type EnrichApplyResult = z.infer<typeof EnrichApplyResult>;
|
||||
@@ -390,10 +390,45 @@ export type SearchSettings = z.infer<typeof SearchSettings>;
|
||||
* (识别缓存命中则 0 次);一个每小时 100 条的 RSS 约为每天 2400–7200 次 TMDB 请求,多数是站点独有的短剧 / 综艺,查不到。
|
||||
* 关闭(缺省)时只在命中订阅 / 本地作品别名、已发布的新作品收录规则前置条件通过、用户手动识别或搜索时查询。
|
||||
*/
|
||||
/** 元数据补全缺省白名单:视频平台、短剧平台、百科 / 影视资料站(不含任何 PT 站) */
|
||||
export const DEFAULT_ENRICH_DOMAINS: string[] = [
|
||||
"baike.baidu.com",
|
||||
"douban.com",
|
||||
"hongguoduanju.com",
|
||||
"wikipedia.org",
|
||||
"iqiyi.com",
|
||||
"v.qq.com",
|
||||
"youku.com",
|
||||
"mgtv.com",
|
||||
"bilibili.com",
|
||||
"douyin.com",
|
||||
"kuaishou.com",
|
||||
"ixigua.com",
|
||||
"baike.sogou.com",
|
||||
"mydramalist.com",
|
||||
];
|
||||
|
||||
export const IdentitySettings = z.object({
|
||||
eagerTmdbLookup: z.boolean(),
|
||||
/** 待识别池里 unmatched 条目多少天没有新证据就归档(仍可按状态筛出、可手动指定) */
|
||||
poolArchiveDays: z.number().int().min(1).max(3650),
|
||||
/**
|
||||
* META-1:订阅别名 / 新作品收录规则命中、TMDB 却查不到时自动尝试一次元数据补全(只出建议,不建作品)。
|
||||
* 每次约 1–2 次网页搜索 + 最多 3 次读取网页 + 1 次模型调用。
|
||||
*/
|
||||
autoEnrich: z.boolean(),
|
||||
/** 元数据补全每日上限(手动 + 自动;命中 7 天标题缓存不计);0 = 停用补全 */
|
||||
enrichDailyBudget: z.number().int().min(0).max(500),
|
||||
/** 元数据补全只读取这些域名(含子域名)的网页:平台站点、百科类;已登记站点的域名一律不读 */
|
||||
enrichDomains: z
|
||||
.array(
|
||||
z
|
||||
.string()
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.regex(/^[a-z0-9.-]+\.[a-z]{2,}$/),
|
||||
)
|
||||
.max(100),
|
||||
});
|
||||
export type IdentitySettings = z.infer<typeof IdentitySettings>;
|
||||
|
||||
@@ -483,7 +518,13 @@ export const DEFAULT_SETTINGS: GlobalSettings = {
|
||||
defaultDetailCacheSec: 86400,
|
||||
},
|
||||
naming: { ...DEFAULT_NAMING_TEMPLATES, lockFileNameForBound: false },
|
||||
identity: { eagerTmdbLookup: false, poolArchiveDays: 30 },
|
||||
identity: {
|
||||
eagerTmdbLookup: false,
|
||||
poolArchiveDays: 30,
|
||||
autoEnrich: false,
|
||||
enrichDailyBudget: 20,
|
||||
enrichDomains: DEFAULT_ENRICH_DOMAINS,
|
||||
},
|
||||
search: { defaultSiteMode: "default", siteIds: [] },
|
||||
};
|
||||
|
||||
@@ -738,6 +779,9 @@ export const SETTINGS_SECTIONS: readonly SettingsSectionDef[] = [
|
||||
"cache.emptyResultTtlSec",
|
||||
"identity.eagerTmdbLookup",
|
||||
"identity.poolArchiveDays",
|
||||
"identity.autoEnrich",
|
||||
"identity.enrichDailyBudget",
|
||||
"identity.enrichDomains",
|
||||
"cache.globalImageCache",
|
||||
"cache.globalImageCacheDays",
|
||||
],
|
||||
|
||||
@@ -95,6 +95,8 @@ const ROWS: Record<string, { row: Obj; dbOnly?: string[]; apiOnly?: string[] }>
|
||||
IdentityCandidate: { row: C.IdentityCandidate },
|
||||
IdentityRedirect: { row: C.IdentityRedirect },
|
||||
MetadataField: { row: C.MetadataField },
|
||||
// META-1 元数据补全记录(0049)
|
||||
MetadataEnrichment: { row: C.MetadataEnrichment },
|
||||
ResolutionAttempt: { row: C.ResolutionAttempt },
|
||||
SiteSnapshot: { row: C.SiteSnapshot },
|
||||
// W4-C 浏览器实体(0023)
|
||||
@@ -265,6 +267,8 @@ const ENUMS: Record<string, z.ZodEnum> = {
|
||||
IdentityCandidateReason: C.IdentityCandidateReason,
|
||||
IdentityRedirectActorType: C.ActorType,
|
||||
MetadataFieldSource: C.MetadataSource,
|
||||
MetadataEnrichmentStatus: C.EnrichmentStatus,
|
||||
MetadataEnrichmentTrigger: C.EnrichmentTrigger,
|
||||
ResolutionAttemptOutcome: C.ResolutionOutcome,
|
||||
ResolutionAttemptActorType: C.ActorType,
|
||||
SiteSnapshotKind: C.SiteSnapshotKind,
|
||||
|
||||
@@ -104,6 +104,7 @@ const OVERRIDES: Record<string, (v: Record<string, unknown>) => Record<string, u
|
||||
IdentityResolveInput: (v) => v,
|
||||
AssetLocationCreate: (v) => v,
|
||||
ReleaseIdentityAssign: (v) => ({ ...v, external: undefined, candidateIndex: undefined }),
|
||||
MetadataEnrichInput: (v) => ({ ...v, title: undefined }),
|
||||
CacheSettings: (v) => ({ ...v, hardThresholdPercent: 90, resumePercent: 85, cleanupTargetPercent: 80 }),
|
||||
GlobalSettings: () => C.DEFAULT_SETTINGS,
|
||||
// 方案至少 2 个(id 不同)
|
||||
@@ -215,8 +216,8 @@ describe("关键合同行为", () => {
|
||||
expect(spec).toMatchObject(meta);
|
||||
expect(spec.inputSchema).toBe(C.CAPABILITY_IO[id].input);
|
||||
expect(spec.outputSchema).toBe(C.CAPABILITY_IO[id].output);
|
||||
// 例外:site.health_check 只读首页并写回健康列,幂等可重跑
|
||||
if (spec.kind !== "read" && spec.kind !== "preview" && id !== "site.health_check")
|
||||
// 例外:site.health_check 只读首页并写回健康列,幂等可重跑;metadata.enrich 只存补全建议(7 天标题缓存),重跑无副作用
|
||||
if (spec.kind !== "read" && spec.kind !== "preview" && id !== "site.health_check" && id !== "metadata.enrich")
|
||||
expect(spec.replay, id).toBe("never");
|
||||
}
|
||||
});
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
import type { Kysely } from "kysely";
|
||||
import { entityTable, execDdl, oneOf } from "../src/migration-helpers.ts";
|
||||
|
||||
// META-1 TMDB 缺失作品的元数据补全(REQ R03 / R10、§7.3、§7.5):
|
||||
// - metadata_enrichment:一次补全的记录(查询 → 网页搜索 / 读取 → 模型抽取出的建议;应用后指向建出的作品)。
|
||||
// 每日预算按当天新建的行计数(命中归一化标题缓存不新建行);建议只存这里,应用前不写作品表。
|
||||
// - metadata_field / field_override 的来源新增 web:值出自网页原文(Agent 改写 / 归纳的字段记 agent,不记 tmdb);
|
||||
// - metadata_field.evidence:字段证据(来源网页、证据等级、补全记录 id),作品详情"元数据来源"里显示链接。
|
||||
|
||||
const OLD_SOURCES = ["tmdb", "user", "agent", "nfo", "site", "import"] as const;
|
||||
const SOURCES = [...OLD_SOURCES, "web"] as const;
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型
|
||||
export async function up(db: Kysely<any>): Promise<void> {
|
||||
await execDdl(
|
||||
db,
|
||||
`
|
||||
alter table metadata_field drop constraint metadata_field_source_check;
|
||||
alter table metadata_field add constraint metadata_field_source_check ${oneOf("source", SOURCES)};
|
||||
alter table field_override drop constraint field_override_source_check;
|
||||
alter table field_override add constraint field_override_source_check ${oneOf("source", SOURCES)};
|
||||
alter table metadata_field add column evidence jsonb;
|
||||
|
||||
${entityTable(
|
||||
"metadata_enrichment",
|
||||
`
|
||||
identity_candidate_id uuid references identity_candidate (id) on delete set null,
|
||||
normalized_title text not null,
|
||||
query jsonb not null,
|
||||
status text not null default 'suggested' ${oneOf("status", ["suggested", "applied", "failed"])},
|
||||
trigger text not null default 'manual' ${oneOf("trigger", ["manual", "auto", "agent"])},
|
||||
suggestion jsonb,
|
||||
error text,
|
||||
usage jsonb not null default '{}'::jsonb,
|
||||
media_entity_id uuid references media_entity (id) on delete set null,
|
||||
applied_at timestamptz,
|
||||
applied_by text,
|
||||
created_by text`,
|
||||
)}
|
||||
create index metadata_enrichment_created_idx on metadata_enrichment (created_at);
|
||||
create index metadata_enrichment_candidate_idx on metadata_enrichment (identity_candidate_id, created_at desc);
|
||||
create index metadata_enrichment_title_idx on metadata_enrichment (normalized_title);
|
||||
`,
|
||||
);
|
||||
}
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 同上
|
||||
export async function down(db: Kysely<any>): Promise<void> {
|
||||
await execDdl(
|
||||
db,
|
||||
`
|
||||
drop table if exists metadata_enrichment;
|
||||
alter table metadata_field drop column if exists evidence;
|
||||
delete from metadata_field where source = 'web';
|
||||
update field_override set source = 'agent' where source = 'web';
|
||||
alter table metadata_field drop constraint metadata_field_source_check;
|
||||
alter table metadata_field add constraint metadata_field_source_check ${oneOf("source", OLD_SOURCES)};
|
||||
alter table field_override drop constraint field_override_source_check;
|
||||
alter table field_override add constraint field_override_source_check ${oneOf("source", OLD_SOURCES)};
|
||||
`,
|
||||
);
|
||||
}
|
||||
@@ -136,7 +136,7 @@ export type ExternalIdentityVerifyStatus = "unverified" | "ok" | "not_found" | "
|
||||
export type FeedSourcePurpose = "movie" | "series" | "variety" | "promotion" | "search" | "generic";
|
||||
export type FeedSourceKind = "rss" | "atom" | "list" | "query";
|
||||
export type FeedSourceHealthStatus = "unknown" | "healthy" | "degraded" | "failing" | "circuit_open" | "disabled";
|
||||
export type FieldOverrideSource = "tmdb" | "user" | "agent" | "nfo" | "site" | "import";
|
||||
export type FieldOverrideSource = "tmdb" | "user" | "agent" | "nfo" | "site" | "import" | "web";
|
||||
export type FieldOverrideActorType = "user" | "agent" | "system" | "api_key";
|
||||
export type IdentityCandidateStatus = "pending" | "resolved" | "dismissed" | "archived";
|
||||
export type IdentityCandidateReason = "ambiguous" | "no_match" | "proposed" | "lookup_failed" | "unmatched";
|
||||
@@ -160,7 +160,9 @@ export type MediaEntityKind = "movie" | "series" | "variety" | "collection" | "o
|
||||
export type MediaEntityStatus = "provisional" | "active" | "merged" | "archived";
|
||||
export type MediaServerInstanceKind = "emby" | "jellyfin";
|
||||
export type MediaServerInstanceHealthStatus = "unknown" | "healthy" | "degraded" | "offline";
|
||||
export type MetadataFieldSource = "tmdb" | "user" | "agent" | "nfo" | "site" | "import";
|
||||
export type MetadataEnrichmentStatus = "suggested" | "applied" | "failed";
|
||||
export type MetadataEnrichmentTrigger = "manual" | "auto" | "agent";
|
||||
export type MetadataFieldSource = "tmdb" | "user" | "agent" | "nfo" | "site" | "import" | "web";
|
||||
export type ObservationStatus = "new" | "linked" | "ignored" | "error";
|
||||
export type OrganizationFileResultRole = "main" | "subtitle" | "audio" | "bluray_dir" | "nfo" | "image" | "other";
|
||||
export type OrganizationFileResultMode =
|
||||
@@ -924,6 +926,24 @@ export interface MediaServerInstanceTable {
|
||||
updated_at: GeneratedTimestamptz;
|
||||
}
|
||||
|
||||
export interface MetadataEnrichmentTable {
|
||||
id: Generated<string>;
|
||||
identity_candidate_id: string | null;
|
||||
normalized_title: string;
|
||||
query: Json;
|
||||
status: Generated<MetadataEnrichmentStatus>;
|
||||
trigger: Generated<MetadataEnrichmentTrigger>;
|
||||
suggestion: Json | null;
|
||||
error: string | null;
|
||||
usage: Generated<Json>;
|
||||
media_entity_id: string | null;
|
||||
applied_at: Timestamptz | null;
|
||||
applied_by: string | null;
|
||||
created_by: string | null;
|
||||
created_at: GeneratedTimestamptz;
|
||||
updated_at: GeneratedTimestamptz;
|
||||
}
|
||||
|
||||
export interface MetadataFieldTable {
|
||||
id: Generated<string>;
|
||||
media_entity_id: string | null;
|
||||
@@ -936,6 +956,7 @@ export interface MetadataFieldTable {
|
||||
fetched_at: GeneratedTimestamptz;
|
||||
created_at: GeneratedTimestamptz;
|
||||
updated_at: GeneratedTimestamptz;
|
||||
evidence: Json | null;
|
||||
}
|
||||
|
||||
export interface NamingTemplateVersionTable {
|
||||
@@ -1671,6 +1692,7 @@ export interface DB {
|
||||
media_asset: MediaAssetTable;
|
||||
media_entity: MediaEntityTable;
|
||||
media_server_instance: MediaServerInstanceTable;
|
||||
metadata_enrichment: MetadataEnrichmentTable;
|
||||
metadata_field: MetadataFieldTable;
|
||||
naming_template_version: NamingTemplateVersionTable;
|
||||
observation: ObservationTable;
|
||||
@@ -1832,6 +1854,9 @@ export type MediaEntityUpdate = Updateable<MediaEntityTable>;
|
||||
export type MediaServerInstanceRow = Selectable<MediaServerInstanceTable>;
|
||||
export type NewMediaServerInstance = Insertable<MediaServerInstanceTable>;
|
||||
export type MediaServerInstanceUpdate = Updateable<MediaServerInstanceTable>;
|
||||
export type MetadataEnrichmentRow = Selectable<MetadataEnrichmentTable>;
|
||||
export type NewMetadataEnrichment = Insertable<MetadataEnrichmentTable>;
|
||||
export type MetadataEnrichmentUpdate = Updateable<MetadataEnrichmentTable>;
|
||||
export type MetadataFieldRow = Selectable<MetadataFieldTable>;
|
||||
export type NewMetadataField = Insertable<MetadataFieldTable>;
|
||||
export type MetadataFieldUpdate = Updateable<MetadataFieldTable>;
|
||||
|
||||
@@ -75,6 +75,8 @@ const EXPECTED_TABLES = [
|
||||
"identity_redirect",
|
||||
"metadata_field",
|
||||
"field_override",
|
||||
// META-1 元数据补全记录(0049)
|
||||
"metadata_enrichment",
|
||||
// W2-D Agent 装配(0021)
|
||||
"capability_invocation",
|
||||
"agent_run_message",
|
||||
|
||||
Reference in new issue
Block a user