diff --git a/apps/server/src/modules/sources-identity/capabilities.ts b/apps/server/src/modules/sources-identity/capabilities.ts new file mode 100644 index 00000000..718aba9e --- /dev/null +++ b/apps/server/src/modules/sources-identity/capabilities.ts @@ -0,0 +1,187 @@ +import type { CapabilityRegistry } from "@umt/agent"; +import { capabilitySpec, type ExternalEntityType, type SourceRelease } from "@umt/contracts"; +import { sql } from "@umt/db"; +import { normalizeTitle } from "@umt/domain"; +import { toApi, toApiList } from "./rows.ts"; +import type { SourcesRuntime } from "./runtime.ts"; +import { assignReleaseIdentity } from "./service/identity-admin.ts"; +import { resolveRelease } from "./service/resolve.ts"; +import { refreshReleaseDetail } from "./service/snapshot.ts"; +import { liveSearch } from "./service/workbench.ts"; + +// 本模块领域的能力处理器(规格一律取 contracts):catalog.lookup / release.search / release.inspect / identity.resolve。 +// identity.resolve 属 propose 类,授权回调由 W2-D 接入前会被注册中心拒绝(安全默认)。 + +export function registerSourcesCapabilities(registry: CapabilityRegistry, rt: () => SourcesRuntime): void { + registry.register(capabilitySpec("catalog.lookup"), async (input) => { + const { db } = rt().ctx; + const local: { entity: unknown; score: number; matchedAlias: string | null }[] = []; + if (input.externalId) { + const rows = await db + .selectFrom("external_identity as i") + .innerJoin("media_entity as e", "e.id", "i.media_entity_id") + .selectAll("e") + .where("i.provider", "=", input.externalId.provider) + .where("i.entity_type", "=", input.externalId.entityType) + .where("i.external_id", "=", input.externalId.externalId) + .where("i.status", "=", "accepted") + .execute(); + for (const e of rows) local.push({ entity: toApi(e), score: 1, matchedAlias: null }); + } + if (input.query) { + const n = normalizeTitle(input.query); + const rows = await sql & { score: number; matched: string }>` + select distinct on (e.id) e.*, greatest(similarity(a.normalized, ${n}), case when a.normalized = ${n} then 1 else 0 end) as score, + a.alias as matched + from title_alias a join media_entity e on e.id = a.media_entity_id + where e.status <> 'merged' and (a.normalized = ${n} or similarity(a.normalized, ${n}) >= 0.3) + ${input.kind ? sql`and e.kind = ${input.kind}` : sql``} + ${input.year ? sql`and (e.year is null or abs(e.year - ${input.year}) <= 1)` : sql``} + order by e.id, score desc`.execute(db); + for (const r of rows.rows.sort((a, b) => b.score - a.score).slice(0, input.limit)) { + const { score, matched, ...e } = r; + if (!local.some((l) => (l.entity as { id: string }).id === e.id)) { + local.push({ entity: toApi(e), score: Number(score), matchedAlias: matched }); + } + } + } + const remote: { + provider: string; + entityType: ExternalEntityType; + externalId: string; + title: string; + year: number | null; + }[] = []; + const tmdb = rt().tmdb(); + if (input.includeRemote && input.query && tmdb) { + for (const m of (await tmdb.searchAll(input.query, input.year ? { year: input.year } : {})).slice( + 0, + input.limit, + )) { + remote.push({ + provider: "tmdb", + entityType: m.entityType, + externalId: m.externalId, + title: m.title, + year: m.year, + }); + } + } + return { local: local as never, remote }; + }); + + registry.register(capabilitySpec("release.search"), async (input) => { + const r = rt(); + const { db } = r.ctx; + let q = db.selectFrom("source_release").selectAll(); + if (input.siteIds?.length) q = q.where("site_id", "in", input.siteIds); + if (input.mediaEntityId) q = q.where("media_entity_id", "=", input.mediaEntityId); + const text = input.query; + if (text) + q = q.where((eb) => eb.or([eb("title", "ilike", `%${text}%`), sql`similarity(title, ${text}) >= 0.3`])); + const run = () => q.orderBy("first_seen_at", "desc").limit(input.limit).execute(); + let rows = await run(); + const sites = await db + .selectFrom("site") + .selectAll() + .where("enabled", "=", true) + .$if(!!input.siteIds?.length, (b) => b.where("id", "in", input.siteIds!)) + .execute(); + const coverage: { siteId: string; fromSnapshot: boolean; live: boolean; error: string | null }[] = []; + const needLive = input.allowLiveSearch && !!text && rows.length === 0; + for (const s of sites) { + if (needLive) { + const res = await liveSearch(r, s, text!); + coverage.push({ siteId: s.id, fromSnapshot: false, live: res.error === null, error: res.error }); + } else coverage.push({ siteId: s.id, fromSnapshot: true, live: false, error: null }); + } + if (needLive) rows = await run(); + return { items: toApiList(rows), coverage }; + }); + + registry.register(capabilitySpec("release.inspect"), async (input) => { + const r = rt(); + if (input.refreshDetail) await refreshReleaseDetail(r, input.sourceReleaseId); + const row = await r.ctx.db + .selectFrom("source_release") + .selectAll() + .where("id", "=", input.sourceReleaseId) + .executeTakeFirst(); + if (!row) throw new Error("来源资源不存在"); + // 库存(media_asset)归 W2-C,这里只按作品只读查询 + const inventory = row.media_entity_id + ? await r.ctx.db + .selectFrom("media_asset") + .selectAll() + .where("media_entity_id", "=", row.media_entity_id) + .execute() + : []; + return { release: toApi(row), inventory: toApiList(inventory) } as never; + }); + + registry.register(capabilitySpec("identity.resolve"), async (input, ctx) => { + const r = rt(); + let srId = input.sourceReleaseId ?? null; + if (!srId && input.observationId) { + const o = await r.ctx.db + .selectFrom("observation") + .select(["source_release_id"]) + .where("id", "=", input.observationId) + .executeTakeFirst(); + srId = o?.source_release_id ?? null; + } + if (!srId) throw new Error("找不到来源资源"); + const actor = { type: "agent" as const, id: ctx.actorId }; + const c = input.candidate; + if (c?.mediaEntityId || (c?.provider && c.entityType && c.externalId)) { + // Agent 的提议不加人工锁,后续人工仍可改 + const res = await assignReleaseIdentity( + r, + srId, + c.mediaEntityId + ? { mediaEntityId: c.mediaEntityId, lock: false } + : { + external: { + provider: c.provider!, + entityType: c.entityType as "movie" | "tv", + externalId: c.externalId!, + }, + lock: false, + }, + actor, + ); + return { + status: "resolved" as const, + mediaEntityId: res.mediaEntityId, + identities: await identities(r, res.mediaEntityId), + decisionId: null, + }; + } + const res = await resolveRelease(r, { sourceReleaseId: srId, force: true }); + const status: "resolved" | "proposed" | "ambiguous" | "unidentifiable" = + res.outcome === "auto" || res.outcome === "skipped" || res.outcome === "skipped_locked" + ? res.mediaEntityId + ? "resolved" + : "unidentifiable" + : res.outcome === "proposed" + ? "proposed" + : res.outcome === "ambiguous" + ? "ambiguous" + : "unidentifiable"; + return { + status, + mediaEntityId: res.mediaEntityId, + identities: res.mediaEntityId ? await identities(r, res.mediaEntityId) : [], + decisionId: res.decisionId, + }; + }); +} + +async function identities(r: SourcesRuntime, entityId: string) { + const rows = await r.ctx.db + .selectFrom("external_identity") + .selectAll() + .where("media_entity_id", "=", entityId) + .execute(); + return toApiList(rows) as never[]; +} diff --git a/apps/server/src/modules/sources-identity/defs.ts b/apps/server/src/modules/sources-identity/defs.ts new file mode 100644 index 00000000..1b99ac6d --- /dev/null +++ b/apps/server/src/modules/sources-identity/defs.ts @@ -0,0 +1,303 @@ +import { + ContentUnit, + ContentUnitUpdate, + CursorQuery, + cursorPage, + Decision, + DecisionKind, + DecisionResolveInput, + DecisionStatus, + ExternalIdentity, + ExternalIdentityCreate, + ExternalIdentityUpdate, + FeedHealth, + FeedSource, + FeedSourceCreate, + FeedSourceUpdate, + Id, + IdentityRevision, + IdentityStatus, + IdParam, + MediaEntity, + MediaEntityCreate, + MediaEntityMerge, + MediaEntityStatus, + MediaEntityUpdate, + MediaKind, + Observation, + ObservationStatus, + OkResult, + ResolutionStatus, + Site, + SiteAccount, + SiteAccountCreate, + SiteAccountUpdate, + SiteCreate, + SiteUpdate, + SourceRelease, + TitleAlias, + TitleAliasCreate, +} from "@umt/contracts"; +import { z } from "zod"; + +// 与 contracts Api(crud())同路径、同 scope 的端点定义,但保留具体 schema 类型供路由推断入参。 +// scope:读 = read;写 = admin(站点/Feed 配置)或 organize.execute(身份纠错、待处理)。 + +const list = ( + path: string, + summary: string, + row: R, + filters: F, + tags: string[], +) => ({ + method: "get" as const, + path, + scope: "read" as const, + summary, + tags, + query: CursorQuery.extend(filters), + response: cursorPage(row), +}); +const get = (path: string, summary: string, row: R, tags: string[]) => ({ + method: "get" as const, + path, + scope: "read" as const, + summary, + tags, + params: IdParam, + response: row, +}); +const write = ( + method: M, + path: string, + scope: "admin" | "organize.execute", + summary: string, + body: B, + response: R, + tags: string[], + withId = true, +) => ({ + method, + path, + scope, + summary, + tags, + ...(withId ? { params: IdParam } : {}), + body: body as B, + response, +}); + +const TS = ["sites"]; +const TF = ["feed-sources"]; +const TR = ["source-releases"]; +const TE = ["media-entities"]; +const TI = ["identity"]; +const TD = ["decisions"]; + +export const D = { + sites: { + list: list("/sites", "站点列表", Site, {}, TS), + get: get("/sites/{id}", "站点详情", Site, TS), + create: { + method: "post" as const, + path: "/sites", + scope: "admin" as const, + summary: "登记站点", + tags: TS, + body: SiteCreate, + response: Site, + }, + update: write("patch", "/sites/{id}", "admin", "修改站点(启停、快照周期、限流)", SiteUpdate, Site, TS), + }, + siteAccounts: { + list: list("/site-accounts", "站点账号列表", SiteAccount, { siteId: Id.optional() }, TS), + get: get("/site-accounts/{id}", "站点账号详情", SiteAccount, TS), + create: { + method: "post" as const, + path: "/site-accounts", + scope: "admin" as const, + summary: "新建站点账号(凭证只写,存为 secret)", + tags: TS, + body: SiteAccountCreate, + response: SiteAccount, + }, + update: write( + "patch", + "/site-accounts/{id}", + "admin", + "修改站点账号 / 轮换凭证", + SiteAccountUpdate, + SiteAccount, + TS, + ), + }, + feedSources: { + list: list("/feed-sources", "Feed 列表", FeedSource, { siteId: Id.optional(), health: FeedHealth.optional() }, TF), + get: get("/feed-sources/{id}", "Feed 详情", FeedSource, TF), + create: { + method: "post" as const, + path: "/feed-sources", + scope: "admin" as const, + summary: "新建 Feed(url 只写,存为 secret;对外只给脱敏 URL)", + tags: TF, + body: FeedSourceCreate, + response: FeedSource, + }, + update: write("patch", "/feed-sources/{id}", "admin", "修改 Feed", FeedSourceUpdate, FeedSource, TF), + fetchNow: write( + "post", + "/feed-sources/{id}/fetch", + "admin", + "立即抓取(仍受站点限流)", + undefined, + z.object({ taskId: Id }), + TF, + ), + }, + sourceReleases: { + list: list( + "/source-releases", + "站点资源列表", + SourceRelease, + { + siteId: Id.optional(), + mediaEntityId: Id.optional(), + resolutionStatus: ResolutionStatus.optional(), + q: z.string().max(200).optional(), + }, + TR, + ), + get: get("/source-releases/{id}", "站点资源详情", SourceRelease, TR), + }, + observations: { + list: list( + "/observations", + "观察列表", + Observation, + { feedId: Id.optional(), status: ObservationStatus.optional() }, + TR, + ), + get: get("/observations/{id}", "观察详情", Observation, TR), + }, + mediaEntities: { + list: list( + "/media-entities", + "作品列表", + MediaEntity, + { kind: MediaKind.optional(), status: MediaEntityStatus.optional(), q: z.string().max(200).optional() }, + TE, + ), + get: get("/media-entities/{id}", "作品(旧 ID 自动跳转)", MediaEntity, TE), + create: { + method: "post" as const, + path: "/media-entities", + scope: "organize.execute" as const, + summary: "手动建本地作品(TMDB 无记录时可先建档)", + tags: TE, + body: MediaEntityCreate, + response: MediaEntity, + }, + update: write( + "patch", + "/media-entities/{id}", + "organize.execute", + "修改作品(标题/原名/年份按字段覆盖并锁定)", + MediaEntityUpdate, + MediaEntity, + TE, + ), + merge: write( + "post", + "/media-entities/{id}/merge", + "organize.execute", + "合并重复作品(保留旧 ID 跳转)", + MediaEntityMerge, + MediaEntity, + TE, + ), + }, + contentUnits: { + list: list( + "/content-units", + "内容单元列表", + ContentUnit, + { mediaEntityId: Id.optional(), seasonNumber: z.coerce.number().int().optional() }, + TE, + ), + get: get("/content-units/{id}", "内容单元详情", ContentUnit, TE), + update: write( + "patch", + "/content-units/{id}", + "organize.execute", + "修改内容单元(改编号即确认并锁定)", + ContentUnitUpdate, + ContentUnit, + TE, + ), + }, + titleAliases: { + list: list( + "/title-aliases", + "别名列表", + TitleAlias, + { mediaEntityId: Id.optional(), q: z.string().max(200).optional() }, + TI, + ), + create: write("post", "/title-aliases", "organize.execute", "新增别名", TitleAliasCreate, TitleAlias, TI, false), + remove: write("delete", "/title-aliases/{id}", "organize.execute", "删除别名", undefined, OkResult, TI), + }, + externalIdentities: { + list: list( + "/external-identities", + "外部身份列表", + ExternalIdentity, + { mediaEntityId: Id.optional(), provider: z.string().optional(), status: IdentityStatus.optional() }, + TI, + ), + get: get("/external-identities/{id}", "外部身份详情", ExternalIdentity, TI), + create: write( + "post", + "/external-identities", + "organize.execute", + "新增外部身份(accepted 等同人工纠错)", + ExternalIdentityCreate, + ExternalIdentity, + TI, + false, + ), + update: write( + "patch", + "/external-identities/{id}", + "organize.execute", + "修订外部身份(乐观并发)", + ExternalIdentityUpdate, + ExternalIdentity, + TI, + ), + revisions: get( + "/external-identities/{id}/revisions", + "身份修订历史", + z.object({ items: z.array(IdentityRevision) }), + TI, + ), + }, + decisions: { + list: list( + "/decisions", + "待处理事项(缺省只列未关闭)", + Decision, + { status: DecisionStatus.optional(), kind: DecisionKind.optional() }, + TD, + ), + get: get("/decisions/{id}", "待处理事项详情", Decision, TD), + resolve: write( + "post", + "/decisions/{id}/resolve", + "organize.execute", + "处理待处理事项", + DecisionResolveInput, + Decision, + TD, + ), + }, +}; diff --git a/apps/server/src/modules/sources-identity/http.ts b/apps/server/src/modules/sources-identity/http.ts new file mode 100644 index 00000000..cfd6a798 --- /dev/null +++ b/apps/server/src/modules/sources-identity/http.ts @@ -0,0 +1,69 @@ +import { createRoute, type RouteConfig } from "@hono/zod-openapi"; +import { type ApiScope, apiResponse } from "@umt/contracts"; +import type { Context } from "hono"; +import type { z } from "zod"; +import { type AuthEnv, requireScope } from "../../http/auth.ts"; +import type { ModuleApp } from "../types.ts"; +import type { Actor } from "./service/common.ts"; + +// 本模块路由封装:createRoute(进 OpenAPI)+ requireScope + 入参校验 + 出参按合同 parse。 + +export interface RouteDef { + method: "get" | "post" | "patch" | "put" | "delete"; + path: string; + scope: ApiScope; + summary: string; + tags: string[]; + params?: z.ZodObject; + query?: z.ZodType; + body?: z.ZodType; + response: z.ZodType; +} + +type Infer = T extends z.ZodType ? z.infer : Record; + +export interface Input { + params: Infer; + query: Infer; + body: Infer; + actor: Actor; + c: Context; +} + +/** 调用者(W2-A 的 c.var.actor:admin 会话 / API Key)→ 审计/修订用的操作者;管理员记为 user */ +export function actorOf(c: Context): Actor { + const a = c.get("actor") as { type?: string; id?: string } | undefined; + if (!a) return { type: "system", id: null }; + return { type: a.type === "api_key" ? "api_key" : "user", id: a.id ?? null }; +} + +export function route(app: ModuleApp, def: D, handler: (input: Input) => Promise) { + const request: RouteConfig["request"] = {}; + if (def.params) request.params = def.params as never; + if (def.query) request.query = def.query as never; + if (def.body) request.body = { content: { "application/json": { schema: def.body as never } }, required: true }; + const r = createRoute({ + method: def.method, + path: def.path, + summary: def.summary, + tags: def.tags, + middleware: [requireScope(def.scope)] as never, + request, + responses: { + 200: { description: def.summary, content: { "application/json": { schema: apiResponse(def.response) } } }, + }, + }); + // biome-ignore lint/suspicious/noExplicitAny: 路由泛型在封装里擦除,入参已由 zod-openapi 校验 + app.openapi(r, (async (c: any) => { + const input = { + params: def.params ? c.req.valid("param") : {}, + query: def.query ? c.req.valid("query") : {}, + body: def.body ? c.req.valid("json") : {}, + actor: actorOf(c), + c, + } as Input; + const data = def.response.parse(await handler(input)); + return c.json({ ok: true, data }, 200); + // biome-ignore lint/suspicious/noExplicitAny: 同上 + }) as any); +} diff --git a/apps/server/src/modules/sources-identity/index.ts b/apps/server/src/modules/sources-identity/index.ts index 10b93630..1dc56727 100644 --- a/apps/server/src/modules/sources-identity/index.ts +++ b/apps/server/src/modules/sources-identity/index.ts @@ -1,12 +1,69 @@ -import type { ServerModule } from "../types.ts"; +import type { ServerContext, ServerModule } from "../types.ts"; +import { registerSourcesCapabilities } from "./capabilities.ts"; +import { decisionRoutes } from "./routes/decisions.ts"; +import { identityRoutes } from "./routes/identity.ts"; +import { sourcesRoutes } from "./routes/sources.ts"; +import { workbenchRoutes } from "./routes/workbench.ts"; +import { type RuntimeOverrides, SourcesRuntime } from "./runtime.ts"; +import { schedulerTick } from "./scheduler.ts"; +import { registerSourcesTasks, sourcesOutboxRoutes } from "./tasks.ts"; /** - * W2-B 来源/身份/元数据:site、site_account、feed_source;ingestFeed / resolveObservation / refreshMetadata; + * W2-B 来源/身份/元数据:site、site_account、feed_source;ingestFeed / resolveObservation / refreshMetadata / snapshotSite; * observation/source_release 写入与去重;身份解析、待识别池与 decision;站点工作台 API。 - * 发出 release.resolved、decision.created(经 ctx.outbox.emit)。 + * 发出 feed.due、feed.gap_suspected、release.observed(自消费)与 release.resolved、decision.created/updated。 */ -export const sourcesIdentityModule: ServerModule = { - name: "sources-identity", - registerRoutes() {}, - registerTasks() {}, -}; +export function createSourcesIdentityModule(overrides: RuntimeOverrides = {}): ServerModule & { + runtime(ctx: ServerContext): SourcesRuntime; +} { + const runtimes = new WeakMap(); + const runtime = (ctx: ServerContext) => { + let rt = runtimes.get(ctx); + if (!rt) { + rt = new SourcesRuntime(ctx, overrides); + runtimes.set(ctx, rt); + } + return rt; + }; + let timer: ReturnType | null = null; + let ticking: Promise | null = null; + + return { + name: "sources-identity", + runtime, + registerRoutes(app, ctx) { + const rt = () => runtime(ctx); + sourcesRoutes(app, rt); + workbenchRoutes(app, rt); + identityRoutes(app, rt); + decisionRoutes(app, rt); + }, + registerTasks(registry, ctx) { + registerSourcesTasks(registry, () => runtime(ctx)); + }, + outboxRoutes: () => sourcesOutboxRoutes, + registerCapabilities(registry, ctx) { + registerSourcesCapabilities(registry, () => runtime(ctx)); + }, + start(ctx) { + const rt = runtime(ctx); + const tick = () => { + if (ticking) return; + ticking = schedulerTick(rt) + .catch((err) => ctx.logger.warn({ err }, "sources-identity 调度失败")) + .finally(() => { + ticking = null; + }); + }; + timer = setInterval(tick, overrides.schedulerIntervalMs ?? 30_000); + tick(); + }, + async stop() { + if (timer) clearInterval(timer); + timer = null; + await ticking; + }, + }; +} + +export const sourcesIdentityModule = createSourcesIdentityModule(); diff --git a/apps/server/src/modules/sources-identity/routes/decisions.ts b/apps/server/src/modules/sources-identity/routes/decisions.ts new file mode 100644 index 00000000..b57fb259 --- /dev/null +++ b/apps/server/src/modules/sources-identity/routes/decisions.ts @@ -0,0 +1,184 @@ +import { type Decision, DecisionKind, DecisionStatus, Id, type PoolCandidate } from "@umt/contracts"; +import { sql } from "@umt/db"; +import { z } from "zod"; +import { ApiException } from "../../../http/errors.ts"; +import type { ModuleApp } from "../../types.ts"; +import { D } from "../defs.ts"; +import { route } from "../http.ts"; +import { decodeCursor, jsonb, page, toApi, toApiList } from "../rows.ts"; +import type { SourcesRuntime } from "../runtime.ts"; +import { audit } from "../service/common.ts"; +import { assignReleaseIdentity, dismissPoolEntry } from "../service/identity-admin.ts"; + +// 待处理中心数据源:decision 列表/详情/聚合/处理。 +// 身份歧义类由本模块直接执行所选候选;其它类型只记录处理结果并发 decision.updated, +// 需要执行能力的选项由 W2-D 的 resolveDecision 消费。 + +const OPEN = ["open", "investigating", "awaiting_user"] as const; +const T = ["decisions"]; + +const SummaryGroup = z.object({ + kind: DecisionKind, + /** 关联作品(身份歧义等尚未归属作品的为 null) */ + mediaEntityId: Id.nullable(), + mediaEntityTitle: z.string().nullable(), + count: z.number().int(), + maxSeverity: z.enum(["info", "warning", "critical"]), + latestAt: z.string(), +}); + +export function decisionRoutes(app: ModuleApp, rt: () => SourcesRuntime) { + const db = () => rt().ctx.db; + + route(app, D.decisions.list, async ({ query }) => { + const off = decodeCursor(query.cursor); + let q = db().selectFrom("decision").selectAll(); + q = query.status ? q.where("status", "=", query.status) : q.where("status", "in", [...OPEN]); + if (query.kind) q = q.where("kind", "=", query.kind); + const rows = await q + .orderBy(sql`case severity when 'critical' then 0 when 'warning' then 1 else 2 end`) + .orderBy("created_at", "desc") + .orderBy("id") + .offset(off) + .limit(query.limit + 1) + .execute(); + const p = page(rows, off, query.limit); + return { items: toApiList(p.items), nextCursor: p.nextCursor }; + }); + + route( + app, + { + method: "get", + path: "/decisions/summary", + scope: "read", + summary: "未关闭待处理事项按作品与问题类型聚合", + tags: T, + query: z.object({ status: DecisionStatus.optional() }), + response: z.object({ total: z.number().int(), groups: z.array(SummaryGroup) }), + }, + async ({ query }) => { + const statuses = query.status ? [query.status] : [...OPEN]; + // 作品归属:evidence.mediaEntityId,或 subject 为 media_entity,或 subject 为已识别的 source_release + const rows = await sql<{ + kind: z.infer; + entity_id: string | null; + entity_title: string | null; + count: number; + max_sev: number; + latest: Date; + }>` + with d as ( + select d.*, coalesce( + case when d.evidence->>'mediaEntityId' ~ '^[0-9a-f-]{36}$' then (d.evidence->>'mediaEntityId')::uuid end, + case when d.subject_type = 'media_entity' then d.subject_id end, + (select r.media_entity_id from source_release r where d.subject_type = 'source_release' and r.id = d.subject_id) + ) as entity_id + from decision d where d.status = any(${statuses}) + ) + select d.kind, d.entity_id, e.title as entity_title, count(*)::int as count, + max(case d.severity when 'critical' then 2 when 'warning' then 1 else 0 end)::int as max_sev, + max(d.created_at) as latest + from d left join media_entity e on e.id = d.entity_id + group by d.kind, d.entity_id, e.title + order by max_sev desc, latest desc`.execute(db()); + const sev = ["info", "warning", "critical"] as const; + return { + total: rows.rows.reduce((a, r) => a + r.count, 0), + groups: rows.rows.map((r) => ({ + kind: r.kind, + mediaEntityId: r.entity_id, + mediaEntityTitle: r.entity_title, + count: r.count, + maxSeverity: sev[r.max_sev] ?? "info", + latestAt: new Date(r.latest).toISOString(), + })), + }; + }, + ); + + route(app, D.decisions.get, async ({ params }) => { + const row = await db().selectFrom("decision").selectAll().where("id", "=", params.id).executeTakeFirst(); + if (!row) throw new ApiException("not_found", "待处理事项不存在"); + return toApi(row); + }); + + route(app, D.decisions.resolve, async ({ params, body, actor }) => { + const d = await db().selectFrom("decision").selectAll().where("id", "=", params.id).executeTakeFirst(); + if (!d) throw new ApiException("not_found", "待处理事项不存在"); + if (!(OPEN as readonly string[]).includes(d.status)) throw new ApiException("conflict", `事项已是 ${d.status}`); + if (!body.dismiss && !body.optionKey) throw new ApiException("bad_request", "需要 optionKey 或 dismiss"); + const by = `${actor.type}:${actor.id ?? ""}`; + const options = (d.options ?? []) as { key: string; input?: Record }[]; + const opt = body.optionKey ? options.find((o) => o.key === body.optionKey) : undefined; + if (body.optionKey && !opt) throw new ApiException("bad_request", `未知选项 ${body.optionKey}`); + + // 身份歧义:直接执行(选候选 = 人工指定并加锁;忽略 = 待识别池 dismissed) + if (d.kind === "identity_ambiguous" && d.subject_type === "source_release" && d.subject_id) { + if (body.dismiss || opt?.key === "dismiss") { + const pool = await db() + .selectFrom("identity_candidate") + .select(["id"]) + .where("source_release_id", "=", d.subject_id) + .executeTakeFirst(); + if (pool) await dismissPoolEntry(rt(), pool.id, actor, body.note ?? null); + } else if (opt?.input && typeof opt.input.candidateIndex === "number") { + const pool = await db() + .selectFrom("identity_candidate") + .select(["candidates"]) + .where("source_release_id", "=", d.subject_id) + .executeTakeFirst(); + if (!(pool?.candidates as PoolCandidate[] | undefined)?.[opt.input.candidateIndex]) { + throw new ApiException("conflict", "候选已变化,请刷新后重试"); + } + await assignReleaseIdentity( + rt(), + d.subject_id, + { candidateIndex: opt.input.candidateIndex, lock: true, reason: body.note ?? `decision ${d.id}` }, + actor, + ); + } + } + // 上面的执行路径已按 dedupeKey 关闭;其余类型在这里记录处理结果 + const row = await db() + .transaction() + .execute(async (trx) => { + const cur = await trx + .selectFrom("decision") + .selectAll() + .where("id", "=", params.id) + .forUpdate() + .executeTakeFirstOrThrow(); + if (!(OPEN as readonly string[]).includes(cur.status)) return cur; + const status = body.dismiss || opt?.key === "dismiss" ? "dismissed" : "resolved"; + const r = await trx + .updateTable("decision") + .set({ + status, + resolution: jsonb({ optionKey: body.optionKey ?? null, note: body.note ?? null, by }), + resolved_by: by, + resolved_at: new Date(), + }) + .where("id", "=", params.id) + .returningAll() + .executeTakeFirstOrThrow(); + await rt().ctx.outbox.emit( + trx, + "decision.updated", + { decisionId: r.id, status }, + { + idempotencyKey: `decision.updated:${r.id}:${status}`, + }, + ); + await audit( + trx, + actor, + "decision.resolved", + { type: "decision", id: r.id }, + { optionKey: body.optionKey ?? null, status }, + ); + return r; + }); + return toApi(row); + }); +} diff --git a/apps/server/src/modules/sources-identity/routes/identity.ts b/apps/server/src/modules/sources-identity/routes/identity.ts new file mode 100644 index 00000000..9c8f2f87 --- /dev/null +++ b/apps/server/src/modules/sources-identity/routes/identity.ts @@ -0,0 +1,597 @@ +import { randomUUID } from "node:crypto"; +import { + type ContentUnit, + cursorPage, + EntityIdentitySet, + EntityMetadataField, + type ExternalIdentity, + FieldOverrideInput, + Id, + IdentityCandidate, + IdentityPoolItem, + IdentityPoolQuery, + IdParam, + type MediaEntity, + MediaEntityDetail, + OkResult, + ReleaseIdentityAssign, + type TitleAlias, +} from "@umt/contracts"; +import { sql } from "@umt/db"; +import { normalizeTitle } from "@umt/domain"; +import { z } from "zod"; +import { ApiException } from "../../../http/errors.ts"; +import type { ModuleApp } from "../../types.ts"; +import { D } from "../defs.ts"; +import { route } from "../http.ts"; +import { decodeCursor, jsonb, page, toApi, toApiList } from "../rows.ts"; +import type { SourcesRuntime } from "../runtime.ts"; +import { audit } from "../service/common.ts"; +import { resolveEntityId } from "../service/entities.ts"; +import { + assignReleaseIdentity, + dismissPoolEntry, + mergeEntities, + setEntityIdentity, + updateExternalIdentity, +} from "../service/identity-admin.ts"; +import { clearFieldOverride, setFieldOverride } from "../service/metadata.ts"; + +// 身份与元数据 API:作品(列表/详情/纠错/合并/字段覆盖)、内容单元、别名、外部身份、待识别池。 + +const T_ENT = ["media-entities"]; +const T_ID = ["identity"]; +const Accepted = z.object({ taskId: Id }); +const FieldParam = z.object({ id: Id, field: EntityMetadataField }); + +export function identityRoutes(app: ModuleApp, rt: () => SourcesRuntime) { + const db = () => rt().ctx.db; + + // ── 作品 ── + route(app, D.mediaEntities.list, async ({ query }) => { + const off = decodeCursor(query.cursor); + let q = db().selectFrom("media_entity").selectAll(); + if (query.kind) q = q.where("kind", "=", query.kind); + q = query.status ? q.where("status", "=", query.status) : q.where("status", "<>", "merged"); + if (query.q) { + const n = normalizeTitle(query.q); + q = q.where((eb) => + eb.or([ + eb("title", "ilike", `%${query.q}%`), + eb.exists( + eb + .selectFrom("title_alias as a") + .select("a.id") + .whereRef("a.media_entity_id", "=", "media_entity.id") + .where(sql`(a.normalized = ${n} or similarity(a.normalized, ${n}) >= 0.4)`), + ), + ]), + ); + } + const rows = await q + .orderBy("updated_at", "desc") + .orderBy("id") + .offset(off) + .limit(query.limit + 1) + .execute(); + const p = page(rows, off, query.limit); + return { items: toApiList(p.items), nextCursor: p.nextCursor }; + }); + route(app, D.mediaEntities.get, async ({ params }) => { + const id = await resolveEntityId(db(), params.id); + if (!id) throw new ApiException("not_found", "作品不存在"); + return toApi(await db().selectFrom("media_entity").selectAll().where("id", "=", id).executeTakeFirstOrThrow()); + }); + route( + app, + { + method: "get", + path: "/media-entities/{id}/detail", + scope: "read", + summary: "作品详情:外部身份、别名、内容单元、字段来源与覆盖(旧 ID 自动跳转)", + tags: T_ENT, + params: IdParam, + response: MediaEntityDetail, + }, + async ({ params }) => { + const id = await resolveEntityId(db(), params.id); + if (!id) throw new ApiException("not_found", "作品不存在"); + const [entity, identities, aliases, units, fields, overrides] = await Promise.all([ + db().selectFrom("media_entity").selectAll().where("id", "=", id).executeTakeFirstOrThrow(), + db() + .selectFrom("external_identity") + .selectAll() + .where("media_entity_id", "=", id) + .orderBy("created_at") + .execute(), + db().selectFrom("title_alias").selectAll().where("media_entity_id", "=", id).orderBy("created_at").execute(), + db() + .selectFrom("content_unit") + .selectAll() + .where("media_entity_id", "=", id) + .orderBy("season_number") + .orderBy("episode_number") + .execute(), + db().selectFrom("metadata_field").selectAll().where("media_entity_id", "=", id).orderBy("field").execute(), + db().selectFrom("field_override").selectAll().where("media_entity_id", "=", id).orderBy("field").execute(), + ]); + return { + entity: toApi(entity), + redirectedFrom: id === params.id ? null : params.id, + identities: toApiList(identities), + aliases: toApiList(aliases), + contentUnits: toApiList(units), + metadataFields: toApiList(fields), + overrides: toApiList(overrides), + }; + }, + ); + route(app, D.mediaEntities.create, async ({ body, actor }) => { + const row = await db() + .transaction() + .execute(async (trx) => { + const r = await trx + .insertInto("media_entity") + .values({ + kind: body.kind, + title: body.title, + original_title: body.originalTitle ?? null, + year: body.year ?? null, + status: body.status === "merged" ? "active" : (body.status ?? "active"), + metadata: jsonb(body.metadata ?? {}), + field_locks: body.fieldLocks ?? [], + }) + .returningAll() + .executeTakeFirstOrThrow(); + await sql`insert into title_alias (media_entity_id, alias, normalized, kind, source, locked) + values (${r.id}, ${body.title}, ${normalizeTitle(body.title) || body.title}, 'translated', 'user', true) + on conflict do nothing`.execute(trx); + await audit(trx, actor, "media_entity.created", { type: "media_entity", id: r.id }, { title: body.title }); + return r; + }); + return toApi(row); + }); + route(app, D.mediaEntities.update, async ({ params, body, actor }) => { + const id = await resolveEntityId(db(), params.id); + if (!id) throw new ApiException("not_found", "作品不存在"); + // 标题/原名/年份按字段覆盖处理(加锁),其余字段直接更新 + const locked: [z.infer, unknown][] = []; + if (body.title !== undefined) locked.push(["title", body.title]); + if (body.originalTitle !== undefined) locked.push(["originalTitle", body.originalTitle]); + if (body.year !== undefined) locked.push(["year", body.year]); + for (const [field, value] of locked) await setFieldOverride(rt(), id, field, { value, lock: true }, actor); + const patch: Record = {}; + if (body.kind !== undefined) patch.kind = body.kind; + if (body.status !== undefined) patch.status = body.status; + if (Object.keys(patch).length) { + await db().updateTable("media_entity").set(patch).where("id", "=", id).execute(); + await audit(db(), actor, "media_entity.updated", { type: "media_entity", id }, patch); + } + return toApi(await db().selectFrom("media_entity").selectAll().where("id", "=", id).executeTakeFirstOrThrow()); + }); + route(app, D.mediaEntities.merge, async ({ params, body, actor }) => + toApi(await mergeEntities(rt(), params.id, body.intoId, body.reason ?? null, actor)), + ); + route( + app, + { + method: "post", + path: "/media-entities/{id}/identities", + scope: "organize.execute", + summary: "人工纠错外部映射:旧映射退役(留修订),新映射 accepted + 人工锁", + tags: T_ID, + params: IdParam, + body: EntityIdentitySet, + response: z.object({ + mediaEntityId: Id, + changed: z.boolean(), + identityId: Id.optional(), + retired: z.array(Id).optional(), + }), + }, + async ({ params, body, actor }) => setEntityIdentity(rt(), params.id, body, actor), + ); + route( + app, + { + method: "post", + path: "/media-entities/{id}/refresh", + scope: "organize.execute", + summary: "刷新元数据(字段级;用户锁优先,不动路径)", + tags: T_ENT, + params: IdParam, + response: Accepted, + }, + async ({ params }) => { + const id = await resolveEntityId(db(), params.id); + if (!id) throw new ApiException("not_found", "作品不存在"); + const r = await rt().ctx.engine.enqueue( + "refreshMetadata", + { mediaEntityId: id }, + { idempotencyKey: `refreshMetadata:${id}:${Math.floor(Date.now() / 60_000)}`, traceId: randomUUID() }, + ); + return { taskId: r.runId }; + }, + ); + const FieldResult = z.object({ + mediaEntityId: Id, + changedFields: z.array(z.string()), + lockedFields: z.array(z.string()), + }); + route( + app, + { + method: "put", + path: "/media-entities/{id}/fields/{field}", + scope: "organize.execute", + summary: "覆盖作品字段并锁定(自动刷新不再改写)", + tags: T_ENT, + params: FieldParam, + body: FieldOverrideInput, + response: FieldResult, + }, + async ({ params, body, actor }) => setFieldOverride(rt(), params.id, params.field, body, actor), + ); + route( + app, + { + method: "delete", + path: "/media-entities/{id}/fields/{field}", + scope: "organize.execute", + summary: "取消字段覆盖与锁定", + tags: T_ENT, + params: FieldParam, + response: FieldResult, + }, + async ({ params, actor }) => clearFieldOverride(rt(), params.id, params.field, actor), + ); + + // ── 内容单元 ── + route(app, D.contentUnits.list, async ({ query }) => { + const off = decodeCursor(query.cursor); + let q = db().selectFrom("content_unit").selectAll(); + if (query.mediaEntityId) q = q.where("media_entity_id", "=", query.mediaEntityId); + if (query.seasonNumber !== undefined) q = q.where("season_number", "=", query.seasonNumber); + const rows = await q + .orderBy("media_entity_id") + .orderBy("season_number") + .orderBy("episode_number") + .orderBy("id") + .offset(off) + .limit(query.limit + 1) + .execute(); + const p = page(rows, off, query.limit); + return { items: toApiList(p.items), nextCursor: p.nextCursor }; + }); + route(app, D.contentUnits.get, async ({ params }) => { + const row = await db().selectFrom("content_unit").selectAll().where("id", "=", params.id).executeTakeFirst(); + if (!row) throw new ApiException("not_found", "内容单元不存在"); + return toApi(row); + }); + route(app, D.contentUnits.update, async ({ params, body, actor }) => { + const unit = await db().selectFrom("content_unit").selectAll().where("id", "=", params.id).executeTakeFirst(); + if (!unit) throw new ApiException("not_found", "内容单元不存在"); + const numbering = ["seasonNumber", "episodeNumber", "sourceIssue", "airDate"] as const; + const touchesNumbering = numbering.some((k) => body[k] !== undefined); + const patch: Record = {}; + const map: Record = { + parentUnitId: "parent_unit_id", + kind: "kind", + contentType: "content_type", + segment: "segment", + seasonNumber: "season_number", + episodeNumber: "episode_number", + sourceIssue: "source_issue", + airDate: "air_date", + displayOrder: "display_order", + title: "title", + status: "status", + numberingLocked: "numbering_locked", + fieldLocks: "field_locks", + }; + for (const [k, col] of Object.entries(map)) { + const v = (body as Record)[k]; + if (v !== undefined) patch[col] = v; + } + if (body.metadata !== undefined) patch.metadata = jsonb(body.metadata); + // 人工改编号即确认并锁定编号(REQ §5.4:编号可修订,人工确认后刷新不得改) + if (touchesNumbering && body.numberingLocked === undefined) patch.numbering_locked = true; + if (touchesNumbering && body.status === undefined && unit.status === "provisional") patch.status = "confirmed"; + const row = await db() + .updateTable("content_unit") + .set(patch) + .where("id", "=", params.id) + .returningAll() + .executeTakeFirstOrThrow(); + await audit( + db(), + actor, + "content_unit.updated", + { type: "content_unit", id: params.id }, + { fields: Object.keys(patch) }, + ); + return toApi(row); + }); + + // ── 别名 ── + route(app, D.titleAliases.list, async ({ query }) => { + const off = decodeCursor(query.cursor); + let q = db().selectFrom("title_alias").selectAll(); + if (query.mediaEntityId) q = q.where("media_entity_id", "=", query.mediaEntityId); + if (query.q) { + const n = normalizeTitle(query.q); + q = q.where(sql`(normalized = ${n} or similarity(normalized, ${n}) >= 0.3)`); + } + const rows = await q + .orderBy("created_at") + .orderBy("id") + .offset(off) + .limit(query.limit + 1) + .execute(); + const p = page(rows, off, query.limit); + return { items: toApiList(p.items), nextCursor: p.nextCursor }; + }); + route(app, D.titleAliases.create, async ({ body, actor }) => { + const id = await resolveEntityId(db(), body.mediaEntityId); + if (!id) throw new ApiException("not_found", "作品不存在"); + const normalized = normalizeTitle(body.alias); + if (!normalized) throw new ApiException("bad_request", "别名归一化后为空"); + try { + const row = await db() + .insertInto("title_alias") + .values({ + media_entity_id: id, + content_unit_id: body.contentUnitId ?? null, + alias: body.alias, + normalized, + language: body.language ?? null, + kind: body.kind ?? "other", + source: body.source, + locked: body.locked ?? body.source === "user", + }) + .returningAll() + .executeTakeFirstOrThrow(); + await audit(db(), actor, "title_alias.created", { type: "title_alias", id: row.id }, { alias: body.alias }); + return toApi(row); + } catch (e) { + if ((e as { code?: string }).code === "23505") throw new ApiException("conflict", "别名已存在"); + throw e; + } + }); + route(app, D.titleAliases.remove, async ({ params, actor }) => { + const r = await db().deleteFrom("title_alias").where("id", "=", params.id).returning(["id"]).executeTakeFirst(); + if (!r) throw new ApiException("not_found", "别名不存在"); + await audit(db(), actor, "title_alias.deleted", { type: "title_alias", id: params.id }); + return OkResult.parse({ ok: true }); + }); + + // ── 外部身份 ── + route(app, D.externalIdentities.list, async ({ query }) => { + const off = decodeCursor(query.cursor); + let q = db().selectFrom("external_identity").selectAll(); + if (query.mediaEntityId) q = q.where("media_entity_id", "=", query.mediaEntityId); + if (query.provider) q = q.where("provider", "=", query.provider); + if (query.status) q = q.where("status", "=", query.status); + const rows = await q + .orderBy("created_at") + .orderBy("id") + .offset(off) + .limit(query.limit + 1) + .execute(); + const p = page(rows, off, query.limit); + return { items: toApiList(p.items), nextCursor: p.nextCursor }; + }); + route(app, D.externalIdentities.get, async ({ params }) => { + const row = await db().selectFrom("external_identity").selectAll().where("id", "=", params.id).executeTakeFirst(); + if (!row) throw new ApiException("not_found", "外部身份不存在"); + return toApi(row); + }); + route(app, D.externalIdentities.create, async ({ body, actor }) => { + if (!body.mediaEntityId) throw new ApiException("bad_request", "v0.1 只支持作品级外部身份"); + const entityId = await resolveEntityId(db(), body.mediaEntityId); + if (!entityId) throw new ApiException("not_found", "作品不存在"); + if ((body.status ?? "proposed") === "accepted") { + const r = await setEntityIdentity( + rt(), + entityId, + { + provider: body.provider, + entityType: body.entityType as "movie" | "tv", + externalId: body.externalId, + lock: body.locked ?? true, + }, + actor, + ); + if (!r.identityId) throw new ApiException("conflict", "该外部身份已是此作品的 accepted 映射"); + return toApi( + await db().selectFrom("external_identity").selectAll().where("id", "=", r.identityId).executeTakeFirstOrThrow(), + ); + } + const row = await db() + .transaction() + .execute(async (trx) => { + const r = await trx + .insertInto("external_identity") + .values({ + media_entity_id: entityId, + provider: body.provider, + entity_type: body.entityType, + external_id: body.externalId, + status: body.status ?? "proposed", + evidence_level: body.evidenceLevel ?? "manual", + evidence: jsonb(body.evidence ?? {}), + locked: body.locked ?? false, + extra: jsonb(body.extra ?? {}), + }) + .returningAll() + .executeTakeFirstOrThrow(); + await trx + .insertInto("identity_revision") + .values({ + external_identity_id: r.id, + revision: 1, + action: "created", + to_status: r.status, + after: jsonb({ status: r.status }), + actor_type: actor.type, + actor_id: actor.id, + }) + .execute(); + return r; + }); + return toApi(row); + }); + route(app, D.externalIdentities.update, async ({ params, body, actor }) => + toApi(await updateExternalIdentity(rt(), params.id, body, actor)), + ); + route(app, D.externalIdentities.revisions, async ({ params }) => { + const rows = await db() + .selectFrom("identity_revision") + .selectAll() + .where("external_identity_id", "=", params.id) + .orderBy("revision") + .execute(); + return { items: toApiList(rows) }; + }); + + // ── 待识别池 ── + route( + app, + { + method: "get", + path: "/identity-pool", + scope: "read", + summary: "待识别池(可按原始标题检索;候选不建正式作品)", + tags: T_ID, + query: IdentityPoolQuery, + response: cursorPage(IdentityPoolItem), + }, + async ({ query }) => { + const off = decodeCursor(query.cursor); + let q = db() + .selectFrom("identity_candidate as p") + .innerJoin("source_release as r", "r.id", "p.source_release_id") + .selectAll("p") + .select([ + "r.site_id as r_site_id", + "r.site_release_id as r_site_release_id", + "r.title as r_title", + "r.subtitle as r_subtitle", + "r.category_raw as r_category_raw", + "r.first_seen_at as r_first_seen_at", + "r.resolution_status as r_resolution_status", + ]); + q = q.where("p.status", "=", query.status ?? "pending"); + if (query.reason) q = q.where("p.reason", "=", query.reason); + if (query.siteId) q = q.where("r.site_id", "=", query.siteId); + if (query.q) { + const n = normalizeTitle(query.q); + q = q.where((eb) => + eb.or([ + eb("r.title", "ilike", `%${query.q}%`), + eb("r.subtitle", "ilike", `%${query.q}%`), + sql`(p.query->>'normalized') = ${n}`, + sql`similarity(r.title, ${query.q}) >= 0.3`, + ]), + ); + } + const rows = await q + .orderBy("p.updated_at", "desc") + .orderBy("p.id") + .offset(off) + .limit(query.limit + 1) + .execute(); + const p = page(rows, off, query.limit); + return { + items: p.items.map((row) => { + const { + r_site_id, + r_site_release_id, + r_title, + r_subtitle, + r_category_raw, + r_first_seen_at, + r_resolution_status, + ...rest + } = row; + return { + ...toApi>(rest), + release: { + id: rest.source_release_id, + siteId: r_site_id, + siteReleaseId: r_site_release_id, + title: r_title, + subtitle: r_subtitle, + categoryRaw: r_category_raw, + firstSeenAt: r_first_seen_at.toISOString(), + resolutionStatus: r_resolution_status, + }, + }; + }), + nextCursor: p.nextCursor, + }; + }, + ); + route( + app, + { + method: "post", + path: "/identity-pool/{id}/assign", + scope: "organize.execute", + summary: "为待识别资源手动指定候选 / 本地作品 / 外部身份", + tags: T_ID, + params: IdParam, + body: ReleaseIdentityAssign, + response: z.object({ mediaEntityId: Id, contentUnitIds: z.array(Id) }), + }, + async ({ params, body, actor }) => { + const pool = await db() + .selectFrom("identity_candidate") + .select(["source_release_id"]) + .where("id", "=", params.id) + .executeTakeFirst(); + if (!pool) throw new ApiException("not_found", "待识别条目不存在"); + return assignReleaseIdentity(rt(), pool.source_release_id, body, actor); + }, + ); + route( + app, + { + method: "post", + path: "/identity-pool/{id}/dismiss", + scope: "organize.execute", + summary: "忽略待识别资源(不再自动复查)", + tags: T_ID, + params: IdParam, + body: z.object({ note: z.string().max(2000).optional() }), + response: IdentityCandidate, + }, + async ({ params, body, actor }) => toApi(await dismissPoolEntry(rt(), params.id, actor, body.note ?? null)), + ); + route( + app, + { + method: "post", + path: "/identity-pool/{id}/retry", + scope: "organize.execute", + summary: "立即重新识别", + tags: T_ID, + params: IdParam, + response: Accepted, + }, + async ({ params }) => { + const pool = await db() + .selectFrom("identity_candidate") + .select(["source_release_id", "attempts"]) + .where("id", "=", params.id) + .executeTakeFirst(); + if (!pool) throw new ApiException("not_found", "待识别条目不存在"); + const r = await rt().ctx.engine.enqueue( + "resolveObservation", + { sourceReleaseId: pool.source_release_id, force: true }, + { idempotencyKey: `resolve:retry:${params.id}:${Math.floor(Date.now() / 10_000)}`, traceId: randomUUID() }, + ); + return { taskId: r.runId }; + }, + ); +} diff --git a/apps/server/src/modules/sources-identity/routes/sources.ts b/apps/server/src/modules/sources-identity/routes/sources.ts new file mode 100644 index 00000000..ef15a375 --- /dev/null +++ b/apps/server/src/modules/sources-identity/routes/sources.ts @@ -0,0 +1,635 @@ +import { randomUUID } from "node:crypto"; +import { listSiteDefinitions, redactUrl, type SiteDefinition } from "@umt/adapters"; +import { + CursorQuery, + cursorPage, + FeedSource, + Id, + IdParam, + type Observation, + ReleaseIdentityAssign, + type SecretPurpose, + Site, + type SiteAccount, + type SiteAdapter, + SiteDefinitionView, + SiteSnapshot, + type SourceRelease, +} from "@umt/contracts"; +import { sql } from "@umt/db"; +import { z } from "zod"; +import { ApiException } from "../../../http/errors.ts"; +import type { ModuleApp } from "../../types.ts"; +import { D } from "../defs.ts"; +import { route } from "../http.ts"; +import { decodeCursor, jsonb, page, toApi, toApiList } from "../rows.ts"; +import type { SourcesRuntime } from "../runtime.ts"; +import { audit, closeDecisions } from "../service/common.ts"; +import { assignReleaseIdentity } from "../service/identity-admin.ts"; + +// 站点、站点账号、Feed、来源资源、观察的管理 API。凭证只写(存 secret,对外只给 secret_ref / 脱敏 URL)。 + +const T_SITES = ["sites"]; +const T_FEEDS = ["feed-sources"]; +const T_RELEASES = ["source-releases"]; + +export function adapterOf(def: SiteDefinition): SiteAdapter { + return def.umt.kind === "mteam" ? "mteam_api" : def.umt.kind === "api" ? "api" : "nexusphp"; +} + +const SECRET_PURPOSE: Record = { + cookie: "site_cookie", + passkey: "site_passkey", + api_key: "site_api_key", + password: "site_password", +}; + +const Accepted = z.object({ taskId: Id }); + +export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) { + const db = () => rt().ctx.db; + + // ── 站点注册表 ── + route( + app, + { + method: "get", + path: "/site-definitions", + scope: "read", + summary: "随产品发布的站点适配器定义(含是否已登记)", + tags: T_SITES, + response: z.object({ items: z.array(SiteDefinitionView) }), + }, + async () => { + const rows = await db().selectFrom("site").select(["id", "key"]).execute(); + const byKey = new Map(rows.map((r) => [r.key, r.id])); + return { + items: listSiteDefinitions().map((d) => ({ + key: d.id, + name: d.umt.kind === "nexusphp" ? (d.umt.displayName ?? d.name) : d.umt.displayName, + adapter: adapterOf(d), + baseUrl: d.domain, + shippedEnabled: d.enabled !== false, + definitionVersion: d.umt.version, + capabilities: d.umt.capabilities, + defaultRateLimit: d.umt.rateLimit, + authSupported: ("authSupported" in d.umt ? d.umt.authSupported : undefined) ?? [], + siteId: byKey.get(d.id) ?? null, + })), + }; + }, + ); + + // ── 站点 ── + route(app, D.sites.list, async ({ query }) => { + const off = decodeCursor(query.cursor); + const rows = await db() + .selectFrom("site") + .selectAll() + .orderBy("key") + .offset(off) + .limit(query.limit + 1) + .execute(); + const p = page(rows, off, query.limit); + return { items: toApiList(p.items), nextCursor: p.nextCursor }; + }); + route(app, D.sites.get, async ({ params }) => toApi(await getSite(params.id))); + route(app, D.sites.create, async ({ body, actor }) => { + const def = listSiteDefinitions().find((d) => d.id === body.key); + if (def && adapterOf(def) !== body.adapter) { + throw new ApiException("bad_request", `站点 ${body.key} 的适配器类型应为 ${adapterOf(def)}`); + } + if (!def && body.adapter !== "generic_rss") { + throw new ApiException("bad_request", `注册表中没有站点 ${body.key};未适配站点只能以 generic_rss 登记`); + } + try { + const row = await db() + .insertInto("site") + .values({ + key: body.key, + name: body.name, + adapter: body.adapter, + base_url: body.baseUrl, + enabled: body.enabled ?? true, + rate_limit: jsonb(body.rateLimit ?? {}), + config: jsonb(body.config ?? {}), + snapshot_enabled: body.snapshotEnabled ?? false, + snapshot_interval_sec: body.snapshotIntervalSec ?? null, + }) + .returningAll() + .executeTakeFirstOrThrow(); + await audit(db(), actor, "site.created", { type: "site", id: row.id }, { key: body.key }); + return toApi(row); + } catch (e) { + if ((e as { code?: string }).code === "23505") throw new ApiException("conflict", `站点 ${body.key} 已登记`); + throw e; + } + }); + route(app, D.sites.update, async ({ params, body, actor }) => { + await getSite(params.id); + const patch: Record = {}; + if (body.name !== undefined) patch.name = body.name; + if (body.adapter !== undefined) patch.adapter = body.adapter; + if (body.baseUrl !== undefined) patch.base_url = body.baseUrl; + if (body.rateLimit !== undefined) patch.rate_limit = jsonb(body.rateLimit); + if (body.config !== undefined) patch.config = jsonb(body.config); + if (body.snapshotEnabled !== undefined) patch.snapshot_enabled = body.snapshotEnabled; + if (body.snapshotIntervalSec !== undefined) patch.snapshot_interval_sec = body.snapshotIntervalSec; + if (body.enabled !== undefined) return toApi(await setSiteEnabled(params.id, body.enabled, actor)); + const row = await db() + .updateTable("site") + .set(patch) + .where("id", "=", params.id) + .returningAll() + .executeTakeFirstOrThrow(); + await audit(db(), actor, "site.updated", { type: "site", id: params.id }, { fields: Object.keys(patch) }); + return toApi(row); + }); + for (const enabled of [true, false]) { + route( + app, + { + method: "post", + path: `/sites/{id}/${enabled ? "enable" : "disable"}`, + scope: "admin", + summary: enabled ? "开启站点" : "关闭站点(停止网络访问与新候选,历史资源仍可查)", + tags: T_SITES, + params: IdParam, + response: Site, + }, + async ({ params, actor }) => toApi(await setSiteEnabled(params.id, enabled, actor)), + ); + } + route( + app, + { + method: "post", + path: "/sites/{id}/snapshot", + scope: "admin", + summary: "立即同步站点快照(受站点限流)", + tags: T_SITES, + params: IdParam, + body: z.object({ maxPages: z.number().int().min(1).max(20).optional() }), + response: Accepted, + }, + async ({ params, body }) => { + const s = await getSite(params.id); + if (!s.enabled) throw new ApiException("precondition_failed", "站点已关闭"); + const r = await rt().ctx.engine.enqueue( + "snapshotSite", + { siteId: s.id, kind: "manual", ...(body.maxPages ? { maxPages: body.maxPages } : {}) }, + { idempotencyKey: `snapshotSite:manual:${s.id}:${Math.floor(Date.now() / 10_000)}`, traceId: randomUUID() }, + ); + return { taskId: r.runId }; + }, + ); + route( + app, + { + method: "get", + path: "/sites/{id}/snapshots", + scope: "read", + summary: "站点快照/补查记录(覆盖范围与新鲜度)", + tags: T_SITES, + params: IdParam, + query: CursorQuery, + response: cursorPage(SiteSnapshot), + }, + async ({ params, query }) => { + const off = decodeCursor(query.cursor); + const rows = await db() + .selectFrom("site_snapshot") + .selectAll() + .where("site_id", "=", params.id) + .orderBy("started_at", "desc") + .offset(off) + .limit(query.limit + 1) + .execute(); + const p = page(rows, off, query.limit); + return { items: toApiList(p.items), nextCursor: p.nextCursor }; + }, + ); + + // ── 站点账号 ── + route(app, D.siteAccounts.list, async ({ query }) => { + const off = decodeCursor(query.cursor); + let q = db().selectFrom("site_account").selectAll(); + if (query.siteId) q = q.where("site_id", "=", query.siteId); + const rows = await q + .orderBy("created_at") + .offset(off) + .limit(query.limit + 1) + .execute(); + const p = page(rows, off, query.limit); + return { items: toApiList(p.items), nextCursor: p.nextCursor }; + }); + route(app, D.siteAccounts.get, async ({ params }) => { + const row = await db().selectFrom("site_account").selectAll().where("id", "=", params.id).executeTakeFirst(); + if (!row) throw new ApiException("not_found", "站点账号不存在"); + return toApi(row); + }); + route(app, D.siteAccounts.create, async ({ body, actor }) => { + await getSite(body.siteId); + const ref = await rt().ctx.secrets.create({ + purpose: SECRET_PURPOSE[body.authKind]!, + plaintext: body.credential, + label: `site_account:${body.label}`, + }); + try { + const row = await db() + .insertInto("site_account") + .values({ + site_id: body.siteId, + label: body.label, + auth_kind: body.authKind, + secret_ref: ref, + enabled: body.enabled ?? true, + }) + .returningAll() + .executeTakeFirstOrThrow(); + await audit(db(), actor, "site_account.created", { type: "site_account", id: row.id }, { siteId: body.siteId }); + return toApi(row); + } catch (e) { + await rt() + .ctx.secrets.remove(ref) + .catch(() => {}); + if ((e as { code?: string }).code === "23505") throw new ApiException("conflict", "同站点已有同名账号"); + throw e; + } + }); + route(app, D.siteAccounts.update, async ({ params, body, actor }) => { + const acc = await db().selectFrom("site_account").selectAll().where("id", "=", params.id).executeTakeFirst(); + if (!acc) throw new ApiException("not_found", "站点账号不存在"); + const patch: Record = {}; + if (body.label !== undefined) patch.label = body.label; + if (body.enabled !== undefined) patch.enabled = body.enabled; + if (body.authKind !== undefined) patch.auth_kind = body.authKind; + if (body.credential !== undefined) { + if (acc.secret_ref) await rt().ctx.secrets.replace(acc.secret_ref, body.credential); + else { + patch.secret_ref = await rt().ctx.secrets.create({ + purpose: SECRET_PURPOSE[body.authKind ?? acc.auth_kind]!, + plaintext: body.credential, + label: `site_account:${acc.label}`, + }); + } + // 换了凭证:状态回到未验证,关闭"登录失效"待处理,站点健康复位 + patch.status = "unknown"; + } + const row = await db() + .transaction() + .execute(async (trx) => { + const r = await trx + .updateTable("site_account") + .set(patch) + .where("id", "=", params.id) + .returningAll() + .executeTakeFirstOrThrow(); + if (body.credential !== undefined) { + await trx + .updateTable("site") + .set({ health_status: "unknown" }) + .where("id", "=", acc.site_id) + .where("health_status", "=", "failing") + .execute(); + await closeDecisions( + rt().ctx, + trx, + `site-auth:${acc.site_id}`, + "resolved", + { by: "credential_updated" }, + `${actor.type}:${actor.id}`, + ); + } + await audit( + trx, + actor, + "site_account.updated", + { type: "site_account", id: params.id }, + { + fields: Object.keys(patch).filter((k) => k !== "secret_ref"), + credentialRotated: body.credential !== undefined, + }, + ); + return r; + }); + return toApi(row); + }); + + // ── Feed ── + route(app, D.feedSources.list, async ({ query }) => { + const off = decodeCursor(query.cursor); + let q = db().selectFrom("feed_source").selectAll(); + if (query.siteId) q = q.where("site_id", "=", query.siteId); + if (query.health) q = q.where("health_status", "=", query.health); + const rows = await q + .orderBy("created_at") + .offset(off) + .limit(query.limit + 1) + .execute(); + const p = page(rows, off, query.limit); + return { items: toApiList(p.items), nextCursor: p.nextCursor }; + }); + route(app, D.feedSources.get, async ({ params }) => toApi(await getFeed(params.id))); + route(app, D.feedSources.create, async ({ body, actor }) => { + if (body.siteId) await getSite(body.siteId); + if (body.siteAccountId) await checkAccount(body.siteAccountId, body.siteId ?? null); + const ref = await rt().ctx.secrets.create({ purpose: "feed_url", plaintext: body.url, label: `feed:${body.name}` }); + const row = await db() + .insertInto("feed_source") + .values({ + site_id: body.siteId ?? null, + site_account_id: body.siteAccountId ?? null, + name: body.name, + purpose: body.purpose ?? "generic", + kind: body.kind ?? "rss", + url_secret_ref: ref, + url_display: redactUrl(body.url), + enabled: body.enabled ?? true, + poll_interval_sec: body.pollIntervalSec ?? 900, + filter: jsonb(body.filter ?? {}), + }) + .returningAll() + .executeTakeFirstOrThrow(); + await audit(db(), actor, "feed_source.created", { type: "feed_source", id: row.id }, { url: row.url_display }); + return toApi(row); + }); + route(app, D.feedSources.update, async ({ params, body, actor }) => { + const feed = await getFeed(params.id); + const patch: Record = {}; + if (body.siteId !== undefined) { + if (body.siteId) await getSite(body.siteId); + patch.site_id = body.siteId; + } + if (body.siteAccountId !== undefined) { + if (body.siteAccountId) await checkAccount(body.siteAccountId, body.siteId ?? feed.site_id); + patch.site_account_id = body.siteAccountId; + } + if (body.name !== undefined) patch.name = body.name; + if (body.purpose !== undefined) patch.purpose = body.purpose; + if (body.kind !== undefined) patch.kind = body.kind; + if (body.enabled !== undefined) { + patch.enabled = body.enabled; + patch.health_status = body.enabled ? "unknown" : "disabled"; + } + if (body.pollIntervalSec !== undefined) patch.poll_interval_sec = body.pollIntervalSec; + if (body.filter !== undefined) patch.filter = jsonb(body.filter); + if (body.url !== undefined) { + if (feed.url_secret_ref) await rt().ctx.secrets.replace(feed.url_secret_ref, body.url); + else + patch.url_secret_ref = await rt().ctx.secrets.create({ + purpose: "feed_url", + plaintext: body.url, + label: `feed:${feed.name}`, + }); + patch.url_display = redactUrl(body.url); + // 换了链接:清空条件请求缓存(游标保留,用于去重) + patch.etag = null; + patch.last_modified = null; + } + const row = await db() + .updateTable("feed_source") + .set(patch) + .where("id", "=", params.id) + .returningAll() + .executeTakeFirstOrThrow(); + await audit( + db(), + actor, + "feed_source.updated", + { type: "feed_source", id: params.id }, + { + fields: Object.keys(patch).filter((k) => k !== "url_secret_ref"), + }, + ); + return toApi(row); + }); + for (const enabled of [true, false]) { + route( + app, + { + method: "post", + path: `/feed-sources/{id}/${enabled ? "enable" : "disable"}`, + scope: "admin", + summary: enabled ? "启用 Feed" : "停用 Feed", + tags: T_FEEDS, + params: IdParam, + response: FeedSource, + }, + async ({ params, actor }) => { + await getFeed(params.id); + const row = await db() + .updateTable("feed_source") + .set({ + enabled, + health_status: enabled ? "unknown" : "disabled", + ...(enabled ? { next_fetch_at: new Date() } : {}), + }) + .where("id", "=", params.id) + .returningAll() + .executeTakeFirstOrThrow(); + await audit(db(), actor, enabled ? "feed_source.enabled" : "feed_source.disabled", { + type: "feed_source", + id: params.id, + }); + return toApi(row); + }, + ); + } + route( + app, + { + method: "post", + path: "/feed-sources/{id}/reset", + scope: "admin", + summary: "手动恢复:关闭熔断、清空退避(凭证已更新后使用)", + tags: T_FEEDS, + params: IdParam, + response: FeedSource, + }, + async ({ params, actor }) => { + const feed = await getFeed(params.id); + const cursor = (feed.cursor ?? {}) as { window?: unknown }; + const row = await db() + .transaction() + .execute(async (trx) => { + const r = await trx + .updateTable("feed_source") + .set({ + consecutive_failures: 0, + backoff_until: null, + last_error: null, + health_status: feed.enabled ? "unknown" : "disabled", + next_fetch_at: new Date(), + cursor: jsonb({ window: cursor.window ?? null }), + }) + .where("id", "=", params.id) + .returningAll() + .executeTakeFirstOrThrow(); + if (feed.site_account_id) { + await trx + .updateTable("site_account") + .set({ status: "unknown" }) + .where("id", "=", feed.site_account_id) + .where("status", "in", ["invalid", "expired"]) + .execute(); + } + await closeDecisions( + rt().ctx, + trx, + `feed-auth:${params.id}`, + "resolved", + { by: "reset" }, + `${actor.type}:${actor.id}`, + ); + await audit(trx, actor, "feed_source.reset", { type: "feed_source", id: params.id }); + return r; + }); + return toApi(row); + }, + ); + route(app, D.feedSources.fetchNow, async ({ params }) => { + const feed = await getFeed(params.id); + if (!feed.enabled) throw new ApiException("precondition_failed", "Feed 已停用"); + const r = await rt().ctx.engine.enqueue( + "ingestFeed", + { feedSourceId: feed.id, manual: true }, + { idempotencyKey: `ingestFeed:manual:${feed.id}:${Math.floor(Date.now() / 10_000)}`, traceId: randomUUID() }, + ); + return { taskId: r.runId }; + }); + + // ── 来源资源 / 观察 ── + route(app, D.sourceReleases.list, async ({ query }) => { + const off = decodeCursor(query.cursor); + let q = db().selectFrom("source_release").selectAll(); + if (query.siteId) q = q.where("site_id", "=", query.siteId); + if (query.mediaEntityId) q = q.where("media_entity_id", "=", query.mediaEntityId); + if (query.resolutionStatus) q = q.where("resolution_status", "=", query.resolutionStatus); + if (query.q) + q = q.where((eb) => + eb.or([eb("title", "ilike", `%${query.q}%`), sql`similarity(title, ${query.q}) >= 0.3`]), + ); + const rows = await q + .orderBy("first_seen_at", "desc") + .orderBy("id") + .offset(off) + .limit(query.limit + 1) + .execute(); + const p = page(rows, off, query.limit); + return { items: toApiList(p.items), nextCursor: p.nextCursor }; + }); + route(app, D.sourceReleases.get, async ({ params }) => { + const row = await db().selectFrom("source_release").selectAll().where("id", "=", params.id).executeTakeFirst(); + if (!row) throw new ApiException("not_found", "来源资源不存在"); + return toApi(row); + }); + route( + app, + { + method: "post", + path: "/source-releases/{id}/identity", + scope: "organize.execute", + summary: "人工指定来源资源的作品/内容单元(加人工锁,自动识别不再改写)", + tags: T_RELEASES, + params: IdParam, + body: ReleaseIdentityAssign, + response: z.object({ mediaEntityId: Id, contentUnitIds: z.array(Id) }), + }, + async ({ params, body, actor }) => assignReleaseIdentity(rt(), params.id, body, actor), + ); + route( + app, + { + method: "post", + path: "/source-releases/{id}/unlock", + scope: "organize.execute", + summary: "解除来源资源的人工锁并重新识别", + tags: T_RELEASES, + params: IdParam, + response: Accepted, + }, + async ({ params, actor }) => { + const row = await db() + .updateTable("source_release") + .set({ resolution_locked: false }) + .where("id", "=", params.id) + .returning(["id", "site_id"]) + .executeTakeFirst(); + if (!row) throw new ApiException("not_found", "来源资源不存在"); + await audit(db(), actor, "identity.release_unlocked", { type: "source_release", id: params.id }); + const r = await rt().ctx.engine.enqueue( + "resolveObservation", + { sourceReleaseId: params.id, force: true }, + { idempotencyKey: `resolve:unlock:${params.id}:${Date.now()}`, traceId: randomUUID() }, + ); + return { taskId: r.runId }; + }, + ); + route(app, D.observations.list, async ({ query }) => { + const off = decodeCursor(query.cursor); + let q = db().selectFrom("observation").selectAll(); + if (query.feedId) q = q.where("feed_id", "=", query.feedId); + if (query.status) q = q.where("status", "=", query.status); + const rows = await q + .orderBy("first_seen_at", "desc") + .orderBy("id") + .offset(off) + .limit(query.limit + 1) + .execute(); + const p = page(rows, off, query.limit); + return { items: toApiList(p.items), nextCursor: p.nextCursor }; + }); + route(app, D.observations.get, async ({ params }) => { + const row = await db().selectFrom("observation").selectAll().where("id", "=", params.id).executeTakeFirst(); + if (!row) throw new ApiException("not_found", "观察不存在"); + return toApi(row); + }); + + // ── 内部辅助 ── + async function getSite(id: string) { + const row = await db().selectFrom("site").selectAll().where("id", "=", id).executeTakeFirst(); + if (!row) throw new ApiException("not_found", "站点不存在"); + return row; + } + async function getFeed(id: string) { + const row = await db().selectFrom("feed_source").selectAll().where("id", "=", id).executeTakeFirst(); + if (!row) throw new ApiException("not_found", "Feed 不存在"); + return row; + } + async function checkAccount(accountId: string, siteId: string | null) { + const acc = await db() + .selectFrom("site_account") + .select(["site_id"]) + .where("id", "=", accountId) + .executeTakeFirst(); + if (!acc) throw new ApiException("bad_request", "站点账号不存在"); + if (siteId && acc.site_id !== siteId) throw new ApiException("bad_request", "站点账号不属于该站点"); + } + async function setSiteEnabled( + id: string, + enabled: boolean, + actor: { type: "user" | "agent" | "system" | "api_key"; id: string | null }, + ) { + await getSite(id); + return db() + .transaction() + .execute(async (trx) => { + const row = await trx + .updateTable("site") + .set({ enabled, health_status: enabled ? "unknown" : "disabled" }) + .where("id", "=", id) + .returningAll() + .executeTakeFirstOrThrow(); + // 历史资源仍可查,标注站点关闭 + await trx + .updateTable("source_release") + .set({ availability: enabled ? "active" : "site_disabled" }) + .where("site_id", "=", id) + .where("availability", "=", enabled ? "site_disabled" : "active") + .execute(); + await audit(trx, actor, enabled ? "site.enabled" : "site.disabled", { type: "site", id }); + return row; + }); + } +} diff --git a/apps/server/src/modules/sources-identity/routes/workbench.ts b/apps/server/src/modules/sources-identity/routes/workbench.ts new file mode 100644 index 00000000..2e9979c2 --- /dev/null +++ b/apps/server/src/modules/sources-identity/routes/workbench.ts @@ -0,0 +1,42 @@ +import { Id, WorkbenchReleaseDetail, WorkbenchReleasePage, WorkbenchReleaseQuery } from "@umt/contracts"; +import { z } from "zod"; +import type { ModuleApp } from "../../types.ts"; +import { route } from "../http.ts"; +import type { SourcesRuntime } from "../runtime.ts"; +import { listSiteReleases, releaseDetail } from "../service/workbench.ts"; + +// 站点工作台(整理视图)数据 API。库存叠加(已有资产/Slot/下载)不在这里:条目带 entity.id, +// 前端按作品 ID 二次查询 W2-C 的媒体库端点。 + +const T = ["site-workbench"]; + +export function workbenchRoutes(app: ModuleApp, rt: () => SourcesRuntime) { + route( + app, + { + method: "get", + path: "/sites/{id}/releases", + scope: "read", + summary: "站点资源列表/搜索(快照优先,必要时实时补查;标注来源与更新时间)", + tags: T, + params: z.object({ id: Id }), + query: WorkbenchReleaseQuery, + response: WorkbenchReleasePage, + }, + async ({ params, query }) => listSiteReleases(rt(), params.id, query), + ); + route( + app, + { + method: "get", + path: "/sites/{id}/releases/{rid}", + scope: "read", + summary: "站点资源详情(refresh=true 时实时复核详情;失败保留既有数据)", + tags: T, + params: z.object({ id: Id, rid: Id }), + query: z.object({ refresh: z.enum(["true", "false"]).optional() }), + response: WorkbenchReleaseDetail, + }, + async ({ params, query }) => releaseDetail(rt(), params.id, params.rid, query.refresh === "true"), + ); +} diff --git a/apps/server/src/modules/sources-identity/rows.ts b/apps/server/src/modules/sources-identity/rows.ts new file mode 100644 index 00000000..27dde3d9 --- /dev/null +++ b/apps/server/src/modules/sources-identity/rows.ts @@ -0,0 +1,37 @@ +// db 行(snake_case、Date)→ contracts(camelCase、ISO 串)。只做命名与时间转换,取值语义不变。 + +const camel = (s: string) => s.replace(/_([a-z0-9])/g, (_, c: string) => c.toUpperCase()); + +function conv(v: unknown): unknown { + if (v instanceof Date) return v.toISOString(); + return v; +} + +/** 通用行转换:键转 camelCase,Date → ISO 串;jsonb/数组原样保留 */ +export function toApi(row: object): T { + const out: Record = {}; + for (const [k, v] of Object.entries(row)) out[camel(k)] = conv(v); + return out as T; +} + +export const toApiList = (rows: readonly object[]): T[] => rows.map((r) => toApi(r)); + +/** 偏移游标(不透明 base64url);列表规模小,足够用 */ +export function encodeCursor(offset: number): string { + return Buffer.from(String(offset), "utf8").toString("base64url"); +} + +export function decodeCursor(cursor: string | undefined): number { + if (!cursor) return 0; + const n = Number(Buffer.from(cursor, "base64url").toString("utf8")); + return Number.isSafeInteger(n) && n >= 0 ? n : 0; +} + +/** 取 limit+1 条判断是否还有下一页 */ +export function page(rows: T[], offset: number, limit: number): { items: T[]; nextCursor: string | null } { + const more = rows.length > limit; + return { items: more ? rows.slice(0, limit) : rows, nextCursor: more ? encodeCursor(offset + limit) : null }; +} + +/** jsonb 写入:数组/对象都先 JSON.stringify(pg 会把 JS 数组当 PG 数组) */ +export const jsonb = (v: unknown): string => JSON.stringify(v ?? null); diff --git a/apps/server/src/modules/sources-identity/runtime.ts b/apps/server/src/modules/sources-identity/runtime.ts new file mode 100644 index 00000000..2525e16e --- /dev/null +++ b/apps/server/src/modules/sources-identity/runtime.ts @@ -0,0 +1,143 @@ +import { + createRssFetcher, + createSiteAdapter, + createSiteRateLimiter, + createTmdbClient, + getSiteDefinition, + type RssFetcher, + type SecretResolver, + type SiteAdapter, + type SiteDefinition, + type SiteRateLimiter, + type TmdbClient, +} from "@umt/adapters"; +import type { SiteRow } from "@umt/db"; +import type { ServerContext } from "../types.ts"; + +// 模块运行时依赖:站点适配器与限流器(每站一个实例)、RSS 抓取器、TMDB 客户端、时钟。 +// 测试通过 createSourcesIdentityModule(overrides) 注入 fetch / TMDB / 时钟。 + +export interface RuntimeOverrides { + /** 站点与 RSS 请求使用的 fetch(测试指向 mock) */ + fetch?: typeof globalThis.fetch; + /** TMDB 客户端工厂;返回 null 表示未配置(只做本地召回) */ + tmdb?: (() => TmdbClient | null) | undefined; + now?: () => Date; + /** RSS 抓取器选项覆盖(退避/熔断阈值) */ + rss?: Parameters[0]; + /** 调度 tick 间隔(毫秒),默认 30s */ + schedulerIntervalMs?: number; +} + +interface CachedAdapter { + sig: string; + limiter: SiteRateLimiter; + adapter: SiteAdapter; +} + +export class SourcesRuntime { + readonly now: () => Date; + readonly rss: RssFetcher; + private readonly adapters = new Map(); + private tmdbClient: TmdbClient | null | undefined; + + constructor( + readonly ctx: ServerContext, + readonly overrides: RuntimeOverrides = {}, + ) { + this.now = overrides.now ?? (() => new Date()); + this.rss = createRssFetcher({ + ...(overrides.fetch ? { fetch: overrides.fetch } : {}), + now: this.now, + ...overrides.rss, + }); + } + + /** TMDB:优先注入;否则读环境变量 TMDB_API_KEY / TMDB_ACCESS_TOKEN(W2-A 设置页接管前的过渡) */ + tmdb(): TmdbClient | null { + if (this.tmdbClient !== undefined) return this.tmdbClient; + if (this.overrides.tmdb) { + this.tmdbClient = this.overrides.tmdb(); + return this.tmdbClient; + } + const env = (k: string) => Deno.env.get(k); + const apiKey = env("TMDB_API_KEY"); + const accessToken = env("TMDB_ACCESS_TOKEN"); + const baseUrl = env("TMDB_BASE_URL"); + this.tmdbClient = + apiKey || accessToken + ? createTmdbClient({ + ...(apiKey ? { apiKey } : {}), + ...(accessToken ? { accessToken } : {}), + ...(baseUrl ? { baseUrl } : {}), + ...(this.overrides.fetch ? { fetch: this.overrides.fetch } : {}), + }) + : null; + return this.tmdbClient; + } + + /** + * 站点适配器:每站一个实例,列表/搜索/详情/下载共享同一个限流器。 + * 站点限流参数或基址变化时重建(签名比较)。 + */ + siteAdapter(site: Pick): SiteAdapter { + const def = getSiteDefinition(site.key); + if (!def) throw new Error(`站点 ${site.key} 没有随产品发布的适配器定义`); + const sig = JSON.stringify([site.base_url, site.rate_limit, site.config]); + const hit = this.adapters.get(site.id); + if (hit && hit.sig === sig) return hit.adapter; + const limiter = createSiteRateLimiter(limiterOptions(def, site.rate_limit)); + const cfg = (site.config ?? {}) as { apiBaseUrl?: string }; + const baseUrl = + def.umt.kind === "mteam" ? cfg.apiBaseUrl : sameUrl(site.base_url, def.domain) ? undefined : site.base_url; + const adapter = createSiteAdapter(def, { + secrets: this.siteSecrets(site.id), + limiter, + now: this.now, + ...(this.overrides.fetch ? { fetch: this.overrides.fetch } : {}), + ...(baseUrl ? { baseUrl } : {}), + }); + this.adapters.set(site.id, { sig, limiter, adapter }); + return adapter; + } + + /** 已建立的站点限流器(测试与诊断用) */ + limiterOf(siteId: string): SiteRateLimiter | null { + return this.adapters.get(siteId)?.limiter ?? null; + } + + /** 站点凭证:取该站第一个启用且未失效的账号,按 auth_kind 对应适配器要求的凭证类型;明文不缓存 */ + private siteSecrets(siteId: string): SecretResolver { + const { db, secrets } = this.ctx; + return { + async resolve({ kind }) { + const want = kind === "cookie" ? ["cookie"] : kind === "apiKey" ? ["api_key"] : ["passkey"]; + const acc = await db + .selectFrom("site_account") + .select(["secret_ref"]) + .where("site_id", "=", siteId) + .where("enabled", "=", true) + .where("status", "in", ["unknown", "active"]) + .where("auth_kind", "in", want as never) + .where("secret_ref", "is not", null) + .orderBy("created_at") + .executeTakeFirst(); + if (!acc?.secret_ref) return null; + return secrets.resolve(acc.secret_ref); + }, + }; + } +} + +function sameUrl(a: string, b: string): boolean { + return a.replace(/\/+$/, "") === b.replace(/\/+$/, ""); +} + +/** 站点行 rate_limit 优先,缺省用定义里的 umt.rateLimit */ +export function limiterOptions(def: SiteDefinition, rateLimit: unknown): { minIntervalMs: number; burst: number } { + const rl = (rateLimit ?? {}) as { requestsPerMinute?: number; minIntervalMs?: number; concurrency?: number }; + const minIntervalMs = + rl.minIntervalMs ?? + (rl.requestsPerMinute ? Math.ceil(60_000 / rl.requestsPerMinute) : def.umt.rateLimit.minIntervalMs); + return { minIntervalMs, burst: rl.concurrency ?? def.umt.rateLimit.burst }; +} diff --git a/apps/server/src/modules/sources-identity/scheduler.ts b/apps/server/src/modules/sources-identity/scheduler.ts new file mode 100644 index 00000000..84566c26 --- /dev/null +++ b/apps/server/src/modules/sources-identity/scheduler.ts @@ -0,0 +1,90 @@ +import { randomUUID } from "node:crypto"; +import { sql } from "@umt/db"; +import type { SourcesRuntime } from "./runtime.ts"; + +// worker 后台调度:按 Feed 轮询策略发 feed.due(outbox → ingestFeed)、按站点快照周期入队 snapshotSite、 +// 按待识别池 next_review_at 入队复查。幂等键按时间槽生成,多 worker 并发也不会重复。 + +export interface TickResult { + feedsDue: number; + snapshotsDue: number; + reviewsDue: number; + skipped?: string; +} + +const REVIEW_LEASE_MS = 10 * 60_000; + +export async function schedulerTick(rt: SourcesRuntime): Promise { + const { ctx } = rt; + const automation = await ctx.settings.get("automation"); + // 恢复后只读对账模式:禁止自动写入 + if (automation.readOnlyReconcile) return { feedsDue: 0, snapshotsDue: 0, reviewsDue: 0, skipped: "read_only" }; + const now = rt.now(); + + // Feed:到期(next_fetch_at 空或已到)且不在退避期、站点(若有)开启 + const feedsDue = await ctx.db.transaction().execute(async (trx) => { + const due = await sql<{ id: string; poll_interval_sec: number }>` + select f.id, f.poll_interval_sec from feed_source f left join site s on s.id = f.site_id + where f.enabled and (s.id is null or s.enabled) + and (f.next_fetch_at is null or f.next_fetch_at <= ${now}) + and (f.backoff_until is null or f.backoff_until <= ${now}) + and f.health_status <> 'circuit_open' + order by f.next_fetch_at nulls first limit 100 + for update of f skip locked`.execute(trx); + for (const f of due.rows) { + const slot = Math.floor(now.getTime() / 1000 / f.poll_interval_sec); + await ctx.outbox.emit( + trx, + "feed.due", + { feedId: f.id, reason: "schedule" }, + { idempotencyKey: `feed.due:${f.id}:${slot}` }, + ); + await trx + .updateTable("feed_source") + .set({ next_fetch_at: new Date(now.getTime() + f.poll_interval_sec * 1000) }) + .where("id", "=", f.id) + .execute(); + } + return due.rows.length; + }); + + // 站点快照:开启快照且到周期 + const sites = await sql<{ id: string; snapshot_interval_sec: number }>` + select id, snapshot_interval_sec from site + where enabled and snapshot_enabled and snapshot_interval_sec is not null + and (last_snapshot_at is null or last_snapshot_at + make_interval(secs => snapshot_interval_sec) <= ${now})`.execute( + ctx.db, + ); + for (const s of sites.rows) { + const slot = Math.floor(now.getTime() / 1000 / s.snapshot_interval_sec); + await ctx.engine.enqueue( + "snapshotSite", + { siteId: s.id, kind: "periodic" }, + { idempotencyKey: `snapshotSite:${s.id}:${slot}`, traceId: randomUUID() }, + ); + } + + // 待识别池复查:到期的先顺延一个租约,避免下个 tick 重复入队 + const reviews = await ctx.db.transaction().execute(async (trx) => { + const rows = await sql<{ id: string; source_release_id: string; attempts: number }>` + select id, source_release_id, attempts from identity_candidate + where status = 'pending' and next_review_at is not null and next_review_at <= ${now} + order by next_review_at limit 50 for update skip locked`.execute(trx); + for (const r of rows.rows) { + await trx + .updateTable("identity_candidate") + .set({ next_review_at: new Date(now.getTime() + REVIEW_LEASE_MS) }) + .where("id", "=", r.id) + .execute(); + } + return rows.rows; + }); + for (const r of reviews) { + await ctx.engine.enqueue( + "resolveObservation", + { sourceReleaseId: r.source_release_id, force: true }, + { idempotencyKey: `review:${r.id}:${r.attempts}`, traceId: randomUUID() }, + ); + } + return { feedsDue, snapshotsDue: sites.rows.length, reviewsDue: reviews.length }; +} diff --git a/apps/server/src/modules/sources-identity/service/common.ts b/apps/server/src/modules/sources-identity/service/common.ts new file mode 100644 index 00000000..f73760d7 --- /dev/null +++ b/apps/server/src/modules/sources-identity/service/common.ts @@ -0,0 +1,109 @@ +import type { ActorType, DecisionKind, DecisionOption, Severity } from "@umt/contracts"; +import { type DB, type Kysely, sql, type Transaction } from "@umt/db"; +import type { ServerContext } from "../../types.ts"; +import { jsonb } from "../rows.ts"; + +export type Exec = Kysely | Transaction; + +export interface Actor { + type: ActorType; + id: string | null; +} + +export const SYSTEM: Actor = { type: "system", id: "sources-identity" }; + +/** 审计:只追加 */ +export async function audit( + exec: Exec, + actor: Actor, + action: string, + target: { type: string; id: string | null }, + payload: Record = {}, +): Promise { + await exec + .insertInto("audit_event") + .values({ + actor_type: actor.type, + actor_id: actor.id, + action, + target_type: target.type, + target_id: target.id, + payload: jsonb(payload), + }) + .execute(); +} + +export interface DecisionInput { + kind: DecisionKind; + severity?: Severity; + title: string; + summary?: string | null; + subjectType: string; + subjectId: string | null; + dedupeKey: string; + evidence?: Record; + options?: DecisionOption[]; + nextReviewAt?: Date | null; +} + +/** + * 开一个待处理事项;同 dedupeKey 已有未关闭条目时只刷新证据与选项(不重复通知)。 + * 新建时同事务发 decision.created。 + */ +export async function openDecision( + ctx: ServerContext, + trx: Exec, + d: DecisionInput, +): Promise<{ id: string; created: boolean }> { + const row = await sql<{ id: string; inserted: boolean }>` + insert into decision (kind, severity, title, summary, subject_type, subject_id, dedupe_key, evidence, options, next_review_at) + values (${d.kind}, ${d.severity ?? "warning"}, ${d.title}, ${d.summary ?? null}, ${d.subjectType}, ${d.subjectId}, + ${d.dedupeKey}, ${jsonb(d.evidence ?? {})}::jsonb, ${jsonb(d.options ?? [])}::jsonb, ${d.nextReviewAt ?? null}) + on conflict (dedupe_key) where status in ('open', 'investigating', 'awaiting_user') + do update set evidence = excluded.evidence, options = excluded.options, summary = excluded.summary, + title = excluded.title, attempts = decision.attempts + 1 + returning id, (xmax = 0) as inserted`.execute(trx); + const r = row.rows[0]!; + if (r.inserted) { + await ctx.outbox.emit( + trx, + "decision.created", + { decisionId: r.id, kind: d.kind, severity: d.severity ?? "warning", title: d.title }, + { idempotencyKey: `decision.created:${r.id}` }, + ); + } + return { id: r.id, created: r.inserted }; +} + +/** 关闭同 dedupeKey 的未关闭事项(身份已被人工/自动确认等),逐条发 decision.updated */ +export async function closeDecisions( + ctx: ServerContext, + trx: Exec, + dedupeKey: string, + status: "resolved" | "dismissed", + resolution: Record, + by: string, +): Promise { + const rows = await trx + .updateTable("decision") + .set({ status, resolution: jsonb(resolution), resolved_by: by, resolved_at: new Date() }) + .where("dedupe_key", "=", dedupeKey) + .where("status", "in", ["open", "investigating", "awaiting_user"]) + .returning(["id"]) + .execute(); + for (const r of rows) { + await ctx.outbox.emit( + trx, + "decision.updated", + { decisionId: r.id, status }, + { + idempotencyKey: `decision.updated:${r.id}:${status}`, + }, + ); + } + return rows.map((r) => r.id); +} + +/** 截断长文本(描述等只留摘录) */ +export const clip = (s: string | undefined | null, n = 2000): string | null => + s == null ? null : s.length > n ? `${s.slice(0, n)}…` : s; diff --git a/apps/server/src/modules/sources-identity/service/entities.ts b/apps/server/src/modules/sources-identity/service/entities.ts new file mode 100644 index 00000000..928d0ec3 --- /dev/null +++ b/apps/server/src/modules/sources-identity/service/entities.ts @@ -0,0 +1,221 @@ +import type { MetadataCandidate } from "@umt/adapters"; +import type { AliasKind, AliasSource, EvidenceLevel, MediaKind } from "@umt/contracts"; +import { sql } from "@umt/db"; +import { normalizeTitle, type ParsedRelease } from "@umt/domain"; +import { toContractMediaKind } from "@umt/mapping"; +import { jsonb } from "../rows.ts"; +import type { Actor, Exec } from "./common.ts"; + +// 本地作品 / 内容单元 / 外部身份 / 别名的写入辅助。所有写入在调用方事务内完成。 + +/** 沿 merged_into_id 找到最终实体(防环,最多 10 跳);不存在返回 null */ +export async function resolveEntityId(exec: Exec, id: string): Promise { + let cur = id; + for (let i = 0; i < 10; i++) { + const row = await exec + .selectFrom("media_entity") + .select(["id", "status", "merged_into_id"]) + .where("id", "=", cur) + .executeTakeFirst(); + if (!row) return null; + if (row.status !== "merged" || !row.merged_into_id) return row.id; + cur = row.merged_into_id; + } + return cur; +} + +/** 已接受的外部身份 → 本地作品(跟随合并跳转) */ +export async function entityByExternal( + exec: Exec, + ref: { provider: string; entityType: string; externalId: string }, +): Promise { + const row = await exec + .selectFrom("external_identity") + .select(["media_entity_id"]) + .where("provider", "=", ref.provider) + .where("entity_type", "=", ref.entityType as never) + .where("external_id", "=", ref.externalId) + .where("status", "=", "accepted") + .where("media_entity_id", "is not", null) + .executeTakeFirst(); + return row?.media_entity_id ? resolveEntityId(exec, row.media_entity_id) : null; +} + +export async function addAlias( + exec: Exec, + entityId: string, + alias: string, + kind: AliasKind, + source: AliasSource, + language: string | null = null, + locked = false, +): Promise { + const normalized = normalizeTitle(alias); + if (!normalized) return; + await sql`insert into title_alias (media_entity_id, content_unit_id, alias, normalized, language, kind, source, locked) + values (${entityId}, null, ${alias}, ${normalized}, ${language}, ${kind}, ${source}, ${locked}) + on conflict (media_entity_id, content_unit_id, normalized, kind) do nothing`.execute(exec); +} + +/** 写一条外部身份(accepted)+ 修订 created;同一外部身份已被其它作品接受时返回 null(交给调用方处理冲突) */ +export async function acceptIdentity( + exec: Exec, + entityId: string, + ref: { provider: string; entityType: string; externalId: string }, + level: EvidenceLevel, + evidence: Record, + actor: Actor, + opts: { locked?: boolean; reason?: string | null } = {}, +): Promise { + const ins = await sql<{ id: string }>` + insert into external_identity (media_entity_id, provider, entity_type, external_id, status, evidence_level, evidence, locked, valid_from, verified_at, verify_status) + values (${entityId}, ${ref.provider}, ${ref.entityType}, ${ref.externalId}, 'accepted', ${level}, ${jsonb(evidence)}::jsonb, + ${opts.locked ?? false}, now(), now(), 'ok') + on conflict do nothing + returning id`.execute(exec); + const id = ins.rows[0]?.id; + if (!id) return null; + await exec + .insertInto("identity_revision") + .values({ + external_identity_id: id, + revision: 1, + action: "created", + from_status: null, + to_status: "accepted", + after: jsonb({ ...ref, mediaEntityId: entityId, evidenceLevel: level, locked: opts.locked ?? false }), + actor_type: actor.type, + actor_id: actor.id, + reason: opts.reason ?? null, + }) + .execute(); + return id; +} + +/** 综艺判定:站点分类名含"综艺"或标题带期号/播出日期 */ +export function looksLikeVariety(parsed: ParsedRelease | null, categoryName: string | null): boolean { + if (categoryName && /综艺|variety|show/i.test(categoryName)) return true; + return !!parsed && (parsed.issue !== undefined || parsed.airDate !== undefined); +} + +/** + * 按 TMDB 详情建本地作品:外部身份 accepted(+IMDb)、别名(原名/译名/别名)。 + * 已有同一 TMDB 身份的作品(含并发创建)直接复用。 + */ +export async function createEntityFromTmdb( + exec: Exec, + m: MetadataCandidate, + opts: { level: EvidenceLevel; evidence: Record; actor: Actor; variety?: boolean; locked?: boolean }, +): Promise<{ entityId: string; created: boolean }> { + const ref = { provider: "tmdb", entityType: m.entityType, externalId: m.externalId }; + const existing = await entityByExternal(exec, ref); + if (existing) return { entityId: existing, created: false }; + const kind: MediaKind = toContractMediaKind(m.entityType, { variety: !!opts.variety }); + const entity = await exec + .insertInto("media_entity") + .values({ + kind, + title: m.title, + original_title: m.originalTitle, + year: m.year, + status: "active", + metadata: jsonb({}), + }) + .returning(["id"]) + .executeTakeFirstOrThrow(); + const idRow = await acceptIdentity(exec, entity.id, ref, opts.level, opts.evidence, opts.actor, { + locked: opts.locked ?? false, + }); + if (!idRow) { + // 并发:另一个事务刚接受了同一身份。回滚本事务的新作品由调用方重试处理 + throw new IdentityConflictError(ref); + } + if (m.externalIds?.imdb) { + await acceptIdentity( + exec, + entity.id, + { provider: "imdb", entityType: m.entityType, externalId: m.externalIds.imdb }, + opts.level, + { via: "tmdb", tmdbId: m.externalId }, + opts.actor, + ); + } + await seedAliases(exec, entity.id, m); + return { entityId: entity.id, created: true }; +} + +export async function seedAliases(exec: Exec, entityId: string, m: MetadataCandidate): Promise { + await addAlias(exec, entityId, m.title, "translated", "tmdb", null); + if (m.originalTitle) await addAlias(exec, entityId, m.originalTitle, "original", "tmdb", m.originalLanguage); + for (const t of m.titles) { + if (t.kind === "localized") await addAlias(exec, entityId, t.text, "translated", "tmdb", t.lang); + } + for (const a of m.aliases.slice(0, 50)) await addAlias(exec, entityId, a, "other", "tmdb", null); +} + +export class IdentityConflictError extends Error { + constructor(readonly ref: { provider: string; entityType: string; externalId: string }) { + super(`外部身份 ${ref.provider}:${ref.entityType}/${ref.externalId} 已被其它作品接受`); + } +} + +/** + * 按解析出的季/集找或建内容单元(临时 provisional 允许)。电影返回空。 + * 返回:集单元(有集号时)或季单元;primary 为写入 source_release.content_unit_id 的那个。 + */ +export async function ensureUnits( + exec: Exec, + entityId: string, + kind: MediaKind, + parsed: ParsedRelease | null, +): Promise<{ unitIds: string[]; primary: string | null }> { + if (!parsed || kind === "movie" || parsed.season === undefined) return { unitIds: [], primary: null }; + const seasons: number[] = []; + const last = parsed.seasonEnd ?? parsed.season; + for (let s = parsed.season; s <= last && seasons.length < 50; s++) seasons.push(s); + const seasonIds: string[] = []; + for (const s of seasons) seasonIds.push(await findOrCreateUnit(exec, entityId, "season", s, null, null)); + if (parsed.episodes && seasons.length === 1) { + const ids: string[] = []; + const end = Math.min(parsed.episodes.end, parsed.episodes.start + 199); + for (let e = parsed.episodes.start; e <= end; e++) { + ids.push(await findOrCreateUnit(exec, entityId, "episode", parsed.season, e, seasonIds[0]!)); + } + return { unitIds: ids, primary: ids.length === 1 ? ids[0]! : seasonIds[0]! }; + } + return { unitIds: seasonIds, primary: seasonIds[0] ?? null }; +} + +async function findOrCreateUnit( + exec: Exec, + entityId: string, + kind: "season" | "episode", + season: number, + episode: number | null, + parentId: string | null, +): Promise { + let q = exec + .selectFrom("content_unit") + .select(["id"]) + .where("media_entity_id", "=", entityId) + .where("kind", "=", kind) + .where("season_number", "=", season) + .where("status", "<>", "retired"); + q = episode === null ? q.where("episode_number", "is", null) : q.where("episode_number", "=", episode); + const hit = await q.orderBy("created_at").executeTakeFirst(); + if (hit) return hit.id; + const row = await exec + .insertInto("content_unit") + .values({ + media_entity_id: entityId, + parent_unit_id: parentId, + kind, + season_number: season, + episode_number: episode, + status: "provisional", + display_order: episode ?? season, + }) + .returning(["id"]) + .executeTakeFirstOrThrow(); + return row.id; +} diff --git a/apps/server/src/modules/sources-identity/service/feeds.ts b/apps/server/src/modules/sources-identity/service/feeds.ts new file mode 100644 index 00000000..48fc0dc2 --- /dev/null +++ b/apps/server/src/modules/sources-identity/service/feeds.ts @@ -0,0 +1,320 @@ +import { createHash } from "node:crypto"; +import { + applyFeedItems, + emptyFeedCursor, + type FeedCursor, + type FeedFetchResult, + type FeedFetchState, + type FeedItem, +} from "@umt/adapters"; +import type { FeedHealth } from "@umt/contracts"; +import { type FeedSourceRow, sql } from "@umt/db"; +import { jsonb } from "../rows.ts"; +import type { SourcesRuntime } from "../runtime.ts"; +import { clip, type Exec, openDecision } from "./common.ts"; + +// ingestFeed:RSS 抓取 → observation(feed+guid 唯一,摘要变化 = 更新)→ source_release(site+站点资源 ID 聚合)。 +// FeedFetchState / FeedCursor 持久化在 feed_source 行;enclosure 只存脱敏值(适配器已脱敏)。 + +/** feed_source.cursor 的形状:重叠窗口游标 + 熔断打开时间 */ +interface StoredCursor { + window?: FeedCursor; + circuitOpenedAt?: string; +} + +const AUTH_BACKOFF_MS = 24 * 3600_000; + +export interface IngestResult { + status: "ok" | "not_modified" | "skipped" | "error" | "auth_failed"; + reason?: string; + newItems: number; + updatedItems: number; + newReleases: number; + possibleGap: boolean; +} + +/** 从 Feed 条目推出站点资源 ID:详情/下载链接的 id 参数或 /detail/ 路径;都没有返回 null */ +export function extractSiteReleaseId(item: Pick): string | null { + for (const u of [item.link, item.enclosureUrl, item.guid]) { + if (!u) continue; + try { + const url = new URL(u); + const q = url.searchParams.get("id") ?? url.searchParams.get("tid") ?? url.searchParams.get("torrentid"); + if (q && /^[\w-]{1,64}$/.test(q)) return q; + const m = /\/(?:detail|details|torrents?|t)\/(\d+)/.exec(url.pathname); + if (m?.[1]) return m[1]; + } catch { + // 不是 URL(纯 GUID) + } + } + if (/^\d{1,20}$/.test(item.guid)) return item.guid; + return null; +} + +export function stateFromRow(f: FeedSourceRow): FeedFetchState { + const cur = (f.cursor ?? {}) as StoredCursor; + const err = f.last_error as FeedFetchState["lastError"] | null; + return { + ...(f.etag ? { etag: f.etag } : {}), + ...(f.last_modified ? { lastModified: f.last_modified } : {}), + consecutiveFailures: f.consecutive_failures, + ...(f.backoff_until ? { nextAttemptAt: f.backoff_until.toISOString() } : {}), + circuit: f.health_status === "circuit_open" ? "open" : "closed", + ...(cur.circuitOpenedAt ? { circuitOpenedAt: cur.circuitOpenedAt } : {}), + ...(f.last_success_at ? { lastSuccessAt: f.last_success_at.toISOString() } : {}), + ...(err && typeof err === "object" && "kind" in err ? { lastError: err } : {}), + }; +} + +function healthOf(r: FeedFetchResult): FeedHealth { + if (r.kind === "ok" || r.kind === "not_modified") return "healthy"; + if (r.state.circuit === "open") return "circuit_open"; + if (r.kind === "skipped") return "degraded"; + return r.state.consecutiveFailures >= 3 ? "failing" : "degraded"; +} + +function isAuthFailure(r: FeedFetchResult): boolean { + if (r.kind !== "error") return false; + return r.error.kind === "expired" || r.error.httpStatus === 401 || r.error.httpStatus === 403; +} + +export async function ingestFeed( + rt: SourcesRuntime, + feedId: string, + opts: { manual?: boolean } = {}, +): Promise { + const { db, secrets } = rt.ctx; + const empty = { newItems: 0, updatedItems: 0, newReleases: 0, possibleGap: false }; + const feed = await db.selectFrom("feed_source").selectAll().where("id", "=", feedId).executeTakeFirst(); + if (!feed) return { status: "skipped", reason: "not_found", ...empty }; + if (!feed.enabled) return { status: "skipped", reason: "disabled", ...empty }; + const site = feed.site_id + ? await db.selectFrom("site").selectAll().where("id", "=", feed.site_id).executeTakeFirst() + : undefined; + if (site && !site.enabled) return { status: "skipped", reason: "site_disabled", ...empty }; + if (!feed.url_secret_ref) return { status: "skipped", reason: "no_url", ...empty }; + + const url = await secrets.resolve(feed.url_secret_ref); + const result = await rt.rss.fetchFeed(url, stateFromRow(feed), { ignoreBackoff: !!opts.manual }); + const now = rt.now(); + const auth = isAuthFailure(result); + const health: FeedHealth = auth ? "failing" : healthOf(result); + const cursor = (feed.cursor ?? {}) as StoredCursor; + const state = result.state; + const pollNext = new Date(now.getTime() + feed.poll_interval_sec * 1000); + const backoffUntil = auth + ? new Date(now.getTime() + AUTH_BACKOFF_MS) + : state.nextAttemptAt && result.kind !== "ok" && result.kind !== "not_modified" + ? new Date(state.nextAttemptAt) + : null; + + return db.transaction().execute(async (trx) => { + let out: IngestResult = { status: result.kind === "ok" ? "ok" : result.kind, ...empty }; + let window = cursor.window ?? emptyFeedCursor(); + if (result.kind === "ok") { + const applied = applyFeedItems(window, result.feed.items); + window = applied.cursor; + const counts = await writeItems(rt, trx, feed, site?.id ?? null, applied.newItems, applied.updatedItems); + const unchanged = result.feed.items + .filter((i) => !applied.newItems.includes(i) && !applied.updatedItems.includes(i)) + .map((i) => i.guid); + if (unchanged.length) await touchUnchanged(trx, feed.id, unchanged, now); + out = { + status: "ok", + newItems: applied.newItems.length, + updatedItems: applied.updatedItems.length, + newReleases: counts.newReleases, + possibleGap: applied.possibleGap, + }; + if (applied.possibleGap) { + await rt.ctx.outbox.emit( + trx, + "feed.gap_suspected", + { + feedId: feed.id, + siteId: site?.id ?? null, + previousFetchedAt: cursor.window?.lastFetchedAt ?? null, + detectedAt: now.toISOString(), + }, + { idempotencyKey: `feed.gap:${feed.id}:${now.toISOString()}` }, + ); + } + } + + const lastError = auth + ? { ...(state.lastError ?? {}), kind: "auth", message: "Feed 凭证失效或过期(401/403/过期提示),已停止自动重试" } + : (state.lastError ?? null); + const nextCursor: StoredCursor = { + window, + ...(state.circuit === "open" && state.circuitOpenedAt ? { circuitOpenedAt: state.circuitOpenedAt } : {}), + }; + await trx + .updateTable("feed_source") + .set({ + etag: state.etag ?? null, + last_modified: state.lastModified ?? null, + consecutive_failures: state.consecutiveFailures, + backoff_until: backoffUntil, + health_status: health, + last_error: result.kind === "ok" || result.kind === "not_modified" ? null : jsonb(lastError), + cursor: jsonb(nextCursor), + last_fetch_at: result.kind === "skipped" ? feed.last_fetch_at : now, + last_success_at: state.lastSuccessAt ? new Date(state.lastSuccessAt) : feed.last_success_at, + next_fetch_at: backoffUntil && backoffUntil > pollNext ? backoffUntil : pollNext, + }) + .where("id", "=", feed.id) + .execute(); + + if (health !== feed.health_status) { + await rt.ctx.outbox.emit(trx, "feed.health_changed", { + feedId: feed.id, + from: feed.health_status, + to: health, + reason: + lastError && typeof lastError === "object" ? String((lastError as { message?: string }).message ?? "") : null, + }); + } + if (result.kind === "ok" || result.kind === "not_modified") { + await rt.ctx.outbox.emit(trx, "feed.fetched", { + feedId: feed.id, + newItems: out.newItems, + updatedItems: out.updatedItems, + notModified: result.kind === "not_modified", + }); + } + + if (auth) { + if (feed.site_account_id) { + await trx + .updateTable("site_account") + .set({ status: "invalid", last_verified_at: now }) + .where("id", "=", feed.site_account_id) + .execute(); + } + await openDecision(rt.ctx, trx, { + kind: "source_unreachable", + severity: "warning", + title: `Feed「${feed.name}」凭证失效`, + summary: "RSS 返回 401/403 或过期提示;已停止自动重试。请更新 Feed 链接或站点账号凭证后手动恢复。", + subjectType: "feed_source", + subjectId: feed.id, + dedupeKey: `feed-auth:${feed.id}`, + evidence: { feedId: feed.id, siteId: feed.site_id, siteAccountId: feed.site_account_id, error: lastError }, + options: [ + { key: "reset", label: "已更新凭证,恢复抓取" }, + { key: "disable", label: "停用该 Feed" }, + ], + }); + out = { ...out, status: "auth_failed", reason: "auth" }; + } + return out; + }); +} + +/** 未变化条目:只刷新 last_seen_at(observation 与其 source_release) */ +async function touchUnchanged(trx: Exec, feedId: string, guids: string[], now: Date) { + const rows = await trx + .updateTable("observation") + .set({ last_seen_at: now, fetched_at: now }) + .where("feed_id", "=", feedId) + .where("guid", "in", guids) + .returning(["source_release_id"]) + .execute(); + const ids = rows.map((r) => r.source_release_id).filter((x): x is string => !!x); + if (ids.length) await trx.updateTable("source_release").set({ last_seen_at: now }).where("id", "in", ids).execute(); +} + +function observationRaw(item: FeedItem) { + return { + link: item.link, + // 只存脱敏后的 enclosure(凭据参数为 [REDACTED]) + enclosureUrl: item.enclosureUrl ?? null, + enclosureHasSecret: item.enclosureHasSecret ?? false, + enclosureType: item.enclosureType ?? null, + size: item.size, + categories: item.categories, + nickname: item.nickname ?? null, + description: clip(item.description, 1000), + rawDigest: item.rawDigest, + }; +} + +async function writeItems( + rt: SourcesRuntime, + trx: Exec, + feed: FeedSourceRow, + siteId: string | null, + newItems: FeedItem[], + updatedItems: FeedItem[], +): Promise<{ newReleases: number }> { + let newReleases = 0; + const updated = new Set(updatedItems); + for (const item of [...newItems, ...updatedItems]) { + const raw = observationRaw(item); + const obs = ( + await sql<{ id: string; source_release_id: string | null; prev_digest: string | null }>` + with prev as (select content_digest from observation where feed_id = ${feed.id} and guid = ${item.guid}) + insert into observation (feed_id, guid, content_digest, title, link_redacted, raw, source_published_at, fetched_at) + values (${feed.id}, ${item.guid}, ${item.rawDigest}, ${item.title}, ${item.link}, ${jsonb(raw)}::jsonb, + ${item.publishedAt ?? null}, ${item.fetchedAt}) + on conflict (feed_id, guid) do update set + content_digest = excluded.content_digest, title = excluded.title, link_redacted = excluded.link_redacted, + raw = excluded.raw, source_published_at = coalesce(excluded.source_published_at, observation.source_published_at), + fetched_at = excluded.fetched_at, last_seen_at = now() + returning id, source_release_id, (select content_digest from prev) as prev_digest`.execute(trx) + ).rows[0]!; + + if (!siteId) { + // 无站点的通用 Feed:只留观察,无法按站点资源聚合 + await trx + .updateTable("observation") + .set({ status: "ignored", last_error: jsonb({ reason: "no_site" }) }) + .where("id", "=", obs.id) + .execute(); + continue; + } + const srid = + extractSiteReleaseId(item) ?? `guid:${createHash("sha1").update(item.guid).digest("hex").slice(0, 20)}`; + const contentChanged = updated.has(item) || (obs.prev_digest !== null && obs.prev_digest !== item.rawDigest); + const sizeBytes = item.size > 0 ? item.size : null; + const rel = ( + await sql<{ id: string; inserted: boolean; prev_title: string | null; resolution_locked: boolean }>` + with prev as (select title from source_release where site_id = ${siteId} and site_release_id = ${srid}) + insert into source_release (site_id, site_release_id, title, category_raw, size_bytes, source_published_at, raw) + values (${siteId}, ${srid}, ${item.title}, ${item.categories[0] ?? null}, ${sizeBytes}, ${item.publishedAt ?? null}, + ${jsonb({ rss: { link: item.link, categories: item.categories } })}::jsonb) + on conflict (site_id, site_release_id) do update set + last_seen_at = now(), + title = case when ${contentChanged} then excluded.title else source_release.title end, + size_bytes = case when ${contentChanged} then coalesce(excluded.size_bytes, source_release.size_bytes) + else coalesce(source_release.size_bytes, excluded.size_bytes) end, + source_published_at = coalesce(source_release.source_published_at, excluded.source_published_at), + category_raw = coalesce(source_release.category_raw, excluded.category_raw) + returning id, (xmax = 0) as inserted, (select title from prev) as prev_title, resolution_locked`.execute(trx) + ).rows[0]!; + await trx + .updateTable("observation") + .set({ source_release_id: rel.id, status: "linked" }) + .where("id", "=", obs.id) + .execute(); + + if (rel.inserted) { + newReleases++; + await rt.ctx.outbox.emit( + trx, + "release.observed", + { sourceReleaseId: rel.id, observationId: obs.id, siteId, isNew: true }, + { idempotencyKey: `release.observed:${rel.id}` }, + ); + } else if (contentChanged && rel.prev_title !== item.title && !rel.resolution_locked) { + // 标题修正:按新标题重新识别(证据版本变化) + await rt.ctx.outbox.emit( + trx, + "release.observed", + { sourceReleaseId: rel.id, observationId: obs.id, siteId, isNew: false }, + { idempotencyKey: `release.observed:${rel.id}:${item.rawDigest}` }, + ); + } + } + return { newReleases }; +} diff --git a/apps/server/src/modules/sources-identity/service/identity-admin.ts b/apps/server/src/modules/sources-identity/service/identity-admin.ts new file mode 100644 index 00000000..99f53215 --- /dev/null +++ b/apps/server/src/modules/sources-identity/service/identity-admin.ts @@ -0,0 +1,472 @@ +import { TmdbError } from "@umt/adapters"; +import type { + EntityIdentitySet, + ExternalIdentityUpdate, + ExternalRef, + PoolCandidate, + ReleaseIdentityAssign, +} from "@umt/contracts"; +import { type ExternalIdentityRow, sql } from "@umt/db"; +import { parseRelease } from "@umt/domain"; +import { toParsedColumns } from "@umt/mapping"; +import { ApiException } from "../../../http/errors.ts"; +import { jsonb } from "../rows.ts"; +import type { SourcesRuntime } from "../runtime.ts"; +import { type Actor, audit, closeDecisions, type Exec } from "./common.ts"; +import { + acceptIdentity, + createEntityFromTmdb, + ensureUnits, + entityByExternal, + IdentityConflictError, + looksLikeVariety, + resolveEntityId, +} from "./entities.ts"; +import { recordAttempt } from "./resolve.ts"; + +// 人工纠错:指定来源资源身份(加锁)、改作品外部映射(新增修订、旧映射退役)、合并作品(旧 ID 跳转)。 +// 所有变更留 identity_revision / resolution_attempt / audit_event,不直接覆盖历史。 + +const actorName = (a: Actor) => `${a.type}:${a.id ?? ""}`; + +/** 外部身份 → 本地作品;未登记且提供方为 TMDB 时按详情建档(TMDB 失败给 502,不当作不存在) */ +async function entityForExternal(rt: SourcesRuntime, exec: Exec, ref: ExternalRef, actor: Actor, variety: boolean) { + const hit = await entityByExternal(exec, ref); + if (hit) return hit; + if (ref.provider !== "tmdb") + throw new ApiException("bad_request", `未登记的外部身份 ${ref.provider}:${ref.externalId},只支持按 TMDB 建档`); + const tmdb = rt.tmdb(); + if (!tmdb) throw new ApiException("precondition_failed", "未配置 TMDB,无法按外部身份建档"); + let detail: Awaited>; + try { + detail = await tmdb.details(ref.entityType, ref.externalId); + } catch (e) { + if (e instanceof TmdbError && e.kind === "notfound") + throw new ApiException("not_found", `TMDB 无此条目 ${ref.entityType}/${ref.externalId}`); + if (e instanceof TmdbError) throw new ApiException("upstream_unavailable", `TMDB 查询失败:${e.kind}`); + throw e; + } + const r = await createEntityFromTmdb(exec, detail, { + level: "manual", + evidence: { by: actorName(actor) }, + actor, + variety, + locked: true, + }); + return r.entityId; +} + +export async function assignReleaseIdentity( + rt: SourcesRuntime, + srId: string, + input: ReleaseIdentityAssign, + actor: Actor, +): Promise<{ mediaEntityId: string; contentUnitIds: string[] }> { + const { ctx } = rt; + const release = await ctx.db.selectFrom("source_release").selectAll().where("id", "=", srId).executeTakeFirst(); + if (!release) throw new ApiException("not_found", "来源资源不存在"); + const parsed = parseRelease(release.title, release.subtitle ? { subtitle: release.subtitle } : {}); + const site = ((release.raw as Record)?._site ?? {}) as { categoryName?: string }; + const variety = looksLikeVariety(parsed, site.categoryName ?? release.category_raw); + + let ref: ExternalRef | null = input.external ?? null; + let localId: string | null = input.mediaEntityId ?? null; + if (input.candidateIndex !== undefined) { + const pool = await ctx.db + .selectFrom("identity_candidate") + .select(["candidates"]) + .where("source_release_id", "=", srId) + .executeTakeFirst(); + const c = ((pool?.candidates ?? []) as PoolCandidate[])[input.candidateIndex]; + if (!c) throw new ApiException("bad_request", "候选序号无效"); + if (c.localEntityId) localId = c.localEntityId; + else ref = { provider: c.provider, entityType: c.entityType, externalId: c.externalId }; + } + + for (let attempt = 0; ; attempt++) { + try { + return await ctx.db.transaction().execute(async (trx) => { + let entityId: string; + if (localId) { + const r = await resolveEntityId(trx, localId); + if (!r) throw new ApiException("not_found", "本地作品不存在"); + entityId = r; + } else { + entityId = await entityForExternal(rt, trx, ref!, actor, variety); + } + const entity = await trx + .selectFrom("media_entity") + .select(["id", "kind"]) + .where("id", "=", entityId) + .executeTakeFirstOrThrow(); + let unitIds: string[]; + let primary: string | null; + if (input.contentUnitId) { + const u = await trx + .selectFrom("content_unit") + .select(["id", "media_entity_id"]) + .where("id", "=", input.contentUnitId) + .executeTakeFirst(); + if (!u || u.media_entity_id !== entityId) throw new ApiException("bad_request", "内容单元不属于该作品"); + unitIds = [u.id]; + primary = u.id; + } else if (input.contentUnitId === null) { + unitIds = []; + primary = null; + } else { + const r = await ensureUnits(trx, entityId, entity.kind, parsed); + unitIds = r.unitIds; + primary = r.primary; + } + const cols = toParsedColumns(parsed); + await trx + .updateTable("source_release") + .set({ + parser_version: cols.parserVersion, + parsed: jsonb(cols.parsed), + media_entity_id: entityId, + content_unit_id: primary, + resolution_status: "resolved", + resolution_locked: input.lock, + resolution_evidence: jsonb({ + level: "manual", + by: actorName(actor), + reason: input.reason ?? null, + previous: { mediaEntityId: release.media_entity_id, contentUnitId: release.content_unit_id }, + }), + }) + .where("id", "=", srId) + .execute(); + await trx + .updateTable("identity_candidate") + .set({ + status: "resolved", + resolved_media_entity_id: entityId, + resolved_by: actorName(actor), + resolved_at: new Date(), + next_review_at: null, + }) + .where("source_release_id", "=", srId) + .execute(); + await closeDecisions( + ctx, + trx, + `identity:${srId}`, + "resolved", + { mediaEntityId: entityId, by: "manual" }, + actorName(actor), + ); + await recordAttempt( + trx, + srId, + null, + "manual", + input.reason ?? null, + "manual", + 1, + entityId, + { mediaEntityId: entityId }, + null, + actor, + ); + await audit( + trx, + actor, + "identity.release_assigned", + { type: "source_release", id: srId }, + { + mediaEntityId: entityId, + contentUnitIds: unitIds, + locked: input.lock, + previous: release.media_entity_id, + }, + ); + await ctx.outbox.emit(trx, "release.resolved", { + sourceReleaseId: srId, + observationId: null, + mediaEntityId: entityId, + contentUnitIds: unitIds, + evidenceLevel: "manual", + }); + return { mediaEntityId: entityId, contentUnitIds: unitIds }; + }); + } catch (e) { + if (e instanceof IdentityConflictError && attempt < 2) continue; + throw e; + } + } +} + +async function nextRevision(exec: Exec, identityId: string): Promise { + const r = await exec + .selectFrom("identity_revision") + .select((eb) => eb.fn.max("revision").as("m")) + .where("external_identity_id", "=", identityId) + .executeTakeFirst(); + return (Number(r?.m) || 0) + 1; +} + +async function retireIdentity(exec: Exec, row: ExternalIdentityRow, actor: Actor, reason: string | null) { + const rev = Math.max(row.revision + 1, await nextRevision(exec, row.id)); + await exec + .updateTable("external_identity") + .set({ status: "retired", valid_until: new Date(), revision: rev, locked: true }) + .where("id", "=", row.id) + .execute(); + await exec + .insertInto("identity_revision") + .values({ + external_identity_id: row.id, + revision: rev, + action: "status_changed", + from_status: row.status, + to_status: "retired", + before: jsonb({ status: row.status, locked: row.locked }), + after: jsonb({ status: "retired", locked: true }), + actor_type: actor.type, + actor_id: actor.id, + reason, + }) + .execute(); +} + +/** + * 改作品的外部映射:同 provider+类型的旧 accepted 映射退役(留修订并锁定,防止自动匹配改回), + * 新映射 accepted + 人工锁。目标身份已被其它作品接受时 409(应走合并)。 + */ +export async function setEntityIdentity(rt: SourcesRuntime, entityId: string, input: EntityIdentitySet, actor: Actor) { + const { ctx } = rt; + return ctx.db.transaction().execute(async (trx) => { + const id = await resolveEntityId(trx, entityId); + if (!id) throw new ApiException("not_found", "作品不存在"); + const other = await entityByExternal(trx, input); + if (other && other !== id) { + throw new ApiException("conflict", `该外部身份已属于作品 ${other},如为重复作品请使用合并`); + } + if (other === id) return { mediaEntityId: id, changed: false }; + const olds = await trx + .selectFrom("external_identity") + .selectAll() + .where("media_entity_id", "=", id) + .where("provider", "=", input.provider) + .where("entity_type", "=", input.entityType) + .where("status", "=", "accepted") + .execute(); + for (const o of olds) await retireIdentity(trx, o, actor, input.reason ?? null); + // 同一外部身份的旧候选(proposed/retired)不复用,新建一行保留历史 + const newId = await acceptIdentity(trx, id, input, "manual", { by: actorName(actor) }, actor, { + locked: input.lock, + reason: input.reason ?? null, + }); + if (!newId) throw new ApiException("conflict", "外部身份唯一约束冲突"); + await audit( + trx, + actor, + "identity.mapping_changed", + { type: "media_entity", id }, + { + ...input, + retired: olds.map((o) => o.id), + identityId: newId, + }, + ); + return { mediaEntityId: id, changed: true, identityId: newId, retired: olds.map((o) => o.id) }; + }); +} + +/** 外部身份字段修订(乐观并发 expectedRevision);每次变更写 identity_revision */ +export async function updateExternalIdentity( + rt: SourcesRuntime, + id: string, + input: ExternalIdentityUpdate, + actor: Actor, +) { + return rt.ctx.db.transaction().execute(async (trx) => { + const row = await trx + .selectFrom("external_identity") + .selectAll() + .where("id", "=", id) + .forUpdate() + .executeTakeFirst(); + if (!row) throw new ApiException("not_found", "外部身份不存在"); + if (row.revision !== input.expectedRevision) { + throw new ApiException("precondition_failed", `修订号已变化(当前 ${row.revision})`); + } + const rev = Math.max(row.revision + 1, await nextRevision(trx, id)); + const patch: Record = { revision: rev }; + if (input.status !== undefined) patch.status = input.status; + if (input.evidenceLevel !== undefined) patch.evidence_level = input.evidenceLevel; + if (input.evidence !== undefined) patch.evidence = jsonb(input.evidence); + if (input.locked !== undefined) patch.locked = input.locked; + if (input.validUntil !== undefined) patch.valid_until = input.validUntil; + if (input.extra !== undefined) patch.extra = jsonb(input.extra); + if (input.status === "retired" && input.validUntil === undefined) patch.valid_until = new Date(); + let updated: ExternalIdentityRow; + try { + updated = await trx + .updateTable("external_identity") + .set(patch) + .where("id", "=", id) + .returningAll() + .executeTakeFirstOrThrow(); + } catch (e) { + if ((e as { code?: string }).code === "23505") + throw new ApiException("conflict", "同一外部身份已有 accepted 映射"); + throw e; + } + const action = + input.status !== undefined && input.status !== row.status + ? "status_changed" + : input.locked !== undefined && input.locked !== row.locked + ? input.locked + ? "locked" + : "unlocked" + : "evidence_updated"; + await trx + .insertInto("identity_revision") + .values({ + external_identity_id: id, + revision: rev, + action, + from_status: row.status, + to_status: updated.status, + before: jsonb({ status: row.status, locked: row.locked, evidenceLevel: row.evidence_level }), + after: jsonb({ status: updated.status, locked: updated.locked, evidenceLevel: updated.evidence_level }), + actor_type: actor.type, + actor_id: actor.id, + reason: input.reason ?? null, + }) + .execute(); + await audit(trx, actor, "identity.updated", { type: "external_identity", id }, { action, revision: rev }); + return updated; + }); +} + +/** + * 合并重复作品:from → into。旧 ID 保留(status=merged + merged_into_id)并写 identity_redirect; + * 本模块的引用(来源资源、别名、待识别池、外部身份)迁到目标;其它模块的表按 merged_into_id 跳转解析。 + */ +export async function mergeEntities( + rt: SourcesRuntime, + fromId: string, + intoId: string, + reason: string | null, + actor: Actor, +) { + if (fromId === intoId) throw new ApiException("bad_request", "不能合并到自身"); + return rt.ctx.db.transaction().execute(async (trx) => { + const from = await trx + .selectFrom("media_entity") + .selectAll() + .where("id", "=", fromId) + .forUpdate() + .executeTakeFirst(); + const intoResolved = await resolveEntityId(trx, intoId); + if (!from) throw new ApiException("not_found", "源作品不存在"); + if (!intoResolved) throw new ApiException("not_found", "目标作品不存在"); + if (from.status === "merged") throw new ApiException("conflict", "源作品已被合并"); + if (intoResolved === fromId) throw new ApiException("conflict", "目标作品已合并到源作品,会形成环"); + + // 外部身份:目标已有同 provider+类型 accepted 的,源的退役;否则迁到目标 + const idents = await trx + .selectFrom("external_identity") + .selectAll() + .where("media_entity_id", "=", fromId) + .where("status", "=", "accepted") + .execute(); + for (const i of idents) { + const clash = await trx + .selectFrom("external_identity") + .select(["id"]) + .where("media_entity_id", "=", intoResolved) + .where("provider", "=", i.provider) + .where("entity_type", "=", i.entity_type) + .where("status", "=", "accepted") + .executeTakeFirst(); + if (clash) { + await retireIdentity(trx, i, actor, `merged into ${intoResolved}`); + continue; + } + const rev = Math.max(i.revision + 1, await nextRevision(trx, i.id)); + await trx + .updateTable("external_identity") + .set({ media_entity_id: intoResolved, revision: rev }) + .where("id", "=", i.id) + .execute(); + await trx + .insertInto("identity_revision") + .values({ + external_identity_id: i.id, + revision: rev, + action: "relinked", + from_status: i.status, + to_status: i.status, + before: jsonb({ mediaEntityId: fromId }), + after: jsonb({ mediaEntityId: intoResolved }), + actor_type: actor.type, + actor_id: actor.id, + reason, + }) + .execute(); + } + await sql`insert into title_alias (media_entity_id, content_unit_id, alias, normalized, language, kind, source, locked) + select ${intoResolved}, null, alias, normalized, language, kind, source, locked from title_alias + where media_entity_id = ${fromId} and content_unit_id is null + on conflict (media_entity_id, content_unit_id, normalized, kind) do nothing`.execute(trx); + await trx + .updateTable("source_release") + .set({ media_entity_id: intoResolved }) + .where("media_entity_id", "=", fromId) + .execute(); + await trx + .updateTable("identity_candidate") + .set({ resolved_media_entity_id: intoResolved }) + .where("resolved_media_entity_id", "=", fromId) + .execute(); + await trx + .updateTable("media_entity") + .set({ status: "merged", merged_into_id: intoResolved }) + .where("id", "=", fromId) + .execute(); + await trx + .insertInto("identity_redirect") + .values({ + from_entity_id: fromId, + to_entity_id: intoResolved, + reason, + actor_type: actor.type, + actor_id: actor.id, + }) + .execute(); + await audit( + trx, + actor, + "identity.entity_merged", + { type: "media_entity", id: fromId }, + { intoId: intoResolved, reason }, + ); + return trx.selectFrom("media_entity").selectAll().where("id", "=", intoResolved).executeTakeFirstOrThrow(); + }); +} + +export async function dismissPoolEntry(rt: SourcesRuntime, poolId: string, actor: Actor, note: string | null) { + const { ctx } = rt; + return ctx.db.transaction().execute(async (trx) => { + const row = await trx + .updateTable("identity_candidate") + .set({ status: "dismissed", next_review_at: null, resolved_by: actorName(actor), resolved_at: new Date() }) + .where("id", "=", poolId) + .returningAll() + .executeTakeFirst(); + if (!row) throw new ApiException("not_found", "待识别条目不存在"); + await trx + .updateTable("source_release") + .set({ resolution_status: "rejected" }) + .where("id", "=", row.source_release_id) + .where("resolution_locked", "=", false) + .execute(); + await closeDecisions(ctx, trx, `identity:${row.source_release_id}`, "dismissed", { note }, actorName(actor)); + await audit(trx, actor, "identity.pool_dismissed", { type: "identity_candidate", id: poolId }, { note }); + return row; + }); +} diff --git a/apps/server/src/modules/sources-identity/service/metadata.ts b/apps/server/src/modules/sources-identity/service/metadata.ts new file mode 100644 index 00000000..66d4be09 --- /dev/null +++ b/apps/server/src/modules/sources-identity/service/metadata.ts @@ -0,0 +1,254 @@ +import { type MetadataCandidate, TmdbError } from "@umt/adapters"; +import type { EntityMetadataField, FieldOverrideInput, FinalFieldValue, MetadataSource } from "@umt/contracts"; +import { sql } from "@umt/db"; +import { ApiException } from "../../../http/errors.ts"; +import { jsonb } from "../rows.ts"; +import type { SourcesRuntime } from "../runtime.ts"; +import { type Actor, audit, type Exec } from "./common.ts"; +import { resolveEntityId, seedAliases } from "./entities.ts"; + +// refreshMetadata:字段级刷新(REQ §7.3)。来源值写 metadata_field;最终值 = 覆盖(锁)> 已锁字段保持 > 来源值。 +// 只更新 media_entity 的展示字段与别名,不触碰 path_binding(路径绑定不随元数据变化)。 + +/** 最终值投影到 media_entity 列的字段 */ +const COLUMN_FIELDS: Partial> = { + title: "title", + originalTitle: "original_title", + year: "year", +}; + +function tmdbFields(m: MetadataCandidate): Partial> { + return { + title: m.title, + originalTitle: m.originalTitle, + year: m.year, + overview: m.overview, + releaseDate: m.releaseDate, + originalLanguage: m.originalLanguage, + originCountry: m.originCountry, + }; +} + +export interface RefreshResult { + status: "refreshed" | "skipped" | "failed"; + reason?: string; + changedFields: string[]; + lockedFields: string[]; +} + +export async function refreshMetadata(rt: SourcesRuntime, mediaEntityId: string): Promise { + const { ctx } = rt; + const id = await resolveEntityId(ctx.db, mediaEntityId); + if (!id) return { status: "skipped", reason: "not_found", changedFields: [], lockedFields: [] }; + const ident = await ctx.db + .selectFrom("external_identity") + .select(["entity_type", "external_id"]) + .where("media_entity_id", "=", id) + .where("provider", "=", "tmdb") + .where("status", "=", "accepted") + .executeTakeFirst(); + const tmdb = rt.tmdb(); + if (!ident || !tmdb || (ident.entity_type !== "movie" && ident.entity_type !== "tv")) { + return { + status: "skipped", + reason: ident ? "tmdb_not_configured" : "no_tmdb_identity", + changedFields: [], + lockedFields: [], + }; + } + let detail: MetadataCandidate; + try { + detail = await tmdb.details(ident.entity_type, ident.external_id); + } catch (e) { + // 404/超时/鉴权失败只记录校验状态,不是删除映射的依据 + if (e instanceof TmdbError) { + await ctx.db + .updateTable("external_identity") + .set({ verify_status: e.kind === "notfound" ? "not_found" : "error", verified_at: rt.now() }) + .where("media_entity_id", "=", id) + .where("provider", "=", "tmdb") + .where("status", "=", "accepted") + .execute(); + if (e.kind === "notfound" || e.kind === "auth") { + return { status: "failed", reason: e.kind, changedFields: [], lockedFields: [] }; + } + } + throw e; + } + const now = rt.now(); + return ctx.db.transaction().execute(async (trx) => { + const fields = tmdbFields(detail); + for (const [field, value] of Object.entries(fields)) { + await sql`insert into metadata_field (media_entity_id, content_unit_id, field, value, language, source, source_revision, fetched_at) + values (${id}, null, ${field}, ${jsonb(value)}::jsonb, ${null}, 'tmdb', ${`tmdb:${ident.entity_type}/${ident.external_id}`}, ${now}) + on conflict (media_entity_id, content_unit_id, field, source, language) do update set + value = excluded.value, source_revision = excluded.source_revision, fetched_at = excluded.fetched_at`.execute( + trx, + ); + } + await trx + .updateTable("external_identity") + .set({ verify_status: "ok", verified_at: now }) + .where("media_entity_id", "=", id) + .where("provider", "=", "tmdb") + .where("status", "=", "accepted") + .execute(); + await seedAliases(trx, id, detail); + const r = await project(trx, id, now); + await trx.updateTable("media_entity").set({ last_metadata_refresh_at: now }).where("id", "=", id).execute(); + return { status: "refreshed" as const, ...r }; + }); +} + +/** + * 重新投影最终字段:覆盖(用户锁)> field_locks 中已锁字段保持现值 > tmdb 来源值。 + * 返回被改变与被锁住的字段。 + */ +export async function project( + trx: Exec, + entityId: string, + now: Date, +): Promise<{ changedFields: string[]; lockedFields: string[] }> { + const e = await trx + .selectFrom("media_entity") + .selectAll() + .where("id", "=", entityId) + .forUpdate() + .executeTakeFirstOrThrow(); + const overrides = await trx + .selectFrom("field_override") + .selectAll() + .where("media_entity_id", "=", entityId) + .execute(); + const sources = await trx + .selectFrom("metadata_field") + .selectAll() + .where("media_entity_id", "=", entityId) + .orderBy("fetched_at", "desc") + .execute(); + const meta = { ...((e.metadata ?? {}) as Record) }; + const locks = new Set(e.field_locks); + const changed: string[] = []; + const lockedFields: string[] = []; + const colPatch: Record = {}; + const fieldNames = new Set([...sources.map((s) => s.field), ...overrides.map((o) => o.field)]); + for (const field of fieldNames) { + const ov = overrides.find((o) => o.field === field); + let next: FinalFieldValue | null = null; + if (ov) { + next = { + value: ov.value, + source: ov.source as MetadataSource, + language: null, + at: ov.updated_at.toISOString(), + locked: ov.locked, + }; + if (ov.locked) lockedFields.push(field); + } else if (locks.has(field)) { + lockedFields.push(field); + continue; + } else { + const src = + sources.find((s) => s.field === field && s.source === "tmdb") ?? sources.find((s) => s.field === field); + if (!src) continue; + next = { + value: src.value, + source: src.source as MetadataSource, + language: src.language, + at: src.fetched_at.toISOString(), + locked: false, + }; + } + const prev = meta[field]; + if (!prev || JSON.stringify(prev.value) !== JSON.stringify(next.value) || prev.source !== next.source) { + changed.push(field); + } + meta[field] = next; + const col = COLUMN_FIELDS[field as EntityMetadataField]; + if (col) { + const v = next.value; + if (col === "year") colPatch.year = typeof v === "number" ? v : null; + else if (col === "title") { + if (typeof v === "string" && v) colPatch.title = v; + } else colPatch.original_title = typeof v === "string" ? v : null; + } + } + await trx + .updateTable("media_entity") + .set({ ...colPatch, metadata: jsonb(meta), updated_at: now }) + .where("id", "=", entityId) + .execute(); + return { changedFields: changed, lockedFields }; +} + +/** 用户覆盖字段并(默认)锁定:之后的自动刷新不得改写 */ +export async function setFieldOverride( + rt: SourcesRuntime, + entityId: string, + field: EntityMetadataField, + input: FieldOverrideInput, + actor: Actor, +) { + return rt.ctx.db.transaction().execute(async (trx) => { + const id = await resolveEntityId(trx, entityId); + if (!id) throw new ApiException("not_found", "作品不存在"); + if (field === "title" && (typeof input.value !== "string" || !input.value.trim())) { + throw new ApiException("bad_request", "title 必须是非空字符串"); + } + if ( + field === "year" && + input.value !== null && + (typeof input.value !== "number" || !Number.isInteger(input.value)) + ) { + throw new ApiException("bad_request", "year 必须是整数或 null"); + } + await sql`insert into field_override (media_entity_id, content_unit_id, field, value, locked, source, reason, actor_type, actor_id) + values (${id}, null, ${field}, ${jsonb(input.value)}::jsonb, ${input.lock}, 'user', ${input.reason ?? null}, ${actor.type}, ${actor.id}) + on conflict (media_entity_id, content_unit_id, field) do update set + value = excluded.value, locked = excluded.locked, reason = excluded.reason, + actor_type = excluded.actor_type, actor_id = excluded.actor_id`.execute(trx); + const e = await trx + .selectFrom("media_entity") + .select(["field_locks"]) + .where("id", "=", id) + .executeTakeFirstOrThrow(); + const locks = new Set(e.field_locks); + if (input.lock) locks.add(field); + else locks.delete(field); + await trx + .updateTable("media_entity") + .set({ field_locks: [...locks].sort() }) + .where("id", "=", id) + .execute(); + const r = await project(trx, id, rt.now()); + await audit(trx, actor, "metadata.field_override", { type: "media_entity", id }, { field, lock: input.lock }); + return { mediaEntityId: id, ...r }; + }); +} + +/** 取消覆盖与锁定:回到来源值 */ +export async function clearFieldOverride( + rt: SourcesRuntime, + entityId: string, + field: EntityMetadataField, + actor: Actor, +) { + return rt.ctx.db.transaction().execute(async (trx) => { + const id = await resolveEntityId(trx, entityId); + if (!id) throw new ApiException("not_found", "作品不存在"); + await trx.deleteFrom("field_override").where("media_entity_id", "=", id).where("field", "=", field).execute(); + const e = await trx + .selectFrom("media_entity") + .select(["field_locks"]) + .where("id", "=", id) + .executeTakeFirstOrThrow(); + await trx + .updateTable("media_entity") + .set({ field_locks: e.field_locks.filter((f) => f !== field) }) + .where("id", "=", id) + .execute(); + const r = await project(trx, id, rt.now()); + await audit(trx, actor, "metadata.field_override_cleared", { type: "media_entity", id }, { field }); + return { mediaEntityId: id, ...r }; + }); +} diff --git a/apps/server/src/modules/sources-identity/service/resolve.ts b/apps/server/src/modules/sources-identity/service/resolve.ts new file mode 100644 index 00000000..c4ea9902 --- /dev/null +++ b/apps/server/src/modules/sources-identity/service/resolve.ts @@ -0,0 +1,637 @@ +import { createHash } from "node:crypto"; +import { type MetadataCandidate, TmdbError } from "@umt/adapters"; +import type { EvidenceLevel, PoolCandidate } from "@umt/contracts"; +import { type SourceReleaseRow, sql } from "@umt/db"; +import { + type IdentityCandidate as DomainCandidate, + type IdentityResolution, + normalizeTitle, + PARSER_VERSION, + type ParsedRelease, + parseRelease, + queryTitles, + type RankedCandidate, + resolveIdentity, +} from "@umt/domain"; +import { + localIdentityCandidate, + toContractEvidenceLevel, + toContractResolveStatus, + toParsedColumns, + toReleaseResolutionStatus, +} from "@umt/mapping"; +import { jsonb } from "../rows.ts"; +import type { SourcesRuntime } from "../runtime.ts"; +import { type Actor, closeDecisions, type Exec, openDecision, SYSTEM } from "./common.ts"; +import { + addAlias, + createEntityFromTmdb, + ensureUnits, + entityByExternal, + IdentityConflictError, + looksLikeVariety, + resolveEntityId, +} from "./entities.ts"; + +// resolveObservation:parseRelease → 本地召回(别名 pg_trgm + 已有外部身份 + 人工锁)→ TMDB 候选 → domain.resolveIdentity。 +// auto 才建/关联正式作品;ambiguous / no_match / 查询失败进待识别池(候选只存 JSON),有界退避复查。 + +const MAX_REVIEW_ATTEMPTS = 8; +const TRGM_THRESHOLD = 0.45; + +export interface ResolveInput { + observationId?: string | undefined; + sourceReleaseId?: string | undefined; + /** 复查/手动重试:忽略"已识别且证据未变"的短路 */ + force?: boolean | undefined; +} + +export interface ResolveResult { + outcome: "auto" | "proposed" | "ambiguous" | "no_match" | "lookup_failed" | "skipped_locked" | "skipped"; + sourceReleaseId: string | null; + mediaEntityId: string | null; + contentUnitIds: string[]; + decisionId: string | null; + reason: string; +} + +/** 候选的稳定键 */ +const candKey = (c: DomainCandidate) => + c.localEntityId ? `local:${c.localEntityId}` : `${c.provider}:${c.entityType}:${c.externalId}`; + +export function evidenceVersion(parsed: ParsedRelease | null, ranked: RankedCandidate[]): string { + const h = createHash("sha1") + .update( + JSON.stringify([parsed?.title ?? null, parsed?.year ?? null, ranked.map((r) => [candKey(r.candidate), r.level])]), + ) + .digest("hex") + .slice(0, 16); + return `p${PARSER_VERSION}:${h}`; +} + +function yearOf(d: string | null | undefined): number | undefined { + return d && /^\d{4}/.test(d) ? Number(d.slice(0, 4)) : undefined; +} + +export function tmdbToCandidate(m: MetadataCandidate): DomainCandidate { + const titles = [m.title, ...(m.originalTitle ? [m.originalTitle] : [])]; + for (const t of m.titles) if (t.kind !== "alias" && !titles.includes(t.text)) titles.push(t.text); + const aliases = [...new Set([...m.aliases, ...m.titles.filter((t) => t.kind === "alias").map((t) => t.text)])]; + const seasonYears = m.seasons?.map((s) => yearOf(s.airDate)).filter((y): y is number => y !== undefined); + return { + provider: "tmdb", + entityType: m.entityType, + externalId: m.externalId, + titles, + ...(m.year !== null ? { year: m.year } : {}), + aliases, + ...(seasonYears?.length ? { seasonYears } : {}), + }; +} + +function poolCandidate(r: RankedCandidate): PoolCandidate { + const c = r.candidate; + return { + provider: c.provider, + entityType: c.entityType, + externalId: c.externalId, + title: c.titles[0] ?? c.externalId, + year: c.year ?? null, + localEntityId: c.localEntityId ?? null, + level: r.level, + evidence: r.evidence, + }; +} + +/** 本地召回:显式外部 ID → 已接受映射;别名精确/trgm 模糊(跟随合并跳转),构造本地候选 */ +async function localCandidates( + exec: Exec, + parsed: ParsedRelease, + release: SourceReleaseRow, +): Promise { + const ids = new Set(); + const site = ((release.raw as Record)?._site ?? {}) as { imdbId?: string }; + const explicit = [...parsed.explicitIds, ...(site.imdbId ? [{ provider: "imdb" as const, id: site.imdbId }] : [])]; + for (const x of explicit) { + const row = await exec + .selectFrom("external_identity") + .select(["media_entity_id"]) + .where("provider", "=", x.provider) + .where("external_id", "=", x.id) + .where("status", "=", "accepted") + .where("media_entity_id", "is not", null) + .execute(); + for (const r of row) if (r.media_entity_id) ids.add(r.media_entity_id); + } + for (const q of queryTitles(parsed)) { + const rows = await sql<{ media_entity_id: string }>` + select distinct media_entity_id from title_alias + where normalized = ${q} or (length(${q}) >= 3 and similarity(normalized, ${q}) >= ${TRGM_THRESHOLD}) + limit 20`.execute(exec); + for (const r of rows.rows) ids.add(r.media_entity_id); + } + const final = new Set(); + for (const id of ids) { + const r = await resolveEntityId(exec, id); + if (r) final.add(r); + } + const out: DomainCandidate[] = []; + for (const id of final) { + const e = await exec.selectFrom("media_entity").selectAll().where("id", "=", id).executeTakeFirst(); + if (!e || e.status === "archived") continue; + const aliases = await exec.selectFrom("title_alias").select(["alias"]).where("media_entity_id", "=", id).execute(); + const idents = await exec + .selectFrom("external_identity") + .select(["provider", "external_id", "status"]) + .where("media_entity_id", "=", id) + .execute(); + const c = localIdentityCandidate( + { id: e.id, kind: e.kind, title: e.title, originalTitle: e.original_title, year: e.year }, + aliases.map((a) => a.alias), + idents.map((i) => ({ provider: i.provider, externalId: i.external_id, status: i.status })), + ); + if (c) out.push(c); + } + return out; +} + +async function tmdbCandidates(rt: SourcesRuntime, parsed: ParsedRelease): Promise { + const tmdb = rt.tmdb(); + if (!tmdb) return []; + const queries = [parsed.titleCn, parsed.titleEn, parsed.title].filter((t): t is string => !!t); + const seen = new Map(); + for (const q of [...new Set(queries)].slice(0, 2)) { + for (const m of await tmdb.searchAll(q)) { + const k = `${m.entityType}:${m.externalId}`; + if (!seen.has(k)) seen.set(k, m); + } + } + return [...seen.values()].slice(0, 20); +} + +async function loadRelease(rt: SourcesRuntime, input: ResolveInput) { + const { db } = rt.ctx; + const observationId = input.observationId ?? null; + let srId = input.sourceReleaseId ?? null; + if (!srId && observationId) { + const o = await db + .selectFrom("observation") + .select(["source_release_id"]) + .where("id", "=", observationId) + .executeTakeFirst(); + srId = o?.source_release_id ?? null; + } + if (!srId) return { release: null, observationId }; + const release = await db.selectFrom("source_release").selectAll().where("id", "=", srId).executeTakeFirst(); + return { release: release ?? null, observationId }; +} + +export async function resolveRelease(rt: SourcesRuntime, input: ResolveInput): Promise { + const { ctx } = rt; + const { release, observationId } = await loadRelease(rt, input); + const base = { mediaEntityId: null, contentUnitIds: [] as string[], decisionId: null }; + if (!release) return { outcome: "skipped", sourceReleaseId: null, reason: "release_not_found", ...base }; + + const pool = await ctx.db + .selectFrom("identity_candidate") + .selectAll() + .where("source_release_id", "=", release.id) + .executeTakeFirst(); + if (release.resolution_locked) { + await recordAttempt( + ctx.db, + release.id, + observationId, + "skipped_locked", + "manual_lock", + "locked", + 0, + release.media_entity_id, + ); + return { + outcome: "skipped_locked", + sourceReleaseId: release.id, + reason: "manual_lock", + ...base, + mediaEntityId: release.media_entity_id, + }; + } + if (pool?.status === "dismissed" && !input.force) { + return { outcome: "skipped", sourceReleaseId: release.id, reason: "dismissed", ...base }; + } + + // 已识别:除非强制(复查/标题更新/人工重试),不重复召回 + if (!input.force && release.resolution_status === "resolved" && release.media_entity_id) { + return { + outcome: "skipped", + sourceReleaseId: release.id, + reason: "already_resolved", + ...base, + mediaEntityId: release.media_entity_id, + }; + } + const parsed = parseRelease(release.title, release.subtitle ? { subtitle: release.subtitle } : {}); + const cols = toParsedColumns(parsed); + if (!parsed.title || parsed.auxiliaryOnly) { + return toPool(rt, release, observationId, parsed, null, [], "no_match", "unparseable_title", null); + } + + // 召回:本地优先,再 TMDB;TMDB 失败不等于无结果 + const local = await localCandidates(ctx.db, parsed, release); + let remote: MetadataCandidate[] = []; + let lookupError: string | null = null; + try { + remote = await tmdbCandidates(rt, parsed); + } catch (e) { + if (!(e instanceof TmdbError)) throw e; + lookupError = `${e.kind}: ${e.message}`; + } + const candidates = [...local, ...remote.map(tmdbToCandidate)]; + const resolution = resolveIdentity(parsed, candidates); + const version = evidenceVersion(parsed, resolution.ranked); + + // 已识别且证据未变:幂等短路(强制复查时) + if ( + release.resolution_status === "resolved" && + release.media_entity_id && + (release.resolution_evidence as { evidenceVersion?: string })?.evidenceVersion === version + ) { + return { + outcome: "skipped", + sourceReleaseId: release.id, + reason: "unchanged", + ...base, + mediaEntityId: release.media_entity_id, + }; + } + + const automation = await ctx.settings.get("automation"); + const autoAccept = automation.enabled && !automation.readOnlyReconcile; + if (resolution.status === "auto" && resolution.best && autoAccept) { + const best = resolution.best; + const remoteBest = + best.candidate.provider === "tmdb" + ? remote.find((m) => m.externalId === best.candidate.externalId && m.entityType === best.candidate.entityType) + : undefined; + let detailed = remoteBest; + if (remoteBest && !remoteBest.detailed) { + const existing = await entityByExternal(ctx.db, { + provider: "tmdb", + entityType: remoteBest.entityType, + externalId: remoteBest.externalId, + }); + if (!existing) { + try { + detailed = await rt.tmdb()!.details(remoteBest.entityType, remoteBest.externalId); + } catch (e) { + if (!(e instanceof TmdbError)) throw e; + detailed = remoteBest; + } + } + } + return bindAuto(rt, release, observationId, parsed, cols, resolution, version, detailed ?? null); + } + + if (resolution.status === "auto" && !autoAccept) { + return toPool( + rt, + release, + observationId, + parsed, + resolution, + resolution.ranked, + "proposed", + "automation_off", + version, + ); + } + if (lookupError && resolution.status !== "ambiguous") { + return toPool( + rt, + release, + observationId, + parsed, + resolution, + resolution.ranked, + "lookup_failed", + lookupError, + version, + ); + } + return toPool( + rt, + release, + observationId, + parsed, + resolution, + resolution.ranked, + resolution.status === "ambiguous" ? "ambiguous" : "no_match", + resolution.reason, + version, + ); +} + +async function recordAttempt( + exec: Exec, + srId: string, + observationId: string | null, + outcome: "auto" | "proposed" | "ambiguous" | "no_match" | "lookup_failed" | "manual" | "skipped_locked", + reason: string | null, + version: string, + count: number, + entityId: string | null, + best: unknown = null, + error: unknown = null, + actor: Actor = SYSTEM, +): Promise { + await exec + .insertInto("resolution_attempt") + .values({ + source_release_id: srId, + observation_id: observationId, + outcome, + reason, + evidence_version: version, + candidate_count: count, + media_entity_id: entityId, + best: best === null ? null : jsonb(best), + error: error === null ? null : jsonb(error), + actor_type: actor.type, + actor_id: actor.id, + }) + .execute(); +} + +function rankedSummary(r: RankedCandidate | undefined) { + return r ? { ...poolCandidate(r), titleMatch: r.titleMatch, yearMatch: r.yearMatch, typeMatch: r.typeMatch } : null; +} + +async function bindAuto( + rt: SourcesRuntime, + release: SourceReleaseRow, + observationId: string | null, + parsed: ParsedRelease, + cols: ReturnType, + resolution: IdentityResolution, + version: string, + tmdbDetail: MetadataCandidate | null, +): Promise { + const { ctx } = rt; + const best = resolution.best!; + const level: EvidenceLevel = toContractEvidenceLevel(best.level) ?? "needs_review"; + const site = ((release.raw as Record)?._site ?? {}) as { categoryName?: string }; + const variety = looksLikeVariety(parsed, site.categoryName ?? release.category_raw); + for (let attempt = 0; ; attempt++) { + try { + return await ctx.db.transaction().execute(async (trx) => { + let entityId: string | null = null; + if (best.candidate.localEntityId) entityId = await resolveEntityId(trx, best.candidate.localEntityId); + if (!entityId && best.candidate.provider === "tmdb") { + entityId = await entityByExternal(trx, { + provider: "tmdb", + entityType: best.candidate.entityType, + externalId: best.candidate.externalId, + }); + if (!entityId && tmdbDetail) { + entityId = ( + await createEntityFromTmdb(trx, tmdbDetail, { + level, + evidence: { + sourceReleaseId: release.id, + rules: best.evidence, + titleMatch: best.titleMatch, + yearMatch: best.yearMatch, + }, + actor: SYSTEM, + variety, + }) + ).entityId; + } + } + if (!entityId) + throw new Error(`auto 结果缺少可绑定的本地作品(${best.candidate.provider}:${best.candidate.externalId})`); + const entity = await trx + .selectFrom("media_entity") + .select(["id", "kind"]) + .where("id", "=", entityId) + .executeTakeFirstOrThrow(); + const units = await ensureUnits(trx, entityId, entity.kind, parsed); + // 站点标题作为别名:下次同名资源可直接本地召回 + if (parsed.titleCn) await addAlias(trx, entityId, parsed.titleCn, "site_title", "site", "zh"); + if (parsed.titleEn) await addAlias(trx, entityId, parsed.titleEn, "site_title", "site", "en"); + + const evidence = { + level, + reason: resolution.reason, + evidenceVersion: version, + best: rankedSummary(best), + candidateCount: resolution.ranked.length, + }; + await trx + .updateTable("source_release") + .set({ + parser_version: cols.parserVersion, + parsed: jsonb(cols.parsed), + media_entity_id: entityId, + content_unit_id: units.primary, + resolution_status: toReleaseResolutionStatus(toContractResolveStatus("auto", { autoAccept: true })), + resolution_evidence: jsonb(evidence), + }) + .where("id", "=", release.id) + .where("resolution_locked", "=", false) + .execute(); + await trx + .updateTable("identity_candidate") + .set({ + status: "resolved", + resolved_media_entity_id: entityId, + resolved_by: "system", + resolved_at: new Date(), + next_review_at: null, + }) + .where("source_release_id", "=", release.id) + .where("status", "=", "pending") + .execute(); + await closeDecisions( + ctx, + trx, + `identity:${release.id}`, + "resolved", + { mediaEntityId: entityId, by: "auto" }, + "system", + ); + await recordAttempt( + trx, + release.id, + observationId, + "auto", + resolution.reason, + version, + resolution.ranked.length, + entityId, + rankedSummary(best), + ); + await ctx.outbox.emit( + trx, + "release.resolved", + { + sourceReleaseId: release.id, + observationId, + mediaEntityId: entityId, + contentUnitIds: units.unitIds, + evidenceLevel: level, + }, + { idempotencyKey: `release.resolved:${release.id}:${entityId}:${version}` }, + ); + return { + outcome: "auto" as const, + sourceReleaseId: release.id, + mediaEntityId: entityId, + contentUnitIds: units.unitIds, + decisionId: null, + reason: resolution.reason, + }; + }); + } catch (e) { + if (e instanceof IdentityConflictError && attempt < 2) continue; + throw e; + } + } +} + +function reviewDelayMs(reason: string, attempts: number): number | null { + if (attempts >= MAX_REVIEW_ATTEMPTS) return null; + const base = reason === "lookup_failed" ? 15 * 60_000 : 3600_000; + return Math.min(base * 2 ** Math.max(0, attempts - 1), 7 * 24 * 3600_000); +} + +async function toPool( + rt: SourcesRuntime, + release: SourceReleaseRow, + observationId: string | null, + parsed: ParsedRelease, + resolution: IdentityResolution | null, + ranked: RankedCandidate[], + reason: "ambiguous" | "no_match" | "proposed" | "lookup_failed", + detail: string, + versionIn: string | null, +): Promise { + const { ctx } = rt; + const version = versionIn ?? evidenceVersion(parsed, ranked); + const cols = toParsedColumns(parsed); + const candidates = ranked + .filter((r) => r.level !== "rejected") + .slice(0, 10) + .map(poolCandidate); + const now = rt.now(); + const query = { + title: parsed.title ?? null, + titleCn: parsed.titleCn ?? null, + titleEn: parsed.titleEn ?? null, + normalized: parsed.title ? normalizeTitle(parsed.title) : null, + year: parsed.year ?? null, + season: parsed.season ?? null, + episodes: parsed.episodes ?? null, + }; + const contractStatus = + reason === "proposed" + ? toContractResolveStatus("auto", { autoAccept: false }) + : reason === "ambiguous" + ? "ambiguous" + : "unidentifiable"; + + return ctx.db.transaction().execute(async (trx) => { + const prev = await trx + .selectFrom("identity_candidate") + .select(["id", "attempts", "evidence_version"]) + .where("source_release_id", "=", release.id) + .executeTakeFirst(); + // 证据版本变化 = 新证据:从头计数;同一证据反复复查按退避递增,达上限停止自动复查(仍可检索) + const attempts = prev && prev.evidence_version === version ? prev.attempts + 1 : 1; + const delay = reviewDelayMs(reason, attempts); + const nextReviewAt = delay === null ? null : new Date(now.getTime() + delay); + + let decisionId: string | null = null; + if ((reason === "ambiguous" || reason === "proposed") && candidates.length > 0) { + const d = await openDecision(ctx, trx, { + kind: "identity_ambiguous", + severity: "info", + title: `待确认身份:${release.title}`, + summary: `${candidates.length} 个候选证据等级相近(${detail}),请选择或忽略`, + subjectType: "source_release", + subjectId: release.id, + dedupeKey: `identity:${release.id}`, + evidence: { + sourceReleaseId: release.id, + siteId: release.site_id, + reason, + detail, + evidenceVersion: version, + query, + }, + options: [ + ...candidates.slice(0, 5).map((c, i) => ({ + key: `c${i}`, + label: `${c.title}${c.year ? ` (${c.year})` : ""} · ${c.localEntityId ? "本地作品" : `${c.provider}:${c.entityType}/${c.externalId}`}`, + input: { sourceReleaseId: release.id, candidateIndex: i }, + })), + { key: "dismiss", label: "忽略此资源" }, + ], + nextReviewAt, + }); + decisionId = d.id; + } + + await sql` + insert into identity_candidate (source_release_id, status, reason, query, candidates, evidence_version, attempts, + last_attempt_at, next_review_at, decision_id) + values (${release.id}, 'pending', ${reason}, ${jsonb(query)}::jsonb, ${jsonb(candidates)}::jsonb, ${version}, ${attempts}, + ${now}, ${nextReviewAt}, ${decisionId}) + on conflict (source_release_id) do update set + status = 'pending', reason = excluded.reason, query = excluded.query, candidates = excluded.candidates, + evidence_version = excluded.evidence_version, attempts = excluded.attempts, last_attempt_at = excluded.last_attempt_at, + next_review_at = excluded.next_review_at, decision_id = coalesce(excluded.decision_id, identity_candidate.decision_id), + resolved_media_entity_id = null, resolved_by = null, resolved_at = null`.execute(trx); + + await trx + .updateTable("source_release") + .set({ + parser_version: cols.parserVersion, + parsed: jsonb(cols.parsed), + resolution_status: toReleaseResolutionStatus(contractStatus), + resolution_evidence: jsonb({ reason, detail, evidenceVersion: version, candidateCount: candidates.length }), + }) + .where("id", "=", release.id) + .where("resolution_locked", "=", false) + .execute(); + if (observationId) { + await trx + .updateTable("observation") + .set((eb) => ({ resolve_attempts: eb("resolve_attempts", "+", 1), next_resolve_at: nextReviewAt })) + .where("id", "=", observationId) + .execute(); + } + await recordAttempt( + trx, + release.id, + observationId, + reason, + detail, + version, + candidates.length, + null, + rankedSummary(resolution?.best), + reason === "lookup_failed" ? { message: detail } : null, + ); + return { + outcome: reason, + sourceReleaseId: release.id, + mediaEntityId: null, + contentUnitIds: [], + decisionId, + reason: detail, + }; + }); +} + +export { recordAttempt }; diff --git a/apps/server/src/modules/sources-identity/service/snapshot.ts b/apps/server/src/modules/sources-identity/service/snapshot.ts new file mode 100644 index 00000000..aef6f370 --- /dev/null +++ b/apps/server/src/modules/sources-identity/service/snapshot.ts @@ -0,0 +1,229 @@ +import { SiteError, type SitePage, type SiteRelease } from "@umt/adapters"; +import type { SiteSnapshotKind } from "@umt/contracts"; +import { type SiteRow, sql } from "@umt/db"; +import { parseRelease } from "@umt/domain"; +import { siteReleaseToSourceRelease } from "@umt/mapping"; +import { jsonb } from "../rows.ts"; +import type { SourcesRuntime } from "../runtime.ts"; +import { type Exec, openDecision } from "./common.ts"; + +// 站点快照(周期列表)、补漏(Feed 疑似缺口,有页数预算)、实时补查(工作台搜索)。 +// 所有请求经站点共享限流器;结果按 site+站点资源 ID upsert 到 source_release,并记录 site_snapshot 覆盖范围。 + +export interface SnapshotInput { + siteId: string; + kind: SiteSnapshotKind; + query?: string | undefined; + /** 页数预算(1 起连续页);默认 site.config.snapshotPages ?? 1,上限 20 */ + maxPages?: number | undefined; +} + +export interface SnapshotResult { + status: "ok" | "partial" | "failed" | "skipped"; + reason?: string; + snapshotId: string | null; + pages: number[]; + itemCount: number; + newCount: number; + releaseIds: string[]; + error?: string; +} + +/** SiteRelease → source_release upsert;新行发 release.observed */ +export async function upsertSiteRelease( + rt: SourcesRuntime, + trx: Exec, + site: Pick, + r: SiteRelease, +): Promise<{ id: string; inserted: boolean }> { + const parsed = parseRelease(r.title, r.subtitle ? { subtitle: r.subtitle } : {}); + const c = siteReleaseToSourceRelease(r, { siteId: site.id, parsed }); + const row = ( + await sql<{ id: string; inserted: boolean }>` + insert into source_release (site_id, site_release_id, title, subtitle, category_raw, tags_raw, size_bytes, infohash_v1, + infohash_v2, promotion, promotion_expires_at, seeders, leechers, source_published_at, parsed, parser_version, raw, + detail_fetched_at) + values (${site.id}, ${c.siteReleaseId}, ${c.title}, ${c.subtitle ?? null}, ${c.categoryRaw ?? null}, ${c.tagsRaw ?? []}, + ${c.sizeBytes ?? null}, ${c.infohashV1 ?? null}, ${c.infohashV2 ?? null}, ${jsonb(c.promotion ?? {})}::jsonb, + ${c.promotionExpiresAt ?? null}, ${c.seeders ?? null}, ${c.leechers ?? null}, ${c.sourcePublishedAt ?? null}, + ${jsonb(c.parsed ?? {})}::jsonb, ${c.parserVersion ?? null}, ${jsonb(c.raw ?? {})}::jsonb, ${null}) + on conflict (site_id, site_release_id) do update set + title = excluded.title, subtitle = coalesce(excluded.subtitle, source_release.subtitle), + category_raw = coalesce(excluded.category_raw, source_release.category_raw), tags_raw = excluded.tags_raw, + size_bytes = coalesce(excluded.size_bytes, source_release.size_bytes), + infohash_v1 = coalesce(excluded.infohash_v1, source_release.infohash_v1), + infohash_v2 = coalesce(excluded.infohash_v2, source_release.infohash_v2), + promotion = excluded.promotion, promotion_expires_at = excluded.promotion_expires_at, + seeders = excluded.seeders, leechers = excluded.leechers, + source_published_at = coalesce(source_release.source_published_at, excluded.source_published_at), + raw = source_release.raw || excluded.raw, last_seen_at = now(), + availability = case when source_release.availability = 'expired' then 'active' else source_release.availability end + returning id, (xmax = 0) as inserted`.execute(trx) + ).rows[0]!; + if (row.inserted) { + await rt.ctx.outbox.emit( + trx, + "release.observed", + { sourceReleaseId: row.id, observationId: null, siteId: site.id, isNew: true }, + { idempotencyKey: `release.observed:${row.id}` }, + ); + } + return row; +} + +/** 凭据失效:账号标失效 + 站点健康 failing + 待处理(不重试) */ +async function onAuthError(rt: SourcesRuntime, trx: Exec, site: SiteRow, e: SiteError) { + await trx + .updateTable("site_account") + .set({ status: e.kind === "auth_expired" ? "expired" : "invalid", last_verified_at: rt.now() }) + .where("site_id", "=", site.id) + .where("enabled", "=", true) + .where("status", "in", ["unknown", "active"]) + .execute(); + await trx.updateTable("site").set({ health_status: "failing" }).where("id", "=", site.id).execute(); + await openDecision(rt.ctx, trx, { + kind: "source_unreachable", + severity: "warning", + title: `站点「${site.name}」登录失效`, + summary: `${e.kind}:${e.message}。已停止该站自动访问,请更新账号凭证后恢复。`, + subjectType: "site", + subjectId: site.id, + dedupeKey: `site-auth:${site.id}`, + evidence: { siteId: site.id, kind: e.kind, httpStatus: e.httpStatus ?? null }, + options: [{ key: "reset", label: "已更新凭证,恢复访问" }], + }); +} + +export async function snapshotSite(rt: SourcesRuntime, input: SnapshotInput): Promise { + const { db } = rt.ctx; + const none = { snapshotId: null, pages: [], itemCount: 0, newCount: 0, releaseIds: [] }; + const site = await db.selectFrom("site").selectAll().where("id", "=", input.siteId).executeTakeFirst(); + if (!site) return { status: "skipped", reason: "not_found", ...none }; + if (!site.enabled) return { status: "skipped", reason: "site_disabled", ...none }; + if (site.health_status === "failing" && input.kind !== "manual" && input.kind !== "live_search") { + const invalid = await db + .selectFrom("site_account") + .select(["id"]) + .where("site_id", "=", site.id) + .where("status", "in", ["invalid", "expired"]) + .executeTakeFirst(); + if (invalid) return { status: "skipped", reason: "auth_invalid", ...none }; + } + let adapter: ReturnType; + try { + adapter = rt.siteAdapter(site); + } catch (e) { + return { status: "skipped", reason: (e as Error).message, ...none }; + } + const cfgPages = Number((site.config as { snapshotPages?: number })?.snapshotPages) || 1; + const maxPages = Math.min(20, Math.max(1, input.maxPages ?? cfgPages)); + const snap = await db + .insertInto("site_snapshot") + .values({ + site_id: site.id, + kind: input.kind, + query: input.query ?? null, + filters: jsonb({}), + definition_version: adapter.definitionVersion, + started_at: rt.now(), + }) + .returning(["id"]) + .executeTakeFirstOrThrow(); + + const pages: number[] = []; + const releases: SiteRelease[] = []; + let error: SiteError | Error | null = null; + for (let p = 1; p <= maxPages; p++) { + try { + const res: SitePage = input.query ? await adapter.search(input.query, { page: p }) : await adapter.list(p); + pages.push(p); + releases.push(...res.items); + if (res.items.length === 0 || (res.totalPages !== null && p >= res.totalPages)) break; + } catch (e) { + error = e as Error; + break; + } + } + + return db.transaction().execute(async (trx) => { + const ids: string[] = []; + let newCount = 0; + for (const r of releases) { + const row = await upsertSiteRelease(rt, trx, site, r); + ids.push(row.id); + if (row.inserted) newCount++; + } + const status = error ? (pages.length ? "partial" : "failed") : "ok"; + const errJson = error + ? { kind: error instanceof SiteError ? error.kind : "internal", message: error.message } + : null; + await trx + .updateTable("site_snapshot") + .set({ + status, + pages: jsonb(pages), + item_count: releases.length, + new_count: newCount, + error: errJson ? jsonb(errJson) : null, + finished_at: rt.now(), + }) + .where("id", "=", snap.id) + .execute(); + if (error instanceof SiteError && error.isAuthError) { + await onAuthError(rt, trx, site, error); + } else if (input.kind !== "live_search") { + await trx + .updateTable("site") + .set({ + ...(status !== "failed" ? { last_snapshot_at: rt.now() } : {}), + health_status: status === "ok" ? "healthy" : "degraded", + snapshot_cursor: jsonb({ + lastKind: input.kind, + lastPages: pages, + lastItemCount: releases.length, + lastStatus: status, + lastError: errJson, + at: rt.now().toISOString(), + }), + }) + .where("id", "=", site.id) + .execute(); + } + return { + status, + snapshotId: snap.id, + pages, + itemCount: releases.length, + newCount, + releaseIds: ids, + ...(errJson ? { error: `${errJson.kind}: ${errJson.message}` } : {}), + }; + }); +} + +/** 详情刷新(工作台 / release.inspect):失败保留既有数据并返回原因 */ +export async function refreshReleaseDetail(rt: SourcesRuntime, srId: string): Promise { + const { db } = rt.ctx; + const rel = await db.selectFrom("source_release").selectAll().where("id", "=", srId).executeTakeFirst(); + if (!rel) return "not_found"; + const site = await db.selectFrom("site").selectAll().where("id", "=", rel.site_id).executeTakeFirst(); + if (!site?.enabled) return "site_disabled"; + let detail: SiteRelease; + try { + detail = await rt.siteAdapter(site).detail(rel.site_release_id); + } catch (e) { + if (e instanceof SiteError) { + if (e.isAuthError) await db.transaction().execute((trx) => onAuthError(rt, trx, site, e)); + if (e.kind === "not_found") { + await db.updateTable("source_release").set({ availability: "expired" }).where("id", "=", srId).execute(); + } + return `${e.kind}: ${e.message}`; + } + return (e as Error).message; + } + await db.transaction().execute(async (trx) => { + await upsertSiteRelease(rt, trx, site, detail); + await trx.updateTable("source_release").set({ detail_fetched_at: rt.now() }).where("id", "=", srId).execute(); + }); + return null; +} diff --git a/apps/server/src/modules/sources-identity/service/workbench.ts b/apps/server/src/modules/sources-identity/service/workbench.ts new file mode 100644 index 00000000..9adffafc --- /dev/null +++ b/apps/server/src/modules/sources-identity/service/workbench.ts @@ -0,0 +1,218 @@ +import type { + ReleaseOrigin, + WorkbenchCoverage, + WorkbenchRelease, + WorkbenchReleaseDetail, + WorkbenchReleaseQuery, +} from "@umt/contracts"; +import { type SiteRow, sql } from "@umt/db"; +import { normalizeTitle } from "@umt/domain"; +import { ApiException } from "../../../http/errors.ts"; +import { decodeCursor, page, toApi } from "../rows.ts"; +import type { SourcesRuntime } from "../runtime.ts"; +import { refreshReleaseDetail, snapshotSite } from "./snapshot.ts"; + +// 站点工作台(整理视图):本地快照优先;本地无结果 / 快照过期 / 用户要求时对已开启站点实时补查, +// 并发相同搜索合并,失败短期退避。库存叠加由前端按 mediaEntityId 查 W2-C 端点。 + +const LIVE_FAIL_BACKOFF_MS = 60_000; + +interface LiveState { + inflight: Map>; + failedUntil: Map; +} + +const liveStates = new WeakMap(); +function liveState(rt: SourcesRuntime): LiveState { + let s = liveStates.get(rt); + if (!s) { + s = { inflight: new Map(), failedUntil: new Map() }; + liveStates.set(rt, s); + } + return s; +} + +/** 合并并发的相同实时搜索;失败后 LIVE_FAIL_BACKOFF_MS 内直接返回上次错误 */ +export async function liveSearch( + rt: SourcesRuntime, + site: SiteRow, + q: string, +): Promise<{ error: string | null; coalesced: boolean }> { + const st = liveState(rt); + const key = `${site.id}|${normalizeTitle(q) || q.toLowerCase()}`; + const failed = st.failedUntil.get(key); + if (failed && failed.until > rt.now().getTime()) return { error: `${failed.error}(退避中)`, coalesced: false }; + const running = st.inflight.get(key); + if (running) return { ...(await running), coalesced: true }; + const p = (async () => { + const r = await snapshotSite(rt, { siteId: site.id, kind: "live_search", query: q, maxPages: 1 }); + const error = r.status === "failed" || r.status === "skipped" ? (r.error ?? r.reason ?? "failed") : null; + if (error) st.failedUntil.set(key, { until: rt.now().getTime() + LIVE_FAIL_BACKOFF_MS, error }); + else st.failedUntil.delete(key); + return { error }; + })().finally(() => st.inflight.delete(key)); + st.inflight.set(key, p); + return { ...(await p), coalesced: false }; +} + +async function lastSnapshotAt(rt: SourcesRuntime, siteId: string): Promise { + const r = await rt.ctx.db + .selectFrom("site_snapshot") + .select(["finished_at"]) + .where("site_id", "=", siteId) + .where("kind", "in", ["periodic", "manual", "backfill"]) + .where("status", "in", ["ok", "partial"]) + .orderBy("started_at", "desc") + .executeTakeFirst(); + return r?.finished_at ?? null; +} + +interface ReleaseJoinRow { + [k: string]: unknown; + observed_by_feeds: number; + entity_title: string | null; + entity_year: number | null; + entity_kind: string | null; +} + +function toWorkbench(row: ReleaseJoinRow): WorkbenchRelease { + const { observed_by_feeds, entity_title, entity_year, entity_kind, ...rest } = row; + const base = toApi>(rest); + const raw = (base.raw ?? {}) as Record; + const origin: ReleaseOrigin = raw._site ? "snapshot" : Number(observed_by_feeds) > 0 ? "feed" : "unknown"; + return { + ...base, + origin, + dataUpdatedAt: base.lastSeenAt, + observedByFeeds: Number(observed_by_feeds), + entity: + base.mediaEntityId && entity_title + ? { id: base.mediaEntityId, title: entity_title, year: entity_year, kind: entity_kind ?? "other" } + : null, + }; +} + +function releaseSelect(rt: SourcesRuntime) { + return rt.ctx.db + .selectFrom("source_release as r") + .leftJoin("media_entity as e", "e.id", "r.media_entity_id") + .selectAll("r") + .select([ + "e.title as entity_title", + "e.year as entity_year", + "e.kind as entity_kind", + sql`(select count(distinct o.feed_id) from observation o where o.source_release_id = r.id)::int`.as( + "observed_by_feeds", + ), + ]); +} + +export async function listSiteReleases( + rt: SourcesRuntime, + siteId: string, + q: WorkbenchReleaseQuery, +): Promise<{ items: WorkbenchRelease[]; nextCursor: string | null; coverage: WorkbenchCoverage }> { + const site = await rt.ctx.db.selectFrom("site").selectAll().where("id", "=", siteId).executeTakeFirst(); + if (!site) throw new ApiException("not_found", "站点不存在"); + const snapAt = await lastSnapshotAt(rt, site.id); + const periodMs = (site.snapshot_interval_sec ?? 3600) * 1000; + const stale = !snapAt || rt.now().getTime() - snapAt.getTime() > periodMs * 2; + const offset = decodeCursor(q.cursor); + + const run = async () => { + let query = releaseSelect(rt).where("r.site_id", "=", site.id); + if (q.q) { + const like = `%${q.q.replace(/[\\%_]/g, (m) => `\\${m}`)}%`; + query = query.where((eb) => + eb.or([ + eb("r.title", "ilike", like), + eb("r.subtitle", "ilike", like), + eb("r.site_release_id", "=", q.q!), + sql`similarity(r.title, ${q.q}) >= 0.3`, + ]), + ); + } + if (q.category) query = query.where("r.category_raw", "=", q.category); + if (q.promotion) query = query.where(sql`r.promotion->>'kind'`, "=", q.promotion); + if (q.resolution) query = query.where("r.resolution_status", "=", q.resolution); + return (await query + .orderBy(sql`coalesce(r.source_published_at, r.first_seen_at)`, "desc") + .orderBy("r.id") + .offset(offset) + .limit(q.limit + 1) + .execute()) as unknown as ReleaseJoinRow[]; + }; + + let rows = await run(); + let live = false; + let liveError: string | null = null; + let coalesced = false; + const wantLive = q.live === "true" || (q.live !== "false" && offset === 0 && !!q.q && (rows.length === 0 || stale)); + if (wantLive && q.q) { + if (!site.enabled) liveError = "站点已关闭,只展示历史数据"; + else { + const r = await liveSearch(rt, site, q.q); + live = r.error === null; + liveError = r.error; + coalesced = r.coalesced; + if (live) rows = await run(); + } + } + const pg = page(rows, offset, q.limit); + return { + items: pg.items.map(toWorkbench), + nextCursor: pg.nextCursor, + coverage: { + siteEnabled: site.enabled, + lastSnapshotAt: snapAt?.toISOString() ?? null, + stale, + fromSnapshot: rows.length > 0, + live, + liveError, + coalesced, + }, + }; +} + +export async function releaseDetail( + rt: SourcesRuntime, + siteId: string, + releaseId: string, + refresh: boolean, +): Promise { + const find = async () => + (await releaseSelect(rt) + .where("r.site_id", "=", siteId) + .where("r.id", "=", releaseId) + .executeTakeFirst()) as unknown as ReleaseJoinRow | undefined; + if (!(await find())) throw new ApiException("not_found", "资源不存在"); + const refreshError = refresh ? await refreshReleaseDetail(rt, releaseId) : null; + const row = (await find())!; + const release = toWorkbench(row); + const obs = await rt.ctx.db + .selectFrom("observation as o") + .innerJoin("feed_source as f", "f.id", "o.feed_id") + .select(["o.id", "o.feed_id", "f.name", "o.first_seen_at", "o.last_seen_at"]) + .where("o.source_release_id", "=", releaseId) + .orderBy("o.first_seen_at") + .execute(); + const promo = (release.promotion ?? {}) as { observedAt?: string }; + return { + release, + observations: obs.map((o) => ({ + observationId: o.id, + feedId: o.feed_id, + feedName: o.name, + firstSeenAt: o.first_seen_at.toISOString(), + lastSeenAt: o.last_seen_at.toISOString(), + })), + volatile: { + promotionObservedAt: promo.observedAt ?? null, + promotionExpiresAt: release.promotionExpiresAt, + peersObservedAt: release.lastSeenAt, + detailFetchedAt: release.detailFetchedAt, + recheckBeforeDownload: true, + }, + refreshError, + }; +} diff --git a/apps/server/src/modules/sources-identity/tasks.ts b/apps/server/src/modules/sources-identity/tasks.ts new file mode 100644 index 00000000..c662c3a7 --- /dev/null +++ b/apps/server/src/modules/sources-identity/tasks.ts @@ -0,0 +1,92 @@ +import { defineTask, type OutboxRouteFn, type TaskRegistry } from "@umt/tasks"; +import { z } from "zod"; +import type { SourcesRuntime } from "./runtime.ts"; +import { ingestFeed } from "./service/feeds.ts"; +import { refreshMetadata } from "./service/metadata.ts"; +import { resolveRelease } from "./service/resolve.ts"; +import { snapshotSite } from "./service/snapshot.ts"; + +// 持久任务:输入只放业务 ID,处理器读业务表。外部请求失败的退避/熔断由适配器状态与业务表承担, +// 凭据失效返回正常结果(不重试);只有意外错误(DB、TMDB 网络)抛出交给引擎重试。 + +const NETWORK_RETRY = { + maximumAttempts: 5, + initialInterval: "1s", + backoffCoefficient: 2, + maximumInterval: "60s", +} as const; + +const id = z.string().uuid(); + +/** Feed 疑似缺口时的补漏页数预算 */ +export const BACKFILL_MAX_PAGES = 3; + +export function registerSourcesTasks(registry: TaskRegistry, rt: () => SourcesRuntime): void { + registry.register( + defineTask({ + name: "ingestFeed", + version: 1, + inputSchema: z.object({ feedSourceId: id, manual: z.boolean().optional() }), + retryPolicy: NETWORK_RETRY, + handler: (t) => t.step.run("ingest", () => ingestFeed(rt(), t.input.feedSourceId, { manual: !!t.input.manual })), + }), + defineTask({ + name: "resolveObservation", + version: 1, + inputSchema: z + .object({ observationId: id.nullish(), sourceReleaseId: id.nullish(), force: z.boolean().optional() }) + .refine((v) => !!v.observationId || !!v.sourceReleaseId, { message: "需要 observationId 或 sourceReleaseId" }), + retryPolicy: NETWORK_RETRY, + handler: (t) => + t.step.run("resolve", () => + resolveRelease(rt(), { + observationId: t.input.observationId ?? undefined, + sourceReleaseId: t.input.sourceReleaseId ?? undefined, + force: t.input.force, + }), + ), + }), + defineTask({ + name: "refreshMetadata", + version: 1, + inputSchema: z.object({ mediaEntityId: id }), + retryPolicy: NETWORK_RETRY, + handler: (t) => t.step.run("refresh", () => refreshMetadata(rt(), t.input.mediaEntityId)), + }), + defineTask({ + name: "snapshotSite", + version: 1, + inputSchema: z.object({ + siteId: id, + kind: z.enum(["periodic", "backfill", "manual"]).default("periodic"), + maxPages: z.number().int().min(1).max(20).optional(), + }), + retryPolicy: NETWORK_RETRY, + handler: (t) => + t.step.run("snapshot", () => + snapshotSite(rt(), { siteId: t.input.siteId, kind: t.input.kind, maxPages: t.input.maxPages }), + ), + }), + ); +} + +type P = Record; + +export const sourcesOutboxRoutes: Record = { + "feed.due": (ev) => { + const p = ev.payload as P; + return { task: "ingestFeed", input: { feedSourceId: p.feedId, manual: p.reason === "manual" } }; + }, + "release.observed": (ev) => { + const p = ev.payload as P; + return { + task: "resolveObservation", + input: { sourceReleaseId: p.sourceReleaseId, observationId: p.observationId }, + }; + }, + "feed.gap_suspected": (ev) => { + const p = ev.payload as P; + if (!p.siteId) return null; + return { task: "snapshotSite", input: { siteId: p.siteId, kind: "backfill", maxPages: BACKFILL_MAX_PAGES } }; + }, +}; diff --git a/apps/server/test/modules.test.ts b/apps/server/test/modules.test.ts index d1ddd0f5..c762893b 100644 --- a/apps/server/test/modules.test.ts +++ b/apps/server/test/modules.test.ts @@ -31,7 +31,7 @@ const pingModule: ServerModule = { registerTasks(registry) { registry.register( defineTask({ - name: "ingestFeed", + name: "testPing", version: 1, inputSchema: z.object({ feedSourceId: z.string() }), handler: async () => "real", @@ -75,8 +75,8 @@ describe("ServerModule 骨架", () => { it("任务注册表:模块任务优先,其余 §3.1-8 任务由空壳兜底;重复注册报错", async () => { const reg = buildTaskRegistry([...SERVER_MODULES, pingModule], fakeCtx); - expect(reg.names()).toEqual([...TASK_NAMES].sort()); - const ingest = reg.latest("ingestFeed"); + expect(reg.names()).toEqual(expect.arrayContaining([...TASK_NAMES, "testPing"])); + const ingest = reg.latest("testPing"); expect(await ingest.handler({ input: {} } as never)).toBe("real"); expect(() => buildTaskRegistry([pingModule, { ...pingModule, name: "dup" }], fakeCtx)).toThrow(/重复注册/); }); @@ -85,6 +85,7 @@ describe("ServerModule 骨架", () => { const merged = Object.keys(collectOutboxRoutes([...SERVER_MODULES, pingModule], fakeCtx)); expect(merged).toContain("test.ping"); expect(merged).toContain("release.resolved"); + expect(merged).toEqual(expect.arrayContaining(["feed.due", "release.observed", "feed.gap_suspected"])); expect(() => collectOutboxRoutes([pingModule, { ...pingModule, name: "dup" }], fakeCtx)).toThrow(/重复认领/); }); diff --git a/apps/server/test/sources-identity/api.test.ts b/apps/server/test/sources-identity/api.test.ts new file mode 100644 index 00000000..8e4693d8 --- /dev/null +++ b/apps/server/test/sources-identity/api.test.ts @@ -0,0 +1,238 @@ +import { createMockNexusPhpSite, type MockNexusPhpSite } from "@umt/testkit"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { snapshotSite } from "../../src/modules/sources-identity/service/snapshot.ts"; +import { liveSearch } from "../../src/modules/sources-identity/service/workbench.ts"; +import { createHarness, EXEC_KEY, type Harness, json, pgAvailable, READ_KEY } from "./harness.ts"; + +const up = await pgAvailable(); + +describe.skipIf(!up)("来源管理与站点工作台 API", () => { + let h: Harness; + let mock: MockNexusPhpSite; + let siteId: string; + + beforeAll(async () => { + mock = await createMockNexusPhpSite(); + h = await createHarness("api"); + }); + afterAll(async () => { + await h?.close(); + await mock?.close(); + }); + + it("鉴权:无 key 401,只读 key 写站点 403,organize.execute 不能改站点配置", async () => { + expect((await h.app.request("/api/v1/sites")).status).toBe(401); + expect((await h.req("/sites", { key: READ_KEY })).status).toBe(200); + const body = { key: "audiences", name: "观众", adapter: "nexusphp", baseUrl: `${mock.baseUrl}/` }; + expect((await h.req("/sites", { key: READ_KEY, body })).status).toBe(403); + expect((await h.req("/sites", { key: EXEC_KEY, body })).status).toBe(403); + }); + + it("站点注册表 + 登记站点;适配器类型与定义不符 400;重复 409;校验失败 422", async () => { + const defs = await json(await h.req("/site-definitions")); + expect(defs.status).toBe(200); + const aud = defs.body.data.items.find((d: { key: string }) => d.key === "audiences"); + expect(aud).toMatchObject({ adapter: "nexusphp", siteId: null, shippedEnabled: true }); + expect(defs.body.data.items.find((d: { key: string }) => d.key === "mteam")?.adapter).toBe("mteam_api"); + + const bad = await json( + await h.req("/sites", { + body: { key: "audiences", name: "观众", adapter: "mteam_api", baseUrl: `${mock.baseUrl}/` }, + }), + ); + expect(bad.status).toBe(400); + const unknown = await json( + await h.req("/sites", { body: { key: "nosuch", name: "x", adapter: "nexusphp", baseUrl: "https://x.example/" } }), + ); + expect(unknown.status).toBe(400); + expect( + (await h.req("/sites", { body: { key: "BAD KEY", name: "x", adapter: "nexusphp", baseUrl: "nope" } })).status, + ).toBe(422); + + const ok = await json( + await h.req("/sites", { + body: { + key: "audiences", + name: "观众", + adapter: "nexusphp", + baseUrl: `${mock.baseUrl}/`, + rateLimit: { minIntervalMs: 0, concurrency: 4 }, + snapshotEnabled: true, + snapshotIntervalSec: 3600, + }, + }), + ); + expect(ok.status).toBe(200); + siteId = ok.body.data.id; + expect( + ( + await h.req("/sites", { + body: { key: "audiences", name: "观众", adapter: "nexusphp", baseUrl: `${mock.baseUrl}/` }, + }) + ).status, + ).toBe(409); + const defs2 = await json(await h.req("/site-definitions")); + expect(defs2.body.data.items.find((d: { key: string }) => d.key === "audiences").siteId).toBe(siteId); + }); + + it("站点账号:凭证只写(存 secret),响应里只有 secret_ref", async () => { + const r = await json( + await h.req("/site-accounts", { body: { siteId, label: "主号", authKind: "cookie", credential: mock.cookie } }), + ); + expect(r.status).toBe(200); + expect(JSON.stringify(r.body)).not.toContain(mock.cookie); + expect(h.secrets.plain.get(r.body.data.secretRef)).toBe(mock.cookie); + expect( + (await h.req("/site-accounts", { body: { siteId, label: "主号", authKind: "cookie", credential: "x" } })).status, + ).toBe(409); + }); + + it("Feed CRUD:URL 存 secret,对外只给脱敏 URL;启停;立即抓取入队", async () => { + const url = "https://audiences.example/torrentrss.php?rows=50&passkey=abcdef0123456789"; + const c = await json( + await h.req("/feed-sources", { body: { siteId, name: "电影", purpose: "movie", url, pollIntervalSec: 600 } }), + ); + expect(c.status).toBe(200); + expect(c.body.data.urlDisplay).toContain("passkey=%5BREDACTED%5D".slice(0, 8)); + expect(JSON.stringify(c.body)).not.toContain("abcdef0123456789"); + const id = c.body.data.id; + expect((await h.req("/feed-sources", { body: { siteId, name: "x", url: "not a url" } })).status).toBe(422); + const off = await json(await h.req(`/feed-sources/${id}/disable`, { body: {} })); + expect(off.body.data).toMatchObject({ enabled: false, healthStatus: "disabled" }); + expect((await h.req(`/feed-sources/${id}/fetch`, { body: {} })).status).toBe(412); + await h.req(`/feed-sources/${id}/enable`, { body: {} }); + const f = await json(await h.req(`/feed-sources/${id}/fetch`, { body: {} })); + expect(f.status).toBe(200); + expect(h.engine.enqueued.at(-1)).toMatchObject({ task: "ingestFeed", input: { feedSourceId: id, manual: true } }); + const patched = await json( + await h.req(`/feed-sources/${id}`, { + method: "PATCH", + body: { url: "https://audiences.example/rss?passkey=NEWPASSKEY99" }, + }), + ); + expect(JSON.stringify(patched.body)).not.toContain("NEWPASSKEY99"); + expect(h.secrets.plain.get(patched.body.data.urlSecretRef)).toContain("NEWPASSKEY99"); + expect((await h.req("/feed-sources/00000000-0000-4000-8000-000000000000")).status).toBe(404); + }); + + it("快照:列表页 upsert 到 source_release,记录覆盖范围;站点级限流器每站一个实例", async () => { + const r = await snapshotSite(h.rt, { siteId, kind: "periodic" }); + expect(r.status).toBe("ok"); + expect(r.pages).toEqual([1]); + expect(r.newCount).toBe(r.itemCount); + const limiter = h.rt.limiterOf(siteId); + expect(limiter).not.toBeNull(); + await snapshotSite(h.rt, { siteId, kind: "manual" }); + expect(h.rt.limiterOf(siteId)).toBe(limiter); + expect(limiter!.stats.granted).toBeGreaterThanOrEqual(2); + const snaps = await json(await h.req(`/sites/${siteId}/snapshots`)); + expect(snaps.body.data.items[0]).toMatchObject({ kind: "manual", status: "ok", pages: [1] }); + const observed = await h.outbox("release.observed"); + expect(observed.length).toBe(r.newCount); + const site = await json(await h.req(`/sites/${siteId}`)); + expect(site.body.data.lastSnapshotAt).not.toBeNull(); + }); + + it("工作台列表:快照优先,标注来源与更新时间;筛选与分页", async () => { + const l = await json(await h.req(`/sites/${siteId}/releases?limit=2`)); + expect(l.status).toBe(200); + expect(l.body.data.items).toHaveLength(2); + expect(l.body.data.nextCursor).not.toBeNull(); + expect(l.body.data.coverage).toMatchObject({ siteEnabled: true, stale: false, live: false }); + expect(l.body.data.items[0]).toMatchObject({ origin: "snapshot", entity: null }); + expect(l.body.data.items[0].dataUpdatedAt).toBeTruthy(); + const p2 = await json(await h.req(`/sites/${siteId}/releases?limit=2&cursor=${l.body.data.nextCursor}`)); + expect(p2.body.data.items[0].id).not.toBe(l.body.data.items[0].id); + const free = await json(await h.req(`/sites/${siteId}/releases?promotion=free`)); + expect(free.body.data.items.every((i: { promotion: { kind: string } }) => i.promotion.kind === "free")).toBe(true); + const q = await json(await h.req(`/sites/${siteId}/releases?q=Dune&live=false`)); + expect(q.body.data.items.map((i: { siteReleaseId: string }) => i.siteReleaseId)).toContain("812345"); + }); + + it("实时补查:并发相同搜索合并为一次站点请求;API 标注实时来源", async () => { + const site = await h.db.selectFrom("site").selectAll().where("id", "=", siteId).executeTakeFirstOrThrow(); + const searches = () => mock.requests.filter((r) => r.path.includes("search=")).length; + const before = searches(); + const [a, b] = await Promise.all([liveSearch(h.rt, site, "Bear"), liveSearch(h.rt, site, "bear")]); + expect(searches() - before).toBe(1); + expect([a.coalesced, b.coalesced].sort()).toEqual([false, true]); + expect(a.error).toBeNull(); + const r = await json(await h.req(`/sites/${siteId}/releases?q=Last&live=true`)); + expect(r.body.data.coverage).toMatchObject({ live: true, liveError: null }); + expect(r.body.data.items.map((i: { siteReleaseId: string }) => i.siteReleaseId)).toContain("812348"); + const snaps = await h.db.selectFrom("site_snapshot").selectAll().where("kind", "=", "live_search").execute(); + expect(snaps).toHaveLength(2); + }); + + it("详情:易变字段复核时间;refresh 失败保留既有数据", async () => { + const rel = await h.db + .selectFrom("source_release") + .selectAll() + .where("site_release_id", "=", "812345") + .executeTakeFirstOrThrow(); + const d = await json(await h.req(`/sites/${siteId}/releases/${rel.id}?refresh=true`)); + expect(d.status).toBe(200); + expect(d.body.data.refreshError).toBeNull(); + expect(d.body.data.volatile.recheckBeforeDownload).toBe(true); + expect(d.body.data.volatile.detailFetchedAt).not.toBeNull(); + mock.setLoggedIn(false); + const d2 = await json(await h.req(`/sites/${siteId}/releases/${rel.id}?refresh=true`)); + expect(d2.status).toBe(200); + expect(d2.body.data.refreshError).toMatch(/auth/); + expect(d2.body.data.release.title).toBe(rel.title); + // 登录失效 → 账号标失效 + 待处理,周期快照不再访问 + const acc = await h.db + .selectFrom("site_account") + .selectAll() + .where("site_id", "=", siteId) + .executeTakeFirstOrThrow(); + expect(["invalid", "expired"]).toContain(acc.status); + expect( + (await h.db.selectFrom("decision").selectAll().where("dedupe_key", "=", `site-auth:${siteId}`).execute()).length, + ).toBe(1); + const n = mock.requests.length; + expect((await snapshotSite(h.rt, { siteId, kind: "periodic" })).reason).toBe("auth_invalid"); + expect(mock.requests.length).toBe(n); + // 换凭证:账号复位、待处理关闭 + mock.setLoggedIn(true); + await h.req(`/site-accounts/${acc.id}`, { method: "PATCH", body: { credential: mock.cookie } }); + const d3 = await h.db + .selectFrom("decision") + .select(["status"]) + .where("dedupe_key", "=", `site-auth:${siteId}`) + .executeTakeFirstOrThrow(); + expect(d3.status).toBe("resolved"); + expect((await snapshotSite(h.rt, { siteId, kind: "periodic" })).status).toBe("ok"); + }); + + it("关闭站点:停止网络访问,历史资源仍可查并标注 site_disabled", async () => { + const off = await json(await h.req(`/sites/${siteId}/disable`, { body: {} })); + expect(off.body.data).toMatchObject({ enabled: false, healthStatus: "disabled" }); + const n = mock.requests.length; + const l = await json(await h.req(`/sites/${siteId}/releases?q=Nothing.Here&live=true`)); + expect(l.body.data.coverage).toMatchObject({ siteEnabled: false, live: false }); + expect(l.body.data.coverage.liveError).toMatch(/关闭/); + expect(mock.requests.length).toBe(n); + const all = await json(await h.req(`/sites/${siteId}/releases`)); + expect(all.body.data.items.length).toBeGreaterThan(0); + expect(all.body.data.items.every((i: { availability: string }) => i.availability === "site_disabled")).toBe(true); + expect((await h.req(`/sites/${siteId}/snapshot`, { body: {} })).status).toBe(412); + await h.req(`/sites/${siteId}/enable`, { body: {} }); + const back = await json(await h.req(`/sites/${siteId}/releases?limit=1`)); + expect(back.body.data.items[0].availability).toBe("active"); + }); + + it("来源资源 / 观察 / 作品列表与 OpenAPI 文档", async () => { + const l = await json(await h.req(`/source-releases?siteId=${siteId}&q=Dune`)); + expect(l.body.data.items.length).toBeGreaterThan(0); + expect((await h.req("/source-releases/not-a-uuid")).status).toBe(422); + expect((await json(await h.req("/observations"))).status).toBe(200); + expect((await json(await h.req("/media-entities"))).status).toBe(200); + expect((await json(await h.req("/identity-pool"))).body.data.items).toEqual([]); + const doc = await json(await h.app.request("/api/openapi.json")); + const paths = Object.keys((doc.body as unknown as { paths: Record }).paths); + expect(paths).toEqual( + expect.arrayContaining(["/api/v1/sites/{id}/releases", "/api/v1/identity-pool", "/api/v1/decisions/summary"]), + ); + }); +}); diff --git a/apps/server/test/sources-identity/harness.ts b/apps/server/test/sources-identity/harness.ts new file mode 100644 index 00000000..1c54ca0f --- /dev/null +++ b/apps/server/test/sources-identity/harness.ts @@ -0,0 +1,278 @@ +import { randomUUID } from "node:crypto"; +import { createTmdbClient } from "@umt/adapters"; +import { DEFAULT_SETTINGS, type GlobalSettings, type SettingsKey } from "@umt/contracts"; +import { type DB, type Kysely, migrateToLatest, sql } from "@umt/db"; +import type { TaskEngine } from "@umt/tasks"; +import { Kysely as K, PostgresDialect } from "kysely"; +import pg from "pg"; +import { createOutboxPort } from "../../src/context.ts"; +import { createApp } from "../../src/http/app.ts"; +import { EventBus } from "../../src/http/sse.ts"; +import { createSourcesIdentityModule } from "../../src/modules/sources-identity/index.ts"; +import type { RuntimeOverrides } from "../../src/modules/sources-identity/runtime.ts"; +import type { SecretStore, ServerContext, SettingsReader } from "../../src/modules/types.ts"; +import { DEV_KEY, memoryLogger, memoryStore } from "../helpers.ts"; + +export const READ_KEY = "w2b-read-key"; +export const EXEC_KEY = "w2b-exec-key"; + +// W2-B 集成测试夹具:独立 PG schema + 真实迁移 + fake 引擎/凭证 + 模块运行时。 + +export const PG_URL = process.env.DATABASE_URL ?? "postgres://umt:umt@localhost:55433/umt"; + +export async function pgAvailable(): Promise { + const c = new pg.Client({ connectionString: PG_URL, connectionTimeoutMillis: 2000 }); + try { + await c.connect(); + await c.end(); + return true; + } catch { + return false; + } +} + +/** 内存凭证库(W2-A 的 SecretStore 实现前);记录明文以便断言"明文不入业务表" */ +export function fakeSecrets(db: () => Kysely): SecretStore & { plain: Map } { + const plain = new Map(); + return { + plain, + async resolve(ref) { + const v = plain.get(ref); + if (v === undefined) throw new Error(`unknown secret ${ref}`); + return v; + }, + async create({ plaintext, purpose }) { + const id = randomUUID(); + plain.set(id, plaintext); + // 业务表 secret_ref 有外键:落一行占位(密文不是真的) + await db() + .insertInto("secret") + .values({ id, name: `test-${id}`, purpose, ciphertext: Buffer.from("x"), nonce: Buffer.from("x") }) + .execute(); + return id; + }, + async replace(ref, plaintext) { + plain.set(ref, plaintext); + }, + async remove(ref) { + plain.delete(ref); + }, + }; +} + +/** 每次读库的设置读取(不缓存,测试里改 settings 表立即生效) */ +export function plainSettings(db: () => Kysely): SettingsReader { + const all = async (): Promise => { + const rows = await db().selectFrom("settings").select(["key", "value"]).execute(); + const out = { ...DEFAULT_SETTINGS } as Record; + for (const r of rows) { + const def = (DEFAULT_SETTINGS as Record)[r.key]; + if (def) out[r.key] = { ...def, ...(r.value as object) }; + } + return out as GlobalSettings; + }; + return { all, get: async (k: K) => (await all())[k] } as SettingsReader; +} + +export interface Enqueued { + task: string; + input: unknown; + key: string; +} + +export function fakeEngine(): TaskEngine & { enqueued: Enqueued[] } { + const enqueued: Enqueued[] = []; + const keys = new Set(); + return { + enqueued, + async enqueue(task, input, opts) { + const dedup = keys.has(opts.idempotencyKey); + if (!dedup) { + keys.add(opts.idempotencyKey); + enqueued.push({ task, input, key: opts.idempotencyKey }); + } + return { runId: randomUUID(), deduplicated: dedup }; + }, + async signal() { + return { delivered: 0 }; + }, + async cancel() { + return false; + }, + async get() { + return null; + }, + async list() { + return []; + }, + async startWorker() {}, + async stop() {}, + }; +} + +export interface Harness { + db: Kysely; + ctx: ServerContext; + engine: ReturnType; + secrets: ReturnType; + module: ReturnType; + rt: ReturnType["runtime"]>; + app: ReturnType; + /** 带 dev admin key 的请求 */ + req(path: string, init?: { method?: string; body?: unknown; key?: string }): Promise; + outbox(type?: string): Promise<{ type: string; payload: Record; idempotency_key: string | null }[]>; + close(): Promise; +} + +export interface HarnessOptions extends RuntimeOverrides { + tmdbUrl?: string; +} + +export async function createHarness(name: string, opts: HarnessOptions = {}): Promise { + const schema = `w2b_${name}_${process.pid.toString(36)}${Math.random().toString(36).slice(2, 7)}`; + const admin = new K({ + dialect: new PostgresDialect({ pool: new pg.Pool({ connectionString: PG_URL, max: 1 }) }), + }); + await sql.raw(`create schema ${schema}`).execute(admin); + // 与 @umt/db createDb 相同的 int8 解析(createDb 导入即注册全局解析器) + await import("@umt/db"); + const db = new K({ + dialect: new PostgresDialect({ + pool: new pg.Pool({ connectionString: PG_URL, max: 8, options: `-c search_path=${schema},public` }), + }), + }) as unknown as Kysely; + await migrateToLatest(db, { migrationTableSchema: schema }); + + const { logger } = memoryLogger(); + const engine = fakeEngine(); + const secrets = fakeSecrets(() => db); + const outboxPort = createOutboxPort(); + const ctx = { + role: "all", + db, + config: { NODE_ENV: "development", DEV_ADMIN_KEY: DEV_KEY } as never, + logger, + events: new EventBus(5), + engine, + outbox: outboxPort, + secrets, + settings: plainSettings(() => db), + capabilities: {} as never, + } as ServerContext; + const { tmdbUrl, ...overrides } = opts; + const module = createSourcesIdentityModule({ + ...overrides, + tmdb: + overrides.tmdb ?? + (() => (tmdbUrl ? createTmdbClient({ apiKey: "test-tmdb-key", baseUrl: `${tmdbUrl}/3`, retries: 0 }) : null)), + }); + const rt = module.runtime(ctx); + const app = createApp({ + config: { NODE_ENV: "development", DEV_ADMIN_KEY: DEV_KEY }, + logger, + bus: new EventBus(5), + role: "all", + version: "0.0.0-test", + modules: [module], + ctx, + apiKeyStore: memoryStore({ + [READ_KEY]: { id: "reader", scopes: ["read"] }, + [EXEC_KEY]: { id: "operator", scopes: ["organize.execute"] }, + }), + }); + return { + db, + ctx, + engine, + secrets, + module, + rt, + app, + req(path, init = {}) { + const headers: Record = { authorization: `Bearer ${init.key ?? DEV_KEY}` }; + if (init.body !== undefined) headers["content-type"] = "application/json"; + return Promise.resolve( + app.request(`/api/v1${path}`, { + method: init.method ?? (init.body !== undefined ? "POST" : "GET"), + headers, + ...(init.body !== undefined ? { body: JSON.stringify(init.body) } : {}), + }), + ); + }, + async outbox(type) { + let q = db.selectFrom("outbox_event").select(["type", "payload", "idempotency_key"]).orderBy("at").orderBy("id"); + if (type) q = q.where("type", "=", type); + return (await q.execute()) as never; + }, + async close() { + await db.destroy(); + await sql.raw(`drop schema if exists ${schema} cascade`).execute(admin); + await admin.destroy(); + }, + }; +} + +/** 登记一个指向 mock 的站点 + 可选账号 */ +export async function seedSite( + h: Harness, + opts: { key?: string; baseUrl: string; cookie?: string; adapter?: "nexusphp" | "mteam_api" | "generic_rss" }, +) { + const site = await h.db + .insertInto("site") + .values({ + key: opts.key ?? "audiences", + name: opts.key ?? "观众", + adapter: opts.adapter ?? "nexusphp", + base_url: opts.baseUrl, + rate_limit: JSON.stringify({ minIntervalMs: 0, concurrency: 5 }), + snapshot_enabled: true, + snapshot_interval_sec: 3600, + }) + .returningAll() + .executeTakeFirstOrThrow(); + let accountId: string | null = null; + if (opts.cookie) { + const ref = await h.secrets.create({ purpose: "site_cookie", plaintext: opts.cookie }); + const acc = await h.db + .insertInto("site_account") + .values({ site_id: site.id, label: "main", auth_kind: "cookie", secret_ref: ref }) + .returning(["id"]) + .executeTakeFirstOrThrow(); + accountId = acc.id; + } + return { site, accountId }; +} + +/** 登记 Feed(URL 存 fake secret;feed_source.url_secret_ref 外键指向 secret 表,这里补一行占位) */ +export async function seedFeed( + h: Harness, + opts: { + siteId: string | null; + url: string; + name?: string; + accountId?: string | null; + purpose?: "movie" | "promotion" | "generic"; + }, +) { + const ref = await h.secrets.create({ purpose: "feed_url", plaintext: opts.url }); + return h.db + .insertInto("feed_source") + .values({ + site_id: opts.siteId, + site_account_id: opts.accountId ?? null, + name: opts.name ?? "feed", + purpose: opts.purpose ?? "generic", + url_secret_ref: ref, + url_display: opts.url.replace(/passkey=[^&]+/, "passkey=[REDACTED]"), + poll_interval_sec: 300, + }) + .returningAll() + .executeTakeFirstOrThrow(); +} + +// biome-ignore lint/suspicious/noExplicitAny: 测试断言用 +type AnyJson = any; +export const json = async (r: Response) => ({ + status: r.status, + body: (await r.json()) as { ok: boolean; data: AnyJson; error?: AnyJson }, +}); diff --git a/apps/server/test/sources-identity/ingest.test.ts b/apps/server/test/sources-identity/ingest.test.ts new file mode 100644 index 00000000..8baf0fb4 --- /dev/null +++ b/apps/server/test/sources-identity/ingest.test.ts @@ -0,0 +1,213 @@ +import { createMockRssServer, type MockFeedItem, type MockRssServer } from "@umt/testkit"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { schedulerTick } from "../../src/modules/sources-identity/scheduler.ts"; +import { extractSiteReleaseId, ingestFeed } from "../../src/modules/sources-identity/service/feeds.ts"; +import { sourcesOutboxRoutes } from "../../src/modules/sources-identity/tasks.ts"; +import { createHarness, type Harness, pgAvailable, seedFeed, seedSite } from "./harness.ts"; + +const up = await pgAvailable(); +const BASE = "https://audiences.example"; + +const item = (guid: string, id: string, title: string, extra: Partial = {}): MockFeedItem => ({ + guid, + title, + link: `${BASE}/details.php?id=${id}&hit=1`, + enclosureUrl: `${BASE}/download.php?id=${id}&passkey=SECRETPK123`, + size: 1024 * 1024 * 1024, + pubDate: new Date("2026-09-20T08:00:00Z"), + categories: ["电影"], + ...extra, +}); + +describe.skipIf(!up)("ingestFeed", () => { + let h: Harness; + const servers: MockRssServer[] = []; + let siteId: string; + let accountId: string | null; + + beforeAll(async () => { + h = await createHarness("ingest"); + const s = await seedSite(h, { baseUrl: `${BASE}/`, cookie: "c=1" }); + siteId = s.site.id; + accountId = s.accountId; + }); + afterAll(async () => { + for (const s of servers) await s.close(); + await h?.close(); + }); + + const feedServer = async (items: MockFeedItem[], passkey = "FEEDPK") => { + const s = await createMockRssServer({ items, passkey }); + servers.push(s); + return s; + }; + + it("从链接推出站点资源 ID", () => { + expect(extractSiteReleaseId({ guid: "x", link: `${BASE}/details.php?id=812345&hit=1` })).toBe("812345"); + expect(extractSiteReleaseId({ guid: "x", link: "https://kp.m-team.cc/detail/912233" })).toBe("912233"); + expect(extractSiteReleaseId({ guid: "abc", link: null })).toBeNull(); + expect(extractSiteReleaseId({ guid: "12345", link: null })).toBe("12345"); + }); + + it("三个 Feed 看到同一资源只建一条 source_release,保留三条观察;release.observed 只发一次", async () => { + const title = "Dune.Part.Two.2024.2160p.UHD.BluRay.REMUX-FraMeSToR"; + const a = await feedServer([item("movie-1", "812345", title)]); + const b = await feedServer([item("promo-xyz", "812345", title)]); + const c = await feedServer([item("https://audiences.example/details.php?id=812345", "812345", title)]); + const feeds = [ + await seedFeed(h, { siteId, url: a.feedUrl, name: "电影", purpose: "movie", accountId }), + await seedFeed(h, { siteId, url: b.feedUrl, name: "促销", purpose: "promotion" }), + await seedFeed(h, { siteId, url: c.feedUrl, name: "综合" }), + ]; + for (const f of feeds) expect((await ingestFeed(h.rt, f.id)).status).toBe("ok"); + + const rels = await h.db.selectFrom("source_release").selectAll().where("site_release_id", "=", "812345").execute(); + expect(rels).toHaveLength(1); + const obs = await h.db.selectFrom("observation").selectAll().where("source_release_id", "=", rels[0]!.id).execute(); + expect(new Set(obs.map((o) => o.feed_id)).size).toBe(3); + expect(obs.every((o) => o.status === "linked")).toBe(true); + const observed = (await h.outbox("release.observed")).filter((e) => e.payload.sourceReleaseId === rels[0]!.id); + expect(observed).toHaveLength(1); + expect(observed[0]!.payload.isNew).toBe(true); + // 时间:来源发布时间与首次发现分开保存 + expect(rels[0]!.source_published_at?.toISOString()).toBe("2026-09-20T08:00:00.000Z"); + expect(rels[0]!.first_seen_at.getTime()).toBeGreaterThan(rels[0]!.source_published_at!.getTime()); + // feed 状态持久化:游标与条件请求头 + const f0 = await h.db + .selectFrom("feed_source") + .selectAll() + .where("id", "=", feeds[0]!.id) + .executeTakeFirstOrThrow(); + expect(f0.health_status).toBe("healthy"); + expect(f0.etag).toBeTruthy(); + expect((f0.cursor as { window: { lastGuidSet: string[] } }).window.lastGuidSet).toContain("movie-1"); + expect(f0.next_fetch_at!.getTime()).toBeGreaterThan(Date.now()); + }); + + it("enclosure/passkey 只存脱敏值;未变化时 304 不产生新条目", async () => { + const s = await feedServer([item("g-1", "900001", "Some.Movie.2020.1080p.WEB-DL-GRP")], "TOPSECRETKEY"); + const f = await seedFeed(h, { siteId, url: s.feedUrl, name: "脱敏" }); + await ingestFeed(h.rt, f.id); + const dump = + JSON.stringify(await h.db.selectFrom("observation").selectAll().where("feed_id", "=", f.id).execute()) + + JSON.stringify( + await h.db.selectFrom("source_release").selectAll().where("site_release_id", "=", "900001").execute(), + ) + + JSON.stringify(await h.db.selectFrom("feed_source").selectAll().where("id", "=", f.id).execute()); + expect(dump).not.toContain("SECRETPK123"); + expect(dump).not.toContain("TOPSECRETKEY"); + expect(dump).toContain("[REDACTED]"); + const again = await ingestFeed(h.rt, f.id); + expect(again.status).toBe("not_modified"); + expect(s.notModifiedCount).toBe(1); + }); + + it("同 GUID 内容变化视为更新:观察与资源标题更新,重新触发识别", async () => { + const s = await feedServer([item("upd-1", "900100", "Show.2021.1080p.WEB-DL-AAA")]); + const f = await seedFeed(h, { siteId, url: s.feedUrl, name: "更新" }); + await ingestFeed(h.rt, f.id); + s.updateItem("upd-1", { title: "Show.2021.2160p.WEB-DL-AAA" }); + const r = await ingestFeed(h.rt, f.id); + expect(r).toMatchObject({ status: "ok", newItems: 0, updatedItems: 1, newReleases: 0 }); + const obs = await h.db.selectFrom("observation").selectAll().where("feed_id", "=", f.id).execute(); + expect(obs).toHaveLength(1); + expect(obs[0]!.title).toBe("Show.2021.2160p.WEB-DL-AAA"); + const rel = await h.db + .selectFrom("source_release") + .selectAll() + .where("site_release_id", "=", "900100") + .executeTakeFirstOrThrow(); + expect(rel.title).toBe("Show.2021.2160p.WEB-DL-AAA"); + const ev = (await h.outbox("release.observed")).filter((e) => e.payload.sourceReleaseId === rel.id); + expect(ev.map((e) => e.payload.isNew)).toEqual([true, false]); + }); + + it("窗口零重叠 → feed.gap_suspected,并路由到有预算的补漏快照", async () => { + const s = await feedServer([item("gap-1", "900201", "A.2020.1080p-X"), item("gap-2", "900202", "B.2020.1080p-X")]); + const f = await seedFeed(h, { siteId, url: s.feedUrl, name: "缺口" }); + await ingestFeed(h.rt, f.id); + s.setItems([item("gap-9", "900209", "C.2020.1080p-X")]); + const r = await ingestFeed(h.rt, f.id); + expect(r.possibleGap).toBe(true); + const gap = (await h.outbox("feed.gap_suspected")).find((e) => e.payload.feedId === f.id); + expect(gap).toBeDefined(); + const route = sourcesOutboxRoutes["feed.gap_suspected"]!({ + id: "1", + type: "feed.gap_suspected", + payload: gap!.payload, + at: new Date(), + }); + expect(route).toMatchObject({ task: "snapshotSite", input: { siteId, kind: "backfill", maxPages: 3 } }); + }); + + it("登录失效(403):账号标失效 + 待处理,不重试", async () => { + const s = await feedServer([item("auth-1", "900300", "X.2020.1080p-Y")], "RIGHTKEY"); + const badUrl = s.feedUrl.replace("RIGHTKEY", "WRONGKEY"); + const f = await seedFeed(h, { siteId, url: badUrl, name: "失效", accountId }); + const r = await ingestFeed(h.rt, f.id); + expect(r.status).toBe("auth_failed"); + const acc = await h.db + .selectFrom("site_account") + .selectAll() + .where("id", "=", accountId!) + .executeTakeFirstOrThrow(); + expect(acc.status).toBe("invalid"); + const d = await h.db + .selectFrom("decision") + .selectAll() + .where("dedupe_key", "=", `feed-auth:${f.id}`) + .executeTakeFirstOrThrow(); + expect(d.kind).toBe("source_unreachable"); + expect(JSON.stringify(d)).not.toContain("WRONGKEY"); + const row = await h.db.selectFrom("feed_source").selectAll().where("id", "=", f.id).executeTakeFirstOrThrow(); + expect(row.health_status).toBe("failing"); + expect(row.backoff_until!.getTime()).toBeGreaterThan(Date.now() + 23 * 3600_000); + // 再次调度:退避期内不发请求 + const before = s.requests.length; + const again = await ingestFeed(h.rt, f.id); + expect(again.status).toBe("skipped"); + expect(s.requests.length).toBe(before); + // 调度器也不会把它当作到期 + await h.db + .updateTable("feed_source") + .set({ next_fetch_at: new Date(0) }) + .where("id", "=", f.id) + .execute(); + await schedulerTick(h.rt); + const due = (await h.outbox("feed.due")).filter((e) => e.payload.feedId === f.id); + expect(due).toHaveLength(0); + // 恢复账号给后续用例 + await h.db.updateTable("site_account").set({ status: "active" }).where("id", "=", accountId!).execute(); + }); + + it("调度器:到期 Feed 发 feed.due(幂等),快照到期入队 snapshotSite", async () => { + const s = await feedServer([item("sch-1", "900400", "S.2020.1080p-Z")]); + const f = await seedFeed(h, { siteId, url: s.feedUrl, name: "调度" }); + const t1 = await schedulerTick(h.rt); + expect(t1.feedsDue).toBeGreaterThanOrEqual(1); + const due = (await h.outbox("feed.due")).filter((e) => e.payload.feedId === f.id); + expect(due).toHaveLength(1); + expect( + sourcesOutboxRoutes["feed.due"]!({ id: "1", type: "feed.due", payload: due[0]!.payload, at: new Date() }), + ).toEqual({ + task: "ingestFeed", + input: { feedSourceId: f.id, manual: false }, + }); + // next_fetch_at 已顺延:紧接着的 tick 不再重复 + await schedulerTick(h.rt); + expect((await h.outbox("feed.due")).filter((e) => e.payload.feedId === f.id)).toHaveLength(1); + expect( + h.engine.enqueued.some((e) => e.task === "snapshotSite" && (e.input as { siteId: string }).siteId === siteId), + ).toBe(true); + }); + + it("站点关闭后 Feed 不再抓取", async () => { + const s = await feedServer([item("off-1", "900500", "O.2020.1080p-Z")]); + const f = await seedFeed(h, { siteId, url: s.feedUrl, name: "关站" }); + await h.db.updateTable("site").set({ enabled: false }).where("id", "=", siteId).execute(); + const r = await ingestFeed(h.rt, f.id); + expect(r).toMatchObject({ status: "skipped", reason: "site_disabled" }); + expect(s.requests.length).toBe(0); + await h.db.updateTable("site").set({ enabled: true }).where("id", "=", siteId).execute(); + }); +}); diff --git a/apps/server/test/sources-identity/mteam-tasks.test.ts b/apps/server/test/sources-identity/mteam-tasks.test.ts new file mode 100644 index 00000000..5ffc3428 --- /dev/null +++ b/apps/server/test/sources-identity/mteam-tasks.test.ts @@ -0,0 +1,97 @@ +import { TaskRegistry } from "@umt/tasks"; +import { createMockMteamApi, type MockMteamApi } from "@umt/testkit"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { snapshotSite } from "../../src/modules/sources-identity/service/snapshot.ts"; +import { createHarness, type Harness, pgAvailable } from "./harness.ts"; + +const up = await pgAvailable(); + +describe.skipIf(!up)("M-Team 快照与任务注册", () => { + let h: Harness; + let api: MockMteamApi; + let siteId: string; + + beforeAll(async () => { + api = await createMockMteamApi(); + h = await createHarness("mteam"); + const site = await h.db + .insertInto("site") + .values({ + key: "mteam", + name: "馒头", + adapter: "mteam_api", + base_url: "https://kp.m-team.cc/", + rate_limit: JSON.stringify({ minIntervalMs: 0 }), + config: JSON.stringify({ apiBaseUrl: api.baseUrl }), + }) + .returningAll() + .executeTakeFirstOrThrow(); + siteId = site.id; + const ref = await h.secrets.create({ purpose: "site_api_key", plaintext: api.apiKey }); + await h.db + .insertInto("site_account") + .values({ site_id: siteId, label: "api", auth_kind: "api_key", secret_ref: ref }) + .execute(); + }); + afterAll(async () => { + await h?.close(); + await api?.close(); + }); + + it("API Key 站点快照:原始分类保留,促销带观察/过期时间;key 不入库", async () => { + const r = await snapshotSite(h.rt, { siteId, kind: "periodic" }); + expect(r.status).toBe("ok"); + expect(r.itemCount).toBeGreaterThan(0); + const rows = await h.db.selectFrom("source_release").selectAll().where("site_id", "=", siteId).execute(); + const sports = rows.find((x) => x.site_release_id === "912237"); + expect(sports?.category_raw).toBe("407"); + const promo = rows.find((x) => x.site_release_id === "912233")!; + expect((promo.promotion as { observedAt?: string }).observedAt).toBeTruthy(); + expect(JSON.stringify(rows)).not.toContain(api.apiKey); + }); + + it("key 失效:账号标失效 + 待处理,快照结果 failed", async () => { + api.setFailMode("banned_key"); + const r = await snapshotSite(h.rt, { siteId, kind: "manual" }); + expect(r.status).toBe("failed"); + const acc = await h.db + .selectFrom("site_account") + .select(["status"]) + .where("site_id", "=", siteId) + .executeTakeFirstOrThrow(); + expect(["invalid", "expired"]).toContain(acc.status); + const d = await h.db + .selectFrom("decision") + .selectAll() + .where("dedupe_key", "=", `site-auth:${siteId}`) + .executeTakeFirstOrThrow(); + expect(d.status).toBe("open"); + api.setFailMode(null); + }); + + it("模块注册 4 个任务与 3 条 outbox 路由;任务处理器经 step.run 调服务", async () => { + const reg = new TaskRegistry(); + h.module.registerTasks(reg, h.ctx); + expect(reg.names()).toEqual(["ingestFeed", "refreshMetadata", "resolveObservation", "snapshotSite"]); + expect(Object.keys(h.module.outboxRoutes!(h.ctx)).sort()).toEqual([ + "feed.due", + "feed.gap_suspected", + "release.observed", + ]); + const def = reg.latest("resolveObservation"); + expect(def.inputSchema.safeParse({}).success).toBe(false); + expect(def.inputSchema.safeParse({ observationId: "00000000-0000-4000-8000-000000000000" }).success).toBe(true); + const steps: string[] = []; + const out = await reg.latest("snapshotSite").handler({ + input: { siteId, kind: "periodic" }, + step: { + run: async (name: string, fn: () => unknown) => { + steps.push(name); + return fn(); + }, + }, + } as never); + expect(steps).toEqual(["snapshot"]); + expect(out).toMatchObject({ status: "skipped", reason: "auth_invalid" }); + }); +}); diff --git a/apps/server/test/sources-identity/resolve.test.ts b/apps/server/test/sources-identity/resolve.test.ts new file mode 100644 index 00000000..711c155c --- /dev/null +++ b/apps/server/test/sources-identity/resolve.test.ts @@ -0,0 +1,403 @@ +import { CapabilityRegistry } from "@umt/agent"; +import { createMockTmdb, type MockTmdb, TMDB_FIXTURES, type TmdbFixture } from "@umt/testkit"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { refreshMetadata } from "../../src/modules/sources-identity/service/metadata.ts"; +import { resolveRelease } from "../../src/modules/sources-identity/service/resolve.ts"; +import { createHarness, type Harness, json, pgAvailable, seedSite } from "./harness.ts"; + +const up = await pgAvailable(); + +const twin = (id: number): TmdbFixture => ({ + type: "movie", + id, + original_title: "Twins", + original_language: "en", + origin_country: ["US"], + date: "1988-12-09", + titles: { "zh-CN": "龙兄鼠弟", "en-US": "Twins" }, + popularity: 10 + id / 1000, + alternative_titles: [], + external_ids: {}, +}); + +describe.skipIf(!up)("resolveObservation / 身份与元数据", () => { + let h: Harness; + let tmdb: MockTmdb; + let siteId: string; + let n = 0; + + beforeAll(async () => { + tmdb = await createMockTmdb({ fixtures: [...TMDB_FIXTURES, twin(111), twin(112)] }); + h = await createHarness("resolve", { tmdbUrl: tmdb.url }); + siteId = (await seedSite(h, { baseUrl: "https://audiences.example/" })).site.id; + }); + afterAll(async () => { + await h?.close(); + await tmdb?.close(); + }); + + const release = async (title: string, extra: { subtitle?: string; category?: string } = {}) => + h.db + .insertInto("source_release") + .values({ + site_id: siteId, + site_release_id: String(700000 + ++n), + title, + subtitle: extra.subtitle ?? null, + category_raw: extra.category ?? null, + }) + .returningAll() + .executeTakeFirstOrThrow(); + + let oppId: string; + + it("auto:按 TMDB 建本地作品 + accepted 外部身份 + 别名,发 release.resolved", async () => { + const r = await release("Oppenheimer.2023.2160p.UHD.BluRay.REMUX.HDR.HEVC-FGT"); + const res = await resolveRelease(h.rt, { sourceReleaseId: r.id }); + expect(res.outcome).toBe("auto"); + oppId = res.mediaEntityId!; + const e = await h.db.selectFrom("media_entity").selectAll().where("id", "=", oppId).executeTakeFirstOrThrow(); + expect(e).toMatchObject({ kind: "movie", year: 2023, status: "active" }); + const ids = await h.db.selectFrom("external_identity").selectAll().where("media_entity_id", "=", oppId).execute(); + expect(ids.map((i) => `${i.provider}:${i.entity_type}:${i.external_id}:${i.status}`).sort()).toEqual([ + "imdb:movie:tt15398776:accepted", + "tmdb:movie:872585:accepted", + ]); + const revs = await h.db.selectFrom("identity_revision").selectAll().execute(); + expect(revs.length).toBeGreaterThanOrEqual(2); + const aliases = await h.db + .selectFrom("title_alias") + .select(["alias"]) + .where("media_entity_id", "=", oppId) + .execute(); + expect(aliases.map((a) => a.alias)).toEqual(expect.arrayContaining(["奥本海默", "Oppenheimer"])); + const rel = await h.db.selectFrom("source_release").selectAll().where("id", "=", r.id).executeTakeFirstOrThrow(); + expect(rel).toMatchObject({ resolution_status: "resolved", media_entity_id: oppId, parser_version: "1" }); + expect((rel.parsed as { year: number }).year).toBe(2023); + const ev = (await h.outbox("release.resolved")).find((x) => x.payload.sourceReleaseId === r.id); + expect(ev?.payload).toMatchObject({ mediaEntityId: oppId, evidenceLevel: "rule_confirmed", contentUnitIds: [] }); + // 幂等:证据未变的重复识别直接短路 + expect((await resolveRelease(h.rt, { sourceReleaseId: r.id })).outcome).toBe("skipped"); + }); + + it("同一作品的其它资源经本地召回复用同一实体,不重复建档", async () => { + const r = await release("奥本海默.Oppenheimer.2023.1080p.WEB-DL.H264-GRP"); + const res = await resolveRelease(h.rt, { sourceReleaseId: r.id }); + expect(res).toMatchObject({ outcome: "auto", mediaEntityId: oppId }); + const count = await h.db + .selectFrom("media_entity") + .select((eb) => eb.fn.countAll().as("c")) + .executeTakeFirstOrThrow(); + expect(Number(count.c)).toBe(1); + }); + + it("剧集:建 series 作品与临时季/集单元", async () => { + const r = await release("The.Bear.S03E02.2160p.DSNP.WEB-DL.DDP5.1.H.265-FLUX", { subtitle: "熊家餐馆 第三季" }); + const res = await resolveRelease(h.rt, { sourceReleaseId: r.id }); + expect(res.outcome).toBe("auto"); + const units = await h.db + .selectFrom("content_unit") + .selectAll() + .where("media_entity_id", "=", res.mediaEntityId!) + .orderBy("kind") + .execute(); + expect(units.map((u) => `${u.kind}:${u.season_number}:${u.episode_number}:${u.status}`)).toEqual([ + "episode:3:2:provisional", + "season:3:null:provisional", + ]); + expect(res.contentUnitIds).toEqual([units[0]!.id]); + const e = await h.db + .selectFrom("media_entity") + .select(["kind"]) + .where("id", "=", res.mediaEntityId!) + .executeTakeFirstOrThrow(); + expect(e.kind).toBe("series"); + }); + + let twinsRelease: string; + it("ambiguous:进待识别池 + 待处理事项,不建正式作品", async () => { + const before = await h.db.selectFrom("media_entity").select("id").execute(); + const r = await release("Twins.1988.1080p.BluRay.x264-GRP"); + twinsRelease = r.id; + const res = await resolveRelease(h.rt, { sourceReleaseId: r.id }); + expect(res.outcome).toBe("ambiguous"); + expect((await h.db.selectFrom("media_entity").select("id").execute()).length).toBe(before.length); + const pool = await h.db + .selectFrom("identity_candidate") + .selectAll() + .where("source_release_id", "=", r.id) + .executeTakeFirstOrThrow(); + expect(pool).toMatchObject({ status: "pending", reason: "ambiguous", attempts: 1 }); + expect((pool.candidates as unknown[]).length).toBe(2); + expect(pool.next_review_at!.getTime()).toBeGreaterThan(Date.now()); + const d = await h.db.selectFrom("decision").selectAll().where("id", "=", res.decisionId!).executeTakeFirstOrThrow(); + expect(d).toMatchObject({ kind: "identity_ambiguous", status: "open", subject_id: r.id }); + expect((d.options as { key: string }[]).map((o) => o.key)).toEqual(["c0", "c1", "dismiss"]); + expect((await h.outbox("decision.created")).some((e) => e.payload.decisionId === d.id)).toBe(true); + const rel = await h.db.selectFrom("source_release").selectAll().where("id", "=", r.id).executeTakeFirstOrThrow(); + expect(rel.resolution_status).toBe("ambiguous"); + expect(rel.media_entity_id).toBeNull(); + const attempts = await h.db + .selectFrom("resolution_attempt") + .selectAll() + .where("source_release_id", "=", r.id) + .execute(); + expect(attempts.map((a) => a.outcome)).toEqual(["ambiguous"]); + // 同一证据再次复查:计数递增(退避加长),不重复开待处理 + await resolveRelease(h.rt, { sourceReleaseId: r.id, force: true }); + const pool2 = await h.db + .selectFrom("identity_candidate") + .selectAll() + .where("source_release_id", "=", r.id) + .executeTakeFirstOrThrow(); + expect(pool2.attempts).toBe(2); + expect(pool2.next_review_at!.getTime()).toBeGreaterThan(pool.next_review_at!.getTime()); + expect( + await h.db.selectFrom("decision").select("id").where("dedupe_key", "=", `identity:${r.id}`).execute(), + ).toHaveLength(1); + }); + + it("待识别池可按原始标题检索;经待处理中心选候选 = 人工指定并加锁", async () => { + const list = await json(await h.req("/identity-pool?q=Twins")); + expect(list.status).toBe(200); + expect(list.body.data.items.map((i: { sourceReleaseId: string }) => i.sourceReleaseId)).toContain(twinsRelease); + const pool = list.body.data.items.find((i: { sourceReleaseId: string }) => i.sourceReleaseId === twinsRelease); + const res = await json(await h.req(`/decisions/${pool.decisionId}/resolve`, { body: { optionKey: "c1" } })); + expect(res.status).toBe(200); + expect(res.body.data.status).toBe("resolved"); + const rel = await h.db + .selectFrom("source_release") + .selectAll() + .where("id", "=", twinsRelease) + .executeTakeFirstOrThrow(); + expect(rel.resolution_locked).toBe(true); + const chosen = pool.candidates[1]; + const ident = await h.db + .selectFrom("external_identity") + .selectAll() + .where("media_entity_id", "=", rel.media_entity_id!) + .where("provider", "=", "tmdb") + .executeTakeFirstOrThrow(); + expect(ident).toMatchObject({ + external_id: chosen.externalId, + evidence_level: "manual", + locked: true, + status: "accepted", + }); + const p = await h.db + .selectFrom("identity_candidate") + .selectAll() + .where("source_release_id", "=", twinsRelease) + .executeTakeFirstOrThrow(); + expect(p.status).toBe("resolved"); + const ev = (await h.outbox("release.resolved")).find((x) => x.payload.sourceReleaseId === twinsRelease); + expect(ev?.payload.evidenceLevel).toBe("manual"); + // 人工锁:自动识别不再改写 + const again = await resolveRelease(h.rt, { sourceReleaseId: twinsRelease, force: true }); + expect(again.outcome).toBe("skipped_locked"); + const after = await h.db + .selectFrom("source_release") + .select(["media_entity_id"]) + .where("id", "=", twinsRelease) + .executeTakeFirstOrThrow(); + expect(after.media_entity_id).toBe(rel.media_entity_id); + }); + + it("no_match 只进池不开待处理;TMDB 失败记为 lookup_failed 而不是无结果", async () => { + const r1 = await release("Zzqxv.Nonexistent.2019.1080p.WEB-DL-NOPE"); + const a = await resolveRelease(h.rt, { sourceReleaseId: r1.id }); + expect(a.outcome).toBe("no_match"); + expect(a.decisionId).toBeNull(); + const rel = await h.db + .selectFrom("source_release") + .select(["resolution_status"]) + .where("id", "=", r1.id) + .executeTakeFirstOrThrow(); + expect(rel.resolution_status).toBe("unidentifiable"); + + tmdb.fault(/\/search\//, { status: 500, times: 20 }); + const r2 = await release("Qwerty.Unknown.2020.1080p.WEB-DL-NOPE"); + const b = await resolveRelease(h.rt, { sourceReleaseId: r2.id }); + expect(b.outcome).toBe("lookup_failed"); + const pool = await h.db + .selectFrom("identity_candidate") + .selectAll() + .where("source_release_id", "=", r2.id) + .executeTakeFirstOrThrow(); + expect(pool.reason).toBe("lookup_failed"); + // 查询失败的复查间隔短于无结果 + expect(pool.next_review_at!.getTime() - Date.now()).toBeLessThan(20 * 60_000); + tmdb.clearFaults(); + }); + + it("自动化关闭:证据足够也只提议(proposed),进池待确认", async () => { + await h.db + .insertInto("settings") + .values({ key: "automation", value: JSON.stringify({ enabled: false }) } as never) + .execute(); + const r = await release("Dune.Part.Two.2024.2160p.UHD.BluRay.REMUX-FraMeSToR", { subtitle: "沙丘2" }); + const res = await resolveRelease(h.rt, { sourceReleaseId: r.id }); + expect(res.outcome).toBe("proposed"); + expect(res.decisionId).not.toBeNull(); + await h.db.deleteFrom("settings").where("key", "=", "automation").execute(); + }); + + it("refreshMetadata:字段级刷新;用户锁定字段不被覆盖,解锁后回到来源值", async () => { + const r1 = await refreshMetadata(h.rt, oppId); + expect(r1.status).toBe("refreshed"); + const fields = await h.db.selectFrom("metadata_field").selectAll().where("media_entity_id", "=", oppId).execute(); + expect(fields.map((f) => f.field)).toEqual(expect.arrayContaining(["title", "overview", "year"])); + let e = await h.db.selectFrom("media_entity").selectAll().where("id", "=", oppId).executeTakeFirstOrThrow(); + expect(e.title).toBe("奥本海默"); + expect((e.metadata as Record).overview?.source).toBe("tmdb"); + + const put = await json( + await h.req(`/media-entities/${oppId}/fields/title`, { + method: "PUT", + body: { value: "奥本海默(IMAX)", reason: "偏好" }, + }), + ); + expect(put.status).toBe(200); + expect(put.body.data.lockedFields).toContain("title"); + const r2 = await refreshMetadata(h.rt, oppId); + expect(r2.lockedFields).toContain("title"); + e = await h.db.selectFrom("media_entity").selectAll().where("id", "=", oppId).executeTakeFirstOrThrow(); + expect(e.title).toBe("奥本海默(IMAX)"); + expect(e.field_locks).toContain("title"); + expect((e.metadata as Record).title).toMatchObject({ + source: "user", + locked: true, + }); + + const del = await json(await h.req(`/media-entities/${oppId}/fields/title`, { method: "DELETE" })); + expect(del.status).toBe(200); + e = await h.db.selectFrom("media_entity").selectAll().where("id", "=", oppId).executeTakeFirstOrThrow(); + expect(e.title).toBe("奥本海默"); + expect(e.field_locks).not.toContain("title"); + }); + + it("人工纠错映射:旧映射退役留修订,新映射 accepted + 锁;自动匹配不改回", async () => { + const res = await json( + await h.req(`/media-entities/${oppId}/identities`, { + body: { provider: "tmdb", entityType: "movie", externalId: "693134", reason: "误匹配" }, + }), + ); + expect(res.status).toBe(200); + expect(res.body.data.changed).toBe(true); + const ids = await h.db + .selectFrom("external_identity") + .selectAll() + .where("media_entity_id", "=", oppId) + .where("provider", "=", "tmdb") + .execute(); + const old = ids.find((i) => i.external_id === "872585")!; + expect(old).toMatchObject({ status: "retired", locked: true }); + expect(ids.find((i) => i.external_id === "693134")).toMatchObject({ + status: "accepted", + locked: true, + evidence_level: "manual", + }); + const revs = await json(await h.req(`/external-identities/${old.id}/revisions`)); + expect(revs.body.data.items.map((r: { action: string }) => r.action)).toEqual(["created", "status_changed"]); + + // 新来一条奥本海默资源:TMDB 872585 不再自动绑回该作品 + const r = await release("Oppenheimer.2023.1080p.BluRay.x264-OTHER"); + await resolveRelease(h.rt, { sourceReleaseId: r.id }); + const accepted = await h.db + .selectFrom("external_identity") + .select(["media_entity_id"]) + .where("provider", "=", "tmdb") + .where("external_id", "=", "872585") + .where("status", "=", "accepted") + .execute(); + expect(accepted.every((a) => a.media_entity_id !== oppId)).toBe(true); + const still = await h.db + .selectFrom("external_identity") + .select(["external_id"]) + .where("media_entity_id", "=", oppId) + .where("provider", "=", "tmdb") + .where("status", "=", "accepted") + .executeTakeFirstOrThrow(); + expect(still.external_id).toBe("693134"); + + // 目标身份已属于其它作品 → 409 + const bear = await h.db + .selectFrom("external_identity") + .select(["media_entity_id"]) + .where("external_id", "=", "136315") + .executeTakeFirstOrThrow(); + const conflict = await json( + await h.req(`/media-entities/${oppId}/identities`, { + body: { provider: "tmdb", entityType: "tv", externalId: "136315" }, + }), + ); + expect(conflict.status).toBe(409); + expect(bear.media_entity_id).not.toBe(oppId); + }); + + it("合并重复作品:旧 ID 跳转,来源资源与别名迁到目标", async () => { + const dup = await json( + await h.req("/media-entities", { body: { kind: "movie", title: "奥本海默 重复", year: 2023 } }), + ); + expect(dup.status).toBe(200); + const dupId = dup.body.data.id; + const r = await release("Dup.Title.2023.1080p-X"); + await h.db.updateTable("source_release").set({ media_entity_id: dupId }).where("id", "=", r.id).execute(); + const m = await json(await h.req(`/media-entities/${dupId}/merge`, { body: { intoId: oppId, reason: "重复" } })); + expect(m.status).toBe(200); + expect(m.body.data.id).toBe(oppId); + const detail = await json(await h.req(`/media-entities/${dupId}/detail`)); + expect(detail.body.data.entity.id).toBe(oppId); + expect(detail.body.data.redirectedFrom).toBe(dupId); + expect(detail.body.data.aliases.map((a: { alias: string }) => a.alias)).toContain("奥本海默 重复"); + const moved = await h.db + .selectFrom("source_release") + .select(["media_entity_id"]) + .where("id", "=", r.id) + .executeTakeFirstOrThrow(); + expect(moved.media_entity_id).toBe(oppId); + const redirect = await h.db + .selectFrom("identity_redirect") + .selectAll() + .where("from_entity_id", "=", dupId) + .executeTakeFirstOrThrow(); + expect(redirect.to_entity_id).toBe(oppId); + // 再合并一次 → 409 + expect((await h.req(`/media-entities/${dupId}/merge`, { body: { intoId: oppId } })).status).toBe(409); + }); + + it("待处理中心聚合与列表", async () => { + const s = await json(await h.req("/decisions/summary")); + expect(s.status).toBe(200); + expect(s.body.data.total).toBeGreaterThanOrEqual(1); + expect(s.body.data.groups.some((g: { kind: string }) => g.kind === "identity_ambiguous")).toBe(true); + const l = await json(await h.req("/decisions?kind=identity_ambiguous")); + expect( + l.body.data.items.every((d: { status: string }) => ["open", "investigating", "awaiting_user"].includes(d.status)), + ).toBe(true); + // 已关闭的事项不可再处理 + const closed = await h.db + .selectFrom("decision") + .select(["id"]) + .where("status", "=", "resolved") + .executeTakeFirstOrThrow(); + expect((await h.req(`/decisions/${closed.id}/resolve`, { body: { dismiss: true } })).status).toBe(409); + }); + + it("能力:catalog.lookup / release.search 可调用;identity.resolve(propose)未授权被拒", async () => { + const reg = new CapabilityRegistry({}); + h.module.registerCapabilities!(reg, h.ctx); + const c = { actorId: "t", taskId: "t", scopeIds: [], signal: new AbortController().signal }; + const look = (await reg.invoke("catalog.lookup", { query: "奥本海默", includeRemote: true }, c)) as { + local: { entity: { id: string } }[]; + remote: { externalId: string }[]; + }; + expect(look.local.map((l) => l.entity.id)).toContain(oppId); + expect(look.remote.map((r) => r.externalId)).toContain("872585"); + const search = (await reg.invoke("release.search", { query: "Oppenheimer", allowLiveSearch: false }, c)) as { + items: unknown[]; + }; + expect(search.items.length).toBeGreaterThan(0); + await expect(reg.invoke("identity.resolve", { sourceReleaseId: twinsRelease }, c)).rejects.toThrow(); + }); +}); diff --git a/packages/contracts/src/content.ts b/packages/contracts/src/content.ts index 9f2f29b0..af53ce5c 100644 --- a/packages/contracts/src/content.ts +++ b/packages/contracts/src/content.ts @@ -1,5 +1,6 @@ import { z } from "zod"; -import { DateOnly, EntityBase, Id, JsonObject, SERVER_MANAGED, Timestamp } from "./common.ts"; +import { ActorType, DateOnly, EntityBase, Id, JsonObject, SERVER_MANAGED, Timestamp } from "./common.ts"; +import { ExternalIdentity } from "./identity.ts"; // 内容域:本地作品、内容单元、别名(REQ §3.1、§5.1、§5.4、§7.5) // 本地 UUID 是主身份;季/集编号、外部 ID 都可修订,不是唯一身份。 @@ -127,3 +128,82 @@ export const TitleAliasCreate = TitleAlias.pick({ export type TitleAliasCreate = z.infer; export const TitleAliasUpdate = TitleAliasCreate.omit({ mediaEntityId: true }).partial(); export type TitleAliasUpdate = z.infer; + +// ── W2-B:字段级元数据(REQ §7.3)与作品详情 ── + +export const MetadataSource = z.enum(["tmdb", "user", "agent", "nfo", "site", "import"]); +export type MetadataSource = z.infer; + +/** 某来源给出的一个候选字段值(刷新只写这里,不直接改最终展示值) */ +export const MetadataField = EntityBase.extend({ + mediaEntityId: Id.nullable(), + contentUnitId: Id.nullable(), + field: z.string().min(1), + value: z.unknown(), + language: z.string().nullable(), + source: MetadataSource, + sourceRevision: z.string().nullable(), + fetchedAt: Timestamp, +}); +export type MetadataField = z.infer; + +/** 字段覆盖:用户锁定优先于任何自动刷新 */ +export const FieldOverride = EntityBase.extend({ + mediaEntityId: Id.nullable(), + contentUnitId: Id.nullable(), + field: z.string().min(1), + value: z.unknown(), + locked: z.boolean(), + source: MetadataSource, + reason: z.string().nullable(), + actorType: ActorType, + actorId: z.string().nullable(), +}); +export type FieldOverride = z.infer; + +/** 可覆盖/锁定的作品字段;title/originalTitle/year 同时投影到 media_entity 列 */ +export const EntityMetadataField = z.enum([ + "title", + "originalTitle", + "year", + "overview", + "releaseDate", + "originalLanguage", + "originCountry", + "posterPath", +]); +export type EntityMetadataField = z.infer; + +export const FieldOverrideInput = z.object({ + value: z.unknown(), + lock: z.boolean().default(true), + reason: z.string().max(2000).optional(), +}); +export type FieldOverrideInput = z.infer; + +/** media_entity.metadata 中每个最终字段的形状 */ +export const FinalFieldValue = z.object({ + value: z.unknown(), + source: MetadataSource, + language: z.string().nullable(), + at: Timestamp, + locked: z.boolean(), +}); +export type FinalFieldValue = z.infer; + +/** 作品详情:外部身份、别名、内容单元、字段来源与覆盖 */ +export const MediaEntityDetail = z.object({ + entity: MediaEntity, + /** 请求的是已合并的旧 ID 时,指向跳转后的实体 */ + redirectedFrom: Id.nullable(), + identities: z.array(ExternalIdentity), + aliases: z.array(TitleAlias), + contentUnits: z.array(ContentUnit), + metadataFields: z.array(MetadataField), + overrides: z.array(FieldOverride), +}); +export type MediaEntityDetail = z.infer; + +/** 合并重复作品(保留旧 ID 跳转) */ +export const MediaEntityMerge = z.object({ intoId: Id, reason: z.string().max(2000).optional() }); +export type MediaEntityMerge = z.infer; diff --git a/packages/contracts/src/events.ts b/packages/contracts/src/events.ts index d26ffdc4..8c6eb61d 100644 --- a/packages/contracts/src/events.ts +++ b/packages/contracts/src/events.ts @@ -4,7 +4,7 @@ import { Id, NonNegInt, Timestamp } from "./common.ts"; import { DownloadStatus, PauseReason, TaskKind, TaskStatus } from "./execution.ts"; import { EvidenceLevel } from "./identity.ts"; import { OrganizationStatus } from "./organization.ts"; -import { FeedHealth } from "./sources.ts"; +import { FeedDuePayload, FeedGapSuspectedPayload, FeedHealth, ReleaseObservedPayload } from "./sources.ts"; /** SSE 事件信封:seq 在进程内单调递增,客户端断线后用 ?after= 补读 */ export const EventEnvelope = z.object({ @@ -41,7 +41,10 @@ export const EventType = { feedFetched: "feed.fetched", feedHealthChanged: "feed.health_changed", + feedDue: "feed.due", + feedGapSuspected: "feed.gap_suspected", + releaseObserved: "release.observed", releaseResolved: "release.resolved", decisionCreated: "decision.created", @@ -175,6 +178,9 @@ export const EVENT_PAYLOADS = { "agent.tool_call": AgentToolCallPayload, "feed.fetched": FeedFetchedPayload, "feed.health_changed": FeedHealthChangedPayload, + "feed.due": FeedDuePayload, + "feed.gap_suspected": FeedGapSuspectedPayload, + "release.observed": ReleaseObservedPayload, "release.resolved": ReleaseResolvedPayload, "decision.created": DecisionCreatedPayload, "decision.updated": DecisionUpdatedPayload, diff --git a/packages/contracts/src/identity.ts b/packages/contracts/src/identity.ts index f284bbc5..73339310 100644 --- a/packages/contracts/src/identity.ts +++ b/packages/contracts/src/identity.ts @@ -119,3 +119,142 @@ export const IdentityRevisionCreate = IdentityRevision.pick({ reason: true, }).partial({ fromStatus: true, toStatus: true, before: true, actorId: true, reason: true }); export type IdentityRevisionCreate = z.infer; + +// ── W2-B:待识别池、识别尝试、合并跳转与人工纠错(REQ §7.2、§7.5) ── + +export const IdentityCandidateStatus = z.enum(["pending", "resolved", "dismissed", "archived"]); +export type IdentityCandidateStatus = z.infer; +/** proposed = 证据足够但自动化关闭/只读,待确认 */ +export const IdentityCandidateReason = z.enum(["ambiguous", "no_match", "proposed", "lookup_failed"]); +export type IdentityCandidateReason = z.infer; + +/** 待识别池里的一个候选(只存 JSON,不建正式作品) */ +export const PoolCandidate = z.object({ + provider: z.string(), + entityType: z.enum(["movie", "tv"]), + externalId: z.string(), + title: z.string(), + year: z.number().int().nullable(), + /** 本地候选的作品 ID */ + localEntityId: Id.nullable(), + level: z.enum(["rule_confirmed", "multi_evidence", "needs_confirm", "rejected"]), + evidence: z.array(z.string()), +}); +export type PoolCandidate = z.infer; + +/** 待识别池条目:每条 source_release 至多一条;失败原因、证据版本与下一次复查时间 */ +export const IdentityCandidate = EntityBase.extend({ + sourceReleaseId: Id, + status: IdentityCandidateStatus, + reason: IdentityCandidateReason, + /** 解析出的检索条件(标题/年份/季集) */ + query: JsonObject, + candidates: z.array(PoolCandidate), + evidenceVersion: z.string(), + attempts: z.number().int().nonnegative(), + lastAttemptAt: Timestamp.nullable(), + nextReviewAt: Timestamp.nullable(), + decisionId: Id.nullable(), + resolvedMediaEntityId: Id.nullable(), + resolvedBy: z.string().nullable(), + resolvedAt: Timestamp.nullable(), +}); +export type IdentityCandidate = z.infer; + +export const ResolutionOutcome = z.enum([ + "auto", + "proposed", + "ambiguous", + "no_match", + "lookup_failed", + "manual", + "skipped_locked", +]); +export type ResolutionOutcome = z.infer; + +/** 一次识别尝试(只追加) */ +export const ResolutionAttempt = EntityBase.extend({ + sourceReleaseId: Id, + observationId: Id.nullable(), + outcome: ResolutionOutcome, + reason: z.string().nullable(), + evidenceVersion: z.string(), + candidateCount: z.number().int().nonnegative(), + mediaEntityId: Id.nullable(), + best: JsonObject.nullable(), + error: JsonObject.nullable(), + actorType: ActorType, + actorId: z.string().nullable(), +}); +export type ResolutionAttempt = z.infer; + +/** 作品合并后的旧 ID 跳转 */ +export const IdentityRedirect = EntityBase.extend({ + fromEntityId: Id, + toEntityId: Id, + reason: z.string().nullable(), + actorType: ActorType, + actorId: z.string().nullable(), +}); +export type IdentityRedirect = z.infer; + +/** 外部身份引用(provider + 类型 + ID) */ +export const ExternalRef = z.object({ + provider: IdentityProvider, + entityType: z.enum(["movie", "tv"]), + externalId: z.string().min(1), +}); +export type ExternalRef = z.infer; + +/** + * 人工指定来源资源的身份:本地作品 / 外部身份(未登记时按外部详情建档)/ 池内候选序号,三选一。 + * 指定后加人工锁,自动识别不再改写。 + */ +export const ReleaseIdentityAssign = z + .object({ + mediaEntityId: Id.optional(), + external: ExternalRef.optional(), + candidateIndex: z.number().int().min(0).optional(), + /** 同时关联到某个内容单元(季/集);缺省按解析出的季集建临时单元 */ + contentUnitId: Id.nullable().optional(), + lock: z.boolean().default(true), + reason: z.string().max(2000).optional(), + }) + .refine((v) => [v.mediaEntityId, v.external, v.candidateIndex].filter((x) => x !== undefined).length === 1, { + message: "mediaEntityId / external / candidateIndex 必须且只能给一个", + }); +export type ReleaseIdentityAssign = z.infer; + +/** 改作品的外部映射:旧的同 provider+类型 accepted 映射退役(留修订),新映射 accepted + 人工锁 */ +export const EntityIdentitySet = ExternalRef.extend({ + reason: z.string().max(2000).optional(), + /** 默认加人工锁 */ + lock: z.boolean().default(true), +}); +export type EntityIdentitySet = z.infer; + +export const IdentityPoolQuery = z.object({ + status: IdentityCandidateStatus.optional(), + reason: IdentityCandidateReason.optional(), + /** 按原始标题/解析标题检索(已观察到的资源必须能从待识别入口找回) */ + q: z.string().trim().min(1).max(200).optional(), + siteId: Id.optional(), + cursor: z.string().min(1).optional(), + limit: z.coerce.number().int().min(1).max(200).default(50), +}); +export type IdentityPoolQuery = z.infer; + +/** 待识别池列表条目:附来源资源摘要 */ +export const IdentityPoolItem = IdentityCandidate.extend({ + release: z.object({ + id: Id, + siteId: Id, + siteReleaseId: z.string(), + title: z.string(), + subtitle: z.string().nullable(), + categoryRaw: z.string().nullable(), + firstSeenAt: Timestamp, + resolutionStatus: z.string(), + }), +}); +export type IdentityPoolItem = z.infer; diff --git a/packages/contracts/src/sources.ts b/packages/contracts/src/sources.ts index 0ef86306..e2a7838a 100644 --- a/packages/contracts/src/sources.ts +++ b/packages/contracts/src/sources.ts @@ -4,7 +4,8 @@ import { EntityBase, Id, JsonObject, NonNegInt, SERVER_MANAGED, Timestamp } from // 来源域:站点、站点账号、Feed、站点资源、观察(REQ §4) // 含 passkey 的 Feed URL 只以 urlSecretRef 保存,对外只给脱敏的 urlDisplay。 -export const SiteAdapter = z.enum(["nexusphp", "mteam_api", "generic_rss"]); +/** api = MP 专用 API spider(Haidan/HDDolby/RousiPro/SunnyPT/TNode/TorrentLeech/Yema) */ +export const SiteAdapter = z.enum(["nexusphp", "mteam_api", "api", "generic_rss"]); export type SiteAdapter = z.infer; export const SiteHealth = z.enum(["unknown", "healthy", "degraded", "failing", "disabled"]); export type SiteHealth = z.infer; @@ -180,6 +181,8 @@ export const SourceRelease = EntityBase.extend({ resolutionEvidence: JsonObject, availability: ReleaseAvailability, raw: JsonObject, + /** 人工确认的身份关联:自动识别/复查不得改写 mediaEntityId/contentUnitId */ + resolutionLocked: z.boolean(), }); export type SourceRelease = z.infer; /** 由适配器/任务写入(按 siteId+siteReleaseId upsert) */ @@ -209,6 +212,7 @@ export const SourceReleaseCreate = SourceRelease.omit({ resolutionEvidence: true, availability: true, raw: true, + resolutionLocked: true, }); export type SourceReleaseCreate = z.infer; export const SourceReleaseUpdate = SourceReleaseCreate.omit({ siteId: true, siteReleaseId: true }).partial(); @@ -259,3 +263,143 @@ export const ObservationUpdate = Observation.pick({ lastError: true, }).partial(); export type ObservationUpdate = z.infer; + +// ── W2-B:站点快照、调度事件与站点工作台 ── + +export const SiteSnapshotKind = z.enum(["periodic", "backfill", "live_search", "manual"]); +export type SiteSnapshotKind = z.infer; +export const SiteSnapshotStatus = z.enum(["running", "ok", "partial", "failed"]); +export type SiteSnapshotStatus = z.infer; + +/** 一次列表快照 / 实时补查 / 补漏的覆盖范围与结果(不把一次分页伪称全站快照,REQ §4.6) */ +export const SiteSnapshot = EntityBase.extend({ + siteId: Id, + kind: SiteSnapshotKind, + query: z.string().nullable(), + filters: JsonObject, + /** 实际抓取的页码(1 起) */ + pages: z.array(z.number().int().positive()), + itemCount: NonNegInt, + newCount: NonNegInt, + status: SiteSnapshotStatus, + error: JsonObject.nullable(), + definitionVersion: z.string().nullable(), + startedAt: Timestamp, + finishedAt: Timestamp.nullable(), +}); +export type SiteSnapshot = z.infer; + +/** 站点注册表条目(随产品发布的适配器定义)+ 是否已在本系统登记 */ +export const SiteDefinitionView = z.object({ + key: z.string(), + name: z.string(), + adapter: SiteAdapter, + baseUrl: z.string(), + /** 定义是否随产品启用(未实测站点为 false) */ + shippedEnabled: z.boolean(), + definitionVersion: z.string(), + capabilities: z.object({ + list: z.boolean(), + search: z.boolean(), + detail: z.boolean(), + download: z.boolean(), + rss: z.boolean(), + }), + defaultRateLimit: z.object({ minIntervalMs: NonNegInt, burst: z.number().int().positive() }), + authSupported: z.array(z.string()), + /** 已登记的 site 行 ID;未登记为 null */ + siteId: Id.nullable(), +}); +export type SiteDefinitionView = z.infer; + +/** Feed 到期(调度器发出,outbox 路由到 ingestFeed) */ +export const FeedDuePayload = z.object({ feedId: Id, reason: z.enum(["schedule", "manual"]) }); +/** 新的 source_release 被观察到(或同一资源标题更新),outbox 路由到 resolveObservation */ +export const ReleaseObservedPayload = z.object({ + sourceReleaseId: Id, + observationId: Id.nullable(), + siteId: Id, + isNew: z.boolean(), +}); +/** Feed 窗口与上次零重叠:疑似缺口,路由到有预算的列表补漏(snapshotSite backfill) */ +export const FeedGapSuspectedPayload = z.object({ + feedId: Id, + siteId: Id.nullable(), + previousFetchedAt: Timestamp.nullable(), + detectedAt: Timestamp, +}); + +/** 站点工作台列表查询(整理视图):快照优先,必要时实时补查 */ +export const WorkbenchReleaseQuery = z.object({ + q: z.string().trim().min(1).max(200).optional(), + category: z.string().max(100).optional(), + /** 只看某类促销(free/2xfree/50%…) */ + promotion: z.string().max(20).optional(), + resolution: ResolutionStatus.optional(), + /** true = 强制实时补查;false = 只看本地快照;缺省 = 本地无结果或快照过期时补查 */ + live: z.enum(["true", "false"]).optional(), + cursor: z.string().min(1).optional(), + limit: z.coerce.number().int().min(1).max(200).default(50), +}); +export type WorkbenchReleaseQuery = z.infer; + +/** 数据来源:快照 / Feed 观察 / 实时补查 */ +export const ReleaseOrigin = z.enum(["snapshot", "feed", "live", "unknown"]); +export type ReleaseOrigin = z.infer; + +/** + * 工作台条目:source_release + 来源标注 + 本地实体关联。 + * 库存叠加(已有资产/Slot/下载状态)由前端按 mediaEntityId 二次查询 W2-C 端点,不在这里猜。 + */ +export const WorkbenchRelease = SourceRelease.extend({ + origin: ReleaseOrigin, + /** 本条数据最近一次被站点确认的时间(快照/Feed/实时) */ + dataUpdatedAt: Timestamp, + observedByFeeds: NonNegInt, + entity: z.object({ id: Id, title: z.string(), year: z.number().int().nullable(), kind: z.string() }).nullable(), +}); +export type WorkbenchRelease = z.infer; + +export const WorkbenchCoverage = z.object({ + siteEnabled: z.boolean(), + lastSnapshotAt: Timestamp.nullable(), + /** 快照过期(超过 2 个周期或从未快照) */ + stale: z.boolean(), + fromSnapshot: z.boolean(), + live: z.boolean(), + liveError: z.string().nullable(), + /** 同一搜索正在进行时合并到已有请求 */ + coalesced: z.boolean(), +}); +export type WorkbenchCoverage = z.infer; + +export const WorkbenchReleasePage = z.object({ + items: z.array(WorkbenchRelease), + nextCursor: z.string().nullable(), + coverage: WorkbenchCoverage, +}); +export type WorkbenchReleasePage = z.infer; + +export const WorkbenchReleaseDetail = z.object({ + release: WorkbenchRelease, + observations: z.array( + z.object({ + observationId: Id, + feedId: Id, + feedName: z.string(), + firstSeenAt: Timestamp, + lastSeenAt: Timestamp, + }), + ), + /** 易变字段的复核时间:缓存命中不能替代下载前复核(REQ §4.6) */ + volatile: z.object({ + promotionObservedAt: Timestamp.nullable(), + promotionExpiresAt: Timestamp.nullable(), + peersObservedAt: Timestamp, + detailFetchedAt: Timestamp.nullable(), + recheckBeforeDownload: z.literal(true), + }), + /** 刷新详情失败时保留既有数据并给出原因 */ + refreshError: z.string().nullable(), +}); +export type WorkbenchReleaseDetail = z.infer; diff --git a/packages/contracts/test/db-sync.test.ts b/packages/contracts/test/db-sync.test.ts index 8420afd3..30fad541 100644 --- a/packages/contracts/test/db-sync.test.ts +++ b/packages/contracts/test/db-sync.test.ts @@ -79,6 +79,13 @@ const ROWS: Record Subscription: { row: C.Subscription }, Task: { row: C.Task }, TitleAlias: { row: C.TitleAlias }, + // W2-B 来源/身份/元数据(0018–0020) + FieldOverride: { row: C.FieldOverride }, + IdentityCandidate: { row: C.IdentityCandidate }, + IdentityRedirect: { row: C.IdentityRedirect }, + MetadataField: { row: C.MetadataField }, + ResolutionAttempt: { row: C.ResolutionAttempt }, + SiteSnapshot: { row: C.SiteSnapshot }, }; /** 引擎自有表(@umt/tasks 内部使用,不经 API 暴露),不要求 Row schema */ @@ -186,6 +193,17 @@ const ENUMS: Record = { TaskStatus: C.TaskStatus, TitleAliasKind: C.AliasKind, TitleAliasSource: C.AliasSource, + // W2-B(0018–0020) + FieldOverrideSource: C.MetadataSource, + FieldOverrideActorType: C.ActorType, + IdentityCandidateStatus: C.IdentityCandidateStatus, + IdentityCandidateReason: C.IdentityCandidateReason, + IdentityRedirectActorType: C.ActorType, + MetadataFieldSource: C.MetadataSource, + ResolutionAttemptOutcome: C.ResolutionOutcome, + ResolutionAttemptActorType: C.ActorType, + SiteSnapshotKind: C.SiteSnapshotKind, + SiteSnapshotStatus: C.SiteSnapshotStatus, }; describe("contracts ⇔ db 表结构同步", () => { diff --git a/packages/contracts/test/schemas.test.ts b/packages/contracts/test/schemas.test.ts index cd0874d8..370edba1 100644 --- a/packages/contracts/test/schemas.test.ts +++ b/packages/contracts/test/schemas.test.ts @@ -91,6 +91,7 @@ const OVERRIDES: Record) => Record v, IdentityResolveInput: (v) => v, AssetLocationCreate: (v) => v, + ReleaseIdentityAssign: (v) => ({ ...v, external: undefined, candidateIndex: undefined }), CacheSettings: (v) => ({ ...v, hardThresholdPercent: 90, resumePercent: 85, cleanupTargetPercent: 80 }), GlobalSettings: () => C.DEFAULT_SETTINGS, EngineStats: (v) => ({ ...v, byStatus: { pending: 1 } }), diff --git a/packages/db/README.md b/packages/db/README.md index 64866ac0..bab8c655 100644 --- a/packages/db/README.md +++ b/packages/db/README.md @@ -29,6 +29,9 @@ PostgreSQL 访问层:Kysely 客户端与连接池、表类型(`src/types.ts`)、 | `0012_organization` | organization_record(change_seq 序列 + 触发器)、organization_file_result;回填 asset_operation.organization_record_id | | `0013_tasks_tables` | 并入 @umt/tasks 自有表:tasks_run_meta、tasks_fact、tasks_op_ledger(if not exists,可接管 W1 旧库) | | `0014_agent_run_event` | agent_run_event:Agent 运行事件,主键 (run_id, seq),删 run 级联 | +| `0018_sources_snapshot` | site.adapter 增加 `api`;source_release.resolution_locked(人工锁);site_snapshot(快照/补漏/实时补查的覆盖范围) | +| `0019_identity_pool` | identity_candidate(待识别池)、resolution_attempt(识别尝试)、identity_redirect(合并跳转) | +| `0020_metadata_fields` | metadata_field(来源候选字段)、field_override(用户覆盖与锁定) | ## 表清单(45 张) @@ -59,6 +62,11 @@ PostgreSQL 访问层:Kysely 客户端与连接池、表类型(`src/types.ts`)、 | `title_alias` | 原名/译名/简称/站点原标题;`normalized` 上建 gin trgm 索引做模糊召回 | | `external_identity` | 本地对象 ↔ 外部 ID;**`(provider, entity_type, external_id)` 对 accepted 部分唯一**,同一本地对象每个 provider+type 只有一个 accepted;候选(proposed/conflicted)可并存;404/超时只记 `verify_status` | | `identity_revision` | 身份每次变更的修订历史,`(external_identity_id, revision)` 唯一,只追加 | +| `identity_candidate` | 待识别池:每条 source_release 至多一条;候选只存 JSON(不建正式作品),保存原因/证据版本/复查时间(有界退避) | +| `resolution_attempt` | 每次识别尝试的结果与证据版本,只追加 | +| `identity_redirect` | 作品合并后的旧 ID 跳转,`from_entity_id` 唯一 | +| `metadata_field` | 各来源的候选字段值(来源/语言/时间/来源修订);刷新只写这里 | +| `field_override` | 用户字段覆盖与锁定,优先于自动来源;目标 + 字段唯一 | ### 来源 | 表 | 用途与关键不变量 | @@ -68,6 +76,7 @@ PostgreSQL 访问层:Kysely 客户端与连接池、表类型(`src/types.ts`)、 | `feed_source` | RSS/列表/查询入口;真实 URL(含 passkey)只在 `url_secret_ref`,对外只给 `url_display`;按 `next_fetch_at` 调度 | | `source_release` | 站点资源;**`(site_id, site_release_id)` 唯一**,多 Feed 观察汇到同一行;`source_published_at`/`first_seen_at` 分开 | | `observation` | 一次原始观察;**`(feed_id, guid)` 唯一**,`content_digest` 变化即条目更新 | +| `site_snapshot` | 一次列表快照 / 补漏 / 实时补查:页码、条目数、状态与错误(新鲜度来源) | ### 文件 | 表 | 用途与关键不变量 | diff --git a/packages/db/migrations/0018_sources_snapshot.ts b/packages/db/migrations/0018_sources_snapshot.ts new file mode 100644 index 00000000..679c4ebc --- /dev/null +++ b/packages/db/migrations/0018_sources_snapshot.ts @@ -0,0 +1,51 @@ +import type { Kysely } from "kysely"; +import { dropTables, entityTable, execDdl, oneOf } from "../src/migration-helpers.ts"; + +// W2-B 来源补充(REQ §4.5、§4.6、§7.5): +// - site.adapter 增加 api(MP 专用 API spider:Haidan/HDDolby/RousiPro/…,见 adapters/sites/spiders); +// - source_release.resolution_locked:人工确认的身份关联,自动识别不得改写; +// - site_snapshot:每次列表快照/实时补查/补漏的覆盖范围与新鲜度(不把一次分页伪称全站快照)。 + +// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型 +export async function up(db: Kysely): Promise { + await execDdl( + db, + ` +alter table site drop constraint site_adapter_check; +alter table site add constraint site_adapter_check ${oneOf("adapter", ["nexusphp", "mteam_api", "api", "generic_rss"])}; + +alter table source_release add column resolution_locked boolean not null default false; + +${entityTable( + "site_snapshot", + ` + site_id uuid not null references site (id) on delete cascade, + kind text not null ${oneOf("kind", ["periodic", "backfill", "live_search", "manual"])}, + query text, + filters jsonb not null default '{}'::jsonb, + pages jsonb not null default '[]'::jsonb, + item_count integer not null default 0, + new_count integer not null default 0, + status text not null default 'running' ${oneOf("status", ["running", "ok", "partial", "failed"])}, + error jsonb, + definition_version text, + started_at timestamptz not null default now(), + finished_at timestamptz`, +)} +create index site_snapshot_site_idx on site_snapshot (site_id, started_at desc); +`, + ); +} + +// biome-ignore lint/suspicious/noExplicitAny: 同上 +export async function down(db: Kysely): Promise { + await execDdl( + db, + ` +${dropTables(["site_snapshot"])} +alter table source_release drop column if exists resolution_locked; +alter table site drop constraint site_adapter_check; +alter table site add constraint site_adapter_check ${oneOf("adapter", ["nexusphp", "mteam_api", "generic_rss"])}; +`, + ); +} diff --git a/packages/db/migrations/0019_identity_pool.ts b/packages/db/migrations/0019_identity_pool.ts new file mode 100644 index 00000000..f67f80f8 --- /dev/null +++ b/packages/db/migrations/0019_identity_pool.ts @@ -0,0 +1,74 @@ +import type { Kysely } from "kysely"; +import { dropTables, entityTable, execDdl, oneOf } from "../src/migration-helpers.ts"; + +// W2-B 待识别池与身份治理(REQ §7.2、§7.5): +// - identity_candidate:每条 source_release 至多一条待识别记录,候选只存 JSON,不逐个建为正式作品; +// 失败保存原因、证据版本与下一次复查时间(有界退避),新证据到来时重试; +// - resolution_attempt:每次识别尝试的结果与证据版本(只追加); +// - identity_redirect:作品合并后的旧 ID 跳转历史(与 media_entity.merged_into_id 一致,可修订)。 + +const ACTOR = ["user", "agent", "system", "api_key"] as const; + +// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型 +export async function up(db: Kysely): Promise { + await execDdl( + db, + ` +${entityTable( + "identity_candidate", + ` + source_release_id uuid not null references source_release (id) on delete cascade, + status text not null default 'pending' ${oneOf("status", ["pending", "resolved", "dismissed", "archived"])}, + reason text not null ${oneOf("reason", ["ambiguous", "no_match", "proposed", "lookup_failed"])}, + query jsonb not null default '{}'::jsonb, + candidates jsonb not null default '[]'::jsonb, + evidence_version text not null, + attempts integer not null default 0, + last_attempt_at timestamptz, + next_review_at timestamptz, + decision_id uuid references decision (id) on delete set null, + resolved_media_entity_id uuid references media_entity (id) on delete set null, + resolved_by text, + resolved_at timestamptz, + constraint identity_candidate_release_key unique (source_release_id)`, +)} +create index identity_candidate_review_idx on identity_candidate (next_review_at) where status = 'pending'; +create index identity_candidate_status_idx on identity_candidate (status, updated_at desc); + +${entityTable( + "resolution_attempt", + ` + source_release_id uuid not null references source_release (id) on delete cascade, + observation_id uuid references observation (id) on delete set null, + outcome text not null ${oneOf("outcome", ["auto", "proposed", "ambiguous", "no_match", "lookup_failed", "manual", "skipped_locked"])}, + reason text, + evidence_version text not null, + candidate_count integer not null default 0, + media_entity_id uuid references media_entity (id) on delete set null, + best jsonb, + error jsonb, + actor_type text not null default 'system' ${oneOf("actor_type", ACTOR)}, + actor_id text`, +)} +create index resolution_attempt_release_idx on resolution_attempt (source_release_id, created_at desc); + +${entityTable( + "identity_redirect", + ` + from_entity_id uuid not null references media_entity (id) on delete cascade, + to_entity_id uuid not null references media_entity (id) on delete cascade, + reason text, + actor_type text not null ${oneOf("actor_type", ACTOR)}, + actor_id text, + constraint identity_redirect_from_key unique (from_entity_id), + constraint identity_redirect_not_self check (from_entity_id <> to_entity_id)`, +)} +create index identity_redirect_to_idx on identity_redirect (to_entity_id); +`, + ); +} + +// biome-ignore lint/suspicious/noExplicitAny: 同上 +export async function down(db: Kysely): Promise { + await execDdl(db, dropTables(["identity_redirect", "resolution_attempt", "identity_candidate"])); +} diff --git a/packages/db/migrations/0020_metadata_fields.ts b/packages/db/migrations/0020_metadata_fields.ts new file mode 100644 index 00000000..60921a8c --- /dev/null +++ b/packages/db/migrations/0020_metadata_fields.ts @@ -0,0 +1,54 @@ +import type { Kysely } from "kysely"; +import { dropTables, entityTable, execDdl, oneOf } from "../src/migration-helpers.ts"; + +// W2-B 字段级元数据协调(REQ §7.3): +// - metadata_field:各来源的候选字段值(来源/语言/时间/来源修订),刷新只写这里; +// - field_override:用户(或 Agent 生成并确认)的字段覆盖与锁定,优先于任何自动来源; +// 最终展示值投影到 media_entity.metadata / content_unit.metadata。路径绑定不受元数据刷新影响。 + +const SOURCE = ["tmdb", "user", "agent", "nfo", "site", "import"] as const; +const target = "constraint {t}_target_check check (num_nonnulls(media_entity_id, content_unit_id) = 1)"; + +// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型 +export async function up(db: Kysely): Promise { + await execDdl( + db, + ` +${entityTable( + "metadata_field", + ` + media_entity_id uuid references media_entity (id) on delete cascade, + content_unit_id uuid references content_unit (id) on delete cascade, + field text not null, + value jsonb, + language text, + source text not null ${oneOf("source", SOURCE)}, + source_revision text, + fetched_at timestamptz not null default now(), + ${target.replace("{t}", "metadata_field")}`, +)} +create unique index metadata_field_key on metadata_field (media_entity_id, content_unit_id, field, source, language) nulls not distinct; + +${entityTable( + "field_override", + ` + media_entity_id uuid references media_entity (id) on delete cascade, + content_unit_id uuid references content_unit (id) on delete cascade, + field text not null, + value jsonb, + locked boolean not null default true, + source text not null default 'user' ${oneOf("source", SOURCE)}, + reason text, + actor_type text not null ${oneOf("actor_type", ["user", "agent", "system", "api_key"])}, + actor_id text, + ${target.replace("{t}", "field_override")}`, +)} +create unique index field_override_key on field_override (media_entity_id, content_unit_id, field) nulls not distinct; +`, + ); +} + +// biome-ignore lint/suspicious/noExplicitAny: 同上 +export async function down(db: Kysely): Promise { + await execDdl(db, dropTables(["field_override", "metadata_field"])); +} diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index 2abb363d..91565a3a 100644 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -117,6 +117,11 @@ export type ExternalIdentityVerifyStatus = "unverified" | "ok" | "not_found" | " export type FeedSourcePurpose = "movie" | "series" | "variety" | "promotion" | "search" | "generic"; export type FeedSourceKind = "rss" | "atom" | "list" | "query"; export type FeedSourceHealthStatus = "unknown" | "healthy" | "degraded" | "failing" | "circuit_open" | "disabled"; +export type FieldOverrideSource = "tmdb" | "user" | "agent" | "nfo" | "site" | "import"; +export type FieldOverrideActorType = "user" | "agent" | "system" | "api_key"; +export type IdentityCandidateStatus = "pending" | "resolved" | "dismissed" | "archived"; +export type IdentityCandidateReason = "ambiguous" | "no_match" | "proposed" | "lookup_failed"; +export type IdentityRedirectActorType = "user" | "agent" | "system" | "api_key"; export type IdentityRevisionAction = | "created" | "status_changed" @@ -136,6 +141,7 @@ export type MediaEntityKind = "movie" | "series" | "variety" | "collection" | "o export type MediaEntityStatus = "provisional" | "active" | "merged" | "archived"; export type MediaServerInstanceKind = "emby" | "jellyfin"; export type MediaServerInstanceHealthStatus = "unknown" | "healthy" | "degraded" | "offline"; +export type MetadataFieldSource = "tmdb" | "user" | "agent" | "nfo" | "site" | "import"; export type ObservationStatus = "new" | "linked" | "ignored" | "error"; export type OrganizationFileResultRole = "main" | "subtitle" | "audio" | "bluray_dir" | "nfo" | "image" | "other"; export type OrganizationFileResultMode = "copy" | "move" | "rename" | "relocate" | "hardlink"; @@ -158,6 +164,15 @@ export type PreferenceTagStatus = "active" | "disabled" | "deleted"; export type ReconcileRunKind = "identity" | "metadata" | "asset" | "acquisition" | "server"; export type ReconcileRunMode = "normal" | "read_only"; export type ReconcileRunStatus = "queued" | "running" | "succeeded" | "partial" | "failed" | "cancelled"; +export type ResolutionAttemptOutcome = + | "auto" + | "proposed" + | "ambiguous" + | "no_match" + | "lookup_failed" + | "manual" + | "skipped_locked"; +export type ResolutionAttemptActorType = "user" | "agent" | "system" | "api_key"; export type SecretPurpose = | "site_cookie" | "site_passkey" @@ -171,11 +186,13 @@ export type SecretPurpose = | "other"; export type ServerMappingItemType = "movie" | "series" | "season" | "episode" | "other"; export type ServerMappingMatchStatus = "matched" | "ambiguous" | "unmatched" | "stale"; -export type SiteAdapter = "nexusphp" | "mteam_api" | "generic_rss"; +export type SiteAdapter = "nexusphp" | "mteam_api" | "api" | "generic_rss"; export type SiteHealthStatus = "unknown" | "healthy" | "degraded" | "failing" | "disabled"; export type SiteCircuitState = "closed" | "open" | "half_open"; export type SiteAccountAuthKind = "cookie" | "passkey" | "api_key" | "password"; export type SiteAccountStatus = "unknown" | "active" | "invalid" | "expired"; +export type SiteSnapshotKind = "periodic" | "backfill" | "live_search" | "manual"; +export type SiteSnapshotStatus = "running" | "ok" | "partial" | "failed"; export type SourceReleaseResolutionStatus = "unresolved" | "resolved" | "ambiguous" | "unidentifiable" | "rejected"; export type SourceReleaseAvailability = "active" | "site_disabled" | "expired" | "unknown"; export type SubscriptionMediaKind = "movie" | "series" | "variety"; @@ -573,6 +590,51 @@ export interface FeedSourceTable { updated_at: GeneratedTimestamptz; } +export interface FieldOverrideTable { + id: Generated; + media_entity_id: string | null; + content_unit_id: string | null; + field: string; + value: Json | null; + locked: Generated; + source: Generated; + reason: string | null; + actor_type: FieldOverrideActorType; + actor_id: string | null; + created_at: GeneratedTimestamptz; + updated_at: GeneratedTimestamptz; +} + +export interface IdentityCandidateTable { + id: Generated; + source_release_id: string; + status: Generated; + reason: IdentityCandidateReason; + query: Generated; + candidates: Generated; + evidence_version: string; + attempts: Generated; + last_attempt_at: Timestamptz | null; + next_review_at: Timestamptz | null; + decision_id: string | null; + resolved_media_entity_id: string | null; + resolved_by: string | null; + resolved_at: Timestamptz | null; + created_at: GeneratedTimestamptz; + updated_at: GeneratedTimestamptz; +} + +export interface IdentityRedirectTable { + id: Generated; + from_entity_id: string; + to_entity_id: string; + reason: string | null; + actor_type: IdentityRedirectActorType; + actor_id: string | null; + created_at: GeneratedTimestamptz; + updated_at: GeneratedTimestamptz; +} + export interface IdentityRevisionTable { id: Generated; external_identity_id: string; @@ -665,6 +727,20 @@ export interface MediaServerInstanceTable { updated_at: GeneratedTimestamptz; } +export interface MetadataFieldTable { + id: Generated; + media_entity_id: string | null; + content_unit_id: string | null; + field: string; + value: Json | null; + language: string | null; + source: MetadataFieldSource; + source_revision: string | null; + fetched_at: GeneratedTimestamptz; + created_at: GeneratedTimestamptz; + updated_at: GeneratedTimestamptz; +} + export interface ObservationTable { id: Generated; feed_id: string; @@ -855,6 +931,23 @@ export interface ReconcileRunTable { updated_at: GeneratedTimestamptz; } +export interface ResolutionAttemptTable { + id: Generated; + source_release_id: string; + observation_id: string | null; + outcome: ResolutionAttemptOutcome; + reason: string | null; + evidence_version: string; + candidate_count: Generated; + media_entity_id: string | null; + best: Json | null; + error: Json | null; + actor_type: Generated; + actor_id: string | null; + created_at: GeneratedTimestamptz; + updated_at: GeneratedTimestamptz; +} + export interface SecretTable { id: Generated; name: string; @@ -929,6 +1022,24 @@ export interface SiteAccountTable { updated_at: GeneratedTimestamptz; } +export interface SiteSnapshotTable { + id: Generated; + site_id: string; + kind: SiteSnapshotKind; + query: string | null; + filters: Generated; + pages: Generated; + item_count: Generated; + new_count: Generated; + status: Generated; + error: Json | null; + definition_version: string | null; + started_at: GeneratedTimestamptz; + finished_at: Timestamptz | null; + created_at: GeneratedTimestamptz; + updated_at: GeneratedTimestamptz; +} + export interface SourceReleaseTable { id: Generated; site_id: string; @@ -959,6 +1070,7 @@ export interface SourceReleaseTable { raw: Generated; created_at: GeneratedTimestamptz; updated_at: GeneratedTimestamptz; + resolution_locked: Generated; } export interface SseEventTable { @@ -1082,12 +1194,16 @@ export interface DB { downloader_instance: DownloaderInstanceTable; external_identity: ExternalIdentityTable; feed_source: FeedSourceTable; + field_override: FieldOverrideTable; + identity_candidate: IdentityCandidateTable; + identity_redirect: IdentityRedirectTable; identity_revision: IdentityRevisionTable; library_root: LibraryRootTable; login_attempt: LoginAttemptTable; media_asset: MediaAssetTable; media_entity: MediaEntityTable; media_server_instance: MediaServerInstanceTable; + metadata_field: MetadataFieldTable; observation: ObservationTable; organization_file_result: OrganizationFileResultTable; organization_record: OrganizationRecordTable; @@ -1098,11 +1214,13 @@ export interface DB { preference_suppression: PreferenceSuppressionTable; preference_tag: PreferenceTagTable; reconcile_run: ReconcileRunTable; + resolution_attempt: ResolutionAttemptTable; secret: SecretTable; server_mapping: ServerMappingTable; settings: SettingsTable; site: SiteTable; site_account: SiteAccountTable; + site_snapshot: SiteSnapshotTable; source_release: SourceReleaseTable; sse_event: SseEventTable; subscription: SubscriptionTable; @@ -1173,6 +1291,15 @@ export type ExternalIdentityUpdate = Updateable; export type FeedSourceRow = Selectable; export type NewFeedSource = Insertable; export type FeedSourceUpdate = Updateable; +export type FieldOverrideRow = Selectable; +export type NewFieldOverride = Insertable; +export type FieldOverrideUpdate = Updateable; +export type IdentityCandidateRow = Selectable; +export type NewIdentityCandidate = Insertable; +export type IdentityCandidateUpdate = Updateable; +export type IdentityRedirectRow = Selectable; +export type NewIdentityRedirect = Insertable; +export type IdentityRedirectUpdate = Updateable; export type IdentityRevisionRow = Selectable; export type NewIdentityRevision = Insertable; export type IdentityRevisionUpdate = Updateable; @@ -1191,6 +1318,9 @@ export type MediaEntityUpdate = Updateable; export type MediaServerInstanceRow = Selectable; export type NewMediaServerInstance = Insertable; export type MediaServerInstanceUpdate = Updateable; +export type MetadataFieldRow = Selectable; +export type NewMetadataField = Insertable; +export type MetadataFieldUpdate = Updateable; export type ObservationRow = Selectable; export type NewObservation = Insertable; export type ObservationUpdate = Updateable; @@ -1221,6 +1351,9 @@ export type PreferenceTagUpdate = Updateable; export type ReconcileRunRow = Selectable; export type NewReconcileRun = Insertable; export type ReconcileRunUpdate = Updateable; +export type ResolutionAttemptRow = Selectable; +export type NewResolutionAttempt = Insertable; +export type ResolutionAttemptUpdate = Updateable; export type SecretRow = Selectable; export type NewSecret = Insertable; export type SecretUpdate = Updateable; @@ -1236,6 +1369,9 @@ export type SiteUpdate = Updateable; export type SiteAccountRow = Selectable; export type NewSiteAccount = Insertable; export type SiteAccountUpdate = Updateable; +export type SiteSnapshotRow = Selectable; +export type NewSiteSnapshot = Insertable; +export type SiteSnapshotUpdate = Updateable; export type SourceReleaseRow = Selectable; export type NewSourceRelease = Insertable; export type SourceReleaseUpdate = Updateable; diff --git a/packages/db/test/schema.test.ts b/packages/db/test/schema.test.ts index c876f187..ff5da3d3 100644 --- a/packages/db/test/schema.test.ts +++ b/packages/db/test/schema.test.ts @@ -68,6 +68,13 @@ const EXPECTED_TABLES = [ "tasks_fact", "tasks_op_ledger", "agent_run_event", + // W2-B 来源/身份/元数据(0018–0020) + "site_snapshot", + "identity_candidate", + "resolution_attempt", + "identity_redirect", + "metadata_field", + "field_override", // 登录门禁(0015)与 SSE 多进程总线(0016) "admin_user", "admin_session", diff --git a/packages/mapping/test/convert.test.ts b/packages/mapping/test/convert.test.ts index bdd56d9e..d15a07e1 100644 --- a/packages/mapping/test/convert.test.ts +++ b/packages/mapping/test/convert.test.ts @@ -258,6 +258,7 @@ describe("SiteRelease ↔ source_release", () => { resolutionStatus: "unresolved", resolutionEvidence: {}, availability: "active", + resolutionLocked: false, }; expect(C.SourceRelease.safeParse(row).success).toBe(true); const { downloadUrl: _d, ...expected } = site;