feat(server): W2-B sources, identity resolution, metadata and site workbench

- Sites / accounts / feeds: registry view (/site-definitions), site CRUD with
  enable/disable (history stays queryable, marked site_disabled), accounts and
  feed URLs stored as secrets (only secret_ref / redacted urlDisplay exposed),
  feed enable/disable/reset/fetch; one shared rate limiter + adapter per site.
- Tasks: ingestFeed (FeedFetchState/FeedCursor on feed_source, observation
  feed+guid upsert with digest-change updates, source_release site+id
  aggregation across feeds, auth failure -> account invalid + decision, no
  retry, gap -> feed.gap_suspected -> budgeted backfill snapshot),
  resolveObservation (parseRelease -> local recall via title_alias pg_trgm +
  accepted identities -> TMDB -> resolveIdentity; auto binds entity /
  provisional units / identities / aliases, otherwise identity_candidate pool
  with evidence version and bounded review backoff; manual lock respected),
  refreshMetadata (metadata_field + field_override, user locks win, paths
  untouched), snapshotSite (site_snapshot coverage records).
- Events: feed.due, feed.gap_suspected, release.observed (self-consumed),
  release.resolved, decision.created/updated; scheduler in worker start().
- Identity API: entity detail with redirects, manual mapping correction with
  identity_revision, merge with identity_redirect, pool list/assign/dismiss/
  retry, content unit numbering lock, aliases, external identities.
- Decisions: list/summary (by entity and kind)/detail/resolve; identity
  options executed directly.
- Workbench: /sites/{id}/releases (snapshot first, coalesced live search with
  backoff, origin + freshness) and /sites/{id}/releases/{rid} (volatile field
  recheck times, refresh keeps data on failure).
- Capabilities: catalog.lookup, release.search, release.inspect,
  identity.resolve.
- Migrations 0018-0020: site.adapter api, source_release.resolution_locked,
  site_snapshot, identity_candidate, resolution_attempt, identity_redirect,
  metadata_field, field_override.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed 2026-09-27 20:12:01 +08:00
1 parent 829ca4f7ac
commit a24bda952b
39 files changed
+6861 -15

No files matched your search

@@ -0,0 +1,187 @@
import type { CapabilityRegistry } from "@umt/agent";
import { capabilitySpec, type ExternalEntityType, type SourceRelease } from "@umt/contracts";
import { sql } from "@umt/db";
import { normalizeTitle } from "@umt/domain";
import { toApi, toApiList } from "./rows.ts";
import type { SourcesRuntime } from "./runtime.ts";
import { assignReleaseIdentity } from "./service/identity-admin.ts";
import { resolveRelease } from "./service/resolve.ts";
import { refreshReleaseDetail } from "./service/snapshot.ts";
import { liveSearch } from "./service/workbench.ts";
// 本模块领域的能力处理器(规格一律取 contracts):catalog.lookup / release.search / release.inspect / identity.resolve。
// identity.resolve 属 propose 类,授权回调由 W2-D 接入前会被注册中心拒绝(安全默认)。
export function registerSourcesCapabilities(registry: CapabilityRegistry, rt: () => SourcesRuntime): void {
registry.register(capabilitySpec("catalog.lookup"), async (input) => {
const { db } = rt().ctx;
const local: { entity: unknown; score: number; matchedAlias: string | null }[] = [];
if (input.externalId) {
const rows = await db
.selectFrom("external_identity as i")
.innerJoin("media_entity as e", "e.id", "i.media_entity_id")
.selectAll("e")
.where("i.provider", "=", input.externalId.provider)
.where("i.entity_type", "=", input.externalId.entityType)
.where("i.external_id", "=", input.externalId.externalId)
.where("i.status", "=", "accepted")
.execute();
for (const e of rows) local.push({ entity: toApi(e), score: 1, matchedAlias: null });
}
if (input.query) {
const n = normalizeTitle(input.query);
const rows = await sql<Record<string, unknown> & { score: number; matched: string }>`
select distinct on (e.id) e.*, greatest(similarity(a.normalized, ${n}), case when a.normalized = ${n} then 1 else 0 end) as score,
a.alias as matched
from title_alias a join media_entity e on e.id = a.media_entity_id
where e.status <> 'merged' and (a.normalized = ${n} or similarity(a.normalized, ${n}) >= 0.3)
${input.kind ? sql`and e.kind = ${input.kind}` : sql``}
${input.year ? sql`and (e.year is null or abs(e.year - ${input.year}) <= 1)` : sql``}
order by e.id, score desc`.execute(db);
for (const r of rows.rows.sort((a, b) => b.score - a.score).slice(0, input.limit)) {
const { score, matched, ...e } = r;
if (!local.some((l) => (l.entity as { id: string }).id === e.id)) {
local.push({ entity: toApi(e), score: Number(score), matchedAlias: matched });
}
}
}
const remote: {
provider: string;
entityType: ExternalEntityType;
externalId: string;
title: string;
year: number | null;
}[] = [];
const tmdb = rt().tmdb();
if (input.includeRemote && input.query && tmdb) {
for (const m of (await tmdb.searchAll(input.query, input.year ? { year: input.year } : {})).slice(
0,
input.limit,
)) {
remote.push({
provider: "tmdb",
entityType: m.entityType,
externalId: m.externalId,
title: m.title,
year: m.year,
});
}
}
return { local: local as never, remote };
});
registry.register(capabilitySpec("release.search"), async (input) => {
const r = rt();
const { db } = r.ctx;
let q = db.selectFrom("source_release").selectAll();
if (input.siteIds?.length) q = q.where("site_id", "in", input.siteIds);
if (input.mediaEntityId) q = q.where("media_entity_id", "=", input.mediaEntityId);
const text = input.query;
if (text)
q = q.where((eb) => eb.or([eb("title", "ilike", `%${text}%`), sql<boolean>`similarity(title, ${text}) >= 0.3`]));
const run = () => q.orderBy("first_seen_at", "desc").limit(input.limit).execute();
let rows = await run();
const sites = await db
.selectFrom("site")
.selectAll()
.where("enabled", "=", true)
.$if(!!input.siteIds?.length, (b) => b.where("id", "in", input.siteIds!))
.execute();
const coverage: { siteId: string; fromSnapshot: boolean; live: boolean; error: string | null }[] = [];
const needLive = input.allowLiveSearch && !!text && rows.length === 0;
for (const s of sites) {
if (needLive) {
const res = await liveSearch(r, s, text!);
coverage.push({ siteId: s.id, fromSnapshot: false, live: res.error === null, error: res.error });
} else coverage.push({ siteId: s.id, fromSnapshot: true, live: false, error: null });
}
if (needLive) rows = await run();
return { items: toApiList<SourceRelease>(rows), coverage };
});
registry.register(capabilitySpec("release.inspect"), async (input) => {
const r = rt();
if (input.refreshDetail) await refreshReleaseDetail(r, input.sourceReleaseId);
const row = await r.ctx.db
.selectFrom("source_release")
.selectAll()
.where("id", "=", input.sourceReleaseId)
.executeTakeFirst();
if (!row) throw new Error("来源资源不存在");
// 库存(media_asset)归 W2-C,这里只按作品只读查询
const inventory = row.media_entity_id
? await r.ctx.db
.selectFrom("media_asset")
.selectAll()
.where("media_entity_id", "=", row.media_entity_id)
.execute()
: [];
return { release: toApi(row), inventory: toApiList(inventory) } as never;
});
registry.register(capabilitySpec("identity.resolve"), async (input, ctx) => {
const r = rt();
let srId = input.sourceReleaseId ?? null;
if (!srId && input.observationId) {
const o = await r.ctx.db
.selectFrom("observation")
.select(["source_release_id"])
.where("id", "=", input.observationId)
.executeTakeFirst();
srId = o?.source_release_id ?? null;
}
if (!srId) throw new Error("找不到来源资源");
const actor = { type: "agent" as const, id: ctx.actorId };
const c = input.candidate;
if (c?.mediaEntityId || (c?.provider && c.entityType && c.externalId)) {
// Agent 的提议不加人工锁,后续人工仍可改
const res = await assignReleaseIdentity(
r,
srId,
c.mediaEntityId
? { mediaEntityId: c.mediaEntityId, lock: false }
: {
external: {
provider: c.provider!,
entityType: c.entityType as "movie" | "tv",
externalId: c.externalId!,
},
lock: false,
},
actor,
);
return {
status: "resolved" as const,
mediaEntityId: res.mediaEntityId,
identities: await identities(r, res.mediaEntityId),
decisionId: null,
};
}
const res = await resolveRelease(r, { sourceReleaseId: srId, force: true });
const status: "resolved" | "proposed" | "ambiguous" | "unidentifiable" =
res.outcome === "auto" || res.outcome === "skipped" || res.outcome === "skipped_locked"
? res.mediaEntityId
? "resolved"
: "unidentifiable"
: res.outcome === "proposed"
? "proposed"
: res.outcome === "ambiguous"
? "ambiguous"
: "unidentifiable";
return {
status,
mediaEntityId: res.mediaEntityId,
identities: res.mediaEntityId ? await identities(r, res.mediaEntityId) : [],
decisionId: res.decisionId,
};
});
}
async function identities(r: SourcesRuntime, entityId: string) {
const rows = await r.ctx.db
.selectFrom("external_identity")
.selectAll()
.where("media_entity_id", "=", entityId)
.execute();
return toApiList(rows) as never[];
}
@@ -0,0 +1,303 @@
import {
ContentUnit,
ContentUnitUpdate,
CursorQuery,
cursorPage,
Decision,
DecisionKind,
DecisionResolveInput,
DecisionStatus,
ExternalIdentity,
ExternalIdentityCreate,
ExternalIdentityUpdate,
FeedHealth,
FeedSource,
FeedSourceCreate,
FeedSourceUpdate,
Id,
IdentityRevision,
IdentityStatus,
IdParam,
MediaEntity,
MediaEntityCreate,
MediaEntityMerge,
MediaEntityStatus,
MediaEntityUpdate,
MediaKind,
Observation,
ObservationStatus,
OkResult,
ResolutionStatus,
Site,
SiteAccount,
SiteAccountCreate,
SiteAccountUpdate,
SiteCreate,
SiteUpdate,
SourceRelease,
TitleAlias,
TitleAliasCreate,
} from "@umt/contracts";
import { z } from "zod";
// 与 contracts Api(crud())同路径、同 scope 的端点定义,但保留具体 schema 类型供路由推断入参。
// scope:读 = read;写 = admin(站点/Feed 配置)或 organize.execute(身份纠错、待处理)。
const list = <R extends z.ZodType, F extends z.ZodRawShape>(
path: string,
summary: string,
row: R,
filters: F,
tags: string[],
) => ({
method: "get" as const,
path,
scope: "read" as const,
summary,
tags,
query: CursorQuery.extend(filters),
response: cursorPage(row),
});
const get = <R extends z.ZodType>(path: string, summary: string, row: R, tags: string[]) => ({
method: "get" as const,
path,
scope: "read" as const,
summary,
tags,
params: IdParam,
response: row,
});
const write = <M extends "post" | "patch" | "delete", B extends z.ZodType | undefined, R extends z.ZodType>(
method: M,
path: string,
scope: "admin" | "organize.execute",
summary: string,
body: B,
response: R,
tags: string[],
withId = true,
) => ({
method,
path,
scope,
summary,
tags,
...(withId ? { params: IdParam } : {}),
body: body as B,
response,
});
const TS = ["sites"];
const TF = ["feed-sources"];
const TR = ["source-releases"];
const TE = ["media-entities"];
const TI = ["identity"];
const TD = ["decisions"];
export const D = {
sites: {
list: list("/sites", "站点列表", Site, {}, TS),
get: get("/sites/{id}", "站点详情", Site, TS),
create: {
method: "post" as const,
path: "/sites",
scope: "admin" as const,
summary: "登记站点",
tags: TS,
body: SiteCreate,
response: Site,
},
update: write("patch", "/sites/{id}", "admin", "修改站点(启停、快照周期、限流)", SiteUpdate, Site, TS),
},
siteAccounts: {
list: list("/site-accounts", "站点账号列表", SiteAccount, { siteId: Id.optional() }, TS),
get: get("/site-accounts/{id}", "站点账号详情", SiteAccount, TS),
create: {
method: "post" as const,
path: "/site-accounts",
scope: "admin" as const,
summary: "新建站点账号(凭证只写,存为 secret)",
tags: TS,
body: SiteAccountCreate,
response: SiteAccount,
},
update: write(
"patch",
"/site-accounts/{id}",
"admin",
"修改站点账号 / 轮换凭证",
SiteAccountUpdate,
SiteAccount,
TS,
),
},
feedSources: {
list: list("/feed-sources", "Feed 列表", FeedSource, { siteId: Id.optional(), health: FeedHealth.optional() }, TF),
get: get("/feed-sources/{id}", "Feed 详情", FeedSource, TF),
create: {
method: "post" as const,
path: "/feed-sources",
scope: "admin" as const,
summary: "新建 Feed(url 只写,存为 secret;对外只给脱敏 URL)",
tags: TF,
body: FeedSourceCreate,
response: FeedSource,
},
update: write("patch", "/feed-sources/{id}", "admin", "修改 Feed", FeedSourceUpdate, FeedSource, TF),
fetchNow: write(
"post",
"/feed-sources/{id}/fetch",
"admin",
"立即抓取(仍受站点限流)",
undefined,
z.object({ taskId: Id }),
TF,
),
},
sourceReleases: {
list: list(
"/source-releases",
"站点资源列表",
SourceRelease,
{
siteId: Id.optional(),
mediaEntityId: Id.optional(),
resolutionStatus: ResolutionStatus.optional(),
q: z.string().max(200).optional(),
},
TR,
),
get: get("/source-releases/{id}", "站点资源详情", SourceRelease, TR),
},
observations: {
list: list(
"/observations",
"观察列表",
Observation,
{ feedId: Id.optional(), status: ObservationStatus.optional() },
TR,
),
get: get("/observations/{id}", "观察详情", Observation, TR),
},
mediaEntities: {
list: list(
"/media-entities",
"作品列表",
MediaEntity,
{ kind: MediaKind.optional(), status: MediaEntityStatus.optional(), q: z.string().max(200).optional() },
TE,
),
get: get("/media-entities/{id}", "作品(旧 ID 自动跳转)", MediaEntity, TE),
create: {
method: "post" as const,
path: "/media-entities",
scope: "organize.execute" as const,
summary: "手动建本地作品(TMDB 无记录时可先建档)",
tags: TE,
body: MediaEntityCreate,
response: MediaEntity,
},
update: write(
"patch",
"/media-entities/{id}",
"organize.execute",
"修改作品(标题/原名/年份按字段覆盖并锁定)",
MediaEntityUpdate,
MediaEntity,
TE,
),
merge: write(
"post",
"/media-entities/{id}/merge",
"organize.execute",
"合并重复作品(保留旧 ID 跳转)",
MediaEntityMerge,
MediaEntity,
TE,
),
},
contentUnits: {
list: list(
"/content-units",
"内容单元列表",
ContentUnit,
{ mediaEntityId: Id.optional(), seasonNumber: z.coerce.number().int().optional() },
TE,
),
get: get("/content-units/{id}", "内容单元详情", ContentUnit, TE),
update: write(
"patch",
"/content-units/{id}",
"organize.execute",
"修改内容单元(改编号即确认并锁定)",
ContentUnitUpdate,
ContentUnit,
TE,
),
},
titleAliases: {
list: list(
"/title-aliases",
"别名列表",
TitleAlias,
{ mediaEntityId: Id.optional(), q: z.string().max(200).optional() },
TI,
),
create: write("post", "/title-aliases", "organize.execute", "新增别名", TitleAliasCreate, TitleAlias, TI, false),
remove: write("delete", "/title-aliases/{id}", "organize.execute", "删除别名", undefined, OkResult, TI),
},
externalIdentities: {
list: list(
"/external-identities",
"外部身份列表",
ExternalIdentity,
{ mediaEntityId: Id.optional(), provider: z.string().optional(), status: IdentityStatus.optional() },
TI,
),
get: get("/external-identities/{id}", "外部身份详情", ExternalIdentity, TI),
create: write(
"post",
"/external-identities",
"organize.execute",
"新增外部身份(accepted 等同人工纠错)",
ExternalIdentityCreate,
ExternalIdentity,
TI,
false,
),
update: write(
"patch",
"/external-identities/{id}",
"organize.execute",
"修订外部身份(乐观并发)",
ExternalIdentityUpdate,
ExternalIdentity,
TI,
),
revisions: get(
"/external-identities/{id}/revisions",
"身份修订历史",
z.object({ items: z.array(IdentityRevision) }),
TI,
),
},
decisions: {
list: list(
"/decisions",
"待处理事项(缺省只列未关闭)",
Decision,
{ status: DecisionStatus.optional(), kind: DecisionKind.optional() },
TD,
),
get: get("/decisions/{id}", "待处理事项详情", Decision, TD),
resolve: write(
"post",
"/decisions/{id}/resolve",
"organize.execute",
"处理待处理事项",
DecisionResolveInput,
Decision,
TD,
),
},
};
@@ -0,0 +1,69 @@
import { createRoute, type RouteConfig } from "@hono/zod-openapi";
import { type ApiScope, apiResponse } from "@umt/contracts";
import type { Context } from "hono";
import type { z } from "zod";
import { type AuthEnv, requireScope } from "../../http/auth.ts";
import type { ModuleApp } from "../types.ts";
import type { Actor } from "./service/common.ts";
// 本模块路由封装:createRoute(进 OpenAPI)+ requireScope + 入参校验 + 出参按合同 parse。
export interface RouteDef {
method: "get" | "post" | "patch" | "put" | "delete";
path: string;
scope: ApiScope;
summary: string;
tags: string[];
params?: z.ZodObject;
query?: z.ZodType;
body?: z.ZodType;
response: z.ZodType;
}
type Infer<T> = T extends z.ZodType ? z.infer<T> : Record<string, never>;
export interface Input<D extends RouteDef> {
params: Infer<D["params"]>;
query: Infer<D["query"]>;
body: Infer<D["body"]>;
actor: Actor;
c: Context<AuthEnv>;
}
/** 调用者(W2-A 的 c.var.actor:admin 会话 / API Key)→ 审计/修订用的操作者;管理员记为 user */
export function actorOf(c: Context<AuthEnv>): Actor {
const a = c.get("actor") as { type?: string; id?: string } | undefined;
if (!a) return { type: "system", id: null };
return { type: a.type === "api_key" ? "api_key" : "user", id: a.id ?? null };
}
export function route<D extends RouteDef>(app: ModuleApp, def: D, handler: (input: Input<D>) => Promise<unknown>) {
const request: RouteConfig["request"] = {};
if (def.params) request.params = def.params as never;
if (def.query) request.query = def.query as never;
if (def.body) request.body = { content: { "application/json": { schema: def.body as never } }, required: true };
const r = createRoute({
method: def.method,
path: def.path,
summary: def.summary,
tags: def.tags,
middleware: [requireScope(def.scope)] as never,
request,
responses: {
200: { description: def.summary, content: { "application/json": { schema: apiResponse(def.response) } } },
},
});
// biome-ignore lint/suspicious/noExplicitAny: 路由泛型在封装里擦除,入参已由 zod-openapi 校验
app.openapi(r, (async (c: any) => {
const input = {
params: def.params ? c.req.valid("param") : {},
query: def.query ? c.req.valid("query") : {},
body: def.body ? c.req.valid("json") : {},
actor: actorOf(c),
c,
} as Input<D>;
const data = def.response.parse(await handler(input));
return c.json({ ok: true, data }, 200);
// biome-ignore lint/suspicious/noExplicitAny: 同上
}) as any);
}
@@ -1,12 +1,69 @@
import type { ServerModule } from "../types.ts";
import type { ServerContext, ServerModule } from "../types.ts";
import { registerSourcesCapabilities } from "./capabilities.ts";
import { decisionRoutes } from "./routes/decisions.ts";
import { identityRoutes } from "./routes/identity.ts";
import { sourcesRoutes } from "./routes/sources.ts";
import { workbenchRoutes } from "./routes/workbench.ts";
import { type RuntimeOverrides, SourcesRuntime } from "./runtime.ts";
import { schedulerTick } from "./scheduler.ts";
import { registerSourcesTasks, sourcesOutboxRoutes } from "./tasks.ts";
/**
* W2-B 来源/身份/元数据:site、site_account、feed_source;ingestFeed / resolveObservation / refreshMetadata;
* W2-B 来源/身份/元数据:site、site_account、feed_source;ingestFeed / resolveObservation / refreshMetadata / snapshotSite;
* observation/source_release 写入与去重;身份解析、待识别池与 decision;站点工作台 API。
* 发出 release.resolved、decision.created(经 ctx.outbox.emit)。
* 发出 feed.due、feed.gap_suspected、release.observed(自消费)与 release.resolved、decision.created/updated。
*/
export const sourcesIdentityModule: ServerModule = {
name: "sources-identity",
registerRoutes() {},
registerTasks() {},
};
export function createSourcesIdentityModule(overrides: RuntimeOverrides = {}): ServerModule & {
runtime(ctx: ServerContext): SourcesRuntime;
} {
const runtimes = new WeakMap<ServerContext, SourcesRuntime>();
const runtime = (ctx: ServerContext) => {
let rt = runtimes.get(ctx);
if (!rt) {
rt = new SourcesRuntime(ctx, overrides);
runtimes.set(ctx, rt);
}
return rt;
};
let timer: ReturnType<typeof setInterval> | null = null;
let ticking: Promise<unknown> | null = null;
return {
name: "sources-identity",
runtime,
registerRoutes(app, ctx) {
const rt = () => runtime(ctx);
sourcesRoutes(app, rt);
workbenchRoutes(app, rt);
identityRoutes(app, rt);
decisionRoutes(app, rt);
},
registerTasks(registry, ctx) {
registerSourcesTasks(registry, () => runtime(ctx));
},
outboxRoutes: () => sourcesOutboxRoutes,
registerCapabilities(registry, ctx) {
registerSourcesCapabilities(registry, () => runtime(ctx));
},
start(ctx) {
const rt = runtime(ctx);
const tick = () => {
if (ticking) return;
ticking = schedulerTick(rt)
.catch((err) => ctx.logger.warn({ err }, "sources-identity 调度失败"))
.finally(() => {
ticking = null;
});
};
timer = setInterval(tick, overrides.schedulerIntervalMs ?? 30_000);
tick();
},
async stop() {
if (timer) clearInterval(timer);
timer = null;
await ticking;
},
};
}
export const sourcesIdentityModule = createSourcesIdentityModule();
@@ -0,0 +1,184 @@
import { type Decision, DecisionKind, DecisionStatus, Id, type PoolCandidate } from "@umt/contracts";
import { sql } from "@umt/db";
import { z } from "zod";
import { ApiException } from "../../../http/errors.ts";
import type { ModuleApp } from "../../types.ts";
import { D } from "../defs.ts";
import { route } from "../http.ts";
import { decodeCursor, jsonb, page, toApi, toApiList } from "../rows.ts";
import type { SourcesRuntime } from "../runtime.ts";
import { audit } from "../service/common.ts";
import { assignReleaseIdentity, dismissPoolEntry } from "../service/identity-admin.ts";
// 待处理中心数据源:decision 列表/详情/聚合/处理。
// 身份歧义类由本模块直接执行所选候选;其它类型只记录处理结果并发 decision.updated,
// 需要执行能力的选项由 W2-D 的 resolveDecision 消费。
const OPEN = ["open", "investigating", "awaiting_user"] as const;
const T = ["decisions"];
const SummaryGroup = z.object({
kind: DecisionKind,
/** 关联作品(身份歧义等尚未归属作品的为 null) */
mediaEntityId: Id.nullable(),
mediaEntityTitle: z.string().nullable(),
count: z.number().int(),
maxSeverity: z.enum(["info", "warning", "critical"]),
latestAt: z.string(),
});
export function decisionRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
const db = () => rt().ctx.db;
route(app, D.decisions.list, async ({ query }) => {
const off = decodeCursor(query.cursor);
let q = db().selectFrom("decision").selectAll();
q = query.status ? q.where("status", "=", query.status) : q.where("status", "in", [...OPEN]);
if (query.kind) q = q.where("kind", "=", query.kind);
const rows = await q
.orderBy(sql`case severity when 'critical' then 0 when 'warning' then 1 else 2 end`)
.orderBy("created_at", "desc")
.orderBy("id")
.offset(off)
.limit(query.limit + 1)
.execute();
const p = page(rows, off, query.limit);
return { items: toApiList<Decision>(p.items), nextCursor: p.nextCursor };
});
route(
app,
{
method: "get",
path: "/decisions/summary",
scope: "read",
summary: "未关闭待处理事项按作品与问题类型聚合",
tags: T,
query: z.object({ status: DecisionStatus.optional() }),
response: z.object({ total: z.number().int(), groups: z.array(SummaryGroup) }),
},
async ({ query }) => {
const statuses = query.status ? [query.status] : [...OPEN];
// 作品归属:evidence.mediaEntityId,或 subject 为 media_entity,或 subject 为已识别的 source_release
const rows = await sql<{
kind: z.infer<typeof DecisionKind>;
entity_id: string | null;
entity_title: string | null;
count: number;
max_sev: number;
latest: Date;
}>`
with d as (
select d.*, coalesce(
case when d.evidence->>'mediaEntityId' ~ '^[0-9a-f-]{36}$' then (d.evidence->>'mediaEntityId')::uuid end,
case when d.subject_type = 'media_entity' then d.subject_id end,
(select r.media_entity_id from source_release r where d.subject_type = 'source_release' and r.id = d.subject_id)
) as entity_id
from decision d where d.status = any(${statuses})
)
select d.kind, d.entity_id, e.title as entity_title, count(*)::int as count,
max(case d.severity when 'critical' then 2 when 'warning' then 1 else 0 end)::int as max_sev,
max(d.created_at) as latest
from d left join media_entity e on e.id = d.entity_id
group by d.kind, d.entity_id, e.title
order by max_sev desc, latest desc`.execute(db());
const sev = ["info", "warning", "critical"] as const;
return {
total: rows.rows.reduce((a, r) => a + r.count, 0),
groups: rows.rows.map((r) => ({
kind: r.kind,
mediaEntityId: r.entity_id,
mediaEntityTitle: r.entity_title,
count: r.count,
maxSeverity: sev[r.max_sev] ?? "info",
latestAt: new Date(r.latest).toISOString(),
})),
};
},
);
route(app, D.decisions.get, async ({ params }) => {
const row = await db().selectFrom("decision").selectAll().where("id", "=", params.id).executeTakeFirst();
if (!row) throw new ApiException("not_found", "待处理事项不存在");
return toApi(row);
});
route(app, D.decisions.resolve, async ({ params, body, actor }) => {
const d = await db().selectFrom("decision").selectAll().where("id", "=", params.id).executeTakeFirst();
if (!d) throw new ApiException("not_found", "待处理事项不存在");
if (!(OPEN as readonly string[]).includes(d.status)) throw new ApiException("conflict", `事项已是 ${d.status}`);
if (!body.dismiss && !body.optionKey) throw new ApiException("bad_request", "需要 optionKey 或 dismiss");
const by = `${actor.type}:${actor.id ?? ""}`;
const options = (d.options ?? []) as { key: string; input?: Record<string, unknown> }[];
const opt = body.optionKey ? options.find((o) => o.key === body.optionKey) : undefined;
if (body.optionKey && !opt) throw new ApiException("bad_request", `未知选项 ${body.optionKey}`);
// 身份歧义:直接执行(选候选 = 人工指定并加锁;忽略 = 待识别池 dismissed)
if (d.kind === "identity_ambiguous" && d.subject_type === "source_release" && d.subject_id) {
if (body.dismiss || opt?.key === "dismiss") {
const pool = await db()
.selectFrom("identity_candidate")
.select(["id"])
.where("source_release_id", "=", d.subject_id)
.executeTakeFirst();
if (pool) await dismissPoolEntry(rt(), pool.id, actor, body.note ?? null);
} else if (opt?.input && typeof opt.input.candidateIndex === "number") {
const pool = await db()
.selectFrom("identity_candidate")
.select(["candidates"])
.where("source_release_id", "=", d.subject_id)
.executeTakeFirst();
if (!(pool?.candidates as PoolCandidate[] | undefined)?.[opt.input.candidateIndex]) {
throw new ApiException("conflict", "候选已变化,请刷新后重试");
}
await assignReleaseIdentity(
rt(),
d.subject_id,
{ candidateIndex: opt.input.candidateIndex, lock: true, reason: body.note ?? `decision ${d.id}` },
actor,
);
}
}
// 上面的执行路径已按 dedupeKey 关闭;其余类型在这里记录处理结果
const row = await db()
.transaction()
.execute(async (trx) => {
const cur = await trx
.selectFrom("decision")
.selectAll()
.where("id", "=", params.id)
.forUpdate()
.executeTakeFirstOrThrow();
if (!(OPEN as readonly string[]).includes(cur.status)) return cur;
const status = body.dismiss || opt?.key === "dismiss" ? "dismissed" : "resolved";
const r = await trx
.updateTable("decision")
.set({
status,
resolution: jsonb({ optionKey: body.optionKey ?? null, note: body.note ?? null, by }),
resolved_by: by,
resolved_at: new Date(),
})
.where("id", "=", params.id)
.returningAll()
.executeTakeFirstOrThrow();
await rt().ctx.outbox.emit(
trx,
"decision.updated",
{ decisionId: r.id, status },
{
idempotencyKey: `decision.updated:${r.id}:${status}`,
},
);
await audit(
trx,
actor,
"decision.resolved",
{ type: "decision", id: r.id },
{ optionKey: body.optionKey ?? null, status },
);
return r;
});
return toApi(row);
});
}
@@ -0,0 +1,597 @@
import { randomUUID } from "node:crypto";
import {
type ContentUnit,
cursorPage,
EntityIdentitySet,
EntityMetadataField,
type ExternalIdentity,
FieldOverrideInput,
Id,
IdentityCandidate,
IdentityPoolItem,
IdentityPoolQuery,
IdParam,
type MediaEntity,
MediaEntityDetail,
OkResult,
ReleaseIdentityAssign,
type TitleAlias,
} from "@umt/contracts";
import { sql } from "@umt/db";
import { normalizeTitle } from "@umt/domain";
import { z } from "zod";
import { ApiException } from "../../../http/errors.ts";
import type { ModuleApp } from "../../types.ts";
import { D } from "../defs.ts";
import { route } from "../http.ts";
import { decodeCursor, jsonb, page, toApi, toApiList } from "../rows.ts";
import type { SourcesRuntime } from "../runtime.ts";
import { audit } from "../service/common.ts";
import { resolveEntityId } from "../service/entities.ts";
import {
assignReleaseIdentity,
dismissPoolEntry,
mergeEntities,
setEntityIdentity,
updateExternalIdentity,
} from "../service/identity-admin.ts";
import { clearFieldOverride, setFieldOverride } from "../service/metadata.ts";
// 身份与元数据 API:作品(列表/详情/纠错/合并/字段覆盖)、内容单元、别名、外部身份、待识别池。
const T_ENT = ["media-entities"];
const T_ID = ["identity"];
const Accepted = z.object({ taskId: Id });
const FieldParam = z.object({ id: Id, field: EntityMetadataField });
export function identityRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
const db = () => rt().ctx.db;
// ── 作品 ──
route(app, D.mediaEntities.list, async ({ query }) => {
const off = decodeCursor(query.cursor);
let q = db().selectFrom("media_entity").selectAll();
if (query.kind) q = q.where("kind", "=", query.kind);
q = query.status ? q.where("status", "=", query.status) : q.where("status", "<>", "merged");
if (query.q) {
const n = normalizeTitle(query.q);
q = q.where((eb) =>
eb.or([
eb("title", "ilike", `%${query.q}%`),
eb.exists(
eb
.selectFrom("title_alias as a")
.select("a.id")
.whereRef("a.media_entity_id", "=", "media_entity.id")
.where(sql<boolean>`(a.normalized = ${n} or similarity(a.normalized, ${n}) >= 0.4)`),
),
]),
);
}
const rows = await q
.orderBy("updated_at", "desc")
.orderBy("id")
.offset(off)
.limit(query.limit + 1)
.execute();
const p = page(rows, off, query.limit);
return { items: toApiList<MediaEntity>(p.items), nextCursor: p.nextCursor };
});
route(app, D.mediaEntities.get, async ({ params }) => {
const id = await resolveEntityId(db(), params.id);
if (!id) throw new ApiException("not_found", "作品不存在");
return toApi(await db().selectFrom("media_entity").selectAll().where("id", "=", id).executeTakeFirstOrThrow());
});
route(
app,
{
method: "get",
path: "/media-entities/{id}/detail",
scope: "read",
summary: "作品详情:外部身份、别名、内容单元、字段来源与覆盖(旧 ID 自动跳转)",
tags: T_ENT,
params: IdParam,
response: MediaEntityDetail,
},
async ({ params }) => {
const id = await resolveEntityId(db(), params.id);
if (!id) throw new ApiException("not_found", "作品不存在");
const [entity, identities, aliases, units, fields, overrides] = await Promise.all([
db().selectFrom("media_entity").selectAll().where("id", "=", id).executeTakeFirstOrThrow(),
db()
.selectFrom("external_identity")
.selectAll()
.where("media_entity_id", "=", id)
.orderBy("created_at")
.execute(),
db().selectFrom("title_alias").selectAll().where("media_entity_id", "=", id).orderBy("created_at").execute(),
db()
.selectFrom("content_unit")
.selectAll()
.where("media_entity_id", "=", id)
.orderBy("season_number")
.orderBy("episode_number")
.execute(),
db().selectFrom("metadata_field").selectAll().where("media_entity_id", "=", id).orderBy("field").execute(),
db().selectFrom("field_override").selectAll().where("media_entity_id", "=", id).orderBy("field").execute(),
]);
return {
entity: toApi(entity),
redirectedFrom: id === params.id ? null : params.id,
identities: toApiList(identities),
aliases: toApiList(aliases),
contentUnits: toApiList(units),
metadataFields: toApiList(fields),
overrides: toApiList(overrides),
};
},
);
route(app, D.mediaEntities.create, async ({ body, actor }) => {
const row = await db()
.transaction()
.execute(async (trx) => {
const r = await trx
.insertInto("media_entity")
.values({
kind: body.kind,
title: body.title,
original_title: body.originalTitle ?? null,
year: body.year ?? null,
status: body.status === "merged" ? "active" : (body.status ?? "active"),
metadata: jsonb(body.metadata ?? {}),
field_locks: body.fieldLocks ?? [],
})
.returningAll()
.executeTakeFirstOrThrow();
await sql`insert into title_alias (media_entity_id, alias, normalized, kind, source, locked)
values (${r.id}, ${body.title}, ${normalizeTitle(body.title) || body.title}, 'translated', 'user', true)
on conflict do nothing`.execute(trx);
await audit(trx, actor, "media_entity.created", { type: "media_entity", id: r.id }, { title: body.title });
return r;
});
return toApi(row);
});
route(app, D.mediaEntities.update, async ({ params, body, actor }) => {
const id = await resolveEntityId(db(), params.id);
if (!id) throw new ApiException("not_found", "作品不存在");
// 标题/原名/年份按字段覆盖处理(加锁),其余字段直接更新
const locked: [z.infer<typeof EntityMetadataField>, unknown][] = [];
if (body.title !== undefined) locked.push(["title", body.title]);
if (body.originalTitle !== undefined) locked.push(["originalTitle", body.originalTitle]);
if (body.year !== undefined) locked.push(["year", body.year]);
for (const [field, value] of locked) await setFieldOverride(rt(), id, field, { value, lock: true }, actor);
const patch: Record<string, unknown> = {};
if (body.kind !== undefined) patch.kind = body.kind;
if (body.status !== undefined) patch.status = body.status;
if (Object.keys(patch).length) {
await db().updateTable("media_entity").set(patch).where("id", "=", id).execute();
await audit(db(), actor, "media_entity.updated", { type: "media_entity", id }, patch);
}
return toApi(await db().selectFrom("media_entity").selectAll().where("id", "=", id).executeTakeFirstOrThrow());
});
route(app, D.mediaEntities.merge, async ({ params, body, actor }) =>
toApi(await mergeEntities(rt(), params.id, body.intoId, body.reason ?? null, actor)),
);
route(
app,
{
method: "post",
path: "/media-entities/{id}/identities",
scope: "organize.execute",
summary: "人工纠错外部映射:旧映射退役(留修订),新映射 accepted + 人工锁",
tags: T_ID,
params: IdParam,
body: EntityIdentitySet,
response: z.object({
mediaEntityId: Id,
changed: z.boolean(),
identityId: Id.optional(),
retired: z.array(Id).optional(),
}),
},
async ({ params, body, actor }) => setEntityIdentity(rt(), params.id, body, actor),
);
route(
app,
{
method: "post",
path: "/media-entities/{id}/refresh",
scope: "organize.execute",
summary: "刷新元数据(字段级;用户锁优先,不动路径)",
tags: T_ENT,
params: IdParam,
response: Accepted,
},
async ({ params }) => {
const id = await resolveEntityId(db(), params.id);
if (!id) throw new ApiException("not_found", "作品不存在");
const r = await rt().ctx.engine.enqueue(
"refreshMetadata",
{ mediaEntityId: id },
{ idempotencyKey: `refreshMetadata:${id}:${Math.floor(Date.now() / 60_000)}`, traceId: randomUUID() },
);
return { taskId: r.runId };
},
);
const FieldResult = z.object({
mediaEntityId: Id,
changedFields: z.array(z.string()),
lockedFields: z.array(z.string()),
});
route(
app,
{
method: "put",
path: "/media-entities/{id}/fields/{field}",
scope: "organize.execute",
summary: "覆盖作品字段并锁定(自动刷新不再改写)",
tags: T_ENT,
params: FieldParam,
body: FieldOverrideInput,
response: FieldResult,
},
async ({ params, body, actor }) => setFieldOverride(rt(), params.id, params.field, body, actor),
);
route(
app,
{
method: "delete",
path: "/media-entities/{id}/fields/{field}",
scope: "organize.execute",
summary: "取消字段覆盖与锁定",
tags: T_ENT,
params: FieldParam,
response: FieldResult,
},
async ({ params, actor }) => clearFieldOverride(rt(), params.id, params.field, actor),
);
// ── 内容单元 ──
route(app, D.contentUnits.list, async ({ query }) => {
const off = decodeCursor(query.cursor);
let q = db().selectFrom("content_unit").selectAll();
if (query.mediaEntityId) q = q.where("media_entity_id", "=", query.mediaEntityId);
if (query.seasonNumber !== undefined) q = q.where("season_number", "=", query.seasonNumber);
const rows = await q
.orderBy("media_entity_id")
.orderBy("season_number")
.orderBy("episode_number")
.orderBy("id")
.offset(off)
.limit(query.limit + 1)
.execute();
const p = page(rows, off, query.limit);
return { items: toApiList<ContentUnit>(p.items), nextCursor: p.nextCursor };
});
route(app, D.contentUnits.get, async ({ params }) => {
const row = await db().selectFrom("content_unit").selectAll().where("id", "=", params.id).executeTakeFirst();
if (!row) throw new ApiException("not_found", "内容单元不存在");
return toApi(row);
});
route(app, D.contentUnits.update, async ({ params, body, actor }) => {
const unit = await db().selectFrom("content_unit").selectAll().where("id", "=", params.id).executeTakeFirst();
if (!unit) throw new ApiException("not_found", "内容单元不存在");
const numbering = ["seasonNumber", "episodeNumber", "sourceIssue", "airDate"] as const;
const touchesNumbering = numbering.some((k) => body[k] !== undefined);
const patch: Record<string, unknown> = {};
const map: Record<string, string> = {
parentUnitId: "parent_unit_id",
kind: "kind",
contentType: "content_type",
segment: "segment",
seasonNumber: "season_number",
episodeNumber: "episode_number",
sourceIssue: "source_issue",
airDate: "air_date",
displayOrder: "display_order",
title: "title",
status: "status",
numberingLocked: "numbering_locked",
fieldLocks: "field_locks",
};
for (const [k, col] of Object.entries(map)) {
const v = (body as Record<string, unknown>)[k];
if (v !== undefined) patch[col] = v;
}
if (body.metadata !== undefined) patch.metadata = jsonb(body.metadata);
// 人工改编号即确认并锁定编号(REQ §5.4:编号可修订,人工确认后刷新不得改)
if (touchesNumbering && body.numberingLocked === undefined) patch.numbering_locked = true;
if (touchesNumbering && body.status === undefined && unit.status === "provisional") patch.status = "confirmed";
const row = await db()
.updateTable("content_unit")
.set(patch)
.where("id", "=", params.id)
.returningAll()
.executeTakeFirstOrThrow();
await audit(
db(),
actor,
"content_unit.updated",
{ type: "content_unit", id: params.id },
{ fields: Object.keys(patch) },
);
return toApi(row);
});
// ── 别名 ──
route(app, D.titleAliases.list, async ({ query }) => {
const off = decodeCursor(query.cursor);
let q = db().selectFrom("title_alias").selectAll();
if (query.mediaEntityId) q = q.where("media_entity_id", "=", query.mediaEntityId);
if (query.q) {
const n = normalizeTitle(query.q);
q = q.where(sql<boolean>`(normalized = ${n} or similarity(normalized, ${n}) >= 0.3)`);
}
const rows = await q
.orderBy("created_at")
.orderBy("id")
.offset(off)
.limit(query.limit + 1)
.execute();
const p = page(rows, off, query.limit);
return { items: toApiList<TitleAlias>(p.items), nextCursor: p.nextCursor };
});
route(app, D.titleAliases.create, async ({ body, actor }) => {
const id = await resolveEntityId(db(), body.mediaEntityId);
if (!id) throw new ApiException("not_found", "作品不存在");
const normalized = normalizeTitle(body.alias);
if (!normalized) throw new ApiException("bad_request", "别名归一化后为空");
try {
const row = await db()
.insertInto("title_alias")
.values({
media_entity_id: id,
content_unit_id: body.contentUnitId ?? null,
alias: body.alias,
normalized,
language: body.language ?? null,
kind: body.kind ?? "other",
source: body.source,
locked: body.locked ?? body.source === "user",
})
.returningAll()
.executeTakeFirstOrThrow();
await audit(db(), actor, "title_alias.created", { type: "title_alias", id: row.id }, { alias: body.alias });
return toApi(row);
} catch (e) {
if ((e as { code?: string }).code === "23505") throw new ApiException("conflict", "别名已存在");
throw e;
}
});
route(app, D.titleAliases.remove, async ({ params, actor }) => {
const r = await db().deleteFrom("title_alias").where("id", "=", params.id).returning(["id"]).executeTakeFirst();
if (!r) throw new ApiException("not_found", "别名不存在");
await audit(db(), actor, "title_alias.deleted", { type: "title_alias", id: params.id });
return OkResult.parse({ ok: true });
});
// ── 外部身份 ──
route(app, D.externalIdentities.list, async ({ query }) => {
const off = decodeCursor(query.cursor);
let q = db().selectFrom("external_identity").selectAll();
if (query.mediaEntityId) q = q.where("media_entity_id", "=", query.mediaEntityId);
if (query.provider) q = q.where("provider", "=", query.provider);
if (query.status) q = q.where("status", "=", query.status);
const rows = await q
.orderBy("created_at")
.orderBy("id")
.offset(off)
.limit(query.limit + 1)
.execute();
const p = page(rows, off, query.limit);
return { items: toApiList<ExternalIdentity>(p.items), nextCursor: p.nextCursor };
});
route(app, D.externalIdentities.get, async ({ params }) => {
const row = await db().selectFrom("external_identity").selectAll().where("id", "=", params.id).executeTakeFirst();
if (!row) throw new ApiException("not_found", "外部身份不存在");
return toApi(row);
});
route(app, D.externalIdentities.create, async ({ body, actor }) => {
if (!body.mediaEntityId) throw new ApiException("bad_request", "v0.1 只支持作品级外部身份");
const entityId = await resolveEntityId(db(), body.mediaEntityId);
if (!entityId) throw new ApiException("not_found", "作品不存在");
if ((body.status ?? "proposed") === "accepted") {
const r = await setEntityIdentity(
rt(),
entityId,
{
provider: body.provider,
entityType: body.entityType as "movie" | "tv",
externalId: body.externalId,
lock: body.locked ?? true,
},
actor,
);
if (!r.identityId) throw new ApiException("conflict", "该外部身份已是此作品的 accepted 映射");
return toApi(
await db().selectFrom("external_identity").selectAll().where("id", "=", r.identityId).executeTakeFirstOrThrow(),
);
}
const row = await db()
.transaction()
.execute(async (trx) => {
const r = await trx
.insertInto("external_identity")
.values({
media_entity_id: entityId,
provider: body.provider,
entity_type: body.entityType,
external_id: body.externalId,
status: body.status ?? "proposed",
evidence_level: body.evidenceLevel ?? "manual",
evidence: jsonb(body.evidence ?? {}),
locked: body.locked ?? false,
extra: jsonb(body.extra ?? {}),
})
.returningAll()
.executeTakeFirstOrThrow();
await trx
.insertInto("identity_revision")
.values({
external_identity_id: r.id,
revision: 1,
action: "created",
to_status: r.status,
after: jsonb({ status: r.status }),
actor_type: actor.type,
actor_id: actor.id,
})
.execute();
return r;
});
return toApi(row);
});
route(app, D.externalIdentities.update, async ({ params, body, actor }) =>
toApi(await updateExternalIdentity(rt(), params.id, body, actor)),
);
route(app, D.externalIdentities.revisions, async ({ params }) => {
const rows = await db()
.selectFrom("identity_revision")
.selectAll()
.where("external_identity_id", "=", params.id)
.orderBy("revision")
.execute();
return { items: toApiList(rows) };
});
// ── 待识别池 ──
route(
app,
{
method: "get",
path: "/identity-pool",
scope: "read",
summary: "待识别池(可按原始标题检索;候选不建正式作品)",
tags: T_ID,
query: IdentityPoolQuery,
response: cursorPage(IdentityPoolItem),
},
async ({ query }) => {
const off = decodeCursor(query.cursor);
let q = db()
.selectFrom("identity_candidate as p")
.innerJoin("source_release as r", "r.id", "p.source_release_id")
.selectAll("p")
.select([
"r.site_id as r_site_id",
"r.site_release_id as r_site_release_id",
"r.title as r_title",
"r.subtitle as r_subtitle",
"r.category_raw as r_category_raw",
"r.first_seen_at as r_first_seen_at",
"r.resolution_status as r_resolution_status",
]);
q = q.where("p.status", "=", query.status ?? "pending");
if (query.reason) q = q.where("p.reason", "=", query.reason);
if (query.siteId) q = q.where("r.site_id", "=", query.siteId);
if (query.q) {
const n = normalizeTitle(query.q);
q = q.where((eb) =>
eb.or([
eb("r.title", "ilike", `%${query.q}%`),
eb("r.subtitle", "ilike", `%${query.q}%`),
sql<boolean>`(p.query->>'normalized') = ${n}`,
sql<boolean>`similarity(r.title, ${query.q}) >= 0.3`,
]),
);
}
const rows = await q
.orderBy("p.updated_at", "desc")
.orderBy("p.id")
.offset(off)
.limit(query.limit + 1)
.execute();
const p = page(rows, off, query.limit);
return {
items: p.items.map((row) => {
const {
r_site_id,
r_site_release_id,
r_title,
r_subtitle,
r_category_raw,
r_first_seen_at,
r_resolution_status,
...rest
} = row;
return {
...toApi<Record<string, unknown>>(rest),
release: {
id: rest.source_release_id,
siteId: r_site_id,
siteReleaseId: r_site_release_id,
title: r_title,
subtitle: r_subtitle,
categoryRaw: r_category_raw,
firstSeenAt: r_first_seen_at.toISOString(),
resolutionStatus: r_resolution_status,
},
};
}),
nextCursor: p.nextCursor,
};
},
);
route(
app,
{
method: "post",
path: "/identity-pool/{id}/assign",
scope: "organize.execute",
summary: "为待识别资源手动指定候选 / 本地作品 / 外部身份",
tags: T_ID,
params: IdParam,
body: ReleaseIdentityAssign,
response: z.object({ mediaEntityId: Id, contentUnitIds: z.array(Id) }),
},
async ({ params, body, actor }) => {
const pool = await db()
.selectFrom("identity_candidate")
.select(["source_release_id"])
.where("id", "=", params.id)
.executeTakeFirst();
if (!pool) throw new ApiException("not_found", "待识别条目不存在");
return assignReleaseIdentity(rt(), pool.source_release_id, body, actor);
},
);
route(
app,
{
method: "post",
path: "/identity-pool/{id}/dismiss",
scope: "organize.execute",
summary: "忽略待识别资源(不再自动复查)",
tags: T_ID,
params: IdParam,
body: z.object({ note: z.string().max(2000).optional() }),
response: IdentityCandidate,
},
async ({ params, body, actor }) => toApi(await dismissPoolEntry(rt(), params.id, actor, body.note ?? null)),
);
route(
app,
{
method: "post",
path: "/identity-pool/{id}/retry",
scope: "organize.execute",
summary: "立即重新识别",
tags: T_ID,
params: IdParam,
response: Accepted,
},
async ({ params }) => {
const pool = await db()
.selectFrom("identity_candidate")
.select(["source_release_id", "attempts"])
.where("id", "=", params.id)
.executeTakeFirst();
if (!pool) throw new ApiException("not_found", "待识别条目不存在");
const r = await rt().ctx.engine.enqueue(
"resolveObservation",
{ sourceReleaseId: pool.source_release_id, force: true },
{ idempotencyKey: `resolve:retry:${params.id}:${Math.floor(Date.now() / 10_000)}`, traceId: randomUUID() },
);
return { taskId: r.runId };
},
);
}
@@ -0,0 +1,635 @@
import { randomUUID } from "node:crypto";
import { listSiteDefinitions, redactUrl, type SiteDefinition } from "@umt/adapters";
import {
CursorQuery,
cursorPage,
FeedSource,
Id,
IdParam,
type Observation,
ReleaseIdentityAssign,
type SecretPurpose,
Site,
type SiteAccount,
type SiteAdapter,
SiteDefinitionView,
SiteSnapshot,
type SourceRelease,
} from "@umt/contracts";
import { sql } from "@umt/db";
import { z } from "zod";
import { ApiException } from "../../../http/errors.ts";
import type { ModuleApp } from "../../types.ts";
import { D } from "../defs.ts";
import { route } from "../http.ts";
import { decodeCursor, jsonb, page, toApi, toApiList } from "../rows.ts";
import type { SourcesRuntime } from "../runtime.ts";
import { audit, closeDecisions } from "../service/common.ts";
import { assignReleaseIdentity } from "../service/identity-admin.ts";
// 站点、站点账号、Feed、来源资源、观察的管理 API。凭证只写(存 secret,对外只给 secret_ref / 脱敏 URL)。
const T_SITES = ["sites"];
const T_FEEDS = ["feed-sources"];
const T_RELEASES = ["source-releases"];
export function adapterOf(def: SiteDefinition): SiteAdapter {
return def.umt.kind === "mteam" ? "mteam_api" : def.umt.kind === "api" ? "api" : "nexusphp";
}
const SECRET_PURPOSE: Record<string, SecretPurpose> = {
cookie: "site_cookie",
passkey: "site_passkey",
api_key: "site_api_key",
password: "site_password",
};
const Accepted = z.object({ taskId: Id });
export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
const db = () => rt().ctx.db;
// ── 站点注册表 ──
route(
app,
{
method: "get",
path: "/site-definitions",
scope: "read",
summary: "随产品发布的站点适配器定义(含是否已登记)",
tags: T_SITES,
response: z.object({ items: z.array(SiteDefinitionView) }),
},
async () => {
const rows = await db().selectFrom("site").select(["id", "key"]).execute();
const byKey = new Map(rows.map((r) => [r.key, r.id]));
return {
items: listSiteDefinitions().map((d) => ({
key: d.id,
name: d.umt.kind === "nexusphp" ? (d.umt.displayName ?? d.name) : d.umt.displayName,
adapter: adapterOf(d),
baseUrl: d.domain,
shippedEnabled: d.enabled !== false,
definitionVersion: d.umt.version,
capabilities: d.umt.capabilities,
defaultRateLimit: d.umt.rateLimit,
authSupported: ("authSupported" in d.umt ? d.umt.authSupported : undefined) ?? [],
siteId: byKey.get(d.id) ?? null,
})),
};
},
);
// ── 站点 ──
route(app, D.sites.list, async ({ query }) => {
const off = decodeCursor(query.cursor);
const rows = await db()
.selectFrom("site")
.selectAll()
.orderBy("key")
.offset(off)
.limit(query.limit + 1)
.execute();
const p = page(rows, off, query.limit);
return { items: toApiList<Site>(p.items), nextCursor: p.nextCursor };
});
route(app, D.sites.get, async ({ params }) => toApi(await getSite(params.id)));
route(app, D.sites.create, async ({ body, actor }) => {
const def = listSiteDefinitions().find((d) => d.id === body.key);
if (def && adapterOf(def) !== body.adapter) {
throw new ApiException("bad_request", `站点 ${body.key} 的适配器类型应为 ${adapterOf(def)}`);
}
if (!def && body.adapter !== "generic_rss") {
throw new ApiException("bad_request", `注册表中没有站点 ${body.key};未适配站点只能以 generic_rss 登记`);
}
try {
const row = await db()
.insertInto("site")
.values({
key: body.key,
name: body.name,
adapter: body.adapter,
base_url: body.baseUrl,
enabled: body.enabled ?? true,
rate_limit: jsonb(body.rateLimit ?? {}),
config: jsonb(body.config ?? {}),
snapshot_enabled: body.snapshotEnabled ?? false,
snapshot_interval_sec: body.snapshotIntervalSec ?? null,
})
.returningAll()
.executeTakeFirstOrThrow();
await audit(db(), actor, "site.created", { type: "site", id: row.id }, { key: body.key });
return toApi(row);
} catch (e) {
if ((e as { code?: string }).code === "23505") throw new ApiException("conflict", `站点 ${body.key} 已登记`);
throw e;
}
});
route(app, D.sites.update, async ({ params, body, actor }) => {
await getSite(params.id);
const patch: Record<string, unknown> = {};
if (body.name !== undefined) patch.name = body.name;
if (body.adapter !== undefined) patch.adapter = body.adapter;
if (body.baseUrl !== undefined) patch.base_url = body.baseUrl;
if (body.rateLimit !== undefined) patch.rate_limit = jsonb(body.rateLimit);
if (body.config !== undefined) patch.config = jsonb(body.config);
if (body.snapshotEnabled !== undefined) patch.snapshot_enabled = body.snapshotEnabled;
if (body.snapshotIntervalSec !== undefined) patch.snapshot_interval_sec = body.snapshotIntervalSec;
if (body.enabled !== undefined) return toApi(await setSiteEnabled(params.id, body.enabled, actor));
const row = await db()
.updateTable("site")
.set(patch)
.where("id", "=", params.id)
.returningAll()
.executeTakeFirstOrThrow();
await audit(db(), actor, "site.updated", { type: "site", id: params.id }, { fields: Object.keys(patch) });
return toApi(row);
});
for (const enabled of [true, false]) {
route(
app,
{
method: "post",
path: `/sites/{id}/${enabled ? "enable" : "disable"}`,
scope: "admin",
summary: enabled ? "开启站点" : "关闭站点(停止网络访问与新候选,历史资源仍可查)",
tags: T_SITES,
params: IdParam,
response: Site,
},
async ({ params, actor }) => toApi(await setSiteEnabled(params.id, enabled, actor)),
);
}
route(
app,
{
method: "post",
path: "/sites/{id}/snapshot",
scope: "admin",
summary: "立即同步站点快照(受站点限流)",
tags: T_SITES,
params: IdParam,
body: z.object({ maxPages: z.number().int().min(1).max(20).optional() }),
response: Accepted,
},
async ({ params, body }) => {
const s = await getSite(params.id);
if (!s.enabled) throw new ApiException("precondition_failed", "站点已关闭");
const r = await rt().ctx.engine.enqueue(
"snapshotSite",
{ siteId: s.id, kind: "manual", ...(body.maxPages ? { maxPages: body.maxPages } : {}) },
{ idempotencyKey: `snapshotSite:manual:${s.id}:${Math.floor(Date.now() / 10_000)}`, traceId: randomUUID() },
);
return { taskId: r.runId };
},
);
route(
app,
{
method: "get",
path: "/sites/{id}/snapshots",
scope: "read",
summary: "站点快照/补查记录(覆盖范围与新鲜度)",
tags: T_SITES,
params: IdParam,
query: CursorQuery,
response: cursorPage(SiteSnapshot),
},
async ({ params, query }) => {
const off = decodeCursor(query.cursor);
const rows = await db()
.selectFrom("site_snapshot")
.selectAll()
.where("site_id", "=", params.id)
.orderBy("started_at", "desc")
.offset(off)
.limit(query.limit + 1)
.execute();
const p = page(rows, off, query.limit);
return { items: toApiList(p.items), nextCursor: p.nextCursor };
},
);
// ── 站点账号 ──
route(app, D.siteAccounts.list, async ({ query }) => {
const off = decodeCursor(query.cursor);
let q = db().selectFrom("site_account").selectAll();
if (query.siteId) q = q.where("site_id", "=", query.siteId);
const rows = await q
.orderBy("created_at")
.offset(off)
.limit(query.limit + 1)
.execute();
const p = page(rows, off, query.limit);
return { items: toApiList<SiteAccount>(p.items), nextCursor: p.nextCursor };
});
route(app, D.siteAccounts.get, async ({ params }) => {
const row = await db().selectFrom("site_account").selectAll().where("id", "=", params.id).executeTakeFirst();
if (!row) throw new ApiException("not_found", "站点账号不存在");
return toApi(row);
});
route(app, D.siteAccounts.create, async ({ body, actor }) => {
await getSite(body.siteId);
const ref = await rt().ctx.secrets.create({
purpose: SECRET_PURPOSE[body.authKind]!,
plaintext: body.credential,
label: `site_account:${body.label}`,
});
try {
const row = await db()
.insertInto("site_account")
.values({
site_id: body.siteId,
label: body.label,
auth_kind: body.authKind,
secret_ref: ref,
enabled: body.enabled ?? true,
})
.returningAll()
.executeTakeFirstOrThrow();
await audit(db(), actor, "site_account.created", { type: "site_account", id: row.id }, { siteId: body.siteId });
return toApi(row);
} catch (e) {
await rt()
.ctx.secrets.remove(ref)
.catch(() => {});
if ((e as { code?: string }).code === "23505") throw new ApiException("conflict", "同站点已有同名账号");
throw e;
}
});
route(app, D.siteAccounts.update, async ({ params, body, actor }) => {
const acc = await db().selectFrom("site_account").selectAll().where("id", "=", params.id).executeTakeFirst();
if (!acc) throw new ApiException("not_found", "站点账号不存在");
const patch: Record<string, unknown> = {};
if (body.label !== undefined) patch.label = body.label;
if (body.enabled !== undefined) patch.enabled = body.enabled;
if (body.authKind !== undefined) patch.auth_kind = body.authKind;
if (body.credential !== undefined) {
if (acc.secret_ref) await rt().ctx.secrets.replace(acc.secret_ref, body.credential);
else {
patch.secret_ref = await rt().ctx.secrets.create({
purpose: SECRET_PURPOSE[body.authKind ?? acc.auth_kind]!,
plaintext: body.credential,
label: `site_account:${acc.label}`,
});
}
// 换了凭证:状态回到未验证,关闭"登录失效"待处理,站点健康复位
patch.status = "unknown";
}
const row = await db()
.transaction()
.execute(async (trx) => {
const r = await trx
.updateTable("site_account")
.set(patch)
.where("id", "=", params.id)
.returningAll()
.executeTakeFirstOrThrow();
if (body.credential !== undefined) {
await trx
.updateTable("site")
.set({ health_status: "unknown" })
.where("id", "=", acc.site_id)
.where("health_status", "=", "failing")
.execute();
await closeDecisions(
rt().ctx,
trx,
`site-auth:${acc.site_id}`,
"resolved",
{ by: "credential_updated" },
`${actor.type}:${actor.id}`,
);
}
await audit(
trx,
actor,
"site_account.updated",
{ type: "site_account", id: params.id },
{
fields: Object.keys(patch).filter((k) => k !== "secret_ref"),
credentialRotated: body.credential !== undefined,
},
);
return r;
});
return toApi(row);
});
// ── Feed ──
route(app, D.feedSources.list, async ({ query }) => {
const off = decodeCursor(query.cursor);
let q = db().selectFrom("feed_source").selectAll();
if (query.siteId) q = q.where("site_id", "=", query.siteId);
if (query.health) q = q.where("health_status", "=", query.health);
const rows = await q
.orderBy("created_at")
.offset(off)
.limit(query.limit + 1)
.execute();
const p = page(rows, off, query.limit);
return { items: toApiList<FeedSource>(p.items), nextCursor: p.nextCursor };
});
route(app, D.feedSources.get, async ({ params }) => toApi(await getFeed(params.id)));
route(app, D.feedSources.create, async ({ body, actor }) => {
if (body.siteId) await getSite(body.siteId);
if (body.siteAccountId) await checkAccount(body.siteAccountId, body.siteId ?? null);
const ref = await rt().ctx.secrets.create({ purpose: "feed_url", plaintext: body.url, label: `feed:${body.name}` });
const row = await db()
.insertInto("feed_source")
.values({
site_id: body.siteId ?? null,
site_account_id: body.siteAccountId ?? null,
name: body.name,
purpose: body.purpose ?? "generic",
kind: body.kind ?? "rss",
url_secret_ref: ref,
url_display: redactUrl(body.url),
enabled: body.enabled ?? true,
poll_interval_sec: body.pollIntervalSec ?? 900,
filter: jsonb(body.filter ?? {}),
})
.returningAll()
.executeTakeFirstOrThrow();
await audit(db(), actor, "feed_source.created", { type: "feed_source", id: row.id }, { url: row.url_display });
return toApi(row);
});
route(app, D.feedSources.update, async ({ params, body, actor }) => {
const feed = await getFeed(params.id);
const patch: Record<string, unknown> = {};
if (body.siteId !== undefined) {
if (body.siteId) await getSite(body.siteId);
patch.site_id = body.siteId;
}
if (body.siteAccountId !== undefined) {
if (body.siteAccountId) await checkAccount(body.siteAccountId, body.siteId ?? feed.site_id);
patch.site_account_id = body.siteAccountId;
}
if (body.name !== undefined) patch.name = body.name;
if (body.purpose !== undefined) patch.purpose = body.purpose;
if (body.kind !== undefined) patch.kind = body.kind;
if (body.enabled !== undefined) {
patch.enabled = body.enabled;
patch.health_status = body.enabled ? "unknown" : "disabled";
}
if (body.pollIntervalSec !== undefined) patch.poll_interval_sec = body.pollIntervalSec;
if (body.filter !== undefined) patch.filter = jsonb(body.filter);
if (body.url !== undefined) {
if (feed.url_secret_ref) await rt().ctx.secrets.replace(feed.url_secret_ref, body.url);
else
patch.url_secret_ref = await rt().ctx.secrets.create({
purpose: "feed_url",
plaintext: body.url,
label: `feed:${feed.name}`,
});
patch.url_display = redactUrl(body.url);
// 换了链接:清空条件请求缓存(游标保留,用于去重)
patch.etag = null;
patch.last_modified = null;
}
const row = await db()
.updateTable("feed_source")
.set(patch)
.where("id", "=", params.id)
.returningAll()
.executeTakeFirstOrThrow();
await audit(
db(),
actor,
"feed_source.updated",
{ type: "feed_source", id: params.id },
{
fields: Object.keys(patch).filter((k) => k !== "url_secret_ref"),
},
);
return toApi(row);
});
for (const enabled of [true, false]) {
route(
app,
{
method: "post",
path: `/feed-sources/{id}/${enabled ? "enable" : "disable"}`,
scope: "admin",
summary: enabled ? "启用 Feed" : "停用 Feed",
tags: T_FEEDS,
params: IdParam,
response: FeedSource,
},
async ({ params, actor }) => {
await getFeed(params.id);
const row = await db()
.updateTable("feed_source")
.set({
enabled,
health_status: enabled ? "unknown" : "disabled",
...(enabled ? { next_fetch_at: new Date() } : {}),
})
.where("id", "=", params.id)
.returningAll()
.executeTakeFirstOrThrow();
await audit(db(), actor, enabled ? "feed_source.enabled" : "feed_source.disabled", {
type: "feed_source",
id: params.id,
});
return toApi(row);
},
);
}
route(
app,
{
method: "post",
path: "/feed-sources/{id}/reset",
scope: "admin",
summary: "手动恢复:关闭熔断、清空退避(凭证已更新后使用)",
tags: T_FEEDS,
params: IdParam,
response: FeedSource,
},
async ({ params, actor }) => {
const feed = await getFeed(params.id);
const cursor = (feed.cursor ?? {}) as { window?: unknown };
const row = await db()
.transaction()
.execute(async (trx) => {
const r = await trx
.updateTable("feed_source")
.set({
consecutive_failures: 0,
backoff_until: null,
last_error: null,
health_status: feed.enabled ? "unknown" : "disabled",
next_fetch_at: new Date(),
cursor: jsonb({ window: cursor.window ?? null }),
})
.where("id", "=", params.id)
.returningAll()
.executeTakeFirstOrThrow();
if (feed.site_account_id) {
await trx
.updateTable("site_account")
.set({ status: "unknown" })
.where("id", "=", feed.site_account_id)
.where("status", "in", ["invalid", "expired"])
.execute();
}
await closeDecisions(
rt().ctx,
trx,
`feed-auth:${params.id}`,
"resolved",
{ by: "reset" },
`${actor.type}:${actor.id}`,
);
await audit(trx, actor, "feed_source.reset", { type: "feed_source", id: params.id });
return r;
});
return toApi(row);
},
);
route(app, D.feedSources.fetchNow, async ({ params }) => {
const feed = await getFeed(params.id);
if (!feed.enabled) throw new ApiException("precondition_failed", "Feed 已停用");
const r = await rt().ctx.engine.enqueue(
"ingestFeed",
{ feedSourceId: feed.id, manual: true },
{ idempotencyKey: `ingestFeed:manual:${feed.id}:${Math.floor(Date.now() / 10_000)}`, traceId: randomUUID() },
);
return { taskId: r.runId };
});
// ── 来源资源 / 观察 ──
route(app, D.sourceReleases.list, async ({ query }) => {
const off = decodeCursor(query.cursor);
let q = db().selectFrom("source_release").selectAll();
if (query.siteId) q = q.where("site_id", "=", query.siteId);
if (query.mediaEntityId) q = q.where("media_entity_id", "=", query.mediaEntityId);
if (query.resolutionStatus) q = q.where("resolution_status", "=", query.resolutionStatus);
if (query.q)
q = q.where((eb) =>
eb.or([eb("title", "ilike", `%${query.q}%`), sql<boolean>`similarity(title, ${query.q}) >= 0.3`]),
);
const rows = await q
.orderBy("first_seen_at", "desc")
.orderBy("id")
.offset(off)
.limit(query.limit + 1)
.execute();
const p = page(rows, off, query.limit);
return { items: toApiList<SourceRelease>(p.items), nextCursor: p.nextCursor };
});
route(app, D.sourceReleases.get, async ({ params }) => {
const row = await db().selectFrom("source_release").selectAll().where("id", "=", params.id).executeTakeFirst();
if (!row) throw new ApiException("not_found", "来源资源不存在");
return toApi(row);
});
route(
app,
{
method: "post",
path: "/source-releases/{id}/identity",
scope: "organize.execute",
summary: "人工指定来源资源的作品/内容单元(加人工锁,自动识别不再改写)",
tags: T_RELEASES,
params: IdParam,
body: ReleaseIdentityAssign,
response: z.object({ mediaEntityId: Id, contentUnitIds: z.array(Id) }),
},
async ({ params, body, actor }) => assignReleaseIdentity(rt(), params.id, body, actor),
);
route(
app,
{
method: "post",
path: "/source-releases/{id}/unlock",
scope: "organize.execute",
summary: "解除来源资源的人工锁并重新识别",
tags: T_RELEASES,
params: IdParam,
response: Accepted,
},
async ({ params, actor }) => {
const row = await db()
.updateTable("source_release")
.set({ resolution_locked: false })
.where("id", "=", params.id)
.returning(["id", "site_id"])
.executeTakeFirst();
if (!row) throw new ApiException("not_found", "来源资源不存在");
await audit(db(), actor, "identity.release_unlocked", { type: "source_release", id: params.id });
const r = await rt().ctx.engine.enqueue(
"resolveObservation",
{ sourceReleaseId: params.id, force: true },
{ idempotencyKey: `resolve:unlock:${params.id}:${Date.now()}`, traceId: randomUUID() },
);
return { taskId: r.runId };
},
);
route(app, D.observations.list, async ({ query }) => {
const off = decodeCursor(query.cursor);
let q = db().selectFrom("observation").selectAll();
if (query.feedId) q = q.where("feed_id", "=", query.feedId);
if (query.status) q = q.where("status", "=", query.status);
const rows = await q
.orderBy("first_seen_at", "desc")
.orderBy("id")
.offset(off)
.limit(query.limit + 1)
.execute();
const p = page(rows, off, query.limit);
return { items: toApiList<Observation>(p.items), nextCursor: p.nextCursor };
});
route(app, D.observations.get, async ({ params }) => {
const row = await db().selectFrom("observation").selectAll().where("id", "=", params.id).executeTakeFirst();
if (!row) throw new ApiException("not_found", "观察不存在");
return toApi(row);
});
// ── 内部辅助 ──
async function getSite(id: string) {
const row = await db().selectFrom("site").selectAll().where("id", "=", id).executeTakeFirst();
if (!row) throw new ApiException("not_found", "站点不存在");
return row;
}
async function getFeed(id: string) {
const row = await db().selectFrom("feed_source").selectAll().where("id", "=", id).executeTakeFirst();
if (!row) throw new ApiException("not_found", "Feed 不存在");
return row;
}
async function checkAccount(accountId: string, siteId: string | null) {
const acc = await db()
.selectFrom("site_account")
.select(["site_id"])
.where("id", "=", accountId)
.executeTakeFirst();
if (!acc) throw new ApiException("bad_request", "站点账号不存在");
if (siteId && acc.site_id !== siteId) throw new ApiException("bad_request", "站点账号不属于该站点");
}
async function setSiteEnabled(
id: string,
enabled: boolean,
actor: { type: "user" | "agent" | "system" | "api_key"; id: string | null },
) {
await getSite(id);
return db()
.transaction()
.execute(async (trx) => {
const row = await trx
.updateTable("site")
.set({ enabled, health_status: enabled ? "unknown" : "disabled" })
.where("id", "=", id)
.returningAll()
.executeTakeFirstOrThrow();
// 历史资源仍可查,标注站点关闭
await trx
.updateTable("source_release")
.set({ availability: enabled ? "active" : "site_disabled" })
.where("site_id", "=", id)
.where("availability", "=", enabled ? "site_disabled" : "active")
.execute();
await audit(trx, actor, enabled ? "site.enabled" : "site.disabled", { type: "site", id });
return row;
});
}
}
@@ -0,0 +1,42 @@
import { Id, WorkbenchReleaseDetail, WorkbenchReleasePage, WorkbenchReleaseQuery } from "@umt/contracts";
import { z } from "zod";
import type { ModuleApp } from "../../types.ts";
import { route } from "../http.ts";
import type { SourcesRuntime } from "../runtime.ts";
import { listSiteReleases, releaseDetail } from "../service/workbench.ts";
// 站点工作台(整理视图)数据 API。库存叠加(已有资产/Slot/下载)不在这里:条目带 entity.id,
// 前端按作品 ID 二次查询 W2-C 的媒体库端点。
const T = ["site-workbench"];
export function workbenchRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
route(
app,
{
method: "get",
path: "/sites/{id}/releases",
scope: "read",
summary: "站点资源列表/搜索(快照优先,必要时实时补查;标注来源与更新时间)",
tags: T,
params: z.object({ id: Id }),
query: WorkbenchReleaseQuery,
response: WorkbenchReleasePage,
},
async ({ params, query }) => listSiteReleases(rt(), params.id, query),
);
route(
app,
{
method: "get",
path: "/sites/{id}/releases/{rid}",
scope: "read",
summary: "站点资源详情(refresh=true 时实时复核详情;失败保留既有数据)",
tags: T,
params: z.object({ id: Id, rid: Id }),
query: z.object({ refresh: z.enum(["true", "false"]).optional() }),
response: WorkbenchReleaseDetail,
},
async ({ params, query }) => releaseDetail(rt(), params.id, params.rid, query.refresh === "true"),
);
}
@@ -0,0 +1,37 @@
// db 行(snake_case、Date)→ contracts(camelCase、ISO 串)。只做命名与时间转换,取值语义不变。
const camel = (s: string) => s.replace(/_([a-z0-9])/g, (_, c: string) => c.toUpperCase());
function conv(v: unknown): unknown {
if (v instanceof Date) return v.toISOString();
return v;
}
/** 通用行转换:键转 camelCase,Date → ISO 串;jsonb/数组原样保留 */
export function toApi<T>(row: object): T {
const out: Record<string, unknown> = {};
for (const [k, v] of Object.entries(row)) out[camel(k)] = conv(v);
return out as T;
}
export const toApiList = <T>(rows: readonly object[]): T[] => rows.map((r) => toApi<T>(r));
/** 偏移游标(不透明 base64url);列表规模小,足够用 */
export function encodeCursor(offset: number): string {
return Buffer.from(String(offset), "utf8").toString("base64url");
}
export function decodeCursor(cursor: string | undefined): number {
if (!cursor) return 0;
const n = Number(Buffer.from(cursor, "base64url").toString("utf8"));
return Number.isSafeInteger(n) && n >= 0 ? n : 0;
}
/** 取 limit+1 条判断是否还有下一页 */
export function page<T>(rows: T[], offset: number, limit: number): { items: T[]; nextCursor: string | null } {
const more = rows.length > limit;
return { items: more ? rows.slice(0, limit) : rows, nextCursor: more ? encodeCursor(offset + limit) : null };
}
/** jsonb 写入:数组/对象都先 JSON.stringify(pg 会把 JS 数组当 PG 数组) */
export const jsonb = (v: unknown): string => JSON.stringify(v ?? null);
@@ -0,0 +1,143 @@
import {
createRssFetcher,
createSiteAdapter,
createSiteRateLimiter,
createTmdbClient,
getSiteDefinition,
type RssFetcher,
type SecretResolver,
type SiteAdapter,
type SiteDefinition,
type SiteRateLimiter,
type TmdbClient,
} from "@umt/adapters";
import type { SiteRow } from "@umt/db";
import type { ServerContext } from "../types.ts";
// 模块运行时依赖:站点适配器与限流器(每站一个实例)、RSS 抓取器、TMDB 客户端、时钟。
// 测试通过 createSourcesIdentityModule(overrides) 注入 fetch / TMDB / 时钟。
export interface RuntimeOverrides {
/** 站点与 RSS 请求使用的 fetch(测试指向 mock) */
fetch?: typeof globalThis.fetch;
/** TMDB 客户端工厂;返回 null 表示未配置(只做本地召回) */
tmdb?: (() => TmdbClient | null) | undefined;
now?: () => Date;
/** RSS 抓取器选项覆盖(退避/熔断阈值) */
rss?: Parameters<typeof createRssFetcher>[0];
/** 调度 tick 间隔(毫秒),默认 30s */
schedulerIntervalMs?: number;
}
interface CachedAdapter {
sig: string;
limiter: SiteRateLimiter;
adapter: SiteAdapter;
}
export class SourcesRuntime {
readonly now: () => Date;
readonly rss: RssFetcher;
private readonly adapters = new Map<string, CachedAdapter>();
private tmdbClient: TmdbClient | null | undefined;
constructor(
readonly ctx: ServerContext,
readonly overrides: RuntimeOverrides = {},
) {
this.now = overrides.now ?? (() => new Date());
this.rss = createRssFetcher({
...(overrides.fetch ? { fetch: overrides.fetch } : {}),
now: this.now,
...overrides.rss,
});
}
/** TMDB:优先注入;否则读环境变量 TMDB_API_KEY / TMDB_ACCESS_TOKEN(W2-A 设置页接管前的过渡) */
tmdb(): TmdbClient | null {
if (this.tmdbClient !== undefined) return this.tmdbClient;
if (this.overrides.tmdb) {
this.tmdbClient = this.overrides.tmdb();
return this.tmdbClient;
}
const env = (k: string) => Deno.env.get(k);
const apiKey = env("TMDB_API_KEY");
const accessToken = env("TMDB_ACCESS_TOKEN");
const baseUrl = env("TMDB_BASE_URL");
this.tmdbClient =
apiKey || accessToken
? createTmdbClient({
...(apiKey ? { apiKey } : {}),
...(accessToken ? { accessToken } : {}),
...(baseUrl ? { baseUrl } : {}),
...(this.overrides.fetch ? { fetch: this.overrides.fetch } : {}),
})
: null;
return this.tmdbClient;
}
/**
* 站点适配器:每站一个实例,列表/搜索/详情/下载共享同一个限流器。
* 站点限流参数或基址变化时重建(签名比较)。
*/
siteAdapter(site: Pick<SiteRow, "id" | "key" | "base_url" | "rate_limit" | "config">): SiteAdapter {
const def = getSiteDefinition(site.key);
if (!def) throw new Error(`站点 ${site.key} 没有随产品发布的适配器定义`);
const sig = JSON.stringify([site.base_url, site.rate_limit, site.config]);
const hit = this.adapters.get(site.id);
if (hit && hit.sig === sig) return hit.adapter;
const limiter = createSiteRateLimiter(limiterOptions(def, site.rate_limit));
const cfg = (site.config ?? {}) as { apiBaseUrl?: string };
const baseUrl =
def.umt.kind === "mteam" ? cfg.apiBaseUrl : sameUrl(site.base_url, def.domain) ? undefined : site.base_url;
const adapter = createSiteAdapter(def, {
secrets: this.siteSecrets(site.id),
limiter,
now: this.now,
...(this.overrides.fetch ? { fetch: this.overrides.fetch } : {}),
...(baseUrl ? { baseUrl } : {}),
});
this.adapters.set(site.id, { sig, limiter, adapter });
return adapter;
}
/** 已建立的站点限流器(测试与诊断用) */
limiterOf(siteId: string): SiteRateLimiter | null {
return this.adapters.get(siteId)?.limiter ?? null;
}
/** 站点凭证:取该站第一个启用且未失效的账号,按 auth_kind 对应适配器要求的凭证类型;明文不缓存 */
private siteSecrets(siteId: string): SecretResolver {
const { db, secrets } = this.ctx;
return {
async resolve({ kind }) {
const want = kind === "cookie" ? ["cookie"] : kind === "apiKey" ? ["api_key"] : ["passkey"];
const acc = await db
.selectFrom("site_account")
.select(["secret_ref"])
.where("site_id", "=", siteId)
.where("enabled", "=", true)
.where("status", "in", ["unknown", "active"])
.where("auth_kind", "in", want as never)
.where("secret_ref", "is not", null)
.orderBy("created_at")
.executeTakeFirst();
if (!acc?.secret_ref) return null;
return secrets.resolve(acc.secret_ref);
},
};
}
}
function sameUrl(a: string, b: string): boolean {
return a.replace(/\/+$/, "") === b.replace(/\/+$/, "");
}
/** 站点行 rate_limit 优先,缺省用定义里的 umt.rateLimit */
export function limiterOptions(def: SiteDefinition, rateLimit: unknown): { minIntervalMs: number; burst: number } {
const rl = (rateLimit ?? {}) as { requestsPerMinute?: number; minIntervalMs?: number; concurrency?: number };
const minIntervalMs =
rl.minIntervalMs ??
(rl.requestsPerMinute ? Math.ceil(60_000 / rl.requestsPerMinute) : def.umt.rateLimit.minIntervalMs);
return { minIntervalMs, burst: rl.concurrency ?? def.umt.rateLimit.burst };
}
@@ -0,0 +1,90 @@
import { randomUUID } from "node:crypto";
import { sql } from "@umt/db";
import type { SourcesRuntime } from "./runtime.ts";
// worker 后台调度:按 Feed 轮询策略发 feed.due(outbox → ingestFeed)、按站点快照周期入队 snapshotSite、
// 按待识别池 next_review_at 入队复查。幂等键按时间槽生成,多 worker 并发也不会重复。
export interface TickResult {
feedsDue: number;
snapshotsDue: number;
reviewsDue: number;
skipped?: string;
}
const REVIEW_LEASE_MS = 10 * 60_000;
export async function schedulerTick(rt: SourcesRuntime): Promise<TickResult> {
const { ctx } = rt;
const automation = await ctx.settings.get("automation");
// 恢复后只读对账模式:禁止自动写入
if (automation.readOnlyReconcile) return { feedsDue: 0, snapshotsDue: 0, reviewsDue: 0, skipped: "read_only" };
const now = rt.now();
// Feed:到期(next_fetch_at 空或已到)且不在退避期、站点(若有)开启
const feedsDue = await ctx.db.transaction().execute(async (trx) => {
const due = await sql<{ id: string; poll_interval_sec: number }>`
select f.id, f.poll_interval_sec from feed_source f left join site s on s.id = f.site_id
where f.enabled and (s.id is null or s.enabled)
and (f.next_fetch_at is null or f.next_fetch_at <= ${now})
and (f.backoff_until is null or f.backoff_until <= ${now})
and f.health_status <> 'circuit_open'
order by f.next_fetch_at nulls first limit 100
for update of f skip locked`.execute(trx);
for (const f of due.rows) {
const slot = Math.floor(now.getTime() / 1000 / f.poll_interval_sec);
await ctx.outbox.emit(
trx,
"feed.due",
{ feedId: f.id, reason: "schedule" },
{ idempotencyKey: `feed.due:${f.id}:${slot}` },
);
await trx
.updateTable("feed_source")
.set({ next_fetch_at: new Date(now.getTime() + f.poll_interval_sec * 1000) })
.where("id", "=", f.id)
.execute();
}
return due.rows.length;
});
// 站点快照:开启快照且到周期
const sites = await sql<{ id: string; snapshot_interval_sec: number }>`
select id, snapshot_interval_sec from site
where enabled and snapshot_enabled and snapshot_interval_sec is not null
and (last_snapshot_at is null or last_snapshot_at + make_interval(secs => snapshot_interval_sec) <= ${now})`.execute(
ctx.db,
);
for (const s of sites.rows) {
const slot = Math.floor(now.getTime() / 1000 / s.snapshot_interval_sec);
await ctx.engine.enqueue(
"snapshotSite",
{ siteId: s.id, kind: "periodic" },
{ idempotencyKey: `snapshotSite:${s.id}:${slot}`, traceId: randomUUID() },
);
}
// 待识别池复查:到期的先顺延一个租约,避免下个 tick 重复入队
const reviews = await ctx.db.transaction().execute(async (trx) => {
const rows = await sql<{ id: string; source_release_id: string; attempts: number }>`
select id, source_release_id, attempts from identity_candidate
where status = 'pending' and next_review_at is not null and next_review_at <= ${now}
order by next_review_at limit 50 for update skip locked`.execute(trx);
for (const r of rows.rows) {
await trx
.updateTable("identity_candidate")
.set({ next_review_at: new Date(now.getTime() + REVIEW_LEASE_MS) })
.where("id", "=", r.id)
.execute();
}
return rows.rows;
});
for (const r of reviews) {
await ctx.engine.enqueue(
"resolveObservation",
{ sourceReleaseId: r.source_release_id, force: true },
{ idempotencyKey: `review:${r.id}:${r.attempts}`, traceId: randomUUID() },
);
}
return { feedsDue, snapshotsDue: sites.rows.length, reviewsDue: reviews.length };
}
@@ -0,0 +1,109 @@
import type { ActorType, DecisionKind, DecisionOption, Severity } from "@umt/contracts";
import { type DB, type Kysely, sql, type Transaction } from "@umt/db";
import type { ServerContext } from "../../types.ts";
import { jsonb } from "../rows.ts";
export type Exec = Kysely<DB> | Transaction<DB>;
export interface Actor {
type: ActorType;
id: string | null;
}
export const SYSTEM: Actor = { type: "system", id: "sources-identity" };
/** 审计:只追加 */
export async function audit(
exec: Exec,
actor: Actor,
action: string,
target: { type: string; id: string | null },
payload: Record<string, unknown> = {},
): Promise<void> {
await exec
.insertInto("audit_event")
.values({
actor_type: actor.type,
actor_id: actor.id,
action,
target_type: target.type,
target_id: target.id,
payload: jsonb(payload),
})
.execute();
}
export interface DecisionInput {
kind: DecisionKind;
severity?: Severity;
title: string;
summary?: string | null;
subjectType: string;
subjectId: string | null;
dedupeKey: string;
evidence?: Record<string, unknown>;
options?: DecisionOption[];
nextReviewAt?: Date | null;
}
/**
* 开一个待处理事项;同 dedupeKey 已有未关闭条目时只刷新证据与选项(不重复通知)。
* 新建时同事务发 decision.created。
*/
export async function openDecision(
ctx: ServerContext,
trx: Exec,
d: DecisionInput,
): Promise<{ id: string; created: boolean }> {
const row = await sql<{ id: string; inserted: boolean }>`
insert into decision (kind, severity, title, summary, subject_type, subject_id, dedupe_key, evidence, options, next_review_at)
values (${d.kind}, ${d.severity ?? "warning"}, ${d.title}, ${d.summary ?? null}, ${d.subjectType}, ${d.subjectId},
${d.dedupeKey}, ${jsonb(d.evidence ?? {})}::jsonb, ${jsonb(d.options ?? [])}::jsonb, ${d.nextReviewAt ?? null})
on conflict (dedupe_key) where status in ('open', 'investigating', 'awaiting_user')
do update set evidence = excluded.evidence, options = excluded.options, summary = excluded.summary,
title = excluded.title, attempts = decision.attempts + 1
returning id, (xmax = 0) as inserted`.execute(trx);
const r = row.rows[0]!;
if (r.inserted) {
await ctx.outbox.emit(
trx,
"decision.created",
{ decisionId: r.id, kind: d.kind, severity: d.severity ?? "warning", title: d.title },
{ idempotencyKey: `decision.created:${r.id}` },
);
}
return { id: r.id, created: r.inserted };
}
/** 关闭同 dedupeKey 的未关闭事项(身份已被人工/自动确认等),逐条发 decision.updated */
export async function closeDecisions(
ctx: ServerContext,
trx: Exec,
dedupeKey: string,
status: "resolved" | "dismissed",
resolution: Record<string, unknown>,
by: string,
): Promise<string[]> {
const rows = await trx
.updateTable("decision")
.set({ status, resolution: jsonb(resolution), resolved_by: by, resolved_at: new Date() })
.where("dedupe_key", "=", dedupeKey)
.where("status", "in", ["open", "investigating", "awaiting_user"])
.returning(["id"])
.execute();
for (const r of rows) {
await ctx.outbox.emit(
trx,
"decision.updated",
{ decisionId: r.id, status },
{
idempotencyKey: `decision.updated:${r.id}:${status}`,
},
);
}
return rows.map((r) => r.id);
}
/** 截断长文本(描述等只留摘录) */
export const clip = (s: string | undefined | null, n = 2000): string | null =>
s == null ? null : s.length > n ? `${s.slice(0, n)}…` : s;
@@ -0,0 +1,221 @@
import type { MetadataCandidate } from "@umt/adapters";
import type { AliasKind, AliasSource, EvidenceLevel, MediaKind } from "@umt/contracts";
import { sql } from "@umt/db";
import { normalizeTitle, type ParsedRelease } from "@umt/domain";
import { toContractMediaKind } from "@umt/mapping";
import { jsonb } from "../rows.ts";
import type { Actor, Exec } from "./common.ts";
// 本地作品 / 内容单元 / 外部身份 / 别名的写入辅助。所有写入在调用方事务内完成。
/** 沿 merged_into_id 找到最终实体(防环,最多 10 跳);不存在返回 null */
export async function resolveEntityId(exec: Exec, id: string): Promise<string | null> {
let cur = id;
for (let i = 0; i < 10; i++) {
const row = await exec
.selectFrom("media_entity")
.select(["id", "status", "merged_into_id"])
.where("id", "=", cur)
.executeTakeFirst();
if (!row) return null;
if (row.status !== "merged" || !row.merged_into_id) return row.id;
cur = row.merged_into_id;
}
return cur;
}
/** 已接受的外部身份 → 本地作品(跟随合并跳转) */
export async function entityByExternal(
exec: Exec,
ref: { provider: string; entityType: string; externalId: string },
): Promise<string | null> {
const row = await exec
.selectFrom("external_identity")
.select(["media_entity_id"])
.where("provider", "=", ref.provider)
.where("entity_type", "=", ref.entityType as never)
.where("external_id", "=", ref.externalId)
.where("status", "=", "accepted")
.where("media_entity_id", "is not", null)
.executeTakeFirst();
return row?.media_entity_id ? resolveEntityId(exec, row.media_entity_id) : null;
}
export async function addAlias(
exec: Exec,
entityId: string,
alias: string,
kind: AliasKind,
source: AliasSource,
language: string | null = null,
locked = false,
): Promise<void> {
const normalized = normalizeTitle(alias);
if (!normalized) return;
await sql`insert into title_alias (media_entity_id, content_unit_id, alias, normalized, language, kind, source, locked)
values (${entityId}, null, ${alias}, ${normalized}, ${language}, ${kind}, ${source}, ${locked})
on conflict (media_entity_id, content_unit_id, normalized, kind) do nothing`.execute(exec);
}
/** 写一条外部身份(accepted)+ 修订 created;同一外部身份已被其它作品接受时返回 null(交给调用方处理冲突) */
export async function acceptIdentity(
exec: Exec,
entityId: string,
ref: { provider: string; entityType: string; externalId: string },
level: EvidenceLevel,
evidence: Record<string, unknown>,
actor: Actor,
opts: { locked?: boolean; reason?: string | null } = {},
): Promise<string | null> {
const ins = await sql<{ id: string }>`
insert into external_identity (media_entity_id, provider, entity_type, external_id, status, evidence_level, evidence, locked, valid_from, verified_at, verify_status)
values (${entityId}, ${ref.provider}, ${ref.entityType}, ${ref.externalId}, 'accepted', ${level}, ${jsonb(evidence)}::jsonb,
${opts.locked ?? false}, now(), now(), 'ok')
on conflict do nothing
returning id`.execute(exec);
const id = ins.rows[0]?.id;
if (!id) return null;
await exec
.insertInto("identity_revision")
.values({
external_identity_id: id,
revision: 1,
action: "created",
from_status: null,
to_status: "accepted",
after: jsonb({ ...ref, mediaEntityId: entityId, evidenceLevel: level, locked: opts.locked ?? false }),
actor_type: actor.type,
actor_id: actor.id,
reason: opts.reason ?? null,
})
.execute();
return id;
}
/** 综艺判定:站点分类名含"综艺"或标题带期号/播出日期 */
export function looksLikeVariety(parsed: ParsedRelease | null, categoryName: string | null): boolean {
if (categoryName && /综艺|variety|show/i.test(categoryName)) return true;
return !!parsed && (parsed.issue !== undefined || parsed.airDate !== undefined);
}
/**
* 按 TMDB 详情建本地作品:外部身份 accepted(+IMDb)、别名(原名/译名/别名)。
* 已有同一 TMDB 身份的作品(含并发创建)直接复用。
*/
export async function createEntityFromTmdb(
exec: Exec,
m: MetadataCandidate,
opts: { level: EvidenceLevel; evidence: Record<string, unknown>; actor: Actor; variety?: boolean; locked?: boolean },
): Promise<{ entityId: string; created: boolean }> {
const ref = { provider: "tmdb", entityType: m.entityType, externalId: m.externalId };
const existing = await entityByExternal(exec, ref);
if (existing) return { entityId: existing, created: false };
const kind: MediaKind = toContractMediaKind(m.entityType, { variety: !!opts.variety });
const entity = await exec
.insertInto("media_entity")
.values({
kind,
title: m.title,
original_title: m.originalTitle,
year: m.year,
status: "active",
metadata: jsonb({}),
})
.returning(["id"])
.executeTakeFirstOrThrow();
const idRow = await acceptIdentity(exec, entity.id, ref, opts.level, opts.evidence, opts.actor, {
locked: opts.locked ?? false,
});
if (!idRow) {
// 并发:另一个事务刚接受了同一身份。回滚本事务的新作品由调用方重试处理
throw new IdentityConflictError(ref);
}
if (m.externalIds?.imdb) {
await acceptIdentity(
exec,
entity.id,
{ provider: "imdb", entityType: m.entityType, externalId: m.externalIds.imdb },
opts.level,
{ via: "tmdb", tmdbId: m.externalId },
opts.actor,
);
}
await seedAliases(exec, entity.id, m);
return { entityId: entity.id, created: true };
}
export async function seedAliases(exec: Exec, entityId: string, m: MetadataCandidate): Promise<void> {
await addAlias(exec, entityId, m.title, "translated", "tmdb", null);
if (m.originalTitle) await addAlias(exec, entityId, m.originalTitle, "original", "tmdb", m.originalLanguage);
for (const t of m.titles) {
if (t.kind === "localized") await addAlias(exec, entityId, t.text, "translated", "tmdb", t.lang);
}
for (const a of m.aliases.slice(0, 50)) await addAlias(exec, entityId, a, "other", "tmdb", null);
}
export class IdentityConflictError extends Error {
constructor(readonly ref: { provider: string; entityType: string; externalId: string }) {
super(`外部身份 ${ref.provider}:${ref.entityType}/${ref.externalId} 已被其它作品接受`);
}
}
/**
* 按解析出的季/集找或建内容单元(临时 provisional 允许)。电影返回空。
* 返回:集单元(有集号时)或季单元;primary 为写入 source_release.content_unit_id 的那个。
*/
export async function ensureUnits(
exec: Exec,
entityId: string,
kind: MediaKind,
parsed: ParsedRelease | null,
): Promise<{ unitIds: string[]; primary: string | null }> {
if (!parsed || kind === "movie" || parsed.season === undefined) return { unitIds: [], primary: null };
const seasons: number[] = [];
const last = parsed.seasonEnd ?? parsed.season;
for (let s = parsed.season; s <= last && seasons.length < 50; s++) seasons.push(s);
const seasonIds: string[] = [];
for (const s of seasons) seasonIds.push(await findOrCreateUnit(exec, entityId, "season", s, null, null));
if (parsed.episodes && seasons.length === 1) {
const ids: string[] = [];
const end = Math.min(parsed.episodes.end, parsed.episodes.start + 199);
for (let e = parsed.episodes.start; e <= end; e++) {
ids.push(await findOrCreateUnit(exec, entityId, "episode", parsed.season, e, seasonIds[0]!));
}
return { unitIds: ids, primary: ids.length === 1 ? ids[0]! : seasonIds[0]! };
}
return { unitIds: seasonIds, primary: seasonIds[0] ?? null };
}
async function findOrCreateUnit(
exec: Exec,
entityId: string,
kind: "season" | "episode",
season: number,
episode: number | null,
parentId: string | null,
): Promise<string> {
let q = exec
.selectFrom("content_unit")
.select(["id"])
.where("media_entity_id", "=", entityId)
.where("kind", "=", kind)
.where("season_number", "=", season)
.where("status", "<>", "retired");
q = episode === null ? q.where("episode_number", "is", null) : q.where("episode_number", "=", episode);
const hit = await q.orderBy("created_at").executeTakeFirst();
if (hit) return hit.id;
const row = await exec
.insertInto("content_unit")
.values({
media_entity_id: entityId,
parent_unit_id: parentId,
kind,
season_number: season,
episode_number: episode,
status: "provisional",
display_order: episode ?? season,
})
.returning(["id"])
.executeTakeFirstOrThrow();
return row.id;
}
@@ -0,0 +1,320 @@
import { createHash } from "node:crypto";
import {
applyFeedItems,
emptyFeedCursor,
type FeedCursor,
type FeedFetchResult,
type FeedFetchState,
type FeedItem,
} from "@umt/adapters";
import type { FeedHealth } from "@umt/contracts";
import { type FeedSourceRow, sql } from "@umt/db";
import { jsonb } from "../rows.ts";
import type { SourcesRuntime } from "../runtime.ts";
import { clip, type Exec, openDecision } from "./common.ts";
// ingestFeed:RSS 抓取 → observation(feed+guid 唯一,摘要变化 = 更新)→ source_release(site+站点资源 ID 聚合)。
// FeedFetchState / FeedCursor 持久化在 feed_source 行;enclosure 只存脱敏值(适配器已脱敏)。
/** feed_source.cursor 的形状:重叠窗口游标 + 熔断打开时间 */
interface StoredCursor {
window?: FeedCursor;
circuitOpenedAt?: string;
}
const AUTH_BACKOFF_MS = 24 * 3600_000;
export interface IngestResult {
status: "ok" | "not_modified" | "skipped" | "error" | "auth_failed";
reason?: string;
newItems: number;
updatedItems: number;
newReleases: number;
possibleGap: boolean;
}
/** 从 Feed 条目推出站点资源 ID:详情/下载链接的 id 参数或 /detail/<id> 路径;都没有返回 null */
export function extractSiteReleaseId(item: Pick<FeedItem, "guid" | "link" | "enclosureUrl">): string | null {
for (const u of [item.link, item.enclosureUrl, item.guid]) {
if (!u) continue;
try {
const url = new URL(u);
const q = url.searchParams.get("id") ?? url.searchParams.get("tid") ?? url.searchParams.get("torrentid");
if (q && /^[\w-]{1,64}$/.test(q)) return q;
const m = /\/(?:detail|details|torrents?|t)\/(\d+)/.exec(url.pathname);
if (m?.[1]) return m[1];
} catch {
// 不是 URL(纯 GUID)
}
}
if (/^\d{1,20}$/.test(item.guid)) return item.guid;
return null;
}
export function stateFromRow(f: FeedSourceRow): FeedFetchState {
const cur = (f.cursor ?? {}) as StoredCursor;
const err = f.last_error as FeedFetchState["lastError"] | null;
return {
...(f.etag ? { etag: f.etag } : {}),
...(f.last_modified ? { lastModified: f.last_modified } : {}),
consecutiveFailures: f.consecutive_failures,
...(f.backoff_until ? { nextAttemptAt: f.backoff_until.toISOString() } : {}),
circuit: f.health_status === "circuit_open" ? "open" : "closed",
...(cur.circuitOpenedAt ? { circuitOpenedAt: cur.circuitOpenedAt } : {}),
...(f.last_success_at ? { lastSuccessAt: f.last_success_at.toISOString() } : {}),
...(err && typeof err === "object" && "kind" in err ? { lastError: err } : {}),
};
}
function healthOf(r: FeedFetchResult): FeedHealth {
if (r.kind === "ok" || r.kind === "not_modified") return "healthy";
if (r.state.circuit === "open") return "circuit_open";
if (r.kind === "skipped") return "degraded";
return r.state.consecutiveFailures >= 3 ? "failing" : "degraded";
}
function isAuthFailure(r: FeedFetchResult): boolean {
if (r.kind !== "error") return false;
return r.error.kind === "expired" || r.error.httpStatus === 401 || r.error.httpStatus === 403;
}
export async function ingestFeed(
rt: SourcesRuntime,
feedId: string,
opts: { manual?: boolean } = {},
): Promise<IngestResult> {
const { db, secrets } = rt.ctx;
const empty = { newItems: 0, updatedItems: 0, newReleases: 0, possibleGap: false };
const feed = await db.selectFrom("feed_source").selectAll().where("id", "=", feedId).executeTakeFirst();
if (!feed) return { status: "skipped", reason: "not_found", ...empty };
if (!feed.enabled) return { status: "skipped", reason: "disabled", ...empty };
const site = feed.site_id
? await db.selectFrom("site").selectAll().where("id", "=", feed.site_id).executeTakeFirst()
: undefined;
if (site && !site.enabled) return { status: "skipped", reason: "site_disabled", ...empty };
if (!feed.url_secret_ref) return { status: "skipped", reason: "no_url", ...empty };
const url = await secrets.resolve(feed.url_secret_ref);
const result = await rt.rss.fetchFeed(url, stateFromRow(feed), { ignoreBackoff: !!opts.manual });
const now = rt.now();
const auth = isAuthFailure(result);
const health: FeedHealth = auth ? "failing" : healthOf(result);
const cursor = (feed.cursor ?? {}) as StoredCursor;
const state = result.state;
const pollNext = new Date(now.getTime() + feed.poll_interval_sec * 1000);
const backoffUntil = auth
? new Date(now.getTime() + AUTH_BACKOFF_MS)
: state.nextAttemptAt && result.kind !== "ok" && result.kind !== "not_modified"
? new Date(state.nextAttemptAt)
: null;
return db.transaction().execute(async (trx) => {
let out: IngestResult = { status: result.kind === "ok" ? "ok" : result.kind, ...empty };
let window = cursor.window ?? emptyFeedCursor();
if (result.kind === "ok") {
const applied = applyFeedItems(window, result.feed.items);
window = applied.cursor;
const counts = await writeItems(rt, trx, feed, site?.id ?? null, applied.newItems, applied.updatedItems);
const unchanged = result.feed.items
.filter((i) => !applied.newItems.includes(i) && !applied.updatedItems.includes(i))
.map((i) => i.guid);
if (unchanged.length) await touchUnchanged(trx, feed.id, unchanged, now);
out = {
status: "ok",
newItems: applied.newItems.length,
updatedItems: applied.updatedItems.length,
newReleases: counts.newReleases,
possibleGap: applied.possibleGap,
};
if (applied.possibleGap) {
await rt.ctx.outbox.emit(
trx,
"feed.gap_suspected",
{
feedId: feed.id,
siteId: site?.id ?? null,
previousFetchedAt: cursor.window?.lastFetchedAt ?? null,
detectedAt: now.toISOString(),
},
{ idempotencyKey: `feed.gap:${feed.id}:${now.toISOString()}` },
);
}
}
const lastError = auth
? { ...(state.lastError ?? {}), kind: "auth", message: "Feed 凭证失效或过期(401/403/过期提示),已停止自动重试" }
: (state.lastError ?? null);
const nextCursor: StoredCursor = {
window,
...(state.circuit === "open" && state.circuitOpenedAt ? { circuitOpenedAt: state.circuitOpenedAt } : {}),
};
await trx
.updateTable("feed_source")
.set({
etag: state.etag ?? null,
last_modified: state.lastModified ?? null,
consecutive_failures: state.consecutiveFailures,
backoff_until: backoffUntil,
health_status: health,
last_error: result.kind === "ok" || result.kind === "not_modified" ? null : jsonb(lastError),
cursor: jsonb(nextCursor),
last_fetch_at: result.kind === "skipped" ? feed.last_fetch_at : now,
last_success_at: state.lastSuccessAt ? new Date(state.lastSuccessAt) : feed.last_success_at,
next_fetch_at: backoffUntil && backoffUntil > pollNext ? backoffUntil : pollNext,
})
.where("id", "=", feed.id)
.execute();
if (health !== feed.health_status) {
await rt.ctx.outbox.emit(trx, "feed.health_changed", {
feedId: feed.id,
from: feed.health_status,
to: health,
reason:
lastError && typeof lastError === "object" ? String((lastError as { message?: string }).message ?? "") : null,
});
}
if (result.kind === "ok" || result.kind === "not_modified") {
await rt.ctx.outbox.emit(trx, "feed.fetched", {
feedId: feed.id,
newItems: out.newItems,
updatedItems: out.updatedItems,
notModified: result.kind === "not_modified",
});
}
if (auth) {
if (feed.site_account_id) {
await trx
.updateTable("site_account")
.set({ status: "invalid", last_verified_at: now })
.where("id", "=", feed.site_account_id)
.execute();
}
await openDecision(rt.ctx, trx, {
kind: "source_unreachable",
severity: "warning",
title: `Feed「${feed.name}」凭证失效`,
summary: "RSS 返回 401/403 或过期提示;已停止自动重试。请更新 Feed 链接或站点账号凭证后手动恢复。",
subjectType: "feed_source",
subjectId: feed.id,
dedupeKey: `feed-auth:${feed.id}`,
evidence: { feedId: feed.id, siteId: feed.site_id, siteAccountId: feed.site_account_id, error: lastError },
options: [
{ key: "reset", label: "已更新凭证,恢复抓取" },
{ key: "disable", label: "停用该 Feed" },
],
});
out = { ...out, status: "auth_failed", reason: "auth" };
}
return out;
});
}
/** 未变化条目:只刷新 last_seen_at(observation 与其 source_release) */
async function touchUnchanged(trx: Exec, feedId: string, guids: string[], now: Date) {
const rows = await trx
.updateTable("observation")
.set({ last_seen_at: now, fetched_at: now })
.where("feed_id", "=", feedId)
.where("guid", "in", guids)
.returning(["source_release_id"])
.execute();
const ids = rows.map((r) => r.source_release_id).filter((x): x is string => !!x);
if (ids.length) await trx.updateTable("source_release").set({ last_seen_at: now }).where("id", "in", ids).execute();
}
function observationRaw(item: FeedItem) {
return {
link: item.link,
// 只存脱敏后的 enclosure(凭据参数为 [REDACTED])
enclosureUrl: item.enclosureUrl ?? null,
enclosureHasSecret: item.enclosureHasSecret ?? false,
enclosureType: item.enclosureType ?? null,
size: item.size,
categories: item.categories,
nickname: item.nickname ?? null,
description: clip(item.description, 1000),
rawDigest: item.rawDigest,
};
}
async function writeItems(
rt: SourcesRuntime,
trx: Exec,
feed: FeedSourceRow,
siteId: string | null,
newItems: FeedItem[],
updatedItems: FeedItem[],
): Promise<{ newReleases: number }> {
let newReleases = 0;
const updated = new Set(updatedItems);
for (const item of [...newItems, ...updatedItems]) {
const raw = observationRaw(item);
const obs = (
await sql<{ id: string; source_release_id: string | null; prev_digest: string | null }>`
with prev as (select content_digest from observation where feed_id = ${feed.id} and guid = ${item.guid})
insert into observation (feed_id, guid, content_digest, title, link_redacted, raw, source_published_at, fetched_at)
values (${feed.id}, ${item.guid}, ${item.rawDigest}, ${item.title}, ${item.link}, ${jsonb(raw)}::jsonb,
${item.publishedAt ?? null}, ${item.fetchedAt})
on conflict (feed_id, guid) do update set
content_digest = excluded.content_digest, title = excluded.title, link_redacted = excluded.link_redacted,
raw = excluded.raw, source_published_at = coalesce(excluded.source_published_at, observation.source_published_at),
fetched_at = excluded.fetched_at, last_seen_at = now()
returning id, source_release_id, (select content_digest from prev) as prev_digest`.execute(trx)
).rows[0]!;
if (!siteId) {
// 无站点的通用 Feed:只留观察,无法按站点资源聚合
await trx
.updateTable("observation")
.set({ status: "ignored", last_error: jsonb({ reason: "no_site" }) })
.where("id", "=", obs.id)
.execute();
continue;
}
const srid =
extractSiteReleaseId(item) ?? `guid:${createHash("sha1").update(item.guid).digest("hex").slice(0, 20)}`;
const contentChanged = updated.has(item) || (obs.prev_digest !== null && obs.prev_digest !== item.rawDigest);
const sizeBytes = item.size > 0 ? item.size : null;
const rel = (
await sql<{ id: string; inserted: boolean; prev_title: string | null; resolution_locked: boolean }>`
with prev as (select title from source_release where site_id = ${siteId} and site_release_id = ${srid})
insert into source_release (site_id, site_release_id, title, category_raw, size_bytes, source_published_at, raw)
values (${siteId}, ${srid}, ${item.title}, ${item.categories[0] ?? null}, ${sizeBytes}, ${item.publishedAt ?? null},
${jsonb({ rss: { link: item.link, categories: item.categories } })}::jsonb)
on conflict (site_id, site_release_id) do update set
last_seen_at = now(),
title = case when ${contentChanged} then excluded.title else source_release.title end,
size_bytes = case when ${contentChanged} then coalesce(excluded.size_bytes, source_release.size_bytes)
else coalesce(source_release.size_bytes, excluded.size_bytes) end,
source_published_at = coalesce(source_release.source_published_at, excluded.source_published_at),
category_raw = coalesce(source_release.category_raw, excluded.category_raw)
returning id, (xmax = 0) as inserted, (select title from prev) as prev_title, resolution_locked`.execute(trx)
).rows[0]!;
await trx
.updateTable("observation")
.set({ source_release_id: rel.id, status: "linked" })
.where("id", "=", obs.id)
.execute();
if (rel.inserted) {
newReleases++;
await rt.ctx.outbox.emit(
trx,
"release.observed",
{ sourceReleaseId: rel.id, observationId: obs.id, siteId, isNew: true },
{ idempotencyKey: `release.observed:${rel.id}` },
);
} else if (contentChanged && rel.prev_title !== item.title && !rel.resolution_locked) {
// 标题修正:按新标题重新识别(证据版本变化)
await rt.ctx.outbox.emit(
trx,
"release.observed",
{ sourceReleaseId: rel.id, observationId: obs.id, siteId, isNew: false },
{ idempotencyKey: `release.observed:${rel.id}:${item.rawDigest}` },
);
}
}
return { newReleases };
}
@@ -0,0 +1,472 @@
import { TmdbError } from "@umt/adapters";
import type {
EntityIdentitySet,
ExternalIdentityUpdate,
ExternalRef,
PoolCandidate,
ReleaseIdentityAssign,
} from "@umt/contracts";
import { type ExternalIdentityRow, sql } from "@umt/db";
import { parseRelease } from "@umt/domain";
import { toParsedColumns } from "@umt/mapping";
import { ApiException } from "../../../http/errors.ts";
import { jsonb } from "../rows.ts";
import type { SourcesRuntime } from "../runtime.ts";
import { type Actor, audit, closeDecisions, type Exec } from "./common.ts";
import {
acceptIdentity,
createEntityFromTmdb,
ensureUnits,
entityByExternal,
IdentityConflictError,
looksLikeVariety,
resolveEntityId,
} from "./entities.ts";
import { recordAttempt } from "./resolve.ts";
// 人工纠错:指定来源资源身份(加锁)、改作品外部映射(新增修订、旧映射退役)、合并作品(旧 ID 跳转)。
// 所有变更留 identity_revision / resolution_attempt / audit_event,不直接覆盖历史。
const actorName = (a: Actor) => `${a.type}:${a.id ?? ""}`;
/** 外部身份 → 本地作品;未登记且提供方为 TMDB 时按详情建档(TMDB 失败给 502,不当作不存在) */
async function entityForExternal(rt: SourcesRuntime, exec: Exec, ref: ExternalRef, actor: Actor, variety: boolean) {
const hit = await entityByExternal(exec, ref);
if (hit) return hit;
if (ref.provider !== "tmdb")
throw new ApiException("bad_request", `未登记的外部身份 ${ref.provider}:${ref.externalId},只支持按 TMDB 建档`);
const tmdb = rt.tmdb();
if (!tmdb) throw new ApiException("precondition_failed", "未配置 TMDB,无法按外部身份建档");
let detail: Awaited<ReturnType<typeof tmdb.details>>;
try {
detail = await tmdb.details(ref.entityType, ref.externalId);
} catch (e) {
if (e instanceof TmdbError && e.kind === "notfound")
throw new ApiException("not_found", `TMDB 无此条目 ${ref.entityType}/${ref.externalId}`);
if (e instanceof TmdbError) throw new ApiException("upstream_unavailable", `TMDB 查询失败:${e.kind}`);
throw e;
}
const r = await createEntityFromTmdb(exec, detail, {
level: "manual",
evidence: { by: actorName(actor) },
actor,
variety,
locked: true,
});
return r.entityId;
}
export async function assignReleaseIdentity(
rt: SourcesRuntime,
srId: string,
input: ReleaseIdentityAssign,
actor: Actor,
): Promise<{ mediaEntityId: string; contentUnitIds: string[] }> {
const { ctx } = rt;
const release = await ctx.db.selectFrom("source_release").selectAll().where("id", "=", srId).executeTakeFirst();
if (!release) throw new ApiException("not_found", "来源资源不存在");
const parsed = parseRelease(release.title, release.subtitle ? { subtitle: release.subtitle } : {});
const site = ((release.raw as Record<string, unknown>)?._site ?? {}) as { categoryName?: string };
const variety = looksLikeVariety(parsed, site.categoryName ?? release.category_raw);
let ref: ExternalRef | null = input.external ?? null;
let localId: string | null = input.mediaEntityId ?? null;
if (input.candidateIndex !== undefined) {
const pool = await ctx.db
.selectFrom("identity_candidate")
.select(["candidates"])
.where("source_release_id", "=", srId)
.executeTakeFirst();
const c = ((pool?.candidates ?? []) as PoolCandidate[])[input.candidateIndex];
if (!c) throw new ApiException("bad_request", "候选序号无效");
if (c.localEntityId) localId = c.localEntityId;
else ref = { provider: c.provider, entityType: c.entityType, externalId: c.externalId };
}
for (let attempt = 0; ; attempt++) {
try {
return await ctx.db.transaction().execute(async (trx) => {
let entityId: string;
if (localId) {
const r = await resolveEntityId(trx, localId);
if (!r) throw new ApiException("not_found", "本地作品不存在");
entityId = r;
} else {
entityId = await entityForExternal(rt, trx, ref!, actor, variety);
}
const entity = await trx
.selectFrom("media_entity")
.select(["id", "kind"])
.where("id", "=", entityId)
.executeTakeFirstOrThrow();
let unitIds: string[];
let primary: string | null;
if (input.contentUnitId) {
const u = await trx
.selectFrom("content_unit")
.select(["id", "media_entity_id"])
.where("id", "=", input.contentUnitId)
.executeTakeFirst();
if (!u || u.media_entity_id !== entityId) throw new ApiException("bad_request", "内容单元不属于该作品");
unitIds = [u.id];
primary = u.id;
} else if (input.contentUnitId === null) {
unitIds = [];
primary = null;
} else {
const r = await ensureUnits(trx, entityId, entity.kind, parsed);
unitIds = r.unitIds;
primary = r.primary;
}
const cols = toParsedColumns(parsed);
await trx
.updateTable("source_release")
.set({
parser_version: cols.parserVersion,
parsed: jsonb(cols.parsed),
media_entity_id: entityId,
content_unit_id: primary,
resolution_status: "resolved",
resolution_locked: input.lock,
resolution_evidence: jsonb({
level: "manual",
by: actorName(actor),
reason: input.reason ?? null,
previous: { mediaEntityId: release.media_entity_id, contentUnitId: release.content_unit_id },
}),
})
.where("id", "=", srId)
.execute();
await trx
.updateTable("identity_candidate")
.set({
status: "resolved",
resolved_media_entity_id: entityId,
resolved_by: actorName(actor),
resolved_at: new Date(),
next_review_at: null,
})
.where("source_release_id", "=", srId)
.execute();
await closeDecisions(
ctx,
trx,
`identity:${srId}`,
"resolved",
{ mediaEntityId: entityId, by: "manual" },
actorName(actor),
);
await recordAttempt(
trx,
srId,
null,
"manual",
input.reason ?? null,
"manual",
1,
entityId,
{ mediaEntityId: entityId },
null,
actor,
);
await audit(
trx,
actor,
"identity.release_assigned",
{ type: "source_release", id: srId },
{
mediaEntityId: entityId,
contentUnitIds: unitIds,
locked: input.lock,
previous: release.media_entity_id,
},
);
await ctx.outbox.emit(trx, "release.resolved", {
sourceReleaseId: srId,
observationId: null,
mediaEntityId: entityId,
contentUnitIds: unitIds,
evidenceLevel: "manual",
});
return { mediaEntityId: entityId, contentUnitIds: unitIds };
});
} catch (e) {
if (e instanceof IdentityConflictError && attempt < 2) continue;
throw e;
}
}
}
async function nextRevision(exec: Exec, identityId: string): Promise<number> {
const r = await exec
.selectFrom("identity_revision")
.select((eb) => eb.fn.max("revision").as("m"))
.where("external_identity_id", "=", identityId)
.executeTakeFirst();
return (Number(r?.m) || 0) + 1;
}
async function retireIdentity(exec: Exec, row: ExternalIdentityRow, actor: Actor, reason: string | null) {
const rev = Math.max(row.revision + 1, await nextRevision(exec, row.id));
await exec
.updateTable("external_identity")
.set({ status: "retired", valid_until: new Date(), revision: rev, locked: true })
.where("id", "=", row.id)
.execute();
await exec
.insertInto("identity_revision")
.values({
external_identity_id: row.id,
revision: rev,
action: "status_changed",
from_status: row.status,
to_status: "retired",
before: jsonb({ status: row.status, locked: row.locked }),
after: jsonb({ status: "retired", locked: true }),
actor_type: actor.type,
actor_id: actor.id,
reason,
})
.execute();
}
/**
* 改作品的外部映射:同 provider+类型的旧 accepted 映射退役(留修订并锁定,防止自动匹配改回),
* 新映射 accepted + 人工锁。目标身份已被其它作品接受时 409(应走合并)。
*/
export async function setEntityIdentity(rt: SourcesRuntime, entityId: string, input: EntityIdentitySet, actor: Actor) {
const { ctx } = rt;
return ctx.db.transaction().execute(async (trx) => {
const id = await resolveEntityId(trx, entityId);
if (!id) throw new ApiException("not_found", "作品不存在");
const other = await entityByExternal(trx, input);
if (other && other !== id) {
throw new ApiException("conflict", `该外部身份已属于作品 ${other},如为重复作品请使用合并`);
}
if (other === id) return { mediaEntityId: id, changed: false };
const olds = await trx
.selectFrom("external_identity")
.selectAll()
.where("media_entity_id", "=", id)
.where("provider", "=", input.provider)
.where("entity_type", "=", input.entityType)
.where("status", "=", "accepted")
.execute();
for (const o of olds) await retireIdentity(trx, o, actor, input.reason ?? null);
// 同一外部身份的旧候选(proposed/retired)不复用,新建一行保留历史
const newId = await acceptIdentity(trx, id, input, "manual", { by: actorName(actor) }, actor, {
locked: input.lock,
reason: input.reason ?? null,
});
if (!newId) throw new ApiException("conflict", "外部身份唯一约束冲突");
await audit(
trx,
actor,
"identity.mapping_changed",
{ type: "media_entity", id },
{
...input,
retired: olds.map((o) => o.id),
identityId: newId,
},
);
return { mediaEntityId: id, changed: true, identityId: newId, retired: olds.map((o) => o.id) };
});
}
/** 外部身份字段修订(乐观并发 expectedRevision);每次变更写 identity_revision */
export async function updateExternalIdentity(
rt: SourcesRuntime,
id: string,
input: ExternalIdentityUpdate,
actor: Actor,
) {
return rt.ctx.db.transaction().execute(async (trx) => {
const row = await trx
.selectFrom("external_identity")
.selectAll()
.where("id", "=", id)
.forUpdate()
.executeTakeFirst();
if (!row) throw new ApiException("not_found", "外部身份不存在");
if (row.revision !== input.expectedRevision) {
throw new ApiException("precondition_failed", `修订号已变化(当前 ${row.revision})`);
}
const rev = Math.max(row.revision + 1, await nextRevision(trx, id));
const patch: Record<string, unknown> = { revision: rev };
if (input.status !== undefined) patch.status = input.status;
if (input.evidenceLevel !== undefined) patch.evidence_level = input.evidenceLevel;
if (input.evidence !== undefined) patch.evidence = jsonb(input.evidence);
if (input.locked !== undefined) patch.locked = input.locked;
if (input.validUntil !== undefined) patch.valid_until = input.validUntil;
if (input.extra !== undefined) patch.extra = jsonb(input.extra);
if (input.status === "retired" && input.validUntil === undefined) patch.valid_until = new Date();
let updated: ExternalIdentityRow;
try {
updated = await trx
.updateTable("external_identity")
.set(patch)
.where("id", "=", id)
.returningAll()
.executeTakeFirstOrThrow();
} catch (e) {
if ((e as { code?: string }).code === "23505")
throw new ApiException("conflict", "同一外部身份已有 accepted 映射");
throw e;
}
const action =
input.status !== undefined && input.status !== row.status
? "status_changed"
: input.locked !== undefined && input.locked !== row.locked
? input.locked
? "locked"
: "unlocked"
: "evidence_updated";
await trx
.insertInto("identity_revision")
.values({
external_identity_id: id,
revision: rev,
action,
from_status: row.status,
to_status: updated.status,
before: jsonb({ status: row.status, locked: row.locked, evidenceLevel: row.evidence_level }),
after: jsonb({ status: updated.status, locked: updated.locked, evidenceLevel: updated.evidence_level }),
actor_type: actor.type,
actor_id: actor.id,
reason: input.reason ?? null,
})
.execute();
await audit(trx, actor, "identity.updated", { type: "external_identity", id }, { action, revision: rev });
return updated;
});
}
/**
* 合并重复作品:from → into。旧 ID 保留(status=merged + merged_into_id)并写 identity_redirect;
* 本模块的引用(来源资源、别名、待识别池、外部身份)迁到目标;其它模块的表按 merged_into_id 跳转解析。
*/
export async function mergeEntities(
rt: SourcesRuntime,
fromId: string,
intoId: string,
reason: string | null,
actor: Actor,
) {
if (fromId === intoId) throw new ApiException("bad_request", "不能合并到自身");
return rt.ctx.db.transaction().execute(async (trx) => {
const from = await trx
.selectFrom("media_entity")
.selectAll()
.where("id", "=", fromId)
.forUpdate()
.executeTakeFirst();
const intoResolved = await resolveEntityId(trx, intoId);
if (!from) throw new ApiException("not_found", "源作品不存在");
if (!intoResolved) throw new ApiException("not_found", "目标作品不存在");
if (from.status === "merged") throw new ApiException("conflict", "源作品已被合并");
if (intoResolved === fromId) throw new ApiException("conflict", "目标作品已合并到源作品,会形成环");
// 外部身份:目标已有同 provider+类型 accepted 的,源的退役;否则迁到目标
const idents = await trx
.selectFrom("external_identity")
.selectAll()
.where("media_entity_id", "=", fromId)
.where("status", "=", "accepted")
.execute();
for (const i of idents) {
const clash = await trx
.selectFrom("external_identity")
.select(["id"])
.where("media_entity_id", "=", intoResolved)
.where("provider", "=", i.provider)
.where("entity_type", "=", i.entity_type)
.where("status", "=", "accepted")
.executeTakeFirst();
if (clash) {
await retireIdentity(trx, i, actor, `merged into ${intoResolved}`);
continue;
}
const rev = Math.max(i.revision + 1, await nextRevision(trx, i.id));
await trx
.updateTable("external_identity")
.set({ media_entity_id: intoResolved, revision: rev })
.where("id", "=", i.id)
.execute();
await trx
.insertInto("identity_revision")
.values({
external_identity_id: i.id,
revision: rev,
action: "relinked",
from_status: i.status,
to_status: i.status,
before: jsonb({ mediaEntityId: fromId }),
after: jsonb({ mediaEntityId: intoResolved }),
actor_type: actor.type,
actor_id: actor.id,
reason,
})
.execute();
}
await sql`insert into title_alias (media_entity_id, content_unit_id, alias, normalized, language, kind, source, locked)
select ${intoResolved}, null, alias, normalized, language, kind, source, locked from title_alias
where media_entity_id = ${fromId} and content_unit_id is null
on conflict (media_entity_id, content_unit_id, normalized, kind) do nothing`.execute(trx);
await trx
.updateTable("source_release")
.set({ media_entity_id: intoResolved })
.where("media_entity_id", "=", fromId)
.execute();
await trx
.updateTable("identity_candidate")
.set({ resolved_media_entity_id: intoResolved })
.where("resolved_media_entity_id", "=", fromId)
.execute();
await trx
.updateTable("media_entity")
.set({ status: "merged", merged_into_id: intoResolved })
.where("id", "=", fromId)
.execute();
await trx
.insertInto("identity_redirect")
.values({
from_entity_id: fromId,
to_entity_id: intoResolved,
reason,
actor_type: actor.type,
actor_id: actor.id,
})
.execute();
await audit(
trx,
actor,
"identity.entity_merged",
{ type: "media_entity", id: fromId },
{ intoId: intoResolved, reason },
);
return trx.selectFrom("media_entity").selectAll().where("id", "=", intoResolved).executeTakeFirstOrThrow();
});
}
export async function dismissPoolEntry(rt: SourcesRuntime, poolId: string, actor: Actor, note: string | null) {
const { ctx } = rt;
return ctx.db.transaction().execute(async (trx) => {
const row = await trx
.updateTable("identity_candidate")
.set({ status: "dismissed", next_review_at: null, resolved_by: actorName(actor), resolved_at: new Date() })
.where("id", "=", poolId)
.returningAll()
.executeTakeFirst();
if (!row) throw new ApiException("not_found", "待识别条目不存在");
await trx
.updateTable("source_release")
.set({ resolution_status: "rejected" })
.where("id", "=", row.source_release_id)
.where("resolution_locked", "=", false)
.execute();
await closeDecisions(ctx, trx, `identity:${row.source_release_id}`, "dismissed", { note }, actorName(actor));
await audit(trx, actor, "identity.pool_dismissed", { type: "identity_candidate", id: poolId }, { note });
return row;
});
}
@@ -0,0 +1,254 @@
import { type MetadataCandidate, TmdbError } from "@umt/adapters";
import type { EntityMetadataField, FieldOverrideInput, FinalFieldValue, MetadataSource } from "@umt/contracts";
import { sql } from "@umt/db";
import { ApiException } from "../../../http/errors.ts";
import { jsonb } from "../rows.ts";
import type { SourcesRuntime } from "../runtime.ts";
import { type Actor, audit, type Exec } from "./common.ts";
import { resolveEntityId, seedAliases } from "./entities.ts";
// refreshMetadata:字段级刷新(REQ §7.3)。来源值写 metadata_field;最终值 = 覆盖(锁)> 已锁字段保持 > 来源值。
// 只更新 media_entity 的展示字段与别名,不触碰 path_binding(路径绑定不随元数据变化)。
/** 最终值投影到 media_entity 列的字段 */
const COLUMN_FIELDS: Partial<Record<EntityMetadataField, "title" | "original_title" | "year">> = {
title: "title",
originalTitle: "original_title",
year: "year",
};
function tmdbFields(m: MetadataCandidate): Partial<Record<EntityMetadataField, unknown>> {
return {
title: m.title,
originalTitle: m.originalTitle,
year: m.year,
overview: m.overview,
releaseDate: m.releaseDate,
originalLanguage: m.originalLanguage,
originCountry: m.originCountry,
};
}
export interface RefreshResult {
status: "refreshed" | "skipped" | "failed";
reason?: string;
changedFields: string[];
lockedFields: string[];
}
export async function refreshMetadata(rt: SourcesRuntime, mediaEntityId: string): Promise<RefreshResult> {
const { ctx } = rt;
const id = await resolveEntityId(ctx.db, mediaEntityId);
if (!id) return { status: "skipped", reason: "not_found", changedFields: [], lockedFields: [] };
const ident = await ctx.db
.selectFrom("external_identity")
.select(["entity_type", "external_id"])
.where("media_entity_id", "=", id)
.where("provider", "=", "tmdb")
.where("status", "=", "accepted")
.executeTakeFirst();
const tmdb = rt.tmdb();
if (!ident || !tmdb || (ident.entity_type !== "movie" && ident.entity_type !== "tv")) {
return {
status: "skipped",
reason: ident ? "tmdb_not_configured" : "no_tmdb_identity",
changedFields: [],
lockedFields: [],
};
}
let detail: MetadataCandidate;
try {
detail = await tmdb.details(ident.entity_type, ident.external_id);
} catch (e) {
// 404/超时/鉴权失败只记录校验状态,不是删除映射的依据
if (e instanceof TmdbError) {
await ctx.db
.updateTable("external_identity")
.set({ verify_status: e.kind === "notfound" ? "not_found" : "error", verified_at: rt.now() })
.where("media_entity_id", "=", id)
.where("provider", "=", "tmdb")
.where("status", "=", "accepted")
.execute();
if (e.kind === "notfound" || e.kind === "auth") {
return { status: "failed", reason: e.kind, changedFields: [], lockedFields: [] };
}
}
throw e;
}
const now = rt.now();
return ctx.db.transaction().execute(async (trx) => {
const fields = tmdbFields(detail);
for (const [field, value] of Object.entries(fields)) {
await sql`insert into metadata_field (media_entity_id, content_unit_id, field, value, language, source, source_revision, fetched_at)
values (${id}, null, ${field}, ${jsonb(value)}::jsonb, ${null}, 'tmdb', ${`tmdb:${ident.entity_type}/${ident.external_id}`}, ${now})
on conflict (media_entity_id, content_unit_id, field, source, language) do update set
value = excluded.value, source_revision = excluded.source_revision, fetched_at = excluded.fetched_at`.execute(
trx,
);
}
await trx
.updateTable("external_identity")
.set({ verify_status: "ok", verified_at: now })
.where("media_entity_id", "=", id)
.where("provider", "=", "tmdb")
.where("status", "=", "accepted")
.execute();
await seedAliases(trx, id, detail);
const r = await project(trx, id, now);
await trx.updateTable("media_entity").set({ last_metadata_refresh_at: now }).where("id", "=", id).execute();
return { status: "refreshed" as const, ...r };
});
}
/**
* 重新投影最终字段:覆盖(用户锁)> field_locks 中已锁字段保持现值 > tmdb 来源值。
* 返回被改变与被锁住的字段。
*/
export async function project(
trx: Exec,
entityId: string,
now: Date,
): Promise<{ changedFields: string[]; lockedFields: string[] }> {
const e = await trx
.selectFrom("media_entity")
.selectAll()
.where("id", "=", entityId)
.forUpdate()
.executeTakeFirstOrThrow();
const overrides = await trx
.selectFrom("field_override")
.selectAll()
.where("media_entity_id", "=", entityId)
.execute();
const sources = await trx
.selectFrom("metadata_field")
.selectAll()
.where("media_entity_id", "=", entityId)
.orderBy("fetched_at", "desc")
.execute();
const meta = { ...((e.metadata ?? {}) as Record<string, FinalFieldValue>) };
const locks = new Set(e.field_locks);
const changed: string[] = [];
const lockedFields: string[] = [];
const colPatch: Record<string, unknown> = {};
const fieldNames = new Set([...sources.map((s) => s.field), ...overrides.map((o) => o.field)]);
for (const field of fieldNames) {
const ov = overrides.find((o) => o.field === field);
let next: FinalFieldValue | null = null;
if (ov) {
next = {
value: ov.value,
source: ov.source as MetadataSource,
language: null,
at: ov.updated_at.toISOString(),
locked: ov.locked,
};
if (ov.locked) lockedFields.push(field);
} else if (locks.has(field)) {
lockedFields.push(field);
continue;
} else {
const src =
sources.find((s) => s.field === field && s.source === "tmdb") ?? sources.find((s) => s.field === field);
if (!src) continue;
next = {
value: src.value,
source: src.source as MetadataSource,
language: src.language,
at: src.fetched_at.toISOString(),
locked: false,
};
}
const prev = meta[field];
if (!prev || JSON.stringify(prev.value) !== JSON.stringify(next.value) || prev.source !== next.source) {
changed.push(field);
}
meta[field] = next;
const col = COLUMN_FIELDS[field as EntityMetadataField];
if (col) {
const v = next.value;
if (col === "year") colPatch.year = typeof v === "number" ? v : null;
else if (col === "title") {
if (typeof v === "string" && v) colPatch.title = v;
} else colPatch.original_title = typeof v === "string" ? v : null;
}
}
await trx
.updateTable("media_entity")
.set({ ...colPatch, metadata: jsonb(meta), updated_at: now })
.where("id", "=", entityId)
.execute();
return { changedFields: changed, lockedFields };
}
/** 用户覆盖字段并(默认)锁定:之后的自动刷新不得改写 */
export async function setFieldOverride(
rt: SourcesRuntime,
entityId: string,
field: EntityMetadataField,
input: FieldOverrideInput,
actor: Actor,
) {
return rt.ctx.db.transaction().execute(async (trx) => {
const id = await resolveEntityId(trx, entityId);
if (!id) throw new ApiException("not_found", "作品不存在");
if (field === "title" && (typeof input.value !== "string" || !input.value.trim())) {
throw new ApiException("bad_request", "title 必须是非空字符串");
}
if (
field === "year" &&
input.value !== null &&
(typeof input.value !== "number" || !Number.isInteger(input.value))
) {
throw new ApiException("bad_request", "year 必须是整数或 null");
}
await sql`insert into field_override (media_entity_id, content_unit_id, field, value, locked, source, reason, actor_type, actor_id)
values (${id}, null, ${field}, ${jsonb(input.value)}::jsonb, ${input.lock}, 'user', ${input.reason ?? null}, ${actor.type}, ${actor.id})
on conflict (media_entity_id, content_unit_id, field) do update set
value = excluded.value, locked = excluded.locked, reason = excluded.reason,
actor_type = excluded.actor_type, actor_id = excluded.actor_id`.execute(trx);
const e = await trx
.selectFrom("media_entity")
.select(["field_locks"])
.where("id", "=", id)
.executeTakeFirstOrThrow();
const locks = new Set(e.field_locks);
if (input.lock) locks.add(field);
else locks.delete(field);
await trx
.updateTable("media_entity")
.set({ field_locks: [...locks].sort() })
.where("id", "=", id)
.execute();
const r = await project(trx, id, rt.now());
await audit(trx, actor, "metadata.field_override", { type: "media_entity", id }, { field, lock: input.lock });
return { mediaEntityId: id, ...r };
});
}
/** 取消覆盖与锁定:回到来源值 */
export async function clearFieldOverride(
rt: SourcesRuntime,
entityId: string,
field: EntityMetadataField,
actor: Actor,
) {
return rt.ctx.db.transaction().execute(async (trx) => {
const id = await resolveEntityId(trx, entityId);
if (!id) throw new ApiException("not_found", "作品不存在");
await trx.deleteFrom("field_override").where("media_entity_id", "=", id).where("field", "=", field).execute();
const e = await trx
.selectFrom("media_entity")
.select(["field_locks"])
.where("id", "=", id)
.executeTakeFirstOrThrow();
await trx
.updateTable("media_entity")
.set({ field_locks: e.field_locks.filter((f) => f !== field) })
.where("id", "=", id)
.execute();
const r = await project(trx, id, rt.now());
await audit(trx, actor, "metadata.field_override_cleared", { type: "media_entity", id }, { field });
return { mediaEntityId: id, ...r };
});
}
@@ -0,0 +1,637 @@
import { createHash } from "node:crypto";
import { type MetadataCandidate, TmdbError } from "@umt/adapters";
import type { EvidenceLevel, PoolCandidate } from "@umt/contracts";
import { type SourceReleaseRow, sql } from "@umt/db";
import {
type IdentityCandidate as DomainCandidate,
type IdentityResolution,
normalizeTitle,
PARSER_VERSION,
type ParsedRelease,
parseRelease,
queryTitles,
type RankedCandidate,
resolveIdentity,
} from "@umt/domain";
import {
localIdentityCandidate,
toContractEvidenceLevel,
toContractResolveStatus,
toParsedColumns,
toReleaseResolutionStatus,
} from "@umt/mapping";
import { jsonb } from "../rows.ts";
import type { SourcesRuntime } from "../runtime.ts";
import { type Actor, closeDecisions, type Exec, openDecision, SYSTEM } from "./common.ts";
import {
addAlias,
createEntityFromTmdb,
ensureUnits,
entityByExternal,
IdentityConflictError,
looksLikeVariety,
resolveEntityId,
} from "./entities.ts";
// resolveObservation:parseRelease → 本地召回(别名 pg_trgm + 已有外部身份 + 人工锁)→ TMDB 候选 → domain.resolveIdentity。
// auto 才建/关联正式作品;ambiguous / no_match / 查询失败进待识别池(候选只存 JSON),有界退避复查。
const MAX_REVIEW_ATTEMPTS = 8;
const TRGM_THRESHOLD = 0.45;
export interface ResolveInput {
observationId?: string | undefined;
sourceReleaseId?: string | undefined;
/** 复查/手动重试:忽略"已识别且证据未变"的短路 */
force?: boolean | undefined;
}
export interface ResolveResult {
outcome: "auto" | "proposed" | "ambiguous" | "no_match" | "lookup_failed" | "skipped_locked" | "skipped";
sourceReleaseId: string | null;
mediaEntityId: string | null;
contentUnitIds: string[];
decisionId: string | null;
reason: string;
}
/** 候选的稳定键 */
const candKey = (c: DomainCandidate) =>
c.localEntityId ? `local:${c.localEntityId}` : `${c.provider}:${c.entityType}:${c.externalId}`;
export function evidenceVersion(parsed: ParsedRelease | null, ranked: RankedCandidate[]): string {
const h = createHash("sha1")
.update(
JSON.stringify([parsed?.title ?? null, parsed?.year ?? null, ranked.map((r) => [candKey(r.candidate), r.level])]),
)
.digest("hex")
.slice(0, 16);
return `p${PARSER_VERSION}:${h}`;
}
function yearOf(d: string | null | undefined): number | undefined {
return d && /^\d{4}/.test(d) ? Number(d.slice(0, 4)) : undefined;
}
export function tmdbToCandidate(m: MetadataCandidate): DomainCandidate {
const titles = [m.title, ...(m.originalTitle ? [m.originalTitle] : [])];
for (const t of m.titles) if (t.kind !== "alias" && !titles.includes(t.text)) titles.push(t.text);
const aliases = [...new Set([...m.aliases, ...m.titles.filter((t) => t.kind === "alias").map((t) => t.text)])];
const seasonYears = m.seasons?.map((s) => yearOf(s.airDate)).filter((y): y is number => y !== undefined);
return {
provider: "tmdb",
entityType: m.entityType,
externalId: m.externalId,
titles,
...(m.year !== null ? { year: m.year } : {}),
aliases,
...(seasonYears?.length ? { seasonYears } : {}),
};
}
function poolCandidate(r: RankedCandidate): PoolCandidate {
const c = r.candidate;
return {
provider: c.provider,
entityType: c.entityType,
externalId: c.externalId,
title: c.titles[0] ?? c.externalId,
year: c.year ?? null,
localEntityId: c.localEntityId ?? null,
level: r.level,
evidence: r.evidence,
};
}
/** 本地召回:显式外部 ID → 已接受映射;别名精确/trgm 模糊(跟随合并跳转),构造本地候选 */
async function localCandidates(
exec: Exec,
parsed: ParsedRelease,
release: SourceReleaseRow,
): Promise<DomainCandidate[]> {
const ids = new Set<string>();
const site = ((release.raw as Record<string, unknown>)?._site ?? {}) as { imdbId?: string };
const explicit = [...parsed.explicitIds, ...(site.imdbId ? [{ provider: "imdb" as const, id: site.imdbId }] : [])];
for (const x of explicit) {
const row = await exec
.selectFrom("external_identity")
.select(["media_entity_id"])
.where("provider", "=", x.provider)
.where("external_id", "=", x.id)
.where("status", "=", "accepted")
.where("media_entity_id", "is not", null)
.execute();
for (const r of row) if (r.media_entity_id) ids.add(r.media_entity_id);
}
for (const q of queryTitles(parsed)) {
const rows = await sql<{ media_entity_id: string }>`
select distinct media_entity_id from title_alias
where normalized = ${q} or (length(${q}) >= 3 and similarity(normalized, ${q}) >= ${TRGM_THRESHOLD})
limit 20`.execute(exec);
for (const r of rows.rows) ids.add(r.media_entity_id);
}
const final = new Set<string>();
for (const id of ids) {
const r = await resolveEntityId(exec, id);
if (r) final.add(r);
}
const out: DomainCandidate[] = [];
for (const id of final) {
const e = await exec.selectFrom("media_entity").selectAll().where("id", "=", id).executeTakeFirst();
if (!e || e.status === "archived") continue;
const aliases = await exec.selectFrom("title_alias").select(["alias"]).where("media_entity_id", "=", id).execute();
const idents = await exec
.selectFrom("external_identity")
.select(["provider", "external_id", "status"])
.where("media_entity_id", "=", id)
.execute();
const c = localIdentityCandidate(
{ id: e.id, kind: e.kind, title: e.title, originalTitle: e.original_title, year: e.year },
aliases.map((a) => a.alias),
idents.map((i) => ({ provider: i.provider, externalId: i.external_id, status: i.status })),
);
if (c) out.push(c);
}
return out;
}
async function tmdbCandidates(rt: SourcesRuntime, parsed: ParsedRelease): Promise<MetadataCandidate[]> {
const tmdb = rt.tmdb();
if (!tmdb) return [];
const queries = [parsed.titleCn, parsed.titleEn, parsed.title].filter((t): t is string => !!t);
const seen = new Map<string, MetadataCandidate>();
for (const q of [...new Set(queries)].slice(0, 2)) {
for (const m of await tmdb.searchAll(q)) {
const k = `${m.entityType}:${m.externalId}`;
if (!seen.has(k)) seen.set(k, m);
}
}
return [...seen.values()].slice(0, 20);
}
async function loadRelease(rt: SourcesRuntime, input: ResolveInput) {
const { db } = rt.ctx;
const observationId = input.observationId ?? null;
let srId = input.sourceReleaseId ?? null;
if (!srId && observationId) {
const o = await db
.selectFrom("observation")
.select(["source_release_id"])
.where("id", "=", observationId)
.executeTakeFirst();
srId = o?.source_release_id ?? null;
}
if (!srId) return { release: null, observationId };
const release = await db.selectFrom("source_release").selectAll().where("id", "=", srId).executeTakeFirst();
return { release: release ?? null, observationId };
}
export async function resolveRelease(rt: SourcesRuntime, input: ResolveInput): Promise<ResolveResult> {
const { ctx } = rt;
const { release, observationId } = await loadRelease(rt, input);
const base = { mediaEntityId: null, contentUnitIds: [] as string[], decisionId: null };
if (!release) return { outcome: "skipped", sourceReleaseId: null, reason: "release_not_found", ...base };
const pool = await ctx.db
.selectFrom("identity_candidate")
.selectAll()
.where("source_release_id", "=", release.id)
.executeTakeFirst();
if (release.resolution_locked) {
await recordAttempt(
ctx.db,
release.id,
observationId,
"skipped_locked",
"manual_lock",
"locked",
0,
release.media_entity_id,
);
return {
outcome: "skipped_locked",
sourceReleaseId: release.id,
reason: "manual_lock",
...base,
mediaEntityId: release.media_entity_id,
};
}
if (pool?.status === "dismissed" && !input.force) {
return { outcome: "skipped", sourceReleaseId: release.id, reason: "dismissed", ...base };
}
// 已识别:除非强制(复查/标题更新/人工重试),不重复召回
if (!input.force && release.resolution_status === "resolved" && release.media_entity_id) {
return {
outcome: "skipped",
sourceReleaseId: release.id,
reason: "already_resolved",
...base,
mediaEntityId: release.media_entity_id,
};
}
const parsed = parseRelease(release.title, release.subtitle ? { subtitle: release.subtitle } : {});
const cols = toParsedColumns(parsed);
if (!parsed.title || parsed.auxiliaryOnly) {
return toPool(rt, release, observationId, parsed, null, [], "no_match", "unparseable_title", null);
}
// 召回:本地优先,再 TMDB;TMDB 失败不等于无结果
const local = await localCandidates(ctx.db, parsed, release);
let remote: MetadataCandidate[] = [];
let lookupError: string | null = null;
try {
remote = await tmdbCandidates(rt, parsed);
} catch (e) {
if (!(e instanceof TmdbError)) throw e;
lookupError = `${e.kind}: ${e.message}`;
}
const candidates = [...local, ...remote.map(tmdbToCandidate)];
const resolution = resolveIdentity(parsed, candidates);
const version = evidenceVersion(parsed, resolution.ranked);
// 已识别且证据未变:幂等短路(强制复查时)
if (
release.resolution_status === "resolved" &&
release.media_entity_id &&
(release.resolution_evidence as { evidenceVersion?: string })?.evidenceVersion === version
) {
return {
outcome: "skipped",
sourceReleaseId: release.id,
reason: "unchanged",
...base,
mediaEntityId: release.media_entity_id,
};
}
const automation = await ctx.settings.get("automation");
const autoAccept = automation.enabled && !automation.readOnlyReconcile;
if (resolution.status === "auto" && resolution.best && autoAccept) {
const best = resolution.best;
const remoteBest =
best.candidate.provider === "tmdb"
? remote.find((m) => m.externalId === best.candidate.externalId && m.entityType === best.candidate.entityType)
: undefined;
let detailed = remoteBest;
if (remoteBest && !remoteBest.detailed) {
const existing = await entityByExternal(ctx.db, {
provider: "tmdb",
entityType: remoteBest.entityType,
externalId: remoteBest.externalId,
});
if (!existing) {
try {
detailed = await rt.tmdb()!.details(remoteBest.entityType, remoteBest.externalId);
} catch (e) {
if (!(e instanceof TmdbError)) throw e;
detailed = remoteBest;
}
}
}
return bindAuto(rt, release, observationId, parsed, cols, resolution, version, detailed ?? null);
}
if (resolution.status === "auto" && !autoAccept) {
return toPool(
rt,
release,
observationId,
parsed,
resolution,
resolution.ranked,
"proposed",
"automation_off",
version,
);
}
if (lookupError && resolution.status !== "ambiguous") {
return toPool(
rt,
release,
observationId,
parsed,
resolution,
resolution.ranked,
"lookup_failed",
lookupError,
version,
);
}
return toPool(
rt,
release,
observationId,
parsed,
resolution,
resolution.ranked,
resolution.status === "ambiguous" ? "ambiguous" : "no_match",
resolution.reason,
version,
);
}
async function recordAttempt(
exec: Exec,
srId: string,
observationId: string | null,
outcome: "auto" | "proposed" | "ambiguous" | "no_match" | "lookup_failed" | "manual" | "skipped_locked",
reason: string | null,
version: string,
count: number,
entityId: string | null,
best: unknown = null,
error: unknown = null,
actor: Actor = SYSTEM,
): Promise<void> {
await exec
.insertInto("resolution_attempt")
.values({
source_release_id: srId,
observation_id: observationId,
outcome,
reason,
evidence_version: version,
candidate_count: count,
media_entity_id: entityId,
best: best === null ? null : jsonb(best),
error: error === null ? null : jsonb(error),
actor_type: actor.type,
actor_id: actor.id,
})
.execute();
}
function rankedSummary(r: RankedCandidate | undefined) {
return r ? { ...poolCandidate(r), titleMatch: r.titleMatch, yearMatch: r.yearMatch, typeMatch: r.typeMatch } : null;
}
async function bindAuto(
rt: SourcesRuntime,
release: SourceReleaseRow,
observationId: string | null,
parsed: ParsedRelease,
cols: ReturnType<typeof toParsedColumns>,
resolution: IdentityResolution,
version: string,
tmdbDetail: MetadataCandidate | null,
): Promise<ResolveResult> {
const { ctx } = rt;
const best = resolution.best!;
const level: EvidenceLevel = toContractEvidenceLevel(best.level) ?? "needs_review";
const site = ((release.raw as Record<string, unknown>)?._site ?? {}) as { categoryName?: string };
const variety = looksLikeVariety(parsed, site.categoryName ?? release.category_raw);
for (let attempt = 0; ; attempt++) {
try {
return await ctx.db.transaction().execute(async (trx) => {
let entityId: string | null = null;
if (best.candidate.localEntityId) entityId = await resolveEntityId(trx, best.candidate.localEntityId);
if (!entityId && best.candidate.provider === "tmdb") {
entityId = await entityByExternal(trx, {
provider: "tmdb",
entityType: best.candidate.entityType,
externalId: best.candidate.externalId,
});
if (!entityId && tmdbDetail) {
entityId = (
await createEntityFromTmdb(trx, tmdbDetail, {
level,
evidence: {
sourceReleaseId: release.id,
rules: best.evidence,
titleMatch: best.titleMatch,
yearMatch: best.yearMatch,
},
actor: SYSTEM,
variety,
})
).entityId;
}
}
if (!entityId)
throw new Error(`auto 结果缺少可绑定的本地作品(${best.candidate.provider}:${best.candidate.externalId})`);
const entity = await trx
.selectFrom("media_entity")
.select(["id", "kind"])
.where("id", "=", entityId)
.executeTakeFirstOrThrow();
const units = await ensureUnits(trx, entityId, entity.kind, parsed);
// 站点标题作为别名:下次同名资源可直接本地召回
if (parsed.titleCn) await addAlias(trx, entityId, parsed.titleCn, "site_title", "site", "zh");
if (parsed.titleEn) await addAlias(trx, entityId, parsed.titleEn, "site_title", "site", "en");
const evidence = {
level,
reason: resolution.reason,
evidenceVersion: version,
best: rankedSummary(best),
candidateCount: resolution.ranked.length,
};
await trx
.updateTable("source_release")
.set({
parser_version: cols.parserVersion,
parsed: jsonb(cols.parsed),
media_entity_id: entityId,
content_unit_id: units.primary,
resolution_status: toReleaseResolutionStatus(toContractResolveStatus("auto", { autoAccept: true })),
resolution_evidence: jsonb(evidence),
})
.where("id", "=", release.id)
.where("resolution_locked", "=", false)
.execute();
await trx
.updateTable("identity_candidate")
.set({
status: "resolved",
resolved_media_entity_id: entityId,
resolved_by: "system",
resolved_at: new Date(),
next_review_at: null,
})
.where("source_release_id", "=", release.id)
.where("status", "=", "pending")
.execute();
await closeDecisions(
ctx,
trx,
`identity:${release.id}`,
"resolved",
{ mediaEntityId: entityId, by: "auto" },
"system",
);
await recordAttempt(
trx,
release.id,
observationId,
"auto",
resolution.reason,
version,
resolution.ranked.length,
entityId,
rankedSummary(best),
);
await ctx.outbox.emit(
trx,
"release.resolved",
{
sourceReleaseId: release.id,
observationId,
mediaEntityId: entityId,
contentUnitIds: units.unitIds,
evidenceLevel: level,
},
{ idempotencyKey: `release.resolved:${release.id}:${entityId}:${version}` },
);
return {
outcome: "auto" as const,
sourceReleaseId: release.id,
mediaEntityId: entityId,
contentUnitIds: units.unitIds,
decisionId: null,
reason: resolution.reason,
};
});
} catch (e) {
if (e instanceof IdentityConflictError && attempt < 2) continue;
throw e;
}
}
}
function reviewDelayMs(reason: string, attempts: number): number | null {
if (attempts >= MAX_REVIEW_ATTEMPTS) return null;
const base = reason === "lookup_failed" ? 15 * 60_000 : 3600_000;
return Math.min(base * 2 ** Math.max(0, attempts - 1), 7 * 24 * 3600_000);
}
async function toPool(
rt: SourcesRuntime,
release: SourceReleaseRow,
observationId: string | null,
parsed: ParsedRelease,
resolution: IdentityResolution | null,
ranked: RankedCandidate[],
reason: "ambiguous" | "no_match" | "proposed" | "lookup_failed",
detail: string,
versionIn: string | null,
): Promise<ResolveResult> {
const { ctx } = rt;
const version = versionIn ?? evidenceVersion(parsed, ranked);
const cols = toParsedColumns(parsed);
const candidates = ranked
.filter((r) => r.level !== "rejected")
.slice(0, 10)
.map(poolCandidate);
const now = rt.now();
const query = {
title: parsed.title ?? null,
titleCn: parsed.titleCn ?? null,
titleEn: parsed.titleEn ?? null,
normalized: parsed.title ? normalizeTitle(parsed.title) : null,
year: parsed.year ?? null,
season: parsed.season ?? null,
episodes: parsed.episodes ?? null,
};
const contractStatus =
reason === "proposed"
? toContractResolveStatus("auto", { autoAccept: false })
: reason === "ambiguous"
? "ambiguous"
: "unidentifiable";
return ctx.db.transaction().execute(async (trx) => {
const prev = await trx
.selectFrom("identity_candidate")
.select(["id", "attempts", "evidence_version"])
.where("source_release_id", "=", release.id)
.executeTakeFirst();
// 证据版本变化 = 新证据:从头计数;同一证据反复复查按退避递增,达上限停止自动复查(仍可检索)
const attempts = prev && prev.evidence_version === version ? prev.attempts + 1 : 1;
const delay = reviewDelayMs(reason, attempts);
const nextReviewAt = delay === null ? null : new Date(now.getTime() + delay);
let decisionId: string | null = null;
if ((reason === "ambiguous" || reason === "proposed") && candidates.length > 0) {
const d = await openDecision(ctx, trx, {
kind: "identity_ambiguous",
severity: "info",
title: `待确认身份:${release.title}`,
summary: `${candidates.length} 个候选证据等级相近(${detail}),请选择或忽略`,
subjectType: "source_release",
subjectId: release.id,
dedupeKey: `identity:${release.id}`,
evidence: {
sourceReleaseId: release.id,
siteId: release.site_id,
reason,
detail,
evidenceVersion: version,
query,
},
options: [
...candidates.slice(0, 5).map((c, i) => ({
key: `c${i}`,
label: `${c.title}${c.year ? ` (${c.year})` : ""} · ${c.localEntityId ? "本地作品" : `${c.provider}:${c.entityType}/${c.externalId}`}`,
input: { sourceReleaseId: release.id, candidateIndex: i },
})),
{ key: "dismiss", label: "忽略此资源" },
],
nextReviewAt,
});
decisionId = d.id;
}
await sql`
insert into identity_candidate (source_release_id, status, reason, query, candidates, evidence_version, attempts,
last_attempt_at, next_review_at, decision_id)
values (${release.id}, 'pending', ${reason}, ${jsonb(query)}::jsonb, ${jsonb(candidates)}::jsonb, ${version}, ${attempts},
${now}, ${nextReviewAt}, ${decisionId})
on conflict (source_release_id) do update set
status = 'pending', reason = excluded.reason, query = excluded.query, candidates = excluded.candidates,
evidence_version = excluded.evidence_version, attempts = excluded.attempts, last_attempt_at = excluded.last_attempt_at,
next_review_at = excluded.next_review_at, decision_id = coalesce(excluded.decision_id, identity_candidate.decision_id),
resolved_media_entity_id = null, resolved_by = null, resolved_at = null`.execute(trx);
await trx
.updateTable("source_release")
.set({
parser_version: cols.parserVersion,
parsed: jsonb(cols.parsed),
resolution_status: toReleaseResolutionStatus(contractStatus),
resolution_evidence: jsonb({ reason, detail, evidenceVersion: version, candidateCount: candidates.length }),
})
.where("id", "=", release.id)
.where("resolution_locked", "=", false)
.execute();
if (observationId) {
await trx
.updateTable("observation")
.set((eb) => ({ resolve_attempts: eb("resolve_attempts", "+", 1), next_resolve_at: nextReviewAt }))
.where("id", "=", observationId)
.execute();
}
await recordAttempt(
trx,
release.id,
observationId,
reason,
detail,
version,
candidates.length,
null,
rankedSummary(resolution?.best),
reason === "lookup_failed" ? { message: detail } : null,
);
return {
outcome: reason,
sourceReleaseId: release.id,
mediaEntityId: null,
contentUnitIds: [],
decisionId,
reason: detail,
};
});
}
export { recordAttempt };
@@ -0,0 +1,229 @@
import { SiteError, type SitePage, type SiteRelease } from "@umt/adapters";
import type { SiteSnapshotKind } from "@umt/contracts";
import { type SiteRow, sql } from "@umt/db";
import { parseRelease } from "@umt/domain";
import { siteReleaseToSourceRelease } from "@umt/mapping";
import { jsonb } from "../rows.ts";
import type { SourcesRuntime } from "../runtime.ts";
import { type Exec, openDecision } from "./common.ts";
// 站点快照(周期列表)、补漏(Feed 疑似缺口,有页数预算)、实时补查(工作台搜索)。
// 所有请求经站点共享限流器;结果按 site+站点资源 ID upsert 到 source_release,并记录 site_snapshot 覆盖范围。
export interface SnapshotInput {
siteId: string;
kind: SiteSnapshotKind;
query?: string | undefined;
/** 页数预算(1 起连续页);默认 site.config.snapshotPages ?? 1,上限 20 */
maxPages?: number | undefined;
}
export interface SnapshotResult {
status: "ok" | "partial" | "failed" | "skipped";
reason?: string;
snapshotId: string | null;
pages: number[];
itemCount: number;
newCount: number;
releaseIds: string[];
error?: string;
}
/** SiteRelease → source_release upsert;新行发 release.observed */
export async function upsertSiteRelease(
rt: SourcesRuntime,
trx: Exec,
site: Pick<SiteRow, "id">,
r: SiteRelease,
): Promise<{ id: string; inserted: boolean }> {
const parsed = parseRelease(r.title, r.subtitle ? { subtitle: r.subtitle } : {});
const c = siteReleaseToSourceRelease(r, { siteId: site.id, parsed });
const row = (
await sql<{ id: string; inserted: boolean }>`
insert into source_release (site_id, site_release_id, title, subtitle, category_raw, tags_raw, size_bytes, infohash_v1,
infohash_v2, promotion, promotion_expires_at, seeders, leechers, source_published_at, parsed, parser_version, raw,
detail_fetched_at)
values (${site.id}, ${c.siteReleaseId}, ${c.title}, ${c.subtitle ?? null}, ${c.categoryRaw ?? null}, ${c.tagsRaw ?? []},
${c.sizeBytes ?? null}, ${c.infohashV1 ?? null}, ${c.infohashV2 ?? null}, ${jsonb(c.promotion ?? {})}::jsonb,
${c.promotionExpiresAt ?? null}, ${c.seeders ?? null}, ${c.leechers ?? null}, ${c.sourcePublishedAt ?? null},
${jsonb(c.parsed ?? {})}::jsonb, ${c.parserVersion ?? null}, ${jsonb(c.raw ?? {})}::jsonb, ${null})
on conflict (site_id, site_release_id) do update set
title = excluded.title, subtitle = coalesce(excluded.subtitle, source_release.subtitle),
category_raw = coalesce(excluded.category_raw, source_release.category_raw), tags_raw = excluded.tags_raw,
size_bytes = coalesce(excluded.size_bytes, source_release.size_bytes),
infohash_v1 = coalesce(excluded.infohash_v1, source_release.infohash_v1),
infohash_v2 = coalesce(excluded.infohash_v2, source_release.infohash_v2),
promotion = excluded.promotion, promotion_expires_at = excluded.promotion_expires_at,
seeders = excluded.seeders, leechers = excluded.leechers,
source_published_at = coalesce(source_release.source_published_at, excluded.source_published_at),
raw = source_release.raw || excluded.raw, last_seen_at = now(),
availability = case when source_release.availability = 'expired' then 'active' else source_release.availability end
returning id, (xmax = 0) as inserted`.execute(trx)
).rows[0]!;
if (row.inserted) {
await rt.ctx.outbox.emit(
trx,
"release.observed",
{ sourceReleaseId: row.id, observationId: null, siteId: site.id, isNew: true },
{ idempotencyKey: `release.observed:${row.id}` },
);
}
return row;
}
/** 凭据失效:账号标失效 + 站点健康 failing + 待处理(不重试) */
async function onAuthError(rt: SourcesRuntime, trx: Exec, site: SiteRow, e: SiteError) {
await trx
.updateTable("site_account")
.set({ status: e.kind === "auth_expired" ? "expired" : "invalid", last_verified_at: rt.now() })
.where("site_id", "=", site.id)
.where("enabled", "=", true)
.where("status", "in", ["unknown", "active"])
.execute();
await trx.updateTable("site").set({ health_status: "failing" }).where("id", "=", site.id).execute();
await openDecision(rt.ctx, trx, {
kind: "source_unreachable",
severity: "warning",
title: `站点「${site.name}」登录失效`,
summary: `${e.kind}:${e.message}。已停止该站自动访问,请更新账号凭证后恢复。`,
subjectType: "site",
subjectId: site.id,
dedupeKey: `site-auth:${site.id}`,
evidence: { siteId: site.id, kind: e.kind, httpStatus: e.httpStatus ?? null },
options: [{ key: "reset", label: "已更新凭证,恢复访问" }],
});
}
export async function snapshotSite(rt: SourcesRuntime, input: SnapshotInput): Promise<SnapshotResult> {
const { db } = rt.ctx;
const none = { snapshotId: null, pages: [], itemCount: 0, newCount: 0, releaseIds: [] };
const site = await db.selectFrom("site").selectAll().where("id", "=", input.siteId).executeTakeFirst();
if (!site) return { status: "skipped", reason: "not_found", ...none };
if (!site.enabled) return { status: "skipped", reason: "site_disabled", ...none };
if (site.health_status === "failing" && input.kind !== "manual" && input.kind !== "live_search") {
const invalid = await db
.selectFrom("site_account")
.select(["id"])
.where("site_id", "=", site.id)
.where("status", "in", ["invalid", "expired"])
.executeTakeFirst();
if (invalid) return { status: "skipped", reason: "auth_invalid", ...none };
}
let adapter: ReturnType<SourcesRuntime["siteAdapter"]>;
try {
adapter = rt.siteAdapter(site);
} catch (e) {
return { status: "skipped", reason: (e as Error).message, ...none };
}
const cfgPages = Number((site.config as { snapshotPages?: number })?.snapshotPages) || 1;
const maxPages = Math.min(20, Math.max(1, input.maxPages ?? cfgPages));
const snap = await db
.insertInto("site_snapshot")
.values({
site_id: site.id,
kind: input.kind,
query: input.query ?? null,
filters: jsonb({}),
definition_version: adapter.definitionVersion,
started_at: rt.now(),
})
.returning(["id"])
.executeTakeFirstOrThrow();
const pages: number[] = [];
const releases: SiteRelease[] = [];
let error: SiteError | Error | null = null;
for (let p = 1; p <= maxPages; p++) {
try {
const res: SitePage = input.query ? await adapter.search(input.query, { page: p }) : await adapter.list(p);
pages.push(p);
releases.push(...res.items);
if (res.items.length === 0 || (res.totalPages !== null && p >= res.totalPages)) break;
} catch (e) {
error = e as Error;
break;
}
}
return db.transaction().execute(async (trx) => {
const ids: string[] = [];
let newCount = 0;
for (const r of releases) {
const row = await upsertSiteRelease(rt, trx, site, r);
ids.push(row.id);
if (row.inserted) newCount++;
}
const status = error ? (pages.length ? "partial" : "failed") : "ok";
const errJson = error
? { kind: error instanceof SiteError ? error.kind : "internal", message: error.message }
: null;
await trx
.updateTable("site_snapshot")
.set({
status,
pages: jsonb(pages),
item_count: releases.length,
new_count: newCount,
error: errJson ? jsonb(errJson) : null,
finished_at: rt.now(),
})
.where("id", "=", snap.id)
.execute();
if (error instanceof SiteError && error.isAuthError) {
await onAuthError(rt, trx, site, error);
} else if (input.kind !== "live_search") {
await trx
.updateTable("site")
.set({
...(status !== "failed" ? { last_snapshot_at: rt.now() } : {}),
health_status: status === "ok" ? "healthy" : "degraded",
snapshot_cursor: jsonb({
lastKind: input.kind,
lastPages: pages,
lastItemCount: releases.length,
lastStatus: status,
lastError: errJson,
at: rt.now().toISOString(),
}),
})
.where("id", "=", site.id)
.execute();
}
return {
status,
snapshotId: snap.id,
pages,
itemCount: releases.length,
newCount,
releaseIds: ids,
...(errJson ? { error: `${errJson.kind}: ${errJson.message}` } : {}),
};
});
}
/** 详情刷新(工作台 / release.inspect):失败保留既有数据并返回原因 */
export async function refreshReleaseDetail(rt: SourcesRuntime, srId: string): Promise<string | null> {
const { db } = rt.ctx;
const rel = await db.selectFrom("source_release").selectAll().where("id", "=", srId).executeTakeFirst();
if (!rel) return "not_found";
const site = await db.selectFrom("site").selectAll().where("id", "=", rel.site_id).executeTakeFirst();
if (!site?.enabled) return "site_disabled";
let detail: SiteRelease;
try {
detail = await rt.siteAdapter(site).detail(rel.site_release_id);
} catch (e) {
if (e instanceof SiteError) {
if (e.isAuthError) await db.transaction().execute((trx) => onAuthError(rt, trx, site, e));
if (e.kind === "not_found") {
await db.updateTable("source_release").set({ availability: "expired" }).where("id", "=", srId).execute();
}
return `${e.kind}: ${e.message}`;
}
return (e as Error).message;
}
await db.transaction().execute(async (trx) => {
await upsertSiteRelease(rt, trx, site, detail);
await trx.updateTable("source_release").set({ detail_fetched_at: rt.now() }).where("id", "=", srId).execute();
});
return null;
}
@@ -0,0 +1,218 @@
import type {
ReleaseOrigin,
WorkbenchCoverage,
WorkbenchRelease,
WorkbenchReleaseDetail,
WorkbenchReleaseQuery,
} from "@umt/contracts";
import { type SiteRow, sql } from "@umt/db";
import { normalizeTitle } from "@umt/domain";
import { ApiException } from "../../../http/errors.ts";
import { decodeCursor, page, toApi } from "../rows.ts";
import type { SourcesRuntime } from "../runtime.ts";
import { refreshReleaseDetail, snapshotSite } from "./snapshot.ts";
// 站点工作台(整理视图):本地快照优先;本地无结果 / 快照过期 / 用户要求时对已开启站点实时补查,
// 并发相同搜索合并,失败短期退避。库存叠加由前端按 mediaEntityId 查 W2-C 端点。
const LIVE_FAIL_BACKOFF_MS = 60_000;
interface LiveState {
inflight: Map<string, Promise<{ error: string | null }>>;
failedUntil: Map<string, { until: number; error: string }>;
}
const liveStates = new WeakMap<SourcesRuntime, LiveState>();
function liveState(rt: SourcesRuntime): LiveState {
let s = liveStates.get(rt);
if (!s) {
s = { inflight: new Map(), failedUntil: new Map() };
liveStates.set(rt, s);
}
return s;
}
/** 合并并发的相同实时搜索;失败后 LIVE_FAIL_BACKOFF_MS 内直接返回上次错误 */
export async function liveSearch(
rt: SourcesRuntime,
site: SiteRow,
q: string,
): Promise<{ error: string | null; coalesced: boolean }> {
const st = liveState(rt);
const key = `${site.id}|${normalizeTitle(q) || q.toLowerCase()}`;
const failed = st.failedUntil.get(key);
if (failed && failed.until > rt.now().getTime()) return { error: `${failed.error}(退避中)`, coalesced: false };
const running = st.inflight.get(key);
if (running) return { ...(await running), coalesced: true };
const p = (async () => {
const r = await snapshotSite(rt, { siteId: site.id, kind: "live_search", query: q, maxPages: 1 });
const error = r.status === "failed" || r.status === "skipped" ? (r.error ?? r.reason ?? "failed") : null;
if (error) st.failedUntil.set(key, { until: rt.now().getTime() + LIVE_FAIL_BACKOFF_MS, error });
else st.failedUntil.delete(key);
return { error };
})().finally(() => st.inflight.delete(key));
st.inflight.set(key, p);
return { ...(await p), coalesced: false };
}
async function lastSnapshotAt(rt: SourcesRuntime, siteId: string): Promise<Date | null> {
const r = await rt.ctx.db
.selectFrom("site_snapshot")
.select(["finished_at"])
.where("site_id", "=", siteId)
.where("kind", "in", ["periodic", "manual", "backfill"])
.where("status", "in", ["ok", "partial"])
.orderBy("started_at", "desc")
.executeTakeFirst();
return r?.finished_at ?? null;
}
interface ReleaseJoinRow {
[k: string]: unknown;
observed_by_feeds: number;
entity_title: string | null;
entity_year: number | null;
entity_kind: string | null;
}
function toWorkbench(row: ReleaseJoinRow): WorkbenchRelease {
const { observed_by_feeds, entity_title, entity_year, entity_kind, ...rest } = row;
const base = toApi<Omit<WorkbenchRelease, "origin" | "dataUpdatedAt" | "observedByFeeds" | "entity">>(rest);
const raw = (base.raw ?? {}) as Record<string, unknown>;
const origin: ReleaseOrigin = raw._site ? "snapshot" : Number(observed_by_feeds) > 0 ? "feed" : "unknown";
return {
...base,
origin,
dataUpdatedAt: base.lastSeenAt,
observedByFeeds: Number(observed_by_feeds),
entity:
base.mediaEntityId && entity_title
? { id: base.mediaEntityId, title: entity_title, year: entity_year, kind: entity_kind ?? "other" }
: null,
};
}
function releaseSelect(rt: SourcesRuntime) {
return rt.ctx.db
.selectFrom("source_release as r")
.leftJoin("media_entity as e", "e.id", "r.media_entity_id")
.selectAll("r")
.select([
"e.title as entity_title",
"e.year as entity_year",
"e.kind as entity_kind",
sql<number>`(select count(distinct o.feed_id) from observation o where o.source_release_id = r.id)::int`.as(
"observed_by_feeds",
),
]);
}
export async function listSiteReleases(
rt: SourcesRuntime,
siteId: string,
q: WorkbenchReleaseQuery,
): Promise<{ items: WorkbenchRelease[]; nextCursor: string | null; coverage: WorkbenchCoverage }> {
const site = await rt.ctx.db.selectFrom("site").selectAll().where("id", "=", siteId).executeTakeFirst();
if (!site) throw new ApiException("not_found", "站点不存在");
const snapAt = await lastSnapshotAt(rt, site.id);
const periodMs = (site.snapshot_interval_sec ?? 3600) * 1000;
const stale = !snapAt || rt.now().getTime() - snapAt.getTime() > periodMs * 2;
const offset = decodeCursor(q.cursor);
const run = async () => {
let query = releaseSelect(rt).where("r.site_id", "=", site.id);
if (q.q) {
const like = `%${q.q.replace(/[\\%_]/g, (m) => `\\${m}`)}%`;
query = query.where((eb) =>
eb.or([
eb("r.title", "ilike", like),
eb("r.subtitle", "ilike", like),
eb("r.site_release_id", "=", q.q!),
sql<boolean>`similarity(r.title, ${q.q}) >= 0.3`,
]),
);
}
if (q.category) query = query.where("r.category_raw", "=", q.category);
if (q.promotion) query = query.where(sql`r.promotion->>'kind'`, "=", q.promotion);
if (q.resolution) query = query.where("r.resolution_status", "=", q.resolution);
return (await query
.orderBy(sql`coalesce(r.source_published_at, r.first_seen_at)`, "desc")
.orderBy("r.id")
.offset(offset)
.limit(q.limit + 1)
.execute()) as unknown as ReleaseJoinRow[];
};
let rows = await run();
let live = false;
let liveError: string | null = null;
let coalesced = false;
const wantLive = q.live === "true" || (q.live !== "false" && offset === 0 && !!q.q && (rows.length === 0 || stale));
if (wantLive && q.q) {
if (!site.enabled) liveError = "站点已关闭,只展示历史数据";
else {
const r = await liveSearch(rt, site, q.q);
live = r.error === null;
liveError = r.error;
coalesced = r.coalesced;
if (live) rows = await run();
}
}
const pg = page(rows, offset, q.limit);
return {
items: pg.items.map(toWorkbench),
nextCursor: pg.nextCursor,
coverage: {
siteEnabled: site.enabled,
lastSnapshotAt: snapAt?.toISOString() ?? null,
stale,
fromSnapshot: rows.length > 0,
live,
liveError,
coalesced,
},
};
}
export async function releaseDetail(
rt: SourcesRuntime,
siteId: string,
releaseId: string,
refresh: boolean,
): Promise<WorkbenchReleaseDetail> {
const find = async () =>
(await releaseSelect(rt)
.where("r.site_id", "=", siteId)
.where("r.id", "=", releaseId)
.executeTakeFirst()) as unknown as ReleaseJoinRow | undefined;
if (!(await find())) throw new ApiException("not_found", "资源不存在");
const refreshError = refresh ? await refreshReleaseDetail(rt, releaseId) : null;
const row = (await find())!;
const release = toWorkbench(row);
const obs = await rt.ctx.db
.selectFrom("observation as o")
.innerJoin("feed_source as f", "f.id", "o.feed_id")
.select(["o.id", "o.feed_id", "f.name", "o.first_seen_at", "o.last_seen_at"])
.where("o.source_release_id", "=", releaseId)
.orderBy("o.first_seen_at")
.execute();
const promo = (release.promotion ?? {}) as { observedAt?: string };
return {
release,
observations: obs.map((o) => ({
observationId: o.id,
feedId: o.feed_id,
feedName: o.name,
firstSeenAt: o.first_seen_at.toISOString(),
lastSeenAt: o.last_seen_at.toISOString(),
})),
volatile: {
promotionObservedAt: promo.observedAt ?? null,
promotionExpiresAt: release.promotionExpiresAt,
peersObservedAt: release.lastSeenAt,
detailFetchedAt: release.detailFetchedAt,
recheckBeforeDownload: true,
},
refreshError,
};
}
@@ -0,0 +1,92 @@
import { defineTask, type OutboxRouteFn, type TaskRegistry } from "@umt/tasks";
import { z } from "zod";
import type { SourcesRuntime } from "./runtime.ts";
import { ingestFeed } from "./service/feeds.ts";
import { refreshMetadata } from "./service/metadata.ts";
import { resolveRelease } from "./service/resolve.ts";
import { snapshotSite } from "./service/snapshot.ts";
// 持久任务:输入只放业务 ID,处理器读业务表。外部请求失败的退避/熔断由适配器状态与业务表承担,
// 凭据失效返回正常结果(不重试);只有意外错误(DB、TMDB 网络)抛出交给引擎重试。
const NETWORK_RETRY = {
maximumAttempts: 5,
initialInterval: "1s",
backoffCoefficient: 2,
maximumInterval: "60s",
} as const;
const id = z.string().uuid();
/** Feed 疑似缺口时的补漏页数预算 */
export const BACKFILL_MAX_PAGES = 3;
export function registerSourcesTasks(registry: TaskRegistry, rt: () => SourcesRuntime): void {
registry.register(
defineTask({
name: "ingestFeed",
version: 1,
inputSchema: z.object({ feedSourceId: id, manual: z.boolean().optional() }),
retryPolicy: NETWORK_RETRY,
handler: (t) => t.step.run("ingest", () => ingestFeed(rt(), t.input.feedSourceId, { manual: !!t.input.manual })),
}),
defineTask({
name: "resolveObservation",
version: 1,
inputSchema: z
.object({ observationId: id.nullish(), sourceReleaseId: id.nullish(), force: z.boolean().optional() })
.refine((v) => !!v.observationId || !!v.sourceReleaseId, { message: "需要 observationId 或 sourceReleaseId" }),
retryPolicy: NETWORK_RETRY,
handler: (t) =>
t.step.run("resolve", () =>
resolveRelease(rt(), {
observationId: t.input.observationId ?? undefined,
sourceReleaseId: t.input.sourceReleaseId ?? undefined,
force: t.input.force,
}),
),
}),
defineTask({
name: "refreshMetadata",
version: 1,
inputSchema: z.object({ mediaEntityId: id }),
retryPolicy: NETWORK_RETRY,
handler: (t) => t.step.run("refresh", () => refreshMetadata(rt(), t.input.mediaEntityId)),
}),
defineTask({
name: "snapshotSite",
version: 1,
inputSchema: z.object({
siteId: id,
kind: z.enum(["periodic", "backfill", "manual"]).default("periodic"),
maxPages: z.number().int().min(1).max(20).optional(),
}),
retryPolicy: NETWORK_RETRY,
handler: (t) =>
t.step.run("snapshot", () =>
snapshotSite(rt(), { siteId: t.input.siteId, kind: t.input.kind, maxPages: t.input.maxPages }),
),
}),
);
}
type P = Record<string, unknown>;
export const sourcesOutboxRoutes: Record<string, OutboxRouteFn> = {
"feed.due": (ev) => {
const p = ev.payload as P;
return { task: "ingestFeed", input: { feedSourceId: p.feedId, manual: p.reason === "manual" } };
},
"release.observed": (ev) => {
const p = ev.payload as P;
return {
task: "resolveObservation",
input: { sourceReleaseId: p.sourceReleaseId, observationId: p.observationId },
};
},
"feed.gap_suspected": (ev) => {
const p = ev.payload as P;
if (!p.siteId) return null;
return { task: "snapshotSite", input: { siteId: p.siteId, kind: "backfill", maxPages: BACKFILL_MAX_PAGES } };
},
};