feat(server): W2-B sources, identity resolution, metadata and site workbench

- Sites / accounts / feeds: registry view (/site-definitions), site CRUD with
  enable/disable (history stays queryable, marked site_disabled), accounts and
  feed URLs stored as secrets (only secret_ref / redacted urlDisplay exposed),
  feed enable/disable/reset/fetch; one shared rate limiter + adapter per site.
- Tasks: ingestFeed (FeedFetchState/FeedCursor on feed_source, observation
  feed+guid upsert with digest-change updates, source_release site+id
  aggregation across feeds, auth failure -> account invalid + decision, no
  retry, gap -> feed.gap_suspected -> budgeted backfill snapshot),
  resolveObservation (parseRelease -> local recall via title_alias pg_trgm +
  accepted identities -> TMDB -> resolveIdentity; auto binds entity /
  provisional units / identities / aliases, otherwise identity_candidate pool
  with evidence version and bounded review backoff; manual lock respected),
  refreshMetadata (metadata_field + field_override, user locks win, paths
  untouched), snapshotSite (site_snapshot coverage records).
- Events: feed.due, feed.gap_suspected, release.observed (self-consumed),
  release.resolved, decision.created/updated; scheduler in worker start().
- Identity API: entity detail with redirects, manual mapping correction with
  identity_revision, merge with identity_redirect, pool list/assign/dismiss/
  retry, content unit numbering lock, aliases, external identities.
- Decisions: list/summary (by entity and kind)/detail/resolve; identity
  options executed directly.
- Workbench: /sites/{id}/releases (snapshot first, coalesced live search with
  backoff, origin + freshness) and /sites/{id}/releases/{rid} (volatile field
  recheck times, refresh keeps data on failure).
- Capabilities: catalog.lookup, release.search, release.inspect,
  identity.resolve.
- Migrations 0018-0020: site.adapter api, source_release.resolution_locked,
  site_snapshot, identity_candidate, resolution_attempt, identity_redirect,
  metadata_field, field_override.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed 2026-09-27 20:12:01 +08:00
1 parent 829ca4f7ac
commit a24bda952b
39 files changed
+6861 -15

No files matched your search

@@ -0,0 +1,187 @@
import type { CapabilityRegistry } from "@umt/agent";
import { capabilitySpec, type ExternalEntityType, type SourceRelease } from "@umt/contracts";
import { sql } from "@umt/db";
import { normalizeTitle } from "@umt/domain";
import { toApi, toApiList } from "./rows.ts";
import type { SourcesRuntime } from "./runtime.ts";
import { assignReleaseIdentity } from "./service/identity-admin.ts";
import { resolveRelease } from "./service/resolve.ts";
import { refreshReleaseDetail } from "./service/snapshot.ts";
import { liveSearch } from "./service/workbench.ts";
// 本模块领域的能力处理器(规格一律取 contracts):catalog.lookup / release.search / release.inspect / identity.resolve。
// identity.resolve 属 propose 类,授权回调由 W2-D 接入前会被注册中心拒绝(安全默认)。
export function registerSourcesCapabilities(registry: CapabilityRegistry, rt: () => SourcesRuntime): void {
registry.register(capabilitySpec("catalog.lookup"), async (input) => {
const { db } = rt().ctx;
const local: { entity: unknown; score: number; matchedAlias: string | null }[] = [];
if (input.externalId) {
const rows = await db
.selectFrom("external_identity as i")
.innerJoin("media_entity as e", "e.id", "i.media_entity_id")
.selectAll("e")
.where("i.provider", "=", input.externalId.provider)
.where("i.entity_type", "=", input.externalId.entityType)
.where("i.external_id", "=", input.externalId.externalId)
.where("i.status", "=", "accepted")
.execute();
for (const e of rows) local.push({ entity: toApi(e), score: 1, matchedAlias: null });
}
if (input.query) {
const n = normalizeTitle(input.query);
const rows = await sql<Record<string, unknown> & { score: number; matched: string }>`
select distinct on (e.id) e.*, greatest(similarity(a.normalized, ${n}), case when a.normalized = ${n} then 1 else 0 end) as score,
a.alias as matched
from title_alias a join media_entity e on e.id = a.media_entity_id
where e.status <> 'merged' and (a.normalized = ${n} or similarity(a.normalized, ${n}) >= 0.3)
${input.kind ? sql`and e.kind = ${input.kind}` : sql``}
${input.year ? sql`and (e.year is null or abs(e.year - ${input.year}) <= 1)` : sql``}
order by e.id, score desc`.execute(db);
for (const r of rows.rows.sort((a, b) => b.score - a.score).slice(0, input.limit)) {
const { score, matched, ...e } = r;
if (!local.some((l) => (l.entity as { id: string }).id === e.id)) {
local.push({ entity: toApi(e), score: Number(score), matchedAlias: matched });
}
}
}
const remote: {
provider: string;
entityType: ExternalEntityType;
externalId: string;
title: string;
year: number | null;
}[] = [];
const tmdb = rt().tmdb();
if (input.includeRemote && input.query && tmdb) {
for (const m of (await tmdb.searchAll(input.query, input.year ? { year: input.year } : {})).slice(
0,
input.limit,
)) {
remote.push({
provider: "tmdb",
entityType: m.entityType,
externalId: m.externalId,
title: m.title,
year: m.year,
});
}
}
return { local: local as never, remote };
});
registry.register(capabilitySpec("release.search"), async (input) => {
const r = rt();
const { db } = r.ctx;
let q = db.selectFrom("source_release").selectAll();
if (input.siteIds?.length) q = q.where("site_id", "in", input.siteIds);
if (input.mediaEntityId) q = q.where("media_entity_id", "=", input.mediaEntityId);
const text = input.query;
if (text)
q = q.where((eb) => eb.or([eb("title", "ilike", `%${text}%`), sql<boolean>`similarity(title, ${text}) >= 0.3`]));
const run = () => q.orderBy("first_seen_at", "desc").limit(input.limit).execute();
let rows = await run();
const sites = await db
.selectFrom("site")
.selectAll()
.where("enabled", "=", true)
.$if(!!input.siteIds?.length, (b) => b.where("id", "in", input.siteIds!))
.execute();
const coverage: { siteId: string; fromSnapshot: boolean; live: boolean; error: string | null }[] = [];
const needLive = input.allowLiveSearch && !!text && rows.length === 0;
for (const s of sites) {
if (needLive) {
const res = await liveSearch(r, s, text!);
coverage.push({ siteId: s.id, fromSnapshot: false, live: res.error === null, error: res.error });
} else coverage.push({ siteId: s.id, fromSnapshot: true, live: false, error: null });
}
if (needLive) rows = await run();
return { items: toApiList<SourceRelease>(rows), coverage };
});
registry.register(capabilitySpec("release.inspect"), async (input) => {
const r = rt();
if (input.refreshDetail) await refreshReleaseDetail(r, input.sourceReleaseId);
const row = await r.ctx.db
.selectFrom("source_release")
.selectAll()
.where("id", "=", input.sourceReleaseId)
.executeTakeFirst();
if (!row) throw new Error("来源资源不存在");
// 库存(media_asset)归 W2-C,这里只按作品只读查询
const inventory = row.media_entity_id
? await r.ctx.db
.selectFrom("media_asset")
.selectAll()
.where("media_entity_id", "=", row.media_entity_id)
.execute()
: [];
return { release: toApi(row), inventory: toApiList(inventory) } as never;
});
registry.register(capabilitySpec("identity.resolve"), async (input, ctx) => {
const r = rt();
let srId = input.sourceReleaseId ?? null;
if (!srId && input.observationId) {
const o = await r.ctx.db
.selectFrom("observation")
.select(["source_release_id"])
.where("id", "=", input.observationId)
.executeTakeFirst();
srId = o?.source_release_id ?? null;
}
if (!srId) throw new Error("找不到来源资源");
const actor = { type: "agent" as const, id: ctx.actorId };
const c = input.candidate;
if (c?.mediaEntityId || (c?.provider && c.entityType && c.externalId)) {
// Agent 的提议不加人工锁,后续人工仍可改
const res = await assignReleaseIdentity(
r,
srId,
c.mediaEntityId
? { mediaEntityId: c.mediaEntityId, lock: false }
: {
external: {
provider: c.provider!,
entityType: c.entityType as "movie" | "tv",
externalId: c.externalId!,
},
lock: false,
},
actor,
);
return {
status: "resolved" as const,
mediaEntityId: res.mediaEntityId,
identities: await identities(r, res.mediaEntityId),
decisionId: null,
};
}
const res = await resolveRelease(r, { sourceReleaseId: srId, force: true });
const status: "resolved" | "proposed" | "ambiguous" | "unidentifiable" =
res.outcome === "auto" || res.outcome === "skipped" || res.outcome === "skipped_locked"
? res.mediaEntityId
? "resolved"
: "unidentifiable"
: res.outcome === "proposed"
? "proposed"
: res.outcome === "ambiguous"
? "ambiguous"
: "unidentifiable";
return {
status,
mediaEntityId: res.mediaEntityId,
identities: res.mediaEntityId ? await identities(r, res.mediaEntityId) : [],
decisionId: res.decisionId,
};
});
}
async function identities(r: SourcesRuntime, entityId: string) {
const rows = await r.ctx.db
.selectFrom("external_identity")
.selectAll()
.where("media_entity_id", "=", entityId)
.execute();
return toApiList(rows) as never[];
}
@@ -0,0 +1,303 @@
import {
ContentUnit,
ContentUnitUpdate,
CursorQuery,
cursorPage,
Decision,
DecisionKind,
DecisionResolveInput,
DecisionStatus,
ExternalIdentity,
ExternalIdentityCreate,
ExternalIdentityUpdate,
FeedHealth,
FeedSource,
FeedSourceCreate,
FeedSourceUpdate,
Id,
IdentityRevision,
IdentityStatus,
IdParam,
MediaEntity,
MediaEntityCreate,
MediaEntityMerge,
MediaEntityStatus,
MediaEntityUpdate,
MediaKind,
Observation,
ObservationStatus,
OkResult,
ResolutionStatus,
Site,
SiteAccount,
SiteAccountCreate,
SiteAccountUpdate,
SiteCreate,
SiteUpdate,
SourceRelease,
TitleAlias,
TitleAliasCreate,
} from "@umt/contracts";
import { z } from "zod";
// 与 contracts Api(crud())同路径、同 scope 的端点定义,但保留具体 schema 类型供路由推断入参。
// scope:读 = read;写 = admin(站点/Feed 配置)或 organize.execute(身份纠错、待处理)。
const list = <R extends z.ZodType, F extends z.ZodRawShape>(
path: string,
summary: string,
row: R,
filters: F,
tags: string[],
) => ({
method: "get" as const,
path,
scope: "read" as const,
summary,
tags,
query: CursorQuery.extend(filters),
response: cursorPage(row),
});
const get = <R extends z.ZodType>(path: string, summary: string, row: R, tags: string[]) => ({
method: "get" as const,
path,
scope: "read" as const,
summary,
tags,
params: IdParam,
response: row,
});
const write = <M extends "post" | "patch" | "delete", B extends z.ZodType | undefined, R extends z.ZodType>(
method: M,
path: string,
scope: "admin" | "organize.execute",
summary: string,
body: B,
response: R,
tags: string[],
withId = true,
) => ({
method,
path,
scope,
summary,
tags,
...(withId ? { params: IdParam } : {}),
body: body as B,
response,
});
const TS = ["sites"];
const TF = ["feed-sources"];
const TR = ["source-releases"];
const TE = ["media-entities"];
const TI = ["identity"];
const TD = ["decisions"];
export const D = {
sites: {
list: list("/sites", "站点列表", Site, {}, TS),
get: get("/sites/{id}", "站点详情", Site, TS),
create: {
method: "post" as const,
path: "/sites",
scope: "admin" as const,
summary: "登记站点",
tags: TS,
body: SiteCreate,
response: Site,
},
update: write("patch", "/sites/{id}", "admin", "修改站点(启停、快照周期、限流)", SiteUpdate, Site, TS),
},
siteAccounts: {
list: list("/site-accounts", "站点账号列表", SiteAccount, { siteId: Id.optional() }, TS),
get: get("/site-accounts/{id}", "站点账号详情", SiteAccount, TS),
create: {
method: "post" as const,
path: "/site-accounts",
scope: "admin" as const,
summary: "新建站点账号(凭证只写,存为 secret)",
tags: TS,
body: SiteAccountCreate,
response: SiteAccount,
},
update: write(
"patch",
"/site-accounts/{id}",
"admin",
"修改站点账号 / 轮换凭证",
SiteAccountUpdate,
SiteAccount,
TS,
),
},
feedSources: {
list: list("/feed-sources", "Feed 列表", FeedSource, { siteId: Id.optional(), health: FeedHealth.optional() }, TF),
get: get("/feed-sources/{id}", "Feed 详情", FeedSource, TF),
create: {
method: "post" as const,
path: "/feed-sources",
scope: "admin" as const,
summary: "新建 Feed(url 只写,存为 secret;对外只给脱敏 URL)",
tags: TF,
body: FeedSourceCreate,
response: FeedSource,
},
update: write("patch", "/feed-sources/{id}", "admin", "修改 Feed", FeedSourceUpdate, FeedSource, TF),
fetchNow: write(
"post",
"/feed-sources/{id}/fetch",
"admin",
"立即抓取(仍受站点限流)",
undefined,
z.object({ taskId: Id }),
TF,
),
},
sourceReleases: {
list: list(
"/source-releases",
"站点资源列表",
SourceRelease,
{
siteId: Id.optional(),
mediaEntityId: Id.optional(),
resolutionStatus: ResolutionStatus.optional(),
q: z.string().max(200).optional(),
},
TR,
),
get: get("/source-releases/{id}", "站点资源详情", SourceRelease, TR),
},
observations: {
list: list(
"/observations",
"观察列表",
Observation,
{ feedId: Id.optional(), status: ObservationStatus.optional() },
TR,
),
get: get("/observations/{id}", "观察详情", Observation, TR),
},
mediaEntities: {
list: list(
"/media-entities",
"作品列表",
MediaEntity,
{ kind: MediaKind.optional(), status: MediaEntityStatus.optional(), q: z.string().max(200).optional() },
TE,
),
get: get("/media-entities/{id}", "作品(旧 ID 自动跳转)", MediaEntity, TE),
create: {
method: "post" as const,
path: "/media-entities",
scope: "organize.execute" as const,
summary: "手动建本地作品(TMDB 无记录时可先建档)",
tags: TE,
body: MediaEntityCreate,
response: MediaEntity,
},
update: write(
"patch",
"/media-entities/{id}",
"organize.execute",
"修改作品(标题/原名/年份按字段覆盖并锁定)",
MediaEntityUpdate,
MediaEntity,
TE,
),
merge: write(
"post",
"/media-entities/{id}/merge",
"organize.execute",
"合并重复作品(保留旧 ID 跳转)",
MediaEntityMerge,
MediaEntity,
TE,
),
},
contentUnits: {
list: list(
"/content-units",
"内容单元列表",
ContentUnit,
{ mediaEntityId: Id.optional(), seasonNumber: z.coerce.number().int().optional() },
TE,
),
get: get("/content-units/{id}", "内容单元详情", ContentUnit, TE),
update: write(
"patch",
"/content-units/{id}",
"organize.execute",
"修改内容单元(改编号即确认并锁定)",
ContentUnitUpdate,
ContentUnit,
TE,
),
},
titleAliases: {
list: list(
"/title-aliases",
"别名列表",
TitleAlias,
{ mediaEntityId: Id.optional(), q: z.string().max(200).optional() },
TI,
),
create: write("post", "/title-aliases", "organize.execute", "新增别名", TitleAliasCreate, TitleAlias, TI, false),
remove: write("delete", "/title-aliases/{id}", "organize.execute", "删除别名", undefined, OkResult, TI),
},
externalIdentities: {
list: list(
"/external-identities",
"外部身份列表",
ExternalIdentity,
{ mediaEntityId: Id.optional(), provider: z.string().optional(), status: IdentityStatus.optional() },
TI,
),
get: get("/external-identities/{id}", "外部身份详情", ExternalIdentity, TI),
create: write(
"post",
"/external-identities",
"organize.execute",
"新增外部身份(accepted 等同人工纠错)",
ExternalIdentityCreate,
ExternalIdentity,
TI,
false,
),
update: write(
"patch",
"/external-identities/{id}",
"organize.execute",
"修订外部身份(乐观并发)",
ExternalIdentityUpdate,
ExternalIdentity,
TI,
),
revisions: get(
"/external-identities/{id}/revisions",
"身份修订历史",
z.object({ items: z.array(IdentityRevision) }),
TI,
),
},
decisions: {
list: list(
"/decisions",
"待处理事项(缺省只列未关闭)",
Decision,
{ status: DecisionStatus.optional(), kind: DecisionKind.optional() },
TD,
),
get: get("/decisions/{id}", "待处理事项详情", Decision, TD),
resolve: write(
"post",
"/decisions/{id}/resolve",
"organize.execute",
"处理待处理事项",
DecisionResolveInput,
Decision,
TD,
),
},
};
@@ -0,0 +1,69 @@
import { createRoute, type RouteConfig } from "@hono/zod-openapi";
import { type ApiScope, apiResponse } from "@umt/contracts";
import type { Context } from "hono";
import type { z } from "zod";
import { type AuthEnv, requireScope } from "../../http/auth.ts";
import type { ModuleApp } from "../types.ts";
import type { Actor } from "./service/common.ts";
// 本模块路由封装:createRoute(进 OpenAPI)+ requireScope + 入参校验 + 出参按合同 parse。
export interface RouteDef {
method: "get" | "post" | "patch" | "put" | "delete";
path: string;
scope: ApiScope;
summary: string;
tags: string[];
params?: z.ZodObject;
query?: z.ZodType;
body?: z.ZodType;
response: z.ZodType;
}
type Infer<T> = T extends z.ZodType ? z.infer<T> : Record<string, never>;
export interface Input<D extends RouteDef> {
params: Infer<D["params"]>;
query: Infer<D["query"]>;
body: Infer<D["body"]>;
actor: Actor;
c: Context<AuthEnv>;
}
/** 调用者(W2-A 的 c.var.actor:admin 会话 / API Key)→ 审计/修订用的操作者;管理员记为 user */
export function actorOf(c: Context<AuthEnv>): Actor {
const a = c.get("actor") as { type?: string; id?: string } | undefined;
if (!a) return { type: "system", id: null };
return { type: a.type === "api_key" ? "api_key" : "user", id: a.id ?? null };
}
export function route<D extends RouteDef>(app: ModuleApp, def: D, handler: (input: Input<D>) => Promise<unknown>) {
const request: RouteConfig["request"] = {};
if (def.params) request.params = def.params as never;
if (def.query) request.query = def.query as never;
if (def.body) request.body = { content: { "application/json": { schema: def.body as never } }, required: true };
const r = createRoute({
method: def.method,
path: def.path,
summary: def.summary,
tags: def.tags,
middleware: [requireScope(def.scope)] as never,
request,
responses: {
200: { description: def.summary, content: { "application/json": { schema: apiResponse(def.response) } } },
},
});
// biome-ignore lint/suspicious/noExplicitAny: 路由泛型在封装里擦除,入参已由 zod-openapi 校验
app.openapi(r, (async (c: any) => {
const input = {
params: def.params ? c.req.valid("param") : {},
query: def.query ? c.req.valid("query") : {},
body: def.body ? c.req.valid("json") : {},
actor: actorOf(c),
c,
} as Input<D>;
const data = def.response.parse(await handler(input));
return c.json({ ok: true, data }, 200);
// biome-ignore lint/suspicious/noExplicitAny: 同上
}) as any);
}
@@ -1,12 +1,69 @@
import type { ServerModule } from "../types.ts";
import type { ServerContext, ServerModule } from "../types.ts";
import { registerSourcesCapabilities } from "./capabilities.ts";
import { decisionRoutes } from "./routes/decisions.ts";
import { identityRoutes } from "./routes/identity.ts";
import { sourcesRoutes } from "./routes/sources.ts";
import { workbenchRoutes } from "./routes/workbench.ts";
import { type RuntimeOverrides, SourcesRuntime } from "./runtime.ts";
import { schedulerTick } from "./scheduler.ts";
import { registerSourcesTasks, sourcesOutboxRoutes } from "./tasks.ts";
/**
* W2-B 来源/身份/元数据:site、site_account、feed_source;ingestFeed / resolveObservation / refreshMetadata;
* W2-B 来源/身份/元数据:site、site_account、feed_source;ingestFeed / resolveObservation / refreshMetadata / snapshotSite;
* observation/source_release 写入与去重;身份解析、待识别池与 decision;站点工作台 API。
* 发出 release.resolved、decision.created(经 ctx.outbox.emit)。
* 发出 feed.due、feed.gap_suspected、release.observed(自消费)与 release.resolved、decision.created/updated。
*/
export const sourcesIdentityModule: ServerModule = {
name: "sources-identity",
registerRoutes() {},
registerTasks() {},
};
export function createSourcesIdentityModule(overrides: RuntimeOverrides = {}): ServerModule & {
runtime(ctx: ServerContext): SourcesRuntime;
} {
const runtimes = new WeakMap<ServerContext, SourcesRuntime>();
const runtime = (ctx: ServerContext) => {
let rt = runtimes.get(ctx);
if (!rt) {
rt = new SourcesRuntime(ctx, overrides);
runtimes.set(ctx, rt);
}
return rt;
};
let timer: ReturnType<typeof setInterval> | null = null;
let ticking: Promise<unknown> | null = null;
return {
name: "sources-identity",
runtime,
registerRoutes(app, ctx) {
const rt = () => runtime(ctx);
sourcesRoutes(app, rt);
workbenchRoutes(app, rt);
identityRoutes(app, rt);
decisionRoutes(app, rt);
},
registerTasks(registry, ctx) {
registerSourcesTasks(registry, () => runtime(ctx));
},
outboxRoutes: () => sourcesOutboxRoutes,
registerCapabilities(registry, ctx) {
registerSourcesCapabilities(registry, () => runtime(ctx));
},
start(ctx) {
const rt = runtime(ctx);
const tick = () => {
if (ticking) return;
ticking = schedulerTick(rt)
.catch((err) => ctx.logger.warn({ err }, "sources-identity 调度失败"))
.finally(() => {
ticking = null;
});
};
timer = setInterval(tick, overrides.schedulerIntervalMs ?? 30_000);
tick();
},
async stop() {
if (timer) clearInterval(timer);
timer = null;
await ticking;
},
};
}
export const sourcesIdentityModule = createSourcesIdentityModule();
@@ -0,0 +1,184 @@
import { type Decision, DecisionKind, DecisionStatus, Id, type PoolCandidate } from "@umt/contracts";
import { sql } from "@umt/db";
import { z } from "zod";
import { ApiException } from "../../../http/errors.ts";
import type { ModuleApp } from "../../types.ts";
import { D } from "../defs.ts";
import { route } from "../http.ts";
import { decodeCursor, jsonb, page, toApi, toApiList } from "../rows.ts";
import type { SourcesRuntime } from "../runtime.ts";
import { audit } from "../service/common.ts";
import { assignReleaseIdentity, dismissPoolEntry } from "../service/identity-admin.ts";
// 待处理中心数据源:decision 列表/详情/聚合/处理。
// 身份歧义类由本模块直接执行所选候选;其它类型只记录处理结果并发 decision.updated,
// 需要执行能力的选项由 W2-D 的 resolveDecision 消费。
const OPEN = ["open", "investigating", "awaiting_user"] as const;
const T = ["decisions"];
const SummaryGroup = z.object({
kind: DecisionKind,
/** 关联作品(身份歧义等尚未归属作品的为 null) */
mediaEntityId: Id.nullable(),
mediaEntityTitle: z.string().nullable(),
count: z.number().int(),
maxSeverity: z.enum(["info", "warning", "critical"]),
latestAt: z.string(),
});
export function decisionRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
const db = () => rt().ctx.db;
route(app, D.decisions.list, async ({ query }) => {
const off = decodeCursor(query.cursor);
let q = db().selectFrom("decision").selectAll();
q = query.status ? q.where("status", "=", query.status) : q.where("status", "in", [...OPEN]);
if (query.kind) q = q.where("kind", "=", query.kind);
const rows = await q
.orderBy(sql`case severity when 'critical' then 0 when 'warning' then 1 else 2 end`)
.orderBy("created_at", "desc")
.orderBy("id")
.offset(off)
.limit(query.limit + 1)
.execute();
const p = page(rows, off, query.limit);
return { items: toApiList<Decision>(p.items), nextCursor: p.nextCursor };
});
route(
app,
{
method: "get",
path: "/decisions/summary",
scope: "read",
summary: "未关闭待处理事项按作品与问题类型聚合",
tags: T,
query: z.object({ status: DecisionStatus.optional() }),
response: z.object({ total: z.number().int(), groups: z.array(SummaryGroup) }),
},
async ({ query }) => {
const statuses = query.status ? [query.status] : [...OPEN];
// 作品归属:evidence.mediaEntityId,或 subject 为 media_entity,或 subject 为已识别的 source_release
const rows = await sql<{
kind: z.infer<typeof DecisionKind>;
entity_id: string | null;
entity_title: string | null;
count: number;
max_sev: number;
latest: Date;
}>`
with d as (
select d.*, coalesce(
case when d.evidence->>'mediaEntityId' ~ '^[0-9a-f-]{36}$' then (d.evidence->>'mediaEntityId')::uuid end,
case when d.subject_type = 'media_entity' then d.subject_id end,
(select r.media_entity_id from source_release r where d.subject_type = 'source_release' and r.id = d.subject_id)
) as entity_id
from decision d where d.status = any(${statuses})
)
select d.kind, d.entity_id, e.title as entity_title, count(*)::int as count,
max(case d.severity when 'critical' then 2 when 'warning' then 1 else 0 end)::int as max_sev,
max(d.created_at) as latest
from d left join media_entity e on e.id = d.entity_id
group by d.kind, d.entity_id, e.title
order by max_sev desc, latest desc`.execute(db());
const sev = ["info", "warning", "critical"] as const;
return {
total: rows.rows.reduce((a, r) => a + r.count, 0),
groups: rows.rows.map((r) => ({
kind: r.kind,
mediaEntityId: r.entity_id,
mediaEntityTitle: r.entity_title,
count: r.count,
maxSeverity: sev[r.max_sev] ?? "info",
latestAt: new Date(r.latest).toISOString(),
})),
};
},
);
route(app, D.decisions.get, async ({ params }) => {
const row = await db().selectFrom("decision").selectAll().where("id", "=", params.id).executeTakeFirst();
if (!row) throw new ApiException("not_found", "待处理事项不存在");
return toApi(row);
});
route(app, D.decisions.resolve, async ({ params, body, actor }) => {
const d = await db().selectFrom("decision").selectAll().where("id", "=", params.id).executeTakeFirst();
if (!d) throw new ApiException("not_found", "待处理事项不存在");
if (!(OPEN as readonly string[]).includes(d.status)) throw new ApiException("conflict", `事项已是 ${d.status}`);
if (!body.dismiss && !body.optionKey) throw new ApiException("bad_request", "需要 optionKey 或 dismiss");
const by = `${actor.type}:${actor.id ?? ""}`;
const options = (d.options ?? []) as { key: string; input?: Record<string, unknown> }[];
const opt = body.optionKey ? options.find((o) => o.key === body.optionKey) : undefined;
if (body.optionKey && !opt) throw new ApiException("bad_request", `未知选项 ${body.optionKey}`);
// 身份歧义:直接执行(选候选 = 人工指定并加锁;忽略 = 待识别池 dismissed)
if (d.kind === "identity_ambiguous" && d.subject_type === "source_release" && d.subject_id) {
if (body.dismiss || opt?.key === "dismiss") {
const pool = await db()
.selectFrom("identity_candidate")
.select(["id"])
.where("source_release_id", "=", d.subject_id)
.executeTakeFirst();
if (pool) await dismissPoolEntry(rt(), pool.id, actor, body.note ?? null);
} else if (opt?.input && typeof opt.input.candidateIndex === "number") {
const pool = await db()
.selectFrom("identity_candidate")
.select(["candidates"])
.where("source_release_id", "=", d.subject_id)
.executeTakeFirst();
if (!(pool?.candidates as PoolCandidate[] | undefined)?.[opt.input.candidateIndex]) {
throw new ApiException("conflict", "候选已变化,请刷新后重试");
}
await assignReleaseIdentity(
rt(),
d.subject_id,
{ candidateIndex: opt.input.candidateIndex, lock: true, reason: body.note ?? `decision ${d.id}` },
actor,
);
}
}
// 上面的执行路径已按 dedupeKey 关闭;其余类型在这里记录处理结果
const row = await db()
.transaction()
.execute(async (trx) => {
const cur = await trx
.selectFrom("decision")
.selectAll()
.where("id", "=", params.id)
.forUpdate()
.executeTakeFirstOrThrow();
if (!(OPEN as readonly string[]).includes(cur.status)) return cur;
const status = body.dismiss || opt?.key === "dismiss" ? "dismissed" : "resolved";
const r = await trx
.updateTable("decision")
.set({
status,
resolution: jsonb({ optionKey: body.optionKey ?? null, note: body.note ?? null, by }),
resolved_by: by,
resolved_at: new Date(),
})
.where("id", "=", params.id)
.returningAll()
.executeTakeFirstOrThrow();
await rt().ctx.outbox.emit(
trx,
"decision.updated",
{ decisionId: r.id, status },
{
idempotencyKey: `decision.updated:${r.id}:${status}`,
},
);
await audit(
trx,
actor,
"decision.resolved",
{ type: "decision", id: r.id },
{ optionKey: body.optionKey ?? null, status },
);
return r;
});
return toApi(row);
});
}
@@ -0,0 +1,597 @@
import { randomUUID } from "node:crypto";
import {
type ContentUnit,
cursorPage,
EntityIdentitySet,
EntityMetadataField,
type ExternalIdentity,
FieldOverrideInput,
Id,
IdentityCandidate,
IdentityPoolItem,
IdentityPoolQuery,
IdParam,
type MediaEntity,
MediaEntityDetail,
OkResult,
ReleaseIdentityAssign,
type TitleAlias,
} from "@umt/contracts";
import { sql } from "@umt/db";
import { normalizeTitle } from "@umt/domain";
import { z } from "zod";
import { ApiException } from "../../../http/errors.ts";
import type { ModuleApp } from "../../types.ts";
import { D } from "../defs.ts";
import { route } from "../http.ts";
import { decodeCursor, jsonb, page, toApi, toApiList } from "../rows.ts";
import type { SourcesRuntime } from "../runtime.ts";
import { audit } from "../service/common.ts";
import { resolveEntityId } from "../service/entities.ts";
import {
assignReleaseIdentity,
dismissPoolEntry,
mergeEntities,
setEntityIdentity,
updateExternalIdentity,
} from "../service/identity-admin.ts";
import { clearFieldOverride, setFieldOverride } from "../service/metadata.ts";
// 身份与元数据 API:作品(列表/详情/纠错/合并/字段覆盖)、内容单元、别名、外部身份、待识别池。
const T_ENT = ["media-entities"];
const T_ID = ["identity"];
const Accepted = z.object({ taskId: Id });
const FieldParam = z.object({ id: Id, field: EntityMetadataField });
export function identityRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
const db = () => rt().ctx.db;
// ── 作品 ──
route(app, D.mediaEntities.list, async ({ query }) => {
const off = decodeCursor(query.cursor);
let q = db().selectFrom("media_entity").selectAll();
if (query.kind) q = q.where("kind", "=", query.kind);
q = query.status ? q.where("status", "=", query.status) : q.where("status", "<>", "merged");
if (query.q) {
const n = normalizeTitle(query.q);
q = q.where((eb) =>
eb.or([
eb("title", "ilike", `%${query.q}%`),
eb.exists(
eb
.selectFrom("title_alias as a")
.select("a.id")
.whereRef("a.media_entity_id", "=", "media_entity.id")
.where(sql<boolean>`(a.normalized = ${n} or similarity(a.normalized, ${n}) >= 0.4)`),
),
]),
);
}
const rows = await q
.orderBy("updated_at", "desc")
.orderBy("id")
.offset(off)
.limit(query.limit + 1)
.execute();
const p = page(rows, off, query.limit);
return { items: toApiList<MediaEntity>(p.items), nextCursor: p.nextCursor };
});
route(app, D.mediaEntities.get, async ({ params }) => {
const id = await resolveEntityId(db(), params.id);
if (!id) throw new ApiException("not_found", "作品不存在");
return toApi(await db().selectFrom("media_entity").selectAll().where("id", "=", id).executeTakeFirstOrThrow());
});
route(
app,
{
method: "get",
path: "/media-entities/{id}/detail",
scope: "read",
summary: "作品详情:外部身份、别名、内容单元、字段来源与覆盖(旧 ID 自动跳转)",
tags: T_ENT,
params: IdParam,
response: MediaEntityDetail,
},
async ({ params }) => {
const id = await resolveEntityId(db(), params.id);
if (!id) throw new ApiException("not_found", "作品不存在");
const [entity, identities, aliases, units, fields, overrides] = await Promise.all([
db().selectFrom("media_entity").selectAll().where("id", "=", id).executeTakeFirstOrThrow(),
db()
.selectFrom("external_identity")
.selectAll()
.where("media_entity_id", "=", id)
.orderBy("created_at")
.execute(),
db().selectFrom("title_alias").selectAll().where("media_entity_id", "=", id).orderBy("created_at").execute(),
db()
.selectFrom("content_unit")
.selectAll()
.where("media_entity_id", "=", id)
.orderBy("season_number")
.orderBy("episode_number")
.execute(),
db().selectFrom("metadata_field").selectAll().where("media_entity_id", "=", id).orderBy("field").execute(),
db().selectFrom("field_override").selectAll().where("media_entity_id", "=", id).orderBy("field").execute(),
]);
return {
entity: toApi(entity),
redirectedFrom: id === params.id ? null : params.id,
identities: toApiList(identities),
aliases: toApiList(aliases),
contentUnits: toApiList(units),
metadataFields: toApiList(fields),
overrides: toApiList(overrides),
};
},
);
route(app, D.mediaEntities.create, async ({ body, actor }) => {
const row = await db()
.transaction()
.execute(async (trx) => {
const r = await trx
.insertInto("media_entity")
.values({
kind: body.kind,
title: body.title,
original_title: body.originalTitle ?? null,
year: body.year ?? null,
status: body.status === "merged" ? "active" : (body.status ?? "active"),
metadata: jsonb(body.metadata ?? {}),
field_locks: body.fieldLocks ?? [],
})
.returningAll()
.executeTakeFirstOrThrow();
await sql`insert into title_alias (media_entity_id, alias, normalized, kind, source, locked)
values (${r.id}, ${body.title}, ${normalizeTitle(body.title) || body.title}, 'translated', 'user', true)
on conflict do nothing`.execute(trx);
await audit(trx, actor, "media_entity.created", { type: "media_entity", id: r.id }, { title: body.title });
return r;
});
return toApi(row);
});
route(app, D.mediaEntities.update, async ({ params, body, actor }) => {
const id = await resolveEntityId(db(), params.id);
if (!id) throw new ApiException("not_found", "作品不存在");
// 标题/原名/年份按字段覆盖处理(加锁),其余字段直接更新
const locked: [z.infer<typeof EntityMetadataField>, unknown][] = [];
if (body.title !== undefined) locked.push(["title", body.title]);
if (body.originalTitle !== undefined) locked.push(["originalTitle", body.originalTitle]);
if (body.year !== undefined) locked.push(["year", body.year]);
for (const [field, value] of locked) await setFieldOverride(rt(), id, field, { value, lock: true }, actor);
const patch: Record<string, unknown> = {};
if (body.kind !== undefined) patch.kind = body.kind;
if (body.status !== undefined) patch.status = body.status;
if (Object.keys(patch).length) {
await db().updateTable("media_entity").set(patch).where("id", "=", id).execute();
await audit(db(), actor, "media_entity.updated", { type: "media_entity", id }, patch);
}
return toApi(await db().selectFrom("media_entity").selectAll().where("id", "=", id).executeTakeFirstOrThrow());
});
route(app, D.mediaEntities.merge, async ({ params, body, actor }) =>
toApi(await mergeEntities(rt(), params.id, body.intoId, body.reason ?? null, actor)),
);
route(
app,
{
method: "post",
path: "/media-entities/{id}/identities",
scope: "organize.execute",
summary: "人工纠错外部映射:旧映射退役(留修订),新映射 accepted + 人工锁",
tags: T_ID,
params: IdParam,
body: EntityIdentitySet,
response: z.object({
mediaEntityId: Id,
changed: z.boolean(),
identityId: Id.optional(),
retired: z.array(Id).optional(),
}),
},
async ({ params, body, actor }) => setEntityIdentity(rt(), params.id, body, actor),
);
route(
app,
{
method: "post",
path: "/media-entities/{id}/refresh",
scope: "organize.execute",
summary: "刷新元数据(字段级;用户锁优先,不动路径)",
tags: T_ENT,
params: IdParam,
response: Accepted,
},
async ({ params }) => {
const id = await resolveEntityId(db(), params.id);
if (!id) throw new ApiException("not_found", "作品不存在");
const r = await rt().ctx.engine.enqueue(
"refreshMetadata",
{ mediaEntityId: id },
{ idempotencyKey: `refreshMetadata:${id}:${Math.floor(Date.now() / 60_000)}`, traceId: randomUUID() },
);
return { taskId: r.runId };
},
);
const FieldResult = z.object({
mediaEntityId: Id,
changedFields: z.array(z.string()),
lockedFields: z.array(z.string()),
});
route(
app,
{
method: "put",
path: "/media-entities/{id}/fields/{field}",
scope: "organize.execute",
summary: "覆盖作品字段并锁定(自动刷新不再改写)",
tags: T_ENT,
params: FieldParam,
body: FieldOverrideInput,
response: FieldResult,
},
async ({ params, body, actor }) => setFieldOverride(rt(), params.id, params.field, body, actor),
);
route(
app,
{
method: "delete",
path: "/media-entities/{id}/fields/{field}",
scope: "organize.execute",
summary: "取消字段覆盖与锁定",
tags: T_ENT,
params: FieldParam,
response: FieldResult,
},
async ({ params, actor }) => clearFieldOverride(rt(), params.id, params.field, actor),
);
// ── 内容单元 ──
route(app, D.contentUnits.list, async ({ query }) => {
const off = decodeCursor(query.cursor);
let q = db().selectFrom("content_unit").selectAll();
if (query.mediaEntityId) q = q.where("media_entity_id", "=", query.mediaEntityId);
if (query.seasonNumber !== undefined) q = q.where("season_number", "=", query.seasonNumber);
const rows = await q
.orderBy("media_entity_id")
.orderBy("season_number")
.orderBy("episode_number")
.orderBy("id")
.offset(off)
.limit(query.limit + 1)
.execute();
const p = page(rows, off, query.limit);
return { items: toApiList<ContentUnit>(p.items), nextCursor: p.nextCursor };
});
route(app, D.contentUnits.get, async ({ params }) => {
const row = await db().selectFrom("content_unit").selectAll().where("id", "=", params.id).executeTakeFirst();
if (!row) throw new ApiException("not_found", "内容单元不存在");
return toApi(row);
});
route(app, D.contentUnits.update, async ({ params, body, actor }) => {
const unit = await db().selectFrom("content_unit").selectAll().where("id", "=", params.id).executeTakeFirst();
if (!unit) throw new ApiException("not_found", "内容单元不存在");
const numbering = ["seasonNumber", "episodeNumber", "sourceIssue", "airDate"] as const;
const touchesNumbering = numbering.some((k) => body[k] !== undefined);
const patch: Record<string, unknown> = {};
const map: Record<string, string> = {
parentUnitId: "parent_unit_id",
kind: "kind",
contentType: "content_type",
segment: "segment",
seasonNumber: "season_number",
episodeNumber: "episode_number",
sourceIssue: "source_issue",
airDate: "air_date",
displayOrder: "display_order",
title: "title",
status: "status",
numberingLocked: "numbering_locked",
fieldLocks: "field_locks",
};
for (const [k, col] of Object.entries(map)) {
const v = (body as Record<string, unknown>)[k];
if (v !== undefined) patch[col] = v;
}
if (body.metadata !== undefined) patch.metadata = jsonb(body.metadata);
// 人工改编号即确认并锁定编号(REQ §5.4:编号可修订,人工确认后刷新不得改)
if (touchesNumbering && body.numberingLocked === undefined) patch.numbering_locked = true;
if (touchesNumbering && body.status === undefined && unit.status === "provisional") patch.status = "confirmed";
const row = await db()
.updateTable("content_unit")
.set(patch)
.where("id", "=", params.id)
.returningAll()
.executeTakeFirstOrThrow();
await audit(
db(),
actor,
"content_unit.updated",
{ type: "content_unit", id: params.id },
{ fields: Object.keys(patch) },
);
return toApi(row);
});
// ── 别名 ──
route(app, D.titleAliases.list, async ({ query }) => {
const off = decodeCursor(query.cursor);
let q = db().selectFrom("title_alias").selectAll();
if (query.mediaEntityId) q = q.where("media_entity_id", "=", query.mediaEntityId);
if (query.q) {
const n = normalizeTitle(query.q);
q = q.where(sql<boolean>`(normalized = ${n} or similarity(normalized, ${n}) >= 0.3)`);
}
const rows = await q
.orderBy("created_at")
.orderBy("id")
.offset(off)
.limit(query.limit + 1)
.execute();
const p = page(rows, off, query.limit);
return { items: toApiList<TitleAlias>(p.items), nextCursor: p.nextCursor };
});
route(app, D.titleAliases.create, async ({ body, actor }) => {
const id = await resolveEntityId(db(), body.mediaEntityId);
if (!id) throw new ApiException("not_found", "作品不存在");
const normalized = normalizeTitle(body.alias);
if (!normalized) throw new ApiException("bad_request", "别名归一化后为空");
try {
const row = await db()
.insertInto("title_alias")
.values({
media_entity_id: id,
content_unit_id: body.contentUnitId ?? null,
alias: body.alias,
normalized,
language: body.language ?? null,
kind: body.kind ?? "other",
source: body.source,
locked: body.locked ?? body.source === "user",
})
.returningAll()
.executeTakeFirstOrThrow();
await audit(db(), actor, "title_alias.created", { type: "title_alias", id: row.id }, { alias: body.alias });
return toApi(row);
} catch (e) {
if ((e as { code?: string }).code === "23505") throw new ApiException("conflict", "别名已存在");
throw e;
}
});
route(app, D.titleAliases.remove, async ({ params, actor }) => {
const r = await db().deleteFrom("title_alias").where("id", "=", params.id).returning(["id"]).executeTakeFirst();
if (!r) throw new ApiException("not_found", "别名不存在");
await audit(db(), actor, "title_alias.deleted", { type: "title_alias", id: params.id });
return OkResult.parse({ ok: true });
});
// ── 外部身份 ──
route(app, D.externalIdentities.list, async ({ query }) => {
const off = decodeCursor(query.cursor);
let q = db().selectFrom("external_identity").selectAll();
if (query.mediaEntityId) q = q.where("media_entity_id", "=", query.mediaEntityId);
if (query.provider) q = q.where("provider", "=", query.provider);
if (query.status) q = q.where("status", "=", query.status);
const rows = await q
.orderBy("created_at")
.orderBy("id")
.offset(off)
.limit(query.limit + 1)
.execute();
const p = page(rows, off, query.limit);
return { items: toApiList<ExternalIdentity>(p.items), nextCursor: p.nextCursor };
});
route(app, D.externalIdentities.get, async ({ params }) => {
const row = await db().selectFrom("external_identity").selectAll().where("id", "=", params.id).executeTakeFirst();
if (!row) throw new ApiException("not_found", "外部身份不存在");
return toApi(row);
});
route(app, D.externalIdentities.create, async ({ body, actor }) => {
if (!body.mediaEntityId) throw new ApiException("bad_request", "v0.1 只支持作品级外部身份");
const entityId = await resolveEntityId(db(), body.mediaEntityId);
if (!entityId) throw new ApiException("not_found", "作品不存在");
if ((body.status ?? "proposed") === "accepted") {
const r = await setEntityIdentity(
rt(),
entityId,
{
provider: body.provider,
entityType: body.entityType as "movie" | "tv",
externalId: body.externalId,
lock: body.locked ?? true,
},
actor,
);
if (!r.identityId) throw new ApiException("conflict", "该外部身份已是此作品的 accepted 映射");
return toApi(
await db().selectFrom("external_identity").selectAll().where("id", "=", r.identityId).executeTakeFirstOrThrow(),
);
}
const row = await db()
.transaction()
.execute(async (trx) => {
const r = await trx
.insertInto("external_identity")
.values({
media_entity_id: entityId,
provider: body.provider,
entity_type: body.entityType,
external_id: body.externalId,
status: body.status ?? "proposed",
evidence_level: body.evidenceLevel ?? "manual",
evidence: jsonb(body.evidence ?? {}),
locked: body.locked ?? false,
extra: jsonb(body.extra ?? {}),
})
.returningAll()
.executeTakeFirstOrThrow();
await trx
.insertInto("identity_revision")
.values({
external_identity_id: r.id,
revision: 1,
action: "created",
to_status: r.status,
after: jsonb({ status: r.status }),
actor_type: actor.type,
actor_id: actor.id,
})
.execute();
return r;
});
return toApi(row);
});
route(app, D.externalIdentities.update, async ({ params, body, actor }) =>
toApi(await updateExternalIdentity(rt(), params.id, body, actor)),
);
route(app, D.externalIdentities.revisions, async ({ params }) => {
const rows = await db()
.selectFrom("identity_revision")
.selectAll()
.where("external_identity_id", "=", params.id)
.orderBy("revision")
.execute();
return { items: toApiList(rows) };
});
// ── 待识别池 ──
route(
app,
{
method: "get",
path: "/identity-pool",
scope: "read",
summary: "待识别池(可按原始标题检索;候选不建正式作品)",
tags: T_ID,
query: IdentityPoolQuery,
response: cursorPage(IdentityPoolItem),
},
async ({ query }) => {
const off = decodeCursor(query.cursor);
let q = db()
.selectFrom("identity_candidate as p")
.innerJoin("source_release as r", "r.id", "p.source_release_id")
.selectAll("p")
.select([
"r.site_id as r_site_id",
"r.site_release_id as r_site_release_id",
"r.title as r_title",
"r.subtitle as r_subtitle",
"r.category_raw as r_category_raw",
"r.first_seen_at as r_first_seen_at",
"r.resolution_status as r_resolution_status",
]);
q = q.where("p.status", "=", query.status ?? "pending");
if (query.reason) q = q.where("p.reason", "=", query.reason);
if (query.siteId) q = q.where("r.site_id", "=", query.siteId);
if (query.q) {
const n = normalizeTitle(query.q);
q = q.where((eb) =>
eb.or([
eb("r.title", "ilike", `%${query.q}%`),
eb("r.subtitle", "ilike", `%${query.q}%`),
sql<boolean>`(p.query->>'normalized') = ${n}`,
sql<boolean>`similarity(r.title, ${query.q}) >= 0.3`,
]),
);
}
const rows = await q
.orderBy("p.updated_at", "desc")
.orderBy("p.id")
.offset(off)
.limit(query.limit + 1)
.execute();
const p = page(rows, off, query.limit);
return {
items: p.items.map((row) => {
const {
r_site_id,
r_site_release_id,
r_title,
r_subtitle,
r_category_raw,
r_first_seen_at,
r_resolution_status,
...rest
} = row;
return {
...toApi<Record<string, unknown>>(rest),
release: {
id: rest.source_release_id,
siteId: r_site_id,
siteReleaseId: r_site_release_id,
title: r_title,
subtitle: r_subtitle,
categoryRaw: r_category_raw,
firstSeenAt: r_first_seen_at.toISOString(),
resolutionStatus: r_resolution_status,
},
};
}),
nextCursor: p.nextCursor,
};
},
);
route(
app,
{
method: "post",
path: "/identity-pool/{id}/assign",
scope: "organize.execute",
summary: "为待识别资源手动指定候选 / 本地作品 / 外部身份",
tags: T_ID,
params: IdParam,
body: ReleaseIdentityAssign,
response: z.object({ mediaEntityId: Id, contentUnitIds: z.array(Id) }),
},
async ({ params, body, actor }) => {
const pool = await db()
.selectFrom("identity_candidate")
.select(["source_release_id"])
.where("id", "=", params.id)
.executeTakeFirst();
if (!pool) throw new ApiException("not_found", "待识别条目不存在");
return assignReleaseIdentity(rt(), pool.source_release_id, body, actor);
},
);
route(
app,
{
method: "post",
path: "/identity-pool/{id}/dismiss",
scope: "organize.execute",
summary: "忽略待识别资源(不再自动复查)",
tags: T_ID,
params: IdParam,
body: z.object({ note: z.string().max(2000).optional() }),
response: IdentityCandidate,
},
async ({ params, body, actor }) => toApi(await dismissPoolEntry(rt(), params.id, actor, body.note ?? null)),
);
route(
app,
{
method: "post",
path: "/identity-pool/{id}/retry",
scope: "organize.execute",
summary: "立即重新识别",
tags: T_ID,
params: IdParam,
response: Accepted,
},
async ({ params }) => {
const pool = await db()
.selectFrom("identity_candidate")
.select(["source_release_id", "attempts"])
.where("id", "=", params.id)
.executeTakeFirst();
if (!pool) throw new ApiException("not_found", "待识别条目不存在");
const r = await rt().ctx.engine.enqueue(
"resolveObservation",
{ sourceReleaseId: pool.source_release_id, force: true },
{ idempotencyKey: `resolve:retry:${params.id}:${Math.floor(Date.now() / 10_000)}`, traceId: randomUUID() },
);
return { taskId: r.runId };
},
);
}
@@ -0,0 +1,635 @@
import { randomUUID } from "node:crypto";
import { listSiteDefinitions, redactUrl, type SiteDefinition } from "@umt/adapters";
import {
CursorQuery,
cursorPage,
FeedSource,
Id,
IdParam,
type Observation,
ReleaseIdentityAssign,
type SecretPurpose,
Site,
type SiteAccount,
type SiteAdapter,
SiteDefinitionView,
SiteSnapshot,
type SourceRelease,
} from "@umt/contracts";
import { sql } from "@umt/db";
import { z } from "zod";
import { ApiException } from "../../../http/errors.ts";
import type { ModuleApp } from "../../types.ts";
import { D } from "../defs.ts";
import { route } from "../http.ts";
import { decodeCursor, jsonb, page, toApi, toApiList } from "../rows.ts";
import type { SourcesRuntime } from "../runtime.ts";
import { audit, closeDecisions } from "../service/common.ts";
import { assignReleaseIdentity } from "../service/identity-admin.ts";
// 站点、站点账号、Feed、来源资源、观察的管理 API。凭证只写(存 secret,对外只给 secret_ref / 脱敏 URL)。
const T_SITES = ["sites"];
const T_FEEDS = ["feed-sources"];
const T_RELEASES = ["source-releases"];
export function adapterOf(def: SiteDefinition): SiteAdapter {
return def.umt.kind === "mteam" ? "mteam_api" : def.umt.kind === "api" ? "api" : "nexusphp";
}
const SECRET_PURPOSE: Record<string, SecretPurpose> = {
cookie: "site_cookie",
passkey: "site_passkey",
api_key: "site_api_key",
password: "site_password",
};
const Accepted = z.object({ taskId: Id });
export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
const db = () => rt().ctx.db;
// ── 站点注册表 ──
route(
app,
{
method: "get",
path: "/site-definitions",
scope: "read",
summary: "随产品发布的站点适配器定义(含是否已登记)",
tags: T_SITES,
response: z.object({ items: z.array(SiteDefinitionView) }),
},
async () => {
const rows = await db().selectFrom("site").select(["id", "key"]).execute();
const byKey = new Map(rows.map((r) => [r.key, r.id]));
return {
items: listSiteDefinitions().map((d) => ({
key: d.id,
name: d.umt.kind === "nexusphp" ? (d.umt.displayName ?? d.name) : d.umt.displayName,
adapter: adapterOf(d),
baseUrl: d.domain,
shippedEnabled: d.enabled !== false,
definitionVersion: d.umt.version,
capabilities: d.umt.capabilities,
defaultRateLimit: d.umt.rateLimit,
authSupported: ("authSupported" in d.umt ? d.umt.authSupported : undefined) ?? [],
siteId: byKey.get(d.id) ?? null,
})),
};
},
);
// ── 站点 ──
route(app, D.sites.list, async ({ query }) => {
const off = decodeCursor(query.cursor);
const rows = await db()
.selectFrom("site")
.selectAll()
.orderBy("key")
.offset(off)
.limit(query.limit + 1)
.execute();
const p = page(rows, off, query.limit);
return { items: toApiList<Site>(p.items), nextCursor: p.nextCursor };
});
route(app, D.sites.get, async ({ params }) => toApi(await getSite(params.id)));
route(app, D.sites.create, async ({ body, actor }) => {
const def = listSiteDefinitions().find((d) => d.id === body.key);
if (def && adapterOf(def) !== body.adapter) {
throw new ApiException("bad_request", `站点 ${body.key} 的适配器类型应为 ${adapterOf(def)}`);
}
if (!def && body.adapter !== "generic_rss") {
throw new ApiException("bad_request", `注册表中没有站点 ${body.key};未适配站点只能以 generic_rss 登记`);
}
try {
const row = await db()
.insertInto("site")
.values({
key: body.key,
name: body.name,
adapter: body.adapter,
base_url: body.baseUrl,
enabled: body.enabled ?? true,
rate_limit: jsonb(body.rateLimit ?? {}),
config: jsonb(body.config ?? {}),
snapshot_enabled: body.snapshotEnabled ?? false,
snapshot_interval_sec: body.snapshotIntervalSec ?? null,
})
.returningAll()
.executeTakeFirstOrThrow();
await audit(db(), actor, "site.created", { type: "site", id: row.id }, { key: body.key });
return toApi(row);
} catch (e) {
if ((e as { code?: string }).code === "23505") throw new ApiException("conflict", `站点 ${body.key} 已登记`);
throw e;
}
});
route(app, D.sites.update, async ({ params, body, actor }) => {
await getSite(params.id);
const patch: Record<string, unknown> = {};
if (body.name !== undefined) patch.name = body.name;
if (body.adapter !== undefined) patch.adapter = body.adapter;
if (body.baseUrl !== undefined) patch.base_url = body.baseUrl;
if (body.rateLimit !== undefined) patch.rate_limit = jsonb(body.rateLimit);
if (body.config !== undefined) patch.config = jsonb(body.config);
if (body.snapshotEnabled !== undefined) patch.snapshot_enabled = body.snapshotEnabled;
if (body.snapshotIntervalSec !== undefined) patch.snapshot_interval_sec = body.snapshotIntervalSec;
if (body.enabled !== undefined) return toApi(await setSiteEnabled(params.id, body.enabled, actor));
const row = await db()
.updateTable("site")
.set(patch)
.where("id", "=", params.id)
.returningAll()
.executeTakeFirstOrThrow();
await audit(db(), actor, "site.updated", { type: "site", id: params.id }, { fields: Object.keys(patch) });
return toApi(row);
});
for (const enabled of [true, false]) {
route(
app,
{
method: "post",
path: `/sites/{id}/${enabled ? "enable" : "disable"}`,
scope: "admin",
summary: enabled ? "开启站点" : "关闭站点(停止网络访问与新候选,历史资源仍可查)",
tags: T_SITES,
params: IdParam,
response: Site,
},
async ({ params, actor }) => toApi(await setSiteEnabled(params.id, enabled, actor)),
);
}
route(
app,
{
method: "post",
path: "/sites/{id}/snapshot",
scope: "admin",
summary: "立即同步站点快照(受站点限流)",
tags: T_SITES,
params: IdParam,
body: z.object({ maxPages: z.number().int().min(1).max(20).optional() }),
response: Accepted,
},
async ({ params, body }) => {
const s = await getSite(params.id);
if (!s.enabled) throw new ApiException("precondition_failed", "站点已关闭");
const r = await rt().ctx.engine.enqueue(
"snapshotSite",
{ siteId: s.id, kind: "manual", ...(body.maxPages ? { maxPages: body.maxPages } : {}) },
{ idempotencyKey: `snapshotSite:manual:${s.id}:${Math.floor(Date.now() / 10_000)}`, traceId: randomUUID() },
);
return { taskId: r.runId };
},
);
route(
app,
{
method: "get",
path: "/sites/{id}/snapshots",
scope: "read",
summary: "站点快照/补查记录(覆盖范围与新鲜度)",
tags: T_SITES,
params: IdParam,
query: CursorQuery,
response: cursorPage(SiteSnapshot),
},
async ({ params, query }) => {
const off = decodeCursor(query.cursor);
const rows = await db()
.selectFrom("site_snapshot")
.selectAll()
.where("site_id", "=", params.id)
.orderBy("started_at", "desc")
.offset(off)
.limit(query.limit + 1)
.execute();
const p = page(rows, off, query.limit);
return { items: toApiList(p.items), nextCursor: p.nextCursor };
},
);
// ── 站点账号 ──
route(app, D.siteAccounts.list, async ({ query }) => {
const off = decodeCursor(query.cursor);
let q = db().selectFrom("site_account").selectAll();
if (query.siteId) q = q.where("site_id", "=", query.siteId);
const rows = await q
.orderBy("created_at")
.offset(off)
.limit(query.limit + 1)
.execute();
const p = page(rows, off, query.limit);
return { items: toApiList<SiteAccount>(p.items), nextCursor: p.nextCursor };
});
route(app, D.siteAccounts.get, async ({ params }) => {
const row = await db().selectFrom("site_account").selectAll().where("id", "=", params.id).executeTakeFirst();
if (!row) throw new ApiException("not_found", "站点账号不存在");
return toApi(row);
});
route(app, D.siteAccounts.create, async ({ body, actor }) => {
await getSite(body.siteId);
const ref = await rt().ctx.secrets.create({
purpose: SECRET_PURPOSE[body.authKind]!,
plaintext: body.credential,
label: `site_account:${body.label}`,
});
try {
const row = await db()
.insertInto("site_account")
.values({
site_id: body.siteId,
label: body.label,
auth_kind: body.authKind,
secret_ref: ref,
enabled: body.enabled ?? true,
})
.returningAll()
.executeTakeFirstOrThrow();
await audit(db(), actor, "site_account.created", { type: "site_account", id: row.id }, { siteId: body.siteId });
return toApi(row);
} catch (e) {
await rt()
.ctx.secrets.remove(ref)
.catch(() => {});
if ((e as { code?: string }).code === "23505") throw new ApiException("conflict", "同站点已有同名账号");
throw e;
}
});
route(app, D.siteAccounts.update, async ({ params, body, actor }) => {
const acc = await db().selectFrom("site_account").selectAll().where("id", "=", params.id).executeTakeFirst();
if (!acc) throw new ApiException("not_found", "站点账号不存在");
const patch: Record<string, unknown> = {};
if (body.label !== undefined) patch.label = body.label;
if (body.enabled !== undefined) patch.enabled = body.enabled;
if (body.authKind !== undefined) patch.auth_kind = body.authKind;
if (body.credential !== undefined) {
if (acc.secret_ref) await rt().ctx.secrets.replace(acc.secret_ref, body.credential);
else {
patch.secret_ref = await rt().ctx.secrets.create({
purpose: SECRET_PURPOSE[body.authKind ?? acc.auth_kind]!,
plaintext: body.credential,
label: `site_account:${acc.label}`,
});
}
// 换了凭证:状态回到未验证,关闭"登录失效"待处理,站点健康复位
patch.status = "unknown";
}
const row = await db()
.transaction()
.execute(async (trx) => {
const r = await trx
.updateTable("site_account")
.set(patch)
.where("id", "=", params.id)
.returningAll()
.executeTakeFirstOrThrow();
if (body.credential !== undefined) {
await trx
.updateTable("site")
.set({ health_status: "unknown" })
.where("id", "=", acc.site_id)
.where("health_status", "=", "failing")
.execute();
await closeDecisions(
rt().ctx,
trx,
`site-auth:${acc.site_id}`,
"resolved",
{ by: "credential_updated" },
`${actor.type}:${actor.id}`,
);
}
await audit(
trx,
actor,
"site_account.updated",
{ type: "site_account", id: params.id },
{
fields: Object.keys(patch).filter((k) => k !== "secret_ref"),
credentialRotated: body.credential !== undefined,
},
);
return r;
});
return toApi(row);
});
// ── Feed ──
route(app, D.feedSources.list, async ({ query }) => {
const off = decodeCursor(query.cursor);
let q = db().selectFrom("feed_source").selectAll();
if (query.siteId) q = q.where("site_id", "=", query.siteId);
if (query.health) q = q.where("health_status", "=", query.health);
const rows = await q
.orderBy("created_at")
.offset(off)
.limit(query.limit + 1)
.execute();
const p = page(rows, off, query.limit);
return { items: toApiList<FeedSource>(p.items), nextCursor: p.nextCursor };
});
route(app, D.feedSources.get, async ({ params }) => toApi(await getFeed(params.id)));
route(app, D.feedSources.create, async ({ body, actor }) => {
if (body.siteId) await getSite(body.siteId);
if (body.siteAccountId) await checkAccount(body.siteAccountId, body.siteId ?? null);
const ref = await rt().ctx.secrets.create({ purpose: "feed_url", plaintext: body.url, label: `feed:${body.name}` });
const row = await db()
.insertInto("feed_source")
.values({
site_id: body.siteId ?? null,
site_account_id: body.siteAccountId ?? null,
name: body.name,
purpose: body.purpose ?? "generic",
kind: body.kind ?? "rss",
url_secret_ref: ref,
url_display: redactUrl(body.url),
enabled: body.enabled ?? true,
poll_interval_sec: body.pollIntervalSec ?? 900,
filter: jsonb(body.filter ?? {}),
})
.returningAll()
.executeTakeFirstOrThrow();
await audit(db(), actor, "feed_source.created", { type: "feed_source", id: row.id }, { url: row.url_display });
return toApi(row);
});
route(app, D.feedSources.update, async ({ params, body, actor }) => {
const feed = await getFeed(params.id);
const patch: Record<string, unknown> = {};
if (body.siteId !== undefined) {
if (body.siteId) await getSite(body.siteId);
patch.site_id = body.siteId;
}
if (body.siteAccountId !== undefined) {
if (body.siteAccountId) await checkAccount(body.siteAccountId, body.siteId ?? feed.site_id);
patch.site_account_id = body.siteAccountId;
}
if (body.name !== undefined) patch.name = body.name;
if (body.purpose !== undefined) patch.purpose = body.purpose;
if (body.kind !== undefined) patch.kind = body.kind;
if (body.enabled !== undefined) {
patch.enabled = body.enabled;
patch.health_status = body.enabled ? "unknown" : "disabled";
}
if (body.pollIntervalSec !== undefined) patch.poll_interval_sec = body.pollIntervalSec;
if (body.filter !== undefined) patch.filter = jsonb(body.filter);
if (body.url !== undefined) {
if (feed.url_secret_ref) await rt().ctx.secrets.replace(feed.url_secret_ref, body.url);
else
patch.url_secret_ref = await rt().ctx.secrets.create({
purpose: "feed_url",
plaintext: body.url,
label: `feed:${feed.name}`,
});
patch.url_display = redactUrl(body.url);
// 换了链接:清空条件请求缓存(游标保留,用于去重)
patch.etag = null;
patch.last_modified = null;
}
const row = await db()
.updateTable("feed_source")
.set(patch)
.where("id", "=", params.id)
.returningAll()
.executeTakeFirstOrThrow();
await audit(
db(),
actor,
"feed_source.updated",
{ type: "feed_source", id: params.id },
{
fields: Object.keys(patch).filter((k) => k !== "url_secret_ref"),
},
);
return toApi(row);
});
for (const enabled of [true, false]) {
route(
app,
{
method: "post",
path: `/feed-sources/{id}/${enabled ? "enable" : "disable"}`,
scope: "admin",
summary: enabled ? "启用 Feed" : "停用 Feed",
tags: T_FEEDS,
params: IdParam,
response: FeedSource,
},
async ({ params, actor }) => {
await getFeed(params.id);
const row = await db()
.updateTable("feed_source")
.set({
enabled,
health_status: enabled ? "unknown" : "disabled",
...(enabled ? { next_fetch_at: new Date() } : {}),
})
.where("id", "=", params.id)
.returningAll()
.executeTakeFirstOrThrow();
await audit(db(), actor, enabled ? "feed_source.enabled" : "feed_source.disabled", {
type: "feed_source",
id: params.id,
});
return toApi(row);
},
);
}
route(
app,
{
method: "post",
path: "/feed-sources/{id}/reset",
scope: "admin",
summary: "手动恢复:关闭熔断、清空退避(凭证已更新后使用)",
tags: T_FEEDS,
params: IdParam,
response: FeedSource,
},
async ({ params, actor }) => {
const feed = await getFeed(params.id);
const cursor = (feed.cursor ?? {}) as { window?: unknown };
const row = await db()
.transaction()
.execute(async (trx) => {
const r = await trx
.updateTable("feed_source")
.set({
consecutive_failures: 0,
backoff_until: null,
last_error: null,
health_status: feed.enabled ? "unknown" : "disabled",
next_fetch_at: new Date(),
cursor: jsonb({ window: cursor.window ?? null }),
})
.where("id", "=", params.id)
.returningAll()
.executeTakeFirstOrThrow();
if (feed.site_account_id) {
await trx
.updateTable("site_account")
.set({ status: "unknown" })
.where("id", "=", feed.site_account_id)
.where("status", "in", ["invalid", "expired"])
.execute();
}
await closeDecisions(
rt().ctx,
trx,
`feed-auth:${params.id}`,
"resolved",
{ by: "reset" },
`${actor.type}:${actor.id}`,
);
await audit(trx, actor, "feed_source.reset", { type: "feed_source", id: params.id });
return r;
});
return toApi(row);
},
);
route(app, D.feedSources.fetchNow, async ({ params }) => {
const feed = await getFeed(params.id);
if (!feed.enabled) throw new ApiException("precondition_failed", "Feed 已停用");
const r = await rt().ctx.engine.enqueue(
"ingestFeed",
{ feedSourceId: feed.id, manual: true },
{ idempotencyKey: `ingestFeed:manual:${feed.id}:${Math.floor(Date.now() / 10_000)}`, traceId: randomUUID() },
);
return { taskId: r.runId };
});
// ── 来源资源 / 观察 ──
route(app, D.sourceReleases.list, async ({ query }) => {
const off = decodeCursor(query.cursor);
let q = db().selectFrom("source_release").selectAll();
if (query.siteId) q = q.where("site_id", "=", query.siteId);
if (query.mediaEntityId) q = q.where("media_entity_id", "=", query.mediaEntityId);
if (query.resolutionStatus) q = q.where("resolution_status", "=", query.resolutionStatus);
if (query.q)
q = q.where((eb) =>
eb.or([eb("title", "ilike", `%${query.q}%`), sql<boolean>`similarity(title, ${query.q}) >= 0.3`]),
);
const rows = await q
.orderBy("first_seen_at", "desc")
.orderBy("id")
.offset(off)
.limit(query.limit + 1)
.execute();
const p = page(rows, off, query.limit);
return { items: toApiList<SourceRelease>(p.items), nextCursor: p.nextCursor };
});
route(app, D.sourceReleases.get, async ({ params }) => {
const row = await db().selectFrom("source_release").selectAll().where("id", "=", params.id).executeTakeFirst();
if (!row) throw new ApiException("not_found", "来源资源不存在");
return toApi(row);
});
route(
app,
{
method: "post",
path: "/source-releases/{id}/identity",
scope: "organize.execute",
summary: "人工指定来源资源的作品/内容单元(加人工锁,自动识别不再改写)",
tags: T_RELEASES,
params: IdParam,
body: ReleaseIdentityAssign,
response: z.object({ mediaEntityId: Id, contentUnitIds: z.array(Id) }),
},
async ({ params, body, actor }) => assignReleaseIdentity(rt(), params.id, body, actor),
);
route(
app,
{
method: "post",
path: "/source-releases/{id}/unlock",
scope: "organize.execute",
summary: "解除来源资源的人工锁并重新识别",
tags: T_RELEASES,
params: IdParam,
response: Accepted,
},
async ({ params, actor }) => {
const row = await db()
.updateTable("source_release")
.set({ resolution_locked: false })
.where("id", "=", params.id)
.returning(["id", "site_id"])
.executeTakeFirst();
if (!row) throw new ApiException("not_found", "来源资源不存在");
await audit(db(), actor, "identity.release_unlocked", { type: "source_release", id: params.id });
const r = await rt().ctx.engine.enqueue(
"resolveObservation",
{ sourceReleaseId: params.id, force: true },
{ idempotencyKey: `resolve:unlock:${params.id}:${Date.now()}`, traceId: randomUUID() },
);
return { taskId: r.runId };
},
);
route(app, D.observations.list, async ({ query }) => {
const off = decodeCursor(query.cursor);
let q = db().selectFrom("observation").selectAll();
if (query.feedId) q = q.where("feed_id", "=", query.feedId);
if (query.status) q = q.where("status", "=", query.status);
const rows = await q
.orderBy("first_seen_at", "desc")
.orderBy("id")
.offset(off)
.limit(query.limit + 1)
.execute();
const p = page(rows, off, query.limit);
return { items: toApiList<Observation>(p.items), nextCursor: p.nextCursor };
});
route(app, D.observations.get, async ({ params }) => {
const row = await db().selectFrom("observation").selectAll().where("id", "=", params.id).executeTakeFirst();
if (!row) throw new ApiException("not_found", "观察不存在");
return toApi(row);
});
// ── 内部辅助 ──
async function getSite(id: string) {
const row = await db().selectFrom("site").selectAll().where("id", "=", id).executeTakeFirst();
if (!row) throw new ApiException("not_found", "站点不存在");
return row;
}
async function getFeed(id: string) {
const row = await db().selectFrom("feed_source").selectAll().where("id", "=", id).executeTakeFirst();
if (!row) throw new ApiException("not_found", "Feed 不存在");
return row;
}
async function checkAccount(accountId: string, siteId: string | null) {
const acc = await db()
.selectFrom("site_account")
.select(["site_id"])
.where("id", "=", accountId)
.executeTakeFirst();
if (!acc) throw new ApiException("bad_request", "站点账号不存在");
if (siteId && acc.site_id !== siteId) throw new ApiException("bad_request", "站点账号不属于该站点");
}
async function setSiteEnabled(
id: string,
enabled: boolean,
actor: { type: "user" | "agent" | "system" | "api_key"; id: string | null },
) {
await getSite(id);
return db()
.transaction()
.execute(async (trx) => {
const row = await trx
.updateTable("site")
.set({ enabled, health_status: enabled ? "unknown" : "disabled" })
.where("id", "=", id)
.returningAll()
.executeTakeFirstOrThrow();
// 历史资源仍可查,标注站点关闭
await trx
.updateTable("source_release")
.set({ availability: enabled ? "active" : "site_disabled" })
.where("site_id", "=", id)
.where("availability", "=", enabled ? "site_disabled" : "active")
.execute();
await audit(trx, actor, enabled ? "site.enabled" : "site.disabled", { type: "site", id });
return row;
});
}
}
@@ -0,0 +1,42 @@
import { Id, WorkbenchReleaseDetail, WorkbenchReleasePage, WorkbenchReleaseQuery } from "@umt/contracts";
import { z } from "zod";
import type { ModuleApp } from "../../types.ts";
import { route } from "../http.ts";
import type { SourcesRuntime } from "../runtime.ts";
import { listSiteReleases, releaseDetail } from "../service/workbench.ts";
// 站点工作台(整理视图)数据 API。库存叠加(已有资产/Slot/下载)不在这里:条目带 entity.id,
// 前端按作品 ID 二次查询 W2-C 的媒体库端点。
const T = ["site-workbench"];
export function workbenchRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
route(
app,
{
method: "get",
path: "/sites/{id}/releases",
scope: "read",
summary: "站点资源列表/搜索(快照优先,必要时实时补查;标注来源与更新时间)",
tags: T,
params: z.object({ id: Id }),
query: WorkbenchReleaseQuery,
response: WorkbenchReleasePage,
},
async ({ params, query }) => listSiteReleases(rt(), params.id, query),
);
route(
app,
{
method: "get",
path: "/sites/{id}/releases/{rid}",
scope: "read",
summary: "站点资源详情(refresh=true 时实时复核详情;失败保留既有数据)",
tags: T,
params: z.object({ id: Id, rid: Id }),
query: z.object({ refresh: z.enum(["true", "false"]).optional() }),
response: WorkbenchReleaseDetail,
},
async ({ params, query }) => releaseDetail(rt(), params.id, params.rid, query.refresh === "true"),
);
}
@@ -0,0 +1,37 @@
// db 行(snake_case、Date)→ contracts(camelCase、ISO 串)。只做命名与时间转换,取值语义不变。
const camel = (s: string) => s.replace(/_([a-z0-9])/g, (_, c: string) => c.toUpperCase());
function conv(v: unknown): unknown {
if (v instanceof Date) return v.toISOString();
return v;
}
/** 通用行转换:键转 camelCase,Date → ISO 串;jsonb/数组原样保留 */
export function toApi<T>(row: object): T {
const out: Record<string, unknown> = {};
for (const [k, v] of Object.entries(row)) out[camel(k)] = conv(v);
return out as T;
}
export const toApiList = <T>(rows: readonly object[]): T[] => rows.map((r) => toApi<T>(r));
/** 偏移游标(不透明 base64url);列表规模小,足够用 */
export function encodeCursor(offset: number): string {
return Buffer.from(String(offset), "utf8").toString("base64url");
}
export function decodeCursor(cursor: string | undefined): number {
if (!cursor) return 0;
const n = Number(Buffer.from(cursor, "base64url").toString("utf8"));
return Number.isSafeInteger(n) && n >= 0 ? n : 0;
}
/** 取 limit+1 条判断是否还有下一页 */
export function page<T>(rows: T[], offset: number, limit: number): { items: T[]; nextCursor: string | null } {
const more = rows.length > limit;
return { items: more ? rows.slice(0, limit) : rows, nextCursor: more ? encodeCursor(offset + limit) : null };
}
/** jsonb 写入:数组/对象都先 JSON.stringify(pg 会把 JS 数组当 PG 数组) */
export const jsonb = (v: unknown): string => JSON.stringify(v ?? null);
@@ -0,0 +1,143 @@
import {
createRssFetcher,
createSiteAdapter,
createSiteRateLimiter,
createTmdbClient,
getSiteDefinition,
type RssFetcher,
type SecretResolver,
type SiteAdapter,
type SiteDefinition,
type SiteRateLimiter,
type TmdbClient,
} from "@umt/adapters";
import type { SiteRow } from "@umt/db";
import type { ServerContext } from "../types.ts";
// 模块运行时依赖:站点适配器与限流器(每站一个实例)、RSS 抓取器、TMDB 客户端、时钟。
// 测试通过 createSourcesIdentityModule(overrides) 注入 fetch / TMDB / 时钟。
export interface RuntimeOverrides {
/** 站点与 RSS 请求使用的 fetch(测试指向 mock) */
fetch?: typeof globalThis.fetch;
/** TMDB 客户端工厂;返回 null 表示未配置(只做本地召回) */
tmdb?: (() => TmdbClient | null) | undefined;
now?: () => Date;
/** RSS 抓取器选项覆盖(退避/熔断阈值) */
rss?: Parameters<typeof createRssFetcher>[0];
/** 调度 tick 间隔(毫秒),默认 30s */
schedulerIntervalMs?: number;
}
interface CachedAdapter {
sig: string;
limiter: SiteRateLimiter;
adapter: SiteAdapter;
}
export class SourcesRuntime {
readonly now: () => Date;
readonly rss: RssFetcher;
private readonly adapters = new Map<string, CachedAdapter>();
private tmdbClient: TmdbClient | null | undefined;
constructor(
readonly ctx: ServerContext,
readonly overrides: RuntimeOverrides = {},
) {
this.now = overrides.now ?? (() => new Date());
this.rss = createRssFetcher({
...(overrides.fetch ? { fetch: overrides.fetch } : {}),
now: this.now,
...overrides.rss,
});
}
/** TMDB:优先注入;否则读环境变量 TMDB_API_KEY / TMDB_ACCESS_TOKEN(W2-A 设置页接管前的过渡) */
tmdb(): TmdbClient | null {
if (this.tmdbClient !== undefined) return this.tmdbClient;
if (this.overrides.tmdb) {
this.tmdbClient = this.overrides.tmdb();
return this.tmdbClient;
}
const env = (k: string) => Deno.env.get(k);
const apiKey = env("TMDB_API_KEY");
const accessToken = env("TMDB_ACCESS_TOKEN");
const baseUrl = env("TMDB_BASE_URL");
this.tmdbClient =
apiKey || accessToken
? createTmdbClient({
...(apiKey ? { apiKey } : {}),
...(accessToken ? { accessToken } : {}),
...(baseUrl ? { baseUrl } : {}),
...(this.overrides.fetch ? { fetch: this.overrides.fetch } : {}),
})
: null;
return this.tmdbClient;
}
/**
* 站点适配器:每站一个实例,列表/搜索/详情/下载共享同一个限流器。
* 站点限流参数或基址变化时重建(签名比较)。
*/
siteAdapter(site: Pick<SiteRow, "id" | "key" | "base_url" | "rate_limit" | "config">): SiteAdapter {
const def = getSiteDefinition(site.key);
if (!def) throw new Error(`站点 ${site.key} 没有随产品发布的适配器定义`);
const sig = JSON.stringify([site.base_url, site.rate_limit, site.config]);
const hit = this.adapters.get(site.id);
if (hit && hit.sig === sig) return hit.adapter;
const limiter = createSiteRateLimiter(limiterOptions(def, site.rate_limit));
const cfg = (site.config ?? {}) as { apiBaseUrl?: string };
const baseUrl =
def.umt.kind === "mteam" ? cfg.apiBaseUrl : sameUrl(site.base_url, def.domain) ? undefined : site.base_url;
const adapter = createSiteAdapter(def, {
secrets: this.siteSecrets(site.id),
limiter,
now: this.now,
...(this.overrides.fetch ? { fetch: this.overrides.fetch } : {}),
...(baseUrl ? { baseUrl } : {}),
});
this.adapters.set(site.id, { sig, limiter, adapter });
return adapter;
}
/** 已建立的站点限流器(测试与诊断用) */
limiterOf(siteId: string): SiteRateLimiter | null {
return this.adapters.get(siteId)?.limiter ?? null;
}
/** 站点凭证:取该站第一个启用且未失效的账号,按 auth_kind 对应适配器要求的凭证类型;明文不缓存 */
private siteSecrets(siteId: string): SecretResolver {
const { db, secrets } = this.ctx;
return {
async resolve({ kind }) {
const want = kind === "cookie" ? ["cookie"] : kind === "apiKey" ? ["api_key"] : ["passkey"];
const acc = await db
.selectFrom("site_account")
.select(["secret_ref"])
.where("site_id", "=", siteId)
.where("enabled", "=", true)
.where("status", "in", ["unknown", "active"])
.where("auth_kind", "in", want as never)
.where("secret_ref", "is not", null)
.orderBy("created_at")
.executeTakeFirst();
if (!acc?.secret_ref) return null;
return secrets.resolve(acc.secret_ref);
},
};
}
}
function sameUrl(a: string, b: string): boolean {
return a.replace(/\/+$/, "") === b.replace(/\/+$/, "");
}
/** 站点行 rate_limit 优先,缺省用定义里的 umt.rateLimit */
export function limiterOptions(def: SiteDefinition, rateLimit: unknown): { minIntervalMs: number; burst: number } {
const rl = (rateLimit ?? {}) as { requestsPerMinute?: number; minIntervalMs?: number; concurrency?: number };
const minIntervalMs =
rl.minIntervalMs ??
(rl.requestsPerMinute ? Math.ceil(60_000 / rl.requestsPerMinute) : def.umt.rateLimit.minIntervalMs);
return { minIntervalMs, burst: rl.concurrency ?? def.umt.rateLimit.burst };
}
@@ -0,0 +1,90 @@
import { randomUUID } from "node:crypto";
import { sql } from "@umt/db";
import type { SourcesRuntime } from "./runtime.ts";
// worker 后台调度:按 Feed 轮询策略发 feed.due(outbox → ingestFeed)、按站点快照周期入队 snapshotSite、
// 按待识别池 next_review_at 入队复查。幂等键按时间槽生成,多 worker 并发也不会重复。
export interface TickResult {
feedsDue: number;
snapshotsDue: number;
reviewsDue: number;
skipped?: string;
}
const REVIEW_LEASE_MS = 10 * 60_000;
export async function schedulerTick(rt: SourcesRuntime): Promise<TickResult> {
const { ctx } = rt;
const automation = await ctx.settings.get("automation");
// 恢复后只读对账模式:禁止自动写入
if (automation.readOnlyReconcile) return { feedsDue: 0, snapshotsDue: 0, reviewsDue: 0, skipped: "read_only" };
const now = rt.now();
// Feed:到期(next_fetch_at 空或已到)且不在退避期、站点(若有)开启
const feedsDue = await ctx.db.transaction().execute(async (trx) => {
const due = await sql<{ id: string; poll_interval_sec: number }>`
select f.id, f.poll_interval_sec from feed_source f left join site s on s.id = f.site_id
where f.enabled and (s.id is null or s.enabled)
and (f.next_fetch_at is null or f.next_fetch_at <= ${now})
and (f.backoff_until is null or f.backoff_until <= ${now})
and f.health_status <> 'circuit_open'
order by f.next_fetch_at nulls first limit 100
for update of f skip locked`.execute(trx);
for (const f of due.rows) {
const slot = Math.floor(now.getTime() / 1000 / f.poll_interval_sec);
await ctx.outbox.emit(
trx,
"feed.due",
{ feedId: f.id, reason: "schedule" },
{ idempotencyKey: `feed.due:${f.id}:${slot}` },
);
await trx
.updateTable("feed_source")
.set({ next_fetch_at: new Date(now.getTime() + f.poll_interval_sec * 1000) })
.where("id", "=", f.id)
.execute();
}
return due.rows.length;
});
// 站点快照:开启快照且到周期
const sites = await sql<{ id: string; snapshot_interval_sec: number }>`
select id, snapshot_interval_sec from site
where enabled and snapshot_enabled and snapshot_interval_sec is not null
and (last_snapshot_at is null or last_snapshot_at + make_interval(secs => snapshot_interval_sec) <= ${now})`.execute(
ctx.db,
);
for (const s of sites.rows) {
const slot = Math.floor(now.getTime() / 1000 / s.snapshot_interval_sec);
await ctx.engine.enqueue(
"snapshotSite",
{ siteId: s.id, kind: "periodic" },
{ idempotencyKey: `snapshotSite:${s.id}:${slot}`, traceId: randomUUID() },
);
}
// 待识别池复查:到期的先顺延一个租约,避免下个 tick 重复入队
const reviews = await ctx.db.transaction().execute(async (trx) => {
const rows = await sql<{ id: string; source_release_id: string; attempts: number }>`
select id, source_release_id, attempts from identity_candidate
where status = 'pending' and next_review_at is not null and next_review_at <= ${now}
order by next_review_at limit 50 for update skip locked`.execute(trx);
for (const r of rows.rows) {
await trx
.updateTable("identity_candidate")
.set({ next_review_at: new Date(now.getTime() + REVIEW_LEASE_MS) })
.where("id", "=", r.id)
.execute();
}
return rows.rows;
});
for (const r of reviews) {
await ctx.engine.enqueue(
"resolveObservation",
{ sourceReleaseId: r.source_release_id, force: true },
{ idempotencyKey: `review:${r.id}:${r.attempts}`, traceId: randomUUID() },
);
}
return { feedsDue, snapshotsDue: sites.rows.length, reviewsDue: reviews.length };
}
@@ -0,0 +1,109 @@
import type { ActorType, DecisionKind, DecisionOption, Severity } from "@umt/contracts";
import { type DB, type Kysely, sql, type Transaction } from "@umt/db";
import type { ServerContext } from "../../types.ts";
import { jsonb } from "../rows.ts";
export type Exec = Kysely<DB> | Transaction<DB>;
export interface Actor {
type: ActorType;
id: string | null;
}
export const SYSTEM: Actor = { type: "system", id: "sources-identity" };
/** 审计:只追加 */
export async function audit(
exec: Exec,
actor: Actor,
action: string,
target: { type: string; id: string | null },
payload: Record<string, unknown> = {},
): Promise<void> {
await exec
.insertInto("audit_event")
.values({
actor_type: actor.type,
actor_id: actor.id,
action,
target_type: target.type,
target_id: target.id,
payload: jsonb(payload),
})
.execute();
}
export interface DecisionInput {
kind: DecisionKind;
severity?: Severity;
title: string;
summary?: string | null;
subjectType: string;
subjectId: string | null;
dedupeKey: string;
evidence?: Record<string, unknown>;
options?: DecisionOption[];
nextReviewAt?: Date | null;
}
/**
* 开一个待处理事项;同 dedupeKey 已有未关闭条目时只刷新证据与选项(不重复通知)。
* 新建时同事务发 decision.created。
*/
export async function openDecision(
ctx: ServerContext,
trx: Exec,
d: DecisionInput,
): Promise<{ id: string; created: boolean }> {
const row = await sql<{ id: string; inserted: boolean }>`
insert into decision (kind, severity, title, summary, subject_type, subject_id, dedupe_key, evidence, options, next_review_at)
values (${d.kind}, ${d.severity ?? "warning"}, ${d.title}, ${d.summary ?? null}, ${d.subjectType}, ${d.subjectId},
${d.dedupeKey}, ${jsonb(d.evidence ?? {})}::jsonb, ${jsonb(d.options ?? [])}::jsonb, ${d.nextReviewAt ?? null})
on conflict (dedupe_key) where status in ('open', 'investigating', 'awaiting_user')
do update set evidence = excluded.evidence, options = excluded.options, summary = excluded.summary,
title = excluded.title, attempts = decision.attempts + 1
returning id, (xmax = 0) as inserted`.execute(trx);
const r = row.rows[0]!;
if (r.inserted) {
await ctx.outbox.emit(
trx,
"decision.created",
{ decisionId: r.id, kind: d.kind, severity: d.severity ?? "warning", title: d.title },
{ idempotencyKey: `decision.created:${r.id}` },
);
}
return { id: r.id, created: r.inserted };
}
/** 关闭同 dedupeKey 的未关闭事项(身份已被人工/自动确认等),逐条发 decision.updated */
export async function closeDecisions(
ctx: ServerContext,
trx: Exec,
dedupeKey: string,
status: "resolved" | "dismissed",
resolution: Record<string, unknown>,
by: string,
): Promise<string[]> {
const rows = await trx
.updateTable("decision")
.set({ status, resolution: jsonb(resolution), resolved_by: by, resolved_at: new Date() })
.where("dedupe_key", "=", dedupeKey)
.where("status", "in", ["open", "investigating", "awaiting_user"])
.returning(["id"])
.execute();
for (const r of rows) {
await ctx.outbox.emit(
trx,
"decision.updated",
{ decisionId: r.id, status },
{
idempotencyKey: `decision.updated:${r.id}:${status}`,
},
);
}
return rows.map((r) => r.id);
}
/** 截断长文本(描述等只留摘录) */
export const clip = (s: string | undefined | null, n = 2000): string | null =>
s == null ? null : s.length > n ? `${s.slice(0, n)}…` : s;
@@ -0,0 +1,221 @@
import type { MetadataCandidate } from "@umt/adapters";
import type { AliasKind, AliasSource, EvidenceLevel, MediaKind } from "@umt/contracts";
import { sql } from "@umt/db";
import { normalizeTitle, type ParsedRelease } from "@umt/domain";
import { toContractMediaKind } from "@umt/mapping";
import { jsonb } from "../rows.ts";
import type { Actor, Exec } from "./common.ts";
// 本地作品 / 内容单元 / 外部身份 / 别名的写入辅助。所有写入在调用方事务内完成。
/** 沿 merged_into_id 找到最终实体(防环,最多 10 跳);不存在返回 null */
export async function resolveEntityId(exec: Exec, id: string): Promise<string | null> {
let cur = id;
for (let i = 0; i < 10; i++) {
const row = await exec
.selectFrom("media_entity")
.select(["id", "status", "merged_into_id"])
.where("id", "=", cur)
.executeTakeFirst();
if (!row) return null;
if (row.status !== "merged" || !row.merged_into_id) return row.id;
cur = row.merged_into_id;
}
return cur;
}
/** 已接受的外部身份 → 本地作品(跟随合并跳转) */
export async function entityByExternal(
exec: Exec,
ref: { provider: string; entityType: string; externalId: string },
): Promise<string | null> {
const row = await exec
.selectFrom("external_identity")
.select(["media_entity_id"])
.where("provider", "=", ref.provider)
.where("entity_type", "=", ref.entityType as never)
.where("external_id", "=", ref.externalId)
.where("status", "=", "accepted")
.where("media_entity_id", "is not", null)
.executeTakeFirst();
return row?.media_entity_id ? resolveEntityId(exec, row.media_entity_id) : null;
}
export async function addAlias(
exec: Exec,
entityId: string,
alias: string,
kind: AliasKind,
source: AliasSource,
language: string | null = null,
locked = false,
): Promise<void> {
const normalized = normalizeTitle(alias);
if (!normalized) return;
await sql`insert into title_alias (media_entity_id, content_unit_id, alias, normalized, language, kind, source, locked)
values (${entityId}, null, ${alias}, ${normalized}, ${language}, ${kind}, ${source}, ${locked})
on conflict (media_entity_id, content_unit_id, normalized, kind) do nothing`.execute(exec);
}
/** 写一条外部身份(accepted)+ 修订 created;同一外部身份已被其它作品接受时返回 null(交给调用方处理冲突) */
export async function acceptIdentity(
exec: Exec,
entityId: string,
ref: { provider: string; entityType: string; externalId: string },
level: EvidenceLevel,
evidence: Record<string, unknown>,
actor: Actor,
opts: { locked?: boolean; reason?: string | null } = {},
): Promise<string | null> {
const ins = await sql<{ id: string }>`
insert into external_identity (media_entity_id, provider, entity_type, external_id, status, evidence_level, evidence, locked, valid_from, verified_at, verify_status)
values (${entityId}, ${ref.provider}, ${ref.entityType}, ${ref.externalId}, 'accepted', ${level}, ${jsonb(evidence)}::jsonb,
${opts.locked ?? false}, now(), now(), 'ok')
on conflict do nothing
returning id`.execute(exec);
const id = ins.rows[0]?.id;
if (!id) return null;
await exec
.insertInto("identity_revision")
.values({
external_identity_id: id,
revision: 1,
action: "created",
from_status: null,
to_status: "accepted",
after: jsonb({ ...ref, mediaEntityId: entityId, evidenceLevel: level, locked: opts.locked ?? false }),
actor_type: actor.type,
actor_id: actor.id,
reason: opts.reason ?? null,
})
.execute();
return id;
}
/** 综艺判定:站点分类名含"综艺"或标题带期号/播出日期 */
export function looksLikeVariety(parsed: ParsedRelease | null, categoryName: string | null): boolean {
if (categoryName && /综艺|variety|show/i.test(categoryName)) return true;
return !!parsed && (parsed.issue !== undefined || parsed.airDate !== undefined);
}
/**
* 按 TMDB 详情建本地作品:外部身份 accepted(+IMDb)、别名(原名/译名/别名)。
* 已有同一 TMDB 身份的作品(含并发创建)直接复用。
*/
export async function createEntityFromTmdb(
exec: Exec,
m: MetadataCandidate,
opts: { level: EvidenceLevel; evidence: Record<string, unknown>; actor: Actor; variety?: boolean; locked?: boolean },
): Promise<{ entityId: string; created: boolean }> {
const ref = { provider: "tmdb", entityType: m.entityType, externalId: m.externalId };
const existing = await entityByExternal(exec, ref);
if (existing) return { entityId: existing, created: false };
const kind: MediaKind = toContractMediaKind(m.entityType, { variety: !!opts.variety });
const entity = await exec
.insertInto("media_entity")
.values({
kind,
title: m.title,
original_title: m.originalTitle,
year: m.year,
status: "active",
metadata: jsonb({}),
})
.returning(["id"])
.executeTakeFirstOrThrow();
const idRow = await acceptIdentity(exec, entity.id, ref, opts.level, opts.evidence, opts.actor, {
locked: opts.locked ?? false,
});
if (!idRow) {
// 并发:另一个事务刚接受了同一身份。回滚本事务的新作品由调用方重试处理
throw new IdentityConflictError(ref);
}
if (m.externalIds?.imdb) {
await acceptIdentity(
exec,
entity.id,
{ provider: "imdb", entityType: m.entityType, externalId: m.externalIds.imdb },
opts.level,
{ via: "tmdb", tmdbId: m.externalId },
opts.actor,
);
}
await seedAliases(exec, entity.id, m);
return { entityId: entity.id, created: true };
}
export async function seedAliases(exec: Exec, entityId: string, m: MetadataCandidate): Promise<void> {
await addAlias(exec, entityId, m.title, "translated", "tmdb", null);
if (m.originalTitle) await addAlias(exec, entityId, m.originalTitle, "original", "tmdb", m.originalLanguage);
for (const t of m.titles) {
if (t.kind === "localized") await addAlias(exec, entityId, t.text, "translated", "tmdb", t.lang);
}
for (const a of m.aliases.slice(0, 50)) await addAlias(exec, entityId, a, "other", "tmdb", null);
}
export class IdentityConflictError extends Error {
constructor(readonly ref: { provider: string; entityType: string; externalId: string }) {
super(`外部身份 ${ref.provider}:${ref.entityType}/${ref.externalId} 已被其它作品接受`);
}
}
/**
* 按解析出的季/集找或建内容单元(临时 provisional 允许)。电影返回空。
* 返回:集单元(有集号时)或季单元;primary 为写入 source_release.content_unit_id 的那个。
*/
export async function ensureUnits(
exec: Exec,
entityId: string,
kind: MediaKind,
parsed: ParsedRelease | null,
): Promise<{ unitIds: string[]; primary: string | null }> {
if (!parsed || kind === "movie" || parsed.season === undefined) return { unitIds: [], primary: null };
const seasons: number[] = [];
const last = parsed.seasonEnd ?? parsed.season;
for (let s = parsed.season; s <= last && seasons.length < 50; s++) seasons.push(s);
const seasonIds: string[] = [];
for (const s of seasons) seasonIds.push(await findOrCreateUnit(exec, entityId, "season", s, null, null));
if (parsed.episodes && seasons.length === 1) {
const ids: string[] = [];
const end = Math.min(parsed.episodes.end, parsed.episodes.start + 199);
for (let e = parsed.episodes.start; e <= end; e++) {
ids.push(await findOrCreateUnit(exec, entityId, "episode", parsed.season, e, seasonIds[0]!));
}
return { unitIds: ids, primary: ids.length === 1 ? ids[0]! : seasonIds[0]! };
}
return { unitIds: seasonIds, primary: seasonIds[0] ?? null };
}
async function findOrCreateUnit(
exec: Exec,
entityId: string,
kind: "season" | "episode",
season: number,
episode: number | null,
parentId: string | null,
): Promise<string> {
let q = exec
.selectFrom("content_unit")
.select(["id"])
.where("media_entity_id", "=", entityId)
.where("kind", "=", kind)
.where("season_number", "=", season)
.where("status", "<>", "retired");
q = episode === null ? q.where("episode_number", "is", null) : q.where("episode_number", "=", episode);
const hit = await q.orderBy("created_at").executeTakeFirst();
if (hit) return hit.id;
const row = await exec
.insertInto("content_unit")
.values({
media_entity_id: entityId,
parent_unit_id: parentId,
kind,
season_number: season,
episode_number: episode,
status: "provisional",
display_order: episode ?? season,
})
.returning(["id"])
.executeTakeFirstOrThrow();
return row.id;
}
@@ -0,0 +1,320 @@
import { createHash } from "node:crypto";
import {
applyFeedItems,
emptyFeedCursor,
type FeedCursor,
type FeedFetchResult,
type FeedFetchState,
type FeedItem,
} from "@umt/adapters";
import type { FeedHealth } from "@umt/contracts";
import { type FeedSourceRow, sql } from "@umt/db";
import { jsonb } from "../rows.ts";
import type { SourcesRuntime } from "../runtime.ts";
import { clip, type Exec, openDecision } from "./common.ts";
// ingestFeed:RSS 抓取 → observation(feed+guid 唯一,摘要变化 = 更新)→ source_release(site+站点资源 ID 聚合)。
// FeedFetchState / FeedCursor 持久化在 feed_source 行;enclosure 只存脱敏值(适配器已脱敏)。
/** feed_source.cursor 的形状:重叠窗口游标 + 熔断打开时间 */
interface StoredCursor {
window?: FeedCursor;
circuitOpenedAt?: string;
}
const AUTH_BACKOFF_MS = 24 * 3600_000;
export interface IngestResult {
status: "ok" | "not_modified" | "skipped" | "error" | "auth_failed";
reason?: string;
newItems: number;
updatedItems: number;
newReleases: number;
possibleGap: boolean;
}
/** 从 Feed 条目推出站点资源 ID:详情/下载链接的 id 参数或 /detail/<id> 路径;都没有返回 null */
export function extractSiteReleaseId(item: Pick<FeedItem, "guid" | "link" | "enclosureUrl">): string | null {
for (const u of [item.link, item.enclosureUrl, item.guid]) {
if (!u) continue;
try {
const url = new URL(u);
const q = url.searchParams.get("id") ?? url.searchParams.get("tid") ?? url.searchParams.get("torrentid");
if (q && /^[\w-]{1,64}$/.test(q)) return q;
const m = /\/(?:detail|details|torrents?|t)\/(\d+)/.exec(url.pathname);
if (m?.[1]) return m[1];
} catch {
// 不是 URL(纯 GUID)
}
}
if (/^\d{1,20}$/.test(item.guid)) return item.guid;
return null;
}
export function stateFromRow(f: FeedSourceRow): FeedFetchState {
const cur = (f.cursor ?? {}) as StoredCursor;
const err = f.last_error as FeedFetchState["lastError"] | null;
return {
...(f.etag ? { etag: f.etag } : {}),
...(f.last_modified ? { lastModified: f.last_modified } : {}),
consecutiveFailures: f.consecutive_failures,
...(f.backoff_until ? { nextAttemptAt: f.backoff_until.toISOString() } : {}),
circuit: f.health_status === "circuit_open" ? "open" : "closed",
...(cur.circuitOpenedAt ? { circuitOpenedAt: cur.circuitOpenedAt } : {}),
...(f.last_success_at ? { lastSuccessAt: f.last_success_at.toISOString() } : {}),
...(err && typeof err === "object" && "kind" in err ? { lastError: err } : {}),
};
}
function healthOf(r: FeedFetchResult): FeedHealth {
if (r.kind === "ok" || r.kind === "not_modified") return "healthy";
if (r.state.circuit === "open") return "circuit_open";
if (r.kind === "skipped") return "degraded";
return r.state.consecutiveFailures >= 3 ? "failing" : "degraded";
}
function isAuthFailure(r: FeedFetchResult): boolean {
if (r.kind !== "error") return false;
return r.error.kind === "expired" || r.error.httpStatus === 401 || r.error.httpStatus === 403;
}
export async function ingestFeed(
rt: SourcesRuntime,
feedId: string,
opts: { manual?: boolean } = {},
): Promise<IngestResult> {
const { db, secrets } = rt.ctx;
const empty = { newItems: 0, updatedItems: 0, newReleases: 0, possibleGap: false };
const feed = await db.selectFrom("feed_source").selectAll().where("id", "=", feedId).executeTakeFirst();
if (!feed) return { status: "skipped", reason: "not_found", ...empty };
if (!feed.enabled) return { status: "skipped", reason: "disabled", ...empty };
const site = feed.site_id
? await db.selectFrom("site").selectAll().where("id", "=", feed.site_id).executeTakeFirst()
: undefined;
if (site && !site.enabled) return { status: "skipped", reason: "site_disabled", ...empty };
if (!feed.url_secret_ref) return { status: "skipped", reason: "no_url", ...empty };
const url = await secrets.resolve(feed.url_secret_ref);
const result = await rt.rss.fetchFeed(url, stateFromRow(feed), { ignoreBackoff: !!opts.manual });
const now = rt.now();
const auth = isAuthFailure(result);
const health: FeedHealth = auth ? "failing" : healthOf(result);
const cursor = (feed.cursor ?? {}) as StoredCursor;
const state = result.state;
const pollNext = new Date(now.getTime() + feed.poll_interval_sec * 1000);
const backoffUntil = auth
? new Date(now.getTime() + AUTH_BACKOFF_MS)
: state.nextAttemptAt && result.kind !== "ok" && result.kind !== "not_modified"
? new Date(state.nextAttemptAt)
: null;
return db.transaction().execute(async (trx) => {
let out: IngestResult = { status: result.kind === "ok" ? "ok" : result.kind, ...empty };
let window = cursor.window ?? emptyFeedCursor();
if (result.kind === "ok") {
const applied = applyFeedItems(window, result.feed.items);
window = applied.cursor;
const counts = await writeItems(rt, trx, feed, site?.id ?? null, applied.newItems, applied.updatedItems);
const unchanged = result.feed.items
.filter((i) => !applied.newItems.includes(i) && !applied.updatedItems.includes(i))
.map((i) => i.guid);
if (unchanged.length) await touchUnchanged(trx, feed.id, unchanged, now);
out = {
status: "ok",
newItems: applied.newItems.length,
updatedItems: applied.updatedItems.length,
newReleases: counts.newReleases,
possibleGap: applied.possibleGap,
};
if (applied.possibleGap) {
await rt.ctx.outbox.emit(
trx,
"feed.gap_suspected",
{
feedId: feed.id,
siteId: site?.id ?? null,
previousFetchedAt: cursor.window?.lastFetchedAt ?? null,
detectedAt: now.toISOString(),
},
{ idempotencyKey: `feed.gap:${feed.id}:${now.toISOString()}` },
);
}
}
const lastError = auth
? { ...(state.lastError ?? {}), kind: "auth", message: "Feed 凭证失效或过期(401/403/过期提示),已停止自动重试" }
: (state.lastError ?? null);
const nextCursor: StoredCursor = {
window,
...(state.circuit === "open" && state.circuitOpenedAt ? { circuitOpenedAt: state.circuitOpenedAt } : {}),
};
await trx
.updateTable("feed_source")
.set({
etag: state.etag ?? null,
last_modified: state.lastModified ?? null,
consecutive_failures: state.consecutiveFailures,
backoff_until: backoffUntil,
health_status: health,
last_error: result.kind === "ok" || result.kind === "not_modified" ? null : jsonb(lastError),
cursor: jsonb(nextCursor),
last_fetch_at: result.kind === "skipped" ? feed.last_fetch_at : now,
last_success_at: state.lastSuccessAt ? new Date(state.lastSuccessAt) : feed.last_success_at,
next_fetch_at: backoffUntil && backoffUntil > pollNext ? backoffUntil : pollNext,
})
.where("id", "=", feed.id)
.execute();
if (health !== feed.health_status) {
await rt.ctx.outbox.emit(trx, "feed.health_changed", {
feedId: feed.id,
from: feed.health_status,
to: health,
reason:
lastError && typeof lastError === "object" ? String((lastError as { message?: string }).message ?? "") : null,
});
}
if (result.kind === "ok" || result.kind === "not_modified") {
await rt.ctx.outbox.emit(trx, "feed.fetched", {
feedId: feed.id,
newItems: out.newItems,
updatedItems: out.updatedItems,
notModified: result.kind === "not_modified",
});
}
if (auth) {
if (feed.site_account_id) {
await trx
.updateTable("site_account")
.set({ status: "invalid", last_verified_at: now })
.where("id", "=", feed.site_account_id)
.execute();
}
await openDecision(rt.ctx, trx, {
kind: "source_unreachable",
severity: "warning",
title: `Feed「${feed.name}」凭证失效`,
summary: "RSS 返回 401/403 或过期提示;已停止自动重试。请更新 Feed 链接或站点账号凭证后手动恢复。",
subjectType: "feed_source",
subjectId: feed.id,
dedupeKey: `feed-auth:${feed.id}`,
evidence: { feedId: feed.id, siteId: feed.site_id, siteAccountId: feed.site_account_id, error: lastError },
options: [
{ key: "reset", label: "已更新凭证,恢复抓取" },
{ key: "disable", label: "停用该 Feed" },
],
});
out = { ...out, status: "auth_failed", reason: "auth" };
}
return out;
});
}
/** 未变化条目:只刷新 last_seen_at(observation 与其 source_release) */
async function touchUnchanged(trx: Exec, feedId: string, guids: string[], now: Date) {
const rows = await trx
.updateTable("observation")
.set({ last_seen_at: now, fetched_at: now })
.where("feed_id", "=", feedId)
.where("guid", "in", guids)
.returning(["source_release_id"])
.execute();
const ids = rows.map((r) => r.source_release_id).filter((x): x is string => !!x);
if (ids.length) await trx.updateTable("source_release").set({ last_seen_at: now }).where("id", "in", ids).execute();
}
function observationRaw(item: FeedItem) {
return {
link: item.link,
// 只存脱敏后的 enclosure(凭据参数为 [REDACTED])
enclosureUrl: item.enclosureUrl ?? null,
enclosureHasSecret: item.enclosureHasSecret ?? false,
enclosureType: item.enclosureType ?? null,
size: item.size,
categories: item.categories,
nickname: item.nickname ?? null,
description: clip(item.description, 1000),
rawDigest: item.rawDigest,
};
}
async function writeItems(
rt: SourcesRuntime,
trx: Exec,
feed: FeedSourceRow,
siteId: string | null,
newItems: FeedItem[],
updatedItems: FeedItem[],
): Promise<{ newReleases: number }> {
let newReleases = 0;
const updated = new Set(updatedItems);
for (const item of [...newItems, ...updatedItems]) {
const raw = observationRaw(item);
const obs = (
await sql<{ id: string; source_release_id: string | null; prev_digest: string | null }>`
with prev as (select content_digest from observation where feed_id = ${feed.id} and guid = ${item.guid})
insert into observation (feed_id, guid, content_digest, title, link_redacted, raw, source_published_at, fetched_at)
values (${feed.id}, ${item.guid}, ${item.rawDigest}, ${item.title}, ${item.link}, ${jsonb(raw)}::jsonb,
${item.publishedAt ?? null}, ${item.fetchedAt})
on conflict (feed_id, guid) do update set
content_digest = excluded.content_digest, title = excluded.title, link_redacted = excluded.link_redacted,
raw = excluded.raw, source_published_at = coalesce(excluded.source_published_at, observation.source_published_at),
fetched_at = excluded.fetched_at, last_seen_at = now()
returning id, source_release_id, (select content_digest from prev) as prev_digest`.execute(trx)
).rows[0]!;
if (!siteId) {
// 无站点的通用 Feed:只留观察,无法按站点资源聚合
await trx
.updateTable("observation")
.set({ status: "ignored", last_error: jsonb({ reason: "no_site" }) })
.where("id", "=", obs.id)
.execute();
continue;
}
const srid =
extractSiteReleaseId(item) ?? `guid:${createHash("sha1").update(item.guid).digest("hex").slice(0, 20)}`;
const contentChanged = updated.has(item) || (obs.prev_digest !== null && obs.prev_digest !== item.rawDigest);
const sizeBytes = item.size > 0 ? item.size : null;
const rel = (
await sql<{ id: string; inserted: boolean; prev_title: string | null; resolution_locked: boolean }>`
with prev as (select title from source_release where site_id = ${siteId} and site_release_id = ${srid})
insert into source_release (site_id, site_release_id, title, category_raw, size_bytes, source_published_at, raw)
values (${siteId}, ${srid}, ${item.title}, ${item.categories[0] ?? null}, ${sizeBytes}, ${item.publishedAt ?? null},
${jsonb({ rss: { link: item.link, categories: item.categories } })}::jsonb)
on conflict (site_id, site_release_id) do update set
last_seen_at = now(),
title = case when ${contentChanged} then excluded.title else source_release.title end,
size_bytes = case when ${contentChanged} then coalesce(excluded.size_bytes, source_release.size_bytes)
else coalesce(source_release.size_bytes, excluded.size_bytes) end,
source_published_at = coalesce(source_release.source_published_at, excluded.source_published_at),
category_raw = coalesce(source_release.category_raw, excluded.category_raw)
returning id, (xmax = 0) as inserted, (select title from prev) as prev_title, resolution_locked`.execute(trx)
).rows[0]!;
await trx
.updateTable("observation")
.set({ source_release_id: rel.id, status: "linked" })
.where("id", "=", obs.id)
.execute();
if (rel.inserted) {
newReleases++;
await rt.ctx.outbox.emit(
trx,
"release.observed",
{ sourceReleaseId: rel.id, observationId: obs.id, siteId, isNew: true },
{ idempotencyKey: `release.observed:${rel.id}` },
);
} else if (contentChanged && rel.prev_title !== item.title && !rel.resolution_locked) {
// 标题修正:按新标题重新识别(证据版本变化)
await rt.ctx.outbox.emit(
trx,
"release.observed",
{ sourceReleaseId: rel.id, observationId: obs.id, siteId, isNew: false },
{ idempotencyKey: `release.observed:${rel.id}:${item.rawDigest}` },
);
}
}
return { newReleases };
}
@@ -0,0 +1,472 @@
import { TmdbError } from "@umt/adapters";
import type {
EntityIdentitySet,
ExternalIdentityUpdate,
ExternalRef,
PoolCandidate,
ReleaseIdentityAssign,
} from "@umt/contracts";
import { type ExternalIdentityRow, sql } from "@umt/db";
import { parseRelease } from "@umt/domain";
import { toParsedColumns } from "@umt/mapping";
import { ApiException } from "../../../http/errors.ts";
import { jsonb } from "../rows.ts";
import type { SourcesRuntime } from "../runtime.ts";
import { type Actor, audit, closeDecisions, type Exec } from "./common.ts";
import {
acceptIdentity,
createEntityFromTmdb,
ensureUnits,
entityByExternal,
IdentityConflictError,
looksLikeVariety,
resolveEntityId,
} from "./entities.ts";
import { recordAttempt } from "./resolve.ts";
// 人工纠错:指定来源资源身份(加锁)、改作品外部映射(新增修订、旧映射退役)、合并作品(旧 ID 跳转)。
// 所有变更留 identity_revision / resolution_attempt / audit_event,不直接覆盖历史。
const actorName = (a: Actor) => `${a.type}:${a.id ?? ""}`;
/** 外部身份 → 本地作品;未登记且提供方为 TMDB 时按详情建档(TMDB 失败给 502,不当作不存在) */
async function entityForExternal(rt: SourcesRuntime, exec: Exec, ref: ExternalRef, actor: Actor, variety: boolean) {
const hit = await entityByExternal(exec, ref);
if (hit) return hit;
if (ref.provider !== "tmdb")
throw new ApiException("bad_request", `未登记的外部身份 ${ref.provider}:${ref.externalId},只支持按 TMDB 建档`);
const tmdb = rt.tmdb();
if (!tmdb) throw new ApiException("precondition_failed", "未配置 TMDB,无法按外部身份建档");
let detail: Awaited<ReturnType<typeof tmdb.details>>;
try {
detail = await tmdb.details(ref.entityType, ref.externalId);
} catch (e) {
if (e instanceof TmdbError && e.kind === "notfound")
throw new ApiException("not_found", `TMDB 无此条目 ${ref.entityType}/${ref.externalId}`);
if (e instanceof TmdbError) throw new ApiException("upstream_unavailable", `TMDB 查询失败:${e.kind}`);
throw e;
}
const r = await createEntityFromTmdb(exec, detail, {
level: "manual",
evidence: { by: actorName(actor) },
actor,
variety,
locked: true,
});
return r.entityId;
}
export async function assignReleaseIdentity(
rt: SourcesRuntime,
srId: string,
input: ReleaseIdentityAssign,
actor: Actor,
): Promise<{ mediaEntityId: string; contentUnitIds: string[] }> {
const { ctx } = rt;
const release = await ctx.db.selectFrom("source_release").selectAll().where("id", "=", srId).executeTakeFirst();
if (!release) throw new ApiException("not_found", "来源资源不存在");
const parsed = parseRelease(release.title, release.subtitle ? { subtitle: release.subtitle } : {});
const site = ((release.raw as Record<string, unknown>)?._site ?? {}) as { categoryName?: string };
const variety = looksLikeVariety(parsed, site.categoryName ?? release.category_raw);
let ref: ExternalRef | null = input.external ?? null;
let localId: string | null = input.mediaEntityId ?? null;
if (input.candidateIndex !== undefined) {
const pool = await ctx.db
.selectFrom("identity_candidate")
.select(["candidates"])
.where("source_release_id", "=", srId)
.executeTakeFirst();
const c = ((pool?.candidates ?? []) as PoolCandidate[])[input.candidateIndex];
if (!c) throw new ApiException("bad_request", "候选序号无效");
if (c.localEntityId) localId = c.localEntityId;
else ref = { provider: c.provider, entityType: c.entityType, externalId: c.externalId };
}
for (let attempt = 0; ; attempt++) {
try {
return await ctx.db.transaction().execute(async (trx) => {
let entityId: string;
if (localId) {
const r = await resolveEntityId(trx, localId);
if (!r) throw new ApiException("not_found", "本地作品不存在");
entityId = r;
} else {
entityId = await entityForExternal(rt, trx, ref!, actor, variety);
}
const entity = await trx
.selectFrom("media_entity")
.select(["id", "kind"])
.where("id", "=", entityId)
.executeTakeFirstOrThrow();
let unitIds: string[];
let primary: string | null;
if (input.contentUnitId) {
const u = await trx
.selectFrom("content_unit")
.select(["id", "media_entity_id"])
.where("id", "=", input.contentUnitId)
.executeTakeFirst();
if (!u || u.media_entity_id !== entityId) throw new ApiException("bad_request", "内容单元不属于该作品");
unitIds = [u.id];
primary = u.id;
} else if (input.contentUnitId === null) {
unitIds = [];
primary = null;
} else {
const r = await ensureUnits(trx, entityId, entity.kind, parsed);
unitIds = r.unitIds;
primary = r.primary;
}
const cols = toParsedColumns(parsed);
await trx
.updateTable("source_release")
.set({
parser_version: cols.parserVersion,
parsed: jsonb(cols.parsed),
media_entity_id: entityId,
content_unit_id: primary,
resolution_status: "resolved",
resolution_locked: input.lock,
resolution_evidence: jsonb({
level: "manual",
by: actorName(actor),
reason: input.reason ?? null,
previous: { mediaEntityId: release.media_entity_id, contentUnitId: release.content_unit_id },
}),
})
.where("id", "=", srId)
.execute();
await trx
.updateTable("identity_candidate")
.set({
status: "resolved",
resolved_media_entity_id: entityId,
resolved_by: actorName(actor),
resolved_at: new Date(),
next_review_at: null,
})
.where("source_release_id", "=", srId)
.execute();
await closeDecisions(
ctx,
trx,
`identity:${srId}`,
"resolved",
{ mediaEntityId: entityId, by: "manual" },
actorName(actor),
);
await recordAttempt(
trx,
srId,
null,
"manual",
input.reason ?? null,
"manual",
1,
entityId,
{ mediaEntityId: entityId },
null,
actor,
);
await audit(
trx,
actor,
"identity.release_assigned",
{ type: "source_release", id: srId },
{
mediaEntityId: entityId,
contentUnitIds: unitIds,
locked: input.lock,
previous: release.media_entity_id,
},
);
await ctx.outbox.emit(trx, "release.resolved", {
sourceReleaseId: srId,
observationId: null,
mediaEntityId: entityId,
contentUnitIds: unitIds,
evidenceLevel: "manual",
});
return { mediaEntityId: entityId, contentUnitIds: unitIds };
});
} catch (e) {
if (e instanceof IdentityConflictError && attempt < 2) continue;
throw e;
}
}
}
async function nextRevision(exec: Exec, identityId: string): Promise<number> {
const r = await exec
.selectFrom("identity_revision")
.select((eb) => eb.fn.max("revision").as("m"))
.where("external_identity_id", "=", identityId)
.executeTakeFirst();
return (Number(r?.m) || 0) + 1;
}
async function retireIdentity(exec: Exec, row: ExternalIdentityRow, actor: Actor, reason: string | null) {
const rev = Math.max(row.revision + 1, await nextRevision(exec, row.id));
await exec
.updateTable("external_identity")
.set({ status: "retired", valid_until: new Date(), revision: rev, locked: true })
.where("id", "=", row.id)
.execute();
await exec
.insertInto("identity_revision")
.values({
external_identity_id: row.id,
revision: rev,
action: "status_changed",
from_status: row.status,
to_status: "retired",
before: jsonb({ status: row.status, locked: row.locked }),
after: jsonb({ status: "retired", locked: true }),
actor_type: actor.type,
actor_id: actor.id,
reason,
})
.execute();
}
/**
* 改作品的外部映射:同 provider+类型的旧 accepted 映射退役(留修订并锁定,防止自动匹配改回),
* 新映射 accepted + 人工锁。目标身份已被其它作品接受时 409(应走合并)。
*/
export async function setEntityIdentity(rt: SourcesRuntime, entityId: string, input: EntityIdentitySet, actor: Actor) {
const { ctx } = rt;
return ctx.db.transaction().execute(async (trx) => {
const id = await resolveEntityId(trx, entityId);
if (!id) throw new ApiException("not_found", "作品不存在");
const other = await entityByExternal(trx, input);
if (other && other !== id) {
throw new ApiException("conflict", `该外部身份已属于作品 ${other},如为重复作品请使用合并`);
}
if (other === id) return { mediaEntityId: id, changed: false };
const olds = await trx
.selectFrom("external_identity")
.selectAll()
.where("media_entity_id", "=", id)
.where("provider", "=", input.provider)
.where("entity_type", "=", input.entityType)
.where("status", "=", "accepted")
.execute();
for (const o of olds) await retireIdentity(trx, o, actor, input.reason ?? null);
// 同一外部身份的旧候选(proposed/retired)不复用,新建一行保留历史
const newId = await acceptIdentity(trx, id, input, "manual", { by: actorName(actor) }, actor, {
locked: input.lock,
reason: input.reason ?? null,
});
if (!newId) throw new ApiException("conflict", "外部身份唯一约束冲突");
await audit(
trx,
actor,
"identity.mapping_changed",
{ type: "media_entity", id },
{
...input,
retired: olds.map((o) => o.id),
identityId: newId,
},
);
return { mediaEntityId: id, changed: true, identityId: newId, retired: olds.map((o) => o.id) };
});
}
/** 外部身份字段修订(乐观并发 expectedRevision);每次变更写 identity_revision */
export async function updateExternalIdentity(
rt: SourcesRuntime,
id: string,
input: ExternalIdentityUpdate,
actor: Actor,
) {
return rt.ctx.db.transaction().execute(async (trx) => {
const row = await trx
.selectFrom("external_identity")
.selectAll()
.where("id", "=", id)
.forUpdate()
.executeTakeFirst();
if (!row) throw new ApiException("not_found", "外部身份不存在");
if (row.revision !== input.expectedRevision) {
throw new ApiException("precondition_failed", `修订号已变化(当前 ${row.revision})`);
}
const rev = Math.max(row.revision + 1, await nextRevision(trx, id));
const patch: Record<string, unknown> = { revision: rev };
if (input.status !== undefined) patch.status = input.status;
if (input.evidenceLevel !== undefined) patch.evidence_level = input.evidenceLevel;
if (input.evidence !== undefined) patch.evidence = jsonb(input.evidence);
if (input.locked !== undefined) patch.locked = input.locked;
if (input.validUntil !== undefined) patch.valid_until = input.validUntil;
if (input.extra !== undefined) patch.extra = jsonb(input.extra);
if (input.status === "retired" && input.validUntil === undefined) patch.valid_until = new Date();
let updated: ExternalIdentityRow;
try {
updated = await trx
.updateTable("external_identity")
.set(patch)
.where("id", "=", id)
.returningAll()
.executeTakeFirstOrThrow();
} catch (e) {
if ((e as { code?: string }).code === "23505")
throw new ApiException("conflict", "同一外部身份已有 accepted 映射");
throw e;
}
const action =
input.status !== undefined && input.status !== row.status
? "status_changed"
: input.locked !== undefined && input.locked !== row.locked
? input.locked
? "locked"
: "unlocked"
: "evidence_updated";
await trx
.insertInto("identity_revision")
.values({
external_identity_id: id,
revision: rev,
action,
from_status: row.status,
to_status: updated.status,
before: jsonb({ status: row.status, locked: row.locked, evidenceLevel: row.evidence_level }),
after: jsonb({ status: updated.status, locked: updated.locked, evidenceLevel: updated.evidence_level }),
actor_type: actor.type,
actor_id: actor.id,
reason: input.reason ?? null,
})
.execute();
await audit(trx, actor, "identity.updated", { type: "external_identity", id }, { action, revision: rev });
return updated;
});
}
/**
* 合并重复作品:from → into。旧 ID 保留(status=merged + merged_into_id)并写 identity_redirect;
* 本模块的引用(来源资源、别名、待识别池、外部身份)迁到目标;其它模块的表按 merged_into_id 跳转解析。
*/
export async function mergeEntities(
rt: SourcesRuntime,
fromId: string,
intoId: string,
reason: string | null,
actor: Actor,
) {
if (fromId === intoId) throw new ApiException("bad_request", "不能合并到自身");
return rt.ctx.db.transaction().execute(async (trx) => {
const from = await trx
.selectFrom("media_entity")
.selectAll()
.where("id", "=", fromId)
.forUpdate()
.executeTakeFirst();
const intoResolved = await resolveEntityId(trx, intoId);
if (!from) throw new ApiException("not_found", "源作品不存在");
if (!intoResolved) throw new ApiException("not_found", "目标作品不存在");
if (from.status === "merged") throw new ApiException("conflict", "源作品已被合并");
if (intoResolved === fromId) throw new ApiException("conflict", "目标作品已合并到源作品,会形成环");
// 外部身份:目标已有同 provider+类型 accepted 的,源的退役;否则迁到目标
const idents = await trx
.selectFrom("external_identity")
.selectAll()
.where("media_entity_id", "=", fromId)
.where("status", "=", "accepted")
.execute();
for (const i of idents) {
const clash = await trx
.selectFrom("external_identity")
.select(["id"])
.where("media_entity_id", "=", intoResolved)
.where("provider", "=", i.provider)
.where("entity_type", "=", i.entity_type)
.where("status", "=", "accepted")
.executeTakeFirst();
if (clash) {
await retireIdentity(trx, i, actor, `merged into ${intoResolved}`);
continue;
}
const rev = Math.max(i.revision + 1, await nextRevision(trx, i.id));
await trx
.updateTable("external_identity")
.set({ media_entity_id: intoResolved, revision: rev })
.where("id", "=", i.id)
.execute();
await trx
.insertInto("identity_revision")
.values({
external_identity_id: i.id,
revision: rev,
action: "relinked",
from_status: i.status,
to_status: i.status,
before: jsonb({ mediaEntityId: fromId }),
after: jsonb({ mediaEntityId: intoResolved }),
actor_type: actor.type,
actor_id: actor.id,
reason,
})
.execute();
}
await sql`insert into title_alias (media_entity_id, content_unit_id, alias, normalized, language, kind, source, locked)
select ${intoResolved}, null, alias, normalized, language, kind, source, locked from title_alias
where media_entity_id = ${fromId} and content_unit_id is null
on conflict (media_entity_id, content_unit_id, normalized, kind) do nothing`.execute(trx);
await trx
.updateTable("source_release")
.set({ media_entity_id: intoResolved })
.where("media_entity_id", "=", fromId)
.execute();
await trx
.updateTable("identity_candidate")
.set({ resolved_media_entity_id: intoResolved })
.where("resolved_media_entity_id", "=", fromId)
.execute();
await trx
.updateTable("media_entity")
.set({ status: "merged", merged_into_id: intoResolved })
.where("id", "=", fromId)
.execute();
await trx
.insertInto("identity_redirect")
.values({
from_entity_id: fromId,
to_entity_id: intoResolved,
reason,
actor_type: actor.type,
actor_id: actor.id,
})
.execute();
await audit(
trx,
actor,
"identity.entity_merged",
{ type: "media_entity", id: fromId },
{ intoId: intoResolved, reason },
);
return trx.selectFrom("media_entity").selectAll().where("id", "=", intoResolved).executeTakeFirstOrThrow();
});
}
export async function dismissPoolEntry(rt: SourcesRuntime, poolId: string, actor: Actor, note: string | null) {
const { ctx } = rt;
return ctx.db.transaction().execute(async (trx) => {
const row = await trx
.updateTable("identity_candidate")
.set({ status: "dismissed", next_review_at: null, resolved_by: actorName(actor), resolved_at: new Date() })
.where("id", "=", poolId)
.returningAll()
.executeTakeFirst();
if (!row) throw new ApiException("not_found", "待识别条目不存在");
await trx
.updateTable("source_release")
.set({ resolution_status: "rejected" })
.where("id", "=", row.source_release_id)
.where("resolution_locked", "=", false)
.execute();
await closeDecisions(ctx, trx, `identity:${row.source_release_id}`, "dismissed", { note }, actorName(actor));
await audit(trx, actor, "identity.pool_dismissed", { type: "identity_candidate", id: poolId }, { note });
return row;
});
}
@@ -0,0 +1,254 @@
import { type MetadataCandidate, TmdbError } from "@umt/adapters";
import type { EntityMetadataField, FieldOverrideInput, FinalFieldValue, MetadataSource } from "@umt/contracts";
import { sql } from "@umt/db";
import { ApiException } from "../../../http/errors.ts";
import { jsonb } from "../rows.ts";
import type { SourcesRuntime } from "../runtime.ts";
import { type Actor, audit, type Exec } from "./common.ts";
import { resolveEntityId, seedAliases } from "./entities.ts";
// refreshMetadata:字段级刷新(REQ §7.3)。来源值写 metadata_field;最终值 = 覆盖(锁)> 已锁字段保持 > 来源值。
// 只更新 media_entity 的展示字段与别名,不触碰 path_binding(路径绑定不随元数据变化)。
/** 最终值投影到 media_entity 列的字段 */
const COLUMN_FIELDS: Partial<Record<EntityMetadataField, "title" | "original_title" | "year">> = {
title: "title",
originalTitle: "original_title",
year: "year",
};
function tmdbFields(m: MetadataCandidate): Partial<Record<EntityMetadataField, unknown>> {
return {
title: m.title,
originalTitle: m.originalTitle,
year: m.year,
overview: m.overview,
releaseDate: m.releaseDate,
originalLanguage: m.originalLanguage,
originCountry: m.originCountry,
};
}
export interface RefreshResult {
status: "refreshed" | "skipped" | "failed";
reason?: string;
changedFields: string[];
lockedFields: string[];
}
export async function refreshMetadata(rt: SourcesRuntime, mediaEntityId: string): Promise<RefreshResult> {
const { ctx } = rt;
const id = await resolveEntityId(ctx.db, mediaEntityId);
if (!id) return { status: "skipped", reason: "not_found", changedFields: [], lockedFields: [] };
const ident = await ctx.db
.selectFrom("external_identity")
.select(["entity_type", "external_id"])
.where("media_entity_id", "=", id)
.where("provider", "=", "tmdb")
.where("status", "=", "accepted")
.executeTakeFirst();
const tmdb = rt.tmdb();
if (!ident || !tmdb || (ident.entity_type !== "movie" && ident.entity_type !== "tv")) {
return {
status: "skipped",
reason: ident ? "tmdb_not_configured" : "no_tmdb_identity",
changedFields: [],
lockedFields: [],
};
}
let detail: MetadataCandidate;
try {
detail = await tmdb.details(ident.entity_type, ident.external_id);
} catch (e) {
// 404/超时/鉴权失败只记录校验状态,不是删除映射的依据
if (e instanceof TmdbError) {
await ctx.db
.updateTable("external_identity")
.set({ verify_status: e.kind === "notfound" ? "not_found" : "error", verified_at: rt.now() })
.where("media_entity_id", "=", id)
.where("provider", "=", "tmdb")
.where("status", "=", "accepted")
.execute();
if (e.kind === "notfound" || e.kind === "auth") {
return { status: "failed", reason: e.kind, changedFields: [], lockedFields: [] };
}
}
throw e;
}
const now = rt.now();
return ctx.db.transaction().execute(async (trx) => {
const fields = tmdbFields(detail);
for (const [field, value] of Object.entries(fields)) {
await sql`insert into metadata_field (media_entity_id, content_unit_id, field, value, language, source, source_revision, fetched_at)
values (${id}, null, ${field}, ${jsonb(value)}::jsonb, ${null}, 'tmdb', ${`tmdb:${ident.entity_type}/${ident.external_id}`}, ${now})
on conflict (media_entity_id, content_unit_id, field, source, language) do update set
value = excluded.value, source_revision = excluded.source_revision, fetched_at = excluded.fetched_at`.execute(
trx,
);
}
await trx
.updateTable("external_identity")
.set({ verify_status: "ok", verified_at: now })
.where("media_entity_id", "=", id)
.where("provider", "=", "tmdb")
.where("status", "=", "accepted")
.execute();
await seedAliases(trx, id, detail);
const r = await project(trx, id, now);
await trx.updateTable("media_entity").set({ last_metadata_refresh_at: now }).where("id", "=", id).execute();
return { status: "refreshed" as const, ...r };
});
}
/**
* 重新投影最终字段:覆盖(用户锁)> field_locks 中已锁字段保持现值 > tmdb 来源值。
* 返回被改变与被锁住的字段。
*/
export async function project(
trx: Exec,
entityId: string,
now: Date,
): Promise<{ changedFields: string[]; lockedFields: string[] }> {
const e = await trx
.selectFrom("media_entity")
.selectAll()
.where("id", "=", entityId)
.forUpdate()
.executeTakeFirstOrThrow();
const overrides = await trx
.selectFrom("field_override")
.selectAll()
.where("media_entity_id", "=", entityId)
.execute();
const sources = await trx
.selectFrom("metadata_field")
.selectAll()
.where("media_entity_id", "=", entityId)
.orderBy("fetched_at", "desc")
.execute();
const meta = { ...((e.metadata ?? {}) as Record<string, FinalFieldValue>) };
const locks = new Set(e.field_locks);
const changed: string[] = [];
const lockedFields: string[] = [];
const colPatch: Record<string, unknown> = {};
const fieldNames = new Set([...sources.map((s) => s.field), ...overrides.map((o) => o.field)]);
for (const field of fieldNames) {
const ov = overrides.find((o) => o.field === field);
let next: FinalFieldValue | null = null;
if (ov) {
next = {
value: ov.value,
source: ov.source as MetadataSource,
language: null,
at: ov.updated_at.toISOString(),
locked: ov.locked,
};
if (ov.locked) lockedFields.push(field);
} else if (locks.has(field)) {
lockedFields.push(field);
continue;
} else {
const src =
sources.find((s) => s.field === field && s.source === "tmdb") ?? sources.find((s) => s.field === field);
if (!src) continue;
next = {
value: src.value,
source: src.source as MetadataSource,
language: src.language,
at: src.fetched_at.toISOString(),
locked: false,
};
}
const prev = meta[field];
if (!prev || JSON.stringify(prev.value) !== JSON.stringify(next.value) || prev.source !== next.source) {
changed.push(field);
}
meta[field] = next;
const col = COLUMN_FIELDS[field as EntityMetadataField];
if (col) {
const v = next.value;
if (col === "year") colPatch.year = typeof v === "number" ? v : null;
else if (col === "title") {
if (typeof v === "string" && v) colPatch.title = v;
} else colPatch.original_title = typeof v === "string" ? v : null;
}
}
await trx
.updateTable("media_entity")
.set({ ...colPatch, metadata: jsonb(meta), updated_at: now })
.where("id", "=", entityId)
.execute();
return { changedFields: changed, lockedFields };
}
/** 用户覆盖字段并(默认)锁定:之后的自动刷新不得改写 */
export async function setFieldOverride(
rt: SourcesRuntime,
entityId: string,
field: EntityMetadataField,
input: FieldOverrideInput,
actor: Actor,
) {
return rt.ctx.db.transaction().execute(async (trx) => {
const id = await resolveEntityId(trx, entityId);
if (!id) throw new ApiException("not_found", "作品不存在");
if (field === "title" && (typeof input.value !== "string" || !input.value.trim())) {
throw new ApiException("bad_request", "title 必须是非空字符串");
}
if (
field === "year" &&
input.value !== null &&
(typeof input.value !== "number" || !Number.isInteger(input.value))
) {
throw new ApiException("bad_request", "year 必须是整数或 null");
}
await sql`insert into field_override (media_entity_id, content_unit_id, field, value, locked, source, reason, actor_type, actor_id)
values (${id}, null, ${field}, ${jsonb(input.value)}::jsonb, ${input.lock}, 'user', ${input.reason ?? null}, ${actor.type}, ${actor.id})
on conflict (media_entity_id, content_unit_id, field) do update set
value = excluded.value, locked = excluded.locked, reason = excluded.reason,
actor_type = excluded.actor_type, actor_id = excluded.actor_id`.execute(trx);
const e = await trx
.selectFrom("media_entity")
.select(["field_locks"])
.where("id", "=", id)
.executeTakeFirstOrThrow();
const locks = new Set(e.field_locks);
if (input.lock) locks.add(field);
else locks.delete(field);
await trx
.updateTable("media_entity")
.set({ field_locks: [...locks].sort() })
.where("id", "=", id)
.execute();
const r = await project(trx, id, rt.now());
await audit(trx, actor, "metadata.field_override", { type: "media_entity", id }, { field, lock: input.lock });
return { mediaEntityId: id, ...r };
});
}
/** 取消覆盖与锁定:回到来源值 */
export async function clearFieldOverride(
rt: SourcesRuntime,
entityId: string,
field: EntityMetadataField,
actor: Actor,
) {
return rt.ctx.db.transaction().execute(async (trx) => {
const id = await resolveEntityId(trx, entityId);
if (!id) throw new ApiException("not_found", "作品不存在");
await trx.deleteFrom("field_override").where("media_entity_id", "=", id).where("field", "=", field).execute();
const e = await trx
.selectFrom("media_entity")
.select(["field_locks"])
.where("id", "=", id)
.executeTakeFirstOrThrow();
await trx
.updateTable("media_entity")
.set({ field_locks: e.field_locks.filter((f) => f !== field) })
.where("id", "=", id)
.execute();
const r = await project(trx, id, rt.now());
await audit(trx, actor, "metadata.field_override_cleared", { type: "media_entity", id }, { field });
return { mediaEntityId: id, ...r };
});
}
@@ -0,0 +1,637 @@
import { createHash } from "node:crypto";
import { type MetadataCandidate, TmdbError } from "@umt/adapters";
import type { EvidenceLevel, PoolCandidate } from "@umt/contracts";
import { type SourceReleaseRow, sql } from "@umt/db";
import {
type IdentityCandidate as DomainCandidate,
type IdentityResolution,
normalizeTitle,
PARSER_VERSION,
type ParsedRelease,
parseRelease,
queryTitles,
type RankedCandidate,
resolveIdentity,
} from "@umt/domain";
import {
localIdentityCandidate,
toContractEvidenceLevel,
toContractResolveStatus,
toParsedColumns,
toReleaseResolutionStatus,
} from "@umt/mapping";
import { jsonb } from "../rows.ts";
import type { SourcesRuntime } from "../runtime.ts";
import { type Actor, closeDecisions, type Exec, openDecision, SYSTEM } from "./common.ts";
import {
addAlias,
createEntityFromTmdb,
ensureUnits,
entityByExternal,
IdentityConflictError,
looksLikeVariety,
resolveEntityId,
} from "./entities.ts";
// resolveObservation:parseRelease → 本地召回(别名 pg_trgm + 已有外部身份 + 人工锁)→ TMDB 候选 → domain.resolveIdentity。
// auto 才建/关联正式作品;ambiguous / no_match / 查询失败进待识别池(候选只存 JSON),有界退避复查。
const MAX_REVIEW_ATTEMPTS = 8;
const TRGM_THRESHOLD = 0.45;
export interface ResolveInput {
observationId?: string | undefined;
sourceReleaseId?: string | undefined;
/** 复查/手动重试:忽略"已识别且证据未变"的短路 */
force?: boolean | undefined;
}
export interface ResolveResult {
outcome: "auto" | "proposed" | "ambiguous" | "no_match" | "lookup_failed" | "skipped_locked" | "skipped";
sourceReleaseId: string | null;
mediaEntityId: string | null;
contentUnitIds: string[];
decisionId: string | null;
reason: string;
}
/** 候选的稳定键 */
const candKey = (c: DomainCandidate) =>
c.localEntityId ? `local:${c.localEntityId}` : `${c.provider}:${c.entityType}:${c.externalId}`;
export function evidenceVersion(parsed: ParsedRelease | null, ranked: RankedCandidate[]): string {
const h = createHash("sha1")
.update(
JSON.stringify([parsed?.title ?? null, parsed?.year ?? null, ranked.map((r) => [candKey(r.candidate), r.level])]),
)
.digest("hex")
.slice(0, 16);
return `p${PARSER_VERSION}:${h}`;
}
function yearOf(d: string | null | undefined): number | undefined {
return d && /^\d{4}/.test(d) ? Number(d.slice(0, 4)) : undefined;
}
export function tmdbToCandidate(m: MetadataCandidate): DomainCandidate {
const titles = [m.title, ...(m.originalTitle ? [m.originalTitle] : [])];
for (const t of m.titles) if (t.kind !== "alias" && !titles.includes(t.text)) titles.push(t.text);
const aliases = [...new Set([...m.aliases, ...m.titles.filter((t) => t.kind === "alias").map((t) => t.text)])];
const seasonYears = m.seasons?.map((s) => yearOf(s.airDate)).filter((y): y is number => y !== undefined);
return {
provider: "tmdb",
entityType: m.entityType,
externalId: m.externalId,
titles,
...(m.year !== null ? { year: m.year } : {}),
aliases,
...(seasonYears?.length ? { seasonYears } : {}),
};
}
function poolCandidate(r: RankedCandidate): PoolCandidate {
const c = r.candidate;
return {
provider: c.provider,
entityType: c.entityType,
externalId: c.externalId,
title: c.titles[0] ?? c.externalId,
year: c.year ?? null,
localEntityId: c.localEntityId ?? null,
level: r.level,
evidence: r.evidence,
};
}
/** 本地召回:显式外部 ID → 已接受映射;别名精确/trgm 模糊(跟随合并跳转),构造本地候选 */
async function localCandidates(
exec: Exec,
parsed: ParsedRelease,
release: SourceReleaseRow,
): Promise<DomainCandidate[]> {
const ids = new Set<string>();
const site = ((release.raw as Record<string, unknown>)?._site ?? {}) as { imdbId?: string };
const explicit = [...parsed.explicitIds, ...(site.imdbId ? [{ provider: "imdb" as const, id: site.imdbId }] : [])];
for (const x of explicit) {
const row = await exec
.selectFrom("external_identity")
.select(["media_entity_id"])
.where("provider", "=", x.provider)
.where("external_id", "=", x.id)
.where("status", "=", "accepted")
.where("media_entity_id", "is not", null)
.execute();
for (const r of row) if (r.media_entity_id) ids.add(r.media_entity_id);
}
for (const q of queryTitles(parsed)) {
const rows = await sql<{ media_entity_id: string }>`
select distinct media_entity_id from title_alias
where normalized = ${q} or (length(${q}) >= 3 and similarity(normalized, ${q}) >= ${TRGM_THRESHOLD})
limit 20`.execute(exec);
for (const r of rows.rows) ids.add(r.media_entity_id);
}
const final = new Set<string>();
for (const id of ids) {
const r = await resolveEntityId(exec, id);
if (r) final.add(r);
}
const out: DomainCandidate[] = [];
for (const id of final) {
const e = await exec.selectFrom("media_entity").selectAll().where("id", "=", id).executeTakeFirst();
if (!e || e.status === "archived") continue;
const aliases = await exec.selectFrom("title_alias").select(["alias"]).where("media_entity_id", "=", id).execute();
const idents = await exec
.selectFrom("external_identity")
.select(["provider", "external_id", "status"])
.where("media_entity_id", "=", id)
.execute();
const c = localIdentityCandidate(
{ id: e.id, kind: e.kind, title: e.title, originalTitle: e.original_title, year: e.year },
aliases.map((a) => a.alias),
idents.map((i) => ({ provider: i.provider, externalId: i.external_id, status: i.status })),
);
if (c) out.push(c);
}
return out;
}
async function tmdbCandidates(rt: SourcesRuntime, parsed: ParsedRelease): Promise<MetadataCandidate[]> {
const tmdb = rt.tmdb();
if (!tmdb) return [];
const queries = [parsed.titleCn, parsed.titleEn, parsed.title].filter((t): t is string => !!t);
const seen = new Map<string, MetadataCandidate>();
for (const q of [...new Set(queries)].slice(0, 2)) {
for (const m of await tmdb.searchAll(q)) {
const k = `${m.entityType}:${m.externalId}`;
if (!seen.has(k)) seen.set(k, m);
}
}
return [...seen.values()].slice(0, 20);
}
async function loadRelease(rt: SourcesRuntime, input: ResolveInput) {
const { db } = rt.ctx;
const observationId = input.observationId ?? null;
let srId = input.sourceReleaseId ?? null;
if (!srId && observationId) {
const o = await db
.selectFrom("observation")
.select(["source_release_id"])
.where("id", "=", observationId)
.executeTakeFirst();
srId = o?.source_release_id ?? null;
}
if (!srId) return { release: null, observationId };
const release = await db.selectFrom("source_release").selectAll().where("id", "=", srId).executeTakeFirst();
return { release: release ?? null, observationId };
}
export async function resolveRelease(rt: SourcesRuntime, input: ResolveInput): Promise<ResolveResult> {
const { ctx } = rt;
const { release, observationId } = await loadRelease(rt, input);
const base = { mediaEntityId: null, contentUnitIds: [] as string[], decisionId: null };
if (!release) return { outcome: "skipped", sourceReleaseId: null, reason: "release_not_found", ...base };
const pool = await ctx.db
.selectFrom("identity_candidate")
.selectAll()
.where("source_release_id", "=", release.id)
.executeTakeFirst();
if (release.resolution_locked) {
await recordAttempt(
ctx.db,
release.id,
observationId,
"skipped_locked",
"manual_lock",
"locked",
0,
release.media_entity_id,
);
return {
outcome: "skipped_locked",
sourceReleaseId: release.id,
reason: "manual_lock",
...base,
mediaEntityId: release.media_entity_id,
};
}
if (pool?.status === "dismissed" && !input.force) {
return { outcome: "skipped", sourceReleaseId: release.id, reason: "dismissed", ...base };
}
// 已识别:除非强制(复查/标题更新/人工重试),不重复召回
if (!input.force && release.resolution_status === "resolved" && release.media_entity_id) {
return {
outcome: "skipped",
sourceReleaseId: release.id,
reason: "already_resolved",
...base,
mediaEntityId: release.media_entity_id,
};
}
const parsed = parseRelease(release.title, release.subtitle ? { subtitle: release.subtitle } : {});
const cols = toParsedColumns(parsed);
if (!parsed.title || parsed.auxiliaryOnly) {
return toPool(rt, release, observationId, parsed, null, [], "no_match", "unparseable_title", null);
}
// 召回:本地优先,再 TMDB;TMDB 失败不等于无结果
const local = await localCandidates(ctx.db, parsed, release);
let remote: MetadataCandidate[] = [];
let lookupError: string | null = null;
try {
remote = await tmdbCandidates(rt, parsed);
} catch (e) {
if (!(e instanceof TmdbError)) throw e;
lookupError = `${e.kind}: ${e.message}`;
}
const candidates = [...local, ...remote.map(tmdbToCandidate)];
const resolution = resolveIdentity(parsed, candidates);
const version = evidenceVersion(parsed, resolution.ranked);
// 已识别且证据未变:幂等短路(强制复查时)
if (
release.resolution_status === "resolved" &&
release.media_entity_id &&
(release.resolution_evidence as { evidenceVersion?: string })?.evidenceVersion === version
) {
return {
outcome: "skipped",
sourceReleaseId: release.id,
reason: "unchanged",
...base,
mediaEntityId: release.media_entity_id,
};
}
const automation = await ctx.settings.get("automation");
const autoAccept = automation.enabled && !automation.readOnlyReconcile;
if (resolution.status === "auto" && resolution.best && autoAccept) {
const best = resolution.best;
const remoteBest =
best.candidate.provider === "tmdb"
? remote.find((m) => m.externalId === best.candidate.externalId && m.entityType === best.candidate.entityType)
: undefined;
let detailed = remoteBest;
if (remoteBest && !remoteBest.detailed) {
const existing = await entityByExternal(ctx.db, {
provider: "tmdb",
entityType: remoteBest.entityType,
externalId: remoteBest.externalId,
});
if (!existing) {
try {
detailed = await rt.tmdb()!.details(remoteBest.entityType, remoteBest.externalId);
} catch (e) {
if (!(e instanceof TmdbError)) throw e;
detailed = remoteBest;
}
}
}
return bindAuto(rt, release, observationId, parsed, cols, resolution, version, detailed ?? null);
}
if (resolution.status === "auto" && !autoAccept) {
return toPool(
rt,
release,
observationId,
parsed,
resolution,
resolution.ranked,
"proposed",
"automation_off",
version,
);
}
if (lookupError && resolution.status !== "ambiguous") {
return toPool(
rt,
release,
observationId,
parsed,
resolution,
resolution.ranked,
"lookup_failed",
lookupError,
version,
);
}
return toPool(
rt,
release,
observationId,
parsed,
resolution,
resolution.ranked,
resolution.status === "ambiguous" ? "ambiguous" : "no_match",
resolution.reason,
version,
);
}
async function recordAttempt(
exec: Exec,
srId: string,
observationId: string | null,
outcome: "auto" | "proposed" | "ambiguous" | "no_match" | "lookup_failed" | "manual" | "skipped_locked",
reason: string | null,
version: string,
count: number,
entityId: string | null,
best: unknown = null,
error: unknown = null,
actor: Actor = SYSTEM,
): Promise<void> {
await exec
.insertInto("resolution_attempt")
.values({
source_release_id: srId,
observation_id: observationId,
outcome,
reason,
evidence_version: version,
candidate_count: count,
media_entity_id: entityId,
best: best === null ? null : jsonb(best),
error: error === null ? null : jsonb(error),
actor_type: actor.type,
actor_id: actor.id,
})
.execute();
}
function rankedSummary(r: RankedCandidate | undefined) {
return r ? { ...poolCandidate(r), titleMatch: r.titleMatch, yearMatch: r.yearMatch, typeMatch: r.typeMatch } : null;
}
async function bindAuto(
rt: SourcesRuntime,
release: SourceReleaseRow,
observationId: string | null,
parsed: ParsedRelease,
cols: ReturnType<typeof toParsedColumns>,
resolution: IdentityResolution,
version: string,
tmdbDetail: MetadataCandidate | null,
): Promise<ResolveResult> {
const { ctx } = rt;
const best = resolution.best!;
const level: EvidenceLevel = toContractEvidenceLevel(best.level) ?? "needs_review";
const site = ((release.raw as Record<string, unknown>)?._site ?? {}) as { categoryName?: string };
const variety = looksLikeVariety(parsed, site.categoryName ?? release.category_raw);
for (let attempt = 0; ; attempt++) {
try {
return await ctx.db.transaction().execute(async (trx) => {
let entityId: string | null = null;
if (best.candidate.localEntityId) entityId = await resolveEntityId(trx, best.candidate.localEntityId);
if (!entityId && best.candidate.provider === "tmdb") {
entityId = await entityByExternal(trx, {
provider: "tmdb",
entityType: best.candidate.entityType,
externalId: best.candidate.externalId,
});
if (!entityId && tmdbDetail) {
entityId = (
await createEntityFromTmdb(trx, tmdbDetail, {
level,
evidence: {
sourceReleaseId: release.id,
rules: best.evidence,
titleMatch: best.titleMatch,
yearMatch: best.yearMatch,
},
actor: SYSTEM,
variety,
})
).entityId;
}
}
if (!entityId)
throw new Error(`auto 结果缺少可绑定的本地作品(${best.candidate.provider}:${best.candidate.externalId})`);
const entity = await trx
.selectFrom("media_entity")
.select(["id", "kind"])
.where("id", "=", entityId)
.executeTakeFirstOrThrow();
const units = await ensureUnits(trx, entityId, entity.kind, parsed);
// 站点标题作为别名:下次同名资源可直接本地召回
if (parsed.titleCn) await addAlias(trx, entityId, parsed.titleCn, "site_title", "site", "zh");
if (parsed.titleEn) await addAlias(trx, entityId, parsed.titleEn, "site_title", "site", "en");
const evidence = {
level,
reason: resolution.reason,
evidenceVersion: version,
best: rankedSummary(best),
candidateCount: resolution.ranked.length,
};
await trx
.updateTable("source_release")
.set({
parser_version: cols.parserVersion,
parsed: jsonb(cols.parsed),
media_entity_id: entityId,
content_unit_id: units.primary,
resolution_status: toReleaseResolutionStatus(toContractResolveStatus("auto", { autoAccept: true })),
resolution_evidence: jsonb(evidence),
})
.where("id", "=", release.id)
.where("resolution_locked", "=", false)
.execute();
await trx
.updateTable("identity_candidate")
.set({
status: "resolved",
resolved_media_entity_id: entityId,
resolved_by: "system",
resolved_at: new Date(),
next_review_at: null,
})
.where("source_release_id", "=", release.id)
.where("status", "=", "pending")
.execute();
await closeDecisions(
ctx,
trx,
`identity:${release.id}`,
"resolved",
{ mediaEntityId: entityId, by: "auto" },
"system",
);
await recordAttempt(
trx,
release.id,
observationId,
"auto",
resolution.reason,
version,
resolution.ranked.length,
entityId,
rankedSummary(best),
);
await ctx.outbox.emit(
trx,
"release.resolved",
{
sourceReleaseId: release.id,
observationId,
mediaEntityId: entityId,
contentUnitIds: units.unitIds,
evidenceLevel: level,
},
{ idempotencyKey: `release.resolved:${release.id}:${entityId}:${version}` },
);
return {
outcome: "auto" as const,
sourceReleaseId: release.id,
mediaEntityId: entityId,
contentUnitIds: units.unitIds,
decisionId: null,
reason: resolution.reason,
};
});
} catch (e) {
if (e instanceof IdentityConflictError && attempt < 2) continue;
throw e;
}
}
}
function reviewDelayMs(reason: string, attempts: number): number | null {
if (attempts >= MAX_REVIEW_ATTEMPTS) return null;
const base = reason === "lookup_failed" ? 15 * 60_000 : 3600_000;
return Math.min(base * 2 ** Math.max(0, attempts - 1), 7 * 24 * 3600_000);
}
async function toPool(
rt: SourcesRuntime,
release: SourceReleaseRow,
observationId: string | null,
parsed: ParsedRelease,
resolution: IdentityResolution | null,
ranked: RankedCandidate[],
reason: "ambiguous" | "no_match" | "proposed" | "lookup_failed",
detail: string,
versionIn: string | null,
): Promise<ResolveResult> {
const { ctx } = rt;
const version = versionIn ?? evidenceVersion(parsed, ranked);
const cols = toParsedColumns(parsed);
const candidates = ranked
.filter((r) => r.level !== "rejected")
.slice(0, 10)
.map(poolCandidate);
const now = rt.now();
const query = {
title: parsed.title ?? null,
titleCn: parsed.titleCn ?? null,
titleEn: parsed.titleEn ?? null,
normalized: parsed.title ? normalizeTitle(parsed.title) : null,
year: parsed.year ?? null,
season: parsed.season ?? null,
episodes: parsed.episodes ?? null,
};
const contractStatus =
reason === "proposed"
? toContractResolveStatus("auto", { autoAccept: false })
: reason === "ambiguous"
? "ambiguous"
: "unidentifiable";
return ctx.db.transaction().execute(async (trx) => {
const prev = await trx
.selectFrom("identity_candidate")
.select(["id", "attempts", "evidence_version"])
.where("source_release_id", "=", release.id)
.executeTakeFirst();
// 证据版本变化 = 新证据:从头计数;同一证据反复复查按退避递增,达上限停止自动复查(仍可检索)
const attempts = prev && prev.evidence_version === version ? prev.attempts + 1 : 1;
const delay = reviewDelayMs(reason, attempts);
const nextReviewAt = delay === null ? null : new Date(now.getTime() + delay);
let decisionId: string | null = null;
if ((reason === "ambiguous" || reason === "proposed") && candidates.length > 0) {
const d = await openDecision(ctx, trx, {
kind: "identity_ambiguous",
severity: "info",
title: `待确认身份:${release.title}`,
summary: `${candidates.length} 个候选证据等级相近(${detail}),请选择或忽略`,
subjectType: "source_release",
subjectId: release.id,
dedupeKey: `identity:${release.id}`,
evidence: {
sourceReleaseId: release.id,
siteId: release.site_id,
reason,
detail,
evidenceVersion: version,
query,
},
options: [
...candidates.slice(0, 5).map((c, i) => ({
key: `c${i}`,
label: `${c.title}${c.year ? ` (${c.year})` : ""} · ${c.localEntityId ? "本地作品" : `${c.provider}:${c.entityType}/${c.externalId}`}`,
input: { sourceReleaseId: release.id, candidateIndex: i },
})),
{ key: "dismiss", label: "忽略此资源" },
],
nextReviewAt,
});
decisionId = d.id;
}
await sql`
insert into identity_candidate (source_release_id, status, reason, query, candidates, evidence_version, attempts,
last_attempt_at, next_review_at, decision_id)
values (${release.id}, 'pending', ${reason}, ${jsonb(query)}::jsonb, ${jsonb(candidates)}::jsonb, ${version}, ${attempts},
${now}, ${nextReviewAt}, ${decisionId})
on conflict (source_release_id) do update set
status = 'pending', reason = excluded.reason, query = excluded.query, candidates = excluded.candidates,
evidence_version = excluded.evidence_version, attempts = excluded.attempts, last_attempt_at = excluded.last_attempt_at,
next_review_at = excluded.next_review_at, decision_id = coalesce(excluded.decision_id, identity_candidate.decision_id),
resolved_media_entity_id = null, resolved_by = null, resolved_at = null`.execute(trx);
await trx
.updateTable("source_release")
.set({
parser_version: cols.parserVersion,
parsed: jsonb(cols.parsed),
resolution_status: toReleaseResolutionStatus(contractStatus),
resolution_evidence: jsonb({ reason, detail, evidenceVersion: version, candidateCount: candidates.length }),
})
.where("id", "=", release.id)
.where("resolution_locked", "=", false)
.execute();
if (observationId) {
await trx
.updateTable("observation")
.set((eb) => ({ resolve_attempts: eb("resolve_attempts", "+", 1), next_resolve_at: nextReviewAt }))
.where("id", "=", observationId)
.execute();
}
await recordAttempt(
trx,
release.id,
observationId,
reason,
detail,
version,
candidates.length,
null,
rankedSummary(resolution?.best),
reason === "lookup_failed" ? { message: detail } : null,
);
return {
outcome: reason,
sourceReleaseId: release.id,
mediaEntityId: null,
contentUnitIds: [],
decisionId,
reason: detail,
};
});
}
export { recordAttempt };
@@ -0,0 +1,229 @@
import { SiteError, type SitePage, type SiteRelease } from "@umt/adapters";
import type { SiteSnapshotKind } from "@umt/contracts";
import { type SiteRow, sql } from "@umt/db";
import { parseRelease } from "@umt/domain";
import { siteReleaseToSourceRelease } from "@umt/mapping";
import { jsonb } from "../rows.ts";
import type { SourcesRuntime } from "../runtime.ts";
import { type Exec, openDecision } from "./common.ts";
// 站点快照(周期列表)、补漏(Feed 疑似缺口,有页数预算)、实时补查(工作台搜索)。
// 所有请求经站点共享限流器;结果按 site+站点资源 ID upsert 到 source_release,并记录 site_snapshot 覆盖范围。
export interface SnapshotInput {
siteId: string;
kind: SiteSnapshotKind;
query?: string | undefined;
/** 页数预算(1 起连续页);默认 site.config.snapshotPages ?? 1,上限 20 */
maxPages?: number | undefined;
}
export interface SnapshotResult {
status: "ok" | "partial" | "failed" | "skipped";
reason?: string;
snapshotId: string | null;
pages: number[];
itemCount: number;
newCount: number;
releaseIds: string[];
error?: string;
}
/** SiteRelease → source_release upsert;新行发 release.observed */
export async function upsertSiteRelease(
rt: SourcesRuntime,
trx: Exec,
site: Pick<SiteRow, "id">,
r: SiteRelease,
): Promise<{ id: string; inserted: boolean }> {
const parsed = parseRelease(r.title, r.subtitle ? { subtitle: r.subtitle } : {});
const c = siteReleaseToSourceRelease(r, { siteId: site.id, parsed });
const row = (
await sql<{ id: string; inserted: boolean }>`
insert into source_release (site_id, site_release_id, title, subtitle, category_raw, tags_raw, size_bytes, infohash_v1,
infohash_v2, promotion, promotion_expires_at, seeders, leechers, source_published_at, parsed, parser_version, raw,
detail_fetched_at)
values (${site.id}, ${c.siteReleaseId}, ${c.title}, ${c.subtitle ?? null}, ${c.categoryRaw ?? null}, ${c.tagsRaw ?? []},
${c.sizeBytes ?? null}, ${c.infohashV1 ?? null}, ${c.infohashV2 ?? null}, ${jsonb(c.promotion ?? {})}::jsonb,
${c.promotionExpiresAt ?? null}, ${c.seeders ?? null}, ${c.leechers ?? null}, ${c.sourcePublishedAt ?? null},
${jsonb(c.parsed ?? {})}::jsonb, ${c.parserVersion ?? null}, ${jsonb(c.raw ?? {})}::jsonb, ${null})
on conflict (site_id, site_release_id) do update set
title = excluded.title, subtitle = coalesce(excluded.subtitle, source_release.subtitle),
category_raw = coalesce(excluded.category_raw, source_release.category_raw), tags_raw = excluded.tags_raw,
size_bytes = coalesce(excluded.size_bytes, source_release.size_bytes),
infohash_v1 = coalesce(excluded.infohash_v1, source_release.infohash_v1),
infohash_v2 = coalesce(excluded.infohash_v2, source_release.infohash_v2),
promotion = excluded.promotion, promotion_expires_at = excluded.promotion_expires_at,
seeders = excluded.seeders, leechers = excluded.leechers,
source_published_at = coalesce(source_release.source_published_at, excluded.source_published_at),
raw = source_release.raw || excluded.raw, last_seen_at = now(),
availability = case when source_release.availability = 'expired' then 'active' else source_release.availability end
returning id, (xmax = 0) as inserted`.execute(trx)
).rows[0]!;
if (row.inserted) {
await rt.ctx.outbox.emit(
trx,
"release.observed",
{ sourceReleaseId: row.id, observationId: null, siteId: site.id, isNew: true },
{ idempotencyKey: `release.observed:${row.id}` },
);
}
return row;
}
/** 凭据失效:账号标失效 + 站点健康 failing + 待处理(不重试) */
async function onAuthError(rt: SourcesRuntime, trx: Exec, site: SiteRow, e: SiteError) {
await trx
.updateTable("site_account")
.set({ status: e.kind === "auth_expired" ? "expired" : "invalid", last_verified_at: rt.now() })
.where("site_id", "=", site.id)
.where("enabled", "=", true)
.where("status", "in", ["unknown", "active"])
.execute();
await trx.updateTable("site").set({ health_status: "failing" }).where("id", "=", site.id).execute();
await openDecision(rt.ctx, trx, {
kind: "source_unreachable",
severity: "warning",
title: `站点「${site.name}」登录失效`,
summary: `${e.kind}:${e.message}。已停止该站自动访问,请更新账号凭证后恢复。`,
subjectType: "site",
subjectId: site.id,
dedupeKey: `site-auth:${site.id}`,
evidence: { siteId: site.id, kind: e.kind, httpStatus: e.httpStatus ?? null },
options: [{ key: "reset", label: "已更新凭证,恢复访问" }],
});
}
export async function snapshotSite(rt: SourcesRuntime, input: SnapshotInput): Promise<SnapshotResult> {
const { db } = rt.ctx;
const none = { snapshotId: null, pages: [], itemCount: 0, newCount: 0, releaseIds: [] };
const site = await db.selectFrom("site").selectAll().where("id", "=", input.siteId).executeTakeFirst();
if (!site) return { status: "skipped", reason: "not_found", ...none };
if (!site.enabled) return { status: "skipped", reason: "site_disabled", ...none };
if (site.health_status === "failing" && input.kind !== "manual" && input.kind !== "live_search") {
const invalid = await db
.selectFrom("site_account")
.select(["id"])
.where("site_id", "=", site.id)
.where("status", "in", ["invalid", "expired"])
.executeTakeFirst();
if (invalid) return { status: "skipped", reason: "auth_invalid", ...none };
}
let adapter: ReturnType<SourcesRuntime["siteAdapter"]>;
try {
adapter = rt.siteAdapter(site);
} catch (e) {
return { status: "skipped", reason: (e as Error).message, ...none };
}
const cfgPages = Number((site.config as { snapshotPages?: number })?.snapshotPages) || 1;
const maxPages = Math.min(20, Math.max(1, input.maxPages ?? cfgPages));
const snap = await db
.insertInto("site_snapshot")
.values({
site_id: site.id,
kind: input.kind,
query: input.query ?? null,
filters: jsonb({}),
definition_version: adapter.definitionVersion,
started_at: rt.now(),
})
.returning(["id"])
.executeTakeFirstOrThrow();
const pages: number[] = [];
const releases: SiteRelease[] = [];
let error: SiteError | Error | null = null;
for (let p = 1; p <= maxPages; p++) {
try {
const res: SitePage = input.query ? await adapter.search(input.query, { page: p }) : await adapter.list(p);
pages.push(p);
releases.push(...res.items);
if (res.items.length === 0 || (res.totalPages !== null && p >= res.totalPages)) break;
} catch (e) {
error = e as Error;
break;
}
}
return db.transaction().execute(async (trx) => {
const ids: string[] = [];
let newCount = 0;
for (const r of releases) {
const row = await upsertSiteRelease(rt, trx, site, r);
ids.push(row.id);
if (row.inserted) newCount++;
}
const status = error ? (pages.length ? "partial" : "failed") : "ok";
const errJson = error
? { kind: error instanceof SiteError ? error.kind : "internal", message: error.message }
: null;
await trx
.updateTable("site_snapshot")
.set({
status,
pages: jsonb(pages),
item_count: releases.length,
new_count: newCount,
error: errJson ? jsonb(errJson) : null,
finished_at: rt.now(),
})
.where("id", "=", snap.id)
.execute();
if (error instanceof SiteError && error.isAuthError) {
await onAuthError(rt, trx, site, error);
} else if (input.kind !== "live_search") {
await trx
.updateTable("site")
.set({
...(status !== "failed" ? { last_snapshot_at: rt.now() } : {}),
health_status: status === "ok" ? "healthy" : "degraded",
snapshot_cursor: jsonb({
lastKind: input.kind,
lastPages: pages,
lastItemCount: releases.length,
lastStatus: status,
lastError: errJson,
at: rt.now().toISOString(),
}),
})
.where("id", "=", site.id)
.execute();
}
return {
status,
snapshotId: snap.id,
pages,
itemCount: releases.length,
newCount,
releaseIds: ids,
...(errJson ? { error: `${errJson.kind}: ${errJson.message}` } : {}),
};
});
}
/** 详情刷新(工作台 / release.inspect):失败保留既有数据并返回原因 */
export async function refreshReleaseDetail(rt: SourcesRuntime, srId: string): Promise<string | null> {
const { db } = rt.ctx;
const rel = await db.selectFrom("source_release").selectAll().where("id", "=", srId).executeTakeFirst();
if (!rel) return "not_found";
const site = await db.selectFrom("site").selectAll().where("id", "=", rel.site_id).executeTakeFirst();
if (!site?.enabled) return "site_disabled";
let detail: SiteRelease;
try {
detail = await rt.siteAdapter(site).detail(rel.site_release_id);
} catch (e) {
if (e instanceof SiteError) {
if (e.isAuthError) await db.transaction().execute((trx) => onAuthError(rt, trx, site, e));
if (e.kind === "not_found") {
await db.updateTable("source_release").set({ availability: "expired" }).where("id", "=", srId).execute();
}
return `${e.kind}: ${e.message}`;
}
return (e as Error).message;
}
await db.transaction().execute(async (trx) => {
await upsertSiteRelease(rt, trx, site, detail);
await trx.updateTable("source_release").set({ detail_fetched_at: rt.now() }).where("id", "=", srId).execute();
});
return null;
}
@@ -0,0 +1,218 @@
import type {
ReleaseOrigin,
WorkbenchCoverage,
WorkbenchRelease,
WorkbenchReleaseDetail,
WorkbenchReleaseQuery,
} from "@umt/contracts";
import { type SiteRow, sql } from "@umt/db";
import { normalizeTitle } from "@umt/domain";
import { ApiException } from "../../../http/errors.ts";
import { decodeCursor, page, toApi } from "../rows.ts";
import type { SourcesRuntime } from "../runtime.ts";
import { refreshReleaseDetail, snapshotSite } from "./snapshot.ts";
// 站点工作台(整理视图):本地快照优先;本地无结果 / 快照过期 / 用户要求时对已开启站点实时补查,
// 并发相同搜索合并,失败短期退避。库存叠加由前端按 mediaEntityId 查 W2-C 端点。
const LIVE_FAIL_BACKOFF_MS = 60_000;
interface LiveState {
inflight: Map<string, Promise<{ error: string | null }>>;
failedUntil: Map<string, { until: number; error: string }>;
}
const liveStates = new WeakMap<SourcesRuntime, LiveState>();
function liveState(rt: SourcesRuntime): LiveState {
let s = liveStates.get(rt);
if (!s) {
s = { inflight: new Map(), failedUntil: new Map() };
liveStates.set(rt, s);
}
return s;
}
/** 合并并发的相同实时搜索;失败后 LIVE_FAIL_BACKOFF_MS 内直接返回上次错误 */
export async function liveSearch(
rt: SourcesRuntime,
site: SiteRow,
q: string,
): Promise<{ error: string | null; coalesced: boolean }> {
const st = liveState(rt);
const key = `${site.id}|${normalizeTitle(q) || q.toLowerCase()}`;
const failed = st.failedUntil.get(key);
if (failed && failed.until > rt.now().getTime()) return { error: `${failed.error}(退避中)`, coalesced: false };
const running = st.inflight.get(key);
if (running) return { ...(await running), coalesced: true };
const p = (async () => {
const r = await snapshotSite(rt, { siteId: site.id, kind: "live_search", query: q, maxPages: 1 });
const error = r.status === "failed" || r.status === "skipped" ? (r.error ?? r.reason ?? "failed") : null;
if (error) st.failedUntil.set(key, { until: rt.now().getTime() + LIVE_FAIL_BACKOFF_MS, error });
else st.failedUntil.delete(key);
return { error };
})().finally(() => st.inflight.delete(key));
st.inflight.set(key, p);
return { ...(await p), coalesced: false };
}
async function lastSnapshotAt(rt: SourcesRuntime, siteId: string): Promise<Date | null> {
const r = await rt.ctx.db
.selectFrom("site_snapshot")
.select(["finished_at"])
.where("site_id", "=", siteId)
.where("kind", "in", ["periodic", "manual", "backfill"])
.where("status", "in", ["ok", "partial"])
.orderBy("started_at", "desc")
.executeTakeFirst();
return r?.finished_at ?? null;
}
interface ReleaseJoinRow {
[k: string]: unknown;
observed_by_feeds: number;
entity_title: string | null;
entity_year: number | null;
entity_kind: string | null;
}
function toWorkbench(row: ReleaseJoinRow): WorkbenchRelease {
const { observed_by_feeds, entity_title, entity_year, entity_kind, ...rest } = row;
const base = toApi<Omit<WorkbenchRelease, "origin" | "dataUpdatedAt" | "observedByFeeds" | "entity">>(rest);
const raw = (base.raw ?? {}) as Record<string, unknown>;
const origin: ReleaseOrigin = raw._site ? "snapshot" : Number(observed_by_feeds) > 0 ? "feed" : "unknown";
return {
...base,
origin,
dataUpdatedAt: base.lastSeenAt,
observedByFeeds: Number(observed_by_feeds),
entity:
base.mediaEntityId && entity_title
? { id: base.mediaEntityId, title: entity_title, year: entity_year, kind: entity_kind ?? "other" }
: null,
};
}
function releaseSelect(rt: SourcesRuntime) {
return rt.ctx.db
.selectFrom("source_release as r")
.leftJoin("media_entity as e", "e.id", "r.media_entity_id")
.selectAll("r")
.select([
"e.title as entity_title",
"e.year as entity_year",
"e.kind as entity_kind",
sql<number>`(select count(distinct o.feed_id) from observation o where o.source_release_id = r.id)::int`.as(
"observed_by_feeds",
),
]);
}
export async function listSiteReleases(
rt: SourcesRuntime,
siteId: string,
q: WorkbenchReleaseQuery,
): Promise<{ items: WorkbenchRelease[]; nextCursor: string | null; coverage: WorkbenchCoverage }> {
const site = await rt.ctx.db.selectFrom("site").selectAll().where("id", "=", siteId).executeTakeFirst();
if (!site) throw new ApiException("not_found", "站点不存在");
const snapAt = await lastSnapshotAt(rt, site.id);
const periodMs = (site.snapshot_interval_sec ?? 3600) * 1000;
const stale = !snapAt || rt.now().getTime() - snapAt.getTime() > periodMs * 2;
const offset = decodeCursor(q.cursor);
const run = async () => {
let query = releaseSelect(rt).where("r.site_id", "=", site.id);
if (q.q) {
const like = `%${q.q.replace(/[\\%_]/g, (m) => `\\${m}`)}%`;
query = query.where((eb) =>
eb.or([
eb("r.title", "ilike", like),
eb("r.subtitle", "ilike", like),
eb("r.site_release_id", "=", q.q!),
sql<boolean>`similarity(r.title, ${q.q}) >= 0.3`,
]),
);
}
if (q.category) query = query.where("r.category_raw", "=", q.category);
if (q.promotion) query = query.where(sql`r.promotion->>'kind'`, "=", q.promotion);
if (q.resolution) query = query.where("r.resolution_status", "=", q.resolution);
return (await query
.orderBy(sql`coalesce(r.source_published_at, r.first_seen_at)`, "desc")
.orderBy("r.id")
.offset(offset)
.limit(q.limit + 1)
.execute()) as unknown as ReleaseJoinRow[];
};
let rows = await run();
let live = false;
let liveError: string | null = null;
let coalesced = false;
const wantLive = q.live === "true" || (q.live !== "false" && offset === 0 && !!q.q && (rows.length === 0 || stale));
if (wantLive && q.q) {
if (!site.enabled) liveError = "站点已关闭,只展示历史数据";
else {
const r = await liveSearch(rt, site, q.q);
live = r.error === null;
liveError = r.error;
coalesced = r.coalesced;
if (live) rows = await run();
}
}
const pg = page(rows, offset, q.limit);
return {
items: pg.items.map(toWorkbench),
nextCursor: pg.nextCursor,
coverage: {
siteEnabled: site.enabled,
lastSnapshotAt: snapAt?.toISOString() ?? null,
stale,
fromSnapshot: rows.length > 0,
live,
liveError,
coalesced,
},
};
}
export async function releaseDetail(
rt: SourcesRuntime,
siteId: string,
releaseId: string,
refresh: boolean,
): Promise<WorkbenchReleaseDetail> {
const find = async () =>
(await releaseSelect(rt)
.where("r.site_id", "=", siteId)
.where("r.id", "=", releaseId)
.executeTakeFirst()) as unknown as ReleaseJoinRow | undefined;
if (!(await find())) throw new ApiException("not_found", "资源不存在");
const refreshError = refresh ? await refreshReleaseDetail(rt, releaseId) : null;
const row = (await find())!;
const release = toWorkbench(row);
const obs = await rt.ctx.db
.selectFrom("observation as o")
.innerJoin("feed_source as f", "f.id", "o.feed_id")
.select(["o.id", "o.feed_id", "f.name", "o.first_seen_at", "o.last_seen_at"])
.where("o.source_release_id", "=", releaseId)
.orderBy("o.first_seen_at")
.execute();
const promo = (release.promotion ?? {}) as { observedAt?: string };
return {
release,
observations: obs.map((o) => ({
observationId: o.id,
feedId: o.feed_id,
feedName: o.name,
firstSeenAt: o.first_seen_at.toISOString(),
lastSeenAt: o.last_seen_at.toISOString(),
})),
volatile: {
promotionObservedAt: promo.observedAt ?? null,
promotionExpiresAt: release.promotionExpiresAt,
peersObservedAt: release.lastSeenAt,
detailFetchedAt: release.detailFetchedAt,
recheckBeforeDownload: true,
},
refreshError,
};
}
@@ -0,0 +1,92 @@
import { defineTask, type OutboxRouteFn, type TaskRegistry } from "@umt/tasks";
import { z } from "zod";
import type { SourcesRuntime } from "./runtime.ts";
import { ingestFeed } from "./service/feeds.ts";
import { refreshMetadata } from "./service/metadata.ts";
import { resolveRelease } from "./service/resolve.ts";
import { snapshotSite } from "./service/snapshot.ts";
// 持久任务:输入只放业务 ID,处理器读业务表。外部请求失败的退避/熔断由适配器状态与业务表承担,
// 凭据失效返回正常结果(不重试);只有意外错误(DB、TMDB 网络)抛出交给引擎重试。
const NETWORK_RETRY = {
maximumAttempts: 5,
initialInterval: "1s",
backoffCoefficient: 2,
maximumInterval: "60s",
} as const;
const id = z.string().uuid();
/** Feed 疑似缺口时的补漏页数预算 */
export const BACKFILL_MAX_PAGES = 3;
export function registerSourcesTasks(registry: TaskRegistry, rt: () => SourcesRuntime): void {
registry.register(
defineTask({
name: "ingestFeed",
version: 1,
inputSchema: z.object({ feedSourceId: id, manual: z.boolean().optional() }),
retryPolicy: NETWORK_RETRY,
handler: (t) => t.step.run("ingest", () => ingestFeed(rt(), t.input.feedSourceId, { manual: !!t.input.manual })),
}),
defineTask({
name: "resolveObservation",
version: 1,
inputSchema: z
.object({ observationId: id.nullish(), sourceReleaseId: id.nullish(), force: z.boolean().optional() })
.refine((v) => !!v.observationId || !!v.sourceReleaseId, { message: "需要 observationId 或 sourceReleaseId" }),
retryPolicy: NETWORK_RETRY,
handler: (t) =>
t.step.run("resolve", () =>
resolveRelease(rt(), {
observationId: t.input.observationId ?? undefined,
sourceReleaseId: t.input.sourceReleaseId ?? undefined,
force: t.input.force,
}),
),
}),
defineTask({
name: "refreshMetadata",
version: 1,
inputSchema: z.object({ mediaEntityId: id }),
retryPolicy: NETWORK_RETRY,
handler: (t) => t.step.run("refresh", () => refreshMetadata(rt(), t.input.mediaEntityId)),
}),
defineTask({
name: "snapshotSite",
version: 1,
inputSchema: z.object({
siteId: id,
kind: z.enum(["periodic", "backfill", "manual"]).default("periodic"),
maxPages: z.number().int().min(1).max(20).optional(),
}),
retryPolicy: NETWORK_RETRY,
handler: (t) =>
t.step.run("snapshot", () =>
snapshotSite(rt(), { siteId: t.input.siteId, kind: t.input.kind, maxPages: t.input.maxPages }),
),
}),
);
}
type P = Record<string, unknown>;
export const sourcesOutboxRoutes: Record<string, OutboxRouteFn> = {
"feed.due": (ev) => {
const p = ev.payload as P;
return { task: "ingestFeed", input: { feedSourceId: p.feedId, manual: p.reason === "manual" } };
},
"release.observed": (ev) => {
const p = ev.payload as P;
return {
task: "resolveObservation",
input: { sourceReleaseId: p.sourceReleaseId, observationId: p.observationId },
};
},
"feed.gap_suspected": (ev) => {
const p = ev.payload as P;
if (!p.siteId) return null;
return { task: "snapshotSite", input: { siteId: p.siteId, kind: "backfill", maxPages: BACKFILL_MAX_PAGES } };
},
};
+4 -3
View File
@@ -31,7 +31,7 @@ const pingModule: ServerModule = {
registerTasks(registry) {
registry.register(
defineTask({
name: "ingestFeed",
name: "testPing",
version: 1,
inputSchema: z.object({ feedSourceId: z.string() }),
handler: async () => "real",
@@ -75,8 +75,8 @@ describe("ServerModule 骨架", () => {
it("任务注册表:模块任务优先,其余 §3.1-8 任务由空壳兜底;重复注册报错", async () => {
const reg = buildTaskRegistry([...SERVER_MODULES, pingModule], fakeCtx);
expect(reg.names()).toEqual([...TASK_NAMES].sort());
const ingest = reg.latest("ingestFeed");
expect(reg.names()).toEqual(expect.arrayContaining([...TASK_NAMES, "testPing"]));
const ingest = reg.latest("testPing");
expect(await ingest.handler({ input: {} } as never)).toBe("real");
expect(() => buildTaskRegistry([pingModule, { ...pingModule, name: "dup" }], fakeCtx)).toThrow(/重复注册/);
});
@@ -85,6 +85,7 @@ describe("ServerModule 骨架", () => {
const merged = Object.keys(collectOutboxRoutes([...SERVER_MODULES, pingModule], fakeCtx));
expect(merged).toContain("test.ping");
expect(merged).toContain("release.resolved");
expect(merged).toEqual(expect.arrayContaining(["feed.due", "release.observed", "feed.gap_suspected"]));
expect(() => collectOutboxRoutes([pingModule, { ...pingModule, name: "dup" }], fakeCtx)).toThrow(/重复认领/);
});
@@ -0,0 +1,238 @@
import { createMockNexusPhpSite, type MockNexusPhpSite } from "@umt/testkit";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { snapshotSite } from "../../src/modules/sources-identity/service/snapshot.ts";
import { liveSearch } from "../../src/modules/sources-identity/service/workbench.ts";
import { createHarness, EXEC_KEY, type Harness, json, pgAvailable, READ_KEY } from "./harness.ts";
const up = await pgAvailable();
describe.skipIf(!up)("来源管理与站点工作台 API", () => {
let h: Harness;
let mock: MockNexusPhpSite;
let siteId: string;
beforeAll(async () => {
mock = await createMockNexusPhpSite();
h = await createHarness("api");
});
afterAll(async () => {
await h?.close();
await mock?.close();
});
it("鉴权:无 key 401,只读 key 写站点 403,organize.execute 不能改站点配置", async () => {
expect((await h.app.request("/api/v1/sites")).status).toBe(401);
expect((await h.req("/sites", { key: READ_KEY })).status).toBe(200);
const body = { key: "audiences", name: "观众", adapter: "nexusphp", baseUrl: `${mock.baseUrl}/` };
expect((await h.req("/sites", { key: READ_KEY, body })).status).toBe(403);
expect((await h.req("/sites", { key: EXEC_KEY, body })).status).toBe(403);
});
it("站点注册表 + 登记站点;适配器类型与定义不符 400;重复 409;校验失败 422", async () => {
const defs = await json(await h.req("/site-definitions"));
expect(defs.status).toBe(200);
const aud = defs.body.data.items.find((d: { key: string }) => d.key === "audiences");
expect(aud).toMatchObject({ adapter: "nexusphp", siteId: null, shippedEnabled: true });
expect(defs.body.data.items.find((d: { key: string }) => d.key === "mteam")?.adapter).toBe("mteam_api");
const bad = await json(
await h.req("/sites", {
body: { key: "audiences", name: "观众", adapter: "mteam_api", baseUrl: `${mock.baseUrl}/` },
}),
);
expect(bad.status).toBe(400);
const unknown = await json(
await h.req("/sites", { body: { key: "nosuch", name: "x", adapter: "nexusphp", baseUrl: "https://x.example/" } }),
);
expect(unknown.status).toBe(400);
expect(
(await h.req("/sites", { body: { key: "BAD KEY", name: "x", adapter: "nexusphp", baseUrl: "nope" } })).status,
).toBe(422);
const ok = await json(
await h.req("/sites", {
body: {
key: "audiences",
name: "观众",
adapter: "nexusphp",
baseUrl: `${mock.baseUrl}/`,
rateLimit: { minIntervalMs: 0, concurrency: 4 },
snapshotEnabled: true,
snapshotIntervalSec: 3600,
},
}),
);
expect(ok.status).toBe(200);
siteId = ok.body.data.id;
expect(
(
await h.req("/sites", {
body: { key: "audiences", name: "观众", adapter: "nexusphp", baseUrl: `${mock.baseUrl}/` },
})
).status,
).toBe(409);
const defs2 = await json(await h.req("/site-definitions"));
expect(defs2.body.data.items.find((d: { key: string }) => d.key === "audiences").siteId).toBe(siteId);
});
it("站点账号:凭证只写(存 secret),响应里只有 secret_ref", async () => {
const r = await json(
await h.req("/site-accounts", { body: { siteId, label: "主号", authKind: "cookie", credential: mock.cookie } }),
);
expect(r.status).toBe(200);
expect(JSON.stringify(r.body)).not.toContain(mock.cookie);
expect(h.secrets.plain.get(r.body.data.secretRef)).toBe(mock.cookie);
expect(
(await h.req("/site-accounts", { body: { siteId, label: "主号", authKind: "cookie", credential: "x" } })).status,
).toBe(409);
});
it("Feed CRUD:URL 存 secret,对外只给脱敏 URL;启停;立即抓取入队", async () => {
const url = "https://audiences.example/torrentrss.php?rows=50&passkey=abcdef0123456789";
const c = await json(
await h.req("/feed-sources", { body: { siteId, name: "电影", purpose: "movie", url, pollIntervalSec: 600 } }),
);
expect(c.status).toBe(200);
expect(c.body.data.urlDisplay).toContain("passkey=%5BREDACTED%5D".slice(0, 8));
expect(JSON.stringify(c.body)).not.toContain("abcdef0123456789");
const id = c.body.data.id;
expect((await h.req("/feed-sources", { body: { siteId, name: "x", url: "not a url" } })).status).toBe(422);
const off = await json(await h.req(`/feed-sources/${id}/disable`, { body: {} }));
expect(off.body.data).toMatchObject({ enabled: false, healthStatus: "disabled" });
expect((await h.req(`/feed-sources/${id}/fetch`, { body: {} })).status).toBe(412);
await h.req(`/feed-sources/${id}/enable`, { body: {} });
const f = await json(await h.req(`/feed-sources/${id}/fetch`, { body: {} }));
expect(f.status).toBe(200);
expect(h.engine.enqueued.at(-1)).toMatchObject({ task: "ingestFeed", input: { feedSourceId: id, manual: true } });
const patched = await json(
await h.req(`/feed-sources/${id}`, {
method: "PATCH",
body: { url: "https://audiences.example/rss?passkey=NEWPASSKEY99" },
}),
);
expect(JSON.stringify(patched.body)).not.toContain("NEWPASSKEY99");
expect(h.secrets.plain.get(patched.body.data.urlSecretRef)).toContain("NEWPASSKEY99");
expect((await h.req("/feed-sources/00000000-0000-4000-8000-000000000000")).status).toBe(404);
});
it("快照:列表页 upsert 到 source_release,记录覆盖范围;站点级限流器每站一个实例", async () => {
const r = await snapshotSite(h.rt, { siteId, kind: "periodic" });
expect(r.status).toBe("ok");
expect(r.pages).toEqual([1]);
expect(r.newCount).toBe(r.itemCount);
const limiter = h.rt.limiterOf(siteId);
expect(limiter).not.toBeNull();
await snapshotSite(h.rt, { siteId, kind: "manual" });
expect(h.rt.limiterOf(siteId)).toBe(limiter);
expect(limiter!.stats.granted).toBeGreaterThanOrEqual(2);
const snaps = await json(await h.req(`/sites/${siteId}/snapshots`));
expect(snaps.body.data.items[0]).toMatchObject({ kind: "manual", status: "ok", pages: [1] });
const observed = await h.outbox("release.observed");
expect(observed.length).toBe(r.newCount);
const site = await json(await h.req(`/sites/${siteId}`));
expect(site.body.data.lastSnapshotAt).not.toBeNull();
});
it("工作台列表:快照优先,标注来源与更新时间;筛选与分页", async () => {
const l = await json(await h.req(`/sites/${siteId}/releases?limit=2`));
expect(l.status).toBe(200);
expect(l.body.data.items).toHaveLength(2);
expect(l.body.data.nextCursor).not.toBeNull();
expect(l.body.data.coverage).toMatchObject({ siteEnabled: true, stale: false, live: false });
expect(l.body.data.items[0]).toMatchObject({ origin: "snapshot", entity: null });
expect(l.body.data.items[0].dataUpdatedAt).toBeTruthy();
const p2 = await json(await h.req(`/sites/${siteId}/releases?limit=2&cursor=${l.body.data.nextCursor}`));
expect(p2.body.data.items[0].id).not.toBe(l.body.data.items[0].id);
const free = await json(await h.req(`/sites/${siteId}/releases?promotion=free`));
expect(free.body.data.items.every((i: { promotion: { kind: string } }) => i.promotion.kind === "free")).toBe(true);
const q = await json(await h.req(`/sites/${siteId}/releases?q=Dune&live=false`));
expect(q.body.data.items.map((i: { siteReleaseId: string }) => i.siteReleaseId)).toContain("812345");
});
it("实时补查:并发相同搜索合并为一次站点请求;API 标注实时来源", async () => {
const site = await h.db.selectFrom("site").selectAll().where("id", "=", siteId).executeTakeFirstOrThrow();
const searches = () => mock.requests.filter((r) => r.path.includes("search=")).length;
const before = searches();
const [a, b] = await Promise.all([liveSearch(h.rt, site, "Bear"), liveSearch(h.rt, site, "bear")]);
expect(searches() - before).toBe(1);
expect([a.coalesced, b.coalesced].sort()).toEqual([false, true]);
expect(a.error).toBeNull();
const r = await json(await h.req(`/sites/${siteId}/releases?q=Last&live=true`));
expect(r.body.data.coverage).toMatchObject({ live: true, liveError: null });
expect(r.body.data.items.map((i: { siteReleaseId: string }) => i.siteReleaseId)).toContain("812348");
const snaps = await h.db.selectFrom("site_snapshot").selectAll().where("kind", "=", "live_search").execute();
expect(snaps).toHaveLength(2);
});
it("详情:易变字段复核时间;refresh 失败保留既有数据", async () => {
const rel = await h.db
.selectFrom("source_release")
.selectAll()
.where("site_release_id", "=", "812345")
.executeTakeFirstOrThrow();
const d = await json(await h.req(`/sites/${siteId}/releases/${rel.id}?refresh=true`));
expect(d.status).toBe(200);
expect(d.body.data.refreshError).toBeNull();
expect(d.body.data.volatile.recheckBeforeDownload).toBe(true);
expect(d.body.data.volatile.detailFetchedAt).not.toBeNull();
mock.setLoggedIn(false);
const d2 = await json(await h.req(`/sites/${siteId}/releases/${rel.id}?refresh=true`));
expect(d2.status).toBe(200);
expect(d2.body.data.refreshError).toMatch(/auth/);
expect(d2.body.data.release.title).toBe(rel.title);
// 登录失效 → 账号标失效 + 待处理,周期快照不再访问
const acc = await h.db
.selectFrom("site_account")
.selectAll()
.where("site_id", "=", siteId)
.executeTakeFirstOrThrow();
expect(["invalid", "expired"]).toContain(acc.status);
expect(
(await h.db.selectFrom("decision").selectAll().where("dedupe_key", "=", `site-auth:${siteId}`).execute()).length,
).toBe(1);
const n = mock.requests.length;
expect((await snapshotSite(h.rt, { siteId, kind: "periodic" })).reason).toBe("auth_invalid");
expect(mock.requests.length).toBe(n);
// 换凭证:账号复位、待处理关闭
mock.setLoggedIn(true);
await h.req(`/site-accounts/${acc.id}`, { method: "PATCH", body: { credential: mock.cookie } });
const d3 = await h.db
.selectFrom("decision")
.select(["status"])
.where("dedupe_key", "=", `site-auth:${siteId}`)
.executeTakeFirstOrThrow();
expect(d3.status).toBe("resolved");
expect((await snapshotSite(h.rt, { siteId, kind: "periodic" })).status).toBe("ok");
});
it("关闭站点:停止网络访问,历史资源仍可查并标注 site_disabled", async () => {
const off = await json(await h.req(`/sites/${siteId}/disable`, { body: {} }));
expect(off.body.data).toMatchObject({ enabled: false, healthStatus: "disabled" });
const n = mock.requests.length;
const l = await json(await h.req(`/sites/${siteId}/releases?q=Nothing.Here&live=true`));
expect(l.body.data.coverage).toMatchObject({ siteEnabled: false, live: false });
expect(l.body.data.coverage.liveError).toMatch(/关闭/);
expect(mock.requests.length).toBe(n);
const all = await json(await h.req(`/sites/${siteId}/releases`));
expect(all.body.data.items.length).toBeGreaterThan(0);
expect(all.body.data.items.every((i: { availability: string }) => i.availability === "site_disabled")).toBe(true);
expect((await h.req(`/sites/${siteId}/snapshot`, { body: {} })).status).toBe(412);
await h.req(`/sites/${siteId}/enable`, { body: {} });
const back = await json(await h.req(`/sites/${siteId}/releases?limit=1`));
expect(back.body.data.items[0].availability).toBe("active");
});
it("来源资源 / 观察 / 作品列表与 OpenAPI 文档", async () => {
const l = await json(await h.req(`/source-releases?siteId=${siteId}&q=Dune`));
expect(l.body.data.items.length).toBeGreaterThan(0);
expect((await h.req("/source-releases/not-a-uuid")).status).toBe(422);
expect((await json(await h.req("/observations"))).status).toBe(200);
expect((await json(await h.req("/media-entities"))).status).toBe(200);
expect((await json(await h.req("/identity-pool"))).body.data.items).toEqual([]);
const doc = await json(await h.app.request("/api/openapi.json"));
const paths = Object.keys((doc.body as unknown as { paths: Record<string, unknown> }).paths);
expect(paths).toEqual(
expect.arrayContaining(["/api/v1/sites/{id}/releases", "/api/v1/identity-pool", "/api/v1/decisions/summary"]),
);
});
});
@@ -0,0 +1,278 @@
import { randomUUID } from "node:crypto";
import { createTmdbClient } from "@umt/adapters";
import { DEFAULT_SETTINGS, type GlobalSettings, type SettingsKey } from "@umt/contracts";
import { type DB, type Kysely, migrateToLatest, sql } from "@umt/db";
import type { TaskEngine } from "@umt/tasks";
import { Kysely as K, PostgresDialect } from "kysely";
import pg from "pg";
import { createOutboxPort } from "../../src/context.ts";
import { createApp } from "../../src/http/app.ts";
import { EventBus } from "../../src/http/sse.ts";
import { createSourcesIdentityModule } from "../../src/modules/sources-identity/index.ts";
import type { RuntimeOverrides } from "../../src/modules/sources-identity/runtime.ts";
import type { SecretStore, ServerContext, SettingsReader } from "../../src/modules/types.ts";
import { DEV_KEY, memoryLogger, memoryStore } from "../helpers.ts";
export const READ_KEY = "w2b-read-key";
export const EXEC_KEY = "w2b-exec-key";
// W2-B 集成测试夹具:独立 PG schema + 真实迁移 + fake 引擎/凭证 + 模块运行时。
export const PG_URL = process.env.DATABASE_URL ?? "postgres://umt:umt@localhost:55433/umt";
export async function pgAvailable(): Promise<boolean> {
const c = new pg.Client({ connectionString: PG_URL, connectionTimeoutMillis: 2000 });
try {
await c.connect();
await c.end();
return true;
} catch {
return false;
}
}
/** 内存凭证库(W2-A 的 SecretStore 实现前);记录明文以便断言"明文不入业务表" */
export function fakeSecrets(db: () => Kysely<DB>): SecretStore & { plain: Map<string, string> } {
const plain = new Map<string, string>();
return {
plain,
async resolve(ref) {
const v = plain.get(ref);
if (v === undefined) throw new Error(`unknown secret ${ref}`);
return v;
},
async create({ plaintext, purpose }) {
const id = randomUUID();
plain.set(id, plaintext);
// 业务表 secret_ref 有外键:落一行占位(密文不是真的)
await db()
.insertInto("secret")
.values({ id, name: `test-${id}`, purpose, ciphertext: Buffer.from("x"), nonce: Buffer.from("x") })
.execute();
return id;
},
async replace(ref, plaintext) {
plain.set(ref, plaintext);
},
async remove(ref) {
plain.delete(ref);
},
};
}
/** 每次读库的设置读取(不缓存,测试里改 settings 表立即生效) */
export function plainSettings(db: () => Kysely<DB>): SettingsReader {
const all = async (): Promise<GlobalSettings> => {
const rows = await db().selectFrom("settings").select(["key", "value"]).execute();
const out = { ...DEFAULT_SETTINGS } as Record<string, unknown>;
for (const r of rows) {
const def = (DEFAULT_SETTINGS as Record<string, object>)[r.key];
if (def) out[r.key] = { ...def, ...(r.value as object) };
}
return out as GlobalSettings;
};
return { all, get: async <K extends SettingsKey>(k: K) => (await all())[k] } as SettingsReader;
}
export interface Enqueued {
task: string;
input: unknown;
key: string;
}
export function fakeEngine(): TaskEngine & { enqueued: Enqueued[] } {
const enqueued: Enqueued[] = [];
const keys = new Set<string>();
return {
enqueued,
async enqueue(task, input, opts) {
const dedup = keys.has(opts.idempotencyKey);
if (!dedup) {
keys.add(opts.idempotencyKey);
enqueued.push({ task, input, key: opts.idempotencyKey });
}
return { runId: randomUUID(), deduplicated: dedup };
},
async signal() {
return { delivered: 0 };
},
async cancel() {
return false;
},
async get() {
return null;
},
async list() {
return [];
},
async startWorker() {},
async stop() {},
};
}
export interface Harness {
db: Kysely<DB>;
ctx: ServerContext;
engine: ReturnType<typeof fakeEngine>;
secrets: ReturnType<typeof fakeSecrets>;
module: ReturnType<typeof createSourcesIdentityModule>;
rt: ReturnType<ReturnType<typeof createSourcesIdentityModule>["runtime"]>;
app: ReturnType<typeof createApp>;
/** 带 dev admin key 的请求 */
req(path: string, init?: { method?: string; body?: unknown; key?: string }): Promise<Response>;
outbox(type?: string): Promise<{ type: string; payload: Record<string, unknown>; idempotency_key: string | null }[]>;
close(): Promise<void>;
}
export interface HarnessOptions extends RuntimeOverrides {
tmdbUrl?: string;
}
export async function createHarness(name: string, opts: HarnessOptions = {}): Promise<Harness> {
const schema = `w2b_${name}_${process.pid.toString(36)}${Math.random().toString(36).slice(2, 7)}`;
const admin = new K<unknown>({
dialect: new PostgresDialect({ pool: new pg.Pool({ connectionString: PG_URL, max: 1 }) }),
});
await sql.raw(`create schema ${schema}`).execute(admin);
// 与 @umt/db createDb 相同的 int8 解析(createDb 导入即注册全局解析器)
await import("@umt/db");
const db = new K<DB>({
dialect: new PostgresDialect({
pool: new pg.Pool({ connectionString: PG_URL, max: 8, options: `-c search_path=${schema},public` }),
}),
}) as unknown as Kysely<DB>;
await migrateToLatest(db, { migrationTableSchema: schema });
const { logger } = memoryLogger();
const engine = fakeEngine();
const secrets = fakeSecrets(() => db);
const outboxPort = createOutboxPort();
const ctx = {
role: "all",
db,
config: { NODE_ENV: "development", DEV_ADMIN_KEY: DEV_KEY } as never,
logger,
events: new EventBus(5),
engine,
outbox: outboxPort,
secrets,
settings: plainSettings(() => db),
capabilities: {} as never,
} as ServerContext;
const { tmdbUrl, ...overrides } = opts;
const module = createSourcesIdentityModule({
...overrides,
tmdb:
overrides.tmdb ??
(() => (tmdbUrl ? createTmdbClient({ apiKey: "test-tmdb-key", baseUrl: `${tmdbUrl}/3`, retries: 0 }) : null)),
});
const rt = module.runtime(ctx);
const app = createApp({
config: { NODE_ENV: "development", DEV_ADMIN_KEY: DEV_KEY },
logger,
bus: new EventBus(5),
role: "all",
version: "0.0.0-test",
modules: [module],
ctx,
apiKeyStore: memoryStore({
[READ_KEY]: { id: "reader", scopes: ["read"] },
[EXEC_KEY]: { id: "operator", scopes: ["organize.execute"] },
}),
});
return {
db,
ctx,
engine,
secrets,
module,
rt,
app,
req(path, init = {}) {
const headers: Record<string, string> = { authorization: `Bearer ${init.key ?? DEV_KEY}` };
if (init.body !== undefined) headers["content-type"] = "application/json";
return Promise.resolve(
app.request(`/api/v1${path}`, {
method: init.method ?? (init.body !== undefined ? "POST" : "GET"),
headers,
...(init.body !== undefined ? { body: JSON.stringify(init.body) } : {}),
}),
);
},
async outbox(type) {
let q = db.selectFrom("outbox_event").select(["type", "payload", "idempotency_key"]).orderBy("at").orderBy("id");
if (type) q = q.where("type", "=", type);
return (await q.execute()) as never;
},
async close() {
await db.destroy();
await sql.raw(`drop schema if exists ${schema} cascade`).execute(admin);
await admin.destroy();
},
};
}
/** 登记一个指向 mock 的站点 + 可选账号 */
export async function seedSite(
h: Harness,
opts: { key?: string; baseUrl: string; cookie?: string; adapter?: "nexusphp" | "mteam_api" | "generic_rss" },
) {
const site = await h.db
.insertInto("site")
.values({
key: opts.key ?? "audiences",
name: opts.key ?? "观众",
adapter: opts.adapter ?? "nexusphp",
base_url: opts.baseUrl,
rate_limit: JSON.stringify({ minIntervalMs: 0, concurrency: 5 }),
snapshot_enabled: true,
snapshot_interval_sec: 3600,
})
.returningAll()
.executeTakeFirstOrThrow();
let accountId: string | null = null;
if (opts.cookie) {
const ref = await h.secrets.create({ purpose: "site_cookie", plaintext: opts.cookie });
const acc = await h.db
.insertInto("site_account")
.values({ site_id: site.id, label: "main", auth_kind: "cookie", secret_ref: ref })
.returning(["id"])
.executeTakeFirstOrThrow();
accountId = acc.id;
}
return { site, accountId };
}
/** 登记 Feed(URL 存 fake secret;feed_source.url_secret_ref 外键指向 secret 表,这里补一行占位) */
export async function seedFeed(
h: Harness,
opts: {
siteId: string | null;
url: string;
name?: string;
accountId?: string | null;
purpose?: "movie" | "promotion" | "generic";
},
) {
const ref = await h.secrets.create({ purpose: "feed_url", plaintext: opts.url });
return h.db
.insertInto("feed_source")
.values({
site_id: opts.siteId,
site_account_id: opts.accountId ?? null,
name: opts.name ?? "feed",
purpose: opts.purpose ?? "generic",
url_secret_ref: ref,
url_display: opts.url.replace(/passkey=[^&]+/, "passkey=[REDACTED]"),
poll_interval_sec: 300,
})
.returningAll()
.executeTakeFirstOrThrow();
}
// biome-ignore lint/suspicious/noExplicitAny: 测试断言用
type AnyJson = any;
export const json = async (r: Response) => ({
status: r.status,
body: (await r.json()) as { ok: boolean; data: AnyJson; error?: AnyJson },
});
@@ -0,0 +1,213 @@
import { createMockRssServer, type MockFeedItem, type MockRssServer } from "@umt/testkit";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { schedulerTick } from "../../src/modules/sources-identity/scheduler.ts";
import { extractSiteReleaseId, ingestFeed } from "../../src/modules/sources-identity/service/feeds.ts";
import { sourcesOutboxRoutes } from "../../src/modules/sources-identity/tasks.ts";
import { createHarness, type Harness, pgAvailable, seedFeed, seedSite } from "./harness.ts";
const up = await pgAvailable();
const BASE = "https://audiences.example";
const item = (guid: string, id: string, title: string, extra: Partial<MockFeedItem> = {}): MockFeedItem => ({
guid,
title,
link: `${BASE}/details.php?id=${id}&hit=1`,
enclosureUrl: `${BASE}/download.php?id=${id}&passkey=SECRETPK123`,
size: 1024 * 1024 * 1024,
pubDate: new Date("2026-09-20T08:00:00Z"),
categories: ["电影"],
...extra,
});
describe.skipIf(!up)("ingestFeed", () => {
let h: Harness;
const servers: MockRssServer[] = [];
let siteId: string;
let accountId: string | null;
beforeAll(async () => {
h = await createHarness("ingest");
const s = await seedSite(h, { baseUrl: `${BASE}/`, cookie: "c=1" });
siteId = s.site.id;
accountId = s.accountId;
});
afterAll(async () => {
for (const s of servers) await s.close();
await h?.close();
});
const feedServer = async (items: MockFeedItem[], passkey = "FEEDPK") => {
const s = await createMockRssServer({ items, passkey });
servers.push(s);
return s;
};
it("从链接推出站点资源 ID", () => {
expect(extractSiteReleaseId({ guid: "x", link: `${BASE}/details.php?id=812345&hit=1` })).toBe("812345");
expect(extractSiteReleaseId({ guid: "x", link: "https://kp.m-team.cc/detail/912233" })).toBe("912233");
expect(extractSiteReleaseId({ guid: "abc", link: null })).toBeNull();
expect(extractSiteReleaseId({ guid: "12345", link: null })).toBe("12345");
});
it("三个 Feed 看到同一资源只建一条 source_release,保留三条观察;release.observed 只发一次", async () => {
const title = "Dune.Part.Two.2024.2160p.UHD.BluRay.REMUX-FraMeSToR";
const a = await feedServer([item("movie-1", "812345", title)]);
const b = await feedServer([item("promo-xyz", "812345", title)]);
const c = await feedServer([item("https://audiences.example/details.php?id=812345", "812345", title)]);
const feeds = [
await seedFeed(h, { siteId, url: a.feedUrl, name: "电影", purpose: "movie", accountId }),
await seedFeed(h, { siteId, url: b.feedUrl, name: "促销", purpose: "promotion" }),
await seedFeed(h, { siteId, url: c.feedUrl, name: "综合" }),
];
for (const f of feeds) expect((await ingestFeed(h.rt, f.id)).status).toBe("ok");
const rels = await h.db.selectFrom("source_release").selectAll().where("site_release_id", "=", "812345").execute();
expect(rels).toHaveLength(1);
const obs = await h.db.selectFrom("observation").selectAll().where("source_release_id", "=", rels[0]!.id).execute();
expect(new Set(obs.map((o) => o.feed_id)).size).toBe(3);
expect(obs.every((o) => o.status === "linked")).toBe(true);
const observed = (await h.outbox("release.observed")).filter((e) => e.payload.sourceReleaseId === rels[0]!.id);
expect(observed).toHaveLength(1);
expect(observed[0]!.payload.isNew).toBe(true);
// 时间:来源发布时间与首次发现分开保存
expect(rels[0]!.source_published_at?.toISOString()).toBe("2026-09-20T08:00:00.000Z");
expect(rels[0]!.first_seen_at.getTime()).toBeGreaterThan(rels[0]!.source_published_at!.getTime());
// feed 状态持久化:游标与条件请求头
const f0 = await h.db
.selectFrom("feed_source")
.selectAll()
.where("id", "=", feeds[0]!.id)
.executeTakeFirstOrThrow();
expect(f0.health_status).toBe("healthy");
expect(f0.etag).toBeTruthy();
expect((f0.cursor as { window: { lastGuidSet: string[] } }).window.lastGuidSet).toContain("movie-1");
expect(f0.next_fetch_at!.getTime()).toBeGreaterThan(Date.now());
});
it("enclosure/passkey 只存脱敏值;未变化时 304 不产生新条目", async () => {
const s = await feedServer([item("g-1", "900001", "Some.Movie.2020.1080p.WEB-DL-GRP")], "TOPSECRETKEY");
const f = await seedFeed(h, { siteId, url: s.feedUrl, name: "脱敏" });
await ingestFeed(h.rt, f.id);
const dump =
JSON.stringify(await h.db.selectFrom("observation").selectAll().where("feed_id", "=", f.id).execute()) +
JSON.stringify(
await h.db.selectFrom("source_release").selectAll().where("site_release_id", "=", "900001").execute(),
) +
JSON.stringify(await h.db.selectFrom("feed_source").selectAll().where("id", "=", f.id).execute());
expect(dump).not.toContain("SECRETPK123");
expect(dump).not.toContain("TOPSECRETKEY");
expect(dump).toContain("[REDACTED]");
const again = await ingestFeed(h.rt, f.id);
expect(again.status).toBe("not_modified");
expect(s.notModifiedCount).toBe(1);
});
it("同 GUID 内容变化视为更新:观察与资源标题更新,重新触发识别", async () => {
const s = await feedServer([item("upd-1", "900100", "Show.2021.1080p.WEB-DL-AAA")]);
const f = await seedFeed(h, { siteId, url: s.feedUrl, name: "更新" });
await ingestFeed(h.rt, f.id);
s.updateItem("upd-1", { title: "Show.2021.2160p.WEB-DL-AAA" });
const r = await ingestFeed(h.rt, f.id);
expect(r).toMatchObject({ status: "ok", newItems: 0, updatedItems: 1, newReleases: 0 });
const obs = await h.db.selectFrom("observation").selectAll().where("feed_id", "=", f.id).execute();
expect(obs).toHaveLength(1);
expect(obs[0]!.title).toBe("Show.2021.2160p.WEB-DL-AAA");
const rel = await h.db
.selectFrom("source_release")
.selectAll()
.where("site_release_id", "=", "900100")
.executeTakeFirstOrThrow();
expect(rel.title).toBe("Show.2021.2160p.WEB-DL-AAA");
const ev = (await h.outbox("release.observed")).filter((e) => e.payload.sourceReleaseId === rel.id);
expect(ev.map((e) => e.payload.isNew)).toEqual([true, false]);
});
it("窗口零重叠 → feed.gap_suspected,并路由到有预算的补漏快照", async () => {
const s = await feedServer([item("gap-1", "900201", "A.2020.1080p-X"), item("gap-2", "900202", "B.2020.1080p-X")]);
const f = await seedFeed(h, { siteId, url: s.feedUrl, name: "缺口" });
await ingestFeed(h.rt, f.id);
s.setItems([item("gap-9", "900209", "C.2020.1080p-X")]);
const r = await ingestFeed(h.rt, f.id);
expect(r.possibleGap).toBe(true);
const gap = (await h.outbox("feed.gap_suspected")).find((e) => e.payload.feedId === f.id);
expect(gap).toBeDefined();
const route = sourcesOutboxRoutes["feed.gap_suspected"]!({
id: "1",
type: "feed.gap_suspected",
payload: gap!.payload,
at: new Date(),
});
expect(route).toMatchObject({ task: "snapshotSite", input: { siteId, kind: "backfill", maxPages: 3 } });
});
it("登录失效(403):账号标失效 + 待处理,不重试", async () => {
const s = await feedServer([item("auth-1", "900300", "X.2020.1080p-Y")], "RIGHTKEY");
const badUrl = s.feedUrl.replace("RIGHTKEY", "WRONGKEY");
const f = await seedFeed(h, { siteId, url: badUrl, name: "失效", accountId });
const r = await ingestFeed(h.rt, f.id);
expect(r.status).toBe("auth_failed");
const acc = await h.db
.selectFrom("site_account")
.selectAll()
.where("id", "=", accountId!)
.executeTakeFirstOrThrow();
expect(acc.status).toBe("invalid");
const d = await h.db
.selectFrom("decision")
.selectAll()
.where("dedupe_key", "=", `feed-auth:${f.id}`)
.executeTakeFirstOrThrow();
expect(d.kind).toBe("source_unreachable");
expect(JSON.stringify(d)).not.toContain("WRONGKEY");
const row = await h.db.selectFrom("feed_source").selectAll().where("id", "=", f.id).executeTakeFirstOrThrow();
expect(row.health_status).toBe("failing");
expect(row.backoff_until!.getTime()).toBeGreaterThan(Date.now() + 23 * 3600_000);
// 再次调度:退避期内不发请求
const before = s.requests.length;
const again = await ingestFeed(h.rt, f.id);
expect(again.status).toBe("skipped");
expect(s.requests.length).toBe(before);
// 调度器也不会把它当作到期
await h.db
.updateTable("feed_source")
.set({ next_fetch_at: new Date(0) })
.where("id", "=", f.id)
.execute();
await schedulerTick(h.rt);
const due = (await h.outbox("feed.due")).filter((e) => e.payload.feedId === f.id);
expect(due).toHaveLength(0);
// 恢复账号给后续用例
await h.db.updateTable("site_account").set({ status: "active" }).where("id", "=", accountId!).execute();
});
it("调度器:到期 Feed 发 feed.due(幂等),快照到期入队 snapshotSite", async () => {
const s = await feedServer([item("sch-1", "900400", "S.2020.1080p-Z")]);
const f = await seedFeed(h, { siteId, url: s.feedUrl, name: "调度" });
const t1 = await schedulerTick(h.rt);
expect(t1.feedsDue).toBeGreaterThanOrEqual(1);
const due = (await h.outbox("feed.due")).filter((e) => e.payload.feedId === f.id);
expect(due).toHaveLength(1);
expect(
sourcesOutboxRoutes["feed.due"]!({ id: "1", type: "feed.due", payload: due[0]!.payload, at: new Date() }),
).toEqual({
task: "ingestFeed",
input: { feedSourceId: f.id, manual: false },
});
// next_fetch_at 已顺延:紧接着的 tick 不再重复
await schedulerTick(h.rt);
expect((await h.outbox("feed.due")).filter((e) => e.payload.feedId === f.id)).toHaveLength(1);
expect(
h.engine.enqueued.some((e) => e.task === "snapshotSite" && (e.input as { siteId: string }).siteId === siteId),
).toBe(true);
});
it("站点关闭后 Feed 不再抓取", async () => {
const s = await feedServer([item("off-1", "900500", "O.2020.1080p-Z")]);
const f = await seedFeed(h, { siteId, url: s.feedUrl, name: "关站" });
await h.db.updateTable("site").set({ enabled: false }).where("id", "=", siteId).execute();
const r = await ingestFeed(h.rt, f.id);
expect(r).toMatchObject({ status: "skipped", reason: "site_disabled" });
expect(s.requests.length).toBe(0);
await h.db.updateTable("site").set({ enabled: true }).where("id", "=", siteId).execute();
});
});
@@ -0,0 +1,97 @@
import { TaskRegistry } from "@umt/tasks";
import { createMockMteamApi, type MockMteamApi } from "@umt/testkit";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { snapshotSite } from "../../src/modules/sources-identity/service/snapshot.ts";
import { createHarness, type Harness, pgAvailable } from "./harness.ts";
const up = await pgAvailable();
describe.skipIf(!up)("M-Team 快照与任务注册", () => {
let h: Harness;
let api: MockMteamApi;
let siteId: string;
beforeAll(async () => {
api = await createMockMteamApi();
h = await createHarness("mteam");
const site = await h.db
.insertInto("site")
.values({
key: "mteam",
name: "馒头",
adapter: "mteam_api",
base_url: "https://kp.m-team.cc/",
rate_limit: JSON.stringify({ minIntervalMs: 0 }),
config: JSON.stringify({ apiBaseUrl: api.baseUrl }),
})
.returningAll()
.executeTakeFirstOrThrow();
siteId = site.id;
const ref = await h.secrets.create({ purpose: "site_api_key", plaintext: api.apiKey });
await h.db
.insertInto("site_account")
.values({ site_id: siteId, label: "api", auth_kind: "api_key", secret_ref: ref })
.execute();
});
afterAll(async () => {
await h?.close();
await api?.close();
});
it("API Key 站点快照:原始分类保留,促销带观察/过期时间;key 不入库", async () => {
const r = await snapshotSite(h.rt, { siteId, kind: "periodic" });
expect(r.status).toBe("ok");
expect(r.itemCount).toBeGreaterThan(0);
const rows = await h.db.selectFrom("source_release").selectAll().where("site_id", "=", siteId).execute();
const sports = rows.find((x) => x.site_release_id === "912237");
expect(sports?.category_raw).toBe("407");
const promo = rows.find((x) => x.site_release_id === "912233")!;
expect((promo.promotion as { observedAt?: string }).observedAt).toBeTruthy();
expect(JSON.stringify(rows)).not.toContain(api.apiKey);
});
it("key 失效:账号标失效 + 待处理,快照结果 failed", async () => {
api.setFailMode("banned_key");
const r = await snapshotSite(h.rt, { siteId, kind: "manual" });
expect(r.status).toBe("failed");
const acc = await h.db
.selectFrom("site_account")
.select(["status"])
.where("site_id", "=", siteId)
.executeTakeFirstOrThrow();
expect(["invalid", "expired"]).toContain(acc.status);
const d = await h.db
.selectFrom("decision")
.selectAll()
.where("dedupe_key", "=", `site-auth:${siteId}`)
.executeTakeFirstOrThrow();
expect(d.status).toBe("open");
api.setFailMode(null);
});
it("模块注册 4 个任务与 3 条 outbox 路由;任务处理器经 step.run 调服务", async () => {
const reg = new TaskRegistry();
h.module.registerTasks(reg, h.ctx);
expect(reg.names()).toEqual(["ingestFeed", "refreshMetadata", "resolveObservation", "snapshotSite"]);
expect(Object.keys(h.module.outboxRoutes!(h.ctx)).sort()).toEqual([
"feed.due",
"feed.gap_suspected",
"release.observed",
]);
const def = reg.latest("resolveObservation");
expect(def.inputSchema.safeParse({}).success).toBe(false);
expect(def.inputSchema.safeParse({ observationId: "00000000-0000-4000-8000-000000000000" }).success).toBe(true);
const steps: string[] = [];
const out = await reg.latest("snapshotSite").handler({
input: { siteId, kind: "periodic" },
step: {
run: async (name: string, fn: () => unknown) => {
steps.push(name);
return fn();
},
},
} as never);
expect(steps).toEqual(["snapshot"]);
expect(out).toMatchObject({ status: "skipped", reason: "auth_invalid" });
});
});
@@ -0,0 +1,403 @@
import { CapabilityRegistry } from "@umt/agent";
import { createMockTmdb, type MockTmdb, TMDB_FIXTURES, type TmdbFixture } from "@umt/testkit";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { refreshMetadata } from "../../src/modules/sources-identity/service/metadata.ts";
import { resolveRelease } from "../../src/modules/sources-identity/service/resolve.ts";
import { createHarness, type Harness, json, pgAvailable, seedSite } from "./harness.ts";
const up = await pgAvailable();
const twin = (id: number): TmdbFixture => ({
type: "movie",
id,
original_title: "Twins",
original_language: "en",
origin_country: ["US"],
date: "1988-12-09",
titles: { "zh-CN": "龙兄鼠弟", "en-US": "Twins" },
popularity: 10 + id / 1000,
alternative_titles: [],
external_ids: {},
});
describe.skipIf(!up)("resolveObservation / 身份与元数据", () => {
let h: Harness;
let tmdb: MockTmdb;
let siteId: string;
let n = 0;
beforeAll(async () => {
tmdb = await createMockTmdb({ fixtures: [...TMDB_FIXTURES, twin(111), twin(112)] });
h = await createHarness("resolve", { tmdbUrl: tmdb.url });
siteId = (await seedSite(h, { baseUrl: "https://audiences.example/" })).site.id;
});
afterAll(async () => {
await h?.close();
await tmdb?.close();
});
const release = async (title: string, extra: { subtitle?: string; category?: string } = {}) =>
h.db
.insertInto("source_release")
.values({
site_id: siteId,
site_release_id: String(700000 + ++n),
title,
subtitle: extra.subtitle ?? null,
category_raw: extra.category ?? null,
})
.returningAll()
.executeTakeFirstOrThrow();
let oppId: string;
it("auto:按 TMDB 建本地作品 + accepted 外部身份 + 别名,发 release.resolved", async () => {
const r = await release("Oppenheimer.2023.2160p.UHD.BluRay.REMUX.HDR.HEVC-FGT");
const res = await resolveRelease(h.rt, { sourceReleaseId: r.id });
expect(res.outcome).toBe("auto");
oppId = res.mediaEntityId!;
const e = await h.db.selectFrom("media_entity").selectAll().where("id", "=", oppId).executeTakeFirstOrThrow();
expect(e).toMatchObject({ kind: "movie", year: 2023, status: "active" });
const ids = await h.db.selectFrom("external_identity").selectAll().where("media_entity_id", "=", oppId).execute();
expect(ids.map((i) => `${i.provider}:${i.entity_type}:${i.external_id}:${i.status}`).sort()).toEqual([
"imdb:movie:tt15398776:accepted",
"tmdb:movie:872585:accepted",
]);
const revs = await h.db.selectFrom("identity_revision").selectAll().execute();
expect(revs.length).toBeGreaterThanOrEqual(2);
const aliases = await h.db
.selectFrom("title_alias")
.select(["alias"])
.where("media_entity_id", "=", oppId)
.execute();
expect(aliases.map((a) => a.alias)).toEqual(expect.arrayContaining(["奥本海默", "Oppenheimer"]));
const rel = await h.db.selectFrom("source_release").selectAll().where("id", "=", r.id).executeTakeFirstOrThrow();
expect(rel).toMatchObject({ resolution_status: "resolved", media_entity_id: oppId, parser_version: "1" });
expect((rel.parsed as { year: number }).year).toBe(2023);
const ev = (await h.outbox("release.resolved")).find((x) => x.payload.sourceReleaseId === r.id);
expect(ev?.payload).toMatchObject({ mediaEntityId: oppId, evidenceLevel: "rule_confirmed", contentUnitIds: [] });
// 幂等:证据未变的重复识别直接短路
expect((await resolveRelease(h.rt, { sourceReleaseId: r.id })).outcome).toBe("skipped");
});
it("同一作品的其它资源经本地召回复用同一实体,不重复建档", async () => {
const r = await release("奥本海默.Oppenheimer.2023.1080p.WEB-DL.H264-GRP");
const res = await resolveRelease(h.rt, { sourceReleaseId: r.id });
expect(res).toMatchObject({ outcome: "auto", mediaEntityId: oppId });
const count = await h.db
.selectFrom("media_entity")
.select((eb) => eb.fn.countAll<number>().as("c"))
.executeTakeFirstOrThrow();
expect(Number(count.c)).toBe(1);
});
it("剧集:建 series 作品与临时季/集单元", async () => {
const r = await release("The.Bear.S03E02.2160p.DSNP.WEB-DL.DDP5.1.H.265-FLUX", { subtitle: "熊家餐馆 第三季" });
const res = await resolveRelease(h.rt, { sourceReleaseId: r.id });
expect(res.outcome).toBe("auto");
const units = await h.db
.selectFrom("content_unit")
.selectAll()
.where("media_entity_id", "=", res.mediaEntityId!)
.orderBy("kind")
.execute();
expect(units.map((u) => `${u.kind}:${u.season_number}:${u.episode_number}:${u.status}`)).toEqual([
"episode:3:2:provisional",
"season:3:null:provisional",
]);
expect(res.contentUnitIds).toEqual([units[0]!.id]);
const e = await h.db
.selectFrom("media_entity")
.select(["kind"])
.where("id", "=", res.mediaEntityId!)
.executeTakeFirstOrThrow();
expect(e.kind).toBe("series");
});
let twinsRelease: string;
it("ambiguous:进待识别池 + 待处理事项,不建正式作品", async () => {
const before = await h.db.selectFrom("media_entity").select("id").execute();
const r = await release("Twins.1988.1080p.BluRay.x264-GRP");
twinsRelease = r.id;
const res = await resolveRelease(h.rt, { sourceReleaseId: r.id });
expect(res.outcome).toBe("ambiguous");
expect((await h.db.selectFrom("media_entity").select("id").execute()).length).toBe(before.length);
const pool = await h.db
.selectFrom("identity_candidate")
.selectAll()
.where("source_release_id", "=", r.id)
.executeTakeFirstOrThrow();
expect(pool).toMatchObject({ status: "pending", reason: "ambiguous", attempts: 1 });
expect((pool.candidates as unknown[]).length).toBe(2);
expect(pool.next_review_at!.getTime()).toBeGreaterThan(Date.now());
const d = await h.db.selectFrom("decision").selectAll().where("id", "=", res.decisionId!).executeTakeFirstOrThrow();
expect(d).toMatchObject({ kind: "identity_ambiguous", status: "open", subject_id: r.id });
expect((d.options as { key: string }[]).map((o) => o.key)).toEqual(["c0", "c1", "dismiss"]);
expect((await h.outbox("decision.created")).some((e) => e.payload.decisionId === d.id)).toBe(true);
const rel = await h.db.selectFrom("source_release").selectAll().where("id", "=", r.id).executeTakeFirstOrThrow();
expect(rel.resolution_status).toBe("ambiguous");
expect(rel.media_entity_id).toBeNull();
const attempts = await h.db
.selectFrom("resolution_attempt")
.selectAll()
.where("source_release_id", "=", r.id)
.execute();
expect(attempts.map((a) => a.outcome)).toEqual(["ambiguous"]);
// 同一证据再次复查:计数递增(退避加长),不重复开待处理
await resolveRelease(h.rt, { sourceReleaseId: r.id, force: true });
const pool2 = await h.db
.selectFrom("identity_candidate")
.selectAll()
.where("source_release_id", "=", r.id)
.executeTakeFirstOrThrow();
expect(pool2.attempts).toBe(2);
expect(pool2.next_review_at!.getTime()).toBeGreaterThan(pool.next_review_at!.getTime());
expect(
await h.db.selectFrom("decision").select("id").where("dedupe_key", "=", `identity:${r.id}`).execute(),
).toHaveLength(1);
});
it("待识别池可按原始标题检索;经待处理中心选候选 = 人工指定并加锁", async () => {
const list = await json(await h.req("/identity-pool?q=Twins"));
expect(list.status).toBe(200);
expect(list.body.data.items.map((i: { sourceReleaseId: string }) => i.sourceReleaseId)).toContain(twinsRelease);
const pool = list.body.data.items.find((i: { sourceReleaseId: string }) => i.sourceReleaseId === twinsRelease);
const res = await json(await h.req(`/decisions/${pool.decisionId}/resolve`, { body: { optionKey: "c1" } }));
expect(res.status).toBe(200);
expect(res.body.data.status).toBe("resolved");
const rel = await h.db
.selectFrom("source_release")
.selectAll()
.where("id", "=", twinsRelease)
.executeTakeFirstOrThrow();
expect(rel.resolution_locked).toBe(true);
const chosen = pool.candidates[1];
const ident = await h.db
.selectFrom("external_identity")
.selectAll()
.where("media_entity_id", "=", rel.media_entity_id!)
.where("provider", "=", "tmdb")
.executeTakeFirstOrThrow();
expect(ident).toMatchObject({
external_id: chosen.externalId,
evidence_level: "manual",
locked: true,
status: "accepted",
});
const p = await h.db
.selectFrom("identity_candidate")
.selectAll()
.where("source_release_id", "=", twinsRelease)
.executeTakeFirstOrThrow();
expect(p.status).toBe("resolved");
const ev = (await h.outbox("release.resolved")).find((x) => x.payload.sourceReleaseId === twinsRelease);
expect(ev?.payload.evidenceLevel).toBe("manual");
// 人工锁:自动识别不再改写
const again = await resolveRelease(h.rt, { sourceReleaseId: twinsRelease, force: true });
expect(again.outcome).toBe("skipped_locked");
const after = await h.db
.selectFrom("source_release")
.select(["media_entity_id"])
.where("id", "=", twinsRelease)
.executeTakeFirstOrThrow();
expect(after.media_entity_id).toBe(rel.media_entity_id);
});
it("no_match 只进池不开待处理;TMDB 失败记为 lookup_failed 而不是无结果", async () => {
const r1 = await release("Zzqxv.Nonexistent.2019.1080p.WEB-DL-NOPE");
const a = await resolveRelease(h.rt, { sourceReleaseId: r1.id });
expect(a.outcome).toBe("no_match");
expect(a.decisionId).toBeNull();
const rel = await h.db
.selectFrom("source_release")
.select(["resolution_status"])
.where("id", "=", r1.id)
.executeTakeFirstOrThrow();
expect(rel.resolution_status).toBe("unidentifiable");
tmdb.fault(/\/search\//, { status: 500, times: 20 });
const r2 = await release("Qwerty.Unknown.2020.1080p.WEB-DL-NOPE");
const b = await resolveRelease(h.rt, { sourceReleaseId: r2.id });
expect(b.outcome).toBe("lookup_failed");
const pool = await h.db
.selectFrom("identity_candidate")
.selectAll()
.where("source_release_id", "=", r2.id)
.executeTakeFirstOrThrow();
expect(pool.reason).toBe("lookup_failed");
// 查询失败的复查间隔短于无结果
expect(pool.next_review_at!.getTime() - Date.now()).toBeLessThan(20 * 60_000);
tmdb.clearFaults();
});
it("自动化关闭:证据足够也只提议(proposed),进池待确认", async () => {
await h.db
.insertInto("settings")
.values({ key: "automation", value: JSON.stringify({ enabled: false }) } as never)
.execute();
const r = await release("Dune.Part.Two.2024.2160p.UHD.BluRay.REMUX-FraMeSToR", { subtitle: "沙丘2" });
const res = await resolveRelease(h.rt, { sourceReleaseId: r.id });
expect(res.outcome).toBe("proposed");
expect(res.decisionId).not.toBeNull();
await h.db.deleteFrom("settings").where("key", "=", "automation").execute();
});
it("refreshMetadata:字段级刷新;用户锁定字段不被覆盖,解锁后回到来源值", async () => {
const r1 = await refreshMetadata(h.rt, oppId);
expect(r1.status).toBe("refreshed");
const fields = await h.db.selectFrom("metadata_field").selectAll().where("media_entity_id", "=", oppId).execute();
expect(fields.map((f) => f.field)).toEqual(expect.arrayContaining(["title", "overview", "year"]));
let e = await h.db.selectFrom("media_entity").selectAll().where("id", "=", oppId).executeTakeFirstOrThrow();
expect(e.title).toBe("奥本海默");
expect((e.metadata as Record<string, { source: string }>).overview?.source).toBe("tmdb");
const put = await json(
await h.req(`/media-entities/${oppId}/fields/title`, {
method: "PUT",
body: { value: "奥本海默(IMAX)", reason: "偏好" },
}),
);
expect(put.status).toBe(200);
expect(put.body.data.lockedFields).toContain("title");
const r2 = await refreshMetadata(h.rt, oppId);
expect(r2.lockedFields).toContain("title");
e = await h.db.selectFrom("media_entity").selectAll().where("id", "=", oppId).executeTakeFirstOrThrow();
expect(e.title).toBe("奥本海默(IMAX)");
expect(e.field_locks).toContain("title");
expect((e.metadata as Record<string, { source: string; locked: boolean }>).title).toMatchObject({
source: "user",
locked: true,
});
const del = await json(await h.req(`/media-entities/${oppId}/fields/title`, { method: "DELETE" }));
expect(del.status).toBe(200);
e = await h.db.selectFrom("media_entity").selectAll().where("id", "=", oppId).executeTakeFirstOrThrow();
expect(e.title).toBe("奥本海默");
expect(e.field_locks).not.toContain("title");
});
it("人工纠错映射:旧映射退役留修订,新映射 accepted + 锁;自动匹配不改回", async () => {
const res = await json(
await h.req(`/media-entities/${oppId}/identities`, {
body: { provider: "tmdb", entityType: "movie", externalId: "693134", reason: "误匹配" },
}),
);
expect(res.status).toBe(200);
expect(res.body.data.changed).toBe(true);
const ids = await h.db
.selectFrom("external_identity")
.selectAll()
.where("media_entity_id", "=", oppId)
.where("provider", "=", "tmdb")
.execute();
const old = ids.find((i) => i.external_id === "872585")!;
expect(old).toMatchObject({ status: "retired", locked: true });
expect(ids.find((i) => i.external_id === "693134")).toMatchObject({
status: "accepted",
locked: true,
evidence_level: "manual",
});
const revs = await json(await h.req(`/external-identities/${old.id}/revisions`));
expect(revs.body.data.items.map((r: { action: string }) => r.action)).toEqual(["created", "status_changed"]);
// 新来一条奥本海默资源:TMDB 872585 不再自动绑回该作品
const r = await release("Oppenheimer.2023.1080p.BluRay.x264-OTHER");
await resolveRelease(h.rt, { sourceReleaseId: r.id });
const accepted = await h.db
.selectFrom("external_identity")
.select(["media_entity_id"])
.where("provider", "=", "tmdb")
.where("external_id", "=", "872585")
.where("status", "=", "accepted")
.execute();
expect(accepted.every((a) => a.media_entity_id !== oppId)).toBe(true);
const still = await h.db
.selectFrom("external_identity")
.select(["external_id"])
.where("media_entity_id", "=", oppId)
.where("provider", "=", "tmdb")
.where("status", "=", "accepted")
.executeTakeFirstOrThrow();
expect(still.external_id).toBe("693134");
// 目标身份已属于其它作品 → 409
const bear = await h.db
.selectFrom("external_identity")
.select(["media_entity_id"])
.where("external_id", "=", "136315")
.executeTakeFirstOrThrow();
const conflict = await json(
await h.req(`/media-entities/${oppId}/identities`, {
body: { provider: "tmdb", entityType: "tv", externalId: "136315" },
}),
);
expect(conflict.status).toBe(409);
expect(bear.media_entity_id).not.toBe(oppId);
});
it("合并重复作品:旧 ID 跳转,来源资源与别名迁到目标", async () => {
const dup = await json(
await h.req("/media-entities", { body: { kind: "movie", title: "奥本海默 重复", year: 2023 } }),
);
expect(dup.status).toBe(200);
const dupId = dup.body.data.id;
const r = await release("Dup.Title.2023.1080p-X");
await h.db.updateTable("source_release").set({ media_entity_id: dupId }).where("id", "=", r.id).execute();
const m = await json(await h.req(`/media-entities/${dupId}/merge`, { body: { intoId: oppId, reason: "重复" } }));
expect(m.status).toBe(200);
expect(m.body.data.id).toBe(oppId);
const detail = await json(await h.req(`/media-entities/${dupId}/detail`));
expect(detail.body.data.entity.id).toBe(oppId);
expect(detail.body.data.redirectedFrom).toBe(dupId);
expect(detail.body.data.aliases.map((a: { alias: string }) => a.alias)).toContain("奥本海默 重复");
const moved = await h.db
.selectFrom("source_release")
.select(["media_entity_id"])
.where("id", "=", r.id)
.executeTakeFirstOrThrow();
expect(moved.media_entity_id).toBe(oppId);
const redirect = await h.db
.selectFrom("identity_redirect")
.selectAll()
.where("from_entity_id", "=", dupId)
.executeTakeFirstOrThrow();
expect(redirect.to_entity_id).toBe(oppId);
// 再合并一次 → 409
expect((await h.req(`/media-entities/${dupId}/merge`, { body: { intoId: oppId } })).status).toBe(409);
});
it("待处理中心聚合与列表", async () => {
const s = await json(await h.req("/decisions/summary"));
expect(s.status).toBe(200);
expect(s.body.data.total).toBeGreaterThanOrEqual(1);
expect(s.body.data.groups.some((g: { kind: string }) => g.kind === "identity_ambiguous")).toBe(true);
const l = await json(await h.req("/decisions?kind=identity_ambiguous"));
expect(
l.body.data.items.every((d: { status: string }) => ["open", "investigating", "awaiting_user"].includes(d.status)),
).toBe(true);
// 已关闭的事项不可再处理
const closed = await h.db
.selectFrom("decision")
.select(["id"])
.where("status", "=", "resolved")
.executeTakeFirstOrThrow();
expect((await h.req(`/decisions/${closed.id}/resolve`, { body: { dismiss: true } })).status).toBe(409);
});
it("能力:catalog.lookup / release.search 可调用;identity.resolve(propose)未授权被拒", async () => {
const reg = new CapabilityRegistry({});
h.module.registerCapabilities!(reg, h.ctx);
const c = { actorId: "t", taskId: "t", scopeIds: [], signal: new AbortController().signal };
const look = (await reg.invoke("catalog.lookup", { query: "奥本海默", includeRemote: true }, c)) as {
local: { entity: { id: string } }[];
remote: { externalId: string }[];
};
expect(look.local.map((l) => l.entity.id)).toContain(oppId);
expect(look.remote.map((r) => r.externalId)).toContain("872585");
const search = (await reg.invoke("release.search", { query: "Oppenheimer", allowLiveSearch: false }, c)) as {
items: unknown[];
};
expect(search.items.length).toBeGreaterThan(0);
await expect(reg.invoke("identity.resolve", { sourceReleaseId: twinsRelease }, c)).rejects.toThrow();
});
});
+81 -1
View File
@@ -1,5 +1,6 @@
import { z } from "zod";
import { DateOnly, EntityBase, Id, JsonObject, SERVER_MANAGED, Timestamp } from "./common.ts";
import { ActorType, DateOnly, EntityBase, Id, JsonObject, SERVER_MANAGED, Timestamp } from "./common.ts";
import { ExternalIdentity } from "./identity.ts";
// 内容域:本地作品、内容单元、别名(REQ §3.1、§5.1、§5.4、§7.5)
// 本地 UUID 是主身份;季/集编号、外部 ID 都可修订,不是唯一身份。
@@ -127,3 +128,82 @@ export const TitleAliasCreate = TitleAlias.pick({
export type TitleAliasCreate = z.infer<typeof TitleAliasCreate>;
export const TitleAliasUpdate = TitleAliasCreate.omit({ mediaEntityId: true }).partial();
export type TitleAliasUpdate = z.infer<typeof TitleAliasUpdate>;
// ── W2-B:字段级元数据(REQ §7.3)与作品详情 ──
export const MetadataSource = z.enum(["tmdb", "user", "agent", "nfo", "site", "import"]);
export type MetadataSource = z.infer<typeof MetadataSource>;
/** 某来源给出的一个候选字段值(刷新只写这里,不直接改最终展示值) */
export const MetadataField = EntityBase.extend({
mediaEntityId: Id.nullable(),
contentUnitId: Id.nullable(),
field: z.string().min(1),
value: z.unknown(),
language: z.string().nullable(),
source: MetadataSource,
sourceRevision: z.string().nullable(),
fetchedAt: Timestamp,
});
export type MetadataField = z.infer<typeof MetadataField>;
/** 字段覆盖:用户锁定优先于任何自动刷新 */
export const FieldOverride = EntityBase.extend({
mediaEntityId: Id.nullable(),
contentUnitId: Id.nullable(),
field: z.string().min(1),
value: z.unknown(),
locked: z.boolean(),
source: MetadataSource,
reason: z.string().nullable(),
actorType: ActorType,
actorId: z.string().nullable(),
});
export type FieldOverride = z.infer<typeof FieldOverride>;
/** 可覆盖/锁定的作品字段;title/originalTitle/year 同时投影到 media_entity 列 */
export const EntityMetadataField = z.enum([
"title",
"originalTitle",
"year",
"overview",
"releaseDate",
"originalLanguage",
"originCountry",
"posterPath",
]);
export type EntityMetadataField = z.infer<typeof EntityMetadataField>;
export const FieldOverrideInput = z.object({
value: z.unknown(),
lock: z.boolean().default(true),
reason: z.string().max(2000).optional(),
});
export type FieldOverrideInput = z.infer<typeof FieldOverrideInput>;
/** media_entity.metadata 中每个最终字段的形状 */
export const FinalFieldValue = z.object({
value: z.unknown(),
source: MetadataSource,
language: z.string().nullable(),
at: Timestamp,
locked: z.boolean(),
});
export type FinalFieldValue = z.infer<typeof FinalFieldValue>;
/** 作品详情:外部身份、别名、内容单元、字段来源与覆盖 */
export const MediaEntityDetail = z.object({
entity: MediaEntity,
/** 请求的是已合并的旧 ID 时,指向跳转后的实体 */
redirectedFrom: Id.nullable(),
identities: z.array(ExternalIdentity),
aliases: z.array(TitleAlias),
contentUnits: z.array(ContentUnit),
metadataFields: z.array(MetadataField),
overrides: z.array(FieldOverride),
});
export type MediaEntityDetail = z.infer<typeof MediaEntityDetail>;
/** 合并重复作品(保留旧 ID 跳转) */
export const MediaEntityMerge = z.object({ intoId: Id, reason: z.string().max(2000).optional() });
export type MediaEntityMerge = z.infer<typeof MediaEntityMerge>;
+7 -1
View File
@@ -4,7 +4,7 @@ import { Id, NonNegInt, Timestamp } from "./common.ts";
import { DownloadStatus, PauseReason, TaskKind, TaskStatus } from "./execution.ts";
import { EvidenceLevel } from "./identity.ts";
import { OrganizationStatus } from "./organization.ts";
import { FeedHealth } from "./sources.ts";
import { FeedDuePayload, FeedGapSuspectedPayload, FeedHealth, ReleaseObservedPayload } from "./sources.ts";
/** SSE 事件信封:seq 在进程内单调递增,客户端断线后用 ?after=<seq> 补读 */
export const EventEnvelope = z.object({
@@ -41,7 +41,10 @@ export const EventType = {
feedFetched: "feed.fetched",
feedHealthChanged: "feed.health_changed",
feedDue: "feed.due",
feedGapSuspected: "feed.gap_suspected",
releaseObserved: "release.observed",
releaseResolved: "release.resolved",
decisionCreated: "decision.created",
@@ -175,6 +178,9 @@ export const EVENT_PAYLOADS = {
"agent.tool_call": AgentToolCallPayload,
"feed.fetched": FeedFetchedPayload,
"feed.health_changed": FeedHealthChangedPayload,
"feed.due": FeedDuePayload,
"feed.gap_suspected": FeedGapSuspectedPayload,
"release.observed": ReleaseObservedPayload,
"release.resolved": ReleaseResolvedPayload,
"decision.created": DecisionCreatedPayload,
"decision.updated": DecisionUpdatedPayload,
+139
View File
@@ -119,3 +119,142 @@ export const IdentityRevisionCreate = IdentityRevision.pick({
reason: true,
}).partial({ fromStatus: true, toStatus: true, before: true, actorId: true, reason: true });
export type IdentityRevisionCreate = z.infer<typeof IdentityRevisionCreate>;
// ── W2-B:待识别池、识别尝试、合并跳转与人工纠错(REQ §7.2、§7.5) ──
export const IdentityCandidateStatus = z.enum(["pending", "resolved", "dismissed", "archived"]);
export type IdentityCandidateStatus = z.infer<typeof IdentityCandidateStatus>;
/** proposed = 证据足够但自动化关闭/只读,待确认 */
export const IdentityCandidateReason = z.enum(["ambiguous", "no_match", "proposed", "lookup_failed"]);
export type IdentityCandidateReason = z.infer<typeof IdentityCandidateReason>;
/** 待识别池里的一个候选(只存 JSON,不建正式作品) */
export const PoolCandidate = z.object({
provider: z.string(),
entityType: z.enum(["movie", "tv"]),
externalId: z.string(),
title: z.string(),
year: z.number().int().nullable(),
/** 本地候选的作品 ID */
localEntityId: Id.nullable(),
level: z.enum(["rule_confirmed", "multi_evidence", "needs_confirm", "rejected"]),
evidence: z.array(z.string()),
});
export type PoolCandidate = z.infer<typeof PoolCandidate>;
/** 待识别池条目:每条 source_release 至多一条;失败原因、证据版本与下一次复查时间 */
export const IdentityCandidate = EntityBase.extend({
sourceReleaseId: Id,
status: IdentityCandidateStatus,
reason: IdentityCandidateReason,
/** 解析出的检索条件(标题/年份/季集) */
query: JsonObject,
candidates: z.array(PoolCandidate),
evidenceVersion: z.string(),
attempts: z.number().int().nonnegative(),
lastAttemptAt: Timestamp.nullable(),
nextReviewAt: Timestamp.nullable(),
decisionId: Id.nullable(),
resolvedMediaEntityId: Id.nullable(),
resolvedBy: z.string().nullable(),
resolvedAt: Timestamp.nullable(),
});
export type IdentityCandidate = z.infer<typeof IdentityCandidate>;
export const ResolutionOutcome = z.enum([
"auto",
"proposed",
"ambiguous",
"no_match",
"lookup_failed",
"manual",
"skipped_locked",
]);
export type ResolutionOutcome = z.infer<typeof ResolutionOutcome>;
/** 一次识别尝试(只追加) */
export const ResolutionAttempt = EntityBase.extend({
sourceReleaseId: Id,
observationId: Id.nullable(),
outcome: ResolutionOutcome,
reason: z.string().nullable(),
evidenceVersion: z.string(),
candidateCount: z.number().int().nonnegative(),
mediaEntityId: Id.nullable(),
best: JsonObject.nullable(),
error: JsonObject.nullable(),
actorType: ActorType,
actorId: z.string().nullable(),
});
export type ResolutionAttempt = z.infer<typeof ResolutionAttempt>;
/** 作品合并后的旧 ID 跳转 */
export const IdentityRedirect = EntityBase.extend({
fromEntityId: Id,
toEntityId: Id,
reason: z.string().nullable(),
actorType: ActorType,
actorId: z.string().nullable(),
});
export type IdentityRedirect = z.infer<typeof IdentityRedirect>;
/** 外部身份引用(provider + 类型 + ID) */
export const ExternalRef = z.object({
provider: IdentityProvider,
entityType: z.enum(["movie", "tv"]),
externalId: z.string().min(1),
});
export type ExternalRef = z.infer<typeof ExternalRef>;
/**
* 人工指定来源资源的身份:本地作品 / 外部身份(未登记时按外部详情建档)/ 池内候选序号,三选一。
* 指定后加人工锁,自动识别不再改写。
*/
export const ReleaseIdentityAssign = z
.object({
mediaEntityId: Id.optional(),
external: ExternalRef.optional(),
candidateIndex: z.number().int().min(0).optional(),
/** 同时关联到某个内容单元(季/集);缺省按解析出的季集建临时单元 */
contentUnitId: Id.nullable().optional(),
lock: z.boolean().default(true),
reason: z.string().max(2000).optional(),
})
.refine((v) => [v.mediaEntityId, v.external, v.candidateIndex].filter((x) => x !== undefined).length === 1, {
message: "mediaEntityId / external / candidateIndex 必须且只能给一个",
});
export type ReleaseIdentityAssign = z.infer<typeof ReleaseIdentityAssign>;
/** 改作品的外部映射:旧的同 provider+类型 accepted 映射退役(留修订),新映射 accepted + 人工锁 */
export const EntityIdentitySet = ExternalRef.extend({
reason: z.string().max(2000).optional(),
/** 默认加人工锁 */
lock: z.boolean().default(true),
});
export type EntityIdentitySet = z.infer<typeof EntityIdentitySet>;
export const IdentityPoolQuery = z.object({
status: IdentityCandidateStatus.optional(),
reason: IdentityCandidateReason.optional(),
/** 按原始标题/解析标题检索(已观察到的资源必须能从待识别入口找回) */
q: z.string().trim().min(1).max(200).optional(),
siteId: Id.optional(),
cursor: z.string().min(1).optional(),
limit: z.coerce.number().int().min(1).max(200).default(50),
});
export type IdentityPoolQuery = z.infer<typeof IdentityPoolQuery>;
/** 待识别池列表条目:附来源资源摘要 */
export const IdentityPoolItem = IdentityCandidate.extend({
release: z.object({
id: Id,
siteId: Id,
siteReleaseId: z.string(),
title: z.string(),
subtitle: z.string().nullable(),
categoryRaw: z.string().nullable(),
firstSeenAt: Timestamp,
resolutionStatus: z.string(),
}),
});
export type IdentityPoolItem = z.infer<typeof IdentityPoolItem>;
+145 -1
View File
@@ -4,7 +4,8 @@ import { EntityBase, Id, JsonObject, NonNegInt, SERVER_MANAGED, Timestamp } from
// 来源域:站点、站点账号、Feed、站点资源、观察(REQ §4)
// 含 passkey 的 Feed URL 只以 urlSecretRef 保存,对外只给脱敏的 urlDisplay。
export const SiteAdapter = z.enum(["nexusphp", "mteam_api", "generic_rss"]);
/** api = MP 专用 API spider(Haidan/HDDolby/RousiPro/SunnyPT/TNode/TorrentLeech/Yema) */
export const SiteAdapter = z.enum(["nexusphp", "mteam_api", "api", "generic_rss"]);
export type SiteAdapter = z.infer<typeof SiteAdapter>;
export const SiteHealth = z.enum(["unknown", "healthy", "degraded", "failing", "disabled"]);
export type SiteHealth = z.infer<typeof SiteHealth>;
@@ -180,6 +181,8 @@ export const SourceRelease = EntityBase.extend({
resolutionEvidence: JsonObject,
availability: ReleaseAvailability,
raw: JsonObject,
/** 人工确认的身份关联:自动识别/复查不得改写 mediaEntityId/contentUnitId */
resolutionLocked: z.boolean(),
});
export type SourceRelease = z.infer<typeof SourceRelease>;
/** 由适配器/任务写入(按 siteId+siteReleaseId upsert) */
@@ -209,6 +212,7 @@ export const SourceReleaseCreate = SourceRelease.omit({
resolutionEvidence: true,
availability: true,
raw: true,
resolutionLocked: true,
});
export type SourceReleaseCreate = z.infer<typeof SourceReleaseCreate>;
export const SourceReleaseUpdate = SourceReleaseCreate.omit({ siteId: true, siteReleaseId: true }).partial();
@@ -259,3 +263,143 @@ export const ObservationUpdate = Observation.pick({
lastError: true,
}).partial();
export type ObservationUpdate = z.infer<typeof ObservationUpdate>;
// ── W2-B:站点快照、调度事件与站点工作台 ──
export const SiteSnapshotKind = z.enum(["periodic", "backfill", "live_search", "manual"]);
export type SiteSnapshotKind = z.infer<typeof SiteSnapshotKind>;
export const SiteSnapshotStatus = z.enum(["running", "ok", "partial", "failed"]);
export type SiteSnapshotStatus = z.infer<typeof SiteSnapshotStatus>;
/** 一次列表快照 / 实时补查 / 补漏的覆盖范围与结果(不把一次分页伪称全站快照,REQ §4.6) */
export const SiteSnapshot = EntityBase.extend({
siteId: Id,
kind: SiteSnapshotKind,
query: z.string().nullable(),
filters: JsonObject,
/** 实际抓取的页码(1 起) */
pages: z.array(z.number().int().positive()),
itemCount: NonNegInt,
newCount: NonNegInt,
status: SiteSnapshotStatus,
error: JsonObject.nullable(),
definitionVersion: z.string().nullable(),
startedAt: Timestamp,
finishedAt: Timestamp.nullable(),
});
export type SiteSnapshot = z.infer<typeof SiteSnapshot>;
/** 站点注册表条目(随产品发布的适配器定义)+ 是否已在本系统登记 */
export const SiteDefinitionView = z.object({
key: z.string(),
name: z.string(),
adapter: SiteAdapter,
baseUrl: z.string(),
/** 定义是否随产品启用(未实测站点为 false) */
shippedEnabled: z.boolean(),
definitionVersion: z.string(),
capabilities: z.object({
list: z.boolean(),
search: z.boolean(),
detail: z.boolean(),
download: z.boolean(),
rss: z.boolean(),
}),
defaultRateLimit: z.object({ minIntervalMs: NonNegInt, burst: z.number().int().positive() }),
authSupported: z.array(z.string()),
/** 已登记的 site 行 ID;未登记为 null */
siteId: Id.nullable(),
});
export type SiteDefinitionView = z.infer<typeof SiteDefinitionView>;
/** Feed 到期(调度器发出,outbox 路由到 ingestFeed) */
export const FeedDuePayload = z.object({ feedId: Id, reason: z.enum(["schedule", "manual"]) });
/** 新的 source_release 被观察到(或同一资源标题更新),outbox 路由到 resolveObservation */
export const ReleaseObservedPayload = z.object({
sourceReleaseId: Id,
observationId: Id.nullable(),
siteId: Id,
isNew: z.boolean(),
});
/** Feed 窗口与上次零重叠:疑似缺口,路由到有预算的列表补漏(snapshotSite backfill) */
export const FeedGapSuspectedPayload = z.object({
feedId: Id,
siteId: Id.nullable(),
previousFetchedAt: Timestamp.nullable(),
detectedAt: Timestamp,
});
/** 站点工作台列表查询(整理视图):快照优先,必要时实时补查 */
export const WorkbenchReleaseQuery = z.object({
q: z.string().trim().min(1).max(200).optional(),
category: z.string().max(100).optional(),
/** 只看某类促销(free/2xfree/50%…) */
promotion: z.string().max(20).optional(),
resolution: ResolutionStatus.optional(),
/** true = 强制实时补查;false = 只看本地快照;缺省 = 本地无结果或快照过期时补查 */
live: z.enum(["true", "false"]).optional(),
cursor: z.string().min(1).optional(),
limit: z.coerce.number().int().min(1).max(200).default(50),
});
export type WorkbenchReleaseQuery = z.infer<typeof WorkbenchReleaseQuery>;
/** 数据来源:快照 / Feed 观察 / 实时补查 */
export const ReleaseOrigin = z.enum(["snapshot", "feed", "live", "unknown"]);
export type ReleaseOrigin = z.infer<typeof ReleaseOrigin>;
/**
* 工作台条目:source_release + 来源标注 + 本地实体关联。
* 库存叠加(已有资产/Slot/下载状态)由前端按 mediaEntityId 二次查询 W2-C 端点,不在这里猜。
*/
export const WorkbenchRelease = SourceRelease.extend({
origin: ReleaseOrigin,
/** 本条数据最近一次被站点确认的时间(快照/Feed/实时) */
dataUpdatedAt: Timestamp,
observedByFeeds: NonNegInt,
entity: z.object({ id: Id, title: z.string(), year: z.number().int().nullable(), kind: z.string() }).nullable(),
});
export type WorkbenchRelease = z.infer<typeof WorkbenchRelease>;
export const WorkbenchCoverage = z.object({
siteEnabled: z.boolean(),
lastSnapshotAt: Timestamp.nullable(),
/** 快照过期(超过 2 个周期或从未快照) */
stale: z.boolean(),
fromSnapshot: z.boolean(),
live: z.boolean(),
liveError: z.string().nullable(),
/** 同一搜索正在进行时合并到已有请求 */
coalesced: z.boolean(),
});
export type WorkbenchCoverage = z.infer<typeof WorkbenchCoverage>;
export const WorkbenchReleasePage = z.object({
items: z.array(WorkbenchRelease),
nextCursor: z.string().nullable(),
coverage: WorkbenchCoverage,
});
export type WorkbenchReleasePage = z.infer<typeof WorkbenchReleasePage>;
export const WorkbenchReleaseDetail = z.object({
release: WorkbenchRelease,
observations: z.array(
z.object({
observationId: Id,
feedId: Id,
feedName: z.string(),
firstSeenAt: Timestamp,
lastSeenAt: Timestamp,
}),
),
/** 易变字段的复核时间:缓存命中不能替代下载前复核(REQ §4.6) */
volatile: z.object({
promotionObservedAt: Timestamp.nullable(),
promotionExpiresAt: Timestamp.nullable(),
peersObservedAt: Timestamp,
detailFetchedAt: Timestamp.nullable(),
recheckBeforeDownload: z.literal(true),
}),
/** 刷新详情失败时保留既有数据并给出原因 */
refreshError: z.string().nullable(),
});
export type WorkbenchReleaseDetail = z.infer<typeof WorkbenchReleaseDetail>;
+18
View File
@@ -79,6 +79,13 @@ const ROWS: Record<string, { row: Obj; dbOnly?: string[]; apiOnly?: string[] }>
Subscription: { row: C.Subscription },
Task: { row: C.Task },
TitleAlias: { row: C.TitleAlias },
// W2-B 来源/身份/元数据(0018–0020)
FieldOverride: { row: C.FieldOverride },
IdentityCandidate: { row: C.IdentityCandidate },
IdentityRedirect: { row: C.IdentityRedirect },
MetadataField: { row: C.MetadataField },
ResolutionAttempt: { row: C.ResolutionAttempt },
SiteSnapshot: { row: C.SiteSnapshot },
};
/** 引擎自有表(@umt/tasks 内部使用,不经 API 暴露),不要求 Row schema */
@@ -186,6 +193,17 @@ const ENUMS: Record<string, z.ZodEnum> = {
TaskStatus: C.TaskStatus,
TitleAliasKind: C.AliasKind,
TitleAliasSource: C.AliasSource,
// W2-B(0018–0020)
FieldOverrideSource: C.MetadataSource,
FieldOverrideActorType: C.ActorType,
IdentityCandidateStatus: C.IdentityCandidateStatus,
IdentityCandidateReason: C.IdentityCandidateReason,
IdentityRedirectActorType: C.ActorType,
MetadataFieldSource: C.MetadataSource,
ResolutionAttemptOutcome: C.ResolutionOutcome,
ResolutionAttemptActorType: C.ActorType,
SiteSnapshotKind: C.SiteSnapshotKind,
SiteSnapshotStatus: C.SiteSnapshotStatus,
};
describe("contracts ⇔ db 表结构同步", () => {
+1
View File
@@ -91,6 +91,7 @@ const OVERRIDES: Record<string, (v: Record<string, unknown>) => Record<string, u
CatalogLookupInput: (v) => v,
IdentityResolveInput: (v) => v,
AssetLocationCreate: (v) => v,
ReleaseIdentityAssign: (v) => ({ ...v, external: undefined, candidateIndex: undefined }),
CacheSettings: (v) => ({ ...v, hardThresholdPercent: 90, resumePercent: 85, cleanupTargetPercent: 80 }),
GlobalSettings: () => C.DEFAULT_SETTINGS,
EngineStats: (v) => ({ ...v, byStatus: { pending: 1 } }),
+9
View File
@@ -29,6 +29,9 @@ PostgreSQL 访问层:Kysely 客户端与连接池、表类型(`src/types.ts`)、
| `0012_organization` | organization_record(change_seq 序列 + 触发器)、organization_file_result;回填 asset_operation.organization_record_id |
| `0013_tasks_tables` | 并入 @umt/tasks 自有表:tasks_run_meta、tasks_fact、tasks_op_ledger(if not exists,可接管 W1 旧库) |
| `0014_agent_run_event` | agent_run_event:Agent 运行事件,主键 (run_id, seq),删 run 级联 |
| `0018_sources_snapshot` | site.adapter 增加 `api`;source_release.resolution_locked(人工锁);site_snapshot(快照/补漏/实时补查的覆盖范围) |
| `0019_identity_pool` | identity_candidate(待识别池)、resolution_attempt(识别尝试)、identity_redirect(合并跳转) |
| `0020_metadata_fields` | metadata_field(来源候选字段)、field_override(用户覆盖与锁定) |
## 表清单(45 张)
@@ -59,6 +62,11 @@ PostgreSQL 访问层:Kysely 客户端与连接池、表类型(`src/types.ts`)、
| `title_alias` | 原名/译名/简称/站点原标题;`normalized` 上建 gin trgm 索引做模糊召回 |
| `external_identity` | 本地对象 ↔ 外部 ID;**`(provider, entity_type, external_id)` 对 accepted 部分唯一**,同一本地对象每个 provider+type 只有一个 accepted;候选(proposed/conflicted)可并存;404/超时只记 `verify_status` |
| `identity_revision` | 身份每次变更的修订历史,`(external_identity_id, revision)` 唯一,只追加 |
| `identity_candidate` | 待识别池:每条 source_release 至多一条;候选只存 JSON(不建正式作品),保存原因/证据版本/复查时间(有界退避) |
| `resolution_attempt` | 每次识别尝试的结果与证据版本,只追加 |
| `identity_redirect` | 作品合并后的旧 ID 跳转,`from_entity_id` 唯一 |
| `metadata_field` | 各来源的候选字段值(来源/语言/时间/来源修订);刷新只写这里 |
| `field_override` | 用户字段覆盖与锁定,优先于自动来源;目标 + 字段唯一 |
### 来源
| 表 | 用途与关键不变量 |
@@ -68,6 +76,7 @@ PostgreSQL 访问层:Kysely 客户端与连接池、表类型(`src/types.ts`)、
| `feed_source` | RSS/列表/查询入口;真实 URL(含 passkey)只在 `url_secret_ref`,对外只给 `url_display`;按 `next_fetch_at` 调度 |
| `source_release` | 站点资源;**`(site_id, site_release_id)` 唯一**,多 Feed 观察汇到同一行;`source_published_at`/`first_seen_at` 分开 |
| `observation` | 一次原始观察;**`(feed_id, guid)` 唯一**,`content_digest` 变化即条目更新 |
| `site_snapshot` | 一次列表快照 / 补漏 / 实时补查:页码、条目数、状态与错误(新鲜度来源) |
### 文件
| 表 | 用途与关键不变量 |
@@ -0,0 +1,51 @@
import type { Kysely } from "kysely";
import { dropTables, entityTable, execDdl, oneOf } from "../src/migration-helpers.ts";
// W2-B 来源补充(REQ §4.5、§4.6、§7.5):
// - site.adapter 增加 api(MP 专用 API spider:Haidan/HDDolby/RousiPro/…,见 adapters/sites/spiders);
// - source_release.resolution_locked:人工确认的身份关联,自动识别不得改写;
// - site_snapshot:每次列表快照/实时补查/补漏的覆盖范围与新鲜度(不把一次分页伪称全站快照)。
// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型
export async function up(db: Kysely<any>): Promise<void> {
await execDdl(
db,
`
alter table site drop constraint site_adapter_check;
alter table site add constraint site_adapter_check ${oneOf("adapter", ["nexusphp", "mteam_api", "api", "generic_rss"])};
alter table source_release add column resolution_locked boolean not null default false;
${entityTable(
"site_snapshot",
`
site_id uuid not null references site (id) on delete cascade,
kind text not null ${oneOf("kind", ["periodic", "backfill", "live_search", "manual"])},
query text,
filters jsonb not null default '{}'::jsonb,
pages jsonb not null default '[]'::jsonb,
item_count integer not null default 0,
new_count integer not null default 0,
status text not null default 'running' ${oneOf("status", ["running", "ok", "partial", "failed"])},
error jsonb,
definition_version text,
started_at timestamptz not null default now(),
finished_at timestamptz`,
)}
create index site_snapshot_site_idx on site_snapshot (site_id, started_at desc);
`,
);
}
// biome-ignore lint/suspicious/noExplicitAny: 同上
export async function down(db: Kysely<any>): Promise<void> {
await execDdl(
db,
`
${dropTables(["site_snapshot"])}
alter table source_release drop column if exists resolution_locked;
alter table site drop constraint site_adapter_check;
alter table site add constraint site_adapter_check ${oneOf("adapter", ["nexusphp", "mteam_api", "generic_rss"])};
`,
);
}
@@ -0,0 +1,74 @@
import type { Kysely } from "kysely";
import { dropTables, entityTable, execDdl, oneOf } from "../src/migration-helpers.ts";
// W2-B 待识别池与身份治理(REQ §7.2、§7.5):
// - identity_candidate:每条 source_release 至多一条待识别记录,候选只存 JSON,不逐个建为正式作品;
// 失败保存原因、证据版本与下一次复查时间(有界退避),新证据到来时重试;
// - resolution_attempt:每次识别尝试的结果与证据版本(只追加);
// - identity_redirect:作品合并后的旧 ID 跳转历史(与 media_entity.merged_into_id 一致,可修订)。
const ACTOR = ["user", "agent", "system", "api_key"] as const;
// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型
export async function up(db: Kysely<any>): Promise<void> {
await execDdl(
db,
`
${entityTable(
"identity_candidate",
`
source_release_id uuid not null references source_release (id) on delete cascade,
status text not null default 'pending' ${oneOf("status", ["pending", "resolved", "dismissed", "archived"])},
reason text not null ${oneOf("reason", ["ambiguous", "no_match", "proposed", "lookup_failed"])},
query jsonb not null default '{}'::jsonb,
candidates jsonb not null default '[]'::jsonb,
evidence_version text not null,
attempts integer not null default 0,
last_attempt_at timestamptz,
next_review_at timestamptz,
decision_id uuid references decision (id) on delete set null,
resolved_media_entity_id uuid references media_entity (id) on delete set null,
resolved_by text,
resolved_at timestamptz,
constraint identity_candidate_release_key unique (source_release_id)`,
)}
create index identity_candidate_review_idx on identity_candidate (next_review_at) where status = 'pending';
create index identity_candidate_status_idx on identity_candidate (status, updated_at desc);
${entityTable(
"resolution_attempt",
`
source_release_id uuid not null references source_release (id) on delete cascade,
observation_id uuid references observation (id) on delete set null,
outcome text not null ${oneOf("outcome", ["auto", "proposed", "ambiguous", "no_match", "lookup_failed", "manual", "skipped_locked"])},
reason text,
evidence_version text not null,
candidate_count integer not null default 0,
media_entity_id uuid references media_entity (id) on delete set null,
best jsonb,
error jsonb,
actor_type text not null default 'system' ${oneOf("actor_type", ACTOR)},
actor_id text`,
)}
create index resolution_attempt_release_idx on resolution_attempt (source_release_id, created_at desc);
${entityTable(
"identity_redirect",
`
from_entity_id uuid not null references media_entity (id) on delete cascade,
to_entity_id uuid not null references media_entity (id) on delete cascade,
reason text,
actor_type text not null ${oneOf("actor_type", ACTOR)},
actor_id text,
constraint identity_redirect_from_key unique (from_entity_id),
constraint identity_redirect_not_self check (from_entity_id <> to_entity_id)`,
)}
create index identity_redirect_to_idx on identity_redirect (to_entity_id);
`,
);
}
// biome-ignore lint/suspicious/noExplicitAny: 同上
export async function down(db: Kysely<any>): Promise<void> {
await execDdl(db, dropTables(["identity_redirect", "resolution_attempt", "identity_candidate"]));
}
@@ -0,0 +1,54 @@
import type { Kysely } from "kysely";
import { dropTables, entityTable, execDdl, oneOf } from "../src/migration-helpers.ts";
// W2-B 字段级元数据协调(REQ §7.3):
// - metadata_field:各来源的候选字段值(来源/语言/时间/来源修订),刷新只写这里;
// - field_override:用户(或 Agent 生成并确认)的字段覆盖与锁定,优先于任何自动来源;
// 最终展示值投影到 media_entity.metadata / content_unit.metadata。路径绑定不受元数据刷新影响。
const SOURCE = ["tmdb", "user", "agent", "nfo", "site", "import"] as const;
const target = "constraint {t}_target_check check (num_nonnulls(media_entity_id, content_unit_id) = 1)";
// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型
export async function up(db: Kysely<any>): Promise<void> {
await execDdl(
db,
`
${entityTable(
"metadata_field",
`
media_entity_id uuid references media_entity (id) on delete cascade,
content_unit_id uuid references content_unit (id) on delete cascade,
field text not null,
value jsonb,
language text,
source text not null ${oneOf("source", SOURCE)},
source_revision text,
fetched_at timestamptz not null default now(),
${target.replace("{t}", "metadata_field")}`,
)}
create unique index metadata_field_key on metadata_field (media_entity_id, content_unit_id, field, source, language) nulls not distinct;
${entityTable(
"field_override",
`
media_entity_id uuid references media_entity (id) on delete cascade,
content_unit_id uuid references content_unit (id) on delete cascade,
field text not null,
value jsonb,
locked boolean not null default true,
source text not null default 'user' ${oneOf("source", SOURCE)},
reason text,
actor_type text not null ${oneOf("actor_type", ["user", "agent", "system", "api_key"])},
actor_id text,
${target.replace("{t}", "field_override")}`,
)}
create unique index field_override_key on field_override (media_entity_id, content_unit_id, field) nulls not distinct;
`,
);
}
// biome-ignore lint/suspicious/noExplicitAny: 同上
export async function down(db: Kysely<any>): Promise<void> {
await execDdl(db, dropTables(["field_override", "metadata_field"]));
}
+137 -1
View File
@@ -117,6 +117,11 @@ export type ExternalIdentityVerifyStatus = "unverified" | "ok" | "not_found" | "
export type FeedSourcePurpose = "movie" | "series" | "variety" | "promotion" | "search" | "generic";
export type FeedSourceKind = "rss" | "atom" | "list" | "query";
export type FeedSourceHealthStatus = "unknown" | "healthy" | "degraded" | "failing" | "circuit_open" | "disabled";
export type FieldOverrideSource = "tmdb" | "user" | "agent" | "nfo" | "site" | "import";
export type FieldOverrideActorType = "user" | "agent" | "system" | "api_key";
export type IdentityCandidateStatus = "pending" | "resolved" | "dismissed" | "archived";
export type IdentityCandidateReason = "ambiguous" | "no_match" | "proposed" | "lookup_failed";
export type IdentityRedirectActorType = "user" | "agent" | "system" | "api_key";
export type IdentityRevisionAction =
| "created"
| "status_changed"
@@ -136,6 +141,7 @@ export type MediaEntityKind = "movie" | "series" | "variety" | "collection" | "o
export type MediaEntityStatus = "provisional" | "active" | "merged" | "archived";
export type MediaServerInstanceKind = "emby" | "jellyfin";
export type MediaServerInstanceHealthStatus = "unknown" | "healthy" | "degraded" | "offline";
export type MetadataFieldSource = "tmdb" | "user" | "agent" | "nfo" | "site" | "import";
export type ObservationStatus = "new" | "linked" | "ignored" | "error";
export type OrganizationFileResultRole = "main" | "subtitle" | "audio" | "bluray_dir" | "nfo" | "image" | "other";
export type OrganizationFileResultMode = "copy" | "move" | "rename" | "relocate" | "hardlink";
@@ -158,6 +164,15 @@ export type PreferenceTagStatus = "active" | "disabled" | "deleted";
export type ReconcileRunKind = "identity" | "metadata" | "asset" | "acquisition" | "server";
export type ReconcileRunMode = "normal" | "read_only";
export type ReconcileRunStatus = "queued" | "running" | "succeeded" | "partial" | "failed" | "cancelled";
export type ResolutionAttemptOutcome =
| "auto"
| "proposed"
| "ambiguous"
| "no_match"
| "lookup_failed"
| "manual"
| "skipped_locked";
export type ResolutionAttemptActorType = "user" | "agent" | "system" | "api_key";
export type SecretPurpose =
| "site_cookie"
| "site_passkey"
@@ -171,11 +186,13 @@ export type SecretPurpose =
| "other";
export type ServerMappingItemType = "movie" | "series" | "season" | "episode" | "other";
export type ServerMappingMatchStatus = "matched" | "ambiguous" | "unmatched" | "stale";
export type SiteAdapter = "nexusphp" | "mteam_api" | "generic_rss";
export type SiteAdapter = "nexusphp" | "mteam_api" | "api" | "generic_rss";
export type SiteHealthStatus = "unknown" | "healthy" | "degraded" | "failing" | "disabled";
export type SiteCircuitState = "closed" | "open" | "half_open";
export type SiteAccountAuthKind = "cookie" | "passkey" | "api_key" | "password";
export type SiteAccountStatus = "unknown" | "active" | "invalid" | "expired";
export type SiteSnapshotKind = "periodic" | "backfill" | "live_search" | "manual";
export type SiteSnapshotStatus = "running" | "ok" | "partial" | "failed";
export type SourceReleaseResolutionStatus = "unresolved" | "resolved" | "ambiguous" | "unidentifiable" | "rejected";
export type SourceReleaseAvailability = "active" | "site_disabled" | "expired" | "unknown";
export type SubscriptionMediaKind = "movie" | "series" | "variety";
@@ -573,6 +590,51 @@ export interface FeedSourceTable {
updated_at: GeneratedTimestamptz;
}
export interface FieldOverrideTable {
id: Generated<string>;
media_entity_id: string | null;
content_unit_id: string | null;
field: string;
value: Json | null;
locked: Generated<boolean>;
source: Generated<FieldOverrideSource>;
reason: string | null;
actor_type: FieldOverrideActorType;
actor_id: string | null;
created_at: GeneratedTimestamptz;
updated_at: GeneratedTimestamptz;
}
export interface IdentityCandidateTable {
id: Generated<string>;
source_release_id: string;
status: Generated<IdentityCandidateStatus>;
reason: IdentityCandidateReason;
query: Generated<Json>;
candidates: Generated<Json>;
evidence_version: string;
attempts: Generated<number>;
last_attempt_at: Timestamptz | null;
next_review_at: Timestamptz | null;
decision_id: string | null;
resolved_media_entity_id: string | null;
resolved_by: string | null;
resolved_at: Timestamptz | null;
created_at: GeneratedTimestamptz;
updated_at: GeneratedTimestamptz;
}
export interface IdentityRedirectTable {
id: Generated<string>;
from_entity_id: string;
to_entity_id: string;
reason: string | null;
actor_type: IdentityRedirectActorType;
actor_id: string | null;
created_at: GeneratedTimestamptz;
updated_at: GeneratedTimestamptz;
}
export interface IdentityRevisionTable {
id: Generated<string>;
external_identity_id: string;
@@ -665,6 +727,20 @@ export interface MediaServerInstanceTable {
updated_at: GeneratedTimestamptz;
}
export interface MetadataFieldTable {
id: Generated<string>;
media_entity_id: string | null;
content_unit_id: string | null;
field: string;
value: Json | null;
language: string | null;
source: MetadataFieldSource;
source_revision: string | null;
fetched_at: GeneratedTimestamptz;
created_at: GeneratedTimestamptz;
updated_at: GeneratedTimestamptz;
}
export interface ObservationTable {
id: Generated<string>;
feed_id: string;
@@ -855,6 +931,23 @@ export interface ReconcileRunTable {
updated_at: GeneratedTimestamptz;
}
export interface ResolutionAttemptTable {
id: Generated<string>;
source_release_id: string;
observation_id: string | null;
outcome: ResolutionAttemptOutcome;
reason: string | null;
evidence_version: string;
candidate_count: Generated<number>;
media_entity_id: string | null;
best: Json | null;
error: Json | null;
actor_type: Generated<ResolutionAttemptActorType>;
actor_id: string | null;
created_at: GeneratedTimestamptz;
updated_at: GeneratedTimestamptz;
}
export interface SecretTable {
id: Generated<string>;
name: string;
@@ -929,6 +1022,24 @@ export interface SiteAccountTable {
updated_at: GeneratedTimestamptz;
}
export interface SiteSnapshotTable {
id: Generated<string>;
site_id: string;
kind: SiteSnapshotKind;
query: string | null;
filters: Generated<Json>;
pages: Generated<Json>;
item_count: Generated<number>;
new_count: Generated<number>;
status: Generated<SiteSnapshotStatus>;
error: Json | null;
definition_version: string | null;
started_at: GeneratedTimestamptz;
finished_at: Timestamptz | null;
created_at: GeneratedTimestamptz;
updated_at: GeneratedTimestamptz;
}
export interface SourceReleaseTable {
id: Generated<string>;
site_id: string;
@@ -959,6 +1070,7 @@ export interface SourceReleaseTable {
raw: Generated<Json>;
created_at: GeneratedTimestamptz;
updated_at: GeneratedTimestamptz;
resolution_locked: Generated<boolean>;
}
export interface SseEventTable {
@@ -1082,12 +1194,16 @@ export interface DB {
downloader_instance: DownloaderInstanceTable;
external_identity: ExternalIdentityTable;
feed_source: FeedSourceTable;
field_override: FieldOverrideTable;
identity_candidate: IdentityCandidateTable;
identity_redirect: IdentityRedirectTable;
identity_revision: IdentityRevisionTable;
library_root: LibraryRootTable;
login_attempt: LoginAttemptTable;
media_asset: MediaAssetTable;
media_entity: MediaEntityTable;
media_server_instance: MediaServerInstanceTable;
metadata_field: MetadataFieldTable;
observation: ObservationTable;
organization_file_result: OrganizationFileResultTable;
organization_record: OrganizationRecordTable;
@@ -1098,11 +1214,13 @@ export interface DB {
preference_suppression: PreferenceSuppressionTable;
preference_tag: PreferenceTagTable;
reconcile_run: ReconcileRunTable;
resolution_attempt: ResolutionAttemptTable;
secret: SecretTable;
server_mapping: ServerMappingTable;
settings: SettingsTable;
site: SiteTable;
site_account: SiteAccountTable;
site_snapshot: SiteSnapshotTable;
source_release: SourceReleaseTable;
sse_event: SseEventTable;
subscription: SubscriptionTable;
@@ -1173,6 +1291,15 @@ export type ExternalIdentityUpdate = Updateable<ExternalIdentityTable>;
export type FeedSourceRow = Selectable<FeedSourceTable>;
export type NewFeedSource = Insertable<FeedSourceTable>;
export type FeedSourceUpdate = Updateable<FeedSourceTable>;
export type FieldOverrideRow = Selectable<FieldOverrideTable>;
export type NewFieldOverride = Insertable<FieldOverrideTable>;
export type FieldOverrideUpdate = Updateable<FieldOverrideTable>;
export type IdentityCandidateRow = Selectable<IdentityCandidateTable>;
export type NewIdentityCandidate = Insertable<IdentityCandidateTable>;
export type IdentityCandidateUpdate = Updateable<IdentityCandidateTable>;
export type IdentityRedirectRow = Selectable<IdentityRedirectTable>;
export type NewIdentityRedirect = Insertable<IdentityRedirectTable>;
export type IdentityRedirectUpdate = Updateable<IdentityRedirectTable>;
export type IdentityRevisionRow = Selectable<IdentityRevisionTable>;
export type NewIdentityRevision = Insertable<IdentityRevisionTable>;
export type IdentityRevisionUpdate = Updateable<IdentityRevisionTable>;
@@ -1191,6 +1318,9 @@ export type MediaEntityUpdate = Updateable<MediaEntityTable>;
export type MediaServerInstanceRow = Selectable<MediaServerInstanceTable>;
export type NewMediaServerInstance = Insertable<MediaServerInstanceTable>;
export type MediaServerInstanceUpdate = Updateable<MediaServerInstanceTable>;
export type MetadataFieldRow = Selectable<MetadataFieldTable>;
export type NewMetadataField = Insertable<MetadataFieldTable>;
export type MetadataFieldUpdate = Updateable<MetadataFieldTable>;
export type ObservationRow = Selectable<ObservationTable>;
export type NewObservation = Insertable<ObservationTable>;
export type ObservationUpdate = Updateable<ObservationTable>;
@@ -1221,6 +1351,9 @@ export type PreferenceTagUpdate = Updateable<PreferenceTagTable>;
export type ReconcileRunRow = Selectable<ReconcileRunTable>;
export type NewReconcileRun = Insertable<ReconcileRunTable>;
export type ReconcileRunUpdate = Updateable<ReconcileRunTable>;
export type ResolutionAttemptRow = Selectable<ResolutionAttemptTable>;
export type NewResolutionAttempt = Insertable<ResolutionAttemptTable>;
export type ResolutionAttemptUpdate = Updateable<ResolutionAttemptTable>;
export type SecretRow = Selectable<SecretTable>;
export type NewSecret = Insertable<SecretTable>;
export type SecretUpdate = Updateable<SecretTable>;
@@ -1236,6 +1369,9 @@ export type SiteUpdate = Updateable<SiteTable>;
export type SiteAccountRow = Selectable<SiteAccountTable>;
export type NewSiteAccount = Insertable<SiteAccountTable>;
export type SiteAccountUpdate = Updateable<SiteAccountTable>;
export type SiteSnapshotRow = Selectable<SiteSnapshotTable>;
export type NewSiteSnapshot = Insertable<SiteSnapshotTable>;
export type SiteSnapshotUpdate = Updateable<SiteSnapshotTable>;
export type SourceReleaseRow = Selectable<SourceReleaseTable>;
export type NewSourceRelease = Insertable<SourceReleaseTable>;
export type SourceReleaseUpdate = Updateable<SourceReleaseTable>;
+7
View File
@@ -68,6 +68,13 @@ const EXPECTED_TABLES = [
"tasks_fact",
"tasks_op_ledger",
"agent_run_event",
// W2-B 来源/身份/元数据(0018–0020)
"site_snapshot",
"identity_candidate",
"resolution_attempt",
"identity_redirect",
"metadata_field",
"field_override",
// 登录门禁(0015)与 SSE 多进程总线(0016)
"admin_user",
"admin_session",
+1
View File
@@ -258,6 +258,7 @@ describe("SiteRelease ↔ source_release", () => {
resolutionStatus: "unresolved",
resolutionEvidence: {},
availability: "active",
resolutionLocked: false,
};
expect(C.SourceRelease.safeParse(row).success).toBe(true);
const { downloadUrl: _d, ...expected } = site;