feat(server): W2-B sources, identity resolution, metadata and site workbench
- Sites / accounts / feeds: registry view (/site-definitions), site CRUD with
enable/disable (history stays queryable, marked site_disabled), accounts and
feed URLs stored as secrets (only secret_ref / redacted urlDisplay exposed),
feed enable/disable/reset/fetch; one shared rate limiter + adapter per site.
- Tasks: ingestFeed (FeedFetchState/FeedCursor on feed_source, observation
feed+guid upsert with digest-change updates, source_release site+id
aggregation across feeds, auth failure -> account invalid + decision, no
retry, gap -> feed.gap_suspected -> budgeted backfill snapshot),
resolveObservation (parseRelease -> local recall via title_alias pg_trgm +
accepted identities -> TMDB -> resolveIdentity; auto binds entity /
provisional units / identities / aliases, otherwise identity_candidate pool
with evidence version and bounded review backoff; manual lock respected),
refreshMetadata (metadata_field + field_override, user locks win, paths
untouched), snapshotSite (site_snapshot coverage records).
- Events: feed.due, feed.gap_suspected, release.observed (self-consumed),
release.resolved, decision.created/updated; scheduler in worker start().
- Identity API: entity detail with redirects, manual mapping correction with
identity_revision, merge with identity_redirect, pool list/assign/dismiss/
retry, content unit numbering lock, aliases, external identities.
- Decisions: list/summary (by entity and kind)/detail/resolve; identity
options executed directly.
- Workbench: /sites/{id}/releases (snapshot first, coalesced live search with
backoff, origin + freshness) and /sites/{id}/releases/{rid} (volatile field
recheck times, refresh keeps data on failure).
- Capabilities: catalog.lookup, release.search, release.inspect,
identity.resolve.
- Migrations 0018-0020: site.adapter api, source_release.resolution_locked,
site_snapshot, identity_candidate, resolution_attempt, identity_redirect,
metadata_field, field_override.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
829ca4f7ac
commit
a24bda952b
39 files changed
+6861
-15
No files matched your search
@@ -0,0 +1,187 @@
|
||||
import type { CapabilityRegistry } from "@umt/agent";
|
||||
import { capabilitySpec, type ExternalEntityType, type SourceRelease } from "@umt/contracts";
|
||||
import { sql } from "@umt/db";
|
||||
import { normalizeTitle } from "@umt/domain";
|
||||
import { toApi, toApiList } from "./rows.ts";
|
||||
import type { SourcesRuntime } from "./runtime.ts";
|
||||
import { assignReleaseIdentity } from "./service/identity-admin.ts";
|
||||
import { resolveRelease } from "./service/resolve.ts";
|
||||
import { refreshReleaseDetail } from "./service/snapshot.ts";
|
||||
import { liveSearch } from "./service/workbench.ts";
|
||||
|
||||
// 本模块领域的能力处理器(规格一律取 contracts):catalog.lookup / release.search / release.inspect / identity.resolve。
|
||||
// identity.resolve 属 propose 类,授权回调由 W2-D 接入前会被注册中心拒绝(安全默认)。
|
||||
|
||||
export function registerSourcesCapabilities(registry: CapabilityRegistry, rt: () => SourcesRuntime): void {
|
||||
registry.register(capabilitySpec("catalog.lookup"), async (input) => {
|
||||
const { db } = rt().ctx;
|
||||
const local: { entity: unknown; score: number; matchedAlias: string | null }[] = [];
|
||||
if (input.externalId) {
|
||||
const rows = await db
|
||||
.selectFrom("external_identity as i")
|
||||
.innerJoin("media_entity as e", "e.id", "i.media_entity_id")
|
||||
.selectAll("e")
|
||||
.where("i.provider", "=", input.externalId.provider)
|
||||
.where("i.entity_type", "=", input.externalId.entityType)
|
||||
.where("i.external_id", "=", input.externalId.externalId)
|
||||
.where("i.status", "=", "accepted")
|
||||
.execute();
|
||||
for (const e of rows) local.push({ entity: toApi(e), score: 1, matchedAlias: null });
|
||||
}
|
||||
if (input.query) {
|
||||
const n = normalizeTitle(input.query);
|
||||
const rows = await sql<Record<string, unknown> & { score: number; matched: string }>`
|
||||
select distinct on (e.id) e.*, greatest(similarity(a.normalized, ${n}), case when a.normalized = ${n} then 1 else 0 end) as score,
|
||||
a.alias as matched
|
||||
from title_alias a join media_entity e on e.id = a.media_entity_id
|
||||
where e.status <> 'merged' and (a.normalized = ${n} or similarity(a.normalized, ${n}) >= 0.3)
|
||||
${input.kind ? sql`and e.kind = ${input.kind}` : sql``}
|
||||
${input.year ? sql`and (e.year is null or abs(e.year - ${input.year}) <= 1)` : sql``}
|
||||
order by e.id, score desc`.execute(db);
|
||||
for (const r of rows.rows.sort((a, b) => b.score - a.score).slice(0, input.limit)) {
|
||||
const { score, matched, ...e } = r;
|
||||
if (!local.some((l) => (l.entity as { id: string }).id === e.id)) {
|
||||
local.push({ entity: toApi(e), score: Number(score), matchedAlias: matched });
|
||||
}
|
||||
}
|
||||
}
|
||||
const remote: {
|
||||
provider: string;
|
||||
entityType: ExternalEntityType;
|
||||
externalId: string;
|
||||
title: string;
|
||||
year: number | null;
|
||||
}[] = [];
|
||||
const tmdb = rt().tmdb();
|
||||
if (input.includeRemote && input.query && tmdb) {
|
||||
for (const m of (await tmdb.searchAll(input.query, input.year ? { year: input.year } : {})).slice(
|
||||
0,
|
||||
input.limit,
|
||||
)) {
|
||||
remote.push({
|
||||
provider: "tmdb",
|
||||
entityType: m.entityType,
|
||||
externalId: m.externalId,
|
||||
title: m.title,
|
||||
year: m.year,
|
||||
});
|
||||
}
|
||||
}
|
||||
return { local: local as never, remote };
|
||||
});
|
||||
|
||||
registry.register(capabilitySpec("release.search"), async (input) => {
|
||||
const r = rt();
|
||||
const { db } = r.ctx;
|
||||
let q = db.selectFrom("source_release").selectAll();
|
||||
if (input.siteIds?.length) q = q.where("site_id", "in", input.siteIds);
|
||||
if (input.mediaEntityId) q = q.where("media_entity_id", "=", input.mediaEntityId);
|
||||
const text = input.query;
|
||||
if (text)
|
||||
q = q.where((eb) => eb.or([eb("title", "ilike", `%${text}%`), sql<boolean>`similarity(title, ${text}) >= 0.3`]));
|
||||
const run = () => q.orderBy("first_seen_at", "desc").limit(input.limit).execute();
|
||||
let rows = await run();
|
||||
const sites = await db
|
||||
.selectFrom("site")
|
||||
.selectAll()
|
||||
.where("enabled", "=", true)
|
||||
.$if(!!input.siteIds?.length, (b) => b.where("id", "in", input.siteIds!))
|
||||
.execute();
|
||||
const coverage: { siteId: string; fromSnapshot: boolean; live: boolean; error: string | null }[] = [];
|
||||
const needLive = input.allowLiveSearch && !!text && rows.length === 0;
|
||||
for (const s of sites) {
|
||||
if (needLive) {
|
||||
const res = await liveSearch(r, s, text!);
|
||||
coverage.push({ siteId: s.id, fromSnapshot: false, live: res.error === null, error: res.error });
|
||||
} else coverage.push({ siteId: s.id, fromSnapshot: true, live: false, error: null });
|
||||
}
|
||||
if (needLive) rows = await run();
|
||||
return { items: toApiList<SourceRelease>(rows), coverage };
|
||||
});
|
||||
|
||||
registry.register(capabilitySpec("release.inspect"), async (input) => {
|
||||
const r = rt();
|
||||
if (input.refreshDetail) await refreshReleaseDetail(r, input.sourceReleaseId);
|
||||
const row = await r.ctx.db
|
||||
.selectFrom("source_release")
|
||||
.selectAll()
|
||||
.where("id", "=", input.sourceReleaseId)
|
||||
.executeTakeFirst();
|
||||
if (!row) throw new Error("来源资源不存在");
|
||||
// 库存(media_asset)归 W2-C,这里只按作品只读查询
|
||||
const inventory = row.media_entity_id
|
||||
? await r.ctx.db
|
||||
.selectFrom("media_asset")
|
||||
.selectAll()
|
||||
.where("media_entity_id", "=", row.media_entity_id)
|
||||
.execute()
|
||||
: [];
|
||||
return { release: toApi(row), inventory: toApiList(inventory) } as never;
|
||||
});
|
||||
|
||||
registry.register(capabilitySpec("identity.resolve"), async (input, ctx) => {
|
||||
const r = rt();
|
||||
let srId = input.sourceReleaseId ?? null;
|
||||
if (!srId && input.observationId) {
|
||||
const o = await r.ctx.db
|
||||
.selectFrom("observation")
|
||||
.select(["source_release_id"])
|
||||
.where("id", "=", input.observationId)
|
||||
.executeTakeFirst();
|
||||
srId = o?.source_release_id ?? null;
|
||||
}
|
||||
if (!srId) throw new Error("找不到来源资源");
|
||||
const actor = { type: "agent" as const, id: ctx.actorId };
|
||||
const c = input.candidate;
|
||||
if (c?.mediaEntityId || (c?.provider && c.entityType && c.externalId)) {
|
||||
// Agent 的提议不加人工锁,后续人工仍可改
|
||||
const res = await assignReleaseIdentity(
|
||||
r,
|
||||
srId,
|
||||
c.mediaEntityId
|
||||
? { mediaEntityId: c.mediaEntityId, lock: false }
|
||||
: {
|
||||
external: {
|
||||
provider: c.provider!,
|
||||
entityType: c.entityType as "movie" | "tv",
|
||||
externalId: c.externalId!,
|
||||
},
|
||||
lock: false,
|
||||
},
|
||||
actor,
|
||||
);
|
||||
return {
|
||||
status: "resolved" as const,
|
||||
mediaEntityId: res.mediaEntityId,
|
||||
identities: await identities(r, res.mediaEntityId),
|
||||
decisionId: null,
|
||||
};
|
||||
}
|
||||
const res = await resolveRelease(r, { sourceReleaseId: srId, force: true });
|
||||
const status: "resolved" | "proposed" | "ambiguous" | "unidentifiable" =
|
||||
res.outcome === "auto" || res.outcome === "skipped" || res.outcome === "skipped_locked"
|
||||
? res.mediaEntityId
|
||||
? "resolved"
|
||||
: "unidentifiable"
|
||||
: res.outcome === "proposed"
|
||||
? "proposed"
|
||||
: res.outcome === "ambiguous"
|
||||
? "ambiguous"
|
||||
: "unidentifiable";
|
||||
return {
|
||||
status,
|
||||
mediaEntityId: res.mediaEntityId,
|
||||
identities: res.mediaEntityId ? await identities(r, res.mediaEntityId) : [],
|
||||
decisionId: res.decisionId,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function identities(r: SourcesRuntime, entityId: string) {
|
||||
const rows = await r.ctx.db
|
||||
.selectFrom("external_identity")
|
||||
.selectAll()
|
||||
.where("media_entity_id", "=", entityId)
|
||||
.execute();
|
||||
return toApiList(rows) as never[];
|
||||
}
|
||||
@@ -0,0 +1,303 @@
|
||||
import {
|
||||
ContentUnit,
|
||||
ContentUnitUpdate,
|
||||
CursorQuery,
|
||||
cursorPage,
|
||||
Decision,
|
||||
DecisionKind,
|
||||
DecisionResolveInput,
|
||||
DecisionStatus,
|
||||
ExternalIdentity,
|
||||
ExternalIdentityCreate,
|
||||
ExternalIdentityUpdate,
|
||||
FeedHealth,
|
||||
FeedSource,
|
||||
FeedSourceCreate,
|
||||
FeedSourceUpdate,
|
||||
Id,
|
||||
IdentityRevision,
|
||||
IdentityStatus,
|
||||
IdParam,
|
||||
MediaEntity,
|
||||
MediaEntityCreate,
|
||||
MediaEntityMerge,
|
||||
MediaEntityStatus,
|
||||
MediaEntityUpdate,
|
||||
MediaKind,
|
||||
Observation,
|
||||
ObservationStatus,
|
||||
OkResult,
|
||||
ResolutionStatus,
|
||||
Site,
|
||||
SiteAccount,
|
||||
SiteAccountCreate,
|
||||
SiteAccountUpdate,
|
||||
SiteCreate,
|
||||
SiteUpdate,
|
||||
SourceRelease,
|
||||
TitleAlias,
|
||||
TitleAliasCreate,
|
||||
} from "@umt/contracts";
|
||||
import { z } from "zod";
|
||||
|
||||
// 与 contracts Api(crud())同路径、同 scope 的端点定义,但保留具体 schema 类型供路由推断入参。
|
||||
// scope:读 = read;写 = admin(站点/Feed 配置)或 organize.execute(身份纠错、待处理)。
|
||||
|
||||
const list = <R extends z.ZodType, F extends z.ZodRawShape>(
|
||||
path: string,
|
||||
summary: string,
|
||||
row: R,
|
||||
filters: F,
|
||||
tags: string[],
|
||||
) => ({
|
||||
method: "get" as const,
|
||||
path,
|
||||
scope: "read" as const,
|
||||
summary,
|
||||
tags,
|
||||
query: CursorQuery.extend(filters),
|
||||
response: cursorPage(row),
|
||||
});
|
||||
const get = <R extends z.ZodType>(path: string, summary: string, row: R, tags: string[]) => ({
|
||||
method: "get" as const,
|
||||
path,
|
||||
scope: "read" as const,
|
||||
summary,
|
||||
tags,
|
||||
params: IdParam,
|
||||
response: row,
|
||||
});
|
||||
const write = <M extends "post" | "patch" | "delete", B extends z.ZodType | undefined, R extends z.ZodType>(
|
||||
method: M,
|
||||
path: string,
|
||||
scope: "admin" | "organize.execute",
|
||||
summary: string,
|
||||
body: B,
|
||||
response: R,
|
||||
tags: string[],
|
||||
withId = true,
|
||||
) => ({
|
||||
method,
|
||||
path,
|
||||
scope,
|
||||
summary,
|
||||
tags,
|
||||
...(withId ? { params: IdParam } : {}),
|
||||
body: body as B,
|
||||
response,
|
||||
});
|
||||
|
||||
const TS = ["sites"];
|
||||
const TF = ["feed-sources"];
|
||||
const TR = ["source-releases"];
|
||||
const TE = ["media-entities"];
|
||||
const TI = ["identity"];
|
||||
const TD = ["decisions"];
|
||||
|
||||
export const D = {
|
||||
sites: {
|
||||
list: list("/sites", "站点列表", Site, {}, TS),
|
||||
get: get("/sites/{id}", "站点详情", Site, TS),
|
||||
create: {
|
||||
method: "post" as const,
|
||||
path: "/sites",
|
||||
scope: "admin" as const,
|
||||
summary: "登记站点",
|
||||
tags: TS,
|
||||
body: SiteCreate,
|
||||
response: Site,
|
||||
},
|
||||
update: write("patch", "/sites/{id}", "admin", "修改站点(启停、快照周期、限流)", SiteUpdate, Site, TS),
|
||||
},
|
||||
siteAccounts: {
|
||||
list: list("/site-accounts", "站点账号列表", SiteAccount, { siteId: Id.optional() }, TS),
|
||||
get: get("/site-accounts/{id}", "站点账号详情", SiteAccount, TS),
|
||||
create: {
|
||||
method: "post" as const,
|
||||
path: "/site-accounts",
|
||||
scope: "admin" as const,
|
||||
summary: "新建站点账号(凭证只写,存为 secret)",
|
||||
tags: TS,
|
||||
body: SiteAccountCreate,
|
||||
response: SiteAccount,
|
||||
},
|
||||
update: write(
|
||||
"patch",
|
||||
"/site-accounts/{id}",
|
||||
"admin",
|
||||
"修改站点账号 / 轮换凭证",
|
||||
SiteAccountUpdate,
|
||||
SiteAccount,
|
||||
TS,
|
||||
),
|
||||
},
|
||||
feedSources: {
|
||||
list: list("/feed-sources", "Feed 列表", FeedSource, { siteId: Id.optional(), health: FeedHealth.optional() }, TF),
|
||||
get: get("/feed-sources/{id}", "Feed 详情", FeedSource, TF),
|
||||
create: {
|
||||
method: "post" as const,
|
||||
path: "/feed-sources",
|
||||
scope: "admin" as const,
|
||||
summary: "新建 Feed(url 只写,存为 secret;对外只给脱敏 URL)",
|
||||
tags: TF,
|
||||
body: FeedSourceCreate,
|
||||
response: FeedSource,
|
||||
},
|
||||
update: write("patch", "/feed-sources/{id}", "admin", "修改 Feed", FeedSourceUpdate, FeedSource, TF),
|
||||
fetchNow: write(
|
||||
"post",
|
||||
"/feed-sources/{id}/fetch",
|
||||
"admin",
|
||||
"立即抓取(仍受站点限流)",
|
||||
undefined,
|
||||
z.object({ taskId: Id }),
|
||||
TF,
|
||||
),
|
||||
},
|
||||
sourceReleases: {
|
||||
list: list(
|
||||
"/source-releases",
|
||||
"站点资源列表",
|
||||
SourceRelease,
|
||||
{
|
||||
siteId: Id.optional(),
|
||||
mediaEntityId: Id.optional(),
|
||||
resolutionStatus: ResolutionStatus.optional(),
|
||||
q: z.string().max(200).optional(),
|
||||
},
|
||||
TR,
|
||||
),
|
||||
get: get("/source-releases/{id}", "站点资源详情", SourceRelease, TR),
|
||||
},
|
||||
observations: {
|
||||
list: list(
|
||||
"/observations",
|
||||
"观察列表",
|
||||
Observation,
|
||||
{ feedId: Id.optional(), status: ObservationStatus.optional() },
|
||||
TR,
|
||||
),
|
||||
get: get("/observations/{id}", "观察详情", Observation, TR),
|
||||
},
|
||||
mediaEntities: {
|
||||
list: list(
|
||||
"/media-entities",
|
||||
"作品列表",
|
||||
MediaEntity,
|
||||
{ kind: MediaKind.optional(), status: MediaEntityStatus.optional(), q: z.string().max(200).optional() },
|
||||
TE,
|
||||
),
|
||||
get: get("/media-entities/{id}", "作品(旧 ID 自动跳转)", MediaEntity, TE),
|
||||
create: {
|
||||
method: "post" as const,
|
||||
path: "/media-entities",
|
||||
scope: "organize.execute" as const,
|
||||
summary: "手动建本地作品(TMDB 无记录时可先建档)",
|
||||
tags: TE,
|
||||
body: MediaEntityCreate,
|
||||
response: MediaEntity,
|
||||
},
|
||||
update: write(
|
||||
"patch",
|
||||
"/media-entities/{id}",
|
||||
"organize.execute",
|
||||
"修改作品(标题/原名/年份按字段覆盖并锁定)",
|
||||
MediaEntityUpdate,
|
||||
MediaEntity,
|
||||
TE,
|
||||
),
|
||||
merge: write(
|
||||
"post",
|
||||
"/media-entities/{id}/merge",
|
||||
"organize.execute",
|
||||
"合并重复作品(保留旧 ID 跳转)",
|
||||
MediaEntityMerge,
|
||||
MediaEntity,
|
||||
TE,
|
||||
),
|
||||
},
|
||||
contentUnits: {
|
||||
list: list(
|
||||
"/content-units",
|
||||
"内容单元列表",
|
||||
ContentUnit,
|
||||
{ mediaEntityId: Id.optional(), seasonNumber: z.coerce.number().int().optional() },
|
||||
TE,
|
||||
),
|
||||
get: get("/content-units/{id}", "内容单元详情", ContentUnit, TE),
|
||||
update: write(
|
||||
"patch",
|
||||
"/content-units/{id}",
|
||||
"organize.execute",
|
||||
"修改内容单元(改编号即确认并锁定)",
|
||||
ContentUnitUpdate,
|
||||
ContentUnit,
|
||||
TE,
|
||||
),
|
||||
},
|
||||
titleAliases: {
|
||||
list: list(
|
||||
"/title-aliases",
|
||||
"别名列表",
|
||||
TitleAlias,
|
||||
{ mediaEntityId: Id.optional(), q: z.string().max(200).optional() },
|
||||
TI,
|
||||
),
|
||||
create: write("post", "/title-aliases", "organize.execute", "新增别名", TitleAliasCreate, TitleAlias, TI, false),
|
||||
remove: write("delete", "/title-aliases/{id}", "organize.execute", "删除别名", undefined, OkResult, TI),
|
||||
},
|
||||
externalIdentities: {
|
||||
list: list(
|
||||
"/external-identities",
|
||||
"外部身份列表",
|
||||
ExternalIdentity,
|
||||
{ mediaEntityId: Id.optional(), provider: z.string().optional(), status: IdentityStatus.optional() },
|
||||
TI,
|
||||
),
|
||||
get: get("/external-identities/{id}", "外部身份详情", ExternalIdentity, TI),
|
||||
create: write(
|
||||
"post",
|
||||
"/external-identities",
|
||||
"organize.execute",
|
||||
"新增外部身份(accepted 等同人工纠错)",
|
||||
ExternalIdentityCreate,
|
||||
ExternalIdentity,
|
||||
TI,
|
||||
false,
|
||||
),
|
||||
update: write(
|
||||
"patch",
|
||||
"/external-identities/{id}",
|
||||
"organize.execute",
|
||||
"修订外部身份(乐观并发)",
|
||||
ExternalIdentityUpdate,
|
||||
ExternalIdentity,
|
||||
TI,
|
||||
),
|
||||
revisions: get(
|
||||
"/external-identities/{id}/revisions",
|
||||
"身份修订历史",
|
||||
z.object({ items: z.array(IdentityRevision) }),
|
||||
TI,
|
||||
),
|
||||
},
|
||||
decisions: {
|
||||
list: list(
|
||||
"/decisions",
|
||||
"待处理事项(缺省只列未关闭)",
|
||||
Decision,
|
||||
{ status: DecisionStatus.optional(), kind: DecisionKind.optional() },
|
||||
TD,
|
||||
),
|
||||
get: get("/decisions/{id}", "待处理事项详情", Decision, TD),
|
||||
resolve: write(
|
||||
"post",
|
||||
"/decisions/{id}/resolve",
|
||||
"organize.execute",
|
||||
"处理待处理事项",
|
||||
DecisionResolveInput,
|
||||
Decision,
|
||||
TD,
|
||||
),
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,69 @@
|
||||
import { createRoute, type RouteConfig } from "@hono/zod-openapi";
|
||||
import { type ApiScope, apiResponse } from "@umt/contracts";
|
||||
import type { Context } from "hono";
|
||||
import type { z } from "zod";
|
||||
import { type AuthEnv, requireScope } from "../../http/auth.ts";
|
||||
import type { ModuleApp } from "../types.ts";
|
||||
import type { Actor } from "./service/common.ts";
|
||||
|
||||
// 本模块路由封装:createRoute(进 OpenAPI)+ requireScope + 入参校验 + 出参按合同 parse。
|
||||
|
||||
export interface RouteDef {
|
||||
method: "get" | "post" | "patch" | "put" | "delete";
|
||||
path: string;
|
||||
scope: ApiScope;
|
||||
summary: string;
|
||||
tags: string[];
|
||||
params?: z.ZodObject;
|
||||
query?: z.ZodType;
|
||||
body?: z.ZodType;
|
||||
response: z.ZodType;
|
||||
}
|
||||
|
||||
type Infer<T> = T extends z.ZodType ? z.infer<T> : Record<string, never>;
|
||||
|
||||
export interface Input<D extends RouteDef> {
|
||||
params: Infer<D["params"]>;
|
||||
query: Infer<D["query"]>;
|
||||
body: Infer<D["body"]>;
|
||||
actor: Actor;
|
||||
c: Context<AuthEnv>;
|
||||
}
|
||||
|
||||
/** 调用者(W2-A 的 c.var.actor:admin 会话 / API Key)→ 审计/修订用的操作者;管理员记为 user */
|
||||
export function actorOf(c: Context<AuthEnv>): Actor {
|
||||
const a = c.get("actor") as { type?: string; id?: string } | undefined;
|
||||
if (!a) return { type: "system", id: null };
|
||||
return { type: a.type === "api_key" ? "api_key" : "user", id: a.id ?? null };
|
||||
}
|
||||
|
||||
export function route<D extends RouteDef>(app: ModuleApp, def: D, handler: (input: Input<D>) => Promise<unknown>) {
|
||||
const request: RouteConfig["request"] = {};
|
||||
if (def.params) request.params = def.params as never;
|
||||
if (def.query) request.query = def.query as never;
|
||||
if (def.body) request.body = { content: { "application/json": { schema: def.body as never } }, required: true };
|
||||
const r = createRoute({
|
||||
method: def.method,
|
||||
path: def.path,
|
||||
summary: def.summary,
|
||||
tags: def.tags,
|
||||
middleware: [requireScope(def.scope)] as never,
|
||||
request,
|
||||
responses: {
|
||||
200: { description: def.summary, content: { "application/json": { schema: apiResponse(def.response) } } },
|
||||
},
|
||||
});
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 路由泛型在封装里擦除,入参已由 zod-openapi 校验
|
||||
app.openapi(r, (async (c: any) => {
|
||||
const input = {
|
||||
params: def.params ? c.req.valid("param") : {},
|
||||
query: def.query ? c.req.valid("query") : {},
|
||||
body: def.body ? c.req.valid("json") : {},
|
||||
actor: actorOf(c),
|
||||
c,
|
||||
} as Input<D>;
|
||||
const data = def.response.parse(await handler(input));
|
||||
return c.json({ ok: true, data }, 200);
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 同上
|
||||
}) as any);
|
||||
}
|
||||
@@ -1,12 +1,69 @@
|
||||
import type { ServerModule } from "../types.ts";
|
||||
import type { ServerContext, ServerModule } from "../types.ts";
|
||||
import { registerSourcesCapabilities } from "./capabilities.ts";
|
||||
import { decisionRoutes } from "./routes/decisions.ts";
|
||||
import { identityRoutes } from "./routes/identity.ts";
|
||||
import { sourcesRoutes } from "./routes/sources.ts";
|
||||
import { workbenchRoutes } from "./routes/workbench.ts";
|
||||
import { type RuntimeOverrides, SourcesRuntime } from "./runtime.ts";
|
||||
import { schedulerTick } from "./scheduler.ts";
|
||||
import { registerSourcesTasks, sourcesOutboxRoutes } from "./tasks.ts";
|
||||
|
||||
/**
|
||||
* W2-B 来源/身份/元数据:site、site_account、feed_source;ingestFeed / resolveObservation / refreshMetadata;
|
||||
* W2-B 来源/身份/元数据:site、site_account、feed_source;ingestFeed / resolveObservation / refreshMetadata / snapshotSite;
|
||||
* observation/source_release 写入与去重;身份解析、待识别池与 decision;站点工作台 API。
|
||||
* 发出 release.resolved、decision.created(经 ctx.outbox.emit)。
|
||||
* 发出 feed.due、feed.gap_suspected、release.observed(自消费)与 release.resolved、decision.created/updated。
|
||||
*/
|
||||
export const sourcesIdentityModule: ServerModule = {
|
||||
name: "sources-identity",
|
||||
registerRoutes() {},
|
||||
registerTasks() {},
|
||||
};
|
||||
export function createSourcesIdentityModule(overrides: RuntimeOverrides = {}): ServerModule & {
|
||||
runtime(ctx: ServerContext): SourcesRuntime;
|
||||
} {
|
||||
const runtimes = new WeakMap<ServerContext, SourcesRuntime>();
|
||||
const runtime = (ctx: ServerContext) => {
|
||||
let rt = runtimes.get(ctx);
|
||||
if (!rt) {
|
||||
rt = new SourcesRuntime(ctx, overrides);
|
||||
runtimes.set(ctx, rt);
|
||||
}
|
||||
return rt;
|
||||
};
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
let ticking: Promise<unknown> | null = null;
|
||||
|
||||
return {
|
||||
name: "sources-identity",
|
||||
runtime,
|
||||
registerRoutes(app, ctx) {
|
||||
const rt = () => runtime(ctx);
|
||||
sourcesRoutes(app, rt);
|
||||
workbenchRoutes(app, rt);
|
||||
identityRoutes(app, rt);
|
||||
decisionRoutes(app, rt);
|
||||
},
|
||||
registerTasks(registry, ctx) {
|
||||
registerSourcesTasks(registry, () => runtime(ctx));
|
||||
},
|
||||
outboxRoutes: () => sourcesOutboxRoutes,
|
||||
registerCapabilities(registry, ctx) {
|
||||
registerSourcesCapabilities(registry, () => runtime(ctx));
|
||||
},
|
||||
start(ctx) {
|
||||
const rt = runtime(ctx);
|
||||
const tick = () => {
|
||||
if (ticking) return;
|
||||
ticking = schedulerTick(rt)
|
||||
.catch((err) => ctx.logger.warn({ err }, "sources-identity 调度失败"))
|
||||
.finally(() => {
|
||||
ticking = null;
|
||||
});
|
||||
};
|
||||
timer = setInterval(tick, overrides.schedulerIntervalMs ?? 30_000);
|
||||
tick();
|
||||
},
|
||||
async stop() {
|
||||
if (timer) clearInterval(timer);
|
||||
timer = null;
|
||||
await ticking;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export const sourcesIdentityModule = createSourcesIdentityModule();
|
||||
@@ -0,0 +1,184 @@
|
||||
import { type Decision, DecisionKind, DecisionStatus, Id, type PoolCandidate } from "@umt/contracts";
|
||||
import { sql } from "@umt/db";
|
||||
import { z } from "zod";
|
||||
import { ApiException } from "../../../http/errors.ts";
|
||||
import type { ModuleApp } from "../../types.ts";
|
||||
import { D } from "../defs.ts";
|
||||
import { route } from "../http.ts";
|
||||
import { decodeCursor, jsonb, page, toApi, toApiList } from "../rows.ts";
|
||||
import type { SourcesRuntime } from "../runtime.ts";
|
||||
import { audit } from "../service/common.ts";
|
||||
import { assignReleaseIdentity, dismissPoolEntry } from "../service/identity-admin.ts";
|
||||
|
||||
// 待处理中心数据源:decision 列表/详情/聚合/处理。
|
||||
// 身份歧义类由本模块直接执行所选候选;其它类型只记录处理结果并发 decision.updated,
|
||||
// 需要执行能力的选项由 W2-D 的 resolveDecision 消费。
|
||||
|
||||
const OPEN = ["open", "investigating", "awaiting_user"] as const;
|
||||
const T = ["decisions"];
|
||||
|
||||
const SummaryGroup = z.object({
|
||||
kind: DecisionKind,
|
||||
/** 关联作品(身份歧义等尚未归属作品的为 null) */
|
||||
mediaEntityId: Id.nullable(),
|
||||
mediaEntityTitle: z.string().nullable(),
|
||||
count: z.number().int(),
|
||||
maxSeverity: z.enum(["info", "warning", "critical"]),
|
||||
latestAt: z.string(),
|
||||
});
|
||||
|
||||
export function decisionRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
|
||||
const db = () => rt().ctx.db;
|
||||
|
||||
route(app, D.decisions.list, async ({ query }) => {
|
||||
const off = decodeCursor(query.cursor);
|
||||
let q = db().selectFrom("decision").selectAll();
|
||||
q = query.status ? q.where("status", "=", query.status) : q.where("status", "in", [...OPEN]);
|
||||
if (query.kind) q = q.where("kind", "=", query.kind);
|
||||
const rows = await q
|
||||
.orderBy(sql`case severity when 'critical' then 0 when 'warning' then 1 else 2 end`)
|
||||
.orderBy("created_at", "desc")
|
||||
.orderBy("id")
|
||||
.offset(off)
|
||||
.limit(query.limit + 1)
|
||||
.execute();
|
||||
const p = page(rows, off, query.limit);
|
||||
return { items: toApiList<Decision>(p.items), nextCursor: p.nextCursor };
|
||||
});
|
||||
|
||||
route(
|
||||
app,
|
||||
{
|
||||
method: "get",
|
||||
path: "/decisions/summary",
|
||||
scope: "read",
|
||||
summary: "未关闭待处理事项按作品与问题类型聚合",
|
||||
tags: T,
|
||||
query: z.object({ status: DecisionStatus.optional() }),
|
||||
response: z.object({ total: z.number().int(), groups: z.array(SummaryGroup) }),
|
||||
},
|
||||
async ({ query }) => {
|
||||
const statuses = query.status ? [query.status] : [...OPEN];
|
||||
// 作品归属:evidence.mediaEntityId,或 subject 为 media_entity,或 subject 为已识别的 source_release
|
||||
const rows = await sql<{
|
||||
kind: z.infer<typeof DecisionKind>;
|
||||
entity_id: string | null;
|
||||
entity_title: string | null;
|
||||
count: number;
|
||||
max_sev: number;
|
||||
latest: Date;
|
||||
}>`
|
||||
with d as (
|
||||
select d.*, coalesce(
|
||||
case when d.evidence->>'mediaEntityId' ~ '^[0-9a-f-]{36}$' then (d.evidence->>'mediaEntityId')::uuid end,
|
||||
case when d.subject_type = 'media_entity' then d.subject_id end,
|
||||
(select r.media_entity_id from source_release r where d.subject_type = 'source_release' and r.id = d.subject_id)
|
||||
) as entity_id
|
||||
from decision d where d.status = any(${statuses})
|
||||
)
|
||||
select d.kind, d.entity_id, e.title as entity_title, count(*)::int as count,
|
||||
max(case d.severity when 'critical' then 2 when 'warning' then 1 else 0 end)::int as max_sev,
|
||||
max(d.created_at) as latest
|
||||
from d left join media_entity e on e.id = d.entity_id
|
||||
group by d.kind, d.entity_id, e.title
|
||||
order by max_sev desc, latest desc`.execute(db());
|
||||
const sev = ["info", "warning", "critical"] as const;
|
||||
return {
|
||||
total: rows.rows.reduce((a, r) => a + r.count, 0),
|
||||
groups: rows.rows.map((r) => ({
|
||||
kind: r.kind,
|
||||
mediaEntityId: r.entity_id,
|
||||
mediaEntityTitle: r.entity_title,
|
||||
count: r.count,
|
||||
maxSeverity: sev[r.max_sev] ?? "info",
|
||||
latestAt: new Date(r.latest).toISOString(),
|
||||
})),
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
route(app, D.decisions.get, async ({ params }) => {
|
||||
const row = await db().selectFrom("decision").selectAll().where("id", "=", params.id).executeTakeFirst();
|
||||
if (!row) throw new ApiException("not_found", "待处理事项不存在");
|
||||
return toApi(row);
|
||||
});
|
||||
|
||||
route(app, D.decisions.resolve, async ({ params, body, actor }) => {
|
||||
const d = await db().selectFrom("decision").selectAll().where("id", "=", params.id).executeTakeFirst();
|
||||
if (!d) throw new ApiException("not_found", "待处理事项不存在");
|
||||
if (!(OPEN as readonly string[]).includes(d.status)) throw new ApiException("conflict", `事项已是 ${d.status}`);
|
||||
if (!body.dismiss && !body.optionKey) throw new ApiException("bad_request", "需要 optionKey 或 dismiss");
|
||||
const by = `${actor.type}:${actor.id ?? ""}`;
|
||||
const options = (d.options ?? []) as { key: string; input?: Record<string, unknown> }[];
|
||||
const opt = body.optionKey ? options.find((o) => o.key === body.optionKey) : undefined;
|
||||
if (body.optionKey && !opt) throw new ApiException("bad_request", `未知选项 ${body.optionKey}`);
|
||||
|
||||
// 身份歧义:直接执行(选候选 = 人工指定并加锁;忽略 = 待识别池 dismissed)
|
||||
if (d.kind === "identity_ambiguous" && d.subject_type === "source_release" && d.subject_id) {
|
||||
if (body.dismiss || opt?.key === "dismiss") {
|
||||
const pool = await db()
|
||||
.selectFrom("identity_candidate")
|
||||
.select(["id"])
|
||||
.where("source_release_id", "=", d.subject_id)
|
||||
.executeTakeFirst();
|
||||
if (pool) await dismissPoolEntry(rt(), pool.id, actor, body.note ?? null);
|
||||
} else if (opt?.input && typeof opt.input.candidateIndex === "number") {
|
||||
const pool = await db()
|
||||
.selectFrom("identity_candidate")
|
||||
.select(["candidates"])
|
||||
.where("source_release_id", "=", d.subject_id)
|
||||
.executeTakeFirst();
|
||||
if (!(pool?.candidates as PoolCandidate[] | undefined)?.[opt.input.candidateIndex]) {
|
||||
throw new ApiException("conflict", "候选已变化,请刷新后重试");
|
||||
}
|
||||
await assignReleaseIdentity(
|
||||
rt(),
|
||||
d.subject_id,
|
||||
{ candidateIndex: opt.input.candidateIndex, lock: true, reason: body.note ?? `decision ${d.id}` },
|
||||
actor,
|
||||
);
|
||||
}
|
||||
}
|
||||
// 上面的执行路径已按 dedupeKey 关闭;其余类型在这里记录处理结果
|
||||
const row = await db()
|
||||
.transaction()
|
||||
.execute(async (trx) => {
|
||||
const cur = await trx
|
||||
.selectFrom("decision")
|
||||
.selectAll()
|
||||
.where("id", "=", params.id)
|
||||
.forUpdate()
|
||||
.executeTakeFirstOrThrow();
|
||||
if (!(OPEN as readonly string[]).includes(cur.status)) return cur;
|
||||
const status = body.dismiss || opt?.key === "dismiss" ? "dismissed" : "resolved";
|
||||
const r = await trx
|
||||
.updateTable("decision")
|
||||
.set({
|
||||
status,
|
||||
resolution: jsonb({ optionKey: body.optionKey ?? null, note: body.note ?? null, by }),
|
||||
resolved_by: by,
|
||||
resolved_at: new Date(),
|
||||
})
|
||||
.where("id", "=", params.id)
|
||||
.returningAll()
|
||||
.executeTakeFirstOrThrow();
|
||||
await rt().ctx.outbox.emit(
|
||||
trx,
|
||||
"decision.updated",
|
||||
{ decisionId: r.id, status },
|
||||
{
|
||||
idempotencyKey: `decision.updated:${r.id}:${status}`,
|
||||
},
|
||||
);
|
||||
await audit(
|
||||
trx,
|
||||
actor,
|
||||
"decision.resolved",
|
||||
{ type: "decision", id: r.id },
|
||||
{ optionKey: body.optionKey ?? null, status },
|
||||
);
|
||||
return r;
|
||||
});
|
||||
return toApi(row);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,597 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import {
|
||||
type ContentUnit,
|
||||
cursorPage,
|
||||
EntityIdentitySet,
|
||||
EntityMetadataField,
|
||||
type ExternalIdentity,
|
||||
FieldOverrideInput,
|
||||
Id,
|
||||
IdentityCandidate,
|
||||
IdentityPoolItem,
|
||||
IdentityPoolQuery,
|
||||
IdParam,
|
||||
type MediaEntity,
|
||||
MediaEntityDetail,
|
||||
OkResult,
|
||||
ReleaseIdentityAssign,
|
||||
type TitleAlias,
|
||||
} from "@umt/contracts";
|
||||
import { sql } from "@umt/db";
|
||||
import { normalizeTitle } from "@umt/domain";
|
||||
import { z } from "zod";
|
||||
import { ApiException } from "../../../http/errors.ts";
|
||||
import type { ModuleApp } from "../../types.ts";
|
||||
import { D } from "../defs.ts";
|
||||
import { route } from "../http.ts";
|
||||
import { decodeCursor, jsonb, page, toApi, toApiList } from "../rows.ts";
|
||||
import type { SourcesRuntime } from "../runtime.ts";
|
||||
import { audit } from "../service/common.ts";
|
||||
import { resolveEntityId } from "../service/entities.ts";
|
||||
import {
|
||||
assignReleaseIdentity,
|
||||
dismissPoolEntry,
|
||||
mergeEntities,
|
||||
setEntityIdentity,
|
||||
updateExternalIdentity,
|
||||
} from "../service/identity-admin.ts";
|
||||
import { clearFieldOverride, setFieldOverride } from "../service/metadata.ts";
|
||||
|
||||
// 身份与元数据 API:作品(列表/详情/纠错/合并/字段覆盖)、内容单元、别名、外部身份、待识别池。
|
||||
|
||||
const T_ENT = ["media-entities"];
|
||||
const T_ID = ["identity"];
|
||||
const Accepted = z.object({ taskId: Id });
|
||||
const FieldParam = z.object({ id: Id, field: EntityMetadataField });
|
||||
|
||||
export function identityRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
|
||||
const db = () => rt().ctx.db;
|
||||
|
||||
// ── 作品 ──
|
||||
route(app, D.mediaEntities.list, async ({ query }) => {
|
||||
const off = decodeCursor(query.cursor);
|
||||
let q = db().selectFrom("media_entity").selectAll();
|
||||
if (query.kind) q = q.where("kind", "=", query.kind);
|
||||
q = query.status ? q.where("status", "=", query.status) : q.where("status", "<>", "merged");
|
||||
if (query.q) {
|
||||
const n = normalizeTitle(query.q);
|
||||
q = q.where((eb) =>
|
||||
eb.or([
|
||||
eb("title", "ilike", `%${query.q}%`),
|
||||
eb.exists(
|
||||
eb
|
||||
.selectFrom("title_alias as a")
|
||||
.select("a.id")
|
||||
.whereRef("a.media_entity_id", "=", "media_entity.id")
|
||||
.where(sql<boolean>`(a.normalized = ${n} or similarity(a.normalized, ${n}) >= 0.4)`),
|
||||
),
|
||||
]),
|
||||
);
|
||||
}
|
||||
const rows = await q
|
||||
.orderBy("updated_at", "desc")
|
||||
.orderBy("id")
|
||||
.offset(off)
|
||||
.limit(query.limit + 1)
|
||||
.execute();
|
||||
const p = page(rows, off, query.limit);
|
||||
return { items: toApiList<MediaEntity>(p.items), nextCursor: p.nextCursor };
|
||||
});
|
||||
route(app, D.mediaEntities.get, async ({ params }) => {
|
||||
const id = await resolveEntityId(db(), params.id);
|
||||
if (!id) throw new ApiException("not_found", "作品不存在");
|
||||
return toApi(await db().selectFrom("media_entity").selectAll().where("id", "=", id).executeTakeFirstOrThrow());
|
||||
});
|
||||
route(
|
||||
app,
|
||||
{
|
||||
method: "get",
|
||||
path: "/media-entities/{id}/detail",
|
||||
scope: "read",
|
||||
summary: "作品详情:外部身份、别名、内容单元、字段来源与覆盖(旧 ID 自动跳转)",
|
||||
tags: T_ENT,
|
||||
params: IdParam,
|
||||
response: MediaEntityDetail,
|
||||
},
|
||||
async ({ params }) => {
|
||||
const id = await resolveEntityId(db(), params.id);
|
||||
if (!id) throw new ApiException("not_found", "作品不存在");
|
||||
const [entity, identities, aliases, units, fields, overrides] = await Promise.all([
|
||||
db().selectFrom("media_entity").selectAll().where("id", "=", id).executeTakeFirstOrThrow(),
|
||||
db()
|
||||
.selectFrom("external_identity")
|
||||
.selectAll()
|
||||
.where("media_entity_id", "=", id)
|
||||
.orderBy("created_at")
|
||||
.execute(),
|
||||
db().selectFrom("title_alias").selectAll().where("media_entity_id", "=", id).orderBy("created_at").execute(),
|
||||
db()
|
||||
.selectFrom("content_unit")
|
||||
.selectAll()
|
||||
.where("media_entity_id", "=", id)
|
||||
.orderBy("season_number")
|
||||
.orderBy("episode_number")
|
||||
.execute(),
|
||||
db().selectFrom("metadata_field").selectAll().where("media_entity_id", "=", id).orderBy("field").execute(),
|
||||
db().selectFrom("field_override").selectAll().where("media_entity_id", "=", id).orderBy("field").execute(),
|
||||
]);
|
||||
return {
|
||||
entity: toApi(entity),
|
||||
redirectedFrom: id === params.id ? null : params.id,
|
||||
identities: toApiList(identities),
|
||||
aliases: toApiList(aliases),
|
||||
contentUnits: toApiList(units),
|
||||
metadataFields: toApiList(fields),
|
||||
overrides: toApiList(overrides),
|
||||
};
|
||||
},
|
||||
);
|
||||
route(app, D.mediaEntities.create, async ({ body, actor }) => {
|
||||
const row = await db()
|
||||
.transaction()
|
||||
.execute(async (trx) => {
|
||||
const r = await trx
|
||||
.insertInto("media_entity")
|
||||
.values({
|
||||
kind: body.kind,
|
||||
title: body.title,
|
||||
original_title: body.originalTitle ?? null,
|
||||
year: body.year ?? null,
|
||||
status: body.status === "merged" ? "active" : (body.status ?? "active"),
|
||||
metadata: jsonb(body.metadata ?? {}),
|
||||
field_locks: body.fieldLocks ?? [],
|
||||
})
|
||||
.returningAll()
|
||||
.executeTakeFirstOrThrow();
|
||||
await sql`insert into title_alias (media_entity_id, alias, normalized, kind, source, locked)
|
||||
values (${r.id}, ${body.title}, ${normalizeTitle(body.title) || body.title}, 'translated', 'user', true)
|
||||
on conflict do nothing`.execute(trx);
|
||||
await audit(trx, actor, "media_entity.created", { type: "media_entity", id: r.id }, { title: body.title });
|
||||
return r;
|
||||
});
|
||||
return toApi(row);
|
||||
});
|
||||
route(app, D.mediaEntities.update, async ({ params, body, actor }) => {
|
||||
const id = await resolveEntityId(db(), params.id);
|
||||
if (!id) throw new ApiException("not_found", "作品不存在");
|
||||
// 标题/原名/年份按字段覆盖处理(加锁),其余字段直接更新
|
||||
const locked: [z.infer<typeof EntityMetadataField>, unknown][] = [];
|
||||
if (body.title !== undefined) locked.push(["title", body.title]);
|
||||
if (body.originalTitle !== undefined) locked.push(["originalTitle", body.originalTitle]);
|
||||
if (body.year !== undefined) locked.push(["year", body.year]);
|
||||
for (const [field, value] of locked) await setFieldOverride(rt(), id, field, { value, lock: true }, actor);
|
||||
const patch: Record<string, unknown> = {};
|
||||
if (body.kind !== undefined) patch.kind = body.kind;
|
||||
if (body.status !== undefined) patch.status = body.status;
|
||||
if (Object.keys(patch).length) {
|
||||
await db().updateTable("media_entity").set(patch).where("id", "=", id).execute();
|
||||
await audit(db(), actor, "media_entity.updated", { type: "media_entity", id }, patch);
|
||||
}
|
||||
return toApi(await db().selectFrom("media_entity").selectAll().where("id", "=", id).executeTakeFirstOrThrow());
|
||||
});
|
||||
route(app, D.mediaEntities.merge, async ({ params, body, actor }) =>
|
||||
toApi(await mergeEntities(rt(), params.id, body.intoId, body.reason ?? null, actor)),
|
||||
);
|
||||
route(
|
||||
app,
|
||||
{
|
||||
method: "post",
|
||||
path: "/media-entities/{id}/identities",
|
||||
scope: "organize.execute",
|
||||
summary: "人工纠错外部映射:旧映射退役(留修订),新映射 accepted + 人工锁",
|
||||
tags: T_ID,
|
||||
params: IdParam,
|
||||
body: EntityIdentitySet,
|
||||
response: z.object({
|
||||
mediaEntityId: Id,
|
||||
changed: z.boolean(),
|
||||
identityId: Id.optional(),
|
||||
retired: z.array(Id).optional(),
|
||||
}),
|
||||
},
|
||||
async ({ params, body, actor }) => setEntityIdentity(rt(), params.id, body, actor),
|
||||
);
|
||||
route(
|
||||
app,
|
||||
{
|
||||
method: "post",
|
||||
path: "/media-entities/{id}/refresh",
|
||||
scope: "organize.execute",
|
||||
summary: "刷新元数据(字段级;用户锁优先,不动路径)",
|
||||
tags: T_ENT,
|
||||
params: IdParam,
|
||||
response: Accepted,
|
||||
},
|
||||
async ({ params }) => {
|
||||
const id = await resolveEntityId(db(), params.id);
|
||||
if (!id) throw new ApiException("not_found", "作品不存在");
|
||||
const r = await rt().ctx.engine.enqueue(
|
||||
"refreshMetadata",
|
||||
{ mediaEntityId: id },
|
||||
{ idempotencyKey: `refreshMetadata:${id}:${Math.floor(Date.now() / 60_000)}`, traceId: randomUUID() },
|
||||
);
|
||||
return { taskId: r.runId };
|
||||
},
|
||||
);
|
||||
const FieldResult = z.object({
|
||||
mediaEntityId: Id,
|
||||
changedFields: z.array(z.string()),
|
||||
lockedFields: z.array(z.string()),
|
||||
});
|
||||
route(
|
||||
app,
|
||||
{
|
||||
method: "put",
|
||||
path: "/media-entities/{id}/fields/{field}",
|
||||
scope: "organize.execute",
|
||||
summary: "覆盖作品字段并锁定(自动刷新不再改写)",
|
||||
tags: T_ENT,
|
||||
params: FieldParam,
|
||||
body: FieldOverrideInput,
|
||||
response: FieldResult,
|
||||
},
|
||||
async ({ params, body, actor }) => setFieldOverride(rt(), params.id, params.field, body, actor),
|
||||
);
|
||||
route(
|
||||
app,
|
||||
{
|
||||
method: "delete",
|
||||
path: "/media-entities/{id}/fields/{field}",
|
||||
scope: "organize.execute",
|
||||
summary: "取消字段覆盖与锁定",
|
||||
tags: T_ENT,
|
||||
params: FieldParam,
|
||||
response: FieldResult,
|
||||
},
|
||||
async ({ params, actor }) => clearFieldOverride(rt(), params.id, params.field, actor),
|
||||
);
|
||||
|
||||
// ── 内容单元 ──
|
||||
route(app, D.contentUnits.list, async ({ query }) => {
|
||||
const off = decodeCursor(query.cursor);
|
||||
let q = db().selectFrom("content_unit").selectAll();
|
||||
if (query.mediaEntityId) q = q.where("media_entity_id", "=", query.mediaEntityId);
|
||||
if (query.seasonNumber !== undefined) q = q.where("season_number", "=", query.seasonNumber);
|
||||
const rows = await q
|
||||
.orderBy("media_entity_id")
|
||||
.orderBy("season_number")
|
||||
.orderBy("episode_number")
|
||||
.orderBy("id")
|
||||
.offset(off)
|
||||
.limit(query.limit + 1)
|
||||
.execute();
|
||||
const p = page(rows, off, query.limit);
|
||||
return { items: toApiList<ContentUnit>(p.items), nextCursor: p.nextCursor };
|
||||
});
|
||||
route(app, D.contentUnits.get, async ({ params }) => {
|
||||
const row = await db().selectFrom("content_unit").selectAll().where("id", "=", params.id).executeTakeFirst();
|
||||
if (!row) throw new ApiException("not_found", "内容单元不存在");
|
||||
return toApi(row);
|
||||
});
|
||||
route(app, D.contentUnits.update, async ({ params, body, actor }) => {
|
||||
const unit = await db().selectFrom("content_unit").selectAll().where("id", "=", params.id).executeTakeFirst();
|
||||
if (!unit) throw new ApiException("not_found", "内容单元不存在");
|
||||
const numbering = ["seasonNumber", "episodeNumber", "sourceIssue", "airDate"] as const;
|
||||
const touchesNumbering = numbering.some((k) => body[k] !== undefined);
|
||||
const patch: Record<string, unknown> = {};
|
||||
const map: Record<string, string> = {
|
||||
parentUnitId: "parent_unit_id",
|
||||
kind: "kind",
|
||||
contentType: "content_type",
|
||||
segment: "segment",
|
||||
seasonNumber: "season_number",
|
||||
episodeNumber: "episode_number",
|
||||
sourceIssue: "source_issue",
|
||||
airDate: "air_date",
|
||||
displayOrder: "display_order",
|
||||
title: "title",
|
||||
status: "status",
|
||||
numberingLocked: "numbering_locked",
|
||||
fieldLocks: "field_locks",
|
||||
};
|
||||
for (const [k, col] of Object.entries(map)) {
|
||||
const v = (body as Record<string, unknown>)[k];
|
||||
if (v !== undefined) patch[col] = v;
|
||||
}
|
||||
if (body.metadata !== undefined) patch.metadata = jsonb(body.metadata);
|
||||
// 人工改编号即确认并锁定编号(REQ §5.4:编号可修订,人工确认后刷新不得改)
|
||||
if (touchesNumbering && body.numberingLocked === undefined) patch.numbering_locked = true;
|
||||
if (touchesNumbering && body.status === undefined && unit.status === "provisional") patch.status = "confirmed";
|
||||
const row = await db()
|
||||
.updateTable("content_unit")
|
||||
.set(patch)
|
||||
.where("id", "=", params.id)
|
||||
.returningAll()
|
||||
.executeTakeFirstOrThrow();
|
||||
await audit(
|
||||
db(),
|
||||
actor,
|
||||
"content_unit.updated",
|
||||
{ type: "content_unit", id: params.id },
|
||||
{ fields: Object.keys(patch) },
|
||||
);
|
||||
return toApi(row);
|
||||
});
|
||||
|
||||
// ── 别名 ──
|
||||
route(app, D.titleAliases.list, async ({ query }) => {
|
||||
const off = decodeCursor(query.cursor);
|
||||
let q = db().selectFrom("title_alias").selectAll();
|
||||
if (query.mediaEntityId) q = q.where("media_entity_id", "=", query.mediaEntityId);
|
||||
if (query.q) {
|
||||
const n = normalizeTitle(query.q);
|
||||
q = q.where(sql<boolean>`(normalized = ${n} or similarity(normalized, ${n}) >= 0.3)`);
|
||||
}
|
||||
const rows = await q
|
||||
.orderBy("created_at")
|
||||
.orderBy("id")
|
||||
.offset(off)
|
||||
.limit(query.limit + 1)
|
||||
.execute();
|
||||
const p = page(rows, off, query.limit);
|
||||
return { items: toApiList<TitleAlias>(p.items), nextCursor: p.nextCursor };
|
||||
});
|
||||
route(app, D.titleAliases.create, async ({ body, actor }) => {
|
||||
const id = await resolveEntityId(db(), body.mediaEntityId);
|
||||
if (!id) throw new ApiException("not_found", "作品不存在");
|
||||
const normalized = normalizeTitle(body.alias);
|
||||
if (!normalized) throw new ApiException("bad_request", "别名归一化后为空");
|
||||
try {
|
||||
const row = await db()
|
||||
.insertInto("title_alias")
|
||||
.values({
|
||||
media_entity_id: id,
|
||||
content_unit_id: body.contentUnitId ?? null,
|
||||
alias: body.alias,
|
||||
normalized,
|
||||
language: body.language ?? null,
|
||||
kind: body.kind ?? "other",
|
||||
source: body.source,
|
||||
locked: body.locked ?? body.source === "user",
|
||||
})
|
||||
.returningAll()
|
||||
.executeTakeFirstOrThrow();
|
||||
await audit(db(), actor, "title_alias.created", { type: "title_alias", id: row.id }, { alias: body.alias });
|
||||
return toApi(row);
|
||||
} catch (e) {
|
||||
if ((e as { code?: string }).code === "23505") throw new ApiException("conflict", "别名已存在");
|
||||
throw e;
|
||||
}
|
||||
});
|
||||
route(app, D.titleAliases.remove, async ({ params, actor }) => {
|
||||
const r = await db().deleteFrom("title_alias").where("id", "=", params.id).returning(["id"]).executeTakeFirst();
|
||||
if (!r) throw new ApiException("not_found", "别名不存在");
|
||||
await audit(db(), actor, "title_alias.deleted", { type: "title_alias", id: params.id });
|
||||
return OkResult.parse({ ok: true });
|
||||
});
|
||||
|
||||
// ── 外部身份 ──
|
||||
route(app, D.externalIdentities.list, async ({ query }) => {
|
||||
const off = decodeCursor(query.cursor);
|
||||
let q = db().selectFrom("external_identity").selectAll();
|
||||
if (query.mediaEntityId) q = q.where("media_entity_id", "=", query.mediaEntityId);
|
||||
if (query.provider) q = q.where("provider", "=", query.provider);
|
||||
if (query.status) q = q.where("status", "=", query.status);
|
||||
const rows = await q
|
||||
.orderBy("created_at")
|
||||
.orderBy("id")
|
||||
.offset(off)
|
||||
.limit(query.limit + 1)
|
||||
.execute();
|
||||
const p = page(rows, off, query.limit);
|
||||
return { items: toApiList<ExternalIdentity>(p.items), nextCursor: p.nextCursor };
|
||||
});
|
||||
route(app, D.externalIdentities.get, async ({ params }) => {
|
||||
const row = await db().selectFrom("external_identity").selectAll().where("id", "=", params.id).executeTakeFirst();
|
||||
if (!row) throw new ApiException("not_found", "外部身份不存在");
|
||||
return toApi(row);
|
||||
});
|
||||
route(app, D.externalIdentities.create, async ({ body, actor }) => {
|
||||
if (!body.mediaEntityId) throw new ApiException("bad_request", "v0.1 只支持作品级外部身份");
|
||||
const entityId = await resolveEntityId(db(), body.mediaEntityId);
|
||||
if (!entityId) throw new ApiException("not_found", "作品不存在");
|
||||
if ((body.status ?? "proposed") === "accepted") {
|
||||
const r = await setEntityIdentity(
|
||||
rt(),
|
||||
entityId,
|
||||
{
|
||||
provider: body.provider,
|
||||
entityType: body.entityType as "movie" | "tv",
|
||||
externalId: body.externalId,
|
||||
lock: body.locked ?? true,
|
||||
},
|
||||
actor,
|
||||
);
|
||||
if (!r.identityId) throw new ApiException("conflict", "该外部身份已是此作品的 accepted 映射");
|
||||
return toApi(
|
||||
await db().selectFrom("external_identity").selectAll().where("id", "=", r.identityId).executeTakeFirstOrThrow(),
|
||||
);
|
||||
}
|
||||
const row = await db()
|
||||
.transaction()
|
||||
.execute(async (trx) => {
|
||||
const r = await trx
|
||||
.insertInto("external_identity")
|
||||
.values({
|
||||
media_entity_id: entityId,
|
||||
provider: body.provider,
|
||||
entity_type: body.entityType,
|
||||
external_id: body.externalId,
|
||||
status: body.status ?? "proposed",
|
||||
evidence_level: body.evidenceLevel ?? "manual",
|
||||
evidence: jsonb(body.evidence ?? {}),
|
||||
locked: body.locked ?? false,
|
||||
extra: jsonb(body.extra ?? {}),
|
||||
})
|
||||
.returningAll()
|
||||
.executeTakeFirstOrThrow();
|
||||
await trx
|
||||
.insertInto("identity_revision")
|
||||
.values({
|
||||
external_identity_id: r.id,
|
||||
revision: 1,
|
||||
action: "created",
|
||||
to_status: r.status,
|
||||
after: jsonb({ status: r.status }),
|
||||
actor_type: actor.type,
|
||||
actor_id: actor.id,
|
||||
})
|
||||
.execute();
|
||||
return r;
|
||||
});
|
||||
return toApi(row);
|
||||
});
|
||||
route(app, D.externalIdentities.update, async ({ params, body, actor }) =>
|
||||
toApi(await updateExternalIdentity(rt(), params.id, body, actor)),
|
||||
);
|
||||
route(app, D.externalIdentities.revisions, async ({ params }) => {
|
||||
const rows = await db()
|
||||
.selectFrom("identity_revision")
|
||||
.selectAll()
|
||||
.where("external_identity_id", "=", params.id)
|
||||
.orderBy("revision")
|
||||
.execute();
|
||||
return { items: toApiList(rows) };
|
||||
});
|
||||
|
||||
// ── 待识别池 ──
|
||||
route(
|
||||
app,
|
||||
{
|
||||
method: "get",
|
||||
path: "/identity-pool",
|
||||
scope: "read",
|
||||
summary: "待识别池(可按原始标题检索;候选不建正式作品)",
|
||||
tags: T_ID,
|
||||
query: IdentityPoolQuery,
|
||||
response: cursorPage(IdentityPoolItem),
|
||||
},
|
||||
async ({ query }) => {
|
||||
const off = decodeCursor(query.cursor);
|
||||
let q = db()
|
||||
.selectFrom("identity_candidate as p")
|
||||
.innerJoin("source_release as r", "r.id", "p.source_release_id")
|
||||
.selectAll("p")
|
||||
.select([
|
||||
"r.site_id as r_site_id",
|
||||
"r.site_release_id as r_site_release_id",
|
||||
"r.title as r_title",
|
||||
"r.subtitle as r_subtitle",
|
||||
"r.category_raw as r_category_raw",
|
||||
"r.first_seen_at as r_first_seen_at",
|
||||
"r.resolution_status as r_resolution_status",
|
||||
]);
|
||||
q = q.where("p.status", "=", query.status ?? "pending");
|
||||
if (query.reason) q = q.where("p.reason", "=", query.reason);
|
||||
if (query.siteId) q = q.where("r.site_id", "=", query.siteId);
|
||||
if (query.q) {
|
||||
const n = normalizeTitle(query.q);
|
||||
q = q.where((eb) =>
|
||||
eb.or([
|
||||
eb("r.title", "ilike", `%${query.q}%`),
|
||||
eb("r.subtitle", "ilike", `%${query.q}%`),
|
||||
sql<boolean>`(p.query->>'normalized') = ${n}`,
|
||||
sql<boolean>`similarity(r.title, ${query.q}) >= 0.3`,
|
||||
]),
|
||||
);
|
||||
}
|
||||
const rows = await q
|
||||
.orderBy("p.updated_at", "desc")
|
||||
.orderBy("p.id")
|
||||
.offset(off)
|
||||
.limit(query.limit + 1)
|
||||
.execute();
|
||||
const p = page(rows, off, query.limit);
|
||||
return {
|
||||
items: p.items.map((row) => {
|
||||
const {
|
||||
r_site_id,
|
||||
r_site_release_id,
|
||||
r_title,
|
||||
r_subtitle,
|
||||
r_category_raw,
|
||||
r_first_seen_at,
|
||||
r_resolution_status,
|
||||
...rest
|
||||
} = row;
|
||||
return {
|
||||
...toApi<Record<string, unknown>>(rest),
|
||||
release: {
|
||||
id: rest.source_release_id,
|
||||
siteId: r_site_id,
|
||||
siteReleaseId: r_site_release_id,
|
||||
title: r_title,
|
||||
subtitle: r_subtitle,
|
||||
categoryRaw: r_category_raw,
|
||||
firstSeenAt: r_first_seen_at.toISOString(),
|
||||
resolutionStatus: r_resolution_status,
|
||||
},
|
||||
};
|
||||
}),
|
||||
nextCursor: p.nextCursor,
|
||||
};
|
||||
},
|
||||
);
|
||||
route(
|
||||
app,
|
||||
{
|
||||
method: "post",
|
||||
path: "/identity-pool/{id}/assign",
|
||||
scope: "organize.execute",
|
||||
summary: "为待识别资源手动指定候选 / 本地作品 / 外部身份",
|
||||
tags: T_ID,
|
||||
params: IdParam,
|
||||
body: ReleaseIdentityAssign,
|
||||
response: z.object({ mediaEntityId: Id, contentUnitIds: z.array(Id) }),
|
||||
},
|
||||
async ({ params, body, actor }) => {
|
||||
const pool = await db()
|
||||
.selectFrom("identity_candidate")
|
||||
.select(["source_release_id"])
|
||||
.where("id", "=", params.id)
|
||||
.executeTakeFirst();
|
||||
if (!pool) throw new ApiException("not_found", "待识别条目不存在");
|
||||
return assignReleaseIdentity(rt(), pool.source_release_id, body, actor);
|
||||
},
|
||||
);
|
||||
route(
|
||||
app,
|
||||
{
|
||||
method: "post",
|
||||
path: "/identity-pool/{id}/dismiss",
|
||||
scope: "organize.execute",
|
||||
summary: "忽略待识别资源(不再自动复查)",
|
||||
tags: T_ID,
|
||||
params: IdParam,
|
||||
body: z.object({ note: z.string().max(2000).optional() }),
|
||||
response: IdentityCandidate,
|
||||
},
|
||||
async ({ params, body, actor }) => toApi(await dismissPoolEntry(rt(), params.id, actor, body.note ?? null)),
|
||||
);
|
||||
route(
|
||||
app,
|
||||
{
|
||||
method: "post",
|
||||
path: "/identity-pool/{id}/retry",
|
||||
scope: "organize.execute",
|
||||
summary: "立即重新识别",
|
||||
tags: T_ID,
|
||||
params: IdParam,
|
||||
response: Accepted,
|
||||
},
|
||||
async ({ params }) => {
|
||||
const pool = await db()
|
||||
.selectFrom("identity_candidate")
|
||||
.select(["source_release_id", "attempts"])
|
||||
.where("id", "=", params.id)
|
||||
.executeTakeFirst();
|
||||
if (!pool) throw new ApiException("not_found", "待识别条目不存在");
|
||||
const r = await rt().ctx.engine.enqueue(
|
||||
"resolveObservation",
|
||||
{ sourceReleaseId: pool.source_release_id, force: true },
|
||||
{ idempotencyKey: `resolve:retry:${params.id}:${Math.floor(Date.now() / 10_000)}`, traceId: randomUUID() },
|
||||
);
|
||||
return { taskId: r.runId };
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,635 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { listSiteDefinitions, redactUrl, type SiteDefinition } from "@umt/adapters";
|
||||
import {
|
||||
CursorQuery,
|
||||
cursorPage,
|
||||
FeedSource,
|
||||
Id,
|
||||
IdParam,
|
||||
type Observation,
|
||||
ReleaseIdentityAssign,
|
||||
type SecretPurpose,
|
||||
Site,
|
||||
type SiteAccount,
|
||||
type SiteAdapter,
|
||||
SiteDefinitionView,
|
||||
SiteSnapshot,
|
||||
type SourceRelease,
|
||||
} from "@umt/contracts";
|
||||
import { sql } from "@umt/db";
|
||||
import { z } from "zod";
|
||||
import { ApiException } from "../../../http/errors.ts";
|
||||
import type { ModuleApp } from "../../types.ts";
|
||||
import { D } from "../defs.ts";
|
||||
import { route } from "../http.ts";
|
||||
import { decodeCursor, jsonb, page, toApi, toApiList } from "../rows.ts";
|
||||
import type { SourcesRuntime } from "../runtime.ts";
|
||||
import { audit, closeDecisions } from "../service/common.ts";
|
||||
import { assignReleaseIdentity } from "../service/identity-admin.ts";
|
||||
|
||||
// 站点、站点账号、Feed、来源资源、观察的管理 API。凭证只写(存 secret,对外只给 secret_ref / 脱敏 URL)。
|
||||
|
||||
const T_SITES = ["sites"];
|
||||
const T_FEEDS = ["feed-sources"];
|
||||
const T_RELEASES = ["source-releases"];
|
||||
|
||||
export function adapterOf(def: SiteDefinition): SiteAdapter {
|
||||
return def.umt.kind === "mteam" ? "mteam_api" : def.umt.kind === "api" ? "api" : "nexusphp";
|
||||
}
|
||||
|
||||
const SECRET_PURPOSE: Record<string, SecretPurpose> = {
|
||||
cookie: "site_cookie",
|
||||
passkey: "site_passkey",
|
||||
api_key: "site_api_key",
|
||||
password: "site_password",
|
||||
};
|
||||
|
||||
const Accepted = z.object({ taskId: Id });
|
||||
|
||||
export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
|
||||
const db = () => rt().ctx.db;
|
||||
|
||||
// ── 站点注册表 ──
|
||||
route(
|
||||
app,
|
||||
{
|
||||
method: "get",
|
||||
path: "/site-definitions",
|
||||
scope: "read",
|
||||
summary: "随产品发布的站点适配器定义(含是否已登记)",
|
||||
tags: T_SITES,
|
||||
response: z.object({ items: z.array(SiteDefinitionView) }),
|
||||
},
|
||||
async () => {
|
||||
const rows = await db().selectFrom("site").select(["id", "key"]).execute();
|
||||
const byKey = new Map(rows.map((r) => [r.key, r.id]));
|
||||
return {
|
||||
items: listSiteDefinitions().map((d) => ({
|
||||
key: d.id,
|
||||
name: d.umt.kind === "nexusphp" ? (d.umt.displayName ?? d.name) : d.umt.displayName,
|
||||
adapter: adapterOf(d),
|
||||
baseUrl: d.domain,
|
||||
shippedEnabled: d.enabled !== false,
|
||||
definitionVersion: d.umt.version,
|
||||
capabilities: d.umt.capabilities,
|
||||
defaultRateLimit: d.umt.rateLimit,
|
||||
authSupported: ("authSupported" in d.umt ? d.umt.authSupported : undefined) ?? [],
|
||||
siteId: byKey.get(d.id) ?? null,
|
||||
})),
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
// ── 站点 ──
|
||||
route(app, D.sites.list, async ({ query }) => {
|
||||
const off = decodeCursor(query.cursor);
|
||||
const rows = await db()
|
||||
.selectFrom("site")
|
||||
.selectAll()
|
||||
.orderBy("key")
|
||||
.offset(off)
|
||||
.limit(query.limit + 1)
|
||||
.execute();
|
||||
const p = page(rows, off, query.limit);
|
||||
return { items: toApiList<Site>(p.items), nextCursor: p.nextCursor };
|
||||
});
|
||||
route(app, D.sites.get, async ({ params }) => toApi(await getSite(params.id)));
|
||||
route(app, D.sites.create, async ({ body, actor }) => {
|
||||
const def = listSiteDefinitions().find((d) => d.id === body.key);
|
||||
if (def && adapterOf(def) !== body.adapter) {
|
||||
throw new ApiException("bad_request", `站点 ${body.key} 的适配器类型应为 ${adapterOf(def)}`);
|
||||
}
|
||||
if (!def && body.adapter !== "generic_rss") {
|
||||
throw new ApiException("bad_request", `注册表中没有站点 ${body.key};未适配站点只能以 generic_rss 登记`);
|
||||
}
|
||||
try {
|
||||
const row = await db()
|
||||
.insertInto("site")
|
||||
.values({
|
||||
key: body.key,
|
||||
name: body.name,
|
||||
adapter: body.adapter,
|
||||
base_url: body.baseUrl,
|
||||
enabled: body.enabled ?? true,
|
||||
rate_limit: jsonb(body.rateLimit ?? {}),
|
||||
config: jsonb(body.config ?? {}),
|
||||
snapshot_enabled: body.snapshotEnabled ?? false,
|
||||
snapshot_interval_sec: body.snapshotIntervalSec ?? null,
|
||||
})
|
||||
.returningAll()
|
||||
.executeTakeFirstOrThrow();
|
||||
await audit(db(), actor, "site.created", { type: "site", id: row.id }, { key: body.key });
|
||||
return toApi(row);
|
||||
} catch (e) {
|
||||
if ((e as { code?: string }).code === "23505") throw new ApiException("conflict", `站点 ${body.key} 已登记`);
|
||||
throw e;
|
||||
}
|
||||
});
|
||||
route(app, D.sites.update, async ({ params, body, actor }) => {
|
||||
await getSite(params.id);
|
||||
const patch: Record<string, unknown> = {};
|
||||
if (body.name !== undefined) patch.name = body.name;
|
||||
if (body.adapter !== undefined) patch.adapter = body.adapter;
|
||||
if (body.baseUrl !== undefined) patch.base_url = body.baseUrl;
|
||||
if (body.rateLimit !== undefined) patch.rate_limit = jsonb(body.rateLimit);
|
||||
if (body.config !== undefined) patch.config = jsonb(body.config);
|
||||
if (body.snapshotEnabled !== undefined) patch.snapshot_enabled = body.snapshotEnabled;
|
||||
if (body.snapshotIntervalSec !== undefined) patch.snapshot_interval_sec = body.snapshotIntervalSec;
|
||||
if (body.enabled !== undefined) return toApi(await setSiteEnabled(params.id, body.enabled, actor));
|
||||
const row = await db()
|
||||
.updateTable("site")
|
||||
.set(patch)
|
||||
.where("id", "=", params.id)
|
||||
.returningAll()
|
||||
.executeTakeFirstOrThrow();
|
||||
await audit(db(), actor, "site.updated", { type: "site", id: params.id }, { fields: Object.keys(patch) });
|
||||
return toApi(row);
|
||||
});
|
||||
for (const enabled of [true, false]) {
|
||||
route(
|
||||
app,
|
||||
{
|
||||
method: "post",
|
||||
path: `/sites/{id}/${enabled ? "enable" : "disable"}`,
|
||||
scope: "admin",
|
||||
summary: enabled ? "开启站点" : "关闭站点(停止网络访问与新候选,历史资源仍可查)",
|
||||
tags: T_SITES,
|
||||
params: IdParam,
|
||||
response: Site,
|
||||
},
|
||||
async ({ params, actor }) => toApi(await setSiteEnabled(params.id, enabled, actor)),
|
||||
);
|
||||
}
|
||||
route(
|
||||
app,
|
||||
{
|
||||
method: "post",
|
||||
path: "/sites/{id}/snapshot",
|
||||
scope: "admin",
|
||||
summary: "立即同步站点快照(受站点限流)",
|
||||
tags: T_SITES,
|
||||
params: IdParam,
|
||||
body: z.object({ maxPages: z.number().int().min(1).max(20).optional() }),
|
||||
response: Accepted,
|
||||
},
|
||||
async ({ params, body }) => {
|
||||
const s = await getSite(params.id);
|
||||
if (!s.enabled) throw new ApiException("precondition_failed", "站点已关闭");
|
||||
const r = await rt().ctx.engine.enqueue(
|
||||
"snapshotSite",
|
||||
{ siteId: s.id, kind: "manual", ...(body.maxPages ? { maxPages: body.maxPages } : {}) },
|
||||
{ idempotencyKey: `snapshotSite:manual:${s.id}:${Math.floor(Date.now() / 10_000)}`, traceId: randomUUID() },
|
||||
);
|
||||
return { taskId: r.runId };
|
||||
},
|
||||
);
|
||||
route(
|
||||
app,
|
||||
{
|
||||
method: "get",
|
||||
path: "/sites/{id}/snapshots",
|
||||
scope: "read",
|
||||
summary: "站点快照/补查记录(覆盖范围与新鲜度)",
|
||||
tags: T_SITES,
|
||||
params: IdParam,
|
||||
query: CursorQuery,
|
||||
response: cursorPage(SiteSnapshot),
|
||||
},
|
||||
async ({ params, query }) => {
|
||||
const off = decodeCursor(query.cursor);
|
||||
const rows = await db()
|
||||
.selectFrom("site_snapshot")
|
||||
.selectAll()
|
||||
.where("site_id", "=", params.id)
|
||||
.orderBy("started_at", "desc")
|
||||
.offset(off)
|
||||
.limit(query.limit + 1)
|
||||
.execute();
|
||||
const p = page(rows, off, query.limit);
|
||||
return { items: toApiList(p.items), nextCursor: p.nextCursor };
|
||||
},
|
||||
);
|
||||
|
||||
// ── 站点账号 ──
|
||||
route(app, D.siteAccounts.list, async ({ query }) => {
|
||||
const off = decodeCursor(query.cursor);
|
||||
let q = db().selectFrom("site_account").selectAll();
|
||||
if (query.siteId) q = q.where("site_id", "=", query.siteId);
|
||||
const rows = await q
|
||||
.orderBy("created_at")
|
||||
.offset(off)
|
||||
.limit(query.limit + 1)
|
||||
.execute();
|
||||
const p = page(rows, off, query.limit);
|
||||
return { items: toApiList<SiteAccount>(p.items), nextCursor: p.nextCursor };
|
||||
});
|
||||
route(app, D.siteAccounts.get, async ({ params }) => {
|
||||
const row = await db().selectFrom("site_account").selectAll().where("id", "=", params.id).executeTakeFirst();
|
||||
if (!row) throw new ApiException("not_found", "站点账号不存在");
|
||||
return toApi(row);
|
||||
});
|
||||
route(app, D.siteAccounts.create, async ({ body, actor }) => {
|
||||
await getSite(body.siteId);
|
||||
const ref = await rt().ctx.secrets.create({
|
||||
purpose: SECRET_PURPOSE[body.authKind]!,
|
||||
plaintext: body.credential,
|
||||
label: `site_account:${body.label}`,
|
||||
});
|
||||
try {
|
||||
const row = await db()
|
||||
.insertInto("site_account")
|
||||
.values({
|
||||
site_id: body.siteId,
|
||||
label: body.label,
|
||||
auth_kind: body.authKind,
|
||||
secret_ref: ref,
|
||||
enabled: body.enabled ?? true,
|
||||
})
|
||||
.returningAll()
|
||||
.executeTakeFirstOrThrow();
|
||||
await audit(db(), actor, "site_account.created", { type: "site_account", id: row.id }, { siteId: body.siteId });
|
||||
return toApi(row);
|
||||
} catch (e) {
|
||||
await rt()
|
||||
.ctx.secrets.remove(ref)
|
||||
.catch(() => {});
|
||||
if ((e as { code?: string }).code === "23505") throw new ApiException("conflict", "同站点已有同名账号");
|
||||
throw e;
|
||||
}
|
||||
});
|
||||
route(app, D.siteAccounts.update, async ({ params, body, actor }) => {
|
||||
const acc = await db().selectFrom("site_account").selectAll().where("id", "=", params.id).executeTakeFirst();
|
||||
if (!acc) throw new ApiException("not_found", "站点账号不存在");
|
||||
const patch: Record<string, unknown> = {};
|
||||
if (body.label !== undefined) patch.label = body.label;
|
||||
if (body.enabled !== undefined) patch.enabled = body.enabled;
|
||||
if (body.authKind !== undefined) patch.auth_kind = body.authKind;
|
||||
if (body.credential !== undefined) {
|
||||
if (acc.secret_ref) await rt().ctx.secrets.replace(acc.secret_ref, body.credential);
|
||||
else {
|
||||
patch.secret_ref = await rt().ctx.secrets.create({
|
||||
purpose: SECRET_PURPOSE[body.authKind ?? acc.auth_kind]!,
|
||||
plaintext: body.credential,
|
||||
label: `site_account:${acc.label}`,
|
||||
});
|
||||
}
|
||||
// 换了凭证:状态回到未验证,关闭"登录失效"待处理,站点健康复位
|
||||
patch.status = "unknown";
|
||||
}
|
||||
const row = await db()
|
||||
.transaction()
|
||||
.execute(async (trx) => {
|
||||
const r = await trx
|
||||
.updateTable("site_account")
|
||||
.set(patch)
|
||||
.where("id", "=", params.id)
|
||||
.returningAll()
|
||||
.executeTakeFirstOrThrow();
|
||||
if (body.credential !== undefined) {
|
||||
await trx
|
||||
.updateTable("site")
|
||||
.set({ health_status: "unknown" })
|
||||
.where("id", "=", acc.site_id)
|
||||
.where("health_status", "=", "failing")
|
||||
.execute();
|
||||
await closeDecisions(
|
||||
rt().ctx,
|
||||
trx,
|
||||
`site-auth:${acc.site_id}`,
|
||||
"resolved",
|
||||
{ by: "credential_updated" },
|
||||
`${actor.type}:${actor.id}`,
|
||||
);
|
||||
}
|
||||
await audit(
|
||||
trx,
|
||||
actor,
|
||||
"site_account.updated",
|
||||
{ type: "site_account", id: params.id },
|
||||
{
|
||||
fields: Object.keys(patch).filter((k) => k !== "secret_ref"),
|
||||
credentialRotated: body.credential !== undefined,
|
||||
},
|
||||
);
|
||||
return r;
|
||||
});
|
||||
return toApi(row);
|
||||
});
|
||||
|
||||
// ── Feed ──
|
||||
route(app, D.feedSources.list, async ({ query }) => {
|
||||
const off = decodeCursor(query.cursor);
|
||||
let q = db().selectFrom("feed_source").selectAll();
|
||||
if (query.siteId) q = q.where("site_id", "=", query.siteId);
|
||||
if (query.health) q = q.where("health_status", "=", query.health);
|
||||
const rows = await q
|
||||
.orderBy("created_at")
|
||||
.offset(off)
|
||||
.limit(query.limit + 1)
|
||||
.execute();
|
||||
const p = page(rows, off, query.limit);
|
||||
return { items: toApiList<FeedSource>(p.items), nextCursor: p.nextCursor };
|
||||
});
|
||||
route(app, D.feedSources.get, async ({ params }) => toApi(await getFeed(params.id)));
|
||||
route(app, D.feedSources.create, async ({ body, actor }) => {
|
||||
if (body.siteId) await getSite(body.siteId);
|
||||
if (body.siteAccountId) await checkAccount(body.siteAccountId, body.siteId ?? null);
|
||||
const ref = await rt().ctx.secrets.create({ purpose: "feed_url", plaintext: body.url, label: `feed:${body.name}` });
|
||||
const row = await db()
|
||||
.insertInto("feed_source")
|
||||
.values({
|
||||
site_id: body.siteId ?? null,
|
||||
site_account_id: body.siteAccountId ?? null,
|
||||
name: body.name,
|
||||
purpose: body.purpose ?? "generic",
|
||||
kind: body.kind ?? "rss",
|
||||
url_secret_ref: ref,
|
||||
url_display: redactUrl(body.url),
|
||||
enabled: body.enabled ?? true,
|
||||
poll_interval_sec: body.pollIntervalSec ?? 900,
|
||||
filter: jsonb(body.filter ?? {}),
|
||||
})
|
||||
.returningAll()
|
||||
.executeTakeFirstOrThrow();
|
||||
await audit(db(), actor, "feed_source.created", { type: "feed_source", id: row.id }, { url: row.url_display });
|
||||
return toApi(row);
|
||||
});
|
||||
route(app, D.feedSources.update, async ({ params, body, actor }) => {
|
||||
const feed = await getFeed(params.id);
|
||||
const patch: Record<string, unknown> = {};
|
||||
if (body.siteId !== undefined) {
|
||||
if (body.siteId) await getSite(body.siteId);
|
||||
patch.site_id = body.siteId;
|
||||
}
|
||||
if (body.siteAccountId !== undefined) {
|
||||
if (body.siteAccountId) await checkAccount(body.siteAccountId, body.siteId ?? feed.site_id);
|
||||
patch.site_account_id = body.siteAccountId;
|
||||
}
|
||||
if (body.name !== undefined) patch.name = body.name;
|
||||
if (body.purpose !== undefined) patch.purpose = body.purpose;
|
||||
if (body.kind !== undefined) patch.kind = body.kind;
|
||||
if (body.enabled !== undefined) {
|
||||
patch.enabled = body.enabled;
|
||||
patch.health_status = body.enabled ? "unknown" : "disabled";
|
||||
}
|
||||
if (body.pollIntervalSec !== undefined) patch.poll_interval_sec = body.pollIntervalSec;
|
||||
if (body.filter !== undefined) patch.filter = jsonb(body.filter);
|
||||
if (body.url !== undefined) {
|
||||
if (feed.url_secret_ref) await rt().ctx.secrets.replace(feed.url_secret_ref, body.url);
|
||||
else
|
||||
patch.url_secret_ref = await rt().ctx.secrets.create({
|
||||
purpose: "feed_url",
|
||||
plaintext: body.url,
|
||||
label: `feed:${feed.name}`,
|
||||
});
|
||||
patch.url_display = redactUrl(body.url);
|
||||
// 换了链接:清空条件请求缓存(游标保留,用于去重)
|
||||
patch.etag = null;
|
||||
patch.last_modified = null;
|
||||
}
|
||||
const row = await db()
|
||||
.updateTable("feed_source")
|
||||
.set(patch)
|
||||
.where("id", "=", params.id)
|
||||
.returningAll()
|
||||
.executeTakeFirstOrThrow();
|
||||
await audit(
|
||||
db(),
|
||||
actor,
|
||||
"feed_source.updated",
|
||||
{ type: "feed_source", id: params.id },
|
||||
{
|
||||
fields: Object.keys(patch).filter((k) => k !== "url_secret_ref"),
|
||||
},
|
||||
);
|
||||
return toApi(row);
|
||||
});
|
||||
for (const enabled of [true, false]) {
|
||||
route(
|
||||
app,
|
||||
{
|
||||
method: "post",
|
||||
path: `/feed-sources/{id}/${enabled ? "enable" : "disable"}`,
|
||||
scope: "admin",
|
||||
summary: enabled ? "启用 Feed" : "停用 Feed",
|
||||
tags: T_FEEDS,
|
||||
params: IdParam,
|
||||
response: FeedSource,
|
||||
},
|
||||
async ({ params, actor }) => {
|
||||
await getFeed(params.id);
|
||||
const row = await db()
|
||||
.updateTable("feed_source")
|
||||
.set({
|
||||
enabled,
|
||||
health_status: enabled ? "unknown" : "disabled",
|
||||
...(enabled ? { next_fetch_at: new Date() } : {}),
|
||||
})
|
||||
.where("id", "=", params.id)
|
||||
.returningAll()
|
||||
.executeTakeFirstOrThrow();
|
||||
await audit(db(), actor, enabled ? "feed_source.enabled" : "feed_source.disabled", {
|
||||
type: "feed_source",
|
||||
id: params.id,
|
||||
});
|
||||
return toApi(row);
|
||||
},
|
||||
);
|
||||
}
|
||||
route(
|
||||
app,
|
||||
{
|
||||
method: "post",
|
||||
path: "/feed-sources/{id}/reset",
|
||||
scope: "admin",
|
||||
summary: "手动恢复:关闭熔断、清空退避(凭证已更新后使用)",
|
||||
tags: T_FEEDS,
|
||||
params: IdParam,
|
||||
response: FeedSource,
|
||||
},
|
||||
async ({ params, actor }) => {
|
||||
const feed = await getFeed(params.id);
|
||||
const cursor = (feed.cursor ?? {}) as { window?: unknown };
|
||||
const row = await db()
|
||||
.transaction()
|
||||
.execute(async (trx) => {
|
||||
const r = await trx
|
||||
.updateTable("feed_source")
|
||||
.set({
|
||||
consecutive_failures: 0,
|
||||
backoff_until: null,
|
||||
last_error: null,
|
||||
health_status: feed.enabled ? "unknown" : "disabled",
|
||||
next_fetch_at: new Date(),
|
||||
cursor: jsonb({ window: cursor.window ?? null }),
|
||||
})
|
||||
.where("id", "=", params.id)
|
||||
.returningAll()
|
||||
.executeTakeFirstOrThrow();
|
||||
if (feed.site_account_id) {
|
||||
await trx
|
||||
.updateTable("site_account")
|
||||
.set({ status: "unknown" })
|
||||
.where("id", "=", feed.site_account_id)
|
||||
.where("status", "in", ["invalid", "expired"])
|
||||
.execute();
|
||||
}
|
||||
await closeDecisions(
|
||||
rt().ctx,
|
||||
trx,
|
||||
`feed-auth:${params.id}`,
|
||||
"resolved",
|
||||
{ by: "reset" },
|
||||
`${actor.type}:${actor.id}`,
|
||||
);
|
||||
await audit(trx, actor, "feed_source.reset", { type: "feed_source", id: params.id });
|
||||
return r;
|
||||
});
|
||||
return toApi(row);
|
||||
},
|
||||
);
|
||||
route(app, D.feedSources.fetchNow, async ({ params }) => {
|
||||
const feed = await getFeed(params.id);
|
||||
if (!feed.enabled) throw new ApiException("precondition_failed", "Feed 已停用");
|
||||
const r = await rt().ctx.engine.enqueue(
|
||||
"ingestFeed",
|
||||
{ feedSourceId: feed.id, manual: true },
|
||||
{ idempotencyKey: `ingestFeed:manual:${feed.id}:${Math.floor(Date.now() / 10_000)}`, traceId: randomUUID() },
|
||||
);
|
||||
return { taskId: r.runId };
|
||||
});
|
||||
|
||||
// ── 来源资源 / 观察 ──
|
||||
route(app, D.sourceReleases.list, async ({ query }) => {
|
||||
const off = decodeCursor(query.cursor);
|
||||
let q = db().selectFrom("source_release").selectAll();
|
||||
if (query.siteId) q = q.where("site_id", "=", query.siteId);
|
||||
if (query.mediaEntityId) q = q.where("media_entity_id", "=", query.mediaEntityId);
|
||||
if (query.resolutionStatus) q = q.where("resolution_status", "=", query.resolutionStatus);
|
||||
if (query.q)
|
||||
q = q.where((eb) =>
|
||||
eb.or([eb("title", "ilike", `%${query.q}%`), sql<boolean>`similarity(title, ${query.q}) >= 0.3`]),
|
||||
);
|
||||
const rows = await q
|
||||
.orderBy("first_seen_at", "desc")
|
||||
.orderBy("id")
|
||||
.offset(off)
|
||||
.limit(query.limit + 1)
|
||||
.execute();
|
||||
const p = page(rows, off, query.limit);
|
||||
return { items: toApiList<SourceRelease>(p.items), nextCursor: p.nextCursor };
|
||||
});
|
||||
route(app, D.sourceReleases.get, async ({ params }) => {
|
||||
const row = await db().selectFrom("source_release").selectAll().where("id", "=", params.id).executeTakeFirst();
|
||||
if (!row) throw new ApiException("not_found", "来源资源不存在");
|
||||
return toApi(row);
|
||||
});
|
||||
route(
|
||||
app,
|
||||
{
|
||||
method: "post",
|
||||
path: "/source-releases/{id}/identity",
|
||||
scope: "organize.execute",
|
||||
summary: "人工指定来源资源的作品/内容单元(加人工锁,自动识别不再改写)",
|
||||
tags: T_RELEASES,
|
||||
params: IdParam,
|
||||
body: ReleaseIdentityAssign,
|
||||
response: z.object({ mediaEntityId: Id, contentUnitIds: z.array(Id) }),
|
||||
},
|
||||
async ({ params, body, actor }) => assignReleaseIdentity(rt(), params.id, body, actor),
|
||||
);
|
||||
route(
|
||||
app,
|
||||
{
|
||||
method: "post",
|
||||
path: "/source-releases/{id}/unlock",
|
||||
scope: "organize.execute",
|
||||
summary: "解除来源资源的人工锁并重新识别",
|
||||
tags: T_RELEASES,
|
||||
params: IdParam,
|
||||
response: Accepted,
|
||||
},
|
||||
async ({ params, actor }) => {
|
||||
const row = await db()
|
||||
.updateTable("source_release")
|
||||
.set({ resolution_locked: false })
|
||||
.where("id", "=", params.id)
|
||||
.returning(["id", "site_id"])
|
||||
.executeTakeFirst();
|
||||
if (!row) throw new ApiException("not_found", "来源资源不存在");
|
||||
await audit(db(), actor, "identity.release_unlocked", { type: "source_release", id: params.id });
|
||||
const r = await rt().ctx.engine.enqueue(
|
||||
"resolveObservation",
|
||||
{ sourceReleaseId: params.id, force: true },
|
||||
{ idempotencyKey: `resolve:unlock:${params.id}:${Date.now()}`, traceId: randomUUID() },
|
||||
);
|
||||
return { taskId: r.runId };
|
||||
},
|
||||
);
|
||||
route(app, D.observations.list, async ({ query }) => {
|
||||
const off = decodeCursor(query.cursor);
|
||||
let q = db().selectFrom("observation").selectAll();
|
||||
if (query.feedId) q = q.where("feed_id", "=", query.feedId);
|
||||
if (query.status) q = q.where("status", "=", query.status);
|
||||
const rows = await q
|
||||
.orderBy("first_seen_at", "desc")
|
||||
.orderBy("id")
|
||||
.offset(off)
|
||||
.limit(query.limit + 1)
|
||||
.execute();
|
||||
const p = page(rows, off, query.limit);
|
||||
return { items: toApiList<Observation>(p.items), nextCursor: p.nextCursor };
|
||||
});
|
||||
route(app, D.observations.get, async ({ params }) => {
|
||||
const row = await db().selectFrom("observation").selectAll().where("id", "=", params.id).executeTakeFirst();
|
||||
if (!row) throw new ApiException("not_found", "观察不存在");
|
||||
return toApi(row);
|
||||
});
|
||||
|
||||
// ── 内部辅助 ──
|
||||
async function getSite(id: string) {
|
||||
const row = await db().selectFrom("site").selectAll().where("id", "=", id).executeTakeFirst();
|
||||
if (!row) throw new ApiException("not_found", "站点不存在");
|
||||
return row;
|
||||
}
|
||||
async function getFeed(id: string) {
|
||||
const row = await db().selectFrom("feed_source").selectAll().where("id", "=", id).executeTakeFirst();
|
||||
if (!row) throw new ApiException("not_found", "Feed 不存在");
|
||||
return row;
|
||||
}
|
||||
async function checkAccount(accountId: string, siteId: string | null) {
|
||||
const acc = await db()
|
||||
.selectFrom("site_account")
|
||||
.select(["site_id"])
|
||||
.where("id", "=", accountId)
|
||||
.executeTakeFirst();
|
||||
if (!acc) throw new ApiException("bad_request", "站点账号不存在");
|
||||
if (siteId && acc.site_id !== siteId) throw new ApiException("bad_request", "站点账号不属于该站点");
|
||||
}
|
||||
async function setSiteEnabled(
|
||||
id: string,
|
||||
enabled: boolean,
|
||||
actor: { type: "user" | "agent" | "system" | "api_key"; id: string | null },
|
||||
) {
|
||||
await getSite(id);
|
||||
return db()
|
||||
.transaction()
|
||||
.execute(async (trx) => {
|
||||
const row = await trx
|
||||
.updateTable("site")
|
||||
.set({ enabled, health_status: enabled ? "unknown" : "disabled" })
|
||||
.where("id", "=", id)
|
||||
.returningAll()
|
||||
.executeTakeFirstOrThrow();
|
||||
// 历史资源仍可查,标注站点关闭
|
||||
await trx
|
||||
.updateTable("source_release")
|
||||
.set({ availability: enabled ? "active" : "site_disabled" })
|
||||
.where("site_id", "=", id)
|
||||
.where("availability", "=", enabled ? "site_disabled" : "active")
|
||||
.execute();
|
||||
await audit(trx, actor, enabled ? "site.enabled" : "site.disabled", { type: "site", id });
|
||||
return row;
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
import { Id, WorkbenchReleaseDetail, WorkbenchReleasePage, WorkbenchReleaseQuery } from "@umt/contracts";
|
||||
import { z } from "zod";
|
||||
import type { ModuleApp } from "../../types.ts";
|
||||
import { route } from "../http.ts";
|
||||
import type { SourcesRuntime } from "../runtime.ts";
|
||||
import { listSiteReleases, releaseDetail } from "../service/workbench.ts";
|
||||
|
||||
// 站点工作台(整理视图)数据 API。库存叠加(已有资产/Slot/下载)不在这里:条目带 entity.id,
|
||||
// 前端按作品 ID 二次查询 W2-C 的媒体库端点。
|
||||
|
||||
const T = ["site-workbench"];
|
||||
|
||||
export function workbenchRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
|
||||
route(
|
||||
app,
|
||||
{
|
||||
method: "get",
|
||||
path: "/sites/{id}/releases",
|
||||
scope: "read",
|
||||
summary: "站点资源列表/搜索(快照优先,必要时实时补查;标注来源与更新时间)",
|
||||
tags: T,
|
||||
params: z.object({ id: Id }),
|
||||
query: WorkbenchReleaseQuery,
|
||||
response: WorkbenchReleasePage,
|
||||
},
|
||||
async ({ params, query }) => listSiteReleases(rt(), params.id, query),
|
||||
);
|
||||
route(
|
||||
app,
|
||||
{
|
||||
method: "get",
|
||||
path: "/sites/{id}/releases/{rid}",
|
||||
scope: "read",
|
||||
summary: "站点资源详情(refresh=true 时实时复核详情;失败保留既有数据)",
|
||||
tags: T,
|
||||
params: z.object({ id: Id, rid: Id }),
|
||||
query: z.object({ refresh: z.enum(["true", "false"]).optional() }),
|
||||
response: WorkbenchReleaseDetail,
|
||||
},
|
||||
async ({ params, query }) => releaseDetail(rt(), params.id, params.rid, query.refresh === "true"),
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
// db 行(snake_case、Date)→ contracts(camelCase、ISO 串)。只做命名与时间转换,取值语义不变。
|
||||
|
||||
const camel = (s: string) => s.replace(/_([a-z0-9])/g, (_, c: string) => c.toUpperCase());
|
||||
|
||||
function conv(v: unknown): unknown {
|
||||
if (v instanceof Date) return v.toISOString();
|
||||
return v;
|
||||
}
|
||||
|
||||
/** 通用行转换:键转 camelCase,Date → ISO 串;jsonb/数组原样保留 */
|
||||
export function toApi<T>(row: object): T {
|
||||
const out: Record<string, unknown> = {};
|
||||
for (const [k, v] of Object.entries(row)) out[camel(k)] = conv(v);
|
||||
return out as T;
|
||||
}
|
||||
|
||||
export const toApiList = <T>(rows: readonly object[]): T[] => rows.map((r) => toApi<T>(r));
|
||||
|
||||
/** 偏移游标(不透明 base64url);列表规模小,足够用 */
|
||||
export function encodeCursor(offset: number): string {
|
||||
return Buffer.from(String(offset), "utf8").toString("base64url");
|
||||
}
|
||||
|
||||
export function decodeCursor(cursor: string | undefined): number {
|
||||
if (!cursor) return 0;
|
||||
const n = Number(Buffer.from(cursor, "base64url").toString("utf8"));
|
||||
return Number.isSafeInteger(n) && n >= 0 ? n : 0;
|
||||
}
|
||||
|
||||
/** 取 limit+1 条判断是否还有下一页 */
|
||||
export function page<T>(rows: T[], offset: number, limit: number): { items: T[]; nextCursor: string | null } {
|
||||
const more = rows.length > limit;
|
||||
return { items: more ? rows.slice(0, limit) : rows, nextCursor: more ? encodeCursor(offset + limit) : null };
|
||||
}
|
||||
|
||||
/** jsonb 写入:数组/对象都先 JSON.stringify(pg 会把 JS 数组当 PG 数组) */
|
||||
export const jsonb = (v: unknown): string => JSON.stringify(v ?? null);
|
||||
@@ -0,0 +1,143 @@
|
||||
import {
|
||||
createRssFetcher,
|
||||
createSiteAdapter,
|
||||
createSiteRateLimiter,
|
||||
createTmdbClient,
|
||||
getSiteDefinition,
|
||||
type RssFetcher,
|
||||
type SecretResolver,
|
||||
type SiteAdapter,
|
||||
type SiteDefinition,
|
||||
type SiteRateLimiter,
|
||||
type TmdbClient,
|
||||
} from "@umt/adapters";
|
||||
import type { SiteRow } from "@umt/db";
|
||||
import type { ServerContext } from "../types.ts";
|
||||
|
||||
// 模块运行时依赖:站点适配器与限流器(每站一个实例)、RSS 抓取器、TMDB 客户端、时钟。
|
||||
// 测试通过 createSourcesIdentityModule(overrides) 注入 fetch / TMDB / 时钟。
|
||||
|
||||
export interface RuntimeOverrides {
|
||||
/** 站点与 RSS 请求使用的 fetch(测试指向 mock) */
|
||||
fetch?: typeof globalThis.fetch;
|
||||
/** TMDB 客户端工厂;返回 null 表示未配置(只做本地召回) */
|
||||
tmdb?: (() => TmdbClient | null) | undefined;
|
||||
now?: () => Date;
|
||||
/** RSS 抓取器选项覆盖(退避/熔断阈值) */
|
||||
rss?: Parameters<typeof createRssFetcher>[0];
|
||||
/** 调度 tick 间隔(毫秒),默认 30s */
|
||||
schedulerIntervalMs?: number;
|
||||
}
|
||||
|
||||
interface CachedAdapter {
|
||||
sig: string;
|
||||
limiter: SiteRateLimiter;
|
||||
adapter: SiteAdapter;
|
||||
}
|
||||
|
||||
export class SourcesRuntime {
|
||||
readonly now: () => Date;
|
||||
readonly rss: RssFetcher;
|
||||
private readonly adapters = new Map<string, CachedAdapter>();
|
||||
private tmdbClient: TmdbClient | null | undefined;
|
||||
|
||||
constructor(
|
||||
readonly ctx: ServerContext,
|
||||
readonly overrides: RuntimeOverrides = {},
|
||||
) {
|
||||
this.now = overrides.now ?? (() => new Date());
|
||||
this.rss = createRssFetcher({
|
||||
...(overrides.fetch ? { fetch: overrides.fetch } : {}),
|
||||
now: this.now,
|
||||
...overrides.rss,
|
||||
});
|
||||
}
|
||||
|
||||
/** TMDB:优先注入;否则读环境变量 TMDB_API_KEY / TMDB_ACCESS_TOKEN(W2-A 设置页接管前的过渡) */
|
||||
tmdb(): TmdbClient | null {
|
||||
if (this.tmdbClient !== undefined) return this.tmdbClient;
|
||||
if (this.overrides.tmdb) {
|
||||
this.tmdbClient = this.overrides.tmdb();
|
||||
return this.tmdbClient;
|
||||
}
|
||||
const env = (k: string) => Deno.env.get(k);
|
||||
const apiKey = env("TMDB_API_KEY");
|
||||
const accessToken = env("TMDB_ACCESS_TOKEN");
|
||||
const baseUrl = env("TMDB_BASE_URL");
|
||||
this.tmdbClient =
|
||||
apiKey || accessToken
|
||||
? createTmdbClient({
|
||||
...(apiKey ? { apiKey } : {}),
|
||||
...(accessToken ? { accessToken } : {}),
|
||||
...(baseUrl ? { baseUrl } : {}),
|
||||
...(this.overrides.fetch ? { fetch: this.overrides.fetch } : {}),
|
||||
})
|
||||
: null;
|
||||
return this.tmdbClient;
|
||||
}
|
||||
|
||||
/**
|
||||
* 站点适配器:每站一个实例,列表/搜索/详情/下载共享同一个限流器。
|
||||
* 站点限流参数或基址变化时重建(签名比较)。
|
||||
*/
|
||||
siteAdapter(site: Pick<SiteRow, "id" | "key" | "base_url" | "rate_limit" | "config">): SiteAdapter {
|
||||
const def = getSiteDefinition(site.key);
|
||||
if (!def) throw new Error(`站点 ${site.key} 没有随产品发布的适配器定义`);
|
||||
const sig = JSON.stringify([site.base_url, site.rate_limit, site.config]);
|
||||
const hit = this.adapters.get(site.id);
|
||||
if (hit && hit.sig === sig) return hit.adapter;
|
||||
const limiter = createSiteRateLimiter(limiterOptions(def, site.rate_limit));
|
||||
const cfg = (site.config ?? {}) as { apiBaseUrl?: string };
|
||||
const baseUrl =
|
||||
def.umt.kind === "mteam" ? cfg.apiBaseUrl : sameUrl(site.base_url, def.domain) ? undefined : site.base_url;
|
||||
const adapter = createSiteAdapter(def, {
|
||||
secrets: this.siteSecrets(site.id),
|
||||
limiter,
|
||||
now: this.now,
|
||||
...(this.overrides.fetch ? { fetch: this.overrides.fetch } : {}),
|
||||
...(baseUrl ? { baseUrl } : {}),
|
||||
});
|
||||
this.adapters.set(site.id, { sig, limiter, adapter });
|
||||
return adapter;
|
||||
}
|
||||
|
||||
/** 已建立的站点限流器(测试与诊断用) */
|
||||
limiterOf(siteId: string): SiteRateLimiter | null {
|
||||
return this.adapters.get(siteId)?.limiter ?? null;
|
||||
}
|
||||
|
||||
/** 站点凭证:取该站第一个启用且未失效的账号,按 auth_kind 对应适配器要求的凭证类型;明文不缓存 */
|
||||
private siteSecrets(siteId: string): SecretResolver {
|
||||
const { db, secrets } = this.ctx;
|
||||
return {
|
||||
async resolve({ kind }) {
|
||||
const want = kind === "cookie" ? ["cookie"] : kind === "apiKey" ? ["api_key"] : ["passkey"];
|
||||
const acc = await db
|
||||
.selectFrom("site_account")
|
||||
.select(["secret_ref"])
|
||||
.where("site_id", "=", siteId)
|
||||
.where("enabled", "=", true)
|
||||
.where("status", "in", ["unknown", "active"])
|
||||
.where("auth_kind", "in", want as never)
|
||||
.where("secret_ref", "is not", null)
|
||||
.orderBy("created_at")
|
||||
.executeTakeFirst();
|
||||
if (!acc?.secret_ref) return null;
|
||||
return secrets.resolve(acc.secret_ref);
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function sameUrl(a: string, b: string): boolean {
|
||||
return a.replace(/\/+$/, "") === b.replace(/\/+$/, "");
|
||||
}
|
||||
|
||||
/** 站点行 rate_limit 优先,缺省用定义里的 umt.rateLimit */
|
||||
export function limiterOptions(def: SiteDefinition, rateLimit: unknown): { minIntervalMs: number; burst: number } {
|
||||
const rl = (rateLimit ?? {}) as { requestsPerMinute?: number; minIntervalMs?: number; concurrency?: number };
|
||||
const minIntervalMs =
|
||||
rl.minIntervalMs ??
|
||||
(rl.requestsPerMinute ? Math.ceil(60_000 / rl.requestsPerMinute) : def.umt.rateLimit.minIntervalMs);
|
||||
return { minIntervalMs, burst: rl.concurrency ?? def.umt.rateLimit.burst };
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { sql } from "@umt/db";
|
||||
import type { SourcesRuntime } from "./runtime.ts";
|
||||
|
||||
// worker 后台调度:按 Feed 轮询策略发 feed.due(outbox → ingestFeed)、按站点快照周期入队 snapshotSite、
|
||||
// 按待识别池 next_review_at 入队复查。幂等键按时间槽生成,多 worker 并发也不会重复。
|
||||
|
||||
export interface TickResult {
|
||||
feedsDue: number;
|
||||
snapshotsDue: number;
|
||||
reviewsDue: number;
|
||||
skipped?: string;
|
||||
}
|
||||
|
||||
const REVIEW_LEASE_MS = 10 * 60_000;
|
||||
|
||||
export async function schedulerTick(rt: SourcesRuntime): Promise<TickResult> {
|
||||
const { ctx } = rt;
|
||||
const automation = await ctx.settings.get("automation");
|
||||
// 恢复后只读对账模式:禁止自动写入
|
||||
if (automation.readOnlyReconcile) return { feedsDue: 0, snapshotsDue: 0, reviewsDue: 0, skipped: "read_only" };
|
||||
const now = rt.now();
|
||||
|
||||
// Feed:到期(next_fetch_at 空或已到)且不在退避期、站点(若有)开启
|
||||
const feedsDue = await ctx.db.transaction().execute(async (trx) => {
|
||||
const due = await sql<{ id: string; poll_interval_sec: number }>`
|
||||
select f.id, f.poll_interval_sec from feed_source f left join site s on s.id = f.site_id
|
||||
where f.enabled and (s.id is null or s.enabled)
|
||||
and (f.next_fetch_at is null or f.next_fetch_at <= ${now})
|
||||
and (f.backoff_until is null or f.backoff_until <= ${now})
|
||||
and f.health_status <> 'circuit_open'
|
||||
order by f.next_fetch_at nulls first limit 100
|
||||
for update of f skip locked`.execute(trx);
|
||||
for (const f of due.rows) {
|
||||
const slot = Math.floor(now.getTime() / 1000 / f.poll_interval_sec);
|
||||
await ctx.outbox.emit(
|
||||
trx,
|
||||
"feed.due",
|
||||
{ feedId: f.id, reason: "schedule" },
|
||||
{ idempotencyKey: `feed.due:${f.id}:${slot}` },
|
||||
);
|
||||
await trx
|
||||
.updateTable("feed_source")
|
||||
.set({ next_fetch_at: new Date(now.getTime() + f.poll_interval_sec * 1000) })
|
||||
.where("id", "=", f.id)
|
||||
.execute();
|
||||
}
|
||||
return due.rows.length;
|
||||
});
|
||||
|
||||
// 站点快照:开启快照且到周期
|
||||
const sites = await sql<{ id: string; snapshot_interval_sec: number }>`
|
||||
select id, snapshot_interval_sec from site
|
||||
where enabled and snapshot_enabled and snapshot_interval_sec is not null
|
||||
and (last_snapshot_at is null or last_snapshot_at + make_interval(secs => snapshot_interval_sec) <= ${now})`.execute(
|
||||
ctx.db,
|
||||
);
|
||||
for (const s of sites.rows) {
|
||||
const slot = Math.floor(now.getTime() / 1000 / s.snapshot_interval_sec);
|
||||
await ctx.engine.enqueue(
|
||||
"snapshotSite",
|
||||
{ siteId: s.id, kind: "periodic" },
|
||||
{ idempotencyKey: `snapshotSite:${s.id}:${slot}`, traceId: randomUUID() },
|
||||
);
|
||||
}
|
||||
|
||||
// 待识别池复查:到期的先顺延一个租约,避免下个 tick 重复入队
|
||||
const reviews = await ctx.db.transaction().execute(async (trx) => {
|
||||
const rows = await sql<{ id: string; source_release_id: string; attempts: number }>`
|
||||
select id, source_release_id, attempts from identity_candidate
|
||||
where status = 'pending' and next_review_at is not null and next_review_at <= ${now}
|
||||
order by next_review_at limit 50 for update skip locked`.execute(trx);
|
||||
for (const r of rows.rows) {
|
||||
await trx
|
||||
.updateTable("identity_candidate")
|
||||
.set({ next_review_at: new Date(now.getTime() + REVIEW_LEASE_MS) })
|
||||
.where("id", "=", r.id)
|
||||
.execute();
|
||||
}
|
||||
return rows.rows;
|
||||
});
|
||||
for (const r of reviews) {
|
||||
await ctx.engine.enqueue(
|
||||
"resolveObservation",
|
||||
{ sourceReleaseId: r.source_release_id, force: true },
|
||||
{ idempotencyKey: `review:${r.id}:${r.attempts}`, traceId: randomUUID() },
|
||||
);
|
||||
}
|
||||
return { feedsDue, snapshotsDue: sites.rows.length, reviewsDue: reviews.length };
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
import type { ActorType, DecisionKind, DecisionOption, Severity } from "@umt/contracts";
|
||||
import { type DB, type Kysely, sql, type Transaction } from "@umt/db";
|
||||
import type { ServerContext } from "../../types.ts";
|
||||
import { jsonb } from "../rows.ts";
|
||||
|
||||
export type Exec = Kysely<DB> | Transaction<DB>;
|
||||
|
||||
export interface Actor {
|
||||
type: ActorType;
|
||||
id: string | null;
|
||||
}
|
||||
|
||||
export const SYSTEM: Actor = { type: "system", id: "sources-identity" };
|
||||
|
||||
/** 审计:只追加 */
|
||||
export async function audit(
|
||||
exec: Exec,
|
||||
actor: Actor,
|
||||
action: string,
|
||||
target: { type: string; id: string | null },
|
||||
payload: Record<string, unknown> = {},
|
||||
): Promise<void> {
|
||||
await exec
|
||||
.insertInto("audit_event")
|
||||
.values({
|
||||
actor_type: actor.type,
|
||||
actor_id: actor.id,
|
||||
action,
|
||||
target_type: target.type,
|
||||
target_id: target.id,
|
||||
payload: jsonb(payload),
|
||||
})
|
||||
.execute();
|
||||
}
|
||||
|
||||
export interface DecisionInput {
|
||||
kind: DecisionKind;
|
||||
severity?: Severity;
|
||||
title: string;
|
||||
summary?: string | null;
|
||||
subjectType: string;
|
||||
subjectId: string | null;
|
||||
dedupeKey: string;
|
||||
evidence?: Record<string, unknown>;
|
||||
options?: DecisionOption[];
|
||||
nextReviewAt?: Date | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 开一个待处理事项;同 dedupeKey 已有未关闭条目时只刷新证据与选项(不重复通知)。
|
||||
* 新建时同事务发 decision.created。
|
||||
*/
|
||||
export async function openDecision(
|
||||
ctx: ServerContext,
|
||||
trx: Exec,
|
||||
d: DecisionInput,
|
||||
): Promise<{ id: string; created: boolean }> {
|
||||
const row = await sql<{ id: string; inserted: boolean }>`
|
||||
insert into decision (kind, severity, title, summary, subject_type, subject_id, dedupe_key, evidence, options, next_review_at)
|
||||
values (${d.kind}, ${d.severity ?? "warning"}, ${d.title}, ${d.summary ?? null}, ${d.subjectType}, ${d.subjectId},
|
||||
${d.dedupeKey}, ${jsonb(d.evidence ?? {})}::jsonb, ${jsonb(d.options ?? [])}::jsonb, ${d.nextReviewAt ?? null})
|
||||
on conflict (dedupe_key) where status in ('open', 'investigating', 'awaiting_user')
|
||||
do update set evidence = excluded.evidence, options = excluded.options, summary = excluded.summary,
|
||||
title = excluded.title, attempts = decision.attempts + 1
|
||||
returning id, (xmax = 0) as inserted`.execute(trx);
|
||||
const r = row.rows[0]!;
|
||||
if (r.inserted) {
|
||||
await ctx.outbox.emit(
|
||||
trx,
|
||||
"decision.created",
|
||||
{ decisionId: r.id, kind: d.kind, severity: d.severity ?? "warning", title: d.title },
|
||||
{ idempotencyKey: `decision.created:${r.id}` },
|
||||
);
|
||||
}
|
||||
return { id: r.id, created: r.inserted };
|
||||
}
|
||||
|
||||
/** 关闭同 dedupeKey 的未关闭事项(身份已被人工/自动确认等),逐条发 decision.updated */
|
||||
export async function closeDecisions(
|
||||
ctx: ServerContext,
|
||||
trx: Exec,
|
||||
dedupeKey: string,
|
||||
status: "resolved" | "dismissed",
|
||||
resolution: Record<string, unknown>,
|
||||
by: string,
|
||||
): Promise<string[]> {
|
||||
const rows = await trx
|
||||
.updateTable("decision")
|
||||
.set({ status, resolution: jsonb(resolution), resolved_by: by, resolved_at: new Date() })
|
||||
.where("dedupe_key", "=", dedupeKey)
|
||||
.where("status", "in", ["open", "investigating", "awaiting_user"])
|
||||
.returning(["id"])
|
||||
.execute();
|
||||
for (const r of rows) {
|
||||
await ctx.outbox.emit(
|
||||
trx,
|
||||
"decision.updated",
|
||||
{ decisionId: r.id, status },
|
||||
{
|
||||
idempotencyKey: `decision.updated:${r.id}:${status}`,
|
||||
},
|
||||
);
|
||||
}
|
||||
return rows.map((r) => r.id);
|
||||
}
|
||||
|
||||
/** 截断长文本(描述等只留摘录) */
|
||||
export const clip = (s: string | undefined | null, n = 2000): string | null =>
|
||||
s == null ? null : s.length > n ? `${s.slice(0, n)}…` : s;
|
||||
@@ -0,0 +1,221 @@
|
||||
import type { MetadataCandidate } from "@umt/adapters";
|
||||
import type { AliasKind, AliasSource, EvidenceLevel, MediaKind } from "@umt/contracts";
|
||||
import { sql } from "@umt/db";
|
||||
import { normalizeTitle, type ParsedRelease } from "@umt/domain";
|
||||
import { toContractMediaKind } from "@umt/mapping";
|
||||
import { jsonb } from "../rows.ts";
|
||||
import type { Actor, Exec } from "./common.ts";
|
||||
|
||||
// 本地作品 / 内容单元 / 外部身份 / 别名的写入辅助。所有写入在调用方事务内完成。
|
||||
|
||||
/** 沿 merged_into_id 找到最终实体(防环,最多 10 跳);不存在返回 null */
|
||||
export async function resolveEntityId(exec: Exec, id: string): Promise<string | null> {
|
||||
let cur = id;
|
||||
for (let i = 0; i < 10; i++) {
|
||||
const row = await exec
|
||||
.selectFrom("media_entity")
|
||||
.select(["id", "status", "merged_into_id"])
|
||||
.where("id", "=", cur)
|
||||
.executeTakeFirst();
|
||||
if (!row) return null;
|
||||
if (row.status !== "merged" || !row.merged_into_id) return row.id;
|
||||
cur = row.merged_into_id;
|
||||
}
|
||||
return cur;
|
||||
}
|
||||
|
||||
/** 已接受的外部身份 → 本地作品(跟随合并跳转) */
|
||||
export async function entityByExternal(
|
||||
exec: Exec,
|
||||
ref: { provider: string; entityType: string; externalId: string },
|
||||
): Promise<string | null> {
|
||||
const row = await exec
|
||||
.selectFrom("external_identity")
|
||||
.select(["media_entity_id"])
|
||||
.where("provider", "=", ref.provider)
|
||||
.where("entity_type", "=", ref.entityType as never)
|
||||
.where("external_id", "=", ref.externalId)
|
||||
.where("status", "=", "accepted")
|
||||
.where("media_entity_id", "is not", null)
|
||||
.executeTakeFirst();
|
||||
return row?.media_entity_id ? resolveEntityId(exec, row.media_entity_id) : null;
|
||||
}
|
||||
|
||||
export async function addAlias(
|
||||
exec: Exec,
|
||||
entityId: string,
|
||||
alias: string,
|
||||
kind: AliasKind,
|
||||
source: AliasSource,
|
||||
language: string | null = null,
|
||||
locked = false,
|
||||
): Promise<void> {
|
||||
const normalized = normalizeTitle(alias);
|
||||
if (!normalized) return;
|
||||
await sql`insert into title_alias (media_entity_id, content_unit_id, alias, normalized, language, kind, source, locked)
|
||||
values (${entityId}, null, ${alias}, ${normalized}, ${language}, ${kind}, ${source}, ${locked})
|
||||
on conflict (media_entity_id, content_unit_id, normalized, kind) do nothing`.execute(exec);
|
||||
}
|
||||
|
||||
/** 写一条外部身份(accepted)+ 修订 created;同一外部身份已被其它作品接受时返回 null(交给调用方处理冲突) */
|
||||
export async function acceptIdentity(
|
||||
exec: Exec,
|
||||
entityId: string,
|
||||
ref: { provider: string; entityType: string; externalId: string },
|
||||
level: EvidenceLevel,
|
||||
evidence: Record<string, unknown>,
|
||||
actor: Actor,
|
||||
opts: { locked?: boolean; reason?: string | null } = {},
|
||||
): Promise<string | null> {
|
||||
const ins = await sql<{ id: string }>`
|
||||
insert into external_identity (media_entity_id, provider, entity_type, external_id, status, evidence_level, evidence, locked, valid_from, verified_at, verify_status)
|
||||
values (${entityId}, ${ref.provider}, ${ref.entityType}, ${ref.externalId}, 'accepted', ${level}, ${jsonb(evidence)}::jsonb,
|
||||
${opts.locked ?? false}, now(), now(), 'ok')
|
||||
on conflict do nothing
|
||||
returning id`.execute(exec);
|
||||
const id = ins.rows[0]?.id;
|
||||
if (!id) return null;
|
||||
await exec
|
||||
.insertInto("identity_revision")
|
||||
.values({
|
||||
external_identity_id: id,
|
||||
revision: 1,
|
||||
action: "created",
|
||||
from_status: null,
|
||||
to_status: "accepted",
|
||||
after: jsonb({ ...ref, mediaEntityId: entityId, evidenceLevel: level, locked: opts.locked ?? false }),
|
||||
actor_type: actor.type,
|
||||
actor_id: actor.id,
|
||||
reason: opts.reason ?? null,
|
||||
})
|
||||
.execute();
|
||||
return id;
|
||||
}
|
||||
|
||||
/** 综艺判定:站点分类名含"综艺"或标题带期号/播出日期 */
|
||||
export function looksLikeVariety(parsed: ParsedRelease | null, categoryName: string | null): boolean {
|
||||
if (categoryName && /综艺|variety|show/i.test(categoryName)) return true;
|
||||
return !!parsed && (parsed.issue !== undefined || parsed.airDate !== undefined);
|
||||
}
|
||||
|
||||
/**
|
||||
* 按 TMDB 详情建本地作品:外部身份 accepted(+IMDb)、别名(原名/译名/别名)。
|
||||
* 已有同一 TMDB 身份的作品(含并发创建)直接复用。
|
||||
*/
|
||||
export async function createEntityFromTmdb(
|
||||
exec: Exec,
|
||||
m: MetadataCandidate,
|
||||
opts: { level: EvidenceLevel; evidence: Record<string, unknown>; actor: Actor; variety?: boolean; locked?: boolean },
|
||||
): Promise<{ entityId: string; created: boolean }> {
|
||||
const ref = { provider: "tmdb", entityType: m.entityType, externalId: m.externalId };
|
||||
const existing = await entityByExternal(exec, ref);
|
||||
if (existing) return { entityId: existing, created: false };
|
||||
const kind: MediaKind = toContractMediaKind(m.entityType, { variety: !!opts.variety });
|
||||
const entity = await exec
|
||||
.insertInto("media_entity")
|
||||
.values({
|
||||
kind,
|
||||
title: m.title,
|
||||
original_title: m.originalTitle,
|
||||
year: m.year,
|
||||
status: "active",
|
||||
metadata: jsonb({}),
|
||||
})
|
||||
.returning(["id"])
|
||||
.executeTakeFirstOrThrow();
|
||||
const idRow = await acceptIdentity(exec, entity.id, ref, opts.level, opts.evidence, opts.actor, {
|
||||
locked: opts.locked ?? false,
|
||||
});
|
||||
if (!idRow) {
|
||||
// 并发:另一个事务刚接受了同一身份。回滚本事务的新作品由调用方重试处理
|
||||
throw new IdentityConflictError(ref);
|
||||
}
|
||||
if (m.externalIds?.imdb) {
|
||||
await acceptIdentity(
|
||||
exec,
|
||||
entity.id,
|
||||
{ provider: "imdb", entityType: m.entityType, externalId: m.externalIds.imdb },
|
||||
opts.level,
|
||||
{ via: "tmdb", tmdbId: m.externalId },
|
||||
opts.actor,
|
||||
);
|
||||
}
|
||||
await seedAliases(exec, entity.id, m);
|
||||
return { entityId: entity.id, created: true };
|
||||
}
|
||||
|
||||
export async function seedAliases(exec: Exec, entityId: string, m: MetadataCandidate): Promise<void> {
|
||||
await addAlias(exec, entityId, m.title, "translated", "tmdb", null);
|
||||
if (m.originalTitle) await addAlias(exec, entityId, m.originalTitle, "original", "tmdb", m.originalLanguage);
|
||||
for (const t of m.titles) {
|
||||
if (t.kind === "localized") await addAlias(exec, entityId, t.text, "translated", "tmdb", t.lang);
|
||||
}
|
||||
for (const a of m.aliases.slice(0, 50)) await addAlias(exec, entityId, a, "other", "tmdb", null);
|
||||
}
|
||||
|
||||
export class IdentityConflictError extends Error {
|
||||
constructor(readonly ref: { provider: string; entityType: string; externalId: string }) {
|
||||
super(`外部身份 ${ref.provider}:${ref.entityType}/${ref.externalId} 已被其它作品接受`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 按解析出的季/集找或建内容单元(临时 provisional 允许)。电影返回空。
|
||||
* 返回:集单元(有集号时)或季单元;primary 为写入 source_release.content_unit_id 的那个。
|
||||
*/
|
||||
export async function ensureUnits(
|
||||
exec: Exec,
|
||||
entityId: string,
|
||||
kind: MediaKind,
|
||||
parsed: ParsedRelease | null,
|
||||
): Promise<{ unitIds: string[]; primary: string | null }> {
|
||||
if (!parsed || kind === "movie" || parsed.season === undefined) return { unitIds: [], primary: null };
|
||||
const seasons: number[] = [];
|
||||
const last = parsed.seasonEnd ?? parsed.season;
|
||||
for (let s = parsed.season; s <= last && seasons.length < 50; s++) seasons.push(s);
|
||||
const seasonIds: string[] = [];
|
||||
for (const s of seasons) seasonIds.push(await findOrCreateUnit(exec, entityId, "season", s, null, null));
|
||||
if (parsed.episodes && seasons.length === 1) {
|
||||
const ids: string[] = [];
|
||||
const end = Math.min(parsed.episodes.end, parsed.episodes.start + 199);
|
||||
for (let e = parsed.episodes.start; e <= end; e++) {
|
||||
ids.push(await findOrCreateUnit(exec, entityId, "episode", parsed.season, e, seasonIds[0]!));
|
||||
}
|
||||
return { unitIds: ids, primary: ids.length === 1 ? ids[0]! : seasonIds[0]! };
|
||||
}
|
||||
return { unitIds: seasonIds, primary: seasonIds[0] ?? null };
|
||||
}
|
||||
|
||||
async function findOrCreateUnit(
|
||||
exec: Exec,
|
||||
entityId: string,
|
||||
kind: "season" | "episode",
|
||||
season: number,
|
||||
episode: number | null,
|
||||
parentId: string | null,
|
||||
): Promise<string> {
|
||||
let q = exec
|
||||
.selectFrom("content_unit")
|
||||
.select(["id"])
|
||||
.where("media_entity_id", "=", entityId)
|
||||
.where("kind", "=", kind)
|
||||
.where("season_number", "=", season)
|
||||
.where("status", "<>", "retired");
|
||||
q = episode === null ? q.where("episode_number", "is", null) : q.where("episode_number", "=", episode);
|
||||
const hit = await q.orderBy("created_at").executeTakeFirst();
|
||||
if (hit) return hit.id;
|
||||
const row = await exec
|
||||
.insertInto("content_unit")
|
||||
.values({
|
||||
media_entity_id: entityId,
|
||||
parent_unit_id: parentId,
|
||||
kind,
|
||||
season_number: season,
|
||||
episode_number: episode,
|
||||
status: "provisional",
|
||||
display_order: episode ?? season,
|
||||
})
|
||||
.returning(["id"])
|
||||
.executeTakeFirstOrThrow();
|
||||
return row.id;
|
||||
}
|
||||
@@ -0,0 +1,320 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
applyFeedItems,
|
||||
emptyFeedCursor,
|
||||
type FeedCursor,
|
||||
type FeedFetchResult,
|
||||
type FeedFetchState,
|
||||
type FeedItem,
|
||||
} from "@umt/adapters";
|
||||
import type { FeedHealth } from "@umt/contracts";
|
||||
import { type FeedSourceRow, sql } from "@umt/db";
|
||||
import { jsonb } from "../rows.ts";
|
||||
import type { SourcesRuntime } from "../runtime.ts";
|
||||
import { clip, type Exec, openDecision } from "./common.ts";
|
||||
|
||||
// ingestFeed:RSS 抓取 → observation(feed+guid 唯一,摘要变化 = 更新)→ source_release(site+站点资源 ID 聚合)。
|
||||
// FeedFetchState / FeedCursor 持久化在 feed_source 行;enclosure 只存脱敏值(适配器已脱敏)。
|
||||
|
||||
/** feed_source.cursor 的形状:重叠窗口游标 + 熔断打开时间 */
|
||||
interface StoredCursor {
|
||||
window?: FeedCursor;
|
||||
circuitOpenedAt?: string;
|
||||
}
|
||||
|
||||
const AUTH_BACKOFF_MS = 24 * 3600_000;
|
||||
|
||||
export interface IngestResult {
|
||||
status: "ok" | "not_modified" | "skipped" | "error" | "auth_failed";
|
||||
reason?: string;
|
||||
newItems: number;
|
||||
updatedItems: number;
|
||||
newReleases: number;
|
||||
possibleGap: boolean;
|
||||
}
|
||||
|
||||
/** 从 Feed 条目推出站点资源 ID:详情/下载链接的 id 参数或 /detail/<id> 路径;都没有返回 null */
|
||||
export function extractSiteReleaseId(item: Pick<FeedItem, "guid" | "link" | "enclosureUrl">): string | null {
|
||||
for (const u of [item.link, item.enclosureUrl, item.guid]) {
|
||||
if (!u) continue;
|
||||
try {
|
||||
const url = new URL(u);
|
||||
const q = url.searchParams.get("id") ?? url.searchParams.get("tid") ?? url.searchParams.get("torrentid");
|
||||
if (q && /^[\w-]{1,64}$/.test(q)) return q;
|
||||
const m = /\/(?:detail|details|torrents?|t)\/(\d+)/.exec(url.pathname);
|
||||
if (m?.[1]) return m[1];
|
||||
} catch {
|
||||
// 不是 URL(纯 GUID)
|
||||
}
|
||||
}
|
||||
if (/^\d{1,20}$/.test(item.guid)) return item.guid;
|
||||
return null;
|
||||
}
|
||||
|
||||
export function stateFromRow(f: FeedSourceRow): FeedFetchState {
|
||||
const cur = (f.cursor ?? {}) as StoredCursor;
|
||||
const err = f.last_error as FeedFetchState["lastError"] | null;
|
||||
return {
|
||||
...(f.etag ? { etag: f.etag } : {}),
|
||||
...(f.last_modified ? { lastModified: f.last_modified } : {}),
|
||||
consecutiveFailures: f.consecutive_failures,
|
||||
...(f.backoff_until ? { nextAttemptAt: f.backoff_until.toISOString() } : {}),
|
||||
circuit: f.health_status === "circuit_open" ? "open" : "closed",
|
||||
...(cur.circuitOpenedAt ? { circuitOpenedAt: cur.circuitOpenedAt } : {}),
|
||||
...(f.last_success_at ? { lastSuccessAt: f.last_success_at.toISOString() } : {}),
|
||||
...(err && typeof err === "object" && "kind" in err ? { lastError: err } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function healthOf(r: FeedFetchResult): FeedHealth {
|
||||
if (r.kind === "ok" || r.kind === "not_modified") return "healthy";
|
||||
if (r.state.circuit === "open") return "circuit_open";
|
||||
if (r.kind === "skipped") return "degraded";
|
||||
return r.state.consecutiveFailures >= 3 ? "failing" : "degraded";
|
||||
}
|
||||
|
||||
function isAuthFailure(r: FeedFetchResult): boolean {
|
||||
if (r.kind !== "error") return false;
|
||||
return r.error.kind === "expired" || r.error.httpStatus === 401 || r.error.httpStatus === 403;
|
||||
}
|
||||
|
||||
export async function ingestFeed(
|
||||
rt: SourcesRuntime,
|
||||
feedId: string,
|
||||
opts: { manual?: boolean } = {},
|
||||
): Promise<IngestResult> {
|
||||
const { db, secrets } = rt.ctx;
|
||||
const empty = { newItems: 0, updatedItems: 0, newReleases: 0, possibleGap: false };
|
||||
const feed = await db.selectFrom("feed_source").selectAll().where("id", "=", feedId).executeTakeFirst();
|
||||
if (!feed) return { status: "skipped", reason: "not_found", ...empty };
|
||||
if (!feed.enabled) return { status: "skipped", reason: "disabled", ...empty };
|
||||
const site = feed.site_id
|
||||
? await db.selectFrom("site").selectAll().where("id", "=", feed.site_id).executeTakeFirst()
|
||||
: undefined;
|
||||
if (site && !site.enabled) return { status: "skipped", reason: "site_disabled", ...empty };
|
||||
if (!feed.url_secret_ref) return { status: "skipped", reason: "no_url", ...empty };
|
||||
|
||||
const url = await secrets.resolve(feed.url_secret_ref);
|
||||
const result = await rt.rss.fetchFeed(url, stateFromRow(feed), { ignoreBackoff: !!opts.manual });
|
||||
const now = rt.now();
|
||||
const auth = isAuthFailure(result);
|
||||
const health: FeedHealth = auth ? "failing" : healthOf(result);
|
||||
const cursor = (feed.cursor ?? {}) as StoredCursor;
|
||||
const state = result.state;
|
||||
const pollNext = new Date(now.getTime() + feed.poll_interval_sec * 1000);
|
||||
const backoffUntil = auth
|
||||
? new Date(now.getTime() + AUTH_BACKOFF_MS)
|
||||
: state.nextAttemptAt && result.kind !== "ok" && result.kind !== "not_modified"
|
||||
? new Date(state.nextAttemptAt)
|
||||
: null;
|
||||
|
||||
return db.transaction().execute(async (trx) => {
|
||||
let out: IngestResult = { status: result.kind === "ok" ? "ok" : result.kind, ...empty };
|
||||
let window = cursor.window ?? emptyFeedCursor();
|
||||
if (result.kind === "ok") {
|
||||
const applied = applyFeedItems(window, result.feed.items);
|
||||
window = applied.cursor;
|
||||
const counts = await writeItems(rt, trx, feed, site?.id ?? null, applied.newItems, applied.updatedItems);
|
||||
const unchanged = result.feed.items
|
||||
.filter((i) => !applied.newItems.includes(i) && !applied.updatedItems.includes(i))
|
||||
.map((i) => i.guid);
|
||||
if (unchanged.length) await touchUnchanged(trx, feed.id, unchanged, now);
|
||||
out = {
|
||||
status: "ok",
|
||||
newItems: applied.newItems.length,
|
||||
updatedItems: applied.updatedItems.length,
|
||||
newReleases: counts.newReleases,
|
||||
possibleGap: applied.possibleGap,
|
||||
};
|
||||
if (applied.possibleGap) {
|
||||
await rt.ctx.outbox.emit(
|
||||
trx,
|
||||
"feed.gap_suspected",
|
||||
{
|
||||
feedId: feed.id,
|
||||
siteId: site?.id ?? null,
|
||||
previousFetchedAt: cursor.window?.lastFetchedAt ?? null,
|
||||
detectedAt: now.toISOString(),
|
||||
},
|
||||
{ idempotencyKey: `feed.gap:${feed.id}:${now.toISOString()}` },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const lastError = auth
|
||||
? { ...(state.lastError ?? {}), kind: "auth", message: "Feed 凭证失效或过期(401/403/过期提示),已停止自动重试" }
|
||||
: (state.lastError ?? null);
|
||||
const nextCursor: StoredCursor = {
|
||||
window,
|
||||
...(state.circuit === "open" && state.circuitOpenedAt ? { circuitOpenedAt: state.circuitOpenedAt } : {}),
|
||||
};
|
||||
await trx
|
||||
.updateTable("feed_source")
|
||||
.set({
|
||||
etag: state.etag ?? null,
|
||||
last_modified: state.lastModified ?? null,
|
||||
consecutive_failures: state.consecutiveFailures,
|
||||
backoff_until: backoffUntil,
|
||||
health_status: health,
|
||||
last_error: result.kind === "ok" || result.kind === "not_modified" ? null : jsonb(lastError),
|
||||
cursor: jsonb(nextCursor),
|
||||
last_fetch_at: result.kind === "skipped" ? feed.last_fetch_at : now,
|
||||
last_success_at: state.lastSuccessAt ? new Date(state.lastSuccessAt) : feed.last_success_at,
|
||||
next_fetch_at: backoffUntil && backoffUntil > pollNext ? backoffUntil : pollNext,
|
||||
})
|
||||
.where("id", "=", feed.id)
|
||||
.execute();
|
||||
|
||||
if (health !== feed.health_status) {
|
||||
await rt.ctx.outbox.emit(trx, "feed.health_changed", {
|
||||
feedId: feed.id,
|
||||
from: feed.health_status,
|
||||
to: health,
|
||||
reason:
|
||||
lastError && typeof lastError === "object" ? String((lastError as { message?: string }).message ?? "") : null,
|
||||
});
|
||||
}
|
||||
if (result.kind === "ok" || result.kind === "not_modified") {
|
||||
await rt.ctx.outbox.emit(trx, "feed.fetched", {
|
||||
feedId: feed.id,
|
||||
newItems: out.newItems,
|
||||
updatedItems: out.updatedItems,
|
||||
notModified: result.kind === "not_modified",
|
||||
});
|
||||
}
|
||||
|
||||
if (auth) {
|
||||
if (feed.site_account_id) {
|
||||
await trx
|
||||
.updateTable("site_account")
|
||||
.set({ status: "invalid", last_verified_at: now })
|
||||
.where("id", "=", feed.site_account_id)
|
||||
.execute();
|
||||
}
|
||||
await openDecision(rt.ctx, trx, {
|
||||
kind: "source_unreachable",
|
||||
severity: "warning",
|
||||
title: `Feed「${feed.name}」凭证失效`,
|
||||
summary: "RSS 返回 401/403 或过期提示;已停止自动重试。请更新 Feed 链接或站点账号凭证后手动恢复。",
|
||||
subjectType: "feed_source",
|
||||
subjectId: feed.id,
|
||||
dedupeKey: `feed-auth:${feed.id}`,
|
||||
evidence: { feedId: feed.id, siteId: feed.site_id, siteAccountId: feed.site_account_id, error: lastError },
|
||||
options: [
|
||||
{ key: "reset", label: "已更新凭证,恢复抓取" },
|
||||
{ key: "disable", label: "停用该 Feed" },
|
||||
],
|
||||
});
|
||||
out = { ...out, status: "auth_failed", reason: "auth" };
|
||||
}
|
||||
return out;
|
||||
});
|
||||
}
|
||||
|
||||
/** 未变化条目:只刷新 last_seen_at(observation 与其 source_release) */
|
||||
async function touchUnchanged(trx: Exec, feedId: string, guids: string[], now: Date) {
|
||||
const rows = await trx
|
||||
.updateTable("observation")
|
||||
.set({ last_seen_at: now, fetched_at: now })
|
||||
.where("feed_id", "=", feedId)
|
||||
.where("guid", "in", guids)
|
||||
.returning(["source_release_id"])
|
||||
.execute();
|
||||
const ids = rows.map((r) => r.source_release_id).filter((x): x is string => !!x);
|
||||
if (ids.length) await trx.updateTable("source_release").set({ last_seen_at: now }).where("id", "in", ids).execute();
|
||||
}
|
||||
|
||||
function observationRaw(item: FeedItem) {
|
||||
return {
|
||||
link: item.link,
|
||||
// 只存脱敏后的 enclosure(凭据参数为 [REDACTED])
|
||||
enclosureUrl: item.enclosureUrl ?? null,
|
||||
enclosureHasSecret: item.enclosureHasSecret ?? false,
|
||||
enclosureType: item.enclosureType ?? null,
|
||||
size: item.size,
|
||||
categories: item.categories,
|
||||
nickname: item.nickname ?? null,
|
||||
description: clip(item.description, 1000),
|
||||
rawDigest: item.rawDigest,
|
||||
};
|
||||
}
|
||||
|
||||
async function writeItems(
|
||||
rt: SourcesRuntime,
|
||||
trx: Exec,
|
||||
feed: FeedSourceRow,
|
||||
siteId: string | null,
|
||||
newItems: FeedItem[],
|
||||
updatedItems: FeedItem[],
|
||||
): Promise<{ newReleases: number }> {
|
||||
let newReleases = 0;
|
||||
const updated = new Set(updatedItems);
|
||||
for (const item of [...newItems, ...updatedItems]) {
|
||||
const raw = observationRaw(item);
|
||||
const obs = (
|
||||
await sql<{ id: string; source_release_id: string | null; prev_digest: string | null }>`
|
||||
with prev as (select content_digest from observation where feed_id = ${feed.id} and guid = ${item.guid})
|
||||
insert into observation (feed_id, guid, content_digest, title, link_redacted, raw, source_published_at, fetched_at)
|
||||
values (${feed.id}, ${item.guid}, ${item.rawDigest}, ${item.title}, ${item.link}, ${jsonb(raw)}::jsonb,
|
||||
${item.publishedAt ?? null}, ${item.fetchedAt})
|
||||
on conflict (feed_id, guid) do update set
|
||||
content_digest = excluded.content_digest, title = excluded.title, link_redacted = excluded.link_redacted,
|
||||
raw = excluded.raw, source_published_at = coalesce(excluded.source_published_at, observation.source_published_at),
|
||||
fetched_at = excluded.fetched_at, last_seen_at = now()
|
||||
returning id, source_release_id, (select content_digest from prev) as prev_digest`.execute(trx)
|
||||
).rows[0]!;
|
||||
|
||||
if (!siteId) {
|
||||
// 无站点的通用 Feed:只留观察,无法按站点资源聚合
|
||||
await trx
|
||||
.updateTable("observation")
|
||||
.set({ status: "ignored", last_error: jsonb({ reason: "no_site" }) })
|
||||
.where("id", "=", obs.id)
|
||||
.execute();
|
||||
continue;
|
||||
}
|
||||
const srid =
|
||||
extractSiteReleaseId(item) ?? `guid:${createHash("sha1").update(item.guid).digest("hex").slice(0, 20)}`;
|
||||
const contentChanged = updated.has(item) || (obs.prev_digest !== null && obs.prev_digest !== item.rawDigest);
|
||||
const sizeBytes = item.size > 0 ? item.size : null;
|
||||
const rel = (
|
||||
await sql<{ id: string; inserted: boolean; prev_title: string | null; resolution_locked: boolean }>`
|
||||
with prev as (select title from source_release where site_id = ${siteId} and site_release_id = ${srid})
|
||||
insert into source_release (site_id, site_release_id, title, category_raw, size_bytes, source_published_at, raw)
|
||||
values (${siteId}, ${srid}, ${item.title}, ${item.categories[0] ?? null}, ${sizeBytes}, ${item.publishedAt ?? null},
|
||||
${jsonb({ rss: { link: item.link, categories: item.categories } })}::jsonb)
|
||||
on conflict (site_id, site_release_id) do update set
|
||||
last_seen_at = now(),
|
||||
title = case when ${contentChanged} then excluded.title else source_release.title end,
|
||||
size_bytes = case when ${contentChanged} then coalesce(excluded.size_bytes, source_release.size_bytes)
|
||||
else coalesce(source_release.size_bytes, excluded.size_bytes) end,
|
||||
source_published_at = coalesce(source_release.source_published_at, excluded.source_published_at),
|
||||
category_raw = coalesce(source_release.category_raw, excluded.category_raw)
|
||||
returning id, (xmax = 0) as inserted, (select title from prev) as prev_title, resolution_locked`.execute(trx)
|
||||
).rows[0]!;
|
||||
await trx
|
||||
.updateTable("observation")
|
||||
.set({ source_release_id: rel.id, status: "linked" })
|
||||
.where("id", "=", obs.id)
|
||||
.execute();
|
||||
|
||||
if (rel.inserted) {
|
||||
newReleases++;
|
||||
await rt.ctx.outbox.emit(
|
||||
trx,
|
||||
"release.observed",
|
||||
{ sourceReleaseId: rel.id, observationId: obs.id, siteId, isNew: true },
|
||||
{ idempotencyKey: `release.observed:${rel.id}` },
|
||||
);
|
||||
} else if (contentChanged && rel.prev_title !== item.title && !rel.resolution_locked) {
|
||||
// 标题修正:按新标题重新识别(证据版本变化)
|
||||
await rt.ctx.outbox.emit(
|
||||
trx,
|
||||
"release.observed",
|
||||
{ sourceReleaseId: rel.id, observationId: obs.id, siteId, isNew: false },
|
||||
{ idempotencyKey: `release.observed:${rel.id}:${item.rawDigest}` },
|
||||
);
|
||||
}
|
||||
}
|
||||
return { newReleases };
|
||||
}
|
||||
@@ -0,0 +1,472 @@
|
||||
import { TmdbError } from "@umt/adapters";
|
||||
import type {
|
||||
EntityIdentitySet,
|
||||
ExternalIdentityUpdate,
|
||||
ExternalRef,
|
||||
PoolCandidate,
|
||||
ReleaseIdentityAssign,
|
||||
} from "@umt/contracts";
|
||||
import { type ExternalIdentityRow, sql } from "@umt/db";
|
||||
import { parseRelease } from "@umt/domain";
|
||||
import { toParsedColumns } from "@umt/mapping";
|
||||
import { ApiException } from "../../../http/errors.ts";
|
||||
import { jsonb } from "../rows.ts";
|
||||
import type { SourcesRuntime } from "../runtime.ts";
|
||||
import { type Actor, audit, closeDecisions, type Exec } from "./common.ts";
|
||||
import {
|
||||
acceptIdentity,
|
||||
createEntityFromTmdb,
|
||||
ensureUnits,
|
||||
entityByExternal,
|
||||
IdentityConflictError,
|
||||
looksLikeVariety,
|
||||
resolveEntityId,
|
||||
} from "./entities.ts";
|
||||
import { recordAttempt } from "./resolve.ts";
|
||||
|
||||
// 人工纠错:指定来源资源身份(加锁)、改作品外部映射(新增修订、旧映射退役)、合并作品(旧 ID 跳转)。
|
||||
// 所有变更留 identity_revision / resolution_attempt / audit_event,不直接覆盖历史。
|
||||
|
||||
const actorName = (a: Actor) => `${a.type}:${a.id ?? ""}`;
|
||||
|
||||
/** 外部身份 → 本地作品;未登记且提供方为 TMDB 时按详情建档(TMDB 失败给 502,不当作不存在) */
|
||||
async function entityForExternal(rt: SourcesRuntime, exec: Exec, ref: ExternalRef, actor: Actor, variety: boolean) {
|
||||
const hit = await entityByExternal(exec, ref);
|
||||
if (hit) return hit;
|
||||
if (ref.provider !== "tmdb")
|
||||
throw new ApiException("bad_request", `未登记的外部身份 ${ref.provider}:${ref.externalId},只支持按 TMDB 建档`);
|
||||
const tmdb = rt.tmdb();
|
||||
if (!tmdb) throw new ApiException("precondition_failed", "未配置 TMDB,无法按外部身份建档");
|
||||
let detail: Awaited<ReturnType<typeof tmdb.details>>;
|
||||
try {
|
||||
detail = await tmdb.details(ref.entityType, ref.externalId);
|
||||
} catch (e) {
|
||||
if (e instanceof TmdbError && e.kind === "notfound")
|
||||
throw new ApiException("not_found", `TMDB 无此条目 ${ref.entityType}/${ref.externalId}`);
|
||||
if (e instanceof TmdbError) throw new ApiException("upstream_unavailable", `TMDB 查询失败:${e.kind}`);
|
||||
throw e;
|
||||
}
|
||||
const r = await createEntityFromTmdb(exec, detail, {
|
||||
level: "manual",
|
||||
evidence: { by: actorName(actor) },
|
||||
actor,
|
||||
variety,
|
||||
locked: true,
|
||||
});
|
||||
return r.entityId;
|
||||
}
|
||||
|
||||
export async function assignReleaseIdentity(
|
||||
rt: SourcesRuntime,
|
||||
srId: string,
|
||||
input: ReleaseIdentityAssign,
|
||||
actor: Actor,
|
||||
): Promise<{ mediaEntityId: string; contentUnitIds: string[] }> {
|
||||
const { ctx } = rt;
|
||||
const release = await ctx.db.selectFrom("source_release").selectAll().where("id", "=", srId).executeTakeFirst();
|
||||
if (!release) throw new ApiException("not_found", "来源资源不存在");
|
||||
const parsed = parseRelease(release.title, release.subtitle ? { subtitle: release.subtitle } : {});
|
||||
const site = ((release.raw as Record<string, unknown>)?._site ?? {}) as { categoryName?: string };
|
||||
const variety = looksLikeVariety(parsed, site.categoryName ?? release.category_raw);
|
||||
|
||||
let ref: ExternalRef | null = input.external ?? null;
|
||||
let localId: string | null = input.mediaEntityId ?? null;
|
||||
if (input.candidateIndex !== undefined) {
|
||||
const pool = await ctx.db
|
||||
.selectFrom("identity_candidate")
|
||||
.select(["candidates"])
|
||||
.where("source_release_id", "=", srId)
|
||||
.executeTakeFirst();
|
||||
const c = ((pool?.candidates ?? []) as PoolCandidate[])[input.candidateIndex];
|
||||
if (!c) throw new ApiException("bad_request", "候选序号无效");
|
||||
if (c.localEntityId) localId = c.localEntityId;
|
||||
else ref = { provider: c.provider, entityType: c.entityType, externalId: c.externalId };
|
||||
}
|
||||
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
try {
|
||||
return await ctx.db.transaction().execute(async (trx) => {
|
||||
let entityId: string;
|
||||
if (localId) {
|
||||
const r = await resolveEntityId(trx, localId);
|
||||
if (!r) throw new ApiException("not_found", "本地作品不存在");
|
||||
entityId = r;
|
||||
} else {
|
||||
entityId = await entityForExternal(rt, trx, ref!, actor, variety);
|
||||
}
|
||||
const entity = await trx
|
||||
.selectFrom("media_entity")
|
||||
.select(["id", "kind"])
|
||||
.where("id", "=", entityId)
|
||||
.executeTakeFirstOrThrow();
|
||||
let unitIds: string[];
|
||||
let primary: string | null;
|
||||
if (input.contentUnitId) {
|
||||
const u = await trx
|
||||
.selectFrom("content_unit")
|
||||
.select(["id", "media_entity_id"])
|
||||
.where("id", "=", input.contentUnitId)
|
||||
.executeTakeFirst();
|
||||
if (!u || u.media_entity_id !== entityId) throw new ApiException("bad_request", "内容单元不属于该作品");
|
||||
unitIds = [u.id];
|
||||
primary = u.id;
|
||||
} else if (input.contentUnitId === null) {
|
||||
unitIds = [];
|
||||
primary = null;
|
||||
} else {
|
||||
const r = await ensureUnits(trx, entityId, entity.kind, parsed);
|
||||
unitIds = r.unitIds;
|
||||
primary = r.primary;
|
||||
}
|
||||
const cols = toParsedColumns(parsed);
|
||||
await trx
|
||||
.updateTable("source_release")
|
||||
.set({
|
||||
parser_version: cols.parserVersion,
|
||||
parsed: jsonb(cols.parsed),
|
||||
media_entity_id: entityId,
|
||||
content_unit_id: primary,
|
||||
resolution_status: "resolved",
|
||||
resolution_locked: input.lock,
|
||||
resolution_evidence: jsonb({
|
||||
level: "manual",
|
||||
by: actorName(actor),
|
||||
reason: input.reason ?? null,
|
||||
previous: { mediaEntityId: release.media_entity_id, contentUnitId: release.content_unit_id },
|
||||
}),
|
||||
})
|
||||
.where("id", "=", srId)
|
||||
.execute();
|
||||
await trx
|
||||
.updateTable("identity_candidate")
|
||||
.set({
|
||||
status: "resolved",
|
||||
resolved_media_entity_id: entityId,
|
||||
resolved_by: actorName(actor),
|
||||
resolved_at: new Date(),
|
||||
next_review_at: null,
|
||||
})
|
||||
.where("source_release_id", "=", srId)
|
||||
.execute();
|
||||
await closeDecisions(
|
||||
ctx,
|
||||
trx,
|
||||
`identity:${srId}`,
|
||||
"resolved",
|
||||
{ mediaEntityId: entityId, by: "manual" },
|
||||
actorName(actor),
|
||||
);
|
||||
await recordAttempt(
|
||||
trx,
|
||||
srId,
|
||||
null,
|
||||
"manual",
|
||||
input.reason ?? null,
|
||||
"manual",
|
||||
1,
|
||||
entityId,
|
||||
{ mediaEntityId: entityId },
|
||||
null,
|
||||
actor,
|
||||
);
|
||||
await audit(
|
||||
trx,
|
||||
actor,
|
||||
"identity.release_assigned",
|
||||
{ type: "source_release", id: srId },
|
||||
{
|
||||
mediaEntityId: entityId,
|
||||
contentUnitIds: unitIds,
|
||||
locked: input.lock,
|
||||
previous: release.media_entity_id,
|
||||
},
|
||||
);
|
||||
await ctx.outbox.emit(trx, "release.resolved", {
|
||||
sourceReleaseId: srId,
|
||||
observationId: null,
|
||||
mediaEntityId: entityId,
|
||||
contentUnitIds: unitIds,
|
||||
evidenceLevel: "manual",
|
||||
});
|
||||
return { mediaEntityId: entityId, contentUnitIds: unitIds };
|
||||
});
|
||||
} catch (e) {
|
||||
if (e instanceof IdentityConflictError && attempt < 2) continue;
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function nextRevision(exec: Exec, identityId: string): Promise<number> {
|
||||
const r = await exec
|
||||
.selectFrom("identity_revision")
|
||||
.select((eb) => eb.fn.max("revision").as("m"))
|
||||
.where("external_identity_id", "=", identityId)
|
||||
.executeTakeFirst();
|
||||
return (Number(r?.m) || 0) + 1;
|
||||
}
|
||||
|
||||
async function retireIdentity(exec: Exec, row: ExternalIdentityRow, actor: Actor, reason: string | null) {
|
||||
const rev = Math.max(row.revision + 1, await nextRevision(exec, row.id));
|
||||
await exec
|
||||
.updateTable("external_identity")
|
||||
.set({ status: "retired", valid_until: new Date(), revision: rev, locked: true })
|
||||
.where("id", "=", row.id)
|
||||
.execute();
|
||||
await exec
|
||||
.insertInto("identity_revision")
|
||||
.values({
|
||||
external_identity_id: row.id,
|
||||
revision: rev,
|
||||
action: "status_changed",
|
||||
from_status: row.status,
|
||||
to_status: "retired",
|
||||
before: jsonb({ status: row.status, locked: row.locked }),
|
||||
after: jsonb({ status: "retired", locked: true }),
|
||||
actor_type: actor.type,
|
||||
actor_id: actor.id,
|
||||
reason,
|
||||
})
|
||||
.execute();
|
||||
}
|
||||
|
||||
/**
|
||||
* 改作品的外部映射:同 provider+类型的旧 accepted 映射退役(留修订并锁定,防止自动匹配改回),
|
||||
* 新映射 accepted + 人工锁。目标身份已被其它作品接受时 409(应走合并)。
|
||||
*/
|
||||
export async function setEntityIdentity(rt: SourcesRuntime, entityId: string, input: EntityIdentitySet, actor: Actor) {
|
||||
const { ctx } = rt;
|
||||
return ctx.db.transaction().execute(async (trx) => {
|
||||
const id = await resolveEntityId(trx, entityId);
|
||||
if (!id) throw new ApiException("not_found", "作品不存在");
|
||||
const other = await entityByExternal(trx, input);
|
||||
if (other && other !== id) {
|
||||
throw new ApiException("conflict", `该外部身份已属于作品 ${other},如为重复作品请使用合并`);
|
||||
}
|
||||
if (other === id) return { mediaEntityId: id, changed: false };
|
||||
const olds = await trx
|
||||
.selectFrom("external_identity")
|
||||
.selectAll()
|
||||
.where("media_entity_id", "=", id)
|
||||
.where("provider", "=", input.provider)
|
||||
.where("entity_type", "=", input.entityType)
|
||||
.where("status", "=", "accepted")
|
||||
.execute();
|
||||
for (const o of olds) await retireIdentity(trx, o, actor, input.reason ?? null);
|
||||
// 同一外部身份的旧候选(proposed/retired)不复用,新建一行保留历史
|
||||
const newId = await acceptIdentity(trx, id, input, "manual", { by: actorName(actor) }, actor, {
|
||||
locked: input.lock,
|
||||
reason: input.reason ?? null,
|
||||
});
|
||||
if (!newId) throw new ApiException("conflict", "外部身份唯一约束冲突");
|
||||
await audit(
|
||||
trx,
|
||||
actor,
|
||||
"identity.mapping_changed",
|
||||
{ type: "media_entity", id },
|
||||
{
|
||||
...input,
|
||||
retired: olds.map((o) => o.id),
|
||||
identityId: newId,
|
||||
},
|
||||
);
|
||||
return { mediaEntityId: id, changed: true, identityId: newId, retired: olds.map((o) => o.id) };
|
||||
});
|
||||
}
|
||||
|
||||
/** 外部身份字段修订(乐观并发 expectedRevision);每次变更写 identity_revision */
|
||||
export async function updateExternalIdentity(
|
||||
rt: SourcesRuntime,
|
||||
id: string,
|
||||
input: ExternalIdentityUpdate,
|
||||
actor: Actor,
|
||||
) {
|
||||
return rt.ctx.db.transaction().execute(async (trx) => {
|
||||
const row = await trx
|
||||
.selectFrom("external_identity")
|
||||
.selectAll()
|
||||
.where("id", "=", id)
|
||||
.forUpdate()
|
||||
.executeTakeFirst();
|
||||
if (!row) throw new ApiException("not_found", "外部身份不存在");
|
||||
if (row.revision !== input.expectedRevision) {
|
||||
throw new ApiException("precondition_failed", `修订号已变化(当前 ${row.revision})`);
|
||||
}
|
||||
const rev = Math.max(row.revision + 1, await nextRevision(trx, id));
|
||||
const patch: Record<string, unknown> = { revision: rev };
|
||||
if (input.status !== undefined) patch.status = input.status;
|
||||
if (input.evidenceLevel !== undefined) patch.evidence_level = input.evidenceLevel;
|
||||
if (input.evidence !== undefined) patch.evidence = jsonb(input.evidence);
|
||||
if (input.locked !== undefined) patch.locked = input.locked;
|
||||
if (input.validUntil !== undefined) patch.valid_until = input.validUntil;
|
||||
if (input.extra !== undefined) patch.extra = jsonb(input.extra);
|
||||
if (input.status === "retired" && input.validUntil === undefined) patch.valid_until = new Date();
|
||||
let updated: ExternalIdentityRow;
|
||||
try {
|
||||
updated = await trx
|
||||
.updateTable("external_identity")
|
||||
.set(patch)
|
||||
.where("id", "=", id)
|
||||
.returningAll()
|
||||
.executeTakeFirstOrThrow();
|
||||
} catch (e) {
|
||||
if ((e as { code?: string }).code === "23505")
|
||||
throw new ApiException("conflict", "同一外部身份已有 accepted 映射");
|
||||
throw e;
|
||||
}
|
||||
const action =
|
||||
input.status !== undefined && input.status !== row.status
|
||||
? "status_changed"
|
||||
: input.locked !== undefined && input.locked !== row.locked
|
||||
? input.locked
|
||||
? "locked"
|
||||
: "unlocked"
|
||||
: "evidence_updated";
|
||||
await trx
|
||||
.insertInto("identity_revision")
|
||||
.values({
|
||||
external_identity_id: id,
|
||||
revision: rev,
|
||||
action,
|
||||
from_status: row.status,
|
||||
to_status: updated.status,
|
||||
before: jsonb({ status: row.status, locked: row.locked, evidenceLevel: row.evidence_level }),
|
||||
after: jsonb({ status: updated.status, locked: updated.locked, evidenceLevel: updated.evidence_level }),
|
||||
actor_type: actor.type,
|
||||
actor_id: actor.id,
|
||||
reason: input.reason ?? null,
|
||||
})
|
||||
.execute();
|
||||
await audit(trx, actor, "identity.updated", { type: "external_identity", id }, { action, revision: rev });
|
||||
return updated;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 合并重复作品:from → into。旧 ID 保留(status=merged + merged_into_id)并写 identity_redirect;
|
||||
* 本模块的引用(来源资源、别名、待识别池、外部身份)迁到目标;其它模块的表按 merged_into_id 跳转解析。
|
||||
*/
|
||||
export async function mergeEntities(
|
||||
rt: SourcesRuntime,
|
||||
fromId: string,
|
||||
intoId: string,
|
||||
reason: string | null,
|
||||
actor: Actor,
|
||||
) {
|
||||
if (fromId === intoId) throw new ApiException("bad_request", "不能合并到自身");
|
||||
return rt.ctx.db.transaction().execute(async (trx) => {
|
||||
const from = await trx
|
||||
.selectFrom("media_entity")
|
||||
.selectAll()
|
||||
.where("id", "=", fromId)
|
||||
.forUpdate()
|
||||
.executeTakeFirst();
|
||||
const intoResolved = await resolveEntityId(trx, intoId);
|
||||
if (!from) throw new ApiException("not_found", "源作品不存在");
|
||||
if (!intoResolved) throw new ApiException("not_found", "目标作品不存在");
|
||||
if (from.status === "merged") throw new ApiException("conflict", "源作品已被合并");
|
||||
if (intoResolved === fromId) throw new ApiException("conflict", "目标作品已合并到源作品,会形成环");
|
||||
|
||||
// 外部身份:目标已有同 provider+类型 accepted 的,源的退役;否则迁到目标
|
||||
const idents = await trx
|
||||
.selectFrom("external_identity")
|
||||
.selectAll()
|
||||
.where("media_entity_id", "=", fromId)
|
||||
.where("status", "=", "accepted")
|
||||
.execute();
|
||||
for (const i of idents) {
|
||||
const clash = await trx
|
||||
.selectFrom("external_identity")
|
||||
.select(["id"])
|
||||
.where("media_entity_id", "=", intoResolved)
|
||||
.where("provider", "=", i.provider)
|
||||
.where("entity_type", "=", i.entity_type)
|
||||
.where("status", "=", "accepted")
|
||||
.executeTakeFirst();
|
||||
if (clash) {
|
||||
await retireIdentity(trx, i, actor, `merged into ${intoResolved}`);
|
||||
continue;
|
||||
}
|
||||
const rev = Math.max(i.revision + 1, await nextRevision(trx, i.id));
|
||||
await trx
|
||||
.updateTable("external_identity")
|
||||
.set({ media_entity_id: intoResolved, revision: rev })
|
||||
.where("id", "=", i.id)
|
||||
.execute();
|
||||
await trx
|
||||
.insertInto("identity_revision")
|
||||
.values({
|
||||
external_identity_id: i.id,
|
||||
revision: rev,
|
||||
action: "relinked",
|
||||
from_status: i.status,
|
||||
to_status: i.status,
|
||||
before: jsonb({ mediaEntityId: fromId }),
|
||||
after: jsonb({ mediaEntityId: intoResolved }),
|
||||
actor_type: actor.type,
|
||||
actor_id: actor.id,
|
||||
reason,
|
||||
})
|
||||
.execute();
|
||||
}
|
||||
await sql`insert into title_alias (media_entity_id, content_unit_id, alias, normalized, language, kind, source, locked)
|
||||
select ${intoResolved}, null, alias, normalized, language, kind, source, locked from title_alias
|
||||
where media_entity_id = ${fromId} and content_unit_id is null
|
||||
on conflict (media_entity_id, content_unit_id, normalized, kind) do nothing`.execute(trx);
|
||||
await trx
|
||||
.updateTable("source_release")
|
||||
.set({ media_entity_id: intoResolved })
|
||||
.where("media_entity_id", "=", fromId)
|
||||
.execute();
|
||||
await trx
|
||||
.updateTable("identity_candidate")
|
||||
.set({ resolved_media_entity_id: intoResolved })
|
||||
.where("resolved_media_entity_id", "=", fromId)
|
||||
.execute();
|
||||
await trx
|
||||
.updateTable("media_entity")
|
||||
.set({ status: "merged", merged_into_id: intoResolved })
|
||||
.where("id", "=", fromId)
|
||||
.execute();
|
||||
await trx
|
||||
.insertInto("identity_redirect")
|
||||
.values({
|
||||
from_entity_id: fromId,
|
||||
to_entity_id: intoResolved,
|
||||
reason,
|
||||
actor_type: actor.type,
|
||||
actor_id: actor.id,
|
||||
})
|
||||
.execute();
|
||||
await audit(
|
||||
trx,
|
||||
actor,
|
||||
"identity.entity_merged",
|
||||
{ type: "media_entity", id: fromId },
|
||||
{ intoId: intoResolved, reason },
|
||||
);
|
||||
return trx.selectFrom("media_entity").selectAll().where("id", "=", intoResolved).executeTakeFirstOrThrow();
|
||||
});
|
||||
}
|
||||
|
||||
export async function dismissPoolEntry(rt: SourcesRuntime, poolId: string, actor: Actor, note: string | null) {
|
||||
const { ctx } = rt;
|
||||
return ctx.db.transaction().execute(async (trx) => {
|
||||
const row = await trx
|
||||
.updateTable("identity_candidate")
|
||||
.set({ status: "dismissed", next_review_at: null, resolved_by: actorName(actor), resolved_at: new Date() })
|
||||
.where("id", "=", poolId)
|
||||
.returningAll()
|
||||
.executeTakeFirst();
|
||||
if (!row) throw new ApiException("not_found", "待识别条目不存在");
|
||||
await trx
|
||||
.updateTable("source_release")
|
||||
.set({ resolution_status: "rejected" })
|
||||
.where("id", "=", row.source_release_id)
|
||||
.where("resolution_locked", "=", false)
|
||||
.execute();
|
||||
await closeDecisions(ctx, trx, `identity:${row.source_release_id}`, "dismissed", { note }, actorName(actor));
|
||||
await audit(trx, actor, "identity.pool_dismissed", { type: "identity_candidate", id: poolId }, { note });
|
||||
return row;
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,254 @@
|
||||
import { type MetadataCandidate, TmdbError } from "@umt/adapters";
|
||||
import type { EntityMetadataField, FieldOverrideInput, FinalFieldValue, MetadataSource } from "@umt/contracts";
|
||||
import { sql } from "@umt/db";
|
||||
import { ApiException } from "../../../http/errors.ts";
|
||||
import { jsonb } from "../rows.ts";
|
||||
import type { SourcesRuntime } from "../runtime.ts";
|
||||
import { type Actor, audit, type Exec } from "./common.ts";
|
||||
import { resolveEntityId, seedAliases } from "./entities.ts";
|
||||
|
||||
// refreshMetadata:字段级刷新(REQ §7.3)。来源值写 metadata_field;最终值 = 覆盖(锁)> 已锁字段保持 > 来源值。
|
||||
// 只更新 media_entity 的展示字段与别名,不触碰 path_binding(路径绑定不随元数据变化)。
|
||||
|
||||
/** 最终值投影到 media_entity 列的字段 */
|
||||
const COLUMN_FIELDS: Partial<Record<EntityMetadataField, "title" | "original_title" | "year">> = {
|
||||
title: "title",
|
||||
originalTitle: "original_title",
|
||||
year: "year",
|
||||
};
|
||||
|
||||
function tmdbFields(m: MetadataCandidate): Partial<Record<EntityMetadataField, unknown>> {
|
||||
return {
|
||||
title: m.title,
|
||||
originalTitle: m.originalTitle,
|
||||
year: m.year,
|
||||
overview: m.overview,
|
||||
releaseDate: m.releaseDate,
|
||||
originalLanguage: m.originalLanguage,
|
||||
originCountry: m.originCountry,
|
||||
};
|
||||
}
|
||||
|
||||
export interface RefreshResult {
|
||||
status: "refreshed" | "skipped" | "failed";
|
||||
reason?: string;
|
||||
changedFields: string[];
|
||||
lockedFields: string[];
|
||||
}
|
||||
|
||||
export async function refreshMetadata(rt: SourcesRuntime, mediaEntityId: string): Promise<RefreshResult> {
|
||||
const { ctx } = rt;
|
||||
const id = await resolveEntityId(ctx.db, mediaEntityId);
|
||||
if (!id) return { status: "skipped", reason: "not_found", changedFields: [], lockedFields: [] };
|
||||
const ident = await ctx.db
|
||||
.selectFrom("external_identity")
|
||||
.select(["entity_type", "external_id"])
|
||||
.where("media_entity_id", "=", id)
|
||||
.where("provider", "=", "tmdb")
|
||||
.where("status", "=", "accepted")
|
||||
.executeTakeFirst();
|
||||
const tmdb = rt.tmdb();
|
||||
if (!ident || !tmdb || (ident.entity_type !== "movie" && ident.entity_type !== "tv")) {
|
||||
return {
|
||||
status: "skipped",
|
||||
reason: ident ? "tmdb_not_configured" : "no_tmdb_identity",
|
||||
changedFields: [],
|
||||
lockedFields: [],
|
||||
};
|
||||
}
|
||||
let detail: MetadataCandidate;
|
||||
try {
|
||||
detail = await tmdb.details(ident.entity_type, ident.external_id);
|
||||
} catch (e) {
|
||||
// 404/超时/鉴权失败只记录校验状态,不是删除映射的依据
|
||||
if (e instanceof TmdbError) {
|
||||
await ctx.db
|
||||
.updateTable("external_identity")
|
||||
.set({ verify_status: e.kind === "notfound" ? "not_found" : "error", verified_at: rt.now() })
|
||||
.where("media_entity_id", "=", id)
|
||||
.where("provider", "=", "tmdb")
|
||||
.where("status", "=", "accepted")
|
||||
.execute();
|
||||
if (e.kind === "notfound" || e.kind === "auth") {
|
||||
return { status: "failed", reason: e.kind, changedFields: [], lockedFields: [] };
|
||||
}
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
const now = rt.now();
|
||||
return ctx.db.transaction().execute(async (trx) => {
|
||||
const fields = tmdbFields(detail);
|
||||
for (const [field, value] of Object.entries(fields)) {
|
||||
await sql`insert into metadata_field (media_entity_id, content_unit_id, field, value, language, source, source_revision, fetched_at)
|
||||
values (${id}, null, ${field}, ${jsonb(value)}::jsonb, ${null}, 'tmdb', ${`tmdb:${ident.entity_type}/${ident.external_id}`}, ${now})
|
||||
on conflict (media_entity_id, content_unit_id, field, source, language) do update set
|
||||
value = excluded.value, source_revision = excluded.source_revision, fetched_at = excluded.fetched_at`.execute(
|
||||
trx,
|
||||
);
|
||||
}
|
||||
await trx
|
||||
.updateTable("external_identity")
|
||||
.set({ verify_status: "ok", verified_at: now })
|
||||
.where("media_entity_id", "=", id)
|
||||
.where("provider", "=", "tmdb")
|
||||
.where("status", "=", "accepted")
|
||||
.execute();
|
||||
await seedAliases(trx, id, detail);
|
||||
const r = await project(trx, id, now);
|
||||
await trx.updateTable("media_entity").set({ last_metadata_refresh_at: now }).where("id", "=", id).execute();
|
||||
return { status: "refreshed" as const, ...r };
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 重新投影最终字段:覆盖(用户锁)> field_locks 中已锁字段保持现值 > tmdb 来源值。
|
||||
* 返回被改变与被锁住的字段。
|
||||
*/
|
||||
export async function project(
|
||||
trx: Exec,
|
||||
entityId: string,
|
||||
now: Date,
|
||||
): Promise<{ changedFields: string[]; lockedFields: string[] }> {
|
||||
const e = await trx
|
||||
.selectFrom("media_entity")
|
||||
.selectAll()
|
||||
.where("id", "=", entityId)
|
||||
.forUpdate()
|
||||
.executeTakeFirstOrThrow();
|
||||
const overrides = await trx
|
||||
.selectFrom("field_override")
|
||||
.selectAll()
|
||||
.where("media_entity_id", "=", entityId)
|
||||
.execute();
|
||||
const sources = await trx
|
||||
.selectFrom("metadata_field")
|
||||
.selectAll()
|
||||
.where("media_entity_id", "=", entityId)
|
||||
.orderBy("fetched_at", "desc")
|
||||
.execute();
|
||||
const meta = { ...((e.metadata ?? {}) as Record<string, FinalFieldValue>) };
|
||||
const locks = new Set(e.field_locks);
|
||||
const changed: string[] = [];
|
||||
const lockedFields: string[] = [];
|
||||
const colPatch: Record<string, unknown> = {};
|
||||
const fieldNames = new Set([...sources.map((s) => s.field), ...overrides.map((o) => o.field)]);
|
||||
for (const field of fieldNames) {
|
||||
const ov = overrides.find((o) => o.field === field);
|
||||
let next: FinalFieldValue | null = null;
|
||||
if (ov) {
|
||||
next = {
|
||||
value: ov.value,
|
||||
source: ov.source as MetadataSource,
|
||||
language: null,
|
||||
at: ov.updated_at.toISOString(),
|
||||
locked: ov.locked,
|
||||
};
|
||||
if (ov.locked) lockedFields.push(field);
|
||||
} else if (locks.has(field)) {
|
||||
lockedFields.push(field);
|
||||
continue;
|
||||
} else {
|
||||
const src =
|
||||
sources.find((s) => s.field === field && s.source === "tmdb") ?? sources.find((s) => s.field === field);
|
||||
if (!src) continue;
|
||||
next = {
|
||||
value: src.value,
|
||||
source: src.source as MetadataSource,
|
||||
language: src.language,
|
||||
at: src.fetched_at.toISOString(),
|
||||
locked: false,
|
||||
};
|
||||
}
|
||||
const prev = meta[field];
|
||||
if (!prev || JSON.stringify(prev.value) !== JSON.stringify(next.value) || prev.source !== next.source) {
|
||||
changed.push(field);
|
||||
}
|
||||
meta[field] = next;
|
||||
const col = COLUMN_FIELDS[field as EntityMetadataField];
|
||||
if (col) {
|
||||
const v = next.value;
|
||||
if (col === "year") colPatch.year = typeof v === "number" ? v : null;
|
||||
else if (col === "title") {
|
||||
if (typeof v === "string" && v) colPatch.title = v;
|
||||
} else colPatch.original_title = typeof v === "string" ? v : null;
|
||||
}
|
||||
}
|
||||
await trx
|
||||
.updateTable("media_entity")
|
||||
.set({ ...colPatch, metadata: jsonb(meta), updated_at: now })
|
||||
.where("id", "=", entityId)
|
||||
.execute();
|
||||
return { changedFields: changed, lockedFields };
|
||||
}
|
||||
|
||||
/** 用户覆盖字段并(默认)锁定:之后的自动刷新不得改写 */
|
||||
export async function setFieldOverride(
|
||||
rt: SourcesRuntime,
|
||||
entityId: string,
|
||||
field: EntityMetadataField,
|
||||
input: FieldOverrideInput,
|
||||
actor: Actor,
|
||||
) {
|
||||
return rt.ctx.db.transaction().execute(async (trx) => {
|
||||
const id = await resolveEntityId(trx, entityId);
|
||||
if (!id) throw new ApiException("not_found", "作品不存在");
|
||||
if (field === "title" && (typeof input.value !== "string" || !input.value.trim())) {
|
||||
throw new ApiException("bad_request", "title 必须是非空字符串");
|
||||
}
|
||||
if (
|
||||
field === "year" &&
|
||||
input.value !== null &&
|
||||
(typeof input.value !== "number" || !Number.isInteger(input.value))
|
||||
) {
|
||||
throw new ApiException("bad_request", "year 必须是整数或 null");
|
||||
}
|
||||
await sql`insert into field_override (media_entity_id, content_unit_id, field, value, locked, source, reason, actor_type, actor_id)
|
||||
values (${id}, null, ${field}, ${jsonb(input.value)}::jsonb, ${input.lock}, 'user', ${input.reason ?? null}, ${actor.type}, ${actor.id})
|
||||
on conflict (media_entity_id, content_unit_id, field) do update set
|
||||
value = excluded.value, locked = excluded.locked, reason = excluded.reason,
|
||||
actor_type = excluded.actor_type, actor_id = excluded.actor_id`.execute(trx);
|
||||
const e = await trx
|
||||
.selectFrom("media_entity")
|
||||
.select(["field_locks"])
|
||||
.where("id", "=", id)
|
||||
.executeTakeFirstOrThrow();
|
||||
const locks = new Set(e.field_locks);
|
||||
if (input.lock) locks.add(field);
|
||||
else locks.delete(field);
|
||||
await trx
|
||||
.updateTable("media_entity")
|
||||
.set({ field_locks: [...locks].sort() })
|
||||
.where("id", "=", id)
|
||||
.execute();
|
||||
const r = await project(trx, id, rt.now());
|
||||
await audit(trx, actor, "metadata.field_override", { type: "media_entity", id }, { field, lock: input.lock });
|
||||
return { mediaEntityId: id, ...r };
|
||||
});
|
||||
}
|
||||
|
||||
/** 取消覆盖与锁定:回到来源值 */
|
||||
export async function clearFieldOverride(
|
||||
rt: SourcesRuntime,
|
||||
entityId: string,
|
||||
field: EntityMetadataField,
|
||||
actor: Actor,
|
||||
) {
|
||||
return rt.ctx.db.transaction().execute(async (trx) => {
|
||||
const id = await resolveEntityId(trx, entityId);
|
||||
if (!id) throw new ApiException("not_found", "作品不存在");
|
||||
await trx.deleteFrom("field_override").where("media_entity_id", "=", id).where("field", "=", field).execute();
|
||||
const e = await trx
|
||||
.selectFrom("media_entity")
|
||||
.select(["field_locks"])
|
||||
.where("id", "=", id)
|
||||
.executeTakeFirstOrThrow();
|
||||
await trx
|
||||
.updateTable("media_entity")
|
||||
.set({ field_locks: e.field_locks.filter((f) => f !== field) })
|
||||
.where("id", "=", id)
|
||||
.execute();
|
||||
const r = await project(trx, id, rt.now());
|
||||
await audit(trx, actor, "metadata.field_override_cleared", { type: "media_entity", id }, { field });
|
||||
return { mediaEntityId: id, ...r };
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,637 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { type MetadataCandidate, TmdbError } from "@umt/adapters";
|
||||
import type { EvidenceLevel, PoolCandidate } from "@umt/contracts";
|
||||
import { type SourceReleaseRow, sql } from "@umt/db";
|
||||
import {
|
||||
type IdentityCandidate as DomainCandidate,
|
||||
type IdentityResolution,
|
||||
normalizeTitle,
|
||||
PARSER_VERSION,
|
||||
type ParsedRelease,
|
||||
parseRelease,
|
||||
queryTitles,
|
||||
type RankedCandidate,
|
||||
resolveIdentity,
|
||||
} from "@umt/domain";
|
||||
import {
|
||||
localIdentityCandidate,
|
||||
toContractEvidenceLevel,
|
||||
toContractResolveStatus,
|
||||
toParsedColumns,
|
||||
toReleaseResolutionStatus,
|
||||
} from "@umt/mapping";
|
||||
import { jsonb } from "../rows.ts";
|
||||
import type { SourcesRuntime } from "../runtime.ts";
|
||||
import { type Actor, closeDecisions, type Exec, openDecision, SYSTEM } from "./common.ts";
|
||||
import {
|
||||
addAlias,
|
||||
createEntityFromTmdb,
|
||||
ensureUnits,
|
||||
entityByExternal,
|
||||
IdentityConflictError,
|
||||
looksLikeVariety,
|
||||
resolveEntityId,
|
||||
} from "./entities.ts";
|
||||
|
||||
// resolveObservation:parseRelease → 本地召回(别名 pg_trgm + 已有外部身份 + 人工锁)→ TMDB 候选 → domain.resolveIdentity。
|
||||
// auto 才建/关联正式作品;ambiguous / no_match / 查询失败进待识别池(候选只存 JSON),有界退避复查。
|
||||
|
||||
const MAX_REVIEW_ATTEMPTS = 8;
|
||||
const TRGM_THRESHOLD = 0.45;
|
||||
|
||||
export interface ResolveInput {
|
||||
observationId?: string | undefined;
|
||||
sourceReleaseId?: string | undefined;
|
||||
/** 复查/手动重试:忽略"已识别且证据未变"的短路 */
|
||||
force?: boolean | undefined;
|
||||
}
|
||||
|
||||
export interface ResolveResult {
|
||||
outcome: "auto" | "proposed" | "ambiguous" | "no_match" | "lookup_failed" | "skipped_locked" | "skipped";
|
||||
sourceReleaseId: string | null;
|
||||
mediaEntityId: string | null;
|
||||
contentUnitIds: string[];
|
||||
decisionId: string | null;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
/** 候选的稳定键 */
|
||||
const candKey = (c: DomainCandidate) =>
|
||||
c.localEntityId ? `local:${c.localEntityId}` : `${c.provider}:${c.entityType}:${c.externalId}`;
|
||||
|
||||
export function evidenceVersion(parsed: ParsedRelease | null, ranked: RankedCandidate[]): string {
|
||||
const h = createHash("sha1")
|
||||
.update(
|
||||
JSON.stringify([parsed?.title ?? null, parsed?.year ?? null, ranked.map((r) => [candKey(r.candidate), r.level])]),
|
||||
)
|
||||
.digest("hex")
|
||||
.slice(0, 16);
|
||||
return `p${PARSER_VERSION}:${h}`;
|
||||
}
|
||||
|
||||
function yearOf(d: string | null | undefined): number | undefined {
|
||||
return d && /^\d{4}/.test(d) ? Number(d.slice(0, 4)) : undefined;
|
||||
}
|
||||
|
||||
export function tmdbToCandidate(m: MetadataCandidate): DomainCandidate {
|
||||
const titles = [m.title, ...(m.originalTitle ? [m.originalTitle] : [])];
|
||||
for (const t of m.titles) if (t.kind !== "alias" && !titles.includes(t.text)) titles.push(t.text);
|
||||
const aliases = [...new Set([...m.aliases, ...m.titles.filter((t) => t.kind === "alias").map((t) => t.text)])];
|
||||
const seasonYears = m.seasons?.map((s) => yearOf(s.airDate)).filter((y): y is number => y !== undefined);
|
||||
return {
|
||||
provider: "tmdb",
|
||||
entityType: m.entityType,
|
||||
externalId: m.externalId,
|
||||
titles,
|
||||
...(m.year !== null ? { year: m.year } : {}),
|
||||
aliases,
|
||||
...(seasonYears?.length ? { seasonYears } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function poolCandidate(r: RankedCandidate): PoolCandidate {
|
||||
const c = r.candidate;
|
||||
return {
|
||||
provider: c.provider,
|
||||
entityType: c.entityType,
|
||||
externalId: c.externalId,
|
||||
title: c.titles[0] ?? c.externalId,
|
||||
year: c.year ?? null,
|
||||
localEntityId: c.localEntityId ?? null,
|
||||
level: r.level,
|
||||
evidence: r.evidence,
|
||||
};
|
||||
}
|
||||
|
||||
/** 本地召回:显式外部 ID → 已接受映射;别名精确/trgm 模糊(跟随合并跳转),构造本地候选 */
|
||||
async function localCandidates(
|
||||
exec: Exec,
|
||||
parsed: ParsedRelease,
|
||||
release: SourceReleaseRow,
|
||||
): Promise<DomainCandidate[]> {
|
||||
const ids = new Set<string>();
|
||||
const site = ((release.raw as Record<string, unknown>)?._site ?? {}) as { imdbId?: string };
|
||||
const explicit = [...parsed.explicitIds, ...(site.imdbId ? [{ provider: "imdb" as const, id: site.imdbId }] : [])];
|
||||
for (const x of explicit) {
|
||||
const row = await exec
|
||||
.selectFrom("external_identity")
|
||||
.select(["media_entity_id"])
|
||||
.where("provider", "=", x.provider)
|
||||
.where("external_id", "=", x.id)
|
||||
.where("status", "=", "accepted")
|
||||
.where("media_entity_id", "is not", null)
|
||||
.execute();
|
||||
for (const r of row) if (r.media_entity_id) ids.add(r.media_entity_id);
|
||||
}
|
||||
for (const q of queryTitles(parsed)) {
|
||||
const rows = await sql<{ media_entity_id: string }>`
|
||||
select distinct media_entity_id from title_alias
|
||||
where normalized = ${q} or (length(${q}) >= 3 and similarity(normalized, ${q}) >= ${TRGM_THRESHOLD})
|
||||
limit 20`.execute(exec);
|
||||
for (const r of rows.rows) ids.add(r.media_entity_id);
|
||||
}
|
||||
const final = new Set<string>();
|
||||
for (const id of ids) {
|
||||
const r = await resolveEntityId(exec, id);
|
||||
if (r) final.add(r);
|
||||
}
|
||||
const out: DomainCandidate[] = [];
|
||||
for (const id of final) {
|
||||
const e = await exec.selectFrom("media_entity").selectAll().where("id", "=", id).executeTakeFirst();
|
||||
if (!e || e.status === "archived") continue;
|
||||
const aliases = await exec.selectFrom("title_alias").select(["alias"]).where("media_entity_id", "=", id).execute();
|
||||
const idents = await exec
|
||||
.selectFrom("external_identity")
|
||||
.select(["provider", "external_id", "status"])
|
||||
.where("media_entity_id", "=", id)
|
||||
.execute();
|
||||
const c = localIdentityCandidate(
|
||||
{ id: e.id, kind: e.kind, title: e.title, originalTitle: e.original_title, year: e.year },
|
||||
aliases.map((a) => a.alias),
|
||||
idents.map((i) => ({ provider: i.provider, externalId: i.external_id, status: i.status })),
|
||||
);
|
||||
if (c) out.push(c);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async function tmdbCandidates(rt: SourcesRuntime, parsed: ParsedRelease): Promise<MetadataCandidate[]> {
|
||||
const tmdb = rt.tmdb();
|
||||
if (!tmdb) return [];
|
||||
const queries = [parsed.titleCn, parsed.titleEn, parsed.title].filter((t): t is string => !!t);
|
||||
const seen = new Map<string, MetadataCandidate>();
|
||||
for (const q of [...new Set(queries)].slice(0, 2)) {
|
||||
for (const m of await tmdb.searchAll(q)) {
|
||||
const k = `${m.entityType}:${m.externalId}`;
|
||||
if (!seen.has(k)) seen.set(k, m);
|
||||
}
|
||||
}
|
||||
return [...seen.values()].slice(0, 20);
|
||||
}
|
||||
|
||||
async function loadRelease(rt: SourcesRuntime, input: ResolveInput) {
|
||||
const { db } = rt.ctx;
|
||||
const observationId = input.observationId ?? null;
|
||||
let srId = input.sourceReleaseId ?? null;
|
||||
if (!srId && observationId) {
|
||||
const o = await db
|
||||
.selectFrom("observation")
|
||||
.select(["source_release_id"])
|
||||
.where("id", "=", observationId)
|
||||
.executeTakeFirst();
|
||||
srId = o?.source_release_id ?? null;
|
||||
}
|
||||
if (!srId) return { release: null, observationId };
|
||||
const release = await db.selectFrom("source_release").selectAll().where("id", "=", srId).executeTakeFirst();
|
||||
return { release: release ?? null, observationId };
|
||||
}
|
||||
|
||||
export async function resolveRelease(rt: SourcesRuntime, input: ResolveInput): Promise<ResolveResult> {
|
||||
const { ctx } = rt;
|
||||
const { release, observationId } = await loadRelease(rt, input);
|
||||
const base = { mediaEntityId: null, contentUnitIds: [] as string[], decisionId: null };
|
||||
if (!release) return { outcome: "skipped", sourceReleaseId: null, reason: "release_not_found", ...base };
|
||||
|
||||
const pool = await ctx.db
|
||||
.selectFrom("identity_candidate")
|
||||
.selectAll()
|
||||
.where("source_release_id", "=", release.id)
|
||||
.executeTakeFirst();
|
||||
if (release.resolution_locked) {
|
||||
await recordAttempt(
|
||||
ctx.db,
|
||||
release.id,
|
||||
observationId,
|
||||
"skipped_locked",
|
||||
"manual_lock",
|
||||
"locked",
|
||||
0,
|
||||
release.media_entity_id,
|
||||
);
|
||||
return {
|
||||
outcome: "skipped_locked",
|
||||
sourceReleaseId: release.id,
|
||||
reason: "manual_lock",
|
||||
...base,
|
||||
mediaEntityId: release.media_entity_id,
|
||||
};
|
||||
}
|
||||
if (pool?.status === "dismissed" && !input.force) {
|
||||
return { outcome: "skipped", sourceReleaseId: release.id, reason: "dismissed", ...base };
|
||||
}
|
||||
|
||||
// 已识别:除非强制(复查/标题更新/人工重试),不重复召回
|
||||
if (!input.force && release.resolution_status === "resolved" && release.media_entity_id) {
|
||||
return {
|
||||
outcome: "skipped",
|
||||
sourceReleaseId: release.id,
|
||||
reason: "already_resolved",
|
||||
...base,
|
||||
mediaEntityId: release.media_entity_id,
|
||||
};
|
||||
}
|
||||
const parsed = parseRelease(release.title, release.subtitle ? { subtitle: release.subtitle } : {});
|
||||
const cols = toParsedColumns(parsed);
|
||||
if (!parsed.title || parsed.auxiliaryOnly) {
|
||||
return toPool(rt, release, observationId, parsed, null, [], "no_match", "unparseable_title", null);
|
||||
}
|
||||
|
||||
// 召回:本地优先,再 TMDB;TMDB 失败不等于无结果
|
||||
const local = await localCandidates(ctx.db, parsed, release);
|
||||
let remote: MetadataCandidate[] = [];
|
||||
let lookupError: string | null = null;
|
||||
try {
|
||||
remote = await tmdbCandidates(rt, parsed);
|
||||
} catch (e) {
|
||||
if (!(e instanceof TmdbError)) throw e;
|
||||
lookupError = `${e.kind}: ${e.message}`;
|
||||
}
|
||||
const candidates = [...local, ...remote.map(tmdbToCandidate)];
|
||||
const resolution = resolveIdentity(parsed, candidates);
|
||||
const version = evidenceVersion(parsed, resolution.ranked);
|
||||
|
||||
// 已识别且证据未变:幂等短路(强制复查时)
|
||||
if (
|
||||
release.resolution_status === "resolved" &&
|
||||
release.media_entity_id &&
|
||||
(release.resolution_evidence as { evidenceVersion?: string })?.evidenceVersion === version
|
||||
) {
|
||||
return {
|
||||
outcome: "skipped",
|
||||
sourceReleaseId: release.id,
|
||||
reason: "unchanged",
|
||||
...base,
|
||||
mediaEntityId: release.media_entity_id,
|
||||
};
|
||||
}
|
||||
|
||||
const automation = await ctx.settings.get("automation");
|
||||
const autoAccept = automation.enabled && !automation.readOnlyReconcile;
|
||||
if (resolution.status === "auto" && resolution.best && autoAccept) {
|
||||
const best = resolution.best;
|
||||
const remoteBest =
|
||||
best.candidate.provider === "tmdb"
|
||||
? remote.find((m) => m.externalId === best.candidate.externalId && m.entityType === best.candidate.entityType)
|
||||
: undefined;
|
||||
let detailed = remoteBest;
|
||||
if (remoteBest && !remoteBest.detailed) {
|
||||
const existing = await entityByExternal(ctx.db, {
|
||||
provider: "tmdb",
|
||||
entityType: remoteBest.entityType,
|
||||
externalId: remoteBest.externalId,
|
||||
});
|
||||
if (!existing) {
|
||||
try {
|
||||
detailed = await rt.tmdb()!.details(remoteBest.entityType, remoteBest.externalId);
|
||||
} catch (e) {
|
||||
if (!(e instanceof TmdbError)) throw e;
|
||||
detailed = remoteBest;
|
||||
}
|
||||
}
|
||||
}
|
||||
return bindAuto(rt, release, observationId, parsed, cols, resolution, version, detailed ?? null);
|
||||
}
|
||||
|
||||
if (resolution.status === "auto" && !autoAccept) {
|
||||
return toPool(
|
||||
rt,
|
||||
release,
|
||||
observationId,
|
||||
parsed,
|
||||
resolution,
|
||||
resolution.ranked,
|
||||
"proposed",
|
||||
"automation_off",
|
||||
version,
|
||||
);
|
||||
}
|
||||
if (lookupError && resolution.status !== "ambiguous") {
|
||||
return toPool(
|
||||
rt,
|
||||
release,
|
||||
observationId,
|
||||
parsed,
|
||||
resolution,
|
||||
resolution.ranked,
|
||||
"lookup_failed",
|
||||
lookupError,
|
||||
version,
|
||||
);
|
||||
}
|
||||
return toPool(
|
||||
rt,
|
||||
release,
|
||||
observationId,
|
||||
parsed,
|
||||
resolution,
|
||||
resolution.ranked,
|
||||
resolution.status === "ambiguous" ? "ambiguous" : "no_match",
|
||||
resolution.reason,
|
||||
version,
|
||||
);
|
||||
}
|
||||
|
||||
async function recordAttempt(
|
||||
exec: Exec,
|
||||
srId: string,
|
||||
observationId: string | null,
|
||||
outcome: "auto" | "proposed" | "ambiguous" | "no_match" | "lookup_failed" | "manual" | "skipped_locked",
|
||||
reason: string | null,
|
||||
version: string,
|
||||
count: number,
|
||||
entityId: string | null,
|
||||
best: unknown = null,
|
||||
error: unknown = null,
|
||||
actor: Actor = SYSTEM,
|
||||
): Promise<void> {
|
||||
await exec
|
||||
.insertInto("resolution_attempt")
|
||||
.values({
|
||||
source_release_id: srId,
|
||||
observation_id: observationId,
|
||||
outcome,
|
||||
reason,
|
||||
evidence_version: version,
|
||||
candidate_count: count,
|
||||
media_entity_id: entityId,
|
||||
best: best === null ? null : jsonb(best),
|
||||
error: error === null ? null : jsonb(error),
|
||||
actor_type: actor.type,
|
||||
actor_id: actor.id,
|
||||
})
|
||||
.execute();
|
||||
}
|
||||
|
||||
function rankedSummary(r: RankedCandidate | undefined) {
|
||||
return r ? { ...poolCandidate(r), titleMatch: r.titleMatch, yearMatch: r.yearMatch, typeMatch: r.typeMatch } : null;
|
||||
}
|
||||
|
||||
async function bindAuto(
|
||||
rt: SourcesRuntime,
|
||||
release: SourceReleaseRow,
|
||||
observationId: string | null,
|
||||
parsed: ParsedRelease,
|
||||
cols: ReturnType<typeof toParsedColumns>,
|
||||
resolution: IdentityResolution,
|
||||
version: string,
|
||||
tmdbDetail: MetadataCandidate | null,
|
||||
): Promise<ResolveResult> {
|
||||
const { ctx } = rt;
|
||||
const best = resolution.best!;
|
||||
const level: EvidenceLevel = toContractEvidenceLevel(best.level) ?? "needs_review";
|
||||
const site = ((release.raw as Record<string, unknown>)?._site ?? {}) as { categoryName?: string };
|
||||
const variety = looksLikeVariety(parsed, site.categoryName ?? release.category_raw);
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
try {
|
||||
return await ctx.db.transaction().execute(async (trx) => {
|
||||
let entityId: string | null = null;
|
||||
if (best.candidate.localEntityId) entityId = await resolveEntityId(trx, best.candidate.localEntityId);
|
||||
if (!entityId && best.candidate.provider === "tmdb") {
|
||||
entityId = await entityByExternal(trx, {
|
||||
provider: "tmdb",
|
||||
entityType: best.candidate.entityType,
|
||||
externalId: best.candidate.externalId,
|
||||
});
|
||||
if (!entityId && tmdbDetail) {
|
||||
entityId = (
|
||||
await createEntityFromTmdb(trx, tmdbDetail, {
|
||||
level,
|
||||
evidence: {
|
||||
sourceReleaseId: release.id,
|
||||
rules: best.evidence,
|
||||
titleMatch: best.titleMatch,
|
||||
yearMatch: best.yearMatch,
|
||||
},
|
||||
actor: SYSTEM,
|
||||
variety,
|
||||
})
|
||||
).entityId;
|
||||
}
|
||||
}
|
||||
if (!entityId)
|
||||
throw new Error(`auto 结果缺少可绑定的本地作品(${best.candidate.provider}:${best.candidate.externalId})`);
|
||||
const entity = await trx
|
||||
.selectFrom("media_entity")
|
||||
.select(["id", "kind"])
|
||||
.where("id", "=", entityId)
|
||||
.executeTakeFirstOrThrow();
|
||||
const units = await ensureUnits(trx, entityId, entity.kind, parsed);
|
||||
// 站点标题作为别名:下次同名资源可直接本地召回
|
||||
if (parsed.titleCn) await addAlias(trx, entityId, parsed.titleCn, "site_title", "site", "zh");
|
||||
if (parsed.titleEn) await addAlias(trx, entityId, parsed.titleEn, "site_title", "site", "en");
|
||||
|
||||
const evidence = {
|
||||
level,
|
||||
reason: resolution.reason,
|
||||
evidenceVersion: version,
|
||||
best: rankedSummary(best),
|
||||
candidateCount: resolution.ranked.length,
|
||||
};
|
||||
await trx
|
||||
.updateTable("source_release")
|
||||
.set({
|
||||
parser_version: cols.parserVersion,
|
||||
parsed: jsonb(cols.parsed),
|
||||
media_entity_id: entityId,
|
||||
content_unit_id: units.primary,
|
||||
resolution_status: toReleaseResolutionStatus(toContractResolveStatus("auto", { autoAccept: true })),
|
||||
resolution_evidence: jsonb(evidence),
|
||||
})
|
||||
.where("id", "=", release.id)
|
||||
.where("resolution_locked", "=", false)
|
||||
.execute();
|
||||
await trx
|
||||
.updateTable("identity_candidate")
|
||||
.set({
|
||||
status: "resolved",
|
||||
resolved_media_entity_id: entityId,
|
||||
resolved_by: "system",
|
||||
resolved_at: new Date(),
|
||||
next_review_at: null,
|
||||
})
|
||||
.where("source_release_id", "=", release.id)
|
||||
.where("status", "=", "pending")
|
||||
.execute();
|
||||
await closeDecisions(
|
||||
ctx,
|
||||
trx,
|
||||
`identity:${release.id}`,
|
||||
"resolved",
|
||||
{ mediaEntityId: entityId, by: "auto" },
|
||||
"system",
|
||||
);
|
||||
await recordAttempt(
|
||||
trx,
|
||||
release.id,
|
||||
observationId,
|
||||
"auto",
|
||||
resolution.reason,
|
||||
version,
|
||||
resolution.ranked.length,
|
||||
entityId,
|
||||
rankedSummary(best),
|
||||
);
|
||||
await ctx.outbox.emit(
|
||||
trx,
|
||||
"release.resolved",
|
||||
{
|
||||
sourceReleaseId: release.id,
|
||||
observationId,
|
||||
mediaEntityId: entityId,
|
||||
contentUnitIds: units.unitIds,
|
||||
evidenceLevel: level,
|
||||
},
|
||||
{ idempotencyKey: `release.resolved:${release.id}:${entityId}:${version}` },
|
||||
);
|
||||
return {
|
||||
outcome: "auto" as const,
|
||||
sourceReleaseId: release.id,
|
||||
mediaEntityId: entityId,
|
||||
contentUnitIds: units.unitIds,
|
||||
decisionId: null,
|
||||
reason: resolution.reason,
|
||||
};
|
||||
});
|
||||
} catch (e) {
|
||||
if (e instanceof IdentityConflictError && attempt < 2) continue;
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function reviewDelayMs(reason: string, attempts: number): number | null {
|
||||
if (attempts >= MAX_REVIEW_ATTEMPTS) return null;
|
||||
const base = reason === "lookup_failed" ? 15 * 60_000 : 3600_000;
|
||||
return Math.min(base * 2 ** Math.max(0, attempts - 1), 7 * 24 * 3600_000);
|
||||
}
|
||||
|
||||
async function toPool(
|
||||
rt: SourcesRuntime,
|
||||
release: SourceReleaseRow,
|
||||
observationId: string | null,
|
||||
parsed: ParsedRelease,
|
||||
resolution: IdentityResolution | null,
|
||||
ranked: RankedCandidate[],
|
||||
reason: "ambiguous" | "no_match" | "proposed" | "lookup_failed",
|
||||
detail: string,
|
||||
versionIn: string | null,
|
||||
): Promise<ResolveResult> {
|
||||
const { ctx } = rt;
|
||||
const version = versionIn ?? evidenceVersion(parsed, ranked);
|
||||
const cols = toParsedColumns(parsed);
|
||||
const candidates = ranked
|
||||
.filter((r) => r.level !== "rejected")
|
||||
.slice(0, 10)
|
||||
.map(poolCandidate);
|
||||
const now = rt.now();
|
||||
const query = {
|
||||
title: parsed.title ?? null,
|
||||
titleCn: parsed.titleCn ?? null,
|
||||
titleEn: parsed.titleEn ?? null,
|
||||
normalized: parsed.title ? normalizeTitle(parsed.title) : null,
|
||||
year: parsed.year ?? null,
|
||||
season: parsed.season ?? null,
|
||||
episodes: parsed.episodes ?? null,
|
||||
};
|
||||
const contractStatus =
|
||||
reason === "proposed"
|
||||
? toContractResolveStatus("auto", { autoAccept: false })
|
||||
: reason === "ambiguous"
|
||||
? "ambiguous"
|
||||
: "unidentifiable";
|
||||
|
||||
return ctx.db.transaction().execute(async (trx) => {
|
||||
const prev = await trx
|
||||
.selectFrom("identity_candidate")
|
||||
.select(["id", "attempts", "evidence_version"])
|
||||
.where("source_release_id", "=", release.id)
|
||||
.executeTakeFirst();
|
||||
// 证据版本变化 = 新证据:从头计数;同一证据反复复查按退避递增,达上限停止自动复查(仍可检索)
|
||||
const attempts = prev && prev.evidence_version === version ? prev.attempts + 1 : 1;
|
||||
const delay = reviewDelayMs(reason, attempts);
|
||||
const nextReviewAt = delay === null ? null : new Date(now.getTime() + delay);
|
||||
|
||||
let decisionId: string | null = null;
|
||||
if ((reason === "ambiguous" || reason === "proposed") && candidates.length > 0) {
|
||||
const d = await openDecision(ctx, trx, {
|
||||
kind: "identity_ambiguous",
|
||||
severity: "info",
|
||||
title: `待确认身份:${release.title}`,
|
||||
summary: `${candidates.length} 个候选证据等级相近(${detail}),请选择或忽略`,
|
||||
subjectType: "source_release",
|
||||
subjectId: release.id,
|
||||
dedupeKey: `identity:${release.id}`,
|
||||
evidence: {
|
||||
sourceReleaseId: release.id,
|
||||
siteId: release.site_id,
|
||||
reason,
|
||||
detail,
|
||||
evidenceVersion: version,
|
||||
query,
|
||||
},
|
||||
options: [
|
||||
...candidates.slice(0, 5).map((c, i) => ({
|
||||
key: `c${i}`,
|
||||
label: `${c.title}${c.year ? ` (${c.year})` : ""} · ${c.localEntityId ? "本地作品" : `${c.provider}:${c.entityType}/${c.externalId}`}`,
|
||||
input: { sourceReleaseId: release.id, candidateIndex: i },
|
||||
})),
|
||||
{ key: "dismiss", label: "忽略此资源" },
|
||||
],
|
||||
nextReviewAt,
|
||||
});
|
||||
decisionId = d.id;
|
||||
}
|
||||
|
||||
await sql`
|
||||
insert into identity_candidate (source_release_id, status, reason, query, candidates, evidence_version, attempts,
|
||||
last_attempt_at, next_review_at, decision_id)
|
||||
values (${release.id}, 'pending', ${reason}, ${jsonb(query)}::jsonb, ${jsonb(candidates)}::jsonb, ${version}, ${attempts},
|
||||
${now}, ${nextReviewAt}, ${decisionId})
|
||||
on conflict (source_release_id) do update set
|
||||
status = 'pending', reason = excluded.reason, query = excluded.query, candidates = excluded.candidates,
|
||||
evidence_version = excluded.evidence_version, attempts = excluded.attempts, last_attempt_at = excluded.last_attempt_at,
|
||||
next_review_at = excluded.next_review_at, decision_id = coalesce(excluded.decision_id, identity_candidate.decision_id),
|
||||
resolved_media_entity_id = null, resolved_by = null, resolved_at = null`.execute(trx);
|
||||
|
||||
await trx
|
||||
.updateTable("source_release")
|
||||
.set({
|
||||
parser_version: cols.parserVersion,
|
||||
parsed: jsonb(cols.parsed),
|
||||
resolution_status: toReleaseResolutionStatus(contractStatus),
|
||||
resolution_evidence: jsonb({ reason, detail, evidenceVersion: version, candidateCount: candidates.length }),
|
||||
})
|
||||
.where("id", "=", release.id)
|
||||
.where("resolution_locked", "=", false)
|
||||
.execute();
|
||||
if (observationId) {
|
||||
await trx
|
||||
.updateTable("observation")
|
||||
.set((eb) => ({ resolve_attempts: eb("resolve_attempts", "+", 1), next_resolve_at: nextReviewAt }))
|
||||
.where("id", "=", observationId)
|
||||
.execute();
|
||||
}
|
||||
await recordAttempt(
|
||||
trx,
|
||||
release.id,
|
||||
observationId,
|
||||
reason,
|
||||
detail,
|
||||
version,
|
||||
candidates.length,
|
||||
null,
|
||||
rankedSummary(resolution?.best),
|
||||
reason === "lookup_failed" ? { message: detail } : null,
|
||||
);
|
||||
return {
|
||||
outcome: reason,
|
||||
sourceReleaseId: release.id,
|
||||
mediaEntityId: null,
|
||||
contentUnitIds: [],
|
||||
decisionId,
|
||||
reason: detail,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export { recordAttempt };
|
||||
@@ -0,0 +1,229 @@
|
||||
import { SiteError, type SitePage, type SiteRelease } from "@umt/adapters";
|
||||
import type { SiteSnapshotKind } from "@umt/contracts";
|
||||
import { type SiteRow, sql } from "@umt/db";
|
||||
import { parseRelease } from "@umt/domain";
|
||||
import { siteReleaseToSourceRelease } from "@umt/mapping";
|
||||
import { jsonb } from "../rows.ts";
|
||||
import type { SourcesRuntime } from "../runtime.ts";
|
||||
import { type Exec, openDecision } from "./common.ts";
|
||||
|
||||
// 站点快照(周期列表)、补漏(Feed 疑似缺口,有页数预算)、实时补查(工作台搜索)。
|
||||
// 所有请求经站点共享限流器;结果按 site+站点资源 ID upsert 到 source_release,并记录 site_snapshot 覆盖范围。
|
||||
|
||||
export interface SnapshotInput {
|
||||
siteId: string;
|
||||
kind: SiteSnapshotKind;
|
||||
query?: string | undefined;
|
||||
/** 页数预算(1 起连续页);默认 site.config.snapshotPages ?? 1,上限 20 */
|
||||
maxPages?: number | undefined;
|
||||
}
|
||||
|
||||
export interface SnapshotResult {
|
||||
status: "ok" | "partial" | "failed" | "skipped";
|
||||
reason?: string;
|
||||
snapshotId: string | null;
|
||||
pages: number[];
|
||||
itemCount: number;
|
||||
newCount: number;
|
||||
releaseIds: string[];
|
||||
error?: string;
|
||||
}
|
||||
|
||||
/** SiteRelease → source_release upsert;新行发 release.observed */
|
||||
export async function upsertSiteRelease(
|
||||
rt: SourcesRuntime,
|
||||
trx: Exec,
|
||||
site: Pick<SiteRow, "id">,
|
||||
r: SiteRelease,
|
||||
): Promise<{ id: string; inserted: boolean }> {
|
||||
const parsed = parseRelease(r.title, r.subtitle ? { subtitle: r.subtitle } : {});
|
||||
const c = siteReleaseToSourceRelease(r, { siteId: site.id, parsed });
|
||||
const row = (
|
||||
await sql<{ id: string; inserted: boolean }>`
|
||||
insert into source_release (site_id, site_release_id, title, subtitle, category_raw, tags_raw, size_bytes, infohash_v1,
|
||||
infohash_v2, promotion, promotion_expires_at, seeders, leechers, source_published_at, parsed, parser_version, raw,
|
||||
detail_fetched_at)
|
||||
values (${site.id}, ${c.siteReleaseId}, ${c.title}, ${c.subtitle ?? null}, ${c.categoryRaw ?? null}, ${c.tagsRaw ?? []},
|
||||
${c.sizeBytes ?? null}, ${c.infohashV1 ?? null}, ${c.infohashV2 ?? null}, ${jsonb(c.promotion ?? {})}::jsonb,
|
||||
${c.promotionExpiresAt ?? null}, ${c.seeders ?? null}, ${c.leechers ?? null}, ${c.sourcePublishedAt ?? null},
|
||||
${jsonb(c.parsed ?? {})}::jsonb, ${c.parserVersion ?? null}, ${jsonb(c.raw ?? {})}::jsonb, ${null})
|
||||
on conflict (site_id, site_release_id) do update set
|
||||
title = excluded.title, subtitle = coalesce(excluded.subtitle, source_release.subtitle),
|
||||
category_raw = coalesce(excluded.category_raw, source_release.category_raw), tags_raw = excluded.tags_raw,
|
||||
size_bytes = coalesce(excluded.size_bytes, source_release.size_bytes),
|
||||
infohash_v1 = coalesce(excluded.infohash_v1, source_release.infohash_v1),
|
||||
infohash_v2 = coalesce(excluded.infohash_v2, source_release.infohash_v2),
|
||||
promotion = excluded.promotion, promotion_expires_at = excluded.promotion_expires_at,
|
||||
seeders = excluded.seeders, leechers = excluded.leechers,
|
||||
source_published_at = coalesce(source_release.source_published_at, excluded.source_published_at),
|
||||
raw = source_release.raw || excluded.raw, last_seen_at = now(),
|
||||
availability = case when source_release.availability = 'expired' then 'active' else source_release.availability end
|
||||
returning id, (xmax = 0) as inserted`.execute(trx)
|
||||
).rows[0]!;
|
||||
if (row.inserted) {
|
||||
await rt.ctx.outbox.emit(
|
||||
trx,
|
||||
"release.observed",
|
||||
{ sourceReleaseId: row.id, observationId: null, siteId: site.id, isNew: true },
|
||||
{ idempotencyKey: `release.observed:${row.id}` },
|
||||
);
|
||||
}
|
||||
return row;
|
||||
}
|
||||
|
||||
/** 凭据失效:账号标失效 + 站点健康 failing + 待处理(不重试) */
|
||||
async function onAuthError(rt: SourcesRuntime, trx: Exec, site: SiteRow, e: SiteError) {
|
||||
await trx
|
||||
.updateTable("site_account")
|
||||
.set({ status: e.kind === "auth_expired" ? "expired" : "invalid", last_verified_at: rt.now() })
|
||||
.where("site_id", "=", site.id)
|
||||
.where("enabled", "=", true)
|
||||
.where("status", "in", ["unknown", "active"])
|
||||
.execute();
|
||||
await trx.updateTable("site").set({ health_status: "failing" }).where("id", "=", site.id).execute();
|
||||
await openDecision(rt.ctx, trx, {
|
||||
kind: "source_unreachable",
|
||||
severity: "warning",
|
||||
title: `站点「${site.name}」登录失效`,
|
||||
summary: `${e.kind}:${e.message}。已停止该站自动访问,请更新账号凭证后恢复。`,
|
||||
subjectType: "site",
|
||||
subjectId: site.id,
|
||||
dedupeKey: `site-auth:${site.id}`,
|
||||
evidence: { siteId: site.id, kind: e.kind, httpStatus: e.httpStatus ?? null },
|
||||
options: [{ key: "reset", label: "已更新凭证,恢复访问" }],
|
||||
});
|
||||
}
|
||||
|
||||
export async function snapshotSite(rt: SourcesRuntime, input: SnapshotInput): Promise<SnapshotResult> {
|
||||
const { db } = rt.ctx;
|
||||
const none = { snapshotId: null, pages: [], itemCount: 0, newCount: 0, releaseIds: [] };
|
||||
const site = await db.selectFrom("site").selectAll().where("id", "=", input.siteId).executeTakeFirst();
|
||||
if (!site) return { status: "skipped", reason: "not_found", ...none };
|
||||
if (!site.enabled) return { status: "skipped", reason: "site_disabled", ...none };
|
||||
if (site.health_status === "failing" && input.kind !== "manual" && input.kind !== "live_search") {
|
||||
const invalid = await db
|
||||
.selectFrom("site_account")
|
||||
.select(["id"])
|
||||
.where("site_id", "=", site.id)
|
||||
.where("status", "in", ["invalid", "expired"])
|
||||
.executeTakeFirst();
|
||||
if (invalid) return { status: "skipped", reason: "auth_invalid", ...none };
|
||||
}
|
||||
let adapter: ReturnType<SourcesRuntime["siteAdapter"]>;
|
||||
try {
|
||||
adapter = rt.siteAdapter(site);
|
||||
} catch (e) {
|
||||
return { status: "skipped", reason: (e as Error).message, ...none };
|
||||
}
|
||||
const cfgPages = Number((site.config as { snapshotPages?: number })?.snapshotPages) || 1;
|
||||
const maxPages = Math.min(20, Math.max(1, input.maxPages ?? cfgPages));
|
||||
const snap = await db
|
||||
.insertInto("site_snapshot")
|
||||
.values({
|
||||
site_id: site.id,
|
||||
kind: input.kind,
|
||||
query: input.query ?? null,
|
||||
filters: jsonb({}),
|
||||
definition_version: adapter.definitionVersion,
|
||||
started_at: rt.now(),
|
||||
})
|
||||
.returning(["id"])
|
||||
.executeTakeFirstOrThrow();
|
||||
|
||||
const pages: number[] = [];
|
||||
const releases: SiteRelease[] = [];
|
||||
let error: SiteError | Error | null = null;
|
||||
for (let p = 1; p <= maxPages; p++) {
|
||||
try {
|
||||
const res: SitePage = input.query ? await adapter.search(input.query, { page: p }) : await adapter.list(p);
|
||||
pages.push(p);
|
||||
releases.push(...res.items);
|
||||
if (res.items.length === 0 || (res.totalPages !== null && p >= res.totalPages)) break;
|
||||
} catch (e) {
|
||||
error = e as Error;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return db.transaction().execute(async (trx) => {
|
||||
const ids: string[] = [];
|
||||
let newCount = 0;
|
||||
for (const r of releases) {
|
||||
const row = await upsertSiteRelease(rt, trx, site, r);
|
||||
ids.push(row.id);
|
||||
if (row.inserted) newCount++;
|
||||
}
|
||||
const status = error ? (pages.length ? "partial" : "failed") : "ok";
|
||||
const errJson = error
|
||||
? { kind: error instanceof SiteError ? error.kind : "internal", message: error.message }
|
||||
: null;
|
||||
await trx
|
||||
.updateTable("site_snapshot")
|
||||
.set({
|
||||
status,
|
||||
pages: jsonb(pages),
|
||||
item_count: releases.length,
|
||||
new_count: newCount,
|
||||
error: errJson ? jsonb(errJson) : null,
|
||||
finished_at: rt.now(),
|
||||
})
|
||||
.where("id", "=", snap.id)
|
||||
.execute();
|
||||
if (error instanceof SiteError && error.isAuthError) {
|
||||
await onAuthError(rt, trx, site, error);
|
||||
} else if (input.kind !== "live_search") {
|
||||
await trx
|
||||
.updateTable("site")
|
||||
.set({
|
||||
...(status !== "failed" ? { last_snapshot_at: rt.now() } : {}),
|
||||
health_status: status === "ok" ? "healthy" : "degraded",
|
||||
snapshot_cursor: jsonb({
|
||||
lastKind: input.kind,
|
||||
lastPages: pages,
|
||||
lastItemCount: releases.length,
|
||||
lastStatus: status,
|
||||
lastError: errJson,
|
||||
at: rt.now().toISOString(),
|
||||
}),
|
||||
})
|
||||
.where("id", "=", site.id)
|
||||
.execute();
|
||||
}
|
||||
return {
|
||||
status,
|
||||
snapshotId: snap.id,
|
||||
pages,
|
||||
itemCount: releases.length,
|
||||
newCount,
|
||||
releaseIds: ids,
|
||||
...(errJson ? { error: `${errJson.kind}: ${errJson.message}` } : {}),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/** 详情刷新(工作台 / release.inspect):失败保留既有数据并返回原因 */
|
||||
export async function refreshReleaseDetail(rt: SourcesRuntime, srId: string): Promise<string | null> {
|
||||
const { db } = rt.ctx;
|
||||
const rel = await db.selectFrom("source_release").selectAll().where("id", "=", srId).executeTakeFirst();
|
||||
if (!rel) return "not_found";
|
||||
const site = await db.selectFrom("site").selectAll().where("id", "=", rel.site_id).executeTakeFirst();
|
||||
if (!site?.enabled) return "site_disabled";
|
||||
let detail: SiteRelease;
|
||||
try {
|
||||
detail = await rt.siteAdapter(site).detail(rel.site_release_id);
|
||||
} catch (e) {
|
||||
if (e instanceof SiteError) {
|
||||
if (e.isAuthError) await db.transaction().execute((trx) => onAuthError(rt, trx, site, e));
|
||||
if (e.kind === "not_found") {
|
||||
await db.updateTable("source_release").set({ availability: "expired" }).where("id", "=", srId).execute();
|
||||
}
|
||||
return `${e.kind}: ${e.message}`;
|
||||
}
|
||||
return (e as Error).message;
|
||||
}
|
||||
await db.transaction().execute(async (trx) => {
|
||||
await upsertSiteRelease(rt, trx, site, detail);
|
||||
await trx.updateTable("source_release").set({ detail_fetched_at: rt.now() }).where("id", "=", srId).execute();
|
||||
});
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
import type {
|
||||
ReleaseOrigin,
|
||||
WorkbenchCoverage,
|
||||
WorkbenchRelease,
|
||||
WorkbenchReleaseDetail,
|
||||
WorkbenchReleaseQuery,
|
||||
} from "@umt/contracts";
|
||||
import { type SiteRow, sql } from "@umt/db";
|
||||
import { normalizeTitle } from "@umt/domain";
|
||||
import { ApiException } from "../../../http/errors.ts";
|
||||
import { decodeCursor, page, toApi } from "../rows.ts";
|
||||
import type { SourcesRuntime } from "../runtime.ts";
|
||||
import { refreshReleaseDetail, snapshotSite } from "./snapshot.ts";
|
||||
|
||||
// 站点工作台(整理视图):本地快照优先;本地无结果 / 快照过期 / 用户要求时对已开启站点实时补查,
|
||||
// 并发相同搜索合并,失败短期退避。库存叠加由前端按 mediaEntityId 查 W2-C 端点。
|
||||
|
||||
const LIVE_FAIL_BACKOFF_MS = 60_000;
|
||||
|
||||
interface LiveState {
|
||||
inflight: Map<string, Promise<{ error: string | null }>>;
|
||||
failedUntil: Map<string, { until: number; error: string }>;
|
||||
}
|
||||
|
||||
const liveStates = new WeakMap<SourcesRuntime, LiveState>();
|
||||
function liveState(rt: SourcesRuntime): LiveState {
|
||||
let s = liveStates.get(rt);
|
||||
if (!s) {
|
||||
s = { inflight: new Map(), failedUntil: new Map() };
|
||||
liveStates.set(rt, s);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
/** 合并并发的相同实时搜索;失败后 LIVE_FAIL_BACKOFF_MS 内直接返回上次错误 */
|
||||
export async function liveSearch(
|
||||
rt: SourcesRuntime,
|
||||
site: SiteRow,
|
||||
q: string,
|
||||
): Promise<{ error: string | null; coalesced: boolean }> {
|
||||
const st = liveState(rt);
|
||||
const key = `${site.id}|${normalizeTitle(q) || q.toLowerCase()}`;
|
||||
const failed = st.failedUntil.get(key);
|
||||
if (failed && failed.until > rt.now().getTime()) return { error: `${failed.error}(退避中)`, coalesced: false };
|
||||
const running = st.inflight.get(key);
|
||||
if (running) return { ...(await running), coalesced: true };
|
||||
const p = (async () => {
|
||||
const r = await snapshotSite(rt, { siteId: site.id, kind: "live_search", query: q, maxPages: 1 });
|
||||
const error = r.status === "failed" || r.status === "skipped" ? (r.error ?? r.reason ?? "failed") : null;
|
||||
if (error) st.failedUntil.set(key, { until: rt.now().getTime() + LIVE_FAIL_BACKOFF_MS, error });
|
||||
else st.failedUntil.delete(key);
|
||||
return { error };
|
||||
})().finally(() => st.inflight.delete(key));
|
||||
st.inflight.set(key, p);
|
||||
return { ...(await p), coalesced: false };
|
||||
}
|
||||
|
||||
async function lastSnapshotAt(rt: SourcesRuntime, siteId: string): Promise<Date | null> {
|
||||
const r = await rt.ctx.db
|
||||
.selectFrom("site_snapshot")
|
||||
.select(["finished_at"])
|
||||
.where("site_id", "=", siteId)
|
||||
.where("kind", "in", ["periodic", "manual", "backfill"])
|
||||
.where("status", "in", ["ok", "partial"])
|
||||
.orderBy("started_at", "desc")
|
||||
.executeTakeFirst();
|
||||
return r?.finished_at ?? null;
|
||||
}
|
||||
|
||||
interface ReleaseJoinRow {
|
||||
[k: string]: unknown;
|
||||
observed_by_feeds: number;
|
||||
entity_title: string | null;
|
||||
entity_year: number | null;
|
||||
entity_kind: string | null;
|
||||
}
|
||||
|
||||
function toWorkbench(row: ReleaseJoinRow): WorkbenchRelease {
|
||||
const { observed_by_feeds, entity_title, entity_year, entity_kind, ...rest } = row;
|
||||
const base = toApi<Omit<WorkbenchRelease, "origin" | "dataUpdatedAt" | "observedByFeeds" | "entity">>(rest);
|
||||
const raw = (base.raw ?? {}) as Record<string, unknown>;
|
||||
const origin: ReleaseOrigin = raw._site ? "snapshot" : Number(observed_by_feeds) > 0 ? "feed" : "unknown";
|
||||
return {
|
||||
...base,
|
||||
origin,
|
||||
dataUpdatedAt: base.lastSeenAt,
|
||||
observedByFeeds: Number(observed_by_feeds),
|
||||
entity:
|
||||
base.mediaEntityId && entity_title
|
||||
? { id: base.mediaEntityId, title: entity_title, year: entity_year, kind: entity_kind ?? "other" }
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
function releaseSelect(rt: SourcesRuntime) {
|
||||
return rt.ctx.db
|
||||
.selectFrom("source_release as r")
|
||||
.leftJoin("media_entity as e", "e.id", "r.media_entity_id")
|
||||
.selectAll("r")
|
||||
.select([
|
||||
"e.title as entity_title",
|
||||
"e.year as entity_year",
|
||||
"e.kind as entity_kind",
|
||||
sql<number>`(select count(distinct o.feed_id) from observation o where o.source_release_id = r.id)::int`.as(
|
||||
"observed_by_feeds",
|
||||
),
|
||||
]);
|
||||
}
|
||||
|
||||
export async function listSiteReleases(
|
||||
rt: SourcesRuntime,
|
||||
siteId: string,
|
||||
q: WorkbenchReleaseQuery,
|
||||
): Promise<{ items: WorkbenchRelease[]; nextCursor: string | null; coverage: WorkbenchCoverage }> {
|
||||
const site = await rt.ctx.db.selectFrom("site").selectAll().where("id", "=", siteId).executeTakeFirst();
|
||||
if (!site) throw new ApiException("not_found", "站点不存在");
|
||||
const snapAt = await lastSnapshotAt(rt, site.id);
|
||||
const periodMs = (site.snapshot_interval_sec ?? 3600) * 1000;
|
||||
const stale = !snapAt || rt.now().getTime() - snapAt.getTime() > periodMs * 2;
|
||||
const offset = decodeCursor(q.cursor);
|
||||
|
||||
const run = async () => {
|
||||
let query = releaseSelect(rt).where("r.site_id", "=", site.id);
|
||||
if (q.q) {
|
||||
const like = `%${q.q.replace(/[\\%_]/g, (m) => `\\${m}`)}%`;
|
||||
query = query.where((eb) =>
|
||||
eb.or([
|
||||
eb("r.title", "ilike", like),
|
||||
eb("r.subtitle", "ilike", like),
|
||||
eb("r.site_release_id", "=", q.q!),
|
||||
sql<boolean>`similarity(r.title, ${q.q}) >= 0.3`,
|
||||
]),
|
||||
);
|
||||
}
|
||||
if (q.category) query = query.where("r.category_raw", "=", q.category);
|
||||
if (q.promotion) query = query.where(sql`r.promotion->>'kind'`, "=", q.promotion);
|
||||
if (q.resolution) query = query.where("r.resolution_status", "=", q.resolution);
|
||||
return (await query
|
||||
.orderBy(sql`coalesce(r.source_published_at, r.first_seen_at)`, "desc")
|
||||
.orderBy("r.id")
|
||||
.offset(offset)
|
||||
.limit(q.limit + 1)
|
||||
.execute()) as unknown as ReleaseJoinRow[];
|
||||
};
|
||||
|
||||
let rows = await run();
|
||||
let live = false;
|
||||
let liveError: string | null = null;
|
||||
let coalesced = false;
|
||||
const wantLive = q.live === "true" || (q.live !== "false" && offset === 0 && !!q.q && (rows.length === 0 || stale));
|
||||
if (wantLive && q.q) {
|
||||
if (!site.enabled) liveError = "站点已关闭,只展示历史数据";
|
||||
else {
|
||||
const r = await liveSearch(rt, site, q.q);
|
||||
live = r.error === null;
|
||||
liveError = r.error;
|
||||
coalesced = r.coalesced;
|
||||
if (live) rows = await run();
|
||||
}
|
||||
}
|
||||
const pg = page(rows, offset, q.limit);
|
||||
return {
|
||||
items: pg.items.map(toWorkbench),
|
||||
nextCursor: pg.nextCursor,
|
||||
coverage: {
|
||||
siteEnabled: site.enabled,
|
||||
lastSnapshotAt: snapAt?.toISOString() ?? null,
|
||||
stale,
|
||||
fromSnapshot: rows.length > 0,
|
||||
live,
|
||||
liveError,
|
||||
coalesced,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function releaseDetail(
|
||||
rt: SourcesRuntime,
|
||||
siteId: string,
|
||||
releaseId: string,
|
||||
refresh: boolean,
|
||||
): Promise<WorkbenchReleaseDetail> {
|
||||
const find = async () =>
|
||||
(await releaseSelect(rt)
|
||||
.where("r.site_id", "=", siteId)
|
||||
.where("r.id", "=", releaseId)
|
||||
.executeTakeFirst()) as unknown as ReleaseJoinRow | undefined;
|
||||
if (!(await find())) throw new ApiException("not_found", "资源不存在");
|
||||
const refreshError = refresh ? await refreshReleaseDetail(rt, releaseId) : null;
|
||||
const row = (await find())!;
|
||||
const release = toWorkbench(row);
|
||||
const obs = await rt.ctx.db
|
||||
.selectFrom("observation as o")
|
||||
.innerJoin("feed_source as f", "f.id", "o.feed_id")
|
||||
.select(["o.id", "o.feed_id", "f.name", "o.first_seen_at", "o.last_seen_at"])
|
||||
.where("o.source_release_id", "=", releaseId)
|
||||
.orderBy("o.first_seen_at")
|
||||
.execute();
|
||||
const promo = (release.promotion ?? {}) as { observedAt?: string };
|
||||
return {
|
||||
release,
|
||||
observations: obs.map((o) => ({
|
||||
observationId: o.id,
|
||||
feedId: o.feed_id,
|
||||
feedName: o.name,
|
||||
firstSeenAt: o.first_seen_at.toISOString(),
|
||||
lastSeenAt: o.last_seen_at.toISOString(),
|
||||
})),
|
||||
volatile: {
|
||||
promotionObservedAt: promo.observedAt ?? null,
|
||||
promotionExpiresAt: release.promotionExpiresAt,
|
||||
peersObservedAt: release.lastSeenAt,
|
||||
detailFetchedAt: release.detailFetchedAt,
|
||||
recheckBeforeDownload: true,
|
||||
},
|
||||
refreshError,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
import { defineTask, type OutboxRouteFn, type TaskRegistry } from "@umt/tasks";
|
||||
import { z } from "zod";
|
||||
import type { SourcesRuntime } from "./runtime.ts";
|
||||
import { ingestFeed } from "./service/feeds.ts";
|
||||
import { refreshMetadata } from "./service/metadata.ts";
|
||||
import { resolveRelease } from "./service/resolve.ts";
|
||||
import { snapshotSite } from "./service/snapshot.ts";
|
||||
|
||||
// 持久任务:输入只放业务 ID,处理器读业务表。外部请求失败的退避/熔断由适配器状态与业务表承担,
|
||||
// 凭据失效返回正常结果(不重试);只有意外错误(DB、TMDB 网络)抛出交给引擎重试。
|
||||
|
||||
const NETWORK_RETRY = {
|
||||
maximumAttempts: 5,
|
||||
initialInterval: "1s",
|
||||
backoffCoefficient: 2,
|
||||
maximumInterval: "60s",
|
||||
} as const;
|
||||
|
||||
const id = z.string().uuid();
|
||||
|
||||
/** Feed 疑似缺口时的补漏页数预算 */
|
||||
export const BACKFILL_MAX_PAGES = 3;
|
||||
|
||||
export function registerSourcesTasks(registry: TaskRegistry, rt: () => SourcesRuntime): void {
|
||||
registry.register(
|
||||
defineTask({
|
||||
name: "ingestFeed",
|
||||
version: 1,
|
||||
inputSchema: z.object({ feedSourceId: id, manual: z.boolean().optional() }),
|
||||
retryPolicy: NETWORK_RETRY,
|
||||
handler: (t) => t.step.run("ingest", () => ingestFeed(rt(), t.input.feedSourceId, { manual: !!t.input.manual })),
|
||||
}),
|
||||
defineTask({
|
||||
name: "resolveObservation",
|
||||
version: 1,
|
||||
inputSchema: z
|
||||
.object({ observationId: id.nullish(), sourceReleaseId: id.nullish(), force: z.boolean().optional() })
|
||||
.refine((v) => !!v.observationId || !!v.sourceReleaseId, { message: "需要 observationId 或 sourceReleaseId" }),
|
||||
retryPolicy: NETWORK_RETRY,
|
||||
handler: (t) =>
|
||||
t.step.run("resolve", () =>
|
||||
resolveRelease(rt(), {
|
||||
observationId: t.input.observationId ?? undefined,
|
||||
sourceReleaseId: t.input.sourceReleaseId ?? undefined,
|
||||
force: t.input.force,
|
||||
}),
|
||||
),
|
||||
}),
|
||||
defineTask({
|
||||
name: "refreshMetadata",
|
||||
version: 1,
|
||||
inputSchema: z.object({ mediaEntityId: id }),
|
||||
retryPolicy: NETWORK_RETRY,
|
||||
handler: (t) => t.step.run("refresh", () => refreshMetadata(rt(), t.input.mediaEntityId)),
|
||||
}),
|
||||
defineTask({
|
||||
name: "snapshotSite",
|
||||
version: 1,
|
||||
inputSchema: z.object({
|
||||
siteId: id,
|
||||
kind: z.enum(["periodic", "backfill", "manual"]).default("periodic"),
|
||||
maxPages: z.number().int().min(1).max(20).optional(),
|
||||
}),
|
||||
retryPolicy: NETWORK_RETRY,
|
||||
handler: (t) =>
|
||||
t.step.run("snapshot", () =>
|
||||
snapshotSite(rt(), { siteId: t.input.siteId, kind: t.input.kind, maxPages: t.input.maxPages }),
|
||||
),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
type P = Record<string, unknown>;
|
||||
|
||||
export const sourcesOutboxRoutes: Record<string, OutboxRouteFn> = {
|
||||
"feed.due": (ev) => {
|
||||
const p = ev.payload as P;
|
||||
return { task: "ingestFeed", input: { feedSourceId: p.feedId, manual: p.reason === "manual" } };
|
||||
},
|
||||
"release.observed": (ev) => {
|
||||
const p = ev.payload as P;
|
||||
return {
|
||||
task: "resolveObservation",
|
||||
input: { sourceReleaseId: p.sourceReleaseId, observationId: p.observationId },
|
||||
};
|
||||
},
|
||||
"feed.gap_suspected": (ev) => {
|
||||
const p = ev.payload as P;
|
||||
if (!p.siteId) return null;
|
||||
return { task: "snapshotSite", input: { siteId: p.siteId, kind: "backfill", maxPages: BACKFILL_MAX_PAGES } };
|
||||
},
|
||||
};
|
||||
Reference in new issue
Block a user