feat(server): W2-B sources, identity resolution, metadata and site workbench

- Sites / accounts / feeds: registry view (/site-definitions), site CRUD with
  enable/disable (history stays queryable, marked site_disabled), accounts and
  feed URLs stored as secrets (only secret_ref / redacted urlDisplay exposed),
  feed enable/disable/reset/fetch; one shared rate limiter + adapter per site.
- Tasks: ingestFeed (FeedFetchState/FeedCursor on feed_source, observation
  feed+guid upsert with digest-change updates, source_release site+id
  aggregation across feeds, auth failure -> account invalid + decision, no
  retry, gap -> feed.gap_suspected -> budgeted backfill snapshot),
  resolveObservation (parseRelease -> local recall via title_alias pg_trgm +
  accepted identities -> TMDB -> resolveIdentity; auto binds entity /
  provisional units / identities / aliases, otherwise identity_candidate pool
  with evidence version and bounded review backoff; manual lock respected),
  refreshMetadata (metadata_field + field_override, user locks win, paths
  untouched), snapshotSite (site_snapshot coverage records).
- Events: feed.due, feed.gap_suspected, release.observed (self-consumed),
  release.resolved, decision.created/updated; scheduler in worker start().
- Identity API: entity detail with redirects, manual mapping correction with
  identity_revision, merge with identity_redirect, pool list/assign/dismiss/
  retry, content unit numbering lock, aliases, external identities.
- Decisions: list/summary (by entity and kind)/detail/resolve; identity
  options executed directly.
- Workbench: /sites/{id}/releases (snapshot first, coalesced live search with
  backoff, origin + freshness) and /sites/{id}/releases/{rid} (volatile field
  recheck times, refresh keeps data on failure).
- Capabilities: catalog.lookup, release.search, release.inspect,
  identity.resolve.
- Migrations 0018-0020: site.adapter api, source_release.resolution_locked,
  site_snapshot, identity_candidate, resolution_attempt, identity_redirect,
  metadata_field, field_override.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed 2026-09-27 20:12:01 +08:00
1 parent 829ca4f7ac
commit a24bda952b
39 files changed
+6861 -15

No files matched your search

+81 -1
View File
@@ -1,5 +1,6 @@
import { z } from "zod";
import { DateOnly, EntityBase, Id, JsonObject, SERVER_MANAGED, Timestamp } from "./common.ts";
import { ActorType, DateOnly, EntityBase, Id, JsonObject, SERVER_MANAGED, Timestamp } from "./common.ts";
import { ExternalIdentity } from "./identity.ts";
// 内容域:本地作品、内容单元、别名(REQ §3.1、§5.1、§5.4、§7.5)
// 本地 UUID 是主身份;季/集编号、外部 ID 都可修订,不是唯一身份。
@@ -127,3 +128,82 @@ export const TitleAliasCreate = TitleAlias.pick({
export type TitleAliasCreate = z.infer<typeof TitleAliasCreate>;
export const TitleAliasUpdate = TitleAliasCreate.omit({ mediaEntityId: true }).partial();
export type TitleAliasUpdate = z.infer<typeof TitleAliasUpdate>;
// ── W2-B:字段级元数据(REQ §7.3)与作品详情 ──
export const MetadataSource = z.enum(["tmdb", "user", "agent", "nfo", "site", "import"]);
export type MetadataSource = z.infer<typeof MetadataSource>;
/** 某来源给出的一个候选字段值(刷新只写这里,不直接改最终展示值) */
export const MetadataField = EntityBase.extend({
mediaEntityId: Id.nullable(),
contentUnitId: Id.nullable(),
field: z.string().min(1),
value: z.unknown(),
language: z.string().nullable(),
source: MetadataSource,
sourceRevision: z.string().nullable(),
fetchedAt: Timestamp,
});
export type MetadataField = z.infer<typeof MetadataField>;
/** 字段覆盖:用户锁定优先于任何自动刷新 */
export const FieldOverride = EntityBase.extend({
mediaEntityId: Id.nullable(),
contentUnitId: Id.nullable(),
field: z.string().min(1),
value: z.unknown(),
locked: z.boolean(),
source: MetadataSource,
reason: z.string().nullable(),
actorType: ActorType,
actorId: z.string().nullable(),
});
export type FieldOverride = z.infer<typeof FieldOverride>;
/** 可覆盖/锁定的作品字段;title/originalTitle/year 同时投影到 media_entity 列 */
export const EntityMetadataField = z.enum([
"title",
"originalTitle",
"year",
"overview",
"releaseDate",
"originalLanguage",
"originCountry",
"posterPath",
]);
export type EntityMetadataField = z.infer<typeof EntityMetadataField>;
export const FieldOverrideInput = z.object({
value: z.unknown(),
lock: z.boolean().default(true),
reason: z.string().max(2000).optional(),
});
export type FieldOverrideInput = z.infer<typeof FieldOverrideInput>;
/** media_entity.metadata 中每个最终字段的形状 */
export const FinalFieldValue = z.object({
value: z.unknown(),
source: MetadataSource,
language: z.string().nullable(),
at: Timestamp,
locked: z.boolean(),
});
export type FinalFieldValue = z.infer<typeof FinalFieldValue>;
/** 作品详情:外部身份、别名、内容单元、字段来源与覆盖 */
export const MediaEntityDetail = z.object({
entity: MediaEntity,
/** 请求的是已合并的旧 ID 时,指向跳转后的实体 */
redirectedFrom: Id.nullable(),
identities: z.array(ExternalIdentity),
aliases: z.array(TitleAlias),
contentUnits: z.array(ContentUnit),
metadataFields: z.array(MetadataField),
overrides: z.array(FieldOverride),
});
export type MediaEntityDetail = z.infer<typeof MediaEntityDetail>;
/** 合并重复作品(保留旧 ID 跳转) */
export const MediaEntityMerge = z.object({ intoId: Id, reason: z.string().max(2000).optional() });
export type MediaEntityMerge = z.infer<typeof MediaEntityMerge>;
+7 -1
View File
@@ -4,7 +4,7 @@ import { Id, NonNegInt, Timestamp } from "./common.ts";
import { DownloadStatus, PauseReason, TaskKind, TaskStatus } from "./execution.ts";
import { EvidenceLevel } from "./identity.ts";
import { OrganizationStatus } from "./organization.ts";
import { FeedHealth } from "./sources.ts";
import { FeedDuePayload, FeedGapSuspectedPayload, FeedHealth, ReleaseObservedPayload } from "./sources.ts";
/** SSE 事件信封:seq 在进程内单调递增,客户端断线后用 ?after=<seq> 补读 */
export const EventEnvelope = z.object({
@@ -41,7 +41,10 @@ export const EventType = {
feedFetched: "feed.fetched",
feedHealthChanged: "feed.health_changed",
feedDue: "feed.due",
feedGapSuspected: "feed.gap_suspected",
releaseObserved: "release.observed",
releaseResolved: "release.resolved",
decisionCreated: "decision.created",
@@ -175,6 +178,9 @@ export const EVENT_PAYLOADS = {
"agent.tool_call": AgentToolCallPayload,
"feed.fetched": FeedFetchedPayload,
"feed.health_changed": FeedHealthChangedPayload,
"feed.due": FeedDuePayload,
"feed.gap_suspected": FeedGapSuspectedPayload,
"release.observed": ReleaseObservedPayload,
"release.resolved": ReleaseResolvedPayload,
"decision.created": DecisionCreatedPayload,
"decision.updated": DecisionUpdatedPayload,
+139
View File
@@ -119,3 +119,142 @@ export const IdentityRevisionCreate = IdentityRevision.pick({
reason: true,
}).partial({ fromStatus: true, toStatus: true, before: true, actorId: true, reason: true });
export type IdentityRevisionCreate = z.infer<typeof IdentityRevisionCreate>;
// ── W2-B:待识别池、识别尝试、合并跳转与人工纠错(REQ §7.2、§7.5) ──
export const IdentityCandidateStatus = z.enum(["pending", "resolved", "dismissed", "archived"]);
export type IdentityCandidateStatus = z.infer<typeof IdentityCandidateStatus>;
/** proposed = 证据足够但自动化关闭/只读,待确认 */
export const IdentityCandidateReason = z.enum(["ambiguous", "no_match", "proposed", "lookup_failed"]);
export type IdentityCandidateReason = z.infer<typeof IdentityCandidateReason>;
/** 待识别池里的一个候选(只存 JSON,不建正式作品) */
export const PoolCandidate = z.object({
provider: z.string(),
entityType: z.enum(["movie", "tv"]),
externalId: z.string(),
title: z.string(),
year: z.number().int().nullable(),
/** 本地候选的作品 ID */
localEntityId: Id.nullable(),
level: z.enum(["rule_confirmed", "multi_evidence", "needs_confirm", "rejected"]),
evidence: z.array(z.string()),
});
export type PoolCandidate = z.infer<typeof PoolCandidate>;
/** 待识别池条目:每条 source_release 至多一条;失败原因、证据版本与下一次复查时间 */
export const IdentityCandidate = EntityBase.extend({
sourceReleaseId: Id,
status: IdentityCandidateStatus,
reason: IdentityCandidateReason,
/** 解析出的检索条件(标题/年份/季集) */
query: JsonObject,
candidates: z.array(PoolCandidate),
evidenceVersion: z.string(),
attempts: z.number().int().nonnegative(),
lastAttemptAt: Timestamp.nullable(),
nextReviewAt: Timestamp.nullable(),
decisionId: Id.nullable(),
resolvedMediaEntityId: Id.nullable(),
resolvedBy: z.string().nullable(),
resolvedAt: Timestamp.nullable(),
});
export type IdentityCandidate = z.infer<typeof IdentityCandidate>;
export const ResolutionOutcome = z.enum([
"auto",
"proposed",
"ambiguous",
"no_match",
"lookup_failed",
"manual",
"skipped_locked",
]);
export type ResolutionOutcome = z.infer<typeof ResolutionOutcome>;
/** 一次识别尝试(只追加) */
export const ResolutionAttempt = EntityBase.extend({
sourceReleaseId: Id,
observationId: Id.nullable(),
outcome: ResolutionOutcome,
reason: z.string().nullable(),
evidenceVersion: z.string(),
candidateCount: z.number().int().nonnegative(),
mediaEntityId: Id.nullable(),
best: JsonObject.nullable(),
error: JsonObject.nullable(),
actorType: ActorType,
actorId: z.string().nullable(),
});
export type ResolutionAttempt = z.infer<typeof ResolutionAttempt>;
/** 作品合并后的旧 ID 跳转 */
export const IdentityRedirect = EntityBase.extend({
fromEntityId: Id,
toEntityId: Id,
reason: z.string().nullable(),
actorType: ActorType,
actorId: z.string().nullable(),
});
export type IdentityRedirect = z.infer<typeof IdentityRedirect>;
/** 外部身份引用(provider + 类型 + ID) */
export const ExternalRef = z.object({
provider: IdentityProvider,
entityType: z.enum(["movie", "tv"]),
externalId: z.string().min(1),
});
export type ExternalRef = z.infer<typeof ExternalRef>;
/**
* 人工指定来源资源的身份:本地作品 / 外部身份(未登记时按外部详情建档)/ 池内候选序号,三选一。
* 指定后加人工锁,自动识别不再改写。
*/
export const ReleaseIdentityAssign = z
.object({
mediaEntityId: Id.optional(),
external: ExternalRef.optional(),
candidateIndex: z.number().int().min(0).optional(),
/** 同时关联到某个内容单元(季/集);缺省按解析出的季集建临时单元 */
contentUnitId: Id.nullable().optional(),
lock: z.boolean().default(true),
reason: z.string().max(2000).optional(),
})
.refine((v) => [v.mediaEntityId, v.external, v.candidateIndex].filter((x) => x !== undefined).length === 1, {
message: "mediaEntityId / external / candidateIndex 必须且只能给一个",
});
export type ReleaseIdentityAssign = z.infer<typeof ReleaseIdentityAssign>;
/** 改作品的外部映射:旧的同 provider+类型 accepted 映射退役(留修订),新映射 accepted + 人工锁 */
export const EntityIdentitySet = ExternalRef.extend({
reason: z.string().max(2000).optional(),
/** 默认加人工锁 */
lock: z.boolean().default(true),
});
export type EntityIdentitySet = z.infer<typeof EntityIdentitySet>;
export const IdentityPoolQuery = z.object({
status: IdentityCandidateStatus.optional(),
reason: IdentityCandidateReason.optional(),
/** 按原始标题/解析标题检索(已观察到的资源必须能从待识别入口找回) */
q: z.string().trim().min(1).max(200).optional(),
siteId: Id.optional(),
cursor: z.string().min(1).optional(),
limit: z.coerce.number().int().min(1).max(200).default(50),
});
export type IdentityPoolQuery = z.infer<typeof IdentityPoolQuery>;
/** 待识别池列表条目:附来源资源摘要 */
export const IdentityPoolItem = IdentityCandidate.extend({
release: z.object({
id: Id,
siteId: Id,
siteReleaseId: z.string(),
title: z.string(),
subtitle: z.string().nullable(),
categoryRaw: z.string().nullable(),
firstSeenAt: Timestamp,
resolutionStatus: z.string(),
}),
});
export type IdentityPoolItem = z.infer<typeof IdentityPoolItem>;
+145 -1
View File
@@ -4,7 +4,8 @@ import { EntityBase, Id, JsonObject, NonNegInt, SERVER_MANAGED, Timestamp } from
// 来源域:站点、站点账号、Feed、站点资源、观察(REQ §4)
// 含 passkey 的 Feed URL 只以 urlSecretRef 保存,对外只给脱敏的 urlDisplay。
export const SiteAdapter = z.enum(["nexusphp", "mteam_api", "generic_rss"]);
/** api = MP 专用 API spider(Haidan/HDDolby/RousiPro/SunnyPT/TNode/TorrentLeech/Yema) */
export const SiteAdapter = z.enum(["nexusphp", "mteam_api", "api", "generic_rss"]);
export type SiteAdapter = z.infer<typeof SiteAdapter>;
export const SiteHealth = z.enum(["unknown", "healthy", "degraded", "failing", "disabled"]);
export type SiteHealth = z.infer<typeof SiteHealth>;
@@ -180,6 +181,8 @@ export const SourceRelease = EntityBase.extend({
resolutionEvidence: JsonObject,
availability: ReleaseAvailability,
raw: JsonObject,
/** 人工确认的身份关联:自动识别/复查不得改写 mediaEntityId/contentUnitId */
resolutionLocked: z.boolean(),
});
export type SourceRelease = z.infer<typeof SourceRelease>;
/** 由适配器/任务写入(按 siteId+siteReleaseId upsert) */
@@ -209,6 +212,7 @@ export const SourceReleaseCreate = SourceRelease.omit({
resolutionEvidence: true,
availability: true,
raw: true,
resolutionLocked: true,
});
export type SourceReleaseCreate = z.infer<typeof SourceReleaseCreate>;
export const SourceReleaseUpdate = SourceReleaseCreate.omit({ siteId: true, siteReleaseId: true }).partial();
@@ -259,3 +263,143 @@ export const ObservationUpdate = Observation.pick({
lastError: true,
}).partial();
export type ObservationUpdate = z.infer<typeof ObservationUpdate>;
// ── W2-B:站点快照、调度事件与站点工作台 ──
export const SiteSnapshotKind = z.enum(["periodic", "backfill", "live_search", "manual"]);
export type SiteSnapshotKind = z.infer<typeof SiteSnapshotKind>;
export const SiteSnapshotStatus = z.enum(["running", "ok", "partial", "failed"]);
export type SiteSnapshotStatus = z.infer<typeof SiteSnapshotStatus>;
/** 一次列表快照 / 实时补查 / 补漏的覆盖范围与结果(不把一次分页伪称全站快照,REQ §4.6) */
export const SiteSnapshot = EntityBase.extend({
siteId: Id,
kind: SiteSnapshotKind,
query: z.string().nullable(),
filters: JsonObject,
/** 实际抓取的页码(1 起) */
pages: z.array(z.number().int().positive()),
itemCount: NonNegInt,
newCount: NonNegInt,
status: SiteSnapshotStatus,
error: JsonObject.nullable(),
definitionVersion: z.string().nullable(),
startedAt: Timestamp,
finishedAt: Timestamp.nullable(),
});
export type SiteSnapshot = z.infer<typeof SiteSnapshot>;
/** 站点注册表条目(随产品发布的适配器定义)+ 是否已在本系统登记 */
export const SiteDefinitionView = z.object({
key: z.string(),
name: z.string(),
adapter: SiteAdapter,
baseUrl: z.string(),
/** 定义是否随产品启用(未实测站点为 false) */
shippedEnabled: z.boolean(),
definitionVersion: z.string(),
capabilities: z.object({
list: z.boolean(),
search: z.boolean(),
detail: z.boolean(),
download: z.boolean(),
rss: z.boolean(),
}),
defaultRateLimit: z.object({ minIntervalMs: NonNegInt, burst: z.number().int().positive() }),
authSupported: z.array(z.string()),
/** 已登记的 site 行 ID;未登记为 null */
siteId: Id.nullable(),
});
export type SiteDefinitionView = z.infer<typeof SiteDefinitionView>;
/** Feed 到期(调度器发出,outbox 路由到 ingestFeed) */
export const FeedDuePayload = z.object({ feedId: Id, reason: z.enum(["schedule", "manual"]) });
/** 新的 source_release 被观察到(或同一资源标题更新),outbox 路由到 resolveObservation */
export const ReleaseObservedPayload = z.object({
sourceReleaseId: Id,
observationId: Id.nullable(),
siteId: Id,
isNew: z.boolean(),
});
/** Feed 窗口与上次零重叠:疑似缺口,路由到有预算的列表补漏(snapshotSite backfill) */
export const FeedGapSuspectedPayload = z.object({
feedId: Id,
siteId: Id.nullable(),
previousFetchedAt: Timestamp.nullable(),
detectedAt: Timestamp,
});
/** 站点工作台列表查询(整理视图):快照优先,必要时实时补查 */
export const WorkbenchReleaseQuery = z.object({
q: z.string().trim().min(1).max(200).optional(),
category: z.string().max(100).optional(),
/** 只看某类促销(free/2xfree/50%…) */
promotion: z.string().max(20).optional(),
resolution: ResolutionStatus.optional(),
/** true = 强制实时补查;false = 只看本地快照;缺省 = 本地无结果或快照过期时补查 */
live: z.enum(["true", "false"]).optional(),
cursor: z.string().min(1).optional(),
limit: z.coerce.number().int().min(1).max(200).default(50),
});
export type WorkbenchReleaseQuery = z.infer<typeof WorkbenchReleaseQuery>;
/** 数据来源:快照 / Feed 观察 / 实时补查 */
export const ReleaseOrigin = z.enum(["snapshot", "feed", "live", "unknown"]);
export type ReleaseOrigin = z.infer<typeof ReleaseOrigin>;
/**
* 工作台条目:source_release + 来源标注 + 本地实体关联。
* 库存叠加(已有资产/Slot/下载状态)由前端按 mediaEntityId 二次查询 W2-C 端点,不在这里猜。
*/
export const WorkbenchRelease = SourceRelease.extend({
origin: ReleaseOrigin,
/** 本条数据最近一次被站点确认的时间(快照/Feed/实时) */
dataUpdatedAt: Timestamp,
observedByFeeds: NonNegInt,
entity: z.object({ id: Id, title: z.string(), year: z.number().int().nullable(), kind: z.string() }).nullable(),
});
export type WorkbenchRelease = z.infer<typeof WorkbenchRelease>;
export const WorkbenchCoverage = z.object({
siteEnabled: z.boolean(),
lastSnapshotAt: Timestamp.nullable(),
/** 快照过期(超过 2 个周期或从未快照) */
stale: z.boolean(),
fromSnapshot: z.boolean(),
live: z.boolean(),
liveError: z.string().nullable(),
/** 同一搜索正在进行时合并到已有请求 */
coalesced: z.boolean(),
});
export type WorkbenchCoverage = z.infer<typeof WorkbenchCoverage>;
export const WorkbenchReleasePage = z.object({
items: z.array(WorkbenchRelease),
nextCursor: z.string().nullable(),
coverage: WorkbenchCoverage,
});
export type WorkbenchReleasePage = z.infer<typeof WorkbenchReleasePage>;
export const WorkbenchReleaseDetail = z.object({
release: WorkbenchRelease,
observations: z.array(
z.object({
observationId: Id,
feedId: Id,
feedName: z.string(),
firstSeenAt: Timestamp,
lastSeenAt: Timestamp,
}),
),
/** 易变字段的复核时间:缓存命中不能替代下载前复核(REQ §4.6) */
volatile: z.object({
promotionObservedAt: Timestamp.nullable(),
promotionExpiresAt: Timestamp.nullable(),
peersObservedAt: Timestamp,
detailFetchedAt: Timestamp.nullable(),
recheckBeforeDownload: z.literal(true),
}),
/** 刷新详情失败时保留既有数据并给出原因 */
refreshError: z.string().nullable(),
});
export type WorkbenchReleaseDetail = z.infer<typeof WorkbenchReleaseDetail>;
+18
View File
@@ -79,6 +79,13 @@ const ROWS: Record<string, { row: Obj; dbOnly?: string[]; apiOnly?: string[] }>
Subscription: { row: C.Subscription },
Task: { row: C.Task },
TitleAlias: { row: C.TitleAlias },
// W2-B 来源/身份/元数据(0018–0020)
FieldOverride: { row: C.FieldOverride },
IdentityCandidate: { row: C.IdentityCandidate },
IdentityRedirect: { row: C.IdentityRedirect },
MetadataField: { row: C.MetadataField },
ResolutionAttempt: { row: C.ResolutionAttempt },
SiteSnapshot: { row: C.SiteSnapshot },
};
/** 引擎自有表(@umt/tasks 内部使用,不经 API 暴露),不要求 Row schema */
@@ -186,6 +193,17 @@ const ENUMS: Record<string, z.ZodEnum> = {
TaskStatus: C.TaskStatus,
TitleAliasKind: C.AliasKind,
TitleAliasSource: C.AliasSource,
// W2-B(0018–0020)
FieldOverrideSource: C.MetadataSource,
FieldOverrideActorType: C.ActorType,
IdentityCandidateStatus: C.IdentityCandidateStatus,
IdentityCandidateReason: C.IdentityCandidateReason,
IdentityRedirectActorType: C.ActorType,
MetadataFieldSource: C.MetadataSource,
ResolutionAttemptOutcome: C.ResolutionOutcome,
ResolutionAttemptActorType: C.ActorType,
SiteSnapshotKind: C.SiteSnapshotKind,
SiteSnapshotStatus: C.SiteSnapshotStatus,
};
describe("contracts ⇔ db 表结构同步", () => {
+1
View File
@@ -91,6 +91,7 @@ const OVERRIDES: Record<string, (v: Record<string, unknown>) => Record<string, u
CatalogLookupInput: (v) => v,
IdentityResolveInput: (v) => v,
AssetLocationCreate: (v) => v,
ReleaseIdentityAssign: (v) => ({ ...v, external: undefined, candidateIndex: undefined }),
CacheSettings: (v) => ({ ...v, hardThresholdPercent: 90, resumePercent: 85, cleanupTargetPercent: 80 }),
GlobalSettings: () => C.DEFAULT_SETTINGS,
EngineStats: (v) => ({ ...v, byStatus: { pending: 1 } }),