feat(server): W2-B sources, identity resolution, metadata and site workbench
- Sites / accounts / feeds: registry view (/site-definitions), site CRUD with
enable/disable (history stays queryable, marked site_disabled), accounts and
feed URLs stored as secrets (only secret_ref / redacted urlDisplay exposed),
feed enable/disable/reset/fetch; one shared rate limiter + adapter per site.
- Tasks: ingestFeed (FeedFetchState/FeedCursor on feed_source, observation
feed+guid upsert with digest-change updates, source_release site+id
aggregation across feeds, auth failure -> account invalid + decision, no
retry, gap -> feed.gap_suspected -> budgeted backfill snapshot),
resolveObservation (parseRelease -> local recall via title_alias pg_trgm +
accepted identities -> TMDB -> resolveIdentity; auto binds entity /
provisional units / identities / aliases, otherwise identity_candidate pool
with evidence version and bounded review backoff; manual lock respected),
refreshMetadata (metadata_field + field_override, user locks win, paths
untouched), snapshotSite (site_snapshot coverage records).
- Events: feed.due, feed.gap_suspected, release.observed (self-consumed),
release.resolved, decision.created/updated; scheduler in worker start().
- Identity API: entity detail with redirects, manual mapping correction with
identity_revision, merge with identity_redirect, pool list/assign/dismiss/
retry, content unit numbering lock, aliases, external identities.
- Decisions: list/summary (by entity and kind)/detail/resolve; identity
options executed directly.
- Workbench: /sites/{id}/releases (snapshot first, coalesced live search with
backoff, origin + freshness) and /sites/{id}/releases/{rid} (volatile field
recheck times, refresh keeps data on failure).
- Capabilities: catalog.lookup, release.search, release.inspect,
identity.resolve.
- Migrations 0018-0020: site.adapter api, source_release.resolution_locked,
site_snapshot, identity_candidate, resolution_attempt, identity_redirect,
metadata_field, field_override.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
829ca4f7ac
commit
a24bda952b
39 files changed
+6861
-15
No files matched your search
@@ -4,7 +4,8 @@ import { EntityBase, Id, JsonObject, NonNegInt, SERVER_MANAGED, Timestamp } from
|
||||
// 来源域:站点、站点账号、Feed、站点资源、观察(REQ §4)
|
||||
// 含 passkey 的 Feed URL 只以 urlSecretRef 保存,对外只给脱敏的 urlDisplay。
|
||||
|
||||
export const SiteAdapter = z.enum(["nexusphp", "mteam_api", "generic_rss"]);
|
||||
/** api = MP 专用 API spider(Haidan/HDDolby/RousiPro/SunnyPT/TNode/TorrentLeech/Yema) */
|
||||
export const SiteAdapter = z.enum(["nexusphp", "mteam_api", "api", "generic_rss"]);
|
||||
export type SiteAdapter = z.infer<typeof SiteAdapter>;
|
||||
export const SiteHealth = z.enum(["unknown", "healthy", "degraded", "failing", "disabled"]);
|
||||
export type SiteHealth = z.infer<typeof SiteHealth>;
|
||||
@@ -180,6 +181,8 @@ export const SourceRelease = EntityBase.extend({
|
||||
resolutionEvidence: JsonObject,
|
||||
availability: ReleaseAvailability,
|
||||
raw: JsonObject,
|
||||
/** 人工确认的身份关联:自动识别/复查不得改写 mediaEntityId/contentUnitId */
|
||||
resolutionLocked: z.boolean(),
|
||||
});
|
||||
export type SourceRelease = z.infer<typeof SourceRelease>;
|
||||
/** 由适配器/任务写入(按 siteId+siteReleaseId upsert) */
|
||||
@@ -209,6 +212,7 @@ export const SourceReleaseCreate = SourceRelease.omit({
|
||||
resolutionEvidence: true,
|
||||
availability: true,
|
||||
raw: true,
|
||||
resolutionLocked: true,
|
||||
});
|
||||
export type SourceReleaseCreate = z.infer<typeof SourceReleaseCreate>;
|
||||
export const SourceReleaseUpdate = SourceReleaseCreate.omit({ siteId: true, siteReleaseId: true }).partial();
|
||||
@@ -259,3 +263,143 @@ export const ObservationUpdate = Observation.pick({
|
||||
lastError: true,
|
||||
}).partial();
|
||||
export type ObservationUpdate = z.infer<typeof ObservationUpdate>;
|
||||
|
||||
// ── W2-B:站点快照、调度事件与站点工作台 ──
|
||||
|
||||
export const SiteSnapshotKind = z.enum(["periodic", "backfill", "live_search", "manual"]);
|
||||
export type SiteSnapshotKind = z.infer<typeof SiteSnapshotKind>;
|
||||
export const SiteSnapshotStatus = z.enum(["running", "ok", "partial", "failed"]);
|
||||
export type SiteSnapshotStatus = z.infer<typeof SiteSnapshotStatus>;
|
||||
|
||||
/** 一次列表快照 / 实时补查 / 补漏的覆盖范围与结果(不把一次分页伪称全站快照,REQ §4.6) */
|
||||
export const SiteSnapshot = EntityBase.extend({
|
||||
siteId: Id,
|
||||
kind: SiteSnapshotKind,
|
||||
query: z.string().nullable(),
|
||||
filters: JsonObject,
|
||||
/** 实际抓取的页码(1 起) */
|
||||
pages: z.array(z.number().int().positive()),
|
||||
itemCount: NonNegInt,
|
||||
newCount: NonNegInt,
|
||||
status: SiteSnapshotStatus,
|
||||
error: JsonObject.nullable(),
|
||||
definitionVersion: z.string().nullable(),
|
||||
startedAt: Timestamp,
|
||||
finishedAt: Timestamp.nullable(),
|
||||
});
|
||||
export type SiteSnapshot = z.infer<typeof SiteSnapshot>;
|
||||
|
||||
/** 站点注册表条目(随产品发布的适配器定义)+ 是否已在本系统登记 */
|
||||
export const SiteDefinitionView = z.object({
|
||||
key: z.string(),
|
||||
name: z.string(),
|
||||
adapter: SiteAdapter,
|
||||
baseUrl: z.string(),
|
||||
/** 定义是否随产品启用(未实测站点为 false) */
|
||||
shippedEnabled: z.boolean(),
|
||||
definitionVersion: z.string(),
|
||||
capabilities: z.object({
|
||||
list: z.boolean(),
|
||||
search: z.boolean(),
|
||||
detail: z.boolean(),
|
||||
download: z.boolean(),
|
||||
rss: z.boolean(),
|
||||
}),
|
||||
defaultRateLimit: z.object({ minIntervalMs: NonNegInt, burst: z.number().int().positive() }),
|
||||
authSupported: z.array(z.string()),
|
||||
/** 已登记的 site 行 ID;未登记为 null */
|
||||
siteId: Id.nullable(),
|
||||
});
|
||||
export type SiteDefinitionView = z.infer<typeof SiteDefinitionView>;
|
||||
|
||||
/** Feed 到期(调度器发出,outbox 路由到 ingestFeed) */
|
||||
export const FeedDuePayload = z.object({ feedId: Id, reason: z.enum(["schedule", "manual"]) });
|
||||
/** 新的 source_release 被观察到(或同一资源标题更新),outbox 路由到 resolveObservation */
|
||||
export const ReleaseObservedPayload = z.object({
|
||||
sourceReleaseId: Id,
|
||||
observationId: Id.nullable(),
|
||||
siteId: Id,
|
||||
isNew: z.boolean(),
|
||||
});
|
||||
/** Feed 窗口与上次零重叠:疑似缺口,路由到有预算的列表补漏(snapshotSite backfill) */
|
||||
export const FeedGapSuspectedPayload = z.object({
|
||||
feedId: Id,
|
||||
siteId: Id.nullable(),
|
||||
previousFetchedAt: Timestamp.nullable(),
|
||||
detectedAt: Timestamp,
|
||||
});
|
||||
|
||||
/** 站点工作台列表查询(整理视图):快照优先,必要时实时补查 */
|
||||
export const WorkbenchReleaseQuery = z.object({
|
||||
q: z.string().trim().min(1).max(200).optional(),
|
||||
category: z.string().max(100).optional(),
|
||||
/** 只看某类促销(free/2xfree/50%…) */
|
||||
promotion: z.string().max(20).optional(),
|
||||
resolution: ResolutionStatus.optional(),
|
||||
/** true = 强制实时补查;false = 只看本地快照;缺省 = 本地无结果或快照过期时补查 */
|
||||
live: z.enum(["true", "false"]).optional(),
|
||||
cursor: z.string().min(1).optional(),
|
||||
limit: z.coerce.number().int().min(1).max(200).default(50),
|
||||
});
|
||||
export type WorkbenchReleaseQuery = z.infer<typeof WorkbenchReleaseQuery>;
|
||||
|
||||
/** 数据来源:快照 / Feed 观察 / 实时补查 */
|
||||
export const ReleaseOrigin = z.enum(["snapshot", "feed", "live", "unknown"]);
|
||||
export type ReleaseOrigin = z.infer<typeof ReleaseOrigin>;
|
||||
|
||||
/**
|
||||
* 工作台条目:source_release + 来源标注 + 本地实体关联。
|
||||
* 库存叠加(已有资产/Slot/下载状态)由前端按 mediaEntityId 二次查询 W2-C 端点,不在这里猜。
|
||||
*/
|
||||
export const WorkbenchRelease = SourceRelease.extend({
|
||||
origin: ReleaseOrigin,
|
||||
/** 本条数据最近一次被站点确认的时间(快照/Feed/实时) */
|
||||
dataUpdatedAt: Timestamp,
|
||||
observedByFeeds: NonNegInt,
|
||||
entity: z.object({ id: Id, title: z.string(), year: z.number().int().nullable(), kind: z.string() }).nullable(),
|
||||
});
|
||||
export type WorkbenchRelease = z.infer<typeof WorkbenchRelease>;
|
||||
|
||||
export const WorkbenchCoverage = z.object({
|
||||
siteEnabled: z.boolean(),
|
||||
lastSnapshotAt: Timestamp.nullable(),
|
||||
/** 快照过期(超过 2 个周期或从未快照) */
|
||||
stale: z.boolean(),
|
||||
fromSnapshot: z.boolean(),
|
||||
live: z.boolean(),
|
||||
liveError: z.string().nullable(),
|
||||
/** 同一搜索正在进行时合并到已有请求 */
|
||||
coalesced: z.boolean(),
|
||||
});
|
||||
export type WorkbenchCoverage = z.infer<typeof WorkbenchCoverage>;
|
||||
|
||||
export const WorkbenchReleasePage = z.object({
|
||||
items: z.array(WorkbenchRelease),
|
||||
nextCursor: z.string().nullable(),
|
||||
coverage: WorkbenchCoverage,
|
||||
});
|
||||
export type WorkbenchReleasePage = z.infer<typeof WorkbenchReleasePage>;
|
||||
|
||||
export const WorkbenchReleaseDetail = z.object({
|
||||
release: WorkbenchRelease,
|
||||
observations: z.array(
|
||||
z.object({
|
||||
observationId: Id,
|
||||
feedId: Id,
|
||||
feedName: z.string(),
|
||||
firstSeenAt: Timestamp,
|
||||
lastSeenAt: Timestamp,
|
||||
}),
|
||||
),
|
||||
/** 易变字段的复核时间:缓存命中不能替代下载前复核(REQ §4.6) */
|
||||
volatile: z.object({
|
||||
promotionObservedAt: Timestamp.nullable(),
|
||||
promotionExpiresAt: Timestamp.nullable(),
|
||||
peersObservedAt: Timestamp,
|
||||
detailFetchedAt: Timestamp.nullable(),
|
||||
recheckBeforeDownload: z.literal(true),
|
||||
}),
|
||||
/** 刷新详情失败时保留既有数据并给出原因 */
|
||||
refreshError: z.string().nullable(),
|
||||
});
|
||||
export type WorkbenchReleaseDetail = z.infer<typeof WorkbenchReleaseDetail>;
|
||||
Reference in new issue
Block a user