feat(server): W2-B sources, identity resolution, metadata and site workbench
- Sites / accounts / feeds: registry view (/site-definitions), site CRUD with
enable/disable (history stays queryable, marked site_disabled), accounts and
feed URLs stored as secrets (only secret_ref / redacted urlDisplay exposed),
feed enable/disable/reset/fetch; one shared rate limiter + adapter per site.
- Tasks: ingestFeed (FeedFetchState/FeedCursor on feed_source, observation
feed+guid upsert with digest-change updates, source_release site+id
aggregation across feeds, auth failure -> account invalid + decision, no
retry, gap -> feed.gap_suspected -> budgeted backfill snapshot),
resolveObservation (parseRelease -> local recall via title_alias pg_trgm +
accepted identities -> TMDB -> resolveIdentity; auto binds entity /
provisional units / identities / aliases, otherwise identity_candidate pool
with evidence version and bounded review backoff; manual lock respected),
refreshMetadata (metadata_field + field_override, user locks win, paths
untouched), snapshotSite (site_snapshot coverage records).
- Events: feed.due, feed.gap_suspected, release.observed (self-consumed),
release.resolved, decision.created/updated; scheduler in worker start().
- Identity API: entity detail with redirects, manual mapping correction with
identity_revision, merge with identity_redirect, pool list/assign/dismiss/
retry, content unit numbering lock, aliases, external identities.
- Decisions: list/summary (by entity and kind)/detail/resolve; identity
options executed directly.
- Workbench: /sites/{id}/releases (snapshot first, coalesced live search with
backoff, origin + freshness) and /sites/{id}/releases/{rid} (volatile field
recheck times, refresh keeps data on failure).
- Capabilities: catalog.lookup, release.search, release.inspect,
identity.resolve.
- Migrations 0018-0020: site.adapter api, source_release.resolution_locked,
site_snapshot, identity_candidate, resolution_attempt, identity_redirect,
metadata_field, field_override.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
829ca4f7ac
commit
a24bda952b
39 files changed
+6861
-15
No files matched your search
@@ -0,0 +1,51 @@
|
||||
import type { Kysely } from "kysely";
|
||||
import { dropTables, entityTable, execDdl, oneOf } from "../src/migration-helpers.ts";
|
||||
|
||||
// W2-B 来源补充(REQ §4.5、§4.6、§7.5):
|
||||
// - site.adapter 增加 api(MP 专用 API spider:Haidan/HDDolby/RousiPro/…,见 adapters/sites/spiders);
|
||||
// - source_release.resolution_locked:人工确认的身份关联,自动识别不得改写;
|
||||
// - site_snapshot:每次列表快照/实时补查/补漏的覆盖范围与新鲜度(不把一次分页伪称全站快照)。
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型
|
||||
export async function up(db: Kysely<any>): Promise<void> {
|
||||
await execDdl(
|
||||
db,
|
||||
`
|
||||
alter table site drop constraint site_adapter_check;
|
||||
alter table site add constraint site_adapter_check ${oneOf("adapter", ["nexusphp", "mteam_api", "api", "generic_rss"])};
|
||||
|
||||
alter table source_release add column resolution_locked boolean not null default false;
|
||||
|
||||
${entityTable(
|
||||
"site_snapshot",
|
||||
`
|
||||
site_id uuid not null references site (id) on delete cascade,
|
||||
kind text not null ${oneOf("kind", ["periodic", "backfill", "live_search", "manual"])},
|
||||
query text,
|
||||
filters jsonb not null default '{}'::jsonb,
|
||||
pages jsonb not null default '[]'::jsonb,
|
||||
item_count integer not null default 0,
|
||||
new_count integer not null default 0,
|
||||
status text not null default 'running' ${oneOf("status", ["running", "ok", "partial", "failed"])},
|
||||
error jsonb,
|
||||
definition_version text,
|
||||
started_at timestamptz not null default now(),
|
||||
finished_at timestamptz`,
|
||||
)}
|
||||
create index site_snapshot_site_idx on site_snapshot (site_id, started_at desc);
|
||||
`,
|
||||
);
|
||||
}
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 同上
|
||||
export async function down(db: Kysely<any>): Promise<void> {
|
||||
await execDdl(
|
||||
db,
|
||||
`
|
||||
${dropTables(["site_snapshot"])}
|
||||
alter table source_release drop column if exists resolution_locked;
|
||||
alter table site drop constraint site_adapter_check;
|
||||
alter table site add constraint site_adapter_check ${oneOf("adapter", ["nexusphp", "mteam_api", "generic_rss"])};
|
||||
`,
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
import type { Kysely } from "kysely";
|
||||
import { dropTables, entityTable, execDdl, oneOf } from "../src/migration-helpers.ts";
|
||||
|
||||
// W2-B 待识别池与身份治理(REQ §7.2、§7.5):
|
||||
// - identity_candidate:每条 source_release 至多一条待识别记录,候选只存 JSON,不逐个建为正式作品;
|
||||
// 失败保存原因、证据版本与下一次复查时间(有界退避),新证据到来时重试;
|
||||
// - resolution_attempt:每次识别尝试的结果与证据版本(只追加);
|
||||
// - identity_redirect:作品合并后的旧 ID 跳转历史(与 media_entity.merged_into_id 一致,可修订)。
|
||||
|
||||
const ACTOR = ["user", "agent", "system", "api_key"] as const;
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型
|
||||
export async function up(db: Kysely<any>): Promise<void> {
|
||||
await execDdl(
|
||||
db,
|
||||
`
|
||||
${entityTable(
|
||||
"identity_candidate",
|
||||
`
|
||||
source_release_id uuid not null references source_release (id) on delete cascade,
|
||||
status text not null default 'pending' ${oneOf("status", ["pending", "resolved", "dismissed", "archived"])},
|
||||
reason text not null ${oneOf("reason", ["ambiguous", "no_match", "proposed", "lookup_failed"])},
|
||||
query jsonb not null default '{}'::jsonb,
|
||||
candidates jsonb not null default '[]'::jsonb,
|
||||
evidence_version text not null,
|
||||
attempts integer not null default 0,
|
||||
last_attempt_at timestamptz,
|
||||
next_review_at timestamptz,
|
||||
decision_id uuid references decision (id) on delete set null,
|
||||
resolved_media_entity_id uuid references media_entity (id) on delete set null,
|
||||
resolved_by text,
|
||||
resolved_at timestamptz,
|
||||
constraint identity_candidate_release_key unique (source_release_id)`,
|
||||
)}
|
||||
create index identity_candidate_review_idx on identity_candidate (next_review_at) where status = 'pending';
|
||||
create index identity_candidate_status_idx on identity_candidate (status, updated_at desc);
|
||||
|
||||
${entityTable(
|
||||
"resolution_attempt",
|
||||
`
|
||||
source_release_id uuid not null references source_release (id) on delete cascade,
|
||||
observation_id uuid references observation (id) on delete set null,
|
||||
outcome text not null ${oneOf("outcome", ["auto", "proposed", "ambiguous", "no_match", "lookup_failed", "manual", "skipped_locked"])},
|
||||
reason text,
|
||||
evidence_version text not null,
|
||||
candidate_count integer not null default 0,
|
||||
media_entity_id uuid references media_entity (id) on delete set null,
|
||||
best jsonb,
|
||||
error jsonb,
|
||||
actor_type text not null default 'system' ${oneOf("actor_type", ACTOR)},
|
||||
actor_id text`,
|
||||
)}
|
||||
create index resolution_attempt_release_idx on resolution_attempt (source_release_id, created_at desc);
|
||||
|
||||
${entityTable(
|
||||
"identity_redirect",
|
||||
`
|
||||
from_entity_id uuid not null references media_entity (id) on delete cascade,
|
||||
to_entity_id uuid not null references media_entity (id) on delete cascade,
|
||||
reason text,
|
||||
actor_type text not null ${oneOf("actor_type", ACTOR)},
|
||||
actor_id text,
|
||||
constraint identity_redirect_from_key unique (from_entity_id),
|
||||
constraint identity_redirect_not_self check (from_entity_id <> to_entity_id)`,
|
||||
)}
|
||||
create index identity_redirect_to_idx on identity_redirect (to_entity_id);
|
||||
`,
|
||||
);
|
||||
}
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 同上
|
||||
export async function down(db: Kysely<any>): Promise<void> {
|
||||
await execDdl(db, dropTables(["identity_redirect", "resolution_attempt", "identity_candidate"]));
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
import type { Kysely } from "kysely";
|
||||
import { dropTables, entityTable, execDdl, oneOf } from "../src/migration-helpers.ts";
|
||||
|
||||
// W2-B 字段级元数据协调(REQ §7.3):
|
||||
// - metadata_field:各来源的候选字段值(来源/语言/时间/来源修订),刷新只写这里;
|
||||
// - field_override:用户(或 Agent 生成并确认)的字段覆盖与锁定,优先于任何自动来源;
|
||||
// 最终展示值投影到 media_entity.metadata / content_unit.metadata。路径绑定不受元数据刷新影响。
|
||||
|
||||
const SOURCE = ["tmdb", "user", "agent", "nfo", "site", "import"] as const;
|
||||
const target = "constraint {t}_target_check check (num_nonnulls(media_entity_id, content_unit_id) = 1)";
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型
|
||||
export async function up(db: Kysely<any>): Promise<void> {
|
||||
await execDdl(
|
||||
db,
|
||||
`
|
||||
${entityTable(
|
||||
"metadata_field",
|
||||
`
|
||||
media_entity_id uuid references media_entity (id) on delete cascade,
|
||||
content_unit_id uuid references content_unit (id) on delete cascade,
|
||||
field text not null,
|
||||
value jsonb,
|
||||
language text,
|
||||
source text not null ${oneOf("source", SOURCE)},
|
||||
source_revision text,
|
||||
fetched_at timestamptz not null default now(),
|
||||
${target.replace("{t}", "metadata_field")}`,
|
||||
)}
|
||||
create unique index metadata_field_key on metadata_field (media_entity_id, content_unit_id, field, source, language) nulls not distinct;
|
||||
|
||||
${entityTable(
|
||||
"field_override",
|
||||
`
|
||||
media_entity_id uuid references media_entity (id) on delete cascade,
|
||||
content_unit_id uuid references content_unit (id) on delete cascade,
|
||||
field text not null,
|
||||
value jsonb,
|
||||
locked boolean not null default true,
|
||||
source text not null default 'user' ${oneOf("source", SOURCE)},
|
||||
reason text,
|
||||
actor_type text not null ${oneOf("actor_type", ["user", "agent", "system", "api_key"])},
|
||||
actor_id text,
|
||||
${target.replace("{t}", "field_override")}`,
|
||||
)}
|
||||
create unique index field_override_key on field_override (media_entity_id, content_unit_id, field) nulls not distinct;
|
||||
`,
|
||||
);
|
||||
}
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 同上
|
||||
export async function down(db: Kysely<any>): Promise<void> {
|
||||
await execDdl(db, dropTables(["field_override", "metadata_field"]));
|
||||
}
|
||||
Reference in new issue
Block a user