diff --git a/apps/server/test/sources-identity/identity-cleanup.test.ts b/apps/server/test/sources-identity/identity-cleanup.test.ts new file mode 100644 index 00000000..69daeaa6 --- /dev/null +++ b/apps/server/test/sources-identity/identity-cleanup.test.ts @@ -0,0 +1,163 @@ +import { execFileSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { sql } from "kysely"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { createHarness, type Harness, PG_URL, pgAvailable } from "./harness.ts"; + +// FIX-6 scripts/identity-cleanup.ts:孤立单元只删资源解析建出的临时单元,TMDB / 元数据来源的季集账本单元不删。 + +const up = await pgAvailable(); +const ROOT = fileURLToPath(new URL("../../../../", import.meta.url)); +const SCRIPT = fileURLToPath(new URL("../../../../scripts/identity-cleanup.ts", import.meta.url)); + +describe.skipIf(!up)("identity-cleanup 孤立单元判定", () => { + let h: Harness; + let url: string; + let tmdbShow: string; + let localShow: string; + let mappedNoLayout: string; + + const run = (...args: string[]) => + execFileSync(Deno.execPath(), ["run", "-A", SCRIPT, ...args], { + cwd: ROOT, + env: { ...process.env, DATABASE_URL: url, NO_COLOR: "1" }, + encoding: "utf8", + }); + + const unit = async ( + entityId: string, + kind: "season" | "episode", + season: number, + episode: number | null, + extra: Record = {}, + ) => + ( + await h.db + .insertInto("content_unit") + .values({ + media_entity_id: entityId, + kind, + season_number: season, + episode_number: episode, + status: "provisional", + ...extra, + } as never) + .returning("id") + .executeTakeFirstOrThrow() + ).id; + + const entity = async (title: string, tmdbId: string | null, seasonCount: number | null) => { + const id = ( + await h.db + .insertInto("media_entity") + .values({ + kind: "series", + title, + status: "active", + metadata: JSON.stringify( + seasonCount === null ? {} : { seasonCount: { value: seasonCount, source: "tmdb", locked: false } }, + ), + } as never) + .returning("id") + .executeTakeFirstOrThrow() + ).id; + if (tmdbId) + await h.db + .insertInto("external_identity") + .values({ + media_entity_id: id, + provider: "tmdb", + entity_type: "tv", + external_id: tmdbId, + status: "accepted", + } as never) + .execute(); + return id; + }; + + const labels = async (entityId: string) => + ( + await h.db + .selectFrom("content_unit") + .select(["kind", "season_number", "episode_number"]) + .where("media_entity_id", "=", entityId) + .execute() + ) + .map((u) => (u.kind === "season" ? `S${u.season_number}` : `S${u.season_number}E${u.episode_number}`)) + .sort(); + + beforeAll(async () => { + h = await createHarness("idcleanup"); + const schema = (await sql<{ s: string }>`select current_schema() as s`.execute(h.db)).rows[0]!.s; + const u = new URL(PG_URL); + u.searchParams.set("options", `-c search_path=${schema},public`); + url = u.toString(); + + // TMDB 映射的剧(季数 1):S01 带 TMDB 总集数 3,集单元都还是 provisional、没有字段来源(旧版本建的账本单元) + tmdbShow = await entity("人生切割术", "95396", 1); + const s1 = await unit(tmdbShow, "season", 1, null, { episode_total: 3, episode_total_source: "tmdb" }); + for (const e of [1, 2, 3]) await unit(tmdbShow, "episode", 1, e, { parent_unit_id: s1 }); + // 带 TMDB 字段来源的集 + const e4 = await unit(tmdbShow, "episode", 1, 4, { parent_unit_id: s1, status: "confirmed" }); + await h.db + .insertInto("metadata_field") + .values({ content_unit_id: e4, field: "name", value: JSON.stringify("第 4 集"), source: "tmdb" } as never) + .execute(); + // 特别篇季(季号 0)在外部季集结构内 + await unit(tmdbShow, "season", 0, null); + // 错挂资源建出的:超出 TMDB 季数 / 超出该季总集数 + const s5 = await unit(tmdbShow, "season", 5, null); + await unit(tmdbShow, "episode", 5, 1, { parent_unit_id: s5 }); + await unit(tmdbShow, "episode", 1, 9, { parent_unit_id: s1 }); + + // 没有外部映射的作品:临时孤立单元删除;带元数据标题的保留(连同季) + localShow = await entity("东北有个周东北", null, null); + const ls1 = await unit(localShow, "season", 1, null); + await unit(localShow, "episode", 1, 1, { parent_unit_id: ls1 }); + await unit(localShow, "episode", 1, 2, { parent_unit_id: ls1, title: "手工命名" }); + const ls2 = await unit(localShow, "season", 2, null); + await unit(localShow, "episode", 2, 1, { parent_unit_id: ls2 }); + + // 有外部映射但季数未知:整部作品都算在外部结构内,不删 + mappedNoLayout = await entity("沙丘:预言", "90228", null); + const m0 = await unit(mappedNoLayout, "season", 0, null); + for (let e = 1; e <= 3; e++) await unit(mappedNoLayout, "episode", 0, e, { parent_unit_id: m0 }); + }); + afterAll(async () => { + await h?.close(); + }); + + it("--dry-run 按类别列出删除 / 保留,不改库", async () => { + const before = await labels(tmdbShow); + const out = run("--dry-run"); + expect(out).toContain("DRY-RUN"); + expect(out).toContain("## 孤立的临时内容单元:删除 6 条"); + expect(out).toContain("删除·作品没有外部映射:3 条\n - 《东北有个周东北》:S02 S01E01 S02E01"); + expect(out).toContain("删除·超出外部季集结构:3 条\n - 《人生切割术》:S05 S01E09 S05E01"); + expect(out).toContain("保留·在作品的外部季集结构内"); + expect(out).toContain("保留·TMDB / 元数据 / 人工来源"); + expect(await labels(tmdbShow)).toEqual(before); + expect(() => run("--dry-run", "--apply")).toThrow(); + }); + + it("--apply:TMDB 季集账本单元不删,临时孤立单元删除", async () => { + run("--apply"); + expect(await labels(tmdbShow)).toEqual(["S0", "S1", "S1E1", "S1E2", "S1E3", "S1E4"]); + expect(await labels(localShow)).toEqual(["S1", "S1E2"]); + expect(await labels(mappedNoLayout)).toEqual(["S0", "S0E1", "S0E2", "S0E3"]); + const audit = await h.db + .selectFrom("audit_event") + .select("payload") + .where("action", "=", "identity.cleanup") + .executeTakeFirstOrThrow(); + const orphans = (audit.payload as { orphanUnits: { reason: string }[] }).orphanUnits; + expect(orphans.map((o) => o.reason).sort()).toEqual([ + "no_external", + "no_external", + "no_external", + "outside_external", + "outside_external", + "outside_external", + ]); + }); +}); diff --git a/docs/runbook/identity.md b/docs/runbook/identity.md index f2043081..235769db 100644 --- a/docs/runbook/identity.md +++ b/docs/runbook/identity.md @@ -106,7 +106,7 @@ DATABASE_URL=postgres://umt:<密码>@$PGIP:5432/umt deno run -A scripts/identity ## 冲突别名与临时内容单元清理(FIX-5,`scripts/identity-cleanup.ts`) 守卫上线前登记的错误别名、重复 / 孤立的临时内容单元用这个脚本清理。缺省只读,`--apply` 在一个事务里执行并写审计 -`identity.cleanup`;`--keep-orphans` 不删孤立单元;`--entity ` 只处理一部作品。连接方式同上。 +`identity.cleanup`;`--dry-run` 显式只读;`--keep-orphans` 不删孤立单元;`--entity ` 只处理一部作品。连接方式同上。 ```sh DATABASE_URL=postgres://umt:<密码>@$PGIP:5432/umt deno run -A scripts/identity-cleanup.ts # 看清单 @@ -118,8 +118,19 @@ DATABASE_URL=postgres://umt:<密码>@$PGIP:5432/umt deno run -A scripts/identity 本作品下带这个名字的资源都解析出年份且都与作品年份差超过 1 年 → 删。 - 重复的临时内容单元:同作品同类型同季同集号(未退役)有多条且有临时的,保留被引用最多的(同数时 confirmed 优先、再取最早的), 其余的引用改指向保留的那条(撞唯一约束的元数据行直接删),子单元改挂,然后删除。 -- 孤立的临时内容单元:`provisional`、没有资源 / 资产 / 订阅 / 下载 / 整理等任何引用、没有 TMDB 来源字段(不是 TMDB 已知的集)、 - 没有保留的子单元 → 删除(字段来源 / 覆盖 / 单元别名一并删)。TMDB 已知的集即使没有资源也保留。 +- 孤立的临时内容单元(FIX-6 收紧):只删**资源解析建出、外部元数据不认识**的临时单元。同时满足才删:`provisional`;没有资源 / 资产 / + 订阅 / 下载 / 整理等任何引用;没有任何元数据痕迹(字段来源 / 字段覆盖 / 标题 / 播出日期 / 总集数来源 / metadata 非空); + 作品**没有**已接受的外部身份,或这个季 / 集**超出**作品的外部季集结构(季号 > TMDB 季数,或集号 > 该季已知总集数; + 季数未知时整部作品都不删,特别篇季 0 总在结构内);没有保留的子单元。字段来源 / 覆盖 / 单元别名随之删除。 + **TMDB / 元数据来源的季集账本单元永不删除**,即使没有资源、状态还是 provisional。 +- 清单按类别输出:删除(作品没有外部映射 / 超出外部季集结构)与保留(仍被引用 / TMDB·元数据·人工来源 / 在外部季集结构内 / + 有保留的子单元),`--dry-run` 显式只读(与 `--apply` 互斥)。先看「删除」两类里有没有正常作品,再 `--apply`; + 审计 `identity.cleanup` 的 `orphanUnits` 记每条删除的单元与原因。 +- 事故(FIX-6,2026-09-29):FIX-5 版本只看"无引用 + 没有 TMDB 字段来源",把旧版本建的、还没回写字段来源的 TMDB 季集账本单元 + (《人生切割术》S00–S03 与 20 集、《沙丘:预言》S00 与 17 集等)当孤立删除。恢复:对受影响作品跑 full 元数据刷新—— + 计划页执行「元数据语言回刷」(`POST /schedules/metadata.backfill_languages/run-now`,停用状态也可手动执行)或逐个 + `POST /media-entities/{id}/refresh?full=1`,正片季的季 / 集单元会按 TMDB 重建为 confirmed。**季 0(特别篇)不在 full 刷新范围内**, + 删掉的特别篇单元只能等资源 / 文件再次命中时懒建。 ## 排查 diff --git a/scripts/identity-cleanup.ts b/scripts/identity-cleanup.ts index 1c5657a7..4d3d8800 100644 --- a/scripts/identity-cleanup.ts +++ b/scripts/identity-cleanup.ts @@ -1,7 +1,7 @@ // 身份数据清理(FIX-5):删除冲突别名、合并重复的临时内容单元、删除孤立的临时内容单元。默认只读(dry-run),加 --apply 才改库。 // // 用法(DATABASE_URL 必须显式给出,脚本不带默认库): -// DATABASE_URL=postgres://umt:***@:5432/umt deno run -A scripts/identity-cleanup.ts # 看清单 +// DATABASE_URL=postgres://umt:***@:5432/umt deno run -A scripts/identity-cleanup.ts [--dry-run] # 看清单(按类别列删除 / 保留) // DATABASE_URL=… deno run -A scripts/identity-cleanup.ts --apply # 执行(单个事务) // DATABASE_URL=… deno run -A scripts/identity-cleanup.ts --keep-orphans # 不删孤立单元 // DATABASE_URL=… deno run -A scripts/identity-cleanup.ts --entity <作品 id> # 只看 / 只处理一部作品 @@ -14,8 +14,10 @@ // 2. 重复的临时内容单元:同一作品、同一类型、同季同集号有多条(未退役)且至少一条是临时(provisional)的, // 保留被引用最多的那条(同数时 confirmed 优先,再取最早建的),其余的引用改指向保留的那条后删除; // 改指向会撞唯一约束的行(同一字段来源 / 别名已在保留的单元上)直接删掉。 -// 3. 孤立的临时内容单元(--keep-orphans 跳过):provisional、没有任何资源 / 资产 / 订阅 / 下载 / 整理等引用、 -// 没有 TMDB 来源的字段(不是 TMDB 已知的季 / 集)、也没有子单元的,删除;字段来源 / 覆盖 / 单元别名随之删除。 +// 3. 孤立的临时内容单元(--keep-orphans 跳过):只删资源解析建出的临时单元——provisional、没有任何资源 / 资产 / 订阅 / +// 下载 / 整理等引用、没有任何元数据痕迹(字段来源 / 覆盖 / 标题 / 播出日期 / 总集数来源),且作品没有外部映射或 +// 这个季 / 集超出作品的外部季集结构;没有保留的子单元。TMDB / 元数据来源的单元永不删除(FIX-6)。 +// 字段来源 / 覆盖 / 单元别名随之删除。 // --apply 在一个事务里执行,并写一条审计事件(action = identity.cleanup)。 import { normalizeTitle, parseRelease } from "@umt/domain"; @@ -23,6 +25,10 @@ import pg from "pg"; const argv = [...Deno.args]; const APPLY = argv.includes("--apply"); +if (APPLY && argv.includes("--dry-run")) { + console.error("--apply 与 --dry-run 不能同时给出"); + Deno.exit(2); +} const KEEP_ORPHANS = argv.includes("--keep-orphans"); const entityArg = argv.indexOf("--entity"); const ONLY_ENTITY = entityArg >= 0 ? (argv[entityArg + 1] ?? null) : null; @@ -274,34 +280,121 @@ for (const m of merges) { } // ---------------------------------------------------------------- 3. 孤立的临时内容单元 +// 只删"资源解析建出来、外部元数据不认识"的临时单元。以下任一成立即保留(FIX-6:曾把 TMDB 季集账本单元当孤立删掉): +// - metadata:单元自身带元数据痕迹——有任何字段来源 / 字段覆盖,或标题、播出日期、总集数来源、metadata 非空 +// (资源解析建单元时这些都不写,有就说明来自 TMDB 等元数据或人工); +// - external:作品有已接受的外部身份,且这个季 / 集落在外部季集结构里(季号 0..季数,季数未知时整部作品都算; +// 集号不超过该季已知总集数,总集数未知时都算)。 +// 保留的单元连同其季单元都不删。 const dropping = new Set(merges.flatMap((m) => m.drop.map((d) => d.id))); -let orphans: UnitRow[] = []; +type OrphanReason = "no_external" | "outside_external"; +type KeepReason = "referenced" | "metadata" | "external" | "has_children"; +let orphans: (UnitRow & { reason: OrphanReason })[] = []; if (!KEEP_ORPHANS) { - const cand = await q( - `select u.id, u.media_entity_id, u.kind, u.season_number, u.episode_number, u.status, u.created_at + const cand = await q( + `select u.id, u.media_entity_id, u.kind, u.season_number, u.episode_number, u.status, u.created_at, + (u.title is not null or u.air_date is not null or u.episode_total_source is not null + or u.metadata <> '{}'::jsonb + or exists (select 1 from metadata_field f where f.content_unit_id = u.id) + or exists (select 1 from field_override o where o.content_unit_id = u.id)) as has_meta from content_unit u where u.status = 'provisional' ${unitFilter} - and not exists (select 1 from metadata_field f where f.content_unit_id = u.id and f.source = 'tmdb') order by u.media_entity_id, u.kind desc, u.season_number, u.episode_number`, ONLY_ENTITY ? [ONLY_ENTITY] : [], ); + const candEntities = [...new Set(cand.map((u) => u.media_entity_id))]; + // 外部季集结构:已接受外部身份的作品 → 季数(null = 未知)与各季已知总集数 + const layout = new Map }>(); + for (const r of await q<{ id: string; season_count: number | null }>( + `select e.id, + coalesce( + case when jsonb_typeof(e.metadata->'seasonCount'->'value') = 'number' + then (e.metadata->'seasonCount'->>'value')::int end, + case when jsonb_typeof(e.metadata->'seasonCount') = 'number' then (e.metadata->>'seasonCount')::int end, + (select case when jsonb_typeof(f.value) = 'number' then (f.value #>> '{}')::int end + from metadata_field f where f.media_entity_id = e.id and f.content_unit_id is null + and f.field = 'seasonCount' order by f.updated_at desc limit 1) + ) as season_count + from media_entity e + where e.id = any($1::uuid[]) + and exists (select 1 from external_identity x where x.media_entity_id = e.id and x.status = 'accepted')`, + [candEntities], + )) + layout.set(r.id, { seasonCount: r.season_count, totals: new Map() }); + for (const r of await q<{ media_entity_id: string; season_number: number; total: number }>( + `select media_entity_id, season_number, max(episode_total)::int as total from content_unit + where kind = 'season' and episode_total is not null and episode_total_source is not null + and media_entity_id = any($1::uuid[]) group by 1, 2`, + [[...layout.keys()]], + )) + layout.get(r.media_entity_id)?.totals.set(r.season_number, r.total); + const inExternalLayout = (u: UnitRow): boolean | null => { + const l = layout.get(u.media_entity_id); + if (!l) return null; // 作品没有外部映射 + const s = u.season_number; + if (s === null) return true; + if (l.seasonCount !== null && (s < 0 || s > l.seasonCount)) return false; + if (u.kind !== "episode" || u.episode_number === null) return true; + const total = l.totals.get(s); + return total === undefined || u.episode_number <= total; + }; + const refs = await refCounts(cand.map((u) => u.id)); - const unused = cand.filter((u) => (refs.get(u.id) ?? 0) === 0 && !dropping.has(u.id)); - const unusedIds = new Set(unused.map((u) => u.id)); + const keep = new Map(); + const drop = new Map(); + for (const u of cand) { + if (dropping.has(u.id)) continue; + if ((refs.get(u.id) ?? 0) > 0) keep.set(u.id, "referenced"); + else if (u.has_meta) keep.set(u.id, "metadata"); + else { + const inside = inExternalLayout(u); + if (inside === true) keep.set(u.id, "external"); + else drop.set(u.id, inside === null ? "no_external" : "outside_external"); + } + } // 季单元:子单元全都要删(或没有子单元)才删 const children = await q<{ parent_unit_id: string; id: string }>( `select parent_unit_id, id from content_unit where parent_unit_id = any($1::uuid[])`, - [unused.filter((u) => u.kind === "season").map((u) => u.id)], + [cand.filter((u) => u.kind === "season" && drop.has(u.id)).map((u) => u.id)], ); - orphans = unused.filter( - (u) => - u.kind !== "season" || - children.filter((c) => c.parent_unit_id === u.id).every((c) => unusedIds.has(c.id) || dropping.has(c.id)), - ); - console.log(`\n## 孤立的临时内容单元:${orphans.length} 条`); - const byEntity = new Map(); - for (const u of orphans) byEntity.set(u.media_entity_id, [...(byEntity.get(u.media_entity_id) ?? []), u]); - for (const [eid, list] of byEntity) console.log(` - 《${await entityTitle(eid)}》:${list.map(unitLabel).join(" ")}`); + for (const u of cand) + if ( + u.kind === "season" && + drop.has(u.id) && + !children.filter((c) => c.parent_unit_id === u.id).every((c) => drop.has(c.id) || dropping.has(c.id)) + ) { + drop.delete(u.id); + keep.set(u.id, "has_children"); + } + orphans = cand.filter((u) => drop.has(u.id)).map((u) => ({ ...u, reason: drop.get(u.id)! })); + + const ORPHAN_TEXT: Record = { + no_external: "作品没有外部映射", + outside_external: "超出外部季集结构", + }; + const KEEP_TEXT: Record = { + referenced: "仍被资源 / 资产 / 订阅等引用", + metadata: "TMDB / 元数据 / 人工来源", + external: "在作品的外部季集结构内", + has_children: "有保留的子单元", + }; + const printGroup = async (list: UnitRow[]) => { + const byEntity = new Map(); + for (const u of list) byEntity.set(u.media_entity_id, [...(byEntity.get(u.media_entity_id) ?? []), u]); + for (const [eid, l] of byEntity) console.log(` - 《${await entityTitle(eid)}》:${l.map(unitLabel).join(" ")}`); + }; + console.log(`\n## 孤立的临时内容单元:删除 ${orphans.length} 条`); + for (const r of Object.keys(ORPHAN_TEXT) as OrphanReason[]) { + const l = orphans.filter((u) => u.reason === r); + console.log(` - 删除·${ORPHAN_TEXT[r]}:${l.length} 条`); + await printGroup(l); + } + console.log(`\n## 临时内容单元保留(不删):${keep.size} 条`); + for (const r of Object.keys(KEEP_TEXT) as KeepReason[]) { + const l = cand.filter((u) => keep.get(u.id) === r); + console.log(` - 保留·${KEEP_TEXT[r]}:${l.length} 条`); + if (r !== "referenced") await printGroup(l); + } } else console.log("\n## 孤立的临时内容单元:跳过(--keep-orphans)"); // ---------------------------------------------------------------- 执行 @@ -360,7 +453,12 @@ if (APPLY) { JSON.stringify({ aliases: aliasDeletes.map((a) => ({ entityId: a.media_entity_id, alias: a.alias, reason: a.reason })), mergedUnits: merges.map((m) => ({ keep: m.keep.id, drop: m.drop.map((d) => d.id) })), - orphanUnits: orphans.length, + orphanUnits: orphans.map((u) => ({ + id: u.id, + entityId: u.media_entity_id, + unit: unitLabel(u), + reason: u.reason, + })), }), ], );