Merge branch 'fix6'

apps/server/test/sources-identity/identity-cleanup.test.ts

@@ -0,0 +1,163 @@
+import { execFileSync } from "node:child_process";
+import { fileURLToPath } from "node:url";
+import { sql } from "kysely";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { createHarness, type Harness, PG_URL, pgAvailable } from "./harness.ts";
+
+// FIX-6 scripts/identity-cleanup.ts:孤立单元只删资源解析建出的临时单元,TMDB / 元数据来源的季集账本单元不删。
+
+const up = await pgAvailable();
+const ROOT = fileURLToPath(new URL("../../../../", import.meta.url));
+const SCRIPT = fileURLToPath(new URL("../../../../scripts/identity-cleanup.ts", import.meta.url));
+
+describe.skipIf(!up)("identity-cleanup 孤立单元判定", () => {
+  let h: Harness;
+  let url: string;
+  let tmdbShow: string;
+  let localShow: string;
+  let mappedNoLayout: string;
+
+  const run = (...args: string[]) =>
+    execFileSync(Deno.execPath(), ["run", "-A", SCRIPT, ...args], {
+      cwd: ROOT,
+      env: { ...process.env, DATABASE_URL: url, NO_COLOR: "1" },
+      encoding: "utf8",
+    });
+
+  const unit = async (
+    entityId: string,
+    kind: "season" | "episode",
+    season: number,
+    episode: number | null,
+    extra: Record<string, unknown> = {},
+  ) =>
+    (
+      await h.db
+        .insertInto("content_unit")
+        .values({
+          media_entity_id: entityId,
+          kind,
+          season_number: season,
+          episode_number: episode,
+          status: "provisional",
+          ...extra,
+        } as never)
+        .returning("id")
+        .executeTakeFirstOrThrow()
+    ).id;
+
+  const entity = async (title: string, tmdbId: string | null, seasonCount: number | null) => {
+    const id = (
+      await h.db
+        .insertInto("media_entity")
+        .values({
+          kind: "series",
+          title,
+          status: "active",
+          metadata: JSON.stringify(
+            seasonCount === null ? {} : { seasonCount: { value: seasonCount, source: "tmdb", locked: false } },
+          ),
+        } as never)
+        .returning("id")
+        .executeTakeFirstOrThrow()
+    ).id;
+    if (tmdbId)
+      await h.db
+        .insertInto("external_identity")
+        .values({
+          media_entity_id: id,
+          provider: "tmdb",
+          entity_type: "tv",
+          external_id: tmdbId,
+          status: "accepted",
+        } as never)
+        .execute();
+    return id;
+  };
+
+  const labels = async (entityId: string) =>
+    (
+      await h.db
+        .selectFrom("content_unit")
+        .select(["kind", "season_number", "episode_number"])
+        .where("media_entity_id", "=", entityId)
+        .execute()
+    )
+      .map((u) => (u.kind === "season" ? `S${u.season_number}` : `S${u.season_number}E${u.episode_number}`))
+      .sort();
+
+  beforeAll(async () => {
+    h = await createHarness("idcleanup");
+    const schema = (await sql<{ s: string }>`select current_schema() as s`.execute(h.db)).rows[0]!.s;
+    const u = new URL(PG_URL);
+    u.searchParams.set("options", `-c search_path=${schema},public`);
+    url = u.toString();
+
+    // TMDB 映射的剧(季数 1):S01 带 TMDB 总集数 3,集单元都还是 provisional、没有字段来源(旧版本建的账本单元)
+    tmdbShow = await entity("人生切割术", "95396", 1);
+    const s1 = await unit(tmdbShow, "season", 1, null, { episode_total: 3, episode_total_source: "tmdb" });
+    for (const e of [1, 2, 3]) await unit(tmdbShow, "episode", 1, e, { parent_unit_id: s1 });
+    // 带 TMDB 字段来源的集
+    const e4 = await unit(tmdbShow, "episode", 1, 4, { parent_unit_id: s1, status: "confirmed" });
+    await h.db
+      .insertInto("metadata_field")
+      .values({ content_unit_id: e4, field: "name", value: JSON.stringify("第 4 集"), source: "tmdb" } as never)
+      .execute();
+    // 特别篇季(季号 0)在外部季集结构内
+    await unit(tmdbShow, "season", 0, null);
+    // 错挂资源建出的:超出 TMDB 季数 / 超出该季总集数
+    const s5 = await unit(tmdbShow, "season", 5, null);
+    await unit(tmdbShow, "episode", 5, 1, { parent_unit_id: s5 });
+    await unit(tmdbShow, "episode", 1, 9, { parent_unit_id: s1 });
+
+    // 没有外部映射的作品:临时孤立单元删除;带元数据标题的保留(连同季)
+    localShow = await entity("东北有个周东北", null, null);
+    const ls1 = await unit(localShow, "season", 1, null);
+    await unit(localShow, "episode", 1, 1, { parent_unit_id: ls1 });
+    await unit(localShow, "episode", 1, 2, { parent_unit_id: ls1, title: "手工命名" });
+    const ls2 = await unit(localShow, "season", 2, null);
+    await unit(localShow, "episode", 2, 1, { parent_unit_id: ls2 });
+
+    // 有外部映射但季数未知:整部作品都算在外部结构内,不删
+    mappedNoLayout = await entity("沙丘：预言", "90228", null);
+    const m0 = await unit(mappedNoLayout, "season", 0, null);
+    for (let e = 1; e <= 3; e++) await unit(mappedNoLayout, "episode", 0, e, { parent_unit_id: m0 });
+  });
+  afterAll(async () => {
+    await h?.close();
+  });
+
+  it("--dry-run 按类别列出删除 / 保留,不改库", async () => {
+    const before = await labels(tmdbShow);
+    const out = run("--dry-run");
+    expect(out).toContain("DRY-RUN");
+    expect(out).toContain("## 孤立的临时内容单元:删除 6 条");
+    expect(out).toContain("删除·作品没有外部映射:3 条\n    - 《东北有个周东北》:S02 S01E01 S02E01");
+    expect(out).toContain("删除·超出外部季集结构:3 条\n    - 《人生切割术》:S05 S01E09 S05E01");
+    expect(out).toContain("保留·在作品的外部季集结构内");
+    expect(out).toContain("保留·TMDB / 元数据 / 人工来源");
+    expect(await labels(tmdbShow)).toEqual(before);
+    expect(() => run("--dry-run", "--apply")).toThrow();
+  });
+
+  it("--apply:TMDB 季集账本单元不删,临时孤立单元删除", async () => {
+    run("--apply");
+    expect(await labels(tmdbShow)).toEqual(["S0", "S1", "S1E1", "S1E2", "S1E3", "S1E4"]);
+    expect(await labels(localShow)).toEqual(["S1", "S1E2"]);
+    expect(await labels(mappedNoLayout)).toEqual(["S0", "S0E1", "S0E2", "S0E3"]);
+    const audit = await h.db
+      .selectFrom("audit_event")
+      .select("payload")
+      .where("action", "=", "identity.cleanup")
+      .executeTakeFirstOrThrow();
+    const orphans = (audit.payload as { orphanUnits: { reason: string }[] }).orphanUnits;
+    expect(orphans.map((o) => o.reason).sort()).toEqual([
+      "no_external",
+      "no_external",
+      "no_external",
+      "outside_external",
+      "outside_external",
+      "outside_external",
+    ]);
+  });
+});

docs/runbook/identity.md

@@ -106,7 +106,7 @@ DATABASE_URL=postgres://umt:<密码>@$PGIP:5432/umt deno run -A scripts/identity
 ## 冲突别名与临时内容单元清理(FIX-5,`scripts/identity-cleanup.ts`)
 
 守卫上线前登记的错误别名、重复 / 孤立的临时内容单元用这个脚本清理。缺省只读,`--apply` 在一个事务里执行并写审计
-`identity.cleanup`;`--keep-orphans` 不删孤立单元;`--entity <id>` 只处理一部作品。连接方式同上。
+`identity.cleanup`;`--dry-run` 显式只读;`--keep-orphans` 不删孤立单元;`--entity <id>` 只处理一部作品。连接方式同上。
 
 ```sh
 DATABASE_URL=postgres://umt:<密码>@$PGIP:5432/umt deno run -A scripts/identity-cleanup.ts            # 看清单
@@ -118,8 +118,19 @@ DATABASE_URL=postgres://umt:<密码>@$PGIP:5432/umt deno run -A scripts/identity
   本作品下带这个名字的资源都解析出年份且都与作品年份差超过 1 年 → 删。
 - 重复的临时内容单元:同作品同类型同季同集号(未退役)有多条且有临时的,保留被引用最多的(同数时 confirmed 优先、再取最早的),
   其余的引用改指向保留的那条(撞唯一约束的元数据行直接删),子单元改挂,然后删除。
-- 孤立的临时内容单元:`provisional`、没有资源 / 资产 / 订阅 / 下载 / 整理等任何引用、没有 TMDB 来源字段(不是 TMDB 已知的集)、
-  没有保留的子单元 → 删除(字段来源 / 覆盖 / 单元别名一并删)。TMDB 已知的集即使没有资源也保留。
+- 孤立的临时内容单元(FIX-6 收紧):只删**资源解析建出、外部元数据不认识**的临时单元。同时满足才删:`provisional`;没有资源 / 资产 /
+  订阅 / 下载 / 整理等任何引用;没有任何元数据痕迹(字段来源 / 字段覆盖 / 标题 / 播出日期 / 总集数来源 / metadata 非空);
+  作品**没有**已接受的外部身份,或这个季 / 集**超出**作品的外部季集结构(季号 > TMDB 季数,或集号 > 该季已知总集数;
+  季数未知时整部作品都不删,特别篇季 0 总在结构内);没有保留的子单元。字段来源 / 覆盖 / 单元别名随之删除。
+  **TMDB / 元数据来源的季集账本单元永不删除**,即使没有资源、状态还是 provisional。
+- 清单按类别输出:删除(作品没有外部映射 / 超出外部季集结构)与保留(仍被引用 / TMDB·元数据·人工来源 / 在外部季集结构内 /
+  有保留的子单元),`--dry-run` 显式只读(与 `--apply` 互斥)。先看「删除」两类里有没有正常作品,再 `--apply`;
+  审计 `identity.cleanup` 的 `orphanUnits` 记每条删除的单元与原因。
+- 事故(FIX-6,2026-09-29):FIX-5 版本只看"无引用 + 没有 TMDB 字段来源",把旧版本建的、还没回写字段来源的 TMDB 季集账本单元
+  (《人生切割术》S00–S03 与 20 集、《沙丘：预言》S00 与 17 集等)当孤立删除。恢复:对受影响作品跑 full 元数据刷新——
+  计划页执行「元数据语言回刷」(`POST /schedules/metadata.backfill_languages/run-now`,停用状态也可手动执行)或逐个
+  `POST /media-entities/{id}/refresh?full=1`,正片季的季 / 集单元会按 TMDB 重建为 confirmed。**季 0(特别篇)不在 full 刷新范围内**,
+  删掉的特别篇单元只能等资源 / 文件再次命中时懒建。
 
 ## 排查
 

scripts/identity-cleanup.ts

@@ -1,7 +1,7 @@
 // 身份数据清理(FIX-5):删除冲突别名、合并重复的临时内容单元、删除孤立的临时内容单元。默认只读(dry-run),加 --apply 才改库。
 //
 // 用法(DATABASE_URL 必须显式给出,脚本不带默认库):
-//   DATABASE_URL=postgres://umt:***@<host>:5432/umt deno run -A scripts/identity-cleanup.ts                 # 看清单
+//   DATABASE_URL=postgres://umt:***@<host>:5432/umt deno run -A scripts/identity-cleanup.ts [--dry-run]       # 看清单(按类别列删除 / 保留)
 //   DATABASE_URL=… deno run -A scripts/identity-cleanup.ts --apply                                          # 执行(单个事务)
 //   DATABASE_URL=… deno run -A scripts/identity-cleanup.ts --keep-orphans                                   # 不删孤立单元
 //   DATABASE_URL=… deno run -A scripts/identity-cleanup.ts --entity <作品 id>                              # 只看 / 只处理一部作品
@@ -14,8 +14,10 @@
 // 2. 重复的临时内容单元:同一作品、同一类型、同季同集号有多条(未退役)且至少一条是临时(provisional)的,
 //    保留被引用最多的那条(同数时 confirmed 优先,再取最早建的),其余的引用改指向保留的那条后删除;
 //    改指向会撞唯一约束的行(同一字段来源 / 别名已在保留的单元上)直接删掉。
-// 3. 孤立的临时内容单元(--keep-orphans 跳过):provisional、没有任何资源 / 资产 / 订阅 / 下载 / 整理等引用、
-//    没有 TMDB 来源的字段(不是 TMDB 已知的季 / 集)、也没有子单元的,删除;字段来源 / 覆盖 / 单元别名随之删除。
+// 3. 孤立的临时内容单元(--keep-orphans 跳过):只删资源解析建出的临时单元——provisional、没有任何资源 / 资产 / 订阅 /
+//    下载 / 整理等引用、没有任何元数据痕迹(字段来源 / 覆盖 / 标题 / 播出日期 / 总集数来源),且作品没有外部映射或
+//    这个季 / 集超出作品的外部季集结构;没有保留的子单元。TMDB / 元数据来源的单元永不删除(FIX-6)。
+//    字段来源 / 覆盖 / 单元别名随之删除。
 // --apply 在一个事务里执行,并写一条审计事件(action = identity.cleanup)。
 
 import { normalizeTitle, parseRelease } from "@umt/domain";
@@ -23,6 +25,10 @@ import pg from "pg";
 
 const argv = [...Deno.args];
 const APPLY = argv.includes("--apply");
+if (APPLY && argv.includes("--dry-run")) {
+  console.error("--apply 与 --dry-run 不能同时给出");
+  Deno.exit(2);
+}
 const KEEP_ORPHANS = argv.includes("--keep-orphans");
 const entityArg = argv.indexOf("--entity");
 const ONLY_ENTITY = entityArg >= 0 ? (argv[entityArg + 1] ?? null) : null;
@@ -274,34 +280,121 @@ for (const m of merges) {
 }
 
 // ---------------------------------------------------------------- 3. 孤立的临时内容单元
+// 只删"资源解析建出来、外部元数据不认识"的临时单元。以下任一成立即保留(FIX-6:曾把 TMDB 季集账本单元当孤立删掉):
+//   - metadata:单元自身带元数据痕迹——有任何字段来源 / 字段覆盖,或标题、播出日期、总集数来源、metadata 非空
+//     (资源解析建单元时这些都不写,有就说明来自 TMDB 等元数据或人工);
+//   - external:作品有已接受的外部身份,且这个季 / 集落在外部季集结构里(季号 0..季数,季数未知时整部作品都算;
+//     集号不超过该季已知总集数,总集数未知时都算)。
+// 保留的单元连同其季单元都不删。
 const dropping = new Set(merges.flatMap((m) => m.drop.map((d) => d.id)));
-let orphans: UnitRow[] = [];
+type OrphanReason = "no_external" | "outside_external";
+type KeepReason = "referenced" | "metadata" | "external" | "has_children";
+let orphans: (UnitRow & { reason: OrphanReason })[] = [];
 if (!KEEP_ORPHANS) {
-  const cand = await q<UnitRow>(
-    `select u.id, u.media_entity_id, u.kind, u.season_number, u.episode_number, u.status, u.created_at
+  const cand = await q<UnitRow & { has_meta: boolean }>(
+    `select u.id, u.media_entity_id, u.kind, u.season_number, u.episode_number, u.status, u.created_at,
+            (u.title is not null or u.air_date is not null or u.episode_total_source is not null
+             or u.metadata <> '{}'::jsonb
+             or exists (select 1 from metadata_field f where f.content_unit_id = u.id)
+             or exists (select 1 from field_override o where o.content_unit_id = u.id)) as has_meta
        from content_unit u
       where u.status = 'provisional' ${unitFilter}
-        and not exists (select 1 from metadata_field f where f.content_unit_id = u.id and f.source = 'tmdb')
       order by u.media_entity_id, u.kind desc, u.season_number, u.episode_number`,
     ONLY_ENTITY ? [ONLY_ENTITY] : [],
   );
+  const candEntities = [...new Set(cand.map((u) => u.media_entity_id))];
+  // 外部季集结构:已接受外部身份的作品 → 季数(null = 未知)与各季已知总集数
+  const layout = new Map<string, { seasonCount: number | null; totals: Map<number, number> }>();
+  for (const r of await q<{ id: string; season_count: number | null }>(
+    `select e.id,
+            coalesce(
+              case when jsonb_typeof(e.metadata->'seasonCount'->'value') = 'number'
+                   then (e.metadata->'seasonCount'->>'value')::int end,
+              case when jsonb_typeof(e.metadata->'seasonCount') = 'number' then (e.metadata->>'seasonCount')::int end,
+              (select case when jsonb_typeof(f.value) = 'number' then (f.value #>> '{}')::int end
+                 from metadata_field f where f.media_entity_id = e.id and f.content_unit_id is null
+                  and f.field = 'seasonCount' order by f.updated_at desc limit 1)
+            ) as season_count
+       from media_entity e
+      where e.id = any($1::uuid[])
+        and exists (select 1 from external_identity x where x.media_entity_id = e.id and x.status = 'accepted')`,
+    [candEntities],
+  ))
+    layout.set(r.id, { seasonCount: r.season_count, totals: new Map() });
+  for (const r of await q<{ media_entity_id: string; season_number: number; total: number }>(
+    `select media_entity_id, season_number, max(episode_total)::int as total from content_unit
+      where kind = 'season' and episode_total is not null and episode_total_source is not null
+        and media_entity_id = any($1::uuid[]) group by 1, 2`,
+    [[...layout.keys()]],
+  ))
+    layout.get(r.media_entity_id)?.totals.set(r.season_number, r.total);
+  const inExternalLayout = (u: UnitRow): boolean | null => {
+    const l = layout.get(u.media_entity_id);
+    if (!l) return null; // 作品没有外部映射
+    const s = u.season_number;
+    if (s === null) return true;
+    if (l.seasonCount !== null && (s < 0 || s > l.seasonCount)) return false;
+    if (u.kind !== "episode" || u.episode_number === null) return true;
+    const total = l.totals.get(s);
+    return total === undefined || u.episode_number <= total;
+  };
+
   const refs = await refCounts(cand.map((u) => u.id));
-  const unused = cand.filter((u) => (refs.get(u.id) ?? 0) === 0 && !dropping.has(u.id));
-  const unusedIds = new Set(unused.map((u) => u.id));
+  const keep = new Map<string, KeepReason>();
+  const drop = new Map<string, OrphanReason>();
+  for (const u of cand) {
+    if (dropping.has(u.id)) continue;
+    if ((refs.get(u.id) ?? 0) > 0) keep.set(u.id, "referenced");
+    else if (u.has_meta) keep.set(u.id, "metadata");
+    else {
+      const inside = inExternalLayout(u);
+      if (inside === true) keep.set(u.id, "external");
+      else drop.set(u.id, inside === null ? "no_external" : "outside_external");
+    }
+  }
   // 季单元:子单元全都要删(或没有子单元)才删
   const children = await q<{ parent_unit_id: string; id: string }>(
     `select parent_unit_id, id from content_unit where parent_unit_id = any($1::uuid[])`,
-    [unused.filter((u) => u.kind === "season").map((u) => u.id)],
+    [cand.filter((u) => u.kind === "season" && drop.has(u.id)).map((u) => u.id)],
   );
-  orphans = unused.filter(
-    (u) =>
-      u.kind !== "season" ||
-      children.filter((c) => c.parent_unit_id === u.id).every((c) => unusedIds.has(c.id) || dropping.has(c.id)),
-  );
-  console.log(`\n## 孤立的临时内容单元:${orphans.length} 条`);
+  for (const u of cand)
+    if (
+      u.kind === "season" &&
+      drop.has(u.id) &&
+      !children.filter((c) => c.parent_unit_id === u.id).every((c) => drop.has(c.id) || dropping.has(c.id))
+    ) {
+      drop.delete(u.id);
+      keep.set(u.id, "has_children");
+    }
+  orphans = cand.filter((u) => drop.has(u.id)).map((u) => ({ ...u, reason: drop.get(u.id)! }));
+
+  const ORPHAN_TEXT: Record<OrphanReason, string> = {
+    no_external: "作品没有外部映射",
+    outside_external: "超出外部季集结构",
+  };
+  const KEEP_TEXT: Record<KeepReason, string> = {
+    referenced: "仍被资源 / 资产 / 订阅等引用",
+    metadata: "TMDB / 元数据 / 人工来源",
+    external: "在作品的外部季集结构内",
+    has_children: "有保留的子单元",
+  };
+  const printGroup = async (list: UnitRow[]) => {
     const byEntity = new Map<string, UnitRow[]>();
-  for (const u of orphans) byEntity.set(u.media_entity_id, [...(byEntity.get(u.media_entity_id) ?? []), u]);
-  for (const [eid, list] of byEntity) console.log(`  - 《${await entityTitle(eid)}》:${list.map(unitLabel).join(" ")}`);
+    for (const u of list) byEntity.set(u.media_entity_id, [...(byEntity.get(u.media_entity_id) ?? []), u]);
+    for (const [eid, l] of byEntity) console.log(`    - 《${await entityTitle(eid)}》:${l.map(unitLabel).join(" ")}`);
+  };
+  console.log(`\n## 孤立的临时内容单元:删除 ${orphans.length} 条`);
+  for (const r of Object.keys(ORPHAN_TEXT) as OrphanReason[]) {
+    const l = orphans.filter((u) => u.reason === r);
+    console.log(`  - 删除·${ORPHAN_TEXT[r]}:${l.length} 条`);
+    await printGroup(l);
+  }
+  console.log(`\n## 临时内容单元保留(不删):${keep.size} 条`);
+  for (const r of Object.keys(KEEP_TEXT) as KeepReason[]) {
+    const l = cand.filter((u) => keep.get(u.id) === r);
+    console.log(`  - 保留·${KEEP_TEXT[r]}:${l.length} 条`);
+    if (r !== "referenced") await printGroup(l);
+  }
 } else console.log("\n## 孤立的临时内容单元:跳过(--keep-orphans)");
 
 // ---------------------------------------------------------------- 执行
@@ -360,7 +453,12 @@ if (APPLY) {
         JSON.stringify({
           aliases: aliasDeletes.map((a) => ({ entityId: a.media_entity_id, alias: a.alias, reason: a.reason })),
           mergedUnits: merges.map((m) => ({ keep: m.keep.id, drop: m.drop.map((d) => d.id) })),
-          orphanUnits: orphans.length,
+          orphanUnits: orphans.map((u) => ({
+            id: u.id,
+            entityId: u.media_entity_id,
+            unit: unitLabel(u),
+            reason: u.reason,
+          })),
         }),
       ],
     );