feat(identity): content unit data hygiene — specials, provisional promotion, duplicate merge (DH-1)

- metadata.includeSpecials (default true): full refresh and the language backfill fetch TMDB season 0 (specials) when present, like MP; season 0 stays out of the episode-total ledger - refreshMetadata promotes provisional season/episode units that match the TMDB season/episode layout to confirmed (source=tmdb), keeping ids and all references; non-matching units stay provisional - content_unit.source (migration 0053): release / file / tmdb / user - duplicate units (same entity/kind/season/episode/content type/segment) are merged as a refresh step and via `scripts/identity-cleanup.ts --merge-duplicates` (dry-run by default); keeper = most referenced, then confirmed, then earliest; references auto-discovered via FKs and repointed - ensureUnits takes a per-unit advisory xact lock to stop concurrent lazy creation from producing duplicates - runbooks updated; user DB read-only dry-run: 征途 S01E09/E10 4 units each -> 1 each (6 removed), not applied Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-09-29 20:49:56 +08:00
1 parent 2847512501
commit 044e718ec6
@@ -1,27 +1,36 @@
-// 身份数据清理(FIX-5):删除冲突别名、合并重复的临时内容单元、删除孤立的临时内容单元。默认只读(dry-run),加 --apply 才改库。
+// 身份数据清理(FIX-5):删除冲突别名、合并重复的内容单元、删除孤立的临时内容单元。默认只读(dry-run),加 --apply 才改库。
 //
 // 用法(DATABASE_URL 必须显式给出,脚本不带默认库):
 //   DATABASE_URL=postgres://umt:***@<host>:5432/umt deno run -A scripts/identity-cleanup.ts [--dry-run]       # 看清单(按类别列删除 / 保留)
 //   DATABASE_URL=… deno run -A scripts/identity-cleanup.ts --apply                                          # 执行(单个事务)
 //   DATABASE_URL=… deno run -A scripts/identity-cleanup.ts --keep-orphans                                   # 不删孤立单元
 //   DATABASE_URL=… deno run -A scripts/identity-cleanup.ts --entity <作品 id>                              # 只看 / 只处理一部作品
+//   DATABASE_URL=… deno run -A scripts/identity-cleanup.ts --merge-duplicates [--apply]                    # 只做第 2 步:合并重复单元(DH-1)
 //
 // 1. 冲突别名:只处理自动识别登记的站点别名(title_alias.source = 'site',作品级)。以下任一成立即删除:
 //    - shared:同一归一化别名还挂在别的作品上,且它不是本作品自己的名字(主标题 / 原标题 / TMDB 译名与别名);
 //    - script:作品有已接受的外部身份,且同一种文字(中文 / 拉丁)下已有确认的外部标题,这个别名对不上任何一个
 //      (中文标题的拼音写法视为相容,保留);
 //    - year:本作品下带这个名字的资源都解析出了年份,且都与作品年份相差超过 1 年。
-// 2. 重复的临时内容单元:同一作品、同一类型、同季同集号有多条(未退役)且至少一条是临时(provisional)的,
-//    保留被引用最多的那条(同数时 confirmed 优先,再取最早建的),其余的引用改指向保留的那条后删除;
-//    改指向会撞唯一约束的行(同一字段来源 / 别名已在保留的单元上)直接删掉。
+// 2. 重复的内容单元(DH-1,与 refreshMetadata 共用 apps/server/.../unit-merge.ts):同一作品、同一类型、同季同集号、
+//    同内容类型与分段有多条未退役单元时,保留被引用最多的那条(同数时 confirmed 优先,再取最早建的),其余的全部引用
+//    (资源覆盖、资产、下载计划、订阅覆盖、字段来源…按外键自动发现)改指向保留的那条后删除;
+//    改指向会撞唯一约束的行(同一行已在保留的单元上)直接删掉。
 // 3. 孤立的临时内容单元(--keep-orphans 跳过):只删资源解析建出的临时单元——provisional、没有任何资源 / 资产 / 订阅 /
 //    下载 / 整理等引用、没有任何元数据痕迹(字段来源 / 覆盖 / 标题 / 播出日期 / 总集数来源),且作品没有外部映射或
 //    这个季 / 集超出作品的外部季集结构;没有保留的子单元。TMDB / 元数据来源的单元永不删除(FIX-6)。
 //    字段来源 / 覆盖 / 单元别名随之删除。
 // --apply 在一个事务里执行,并写一条审计事件(action = identity.cleanup)。

+import type { DB } from "@umt/db";
 import { normalizeTitle, parseRelease } from "@umt/domain";
+import { Kysely, PostgresDialect } from "kysely";
 import pg from "pg";
+import {
+  applyUnitMerges,
+  planUnitMerges,
+  type UnitMergeGroup,
+} from "../apps/server/src/modules/sources-identity/service/unit-merge.ts";

 const argv = [...Deno.args];
 const APPLY = argv.includes("--apply");
@@ -29,7 +38,8 @@ if (APPLY && argv.includes("--dry-run")) {
  console.error("--apply 与 --dry-run 不能同时给出");
  Deno.exit(2);
 }
-const KEEP_ORPHANS = argv.includes("--keep-orphans");
+const MERGE_ONLY = argv.includes("--merge-duplicates");
+const KEEP_ORPHANS = argv.includes("--keep-orphans") || MERGE_ONLY;
 const entityArg = argv.indexOf("--entity");
 const ONLY_ENTITY = entityArg >= 0 ? (argv[entityArg + 1] ?? null) : null;
 const url = Deno.env.get("DATABASE_URL");
@@ -39,8 +49,16 @@ if (!url) {
 }
 const db = new pg.Client({ connectionString: url });
 await db.connect();
+// 合并模块用 Kysely:与 q() 共用同一个连接(同一事务)
+const kdb = new Kysely<DB>({
+  dialect: new PostgresDialect({
+    pool: { connect: async () => Object.assign(db, { release() {} }), end: async () => {} } as never,
+  }),
+});
 const q = async <T>(text: string, values: unknown[] = []) => (await db.query(text, values)).rows as T[];
-console.log(`# identity-cleanup (${APPLY ? "APPLY" : "DRY-RUN"})${KEEP_ORPHANS ? " --keep-orphans" : ""}\n`);
+console.log(
+  `# identity-cleanup (${APPLY ? "APPLY" : "DRY-RUN"})${MERGE_ONLY ? " --merge-duplicates" : KEEP_ORPHANS ? " --keep-orphans" : ""}\n`,
+);

 const isHan = (s: string) => /\p{Script=Han}/u.test(s);

@@ -80,11 +98,13 @@ interface EntityRow {
 }
 const entityFilter = ONLY_ENTITY ? "and a.media_entity_id = $1" : "";
 const entityVals = ONLY_ENTITY ? [ONLY_ENTITY] : [];
-const siteAliases = await q<AliasRow>(
-  `select a.id, a.media_entity_id, a.alias, a.normalized, a.source from title_alias a
-    where a.source = 'site' and a.content_unit_id is null ${entityFilter}`,
-  entityVals,
-);
+const siteAliases = MERGE_ONLY
+  ? []
+  : await q<AliasRow>(
+      `select a.id, a.media_entity_id, a.alias, a.normalized, a.source from title_alias a
+        where a.source = 'site' and a.content_unit_id is null ${entityFilter}`,
+      entityVals,
+    );
 const entityIds = [...new Set(siteAliases.map((a) => a.media_entity_id))];
 const entities = new Map(
  (
@@ -183,7 +203,7 @@ const entityTitle = async (id: string) =>
  entities.get(id)?.title ??
  (await q<{ title: string }>(`select title from media_entity where id = $1`, [id]))[0]?.title ??
  id;
-console.log(`## 冲突别名:${aliasDeletes.length} 个`);
+if (!MERGE_ONLY) console.log(`## 冲突别名:${aliasDeletes.length} 个`);
 for (const eid of [...new Set(aliasDeletes.map((a) => a.media_entity_id))]) {
  const e = entities.get(eid)!;
  console.log(`\n### ${e.title}(${e.year ?? "-"}) ${eid}`);
@@ -228,7 +248,7 @@ async function refCounts(ids: string[]): Promise<Map<string, number>> {
  return out;
 }

-// ---------------------------------------------------------------- 2. 重复的临时内容单元
+// ---------------------------------------------------------------- 2. 重复的内容单元
 interface UnitRow {
  id: string;
  media_entity_id: string;
@@ -239,43 +259,19 @@ interface UnitRow {
  created_at: Date;
 }
 const unitFilter = ONLY_ENTITY ? "and u.media_entity_id = $1" : "";
-const dupRows = await q<UnitRow>(
-  `select u.id, u.media_entity_id, u.kind, u.season_number, u.episode_number, u.status, u.created_at
-     from content_unit u
-     join (select media_entity_id, kind, season_number, episode_number from content_unit
-            where status <> 'retired' group by 1, 2, 3, 4
-           having count(*) > 1 and bool_or(status = 'provisional')) g
-       on g.media_entity_id = u.media_entity_id and g.kind = u.kind
-      and g.season_number is not distinct from u.season_number and g.episode_number is not distinct from u.episode_number
-    where u.status <> 'retired' ${unitFilter}
-    order by u.media_entity_id, u.season_number, u.episode_number, u.created_at`,
-  ONLY_ENTITY ? [ONLY_ENTITY] : [],
-);
-const dupRefs = await refCounts(dupRows.map((u) => u.id));
-const groups = new Map<string, UnitRow[]>();
-for (const u of dupRows) {
-  const k = `${u.media_entity_id}|${u.kind}|${u.season_number}|${u.episode_number}`;
-  groups.set(k, [...(groups.get(k) ?? []), u]);
-}
-const merges: { keep: UnitRow; drop: UnitRow[] }[] = [];
-for (const list of groups.values()) {
-  const sorted = [...list].sort(
-    (a, b) =>
-      (dupRefs.get(b.id) ?? 0) - (dupRefs.get(a.id) ?? 0) ||
-      Number(b.status === "confirmed") - Number(a.status === "confirmed") ||
-      new Date(a.created_at).getTime() - new Date(b.created_at).getTime(),
-  );
-  merges.push({ keep: sorted[0]!, drop: sorted.slice(1) });
-}
+const merges: UnitMergeGroup[] = await planUnitMerges(kdb, ONLY_ENTITY ? { mediaEntityId: ONLY_ENTITY } : {});
 const unitLabel = (u: Pick<UnitRow, "kind" | "season_number" | "episode_number">) =>
  u.kind === "season"
    ? `S${String(u.season_number ?? 0).padStart(2, "0")}`
    : `S${String(u.season_number ?? 0).padStart(2, "0")}E${String(u.episode_number ?? 0).padStart(2, "0")}`;
-console.log(`\n## 重复的临时内容单元:${merges.length} 组,删除 ${merges.reduce((s, m) => s + m.drop.length, 0)} 条`);
+const groupLabel = (g: UnitMergeGroup) =>
+  unitLabel({ kind: g.kind, season_number: g.seasonNumber, episode_number: g.episodeNumber }) +
+  (g.contentType !== "main" || g.segment !== "full" ? `(${g.contentType}/${g.segment})` : "");
+console.log(`\n## 重复的内容单元:${merges.length} 组,删除 ${merges.reduce((s, m) => s + m.drop.length, 0)} 条`);
 for (const m of merges) {
  console.log(
-    `  - 《${await entityTitle(m.keep.media_entity_id)}》${unitLabel(m.keep)}:保留 ${m.keep.id}(${m.keep.status},引用 ${dupRefs.get(m.keep.id) ?? 0}),` +
-      `合并 ${m.drop.map((d) => `${d.id}(${d.status},引用 ${dupRefs.get(d.id) ?? 0})`).join("、")}`,
+    `  - 《${await entityTitle(m.mediaEntityId)}》${groupLabel(m)}:保留 ${m.keep.id}(${m.keep.status},引用 ${m.keep.refs}),` +
+      `合并 ${m.drop.map((d) => `${d.id}(${d.status},引用 ${d.refs})`).join("、")}`,
  );
 }

@@ -395,7 +391,7 @@ if (!KEEP_ORPHANS) {
    console.log(`  - 保留·${KEEP_TEXT[r]}:${l.length} 条`);
    if (r !== "referenced") await printGroup(l);
  }
-} else console.log("\n## 孤立的临时内容单元:跳过(--keep-orphans)");
+} else if (!MERGE_ONLY) console.log("\n## 孤立的临时内容单元:跳过(--keep-orphans)");

 // ---------------------------------------------------------------- 执行
 if (APPLY) {
@@ -403,37 +399,7 @@ if (APPLY) {
  try {
    if (aliasDeletes.length)
      await q(`delete from title_alias where id = any($1::uuid[])`, [aliasDeletes.map((a) => a.id)]);
-    let repointed = 0;
-    let droppedRows = 0;
-    const allRefTables = fks.filter((f) => f.table !== "content_unit");
-    for (const m of merges) {
-      for (const d of m.drop) {
-        for (const f of allRefTables) {
-          const rows = await q<{ ctid: string }>(
-            `select ctid::text from ${quoteIdent(f.table)} where ${quoteIdent(f.column)} = $1`,
-            [d.id],
-          );
-          for (const r of rows) {
-            await db.query("savepoint sp");
-            try {
-              await q(`update ${quoteIdent(f.table)} set ${quoteIdent(f.column)} = $1 where ctid = $2::tid`, [
-                m.keep.id,
-                r.ctid,
-              ]);
-              await db.query("release savepoint sp");
-              repointed++;
-            } catch (e) {
-              await db.query("rollback to savepoint sp");
-              if ((e as { code?: string }).code !== "23505") throw e;
-              await q(`delete from ${quoteIdent(f.table)} where ctid = $1::tid`, [r.ctid]);
-              droppedRows++;
-            }
-          }
-        }
-        await q(`update content_unit set parent_unit_id = $1 where parent_unit_id = $2`, [m.keep.id, d.id]);
-        await q(`delete from content_unit where id = $1`, [d.id]);
-      }
-    }
+    const merged = await applyUnitMerges(kdb, merges);
    // 孤立单元:先集后季
    const orphanIds = orphans
      .sort((a, b) => Number(a.kind === "season") - Number(b.kind === "season"))
@@ -453,6 +419,7 @@ if (APPLY) {
        JSON.stringify({
          aliases: aliasDeletes.map((a) => ({ entityId: a.media_entity_id, alias: a.alias, reason: a.reason })),
          mergedUnits: merges.map((m) => ({ keep: m.keep.id, drop: m.drop.map((d) => d.id) })),
+          mode: MERGE_ONLY ? "merge-duplicates" : "full",
          orphanUnits: orphans.map((u) => ({
            id: u.id,
            entityId: u.media_entity_id,
@@ -464,7 +431,7 @@ if (APPLY) {
    );
    await db.query("commit");
    console.log(
-      `\n已执行:删除 ${aliasDeletes.length} 个别名;合并 ${dropping.size} 个重复单元(改指向 ${repointed} 行,删除冲突行 ${droppedRows} 行);` +
+      `\n已执行:删除 ${aliasDeletes.length} 个别名;合并 ${merged.merged} 个重复单元(改指向 ${merged.repointed} 行,删除冲突行 ${merged.droppedRows} 行);` +
        `删除 ${orphans.length} 个孤立单元`,
    );
  } catch (e) {