feat(scripts): TMDB 多语言别名清理脚本与保留规则文档
Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
7efbe25c17
commit
ce93ea3a93
5 files changed
+453
No files matched your search
@@ -0,0 +1,187 @@
|
||||
// TMDB 多语言别名清理(用户规则 2026-10-06):TMDB 的多语言数据只保留简中 / 繁中 / 英文 / 作品原语言。
|
||||
// 规则见 packages/adapters/src/tmdb/language-policy.ts、docs/runbook/identity.md「TMDB 多语言保留规则」。
|
||||
// 默认只读(dry-run),加 --apply 才删(单个事务,写一条审计 metadata.tmdb_alias_language_cleanup)。
|
||||
//
|
||||
// 用法(DATABASE_URL 必须显式给出,脚本不带默认库):
|
||||
// DATABASE_URL=postgres://umt:***@<host>:5432/umt deno run -A scripts/tmdb-alias-language-cleanup.ts # 看统计与样例
|
||||
// DATABASE_URL=… deno run -A scripts/tmdb-alias-language-cleanup.ts --apply # 删除
|
||||
// DATABASE_URL=… deno run -A scripts/tmdb-alias-language-cleanup.ts --entity <作品 id> # 只看 / 只处理一部作品
|
||||
// DATABASE_URL=… deno run -A scripts/tmdb-alias-language-cleanup.ts --samples 20 # 每类样例条数(缺省 5)
|
||||
//
|
||||
// 别名(title_alias)判定——只动 source = 'tmdb' 且未锁定的行;人工(user)/ 站点(site)/ 智能体(agent)/ NFO / 导入来源与
|
||||
// 锁定的行一律不动:
|
||||
// - kind = original(原名)保留;与作品主标题 / 原标题归一化相同的保留(作品自己的名字);
|
||||
// - language 非空:规范化后是简中(zh-CN / zh-SG / zh-Hans / zh)、繁中(zh-TW / zh-HK / zh-MO / zh-Hant)、英文(en-*)、
|
||||
// 作品原语言(media_entity.metadata.originalLanguage,没有时取 TMDB 字段来源)→ 保留,否则删除(原因 language);
|
||||
// - language 为空(旧版建档写的 other 等):按文字判断——汉字(不含假名 / 谚文)、拉丁字母、没有字母的保留;
|
||||
// 其它文字(西里尔 / 阿拉伯 / 希伯来 / 泰文 / 谚文 / 假名 …)只在与作品原语言相符时保留,否则删除(原因 script)。
|
||||
// 拉丁字母无法区分英 / 法 / 西 / 葡,一律按英文保留。
|
||||
// 字段来源(metadata_field,来源 tmdb)里语言不在保留范围的标题 / 简介 / 集名只报告不删:删了会让最终值暂时没有来源,
|
||||
// 对这些作品跑一次元数据刷新(语言链已按作品收敛)即被同字段的保留语言值替换(同字段换语言时删旧行)。
|
||||
// 作品与单元的 metadata 只存按语言链取到的单个值,不存 translations 列表,没有别的多语言数据要收敛。
|
||||
|
||||
import { allowedLanguage, untaggedTextAllowed } from "@umt/adapters";
|
||||
import { normalizeTitle } from "@umt/domain";
|
||||
import pg from "pg";
|
||||
|
||||
const argv = [...Deno.args];
|
||||
const APPLY = argv.includes("--apply");
|
||||
if (APPLY && argv.includes("--dry-run")) {
|
||||
console.error("--apply 与 --dry-run 不能同时给出");
|
||||
Deno.exit(2);
|
||||
}
|
||||
const argOf = (k: string) => {
|
||||
const i = argv.indexOf(k);
|
||||
return i >= 0 ? (argv[i + 1] ?? null) : null;
|
||||
};
|
||||
const ONLY_ENTITY = argOf("--entity");
|
||||
const SAMPLES = Number(argOf("--samples") ?? 5) || 5;
|
||||
const url = Deno.env.get("DATABASE_URL");
|
||||
if (!url) {
|
||||
console.error("需要 DATABASE_URL(脚本不带默认库)");
|
||||
Deno.exit(2);
|
||||
}
|
||||
const db = new pg.Client({ connectionString: url });
|
||||
await db.connect();
|
||||
const q = async <T>(text: string, values: unknown[] = []) => (await db.query(text, values)).rows as T[];
|
||||
console.log(
|
||||
`# tmdb-alias-language-cleanup (${APPLY ? "APPLY" : "DRY-RUN"})${ONLY_ENTITY ? ` --entity ${ONLY_ENTITY}` : ""}\n`,
|
||||
);
|
||||
|
||||
interface Row {
|
||||
id: string;
|
||||
media_entity_id: string;
|
||||
alias: string;
|
||||
normalized: string;
|
||||
language: string | null;
|
||||
kind: string;
|
||||
title: string;
|
||||
original_title: string | null;
|
||||
orig_lang: string | null;
|
||||
}
|
||||
|
||||
// 作品原语言:投影值优先,没有时取 tmdb 字段来源
|
||||
const ORIG_LANG = `coalesce(e.metadata->'originalLanguage'->>'value',
|
||||
(select f.value#>>'{}' from metadata_field f where f.media_entity_id = e.id and f.content_unit_id is null
|
||||
and f.field = 'originalLanguage' and f.source = 'tmdb' order by f.fetched_at desc limit 1))`;
|
||||
|
||||
const rows = await q<Row>(
|
||||
`select a.id, a.media_entity_id, a.alias, a.normalized, a.language, a.kind::text as kind,
|
||||
e.title, e.original_title, ${ORIG_LANG} as orig_lang
|
||||
from title_alias a join media_entity e on e.id = a.media_entity_id
|
||||
where a.source = 'tmdb' and not a.locked and ($1::uuid is null or a.media_entity_id = $1::uuid)
|
||||
order by e.title, a.alias`,
|
||||
[ONLY_ENTITY],
|
||||
);
|
||||
const other = await q<{ source: string; locked: boolean; n: number }>(
|
||||
`select source::text as source, locked, count(*)::int as n from title_alias
|
||||
where (source <> 'tmdb' or locked) and ($1::uuid is null or media_entity_id = $1::uuid)
|
||||
group by 1, 2 order by 1, 2`,
|
||||
[ONLY_ENTITY],
|
||||
);
|
||||
|
||||
type Verdict = { del: false; why: string } | { del: true; why: "language" | "script" };
|
||||
function judge(r: Row): Verdict {
|
||||
if (r.kind === "original") return { del: false, why: "原名" };
|
||||
const own = [r.title, r.original_title].filter((x): x is string => !!x).map(normalizeTitle);
|
||||
if (own.includes(r.normalized)) return { del: false, why: "作品自己的名字" };
|
||||
const work = { originalLanguage: r.orig_lang };
|
||||
if (r.language !== null)
|
||||
return allowedLanguage(r.language, work) ? { del: false, why: "保留语言" } : { del: true, why: "language" };
|
||||
return untaggedTextAllowed(r.alias, work) ? { del: false, why: "无语言·文字相符" } : { del: true, why: "script" };
|
||||
}
|
||||
|
||||
const deletes: (Row & { why: string })[] = [];
|
||||
const keepBy = new Map<string, number>();
|
||||
for (const r of rows) {
|
||||
const v = judge(r);
|
||||
if (v.del) deletes.push({ ...r, why: v.why });
|
||||
else keepBy.set(v.why, (keepBy.get(v.why) ?? 0) + 1);
|
||||
}
|
||||
|
||||
console.log(
|
||||
`## 别名(来源 tmdb、未锁定):${rows.length} 条,删除 ${deletes.length} 条,保留 ${rows.length - deletes.length} 条`,
|
||||
);
|
||||
for (const [why, n] of [...keepBy].sort((a, b) => b[1] - a[1])) console.log(` 保留·${why}:${n} 条`);
|
||||
console.log(
|
||||
` 不动(非 tmdb 来源或已锁定):${other.map((o) => `${o.source}${o.locked ? "·锁定" : ""} ${o.n}`).join(",") || "0"}`,
|
||||
);
|
||||
const groups = new Map<string, (Row & { why: string })[]>();
|
||||
for (const d of deletes) {
|
||||
const k = `${d.language ?? "(空)"} / ${d.kind}`;
|
||||
groups.set(k, [...(groups.get(k) ?? []), d]);
|
||||
}
|
||||
if (deletes.length) console.log(`\n### 删除:按语言 / kind`);
|
||||
for (const [k, list] of [...groups].sort((a, b) => b[1].length - a[1].length)) {
|
||||
console.log(` ${k}:${list.length} 条`);
|
||||
for (const d of list.slice(0, SAMPLES))
|
||||
console.log(` - 《${d.title}》(原语言 ${d.orig_lang ?? "未知"}):${d.alias}`);
|
||||
}
|
||||
|
||||
// 字段来源里语言不在保留范围的(只报告)
|
||||
const fields = await q<{
|
||||
entity_id: string;
|
||||
title: string;
|
||||
orig_lang: string | null;
|
||||
field: string;
|
||||
language: string;
|
||||
unit: boolean;
|
||||
}>(
|
||||
`select e.id as entity_id, e.title, ${ORIG_LANG} as orig_lang, f.field, f.language, f.content_unit_id is not null as unit
|
||||
from metadata_field f
|
||||
left join content_unit u on u.id = f.content_unit_id
|
||||
join media_entity e on e.id = coalesce(f.media_entity_id, u.media_entity_id)
|
||||
where f.source = 'tmdb' and f.language is not null and ($1::uuid is null or e.id = $1::uuid)`,
|
||||
[ONLY_ENTITY],
|
||||
);
|
||||
const badFields = fields.filter((f) => !allowedLanguage(f.language, { originalLanguage: f.orig_lang }));
|
||||
console.log(`\n## 字段来源(tmdb)语言不在保留范围:${badFields.length} 条(只报告,对涉及的作品跑元数据刷新即收敛)`);
|
||||
const byLang = new Map<string, number>();
|
||||
for (const f of badFields)
|
||||
byLang.set(
|
||||
`${f.language} / ${f.unit ? "单元" : "作品"}·${f.field}`,
|
||||
(byLang.get(`${f.language} / ${f.unit ? "单元" : "作品"}·${f.field}`) ?? 0) + 1,
|
||||
);
|
||||
for (const [k, n] of [...byLang].sort((a, b) => b[1] - a[1])) console.log(` ${k}:${n} 条`);
|
||||
const badEntities = [...new Map(badFields.map((f) => [f.entity_id, f.title])).entries()];
|
||||
for (const [id, title] of badEntities.slice(0, SAMPLES * 4)) console.log(` - 《${title}》 ${id}`);
|
||||
|
||||
if (!APPLY) {
|
||||
console.log(`\n(dry-run,未改库;确认后加 --apply)`);
|
||||
} else if (deletes.length) {
|
||||
try {
|
||||
await db.query("begin");
|
||||
const ids = deletes.map((d) => d.id);
|
||||
const res = await db.query(
|
||||
`delete from title_alias where id = any($1::uuid[]) and source = 'tmdb' and not locked`,
|
||||
[ids],
|
||||
);
|
||||
await db.query(
|
||||
`insert into audit_event (actor_type, actor_id, action, target_type, target_id, payload)
|
||||
values ('system', 'tmdb-alias-language-cleanup', 'metadata.tmdb_alias_language_cleanup', 'media_entity', $1, $2::jsonb)`,
|
||||
[
|
||||
ONLY_ENTITY,
|
||||
JSON.stringify({
|
||||
deleted: res.rowCount,
|
||||
aliases: deletes.map((d) => ({
|
||||
entityId: d.media_entity_id,
|
||||
alias: d.alias,
|
||||
language: d.language,
|
||||
kind: d.kind,
|
||||
reason: d.why,
|
||||
})),
|
||||
}),
|
||||
],
|
||||
);
|
||||
await db.query("commit");
|
||||
console.log(`\n已删除 ${res.rowCount} 条别名`);
|
||||
} catch (e) {
|
||||
await db.query("rollback");
|
||||
console.error("执行失败,已回滚:", (e as Error).message);
|
||||
await db.end();
|
||||
Deno.exit(1);
|
||||
}
|
||||
} else {
|
||||
console.log(`\n没有要删除的别名`);
|
||||
}
|
||||
await db.end();
|
||||
Reference in new issue
Block a user