merge: fix/x-tmdb-lang
Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei
This commit is contained in:
commit
cfe9ad1a71
16 files changed
+914
-49
No files matched your search
@@ -1,4 +1,4 @@
|
||||
import type { MetadataCandidate } from "@umt/adapters";
|
||||
import { allowedLanguage, type MetadataCandidate, textMatchesLanguage } from "@umt/adapters";
|
||||
import type { AliasKind, AliasSource, EvidenceLevel, MediaKind } from "@umt/contracts";
|
||||
import { sql } from "@umt/db";
|
||||
import { normalizeTitle, type ParsedRelease } from "@umt/domain";
|
||||
@@ -171,6 +171,11 @@ export async function createEntityFromTmdb(
|
||||
return { entityId: entity.id, created: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* TMDB 详情登记别名(来源 tmdb)。多语言保留规则(adapters language-policy.ts):details() 已只留简中 / 繁中 / 英文 /
|
||||
* 原语言的本地化名、译名与别名(别名带按地区 + 文字推断的语言),这里按语言写入,不再写空语言的 other。
|
||||
* 本地化名的文字与语言不符时(TMDB 缺该语言译名时回退成原名)不按该语言登记。
|
||||
*/
|
||||
export async function seedAliases(exec: Exec, entityId: string, m: MetadataCandidate): Promise<void> {
|
||||
// 同一文本只登记一次(按首个出现的类别):TMDB 的译名/别名列表常重复主标题与原名
|
||||
const seen = new Set<string>();
|
||||
@@ -180,12 +185,21 @@ export async function seedAliases(exec: Exec, entityId: string, m: MetadataCandi
|
||||
seen.add(n);
|
||||
await addAlias(exec, entityId, text, kind, "tmdb", lang);
|
||||
};
|
||||
await add(m.title, "translated", null);
|
||||
const work = { originalLanguage: m.originalLanguage, originCountry: m.originCountry };
|
||||
if (m.originalTitle) await add(m.originalTitle, "original", m.originalLanguage);
|
||||
let n = 0;
|
||||
for (const t of m.titles) {
|
||||
if (t.kind === "localized") await add(t.text, "translated", t.lang);
|
||||
if (t.kind === "original" || !t.lang || n >= 50) continue;
|
||||
const lang = allowedLanguage(t.lang, work);
|
||||
if (!lang || !textMatchesLanguage(lang, t.text)) continue;
|
||||
const kind: AliasKind = t.kind === "alias" ? (lang.endsWith("-Latn") ? "romanized" : "other") : "translated";
|
||||
await add(t.text, kind, lang);
|
||||
n++;
|
||||
}
|
||||
for (const a of m.aliases.slice(0, 50)) await add(a, "other", null);
|
||||
// 主标题(建档时可能来自语言链):上面没登记到时补一条,语言按同文本的本地化名(文字相符才标)
|
||||
const loc = m.titles.find((t) => t.text === m.title && t.kind === "localized" && t.lang);
|
||||
const titleLang = loc?.lang && textMatchesLanguage(loc.lang, m.title) ? allowedLanguage(loc.lang, work) : null;
|
||||
await add(m.title, "translated", titleLang);
|
||||
}
|
||||
|
||||
export class IdentityConflictError extends Error {
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import {
|
||||
DEFAULT_METADATA_LANGUAGES,
|
||||
scriptMatches,
|
||||
type TmdbRichDetail,
|
||||
type TmdbRichEpisode,
|
||||
type TmdbRichSeasonDetail,
|
||||
textMatchesLanguage,
|
||||
} from "@umt/adapters";
|
||||
import type {
|
||||
EntityMetadataField,
|
||||
@@ -25,6 +25,7 @@ import { ensureSeason, promoteUnits } from "./metadata-totals.ts";
|
||||
// - 作品:TMDB 富详情(fetchRichDetail)按设置 metadata.languages 取标题 / 简介 / 标语(MD-2:简介 / 标语缺省只回退到中文变体),另存中文名 titleZh、类型、评分、
|
||||
// 状态、时长、季数、主创、外部 ID;来源值写 metadata_field(language = 实际取到的语言),同字段换语言时删旧行;
|
||||
// - 别名:translations / alternative_titles 写 title_alias(来源 tmdb,标语言);已有同文本别名只补空的语言;
|
||||
// 只写简中 / 繁中 / 英文 / 原语言(多语言保留规则,adapters language-policy.ts);
|
||||
// - 季 / 集:fetchRichSeason 的每集 name / overview / airDate / runtime / stillPath / vote 写 metadata_field(content_unit_id),
|
||||
// 投影到 content_unit.metadata(字段级,可锁),name → title 列、airDate → air_date 列;只写变化的字段,
|
||||
// 不改编号、不碰路径绑定与资产覆盖。集单元不存在时按 TMDB 建(confirmed,挂季单元)。
|
||||
@@ -152,26 +153,22 @@ export async function writeRichEntityFields(
|
||||
);
|
||||
}
|
||||
|
||||
/** 别名的地区 → 语言:只在文字相符时标(CN 地区的英文别名不标 zh-CN) */
|
||||
function altLanguage(region: string | null, text: string): string | null {
|
||||
if (!region) return null;
|
||||
if (["CN", "SG", "TW", "HK", "MO"].includes(region)) return scriptMatches("zh", text) ? `zh-${region}` : null;
|
||||
if (region === "JP") return scriptMatches("ja", text) ? "ja-JP" : null;
|
||||
if (region === "KR") return scriptMatches("ko", text) ? "ko-KR" : null;
|
||||
if (["US", "GB", "CA", "AU", "NZ", "IE"].includes(region)) return scriptMatches("en", text) ? `en-${region}` : null;
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* translations(译名,kind translated)与 alternative_titles(别名,kind other)写 title_alias,来源 tmdb、标语言。
|
||||
* translations(译名,kind translated)与 alternative_titles(别名,kind other;原产地区罗马字 romanized)写 title_alias,
|
||||
* 来源 tmdb、标语言。多语言保留规则:fetchRichDetail 已只留简中 / 繁中 / 英文 / 原语言的译名,别名带按地区 + 文字推断的语言
|
||||
* (推断不出的已丢弃);这里再要求译名文字与语言相符。
|
||||
* 同作品已有同文本别名时不再新增,只给来源 tmdb 且语言为空的那行补语言。最多 80 条。
|
||||
*/
|
||||
export async function tagTitleAliases(exec: Exec, entityId: string, rich: TmdbRichDetail): Promise<number> {
|
||||
const want: { text: string; kind: "translated" | "other"; language: string | null }[] = [];
|
||||
type Kind = "translated" | "other" | "romanized";
|
||||
const want: { text: string; kind: Kind; language: string | null }[] = [];
|
||||
if (rich.titleZh) want.push({ text: rich.titleZh, kind: "translated", language: rich.titleZhLanguage });
|
||||
for (const t of rich.translations) if (t.title) want.push({ text: t.title, kind: "translated", language: t.lang });
|
||||
for (const t of rich.translations)
|
||||
if (t.title && textMatchesLanguage(t.lang, t.title))
|
||||
want.push({ text: t.title, kind: "translated", language: t.lang });
|
||||
for (const a of rich.alternativeTitles)
|
||||
want.push({ text: a.title, kind: "other", language: altLanguage(a.region, a.title) });
|
||||
if (a.language)
|
||||
want.push({ text: a.title, kind: a.language.endsWith("-Latn") ? "romanized" : "other", language: a.language });
|
||||
const rows = await exec
|
||||
.selectFrom("title_alias")
|
||||
.select(["id", "normalized", "language", "source"])
|
||||
|
||||
@@ -361,7 +361,14 @@ export async function refreshMetadata(
|
||||
creatable = new Set(opts.full ? want : tracked);
|
||||
for (const n of want) {
|
||||
try {
|
||||
richSeasons.push(await fetchRichSeason(tmdb, ident.external_id, n, langs, { now: rt.now(), ...textOpts }));
|
||||
richSeasons.push(
|
||||
await fetchRichSeason(tmdb, ident.external_id, n, langs, {
|
||||
now: rt.now(),
|
||||
...textOpts,
|
||||
// 语言链按作品原语言收敛(多语言保留规则)
|
||||
originalLanguage: richDetail?.originalLanguage ?? detail.originalLanguage,
|
||||
}),
|
||||
);
|
||||
} catch (e) {
|
||||
ctx.logger.debug({ entityId: id, season: n, err: { message: (e as Error).message } }, "季详情获取失败");
|
||||
}
|
||||
|
||||
@@ -0,0 +1,216 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { createMockTmdb, type MockTmdb, type TmdbFixture } from "@umt/testkit";
|
||||
import { sql } from "kysely";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { createEntityFromTmdbCandidate } from "../../src/modules/sources-identity/service/entity-from-candidate.ts";
|
||||
import { refreshMetadata } from "../../src/modules/sources-identity/service/metadata.ts";
|
||||
import { createHarness, type Harness, PG_URL, pgAvailable } from "./harness.ts";
|
||||
|
||||
// TMDB 多语言保留规则(用户规则 2026-10-06):建档 / 刷新只写简中 / 繁中 / 英文 / 原语言的别名与字段;
|
||||
// scripts/tmdb-alias-language-cleanup.ts 只删 TMDB 来源、语言不在保留范围的别名。
|
||||
|
||||
const up = await pgAvailable();
|
||||
const ROOT = fileURLToPath(new URL("../../../../", import.meta.url));
|
||||
const SCRIPT = fileURLToPath(new URL("../../../../scripts/tmdb-alias-language-cleanup.ts", import.meta.url));
|
||||
|
||||
/** 日语原作的剧:各种语言的译名与别名 */
|
||||
const AOT: TmdbFixture = {
|
||||
type: "tv",
|
||||
id: 1429,
|
||||
original_title: "進撃の巨人",
|
||||
original_language: "ja",
|
||||
origin_country: ["JP"],
|
||||
date: "2013-04-07",
|
||||
titles: { "zh-CN": "进击的巨人", "en-US": "Attack on Titan", "ja-JP": "進撃の巨人", "zh-TW": "進擊的巨人" },
|
||||
overviews: { "zh-CN": "巨人吃人的世界。", "ja-JP": "巨人が人を喰う世界。", "fr-FR": "Un monde de titans." },
|
||||
popularity: 200,
|
||||
alternative_titles: [
|
||||
{ iso_3166_1: "JP", title: "Shingeki no Kyojin", type: "Romaji" },
|
||||
{ iso_3166_1: "US", title: "AoT", type: "" },
|
||||
{ iso_3166_1: "BR", title: "Ataque dos Titãs", type: "" },
|
||||
{ iso_3166_1: "RU", title: "Атака на титанов", type: "" },
|
||||
{ iso_3166_1: "CN", title: "巨人", type: "简称" },
|
||||
{ iso_3166_1: "CN", title: "Attack Titan CN", type: "" },
|
||||
{ iso_3166_1: "KR", title: "진격의 거인", type: "" },
|
||||
],
|
||||
extra_translations: [
|
||||
{ iso_639_1: "ru", iso_3166_1: "RU", title: "Атака титанов" },
|
||||
{ iso_639_1: "fr", iso_3166_1: "FR", title: "L'Attaque des Titans" },
|
||||
{ iso_639_1: "ko", iso_3166_1: "KR", title: "진격의 거인" },
|
||||
{ iso_639_1: "th", iso_3166_1: "TH", title: "ผ่าพิภพไททัน" },
|
||||
],
|
||||
external_ids: { imdb_id: "tt2560140", tvdb_id: 267440 },
|
||||
poster_path: "/aot.jpg",
|
||||
rich: { genres: ["动画"], status: "Ended" },
|
||||
seasons: [
|
||||
{
|
||||
season_number: 1,
|
||||
air_date: "2013-04-07",
|
||||
names: { "zh-CN": "第 1 季", "en-US": "Season 1" },
|
||||
episodes: [
|
||||
{
|
||||
episode_number: 1,
|
||||
air_date: "2013-04-07",
|
||||
names: { "fr-FR": "À toi, dans 2000 ans", "ja-JP": "二千年後の君へ" },
|
||||
overviews: { "fr-FR": "Le mur tombe.", "ja-JP": "壁が壊される。" },
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const ALLOWED = /^(zh(-(CN|SG|TW|HK|MO|Hans|Hant))?|en(-[A-Z]{2})?|ja(-JP|-Latn)?)$/;
|
||||
|
||||
describe.skipIf(!up)("TMDB 多语言保留规则", () => {
|
||||
let h: Harness;
|
||||
let tmdb: MockTmdb;
|
||||
let id: string;
|
||||
let url: string;
|
||||
const actor = { type: "user" as const, id: "tester" };
|
||||
|
||||
const run = (...args: string[]) =>
|
||||
execFileSync(Deno.execPath(), ["run", "-A", SCRIPT, ...args], {
|
||||
cwd: ROOT,
|
||||
env: { ...process.env, DATABASE_URL: url, NO_COLOR: "1" },
|
||||
encoding: "utf8",
|
||||
});
|
||||
const aliases = () =>
|
||||
h.db
|
||||
.selectFrom("title_alias")
|
||||
.select(["alias", "language", "kind", "source", "locked"])
|
||||
.where("media_entity_id", "=", id)
|
||||
.orderBy("alias")
|
||||
.execute();
|
||||
|
||||
beforeAll(async () => {
|
||||
tmdb = await createMockTmdb({ fixtures: [AOT] });
|
||||
h = await createHarness("tmdblang", { tmdbUrl: tmdb.url, now: () => new Date("2026-10-06T00:00:00Z") });
|
||||
const schema = (await sql<{ s: string }>`select current_schema() as s`.execute(h.db)).rows[0]!.s;
|
||||
const u = new URL(PG_URL);
|
||||
u.searchParams.set("options", `-c search_path=${schema},public`);
|
||||
url = u.toString();
|
||||
// 用户在语言链里放了 fr-FR(不是原语言,应被跳过)与 ja-JP(是原语言,保留)
|
||||
await h.db
|
||||
.insertInto("settings")
|
||||
.values({ key: "metadata", value: JSON.stringify({ languages: ["fr-FR", "zh-CN", "ja-JP", "en-US"] }) } as never)
|
||||
.execute();
|
||||
});
|
||||
afterAll(async () => {
|
||||
await h?.close();
|
||||
await tmdb?.close();
|
||||
});
|
||||
|
||||
it("建档:别名只含保留语言,不写空语言的 other;罗马字记 ja-Latn", async () => {
|
||||
const r = await createEntityFromTmdbCandidate(h.db, () => h.rt.tmdb(), { externalId: "1429", kind: "tv" }, actor);
|
||||
id = r.mediaEntityId;
|
||||
const list = await aliases();
|
||||
const texts = list.map((a) => a.alias);
|
||||
for (const bad of [
|
||||
"Атака титанов",
|
||||
"Атака на титанов",
|
||||
"L'Attaque des Titans",
|
||||
"진격의 거인",
|
||||
"ผ่าพิภพไททัน",
|
||||
"Ataque dos Titãs",
|
||||
"Attack Titan CN",
|
||||
])
|
||||
expect(texts).not.toContain(bad);
|
||||
expect(list.find((a) => a.alias === "進撃の巨人")).toMatchObject({ kind: "original", language: "ja" });
|
||||
expect(list.find((a) => a.alias === "Shingeki no Kyojin")).toMatchObject({
|
||||
kind: "romanized",
|
||||
language: "ja-Latn",
|
||||
});
|
||||
expect(list.find((a) => a.alias === "巨人")).toMatchObject({ kind: "other", language: "zh-CN" });
|
||||
expect(list.find((a) => a.alias === "AoT")).toMatchObject({ language: "en-US" });
|
||||
expect(list.filter((a) => a.kind === "other" && a.language === null)).toEqual([]);
|
||||
for (const a of list) if (a.language !== null) expect(a.language).toMatch(ALLOWED);
|
||||
});
|
||||
|
||||
it("刷新:语言链去掉 fr-FR,标题 / 简介 / 集名不取法语;别名仍只含保留语言", async () => {
|
||||
const res = await refreshMetadata(h.rt, id, { full: true });
|
||||
expect(res.status).toBe("refreshed");
|
||||
expect(res.titles).toMatchObject({ title: "进击的巨人", language: "zh-CN" });
|
||||
const fields = await h.db
|
||||
.selectFrom("metadata_field")
|
||||
.leftJoin("content_unit", "content_unit.id", "metadata_field.content_unit_id")
|
||||
.select(["metadata_field.field", "metadata_field.language", "metadata_field.value"])
|
||||
.where((eb) =>
|
||||
eb.or([eb("metadata_field.media_entity_id", "=", id), eb("content_unit.media_entity_id", "=", id)]),
|
||||
)
|
||||
.where("metadata_field.language", "is not", null)
|
||||
.execute();
|
||||
expect(fields.length).toBeGreaterThan(0);
|
||||
for (const f of fields) expect(f.language).toMatch(ALLOWED);
|
||||
const ep = await h.db
|
||||
.selectFrom("content_unit")
|
||||
.select(["title", "metadata"])
|
||||
.where("media_entity_id", "=", id)
|
||||
.where("kind", "=", "episode")
|
||||
.executeTakeFirstOrThrow();
|
||||
// 集名:fr-FR 被跳过,ja-JP(原语言)可用
|
||||
expect(ep.title).toBe("二千年後の君へ");
|
||||
expect((ep.metadata as Record<string, { language: string | null }>).name?.language).toBe("ja-JP");
|
||||
const list = await aliases();
|
||||
expect(list.filter((a) => a.kind === "other" && a.language === null)).toEqual([]);
|
||||
for (const a of list) if (a.language !== null) expect(a.language).toMatch(ALLOWED);
|
||||
expect(list.map((a) => a.alias)).not.toContain("L'Attaque des Titans");
|
||||
});
|
||||
|
||||
it("清理脚本:dry-run 统计、apply 只删 TMDB 来源的不合规别名,人工 / 站点 / 锁定的保留", async () => {
|
||||
// 旧版本写下的数据:各语言译名(translated)、空语言 other;以及人工 / 站点 / 锁定的同类文本
|
||||
const add = (alias: string, language: string | null, kind: string, source = "tmdb", locked = false) =>
|
||||
sql`insert into title_alias (media_entity_id, content_unit_id, alias, normalized, language, kind, source, locked)
|
||||
values (${id}, null, ${alias}, ${alias.toLowerCase()}, ${language}, ${kind}, ${source}, ${locked})`.execute(
|
||||
h.db,
|
||||
);
|
||||
await add("Атака титанов", "ru-RU", "translated");
|
||||
await add("Ataque dos Titãs", "pt-BR", "translated");
|
||||
await add("L'Attaque des Titans", "fr-FR", "translated");
|
||||
await add("Атака на титанов", null, "other");
|
||||
await add("진격의 거인", null, "other");
|
||||
await add("Ataque Titan", null, "other"); // 拉丁:按英文保留
|
||||
await add("ผ่าพิภพไททัน", "th-TH", "translated", "user"); // 人工:不动
|
||||
await add("Атака гигантов", "ru-RU", "site_title", "site"); // 站点:不动
|
||||
await add("Attaque verrouillée", "fr-FR", "translated", "tmdb", true); // 锁定:不动
|
||||
// 字段来源里的法语值(旧语言链写下的):只报告
|
||||
await sql`insert into metadata_field (media_entity_id, content_unit_id, field, value, language, source, source_revision, fetched_at)
|
||||
values (${id}, null, 'tagline', '"Le mur"'::jsonb, 'fr-FR', 'tmdb', 'x', now())`.execute(h.db);
|
||||
|
||||
const before = (await aliases()).length;
|
||||
const out = run("--dry-run");
|
||||
expect(out).toContain("DRY-RUN");
|
||||
expect(out).toMatch(/删除 5 条/);
|
||||
expect(out).toContain("ru-RU / translated:1 条");
|
||||
expect(out).toContain("(空) / other:2 条");
|
||||
expect(out).toContain("不动(非 tmdb 来源或已锁定):");
|
||||
expect(out).toContain("## 字段来源(tmdb)语言不在保留范围:1 条");
|
||||
expect(out).toContain("fr-FR / 作品·tagline:1 条");
|
||||
expect((await aliases()).length).toBe(before);
|
||||
|
||||
const applied = run("--apply");
|
||||
expect(applied).toContain("已删除 5 条别名");
|
||||
const after = await aliases();
|
||||
const texts = after.map((a) => a.alias);
|
||||
for (const gone of ["Атака титанов", "Ataque dos Titãs", "L'Attaque des Titans", "Атака на титанов", "진격의 거인"])
|
||||
expect(texts).not.toContain(gone);
|
||||
for (const kept of [
|
||||
"Ataque Titan",
|
||||
"ผ่าพิภพไททัน",
|
||||
"Атака гигантов",
|
||||
"Attaque verrouillée",
|
||||
"進撃の巨人",
|
||||
"Shingeki no Kyojin",
|
||||
])
|
||||
expect(texts).toContain(kept);
|
||||
expect(after.length).toBe(before - 5);
|
||||
const audit = await h.db
|
||||
.selectFrom("audit_event")
|
||||
.select(["payload"])
|
||||
.where("action", "=", "metadata.tmdb_alias_language_cleanup")
|
||||
.executeTakeFirstOrThrow();
|
||||
expect((audit.payload as { deleted: number }).deleted).toBe(5);
|
||||
// 再跑一遍没有要删的
|
||||
expect(run()).toContain("删除 0 条");
|
||||
});
|
||||
});
|
||||
@@ -69,7 +69,7 @@ export const LANGUAGE_CARD: CardSpec = {
|
||||
label: "语言回退链",
|
||||
kind: "text",
|
||||
placeholder: DEFAULT_LANGUAGES.join(", "),
|
||||
hint: "标题和每集信息按顺序取第一个有值的语言,简介默认只取链里的中文;中文名取链中第一个中文值。逗号分隔,如 zh-CN, zh-SG",
|
||||
hint: "标题和每集信息按顺序取第一个有值的语言,简介默认只取链里的中文;中文名取链中第一个中文值。只认简繁中文、英文和作品原语言(如日语作品的 ja-JP),其它语言跳过。逗号分隔,如 zh-CN, zh-SG",
|
||||
wide: true,
|
||||
},
|
||||
{
|
||||
|
||||
@@ -336,6 +336,7 @@ export interface TmdbPort { searchMovie(q: string, year?: number): Promise<TmdbS
|
||||
| --- | --- | --- |
|
||||
| 查询名顺序、按类型分步计划、年份容差 ±1、标题精确匹配优先、别名回退、季年份匹配、无类型 id 的双查加消歧、连接失败和不存在分开 | ① 所有 direct 或 alias 结果**只作为候选**,按证据打分输出 `candidates[]`;分数够高并且只有一个候选时才自动绑定,否则进入待识别池(REQ §7.5);② 别名回退要**限制拉详情的次数**(例如前 5 个),避免上游那样对每个候选都请求一次;③ 已经存在本地实体时,先查 `title_alias`(pg_trgm)召回本地实体,再查 TMDB;④ TMDB 缺失或失败**不阻断**有本地证据的入库(REQ §14.1.3) | 本地 UUID 优先;控制成本;REQ 要求不因外部服务阻断 |
|
||||
| 中文标题补全 | 放进元数据协调层(REQ §7.3 字段级来源),**不改变路径绑定** | REQ §7.1 |
|
||||
| 别名与译名(`__get_names` 用全部 `alternative_titles` + 全部 `translations`;`__update_tmdbinfo_extra_title` 取 en / hk / tw / sg 译名,`__update_tmdbinfo_cn_title` 取 CN 地区别名) | **多语言保留规则**(2026-10-06 用户规则,`packages/adapters/src/tmdb/language-policy.ts`):译名、别名、简介、标语只留简中(zh-CN / zh-SG / zh-Hans)、繁中(zh-TW / zh-HK / zh-MO / zh-Hant)、英文(en-*)、作品原语言;`alternative_titles` 按地区推断语言(CN / SG 简中、TW / HK / MO 繁中、英语区英文、原产地区原语言)并校验文字,推断不出或文字不符的丢弃;原产地区的罗马字别名(type 含 roma)记 `<原语言>-Latn`。识别比较(`details().aliases`、`match.ts` 别名回退、`tmdbToCandidate`、按作品搜资源的 `work-match.ts`)同样只用这些 | 用户规则:俄 / 葡 / 越 / 西 / 泰等与作品无关的语言别名只会制造误匹配和噪声;MP 的 en / hk / tw / sg / CN 取值都在保留范围内 |
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -292,6 +292,53 @@ DATABASE_URL=postgres://umt:<密码>@$PGIP:5432/umt deno run -A scripts/identity
|
||||
`POST /media-entities/{id}/refresh?full=1`,季 / 集单元会按 TMDB 重建为 confirmed。季 0(特别篇)自 DH-1 起也在 full 刷新
|
||||
范围内(`metadata.includeSpecials` 缺省开),回刷即可重建。
|
||||
|
||||
## TMDB 多语言保留规则(2026-10-06)
|
||||
|
||||
用户规则:「除了简繁、英文,还有源语言之外,不需要获取其他更多的别名」,TMDB 的多语言数据整体按这个保留。
|
||||
规则是一个纯函数模块 `packages/adapters/src/tmdb/language-policy.ts`(单测 `language-policy.test.ts`)。
|
||||
|
||||
- **保留的语言**:简中(zh-CN / zh-SG / zh-Hans,及不带地区的 zh)、繁中(zh-TW / zh-HK / zh-MO / zh-Hant)、英文(en-*)、
|
||||
作品原语言(TMDB `original_language`,按主语言比较,如 ja → ja-JP / ja 都算;原语言是中文 / 英文时不额外增加,
|
||||
原语言是中文时 zh 的其它地区也算原语言)。原名(`original_title / original_name`)总是保留。
|
||||
- **alternative_titles**(只有地区没有语言):CN / SG → 简中,TW / HK / MO → 繁中,US / GB / AU / CA / NZ / IE → 英文,
|
||||
作品 `origin_country` 里的地区 → 原语言;再校验文字:中文要汉字且不含假名 / 谚文,带地区时看简繁特有字(明显是另一种的不要),
|
||||
日文要假名或汉字,韩文要谚文,泰 / 俄 / 阿 / 希伯来 / 印地等按各自文字,英文和其它拉丁语言要拉丁字母且不含非拉丁文字。
|
||||
推断不出(没有地区、地区不在上面)或文字都不符的丢弃。例外:原产地区、type 含 roma(TMDB 的 "Romaji")的罗马字别名记
|
||||
`<原语言>-Latn`(kind `romanized`),资源标题常写罗马字。
|
||||
- **应用位置**:`TmdbClient.details()`(识别比较用的 `aliases` / `titles`,`match.ts` 别名回退、`tmdbToCandidate`、
|
||||
`work-match.ts` 按作品搜资源都只看到保留语言)、`fetchRichDetail`(`translations` / `alternativeTitles` 只留保留语言,
|
||||
别名带推断语言)、建档 `seedAliases` 与刷新 `tagTitleAliases` 写 `title_alias`(语言写规范化后的值,不再写空语言的 `other`;
|
||||
本地化名文字与语言不符时——TMDB 缺该语言译名时回退成原名——不按该语言登记)。
|
||||
- **语言链**(设置 `metadata.languages`):缺省 `zh-CN, zh-SG, zh-TW, zh-HK, en-US` 全在保留范围。设置仍允许填任意 `xx-YY`,
|
||||
但按作品收敛:不在保留范围的语言(对这部作品既不是中英文也不是原语言)从链里去掉,标题 / 简介 / 标语 / 季集名都不取它;
|
||||
请求语言(链首)被去掉时请求语言下的值不拿来兜底;链全被去掉时用缺省链。季集详情不知道原语言时(链里有中英文以外的语言)
|
||||
先读一次作品详情(走请求缓存)。
|
||||
- 作品与单元的 `metadata` 只存按语言链取到的单个值(带语言),不存 translations 列表,没有别的多语言数据。
|
||||
|
||||
### 存量清理(`scripts/tmdb-alias-language-cleanup.ts`)
|
||||
|
||||
缺省只读(`--dry-run` 显式只读,与 `--apply` 互斥);`--apply` 在一个事务里删除并写审计 `metadata.tmdb_alias_language_cleanup`
|
||||
(payload 列出每条删除的作品 / 别名 / 语言 / kind / 原因);`--entity <id>` 只处理一部作品;`--samples N` 每类样例条数(缺省 5)。
|
||||
连接方式同上(用户栈的 PG 取容器 IP):
|
||||
|
||||
```sh
|
||||
DATABASE_URL=postgres://umt:<密码>@$PGIP:5432/umt deno run -A scripts/tmdb-alias-language-cleanup.ts # 看统计与样例
|
||||
DATABASE_URL=postgres://umt:<密码>@$PGIP:5432/umt deno run -A scripts/tmdb-alias-language-cleanup.ts --apply # 逐类目过目后执行
|
||||
```
|
||||
|
||||
- **判定口径**(`title_alias.source` 能区分来源):只动 `source = tmdb` 且未锁定的行;`user` / `site` / `agent` / `nfo` / `import`
|
||||
来源与锁定的行一律不动(清单里只报数量)。tmdb 行中:`kind = original` 保留;与作品主标题 / 原标题归一化相同的保留;
|
||||
`language` 非空 → 规范化后在保留范围(原语言取 `media_entity.metadata.originalLanguage`,没有时取 tmdb 字段来源)保留,否则删
|
||||
(原因 `language`);`language` 为空(旧版建档写的 `other` 等)→ 汉字(不含假名 / 谚文)、拉丁字母、没有字母的保留,其它文字
|
||||
只在与原语言相符时保留,否则删(原因 `script`)。拉丁字母分不出英 / 法 / 西 / 葡,空语言的拉丁别名一律按英文保留——
|
||||
要彻底收敛这部分,对作品跑一次元数据刷新不会删它们(刷新只增不删),需要时人工处理。
|
||||
- **字段来源**:`metadata_field`(来源 tmdb)里语言不在保留范围的值只报告不删(删了会让最终值暂时没有来源)。对清单里的作品
|
||||
跑一次元数据刷新即收敛(同字段换语言时删旧行),批量用计划页「元数据语言回刷」(`metadata.backfill_languages`)。
|
||||
- 输出:别名总数 / 删除 / 保留,保留按原因计数;删除按「语言 / kind」分组计数,每组列样例(作品、原语言、别名);
|
||||
字段来源按「语言 / 作品或单元·字段」计数并列涉及的作品。
|
||||
- **对部署库**(lessons 第 22 条):先 dry-run,协调者逐类目看清单(重点看删除组里有没有人工补录却被记成 tmdb 来源的、
|
||||
原语言为空导致误删的日 / 韩 / 泰文别名)再 `--apply`。原语言为空的作品只保留中英文,先对这些作品刷新元数据补上原语言再清理更稳。
|
||||
|
||||
## 排查
|
||||
|
||||
```sql
|
||||
|
||||
@@ -9,6 +9,8 @@
|
||||
- 季 / 集:先按链首请求 `/tv/{id}/season/{n}`;已播出的集缺标题(TMDB 占位名"第 N 集""Episode N")或缺简介时,按链逐个语言补拉同一端点,补齐即停(简介只从简介链里的语言取;只缺简介且剩下的语言都不在简介链里时不再补拉);链走完仍没有标题的集保留占位名(语言记为空)。未播出的集不触发补拉。
|
||||
- 每个字段的来源值写 `metadata_field`,`language` 记实际取到的语言;同字段语言变了,旧语言的 tmdb 行删掉。
|
||||
- `translations` 与 `alternative_titles` 写 `title_alias`(来源 tmdb,标语言;别名只在地区与文字相符时标,如 CN + 汉字 → zh-CN)。已有同文本别名(繁简归一化后同键)不重复登记,只给语言为空的 tmdb 行补语言。
|
||||
- 多语言保留规则(2026-10-06):TMDB 的译名 / 别名 / 简介 / 标语只保留简中、繁中、英文和作品原语言,语言链按作品收敛
|
||||
(链里的 ja-JP 只对日语原作生效,fr-FR 这类对原语言不是它的作品跳过)。规则、清理脚本见 `identity.md`「TMDB 多语言保留规则」。
|
||||
|
||||
## 刷新
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
*/
|
||||
import { redactUrl } from "@umt/contracts";
|
||||
import { RequestLimiter, TtlLruCache } from "./cache.ts";
|
||||
import { allowedLanguage, altTitleLanguage, isAllowedLanguage } from "./language-policy.ts";
|
||||
|
||||
export type TmdbErrorKind = "auth" | "network" | "timeout" | "notfound" | "rate" | "protocol";
|
||||
|
||||
@@ -27,7 +28,7 @@ export type MediaEntityType = "movie" | "tv";
|
||||
|
||||
export interface CandidateTitle {
|
||||
text: string;
|
||||
/** BCP47,如 zh-CN;原名为原始语言;别名为 null */
|
||||
/** BCP47,如 zh-CN;原名为原始语言;别名为按地区 + 文字推断的语言(language-policy altTitleLanguage) */
|
||||
lang: string | null;
|
||||
kind: "localized" | "original" | "alias" | "translation";
|
||||
/** 别名/翻译的地区(iso_3166_1) */
|
||||
@@ -395,9 +396,13 @@ export class TmdbClient {
|
||||
),
|
||||
);
|
||||
const c = this.fromBrief(type, first!.j, first!.lang);
|
||||
// 多语言保留规则(language-policy.ts):本地化名 / 译名只留简中 / 繁中 / 英文 / 原语言;别名按地区 + 文字推断语言,
|
||||
// 推断不出的不要。MP __get_names 把全部别名与译名都计入可匹配名称,这里收窄(用户规则 2026-10-06)
|
||||
const work = { originalLanguage: c.originalLanguage, originCountry: c.originCountry };
|
||||
c.titles = c.titles.filter((t) => t.kind !== "localized" || isAllowedLanguage(t.lang, work));
|
||||
for (const { lang, j } of rest) {
|
||||
const t = s(type === "movie" ? j.title : j.name);
|
||||
if (t) TmdbClient.mergeTitle(c, { text: t, lang, kind: "localized" });
|
||||
if (t && isAllowedLanguage(lang, work)) TmdbClient.mergeTitle(c, { text: t, lang, kind: "localized" });
|
||||
}
|
||||
const alt = first!.j.alternative_titles as Json | undefined;
|
||||
const altList = ((type === "movie" ? alt?.titles : alt?.results) ?? []) as Json[];
|
||||
@@ -405,25 +410,24 @@ export class TmdbClient {
|
||||
for (const a of altList) {
|
||||
const text = s(a.title);
|
||||
if (!text) continue;
|
||||
aliases.add(text);
|
||||
const region = s(a.iso_3166_1);
|
||||
c.titles.push({ text, lang: null, kind: "alias", ...(region ? { region } : {}) });
|
||||
const lang = altTitleLanguage({ title: text, region, type: s(a.type) }, work);
|
||||
if (!lang) continue;
|
||||
aliases.add(text);
|
||||
c.titles.push({ text, lang, kind: "alias", ...(region ? { region } : {}) });
|
||||
}
|
||||
// translations:MP __get_names 同样把各语言译名计入可匹配名称
|
||||
// translations:MP __get_names 同样把各语言译名计入可匹配名称(这里只留保留语言)
|
||||
const trs = (((first!.j.translations as Json | undefined)?.translations ?? []) as Json[]).filter(Boolean);
|
||||
for (const tr of trs) {
|
||||
const data = tr.data as Json | undefined;
|
||||
const text = s(type === "movie" ? data?.title : data?.name);
|
||||
if (!text) continue;
|
||||
aliases.add(text);
|
||||
const l = s(tr.iso_639_1);
|
||||
const region = s(tr.iso_3166_1);
|
||||
TmdbClient.mergeTitle(c, {
|
||||
text,
|
||||
lang: l ? (region ? `${l}-${region}` : l) : null,
|
||||
kind: "translation",
|
||||
...(region ? { region } : {}),
|
||||
});
|
||||
const lang = allowedLanguage(l ? (region ? `${l}-${region}` : l) : null, work);
|
||||
if (!lang) continue;
|
||||
aliases.add(text);
|
||||
TmdbClient.mergeTitle(c, { text, lang, kind: "translation", ...(region ? { region } : {}) });
|
||||
}
|
||||
c.aliases = [...aliases];
|
||||
const ext = first!.j.external_ids as Json | undefined;
|
||||
|
||||
@@ -6,8 +6,11 @@
|
||||
* 请求缓存语义同 client.ts(MP tmdbv3api request 的 @cached:成功才缓存、空结果短 TTL)。
|
||||
* 调整:标题 / 简介按语言回退链取(translations.ts);季集在首选语言缺标题(占位名)或缺简介时按链逐个语言补拉,
|
||||
* 链走完或全部补齐即停。MD-2:简介 / 标语只按 overviewLanguages(缺省只到中文变体,不回退英文)取,取不到留空。主创只留前 10 位演员 + 导演 / 主创(created_by),头像只存路径(前端经图片代理)。
|
||||
* 多语言保留规则(language-policy.ts):translations / alternative_titles 只留简中 / 繁中 / 英文 / 原语言,
|
||||
* 语言链按作品收敛(去掉既不是中英文、也不是作品原语言的语言)。
|
||||
*/
|
||||
import type { MediaEntityType, TmdbClient } from "./client.ts";
|
||||
import { allowedLanguageChain, altTitleLanguage, isAllowedLanguage, type WorkLanguages } from "./language-policy.ts";
|
||||
import {
|
||||
DEFAULT_METADATA_LANGUAGES,
|
||||
isPlaceholderEpisodeName,
|
||||
@@ -85,7 +88,7 @@ export interface TmdbRichDetail {
|
||||
translations: TmdbTranslation[];
|
||||
alternativeTitles: TmdbAltTitle[];
|
||||
seasons: TmdbRichSeason[];
|
||||
/** 本次使用的语言链 */
|
||||
/** 本次使用的语言链(按作品收敛后) */
|
||||
languages: string[];
|
||||
}
|
||||
|
||||
@@ -126,6 +129,12 @@ export interface RichTextOptions {
|
||||
const chainOf = (languages?: readonly string[]) =>
|
||||
languages?.length ? [...languages] : [...DEFAULT_METADATA_LANGUAGES];
|
||||
|
||||
/** 语言链按作品收敛(保留规则);全被去掉时用缺省链 */
|
||||
export function workLanguageChain(languages: readonly string[] | undefined, work: WorkLanguages): string[] {
|
||||
const eff = allowedLanguageChain(chainOf(languages), work);
|
||||
return eff.length ? eff : allowedLanguageChain(DEFAULT_METADATA_LANGUAGES, work);
|
||||
}
|
||||
|
||||
function credits(raw: unknown, type: MediaEntityType, createdBy: unknown): { cast: TmdbCredit[]; crew: TmdbCredit[] } {
|
||||
const c = (raw ?? {}) as Json;
|
||||
const cast = ((Array.isArray(c.cast) ? c.cast : []) as Json[])
|
||||
@@ -165,24 +174,40 @@ export async function fetchRichDetail(
|
||||
languages?: readonly string[],
|
||||
opts: RichTextOptions = {},
|
||||
): Promise<TmdbRichDetail> {
|
||||
const chain = chainOf(languages);
|
||||
const textChain = overviewLanguages(chain, opts.overviewNonChineseFallback);
|
||||
const requested = chainOf(languages);
|
||||
const j = await client.getJson<Json>(`/${type}/${id}`, {
|
||||
language: chain[0],
|
||||
language: requested[0],
|
||||
append_to_response: "alternative_titles,credits,external_ids,translations",
|
||||
});
|
||||
const titleKey = type === "movie" ? "title" : "name";
|
||||
const translations = parseTranslations(j.translations, titleKey);
|
||||
const altTitles = parseAltTitles(j.alternative_titles);
|
||||
const originalTitle = s(type === "movie" ? j.original_title : j.original_name);
|
||||
const originalLanguage = s(j.original_language);
|
||||
const primary = s(j[titleKey]);
|
||||
const originCountry = Array.isArray(j.origin_country)
|
||||
? (j.origin_country as unknown[]).filter((x): x is string => typeof x === "string")
|
||||
: [];
|
||||
const work: WorkLanguages = { originalLanguage, originCountry };
|
||||
// 保留规则:译名只留允许语言;别名按地区 + 文字推断语言,推断不出的丢弃;语言链按作品收敛
|
||||
const translations = parseTranslations(j.translations, titleKey).filter((t) => isAllowedLanguage(t.lang, work));
|
||||
const altTitles = parseAltTitles(j.alternative_titles)
|
||||
.map((a) => ({ ...a, language: altTitleLanguage(a, work) }))
|
||||
.filter((a) => a.language);
|
||||
const chain = workLanguageChain(requested, work);
|
||||
const textChain = overviewLanguages(chain, opts.overviewNonChineseFallback);
|
||||
// 请求语言不在保留范围内时,请求语言下的值不可信(是被去掉的语言),不拿来兜底
|
||||
const reqLang = isAllowedLanguage(requested[0], work) ? requested[0]! : null;
|
||||
const primary = reqLang ? s(j[titleKey]) : null;
|
||||
const titleSrc = { translations, altTitles, originalTitle, originalLanguage, primary };
|
||||
const title = pickTitle(chain, titleSrc);
|
||||
const zh = pickChineseTitle(chain, titleSrc);
|
||||
// 请求语言 = 链首,链首总在 textChain 里(见 overviewLanguages),所以 primary 可以兜底
|
||||
const overview = pickText(textChain, translations, "overview", { value: s(j.overview), language: chain[0] ?? null });
|
||||
const tagline = pickText(textChain, translations, "tagline", { value: s(j.tagline), language: chain[0] ?? null });
|
||||
const overview = pickText(textChain, translations, "overview", {
|
||||
value: reqLang ? s(j.overview) : null,
|
||||
language: reqLang,
|
||||
});
|
||||
const tagline = pickText(textChain, translations, "tagline", {
|
||||
value: reqLang ? s(j.tagline) : null,
|
||||
language: reqLang,
|
||||
});
|
||||
const date = s(type === "movie" ? j.release_date : j.first_air_date);
|
||||
const ext = (j.external_ids ?? {}) as Json;
|
||||
const runtimes = Array.isArray(j.episode_run_time) ? (j.episode_run_time as unknown[]) : [];
|
||||
@@ -190,7 +215,7 @@ export async function fetchRichDetail(
|
||||
return {
|
||||
entityType: type,
|
||||
externalId: String(id),
|
||||
title: title?.value ?? primary ?? originalTitle ?? "",
|
||||
title: title?.value ?? primary ?? originalTitle ?? s(j[titleKey]) ?? "",
|
||||
titleLanguage: title?.language ?? null,
|
||||
titleZh: zh?.value ?? null,
|
||||
titleZhLanguage: zh?.language ?? null,
|
||||
@@ -212,9 +237,7 @@ export async function fetchRichDetail(
|
||||
networks: ((Array.isArray(j.networks) ? j.networks : []) as Json[])
|
||||
.map((x) => s(x.name))
|
||||
.filter((x): x is string => !!x),
|
||||
originCountry: Array.isArray(j.origin_country)
|
||||
? (j.origin_country as unknown[]).filter((x): x is string => typeof x === "string")
|
||||
: [],
|
||||
originCountry,
|
||||
posterPath: s(j.poster_path),
|
||||
backdropPath: s(j.backdrop_path),
|
||||
cast,
|
||||
@@ -268,9 +291,18 @@ export async function fetchRichSeason(
|
||||
tvId: string | number,
|
||||
seasonNumber: number,
|
||||
languages?: readonly string[],
|
||||
opts: { now?: Date } & RichTextOptions = {},
|
||||
opts: { now?: Date; originalLanguage?: string | null } & RichTextOptions = {},
|
||||
): Promise<TmdbRichSeasonDetail> {
|
||||
const chain = chainOf(languages);
|
||||
const requested = chainOf(languages);
|
||||
// 保留规则:链里有中英文以外的语言时要知道作品原语言才能判断;调用方没给就读一次作品详情(请求缓存)
|
||||
let originalLanguage = opts.originalLanguage;
|
||||
if (originalLanguage === undefined && allowedLanguageChain(requested, {}).length !== requested.length) {
|
||||
originalLanguage = await client
|
||||
.getJson<Json>(`/tv/${tvId}`, { language: requested[0] })
|
||||
.then((d) => s(d.original_language))
|
||||
.catch(() => null);
|
||||
}
|
||||
const chain = workLanguageChain(requested, { originalLanguage: originalLanguage ?? null });
|
||||
const first = chain[0]!;
|
||||
// MD-2:简介只从这些语言取(缺省只到中文变体);标题仍按完整链
|
||||
const textChain = overviewLanguages(chain, opts.overviewNonChineseFallback);
|
||||
|
||||
@@ -2,6 +2,7 @@ export { RequestLimiter, TtlLruCache } from "./cache.ts";
|
||||
export * from "./client.ts";
|
||||
export * from "./details.ts";
|
||||
export * from "./episode-groups.ts";
|
||||
export * from "./language-policy.ts";
|
||||
export * from "./match.ts";
|
||||
export * from "./recall.ts";
|
||||
export * from "./translations.ts";
|
||||
@@ -0,0 +1,146 @@
|
||||
import { createMockTmdb, type MockTmdb, TMDB_FIXTURES } from "@umt/testkit";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
allowedLanguage,
|
||||
allowedLanguageChain,
|
||||
altTitleLanguage,
|
||||
chineseScriptOf,
|
||||
createTmdbClient,
|
||||
fetchRichDetail,
|
||||
textMatchesLanguage,
|
||||
untaggedTextAllowed,
|
||||
} from "./index.ts";
|
||||
|
||||
const JA = { originalLanguage: "ja", originCountry: ["JP"] };
|
||||
const KO = { originalLanguage: "ko", originCountry: ["KR"] };
|
||||
const TH = { originalLanguage: "th", originCountry: ["TH"] };
|
||||
const ZH = { originalLanguage: "zh", originCountry: ["CN"] };
|
||||
const EN = { originalLanguage: "en", originCountry: ["US"] };
|
||||
|
||||
describe("多语言保留规则:语言标签", () => {
|
||||
it("简中 / 繁中 / 英文总是保留,规范化大小写", () => {
|
||||
for (const tag of ["zh-CN", "zh-SG", "zh-Hans", "zh-TW", "zh-HK", "zh-MO", "zh-Hant", "zh", "en-US", "en-GB", "en"])
|
||||
expect(allowedLanguage(tag, EN)).toBe(tag);
|
||||
expect(allowedLanguage("zh-cn", EN)).toBe("zh-CN");
|
||||
expect(allowedLanguage("zh_hant", EN)).toBe("zh-Hant");
|
||||
expect(allowedLanguage("EN-au", EN)).toBe("en-AU");
|
||||
});
|
||||
|
||||
it("其它语言只在是作品原语言时保留", () => {
|
||||
for (const tag of ["ru-RU", "pt-BR", "vi-VN", "es-ES", "th-TH", "bg-BG", "he-IL", "fr-FR", "ja-JP", "ko-KR"])
|
||||
expect(allowedLanguage(tag, EN)).toBeNull();
|
||||
expect(allowedLanguage("ja-JP", JA)).toBe("ja-JP");
|
||||
expect(allowedLanguage("ja", JA)).toBe("ja");
|
||||
expect(allowedLanguage("ko-KR", JA)).toBeNull();
|
||||
expect(allowedLanguage("ko-KR", KO)).toBe("ko-KR");
|
||||
expect(allowedLanguage("th-TH", TH)).toBe("th-TH");
|
||||
// 原语言是中文:中文的其它地区也算原语言;不额外增加别的语言
|
||||
expect(allowedLanguage("zh-MY", ZH)).toBe("zh-MY");
|
||||
expect(allowedLanguage("zh-MY", EN)).toBeNull();
|
||||
expect(allowedLanguage("ja-JP", ZH)).toBeNull();
|
||||
// 没有原语言信息:只有中英文
|
||||
expect(allowedLanguage("ja-JP", {})).toBeNull();
|
||||
expect(allowedLanguage(null, JA)).toBeNull();
|
||||
expect(allowedLanguage("", JA)).toBeNull();
|
||||
});
|
||||
|
||||
it("语言链按作品收敛:去掉不保留的语言,保持顺序", () => {
|
||||
const chain = ["ja-JP", "zh-CN", "fr-FR", "zh-TW", "en-US"];
|
||||
expect(allowedLanguageChain(chain, JA)).toEqual(chain.filter((x) => x !== "fr-FR"));
|
||||
expect(allowedLanguageChain(chain, EN)).toEqual(["zh-CN", "zh-TW", "en-US"]);
|
||||
expect(allowedLanguageChain(["fr-FR"], EN)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("多语言保留规则:文字校验", () => {
|
||||
it("简繁特有字", () => {
|
||||
expect(chineseScriptOf("人生切割术")).toBe("zh-Hans");
|
||||
expect(chineseScriptOf("人生切割術")).toBe("zh-Hant");
|
||||
expect(chineseScriptOf("沙丘")).toBeNull();
|
||||
});
|
||||
|
||||
it("各语言的文字", () => {
|
||||
expect(textMatchesLanguage("zh-CN", "奥本海默传")).toBe(true);
|
||||
expect(textMatchesLanguage("zh-CN", "奧本海默傳")).toBe(false); // 明显繁体
|
||||
expect(textMatchesLanguage("zh-TW", "最後生還者")).toBe(true);
|
||||
expect(textMatchesLanguage("zh-TW", "最后的生还者")).toBe(false);
|
||||
expect(textMatchesLanguage("zh", "最後生還者")).toBe(true); // 不带地区不看简繁
|
||||
expect(textMatchesLanguage("zh-CN", "Severance CN")).toBe(false);
|
||||
expect(textMatchesLanguage("zh-CN", "進撃の巨人")).toBe(false);
|
||||
expect(textMatchesLanguage("ja-JP", "進撃の巨人")).toBe(true);
|
||||
expect(textMatchesLanguage("ja-JP", "Shingeki no Kyojin")).toBe(false);
|
||||
expect(textMatchesLanguage("ko-KR", "오징어 게임")).toBe(true);
|
||||
expect(textMatchesLanguage("ko-KR", "Squid Game")).toBe(false);
|
||||
expect(textMatchesLanguage("th-TH", "ฉลาดเกมส์โกง")).toBe(true);
|
||||
expect(textMatchesLanguage("th-TH", "Bad Genius")).toBe(false);
|
||||
expect(textMatchesLanguage("en-US", "Bad Genius")).toBe(true);
|
||||
expect(textMatchesLanguage("en-US", "Очень странные дела")).toBe(false);
|
||||
expect(textMatchesLanguage("ru-RU", "Очень странные дела")).toBe(true);
|
||||
expect(textMatchesLanguage("ja-Latn", "Shingeki no Kyojin")).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("多语言保留规则:alternative_titles 按地区推断语言", () => {
|
||||
it("CN / SG → 简中,TW / HK / MO → 繁中;文字不符丢弃", () => {
|
||||
expect(altTitleLanguage({ title: "人生切割术", region: "CN" }, EN)).toBe("zh-CN");
|
||||
expect(altTitleLanguage({ title: "生活割離術", region: "HK" }, EN)).toBe("zh-HK");
|
||||
expect(altTitleLanguage({ title: "人生切割術", region: "CN" }, EN)).toBeNull();
|
||||
expect(altTitleLanguage({ title: "Severance CN", region: "CN" }, EN)).toBeNull();
|
||||
expect(altTitleLanguage({ title: "奧本海默", region: "TW" }, EN)).toBe("zh-TW");
|
||||
});
|
||||
|
||||
it("英语区 → 英文;其它地区只在是原产地区时 → 原语言", () => {
|
||||
expect(altTitleLanguage({ title: "Dune 2", region: "US" }, EN)).toBe("en-US");
|
||||
expect(altTitleLanguage({ title: "Attack on Titan", region: "GB" }, JA)).toBe("en-GB");
|
||||
expect(altTitleLanguage({ title: "Дюна 2", region: "RU" }, EN)).toBeNull();
|
||||
expect(altTitleLanguage({ title: "Duna: Parte Dois", region: "BR" }, EN)).toBeNull();
|
||||
expect(altTitleLanguage({ title: "進撃の巨人", region: "JP" }, JA)).toBe("ja-JP");
|
||||
expect(altTitleLanguage({ title: "進撃の巨人", region: "JP" }, EN)).toBeNull();
|
||||
expect(altTitleLanguage({ title: "오징어 게임", region: "KR" }, KO)).toBe("ko-KR");
|
||||
expect(altTitleLanguage({ title: "ฉลาดเกมส์โกง", region: "TH" }, TH)).toBe("th-TH");
|
||||
expect(altTitleLanguage({ title: "Bad Genius", region: "TH" }, TH)).toBeNull();
|
||||
expect(altTitleLanguage({ title: "流浪地球 2", region: "CN" }, ZH)).toBe("zh-CN");
|
||||
expect(altTitleLanguage({ title: "The Wandering Earth 2", region: "US" }, ZH)).toBe("en-US");
|
||||
expect(altTitleLanguage({ title: "无地区", region: null }, ZH)).toBeNull();
|
||||
});
|
||||
|
||||
it("原产地区的罗马字别名记原语言拉丁转写", () => {
|
||||
expect(altTitleLanguage({ title: "Shingeki no Kyojin", region: "JP", type: "Romaji" }, JA)).toBe("ja-Latn");
|
||||
expect(altTitleLanguage({ title: "Shingeki no Kyojin", region: "JP", type: "" }, JA)).toBeNull();
|
||||
expect(altTitleLanguage({ title: "Shingeki no Kyojin", region: "FR", type: "Romaji" }, JA)).toBeNull();
|
||||
});
|
||||
|
||||
it("没有语言标记的旧别名按文字判断", () => {
|
||||
expect(untaggedTextAllowed("人生切割术", EN)).toBe(true);
|
||||
expect(untaggedTextAllowed("Severance", EN)).toBe(true);
|
||||
expect(untaggedTextAllowed("Разделение", EN)).toBe(false);
|
||||
expect(untaggedTextAllowed("세브란스", EN)).toBe(false);
|
||||
expect(untaggedTextAllowed("세브란스", KO)).toBe(true);
|
||||
expect(untaggedTextAllowed("ฉลาดเกมส์โกง", TH)).toBe(true);
|
||||
expect(untaggedTextAllowed("2049", EN)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("多语言保留规则:TMDB 详情", () => {
|
||||
let mock: MockTmdb | null = null;
|
||||
afterEach(async () => {
|
||||
await mock?.close();
|
||||
mock = null;
|
||||
});
|
||||
|
||||
it("details() 与富详情只留保留语言的译名与别名(奥本海默的 ja-JP 译名不要)", async () => {
|
||||
mock = await createMockTmdb({ fixtures: TMDB_FIXTURES });
|
||||
const tmdb = createTmdbClient({ baseUrl: `${mock.url}/3`, apiKey: "test-tmdb-key", retries: 0, timeoutMs: 1000 });
|
||||
const d = await tmdb.details("movie", 872585);
|
||||
expect(d.aliases).not.toContain("オッペンハイマー");
|
||||
expect(d.aliases).toEqual(expect.arrayContaining(["奥本海默传", "奧本海默"]));
|
||||
expect(d.titles.find((t) => t.text === "奥本海默传")).toMatchObject({ kind: "alias", lang: "zh-CN" });
|
||||
const rich = await fetchRichDetail(tmdb, "movie", 872585, ["ja-JP", "zh-CN", "en-US"]);
|
||||
expect(rich.translations.map((t) => t.lang).sort()).toEqual(["en-US", "zh-CN"]);
|
||||
expect(rich.alternativeTitles.map((a) => a.language).sort()).toEqual(["zh-CN", "zh-TW"]);
|
||||
// 请求语言 ja-JP 不在保留范围:语言链收敛,不用 ja-JP 的值兜底
|
||||
expect(rich.languages).toEqual(["zh-CN", "en-US"]);
|
||||
expect(rich.title).toBe("奥本海默");
|
||||
expect(rich.titleLanguage).toBe("zh-CN");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,207 @@
|
||||
/**
|
||||
* TMDB 多语言保留规则(用户规则 2026-10-06):TMDB 的译名 / 别名 / 简介 / 标语只保留
|
||||
* 简体中文(zh-CN / zh-SG / zh-Hans,及不带地区的 zh)、繁体中文(zh-TW / zh-HK / zh-MO / zh-Hant)、
|
||||
* 英文(en-*)、作品原语言(original_language,如 ja / ko / th;原语言是中文 / 英文时不额外增加)。
|
||||
* 原名(original_title / original_name)总是保留,不经过这里。
|
||||
*
|
||||
* alternative_titles 只有地区(iso_3166_1)没有语言:按地区推断——CN / SG → 简中,TW / HK / MO → 繁中,
|
||||
* US / GB / AU / CA / NZ / IE → 英文,作品 origin_country 里的地区 → 原语言;再按文字校验(汉字简繁 / 假名 / 谚文 /
|
||||
* 泰文 / 西里尔 / 拉丁 …),对不上的丢弃。例外:原产地区的罗马字别名(type 含 roma,如 TMDB 的 "Romaji")记为
|
||||
* 原语言的拉丁转写 `<原语言>-Latn`,资源标题常用罗马字。
|
||||
* MP 对照:tmdbapi.__get_names 把全部 alternative_titles 与全部 translations 都当可匹配名称;
|
||||
* __update_tmdbinfo_extra_title 只取 en / hk / tw / sg 译名,__update_tmdbinfo_cn_title 取 CN 地区别名。
|
||||
*/
|
||||
|
||||
export interface WorkLanguages {
|
||||
/** TMDB original_language(ISO 639-1,如 ja) */
|
||||
originalLanguage?: string | null;
|
||||
/** TMDB origin_country(ISO 3166-1,如 ["JP"]) */
|
||||
originCountry?: readonly string[] | null;
|
||||
}
|
||||
|
||||
const HANS_REGIONS = new Set(["CN", "SG"]);
|
||||
const HANT_REGIONS = new Set(["TW", "HK", "MO"]);
|
||||
/** 英语区:别名按地区推断为英文 */
|
||||
export const ENGLISH_REGIONS: ReadonlySet<string> = new Set(["US", "GB", "AU", "CA", "NZ", "IE"]);
|
||||
|
||||
interface Tag {
|
||||
lang: string;
|
||||
script: string | null;
|
||||
region: string | null;
|
||||
}
|
||||
|
||||
function parseTag(tag: string | null | undefined): Tag | null {
|
||||
if (!tag) return null;
|
||||
const parts = tag.trim().split(/[-_]/).filter(Boolean);
|
||||
const lang = parts[0]?.toLowerCase();
|
||||
if (!lang || !/^[a-z]{2,3}$/.test(lang)) return null;
|
||||
let script: string | null = null;
|
||||
let region: string | null = null;
|
||||
for (const p of parts.slice(1)) {
|
||||
if (/^[A-Za-z]{4}$/.test(p)) script = p[0]!.toUpperCase() + p.slice(1).toLowerCase();
|
||||
else if (/^([A-Za-z]{2}|\d{3})$/.test(p)) region = p.toUpperCase();
|
||||
}
|
||||
return { lang, script, region };
|
||||
}
|
||||
|
||||
const formatTag = (t: Tag) => [t.lang, t.script, t.region].filter(Boolean).join("-");
|
||||
const primaryOf = (lang: string | null | undefined) => parseTag(lang)?.lang ?? null;
|
||||
|
||||
/** 中文标签的简繁:zh-Hans / zh-CN / zh-SG / zh → 简;zh-Hant / zh-TW / zh-HK / zh-MO → 繁;其它地区 null */
|
||||
export function chineseVariant(tag: string | null | undefined): "zh-Hans" | "zh-Hant" | null {
|
||||
const t = parseTag(tag);
|
||||
if (t?.lang !== "zh") return null;
|
||||
if (t.script === "Hans") return "zh-Hans";
|
||||
if (t.script === "Hant") return "zh-Hant";
|
||||
if (!t.region || HANS_REGIONS.has(t.region)) return "zh-Hans";
|
||||
if (HANT_REGIONS.has(t.region)) return "zh-Hant";
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 语言标签是否在保留范围内;在范围内返回规范化后的标签(zh-cn → zh-CN,zh_hant → zh-Hant),否则 null。
|
||||
* 原语言按主语言比较(作品原语言 ja 时 ja-JP / ja 都保留)。
|
||||
*/
|
||||
export function allowedLanguage(tag: string | null | undefined, work: WorkLanguages = {}): string | null {
|
||||
const t = parseTag(tag);
|
||||
if (!t) return null;
|
||||
const canonical = formatTag(t);
|
||||
if (t.lang === "en") return canonical;
|
||||
const orig = primaryOf(work.originalLanguage);
|
||||
if (orig && t.lang === orig) return canonical;
|
||||
if (t.lang === "zh") return chineseVariant(canonical) ? canonical : null;
|
||||
return null;
|
||||
}
|
||||
|
||||
export const isAllowedLanguage = (tag: string | null | undefined, work: WorkLanguages = {}): boolean =>
|
||||
allowedLanguage(tag, work) !== null;
|
||||
|
||||
/** 语言链按作品收敛:去掉不在保留范围内的语言(保持顺序、去重);全被去掉时返回空数组 */
|
||||
export function allowedLanguageChain(chain: readonly string[], work: WorkLanguages = {}): string[] {
|
||||
const out: string[] = [];
|
||||
for (const tag of chain) {
|
||||
const a = allowedLanguage(tag, work);
|
||||
if (a && !out.includes(a)) out.push(a);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ── 文字校验 ──
|
||||
|
||||
const HAN = /\p{Script=Han}/u;
|
||||
const KANA = /[\p{Script=Hiragana}\p{Script=Katakana}]/u;
|
||||
const HANGUL = /\p{Script=Hangul}/u;
|
||||
const LATIN = /\p{Script=Latin}/u;
|
||||
/** 非拉丁的"强"文字:拉丁语言的标题里不该出现 */
|
||||
const NON_LATIN =
|
||||
/[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}\p{Script=Thai}\p{Script=Cyrillic}\p{Script=Arabic}\p{Script=Hebrew}\p{Script=Devanagari}\p{Script=Greek}]/u;
|
||||
|
||||
const SCRIPT_OF: Record<string, RegExp> = {
|
||||
th: /\p{Script=Thai}/u,
|
||||
ru: /\p{Script=Cyrillic}/u,
|
||||
uk: /\p{Script=Cyrillic}/u,
|
||||
bg: /\p{Script=Cyrillic}/u,
|
||||
sr: /\p{Script=Cyrillic}/u,
|
||||
be: /\p{Script=Cyrillic}/u,
|
||||
kk: /\p{Script=Cyrillic}/u,
|
||||
mk: /\p{Script=Cyrillic}/u,
|
||||
mn: /\p{Script=Cyrillic}/u,
|
||||
ar: /\p{Script=Arabic}/u,
|
||||
fa: /\p{Script=Arabic}/u,
|
||||
ur: /\p{Script=Arabic}/u,
|
||||
he: /\p{Script=Hebrew}/u,
|
||||
hi: /\p{Script=Devanagari}/u,
|
||||
mr: /\p{Script=Devanagari}/u,
|
||||
ne: /\p{Script=Devanagari}/u,
|
||||
bn: /\p{Script=Bengali}/u,
|
||||
ta: /\p{Script=Tamil}/u,
|
||||
te: /\p{Script=Telugu}/u,
|
||||
ml: /\p{Script=Malayalam}/u,
|
||||
kn: /\p{Script=Kannada}/u,
|
||||
el: /\p{Script=Greek}/u,
|
||||
ka: /\p{Script=Georgian}/u,
|
||||
hy: /\p{Script=Armenian}/u,
|
||||
km: /\p{Script=Khmer}/u,
|
||||
lo: /\p{Script=Lao}/u,
|
||||
my: /\p{Script=Myanmar}/u,
|
||||
};
|
||||
|
||||
// 简繁不同形的常用字(影视标题常见),各一份;只用于"明显是另一种"的判断,两边都没有时不否决
|
||||
const TRAD_CHARS =
|
||||
"們這個說時會來對後與麼還沒讓從無過開關問題點學實現發見頭聽話體愛應經義東車長門國電腦網買賣歡寫讀樣種邊戰爭傳龍鳳驚險夢戀劍俠島鬥獸獵殺將軍漢華歲親媽飛機進錄離雙號達偵線紅綠藍黃銀鐵鋼風雲龜馬魚鳥雞貓獅書畫樂聲劇場戲錢貴慶萬歷憶夠壞隊員陽陰靈屍謎諜襲獄惡復紀傑紳蘭麗嬌婦兒孫師醫術獨勝敗強鎮縣區莊園間塵煙燈熱氣溫聖誕節單狀態舊頁顏預選樓廳寶滅絕斷續終結細織純給約級總絲鄉歸處偉劉張陳楊趙吳鄭謝許蘇盧蔣韓鄧馮譚齊顧錦繡濤潔漁灣臺審計記憶護衛權勢語誤談識詩誰變奪奮緣獻歐擊鏡響頂順";
|
||||
const SIMP_CHARS =
|
||||
"们这个说时会来对与么还没让从无过开关问题点学实现发见头听话体爱应经义东车长门国电脑网买卖欢写读样种边战争传龙凤惊险梦恋剑侠岛兽猎杀将军汉华岁亲妈飞机进录离双号达侦线红绿蓝黄银铁钢风龟马鱼鸟鸡猫狮书画乐声剧场戏钱贵庆万历忆够坏队员阳阴灵尸谜谍袭狱恶复纪杰绅兰丽娇妇儿孙师医术独胜败强镇县区庄园间尘烟灯热气温圣诞节单状态旧页颜预选楼厅宝灭绝断续终结细织纯给约级总丝乡归处伟刘张陈杨赵吴郑谢许苏卢蒋韩邓冯谭齐顾锦绣涛洁渔湾审计记忆护卫权势语误谈识诗谁变夺奋缘献欧击镜响顶顺";
|
||||
const TRAD = new Set([...TRAD_CHARS].filter((c) => !SIMP_CHARS.includes(c)));
|
||||
const SIMP = new Set([...SIMP_CHARS].filter((c) => !TRAD_CHARS.includes(c)));
|
||||
|
||||
/** 汉字文本的简繁倾向:只有繁体特有字 → hant;只有简体特有字 → hans;都有或都没有 → null */
|
||||
export function chineseScriptOf(text: string): "zh-Hans" | "zh-Hant" | null {
|
||||
let t = 0;
|
||||
let s = 0;
|
||||
for (const c of text) {
|
||||
if (TRAD.has(c)) t++;
|
||||
else if (SIMP.has(c)) s++;
|
||||
}
|
||||
if (t && !s) return "zh-Hant";
|
||||
if (s && !t) return "zh-Hans";
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 标题文字与语言标签是否相符:zh 要求汉字且不含假名 / 谚文(带地区或 Hans / Hant 时再看简繁特有字,明显是另一种的不符);
|
||||
* ja 要求假名或汉字;ko 要求谚文;泰 / 俄 / 阿 / 希伯来 / 印地等按各自文字;`-Latn` 与其余语言按拉丁字母且不含非拉丁强文字。
|
||||
*/
|
||||
export function textMatchesLanguage(tag: string, text: string): boolean {
|
||||
const t = parseTag(tag);
|
||||
if (!t) return false;
|
||||
if (t.script === "Latn") return LATIN.test(text) && !NON_LATIN.test(text);
|
||||
if (t.lang === "zh") {
|
||||
if (!HAN.test(text) || KANA.test(text) || HANGUL.test(text)) return false;
|
||||
const want = chineseVariant(tag);
|
||||
const got = chineseScriptOf(text);
|
||||
return !want || !got || want === got || !(t.region || t.script);
|
||||
}
|
||||
if (t.lang === "ja") return (KANA.test(text) || HAN.test(text)) && !HANGUL.test(text);
|
||||
if (t.lang === "ko") return HANGUL.test(text);
|
||||
const re = SCRIPT_OF[t.lang];
|
||||
if (re) return re.test(text);
|
||||
return LATIN.test(text) && !NON_LATIN.test(text);
|
||||
}
|
||||
|
||||
/**
|
||||
* alternative_titles 的一条(只有地区)推断语言:地区候选依次为 简中(CN / SG)/ 繁中(TW / HK / MO)/ 英文(英语区)/
|
||||
* 原语言(作品原产地区);取第一个文字相符的。原产地区的罗马字别名(type 含 roma)记 `<原语言>-Latn`。
|
||||
* 推断不出或文字都不符返回 null(该别名不保留)。
|
||||
*/
|
||||
export function altTitleLanguage(
|
||||
alt: { title: string; region: string | null; type?: string | null },
|
||||
work: WorkLanguages = {},
|
||||
): string | null {
|
||||
const region = alt.region?.toUpperCase() ?? null;
|
||||
if (!region) return null;
|
||||
const orig = primaryOf(work.originalLanguage);
|
||||
const home = (work.originCountry ?? []).some((c) => c.toUpperCase() === region);
|
||||
const tries: string[] = [];
|
||||
if (HANS_REGIONS.has(region) || HANT_REGIONS.has(region)) tries.push(`zh-${region}`);
|
||||
if (ENGLISH_REGIONS.has(region)) tries.push(`en-${region}`);
|
||||
if (home && orig) tries.push(`${orig}-${region}`);
|
||||
for (const tag of tries) {
|
||||
const a = allowedLanguage(tag, work);
|
||||
if (a && textMatchesLanguage(a, alt.title)) return a;
|
||||
}
|
||||
if (home && orig && orig !== "en" && /roma/i.test(alt.type ?? "") && textMatchesLanguage("und-Latn", alt.title))
|
||||
return `${orig}-Latn`;
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 没有语言标记的已存别名(旧数据)按文字判断是否可能属于保留语言:汉字(不含假名 / 谚文)→ 中文;拉丁 → 英文(无法区分
|
||||
* 法 / 西 / 葡等拉丁语言,按英文保留);其余文字只在与原语言相符时保留。没有字母的(纯数字 / 符号)保留。
|
||||
*/
|
||||
export function untaggedTextAllowed(text: string, work: WorkLanguages = {}): boolean {
|
||||
if (!/\p{L}/u.test(text)) return true;
|
||||
if (textMatchesLanguage("zh", text)) return true;
|
||||
if (LATIN.test(text) && !NON_LATIN.test(text)) return true;
|
||||
const orig = primaryOf(work.originalLanguage);
|
||||
return !!orig && orig !== "zh" && orig !== "en" && textMatchesLanguage(orig, text);
|
||||
}
|
||||
@@ -25,6 +25,8 @@ export interface TmdbAltTitle {
|
||||
title: string;
|
||||
region: string | null;
|
||||
type: string | null;
|
||||
/** 按地区 + 文字推断的语言(language-policy altTitleLanguage);fetchRichDetail 只留推断得出的 */
|
||||
language?: string | null;
|
||||
}
|
||||
|
||||
/** 取到的值与其来源语言;via 说明来自哪里(给元数据来源页和测试看) */
|
||||
|
||||
@@ -256,7 +256,9 @@ export const MetadataSettings = z.object({
|
||||
tmdbBaseUrl: z.url().nullable(),
|
||||
/**
|
||||
* 元数据语言回退链(MD-1):作品 / 季 / 集的标题与简介按顺序取第一个非空值;中文名(titleZh)取链中第一个中文值。
|
||||
* BCP47 语言-地区,如 zh-CN;缺省 zh-CN → zh-SG → zh-TW → zh-HK → en-US
|
||||
* BCP47 语言-地区,如 zh-CN;缺省 zh-CN → zh-SG → zh-TW → zh-HK → en-US。
|
||||
* 多语言保留规则(adapters tmdb/language-policy.ts):只有简中(zh-CN / zh-SG)、繁中(zh-TW / zh-HK / zh-MO)、
|
||||
* 英文(en-*)和作品原语言生效;其它语言(如 ja-JP)只在作品原语言是它时生效,否则按作品跳过
|
||||
*/
|
||||
languages: z
|
||||
.array(z.string().regex(/^[a-z]{2,3}(-[A-Z]{2})?$/))
|
||||
|
||||
@@ -0,0 +1,187 @@
|
||||
// TMDB 多语言别名清理(用户规则 2026-10-06):TMDB 的多语言数据只保留简中 / 繁中 / 英文 / 作品原语言。
|
||||
// 规则见 packages/adapters/src/tmdb/language-policy.ts、docs/runbook/identity.md「TMDB 多语言保留规则」。
|
||||
// 默认只读(dry-run),加 --apply 才删(单个事务,写一条审计 metadata.tmdb_alias_language_cleanup)。
|
||||
//
|
||||
// 用法(DATABASE_URL 必须显式给出,脚本不带默认库):
|
||||
// DATABASE_URL=postgres://umt:***@<host>:5432/umt deno run -A scripts/tmdb-alias-language-cleanup.ts # 看统计与样例
|
||||
// DATABASE_URL=… deno run -A scripts/tmdb-alias-language-cleanup.ts --apply # 删除
|
||||
// DATABASE_URL=… deno run -A scripts/tmdb-alias-language-cleanup.ts --entity <作品 id> # 只看 / 只处理一部作品
|
||||
// DATABASE_URL=… deno run -A scripts/tmdb-alias-language-cleanup.ts --samples 20 # 每类样例条数(缺省 5)
|
||||
//
|
||||
// 别名(title_alias)判定——只动 source = 'tmdb' 且未锁定的行;人工(user)/ 站点(site)/ 智能体(agent)/ NFO / 导入来源与
|
||||
// 锁定的行一律不动:
|
||||
// - kind = original(原名)保留;与作品主标题 / 原标题归一化相同的保留(作品自己的名字);
|
||||
// - language 非空:规范化后是简中(zh-CN / zh-SG / zh-Hans / zh)、繁中(zh-TW / zh-HK / zh-MO / zh-Hant)、英文(en-*)、
|
||||
// 作品原语言(media_entity.metadata.originalLanguage,没有时取 TMDB 字段来源)→ 保留,否则删除(原因 language);
|
||||
// - language 为空(旧版建档写的 other 等):按文字判断——汉字(不含假名 / 谚文)、拉丁字母、没有字母的保留;
|
||||
// 其它文字(西里尔 / 阿拉伯 / 希伯来 / 泰文 / 谚文 / 假名 …)只在与作品原语言相符时保留,否则删除(原因 script)。
|
||||
// 拉丁字母无法区分英 / 法 / 西 / 葡,一律按英文保留。
|
||||
// 字段来源(metadata_field,来源 tmdb)里语言不在保留范围的标题 / 简介 / 集名只报告不删:删了会让最终值暂时没有来源,
|
||||
// 对这些作品跑一次元数据刷新(语言链已按作品收敛)即被同字段的保留语言值替换(同字段换语言时删旧行)。
|
||||
// 作品与单元的 metadata 只存按语言链取到的单个值,不存 translations 列表,没有别的多语言数据要收敛。
|
||||
|
||||
import { allowedLanguage, untaggedTextAllowed } from "@umt/adapters";
|
||||
import { normalizeTitle } from "@umt/domain";
|
||||
import pg from "pg";
|
||||
|
||||
const argv = [...Deno.args];
|
||||
const APPLY = argv.includes("--apply");
|
||||
if (APPLY && argv.includes("--dry-run")) {
|
||||
console.error("--apply 与 --dry-run 不能同时给出");
|
||||
Deno.exit(2);
|
||||
}
|
||||
const argOf = (k: string) => {
|
||||
const i = argv.indexOf(k);
|
||||
return i >= 0 ? (argv[i + 1] ?? null) : null;
|
||||
};
|
||||
const ONLY_ENTITY = argOf("--entity");
|
||||
const SAMPLES = Number(argOf("--samples") ?? 5) || 5;
|
||||
const url = Deno.env.get("DATABASE_URL");
|
||||
if (!url) {
|
||||
console.error("需要 DATABASE_URL(脚本不带默认库)");
|
||||
Deno.exit(2);
|
||||
}
|
||||
const db = new pg.Client({ connectionString: url });
|
||||
await db.connect();
|
||||
const q = async <T>(text: string, values: unknown[] = []) => (await db.query(text, values)).rows as T[];
|
||||
console.log(
|
||||
`# tmdb-alias-language-cleanup (${APPLY ? "APPLY" : "DRY-RUN"})${ONLY_ENTITY ? ` --entity ${ONLY_ENTITY}` : ""}\n`,
|
||||
);
|
||||
|
||||
interface Row {
|
||||
id: string;
|
||||
media_entity_id: string;
|
||||
alias: string;
|
||||
normalized: string;
|
||||
language: string | null;
|
||||
kind: string;
|
||||
title: string;
|
||||
original_title: string | null;
|
||||
orig_lang: string | null;
|
||||
}
|
||||
|
||||
// 作品原语言:投影值优先,没有时取 tmdb 字段来源
|
||||
const ORIG_LANG = `coalesce(e.metadata->'originalLanguage'->>'value',
|
||||
(select f.value#>>'{}' from metadata_field f where f.media_entity_id = e.id and f.content_unit_id is null
|
||||
and f.field = 'originalLanguage' and f.source = 'tmdb' order by f.fetched_at desc limit 1))`;
|
||||
|
||||
const rows = await q<Row>(
|
||||
`select a.id, a.media_entity_id, a.alias, a.normalized, a.language, a.kind::text as kind,
|
||||
e.title, e.original_title, ${ORIG_LANG} as orig_lang
|
||||
from title_alias a join media_entity e on e.id = a.media_entity_id
|
||||
where a.source = 'tmdb' and not a.locked and ($1::uuid is null or a.media_entity_id = $1::uuid)
|
||||
order by e.title, a.alias`,
|
||||
[ONLY_ENTITY],
|
||||
);
|
||||
const other = await q<{ source: string; locked: boolean; n: number }>(
|
||||
`select source::text as source, locked, count(*)::int as n from title_alias
|
||||
where (source <> 'tmdb' or locked) and ($1::uuid is null or media_entity_id = $1::uuid)
|
||||
group by 1, 2 order by 1, 2`,
|
||||
[ONLY_ENTITY],
|
||||
);
|
||||
|
||||
type Verdict = { del: false; why: string } | { del: true; why: "language" | "script" };
|
||||
function judge(r: Row): Verdict {
|
||||
if (r.kind === "original") return { del: false, why: "原名" };
|
||||
const own = [r.title, r.original_title].filter((x): x is string => !!x).map(normalizeTitle);
|
||||
if (own.includes(r.normalized)) return { del: false, why: "作品自己的名字" };
|
||||
const work = { originalLanguage: r.orig_lang };
|
||||
if (r.language !== null)
|
||||
return allowedLanguage(r.language, work) ? { del: false, why: "保留语言" } : { del: true, why: "language" };
|
||||
return untaggedTextAllowed(r.alias, work) ? { del: false, why: "无语言·文字相符" } : { del: true, why: "script" };
|
||||
}
|
||||
|
||||
const deletes: (Row & { why: string })[] = [];
|
||||
const keepBy = new Map<string, number>();
|
||||
for (const r of rows) {
|
||||
const v = judge(r);
|
||||
if (v.del) deletes.push({ ...r, why: v.why });
|
||||
else keepBy.set(v.why, (keepBy.get(v.why) ?? 0) + 1);
|
||||
}
|
||||
|
||||
console.log(
|
||||
`## 别名(来源 tmdb、未锁定):${rows.length} 条,删除 ${deletes.length} 条,保留 ${rows.length - deletes.length} 条`,
|
||||
);
|
||||
for (const [why, n] of [...keepBy].sort((a, b) => b[1] - a[1])) console.log(` 保留·${why}:${n} 条`);
|
||||
console.log(
|
||||
` 不动(非 tmdb 来源或已锁定):${other.map((o) => `${o.source}${o.locked ? "·锁定" : ""} ${o.n}`).join(",") || "0"}`,
|
||||
);
|
||||
const groups = new Map<string, (Row & { why: string })[]>();
|
||||
for (const d of deletes) {
|
||||
const k = `${d.language ?? "(空)"} / ${d.kind}`;
|
||||
groups.set(k, [...(groups.get(k) ?? []), d]);
|
||||
}
|
||||
if (deletes.length) console.log(`\n### 删除:按语言 / kind`);
|
||||
for (const [k, list] of [...groups].sort((a, b) => b[1].length - a[1].length)) {
|
||||
console.log(` ${k}:${list.length} 条`);
|
||||
for (const d of list.slice(0, SAMPLES))
|
||||
console.log(` - 《${d.title}》(原语言 ${d.orig_lang ?? "未知"}):${d.alias}`);
|
||||
}
|
||||
|
||||
// 字段来源里语言不在保留范围的(只报告)
|
||||
const fields = await q<{
|
||||
entity_id: string;
|
||||
title: string;
|
||||
orig_lang: string | null;
|
||||
field: string;
|
||||
language: string;
|
||||
unit: boolean;
|
||||
}>(
|
||||
`select e.id as entity_id, e.title, ${ORIG_LANG} as orig_lang, f.field, f.language, f.content_unit_id is not null as unit
|
||||
from metadata_field f
|
||||
left join content_unit u on u.id = f.content_unit_id
|
||||
join media_entity e on e.id = coalesce(f.media_entity_id, u.media_entity_id)
|
||||
where f.source = 'tmdb' and f.language is not null and ($1::uuid is null or e.id = $1::uuid)`,
|
||||
[ONLY_ENTITY],
|
||||
);
|
||||
const badFields = fields.filter((f) => !allowedLanguage(f.language, { originalLanguage: f.orig_lang }));
|
||||
console.log(`\n## 字段来源(tmdb)语言不在保留范围:${badFields.length} 条(只报告,对涉及的作品跑元数据刷新即收敛)`);
|
||||
const byLang = new Map<string, number>();
|
||||
for (const f of badFields)
|
||||
byLang.set(
|
||||
`${f.language} / ${f.unit ? "单元" : "作品"}·${f.field}`,
|
||||
(byLang.get(`${f.language} / ${f.unit ? "单元" : "作品"}·${f.field}`) ?? 0) + 1,
|
||||
);
|
||||
for (const [k, n] of [...byLang].sort((a, b) => b[1] - a[1])) console.log(` ${k}:${n} 条`);
|
||||
const badEntities = [...new Map(badFields.map((f) => [f.entity_id, f.title])).entries()];
|
||||
for (const [id, title] of badEntities.slice(0, SAMPLES * 4)) console.log(` - 《${title}》 ${id}`);
|
||||
|
||||
if (!APPLY) {
|
||||
console.log(`\n(dry-run,未改库;确认后加 --apply)`);
|
||||
} else if (deletes.length) {
|
||||
try {
|
||||
await db.query("begin");
|
||||
const ids = deletes.map((d) => d.id);
|
||||
const res = await db.query(
|
||||
`delete from title_alias where id = any($1::uuid[]) and source = 'tmdb' and not locked`,
|
||||
[ids],
|
||||
);
|
||||
await db.query(
|
||||
`insert into audit_event (actor_type, actor_id, action, target_type, target_id, payload)
|
||||
values ('system', 'tmdb-alias-language-cleanup', 'metadata.tmdb_alias_language_cleanup', 'media_entity', $1, $2::jsonb)`,
|
||||
[
|
||||
ONLY_ENTITY,
|
||||
JSON.stringify({
|
||||
deleted: res.rowCount,
|
||||
aliases: deletes.map((d) => ({
|
||||
entityId: d.media_entity_id,
|
||||
alias: d.alias,
|
||||
language: d.language,
|
||||
kind: d.kind,
|
||||
reason: d.why,
|
||||
})),
|
||||
}),
|
||||
],
|
||||
);
|
||||
await db.query("commit");
|
||||
console.log(`\n已删除 ${res.rowCount} 条别名`);
|
||||
} catch (e) {
|
||||
await db.query("rollback");
|
||||
console.error("执行失败,已回滚:", (e as Error).message);
|
||||
await db.end();
|
||||
Deno.exit(1);
|
||||
}
|
||||
} else {
|
||||
console.log(`\n没有要删除的别名`);
|
||||
}
|
||||
await db.end();
|
||||
Reference in new issue
Block a user