feat(scripts): identity-reevaluate with --rollback-wrong; docs for ID-2
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
03de030371
commit
42e08b30dc
4 files changed
+378
-1
No files matched your search
@@ -0,0 +1,236 @@
|
||||
// 身份识别重评(ID-2):匹配规则更新后,处理存量的待识别事项与错误映射。默认只读(dry-run),加 --apply 才改库。
|
||||
//
|
||||
// 用法(DATABASE_URL 必须显式给出,脚本不带默认库):
|
||||
// DATABASE_URL=postgres://umt:***@<host>:5432/umt deno run -A scripts/identity-reevaluate.ts
|
||||
// DATABASE_URL=… deno run -A scripts/identity-reevaluate.ts --rollback-wrong # 另查分类词别名造成的错误映射
|
||||
// DATABASE_URL=… deno run -A scripts/identity-reevaluate.ts --rollback-wrong --apply # 真正执行
|
||||
//
|
||||
// 1. open 的 identity_ambiguous:按新解析规则重新解析每个成员资源并给出预估(只用池里存的候选,不查 TMDB);
|
||||
// --apply 时把这些池条目的 next_review_at 置为现在、attempts 清零,由运行中的服务(已部署新规则)的池复查
|
||||
// 重新识别——能自动确认的会绑定作品、建映射,并从合并事项里摘除(组空即关闭)。脚本自己不查 TMDB、不建作品。
|
||||
// 2. --rollback-wrong:找出站点来源(source=site)且是分类词 / 停用词 / 单字的别名(例如 RSS 前缀「[电视剧]」被当成
|
||||
// 中文名登记的「电视剧」),对挂着这类别名的作品,用新解析 + 作品自身名称(主标题 / 原标题 / 非站点来源别名)
|
||||
// 重新核对已绑定的资源:对不上的视为经错误别名误并入 → --apply 时解除绑定、退回待识别池(立即复查),
|
||||
// 删除该错误别名以及由误并入资源登记的站点别名(链式扩散的那些),各记一条 resolution_attempt(manual)。
|
||||
// 内容单元(季 / 集)不删,只列出已无资源引用的单元。
|
||||
|
||||
import {
|
||||
type IdentityCandidate,
|
||||
isGenericAliasName,
|
||||
normalizeTitle,
|
||||
parseRelease,
|
||||
queryTitles,
|
||||
resolveIdentity,
|
||||
scoreCandidate,
|
||||
} from "@umt/domain";
|
||||
import pg from "pg";
|
||||
|
||||
const args = new Set(Deno.args);
|
||||
const APPLY = args.has("--apply");
|
||||
const ROLLBACK = args.has("--rollback-wrong");
|
||||
const url = Deno.env.get("DATABASE_URL");
|
||||
if (!url) {
|
||||
console.error("需要 DATABASE_URL(脚本不带默认库)");
|
||||
Deno.exit(2);
|
||||
}
|
||||
const db = new pg.Client({ connectionString: url });
|
||||
await db.connect();
|
||||
const q = async <T>(text: string, values: unknown[] = []) => (await db.query(text, values)).rows as T[];
|
||||
const mode = APPLY ? "APPLY" : "DRY-RUN";
|
||||
console.log(`# identity-reevaluate (${mode})${ROLLBACK ? " + rollback-wrong" : ""}\n`);
|
||||
|
||||
interface PoolRow {
|
||||
pool_id: string;
|
||||
decision_id: string;
|
||||
decision_title: string;
|
||||
source_release_id: string;
|
||||
title: string;
|
||||
subtitle: string | null;
|
||||
candidates: {
|
||||
provider: string;
|
||||
entityType: "movie" | "tv";
|
||||
externalId: string;
|
||||
title: string;
|
||||
year: number | null;
|
||||
}[];
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- 1. open identity_ambiguous
|
||||
const pool = await q<PoolRow>(
|
||||
`select c.id as pool_id, d.id as decision_id, d.title as decision_title, r.id as source_release_id, r.title, r.subtitle,
|
||||
c.candidates
|
||||
from decision d
|
||||
join identity_candidate c on c.decision_id = d.id and c.status = 'pending'
|
||||
join source_release r on r.id = c.source_release_id
|
||||
where d.kind = 'identity_ambiguous' and d.status in ('open', 'investigating', 'awaiting_user')
|
||||
order by d.id, r.title`,
|
||||
);
|
||||
console.log(
|
||||
`## open identity_ambiguous:${new Set(pool.map((p) => p.decision_id)).size} 条事项,${pool.length} 个成员资源`,
|
||||
);
|
||||
let predictedAuto = 0;
|
||||
for (const p of pool) {
|
||||
const parsed = parseRelease(p.title, p.subtitle ? { subtitle: p.subtitle } : {});
|
||||
const cands: IdentityCandidate[] = (p.candidates ?? []).map((c) => ({
|
||||
provider: c.provider,
|
||||
entityType: c.entityType,
|
||||
externalId: c.externalId,
|
||||
titles: [c.title],
|
||||
aliases: [],
|
||||
...(c.year != null ? { year: c.year } : {}),
|
||||
}));
|
||||
const res = resolveIdentity(parsed, cands);
|
||||
if (res.status === "auto") predictedAuto++;
|
||||
const best = res.best?.candidate;
|
||||
console.log(
|
||||
`- [${res.status}${res.status === "auto" ? ` → ${best?.provider}:${best?.entityType}/${best?.externalId} ${best?.titles[0]}` : ""}] ` +
|
||||
`${p.title} (中文名 ${parsed.titleCn ?? "-"},英文名 ${parsed.titleEn ?? "-"},年份 ${parsed.year ?? "-"})`,
|
||||
);
|
||||
}
|
||||
console.log(
|
||||
`\n预估(只用池里存的候选,服务复查时会按新规则重新查 TMDB 并比对别名,结果以复查为准):${predictedAuto}/${pool.length} 可自动确认`,
|
||||
);
|
||||
if (APPLY && pool.length) {
|
||||
await q(`update identity_candidate set next_review_at = now(), attempts = 0 where id = any($1::uuid[])`, [
|
||||
pool.map((p) => p.pool_id),
|
||||
]);
|
||||
console.log(`已把 ${pool.length} 个池条目排入立即复查(由运行中的服务处理)`);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- 2. rollback-wrong
|
||||
if (ROLLBACK) {
|
||||
interface AliasRow {
|
||||
id: string;
|
||||
media_entity_id: string;
|
||||
alias: string;
|
||||
normalized: string;
|
||||
source: string;
|
||||
}
|
||||
const siteAliases = await q<AliasRow>(
|
||||
`select id, media_entity_id, alias, normalized, source from title_alias where source = 'site' and content_unit_id is null`,
|
||||
);
|
||||
const poison = siteAliases.filter((a) => isGenericAliasName(a.normalized));
|
||||
const entities = [...new Set(poison.map((a) => a.media_entity_id))];
|
||||
console.log(`\n## rollback-wrong:${poison.length} 个分类词 / 通用词站点别名,涉及 ${entities.length} 部作品`);
|
||||
const toUnbind: { id: string; title: string; subtitle: string | null; entityId: string }[] = [];
|
||||
const aliasIdsToDelete = new Set(poison.map((a) => a.id));
|
||||
for (const entityId of entities) {
|
||||
const [e] = await q<{
|
||||
id: string;
|
||||
kind: string;
|
||||
title: string;
|
||||
original_title: string | null;
|
||||
year: number | null;
|
||||
}>(`select id, kind, title, original_title, year from media_entity where id = $1`, [entityId]);
|
||||
if (!e) continue;
|
||||
const all = await q<AliasRow>(
|
||||
`select id, media_entity_id, alias, normalized, source from title_alias where media_entity_id = $1`,
|
||||
[entityId],
|
||||
);
|
||||
// 作品自身名称:主标题 / 原标题 / 非站点来源的别名(TMDB 译名等)
|
||||
const legit: IdentityCandidate = {
|
||||
provider: "local",
|
||||
entityType: e.kind === "movie" ? "movie" : "tv",
|
||||
externalId: e.id,
|
||||
titles: [e.title, ...(e.original_title ? [e.original_title] : [])],
|
||||
aliases: all.filter((a) => a.source !== "site").map((a) => a.alias),
|
||||
...(e.year != null ? { year: e.year } : {}),
|
||||
};
|
||||
const legitNorm = new Set([...legit.titles, ...legit.aliases].map(normalizeTitle));
|
||||
const releases = await q<{ id: string; title: string; subtitle: string | null; parsed: Record<string, unknown> }>(
|
||||
`select id, title, subtitle, parsed from source_release where media_entity_id = $1 and not resolution_locked`,
|
||||
[entityId],
|
||||
);
|
||||
const wrongNames = new Set<string>();
|
||||
let kept = 0;
|
||||
for (const r of releases) {
|
||||
const parsed = parseRelease(r.title, r.subtitle ? { subtitle: r.subtitle } : {});
|
||||
const s = scoreCandidate(parsed, legit);
|
||||
if (s.titleMatch === "exact" || s.titleMatch === "alias") {
|
||||
kept++;
|
||||
continue;
|
||||
}
|
||||
toUnbind.push({ id: r.id, title: r.title, subtitle: r.subtitle, entityId });
|
||||
for (const n of queryTitles(parsed)) wrongNames.add(n);
|
||||
for (const k of ["titleCn", "titleEn", "title"]) {
|
||||
const v = r.parsed?.[k];
|
||||
if (typeof v === "string" && v) wrongNames.add(normalizeTitle(v));
|
||||
}
|
||||
}
|
||||
const chain = all.filter(
|
||||
(a) => a.source === "site" && !legitNorm.has(a.normalized) && wrongNames.has(a.normalized),
|
||||
);
|
||||
for (const a of chain) aliasIdsToDelete.add(a.id);
|
||||
console.log(
|
||||
`\n### ${e.title}(${e.year ?? "-"}):已绑定 ${releases.length} 条,核对通过 ${kept} 条,误并入 ${releases.length - kept} 条`,
|
||||
);
|
||||
console.log(
|
||||
` 删除别名:${[...new Set([...poison.filter((a) => a.media_entity_id === entityId), ...chain].map((a) => `「${a.alias}」`))].join(" ") || "-"}`,
|
||||
);
|
||||
for (const r of toUnbind.filter((x) => x.entityId === entityId)) console.log(` - 解除:${r.title}`);
|
||||
}
|
||||
// 引用了这些资源的其他记录(只报告,不改)
|
||||
if (toUnbind.length) {
|
||||
const refs = await q<{ table_name: string }>(
|
||||
`select table_name from information_schema.columns
|
||||
where table_schema = 'public' and column_name = 'source_release_id'
|
||||
and table_name not in ('identity_candidate', 'resolution_attempt', 'observation')`,
|
||||
);
|
||||
for (const { table_name } of refs) {
|
||||
const [c] = await q<{ n: number }>(
|
||||
`select count(*)::int as n from "${table_name}" where source_release_id = any($1::uuid[])`,
|
||||
[toUnbind.map((r) => r.id)],
|
||||
);
|
||||
if (c && c.n > 0) console.log(` 注意:${table_name} 里有 ${c.n} 行引用了误并入的资源(不自动处理,请人工核对)`);
|
||||
}
|
||||
}
|
||||
console.log(`\n合计:解除 ${toUnbind.length} 条资源绑定,删除 ${aliasIdsToDelete.size} 个站点别名`);
|
||||
if (APPLY && (toUnbind.length || aliasIdsToDelete.size)) {
|
||||
await db.query("begin");
|
||||
try {
|
||||
await q(`delete from title_alias where id = any($1::uuid[])`, [[...aliasIdsToDelete]]);
|
||||
for (const r of toUnbind) {
|
||||
const parsed = parseRelease(r.title, r.subtitle ? { subtitle: r.subtitle } : {});
|
||||
await q(
|
||||
`update source_release set media_entity_id = null, content_unit_id = null, resolution_status = 'unresolved',
|
||||
resolution_evidence = $2::jsonb, updated_at = now()
|
||||
where id = $1 and not resolution_locked`,
|
||||
[r.id, JSON.stringify({ reason: "rollback_wrong_alias", previousEntityId: r.entityId })],
|
||||
);
|
||||
const normalized = parsed.title ? normalizeTitle(parsed.title) || null : null;
|
||||
await q(
|
||||
`insert into identity_candidate (source_release_id, status, reason, query, normalized_title, candidates,
|
||||
evidence_version, attempts, last_attempt_at, next_review_at)
|
||||
values ($1, 'pending', 'no_match', $2::jsonb, $3, '[]'::jsonb, 'rollback', 0, now(), now())
|
||||
on conflict (source_release_id) do update set status = 'pending', reason = 'no_match', attempts = 0,
|
||||
next_review_at = now(), resolved_media_entity_id = null, resolved_by = null, resolved_at = null`,
|
||||
[r.id, JSON.stringify({ title: parsed.title ?? null, rollbackFrom: r.entityId }), normalized],
|
||||
);
|
||||
await q(
|
||||
`insert into resolution_attempt (source_release_id, outcome, reason, evidence_version, candidate_count,
|
||||
media_entity_id, actor_type, actor_id)
|
||||
values ($1, 'manual', 'rollback_wrong_alias', 'rollback', 0, null, 'system', 'identity-reevaluate')`,
|
||||
[r.id],
|
||||
);
|
||||
}
|
||||
await db.query("commit");
|
||||
console.log("已执行:资源退回待识别池并排入立即复查(由运行中的服务按新规则重新识别)");
|
||||
} catch (e) {
|
||||
await db.query("rollback");
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
const orphan = await q<{ n: number }>(
|
||||
`select count(*)::int as n from content_unit u
|
||||
where u.media_entity_id = any($1::uuid[])
|
||||
and not exists (select 1 from source_release r where r.content_unit_id = u.id)`,
|
||||
[entities],
|
||||
).catch(() => [{ n: -1 }]);
|
||||
if (orphan[0] && orphan[0].n > 0)
|
||||
console.log(
|
||||
`提示:这些作品下${APPLY ? "现在" : "当前"}有 ${orphan[0].n} 个内容单元没有资源引用(脚本不删除,需要时人工清理)`,
|
||||
);
|
||||
}
|
||||
|
||||
if (!APPLY) console.log("\n(dry-run:未改库;确认后加 --apply 执行)");
|
||||
await db.end();
|
||||
Reference in new issue
Block a user