feat(metadata): overview falls back to Chinese variants only (MD-2)

- adapters: overviewLanguages() limits overview/tagline chain to Chinese
  variants (zh-CN→zh-SG→zh-TW→zh-HK); titles keep full chain; season
  supplement fetch skips languages useless for overview
- setting metadata.overviewNonChineseFallback (default off) + UI switch
- media detail shows "暂无简介" for empty entity/episode overview
- scripts/overview-recompute.ts: recompute stored non-Chinese overviews
  from kv_cache only (no TMDB requests), dry-run by default, --apply
- tests: chain unit tests, Heart Signal (103147) fixture, server
  recompute test, MP differential for episode overview
- runbook + CHANGELOG

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed 2026-09-30 18:04:23 +08:00
1 parent 081db4651c
commit f9901b514b
22 files changed
+887 -38

No files matched your search

@@ -23,7 +23,7 @@ import {
createEntityFromTmdbCandidate,
enqueueNewEntityRefresh,
} from "../sources-identity/service/entity-from-candidate.ts";
import { metadataLanguages } from "../sources-identity/service/metadata-rich.ts";
import { metadataLanguages, metadataTextOptions } from "../sources-identity/service/metadata-rich.ts";
import { identifyParsed, type NameIdentifyResult } from "../sources-identity/service/resolve.ts";
import type { ServerContext } from "../types.ts";
@@ -133,7 +133,7 @@ export function createPorts(
() => rt().tmdb(),
{ kind: ref.type, externalId: ref.id },
{ type, id: actor.id },
{ languages: await metadataLanguages(ctx) },
{ languages: await metadataLanguages(ctx), ...(await metadataTextOptions(ctx)) },
);
if (r.created) await enqueueNewEntityRefresh(ctx, r.mediaEntityId);
return r.mediaEntityId;
@@ -5,7 +5,7 @@ import {
createEntityFromTmdbCandidate,
enqueueNewEntityRefresh,
} from "../sources-identity/service/entity-from-candidate.ts";
import { metadataLanguages } from "../sources-identity/service/metadata-rich.ts";
import { metadataLanguages, metadataTextOptions } from "../sources-identity/service/metadata-rich.ts";
import { actorOfP, FilesError, type FmDeps } from "./common.ts";
import { audit } from "./ledger.ts";
@@ -24,7 +24,10 @@ export async function entityFromCandidate(
): Promise<FilesEntityFromCandidateResult> {
try {
const languages = await metadataLanguages(d.ctx);
const r = await createEntityFromTmdbCandidate(d.ctx.db, d.tmdb, ref, actorOf(principal), { languages });
const r = await createEntityFromTmdbCandidate(d.ctx.db, d.tmdb, ref, actorOf(principal), {
languages,
...(await metadataTextOptions(d.ctx)),
});
if (r.created) {
await audit(d, principal, "entity_from_candidate", { ...ref, mediaEntityId: r.mediaEntityId });
// 订阅建档 / 搜索建档 / 整理向导都走这里:建档即入队 full 刷新(每集信息、富字段)
+3 -3
View File
@@ -28,7 +28,7 @@ import type { z } from "zod";
import { type AuthEnv, requireScope } from "../../http/auth.ts";
import { ApiException } from "../../http/errors.ts";
import type { SourcesRuntime } from "../sources-identity/runtime.ts";
import { metadataLanguages } from "../sources-identity/service/metadata-rich.ts";
import { metadataLanguages, metadataTextOptions } from "../sources-identity/service/metadata-rich.ts";
import { upsertSiteRelease } from "../sources-identity/service/snapshot.ts";
import type { ModuleApp, ServerContext } from "../types.ts";
import { matchActions } from "./actions.ts";
@@ -169,7 +169,7 @@ export function registerSearchRoutes(app: ModuleApp, ctx: ServerContext, d: Sear
try {
const langs = await metadataLanguages(ctx);
return await raceAbort(
fetchRichDetail(client, type, id, langs),
fetchRichDetail(client, type, id, langs, await metadataTextOptions(ctx)),
AbortSignal.timeout(d.siteTimeoutMs),
"TMDB 查询超时",
);
@@ -413,7 +413,7 @@ export function registerSearchRoutes(app: ModuleApp, ctx: ServerContext, d: Sear
try {
const langs = await metadataLanguages(ctx);
const sd = await raceAbort(
fetchRichSeason(client, p.id, p.season, langs),
fetchRichSeason(client, p.id, p.season, langs, await metadataTextOptions(ctx)),
AbortSignal.timeout(d.siteTimeoutMs * 2),
"TMDB 查询超时",
);
@@ -1,4 +1,10 @@
import { fetchRichDetail, type MetadataCandidate, type TmdbClient, TmdbError } from "@umt/adapters";
import {
fetchRichDetail,
type MetadataCandidate,
type RichTextOptions,
type TmdbClient,
TmdbError,
} from "@umt/adapters";
import type { MediaKind } from "@umt/contracts";
import type { DB, Kysely } from "@umt/db";
import type { ParsedRelease } from "@umt/domain";
@@ -26,11 +32,12 @@ export async function withLanguageChain(
client: TmdbClient,
detail: MetadataCandidate,
languages: readonly string[] | undefined,
textOpts: RichTextOptions = {},
): Promise<MetadataCandidate> {
if (!languages?.length || typeof client.getJson !== "function") return detail;
if (detail.entityType !== "movie" && detail.entityType !== "tv") return detail;
try {
const rich = await fetchRichDetail(client, detail.entityType, detail.externalId, languages);
const rich = await fetchRichDetail(client, detail.entityType, detail.externalId, languages, textOpts);
return { ...detail, title: rich.title || detail.title, overview: rich.overview ?? detail.overview };
} catch (e) {
if (e instanceof TmdbError) return detail;
@@ -74,7 +81,7 @@ export async function createEntityFromTmdbCandidate(
tmdb: () => Promise<TmdbClient | null>,
ref: { externalId: string; kind: "movie" | "tv" },
actor: Actor,
opts: { languages?: readonly string[] } = {},
opts: { languages?: readonly string[] } & RichTextOptions = {},
): Promise<EntityFromCandidate> {
const extRef = { provider: "tmdb", entityType: ref.kind, externalId: ref.externalId };
const existing = await entityByExternal(db, extRef);
@@ -91,7 +98,9 @@ export async function createEntityFromTmdbCandidate(
if (e instanceof TmdbError) throw new CandidateError("upstream", `TMDB 查询失败:${e.kind}`);
throw e;
}
detail = await withLanguageChain(client, detail, opts.languages);
detail = await withLanguageChain(client, detail, opts.languages, {
overviewNonChineseFallback: opts.overviewNonChineseFallback === true,
});
for (let attempt = 0; ; attempt++) {
try {
@@ -24,7 +24,7 @@ import {
} from "./entities.ts";
import { enqueueNewEntityRefresh, withLanguageChain } from "./entity-from-candidate.ts";
import { detachFromIdentityDecisions } from "./identity-decisions.ts";
import { metadataLanguages } from "./metadata-rich.ts";
import { metadataLanguages, metadataTextOptions } from "./metadata-rich.ts";
import { recordAttempt } from "./resolve.ts";
import { backfillRecognizeCache } from "./tmdb-recognize-cache.ts";
@@ -59,7 +59,7 @@ async function entityForExternal(
if (e instanceof TmdbError) throw new ApiException("upstream_unavailable", `TMDB 查询失败:${e.kind}`);
throw e;
}
detail = await withLanguageChain(tmdb, detail, await metadataLanguages(rt.ctx));
detail = await withLanguageChain(tmdb, detail, await metadataLanguages(rt.ctx), await metadataTextOptions(rt.ctx));
const r = await createEntityFromTmdb(exec, detail, {
level: "manual",
evidence: { by: actorName(actor) },
@@ -22,7 +22,7 @@ import { type Actor, audit, type Exec } from "./common.ts";
import { ensureSeason, promoteUnits } from "./metadata-totals.ts";
// MD-1 语言回退链 + 每集元数据(REQ §7.3 字段级):
// - 作品:TMDB 富详情(fetchRichDetail)按设置 metadata.languages 取标题 / 简介 / 标语,另存中文名 titleZh、类型、评分、
// - 作品:TMDB 富详情(fetchRichDetail)按设置 metadata.languages 取标题 / 简介 / 标语(MD-2:简介 / 标语缺省只回退到中文变体),另存中文名 titleZh、类型、评分、
// 状态、时长、季数、主创、外部 ID;来源值写 metadata_field(language = 实际取到的语言),同字段换语言时删旧行;
// - 别名:translations / alternative_titles 写 title_alias(来源 tmdb,标语言);已有同文本别名只补空的语言;
// - 季 / 集:fetchRichSeason 的每集 name / overview / airDate / runtime / stillPath / vote 写 metadata_field(content_unit_id),
@@ -40,6 +40,15 @@ export async function metadataLanguages(ctx: ServerContext): Promise<string[]> {
return langs?.length ? langs : [...DEFAULT_METADATA_LANGUAGES];
}
/** 设置 metadata.overviewNonChineseFallback(MD-2):简介 / 标语是否允许回退到非中文;缺省 false */
export async function metadataTextOptions(ctx: ServerContext): Promise<{ overviewNonChineseFallback: boolean }> {
const m = await ctx.settings.get("metadata").catch(() => null);
return {
overviewNonChineseFallback:
(m as { overviewNonChineseFallback?: boolean } | null)?.overviewNonChineseFallback === true,
};
}
/** 设置 metadata.includeSpecials:full 刷新 / 回刷是否包含季 0(特别篇);缺省 true */
export async function metadataIncludeSpecials(ctx: ServerContext): Promise<boolean> {
const m = await ctx.settings.get("metadata").catch(() => null);
@@ -82,7 +91,7 @@ const same = (a: unknown, b: unknown) => JSON.stringify(a ?? null) === JSON.stri
* 写一组 tmdb 来源字段(作品或内容单元):值和语言都没变的跳过;语言变了删掉该字段其它语言的 tmdb 行再写。
* 返回写入(新增 / 变化)的字段名。
*/
async function writeSourceFields(
export async function writeSourceFields(
exec: Exec,
target: { mediaEntityId: string } | { contentUnitId: string },
fields: Record<string, FieldValue>,
@@ -25,6 +25,7 @@ import {
applySeasonMetadata,
metadataIncludeSpecials,
metadataLanguages,
metadataTextOptions,
type SeasonMetadataResult,
seasonsInLibrary,
seasonsToRefresh,
@@ -189,13 +190,14 @@ export async function refreshMetadata(
// MD-1 富详情(语言回退链、类型、评分、主创…)与每集信息;注入的简化客户端没有 getJson 时走旧路径
const rich = typeof tmdb.getJson === "function";
const langs = rich ? await metadataLanguages(ctx) : [];
const textOpts = await metadataTextOptions(ctx);
let richDetail: TmdbRichDetail | null = null;
const richSeasons: TmdbRichSeasonDetail[] = [];
// 可以新建集单元的季:full 刷新的全部季、追踪中订阅的季;其余季只更新已有集单元
let creatable = new Set<number>();
if (rich) {
try {
richDetail = await fetchRichDetail(tmdb, ident.entity_type, ident.external_id, langs);
richDetail = await fetchRichDetail(tmdb, ident.entity_type, ident.external_id, langs, textOpts);
} catch (e) {
if (!(e instanceof TmdbError)) throw e;
ctx.logger.debug({ entityId: id, err: { message: e.message } }, "TMDB 富详情获取失败,按基础详情刷新");
@@ -213,7 +215,7 @@ export async function refreshMetadata(
creatable = new Set(opts.full ? want : tracked);
for (const n of want) {
try {
richSeasons.push(await fetchRichSeason(tmdb, ident.external_id, n, langs, { now: rt.now() }));
richSeasons.push(await fetchRichSeason(tmdb, ident.external_id, n, langs, { now: rt.now(), ...textOpts }));
} catch (e) {
ctx.logger.debug({ entityId: id, season: n, err: { message: (e as Error).message } }, "季详情获取失败");
}
@@ -0,0 +1,289 @@
import {
fetchRichDetail,
fetchRichSeason,
overviewLanguages,
type TmdbClient,
TmdbError,
tmdbRequestKey,
} from "@umt/adapters";
import { sql } from "@umt/db";
import { CACHE_REGIONS } from "../../../shared-cache.ts";
import { type Actor, audit, type Exec } from "./common.ts";
import { project } from "./metadata.ts";
import { projectUnit, writeSourceFields } from "./metadata-rich.ts";
// MD-2 存量简介重算:MD-1 的回退链会把 en-US 等非中文简介填进作品 / 季 / 集的简介;新规则下简介只回退到中文变体
// (见 adapters overviewLanguages)。这里找出来源 tmdb、语言不在简介链里的简介,**不请求 TMDB**,只读 kv_cache
// (region tmdb.request)里当初取值用过的响应,按新规则重算:
// - 缓存里有响应:用同一套 fetchRichDetail / fetchRichSeason 重算(缓存读取不续期、不改最近使用时间,过期未清理的行也用);
// - 缓存里没有(已淘汰):当初能落到非中文,说明当时链里的中文变体都没有简介,按新规则直接清空(basis = no_cache);
// - 作品另按缓存重算标语(标语不记语言,只能在缓存命中时重算)。
// 写入走字段来源表 + 投影(与刷新同一套 writeSourceFields / project / projectUnit),用户锁定 / 覆盖的字段最终值不变。
// 入口:scripts/overview-recompute.ts(缺省 dry-run,--apply 写库)。
export interface OverviewValue {
value: string | null;
language: string | null;
}
export interface OverviewChange {
target: "entity" | "season" | "episode";
mediaEntityId: string;
entityTitle: string;
contentUnitId: string | null;
season: number | null;
episode: number | null;
field: "overview" | "tagline";
from: OverviewValue;
to: OverviewValue;
/** cache = 按缓存的 TMDB 响应重算;no_cache = 缓存里没有,按"当时中文为空"清空 */
basis: "cache" | "no_cache";
/** tmdb 来源修订(写 metadata_field.source_revision) */
revision: string;
}
export interface OverviewRecomputePlan {
languages: string[];
overviewLanguages: string[];
/** 语言不在简介链里的简介行数(作品 / 季与集) */
scanned: { entities: number; units: number };
changes: OverviewChange[];
/** 读缓存次数 / 未命中次数 */
cache: { reads: number; misses: number };
}
type CacheClient = Pick<TmdbClient, "getJson"> & { reads: number; misses: number };
/** 只读 kv_cache 的 TMDB 客户端:命中返回缓存值,未命中抛 notfound(fetchRichSeason 的补拉遇错即停) */
export function cacheOnlyTmdb(exec: Exec): CacheClient {
const c: CacheClient = {
reads: 0,
misses: 0,
async getJson<T>(path: string, params: Record<string, string | undefined> = {}): Promise<T> {
const key = tmdbRequestKey(path, params);
c.reads++;
const r = await sql<{ value: unknown }>`select value from kv_cache
where region = ${CACHE_REGIONS.tmdbRequest} and key = ${key}`.execute(exec);
const v = r.rows[0]?.value;
if (v === undefined || v === null) {
c.misses++;
throw new TmdbError("notfound", `未缓存 ${key}`, { path });
}
return v as T;
},
};
return c;
}
const same = (a: OverviewValue, b: OverviewValue) =>
(a.value ?? null) === (b.value ?? null) && a.language === b.language;
const str = (v: unknown): string | null => (typeof v === "string" ? v : null);
/** 生成重算清单(只读;不请求 TMDB) */
export async function planOverviewRecompute(
exec: Exec,
opts: { languages: readonly string[]; overviewNonChineseFallback?: boolean; now?: Date; entityId?: string | null },
): Promise<OverviewRecomputePlan> {
const languages = [...opts.languages];
const textLangs = overviewLanguages(languages, opts.overviewNonChineseFallback === true);
const only = opts.entityId ?? null;
const tmdb = cacheOnlyTmdb(exec);
const changes: OverviewChange[] = [];
// ── 作品 ──
const ents = await sql<{
id: string;
title: string;
value: unknown;
language: string;
entity_type: "movie" | "tv";
external_id: string;
}>`
select distinct on (f.media_entity_id) f.media_entity_id as id, e.title, f.value, f.language, x.entity_type, x.external_id
from metadata_field f
join media_entity e on e.id = f.media_entity_id
join external_identity x on x.media_entity_id = e.id and x.provider = 'tmdb' and x.status = 'accepted'
and x.entity_type in ('movie', 'tv')
where f.content_unit_id is null and f.source = 'tmdb' and f.field = 'overview'
and f.language is not null and not (f.language = any(${textLangs}::text[]))
and (${only}::uuid is null or f.media_entity_id = ${only}::uuid)
order by f.media_entity_id, x.created_at`.execute(exec);
for (const e of ents.rows) {
const revision = `tmdb:${e.entity_type}/${e.external_id}`;
const base = {
target: "entity" as const,
mediaEntityId: e.id,
entityTitle: e.title,
contentUnitId: null,
season: null,
episode: null,
revision,
};
const from = { value: str(e.value), language: e.language };
try {
const rich = await fetchRichDetail(tmdb, e.entity_type, e.external_id, languages, {
overviewNonChineseFallback: opts.overviewNonChineseFallback === true,
});
const to = { value: rich.overview, language: rich.overviewLanguage };
if (!same(from, to)) changes.push({ ...base, field: "overview", from, to, basis: "cache" });
const tag = await exec
.selectFrom("metadata_field")
.select(["value", "language"])
.where("media_entity_id", "=", e.id)
.where("content_unit_id", "is", null)
.where("source", "=", "tmdb")
.where("field", "=", "tagline")
.executeTakeFirst();
const tagFrom = { value: str(tag?.value), language: tag?.language ?? null };
const tagTo = { value: rich.tagline, language: null };
if (tag && !same(tagFrom, tagTo))
changes.push({ ...base, field: "tagline", from: tagFrom, to: tagTo, basis: "cache" });
} catch (err) {
if (!(err instanceof TmdbError)) throw err;
changes.push({ ...base, field: "overview", from, to: { value: null, language: null }, basis: "no_cache" });
}
}
// ── 季 / 集 ──
const units = await sql<{
unit_id: string;
kind: string;
season_number: number | null;
episode_number: number | null;
media_entity_id: string;
title: string;
value: unknown;
language: string;
external_id: string;
}>`
select distinct on (f.content_unit_id) f.content_unit_id as unit_id, u.kind, u.season_number, u.episode_number,
u.media_entity_id, e.title, f.value, f.language, x.external_id
from metadata_field f
join content_unit u on u.id = f.content_unit_id
join media_entity e on e.id = u.media_entity_id
join external_identity x on x.media_entity_id = e.id and x.provider = 'tmdb' and x.status = 'accepted'
and x.entity_type = 'tv'
where f.media_entity_id is null and f.source = 'tmdb' and f.field = 'overview'
and f.language is not null and not (f.language = any(${textLangs}::text[]))
and u.status <> 'retired' and u.kind in ('season', 'episode') and u.season_number is not null
and (${only}::uuid is null or u.media_entity_id = ${only}::uuid)
order by f.content_unit_id, x.created_at`.execute(exec);
const groups = new Map<string, typeof units.rows>();
for (const u of units.rows) {
const k = `${u.media_entity_id}|${u.external_id}|${u.season_number}`;
groups.set(k, [...(groups.get(k) ?? []), u]);
}
for (const rows of groups.values()) {
const first = rows[0]!;
const season = first.season_number!;
let sd: Awaited<ReturnType<typeof fetchRichSeason>> | null = null;
try {
sd = await fetchRichSeason(tmdb, first.external_id, season, languages, {
now: opts.now ?? new Date(),
overviewNonChineseFallback: opts.overviewNonChineseFallback === true,
});
} catch (err) {
if (!(err instanceof TmdbError)) throw err;
}
const sorted = [...rows].sort((a, b) => (a.episode_number ?? -1) - (b.episode_number ?? -1));
for (const u of sorted) {
const from = { value: str(u.value), language: u.language };
let to: OverviewValue = { value: null, language: null };
let basis: OverviewChange["basis"] = "no_cache";
if (sd && u.kind === "season") {
to = { value: sd.overview, language: sd.overviewLanguage };
basis = "cache";
} else if (sd) {
const ep = sd.episodes.find((x) => x.episodeNumber === u.episode_number);
if (ep) {
to = { value: ep.overview, language: ep.overviewLanguage };
basis = "cache";
}
}
if (same(from, to)) continue;
changes.push({
target: u.kind === "season" ? "season" : "episode",
mediaEntityId: u.media_entity_id,
entityTitle: u.title,
contentUnitId: u.unit_id,
season,
episode: u.kind === "season" ? null : u.episode_number,
field: "overview",
from,
to,
basis,
revision: `tmdb:tv/${u.external_id}`,
});
}
}
return {
languages,
overviewLanguages: textLangs,
scanned: { entities: ents.rows.length, units: units.rows.length },
changes,
cache: { reads: tmdb.reads, misses: tmdb.misses },
};
}
/**
* 按清单写库:同刷新一样写字段来源(语言变了删旧语言行)并投影;锁定 / 覆盖的字段最终值不变。
* 调用方负责事务(脚本在一个事务里执行)。返回投影后最终值有变化的作品 / 单元数。
*/
export async function applyOverviewRecompute(
exec: Exec,
plan: OverviewRecomputePlan,
opts: { now: Date; actor?: Actor },
): Promise<{ entities: number; units: number }> {
const byTarget = new Map<string, OverviewChange[]>();
for (const c of plan.changes) {
const k = c.contentUnitId ? `u:${c.contentUnitId}` : `e:${c.mediaEntityId}`;
byTarget.set(k, [...(byTarget.get(k) ?? []), c]);
}
let entities = 0;
let units = 0;
for (const list of byTarget.values()) {
const c0 = list[0]!;
const fields = Object.fromEntries(list.map((c) => [c.field, c.to]));
const names = list.map((c) => c.field);
if (c0.contentUnitId) {
const existing = await exec
.selectFrom("metadata_field")
.select(["field", "value", "language"])
.where("content_unit_id", "=", c0.contentUnitId)
.where("media_entity_id", "is", null)
.where("source", "=", "tmdb")
.where("field", "in", names)
.execute();
await writeSourceFields(exec, { contentUnitId: c0.contentUnitId }, fields, existing, c0.revision, opts.now);
const r = await projectUnit(exec, c0.contentUnitId, opts.now);
if (r.changedFields.length) units++;
} else {
const existing = await exec
.selectFrom("metadata_field")
.select(["field", "value", "language"])
.where("media_entity_id", "=", c0.mediaEntityId)
.where("content_unit_id", "is", null)
.where("source", "=", "tmdb")
.where("field", "in", names)
.execute();
await writeSourceFields(exec, { mediaEntityId: c0.mediaEntityId }, fields, existing, c0.revision, opts.now);
const r = await project(exec, c0.mediaEntityId, opts.now);
if (r.changedFields.length) entities++;
}
}
if (plan.changes.length) {
await audit(
exec,
opts.actor ?? { type: "system", id: "overview-recompute" },
"metadata.overview_recompute",
{ type: "metadata", id: null },
{
overviewLanguages: plan.overviewLanguages,
changes: plan.changes.length,
noCache: plan.changes.filter((c) => c.basis === "no_cache").length,
},
);
}
return { entities, units };
}