From 37cf0a398e2ce0a56ff571057fed7a34efc6ae74 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 3 Oct 2026 21:04:57 +0800 Subject: [PATCH] =?UTF-8?q?feat(identity):=20=E6=94=B9=20TMDB=20=E6=98=A0?= =?UTF-8?q?=E5=B0=84=E5=90=8E=E6=B8=85=E6=97=A7=E5=AD=97=E6=AE=B5=E3=80=81?= =?UTF-8?q?full=20=E5=88=B7=E6=96=B0=E5=B9=B6=E8=87=AA=E5=8A=A8=E6=9B=B4?= =?UTF-8?q?=E6=96=B0=E9=87=8D=E5=88=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - setEntityIdentity:TMDB 映射变化时同事务清掉作品及季 / 集单元的 tmdb 来源字段(人工覆盖 / 锁定保留, 海报背景图不再保留旧作品的),退役没锁的 IMDb 映射,提交后排 full 刷新 - refreshMetadata:NFO 相关字段(作品或季 / 集)有变化时调 scheduleEntityRescrape,评分票数变化不触发 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei --- .../service/identity-admin.ts | 34 +++- .../sources-identity/service/metadata-rich.ts | 22 ++- .../sources-identity/service/metadata.ts | 108 ++++++++++- .../identity-change-rescrape.test.ts | 179 ++++++++++++++++++ 4 files changed, 340 insertions(+), 3 deletions(-) create mode 100644 apps/server/test/sources-identity/identity-change-rescrape.test.ts diff --git a/apps/server/src/modules/sources-identity/service/identity-admin.ts b/apps/server/src/modules/sources-identity/service/identity-admin.ts index f4a0b5f4..05b088ad 100644 --- a/apps/server/src/modules/sources-identity/service/identity-admin.ts +++ b/apps/server/src/modules/sources-identity/service/identity-admin.ts @@ -29,6 +29,7 @@ import { settleEpisodeDecision, } from "./episode-resolution.ts"; import { detachFromIdentityDecisions } from "./identity-decisions.ts"; +import { clearTmdbMetadata } from "./metadata.ts"; import { metadataLanguages, metadataTextOptions } from "./metadata-rich.ts"; import { recordAttempt } from "./resolve.ts"; import { backfillRecognizeCache } from "./tmdb-recognize-cache.ts"; @@ -271,10 +272,12 @@ async function retireIdentity(exec: Exec, row: ExternalIdentityRow, actor: Actor /** * 改作品的外部映射:同 provider+类型的旧 accepted 映射退役(留修订并锁定,防止自动匹配改回), * 新映射 accepted + 人工锁。目标身份已被其它作品接受时 409(应走合并)。 + * TMDB 映射变了:同一事务里清掉该作品(及季 / 集单元)tmdb 来源的元数据(人工覆盖 / 锁定保留),提交后排一次 + * full 刷新;刷新写回新身份的字段后,刮削过的作品自动排「更新」模式重刮(refreshMetadata → scheduleEntityRescrape)。 */ export async function setEntityIdentity(rt: SourcesRuntime, entityId: string, input: EntityIdentitySet, actor: Actor) { const { ctx } = rt; - return ctx.db.transaction().execute(async (trx) => { + const r = await ctx.db.transaction().execute(async (trx) => { const id = await resolveEntityId(trx, entityId); if (!id) throw new ApiException("not_found", "作品不存在"); const other = await entityByExternal(trx, input); @@ -297,6 +300,21 @@ export async function setEntityIdentity(rt: SourcesRuntime, entityId: string, in reason: input.reason ?? null, }); if (!newId) throw new ApiException("conflict", "外部身份唯一约束冲突"); + if (input.provider === "tmdb") { + // 旧 TMDB 带出来的 IMDb 映射(没人工锁的)一并退役:NFO 里 IMDb 是 default uniqueid,留着会被认成旧作品; + // 刷新后 NFO 取新详情里的 IMDb + const derived = await trx + .selectFrom("external_identity") + .selectAll() + .where("media_entity_id", "=", id) + .where("content_unit_id", "is", null) + .where("provider", "=", "imdb") + .where("status", "=", "accepted") + .where("locked", "=", false) + .execute(); + for (const o of derived) await retireIdentity(trx, o, actor, "tmdb_mapping_changed"); + await clearTmdbMetadata(trx, id, rt.now()); + } await audit( trx, actor, @@ -310,6 +328,20 @@ export async function setEntityIdentity(rt: SourcesRuntime, entityId: string, in ); return { mediaEntityId: id, changed: true, identityId: newId, retired: olds.map((o) => o.id) }; }); + if (r.changed && input.provider === "tmdb" && r.identityId) + await ctx.engine + .enqueue( + "refreshMetadata", + { mediaEntityId: r.mediaEntityId, full: true }, + { idempotencyKey: `refreshMetadata:identity:${r.identityId}`, traceId: r.mediaEntityId }, + ) + .catch((err: unknown) => + ctx.logger.warn( + { entityId: r.mediaEntityId, err: { message: (err as Error).message } }, + "改映射后元数据刷新没排上", + ), + ); + return r; } /** 外部身份字段修订(乐观并发 expectedRevision);每次变更写 identity_revision */ diff --git a/apps/server/src/modules/sources-identity/service/metadata-rich.ts b/apps/server/src/modules/sources-identity/service/metadata-rich.ts index 4e06d676..34658f15 100644 --- a/apps/server/src/modules/sources-identity/service/metadata-rich.ts +++ b/apps/server/src/modules/sources-identity/service/metadata-rich.ts @@ -315,8 +315,20 @@ export interface SeasonMetadataResult { changedUnits: number; /** DH-1:临时转正式的集单元数(季单元转正不计) */ promoted: number; + /** 写进 NFO 的字段(名称、简介、日期、时长、剧照、季海报)有变化的单元数;刷新后据此决定要不要自动重刮 */ + nfoChangedUnits?: number; } +/** 内容单元里写进 NFO 的字段(评分 / 票数变化不触发自动重刮) */ +export const UNIT_NFO_FIELDS: ReadonlySet = new Set([ + "name", + "overview", + "airDate", + "runtime", + "stillPath", + "posterPath", +]); + /** * 季详情 → 季单元与集单元的字段级元数据。集单元按(作品, 季, 集号)找最早建的未退役单元,没有时: * create = true(full 刷新 / 追踪中的季)建 confirmed 单元;否则只更新已有单元(集单元仍由发布 / 文件懒建)。 @@ -331,7 +343,14 @@ export async function applySeasonMetadata( opts: { create?: boolean } = {}, ): Promise { const create = opts.create ?? true; - const out: SeasonMetadataResult = { season: sd.seasonNumber, episodes: 0, created: 0, changedUnits: 0, promoted: 0 }; + const out: SeasonMetadataResult = { + season: sd.seasonNumber, + episodes: 0, + created: 0, + changedUnits: 0, + promoted: 0, + nfoChangedUnits: 0, + }; if (sd.seasonNumber < 0) return out; const seasonUnit = await ensureSeason(exec, entityId, sd.seasonNumber, sd.name); const units = await exec @@ -399,6 +418,7 @@ export async function applySeasonMetadata( if (written.length || t.fresh) { const r = await projectUnit(exec, t.unitId, now); if (r.changedFields.length) out.changedUnits++; + if (r.changedFields.some((f) => UNIT_NFO_FIELDS.has(f))) out.nfoChangedUnits = (out.nfoChangedUnits ?? 0) + 1; } } out.episodes = eps.length; diff --git a/apps/server/src/modules/sources-identity/service/metadata.ts b/apps/server/src/modules/sources-identity/service/metadata.ts index 34cfa24e..9b704e82 100644 --- a/apps/server/src/modules/sources-identity/service/metadata.ts +++ b/apps/server/src/modules/sources-identity/service/metadata.ts @@ -17,6 +17,7 @@ import type { } from "@umt/contracts"; import { sql } from "@umt/db"; import { ApiException } from "../../../http/errors.ts"; +import { scheduleEntityRescrape } from "../../acquisition-organize/services/scrape.ts"; import { jsonb } from "../rows.ts"; import type { SourcesRuntime } from "../runtime.ts"; import { type Actor, audit, type Exec } from "./common.ts"; @@ -118,6 +119,102 @@ export async function writeEnrichmentFields( } } +/** 作品里写进 NFO 的字段:刷新后这些字段有变化才自动重刮(评分、票数、播出状态等不算) */ +const ENTITY_NFO_FIELDS: ReadonlySet = new Set([ + "title", + "originalTitle", + "year", + "overview", + "tagline", + "releaseDate", + "genres", + "originCountry", + "platform", + "runtime", + "cast", + "crew", + "externalIds", + "posterPath", + "backdropPath", +]); + +/** + * 改作品的 TMDB 映射后:清掉该作品及其季 / 集单元的 tmdb 来源字段(metadata_field 与投影出的最终值), + * 人工覆盖与锁定字段保留;单元的 title / air_date 列随 name / airDate 一起清空。之后由 full 刷新按新身份重新填, + * 避免新身份的 NFO 混进旧作品的标题、简介、IMDb、海报(海报 / 背景图「TMDB 没有时保留旧值」不能保留旧作品的)。 + */ +export async function clearTmdbMetadata(trx: Exec, entityId: string, now: Date): Promise { + const keep = (meta: Record, locks: Set, overridden: Set) => { + const out: Record = {}; + for (const [k, v] of Object.entries(meta)) + if (v?.source !== "tmdb" || locks.has(k) || overridden.has(k) || v.locked) out[k] = v; + return out; + }; + const e = await trx + .selectFrom("media_entity") + .select(["metadata", "field_locks"]) + .where("id", "=", entityId) + .forUpdate() + .executeTakeFirstOrThrow(); + const entityOverrides = await trx + .selectFrom("field_override") + .select("field") + .where("media_entity_id", "=", entityId) + .execute(); + await trx + .deleteFrom("metadata_field") + .where("media_entity_id", "=", entityId) + .where("content_unit_id", "is", null) + .where("source", "=", "tmdb") + .execute(); + await trx + .updateTable("media_entity") + .set({ + metadata: jsonb( + keep( + (e.metadata ?? {}) as Record, + new Set(e.field_locks), + new Set(entityOverrides.map((o) => o.field)), + ), + ), + updated_at: now, + }) + .where("id", "=", entityId) + .execute(); + // 其它来源(web / agent 补全)按原规则重新投影 + await project(trx, entityId, now); + + const units = await trx + .selectFrom("content_unit") + .select(["id", "metadata", "field_locks"]) + .where("media_entity_id", "=", entityId) + .execute(); + if (!units.length) return; + const ids = units.map((u) => u.id); + const unitOverrides = await trx + .selectFrom("field_override") + .select(["content_unit_id", "field"]) + .where("content_unit_id", "in", ids) + .execute(); + await trx.deleteFrom("metadata_field").where("content_unit_id", "in", ids).where("source", "=", "tmdb").execute(); + for (const u of units) { + const meta = (u.metadata ?? {}) as Record; + const overridden = new Set(unitOverrides.filter((o) => o.content_unit_id === u.id).map((o) => o.field)); + const next = keep(meta, new Set(u.field_locks), overridden); + if (Object.keys(next).length === Object.keys(meta).length) continue; + await trx + .updateTable("content_unit") + .set({ + metadata: jsonb(next), + ...(next.name ? {} : { title: null }), + ...(next.airDate ? {} : { air_date: null }), + updated_at: now, + }) + .where("id", "=", u.id) + .execute(); + } +} + export interface RefreshResult { status: "refreshed" | "skipped" | "failed"; reason?: string; @@ -240,7 +337,7 @@ export async function refreshMetadata( } const now = rt.now(); const rev = `tmdb:${ident.entity_type}/${ident.external_id}`; - return ctx.db.transaction().execute(async (trx) => { + const result = await ctx.db.transaction().execute(async (trx) => { if (richDetail) await writeRichEntityFields(trx, id, richDetail, rev, now); else await writeTmdbFields(trx, id, detail, rev, now); await trx @@ -282,6 +379,15 @@ export async function refreshMetadata( : {}), }; }); + // 写进 NFO 的字段有变化:刮削过的作品延后排一次「更新」模式重刮(期间再刷新合进同一条);失败只记日志 + const nfoChanged = + result.changedFields.some((f) => ENTITY_NFO_FIELDS.has(f)) || + (result.episodes ?? []).some((x) => (x.nfoChangedUnits ?? 0) > 0); + if (nfoChanged) + await scheduleEntityRescrape({ ctx }, id).catch((err: unknown) => + ctx.logger.warn({ entityId: id, err: { message: (err as Error).message } }, "元数据刷新后重刮没排上"), + ); + return result; } /** diff --git a/apps/server/test/sources-identity/identity-change-rescrape.test.ts b/apps/server/test/sources-identity/identity-change-rescrape.test.ts new file mode 100644 index 00000000..623c66d6 --- /dev/null +++ b/apps/server/test/sources-identity/identity-change-rescrape.test.ts @@ -0,0 +1,179 @@ +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { createMockTmdb, createTempTree, type MockTmdb, type TempTree, TMDB_FIXTURES } from "@umt/testkit"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import type { AoPorts } from "../../src/modules/acquisition-organize/ports.ts"; +import type { AoDeps } from "../../src/modules/acquisition-organize/services/common.ts"; +import { + RESCRAPE_DELAY_MS, + runScrapeRecord, + SCRAPE_TASK, + scrapeEntity, +} from "../../src/modules/acquisition-organize/services/scrape.ts"; +import { createEntityFromTmdbCandidate } from "../../src/modules/sources-identity/service/entity-from-candidate.ts"; +import { setEntityIdentity } from "../../src/modules/sources-identity/service/identity-admin.ts"; +import { refreshMetadata, setFieldOverride } from "../../src/modules/sources-identity/service/metadata.ts"; +import { createHarness, type Harness, pgAvailable } from "./harness.ts"; + +// 改作品的 TMDB 映射后:清掉旧作品的 tmdb 来源字段(人工锁定的保留)→ full 刷新 → 刮削过的作品延后排一次「更新」模式重刮; +// 重刮后的 NFO 标题 / 简介 / IMDb / 海报都是新作品的,不残留旧作品;刷新期间再刷新合进同一条排队记录(防抖)。 + +const up = await pgAvailable(); +const enc = (s: string) => new TextEncoder().encode(s); +const VIDEO = "Movies/奥本海默 (2023)/Oppenheimer.2023.1080p.mkv"; +const DIR = "Movies/奥本海默 (2023)"; +const NFO = `${DIR}/Oppenheimer.2023.1080p.nfo`; +type Final = { value: unknown; source: string; locked: boolean }; + +describe.skipIf(!up)("改映射后不再新旧混写:清字段 → 刷新 → 更新模式重刮", () => { + let h: Harness; + let tmdb: MockTmdb; + let lib: TempTree; + let d: AoDeps; + let id = ""; + const actor = { type: "user" as const, id: "tester" }; + const enqueueOpts: { task: string; availableAt?: Date | undefined }[] = []; + + beforeAll(async () => { + // 沙丘 2 补一张海报,验证改映射后海报换成新作品的 + const fixtures = TMDB_FIXTURES.map((f) => (f.id === 693134 ? { ...f, poster_path: "/dune.jpg" } : f)); + tmdb = await createMockTmdb({ fixtures }); + h = await createHarness("idrescrape", { tmdbUrl: tmdb.url }); + lib = await createTempTree({ [VIDEO]: 1000 }, { sentinel: true, prefix: "umt-idscr-" }); + const orig = h.engine.enqueue.bind(h.engine); + h.engine.enqueue = async (task, input, o) => { + enqueueOpts.push({ task, availableAt: o.availableAt }); + return orig(task, input, o); + }; + d = { + ctx: h.ctx, + ports: { + now: () => new Date(), + scrapeImage: async (p: string) => ({ content: enc(p), mime: "image/jpeg" }), + scrapeTmdbDetail: async () => null, + scrapeTmdbSeason: async () => null, + } as unknown as AoPorts, + }; + }, 60_000); + afterAll(async () => { + await h?.close(); + await tmdb?.close(); + await lib?.cleanup(); + }); + + const queuedUpdates = () => + h.db + .selectFrom("scrape_record") + .select(["id", "mode", "trigger", "status"]) + .where("media_entity_id", "=", id) + .where("status", "=", "queued") + .execute(); + + it("先按奥本海默建档、刷新、刮削;没刮削过时刷新不排重刮", async () => { + const r = await createEntityFromTmdbCandidate( + h.db, + () => h.rt.tmdb(), + { externalId: "872585", kind: "movie" }, + actor, + ); + id = r.mediaEntityId; + expect((await refreshMetadata(h.rt, id, { full: true })).status).toBe("refreshed"); + expect(h.engine.enqueued.some((x) => x.task === SCRAPE_TASK)).toBe(false); + await setFieldOverride(h.rt, id, "tagline", { value: "我的标语", lock: true }, actor); + + const root = await h.db + .insertInto("library_root") + .values({ name: "lib-idscr", media_kind: "mixed", local_path: lib.root, management_mode: "managed" }) + .returning("id") + .executeTakeFirstOrThrow(); + const asset = await h.db + .insertInto("media_asset") + .values({ media_entity_id: id, kind: "video" }) + .returning("id") + .executeTakeFirstOrThrow(); + await h.db + .insertInto("asset_location") + .values({ media_asset_id: asset.id, library_root_id: root.id, relative_path: VIDEO }) + .execute(); + + const q = await scrapeEntity(d, id, { trigger: "manual" }); + expect(q).toHaveLength(1); + const rec = await runScrapeRecord(d, q[0]!.recordId); + expect(rec).toMatchObject({ status: "succeeded", mode: "update" }); + expect(rec.files.every((f) => f.action !== "written" || /^[0-9a-f]{64}$/.test(f.hash ?? ""))).toBe(true); + const nfo = await readFile(join(lib.root, NFO), "utf8"); + expect(nfo).toContain("奥本海默"); + expect(nfo).toContain("tt15398776"); + expect(nfo).toContain("我的标语"); + expect(await readFile(join(lib.root, DIR, "poster.jpg"), "utf8")).toBe("/oppenheimer.jpg"); + }, 60_000); + + it("改映射到沙丘 2:tmdb 来源字段清空,锁定字段保留,排 full 刷新", async () => { + const before = h.engine.enqueued.length; + const r = await setEntityIdentity( + h.rt, + id, + { provider: "tmdb", entityType: "movie", externalId: "693134", lock: true }, + actor, + ); + expect(r.changed).toBe(true); + const e = await h.db.selectFrom("media_entity").select(["metadata"]).where("id", "=", id).executeTakeFirstOrThrow(); + const meta = e.metadata as Record; + expect(meta.overview).toBeUndefined(); + expect(meta.posterPath).toBeUndefined(); + expect(meta.externalIds).toBeUndefined(); + expect(meta.tagline).toMatchObject({ value: "我的标语", locked: true }); + const tmdbRows = await h.db + .selectFrom("metadata_field") + .select("field") + .where("media_entity_id", "=", id) + .where("source", "=", "tmdb") + .execute(); + expect(tmdbRows).toEqual([]); + expect(h.engine.enqueued.slice(before)).toEqual([ + expect.objectContaining({ task: "refreshMetadata", input: { mediaEntityId: id, full: true } }), + ]); + }); + + it("刷新后延后排一次更新模式重刮;期间再刷新合进同一条", async () => { + const t0 = Date.now(); + expect((await refreshMetadata(h.rt, id, { full: true })).status).toBe("refreshed"); + const queued = await queuedUpdates(); + expect(queued).toEqual([expect.objectContaining({ mode: "update", trigger: "auto" })]); + const opt = enqueueOpts.filter((x) => x.task === SCRAPE_TASK).at(-1); + expect(opt?.availableAt?.getTime()).toBeGreaterThanOrEqual(t0 + RESCRAPE_DELAY_MS - 1000); + + // 防抖:还没开始时再来一次刷新(哪怕字段又变了)不另建记录、不另排任务 + await h.db + .updateTable("metadata_field") + .set({ value: JSON.stringify("旧值") }) + .where("media_entity_id", "=", id) + .where("field", "=", "overview") + .execute(); + const scrapes = h.engine.enqueued.filter((x) => x.task === SCRAPE_TASK).length; + await refreshMetadata(h.rt, id, { full: true }); + expect(await queuedUpdates()).toHaveLength(1); + expect(h.engine.enqueued.filter((x) => x.task === SCRAPE_TASK).length).toBe(scrapes); + }, 60_000); + + it("重刮后 NFO 的标题 / 简介 / IMDb / 海报都是沙丘 2,不残留奥本海默;锁定字段照写", async () => { + const [q] = await queuedUpdates(); + const rec = await runScrapeRecord(d, q!.id); + expect(rec.status).toBe("succeeded"); + expect(rec.files.find((f) => f.role === "movie_nfo")).toMatchObject({ action: "written" }); + const nfo = await readFile(join(lib.root, NFO), "utf8"); + expect(nfo).toContain("沙丘2"); + expect(nfo).toContain("693134"); + expect(nfo).toContain("tt15239678"); + expect(nfo).not.toContain("tt15398776"); + expect(nfo).not.toContain("奥本海默"); + expect(nfo).not.toContain("原子弹之父"); + expect(nfo).toContain("我的标语"); + expect(await readFile(join(lib.root, DIR, "poster.jpg"), "utf8")).toBe("/dune.jpg"); + + // 字段没变的刷新不再排重刮 + const scrapes = h.engine.enqueued.filter((x) => x.task === SCRAPE_TASK).length; + await refreshMetadata(h.rt, id, { full: true }); + expect(h.engine.enqueued.filter((x) => x.task === SCRAPE_TASK).length).toBe(scrapes); + }, 60_000); +});