diff --git a/apps/server/src/modules/acquisition-organize/services/scrape-input.ts b/apps/server/src/modules/acquisition-organize/services/scrape-input.ts index 84857c27..dd4c0a97 100644 --- a/apps/server/src/modules/acquisition-organize/services/scrape-input.ts +++ b/apps/server/src/modules/acquisition-organize/services/scrape-input.ts @@ -1,5 +1,5 @@ import type { TmdbRichDetail, TmdbRichSeasonDetail } from "@umt/adapters"; -import type { NfoEpisode, NfoIds, NfoPerson, NfoSeason, NfoWork } from "@umt/domain"; +import { type NfoEpisode, type NfoIds, type NfoPerson, type NfoSeason, type NfoWork, scrapeKindOf } from "@umt/domain"; import { richEntityFields } from "../../sources-identity/service/metadata-rich.ts"; // SCR-1:本地作品 / 季 / 集的最终字段(media_entity.metadata、content_unit.metadata,字段级 {value, …})→ NFO 输入。 @@ -71,7 +71,7 @@ export function nfoWork(e: EntityForNfo, ids: IdentityMap): NfoWork { douban: ids.douban ?? null, }; const crew = credits(val(m, "crew")); - const isTv = e.kind !== "movie"; + const isTv = scrapeKindOf(e.kind) === "tv"; return { kind: isTv ? "tvshow" : "movie", title: str(val(m, "title")) ?? e.title, @@ -136,6 +136,22 @@ export function nfoEpisode(season: number, episode: number, u: UnitForNfo | unde }; } +/** 单元被元数据来源填过(名称 / 简介 / 日期至少有一个字段,值为空也算:TMDB 就是没有,刷新也补不出来) */ +const filled = (m: unknown) => !!m && typeof m === "object" && ["name", "overview", "airDate"].some((k) => k in m); + +/** + * 季单元缺元数据(没有单元,或从没被元数据来源填过、也没有名称):不写「季 N」占位 NFO。 + * 「缺」指还没取到(整理懒建、改身份后清空),不是 TMDB 本身没写简介。 + */ +export function seasonMetadataMissing(u: UnitForNfo | undefined): boolean { + return !u || (!filled(u.metadata) && !str(u.title)); +} + +/** 集单元缺元数据(没有单元,或从没被填过、也没有标题和播出日期):不写「第 N 集」占位 NFO */ +export function episodeMetadataMissing(u: UnitForNfo | undefined): boolean { + return !u || (!filled(u.metadata) && !str(u.title) && !u.air_date); +} + /** 图片路径:作品海报 / 背景图,季海报,集剧照 */ export function imagePathsOf(entityMeta: unknown) { return { diff --git a/apps/server/src/modules/acquisition-organize/services/scrape.ts b/apps/server/src/modules/acquisition-organize/services/scrape.ts index 1b028829..11ca7fe3 100644 --- a/apps/server/src/modules/acquisition-organize/services/scrape.ts +++ b/apps/server/src/modules/acquisition-organize/services/scrape.ts @@ -1,5 +1,5 @@ -import { randomUUID } from "node:crypto"; -import { lstat, mkdir, readdir, realpath, rename, rm, writeFile } from "node:fs/promises"; +import { createHash, randomUUID } from "node:crypto"; +import { lstat, mkdir, readdir, readFile, realpath, rename, rm, writeFile } from "node:fs/promises"; import { dirname, isAbsolute, join, relative, resolve } from "node:path"; import { checkRoot, DEFAULT_ROOT_SENTINEL } from "@umt/adapters"; import { @@ -19,6 +19,7 @@ import { planScrapeTargets, type ScrapeTarget, type ScrapeVideo, + scrapeKindOf, seasonNfo, seasonOfDirName, workNfo, @@ -31,11 +32,13 @@ import { errMessage, jsonb } from "../util.ts"; import type { AoDeps } from "./common.ts"; import { type EntityForNfo, + episodeMetadataMissing, type IdentityMap, imagePathsOf, nfoEpisode, nfoSeason, nfoWork, + seasonMetadataMissing, tmdbEntity, tmdbSeasonUnits, type UnitForNfo, @@ -51,7 +54,10 @@ import { // 剧集目录 → tvshow.nfo + 海报 / 背景图 + 季目录(Season 1 / S01 / Specials)的 season.nfo 与季海报 + 每集 NFO 与剧照; // 季目录 → season.nfo + 季海报(季目录 poster.jpg 与上级 seasonNN-poster.jpg)+ 每集;单集文件 → 该集 NFO 与剧照。 // - 字段:有本地作品且有元数据时用最终字段(字段锁、人工覆盖优先);否则按 TMDB id 直接取 TMDB(与刷新元数据同一映射)。 -// - 覆盖:missing 只补缺失 / overwrite 全部覆盖;写入走同目录临时文件 + rename;目标必须在登记目录内。 +// - 覆盖:update(缺省)只覆盖本工具上次写入且没被改过的文件(逐文件 sha256 记在 scrape_record.files, +// 改过的记 scrape_user_modified)/ missing 只补缺失 / overwrite 全部覆盖;写入走同目录临时文件 + rename; +// 目标必须在登记目录内。 +// - 季 / 集缺元数据时先按 TMDB id 现取季详情补齐,仍缺就跳过(scrape_metadata_missing,记录 partial),不写占位 NFO。 // - 每次刮削一条 scrape_record(目录、识别结果、逐文件结果、原因);审计 organize.scrape;SSE files.changed。 export const SCRAPE_TASK = "scrapeMedia"; @@ -59,7 +65,10 @@ const IMAGE_CONCURRENCY = 4; const DEDUPE_MS = 30 * 60_000; const TEMP_RE = /(\.uhd-partial|\.!qB|\.part)$/i; -export async function scrapeSettings(d: AoDeps): Promise { +/** 排队一侧只用到 ctx(元数据刷新后自动重刮从 sources-identity 调用) */ +type QueueDeps = Pick; + +export async function scrapeSettings(d: QueueDeps): Promise { const s = await d.ctx.settings.get("organize").catch(() => DEFAULT_SETTINGS.organize); return { ...DEFAULT_SETTINGS.organize, ...s }; } @@ -83,6 +92,8 @@ export interface QueueScrapeInput extends ScrapeLocation { idempotencyKey?: string | undefined; /** 只刮这些视频(库根相对路径;整理后刮削用,作品 / 季目录元数据照常初始化);缺省 = 目录下全部 */ videos?: readonly string[] | undefined; + /** 任务最早开始时间(元数据刷新后的自动重刮延后执行,期间再来的刷新合进同一条排队记录) */ + availableAt?: Date | undefined; } type Row = { @@ -158,7 +169,7 @@ export async function latestEntityScrape(d: AoDeps, entityId: string): Promise { const db = d.ctx.db; @@ -212,7 +223,11 @@ export async function queueScrape( queued = await d.ctx.engine.enqueue( SCRAPE_TASK, { recordId: rec.id }, - { idempotencyKey: input.idempotencyKey ?? `${SCRAPE_TASK}:${rec.id}`, traceId: randomUUID() }, + { + idempotencyKey: input.idempotencyKey ?? `${SCRAPE_TASK}:${rec.id}`, + traceId: randomUUID(), + ...(input.availableAt ? { availableAt: input.availableAt } : {}), + }, ); } catch (err) { // 只有新记录的入队失败才标失败;复用的记录可能已有接受但未关联的 run。 @@ -259,11 +274,9 @@ export async function queueScrape( // ── 作品 → 目录(作品详情「重新刮削」、入库后自动刮削) ── -const isMovieKind = (k: string) => k === "movie" || k === "collection" || k === "other"; - -/** 本地作品 → 识别结果(带已接受的 TMDB id) */ +/** 本地作品 → 识别结果(带已接受的、与作品类型一致的 TMDB id) */ export async function entityScrapeMatch( - d: AoDeps, + d: QueueDeps, entityId: string, source: ScrapeMatch["source"], ): Promise { @@ -273,12 +286,15 @@ export async function entityScrapeMatch( .where("id", "=", entityId) .executeTakeFirst(); if (!e || e.status === "merged") return null; + const kind = scrapeKindOf(e.kind); + // 同一作品可同时有 movie 与 tv 两个 accepted 映射:只取与作品类型一致的那个 const tmdb = await d.ctx.db .selectFrom("external_identity") .select(["external_id"]) .where("media_entity_id", "=", entityId) .where("content_unit_id", "is", null) .where("provider", "=", "tmdb") + .where("entity_type", "=", kind) .where("status", "=", "accepted") .orderBy("updated_at", "desc") .executeTakeFirst(); @@ -286,7 +302,7 @@ export async function entityScrapeMatch( mediaEntityId: e.id, provider: tmdb ? "tmdb" : null, externalId: tmdb?.external_id ?? null, - kind: isMovieKind(e.kind) ? "movie" : "tv", + kind, title: e.title, year: e.year, source, @@ -298,7 +314,7 @@ export async function entityScrapeMatch( * 推不出(视频直接在库根)时对视频文件本身刮削。只取启用的库根里 active 视频的位置。 */ export async function entityScrapeLocations( - d: AoDeps, + d: QueueDeps, entityId: string, kind: "movie" | "tv", ): Promise { @@ -348,13 +364,14 @@ export async function entityScrapeLocations( /** 对作品的目录刮削(每个目录一条记录);没有已入库文件时返回空 */ export async function scrapeEntity( - d: AoDeps, + d: QueueDeps, entityId: string, opts: { mode?: ScrapeMode | undefined; trigger: "auto" | "manual"; organizationRecordId?: string | undefined; actor?: { type: string; id: string | null } | undefined; + availableAt?: Date | undefined; }, ): Promise<{ recordId: string; taskId: string | null; deduplicated: boolean }[]> { const match = await entityScrapeMatch(d, entityId, "library"); @@ -370,6 +387,7 @@ export async function scrapeEntity( trigger: opts.trigger, organizationRecordId: opts.organizationRecordId, actor: opts.actor, + availableAt: opts.availableAt, idempotencyKey: opts.organizationRecordId ? `${SCRAPE_TASK}:record:${opts.organizationRecordId}:${i}` : undefined, @@ -378,6 +396,31 @@ export async function scrapeEntity( return out; } +/** 元数据刷新后自动重刮的延迟:这段时间内再来的刷新合进同一条排队记录(防抖) */ +export const RESCRAPE_DELAY_MS = 2 * 60_000; + +/** + * 元数据刷新(或改身份后的刷新)改变了作品的 NFO 字段:对刮削过的作品(有成功 / 部分成功的刮削记录)排一次 + * 「更新」模式重刮,延后 RESCRAPE_DELAY_MS 执行;还没开始的同目录更新刮削直接复用(queueScrape 合批)。 + * 返回排上(含复用)的记录数;没刮削过的作品不排。 + */ +export async function scheduleEntityRescrape(d: QueueDeps, entityId: string): Promise { + const scraped = await d.ctx.db + .selectFrom("scrape_record") + .select("id") + .where("media_entity_id", "=", entityId) + .where("status", "in", ["succeeded", "partial"]) + .limit(1) + .executeTakeFirst(); + if (!scraped) return 0; + const out = await scrapeEntity(d, entityId, { + mode: "update", + trigger: "auto", + availableAt: new Date(Date.now() + RESCRAPE_DELAY_MS), + }); + return out.length; +} + /** 整理入库成功后(设置打开或本次要求刮削):照 MP 对作品目录刮削,集只刮本次整理的文件。 */ export async function scrapeAfterImport( d: AoDeps, @@ -552,6 +595,50 @@ async function coverageOf( return out; } +const sha256 = (c: Uint8Array | string) => createHash("sha256").update(c).digest("hex"); +/** 旧版本写的文件没有 hash:修改时间不晚于那次刮削结束(加这点余量)就当没被改过 */ +const LEGACY_MTIME_SLACK_MS = 2_000; +const missingMeta = (t: ScrapeTarget): ScrapeFileResult => ({ + path: t.path, + kind: t.kind, + role: t.role, + action: "skipped", + reason: "scrape_metadata_missing", +}); + +interface PrevWrite { + /** 写入时的 sha256;旧版本记录没有 */ + hash: string | null; + /** 那次刮削结束时间(毫秒) */ + at: number; +} + +/** + * 同一登记目录里以前的刮削记录 → 本工具最近一次写入(或核对未变)的文件:路径 → hash。 + * 从新到旧取每个路径第一条带 hash 的结果;旧版本只有 written、没有 hash 的按结束时间兜底。 + */ +async function previousWrites(d: AoDeps, rootId: string, exceptId: string): Promise> { + const rows = await d.ctx.db + .selectFrom("scrape_record") + .select(["files", "finished_at"]) + .where("root_id", "=", rootId) + .where("id", "<>", exceptId) + .where("finished_at", "is not", null) + .orderBy("finished_at", "desc") + .limit(500) + .execute(); + const out = new Map(); + for (const r of rows) { + const at = new Date(r.finished_at!).getTime(); + for (const f of (r.files ?? []) as ScrapeFileResult[]) { + if (out.has(f.path)) continue; + if (f.hash) out.set(f.path, { hash: f.hash, at }); + else if (f.action === "written") out.set(f.path, { hash: null, at }); + } + } + return out; +} + interface Fields { entity: EntityForNfo; ids: IdentityMap; @@ -579,6 +666,8 @@ export async function runScrapeRecord( const mode = rec.mode as ScrapeMode; let match = (rec.match ?? null) as ScrapeMatch | null; const dirs = new Set(); + /** 逐文件结果;兜底异常时也保留已经写了的 */ + const files: ScrapeFileResult[] = []; const finish = async (r: { status: ScrapeRecord["status"]; @@ -681,7 +770,7 @@ export async function runScrapeRecord( const ids: IdentityMap = {}; const idRows = await db .selectFrom("external_identity") - .select(["provider", "external_id"]) + .select(["provider", "external_id", "entity_type"]) .where("media_entity_id", "=", e.id) .where("content_unit_id", "is", null) .where("status", "=", "accepted") @@ -689,6 +778,8 @@ export async function runScrapeRecord( .execute(); for (const r of idRows) { const p = r.provider.toLowerCase() as keyof IdentityMap; + // TMDB 映射只认与刮削类型一致的(电影 / 剧集各一个时不串) + if (p === "tmdb" && r.entity_type !== kind) continue; if (["tmdb", "imdb", "tvdb", "douban"].includes(p) && !ids[p]) ids[p] = r.external_id; } if (!match.externalId && ids.tmdb) match = { ...match, provider: "tmdb", externalId: ids.tmdb }; @@ -757,6 +848,7 @@ export async function runScrapeRecord( } // ── 没有本地字段:按 TMDB 条目直接取 ── + let fromTmdb = false; if (!fields) { if (!match.externalId) return await fail("scrape_no_metadata"); let detail: Awaited>; @@ -770,22 +862,38 @@ export async function runScrapeRecord( const t = tmdbEntity(detail); fields = { entity: t.entity, ids: t.ids, seasonUnit: new Map(), episodeUnit: new Map() }; match = { ...match, provider: "tmdb", title: detail.title, year: detail.year }; - if (kind === "tv") { - const seasons = [...new Set(videos.map((v) => v.season).filter((s): s is number => typeof s === "number"))]; - for (const n of seasons) { - const sd = await d.ports.scrapeTmdbSeason(match.externalId!, n).catch((err) => { - d.ctx.logger.warn({ recordId, season: n, err: { message: errMessage(err) } }, "刮削取 TMDB 季详情失败"); - return null; - }); - if (!sd) continue; - const u = tmdbSeasonUnits(sd); - fields.seasonUnit.set(n, u.season); - for (const [ep, eu] of u.episodes) fields.episodeUnit.set(`${n}:${ep}`, eu); + fromTmdb = true; + } + + // ── 季 / 集元数据:TMDB 分支取每个涉及的季;本地分支只在季 / 集单元缺元数据时按 TMDB id 现取补齐 ── + const { seasonUnit, episodeUnit } = fields; + if (kind === "tv") { + const tmdbId = match.externalId ?? fields.ids.tmdb ?? null; + const seasons = [...new Set(videos.map((v) => v.season).filter((s): s is number => typeof s === "number"))]; + for (const n of seasons.sort((a, b) => a - b)) { + const lacking = + (lay.parts.season && seasonMetadataMissing(seasonUnit.get(n))) || + videos.some( + (v) => + v.season === n && + typeof v.episode === "number" && + episodeMetadataMissing(episodeUnit.get(`${n}:${v.episode}`)), + ); + if (!tmdbId || (!fromTmdb && !lacking)) continue; + const sd = await d.ports.scrapeTmdbSeason(tmdbId, n).catch((err) => { + d.ctx.logger.warn({ recordId, season: n, err: { message: errMessage(err) } }, "刮削取 TMDB 季详情失败"); + return null; + }); + if (!sd) continue; + const u = tmdbSeasonUnits(sd); + // 本地已有的字段(含人工覆盖)不动,只补缺元数据的单元 + if (fromTmdb || seasonMetadataMissing(seasonUnit.get(n))) seasonUnit.set(n, u.season); + for (const [ep, eu] of u.episodes) { + const k = `${n}:${ep}`; + if (fromTmdb || episodeMetadataMissing(episodeUnit.get(k))) episodeUnit.set(k, eu); } } } - - const { seasonUnit, episodeUnit } = fields; const work = nfoWork(fields.entity, fields.ids); const seasonPosters = new Map(); for (const [n, u] of seasonUnit) seasonPosters.set(n, unitPosterPath(u.metadata)); @@ -800,7 +908,9 @@ export async function runScrapeRecord( options: { images: settings.scrapeImages, episodeThumbs: settings.scrapeEpisodeThumbs }, parts: lay.parts, }); - const files: ScrapeFileResult[] = [...noEpisode]; + files.push(...noEpisode); + // 「更新」模式:本工具以前写过的文件(路径 → 写入时的 hash / 时间) + const ours = mode === "update" ? await previousWrites(d, rec.root_id, recordId) : new Map(); const imageCache = new Map>(); const fetchImage = (p: string) => { let hit = imageCache.get(p); @@ -810,21 +920,37 @@ export async function runScrapeRecord( } return hit; }; - const decide = async (t: ScrapeTarget): Promise<{ abs: string } | ScrapeFileResult> => { + type Decision = { abs: string; prevHash: string | null } | ScrapeFileResult; + const decide = async (t: ScrapeTarget): Promise => { const target = resolve(rootReal, t.path); const base = { path: t.path, kind: t.kind, role: t.role }; if (!inside(rootReal, target)) return { ...base, action: "failed", reason: "path_escape" }; const ex = await lstat(target).catch(() => null); - if (ex && (!ex.isFile() || ex.isSymbolicLink())) return { ...base, action: "skipped", reason: "scrape_exists" }; - if (ex && mode === "missing") return { ...base, action: "skipped", reason: "scrape_exists" }; - return { abs: target }; + if (!ex) return { abs: target, prevHash: null }; + if (!ex.isFile() || ex.isSymbolicLink()) return { ...base, action: "skipped", reason: "scrape_exists" }; + if (mode === "missing") return { ...base, action: "skipped", reason: "scrape_exists" }; + if (mode === "overwrite") return { abs: target, prevHash: null }; + // update:只覆盖本工具写过且之后没被改过的 + const prev = ours.get(t.path); + if (!prev) return { ...base, action: "skipped", reason: "scrape_exists" }; + const cur = sha256(await readFile(target)); + const untouched = prev.hash ? prev.hash === cur : ex.mtimeMs <= prev.at + LEGACY_MTIME_SLACK_MS; + if (!untouched) return { ...base, action: "skipped", reason: "scrape_user_modified" }; + return { abs: target, prevHash: cur }; }; - const write = async (t: ScrapeTarget, target: string, content: Uint8Array | string): Promise => { + const write = async ( + t: ScrapeTarget, + dec: { abs: string; prevHash: string | null }, + content: Uint8Array | string, + ): Promise => { const base = { path: t.path, kind: t.kind, role: t.role }; + const hash = sha256(content); + // 更新模式下内容没变就不重写(不打扰媒体服务器) + if (dec.prevHash === hash) return { ...base, action: "skipped", reason: "scrape_unchanged", hash }; try { - await atomicWrite(target, content); + await atomicWrite(dec.abs, content); dirs.add(dirOf(t.path)); - return { ...base, action: "written", reason: null }; + return { ...base, action: "written", reason: null, hash }; } catch (err) { d.ctx.logger.warn({ recordId, path: t.path, err: { message: errMessage(err) } }, "刮削文件写入失败"); return { ...base, action: "failed", reason: "scrape_write_failed" }; @@ -841,9 +967,22 @@ export async function runScrapeRecord( } let xml: string; if (t.role === "movie_nfo" || t.role === "tvshow_nfo") xml = workNfo(work); - else if (t.role === "season_nfo") xml = seasonNfo(nfoSeason(t.season!, seasonUnit.get(t.season!))); - else xml = episodeNfo(nfoEpisode(t.season!, t.episode!, episodeUnit.get(`${t.season}:${t.episode}`))); - files.push(await write(t, dec.abs, xml)); + else if (t.role === "season_nfo") { + const u = seasonUnit.get(t.season!); + if (seasonMetadataMissing(u)) { + files.push(missingMeta(t)); + continue; + } + xml = seasonNfo(nfoSeason(t.season!, u)); + } else { + const u = episodeUnit.get(`${t.season}:${t.episode}`); + if (episodeMetadataMissing(u)) { + files.push(missingMeta(t)); + continue; + } + xml = episodeNfo(nfoEpisode(t.season!, t.episode!, u)); + } + files.push(await write(t, dec, xml)); } // 图片:限并发,失败不影响 NFO const imgs = targets.filter((x) => x.kind === "image" && x.imagePath); @@ -859,23 +998,37 @@ export async function runScrapeRecord( d.ctx.logger.warn({ recordId, image: t.imagePath, err: { message: errMessage(err) } }, "刮削图片下载失败"); return { path: t.path, kind: t.kind, role: t.role, action: "failed", reason: "scrape_image_failed" }; } - return write(t, dec.abs, bytes); + return write(t, dec, bytes); })), ); const written = files.filter((f) => f.action === "written").length; const failed = files.filter((f) => f.action === "failed").length; + // 缺元数据跳过的季 / 集不算成功(要刷新元数据后再刮) + const lacking = files.some((f) => f.reason === "scrape_metadata_missing"); const status: ScrapeRecord["status"] = - failed === 0 ? "succeeded" : written > 0 || files.some((f) => f.action === "skipped") ? "partial" : "failed"; + failed === 0 + ? lacking + ? "partial" + : "succeeded" + : written > 0 || files.some((f) => f.action === "skipped") + ? "partial" + : "failed"; return await finish({ status, - reason: failed ? (files.find((f) => f.action === "failed")?.reason ?? null) : null, + reason: failed + ? (files.find((f) => f.action === "failed")?.reason ?? null) + : lacking + ? "scrape_metadata_missing" + : null, files, }); } catch (err) { if (signal?.aborted) throw err; d.ctx.logger.warn({ recordId, err: { message: errMessage(err) } }, "刮削失败"); - return fail("scrape_write_failed"); + // 不是写文件的错(如查库失败):记内部错误,已写的文件结果照留 + const anyWritten = files.some((f) => f.action === "written"); + return finish({ status: anyWritten ? "partial" : "failed", reason: "scrape_internal_error", files }); } } diff --git a/apps/server/test/acquisition-organize/scrape-accuracy.test.ts b/apps/server/test/acquisition-organize/scrape-accuracy.test.ts new file mode 100644 index 00000000..68a8ce28 --- /dev/null +++ b/apps/server/test/acquisition-organize/scrape-accuracy.test.ts @@ -0,0 +1,327 @@ +import { readFile, stat, utimes, writeFile } from "node:fs/promises"; +import type { TmdbRichSeasonDetail } from "@umt/adapters"; +import type { ScrapeFileResult, ScrapeMatch } from "@umt/contracts"; +import { createTempTree, type TempTree } from "@umt/testkit"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { queueScrape, runScrapeRecord } from "../../src/modules/acquisition-organize/services/scrape.ts"; +import { nfoWork } from "../../src/modules/acquisition-organize/services/scrape-input.ts"; +import { createHarness, type Harness, pgAvailable, seedBase } from "./harness.ts"; + +// 刮削准确度: +// - 「更新」模式(缺省):不存在的写入;本工具上次写入且没被改过的按新内容覆盖(内容没变不重写);被人改过的跳过 +// (scrape_user_modified);不是本工具写的不动;旧版本没有 hash 的记录按修改时间兜底; +// - 集 / 季缺元数据:先按 TMDB id 现取季详情补齐;取不到跳过(scrape_metadata_missing)、记录 partial,不写占位 NFO; +// - 合集 / 其他类作品按电影刮,NFO 根元素是 ; +// - 兜底异常记 scrape_internal_error。 + +const hasPg = await pgAvailable(); +const at = new Date().toISOString(); +const f = (value: unknown, language: string | null = null) => ({ value, source: "tmdb", language, at, locked: false }); +const enc = (s: string) => new TextEncoder().encode(s); + +function season1(names: Record): TmdbRichSeasonDetail { + return { + tvId: "95396", + seasonNumber: 1, + name: "第 1 季", + nameLanguage: "zh-CN", + overview: null, + overviewLanguage: null, + airDate: "2022-02-17", + posterPath: null, + fetchedLanguages: ["zh-CN"], + episodes: Object.entries(names).map(([n, name]) => ({ + seasonNumber: 1, + episodeNumber: Number(n), + name, + nameLanguage: "zh-CN", + overview: `${name}的简介`, + overviewLanguage: "zh-CN", + airDate: "2022-02-25", + runtime: 50, + stillPath: null, + voteAverage: null, + voteCount: null, + })), + }; +} + +describe.skipIf(!hasPg)("刮削准确度:更新模式、缺元数据不写占位、作品类型", () => { + let h: Harness; + let lib: TempTree; + let rootId = ""; + let seasonDetail: (n: number) => Promise = async () => null; + const seasonCalls: number[] = []; + + beforeAll(async () => { + lib = await createTempTree( + { + "Movies/星际穿越 (2014)/Interstellar.2014.mkv": 100, + "Movies/星际穿越 (2014)/fanart.jpg": "别的工具放的背景图", + "TV/人生切割术 (2022)/Season 1/Severance.S01E01.mkv": 100, + "TV/人生切割术 (2022)/Season 1/Severance.S01E02.mkv": 100, + "Movies/合集 (2020)/Collection.2020.mkv": 100, + "Movies/其他 (2021)/Other.2021.mkv": 100, + }, + { sentinel: true, prefix: "umt-scracc-" }, + ); + h = await createHarness("scrape_acc", { + scrapeImage: async (p) => ({ content: enc(p), mime: "image/jpeg" }), + scrapeTmdbSeason: async (_tv, n) => { + seasonCalls.push(n); + return seasonDetail(n); + }, + }); + rootId = (await seedBase(h.db, { libraryPath: lib.root, cachePath: lib.root, qbUrl: "http://127.0.0.1:9" })).rootId; + }, 60_000); + afterAll(async () => { + await h?.close(); + await lib?.cleanup(); + }); + + async function entity(kind: "movie" | "series" | "collection" | "other", title: string, meta: object) { + return ( + await h.db + .insertInto("media_entity") + .values({ kind, title, year: 2020, status: "active", metadata: JSON.stringify(meta) }) + .returning("id") + .executeTakeFirstOrThrow() + ).id; + } + async function scrape( + entityId: string, + kind: "movie" | "tv", + path: string, + mode?: "missing" | "overwrite" | "update", + externalId: string | null = null, + ) { + const match: ScrapeMatch = { + mediaEntityId: entityId, + provider: externalId ? "tmdb" : null, + externalId, + kind, + title: "x", + year: null, + source: "library", + }; + const q = await queueScrape(h.d, { + rootId: `lib:${rootId}`, + rootPath: lib.root, + libraryRootId: rootId, + path, + match, + mode, + trigger: "manual", + }); + return runScrapeRecord(h.d, q.recordId); + } + const fileOf = (files: ScrapeFileResult[], role: string) => files.find((x) => x.role === role); + const setTitle = (id: string, title: string) => + h.db + .updateTable("media_entity") + .set({ metadata: JSON.stringify({ title: f(title), posterPath: f("/inter.jpg") }) }) + .where("id", "=", id) + .execute(); + + const MOVIE_DIR = "Movies/星际穿越 (2014)"; + const MOVIE_NFO = `${MOVIE_DIR}/Interstellar.2014.nfo`; + let movie = ""; + + it("更新模式:不存在的写入并记 hash;不是本工具写的不动", async () => { + movie = await entity("movie", "星际穿越", { title: f("星际穿越"), posterPath: f("/inter.jpg") }); + const r = await scrape(movie, "movie", MOVIE_DIR); + expect(r).toMatchObject({ status: "succeeded", mode: "update" }); + expect(fileOf(r.files, "movie_nfo")).toMatchObject({ + action: "written", + hash: expect.stringMatching(/^[0-9a-f]{64}$/), + }); + expect(fileOf(r.files, "poster")).toMatchObject({ action: "written" }); + expect(await readFile(lib.path(MOVIE_NFO), "utf8")).toContain("星际穿越"); + // fanart.jpg 是别的工具放的:元数据没有背景图,不在目标里;加上背景图后更新模式也不覆盖它 + await h.db + .updateTable("media_entity") + .set({ + metadata: JSON.stringify({ title: f("星际穿越"), posterPath: f("/inter.jpg"), backdropPath: f("/bg.jpg") }), + }) + .where("id", "=", movie) + .execute(); + const again = await scrape(movie, "movie", MOVIE_DIR, "update"); + expect(fileOf(again.files, "fanart")).toMatchObject({ action: "skipped", reason: "scrape_exists" }); + expect(await readFile(lib.path(`${MOVIE_DIR}/fanart.jpg`), "utf8")).toBe("别的工具放的背景图"); + // 内容没变的不重写 + expect(fileOf(again.files, "movie_nfo")).toMatchObject({ action: "skipped", reason: "scrape_unchanged" }); + expect(fileOf(again.files, "poster")).toMatchObject({ action: "skipped", reason: "scrape_unchanged" }); + expect(again.status).toBe("succeeded"); + }); + + it("更新模式:本工具写的、没被改过的按新内容覆盖;被人改过的跳过并记原因", async () => { + await setTitle(movie, "星际穿越(新译名)"); + const upd = await scrape(movie, "movie", MOVIE_DIR, "update"); + expect(fileOf(upd.files, "movie_nfo")).toMatchObject({ action: "written" }); + expect(await readFile(lib.path(MOVIE_NFO), "utf8")).toContain("星际穿越(新译名)"); + + await writeFile(lib.path(MOVIE_NFO), "我手改的"); + await setTitle(movie, "星际穿越(再改)"); + const kept = await scrape(movie, "movie", MOVIE_DIR, "update"); + expect(fileOf(kept.files, "movie_nfo")).toMatchObject({ action: "skipped", reason: "scrape_user_modified" }); + expect(await readFile(lib.path(MOVIE_NFO), "utf8")).toBe("我手改的"); + expect(kept.status).toBe("succeeded"); + // 之后每次更新都还认得出是人改过的(跳过的结果不带 hash,不会把人改的内容当成本工具写的) + const still = await scrape(movie, "movie", MOVIE_DIR, "update"); + expect(fileOf(still.files, "movie_nfo")).toMatchObject({ action: "skipped", reason: "scrape_user_modified" }); + // 全部覆盖照旧改写 + const over = await scrape(movie, "movie", MOVIE_DIR, "overwrite"); + expect(fileOf(over.files, "movie_nfo")).toMatchObject({ action: "written" }); + expect(await readFile(lib.path(MOVIE_NFO), "utf8")).toContain("星际穿越(再改)"); + }); + + it("更新模式:旧版本记录没有 hash 时按修改时间判断", async () => { + // 模拟旧版本:去掉所有记录里的 hash + const rows = await h.db.selectFrom("scrape_record").select(["id", "files", "finished_at"]).execute(); + for (const r of rows) + await h.db + .updateTable("scrape_record") + .set({ + files: JSON.stringify((r.files as ScrapeFileResult[]).map(({ hash: _h, ...x }) => x)), + }) + .where("id", "=", r.id) + .execute(); + const last = rows + .filter((r) => r.finished_at) + .sort((a, b) => +new Date(b.finished_at!) - +new Date(a.finished_at!))[0]!; + const past = new Date(+new Date(last.finished_at!) - 60_000); + await utimes(lib.path(MOVIE_NFO), past, past); + await setTitle(movie, "星际穿越(旧记录)"); + const r = await scrape(movie, "movie", MOVIE_DIR, "update"); + expect(fileOf(r.files, "movie_nfo")).toMatchObject({ action: "written" }); + expect(await readFile(lib.path(MOVIE_NFO), "utf8")).toContain("星际穿越(旧记录)"); + + // 改动时间晚于那次刮削:当作人改过 + const rows2 = await h.db.selectFrom("scrape_record").select(["id", "files"]).execute(); + for (const x of rows2) + await h.db + .updateTable("scrape_record") + .set({ files: JSON.stringify((x.files as ScrapeFileResult[]).map(({ hash: _h, ...y }) => y)) }) + .where("id", "=", x.id) + .execute(); + const future = new Date(Date.now() + 60_000); + await utimes(lib.path(MOVIE_NFO), future, future); + expect((await stat(lib.path(MOVIE_NFO))).mtimeMs).toBeGreaterThan(Date.now()); + await setTitle(movie, "星际穿越(不该写进去)"); + const kept = await scrape(movie, "movie", MOVIE_DIR, "update"); + expect(fileOf(kept.files, "movie_nfo")).toMatchObject({ action: "skipped", reason: "scrape_user_modified" }); + }); + + const SHOW = "TV/人生切割术 (2022)"; + let series = ""; + + it("集单元没有元数据:按 TMDB id 现取季详情补齐,写出正确的集 NFO", async () => { + series = await entity("series", "人生切割术", { + title: f("人生切割术"), + overview: f("马克领导着一个办公室团队。"), + }); + const season = await h.db + .insertInto("content_unit") + .values({ + media_entity_id: series, + kind: "season", + season_number: 1, + status: "confirmed", + title: "第 1 季", + metadata: JSON.stringify({ name: f("第 1 季") }), + }) + .returning("id") + .executeTakeFirstOrThrow(); + await h.db + .insertInto("content_unit") + .values({ + media_entity_id: series, + parent_unit_id: season.id, + kind: "episode", + season_number: 1, + episode_number: 1, + status: "confirmed", + title: "有关地狱的好消息", + metadata: JSON.stringify({ name: f("有关地狱的好消息"), overview: f("第一集简介") }), + }) + .execute(); + // 第 2 集是整理懒建的空单元 + await h.db + .insertInto("content_unit") + .values({ + media_entity_id: series, + parent_unit_id: season.id, + kind: "episode", + season_number: 1, + episode_number: 2, + status: "provisional", + }) + .execute(); + seasonDetail = async (n) => (n === 1 ? season1({ 1: "TMDB 第一集", 2: "半循环" }) : null); + const r = await scrape(series, "tv", SHOW, "update", "95396"); + expect(seasonCalls).toEqual([1]); + expect(r.status).toBe("succeeded"); + const ep2 = await readFile(lib.path(`${SHOW}/Season 1/Severance.S01E02.nfo`), "utf8"); + expect(ep2).toContain("半循环"); + expect(ep2).toContain(""); + // 本地已有的集字段不被 TMDB 现取的值替换 + expect(await readFile(lib.path(`${SHOW}/Season 1/Severance.S01E01.nfo`), "utf8")).toContain( + "有关地狱的好消息", + ); + }); + + it("TMDB 季详情失败:缺元数据的集跳过(scrape_metadata_missing)、记录 partial,不写占位 NFO", async () => { + await writeFile(lib.path(`${SHOW}/Season 1/Severance.S01E03.mkv`), "video"); + seasonDetail = async () => { + throw new Error("TMDB 超时"); + }; + const r = await scrape(series, "tv", SHOW, "update", "95396"); + expect(r).toMatchObject({ status: "partial", reason: "scrape_metadata_missing" }); + expect(r.files.find((x) => x.path.endsWith("S01E03.nfo"))).toMatchObject({ + action: "skipped", + reason: "scrape_metadata_missing", + }); + await expect(stat(lib.path(`${SHOW}/Season 1/Severance.S01E03.nfo`))).rejects.toThrow(); + + // 季单元也没有元数据时同样不写「季 N」占位 + await h.db + .updateTable("content_unit") + .set({ title: null, metadata: JSON.stringify({}) }) + .where("media_entity_id", "=", series) + .where("kind", "=", "season") + .execute(); + const r2 = await scrape(series, "tv", SHOW, "overwrite", "95396"); + expect(fileOf(r2.files, "season_nfo")).toMatchObject({ action: "skipped", reason: "scrape_metadata_missing" }); + expect(r2.status).toBe("partial"); + }); + + it("合集 / 其他类作品按电影刮削,NFO 根元素是 ", async () => { + expect(nfoWork({ kind: "collection", title: "x", original_title: null, year: null, metadata: {} }, {}).kind).toBe( + "movie", + ); + for (const [kind, dir, nfo] of [ + ["collection", "Movies/合集 (2020)", "Collection.2020.nfo"], + ["other", "Movies/其他 (2021)", "Other.2021.nfo"], + ] as const) { + const id = await entity(kind, `${kind} 作品`, { title: f(`${kind} 作品`) }); + const r = await scrape(id, "movie", dir); + expect(r.status).toBe("succeeded"); + const xml = await readFile(lib.path(`${dir}/${nfo}`), "utf8"); + expect(xml).toContain("\n"); + expect(xml).not.toContain(""); + } + }); + + it("兜底异常记 scrape_internal_error(不是写文件失败)", async () => { + h.d.ports.scrapeTmdbSeason = (() => { + throw new Error("意外错误"); + }) as never; + await h.db + .updateTable("content_unit") + .set({ title: null, metadata: JSON.stringify({}) }) + .where("media_entity_id", "=", series) + .execute(); + const r = await scrape(series, "tv", SHOW, "update", "95396"); + expect(r).toMatchObject({ status: "failed", reason: "scrape_internal_error" }); + }); +}); diff --git a/apps/server/test/acquisition-organize/scrape.test.ts b/apps/server/test/acquisition-organize/scrape.test.ts index 7531008e..88e820e4 100644 --- a/apps/server/test/acquisition-organize/scrape.test.ts +++ b/apps/server/test/acquisition-organize/scrape.test.ts @@ -96,7 +96,7 @@ describe.skipIf(!hasPg)("刮削:NFO 与图片写进媒体库目录", () => { } /** 作品详情「重新刮削」的服务端做法:对作品目录排刮削并执行 */ - async function rescrape(entity: string, mode?: "missing" | "overwrite") { + async function rescrape(entity: string, mode?: "missing" | "overwrite" | "update") { const q = await scrapeEntity(h.d, entity, { trigger: "manual", mode }); expect(q.length).toBe(1); return runScrapeRecord(h.d, q[0]!.recordId); @@ -193,7 +193,7 @@ describe.skipIf(!hasPg)("刮削:NFO 与图片写进媒体库目录", () => { expect(first).toMatchObject({ media_entity_id: series, trigger: "auto", - mode: "missing", + mode: "update", // 缺省覆盖方式:更新 organization_record_id: rec, }); diff --git a/apps/server/test/file-manager/organize-in-place.test.ts b/apps/server/test/file-manager/organize-in-place.test.ts index 247e1b52..74cadfc3 100644 --- a/apps/server/test/file-manager/organize-in-place.test.ts +++ b/apps/server/test/file-manager/organize-in-place.test.ts @@ -124,6 +124,12 @@ describe.skipIf(skip)("file manager in-place import", () => { .where("organization_record_id", "=", r.recordId) .execute(); expect(recs).toMatchObject([{ path: title, videos: [ep1] }]); + // 季 / 集单元缺元数据时不写占位 NFO:模拟刷新元数据已填过整理懒建的单元 + await h.db + .updateTable("content_unit") + .set({ metadata: JSON.stringify({ name: { value: "名称", source: "tmdb", language: null, at, locked: false } }) }) + .where("media_entity_id", "=", entity) + .execute(); await runScrapeRecord(h.module.deps().ao, recs[0]!.id); const show = await readdir(join(h.lib, title)); const season = await readdir(join(h.lib, title, "Season 1")); diff --git a/apps/server/test/file-manager/scrape.test.ts b/apps/server/test/file-manager/scrape.test.ts index 4e2a1c62..0fa34f8b 100644 --- a/apps/server/test/file-manager/scrape.test.ts +++ b/apps/server/test/file-manager/scrape.test.ts @@ -277,7 +277,7 @@ describe.skipIf(skip)("files 刮削(SCR-2:任意目录就地刮削)", () => { expect(show).toMatchObject({ status: "succeeded", path: SHOW, - mode: "missing", + mode: "update", trigger: "manual", match: { mediaEntityId: null, diff --git a/packages/contracts/src/reasons.ts b/packages/contracts/src/reasons.ts index 52ca2b65..9efb32b4 100644 --- a/packages/contracts/src/reasons.ts +++ b/packages/contracts/src/reasons.ts @@ -105,7 +105,11 @@ const REASONS: Record = { task_enqueue_failed: { text: "后台任务没排上", action: "稍后重试" }, scrape_no_videos: { text: "目录里没有视频", action: "对作品或季目录刮削" }, scrape_tmdb_unavailable: { text: "TMDB 不可用", action: "检查 TMDB 设置后重新刮削" }, - scrape_exists: { text: "文件已存在，只补缺失时不改" }, + scrape_exists: { text: "文件已存在，不是本工具写的或只补缺失时不改" }, + scrape_unchanged: { text: "内容没有变化" }, + scrape_user_modified: { text: "文件写入后被改过，更新时不覆盖", action: "要替换就选全部覆盖重新刮削" }, + scrape_metadata_missing: { text: "缺少这一季或这一集的元数据，没有写占位文件", action: "刷新元数据后重新刮削" }, + scrape_internal_error: { text: "刮削过程中出错", action: "查看日志后重新刮削" }, scrape_write_failed: { text: "写入目录失败", action: "检查目录权限后重新刮削" }, scrape_image_failed: { text: "图片没下载成功", action: "稍后重新刮削" }, scrape_no_episode: { text: "文件名里认不出集号" }, diff --git a/packages/contracts/src/scrape.ts b/packages/contracts/src/scrape.ts index a8725a39..d5c9ec09 100644 --- a/packages/contracts/src/scrape.ts +++ b/packages/contracts/src/scrape.ts @@ -14,11 +14,16 @@ export const ScrapeFileResult = z.object({ action: z.enum(["written", "skipped", "failed"]), /** 跳过 / 失败原因码(reasons.ts:scrape_exists / scrape_write_failed / scrape_image_failed …) */ reason: z.string().nullable(), + /** 本工具写入(或核对未变)时文件内容的 sha256;「更新」模式据此判断文件之后有没有被人改过 */ + hash: z.string().nullable().optional(), }); export type ScrapeFileResult = z.infer; -/** missing = 只补缺失;overwrite = 全部覆盖 */ -export const ScrapeMode = z.enum(["missing", "overwrite"]); +/** + * missing = 只补缺失;overwrite = 全部覆盖; + * update = 更新(缺省):不存在的写入,本工具上次写入且没被改过的用新内容覆盖,被改过 / 不是本工具写的不动 + */ +export const ScrapeMode = z.enum(["missing", "overwrite", "update"]); export type ScrapeMode = z.infer; export const ScrapeStatus = z.enum(["queued", "running", "succeeded", "partial", "failed"]); diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index be6c8aa0..3f810a2b 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -78,8 +78,11 @@ export const OrganizeSettings = z.object({ minSizeBytes: z.number().int().nonnegative(), /** SCR-1:整理入库成功后自动刮削(往媒体库目录写 NFO 与图片);缺省关(同 MP 目录「刮削」缺省) */ scrapeAfterImport: z.boolean(), - /** SCR-1:目标文件已存在时 missing = 只补缺失(MP「仅缺失」)/ overwrite = 全部覆盖 */ - scrapeMode: z.enum(["missing", "overwrite"]), + /** + * 目标文件已存在时:update = 更新(缺省;只覆盖本工具写过且没被改过的文件)/ missing = 只补缺失(MP「仅缺失」)/ + * overwrite = 全部覆盖 + */ + scrapeMode: z.enum(["missing", "overwrite", "update"]), /** SCR-1:下载海报、背景图、季海报 */ scrapeImages: z.boolean(), /** SCR-1:下载每集剧照(`<视频名>-thumb.jpg`) */ @@ -642,7 +645,7 @@ export const DEFAULT_SETTINGS: GlobalSettings = { allowCrossVolumeMove: true, minSizeBytes: 50 * 1024 * 1024, scrapeAfterImport: false, - scrapeMode: "missing", + scrapeMode: "update", scrapeImages: true, scrapeEpisodeThumbs: true, }, diff --git a/packages/db/migrations/0070_scrape_update_mode.ts b/packages/db/migrations/0070_scrape_update_mode.ts new file mode 100644 index 00000000..0d95c14e --- /dev/null +++ b/packages/db/migrations/0070_scrape_update_mode.ts @@ -0,0 +1,23 @@ +import type { Kysely } from "kysely"; +import { execDdl, oneOf } from "../src/migration-helpers.ts"; + +// 刮削覆盖策略加「更新」(update):只覆盖本工具上次写入且之后没被改过的文件(按 scrape_record.files 里记的 hash 判断), +// 被人改过的跳过(scrape_user_modified)。scrape_record.mode 的 CHECK 增加 update;逐文件 hash 放在 files jsonb 里,不加列。 + +const OLD = ["missing", "overwrite"]; +const NEW = [...OLD, "update"]; + +const swap = (values: readonly string[]) => ` +alter table scrape_record drop constraint if exists scrape_record_mode_check; +alter table scrape_record add constraint scrape_record_mode_check ${oneOf("mode", values)}; +`; + +// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型 +export async function up(db: Kysely): Promise { + await execDdl(db, swap(NEW)); +} + +// biome-ignore lint/suspicious/noExplicitAny: 同上 +export async function down(db: Kysely): Promise { + await execDdl(db, `update scrape_record set mode = 'missing' where mode = 'update'; ${swap(OLD)}`); +} diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index 3f2a61ad..818a267f 100644 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -233,7 +233,7 @@ export type ScheduledJobRunTrigger = "schedule" | "manual" | "skip"; export type ScheduledJobRunStatus = "ok" | "failed" | "skipped" | "running"; export type ScrapeRecordStatus = "queued" | "running" | "succeeded" | "partial" | "failed"; export type ScrapeRecordTrigger = "auto" | "manual"; -export type ScrapeRecordMode = "missing" | "overwrite"; +export type ScrapeRecordMode = "missing" | "overwrite" | "update"; export type SecretPurpose = | "site_cookie" | "site_passkey" diff --git a/packages/domain/src/scrape/layout.ts b/packages/domain/src/scrape/layout.ts index 0b07f80b..69d84a70 100644 --- a/packages/domain/src/scrape/layout.ts +++ b/packages/domain/src/scrape/layout.ts @@ -10,6 +10,14 @@ // 视频直接在库根(没有作品目录)时,电影图片改用 `<视频名>-poster.jpg` / `<视频名>-fanart.jpg`, // 剧集不写 tvshow / 季文件(不能往库根里放 tvshow.nfo)。 +/** + * 作品类型 → 刮削布局与 NFO 根元素:剧集 / 综艺按剧集(tvshow.nfo + 季 / 集),其余(电影、合集、其他)都按电影 + * (`<视频名>.nfo`,根元素 ``)。刮削的范围判定与 NFO 生成都用这一个口径。 + */ +export function scrapeKindOf(entityKind: string): "movie" | "tv" { + return entityKind === "series" || entityKind === "variety" || entityKind === "tv" ? "tv" : "movie"; +} + /** 纯字符串的 POSIX dirname / basename / extname(domain 不依赖 node:path,前端也能打包) */ const posix = { dirname(p: string): string { diff --git a/packages/domain/test/scrape.test.ts b/packages/domain/test/scrape.test.ts index 7b303e91..c9ae6ea3 100644 --- a/packages/domain/test/scrape.test.ts +++ b/packages/domain/test/scrape.test.ts @@ -4,6 +4,7 @@ import { inferWorkDir, isSeasonDirName, planScrapeTargets, + scrapeKindOf, seasonNfo, seasonOfDirName, seasonPosterName, @@ -247,3 +248,10 @@ describe("刮削目标布局", () => { expect(ep.map((x) => x.role)).toEqual(["episode_nfo", "episode_thumb"]); }); }); + +describe("作品类型 → 刮削类型", () => { + it("剧集 / 综艺按剧集,电影 / 合集 / 其他按电影", () => { + expect(["series", "variety", "tv"].map(scrapeKindOf)).toEqual(["tv", "tv", "tv"]); + expect(["movie", "collection", "other"].map(scrapeKindOf)).toEqual(["movie", "movie", "movie"]); + }); +});