diff --git a/apps/server/src/modules/acquisition-organize/__tests__/scrape.test.ts b/apps/server/src/modules/acquisition-organize/__tests__/scrape.test.ts new file mode 100644 index 00000000..df3e75aa --- /dev/null +++ b/apps/server/src/modules/acquisition-organize/__tests__/scrape.test.ts @@ -0,0 +1,299 @@ +import { chmod, readdir, readFile, stat, writeFile } from "node:fs/promises"; +import { join, relative } from "node:path"; +import { createTempTree, type TempTree } from "@umt/testkit"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { executeRecord, planImport } from "../services/organize.ts"; +import { runScrape, SCRAPE_TASK } from "../services/scrape.ts"; +import { createHarness, type Harness, pgAvailable, seedBase, seedEntity } from "./harness.ts"; + +// SCR-1 刮削:入库成功后自动排 scrapeMedia → 写 NFO 与图片(临时库根);只补缺失 / 全部覆盖;图片失败不阻塞 NFO; +// 只登记库根不写;作品详情「重新刮削」接口与刮削状态。 + +const hasPg = await pgAvailable(); +const SIZE = 200_000; +/** 最小 JPEG 头(按文件头认图) */ +const JPEG = new Uint8Array([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10, 0x4a, 0x46, 0x49, 0x46, 0x00]); +const at = new Date().toISOString(); +const f = (value: unknown, language: string | null = null) => ({ value, source: "tmdb", language, at, locked: false }); + +async function tree(root: string): Promise { + const out: string[] = []; + const walk = async (d: string) => { + for (const n of (await readdir(d)).sort()) { + const p = join(d, n); + if ((await stat(p)).isDirectory()) await walk(p); + else out.push(relative(root, p)); + } + }; + await walk(root); + return out.filter((p) => !p.startsWith(".")).sort(); +} + +describe.skipIf(!hasPg)("刮削:NFO 与图片写进媒体库目录", () => { + let h: Harness; + let lib: TempTree; + let src: TempTree; + const fetched: string[] = []; + + beforeAll(async () => { + lib = await createTempTree({}, { sentinel: true, prefix: "umt-scrlib-" }); + src = await createTempTree( + { + "Severance.S01E01.1080p.WEB-DL.H264-GRP.mkv": SIZE, + "Severance.S01E02.1080p.WEB-DL.H264-GRP.mkv": SIZE, + "Interstellar.2014.1080p.BluRay.x264-GRP.mkv": SIZE, + }, + { prefix: "umt-scrsrc-" }, + ); + h = await createHarness("scrape", { + async scrapeImage(p) { + fetched.push(p); + if (p === "/broken.jpg") throw new Error("TMDB 图片返回 HTTP 404"); + return { content: JPEG, mime: "image/jpeg" }; + }, + }); + await seedBase(h.db, { libraryPath: lib.root, cachePath: src.root, qbUrl: "http://127.0.0.1:9" }); + await h.setSettings("organize", { scrapeAfterImport: true }); + }, 60_000); + + afterAll(async () => { + await h?.close(); + for (const t of [lib, src]) await t?.cleanup(); + }); + + async function organize(entity: string, file: string) { + const plan = await planImport(h.d, { + sourcePaths: [src.path(file)], + mediaEntityId: entity, + action: "copy", + trigger: "manual", + actorType: "user", + }); + expect(plan.status).toBe("running"); + const ex = await executeRecord(h.d, plan.recordId!); + expect(ex.status).toBe("succeeded"); + return plan.recordId!; + } + + /** 按引擎记录的入队项跑刮削(假引擎不执行) */ + async function runQueued(entity: string) { + const q = h.engine.enqueued.filter( + (x) => x.task === SCRAPE_TASK && (x.input as { mediaEntityId: string }).mediaEntityId === entity, + ); + expect(q.length).toBeGreaterThan(0); + return runScrape(h.d, q[q.length - 1]!.input as Parameters[1]); + } + + let series = ""; + let movie = ""; + + it("剧集:入库后自动排刮削,写 tvshow / 季 / 集 NFO、海报、季海报、剧照", async () => { + series = await seedEntity(h.db, { kind: "series", title: "人生切割术", year: 2022 }); + await h.db + .updateTable("media_entity") + .set({ + original_title: "Severance", + metadata: JSON.stringify({ + title: f("人生切割术", "zh-CN"), + originalTitle: f("Severance", "en"), + year: f(2022), + releaseDate: f("2022-02-17"), + overview: f("马克领导着一个办公室团队。", "zh-CN"), + genres: f(["剧情", "悬疑"]), + voteAverage: f(8.4), + originCountry: f(["US"]), + platform: f("Apple TV+"), + posterPath: f("/sev.jpg"), + backdropPath: f("/broken.jpg"), + cast: f([ + { id: 4495, name: "亚当·斯科特", originalName: null, role: "Mark Scout", profilePath: "/adam.jpg" }, + ]), + crew: [], + externalIds: f({ imdb: "tt11280740", tvdb: "371980", wikidata: null }), + }), + }) + .where("id", "=", series) + .execute(); + await h.db + .insertInto("external_identity") + .values({ + media_entity_id: series, + provider: "tmdb", + entity_type: "tv", + external_id: "95396", + status: "accepted", + }) + .execute(); + const season = await h.db + .insertInto("content_unit") + .values({ + media_entity_id: series, + kind: "season", + season_number: 1, + status: "confirmed", + title: "第 1 季", + metadata: JSON.stringify({ name: f("第 1 季", "zh-CN"), airDate: f("2022-02-17"), posterPath: f("/s1.jpg") }), + }) + .returning("id") + .executeTakeFirstOrThrow(); + for (const [n, name, still] of [ + [1, "有关地狱的好消息", "/e1.jpg"], + [2, "半循环", null], + ] as const) + await h.db + .insertInto("content_unit") + .values({ + media_entity_id: series, + parent_unit_id: season.id, + kind: "episode", + season_number: 1, + episode_number: n, + status: "confirmed", + title: name, + air_date: "2022-02-17", + metadata: JSON.stringify({ + name: f(name, "zh-CN"), + overview: f(`第 ${n} 集简介`, "zh-CN"), + airDate: f("2022-02-17"), + runtime: f(55), + voteAverage: f(8), + ...(still ? { stillPath: f(still) } : {}), + }), + }) + .execute(); + + const rec = await organize(series, "Severance.S01E01.1080p.WEB-DL.H264-GRP.mkv"); + await organize(series, "Severance.S01E02.1080p.WEB-DL.H264-GRP.mkv"); + const q = h.engine.enqueued.filter((x) => x.task === SCRAPE_TASK); + expect(q[0]!.key).toBe(`${SCRAPE_TASK}:record:${rec}`); + expect(q[0]!.input).toMatchObject({ mediaEntityId: series, trigger: "auto", mode: "missing", recordId: rec }); + + const r = await runQueued(series); + expect(r.status).toBe("partial"); // 背景图下载失败 + const files = await tree(lib.root); + const show = files.find((p) => p.endsWith("tvshow.nfo"))!; + const dir = show.slice(0, -"/tvshow.nfo".length); + const ep1 = files.find((p) => p.includes("S01E01") && p.endsWith(".mkv"))!; + const stem1 = ep1.slice(0, -4); + expect(files).toEqual( + expect.arrayContaining([ + `${dir}/tvshow.nfo`, + `${dir}/poster.jpg`, + `${dir}/season01-poster.jpg`, + `${dir}/Season 1/season.nfo`, + `${dir}/Season 1/poster.jpg`, + `${stem1}.nfo`, + `${stem1}-thumb.jpg`, + ]), + ); + expect(files.some((p) => p.endsWith("fanart.jpg"))).toBe(false); + expect(files.filter((p) => p.endsWith(".nfo"))).toHaveLength(4); + expect(files.some((p) => p.includes(".umt-scrape.tmp"))).toBe(false); + const nfo = await readFile(join(lib.root, show), "utf8"); + expect(nfo.startsWith('\n')).toBe(true); + expect(nfo).toContain('tt11280740'); + expect(nfo).toContain("美国"); + const epNfo = await readFile(join(lib.root, `${stem1}.nfo`), "utf8"); + expect(epNfo).toContain("有关地狱的好消息"); + expect(epNfo).toMatch(/1<\/season>\n {2}1<\/episode>/); + // 同一张季海报只取一次 + expect(fetched.filter((p) => p === "/s1.jpg")).toHaveLength(1); + + const st = await h.api("GET", `/library/${series}`); + expect(st.status).toBe(200); + expect(st.body.data.scrape).toMatchObject({ + status: "partial", + trigger: "auto", + failed: 1, + reason: "scrape_image_failed", + }); + expect(st.body.data.scrape.files.find((x: { role: string }) => x.role === "fanart")).toMatchObject({ + action: "failed", + reason: "scrape_image_failed", + }); + const audit = await h.db + .selectFrom("audit_event") + .select(["action", "target_id"]) + .where("action", "=", "organize.scrape") + .execute(); + expect(audit.some((a) => a.target_id === series)).toBe(true); + }, 60_000); + + it("只补缺失:已有文件(包括别人写的)不动;全部覆盖:改写", async () => { + const files = await tree(lib.root); + const show = join(lib.root, files.find((p) => p.endsWith("tvshow.nfo"))!); + await writeFile(show, "别的工具写的"); + const again = await runScrape(h.d, { mediaEntityId: series, trigger: "manual", mode: "missing" }); + expect(await readFile(show, "utf8")).toBe("别的工具写的"); + expect(again.files.find((x) => x.role === "tvshow_nfo")).toMatchObject({ + action: "skipped", + reason: "scrape_exists", + }); + const over = await runScrape(h.d, { mediaEntityId: series, trigger: "manual", mode: "overwrite" }); + expect(over.written).toBeGreaterThan(0); + expect(await readFile(show, "utf8")).toContain("人生切割术"); + }, 60_000); + + it("电影:<视频名>.nfo + 作品目录 poster / fanart;手动「重新刮削」接口排任务", async () => { + await h.setSettings("organize", { scrapeAfterImport: false }); + movie = await seedEntity(h.db, { kind: "movie", title: "星际穿越", year: 2014 }); + await h.db + .updateTable("media_entity") + .set({ + metadata: JSON.stringify({ + title: f("星际穿越", "zh-CN"), + overview: f("穿越虫洞。", "zh-CN"), + releaseDate: f("2014-11-05"), + runtime: f(169), + posterPath: f("/inter.jpg"), + backdropPath: f("/inter-bg.jpg"), + crew: f([{ id: 525, name: "克里斯托弗·诺兰", originalName: null, role: "Director", profilePath: null }]), + }), + }) + .where("id", "=", movie) + .execute(); + const before = h.engine.enqueued.length; + await organize(movie, "Interstellar.2014.1080p.BluRay.x264-GRP.mkv"); + expect(h.engine.enqueued.slice(before).some((x) => x.task === SCRAPE_TASK)).toBe(false); // 关了自动刮削 + + const res = await h.api("POST", `/library/${movie}/scrape`, {}); + expect(res.status).toBe(202); + expect(res.body.data).toMatchObject({ mediaEntityId: movie, deduplicated: false }); + const dup = await h.api("POST", `/library/${movie}/scrape`, {}); + expect(dup.body.data.deduplicated).toBe(true); + expect((await h.api("GET", `/library/${movie}`)).body.data.scrape.status).toBe("queued"); + + const r = await runQueued(movie); + expect(r.status).toBe("succeeded"); + const files = await tree(lib.root); + const video = files.find((p) => p.includes("星际穿越") && p.endsWith(".mkv"))!; + const dir = video.slice(0, video.lastIndexOf("/")); + expect(files.filter((p) => p.startsWith(`${dir}/`))).toEqual( + [video, `${video.slice(0, -4)}.nfo`, `${dir}/fanart.jpg`, `${dir}/poster.jpg`].sort(), + ); + const nfo = await readFile(join(lib.root, `${video.slice(0, -4)}.nfo`), "utf8"); + expect(nfo).toContain('克里斯托弗·诺兰'); + expect(nfo).toContain("169"); + }, 60_000); + + it("写入失败记原因;只登记库根不写文件;没有入库文件的作品 409", async () => { + const files = await tree(lib.root); + const video = files.find((p) => p.includes("星际穿越") && p.endsWith(".mkv"))!; + const dirAbs = join(lib.root, video.slice(0, video.lastIndexOf("/"))); + if (process.getuid?.() !== 0) { + await chmod(dirAbs, 0o555); + const r = await runScrape(h.d, { mediaEntityId: movie, trigger: "manual", mode: "overwrite" }); + await chmod(dirAbs, 0o755); + expect(r.files.some((x) => x.reason === "scrape_write_failed")).toBe(true); + } + await h.db.updateTable("library_root").set({ management_mode: "register_only" }).execute(); + const ro = await runScrape(h.d, { mediaEntityId: movie, trigger: "manual", mode: "overwrite" }); + expect(ro).toMatchObject({ status: "failed", reason: "scrape_root_read_only", written: 0 }); + await h.db.updateTable("library_root").set({ management_mode: "managed" }).execute(); + + const empty = await seedEntity(h.db, { kind: "movie", title: "没入库", year: 2020 }); + expect((await h.api("POST", `/library/${empty}/scrape`, {})).status).toBe(412); + const none = await runScrape(h.d, { mediaEntityId: empty, trigger: "manual" }); + expect(none).toMatchObject({ status: "failed", reason: "scrape_no_metadata" }); + }, 60_000); +}); diff --git a/apps/server/src/modules/acquisition-organize/handlers.ts b/apps/server/src/modules/acquisition-organize/handlers.ts index 73c2c44b..b8c7af42 100644 --- a/apps/server/src/modules/acquisition-organize/handlers.ts +++ b/apps/server/src/modules/acquisition-organize/handlers.ts @@ -5,6 +5,7 @@ import { type AoDeps, automationBlock, ensureTaskRow, markTask } from "./service import { cleanupDownloadCache, submitDownload } from "./services/downloads.ts"; import { executeRecord, planImport } from "./services/organize.ts"; import { reconcileLibrary } from "./services/reconcile.ts"; +import { runScrape, SCRAPE_TASK } from "./services/scrape.ts"; import { errMessage } from "./util.ts"; // 持久任务处理器。输入只放业务 id;业务状态写业务表;副作用经 withAttemptToken(在 services 里)。 @@ -187,6 +188,25 @@ export function registerAoTasks(registry: TaskRegistry, d: AoDeps) { ); }, }), + // SCR-1 刮削:写 NFO 与图片到媒体库目录(文件道);结果写 media_scrape,不重试(失败的文件在作品详情里重新刮削) + defineTask({ + name: SCRAPE_TASK, + version: 1, + lane: "file", + inputSchema: z.object({ + mediaEntityId: id, + trigger: z.enum(["auto", "manual"]), + mode: z.enum(["missing", "overwrite"]).optional(), + recordId: id.optional(), + actor: z.object({ type: z.string(), id: z.string().nullable() }).optional(), + }), + retryPolicy: { maximumAttempts: 1 }, + handler: async (t) => + t.step.run("scrape", async () => { + const r = await runScrape(d, { ...t.input, runId: t.runId }, t.abortSignal); + return { status: r.status, written: r.written, skipped: r.skipped, failed: r.failed, reason: r.reason }; + }), + }), defineTask({ name: "reconcileLibrary", version: 1, diff --git a/apps/server/src/modules/acquisition-organize/ports.ts b/apps/server/src/modules/acquisition-organize/ports.ts index 6b250c1f..31c071df 100644 --- a/apps/server/src/modules/acquisition-organize/ports.ts +++ b/apps/server/src/modules/acquisition-organize/ports.ts @@ -1,3 +1,4 @@ +import { join } from "node:path"; import { createEmbyClient, createQbClient, @@ -18,6 +19,9 @@ import { import type { ActorType } from "@umt/contracts"; import type { DownloaderInstanceRow, MediaServerInstanceRow, SiteRow, SourceReleaseRow } from "@umt/db"; import type { ParsedRelease } from "@umt/domain"; +import { dataPaths } from "../../config.ts"; +import { externalFetch } from "../../egress.ts"; +import { ImageCache } from "../cache/images.ts"; import { SourcesRuntime } from "../sources-identity/runtime.ts"; import { createEntityFromTmdbCandidate, @@ -79,6 +83,11 @@ export interface AoPorts { ref: { type: "movie" | "tv"; id: string }, actor: { type: string; id: string | null }, ): Promise; + /** + * SCR-1 刮削取图:TMDB 图片路径 → 字节(原图,经图片缓存 DATA_DIR/cache/images 与出口代理)。 + * 失败抛错(由刮削记为该图片失败,不影响 NFO)。 + */ + scrapeImage(tmdbPath: string): Promise<{ content: Uint8Array; mime: string }>; options: AoOptions; } @@ -113,6 +122,7 @@ export function createPorts( } }; + let images: ImageCache | null = null; let sources: SourcesRuntime | null = null; const rt = () => { sources ??= new SourcesRuntime(ctx); @@ -179,6 +189,14 @@ export function createPorts( if (t.kind === "magnet") return { kind: "magnet", uri: t.uri }; return { kind: "url", url: t.url }; }, + async scrapeImage(tmdbPath) { + images ??= new ImageCache({ + dir: join(dataPaths(ctx.config).dataDir, "cache", "images"), + fetch: externalFetch, + timeoutMs: 30_000, + }); + return images.bytes("original", tmdbPath); + }, diskUsage: (p) => diskUsage(p), probe: (p) => ffprobe(p), async mediaServer(inst) { diff --git a/apps/server/src/modules/acquisition-organize/routes/library.ts b/apps/server/src/modules/acquisition-organize/routes/library.ts index 522e9a5d..2a1d4695 100644 --- a/apps/server/src/modules/acquisition-organize/routes/library.ts +++ b/apps/server/src/modules/acquisition-organize/routes/library.ts @@ -14,6 +14,8 @@ import { type LibraryItem, LibraryItem as LibraryItemSchema, LibraryListQuery, + LibraryScrapeRequest, + LibraryScrapeResult, type MediaAsset, type MediaEntity, MediaServerInventory, @@ -25,6 +27,7 @@ import { sql } from "@umt/db"; import { ApiException } from "../../../http/errors.ts"; import type { ModuleApp } from "../../types.ts"; import { type AoDeps, CANDIDATE_ARTIFACT_TITLE } from "../services/common.ts"; +import { enqueueScrape, loadScrapeState } from "../services/scrape.ts"; import { notFound, toApi, toApiList } from "../util.ts"; import { mount, paginate } from "./helpers.ts"; import { registerLibraryVersionRoutes } from "./library-versions.ts"; @@ -404,12 +407,48 @@ export function registerLibraryRoutes(app: ModuleApp, d: AoDeps) { createdAt: new Date(r.created_at).toISOString(), finishedAt: r.finished_at ? new Date(r.finished_at).toISOString() : null, })), + scrape: await loadScrapeState(d, id), }; return detail; }, { tag: TAG }, ); + // SCR-1 刮削:作品详情「重新刮削」;与入库后自动刮削、文件管理「刮削」同一任务 scrapeMedia + mount( + app, + { + method: "post", + path: "/library/{id}/scrape", + scope: "organize.execute", + summary: "刮削:把作品的 NFO 与图片写进媒体库目录(后台任务 scrapeMedia;覆盖方式缺省按设置)", + params: IdParam, + body: LibraryScrapeRequest, + response: LibraryScrapeResult, + }, + async (c, { params, body }) => { + const e = await db().selectFrom("media_entity").select(["id"]).where("id", "=", params.id).executeTakeFirst(); + if (!e) notFound("作品"); + const has = await db() + .selectFrom("media_asset") + .select(["id"]) + .where("media_entity_id", "=", params.id) + .where("status", "=", "active") + .executeTakeFirst(); + if (!has) throw new ApiException("precondition_failed", "作品没有已入库文件。先整理入库,再刮削"); + const actor = c.get("actor"); + const req = (body ?? {}) as LibraryScrapeRequest; + const r = await enqueueScrape(d, { + mediaEntityId: params.id, + trigger: "manual", + mode: req.mode, + actor: { type: actor?.type === "api_key" ? "api_key" : "user", id: actor?.id ?? null }, + }); + return { mediaEntityId: params.id, ...r }; + }, + { tag: TAG, status: 202 }, + ); + mount( app, { diff --git a/apps/server/src/modules/acquisition-organize/services/organize.ts b/apps/server/src/modules/acquisition-organize/services/organize.ts index ac5c86d1..bb49b356 100644 --- a/apps/server/src/modules/acquisition-organize/services/organize.ts +++ b/apps/server/src/modules/acquisition-organize/services/organize.ts @@ -47,6 +47,7 @@ import { effectiveNaming, ensureNamingVersion, loadNamingEntity, namingByVersion import { PathGuardError } from "./organize-guard.ts"; import { type FrozenTransfer, finalizeTransfer, planTransfer, type TransferSpec } from "./organize-transfer.ts"; import { hnrSatisfied } from "./retention.ts"; +import { scrapeAfterImport } from "./scrape.ts"; import { slotComplete as sharedSlotComplete, subscriptionLibraryRoot } from "./subscriptions.ts"; import { CONFLICT_OPTIONS, @@ -1710,6 +1711,9 @@ export async function finalizeRecord(d: AoDeps, recordId: string): Promise; + +const val = (m: unknown, k: string): unknown => ((m ?? {}) as Final)[k]?.value; +const str = (v: unknown): string | null => (typeof v === "string" && v.trim() ? v.trim() : null); +const numOf = (v: unknown): number | null => (typeof v === "number" && Number.isFinite(v) ? v : null); +const strs = (v: unknown): string[] => + Array.isArray(v) ? v.filter((x): x is string => typeof x === "string" && x.trim().length > 0) : []; + +interface Credit { + id?: unknown; + name?: unknown; + role?: unknown; + profilePath?: unknown; +} +const credits = (v: unknown): Credit[] => (Array.isArray(v) ? (v as Credit[]) : []); +const person = (c: Credit): NfoPerson | null => { + const name = str(c.name); + if (!name) return null; + return { tmdbId: numOf(c.id), name, role: str(c.role), profilePath: str(c.profilePath) }; +}; + +/** 制片国:ISO 3166-1 代码 → 中文名(Intl);认不出的保留原码 */ +export function countryNames(codes: readonly string[], locale = "zh-CN"): string[] { + let dn: Intl.DisplayNames | null = null; + try { + dn = new Intl.DisplayNames([locale], { type: "region" }); + } catch { + dn = null; + } + return codes.map((c) => { + if (!/^[A-Z]{2}$/.test(c)) return c; + try { + return dn?.of(c) ?? c; + } catch { + return c; + } + }); +} + +export interface EntityForNfo { + kind: string; + title: string; + original_title: string | null; + year: number | null; + metadata: unknown; +} + +/** 已接受的外部身份:provider → id(tmdb / imdb / tvdb / douban) */ +export type IdentityMap = Partial>; + +export function nfoWork(e: EntityForNfo, ids: IdentityMap): NfoWork { + const m = e.metadata; + const ext = (val(m, "externalIds") ?? {}) as { imdb?: unknown; tvdb?: unknown }; + const nfoIds: NfoIds = { + tmdb: ids.tmdb ?? null, + imdb: ids.imdb ?? str(ext.imdb), + tvdb: ids.tvdb ?? str(ext.tvdb), + douban: ids.douban ?? null, + }; + const crew = credits(val(m, "crew")); + const isTv = e.kind !== "movie"; + return { + kind: isTv ? "tvshow" : "movie", + title: str(val(m, "title")) ?? e.title, + originalTitle: str(val(m, "originalTitle")) ?? e.original_title, + year: numOf(val(m, "year")) ?? e.year, + premiered: str(val(m, "releaseDate")), + overview: str(val(m, "overview")), + tagline: str(val(m, "tagline")), + rating: numOf(val(m, "voteAverage")), + genres: strs(val(m, "genres")), + countries: countryNames(strs(val(m, "originCountry"))), + studios: isTv + ? (str(val(m, "platform")) ?? "") + .split(" / ") + .map((s) => s.trim()) + .filter(Boolean) + : [], + runtime: isTv ? null : numOf(val(m, "runtime")), + directors: crew + .filter((c) => c.role === "Director") + .map(person) + .filter((p): p is NfoPerson => !!p), + actors: credits(val(m, "cast")) + .map(person) + .filter((p): p is NfoPerson => !!p), + ids: nfoIds, + }; +} + +export interface UnitForNfo { + season_number: number | null; + episode_number: number | null; + title: string | null; + air_date: string | null; + metadata: unknown; +} + +export function nfoSeason(season: number, u: UnitForNfo | undefined): NfoSeason { + const m = u?.metadata; + return { + season, + title: str(val(m, "name")) ?? u?.title ?? null, + overview: str(val(m, "overview")), + airDate: str(val(m, "airDate")) ?? u?.air_date ?? null, + }; +} + +/** TMDB 占位集名(「第 N 集」「Episode N」)按没有标题处理,NFO 里由生成器补「第 N 集」 */ +const PLACEHOLDER = /^(第\s*\d+\s*集|Episode\s+\d+)$/i; + +export function nfoEpisode(season: number, episode: number, u: UnitForNfo | undefined): NfoEpisode { + const m = u?.metadata; + const title = str(val(m, "name")) ?? u?.title ?? null; + return { + season, + episode, + title: title && !PLACEHOLDER.test(title) ? title : null, + overview: str(val(m, "overview")), + airDate: str(val(m, "airDate")) ?? u?.air_date ?? null, + rating: numOf(val(m, "voteAverage")), + runtime: numOf(val(m, "runtime")), + }; +} + +/** 图片路径:作品海报 / 背景图,季海报,集剧照 */ +export function imagePathsOf(entityMeta: unknown) { + return { posterPath: str(val(entityMeta, "posterPath")), backdropPath: str(val(entityMeta, "backdropPath")) }; +} +export const unitPosterPath = (m: unknown) => str(val(m, "posterPath")); +export const unitStillPath = (m: unknown) => str(val(m, "stillPath")); diff --git a/apps/server/src/modules/acquisition-organize/services/scrape.ts b/apps/server/src/modules/acquisition-organize/services/scrape.ts new file mode 100644 index 00000000..3e9c7859 --- /dev/null +++ b/apps/server/src/modules/acquisition-organize/services/scrape.ts @@ -0,0 +1,484 @@ +import { randomUUID } from "node:crypto"; +import { lstat, mkdir, rename, rm, writeFile } from "node:fs/promises"; +import { dirname, isAbsolute, join, relative, resolve } from "node:path"; +import { checkRoot, DEFAULT_ROOT_SENTINEL } from "@umt/adapters"; +import { + DEFAULT_SETTINGS, + type LibraryScrapeState, + type OrganizeSettings, + type ScrapeFileResult, + type ScrapeMode, +} from "@umt/contracts"; +import { sql } from "@umt/db"; +import { + episodeNfo, + inferWorkDir, + planScrapeTargets, + type ScrapeTarget, + type ScrapeVideo, + seasonNfo, + workNfo, +} from "@umt/domain"; +import { errMessage, jsonb } from "../util.ts"; +import type { AoDeps } from "./common.ts"; +import { + type IdentityMap, + imagePathsOf, + nfoEpisode, + nfoSeason, + nfoWork, + type UnitForNfo, + unitPosterPath, + unitStillPath, +} from "./scrape-input.ts"; + +// SCR-1 刮削:把作品的最终字段写成 NFO,下载海报 / 背景图 / 季海报 / 剧照,放进媒体库目录(Emby / Jellyfin / Kodi 读取)。 +// - 触发:整理入库成功后(设置 organize.scrapeAfterImport)、作品详情「重新刮削」、文件管理「刮削」,都排同一个任务 scrapeMedia(文件道)。 +// - 目标文件与命名:packages/domain/src/scrape/layout.ts(文件名跟随已入库视频,即命名模板的结果);内容:scrape/nfo.ts。 +// - 覆盖:missing 只补缺失(已存在的跳过)/ overwrite 全部覆盖;NFO 先写,图片后下(并发 4,失败只记该文件,不影响 NFO)。 +// - 写入:同目录临时文件 + rename(原子),UTF-8;只写 managed 库根,库根离线整体失败;目标必须仍在库根内。 +// - 结果:media_scrape 一部作品一行(最近一次,逐文件结果)+ 审计 organize.scrape;SSE files.changed 让文件管理刷新。 + +export const SCRAPE_TASK = "scrapeMedia"; +const IMAGE_CONCURRENCY = 4; + +export interface ScrapeInput { + mediaEntityId: string; + trigger: "auto" | "manual"; + mode?: ScrapeMode | undefined; + recordId?: string | undefined; + /** 手动刮削的发起人(审计) */ + actor?: { type: string; id: string | null } | undefined; +} + +export async function scrapeSettings(d: AoDeps): Promise { + const s = await d.ctx.settings.get("organize").catch(() => DEFAULT_SETTINGS.organize); + return { ...DEFAULT_SETTINGS.organize, ...s }; +} + +/** + * 排刮削任务。同一作品已有排队中(还没开始)的刮削时直接复用(deduplicated); + * 自动刮削按整理记录幂等(同一记录只排一次)。 + */ +export async function enqueueScrape(d: AoDeps, input: ScrapeInput): Promise<{ taskId: string; deduplicated: boolean }> { + const db = d.ctx.db; + const mode = input.mode ?? (await scrapeSettings(d)).scrapeMode; + const cur = await db + .selectFrom("media_scrape") + .select(["status", "engine_run_id", "requested_at"]) + .where("media_entity_id", "=", input.mediaEntityId) + .executeTakeFirst(); + if (cur?.status === "queued" && cur.engine_run_id && Date.now() - new Date(cur.requested_at).getTime() < 30 * 60_000) + return { taskId: cur.engine_run_id, deduplicated: true }; + const key = + input.trigger === "auto" && input.recordId + ? `${SCRAPE_TASK}:record:${input.recordId}` + : `${SCRAPE_TASK}:manual:${input.mediaEntityId}:${randomUUID()}`; + const { runId, deduplicated } = await d.ctx.engine.enqueue( + SCRAPE_TASK, + { + mediaEntityId: input.mediaEntityId, + trigger: input.trigger, + mode, + ...(input.recordId ? { recordId: input.recordId } : {}), + ...(input.actor ? { actor: input.actor } : {}), + }, + { idempotencyKey: key, traceId: randomUUID() }, + ); + if (!deduplicated) + await sql`insert into media_scrape (media_entity_id, status, trigger, mode, record_id, engine_run_id, requested_at, + started_at, finished_at, written, skipped, failed, reason, files) + values (${input.mediaEntityId}, 'queued', ${input.trigger}, ${mode}, ${input.recordId ?? null}, ${runId}, now(), + null, null, 0, 0, 0, null, '[]'::jsonb) + on conflict (media_entity_id) do update set status = 'queued', trigger = excluded.trigger, mode = excluded.mode, + record_id = excluded.record_id, engine_run_id = excluded.engine_run_id, requested_at = now(), updated_at = now()`.execute( + db, + ); + return { taskId: runId, deduplicated: !!deduplicated }; +} + +/** 整理入库成功后:设置打开时自动排刮削(失败只记日志,不影响整理结果) */ +export async function scrapeAfterImport(d: AoDeps, entityId: string | null, recordId: string): Promise { + if (!entityId) return; + try { + const s = await scrapeSettings(d); + if (!s.scrapeAfterImport) return; + await enqueueScrape(d, { mediaEntityId: entityId, trigger: "auto", recordId }); + } catch (err) { + d.ctx.logger.warn({ entityId, recordId, err: { message: errMessage(err) } }, "入库后自动刮削没排上"); + } +} + +export async function loadScrapeState(d: AoDeps, entityId: string): Promise { + const r = await d.ctx.db + .selectFrom("media_scrape") + .selectAll() + .where("media_entity_id", "=", entityId) + .executeTakeFirst(); + if (!r) return null; + const iso = (t: Date | string | null) => (t ? new Date(t).toISOString() : null); + return { + status: r.status as LibraryScrapeState["status"], + trigger: r.trigger as LibraryScrapeState["trigger"], + mode: r.mode as ScrapeMode, + requestedAt: iso(r.requested_at)!, + startedAt: iso(r.started_at), + finishedAt: iso(r.finished_at), + written: r.written, + skipped: r.skipped, + failed: r.failed, + reason: r.reason, + files: (r.files ?? []) as ScrapeFileResult[], + recordId: r.record_id, + taskId: r.engine_run_id, + }; +} + +// ── 执行 ── + +interface RootGroup { + rootId: string; + rootPath: string; + managed: boolean; + sentinel: string | null; + videos: ScrapeVideo[]; +} + +/** 原子写:同目录临时文件 → rename;目录不存在时建(只在库根内) */ +async function atomicWrite(abs: string, content: Uint8Array | string): Promise { + await mkdir(dirname(abs), { recursive: true }); + const tmp = join(dirname(abs), `.${randomUUID()}.umt-scrape.tmp`); + try { + await writeFile(tmp, content); + await rename(tmp, abs); + } catch (e) { + await rm(tmp, { force: true }).catch(() => {}); + throw e; + } +} + +const inside = (root: string, abs: string) => { + const r = relative(root, abs); + return !!r && !r.startsWith("..") && !isAbsolute(r); +}; + +/** 有限并发 map */ +async function pool(items: readonly T[], n: number, fn: (t: T) => Promise): Promise { + const out: R[] = new Array(items.length); + let i = 0; + const worker = async () => { + while (i < items.length) { + const k = i++; + out[k] = await fn(items[k]!); + } + }; + await Promise.all(Array.from({ length: Math.min(n, items.length) }, worker)); + return out; +} + +export interface ScrapeRunResult { + status: LibraryScrapeState["status"]; + written: number; + skipped: number; + failed: number; + reason: string | null; + files: ScrapeFileResult[]; +} + +export async function runScrape( + d: AoDeps, + input: ScrapeInput & { runId?: string }, + signal?: AbortSignal, +): Promise { + const db = d.ctx.db; + const settings = await scrapeSettings(d); + const mode: ScrapeMode = input.mode ?? settings.scrapeMode; + const entityId = input.mediaEntityId; + const now = () => d.ports.now(); + await sql`insert into media_scrape (media_entity_id, status, trigger, mode, record_id, engine_run_id, started_at) + values (${entityId}, 'running', ${input.trigger}, ${mode}, ${input.recordId ?? null}, ${input.runId ?? null}, ${now()}) + on conflict (media_entity_id) do update set status = 'running', trigger = excluded.trigger, mode = excluded.mode, + record_id = coalesce(excluded.record_id, media_scrape.record_id), + engine_run_id = coalesce(excluded.engine_run_id, media_scrape.engine_run_id), + started_at = excluded.started_at, finished_at = null, reason = null, updated_at = now()` + .execute(db) + .catch(() => {}); + + const finish = async (r: ScrapeRunResult, rootDirs: Map> = new Map()) => { + await db + .updateTable("media_scrape") + .set({ + status: r.status, + finished_at: now(), + written: r.written, + skipped: r.skipped, + failed: r.failed, + reason: r.reason, + files: jsonb(r.files), + updated_at: now(), + }) + .where("media_entity_id", "=", entityId) + .execute() + .catch(() => {}); + await db + .insertInto("audit_event") + .values({ + actor_type: input.actor?.type ?? "system", + actor_id: input.actor?.id ?? null, + action: "organize.scrape", + target_type: "media_entity", + target_id: entityId, + payload: jsonb({ + trigger: input.trigger, + mode, + recordId: input.recordId ?? null, + status: r.status, + written: r.written, + skipped: r.skipped, + failed: r.failed, + reason: r.reason, + files: r.files.filter((f) => f.action !== "skipped").slice(0, 200), + }), + }) + .execute() + .catch(() => {}); + for (const [rootId, dirs] of rootDirs) + if (dirs.size) void d.ctx.events.publish("files.changed", { rootId: `lib:${rootId}`, dirs: [...dirs] }); + return r; + }; + const fail = (reason: string) => finish({ status: "failed", written: 0, skipped: 0, failed: 0, reason, files: [] }); + + const e = await db + .selectFrom("media_entity") + .select(["id", "kind", "title", "original_title", "year", "metadata", "status"]) + .where("id", "=", entityId) + .executeTakeFirst(); + if (!e || e.status === "merged") return fail("entity_not_found"); + const kind: "movie" | "tv" = e.kind === "movie" || e.kind === "collection" || e.kind === "other" ? "movie" : "tv"; + const meta = (e.metadata ?? {}) as Record; + if (!Object.keys(meta).length) return fail("scrape_no_metadata"); + + // 已入库的视频位置(库根内) + const locs = await db + .selectFrom("media_asset as a") + .innerJoin("asset_location as l", "l.media_asset_id", "a.id") + .innerJoin("library_root as r", "r.id", "l.library_root_id") + .select([ + "a.id as asset_id", + "l.relative_path", + "r.id as root_id", + "r.local_path", + "r.management_mode", + "r.enabled", + "r.sentinel_file", + ]) + .where("a.media_entity_id", "=", entityId) + .where("a.status", "=", "active") + .where("a.kind", "=", "video") + .where("l.location_kind", "=", "library") + .where("l.status", "in", ["available", "unavailable", "suspected_missing"]) + .execute(); + if (!locs.length) return fail("scrape_no_files"); + + const assetIds = [...new Set(locs.map((l) => l.asset_id))]; + const cov = await db + .selectFrom("asset_unit_coverage") + .select(["media_asset_id", "content_unit_id"]) + .where("media_asset_id", "in", assetIds) + .execute(); + const units = await db + .selectFrom("content_unit") + .select(["id", "kind", "season_number", "episode_number", "title", "air_date", "metadata"]) + .where("media_entity_id", "=", entityId) + .where("status", "!=", "retired") + .orderBy("created_at") + .execute(); + const unitById = new Map(units.map((u) => [u.id, u])); + const seasonUnit = new Map(); + const episodeUnit = new Map(); + for (const u of units) { + if (u.kind === "season" && u.season_number !== null && !seasonUnit.has(u.season_number)) + seasonUnit.set(u.season_number, u); + if (u.kind === "episode" && u.season_number !== null && u.episode_number !== null) { + const k = `${u.season_number}:${u.episode_number}`; + if (!episodeUnit.has(k)) episodeUnit.set(k, u); + } + } + + const groups = new Map(); + for (const l of locs) { + if (!l.enabled) continue; + const g = groups.get(l.root_id) ?? { + rootId: l.root_id, + rootPath: l.local_path, + managed: l.management_mode === "managed", + sentinel: l.sentinel_file, + videos: [], + }; + let season: number | null = null; + let episode: number | null = null; + if (kind === "tv") { + const eps = cov + .filter((c) => c.media_asset_id === l.asset_id) + .map((c) => unitById.get(c.content_unit_id)) + .filter((u) => u?.kind === "episode" && u.season_number !== null && u.episode_number !== null) + .sort((a, b) => a!.season_number! - b!.season_number! || a!.episode_number! - b!.episode_number!); + season = eps[0]?.season_number ?? null; + episode = eps[0]?.episode_number ?? null; + } + g.videos.push({ path: l.relative_path, season, episode }); + groups.set(l.root_id, g); + } + if (!groups.size) return fail("scrape_no_files"); + + const idRows = await db + .selectFrom("external_identity") + .select(["provider", "external_id"]) + .where("media_entity_id", "=", entityId) + .where("content_unit_id", "is", null) + .where("status", "=", "accepted") + .orderBy("updated_at", "desc") + .execute() + .catch(() => [] as { provider: string; external_id: string }[]); + const ids: IdentityMap = {}; + for (const r of idRows) { + const p = r.provider.toLowerCase() as keyof IdentityMap; + if (["tmdb", "imdb", "tvdb", "douban"].includes(p) && !ids[p]) ids[p] = r.external_id; + } + const bindings = await db + .selectFrom("path_binding") + .select(["library_root_id", "relative_path"]) + .where("media_entity_id", "=", entityId) + .where("binding_kind", "=", "entity") + .where("status", "=", "active") + .execute(); + + const work = nfoWork(e, ids); + const art = imagePathsOf(meta); + const seasonPosters = new Map(); + for (const [n, u] of seasonUnit) seasonPosters.set(n, unitPosterPath(u.metadata)); + const stills = new Map(); + for (const [k, u] of episodeUnit) stills.set(k, unitStillPath(u.metadata)); + + const files: ScrapeFileResult[] = []; + const rootDirs = new Map>(); + const imageCache = new Map>(); + const fetchImage = (p: string) => { + let hit = imageCache.get(p); + if (!hit) { + hit = d.ports.scrapeImage(p).then((r) => r.content); + imageCache.set(p, hit); + } + return hit; + }; + + for (const g of groups.values()) { + if (!g.managed) { + files.push( + ...g.videos.map((v) => ({ + path: v.path, + kind: "nfo" as const, + role: kind === "movie" ? "movie_nfo" : "episode_nfo", + action: "skipped" as const, + reason: "scrape_root_read_only", + })), + ); + continue; + } + const health = await checkRoot(g.rootPath, g.sentinel ?? DEFAULT_ROOT_SENTINEL); + if (health.kind !== "online") { + return finish({ + status: "failed", + written: 0, + skipped: 0, + failed: 0, + reason: "library_root_offline", + files, + }); + } + const bound = bindings.find((b) => b.library_root_id === g.rootId)?.relative_path ?? null; + const workDir = bound ?? inferWorkDir(kind, g.videos[0]!.path); + const targets = planScrapeTargets({ + kind, + workDir, + videos: g.videos, + images: { ...art, seasonPosters, stills }, + options: { images: settings.scrapeImages, episodeThumbs: settings.scrapeEpisodeThumbs }, + }); + if (kind === "tv" && !workDir) + files.push({ path: "", kind: "nfo", role: "tvshow_nfo", action: "skipped", reason: "scrape_no_work_dir" }); + const rootAbs = resolve(g.rootPath); + const dirs = rootDirs.get(g.rootId) ?? new Set(); + rootDirs.set(g.rootId, dirs); + + const decide = async (t: ScrapeTarget): Promise<{ abs: string } | ScrapeFileResult> => { + const abs = resolve(rootAbs, t.path); + const base = { path: t.path, kind: t.kind, role: t.role }; + if (!inside(rootAbs, abs)) return { ...base, action: "failed", reason: "path_escape" }; + const st = await lstat(abs).catch(() => null); + if (st && (!st.isFile() || st.isSymbolicLink())) return { ...base, action: "skipped", reason: "scrape_exists" }; + if (st && mode === "missing") return { ...base, action: "skipped", reason: "scrape_exists" }; + return { abs }; + }; + const write = async (t: ScrapeTarget, abs: string, content: Uint8Array | string): Promise => { + const base = { path: t.path, kind: t.kind, role: t.role }; + try { + await atomicWrite(abs, content); + dirs.add(dirname(t.path) === "." ? "" : dirname(t.path)); + return { ...base, action: "written", reason: null }; + } catch (err) { + d.ctx.logger.warn({ entityId, path: t.path, err: { message: errMessage(err) } }, "刮削文件写入失败"); + return { ...base, action: "failed", reason: "scrape_write_failed" }; + } + }; + + // NFO 先写(不依赖网络) + for (const t of targets.filter((x) => x.kind === "nfo")) { + if (signal?.aborted) throw new Error("aborted"); + const dec = await decide(t); + if (!("abs" in dec)) { + files.push(dec); + continue; + } + let xml: string; + if (t.role === "movie_nfo" || t.role === "tvshow_nfo") xml = workNfo(work); + else if (t.role === "season_nfo") xml = seasonNfo(nfoSeason(t.season!, seasonUnit.get(t.season!))); + else xml = episodeNfo(nfoEpisode(t.season!, t.episode!, episodeUnit.get(`${t.season}:${t.episode}`))); + files.push(await write(t, dec.abs, xml)); + } + // 图片:限并发,失败不影响 NFO + const imgs = targets.filter((x) => x.kind === "image" && x.imagePath); + const results = await pool(imgs, IMAGE_CONCURRENCY, async (t): Promise => { + if (signal?.aborted) throw new Error("aborted"); + const dec = await decide(t); + if (!("abs" in dec)) return dec; + let bytes: Uint8Array; + try { + bytes = await fetchImage(t.imagePath!); + } catch (err) { + d.ctx.logger.warn({ entityId, image: t.imagePath, err: { message: errMessage(err) } }, "刮削图片下载失败"); + return { path: t.path, kind: t.kind, role: t.role, action: "failed", reason: "scrape_image_failed" }; + } + return write(t, dec.abs, bytes); + }); + files.push(...results); + } + + const written = files.filter((f) => f.action === "written").length; + const failed = files.filter((f) => f.action === "failed").length; + const skipped = files.filter((f) => f.action === "skipped").length; + const onlyReadOnly = files.length > 0 && files.every((f) => f.reason === "scrape_root_read_only"); + const status: ScrapeRunResult["status"] = onlyReadOnly + ? "failed" + : failed === 0 + ? "succeeded" + : written > 0 || skipped > 0 + ? "partial" + : "failed"; + const reason = onlyReadOnly + ? "scrape_root_read_only" + : failed + ? (files.find((f) => f.action === "failed")?.reason ?? null) + : null; + return finish({ status, written, skipped, failed, reason, files }, rootDirs); +} diff --git a/apps/server/src/modules/cache/images.ts b/apps/server/src/modules/cache/images.ts index dcd961b9..b9a881c8 100644 --- a/apps/server/src/modules/cache/images.ts +++ b/apps/server/src/modules/cache/images.ts @@ -107,6 +107,14 @@ export class ImageCache { return new Response(content as BodyInit, { status: 200, headers }); } + /** + * SCR-1 刮削取图:TMDB 路径(`/xx.jpg`)→ 图片字节。同一缓存目录与出口代理;命中磁盘缓存就不再请求 + * (TMDB 图片路径内容不变,刮削不看 GLOBAL_IMAGE_CACHE 开关)。 + */ + async bytes(size: string, tmdbPath: string): Promise<{ content: Uint8Array; mime: string }> { + return this.fetchImage(this.upstream(size, tmdbPath.replace(/^\/+/, "")), true); + } + /** ImageHelper.fetch_image_with_mime_type */ private async fetchImage(url: string, useCache: boolean): Promise<{ content: Uint8Array; mime: string }> { const path = join(this.opts.dir, imageCachePath(url)); diff --git a/apps/server/src/modules/file-manager/probe.ts b/apps/server/src/modules/file-manager/probe.ts index 590e2c29..5b7e3613 100644 --- a/apps/server/src/modules/file-manager/probe.ts +++ b/apps/server/src/modules/file-manager/probe.ts @@ -13,6 +13,7 @@ import type { } from "@umt/contracts"; import { sql } from "@umt/db"; import type { Principal } from "../../http/auth.ts"; +import { enqueueScrape } from "../acquisition-organize/services/scrape.ts"; import { FilesError, type FmDeps, jsonb, resolveIn } from "./common.ts"; import { audit } from "./ledger.ts"; import { kindOfName, mediaSummary } from "./list.ts"; @@ -20,7 +21,8 @@ import { entitiesUnder } from "./records.ts"; // probe:ffprobe 摘要;已登记资产写回 media_asset.media_info.probe,未登记只缓存 24h(DATA_DIR/cache/probe)。 // scrape:只对已登记作品,等价 POST /media-entities/{id}/refresh(入队 refreshMetadata,字段锁优先、不动路径); -// NFO/图片投影 W5 实现(TODO),请求时返回 not_implemented。 +// writeNfo / images(SCR-1)另排 scrapeMedia,把 NFO 与图片写进媒体库目录(与作品详情「重新刮削」同一任务)。 +// scrape-batch write=true:文件管理「刮削」,按作品去重各排一次 scrapeMedia(不刷新元数据)。 const CACHE_TTL_MS = 24 * 3600_000; @@ -133,13 +135,23 @@ export async function scrape( .where("locked", "=", true) .execute(); const lockedFields = [...new Set([...(e.field_locks ?? []), ...overrides.map((o) => o.field)])].sort(); - // TODO(W5):writeNfo/images → 把最终字段投影成 NFO/海报写到绑定目录(临时文件 + 原子提交;非本系统 NFO 不覆盖) + const scrapeTaskId = + body.writeNfo || body.images + ? ( + await enqueueScrape(d.ao, { + mediaEntityId: e.id, + trigger: "manual", + actor: { type: principal?.kind === "api_key" ? "api_key" : "user", id: principal?.id ?? null }, + }) + ).taskId + : null; const result: FilesScrapeResult = { mediaEntityId: e.id, taskId: runId, lockedFields, - nfo: body.writeNfo ? "not_implemented" : "not_requested", - images: body.images ? "not_implemented" : "not_requested", + nfo: body.writeNfo ? "queued" : "not_requested", + images: body.images ? "queued" : "not_requested", + scrapeTaskId, }; await audit(d, principal, "scrape", { ...result }); return result; @@ -202,17 +214,28 @@ export async function scrapeBatch( skipped.push({ ref: id, reason: "not_registered" }); continue; } - const { runId } = await d.ctx.engine.enqueue( - "refreshMetadata", - { mediaEntityId: id }, - { idempotencyKey: `refreshMetadata:files-batch:${body.idempotencyKey}:${id}`, traceId: randomUUID() }, - ); + const runId = body.write + ? ( + await enqueueScrape(d.ao, { + mediaEntityId: id, + trigger: "manual", + actor: { type: principal?.kind === "api_key" ? "api_key" : "user", id: principal?.id ?? null }, + }) + ).taskId + : ( + await d.ctx.engine.enqueue( + "refreshMetadata", + { mediaEntityId: id }, + { idempotencyKey: `refreshMetadata:files-batch:${body.idempotencyKey}:${id}`, traceId: randomUUID() }, + ) + ).runId; const lockedFields = [ ...new Set([...(e.field_locks ?? []), ...locks.filter((l) => l.media_entity_id === id).map((l) => l.field)]), ].sort(); submitted.push({ mediaEntityId: id, title: e.title, taskId: runId, lockedFields }); } await audit(d, principal, "scrape_batch", { + write: body.write, submitted: submitted.map((s) => s.mediaEntityId), skipped, }); diff --git a/apps/server/test/file-manager/identify-organize.test.ts b/apps/server/test/file-manager/identify-organize.test.ts index 1ddf5f72..07ce5166 100644 --- a/apps/server/test/file-manager/identify-organize.test.ts +++ b/apps/server/test/file-manager/identify-organize.test.ts @@ -305,8 +305,18 @@ describe.skipIf(skip)("files 识别 / 整理 / 探测 / 刮削", () => { const s = await h.api("POST", "/files/scrape", { idempotencyKey: key(), mediaEntityId: film, writeNfo: true }); expect(s.status).toBe(200); - expect(s.body.data).toMatchObject({ mediaEntityId: film, nfo: "not_implemented", images: "not_requested" }); + // SCR-1:writeNfo 另排 scrapeMedia(写 NFO 与图片到媒体库目录) + expect(s.body.data).toMatchObject({ mediaEntityId: film, nfo: "queued", images: "not_requested" }); + expect(s.body.data.scrapeTaskId).toBeTruthy(); expect(h.engine.enqueued.some((e) => e.task === "refreshMetadata")).toBe(true); + expect(h.engine.enqueued.some((e) => e.task === "scrapeMedia")).toBe(true); + const b = await h.api("POST", "/files/scrape-batch", { + idempotencyKey: key(), + mediaEntityIds: [film], + write: true, + }); + expect(b.status).toBe(200); + expect(b.body.data.submitted).toHaveLength(1); const none = await h.api("POST", "/files/scrape", { idempotencyKey: key(), mediaEntityId: show }); expect(none.status).toBe(400); }); diff --git a/apps/server/test/scrape/gen-scrape-mp.py b/apps/server/test/scrape/gen-scrape-mp.py new file mode 100644 index 00000000..4ca4965d --- /dev/null +++ b/apps/server/test/scrape/gen-scrape-mp.py @@ -0,0 +1,51 @@ +"""SCR-1 刮削差分:用 MP 上游的 TmdbScraper(themoviedb/scraper.py)从同一份 TMDB 原始响应生成 NFO,写 scrape-mp.json。 + +用法(MP 仓库根目录,用其 .venv): + cd /opt/moviepilot-plus/upstream/MoviePilot && PYTHONPATH=. \ + .venv/bin/python /opt/uhd_media_tool/apps/server/test/scrape/gen-scrape-mp.py + +只记录行为输出,不复制 MP 源码。MediaInfo 用 MP 自己的 set_tmdb_info(TMDB 详情 → 统一字段), +季 / 集直接喂 TMDB 季详情(与 MP get_metadata_nfo 取到的一致)。图片命名按 MP 代码行为登记在 nfo-mp-diff.test.ts。 +""" + +import json +import pathlib + +from app.domain.context import MediaInfo +from app.domain.projection.tmdb import configure_image_url_builder +from app.modules.themoviedb.scraper import TmdbScraper + +HERE = pathlib.Path(__file__).parent +raw = json.loads((HERE / "tmdb-raw.json").read_text(encoding="utf-8")) + +# 启动组合根里注入的 TMDB 图片地址(settings.TMDB_IMAGE_URL,原图) +configure_image_url_builder(lambda p: f"https://image.tmdb.org/t/p/original/{p.removeprefix('/')}") + +s = TmdbScraper() + + +def xml(doc) -> str: + return doc.toprettyxml(indent=" ", encoding="utf-8").decode("utf-8") + + +def media(info: dict, media_type: str) -> MediaInfo: + mi = MediaInfo() + mi.set_tmdb_info({**info, "media_type": media_type}) + return mi + + +out = {} +out["movie"] = xml(s._TmdbScraper__gen_movie_nfo_file(mediainfo=media(raw["/movie/157336"], "movie"))) +out["tvshow"] = xml(s._TmdbScraper__gen_tv_nfo_file(mediainfo=media(raw["/tv/95396"], "tv"))) +season = raw["/tv/95396/season/1"] +out["season1"] = xml(s._TmdbScraper__gen_tv_season_nfo_file(seasoninfo=season, season=1)) +for ep in season["episodes"]: + n = ep["episode_number"] + out[f"s01e{n:02d}"] = xml( + s._TmdbScraper__gen_tv_episode_nfo_file(tmdbid=95396, episodeinfo=ep, season=1, episode=n) + ) +out["seasonPosterName"] = TmdbScraper.get_season_poster(season, 1)[0] +out["seasonSpecialsPosterName"] = TmdbScraper.get_season_poster({"poster_path": "/x.jpg"}, 0)[0] + +(HERE / "scrape-mp.json").write_text(json.dumps(out, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") +print(f"写入 {len(out)} 项 → scrape-mp.json") diff --git a/apps/server/test/scrape/nfo-deviations.json b/apps/server/test/scrape/nfo-deviations.json new file mode 100644 index 00000000..7f110843 --- /dev/null +++ b/apps/server/test/scrape/nfo-deviations.json @@ -0,0 +1,69 @@ +{ + "_note": "SCR-1 NFO 与 MP 的已登记差异(按样例、按元素)。测试先去掉双方的空元素(MP 缺值写 、空 CDATA、评分 0,我们不写),再去掉这里登记的元素后逐行比较;登记的元素必须确实不同,差异消除后删掉登记。", + "deviations": [ + { + "case": "movie", + "tag": "studio", + "reason": "本地不存 TMDB production_companies,电影不写出品公司(Emby / Jellyfin 刷新时会自己补)" + }, + { + "case": "movie", + "tag": "country", + "reason": "制片国取 TMDB origin_country(代码转中文名);MP 取 production_countries(本例多一个英国)" + }, + { + "case": "movie", + "tag": "tagline", + "reason": "我们额外写标语,MP 不写" + }, + { + "case": "tvshow", + "tag": "studio", + "reason": "剧集写播出平台(TMDB networks,本例 Apple TV+);MP 写出品公司 production_companies" + }, + { + "case": "s01e01", + "tag": "uniqueid", + "reason": "本地不存 TMDB 集 id,集 NFO 不写 uniqueid,媒体服务器按季号集号匹配" + }, + { + "case": "s01e01", + "tag": "tmdbid", + "reason": "同上" + }, + { + "case": "s01e01", + "tag": "runtime", + "reason": "我们额外写集时长,MP 不写" + }, + { + "case": "s01e02", + "tag": "uniqueid", + "reason": "同 s01e01" + }, + { + "case": "s01e02", + "tag": "tmdbid", + "reason": "同 s01e01" + }, + { + "case": "s01e02", + "tag": "runtime", + "reason": "同 s01e01" + } + ], + "images": [ + { + "role": "movie poster / fanart", + "mp": "poster.jpg、backdrop.jpg、fanart.jpg(作品目录)", + "ours": "poster.jpg、fanart.jpg", + "reason": "backdrop.jpg 是 fanart.jpg 的副本,Emby / Jellyfin / Kodi 都认 fanart.jpg,不写重复文件" + }, + { + "role": "episode thumb", + "mp": "<视频名>.jpg", + "ours": "<视频名>-thumb.jpg", + "reason": "Kodi 约定 -thumb 后缀;Jellyfin / Emby 同样识别,且不会和同名海报规则冲突" + } + ] +} diff --git a/apps/server/test/scrape/nfo-mp-diff.test.ts b/apps/server/test/scrape/nfo-mp-diff.test.ts new file mode 100644 index 00000000..1ccabbfa --- /dev/null +++ b/apps/server/test/scrape/nfo-mp-diff.test.ts @@ -0,0 +1,149 @@ +import { readFileSync } from "node:fs"; +import { fetchRichDetail, fetchRichSeason } from "@umt/adapters"; +import { episodeNfo, planScrapeTargets, seasonNfo, seasonPosterName, workNfo } from "@umt/domain"; +import { describe, expect, it } from "vitest"; +import { nfoEpisode, nfoSeason, nfoWork } from "../../src/modules/acquisition-organize/services/scrape-input.ts"; +import { richEntityFields } from "../../src/modules/sources-identity/service/metadata-rich.ts"; + +// SCR-1 NFO 差分:同一份 TMDB 原始响应(tmdb-raw.json),MP 走 set_tmdb_info + TmdbScraper(gen-scrape-mp.py → scrape-mp.json), +// 我们走 fetchRichDetail / fetchRichSeason → 字段投影(metadata)→ scrape-input → domain NFO 生成器。 +// 双方去掉空元素后逐行比较,已登记的差异(nfo-deviations.json)按元素排除并要求确实不同。 + +const read = (f: string) => JSON.parse(readFileSync(new URL(f, import.meta.url), "utf8")); +const raw = read("./tmdb-raw.json") as Record; +const mp = read("./scrape-mp.json") as Record; +const devs = read("./nfo-deviations.json").deviations as { case: string; tag: string }[]; + +const LANGS = ["zh-CN", "zh-SG", "zh-TW", "zh-HK", "en-US"]; +const client = { + async getJson(path: string): Promise { + const r = raw[path]; + if (!r) throw new Error(`fixture 缺 ${path}`); + return structuredClone(r) as T; + }, +}; +const final = (fields: Record) => + Object.fromEntries(Object.entries(fields).map(([k, v]) => [k, { value: v.value }])); + +async function ours(): Promise> { + const out: Record = {}; + const movie = await fetchRichDetail(client, "movie", 157336, LANGS); + out.movie = workNfo( + nfoWork( + { + kind: "movie", + title: movie.title, + original_title: movie.originalTitle, + year: movie.year, + metadata: final(richEntityFields(movie) as never), + }, + { tmdb: "157336" }, + ), + ); + const tv = await fetchRichDetail(client, "tv", 95396, LANGS); + out.tvshow = workNfo( + nfoWork( + { + kind: "series", + title: tv.title, + original_title: tv.originalTitle, + year: tv.year, + metadata: final(richEntityFields(tv) as never), + }, + { tmdb: "95396" }, + ), + ); + const sd = await fetchRichSeason(client, 95396, 1, LANGS, { now: new Date("2026-01-01") }); + out.season1 = seasonNfo( + nfoSeason(1, { + season_number: 1, + episode_number: null, + title: sd.name, + air_date: sd.airDate, + metadata: { name: { value: sd.name }, overview: { value: sd.overview }, airDate: { value: sd.airDate } }, + }), + ); + for (const e of sd.episodes) { + out[`s01e${String(e.episodeNumber).padStart(2, "0")}`] = episodeNfo( + nfoEpisode(1, e.episodeNumber, { + season_number: 1, + episode_number: e.episodeNumber, + title: e.name, + air_date: e.airDate, + metadata: { + name: { value: e.name }, + overview: { value: e.overview }, + airDate: { value: e.airDate }, + runtime: { value: e.runtime }, + voteAverage: { value: e.voteAverage }, + }, + }), + ); + } + return out; +} + +/** 去空元素(MP 缺值写 、空 CDATA、评分 0) */ +const normalize = (xml: string) => + xml + .split("\n") + .filter((l) => l.trim() !== "") + .filter((l) => !/^\s*<[\w]+(\s[^>]*)?\/>$/.test(l)) + .filter((l) => !//.test(l)) + .filter((l) => !/^\s*0<\/rating>$/.test(l)); +const tagOf = (l: string) => /^ {2}<(\w+)/.exec(l)?.[1] ?? null; + +describe("NFO 与 MP 差分", async () => { + const mine = await ours(); + const cases = ["movie", "tvshow", "season1", "s01e01", "s01e02"]; + + it.each(cases)("%s:结构与取值一致(已登记差异除外)", (c) => { + const skip = new Set(devs.filter((d) => d.case === c).map((d) => d.tag)); + const a = normalize(mp[c]!).filter((l) => !skip.has(tagOf(l) ?? "")); + const b = normalize(mine[c]!).filter((l) => !skip.has(tagOf(l) ?? "")); + expect(b).toEqual(a); + for (const tag of skip) { + const ma = normalize(mp[c]!).filter((l) => tagOf(l) === tag); + const mb = normalize(mine[c]!).filter((l) => tagOf(l) === tag); + expect(mb, `登记的差异 ${c}.${tag} 已消失,删掉登记`).not.toEqual(ma); + } + }); + + it("编码与声明:UTF-8、两空格缩进、末尾换行,和 MP 一致", () => { + for (const c of cases) { + expect(mine[c]!.split("\n")[0]).toBe(mp[c]!.split("\n")[0]); + expect(mine[c]!.endsWith("\n")).toBe(true); + } + }); + + it("季海报命名与 MP get_season_poster 一致", () => { + expect(seasonPosterName(1)).toBe(mp.seasonPosterName); + expect(seasonPosterName(0)).toBe(mp.seasonSpecialsPosterName); + }); + + it("文件命名:对照 MP scraping.py 的目标路径(差异见 nfo-deviations.json images)", () => { + const t = planScrapeTargets({ + kind: "tv", + workDir: "人生切割术 (2022)", + videos: [{ path: "人生切割术 (2022)/Season 1/人生切割术 - S01E01 - 第 1 集.mkv", season: 1, episode: 1 }], + images: { + posterPath: "/p.jpg", + backdropPath: "/b.jpg", + seasonPosters: new Map([[1, "/s.jpg"]]), + stills: new Map([["1:1", "/e.jpg"]]), + }, + options: { images: true, episodeThumbs: true }, + }); + // MP:tvshow.nfo(TV)、season.nfo(SEASON)、.nfo(EPISODE);季海报根目录 seasonXX-poster + 季目录 poster + expect(t.map((x) => x.path)).toEqual([ + "人生切割术 (2022)/tvshow.nfo", + "人生切割术 (2022)/poster.jpg", + "人生切割术 (2022)/fanart.jpg", + "人生切割术 (2022)/Season 1/season.nfo", + "人生切割术 (2022)/season01-poster.jpg", + "人生切割术 (2022)/Season 1/poster.jpg", + "人生切割术 (2022)/Season 1/人生切割术 - S01E01 - 第 1 集.nfo", + "人生切割术 (2022)/Season 1/人生切割术 - S01E01 - 第 1 集-thumb.jpg", + ]); + }); +}); diff --git a/apps/server/test/scrape/scrape-mp.json b/apps/server/test/scrape/scrape-mp.json new file mode 100644 index 00000000..54a6cfef --- /dev/null +++ b/apps/server/test/scrape/scrape-mp.json @@ -0,0 +1,9 @@ +{ + "movie": "\n\n 157336\n 157336\n tt0816692\n tt0816692\n \n \n 克里斯托弗·诺兰\n \n 马修·麦康纳\n Actor\n Cooper\n 10297\n https://image.tmdb.org/t/p/original/lCySuYjhXix3FzQdS4oceDDrXKI.jpg\n https://www.themoviedb.org/person/10297\n \n \n 安妮·海瑟薇\n Actor\n Brand\n 1813\n https://image.tmdb.org/t/p/original/s6tflSD20MGz04ZR2R1lZvhmC4Y.jpg\n https://www.themoviedb.org/person/1813\n \n \n 杰西卡·查斯坦\n Actor\n Murph\n 83002\n https://www.themoviedb.org/person/83002\n \n 冒险\n 剧情\n 科幻\n 8.5\n Legendary Pictures\n 英国\n 美国\n 169\n 星际穿越\n Interstellar\n 2014-11-05\n 2014\n\n", + "tvshow": "\n\n 95396\n 95396\n 371980\n 371980\n tt11280740\n tt11280740\n \n \n \n 亚当·斯科特\n Actor\n Mark Scout\n 4495\n https://image.tmdb.org/t/p/original/9ynbGSZ7O7stcCYx1ElmFhUgCKj.jpg\n https://www.themoviedb.org/person/4495\n \n \n 布丽特·洛薇尔\n Actor\n Helly R.\n 1319469\n https://www.themoviedb.org/person/1319469\n \n 剧情\n 悬疑\n Sci-Fi & Fantasy\n 8.4\n Red Hour Productions\n 美国\n 人生切割术\n Severance\n 2022-02-17\n 2022\n -1\n -1\n\n", + "season1": "\n\n \n \n 第 1 季\n 2022-02-17\n 2022-02-17\n 2022\n 1\n\n", + "s01e01": "\n\n 1830836\n 1830836\n 有关地狱的好消息\n \n \n 2022-02-17\n 2022\n 1\n 1\n 8.2\n\n", + "s01e02": "\n\n 3435543\n 3435543\n 半循环\n \n \n 2022-02-18\n 2022\n 1\n 2\n 8.0\n\n", + "seasonPosterName": "season01-poster.jpg", + "seasonSpecialsPosterName": "season-specials-poster.jpg" +} diff --git a/apps/server/test/scrape/tmdb-raw.json b/apps/server/test/scrape/tmdb-raw.json new file mode 100644 index 00000000..b28e4554 --- /dev/null +++ b/apps/server/test/scrape/tmdb-raw.json @@ -0,0 +1,215 @@ +{ + "_note": "SCR-1 差分用 TMDB 原始响应(zh-CN,形态照 TMDB v3;数值只保证合理,不作权威数据)。MP 与我们都从这里出发生成 NFO。", + "/movie/157336": { + "id": 157336, + "title": "星际穿越", + "original_title": "Interstellar", + "original_language": "en", + "origin_country": ["US"], + "release_date": "2014-11-05", + "overview": "在不远的未来,随着地球自然环境的恶化,人类面临着无法生存的威胁。一群宇航员穿越虫洞,为人类寻找新家园。", + "tagline": "我们注定要离开地球", + "runtime": 169, + "vote_average": 8.464, + "vote_count": 37000, + "status": "Released", + "poster_path": "/nCbkOyOMTEwlEV0LtCOvCnwEONA.jpg", + "backdrop_path": "/xJHokMbljvjADYdit5fK5VQsXEG.jpg", + "genres": [{ "id": 12, "name": "冒险" }, { "id": 18, "name": "剧情" }, { "id": 878, "name": "科幻" }], + "production_countries": [{ "iso_3166_1": "GB", "name": "英国" }, { "iso_3166_1": "US", "name": "美国" }], + "production_companies": [{ "id": 923, "name": "Legendary Pictures" }], + "external_ids": { "imdb_id": "tt0816692", "wikidata_id": "Q13417189" }, + "credits": { + "cast": [ + { + "id": 10297, + "name": "马修·麦康纳", + "original_name": "Matthew McConaughey", + "character": "Cooper", + "known_for_department": "Acting", + "order": 0, + "profile_path": "/lCySuYjhXix3FzQdS4oceDDrXKI.jpg" + }, + { + "id": 1813, + "name": "安妮·海瑟薇", + "original_name": "Anne Hathaway", + "character": "Brand", + "known_for_department": "Acting", + "order": 1, + "profile_path": "/s6tflSD20MGz04ZR2R1lZvhmC4Y.jpg" + }, + { + "id": 83002, + "name": "杰西卡·查斯坦", + "original_name": "Jessica Chastain", + "character": "Murph", + "known_for_department": "Acting", + "order": 2, + "profile_path": null + } + ], + "crew": [ + { + "id": 525, + "name": "克里斯托弗·诺兰", + "original_name": "Christopher Nolan", + "job": "Director", + "department": "Directing", + "known_for_department": "Directing", + "profile_path": "/xuAIuYSmsUzKlUMBFGVZaWsY3DZ.jpg" + } + ] + }, + "translations": { + "translations": [ + { + "iso_639_1": "zh", + "iso_3166_1": "CN", + "data": { + "title": "星际穿越", + "overview": "在不远的未来,随着地球自然环境的恶化,人类面临着无法生存的威胁。一群宇航员穿越虫洞,为人类寻找新家园。", + "tagline": "我们注定要离开地球" + } + }, + { + "iso_639_1": "en", + "iso_3166_1": "US", + "data": { + "title": "Interstellar", + "overview": "The adventures of a group of explorers.", + "tagline": "Mankind was born on Earth. It was never meant to die here." + } + } + ] + }, + "alternative_titles": { "titles": [{ "iso_3166_1": "CN", "title": "星际启示录", "type": "" }] } + }, + "/tv/95396": { + "id": 95396, + "name": "人生切割术", + "original_name": "Severance", + "original_language": "en", + "origin_country": ["US"], + "first_air_date": "2022-02-17", + "last_air_date": "2025-03-21", + "overview": "马克领导着一个办公室团队,他们的记忆被手术分割,工作与生活从此一分为二。", + "tagline": "", + "vote_average": 8.395, + "vote_count": 2400, + "status": "Returning Series", + "episode_run_time": [], + "number_of_seasons": 2, + "number_of_episodes": 19, + "poster_path": "/pPHpeI2X1qEd1CS1SeyrdhZ4qnT.jpg", + "backdrop_path": "/hS8cOSG9m8H9dGaIxkRYZ7rRl8F.jpg", + "genres": [ + { "id": 18, "name": "剧情" }, + { "id": 9648, "name": "悬疑" }, + { "id": 10765, "name": "Sci-Fi & Fantasy" } + ], + "networks": [{ "id": 2552, "name": "Apple TV+" }], + "production_countries": [{ "iso_3166_1": "US", "name": "美国" }], + "production_companies": [{ "id": 1, "name": "Red Hour Productions" }], + "created_by": [{ "id": 3442409, "name": "丹·埃里克森", "profile_path": null }], + "external_ids": { "imdb_id": "tt11280740", "tvdb_id": 371980 }, + "credits": { + "cast": [ + { + "id": 4495, + "name": "亚当·斯科特", + "original_name": "Adam Scott", + "character": "Mark Scout", + "known_for_department": "Acting", + "order": 0, + "profile_path": "/9ynbGSZ7O7stcCYx1ElmFhUgCKj.jpg" + }, + { + "id": 1319469, + "name": "布丽特·洛薇尔", + "original_name": "Britt Lower", + "character": "Helly R.", + "known_for_department": "Acting", + "order": 1, + "profile_path": null + } + ], + "crew": [] + }, + "seasons": [ + { + "season_number": 1, + "episode_count": 9, + "name": "第 1 季", + "air_date": "2022-02-17", + "overview": "", + "poster_path": "/lFf6LLrQjYldcZItzOkGmMMigP7.jpg" + }, + { + "season_number": 2, + "episode_count": 10, + "name": "第 2 季", + "air_date": "2025-01-16", + "overview": "", + "poster_path": "/5n9UHSJp9KcHSTnkBPkcQqgWUgL.jpg" + } + ], + "translations": { + "translations": [ + { + "iso_639_1": "zh", + "iso_3166_1": "CN", + "data": { + "name": "人生切割术", + "overview": "马克领导着一个办公室团队,他们的记忆被手术分割,工作与生活从此一分为二。", + "tagline": "" + } + }, + { + "iso_639_1": "en", + "iso_3166_1": "US", + "data": { "name": "Severance", "overview": "Mark leads a team of office workers.", "tagline": "" } + } + ] + }, + "alternative_titles": { "results": [{ "iso_3166_1": "TW", "title": "人生切割術", "type": "" }] } + }, + "/tv/95396/season/1": { + "id": 186498, + "season_number": 1, + "name": "第 1 季", + "overview": "在卢蒙工业公司,员工们接受了一种名为“分割”的手术 & 工作记忆与生活记忆被彻底隔开。", + "air_date": "2022-02-17", + "poster_path": "/lFf6LLrQjYldcZItzOkGmMMigP7.jpg", + "episodes": [ + { + "id": 1830836, + "episode_number": 1, + "season_number": 1, + "name": "有关地狱的好消息", + "overview": "马克升职成为宏观数据精炼部的主管,并迎来一位新员工。", + "air_date": "2022-02-17", + "runtime": 57, + "still_path": "/jM3lJqrr6uD17rBHCM7qc2rH4rM.jpg", + "vote_average": 8.2, + "vote_count": 171, + "crew": [], + "guest_stars": [] + }, + { + "id": 3435543, + "episode_number": 2, + "season_number": 1, + "name": "半循环", + "overview": "马克带海莉参观办公室。", + "air_date": "2022-02-18", + "runtime": 53, + "still_path": "/kg2cNi5r1hhvF5GWWyi3wYO7rqE.jpg", + "vote_average": 8.0, + "vote_count": 150, + "crew": [], + "guest_stars": [] + } + ] + } +} diff --git a/packages/contracts/src/api.ts b/packages/contracts/src/api.ts index 94424c9f..f8ac6664 100644 --- a/packages/contracts/src/api.ts +++ b/packages/contracts/src/api.ts @@ -2238,6 +2238,15 @@ export const Api = { query: libraryVersions.LibraryVersionsQuery, response: libraryVersions.LibraryVersions, }), + scrape: ep({ + method: "post", + path: "/library/{id}/scrape", + scope: "organize.execute", + summary: "刮削:把作品的 NFO 与图片写进媒体库目录(后台任务 scrapeMedia;覆盖方式缺省按设置)", + params: IdParam, + body: organization.LibraryScrapeRequest, + response: organization.LibraryScrapeResult, + }), reconcile: ep({ method: "post", path: "/library/reconcile", diff --git a/packages/contracts/src/files.ts b/packages/contracts/src/files.ts index c3fbabd0..5264742f 100644 --- a/packages/contracts/src/files.ts +++ b/packages/contracts/src/files.ts @@ -685,6 +685,7 @@ export type IdentifyCompletedPayload = z.infer; export const FilesScrapeRequest = z.object({ idempotencyKey: IdempotencyKey, mediaEntityId: Id, + /** SCR-1:true 时另排 scrapeMedia,把 NFO(与 images 时的图片)写进媒体库目录 */ writeNfo: z.boolean().default(false), images: z.boolean().default(false), }); @@ -695,9 +696,11 @@ export const FilesScrapeResult = z.object({ taskId: z.string(), /** 当前被用户锁定、刷新不会改动的字段 */ lockedFields: z.array(z.string()), - /** NFO/图片投影:W5 实现前请求时为 not_implemented */ - nfo: z.enum(["written", "skipped", "not_requested", "not_implemented"]), - images: z.enum(["written", "skipped", "not_requested", "not_implemented"]), + /** NFO/图片投影(SCR-1):queued = 已排 scrapeMedia,结果看作品刮削状态;not_implemented 已不再返回 */ + nfo: z.enum(["written", "skipped", "queued", "not_requested", "not_implemented"]), + images: z.enum(["written", "skipped", "queued", "not_requested", "not_implemented"]), + /** scrapeMedia 引擎 run id(writeNfo / images 时) */ + scrapeTaskId: z.string().nullable().optional(), }); export type FilesScrapeResult = z.infer; @@ -845,6 +848,8 @@ export const FilesScrapeBatchRequest = z.object({ idempotencyKey: IdempotencyKey, items: z.array(FileRef).max(500).optional(), mediaEntityIds: z.array(Id).max(200).optional(), + /** SCR-1:true = 刮削(排 scrapeMedia 写 NFO 与图片,覆盖方式按设置);false(缺省)= 刷新元数据 */ + write: z.boolean().default(false), }); export type FilesScrapeBatchRequest = z.infer; export const FilesScrapeBatchResult = z.object({ diff --git a/packages/contracts/src/organization.ts b/packages/contracts/src/organization.ts index 54f5ad72..b46ea126 100644 --- a/packages/contracts/src/organization.ts +++ b/packages/contracts/src/organization.ts @@ -370,6 +370,81 @@ export const LibraryDroppedPlanView = z.object({ }); export type LibraryDroppedPlanView = z.infer; +/** SCR-1 刮削:一个写入目标(NFO / 图片)的结果;path 为库根内相对路径 */ +export const ScrapeFileResult = z.object({ + path: z.string(), + kind: z.enum(["nfo", "image"]), + /** movie_nfo / tvshow_nfo / season_nfo / episode_nfo / poster / fanart / season_poster / episode_thumb */ + role: z.string(), + action: z.enum(["written", "skipped", "failed"]), + /** 跳过 / 失败原因码(reasons.ts:scrape_exists / scrape_write_failed / scrape_image_failed …) */ + reason: z.string().nullable(), +}); +export type ScrapeFileResult = z.infer; + +export const ScrapeMode = z.enum(["missing", "overwrite"]); +export type ScrapeMode = z.infer; + +export const ScrapeStatus = z.enum(["queued", "running", "succeeded", "partial", "failed"]); +export type ScrapeStatus = z.infer; +export const ScrapeTrigger = z.enum(["auto", "manual"]); +export type ScrapeTrigger = z.infer; + +/** 表 media_scrape 的行(一部作品一行,最近一次刮削) */ +export const MediaScrape = z.object({ + mediaEntityId: Id, + status: ScrapeStatus, + trigger: ScrapeTrigger, + mode: ScrapeMode, + recordId: Id.nullable(), + engineRunId: z.string().nullable(), + requestedAt: Timestamp, + startedAt: Timestamp.nullable(), + finishedAt: Timestamp.nullable(), + written: NonNegInt, + skipped: NonNegInt, + failed: NonNegInt, + reason: z.string().nullable(), + files: z.array(ScrapeFileResult), + createdAt: Timestamp, + updatedAt: Timestamp, +}); +export type MediaScrape = z.infer; + +/** SCR-1 作品刮削状态(最近一次;表 media_scrape) */ +export const LibraryScrapeState = z.object({ + status: ScrapeStatus, + trigger: ScrapeTrigger, + mode: ScrapeMode, + requestedAt: Timestamp, + startedAt: Timestamp.nullable(), + finishedAt: Timestamp.nullable(), + written: NonNegInt, + skipped: NonNegInt, + failed: NonNegInt, + /** 整体失败原因码(没有文件、没有元数据等) */ + reason: z.string().nullable(), + files: z.array(ScrapeFileResult), + /** 触发它的整理记录(入库后自动刮削) */ + recordId: Id.nullable(), + /** 引擎 run id */ + taskId: z.string().nullable(), +}); +export type LibraryScrapeState = z.infer; + +export const LibraryScrapeRequest = z.object({ + /** 缺省按设置 organize.scrapeMode */ + mode: ScrapeMode.optional(), +}); +export type LibraryScrapeRequest = z.infer; +export const LibraryScrapeResult = z.object({ + mediaEntityId: Id, + taskId: z.string(), + /** 同一作品已有排队中的刮削,直接复用 */ + deduplicated: z.boolean(), +}); +export type LibraryScrapeResult = z.infer; + export const LibraryDetail = z.object({ entity: MediaEntity, /** 身份映射状态(含 proposed/conflicted,前端区分展示) */ @@ -403,6 +478,8 @@ export const LibraryDetail = z.object({ }), ) .optional(), + /** SCR-1:最近一次刮削(写 NFO 与图片);没刮削过为 null */ + scrape: LibraryScrapeState.nullable().optional(), }); export type LibraryDetail = z.infer; diff --git a/packages/contracts/src/reasons.ts b/packages/contracts/src/reasons.ts index a0c4f6d9..cd083aac 100644 --- a/packages/contracts/src/reasons.ts +++ b/packages/contracts/src/reasons.ts @@ -87,6 +87,14 @@ const REASONS: Record = { read_only_reconcile: { text: "只读对账中,暂不提交下载" }, rejected: { text: "下载器拒绝了这个种子" }, source_failed: { text: "从站点取种子失败", action: "检查站点登录后重新下载" }, + // SCR-1 刮削(作品刮削状态、逐文件结果) + scrape_no_files: { text: "作品没有已入库文件", action: "先整理入库" }, + scrape_no_metadata: { text: "作品还没有元数据", action: "先刷新元数据" }, + scrape_root_read_only: { text: "媒体库目录是只登记模式,不写文件" }, + scrape_exists: { text: "文件已存在,按设置只补缺失" }, + scrape_write_failed: { text: "写入媒体库目录失败", action: "检查目录权限后重新刮削" }, + scrape_image_failed: { text: "图片没下载成功", action: "稍后重新刮削" }, + scrape_no_work_dir: { text: "视频直接放在媒体库目录下,不写剧集和季的文件" }, }; /** diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index a93c4ac2..a1483faf 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -72,6 +72,14 @@ export const OrganizeSettings = z.object({ allowCrossVolumeMove: z.boolean(), /** 整理 / 文件管理识别时视频文件的最小体积(字节),低于它的视为样片跳过;请求(向导)里给的值优先 */ minSizeBytes: z.number().int().nonnegative(), + /** SCR-1:整理入库成功后自动刮削(往媒体库目录写 NFO 与图片);缺省关(同 MP 目录「刮削」缺省) */ + scrapeAfterImport: z.boolean(), + /** SCR-1:目标文件已存在时 missing = 只补缺失(MP「仅缺失」)/ overwrite = 全部覆盖 */ + scrapeMode: z.enum(["missing", "overwrite"]), + /** SCR-1:下载海报、背景图、季海报 */ + scrapeImages: z.boolean(), + /** SCR-1:下载每集剧照(`<视频名>-thumb.jpg`) */ + scrapeEpisodeThumbs: z.boolean(), }); export type OrganizeSettings = z.infer; @@ -618,7 +626,15 @@ export const DEFAULT_SETTINGS: GlobalSettings = { globalImageCache: false, globalImageCacheDays: 7, }, - organize: { defaultMode: "copy", allowCrossVolumeMove: true, minSizeBytes: 50 * 1024 * 1024 }, + organize: { + defaultMode: "copy", + allowCrossVolumeMove: true, + minSizeBytes: 50 * 1024 * 1024, + scrapeAfterImport: false, + scrapeMode: "missing", + scrapeImages: true, + scrapeEpisodeThumbs: true, + }, retention: { seedRetentionDays: 7, logRetentionDays: 180 }, pi: { defaultUpstreamId: null, @@ -927,17 +943,22 @@ export const SETTINGS_SECTIONS: readonly SettingsSectionDef[] = [ { id: "library", label: "媒体库与目录", - description: "媒体库目录、默认整理方式、命名模板", + description: "媒体库目录、默认整理方式、刮削、命名模板", legacyTabs: [{ tab: "library-roots", anchor: "roots" }], anchors: [ { id: "roots", label: "媒体库目录" }, { id: "organize", label: "整理方式" }, + { id: "scrape", label: "刮削" }, { id: "naming", label: "命名模板" }, ], fields: [ "organize.defaultMode", "organize.allowCrossVolumeMove", "organize.minSizeBytes", + "organize.scrapeAfterImport", + "organize.scrapeMode", + "organize.scrapeImages", + "organize.scrapeEpisodeThumbs", "naming.movie", "naming.tv", "naming.variety", diff --git a/packages/contracts/test/db-sync.test.ts b/packages/contracts/test/db-sync.test.ts index bb763cea..d5a0b5f2 100644 --- a/packages/contracts/test/db-sync.test.ts +++ b/packages/contracts/test/db-sync.test.ts @@ -63,6 +63,7 @@ const ROWS: Record LibraryRoot: { row: C.LibraryRoot }, MediaAsset: { row: C.MediaAsset }, MediaEntity: { row: C.MediaEntity }, + MediaScrape: { row: C.MediaScrape }, MediaServerInstance: { row: C.MediaServerInstance }, Observation: { row: C.Observation }, OrganizationFileResult: { row: C.OrganizationFileResult }, @@ -209,6 +210,9 @@ const ENUMS: Record = { MediaAssetStatus: C.AssetStatus, MediaEntityKind: C.MediaKind, MediaEntityStatus: C.MediaEntityStatus, + MediaScrapeMode: C.ScrapeMode, + MediaScrapeStatus: C.ScrapeStatus, + MediaScrapeTrigger: C.ScrapeTrigger, MediaServerInstanceKind: C.MediaServerKind, MediaServerInstanceHealthStatus: C.InstanceHealth, ObservationStatus: C.ObservationStatus, diff --git a/packages/db/migrations/0057_media_scrape.ts b/packages/db/migrations/0057_media_scrape.ts new file mode 100644 index 00000000..5d63eb61 --- /dev/null +++ b/packages/db/migrations/0057_media_scrape.ts @@ -0,0 +1,38 @@ +import type { Kysely } from "kysely"; +import { execDdl } from "../src/migration-helpers.ts"; + +// SCR-1 刮削:每部作品最近一次刮削(往媒体库目录写 NFO 与图片)的状态与逐文件结果。 +// 一部作品一行,重新刮削覆盖;历史看审计(organize.scrape)与引擎运行记录。 + +// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型 +export async function up(db: Kysely): Promise { + await execDdl( + db, + `create table if not exists media_scrape ( + media_entity_id uuid primary key references media_entity (id) on delete cascade, + status text not null default 'queued' check (status in ('queued', 'running', 'succeeded', 'partial', 'failed')), + trigger text not null default 'manual' check (trigger in ('auto', 'manual')), + mode text not null default 'missing' check (mode in ('missing', 'overwrite')), + record_id uuid references organization_record (id) on delete set null, + engine_run_id text, + requested_at timestamptz not null default now(), + started_at timestamptz, + finished_at timestamptz, + written integer not null default 0, + skipped integer not null default 0, + failed integer not null default 0, + reason text, + files jsonb not null default '[]'::jsonb, + created_at timestamptz not null default now(), + updated_at timestamptz not null default now() + ); + drop trigger if exists media_scrape_touch_updated_at on media_scrape; + create trigger media_scrape_touch_updated_at before update on media_scrape + for each row execute function umt_touch_updated_at();`, + ); +} + +// biome-ignore lint/suspicious/noExplicitAny: 同上 +export async function down(db: Kysely): Promise { + await execDdl(db, "drop table if exists media_scrape;"); +} diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index 61c0b627..01be7d21 100644 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -159,6 +159,9 @@ export type MediaAssetKind = "video" | "bluray_dir" | "subtitle" | "audio" | "nf export type MediaAssetStatus = "active" | "replaced" | "retired" | "missing"; export type MediaEntityKind = "movie" | "series" | "variety" | "collection" | "other"; export type MediaEntityStatus = "provisional" | "active" | "merged" | "archived"; +export type MediaScrapeStatus = "queued" | "running" | "succeeded" | "partial" | "failed"; +export type MediaScrapeTrigger = "auto" | "manual"; +export type MediaScrapeMode = "missing" | "overwrite"; export type MediaServerInstanceKind = "emby" | "jellyfin"; export type MediaServerInstanceHealthStatus = "unknown" | "healthy" | "degraded" | "offline"; export type MetadataEnrichmentStatus = "suggested" | "applied" | "failed"; @@ -915,6 +918,25 @@ export interface MediaEntityTable { naming_override: Json | null; } +export interface MediaScrapeTable { + media_entity_id: string; + status: Generated; + trigger: Generated; + mode: Generated; + record_id: string | null; + engine_run_id: string | null; + requested_at: GeneratedTimestamptz; + started_at: Timestamptz | null; + finished_at: Timestamptz | null; + written: Generated; + skipped: Generated; + failed: Generated; + reason: string | null; + files: Generated; + created_at: GeneratedTimestamptz; + updated_at: GeneratedTimestamptz; +} + export interface MediaServerInstanceTable { id: Generated; name: string; @@ -1698,6 +1720,7 @@ export interface DB { mcp_server_config: McpServerConfigTable; media_asset: MediaAssetTable; media_entity: MediaEntityTable; + media_scrape: MediaScrapeTable; media_server_instance: MediaServerInstanceTable; metadata_enrichment: MetadataEnrichmentTable; metadata_field: MetadataFieldTable; @@ -1858,6 +1881,9 @@ export type MediaAssetUpdate = Updateable; export type MediaEntityRow = Selectable; export type NewMediaEntity = Insertable; export type MediaEntityUpdate = Updateable; +export type MediaScrapeRow = Selectable; +export type NewMediaScrape = Insertable; +export type MediaScrapeUpdate = Updateable; export type MediaServerInstanceRow = Selectable; export type NewMediaServerInstance = Insertable; export type MediaServerInstanceUpdate = Updateable; diff --git a/packages/db/test/schema.test.ts b/packages/db/test/schema.test.ts index c878f3f2..b75fd7c7 100644 --- a/packages/db/test/schema.test.ts +++ b/packages/db/test/schema.test.ts @@ -26,6 +26,7 @@ const EXPECTED_TABLES = [ "path_mapping", // 内容 / 身份 "media_entity", + "media_scrape", "content_unit", "title_alias", "external_identity", @@ -153,6 +154,8 @@ const NON_ENTITY_TABLES = [ "kv_cache", // 站点请求计数:(site_id, hour) 主键 "site_request_count", + // 作品最近一次刮削:media_entity_id 主键(SCR-1) + "media_scrape", ]; describe.skipIf(skip)("schema migrations (real PG)", () => {