From 87bf33b42b738910f7dd089c6eb076f1332ac2d9 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 1 Oct 2026 00:08:17 +0800 Subject: [PATCH] =?UTF-8?q?feat(scrape):=20=E4=BB=BB=E6=84=8F=E7=9B=AE?= =?UTF-8?q?=E5=BD=95=E5=B0=B1=E5=9C=B0=E5=88=AE=E5=89=8A,=E6=8C=89?= =?UTF-8?q?=E8=B7=AF=E5=BE=84=E8=AF=86=E5=88=AB=E4=BD=9C=E5=93=81,?= =?UTF-8?q?=E6=AF=8F=E6=AC=A1=E5=88=AE=E5=89=8A=E4=B8=80=E6=9D=A1=E8=AE=B0?= =?UTF-8?q?=E5=BD=95(SCR-2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - POST /files/scrape 改为对目录 / 视频文件刮削:已登记用作品,否则按路径识别(本地 + TMDB), 认不准返回候选,选定后再提交;未登记直接用 TMDB 字段,addToLibrary 顺手建作品 - 作品详情重新刮削、入库后自动刮削改为对作品目录刮削,同一实现 - 迁移 0058_scrape_record 替代 media_scrape;GET /files/scrape-records - 与 MP ScrapingChain 的文件布局差分(7 个场景,3 处登记偏离) Co-Authored-By: Claude Opus 5.5 --- .../__tests__/scrape.test.ts | 62 +- .../modules/acquisition-organize/handlers.ts | 16 +- .../src/modules/acquisition-organize/ports.ts | 20 + .../acquisition-organize/routes/library.ts | 26 +- .../services/scrape-input.ts | 57 ++ .../acquisition-organize/services/scrape.ts | 955 ++++++++++++------ .../src/modules/file-manager/identify.ts | 34 +- apps/server/src/modules/file-manager/probe.ts | 104 +- .../server/src/modules/file-manager/routes.ts | 9 +- .../server/src/modules/file-manager/scrape.ts | 211 ++++ apps/server/test/file-manager/harness.ts | 30 +- .../file-manager/identify-organize.test.ts | 21 +- apps/server/test/file-manager/scrape.test.ts | 443 ++++++++ apps/server/test/file-manager/v2.test.ts | 6 +- .../test/scrape/gen-scrape-layout-mp.py | 144 +++ apps/server/test/scrape/scrape-layout-mp.json | 75 ++ .../test/scrape/scrape-layout-mp.test.ts | 174 ++++ packages/contracts/src/api.ts | 16 +- packages/contracts/src/files.ts | 66 +- packages/contracts/src/index.ts | 1 + packages/contracts/src/organization.ts | 76 +- packages/contracts/src/reasons.ts | 20 +- packages/contracts/src/scrape.ts | 119 +++ packages/contracts/test/db-sync.test.ts | 8 +- packages/db/migrations/0058_scrape_record.ts | 72 ++ packages/db/src/types.ts | 60 +- packages/db/test/schema.test.ts | 5 +- packages/domain/src/scrape/layout.ts | 45 +- packages/domain/test/scrape.test.ts | 50 +- 29 files changed, 2272 insertions(+), 653 deletions(-) create mode 100644 apps/server/src/modules/file-manager/scrape.ts create mode 100644 apps/server/test/file-manager/scrape.test.ts create mode 100644 apps/server/test/scrape/gen-scrape-layout-mp.py create mode 100644 apps/server/test/scrape/scrape-layout-mp.json create mode 100644 apps/server/test/scrape/scrape-layout-mp.test.ts create mode 100644 packages/contracts/src/scrape.ts create mode 100644 packages/db/migrations/0058_scrape_record.ts diff --git a/apps/server/src/modules/acquisition-organize/__tests__/scrape.test.ts b/apps/server/src/modules/acquisition-organize/__tests__/scrape.test.ts index df3e75aa..a959308a 100644 --- a/apps/server/src/modules/acquisition-organize/__tests__/scrape.test.ts +++ b/apps/server/src/modules/acquisition-organize/__tests__/scrape.test.ts @@ -3,11 +3,11 @@ import { join, relative } from "node:path"; import { createTempTree, type TempTree } from "@umt/testkit"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { executeRecord, planImport } from "../services/organize.ts"; -import { runScrape, SCRAPE_TASK } from "../services/scrape.ts"; +import { runScrapeRecord, SCRAPE_TASK, scrapeEntity } from "../services/scrape.ts"; import { createHarness, type Harness, pgAvailable, seedBase, seedEntity } from "./harness.ts"; -// SCR-1 刮削:入库成功后自动排 scrapeMedia → 写 NFO 与图片(临时库根);只补缺失 / 全部覆盖;图片失败不阻塞 NFO; -// 只登记库根不写;作品详情「重新刮削」接口与刮削状态。 +// 刮削(SCR-1 / SCR-2):入库成功后自动对作品目录排 scrapeMedia → 写 NFO 与图片(临时库根);只补缺失 / 全部覆盖; +// 图片失败不阻塞 NFO;只登记库根不写;作品详情「重新刮削」接口与刮削状态(最近一条刮削记录)。 const hasPg = await pgAvailable(); const SIZE = 200_000; @@ -75,13 +75,26 @@ describe.skipIf(!hasPg)("刮削:NFO 与图片写进媒体库目录", () => { return plan.recordId!; } - /** 按引擎记录的入队项跑刮削(假引擎不执行) */ + /** 按引擎记录的入队项跑刮削(假引擎不执行):该作品最近一条刮削记录 */ async function runQueued(entity: string) { - const q = h.engine.enqueued.filter( - (x) => x.task === SCRAPE_TASK && (x.input as { mediaEntityId: string }).mediaEntityId === entity, - ); - expect(q.length).toBeGreaterThan(0); - return runScrape(h.d, q[q.length - 1]!.input as Parameters[1]); + const recs = await h.db + .selectFrom("scrape_record") + .select(["id"]) + .where("media_entity_id", "=", entity) + .where("status", "=", "queued") + .orderBy("requested_at", "desc") + .execute(); + expect(recs.length).toBeGreaterThan(0); + const q = h.engine.enqueued.filter((x) => x.task === SCRAPE_TASK); + expect(q.some((x) => (x.input as { recordId: string }).recordId === recs[0]!.id)).toBe(true); + return runScrapeRecord(h.d, recs[0]!.id); + } + + /** 作品详情「重新刮削」的服务端做法:对作品目录排刮削并执行 */ + async function rescrape(entity: string, mode?: "missing" | "overwrite") { + const q = await scrapeEntity(h.d, entity, { trigger: "manual", mode }); + expect(q.length).toBe(1); + return runScrapeRecord(h.d, q[0]!.recordId); } let series = ""; @@ -165,8 +178,18 @@ describe.skipIf(!hasPg)("刮削:NFO 与图片写进媒体库目录", () => { const rec = await organize(series, "Severance.S01E01.1080p.WEB-DL.H264-GRP.mkv"); await organize(series, "Severance.S01E02.1080p.WEB-DL.H264-GRP.mkv"); const q = h.engine.enqueued.filter((x) => x.task === SCRAPE_TASK); - expect(q[0]!.key).toBe(`${SCRAPE_TASK}:record:${rec}`); - expect(q[0]!.input).toMatchObject({ mediaEntityId: series, trigger: "auto", mode: "missing", recordId: rec }); + expect(q[0]!.key).toBe(`${SCRAPE_TASK}:record:${rec}:0`); + const first = await h.db + .selectFrom("scrape_record") + .selectAll() + .where("id", "=", (q[0]!.input as { recordId: string }).recordId) + .executeTakeFirstOrThrow(); + expect(first).toMatchObject({ + media_entity_id: series, + trigger: "auto", + mode: "missing", + organization_record_id: rec, + }); const r = await runQueued(series); expect(r.status).toBe("partial"); // 背景图下载失败 @@ -213,23 +236,23 @@ describe.skipIf(!hasPg)("刮削:NFO 与图片写进媒体库目录", () => { }); const audit = await h.db .selectFrom("audit_event") - .select(["action", "target_id"]) + .select(["action", "target_id", "target_type"]) .where("action", "=", "organize.scrape") .execute(); - expect(audit.some((a) => a.target_id === series)).toBe(true); + expect(audit.some((a) => a.target_type === "scrape_record" && a.target_id === st.body.data.scrape.id)).toBe(true); }, 60_000); it("只补缺失:已有文件(包括别人写的)不动;全部覆盖:改写", async () => { const files = await tree(lib.root); const show = join(lib.root, files.find((p) => p.endsWith("tvshow.nfo"))!); await writeFile(show, "别的工具写的"); - const again = await runScrape(h.d, { mediaEntityId: series, trigger: "manual", mode: "missing" }); + const again = await rescrape(series, "missing"); expect(await readFile(show, "utf8")).toBe("别的工具写的"); expect(again.files.find((x) => x.role === "tvshow_nfo")).toMatchObject({ action: "skipped", reason: "scrape_exists", }); - const over = await runScrape(h.d, { mediaEntityId: series, trigger: "manual", mode: "overwrite" }); + const over = await rescrape(series, "overwrite"); expect(over.written).toBeGreaterThan(0); expect(await readFile(show, "utf8")).toContain("人生切割术"); }, 60_000); @@ -276,24 +299,23 @@ describe.skipIf(!hasPg)("刮削:NFO 与图片写进媒体库目录", () => { expect(nfo).toContain("169"); }, 60_000); - it("写入失败记原因;只登记库根不写文件;没有入库文件的作品 409", async () => { + it("写入失败记原因;只登记库根不写文件;没有入库文件的作品 412", async () => { const files = await tree(lib.root); const video = files.find((p) => p.includes("星际穿越") && p.endsWith(".mkv"))!; const dirAbs = join(lib.root, video.slice(0, video.lastIndexOf("/"))); if (process.getuid?.() !== 0) { await chmod(dirAbs, 0o555); - const r = await runScrape(h.d, { mediaEntityId: movie, trigger: "manual", mode: "overwrite" }); + const r = await rescrape(movie, "overwrite"); await chmod(dirAbs, 0o755); expect(r.files.some((x) => x.reason === "scrape_write_failed")).toBe(true); } await h.db.updateTable("library_root").set({ management_mode: "register_only" }).execute(); - const ro = await runScrape(h.d, { mediaEntityId: movie, trigger: "manual", mode: "overwrite" }); + const ro = await rescrape(movie, "overwrite"); expect(ro).toMatchObject({ status: "failed", reason: "scrape_root_read_only", written: 0 }); await h.db.updateTable("library_root").set({ management_mode: "managed" }).execute(); const empty = await seedEntity(h.db, { kind: "movie", title: "没入库", year: 2020 }); expect((await h.api("POST", `/library/${empty}/scrape`, {})).status).toBe(412); - const none = await runScrape(h.d, { mediaEntityId: empty, trigger: "manual" }); - expect(none).toMatchObject({ status: "failed", reason: "scrape_no_metadata" }); + expect(await scrapeEntity(h.d, empty, { trigger: "manual" })).toEqual([]); }, 60_000); }); diff --git a/apps/server/src/modules/acquisition-organize/handlers.ts b/apps/server/src/modules/acquisition-organize/handlers.ts index b8c7af42..87ba68e7 100644 --- a/apps/server/src/modules/acquisition-organize/handlers.ts +++ b/apps/server/src/modules/acquisition-organize/handlers.ts @@ -5,7 +5,7 @@ import { type AoDeps, automationBlock, ensureTaskRow, markTask } from "./service import { cleanupDownloadCache, submitDownload } from "./services/downloads.ts"; import { executeRecord, planImport } from "./services/organize.ts"; import { reconcileLibrary } from "./services/reconcile.ts"; -import { runScrape, SCRAPE_TASK } from "./services/scrape.ts"; +import { runScrapeRecord, SCRAPE_TASK } from "./services/scrape.ts"; import { errMessage } from "./util.ts"; // 持久任务处理器。输入只放业务 id;业务状态写业务表;副作用经 withAttemptToken(在 services 里)。 @@ -188,22 +188,16 @@ export function registerAoTasks(registry: TaskRegistry, d: AoDeps) { ); }, }), - // SCR-1 刮削:写 NFO 与图片到媒体库目录(文件道);结果写 media_scrape,不重试(失败的文件在作品详情里重新刮削) + // 刮削(SCR-1 / SCR-2):按刮削记录在目录里就地写 NFO 与图片(文件道);结果写回 scrape_record,不重试(失败的重新刮削) defineTask({ name: SCRAPE_TASK, - version: 1, + version: 2, lane: "file", - inputSchema: z.object({ - mediaEntityId: id, - trigger: z.enum(["auto", "manual"]), - mode: z.enum(["missing", "overwrite"]).optional(), - recordId: id.optional(), - actor: z.object({ type: z.string(), id: z.string().nullable() }).optional(), - }), + inputSchema: z.object({ recordId: id }), retryPolicy: { maximumAttempts: 1 }, handler: async (t) => t.step.run("scrape", async () => { - const r = await runScrape(d, { ...t.input, runId: t.runId }, t.abortSignal); + const r = await runScrapeRecord(d, t.input.recordId, t.abortSignal); return { status: r.status, written: r.written, skipped: r.skipped, failed: r.failed, reason: r.reason }; }), }), diff --git a/apps/server/src/modules/acquisition-organize/ports.ts b/apps/server/src/modules/acquisition-organize/ports.ts index 31c071df..f55f19f7 100644 --- a/apps/server/src/modules/acquisition-organize/ports.ts +++ b/apps/server/src/modules/acquisition-organize/ports.ts @@ -6,6 +6,8 @@ import { type DownloaderClient, diskUsage, type EmbyClient, + fetchRichDetail, + fetchRichSeason, ffprobe, getSiteAdapter, getSiteDefinition, @@ -13,6 +15,8 @@ import { qbAuthConfig, type SecretResolver, SiteError, + type TmdbRichDetail, + type TmdbRichSeasonDetail, type TorrentSource, type VerifyMethod, } from "@umt/adapters"; @@ -88,6 +92,12 @@ export interface AoPorts { * 失败抛错(由刮削记为该图片失败,不影响 NFO)。 */ scrapeImage(tmdbPath: string): Promise<{ content: Uint8Array; mime: string }>; + /** + * SCR-2 刮削未登记的目录:直接取 TMDB 富详情 / 季详情(按设置的元数据语言链,经 TMDB 缓存)。 + * TMDB 没配置返回 null;条目不存在或上游失败抛错。 + */ + scrapeTmdbDetail(type: "movie" | "tv", id: string): Promise; + scrapeTmdbSeason(tvId: string, season: number): Promise; options: AoOptions; } @@ -197,6 +207,16 @@ export function createPorts( }); return images.bytes("original", tmdbPath); }, + async scrapeTmdbDetail(type, id) { + const client = await rt().tmdb(); + if (!client) return null; + return fetchRichDetail(client, type, id, await metadataLanguages(ctx), await metadataTextOptions(ctx)); + }, + async scrapeTmdbSeason(tvId, season) { + const client = await rt().tmdb(); + if (!client) return null; + return fetchRichSeason(client, tvId, season, await metadataLanguages(ctx), await metadataTextOptions(ctx)); + }, diskUsage: (p) => diskUsage(p), probe: (p) => ffprobe(p), async mediaServer(inst) { diff --git a/apps/server/src/modules/acquisition-organize/routes/library.ts b/apps/server/src/modules/acquisition-organize/routes/library.ts index 2a1d4695..6f4257a6 100644 --- a/apps/server/src/modules/acquisition-organize/routes/library.ts +++ b/apps/server/src/modules/acquisition-organize/routes/library.ts @@ -27,7 +27,7 @@ import { sql } from "@umt/db"; import { ApiException } from "../../../http/errors.ts"; import type { ModuleApp } from "../../types.ts"; import { type AoDeps, CANDIDATE_ARTIFACT_TITLE } from "../services/common.ts"; -import { enqueueScrape, loadScrapeState } from "../services/scrape.ts"; +import { latestEntityScrape, scrapeEntity } from "../services/scrape.ts"; import { notFound, toApi, toApiList } from "../util.ts"; import { mount, paginate } from "./helpers.ts"; import { registerLibraryVersionRoutes } from "./library-versions.ts"; @@ -407,21 +407,21 @@ export function registerLibraryRoutes(app: ModuleApp, d: AoDeps) { createdAt: new Date(r.created_at).toISOString(), finishedAt: r.finished_at ? new Date(r.finished_at).toISOString() : null, })), - scrape: await loadScrapeState(d, id), + scrape: await latestEntityScrape(d, id), }; return detail; }, { tag: TAG }, ); - // SCR-1 刮削:作品详情「重新刮削」;与入库后自动刮削、文件管理「刮削」同一任务 scrapeMedia + // 作品详情「重新刮削」「覆盖重写」:对作品的目录刮削(SCR-2:与文件管理「刮削」、入库后自动刮削同一实现) mount( app, { method: "post", path: "/library/{id}/scrape", scope: "organize.execute", - summary: "刮削：把作品的 NFO 与图片写进媒体库目录（后台任务 scrapeMedia；覆盖方式缺省按设置）", + summary: "刮削：对作品目录就地写 NFO 与图片（后台任务 scrapeMedia；覆盖方式缺省按设置）", params: IdParam, body: LibraryScrapeRequest, response: LibraryScrapeResult, @@ -429,22 +429,20 @@ export function registerLibraryRoutes(app: ModuleApp, d: AoDeps) { async (c, { params, body }) => { const e = await db().selectFrom("media_entity").select(["id"]).where("id", "=", params.id).executeTakeFirst(); if (!e) notFound("作品"); - const has = await db() - .selectFrom("media_asset") - .select(["id"]) - .where("media_entity_id", "=", params.id) - .where("status", "=", "active") - .executeTakeFirst(); - if (!has) throw new ApiException("precondition_failed", "作品没有已入库文件。先整理入库，再刮削"); const actor = c.get("actor"); const req = (body ?? {}) as LibraryScrapeRequest; - const r = await enqueueScrape(d, { - mediaEntityId: params.id, + const recs = await scrapeEntity(d, params.id, { trigger: "manual", mode: req.mode, actor: { type: actor?.type === "api_key" ? "api_key" : "user", id: actor?.id ?? null }, }); - return { mediaEntityId: params.id, ...r }; + if (!recs.length) throw new ApiException("precondition_failed", "作品没有已入库文件。到文件管理里对目录刮削"); + return { + mediaEntityId: params.id, + taskId: recs[0]?.taskId ?? null, + deduplicated: recs.every((r) => r.deduplicated), + recordIds: recs.map((r) => r.recordId), + }; }, { tag: TAG, status: 202 }, ); diff --git a/apps/server/src/modules/acquisition-organize/services/scrape-input.ts b/apps/server/src/modules/acquisition-organize/services/scrape-input.ts index 38d1b436..abe9ba14 100644 --- a/apps/server/src/modules/acquisition-organize/services/scrape-input.ts +++ b/apps/server/src/modules/acquisition-organize/services/scrape-input.ts @@ -1,4 +1,6 @@ +import type { TmdbRichDetail, TmdbRichSeasonDetail } from "@umt/adapters"; import type { NfoEpisode, NfoIds, NfoPerson, NfoSeason, NfoWork } from "@umt/domain"; +import { richEntityFields } from "../../sources-identity/service/metadata-rich.ts"; // SCR-1:本地作品 / 季 / 集的最终字段(media_entity.metadata、content_unit.metadata,字段级 {value, …})→ NFO 输入。 // 纯函数,不查库;差分测试(apps/server/test/scrape/nfo-mp-diff.test.ts)直接调用。 @@ -134,3 +136,58 @@ export function imagePathsOf(entityMeta: unknown) { } export const unitPosterPath = (m: unknown) => str(val(m, "posterPath")); export const unitStillPath = (m: unknown) => str(val(m, "stillPath")); + +// ── SCR-2:未登记目录直接用 TMDB 字段(与刷新元数据同一套映射,形状同 media_entity.metadata / content_unit.metadata) ── + +/** TMDB 富详情 → 作品(NFO 输入);字段映射同刷新元数据(richEntityFields) */ +export function tmdbEntity(r: TmdbRichDetail): { entity: EntityForNfo; ids: IdentityMap } { + return { + entity: { + kind: r.entityType === "movie" ? "movie" : "series", + title: r.title, + original_title: r.originalTitle, + year: r.year, + metadata: richEntityFields(r), + }, + ids: { + tmdb: r.externalId, + ...(r.externalIds.imdb ? { imdb: r.externalIds.imdb } : {}), + ...(r.externalIds.tvdb ? { tvdb: r.externalIds.tvdb } : {}), + }, + }; +} + +const fv = (value: unknown) => ({ value }); + +/** TMDB 季详情 → 季与各集(NFO 输入) */ +export function tmdbSeasonUnits(sd: TmdbRichSeasonDetail): { season: UnitForNfo; episodes: Map } { + const season: UnitForNfo = { + season_number: sd.seasonNumber, + episode_number: null, + title: sd.name, + air_date: sd.airDate, + metadata: { + name: fv(sd.name), + overview: fv(sd.overview), + airDate: fv(sd.airDate), + posterPath: fv(sd.posterPath), + }, + }; + const episodes = new Map(); + for (const e of sd.episodes) + episodes.set(e.episodeNumber, { + season_number: e.seasonNumber, + episode_number: e.episodeNumber, + title: e.name, + air_date: e.airDate, + metadata: { + name: fv(e.name), + overview: fv(e.overview), + airDate: fv(e.airDate?.slice(0, 10) ?? null), + runtime: fv(e.runtime), + stillPath: fv(e.stillPath), + voteAverage: fv(e.voteAverage), + }, + }); + return { season, episodes }; +} diff --git a/apps/server/src/modules/acquisition-organize/services/scrape.ts b/apps/server/src/modules/acquisition-organize/services/scrape.ts index 3e9c7859..6db75878 100644 --- a/apps/server/src/modules/acquisition-organize/services/scrape.ts +++ b/apps/server/src/modules/acquisition-organize/services/scrape.ts @@ -1,150 +1,355 @@ import { randomUUID } from "node:crypto"; -import { lstat, mkdir, rename, rm, writeFile } from "node:fs/promises"; +import { lstat, mkdir, readdir, realpath, rename, rm, writeFile } from "node:fs/promises"; import { dirname, isAbsolute, join, relative, resolve } from "node:path"; import { checkRoot, DEFAULT_ROOT_SENTINEL } from "@umt/adapters"; import { DEFAULT_SETTINGS, - type LibraryScrapeState, type OrganizeSettings, type ScrapeFileResult, + type ScrapeMatch, type ScrapeMode, + type ScrapeRecord, } from "@umt/contracts"; -import { sql } from "@umt/db"; import { episodeNfo, + fileKindOf, inferWorkDir, + isSeasonDirName, + parseReleasePath, planScrapeTargets, type ScrapeTarget, type ScrapeVideo, seasonNfo, + seasonOfDirName, workNfo, } from "@umt/domain"; import { errMessage, jsonb } from "../util.ts"; import type { AoDeps } from "./common.ts"; import { + type EntityForNfo, type IdentityMap, imagePathsOf, nfoEpisode, nfoSeason, nfoWork, + tmdbEntity, + tmdbSeasonUnits, type UnitForNfo, unitPosterPath, unitStillPath, } from "./scrape-input.ts"; -// SCR-1 刮削:把作品的最终字段写成 NFO,下载海报 / 背景图 / 季海报 / 剧照,放进媒体库目录(Emby / Jellyfin / Kodi 读取)。 -// - 触发:整理入库成功后(设置 organize.scrapeAfterImport)、作品详情「重新刮削」、文件管理「刮削」,都排同一个任务 scrapeMedia(文件道)。 -// - 目标文件与命名:packages/domain/src/scrape/layout.ts(文件名跟随已入库视频,即命名模板的结果);内容:scrape/nfo.ts。 -// - 覆盖:missing 只补缺失(已存在的跳过)/ overwrite 全部覆盖;NFO 先写,图片后下(并发 4,失败只记该文件,不影响 NFO)。 -// - 写入:同目录临时文件 + rename(原子),UTF-8;只写 managed 库根,库根离线整体失败;目标必须仍在库根内。 -// - 结果:media_scrape 一部作品一行(最近一次,逐文件结果)+ 审计 organize.scrape;SSE files.changed 让文件管理刷新。 +// 刮削(SCR-1 写 NFO 与图片;SCR-2 照 MP 简化为"对目录 / 视频文件就地刮削"): +// - 对象是登记目录(库根 / 下载根)里的一个目录或视频文件,不要求先登记。识别在调用方做(文件管理按路径识别、 +// 作品详情与入库后自动刮削直接给作品),这里只按识别结果(本地作品或 TMDB 条目)在该目录里写文件。 +// - 范围对照 MP ScrapingChain.scrape_metadata: +// 电影文件 → <视频名>.nfo + 同目录海报 / 背景图;电影目录 → 目录里每个视频的 NFO + 目录海报 / 背景图(蓝光原盘写 <目录名>.nfo); +// 剧集目录 → tvshow.nfo + 海报 / 背景图 + 季目录(Season 1 / S01 / Specials)的 season.nfo 与季海报 + 每集 NFO 与剧照; +// 季目录 → season.nfo + 季海报(季目录 poster.jpg 与上级 seasonNN-poster.jpg)+ 每集;单集文件 → 该集 NFO 与剧照。 +// - 字段:有本地作品且有元数据时用最终字段(字段锁、人工覆盖优先);否则按 TMDB id 直接取 TMDB(与刷新元数据同一映射)。 +// - 覆盖:missing 只补缺失 / overwrite 全部覆盖;写入走同目录临时文件 + rename;目标必须在登记目录内。 +// - 每次刮削一条 scrape_record(目录、识别结果、逐文件结果、原因);审计 organize.scrape;SSE files.changed。 export const SCRAPE_TASK = "scrapeMedia"; const IMAGE_CONCURRENCY = 4; - -export interface ScrapeInput { - mediaEntityId: string; - trigger: "auto" | "manual"; - mode?: ScrapeMode | undefined; - recordId?: string | undefined; - /** 手动刮削的发起人(审计) */ - actor?: { type: string; id: string | null } | undefined; -} +const DEDUPE_MS = 30 * 60_000; +const TEMP_RE = /(\.uhd-partial|\.!qB|\.part)$/i; export async function scrapeSettings(d: AoDeps): Promise { const s = await d.ctx.settings.get("organize").catch(() => DEFAULT_SETTINGS.organize); return { ...DEFAULT_SETTINGS.organize, ...s }; } -/** - * 排刮削任务。同一作品已有排队中(还没开始)的刮削时直接复用(deduplicated); - * 自动刮削按整理记录幂等(同一记录只排一次)。 - */ -export async function enqueueScrape(d: AoDeps, input: ScrapeInput): Promise<{ taskId: string; deduplicated: boolean }> { - const db = d.ctx.db; - const mode = input.mode ?? (await scrapeSettings(d)).scrapeMode; - const cur = await db - .selectFrom("media_scrape") - .select(["status", "engine_run_id", "requested_at"]) - .where("media_entity_id", "=", input.mediaEntityId) - .executeTakeFirst(); - if (cur?.status === "queued" && cur.engine_run_id && Date.now() - new Date(cur.requested_at).getTime() < 30 * 60_000) - return { taskId: cur.engine_run_id, deduplicated: true }; - const key = - input.trigger === "auto" && input.recordId - ? `${SCRAPE_TASK}:record:${input.recordId}` - : `${SCRAPE_TASK}:manual:${input.mediaEntityId}:${randomUUID()}`; - const { runId, deduplicated } = await d.ctx.engine.enqueue( - SCRAPE_TASK, - { - mediaEntityId: input.mediaEntityId, - trigger: input.trigger, - mode, - ...(input.recordId ? { recordId: input.recordId } : {}), - ...(input.actor ? { actor: input.actor } : {}), - }, - { idempotencyKey: key, traceId: randomUUID() }, - ); - if (!deduplicated) - await sql`insert into media_scrape (media_entity_id, status, trigger, mode, record_id, engine_run_id, requested_at, - started_at, finished_at, written, skipped, failed, reason, files) - values (${input.mediaEntityId}, 'queued', ${input.trigger}, ${mode}, ${input.recordId ?? null}, ${runId}, now(), - null, null, 0, 0, 0, null, '[]'::jsonb) - on conflict (media_entity_id) do update set status = 'queued', trigger = excluded.trigger, mode = excluded.mode, - record_id = excluded.record_id, engine_run_id = excluded.engine_run_id, requested_at = now(), updated_at = now()`.execute( - db, - ); - return { taskId: runId, deduplicated: !!deduplicated }; +/** 刮削的位置:登记目录(id + 本机路径)+ 目录内相对路径 */ +export interface ScrapeLocation { + /** `lib:` / `dl::` */ + rootId: string; + rootPath: string; + libraryRootId: string | null; + path: string; } -/** 整理入库成功后:设置打开时自动排刮削(失败只记日志,不影响整理结果) */ -export async function scrapeAfterImport(d: AoDeps, entityId: string | null, recordId: string): Promise { - if (!entityId) return; - try { - const s = await scrapeSettings(d); - if (!s.scrapeAfterImport) return; - await enqueueScrape(d, { mediaEntityId: entityId, trigger: "auto", recordId }); - } catch (err) { - d.ctx.logger.warn({ entityId, recordId, err: { message: errMessage(err) } }, "入库后自动刮削没排上"); - } +export interface QueueScrapeInput extends ScrapeLocation { + match: ScrapeMatch; + mode?: ScrapeMode | undefined; + trigger: "auto" | "manual"; + organizationRecordId?: string | null | undefined; + actor?: { type: string; id: string | null } | undefined; + /** 引擎幂等键(入库后自动刮削按整理记录) */ + idempotencyKey?: string | undefined; } -export async function loadScrapeState(d: AoDeps, entityId: string): Promise { - const r = await d.ctx.db - .selectFrom("media_scrape") - .selectAll() - .where("media_entity_id", "=", entityId) - .executeTakeFirst(); - if (!r) return null; - const iso = (t: Date | string | null) => (t ? new Date(t).toISOString() : null); +type Row = { + id: string; + root_id: string; + path: string; + status: string; + trigger: string; + mode: string; + match: unknown; + written: number; + skipped: number; + failed: number; + reason: string | null; + files: unknown; + organization_record_id: string | null; + engine_run_id: string | null; + requested_at: Date | string; + started_at: Date | string | null; + finished_at: Date | string | null; +}; + +const iso = (t: Date | string | null) => (t ? new Date(t).toISOString() : null); + +export function scrapeRecordView(r: Row): ScrapeRecord { return { - status: r.status as LibraryScrapeState["status"], - trigger: r.trigger as LibraryScrapeState["trigger"], + id: r.id, + root: r.root_id, + path: r.path, + status: r.status as ScrapeRecord["status"], + trigger: r.trigger as ScrapeRecord["trigger"], mode: r.mode as ScrapeMode, - requestedAt: iso(r.requested_at)!, - startedAt: iso(r.started_at), - finishedAt: iso(r.finished_at), + match: (r.match ?? null) as ScrapeMatch | null, written: r.written, skipped: r.skipped, failed: r.failed, reason: r.reason, files: (r.files ?? []) as ScrapeFileResult[], - recordId: r.record_id, + organizationRecordId: r.organization_record_id, taskId: r.engine_run_id, + requestedAt: iso(r.requested_at)!, + startedAt: iso(r.started_at), + finishedAt: iso(r.finished_at), }; } +export async function loadScrapeRecords(d: AoDeps, ids: readonly string[]): Promise { + if (!ids.length) return []; + const rows = await d.ctx.db + .selectFrom("scrape_record") + .selectAll() + .where("id", "in", [...ids]) + .execute(); + const by = new Map(rows.map((r) => [r.id, scrapeRecordView(r)])); + return ids.map((id) => by.get(id)).filter((x): x is ScrapeRecord => !!x); +} + +/** 作品详情:该作品最近一次刮削 */ +export async function latestEntityScrape(d: AoDeps, entityId: string): Promise { + const r = await d.ctx.db + .selectFrom("scrape_record") + .selectAll() + .where("media_entity_id", "=", entityId) + .orderBy("requested_at", "desc") + .limit(1) + .executeTakeFirst(); + return r ? scrapeRecordView(r) : null; +} + +/** + * 建一条刮削记录并排任务 scrapeMedia。同一目录已有排队中(还没开始,30 分钟内)的同方式、同作品刮削时直接复用。 + */ +export async function queueScrape( + d: AoDeps, + input: QueueScrapeInput, +): Promise<{ recordId: string; taskId: string | null; deduplicated: boolean }> { + const db = d.ctx.db; + const mode = input.mode ?? (await scrapeSettings(d)).scrapeMode; + const cur = await db + .selectFrom("scrape_record") + .select(["id", "engine_run_id", "media_entity_id", "match"]) + .where("root_id", "=", input.rootId) + .where("path", "=", input.path) + .where("mode", "=", mode) + .where("status", "=", "queued") + .where("requested_at", ">", new Date(Date.now() - DEDUPE_MS)) + .orderBy("requested_at", "desc") + .executeTakeFirst(); + const sameMatch = + !!cur && + (cur.media_entity_id ?? null) === input.match.mediaEntityId && + ((cur.match as ScrapeMatch | null)?.externalId ?? null) === input.match.externalId; + if (cur && sameMatch && cur.engine_run_id) return { recordId: cur.id, taskId: cur.engine_run_id, deduplicated: true }; + const rec = await db + .insertInto("scrape_record") + .values({ + root_id: input.rootId, + root_path: input.rootPath, + library_root_id: input.libraryRootId, + path: input.path, + media_entity_id: input.match.mediaEntityId, + match: jsonb(input.match), + status: "queued", + trigger: input.trigger, + mode, + organization_record_id: input.organizationRecordId ?? null, + actor_type: input.actor?.type ?? (input.trigger === "auto" ? "system" : null), + actor_id: input.actor?.id ?? null, + }) + .returning("id") + .executeTakeFirstOrThrow(); + try { + const { runId, deduplicated } = await d.ctx.engine.enqueue( + SCRAPE_TASK, + { recordId: rec.id }, + { idempotencyKey: input.idempotencyKey ?? `${SCRAPE_TASK}:${rec.id}`, traceId: randomUUID() }, + ); + if (deduplicated) { + // 同一幂等键已排过(自动刮削重放):这条记录作废,返回已有的 + await db.deleteFrom("scrape_record").where("id", "=", rec.id).execute(); + const prev = await db + .selectFrom("scrape_record") + .select(["id"]) + .where("engine_run_id", "=", runId) + .executeTakeFirst(); + return { recordId: prev?.id ?? rec.id, taskId: runId, deduplicated: true }; + } + await db.updateTable("scrape_record").set({ engine_run_id: runId }).where("id", "=", rec.id).execute(); + return { recordId: rec.id, taskId: runId, deduplicated: false }; + } catch (err) { + await db + .updateTable("scrape_record") + .set({ status: "failed", reason: "task_enqueue_failed", finished_at: new Date() }) + .where("id", "=", rec.id) + .execute() + .catch(() => {}); + throw err; + } +} + +// ── 作品 → 目录(作品详情「重新刮削」、入库后自动刮削) ── + +const isMovieKind = (k: string) => k === "movie" || k === "collection" || k === "other"; + +/** 本地作品 → 识别结果(带已接受的 TMDB id) */ +export async function entityScrapeMatch( + d: AoDeps, + entityId: string, + source: ScrapeMatch["source"], +): Promise { + const e = await d.ctx.db + .selectFrom("media_entity") + .select(["id", "kind", "title", "year", "status"]) + .where("id", "=", entityId) + .executeTakeFirst(); + if (!e || e.status === "merged") return null; + const tmdb = await d.ctx.db + .selectFrom("external_identity") + .select(["external_id"]) + .where("media_entity_id", "=", entityId) + .where("content_unit_id", "is", null) + .where("provider", "=", "tmdb") + .where("status", "=", "accepted") + .orderBy("updated_at", "desc") + .executeTakeFirst(); + return { + mediaEntityId: e.id, + provider: tmdb ? "tmdb" : null, + externalId: tmdb?.external_id ?? null, + kind: isMovieKind(e.kind) ? "movie" : "tv", + title: e.title, + year: e.year, + source, + }; +} + +/** + * 作品的刮削目录:按库根分组,优先作品路径绑定,其余视频按路径推作品目录(剧集父目录像季目录时取上一级); + * 推不出(视频直接在库根)时对视频文件本身刮削。只取启用的库根里 active 视频的位置。 + */ +export async function entityScrapeLocations( + d: AoDeps, + entityId: string, + kind: "movie" | "tv", +): Promise { + const db = d.ctx.db; + const locs = await db + .selectFrom("media_asset as a") + .innerJoin("asset_location as l", "l.media_asset_id", "a.id") + .innerJoin("library_root as r", "r.id", "l.library_root_id") + .select(["l.relative_path", "r.id as root_id", "r.local_path"]) + .where("a.media_entity_id", "=", entityId) + .where("a.status", "=", "active") + .where("a.kind", "in", ["video", "bluray_dir"]) + .where("l.location_kind", "=", "library") + .where("l.status", "in", ["available", "unavailable", "suspected_missing"]) + .where("r.enabled", "=", true) + .orderBy("l.relative_path") + .execute(); + const bindings = await db + .selectFrom("path_binding") + .select(["library_root_id", "relative_path"]) + .where("media_entity_id", "=", entityId) + .where("binding_kind", "=", "entity") + .where("status", "=", "active") + .execute(); + const out = new Map(); + for (const l of locs) { + const bound = bindings.find( + (b) => + b.library_root_id === l.root_id && + !!b.relative_path && + (l.relative_path === b.relative_path || l.relative_path.startsWith(`${b.relative_path}/`)), + ); + const path = bound?.relative_path ?? inferWorkDir(kind, l.relative_path) ?? l.relative_path; + const key = `${l.root_id}\u0000${path}`; + if (!out.has(key)) + out.set(key, { rootId: `lib:${l.root_id}`, rootPath: l.local_path, libraryRootId: l.root_id, path }); + } + // 父目录已在列表里的子路径去掉,避免同一处刮两次 + const all = [...out.values()]; + return all.filter( + (x) => !all.some((y) => y !== x && y.rootId === x.rootId && !!y.path && x.path.startsWith(`${y.path}/`)), + ); +} + +/** 对作品的目录刮削(每个目录一条记录);没有已入库文件时返回空 */ +export async function scrapeEntity( + d: AoDeps, + entityId: string, + opts: { + mode?: ScrapeMode | undefined; + trigger: "auto" | "manual"; + organizationRecordId?: string | undefined; + actor?: { type: string; id: string | null } | undefined; + }, +): Promise<{ recordId: string; taskId: string | null; deduplicated: boolean }[]> { + const match = await entityScrapeMatch(d, entityId, "library"); + if (!match) return []; + const locs = await entityScrapeLocations(d, entityId, match.kind); + const out = []; + for (const [i, loc] of locs.entries()) + out.push( + await queueScrape(d, { + ...loc, + match, + mode: opts.mode, + trigger: opts.trigger, + organizationRecordId: opts.organizationRecordId, + actor: opts.actor, + idempotencyKey: opts.organizationRecordId + ? `${SCRAPE_TASK}:record:${opts.organizationRecordId}:${i}` + : undefined, + }), + ); + return out; +} + +/** 整理入库成功后:设置打开时对作品目录自动刮削(失败只记日志,不影响整理结果) */ +export async function scrapeAfterImport(d: AoDeps, entityId: string | null, recordId: string): Promise { + if (!entityId) return; + try { + const s = await scrapeSettings(d); + if (!s.scrapeAfterImport) return; + await scrapeEntity(d, entityId, { trigger: "auto", organizationRecordId: recordId }); + } catch (err) { + d.ctx.logger.warn({ entityId, recordId, err: { message: errMessage(err) } }, "入库后自动刮削没排上"); + } +} + // ── 执行 ── -interface RootGroup { - rootId: string; - rootPath: string; - managed: boolean; - sentinel: string | null; - videos: ScrapeVideo[]; -} - -/** 原子写:同目录临时文件 → rename;目录不存在时建(只在库根内) */ +/** 原子写:同目录临时文件 → rename;目录不存在时建(只在登记目录内) */ async function atomicWrite(abs: string, content: Uint8Array | string): Promise { await mkdir(dirname(abs), { recursive: true }); const tmp = join(dirname(abs), `.${randomUUID()}.umt-scrape.tmp`); @@ -161,6 +366,9 @@ const inside = (root: string, abs: string) => { const r = relative(root, abs); return !!r && !r.startsWith("..") && !isAbsolute(r); }; +const joinRel = (dir: string, name: string) => (dir ? `${dir}/${name}` : name); +const baseOf = (rel: string) => rel.slice(rel.lastIndexOf("/") + 1); +const dirOf = (rel: string) => (rel.includes("/") ? rel.slice(0, rel.lastIndexOf("/")) : ""); /** 有限并发 map */ async function pool(items: readonly T[], n: number, fn: (t: T) => Promise): Promise { @@ -176,258 +384,329 @@ async function pool(items: readonly T[], n: number, fn: (t: T) => Promise< return out; } -export interface ScrapeRunResult { - status: LibraryScrapeState["status"]; - written: number; - skipped: number; - failed: number; - reason: string | null; - files: ScrapeFileResult[]; +async function listDir(rootReal: string, rel: string) { + const names = await readdir(join(rootReal, rel), { withFileTypes: true }).catch(() => []); + const visible = names.filter((n) => !n.name.startsWith(".")).sort((a, b) => a.name.localeCompare(b.name)); + return { + videos: visible + .filter((n) => n.isFile() && fileKindOf(n.name) === "video" && !TEMP_RE.test(n.name)) + .map((n) => joinRel(rel, n.name)), + dirs: visible.filter((n) => n.isDirectory()).map((n) => n.name), + }; } -export async function runScrape( - d: AoDeps, - input: ScrapeInput & { runId?: string }, - signal?: AbortSignal, -): Promise { - const db = d.ctx.db; - const settings = await scrapeSettings(d); - const mode: ScrapeMode = input.mode ?? settings.scrapeMode; - const entityId = input.mediaEntityId; - const now = () => d.ports.now(); - await sql`insert into media_scrape (media_entity_id, status, trigger, mode, record_id, engine_run_id, started_at) - values (${entityId}, 'running', ${input.trigger}, ${mode}, ${input.recordId ?? null}, ${input.runId ?? null}, ${now()}) - on conflict (media_entity_id) do update set status = 'running', trigger = excluded.trigger, mode = excluded.mode, - record_id = coalesce(excluded.record_id, media_scrape.record_id), - engine_run_id = coalesce(excluded.engine_run_id, media_scrape.engine_run_id), - started_at = excluded.started_at, finished_at = null, reason = null, updated_at = now()` - .execute(db) - .catch(() => {}); +interface Layout { + videos: string[]; + workDir: string | null; + parts: { work: boolean; season: boolean }; + /** 季目录刮削时目录名给出的季号 */ + dirSeason: number | null; +} - const finish = async (r: ScrapeRunResult, rootDirs: Map> = new Map()) => { - await db - .updateTable("media_scrape") +/** 选中的目录 / 文件 → 要刮的视频与范围(对照 MP _handle_movie_* / _handle_tv_*) */ +async function layoutOf(rootReal: string, rel: string, isDir: boolean, kind: "movie" | "tv"): Promise { + const parent = dirOf(rel) || null; + const all = { work: true, season: true }; + if (!isDir) + return kind === "movie" + ? { videos: [rel], workDir: parent, parts: all, dirSeason: null } + : { videos: [rel], workDir: null, parts: { work: false, season: false }, dirSeason: null }; + const here = await listDir(rootReal, rel); + if (kind === "movie") { + // 蓝光原盘目录:写 <目录名>.nfo(借一个同名的虚拟视频路径让布局按目录名命名)+ 目录海报 + if (rel && here.dirs.some((n) => /^(BDMV|VIDEO_TS)$/i.test(n))) + return { videos: [joinRel(rel, `${baseOf(rel)}.bluray`)], workDir: rel, parts: all, dirSeason: null }; + return { videos: here.videos, workDir: rel || null, parts: all, dirSeason: null }; + } + if (rel && isSeasonDirName(baseOf(rel))) + return { + videos: here.videos, + workDir: parent, + parts: { work: false, season: true }, + dirSeason: seasonOfDirName(baseOf(rel)), + }; + const videos = [...here.videos]; + for (const sub of here.dirs) + if (isSeasonDirName(sub)) videos.push(...(await listDir(rootReal, joinRel(rel, sub))).videos); + return { videos, workDir: rel || null, parts: all, dirSeason: null }; +} + +/** 已登记视频的季集(覆盖单元里最小的一集,多集文件按首集) */ +async function coverageOf( + d: AoDeps, + libraryRootId: string, + entityId: string, + videos: readonly string[], + units: readonly { id: string; kind: string; season_number: number | null; episode_number: number | null }[], +): Promise> { + const out = new Map(); + if (!videos.length) return out; + const rows = await d.ctx.db + .selectFrom("asset_location as l") + .innerJoin("media_asset as a", "a.id", "l.media_asset_id") + .innerJoin("asset_unit_coverage as c", "c.media_asset_id", "a.id") + .select(["l.relative_path", "c.content_unit_id"]) + .where("l.library_root_id", "=", libraryRootId) + .where("l.relative_path", "in", [...videos]) + .where("l.status", "<>", "removed") + .where("a.media_entity_id", "=", entityId) + .execute(); + const byId = new Map(units.map((u) => [u.id, u])); + for (const r of rows) { + const u = byId.get(r.content_unit_id); + if (u?.kind !== "episode" || u.season_number === null || u.episode_number === null) continue; + const cur = out.get(r.relative_path); + if (!cur || u.season_number < cur.season || (u.season_number === cur.season && u.episode_number < cur.episode)) + out.set(r.relative_path, { season: u.season_number, episode: u.episode_number }); + } + return out; +} + +interface Fields { + entity: EntityForNfo; + ids: IdentityMap; + seasonUnit: Map; + episodeUnit: Map; +} + +export async function runScrapeRecord(d: AoDeps, recordId: string, signal?: AbortSignal): Promise { + const db = d.ctx.db; + const rec = await db.selectFrom("scrape_record").selectAll().where("id", "=", recordId).executeTakeFirst(); + if (!rec) throw new Error(`刮削记录不存在：${recordId}`); + if (rec.status !== "queued" && rec.status !== "running") return scrapeRecordView(rec); + const now = () => d.ports.now(); + await db + .updateTable("scrape_record") + .set({ status: "running", started_at: now(), reason: null }) + .where("id", "=", recordId) + .execute(); + const settings = await scrapeSettings(d); + const mode = rec.mode as ScrapeMode; + let match = (rec.match ?? null) as ScrapeMatch | null; + const dirs = new Set(); + + const finish = async (r: { + status: ScrapeRecord["status"]; + reason: string | null; + files: ScrapeFileResult[]; + }): Promise => { + const written = r.files.filter((f) => f.action === "written").length; + const failed = r.files.filter((f) => f.action === "failed").length; + const skipped = r.files.filter((f) => f.action === "skipped").length; + const row = await db + .updateTable("scrape_record") .set({ status: r.status, - finished_at: now(), - written: r.written, - skipped: r.skipped, - failed: r.failed, reason: r.reason, files: jsonb(r.files), - updated_at: now(), + written, + skipped, + failed, + match: match ? jsonb(match) : null, + finished_at: now(), }) - .where("media_entity_id", "=", entityId) - .execute() - .catch(() => {}); + .where("id", "=", recordId) + .returningAll() + .executeTakeFirstOrThrow(); await db .insertInto("audit_event") .values({ - actor_type: input.actor?.type ?? "system", - actor_id: input.actor?.id ?? null, + actor_type: rec.actor_type ?? "system", + actor_id: rec.actor_id, action: "organize.scrape", - target_type: "media_entity", - target_id: entityId, + target_type: "scrape_record", + target_id: recordId, payload: jsonb({ - trigger: input.trigger, + root: rec.root_id, + path: rec.path, + trigger: rec.trigger, mode, - recordId: input.recordId ?? null, + match, + organizationRecordId: rec.organization_record_id, status: r.status, - written: r.written, - skipped: r.skipped, - failed: r.failed, + written, + skipped, + failed, reason: r.reason, files: r.files.filter((f) => f.action !== "skipped").slice(0, 200), }), }) .execute() .catch(() => {}); - for (const [rootId, dirs] of rootDirs) - if (dirs.size) void d.ctx.events.publish("files.changed", { rootId: `lib:${rootId}`, dirs: [...dirs] }); - return r; + if (dirs.size) void d.ctx.events.publish("files.changed", { rootId: rec.root_id, dirs: [...dirs] }); + return scrapeRecordView(row); }; - const fail = (reason: string) => finish({ status: "failed", written: 0, skipped: 0, failed: 0, reason, files: [] }); + const fail = (reason: string) => finish({ status: "failed", reason, files: [] }); - const e = await db - .selectFrom("media_entity") - .select(["id", "kind", "title", "original_title", "year", "metadata", "status"]) - .where("id", "=", entityId) - .executeTakeFirst(); - if (!e || e.status === "merged") return fail("entity_not_found"); - const kind: "movie" | "tv" = e.kind === "movie" || e.kind === "collection" || e.kind === "other" ? "movie" : "tv"; - const meta = (e.metadata ?? {}) as Record; - if (!Object.keys(meta).length) return fail("scrape_no_metadata"); + try { + if (!match) return await fail("scrape_no_metadata"); - // 已入库的视频位置(库根内) - const locs = await db - .selectFrom("media_asset as a") - .innerJoin("asset_location as l", "l.media_asset_id", "a.id") - .innerJoin("library_root as r", "r.id", "l.library_root_id") - .select([ - "a.id as asset_id", - "l.relative_path", - "r.id as root_id", - "r.local_path", - "r.management_mode", - "r.enabled", - "r.sentinel_file", - ]) - .where("a.media_entity_id", "=", entityId) - .where("a.status", "=", "active") - .where("a.kind", "=", "video") - .where("l.location_kind", "=", "library") - .where("l.status", "in", ["available", "unavailable", "suspected_missing"]) - .execute(); - if (!locs.length) return fail("scrape_no_files"); - - const assetIds = [...new Set(locs.map((l) => l.asset_id))]; - const cov = await db - .selectFrom("asset_unit_coverage") - .select(["media_asset_id", "content_unit_id"]) - .where("media_asset_id", "in", assetIds) - .execute(); - const units = await db - .selectFrom("content_unit") - .select(["id", "kind", "season_number", "episode_number", "title", "air_date", "metadata"]) - .where("media_entity_id", "=", entityId) - .where("status", "!=", "retired") - .orderBy("created_at") - .execute(); - const unitById = new Map(units.map((u) => [u.id, u])); - const seasonUnit = new Map(); - const episodeUnit = new Map(); - for (const u of units) { - if (u.kind === "season" && u.season_number !== null && !seasonUnit.has(u.season_number)) - seasonUnit.set(u.season_number, u); - if (u.kind === "episode" && u.season_number !== null && u.episode_number !== null) { - const k = `${u.season_number}:${u.episode_number}`; - if (!episodeUnit.has(k)) episodeUnit.set(k, u); + // 登记目录:库根按标记文件 / 管理模式;下载根按目录可访问 + if (rec.library_root_id) { + const lr = await db + .selectFrom("library_root") + .select(["local_path", "management_mode", "enabled", "sentinel_file"]) + .where("id", "=", rec.library_root_id) + .executeTakeFirst(); + if (!lr?.enabled) return await fail("scrape_root_offline"); + if (lr.management_mode !== "managed") return await fail("scrape_root_read_only"); + const health = await checkRoot(lr.local_path, lr.sentinel_file ?? DEFAULT_ROOT_SENTINEL); + if (health.kind !== "online") return await fail("scrape_root_offline"); } - } + const rootReal = await realpath(rec.root_path).catch(() => null); + if (!rootReal) return await fail("scrape_root_offline"); + const abs = resolve(rootReal, rec.path); + if (rec.path && !inside(rootReal, abs)) return await fail("path_escape"); + const st = await lstat(abs).catch(() => null); + if (!st) return await fail("scrape_path_missing"); + const real = await realpath(abs).catch(() => null); + if (!real || (real !== rootReal && !inside(rootReal, real))) return await fail("path_escape"); + const isDir = st.isDirectory(); + if (!isDir && !(st.isFile() && fileKindOf(abs) === "video")) return await fail("scrape_not_video"); - const groups = new Map(); - for (const l of locs) { - if (!l.enabled) continue; - const g = groups.get(l.root_id) ?? { - rootId: l.root_id, - rootPath: l.local_path, - managed: l.management_mode === "managed", - sentinel: l.sentinel_file, - videos: [], - }; - let season: number | null = null; - let episode: number | null = null; - if (kind === "tv") { - const eps = cov - .filter((c) => c.media_asset_id === l.asset_id) - .map((c) => unitById.get(c.content_unit_id)) - .filter((u) => u?.kind === "episode" && u.season_number !== null && u.episode_number !== null) - .sort((a, b) => a!.season_number! - b!.season_number! || a!.episode_number! - b!.episode_number!); - season = eps[0]?.season_number ?? null; - episode = eps[0]?.episode_number ?? null; + const kind = match.kind; + const lay = await layoutOf(rootReal, rec.path, isDir, kind); + if (!lay.videos.length) return await fail("scrape_no_videos"); + + // ── 字段:本地作品(有元数据)优先 ── + let fields: Fields | null = null; + let coverage = new Map(); + if (match.mediaEntityId) { + const e = await db + .selectFrom("media_entity") + .select(["id", "kind", "title", "original_title", "year", "metadata", "status"]) + .where("id", "=", match.mediaEntityId) + .executeTakeFirst(); + if (e && e.status !== "merged") { + const ids: IdentityMap = {}; + const idRows = await db + .selectFrom("external_identity") + .select(["provider", "external_id"]) + .where("media_entity_id", "=", e.id) + .where("content_unit_id", "is", null) + .where("status", "=", "accepted") + .orderBy("updated_at", "desc") + .execute(); + for (const r of idRows) { + const p = r.provider.toLowerCase() as keyof IdentityMap; + if (["tmdb", "imdb", "tvdb", "douban"].includes(p) && !ids[p]) ids[p] = r.external_id; + } + if (!match.externalId && ids.tmdb) match = { ...match, provider: "tmdb", externalId: ids.tmdb }; + if (Object.keys((e.metadata ?? {}) as object).length) { + const units = await db + .selectFrom("content_unit") + .select(["id", "kind", "season_number", "episode_number", "title", "air_date", "metadata"]) + .where("media_entity_id", "=", e.id) + .where("status", "!=", "retired") + .orderBy("created_at") + .execute(); + fields = { entity: e, ids, seasonUnit: new Map(), episodeUnit: new Map() }; + for (const u of units) { + if (u.kind === "season" && u.season_number !== null && !fields.seasonUnit.has(u.season_number)) + fields.seasonUnit.set(u.season_number, u); + if (u.kind === "episode" && u.season_number !== null && u.episode_number !== null) { + const k = `${u.season_number}:${u.episode_number}`; + if (!fields.episodeUnit.has(k)) fields.episodeUnit.set(k, u); + } + } + if (kind === "tv" && rec.library_root_id) + coverage = await coverageOf(d, rec.library_root_id, e.id, lay.videos, units); + } + } } - g.videos.push({ path: l.relative_path, season, episode }); - groups.set(l.root_id, g); - } - if (!groups.size) return fail("scrape_no_files"); - const idRows = await db - .selectFrom("external_identity") - .select(["provider", "external_id"]) - .where("media_entity_id", "=", entityId) - .where("content_unit_id", "is", null) - .where("status", "=", "accepted") - .orderBy("updated_at", "desc") - .execute() - .catch(() => [] as { provider: string; external_id: string }[]); - const ids: IdentityMap = {}; - for (const r of idRows) { - const p = r.provider.toLowerCase() as keyof IdentityMap; - if (["tmdb", "imdb", "tvdb", "douban"].includes(p) && !ids[p]) ids[p] = r.external_id; - } - const bindings = await db - .selectFrom("path_binding") - .select(["library_root_id", "relative_path"]) - .where("media_entity_id", "=", entityId) - .where("binding_kind", "=", "entity") - .where("status", "=", "active") - .execute(); - - const work = nfoWork(e, ids); - const art = imagePathsOf(meta); - const seasonPosters = new Map(); - for (const [n, u] of seasonUnit) seasonPosters.set(n, unitPosterPath(u.metadata)); - const stills = new Map(); - for (const [k, u] of episodeUnit) stills.set(k, unitStillPath(u.metadata)); - - const files: ScrapeFileResult[] = []; - const rootDirs = new Map>(); - const imageCache = new Map>(); - const fetchImage = (p: string) => { - let hit = imageCache.get(p); - if (!hit) { - hit = d.ports.scrapeImage(p).then((r) => r.content); - imageCache.set(p, hit); + // 季 / 集号:已登记的按覆盖单元;其余按路径解析(季目录名兜底,同 MP 对每集重新 MetaInfoPath) + const videos: ScrapeVideo[] = []; + const noEpisode: ScrapeFileResult[] = []; + for (const v of lay.videos) { + if (kind === "movie") { + videos.push({ path: v }); + continue; + } + const cov = coverage.get(v); + if (cov) { + videos.push({ path: v, season: cov.season, episode: cov.episode }); + continue; + } + const p = parseReleasePath(v); + const season = p.season ?? lay.dirSeason ?? seasonOfDirName(baseOf(dirOf(v))) ?? (p.episodes ? 1 : null); + const episode = p.episodes?.start ?? null; + if (episode === null) + noEpisode.push({ path: v, kind: "nfo", role: "episode_nfo", action: "skipped", reason: "scrape_no_episode" }); + videos.push({ path: v, season, episode }); } - return hit; - }; - for (const g of groups.values()) { - if (!g.managed) { - files.push( - ...g.videos.map((v) => ({ - path: v.path, - kind: "nfo" as const, - role: kind === "movie" ? "movie_nfo" : "episode_nfo", - action: "skipped" as const, - reason: "scrape_root_read_only", - })), - ); - continue; + // ── 没有本地字段:按 TMDB 条目直接取 ── + if (!fields) { + if (!match.externalId) return await fail("scrape_no_metadata"); + let detail: Awaited>; + try { + detail = await d.ports.scrapeTmdbDetail(kind, match.externalId); + } catch (err) { + d.ctx.logger.warn({ recordId, err: { message: errMessage(err) } }, "刮削取 TMDB 详情失败"); + return await fail("scrape_tmdb_unavailable"); + } + if (!detail) return await fail("scrape_tmdb_unavailable"); + const t = tmdbEntity(detail); + fields = { entity: t.entity, ids: t.ids, seasonUnit: new Map(), episodeUnit: new Map() }; + match = { ...match, provider: "tmdb", title: detail.title, year: detail.year }; + if (kind === "tv") { + const seasons = [...new Set(videos.map((v) => v.season).filter((s): s is number => typeof s === "number"))]; + for (const n of seasons) { + const sd = await d.ports.scrapeTmdbSeason(match.externalId!, n).catch((err) => { + d.ctx.logger.warn({ recordId, season: n, err: { message: errMessage(err) } }, "刮削取 TMDB 季详情失败"); + return null; + }); + if (!sd) continue; + const u = tmdbSeasonUnits(sd); + fields.seasonUnit.set(n, u.season); + for (const [ep, eu] of u.episodes) fields.episodeUnit.set(`${n}:${ep}`, eu); + } + } } - const health = await checkRoot(g.rootPath, g.sentinel ?? DEFAULT_ROOT_SENTINEL); - if (health.kind !== "online") { - return finish({ - status: "failed", - written: 0, - skipped: 0, - failed: 0, - reason: "library_root_offline", - files, - }); - } - const bound = bindings.find((b) => b.library_root_id === g.rootId)?.relative_path ?? null; - const workDir = bound ?? inferWorkDir(kind, g.videos[0]!.path); + + const { seasonUnit, episodeUnit } = fields; + const work = nfoWork(fields.entity, fields.ids); + const seasonPosters = new Map(); + for (const [n, u] of seasonUnit) seasonPosters.set(n, unitPosterPath(u.metadata)); + const stills = new Map(); + for (const [k, u] of episodeUnit) stills.set(k, unitStillPath(u.metadata)); + const targets = planScrapeTargets({ kind, - workDir, - videos: g.videos, - images: { ...art, seasonPosters, stills }, + workDir: lay.workDir, + videos, + images: { ...imagePathsOf(fields.entity.metadata), seasonPosters, stills }, options: { images: settings.scrapeImages, episodeThumbs: settings.scrapeEpisodeThumbs }, + parts: lay.parts, }); - if (kind === "tv" && !workDir) - files.push({ path: "", kind: "nfo", role: "tvshow_nfo", action: "skipped", reason: "scrape_no_work_dir" }); - const rootAbs = resolve(g.rootPath); - const dirs = rootDirs.get(g.rootId) ?? new Set(); - rootDirs.set(g.rootId, dirs); - - const decide = async (t: ScrapeTarget): Promise<{ abs: string } | ScrapeFileResult> => { - const abs = resolve(rootAbs, t.path); - const base = { path: t.path, kind: t.kind, role: t.role }; - if (!inside(rootAbs, abs)) return { ...base, action: "failed", reason: "path_escape" }; - const st = await lstat(abs).catch(() => null); - if (st && (!st.isFile() || st.isSymbolicLink())) return { ...base, action: "skipped", reason: "scrape_exists" }; - if (st && mode === "missing") return { ...base, action: "skipped", reason: "scrape_exists" }; - return { abs }; + const files: ScrapeFileResult[] = [...noEpisode]; + const imageCache = new Map>(); + const fetchImage = (p: string) => { + let hit = imageCache.get(p); + if (!hit) { + hit = d.ports.scrapeImage(p).then((r) => r.content); + imageCache.set(p, hit); + } + return hit; }; - const write = async (t: ScrapeTarget, abs: string, content: Uint8Array | string): Promise => { + const decide = async (t: ScrapeTarget): Promise<{ abs: string } | ScrapeFileResult> => { + const target = resolve(rootReal, t.path); + const base = { path: t.path, kind: t.kind, role: t.role }; + if (!inside(rootReal, target)) return { ...base, action: "failed", reason: "path_escape" }; + const ex = await lstat(target).catch(() => null); + if (ex && (!ex.isFile() || ex.isSymbolicLink())) return { ...base, action: "skipped", reason: "scrape_exists" }; + if (ex && mode === "missing") return { ...base, action: "skipped", reason: "scrape_exists" }; + return { abs: target }; + }; + const write = async (t: ScrapeTarget, target: string, content: Uint8Array | string): Promise => { const base = { path: t.path, kind: t.kind, role: t.role }; try { - await atomicWrite(abs, content); - dirs.add(dirname(t.path) === "." ? "" : dirname(t.path)); + await atomicWrite(target, content); + dirs.add(dirOf(t.path)); return { ...base, action: "written", reason: null }; } catch (err) { - d.ctx.logger.warn({ entityId, path: t.path, err: { message: errMessage(err) } }, "刮削文件写入失败"); + d.ctx.logger.warn({ recordId, path: t.path, err: { message: errMessage(err) } }, "刮削文件写入失败"); return { ...base, action: "failed", reason: "scrape_write_failed" }; } }; @@ -448,37 +727,45 @@ export async function runScrape( } // 图片:限并发,失败不影响 NFO const imgs = targets.filter((x) => x.kind === "image" && x.imagePath); - const results = await pool(imgs, IMAGE_CONCURRENCY, async (t): Promise => { - if (signal?.aborted) throw new Error("aborted"); - const dec = await decide(t); - if (!("abs" in dec)) return dec; - let bytes: Uint8Array; - try { - bytes = await fetchImage(t.imagePath!); - } catch (err) { - d.ctx.logger.warn({ entityId, image: t.imagePath, err: { message: errMessage(err) } }, "刮削图片下载失败"); - return { path: t.path, kind: t.kind, role: t.role, action: "failed", reason: "scrape_image_failed" }; - } - return write(t, dec.abs, bytes); - }); - files.push(...results); - } + files.push( + ...(await pool(imgs, IMAGE_CONCURRENCY, async (t): Promise => { + if (signal?.aborted) throw new Error("aborted"); + const dec = await decide(t); + if (!("abs" in dec)) return dec; + let bytes: Uint8Array; + try { + bytes = await fetchImage(t.imagePath!); + } catch (err) { + d.ctx.logger.warn({ recordId, image: t.imagePath, err: { message: errMessage(err) } }, "刮削图片下载失败"); + return { path: t.path, kind: t.kind, role: t.role, action: "failed", reason: "scrape_image_failed" }; + } + return write(t, dec.abs, bytes); + })), + ); - const written = files.filter((f) => f.action === "written").length; - const failed = files.filter((f) => f.action === "failed").length; - const skipped = files.filter((f) => f.action === "skipped").length; - const onlyReadOnly = files.length > 0 && files.every((f) => f.reason === "scrape_root_read_only"); - const status: ScrapeRunResult["status"] = onlyReadOnly - ? "failed" - : failed === 0 - ? "succeeded" - : written > 0 || skipped > 0 - ? "partial" - : "failed"; - const reason = onlyReadOnly - ? "scrape_root_read_only" - : failed - ? (files.find((f) => f.action === "failed")?.reason ?? null) - : null; - return finish({ status, written, skipped, failed, reason, files }, rootDirs); + const written = files.filter((f) => f.action === "written").length; + const failed = files.filter((f) => f.action === "failed").length; + const status: ScrapeRecord["status"] = + failed === 0 ? "succeeded" : written > 0 || files.some((f) => f.action === "skipped") ? "partial" : "failed"; + return await finish({ + status, + reason: failed ? (files.find((f) => f.action === "failed")?.reason ?? null) : null, + files, + }); + } catch (err) { + if (signal?.aborted) throw err; + d.ctx.logger.warn({ recordId, err: { message: errMessage(err) } }, "刮削失败"); + return fail("scrape_write_failed"); + } +} + +/** 请求内等刮削结束(最多 waitMs);返回最新记录 */ +export async function waitScrapeRecords(d: AoDeps, ids: readonly string[], waitMs: number): Promise { + const deadline = Date.now() + waitMs; + let recs = await loadScrapeRecords(d, ids); + while (recs.some((r) => r.status === "queued" || r.status === "running") && Date.now() < deadline) { + await new Promise((res) => setTimeout(res, 200)); + recs = await loadScrapeRecords(d, ids); + } + return recs; } diff --git a/apps/server/src/modules/file-manager/identify.ts b/apps/server/src/modules/file-manager/identify.ts index 583fea2a..6c2c5dfc 100644 --- a/apps/server/src/modules/file-manager/identify.ts +++ b/apps/server/src/modules/file-manager/identify.ts @@ -11,7 +11,14 @@ import { type OrganizationStatus, } from "@umt/contracts"; import { sql } from "@umt/db"; -import { type IdentityCandidate, type ParsedRelease, parseReleasePath, queryTitles, rankCandidates } from "@umt/domain"; +import { + type IdentityCandidate, + type ParsedRelease, + parseReleasePath, + queryTitles, + rankCandidates, + resolveIdentity, +} from "@umt/domain"; import { localIdentityCandidate } from "@umt/mapping"; import type { Principal } from "../../http/auth.ts"; import { EXTERNAL_INVOKE_TIMEOUT_MS, sharedInvokeLimiter } from "../agent/invoke-limits.ts"; @@ -177,13 +184,24 @@ export async function runIdentifyRemote( return payload; } -async function identifyCore( +/** 候选键:本地作品 id 或 `provider:externalId`(与 FileIdentifyCandidate 对应) */ +export const candidateKey = (c: { mediaEntityId: string | null; provider: string; externalId: string | null }) => + c.mediaEntityId ?? `${c.provider}:${c.externalId}`; + +export async function identifyCore( d: FmDeps, body: FilesIdentifyRequest, opts: CoreOpts, -): Promise<{ result: FilesIdentifyResult; remotePending: number; pendingPaths: Set }> { +): Promise<{ + result: FilesIdentifyResult; + remotePending: number; + pendingPaths: Set; + /** SCR-2:每个视频按 resolveIdentity 能否自动确认(root\0path → 候选键;不能为 null) */ + autoPicks: Map; +}> { const db = d.ctx.db; const pendingPaths = new Set(); + const autoPicks = new Map(); const { videos, truncated } = await collectVideos(d, body); // G2:最小体积(请求/向导 > settings.organize.minSizeBytes,缺省 50 MiB;设置读不到时不过滤) const orgSettings = (await d.ctx.settings.get("organize").catch(() => null)) as { minSizeBytes?: unknown } | null; @@ -411,6 +429,14 @@ async function identifyCore( ), }; }); + const res = resolveIdentity(parsed, all); + const pick = res.status === "auto" && res.best ? res.best.candidate : null; + autoPicks.set( + `${v.root}\u0000${v.rel}`, + pick && (!kindHint || (kindHint === "movie") === (pick.entityType === "movie")) + ? (pick.localEntityId ?? `${pick.provider}:${pick.externalId}`) + : null, + ); const best = candidates[0]; let contentUnitId: string | null = null; let contentUnit: string | null = null; @@ -454,7 +480,7 @@ async function identifyCore( } if (videos.length === 0 && body.items.length > 0 && !truncated) throw new FilesError("bad_request", "所选项中没有视频文件"); - return { result: { items, truncated, excludedBySize }, remotePending: remoteQueries, pendingPaths }; + return { result: { items, truncated, excludedBySize }, remotePending: remoteQueries, pendingPaths, autoPicks }; } /** TMDB 图片 CDN 缩略图(w154);已是完整 URL 的原样返回 */ diff --git a/apps/server/src/modules/file-manager/probe.ts b/apps/server/src/modules/file-manager/probe.ts index 5b7e3613..b9e04bce 100644 --- a/apps/server/src/modules/file-manager/probe.ts +++ b/apps/server/src/modules/file-manager/probe.ts @@ -8,21 +8,17 @@ import type { FilesProbeResult, FilesScrapeBatchRequest, FilesScrapeBatchResult, - FilesScrapeRequest, - FilesScrapeResult, } from "@umt/contracts"; import { sql } from "@umt/db"; import type { Principal } from "../../http/auth.ts"; -import { enqueueScrape } from "../acquisition-organize/services/scrape.ts"; import { FilesError, type FmDeps, jsonb, resolveIn } from "./common.ts"; import { audit } from "./ledger.ts"; import { kindOfName, mediaSummary } from "./list.ts"; import { entitiesUnder } from "./records.ts"; // probe:ffprobe 摘要;已登记资产写回 media_asset.media_info.probe,未登记只缓存 24h(DATA_DIR/cache/probe)。 -// scrape:只对已登记作品,等价 POST /media-entities/{id}/refresh(入队 refreshMetadata,字段锁优先、不动路径); -// writeNfo / images(SCR-1)另排 scrapeMedia,把 NFO 与图片写进媒体库目录(与作品详情「重新刮削」同一任务)。 -// scrape-batch write=true:文件管理「刮削」,按作品去重各排一次 scrapeMedia(不刷新元数据)。 +// scrape-batch:文件管理「刷新元数据」,按作品去重各排一次 refreshMetadata(字段锁优先、不动路径、不写文件)。 +// 刮削(写 NFO 与图片)见 scrape.ts。 const CACHE_TTL_MS = 24 * 3600_000; @@ -104,62 +100,8 @@ export async function probeFiles(d: FmDeps, body: FilesProbeRequest): Promise { - const db = d.ctx.db; - const e = await db - .selectFrom("media_entity") - .select(["id", "status", "field_locks"]) - .where("id", "=", body.mediaEntityId) - .executeTakeFirst(); - if (!e) throw new FilesError("not_found", "作品不存在"); - const registered = await db - .selectFrom("media_asset") - .select(["id"]) - .where("media_entity_id", "=", e.id) - .where("status", "=", "active") - .executeTakeFirst(); - if (!registered) throw new FilesError("bad_request", "作品没有已入库文件。先识别并整理入库，再刮削"); - const { runId } = await d.ctx.engine.enqueue( - "refreshMetadata", - { mediaEntityId: e.id }, - { idempotencyKey: `refreshMetadata:files:${body.idempotencyKey}`, traceId: randomUUID() }, - ); - const overrides = await db - .selectFrom("field_override") - .select(["field"]) - .where("media_entity_id", "=", e.id) - .where("locked", "=", true) - .execute(); - const lockedFields = [...new Set([...(e.field_locks ?? []), ...overrides.map((o) => o.field)])].sort(); - const scrapeTaskId = - body.writeNfo || body.images - ? ( - await enqueueScrape(d.ao, { - mediaEntityId: e.id, - trigger: "manual", - actor: { type: principal?.kind === "api_key" ? "api_key" : "user", id: principal?.id ?? null }, - }) - ).taskId - : null; - const result: FilesScrapeResult = { - mediaEntityId: e.id, - taskId: runId, - lockedFields, - nfo: body.writeNfo ? "queued" : "not_requested", - images: body.images ? "queued" : "not_requested", - scrapeTaskId, - }; - await audit(d, principal, "scrape", { ...result }); - return result; -} - /** * G5 批量刷新元数据:文件/目录归到其下已登记资产的作品,与显式作品合并去重,逐个入队 refreshMetadata。 - * 只对有 active 资产的作品提交;其余进 skipped(前端汇总"已提交 n 个作品,锁定字段不变")。 */ export async function scrapeBatch( d: FmDeps, @@ -183,18 +125,6 @@ export async function scrapeBatch( const ents = list.length ? await db.selectFrom("media_entity").select(["id", "title", "field_locks"]).where("id", "in", list).execute() : []; - const active = new Set( - list.length - ? ( - await db - .selectFrom("media_asset") - .select(["media_entity_id"]) - .where("media_entity_id", "in", list) - .where("status", "=", "active") - .execute() - ).map((a) => a.media_entity_id) - : [], - ); const locks = list.length ? await db .selectFrom("field_override") @@ -210,34 +140,16 @@ export async function scrapeBatch( skipped.push({ ref: id, reason: "not_found" }); continue; } - if (!active.has(id)) { - skipped.push({ ref: id, reason: "not_registered" }); - continue; - } - const runId = body.write - ? ( - await enqueueScrape(d.ao, { - mediaEntityId: id, - trigger: "manual", - actor: { type: principal?.kind === "api_key" ? "api_key" : "user", id: principal?.id ?? null }, - }) - ).taskId - : ( - await d.ctx.engine.enqueue( - "refreshMetadata", - { mediaEntityId: id }, - { idempotencyKey: `refreshMetadata:files-batch:${body.idempotencyKey}:${id}`, traceId: randomUUID() }, - ) - ).runId; + const { runId } = await d.ctx.engine.enqueue( + "refreshMetadata", + { mediaEntityId: id }, + { idempotencyKey: `refreshMetadata:files-batch:${body.idempotencyKey}:${id}`, traceId: randomUUID() }, + ); const lockedFields = [ ...new Set([...(e.field_locks ?? []), ...locks.filter((l) => l.media_entity_id === id).map((l) => l.field)]), ].sort(); submitted.push({ mediaEntityId: id, title: e.title, taskId: runId, lockedFields }); } - await audit(d, principal, "scrape_batch", { - write: body.write, - submitted: submitted.map((s) => s.mediaEntityId), - skipped, - }); + await audit(d, principal, "scrape_batch", { submitted: submitted.map((s) => s.mediaEntityId), skipped }); return { submitted, skipped }; } diff --git a/apps/server/src/modules/file-manager/routes.ts b/apps/server/src/modules/file-manager/routes.ts index de89ccfe..8d156b19 100644 --- a/apps/server/src/modules/file-manager/routes.ts +++ b/apps/server/src/modules/file-manager/routes.ts @@ -15,8 +15,9 @@ import { listDir, listRoots, statEntry } from "./list.ts"; import { mkdirIn, renameEntry } from "./mutate.ts"; import { organizeFiles } from "./organize.ts"; import { previewFile } from "./preview.ts"; -import { probeFiles, scrape, scrapeBatch } from "./probe.ts"; +import { probeFiles, scrapeBatch } from "./probe.ts"; import { findRecords } from "./records.ts"; +import { scrapeFiles, scrapeRecords } from "./scrape.ts"; import { suggestName } from "./suggest-name.ts"; import { transfer } from "./transfer.ts"; import { deleteFiles, listTrash, purgeTrash, restoreTrash } from "./trash.ts"; @@ -110,7 +111,11 @@ export function registerFileRoutes(app: ModuleApp, d: FmDeps) { mount(app, F.identify, async (c, { body }) => identify(d, c.get("principal"), body, { wait: /^(1|true)$/i.test(c.req.query("wait") ?? "") }), ); - mount(app, F.scrape, async (c, { body }) => idem(d, c, "scrape", body, () => scrape(d, c.get("principal"), body))); + // SCR-2:任意目录 / 视频文件就地刮削(识别 → 写 NFO 与图片);认不准的项返回候选 + mount(app, F.scrape, async (c, { body }) => + idem(d, c, "scrape", body, () => scrapeFiles(d, c.get("principal"), body)), + ); + mount(app, F.scrapeRecords, async (_c, { query }) => scrapeRecords(d, query.ids)); mount(app, F.probe, async (_c, { body }) => probeFiles(d, body)); mount(app, F.records, async (_c, { query }) => findRecords(d, query)); mount(app, F.download, async (c, { query }) => downloadFile(d, c.get("principal"), query), true); diff --git a/apps/server/src/modules/file-manager/scrape.ts b/apps/server/src/modules/file-manager/scrape.ts new file mode 100644 index 00000000..80ca1b90 --- /dev/null +++ b/apps/server/src/modules/file-manager/scrape.ts @@ -0,0 +1,211 @@ +import type { + FileIdentifyCandidate, + FilesScrapeItem, + FilesScrapeNeedsChoice, + FilesScrapeRequest, + FilesScrapeResult, + ScrapeMatch, +} from "@umt/contracts"; +import type { Principal } from "../../http/auth.ts"; +import { + entityScrapeMatch, + loadScrapeRecords, + queueScrape, + scrapeSettings, + waitScrapeRecords, +} from "../acquisition-organize/services/scrape.ts"; +import { actorOfP, FilesError, type FmDeps, resolveIn } from "./common.ts"; +import { candidateKey, identifyCore } from "./identify.ts"; +import { audit } from "./ledger.ts"; +import { kindOfName } from "./list.ts"; +import { entitiesUnder } from "./records.ts"; + +// SCR-2 文件管理「刮削」(照 MP 文件管理的刮削):任意目录 / 视频文件 → 识别作品 → 就地写 NFO 与图片。 +// 识别顺序:请求指定(选定的候选)> 目录里已登记文件所属的唯一作品 > 按路径识别(本地召回 + TMDB,resolveIdentity +// 能自动确认且各视频一致时自动用)。认不准的项进 needsChoice(前 5 候选),选定后带 mediaEntityId / external 再提交。 +// 未登记也能刮削(直接用 TMDB 字段);addToLibrary 时顺手把 TMDB 条目加入媒体库。 +// 每项一条刮削记录,排 scrapeMedia;请求内最多等 transferWaitMs,没刮完的 pending=true,前端按 id 轮询。 + +const RECOGNIZE_TIMEOUT_MS = 15_000; + +type Actor = { type: string; id: string | null }; +const actorOf = (p: Principal | undefined): Actor => ({ + type: p?.kind === "api_key" ? "api_key" : "user", + id: p?.id ?? null, +}); + +/** 路径识别:各视频都能自动确认且是同一作品时返回它;否则返回候选(按出现次数与等级合并,前 5) */ +async function recognize( + d: FmDeps, + principal: Principal | undefined, + it: FilesScrapeItem, +): Promise<{ pick: FileIdentifyCandidate | null; choice: Omit }> { + const r = await identifyCore( + d, + { items: [{ root: it.root, path: it.path }], minSizeBytes: 0 }, + { mode: "remote", actorId: actorOfP(principal), signal: AbortSignal.timeout(RECOGNIZE_TIMEOUT_MS) }, + ); + const items = r.result.items; + const picks = items.map((i) => r.autoPicks.get(`${i.root}\u0000${i.path}`) ?? null); + const agreed = picks.length > 0 && picks.every((p) => p && p === picks[0]) ? picks[0] : null; + const LV = { high: 3, medium: 2, low: 1 } as const; + const merged = new Map(); + for (const i of items) + for (const c of i.candidates) { + const k = candidateKey(c); + const cur = merged.get(k); + if (!cur) merged.set(k, { c, n: 1 }); + else { + cur.n += 1; + if (LV[c.level] > LV[cur.c.level]) cur.c = { ...c, posterUrl: c.posterUrl ?? cur.c.posterUrl }; + } + } + const ranked = [...merged.values()].sort((a, b) => b.n - a.n || LV[b.c.level] - LV[a.c.level]).map((x) => x.c); + const pick = agreed ? (ranked.find((c) => candidateKey(c) === agreed) ?? null) : null; + const first = items[0]?.parsed; + return { + pick, + choice: { + title: first?.title ?? null, + candidates: ranked.slice(0, 5), + warnings: [...new Set(items.flatMap((i) => i.warnings))].slice(0, 5), + }, + }; +} + +/** TMDB 条目 → 识别结果:已有本地作品(已接受的 TMDB 映射)就用它;addToLibrary 时建作品 */ +async function externalMatch( + d: FmDeps, + ext: { externalId: string; kind: "movie" | "tv"; title?: string; year?: number | null }, + source: ScrapeMatch["source"], + addToLibrary: boolean, + actor: Actor, + entities: FilesScrapeResult["entities"], +): Promise { + const local = await d.ctx.db + .selectFrom("external_identity") + .select(["media_entity_id"]) + .where("provider", "=", "tmdb") + .where("entity_type", "=", ext.kind) + .where("external_id", "=", ext.externalId) + .where("status", "=", "accepted") + .where("content_unit_id", "is", null) + .executeTakeFirst(); + let entityId = local?.media_entity_id ?? null; + if (!entityId && addToLibrary) { + entityId = await d.ao.ports.entityFromTmdb({ type: ext.kind, id: ext.externalId }, actor); + const e = await d.ctx.db.selectFrom("media_entity").select(["title"]).where("id", "=", entityId).executeTakeFirst(); + entities.push({ mediaEntityId: entityId, title: e?.title ?? ext.title ?? ext.externalId, created: true }); + } + if (entityId) { + const m = await entityScrapeMatch(d.ao, entityId, source); + if (m) return { ...m, provider: "tmdb", externalId: ext.externalId }; + } + return { + mediaEntityId: null, + provider: "tmdb", + externalId: ext.externalId, + kind: ext.kind, + title: ext.title ?? `TMDB ${ext.externalId}`, + year: ext.year ?? null, + source, + }; +} + +export async function scrapeFiles( + d: FmDeps, + principal: Principal | undefined, + body: FilesScrapeRequest, +): Promise { + const db = d.ctx.db; + const actor = actorOf(principal); + const mode = body.mode ?? (await scrapeSettings(d.ao)).scrapeMode; + const recordIds: string[] = []; + const needsChoice: FilesScrapeResult["needsChoice"] = []; + const skipped: FilesScrapeResult["skipped"] = []; + const entities: FilesScrapeResult["entities"] = []; + for (const it of body.items) { + if (it.mediaEntityId && it.external) throw new FilesError("bad_request", "mediaEntityId 与 external 只能给一个"); + const r = await resolveIn(d, it.root, it.path); + const ref = { root: it.root, path: r.rel }; + if (r.def.kind === "library" && !r.def.modifiable) { + skipped.push({ ...ref, reason: "scrape_root_read_only" }); + continue; + } + const isDir = !!r.lst?.isDirectory(); + if (!isDir && !(r.lst?.isFile() && kindOfName(r.abs) === "video")) { + skipped.push({ ...ref, reason: "scrape_not_video" }); + continue; + } + let match: ScrapeMatch | null = null; + if (it.mediaEntityId) { + match = await entityScrapeMatch(d.ao, it.mediaEntityId, "chosen"); + if (!match) throw new FilesError("not_found", `作品不存在：${it.mediaEntityId}`); + } else if (it.external) { + match = await externalMatch(d, it.external, "chosen", body.addToLibrary, actor, entities); + } else { + // 目录里的文件已登记到唯一作品:直接用(重新刮削已入库的) + const owners = r.def.libraryRootId ? await entitiesUnder(db, r.def.libraryRootId, r.rel, isDir) : []; + if (owners.length === 1) match = await entityScrapeMatch(d.ao, owners[0]!, "library"); + if (!match) { + let rec: Awaited>; + try { + rec = await recognize(d, principal, { ...it, path: r.rel }); + } catch (e) { + if (e instanceof FilesError && e.reason === "bad_request") { + skipped.push({ ...ref, reason: "scrape_no_videos" }); + continue; + } + throw e; + } + const c = rec.pick; + if (c?.mediaEntityId) match = await entityScrapeMatch(d.ao, c.mediaEntityId, "auto"); + else if (c && c.provider === "tmdb" && c.externalId) + match = await externalMatch( + d, + { externalId: c.externalId, kind: c.kind === "movie" ? "movie" : "tv", title: c.title, year: c.year }, + "auto", + body.addToLibrary, + actor, + entities, + ); + if (!match) { + needsChoice.push({ ...ref, ...rec.choice }); + continue; + } + } + } + const q = await queueScrape(d.ao, { + rootId: r.def.id, + rootPath: r.def.hostPath, + libraryRootId: r.def.libraryRootId, + path: r.rel, + match, + mode, + trigger: "manual", + actor, + }); + recordIds.push(q.recordId); + } + const records = recordIds.length ? await waitScrapeRecords(d.ao, recordIds, d.options.transferWaitMs) : []; + await audit(d, principal, "scrape", { + items: body.items.map((i) => `${i.root}:${i.path}`).slice(0, 100), + mode, + recordIds, + needsChoice: needsChoice.map((n) => `${n.root}:${n.path}`), + skipped, + }); + return { + records, + needsChoice, + skipped, + pending: records.some((r) => r.status === "queued" || r.status === "running"), + entities, + }; +} + +export async function scrapeRecords(d: FmDeps, idsParam: string) { + const ids = [...new Set(idsParam.split(",").map((s) => s.trim()))].filter((s) => /^[0-9a-f-]{36}$/i.test(s)); + if (!ids.length || ids.length > 100) throw new FilesError("bad_request", "ids 需要 1–100 个刮削记录 id"); + return { records: await loadScrapeRecords(d.ao, ids) }; +} diff --git a/apps/server/test/file-manager/harness.ts b/apps/server/test/file-manager/harness.ts index 4a811a26..f1009226 100644 --- a/apps/server/test/file-manager/harness.ts +++ b/apps/server/test/file-manager/harness.ts @@ -22,6 +22,7 @@ import { EventBus } from "../../src/http/sse.ts"; import { createLogger } from "../../src/logger.ts"; import { fakeEngine } from "../../src/modules/acquisition-organize/__tests__/harness.ts"; import { executeRecord, type FrozenPlan } from "../../src/modules/acquisition-organize/services/organize.ts"; +import { runScrapeRecord } from "../../src/modules/acquisition-organize/services/scrape.ts"; import { createCapabilityRegistry } from "../../src/modules/agent/capability-registry.ts"; import { createFileManagerModule, type FmOverrides } from "../../src/modules/file-manager/index.ts"; import type { ServerContext } from "../../src/modules/types.ts"; @@ -57,6 +58,17 @@ export async function pgAvailable() { // biome-ignore lint/suspicious/noExplicitAny: 测试断言直接读响应 JSON type Loose = any; +/** catalog.lookup 远端候选(测试用) */ +export interface RemoteHit { + provider: string; + entityType: "movie" | "tv"; + externalId: string; + title: string; + originalTitle?: string | null; + year: number | null; + posterPath?: string | null; +} + export interface FmHarness { db: Kysely; engine: ReturnType; @@ -80,7 +92,11 @@ export async function createFmHarness( name: string, overrides: FmOverrides = {}, /** runTransfers(默认 true):假引擎收到 /files 移动/复制的 verifyAndImport 时就地执行(模拟 worker) */ - opts: { catalog?: boolean; runTransfers?: boolean } = {}, + opts: { + /** true = 内置假 catalog.lookup("Remote Film");函数 = 自定义远端候选(按查询词返回) */ + catalog?: boolean | ((query: string) => RemoteHit[]); + runTransfers?: boolean; + } = {}, ): Promise { const schema = `fm_test_${name}_${process.pid.toString(36)}${Math.random().toString(36).slice(2, 7)}`; const admin = new pg.Client({ connectionString: DB_URL }); @@ -124,9 +140,12 @@ export async function createFmHarness( if (opts.catalog) capabilities.register(capabilitySpec("catalog.lookup"), async (input) => ({ local: [], - remote: String(input.query).includes("Remote") - ? [{ provider: "tmdb", entityType: "movie" as const, externalId: "999", title: "Remote Film", year: 2019 }] - : [], + remote: + typeof opts.catalog === "function" + ? opts.catalog(String(input.query)) + : String(input.query).includes("Remote") + ? [{ provider: "tmdb", entityType: "movie" as const, externalId: "999", title: "Remote Film", year: 2019 }] + : [], })); const ctx: ServerContext = { role: "all", @@ -173,6 +192,9 @@ export async function createFmHarness( engine.enqueue = async (task, input, o) => { const r = await enqueue(task, input, o); const recordId = (input as { recordId?: string }).recordId; + // SCR-2 刮削:模拟 worker 就地执行 scrapeMedia + if (task === "scrapeMedia" && recordId) + setTimeout(() => void runScrapeRecord(module.deps().ao, recordId).catch(() => {}), 0); if (task === "verifyAndImport" && recordId) setTimeout(async () => { try { diff --git a/apps/server/test/file-manager/identify-organize.test.ts b/apps/server/test/file-manager/identify-organize.test.ts index 07ce5166..a8fdc540 100644 --- a/apps/server/test/file-manager/identify-organize.test.ts +++ b/apps/server/test/file-manager/identify-organize.test.ts @@ -14,7 +14,7 @@ import { } from "./harness.ts"; // identify(只读,本地 + 远端召回前 5,不落库)、organize(按作品分组走 planImport,dryRun→execute 一致与源变更)、 -// probe(已登记写回 media_info)、scrape(仅已登记作品,入队 refreshMetadata;NFO 待 W5)。 +// probe(已登记写回 media_info)、刷新元数据(scrape-batch,入队 refreshMetadata)。 const skip = !(await pgAvailable()); @@ -278,7 +278,7 @@ describe.skipIf(skip)("files 识别 / 整理 / 探测 / 刮削", () => { expect(r.status).toBeLessThan(500); }); - it("probe:已登记资产写回 media_info;scrape 仅已登记作品", async () => { + it("probe:已登记资产写回 media_info;刷新元数据按作品入队", async () => { await mkdir(join(h.lib, "Blue Film probe")); await writeFile(join(h.lib, "Blue Film probe", "Blue.Film.2021.mkv"), "x"); const a = await seedAsset(h.db, h, { entityId: film, rel: "Blue Film probe/Blue.Film.2021.mkv" }); @@ -303,21 +303,10 @@ describe.skipIf(skip)("files 识别 / 整理 / 探测 / 刮削", () => { const u2 = await h.api("POST", "/files/probe", { items: [{ root: h.dlRoot, path: "Remote.Film.2019.1080p.mkv" }] }); expect(u2.body.data.items[0].cached).toBe(true); - const s = await h.api("POST", "/files/scrape", { idempotencyKey: key(), mediaEntityId: film, writeNfo: true }); - expect(s.status).toBe(200); - // SCR-1:writeNfo 另排 scrapeMedia(写 NFO 与图片到媒体库目录) - expect(s.body.data).toMatchObject({ mediaEntityId: film, nfo: "queued", images: "not_requested" }); - expect(s.body.data.scrapeTaskId).toBeTruthy(); - expect(h.engine.enqueued.some((e) => e.task === "refreshMetadata")).toBe(true); - expect(h.engine.enqueued.some((e) => e.task === "scrapeMedia")).toBe(true); - const b = await h.api("POST", "/files/scrape-batch", { - idempotencyKey: key(), - mediaEntityIds: [film], - write: true, - }); + // 刷新元数据(scrape-batch):按作品入队 refreshMetadata;刮削(写 NFO)见 scrape.test.ts + const b = await h.api("POST", "/files/scrape-batch", { idempotencyKey: key(), mediaEntityIds: [film] }); expect(b.status).toBe(200); expect(b.body.data.submitted).toHaveLength(1); - const none = await h.api("POST", "/files/scrape", { idempotencyKey: key(), mediaEntityId: show }); - expect(none.status).toBe(400); + expect(h.engine.enqueued.some((e) => e.task === "refreshMetadata")).toBe(true); }); }); diff --git a/apps/server/test/file-manager/scrape.test.ts b/apps/server/test/file-manager/scrape.test.ts new file mode 100644 index 00000000..4e2a1c62 --- /dev/null +++ b/apps/server/test/file-manager/scrape.test.ts @@ -0,0 +1,443 @@ +import { mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises"; +import { join, relative } from "node:path"; +import type { TmdbRichDetail, TmdbRichSeasonDetail } from "@umt/adapters"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { latestEntityScrape, scrapeEntity } from "../../src/modules/acquisition-organize/services/scrape.ts"; +import { + createFmHarness, + EXEC_KEY, + type FmHarness, + key, + pgAvailable, + type RemoteHit, + seedAsset, + seedEntity, +} from "./harness.ts"; + +// SCR-2 刮削照 MP:文件管理任意目录 / 视频文件 → 按路径识别作品(未登记走 TMDB,认不准返回候选)→ 就地写 NFO 与图片。 +// 场景:未登记的《人生切割术》S01 两集 + 《星际穿越》目录一次刮削;歧义候选(Solaris 1972 / 2002)选定后刮削并加入媒体库; +// 只补缺失 / 全部覆盖;季目录与单集文件的范围;已登记目录直接用作品;只登记目录与非视频文件跳过。 + +const skip = !(await pgAvailable()); +const JPEG = new Uint8Array([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10, 0x4a, 0x46, 0x49, 0x46, 0x00]); + +function detail( + p: Partial & Pick, +): TmdbRichDetail { + return { + titleLanguage: "zh-CN", + titleZh: p.title, + titleZhLanguage: "zh-CN", + originalTitle: null, + originalLanguage: "en", + overview: null, + overviewLanguage: null, + tagline: null, + year: null, + releaseDate: null, + lastAirDate: null, + genres: [], + voteAverage: null, + voteCount: null, + status: null, + runtime: null, + seasonCount: null, + episodeCount: null, + networks: [], + originCountry: [], + posterPath: null, + backdropPath: null, + cast: [], + crew: [], + externalIds: { imdb: null, tvdb: null, wikidata: null }, + translations: [], + alternativeTitles: [], + seasons: [], + languages: ["zh-CN"], + ...p, + }; +} + +const DETAILS: Record = { + "tv:95396": detail({ + entityType: "tv", + externalId: "95396", + title: "人生切割术", + originalTitle: "Severance", + year: 2022, + releaseDate: "2022-02-17", + overview: "马克领导着一个办公室团队。", + overviewLanguage: "zh-CN", + genres: ["剧情", "悬疑"], + voteAverage: 8.4, + networks: ["Apple TV+"], + originCountry: ["US"], + posterPath: "/sev.jpg", + backdropPath: "/sev-bg.jpg", + externalIds: { imdb: "tt11280740", tvdb: "371980", wikidata: null }, + }), + "movie:157336": detail({ + entityType: "movie", + externalId: "157336", + title: "星际穿越", + originalTitle: "Interstellar", + year: 2014, + releaseDate: "2014-11-05", + runtime: 169, + posterPath: "/inter.jpg", + backdropPath: "/inter-bg.jpg", + crew: [{ id: 525, name: "克里斯托弗·诺兰", originalName: null, role: "Director", profilePath: null }], + }), + "movie:593": detail({ + entityType: "movie", + externalId: "593", + title: "飞向太空", + originalTitle: "Solaris", + year: 1972, + }), + "movie:2103": detail({ + entityType: "movie", + externalId: "2103", + title: "索拉里斯星", + originalTitle: "Solaris", + year: 2002, + posterPath: "/solaris.jpg", + }), +}; + +const SEASON1: TmdbRichSeasonDetail = { + tvId: "95396", + seasonNumber: 1, + name: "第 1 季", + nameLanguage: "zh-CN", + overview: null, + overviewLanguage: null, + airDate: "2022-02-17", + posterPath: "/s1.jpg", + fetchedLanguages: ["zh-CN"], + episodes: [ + [1, "有关地狱的好消息", "/e1.jpg"], + [2, "半循环", "/e2.jpg"], + ].map(([n, name, still]) => ({ + seasonNumber: 1, + episodeNumber: n as number, + name: name as string, + nameLanguage: "zh-CN", + overview: `第 ${n} 集简介`, + overviewLanguage: "zh-CN", + airDate: "2022-02-17", + runtime: 55, + stillPath: still as string, + voteAverage: 8, + voteCount: 10, + })), +}; + +const REMOTE: Record = { + 人生切割术: [ + { + provider: "tmdb", + entityType: "tv", + externalId: "95396", + title: "人生切割术", + originalTitle: "Severance", + year: 2022, + }, + ], + 星际穿越: [ + { + provider: "tmdb", + entityType: "movie", + externalId: "157336", + title: "星际穿越", + originalTitle: "Interstellar", + year: 2014, + }, + ], + Solaris: [ + { + provider: "tmdb", + entityType: "movie", + externalId: "593", + title: "飞向太空", + originalTitle: "Solaris", + year: 1972, + }, + { + provider: "tmdb", + entityType: "movie", + externalId: "2103", + title: "索拉里斯星", + originalTitle: "Solaris", + year: 2002, + }, + ], +}; + +async function tree(root: string, under = ""): Promise { + const out: string[] = []; + const walk = async (d: string) => { + for (const n of (await readdir(d)).sort()) { + if (n.startsWith(".")) continue; + const p = join(d, n); + if ((await stat(p)).isDirectory()) await walk(p); + else out.push(relative(root, p)); + } + }; + await walk(join(root, under)); + return out.sort(); +} + +const SHOW = "人生切割术 (2022)"; +const EP1 = `${SHOW}/Season 1/人生切割术 - S01E01 - 有关地狱的好消息.mkv`; +const EP2 = `${SHOW}/Season 1/人生切割术 - S01E02 - 半循环.mkv`; +const MOVIE = "星际穿越 (2014)"; +const MOVIE_FILE = `${MOVIE}/星际穿越 (2014) - 1080p.mkv`; + +describe.skipIf(skip)("files 刮削(SCR-2:任意目录就地刮削)", () => { + let h: FmHarness; + const images: string[] = []; + const tmdbCalls: string[] = []; + + beforeAll(async () => { + h = await createFmHarness( + "scrape2", + { + ao: { + async scrapeImage(p) { + images.push(p); + return { content: JPEG, mime: "image/jpeg" }; + }, + async scrapeTmdbDetail(type, id) { + tmdbCalls.push(`${type}:${id}`); + const d = DETAILS[`${type}:${id}`]; + if (!d) throw new Error("TMDB 404"); + return d; + }, + async scrapeTmdbSeason(tvId, n) { + tmdbCalls.push(`season:${tvId}:${n}`); + return tvId === "95396" && n === 1 ? SEASON1 : null; + }, + async entityFromTmdb(ref) { + const d = DETAILS[`${ref.type}:${ref.id}`]!; + const e = await h.db + .insertInto("media_entity") + .values({ + kind: ref.type === "movie" ? "movie" : "series", + title: d.title, + year: d.year, + status: "active", + }) + .returning("id") + .executeTakeFirstOrThrow(); + await h.db + .insertInto("external_identity") + .values({ + media_entity_id: e.id, + provider: "tmdb", + entity_type: ref.type, + external_id: ref.id, + status: "accepted", + }) + .execute(); + return e.id; + }, + }, + }, + { + catalog: (q) => + Object.entries(REMOTE) + .filter(([k]) => q.includes(k) || k.includes(q)) + .flatMap(([, v]) => v), + }, + ); + await mkdir(join(h.lib, SHOW, "Season 1"), { recursive: true }); + await mkdir(join(h.lib, MOVIE)); + await mkdir(join(h.lib, "Solaris")); + for (const f of [EP1, EP2, MOVIE_FILE, "Solaris/Solaris.1080p.BluRay.x264-GRP.mkv"]) + await writeFile(join(h.lib, f), "video-bytes"); + await writeFile(join(h.lib, MOVIE, "readme.txt"), "x"); + }); + afterAll(async () => h?.close()); + + const scrape = (items: unknown[], extra: Record = {}, k = EXEC_KEY) => + h.api("POST", "/files/scrape", { idempotencyKey: key(), items, ...extra }, k); + + it("未登记的剧集目录 + 电影目录:按路径识别 TMDB,就地写 NFO 与图片,不建作品", async () => { + const r = await scrape([ + { root: h.libRoot, path: SHOW }, + { root: h.libRoot, path: MOVIE }, + ]); + expect(r.status).toBe(200); + const data = r.body.data; + expect(data.needsChoice).toEqual([]); + expect(data.pending).toBe(false); + expect(data.records).toHaveLength(2); + const [show, movie] = data.records; + expect(show).toMatchObject({ + status: "succeeded", + path: SHOW, + mode: "missing", + trigger: "manual", + match: { + mediaEntityId: null, + provider: "tmdb", + externalId: "95396", + kind: "tv", + source: "auto", + title: "人生切割术", + }, + }); + expect(movie).toMatchObject({ status: "succeeded", match: { externalId: "157336", kind: "movie" } }); + + const files = await tree(h.lib); + expect(files).toEqual( + [ + `${SHOW}/Season 1/poster.jpg`, + `${SHOW}/Season 1/season.nfo`, + EP1, + EP1.replace(/\.mkv$/, "-thumb.jpg"), + EP1.replace(/\.mkv$/, ".nfo"), + EP2, + EP2.replace(/\.mkv$/, "-thumb.jpg"), + EP2.replace(/\.mkv$/, ".nfo"), + `${SHOW}/fanart.jpg`, + `${SHOW}/poster.jpg`, + `${SHOW}/season01-poster.jpg`, + `${SHOW}/tvshow.nfo`, + `${MOVIE}/fanart.jpg`, + `${MOVIE}/poster.jpg`, + `${MOVIE}/readme.txt`, + MOVIE_FILE, + MOVIE_FILE.replace(/\.mkv$/, ".nfo"), + "Solaris/Solaris.1080p.BluRay.x264-GRP.mkv", + ].sort(), + ); + // 未登记:不建作品,字段直接取 TMDB + expect(await h.db.selectFrom("media_entity").select("id").execute()).toHaveLength(0); + const tvshow = await readFile(join(h.lib, SHOW, "tvshow.nfo"), "utf8"); + expect(tvshow).toContain("人生切割术"); + expect(tvshow).toContain('tt11280740'); + const ep = await readFile(join(h.lib, EP1.replace(/\.mkv$/, ".nfo")), "utf8"); + expect(ep).toContain("有关地狱的好消息"); + expect(ep).toMatch(/1<\/season>\n {2}1<\/episode>/); + const nfo = await readFile(join(h.lib, MOVIE_FILE.replace(/\.mkv$/, ".nfo")), "utf8"); + expect(nfo).toContain('克里斯托弗·诺兰'); + expect(tmdbCalls).toEqual(expect.arrayContaining(["tv:95396", "season:95396:1", "movie:157336"])); + // 记录可按 id 回查;审计按记录 + const back = await h.api("GET", `/files/scrape-records?ids=${show.id},${movie.id}`); + expect(back.body.data.records.map((x: { id: string }) => x.id)).toEqual([show.id, movie.id]); + expect(back.body.data.records[0].files.filter((f: { action: string }) => f.action === "written")).toHaveLength(10); + expect(h.events.some((e) => e.type === "files.changed" && e.payload.rootId === h.libRoot)).toBe(true); + }); + + it("歧义:返回候选;选定 TMDB 条目后刮削,并按需加入媒体库", async () => { + const first = await scrape([{ root: h.libRoot, path: "Solaris" }]); + expect(first.status).toBe(200); + expect(first.body.data.records).toEqual([]); + expect(first.body.data.needsChoice).toHaveLength(1); + const nc = first.body.data.needsChoice[0]; + expect(nc).toMatchObject({ root: h.libRoot, path: "Solaris", title: "Solaris" }); + expect(nc.candidates.map((c: { externalId: string }) => c.externalId).sort()).toEqual(["2103", "593"]); + expect(await tree(h.lib, "Solaris")).toEqual(["Solaris/Solaris.1080p.BluRay.x264-GRP.mkv"]); + + const second = await scrape( + [{ root: h.libRoot, path: "Solaris", external: { provider: "tmdb", externalId: "2103", kind: "movie" } }], + { addToLibrary: true }, + ); + expect(second.status).toBe(200); + const rec = second.body.data.records[0]; + expect(rec).toMatchObject({ + status: "succeeded", + match: { provider: "tmdb", externalId: "2103", source: "chosen", title: "索拉里斯星", year: 2002 }, + }); + expect(second.body.data.entities).toMatchObject([{ title: "索拉里斯星", created: true }]); + expect(rec.match.mediaEntityId).toBe(second.body.data.entities[0].mediaEntityId); + expect(await tree(h.lib, "Solaris")).toEqual([ + "Solaris/Solaris.1080p.BluRay.x264-GRP.mkv", + "Solaris/Solaris.1080p.BluRay.x264-GRP.nfo", + "Solaris/poster.jpg", + ]); + expect(await readFile(join(h.lib, "Solaris/Solaris.1080p.BluRay.x264-GRP.nfo"), "utf8")).toContain( + "索拉里斯星", + ); + // 作品详情取的是该作品最近一条刮削记录 + expect((await latestEntityScrape(h.module.deps().ao, rec.match.mediaEntityId))?.id).toBe(rec.id); + }); + + it("只补缺失跳过已有文件(包括别人写的);全部覆盖改写", async () => { + const show = join(h.lib, SHOW, "tvshow.nfo"); + await writeFile(show, "别的工具写的"); + const miss = await scrape([{ root: h.libRoot, path: SHOW }]); + const rec = miss.body.data.records[0]; + expect(rec).toMatchObject({ status: "succeeded", written: 0 }); + expect(rec.skipped).toBeGreaterThan(0); + expect(await readFile(show, "utf8")).toBe("别的工具写的"); + const over = await scrape([{ root: h.libRoot, path: SHOW }], { mode: "overwrite" }); + expect(over.body.data.records[0]).toMatchObject({ status: "succeeded", mode: "overwrite", skipped: 0 }); + expect(await readFile(show, "utf8")).toContain("人生切割术"); + }); + + it("季目录只写季与各集;单集文件只写该集", async () => { + const season = await scrape([{ root: h.libRoot, path: `${SHOW}/Season 1` }], { mode: "overwrite" }); + const roles = season.body.data.records[0].files.map((f: { path: string; role: string }) => `${f.role}:${f.path}`); + expect(roles.sort()).toEqual( + [ + `season_nfo:${SHOW}/Season 1/season.nfo`, + `season_poster:${SHOW}/season01-poster.jpg`, + `season_poster:${SHOW}/Season 1/poster.jpg`, + `episode_nfo:${EP1.replace(/\.mkv$/, ".nfo")}`, + `episode_nfo:${EP2.replace(/\.mkv$/, ".nfo")}`, + `episode_thumb:${EP1.replace(/\.mkv$/, "-thumb.jpg")}`, + `episode_thumb:${EP2.replace(/\.mkv$/, "-thumb.jpg")}`, + ].sort(), + ); + const one = await scrape([{ root: h.libRoot, path: EP2 }], { mode: "overwrite" }); + expect(one.body.data.records[0].files.map((f: { role: string }) => f.role).sort()).toEqual([ + "episode_nfo", + "episode_thumb", + ]); + }); + + it("已登记的目录直接用作品;作品详情「重新刮削」对作品目录刮削", async () => { + const film = await seedEntity(h.db, { kind: "movie", title: "星际穿越", year: 2014 }); + await h.db + .insertInto("external_identity") + .values({ + media_entity_id: film, + provider: "tmdb", + entity_type: "movie", + external_id: "157336", + status: "accepted", + }) + .execute(); + await seedAsset(h.db, h, { entityId: film, rel: MOVIE_FILE }); + const r = await scrape([{ root: h.libRoot, path: MOVIE }]); + expect(r.body.data.records[0].match).toMatchObject({ + mediaEntityId: film, + source: "library", + externalId: "157336", + }); + // 作品详情「重新刮削」(POST /library/{id}/scrape)的服务端做法 + const again = await scrapeEntity(h.module.deps().ao, film, { trigger: "manual", mode: "overwrite" }); + expect(again).toHaveLength(1); + const recs = await h.api("GET", `/files/scrape-records?ids=${again[0]!.recordId}`); + expect(recs.body.data.records[0]).toMatchObject({ path: MOVIE, root: h.libRoot, mode: "overwrite" }); + }); + + it("只登记目录与非视频文件跳过;没有视频的目录跳过", async () => { + await mkdir(join(h.lib, "空目录")); + const r = await scrape([ + { root: h.libRoot, path: `${MOVIE}/readme.txt` }, + { root: h.libRoot, path: "空目录" }, + ]); + expect(r.body.data.skipped).toEqual([ + { root: h.libRoot, path: `${MOVIE}/readme.txt`, reason: "scrape_not_video" }, + { root: h.libRoot, path: "空目录", reason: "scrape_no_videos" }, + ]); + await h.db.updateTable("library_root").set({ management_mode: "register_only" }).execute(); + const ro = await scrape([{ root: h.libRoot, path: MOVIE }]); + expect(ro.body.data.skipped).toEqual([{ root: h.libRoot, path: MOVIE, reason: "scrape_root_read_only" }]); + await h.db.updateTable("library_root").set({ management_mode: "managed" }).execute(); + }); +}); diff --git a/apps/server/test/file-manager/v2.test.ts b/apps/server/test/file-manager/v2.test.ts index 6fe5bb2d..09cb5c2c 100644 --- a/apps/server/test/file-manager/v2.test.ts +++ b/apps/server/test/file-manager/v2.test.ts @@ -277,7 +277,7 @@ describe.skipIf(skip)("files v2(W9-H)", () => { expect(src.body.data.mediaEntityId).toBe(film); }); - it("G5:批量刷新元数据按作品去重,未登记作品跳过", async () => { + it("G5:批量刷新元数据按作品去重(不要求有已入库文件)", async () => { const reg = await seedEntity(h.db, { kind: "movie", title: "Registered Film", year: 2018 }); await mkdir(join(h.lib, "Registered Film (2018)"), { recursive: true }); await writeFile(join(h.lib, "Registered Film (2018)", "a.mkv"), big); @@ -299,8 +299,8 @@ describe.skipIf(skip)("files v2(W9-H)", () => { EXEC_KEY, ); expect(r.status).toBe(200); - expect(r.body.data.submitted.map((s: Loose) => s.mediaEntityId)).toEqual([reg]); - expect(r.body.data.skipped).toEqual([{ ref: bare, reason: "not_registered" }]); + expect(r.body.data.submitted.map((s: Loose) => s.mediaEntityId)).toEqual([reg, bare]); + expect(r.body.data.skipped).toEqual([]); }); it("G8:目录摘要 — 上一级、子目录是否有下级、视频/已登记计数", async () => { diff --git a/apps/server/test/scrape/gen-scrape-layout-mp.py b/apps/server/test/scrape/gen-scrape-layout-mp.py new file mode 100644 index 00000000..39b8bc6c --- /dev/null +++ b/apps/server/test/scrape/gen-scrape-layout-mp.py @@ -0,0 +1,144 @@ +"""SCR-2 刮削差分(文件布局):用 MP 上游 ScrapingChain.scrape_metadata 的真实决策代码,在临时目录里对同一组目录结构 +"就地刮削",记录 MP 写了哪些文件,写 scrape-layout-mp.json。与 apps/server/test/file-manager/scrape-layout.test.ts 对照。 + +用法(MP 仓库根目录,用其 .venv): + cd /opt/moviepilot-plus/upstream/MoviePilot && PYTHONPATH=. \ + .venv/bin/python /opt/uhd_media_tool/apps/server/test/scrape/gen-scrape-layout-mp.py + +只记录行为输出,不复制 MP 源码。替身: +- 存储:本地目录的最小实现(list_files / get_parent_item / get_file_item / upload_file / contains_bluray_subdirectories); +- NFO 内容:固定占位(内容差分见 gen-scrape-mp.py); +- 图片清单:按 MP themoviedb 模块 get_metadata_img 的规则给(作品 poster + backdrop;季 seasonNN-poster;集 episode-thumb); +- 识别:recognize_media 原样返回传入的作品(路径识别不在此差分范围)。 +刮削策略用 MP 缺省(ScrapingConfig 默认全部 missingonly);手动刮削 overwrite=True(同 MP /media/scrape 接口)。 +""" + +import json +import pathlib +import shutil +import tempfile +from types import SimpleNamespace + +import app.chain.scraping as scraping_mod +from app.chain.scraping import ScrapingChain, ScrapingConfig +from app.domain.context import MediaInfo +from app.schemas.types import MediaSource, MediaType +from app.schemas.workflow import FileItem + +HERE = pathlib.Path(__file__).parent +VIDEO_EXT = (".mkv", ".mp4", ".ts", ".m2ts", ".avi", ".iso") + + +def item(p: pathlib.Path) -> FileItem: + return FileItem( + storage="local", + type="dir" if p.is_dir() else "file", + path=str(p) + ("/" if p.is_dir() else ""), + name=p.name, + basename=p.stem if p.is_file() else p.name, + extension=p.suffix[1:] if p.is_file() else None, + ) + + +class LocalStorage: + def list_files(self, fileitem, recursion=False): + return [item(c) for c in sorted(pathlib.Path(fileitem.path).iterdir())] + + def get_parent_item(self, fileitem): + return item(pathlib.Path(fileitem.path.rstrip("/")).parent) + + def get_file_item(self, storage, path): + p = pathlib.Path(path) + return item(p) if p.exists() else None + + def upload_file(self, fileitem, path, new_name=None): + target = pathlib.Path(fileitem.path) / (new_name or path.name) + shutil.copyfile(path, target) + return item(target) + + def contains_bluray_subdirectories(self, files): + return any(f.type == "dir" and f.name.upper() in ("BDMV", "VIDEO_TS") for f in files) + + +def make_chain() -> ScrapingChain: + c = object.__new__(ScrapingChain) + c.storagechain = LocalStorage() + c.scraping_policies = ScrapingConfig() + c.runtime_config = SimpleNamespace( + video_extensions=VIDEO_EXT, audio_extensions=(".flac", ".mp3"), season_zero_names=("Specials", "SPs") + ) + c.metadata_nfo = lambda **kw: "" + + def metadata_img(mediainfo, season=None, episode=None): + if season is not None: + if episode: + return {"episode-thumb.jpg": "https://img/still.jpg"} + return {f"season{str(season).rjust(2, '0')}-poster.jpg": "https://img/season.jpg"} + return {"poster.jpg": "https://img/poster.jpg", "backdrop.jpg": "https://img/backdrop.jpg"} + + c.metadata_img = metadata_img + + def save_image(fileitem, path, url): + pathlib.Path(path).write_bytes(b"jpg") + + c._download_and_save_image = save_image + return c + + +class FakeMediaChain: + def recognize_media(self, meta=None, **kw): + return CURRENT["media"] + + def recognize_by_meta(self, meta): + return CURRENT["media"] + + +CURRENT: dict = {} +scraping_mod.MediaChain = FakeMediaChain # 集文件重新识别季集后仍是同一作品 + + +def media(kind: MediaType, tmdb_id: int, title: str) -> MediaInfo: + mi = MediaInfo() + mi.type = kind + mi.tmdb_id = tmdb_id + mi.media_source = MediaSource.TMDB + mi.media_id = str(tmdb_id) + mi.title = title + return mi + + +SHOW = "人生切割术 (2022)" +EP1 = f"{SHOW}/Season 1/人生切割术 - S01E01 - 有关地狱的好消息.mkv" +EP2 = f"{SHOW}/Season 1/人生切割术 - S01E02 - 半循环.mkv" +MOVIE = "星际穿越 (2014)" +MOVIE_FILE = f"{MOVIE}/星际穿越 (2014) - 1080p.mkv" +LOOSE = "Interstellar.2014.1080p.BluRay.mkv" +TREE = [EP1, EP2, MOVIE_FILE, LOOSE, "Heat (1995)/BDMV/index.bdmv"] + +SCENARIOS = { + "show_dir": (SHOW, MediaType.TV), + "season_dir": (f"{SHOW}/Season 1", MediaType.TV), + "episode_file": (EP2, MediaType.TV), + "movie_dir": (MOVIE, MediaType.MOVIE), + "movie_file": (MOVIE_FILE, MediaType.MOVIE), + "movie_file_in_root": (LOOSE, MediaType.MOVIE), + "bluray_dir": ("Heat (1995)", MediaType.MOVIE), +} + +out = {} +for name, (target, kind) in SCENARIOS.items(): + root = pathlib.Path(tempfile.mkdtemp(prefix="mp-scr-")) + for f in TREE: + (root / f).parent.mkdir(parents=True, exist_ok=True) + (root / f).write_bytes(b"video") + before = {str(p.relative_to(root)) for p in root.rglob("*") if p.is_file()} + CURRENT["media"] = media(kind, 95396 if kind == MediaType.TV else 157336, "x") + chain = make_chain() + ok, msg = chain.scrape_metadata(fileitem=item(root / target), mediainfo=CURRENT["media"], overwrite=True) + after = {str(p.relative_to(root)) for p in root.rglob("*") if p.is_file()} + out[name] = {"target": target, "ok": ok, "written": sorted(after - before)} + shutil.rmtree(root) + +(HERE / "scrape-layout-mp.json").write_text(json.dumps(out, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") +print(json.dumps(out, ensure_ascii=False, indent=2)) +# 生成后在本仓库跑一次格式化(lint 要求):deno run -A npm:@biomejs/biome check --write apps/server/test/scrape/scrape-layout-mp.json diff --git a/apps/server/test/scrape/scrape-layout-mp.json b/apps/server/test/scrape/scrape-layout-mp.json new file mode 100644 index 00000000..97f15a4d --- /dev/null +++ b/apps/server/test/scrape/scrape-layout-mp.json @@ -0,0 +1,75 @@ +{ + "show_dir": { + "target": "人生切割术 (2022)", + "ok": true, + "written": [ + "人生切割术 (2022)/Season 1/poster.jpg", + "人生切割术 (2022)/Season 1/season.nfo", + "人生切割术 (2022)/Season 1/人生切割术 - S01E01 - 有关地狱的好消息.jpg", + "人生切割术 (2022)/Season 1/人生切割术 - S01E01 - 有关地狱的好消息.nfo", + "人生切割术 (2022)/Season 1/人生切割术 - S01E02 - 半循环.jpg", + "人生切割术 (2022)/Season 1/人生切割术 - S01E02 - 半循环.nfo", + "人生切割术 (2022)/backdrop.jpg", + "人生切割术 (2022)/fanart.jpg", + "人生切割术 (2022)/poster.jpg", + "人生切割术 (2022)/season01-poster.jpg", + "人生切割术 (2022)/tvshow.nfo" + ] + }, + "season_dir": { + "target": "人生切割术 (2022)/Season 1", + "ok": true, + "written": [ + "人生切割术 (2022)/Season 1/poster.jpg", + "人生切割术 (2022)/Season 1/season.nfo", + "人生切割术 (2022)/Season 1/人生切割术 - S01E01 - 有关地狱的好消息.jpg", + "人生切割术 (2022)/Season 1/人生切割术 - S01E01 - 有关地狱的好消息.nfo", + "人生切割术 (2022)/Season 1/人生切割术 - S01E02 - 半循环.jpg", + "人生切割术 (2022)/Season 1/人生切割术 - S01E02 - 半循环.nfo", + "人生切割术 (2022)/season01-poster.jpg" + ] + }, + "episode_file": { + "target": "人生切割术 (2022)/Season 1/人生切割术 - S01E02 - 半循环.mkv", + "ok": true, + "written": [ + "人生切割术 (2022)/Season 1/人生切割术 - S01E02 - 半循环.jpg", + "人生切割术 (2022)/Season 1/人生切割术 - S01E02 - 半循环.nfo" + ] + }, + "movie_dir": { + "target": "星际穿越 (2014)", + "ok": true, + "written": [ + "星际穿越 (2014)/backdrop.jpg", + "星际穿越 (2014)/fanart.jpg", + "星际穿越 (2014)/poster.jpg", + "星际穿越 (2014)/星际穿越 (2014) - 1080p.nfo" + ] + }, + "movie_file": { + "target": "星际穿越 (2014)/星际穿越 (2014) - 1080p.mkv", + "ok": true, + "written": [ + "星际穿越 (2014)/backdrop.jpg", + "星际穿越 (2014)/fanart.jpg", + "星际穿越 (2014)/poster.jpg", + "星际穿越 (2014)/星际穿越 (2014) - 1080p.nfo" + ] + }, + "movie_file_in_root": { + "target": "Interstellar.2014.1080p.BluRay.mkv", + "ok": true, + "written": ["Interstellar.2014.1080p.BluRay.nfo", "backdrop.jpg", "fanart.jpg", "poster.jpg"] + }, + "bluray_dir": { + "target": "Heat (1995)", + "ok": true, + "written": [ + "Heat (1995)/Heat (1995).nfo", + "Heat (1995)/backdrop.jpg", + "Heat (1995)/fanart.jpg", + "Heat (1995)/poster.jpg" + ] + } +} diff --git a/apps/server/test/scrape/scrape-layout-mp.test.ts b/apps/server/test/scrape/scrape-layout-mp.test.ts new file mode 100644 index 00000000..37952076 --- /dev/null +++ b/apps/server/test/scrape/scrape-layout-mp.test.ts @@ -0,0 +1,174 @@ +import { mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises"; +import { dirname, join, relative } from "node:path"; +import type { TmdbRichDetail, TmdbRichSeasonDetail } from "@umt/adapters"; +import type { ScrapeMatch } from "@umt/contracts"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { queueScrape, runScrapeRecord } from "../../src/modules/acquisition-organize/services/scrape.ts"; +import { createFmHarness, type FmHarness, pgAvailable } from "../file-manager/harness.ts"; + +// SCR-2 刮削文件布局差分:同一组目录结构,MP ScrapingChain.scrape_metadata(gen-scrape-layout-mp.py 生成 +// scrape-layout-mp.json)写哪些文件 ⇔ 我们就地刮削写哪些文件。偏离只允许下面 DEVIATIONS 里登记的几类 +// (理由见 docs/runbook/scrape.md「与 MP 的差分」)。 + +const skip = !(await pgAvailable()); +const MP = JSON.parse(await readFile(new URL("./scrape-layout-mp.json", import.meta.url), "utf8")) as Record< + string, + { target: string; ok: boolean; written: string[] } +>; + +const SHOW = "人生切割术 (2022)"; +const EP1 = `${SHOW}/Season 1/人生切割术 - S01E01 - 有关地狱的好消息.mkv`; +const EP2 = `${SHOW}/Season 1/人生切割术 - S01E02 - 半循环.mkv`; +const MOVIE_FILE = "星际穿越 (2014)/星际穿越 (2014) - 1080p.mkv"; +const LOOSE = "Interstellar.2014.1080p.BluRay.mkv"; +const TREE = [EP1, EP2, MOVIE_FILE, LOOSE, "Heat (1995)/BDMV/index.bdmv"]; + +/** 登记的偏离:把 MP 的输出换算成我们应有的输出 */ +const DEVIATIONS: { id: string; apply: (files: string[], scenario: string) => string[] }[] = [ + // 1. 不写 backdrop.jpg 副本,只写 fanart.jpg(Emby / Jellyfin / Kodi 都认 fanart) + { id: "no_backdrop_copy", apply: (fs) => fs.filter((f) => !/(^|\/)backdrop\.jpg$/.test(f)) }, + // 2. 剧照 <视频名>-thumb.jpg(Kodi 约定;MP 写 <视频名>.jpg) + { + id: "episode_thumb_suffix", + apply: (fs) => fs.map((f) => (/ - S\d+E\d+ - [^/]+\.jpg$/.test(f) ? f.replace(/\.jpg$/, "-thumb.jpg") : f)), + }, + // 3. 视频直接在登记目录根:图片写 <视频名>-poster.jpg / -fanart.jpg,不往根里放 poster.jpg(MP 会) + { + id: "root_movie_images", + apply: (fs, s) => + s !== "movie_file_in_root" + ? fs + : fs.map((f) => (f === "poster.jpg" || f === "fanart.jpg" ? `${LOOSE.replace(/\.mkv$/, "")}-${f}` : f)), + }, +]; + +const detail = (entityType: "movie" | "tv", externalId: string): TmdbRichDetail => + ({ + entityType, + externalId, + title: "x", + titleLanguage: "zh-CN", + titleZh: "x", + titleZhLanguage: "zh-CN", + originalTitle: null, + originalLanguage: "en", + overview: null, + overviewLanguage: null, + tagline: null, + year: 2022, + releaseDate: null, + lastAirDate: null, + genres: [], + voteAverage: null, + voteCount: null, + status: null, + runtime: null, + seasonCount: null, + episodeCount: null, + networks: [], + originCountry: [], + posterPath: "/poster.jpg", + backdropPath: "/backdrop.jpg", + cast: [], + crew: [], + externalIds: { imdb: null, tvdb: null, wikidata: null }, + translations: [], + alternativeTitles: [], + seasons: [], + languages: ["zh-CN"], + }) satisfies TmdbRichDetail; + +const season1: TmdbRichSeasonDetail = { + tvId: "95396", + seasonNumber: 1, + name: null, + nameLanguage: null, + overview: null, + overviewLanguage: null, + airDate: null, + posterPath: "/season.jpg", + fetchedLanguages: ["zh-CN"], + episodes: [1, 2].map((n) => ({ + seasonNumber: 1, + episodeNumber: n, + name: null, + nameLanguage: null, + overview: null, + overviewLanguage: null, + airDate: null, + runtime: null, + stillPath: "/still.jpg", + voteAverage: null, + voteCount: null, + })), +}; + +async function files(root: string): Promise> { + const out = new Set(); + const walk = async (d: string) => { + for (const n of await readdir(d).catch(() => [])) { + if (n.startsWith(".")) continue; + const p = join(d, n); + if ((await stat(p)).isDirectory()) await walk(p); + else out.add(relative(root, p)); + } + }; + await walk(root); + return out; +} + +describe.skipIf(skip)("刮削文件布局与 MP 差分", () => { + let h: FmHarness; + beforeAll(async () => { + h = await createFmHarness( + "scrlayout", + { + ao: { + scrapeImage: async () => ({ content: new Uint8Array([0xff, 0xd8, 0xff]), mime: "image/jpeg" }), + scrapeTmdbDetail: async (type, id) => detail(type, id), + scrapeTmdbSeason: async (_tv, n) => (n === 1 ? season1 : null), + }, + }, + { runTransfers: false }, + ); + }); + afterAll(async () => h?.close()); + + for (const [scenario, mp] of Object.entries(MP)) { + it(`${scenario}:${mp.target}`, async () => { + // 每个场景一份独立目录树;"视频直接在根"的场景用下载根(整个根就是这棵树) + const inRoot = scenario === "movie_file_in_root"; + const base = inRoot ? h.dl : join(h.lib, scenario); + for (const f of TREE) { + await mkdir(dirname(join(base, f)), { recursive: true }); + await writeFile(join(base, f), "video"); + } + const before = await files(base); + const tv = mp.target.startsWith(SHOW); + const match: ScrapeMatch = { + mediaEntityId: null, + provider: "tmdb", + externalId: tv ? "95396" : "157336", + kind: tv ? "tv" : "movie", + title: "x", + year: null, + source: "chosen", + }; + const q = await queueScrape(h.module.deps().ao, { + ...(inRoot + ? { rootId: h.dlRoot, rootPath: h.dl, libraryRootId: null } + : { rootId: h.libRoot, rootPath: h.lib, libraryRootId: h.libraryRootId }), + path: inRoot ? mp.target : `${scenario}/${mp.target}`, + match, + mode: "overwrite", + trigger: "manual", + }); + const rec = await runScrapeRecord(h.module.deps().ao, q.recordId); + expect(rec.status).toBe("succeeded"); + const after = await files(base); + const ours = [...after].filter((f) => !before.has(f)).sort(); + const expected = DEVIATIONS.reduce((fs, d) => d.apply(fs, scenario), [...mp.written]).sort(); + expect(ours).toEqual(expected); + }); + } +}); diff --git a/packages/contracts/src/api.ts b/packages/contracts/src/api.ts index f8ac6664..c8304da6 100644 --- a/packages/contracts/src/api.ts +++ b/packages/contracts/src/api.ts @@ -21,6 +21,7 @@ import * as organization from "./organization.ts"; import * as pi from "./pi.ts"; import * as policy from "./policy.ts"; import * as schedules from "./schedules.ts"; +import * as scrape from "./scrape.ts"; import * as search from "./search.ts"; import * as settingsNs from "./settings.ts"; import { GlobalSettings, GlobalSettingsUpdate } from "./settings.ts"; @@ -1551,10 +1552,19 @@ export const Api = { method: "post", path: "/files/scrape", scope: "organize.execute", - summary: "刮削已登记作品（字段级刷新，遵守字段锁；NFO 投影 W5）", + summary: + "刮削：对目录或视频文件就地写 NFO 与图片（按路径识别作品，不要求已登记；认不准时返回候选，选定后再提交）", body: files.FilesScrapeRequest, response: files.FilesScrapeResult, }), + scrapeRecords: ep({ + method: "get", + path: "/files/scrape-records", + scope: "organize.read", + summary: "按 id 查刮削记录（目录、识别结果、逐文件结果）", + query: files.FilesScrapeRecordsQuery, + response: files.FilesScrapeRecordsResult, + }), probe: ep({ method: "post", path: "/files/probe", @@ -2244,8 +2254,8 @@ export const Api = { scope: "organize.execute", summary: "刮削：把作品的 NFO 与图片写进媒体库目录（后台任务 scrapeMedia；覆盖方式缺省按设置）", params: IdParam, - body: organization.LibraryScrapeRequest, - response: organization.LibraryScrapeResult, + body: scrape.LibraryScrapeRequest, + response: scrape.LibraryScrapeResult, }), reconcile: ep({ method: "post", diff --git a/packages/contracts/src/files.ts b/packages/contracts/src/files.ts index 5264742f..05d40531 100644 --- a/packages/contracts/src/files.ts +++ b/packages/contracts/src/files.ts @@ -5,6 +5,7 @@ import { DownloadStatus } from "./execution.ts"; import { InstanceHealth, LibraryMediaKind } from "./instances.ts"; import { OrganizationStatus, PublicOrganizationRecord } from "./organization.ts"; import { SlotStatus } from "./policy.ts"; +import { ScrapeMode, ScrapeRecord } from "./scrape.ts"; // 文件管理(`/files`,docs/plan/06-file-manager-spec.md)。 // 只能浏览已登记的库根与下载根;路径一律用 {root, path}:root 为 FileRootId,path 为根内相对路径("" = 根)。 @@ -680,30 +681,59 @@ export const IdentifyCompletedPayload = z.object({ }); export type IdentifyCompletedPayload = z.infer; -// 刮削(仅已登记作品) +// 刮削(SCR-2,照 MP):任意目录 / 视频文件就地写 NFO 与图片;按路径识别作品(已登记直接用,未登记走 TMDB 匹配), +// 认不准时返回候选(needsChoice),选定后带 mediaEntityId / external 再提交一次。 + +export const FilesScrapeItem = FileRef.extend({ + /** 指定本地作品(选定候选后回传);与 external 二选一 */ + mediaEntityId: Id.optional(), + /** 指定 TMDB 条目(候选里只有 TMDB 的);不登记也能刮削 */ + external: FilesExternalCandidate.optional(), +}); +export type FilesScrapeItem = z.infer; export const FilesScrapeRequest = z.object({ idempotencyKey: IdempotencyKey, - mediaEntityId: Id, - /** SCR-1:true 时另排 scrapeMedia,把 NFO(与 images 时的图片)写进媒体库目录 */ - writeNfo: z.boolean().default(false), - images: z.boolean().default(false), + items: z.array(FilesScrapeItem).min(1).max(100), + /** 缺省按设置 organize.scrapeMode */ + mode: ScrapeMode.optional(), + /** 指定的 TMDB 条目顺手加入媒体库(建作品);不影响刮削本身 */ + addToLibrary: z.boolean().default(false), }); export type FilesScrapeRequest = z.infer; + +export const FilesScrapeNeedsChoice = z.object({ + root: FileRootId, + path: z.string(), + /** 从路径解析出的标题 */ + title: z.string().nullable(), + /** 前 5 个候选(可能为空:让用户搜索) */ + candidates: z.array(FileIdentifyCandidate).max(5), + warnings: z.array(z.string()), +}); +export type FilesScrapeNeedsChoice = z.infer; + export const FilesScrapeResult = z.object({ - mediaEntityId: Id, - /** refreshMetadata 引擎 run id(异步;字段级刷新,用户锁优先,不动路径) */ - taskId: z.string(), - /** 当前被用户锁定、刷新不会改动的字段 */ - lockedFields: z.array(z.string()), - /** NFO/图片投影(SCR-1):queued = 已排 scrapeMedia,结果看作品刮削状态;not_implemented 已不再返回 */ - nfo: z.enum(["written", "skipped", "queued", "not_requested", "not_implemented"]), - images: z.enum(["written", "skipped", "queued", "not_requested", "not_implemented"]), - /** scrapeMedia 引擎 run id(writeNfo / images 时) */ - scrapeTaskId: z.string().nullable().optional(), + /** 已提交的刮削(每项一条记录);pending=true 时仍有排队 / 进行中的,按 id 查 /files/scrape-records */ + records: z.array(ScrapeRecord), + /** 认不准作品的项:选一个候选后带 mediaEntityId / external 重新提交 */ + needsChoice: z.array(FilesScrapeNeedsChoice), + /** 没提交的项:scrape_root_read_only(只登记目录)/ scrape_not_video(不是视频或目录) */ + skipped: z.array(z.object({ root: FileRootId, path: z.string(), reason: z.string() })), + pending: z.boolean(), + /** addToLibrary 时加入媒体库的作品 */ + entities: z.array(z.object({ mediaEntityId: Id, title: z.string(), created: z.boolean() })), }); export type FilesScrapeResult = z.infer; +export const FilesScrapeRecordsQuery = z.object({ + /** 逗号分隔的刮削记录 id(≤100) */ + ids: z.string().min(1).max(4000), +}); +export type FilesScrapeRecordsQuery = z.infer; +export const FilesScrapeRecordsResult = z.object({ records: z.array(ScrapeRecord) }); +export type FilesScrapeRecordsResult = z.infer; + // 探测 export const FilesProbeRequest = z.object({ items: z.array(FileRef).min(1).max(20) }); @@ -843,20 +873,18 @@ export const FilesEntityFromCandidateResult = z.object({ }); export type FilesEntityFromCandidateResult = z.infer; -/** G5:批量刷新元数据(刮削):按作品去重;items 里的文件/目录归到其已登记资产的作品 */ +/** G5:批量刷新元数据:按作品去重;items 里的文件/目录归到其已登记资产的作品(刮削见 /files/scrape) */ export const FilesScrapeBatchRequest = z.object({ idempotencyKey: IdempotencyKey, items: z.array(FileRef).max(500).optional(), mediaEntityIds: z.array(Id).max(200).optional(), - /** SCR-1:true = 刮削(排 scrapeMedia 写 NFO 与图片,覆盖方式按设置);false(缺省)= 刷新元数据 */ - write: z.boolean().default(false), }); export type FilesScrapeBatchRequest = z.infer; export const FilesScrapeBatchResult = z.object({ submitted: z.array( z.object({ mediaEntityId: Id, title: z.string(), taskId: z.string(), lockedFields: z.array(z.string()) }), ), - /** 未提交:not_registered(作品没有已登记资产)/ not_found / no_entity(文件不属于任何已登记作品) */ + /** 未提交:not_found / no_entity(文件不属于任何已登记作品) */ skipped: z.array(z.object({ ref: z.string(), reason: z.string() })), }); export type FilesScrapeBatchResult = z.infer; diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index afc99244..254e638b 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -21,5 +21,6 @@ export * from "./pi.ts"; export * from "./policy.ts"; export * from "./reasons.ts"; export * from "./redact.ts"; +export * from "./scrape.ts"; export * from "./settings.ts"; export * from "./sources.ts"; diff --git a/packages/contracts/src/organization.ts b/packages/contracts/src/organization.ts index b46ea126..34b12c6b 100644 --- a/packages/contracts/src/organization.ts +++ b/packages/contracts/src/organization.ts @@ -6,6 +6,7 @@ import { MediaEntity, MediaKind } from "./content.ts"; import { AcquisitionPlan, DownloadStatus, DownloadTask } from "./execution.ts"; import { ExternalIdentity } from "./identity.ts"; import { AcquisitionSlot, CandidateEvaluation, SlotStatus } from "./policy.ts"; +import { LibraryScrapeState } from "./scrape.ts"; // 整理域:整理记录(批次)+ 逐文件结果,以及对外开放的整理记录 API 合同(REQ §12.4) // 自动与手动整理产生同格式记录;读记录需 organize.read,发起整理需 organize.execute。 @@ -370,81 +371,6 @@ export const LibraryDroppedPlanView = z.object({ }); export type LibraryDroppedPlanView = z.infer; -/** SCR-1 刮削:一个写入目标(NFO / 图片)的结果;path 为库根内相对路径 */ -export const ScrapeFileResult = z.object({ - path: z.string(), - kind: z.enum(["nfo", "image"]), - /** movie_nfo / tvshow_nfo / season_nfo / episode_nfo / poster / fanart / season_poster / episode_thumb */ - role: z.string(), - action: z.enum(["written", "skipped", "failed"]), - /** 跳过 / 失败原因码(reasons.ts:scrape_exists / scrape_write_failed / scrape_image_failed …) */ - reason: z.string().nullable(), -}); -export type ScrapeFileResult = z.infer; - -export const ScrapeMode = z.enum(["missing", "overwrite"]); -export type ScrapeMode = z.infer; - -export const ScrapeStatus = z.enum(["queued", "running", "succeeded", "partial", "failed"]); -export type ScrapeStatus = z.infer; -export const ScrapeTrigger = z.enum(["auto", "manual"]); -export type ScrapeTrigger = z.infer; - -/** 表 media_scrape 的行(一部作品一行,最近一次刮削) */ -export const MediaScrape = z.object({ - mediaEntityId: Id, - status: ScrapeStatus, - trigger: ScrapeTrigger, - mode: ScrapeMode, - recordId: Id.nullable(), - engineRunId: z.string().nullable(), - requestedAt: Timestamp, - startedAt: Timestamp.nullable(), - finishedAt: Timestamp.nullable(), - written: NonNegInt, - skipped: NonNegInt, - failed: NonNegInt, - reason: z.string().nullable(), - files: z.array(ScrapeFileResult), - createdAt: Timestamp, - updatedAt: Timestamp, -}); -export type MediaScrape = z.infer; - -/** SCR-1 作品刮削状态(最近一次;表 media_scrape) */ -export const LibraryScrapeState = z.object({ - status: ScrapeStatus, - trigger: ScrapeTrigger, - mode: ScrapeMode, - requestedAt: Timestamp, - startedAt: Timestamp.nullable(), - finishedAt: Timestamp.nullable(), - written: NonNegInt, - skipped: NonNegInt, - failed: NonNegInt, - /** 整体失败原因码(没有文件、没有元数据等) */ - reason: z.string().nullable(), - files: z.array(ScrapeFileResult), - /** 触发它的整理记录(入库后自动刮削) */ - recordId: Id.nullable(), - /** 引擎 run id */ - taskId: z.string().nullable(), -}); -export type LibraryScrapeState = z.infer; - -export const LibraryScrapeRequest = z.object({ - /** 缺省按设置 organize.scrapeMode */ - mode: ScrapeMode.optional(), -}); -export type LibraryScrapeRequest = z.infer; -export const LibraryScrapeResult = z.object({ - mediaEntityId: Id, - taskId: z.string(), - /** 同一作品已有排队中的刮削,直接复用 */ - deduplicated: z.boolean(), -}); -export type LibraryScrapeResult = z.infer; - export const LibraryDetail = z.object({ entity: MediaEntity, /** 身份映射状态(含 proposed/conflicted,前端区分展示) */ diff --git a/packages/contracts/src/reasons.ts b/packages/contracts/src/reasons.ts index cd083aac..cbb7f320 100644 --- a/packages/contracts/src/reasons.ts +++ b/packages/contracts/src/reasons.ts @@ -87,13 +87,21 @@ const REASONS: Record = { read_only_reconcile: { text: "只读对账中，暂不提交下载" }, rejected: { text: "下载器拒绝了这个种子" }, source_failed: { text: "从站点取种子失败", action: "检查站点登录后重新下载" }, - // SCR-1 刮削(作品刮削状态、逐文件结果) - scrape_no_files: { text: "作品没有已入库文件", action: "先整理入库" }, - scrape_no_metadata: { text: "作品还没有元数据", action: "先刷新元数据" }, - scrape_root_read_only: { text: "媒体库目录是只登记模式，不写文件" }, - scrape_exists: { text: "文件已存在，按设置只补缺失" }, - scrape_write_failed: { text: "写入媒体库目录失败", action: "检查目录权限后重新刮削" }, + // SCR-1 / SCR-2 刮削(刮削记录、逐文件结果) + scrape_no_files: { text: "作品没有已入库文件", action: "到文件管理里对目录刮削" }, + scrape_no_metadata: { text: "没有可用的元数据", action: "指定 TMDB 条目后重新刮削" }, + scrape_root_read_only: { text: "只登记模式的媒体库目录，不写文件" }, + scrape_root_offline: { text: "目录不可访问或未挂载" }, + scrape_not_video: { text: "不是视频文件或目录" }, + scrape_path_missing: { text: "目录或文件已不存在" }, + path_escape: { text: "路径指向登记目录之外" }, + task_enqueue_failed: { text: "后台任务没排上", action: "稍后重试" }, + scrape_no_videos: { text: "目录里没有视频", action: "对作品或季目录刮削" }, + scrape_tmdb_unavailable: { text: "TMDB 不可用", action: "检查 TMDB 设置后重新刮削" }, + scrape_exists: { text: "文件已存在，只补缺失时不改" }, + scrape_write_failed: { text: "写入目录失败", action: "检查目录权限后重新刮削" }, scrape_image_failed: { text: "图片没下载成功", action: "稍后重新刮削" }, + scrape_no_episode: { text: "文件名里认不出集号" }, scrape_no_work_dir: { text: "视频直接放在媒体库目录下，不写剧集和季的文件" }, }; diff --git a/packages/contracts/src/scrape.ts b/packages/contracts/src/scrape.ts new file mode 100644 index 00000000..77154da4 --- /dev/null +++ b/packages/contracts/src/scrape.ts @@ -0,0 +1,119 @@ +import { z } from "zod"; +import { Id, NonNegInt, Timestamp } from "./common.ts"; + +// 刮削(SCR-1 / SCR-2):在目录里就地写 NFO 与图片(Emby / Jellyfin / Kodi 读取)。 +// SCR-2 起照 MP 简化:文件管理里任意目录 / 视频文件都能刮削(按路径识别作品,不要求先登记); +// 作品详情「重新刮削」、入库后自动刮削走同一实现(对作品目录刮削)。每次刮削一条记录(表 scrape_record)。 + +/** 一个写入目标(NFO / 图片)的结果;path 为所在登记目录(库根 / 下载根)内的相对路径 */ +export const ScrapeFileResult = z.object({ + path: z.string(), + kind: z.enum(["nfo", "image"]), + /** movie_nfo / tvshow_nfo / season_nfo / episode_nfo / poster / fanart / season_poster / episode_thumb */ + role: z.string(), + action: z.enum(["written", "skipped", "failed"]), + /** 跳过 / 失败原因码(reasons.ts:scrape_exists / scrape_write_failed / scrape_image_failed …) */ + reason: z.string().nullable(), +}); +export type ScrapeFileResult = z.infer; + +/** missing = 只补缺失;overwrite = 全部覆盖 */ +export const ScrapeMode = z.enum(["missing", "overwrite"]); +export type ScrapeMode = z.infer; + +export const ScrapeStatus = z.enum(["queued", "running", "succeeded", "partial", "failed"]); +export type ScrapeStatus = z.infer; +export const ScrapeTrigger = z.enum(["auto", "manual"]); +export type ScrapeTrigger = z.infer; + +/** 识别结果:library = 目录里的文件已登记到该作品;auto = 按路径唯一识别;chosen = 用户从候选里选的 */ +export const ScrapeMatchSource = z.enum(["library", "auto", "chosen"]); +export type ScrapeMatchSource = z.infer; + +export const ScrapeMatch = z.object({ + /** 本地作品(没有登记时为 null,直接用 TMDB 字段) */ + mediaEntityId: Id.nullable(), + provider: z.literal("tmdb").nullable(), + externalId: z.string().nullable(), + kind: z.enum(["movie", "tv"]), + title: z.string(), + year: z.number().int().nullable(), + source: ScrapeMatchSource, +}); +export type ScrapeMatch = z.infer; + +/** 一次刮削(表 scrape_record):目录、识别结果、写了哪些文件、跳过 / 失败原因 */ +export const ScrapeRecord = z.object({ + id: Id, + /** 登记目录 id(`lib:` / `dl::`) */ + root: z.string(), + /** 目录内相对路径(刮削的目录或视频文件) */ + path: z.string(), + status: ScrapeStatus, + trigger: ScrapeTrigger, + mode: ScrapeMode, + match: ScrapeMatch.nullable(), + written: NonNegInt, + skipped: NonNegInt, + failed: NonNegInt, + /** 整体失败原因码(没有元数据、目录不可访问等) */ + reason: z.string().nullable(), + files: z.array(ScrapeFileResult), + /** 触发它的整理记录(入库后自动刮削) */ + organizationRecordId: Id.nullable(), + /** 引擎 run id */ + taskId: z.string().nullable(), + requestedAt: Timestamp, + startedAt: Timestamp.nullable(), + finishedAt: Timestamp.nullable(), +}); +export type ScrapeRecord = z.infer; + +/** 表 scrape_record 的行(0058) */ +export const ScrapeRecordRow = z.object({ + id: Id, + rootId: z.string(), + rootPath: z.string(), + libraryRootId: Id.nullable(), + path: z.string(), + mediaEntityId: Id.nullable(), + match: ScrapeMatch.nullable(), + status: ScrapeStatus, + trigger: ScrapeTrigger, + mode: ScrapeMode, + organizationRecordId: Id.nullable(), + engineRunId: z.string().nullable(), + actorType: z.string().nullable(), + actorId: z.string().nullable(), + requestedAt: Timestamp, + startedAt: Timestamp.nullable(), + finishedAt: Timestamp.nullable(), + written: NonNegInt, + skipped: NonNegInt, + failed: NonNegInt, + reason: z.string().nullable(), + files: z.array(ScrapeFileResult), + createdAt: Timestamp, + updatedAt: Timestamp, +}); +export type ScrapeRecordRow = z.infer; + +/** 作品详情的刮削状态 = 该作品最近一次刮削记录 */ +export const LibraryScrapeState = ScrapeRecord; +export type LibraryScrapeState = ScrapeRecord; + +export const LibraryScrapeRequest = z.object({ + /** 缺省按设置 organize.scrapeMode */ + mode: ScrapeMode.optional(), +}); +export type LibraryScrapeRequest = z.infer; +export const LibraryScrapeResult = z.object({ + mediaEntityId: Id, + /** 第一条记录的引擎 run id */ + taskId: z.string().nullable(), + /** 同一目录已有排队中的刮削,直接复用 */ + deduplicated: z.boolean(), + /** 每个作品目录一条刮削记录 */ + recordIds: z.array(Id), +}); +export type LibraryScrapeResult = z.infer; diff --git a/packages/contracts/test/db-sync.test.ts b/packages/contracts/test/db-sync.test.ts index d5a0b5f2..2797655d 100644 --- a/packages/contracts/test/db-sync.test.ts +++ b/packages/contracts/test/db-sync.test.ts @@ -63,7 +63,6 @@ const ROWS: Record LibraryRoot: { row: C.LibraryRoot }, MediaAsset: { row: C.MediaAsset }, MediaEntity: { row: C.MediaEntity }, - MediaScrape: { row: C.MediaScrape }, MediaServerInstance: { row: C.MediaServerInstance }, Observation: { row: C.Observation }, OrganizationFileResult: { row: C.OrganizationFileResult }, @@ -76,6 +75,7 @@ const ROWS: Record PreferenceTag: { row: C.PreferenceTag }, ReconcileRun: { row: C.ReconcileRun }, Secret: { row: C.Secret, dbOnly: ["ciphertext", "nonce", "authTag"] }, + ScrapeRecord: { row: C.ScrapeRecordRow }, ServerMapping: { row: C.ServerMapping }, Settings: { row: C.SettingEntry }, // deleted_at:软删除只在服务端过滤(已删站点不出 API);capabilities 由类型 ∩ 站点定义算出(IA-2) @@ -210,9 +210,6 @@ const ENUMS: Record = { MediaAssetStatus: C.AssetStatus, MediaEntityKind: C.MediaKind, MediaEntityStatus: C.MediaEntityStatus, - MediaScrapeMode: C.ScrapeMode, - MediaScrapeStatus: C.ScrapeStatus, - MediaScrapeTrigger: C.ScrapeTrigger, MediaServerInstanceKind: C.MediaServerKind, MediaServerInstanceHealthStatus: C.InstanceHealth, ObservationStatus: C.ObservationStatus, @@ -246,6 +243,9 @@ const ENUMS: Record = { ScheduledJobLastStatus: C.ScheduleRunStatus, ScheduledJobRunTrigger: C.ScheduleTrigger, ScheduledJobRunStatus: C.ScheduleRunStatus, + ScrapeRecordStatus: C.ScrapeStatus, + ScrapeRecordTrigger: C.ScrapeTrigger, + ScrapeRecordMode: C.ScrapeMode, PiUpstreamAuthKind: C.PiAuthKind, PiUpstreamStatus: C.PiUpstreamStatus, PiUpstreamThinkingLevel: C.PiThinkingLevel, diff --git a/packages/db/migrations/0058_scrape_record.ts b/packages/db/migrations/0058_scrape_record.ts new file mode 100644 index 00000000..672b062b --- /dev/null +++ b/packages/db/migrations/0058_scrape_record.ts @@ -0,0 +1,72 @@ +import type { Kysely } from "kysely"; +import { execDdl } from "../src/migration-helpers.ts"; + +// SCR-2 刮削照 MP 简化:刮削对象从"作品"改为"目录 / 视频文件"(不要求先登记),每次刮削一条记录: +// 目录(登记目录 id + 相对路径)、识别结果(本地作品 / TMDB 条目)、写了哪些文件、跳过 / 失败原因。 +// 替代 SCR-1 的 media_scrape(一部作品一行);作品详情取该作品最近一条记录。 + +const MEDIA_SCRAPE = `create table if not exists media_scrape ( + media_entity_id uuid primary key references media_entity (id) on delete cascade, + status text not null default 'queued' check (status in ('queued', 'running', 'succeeded', 'partial', 'failed')), + trigger text not null default 'manual' check (trigger in ('auto', 'manual')), + mode text not null default 'missing' check (mode in ('missing', 'overwrite')), + record_id uuid references organization_record (id) on delete set null, + engine_run_id text, + requested_at timestamptz not null default now(), + started_at timestamptz, + finished_at timestamptz, + written integer not null default 0, + skipped integer not null default 0, + failed integer not null default 0, + reason text, + files jsonb not null default '[]'::jsonb, + created_at timestamptz not null default now(), + updated_at timestamptz not null default now() + ); + drop trigger if exists media_scrape_touch_updated_at on media_scrape; + create trigger media_scrape_touch_updated_at before update on media_scrape + for each row execute function umt_touch_updated_at();`; + +// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型 +export async function up(db: Kysely): Promise { + await execDdl( + db, + `create table if not exists scrape_record ( + id uuid primary key default gen_random_uuid(), + root_id text not null, + root_path text not null, + library_root_id uuid references library_root (id) on delete set null, + path text not null default '', + media_entity_id uuid references media_entity (id) on delete set null, + match jsonb, + status text not null default 'queued' check (status in ('queued', 'running', 'succeeded', 'partial', 'failed')), + trigger text not null default 'manual' check (trigger in ('auto', 'manual')), + mode text not null default 'missing' check (mode in ('missing', 'overwrite')), + organization_record_id uuid references organization_record (id) on delete set null, + engine_run_id text, + actor_type text, + actor_id text, + requested_at timestamptz not null default now(), + started_at timestamptz, + finished_at timestamptz, + written integer not null default 0, + skipped integer not null default 0, + failed integer not null default 0, + reason text, + files jsonb not null default '[]'::jsonb, + created_at timestamptz not null default now(), + updated_at timestamptz not null default now() + ); + create index if not exists scrape_record_entity_idx on scrape_record (media_entity_id, requested_at desc); + create index if not exists scrape_record_path_idx on scrape_record (root_id, path, requested_at desc); + drop trigger if exists scrape_record_touch_updated_at on scrape_record; + create trigger scrape_record_touch_updated_at before update on scrape_record + for each row execute function umt_touch_updated_at(); + drop table if exists media_scrape;`, + ); +} + +// biome-ignore lint/suspicious/noExplicitAny: 同上 +export async function down(db: Kysely): Promise { + await execDdl(db, `drop table if exists scrape_record; ${MEDIA_SCRAPE}`); +} diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index 01be7d21..56605b46 100644 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -159,9 +159,6 @@ export type MediaAssetKind = "video" | "bluray_dir" | "subtitle" | "audio" | "nf export type MediaAssetStatus = "active" | "replaced" | "retired" | "missing"; export type MediaEntityKind = "movie" | "series" | "variety" | "collection" | "other"; export type MediaEntityStatus = "provisional" | "active" | "merged" | "archived"; -export type MediaScrapeStatus = "queued" | "running" | "succeeded" | "partial" | "failed"; -export type MediaScrapeTrigger = "auto" | "manual"; -export type MediaScrapeMode = "missing" | "overwrite"; export type MediaServerInstanceKind = "emby" | "jellyfin"; export type MediaServerInstanceHealthStatus = "unknown" | "healthy" | "degraded" | "offline"; export type MetadataEnrichmentStatus = "suggested" | "applied" | "failed"; @@ -226,6 +223,9 @@ export type ScheduledJobKind = "task" | "agent_check"; export type ScheduledJobLastStatus = "ok" | "failed" | "skipped" | "running"; export type ScheduledJobRunTrigger = "schedule" | "manual" | "skip"; export type ScheduledJobRunStatus = "ok" | "failed" | "skipped" | "running"; +export type ScrapeRecordStatus = "queued" | "running" | "succeeded" | "partial" | "failed"; +export type ScrapeRecordTrigger = "auto" | "manual"; +export type ScrapeRecordMode = "missing" | "overwrite"; export type SecretPurpose = | "site_cookie" | "site_passkey" @@ -918,25 +918,6 @@ export interface MediaEntityTable { naming_override: Json | null; } -export interface MediaScrapeTable { - media_entity_id: string; - status: Generated; - trigger: Generated; - mode: Generated; - record_id: string | null; - engine_run_id: string | null; - requested_at: GeneratedTimestamptz; - started_at: Timestamptz | null; - finished_at: Timestamptz | null; - written: Generated; - skipped: Generated; - failed: Generated; - reason: string | null; - files: Generated; - created_at: GeneratedTimestamptz; - updated_at: GeneratedTimestamptz; -} - export interface MediaServerInstanceTable { id: Generated; name: string; @@ -1345,6 +1326,33 @@ export interface ScheduledJobRunTable { created_at: GeneratedTimestamptz; } +export interface ScrapeRecordTable { + id: Generated; + root_id: string; + root_path: string; + library_root_id: string | null; + path: Generated; + media_entity_id: string | null; + match: Json | null; + status: Generated; + trigger: Generated; + mode: Generated; + organization_record_id: string | null; + engine_run_id: string | null; + actor_type: string | null; + actor_id: string | null; + requested_at: GeneratedTimestamptz; + started_at: Timestamptz | null; + finished_at: Timestamptz | null; + written: Generated; + skipped: Generated; + failed: Generated; + reason: string | null; + files: Generated; + created_at: GeneratedTimestamptz; + updated_at: GeneratedTimestamptz; +} + export interface SecretTable { id: Generated; name: string; @@ -1720,7 +1728,6 @@ export interface DB { mcp_server_config: McpServerConfigTable; media_asset: MediaAssetTable; media_entity: MediaEntityTable; - media_scrape: MediaScrapeTable; media_server_instance: MediaServerInstanceTable; metadata_enrichment: MetadataEnrichmentTable; metadata_field: MetadataFieldTable; @@ -1744,6 +1751,7 @@ export interface DB { rule_group: RuleGroupTable; scheduled_job: ScheduledJobTable; scheduled_job_run: ScheduledJobRunTable; + scrape_record: ScrapeRecordTable; secret: SecretTable; server_mapping: ServerMappingTable; settings: SettingsTable; @@ -1881,9 +1889,6 @@ export type MediaAssetUpdate = Updateable; export type MediaEntityRow = Selectable; export type NewMediaEntity = Insertable; export type MediaEntityUpdate = Updateable; -export type MediaScrapeRow = Selectable; -export type NewMediaScrape = Insertable; -export type MediaScrapeUpdate = Updateable; export type MediaServerInstanceRow = Selectable; export type NewMediaServerInstance = Insertable; export type MediaServerInstanceUpdate = Updateable; @@ -1953,6 +1958,9 @@ export type ScheduledJobUpdate = Updateable; export type ScheduledJobRunRow = Selectable; export type NewScheduledJobRun = Insertable; export type ScheduledJobRunUpdate = Updateable; +export type ScrapeRecordRow = Selectable; +export type NewScrapeRecord = Insertable; +export type ScrapeRecordUpdate = Updateable; export type SecretRow = Selectable; export type NewSecret = Insertable; export type SecretUpdate = Updateable; diff --git a/packages/db/test/schema.test.ts b/packages/db/test/schema.test.ts index b75fd7c7..94026b56 100644 --- a/packages/db/test/schema.test.ts +++ b/packages/db/test/schema.test.ts @@ -26,7 +26,6 @@ const EXPECTED_TABLES = [ "path_mapping", // 内容 / 身份 "media_entity", - "media_scrape", "content_unit", "title_alias", "external_identity", @@ -124,6 +123,8 @@ const EXPECTED_TABLES = [ "kv_cache", // 站点请求计数(0046,按小时聚合) "site_request_count", + // SCR-2 刮削记录(0058,替代 0057 media_scrape) + "scrape_record", ].sort(); /** 不走 entityTable() 的表:追加型/引擎自有表,没有 uuid id + created_at/updated_at 三件套 */ @@ -154,8 +155,6 @@ const NON_ENTITY_TABLES = [ "kv_cache", // 站点请求计数:(site_id, hour) 主键 "site_request_count", - // 作品最近一次刮削:media_entity_id 主键(SCR-1) - "media_scrape", ]; describe.skipIf(skip)("schema migrations (real PG)", () => { diff --git a/packages/domain/src/scrape/layout.ts b/packages/domain/src/scrape/layout.ts index b576a083..0b07f80b 100644 --- a/packages/domain/src/scrape/layout.ts +++ b/packages/domain/src/scrape/layout.ts @@ -61,6 +61,11 @@ export interface ScrapeLayoutInput { videos: readonly ScrapeVideo[]; images: ScrapeImages; options: { images: boolean; episodeThumbs: boolean }; + /** + * SCR-2 就地刮削的范围(对照 MP scrape_metadata 的 init_folder):work = 作品级(剧集 tvshow.nfo / 海报 / 背景图, + * 电影海报 / 背景图);season = 季级(season.nfo / 季海报)。缺省都写。选中单集文件时两者都关,选中季目录时只关 work。 + */ + parts?: { work?: boolean; season?: boolean }; } export interface ScrapeTarget { @@ -76,7 +81,18 @@ export interface ScrapeTarget { imagePath?: string; } -const SEASON_DIR_RE = /^(season\s*\d+|s\d{1,3}|specials?|特别篇|第\s*\d+\s*季)$/i; +const SEASON_DIR_RE = /^(season\s*\d+|s\d{1,3}|specials?|sps?|特别篇|第\s*\d+\s*季)$/i; + +/** 目录名像季目录(Season 1 / S01 / Specials / 第 1 季);对照 MP season_zero_names 与 MetaInfo(dir).begin_season */ +export const isSeasonDirName = (name: string): boolean => SEASON_DIR_RE.test(name.trim()); + +/** 季目录名 → 季号(Specials / SPs / 特别篇 = 0);认不出为 null */ +export function seasonOfDirName(name: string): number | null { + const n = name.trim(); + if (/^(specials?|sps?|特别篇)$/i.test(n)) return 0; + const m = /^(?:season\s*|s)(\d{1,4})$/i.exec(n) ?? /^第\s*(\d{1,4})\s*季$/.exec(n); + return m ? Number(m[1]) : null; +} /** 没有作品绑定时从视频路径推作品目录:电影取父目录;剧集父目录像季目录时取祖父目录 */ export function inferWorkDir(kind: "movie" | "tv", videoPath: string): string | null { @@ -127,7 +143,7 @@ export function planScrapeTargets(input: ScrapeLayoutInput): ScrapeTarget[] { const dir = normDir(posix.dirname(v.path)); add({ path: join(dir, `${stemOf(v.path)}.nfo`), kind: "nfo", role: "movie_nfo", video: v.path }); } - if (!input.options.images) return out; + if (!input.options.images || input.parts?.work === false) return out; const inWork = workDir && videos.some((v) => isUnder(v.path, workDir)); const art = (role: "poster" | "fanart", p: string | null | undefined) => { if (!p) return; @@ -150,8 +166,10 @@ export function planScrapeTargets(input: ScrapeLayoutInput): ScrapeTarget[] { } // 剧集 + const wantWork = input.parts?.work !== false; + const wantSeason = input.parts?.season !== false; const underWork = workDir ? videos.filter((v) => isUnder(v.path, workDir)) : []; - if (workDir && underWork.length) { + if (workDir && underWork.length && wantWork) { add({ path: join(workDir, "tvshow.nfo"), kind: "nfo", role: "tvshow_nfo" }); if (input.options.images) { if (img.posterPath) @@ -169,9 +187,11 @@ export function planScrapeTargets(input: ScrapeLayoutInput): ScrapeTarget[] { imagePath: img.backdropPath, }); } - // 季:按视频所在目录(不等于作品目录的才算季目录) + } + // 季:按视频所在目录(不等于作品目录的才算季目录);没有作品目录时只写季目录里的 season.nfo / poster + if (wantSeason) { const seasons = new Map>(); - for (const v of underWork) { + for (const v of workDir ? underWork : videos) { if (v.season === null || v.season === undefined) continue; const dir = normDir(posix.dirname(v.path)); const set = seasons.get(v.season) ?? new Set(); @@ -182,13 +202,14 @@ export function planScrapeTargets(input: ScrapeLayoutInput): ScrapeTarget[] { for (const d of [...dirs].sort()) add({ path: `${d}/season.nfo`, kind: "nfo", role: "season_nfo", season }); const sp = img.seasonPosters?.get(season); if (input.options.images && sp) { - add({ - path: join(workDir, seasonPosterName(season, extOf(sp))), - kind: "image", - role: "season_poster", - season, - imagePath: sp, - }); + if (workDir) + add({ + path: join(workDir, seasonPosterName(season, extOf(sp))), + kind: "image", + role: "season_poster", + season, + imagePath: sp, + }); for (const d of [...dirs].sort()) add({ path: `${d}/poster${extOf(sp)}`, kind: "image", role: "season_poster", season, imagePath: sp }); } diff --git a/packages/domain/test/scrape.test.ts b/packages/domain/test/scrape.test.ts index f5fcd31b..7b303e91 100644 --- a/packages/domain/test/scrape.test.ts +++ b/packages/domain/test/scrape.test.ts @@ -1,5 +1,14 @@ import { describe, expect, it } from "vitest"; -import { episodeNfo, inferWorkDir, planScrapeTargets, seasonNfo, seasonPosterName, workNfo } from "../src/index.ts"; +import { + episodeNfo, + inferWorkDir, + isSeasonDirName, + planScrapeTargets, + seasonNfo, + seasonOfDirName, + seasonPosterName, + workNfo, +} from "../src/index.ts"; describe("NFO 生成", () => { it("电影:MP 元素顺序,缺值不写,IMDb 为 default", () => { @@ -198,4 +207,43 @@ describe("刮削目标布局", () => { }); expect(t.map((x) => x.path)).toEqual(["a.S01E01.nfo", "a.S01E01-thumb.jpg"]); }); + + it("SCR-2 季目录名:Season 1 / S01 / Specials / SPs / 第 1 季", () => { + for (const n of ["Season 1", "season 12", "S01", "Specials", "SPs", "特别篇", "第 2 季"]) + expect(isSeasonDirName(n)).toBe(true); + for (const n of ["人生切割术 (2022)", "Extras", "Season"]) expect(isSeasonDirName(n)).toBe(false); + expect(seasonOfDirName("Season 3")).toBe(3); + expect(seasonOfDirName("S01")).toBe(1); + expect(seasonOfDirName("Specials")).toBe(0); + expect(seasonOfDirName("第 2 季")).toBe(2); + expect(seasonOfDirName("Extras")).toBeNull(); + }); + + it("SCR-2 范围:选中季目录只写季级与各集(季海报仍写上级),选中单集只写该集", () => { + const videos = [{ path: "人生切割术 (2022)/Season 1/a.S01E01.mkv", season: 1, episode: 1 }]; + const season = planScrapeTargets({ + kind: "tv", + workDir: "人生切割术 (2022)", + videos, + images, + options: { images: true, episodeThumbs: true }, + parts: { work: false }, + }); + expect(season.map((x) => `${x.role}:${x.path}`)).toEqual([ + "season_nfo:人生切割术 (2022)/Season 1/season.nfo", + "season_poster:人生切割术 (2022)/season01-poster.jpg", + "season_poster:人生切割术 (2022)/Season 1/poster.jpg", + "episode_nfo:人生切割术 (2022)/Season 1/a.S01E01.nfo", + "episode_thumb:人生切割术 (2022)/Season 1/a.S01E01-thumb.jpg", + ]); + const ep = planScrapeTargets({ + kind: "tv", + workDir: null, + videos, + images, + options: { images: true, episodeThumbs: true }, + parts: { work: false, season: false }, + }); + expect(ep.map((x) => x.role)).toEqual(["episode_nfo", "episode_thumb"]); + }); });