From 89aa70bb9cc744aebb06b095533ddf8302ffd3bb Mon Sep 17 00:00:00 2001 From: root Date: Sat, 3 Oct 2026 23:21:52 +0800 Subject: [PATCH] =?UTF-8?q?fix(scrape):=20=E5=B9=B3=E9=93=BA=E7=9B=AE?= =?UTF-8?q?=E5=BD=95=E5=8F=AA=E5=88=AE=E6=9C=AC=E4=BD=9C=E5=93=81=E8=A7=86?= =?UTF-8?q?=E9=A2=91=E3=80=81=E5=9B=BE=E7=89=87=E6=8C=89=E8=A7=86=E9=A2=91?= =?UTF-8?q?=E5=90=8D=E5=91=BD=E5=90=8D;=E5=9B=BE=E7=89=87=E6=89=A9?= =?UTF-8?q?=E5=B1=95=E5=90=8D=E6=8C=89=E5=AE=9E=E9=99=85=E7=B1=BB=E5=9E=8B?= =?UTF-8?q?;=E5=8F=96=E6=B6=88=E6=97=B6=E8=AE=B0=E5=BD=95=E4=B8=8D?= =?UTF-8?q?=E5=81=9C=E5=9C=A8=E8=BF=9B=E8=A1=8C=E4=B8=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 作品重刮对电影传入该作品已入库的视频,不给同目录别的电影写本作品 NFO; - 电影目录里有别的作品的视频时海报 / 背景图写 <视频名>-poster / -fanart,不写目录共用的 poster.jpg; - 图片扩展名按下载到的 MIME,URL 认不出或与内容不符时以内容为准; - 任务被取消时记录收尾为失败 scrape_canceled,保留已写文件结果。 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei --- .../acquisition-organize/services/scrape.ts | 126 ++++++++++++++++-- .../scrape-episodes.test.ts | 96 ++++++++++++- docs/runbook/scrape.md | 7 +- packages/domain/src/scrape/layout.ts | 9 +- 4 files changed, 220 insertions(+), 18 deletions(-) diff --git a/apps/server/src/modules/acquisition-organize/services/scrape.ts b/apps/server/src/modules/acquisition-organize/services/scrape.ts index c9d17414..cd51eaeb 100644 --- a/apps/server/src/modules/acquisition-organize/services/scrape.ts +++ b/apps/server/src/modules/acquisition-organize/services/scrape.ts @@ -81,6 +81,8 @@ export interface ScrapeLocation { rootPath: string; libraryRootId: string | null; path: string; + /** 只刮这些视频(作品重刮:电影目录里属于本作品的视频);缺省 = 目录下全部 */ + videos?: readonly string[] | undefined; } export interface QueueScrapeInput extends ScrapeLocation { @@ -341,6 +343,8 @@ export async function entityScrapeLocations( .where("status", "=", "active") .execute(); const out = new Map(); + /** 电影:每个目录里属于本作品的视频(重刮只刮这些,同目录别的电影不写本作品的 NFO);有原盘目录的不限定 */ + const own = new Map(); for (const l of locs) { const bound = bindings.find( (b) => @@ -358,9 +362,26 @@ export async function entityScrapeLocations( } // 父目录已在列表里的子路径去掉,避免同一处刮两次 const all = [...out.values()]; - return all.filter( + const kept = all.filter( (x) => !all.some((y) => y !== x && y.rootId === x.rootId && !!y.path && x.path.startsWith(`${y.path}/`)), ); + if (kind !== "movie") return kept; + for (const l of locs) { + const loc = kept.find( + (x) => + x.libraryRootId === l.root_id && + (!x.path || l.relative_path === x.path || l.relative_path.startsWith(`${x.path}/`)), + ); + if (!loc) continue; + const key = `${loc.rootId}\u0000${loc.path}`; + if (l.kind !== "video") own.set(key, null); + else if (own.get(key) !== null) own.set(key, [...(own.get(key) ?? []), l.relative_path]); + } + return kept.map((x) => { + const videos = own.get(`${x.rootId}\u0000${x.path}`); + // 选中的就是视频文件本身时不用限定 + return videos?.length && !(videos.length === 1 && videos[0] === x.path) ? { ...x, videos } : x; + }); } /** 对作品的目录刮削(每个目录一条记录);没有已入库文件时返回空 */ @@ -611,6 +632,8 @@ async function listDir(rootReal: string, rel: string) { interface Layout { videos: string[]; + /** 选中目录里的全部视频(不进子目录);判断电影目录里有没有别的作品 */ + dirVideos: string[]; workDir: string | null; parts: { work: boolean; season: boolean }; /** 季目录刮削时目录名给出的季号 */ @@ -623,18 +646,31 @@ async function layoutOf(rootReal: string, rel: string, isDir: boolean, kind: "mo const all = { work: true, season: true }; if (!isDir) return kind === "movie" - ? { videos: [rel], workDir: parent, parts: all, dirSeason: null } - : { videos: [rel], workDir: null, parts: { work: false, season: false }, dirSeason: null }; + ? { + videos: [rel], + dirVideos: (await listDir(rootReal, parent ?? "")).videos, + workDir: parent, + parts: all, + dirSeason: null, + } + : { videos: [rel], dirVideos: [], workDir: null, parts: { work: false, season: false }, dirSeason: null }; const here = await listDir(rootReal, rel); if (kind === "movie") { // 蓝光原盘目录:写 <目录名>.nfo(借一个同名的虚拟视频路径让布局按目录名命名)+ 目录海报 if (rel && here.dirs.some((n) => /^(BDMV|VIDEO_TS)$/i.test(n))) - return { videos: [joinRel(rel, `${baseOf(rel)}.bluray`)], workDir: rel, parts: all, dirSeason: null }; - return { videos: here.videos, workDir: rel || null, parts: all, dirSeason: null }; + return { + videos: [joinRel(rel, `${baseOf(rel)}.bluray`)], + dirVideos: [], + workDir: rel, + parts: all, + dirSeason: null, + }; + return { videos: here.videos, dirVideos: here.videos, workDir: rel || null, parts: all, dirSeason: null }; } if (rel && isSeasonDirName(baseOf(rel))) return { videos: here.videos, + dirVideos: here.videos, workDir: parent, parts: { work: false, season: true }, dirSeason: seasonOfDirName(baseOf(rel)), @@ -642,7 +678,7 @@ async function layoutOf(rootReal: string, rel: string, isDir: boolean, kind: "mo const videos = [...here.videos]; for (const sub of here.dirs) if (isSeasonDirName(sub)) videos.push(...(await listDir(rootReal, joinRel(rel, sub))).videos); - return { videos, workDir: rel || null, parts: all, dirSeason: null }; + return { videos, dirVideos: here.videos, workDir: rel || null, parts: all, dirSeason: null }; } /** 已登记视频的季集:覆盖单元里最小的一集为首集,同一季覆盖的其余集一并列出(多集文件每集一个 NFO 块) */ @@ -724,8 +760,53 @@ function episodeByAirDate( return hits.length === 1 ? hits[0]! : null; } +/** 图片 MIME → 扩展名;认不出返回 null(沿用按 URL 定的扩展名) */ +function imageExtOf(mime: string | null | undefined): string | null { + const m = (mime ?? "").split(";")[0]!.trim().toLowerCase(); + return m === "image/jpeg" || m === "image/jpg" + ? ".jpg" + : m === "image/png" + ? ".png" + : m === "image/webp" + ? ".webp" + : null; +} + /** 预期内的跳过(不影响记录状态):已存在、内容没变、被人改过 */ const EXPECTED_SKIPS = new Set(["scrape_exists", "scrape_unchanged", "scrape_user_modified"]); +/** + * 电影目录里有没有别的作品:目录里的视频登记在别的作品下,或(只刮部分视频时)有没登记到本作品、也不在本次范围里的视频。 + * 同一作品的多个版本不算。 + */ +async function movieDirShared( + d: AoDeps, + rec: { library_root_id: string | null; videos: string[] | null }, + lay: Layout, + entityId: string | null, +): Promise { + const others = lay.dirVideos.filter((v) => !lay.videos.includes(v)); + if (!lay.dirVideos.length) return false; + const owners = new Map>(); + if (rec.library_root_id) { + const rows = await d.ctx.db + .selectFrom("asset_location as l") + .innerJoin("media_asset as a", "a.id", "l.media_asset_id") + .select(["l.relative_path", "a.media_entity_id"]) + .where("l.library_root_id", "=", rec.library_root_id) + .where("l.relative_path", "in", lay.dirVideos) + .where("l.status", "<>", "removed") + .where("a.status", "=", "active") + .execute(); + for (const r of rows) + owners.set(r.relative_path, (owners.get(r.relative_path) ?? new Set()).add(r.media_entity_id)); + } + return lay.dirVideos.some((v) => { + const own = owners.get(v); + if (own && [...own].some((e) => e !== entityId)) return true; + return !own?.size && others.includes(v); + }); +} + const sha256 = (c: Uint8Array | string) => createHash("sha256").update(c).digest("hex"); /** 旧版本写的文件没有 hash:修改时间不晚于那次刮削结束(加这点余量)就当没被改过 */ const LEGACY_MTIME_SLACK_MS = 2_000; @@ -937,6 +1018,9 @@ export async function runScrapeRecord( } } + // 电影目录里还有别的作品(自定义平铺模板):图片按视频名命名,不写目录共用的 poster / fanart + const sharedDir = kind === "movie" && (await movieDirShared(d, rec, lay, match.mediaEntityId)); + // 季 / 集号:已登记的按覆盖单元;其余按路径解析(季目录名兜底,同 MP 对每集重新 MetaInfoPath) const videos: ScrapeVideo[] = []; /** 集数待确认(不写分集 NFO,原因 episode_unresolved) */ @@ -1063,14 +1147,15 @@ export async function runScrapeRecord( images: { ...imagePathsOf(fields.entity.metadata), seasonPosters, stills }, options: { images: settings.scrapeImages, episodeThumbs: settings.scrapeEpisodeThumbs }, parts: lay.parts, + sharedDir, }); // 「更新」模式:本工具以前写过的文件(路径 → 写入时的 hash / 时间) const ours = mode === "update" ? await previousWrites(d, rec.root_id, recordId) : new Map(); - const imageCache = new Map>(); + const imageCache = new Map>(); const fetchImage = (p: string) => { let hit = imageCache.get(p); if (!hit) { - hit = d.ports.scrapeImage(p).then((r) => r.content); + hit = d.ports.scrapeImage(p); imageCache.set(p, hit); } return hit; @@ -1147,18 +1232,27 @@ export async function runScrapeRecord( // 图片:限并发,失败不影响 NFO const imgs = targets.filter((x) => x.kind === "image" && x.imagePath); files.push( - ...(await pool(imgs, IMAGE_CONCURRENCY, async (t): Promise => { + ...(await pool(imgs, IMAGE_CONCURRENCY, async (planned0): Promise => { if (signal?.aborted) throw new Error("aborted"); - const dec = await decide(t); + let t = planned0; + let dec = await decide(t); if (!("abs" in dec)) return dec; - let bytes: Uint8Array; + let got: { content: Uint8Array; mime: string }; try { - bytes = await fetchImage(t.imagePath!); + got = await fetchImage(t.imagePath!); } catch (err) { d.ctx.logger.warn({ recordId, image: t.imagePath, err: { message: errMessage(err) } }, "刮削图片下载失败"); return { path: t.path, kind: t.kind, role: t.role, action: "failed", reason: "scrape_image_failed" }; } - return write(t, dec, bytes); + // 扩展名按实际类型(URL 认不出或与内容不符时,例如 Local 封面链接是 webp、内容是 jpeg) + const ext = imageExtOf(got.mime); + const planned = /\.(jpe?g|png|webp)$/i.exec(t.path)?.[0]?.toLowerCase(); + if (ext && planned && ext !== planned && !(ext === ".jpg" && planned === ".jpeg")) { + t = { ...t, path: t.path.slice(0, -planned.length) + ext }; + dec = await decide(t); + if (!("abs" in dec)) return dec; + } + return write(t, dec, got.content); })), ); @@ -1185,7 +1279,11 @@ export async function runScrapeRecord( files, }); } catch (err) { - if (signal?.aborted) throw err; + if (signal?.aborted) { + // 任务被取消(引擎不会再跑这一步):记录收尾为失败,已写的文件结果照留,不停在进行中 + await finish({ status: "failed", reason: "scrape_canceled", files }).catch(() => {}); + throw err; + } d.ctx.logger.warn({ recordId, err: { message: errMessage(err) } }, "刮削失败"); // 不是写文件的错(如查库失败):记内部错误,已写的文件结果照留 const anyWritten = files.some((f) => f.action === "written"); diff --git a/apps/server/test/acquisition-organize/scrape-episodes.test.ts b/apps/server/test/acquisition-organize/scrape-episodes.test.ts index e0a2ffad..1be7bc5b 100644 --- a/apps/server/test/acquisition-organize/scrape-episodes.test.ts +++ b/apps/server/test/acquisition-organize/scrape-episodes.test.ts @@ -50,7 +50,7 @@ describe.skipIf(!hasPg)("刮削:多集文件、综艺日期期数、平铺目录 let lib: TempTree; let rootId = ""; let seasonDetail: (n: number) => Promise = async () => null; - const imageMime = "image/jpeg"; + let imageMime = "image/jpeg"; beforeAll(async () => { lib = await createTempTree( @@ -280,4 +280,98 @@ describe.skipIf(!hasPg)("刮削:多集文件、综艺日期期数、平铺目录 expect(nfo).toContain("5"); expect(r.status).toBe("succeeded"); }); + + async function registerVideo(entityId: string, path: string) { + const a = await h.db + .insertInto("media_asset") + .values({ media_entity_id: entityId, kind: "video", status: "active" }) + .returning("id") + .executeTakeFirstOrThrow(); + await h.db + .insertInto("asset_location") + .values({ + media_asset_id: a.id, + location_kind: "library", + library_root_id: rootId, + relative_path: path, + status: "available", + }) + .execute(); + } + + it("平铺模板:作品重刮只刮本作品的视频,图片按 <视频名>-poster 命名,不写目录共用的 poster", async () => { + const meta = (t: string) => ({ title: f(t), posterPath: f(`/${t}.jpg`), backdropPath: f(`/${t}-bg.jpg`) }); + const inception = await entity("movie", "盗梦空间", meta("盗梦空间"), { id: "27205", type: "movie" }); + const interstellar = await entity("movie", "星际穿越", meta("星际穿越"), { id: "157336", type: "movie" }); + await registerVideo(inception, "Movies/盗梦空间 (2010).mkv"); + await registerVideo(interstellar, "Movies/星际穿越 (2014).mkv"); + const q = await scrapeEntity(h.d, inception, { trigger: "manual", mode: "overwrite" }); + expect(q).toHaveLength(1); + const r = await runScrapeRecord(h.d, q[0]!.recordId); + expect(r.status).toBe("succeeded"); + expect(r.videos).toEqual(["Movies/盗梦空间 (2010).mkv"]); + expect(r.files.map((x) => x.path).sort()).toEqual([ + "Movies/盗梦空间 (2010)-fanart.jpg", + "Movies/盗梦空间 (2010)-poster.jpg", + "Movies/盗梦空间 (2010).nfo", + ]); + await expect(stat(lib.path("Movies/星际穿越 (2014).nfo"))).rejects.toThrow(); + await expect(stat(lib.path("Movies/poster.jpg"))).rejects.toThrow(); + expect(await readFile(lib.path("Movies/盗梦空间 (2010).nfo"), "utf8")).toContain("盗梦空间"); + }); + + it("单部电影的目录照旧写 poster / fanart;图片扩展名按实际类型", async () => { + const avatar = await entity( + "movie", + "阿凡达", + { title: f("阿凡达"), posterPath: f("https://img.example.test/avatar.webp") }, + { id: "19995", type: "movie" }, + ); + await registerVideo(avatar, "Single/阿凡达 (2009)/阿凡达 (2009).mkv"); + imageMime = "image/png"; + try { + const q = await scrapeEntity(h.d, avatar, { trigger: "manual", mode: "overwrite" }); + const r = await runScrapeRecord(h.d, q[0]!.recordId); + expect(r.status).toBe("succeeded"); + expect(r.files.find((x) => x.role === "poster")).toMatchObject({ + path: "Single/阿凡达 (2009)/poster.png", + action: "written", + }); + await expect(stat(lib.path("Single/阿凡达 (2009)/poster.webp"))).rejects.toThrow(); + expect(new Uint8Array(await readFile(lib.path("Single/阿凡达 (2009)/poster.png")))).toEqual(PNG); + } finally { + imageMime = "image/jpeg"; + } + }); + + it("任务被取消:记录收尾为失败(scrape_canceled),不停在进行中", async () => { + const id = await entity("movie", "取消", { title: f("取消") }, { id: "1", type: "movie" }); + const match: ScrapeMatch = { + mediaEntityId: id, + provider: "tmdb", + externalId: "1", + kind: "movie", + title: "取消", + year: null, + source: "library", + }; + const q = await queueScrape(h.d, { + rootId: `lib:${rootId}`, + rootPath: lib.root, + libraryRootId: rootId, + path: "Single/阿凡达 (2009)", + match, + trigger: "manual", + }); + const ac = new AbortController(); + ac.abort(); + await expect(runScrapeRecord(h.d, q.recordId, ac.signal)).rejects.toThrow(); + const row = await h.db + .selectFrom("scrape_record") + .select(["status", "reason", "finished_at"]) + .where("id", "=", q.recordId) + .executeTakeFirstOrThrow(); + expect(row).toMatchObject({ status: "failed", reason: "scrape_canceled" }); + expect(row.finished_at).not.toBeNull(); + }); }); diff --git a/docs/runbook/scrape.md b/docs/runbook/scrape.md index 7b2118e6..0afa2b30 100644 --- a/docs/runbook/scrape.md +++ b/docs/runbook/scrape.md @@ -114,6 +114,11 @@ TMDB 有这一集但没有译名 / 简介(只有占位名和播出日期)不算 | 电影目录 | 目录里每个视频 `<视频名>.nfo`(不进子目录)+ 目录 `poster.jpg` / `fanart.jpg`;蓝光原盘目录(含 `BDMV` / `VIDEO_TS`)写 `<目录名>.nfo` | | 电影文件 | `<视频名>.nfo` + 同目录 `poster.jpg` / `fanart.jpg`;文件直接在登记目录根时图片改为 `<视频名>-poster.jpg` / `-fanart.jpg` | +- 自定义平铺模板(多部电影放在同一目录):目录里有登记在别的作品下的视频,或只刮部分视频时目录里还有没登记到本作品的视频, + 图片改为 `<视频名>-poster.jpg` / `-fanart.jpg`,不写整个目录共用的 `poster.jpg`。作品详情「重新刮削」对电影只刮该作品已入库的视频, + 不给同目录别的电影写本作品的 NFO(同一作品的多个版本照旧共用一张 poster)。 +- 图片扩展名按下载到的实际类型(jpeg / png / webp),URL 认不出或与内容不符时以内容为准。 + - 集号:已登记视频按覆盖的集单元;其余按路径解析(季号缺失时取季目录名,再缺按第 1 季;同 MP 对每集重新 `MetaInfoPath`)。 认不出集号的视频记 `scrape_no_episode` 跳过,记录算部分完成。 - 多集文件(文件名 `E01-E02`,或已登记文件覆盖多个集单元):一个 `<视频名>.nfo` 里按 Kodi 约定每集写一个 `` 块 @@ -232,7 +237,7 @@ cd /opt/uhd_media_tool && deno run -A npm:@biomejs/biome check --write apps/serv - 原因码(`packages/contracts/src/reasons.ts`):`scrape_no_metadata`、`scrape_root_read_only`、`scrape_root_offline`、`scrape_not_video`、 `scrape_no_videos`、`scrape_path_missing`、`scrape_tmdb_unavailable`、`scrape_no_episode`、`scrape_exists`、`scrape_unchanged`、 `scrape_user_modified`、`scrape_metadata_missing`、`scrape_write_failed`、`scrape_image_failed`、`scrape_internal_error`(查库等非写文件错误, - 已写的文件结果保留)、`path_escape`、`task_enqueue_failed`。 + 已写的文件结果保留)、`scrape_canceled`(任务被取消,记录收尾为失败,已写的文件结果保留)、`scrape_not_library`、`path_escape`、`task_enqueue_failed`。 | 现象 | 看哪里 | | --- | --- | diff --git a/packages/domain/src/scrape/layout.ts b/packages/domain/src/scrape/layout.ts index 763d6574..1c51ccc5 100644 --- a/packages/domain/src/scrape/layout.ts +++ b/packages/domain/src/scrape/layout.ts @@ -7,7 +7,7 @@ // - 剧集:作品目录 tvshow.nfo、poster.jpg、fanart.jpg;季目录 season.nfo;季海报同时写作品目录 // `season01-poster.jpg`(特别篇 `season-specials-poster.jpg`)与季目录 `poster.jpg`(MP 同); // - 每集:`<视频名>.nfo`;剧照 `<视频名>-thumb.jpg`(Kodi / Jellyfin / Emby 都认;MP 写 `<视频名>.jpg`)。 -// 视频直接在库根(没有作品目录)时,电影图片改用 `<视频名>-poster.jpg` / `<视频名>-fanart.jpg`, +// 视频直接在库根(没有作品目录)或电影目录里还有别的作品时,电影图片改用 `<视频名>-poster.jpg` / `<视频名>-fanart.jpg`, // 剧集不写 tvshow / 季文件(不能往库根里放 tvshow.nfo)。 /** @@ -79,6 +79,11 @@ export interface ScrapeLayoutInput { * 电影海报 / 背景图);season = 季级(season.nfo / 季海报)。缺省都写。选中单集文件时两者都关,选中季目录时只关 work。 */ parts?: { work?: boolean; season?: boolean }; + /** + * 电影:作品目录里还有别的作品的视频(自定义平铺模板把多部电影放在同一目录)。图片改用 `<视频名>-poster` / + * `<视频名>-fanart`,不写整个目录共用的 poster.jpg / fanart.jpg。 + */ + sharedDir?: boolean; } export interface ScrapeTarget { @@ -159,7 +164,7 @@ export function planScrapeTargets(input: ScrapeLayoutInput): ScrapeTarget[] { add({ path: join(dir, `${stemOf(v.path)}.nfo`), kind: "nfo", role: "movie_nfo", video: v.path }); } if (!input.options.images || input.parts?.work === false) return out; - const inWork = workDir && videos.some((v) => isUnder(v.path, workDir)); + const inWork = workDir && !input.sharedDir && videos.some((v) => isUnder(v.path, workDir)); const art = (role: "poster" | "fanart", p: string | null | undefined) => { if (!p) return; if (inWork) add({ path: join(workDir, `${role}${extOf(p)}`), kind: "image", role, imagePath: p });