From 8c4c7d028a740cc9754273a610b9d5ef9850923d Mon Sep 17 00:00:00 2001 From: root Date: Sat, 3 Oct 2026 23:13:47 +0800 Subject: [PATCH] =?UTF-8?q?feat(scrape):=20=E5=A4=9A=E9=9B=86=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E5=86=99=E5=A4=9A=E4=B8=AA=20episodedetails=20?= =?UTF-8?q?=E5=9D=97,=E7=BB=BC=E8=89=BA=E6=97=A5=E6=9C=9F=E6=9C=9F?= =?UTF-8?q?=E6=95=B0=E6=8C=89=E6=92=AD=E5=87=BA=E6=97=A5=E6=9C=9F=E5=AF=B9?= =?UTF-8?q?=E9=9B=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 多集文件(文件名 E01-E02 或覆盖多个集单元)按 Kodi 约定在一个 NFO 里每集写一块,MP 只写首集,偏离登记在 nfo-deviations.json; - 综艺按日期命名的期数按本地季表或 TMDB 季详情的播出日期对上集,写季号、集号与 aired,不再 scrape_no_episode 跳过; - 认不出集号、集数待确认的跳过让记录记部分完成(已存在 / 没变 / 被人改过仍算完成)。 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei --- .../acquisition-organize/services/scrape.ts | 148 ++++++--- .../scrape-episodes.test.ts | 283 ++++++++++++++++++ apps/server/test/scrape/nfo-deviations.json | 8 +- apps/server/test/scrape/nfo-mp-diff.test.ts | 57 ++-- docs/runbook/scrape.md | 9 +- packages/domain/src/scrape/layout.ts | 12 +- packages/domain/src/scrape/nfo.ts | 22 +- packages/domain/test/scrape.test.ts | 35 +++ 8 files changed, 513 insertions(+), 61 deletions(-) create mode 100644 apps/server/test/acquisition-organize/scrape-episodes.test.ts diff --git a/apps/server/src/modules/acquisition-organize/services/scrape.ts b/apps/server/src/modules/acquisition-organize/services/scrape.ts index 35335ea7..e4df6c9b 100644 --- a/apps/server/src/modules/acquisition-organize/services/scrape.ts +++ b/apps/server/src/modules/acquisition-organize/services/scrape.ts @@ -11,10 +11,10 @@ import { type ScrapeRecord, } from "@umt/contracts"; import { - episodeNfo, fileKindOf, inferWorkDir, isSeasonDirName, + multiEpisodeNfo, parseReleasePath, planScrapeTargets, type ScrapeTarget, @@ -565,15 +565,23 @@ async function layoutOf(rootReal: string, rel: string, isDir: boolean, kind: "mo return { videos, workDir: rel || null, parts: all, dirSeason: null }; } -/** 已登记视频的季集(覆盖单元里最小的一集,多集文件按首集) */ +/** 已登记视频的季集:覆盖单元里最小的一集为首集,同一季覆盖的其余集一并列出(多集文件每集一个 NFO 块) */ +interface Covered { + season: number; + episode: number; + episodes: number[]; +} +/** 一个文件最多按这么多集写 NFO 块(整季合成一个文件的极端情况不至于写出巨型 NFO) */ +const MAX_EPISODES_PER_FILE = 100; + async function coverageOf( d: AoDeps, libraryRootId: string, entityId: string, videos: readonly string[], units: readonly { id: string; kind: string; season_number: number | null; episode_number: number | null }[], -): Promise> { - const out = new Map(); +): Promise> { + const out = new Map(); if (!videos.length) return out; const rows = await d.ctx.db .selectFrom("asset_location as l") @@ -586,16 +594,58 @@ async function coverageOf( .where("a.media_entity_id", "=", entityId) .execute(); const byId = new Map(units.map((u) => [u.id, u])); + const all = new Map(); for (const r of rows) { const u = byId.get(r.content_unit_id); if (u?.kind !== "episode" || u.season_number === null || u.episode_number === null) continue; - const cur = out.get(r.relative_path); - if (!cur || u.season_number < cur.season || (u.season_number === cur.season && u.episode_number < cur.episode)) - out.set(r.relative_path, { season: u.season_number, episode: u.episode_number }); + all.set(r.relative_path, [ + ...(all.get(r.relative_path) ?? []), + { season: u.season_number, episode: u.episode_number }, + ]); + } + for (const [path, list] of all) { + list.sort((a, b) => a.season - b.season || a.episode - b.episode); + const first = list[0]!; + const episodes = [...new Set(list.filter((x) => x.season === first.season).map((x) => x.episode))].slice( + 0, + MAX_EPISODES_PER_FILE, + ); + out.set(path, { season: first.season, episode: first.episode, episodes }); } return out; } +/** 集区间 → 集号列表(多集文件 E01-E02) */ +const rangeOf = (r: { start: number; end: number }): number[] => + Array.from({ length: Math.max(1, Math.min(r.end - r.start + 1, MAX_EPISODES_PER_FILE)) }, (_, i) => r.start + i); + +/** 播出日期统一成 YYYY-MM-DD(解析结果是 YYYYMMDD;单元列可能是 Date) */ +function isoDay(v: unknown): string | null { + if (v instanceof Date) return Number.isNaN(+v) ? null : v.toISOString().slice(0, 10); + if (typeof v !== "string") return null; + const m = /^(\d{4})-?(\d{2})-?(\d{2})/.exec(v.trim()); + return m ? `${m[1]}-${m[2]}-${m[3]}` : null; +} + +/** 按播出日期在集单元里找集(综艺按日期命名的期数):季已知时只看该季;多于一个候选时不猜 */ +function episodeByAirDate( + units: ReadonlyMap, + day: string, + season: number | null, +): { season: number; episode: number } | null { + const hits: { season: number; episode: number }[] = []; + for (const u of units.values()) { + if (u.season_number === null || u.episode_number === null) continue; + if (season !== null && u.season_number !== season) continue; + const m = (u.metadata ?? {}) as { airDate?: { value?: unknown } }; + if (isoDay(m.airDate?.value) === day || isoDay(u.air_date) === day) + hits.push({ season: u.season_number, episode: u.episode_number }); + } + return hits.length === 1 ? hits[0]! : null; +} + +/** 预期内的跳过(不影响记录状态):已存在、内容没变、被人改过 */ +const EXPECTED_SKIPS = new Set(["scrape_exists", "scrape_unchanged", "scrape_user_modified"]); const sha256 = (c: Uint8Array | string) => createHash("sha256").update(c).digest("hex"); /** 旧版本写的文件没有 hash:修改时间不晚于那次刮削结束(加这点余量)就当没被改过 */ const LEGACY_MTIME_SLACK_MS = 2_000; @@ -757,7 +807,7 @@ export async function runScrapeRecord( // ── 字段:本地作品(有元数据)优先 ── let fields: Fields | null = null; - let coverage = new Map(); + let coverage = new Map(); let mapEpisodes: EntityEpisodeMapper | null = null; if (match.mediaEntityId) { const e = await db @@ -809,9 +859,12 @@ export async function runScrapeRecord( // 季 / 集号:已登记的按覆盖单元;其余按路径解析(季目录名兜底,同 MP 对每集重新 MetaInfoPath) const videos: ScrapeVideo[] = []; - const noEpisode: ScrapeFileResult[] = []; + /** 集数待确认(不写分集 NFO,原因 episode_unresolved) */ + const unresolved = new Set(); + /** 只有播出日期、没有集号的视频(综艺按日期命名的期数):路径 → YYYY-MM-DD,季 / 集元数据到位后按日期对集 */ + const dated = new Map(); // 集数映射(批次 3):未登记的文件按本地剧集作品的季表 + 规则把文件名季集换成规范季集再对集元数据; - // 集数待确认的不写分集 NFO(原因 episode_unresolved) + // 综艺按播出日期对上季表里的集 for (const v of lay.videos) { if (kind === "movie") { videos.push({ path: v }); @@ -819,33 +872,29 @@ export async function runScrapeRecord( } const cov = coverage.get(v); if (cov) { - videos.push({ path: v, season: cov.season, episode: cov.episode }); + videos.push({ path: v, season: cov.season, episode: cov.episode, episodes: cov.episodes }); continue; } const p = parseReleasePath(v); let season = p.season ?? lay.dirSeason ?? seasonOfDirName(baseOf(dirOf(v))) ?? (p.episodes ? 1 : null); - let episode = p.episodes?.start ?? null; - if (mapEpisodes && p.episodes) { + let range = p.episodes ?? null; + if (mapEpisodes && (p.episodes || p.airDate)) { const m = mapEpisodes(p); if (m.resolution.method === "unresolved") { - noEpisode.push({ - path: v, - kind: "nfo", - role: "episode_nfo", - action: "skipped", - reason: "episode_unresolved", - }); + unresolved.add(v); videos.push({ path: v, season, episode: null }); continue; } if (m.resolution.canonical) { season = m.resolution.canonical.season; - episode = m.resolution.canonical.episodes.start; + range = m.resolution.canonical.episodes; } } - if (episode === null) - noEpisode.push({ path: v, kind: "nfo", role: "episode_nfo", action: "skipped", reason: "scrape_no_episode" }); - videos.push({ path: v, season, episode }); + if (!range && p.airDate) { + const day = isoDay(p.airDate); + if (day) dated.set(v, day); + } + videos.push({ path: v, season, episode: range?.start ?? null, ...(range ? { episodes: rangeOf(range) } : {}) }); } // ── 没有本地字段:按 TMDB 条目直接取 ── @@ -868,6 +917,21 @@ export async function runScrapeRecord( // ── 季 / 集元数据:TMDB 分支取每个涉及的季;本地分支只在季 / 集单元缺元数据时按 TMDB id 现取补齐 ── const { seasonUnit, episodeUnit } = fields; + const epsOf = (v: ScrapeVideo) => + v.episodes?.length ? v.episodes : typeof v.episode === "number" ? [v.episode] : []; + /** 只有播出日期的视频按日期对集(本地集单元或刚取到的 TMDB 季详情) */ + const resolveDated = () => { + for (const v of videos) { + const day = dated.get(v.path); + if (!day || typeof v.episode === "number") continue; + const hit = episodeByAirDate(episodeUnit, day, v.season ?? null); + if (!hit) continue; + v.season = hit.season; + v.episode = hit.episode; + v.episodes = [hit.episode]; + } + }; + resolveDated(); if (kind === "tv") { const tmdbId = match.externalId ?? fields.ids.tmdb ?? null; const seasons = [...new Set(videos.map((v) => v.season).filter((s): s is number => typeof s === "number"))]; @@ -877,8 +941,8 @@ export async function runScrapeRecord( videos.some( (v) => v.season === n && - typeof v.episode === "number" && - episodeMetadataMissing(episodeUnit.get(`${n}:${v.episode}`)), + (epsOf(v).some((ep) => episodeMetadataMissing(episodeUnit.get(`${n}:${ep}`))) || + (typeof v.episode !== "number" && dated.has(v.path))), ); if (!tmdbId || (!fromTmdb && !lacking)) continue; const sd = await d.ports.scrapeTmdbSeason(tmdbId, n).catch((err) => { @@ -894,6 +958,17 @@ export async function runScrapeRecord( if (fromTmdb || episodeMetadataMissing(episodeUnit.get(k))) episodeUnit.set(k, eu); } } + resolveDated(); + for (const v of videos) { + if (typeof v.episode === "number") continue; + files.push({ + path: v.path, + kind: "nfo", + role: "episode_nfo", + action: "skipped", + reason: unresolved.has(v.path) ? "episode_unresolved" : "scrape_no_episode", + }); + } } const work = nfoWork(fields.entity, fields.ids); const seasonPosters = new Map(); @@ -909,7 +984,6 @@ export async function runScrapeRecord( options: { images: settings.scrapeImages, episodeThumbs: settings.scrapeEpisodeThumbs }, parts: lay.parts, }); - files.push(...noEpisode); // 「更新」模式:本工具以前写过的文件(路径 → 写入时的 hash / 时间) const ours = mode === "update" ? await previousWrites(d, rec.root_id, recordId) : new Map(); const imageCache = new Map>(); @@ -978,12 +1052,15 @@ export async function runScrapeRecord( } xml = seasonNfo(nfoSeason(t.season!, u)); } else { - const u = episodeUnit.get(`${t.season}:${t.episode}`); - if (episodeMetadataMissing(u) && hasTmdbId) { + // 多集文件每集一块;缺元数据的集不写(全缺才跳过整个 NFO) + const eps = (t.episodes?.length ? t.episodes : [t.episode!]).filter( + (ep) => !(hasTmdbId && episodeMetadataMissing(episodeUnit.get(`${t.season}:${ep}`))), + ); + if (!eps.length) { files.push(missingMeta(t)); continue; } - xml = episodeNfo(nfoEpisode(t.season!, t.episode!, u)); + xml = multiEpisodeNfo(eps.map((ep) => nfoEpisode(t.season!, ep, episodeUnit.get(`${t.season}:${ep}`)))); } files.push(await write(t, dec, xml)); } @@ -1007,11 +1084,12 @@ export async function runScrapeRecord( const written = files.filter((f) => f.action === "written").length; const failed = files.filter((f) => f.action === "failed").length; - // 缺元数据跳过的季 / 集不算成功(要刷新元数据后再刮) - const lacking = files.some((f) => f.reason === "scrape_metadata_missing"); + // 缺元数据、认不出集号、集数待确认而跳过的不算成功(要刷新元数据 / 确认集数后再刮); + // 已存在、内容没变、被人改过的跳过是预期的 + const incomplete = files.find((f) => f.action === "skipped" && !EXPECTED_SKIPS.has(f.reason ?? "")); const status: ScrapeRecord["status"] = failed === 0 - ? lacking + ? incomplete ? "partial" : "succeeded" : written > 0 || files.some((f) => f.action === "skipped") @@ -1021,9 +1099,9 @@ export async function runScrapeRecord( status, reason: failed ? (files.find((f) => f.action === "failed")?.reason ?? null) - : lacking + : files.some((f) => f.reason === "scrape_metadata_missing") ? "scrape_metadata_missing" - : null, + : (incomplete?.reason ?? null), files, }); } catch (err) { diff --git a/apps/server/test/acquisition-organize/scrape-episodes.test.ts b/apps/server/test/acquisition-organize/scrape-episodes.test.ts new file mode 100644 index 00000000..e0a2ffad --- /dev/null +++ b/apps/server/test/acquisition-organize/scrape-episodes.test.ts @@ -0,0 +1,283 @@ +import { readFile, stat } from "node:fs/promises"; +import type { TmdbRichSeasonDetail } from "@umt/adapters"; +import type { ScrapeFileResult, ScrapeMatch } from "@umt/contracts"; +import { createTempTree, type TempTree } from "@umt/testkit"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { queueScrape, runScrapeRecord, scrapeEntity } from "../../src/modules/acquisition-organize/services/scrape.ts"; +import { createHarness, type Harness, pgAvailable, seedBase } from "./harness.ts"; + +// 刮削的集与布局: +// - 多集文件(文件名 E01-E02 / 覆盖多个集单元)按 Kodi 约定在一个 NFO 里写多个 ; +// - 综艺按播出日期命名的期数按日期对上集(本地集单元或 TMDB 季详情)写分集 NFO; +// - 作品重刮只刮该作品的视频;同一目录里有别的作品时电影图片按 <视频名>-poster 命名; +// - 认不出集号的跳过算部分完成;图片扩展名按实际类型;取消时记录不停在进行中。 + +const hasPg = await pgAvailable(); +const at = new Date().toISOString(); +const f = (value: unknown) => ({ value, source: "tmdb", language: "zh-CN", at, locked: false }); +const enc = (s: string) => new TextEncoder().encode(s); +const PNG = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + +function seasonOf(n: number, eps: { ep: number; name: string; airDate: string }[]): TmdbRichSeasonDetail { + return { + tvId: "777", + seasonNumber: n, + name: `第 ${n} 季`, + nameLanguage: "zh-CN", + overview: null, + overviewLanguage: null, + airDate: eps[0]?.airDate ?? null, + posterPath: null, + fetchedLanguages: ["zh-CN"], + episodes: eps.map((e) => ({ + seasonNumber: n, + episodeNumber: e.ep, + name: e.name, + nameLanguage: "zh-CN", + overview: `${e.name}简介`, + overviewLanguage: "zh-CN", + airDate: e.airDate, + runtime: null, + stillPath: null, + voteAverage: null, + voteCount: null, + })), + }; +} + +describe.skipIf(!hasPg)("刮削:多集文件、综艺日期期数、平铺目录、状态", () => { + let h: Harness; + let lib: TempTree; + let rootId = ""; + let seasonDetail: (n: number) => Promise = async () => null; + const imageMime = "image/jpeg"; + + beforeAll(async () => { + lib = await createTempTree( + { + "TV/双集剧 (2020)/Season 1/Double.S01E01-E02.mkv": 100, + "TV/双集剧 (2020)/Season 1/Double.S01E03.mkv": 100, + "TV/综艺 (2024)/Season 1/综艺 - 2024-05-03.mkv": 100, + "TV/综艺 (2024)/Season 1/综艺 - 2024-05-10.mkv": 100, + "TV/综艺 (2024)/Season 1/综艺 - 2024-06-01.mkv": 100, + "Movies/盗梦空间 (2010).mkv": 100, + "Movies/星际穿越 (2014).mkv": 100, + "Single/阿凡达 (2009)/阿凡达 (2009).mkv": 100, + }, + { sentinel: true, prefix: "umt-screps-" }, + ); + h = await createHarness("scrape_eps", { + scrapeImage: async (p) => ({ content: imageMime === "image/png" ? PNG : enc(p), mime: imageMime }), + scrapeTmdbSeason: async (_tv, n) => seasonDetail(n), + }); + rootId = (await seedBase(h.db, { libraryPath: lib.root, cachePath: lib.root, qbUrl: "http://127.0.0.1:9" })).rootId; + }, 60_000); + afterAll(async () => { + await h?.close(); + await lib?.cleanup(); + }); + + async function entity(kind: string, title: string, meta: object, tmdb?: { id: string; type: "movie" | "tv" }) { + const id = ( + await h.db + .insertInto("media_entity") + .values({ kind: kind as never, title, year: 2020, status: "active", metadata: JSON.stringify(meta) }) + .returning("id") + .executeTakeFirstOrThrow() + ).id; + if (tmdb) + await h.db + .insertInto("external_identity") + .values({ + media_entity_id: id, + provider: "tmdb", + entity_type: tmdb.type, + external_id: tmdb.id, + status: "accepted", + }) + .execute(); + return id; + } + async function unit(entityId: string, season: number, ep: number | null, meta: object, airDate?: string) { + return ( + await h.db + .insertInto("content_unit") + .values({ + media_entity_id: entityId, + kind: ep === null ? "season" : "episode", + season_number: season, + episode_number: ep, + status: "confirmed", + title: (meta as { name?: { value?: string } }).name?.value ?? null, + air_date: airDate ?? null, + metadata: JSON.stringify(meta), + }) + .returning("id") + .executeTakeFirstOrThrow() + ).id; + } + async function scrape(entityId: string | null, kind: "movie" | "tv", path: string, externalId: string | null = null) { + const match: ScrapeMatch = { + mediaEntityId: entityId, + provider: externalId ? "tmdb" : null, + externalId, + kind, + title: "x", + year: null, + source: entityId ? "library" : "chosen", + }; + const q = await queueScrape(h.d, { + rootId: `lib:${rootId}`, + rootPath: lib.root, + libraryRootId: rootId, + path, + match, + mode: "overwrite", + trigger: "manual", + }); + return runScrapeRecord(h.d, q.recordId); + } + const byPath = (files: ScrapeFileResult[], suffix: string) => files.find((x) => x.path.endsWith(suffix)); + const count = (xml: string, re: RegExp) => xml.match(re)?.length ?? 0; + + it("多集文件(文件名 E01-E02):一个 NFO 里每集一个 ,按集号顺序", async () => { + const id = await entity("series", "双集剧", { title: f("双集剧") }, { id: "500", type: "tv" }); + const s = await unit(id, 1, null, { name: f("第 1 季") }); + for (const [ep, name] of [ + [1, "开端"], + [2, "发展"], + [3, "结局"], + ] as const) + await h.db + .insertInto("content_unit") + .values({ + media_entity_id: id, + parent_unit_id: s, + kind: "episode", + season_number: 1, + episode_number: ep, + status: "confirmed", + title: name, + metadata: JSON.stringify({ name: f(name), overview: f(`${name}简介`) }), + }) + .execute(); + const r = await scrape(id, "tv", "TV/双集剧 (2020)", "500"); + expect(r.status).toBe("succeeded"); + expect(byPath(r.files, "S01E01-E02.nfo")).toMatchObject({ action: "written" }); + const xml = await readFile(lib.path("TV/双集剧 (2020)/Season 1/Double.S01E01-E02.nfo"), "utf8"); + expect(count(xml, /<\?xml /g)).toBe(1); + expect(count(xml, //g)).toBe(2); + expect(xml.indexOf("开端")).toBeLessThan(xml.indexOf("发展")); + const single = await readFile(lib.path("TV/双集剧 (2020)/Season 1/Double.S01E03.nfo"), "utf8"); + expect(count(single, //g)).toBe(1); + }); + + it("多集文件(已登记、覆盖两个集单元):按覆盖单元写两块", async () => { + const id = await entity("series", "覆盖剧", { title: f("覆盖剧") }, { id: "501", type: "tv" }); + const s = await unit(id, 1, null, { name: f("第 1 季") }); + const eps: string[] = []; + for (const [ep, name] of [ + [1, "甲"], + [2, "乙"], + ] as const) + eps.push( + ( + await h.db + .insertInto("content_unit") + .values({ + media_entity_id: id, + parent_unit_id: s, + kind: "episode", + season_number: 1, + episode_number: ep, + status: "confirmed", + title: name, + metadata: JSON.stringify({ name: f(name) }), + }) + .returning("id") + .executeTakeFirstOrThrow() + ).id, + ); + // 文件名没有集号区间,覆盖单元是 E01 + E02 + const path = "TV/覆盖剧 (2020)/Season 1/覆盖剧 - S01E01.mkv"; + await (await import("node:fs/promises")).mkdir(lib.path("TV/覆盖剧 (2020)/Season 1"), { recursive: true }); + await (await import("node:fs/promises")).writeFile(lib.path(path), "video"); + const asset = await h.db + .insertInto("media_asset") + .values({ media_entity_id: id, kind: "video", status: "active" }) + .returning("id") + .executeTakeFirstOrThrow(); + await h.db + .insertInto("asset_location") + .values({ + media_asset_id: asset.id, + location_kind: "library", + library_root_id: rootId, + relative_path: path, + status: "available", + }) + .execute(); + for (const u of eps) + await h.db + .insertInto("asset_unit_coverage") + .values({ media_asset_id: asset.id, content_unit_id: u, coverage: "full" }) + .execute(); + const r = await scrape(id, "tv", "TV/覆盖剧 (2020)", "501"); + expect(r.status).toBe("succeeded"); + const xml = await readFile(lib.path(path.replace(/\.mkv$/, ".nfo")), "utf8"); + expect(count(xml, //g)).toBe(2); + expect(xml).toContain("乙"); + }); + + let variety = ""; + it("综艺按日期命名:按集单元的播出日期对上集,写 与集号;对不上的跳过", async () => { + const id = await entity("variety", "综艺", { title: f("综艺") }, { id: "777", type: "tv" }); + variety = id; + const s = await unit(id, 1, null, { name: f("第 1 季") }); + for (const [ep, name, day] of [ + [1, "第一期", "2024-05-03"], + [2, "第二期", "2024-05-10"], + ] as const) + await h.db + .insertInto("content_unit") + .values({ + media_entity_id: id, + parent_unit_id: s, + kind: "episode", + season_number: 1, + episode_number: ep, + status: "confirmed", + title: name, + air_date: day, + metadata: JSON.stringify({ name: f(name), airDate: f(day) }), + }) + .execute(); + seasonDetail = async () => null; + const r = await scrape(id, "tv", "TV/综艺 (2024)", "777"); + const nfo = await readFile(lib.path("TV/综艺 (2024)/Season 1/综艺 - 2024-05-10.nfo"), "utf8"); + expect(nfo).toContain("第二期"); + expect(nfo).toContain("2024-05-10"); + expect(nfo).toContain("2"); + expect(byPath(r.files, "2024-05-03.nfo")).toMatchObject({ action: "written" }); + // 6 月 1 日这期季表里没有(TMDB 也取不到):跳过,算部分完成 + expect(byPath(r.files, "2024-06-01.mkv")).toMatchObject({ action: "skipped", reason: "scrape_no_episode" }); + expect(r.status).toBe("partial"); + }); + + it("综艺按日期命名:本地没有这一期时按 TMDB 季详情的播出日期对集", async () => { + seasonDetail = async (n) => + n === 1 + ? seasonOf(1, [ + { ep: 1, name: "第一期", airDate: "2024-05-03" }, + { ep: 2, name: "第二期", airDate: "2024-05-10" }, + { ep: 5, name: "六月特辑", airDate: "2024-06-01" }, + ]) + : null; + const r = await scrape(variety, "tv", "TV/综艺 (2024)", "777"); + expect(byPath(r.files, "2024-06-01.nfo")).toMatchObject({ action: "written" }); + const nfo = await readFile(lib.path("TV/综艺 (2024)/Season 1/综艺 - 2024-06-01.nfo"), "utf8"); + expect(nfo).toContain("六月特辑"); + expect(nfo).toContain("5"); + expect(r.status).toBe("succeeded"); + }); +}); diff --git a/apps/server/test/scrape/nfo-deviations.json b/apps/server/test/scrape/nfo-deviations.json index 7f110843..f5f6fa0e 100644 --- a/apps/server/test/scrape/nfo-deviations.json +++ b/apps/server/test/scrape/nfo-deviations.json @@ -65,5 +65,11 @@ "ours": "<视频名>-thumb.jpg", "reason": "Kodi 约定 -thumb 后缀;Jellyfin / Emby 同样识别,且不会和同名海报规则冲突" } - ] + ], + "multiEpisode": { + "case": "s01e01-e02(一个文件覆盖第 1、2 集)", + "mp": "只按首集(begin_episode)写一个 ,与 s01e01 相同", + "ours": "按 Kodi 多集约定写两个 块:第一块同 s01e01,第二块同 s01e02(各自的已登记差异照旧)", + "reason": "只写首集时 Kodi 只显示第一集,第二集在媒体库里缺失;Jellyfin / Emby 也读多块 NFO" + } } diff --git a/apps/server/test/scrape/nfo-mp-diff.test.ts b/apps/server/test/scrape/nfo-mp-diff.test.ts index 1ccabbfa..8756e616 100644 --- a/apps/server/test/scrape/nfo-mp-diff.test.ts +++ b/apps/server/test/scrape/nfo-mp-diff.test.ts @@ -1,6 +1,6 @@ import { readFileSync } from "node:fs"; import { fetchRichDetail, fetchRichSeason } from "@umt/adapters"; -import { episodeNfo, planScrapeTargets, seasonNfo, seasonPosterName, workNfo } from "@umt/domain"; +import { episodeNfo, multiEpisodeNfo, planScrapeTargets, seasonNfo, seasonPosterName, workNfo } from "@umt/domain"; import { describe, expect, it } from "vitest"; import { nfoEpisode, nfoSeason, nfoWork } from "../../src/modules/acquisition-organize/services/scrape-input.ts"; import { richEntityFields } from "../../src/modules/sources-identity/service/metadata-rich.ts"; @@ -12,7 +12,8 @@ import { richEntityFields } from "../../src/modules/sources-identity/service/met const read = (f: string) => JSON.parse(readFileSync(new URL(f, import.meta.url), "utf8")); const raw = read("./tmdb-raw.json") as Record; const mp = read("./scrape-mp.json") as Record; -const devs = read("./nfo-deviations.json").deviations as { case: string; tag: string }[]; +const devFile = read("./nfo-deviations.json"); +const devs = devFile.deviations as { case: string; tag: string }[]; const LANGS = ["zh-CN", "zh-SG", "zh-TW", "zh-HK", "en-US"]; const client = { @@ -63,23 +64,23 @@ async function ours(): Promise> { metadata: { name: { value: sd.name }, overview: { value: sd.overview }, airDate: { value: sd.airDate } }, }), ); - for (const e of sd.episodes) { - out[`s01e${String(e.episodeNumber).padStart(2, "0")}`] = episodeNfo( - nfoEpisode(1, e.episodeNumber, { - season_number: 1, - episode_number: e.episodeNumber, - title: e.name, - air_date: e.airDate, - metadata: { - name: { value: e.name }, - overview: { value: e.overview }, - airDate: { value: e.airDate }, - runtime: { value: e.runtime }, - voteAverage: { value: e.voteAverage }, - }, - }), - ); - } + const eps = sd.episodes.map((e) => + nfoEpisode(1, e.episodeNumber, { + season_number: 1, + episode_number: e.episodeNumber, + title: e.name, + air_date: e.airDate, + metadata: { + name: { value: e.name }, + overview: { value: e.overview }, + airDate: { value: e.airDate }, + runtime: { value: e.runtime }, + voteAverage: { value: e.voteAverage }, + }, + }), + ); + for (const e of eps) out[`s01e${String(e.episode).padStart(2, "0")}`] = episodeNfo(e); + out["s01e01-e02"] = multiEpisodeNfo(eps.slice(0, 2)); return out; } @@ -109,6 +110,24 @@ describe("NFO 与 MP 差分", async () => { } }); + it("多集文件:按登记的偏离写多个 块,每块与 MP 对应单集一致(MP 只写首集)", () => { + expect(devFile.multiEpisode?.reason).toBeTruthy(); + const xml = mine["s01e01-e02"]!; + const [decl, ...body] = xml.split("\n"); + expect(decl).toBe(mp.s01e01!.split("\n")[0]); + const blocks = body + .join("\n") + .split(/(?=)/) + .filter((b) => b.trim()); + expect(blocks).toHaveLength(2); + for (const [i, c] of ["s01e01", "s01e02"].entries()) { + const skip = new Set(devs.filter((d) => d.case === c).map((d) => d.tag)); + const a = normalize(mp[c]!).filter((l) => !skip.has(tagOf(l) ?? "") && !l.startsWith(" !skip.has(tagOf(l) ?? "")); + expect(b).toEqual(a); + } + }); + it("编码与声明:UTF-8、两空格缩进、末尾换行,和 MP 一致", () => { for (const c of cases) { expect(mine[c]!.split("\n")[0]).toBe(mp[c]!.split("\n")[0]); diff --git a/docs/runbook/scrape.md b/docs/runbook/scrape.md index f2c62522..7818f353 100644 --- a/docs/runbook/scrape.md +++ b/docs/runbook/scrape.md @@ -102,7 +102,11 @@ TMDB 有这一集但没有译名 / 简介(只有占位名和播出日期)不算 | 电影文件 | `<视频名>.nfo` + 同目录 `poster.jpg` / `fanart.jpg`;文件直接在登记目录根时图片改为 `<视频名>-poster.jpg` / `-fanart.jpg` | - 集号:已登记视频按覆盖的集单元;其余按路径解析(季号缺失时取季目录名,再缺按第 1 季;同 MP 对每集重新 `MetaInfoPath`)。 - 认不出集号的视频记 `scrape_no_episode` 跳过。多集文件按首集。 + 认不出集号的视频记 `scrape_no_episode` 跳过,记录算部分完成。 +- 多集文件(文件名 `E01-E02`,或已登记文件覆盖多个集单元):一个 `<视频名>.nfo` 里按 Kodi 约定每集写一个 `` 块 + (MP 只写首集,偏离登记在 `nfo-deviations.json` 的 `multiEpisode`);剧照按首集。某一集缺元数据时只写有元数据的那几块,全缺才跳过。 +- 综艺按播出日期命名的期数(默认综艺模板没有集号时用日期):按集单元的播出日期对上集(本地季表,或按 TMDB id 现取的季详情), + 写该集的季号、集号与 ``;日期对不上(或同一天有多集)的跳过 `scrape_no_episode`。 - 字段:本地作品有元数据时用最终字段(字段锁、人工覆盖优先,见 `metadata.md`);否则按 TMDB id 直接取 TMDB(与刷新元数据同一映射)。 TMDB id 只认与刮削类型一致的映射(同一作品同时有 movie / tv 两个 accepted 映射时不串)。 - 作品类型:剧集 / 综艺按剧集刮,电影 / 合集 / 其他按电影刮(`<视频名>.nfo`,根元素 ``),范围判定与 NFO 生成同一个函数 `scrapeKindOf`。 @@ -177,6 +181,7 @@ TMDB 有这一集但没有译名 / 简介(只有占位名和播出日期)不算 `title`、`originaltitle`、`premiered`、`year`;剧集另有 `season=-1`、`episode=-1`。 - 季:`plot`、`outline`、`title`(单元有简介 / 日期但没名称时「季 N」)、`premiered`、`releasedate`、`year`、`seasonnumber`。 - 集:`title`(TMDB 占位名时「第 N 集」)、`plot`、`outline`、`aired`、`year`、`season`、`episode`、`rating`。 + 多集文件每集一个 `` 块。 季 / 集单元整个缺元数据时不写(见「季 / 集缺元数据」)。 - 取值:标题 / 简介 / 标语按元数据语言链(见 `metadata.md`);字段锁与人工覆盖优先(用的是投影后的最终值)。 @@ -189,6 +194,7 @@ TMDB 有这一集但没有译名 / 简介(只有占位名和播出日期)不算 | 制片国取 TMDB `origin_country`(代码转中文名);剧集 `studio` 写播出平台 | 本地不存 `production_countries` / `production_companies` | | 电影不写 `studio` | 同上 | | 集 NFO 不写 `uniqueid` / `tmdbid` | 本地不存 TMDB 集 id;媒体服务器按季号集号匹配 | +| 多集文件每集一个 `` 块(MP 只写首集) | 只写首集时 Kodi 只显示第一集 | | 简介含 `]]>` 时拆成两段 CDATA | MP 的 minidom 直接抛错 | 重新生成 MP 输出(改了样例或跟进上游后): @@ -221,6 +227,7 @@ cd /opt/uhd_media_tool && deno run -A npm:@biomejs/biome check --write apps/serv | 刮削对话框一直让选作品 | 文件名解析不出标题,或 TMDB 有多个同名候选;选一次即可,打开「同时加入媒体库」后下次按已入库作品识别 | | 全部「跳过」 | 覆盖方式是只补缺失,或更新时文件不是本工具写的 / 被人改过(`scrape_exists` / `scrape_user_modified`);要替换选「全部覆盖」或作品详情「覆盖重写」 | | 改了映射 / 刷新了元数据,NFO 没变 | 作品以前没刮削过(不自动重刮);重刮任务要等 2 分钟;NFO 被人改过(`scrape_user_modified`) | +| 记录「部分完成」、原因认不出集号 / 集数待确认 | 文件名没有集号(综艺日期对不上季表),或集数映射待确认;确认集数或刷新元数据后重新刮削 | | 记录「部分成功」、原因缺元数据 | 季 / 集还没取到元数据且 TMDB 季详情取不到;刷新元数据(作品详情「刷新」带 full)后会自动重刮 | | 图片失败、NFO 正常 | TMDB 图片访问(出口代理);稍后重新刮削,只补缺失的图片 | | 「没有可用的元数据」 | 作品没有 TMDB 字段也没有 TMDB id;在文件管理里刮削并指定 TMDB 条目 | diff --git a/packages/domain/src/scrape/layout.ts b/packages/domain/src/scrape/layout.ts index 69d84a70..763d6574 100644 --- a/packages/domain/src/scrape/layout.ts +++ b/packages/domain/src/scrape/layout.ts @@ -49,8 +49,13 @@ export interface ScrapeVideo { /** 库根内相对路径 */ path: string; season?: number | null; - /** 首集集号(多集文件按首集,同 MP begin_episode) */ + /** 首集集号(同 MP begin_episode;剧照、文件命名按首集) */ episode?: number | null; + /** + * 文件覆盖的全部集号(多集文件 E01-E02 / 覆盖多个集单元),含首集、升序;缺省 = 只有首集。 + * 集 NFO 按 Kodi 约定每集写一个 `` 块(MP 只写首集)。 + */ + episodes?: readonly number[] | null; } export interface ScrapeImages { @@ -83,6 +88,8 @@ export interface ScrapeTarget { role: ScrapeTargetRole; season?: number; episode?: number; + /** 集 NFO:文件覆盖的全部集号(多集文件多于一个) */ + episodes?: number[]; /** 这个文件对应的视频(电影 / 集 NFO、剧照) */ video?: string; /** 图片:TMDB 图片路径 */ @@ -227,12 +234,15 @@ export function planScrapeTargets(input: ScrapeLayoutInput): ScrapeTarget[] { if (v.season === null || v.season === undefined || v.episode === null || v.episode === undefined) continue; const dir = normDir(posix.dirname(v.path)); const stem = stemOf(v.path); + const first = v.episode; + const eps = [...new Set([first, ...(v.episodes ?? [])])].sort((a, b) => a - b); add({ path: join(dir, `${stem}.nfo`), kind: "nfo", role: "episode_nfo", season: v.season, episode: v.episode, + episodes: eps, video: v.path, }); const still = img.stills?.get(`${v.season}:${v.episode}`); diff --git a/packages/domain/src/scrape/nfo.ts b/packages/domain/src/scrape/nfo.ts index 5c6d31d7..95ebc2b5 100644 --- a/packages/domain/src/scrape/nfo.ts +++ b/packages/domain/src/scrape/nfo.ts @@ -6,7 +6,8 @@ // - 缺值不写元素(MP 写空元素 ``、评分缺省写 0、空 CDATA); // - 简介里的 `]]>` 拆成两段 CDATA(MP 的 minidom 直接抛错); // - 额外写 豆瓣 id(doubanid + uniqueid type=douban)、标语 tagline、集时长 runtime; -// - 集 NFO 没有 TMDB 集 id(本地不存),不写 uniqueid / tmdbid,媒体服务器按季号集号匹配。 +// - 集 NFO 没有 TMDB 集 id(本地不存),不写 uniqueid / tmdbid,媒体服务器按季号集号匹配; +// - 多集文件(E01-E02)按 Kodi 约定每集一个 `` 块写进同一个 NFO(MP 只写首集)。 export const TMDB_IMAGE_BASE = "https://image.tmdb.org/t/p/original"; @@ -92,9 +93,9 @@ function render(n: Node, indent: string, out: string[]) { else out.push(`${indent}<${n.tag}${attrs}/>`); } -function toXml(root: Node): string { +function toXml(...roots: Node[]): string { const out = ['']; - render(root, "", out); + for (const root of roots) render(root, "", out); return `${out.join("\n")}\n`; } @@ -248,6 +249,19 @@ export function seasonNfo(s: NfoSeason): string { /** 每集 `<视频名>.nfo` */ export function episodeNfo(e: NfoEpisode): string { + return toXml(episodeNode(e)); +} + +/** + * 多集文件的 `<视频名>.nfo`:Kodi 约定(Jellyfin / Emby 同样读取)每集一个 `` 块,依次写在同一个文件里, + * 只有一个 XML 声明。只有一集时与 episodeNfo 相同。 + */ +export function multiEpisodeNfo(list: readonly NfoEpisode[]): string { + if (!list.length) throw new Error("multiEpisodeNfo: 至少一集"); + return toXml(...list.map(episodeNode)); +} + +function episodeNode(e: NfoEpisode): Node { const b = new Builder(); b.text("title", has(e.title) ? e.title : `第 ${e.episode} 集`); b.plot(e.overview); @@ -259,5 +273,5 @@ export function episodeNfo(e: NfoEpisode): string { b.directors(e.directors); b.actors(e.actors, false); if (num(e.runtime) && e.runtime > 0) b.text("runtime", e.runtime); - return toXml({ tag: "episodedetails", children: b.nodes }); + return { tag: "episodedetails", children: b.nodes }; } diff --git a/packages/domain/test/scrape.test.ts b/packages/domain/test/scrape.test.ts index c9ae6ea3..417da9cb 100644 --- a/packages/domain/test/scrape.test.ts +++ b/packages/domain/test/scrape.test.ts @@ -3,6 +3,7 @@ import { episodeNfo, inferWorkDir, isSeasonDirName, + multiEpisodeNfo, planScrapeTargets, scrapeKindOf, seasonNfo, @@ -255,3 +256,37 @@ describe("作品类型 → 刮削类型", () => { expect(["movie", "collection", "other"].map(scrapeKindOf)).toEqual(["movie", "movie", "movie"]); }); }); + +describe("多集文件", () => { + it("集 NFO 每集一个 块,只有一个 XML 声明;一集时与 episodeNfo 相同", () => { + const xml = multiEpisodeNfo([ + { season: 1, episode: 1, title: "上", airDate: "2022-02-17" }, + { season: 1, episode: 2, title: "下", airDate: "2022-02-17" }, + ]); + expect(xml.match(/<\?xml /g)).toHaveLength(1); + expect(xml.match(//g)).toHaveLength(2); + expect(xml).toContain(" 上\n 2022-02-17"); + expect(xml.indexOf("1")).toBeLessThan(xml.indexOf("2")); + const one = { season: 1, episode: 3, title: "单集" }; + expect(multiEpisodeNfo([one])).toBe(episodeNfo(one)); + }); + + it("布局:多集文件只写一个 NFO(按视频名)与首集剧照,目标带全部集号", () => { + const t = planScrapeTargets({ + kind: "tv", + workDir: "剧 (2020)", + videos: [{ path: "剧 (2020)/Season 1/剧 - S01E01-E02.mkv", season: 1, episode: 1, episodes: [2, 1] }], + images: { stills: new Map([["1:1", "/e1.jpg"]]) }, + options: { images: true, episodeThumbs: true }, + parts: { work: false, season: false }, + }); + expect(t).toEqual([ + expect.objectContaining({ + role: "episode_nfo", + path: "剧 (2020)/Season 1/剧 - S01E01-E02.nfo", + episodes: [1, 2], + }), + expect.objectContaining({ role: "episode_thumb", path: "剧 (2020)/Season 1/剧 - S01E01-E02-thumb.jpg" }), + ]); + }); +});