rootandClaude Opus 5.5
f5bc6e9c40
原地入库逐个文件排刮削,单文件刮削只写该集,剧集根目录(tvshow.nfo、海报、季海报)与季目录 (season.nfo)没有元数据。照 MP ScrapingChain.scrape_metadata_event:按作品目录分组,目录元数据照常 初始化,集只刮本次文件;复制 / 移动同样改为只刮本次文件(此前刮整个作品目录)。 - 迁移 0062:scrape_record.videos(null = 目录下全部) - queueScrape 支持 videos;同目录排队中还没开始的刮削合成一批(有一方刮全部则刮全部) - 作品详情「重新刮削」与文件管理刮削不变(目录下全部) Co-Authored-By: Claude Opus 5.5 <[email protected]>
343 lines
14 KiB
TypeScript
343 lines
14 KiB
TypeScript
import { chmod, readdir, readFile, stat, writeFile } from "node:fs/promises";
|
|
import { join, relative } from "node:path";
|
|
import { createTempTree, type TempTree } from "@umt/testkit";
|
|
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
|
import { executeRecord, planImport } from "../../src/modules/acquisition-organize/services/organize.ts";
|
|
import {
|
|
entityScrapeLocations,
|
|
runScrapeRecord,
|
|
SCRAPE_TASK,
|
|
scrapeEntity,
|
|
} from "../../src/modules/acquisition-organize/services/scrape.ts";
|
|
import { createHarness, type Harness, pgAvailable, seedBase, seedEntity } from "./harness.ts";
|
|
|
|
// 刮削(SCR-1 / SCR-2):入库成功后自动对作品目录排 scrapeMedia → 写 NFO 与图片(临时库根);只补缺失 / 全部覆盖;
|
|
// 图片失败不阻塞 NFO;只登记库根不写;作品详情「重新刮削」接口与刮削状态(最近一条刮削记录)。
|
|
|
|
const hasPg = await pgAvailable();
|
|
const SIZE = 200_000;
|
|
/** 最小 JPEG 头(按文件头认图) */
|
|
const JPEG = new Uint8Array([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10, 0x4a, 0x46, 0x49, 0x46, 0x00]);
|
|
const at = new Date().toISOString();
|
|
const f = (value: unknown, language: string | null = null) => ({ value, source: "tmdb", language, at, locked: false });
|
|
|
|
async function tree(root: string): Promise<string[]> {
|
|
const out: string[] = [];
|
|
const walk = async (d: string) => {
|
|
for (const n of (await readdir(d)).sort()) {
|
|
const p = join(d, n);
|
|
if ((await stat(p)).isDirectory()) await walk(p);
|
|
else out.push(relative(root, p));
|
|
}
|
|
};
|
|
await walk(root);
|
|
return out.filter((p) => !p.startsWith(".")).sort();
|
|
}
|
|
|
|
describe.skipIf(!hasPg)("刮削:NFO 与图片写进媒体库目录", () => {
|
|
let h: Harness;
|
|
let lib: TempTree;
|
|
let src: TempTree;
|
|
const fetched: string[] = [];
|
|
|
|
beforeAll(async () => {
|
|
lib = await createTempTree({}, { sentinel: true, prefix: "umt-scrlib-" });
|
|
src = await createTempTree(
|
|
{
|
|
"Severance.S01E01.1080p.WEB-DL.H264-GRP.mkv": SIZE,
|
|
"Severance.S01E02.1080p.WEB-DL.H264-GRP.mkv": SIZE,
|
|
"Interstellar.2014.1080p.BluRay.x264-GRP.mkv": SIZE,
|
|
},
|
|
{ prefix: "umt-scrsrc-" },
|
|
);
|
|
h = await createHarness("scrape", {
|
|
async scrapeImage(p) {
|
|
fetched.push(p);
|
|
if (p === "/broken.jpg") throw new Error("TMDB 图片返回 HTTP 404");
|
|
return { content: JPEG, mime: "image/jpeg" };
|
|
},
|
|
});
|
|
await seedBase(h.db, { libraryPath: lib.root, cachePath: src.root, qbUrl: "http://127.0.0.1:9" });
|
|
await h.setSettings("organize", { scrapeAfterImport: true });
|
|
}, 60_000);
|
|
|
|
afterAll(async () => {
|
|
await h?.close();
|
|
for (const t of [lib, src]) await t?.cleanup();
|
|
});
|
|
|
|
async function organize(entity: string, file: string) {
|
|
const plan = await planImport(h.d, {
|
|
sourcePaths: [src.path(file)],
|
|
mediaEntityId: entity,
|
|
action: "copy",
|
|
trigger: "manual",
|
|
actorType: "user",
|
|
});
|
|
expect(plan.status).toBe("running");
|
|
const ex = await executeRecord(h.d, plan.recordId!);
|
|
expect(ex.status).toBe("succeeded");
|
|
return plan.recordId!;
|
|
}
|
|
|
|
/** 按引擎记录的入队项跑刮削(假引擎不执行):该作品最近一条刮削记录 */
|
|
async function runQueued(entity: string) {
|
|
const recs = await h.db
|
|
.selectFrom("scrape_record")
|
|
.select(["id"])
|
|
.where("media_entity_id", "=", entity)
|
|
.where("status", "=", "queued")
|
|
.orderBy("requested_at", "desc")
|
|
.execute();
|
|
expect(recs.length).toBeGreaterThan(0);
|
|
const q = h.engine.enqueued.filter((x) => x.task === SCRAPE_TASK);
|
|
expect(q.some((x) => (x.input as { recordId: string }).recordId === recs[0]!.id)).toBe(true);
|
|
return runScrapeRecord(h.d, recs[0]!.id);
|
|
}
|
|
|
|
/** 作品详情「重新刮削」的服务端做法:对作品目录排刮削并执行 */
|
|
async function rescrape(entity: string, mode?: "missing" | "overwrite") {
|
|
const q = await scrapeEntity(h.d, entity, { trigger: "manual", mode });
|
|
expect(q.length).toBe(1);
|
|
return runScrapeRecord(h.d, q[0]!.recordId);
|
|
}
|
|
|
|
let series = "";
|
|
let movie = "";
|
|
|
|
it("剧集:入库后自动排刮削,写 tvshow / 季 / 集 NFO、海报、季海报、剧照", async () => {
|
|
series = await seedEntity(h.db, { kind: "series", title: "人生切割术", year: 2022 });
|
|
await h.db
|
|
.updateTable("media_entity")
|
|
.set({
|
|
original_title: "Severance",
|
|
metadata: JSON.stringify({
|
|
title: f("人生切割术", "zh-CN"),
|
|
originalTitle: f("Severance", "en"),
|
|
year: f(2022),
|
|
releaseDate: f("2022-02-17"),
|
|
overview: f("马克领导着一个办公室团队。", "zh-CN"),
|
|
genres: f(["剧情", "悬疑"]),
|
|
voteAverage: f(8.4),
|
|
originCountry: f(["US"]),
|
|
platform: f("Apple TV+"),
|
|
posterPath: f("/sev.jpg"),
|
|
backdropPath: f("/broken.jpg"),
|
|
cast: f([
|
|
{ id: 4495, name: "亚当·斯科特", originalName: null, role: "Mark Scout", profilePath: "/adam.jpg" },
|
|
]),
|
|
crew: [],
|
|
externalIds: f({ imdb: "tt11280740", tvdb: "371980", wikidata: null }),
|
|
}),
|
|
})
|
|
.where("id", "=", series)
|
|
.execute();
|
|
await h.db
|
|
.insertInto("external_identity")
|
|
.values({
|
|
media_entity_id: series,
|
|
provider: "tmdb",
|
|
entity_type: "tv",
|
|
external_id: "95396",
|
|
status: "accepted",
|
|
})
|
|
.execute();
|
|
const season = await h.db
|
|
.insertInto("content_unit")
|
|
.values({
|
|
media_entity_id: series,
|
|
kind: "season",
|
|
season_number: 1,
|
|
status: "confirmed",
|
|
title: "第 1 季",
|
|
metadata: JSON.stringify({ name: f("第 1 季", "zh-CN"), airDate: f("2022-02-17"), posterPath: f("/s1.jpg") }),
|
|
})
|
|
.returning("id")
|
|
.executeTakeFirstOrThrow();
|
|
for (const [n, name, still] of [
|
|
[1, "有关地狱的好消息", "/e1.jpg"],
|
|
[2, "半循环", null],
|
|
] as const)
|
|
await h.db
|
|
.insertInto("content_unit")
|
|
.values({
|
|
media_entity_id: series,
|
|
parent_unit_id: season.id,
|
|
kind: "episode",
|
|
season_number: 1,
|
|
episode_number: n,
|
|
status: "confirmed",
|
|
title: name,
|
|
air_date: "2022-02-17",
|
|
metadata: JSON.stringify({
|
|
name: f(name, "zh-CN"),
|
|
overview: f(`第 ${n} 集简介`, "zh-CN"),
|
|
airDate: f("2022-02-17"),
|
|
runtime: f(55),
|
|
voteAverage: f(8),
|
|
...(still ? { stillPath: f(still) } : {}),
|
|
}),
|
|
})
|
|
.execute();
|
|
|
|
const rec = await organize(series, "Severance.S01E01.1080p.WEB-DL.H264-GRP.mkv");
|
|
await organize(series, "Severance.S01E02.1080p.WEB-DL.H264-GRP.mkv");
|
|
const q = h.engine.enqueued.filter((x) => x.task === SCRAPE_TASK);
|
|
// 照 MP:对作品目录刮,集只刮本次整理的文件;两批还没开始时合成一条
|
|
expect(q[0]!.key.startsWith(`${SCRAPE_TASK}:record:${rec}:`)).toBe(true);
|
|
const first = await h.db
|
|
.selectFrom("scrape_record")
|
|
.selectAll()
|
|
.where("id", "=", (q[0]!.input as { recordId: string }).recordId)
|
|
.executeTakeFirstOrThrow();
|
|
expect(first).toMatchObject({
|
|
media_entity_id: series,
|
|
trigger: "auto",
|
|
mode: "missing",
|
|
organization_record_id: rec,
|
|
});
|
|
|
|
const r = await runQueued(series);
|
|
expect(r.status).toBe("partial"); // 背景图下载失败
|
|
const files = await tree(lib.root);
|
|
const show = files.find((p) => p.endsWith("tvshow.nfo"))!;
|
|
const dir = show.slice(0, -"/tvshow.nfo".length);
|
|
const ep1 = files.find((p) => p.includes("S01E01") && p.endsWith(".mkv"))!;
|
|
const stem1 = ep1.slice(0, -4);
|
|
expect(files).toEqual(
|
|
expect.arrayContaining([
|
|
`${dir}/tvshow.nfo`,
|
|
`${dir}/poster.jpg`,
|
|
`${dir}/season01-poster.jpg`,
|
|
`${dir}/Season 1/season.nfo`,
|
|
`${dir}/Season 1/poster.jpg`,
|
|
`${stem1}.nfo`,
|
|
`${stem1}-thumb.jpg`,
|
|
]),
|
|
);
|
|
expect(files.some((p) => p.endsWith("fanart.jpg"))).toBe(false);
|
|
expect(files.filter((p) => p.endsWith(".nfo"))).toHaveLength(4);
|
|
expect(files.some((p) => p.includes(".umt-scrape.tmp"))).toBe(false);
|
|
const nfo = await readFile(join(lib.root, show), "utf8");
|
|
expect(nfo.startsWith('<?xml version="1.0" encoding="utf-8"?>\n<tvshow>')).toBe(true);
|
|
expect(nfo).toContain('<uniqueid type="imdb" default="true">tt11280740</uniqueid>');
|
|
expect(nfo).toContain("<country>美国</country>");
|
|
const epNfo = await readFile(join(lib.root, `${stem1}.nfo`), "utf8");
|
|
expect(epNfo).toContain("<title>有关地狱的好消息</title>");
|
|
expect(epNfo).toMatch(/<season>1<\/season>\n {2}<episode>1<\/episode>/);
|
|
// 同一张季海报只取一次
|
|
expect(fetched.filter((p) => p === "/s1.jpg")).toHaveLength(1);
|
|
|
|
const st = await h.api("GET", `/library/${series}`);
|
|
expect(st.status).toBe(200);
|
|
expect(st.body.data.scrape).toMatchObject({
|
|
status: "partial",
|
|
trigger: "auto",
|
|
failed: 1,
|
|
reason: "scrape_image_failed",
|
|
});
|
|
expect(st.body.data.scrape.files.find((x: { role: string }) => x.role === "fanart")).toMatchObject({
|
|
action: "failed",
|
|
reason: "scrape_image_failed",
|
|
});
|
|
const audit = await h.db
|
|
.selectFrom("audit_event")
|
|
.select(["action", "target_id", "target_type"])
|
|
.where("action", "=", "organize.scrape")
|
|
.execute();
|
|
expect(audit.some((a) => a.target_type === "scrape_record" && a.target_id === st.body.data.scrape.id)).toBe(true);
|
|
}, 60_000);
|
|
|
|
it("只补缺失:已有文件(包括别人写的)不动;全部覆盖:改写", async () => {
|
|
const files = await tree(lib.root);
|
|
const show = join(lib.root, files.find((p) => p.endsWith("tvshow.nfo"))!);
|
|
await writeFile(show, "<tvshow>别的工具写的</tvshow>");
|
|
const again = await rescrape(series, "missing");
|
|
expect(await readFile(show, "utf8")).toBe("<tvshow>别的工具写的</tvshow>");
|
|
expect(again.files.find((x) => x.role === "tvshow_nfo")).toMatchObject({
|
|
action: "skipped",
|
|
reason: "scrape_exists",
|
|
});
|
|
const over = await rescrape(series, "overwrite");
|
|
expect(over.written).toBeGreaterThan(0);
|
|
expect(await readFile(show, "utf8")).toContain("<title>人生切割术</title>");
|
|
}, 60_000);
|
|
|
|
it("电影:<视频名>.nfo + 作品目录 poster / fanart;手动「重新刮削」接口排任务", async () => {
|
|
await h.setSettings("organize", { scrapeAfterImport: false });
|
|
movie = await seedEntity(h.db, { kind: "movie", title: "星际穿越", year: 2014 });
|
|
await h.db
|
|
.updateTable("media_entity")
|
|
.set({
|
|
metadata: JSON.stringify({
|
|
title: f("星际穿越", "zh-CN"),
|
|
overview: f("穿越虫洞。", "zh-CN"),
|
|
releaseDate: f("2014-11-05"),
|
|
runtime: f(169),
|
|
posterPath: f("/inter.jpg"),
|
|
backdropPath: f("/inter-bg.jpg"),
|
|
crew: f([{ id: 525, name: "克里斯托弗·诺兰", originalName: null, role: "Director", profilePath: null }]),
|
|
}),
|
|
})
|
|
.where("id", "=", movie)
|
|
.execute();
|
|
const before = h.engine.enqueued.length;
|
|
await organize(movie, "Interstellar.2014.1080p.BluRay.x264-GRP.mkv");
|
|
expect(h.engine.enqueued.slice(before).some((x) => x.task === SCRAPE_TASK)).toBe(false); // 关了自动刮削
|
|
|
|
const res = await h.api("POST", `/library/${movie}/scrape`, {});
|
|
expect(res.status).toBe(202);
|
|
expect(res.body.data).toMatchObject({ mediaEntityId: movie, deduplicated: false });
|
|
const dup = await h.api("POST", `/library/${movie}/scrape`, {});
|
|
expect(dup.body.data.deduplicated).toBe(true);
|
|
expect((await h.api("GET", `/library/${movie}`)).body.data.scrape.status).toBe("queued");
|
|
|
|
const r = await runQueued(movie);
|
|
expect(r.status).toBe("succeeded");
|
|
const files = await tree(lib.root);
|
|
const video = files.find((p) => p.includes("星际穿越") && p.endsWith(".mkv"))!;
|
|
const dir = video.slice(0, video.lastIndexOf("/"));
|
|
expect(files.filter((p) => p.startsWith(`${dir}/`))).toEqual(
|
|
[video, `${video.slice(0, -4)}.nfo`, `${dir}/fanart.jpg`, `${dir}/poster.jpg`].sort(),
|
|
);
|
|
const nfo = await readFile(join(lib.root, `${video.slice(0, -4)}.nfo`), "utf8");
|
|
expect(nfo).toContain('<director tmdbid="525">克里斯托弗·诺兰</director>');
|
|
expect(nfo).toContain("<runtime>169</runtime>");
|
|
}, 60_000);
|
|
|
|
it("蓝光原盘资产:原盘目录本身就是刮削目录,不取上一级", async () => {
|
|
const disc = await seedEntity(h.db, { kind: "movie", title: "原盘", year: 2020 });
|
|
const root = await h.db.selectFrom("library_root").select("id").executeTakeFirstOrThrow();
|
|
const asset = await h.db
|
|
.insertInto("media_asset")
|
|
.values({ media_entity_id: disc, kind: "bluray_dir" })
|
|
.returning("id")
|
|
.executeTakeFirstOrThrow();
|
|
await h.db
|
|
.insertInto("asset_location")
|
|
.values({ media_asset_id: asset.id, library_root_id: root.id, relative_path: "Movies/原盘 (2020)" })
|
|
.execute();
|
|
expect((await entityScrapeLocations(h.d, disc, "movie")).map((l) => l.path)).toEqual(["Movies/原盘 (2020)"]);
|
|
});
|
|
|
|
it("写入失败记原因;只登记库根不写文件;没有入库文件的作品 412", async () => {
|
|
const files = await tree(lib.root);
|
|
const video = files.find((p) => p.includes("星际穿越") && p.endsWith(".mkv"))!;
|
|
const dirAbs = join(lib.root, video.slice(0, video.lastIndexOf("/")));
|
|
if (process.getuid?.() !== 0) {
|
|
await chmod(dirAbs, 0o555);
|
|
const r = await rescrape(movie, "overwrite");
|
|
await chmod(dirAbs, 0o755);
|
|
expect(r.files.some((x) => x.reason === "scrape_write_failed")).toBe(true);
|
|
}
|
|
await h.db.updateTable("library_root").set({ management_mode: "register_only" }).execute();
|
|
const ro = await rescrape(movie, "overwrite");
|
|
expect(ro).toMatchObject({ status: "failed", reason: "scrape_root_read_only", written: 0 });
|
|
await h.db.updateTable("library_root").set({ management_mode: "managed" }).execute();
|
|
|
|
const empty = await seedEntity(h.db, { kind: "movie", title: "没入库", year: 2020 });
|
|
expect((await h.api("POST", `/library/${empty}/scrape`, {})).status).toBe(412);
|
|
expect(await scrapeEntity(h.d, empty, { trigger: "manual" })).toEqual([]);
|
|
}, 60_000);
|
|
});
|