feat(server): scrapeMedia 刮削任务,入库后自动刮削(SCR-1)
- 文件道任务 scrapeMedia:NFO 原子写入、图片经图片缓存下载(并发 4,失败不阻塞 NFO)、只补缺失 / 全部覆盖
- 设置 organize.scrapeAfterImport / scrapeMode / scrapeImages / scrapeEpisodeThumbs
- POST /library/{id}/scrape、详情 scrape 状态;files/scrape writeNfo 与 scrape-batch write 走同一任务
- 迁移 0057_media_scrape、原因码、审计 organize.scrape
- 与 MP 输出的 NFO 差分测试(apps/server/test/scrape)
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
861f799615
commit
f38eca419e
24 files changed
+1740
-15
No files matched your search
@@ -0,0 +1,299 @@
|
||||
import { chmod, readdir, readFile, stat, writeFile } from "node:fs/promises";
|
||||
import { join, relative } from "node:path";
|
||||
import { createTempTree, type TempTree } from "@umt/testkit";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { executeRecord, planImport } from "../services/organize.ts";
|
||||
import { runScrape, SCRAPE_TASK } from "../services/scrape.ts";
|
||||
import { createHarness, type Harness, pgAvailable, seedBase, seedEntity } from "./harness.ts";
|
||||
|
||||
// SCR-1 刮削:入库成功后自动排 scrapeMedia → 写 NFO 与图片(临时库根);只补缺失 / 全部覆盖;图片失败不阻塞 NFO;
|
||||
// 只登记库根不写;作品详情「重新刮削」接口与刮削状态。
|
||||
|
||||
const hasPg = await pgAvailable();
|
||||
const SIZE = 200_000;
|
||||
/** 最小 JPEG 头(按文件头认图) */
|
||||
const JPEG = new Uint8Array([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10, 0x4a, 0x46, 0x49, 0x46, 0x00]);
|
||||
const at = new Date().toISOString();
|
||||
const f = (value: unknown, language: string | null = null) => ({ value, source: "tmdb", language, at, locked: false });
|
||||
|
||||
async function tree(root: string): Promise<string[]> {
|
||||
const out: string[] = [];
|
||||
const walk = async (d: string) => {
|
||||
for (const n of (await readdir(d)).sort()) {
|
||||
const p = join(d, n);
|
||||
if ((await stat(p)).isDirectory()) await walk(p);
|
||||
else out.push(relative(root, p));
|
||||
}
|
||||
};
|
||||
await walk(root);
|
||||
return out.filter((p) => !p.startsWith(".")).sort();
|
||||
}
|
||||
|
||||
describe.skipIf(!hasPg)("刮削:NFO 与图片写进媒体库目录", () => {
|
||||
let h: Harness;
|
||||
let lib: TempTree;
|
||||
let src: TempTree;
|
||||
const fetched: string[] = [];
|
||||
|
||||
beforeAll(async () => {
|
||||
lib = await createTempTree({}, { sentinel: true, prefix: "umt-scrlib-" });
|
||||
src = await createTempTree(
|
||||
{
|
||||
"Severance.S01E01.1080p.WEB-DL.H264-GRP.mkv": SIZE,
|
||||
"Severance.S01E02.1080p.WEB-DL.H264-GRP.mkv": SIZE,
|
||||
"Interstellar.2014.1080p.BluRay.x264-GRP.mkv": SIZE,
|
||||
},
|
||||
{ prefix: "umt-scrsrc-" },
|
||||
);
|
||||
h = await createHarness("scrape", {
|
||||
async scrapeImage(p) {
|
||||
fetched.push(p);
|
||||
if (p === "/broken.jpg") throw new Error("TMDB 图片返回 HTTP 404");
|
||||
return { content: JPEG, mime: "image/jpeg" };
|
||||
},
|
||||
});
|
||||
await seedBase(h.db, { libraryPath: lib.root, cachePath: src.root, qbUrl: "http://127.0.0.1:9" });
|
||||
await h.setSettings("organize", { scrapeAfterImport: true });
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await h?.close();
|
||||
for (const t of [lib, src]) await t?.cleanup();
|
||||
});
|
||||
|
||||
async function organize(entity: string, file: string) {
|
||||
const plan = await planImport(h.d, {
|
||||
sourcePaths: [src.path(file)],
|
||||
mediaEntityId: entity,
|
||||
action: "copy",
|
||||
trigger: "manual",
|
||||
actorType: "user",
|
||||
});
|
||||
expect(plan.status).toBe("running");
|
||||
const ex = await executeRecord(h.d, plan.recordId!);
|
||||
expect(ex.status).toBe("succeeded");
|
||||
return plan.recordId!;
|
||||
}
|
||||
|
||||
/** 按引擎记录的入队项跑刮削(假引擎不执行) */
|
||||
async function runQueued(entity: string) {
|
||||
const q = h.engine.enqueued.filter(
|
||||
(x) => x.task === SCRAPE_TASK && (x.input as { mediaEntityId: string }).mediaEntityId === entity,
|
||||
);
|
||||
expect(q.length).toBeGreaterThan(0);
|
||||
return runScrape(h.d, q[q.length - 1]!.input as Parameters<typeof runScrape>[1]);
|
||||
}
|
||||
|
||||
let series = "";
|
||||
let movie = "";
|
||||
|
||||
it("剧集:入库后自动排刮削,写 tvshow / 季 / 集 NFO、海报、季海报、剧照", async () => {
|
||||
series = await seedEntity(h.db, { kind: "series", title: "人生切割术", year: 2022 });
|
||||
await h.db
|
||||
.updateTable("media_entity")
|
||||
.set({
|
||||
original_title: "Severance",
|
||||
metadata: JSON.stringify({
|
||||
title: f("人生切割术", "zh-CN"),
|
||||
originalTitle: f("Severance", "en"),
|
||||
year: f(2022),
|
||||
releaseDate: f("2022-02-17"),
|
||||
overview: f("马克领导着一个办公室团队。", "zh-CN"),
|
||||
genres: f(["剧情", "悬疑"]),
|
||||
voteAverage: f(8.4),
|
||||
originCountry: f(["US"]),
|
||||
platform: f("Apple TV+"),
|
||||
posterPath: f("/sev.jpg"),
|
||||
backdropPath: f("/broken.jpg"),
|
||||
cast: f([
|
||||
{ id: 4495, name: "亚当·斯科特", originalName: null, role: "Mark Scout", profilePath: "/adam.jpg" },
|
||||
]),
|
||||
crew: [],
|
||||
externalIds: f({ imdb: "tt11280740", tvdb: "371980", wikidata: null }),
|
||||
}),
|
||||
})
|
||||
.where("id", "=", series)
|
||||
.execute();
|
||||
await h.db
|
||||
.insertInto("external_identity")
|
||||
.values({
|
||||
media_entity_id: series,
|
||||
provider: "tmdb",
|
||||
entity_type: "tv",
|
||||
external_id: "95396",
|
||||
status: "accepted",
|
||||
})
|
||||
.execute();
|
||||
const season = await h.db
|
||||
.insertInto("content_unit")
|
||||
.values({
|
||||
media_entity_id: series,
|
||||
kind: "season",
|
||||
season_number: 1,
|
||||
status: "confirmed",
|
||||
title: "第 1 季",
|
||||
metadata: JSON.stringify({ name: f("第 1 季", "zh-CN"), airDate: f("2022-02-17"), posterPath: f("/s1.jpg") }),
|
||||
})
|
||||
.returning("id")
|
||||
.executeTakeFirstOrThrow();
|
||||
for (const [n, name, still] of [
|
||||
[1, "有关地狱的好消息", "/e1.jpg"],
|
||||
[2, "半循环", null],
|
||||
] as const)
|
||||
await h.db
|
||||
.insertInto("content_unit")
|
||||
.values({
|
||||
media_entity_id: series,
|
||||
parent_unit_id: season.id,
|
||||
kind: "episode",
|
||||
season_number: 1,
|
||||
episode_number: n,
|
||||
status: "confirmed",
|
||||
title: name,
|
||||
air_date: "2022-02-17",
|
||||
metadata: JSON.stringify({
|
||||
name: f(name, "zh-CN"),
|
||||
overview: f(`第 ${n} 集简介`, "zh-CN"),
|
||||
airDate: f("2022-02-17"),
|
||||
runtime: f(55),
|
||||
voteAverage: f(8),
|
||||
...(still ? { stillPath: f(still) } : {}),
|
||||
}),
|
||||
})
|
||||
.execute();
|
||||
|
||||
const rec = await organize(series, "Severance.S01E01.1080p.WEB-DL.H264-GRP.mkv");
|
||||
await organize(series, "Severance.S01E02.1080p.WEB-DL.H264-GRP.mkv");
|
||||
const q = h.engine.enqueued.filter((x) => x.task === SCRAPE_TASK);
|
||||
expect(q[0]!.key).toBe(`${SCRAPE_TASK}:record:${rec}`);
|
||||
expect(q[0]!.input).toMatchObject({ mediaEntityId: series, trigger: "auto", mode: "missing", recordId: rec });
|
||||
|
||||
const r = await runQueued(series);
|
||||
expect(r.status).toBe("partial"); // 背景图下载失败
|
||||
const files = await tree(lib.root);
|
||||
const show = files.find((p) => p.endsWith("tvshow.nfo"))!;
|
||||
const dir = show.slice(0, -"/tvshow.nfo".length);
|
||||
const ep1 = files.find((p) => p.includes("S01E01") && p.endsWith(".mkv"))!;
|
||||
const stem1 = ep1.slice(0, -4);
|
||||
expect(files).toEqual(
|
||||
expect.arrayContaining([
|
||||
`${dir}/tvshow.nfo`,
|
||||
`${dir}/poster.jpg`,
|
||||
`${dir}/season01-poster.jpg`,
|
||||
`${dir}/Season 1/season.nfo`,
|
||||
`${dir}/Season 1/poster.jpg`,
|
||||
`${stem1}.nfo`,
|
||||
`${stem1}-thumb.jpg`,
|
||||
]),
|
||||
);
|
||||
expect(files.some((p) => p.endsWith("fanart.jpg"))).toBe(false);
|
||||
expect(files.filter((p) => p.endsWith(".nfo"))).toHaveLength(4);
|
||||
expect(files.some((p) => p.includes(".umt-scrape.tmp"))).toBe(false);
|
||||
const nfo = await readFile(join(lib.root, show), "utf8");
|
||||
expect(nfo.startsWith('<?xml version="1.0" encoding="utf-8"?>\n<tvshow>')).toBe(true);
|
||||
expect(nfo).toContain('<uniqueid type="imdb" default="true">tt11280740</uniqueid>');
|
||||
expect(nfo).toContain("<country>美国</country>");
|
||||
const epNfo = await readFile(join(lib.root, `${stem1}.nfo`), "utf8");
|
||||
expect(epNfo).toContain("<title>有关地狱的好消息</title>");
|
||||
expect(epNfo).toMatch(/<season>1<\/season>\n {2}<episode>1<\/episode>/);
|
||||
// 同一张季海报只取一次
|
||||
expect(fetched.filter((p) => p === "/s1.jpg")).toHaveLength(1);
|
||||
|
||||
const st = await h.api("GET", `/library/${series}`);
|
||||
expect(st.status).toBe(200);
|
||||
expect(st.body.data.scrape).toMatchObject({
|
||||
status: "partial",
|
||||
trigger: "auto",
|
||||
failed: 1,
|
||||
reason: "scrape_image_failed",
|
||||
});
|
||||
expect(st.body.data.scrape.files.find((x: { role: string }) => x.role === "fanart")).toMatchObject({
|
||||
action: "failed",
|
||||
reason: "scrape_image_failed",
|
||||
});
|
||||
const audit = await h.db
|
||||
.selectFrom("audit_event")
|
||||
.select(["action", "target_id"])
|
||||
.where("action", "=", "organize.scrape")
|
||||
.execute();
|
||||
expect(audit.some((a) => a.target_id === series)).toBe(true);
|
||||
}, 60_000);
|
||||
|
||||
it("只补缺失:已有文件(包括别人写的)不动;全部覆盖:改写", async () => {
|
||||
const files = await tree(lib.root);
|
||||
const show = join(lib.root, files.find((p) => p.endsWith("tvshow.nfo"))!);
|
||||
await writeFile(show, "<tvshow>别的工具写的</tvshow>");
|
||||
const again = await runScrape(h.d, { mediaEntityId: series, trigger: "manual", mode: "missing" });
|
||||
expect(await readFile(show, "utf8")).toBe("<tvshow>别的工具写的</tvshow>");
|
||||
expect(again.files.find((x) => x.role === "tvshow_nfo")).toMatchObject({
|
||||
action: "skipped",
|
||||
reason: "scrape_exists",
|
||||
});
|
||||
const over = await runScrape(h.d, { mediaEntityId: series, trigger: "manual", mode: "overwrite" });
|
||||
expect(over.written).toBeGreaterThan(0);
|
||||
expect(await readFile(show, "utf8")).toContain("<title>人生切割术</title>");
|
||||
}, 60_000);
|
||||
|
||||
it("电影:<视频名>.nfo + 作品目录 poster / fanart;手动「重新刮削」接口排任务", async () => {
|
||||
await h.setSettings("organize", { scrapeAfterImport: false });
|
||||
movie = await seedEntity(h.db, { kind: "movie", title: "星际穿越", year: 2014 });
|
||||
await h.db
|
||||
.updateTable("media_entity")
|
||||
.set({
|
||||
metadata: JSON.stringify({
|
||||
title: f("星际穿越", "zh-CN"),
|
||||
overview: f("穿越虫洞。", "zh-CN"),
|
||||
releaseDate: f("2014-11-05"),
|
||||
runtime: f(169),
|
||||
posterPath: f("/inter.jpg"),
|
||||
backdropPath: f("/inter-bg.jpg"),
|
||||
crew: f([{ id: 525, name: "克里斯托弗·诺兰", originalName: null, role: "Director", profilePath: null }]),
|
||||
}),
|
||||
})
|
||||
.where("id", "=", movie)
|
||||
.execute();
|
||||
const before = h.engine.enqueued.length;
|
||||
await organize(movie, "Interstellar.2014.1080p.BluRay.x264-GRP.mkv");
|
||||
expect(h.engine.enqueued.slice(before).some((x) => x.task === SCRAPE_TASK)).toBe(false); // 关了自动刮削
|
||||
|
||||
const res = await h.api("POST", `/library/${movie}/scrape`, {});
|
||||
expect(res.status).toBe(202);
|
||||
expect(res.body.data).toMatchObject({ mediaEntityId: movie, deduplicated: false });
|
||||
const dup = await h.api("POST", `/library/${movie}/scrape`, {});
|
||||
expect(dup.body.data.deduplicated).toBe(true);
|
||||
expect((await h.api("GET", `/library/${movie}`)).body.data.scrape.status).toBe("queued");
|
||||
|
||||
const r = await runQueued(movie);
|
||||
expect(r.status).toBe("succeeded");
|
||||
const files = await tree(lib.root);
|
||||
const video = files.find((p) => p.includes("星际穿越") && p.endsWith(".mkv"))!;
|
||||
const dir = video.slice(0, video.lastIndexOf("/"));
|
||||
expect(files.filter((p) => p.startsWith(`${dir}/`))).toEqual(
|
||||
[video, `${video.slice(0, -4)}.nfo`, `${dir}/fanart.jpg`, `${dir}/poster.jpg`].sort(),
|
||||
);
|
||||
const nfo = await readFile(join(lib.root, `${video.slice(0, -4)}.nfo`), "utf8");
|
||||
expect(nfo).toContain('<director tmdbid="525">克里斯托弗·诺兰</director>');
|
||||
expect(nfo).toContain("<runtime>169</runtime>");
|
||||
}, 60_000);
|
||||
|
||||
it("写入失败记原因;只登记库根不写文件;没有入库文件的作品 409", async () => {
|
||||
const files = await tree(lib.root);
|
||||
const video = files.find((p) => p.includes("星际穿越") && p.endsWith(".mkv"))!;
|
||||
const dirAbs = join(lib.root, video.slice(0, video.lastIndexOf("/")));
|
||||
if (process.getuid?.() !== 0) {
|
||||
await chmod(dirAbs, 0o555);
|
||||
const r = await runScrape(h.d, { mediaEntityId: movie, trigger: "manual", mode: "overwrite" });
|
||||
await chmod(dirAbs, 0o755);
|
||||
expect(r.files.some((x) => x.reason === "scrape_write_failed")).toBe(true);
|
||||
}
|
||||
await h.db.updateTable("library_root").set({ management_mode: "register_only" }).execute();
|
||||
const ro = await runScrape(h.d, { mediaEntityId: movie, trigger: "manual", mode: "overwrite" });
|
||||
expect(ro).toMatchObject({ status: "failed", reason: "scrape_root_read_only", written: 0 });
|
||||
await h.db.updateTable("library_root").set({ management_mode: "managed" }).execute();
|
||||
|
||||
const empty = await seedEntity(h.db, { kind: "movie", title: "没入库", year: 2020 });
|
||||
expect((await h.api("POST", `/library/${empty}/scrape`, {})).status).toBe(412);
|
||||
const none = await runScrape(h.d, { mediaEntityId: empty, trigger: "manual" });
|
||||
expect(none).toMatchObject({ status: "failed", reason: "scrape_no_metadata" });
|
||||
}, 60_000);
|
||||
});
|
||||
@@ -5,6 +5,7 @@ import { type AoDeps, automationBlock, ensureTaskRow, markTask } from "./service
|
||||
import { cleanupDownloadCache, submitDownload } from "./services/downloads.ts";
|
||||
import { executeRecord, planImport } from "./services/organize.ts";
|
||||
import { reconcileLibrary } from "./services/reconcile.ts";
|
||||
import { runScrape, SCRAPE_TASK } from "./services/scrape.ts";
|
||||
import { errMessage } from "./util.ts";
|
||||
|
||||
// 持久任务处理器。输入只放业务 id;业务状态写业务表;副作用经 withAttemptToken(在 services 里)。
|
||||
@@ -187,6 +188,25 @@ export function registerAoTasks(registry: TaskRegistry, d: AoDeps) {
|
||||
);
|
||||
},
|
||||
}),
|
||||
// SCR-1 刮削:写 NFO 与图片到媒体库目录(文件道);结果写 media_scrape,不重试(失败的文件在作品详情里重新刮削)
|
||||
defineTask({
|
||||
name: SCRAPE_TASK,
|
||||
version: 1,
|
||||
lane: "file",
|
||||
inputSchema: z.object({
|
||||
mediaEntityId: id,
|
||||
trigger: z.enum(["auto", "manual"]),
|
||||
mode: z.enum(["missing", "overwrite"]).optional(),
|
||||
recordId: id.optional(),
|
||||
actor: z.object({ type: z.string(), id: z.string().nullable() }).optional(),
|
||||
}),
|
||||
retryPolicy: { maximumAttempts: 1 },
|
||||
handler: async (t) =>
|
||||
t.step.run("scrape", async () => {
|
||||
const r = await runScrape(d, { ...t.input, runId: t.runId }, t.abortSignal);
|
||||
return { status: r.status, written: r.written, skipped: r.skipped, failed: r.failed, reason: r.reason };
|
||||
}),
|
||||
}),
|
||||
defineTask({
|
||||
name: "reconcileLibrary",
|
||||
version: 1,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
createEmbyClient,
|
||||
createQbClient,
|
||||
@@ -18,6 +19,9 @@ import {
|
||||
import type { ActorType } from "@umt/contracts";
|
||||
import type { DownloaderInstanceRow, MediaServerInstanceRow, SiteRow, SourceReleaseRow } from "@umt/db";
|
||||
import type { ParsedRelease } from "@umt/domain";
|
||||
import { dataPaths } from "../../config.ts";
|
||||
import { externalFetch } from "../../egress.ts";
|
||||
import { ImageCache } from "../cache/images.ts";
|
||||
import { SourcesRuntime } from "../sources-identity/runtime.ts";
|
||||
import {
|
||||
createEntityFromTmdbCandidate,
|
||||
@@ -79,6 +83,11 @@ export interface AoPorts {
|
||||
ref: { type: "movie" | "tv"; id: string },
|
||||
actor: { type: string; id: string | null },
|
||||
): Promise<string>;
|
||||
/**
|
||||
* SCR-1 刮削取图:TMDB 图片路径 → 字节(原图,经图片缓存 DATA_DIR/cache/images 与出口代理)。
|
||||
* 失败抛错(由刮削记为该图片失败,不影响 NFO)。
|
||||
*/
|
||||
scrapeImage(tmdbPath: string): Promise<{ content: Uint8Array; mime: string }>;
|
||||
options: AoOptions;
|
||||
}
|
||||
|
||||
@@ -113,6 +122,7 @@ export function createPorts(
|
||||
}
|
||||
};
|
||||
|
||||
let images: ImageCache | null = null;
|
||||
let sources: SourcesRuntime | null = null;
|
||||
const rt = () => {
|
||||
sources ??= new SourcesRuntime(ctx);
|
||||
@@ -179,6 +189,14 @@ export function createPorts(
|
||||
if (t.kind === "magnet") return { kind: "magnet", uri: t.uri };
|
||||
return { kind: "url", url: t.url };
|
||||
},
|
||||
async scrapeImage(tmdbPath) {
|
||||
images ??= new ImageCache({
|
||||
dir: join(dataPaths(ctx.config).dataDir, "cache", "images"),
|
||||
fetch: externalFetch,
|
||||
timeoutMs: 30_000,
|
||||
});
|
||||
return images.bytes("original", tmdbPath);
|
||||
},
|
||||
diskUsage: (p) => diskUsage(p),
|
||||
probe: (p) => ffprobe(p),
|
||||
async mediaServer(inst) {
|
||||
|
||||
@@ -14,6 +14,8 @@ import {
|
||||
type LibraryItem,
|
||||
LibraryItem as LibraryItemSchema,
|
||||
LibraryListQuery,
|
||||
LibraryScrapeRequest,
|
||||
LibraryScrapeResult,
|
||||
type MediaAsset,
|
||||
type MediaEntity,
|
||||
MediaServerInventory,
|
||||
@@ -25,6 +27,7 @@ import { sql } from "@umt/db";
|
||||
import { ApiException } from "../../../http/errors.ts";
|
||||
import type { ModuleApp } from "../../types.ts";
|
||||
import { type AoDeps, CANDIDATE_ARTIFACT_TITLE } from "../services/common.ts";
|
||||
import { enqueueScrape, loadScrapeState } from "../services/scrape.ts";
|
||||
import { notFound, toApi, toApiList } from "../util.ts";
|
||||
import { mount, paginate } from "./helpers.ts";
|
||||
import { registerLibraryVersionRoutes } from "./library-versions.ts";
|
||||
@@ -404,12 +407,48 @@ export function registerLibraryRoutes(app: ModuleApp, d: AoDeps) {
|
||||
createdAt: new Date(r.created_at).toISOString(),
|
||||
finishedAt: r.finished_at ? new Date(r.finished_at).toISOString() : null,
|
||||
})),
|
||||
scrape: await loadScrapeState(d, id),
|
||||
};
|
||||
return detail;
|
||||
},
|
||||
{ tag: TAG },
|
||||
);
|
||||
|
||||
// SCR-1 刮削:作品详情「重新刮削」;与入库后自动刮削、文件管理「刮削」同一任务 scrapeMedia
|
||||
mount(
|
||||
app,
|
||||
{
|
||||
method: "post",
|
||||
path: "/library/{id}/scrape",
|
||||
scope: "organize.execute",
|
||||
summary: "刮削：把作品的 NFO 与图片写进媒体库目录（后台任务 scrapeMedia；覆盖方式缺省按设置）",
|
||||
params: IdParam,
|
||||
body: LibraryScrapeRequest,
|
||||
response: LibraryScrapeResult,
|
||||
},
|
||||
async (c, { params, body }) => {
|
||||
const e = await db().selectFrom("media_entity").select(["id"]).where("id", "=", params.id).executeTakeFirst();
|
||||
if (!e) notFound("作品");
|
||||
const has = await db()
|
||||
.selectFrom("media_asset")
|
||||
.select(["id"])
|
||||
.where("media_entity_id", "=", params.id)
|
||||
.where("status", "=", "active")
|
||||
.executeTakeFirst();
|
||||
if (!has) throw new ApiException("precondition_failed", "作品没有已入库文件。先整理入库，再刮削");
|
||||
const actor = c.get("actor");
|
||||
const req = (body ?? {}) as LibraryScrapeRequest;
|
||||
const r = await enqueueScrape(d, {
|
||||
mediaEntityId: params.id,
|
||||
trigger: "manual",
|
||||
mode: req.mode,
|
||||
actor: { type: actor?.type === "api_key" ? "api_key" : "user", id: actor?.id ?? null },
|
||||
});
|
||||
return { mediaEntityId: params.id, ...r };
|
||||
},
|
||||
{ tag: TAG, status: 202 },
|
||||
);
|
||||
|
||||
mount(
|
||||
app,
|
||||
{
|
||||
|
||||
@@ -47,6 +47,7 @@ import { effectiveNaming, ensureNamingVersion, loadNamingEntity, namingByVersion
|
||||
import { PathGuardError } from "./organize-guard.ts";
|
||||
import { type FrozenTransfer, finalizeTransfer, planTransfer, type TransferSpec } from "./organize-transfer.ts";
|
||||
import { hnrSatisfied } from "./retention.ts";
|
||||
import { scrapeAfterImport } from "./scrape.ts";
|
||||
import { slotComplete as sharedSlotComplete, subscriptionLibraryRoot } from "./subscriptions.ts";
|
||||
import {
|
||||
CONFLICT_OPTIONS,
|
||||
@@ -1710,6 +1711,9 @@ export async function finalizeRecord(d: AoDeps, recordId: string): Promise<Execu
|
||||
return { recordId, status, succeeded: counts.file_succeeded, failed: counts.file_failed, move, dt: dt ?? null };
|
||||
});
|
||||
|
||||
// SCR-1:有资产入库时按设置自动刮削(写 NFO 与图片;排任务,不在这里等)
|
||||
if (result.status === "succeeded" || result.status === "partial")
|
||||
await scrapeAfterImport(d, frozenPre.entityId ?? null, recordId);
|
||||
// Move 完成:移除 qB 任务记录,结束下载侧管理(不删数据)
|
||||
if (result.move && result.dt) {
|
||||
const topo = (await loadInstances(db, { enabledOnly: false, ids: [result.dt.instance_id] }))[0];
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
import type { NfoEpisode, NfoIds, NfoPerson, NfoSeason, NfoWork } from "@umt/domain";
|
||||
|
||||
// SCR-1:本地作品 / 季 / 集的最终字段(media_entity.metadata、content_unit.metadata,字段级 {value, …})→ NFO 输入。
|
||||
// 纯函数,不查库;差分测试(apps/server/test/scrape/nfo-mp-diff.test.ts)直接调用。
|
||||
|
||||
type Final = Record<string, { value?: unknown } | undefined>;
|
||||
|
||||
const val = (m: unknown, k: string): unknown => ((m ?? {}) as Final)[k]?.value;
|
||||
const str = (v: unknown): string | null => (typeof v === "string" && v.trim() ? v.trim() : null);
|
||||
const numOf = (v: unknown): number | null => (typeof v === "number" && Number.isFinite(v) ? v : null);
|
||||
const strs = (v: unknown): string[] =>
|
||||
Array.isArray(v) ? v.filter((x): x is string => typeof x === "string" && x.trim().length > 0) : [];
|
||||
|
||||
interface Credit {
|
||||
id?: unknown;
|
||||
name?: unknown;
|
||||
role?: unknown;
|
||||
profilePath?: unknown;
|
||||
}
|
||||
const credits = (v: unknown): Credit[] => (Array.isArray(v) ? (v as Credit[]) : []);
|
||||
const person = (c: Credit): NfoPerson | null => {
|
||||
const name = str(c.name);
|
||||
if (!name) return null;
|
||||
return { tmdbId: numOf(c.id), name, role: str(c.role), profilePath: str(c.profilePath) };
|
||||
};
|
||||
|
||||
/** 制片国:ISO 3166-1 代码 → 中文名(Intl);认不出的保留原码 */
|
||||
export function countryNames(codes: readonly string[], locale = "zh-CN"): string[] {
|
||||
let dn: Intl.DisplayNames | null = null;
|
||||
try {
|
||||
dn = new Intl.DisplayNames([locale], { type: "region" });
|
||||
} catch {
|
||||
dn = null;
|
||||
}
|
||||
return codes.map((c) => {
|
||||
if (!/^[A-Z]{2}$/.test(c)) return c;
|
||||
try {
|
||||
return dn?.of(c) ?? c;
|
||||
} catch {
|
||||
return c;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export interface EntityForNfo {
|
||||
kind: string;
|
||||
title: string;
|
||||
original_title: string | null;
|
||||
year: number | null;
|
||||
metadata: unknown;
|
||||
}
|
||||
|
||||
/** 已接受的外部身份:provider → id(tmdb / imdb / tvdb / douban) */
|
||||
export type IdentityMap = Partial<Record<"tmdb" | "imdb" | "tvdb" | "douban", string>>;
|
||||
|
||||
export function nfoWork(e: EntityForNfo, ids: IdentityMap): NfoWork {
|
||||
const m = e.metadata;
|
||||
const ext = (val(m, "externalIds") ?? {}) as { imdb?: unknown; tvdb?: unknown };
|
||||
const nfoIds: NfoIds = {
|
||||
tmdb: ids.tmdb ?? null,
|
||||
imdb: ids.imdb ?? str(ext.imdb),
|
||||
tvdb: ids.tvdb ?? str(ext.tvdb),
|
||||
douban: ids.douban ?? null,
|
||||
};
|
||||
const crew = credits(val(m, "crew"));
|
||||
const isTv = e.kind !== "movie";
|
||||
return {
|
||||
kind: isTv ? "tvshow" : "movie",
|
||||
title: str(val(m, "title")) ?? e.title,
|
||||
originalTitle: str(val(m, "originalTitle")) ?? e.original_title,
|
||||
year: numOf(val(m, "year")) ?? e.year,
|
||||
premiered: str(val(m, "releaseDate")),
|
||||
overview: str(val(m, "overview")),
|
||||
tagline: str(val(m, "tagline")),
|
||||
rating: numOf(val(m, "voteAverage")),
|
||||
genres: strs(val(m, "genres")),
|
||||
countries: countryNames(strs(val(m, "originCountry"))),
|
||||
studios: isTv
|
||||
? (str(val(m, "platform")) ?? "")
|
||||
.split(" / ")
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean)
|
||||
: [],
|
||||
runtime: isTv ? null : numOf(val(m, "runtime")),
|
||||
directors: crew
|
||||
.filter((c) => c.role === "Director")
|
||||
.map(person)
|
||||
.filter((p): p is NfoPerson => !!p),
|
||||
actors: credits(val(m, "cast"))
|
||||
.map(person)
|
||||
.filter((p): p is NfoPerson => !!p),
|
||||
ids: nfoIds,
|
||||
};
|
||||
}
|
||||
|
||||
export interface UnitForNfo {
|
||||
season_number: number | null;
|
||||
episode_number: number | null;
|
||||
title: string | null;
|
||||
air_date: string | null;
|
||||
metadata: unknown;
|
||||
}
|
||||
|
||||
export function nfoSeason(season: number, u: UnitForNfo | undefined): NfoSeason {
|
||||
const m = u?.metadata;
|
||||
return {
|
||||
season,
|
||||
title: str(val(m, "name")) ?? u?.title ?? null,
|
||||
overview: str(val(m, "overview")),
|
||||
airDate: str(val(m, "airDate")) ?? u?.air_date ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
/** TMDB 占位集名(「第 N 集」「Episode N」)按没有标题处理,NFO 里由生成器补「第 N 集」 */
|
||||
const PLACEHOLDER = /^(第\s*\d+\s*集|Episode\s+\d+)$/i;
|
||||
|
||||
export function nfoEpisode(season: number, episode: number, u: UnitForNfo | undefined): NfoEpisode {
|
||||
const m = u?.metadata;
|
||||
const title = str(val(m, "name")) ?? u?.title ?? null;
|
||||
return {
|
||||
season,
|
||||
episode,
|
||||
title: title && !PLACEHOLDER.test(title) ? title : null,
|
||||
overview: str(val(m, "overview")),
|
||||
airDate: str(val(m, "airDate")) ?? u?.air_date ?? null,
|
||||
rating: numOf(val(m, "voteAverage")),
|
||||
runtime: numOf(val(m, "runtime")),
|
||||
};
|
||||
}
|
||||
|
||||
/** 图片路径:作品海报 / 背景图,季海报,集剧照 */
|
||||
export function imagePathsOf(entityMeta: unknown) {
|
||||
return { posterPath: str(val(entityMeta, "posterPath")), backdropPath: str(val(entityMeta, "backdropPath")) };
|
||||
}
|
||||
export const unitPosterPath = (m: unknown) => str(val(m, "posterPath"));
|
||||
export const unitStillPath = (m: unknown) => str(val(m, "stillPath"));
|
||||
@@ -0,0 +1,484 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { lstat, mkdir, rename, rm, writeFile } from "node:fs/promises";
|
||||
import { dirname, isAbsolute, join, relative, resolve } from "node:path";
|
||||
import { checkRoot, DEFAULT_ROOT_SENTINEL } from "@umt/adapters";
|
||||
import {
|
||||
DEFAULT_SETTINGS,
|
||||
type LibraryScrapeState,
|
||||
type OrganizeSettings,
|
||||
type ScrapeFileResult,
|
||||
type ScrapeMode,
|
||||
} from "@umt/contracts";
|
||||
import { sql } from "@umt/db";
|
||||
import {
|
||||
episodeNfo,
|
||||
inferWorkDir,
|
||||
planScrapeTargets,
|
||||
type ScrapeTarget,
|
||||
type ScrapeVideo,
|
||||
seasonNfo,
|
||||
workNfo,
|
||||
} from "@umt/domain";
|
||||
import { errMessage, jsonb } from "../util.ts";
|
||||
import type { AoDeps } from "./common.ts";
|
||||
import {
|
||||
type IdentityMap,
|
||||
imagePathsOf,
|
||||
nfoEpisode,
|
||||
nfoSeason,
|
||||
nfoWork,
|
||||
type UnitForNfo,
|
||||
unitPosterPath,
|
||||
unitStillPath,
|
||||
} from "./scrape-input.ts";
|
||||
|
||||
// SCR-1 刮削:把作品的最终字段写成 NFO,下载海报 / 背景图 / 季海报 / 剧照,放进媒体库目录(Emby / Jellyfin / Kodi 读取)。
|
||||
// - 触发:整理入库成功后(设置 organize.scrapeAfterImport)、作品详情「重新刮削」、文件管理「刮削」,都排同一个任务 scrapeMedia(文件道)。
|
||||
// - 目标文件与命名:packages/domain/src/scrape/layout.ts(文件名跟随已入库视频,即命名模板的结果);内容:scrape/nfo.ts。
|
||||
// - 覆盖:missing 只补缺失(已存在的跳过)/ overwrite 全部覆盖;NFO 先写,图片后下(并发 4,失败只记该文件,不影响 NFO)。
|
||||
// - 写入:同目录临时文件 + rename(原子),UTF-8;只写 managed 库根,库根离线整体失败;目标必须仍在库根内。
|
||||
// - 结果:media_scrape 一部作品一行(最近一次,逐文件结果)+ 审计 organize.scrape;SSE files.changed 让文件管理刷新。
|
||||
|
||||
export const SCRAPE_TASK = "scrapeMedia";
|
||||
const IMAGE_CONCURRENCY = 4;
|
||||
|
||||
export interface ScrapeInput {
|
||||
mediaEntityId: string;
|
||||
trigger: "auto" | "manual";
|
||||
mode?: ScrapeMode | undefined;
|
||||
recordId?: string | undefined;
|
||||
/** 手动刮削的发起人(审计) */
|
||||
actor?: { type: string; id: string | null } | undefined;
|
||||
}
|
||||
|
||||
export async function scrapeSettings(d: AoDeps): Promise<OrganizeSettings> {
|
||||
const s = await d.ctx.settings.get("organize").catch(() => DEFAULT_SETTINGS.organize);
|
||||
return { ...DEFAULT_SETTINGS.organize, ...s };
|
||||
}
|
||||
|
||||
/**
|
||||
* 排刮削任务。同一作品已有排队中(还没开始)的刮削时直接复用(deduplicated);
|
||||
* 自动刮削按整理记录幂等(同一记录只排一次)。
|
||||
*/
|
||||
export async function enqueueScrape(d: AoDeps, input: ScrapeInput): Promise<{ taskId: string; deduplicated: boolean }> {
|
||||
const db = d.ctx.db;
|
||||
const mode = input.mode ?? (await scrapeSettings(d)).scrapeMode;
|
||||
const cur = await db
|
||||
.selectFrom("media_scrape")
|
||||
.select(["status", "engine_run_id", "requested_at"])
|
||||
.where("media_entity_id", "=", input.mediaEntityId)
|
||||
.executeTakeFirst();
|
||||
if (cur?.status === "queued" && cur.engine_run_id && Date.now() - new Date(cur.requested_at).getTime() < 30 * 60_000)
|
||||
return { taskId: cur.engine_run_id, deduplicated: true };
|
||||
const key =
|
||||
input.trigger === "auto" && input.recordId
|
||||
? `${SCRAPE_TASK}:record:${input.recordId}`
|
||||
: `${SCRAPE_TASK}:manual:${input.mediaEntityId}:${randomUUID()}`;
|
||||
const { runId, deduplicated } = await d.ctx.engine.enqueue(
|
||||
SCRAPE_TASK,
|
||||
{
|
||||
mediaEntityId: input.mediaEntityId,
|
||||
trigger: input.trigger,
|
||||
mode,
|
||||
...(input.recordId ? { recordId: input.recordId } : {}),
|
||||
...(input.actor ? { actor: input.actor } : {}),
|
||||
},
|
||||
{ idempotencyKey: key, traceId: randomUUID() },
|
||||
);
|
||||
if (!deduplicated)
|
||||
await sql`insert into media_scrape (media_entity_id, status, trigger, mode, record_id, engine_run_id, requested_at,
|
||||
started_at, finished_at, written, skipped, failed, reason, files)
|
||||
values (${input.mediaEntityId}, 'queued', ${input.trigger}, ${mode}, ${input.recordId ?? null}, ${runId}, now(),
|
||||
null, null, 0, 0, 0, null, '[]'::jsonb)
|
||||
on conflict (media_entity_id) do update set status = 'queued', trigger = excluded.trigger, mode = excluded.mode,
|
||||
record_id = excluded.record_id, engine_run_id = excluded.engine_run_id, requested_at = now(), updated_at = now()`.execute(
|
||||
db,
|
||||
);
|
||||
return { taskId: runId, deduplicated: !!deduplicated };
|
||||
}
|
||||
|
||||
/** 整理入库成功后:设置打开时自动排刮削(失败只记日志,不影响整理结果) */
|
||||
export async function scrapeAfterImport(d: AoDeps, entityId: string | null, recordId: string): Promise<void> {
|
||||
if (!entityId) return;
|
||||
try {
|
||||
const s = await scrapeSettings(d);
|
||||
if (!s.scrapeAfterImport) return;
|
||||
await enqueueScrape(d, { mediaEntityId: entityId, trigger: "auto", recordId });
|
||||
} catch (err) {
|
||||
d.ctx.logger.warn({ entityId, recordId, err: { message: errMessage(err) } }, "入库后自动刮削没排上");
|
||||
}
|
||||
}
|
||||
|
||||
export async function loadScrapeState(d: AoDeps, entityId: string): Promise<LibraryScrapeState | null> {
|
||||
const r = await d.ctx.db
|
||||
.selectFrom("media_scrape")
|
||||
.selectAll()
|
||||
.where("media_entity_id", "=", entityId)
|
||||
.executeTakeFirst();
|
||||
if (!r) return null;
|
||||
const iso = (t: Date | string | null) => (t ? new Date(t).toISOString() : null);
|
||||
return {
|
||||
status: r.status as LibraryScrapeState["status"],
|
||||
trigger: r.trigger as LibraryScrapeState["trigger"],
|
||||
mode: r.mode as ScrapeMode,
|
||||
requestedAt: iso(r.requested_at)!,
|
||||
startedAt: iso(r.started_at),
|
||||
finishedAt: iso(r.finished_at),
|
||||
written: r.written,
|
||||
skipped: r.skipped,
|
||||
failed: r.failed,
|
||||
reason: r.reason,
|
||||
files: (r.files ?? []) as ScrapeFileResult[],
|
||||
recordId: r.record_id,
|
||||
taskId: r.engine_run_id,
|
||||
};
|
||||
}
|
||||
|
||||
// ── 执行 ──
|
||||
|
||||
interface RootGroup {
|
||||
rootId: string;
|
||||
rootPath: string;
|
||||
managed: boolean;
|
||||
sentinel: string | null;
|
||||
videos: ScrapeVideo[];
|
||||
}
|
||||
|
||||
/** 原子写:同目录临时文件 → rename;目录不存在时建(只在库根内) */
|
||||
async function atomicWrite(abs: string, content: Uint8Array | string): Promise<void> {
|
||||
await mkdir(dirname(abs), { recursive: true });
|
||||
const tmp = join(dirname(abs), `.${randomUUID()}.umt-scrape.tmp`);
|
||||
try {
|
||||
await writeFile(tmp, content);
|
||||
await rename(tmp, abs);
|
||||
} catch (e) {
|
||||
await rm(tmp, { force: true }).catch(() => {});
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
const inside = (root: string, abs: string) => {
|
||||
const r = relative(root, abs);
|
||||
return !!r && !r.startsWith("..") && !isAbsolute(r);
|
||||
};
|
||||
|
||||
/** 有限并发 map */
|
||||
async function pool<T, R>(items: readonly T[], n: number, fn: (t: T) => Promise<R>): Promise<R[]> {
|
||||
const out: R[] = new Array(items.length);
|
||||
let i = 0;
|
||||
const worker = async () => {
|
||||
while (i < items.length) {
|
||||
const k = i++;
|
||||
out[k] = await fn(items[k]!);
|
||||
}
|
||||
};
|
||||
await Promise.all(Array.from({ length: Math.min(n, items.length) }, worker));
|
||||
return out;
|
||||
}
|
||||
|
||||
export interface ScrapeRunResult {
|
||||
status: LibraryScrapeState["status"];
|
||||
written: number;
|
||||
skipped: number;
|
||||
failed: number;
|
||||
reason: string | null;
|
||||
files: ScrapeFileResult[];
|
||||
}
|
||||
|
||||
export async function runScrape(
|
||||
d: AoDeps,
|
||||
input: ScrapeInput & { runId?: string },
|
||||
signal?: AbortSignal,
|
||||
): Promise<ScrapeRunResult> {
|
||||
const db = d.ctx.db;
|
||||
const settings = await scrapeSettings(d);
|
||||
const mode: ScrapeMode = input.mode ?? settings.scrapeMode;
|
||||
const entityId = input.mediaEntityId;
|
||||
const now = () => d.ports.now();
|
||||
await sql`insert into media_scrape (media_entity_id, status, trigger, mode, record_id, engine_run_id, started_at)
|
||||
values (${entityId}, 'running', ${input.trigger}, ${mode}, ${input.recordId ?? null}, ${input.runId ?? null}, ${now()})
|
||||
on conflict (media_entity_id) do update set status = 'running', trigger = excluded.trigger, mode = excluded.mode,
|
||||
record_id = coalesce(excluded.record_id, media_scrape.record_id),
|
||||
engine_run_id = coalesce(excluded.engine_run_id, media_scrape.engine_run_id),
|
||||
started_at = excluded.started_at, finished_at = null, reason = null, updated_at = now()`
|
||||
.execute(db)
|
||||
.catch(() => {});
|
||||
|
||||
const finish = async (r: ScrapeRunResult, rootDirs: Map<string, Set<string>> = new Map()) => {
|
||||
await db
|
||||
.updateTable("media_scrape")
|
||||
.set({
|
||||
status: r.status,
|
||||
finished_at: now(),
|
||||
written: r.written,
|
||||
skipped: r.skipped,
|
||||
failed: r.failed,
|
||||
reason: r.reason,
|
||||
files: jsonb(r.files),
|
||||
updated_at: now(),
|
||||
})
|
||||
.where("media_entity_id", "=", entityId)
|
||||
.execute()
|
||||
.catch(() => {});
|
||||
await db
|
||||
.insertInto("audit_event")
|
||||
.values({
|
||||
actor_type: input.actor?.type ?? "system",
|
||||
actor_id: input.actor?.id ?? null,
|
||||
action: "organize.scrape",
|
||||
target_type: "media_entity",
|
||||
target_id: entityId,
|
||||
payload: jsonb({
|
||||
trigger: input.trigger,
|
||||
mode,
|
||||
recordId: input.recordId ?? null,
|
||||
status: r.status,
|
||||
written: r.written,
|
||||
skipped: r.skipped,
|
||||
failed: r.failed,
|
||||
reason: r.reason,
|
||||
files: r.files.filter((f) => f.action !== "skipped").slice(0, 200),
|
||||
}),
|
||||
})
|
||||
.execute()
|
||||
.catch(() => {});
|
||||
for (const [rootId, dirs] of rootDirs)
|
||||
if (dirs.size) void d.ctx.events.publish("files.changed", { rootId: `lib:${rootId}`, dirs: [...dirs] });
|
||||
return r;
|
||||
};
|
||||
const fail = (reason: string) => finish({ status: "failed", written: 0, skipped: 0, failed: 0, reason, files: [] });
|
||||
|
||||
const e = await db
|
||||
.selectFrom("media_entity")
|
||||
.select(["id", "kind", "title", "original_title", "year", "metadata", "status"])
|
||||
.where("id", "=", entityId)
|
||||
.executeTakeFirst();
|
||||
if (!e || e.status === "merged") return fail("entity_not_found");
|
||||
const kind: "movie" | "tv" = e.kind === "movie" || e.kind === "collection" || e.kind === "other" ? "movie" : "tv";
|
||||
const meta = (e.metadata ?? {}) as Record<string, unknown>;
|
||||
if (!Object.keys(meta).length) return fail("scrape_no_metadata");
|
||||
|
||||
// 已入库的视频位置(库根内)
|
||||
const locs = await db
|
||||
.selectFrom("media_asset as a")
|
||||
.innerJoin("asset_location as l", "l.media_asset_id", "a.id")
|
||||
.innerJoin("library_root as r", "r.id", "l.library_root_id")
|
||||
.select([
|
||||
"a.id as asset_id",
|
||||
"l.relative_path",
|
||||
"r.id as root_id",
|
||||
"r.local_path",
|
||||
"r.management_mode",
|
||||
"r.enabled",
|
||||
"r.sentinel_file",
|
||||
])
|
||||
.where("a.media_entity_id", "=", entityId)
|
||||
.where("a.status", "=", "active")
|
||||
.where("a.kind", "=", "video")
|
||||
.where("l.location_kind", "=", "library")
|
||||
.where("l.status", "in", ["available", "unavailable", "suspected_missing"])
|
||||
.execute();
|
||||
if (!locs.length) return fail("scrape_no_files");
|
||||
|
||||
const assetIds = [...new Set(locs.map((l) => l.asset_id))];
|
||||
const cov = await db
|
||||
.selectFrom("asset_unit_coverage")
|
||||
.select(["media_asset_id", "content_unit_id"])
|
||||
.where("media_asset_id", "in", assetIds)
|
||||
.execute();
|
||||
const units = await db
|
||||
.selectFrom("content_unit")
|
||||
.select(["id", "kind", "season_number", "episode_number", "title", "air_date", "metadata"])
|
||||
.where("media_entity_id", "=", entityId)
|
||||
.where("status", "!=", "retired")
|
||||
.orderBy("created_at")
|
||||
.execute();
|
||||
const unitById = new Map(units.map((u) => [u.id, u]));
|
||||
const seasonUnit = new Map<number, UnitForNfo>();
|
||||
const episodeUnit = new Map<string, UnitForNfo>();
|
||||
for (const u of units) {
|
||||
if (u.kind === "season" && u.season_number !== null && !seasonUnit.has(u.season_number))
|
||||
seasonUnit.set(u.season_number, u);
|
||||
if (u.kind === "episode" && u.season_number !== null && u.episode_number !== null) {
|
||||
const k = `${u.season_number}:${u.episode_number}`;
|
||||
if (!episodeUnit.has(k)) episodeUnit.set(k, u);
|
||||
}
|
||||
}
|
||||
|
||||
const groups = new Map<string, RootGroup>();
|
||||
for (const l of locs) {
|
||||
if (!l.enabled) continue;
|
||||
const g = groups.get(l.root_id) ?? {
|
||||
rootId: l.root_id,
|
||||
rootPath: l.local_path,
|
||||
managed: l.management_mode === "managed",
|
||||
sentinel: l.sentinel_file,
|
||||
videos: [],
|
||||
};
|
||||
let season: number | null = null;
|
||||
let episode: number | null = null;
|
||||
if (kind === "tv") {
|
||||
const eps = cov
|
||||
.filter((c) => c.media_asset_id === l.asset_id)
|
||||
.map((c) => unitById.get(c.content_unit_id))
|
||||
.filter((u) => u?.kind === "episode" && u.season_number !== null && u.episode_number !== null)
|
||||
.sort((a, b) => a!.season_number! - b!.season_number! || a!.episode_number! - b!.episode_number!);
|
||||
season = eps[0]?.season_number ?? null;
|
||||
episode = eps[0]?.episode_number ?? null;
|
||||
}
|
||||
g.videos.push({ path: l.relative_path, season, episode });
|
||||
groups.set(l.root_id, g);
|
||||
}
|
||||
if (!groups.size) return fail("scrape_no_files");
|
||||
|
||||
const idRows = await db
|
||||
.selectFrom("external_identity")
|
||||
.select(["provider", "external_id"])
|
||||
.where("media_entity_id", "=", entityId)
|
||||
.where("content_unit_id", "is", null)
|
||||
.where("status", "=", "accepted")
|
||||
.orderBy("updated_at", "desc")
|
||||
.execute()
|
||||
.catch(() => [] as { provider: string; external_id: string }[]);
|
||||
const ids: IdentityMap = {};
|
||||
for (const r of idRows) {
|
||||
const p = r.provider.toLowerCase() as keyof IdentityMap;
|
||||
if (["tmdb", "imdb", "tvdb", "douban"].includes(p) && !ids[p]) ids[p] = r.external_id;
|
||||
}
|
||||
const bindings = await db
|
||||
.selectFrom("path_binding")
|
||||
.select(["library_root_id", "relative_path"])
|
||||
.where("media_entity_id", "=", entityId)
|
||||
.where("binding_kind", "=", "entity")
|
||||
.where("status", "=", "active")
|
||||
.execute();
|
||||
|
||||
const work = nfoWork(e, ids);
|
||||
const art = imagePathsOf(meta);
|
||||
const seasonPosters = new Map<number, string | null>();
|
||||
for (const [n, u] of seasonUnit) seasonPosters.set(n, unitPosterPath(u.metadata));
|
||||
const stills = new Map<string, string | null>();
|
||||
for (const [k, u] of episodeUnit) stills.set(k, unitStillPath(u.metadata));
|
||||
|
||||
const files: ScrapeFileResult[] = [];
|
||||
const rootDirs = new Map<string, Set<string>>();
|
||||
const imageCache = new Map<string, Promise<Uint8Array>>();
|
||||
const fetchImage = (p: string) => {
|
||||
let hit = imageCache.get(p);
|
||||
if (!hit) {
|
||||
hit = d.ports.scrapeImage(p).then((r) => r.content);
|
||||
imageCache.set(p, hit);
|
||||
}
|
||||
return hit;
|
||||
};
|
||||
|
||||
for (const g of groups.values()) {
|
||||
if (!g.managed) {
|
||||
files.push(
|
||||
...g.videos.map((v) => ({
|
||||
path: v.path,
|
||||
kind: "nfo" as const,
|
||||
role: kind === "movie" ? "movie_nfo" : "episode_nfo",
|
||||
action: "skipped" as const,
|
||||
reason: "scrape_root_read_only",
|
||||
})),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
const health = await checkRoot(g.rootPath, g.sentinel ?? DEFAULT_ROOT_SENTINEL);
|
||||
if (health.kind !== "online") {
|
||||
return finish({
|
||||
status: "failed",
|
||||
written: 0,
|
||||
skipped: 0,
|
||||
failed: 0,
|
||||
reason: "library_root_offline",
|
||||
files,
|
||||
});
|
||||
}
|
||||
const bound = bindings.find((b) => b.library_root_id === g.rootId)?.relative_path ?? null;
|
||||
const workDir = bound ?? inferWorkDir(kind, g.videos[0]!.path);
|
||||
const targets = planScrapeTargets({
|
||||
kind,
|
||||
workDir,
|
||||
videos: g.videos,
|
||||
images: { ...art, seasonPosters, stills },
|
||||
options: { images: settings.scrapeImages, episodeThumbs: settings.scrapeEpisodeThumbs },
|
||||
});
|
||||
if (kind === "tv" && !workDir)
|
||||
files.push({ path: "", kind: "nfo", role: "tvshow_nfo", action: "skipped", reason: "scrape_no_work_dir" });
|
||||
const rootAbs = resolve(g.rootPath);
|
||||
const dirs = rootDirs.get(g.rootId) ?? new Set<string>();
|
||||
rootDirs.set(g.rootId, dirs);
|
||||
|
||||
const decide = async (t: ScrapeTarget): Promise<{ abs: string } | ScrapeFileResult> => {
|
||||
const abs = resolve(rootAbs, t.path);
|
||||
const base = { path: t.path, kind: t.kind, role: t.role };
|
||||
if (!inside(rootAbs, abs)) return { ...base, action: "failed", reason: "path_escape" };
|
||||
const st = await lstat(abs).catch(() => null);
|
||||
if (st && (!st.isFile() || st.isSymbolicLink())) return { ...base, action: "skipped", reason: "scrape_exists" };
|
||||
if (st && mode === "missing") return { ...base, action: "skipped", reason: "scrape_exists" };
|
||||
return { abs };
|
||||
};
|
||||
const write = async (t: ScrapeTarget, abs: string, content: Uint8Array | string): Promise<ScrapeFileResult> => {
|
||||
const base = { path: t.path, kind: t.kind, role: t.role };
|
||||
try {
|
||||
await atomicWrite(abs, content);
|
||||
dirs.add(dirname(t.path) === "." ? "" : dirname(t.path));
|
||||
return { ...base, action: "written", reason: null };
|
||||
} catch (err) {
|
||||
d.ctx.logger.warn({ entityId, path: t.path, err: { message: errMessage(err) } }, "刮削文件写入失败");
|
||||
return { ...base, action: "failed", reason: "scrape_write_failed" };
|
||||
}
|
||||
};
|
||||
|
||||
// NFO 先写(不依赖网络)
|
||||
for (const t of targets.filter((x) => x.kind === "nfo")) {
|
||||
if (signal?.aborted) throw new Error("aborted");
|
||||
const dec = await decide(t);
|
||||
if (!("abs" in dec)) {
|
||||
files.push(dec);
|
||||
continue;
|
||||
}
|
||||
let xml: string;
|
||||
if (t.role === "movie_nfo" || t.role === "tvshow_nfo") xml = workNfo(work);
|
||||
else if (t.role === "season_nfo") xml = seasonNfo(nfoSeason(t.season!, seasonUnit.get(t.season!)));
|
||||
else xml = episodeNfo(nfoEpisode(t.season!, t.episode!, episodeUnit.get(`${t.season}:${t.episode}`)));
|
||||
files.push(await write(t, dec.abs, xml));
|
||||
}
|
||||
// 图片:限并发,失败不影响 NFO
|
||||
const imgs = targets.filter((x) => x.kind === "image" && x.imagePath);
|
||||
const results = await pool(imgs, IMAGE_CONCURRENCY, async (t): Promise<ScrapeFileResult> => {
|
||||
if (signal?.aborted) throw new Error("aborted");
|
||||
const dec = await decide(t);
|
||||
if (!("abs" in dec)) return dec;
|
||||
let bytes: Uint8Array;
|
||||
try {
|
||||
bytes = await fetchImage(t.imagePath!);
|
||||
} catch (err) {
|
||||
d.ctx.logger.warn({ entityId, image: t.imagePath, err: { message: errMessage(err) } }, "刮削图片下载失败");
|
||||
return { path: t.path, kind: t.kind, role: t.role, action: "failed", reason: "scrape_image_failed" };
|
||||
}
|
||||
return write(t, dec.abs, bytes);
|
||||
});
|
||||
files.push(...results);
|
||||
}
|
||||
|
||||
const written = files.filter((f) => f.action === "written").length;
|
||||
const failed = files.filter((f) => f.action === "failed").length;
|
||||
const skipped = files.filter((f) => f.action === "skipped").length;
|
||||
const onlyReadOnly = files.length > 0 && files.every((f) => f.reason === "scrape_root_read_only");
|
||||
const status: ScrapeRunResult["status"] = onlyReadOnly
|
||||
? "failed"
|
||||
: failed === 0
|
||||
? "succeeded"
|
||||
: written > 0 || skipped > 0
|
||||
? "partial"
|
||||
: "failed";
|
||||
const reason = onlyReadOnly
|
||||
? "scrape_root_read_only"
|
||||
: failed
|
||||
? (files.find((f) => f.action === "failed")?.reason ?? null)
|
||||
: null;
|
||||
return finish({ status, written, skipped, failed, reason, files }, rootDirs);
|
||||
}
|
||||
+8
@@ -107,6 +107,14 @@ export class ImageCache {
|
||||
return new Response(content as BodyInit, { status: 200, headers });
|
||||
}
|
||||
|
||||
/**
|
||||
* SCR-1 刮削取图:TMDB 路径(`/xx.jpg`)→ 图片字节。同一缓存目录与出口代理;命中磁盘缓存就不再请求
|
||||
* (TMDB 图片路径内容不变,刮削不看 GLOBAL_IMAGE_CACHE 开关)。
|
||||
*/
|
||||
async bytes(size: string, tmdbPath: string): Promise<{ content: Uint8Array; mime: string }> {
|
||||
return this.fetchImage(this.upstream(size, tmdbPath.replace(/^\/+/, "")), true);
|
||||
}
|
||||
|
||||
/** ImageHelper.fetch_image_with_mime_type */
|
||||
private async fetchImage(url: string, useCache: boolean): Promise<{ content: Uint8Array; mime: string }> {
|
||||
const path = join(this.opts.dir, imageCachePath(url));
|
||||
|
||||
@@ -13,6 +13,7 @@ import type {
|
||||
} from "@umt/contracts";
|
||||
import { sql } from "@umt/db";
|
||||
import type { Principal } from "../../http/auth.ts";
|
||||
import { enqueueScrape } from "../acquisition-organize/services/scrape.ts";
|
||||
import { FilesError, type FmDeps, jsonb, resolveIn } from "./common.ts";
|
||||
import { audit } from "./ledger.ts";
|
||||
import { kindOfName, mediaSummary } from "./list.ts";
|
||||
@@ -20,7 +21,8 @@ import { entitiesUnder } from "./records.ts";
|
||||
|
||||
// probe:ffprobe 摘要;已登记资产写回 media_asset.media_info.probe,未登记只缓存 24h(DATA_DIR/cache/probe)。
|
||||
// scrape:只对已登记作品,等价 POST /media-entities/{id}/refresh(入队 refreshMetadata,字段锁优先、不动路径);
|
||||
// NFO/图片投影 W5 实现(TODO),请求时返回 not_implemented。
|
||||
// writeNfo / images(SCR-1)另排 scrapeMedia,把 NFO 与图片写进媒体库目录(与作品详情「重新刮削」同一任务)。
|
||||
// scrape-batch write=true:文件管理「刮削」,按作品去重各排一次 scrapeMedia(不刷新元数据)。
|
||||
|
||||
const CACHE_TTL_MS = 24 * 3600_000;
|
||||
|
||||
@@ -133,13 +135,23 @@ export async function scrape(
|
||||
.where("locked", "=", true)
|
||||
.execute();
|
||||
const lockedFields = [...new Set([...(e.field_locks ?? []), ...overrides.map((o) => o.field)])].sort();
|
||||
// TODO(W5):writeNfo/images → 把最终字段投影成 NFO/海报写到绑定目录(临时文件 + 原子提交;非本系统 NFO 不覆盖)
|
||||
const scrapeTaskId =
|
||||
body.writeNfo || body.images
|
||||
? (
|
||||
await enqueueScrape(d.ao, {
|
||||
mediaEntityId: e.id,
|
||||
trigger: "manual",
|
||||
actor: { type: principal?.kind === "api_key" ? "api_key" : "user", id: principal?.id ?? null },
|
||||
})
|
||||
).taskId
|
||||
: null;
|
||||
const result: FilesScrapeResult = {
|
||||
mediaEntityId: e.id,
|
||||
taskId: runId,
|
||||
lockedFields,
|
||||
nfo: body.writeNfo ? "not_implemented" : "not_requested",
|
||||
images: body.images ? "not_implemented" : "not_requested",
|
||||
nfo: body.writeNfo ? "queued" : "not_requested",
|
||||
images: body.images ? "queued" : "not_requested",
|
||||
scrapeTaskId,
|
||||
};
|
||||
await audit(d, principal, "scrape", { ...result });
|
||||
return result;
|
||||
@@ -202,17 +214,28 @@ export async function scrapeBatch(
|
||||
skipped.push({ ref: id, reason: "not_registered" });
|
||||
continue;
|
||||
}
|
||||
const { runId } = await d.ctx.engine.enqueue(
|
||||
"refreshMetadata",
|
||||
{ mediaEntityId: id },
|
||||
{ idempotencyKey: `refreshMetadata:files-batch:${body.idempotencyKey}:${id}`, traceId: randomUUID() },
|
||||
);
|
||||
const runId = body.write
|
||||
? (
|
||||
await enqueueScrape(d.ao, {
|
||||
mediaEntityId: id,
|
||||
trigger: "manual",
|
||||
actor: { type: principal?.kind === "api_key" ? "api_key" : "user", id: principal?.id ?? null },
|
||||
})
|
||||
).taskId
|
||||
: (
|
||||
await d.ctx.engine.enqueue(
|
||||
"refreshMetadata",
|
||||
{ mediaEntityId: id },
|
||||
{ idempotencyKey: `refreshMetadata:files-batch:${body.idempotencyKey}:${id}`, traceId: randomUUID() },
|
||||
)
|
||||
).runId;
|
||||
const lockedFields = [
|
||||
...new Set([...(e.field_locks ?? []), ...locks.filter((l) => l.media_entity_id === id).map((l) => l.field)]),
|
||||
].sort();
|
||||
submitted.push({ mediaEntityId: id, title: e.title, taskId: runId, lockedFields });
|
||||
}
|
||||
await audit(d, principal, "scrape_batch", {
|
||||
write: body.write,
|
||||
submitted: submitted.map((s) => s.mediaEntityId),
|
||||
skipped,
|
||||
});
|
||||
|
||||
@@ -305,8 +305,18 @@ describe.skipIf(skip)("files 识别 / 整理 / 探测 / 刮削", () => {
|
||||
|
||||
const s = await h.api("POST", "/files/scrape", { idempotencyKey: key(), mediaEntityId: film, writeNfo: true });
|
||||
expect(s.status).toBe(200);
|
||||
expect(s.body.data).toMatchObject({ mediaEntityId: film, nfo: "not_implemented", images: "not_requested" });
|
||||
// SCR-1:writeNfo 另排 scrapeMedia(写 NFO 与图片到媒体库目录)
|
||||
expect(s.body.data).toMatchObject({ mediaEntityId: film, nfo: "queued", images: "not_requested" });
|
||||
expect(s.body.data.scrapeTaskId).toBeTruthy();
|
||||
expect(h.engine.enqueued.some((e) => e.task === "refreshMetadata")).toBe(true);
|
||||
expect(h.engine.enqueued.some((e) => e.task === "scrapeMedia")).toBe(true);
|
||||
const b = await h.api("POST", "/files/scrape-batch", {
|
||||
idempotencyKey: key(),
|
||||
mediaEntityIds: [film],
|
||||
write: true,
|
||||
});
|
||||
expect(b.status).toBe(200);
|
||||
expect(b.body.data.submitted).toHaveLength(1);
|
||||
const none = await h.api("POST", "/files/scrape", { idempotencyKey: key(), mediaEntityId: show });
|
||||
expect(none.status).toBe(400);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
"""SCR-1 刮削差分:用 MP 上游的 TmdbScraper(themoviedb/scraper.py)从同一份 TMDB 原始响应生成 NFO,写 scrape-mp.json。
|
||||
|
||||
用法(MP 仓库根目录,用其 .venv):
|
||||
cd /opt/moviepilot-plus/upstream/MoviePilot && PYTHONPATH=. \
|
||||
.venv/bin/python /opt/uhd_media_tool/apps/server/test/scrape/gen-scrape-mp.py
|
||||
|
||||
只记录行为输出,不复制 MP 源码。MediaInfo 用 MP 自己的 set_tmdb_info(TMDB 详情 → 统一字段),
|
||||
季 / 集直接喂 TMDB 季详情(与 MP get_metadata_nfo 取到的一致)。图片命名按 MP 代码行为登记在 nfo-mp-diff.test.ts。
|
||||
"""
|
||||
|
||||
import json
|
||||
import pathlib
|
||||
|
||||
from app.domain.context import MediaInfo
|
||||
from app.domain.projection.tmdb import configure_image_url_builder
|
||||
from app.modules.themoviedb.scraper import TmdbScraper
|
||||
|
||||
HERE = pathlib.Path(__file__).parent
|
||||
raw = json.loads((HERE / "tmdb-raw.json").read_text(encoding="utf-8"))
|
||||
|
||||
# 启动组合根里注入的 TMDB 图片地址(settings.TMDB_IMAGE_URL,原图)
|
||||
configure_image_url_builder(lambda p: f"https://image.tmdb.org/t/p/original/{p.removeprefix('/')}")
|
||||
|
||||
s = TmdbScraper()
|
||||
|
||||
|
||||
def xml(doc) -> str:
|
||||
return doc.toprettyxml(indent=" ", encoding="utf-8").decode("utf-8")
|
||||
|
||||
|
||||
def media(info: dict, media_type: str) -> MediaInfo:
|
||||
mi = MediaInfo()
|
||||
mi.set_tmdb_info({**info, "media_type": media_type})
|
||||
return mi
|
||||
|
||||
|
||||
out = {}
|
||||
out["movie"] = xml(s._TmdbScraper__gen_movie_nfo_file(mediainfo=media(raw["/movie/157336"], "movie")))
|
||||
out["tvshow"] = xml(s._TmdbScraper__gen_tv_nfo_file(mediainfo=media(raw["/tv/95396"], "tv")))
|
||||
season = raw["/tv/95396/season/1"]
|
||||
out["season1"] = xml(s._TmdbScraper__gen_tv_season_nfo_file(seasoninfo=season, season=1))
|
||||
for ep in season["episodes"]:
|
||||
n = ep["episode_number"]
|
||||
out[f"s01e{n:02d}"] = xml(
|
||||
s._TmdbScraper__gen_tv_episode_nfo_file(tmdbid=95396, episodeinfo=ep, season=1, episode=n)
|
||||
)
|
||||
out["seasonPosterName"] = TmdbScraper.get_season_poster(season, 1)[0]
|
||||
out["seasonSpecialsPosterName"] = TmdbScraper.get_season_poster({"poster_path": "/x.jpg"}, 0)[0]
|
||||
|
||||
(HERE / "scrape-mp.json").write_text(json.dumps(out, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(f"写入 {len(out)} 项 → scrape-mp.json")
|
||||
@@ -0,0 +1,69 @@
|
||||
{
|
||||
"_note": "SCR-1 NFO 与 MP 的已登记差异(按样例、按元素)。测试先去掉双方的空元素(MP 缺值写 <x/>、空 CDATA、评分 0,我们不写),再去掉这里登记的元素后逐行比较;登记的元素必须确实不同,差异消除后删掉登记。",
|
||||
"deviations": [
|
||||
{
|
||||
"case": "movie",
|
||||
"tag": "studio",
|
||||
"reason": "本地不存 TMDB production_companies,电影不写出品公司(Emby / Jellyfin 刷新时会自己补)"
|
||||
},
|
||||
{
|
||||
"case": "movie",
|
||||
"tag": "country",
|
||||
"reason": "制片国取 TMDB origin_country(代码转中文名);MP 取 production_countries(本例多一个英国)"
|
||||
},
|
||||
{
|
||||
"case": "movie",
|
||||
"tag": "tagline",
|
||||
"reason": "我们额外写标语,MP 不写"
|
||||
},
|
||||
{
|
||||
"case": "tvshow",
|
||||
"tag": "studio",
|
||||
"reason": "剧集写播出平台(TMDB networks,本例 Apple TV+);MP 写出品公司 production_companies"
|
||||
},
|
||||
{
|
||||
"case": "s01e01",
|
||||
"tag": "uniqueid",
|
||||
"reason": "本地不存 TMDB 集 id,集 NFO 不写 uniqueid,媒体服务器按季号集号匹配"
|
||||
},
|
||||
{
|
||||
"case": "s01e01",
|
||||
"tag": "tmdbid",
|
||||
"reason": "同上"
|
||||
},
|
||||
{
|
||||
"case": "s01e01",
|
||||
"tag": "runtime",
|
||||
"reason": "我们额外写集时长,MP 不写"
|
||||
},
|
||||
{
|
||||
"case": "s01e02",
|
||||
"tag": "uniqueid",
|
||||
"reason": "同 s01e01"
|
||||
},
|
||||
{
|
||||
"case": "s01e02",
|
||||
"tag": "tmdbid",
|
||||
"reason": "同 s01e01"
|
||||
},
|
||||
{
|
||||
"case": "s01e02",
|
||||
"tag": "runtime",
|
||||
"reason": "同 s01e01"
|
||||
}
|
||||
],
|
||||
"images": [
|
||||
{
|
||||
"role": "movie poster / fanart",
|
||||
"mp": "poster.jpg、backdrop.jpg、fanart.jpg(作品目录)",
|
||||
"ours": "poster.jpg、fanart.jpg",
|
||||
"reason": "backdrop.jpg 是 fanart.jpg 的副本,Emby / Jellyfin / Kodi 都认 fanart.jpg,不写重复文件"
|
||||
},
|
||||
{
|
||||
"role": "episode thumb",
|
||||
"mp": "<视频名>.jpg",
|
||||
"ours": "<视频名>-thumb.jpg",
|
||||
"reason": "Kodi 约定 -thumb 后缀;Jellyfin / Emby 同样识别,且不会和同名海报规则冲突"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { fetchRichDetail, fetchRichSeason } from "@umt/adapters";
|
||||
import { episodeNfo, planScrapeTargets, seasonNfo, seasonPosterName, workNfo } from "@umt/domain";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { nfoEpisode, nfoSeason, nfoWork } from "../../src/modules/acquisition-organize/services/scrape-input.ts";
|
||||
import { richEntityFields } from "../../src/modules/sources-identity/service/metadata-rich.ts";
|
||||
|
||||
// SCR-1 NFO 差分:同一份 TMDB 原始响应(tmdb-raw.json),MP 走 set_tmdb_info + TmdbScraper(gen-scrape-mp.py → scrape-mp.json),
|
||||
// 我们走 fetchRichDetail / fetchRichSeason → 字段投影(metadata)→ scrape-input → domain NFO 生成器。
|
||||
// 双方去掉空元素后逐行比较,已登记的差异(nfo-deviations.json)按元素排除并要求确实不同。
|
||||
|
||||
const read = (f: string) => JSON.parse(readFileSync(new URL(f, import.meta.url), "utf8"));
|
||||
const raw = read("./tmdb-raw.json") as Record<string, unknown>;
|
||||
const mp = read("./scrape-mp.json") as Record<string, string>;
|
||||
const devs = read("./nfo-deviations.json").deviations as { case: string; tag: string }[];
|
||||
|
||||
const LANGS = ["zh-CN", "zh-SG", "zh-TW", "zh-HK", "en-US"];
|
||||
const client = {
|
||||
async getJson<T>(path: string): Promise<T> {
|
||||
const r = raw[path];
|
||||
if (!r) throw new Error(`fixture 缺 ${path}`);
|
||||
return structuredClone(r) as T;
|
||||
},
|
||||
};
|
||||
const final = (fields: Record<string, { value: unknown }>) =>
|
||||
Object.fromEntries(Object.entries(fields).map(([k, v]) => [k, { value: v.value }]));
|
||||
|
||||
async function ours(): Promise<Record<string, string>> {
|
||||
const out: Record<string, string> = {};
|
||||
const movie = await fetchRichDetail(client, "movie", 157336, LANGS);
|
||||
out.movie = workNfo(
|
||||
nfoWork(
|
||||
{
|
||||
kind: "movie",
|
||||
title: movie.title,
|
||||
original_title: movie.originalTitle,
|
||||
year: movie.year,
|
||||
metadata: final(richEntityFields(movie) as never),
|
||||
},
|
||||
{ tmdb: "157336" },
|
||||
),
|
||||
);
|
||||
const tv = await fetchRichDetail(client, "tv", 95396, LANGS);
|
||||
out.tvshow = workNfo(
|
||||
nfoWork(
|
||||
{
|
||||
kind: "series",
|
||||
title: tv.title,
|
||||
original_title: tv.originalTitle,
|
||||
year: tv.year,
|
||||
metadata: final(richEntityFields(tv) as never),
|
||||
},
|
||||
{ tmdb: "95396" },
|
||||
),
|
||||
);
|
||||
const sd = await fetchRichSeason(client, 95396, 1, LANGS, { now: new Date("2026-01-01") });
|
||||
out.season1 = seasonNfo(
|
||||
nfoSeason(1, {
|
||||
season_number: 1,
|
||||
episode_number: null,
|
||||
title: sd.name,
|
||||
air_date: sd.airDate,
|
||||
metadata: { name: { value: sd.name }, overview: { value: sd.overview }, airDate: { value: sd.airDate } },
|
||||
}),
|
||||
);
|
||||
for (const e of sd.episodes) {
|
||||
out[`s01e${String(e.episodeNumber).padStart(2, "0")}`] = episodeNfo(
|
||||
nfoEpisode(1, e.episodeNumber, {
|
||||
season_number: 1,
|
||||
episode_number: e.episodeNumber,
|
||||
title: e.name,
|
||||
air_date: e.airDate,
|
||||
metadata: {
|
||||
name: { value: e.name },
|
||||
overview: { value: e.overview },
|
||||
airDate: { value: e.airDate },
|
||||
runtime: { value: e.runtime },
|
||||
voteAverage: { value: e.voteAverage },
|
||||
},
|
||||
}),
|
||||
);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** 去空元素(MP 缺值写 <x/>、空 CDATA、评分 0) */
|
||||
const normalize = (xml: string) =>
|
||||
xml
|
||||
.split("\n")
|
||||
.filter((l) => l.trim() !== "")
|
||||
.filter((l) => !/^\s*<[\w]+(\s[^>]*)?\/>$/.test(l))
|
||||
.filter((l) => !/<!\[CDATA\[\]\]>/.test(l))
|
||||
.filter((l) => !/^\s*<rating>0<\/rating>$/.test(l));
|
||||
const tagOf = (l: string) => /^ {2}<(\w+)/.exec(l)?.[1] ?? null;
|
||||
|
||||
describe("NFO 与 MP 差分", async () => {
|
||||
const mine = await ours();
|
||||
const cases = ["movie", "tvshow", "season1", "s01e01", "s01e02"];
|
||||
|
||||
it.each(cases)("%s:结构与取值一致(已登记差异除外)", (c) => {
|
||||
const skip = new Set(devs.filter((d) => d.case === c).map((d) => d.tag));
|
||||
const a = normalize(mp[c]!).filter((l) => !skip.has(tagOf(l) ?? ""));
|
||||
const b = normalize(mine[c]!).filter((l) => !skip.has(tagOf(l) ?? ""));
|
||||
expect(b).toEqual(a);
|
||||
for (const tag of skip) {
|
||||
const ma = normalize(mp[c]!).filter((l) => tagOf(l) === tag);
|
||||
const mb = normalize(mine[c]!).filter((l) => tagOf(l) === tag);
|
||||
expect(mb, `登记的差异 ${c}.${tag} 已消失,删掉登记`).not.toEqual(ma);
|
||||
}
|
||||
});
|
||||
|
||||
it("编码与声明:UTF-8、两空格缩进、末尾换行,和 MP 一致", () => {
|
||||
for (const c of cases) {
|
||||
expect(mine[c]!.split("\n")[0]).toBe(mp[c]!.split("\n")[0]);
|
||||
expect(mine[c]!.endsWith("\n")).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("季海报命名与 MP get_season_poster 一致", () => {
|
||||
expect(seasonPosterName(1)).toBe(mp.seasonPosterName);
|
||||
expect(seasonPosterName(0)).toBe(mp.seasonSpecialsPosterName);
|
||||
});
|
||||
|
||||
it("文件命名:对照 MP scraping.py 的目标路径(差异见 nfo-deviations.json images)", () => {
|
||||
const t = planScrapeTargets({
|
||||
kind: "tv",
|
||||
workDir: "人生切割术 (2022)",
|
||||
videos: [{ path: "人生切割术 (2022)/Season 1/人生切割术 - S01E01 - 第 1 集.mkv", season: 1, episode: 1 }],
|
||||
images: {
|
||||
posterPath: "/p.jpg",
|
||||
backdropPath: "/b.jpg",
|
||||
seasonPosters: new Map([[1, "/s.jpg"]]),
|
||||
stills: new Map([["1:1", "/e.jpg"]]),
|
||||
},
|
||||
options: { images: true, episodeThumbs: true },
|
||||
});
|
||||
// MP:tvshow.nfo(TV)、season.nfo(SEASON)、<stem>.nfo(EPISODE);季海报根目录 seasonXX-poster + 季目录 poster
|
||||
expect(t.map((x) => x.path)).toEqual([
|
||||
"人生切割术 (2022)/tvshow.nfo",
|
||||
"人生切割术 (2022)/poster.jpg",
|
||||
"人生切割术 (2022)/fanart.jpg",
|
||||
"人生切割术 (2022)/Season 1/season.nfo",
|
||||
"人生切割术 (2022)/season01-poster.jpg",
|
||||
"人生切割术 (2022)/Season 1/poster.jpg",
|
||||
"人生切割术 (2022)/Season 1/人生切割术 - S01E01 - 第 1 集.nfo",
|
||||
"人生切割术 (2022)/Season 1/人生切割术 - S01E01 - 第 1 集-thumb.jpg",
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"movie": "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<movie>\n <tmdbid>157336</tmdbid>\n <uniqueid type=\"tmdb\" default=\"false\">157336</uniqueid>\n <imdbid>tt0816692</imdbid>\n <uniqueid type=\"imdb\" default=\"true\">tt0816692</uniqueid>\n <plot><![CDATA[在不远的未来，随着地球自然环境的恶化，人类面临着无法生存的威胁。一群宇航员穿越虫洞，为人类寻找新家园。]]></plot>\n <outline><![CDATA[在不远的未来，随着地球自然环境的恶化，人类面临着无法生存的威胁。一群宇航员穿越虫洞，为人类寻找新家园。]]></outline>\n <director tmdbid=\"525\">克里斯托弗·诺兰</director>\n <actor>\n <name>马修·麦康纳</name>\n <type>Actor</type>\n <role>Cooper</role>\n <tmdbid>10297</tmdbid>\n <thumb>https://image.tmdb.org/t/p/original/lCySuYjhXix3FzQdS4oceDDrXKI.jpg</thumb>\n <profile>https://www.themoviedb.org/person/10297</profile>\n </actor>\n <actor>\n <name>安妮·海瑟薇</name>\n <type>Actor</type>\n <role>Brand</role>\n <tmdbid>1813</tmdbid>\n <thumb>https://image.tmdb.org/t/p/original/s6tflSD20MGz04ZR2R1lZvhmC4Y.jpg</thumb>\n <profile>https://www.themoviedb.org/person/1813</profile>\n </actor>\n <actor>\n <name>杰西卡·查斯坦</name>\n <type>Actor</type>\n <role>Murph</role>\n <tmdbid>83002</tmdbid>\n <profile>https://www.themoviedb.org/person/83002</profile>\n </actor>\n <genre>冒险</genre>\n <genre>剧情</genre>\n <genre>科幻</genre>\n <rating>8.5</rating>\n <studio>Legendary Pictures</studio>\n <country>英国</country>\n <country>美国</country>\n <runtime>169</runtime>\n <title>星际穿越</title>\n <originaltitle>Interstellar</originaltitle>\n <premiered>2014-11-05</premiered>\n <year>2014</year>\n</movie>\n",
|
||||
"tvshow": "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<tvshow>\n <tmdbid>95396</tmdbid>\n <uniqueid type=\"tmdb\" default=\"false\">95396</uniqueid>\n <tvdbid>371980</tvdbid>\n <uniqueid type=\"tvdb\">371980</uniqueid>\n <imdbid>tt11280740</imdbid>\n <uniqueid type=\"imdb\" default=\"true\">tt11280740</uniqueid>\n <plot><![CDATA[马克领导着一个办公室团队，他们的记忆被手术分割，工作与生活从此一分为二。]]></plot>\n <outline><![CDATA[马克领导着一个办公室团队，他们的记忆被手术分割，工作与生活从此一分为二。]]></outline>\n <actor>\n <name>亚当·斯科特</name>\n <type>Actor</type>\n <role>Mark Scout</role>\n <tmdbid>4495</tmdbid>\n <thumb>https://image.tmdb.org/t/p/original/9ynbGSZ7O7stcCYx1ElmFhUgCKj.jpg</thumb>\n <profile>https://www.themoviedb.org/person/4495</profile>\n </actor>\n <actor>\n <name>布丽特·洛薇尔</name>\n <type>Actor</type>\n <role>Helly R.</role>\n <tmdbid>1319469</tmdbid>\n <profile>https://www.themoviedb.org/person/1319469</profile>\n </actor>\n <genre>剧情</genre>\n <genre>悬疑</genre>\n <genre>Sci-Fi & Fantasy</genre>\n <rating>8.4</rating>\n <studio>Red Hour Productions</studio>\n <country>美国</country>\n <title>人生切割术</title>\n <originaltitle>Severance</originaltitle>\n <premiered>2022-02-17</premiered>\n <year>2022</year>\n <season>-1</season>\n <episode>-1</episode>\n</tvshow>\n",
|
||||
"season1": "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<season>\n <plot><![CDATA[在卢蒙工业公司，员工们接受了一种名为“分割”的手术 & 工作记忆与生活记忆被彻底隔开。]]></plot>\n <outline><![CDATA[在卢蒙工业公司，员工们接受了一种名为“分割”的手术 & 工作记忆与生活记忆被彻底隔开。]]></outline>\n <title>第 1 季</title>\n <premiered>2022-02-17</premiered>\n <releasedate>2022-02-17</releasedate>\n <year>2022</year>\n <seasonnumber>1</seasonnumber>\n</season>\n",
|
||||
"s01e01": "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<episodedetails>\n <uniqueid type=\"tmdb\" default=\"true\">1830836</uniqueid>\n <tmdbid>1830836</tmdbid>\n <title>有关地狱的好消息</title>\n <plot><![CDATA[马克升职成为宏观数据精炼部的主管，并迎来一位新员工。]]></plot>\n <outline><![CDATA[马克升职成为宏观数据精炼部的主管，并迎来一位新员工。]]></outline>\n <aired>2022-02-17</aired>\n <year>2022</year>\n <season>1</season>\n <episode>1</episode>\n <rating>8.2</rating>\n</episodedetails>\n",
|
||||
"s01e02": "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<episodedetails>\n <uniqueid type=\"tmdb\" default=\"true\">3435543</uniqueid>\n <tmdbid>3435543</tmdbid>\n <title>半循环</title>\n <plot><![CDATA[马克带海莉参观办公室。]]></plot>\n <outline><![CDATA[马克带海莉参观办公室。]]></outline>\n <aired>2022-02-18</aired>\n <year>2022</year>\n <season>1</season>\n <episode>2</episode>\n <rating>8.0</rating>\n</episodedetails>\n",
|
||||
"seasonPosterName": "season01-poster.jpg",
|
||||
"seasonSpecialsPosterName": "season-specials-poster.jpg"
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
{
|
||||
"_note": "SCR-1 差分用 TMDB 原始响应(zh-CN,形态照 TMDB v3;数值只保证合理,不作权威数据)。MP 与我们都从这里出发生成 NFO。",
|
||||
"/movie/157336": {
|
||||
"id": 157336,
|
||||
"title": "星际穿越",
|
||||
"original_title": "Interstellar",
|
||||
"original_language": "en",
|
||||
"origin_country": ["US"],
|
||||
"release_date": "2014-11-05",
|
||||
"overview": "在不远的未来，随着地球自然环境的恶化，人类面临着无法生存的威胁。一群宇航员穿越虫洞，为人类寻找新家园。",
|
||||
"tagline": "我们注定要离开地球",
|
||||
"runtime": 169,
|
||||
"vote_average": 8.464,
|
||||
"vote_count": 37000,
|
||||
"status": "Released",
|
||||
"poster_path": "/nCbkOyOMTEwlEV0LtCOvCnwEONA.jpg",
|
||||
"backdrop_path": "/xJHokMbljvjADYdit5fK5VQsXEG.jpg",
|
||||
"genres": [{ "id": 12, "name": "冒险" }, { "id": 18, "name": "剧情" }, { "id": 878, "name": "科幻" }],
|
||||
"production_countries": [{ "iso_3166_1": "GB", "name": "英国" }, { "iso_3166_1": "US", "name": "美国" }],
|
||||
"production_companies": [{ "id": 923, "name": "Legendary Pictures" }],
|
||||
"external_ids": { "imdb_id": "tt0816692", "wikidata_id": "Q13417189" },
|
||||
"credits": {
|
||||
"cast": [
|
||||
{
|
||||
"id": 10297,
|
||||
"name": "马修·麦康纳",
|
||||
"original_name": "Matthew McConaughey",
|
||||
"character": "Cooper",
|
||||
"known_for_department": "Acting",
|
||||
"order": 0,
|
||||
"profile_path": "/lCySuYjhXix3FzQdS4oceDDrXKI.jpg"
|
||||
},
|
||||
{
|
||||
"id": 1813,
|
||||
"name": "安妮·海瑟薇",
|
||||
"original_name": "Anne Hathaway",
|
||||
"character": "Brand",
|
||||
"known_for_department": "Acting",
|
||||
"order": 1,
|
||||
"profile_path": "/s6tflSD20MGz04ZR2R1lZvhmC4Y.jpg"
|
||||
},
|
||||
{
|
||||
"id": 83002,
|
||||
"name": "杰西卡·查斯坦",
|
||||
"original_name": "Jessica Chastain",
|
||||
"character": "Murph",
|
||||
"known_for_department": "Acting",
|
||||
"order": 2,
|
||||
"profile_path": null
|
||||
}
|
||||
],
|
||||
"crew": [
|
||||
{
|
||||
"id": 525,
|
||||
"name": "克里斯托弗·诺兰",
|
||||
"original_name": "Christopher Nolan",
|
||||
"job": "Director",
|
||||
"department": "Directing",
|
||||
"known_for_department": "Directing",
|
||||
"profile_path": "/xuAIuYSmsUzKlUMBFGVZaWsY3DZ.jpg"
|
||||
}
|
||||
]
|
||||
},
|
||||
"translations": {
|
||||
"translations": [
|
||||
{
|
||||
"iso_639_1": "zh",
|
||||
"iso_3166_1": "CN",
|
||||
"data": {
|
||||
"title": "星际穿越",
|
||||
"overview": "在不远的未来，随着地球自然环境的恶化，人类面临着无法生存的威胁。一群宇航员穿越虫洞，为人类寻找新家园。",
|
||||
"tagline": "我们注定要离开地球"
|
||||
}
|
||||
},
|
||||
{
|
||||
"iso_639_1": "en",
|
||||
"iso_3166_1": "US",
|
||||
"data": {
|
||||
"title": "Interstellar",
|
||||
"overview": "The adventures of a group of explorers.",
|
||||
"tagline": "Mankind was born on Earth. It was never meant to die here."
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"alternative_titles": { "titles": [{ "iso_3166_1": "CN", "title": "星际启示录", "type": "" }] }
|
||||
},
|
||||
"/tv/95396": {
|
||||
"id": 95396,
|
||||
"name": "人生切割术",
|
||||
"original_name": "Severance",
|
||||
"original_language": "en",
|
||||
"origin_country": ["US"],
|
||||
"first_air_date": "2022-02-17",
|
||||
"last_air_date": "2025-03-21",
|
||||
"overview": "马克领导着一个办公室团队，他们的记忆被手术分割，工作与生活从此一分为二。",
|
||||
"tagline": "",
|
||||
"vote_average": 8.395,
|
||||
"vote_count": 2400,
|
||||
"status": "Returning Series",
|
||||
"episode_run_time": [],
|
||||
"number_of_seasons": 2,
|
||||
"number_of_episodes": 19,
|
||||
"poster_path": "/pPHpeI2X1qEd1CS1SeyrdhZ4qnT.jpg",
|
||||
"backdrop_path": "/hS8cOSG9m8H9dGaIxkRYZ7rRl8F.jpg",
|
||||
"genres": [
|
||||
{ "id": 18, "name": "剧情" },
|
||||
{ "id": 9648, "name": "悬疑" },
|
||||
{ "id": 10765, "name": "Sci-Fi & Fantasy" }
|
||||
],
|
||||
"networks": [{ "id": 2552, "name": "Apple TV+" }],
|
||||
"production_countries": [{ "iso_3166_1": "US", "name": "美国" }],
|
||||
"production_companies": [{ "id": 1, "name": "Red Hour Productions" }],
|
||||
"created_by": [{ "id": 3442409, "name": "丹·埃里克森", "profile_path": null }],
|
||||
"external_ids": { "imdb_id": "tt11280740", "tvdb_id": 371980 },
|
||||
"credits": {
|
||||
"cast": [
|
||||
{
|
||||
"id": 4495,
|
||||
"name": "亚当·斯科特",
|
||||
"original_name": "Adam Scott",
|
||||
"character": "Mark Scout",
|
||||
"known_for_department": "Acting",
|
||||
"order": 0,
|
||||
"profile_path": "/9ynbGSZ7O7stcCYx1ElmFhUgCKj.jpg"
|
||||
},
|
||||
{
|
||||
"id": 1319469,
|
||||
"name": "布丽特·洛薇尔",
|
||||
"original_name": "Britt Lower",
|
||||
"character": "Helly R.",
|
||||
"known_for_department": "Acting",
|
||||
"order": 1,
|
||||
"profile_path": null
|
||||
}
|
||||
],
|
||||
"crew": []
|
||||
},
|
||||
"seasons": [
|
||||
{
|
||||
"season_number": 1,
|
||||
"episode_count": 9,
|
||||
"name": "第 1 季",
|
||||
"air_date": "2022-02-17",
|
||||
"overview": "",
|
||||
"poster_path": "/lFf6LLrQjYldcZItzOkGmMMigP7.jpg"
|
||||
},
|
||||
{
|
||||
"season_number": 2,
|
||||
"episode_count": 10,
|
||||
"name": "第 2 季",
|
||||
"air_date": "2025-01-16",
|
||||
"overview": "",
|
||||
"poster_path": "/5n9UHSJp9KcHSTnkBPkcQqgWUgL.jpg"
|
||||
}
|
||||
],
|
||||
"translations": {
|
||||
"translations": [
|
||||
{
|
||||
"iso_639_1": "zh",
|
||||
"iso_3166_1": "CN",
|
||||
"data": {
|
||||
"name": "人生切割术",
|
||||
"overview": "马克领导着一个办公室团队，他们的记忆被手术分割，工作与生活从此一分为二。",
|
||||
"tagline": ""
|
||||
}
|
||||
},
|
||||
{
|
||||
"iso_639_1": "en",
|
||||
"iso_3166_1": "US",
|
||||
"data": { "name": "Severance", "overview": "Mark leads a team of office workers.", "tagline": "" }
|
||||
}
|
||||
]
|
||||
},
|
||||
"alternative_titles": { "results": [{ "iso_3166_1": "TW", "title": "人生切割術", "type": "" }] }
|
||||
},
|
||||
"/tv/95396/season/1": {
|
||||
"id": 186498,
|
||||
"season_number": 1,
|
||||
"name": "第 1 季",
|
||||
"overview": "在卢蒙工业公司，员工们接受了一种名为“分割”的手术 & 工作记忆与生活记忆被彻底隔开。",
|
||||
"air_date": "2022-02-17",
|
||||
"poster_path": "/lFf6LLrQjYldcZItzOkGmMMigP7.jpg",
|
||||
"episodes": [
|
||||
{
|
||||
"id": 1830836,
|
||||
"episode_number": 1,
|
||||
"season_number": 1,
|
||||
"name": "有关地狱的好消息",
|
||||
"overview": "马克升职成为宏观数据精炼部的主管，并迎来一位新员工。",
|
||||
"air_date": "2022-02-17",
|
||||
"runtime": 57,
|
||||
"still_path": "/jM3lJqrr6uD17rBHCM7qc2rH4rM.jpg",
|
||||
"vote_average": 8.2,
|
||||
"vote_count": 171,
|
||||
"crew": [],
|
||||
"guest_stars": []
|
||||
},
|
||||
{
|
||||
"id": 3435543,
|
||||
"episode_number": 2,
|
||||
"season_number": 1,
|
||||
"name": "半循环",
|
||||
"overview": "马克带海莉参观办公室。",
|
||||
"air_date": "2022-02-18",
|
||||
"runtime": 53,
|
||||
"still_path": "/kg2cNi5r1hhvF5GWWyi3wYO7rqE.jpg",
|
||||
"vote_average": 8.0,
|
||||
"vote_count": 150,
|
||||
"crew": [],
|
||||
"guest_stars": []
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user