feat(server): scrapeMedia 刮削任务,入库后自动刮削(SCR-1)
- 文件道任务 scrapeMedia:NFO 原子写入、图片经图片缓存下载(并发 4,失败不阻塞 NFO)、只补缺失 / 全部覆盖
- 设置 organize.scrapeAfterImport / scrapeMode / scrapeImages / scrapeEpisodeThumbs
- POST /library/{id}/scrape、详情 scrape 状态;files/scrape writeNfo 与 scrape-batch write 走同一任务
- 迁移 0057_media_scrape、原因码、审计 organize.scrape
- 与 MP 输出的 NFO 差分测试(apps/server/test/scrape)
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
861f799615
commit
f38eca419e
24 files changed
+1740
-15
No files matched your search
@@ -0,0 +1,299 @@
|
||||
import { chmod, readdir, readFile, stat, writeFile } from "node:fs/promises";
|
||||
import { join, relative } from "node:path";
|
||||
import { createTempTree, type TempTree } from "@umt/testkit";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { executeRecord, planImport } from "../services/organize.ts";
|
||||
import { runScrape, SCRAPE_TASK } from "../services/scrape.ts";
|
||||
import { createHarness, type Harness, pgAvailable, seedBase, seedEntity } from "./harness.ts";
|
||||
|
||||
// SCR-1 刮削:入库成功后自动排 scrapeMedia → 写 NFO 与图片(临时库根);只补缺失 / 全部覆盖;图片失败不阻塞 NFO;
|
||||
// 只登记库根不写;作品详情「重新刮削」接口与刮削状态。
|
||||
|
||||
const hasPg = await pgAvailable();
|
||||
const SIZE = 200_000;
|
||||
/** 最小 JPEG 头(按文件头认图) */
|
||||
const JPEG = new Uint8Array([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10, 0x4a, 0x46, 0x49, 0x46, 0x00]);
|
||||
const at = new Date().toISOString();
|
||||
const f = (value: unknown, language: string | null = null) => ({ value, source: "tmdb", language, at, locked: false });
|
||||
|
||||
async function tree(root: string): Promise<string[]> {
|
||||
const out: string[] = [];
|
||||
const walk = async (d: string) => {
|
||||
for (const n of (await readdir(d)).sort()) {
|
||||
const p = join(d, n);
|
||||
if ((await stat(p)).isDirectory()) await walk(p);
|
||||
else out.push(relative(root, p));
|
||||
}
|
||||
};
|
||||
await walk(root);
|
||||
return out.filter((p) => !p.startsWith(".")).sort();
|
||||
}
|
||||
|
||||
describe.skipIf(!hasPg)("刮削:NFO 与图片写进媒体库目录", () => {
|
||||
let h: Harness;
|
||||
let lib: TempTree;
|
||||
let src: TempTree;
|
||||
const fetched: string[] = [];
|
||||
|
||||
beforeAll(async () => {
|
||||
lib = await createTempTree({}, { sentinel: true, prefix: "umt-scrlib-" });
|
||||
src = await createTempTree(
|
||||
{
|
||||
"Severance.S01E01.1080p.WEB-DL.H264-GRP.mkv": SIZE,
|
||||
"Severance.S01E02.1080p.WEB-DL.H264-GRP.mkv": SIZE,
|
||||
"Interstellar.2014.1080p.BluRay.x264-GRP.mkv": SIZE,
|
||||
},
|
||||
{ prefix: "umt-scrsrc-" },
|
||||
);
|
||||
h = await createHarness("scrape", {
|
||||
async scrapeImage(p) {
|
||||
fetched.push(p);
|
||||
if (p === "/broken.jpg") throw new Error("TMDB 图片返回 HTTP 404");
|
||||
return { content: JPEG, mime: "image/jpeg" };
|
||||
},
|
||||
});
|
||||
await seedBase(h.db, { libraryPath: lib.root, cachePath: src.root, qbUrl: "http://127.0.0.1:9" });
|
||||
await h.setSettings("organize", { scrapeAfterImport: true });
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await h?.close();
|
||||
for (const t of [lib, src]) await t?.cleanup();
|
||||
});
|
||||
|
||||
async function organize(entity: string, file: string) {
|
||||
const plan = await planImport(h.d, {
|
||||
sourcePaths: [src.path(file)],
|
||||
mediaEntityId: entity,
|
||||
action: "copy",
|
||||
trigger: "manual",
|
||||
actorType: "user",
|
||||
});
|
||||
expect(plan.status).toBe("running");
|
||||
const ex = await executeRecord(h.d, plan.recordId!);
|
||||
expect(ex.status).toBe("succeeded");
|
||||
return plan.recordId!;
|
||||
}
|
||||
|
||||
/** 按引擎记录的入队项跑刮削(假引擎不执行) */
|
||||
async function runQueued(entity: string) {
|
||||
const q = h.engine.enqueued.filter(
|
||||
(x) => x.task === SCRAPE_TASK && (x.input as { mediaEntityId: string }).mediaEntityId === entity,
|
||||
);
|
||||
expect(q.length).toBeGreaterThan(0);
|
||||
return runScrape(h.d, q[q.length - 1]!.input as Parameters<typeof runScrape>[1]);
|
||||
}
|
||||
|
||||
let series = "";
|
||||
let movie = "";
|
||||
|
||||
it("剧集:入库后自动排刮削,写 tvshow / 季 / 集 NFO、海报、季海报、剧照", async () => {
|
||||
series = await seedEntity(h.db, { kind: "series", title: "人生切割术", year: 2022 });
|
||||
await h.db
|
||||
.updateTable("media_entity")
|
||||
.set({
|
||||
original_title: "Severance",
|
||||
metadata: JSON.stringify({
|
||||
title: f("人生切割术", "zh-CN"),
|
||||
originalTitle: f("Severance", "en"),
|
||||
year: f(2022),
|
||||
releaseDate: f("2022-02-17"),
|
||||
overview: f("马克领导着一个办公室团队。", "zh-CN"),
|
||||
genres: f(["剧情", "悬疑"]),
|
||||
voteAverage: f(8.4),
|
||||
originCountry: f(["US"]),
|
||||
platform: f("Apple TV+"),
|
||||
posterPath: f("/sev.jpg"),
|
||||
backdropPath: f("/broken.jpg"),
|
||||
cast: f([
|
||||
{ id: 4495, name: "亚当·斯科特", originalName: null, role: "Mark Scout", profilePath: "/adam.jpg" },
|
||||
]),
|
||||
crew: [],
|
||||
externalIds: f({ imdb: "tt11280740", tvdb: "371980", wikidata: null }),
|
||||
}),
|
||||
})
|
||||
.where("id", "=", series)
|
||||
.execute();
|
||||
await h.db
|
||||
.insertInto("external_identity")
|
||||
.values({
|
||||
media_entity_id: series,
|
||||
provider: "tmdb",
|
||||
entity_type: "tv",
|
||||
external_id: "95396",
|
||||
status: "accepted",
|
||||
})
|
||||
.execute();
|
||||
const season = await h.db
|
||||
.insertInto("content_unit")
|
||||
.values({
|
||||
media_entity_id: series,
|
||||
kind: "season",
|
||||
season_number: 1,
|
||||
status: "confirmed",
|
||||
title: "第 1 季",
|
||||
metadata: JSON.stringify({ name: f("第 1 季", "zh-CN"), airDate: f("2022-02-17"), posterPath: f("/s1.jpg") }),
|
||||
})
|
||||
.returning("id")
|
||||
.executeTakeFirstOrThrow();
|
||||
for (const [n, name, still] of [
|
||||
[1, "有关地狱的好消息", "/e1.jpg"],
|
||||
[2, "半循环", null],
|
||||
] as const)
|
||||
await h.db
|
||||
.insertInto("content_unit")
|
||||
.values({
|
||||
media_entity_id: series,
|
||||
parent_unit_id: season.id,
|
||||
kind: "episode",
|
||||
season_number: 1,
|
||||
episode_number: n,
|
||||
status: "confirmed",
|
||||
title: name,
|
||||
air_date: "2022-02-17",
|
||||
metadata: JSON.stringify({
|
||||
name: f(name, "zh-CN"),
|
||||
overview: f(`第 ${n} 集简介`, "zh-CN"),
|
||||
airDate: f("2022-02-17"),
|
||||
runtime: f(55),
|
||||
voteAverage: f(8),
|
||||
...(still ? { stillPath: f(still) } : {}),
|
||||
}),
|
||||
})
|
||||
.execute();
|
||||
|
||||
const rec = await organize(series, "Severance.S01E01.1080p.WEB-DL.H264-GRP.mkv");
|
||||
await organize(series, "Severance.S01E02.1080p.WEB-DL.H264-GRP.mkv");
|
||||
const q = h.engine.enqueued.filter((x) => x.task === SCRAPE_TASK);
|
||||
expect(q[0]!.key).toBe(`${SCRAPE_TASK}:record:${rec}`);
|
||||
expect(q[0]!.input).toMatchObject({ mediaEntityId: series, trigger: "auto", mode: "missing", recordId: rec });
|
||||
|
||||
const r = await runQueued(series);
|
||||
expect(r.status).toBe("partial"); // 背景图下载失败
|
||||
const files = await tree(lib.root);
|
||||
const show = files.find((p) => p.endsWith("tvshow.nfo"))!;
|
||||
const dir = show.slice(0, -"/tvshow.nfo".length);
|
||||
const ep1 = files.find((p) => p.includes("S01E01") && p.endsWith(".mkv"))!;
|
||||
const stem1 = ep1.slice(0, -4);
|
||||
expect(files).toEqual(
|
||||
expect.arrayContaining([
|
||||
`${dir}/tvshow.nfo`,
|
||||
`${dir}/poster.jpg`,
|
||||
`${dir}/season01-poster.jpg`,
|
||||
`${dir}/Season 1/season.nfo`,
|
||||
`${dir}/Season 1/poster.jpg`,
|
||||
`${stem1}.nfo`,
|
||||
`${stem1}-thumb.jpg`,
|
||||
]),
|
||||
);
|
||||
expect(files.some((p) => p.endsWith("fanart.jpg"))).toBe(false);
|
||||
expect(files.filter((p) => p.endsWith(".nfo"))).toHaveLength(4);
|
||||
expect(files.some((p) => p.includes(".umt-scrape.tmp"))).toBe(false);
|
||||
const nfo = await readFile(join(lib.root, show), "utf8");
|
||||
expect(nfo.startsWith('<?xml version="1.0" encoding="utf-8"?>\n<tvshow>')).toBe(true);
|
||||
expect(nfo).toContain('<uniqueid type="imdb" default="true">tt11280740</uniqueid>');
|
||||
expect(nfo).toContain("<country>美国</country>");
|
||||
const epNfo = await readFile(join(lib.root, `${stem1}.nfo`), "utf8");
|
||||
expect(epNfo).toContain("<title>有关地狱的好消息</title>");
|
||||
expect(epNfo).toMatch(/<season>1<\/season>\n {2}<episode>1<\/episode>/);
|
||||
// 同一张季海报只取一次
|
||||
expect(fetched.filter((p) => p === "/s1.jpg")).toHaveLength(1);
|
||||
|
||||
const st = await h.api("GET", `/library/${series}`);
|
||||
expect(st.status).toBe(200);
|
||||
expect(st.body.data.scrape).toMatchObject({
|
||||
status: "partial",
|
||||
trigger: "auto",
|
||||
failed: 1,
|
||||
reason: "scrape_image_failed",
|
||||
});
|
||||
expect(st.body.data.scrape.files.find((x: { role: string }) => x.role === "fanart")).toMatchObject({
|
||||
action: "failed",
|
||||
reason: "scrape_image_failed",
|
||||
});
|
||||
const audit = await h.db
|
||||
.selectFrom("audit_event")
|
||||
.select(["action", "target_id"])
|
||||
.where("action", "=", "organize.scrape")
|
||||
.execute();
|
||||
expect(audit.some((a) => a.target_id === series)).toBe(true);
|
||||
}, 60_000);
|
||||
|
||||
it("只补缺失:已有文件(包括别人写的)不动;全部覆盖:改写", async () => {
|
||||
const files = await tree(lib.root);
|
||||
const show = join(lib.root, files.find((p) => p.endsWith("tvshow.nfo"))!);
|
||||
await writeFile(show, "<tvshow>别的工具写的</tvshow>");
|
||||
const again = await runScrape(h.d, { mediaEntityId: series, trigger: "manual", mode: "missing" });
|
||||
expect(await readFile(show, "utf8")).toBe("<tvshow>别的工具写的</tvshow>");
|
||||
expect(again.files.find((x) => x.role === "tvshow_nfo")).toMatchObject({
|
||||
action: "skipped",
|
||||
reason: "scrape_exists",
|
||||
});
|
||||
const over = await runScrape(h.d, { mediaEntityId: series, trigger: "manual", mode: "overwrite" });
|
||||
expect(over.written).toBeGreaterThan(0);
|
||||
expect(await readFile(show, "utf8")).toContain("<title>人生切割术</title>");
|
||||
}, 60_000);
|
||||
|
||||
it("电影:<视频名>.nfo + 作品目录 poster / fanart;手动「重新刮削」接口排任务", async () => {
|
||||
await h.setSettings("organize", { scrapeAfterImport: false });
|
||||
movie = await seedEntity(h.db, { kind: "movie", title: "星际穿越", year: 2014 });
|
||||
await h.db
|
||||
.updateTable("media_entity")
|
||||
.set({
|
||||
metadata: JSON.stringify({
|
||||
title: f("星际穿越", "zh-CN"),
|
||||
overview: f("穿越虫洞。", "zh-CN"),
|
||||
releaseDate: f("2014-11-05"),
|
||||
runtime: f(169),
|
||||
posterPath: f("/inter.jpg"),
|
||||
backdropPath: f("/inter-bg.jpg"),
|
||||
crew: f([{ id: 525, name: "克里斯托弗·诺兰", originalName: null, role: "Director", profilePath: null }]),
|
||||
}),
|
||||
})
|
||||
.where("id", "=", movie)
|
||||
.execute();
|
||||
const before = h.engine.enqueued.length;
|
||||
await organize(movie, "Interstellar.2014.1080p.BluRay.x264-GRP.mkv");
|
||||
expect(h.engine.enqueued.slice(before).some((x) => x.task === SCRAPE_TASK)).toBe(false); // 关了自动刮削
|
||||
|
||||
const res = await h.api("POST", `/library/${movie}/scrape`, {});
|
||||
expect(res.status).toBe(202);
|
||||
expect(res.body.data).toMatchObject({ mediaEntityId: movie, deduplicated: false });
|
||||
const dup = await h.api("POST", `/library/${movie}/scrape`, {});
|
||||
expect(dup.body.data.deduplicated).toBe(true);
|
||||
expect((await h.api("GET", `/library/${movie}`)).body.data.scrape.status).toBe("queued");
|
||||
|
||||
const r = await runQueued(movie);
|
||||
expect(r.status).toBe("succeeded");
|
||||
const files = await tree(lib.root);
|
||||
const video = files.find((p) => p.includes("星际穿越") && p.endsWith(".mkv"))!;
|
||||
const dir = video.slice(0, video.lastIndexOf("/"));
|
||||
expect(files.filter((p) => p.startsWith(`${dir}/`))).toEqual(
|
||||
[video, `${video.slice(0, -4)}.nfo`, `${dir}/fanart.jpg`, `${dir}/poster.jpg`].sort(),
|
||||
);
|
||||
const nfo = await readFile(join(lib.root, `${video.slice(0, -4)}.nfo`), "utf8");
|
||||
expect(nfo).toContain('<director tmdbid="525">克里斯托弗·诺兰</director>');
|
||||
expect(nfo).toContain("<runtime>169</runtime>");
|
||||
}, 60_000);
|
||||
|
||||
it("写入失败记原因;只登记库根不写文件;没有入库文件的作品 409", async () => {
|
||||
const files = await tree(lib.root);
|
||||
const video = files.find((p) => p.includes("星际穿越") && p.endsWith(".mkv"))!;
|
||||
const dirAbs = join(lib.root, video.slice(0, video.lastIndexOf("/")));
|
||||
if (process.getuid?.() !== 0) {
|
||||
await chmod(dirAbs, 0o555);
|
||||
const r = await runScrape(h.d, { mediaEntityId: movie, trigger: "manual", mode: "overwrite" });
|
||||
await chmod(dirAbs, 0o755);
|
||||
expect(r.files.some((x) => x.reason === "scrape_write_failed")).toBe(true);
|
||||
}
|
||||
await h.db.updateTable("library_root").set({ management_mode: "register_only" }).execute();
|
||||
const ro = await runScrape(h.d, { mediaEntityId: movie, trigger: "manual", mode: "overwrite" });
|
||||
expect(ro).toMatchObject({ status: "failed", reason: "scrape_root_read_only", written: 0 });
|
||||
await h.db.updateTable("library_root").set({ management_mode: "managed" }).execute();
|
||||
|
||||
const empty = await seedEntity(h.db, { kind: "movie", title: "没入库", year: 2020 });
|
||||
expect((await h.api("POST", `/library/${empty}/scrape`, {})).status).toBe(412);
|
||||
const none = await runScrape(h.d, { mediaEntityId: empty, trigger: "manual" });
|
||||
expect(none).toMatchObject({ status: "failed", reason: "scrape_no_metadata" });
|
||||
}, 60_000);
|
||||
});
|
||||
@@ -5,6 +5,7 @@ import { type AoDeps, automationBlock, ensureTaskRow, markTask } from "./service
|
||||
import { cleanupDownloadCache, submitDownload } from "./services/downloads.ts";
|
||||
import { executeRecord, planImport } from "./services/organize.ts";
|
||||
import { reconcileLibrary } from "./services/reconcile.ts";
|
||||
import { runScrape, SCRAPE_TASK } from "./services/scrape.ts";
|
||||
import { errMessage } from "./util.ts";
|
||||
|
||||
// 持久任务处理器。输入只放业务 id;业务状态写业务表;副作用经 withAttemptToken(在 services 里)。
|
||||
@@ -187,6 +188,25 @@ export function registerAoTasks(registry: TaskRegistry, d: AoDeps) {
|
||||
);
|
||||
},
|
||||
}),
|
||||
// SCR-1 刮削:写 NFO 与图片到媒体库目录(文件道);结果写 media_scrape,不重试(失败的文件在作品详情里重新刮削)
|
||||
defineTask({
|
||||
name: SCRAPE_TASK,
|
||||
version: 1,
|
||||
lane: "file",
|
||||
inputSchema: z.object({
|
||||
mediaEntityId: id,
|
||||
trigger: z.enum(["auto", "manual"]),
|
||||
mode: z.enum(["missing", "overwrite"]).optional(),
|
||||
recordId: id.optional(),
|
||||
actor: z.object({ type: z.string(), id: z.string().nullable() }).optional(),
|
||||
}),
|
||||
retryPolicy: { maximumAttempts: 1 },
|
||||
handler: async (t) =>
|
||||
t.step.run("scrape", async () => {
|
||||
const r = await runScrape(d, { ...t.input, runId: t.runId }, t.abortSignal);
|
||||
return { status: r.status, written: r.written, skipped: r.skipped, failed: r.failed, reason: r.reason };
|
||||
}),
|
||||
}),
|
||||
defineTask({
|
||||
name: "reconcileLibrary",
|
||||
version: 1,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
createEmbyClient,
|
||||
createQbClient,
|
||||
@@ -18,6 +19,9 @@ import {
|
||||
import type { ActorType } from "@umt/contracts";
|
||||
import type { DownloaderInstanceRow, MediaServerInstanceRow, SiteRow, SourceReleaseRow } from "@umt/db";
|
||||
import type { ParsedRelease } from "@umt/domain";
|
||||
import { dataPaths } from "../../config.ts";
|
||||
import { externalFetch } from "../../egress.ts";
|
||||
import { ImageCache } from "../cache/images.ts";
|
||||
import { SourcesRuntime } from "../sources-identity/runtime.ts";
|
||||
import {
|
||||
createEntityFromTmdbCandidate,
|
||||
@@ -79,6 +83,11 @@ export interface AoPorts {
|
||||
ref: { type: "movie" | "tv"; id: string },
|
||||
actor: { type: string; id: string | null },
|
||||
): Promise<string>;
|
||||
/**
|
||||
* SCR-1 刮削取图:TMDB 图片路径 → 字节(原图,经图片缓存 DATA_DIR/cache/images 与出口代理)。
|
||||
* 失败抛错(由刮削记为该图片失败,不影响 NFO)。
|
||||
*/
|
||||
scrapeImage(tmdbPath: string): Promise<{ content: Uint8Array; mime: string }>;
|
||||
options: AoOptions;
|
||||
}
|
||||
|
||||
@@ -113,6 +122,7 @@ export function createPorts(
|
||||
}
|
||||
};
|
||||
|
||||
let images: ImageCache | null = null;
|
||||
let sources: SourcesRuntime | null = null;
|
||||
const rt = () => {
|
||||
sources ??= new SourcesRuntime(ctx);
|
||||
@@ -179,6 +189,14 @@ export function createPorts(
|
||||
if (t.kind === "magnet") return { kind: "magnet", uri: t.uri };
|
||||
return { kind: "url", url: t.url };
|
||||
},
|
||||
async scrapeImage(tmdbPath) {
|
||||
images ??= new ImageCache({
|
||||
dir: join(dataPaths(ctx.config).dataDir, "cache", "images"),
|
||||
fetch: externalFetch,
|
||||
timeoutMs: 30_000,
|
||||
});
|
||||
return images.bytes("original", tmdbPath);
|
||||
},
|
||||
diskUsage: (p) => diskUsage(p),
|
||||
probe: (p) => ffprobe(p),
|
||||
async mediaServer(inst) {
|
||||
|
||||
@@ -14,6 +14,8 @@ import {
|
||||
type LibraryItem,
|
||||
LibraryItem as LibraryItemSchema,
|
||||
LibraryListQuery,
|
||||
LibraryScrapeRequest,
|
||||
LibraryScrapeResult,
|
||||
type MediaAsset,
|
||||
type MediaEntity,
|
||||
MediaServerInventory,
|
||||
@@ -25,6 +27,7 @@ import { sql } from "@umt/db";
|
||||
import { ApiException } from "../../../http/errors.ts";
|
||||
import type { ModuleApp } from "../../types.ts";
|
||||
import { type AoDeps, CANDIDATE_ARTIFACT_TITLE } from "../services/common.ts";
|
||||
import { enqueueScrape, loadScrapeState } from "../services/scrape.ts";
|
||||
import { notFound, toApi, toApiList } from "../util.ts";
|
||||
import { mount, paginate } from "./helpers.ts";
|
||||
import { registerLibraryVersionRoutes } from "./library-versions.ts";
|
||||
@@ -404,12 +407,48 @@ export function registerLibraryRoutes(app: ModuleApp, d: AoDeps) {
|
||||
createdAt: new Date(r.created_at).toISOString(),
|
||||
finishedAt: r.finished_at ? new Date(r.finished_at).toISOString() : null,
|
||||
})),
|
||||
scrape: await loadScrapeState(d, id),
|
||||
};
|
||||
return detail;
|
||||
},
|
||||
{ tag: TAG },
|
||||
);
|
||||
|
||||
// SCR-1 刮削:作品详情「重新刮削」;与入库后自动刮削、文件管理「刮削」同一任务 scrapeMedia
|
||||
mount(
|
||||
app,
|
||||
{
|
||||
method: "post",
|
||||
path: "/library/{id}/scrape",
|
||||
scope: "organize.execute",
|
||||
summary: "刮削：把作品的 NFO 与图片写进媒体库目录（后台任务 scrapeMedia；覆盖方式缺省按设置）",
|
||||
params: IdParam,
|
||||
body: LibraryScrapeRequest,
|
||||
response: LibraryScrapeResult,
|
||||
},
|
||||
async (c, { params, body }) => {
|
||||
const e = await db().selectFrom("media_entity").select(["id"]).where("id", "=", params.id).executeTakeFirst();
|
||||
if (!e) notFound("作品");
|
||||
const has = await db()
|
||||
.selectFrom("media_asset")
|
||||
.select(["id"])
|
||||
.where("media_entity_id", "=", params.id)
|
||||
.where("status", "=", "active")
|
||||
.executeTakeFirst();
|
||||
if (!has) throw new ApiException("precondition_failed", "作品没有已入库文件。先整理入库，再刮削");
|
||||
const actor = c.get("actor");
|
||||
const req = (body ?? {}) as LibraryScrapeRequest;
|
||||
const r = await enqueueScrape(d, {
|
||||
mediaEntityId: params.id,
|
||||
trigger: "manual",
|
||||
mode: req.mode,
|
||||
actor: { type: actor?.type === "api_key" ? "api_key" : "user", id: actor?.id ?? null },
|
||||
});
|
||||
return { mediaEntityId: params.id, ...r };
|
||||
},
|
||||
{ tag: TAG, status: 202 },
|
||||
);
|
||||
|
||||
mount(
|
||||
app,
|
||||
{
|
||||
|
||||
@@ -47,6 +47,7 @@ import { effectiveNaming, ensureNamingVersion, loadNamingEntity, namingByVersion
|
||||
import { PathGuardError } from "./organize-guard.ts";
|
||||
import { type FrozenTransfer, finalizeTransfer, planTransfer, type TransferSpec } from "./organize-transfer.ts";
|
||||
import { hnrSatisfied } from "./retention.ts";
|
||||
import { scrapeAfterImport } from "./scrape.ts";
|
||||
import { slotComplete as sharedSlotComplete, subscriptionLibraryRoot } from "./subscriptions.ts";
|
||||
import {
|
||||
CONFLICT_OPTIONS,
|
||||
@@ -1710,6 +1711,9 @@ export async function finalizeRecord(d: AoDeps, recordId: string): Promise<Execu
|
||||
return { recordId, status, succeeded: counts.file_succeeded, failed: counts.file_failed, move, dt: dt ?? null };
|
||||
});
|
||||
|
||||
// SCR-1:有资产入库时按设置自动刮削(写 NFO 与图片;排任务,不在这里等)
|
||||
if (result.status === "succeeded" || result.status === "partial")
|
||||
await scrapeAfterImport(d, frozenPre.entityId ?? null, recordId);
|
||||
// Move 完成:移除 qB 任务记录,结束下载侧管理(不删数据)
|
||||
if (result.move && result.dt) {
|
||||
const topo = (await loadInstances(db, { enabledOnly: false, ids: [result.dt.instance_id] }))[0];
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
import type { NfoEpisode, NfoIds, NfoPerson, NfoSeason, NfoWork } from "@umt/domain";
|
||||
|
||||
// SCR-1:本地作品 / 季 / 集的最终字段(media_entity.metadata、content_unit.metadata,字段级 {value, …})→ NFO 输入。
|
||||
// 纯函数,不查库;差分测试(apps/server/test/scrape/nfo-mp-diff.test.ts)直接调用。
|
||||
|
||||
type Final = Record<string, { value?: unknown } | undefined>;
|
||||
|
||||
const val = (m: unknown, k: string): unknown => ((m ?? {}) as Final)[k]?.value;
|
||||
const str = (v: unknown): string | null => (typeof v === "string" && v.trim() ? v.trim() : null);
|
||||
const numOf = (v: unknown): number | null => (typeof v === "number" && Number.isFinite(v) ? v : null);
|
||||
const strs = (v: unknown): string[] =>
|
||||
Array.isArray(v) ? v.filter((x): x is string => typeof x === "string" && x.trim().length > 0) : [];
|
||||
|
||||
interface Credit {
|
||||
id?: unknown;
|
||||
name?: unknown;
|
||||
role?: unknown;
|
||||
profilePath?: unknown;
|
||||
}
|
||||
const credits = (v: unknown): Credit[] => (Array.isArray(v) ? (v as Credit[]) : []);
|
||||
const person = (c: Credit): NfoPerson | null => {
|
||||
const name = str(c.name);
|
||||
if (!name) return null;
|
||||
return { tmdbId: numOf(c.id), name, role: str(c.role), profilePath: str(c.profilePath) };
|
||||
};
|
||||
|
||||
/** 制片国:ISO 3166-1 代码 → 中文名(Intl);认不出的保留原码 */
|
||||
export function countryNames(codes: readonly string[], locale = "zh-CN"): string[] {
|
||||
let dn: Intl.DisplayNames | null = null;
|
||||
try {
|
||||
dn = new Intl.DisplayNames([locale], { type: "region" });
|
||||
} catch {
|
||||
dn = null;
|
||||
}
|
||||
return codes.map((c) => {
|
||||
if (!/^[A-Z]{2}$/.test(c)) return c;
|
||||
try {
|
||||
return dn?.of(c) ?? c;
|
||||
} catch {
|
||||
return c;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export interface EntityForNfo {
|
||||
kind: string;
|
||||
title: string;
|
||||
original_title: string | null;
|
||||
year: number | null;
|
||||
metadata: unknown;
|
||||
}
|
||||
|
||||
/** 已接受的外部身份:provider → id(tmdb / imdb / tvdb / douban) */
|
||||
export type IdentityMap = Partial<Record<"tmdb" | "imdb" | "tvdb" | "douban", string>>;
|
||||
|
||||
export function nfoWork(e: EntityForNfo, ids: IdentityMap): NfoWork {
|
||||
const m = e.metadata;
|
||||
const ext = (val(m, "externalIds") ?? {}) as { imdb?: unknown; tvdb?: unknown };
|
||||
const nfoIds: NfoIds = {
|
||||
tmdb: ids.tmdb ?? null,
|
||||
imdb: ids.imdb ?? str(ext.imdb),
|
||||
tvdb: ids.tvdb ?? str(ext.tvdb),
|
||||
douban: ids.douban ?? null,
|
||||
};
|
||||
const crew = credits(val(m, "crew"));
|
||||
const isTv = e.kind !== "movie";
|
||||
return {
|
||||
kind: isTv ? "tvshow" : "movie",
|
||||
title: str(val(m, "title")) ?? e.title,
|
||||
originalTitle: str(val(m, "originalTitle")) ?? e.original_title,
|
||||
year: numOf(val(m, "year")) ?? e.year,
|
||||
premiered: str(val(m, "releaseDate")),
|
||||
overview: str(val(m, "overview")),
|
||||
tagline: str(val(m, "tagline")),
|
||||
rating: numOf(val(m, "voteAverage")),
|
||||
genres: strs(val(m, "genres")),
|
||||
countries: countryNames(strs(val(m, "originCountry"))),
|
||||
studios: isTv
|
||||
? (str(val(m, "platform")) ?? "")
|
||||
.split(" / ")
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean)
|
||||
: [],
|
||||
runtime: isTv ? null : numOf(val(m, "runtime")),
|
||||
directors: crew
|
||||
.filter((c) => c.role === "Director")
|
||||
.map(person)
|
||||
.filter((p): p is NfoPerson => !!p),
|
||||
actors: credits(val(m, "cast"))
|
||||
.map(person)
|
||||
.filter((p): p is NfoPerson => !!p),
|
||||
ids: nfoIds,
|
||||
};
|
||||
}
|
||||
|
||||
export interface UnitForNfo {
|
||||
season_number: number | null;
|
||||
episode_number: number | null;
|
||||
title: string | null;
|
||||
air_date: string | null;
|
||||
metadata: unknown;
|
||||
}
|
||||
|
||||
export function nfoSeason(season: number, u: UnitForNfo | undefined): NfoSeason {
|
||||
const m = u?.metadata;
|
||||
return {
|
||||
season,
|
||||
title: str(val(m, "name")) ?? u?.title ?? null,
|
||||
overview: str(val(m, "overview")),
|
||||
airDate: str(val(m, "airDate")) ?? u?.air_date ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
/** TMDB 占位集名(「第 N 集」「Episode N」)按没有标题处理,NFO 里由生成器补「第 N 集」 */
|
||||
const PLACEHOLDER = /^(第\s*\d+\s*集|Episode\s+\d+)$/i;
|
||||
|
||||
export function nfoEpisode(season: number, episode: number, u: UnitForNfo | undefined): NfoEpisode {
|
||||
const m = u?.metadata;
|
||||
const title = str(val(m, "name")) ?? u?.title ?? null;
|
||||
return {
|
||||
season,
|
||||
episode,
|
||||
title: title && !PLACEHOLDER.test(title) ? title : null,
|
||||
overview: str(val(m, "overview")),
|
||||
airDate: str(val(m, "airDate")) ?? u?.air_date ?? null,
|
||||
rating: numOf(val(m, "voteAverage")),
|
||||
runtime: numOf(val(m, "runtime")),
|
||||
};
|
||||
}
|
||||
|
||||
/** 图片路径:作品海报 / 背景图,季海报,集剧照 */
|
||||
export function imagePathsOf(entityMeta: unknown) {
|
||||
return { posterPath: str(val(entityMeta, "posterPath")), backdropPath: str(val(entityMeta, "backdropPath")) };
|
||||
}
|
||||
export const unitPosterPath = (m: unknown) => str(val(m, "posterPath"));
|
||||
export const unitStillPath = (m: unknown) => str(val(m, "stillPath"));
|
||||
@@ -0,0 +1,484 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { lstat, mkdir, rename, rm, writeFile } from "node:fs/promises";
|
||||
import { dirname, isAbsolute, join, relative, resolve } from "node:path";
|
||||
import { checkRoot, DEFAULT_ROOT_SENTINEL } from "@umt/adapters";
|
||||
import {
|
||||
DEFAULT_SETTINGS,
|
||||
type LibraryScrapeState,
|
||||
type OrganizeSettings,
|
||||
type ScrapeFileResult,
|
||||
type ScrapeMode,
|
||||
} from "@umt/contracts";
|
||||
import { sql } from "@umt/db";
|
||||
import {
|
||||
episodeNfo,
|
||||
inferWorkDir,
|
||||
planScrapeTargets,
|
||||
type ScrapeTarget,
|
||||
type ScrapeVideo,
|
||||
seasonNfo,
|
||||
workNfo,
|
||||
} from "@umt/domain";
|
||||
import { errMessage, jsonb } from "../util.ts";
|
||||
import type { AoDeps } from "./common.ts";
|
||||
import {
|
||||
type IdentityMap,
|
||||
imagePathsOf,
|
||||
nfoEpisode,
|
||||
nfoSeason,
|
||||
nfoWork,
|
||||
type UnitForNfo,
|
||||
unitPosterPath,
|
||||
unitStillPath,
|
||||
} from "./scrape-input.ts";
|
||||
|
||||
// SCR-1 刮削:把作品的最终字段写成 NFO,下载海报 / 背景图 / 季海报 / 剧照,放进媒体库目录(Emby / Jellyfin / Kodi 读取)。
|
||||
// - 触发:整理入库成功后(设置 organize.scrapeAfterImport)、作品详情「重新刮削」、文件管理「刮削」,都排同一个任务 scrapeMedia(文件道)。
|
||||
// - 目标文件与命名:packages/domain/src/scrape/layout.ts(文件名跟随已入库视频,即命名模板的结果);内容:scrape/nfo.ts。
|
||||
// - 覆盖:missing 只补缺失(已存在的跳过)/ overwrite 全部覆盖;NFO 先写,图片后下(并发 4,失败只记该文件,不影响 NFO)。
|
||||
// - 写入:同目录临时文件 + rename(原子),UTF-8;只写 managed 库根,库根离线整体失败;目标必须仍在库根内。
|
||||
// - 结果:media_scrape 一部作品一行(最近一次,逐文件结果)+ 审计 organize.scrape;SSE files.changed 让文件管理刷新。
|
||||
|
||||
export const SCRAPE_TASK = "scrapeMedia";
|
||||
const IMAGE_CONCURRENCY = 4;
|
||||
|
||||
export interface ScrapeInput {
|
||||
mediaEntityId: string;
|
||||
trigger: "auto" | "manual";
|
||||
mode?: ScrapeMode | undefined;
|
||||
recordId?: string | undefined;
|
||||
/** 手动刮削的发起人(审计) */
|
||||
actor?: { type: string; id: string | null } | undefined;
|
||||
}
|
||||
|
||||
export async function scrapeSettings(d: AoDeps): Promise<OrganizeSettings> {
|
||||
const s = await d.ctx.settings.get("organize").catch(() => DEFAULT_SETTINGS.organize);
|
||||
return { ...DEFAULT_SETTINGS.organize, ...s };
|
||||
}
|
||||
|
||||
/**
|
||||
* 排刮削任务。同一作品已有排队中(还没开始)的刮削时直接复用(deduplicated);
|
||||
* 自动刮削按整理记录幂等(同一记录只排一次)。
|
||||
*/
|
||||
export async function enqueueScrape(d: AoDeps, input: ScrapeInput): Promise<{ taskId: string; deduplicated: boolean }> {
|
||||
const db = d.ctx.db;
|
||||
const mode = input.mode ?? (await scrapeSettings(d)).scrapeMode;
|
||||
const cur = await db
|
||||
.selectFrom("media_scrape")
|
||||
.select(["status", "engine_run_id", "requested_at"])
|
||||
.where("media_entity_id", "=", input.mediaEntityId)
|
||||
.executeTakeFirst();
|
||||
if (cur?.status === "queued" && cur.engine_run_id && Date.now() - new Date(cur.requested_at).getTime() < 30 * 60_000)
|
||||
return { taskId: cur.engine_run_id, deduplicated: true };
|
||||
const key =
|
||||
input.trigger === "auto" && input.recordId
|
||||
? `${SCRAPE_TASK}:record:${input.recordId}`
|
||||
: `${SCRAPE_TASK}:manual:${input.mediaEntityId}:${randomUUID()}`;
|
||||
const { runId, deduplicated } = await d.ctx.engine.enqueue(
|
||||
SCRAPE_TASK,
|
||||
{
|
||||
mediaEntityId: input.mediaEntityId,
|
||||
trigger: input.trigger,
|
||||
mode,
|
||||
...(input.recordId ? { recordId: input.recordId } : {}),
|
||||
...(input.actor ? { actor: input.actor } : {}),
|
||||
},
|
||||
{ idempotencyKey: key, traceId: randomUUID() },
|
||||
);
|
||||
if (!deduplicated)
|
||||
await sql`insert into media_scrape (media_entity_id, status, trigger, mode, record_id, engine_run_id, requested_at,
|
||||
started_at, finished_at, written, skipped, failed, reason, files)
|
||||
values (${input.mediaEntityId}, 'queued', ${input.trigger}, ${mode}, ${input.recordId ?? null}, ${runId}, now(),
|
||||
null, null, 0, 0, 0, null, '[]'::jsonb)
|
||||
on conflict (media_entity_id) do update set status = 'queued', trigger = excluded.trigger, mode = excluded.mode,
|
||||
record_id = excluded.record_id, engine_run_id = excluded.engine_run_id, requested_at = now(), updated_at = now()`.execute(
|
||||
db,
|
||||
);
|
||||
return { taskId: runId, deduplicated: !!deduplicated };
|
||||
}
|
||||
|
||||
/** 整理入库成功后:设置打开时自动排刮削(失败只记日志,不影响整理结果) */
|
||||
export async function scrapeAfterImport(d: AoDeps, entityId: string | null, recordId: string): Promise<void> {
|
||||
if (!entityId) return;
|
||||
try {
|
||||
const s = await scrapeSettings(d);
|
||||
if (!s.scrapeAfterImport) return;
|
||||
await enqueueScrape(d, { mediaEntityId: entityId, trigger: "auto", recordId });
|
||||
} catch (err) {
|
||||
d.ctx.logger.warn({ entityId, recordId, err: { message: errMessage(err) } }, "入库后自动刮削没排上");
|
||||
}
|
||||
}
|
||||
|
||||
export async function loadScrapeState(d: AoDeps, entityId: string): Promise<LibraryScrapeState | null> {
|
||||
const r = await d.ctx.db
|
||||
.selectFrom("media_scrape")
|
||||
.selectAll()
|
||||
.where("media_entity_id", "=", entityId)
|
||||
.executeTakeFirst();
|
||||
if (!r) return null;
|
||||
const iso = (t: Date | string | null) => (t ? new Date(t).toISOString() : null);
|
||||
return {
|
||||
status: r.status as LibraryScrapeState["status"],
|
||||
trigger: r.trigger as LibraryScrapeState["trigger"],
|
||||
mode: r.mode as ScrapeMode,
|
||||
requestedAt: iso(r.requested_at)!,
|
||||
startedAt: iso(r.started_at),
|
||||
finishedAt: iso(r.finished_at),
|
||||
written: r.written,
|
||||
skipped: r.skipped,
|
||||
failed: r.failed,
|
||||
reason: r.reason,
|
||||
files: (r.files ?? []) as ScrapeFileResult[],
|
||||
recordId: r.record_id,
|
||||
taskId: r.engine_run_id,
|
||||
};
|
||||
}
|
||||
|
||||
// ── 执行 ──
|
||||
|
||||
interface RootGroup {
|
||||
rootId: string;
|
||||
rootPath: string;
|
||||
managed: boolean;
|
||||
sentinel: string | null;
|
||||
videos: ScrapeVideo[];
|
||||
}
|
||||
|
||||
/** 原子写:同目录临时文件 → rename;目录不存在时建(只在库根内) */
|
||||
async function atomicWrite(abs: string, content: Uint8Array | string): Promise<void> {
|
||||
await mkdir(dirname(abs), { recursive: true });
|
||||
const tmp = join(dirname(abs), `.${randomUUID()}.umt-scrape.tmp`);
|
||||
try {
|
||||
await writeFile(tmp, content);
|
||||
await rename(tmp, abs);
|
||||
} catch (e) {
|
||||
await rm(tmp, { force: true }).catch(() => {});
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
const inside = (root: string, abs: string) => {
|
||||
const r = relative(root, abs);
|
||||
return !!r && !r.startsWith("..") && !isAbsolute(r);
|
||||
};
|
||||
|
||||
/** 有限并发 map */
|
||||
async function pool<T, R>(items: readonly T[], n: number, fn: (t: T) => Promise<R>): Promise<R[]> {
|
||||
const out: R[] = new Array(items.length);
|
||||
let i = 0;
|
||||
const worker = async () => {
|
||||
while (i < items.length) {
|
||||
const k = i++;
|
||||
out[k] = await fn(items[k]!);
|
||||
}
|
||||
};
|
||||
await Promise.all(Array.from({ length: Math.min(n, items.length) }, worker));
|
||||
return out;
|
||||
}
|
||||
|
||||
export interface ScrapeRunResult {
|
||||
status: LibraryScrapeState["status"];
|
||||
written: number;
|
||||
skipped: number;
|
||||
failed: number;
|
||||
reason: string | null;
|
||||
files: ScrapeFileResult[];
|
||||
}
|
||||
|
||||
export async function runScrape(
|
||||
d: AoDeps,
|
||||
input: ScrapeInput & { runId?: string },
|
||||
signal?: AbortSignal,
|
||||
): Promise<ScrapeRunResult> {
|
||||
const db = d.ctx.db;
|
||||
const settings = await scrapeSettings(d);
|
||||
const mode: ScrapeMode = input.mode ?? settings.scrapeMode;
|
||||
const entityId = input.mediaEntityId;
|
||||
const now = () => d.ports.now();
|
||||
await sql`insert into media_scrape (media_entity_id, status, trigger, mode, record_id, engine_run_id, started_at)
|
||||
values (${entityId}, 'running', ${input.trigger}, ${mode}, ${input.recordId ?? null}, ${input.runId ?? null}, ${now()})
|
||||
on conflict (media_entity_id) do update set status = 'running', trigger = excluded.trigger, mode = excluded.mode,
|
||||
record_id = coalesce(excluded.record_id, media_scrape.record_id),
|
||||
engine_run_id = coalesce(excluded.engine_run_id, media_scrape.engine_run_id),
|
||||
started_at = excluded.started_at, finished_at = null, reason = null, updated_at = now()`
|
||||
.execute(db)
|
||||
.catch(() => {});
|
||||
|
||||
const finish = async (r: ScrapeRunResult, rootDirs: Map<string, Set<string>> = new Map()) => {
|
||||
await db
|
||||
.updateTable("media_scrape")
|
||||
.set({
|
||||
status: r.status,
|
||||
finished_at: now(),
|
||||
written: r.written,
|
||||
skipped: r.skipped,
|
||||
failed: r.failed,
|
||||
reason: r.reason,
|
||||
files: jsonb(r.files),
|
||||
updated_at: now(),
|
||||
})
|
||||
.where("media_entity_id", "=", entityId)
|
||||
.execute()
|
||||
.catch(() => {});
|
||||
await db
|
||||
.insertInto("audit_event")
|
||||
.values({
|
||||
actor_type: input.actor?.type ?? "system",
|
||||
actor_id: input.actor?.id ?? null,
|
||||
action: "organize.scrape",
|
||||
target_type: "media_entity",
|
||||
target_id: entityId,
|
||||
payload: jsonb({
|
||||
trigger: input.trigger,
|
||||
mode,
|
||||
recordId: input.recordId ?? null,
|
||||
status: r.status,
|
||||
written: r.written,
|
||||
skipped: r.skipped,
|
||||
failed: r.failed,
|
||||
reason: r.reason,
|
||||
files: r.files.filter((f) => f.action !== "skipped").slice(0, 200),
|
||||
}),
|
||||
})
|
||||
.execute()
|
||||
.catch(() => {});
|
||||
for (const [rootId, dirs] of rootDirs)
|
||||
if (dirs.size) void d.ctx.events.publish("files.changed", { rootId: `lib:${rootId}`, dirs: [...dirs] });
|
||||
return r;
|
||||
};
|
||||
const fail = (reason: string) => finish({ status: "failed", written: 0, skipped: 0, failed: 0, reason, files: [] });
|
||||
|
||||
const e = await db
|
||||
.selectFrom("media_entity")
|
||||
.select(["id", "kind", "title", "original_title", "year", "metadata", "status"])
|
||||
.where("id", "=", entityId)
|
||||
.executeTakeFirst();
|
||||
if (!e || e.status === "merged") return fail("entity_not_found");
|
||||
const kind: "movie" | "tv" = e.kind === "movie" || e.kind === "collection" || e.kind === "other" ? "movie" : "tv";
|
||||
const meta = (e.metadata ?? {}) as Record<string, unknown>;
|
||||
if (!Object.keys(meta).length) return fail("scrape_no_metadata");
|
||||
|
||||
// 已入库的视频位置(库根内)
|
||||
const locs = await db
|
||||
.selectFrom("media_asset as a")
|
||||
.innerJoin("asset_location as l", "l.media_asset_id", "a.id")
|
||||
.innerJoin("library_root as r", "r.id", "l.library_root_id")
|
||||
.select([
|
||||
"a.id as asset_id",
|
||||
"l.relative_path",
|
||||
"r.id as root_id",
|
||||
"r.local_path",
|
||||
"r.management_mode",
|
||||
"r.enabled",
|
||||
"r.sentinel_file",
|
||||
])
|
||||
.where("a.media_entity_id", "=", entityId)
|
||||
.where("a.status", "=", "active")
|
||||
.where("a.kind", "=", "video")
|
||||
.where("l.location_kind", "=", "library")
|
||||
.where("l.status", "in", ["available", "unavailable", "suspected_missing"])
|
||||
.execute();
|
||||
if (!locs.length) return fail("scrape_no_files");
|
||||
|
||||
const assetIds = [...new Set(locs.map((l) => l.asset_id))];
|
||||
const cov = await db
|
||||
.selectFrom("asset_unit_coverage")
|
||||
.select(["media_asset_id", "content_unit_id"])
|
||||
.where("media_asset_id", "in", assetIds)
|
||||
.execute();
|
||||
const units = await db
|
||||
.selectFrom("content_unit")
|
||||
.select(["id", "kind", "season_number", "episode_number", "title", "air_date", "metadata"])
|
||||
.where("media_entity_id", "=", entityId)
|
||||
.where("status", "!=", "retired")
|
||||
.orderBy("created_at")
|
||||
.execute();
|
||||
const unitById = new Map(units.map((u) => [u.id, u]));
|
||||
const seasonUnit = new Map<number, UnitForNfo>();
|
||||
const episodeUnit = new Map<string, UnitForNfo>();
|
||||
for (const u of units) {
|
||||
if (u.kind === "season" && u.season_number !== null && !seasonUnit.has(u.season_number))
|
||||
seasonUnit.set(u.season_number, u);
|
||||
if (u.kind === "episode" && u.season_number !== null && u.episode_number !== null) {
|
||||
const k = `${u.season_number}:${u.episode_number}`;
|
||||
if (!episodeUnit.has(k)) episodeUnit.set(k, u);
|
||||
}
|
||||
}
|
||||
|
||||
const groups = new Map<string, RootGroup>();
|
||||
for (const l of locs) {
|
||||
if (!l.enabled) continue;
|
||||
const g = groups.get(l.root_id) ?? {
|
||||
rootId: l.root_id,
|
||||
rootPath: l.local_path,
|
||||
managed: l.management_mode === "managed",
|
||||
sentinel: l.sentinel_file,
|
||||
videos: [],
|
||||
};
|
||||
let season: number | null = null;
|
||||
let episode: number | null = null;
|
||||
if (kind === "tv") {
|
||||
const eps = cov
|
||||
.filter((c) => c.media_asset_id === l.asset_id)
|
||||
.map((c) => unitById.get(c.content_unit_id))
|
||||
.filter((u) => u?.kind === "episode" && u.season_number !== null && u.episode_number !== null)
|
||||
.sort((a, b) => a!.season_number! - b!.season_number! || a!.episode_number! - b!.episode_number!);
|
||||
season = eps[0]?.season_number ?? null;
|
||||
episode = eps[0]?.episode_number ?? null;
|
||||
}
|
||||
g.videos.push({ path: l.relative_path, season, episode });
|
||||
groups.set(l.root_id, g);
|
||||
}
|
||||
if (!groups.size) return fail("scrape_no_files");
|
||||
|
||||
const idRows = await db
|
||||
.selectFrom("external_identity")
|
||||
.select(["provider", "external_id"])
|
||||
.where("media_entity_id", "=", entityId)
|
||||
.where("content_unit_id", "is", null)
|
||||
.where("status", "=", "accepted")
|
||||
.orderBy("updated_at", "desc")
|
||||
.execute()
|
||||
.catch(() => [] as { provider: string; external_id: string }[]);
|
||||
const ids: IdentityMap = {};
|
||||
for (const r of idRows) {
|
||||
const p = r.provider.toLowerCase() as keyof IdentityMap;
|
||||
if (["tmdb", "imdb", "tvdb", "douban"].includes(p) && !ids[p]) ids[p] = r.external_id;
|
||||
}
|
||||
const bindings = await db
|
||||
.selectFrom("path_binding")
|
||||
.select(["library_root_id", "relative_path"])
|
||||
.where("media_entity_id", "=", entityId)
|
||||
.where("binding_kind", "=", "entity")
|
||||
.where("status", "=", "active")
|
||||
.execute();
|
||||
|
||||
const work = nfoWork(e, ids);
|
||||
const art = imagePathsOf(meta);
|
||||
const seasonPosters = new Map<number, string | null>();
|
||||
for (const [n, u] of seasonUnit) seasonPosters.set(n, unitPosterPath(u.metadata));
|
||||
const stills = new Map<string, string | null>();
|
||||
for (const [k, u] of episodeUnit) stills.set(k, unitStillPath(u.metadata));
|
||||
|
||||
const files: ScrapeFileResult[] = [];
|
||||
const rootDirs = new Map<string, Set<string>>();
|
||||
const imageCache = new Map<string, Promise<Uint8Array>>();
|
||||
const fetchImage = (p: string) => {
|
||||
let hit = imageCache.get(p);
|
||||
if (!hit) {
|
||||
hit = d.ports.scrapeImage(p).then((r) => r.content);
|
||||
imageCache.set(p, hit);
|
||||
}
|
||||
return hit;
|
||||
};
|
||||
|
||||
for (const g of groups.values()) {
|
||||
if (!g.managed) {
|
||||
files.push(
|
||||
...g.videos.map((v) => ({
|
||||
path: v.path,
|
||||
kind: "nfo" as const,
|
||||
role: kind === "movie" ? "movie_nfo" : "episode_nfo",
|
||||
action: "skipped" as const,
|
||||
reason: "scrape_root_read_only",
|
||||
})),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
const health = await checkRoot(g.rootPath, g.sentinel ?? DEFAULT_ROOT_SENTINEL);
|
||||
if (health.kind !== "online") {
|
||||
return finish({
|
||||
status: "failed",
|
||||
written: 0,
|
||||
skipped: 0,
|
||||
failed: 0,
|
||||
reason: "library_root_offline",
|
||||
files,
|
||||
});
|
||||
}
|
||||
const bound = bindings.find((b) => b.library_root_id === g.rootId)?.relative_path ?? null;
|
||||
const workDir = bound ?? inferWorkDir(kind, g.videos[0]!.path);
|
||||
const targets = planScrapeTargets({
|
||||
kind,
|
||||
workDir,
|
||||
videos: g.videos,
|
||||
images: { ...art, seasonPosters, stills },
|
||||
options: { images: settings.scrapeImages, episodeThumbs: settings.scrapeEpisodeThumbs },
|
||||
});
|
||||
if (kind === "tv" && !workDir)
|
||||
files.push({ path: "", kind: "nfo", role: "tvshow_nfo", action: "skipped", reason: "scrape_no_work_dir" });
|
||||
const rootAbs = resolve(g.rootPath);
|
||||
const dirs = rootDirs.get(g.rootId) ?? new Set<string>();
|
||||
rootDirs.set(g.rootId, dirs);
|
||||
|
||||
const decide = async (t: ScrapeTarget): Promise<{ abs: string } | ScrapeFileResult> => {
|
||||
const abs = resolve(rootAbs, t.path);
|
||||
const base = { path: t.path, kind: t.kind, role: t.role };
|
||||
if (!inside(rootAbs, abs)) return { ...base, action: "failed", reason: "path_escape" };
|
||||
const st = await lstat(abs).catch(() => null);
|
||||
if (st && (!st.isFile() || st.isSymbolicLink())) return { ...base, action: "skipped", reason: "scrape_exists" };
|
||||
if (st && mode === "missing") return { ...base, action: "skipped", reason: "scrape_exists" };
|
||||
return { abs };
|
||||
};
|
||||
const write = async (t: ScrapeTarget, abs: string, content: Uint8Array | string): Promise<ScrapeFileResult> => {
|
||||
const base = { path: t.path, kind: t.kind, role: t.role };
|
||||
try {
|
||||
await atomicWrite(abs, content);
|
||||
dirs.add(dirname(t.path) === "." ? "" : dirname(t.path));
|
||||
return { ...base, action: "written", reason: null };
|
||||
} catch (err) {
|
||||
d.ctx.logger.warn({ entityId, path: t.path, err: { message: errMessage(err) } }, "刮削文件写入失败");
|
||||
return { ...base, action: "failed", reason: "scrape_write_failed" };
|
||||
}
|
||||
};
|
||||
|
||||
// NFO 先写(不依赖网络)
|
||||
for (const t of targets.filter((x) => x.kind === "nfo")) {
|
||||
if (signal?.aborted) throw new Error("aborted");
|
||||
const dec = await decide(t);
|
||||
if (!("abs" in dec)) {
|
||||
files.push(dec);
|
||||
continue;
|
||||
}
|
||||
let xml: string;
|
||||
if (t.role === "movie_nfo" || t.role === "tvshow_nfo") xml = workNfo(work);
|
||||
else if (t.role === "season_nfo") xml = seasonNfo(nfoSeason(t.season!, seasonUnit.get(t.season!)));
|
||||
else xml = episodeNfo(nfoEpisode(t.season!, t.episode!, episodeUnit.get(`${t.season}:${t.episode}`)));
|
||||
files.push(await write(t, dec.abs, xml));
|
||||
}
|
||||
// 图片:限并发,失败不影响 NFO
|
||||
const imgs = targets.filter((x) => x.kind === "image" && x.imagePath);
|
||||
const results = await pool(imgs, IMAGE_CONCURRENCY, async (t): Promise<ScrapeFileResult> => {
|
||||
if (signal?.aborted) throw new Error("aborted");
|
||||
const dec = await decide(t);
|
||||
if (!("abs" in dec)) return dec;
|
||||
let bytes: Uint8Array;
|
||||
try {
|
||||
bytes = await fetchImage(t.imagePath!);
|
||||
} catch (err) {
|
||||
d.ctx.logger.warn({ entityId, image: t.imagePath, err: { message: errMessage(err) } }, "刮削图片下载失败");
|
||||
return { path: t.path, kind: t.kind, role: t.role, action: "failed", reason: "scrape_image_failed" };
|
||||
}
|
||||
return write(t, dec.abs, bytes);
|
||||
});
|
||||
files.push(...results);
|
||||
}
|
||||
|
||||
const written = files.filter((f) => f.action === "written").length;
|
||||
const failed = files.filter((f) => f.action === "failed").length;
|
||||
const skipped = files.filter((f) => f.action === "skipped").length;
|
||||
const onlyReadOnly = files.length > 0 && files.every((f) => f.reason === "scrape_root_read_only");
|
||||
const status: ScrapeRunResult["status"] = onlyReadOnly
|
||||
? "failed"
|
||||
: failed === 0
|
||||
? "succeeded"
|
||||
: written > 0 || skipped > 0
|
||||
? "partial"
|
||||
: "failed";
|
||||
const reason = onlyReadOnly
|
||||
? "scrape_root_read_only"
|
||||
: failed
|
||||
? (files.find((f) => f.action === "failed")?.reason ?? null)
|
||||
: null;
|
||||
return finish({ status, written, skipped, failed, reason, files }, rootDirs);
|
||||
}
|
||||
+8
@@ -107,6 +107,14 @@ export class ImageCache {
|
||||
return new Response(content as BodyInit, { status: 200, headers });
|
||||
}
|
||||
|
||||
/**
|
||||
* SCR-1 刮削取图:TMDB 路径(`/xx.jpg`)→ 图片字节。同一缓存目录与出口代理;命中磁盘缓存就不再请求
|
||||
* (TMDB 图片路径内容不变,刮削不看 GLOBAL_IMAGE_CACHE 开关)。
|
||||
*/
|
||||
async bytes(size: string, tmdbPath: string): Promise<{ content: Uint8Array; mime: string }> {
|
||||
return this.fetchImage(this.upstream(size, tmdbPath.replace(/^\/+/, "")), true);
|
||||
}
|
||||
|
||||
/** ImageHelper.fetch_image_with_mime_type */
|
||||
private async fetchImage(url: string, useCache: boolean): Promise<{ content: Uint8Array; mime: string }> {
|
||||
const path = join(this.opts.dir, imageCachePath(url));
|
||||
|
||||
@@ -13,6 +13,7 @@ import type {
|
||||
} from "@umt/contracts";
|
||||
import { sql } from "@umt/db";
|
||||
import type { Principal } from "../../http/auth.ts";
|
||||
import { enqueueScrape } from "../acquisition-organize/services/scrape.ts";
|
||||
import { FilesError, type FmDeps, jsonb, resolveIn } from "./common.ts";
|
||||
import { audit } from "./ledger.ts";
|
||||
import { kindOfName, mediaSummary } from "./list.ts";
|
||||
@@ -20,7 +21,8 @@ import { entitiesUnder } from "./records.ts";
|
||||
|
||||
// probe:ffprobe 摘要;已登记资产写回 media_asset.media_info.probe,未登记只缓存 24h(DATA_DIR/cache/probe)。
|
||||
// scrape:只对已登记作品,等价 POST /media-entities/{id}/refresh(入队 refreshMetadata,字段锁优先、不动路径);
|
||||
// NFO/图片投影 W5 实现(TODO),请求时返回 not_implemented。
|
||||
// writeNfo / images(SCR-1)另排 scrapeMedia,把 NFO 与图片写进媒体库目录(与作品详情「重新刮削」同一任务)。
|
||||
// scrape-batch write=true:文件管理「刮削」,按作品去重各排一次 scrapeMedia(不刷新元数据)。
|
||||
|
||||
const CACHE_TTL_MS = 24 * 3600_000;
|
||||
|
||||
@@ -133,13 +135,23 @@ export async function scrape(
|
||||
.where("locked", "=", true)
|
||||
.execute();
|
||||
const lockedFields = [...new Set([...(e.field_locks ?? []), ...overrides.map((o) => o.field)])].sort();
|
||||
// TODO(W5):writeNfo/images → 把最终字段投影成 NFO/海报写到绑定目录(临时文件 + 原子提交;非本系统 NFO 不覆盖)
|
||||
const scrapeTaskId =
|
||||
body.writeNfo || body.images
|
||||
? (
|
||||
await enqueueScrape(d.ao, {
|
||||
mediaEntityId: e.id,
|
||||
trigger: "manual",
|
||||
actor: { type: principal?.kind === "api_key" ? "api_key" : "user", id: principal?.id ?? null },
|
||||
})
|
||||
).taskId
|
||||
: null;
|
||||
const result: FilesScrapeResult = {
|
||||
mediaEntityId: e.id,
|
||||
taskId: runId,
|
||||
lockedFields,
|
||||
nfo: body.writeNfo ? "not_implemented" : "not_requested",
|
||||
images: body.images ? "not_implemented" : "not_requested",
|
||||
nfo: body.writeNfo ? "queued" : "not_requested",
|
||||
images: body.images ? "queued" : "not_requested",
|
||||
scrapeTaskId,
|
||||
};
|
||||
await audit(d, principal, "scrape", { ...result });
|
||||
return result;
|
||||
@@ -202,17 +214,28 @@ export async function scrapeBatch(
|
||||
skipped.push({ ref: id, reason: "not_registered" });
|
||||
continue;
|
||||
}
|
||||
const { runId } = await d.ctx.engine.enqueue(
|
||||
"refreshMetadata",
|
||||
{ mediaEntityId: id },
|
||||
{ idempotencyKey: `refreshMetadata:files-batch:${body.idempotencyKey}:${id}`, traceId: randomUUID() },
|
||||
);
|
||||
const runId = body.write
|
||||
? (
|
||||
await enqueueScrape(d.ao, {
|
||||
mediaEntityId: id,
|
||||
trigger: "manual",
|
||||
actor: { type: principal?.kind === "api_key" ? "api_key" : "user", id: principal?.id ?? null },
|
||||
})
|
||||
).taskId
|
||||
: (
|
||||
await d.ctx.engine.enqueue(
|
||||
"refreshMetadata",
|
||||
{ mediaEntityId: id },
|
||||
{ idempotencyKey: `refreshMetadata:files-batch:${body.idempotencyKey}:${id}`, traceId: randomUUID() },
|
||||
)
|
||||
).runId;
|
||||
const lockedFields = [
|
||||
...new Set([...(e.field_locks ?? []), ...locks.filter((l) => l.media_entity_id === id).map((l) => l.field)]),
|
||||
].sort();
|
||||
submitted.push({ mediaEntityId: id, title: e.title, taskId: runId, lockedFields });
|
||||
}
|
||||
await audit(d, principal, "scrape_batch", {
|
||||
write: body.write,
|
||||
submitted: submitted.map((s) => s.mediaEntityId),
|
||||
skipped,
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user