fix(scrape): 整理后刮削照 MP 对作品目录刮,集只刮本次整理的文件
原地入库逐个文件排刮削,单文件刮削只写该集,剧集根目录(tvshow.nfo、海报、季海报)与季目录 (season.nfo)没有元数据。照 MP ScrapingChain.scrape_metadata_event:按作品目录分组,目录元数据照常 初始化,集只刮本次文件;复制 / 移动同样改为只刮本次文件(此前刮整个作品目录)。 - 迁移 0062:scrape_record.videos(null = 目录下全部) - queueScrape 支持 videos;同目录排队中还没开始的刮削合成一批(有一方刮全部则刮全部) - 作品详情「重新刮削」与文件管理刮削不变(目录下全部) Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
501b4eb403
commit
f5bc6e9c40
7 files changed
+110
-36
No files matched your search
@@ -77,6 +77,8 @@ export interface QueueScrapeInput extends ScrapeLocation {
|
||||
actor?: { type: string; id: string | null } | undefined;
|
||||
/** 引擎幂等键(入库后自动刮削按整理记录) */
|
||||
idempotencyKey?: string | undefined;
|
||||
/** 只刮这些视频(库根相对路径;整理后刮削用,作品 / 季目录元数据照常初始化);缺省 = 目录下全部 */
|
||||
videos?: readonly string[] | undefined;
|
||||
}
|
||||
|
||||
type Row = {
|
||||
@@ -93,6 +95,7 @@ type Row = {
|
||||
reason: string | null;
|
||||
files: unknown;
|
||||
organization_record_id: string | null;
|
||||
videos: string[] | null;
|
||||
engine_run_id: string | null;
|
||||
requested_at: Date | string;
|
||||
started_at: Date | string | null;
|
||||
@@ -116,6 +119,7 @@ export function scrapeRecordView(r: Row): ScrapeRecord {
|
||||
reason: r.reason,
|
||||
files: (r.files ?? []) as ScrapeFileResult[],
|
||||
organizationRecordId: r.organization_record_id,
|
||||
videos: r.videos ?? null,
|
||||
taskId: r.engine_run_id,
|
||||
requestedAt: iso(r.requested_at)!,
|
||||
startedAt: iso(r.started_at),
|
||||
@@ -157,7 +161,7 @@ export async function queueScrape(
|
||||
const mode = input.mode ?? (await scrapeSettings(d)).scrapeMode;
|
||||
const cur = await db
|
||||
.selectFrom("scrape_record")
|
||||
.select(["id", "engine_run_id", "media_entity_id", "match"])
|
||||
.select(["id", "engine_run_id", "media_entity_id", "match", "videos"])
|
||||
.where("root_id", "=", input.rootId)
|
||||
.where("path", "=", input.path)
|
||||
.where("mode", "=", mode)
|
||||
@@ -165,10 +169,17 @@ export async function queueScrape(
|
||||
.where("requested_at", ">", new Date(Date.now() - DEDUPE_MS))
|
||||
.orderBy("requested_at", "desc")
|
||||
.executeTakeFirst();
|
||||
const videos = input.videos?.length ? [...new Set(input.videos)].sort() : null;
|
||||
const sameMatch =
|
||||
!!cur &&
|
||||
(cur.media_entity_id ?? null) === input.match.mediaEntityId &&
|
||||
((cur.match as ScrapeMatch | null)?.externalId ?? null) === input.match.externalId;
|
||||
if (cur && sameMatch) {
|
||||
// 同一目录还没开始的刮削合成一批(MP 按目标目录合批):有一方刮全部则刮全部,否则合并文件
|
||||
const merged = cur.videos && videos ? [...new Set([...cur.videos, ...videos])].sort() : null;
|
||||
if (JSON.stringify(merged) !== JSON.stringify(cur.videos ? [...cur.videos].sort() : null))
|
||||
await db.updateTable("scrape_record").set({ videos: merged }).where("id", "=", cur.id).execute();
|
||||
}
|
||||
if (cur && sameMatch && cur.engine_run_id) return { recordId: cur.id, taskId: cur.engine_run_id, deduplicated: true };
|
||||
const reused = !!cur && sameMatch;
|
||||
const rec = reused
|
||||
@@ -188,6 +199,7 @@ export async function queueScrape(
|
||||
organization_record_id: input.organizationRecordId ?? null,
|
||||
actor_type: input.actor?.type ?? (input.trigger === "auto" ? "system" : null),
|
||||
actor_id: input.actor?.id ?? null,
|
||||
videos,
|
||||
})
|
||||
.returning("id")
|
||||
.executeTakeFirstOrThrow();
|
||||
@@ -362,7 +374,7 @@ export async function scrapeEntity(
|
||||
return out;
|
||||
}
|
||||
|
||||
/** 整理入库成功后:设置打开或本次 Local 补全要求写本地元数据时,复用作品目录刮削任务。 */
|
||||
/** 整理入库成功后(设置打开或本次要求刮削):照 MP 对作品目录刮削,集只刮本次整理的文件。 */
|
||||
export async function scrapeAfterImport(
|
||||
d: AoDeps,
|
||||
entityId: string | null,
|
||||
@@ -376,40 +388,46 @@ export async function scrapeAfterImport(
|
||||
if (!s.scrapeAfterImport && !opts.force && opts.enabled !== true) return;
|
||||
const organization = await d.ctx.db
|
||||
.selectFrom("organization_record")
|
||||
.select(["action", "target_library_root_id"])
|
||||
.select(["target_library_root_id"])
|
||||
.where("id", "=", recordId)
|
||||
.executeTakeFirst();
|
||||
if (organization?.action === "in_place" && organization.target_library_root_id) {
|
||||
const match = await entityScrapeMatch(d, entityId, "library");
|
||||
const root = await d.ctx.db
|
||||
.selectFrom("library_root")
|
||||
.selectAll()
|
||||
.where("id", "=", organization.target_library_root_id)
|
||||
.executeTakeFirst();
|
||||
if (!match || !root || root.management_mode !== "managed") return;
|
||||
const files = await d.ctx.db
|
||||
.selectFrom("organization_file_result")
|
||||
.select("target_path")
|
||||
.where("organization_record_id", "=", recordId)
|
||||
.where("role", "=", "main")
|
||||
.where("status", "=", "succeeded")
|
||||
.orderBy("ordinal")
|
||||
.execute();
|
||||
for (const file of files)
|
||||
if (file.target_path)
|
||||
await queueScrape(d, {
|
||||
rootId: `lib:${root.id}`,
|
||||
rootPath: root.local_path,
|
||||
libraryRootId: root.id,
|
||||
path: file.target_path,
|
||||
match,
|
||||
trigger: "auto",
|
||||
organizationRecordId: recordId,
|
||||
idempotencyKey: `${SCRAPE_TASK}:record:${recordId}:${file.target_path}`,
|
||||
});
|
||||
return;
|
||||
const match = await entityScrapeMatch(d, entityId, "library");
|
||||
const root = organization?.target_library_root_id
|
||||
? await d.ctx.db
|
||||
.selectFrom("library_root")
|
||||
.selectAll()
|
||||
.where("id", "=", organization.target_library_root_id)
|
||||
.executeTakeFirst()
|
||||
: undefined;
|
||||
if (!match || !root || root.management_mode !== "managed") return;
|
||||
const files = await d.ctx.db
|
||||
.selectFrom("organization_file_result")
|
||||
.select("target_path")
|
||||
.where("organization_record_id", "=", recordId)
|
||||
.where("role", "=", "main")
|
||||
.where("status", "=", "succeeded")
|
||||
.orderBy("ordinal")
|
||||
.execute();
|
||||
// 照 MP 整理后刮削:按作品目录分组,对作品目录刮(作品 / 季的 NFO 与图片照常初始化),集只刮本次整理的文件;
|
||||
// 视频直接在库根(推不出作品目录)时对文件本身刮。
|
||||
const byDir = new Map<string, string[]>();
|
||||
for (const f of files) {
|
||||
if (!f.target_path) continue;
|
||||
const dir = inferWorkDir(match.kind, f.target_path) ?? f.target_path;
|
||||
byDir.set(dir, [...(byDir.get(dir) ?? []), f.target_path]);
|
||||
}
|
||||
await scrapeEntity(d, entityId, { trigger: "auto", organizationRecordId: recordId });
|
||||
for (const [dir, videos] of byDir)
|
||||
await queueScrape(d, {
|
||||
rootId: `lib:${root.id}`,
|
||||
rootPath: root.local_path,
|
||||
libraryRootId: root.id,
|
||||
path: dir,
|
||||
match,
|
||||
trigger: "auto",
|
||||
organizationRecordId: recordId,
|
||||
idempotencyKey: `${SCRAPE_TASK}:record:${recordId}:${dir}`,
|
||||
...(dir === videos[0] ? {} : { videos }),
|
||||
});
|
||||
} catch (err) {
|
||||
d.ctx.logger.warn({ entityId, recordId, err: { message: errMessage(err) } }, "入库后自动刮削没排上");
|
||||
}
|
||||
@@ -639,6 +657,8 @@ export async function runScrapeRecord(
|
||||
|
||||
const kind = match.kind;
|
||||
const lay = await layoutOf(rootReal, rec.path, isDir, kind);
|
||||
// 整理后刮削:目录布局照常(作品 / 季元数据),集只刮本次整理的文件
|
||||
if (rec.videos) lay.videos = rec.videos.filter((v) => !rec.path || v === rec.path || v.startsWith(`${rec.path}/`));
|
||||
if (!lay.videos.length) return await fail("scrape_no_videos");
|
||||
|
||||
// ── 字段:本地作品(有元数据)优先 ──
|
||||
|
||||
@@ -183,7 +183,8 @@ describe.skipIf(!hasPg)("刮削:NFO 与图片写进媒体库目录", () => {
|
||||
const rec = await organize(series, "Severance.S01E01.1080p.WEB-DL.H264-GRP.mkv");
|
||||
await organize(series, "Severance.S01E02.1080p.WEB-DL.H264-GRP.mkv");
|
||||
const q = h.engine.enqueued.filter((x) => x.task === SCRAPE_TASK);
|
||||
expect(q[0]!.key).toBe(`${SCRAPE_TASK}:record:${rec}:0`);
|
||||
// 照 MP:对作品目录刮,集只刮本次整理的文件;两批还没开始时合成一条
|
||||
expect(q[0]!.key.startsWith(`${SCRAPE_TASK}:record:${rec}:`)).toBe(true);
|
||||
const first = await h.db
|
||||
.selectFrom("scrape_record")
|
||||
.selectAll()
|
||||
|
||||
@@ -3,6 +3,7 @@ import { mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { executeRecord } from "../../src/modules/acquisition-organize/services/organize.ts";
|
||||
import { runScrapeRecord } from "../../src/modules/acquisition-organize/services/scrape.ts";
|
||||
import { createFmHarness, type FmHarness, key, pgAvailable, seedEntity } from "./harness.ts";
|
||||
|
||||
const skip = !(await pgAvailable());
|
||||
@@ -100,6 +101,37 @@ describe.skipIf(skip)("file manager in-place import", () => {
|
||||
.execute();
|
||||
expect(units.some((u) => u.season_number === 2 && u.episode_number === 7)).toBe(true);
|
||||
});
|
||||
it("剧集原地入库后照 MP 刮削:剧集根目录与季目录写元数据,集只刮本次登记的", async () => {
|
||||
const title = `Show-${randomUUID().slice(0, 8)}`;
|
||||
const entity = await seedEntity(h.db, { title, kind: "series", year: 2022 });
|
||||
const at = new Date().toISOString();
|
||||
await h.db
|
||||
.updateTable("media_entity")
|
||||
.set({
|
||||
metadata: JSON.stringify({ title: { value: title, source: "tmdb", language: "zh-CN", at, locked: false } }),
|
||||
})
|
||||
.where("id", "=", entity)
|
||||
.execute();
|
||||
await mkdir(join(h.lib, title, "Season 1"), { recursive: true });
|
||||
const ep1 = `${title}/Season 1/original.name.S01E01.mkv`;
|
||||
const ep2 = `${title}/Season 1/original.name.S01E02.mkv`;
|
||||
for (const p of [ep1, ep2]) await writeFile(join(h.lib, p), "video");
|
||||
const r = await queue([{ root: h.libRoot, path: ep1, mediaEntityId: entity }], { scrapeAfterImport: true });
|
||||
expect((await executeRecord(h.module.deps().ao, r.recordId)).status).toBe("succeeded");
|
||||
const recs = await h.db
|
||||
.selectFrom("scrape_record")
|
||||
.select(["id", "path", "videos"])
|
||||
.where("organization_record_id", "=", r.recordId)
|
||||
.execute();
|
||||
expect(recs).toMatchObject([{ path: title, videos: [ep1] }]);
|
||||
await runScrapeRecord(h.module.deps().ao, recs[0]!.id);
|
||||
const show = await readdir(join(h.lib, title));
|
||||
const season = await readdir(join(h.lib, title, "Season 1"));
|
||||
expect(show).toContain("tvshow.nfo");
|
||||
expect(season).toContain("season.nfo");
|
||||
expect(season).toContain("original.name.S01E01.nfo");
|
||||
expect(season).not.toContain("original.name.S01E02.nfo");
|
||||
});
|
||||
it("scrapes only the newly registered file, retaining its organization association on resume", async () => {
|
||||
const s = await source("Scrape");
|
||||
await h.db
|
||||
@@ -118,10 +150,13 @@ describe.skipIf(skip)("file manager in-place import", () => {
|
||||
await executeRecord(h.module.deps().ao, r.recordId);
|
||||
const scrapes = await h.db
|
||||
.selectFrom("scrape_record")
|
||||
.select(["path", "organization_record_id"])
|
||||
.select(["path", "organization_record_id", "videos"])
|
||||
.where("organization_record_id", "=", r.recordId)
|
||||
.execute();
|
||||
expect(scrapes).toEqual([{ path: s.path, organization_record_id: r.recordId }]);
|
||||
// 照 MP:对作品目录刮(目录海报 / 背景照常),视频只刮本次登记的这一个
|
||||
expect(scrapes).toEqual([
|
||||
{ path: s.path.split("/").slice(0, -1).join("/"), organization_record_id: r.recordId, videos: [s.path] },
|
||||
]);
|
||||
expect(await readFile(join(h.lib, s.path.split("/")[0]!, "Unrelated.mkv"), "utf8")).toBe("unrelated video");
|
||||
});
|
||||
it("rejects an unregistered download directory without submitting a task", async () => {
|
||||
|
||||
@@ -765,6 +765,7 @@ function mockScrape(b: {
|
||||
reason: null,
|
||||
files,
|
||||
organizationRecordId: null,
|
||||
videos: null,
|
||||
taskId: `run-${seq}`,
|
||||
requestedAt: now,
|
||||
startedAt: now,
|
||||
|
||||
Reference in new issue
Block a user