feat(scrape): 入库后自动刮削缺省开启,没排上时在整理记录上记「未刮削（原因）」
- scrape_record 状态加 skipped(迁移 0074);目标不在媒体库、库根只登记、作品没有 TMDB id 也没有本地元数据、排队失败时 留一条挂在整理记录上的「未刮削」记录(同原因只记一次),设置关闭不记; - 缺省开启 scrapeAfterImport,只对有 TMDB id 或本地元数据的作品排; - 已结束记录重放与收尾同一口径补排刮削(按整理记录幂等); - 整理详情「整理后刮削」显示未刮削原因,并列出失败与非预期跳过(缺元数据、认不出集号)的文件。 Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
8c4c7d028a
commit
cdf62dbf02
12 files changed
+320
-49
No files matched your search
@@ -82,10 +82,8 @@ export async function executeRecord(
|
||||
await settleSamePathReplacement(d, rec.plan as unknown as FrozenPlan, recordId);
|
||||
await cleanupReplacement(d, rec.plan as unknown as FrozenPlan, recordId);
|
||||
const frozen = rec.plan as unknown as FrozenPlan;
|
||||
if (
|
||||
(frozen.metadataMode === "local" || frozen.scrapeAfterImport !== undefined) &&
|
||||
["succeeded", "partial"].includes(rec.status)
|
||||
) {
|
||||
// 与 finalizeRecord 同一口径(按设置 / 本次选项);排队按整理记录幂等,重放不会重复刮
|
||||
if (["succeeded", "partial"].includes(rec.status)) {
|
||||
await scrapeAfterImport(d, frozen.entityId, recordId, {
|
||||
force: frozen.metadataMode === "local",
|
||||
enabled: frozen.scrapeAfterImport,
|
||||
|
||||
@@ -422,7 +422,55 @@ export async function scheduleEntityRescrape(d: QueueDeps, entityId: string): Pr
|
||||
return out.length;
|
||||
}
|
||||
|
||||
/** 整理入库成功后(设置打开或本次要求刮削):照 MP 对作品目录刮削,集只刮本次整理的文件。 */
|
||||
/**
|
||||
* 入库后自动刮削没排上:写一条 skipped 刮削记录挂在整理记录上(整理详情「整理后刮削」显示「未刮削（原因）」)。
|
||||
* 同一整理记录同一原因只记一次(崩溃重放会再调)。失败只告警。
|
||||
*/
|
||||
async function recordScrapeSkipped(
|
||||
d: AoDeps,
|
||||
recordId: string,
|
||||
reason: string,
|
||||
at: { root?: { id: string; local_path: string } | undefined; path?: string; match?: ScrapeMatch | null },
|
||||
): Promise<void> {
|
||||
try {
|
||||
const dup = await d.ctx.db
|
||||
.selectFrom("scrape_record")
|
||||
.select("id")
|
||||
.where("organization_record_id", "=", recordId)
|
||||
.where("status", "=", "skipped")
|
||||
.where("reason", "=", reason)
|
||||
.executeTakeFirst();
|
||||
if (dup) return;
|
||||
const now = d.ports.now();
|
||||
await d.ctx.db
|
||||
.insertInto("scrape_record")
|
||||
.values({
|
||||
root_id: at.root ? `lib:${at.root.id}` : "",
|
||||
root_path: at.root?.local_path ?? "",
|
||||
library_root_id: at.root?.id ?? null,
|
||||
path: at.path ?? "",
|
||||
media_entity_id: at.match?.mediaEntityId ?? null,
|
||||
match: at.match ? jsonb(at.match) : null,
|
||||
status: "skipped",
|
||||
trigger: "auto",
|
||||
mode: (await scrapeSettings(d)).scrapeMode,
|
||||
organization_record_id: recordId,
|
||||
actor_type: "system",
|
||||
reason,
|
||||
requested_at: now,
|
||||
finished_at: now,
|
||||
})
|
||||
.execute();
|
||||
} catch (err) {
|
||||
d.ctx.logger.warn({ recordId, reason, err: { message: errMessage(err) } }, "记录「未刮削」失败");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 整理入库成功后(设置打开或本次要求刮削):照 MP 对作品目录刮削,集只刮本次整理的文件。
|
||||
* 只对有 TMDB id 或本地元数据(Local)的作品排;没排上(目标不在媒体库、库根只登记、作品没有元数据、排队失败)时
|
||||
* 在整理记录上记「未刮削（原因）」;设置关闭是预期,不记。
|
||||
*/
|
||||
export async function scrapeAfterImport(
|
||||
d: AoDeps,
|
||||
entityId: string | null,
|
||||
@@ -431,23 +479,11 @@ export async function scrapeAfterImport(
|
||||
): Promise<void> {
|
||||
if (!entityId) return;
|
||||
if (opts.enabled === false) return;
|
||||
let root: { id: string; local_path: string; management_mode: string } | undefined;
|
||||
let match: ScrapeMatch | null = null;
|
||||
try {
|
||||
const s = await scrapeSettings(d);
|
||||
if (!s.scrapeAfterImport && !opts.force && opts.enabled !== true) return;
|
||||
const organization = await d.ctx.db
|
||||
.selectFrom("organization_record")
|
||||
.select(["target_library_root_id"])
|
||||
.where("id", "=", recordId)
|
||||
.executeTakeFirst();
|
||||
const match = await entityScrapeMatch(d, entityId, "library");
|
||||
const root = organization?.target_library_root_id
|
||||
? await d.ctx.db
|
||||
.selectFrom("library_root")
|
||||
.selectAll()
|
||||
.where("id", "=", organization.target_library_root_id)
|
||||
.executeTakeFirst()
|
||||
: undefined;
|
||||
if (!match || !root || root.management_mode !== "managed") return;
|
||||
const files = await d.ctx.db
|
||||
.selectFrom("organization_file_result")
|
||||
.select("target_path")
|
||||
@@ -456,6 +492,35 @@ export async function scrapeAfterImport(
|
||||
.where("status", "=", "succeeded")
|
||||
.orderBy("ordinal")
|
||||
.execute();
|
||||
// 本次没有新入库的视频(都已在库等):没什么可刮
|
||||
if (!files.some((f) => f.target_path)) return;
|
||||
const organization = await d.ctx.db
|
||||
.selectFrom("organization_record")
|
||||
.select(["target_library_root_id"])
|
||||
.where("id", "=", recordId)
|
||||
.executeTakeFirst();
|
||||
root = organization?.target_library_root_id
|
||||
? await d.ctx.db
|
||||
.selectFrom("library_root")
|
||||
.select(["id", "local_path", "management_mode"])
|
||||
.where("id", "=", organization.target_library_root_id)
|
||||
.executeTakeFirst()
|
||||
: undefined;
|
||||
match = await entityScrapeMatch(d, entityId, "library");
|
||||
if (!root) return await recordScrapeSkipped(d, recordId, "scrape_not_library", { match });
|
||||
if (root.management_mode !== "managed")
|
||||
return await recordScrapeSkipped(d, recordId, "scrape_root_read_only", { root, match });
|
||||
if (!match) return await recordScrapeSkipped(d, recordId, "scrape_no_metadata", { root });
|
||||
if (!match.externalId) {
|
||||
// 没有 TMDB id:只有本地元数据(Local 模式补全的字段)时才刮
|
||||
const e = await d.ctx.db
|
||||
.selectFrom("media_entity")
|
||||
.select("metadata")
|
||||
.where("id", "=", entityId)
|
||||
.executeTakeFirst();
|
||||
if (!Object.keys((e?.metadata ?? {}) as object).length)
|
||||
return await recordScrapeSkipped(d, recordId, "scrape_no_metadata", { root, match });
|
||||
}
|
||||
// 照 MP 整理后刮削:按作品目录分组,对作品目录刮(作品 / 季的 NFO 与图片照常初始化),集只刮本次整理的文件;
|
||||
// 视频直接在库根(推不出作品目录)时对文件本身刮。
|
||||
const byDir = new Map<string, string[]>();
|
||||
@@ -464,20 +529,35 @@ export async function scrapeAfterImport(
|
||||
const dir = inferWorkDir(match.kind, f.target_path) ?? f.target_path;
|
||||
byDir.set(dir, [...(byDir.get(dir) ?? []), f.target_path]);
|
||||
}
|
||||
for (const [dir, videos] of byDir)
|
||||
await queueScrape(d, {
|
||||
rootId: `lib:${root.id}`,
|
||||
rootPath: root.local_path,
|
||||
libraryRootId: root.id,
|
||||
path: dir,
|
||||
match,
|
||||
trigger: "auto",
|
||||
organizationRecordId: recordId,
|
||||
idempotencyKey: `${SCRAPE_TASK}:record:${recordId}:${dir}`,
|
||||
...(dir === videos[0] ? {} : { videos }),
|
||||
});
|
||||
for (const [dir, videos] of byDir) {
|
||||
try {
|
||||
await queueScrape(d, {
|
||||
rootId: `lib:${root.id}`,
|
||||
rootPath: root.local_path,
|
||||
libraryRootId: root.id,
|
||||
path: dir,
|
||||
match,
|
||||
trigger: "auto",
|
||||
organizationRecordId: recordId,
|
||||
idempotencyKey: `${SCRAPE_TASK}:record:${recordId}:${dir}`,
|
||||
...(dir === videos[0] ? {} : { videos }),
|
||||
});
|
||||
} catch (err) {
|
||||
// 新记录入队失败时 queueScrape 已把它标成 task_enqueue_failed(整理详情可见);这里再兜一条,防止没留下记录
|
||||
d.ctx.logger.warn({ entityId, recordId, dir, err: { message: errMessage(err) } }, "入库后自动刮削没排上");
|
||||
const marked = await d.ctx.db
|
||||
.selectFrom("scrape_record")
|
||||
.select("id")
|
||||
.where("organization_record_id", "=", recordId)
|
||||
.where("path", "=", dir)
|
||||
.where("reason", "=", "task_enqueue_failed")
|
||||
.executeTakeFirst();
|
||||
if (!marked) await recordScrapeSkipped(d, recordId, "task_enqueue_failed", { root, path: dir, match });
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
d.ctx.logger.warn({ entityId, recordId, err: { message: errMessage(err) } }, "入库后自动刮削没排上");
|
||||
await recordScrapeSkipped(d, recordId, "scrape_internal_error", { root, match });
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
import { createTempTree, type TempTree } from "@umt/testkit";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { executeRecord, planImport } from "../../src/modules/acquisition-organize/services/organize.ts";
|
||||
import { SCRAPE_TASK, scrapeAfterImport } from "../../src/modules/acquisition-organize/services/scrape.ts";
|
||||
import { createHarness, type Harness, pgAvailable, seedBase, seedEntity } from "./harness.ts";
|
||||
|
||||
// 入库后自动刮削(缺省开):只对有 TMDB id 或本地元数据的作品排;没排上(库根只登记、作品没有元数据、排队失败)时
|
||||
// 在整理记录上留一条「未刮削（原因）」的刮削记录;设置关闭是预期,不留。
|
||||
|
||||
const hasPg = await pgAvailable();
|
||||
const SIZE = 200_000;
|
||||
|
||||
describe.skipIf(!hasPg)("入库后自动刮削:缺省开启,没排上时整理记录上可见", () => {
|
||||
let h: Harness;
|
||||
let lib: TempTree;
|
||||
let src: TempTree;
|
||||
let rootId = "";
|
||||
|
||||
beforeAll(async () => {
|
||||
lib = await createTempTree({}, { sentinel: true, prefix: "umt-sailib-" });
|
||||
src = await createTempTree(
|
||||
{
|
||||
"Tenet.2020.1080p.BluRay.x264-GRP.mkv": SIZE,
|
||||
"Dune.2021.1080p.BluRay.x264-GRP.mkv": SIZE,
|
||||
"Arrival.2016.1080p.BluRay.x264-GRP.mkv": SIZE,
|
||||
"Heat.1995.1080p.BluRay.x264-GRP.mkv": SIZE,
|
||||
},
|
||||
{ prefix: "umt-saisrc-" },
|
||||
);
|
||||
h = await createHarness("scrape_after_import");
|
||||
rootId = (await seedBase(h.db, { libraryPath: lib.root, cachePath: src.root, qbUrl: "http://127.0.0.1:9" })).rootId;
|
||||
}, 60_000);
|
||||
afterAll(async () => {
|
||||
await h?.close();
|
||||
for (const t of [lib, src]) await t?.cleanup();
|
||||
});
|
||||
|
||||
async function movie(title: string, opts: { tmdb?: string; metadata?: object } = {}) {
|
||||
const id = await seedEntity(h.db, { kind: "movie", title, year: 2020 });
|
||||
if (opts.metadata)
|
||||
await h.db
|
||||
.updateTable("media_entity")
|
||||
.set({ metadata: JSON.stringify(opts.metadata) })
|
||||
.where("id", "=", id)
|
||||
.execute();
|
||||
if (opts.tmdb)
|
||||
await h.db
|
||||
.insertInto("external_identity")
|
||||
.values({
|
||||
media_entity_id: id,
|
||||
provider: "tmdb",
|
||||
entity_type: "movie",
|
||||
external_id: opts.tmdb,
|
||||
status: "accepted",
|
||||
})
|
||||
.execute();
|
||||
return id;
|
||||
}
|
||||
async function organize(entity: string, file: string) {
|
||||
const plan = await planImport(h.d, {
|
||||
sourcePaths: [src.path(file)],
|
||||
mediaEntityId: entity,
|
||||
action: "copy",
|
||||
trigger: "manual",
|
||||
actorType: "user",
|
||||
});
|
||||
const ex = await executeRecord(h.d, plan.recordId!);
|
||||
expect(ex.status).toBe("succeeded");
|
||||
return plan.recordId!;
|
||||
}
|
||||
const recordsOf = (orgId: string) =>
|
||||
h.db
|
||||
.selectFrom("scrape_record")
|
||||
.select(["status", "reason", "trigger"])
|
||||
.where("organization_record_id", "=", orgId)
|
||||
.execute();
|
||||
|
||||
it("缺省设置:有 TMDB id 的作品入库后排上刮削", async () => {
|
||||
const id = await movie("信条", { tmdb: "577922" });
|
||||
const org = await organize(id, "Tenet.2020.1080p.BluRay.x264-GRP.mkv");
|
||||
expect(await recordsOf(org)).toEqual([{ status: "queued", reason: null, trigger: "auto" }]);
|
||||
expect(h.engine.enqueued.some((x) => x.task === SCRAPE_TASK)).toBe(true);
|
||||
});
|
||||
|
||||
it("没有 TMDB id 也没有本地元数据:不排任务,记「未刮削」scrape_no_metadata;重放不重复记", async () => {
|
||||
const before = h.engine.enqueued.filter((x) => x.task === SCRAPE_TASK).length;
|
||||
const id = await movie("沙丘");
|
||||
const org = await organize(id, "Dune.2021.1080p.BluRay.x264-GRP.mkv");
|
||||
expect(await recordsOf(org)).toEqual([{ status: "skipped", reason: "scrape_no_metadata", trigger: "auto" }]);
|
||||
expect(h.engine.enqueued.filter((x) => x.task === SCRAPE_TASK).length).toBe(before);
|
||||
await executeRecord(h.d, org);
|
||||
expect(await recordsOf(org)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("库根是只登记模式:记 scrape_root_read_only;排队失败:记 task_enqueue_failed", async () => {
|
||||
const id = await movie("降临", { tmdb: "329865" });
|
||||
const org = await organize(id, "Arrival.2016.1080p.BluRay.x264-GRP.mkv");
|
||||
await h.db.deleteFrom("scrape_record").where("organization_record_id", "=", org).execute();
|
||||
await h.db.updateTable("library_root").set({ management_mode: "register_only" }).where("id", "=", rootId).execute();
|
||||
try {
|
||||
await scrapeAfterImport(h.d, id, org);
|
||||
expect(await recordsOf(org)).toEqual([{ status: "skipped", reason: "scrape_root_read_only", trigger: "auto" }]);
|
||||
} finally {
|
||||
await h.db.updateTable("library_root").set({ management_mode: "managed" }).where("id", "=", rootId).execute();
|
||||
}
|
||||
await h.db.deleteFrom("scrape_record").where("organization_record_id", "=", org).execute();
|
||||
const enqueue = h.d.ctx.engine.enqueue;
|
||||
h.d.ctx.engine.enqueue = (async () => {
|
||||
throw new Error("引擎不可用");
|
||||
}) as never;
|
||||
try {
|
||||
await scrapeAfterImport(h.d, id, org);
|
||||
} finally {
|
||||
h.d.ctx.engine.enqueue = enqueue;
|
||||
}
|
||||
expect(await recordsOf(org)).toEqual([{ status: "failed", reason: "task_enqueue_failed", trigger: "auto" }]);
|
||||
});
|
||||
|
||||
it("设置关闭:不排也不记", async () => {
|
||||
await h.setSettings("organize", { scrapeAfterImport: false });
|
||||
const id = await movie("盗火线", { tmdb: "949" });
|
||||
const org = await organize(id, "Heat.1995.1080p.BluRay.x264-GRP.mkv");
|
||||
expect(await recordsOf(org)).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -205,9 +205,14 @@ describe.skipIf(skip)("file manager in-place import", () => {
|
||||
try {
|
||||
const r = await queue(s.items, { scrapeAfterImport: true });
|
||||
expect((await executeRecord(h.module.deps().ao, r.recordId)).status).toBe("succeeded");
|
||||
// 不刮削,只在整理记录上留一条「未刮削（只登记模式）」
|
||||
expect(
|
||||
await h.db.selectFrom("scrape_record").select("id").where("organization_record_id", "=", r.recordId).execute(),
|
||||
).toEqual([]);
|
||||
await h.db
|
||||
.selectFrom("scrape_record")
|
||||
.select(["status", "reason"])
|
||||
.where("organization_record_id", "=", r.recordId)
|
||||
.execute(),
|
||||
).toEqual([{ status: "skipped", reason: "scrape_root_read_only" }]);
|
||||
expect(await readdir(join(h.lib, s.path.split("/")[0]!))).toEqual(["original.name.mkv"]);
|
||||
} finally {
|
||||
await h.db.updateTable("library_root").set({ management_mode: "managed" }).where("id", "=", rootId).execute();
|
||||
|
||||
@@ -21,6 +21,7 @@ export const SCRAPE_STATUS: Record<ScrapeRecord["status"], { label: string; tone
|
||||
succeeded: { label: "已完成", tone: "success" },
|
||||
partial: { label: "部分完成", tone: "warning" },
|
||||
failed: { label: "失败", tone: "destructive" },
|
||||
skipped: { label: "未刮削", tone: "neutral" },
|
||||
};
|
||||
|
||||
const ACTION: Record<ScrapeFileResult["action"], { label: string; tone: StatusTone }> = {
|
||||
@@ -82,6 +83,8 @@ function FileRow({ f }: { f: ScrapeFileResult }) {
|
||||
|
||||
/** 不必逐条列出的跳过:已存在(不是本工具写的 / 只补缺失)、内容没变 */
|
||||
const QUIET_SKIPS = new Set(["scrape_exists", "scrape_unchanged"]);
|
||||
/** 预期内的跳过(已存在、没变、被人改过);其余跳过(缺元数据、认不出集号)让记录算部分完成 */
|
||||
export const EXPECTED_SCRAPE_SKIPS = new Set([...QUIET_SKIPS, "scrape_user_modified"]);
|
||||
|
||||
/** 一条刮削记录:状态、目录与识别结果、计数、逐文件结果(缺省不列"已存在"跳过的) */
|
||||
export function ScrapeRecordView({ record: s, showPath = false }: { record: ScrapeRecord; showPath?: boolean }) {
|
||||
|
||||
@@ -1,14 +1,18 @@
|
||||
import { useQuery } from "@tanstack/react-query";
|
||||
import { FilesScrapeRecordsResult } from "@umt/contracts";
|
||||
import { FilesScrapeRecordsResult, type ScrapeFileResult } from "@umt/contracts";
|
||||
import { Link } from "react-router";
|
||||
import { apiParsed } from "~/api/parse.ts";
|
||||
import { ErrorState, LoadingState } from "~/components/console/empty-state.tsx";
|
||||
import { PaginatedList } from "~/components/console/paginated-list.tsx";
|
||||
import { StatusBadge } from "~/components/console/status-badge.tsx";
|
||||
import { SCRAPE_STATUS, scrapeCounts } from "~/components/library/scrape-panel.tsx";
|
||||
import { EXPECTED_SCRAPE_SKIPS, SCRAPE_STATUS, scrapeCounts } from "~/components/library/scrape-panel.tsx";
|
||||
import { reasonLine } from "~/lib/reasons.ts";
|
||||
import { formatDateTime } from "~/lib/utils.ts";
|
||||
|
||||
/** 失败的文件,以及缺元数据、认不出集号等不在预期内的跳过 */
|
||||
const unfinished = (files: ScrapeFileResult[]) =>
|
||||
files.filter((f) => f.action === "failed" || (f.action === "skipped" && !EXPECTED_SCRAPE_SKIPS.has(f.reason ?? "")));
|
||||
|
||||
/** 整理与刮削分别展示结果，避免将刮削失败当作文件搬运失败。 */
|
||||
export function OrganizeScrapeHistory({ recordId }: { recordId: string }) {
|
||||
const query = useQuery({
|
||||
@@ -39,21 +43,36 @@ export function OrganizeScrapeHistory({ recordId }: { recordId: string }) {
|
||||
<p className="break-all font-mono text-meta text-muted-foreground">
|
||||
{record.match?.title ?? record.path} · {record.path || "/"}
|
||||
</p>
|
||||
<p className="text-meta text-muted-foreground">{scrapeCounts(record) || "等待写入 NFO 与图片"}</p>
|
||||
{record.reason ? <p className="text-meta text-destructive">{reasonLine(record.reason)}</p> : null}
|
||||
{record.files.some((file) => file.action === "failed") ? (
|
||||
{record.status === "skipped" ? null : (
|
||||
<p className="text-meta text-muted-foreground">{scrapeCounts(record) || "等待写入 NFO 与图片"}</p>
|
||||
)}
|
||||
{record.reason ? (
|
||||
<p
|
||||
className={
|
||||
record.status === "skipped" ? "text-meta text-muted-foreground" : "text-meta text-destructive"
|
||||
}
|
||||
>
|
||||
{reasonLine(record.reason)}
|
||||
</p>
|
||||
) : null}
|
||||
{unfinished(record.files).length ? (
|
||||
<details className="text-meta">
|
||||
<summary className="cursor-pointer text-muted-foreground">失败文件</summary>
|
||||
<summary className="cursor-pointer text-muted-foreground">失败或没写的文件</summary>
|
||||
<PaginatedList
|
||||
rows={record.files.filter((file) => file.action === "failed")}
|
||||
label="刮削失败文件"
|
||||
rows={unfinished(record.files)}
|
||||
label="刮削失败或跳过的文件"
|
||||
resetKey={record.id}
|
||||
as="ul"
|
||||
className="mt-2 space-y-1"
|
||||
>
|
||||
{(file) => (
|
||||
<li key={`${file.path}:${file.role}`} className="break-all text-destructive">
|
||||
{file.path} · {reasonLine(file.reason) || "写入失败"}
|
||||
<li
|
||||
key={`${file.path}:${file.role}`}
|
||||
className={
|
||||
file.action === "failed" ? "break-all text-destructive" : "break-all text-muted-foreground"
|
||||
}
|
||||
>
|
||||
{file.path} · {reasonLine(file.reason) || (file.action === "failed" ? "写入失败" : "跳过")}
|
||||
</li>
|
||||
)}
|
||||
</PaginatedList>
|
||||
|
||||
Reference in new issue
Block a user