merge: fix/i-scrape

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei
This commit is contained in:
rootandClaude Opus 5.5 committed 2026-10-03 23:50:29 +08:00
commit ee503dd1ea
35 files changed
+1915 -200

No files matched your search

@@ -21,7 +21,12 @@ import { type AoDeps, upsertDecision } from "./common.ts";
import { finalizeRecord, removeMovedDownload } from "./organize-finalize.ts";
import { emitRecord } from "./organize-plan.ts";
import type { FrozenPlan } from "./organize-types.ts";
import { cleanupReplacement, settleSamePathReplacement, stagedReplacementBackup } from "./replacement.ts";
import {
cleanupReplacement,
quarantineNotVideo,
settleSamePathReplacement,
stagedReplacementBackup,
} from "./replacement.ts";
import { scrapeAfterImport } from "./scrape.ts";
// ── 执行 ──
@@ -81,11 +86,10 @@ export async function executeRecord(
if (rec.status !== "running" && rec.status !== "queued") {
await settleSamePathReplacement(d, rec.plan as unknown as FrozenPlan, recordId);
await cleanupReplacement(d, rec.plan as unknown as FrozenPlan, recordId);
await quarantineNotVideo(d, rec.plan as unknown as FrozenPlan, recordId);
const frozen = rec.plan as unknown as FrozenPlan;
if (
(frozen.metadataMode === "local" || frozen.scrapeAfterImport !== undefined) &&
["succeeded", "partial"].includes(rec.status)
) {
// 与 finalizeRecord 同一口径(按设置 / 本次选项);排队按整理记录幂等,重放不会重复刮
if (["succeeded", "partial"].includes(rec.status)) {
await scrapeAfterImport(d, frozen.entityId, recordId, {
force: frozen.metadataMode === "local",
enabled: frozen.scrapeAfterImport,
@@ -767,14 +771,8 @@ export async function transferFile(
},
};
}
const r = await px.crossDeviceMove(
src,
dst,
{
hash: d.ports.options.verifyHash,
},
call,
);
// 跨卷移动删源前一律全量 sha256 校验(抽样校验不够当删源依据;只影响跨卷 Move,复制 / 同卷仍按 verifyHash)
const r = await px.crossDeviceMove(src, dst, { hash: "sha256" }, call);
const failedStep = r.steps.find((st) => st.status === "failed");
return {
status: r.status === "completed" ? "succeeded" : r.status,
@@ -15,11 +15,19 @@ import type { ExecuteResult } from "./organize-exec.ts";
import { emitRecord } from "./organize-plan.ts";
import { finalizeTransfer } from "./organize-transfer.ts";
import type { FrozenPlan } from "./organize-types.ts";
import { cleanupReplacement, settleSamePathReplacement } from "./replacement.ts";
import { cleanupReplacement, quarantineNotVideo, settleSamePathReplacement } from "./replacement.ts";
import { scrapeAfterImport } from "./scrape.ts";
import { slotComplete as sharedSlotComplete } from "./subscriptions.ts";
import { downloadTasksUnder } from "./takeover.ts";
/** 被排除待确认的视频原因(待处理摘要里显示) */
const EXCLUDED_TEXT: Record<string, string> = {
episode_unresolved: "集数待确认",
no_target_dir: "没有季目录",
naming_failed: "命名失败",
title_mismatch: "标题不是本作品",
};
/** 登记资产/位置/绑定 + Slot/计划/下载任务 + 记录终态,同一事务;记录行锁作 fencing */
export async function finalizeRecord(d: AoDeps, recordId: string): Promise<ExecuteResult> {
const db = d.ctx.db;
@@ -82,6 +90,7 @@ export async function finalizeRecord(d: AoDeps, recordId: string): Promise<Execu
.execute();
// 探测结果明确"不是视频" → 该文件核验失败
const notVideo = new Set<string>();
for (const f of files) {
const p = probes.get(f.id) as { status?: string; video?: unknown[] } | undefined;
if (p?.status === "ok" && Array.isArray(p.video) && p.video.length === 0) {
@@ -97,6 +106,21 @@ export async function finalizeRecord(d: AoDeps, recordId: string): Promise<Execu
.where("id", "=", f.id)
.execute();
f.status = "failed";
notVideo.add(f.source_path);
}
}
// 主文件不是视频:附属文件不登记(结果改为跳过 main_failed),新文件随后移入回收站(quarantineNotVideo)
if (notVideo.size) {
const mainOf = new Map(frozen.organize.files.map((pf) => [pf.src, pf.mainSrc]));
for (const f of files) {
if (f.status !== "succeeded" || f.role === "main" || !notVideo.has(mainOf.get(f.source_path) ?? "")) continue;
await trx
.updateTable("organization_file_result")
.set({ status: "skipped", skip_reason: "main_failed" })
.where("id", "=", f.id)
.execute();
f.status = "skipped";
f.skip_reason = "main_failed";
}
}
@@ -145,16 +169,22 @@ export async function finalizeRecord(d: AoDeps, recordId: string): Promise<Execu
const parent = pf?.mainSrc ? assetBySrc.get(pf.mainSrc) : undefined;
if (!isMain && !parent) continue;
let hit = await trx
.selectFrom("asset_location")
.select(["id", "media_asset_id"])
.where("library_root_id", "=", frozen.libraryRootId)
.where("relative_path", "=", f.target_path)
.where("status", "!=", "removed")
.selectFrom("asset_location as l")
.innerJoin("media_asset as a", "a.id", "l.media_asset_id")
.select(["l.id", "l.media_asset_id", "l.status", "a.media_entity_id"])
.where("l.library_root_id", "=", frozen.libraryRootId)
.where("l.relative_path", "=", f.target_path)
.where("l.status", "!=", "removed")
.executeTakeFirst();
// 同一路径已有位置时,这些情况旧资产不能沿用,退役旧位置、新建资产(size / hash / 版本信息都是新文件的):
// - 替换(向导 / 待处理选「替换」= replace_with_verify;洗版只退役选中的旧版本);
// - 旧位置属于别的作品。
if (
hit &&
pf?.action === "replace" &&
frozen.replacement?.assets.some((a) => a.files.some((old) => old.locationId === hit!.id))
((pf?.action === "replace" &&
(!frozen.replacement ||
frozen.replacement.assets.some((a) => a.files.some((old) => old.locationId === hit!.id)))) ||
hit.media_entity_id !== frozen.entityId)
) {
await trx.updateTable("asset_location").set({ status: "removed" }).where("id", "=", hit.id).execute();
await sql`update media_asset set status='retired' where id=${hit.media_asset_id} and not exists(select 1 from asset_location where media_asset_id=${hit.media_asset_id} and status <> 'removed')`.execute(
@@ -166,46 +196,80 @@ export async function finalizeRecord(d: AoDeps, recordId: string): Promise<Execu
let locationId: string;
const st = await statSafe(join(frozen.rootPath, f.target_path));
const factsOf = st.kind === "ok" ? st.facts : null;
const fx = frozen.facts[f.source_path];
const probe = probes.get(f.id);
const facts = {
size_bytes: factsOf?.size ?? f.source_size_bytes,
sample_digest: f.checksum?.startsWith("sample:") ? f.checksum.slice(7) : null,
content_hash: f.checksum?.startsWith("sha256:") ? f.checksum.slice(7) : null,
hash_algorithm: f.checksum?.startsWith("sha256:") ? "sha256" : null,
media_info: jsonb({
...(probe ? { probe } : {}),
_umt: {
quality: fx?.quality ?? {},
unitKeys: fx?.unitKeys ?? [],
},
}),
// LIBV-1:库内文件名 + 探测 + 资源解析合成版本规格
delivery: jsonb(
isMain
? deliveryForAsset(f.target_path, {
...(probe ? { probe } : {}),
_umt: { quality: fx?.quality ?? {} },
})
: fx
? {
...(fx.quality.resolution ? { resolution: fx.quality.resolution } : {}),
...(fx.quality.source ? { sourceType: fx.quality.source } : {}),
...(fx.quality.releaseGroup ? { releaseGroup: fx.quality.releaseGroup.name } : {}),
...(fx.quality.hdr?.length ? { hdr: fx.quality.hdr } : {}),
}
: {},
),
content_edition: fx?.quality.cut ?? null,
};
const locFacts = {
size_bytes: factsOf?.size ?? null,
inode: factsOf?.ino ?? null,
device: factsOf?.dev ?? null,
mtime: factsOf ? new Date(factsOf.mtimeMs) : null,
last_seen_at: now,
last_checked_at: now,
};
if (hit) {
assetId = hit.media_asset_id;
locationId = hit.id;
// 复用的位置之前被判缺失 / 不可用(同名文件重新入库):位置恢复可用,资产事实换成这次的文件
if (hit.status !== "available") {
await trx
.updateTable("asset_location")
.set({ status: "available", ...locFacts })
.where("id", "=", hit.id)
.execute();
await trx
.updateTable("media_asset")
.set({
...facts,
verified_at: now,
...(dt ? { source_release_id: dt.source_release_id, download_task_id: dt.id } : {}),
})
.where("id", "=", assetId)
.execute();
await trx
.updateTable("media_asset")
.set({ status: "active" })
.where("id", "=", assetId)
.where("status", "=", "missing")
.execute();
}
} else {
const fx = frozen.facts[f.source_path];
const probe = probes.get(f.id);
const asset = await trx
.insertInto("media_asset")
.values({
media_entity_id: frozen.entityId,
kind: isMain ? "video" : f.role === "audio" ? "audio" : f.role === "subtitle" ? "subtitle" : "other",
parent_asset_id: parent?.assetId ?? null,
size_bytes: factsOf?.size ?? f.source_size_bytes,
sample_digest: f.checksum?.startsWith("sample:") ? f.checksum.slice(7) : null,
content_hash: f.checksum?.startsWith("sha256:") ? f.checksum.slice(7) : null,
hash_algorithm: f.checksum?.startsWith("sha256:") ? "sha256" : null,
media_info: jsonb({
...(probe ? { probe } : {}),
_umt: {
quality: fx?.quality ?? {},
unitKeys: fx?.unitKeys ?? [],
},
}),
// LIBV-1:库内文件名 + 探测 + 资源解析合成版本规格
delivery: jsonb(
isMain
? deliveryForAsset(f.target_path, {
...(probe ? { probe } : {}),
_umt: { quality: fx?.quality ?? {} },
})
: fx
? {
...(fx.quality.resolution ? { resolution: fx.quality.resolution } : {}),
...(fx.quality.source ? { sourceType: fx.quality.source } : {}),
...(fx.quality.releaseGroup ? { releaseGroup: fx.quality.releaseGroup.name } : {}),
...(fx.quality.hdr?.length ? { hdr: fx.quality.hdr } : {}),
}
: {},
),
content_edition: fx?.quality.cut ?? null,
...facts,
source_release_id: dt?.source_release_id ?? null,
download_task_id: dt?.id ?? null,
status: "active",
@@ -223,13 +287,8 @@ export async function finalizeRecord(d: AoDeps, recordId: string): Promise<Execu
relative_path: f.target_path,
status: "available",
is_primary: true,
size_bytes: factsOf?.size ?? null,
inode: factsOf?.ino ?? null,
device: factsOf?.dev ?? null,
mtime: factsOf ? new Date(factsOf.mtimeMs) : null,
...locFacts,
sidecar_of_location_id: parent?.locationId ?? null,
last_seen_at: now,
last_checked_at: now,
})
.returning("id")
.executeTakeFirstOrThrow();
@@ -299,13 +358,14 @@ export async function finalizeRecord(d: AoDeps, recordId: string): Promise<Execu
.where("id", "=", recordId)
.execute();
if (unresolved.length > 0) {
const shown = unresolved
.slice(0, 5)
.map((f) => `${f.source_path.split("/").pop()}(${f.skip_reason?.replace(/^excluded:/, "")})`);
const shown = unresolved.slice(0, 5).map((f) => {
const code = f.skip_reason?.replace(/^excluded:/, "") ?? "";
return `${f.source_path.split("/").pop()}(${EXCLUDED_TEXT[code] ?? code})`;
});
await upsertDecision(d, trx, {
kind: "other",
title: `有 ${unresolved.length} 个视频没有整理`,
summary: `季集、季目录或命名没能确定,文件留在原处${
summary: `季集、季目录、命名没能确定,或文件标题不是本作品,文件留在原处${
rec.action === "move" ? ",下载任务保留" : ""
}:${shown.join("、")}${unresolved.length > shown.length ? " 等" : ""}`,
subjectType: "organization_record",
@@ -463,6 +523,7 @@ export async function finalizeRecord(d: AoDeps, recordId: string): Promise<Execu
await settleSamePathReplacement(d, frozenPre, recordId);
await cleanupReplacement(d, frozenPre, recordId);
await quarantineNotVideo(d, frozenPre, recordId);
// SCR-1:有资产入库时按设置自动刮削(写 NFO 与图片;排任务,不在这里等)
if (result.status === "succeeded" || result.status === "partial") {
@@ -24,6 +24,7 @@ import {
resolveTargetDirectory,
type SourceFile,
sidecarStem,
titleMismatches,
} from "@umt/domain";
import { organizePlanToFileResults, toDomainPathBinding } from "@umt/mapping";
import {
@@ -382,7 +383,8 @@ export async function planImport(d: AoDeps, req: ImportRequest): Promise<PlanRes
actorId: req.actorId ?? "system:organize",
taskId: req.taskId ?? `organize:${dt?.id ?? rel.id}`,
scopeIds: [],
signal: new AbortController().signal,
// 带超时:上游(模型 / 网页搜索)挂住时不能一直占着 file 道
signal: AbortSignal.timeout(RESOLVE_ORGANIZE_TIMEOUT_MS),
source: "task",
},
)) as { mediaEntityId?: string | null; mode?: "tmdb" | "local" | null };
@@ -637,9 +639,22 @@ export async function planImport(d: AoDeps, req: ImportRequest): Promise<PlanRes
const episodes = mediaType === "tv" ? await planFileEpisodes(d, entity, rel, dt) : null;
const mappedOverrides: Record<string, DomainFileOverride> = {};
const unresolvedEpisodes: NonNullable<FrozenPlan["unresolvedEpisodes"]> = {};
// 文件级标题粗核:混合包 / 合集里标题明显不是本作品的视频不按本作品命名入库(待确认 title_mismatch);
// 用户逐文件指定过的不核
const mismatched = await fileTitleMismatches(
d,
entity,
mediaType,
files.filter((f) => !overrideFor(req.fileOverrides, f.path) && fileKindOf(basename(f.path)) === "video"),
);
for (const f of files) {
let parsed = parseReleasePath(segmentsWithin(f.path, sourceRoot));
if (parsed.fileKind !== "video") continue;
const wrongTitle = mismatched.get(f.path);
if (wrongTitle !== undefined) {
mappedOverrides[f.path] = { exclude: { reason: "title_mismatch", detail: wrongTitle } };
continue;
}
const ov = overrideFor(req.fileOverrides, f.path);
let resolution: EpisodeResolution | undefined;
if (mediaType === "tv" && (ov?.season !== undefined || ov?.episodes)) {
@@ -841,9 +856,12 @@ export async function planImport(d: AoDeps, req: ImportRequest): Promise<PlanRes
const e: ExistingTarget = { path: f.dst, size: st.facts.size };
if (st.facts.size === f.size) {
// 同尺寸已存在的目标:全量 sha256 判定是否同一文件(子进程,见 adapters sha256Full)
e.sha256 = (await sha256Full(probe)).digest;
e.sha256 = await cachedSha256(probe, st.facts);
const src = files.find((x) => x.path === f.src);
if (src) src.sha256 = (await sha256Full(src.path)).digest;
if (src) {
const ss = await statSafe(src.path);
src.sha256 = ss.kind === "ok" ? await cachedSha256(src.path, ss.facts) : (await sha256Full(src.path)).digest;
}
}
existing.push(e);
}
@@ -1221,3 +1239,58 @@ export async function emitRecord(d: AoDeps, exec: Exec, recordId: string): Promi
mediaEntityId: r.media_entity_id,
});
}
/**
* 计划期全量 sha256 按 (路径, 大小, 修改时间, inode) 缓存:向导预览 / 复核 / 执行各算一遍同一对大文件时只算一次。
* 文件被改动(大小 / 修改时间 / inode 变)就重算;只留最近的 SHA_CACHE_MAX 条。
*/
const SHA_CACHE_MAX = 2000;
const shaCache = new Map<string, string>();
export async function cachedSha256(
path: string,
facts: { size: number; mtimeMs: number; ino?: number | bigint | null },
): Promise<string> {
const key = `${path}\u0000${facts.size}\u0000${facts.mtimeMs}\u0000${facts.ino ?? ""}`;
const hit = shaCache.get(key);
if (hit) {
shaCache.delete(key);
shaCache.set(key, hit);
return hit;
}
const digest = (await sha256Full(path)).digest;
shaCache.set(key, digest);
if (shaCache.size > SHA_CACHE_MAX) shaCache.delete(shaCache.keys().next().value!);
return digest;
}
/** 整理时补全元数据(metadata.resolve_organize:模型 + 网页搜索)的超时 */
export const RESOLVE_ORGANIZE_TIMEOUT_MS = 3 * 60_000;
/** 作品标题 / 原名 / 别名 → 这批视频里标题明显不是本作品的(路径 → 文件标题) */
async function fileTitleMismatches(
d: AoDeps,
entity: Pick<MediaEntityRow, "id" | "title" | "original_title" | "metadata">,
mediaType: "movie" | "tv",
videos: readonly { path: string }[],
): Promise<Map<string, string>> {
if (videos.length < 2) return new Map();
const aliases = await d.ctx.db
.selectFrom("title_alias")
.select("alias")
.where("media_entity_id", "=", entity.id)
.where("content_unit_id", "is", null)
.execute();
const meta = (entity.metadata ?? {}) as Record<string, { value?: unknown } | undefined>;
const names = [
entity.title,
entity.original_title,
meta.title?.value,
meta.originalTitle?.value,
...aliases.map((a) => a.alias),
].filter((n): n is string => typeof n === "string" && n.trim().length > 0);
return titleMismatches(
videos.map((v) => ({ path: v.path, name: basename(v.path) })),
names,
mediaType,
);
}
@@ -500,3 +500,107 @@ export async function stagedReplacementBackup(d: AoDeps, file: ReplacementFile):
}
return null;
}
/**
* 收尾判定「不是视频」(ffprobe 没有视频流)的文件:已拷 / 搬进库的新文件和它的附属文件移入回收站(Move 时源已删,
* 回收站里是唯一一份),并开待处理。原地入库不动用户原有的文件;洗版替换由 settleSamePathReplacement 回滚。
* 可重入:目标已不在、或已登记成资产、或大小对不上(不是本次写入的)就不动。
*/
export async function quarantineNotVideo(d: AoDeps, frozen: FrozenPlan, recordId: string): Promise<void> {
const db = d.ctx.db;
const rec = await db
.selectFrom("organization_record")
.select(["action", "download_task_id"])
.where("id", "=", recordId)
.executeTakeFirst();
if (!rec || rec.action === "in_place" || frozen.transfer) return;
const results = await db
.selectFrom("organization_file_result")
.select(["id", "source_path", "target_path", "status", "skip_reason", "error", "source_size_bytes", "role"])
.where("organization_record_id", "=", recordId)
.orderBy("ordinal")
.execute();
const planned = new Map(frozen.organize.files.map((f) => [f.src, f]));
const rejectedMains = new Set(
results
.filter((f) => f.status === "failed" && (f.error as { code?: string } | null)?.code === "NOT_VIDEO")
.map((f) => f.source_path),
);
if (!rejectedMains.size) return;
const targets = results.filter(
(f) =>
f.target_path &&
(rejectedMains.has(f.source_path) ||
(f.status === "skipped" &&
f.skip_reason === "main_failed" &&
rejectedMains.has(planned.get(f.source_path)?.mainSrc ?? ""))),
);
const fsr = fsRuntime(d);
const moved: { path: string; trash: string }[] = [];
const left: { path: string; error: string }[] = [];
for (const f of targets) {
const pf = planned.get(f.source_path);
if (frozen.replacement && pf?.action === "replace") continue;
const rel = f.target_path!;
try {
if ((await fsr.proxy.checkRoot(frozen.rootPath, frozen.sentinel)).kind !== "online") throw new Error("库根离线");
const rootReal = await realpath(frozen.rootPath);
const abs = join(rootReal, rel);
const st = await fsr.proxy.statSafe(abs, { root: rootReal, sentinel: frozen.sentinel });
if (st.kind === "missing") continue;
if (st.kind !== "ok" || !st.facts.isFile || st.facts.isSymbolicLink) throw new Error("目标不是普通文件");
const registered = await db
.selectFrom("asset_location")
.select("id")
.where("library_root_id", "=", frozen.libraryRootId)
.where("relative_path", "=", rel)
.where("status", "<>", "removed")
.executeTakeFirst();
if (registered) continue;
const size = Number(f.source_size_bytes ?? pf?.size ?? -1);
if (size >= 0 && st.facts.size !== size) throw new Error("目标大小与本次写入的不同,保留待核对");
const id = randomUUID();
const now = d.ports.now();
const trash = join(rootReal, ".umt-trash", now.toISOString().slice(0, 10).replace(/-/g, ""), id, rel);
await mkdir(dirname(trash), { recursive: true });
const m = await fsr.proxy.moveSafe(abs, trash);
if (m.status !== "moved") throw new Error(`移入回收站失败:${m.status}`);
const days = Number(Deno.env.get("FILES_TRASH_RETENTION_DAYS") ?? 7) || 7;
await db
.insertInto("trash_item")
.values({
id,
root_id: `lib:${frozen.libraryRootId}`,
library_root_id: frozen.libraryRootId,
root_host_path: rootReal,
original_path: rel,
trash_path: trash,
is_dir: false,
size_bytes: st.facts.size,
actor: "system:organize",
deleted_at: now,
expires_at: new Date(now.getTime() + days * 86_400_000),
asset_snapshot: jsonb({ source: "organize_not_video", recordId, sourcePath: f.source_path }),
})
.execute();
moved.push({ path: rel, trash });
} catch (error) {
left.push({ path: rel, error: error instanceof Error ? error.message : String(error) });
}
}
if (!moved.length && !left.length) return;
const names = [...rejectedMains].map((p) => basename(p));
await upsertDecision(d, db, {
kind: "other",
title: `${rejectedMains.size} 个文件不是视频,没有入库`,
summary: `${names.slice(0, 5).join("、")}${names.length > 5 ? " 等" : ""} 没有视频流(伪装或损坏的文件)。${
moved.length ? "已拷进媒体库的文件移入回收站" : ""
}${left.length ? `${moved.length ? "," : ""}${left.length} 个文件留在媒体库里待核对` : ""}${
rec.action === "move" ? ";源文件已随移动删除,回收站里是唯一一份" : ""
}`,
subjectType: "organization_record",
subjectId: recordId,
dedupeKey: `organize_not_video:${recordId}`,
evidence: { downloadTaskId: rec.download_task_id, moved, left },
});
}
@@ -11,10 +11,10 @@ import {
type ScrapeRecord,
} from "@umt/contracts";
import {
episodeNfo,
fileKindOf,
inferWorkDir,
isSeasonDirName,
multiEpisodeNfo,
parseReleasePath,
planScrapeTargets,
type ScrapeTarget,
@@ -81,6 +81,8 @@ export interface ScrapeLocation {
rootPath: string;
libraryRootId: string | null;
path: string;
/** 只刮这些视频(作品重刮:电影目录里属于本作品的视频);缺省 = 目录下全部 */
videos?: readonly string[] | undefined;
}
export interface QueueScrapeInput extends ScrapeLocation {
@@ -341,6 +343,8 @@ export async function entityScrapeLocations(
.where("status", "=", "active")
.execute();
const out = new Map<string, ScrapeLocation>();
/** 电影:每个目录里属于本作品的视频(重刮只刮这些,同目录别的电影不写本作品的 NFO);有原盘目录的不限定 */
const own = new Map<string, string[] | null>();
for (const l of locs) {
const bound = bindings.find(
(b) =>
@@ -358,9 +362,26 @@ export async function entityScrapeLocations(
}
// 父目录已在列表里的子路径去掉,避免同一处刮两次
const all = [...out.values()];
return all.filter(
const kept = all.filter(
(x) => !all.some((y) => y !== x && y.rootId === x.rootId && !!y.path && x.path.startsWith(`${y.path}/`)),
);
if (kind !== "movie") return kept;
for (const l of locs) {
const loc = kept.find(
(x) =>
x.libraryRootId === l.root_id &&
(!x.path || l.relative_path === x.path || l.relative_path.startsWith(`${x.path}/`)),
);
if (!loc) continue;
const key = `${loc.rootId}\u0000${loc.path}`;
if (l.kind !== "video") own.set(key, null);
else if (own.get(key) !== null) own.set(key, [...(own.get(key) ?? []), l.relative_path]);
}
return kept.map((x) => {
const videos = own.get(`${x.rootId}\u0000${x.path}`);
// 选中的就是视频文件本身时不用限定
return videos?.length && !(videos.length === 1 && videos[0] === x.path) ? { ...x, videos } : x;
});
}
/** 对作品的目录刮削(每个目录一条记录);没有已入库文件时返回空 */
@@ -422,7 +443,55 @@ export async function scheduleEntityRescrape(d: QueueDeps, entityId: string): Pr
return out.length;
}
/** 整理入库成功后(设置打开或本次要求刮削):照 MP 对作品目录刮削,集只刮本次整理的文件。 */
/**
* 入库后自动刮削没排上:写一条 skipped 刮削记录挂在整理记录上(整理详情「整理后刮削」显示「未刮削(原因)」)。
* 同一整理记录同一原因只记一次(崩溃重放会再调)。失败只告警。
*/
async function recordScrapeSkipped(
d: AoDeps,
recordId: string,
reason: string,
at: { root?: { id: string; local_path: string } | undefined; path?: string; match?: ScrapeMatch | null },
): Promise<void> {
try {
const dup = await d.ctx.db
.selectFrom("scrape_record")
.select("id")
.where("organization_record_id", "=", recordId)
.where("status", "=", "skipped")
.where("reason", "=", reason)
.executeTakeFirst();
if (dup) return;
const now = d.ports.now();
await d.ctx.db
.insertInto("scrape_record")
.values({
root_id: at.root ? `lib:${at.root.id}` : "",
root_path: at.root?.local_path ?? "",
library_root_id: at.root?.id ?? null,
path: at.path ?? "",
media_entity_id: at.match?.mediaEntityId ?? null,
match: at.match ? jsonb(at.match) : null,
status: "skipped",
trigger: "auto",
mode: (await scrapeSettings(d)).scrapeMode,
organization_record_id: recordId,
actor_type: "system",
reason,
requested_at: now,
finished_at: now,
})
.execute();
} catch (err) {
d.ctx.logger.warn({ recordId, reason, err: { message: errMessage(err) } }, "记录「未刮削」失败");
}
}
/**
* 整理入库成功后(设置打开或本次要求刮削):照 MP 对作品目录刮削,集只刮本次整理的文件。
* 只对有 TMDB id 或本地元数据(Local)的作品排;没排上(目标不在媒体库、库根只登记、作品没有元数据、排队失败)时
* 在整理记录上记「未刮削(原因)」;设置关闭是预期,不记。
*/
export async function scrapeAfterImport(
d: AoDeps,
entityId: string | null,
@@ -431,23 +500,11 @@ export async function scrapeAfterImport(
): Promise<void> {
if (!entityId) return;
if (opts.enabled === false) return;
let root: { id: string; local_path: string; management_mode: string } | undefined;
let match: ScrapeMatch | null = null;
try {
const s = await scrapeSettings(d);
if (!s.scrapeAfterImport && !opts.force && opts.enabled !== true) return;
const organization = await d.ctx.db
.selectFrom("organization_record")
.select(["target_library_root_id"])
.where("id", "=", recordId)
.executeTakeFirst();
const match = await entityScrapeMatch(d, entityId, "library");
const root = organization?.target_library_root_id
? await d.ctx.db
.selectFrom("library_root")
.selectAll()
.where("id", "=", organization.target_library_root_id)
.executeTakeFirst()
: undefined;
if (!match || !root || root.management_mode !== "managed") return;
const files = await d.ctx.db
.selectFrom("organization_file_result")
.select("target_path")
@@ -456,6 +513,35 @@ export async function scrapeAfterImport(
.where("status", "=", "succeeded")
.orderBy("ordinal")
.execute();
// 本次没有新入库的视频(都已在库等):没什么可刮
if (!files.some((f) => f.target_path)) return;
const organization = await d.ctx.db
.selectFrom("organization_record")
.select(["target_library_root_id"])
.where("id", "=", recordId)
.executeTakeFirst();
root = organization?.target_library_root_id
? await d.ctx.db
.selectFrom("library_root")
.select(["id", "local_path", "management_mode"])
.where("id", "=", organization.target_library_root_id)
.executeTakeFirst()
: undefined;
match = await entityScrapeMatch(d, entityId, "library");
if (!root) return await recordScrapeSkipped(d, recordId, "scrape_not_library", { match });
if (root.management_mode !== "managed")
return await recordScrapeSkipped(d, recordId, "scrape_root_read_only", { root, match });
if (!match) return await recordScrapeSkipped(d, recordId, "scrape_no_metadata", { root });
if (!match.externalId) {
// 没有 TMDB id:只有本地元数据(Local 模式补全的字段)时才刮
const e = await d.ctx.db
.selectFrom("media_entity")
.select("metadata")
.where("id", "=", entityId)
.executeTakeFirst();
if (!Object.keys((e?.metadata ?? {}) as object).length)
return await recordScrapeSkipped(d, recordId, "scrape_no_metadata", { root, match });
}
// 照 MP 整理后刮削:按作品目录分组,对作品目录刮(作品 / 季的 NFO 与图片照常初始化),集只刮本次整理的文件;
// 视频直接在库根(推不出作品目录)时对文件本身刮。
const byDir = new Map<string, string[]>();
@@ -464,20 +550,35 @@ export async function scrapeAfterImport(
const dir = inferWorkDir(match.kind, f.target_path) ?? f.target_path;
byDir.set(dir, [...(byDir.get(dir) ?? []), f.target_path]);
}
for (const [dir, videos] of byDir)
await queueScrape(d, {
rootId: `lib:${root.id}`,
rootPath: root.local_path,
libraryRootId: root.id,
path: dir,
match,
trigger: "auto",
organizationRecordId: recordId,
idempotencyKey: `${SCRAPE_TASK}:record:${recordId}:${dir}`,
...(dir === videos[0] ? {} : { videos }),
});
for (const [dir, videos] of byDir) {
try {
await queueScrape(d, {
rootId: `lib:${root.id}`,
rootPath: root.local_path,
libraryRootId: root.id,
path: dir,
match,
trigger: "auto",
organizationRecordId: recordId,
idempotencyKey: `${SCRAPE_TASK}:record:${recordId}:${dir}`,
...(dir === videos[0] ? {} : { videos }),
});
} catch (err) {
// 新记录入队失败时 queueScrape 已把它标成 task_enqueue_failed(整理详情可见);这里再兜一条,防止没留下记录
d.ctx.logger.warn({ entityId, recordId, dir, err: { message: errMessage(err) } }, "入库后自动刮削没排上");
const marked = await d.ctx.db
.selectFrom("scrape_record")
.select("id")
.where("organization_record_id", "=", recordId)
.where("path", "=", dir)
.where("reason", "=", "task_enqueue_failed")
.executeTakeFirst();
if (!marked) await recordScrapeSkipped(d, recordId, "task_enqueue_failed", { root, path: dir, match });
}
}
} catch (err) {
d.ctx.logger.warn({ entityId, recordId, err: { message: errMessage(err) } }, "入库后自动刮削没排上");
await recordScrapeSkipped(d, recordId, "scrape_internal_error", { root, match });
}
}
@@ -531,6 +632,8 @@ async function listDir(rootReal: string, rel: string) {
interface Layout {
videos: string[];
/** 选中目录里的全部视频(不进子目录);判断电影目录里有没有别的作品 */
dirVideos: string[];
workDir: string | null;
parts: { work: boolean; season: boolean };
/** 季目录刮削时目录名给出的季号 */
@@ -543,18 +646,31 @@ async function layoutOf(rootReal: string, rel: string, isDir: boolean, kind: "mo
const all = { work: true, season: true };
if (!isDir)
return kind === "movie"
? { videos: [rel], workDir: parent, parts: all, dirSeason: null }
: { videos: [rel], workDir: null, parts: { work: false, season: false }, dirSeason: null };
? {
videos: [rel],
dirVideos: (await listDir(rootReal, parent ?? "")).videos,
workDir: parent,
parts: all,
dirSeason: null,
}
: { videos: [rel], dirVideos: [], workDir: null, parts: { work: false, season: false }, dirSeason: null };
const here = await listDir(rootReal, rel);
if (kind === "movie") {
// 蓝光原盘目录:写 <目录名>.nfo(借一个同名的虚拟视频路径让布局按目录名命名)+ 目录海报
if (rel && here.dirs.some((n) => /^(BDMV|VIDEO_TS)$/i.test(n)))
return { videos: [joinRel(rel, `${baseOf(rel)}.bluray`)], workDir: rel, parts: all, dirSeason: null };
return { videos: here.videos, workDir: rel || null, parts: all, dirSeason: null };
return {
videos: [joinRel(rel, `${baseOf(rel)}.bluray`)],
dirVideos: [],
workDir: rel,
parts: all,
dirSeason: null,
};
return { videos: here.videos, dirVideos: here.videos, workDir: rel || null, parts: all, dirSeason: null };
}
if (rel && isSeasonDirName(baseOf(rel)))
return {
videos: here.videos,
dirVideos: here.videos,
workDir: parent,
parts: { work: false, season: true },
dirSeason: seasonOfDirName(baseOf(rel)),
@@ -562,18 +678,26 @@ async function layoutOf(rootReal: string, rel: string, isDir: boolean, kind: "mo
const videos = [...here.videos];
for (const sub of here.dirs)
if (isSeasonDirName(sub)) videos.push(...(await listDir(rootReal, joinRel(rel, sub))).videos);
return { videos, workDir: rel || null, parts: all, dirSeason: null };
return { videos, dirVideos: here.videos, workDir: rel || null, parts: all, dirSeason: null };
}
/** 已登记视频的季集(覆盖单元里最小的一集,多集文件按首集) */
/** 已登记视频的季集:覆盖单元里最小的一集为首集,同一季覆盖的其余集一并列出(多集文件每集一个 NFO 块) */
interface Covered {
season: number;
episode: number;
episodes: number[];
}
/** 一个文件最多按这么多集写 NFO 块(整季合成一个文件的极端情况不至于写出巨型 NFO) */
const MAX_EPISODES_PER_FILE = 100;
async function coverageOf(
d: AoDeps,
libraryRootId: string,
entityId: string,
videos: readonly string[],
units: readonly { id: string; kind: string; season_number: number | null; episode_number: number | null }[],
): Promise<Map<string, { season: number; episode: number }>> {
const out = new Map<string, { season: number; episode: number }>();
): Promise<Map<string, Covered>> {
const out = new Map<string, Covered>();
if (!videos.length) return out;
const rows = await d.ctx.db
.selectFrom("asset_location as l")
@@ -586,16 +710,103 @@ async function coverageOf(
.where("a.media_entity_id", "=", entityId)
.execute();
const byId = new Map(units.map((u) => [u.id, u]));
const all = new Map<string, { season: number; episode: number }[]>();
for (const r of rows) {
const u = byId.get(r.content_unit_id);
if (u?.kind !== "episode" || u.season_number === null || u.episode_number === null) continue;
const cur = out.get(r.relative_path);
if (!cur || u.season_number < cur.season || (u.season_number === cur.season && u.episode_number < cur.episode))
out.set(r.relative_path, { season: u.season_number, episode: u.episode_number });
all.set(r.relative_path, [
...(all.get(r.relative_path) ?? []),
{ season: u.season_number, episode: u.episode_number },
]);
}
for (const [path, list] of all) {
list.sort((a, b) => a.season - b.season || a.episode - b.episode);
const first = list[0]!;
const episodes = [...new Set(list.filter((x) => x.season === first.season).map((x) => x.episode))].slice(
0,
MAX_EPISODES_PER_FILE,
);
out.set(path, { season: first.season, episode: first.episode, episodes });
}
return out;
}
/** 集区间 → 集号列表(多集文件 E01-E02) */
const rangeOf = (r: { start: number; end: number }): number[] =>
Array.from({ length: Math.max(1, Math.min(r.end - r.start + 1, MAX_EPISODES_PER_FILE)) }, (_, i) => r.start + i);
/** 播出日期统一成 YYYY-MM-DD(解析结果是 YYYYMMDD;单元列可能是 Date) */
function isoDay(v: unknown): string | null {
if (v instanceof Date) return Number.isNaN(+v) ? null : v.toISOString().slice(0, 10);
if (typeof v !== "string") return null;
const m = /^(\d{4})-?(\d{2})-?(\d{2})/.exec(v.trim());
return m ? `${m[1]}-${m[2]}-${m[3]}` : null;
}
/** 按播出日期在集单元里找集(综艺按日期命名的期数):季已知时只看该季;多于一个候选时不猜 */
function episodeByAirDate(
units: ReadonlyMap<string, UnitForNfo>,
day: string,
season: number | null,
): { season: number; episode: number } | null {
const hits: { season: number; episode: number }[] = [];
for (const u of units.values()) {
if (u.season_number === null || u.episode_number === null) continue;
if (season !== null && u.season_number !== season) continue;
const m = (u.metadata ?? {}) as { airDate?: { value?: unknown } };
if (isoDay(m.airDate?.value) === day || isoDay(u.air_date) === day)
hits.push({ season: u.season_number, episode: u.episode_number });
}
return hits.length === 1 ? hits[0]! : null;
}
/** 图片 MIME → 扩展名;认不出返回 null(沿用按 URL 定的扩展名) */
function imageExtOf(mime: string | null | undefined): string | null {
const m = (mime ?? "").split(";")[0]!.trim().toLowerCase();
return m === "image/jpeg" || m === "image/jpg"
? ".jpg"
: m === "image/png"
? ".png"
: m === "image/webp"
? ".webp"
: null;
}
/** 预期内的跳过(不影响记录状态):已存在、内容没变、被人改过 */
const EXPECTED_SKIPS = new Set(["scrape_exists", "scrape_unchanged", "scrape_user_modified"]);
/**
* 电影目录里有没有别的作品:目录里的视频登记在别的作品下,或(只刮部分视频时)有没登记到本作品、也不在本次范围里的视频。
* 同一作品的多个版本不算。
*/
async function movieDirShared(
d: AoDeps,
rec: { library_root_id: string | null; videos: string[] | null },
lay: Layout,
entityId: string | null,
): Promise<boolean> {
const others = lay.dirVideos.filter((v) => !lay.videos.includes(v));
if (!lay.dirVideos.length) return false;
const owners = new Map<string, Set<string>>();
if (rec.library_root_id) {
const rows = await d.ctx.db
.selectFrom("asset_location as l")
.innerJoin("media_asset as a", "a.id", "l.media_asset_id")
.select(["l.relative_path", "a.media_entity_id"])
.where("l.library_root_id", "=", rec.library_root_id)
.where("l.relative_path", "in", lay.dirVideos)
.where("l.status", "<>", "removed")
.where("a.status", "=", "active")
.execute();
for (const r of rows)
owners.set(r.relative_path, (owners.get(r.relative_path) ?? new Set()).add(r.media_entity_id ?? ""));
}
return lay.dirVideos.some((v) => {
const own = owners.get(v);
if (own && [...own].some((e) => e !== entityId)) return true;
return !own?.size && others.includes(v);
});
}
const sha256 = (c: Uint8Array | string) => createHash("sha256").update(c).digest("hex");
/** 旧版本写的文件没有 hash:修改时间不晚于那次刮削结束(加这点余量)就当没被改过 */
const LEGACY_MTIME_SLACK_MS = 2_000;
@@ -757,7 +968,7 @@ export async function runScrapeRecord(
// ── 字段:本地作品(有元数据)优先 ──
let fields: Fields | null = null;
let coverage = new Map<string, { season: number; episode: number }>();
let coverage = new Map<string, Covered>();
let mapEpisodes: EntityEpisodeMapper | null = null;
if (match.mediaEntityId) {
const e = await db
@@ -807,11 +1018,17 @@ export async function runScrapeRecord(
}
}
// 电影目录里还有别的作品(自定义平铺模板):图片按视频名命名,不写目录共用的 poster / fanart
const sharedDir = kind === "movie" && (await movieDirShared(d, rec, lay, match.mediaEntityId));
// 季 / 集号:已登记的按覆盖单元;其余按路径解析(季目录名兜底,同 MP 对每集重新 MetaInfoPath)
const videos: ScrapeVideo[] = [];
const noEpisode: ScrapeFileResult[] = [];
/** 集数待确认(不写分集 NFO,原因 episode_unresolved) */
const unresolved = new Set<string>();
/** 只有播出日期、没有集号的视频(综艺按日期命名的期数):路径 → YYYY-MM-DD,季 / 集元数据到位后按日期对集 */
const dated = new Map<string, string>();
// 集数映射(批次 3):未登记的文件按本地剧集作品的季表 + 规则把文件名季集换成规范季集再对集元数据;
// 集数待确认的不写分集 NFO(原因 episode_unresolved)
// 综艺按播出日期对上季表里的集
for (const v of lay.videos) {
if (kind === "movie") {
videos.push({ path: v });
@@ -819,33 +1036,29 @@ export async function runScrapeRecord(
}
const cov = coverage.get(v);
if (cov) {
videos.push({ path: v, season: cov.season, episode: cov.episode });
videos.push({ path: v, season: cov.season, episode: cov.episode, episodes: cov.episodes });
continue;
}
const p = parseReleasePath(v);
let season = p.season ?? lay.dirSeason ?? seasonOfDirName(baseOf(dirOf(v))) ?? (p.episodes ? 1 : null);
let episode = p.episodes?.start ?? null;
if (mapEpisodes && p.episodes) {
let range = p.episodes ?? null;
if (mapEpisodes && (p.episodes || p.airDate)) {
const m = mapEpisodes(p);
if (m.resolution.method === "unresolved") {
noEpisode.push({
path: v,
kind: "nfo",
role: "episode_nfo",
action: "skipped",
reason: "episode_unresolved",
});
unresolved.add(v);
videos.push({ path: v, season, episode: null });
continue;
}
if (m.resolution.canonical) {
season = m.resolution.canonical.season;
episode = m.resolution.canonical.episodes.start;
range = m.resolution.canonical.episodes;
}
}
if (episode === null)
noEpisode.push({ path: v, kind: "nfo", role: "episode_nfo", action: "skipped", reason: "scrape_no_episode" });
videos.push({ path: v, season, episode });
if (!range && p.airDate) {
const day = isoDay(p.airDate);
if (day) dated.set(v, day);
}
videos.push({ path: v, season, episode: range?.start ?? null, ...(range ? { episodes: rangeOf(range) } : {}) });
}
// ── 没有本地字段:按 TMDB 条目直接取 ──
@@ -868,6 +1081,21 @@ export async function runScrapeRecord(
// ── 季 / 集元数据:TMDB 分支取每个涉及的季;本地分支只在季 / 集单元缺元数据时按 TMDB id 现取补齐 ──
const { seasonUnit, episodeUnit } = fields;
const epsOf = (v: ScrapeVideo) =>
v.episodes?.length ? v.episodes : typeof v.episode === "number" ? [v.episode] : [];
/** 只有播出日期的视频按日期对集(本地集单元或刚取到的 TMDB 季详情) */
const resolveDated = () => {
for (const v of videos) {
const day = dated.get(v.path);
if (!day || typeof v.episode === "number") continue;
const hit = episodeByAirDate(episodeUnit, day, v.season ?? null);
if (!hit) continue;
v.season = hit.season;
v.episode = hit.episode;
v.episodes = [hit.episode];
}
};
resolveDated();
if (kind === "tv") {
const tmdbId = match.externalId ?? fields.ids.tmdb ?? null;
const seasons = [...new Set(videos.map((v) => v.season).filter((s): s is number => typeof s === "number"))];
@@ -877,8 +1105,8 @@ export async function runScrapeRecord(
videos.some(
(v) =>
v.season === n &&
typeof v.episode === "number" &&
episodeMetadataMissing(episodeUnit.get(`${n}:${v.episode}`)),
(epsOf(v).some((ep) => episodeMetadataMissing(episodeUnit.get(`${n}:${ep}`))) ||
(typeof v.episode !== "number" && dated.has(v.path))),
);
if (!tmdbId || (!fromTmdb && !lacking)) continue;
const sd = await d.ports.scrapeTmdbSeason(tmdbId, n).catch((err) => {
@@ -894,6 +1122,17 @@ export async function runScrapeRecord(
if (fromTmdb || episodeMetadataMissing(episodeUnit.get(k))) episodeUnit.set(k, eu);
}
}
resolveDated();
for (const v of videos) {
if (typeof v.episode === "number") continue;
files.push({
path: v.path,
kind: "nfo",
role: "episode_nfo",
action: "skipped",
reason: unresolved.has(v.path) ? "episode_unresolved" : "scrape_no_episode",
});
}
}
const work = nfoWork(fields.entity, fields.ids);
const seasonPosters = new Map<number, string | null>();
@@ -908,15 +1147,15 @@ export async function runScrapeRecord(
images: { ...imagePathsOf(fields.entity.metadata), seasonPosters, stills },
options: { images: settings.scrapeImages, episodeThumbs: settings.scrapeEpisodeThumbs },
parts: lay.parts,
sharedDir,
});
files.push(...noEpisode);
// 「更新」模式:本工具以前写过的文件(路径 → 写入时的 hash / 时间)
const ours = mode === "update" ? await previousWrites(d, rec.root_id, recordId) : new Map<string, PrevWrite>();
const imageCache = new Map<string, Promise<Uint8Array>>();
const imageCache = new Map<string, Promise<{ content: Uint8Array; mime: string }>>();
const fetchImage = (p: string) => {
let hit = imageCache.get(p);
if (!hit) {
hit = d.ports.scrapeImage(p).then((r) => r.content);
hit = d.ports.scrapeImage(p);
imageCache.set(p, hit);
}
return hit;
@@ -978,40 +1217,53 @@ export async function runScrapeRecord(
}
xml = seasonNfo(nfoSeason(t.season!, u));
} else {
const u = episodeUnit.get(`${t.season}:${t.episode}`);
if (episodeMetadataMissing(u) && hasTmdbId) {
// 多集文件每集一块;缺元数据的集不写(全缺才跳过整个 NFO)
const eps = (t.episodes?.length ? t.episodes : [t.episode!]).filter(
(ep) => !(hasTmdbId && episodeMetadataMissing(episodeUnit.get(`${t.season}:${ep}`))),
);
if (!eps.length) {
files.push(missingMeta(t));
continue;
}
xml = episodeNfo(nfoEpisode(t.season!, t.episode!, u));
xml = multiEpisodeNfo(eps.map((ep) => nfoEpisode(t.season!, ep, episodeUnit.get(`${t.season}:${ep}`))));
}
files.push(await write(t, dec, xml));
}
// 图片:限并发,失败不影响 NFO
const imgs = targets.filter((x) => x.kind === "image" && x.imagePath);
files.push(
...(await pool(imgs, IMAGE_CONCURRENCY, async (t): Promise<ScrapeFileResult> => {
...(await pool(imgs, IMAGE_CONCURRENCY, async (planned0): Promise<ScrapeFileResult> => {
if (signal?.aborted) throw new Error("aborted");
const dec = await decide(t);
let t = planned0;
let dec = await decide(t);
if (!("abs" in dec)) return dec;
let bytes: Uint8Array;
let got: { content: Uint8Array; mime: string };
try {
bytes = await fetchImage(t.imagePath!);
got = await fetchImage(t.imagePath!);
} catch (err) {
d.ctx.logger.warn({ recordId, image: t.imagePath, err: { message: errMessage(err) } }, "刮削图片下载失败");
return { path: t.path, kind: t.kind, role: t.role, action: "failed", reason: "scrape_image_failed" };
}
return write(t, dec, bytes);
// 扩展名按实际类型(URL 认不出或与内容不符时,例如 Local 封面链接是 webp、内容是 jpeg)
const ext = imageExtOf(got.mime);
const planned = /\.(jpe?g|png|webp)$/i.exec(t.path)?.[0]?.toLowerCase();
if (ext && planned && ext !== planned && !(ext === ".jpg" && planned === ".jpeg")) {
t = { ...t, path: t.path.slice(0, -planned.length) + ext };
dec = await decide(t);
if (!("abs" in dec)) return dec;
}
return write(t, dec, got.content);
})),
);
const written = files.filter((f) => f.action === "written").length;
const failed = files.filter((f) => f.action === "failed").length;
// 缺元数据跳过的季 / 集不算成功(要刷新元数据后再刮)
const lacking = files.some((f) => f.reason === "scrape_metadata_missing");
// 缺元数据、认不出集号、集数待确认而跳过的不算成功(要刷新元数据 / 确认集数后再刮);
// 已存在、内容没变、被人改过的跳过是预期的
const incomplete = files.find((f) => f.action === "skipped" && !EXPECTED_SKIPS.has(f.reason ?? ""));
const status: ScrapeRecord["status"] =
failed === 0
? lacking
? incomplete
? "partial"
: "succeeded"
: written > 0 || files.some((f) => f.action === "skipped")
@@ -1021,13 +1273,17 @@ export async function runScrapeRecord(
status,
reason: failed
? (files.find((f) => f.action === "failed")?.reason ?? null)
: lacking
: files.some((f) => f.reason === "scrape_metadata_missing")
? "scrape_metadata_missing"
: null,
: (incomplete?.reason ?? null),
files,
});
} catch (err) {
if (signal?.aborted) throw err;
if (signal?.aborted) {
// 任务被取消(引擎不会再跑这一步):记录收尾为失败,已写的文件结果照留,不停在进行中
await finish({ status: "failed", reason: "scrape_canceled", files }).catch(() => {});
throw err;
}
d.ctx.logger.warn({ recordId, err: { message: errMessage(err) } }, "刮削失败");
// 不是写文件的错(如查库失败):记内部错误,已写的文件结果照留
const anyWritten = files.some((f) => f.action === "written");
+42 -6
View File
@@ -584,7 +584,10 @@ export async function restoreTrash(
return { restored, failed, recordId };
}
/** 清除:先删行(复核仍存在)再删文件;ids 缺省 = 只清过期项 */
/**
* 清除:先删文件再删行(删文件失败时行留着,下次再清,不留没有登记的孤儿文件);ids 缺省 = 只清过期项,
* 并顺带清掉各库根 `.umt-trash/replaced-*` 里过期、没有回收站登记的旧文件(替换旧文件登记回收站失败时留下的)。
*/
export async function purgeTrash(
d: FmDeps,
principal: Principal | undefined,
@@ -598,18 +601,51 @@ export async function purgeTrash(
for (const row of rows) {
try {
if (!trashPathSafe(d, row)) throw new FilesError("path_escape", "回收站路径异常,拒绝清除");
const gone = await d.ctx.db.deleteFrom("trash_item").where("id", "=", row.id).returning("id").executeTakeFirst();
if (!gone) continue;
await rm(row.trash_path, { recursive: true, force: true });
// 顺带清掉空的 <uuid>/<yyyymmdd> 目录
const uuidDir = row.trash_path.slice(0, row.trash_path.length - row.original_path.length - 1);
await rm(uuidDir, { recursive: true, force: true }).catch(() => undefined);
purged.push(row.id);
const gone = await d.ctx.db.deleteFrom("trash_item").where("id", "=", row.id).returning("id").executeTakeFirst();
if (gone) purged.push(row.id);
} catch (e) {
failed.push({ id: row.id, code: e instanceof FilesError ? e.reason : "io_error", message: errMessage(e) });
}
}
if (purged.length || failed.length)
await audit(d, principal, "trash_purge", { purged, failed, expiredOnly: !body.ids });
const orphans = body.ids ? [] : await purgeReplacedOrphans(d);
if (purged.length || failed.length || orphans.length)
await audit(d, principal, "trash_purge", { purged, failed, orphans, expiredOnly: !body.ids });
return { purged, failed };
}
/** 各启用库根 `.umt-trash/replaced-*`:修改时间早于保留期、里面没有任何回收站登记的,整个删掉;返回删掉的目录 */
async function purgeReplacedOrphans(d: FmDeps): Promise<string[]> {
const out: string[] = [];
const cutoff = d.ao.ports.now().getTime() - d.options.trashRetentionDays * 86_400_000;
const roots = await d.ctx.db
.selectFrom("library_root")
.select(["local_path"])
.where("enabled", "=", true)
.where("management_mode", "=", "managed")
.execute();
for (const r of roots) {
const rootReal = await realpath(r.local_path).catch(() => null);
if (!rootReal) continue;
const bin = join(rootReal, TRASH_DIR);
for (const name of await readdir(bin).catch(() => [] as string[])) {
if (!name.startsWith("replaced-")) continue;
const dir = join(bin, name);
const st = await lstat(dir).catch(() => null);
if (!st?.isDirectory() || st.mtimeMs > cutoff) continue;
const held = await d.ctx.db
.selectFrom("trash_item")
.select("id")
.where("trash_path", "like", likePrefix(dir))
.executeTakeFirst();
if (held) continue;
await rm(dir, { recursive: true, force: true })
.then(() => out.push(dir))
.catch(() => undefined);
}
}
return out;
}