feat(fs): W4-E4 fsproxy hang isolation, partial sweep, replace-with-verify, TIMEOUT retry
- adapters/fs: FsProxy + fsworker run each file op in a killable subprocess;
progress-stall watchdog (FS_HANG_TIMEOUT_MS) for copy/hash, total timeout for
quick ops; SIGTERM -> SIGKILL -> FsError('TIMEOUT'); stat/checkRoot timeouts
never read as missing
- copyWithVerify: onExisting conflict/skip/replace (hardlink backup + atomic
rename, old file to .umt-trash or deleted), sweep stale partials before write,
document why permissions/owner are not copied
- filesMatch/filesMatchSha256 throw FsError on read failure instead of
match=false; crossDeviceMove skips redundant re-hash after sha256 copy
- FsErrorCode TIMEOUT + fsErrorClass; organize executeRecord routes fs ops
through fsproxy, retries TIMEOUT/IO up to FS_TRANSFER_MAX_ATTEMPTS then opens
a decision without deleting the record; reconcile errors no longer reported
as EEXIST
- domain: subtitle .default marker (ORGANIZE_DEFAULT_SUBTITLE_LANG) and
findOtherVersions (same season/episode/part, video only) as upgrade
disposal candidates
- scripts/bench/fs-bench.ts and benchmark record in porting doc
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
36dc048e89
commit
2e2d56184c
21 files changed
+1970
-179
No files matched your search
@@ -30,6 +30,31 @@ export const ConfigSchema = z.object({
|
||||
DATA_DIR: z.string().min(1).default("./data"),
|
||||
/** Agent JSONL 会话目录;缺省 DATA_DIR/agent-sessions(api 与 worker 必须共享同一目录) */
|
||||
AGENT_SESSION_DIR: z.string().min(1).optional(),
|
||||
// ── 文件操作挂死隔离(adapters fsproxy,对标 MP fsproxy)──
|
||||
/** auto:能起 deno 子进程就把单个文件操作放进执行子进程(可 SIGKILL 回收),否则进程内看门狗;inprocess 强制进程内 */
|
||||
FS_PROXY_MODE: z.enum(["auto", "subprocess", "inprocess"]).default("auto"),
|
||||
/** 复制/哈希:进度停滞多久判挂死(ms),超时 SIGTERM→SIGKILL 并以 TIMEOUT 失败 */
|
||||
FS_HANG_TIMEOUT_MS: z.coerce.number().int().min(1000).default(120_000),
|
||||
/** stat/根检查/同卷 move 等快操作的总超时(ms) */
|
||||
FS_QUICK_TIMEOUT_MS: z.coerce.number().int().min(500).default(30_000),
|
||||
/** 执行子进程上报进度心跳的间隔(ms) */
|
||||
FS_PROGRESS_INTERVAL_MS: z.coerce.number().int().min(10).default(1000),
|
||||
/** SIGTERM 到 SIGKILL 的宽限(ms) */
|
||||
FS_KILL_GRACE_MS: z.coerce.number().int().min(100).default(2000),
|
||||
/** 写入前清理目标目录残留 partial 的年龄阈值(ms);0 = 不清 */
|
||||
FS_PARTIAL_MAX_AGE_MS: z.coerce
|
||||
.number()
|
||||
.int()
|
||||
.min(0)
|
||||
.default(24 * 3600_000),
|
||||
/** 单个文件的 TIMEOUT/IO 类失败最多尝试几次(含首次);超限转待处理,不删记录不重下 */
|
||||
FS_TRANSFER_MAX_ATTEMPTS: z.coerce.number().int().min(1).max(10).default(3),
|
||||
/** 替换入库(replace_with_verify)时旧文件去向:trash = 移到库根 .umt-trash;delete = 删除 */
|
||||
FS_REPLACE_OLD: z.enum(["trash", "delete"]).default("trash"),
|
||||
/** 入库目标已存在时的缺省策略:conflict = 转待处理;skip = 跳过;replace_with_verify = 授权后写临时→校验→原子替换 */
|
||||
ORGANIZE_ON_EXISTING: z.enum(["conflict", "skip", "replace_with_verify"]).default("conflict"),
|
||||
/** 随行字幕中此语言加 .default 标记(对照 MP DEFAULT_SUB);不设 = 不加 */
|
||||
ORGANIZE_DEFAULT_SUBTITLE_LANG: z.enum(["zh-cn", "zh-tw", "en", "ja", "ko"]).optional(),
|
||||
});
|
||||
export type Config = z.infer<typeof ConfigSchema>;
|
||||
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { existsSync } from "node:fs";
|
||||
import { readdir, readFile, rm, utimes, writeFile } from "node:fs/promises";
|
||||
import { dirname, join } from "node:path";
|
||||
import { createTempTree, type TempTree } from "@umt/testkit";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { executeRecord, type FrozenPlan, planImport } from "../services/organize.ts";
|
||||
import { createHarness, type Harness, pgAvailable, seedBase, seedEntity } from "./harness.ts";
|
||||
|
||||
// W4-E4 执行侧:replace_with_verify 接线(写临时→校验→原子替换,旧文件进回收目录)、写前清 partial、
|
||||
// 对账比对挂死不误报 EEXIST 而是 TIMEOUT → 有限次重试 → 超限建待处理(不删记录)。
|
||||
|
||||
const hasPg = await pgAvailable();
|
||||
const SIZE = 200_000;
|
||||
|
||||
describe.skipIf(!hasPg)("整理执行:onExisting / partial 清理 / TIMEOUT 重试", () => {
|
||||
let h: Harness;
|
||||
let lib: TempTree;
|
||||
let src: TempTree;
|
||||
|
||||
beforeAll(async () => {
|
||||
lib = await createTempTree({}, { sentinel: true, prefix: "umt-fslib-" });
|
||||
src = await createTempTree(
|
||||
{ "Replace.Me.2020.1080p.BluRay.x264-GRP.mkv": SIZE, "Hang.Case.2019.1080p.BluRay.x264-GRP.mkv": SIZE },
|
||||
{ prefix: "umt-fssrc-" },
|
||||
);
|
||||
h = await createHarness("fsexec");
|
||||
await seedBase(h.db, { libraryPath: lib.root, cachePath: src.root, qbUrl: "http://127.0.0.1:9" });
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await h?.close();
|
||||
for (const t of [lib, src]) await t?.cleanup();
|
||||
});
|
||||
|
||||
const plannedDst = async (entity: string, file: string) => {
|
||||
const dry = await planImport(h.d, {
|
||||
sourcePaths: [src.path(file)],
|
||||
mediaEntityId: entity,
|
||||
action: "copy",
|
||||
trigger: "manual",
|
||||
actorType: "user",
|
||||
dryRun: true,
|
||||
});
|
||||
return dry.plan!.organize.files.find((f) => f.role === "main")!.dst;
|
||||
};
|
||||
|
||||
it("replace_with_verify + 授权:原子替换旧文件,旧文件进 .umt-trash;写前清走 >24h 的 partial", async () => {
|
||||
const e = await seedEntity(h.db, { kind: "movie", title: "替换片", year: 2020 });
|
||||
const file = "Replace.Me.2020.1080p.BluRay.x264-GRP.mkv";
|
||||
const dst = await plannedDst(e, file);
|
||||
await writeFile(dst, "OLD-VERSION").catch(async () => {
|
||||
execFileSync("mkdir", ["-p", dirname(dst)]);
|
||||
await writeFile(dst, "OLD-VERSION");
|
||||
});
|
||||
const stale = join(dirname(dst), ".x.mkv.1.abc.uhd-partial");
|
||||
await writeFile(stale, "half");
|
||||
const old = new Date(Date.now() - 48 * 3600_000);
|
||||
await utimes(stale, old, old);
|
||||
|
||||
// 无授权:阻塞待处理,不执行
|
||||
const blocked = await planImport(h.d, {
|
||||
sourcePaths: [src.path(file)],
|
||||
mediaEntityId: e,
|
||||
action: "copy",
|
||||
trigger: "manual",
|
||||
actorType: "user",
|
||||
onExisting: "replace_with_verify",
|
||||
});
|
||||
expect(blocked.status).toBe("failed");
|
||||
expect(blocked.reason).toContain("replace_requires_grant");
|
||||
|
||||
const plan = await planImport(h.d, {
|
||||
sourcePaths: [src.path(file)],
|
||||
mediaEntityId: e,
|
||||
action: "copy",
|
||||
trigger: "manual",
|
||||
actorType: "user",
|
||||
onExisting: "replace_with_verify",
|
||||
grantReplace: true,
|
||||
});
|
||||
expect(plan.status).toBe("running");
|
||||
const rec = await h.db
|
||||
.selectFrom("organization_record")
|
||||
.select("plan")
|
||||
.where("id", "=", plan.recordId!)
|
||||
.executeTakeFirstOrThrow();
|
||||
expect((rec.plan as unknown as FrozenPlan).organize.files[0]!.action).toBe("replace");
|
||||
|
||||
const ex = await executeRecord(h.d, plan.recordId!);
|
||||
expect(ex.status).toBe("succeeded");
|
||||
expect((await readFile(dst)).equals(await readFile(src.path(file)))).toBe(true);
|
||||
const op = await h.db
|
||||
.selectFrom("asset_operation")
|
||||
.select("result")
|
||||
.where("operation_id", "=", `org:${plan.recordId}:0`)
|
||||
.executeTakeFirstOrThrow();
|
||||
const r = op.result as { replaced?: { disposition: string; path: string } };
|
||||
expect(r.replaced?.disposition).toBe("trash");
|
||||
expect(r.replaced!.path.startsWith(join(lib.root, ".umt-trash"))).toBe(true);
|
||||
expect(await readFile(r.replaced!.path, "utf8")).toBe("OLD-VERSION");
|
||||
expect(existsSync(stale)).toBe(false);
|
||||
expect((await readdir(dirname(dst))).filter((n) => n.startsWith("."))).toEqual([]);
|
||||
}, 60_000);
|
||||
|
||||
it("对账比对挂死 → TIMEOUT(不是 EEXIST)→ 有限次重试 → 超限建待处理,记录保留", async () => {
|
||||
const ctx = h.d.ctx as { config: Record<string, unknown> };
|
||||
const saved = ctx.config;
|
||||
ctx.config = { ...saved, FS_HANG_TIMEOUT_MS: 1000, FS_KILL_GRACE_MS: 300, FS_TRANSFER_MAX_ATTEMPTS: 3 };
|
||||
try {
|
||||
const e = await seedEntity(h.db, { kind: "movie", title: "挂死片", year: 2019 });
|
||||
const file = "Hang.Case.2019.1080p.BluRay.x264-GRP.mkv";
|
||||
const plan = await planImport(h.d, {
|
||||
sourcePaths: [src.path(file)],
|
||||
mediaEntityId: e,
|
||||
action: "copy",
|
||||
trigger: "manual",
|
||||
actorType: "user",
|
||||
});
|
||||
expect(plan.status).toBe("running");
|
||||
const f = (plan.plan ??
|
||||
((
|
||||
await h.db
|
||||
.selectFrom("organization_record")
|
||||
.select("plan")
|
||||
.where("id", "=", plan.recordId!)
|
||||
.executeTakeFirstOrThrow()
|
||||
).plan as unknown as FrozenPlan))!.organize.files[0]!;
|
||||
// 源换成 FIFO(读会卡在 open)、目标放一个同尺寸(0)的文件:对账走 sha256 比对并挂死
|
||||
await rm(f.src);
|
||||
execFileSync("mkfifo", [f.src]);
|
||||
execFileSync("mkdir", ["-p", dirname(f.dst)]);
|
||||
await writeFile(f.dst, "");
|
||||
|
||||
for (const attempt of [1, 2]) {
|
||||
const err = await executeRecord(h.d, plan.recordId!).catch((x) => x);
|
||||
expect(err).toBeInstanceOf(Error);
|
||||
expect(String(err.message)).toContain(`TIMEOUT@${f.src}(第 ${attempt} 次)`);
|
||||
const fr = await h.db
|
||||
.selectFrom("organization_file_result")
|
||||
.select(["status"])
|
||||
.where("organization_record_id", "=", plan.recordId!)
|
||||
.where("role", "=", "main")
|
||||
.executeTakeFirstOrThrow();
|
||||
expect(fr.status).toBe("running");
|
||||
}
|
||||
const ex = await executeRecord(h.d, plan.recordId!);
|
||||
expect(ex.status).toBe("failed");
|
||||
const fr = await h.db
|
||||
.selectFrom("organization_file_result")
|
||||
.select(["status", "error"])
|
||||
.where("organization_record_id", "=", plan.recordId!)
|
||||
.where("role", "=", "main")
|
||||
.executeTakeFirstOrThrow();
|
||||
expect(fr.status).toBe("failed");
|
||||
expect((fr.error as { code: string }).code).toBe("TIMEOUT");
|
||||
const dec = await h.db
|
||||
.selectFrom("decision")
|
||||
.select(["kind", "title", "evidence"])
|
||||
.where("dedupe_key", "=", `fs_retry_exhausted:org:${plan.recordId}:0`)
|
||||
.executeTakeFirstOrThrow();
|
||||
expect(dec.title).toContain("挂死");
|
||||
expect((dec.evidence as { attempts: number }).attempts).toBe(3);
|
||||
// 记录保留,目标未被覆盖/删除
|
||||
const rec = await h.db
|
||||
.selectFrom("organization_record")
|
||||
.select("status")
|
||||
.where("id", "=", plan.recordId!)
|
||||
.executeTakeFirst();
|
||||
expect(rec?.status).toBe("failed");
|
||||
expect(existsSync(f.dst)).toBe(true);
|
||||
await rm(f.src);
|
||||
} finally {
|
||||
ctx.config = saved;
|
||||
}
|
||||
}, 90_000);
|
||||
});
|
||||
@@ -1,14 +1,15 @@
|
||||
import { mkdir, readdir } from "node:fs/promises";
|
||||
import { dirname, join, relative } from "node:path";
|
||||
import {
|
||||
type CopyReceipt,
|
||||
type CrossMoveStepLog,
|
||||
checkRoot,
|
||||
copyWithVerify,
|
||||
crossDeviceMove,
|
||||
DEFAULT_ROOT_SENTINEL,
|
||||
FsError,
|
||||
filesMatchSha256,
|
||||
moveSafe,
|
||||
FsProxy,
|
||||
fsErrorClass,
|
||||
type ReplaceOldPolicy,
|
||||
type StatResult,
|
||||
sha256Full,
|
||||
statSafe,
|
||||
} from "@umt/adapters";
|
||||
@@ -19,7 +20,9 @@ import {
|
||||
buildNamingContext,
|
||||
DEFAULT_TEMPLATES,
|
||||
type PathBinding as DomainBinding,
|
||||
type ExistingPolicy,
|
||||
type ExistingTarget,
|
||||
findOtherVersions,
|
||||
movieCompletion,
|
||||
type NewBinding,
|
||||
type OrganizePlan,
|
||||
@@ -59,6 +62,10 @@ export interface ImportRequest {
|
||||
dryRun?: boolean;
|
||||
/** 手动按路径整理:遍历到的每个文件都经它复核(realpath 在允许的根下),越界抛 PathGuardError */
|
||||
pathGuard?: ((p: string) => Promise<string>) | undefined;
|
||||
/** 目标已存在策略(缺省取 ORGANIZE_ON_EXISTING,再缺省 conflict);replace_with_verify 还需 grantReplace,否则阻塞待授权 */
|
||||
onExisting?: ExistingPolicy | undefined;
|
||||
/** 已获替换授权(审批/手动确认) */
|
||||
grantReplace?: boolean | undefined;
|
||||
}
|
||||
|
||||
/** organization_record.plan 的冻结内容 */
|
||||
@@ -443,6 +450,12 @@ export async function planImport(d: AoDeps, req: ImportRequest): Promise<PlanRes
|
||||
target: { ...target, ...(health.kind === "online" ? { device: health.dev } : {}) },
|
||||
retention: { hnrSatisfied: hnr },
|
||||
minVideoBytes: d.ports.options.minVideoBytes,
|
||||
...(d.ctx.config?.ORGANIZE_DEFAULT_SUBTITLE_LANG
|
||||
? { defaultSubtitleLang: d.ctx.config.ORGANIZE_DEFAULT_SUBTITLE_LANG }
|
||||
: {}),
|
||||
// 目标存在策略只透传给域层规划;执行端按冻结计划的 action(replace/transfer)传 onExisting
|
||||
onExisting: req.onExisting ?? d.ctx.config?.ORGANIZE_ON_EXISTING ?? "conflict",
|
||||
...(req.grantReplace ? { grants: { replace: true } } : {}),
|
||||
} as const;
|
||||
const draft = planOrganize({ ...baseInput, existingTargets: [] });
|
||||
const existing: ExistingTarget[] = [];
|
||||
@@ -644,6 +657,14 @@ type FileOutcome = {
|
||||
/** 全量 sha256 的执行方式(sha256sum 子进程 / 进程内流式) */
|
||||
hashVia?: string;
|
||||
durationMs?: number;
|
||||
/** 本次执行是第几次尝试(账本 attempts) */
|
||||
attempt?: number;
|
||||
/** 失败时的附加证据(如对账阶段比对失败) */
|
||||
evidence?: { phase: string; src: string; dst: string };
|
||||
/** replace:旧文件去向 */
|
||||
replaced?: { disposition: string; size: number; mtimeMs: number; path?: string; error?: string };
|
||||
/** 写前清走的残留 partial */
|
||||
sweptPartials?: string[];
|
||||
};
|
||||
const stepsOf = (s: CrossMoveStepLog[]): StepLog[] => s.map((x) => ({ ...x }));
|
||||
|
||||
@@ -669,9 +690,15 @@ export async function executeRecord(
|
||||
return { recordId, status: rec.status, succeeded: rec.file_succeeded, failed: rec.file_failed };
|
||||
}
|
||||
const frozen = rec.plan as unknown as FrozenPlan;
|
||||
const health = await checkRoot(frozen.rootPath, frozen.sentinel);
|
||||
const fsr = fsRuntime(d);
|
||||
// 根检查也走执行子进程:NFS/FUSE 挂死 → offline(timeout),按根离线重试,绝不当成文件缺失
|
||||
const health = await fsr.proxy.checkRoot(frozen.rootPath, frozen.sentinel);
|
||||
if (health.kind === "offline") throw new Error(`媒体库根离线(${health.reason}),稍后重试`);
|
||||
for (const dir of frozen.targetDirs) await mkdir(dir, { recursive: true });
|
||||
// 写前顺带清本目录残留 partial(对照 MP:只清本目录、>24h;失败吞掉)。copy 路径在 copyWithVerify 内还会再清一次
|
||||
if (fsr.partialMaxAgeMs > 0)
|
||||
for (const dir of frozen.targetDirs)
|
||||
await fsr.proxy.sweepStalePartials(dir, fsr.partialMaxAgeMs).catch(() => [] as string[]);
|
||||
const orgSettings = await d.ctx.settings.get("organize");
|
||||
|
||||
const files = await db
|
||||
@@ -682,10 +709,14 @@ export async function executeRecord(
|
||||
.execute();
|
||||
const planned = new Map(frozen.organize.files.map((f) => [f.src, f]));
|
||||
const failedMains = new Set<string>();
|
||||
/** 本轮 TIMEOUT/IO 类失败、留待任务重试的文件(结果不入账本,文件结果保持 running) */
|
||||
const deferred: { src: string; code: string; message: string; attempt: number }[] = [];
|
||||
for (const fr of files) {
|
||||
if (fr.status !== "queued" && fr.status !== "running") continue;
|
||||
const pf = planned.get(fr.source_path);
|
||||
if (!pf) continue;
|
||||
// 主文件等重试:随行文件本轮也不动
|
||||
if (pf.role === "sidecar" && pf.mainSrc && deferred.some((x) => x.src === pf.mainSrc)) continue;
|
||||
if (pf.role === "sidecar" && pf.mainSrc && failedMains.has(pf.mainSrc)) {
|
||||
await db
|
||||
.updateTable("organization_file_result")
|
||||
@@ -721,79 +752,125 @@ export async function executeRecord(
|
||||
.set({ status: "running", started_at: fr.started_at ?? d.ports.now() })
|
||||
.where("id", "=", fr.id)
|
||||
.execute();
|
||||
const outcome = await withAttemptToken<FileOutcome>(
|
||||
db,
|
||||
opId,
|
||||
async (info) => {
|
||||
const t0 = performance.now();
|
||||
const r = await transferFile(
|
||||
kind,
|
||||
pf.src,
|
||||
pf.dst,
|
||||
pf.size,
|
||||
pf.crossDevice,
|
||||
info.resumed,
|
||||
d,
|
||||
orgSettings.allowCrossVolumeMove,
|
||||
opts.abortSignal,
|
||||
);
|
||||
r.durationMs = Math.round(performance.now() - t0);
|
||||
// 打点:每个文件的耗时/字节/哈希方式,便于判断大文件入库是否拖慢 worker
|
||||
d.ctx.logger.info(
|
||||
{
|
||||
recordId,
|
||||
ordinal: fr.ordinal,
|
||||
let outcome: FileOutcome;
|
||||
try {
|
||||
outcome = await withAttemptToken<FileOutcome>(
|
||||
db,
|
||||
opId,
|
||||
async (info) => {
|
||||
const t0 = performance.now();
|
||||
const r = await transferFile(d, fsr, {
|
||||
kind,
|
||||
status: r.status,
|
||||
bytes: r.bytes ?? pf.size,
|
||||
durationMs: r.durationMs,
|
||||
mbps: r.durationMs > 0 ? Math.round(((r.bytes ?? pf.size) / 1e6 / r.durationMs) * 1000) : null,
|
||||
hashVia: r.hashVia ?? null,
|
||||
src: pf.src,
|
||||
dst: pf.dst,
|
||||
size: pf.size,
|
||||
crossDevice: pf.crossDevice,
|
||||
resumed: info.resumed,
|
||||
},
|
||||
"organize.transfer",
|
||||
);
|
||||
return r;
|
||||
},
|
||||
{
|
||||
...(opts.runId ? { runId: opts.runId } : {}),
|
||||
commit: async (trx, r) => {
|
||||
const now = d.ports.now();
|
||||
const ok = r.status !== "failed";
|
||||
await trx
|
||||
.updateTable("asset_operation")
|
||||
.set({
|
||||
status: ok ? "succeeded" : "failed",
|
||||
result: jsonb(r),
|
||||
...(r.error ? { error: jsonb(r.error) } : {}),
|
||||
finished_at: now,
|
||||
})
|
||||
.where("operation_id", "=", opId)
|
||||
.execute();
|
||||
const op = await trx
|
||||
.selectFrom("asset_operation")
|
||||
.select("id")
|
||||
.where("operation_id", "=", opId)
|
||||
.executeTakeFirstOrThrow();
|
||||
await trx
|
||||
.updateTable("organization_file_result")
|
||||
.set({
|
||||
status: ok ? "succeeded" : "failed",
|
||||
asset_operation_id: op.id,
|
||||
checksum: r.checksum ?? null,
|
||||
error: r.error
|
||||
? jsonb(r.error)
|
||||
: r.status === "attention"
|
||||
? jsonb({ code: "source_not_removed", message: "目标已就绪,源文件未能删除" })
|
||||
: null,
|
||||
finished_at: now,
|
||||
})
|
||||
.where("id", "=", fr.id)
|
||||
.execute();
|
||||
allowCrossVolume: orgSettings.allowCrossVolumeMove,
|
||||
// 计划冻结的目标存在策略:replace = replace_with_verify 已获授权;其余目标此刻若已存在一律冲突
|
||||
onExisting: pf.action === "replace" ? "replace" : "conflict",
|
||||
replaceOld: { mode: fsr.replaceOld, trashDir: join(frozen.rootPath, ".umt-trash") },
|
||||
...(opts.abortSignal ? { signal: opts.abortSignal } : {}),
|
||||
});
|
||||
r.durationMs = Math.round(performance.now() - t0);
|
||||
r.attempt = info.attempt;
|
||||
// TIMEOUT/IO 类:未到上限 → 不入账本、抛出等任务重试;到上限 → 按失败入账并建待处理
|
||||
if (
|
||||
r.status === "failed" &&
|
||||
r.error &&
|
||||
fsErrorClass(r.error.code) === "retry" &&
|
||||
info.attempt < fsr.maxAttempts
|
||||
)
|
||||
throw new RetryableTransfer(r.error, info.attempt);
|
||||
// 打点:每个文件的耗时/字节/哈希方式,便于判断大文件入库是否拖慢 worker
|
||||
d.ctx.logger.info(
|
||||
{
|
||||
recordId,
|
||||
ordinal: fr.ordinal,
|
||||
kind,
|
||||
status: r.status,
|
||||
bytes: r.bytes ?? pf.size,
|
||||
durationMs: r.durationMs,
|
||||
mbps: r.durationMs > 0 ? Math.round(((r.bytes ?? pf.size) / 1e6 / r.durationMs) * 1000) : null,
|
||||
hashVia: r.hashVia ?? null,
|
||||
resumed: info.resumed,
|
||||
},
|
||||
"organize.transfer",
|
||||
);
|
||||
return r;
|
||||
},
|
||||
},
|
||||
);
|
||||
{
|
||||
...(opts.runId ? { runId: opts.runId } : {}),
|
||||
commit: async (trx, r) => {
|
||||
const now = d.ports.now();
|
||||
const ok = r.status !== "failed";
|
||||
await trx
|
||||
.updateTable("asset_operation")
|
||||
.set({
|
||||
status: ok ? "succeeded" : "failed",
|
||||
result: jsonb(r),
|
||||
...(r.error ? { error: jsonb(r.error) } : {}),
|
||||
finished_at: now,
|
||||
})
|
||||
.where("operation_id", "=", opId)
|
||||
.execute();
|
||||
const op = await trx
|
||||
.selectFrom("asset_operation")
|
||||
.select("id")
|
||||
.where("operation_id", "=", opId)
|
||||
.executeTakeFirstOrThrow();
|
||||
await trx
|
||||
.updateTable("organization_file_result")
|
||||
.set({
|
||||
status: ok ? "succeeded" : "failed",
|
||||
asset_operation_id: op.id,
|
||||
checksum: r.checksum ?? null,
|
||||
error: r.error
|
||||
? jsonb(r.error)
|
||||
: r.status === "attention"
|
||||
? jsonb({ code: "source_not_removed", message: "目标已就绪,源文件未能删除" })
|
||||
: null,
|
||||
finished_at: now,
|
||||
})
|
||||
.where("id", "=", fr.id)
|
||||
.execute();
|
||||
},
|
||||
},
|
||||
);
|
||||
} catch (e) {
|
||||
if (!(e instanceof RetryableTransfer)) throw e;
|
||||
deferred.push({ src: pf.src, code: e.error.code, message: e.error.message, attempt: e.attempt });
|
||||
await db
|
||||
.updateTable("asset_operation")
|
||||
.set({ error: jsonb({ ...e.error, attempt: e.attempt, retrying: true }) })
|
||||
.where("operation_id", "=", opId)
|
||||
.execute();
|
||||
continue;
|
||||
}
|
||||
if (outcome.status === "failed" && pf.role === "main") failedMains.add(pf.src);
|
||||
if (outcome.status === "failed" && outcome.error && fsErrorClass(outcome.error.code) === "retry") {
|
||||
await upsertDecision(d, db, {
|
||||
kind: "other",
|
||||
title: outcome.error.code === "TIMEOUT" ? "文件操作挂死,重试已达上限" : "文件读写失败,重试已达上限",
|
||||
summary: `${pf.src} → ${pf.dst}:${outcome.error.message}(已尝试 ${outcome.attempt ?? fsr.maxAttempts} 次;记录与下载保留,不重下)`,
|
||||
subjectType: "organization_record",
|
||||
subjectId: recordId,
|
||||
dedupeKey: `fs_retry_exhausted:${opId}`,
|
||||
evidence: { src: pf.src, dst: pf.dst, error: outcome.error, attempts: outcome.attempt ?? null },
|
||||
options: [{ key: "retry", label: "检查挂载后重新执行整理" }],
|
||||
});
|
||||
}
|
||||
if (outcome.replaced?.disposition === "left") {
|
||||
await upsertDecision(d, db, {
|
||||
kind: "other",
|
||||
title: "替换入库后旧文件未能处置",
|
||||
summary: `${pf.dst} 已替换为新版本;旧文件留在 ${outcome.replaced.path}`,
|
||||
subjectType: "organization_record",
|
||||
subjectId: recordId,
|
||||
dedupeKey: `replace_old_left:${opId}`,
|
||||
evidence: { replaced: outcome.replaced },
|
||||
});
|
||||
}
|
||||
if (outcome.status === "attention") {
|
||||
await upsertDecision(d, db, {
|
||||
kind: "other",
|
||||
@@ -806,74 +883,190 @@ export async function executeRecord(
|
||||
});
|
||||
}
|
||||
}
|
||||
if (deferred.length > 0) {
|
||||
// 交给任务引擎退避重试(executeRecord 幂等:已完成文件直接跳过,账本无结果的文件重跑并先对账)
|
||||
throw new Error(
|
||||
`文件操作可重试失败,等待重试:${deferred.map((x) => `${x.code}@${x.src}(第 ${x.attempt} 次)`).join(";")}`,
|
||||
);
|
||||
}
|
||||
return finalizeRecord(d, recordId);
|
||||
}
|
||||
|
||||
/** TIMEOUT/IO 类失败且未到尝试上限:从 withAttemptToken 内抛出,账本不落结果 */
|
||||
class RetryableTransfer extends Error {
|
||||
constructor(
|
||||
readonly error: { code: string; message: string },
|
||||
readonly attempt: number,
|
||||
) {
|
||||
super(`${error.code}: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
interface FsRuntime {
|
||||
proxy: FsProxy;
|
||||
partialMaxAgeMs: number;
|
||||
maxAttempts: number;
|
||||
replaceOld: ReplaceOldPolicy["mode"];
|
||||
}
|
||||
const fsRuntimes = new WeakMap<object, FsRuntime>();
|
||||
|
||||
/** 按 server 配置(FS_*)建一次 fsproxy,随 ctx.config 缓存 */
|
||||
export function fsRuntime(d: AoDeps): FsRuntime {
|
||||
const c = (d.ctx.config ?? {}) as Partial<import("../../../config.ts").Config>;
|
||||
const hit = fsRuntimes.get(c);
|
||||
if (hit) return hit;
|
||||
const rt: FsRuntime = {
|
||||
proxy: new FsProxy({
|
||||
mode: c.FS_PROXY_MODE ?? "auto",
|
||||
hangTimeoutMs: c.FS_HANG_TIMEOUT_MS ?? 120_000,
|
||||
quickTimeoutMs: c.FS_QUICK_TIMEOUT_MS ?? 30_000,
|
||||
progressIntervalMs: c.FS_PROGRESS_INTERVAL_MS ?? 1000,
|
||||
killGraceMs: c.FS_KILL_GRACE_MS ?? 2000,
|
||||
}),
|
||||
partialMaxAgeMs: c.FS_PARTIAL_MAX_AGE_MS ?? 24 * 3600_000,
|
||||
maxAttempts: c.FS_TRANSFER_MAX_ATTEMPTS ?? 3,
|
||||
replaceOld: c.FS_REPLACE_OLD ?? "trash",
|
||||
};
|
||||
fsRuntimes.set(c, rt);
|
||||
return rt;
|
||||
}
|
||||
|
||||
/** statSafe 结果 → 失败码(失败 ≠ 不存在:只有 missing 才是 SOURCE_MISSING) */
|
||||
function statFailure(st: StatResult, path: string, role: "源" | "目标"): { code: string; message: string } | null {
|
||||
switch (st.kind) {
|
||||
case "ok":
|
||||
return null;
|
||||
case "missing":
|
||||
return role === "源" ? { code: "SOURCE_MISSING", message: `源文件不存在: ${path}` } : null;
|
||||
case "permission_denied":
|
||||
return { code: "EACCES", message: `${role}无权限访问: ${path}` };
|
||||
case "root_offline":
|
||||
return { code: "ROOT_OFFLINE", message: `${role}所在根离线(${st.reason}): ${path}` };
|
||||
case "error":
|
||||
return {
|
||||
code: st.code === "TIMEOUT" ? "TIMEOUT" : "IO",
|
||||
message: `${role}状态读取失败(${st.code}): ${st.message}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function transferFile(
|
||||
kind: "copy" | "move",
|
||||
src: string,
|
||||
dst: string,
|
||||
size: number,
|
||||
crossDevice: boolean | undefined,
|
||||
resumed: boolean,
|
||||
d: AoDeps,
|
||||
allowCrossVolume: boolean,
|
||||
signal?: AbortSignal,
|
||||
fsr: FsRuntime,
|
||||
x: {
|
||||
kind: "copy" | "move";
|
||||
src: string;
|
||||
dst: string;
|
||||
size: number;
|
||||
crossDevice: boolean | undefined;
|
||||
resumed: boolean;
|
||||
allowCrossVolume: boolean;
|
||||
onExisting: "conflict" | "replace";
|
||||
replaceOld: ReplaceOldPolicy;
|
||||
signal?: AbortSignal;
|
||||
},
|
||||
): Promise<FileOutcome> {
|
||||
const { kind, src, dst, size, crossDevice, resumed } = x;
|
||||
const px = fsr.proxy;
|
||||
const call = x.signal ? { signal: x.signal } : {};
|
||||
const fsErr = (e: unknown) => ({ code: e instanceof FsError ? e.code : "IO", message: errMessage(e) });
|
||||
const [s, t] = await Promise.all([statSafe(src), statSafe(dst)]);
|
||||
// 对账:上次 attempt 已完成外部动作但没留下结果
|
||||
if (t.kind === "ok") {
|
||||
if (s.kind === "missing" && kind === "move") {
|
||||
return t.facts.size === size
|
||||
? { status: "succeeded", bytes: size, resumed: true }
|
||||
: { status: "failed", error: { code: "CONFLICT", message: "源已不存在且目标大小不符" } };
|
||||
}
|
||||
if (s.kind === "ok") {
|
||||
const same = await filesMatchSha256(src, dst, signal).catch(() => ({ match: false, via: null }));
|
||||
if (!same.match) return { status: "failed", error: { code: "EEXIST", message: `目标已存在且内容不同: ${dst}` } };
|
||||
if (kind === "copy")
|
||||
return { status: "succeeded", bytes: size, resumed, ...(same.via ? { hashVia: same.via } : {}) };
|
||||
if (crossDevice === true || resumed) {
|
||||
const r = await crossDeviceMove(src, dst, { hash: "sha256" });
|
||||
return {
|
||||
status: r.status === "completed" ? "succeeded" : r.status,
|
||||
bytes: size,
|
||||
steps: stepsOf(r.steps),
|
||||
resumed: true,
|
||||
};
|
||||
const copyReceipt = (r: CopyReceipt): FileOutcome => ({
|
||||
status: "succeeded",
|
||||
bytes: r.bytes,
|
||||
...(r.sha256Via ? { hashVia: r.sha256Via } : {}),
|
||||
...(r.sha256 ? { checksum: `sha256:${r.sha256}` } : r.sampleDigest ? { checksum: `sample:${r.sampleDigest}` } : {}),
|
||||
...(r.replaced ? { replaced: r.replaced } : {}),
|
||||
...(r.sweptPartials?.length ? { sweptPartials: r.sweptPartials } : {}),
|
||||
});
|
||||
const copyOpts = {
|
||||
hash: d.ports.options.verifyHash,
|
||||
onExisting: x.onExisting,
|
||||
replaceOld: x.replaceOld,
|
||||
sweepPartialsOlderThanMs: fsr.partialMaxAgeMs > 0 ? fsr.partialMaxAgeMs : null,
|
||||
};
|
||||
try {
|
||||
const [s, t] = await Promise.all([px.statSafe(src, {}, call), px.statSafe(dst, {}, call)]);
|
||||
// 目标状态读不出(挂死/IO/无权限)≠ 目标不存在:不能据此往下写
|
||||
const tFail = statFailure(t, dst, "目标");
|
||||
if (tFail) return { status: "failed", error: tFail };
|
||||
// 对账:上次 attempt 已完成外部动作但没留下结果
|
||||
if (t.kind === "ok") {
|
||||
if (s.kind === "missing" && kind === "move") {
|
||||
return t.facts.size === size
|
||||
? { status: "succeeded", bytes: size, resumed: true }
|
||||
: { status: "failed", error: { code: "CONFLICT", message: "源已不存在且目标大小不符" } };
|
||||
}
|
||||
if (s.kind === "ok") {
|
||||
// 比对读失败/超时 → 按 IO/TIMEOUT 失败并留证据,不得冒充"内容不同"(EEXIST)
|
||||
let same: { match: boolean; via: string | null };
|
||||
try {
|
||||
same = await px.filesMatchSha256(src, dst, call);
|
||||
} catch (e) {
|
||||
const err = fsErr(e);
|
||||
return {
|
||||
status: "failed",
|
||||
error: { ...err, message: `对账比对失败:${err.message}` },
|
||||
evidence: { phase: "reconcile", src, dst },
|
||||
};
|
||||
}
|
||||
if (same.match) {
|
||||
if (kind === "copy")
|
||||
return { status: "succeeded", bytes: size, resumed, ...(same.via ? { hashVia: same.via } : {}) };
|
||||
if (crossDevice === true || resumed || x.onExisting === "replace") {
|
||||
const r = await px.crossDeviceMove(src, dst, { hash: "sha256" }, call);
|
||||
return {
|
||||
status: r.status === "completed" ? "succeeded" : r.status,
|
||||
bytes: size,
|
||||
steps: stepsOf(r.steps),
|
||||
resumed: true,
|
||||
};
|
||||
}
|
||||
} else if (x.onExisting !== "replace")
|
||||
return { status: "failed", error: { code: "EEXIST", message: `目标已存在且内容不同: ${dst}` } };
|
||||
}
|
||||
}
|
||||
}
|
||||
if (s.kind !== "ok")
|
||||
return { status: "failed", error: { code: "SOURCE_MISSING", message: `源文件不可访问(${s.kind}): ${src}` } };
|
||||
try {
|
||||
if (kind === "copy") {
|
||||
const r = await copyWithVerify(src, dst, { hash: d.ports.options.verifyHash, ...(signal ? { signal } : {}) });
|
||||
return {
|
||||
status: "succeeded",
|
||||
bytes: r.bytes,
|
||||
...(r.sha256Via ? { hashVia: r.sha256Via } : {}),
|
||||
...(r.sha256
|
||||
? { checksum: `sha256:${r.sha256}` }
|
||||
: r.sampleDigest
|
||||
? { checksum: `sample:${r.sampleDigest}` }
|
||||
: {}),
|
||||
};
|
||||
const sFail = statFailure(s, src, "源");
|
||||
if (sFail) return { status: "failed", error: sFail };
|
||||
|
||||
if (x.onExisting === "replace" && t.kind === "ok") {
|
||||
// 替换:新文件写临时 → 校验 → 原子替换,旧文件按 FS_REPLACE_OLD 进回收目录/删除;move 再删源
|
||||
const r = await px.copy(src, dst, copyOpts, call);
|
||||
const out = copyReceipt(r);
|
||||
if (kind === "move") {
|
||||
const del = await px.run({ op: "unlink", path: src }, call).then(
|
||||
() => null,
|
||||
(e) => fsErr(e),
|
||||
);
|
||||
if (del)
|
||||
return {
|
||||
...out,
|
||||
status: "attention",
|
||||
steps: [{ step: "delete_source", status: "failed", at: new Date().toISOString(), error: del }],
|
||||
};
|
||||
}
|
||||
return out;
|
||||
}
|
||||
if (kind === "copy") return copyReceipt(await px.copy(src, dst, copyOpts, call));
|
||||
if (crossDevice !== true) {
|
||||
const m = await moveSafe(src, dst);
|
||||
const m = await px.moveSafe(src, dst, call);
|
||||
if (m.status === "moved") return { status: "succeeded", bytes: size };
|
||||
if (m.status === "exists") return { status: "failed", error: { code: "EEXIST", message: `目标已存在: ${dst}` } };
|
||||
// EXDEV:不静默降级;按设置显式走可恢复三步
|
||||
}
|
||||
if (!allowCrossVolume)
|
||||
if (!x.allowCrossVolume)
|
||||
return {
|
||||
status: "failed",
|
||||
error: { code: "EXDEV", message: "跨卷 Move 未启用(settings.organize.allowCrossVolumeMove)" },
|
||||
};
|
||||
const r = await crossDeviceMove(src, dst, { hash: d.ports.options.verifyHash, ...(signal ? { signal } : {}) });
|
||||
return { status: r.status === "completed" ? "succeeded" : r.status, bytes: size, steps: stepsOf(r.steps) };
|
||||
const r = await px.crossDeviceMove(src, dst, { hash: d.ports.options.verifyHash }, call);
|
||||
const failedStep = r.steps.find((st) => st.status === "failed");
|
||||
return {
|
||||
status: r.status === "completed" ? "succeeded" : r.status,
|
||||
bytes: size,
|
||||
steps: stepsOf(r.steps),
|
||||
// 跨卷失败带上首个失败步骤的错误码(TIMEOUT/IO 走重试,EEXIST/VERIFY_FAILED 需人工)
|
||||
...(r.status === "failed" && failedStep?.error ? { error: failedStep.error } : {}),
|
||||
};
|
||||
} catch (e) {
|
||||
return { status: "failed", error: fsErr(e) };
|
||||
}
|
||||
@@ -895,10 +1088,19 @@ export async function finalizeRecord(d: AoDeps, recordId: string): Promise<Execu
|
||||
.selectAll()
|
||||
.where("organization_record_id", "=", recordId)
|
||||
.execute();
|
||||
// 同位置其它版本(MP latest 口径:同季同集同 Part、只看视频扩展名):升级时作为旧版本处置候选,不删
|
||||
const otherVersions: string[] = [];
|
||||
for (const f of filesPre) {
|
||||
if (f.status === "succeeded" && f.role === "main" && f.target_path) {
|
||||
const r = await d.ports.probe(join(frozenPre.rootPath, f.target_path));
|
||||
const abs = join(frozenPre.rootPath, f.target_path);
|
||||
const r = await d.ports.probe(abs);
|
||||
probes.set(f.id, r);
|
||||
const names = await readdir(dirname(abs)).catch(() => [] as string[]);
|
||||
for (const o of findOtherVersions(
|
||||
abs,
|
||||
names.map((n) => join(dirname(abs), n)),
|
||||
))
|
||||
otherVersions.push(relative(frozenPre.rootPath, o));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1144,7 +1346,12 @@ export async function finalizeRecord(d: AoDeps, recordId: string): Promise<Execu
|
||||
subjectType: "media_asset",
|
||||
subjectId: old,
|
||||
dedupeKey: `old_version:${old}`,
|
||||
evidence: { slotId: slot.id, oldAssetId: old, newAssetId: mainAssets[0] },
|
||||
evidence: {
|
||||
slotId: slot.id,
|
||||
oldAssetId: old,
|
||||
newAssetId: mainAssets[0],
|
||||
...(otherVersions.length ? { otherVersionFiles: otherVersions } : {}),
|
||||
},
|
||||
});
|
||||
}
|
||||
if (complete && slot.subscription_id) {
|
||||
|
||||
Reference in new issue
Block a user