feat(fs): W4-E4 fsproxy hang isolation, partial sweep, replace-with-verify, TIMEOUT retry
- adapters/fs: FsProxy + fsworker run each file op in a killable subprocess;
progress-stall watchdog (FS_HANG_TIMEOUT_MS) for copy/hash, total timeout for
quick ops; SIGTERM -> SIGKILL -> FsError('TIMEOUT'); stat/checkRoot timeouts
never read as missing
- copyWithVerify: onExisting conflict/skip/replace (hardlink backup + atomic
rename, old file to .umt-trash or deleted), sweep stale partials before write,
document why permissions/owner are not copied
- filesMatch/filesMatchSha256 throw FsError on read failure instead of
match=false; crossDeviceMove skips redundant re-hash after sha256 copy
- FsErrorCode TIMEOUT + fsErrorClass; organize executeRecord routes fs ops
through fsproxy, retries TIMEOUT/IO up to FS_TRANSFER_MAX_ATTEMPTS then opens
a decision without deleting the record; reconcile errors no longer reported
as EEXIST
- domain: subtitle .default marker (ORGANIZE_DEFAULT_SUBTITLE_LANG) and
findOtherVersions (same season/episode/part, video only) as upgrade
disposal candidates
- scripts/bench/fs-bench.ts and benchmark record in porting doc
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
36dc048e89
commit
2e2d56184c
21 files changed
+1970
-179
No files matched your search
@@ -0,0 +1,124 @@
|
||||
// 文件操作基准(不进 ci):跨卷 move 1 GB、全量哈希 1 GB、10 万文件扫描;输出耗时与 MB/s。
|
||||
// 用法:deno run -A scripts/bench/fs-bench.ts --base /opt/tmp-bench --other /dev/shm [--size-mb 1024] [--files 100000]
|
||||
// base:被测盘(源文件、扫描树所在);other:跨卷 move 的目标(必须与 base 不同设备)
|
||||
// 结果写进 docs/porting/moviepilot-rust-transfer.md「基准记录」。
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { mkdir, mkdtemp, rm, stat, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { parseArgs } from "node:util";
|
||||
import {
|
||||
crossDeviceMove,
|
||||
FsProxy,
|
||||
resetSha256Probe,
|
||||
scanIncremental,
|
||||
sha256File,
|
||||
sha256Full,
|
||||
writeRootSentinel,
|
||||
} from "../../packages/adapters/src/index.ts";
|
||||
|
||||
const { values } = parseArgs({
|
||||
args: Deno.args,
|
||||
options: {
|
||||
base: { type: "string" },
|
||||
other: { type: "string", default: "/dev/shm" },
|
||||
"size-mb": { type: "string", default: "1024" },
|
||||
files: { type: "string", default: "100000" },
|
||||
},
|
||||
});
|
||||
if (!values.base) {
|
||||
console.error("需要 --base <被测目录>");
|
||||
Deno.exit(2);
|
||||
}
|
||||
const MB = 1024 * 1024;
|
||||
const sizeMb = Number(values["size-mb"]);
|
||||
const nFiles = Number(values.files);
|
||||
const work = await mkdtemp(join(values.base, "umt-bench-"));
|
||||
const other = await mkdtemp(join(values.other!, "umt-bench-x-"));
|
||||
if ((await stat(work)).dev === (await stat(other)).dev) console.warn("警告:base 与 other 在同一设备,move 不跨卷");
|
||||
|
||||
const rows: { case: string; ms: number; mbps: number | null; note?: string }[] = [];
|
||||
async function timed(name: string, bytes: number | null, fn: () => Promise<unknown>, note?: string) {
|
||||
const t0 = performance.now();
|
||||
await fn();
|
||||
const ms = performance.now() - t0;
|
||||
rows.push({
|
||||
case: name,
|
||||
ms: Math.round(ms),
|
||||
mbps: bytes ? Math.round(bytes / MB / (ms / 1000)) : null,
|
||||
...(note ? { note } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
async function makeBig(p: string) {
|
||||
const f = await Deno.open(p, { write: true, create: true, truncate: true });
|
||||
const chunk = randomBytes(8 * MB);
|
||||
for (let i = 0; i < sizeMb / 8; i++) await f.write(chunk);
|
||||
f.close();
|
||||
}
|
||||
|
||||
const bytes = sizeMb * MB;
|
||||
const big = join(work, "big.bin");
|
||||
try {
|
||||
await makeBig(big);
|
||||
// 冷缓存不可控(tmpfs 恒热;nvme 上刚写完也在页缓存里),数字代表热缓存吞吐
|
||||
resetSha256Probe();
|
||||
await timed("hash sha256sum 子进程", bytes, () => sha256Full(big));
|
||||
await timed("hash 进程内流式", bytes, () => sha256File(big));
|
||||
const px = new FsProxy({ mode: "subprocess" });
|
||||
await timed("hash fsproxy 子进程(流式+心跳)", bytes, () => px.sha256(big));
|
||||
|
||||
for (const hash of ["size", "sha256"] as const) {
|
||||
const src = join(work, `mv-${hash}.bin`);
|
||||
await makeBig(src);
|
||||
await timed(`跨卷 move 进程内 hash=${hash}`, bytes, () =>
|
||||
crossDeviceMove(src, join(other, `mv-${hash}.bin`), { hash }),
|
||||
);
|
||||
const src2 = join(work, `mvp-${hash}.bin`);
|
||||
await makeBig(src2);
|
||||
let beats = 0;
|
||||
await timed(`跨卷 move fsproxy hash=${hash}`, bytes, () =>
|
||||
px.crossDeviceMove(src2, join(other, `mvp-${hash}.bin`), { hash }, { onProgress: () => beats++ }),
|
||||
);
|
||||
rows.at(-1)!.note = `心跳 ${beats} 次`;
|
||||
}
|
||||
const cp = join(work, "cp.bin");
|
||||
await makeBig(cp);
|
||||
await timed("对照:cp 跨卷", bytes, async () => {
|
||||
const r = await new Deno.Command("cp", { args: [cp, join(other, "cp.bin")] }).output();
|
||||
if (!r.success) throw new Error("cp failed");
|
||||
});
|
||||
|
||||
// 10 万文件扫描:100 个目录 × 1000 个小文件
|
||||
const tree = join(work, "scan");
|
||||
await mkdir(tree);
|
||||
await writeRootSentinel(tree);
|
||||
const perDir = 1000;
|
||||
const t0 = performance.now();
|
||||
for (let d = 0; d < Math.ceil(nFiles / perDir); d++) {
|
||||
const dir = join(tree, `d${String(d).padStart(3, "0")}`);
|
||||
await mkdir(dir);
|
||||
await Promise.all(
|
||||
Array.from({ length: Math.min(perDir, nFiles - d * perDir) }, (_, i) => writeFile(join(dir, `f${i}.mkv`), "x")),
|
||||
);
|
||||
}
|
||||
const mkMs = Math.round(performance.now() - t0);
|
||||
let entries = 0;
|
||||
let batches = 0;
|
||||
await timed(`扫描 ${nFiles} 文件(scanIncremental,maxStats=2000 分批)`, null, async () => {
|
||||
let cursor: string | null = null;
|
||||
do {
|
||||
const r = await scanIncremental(tree, { cursor, maxStats: 2000 });
|
||||
entries += r.entries.length;
|
||||
batches++;
|
||||
cursor = r.cursor;
|
||||
} while (cursor);
|
||||
});
|
||||
rows.at(-1)!.note = `条目 ${entries},批 ${batches},建树 ${mkMs} ms`;
|
||||
await timed(`扫描 ${nFiles} 文件(单批)`, null, () => scanIncremental(tree, { maxStats: nFiles + 10 }));
|
||||
} finally {
|
||||
await rm(work, { recursive: true, force: true });
|
||||
await rm(other, { recursive: true, force: true });
|
||||
}
|
||||
console.log(`base=${values.base} other=${values.other} size=${sizeMb} MiB files=${nFiles}`);
|
||||
console.table(rows);
|
||||
console.log(JSON.stringify(rows));
|
||||
Reference in new issue
Block a user