perf(fs): 1MiB stream chunks, full sha256 via sha256sum subprocess, walk cap
- copy streams use 1MiB highWaterMark - forced full sha256 uses Deno.Command(sha256sum), falls back to in-process stream; method recorded as hashVia and counted - per-file organize.transfer timing log - ORGANIZE_WALK_MAX_FILES caps manual path walks; dryRun returns truncated Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
473c5e94a5
commit
f9c20c67dc
6 files changed
+223
-27
No files matched your search
@@ -30,6 +30,8 @@ export interface CopyReceipt {
|
||||
bytes: number;
|
||||
verifiedBy: VerifyMethod;
|
||||
sha256?: string;
|
||||
/** sha256 校验时临时文件的全量哈希方式 */
|
||||
sha256Via?: Sha256Via;
|
||||
sampleDigest?: string;
|
||||
partialPath: string;
|
||||
durationMs: number;
|
||||
@@ -48,9 +50,13 @@ async function fsyncPath(p: string, flags = "r"): Promise<void> {
|
||||
}
|
||||
}
|
||||
|
||||
/** 复制/哈希的流块大小:1 MiB(缺省 64 KiB 在大文件上每 GB 要多上万次回调) */
|
||||
export const IO_CHUNK_BYTES = 1024 * 1024;
|
||||
|
||||
/** 进程内流式 sha256(每块之间让出事件循环) */
|
||||
export async function sha256File(p: string, signal?: AbortSignal): Promise<string> {
|
||||
const h = createHash("sha256");
|
||||
await pipeline(createReadStream(p), async (src: AsyncIterable<Buffer>) => {
|
||||
await pipeline(createReadStream(p, { highWaterMark: IO_CHUNK_BYTES }), async (src: AsyncIterable<Buffer>) => {
|
||||
for await (const c of src) {
|
||||
if (signal?.aborted) throw new FsError("ABORTED", "aborted", { path: p });
|
||||
h.update(c);
|
||||
@@ -59,6 +65,85 @@ export async function sha256File(p: string, signal?: AbortSignal): Promise<strin
|
||||
return h.digest("hex");
|
||||
}
|
||||
|
||||
/** 全量 sha256 用的方式:外部 sha256sum 子进程,或进程内流式(回退) */
|
||||
export type Sha256Via = "sha256sum" | "stream";
|
||||
|
||||
export interface Sha256Result {
|
||||
digest: string;
|
||||
via: Sha256Via;
|
||||
}
|
||||
|
||||
const hashCounters: Record<Sha256Via, number> = { sha256sum: 0, stream: 0 };
|
||||
/** 子进程不可用(未安装 / 没有 --allow-run)后不再尝试,直到进程重启 */
|
||||
let sha256sumUnavailable: string | null = null;
|
||||
|
||||
/** 本进程全量哈希方式计数与子进程不可用原因(诊断 / 指标用) */
|
||||
export function sha256Stats(): { counts: Record<Sha256Via, number>; subprocessUnavailable: string | null } {
|
||||
return { counts: { ...hashCounters }, subprocessUnavailable: sha256sumUnavailable };
|
||||
}
|
||||
|
||||
type DenoLike = {
|
||||
Command: new (
|
||||
cmd: string,
|
||||
opts: { args: string[]; stdout: "piped"; stderr: "piped"; signal?: AbortSignal },
|
||||
) => { output(): Promise<{ success: boolean; code: number; stdout: Uint8Array; stderr: Uint8Array }> };
|
||||
errors: { NotFound: new () => Error; NotCapable?: new () => Error; PermissionDenied: new () => Error };
|
||||
};
|
||||
|
||||
async function sha256Subprocess(p: string, signal?: AbortSignal): Promise<string | null> {
|
||||
const D = (globalThis as { Deno?: DenoLike }).Deno;
|
||||
if (!D?.Command) {
|
||||
sha256sumUnavailable = "no_deno_command";
|
||||
return null;
|
||||
}
|
||||
let out: Awaited<ReturnType<InstanceType<DenoLike["Command"]>["output"]>>;
|
||||
try {
|
||||
out = await new D.Command("sha256sum", {
|
||||
args: ["-b", "--", p],
|
||||
stdout: "piped",
|
||||
stderr: "piped",
|
||||
...(signal ? { signal } : {}),
|
||||
}).output();
|
||||
} catch (e) {
|
||||
if (signal?.aborted) throw new FsError("ABORTED", "aborted", { path: p, cause: e });
|
||||
const notCapable = D.errors.NotCapable && e instanceof D.errors.NotCapable;
|
||||
if (e instanceof D.errors.NotFound || e instanceof D.errors.PermissionDenied || notCapable) {
|
||||
sha256sumUnavailable = e instanceof D.errors.NotFound ? "not_installed" : "not_permitted";
|
||||
return null;
|
||||
}
|
||||
throw toFsError(e, p);
|
||||
}
|
||||
if (!out.success) {
|
||||
const msg = new TextDecoder().decode(out.stderr).trim().slice(0, 300);
|
||||
throw new FsError("IO", `sha256sum 失败(${out.code}): ${msg}`, { path: p });
|
||||
}
|
||||
const digest = new TextDecoder().decode(out.stdout).trim().split(/\s+/)[0] ?? "";
|
||||
if (!/^[0-9a-f]{64}$/.test(digest)) throw new FsError("IO", `sha256sum 输出无法解析`, { path: p });
|
||||
return digest;
|
||||
}
|
||||
|
||||
/**
|
||||
* 全量 sha256:优先外部 `sha256sum` 子进程(读盘 + 哈希完全不占本进程事件循环),
|
||||
* 不可用时回退进程内流式哈希。返回所用方式,调用方记进操作结果。
|
||||
*/
|
||||
export async function sha256Full(p: string, signal?: AbortSignal): Promise<Sha256Result> {
|
||||
if (!sha256sumUnavailable) {
|
||||
const digest = await sha256Subprocess(p, signal);
|
||||
if (digest) {
|
||||
hashCounters.sha256sum += 1;
|
||||
return { digest, via: "sha256sum" };
|
||||
}
|
||||
}
|
||||
const digest = await sha256File(p, signal);
|
||||
hashCounters.stream += 1;
|
||||
return { digest, via: "stream" };
|
||||
}
|
||||
|
||||
/** 测试用:清掉"子进程不可用"缓存 */
|
||||
export function resetSha256Probe(): void {
|
||||
sha256sumUnavailable = null;
|
||||
}
|
||||
|
||||
/** 抽样摘要:首/尾/等距 N 块 + 文件大小。抽样一致不等于完整一致性证明(REQ §8.2)。 */
|
||||
export async function sampleDigest(p: string, chunks = 8, chunkBytes = 1024 * 1024): Promise<string> {
|
||||
const fh = await open(p, "r");
|
||||
@@ -141,9 +226,9 @@ export async function copyWithVerify(src: string, dst: string, opts: CopyOptions
|
||||
},
|
||||
});
|
||||
await pipeline(
|
||||
createReadStream(src),
|
||||
createReadStream(src, { highWaterMark: IO_CHUNK_BYTES }),
|
||||
meter,
|
||||
createWriteStream(tmp, { flags: "wx", flush: true }),
|
||||
createWriteStream(tmp, { flags: "wx", flush: true, highWaterMark: IO_CHUNK_BYTES }),
|
||||
opts.signal ? { signal: opts.signal } : {},
|
||||
);
|
||||
await fsyncPath(tmp, "r+");
|
||||
@@ -166,9 +251,11 @@ export async function copyWithVerify(src: string, dst: string, opts: CopyOptions
|
||||
};
|
||||
if (hasher) {
|
||||
const expected = hasher.digest("hex");
|
||||
const actual = await sha256File(tmp, opts.signal);
|
||||
if (actual !== expected) throw new FsError("VERIFY_FAILED", "sha256 mismatch", { path: dst });
|
||||
receipt.sha256 = actual;
|
||||
// 回读临时文件做全量比对:走子进程,不占本进程事件循环
|
||||
const actual = await sha256Full(tmp, opts.signal);
|
||||
if (actual.digest !== expected) throw new FsError("VERIFY_FAILED", "sha256 mismatch", { path: dst });
|
||||
receipt.sha256 = actual.digest;
|
||||
receipt.sha256Via = actual.via;
|
||||
} else if (method === "sample") {
|
||||
const a = await sampleDigest(src, opts.sampleChunks, opts.sampleChunkBytes);
|
||||
const b = await sampleDigest(tmp, opts.sampleChunks, opts.sampleChunkBytes);
|
||||
@@ -187,11 +274,23 @@ export async function copyWithVerify(src: string, dst: string, opts: CopyOptions
|
||||
}
|
||||
}
|
||||
|
||||
/** 两个文件全量 sha256 比对(大小先比);两边并行,各走 sha256Full。返回是否一致与所用方式 */
|
||||
export async function filesMatchSha256(
|
||||
a: string,
|
||||
b: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<{ match: boolean; via: Sha256Via | null }> {
|
||||
const [sa, sb] = await Promise.all([stat(a), stat(b)]);
|
||||
if (sa.size !== sb.size) return { match: false, via: null };
|
||||
const [ha, hb] = await Promise.all([sha256Full(a, signal), sha256Full(b, signal)]);
|
||||
return { match: ha.digest === hb.digest, via: ha.via === hb.via ? ha.via : "stream" };
|
||||
}
|
||||
|
||||
/** 按指定方式比对两个文件是否一致(size 恒比对)。 */
|
||||
export async function filesMatch(a: string, b: string, method: VerifyMethod = "size"): Promise<boolean> {
|
||||
if (method === "sha256") return (await filesMatchSha256(a, b)).match;
|
||||
const [sa, sb] = await Promise.all([stat(a), stat(b)]);
|
||||
if (sa.size !== sb.size) return false;
|
||||
if (method === "sha256") return (await sha256File(a)) === (await sha256File(b));
|
||||
if (method === "sample") return (await sampleDigest(a)) === (await sampleDigest(b));
|
||||
return true;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user