pixel_pop/UHD_Media_Tool
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(fs): 1MiB stream chunks, full sha256 via sha256sum subprocess, walk cap

Browse Source 
- copy streams use 1MiB highWaterMark
- forced full sha256 uses Deno.Command(sha256sum), falls back to in-process stream;
  method recorded as hashVia and counted
- per-file organize.transfer timing log
- ORGANIZE_WALK_MAX_FILES caps manual path walks; dryRun returns truncated

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed 2026-09-27 22:12:16 +08:00
1 parent 473c5e94a5
commit f9c20c67dc
6 files changed
+223 -27
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

apps/server/src/modules/acquisition-organize/routes/organization.ts
+9 -3
View File
@@ -137,15 +137,21 @@ export function registerOrganizationRoutes(app: ModuleApp, d: AoDeps) {
task_id: taskId,
subject_type: "organize_request",
title: "organize_plan",
content: jsonb({ request: body, plan: plan.plan ?? null, reason: plan.reason, status: plan.status }),
content: jsonb({
request: body,
plan: plan.plan ?? null,
reason: plan.reason,
status: plan.status,
truncated: plan.truncated === true,
}),
content_hash: randomUUID(),
})
.returning("id")
.executeTakeFirstOrThrow();
await markTask(db(), taskId, "succeeded", {
output: { planId: art.id, status: plan.status, reason: plan.reason },
output: { planId: art.id, status: plan.status, reason: plan.reason, truncated: plan.truncated === true },
});
return { recordId: null, taskId, planId: art.id };
return { recordId: null, taskId, planId: art.id, truncated: plan.truncated === true };
}
if (plan.status === "running" || plan.status === "queued") {
const { runId } = await d.ctx.engine.enqueue(
apps/server/src/modules/acquisition-organize/services/organize.ts
+58 -14
View File
@@ -7,9 +7,9 @@ import {
crossDeviceMove,
DEFAULT_ROOT_SENTINEL,
FsError,
filesMatch,
filesMatchSha256,
moveSafe,
sha256File,
sha256Full,
statSafe,
} from "@umt/adapters";
import type { ActorType, OrganizationRecord, OrganizationTrigger, PathBinding } from "@umt/contracts";
@@ -81,6 +81,8 @@ export interface FrozenPlan {
export interface PlanResult {
/** dryRun 时为 null */
recordId: string | null;
/** 按路径整理时源目录文件数超过上限(walkMaxFiles),计划只含前 N 个文件(仅 dryRun 会出现) */
truncated?: boolean;
status: OrganizationRecord["status"];
reason: string | null;
plan?: FrozenPlan;
@@ -108,7 +110,8 @@ function segmentsWithin(path: string, root: string | null) {
return { file, ...(parent !== undefined ? { parent } : {}), ...(grandParent !== undefined ? { grandParent } : {}) };
}
async function walk(p: string, out: string[], limit = 5000): Promise<void> {
/** 递归列出文件(跳过隐藏项),最多 limit 个;调用方传 上限+1 判断是否截断 */
async function walk(p: string, out: string[], limit: number): Promise<void> {
const st = await statSafe(p);
if (st.kind !== "ok") throw new FsError("ENOENT", `源路径不可访问: ${p} (${st.kind})`, { path: p });
if (st.facts.isFile) {
@@ -172,6 +175,7 @@ export async function planImport(d: AoDeps, req: ImportRequest): Promise<PlanRes
downloadTaskId: dt?.id ?? null,
};
const dry = req.dryRun === true;
let truncated = false;
const decide_ = (input: Parameters<typeof upsertDecision>[2]) =>
dry ? Promise.resolve("") : upsertDecision(d, db, input);
const fail = async (
@@ -180,7 +184,7 @@ export async function planImport(d: AoDeps, req: ImportRequest): Promise<PlanRes
entityId: string | null = null,
): Promise<PlanResult> =>
dry
? { recordId: null, status: "failed", reason }
? { recordId: null, status: "failed", reason, ...(truncated ? { truncated } : {}) }
: writeFailedRecord(d, {
...base,
action: req.action ?? "copy",
@@ -228,11 +232,19 @@ export async function planImport(d: AoDeps, req: ImportRequest): Promise<PlanRes
}
} else if (req.sourcePaths?.length) {
const paths: string[] = [];
const max = d.ports.options.walkMaxFiles;
try {
for (const p of req.sourcePaths) await walk(p, paths);
// 多走一个用来判断是否超限
for (const p of req.sourcePaths) await walk(p, paths, max + 1);
} catch (err) {
return fail("source_missing", { message: errMessage(err) });
}
if (paths.length > max) {
paths.length = max;
truncated = true;
// 正式执行不接受截断的计划(会漏整理文件);dryRun 返回前 N 个文件的计划 + truncated 标记
if (!dry) return fail("too_many_files", { limit: max });
}
for (const p of paths) {
const st = await statSafe(p);
if (st.kind === "ok") files.push({ path: p, size: st.facts.size, device: st.facts.dev });
@@ -431,9 +443,10 @@ export async function planImport(d: AoDeps, req: ImportRequest): Promise<PlanRes
}
const e: ExistingTarget = { path: f.dst, size: st.facts.size };
if (st.facts.size === f.size) {
e.sha256 = await sha256File(f.dst);
// 同尺寸已存在的目标:全量 sha256 判定是否同一文件(子进程,见 adapters sha256Full)
e.sha256 = (await sha256Full(f.dst)).digest;
const src = files.find((x) => x.path === f.src);
if (src) src.sha256 = await sha256File(src.path);
if (src) src.sha256 = (await sha256Full(src.path)).digest;
}
existing.push(e);
}
@@ -461,7 +474,14 @@ export async function planImport(d: AoDeps, req: ImportRequest): Promise<PlanRes
: null;
const now = d.ports.now();
const targetRel = (abs: string | null | undefined) => (abs ? relative(root.local_path, abs) : null);
if (dry) return { recordId: null, status: blocking.length ? "failed" : "queued", reason, plan: frozen };
if (dry)
return {
recordId: null,
status: blocking.length ? "failed" : "queued",
reason,
plan: frozen,
...(truncated ? { truncated } : {}),
};
const result = await db.transaction().execute(async (trx) => {
const rec = await trx
@@ -609,6 +629,9 @@ type FileOutcome = {
steps?: StepLog[];
error?: { code: string; message: string };
resumed?: boolean;
/** 全量 sha256 的执行方式(sha256sum 子进程 / 进程内流式) */
hashVia?: string;
durationMs?: number;
};
const stepsOf = (s: CrossMoveStepLog[]): StepLog[] => s.map((x) => ({ ...x }));
@@ -689,8 +712,9 @@ export async function executeRecord(
const outcome = await withAttemptToken<FileOutcome>(
db,
opId,
async (info) =>
transferFile(
async (info) => {
const t0 = performance.now();
const r = await transferFile(
kind,
pf.src,
pf.dst,
@@ -700,7 +724,25 @@ export async function executeRecord(
d,
orgSettings.allowCrossVolumeMove,
opts.abortSignal,
),
);
r.durationMs = Math.round(performance.now() - t0);
// 打点:每个文件的耗时/字节/哈希方式,便于判断大文件入库是否拖慢 worker
d.ctx.logger.info(
{
recordId,
ordinal: fr.ordinal,
kind,
status: r.status,
bytes: r.bytes ?? pf.size,
durationMs: r.durationMs,
mbps: r.durationMs > 0 ? Math.round(((r.bytes ?? pf.size) / 1e6 / r.durationMs) * 1000) : null,
hashVia: r.hashVia ?? null,
resumed: info.resumed,
},
"organize.transfer",
);
return r;
},
{
...(opts.runId ? { runId: opts.runId } : {}),
commit: async (trx, r) => {
@@ -776,9 +818,10 @@ async function transferFile(
: { status: "failed", error: { code: "CONFLICT", message: "源已不存在且目标大小不符" } };
}
if (s.kind === "ok") {
const same = await filesMatch(src, dst, "sha256").catch(() => false);
if (!same) return { status: "failed", error: { code: "EEXIST", message: `目标已存在且内容不同: ${dst}` } };
if (kind === "copy") return { status: "succeeded", bytes: size, resumed };
const same = await filesMatchSha256(src, dst, signal).catch(() => ({ match: false, via: null }));
if (!same.match) return { status: "failed", error: { code: "EEXIST", message: `目标已存在且内容不同: ${dst}` } };
if (kind === "copy")
return { status: "succeeded", bytes: size, resumed, ...(same.via ? { hashVia: same.via } : {}) };
if (crossDevice === true || resumed) {
const r = await crossDeviceMove(src, dst, { hash: "sha256" });
return {
@@ -798,6 +841,7 @@ async function transferFile(
return {
status: "succeeded",
bytes: r.bytes,
...(r.sha256Via ? { hashVia: r.sha256Via } : {}),
...(r.sha256
? { checksum: `sha256:${r.sha256}` }
: r.sampleDigest