feat(adapters): qBittorrent, filesystem, TMDB and Emby adapters with testkit mocks (W1-3a)
- qbittorrent: WebUI API v2 client (cookie/API-key auth, add/info/files/filePrio, pause/resume with 5.x stop/start fallback, tags, delete with explicit deleteFiles, maindata), error classes, idempotent submit with local infohash (own bencode) and post-timeout hash/probe-tag lookup; state mapping per MoviePilot - fs: copyWithVerify (tmp + fsync + size/sha256/sample + no-clobber commit), moveSafe (EXDEV surfaced), resumable crossDeviceMove, statSafe/checkRoot with root sentinel, budgeted incremental scan with cursor, ffprobe, statfs usage - tmdb: multi-language search/details/seasons, LRU+TTL cache with coalescing, rate-limit backoff, MoviePilot match chain (name normalization, year +-1) - emby: Emby/Jellyfin items/seasons/episodes/MediaSources, failures -> unknown - testkit: mock qB / TMDB / Emby servers, torrent builder, temp-tree fixtures - docs: W1-3a porting table appended to downloader-and-sites ledger Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
rootandClaude Fable 5.1 committed
1 parent
9c881e737f
commit
ef2e504a94
45 files changed
+5766
-1
No files matched your search
@@ -0,0 +1,5 @@
|
||||
# fs
|
||||
|
||||
本地文件操作:`copyWithVerify`(同目录隐藏临时文件 → fsync → size/sha256/抽样校验 → no-clobber 原子提交 → fsync 目录,保留 mtime,默认不建父目录)、`moveSafe`(同设备 no-clobber 移动,EXDEV 返回 `exdev` 不降级)、`crossDeviceMove`(copy → verify → delete_source,返回步骤日志,可重入,删源失败为 `attention`)、`statSafe`/`checkRoot`(先查库根哨兵 `.uhd-root`,区分 missing / permission_denied / root_offline)、`scanIncremental`(名称排序深度优先、每批 lstat 上限、游标续扫、new/changed/unchanged 标注)、`ffprobe`(无 ffprobe 或失败返回 `unknown`,不抛)、`diskUsage`(statfs,失败返回 `unknown`)、`sweepStalePartials`。
|
||||
|
||||
在本机临时目录与 /dev/shm(跨设备)上做了集成测试;权限拒绝类用例在 root 下跳过。未在真实 NAS/FUSE/网络挂载上验证。
|
||||
@@ -0,0 +1,213 @@
|
||||
// 迁移来源:MoviePilot app/modules/filemanager/storages/local.py _write_atomically/_partial_path/_cleanup_stale_partials/copy(e6820a8);
|
||||
// 调整:fsync + size/sha256/sample 校验、no-clobber 提交(link 原子失败),见 docs/porting/moviepilot-transfer.md §2.6。
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { createReadStream, createWriteStream } from "node:fs";
|
||||
import { link, lstat, mkdir, open, readdir, rename, stat, unlink, utimes } from "node:fs/promises";
|
||||
import { basename, dirname, join } from "node:path";
|
||||
import { Transform } from "node:stream";
|
||||
import { pipeline } from "node:stream/promises";
|
||||
import { errnoOf, FsError, toFsError } from "./errors";
|
||||
|
||||
export type VerifyMethod = "size" | "sha256" | "sample";
|
||||
|
||||
export interface CopyOptions {
|
||||
/** 临时文件后缀,默认 .uhd-partial;临时文件与目标同目录、以 . 开头 */
|
||||
tmpSuffix?: string;
|
||||
/** 校验方式,默认 size(总是比对大小;sha256/sample 额外比对内容) */
|
||||
hash?: VerifyMethod;
|
||||
/** sample 模式的抽样块数与块大小 */
|
||||
sampleChunks?: number;
|
||||
sampleChunkBytes?: number;
|
||||
/** 是否创建目标父目录;默认 false(防止库根离线时在挂载点下写入) */
|
||||
createParents?: boolean;
|
||||
signal?: AbortSignal;
|
||||
onProgress?: (bytesCopied: number, totalBytes: number) => void;
|
||||
}
|
||||
|
||||
export interface CopyReceipt {
|
||||
src: string;
|
||||
dst: string;
|
||||
bytes: number;
|
||||
verifiedBy: VerifyMethod;
|
||||
sha256?: string;
|
||||
sampleDigest?: string;
|
||||
partialPath: string;
|
||||
durationMs: number;
|
||||
}
|
||||
|
||||
async function fsyncPath(p: string, flags = "r"): Promise<void> {
|
||||
let fh: Awaited<ReturnType<typeof open>> | undefined;
|
||||
try {
|
||||
fh = await open(p, flags);
|
||||
await fh.sync();
|
||||
} catch (e) {
|
||||
// 部分文件系统不支持对目录 fsync
|
||||
if (!["EISDIR", "EINVAL", "EBADF", "EPERM", "ENOTSUP"].includes(errnoOf(e) ?? "")) throw e;
|
||||
} finally {
|
||||
await fh?.close();
|
||||
}
|
||||
}
|
||||
|
||||
export async function sha256File(p: string, signal?: AbortSignal): Promise<string> {
|
||||
const h = createHash("sha256");
|
||||
await pipeline(createReadStream(p), async (src: AsyncIterable<Buffer>) => {
|
||||
for await (const c of src) {
|
||||
if (signal?.aborted) throw new FsError("ABORTED", "aborted", { path: p });
|
||||
h.update(c);
|
||||
}
|
||||
});
|
||||
return h.digest("hex");
|
||||
}
|
||||
|
||||
/** 抽样摘要:首/尾/等距 N 块 + 文件大小。抽样一致不等于完整一致性证明(REQ §8.2)。 */
|
||||
export async function sampleDigest(p: string, chunks = 8, chunkBytes = 1024 * 1024): Promise<string> {
|
||||
const fh = await open(p, "r");
|
||||
try {
|
||||
const { size } = await fh.stat();
|
||||
const h = createHash("sha256");
|
||||
h.update(String(size));
|
||||
const n = Math.max(2, chunks);
|
||||
const buf = Buffer.alloc(chunkBytes);
|
||||
const offsets = new Set<number>();
|
||||
for (let i = 0; i < n; i++) offsets.add(Math.max(0, Math.floor(((size - chunkBytes) * i) / (n - 1))));
|
||||
for (const off of [...offsets].sort((a, b) => a - b)) {
|
||||
const { bytesRead } = await fh.read(buf, 0, chunkBytes, off);
|
||||
h.update(buf.subarray(0, bytesRead));
|
||||
}
|
||||
return h.digest("hex");
|
||||
} finally {
|
||||
await fh.close();
|
||||
}
|
||||
}
|
||||
|
||||
export function partialPathFor(dst: string, suffix = ".uhd-partial"): string {
|
||||
return join(dirname(dst), `.${basename(dst)}.${process.pid}.${randomBytes(3).toString("hex")}${suffix}`);
|
||||
}
|
||||
|
||||
/** no-clobber 提交:link(tmp,dst) 在目标存在时原子失败;文件系统不支持硬链接时退回 lstat+rename。 */
|
||||
export async function commitNoClobber(tmp: string, dst: string): Promise<void> {
|
||||
try {
|
||||
await link(tmp, dst);
|
||||
await unlink(tmp);
|
||||
return;
|
||||
} catch (e) {
|
||||
const code = errnoOf(e);
|
||||
if (code === "EEXIST") throw new FsError("EEXIST", `target exists: ${dst}`, { path: dst, cause: e });
|
||||
if (!["EPERM", "ENOTSUP", "EOPNOTSUPP", "ENOSYS", "EMLINK"].includes(code ?? "")) throw toFsError(e, dst);
|
||||
}
|
||||
try {
|
||||
await lstat(dst);
|
||||
throw new FsError("EEXIST", `target exists: ${dst}`, { path: dst });
|
||||
} catch (e) {
|
||||
if (e instanceof FsError) throw e;
|
||||
if (errnoOf(e) !== "ENOENT") throw toFsError(e, dst);
|
||||
}
|
||||
await rename(tmp, dst);
|
||||
}
|
||||
|
||||
/**
|
||||
* 复制并校验:写目标同目录临时文件 → fsync → 校验 → no-clobber 原子提交 → fsync 目录。
|
||||
* 任何失败都会清理临时文件并抛出 FsError;目标已存在抛 EEXIST,绝不覆盖。
|
||||
*/
|
||||
export async function copyWithVerify(src: string, dst: string, opts: CopyOptions = {}): Promise<CopyReceipt> {
|
||||
const started = Date.now();
|
||||
const method = opts.hash ?? "size";
|
||||
let srcStat: Awaited<ReturnType<typeof stat>>;
|
||||
try {
|
||||
srcStat = await stat(src);
|
||||
} catch (e) {
|
||||
throw toFsError(e, src);
|
||||
}
|
||||
if (!srcStat.isFile()) throw new FsError("IO", `source is not a regular file: ${src}`, { path: src });
|
||||
try {
|
||||
await lstat(dst);
|
||||
throw new FsError("EEXIST", `target exists: ${dst}`, { path: dst });
|
||||
} catch (e) {
|
||||
if (e instanceof FsError) throw e;
|
||||
if (errnoOf(e) !== "ENOENT") throw toFsError(e, dst);
|
||||
}
|
||||
if (opts.createParents) await mkdir(dirname(dst), { recursive: true });
|
||||
|
||||
const tmp = partialPathFor(dst, opts.tmpSuffix);
|
||||
const hasher = method === "sha256" ? createHash("sha256") : null;
|
||||
let copied = 0;
|
||||
try {
|
||||
const meter = new Transform({
|
||||
transform(chunk: Buffer, _enc, cb) {
|
||||
copied += chunk.length;
|
||||
hasher?.update(chunk);
|
||||
opts.onProgress?.(copied, srcStat.size);
|
||||
cb(null, chunk);
|
||||
},
|
||||
});
|
||||
await pipeline(
|
||||
createReadStream(src),
|
||||
meter,
|
||||
createWriteStream(tmp, { flags: "wx", flush: true }),
|
||||
opts.signal ? { signal: opts.signal } : {},
|
||||
);
|
||||
await fsyncPath(tmp, "r+");
|
||||
|
||||
const after = await stat(src);
|
||||
if (after.size !== srcStat.size || after.mtimeMs !== srcStat.mtimeMs) {
|
||||
throw new FsError("SOURCE_CHANGED", `source changed during copy: ${src}`, { path: src });
|
||||
}
|
||||
const tmpStat = await stat(tmp);
|
||||
if (tmpStat.size !== srcStat.size || copied !== srcStat.size) {
|
||||
throw new FsError("VERIFY_FAILED", `size mismatch ${tmpStat.size} != ${srcStat.size}`, { path: dst });
|
||||
}
|
||||
const receipt: CopyReceipt = {
|
||||
src,
|
||||
dst,
|
||||
bytes: srcStat.size,
|
||||
verifiedBy: method,
|
||||
partialPath: tmp,
|
||||
durationMs: 0,
|
||||
};
|
||||
if (hasher) {
|
||||
const expected = hasher.digest("hex");
|
||||
const actual = await sha256File(tmp, opts.signal);
|
||||
if (actual !== expected) throw new FsError("VERIFY_FAILED", "sha256 mismatch", { path: dst });
|
||||
receipt.sha256 = actual;
|
||||
} else if (method === "sample") {
|
||||
const a = await sampleDigest(src, opts.sampleChunks, opts.sampleChunkBytes);
|
||||
const b = await sampleDigest(tmp, opts.sampleChunks, opts.sampleChunkBytes);
|
||||
if (a !== b) throw new FsError("VERIFY_FAILED", "sample digest mismatch", { path: dst });
|
||||
receipt.sampleDigest = b;
|
||||
}
|
||||
await utimes(tmp, srcStat.atime, srcStat.mtime);
|
||||
await commitNoClobber(tmp, dst);
|
||||
await fsyncPath(dirname(dst));
|
||||
receipt.durationMs = Date.now() - started;
|
||||
return receipt;
|
||||
} catch (e) {
|
||||
await unlink(tmp).catch(() => undefined);
|
||||
if (opts.signal?.aborted) throw new FsError("ABORTED", "copy aborted", { path: dst, cause: e });
|
||||
throw toFsError(e, dst);
|
||||
}
|
||||
}
|
||||
|
||||
/** 按指定方式比对两个文件是否一致(size 恒比对)。 */
|
||||
export async function filesMatch(a: string, b: string, method: VerifyMethod = "size"): Promise<boolean> {
|
||||
const [sa, sb] = await Promise.all([stat(a), stat(b)]);
|
||||
if (sa.size !== sb.size) return false;
|
||||
if (method === "sha256") return (await sha256File(a)) === (await sha256File(b));
|
||||
if (method === "sample") return (await sampleDigest(a)) === (await sampleDigest(b));
|
||||
return true;
|
||||
}
|
||||
|
||||
/** 清理目录下超过 olderThanMs 的残留临时文件,返回被删除的路径。 */
|
||||
export async function sweepStalePartials(dir: string, olderThanMs: number, suffix = ".uhd-partial"): Promise<string[]> {
|
||||
const removed: string[] = [];
|
||||
const now = Date.now();
|
||||
for (const name of await readdir(dir)) {
|
||||
if (!name.startsWith(".") || !name.endsWith(suffix)) continue;
|
||||
const p = join(dir, name);
|
||||
const st = await lstat(p).catch(() => null);
|
||||
if (st?.isFile() && now - st.mtimeMs > olderThanMs) {
|
||||
await unlink(p).catch(() => undefined);
|
||||
removed.push(p);
|
||||
}
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
export type FsErrorCode =
|
||||
| "EXDEV" // 跨设备,不会自动降级
|
||||
| "EEXIST" // 目标已存在(no-clobber)
|
||||
| "ENOENT"
|
||||
| "EACCES"
|
||||
| "ENOSPC"
|
||||
| "VERIFY_FAILED"
|
||||
| "SOURCE_CHANGED"
|
||||
| "ROOT_OFFLINE"
|
||||
| "ABORTED"
|
||||
| "IO";
|
||||
|
||||
export class FsError extends Error {
|
||||
override readonly name = "FsError";
|
||||
constructor(
|
||||
readonly code: FsErrorCode,
|
||||
message: string,
|
||||
readonly detail: { path?: string; syscall?: string; cause?: unknown } = {},
|
||||
) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
export function errnoOf(e: unknown): string | undefined {
|
||||
return (e as NodeJS.ErrnoException | undefined)?.code;
|
||||
}
|
||||
|
||||
export function toFsError(e: unknown, path?: string): FsError {
|
||||
if (e instanceof FsError) return e;
|
||||
const code = errnoOf(e);
|
||||
const mapped: FsErrorCode =
|
||||
code === "EXDEV" || code === "EEXIST" || code === "ENOENT" || code === "ENOSPC"
|
||||
? code
|
||||
: code === "EACCES" || code === "EPERM"
|
||||
? "EACCES"
|
||||
: (e as Error)?.name === "AbortError"
|
||||
? "ABORTED"
|
||||
: "IO";
|
||||
return new FsError(mapped, (e as Error)?.message ?? String(e), {
|
||||
...(path !== undefined ? { path } : {}),
|
||||
...((e as NodeJS.ErrnoException)?.syscall ? { syscall: (e as NodeJS.ErrnoException).syscall! } : {}),
|
||||
cause: e,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
// 本项目新增:ffprobe 封装与 statfs 磁盘用量(REQ §8.2、§8.4.1);失败一律返回 unknown。
|
||||
import { execFile } from "node:child_process";
|
||||
import { statfs } from "node:fs/promises";
|
||||
import { errnoOf } from "./errors";
|
||||
|
||||
export interface ProbeVideo {
|
||||
codec?: string;
|
||||
profile?: string;
|
||||
width?: number;
|
||||
height?: number;
|
||||
pixFmt?: string;
|
||||
/** 仅在流信息明确时给出;未知不补值 */
|
||||
hdr?: "HDR10" | "HLG" | "DV";
|
||||
frameRate?: string;
|
||||
}
|
||||
export interface ProbeAudio {
|
||||
codec?: string;
|
||||
profile?: string;
|
||||
channels?: number;
|
||||
channelLayout?: string;
|
||||
language?: string;
|
||||
title?: string;
|
||||
}
|
||||
export interface ProbeSubtitle {
|
||||
codec?: string;
|
||||
language?: string;
|
||||
title?: string;
|
||||
forced?: boolean;
|
||||
}
|
||||
|
||||
export type ProbeResult =
|
||||
| {
|
||||
status: "ok";
|
||||
container?: string;
|
||||
durationSec?: number;
|
||||
bitRate?: number;
|
||||
sizeBytes?: number;
|
||||
video: ProbeVideo[];
|
||||
audio: ProbeAudio[];
|
||||
subtitles: ProbeSubtitle[];
|
||||
}
|
||||
| {
|
||||
status: "unknown";
|
||||
reason: "ffprobe_unavailable" | "timeout" | "probe_failed" | "parse_failed";
|
||||
message?: string;
|
||||
};
|
||||
|
||||
export interface ProbeOptions {
|
||||
/** ffprobe 可执行文件,默认 "ffprobe"(PATH 查找) */
|
||||
bin?: string;
|
||||
timeoutMs?: number;
|
||||
}
|
||||
|
||||
const optNum = (v: unknown): number | undefined => {
|
||||
const n = typeof v === "string" ? Number(v) : v;
|
||||
return typeof n === "number" && Number.isFinite(n) ? n : undefined;
|
||||
};
|
||||
const optStr = (v: unknown): string | undefined => (typeof v === "string" && v !== "" ? v : undefined);
|
||||
|
||||
type Stream = Record<string, unknown> & {
|
||||
tags?: Record<string, string>;
|
||||
side_data_list?: { side_data_type?: string }[];
|
||||
};
|
||||
|
||||
function hdrOf(s: Stream): ProbeVideo["hdr"] {
|
||||
if (s.side_data_list?.some((d) => /dovi|dolby vision/i.test(d.side_data_type ?? ""))) return "DV";
|
||||
if (s.color_transfer === "smpte2084") return "HDR10";
|
||||
if (s.color_transfer === "arib-std-b67") return "HLG";
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function clean<T extends object>(o: T): T {
|
||||
return Object.fromEntries(Object.entries(o).filter(([, v]) => v !== undefined)) as T;
|
||||
}
|
||||
|
||||
/** ffprobe 封装:任何失败(含本机无 ffprobe)返回 { status: "unknown" },不抛。 */
|
||||
export function ffprobe(path: string, opts: ProbeOptions = {}): Promise<ProbeResult> {
|
||||
return new Promise((resolve) => {
|
||||
execFile(
|
||||
opts.bin ?? "ffprobe",
|
||||
["-v", "error", "-print_format", "json", "-show_format", "-show_streams", "--", path],
|
||||
{ timeout: opts.timeoutMs ?? 30_000, maxBuffer: 16 * 1024 * 1024 },
|
||||
(err, stdout, stderr) => {
|
||||
if (err) {
|
||||
const e = err as NodeJS.ErrnoException & { killed?: boolean; signal?: string };
|
||||
if (errnoOf(e) === "ENOENT" || errnoOf(e) === "EACCES")
|
||||
return resolve({ status: "unknown", reason: "ffprobe_unavailable" });
|
||||
if (e.killed || e.signal === "SIGTERM") return resolve({ status: "unknown", reason: "timeout" });
|
||||
return resolve({ status: "unknown", reason: "probe_failed", message: String(stderr).slice(0, 500) });
|
||||
}
|
||||
try {
|
||||
const j = JSON.parse(stdout) as { format?: Record<string, unknown>; streams?: Stream[] };
|
||||
const streams = j.streams ?? [];
|
||||
const f = j.format ?? {};
|
||||
resolve(
|
||||
clean({
|
||||
status: "ok" as const,
|
||||
container: optStr(f.format_name),
|
||||
durationSec: optNum(f.duration),
|
||||
bitRate: optNum(f.bit_rate),
|
||||
sizeBytes: optNum(f.size),
|
||||
video: streams
|
||||
.filter((s) => s.codec_type === "video" && !(s.disposition as { attached_pic?: number })?.attached_pic)
|
||||
.map((s) =>
|
||||
clean({
|
||||
codec: optStr(s.codec_name),
|
||||
profile: optStr(s.profile),
|
||||
width: optNum(s.width),
|
||||
height: optNum(s.height),
|
||||
pixFmt: optStr(s.pix_fmt),
|
||||
hdr: hdrOf(s),
|
||||
frameRate: optStr(s.avg_frame_rate) === "0/0" ? undefined : optStr(s.avg_frame_rate),
|
||||
}),
|
||||
),
|
||||
audio: streams
|
||||
.filter((s) => s.codec_type === "audio")
|
||||
.map((s) =>
|
||||
clean({
|
||||
codec: optStr(s.codec_name),
|
||||
profile: optStr(s.profile),
|
||||
channels: optNum(s.channels),
|
||||
channelLayout: optStr(s.channel_layout),
|
||||
language: optStr(s.tags?.language),
|
||||
title: optStr(s.tags?.title),
|
||||
}),
|
||||
),
|
||||
subtitles: streams
|
||||
.filter((s) => s.codec_type === "subtitle")
|
||||
.map((s) =>
|
||||
clean({
|
||||
codec: optStr(s.codec_name),
|
||||
language: optStr(s.tags?.language),
|
||||
title: optStr(s.tags?.title),
|
||||
forced: (s.disposition as { forced?: number } | undefined)?.forced === 1 ? true : undefined,
|
||||
}),
|
||||
),
|
||||
}),
|
||||
);
|
||||
} catch (e) {
|
||||
resolve({ status: "unknown", reason: "parse_failed", message: (e as Error).message });
|
||||
}
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
export type DiskUsage =
|
||||
| {
|
||||
status: "ok";
|
||||
path: string;
|
||||
totalBytes: number;
|
||||
freeBytes: number;
|
||||
/** 非 root 可用空间 */
|
||||
availableBytes: number;
|
||||
usedBytes: number;
|
||||
/** used / (used + available),与 df 的 Use% 一致 */
|
||||
usedRatio: number;
|
||||
}
|
||||
/** 无法读取:调用方应保守阻止新增分配并告警(REQ §8.4.1) */
|
||||
| { status: "unknown"; path: string; code: string; message: string };
|
||||
|
||||
export async function diskUsage(path: string): Promise<DiskUsage> {
|
||||
try {
|
||||
const s = await statfs(path);
|
||||
const total = s.blocks * s.bsize;
|
||||
const free = s.bfree * s.bsize;
|
||||
const avail = s.bavail * s.bsize;
|
||||
const used = total - free;
|
||||
return {
|
||||
status: "ok",
|
||||
path,
|
||||
totalBytes: total,
|
||||
freeBytes: free,
|
||||
availableBytes: avail,
|
||||
usedBytes: used,
|
||||
usedRatio: used + avail > 0 ? used / (used + avail) : 0,
|
||||
};
|
||||
} catch (e) {
|
||||
return { status: "unknown", path, code: errnoOf(e) ?? "UNKNOWN", message: (e as Error)?.message ?? String(e) };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,263 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { chmod, readFile, rm, stat, symlink, unlink, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { createTempTree, findCrossDeviceBase, type TempTree } from "@umt/testkit";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
checkRoot,
|
||||
copyWithVerify,
|
||||
crossDeviceMove,
|
||||
diskUsage,
|
||||
FsError,
|
||||
ffprobe,
|
||||
moveSafe,
|
||||
scanIncremental,
|
||||
statSafe,
|
||||
sweepStalePartials,
|
||||
} from "./index";
|
||||
|
||||
const trees: TempTree[] = [];
|
||||
async function tree(...a: Parameters<typeof createTempTree>) {
|
||||
const t = await createTempTree(...a);
|
||||
trees.push(t);
|
||||
return t;
|
||||
}
|
||||
afterEach(async () => {
|
||||
await Promise.all(trees.splice(0).map((t) => t.cleanup()));
|
||||
});
|
||||
|
||||
const xdevBase = findCrossDeviceBase();
|
||||
const isRoot = process.getuid?.() === 0;
|
||||
const hasFfmpeg = (() => {
|
||||
try {
|
||||
execFileSync("ffmpeg", ["-version"], { stdio: "ignore" });
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
})();
|
||||
|
||||
describe("copyWithVerify", () => {
|
||||
it.each(["size", "sha256", "sample"] as const)("%s 校验后原子提交,保留 mtime,无残留临时文件", async (hash) => {
|
||||
const t = await tree({ "src/a.mkv": 3 * 1024 * 1024 + 17, "dst/": null });
|
||||
const r = await copyWithVerify(t.path("src/a.mkv"), t.path("dst/a.mkv"), { hash, sampleChunkBytes: 4096 });
|
||||
expect(r.bytes).toBe(3 * 1024 * 1024 + 17);
|
||||
expect(r.verifiedBy).toBe(hash);
|
||||
if (hash === "sha256") expect(r.sha256).toMatch(/^[0-9a-f]{64}$/);
|
||||
expect((await readFile(t.path("dst/a.mkv"))).equals(await readFile(t.path("src/a.mkv")))).toBe(true);
|
||||
expect(Math.round((await stat(t.path("dst/a.mkv"))).mtimeMs)).toBe(
|
||||
Math.round((await stat(t.path("src/a.mkv"))).mtimeMs),
|
||||
);
|
||||
expect(await sweepStalePartials(t.path("dst"), -1)).toEqual([]);
|
||||
});
|
||||
|
||||
it("目标已存在 → EEXIST,不覆盖", async () => {
|
||||
const t = await tree({ "a.mkv": "new", "b.mkv": "old" });
|
||||
const e = await copyWithVerify(t.path("a.mkv"), t.path("b.mkv")).catch((x) => x);
|
||||
expect(e).toBeInstanceOf(FsError);
|
||||
expect(e.code).toBe("EEXIST");
|
||||
expect(await readFile(t.path("b.mkv"), "utf8")).toBe("old");
|
||||
});
|
||||
|
||||
it("父目录不存在默认不创建;中止信号清理临时文件", async () => {
|
||||
const t = await tree({ "a.mkv": 1024 * 1024 });
|
||||
expect((await copyWithVerify(t.path("a.mkv"), t.path("lib/x/a.mkv")).catch((e) => e)).code).toBe("ENOENT");
|
||||
const ac = new AbortController();
|
||||
ac.abort();
|
||||
await t.write("lib/.keep", "");
|
||||
const e = await copyWithVerify(t.path("a.mkv"), t.path("lib/a.mkv"), { signal: ac.signal }).catch((x) => x);
|
||||
expect(e.code).toBe("ABORTED");
|
||||
expect(await sweepStalePartials(t.path("lib"), -1)).toEqual([]);
|
||||
await expect(stat(t.path("lib/a.mkv"))).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("残留临时文件清理只删过期的 .uhd-partial", async () => {
|
||||
const t = await tree({ "lib/.x.mkv.1.abc.uhd-partial": "x", "lib/keep.mkv": "k" });
|
||||
const removed = await sweepStalePartials(t.path("lib"), -1);
|
||||
expect(removed).toEqual([t.path("lib/.x.mkv.1.abc.uhd-partial")]);
|
||||
expect(await sweepStalePartials(t.path("lib"), 60_000)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("moveSafe / crossDeviceMove", () => {
|
||||
it("同设备移动;目标存在返回 exists 且不覆盖", async () => {
|
||||
const t = await tree({ "a.mkv": "A", "b.mkv": "B", "c.mkv": "C" });
|
||||
expect(await moveSafe(t.path("a.mkv"), t.path("moved.mkv"))).toEqual({ status: "moved" });
|
||||
await expect(stat(t.path("a.mkv"))).rejects.toThrow();
|
||||
expect(await moveSafe(t.path("b.mkv"), t.path("c.mkv"))).toEqual({ status: "exists" });
|
||||
expect(await readFile(t.path("c.mkv"), "utf8")).toBe("C");
|
||||
const e = await moveSafe(t.path("nope.mkv"), t.path("x.mkv")).catch((x) => x);
|
||||
expect(e.code).toBe("ENOENT");
|
||||
});
|
||||
|
||||
it("link 成功但 unlink 前中断(同 inode)→ 续跑补完删除", async () => {
|
||||
const t = await tree({ "a.mkv": "A" });
|
||||
const { link } = await import("node:fs/promises");
|
||||
await link(t.path("a.mkv"), t.path("b.mkv"));
|
||||
expect(await moveSafe(t.path("a.mkv"), t.path("b.mkv"))).toEqual({ status: "moved" });
|
||||
await expect(stat(t.path("a.mkv"))).rejects.toThrow();
|
||||
});
|
||||
|
||||
it.skipIf(!xdevBase)("跨设备:moveSafe 返回 exdev 不降级;crossDeviceMove 三步完成", async () => {
|
||||
// xdevBase 为 null 时跳过:本机找不到与 tmpdir 不同设备的可写目录
|
||||
const a = await tree({ "a.mkv": 256 * 1024 });
|
||||
const b = await tree({ "lib/": null }, { base: xdevBase! });
|
||||
expect(await moveSafe(a.path("a.mkv"), b.path("lib/a.mkv"))).toEqual({ status: "exdev" });
|
||||
await expect(stat(a.path("a.mkv"))).resolves.toBeTruthy();
|
||||
const r = await crossDeviceMove(a.path("a.mkv"), b.path("lib/a.mkv"), { hash: "sha256" });
|
||||
expect(r.status).toBe("completed");
|
||||
expect(r.steps.map((s) => `${s.step}:${s.status}`)).toEqual(["copy:done", "verify:done", "delete_source:done"]);
|
||||
await expect(stat(a.path("a.mkv"))).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("crossDeviceMove 可重入:目标已一致则跳过复制;源已删且上次已验证 → completed;目标不同 → failed 不删源", async () => {
|
||||
const t = await tree({ "src/a.mkv": "same", "dst/a.mkv": "same", "src/b.mkv": "bbbb", "dst/b.mkv": "cccc" });
|
||||
const r1 = await crossDeviceMove(t.path("src/a.mkv"), t.path("dst/a.mkv"), { hash: "sha256" });
|
||||
expect(r1.steps.map((s) => `${s.step}:${s.status}`)).toEqual(["copy:skipped", "verify:done", "delete_source:done"]);
|
||||
const r2 = await crossDeviceMove(t.path("src/a.mkv"), t.path("dst/a.mkv"), { previous: r1.steps });
|
||||
expect(r2.status).toBe("completed");
|
||||
const r3 = await crossDeviceMove(t.path("src/b.mkv"), t.path("dst/b.mkv"), { hash: "sha256" });
|
||||
expect(r3.status).toBe("failed");
|
||||
expect(await readFile(t.path("src/b.mkv"), "utf8")).toBe("bbbb");
|
||||
const r4 = await crossDeviceMove(t.path("src/none.mkv"), t.path("dst/b.mkv"));
|
||||
expect(r4.status).toBe("attention");
|
||||
});
|
||||
|
||||
it.skipIf(isRoot)("删源失败 → attention(以 root 运行时跳过:root 无视目录权限)", async () => {
|
||||
const t = await tree({ "src/a.mkv": "A", "dst/": null });
|
||||
await chmod(t.path("src"), 0o555);
|
||||
try {
|
||||
const r = await crossDeviceMove(t.path("src/a.mkv"), t.path("dst/a.mkv"));
|
||||
expect(r.status).toBe("attention");
|
||||
expect(r.steps.at(-1)).toMatchObject({ step: "delete_source", status: "failed" });
|
||||
} finally {
|
||||
await chmod(t.path("src"), 0o755);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("statSafe / checkRoot / diskUsage", () => {
|
||||
it("区分正常、缺失、根离线", async () => {
|
||||
const t = await tree({ "movies/a.mkv": "A" }, { sentinel: true });
|
||||
const ok = await statSafe(t.path("movies/a.mkv"), { root: t.root });
|
||||
expect(ok.kind).toBe("ok");
|
||||
if (ok.kind === "ok") expect(ok.facts.size).toBe(1);
|
||||
expect((await statSafe(t.path("movies/b.mkv"), { root: t.root })).kind).toBe("missing");
|
||||
expect(await checkRoot(t.root)).toMatchObject({ kind: "online" });
|
||||
await unlink(t.path(".uhd-root"));
|
||||
expect(await statSafe(t.path("movies/b.mkv"), { root: t.root })).toMatchObject({
|
||||
kind: "root_offline",
|
||||
reason: "sentinel_missing",
|
||||
});
|
||||
expect(await checkRoot(join(t.root, "gone"))).toMatchObject({ kind: "offline", reason: "root_missing" });
|
||||
});
|
||||
|
||||
it.skipIf(isRoot)("权限拒绝 → permission_denied(root 运行时跳过)", async () => {
|
||||
const t = await tree({ "locked/a.mkv": "A" });
|
||||
await chmod(t.path("locked"), 0o000);
|
||||
try {
|
||||
expect((await statSafe(t.path("locked/a.mkv"))).kind).toBe("permission_denied");
|
||||
} finally {
|
||||
await chmod(t.path("locked"), 0o755);
|
||||
}
|
||||
});
|
||||
|
||||
it("磁盘用量:statfs 成功给出比例,路径不存在返回 unknown", async () => {
|
||||
const t = await tree();
|
||||
const u = await diskUsage(t.root);
|
||||
expect(u.status).toBe("ok");
|
||||
if (u.status === "ok") {
|
||||
expect(u.totalBytes).toBeGreaterThan(0);
|
||||
expect(u.usedRatio).toBeGreaterThanOrEqual(0);
|
||||
expect(u.usedRatio).toBeLessThanOrEqual(1);
|
||||
}
|
||||
expect(await diskUsage(join(t.root, "nope"))).toMatchObject({ status: "unknown", code: "ENOENT" });
|
||||
});
|
||||
});
|
||||
|
||||
describe("scanIncremental", () => {
|
||||
it("按 I/O 上限分批,游标续扫覆盖全部且不重复", async () => {
|
||||
const spec: Record<string, string> = {};
|
||||
for (const d of ["A", "B", "C"]) for (let i = 0; i < 4; i++) spec[`${d}/s${i}/f${i}.mkv`] = `${d}${i}`;
|
||||
spec["B/.x.uhd-partial"] = "p";
|
||||
const t = await tree(spec, { sentinel: true });
|
||||
const seen: string[] = [];
|
||||
let cursor: string | null = null;
|
||||
let batches = 0;
|
||||
do {
|
||||
const r = await scanIncremental(t.root, { cursor, maxStats: 5 });
|
||||
expect(r.statCount).toBeLessThanOrEqual(5);
|
||||
seen.push(...r.entries.map((e) => e.relPath));
|
||||
cursor = r.cursor;
|
||||
batches++;
|
||||
if (r.status === "complete") break;
|
||||
expect(r.status).toBe("partial");
|
||||
} while (batches < 10);
|
||||
expect(batches).toBe(3);
|
||||
expect(seen).toEqual([...seen].sort());
|
||||
expect(new Set(seen).size).toBe(12);
|
||||
expect(seen).not.toContain("B/.x.uhd-partial");
|
||||
});
|
||||
|
||||
it("标注 new/changed/unchanged;根离线不扫描;符号链接不跟随", async () => {
|
||||
const t = await tree({ "a.mkv": "A", "b.mkv": "B", "c.mkv": "C" }, { sentinel: true });
|
||||
await symlink(t.path("a.mkv"), t.path("d.mkv"));
|
||||
const first = await scanIncremental(t.root);
|
||||
const prev = new Map(first.entries.map((e) => [e.relPath, e]));
|
||||
await writeFile(t.path("b.mkv"), "BBBB");
|
||||
const second = await scanIncremental(t.root, { previous: (p) => prev.get(p) });
|
||||
const byPath = Object.fromEntries(second.entries.map((e) => [e.relPath, e]));
|
||||
expect(byPath["a.mkv"]!.change).toBe("unchanged");
|
||||
expect(byPath["b.mkv"]!.change).toBe("changed");
|
||||
expect(byPath["d.mkv"]!.isSymlink).toBe(true);
|
||||
await rm(t.path(".uhd-root"));
|
||||
const off = await scanIncremental(t.root);
|
||||
expect(off.status).toBe("root_offline");
|
||||
expect(off.entries).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("ffprobe", () => {
|
||||
it("可执行文件不存在 / 非媒体文件 → unknown,不抛", async () => {
|
||||
const t = await tree({ "a.txt": "hello" });
|
||||
expect(await ffprobe(t.path("a.txt"), { bin: "/nonexistent/ffprobe" })).toEqual({
|
||||
status: "unknown",
|
||||
reason: "ffprobe_unavailable",
|
||||
});
|
||||
const r = await ffprobe(t.path("a.txt"));
|
||||
expect(r.status).toBe("unknown");
|
||||
});
|
||||
|
||||
it.skipIf(!hasFfmpeg)("真实小样片:读出分辨率与音轨(本机无 ffmpeg 时跳过)", async () => {
|
||||
const t = await tree();
|
||||
const out = t.path("s.mkv");
|
||||
execFileSync("ffmpeg", [
|
||||
"-v",
|
||||
"error",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"testsrc=size=320x240:rate=5",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"sine=frequency=440",
|
||||
"-t",
|
||||
"1",
|
||||
"-c:v",
|
||||
"mpeg4",
|
||||
"-c:a",
|
||||
"ac3",
|
||||
"-metadata:s:a:0",
|
||||
"language=jpn",
|
||||
out,
|
||||
]);
|
||||
const r = await ffprobe(out);
|
||||
expect(r.status).toBe("ok");
|
||||
if (r.status === "ok") {
|
||||
expect(r.video[0]).toMatchObject({ codec: "mpeg4", width: 320, height: 240 });
|
||||
expect(r.video[0]!.hdr).toBeUndefined();
|
||||
expect(r.audio[0]).toMatchObject({ codec: "ac3", language: "jpn" });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,6 @@
|
||||
export * from "./copy";
|
||||
export * from "./errors";
|
||||
export * from "./ffprobe";
|
||||
export * from "./move";
|
||||
export * from "./scan";
|
||||
export * from "./stat";
|
||||
@@ -0,0 +1,180 @@
|
||||
// 迁移来源:MoviePilot storages/local.py move 与 transhandler.py materialize_target/delete_move_source(e6820a8);
|
||||
// 调整:EXDEV 显式返回不降级;跨设备拆为 copy→verify→delete_source 可重入三步,删源失败 = attention。
|
||||
import { link, lstat, rename, stat, unlink } from "node:fs/promises";
|
||||
import { dirname } from "node:path";
|
||||
import { type CopyOptions, copyWithVerify, filesMatch, type VerifyMethod } from "./copy";
|
||||
import { errnoOf, FsError, toFsError } from "./errors";
|
||||
|
||||
export type MoveResult = { status: "moved" } | { status: "exdev" } | { status: "exists" };
|
||||
|
||||
/**
|
||||
* 同设备移动(no-clobber)。跨设备返回 { status: "exdev" },**不会**自动降级为复制——
|
||||
* 调用方需显式切换到 crossDeviceMove 并在操作账本中记录(REQ §8.3)。
|
||||
* 其他错误抛 FsError。
|
||||
*/
|
||||
export async function moveSafe(src: string, dst: string): Promise<MoveResult> {
|
||||
let s: Awaited<ReturnType<typeof lstat>>;
|
||||
try {
|
||||
s = await lstat(src);
|
||||
} catch (e) {
|
||||
throw toFsError(e, src);
|
||||
}
|
||||
if (s.isFile()) {
|
||||
try {
|
||||
await link(src, dst);
|
||||
} catch (e) {
|
||||
const code = errnoOf(e);
|
||||
if (code === "EXDEV") return { status: "exdev" };
|
||||
if (code === "EEXIST") {
|
||||
// 上次 link 成功但 unlink 前中断:目标与源是同一 inode → 补完删除源
|
||||
const d = await lstat(dst).catch(() => null);
|
||||
if (d && d.ino === s.ino && d.dev === s.dev) {
|
||||
await unlink(src);
|
||||
return { status: "moved" };
|
||||
}
|
||||
return { status: "exists" };
|
||||
}
|
||||
if (!["EPERM", "ENOTSUP", "EOPNOTSUPP", "ENOSYS", "EMLINK"].includes(code ?? "")) throw toFsError(e, dst);
|
||||
return renameChecked(src, dst);
|
||||
}
|
||||
try {
|
||||
await unlink(src);
|
||||
} catch (e) {
|
||||
throw toFsError(e, src);
|
||||
}
|
||||
return { status: "moved" };
|
||||
}
|
||||
return renameChecked(src, dst);
|
||||
}
|
||||
|
||||
async function renameChecked(src: string, dst: string): Promise<MoveResult> {
|
||||
try {
|
||||
await lstat(dst);
|
||||
return { status: "exists" };
|
||||
} catch (e) {
|
||||
if (errnoOf(e) !== "ENOENT") throw toFsError(e, dst);
|
||||
}
|
||||
try {
|
||||
await rename(src, dst);
|
||||
return { status: "moved" };
|
||||
} catch (e) {
|
||||
if (errnoOf(e) === "EXDEV") return { status: "exdev" };
|
||||
throw toFsError(e, dst);
|
||||
}
|
||||
}
|
||||
|
||||
export type CrossMoveStep = "copy" | "verify" | "delete_source";
|
||||
|
||||
export interface CrossMoveStepLog {
|
||||
step: CrossMoveStep;
|
||||
status: "done" | "skipped" | "failed";
|
||||
at: string;
|
||||
detail?: string;
|
||||
error?: { code: string; message: string };
|
||||
}
|
||||
|
||||
export interface CrossMoveResult {
|
||||
/** completed:三步完成;attention:目标已就绪但删源失败/无法确认;failed:目标未提交,源保持不动 */
|
||||
status: "completed" | "attention" | "failed";
|
||||
steps: CrossMoveStepLog[];
|
||||
}
|
||||
|
||||
export interface CrossMoveOptions extends Omit<CopyOptions, "hash"> {
|
||||
hash?: VerifyMethod;
|
||||
/** 上次运行的步骤日志,用于中断续跑(只作参考,实际仍以文件系统观察为准) */
|
||||
previous?: CrossMoveStepLog[];
|
||||
}
|
||||
|
||||
async function exists(p: string): Promise<boolean> {
|
||||
try {
|
||||
await stat(p);
|
||||
return true;
|
||||
} catch (e) {
|
||||
if (errnoOf(e) === "ENOENT") return false;
|
||||
throw toFsError(e, p);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 跨设备搬运:copy → verify → delete_source,每步返回日志,可重入:
|
||||
* - 目标已存在且与源一致 → copy 记为 skipped(上次已完成);不一致 → failed,不覆盖
|
||||
* - 源已不存在但目标在,且上次日志 verify=done → 视为删源已完成
|
||||
* - 删源失败 → attention(不能报成功)
|
||||
*/
|
||||
export async function crossDeviceMove(src: string, dst: string, opts: CrossMoveOptions = {}): Promise<CrossMoveResult> {
|
||||
const method = opts.hash ?? "size";
|
||||
const steps: CrossMoveStepLog[] = [];
|
||||
const log = (step: CrossMoveStep, status: CrossMoveStepLog["status"], detail?: string, err?: unknown) => {
|
||||
const e = err === undefined ? undefined : toFsError(err);
|
||||
steps.push({
|
||||
step,
|
||||
status,
|
||||
at: new Date().toISOString(),
|
||||
...(detail ? { detail } : {}),
|
||||
...(e ? { error: { code: e.code, message: e.message } } : {}),
|
||||
});
|
||||
};
|
||||
|
||||
let srcExists: boolean;
|
||||
let dstExists: boolean;
|
||||
try {
|
||||
[srcExists, dstExists] = await Promise.all([exists(src), exists(dst)]);
|
||||
} catch (e) {
|
||||
log("copy", "failed", "observe failed", e);
|
||||
return { status: "failed", steps };
|
||||
}
|
||||
|
||||
if (!srcExists) {
|
||||
const verified = opts.previous?.some((s) => s.step === "verify" && s.status === "done");
|
||||
if (dstExists && verified) {
|
||||
log("copy", "skipped", "already applied");
|
||||
log("verify", "skipped", "verified in previous run");
|
||||
log("delete_source", "skipped", "source already removed");
|
||||
return { status: "completed", steps };
|
||||
}
|
||||
log("copy", "failed", dstExists ? "source missing, target cannot be verified" : "source missing", undefined);
|
||||
return { status: dstExists ? "attention" : "failed", steps };
|
||||
}
|
||||
|
||||
if (dstExists) {
|
||||
try {
|
||||
if (await filesMatch(src, dst, method)) log("copy", "skipped", "target already matches source");
|
||||
else {
|
||||
log("copy", "failed", "target exists and differs", new FsError("EEXIST", `target exists: ${dst}`));
|
||||
return { status: "failed", steps };
|
||||
}
|
||||
} catch (e) {
|
||||
log("copy", "failed", "compare failed", e);
|
||||
return { status: "failed", steps };
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
const { previous: _p, ...copyOpts } = opts;
|
||||
const r = await copyWithVerify(src, dst, { ...copyOpts, hash: method });
|
||||
log("copy", "done", `${r.bytes} bytes`);
|
||||
} catch (e) {
|
||||
log("copy", "failed", undefined, e);
|
||||
return { status: "failed", steps };
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
if (!(await filesMatch(src, dst, method))) {
|
||||
log("verify", "failed", undefined, new FsError("VERIFY_FAILED", "target differs from source"));
|
||||
return { status: "failed", steps };
|
||||
}
|
||||
log("verify", "done", method);
|
||||
} catch (e) {
|
||||
log("verify", "failed", undefined, e);
|
||||
return { status: "failed", steps };
|
||||
}
|
||||
|
||||
try {
|
||||
await unlink(src);
|
||||
log("delete_source", "done", dirname(src));
|
||||
} catch (e) {
|
||||
log("delete_source", "failed", undefined, e);
|
||||
return { status: "attention", steps };
|
||||
}
|
||||
return { status: "completed", steps };
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
// 本项目新增(REQ §8.1/§8.2 分批增量扫描与中断续扫);MP 整理监控为全量目录遍历,无游标。
|
||||
import type { Dirent } from "node:fs";
|
||||
import { lstat, readdir } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { errnoOf } from "./errors";
|
||||
import { checkRoot, DEFAULT_ROOT_SENTINEL } from "./stat";
|
||||
|
||||
export interface ScanEntry {
|
||||
/** 相对根目录的路径,/ 分隔 */
|
||||
relPath: string;
|
||||
size: number;
|
||||
mtimeMs: number;
|
||||
ino: number;
|
||||
dev: number;
|
||||
nlink: number;
|
||||
isSymlink: boolean;
|
||||
change: "new" | "changed" | "unchanged";
|
||||
}
|
||||
|
||||
export interface ScanOptions {
|
||||
/** 上次返回的游标;空则从头扫 */
|
||||
cursor?: string | null;
|
||||
/** 本批最多 lstat 的文件数(I/O 上限),默认 2000 */
|
||||
maxStats?: number;
|
||||
/** 本批最多 readdir 的目录数,默认不限 */
|
||||
maxDirs?: number;
|
||||
/** 哨兵文件名;false 表示不检查(不推荐) */
|
||||
sentinel?: string | false;
|
||||
/** 已知记录,用于标注 new/changed/unchanged */
|
||||
previous?: (relPath: string) => { size: number; mtimeMs: number } | undefined;
|
||||
/** 跳过的文件名判定(默认跳过 .uhd-partial 与 .!qB 未完成文件) */
|
||||
skip?: (name: string) => boolean;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export interface ScanResult {
|
||||
status: "complete" | "partial" | "root_offline" | "aborted";
|
||||
entries: ScanEntry[];
|
||||
/** 继续扫描用的游标;complete 时为 null */
|
||||
cursor: string | null;
|
||||
statCount: number;
|
||||
dirCount: number;
|
||||
errors: { relPath: string; code: string }[];
|
||||
}
|
||||
|
||||
const defaultSkip = (n: string) => n.endsWith(".uhd-partial") || n.endsWith(".!qB");
|
||||
|
||||
function encodeCursor(after: string[]): string {
|
||||
return Buffer.from(JSON.stringify({ v: 1, after })).toString("base64url");
|
||||
}
|
||||
function decodeCursor(c: string | null | undefined): string[] | null {
|
||||
if (!c) return null;
|
||||
try {
|
||||
const j = JSON.parse(Buffer.from(c, "base64url").toString()) as { v: number; after: string[] };
|
||||
return j.v === 1 && Array.isArray(j.after) ? j.after : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
function cmp(a: string, b: string): number {
|
||||
return a < b ? -1 : a > b ? 1 : 0;
|
||||
}
|
||||
/** 段序比较(深度优先前序、名称排序的遍历顺序) */
|
||||
function cmpSegs(a: string[], b: string[]): number {
|
||||
for (let i = 0; i < Math.min(a.length, b.length); i++) {
|
||||
const c = cmp(a[i]!, b[i]!);
|
||||
if (c) return c;
|
||||
}
|
||||
return a.length - b.length;
|
||||
}
|
||||
function isPrefix(p: string[], of: string[]): boolean {
|
||||
return p.length < of.length && p.every((s, i) => s === of[i]);
|
||||
}
|
||||
|
||||
/**
|
||||
* 分批增量扫描:确定性顺序(名称排序、深度优先),每批受 maxStats/maxDirs 限制,
|
||||
* 返回游标供下一批续扫;进程中断后用最后持久化的游标续扫,不必从头遍历。
|
||||
* 目录只 readdir 不 stat;文件 lstat(不跟随符号链接)。
|
||||
*/
|
||||
export async function scanIncremental(root: string, opts: ScanOptions = {}): Promise<ScanResult> {
|
||||
const result: ScanResult = { status: "complete", entries: [], cursor: null, statCount: 0, dirCount: 0, errors: [] };
|
||||
if (opts.sentinel !== false) {
|
||||
const h = await checkRoot(root, opts.sentinel ?? DEFAULT_ROOT_SENTINEL);
|
||||
if (h.kind === "offline") return { ...result, status: "root_offline" };
|
||||
}
|
||||
const maxStats = opts.maxStats ?? 2000;
|
||||
const maxDirs = opts.maxDirs ?? Number.POSITIVE_INFINITY;
|
||||
const skip = opts.skip ?? defaultSkip;
|
||||
const sentinelName = opts.sentinel === false ? null : (opts.sentinel ?? DEFAULT_ROOT_SENTINEL);
|
||||
const after = decodeCursor(opts.cursor);
|
||||
let last: string[] | null = after;
|
||||
let stopped: ScanResult["status"] | null = null;
|
||||
|
||||
const walk = async (dirSegs: string[]): Promise<void> => {
|
||||
if (result.dirCount >= maxDirs) {
|
||||
stopped = "partial";
|
||||
return;
|
||||
}
|
||||
let ents: Dirent[];
|
||||
try {
|
||||
ents = await readdir(join(root, ...dirSegs), { withFileTypes: true });
|
||||
result.dirCount++;
|
||||
} catch (e) {
|
||||
result.errors.push({ relPath: dirSegs.join("/"), code: errnoOf(e) ?? "UNKNOWN" });
|
||||
return;
|
||||
}
|
||||
ents.sort((a, b) => cmp(a.name, b.name));
|
||||
for (const d of ents) {
|
||||
if (opts.signal?.aborted) {
|
||||
stopped = "aborted";
|
||||
return;
|
||||
}
|
||||
const segs = [...dirSegs, d.name];
|
||||
if (dirSegs.length === 0 && d.name === sentinelName) continue;
|
||||
if (after && cmpSegs(segs, after) <= 0 && !isPrefix(segs, after)) continue;
|
||||
if (d.isDirectory()) {
|
||||
await walk(segs);
|
||||
if (stopped) return;
|
||||
continue;
|
||||
}
|
||||
if (skip(d.name)) continue;
|
||||
if (result.statCount >= maxStats) {
|
||||
stopped = "partial";
|
||||
return;
|
||||
}
|
||||
const rel = segs.join("/");
|
||||
try {
|
||||
const st = await lstat(join(root, ...segs));
|
||||
result.statCount++;
|
||||
if (st.isDirectory()) continue;
|
||||
const prev = opts.previous?.(rel);
|
||||
result.entries.push({
|
||||
relPath: rel,
|
||||
size: st.size,
|
||||
mtimeMs: st.mtimeMs,
|
||||
ino: st.ino,
|
||||
dev: st.dev,
|
||||
nlink: st.nlink,
|
||||
isSymlink: st.isSymbolicLink(),
|
||||
change: !prev ? "new" : prev.size === st.size && prev.mtimeMs === st.mtimeMs ? "unchanged" : "changed",
|
||||
});
|
||||
} catch (e) {
|
||||
result.statCount++;
|
||||
result.errors.push({ relPath: rel, code: errnoOf(e) ?? "UNKNOWN" });
|
||||
}
|
||||
last = segs;
|
||||
}
|
||||
};
|
||||
|
||||
await walk([]);
|
||||
if (stopped) {
|
||||
result.status = stopped;
|
||||
result.cursor = last ? encodeCursor(last) : (opts.cursor ?? null);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
// 本项目新增(REQ §8.2 挂载根哨兵);MP 无对应实现。
|
||||
import { lstat, stat, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { errnoOf } from "./errors";
|
||||
|
||||
export const DEFAULT_ROOT_SENTINEL = ".uhd-root";
|
||||
|
||||
export interface FileFacts {
|
||||
size: number;
|
||||
mtimeMs: number;
|
||||
ino: number;
|
||||
dev: number;
|
||||
nlink: number;
|
||||
isFile: boolean;
|
||||
isDirectory: boolean;
|
||||
isSymbolicLink: boolean;
|
||||
}
|
||||
|
||||
export type StatResult =
|
||||
| { kind: "ok"; facts: FileFacts }
|
||||
| { kind: "missing" }
|
||||
| { kind: "permission_denied" }
|
||||
/** 挂载根哨兵不存在/不可读:不能据此判定文件缺失 */
|
||||
| { kind: "root_offline"; root: string; reason: string }
|
||||
| { kind: "error"; code: string; message: string };
|
||||
|
||||
export type RootHealth =
|
||||
| { kind: "online"; dev: number }
|
||||
| { kind: "offline"; reason: "sentinel_missing" | "root_missing" | "permission_denied" | "error"; code?: string };
|
||||
|
||||
/** 检查库根/缓存根是否在线:根目录存在且哨兵文件存在。 */
|
||||
export async function checkRoot(root: string, sentinel: string = DEFAULT_ROOT_SENTINEL): Promise<RootHealth> {
|
||||
let dev: number;
|
||||
try {
|
||||
const st = await stat(root);
|
||||
dev = st.dev;
|
||||
} catch (e) {
|
||||
const code = errnoOf(e);
|
||||
if (code === "ENOENT") return { kind: "offline", reason: "root_missing" };
|
||||
if (code === "EACCES" || code === "EPERM") return { kind: "offline", reason: "permission_denied", code };
|
||||
return { kind: "offline", reason: "error", code: code ?? "UNKNOWN" };
|
||||
}
|
||||
try {
|
||||
await stat(join(root, sentinel));
|
||||
return { kind: "online", dev };
|
||||
} catch (e) {
|
||||
const code = errnoOf(e);
|
||||
if (code === "ENOENT") return { kind: "offline", reason: "sentinel_missing" };
|
||||
if (code === "EACCES" || code === "EPERM") return { kind: "offline", reason: "permission_denied", code };
|
||||
return { kind: "offline", reason: "error", code: code ?? "UNKNOWN" };
|
||||
}
|
||||
}
|
||||
|
||||
/** 在根目录写入哨兵文件(首次登记库根时调用)。 */
|
||||
export async function writeRootSentinel(root: string, sentinel: string = DEFAULT_ROOT_SENTINEL): Promise<void> {
|
||||
await writeFile(join(root, sentinel), `uhd-media-tool root sentinel\ncreated=${new Date().toISOString()}\n`, {
|
||||
flag: "wx",
|
||||
}).catch((e) => {
|
||||
if (errnoOf(e) !== "EEXIST") throw e;
|
||||
});
|
||||
}
|
||||
|
||||
function facts(st: Awaited<ReturnType<typeof lstat>>): FileFacts {
|
||||
return {
|
||||
size: Number(st.size),
|
||||
mtimeMs: Number(st.mtimeMs),
|
||||
ino: Number(st.ino),
|
||||
dev: Number(st.dev),
|
||||
nlink: Number(st.nlink),
|
||||
isFile: st.isFile(),
|
||||
isDirectory: st.isDirectory(),
|
||||
isSymbolicLink: st.isSymbolicLink(),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 区分 ENOENT / EACCES / 挂载根离线的 stat。给出 root 时先检查哨兵,
|
||||
* 根离线返回 root_offline 而不是 missing(REQ §8.2)。不抛异常。
|
||||
*/
|
||||
export async function statSafe(path: string, opts: { root?: string; sentinel?: string } = {}): Promise<StatResult> {
|
||||
if (opts.root) {
|
||||
const h = await checkRoot(opts.root, opts.sentinel);
|
||||
if (h.kind === "offline") return { kind: "root_offline", root: opts.root, reason: h.reason };
|
||||
}
|
||||
try {
|
||||
return { kind: "ok", facts: facts(await lstat(path)) };
|
||||
} catch (e) {
|
||||
const code = errnoOf(e);
|
||||
if (code === "ENOENT" || code === "ENOTDIR") return { kind: "missing" };
|
||||
if (code === "EACCES" || code === "EPERM") return { kind: "permission_denied" };
|
||||
return { kind: "error", code: code ?? "UNKNOWN", message: (e as Error)?.message ?? String(e) };
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user