From fed2202e745b87be376c1051975e38e077ed40db Mon Sep 17 00:00:00 2001 From: root Date: Wed, 7 Oct 2026 00:01:21 +0800 Subject: [PATCH] =?UTF-8?q?test(storage):=20=E7=9C=9F=E5=AE=9E=20OpenList?= =?UTF-8?q?=20=E5=BE=80=E8=BF=94=E6=B5=8B=E8=AF=95=E4=B8=8E=20alist-e2e=20?= =?UTF-8?q?=E9=AA=8C=E6=94=B6=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - packages/adapters alist.real.test.ts:上传 / 冲突 / 同驱动与跨驱动复制移动 / 下载比对(环境变量开启) - apps/server alist-organize.real.test.ts:本地 → OpenList(整理复制)→ 本地(移动)→ OpenList 内整理 - scripts/alist-e2e.ts:对任意地址 + 令牌或账号密码跑往返,退出码表示是否全部通过 - 写后目标暂不可见时刷新父目录列表(OpenList 的 get 不认 refresh) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei --- .../test/storage/alist-organize.real.test.ts | 158 ++++++++++++++++++ .../adapters/src/alist/alist.real.test.ts | 102 +++++++++++ packages/adapters/src/alist/client.ts | 10 +- scripts/alist-e2e.ts | 140 ++++++++++++++++ 4 files changed, 408 insertions(+), 2 deletions(-) create mode 100644 apps/server/test/storage/alist-organize.real.test.ts create mode 100644 packages/adapters/src/alist/alist.real.test.ts create mode 100644 scripts/alist-e2e.ts diff --git a/apps/server/test/storage/alist-organize.real.test.ts b/apps/server/test/storage/alist-organize.real.test.ts new file mode 100644 index 00000000..ba706598 --- /dev/null +++ b/apps/server/test/storage/alist-organize.real.test.ts @@ -0,0 +1,158 @@ +import { randomBytes } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { type AlistAuth, AlistClient } from "@umt/adapters"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { executeRecord } from "../../src/modules/acquisition-organize/services/organize.ts"; +import { overrideStorage } from "../../src/storage/registry.ts"; +import { toVPath } from "../../src/storage/vpath.ts"; +import { createFmHarness, type FmHarness, key, pgAvailable, seedEntity } from "../file-manager/harness.ts"; + +// 真实 OpenList 上跑一次整理往返(环境变量开启,缺省跳过;变量同 packages/adapters/src/alist/alist.real.test.ts): +// 本地下载目录 → 整理(复制)到 OpenList 上的媒体库 → 从 OpenList 整理(移动)回本地媒体库 → OpenList 内移动。 + +const URL_ = Deno.env.get("UMT_ALIST_REAL_URL"); +const ROOT = Deno.env.get("UMT_ALIST_REAL_ROOT"); +const auth: AlistAuth | null = Deno.env.get("UMT_ALIST_REAL_TOKEN") + ? { kind: "token", token: Deno.env.get("UMT_ALIST_REAL_TOKEN")! } + : Deno.env.get("UMT_ALIST_REAL_USER") + ? { + kind: "password", + username: Deno.env.get("UMT_ALIST_REAL_USER")!, + password: Deno.env.get("UMT_ALIST_REAL_PASSWORD") ?? "", + } + : null; +const enabled = !!URL_ && !!ROOT && !!auth && (await pgAvailable()); + +describe.skipIf(!enabled)("真实 OpenList:整理往返 本地 → OpenList → 本地", () => { + let h: FmHarness; + let client: AlistClient; + let storageId: string; + let base: string; + let libId: string; + const body = `real-video-${randomBytes(64).toString("hex")}`; + beforeAll(async () => { + h = await createFmHarness("alist_real", {}, { runTransfers: false }); + client = new AlistClient({ baseUrl: URL_!, auth: auth! }); + base = `${ROOT!.replace(/\/+$/, "")}/umt-org-${randomBytes(3).toString("hex")}`; + await client.mkdir(`${base}/library`); + await client.mkdir(`${base}/incoming`); + const row = await h.db + .insertInto("storage_instance") + .values({ name: "真实 OpenList", base_url: URL_!, root_path: base }) + .returning("id") + .executeTakeFirstOrThrow(); + storageId = row.id; + overrideStorage(storageId, { client, name: "真实 OpenList", rootPath: base }); + libId = ( + await h.db + .insertInto("library_root") + .values({ + name: "OpenList 媒体库", + media_kind: "mixed", + local_path: toVPath(storageId, `${base}/library`), + management_mode: "managed", + sentinel_file: null, + }) + .returning("id") + .executeTakeFirstOrThrow() + ).id; + }, 60_000); + afterAll(async () => { + await client?.removePath(base).catch(() => undefined); + await h?.close(); + }); + + async function organize(items: unknown[], action: "copy" | "move", targetLibraryRootId: string) { + const dry = await h.api("POST", "/files/organize", { + idempotencyKey: key(), + items, + action, + targetLibraryRootId, + dryRun: true, + }); + expect(dry.body.data.groups[0].error, JSON.stringify(dry.body)).toBeNull(); + const exec = await h.api("POST", "/files/organize", { + idempotencyKey: key(), + items, + action, + targetLibraryRootId, + dryRun: false, + planIds: [dry.body.data.groups[0].planId], + }); + const recordId = exec.body.data.recordIds[0] as string; + await executeRecord(h.module.deps().ao, recordId); + const rec = await h.db + .selectFrom("organization_record") + .select(["status"]) + .where("id", "=", recordId) + .executeTakeFirstOrThrow(); + const main = await h.db + .selectFrom("organization_file_result") + .select(["target_path", "checksum", "status"]) + .where("organization_record_id", "=", recordId) + .where("role", "=", "main") + .executeTakeFirstOrThrow(); + return { status: rec.status, main }; + } + + it("本地 → OpenList(复制)→ 本地(移动)→ OpenList 内移动", async () => { + const film = await seedEntity(h.db, { kind: "movie", title: "Real Round Trip", year: 2024 }); + const name = "Real.Round.Trip.2024.1080p.WEB-DL.mkv"; + await writeFile(join(h.dl, name), body); + + const up = await organize([{ root: h.dlRoot, path: name, mediaEntityId: film }], "copy", libId); + console.log(`[real] 本地 → OpenList 复制：${up.status}，目标 ${up.main.target_path}，${up.main.checksum}`); + expect(up.status).toBe("succeeded"); + const remoteFile = `${base}/library/${up.main.target_path}`; + expect((await client.get(remoteFile, { refresh: true })).size).toBe(body.length); + + // 从 OpenList 移回本地媒体库(文件管理「移动到」:下载成功后删远端源,已登记位置跟随到本地) + const rel = `library/${up.main.target_path}`; + const src = { root: `st:${storageId}`, path: rel }; + const dryMv = await h.api("POST", "/files/move", { + idempotencyKey: key(), + items: [src], + target: { root: h.libRoot, path: "" }, + mode: "move", + dryRun: true, + }); + expect(dryMv.status, JSON.stringify(dryMv.body)).toBe(200); + const mv = await h.api("POST", "/files/move", { + idempotencyKey: key(), + items: [src], + target: { root: h.libRoot, path: "" }, + mode: "move", + dryRun: false, + planId: dryMv.body.data.planId, + }); + expect(mv.status, JSON.stringify(mv.body)).toBe(200); + const recordId = mv.body.data.executed.recordId as string; + await executeRecord(h.module.deps().ao, recordId); + const back = await h.db + .selectFrom("organization_record") + .select("status") + .where("id", "=", recordId) + .executeTakeFirstOrThrow(); + const localName = up.main.target_path!.split("/").pop()!; + console.log(`[real] OpenList → 本地 移动：${back.status}，目标 ${localName}`); + expect(back.status).toBe("succeeded"); + expect(await readFile(join(h.lib, localName), "utf8")).toBe(body); + expect(await client.stat(remoteFile, { refresh: true })).toBeNull(); + + // OpenList 内整理(同实例 API move):另一部作品放在 OpenList 的 incoming + const film2 = await seedEntity(h.db, { kind: "movie", title: "Real Inside Move", year: 2023 }); + const name2 = "Real.Inside.Move.2023.1080p.WEB-DL.mkv"; + await writeFile(join(h.dl, name2), body); + await client.upload(join(h.dl, name2), `${base}/incoming/${name2}`); + const inside = await organize( + [{ root: `st:${storageId}`, path: `incoming/${name2}`, mediaEntityId: film2 }], + "move", + libId, + ); + console.log(`[real] OpenList 内移动：${inside.status}，目标 ${inside.main.target_path}`); + expect(inside.status).toBe("succeeded"); + expect(await client.stat(`${base}/incoming/${name2}`, { refresh: true })).toBeNull(); + expect((await client.get(`${base}/library/${inside.main.target_path}`, { refresh: true })).size).toBe(body.length); + }, 120_000); +}); diff --git a/packages/adapters/src/alist/alist.real.test.ts b/packages/adapters/src/alist/alist.real.test.ts new file mode 100644 index 00000000..5e670932 --- /dev/null +++ b/packages/adapters/src/alist/alist.real.test.ts @@ -0,0 +1,102 @@ +import { createHash, randomBytes } from "node:crypto"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { type AlistAuth, AlistClient } from "./client.ts"; + +// 真实 Alist / OpenList(环境变量开启,缺省跳过): +// UMT_ALIST_REAL_URL 地址,如 http://127.0.0.1:55244 +// UMT_ALIST_REAL_TOKEN 永久令牌;或 UMT_ALIST_REAL_USER + UMT_ALIST_REAL_PASSWORD +// UMT_ALIST_REAL_ROOT 可写测试目录(会在里面建 umt-it-<随机>/,结束删除),如 /s1 +// UMT_ALIST_REAL_ROOT2 可选:另一个挂载(驱动)下的可写目录,测跨驱动复制 / 移动(异步任务) +// 跑法:docs/runbook/storage-alist.md「真机验收」。 + +const URL_ = Deno.env.get("UMT_ALIST_REAL_URL"); +const ROOT = Deno.env.get("UMT_ALIST_REAL_ROOT"); +const ROOT2 = Deno.env.get("UMT_ALIST_REAL_ROOT2"); +const auth: AlistAuth | null = Deno.env.get("UMT_ALIST_REAL_TOKEN") + ? { kind: "token", token: Deno.env.get("UMT_ALIST_REAL_TOKEN")! } + : Deno.env.get("UMT_ALIST_REAL_USER") + ? { + kind: "password", + username: Deno.env.get("UMT_ALIST_REAL_USER")!, + password: Deno.env.get("UMT_ALIST_REAL_PASSWORD") ?? "", + } + : null; +const enabled = !!URL_ && !!ROOT && !!auth; +const sha = (b: Uint8Array) => createHash("sha256").update(b).digest("hex"); + +describe.skipIf(!enabled)("真实 Alist / OpenList:本地 → Alist → 本地往返", () => { + let c: AlistClient; + let dir: string; + let remote: string; + let remote2: string | null = null; + const data = randomBytes(3 * 1024 * 1024 + 123); + beforeAll(async () => { + c = new AlistClient({ baseUrl: URL_!, auth: auth!, qps: 10, concurrency: 4 }); + dir = await mkdtemp(join(tmpdir(), "umt-alist-real-")); + remote = `${ROOT!.replace(/\/+$/, "")}/umt-it-${randomBytes(3).toString("hex")}`; + if (ROOT2) remote2 = `${ROOT2.replace(/\/+$/, "")}/umt-it-${randomBytes(3).toString("hex")}`; + await writeFile(join(dir, "source.bin"), data); + }); + afterAll(async () => { + await c?.removePath(remote).catch(() => undefined); + if (remote2) await c?.removePath(remote2).catch(() => undefined); + await rm(dir, { recursive: true, force: true }); + }); + + it("连接:版本 / 用户 / 根目录", async () => { + const v = await c.version(); + const me = await c.me(); + console.log(`[real] Alist 版本 ${v}，用户 ${me.username}`); + expect((await c.get(ROOT!)).isDir).toBe(true); + }); + + it("上传(临时名 → 改名、带秒传头)→ 同名冲突不覆盖", async () => { + let last = 0; + const r = await c.upload(join(dir, "source.bin"), `${remote}/电影 A/source.bin`, { onProgress: (d) => (last = d) }); + console.log(`[real] 上传 ${r.bytes} 字节，sha256 ${r.hashes.sha256.slice(0, 12)}…，进度回调到 ${last}`); + expect(r.obj.size).toBe(data.length); + expect(last).toBe(data.length); + const names = (await c.list(`${remote}/电影 A`, { refresh: true })).map((o) => o.name); + expect(names).toEqual(["source.bin"]); + await expect(c.upload(join(dir, "source.bin"), `${remote}/电影 A/source.bin`)).rejects.toMatchObject({ + kind: "conflict", + }); + }); + + it("同驱动复制(改名走暂存目录)/ 移动 / 改名", async () => { + const cp = await c.copyFile(`${remote}/电影 A/source.bin`, `${remote}/library/Movie (2020).bin`); + expect(cp.size).toBe(data.length); + const mv = await c.moveFile(`${remote}/library/Movie (2020).bin`, `${remote}/library2/Moved.bin`); + expect(mv.path).toBe(`${remote}/library2/Moved.bin`); + expect(await c.stat(`${remote}/library/Movie (2020).bin`)).toBeNull(); + const lib = (await c.list(`${remote}/library`, { refresh: true })).map((o) => o.name); + expect(lib.filter((n) => n.startsWith(".umt-stage-"))).toEqual([]); + console.log(`[real] Alist 内复制、移动完成，暂存目录已清理`); + }); + + it.skipIf(!ROOT2)("跨驱动复制 / 移动:轮询 Alist 任务到完成", async () => { + const cp = await c.copyFile(`${remote}/library2/Moved.bin`, `${remote2}/cross/Copied.bin`); + expect(cp.size).toBe(data.length); + const mv = await c.moveFile(`${remote2}/cross/Copied.bin`, `${remote}/back/Back.bin`); + expect(mv.size).toBe(data.length); + console.log(`[real] 跨驱动复制、移动完成(任务轮询)`); + }); + + it("下载回本地:内容与源一致;本地已存在冲突", async () => { + const out = join(dir, "back.bin"); + const r = await c.download(`${remote}/library2/Moved.bin`, out); + const got = new Uint8Array(await readFile(out)); + console.log(`[real] 下载 ${r.bytes} 字节，与源 sha256 一致：${sha(got) === sha(data)}`); + expect(sha(got)).toBe(sha(data)); + await expect(c.download(`${remote}/library2/Moved.bin`, out)).rejects.toMatchObject({ kind: "conflict" }); + }); + + it("不存在 ≠ 失败;容量拿不到返回 null 不报错", async () => { + await expect(c.get(`${remote}/nope.bin`)).rejects.toMatchObject({ kind: "not_found" }); + const u = await c.usage(ROOT!); + console.log(`[real] 容量：${u ? `${u.free}/${u.total}` : "未知"}`); + }); +}); diff --git a/packages/adapters/src/alist/client.ts b/packages/adapters/src/alist/client.ts index 74eac551..edcabceb 100644 --- a/packages/adapters/src/alist/client.ts +++ b/packages/adapters/src/alist/client.ts @@ -596,11 +596,17 @@ export class AlistClient { async waitExists(path: string, opts: { timeoutMs?: number; signal?: AbortSignal } = {}): Promise { const deadline = this.o.now() + (opts.timeoutMs ?? 60_000); let delay = 500; + const p = normAlistPath(path); for (;;) { opts.signal?.throwIfAborted(); - const o = await this.stat(path, { refresh: true }); + const o = await this.stat(p, { refresh: true }); if (o) return o; - if (this.o.now() >= deadline) throw new AlistError("not_found", `写入后目标仍不存在：${normAlistPath(path)}`); + // OpenList 的 get 不认 refresh:刷新父目录列表让缓存失效,再按列表结果判断 + const hit = (await this.list(posix.dirname(p), { refresh: true }).catch(() => [])).find( + (x) => x.name === posix.basename(p), + ); + if (hit) return hit; + if (this.o.now() >= deadline) throw new AlistError("not_found", `写入后目标仍不存在：${p}`); await this.o.sleep(delay); delay = Math.min(delay * 2, 10_000); } diff --git a/scripts/alist-e2e.ts b/scripts/alist-e2e.ts new file mode 100644 index 00000000..30e9af9c --- /dev/null +++ b/scripts/alist-e2e.ts @@ -0,0 +1,140 @@ +// Alist / OpenList 往返验收脚本(docs/runbook/storage-alist.md「真机验收」):对任意地址 + 令牌(或账号密码) +// 用本项目的适配器跑一遍:连接 → 建目录 → 上传(秒传头、临时名改名、进度)→ 同名冲突不覆盖 → 同驱动复制(改名) +// → 移动 → 可选跨驱动复制 / 移动(Alist 异步任务)→ 下载回本地比对 sha256 → 不存在 ≠ 失败 → 清理。 +// +// 用法: +// deno run -A scripts/alist-e2e.ts --url http://127.0.0.1:5244 --token <令牌> --root /s1 [--root2 /s2] [--size-mb 8] +// deno run -A scripts/alist-e2e.ts --url ... --user admin --password <密码> --root /s1 +// 也可用环境变量 UMT_ALIST_REAL_URL / _TOKEN / _USER / _PASSWORD / _ROOT / _ROOT2。退出码 0 = 全部通过。 +import { createHash, randomBytes } from "node:crypto"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { parseArgs } from "node:util"; +import { type AlistAuth, AlistClient, AlistError, redactAlistText } from "../packages/adapters/src/alist/index.ts"; + +const { values: a } = parseArgs({ + args: Deno.args, + options: { + url: { type: "string" }, + token: { type: "string" }, + user: { type: "string" }, + password: { type: "string" }, + root: { type: "string" }, + root2: { type: "string" }, + "size-mb": { type: "string" }, + keep: { type: "boolean" }, + }, +}); +const env = (k: string) => Deno.env.get(`UMT_ALIST_REAL_${k}`); +const url = a.url ?? env("URL"); +const token = a.token ?? env("TOKEN"); +const user = a.user ?? env("USER"); +const password = a.password ?? env("PASSWORD"); +const root = (a.root ?? env("ROOT") ?? "").replace(/\/+$/, ""); +const root2 = (a.root2 ?? env("ROOT2") ?? "").replace(/\/+$/, ""); +if (!url || !root || (!token && !user)) { + console.error( + "用法：--url <地址> (--token <令牌> | --user <用户> --password <密码>) --root <可写目录> [--root2 <另一驱动目录>]", + ); + Deno.exit(2); +} +const auth: AlistAuth = token + ? { kind: "token", token } + : { kind: "password", username: user!, password: password ?? "" }; +const secrets = [token, password]; +const c = new AlistClient({ baseUrl: url, auth }); +const tag = `umt-e2e-${randomBytes(3).toString("hex")}`; +const base = `${root}/${tag}`; +const base2 = root2 ? `${root2}/${tag}` : null; +const size = Math.max(1, Number(a["size-mb"] ?? 8)) * 1024 * 1024 + 7; +const sha = (b: Uint8Array) => createHash("sha256").update(b).digest("hex"); + +let failed = 0; +async function step(name: string, fn: () => Promise) { + const t0 = performance.now(); + try { + const note = await fn(); + console.log(`✓ ${name}（${Math.round(performance.now() - t0)} ms）${note ? `：${note}` : ""}`); + } catch (e) { + failed++; + const msg = e instanceof AlistError ? `${e.kind}：${e.message}` : String((e as Error)?.message ?? e); + console.log(`✗ ${name}（${Math.round(performance.now() - t0)} ms）：${redactAlistText(msg, secrets)}`); + } +} +const expectKind = async (p: Promise, kind: string) => { + try { + await p; + } catch (e) { + if (e instanceof AlistError && e.kind === kind) return; + throw e; + } + throw new Error(`应当失败（${kind}），却成功了`); +}; + +const dir = await mkdtemp(join(tmpdir(), "umt-alist-e2e-")); +const data = randomBytes(size); +await writeFile(join(dir, "source.bin"), data); +console.log( + `Alist 往返验收：${url} 测试目录 ${base}${base2 ? `、${base2}` : ""} 文件 ${(size / 1048576).toFixed(1)} MiB`, +); + +await step("连接（版本、用户、根目录）", async () => { + const v = await c.version(); + const me = await c.me(); + if (!(await c.get(root)).isDir) throw new Error(`${root} 不是目录`); + return `${v ?? "版本未知"}，用户 ${me.username}`; +}); +await step("建目录", async () => { + await c.mkdir(`${base}/电影 A`); +}); +await step("上传（秒传头、临时名 → 改名、进度回调）", async () => { + let last = 0; + const r = await c.upload(join(dir, "source.bin"), `${base}/电影 A/source.bin`, { onProgress: (d) => (last = d) }); + if (last !== size) throw new Error(`进度只到 ${last}/${size}`); + return `${r.bytes} 字节，sha256 ${r.hashes.sha256.slice(0, 12)}…`; +}); +await step("同名不覆盖（上传 / 复制都报冲突）", async () => { + await expectKind(c.upload(join(dir, "source.bin"), `${base}/电影 A/source.bin`), "conflict"); + await c.mkdir(`${base}/dup`); + await c.copyFile(`${base}/电影 A/source.bin`, `${base}/dup/source.bin`); + await expectKind(c.copyFile(`${base}/电影 A/source.bin`, `${base}/dup/source.bin`), "conflict"); +}); +await step("同驱动复制并改名（暂存目录）", async () => { + const o = await c.copyFile(`${base}/电影 A/source.bin`, `${base}/library/Movie (2020).bin`); + const left = (await c.list(`${base}/library`, { refresh: true })).filter((x) => x.name.startsWith(".umt-stage-")); + if (left.length) throw new Error("暂存目录没清理"); + return `${o.size} 字节`; +}); +await step("同驱动移动", async () => { + await c.moveFile(`${base}/library/Movie (2020).bin`, `${base}/moved/Moved.bin`); + if (await c.stat(`${base}/library/Movie (2020).bin`, { refresh: true })) throw new Error("源还在"); +}); +if (base2) { + await step("跨驱动复制 / 移动（等 Alist 任务完成）", async () => { + await c.copyFile(`${base}/moved/Moved.bin`, `${base2}/cross/Copied.bin`); + await c.moveFile(`${base2}/cross/Copied.bin`, `${base}/back/Back.bin`); + return "完成"; + }); +} +await step("下载回本地并比对", async () => { + const out = join(dir, "back.bin"); + const r = await c.download(`${base}/moved/Moved.bin`, out); + const ok = sha(new Uint8Array(await readFile(out))) === sha(data); + if (!ok) throw new Error("内容与源不一致"); + return `${r.bytes} 字节，sha256 一致`; +}); +await step("不存在 ≠ 失败；容量拿不到不报错", async () => { + await expectKind(c.get(`${base}/nope.bin`), "not_found"); + const u = await c.usage(root); + return u ? `剩余 ${(u.free / 1073741824).toFixed(1)} GiB / ${(u.total / 1073741824).toFixed(1)} GiB` : "容量未知"; +}); +if (!a.keep) { + await step("清理测试目录", async () => { + await c.removePath(base); + if (base2) await c.removePath(base2); + }); +} +await rm(dir, { recursive: true, force: true }); +console.log(failed ? `\n${failed} 步失败` : "\n全部通过"); +Deno.exit(failed ? 1 : 0);