feat(storage): 存储实例与 Alist 虚拟路径(迁移 0090、设置接口、注册表、vfs、健康检查)
- storage_instance 表 + secret 用途 storage_credential;令牌 / 密码 / 路径密码只写不读 - /storage-instances CRUD 与测试连接(版本、用户、根路径、容量);被库根引用不能删 - 远端路径 /@alist/<id>/<路径>,vfs 统一本地 / Alist 的 stat、列目录、建目录、读写、移动、删除 - 整理矩阵中涉及 Alist 的传输(storage/transfer.ts)与远端 ffprobe(直链只读头) - 媒体库目录可登记在 Alist 上(不写哨兵,目录可访问即在线);健康检查列出存储实例 Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
5ff24110f8
commit
c2bf8a071a
20 files changed
+1625
-8
No files matched your search
@@ -17,6 +17,7 @@ import type { EventBusPort } from "./http/sse.ts";
|
||||
import { assertCapabilityCount, createCapabilityRegistry } from "./modules/agent/capability-registry.ts";
|
||||
import { createSecretStore, createSettingsReader } from "./modules/auth-settings/provided.ts";
|
||||
import type { OutboxPort, ServerContext, ServerModule, ServerRole } from "./modules/types.ts";
|
||||
import { configureStorage } from "./storage/registry.ts";
|
||||
|
||||
// 进程装配:ServerContext、任务注册表、outbox 路由。api / worker / all 三种角色共用。
|
||||
|
||||
@@ -200,6 +201,7 @@ export async function bootstrap(opts: BootstrapOptions): Promise<Bootstrapped> {
|
||||
// 授权(automation_grant / approval + 自动化总开关)、审计(audit_event)、幂等键与操作账本在这里注入
|
||||
capabilities: createCapabilityRegistry({ logger, db, settings }),
|
||||
};
|
||||
configureStorage(ctx);
|
||||
const tasks = buildTaskRegistry(opts.modules, ctx);
|
||||
for (const m of opts.modules) m.registerCapabilities?.(ctx.capabilities, ctx);
|
||||
ctx.hooks = buildHookRegistry(opts.modules, ctx);
|
||||
|
||||
@@ -7,6 +7,7 @@ import { registerEngineRoutes } from "./engine-routes.ts";
|
||||
import { registerInstanceRoutes } from "./instances-routes.ts";
|
||||
import { registerKeyRoutes } from "./keys-routes.ts";
|
||||
import { registerSettingsRoutes } from "./settings-routes.ts";
|
||||
import { registerStorageRoutes } from "./storage-routes.ts";
|
||||
|
||||
/**
|
||||
* W2-A 认证/设置/实例/引擎面板:管理员登录门禁、settings、api_key、secret 加密服务、
|
||||
@@ -38,6 +39,7 @@ export function createAuthSettingsModule(opts: AuthSettingsModuleOptions = {}):
|
||||
registerSettingsRoutes(app, ctx);
|
||||
registerKeyRoutes(app, ctx);
|
||||
registerInstanceRoutes(app, ctx);
|
||||
registerStorageRoutes(app, ctx);
|
||||
registerEngineRoutes(app, ctx, opts.engineSchema ? { schema: opts.engineSchema } : {});
|
||||
if (opts.setupTokenOnStart !== false && ctx.db) {
|
||||
ensureSetupToken(ctx).catch((err) => ctx.logger.error({ err }, "生成 setup token 失败"));
|
||||
|
||||
@@ -25,6 +25,8 @@ import {
|
||||
import { z } from "zod";
|
||||
import { ApiException } from "../../http/errors.ts";
|
||||
import { redact, redactString } from "../../logger.ts";
|
||||
import { isRemotePath, vcheckRoot } from "../../storage/vfs.ts";
|
||||
import { parseVPath } from "../../storage/vpath.ts";
|
||||
import type { ModuleApp, ServerContext } from "../types.ts";
|
||||
import {
|
||||
afterCursor,
|
||||
@@ -184,6 +186,17 @@ export function registerInstanceRoutes(app: ModuleApp, ctx: ServerContext) {
|
||||
|
||||
// ── library_root ──
|
||||
const sentinel = async (localPath: string, file: string | null | undefined) => {
|
||||
// Alist 库根(/@alist/<存储 id>/<路径>):不写哨兵(同 MP),只确认目录存在且存储可连
|
||||
if (isRemotePath(localPath)) {
|
||||
if (!parseVPath(localPath)) throw new ApiException("validation_failed", "Alist 路径格式不对");
|
||||
const h = await vcheckRoot(localPath);
|
||||
if (h.kind !== "online")
|
||||
throw new ApiException(
|
||||
"precondition_failed",
|
||||
`Alist 目录不可用（${h.reason}），先在 Alist 里建好目录并测试存储连接`,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
const name = file || DEFAULT_ROOT_SENTINEL;
|
||||
try {
|
||||
const st = await stat(localPath);
|
||||
@@ -229,7 +242,9 @@ export function registerInstanceRoutes(app: ModuleApp, ctx: ServerContext) {
|
||||
test("/library-roots/{id}/test", "检查媒体库目录在线(目录 + 哨兵文件)", "library-roots"),
|
||||
async ({ params, actor }) => {
|
||||
const row = await getRow("library_root", params.id);
|
||||
const h = await checkRoot(String(row.local_path), (row.sentinel_file as string | null) ?? DEFAULT_ROOT_SENTINEL);
|
||||
const h = isRemotePath(String(row.local_path))
|
||||
? await vcheckRoot(String(row.local_path))
|
||||
: await checkRoot(String(row.local_path), (row.sentinel_file as string | null) ?? DEFAULT_ROOT_SENTINEL);
|
||||
const ok = h.kind === "online";
|
||||
await update("library_root", row.id, {
|
||||
health_status: ok ? "healthy" : "offline",
|
||||
|
||||
@@ -0,0 +1,264 @@
|
||||
import { AlistError, normAlistPath, redactAlistText } from "@umt/adapters";
|
||||
import {
|
||||
Api,
|
||||
StorageInstance,
|
||||
StorageInstanceCreate,
|
||||
StorageInstanceUpdate,
|
||||
type StorageTestResult,
|
||||
secretInput,
|
||||
} from "@umt/contracts";
|
||||
import { ApiException } from "../../http/errors.ts";
|
||||
import { buildClient, clearStorageCache, configureStorage, resolvePlain } from "../../storage/registry.ts";
|
||||
import type { ModuleApp, ServerContext } from "../types.ts";
|
||||
import {
|
||||
afterCursor,
|
||||
audit,
|
||||
crudDefs,
|
||||
cursorCol,
|
||||
decodeCursor,
|
||||
endpoint,
|
||||
pageOf,
|
||||
pgConflict,
|
||||
toApi,
|
||||
toDb,
|
||||
} from "./endpoint.ts";
|
||||
|
||||
// 存储实例(Alist / OpenList)CRUD + 测试连接(docs/design/storage-alist.md §3、§4)。
|
||||
// 凭据(令牌 / 密码)与路径密码只写不读:省略或空串 = 不变,null = 清除;明文只在 secret 表里加密保存。
|
||||
|
||||
const JSON_COLS = ["options", "health_detail", "version_info"] as const;
|
||||
const tags = ["storage-instances"];
|
||||
|
||||
type Row = Record<string, unknown> & {
|
||||
id: string;
|
||||
secret_ref: string | null;
|
||||
path_password_ref: string | null;
|
||||
auth_kind: "token" | "password" | "guest";
|
||||
username: string | null;
|
||||
};
|
||||
|
||||
export function registerStorageRoutes(app: ModuleApp, ctx: ServerContext) {
|
||||
configureStorage(ctx);
|
||||
const S = {
|
||||
...crudDefs("/storage-instances", "存储实例", {
|
||||
row: StorageInstance,
|
||||
create: StorageInstanceCreate,
|
||||
update: StorageInstanceUpdate,
|
||||
}),
|
||||
test: Api.storageInstances.test,
|
||||
remove: Api.storageInstances.remove,
|
||||
};
|
||||
const db = () => ctx.db;
|
||||
|
||||
const getRow = async (id: string): Promise<Row> => {
|
||||
const r = await db().selectFrom("storage_instance").selectAll().where("id", "=", id).executeTakeFirst();
|
||||
if (!r) throw new ApiException("not_found", "存储实例不存在");
|
||||
return r as unknown as Row;
|
||||
};
|
||||
const normalize = (b: Partial<StorageInstanceCreate>) => {
|
||||
const out: Record<string, unknown> = { ...b };
|
||||
if (b.baseUrl !== undefined) out.baseUrl = b.baseUrl.trim().replace(/\/+$/, "");
|
||||
if (b.rootPath !== undefined) {
|
||||
try {
|
||||
out.rootPath = normAlistPath(b.rootPath);
|
||||
} catch {
|
||||
throw new ApiException("validation_failed", "根路径不合法");
|
||||
}
|
||||
}
|
||||
return out;
|
||||
};
|
||||
const checkAuth = (authKind: string, username: string | null | undefined, hasCredential: boolean) => {
|
||||
if (authKind === "token" && !hasCredential) throw new ApiException("validation_failed", "令牌认证需要填写令牌");
|
||||
if (authKind === "password" && (!username || !hasCredential))
|
||||
throw new ApiException("validation_failed", "账号密码认证需要填写用户名和密码");
|
||||
};
|
||||
const putSecret = async (oldRef: string | null, plaintext: string) => {
|
||||
if (oldRef) {
|
||||
await ctx.secrets.replace(oldRef, plaintext);
|
||||
return oldRef;
|
||||
}
|
||||
return ctx.secrets.create({ purpose: "storage_credential", plaintext });
|
||||
};
|
||||
const view = (r: object) => toApi(r);
|
||||
|
||||
endpoint(app, { ...S.list, tags }, async ({ query }) => {
|
||||
const q0 = query as { cursor?: string; limit: number };
|
||||
const cur = decodeCursor(q0.cursor);
|
||||
let q = db().selectFrom("storage_instance").selectAll().select(cursorCol("created_at"));
|
||||
if (cur) q = q.where(afterCursor("created_at", cur));
|
||||
const rows = await q
|
||||
.orderBy("created_at", "desc")
|
||||
.orderBy("id", "desc")
|
||||
.limit(q0.limit + 1)
|
||||
.execute();
|
||||
const page = pageOf(rows as unknown as { cursor_us: number; id: string }[], q0.limit);
|
||||
return { items: page.items.map(view), nextCursor: page.nextCursor } as never;
|
||||
});
|
||||
endpoint(app, { ...S.get, tags }, async ({ params }) => view(await getRow(params.id)) as never);
|
||||
|
||||
endpoint(app, { ...S.create, tags }, async ({ body, actor }) => {
|
||||
const { credential: rawCredential, pathPassword: rawPw, ...rest } = body;
|
||||
const credential = secretInput(rawCredential) ?? undefined;
|
||||
const pathPassword = secretInput(rawPw) ?? undefined;
|
||||
checkAuth(rest.authKind ?? "token", rest.username, !!credential);
|
||||
const refs: string[] = [];
|
||||
try {
|
||||
const secretRef =
|
||||
credential && (rest.authKind ?? "token") !== "guest"
|
||||
? await ctx.secrets.create({ purpose: "storage_credential", plaintext: credential })
|
||||
: null;
|
||||
if (secretRef) refs.push(secretRef);
|
||||
const pwRef = pathPassword
|
||||
? await ctx.secrets.create({ purpose: "storage_credential", plaintext: pathPassword })
|
||||
: null;
|
||||
if (pwRef) refs.push(pwRef);
|
||||
const row = await db()
|
||||
.insertInto("storage_instance")
|
||||
.values({
|
||||
...(toDb(normalize(rest), JSON_COLS) as { name: string; base_url: string }),
|
||||
secret_ref: secretRef,
|
||||
path_password_ref: pwRef,
|
||||
})
|
||||
.returningAll()
|
||||
.executeTakeFirstOrThrow()
|
||||
.catch((e: unknown) => pgConflict(e, "存储实例名称重复"));
|
||||
await audit(
|
||||
ctx,
|
||||
actor,
|
||||
"storage_instance.create",
|
||||
{ type: "storage_instance", id: row.id },
|
||||
{ name: row.name, authKind: row.auth_kind },
|
||||
);
|
||||
return view(row) as never;
|
||||
} catch (e) {
|
||||
for (const r of refs) await ctx.secrets.remove(r).catch(() => {});
|
||||
throw e;
|
||||
}
|
||||
});
|
||||
|
||||
endpoint(app, { ...S.update, tags }, async ({ params, body, actor }) => {
|
||||
const { credential: rawCredential, pathPassword: rawPw, ...rest } = body;
|
||||
const credential = secretInput(rawCredential);
|
||||
const pathPassword = secretInput(rawPw);
|
||||
const cur = await getRow(params.id);
|
||||
const authKind = rest.authKind ?? cur.auth_kind;
|
||||
const username = rest.username !== undefined ? rest.username : cur.username;
|
||||
const hasCred = credential === null ? false : credential ? true : !!cur.secret_ref;
|
||||
checkAuth(authKind, username, hasCred);
|
||||
const values = toDb(normalize(rest), JSON_COLS);
|
||||
const drop: string[] = [];
|
||||
if (authKind === "guest" || credential === null) {
|
||||
if (cur.secret_ref) {
|
||||
values.secret_ref = null;
|
||||
drop.push(cur.secret_ref);
|
||||
}
|
||||
} else if (credential) values.secret_ref = await putSecret(cur.secret_ref, credential);
|
||||
if (pathPassword === null) {
|
||||
if (cur.path_password_ref) {
|
||||
values.path_password_ref = null;
|
||||
drop.push(cur.path_password_ref);
|
||||
}
|
||||
} else if (pathPassword) values.path_password_ref = await putSecret(cur.path_password_ref, pathPassword);
|
||||
// 改了凭据也要让注册表重建客户端:updated_at 由触发器刷新;无字段变化时补一次 touch
|
||||
const row = await db()
|
||||
.updateTable("storage_instance")
|
||||
.set({ ...(values as Record<string, never>), updated_at: new Date() })
|
||||
.where("id", "=", params.id)
|
||||
.returningAll()
|
||||
.executeTakeFirst()
|
||||
.catch((e: unknown) => pgConflict(e, "存储实例名称重复"));
|
||||
if (!row) throw new ApiException("not_found", "存储实例不存在");
|
||||
for (const r of drop) await ctx.secrets.remove(r).catch(() => {});
|
||||
clearStorageCache(row.id);
|
||||
await audit(
|
||||
ctx,
|
||||
actor,
|
||||
"storage_instance.update",
|
||||
{ type: "storage_instance", id: row.id },
|
||||
{
|
||||
fields: Object.keys(body).map((k) =>
|
||||
k === "credential" || k === "pathPassword" ? `${k}(${body[k] === null ? "cleared" : "set"})` : k,
|
||||
),
|
||||
},
|
||||
);
|
||||
return view(row) as never;
|
||||
});
|
||||
|
||||
endpoint(app, { ...S.remove, tags }, async ({ params, actor }) => {
|
||||
const row = await getRow(params.id);
|
||||
const prefix = `/@alist/${row.id}`;
|
||||
const used = await db()
|
||||
.selectFrom("library_root")
|
||||
.select("name")
|
||||
.where((eb) => eb.or([eb("local_path", "=", prefix), eb("local_path", "like", `${prefix}/%`)]))
|
||||
.execute();
|
||||
if (used.length)
|
||||
throw new ApiException(
|
||||
"conflict",
|
||||
`存储实例仍被媒体库目录使用：${used.map((u) => u.name).join("、")}，先删除或改用其它存储`,
|
||||
);
|
||||
await db().deleteFrom("storage_instance").where("id", "=", row.id).execute();
|
||||
for (const r of [row.secret_ref, row.path_password_ref]) if (r) await ctx.secrets.remove(r).catch(() => {});
|
||||
clearStorageCache(row.id);
|
||||
await audit(ctx, actor, "storage_instance.delete", { type: "storage_instance", id: row.id });
|
||||
return { ok: true as const };
|
||||
});
|
||||
|
||||
endpoint(app, { ...S.test, tags }, async ({ params, actor }) => {
|
||||
const row = await getRow(params.id);
|
||||
const result = await testStorage(ctx, row);
|
||||
await db()
|
||||
.updateTable("storage_instance")
|
||||
.set({
|
||||
health_status: result.healthStatus,
|
||||
last_health_at: new Date(),
|
||||
health_detail: JSON.stringify({ message: result.message, rootOk: result.rootOk, usage: result.usage }),
|
||||
...(result.version ? { version_info: JSON.stringify({ version: result.version, user: result.user }) } : {}),
|
||||
})
|
||||
.where("id", "=", row.id)
|
||||
.execute();
|
||||
clearStorageCache(row.id);
|
||||
await audit(ctx, actor, "storage_instance.test", { type: "storage_instance", id: row.id }, { ok: result.ok });
|
||||
return result;
|
||||
});
|
||||
}
|
||||
|
||||
/** 测试连接(设置页「测试连接」与健康检查共用) */
|
||||
export async function testStorage(
|
||||
ctx: Pick<ServerContext, "secrets" | "logger">,
|
||||
row: Row & { base_url?: unknown; root_path?: unknown; options?: unknown },
|
||||
): Promise<StorageTestResult> {
|
||||
let plain: { credential: string | null; pathPassword: string | null } = { credential: null, pathPassword: null };
|
||||
const fail = (healthStatus: StorageTestResult["healthStatus"], message: string): StorageTestResult => ({
|
||||
ok: false,
|
||||
healthStatus,
|
||||
version: null,
|
||||
user: null,
|
||||
rootOk: false,
|
||||
usage: null,
|
||||
message: redactAlistText(message, [plain.credential, plain.pathPassword]).slice(0, 500),
|
||||
});
|
||||
try {
|
||||
plain = await resolvePlain(ctx.secrets, row);
|
||||
const client = buildClient(row as never, plain);
|
||||
const version = await client.version();
|
||||
const user = row.auth_kind === "guest" ? "guest" : (await client.me()).username;
|
||||
const root = await client.stat(String(row.root_path ?? "/"));
|
||||
const usage = await client.usage(String(row.root_path ?? "/"));
|
||||
if (!root?.isDir) return { ...fail("degraded", `根路径不存在或不是目录：${row.root_path}`), version, user, usage };
|
||||
return { ok: true, healthStatus: "healthy", version, user, rootOk: true, usage, message: null };
|
||||
} catch (e) {
|
||||
if (e instanceof AlistError) {
|
||||
const label =
|
||||
e.kind === "auth"
|
||||
? row.auth_kind === "token"
|
||||
? "令牌失效或无权限"
|
||||
: "登录失败（用户名或密码不对）"
|
||||
: e.kind === "unavailable" || e.kind === "timeout"
|
||||
? "连不上"
|
||||
: "接口出错";
|
||||
return fail("offline", `${label}：${e.message}`);
|
||||
}
|
||||
return fail("offline", String((e as Error)?.message ?? e));
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,9 @@ import {
|
||||
SystemWorkerLane,
|
||||
} from "@umt/contracts";
|
||||
import { type DB, type Kysely, type Selectable, sql } from "@umt/db";
|
||||
import { storageEntry } from "../../storage/registry.ts";
|
||||
import { isRemotePath, vcheckRoot } from "../../storage/vfs.ts";
|
||||
import { toVPath } from "../../storage/vpath.ts";
|
||||
import { CURRENT_HOST_WINDOW_MS, HEARTBEAT_ROW_RETENTION_MS, HEARTBEAT_STALE_MS } from "./heartbeat.ts";
|
||||
|
||||
// GET /system/health 各段的取数(W9-I)。每段独立 try/catch:一段失败只把该段标 unknown,不让整页 500。
|
||||
@@ -214,7 +217,14 @@ export async function loadPathTargets(db: Kysely<DB>) {
|
||||
.where("path_mapping.view_kind", "=", "downloader")
|
||||
.orderBy("path_mapping.local_prefix")
|
||||
.execute();
|
||||
return { roots, maps };
|
||||
const storages = await db
|
||||
.selectFrom("storage_instance")
|
||||
.select(["id", "name", "root_path"])
|
||||
.where("enabled", "=", true)
|
||||
.orderBy("name")
|
||||
.execute()
|
||||
.catch(() => [] as { id: string; name: string; root_path: string }[]);
|
||||
return { roots, maps, storages };
|
||||
}
|
||||
|
||||
export function diskLevel(ratio: number, t: HealthDeps["thresholds"]): HealthLevel {
|
||||
@@ -226,11 +236,14 @@ export async function loadDisksAndMounts(d: HealthDeps): Promise<{ disks: System
|
||||
const usage = d.diskUsage ?? diskUsage;
|
||||
const check = d.checkRoot ?? checkRoot;
|
||||
const ms = d.ioTimeoutMs ?? 3000;
|
||||
const { roots, maps } = await loadPathTargets(d.db);
|
||||
const { roots, maps, storages } = await loadPathTargets(d.db);
|
||||
|
||||
const targets: PathTarget[] = [
|
||||
{ scope: "data_dir", id: null, name: "DATA_DIR", path: d.dataDir },
|
||||
...roots.map((r) => ({ scope: "library_root" as const, id: r.id, name: r.name, path: r.local_path })),
|
||||
// Alist 库根没有本机磁盘可统计(容量在文件管理按 mount_details 显示)
|
||||
...roots
|
||||
.filter((r) => !isRemotePath(r.local_path))
|
||||
.map((r) => ({ scope: "library_root" as const, id: r.id, name: r.name, path: r.local_path })),
|
||||
...maps.map((m) => ({
|
||||
scope: "download_dir" as const,
|
||||
id: m.id,
|
||||
@@ -280,7 +293,12 @@ export async function loadDisksAndMounts(d: HealthDeps): Promise<{ disks: System
|
||||
...roots.map(async (r): Promise<SystemMount> => {
|
||||
const base = { kind: "library_root" as const, id: r.id, name: r.name, path: r.local_path, sentinelChecked: true };
|
||||
try {
|
||||
const h = await withTimeout(check(r.local_path, r.sentinel_file ?? DEFAULT_ROOT_SENTINEL), ms);
|
||||
const h = await withTimeout(
|
||||
isRemotePath(r.local_path)
|
||||
? vcheckRoot(r.local_path)
|
||||
: check(r.local_path, r.sentinel_file ?? DEFAULT_ROOT_SENTINEL),
|
||||
isRemotePath(r.local_path) ? Math.max(ms, 15_000) : ms,
|
||||
);
|
||||
return h.kind === "online"
|
||||
? { ...base, status: "online", reason: null }
|
||||
: { ...base, status: "offline", reason: h.reason };
|
||||
@@ -316,6 +334,27 @@ export async function loadDisksAndMounts(d: HealthDeps): Promise<{ disks: System
|
||||
return { ...base, status: "offline", reason };
|
||||
}
|
||||
}),
|
||||
// Alist / OpenList 实例:连通 + 根路径可访问(令牌失效、连不上都在这里显示)
|
||||
...storages.map(async (s): Promise<SystemMount> => {
|
||||
const path = toVPath(s.id, s.root_path);
|
||||
const base = {
|
||||
kind: "storage_instance" as const,
|
||||
id: s.id,
|
||||
name: s.name,
|
||||
path: s.root_path,
|
||||
sentinelChecked: false,
|
||||
};
|
||||
try {
|
||||
await storageEntry(s.id);
|
||||
const h = await withTimeout(vcheckRoot(path), Math.max(ms, 15_000));
|
||||
if (h.kind === "online") return { ...base, status: "online", reason: null };
|
||||
return { ...base, status: h.reason === "timeout" ? "timeout" : "offline", reason: h.reason };
|
||||
} catch (err) {
|
||||
return err instanceof Timeout
|
||||
? { ...base, status: "timeout", reason: "timeout" }
|
||||
: { ...base, status: "offline", reason: "error" };
|
||||
}
|
||||
}),
|
||||
]);
|
||||
return { disks, mounts };
|
||||
}
|
||||
|
||||
@@ -85,7 +85,10 @@ export async function loadMountedPaths(db: Kysely<DB>): Promise<MountedPath[]> {
|
||||
.where("view_kind", "=", "downloader")
|
||||
.execute();
|
||||
return [
|
||||
...roots.map((r) => ({ kind: "library_root" as const, id: r.id, name: r.name, path: r.local_path })),
|
||||
// Alist 库根(/@alist/…)不是本机挂载,由健康检查按实例连通性判断
|
||||
...roots
|
||||
.filter((r) => !r.local_path.startsWith("/@alist/"))
|
||||
.map((r) => ({ kind: "library_root" as const, id: r.id, name: r.name, path: r.local_path })),
|
||||
...maps.map((m) => ({ kind: "path_mapping" as const, id: m.id, name: m.remote_prefix, path: m.local_prefix })),
|
||||
];
|
||||
}
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
import { ffprobe, type ProbeResult, redactAlistText } from "@umt/adapters";
|
||||
import { vrawUrl } from "./vfs.ts";
|
||||
import { parseVPath } from "./vpath.ts";
|
||||
|
||||
// 远端文件的 ffprobe(docs/design/storage-alist.md §7):不下载,直接对签名直链探测,ffprobe 只读容器头与少量包(HTTP Range)。
|
||||
// 拿不到直链或探测失败返回 unknown(不阻塞,同本地 ffprobe 失败的口径)。直链进错误信息前脱敏。
|
||||
|
||||
export async function probeAny(
|
||||
path: string,
|
||||
local: (p: string) => Promise<ProbeResult> = (p) => ffprobe(p),
|
||||
): Promise<ProbeResult> {
|
||||
if (!parseVPath(path)) return local(path);
|
||||
let url: string;
|
||||
try {
|
||||
url = await vrawUrl(path);
|
||||
} catch (e) {
|
||||
return { status: "unknown", reason: "probe_failed", message: redactAlistText(String((e as Error)?.message ?? e)) };
|
||||
}
|
||||
const r = await ffprobe(url, { inputArgs: ["-rw_timeout", "30000000"], timeoutMs: 60_000 });
|
||||
if (r.status === "unknown" && r.message)
|
||||
return { ...r, message: redactAlistText(r.message.split(url).join("<直链>")) };
|
||||
return r;
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
import { type AlistAuth, AlistClient, AlistError } from "@umt/adapters";
|
||||
import { STORAGE_OPTION_DEFAULTS, type StorageOptions } from "@umt/contracts";
|
||||
import type { DB, Kysely, StorageInstanceRow } from "@umt/db";
|
||||
import type { ServerContext } from "../modules/types.ts";
|
||||
|
||||
// 存储实例注册表(docs/design/storage-alist.md §3):按 id 取实例 + 解密凭据 → AlistClient。
|
||||
// 进程级:模块装配时 configureStorage(ctx);实例行按 updated_at 缓存,配置一改客户端重建(令牌缓存随之作废,同 MP init_storage)。
|
||||
|
||||
export interface StorageEntry {
|
||||
id: string;
|
||||
name: string;
|
||||
enabled: boolean;
|
||||
/** 本系统可用的 Alist 根路径 */
|
||||
rootPath: string;
|
||||
options: Required<StorageOptions>;
|
||||
client: AlistClient;
|
||||
/** 日志脱敏要遮的秘密值 */
|
||||
secrets: string[];
|
||||
}
|
||||
|
||||
interface Source {
|
||||
dataDir?: string | undefined;
|
||||
db: Kysely<DB>;
|
||||
secrets: ServerContext["secrets"];
|
||||
logger?: ServerContext["logger"] | undefined;
|
||||
}
|
||||
|
||||
let source: Source | null = null;
|
||||
const cache = new Map<string, { stamp: string; entry: StorageEntry; checkedAt: number }>();
|
||||
const overrides = new Map<string, StorageEntry>();
|
||||
const RECHECK_MS = 5_000;
|
||||
|
||||
/** 模块装配时调用(幂等;最后一次为准) */
|
||||
export function configureStorage(
|
||||
ctx: Pick<ServerContext, "db" | "secrets"> & {
|
||||
logger?: ServerContext["logger"];
|
||||
config?: Partial<Pick<ServerContext["config"], "DATA_DIR">>;
|
||||
},
|
||||
) {
|
||||
if (source?.db !== ctx.db) cache.clear();
|
||||
source = { db: ctx.db, secrets: ctx.secrets, logger: ctx.logger, dataDir: ctx.config?.DATA_DIR };
|
||||
}
|
||||
|
||||
/** 数据目录(跨存储中转、上传暂存放在 <DATA_DIR>/cache/storage-relay) */
|
||||
export const storageDataDir = (): string | undefined => source?.dataDir;
|
||||
|
||||
/** 测试用:直接给某个存储 id 指定客户端(mock Alist) */
|
||||
export function overrideStorage(
|
||||
id: string,
|
||||
e: { client: AlistClient; name?: string; rootPath?: string; options?: StorageOptions } | null,
|
||||
): void {
|
||||
if (!e) {
|
||||
overrides.delete(id);
|
||||
return;
|
||||
}
|
||||
overrides.set(id, {
|
||||
id,
|
||||
name: e.name ?? "Alist",
|
||||
enabled: true,
|
||||
rootPath: e.rootPath ?? "/",
|
||||
options: { ...STORAGE_OPTION_DEFAULTS, ...(e.options ?? {}) },
|
||||
client: e.client,
|
||||
secrets: [],
|
||||
});
|
||||
}
|
||||
|
||||
export function clearStorageCache(id?: string): void {
|
||||
if (id) cache.delete(id);
|
||||
else cache.clear();
|
||||
}
|
||||
|
||||
export const storageOptions = (o: unknown): Required<StorageOptions> => ({
|
||||
...STORAGE_OPTION_DEFAULTS,
|
||||
...((o && typeof o === "object" ? o : {}) as StorageOptions),
|
||||
});
|
||||
|
||||
/** 由实例行 + 明文凭据建客户端(测试连接、注册表共用) */
|
||||
export function buildClient(
|
||||
row: Pick<StorageInstanceRow, "base_url" | "auth_kind" | "username" | "options">,
|
||||
plain: { credential: string | null; pathPassword: string | null },
|
||||
logger?: ServerContext["logger"],
|
||||
): AlistClient {
|
||||
const o = storageOptions(row.options);
|
||||
let auth: AlistAuth;
|
||||
if (row.auth_kind === "token") {
|
||||
if (!plain.credential) throw new AlistError("auth", "没有配置令牌");
|
||||
auth = { kind: "token", token: plain.credential };
|
||||
} else if (row.auth_kind === "password") {
|
||||
if (!row.username || !plain.credential) throw new AlistError("auth", "账号密码认证需要用户名和密码");
|
||||
auth = { kind: "password", username: row.username, password: plain.credential };
|
||||
} else auth = { kind: "guest" };
|
||||
return new AlistClient({
|
||||
baseUrl: row.base_url,
|
||||
auth,
|
||||
pathPassword: plain.pathPassword,
|
||||
qps: o.qps,
|
||||
concurrency: o.concurrency,
|
||||
timeoutMs: o.timeoutSec * 1000,
|
||||
idleTimeoutMs: o.idleTimeoutSec * 1000,
|
||||
taskTimeoutMs: o.taskTimeoutMin * 60_000,
|
||||
retries: o.retries,
|
||||
uploadAsTask: o.uploadAsTask,
|
||||
...(logger
|
||||
? {
|
||||
logger: {
|
||||
debug: (obj, msg) => logger.debug(obj, msg),
|
||||
warn: (obj, msg) => logger.warn(obj, msg),
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
|
||||
export async function resolvePlain(
|
||||
secrets: Source["secrets"],
|
||||
row: Pick<StorageInstanceRow, "secret_ref" | "path_password_ref">,
|
||||
) {
|
||||
return {
|
||||
credential: row.secret_ref ? await secrets.resolve(row.secret_ref) : null,
|
||||
pathPassword: row.path_password_ref ? await secrets.resolve(row.path_password_ref) : null,
|
||||
};
|
||||
}
|
||||
|
||||
/** 取存储实例;不存在 / 停用 / 未装配抛 AlistError(unavailable,按「存储离线」处理,不当成文件不存在) */
|
||||
export async function storageEntry(id: string, opts: { allowDisabled?: boolean } = {}): Promise<StorageEntry> {
|
||||
const o = overrides.get(id);
|
||||
if (o) return o;
|
||||
if (!source) throw new AlistError("unavailable", "存储注册表未初始化");
|
||||
const hit = cache.get(id);
|
||||
if (hit && Date.now() - hit.checkedAt < RECHECK_MS) return check(hit.entry, opts);
|
||||
const row = await source.db.selectFrom("storage_instance").selectAll().where("id", "=", id).executeTakeFirst();
|
||||
if (!row) throw new AlistError("unavailable", `存储实例不存在：${id}`);
|
||||
const stamp = new Date(row.updated_at).toISOString();
|
||||
if (hit && hit.stamp === stamp) {
|
||||
hit.checkedAt = Date.now();
|
||||
return check(hit.entry, opts);
|
||||
}
|
||||
const plain = await resolvePlain(source.secrets, row);
|
||||
const entry: StorageEntry = {
|
||||
id: row.id,
|
||||
name: row.name,
|
||||
enabled: row.enabled,
|
||||
rootPath: row.root_path,
|
||||
options: storageOptions(row.options),
|
||||
client: buildClient(row, plain, source.logger),
|
||||
secrets: [plain.credential, plain.pathPassword].filter((s): s is string => !!s),
|
||||
};
|
||||
cache.set(id, { stamp, entry, checkedAt: Date.now() });
|
||||
return check(entry, opts);
|
||||
}
|
||||
|
||||
function check(e: StorageEntry, opts: { allowDisabled?: boolean }): StorageEntry {
|
||||
if (!e.enabled && !opts.allowDisabled) throw new AlistError("unavailable", `存储实例已停用：${e.name}`);
|
||||
return e;
|
||||
}
|
||||
|
||||
/** 启用的实例(文件管理根、健康检查) */
|
||||
export async function listStorages(db?: Kysely<DB>): Promise<StorageInstanceRow[]> {
|
||||
const d = db ?? source?.db;
|
||||
if (!d) return [];
|
||||
return d.selectFrom("storage_instance").selectAll().orderBy("name").execute();
|
||||
}
|
||||
|
||||
/** 存储名表(展示「存储名:/路径」) */
|
||||
export async function storageNames(db?: Kysely<DB>): Promise<Map<string, string>> {
|
||||
const m = new Map<string, string>();
|
||||
for (const [id, e] of overrides) m.set(id, e.name);
|
||||
for (const r of await listStorages(db)) m.set(r.id, r.name);
|
||||
return m;
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { mkdir, rename, rm, stat, unlink } from "node:fs/promises";
|
||||
import { basename, dirname, join, posix } from "node:path";
|
||||
import { AlistError, type ProgressCallback } from "@umt/adapters";
|
||||
import { invalidateRemote, relayDir, remote, remoteError } from "./vfs.ts";
|
||||
import { parseVPath, sameStorage } from "./vpath.ts";
|
||||
|
||||
// 整理矩阵中涉及 Alist 的方向(docs/design/storage-alist.md §5):
|
||||
// 本地→Alist 上传、Alist→本地 下载、同实例 Alist→Alist API copy / move、跨实例 下载→上传。
|
||||
// 成功判定与 MP 一致(用户决定):接口成功 + 目标存在,不比对大小 / 哈希;移动在判定通过后删源,删源失败 = attention。
|
||||
// 本地 ↔ 本地不经这里(仍走 fsproxy 与现有核验)。
|
||||
|
||||
export interface RemoteTransferInput {
|
||||
kind: "copy" | "move";
|
||||
src: string;
|
||||
dst: string;
|
||||
/** 目标已存在:conflict 失败;replace 先把旧文件移到 replaceTrashDir(同存储),成功后旧文件留在那里,失败移回 */
|
||||
onExisting: "conflict" | "replace";
|
||||
replaceTrashDir?: string;
|
||||
/** 本次是重试(上次可能已完成外部动作) */
|
||||
resumed?: boolean;
|
||||
signal?: AbortSignal;
|
||||
onProgress?: ProgressCallback;
|
||||
}
|
||||
|
||||
export interface RemoteTransferResult {
|
||||
status: "succeeded" | "failed" | "attention";
|
||||
bytes?: number;
|
||||
/** 顺带得到的本地文件 sha256(上传的源 / 下载的结果),只是事实记录 */
|
||||
checksum?: string;
|
||||
resumed?: boolean;
|
||||
error?: { code: string; message: string };
|
||||
replaced?: { disposition: "trash"; path: string; size: number; mtimeMs: number };
|
||||
steps?: { step: string; status: string; at: string; detail?: string; error?: { code: string; message: string } }[];
|
||||
via: "upload" | "download" | "alist_copy" | "alist_move" | "relay";
|
||||
}
|
||||
|
||||
/** Alist / 本地错误 → 整理账本错误码(可重试的 IO/TIMEOUT 走任务重试) */
|
||||
export function transferErrorCode(e: unknown, role: "源" | "目标" | null = null): { code: string; message: string } {
|
||||
if (e instanceof AlistError) {
|
||||
const code =
|
||||
e.kind === "not_found"
|
||||
? role === "源"
|
||||
? "SOURCE_MISSING"
|
||||
: "IO"
|
||||
: e.kind === "conflict"
|
||||
? "EEXIST"
|
||||
: e.kind === "auth"
|
||||
? "EACCES"
|
||||
: e.kind === "timeout" || e.kind === "rate_limited"
|
||||
? "TIMEOUT"
|
||||
: e.kind === "aborted"
|
||||
? "ABORTED"
|
||||
: "IO";
|
||||
return { code, message: `Alist：${e.message}` };
|
||||
}
|
||||
const c = (e as { code?: string } | null)?.code;
|
||||
const message = (e as Error)?.message ?? String(e);
|
||||
if (c === "ENOENT") return { code: role === "源" ? "SOURCE_MISSING" : "IO", message };
|
||||
if (c === "EEXIST") return { code: "EEXIST", message };
|
||||
if (c === "EACCES" || c === "EPERM") return { code: "EACCES", message };
|
||||
if (c === "ENOSPC") return { code: "ENOSPC", message };
|
||||
if (c === "ETIMEDOUT") return { code: "TIMEOUT", message };
|
||||
if (c === "ABORT_ERR") return { code: "ABORTED", message };
|
||||
return { code: "IO", message };
|
||||
}
|
||||
|
||||
const now = () => new Date().toISOString();
|
||||
|
||||
async function localExists(p: string): Promise<{ size: number; mtimeMs: number } | null> {
|
||||
try {
|
||||
const s = await stat(p);
|
||||
return { size: s.size, mtimeMs: s.mtimeMs };
|
||||
} catch (e) {
|
||||
if ((e as { code?: string }).code === "ENOENT") return null;
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
/** 任一端是远端路径? */
|
||||
export const involvesRemote = (src: string, dst: string) => !!parseVPath(src) || !!parseVPath(dst);
|
||||
|
||||
/** 执行一次涉及远端的复制 / 移动 */
|
||||
export async function remoteTransfer(x: RemoteTransferInput): Promise<RemoteTransferResult> {
|
||||
const rs = parseVPath(x.src);
|
||||
const rd = parseVPath(x.dst);
|
||||
if (rs) invalidateRemote(x.src);
|
||||
if (rd) invalidateRemote(x.dst);
|
||||
const via: RemoteTransferResult["via"] =
|
||||
rs && rd
|
||||
? sameStorage(x.src, x.dst)
|
||||
? x.kind === "copy"
|
||||
? "alist_copy"
|
||||
: "alist_move"
|
||||
: "relay"
|
||||
: rd
|
||||
? "upload"
|
||||
: "download";
|
||||
const fail = (e: unknown, role: "源" | "目标" | null = null): RemoteTransferResult => ({
|
||||
status: "failed",
|
||||
error: transferErrorCode(e, role),
|
||||
via,
|
||||
});
|
||||
|
||||
// ── 源 / 目标状态 ──
|
||||
let srcSize = 0;
|
||||
let srcExists = true;
|
||||
try {
|
||||
if (rs) {
|
||||
const { client } = await remote(x.src);
|
||||
const o = await client.stat(rs.remote);
|
||||
srcExists = !!o;
|
||||
srcSize = o?.size ?? 0;
|
||||
if (o?.isDir) return fail(new AlistError("error", `源是目录：${x.src}`));
|
||||
} else {
|
||||
const s = await localExists(x.src);
|
||||
srcExists = !!s;
|
||||
srcSize = s?.size ?? 0;
|
||||
}
|
||||
} catch (e) {
|
||||
return fail(e, "源");
|
||||
}
|
||||
let dstFacts: { size: number; mtimeMs: number } | null;
|
||||
try {
|
||||
if (rd) {
|
||||
const { client } = await remote(x.dst);
|
||||
const o = await client.stat(rd.remote);
|
||||
dstFacts = o ? { size: o.size, mtimeMs: o.modified } : null;
|
||||
} else dstFacts = await localExists(x.dst);
|
||||
} catch (e) {
|
||||
return fail(e, "目标");
|
||||
}
|
||||
// 对账:上次尝试已完成外部动作但没留下结果(目标都是「临时名 → 改名」落地的,目标在 = 上次已写完)
|
||||
if (dstFacts && x.resumed && x.onExisting !== "replace") {
|
||||
if (x.kind === "move" && srcExists) {
|
||||
const del = await deleteSource(x.src).then(
|
||||
() => null,
|
||||
(e) => transferErrorCode(e),
|
||||
);
|
||||
if (del)
|
||||
return {
|
||||
status: "attention",
|
||||
bytes: dstFacts.size,
|
||||
resumed: true,
|
||||
via,
|
||||
steps: [{ step: "delete_source", status: "failed", at: now(), error: del }],
|
||||
};
|
||||
}
|
||||
return { status: "succeeded", bytes: dstFacts.size, resumed: true, via };
|
||||
}
|
||||
if (!srcExists) {
|
||||
if (x.kind === "move" && dstFacts) return { status: "succeeded", bytes: dstFacts.size, resumed: true, via };
|
||||
return { status: "failed", error: { code: "SOURCE_MISSING", message: `源文件不存在：${x.src}` }, via };
|
||||
}
|
||||
if (dstFacts && x.onExisting !== "replace")
|
||||
return { status: "failed", error: { code: "EEXIST", message: `目标已存在：${x.dst}` }, via };
|
||||
|
||||
// ── 替换:旧目标先移走 ──
|
||||
let parked: { path: string; size: number; mtimeMs: number } | null = null;
|
||||
if (dstFacts && x.onExisting === "replace") {
|
||||
const trashDir = x.replaceTrashDir ?? join(dirname(x.dst), ".umt-trash");
|
||||
const to = join(trashDir, `replaced-${Date.now()}-${randomUUID().slice(0, 6)}`, basename(x.dst));
|
||||
try {
|
||||
await parkOld(x.dst, to);
|
||||
parked = { path: to, size: dstFacts.size, mtimeMs: dstFacts.mtimeMs };
|
||||
} catch (e) {
|
||||
return fail(e, "目标");
|
||||
}
|
||||
}
|
||||
const restoreParked = async () => {
|
||||
if (parked) await parkOld(parked.path, x.dst).catch(() => undefined);
|
||||
};
|
||||
|
||||
// ── 传输 ──
|
||||
let bytes = srcSize;
|
||||
let checksum: string | undefined;
|
||||
try {
|
||||
if (rd && !rs) {
|
||||
const { client } = await remote(x.dst);
|
||||
const r = await client.upload(x.src, rd.remote, {
|
||||
...(x.signal ? { signal: x.signal } : {}),
|
||||
...(x.onProgress ? { onProgress: x.onProgress } : {}),
|
||||
});
|
||||
bytes = r.bytes;
|
||||
checksum = `sha256:${r.hashes.sha256}`;
|
||||
} else if (rs && !rd) {
|
||||
await mkdir(dirname(x.dst), { recursive: true });
|
||||
const { client } = await remote(x.src);
|
||||
const r = await client.download(rs.remote, x.dst, {
|
||||
...(x.signal ? { signal: x.signal } : {}),
|
||||
...(x.onProgress ? { onProgress: x.onProgress } : {}),
|
||||
});
|
||||
bytes = r.bytes;
|
||||
checksum = `sha256:${r.sha256}`;
|
||||
} else if (rs && rd && sameStorage(x.src, x.dst)) {
|
||||
const { client } = await remote(x.src);
|
||||
const opts = x.signal ? { signal: x.signal } : {};
|
||||
const o =
|
||||
x.kind === "copy"
|
||||
? await client.copyFile(rs.remote, rd.remote, opts)
|
||||
: await client.moveFile(rs.remote, rd.remote, opts);
|
||||
bytes = o.size || srcSize;
|
||||
} else if (rs && rd) {
|
||||
// 跨实例:下载到本机中转 → 上传(MP 不支持,我们补上)
|
||||
const tmpDir = join(relayDir(), randomUUID());
|
||||
await mkdir(tmpDir, { recursive: true });
|
||||
const tmp = join(tmpDir, posix.basename(rs.remote));
|
||||
try {
|
||||
const a = await remote(x.src);
|
||||
await a.client.download(rs.remote, tmp, {
|
||||
...(x.signal ? { signal: x.signal } : {}),
|
||||
...(x.onProgress ? { onProgress: (d, t) => x.onProgress?.(d / 2, t) } : {}),
|
||||
});
|
||||
const b = await remote(x.dst);
|
||||
const r = await b.client.upload(tmp, rd.remote, {
|
||||
...(x.signal ? { signal: x.signal } : {}),
|
||||
...(x.onProgress ? { onProgress: (d, t) => x.onProgress?.(t / 2 + d / 2, t) } : {}),
|
||||
});
|
||||
bytes = r.bytes;
|
||||
checksum = `sha256:${r.hashes.sha256}`;
|
||||
} finally {
|
||||
await rm(tmpDir, { recursive: true, force: true }).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
await restoreParked();
|
||||
return fail(e);
|
||||
}
|
||||
const out: RemoteTransferResult = {
|
||||
status: "succeeded",
|
||||
bytes,
|
||||
...(checksum ? { checksum } : {}),
|
||||
via,
|
||||
...(parked ? { replaced: { disposition: "trash" as const, ...parked } } : {}),
|
||||
};
|
||||
// 同实例 API move 自己删了源;其它移动在成功后删源
|
||||
if (x.kind === "move" && via !== "alist_move") {
|
||||
const del = await deleteSource(x.src).then(
|
||||
() => null,
|
||||
(e) => transferErrorCode(e),
|
||||
);
|
||||
if (del)
|
||||
return {
|
||||
...out,
|
||||
status: "attention",
|
||||
steps: [{ step: "delete_source", status: "failed", at: now(), error: del }],
|
||||
};
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async function deleteSource(src: string): Promise<void> {
|
||||
const v = parseVPath(src);
|
||||
if (!v) {
|
||||
await unlink(src);
|
||||
return;
|
||||
}
|
||||
const { client } = await remote(src);
|
||||
try {
|
||||
await client.removePath(v.remote);
|
||||
} catch (e) {
|
||||
throw remoteError(e, src);
|
||||
}
|
||||
}
|
||||
|
||||
/** 同存储内把 from 移到 to(替换时暂放旧文件 / 失败移回) */
|
||||
async function parkOld(from: string, to: string): Promise<void> {
|
||||
const a = parseVPath(from);
|
||||
const b = parseVPath(to);
|
||||
if (!a && !b) {
|
||||
await mkdir(dirname(to), { recursive: true });
|
||||
await rename(from, to);
|
||||
return;
|
||||
}
|
||||
if (!a || !b || a.storageId !== b.storageId) throw new AlistError("error", "替换暂存必须与目标在同一存储");
|
||||
const { client } = await remote(from);
|
||||
await client.moveFile(a.remote, b.remote);
|
||||
}
|
||||
@@ -0,0 +1,356 @@
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { lstat, mkdir, readdir, readFile, realpath, rename, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { basename, dirname, join, posix } from "node:path";
|
||||
import {
|
||||
type AlistClient,
|
||||
AlistError,
|
||||
checkRoot,
|
||||
DEFAULT_ROOT_SENTINEL,
|
||||
type FileFacts,
|
||||
type RootHealth,
|
||||
type StatResult,
|
||||
statSafe,
|
||||
} from "@umt/adapters";
|
||||
import { storageDataDir, storageEntry } from "./registry.ts";
|
||||
import { parseVPath, toVPath, type VPath } from "./vpath.ts";
|
||||
|
||||
// 本地 / Alist 统一的文件操作(docs/design/storage-alist.md §3)。本地分支原样调用 node:fs / adapters,
|
||||
// 远端分支经注册表取 AlistClient。远端错误映射成 errno 风格的 code:不存在 ENOENT、冲突 EEXIST、鉴权 EACCES,
|
||||
// 连接 / 服务端失败 EHOSTDOWN / ETIMEDOUT(文件管理按「根不可用」503 处理,整理按可重试 IO 处理,绝不当成不存在)。
|
||||
|
||||
export { isRemotePath, parseVPath, toVPath } from "./vpath.ts";
|
||||
|
||||
export interface VStats {
|
||||
size: number;
|
||||
mtimeMs: number;
|
||||
ino: number;
|
||||
dev: number;
|
||||
nlink: number;
|
||||
isFile(): boolean;
|
||||
isDirectory(): boolean;
|
||||
isSymbolicLink(): boolean;
|
||||
}
|
||||
|
||||
/** Alist 错误 → 带 errno code 的 Error */
|
||||
export function remoteError(e: unknown, path?: string): Error & { code: string } {
|
||||
if (e instanceof AlistError) {
|
||||
const code =
|
||||
e.kind === "not_found"
|
||||
? "ENOENT"
|
||||
: e.kind === "conflict"
|
||||
? "EEXIST"
|
||||
: e.kind === "auth"
|
||||
? "EACCES"
|
||||
: e.kind === "timeout" || e.kind === "rate_limited"
|
||||
? "ETIMEDOUT"
|
||||
: e.kind === "aborted"
|
||||
? "ABORT_ERR"
|
||||
: "EHOSTDOWN";
|
||||
return Object.assign(new Error(`Alist：${e.message}${path ? `（${path}）` : ""}`), { code, alist: e.kind });
|
||||
}
|
||||
if (e && typeof e === "object" && "code" in e) return e as Error & { code: string };
|
||||
return Object.assign(new Error(String((e as Error)?.message ?? e)), { code: "EIO" });
|
||||
}
|
||||
|
||||
/** 远端对象的伪设备号:同一存储实例相同,与本机设备号不冲突 */
|
||||
export function remoteDev(storageId: string): number {
|
||||
return 2 ** 40 + Number.parseInt(createHash("sha1").update(storageId).digest("hex").slice(0, 8), 16);
|
||||
}
|
||||
|
||||
export async function remote(p: string): Promise<{ v: VPath; client: AlistClient }> {
|
||||
const v = parseVPath(p);
|
||||
if (!v) throw Object.assign(new Error(`不是远端路径：${p}`), { code: "EINVAL" });
|
||||
try {
|
||||
const e = await storageEntry(v.storageId);
|
||||
return { v, client: e.client };
|
||||
} catch (err) {
|
||||
throw remoteError(err, p);
|
||||
}
|
||||
}
|
||||
|
||||
const statsOf = (storageId: string, o: { isDir: boolean; size: number; modified: number }): VStats => ({
|
||||
size: o.isDir ? 0 : o.size,
|
||||
mtimeMs: o.modified,
|
||||
ino: 0,
|
||||
dev: remoteDev(storageId),
|
||||
nlink: 1,
|
||||
isFile: () => !o.isDir,
|
||||
isDirectory: () => o.isDir,
|
||||
isSymbolicLink: () => false,
|
||||
});
|
||||
|
||||
export const factsOf = (s: VStats): FileFacts => ({
|
||||
size: Number(s.size),
|
||||
mtimeMs: Number(s.mtimeMs),
|
||||
ino: Number(s.ino),
|
||||
dev: Number(s.dev),
|
||||
nlink: Number(s.nlink),
|
||||
isFile: s.isFile(),
|
||||
isDirectory: s.isDirectory(),
|
||||
isSymbolicLink: s.isSymbolicLink(),
|
||||
});
|
||||
|
||||
// 远端 stat 短缓存:列目录拿到的子项 stat 在几秒内复用,避免遍历时逐个 get(§9-8 限制请求量);写操作按目录失效
|
||||
const STAT_TTL_MS = 3000;
|
||||
const statCache = new Map<string, { at: number; stats: VStats }>();
|
||||
function cacheStat(p: string, stats: VStats) {
|
||||
statCache.set(p, { at: Date.now(), stats });
|
||||
if (statCache.size > 20_000) statCache.delete(statCache.keys().next().value!);
|
||||
}
|
||||
/** 写后失效:该路径、其子树与父目录 */
|
||||
export function invalidateRemote(p: string): void {
|
||||
const parent = posix.dirname(p);
|
||||
for (const k of statCache.keys()) if (k === p || k.startsWith(`${p}/`) || k === parent) statCache.delete(k);
|
||||
}
|
||||
|
||||
export async function vlstat(p: string, opts: { refresh?: boolean } = {}): Promise<VStats> {
|
||||
if (!parseVPath(p)) return (await lstat(p)) as unknown as VStats;
|
||||
if (!opts.refresh) {
|
||||
const hit = statCache.get(p);
|
||||
if (hit && Date.now() - hit.at < STAT_TTL_MS) return hit.stats;
|
||||
}
|
||||
const { v, client } = await remote(p);
|
||||
try {
|
||||
const st = statsOf(v.storageId, await client.get(v.remote, opts));
|
||||
cacheStat(p, st);
|
||||
return st;
|
||||
} catch (e) {
|
||||
throw remoteError(e, p);
|
||||
}
|
||||
}
|
||||
|
||||
/** 远端没有符号链接:realpath = 规范化后的路径(存在才返回) */
|
||||
export async function vrealpath(p: string): Promise<string> {
|
||||
const v = parseVPath(p);
|
||||
if (!v) return realpath(p);
|
||||
await vlstat(p);
|
||||
return toVPath(v.storageId, v.remote);
|
||||
}
|
||||
|
||||
export interface VDirent {
|
||||
name: string;
|
||||
stats: VStats;
|
||||
}
|
||||
|
||||
/** 列目录(带每项的 stat;远端一次 list 拿全) */
|
||||
export async function vreaddirStats(p: string, opts: { refresh?: boolean } = {}): Promise<VDirent[]> {
|
||||
const v = parseVPath(p);
|
||||
if (!v) {
|
||||
const names = await readdir(p);
|
||||
const out: VDirent[] = [];
|
||||
for (const name of names) {
|
||||
const st = await lstat(join(p, name)).catch(() => null);
|
||||
if (st) out.push({ name, stats: st as unknown as VStats });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
const { client } = await remote(p);
|
||||
try {
|
||||
return (await client.list(v.remote, opts)).map((o) => {
|
||||
const stats = statsOf(v.storageId, o);
|
||||
cacheStat(join(p, o.name), stats);
|
||||
return { name: o.name, stats };
|
||||
});
|
||||
} catch (e) {
|
||||
throw remoteError(e, p);
|
||||
}
|
||||
}
|
||||
|
||||
export interface VDirentLite {
|
||||
name: string;
|
||||
isDirectory(): boolean;
|
||||
isFile(): boolean;
|
||||
isSymbolicLink(): boolean;
|
||||
}
|
||||
|
||||
/** readdir(withFileTypes) 的统一版本;本地照旧 */
|
||||
export async function vreaddirents(p: string): Promise<VDirentLite[]> {
|
||||
if (!parseVPath(p)) return readdir(p, { withFileTypes: true });
|
||||
return (await vreaddirStats(p)).map((d) => ({
|
||||
name: d.name,
|
||||
isDirectory: () => d.stats.isDirectory(),
|
||||
isFile: () => d.stats.isFile(),
|
||||
isSymbolicLink: () => false,
|
||||
}));
|
||||
}
|
||||
|
||||
export async function vreaddir(p: string): Promise<string[]> {
|
||||
if (!parseVPath(p)) return readdir(p);
|
||||
return (await vreaddirStats(p)).map((d) => d.name).sort();
|
||||
}
|
||||
|
||||
/** 同 adapters statSafe 的口径;远端失败(连不上 / 超时 / 鉴权)返回 error,只有明确不存在才是 missing */
|
||||
export async function vstatSafe(p: string, opts: { root?: string; sentinel?: string } = {}): Promise<StatResult> {
|
||||
if (!parseVPath(p)) return statSafe(p, opts);
|
||||
if (opts.root) {
|
||||
const h = await vcheckRoot(opts.root, opts.sentinel);
|
||||
if (h.kind === "offline") return { kind: "root_offline", root: opts.root, reason: h.reason };
|
||||
}
|
||||
try {
|
||||
return { kind: "ok", facts: factsOf(await vlstat(p)) };
|
||||
} catch (e) {
|
||||
const err = remoteError(e, p);
|
||||
if (err.code === "ENOENT") return { kind: "missing" };
|
||||
if (err.code === "EACCES") return { kind: "permission_denied" };
|
||||
return { kind: "error", code: err.code === "ETIMEDOUT" ? "TIMEOUT" : "IO", message: err.message };
|
||||
}
|
||||
}
|
||||
|
||||
/** 根在线:本地按哨兵;远端 = 根路径存在且是目录(Alist 不用哨兵,同 MP) */
|
||||
export async function vcheckRoot(root: string, sentinel: string = DEFAULT_ROOT_SENTINEL): Promise<RootHealth> {
|
||||
const v = parseVPath(root);
|
||||
if (!v) return checkRoot(root, sentinel);
|
||||
try {
|
||||
const st = await vlstat(root);
|
||||
if (!st.isDirectory()) return { kind: "offline", reason: "root_missing" };
|
||||
return { kind: "online", dev: remoteDev(v.storageId) };
|
||||
} catch (e) {
|
||||
const err = remoteError(e, root);
|
||||
if (err.code === "ENOENT") return { kind: "offline", reason: "root_missing" };
|
||||
if (err.code === "EACCES") return { kind: "offline", reason: "permission_denied", code: "EACCES" };
|
||||
if (err.code === "ETIMEDOUT") return { kind: "offline", reason: "timeout" };
|
||||
return { kind: "offline", reason: "error", code: err.code };
|
||||
}
|
||||
}
|
||||
|
||||
export async function vmkdirp(p: string): Promise<void> {
|
||||
const v = parseVPath(p);
|
||||
if (!v) {
|
||||
await mkdir(p, { recursive: true });
|
||||
return;
|
||||
}
|
||||
const { client } = await remote(p);
|
||||
invalidateRemote(p);
|
||||
await client.mkdir(v.remote).catch((e) => {
|
||||
throw remoteError(e, p);
|
||||
});
|
||||
}
|
||||
|
||||
export async function vexists(p: string): Promise<boolean> {
|
||||
try {
|
||||
await vlstat(p);
|
||||
return true;
|
||||
} catch (e) {
|
||||
if ((e as { code?: string }).code === "ENOENT") return false;
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
export async function vreadFile(p: string, maxBytes = 4 * 1024 * 1024): Promise<Uint8Array> {
|
||||
const v = parseVPath(p);
|
||||
if (!v) return new Uint8Array(await readFile(p));
|
||||
const { client } = await remote(p);
|
||||
try {
|
||||
return await client.readFile(v.remote, maxBytes);
|
||||
} catch (e) {
|
||||
throw remoteError(e, p);
|
||||
}
|
||||
}
|
||||
|
||||
/** 本机临时目录(远端写入的本地中转) */
|
||||
export function relayDir(): string {
|
||||
return join(storageDataDir() ?? Deno.env.get("DATA_DIR") ?? tmpdir(), "cache", "storage-relay");
|
||||
}
|
||||
|
||||
/**
|
||||
* 原子写:本地同目录临时文件 → rename;远端本地生成后上传(同 MP 刮削 `_save_file`):
|
||||
* 上传到旁边的临时名,成功后删旧文件(若有)再改名。
|
||||
*/
|
||||
export async function vwriteFileAtomic(p: string, content: Uint8Array | string): Promise<void> {
|
||||
const v = parseVPath(p);
|
||||
if (!v) {
|
||||
await mkdir(dirname(p), { recursive: true });
|
||||
const tmp = `${p}.${randomUUID().slice(0, 8)}.tmp`;
|
||||
try {
|
||||
await writeFile(tmp, content);
|
||||
await rename(tmp, p);
|
||||
} catch (e) {
|
||||
await rm(tmp, { force: true }).catch(() => {});
|
||||
throw e;
|
||||
}
|
||||
return;
|
||||
}
|
||||
const { client } = await remote(p);
|
||||
invalidateRemote(p);
|
||||
const dir = relayDir();
|
||||
await mkdir(dir, { recursive: true });
|
||||
const local = join(dir, `${randomUUID()}-${basename(p)}`);
|
||||
const stage = `${v.remote}.umt-new`;
|
||||
try {
|
||||
await writeFile(local, content);
|
||||
if (await client.stat(stage)) await client.removePath(stage);
|
||||
await client.upload(local, stage);
|
||||
if (await client.stat(v.remote)) await client.removePath(v.remote);
|
||||
await client.rename(stage, posix.basename(v.remote));
|
||||
} catch (e) {
|
||||
await client.removePath(stage).catch(() => undefined);
|
||||
throw remoteError(e, p);
|
||||
} finally {
|
||||
await rm(local, { force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
/** 删除文件或目录(远端 remove;本地 rm -r) */
|
||||
export async function vremove(p: string): Promise<void> {
|
||||
const v = parseVPath(p);
|
||||
if (!v) {
|
||||
await rm(p, { recursive: true, force: true });
|
||||
return;
|
||||
}
|
||||
const { client } = await remote(p);
|
||||
invalidateRemote(p);
|
||||
await client.removePath(v.remote).catch((e) => {
|
||||
throw remoteError(e, p);
|
||||
});
|
||||
}
|
||||
|
||||
/** 同存储内移动 / 改名(远端用 API move,目标已存在 EEXIST;本地 rename 不覆盖由调用方保证) */
|
||||
export async function vmove(src: string, dst: string): Promise<void> {
|
||||
const a = parseVPath(src);
|
||||
const b = parseVPath(dst);
|
||||
if (!a && !b) {
|
||||
await mkdir(dirname(dst), { recursive: true });
|
||||
await rename(src, dst);
|
||||
return;
|
||||
}
|
||||
if (!a || !b || a.storageId !== b.storageId)
|
||||
throw Object.assign(new Error(`跨存储不能直接移动：${src} → ${dst}`), { code: "EXDEV" });
|
||||
const { client } = await remote(src);
|
||||
invalidateRemote(src);
|
||||
invalidateRemote(dst);
|
||||
await client.moveFile(a.remote, b.remote).catch((e) => {
|
||||
throw remoteError(e, src);
|
||||
});
|
||||
}
|
||||
|
||||
/** 远端直链(ffprobe 用;日志前要脱敏) */
|
||||
export async function vrawUrl(p: string): Promise<string> {
|
||||
const { v, client } = await remote(p);
|
||||
return client.rawUrl(v.remote).catch((e) => {
|
||||
throw remoteError(e, p);
|
||||
});
|
||||
}
|
||||
|
||||
/** 同目录不覆盖改名(远端 Alist rename 本身拒绝同名;本地走 adapters moveSafe) */
|
||||
export async function vrenameNoClobber(from: string, to: string): Promise<{ status: "moved" | "exists" | "exdev" }> {
|
||||
const a = parseVPath(from);
|
||||
if (!a) {
|
||||
const { moveSafe } = await import("@umt/adapters");
|
||||
return moveSafe(from, to);
|
||||
}
|
||||
const b = parseVPath(to);
|
||||
if (!b || b.storageId !== a.storageId) return { status: "exdev" };
|
||||
const { client } = await remote(from);
|
||||
invalidateRemote(from);
|
||||
invalidateRemote(to);
|
||||
try {
|
||||
if (await client.stat(b.remote)) return { status: "exists" };
|
||||
if (posix.dirname(a.remote) === posix.dirname(b.remote)) await client.rename(a.remote, posix.basename(b.remote));
|
||||
else await client.moveFile(a.remote, b.remote);
|
||||
return { status: "moved" };
|
||||
} catch (e) {
|
||||
if (e instanceof AlistError && e.kind === "conflict") return { status: "exists" };
|
||||
throw remoteError(e, from);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
import { posix } from "node:path";
|
||||
|
||||
// 远端路径的虚拟前缀(docs/design/storage-alist.md §0、§3):/@alist/<存储 id>/<Alist 内路径>。
|
||||
// 它是合法的 POSIX 绝对路径,join / dirname / relative 照常可用;本机没有 /@alist,
|
||||
// 不认识前缀的旧代码对它做本地 IO 只会得到 ENOENT(失败安全)。
|
||||
|
||||
export const VROOT = "/@alist";
|
||||
const RE = /^\/@alist\/([0-9a-f-]{36})(\/.*)?$/i;
|
||||
|
||||
export interface VPath {
|
||||
storageId: string;
|
||||
/** Alist 内绝对路径(以 / 开头,无尾斜杠) */
|
||||
remote: string;
|
||||
}
|
||||
|
||||
export const isRemotePath = (p: string | null | undefined): boolean =>
|
||||
!!p && (p === VROOT || p.startsWith(`${VROOT}/`));
|
||||
|
||||
export function parseVPath(p: string): VPath | null {
|
||||
if (!isRemotePath(p)) return null;
|
||||
const m = RE.exec(p.replace(/\/+$/, ""));
|
||||
if (!m) return null;
|
||||
const remote = posix.normalize(m[2] ?? "/");
|
||||
if (remote.split("/").includes("..")) return null;
|
||||
return { storageId: m[1]!.toLowerCase(), remote: remote === "/" ? "/" : remote.replace(/\/+$/, "") };
|
||||
}
|
||||
|
||||
export function toVPath(storageId: string, remote: string): string {
|
||||
const r = posix.normalize(`/${remote}`).replace(/\/+$/, "");
|
||||
return `${VROOT}/${storageId}${r === "" || r === "/" ? "" : r}`;
|
||||
}
|
||||
|
||||
/** 同一存储实例? */
|
||||
export function sameStorage(a: string, b: string): boolean {
|
||||
const pa = parseVPath(a);
|
||||
const pb = parseVPath(b);
|
||||
return !!pa && !!pb && pa.storageId === pb.storageId;
|
||||
}
|
||||
|
||||
/** 界面 / 日志展示:「存储名:/路径」;本地原样 */
|
||||
export function displayPath(p: string, names: ReadonlyMap<string, string> = new Map()): string {
|
||||
const v = parseVPath(p);
|
||||
if (!v) return p;
|
||||
return `${names.get(v.storageId) ?? "Alist"}:${v.remote}`;
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
import { createMockAlist, serveMockAlist } from "@umt/adapters/alist-mock";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { buildHealth } from "../../src/modules/system/health.ts";
|
||||
import { clearStorageCache, storageEntry } from "../../src/storage/registry.ts";
|
||||
import { DEV_KEY, json } from "../helpers.ts";
|
||||
import { bearer, createHarness, type Harness, pgAvailable } from "./harness.ts";
|
||||
|
||||
// 存储实例(Alist / OpenList)设置接口:只写凭据、测试连接(版本 / 用户 / 根路径 / 容量)、
|
||||
// 令牌失效 → offline、被库根引用不能删、健康检查里的存储实例(docs/design/storage-alist.md §3、§4)。
|
||||
|
||||
const hasPg = await pgAvailable();
|
||||
|
||||
describe.skipIf(!hasPg)("存储实例(真实 PG + 模拟 Alist HTTP)", () => {
|
||||
let h: Harness;
|
||||
let admin: ReturnType<typeof bearer>;
|
||||
const mock = createMockAlist({
|
||||
token: "perm-token-xyz",
|
||||
username: "alice",
|
||||
password: "pw-alice",
|
||||
mountDetails: { "/s1": { total_space: 1000, free_space: 250, used_space: 750 } },
|
||||
});
|
||||
mock.mkdirp("/s1/media");
|
||||
let srv: { base: string; close: () => Promise<void> };
|
||||
|
||||
beforeAll(async () => {
|
||||
srv = serveMockAlist(mock);
|
||||
h = await createHarness({ engine: false });
|
||||
admin = bearer(h.app, DEV_KEY);
|
||||
}, 60_000);
|
||||
afterAll(async () => {
|
||||
await h?.close();
|
||||
await srv?.close();
|
||||
});
|
||||
|
||||
it("新建(令牌):凭据只写不读;测试连接拿到版本、用户、根路径、容量", async () => {
|
||||
const c = await admin("POST", "/storage-instances", {
|
||||
name: "openlist-a",
|
||||
baseUrl: `${srv.base}/`,
|
||||
rootPath: "/s1/media/",
|
||||
authKind: "token",
|
||||
credential: "perm-token-xyz",
|
||||
pathPassword: "folder-pw",
|
||||
});
|
||||
expect(c.status).toBe(200);
|
||||
const row = (await json(c)).data;
|
||||
expect(row.baseUrl).toBe(srv.base);
|
||||
expect(row.rootPath).toBe("/s1/media");
|
||||
expect(row.secretRef).toBeTruthy();
|
||||
expect(row.pathPasswordRef).toBeTruthy();
|
||||
expect(JSON.stringify(row)).not.toContain("perm-token-xyz");
|
||||
expect(JSON.stringify(row)).not.toContain("folder-pw");
|
||||
expect(await h.ctx.secrets.resolve(row.secretRef)).toBe("perm-token-xyz");
|
||||
|
||||
const t = (await json(await admin("POST", `/storage-instances/${row.id}/test`))).data;
|
||||
expect(t).toMatchObject({ ok: true, healthStatus: "healthy", user: "alice", rootOk: true });
|
||||
expect(t.version).toContain("v4");
|
||||
expect(t.usage).toEqual({ total: 1000, free: 250, used: 750 });
|
||||
const after = (await json(await admin("GET", `/storage-instances/${row.id}`))).data;
|
||||
expect(after.healthStatus).toBe("healthy");
|
||||
});
|
||||
|
||||
it("账号密码:缺密码 422;改成错密码后测试 = offline 且信息里没有密码", async () => {
|
||||
const bad = await admin("POST", "/storage-instances", {
|
||||
name: "pw-missing",
|
||||
baseUrl: srv.base,
|
||||
authKind: "password",
|
||||
username: "alice",
|
||||
});
|
||||
expect(bad.status).toBe(422);
|
||||
const c = (
|
||||
await json(
|
||||
await admin("POST", "/storage-instances", {
|
||||
name: "openlist-pw",
|
||||
baseUrl: srv.base,
|
||||
authKind: "password",
|
||||
username: "alice",
|
||||
credential: "pw-alice",
|
||||
}),
|
||||
)
|
||||
).data;
|
||||
expect((await json(await admin("POST", `/storage-instances/${c.id}/test`))).data.ok).toBe(true);
|
||||
await admin("PATCH", `/storage-instances/${c.id}`, { credential: "wrong-secret-pw" });
|
||||
const t = (await json(await admin("POST", `/storage-instances/${c.id}/test`))).data;
|
||||
expect(t).toMatchObject({ ok: false, healthStatus: "offline" });
|
||||
expect(t.message).toContain("登录失败");
|
||||
expect(t.message).not.toContain("wrong-secret-pw");
|
||||
});
|
||||
|
||||
it("永久令牌失效:测试与健康检查都报不可用", async () => {
|
||||
const c = (
|
||||
await json(
|
||||
await admin("POST", "/storage-instances", {
|
||||
name: "openlist-badtoken",
|
||||
baseUrl: srv.base,
|
||||
rootPath: "/s1",
|
||||
credential: "revoked-token",
|
||||
}),
|
||||
)
|
||||
).data;
|
||||
const t = (await json(await admin("POST", `/storage-instances/${c.id}/test`))).data;
|
||||
expect(t.ok).toBe(false);
|
||||
expect(t.message).toContain("令牌失效");
|
||||
clearStorageCache();
|
||||
const health = await buildHealth({
|
||||
db: h.ctx.db,
|
||||
engineSchema: "openworkflow",
|
||||
dataDir: "/tmp",
|
||||
self: { version: "t", startedAt: new Date(), sseConnections: 0 },
|
||||
thresholds: { warnPercent: 85, criticalPercent: 95 },
|
||||
});
|
||||
const m = health.mounts.find((x) => x.kind === "storage_instance" && x.id === c.id);
|
||||
expect(m).toMatchObject({ status: "offline", reason: "permission_denied" });
|
||||
const ok = health.mounts.find((x) => x.kind === "storage_instance" && x.name === "openlist-a");
|
||||
expect(ok?.status).toBe("online");
|
||||
});
|
||||
|
||||
it("改凭据后注册表重建客户端;清除路径密码", async () => {
|
||||
const row = (await json(await admin("GET", "/storage-instances?limit=50"))).data.items.find(
|
||||
(x: { name: string }) => x.name === "openlist-a",
|
||||
);
|
||||
const e1 = await storageEntry(row.id);
|
||||
expect(e1.secrets).toContain("folder-pw");
|
||||
const r = (await json(await admin("PATCH", `/storage-instances/${row.id}`, { pathPassword: null }))).data;
|
||||
expect(r.pathPasswordRef).toBeNull();
|
||||
const e2 = await storageEntry(row.id);
|
||||
expect(e2.client).not.toBe(e1.client);
|
||||
expect(e2.secrets).not.toContain("folder-pw");
|
||||
});
|
||||
|
||||
it("被媒体库目录引用时不能删;解除后可删,凭据一并删除", async () => {
|
||||
const c = (
|
||||
await json(
|
||||
await admin("POST", "/storage-instances", {
|
||||
name: "to-delete",
|
||||
baseUrl: srv.base,
|
||||
rootPath: "/s1",
|
||||
credential: "perm-token-xyz",
|
||||
}),
|
||||
)
|
||||
).data;
|
||||
const lib = await admin("POST", "/library-roots", {
|
||||
name: "Alist 电影",
|
||||
localPath: `/@alist/${c.id}/s1/media`,
|
||||
mediaKind: "movie",
|
||||
});
|
||||
expect(lib.status).toBe(200);
|
||||
const libRow = (await json(lib)).data;
|
||||
expect(libRow.sentinelFile).toBeNull();
|
||||
const missing = await admin("POST", "/library-roots", {
|
||||
name: "不存在的目录",
|
||||
localPath: `/@alist/${c.id}/s1/nope`,
|
||||
});
|
||||
expect(missing.status).toBe(412);
|
||||
const blocked = await admin("DELETE", `/storage-instances/${c.id}`);
|
||||
expect(blocked.status).toBe(409);
|
||||
const test = (await json(await admin("POST", `/library-roots/${libRow.id}/test`))).data;
|
||||
expect(test.ok).toBe(true);
|
||||
await admin("DELETE", `/library-roots/${libRow.id}`);
|
||||
const del = await admin("DELETE", `/storage-instances/${c.id}`);
|
||||
expect(del.status).toBe(200);
|
||||
await expect(h.ctx.secrets.resolve(c.secretRef)).rejects.toThrow();
|
||||
const audits = await h.ctx.db
|
||||
.selectFrom("audit_event")
|
||||
.select("action")
|
||||
.where("action", "like", "storage_instance.%")
|
||||
.execute();
|
||||
expect(new Set(audits.map((a) => a.action))).toEqual(
|
||||
new Set([
|
||||
"storage_instance.create",
|
||||
"storage_instance.update",
|
||||
"storage_instance.test",
|
||||
"storage_instance.delete",
|
||||
]),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -49,6 +49,8 @@ export interface ProbeOptions {
|
||||
/** ffprobe 可执行文件,默认 "ffprobe"(PATH 查找) */
|
||||
bin?: string;
|
||||
timeoutMs?: number;
|
||||
/** 输入前的附加参数(远端直链用 -rw_timeout 等) */
|
||||
inputArgs?: string[];
|
||||
}
|
||||
|
||||
const optNum = (v: unknown): number | undefined => {
|
||||
@@ -78,7 +80,7 @@ export function ffprobe(path: string, opts: ProbeOptions = {}): Promise<ProbeRes
|
||||
return new Promise((resolve) => {
|
||||
execFile(
|
||||
opts.bin ?? "ffprobe",
|
||||
["-v", "error", "-print_format", "json", "-show_format", "-show_streams", "--", path],
|
||||
["-v", "error", ...(opts.inputArgs ?? []), "-print_format", "json", "-show_format", "-show_streams", "--", path],
|
||||
{ timeout: opts.timeoutMs ?? 30_000, maxBuffer: 16 * 1024 * 1024 },
|
||||
(err, stdout, stderr) => {
|
||||
if (err) {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { z } from "zod";
|
||||
import { IdParam } from "../common.ts";
|
||||
import * as instances from "../instances.ts";
|
||||
import * as storage from "../storage.ts";
|
||||
import { crud, del, ep, instanceTest, R } from "./shared.ts";
|
||||
|
||||
export const instancesApi = {
|
||||
@@ -54,6 +55,27 @@ export const instancesApi = {
|
||||
test: instanceTest("/media-server-instances/{id}/test", "测试媒体服务器连接"),
|
||||
remove: del("/media-server-instances/{id}", "删除媒体服务器实例"),
|
||||
},
|
||||
storageInstances: {
|
||||
...crud(
|
||||
"/storage-instances",
|
||||
"存储实例",
|
||||
{
|
||||
row: storage.StorageInstance,
|
||||
create: storage.StorageInstanceCreate,
|
||||
update: storage.StorageInstanceUpdate,
|
||||
},
|
||||
R,
|
||||
),
|
||||
test: ep({
|
||||
method: "post",
|
||||
path: "/storage-instances/{id}/test",
|
||||
scope: "admin",
|
||||
summary: "测试 Alist / OpenList 连接（版本、登录用户、根路径、容量）",
|
||||
params: IdParam,
|
||||
response: storage.StorageTestResult,
|
||||
}),
|
||||
remove: del("/storage-instances/{id}", "删除存储实例"),
|
||||
},
|
||||
pathMappings: {
|
||||
...crud(
|
||||
"/path-mappings",
|
||||
|
||||
@@ -225,6 +225,10 @@ export const AuditAction = z.enum([
|
||||
"media_server_instance.update",
|
||||
"media_server_instance.delete",
|
||||
"media_server_instance.test",
|
||||
"storage_instance.create",
|
||||
"storage_instance.update",
|
||||
"storage_instance.delete",
|
||||
"storage_instance.test",
|
||||
"path_mapping.create",
|
||||
"path_mapping.update",
|
||||
"path_mapping.delete",
|
||||
|
||||
@@ -31,5 +31,6 @@ export * from "./redact.ts";
|
||||
export * from "./scrape.ts";
|
||||
export * from "./settings.ts";
|
||||
export * from "./sources.ts";
|
||||
export * from "./storage.ts";
|
||||
export * from "./subtitles.ts";
|
||||
export * from "./system-resources.ts";
|
||||
@@ -168,6 +168,7 @@ export const SecretPurpose = z.enum([
|
||||
"media_server_api_key",
|
||||
"metadata_api_key",
|
||||
"llm_api_key",
|
||||
"storage_credential",
|
||||
"other",
|
||||
]);
|
||||
export type SecretPurpose = z.infer<typeof SecretPurpose>;
|
||||
@@ -448,7 +449,8 @@ export const SystemDisk = z.object({
|
||||
export type SystemDisk = z.infer<typeof SystemDisk>;
|
||||
|
||||
export const SystemMount = z.object({
|
||||
kind: z.enum(["library_root", "path_mapping"]),
|
||||
/** storage_instance = Alist / OpenList 实例(连通 + 根路径可访问;令牌失效为 offline/auth) */
|
||||
kind: z.enum(["library_root", "path_mapping", "storage_instance"]),
|
||||
id: z.string(),
|
||||
name: z.string(),
|
||||
path: z.string(),
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
import { z } from "zod";
|
||||
import { EntityBase, Id, JsonObject, Timestamp, WriteOnlySecret } from "./common.ts";
|
||||
import { InstanceHealth } from "./instances.ts";
|
||||
|
||||
// Alist / OpenList 存储实例(docs/design/storage-alist.md §3)。凭据(令牌 / 密码)与路径密码只写不读。
|
||||
|
||||
export const StorageKind = z.enum(["alist"]);
|
||||
export type StorageKind = z.infer<typeof StorageKind>;
|
||||
/** 只用于展示与版本差异;接口相同(MP 的 AlistGo 是同实现的子类) */
|
||||
export const StorageFlavor = z.enum(["alist", "openlist"]);
|
||||
export type StorageFlavor = z.infer<typeof StorageFlavor>;
|
||||
/** token 永久令牌(Alist 后台「设置 → 其他 → 令牌」);password 账号密码换临时令牌;guest 访客 */
|
||||
export const StorageAuthKind = z.enum(["token", "password", "guest"]);
|
||||
export type StorageAuthKind = z.infer<typeof StorageAuthKind>;
|
||||
|
||||
/** 实例高级选项;不填用缺省(括号内) */
|
||||
export const StorageOptions = z
|
||||
.object({
|
||||
/** 每秒请求数(5);0 不限 */
|
||||
qps: z.number().int().min(0).max(100),
|
||||
/** 并发(4);0 不限 */
|
||||
concurrency: z.number().int().min(0).max(32),
|
||||
/** 普通请求超时秒(30) */
|
||||
timeoutSec: z.number().int().min(5).max(600),
|
||||
/** 上传 / 下载无进度空闲超时秒(120) */
|
||||
idleTimeoutSec: z.number().int().min(10).max(3600),
|
||||
/** 等待 Alist 复制 / 移动 / 上传任务的上限分钟(360) */
|
||||
taskTimeoutMin: z.number().int().min(1).max(10080),
|
||||
/** 可重试错误的重试次数(3) */
|
||||
retries: z.number().int().min(0).max(10),
|
||||
/** 上传用 As-Task(false,同 MP) */
|
||||
uploadAsTask: z.boolean(),
|
||||
/** 目录监控最大深度(5,同 MP) */
|
||||
snapshotMaxDepth: z.number().int().min(1).max(20),
|
||||
/** 目录监控按目录修改时间跳过未变子树(true) */
|
||||
snapshotCheckFolderModtime: z.boolean(),
|
||||
})
|
||||
.partial();
|
||||
export type StorageOptions = z.infer<typeof StorageOptions>;
|
||||
|
||||
export const STORAGE_OPTION_DEFAULTS: Required<StorageOptions> = {
|
||||
qps: 5,
|
||||
concurrency: 4,
|
||||
timeoutSec: 30,
|
||||
idleTimeoutSec: 120,
|
||||
taskTimeoutMin: 360,
|
||||
retries: 3,
|
||||
uploadAsTask: false,
|
||||
snapshotMaxDepth: 5,
|
||||
snapshotCheckFolderModtime: true,
|
||||
};
|
||||
|
||||
/** Alist 内路径:以 / 开头,不含 .. 段 */
|
||||
export const AlistPath = z
|
||||
.string()
|
||||
.max(4096)
|
||||
.refine((p) => p.startsWith("/") && !p.includes("\0") && !p.split("/").some((s) => s === ".." || s === "."), {
|
||||
message: "Alist 路径必须以 / 开头，不能含 . 或 ..",
|
||||
});
|
||||
|
||||
export const StorageInstance = EntityBase.extend({
|
||||
name: z.string().min(1).max(100),
|
||||
kind: StorageKind,
|
||||
flavor: StorageFlavor,
|
||||
baseUrl: z.url(),
|
||||
/** 本系统可浏览 / 使用的 Alist 根路径(缺省 /) */
|
||||
rootPath: AlistPath,
|
||||
authKind: StorageAuthKind,
|
||||
username: z.string().nullable(),
|
||||
secretRef: Id.nullable(),
|
||||
pathPasswordRef: Id.nullable(),
|
||||
enabled: z.boolean(),
|
||||
options: StorageOptions,
|
||||
healthStatus: InstanceHealth,
|
||||
lastHealthAt: Timestamp.nullable(),
|
||||
healthDetail: JsonObject,
|
||||
versionInfo: JsonObject,
|
||||
});
|
||||
export type StorageInstance = z.infer<typeof StorageInstance>;
|
||||
|
||||
export const StorageInstanceCreate = StorageInstance.pick({
|
||||
name: true,
|
||||
flavor: true,
|
||||
baseUrl: true,
|
||||
rootPath: true,
|
||||
authKind: true,
|
||||
username: true,
|
||||
enabled: true,
|
||||
options: true,
|
||||
})
|
||||
.partial({ flavor: true, rootPath: true, authKind: true, username: true, enabled: true, options: true })
|
||||
.extend({
|
||||
/** 令牌(authKind=token)或密码(authKind=password);只写 */
|
||||
credential: WriteOnlySecret(4096).optional(),
|
||||
/** 路径密码;只写 */
|
||||
pathPassword: WriteOnlySecret(1024).optional(),
|
||||
});
|
||||
export type StorageInstanceCreate = z.infer<typeof StorageInstanceCreate>;
|
||||
/** 更新:credential / pathPassword 省略或空串 = 不变,null = 清除 */
|
||||
export const StorageInstanceUpdate = StorageInstanceCreate.partial().extend({
|
||||
credential: WriteOnlySecret(4096).nullable().optional(),
|
||||
pathPassword: WriteOnlySecret(1024).nullable().optional(),
|
||||
});
|
||||
export type StorageInstanceUpdate = z.infer<typeof StorageInstanceUpdate>;
|
||||
|
||||
export const StorageUsage = z.object({ total: z.number(), free: z.number(), used: z.number() });
|
||||
export type StorageUsage = z.infer<typeof StorageUsage>;
|
||||
|
||||
export const StorageTestResult = z.object({
|
||||
ok: z.boolean(),
|
||||
healthStatus: InstanceHealth,
|
||||
version: z.string().nullable(),
|
||||
/** 登录用户(访客为 guest) */
|
||||
user: z.string().nullable(),
|
||||
/** 根路径存在且是目录 */
|
||||
rootOk: z.boolean(),
|
||||
/** 容量;驱动不给或令牌不是管理员时为 null(显示「未知」) */
|
||||
usage: StorageUsage.nullable(),
|
||||
message: z.string().nullable(),
|
||||
});
|
||||
export type StorageTestResult = z.infer<typeof StorageTestResult>;
|
||||
@@ -0,0 +1,62 @@
|
||||
import type { Kysely } from "kysely";
|
||||
import { entityTable, execDdl, oneOf } from "../src/migration-helpers.ts";
|
||||
|
||||
// Alist / OpenList 存储实例(docs/design/storage-alist.md §3):远端路径用虚拟前缀 /@alist/<id>/<路径> 表示,
|
||||
// 库根 local_path、整理记录、资产位置里都直接存这个路径,不需要新列。凭据走 secret 表(purpose storage_credential)。
|
||||
|
||||
const PURPOSE_OLD = [
|
||||
"site_cookie",
|
||||
"site_passkey",
|
||||
"site_api_key",
|
||||
"site_password",
|
||||
"feed_url",
|
||||
"downloader_password",
|
||||
"media_server_api_key",
|
||||
"metadata_api_key",
|
||||
"llm_api_key",
|
||||
"other",
|
||||
];
|
||||
const PURPOSE_NEW = [...PURPOSE_OLD.slice(0, -1), "storage_credential", "other"];
|
||||
const HEALTH = ["unknown", "healthy", "degraded", "offline"];
|
||||
|
||||
const swapPurpose = (values: readonly string[]) => `
|
||||
alter table secret drop constraint if exists secret_purpose_check;
|
||||
alter table secret add constraint secret_purpose_check ${oneOf("purpose", values)};
|
||||
`;
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型
|
||||
export async function up(db: Kysely<any>): Promise<void> {
|
||||
await execDdl(db, swapPurpose(PURPOSE_NEW));
|
||||
await execDdl(
|
||||
db,
|
||||
entityTable(
|
||||
"storage_instance",
|
||||
`
|
||||
name text not null unique,
|
||||
kind text not null default 'alist' ${oneOf("kind", ["alist"])},
|
||||
flavor text not null default 'openlist' ${oneOf("flavor", ["alist", "openlist"])},
|
||||
base_url text not null,
|
||||
root_path text not null default '/' check (root_path like '/%'),
|
||||
auth_kind text not null default 'token' ${oneOf("auth_kind", ["token", "password", "guest"])},
|
||||
username text,
|
||||
secret_ref uuid references secret(id) on delete set null,
|
||||
path_password_ref uuid references secret(id) on delete set null,
|
||||
enabled boolean not null default true,
|
||||
options jsonb not null default '{}'::jsonb,
|
||||
health_status text not null default 'unknown' ${oneOf("health_status", HEALTH)},
|
||||
last_health_at timestamptz,
|
||||
health_detail jsonb not null default '{}'::jsonb,
|
||||
version_info jsonb not null default '{}'::jsonb`,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 同上
|
||||
export async function down(db: Kysely<any>): Promise<void> {
|
||||
await execDdl(
|
||||
db,
|
||||
`drop table if exists storage_instance cascade;
|
||||
delete from secret where purpose = 'storage_credential';
|
||||
${swapPurpose(PURPOSE_OLD)}`,
|
||||
);
|
||||
}
|
||||
@@ -246,6 +246,7 @@ export type SecretPurpose =
|
||||
| "media_server_api_key"
|
||||
| "metadata_api_key"
|
||||
| "llm_api_key"
|
||||
| "storage_credential"
|
||||
| "other";
|
||||
export type ServerMappingItemType = "movie" | "series" | "season" | "episode" | "other";
|
||||
export type ServerMappingMatchStatus = "matched" | "ambiguous" | "unmatched" | "stale";
|
||||
@@ -260,6 +261,10 @@ export type SiteSnapshotKind = "periodic" | "backfill" | "live_search" | "manual
|
||||
export type SiteSnapshotStatus = "running" | "ok" | "partial" | "failed";
|
||||
export type SourceReleaseResolutionStatus = "unresolved" | "resolved" | "ambiguous" | "unidentifiable" | "rejected";
|
||||
export type SourceReleaseAvailability = "active" | "site_disabled" | "expired" | "unknown";
|
||||
export type StorageInstanceKind = "alist";
|
||||
export type StorageInstanceFlavor = "alist" | "openlist";
|
||||
export type StorageInstanceAuthKind = "token" | "password" | "guest";
|
||||
export type StorageInstanceHealthStatus = "unknown" | "healthy" | "degraded" | "offline";
|
||||
export type SubscriptionMediaKind = "movie" | "series" | "variety";
|
||||
export type SubscriptionMode = "once" | "continuous";
|
||||
export type SubscriptionSeasonScope = "current_season" | "following_seasons";
|
||||
@@ -1635,6 +1640,27 @@ export interface SseEventTable {
|
||||
payload: Generated<Json>;
|
||||
}
|
||||
|
||||
export interface StorageInstanceTable {
|
||||
id: Generated<string>;
|
||||
name: string;
|
||||
kind: Generated<StorageInstanceKind>;
|
||||
flavor: Generated<StorageInstanceFlavor>;
|
||||
base_url: string;
|
||||
root_path: Generated<string>;
|
||||
auth_kind: Generated<StorageInstanceAuthKind>;
|
||||
username: string | null;
|
||||
secret_ref: string | null;
|
||||
path_password_ref: string | null;
|
||||
enabled: Generated<boolean>;
|
||||
options: Generated<Json>;
|
||||
health_status: Generated<StorageInstanceHealthStatus>;
|
||||
last_health_at: Timestamptz | null;
|
||||
health_detail: Generated<Json>;
|
||||
version_info: Generated<Json>;
|
||||
created_at: GeneratedTimestamptz;
|
||||
updated_at: GeneratedTimestamptz;
|
||||
}
|
||||
|
||||
export interface SubscriptionTable {
|
||||
id: Generated<string>;
|
||||
media_entity_id: string | null;
|
||||
@@ -1869,6 +1895,7 @@ export interface DB {
|
||||
slot_unit: SlotUnitTable;
|
||||
source_release: SourceReleaseTable;
|
||||
sse_event: SseEventTable;
|
||||
storage_instance: StorageInstanceTable;
|
||||
subscription: SubscriptionTable;
|
||||
subscription_history: SubscriptionHistoryTable;
|
||||
subtitle_write: SubtitleWriteTable;
|
||||
@@ -2115,6 +2142,9 @@ export type SourceReleaseUpdate = Updateable<SourceReleaseTable>;
|
||||
export type SseEventRow = Selectable<SseEventTable>;
|
||||
export type NewSseEvent = Insertable<SseEventTable>;
|
||||
export type SseEventUpdate = Updateable<SseEventTable>;
|
||||
export type StorageInstanceRow = Selectable<StorageInstanceTable>;
|
||||
export type NewStorageInstance = Insertable<StorageInstanceTable>;
|
||||
export type StorageInstanceUpdate = Updateable<StorageInstanceTable>;
|
||||
export type SubscriptionRow = Selectable<SubscriptionTable>;
|
||||
export type NewSubscription = Insertable<SubscriptionTable>;
|
||||
export type SubscriptionUpdate = Updateable<SubscriptionTable>;
|
||||
|
||||
Reference in new issue
Block a user