From b8b2a3d2d5dbe1605db8409404390275f9e436a5 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 6 Oct 2026 23:12:26 +0800 Subject: [PATCH 01/11] =?UTF-8?q?docs(design):=20Alist=20/=20OpenList=20?= =?UTF-8?q?=E5=AD=98=E5=82=A8=E8=AE=BE=E8=AE=A1(=E5=AF=B9=E9=BD=90=20MP=20?= =?UTF-8?q?v3=20=E4=B8=8E=E6=94=B9=E8=BF=9B=E9=A1=B9)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei --- docs/design/storage-alist.md | 230 +++++++++++++++++++++++++++++++++++ 1 file changed, 230 insertions(+) create mode 100644 docs/design/storage-alist.md diff --git a/docs/design/storage-alist.md b/docs/design/storage-alist.md new file mode 100644 index 00000000..2942cf97 --- /dev/null +++ b/docs/design/storage-alist.md @@ -0,0 +1,230 @@ +# Alist / OpenList 存储（对齐 MP v3） + +> 对照源（只读，只参考行为）：`/tmp/umt-moviepilot-v3-review-20261001`（v3.0.10-1）`app/modules/filemanager/storages/{__init__,alist,alistgo}.py`、`transhandler.py`（`__transfer_command` 约 1423–1630 行）、`app/chain/scraping.py`（`_save_file` / `_download_and_save_image` → `storagechain.upload_file`）、`app/monitor/{monitor,poller,snapshot}.py`。 +> 真机核对：临时 OpenList v4.2.6 容器（Local 驱动 ×2），接口返回形状见 §2.4。 +> 用户决定（2026-10-06）：推翻 `moviepilot-filemanager.md` 末节「网盘暂不做」，按 MP 做 Alist / OpenList（115 直连、rclone 仍不做）。 + +## 0. 结论 + +- 引入最小的「存储」概念：`storage_instance` 表登记多个 Alist / OpenList 实例（地址、永久令牌或账号密码、可选路径密码、根路径、限速）。本地仍走现有 fsproxy，远端走新适配器 `packages/adapters/src/alist/`。 +- **远端路径用虚拟前缀表示**：`/@alist/<存储 id>/`。它是合法的 POSIX 绝对路径，`join / dirname / relative` 照常可用，整理计划、整理记录、资产位置、回收站里的路径都不用换结构；不认识前缀的旧代码对它做本地 IO 只会得到 ENOENT（本机没有 `/@alist`），天然失败安全。认识前缀的地方经 `apps/server/src/storage/vfs.ts` 分派到适配器（MP 用 `u115:/x` 形式的 `FileURI` 表达同一件事）。 +- 媒体库目录、整理来源、文件管理都能选存储（对照 MP 目录配置「下载目录 / 媒体库目录各带 storage」）：库根的 `local_path` 存虚拟路径；文件管理多一类根 `st:<存储 id>`（整个实例可浏览）；整理来源选这些根即可。 +- 整理矩阵按 MP：本地→Alist 上传、Alist→本地 下载、同实例 Alist→Alist 用 API copy / move、跨实例 下载再上传（MP 不支持，我们补上）。 +- **网盘方向不做核验（用户决定，2026-10-06）**：凡涉及 Alist 的上传、下载、复制、移动，成功判定与 MP 一致——接口成功 + 目标存在即算成功，不比对大小、不比对哈希（不同网盘返回的大小 / 哈希不可靠，容易误判失败）；移动在这个判定通过后删源。本地 ↔ 本地仍走现有核验（fsproxy、跨卷 copy → 全量 sha256 → 删源）不变。 +- 刮削目标在 Alist：NFO / 图片本地生成后上传到对应目录，已存在策略与本地一致（MP 也是先写临时文件再 `upload_file`）。 +- 目录监控：Alist 目录作为整理来源，按 MP 快照语义（深度上限、可选按目录修改时间跳过未变子树），复用现有来源快照表与「两次观察稳定」规则。 +- ffprobe：远端文件不下载，直接对签名直链（`raw_url` / `/d/...?sign=`）跑 ffprobe，只读文件头（HTTP Range）。MP 核心整理不做 ffprobe（只留事件给插件），我们本地做了「不是视频」核验，远端沿用同一核验但只读头。 +- 相对 MP 的 11 项改进（协调者补充、用户认可）见 §9；其中 3、5、6、7 的「核验」部分被用户随后的决定取消（网盘方向不核验），其余照做。 + +## 1. MP 的 Alist 能力清单 → 我们的落点 + +`StorageBase` 抽象方法逐项（`alist.py`；`alistgo.py` 只是换了 schema 名的子类，接口完全相同）： + +| # | MP 能力 | MP 做法 | 我们 | 落点 | +| --- | --- | --- | --- | --- | +| 1 | 配置 | 单实例（`StorageSchema.Alist` / `AlistGo` 各一份）：`url`、`token`、`username`、`password` | 多实例；另加根路径、路径密码、限速、超时、目录监控开关 | `storage_instance` 表、设置 →「存储」 | +| 2 | 令牌 | 永久令牌优先；否则 `/api/auth/login` 换临时 token，缓存 2 天提前 5 分钟（`@cached ttl`） | 同；且遇 401 / token 无效自动重登一次再试（§9-2） | `AlistClient.token()` | +| 3 | check | 能拿到 token 即可用 | `/api/me` + 根路径 `/api/fs/get`；同时取版本 `/api/public/settings` | 测试连接、健康检查 | +| 4 | list | `/api/fs/list`，`per_page=500` 自动翻页（`total` / `pages_total` / `has_more`），`refresh` 透传 | 同；refresh 只在用户点刷新或写后确认目标时带（§9-8） | `AlistClient.list()` | +| 5 | get_item | `/api/fs/get`，失败一律 None | 「不存在」与其它失败分开（§9-1） | `AlistClient.get()` → `AlistNotFound` / `AlistError` | +| 6 | get_item_strict | 只有 `not found / not exist` 当不存在，其余抛 StorageQueryError | 同（这就是我们的缺省语义） | 同上 | +| 7 | create_folder / get_folder | `/api/fs/mkdir`，之后 `_delay_get_item`（sleep 2s×2 + refresh） | `mkdir` 幂等（Alist 本身递归建）；不 sleep 轮询 | `AlistClient.mkdir()` | +| 8 | delete | `/api/fs/remove {dir, names}` | 同；注意 Alist 删不存在的名字也返回成功 | `AlistClient.remove()` | +| 9 | rename | `/api/fs/rename {path, name}` | 同；目标同名存在时 Alist 返回 403 `file [x] exists`，映射为冲突 | `AlistClient.rename()` | +| 10 | download | `/api/fs/get` 取 `raw_url`，没有就拼 `/d?sign=`；8 KB 块流式写 `TEMP_PATH/`，失败时**返回半截文件** | Range 断点续传、签名过期重取、写临时文件成功后改名（§9-7） | `AlistClient.download()` | +| 11 | upload | `PUT /api/fs/put`，头 `File-Path`（URL 编码）、`As-Task`（缺省 false）、`Content-Length`、`Last-Modified`、`X-File-Md5/Sha1/Sha256`（秒传；先整读一遍算）；超时 6000 s；进度回调；之后 `_delay_get_item` 并按需 rename | 同一套头（三种哈希一遍读出）；无进度空闲超时；退避重试；As-Task 等任务完成；**先查同名、传临时名再改名**（Alist 的 put 会静默覆盖同名文件，真机核对过）；成功判定 = 接口成功 + 目标存在（同 MP，不核验）（§9-4/5） | `AlistClient.upload()` | +| 12 | detail | = get_item | 同 | — | +| 13 | copy / copy_item | `/api/fs/copy {src_dir,dst_dir,names}`，按原名复制后 rename；之后 sleep 2s×2 看目标 | 等异步任务完成（`/api/task/copy/info?tid=`），没有任务 id 时轮询到目标出现；名字不同走暂存目录；成功判定 = 任务成功 + 目标存在（§9-3/4） | `AlistClient.copyFile()` | +| 14 | move / move_item | 先在源目录 rename 成新名，再 `/api/fs/move` | 暂存目录方案，失败尽量回滚（§9-4）；跨驱动 move 也是异步任务，同样等待 | `AlistClient.moveFile()` | +| 15 | link / softlink | 不支持（`pass`） | 不支持；整理方式只给复制 / 移动（MP `transtype = {copy, move}`） | 整理弹窗按存储过滤方式 | +| 16 | usage | 未实现（`pass`） | `/api/admin/storage/list` 的 `mount_details`（total/free/used，驱动支持且令牌是管理员时才有）；拿不到显示「未知」不报错（§9-10） | `AlistClient.usage()` | +| 17 | snapshot | 基类通用实现：递归 list，`max_depth=5`，可选按目录 mtime 跳过（OpenList 由 `OPENLIST_SNAPSHOT_CHECK_FOLDER_MODTIME` 控制），保留上次快照里未遍历子树 | 同语义；按实例限速；深度、mtime 开关在实例配置（§8） | `storage/snapshot.ts` | +| 18 | 路径密码 | 接口带 `password` 参数，没有配置入口 | 实例配置可填（加密保存）（§9-11） | `storage_instance.path_password_ref` | +| 19 | AlistGo（alistgo.com 文档的 v3 分支） | 独立 schema、同实现 | 不单列：同一个适配器；实例上记 `flavor`（alist / openlist）只用于展示与版本差异（§2.5） | — | +| 20 | 存储管理动作（扫码登录等 `/storage/manage`） | Alist 无 | 不适用 | — | + +## 2. 适配器 `packages/adapters/src/alist/` + +### 2.1 接口 + +```ts +new AlistClient({ baseUrl, auth: {kind:"token",token} | {kind:"password",username,password} | {kind:"guest"}, + pathPassword?, fetch?, timeoutMs, idleTimeoutMs, retries, qps, concurrency, taskTimeoutMs, logger? }) +.me() .version() .list(path,{refresh}) .get(path,{refresh}) .exists(path) .mkdir(path) .rename(path,newName) +.remove(dir,names) .copyFile(src,dstPath,opts) .moveFile(src,dstPath,opts) .upload(localFile,dstPath,opts) +.download(remotePath,localFile,opts) .rawUrl(path) .usage(path) .waitTask(kind,id,opts) +``` + +错误类型：`AlistNotFound`（`object not found` / `not exist` / `404`）、`AlistConflict`（`exists`）、`AlistAuthError`（HTTP 401/403 或 code 401）、`AlistRateLimited`（429 / `too many` / `rate limit` / `风控`）、`AlistUnavailable`（连不上 / 超时 / 5xx）、`AlistError`（其它业务错误）。`fsCode` 映射到现有 `FsErrorCode`：NotFound→ENOENT、Conflict→EEXIST、Auth→EACCES、RateLimited/Unavailable→TIMEOUT/IO（可重试）、其它→IO。 + +### 2.2 令牌 + +- 永久令牌（Alist 后台「设置 → 其他 → 令牌」）优先，原样放 `Authorization` 头（不加 Bearer，与 MP 一致）。 +- 账号密码：`POST /api/auth/login`（明文密码，与 MP 一致；不用 `/api/auth/login/hash`，避免版本差异）。token 缓存在客户端实例里，TTL 47 h 55 m（MP：2 天提前 5 分钟）；实例配置变更时客户端重建，缓存随之作废（MP `init_storage` 删缓存）。 +- 任一请求得到 401 / `token is invalidated` / `token is expired`：账号密码模式清缓存重登**一次**再试；永久令牌模式直接抛 `AlistAuthError`，健康检查显示「令牌失效」。 +- 访客模式（无令牌无账号）：只用于只读公开目录，写操作 Alist 自己会拒。 + +### 2.3 限速、超时、重试（§9-8） + +- 每实例一个令牌桶（缺省 5 QPS）+ 并发上限（缺省 4）；上传 / 下载数据流不计入 QPS，但计入并发。 +- 普通请求超时 30 s；数据流用**空闲超时**（缺省 120 s 无字节前进即中止，同 fsproxy `hangTimeoutMs` 口径）。 +- 幂等读（list/get/me/task info）与可安全重放的写（mkdir）对可重试错误（连不上、5xx、429/风控）指数退避重试，缺省 3 次（0.5 s→1 s→2 s，429 起步 5 s）；rename/remove/copy/move 不自动重试（调用方按整理账本重试并先对账）。上传 / 下载在流式中断时按 §2.6/2.7 续传或重传。 +- 日志只记方法、路径、Alist code 与耗时；令牌、密码、`sign`、`raw_url` 查询串一律 redact（`redactAlist`），测试覆盖。 + +### 2.4 真机接口形状（OpenList v4.2.6） + +| 场景 | 返回 | +| --- | --- | +| 不存在 | HTTP 200，`{"code":500,"message":"failed to get obj: object not found"}` | +| 令牌无效 | HTTP 200，`{"code":401,"message":"token is invalidated"}` | +| copy（v4 一律异步，含同驱动） | `{"code":200,"data":{"message":"Successfully created 1 copy task(s)","tasks":[{"id":"…","state":0,…}]}}` | +| copy 目标目录不存在 | `{"code":500,"message":"failed to get dst dir: …object not found"}` → 先 mkdir | +| copy / rename 目标已存在 | `{"code":403,"message":"file [x.bin] exists"}` | +| move 同驱动 | `{"code":200,"data":{"message":"Move operations completed immediately"}}` | +| move 跨驱动 | 同 copy，返回 tasks | +| put（As-Task: false） | `{"code":200,"data":null}`；**目标已存在时静默覆盖** | +| put（As-Task: true） | `data.task {id, state, total_bytes}` | +| 任务查询 | `POST /api/task/{copy\|move\|upload}/info?tid=` → `state` 0 待执行 / 1 执行中 / 2 成功 / 3 取消中 / 4 已取消 / 5 出错 / 6 失败中 / 7 失败 / 8 等待重试 / 9 重试前；`error` 为错误文本；不存在 `{"code":404,"message":"task not found"}` | +| 下载 | `/d/?sign=` 支持 Range（206）；签名错 → 401 | +| remove 不存在的名字 | 返回成功（调用方不能据此判断删除了东西） | + +### 2.5 Alist v3 与 OpenList v4 的差异 + +| 项 | Alist v3（≥3.25） | OpenList v4 | 处理 | +| --- | --- | --- | --- | +| copy / move 返回 | 同驱动常为同步、`data:null`；新版本返回 `tasks` | 返回 `tasks`（move 同驱动立即完成） | 有 `tasks` 等任务；没有则按「目标存在且大小与源一致」退避轮询（§9-3） | +| 任务接口 | 旧版在 `/api/admin/task/copy/info`（管理员），新版 `/api/task/…` | `/api/task/…` | 先试 `/api/task/`，404 再试 `/api/admin/task/`，都不行退回大小轮询 | +| `filtered_total` / `pages_total` / `has_more` | 部分版本没有 | 有 | 都按 MP 的判断链兼容 | +| `mount_details`（容量） | 无 | 驱动支持时有 | 无则「未知」 | +| `hash_info` | 有（驱动给才非空） | 同 | 不用于判定（用户决定网盘方向不核验） | +| 版本号 | `/api/public/settings` `version` | 同（如 `v4.2.6 (Commit …)`） | 存进实例 `version_info` 展示 | + +### 2.6 上传（§9-4/5） + +1. 一遍读出本地文件的 md5 / sha1 / sha256（同一个读流），只作秒传头（照 MP 发）；sha256 顺带记为整理记录的 `checksum`（来自本地源，不拿来比对远端）。 +2. 目标已存在：按调用方策略（整理是「冲突」或「替换」）——**不依赖 put 的覆盖**。 +3. 传到同目录临时名 `.uhd-partial`（As-Task 与 MP 缺省一致为 false；实例可改 true，此时轮询任务直到完成）。读流带进度回调、空闲超时、AbortSignal。 +4. 成功判定与 MP 一致：接口成功 + `get(临时名, refresh=true)` 能取到即成功，不比对大小 / 哈希（用户决定）。 +5. `rename` 成目标名（替换时先把旧文件移走，见 §6.3）。失败删除临时名。 +6. 失败按可重试错误退避重传（整文件重传：Alist 的 put 不支持续传）；重试次数用实例配置。 + +### 2.7 下载（§9-7） + +1. `get(path)` 取 `raw_url`（没有就 `/d?sign=`）。 +2. 写到 `<目标>.uhd-partial`；已有 partial（上次中断）时 `Range: bytes=<已有>-` 续传，服务端不给 206 就从头写。 +3. 401/403/410（签名过期）→ 重新 get 取新链接再续，最多 3 次；其它可重试错误退避续传。 +4. 流正常结束即成功（与 MP 一致，不比对大小 / 哈希，用户决定）；边写边算 sha256 只记为 `checksum`（本地文件的事实，不用于判定）。 +5. 成功后 `link + unlink`（不覆盖，同 `commitNoClobber`）改成目标名；中断保留 partial 供续传，24 h 后由现有 partial 清理带走。 + +## 3. 存储抽象（服务端） + +- 表 `storage_instance`（迁移 0090）：`id, name, kind('alist'), flavor('alist'|'openlist'), base_url, root_path, auth_kind('token'|'password'|'guest'), username, secret_ref, path_password_ref, enabled, options jsonb, health_status, last_health_at, health_detail, version_info, created_at, updated_at`。凭据走现有 `secret` 表（`SecretPurpose` 加 `storage_credential`），API 只写不读（与下载器 / 媒体服务器同语义：省略或空串不变、null 清除）。 +- `options`：`qps`(5)、`concurrency`(4)、`timeoutMs`(30000)、`idleTimeoutMs`(120000)、`taskTimeoutMs`(21600000)、`retries`(3)、`uploadAsTask`(false)、`snapshotMaxDepth`(5)、`snapshotCheckFolderModtime`(true，MP 本地缺省 true；OpenList 在 MP 里是开关、缺省关——我们缺省开并可关)。 +- `apps/server/src/storage/`： + - `registry.ts`：按 id 取实例 + 解密凭据 → `AlistClient`（按 `updated_at` 缓存，配置一改就重建）。 + - `vpath.ts`：`/@alist//` 编解码；`displayPath()` 给界面「存储名:/路径」。 + - `vfs.ts`：本地 / 远端统一的 `lstat / realpath / readdir / statSafe / checkRoot / mkdirp / readFile / writeFileAtomic / rename / move / remove`；本地分支原样调用 node:fs / fsproxy。 + - `transfer.ts`：整理矩阵（§5）。 +- 不为抽象重写本地：本地路径照旧走 fsproxy（挂死隔离、跨卷 copy → 核验 → 删源全部不变）；只在会遇到远端路径的调用点换成 vfs。 + +## 4. 设置与界面 + +- 设置 → 媒体库与目录 →「存储」：列表 + 新增 / 编辑（名称、类型 Alist/OpenList、地址、根路径、认证方式 令牌 / 账号密码 / 访客、令牌或密码、路径密码、高级：限速 / 并发 / 超时 / 上传任务模式 / 监控深度 / 按目录时间跳过）+「测试连接」（显示版本、当前用户、根目录是否可写、容量）。 +- 媒体库目录表单多一个「存储」下拉（本机 / 各 Alist 实例）；选 Alist 时路径为 Alist 内路径（可点浏览），不写哨兵文件，健康 = 根目录可访问。 +- 文件管理：左栏「存储」组（对照 MP 存储切换菜单），每个启用实例一个根 `st:`（从实例根路径开始）；Alist 库根照常出现在「媒体库」组。操作：浏览、新建目录、改名、删除、移动到 / 复制到、上传、下载、整理、刮削。 +- 整理来源：来源目录可选 Alist 根；触发方式只能「手动 / 目录快照」（qB 标签对远端无意义）。 + +## 5. 整理矩阵 + +| 源 → 目标 | 复制 | 移动 | 成功判定 | MP | +| --- | --- | --- | --- | --- | +| 本地 → 本地 | 现有（fsproxy、跨卷 copy → sha256 → 删源） | 现有 | 现有核验（不变） | 同 | +| 本地 → Alist | 上传（§2.6） | 上传成功后删本地源；删源失败 = 「已整理，源未删」待处理（同跨卷） | 接口成功 + 目标存在 | 同（上传成功即删源） | +| Alist → 本地 | 下载（§2.7） | 下载成功后删远端源；删源失败同上 | 下载流完整结束（临时文件改名成功） | 下载后 `SystemUtils.move`；失败可能留半截 | +| Alist → Alist 同实例 | API copy（等任务） | API move（等任务） | 任务成功 + 目标存在 | 同（不等任务，sleep 后看目标） | +| Alist → Alist 跨实例 | 下载到 `DATA_DIR/cache/storage-relay/` → 上传 → 删暂存 | 同，上传成功后删远端源 | 两段各自的判定 | **不支持**（`不支持 X 到 Y 的文件整理`） | +| 原地整理（Alist 库根） | 只登记，不动文件 | — | 文件存在 | MP 无 | + +- 硬链接 / 软链接：远端不支持（MP 同），弹窗只给复制 / 移动；原地整理不受影响。 +- 跨卷开关 `allowCrossVolumeMove` 只管本地跨卷；涉及远端的移动一律「复制 / 上传 / 下载成功 → 删源」，不受该开关限制（与 MP 一致：网盘移动就是上传后删源）。 +- 网盘方向不核验大小 / 哈希是用户决定（不同网盘返回的大小 / 哈希不可靠）；本地 ↔ 本地仍走现有核验。 +- 失败路径（都有测试）：上传中断 / 接口失败 / 上传后目标取不到 → 删远端临时文件、**不删源**；下载中断 → 保留 partial、不删源；删源失败 → 记录 attention；可重试错误（连不上、限流、超时）走整理账本现有的「未到上限不入账、任务重试」。 +- 整理记录、失败重试、智能体兜底识别（识别不到作品自动派）、H&R / 下载器引用检查都照旧：它们只看路径字符串与账本；远端源不会有下载器引用，自然放行。 +- 计划阶段：源 / 目标的 stat、目标冲突、同尺寸是否同一文件（本地用 sha256；远端只在两边 hash_info 有同类哈希时判同，否则按冲突）都走 vfs。 +- 撤销：整理记录「删除」可连带删源 / 媒体库文件——Alist 上的文件移进该存储根下的 `.umt-trash/`（同本地布局，回收站可还原）；本地→Alist 的移动撤不回本地源（源已删），说明里写清。 + +## 6. 核验、回收站、替换与 fsproxy 的关系 + +### 6.0 核验口径 + +- 本地 ↔ 本地：现有核验不变（复制按 `verifyHash`，跨卷移动删源前全量 sha256）。 +- 涉及 Alist 的方向：不比对大小 / 哈希（用户决定）。判定只看接口成功与目标存在；整理记录里的 `checksum` 在能顺带得到本地文件 sha256 时记上（上传的源 / 下载的结果），只是事实记录。 + +### 6.1 fsproxy + +本地 IO 仍全部经 fsproxy 子进程（挂死隔离）。远端 IO 在主进程里经 fetch：网络挂死由空闲超时兜住，不会卡系统调用。跨存储传输中「读本地 / 写本地」一侧直接用流（上传读本地文件、下载写本地 partial），其挂死同样由空闲超时判定。 + +### 6.2 回收站 + +文件管理删除、整理记录删除连带的文件、替换下来的旧文件：Alist 上与本地同布局 `<根>/.umt-trash///<相对路径>`，用 Alist 同驱动 move（立即完成）。还原 = move 回去；清除 / 过期 = remove。跨根不需要（回收站总在同一存储根内）。MP 无回收站，Alist 驱动自己的回收站（如 Local 的 `recycle_bin_path`）不依赖。 + +### 6.3 替换（目标已存在且选「替换」/ 洗版） + +本地：现有原子替换。Alist：旧文件先 move 到 `<根>/.umt-trash/replacement-staging/…`，新文件按 §2.6 上传 / 或 copy 到位，成功后旧文件登记回收站；失败把旧文件 move 回原处。MP「总是覆盖」是先删目标再传，我们保留旧文件可还原。 + +### 6.4 「不是视频」隔离 + +ffprobe（§7）判定没有视频流时，远端新文件同样 move 进 `.umt-trash` 并登记回收站。 + +## 7. ffprobe + +- 远端文件：`AlistClient.rawUrl()` 取签名直链，`ffprobe -rw_timeout 30000000 `；ffprobe 只读容器头与少量包（HTTP Range），不全量下载。直链进日志前 redact。 +- 本地 → Alist：收尾探测目标（远端 URL）；URL 探测失败（驱动不给直链、302 到需要 Cookie 的网盘）时退回探测本地源（复制时还在；移动时在删源前已于执行阶段对源探测并缓存）。都拿不到 → 探测结果「未知」，不阻塞（同本地 ffprobe 失败的口径）。 +- MP：核心整理不做 ffprobe，只有 `TransferRenameEventData` 给插件。我们比 MP 多的「不是视频」核验对远端照样生效。 + +## 8. 目录监控（远端快照） + +- 来源配置选 Alist 根 + 触发方式「目录快照」：`walkSource` 对远端走 `storage/snapshot.ts`：BFS 列目录，深度 ≤ `snapshotMaxDepth`（MP 5）；`snapshotCheckFolderModtime` 开时，子目录 `modified` 不晚于上次扫描时间就沿用上次快照里该子树的文件（根目录每轮都列，用于发现删除，同 MP）；列表失败的子树保留旧快照（MP「查询异常时保留基线」）。 +- 之后完全复用现有语义：首扫建基线不整理；新文件 / 变化文件需连续两次观察大小与修改时间一致才整理；结果按现有批次识别上限分批；失败写 `organize_source_snapshot.last_error`，保留旧快照。 +- 扫描间隔沿用来源配置 `intervalSec`（MP 按文件数自适应 `adjust_interval`；我们已有逐来源间隔，不再自适应）。 +- 按实例限速：快照的 list 走同一个令牌桶，不会打爆网盘。 + +## 9. 相对 MP 的改进（协调者补充，用户认可） + +| # | MP 缺陷 | 我们 | 测试 | +| --- | --- | --- | --- | +| 1 | 网络错误 / 非 200 / code≠200 都返回 None，调用方当「不存在」，可能重复上传或误判 | 不存在（`object not found`）抛 `AlistNotFound`；连接失败 / 鉴权 / 服务端错误是其它错误类型；vfs 映射为 ENOENT 与 IO/TIMEOUT/EACCES；整理遇到后者按可重试失败，不当不存在 | `alist/client.test.ts`「错误分类」、整理「连不上不当不存在」 | +| 2 | token 固定缓存 2 天，不处理 401 | 401 / token 无效自动重登一次再试；永久令牌失效直接报鉴权错误并进健康检查 | 「令牌失效重登」「永久令牌失效」 | +| 3 | copy/move 跨驱动是异步任务，MP sleep 2s×2 后 get，大文件必误判 | 有任务 id 就轮询任务（指数退避 0.5 s→最大 10 s，超时可配，缺省 6 h），失败拿任务 `error`；无任务 id 轮询到目标出现为止 | 「异步任务成功 / 失败 / 超时」「无任务 id 等目标出现」 | +| 4 | 先按原名复制再改名，目标有同名会冲突 / 覆盖 | 执行前查目标冲突（按整理冲突策略）；名字不同或原名在目标目录已被占用时先放暂存目录 `.umt-stage-` 改名再移入，失败清理暂存 | 「改名复制走暂存」「目标冲突不覆盖」 | +| 5 | 上传前整读一遍算三种哈希；PUT 绝对超时 6000 s；无重试 | 三种哈希同一遍读出（只作秒传头，照 MP）；空闲超时；退避重试；As-Task 轮询任务到完成 | 「上传进度 / 中断 / 重试 / As-Task」 | +| 6 | 上传后只确认目标存在 | **保持 MP 判定（用户决定）**：接口成功 + 目标存在即成功，不比对大小 / 哈希；只是确认目标时不靠 sleep | 「上传后目标取不到不删源」 | +| 7 | 下载无续传、raw_url 过期不处理、可能留半截文件当成品 | Range 续传、签名过期重取再续、先写临时文件成功后改名（不比对大小 / 哈希，用户决定） | 「下载续传 / 签名过期 / 中断不改名」 | +| 8 | `_delay_get_item` 每次 refresh=true，穿透网盘 API 易触发风控 | 只在写后确认目标与用户手动刷新时 refresh；每实例 QPS + 并发限制（可配，缺省保守），限流 / 风控退避 | 「refresh 只在需要时」「限流退避」 | +| 9 | 快照每次整树递归 | 深度上限、按目录 mtime 跳过未变子树、按实例限速、结果分批 | 「快照深度 / mtime 跳过 / 列表失败保留」 | +| 10 | usage 未实现 | `mount_details` 能拿到就显示，拿不到「未知」不报错 | 「容量未知不报错」 | +| 11 | 路径密码无配置入口 | 实例可配（加密保存），list/get 带上；日志 redact | 「路径密码透传与 redact」 | + +## 10. 不做的部分与理由 + +| MP 有 | 不做 | 理由 | +| --- | --- | --- | +| Alist 的硬链接 / 软链接整理方式 | 不做 | MP 本身也不支持（`link/softlink` 为空实现，`transtype` 只有复制 / 移动） | +| AlistGo 作为独立存储类型 | 合并 | 接口完全相同（MP 子类只改了 schema 名）；实例上记 `flavor` 展示即可 | +| 远端目录监控的按文件数自适应间隔（`adjust_interval`） | 不做 | 我们已有逐来源的扫描间隔配置，再自适应会让用户设的间隔失效 | +| 远端监控「启动前已存在的文件不处理」以外的整理存量 | 已有 | 「整理存量」对远端同样可用（这不是不做，列此说明口径一致） | +| 115 / 阿里云盘 / SMB / rclone 存储 | 不做 | 用户只要求 Alist / OpenList；115 等可以挂到 Alist 下用 | +| 下载到浏览器时先落 MP 临时目录再回传 | 改为流式代理 | 不占本地盘；仍只允许管理员会话 | + +## 11. 迁移 + +- `0090_storage_instance`：`storage_instance` 表；`secret_purpose` 枚举加 `storage_credential`。 +- `0091` 未使用（预留；整理来源快照沿用 `organize_source_snapshot`，库根沿用 `library_root.local_path` 存虚拟路径，不需要新列）。 + +## 12. 测试 + +- 适配器：`packages/adapters/src/alist/*.test.ts`，进程内 mock Alist（`mock-server.ts`）覆盖全部接口、错误码、令牌失效重登、异步任务、上传下载进度与中断、续传、签名过期、限流。 +- 服务端：整理矩阵每个方向复制 / 移动各一条端到端（mock 存储），含「上传中断不删源」「上传后目标取不到不删源」「下载中断不改名不删源」；刮削写远端；目录监控发现远端新文件；文件管理各操作路由。 +- 前端：存储设置表单、文件管理存储切换。 +- 真机：`*.real.test.ts`，`UMT_ALIST_REAL_URL` / `UMT_ALIST_REAL_TOKEN` 等环境变量开启，缺省跳过；跑本地 → OpenList → 本地往返。 +- 真机验收（用户不提供真实 Alist，协调者自起 OpenList 容器）：步骤见 `docs/runbook/storage-alist.md`；脚本 `scripts/alist-e2e.ts` 可对任意 base URL + 令牌跑往返与 Alist 内移动。 From 5ff24110f84fb4a56cfca13eb14fbdb2d5a4ea87 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 6 Oct 2026 23:20:09 +0800 Subject: [PATCH 02/11] =?UTF-8?q?feat(adapters):=20Alist=20/=20OpenList=20?= =?UTF-8?q?=E5=AE=A2=E6=88=B7=E7=AB=AF=E4=B8=8E=E6=B5=8B=E8=AF=95=E7=94=A8?= =?UTF-8?q?=E6=A8=A1=E6=8B=9F=E6=9C=8D=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 登录(永久令牌优先、账号密码换令牌缓存、401 自动重登)、list 分页、get/raw_url、mkdir/rename/remove、 copy/move 轮询异步任务与暂存目录改名、上传(三种哈希一遍读出、临时名、空闲超时、退避重试、As-Task)、 下载(Range 续传、直链过期重取、临时文件改名)、容量、每实例限速与脱敏。 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei --- packages/adapters/deno.json | 2 +- packages/adapters/package.json | 3 +- packages/adapters/src/alist/client.test.ts | 548 ++++++++++++ packages/adapters/src/alist/client.ts | 951 +++++++++++++++++++++ packages/adapters/src/alist/errors.ts | 70 ++ packages/adapters/src/alist/index.ts | 4 + packages/adapters/src/alist/limiter.ts | 61 ++ packages/adapters/src/alist/mock.ts | 491 +++++++++++ packages/adapters/src/alist/redact.ts | 22 + packages/adapters/src/index.ts | 1 + 10 files changed, 2151 insertions(+), 2 deletions(-) create mode 100644 packages/adapters/src/alist/client.test.ts create mode 100644 packages/adapters/src/alist/client.ts create mode 100644 packages/adapters/src/alist/errors.ts create mode 100644 packages/adapters/src/alist/index.ts create mode 100644 packages/adapters/src/alist/limiter.ts create mode 100644 packages/adapters/src/alist/mock.ts create mode 100644 packages/adapters/src/alist/redact.ts diff --git a/packages/adapters/deno.json b/packages/adapters/deno.json index a458dc95..0c71b687 100644 --- a/packages/adapters/deno.json +++ b/packages/adapters/deno.json @@ -1,7 +1,7 @@ { "name": "@umt/adapters", "version": "0.3.34", - "exports": { ".": "./src/index.ts", "./fsworker": "./src/fs/fsworker.ts" }, + "exports": { ".": "./src/index.ts", "./fsworker": "./src/fs/fsworker.ts", "./alist-mock": "./src/alist/mock.ts" }, "tasks": { "test": "deno run -A npm:vitest run", "typecheck": "deno run -A npm:typescript/tsc --noEmit -p tsconfig.json" diff --git a/packages/adapters/package.json b/packages/adapters/package.json index ebe1453a..4a1a00d4 100644 --- a/packages/adapters/package.json +++ b/packages/adapters/package.json @@ -4,6 +4,7 @@ "type": "module", "exports": { ".": "./src/index.ts", - "./fsworker": "./src/fs/fsworker.ts" + "./fsworker": "./src/fs/fsworker.ts", + "./alist-mock": "./src/alist/mock.ts" } } diff --git a/packages/adapters/src/alist/client.test.ts b/packages/adapters/src/alist/client.test.ts new file mode 100644 index 00000000..e2f02cb5 --- /dev/null +++ b/packages/adapters/src/alist/client.test.ts @@ -0,0 +1,548 @@ +import { Buffer } from "node:buffer"; +import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { AlistClient, type AlistClientOptions, hashLocalFile, PARTIAL_SUFFIX } from "./client.ts"; +import { AlistError, classifyAlist } from "./errors.ts"; +import { AlistLimiter } from "./limiter.ts"; +import { createMockAlist, type MockAlistOptions } from "./mock.ts"; +import { redactAlistText, redactAlistUrl } from "./redact.ts"; + +// 适配器:mock Alist 覆盖全部接口、错误码、令牌失效重登、异步任务、上传下载进度与中断(docs/design/storage-alist.md §12)。 + +let dir: string; +beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), "umt-alist-")); +}); +afterEach(async () => { + await rm(dir, { recursive: true, force: true }); +}); + +const noSleep = async () => {}; +function setup(mo: MockAlistOptions = { token: "perm-token" }, co: Partial = {}) { + const mock = createMockAlist(mo); + const client = new AlistClient({ + baseUrl: mock.base, + auth: mo.username + ? { kind: "password", username: mo.username, password: mo.password ?? "" } + : { kind: "token", token: mo.token ?? "" }, + fetch: mock.fetch, + sleep: noSleep, + qps: 0, + concurrency: 0, + ...co, + }); + return { mock, client }; +} +const bytes = (n: number, seed = 7) => { + const b = new Uint8Array(n); + for (let i = 0; i < n; i++) b[i] = (i * 31 + seed) & 0xff; + return b; +}; +const caught = async (p: Promise) => { + try { + await p; + } catch (e) { + return e as AlistError; + } + throw new Error("expected rejection"); +}; + +describe("错误分类(不存在 ≠ 失败)", () => { + it("真机返回形状 → 类型", () => { + expect(classifyAlist(200, 500, "failed to get obj: object not found").kind).toBe("not_found"); + expect(classifyAlist(200, 401, "token is invalidated").kind).toBe("auth"); + expect(classifyAlist(200, 403, "file [x.bin] exists").kind).toBe("conflict"); + expect(classifyAlist(429, null, "Too Many Requests").kind).toBe("rate_limited"); + expect(classifyAlist(502, null, "bad gateway").kind).toBe("unavailable"); + expect(classifyAlist(200, 500, "failed get storage: storage not found; please add a storage first").kind).toBe( + "not_found", + ); + expect(classifyAlist(200, 500, "some driver error").kind).toBe("error"); + expect(new AlistError("unavailable", "x").retryable).toBe(true); + expect(new AlistError("not_found", "x").fsCode).toBe("ENOENT"); + expect(new AlistError("conflict", "x").fsCode).toBe("EEXIST"); + expect(new AlistError("auth", "x").fsCode).toBe("EACCES"); + }); + + it("get:不存在抛 not_found,stat 返回 null;连不上抛 unavailable(不当成不存在)", async () => { + const { mock, client } = setup({ token: "t" }, { retries: 0 }); + expect((await caught(client.get("/s1/nope"))).kind).toBe("not_found"); + expect(await client.stat("/s1/nope")).toBeNull(); + mock.failNext(1); + expect((await caught(client.stat("/s1/nope"))).kind).toBe("unavailable"); + }); + + it("可重试错误退避重试;次数用尽才失败", async () => { + const { mock, client } = setup({ token: "t" }, { retries: 2 }); + mock.put("/s1/a.txt", "hi"); + mock.failNext(2); + expect((await client.get("/s1/a.txt")).size).toBe(2); + expect(client.stats.retries).toBe(2); + mock.failNext(3); + expect((await caught(client.get("/s1/a.txt"))).kind).toBe("unavailable"); + }); + + it("限流 / 风控(429)退避后成功", async () => { + const slept: number[] = []; + const { mock, client } = setup({ token: "t" }, { sleep: async (ms) => void slept.push(ms) }); + mock.put("/s1/a.txt", "hi"); + mock.rateLimitNext(2); + expect((await client.get("/s1/a.txt")).name).toBe("a.txt"); + expect(slept.length).toBe(2); + expect(slept[0]).toBeGreaterThanOrEqual(5000); + }); +}); + +describe("令牌", () => { + it("永久令牌原样放 Authorization", async () => { + const { mock, client } = setup({ token: "perm" }); + await client.me(); + expect(mock.requests.at(-1)!.headers.authorization).toBe("perm"); + }); + + it("永久令牌失效:直接报鉴权错误,不重试", async () => { + const mock = createMockAlist({ token: "right" }); + const client = new AlistClient({ + baseUrl: mock.base, + auth: { kind: "token", token: "wrong" }, + fetch: mock.fetch, + qps: 0, + }); + const e = await caught(client.me()); + expect(e.kind).toBe("auth"); + expect(mock.requests.length).toBe(1); + }); + + it("账号密码:登录换令牌并缓存;令牌失效自动重登一次再试", async () => { + const { mock, client } = setup({ username: "u", password: "p" }); + await client.me(); + await client.me(); + expect(mock.requests.filter((r) => r.path === "/api/auth/login").length).toBe(1); + mock.invalidateSessions(); + expect((await client.me()).username).toBe("u"); + expect(client.stats.relogins).toBe(1); + expect(mock.requests.filter((r) => r.path === "/api/auth/login").length).toBe(2); + }); + + it("密码错误:鉴权错误", async () => { + const mock = createMockAlist({ username: "u", password: "p" }); + const client = new AlistClient({ + baseUrl: mock.base, + auth: { kind: "password", username: "u", password: "bad" }, + fetch: mock.fetch, + qps: 0, + }); + expect((await caught(client.me())).kind).toBe("auth"); + }); + + it("令牌缓存到期(约 2 天)后重新登录", async () => { + let now = 1_000_000; + const { mock, client } = setup({ username: "u", password: "p" }, { now: () => now }); + await client.me(); + now += 2 * 86_400_000; + await client.me(); + expect(mock.requests.filter((r) => r.path === "/api/auth/login").length).toBe(2); + }); +}); + +describe("读接口", () => { + it("list 自动翻页(每页 500),不带 refresh;用户刷新才带", async () => { + const { mock, client } = setup(); + for (let i = 0; i < 1203; i++) mock.put(`/s1/big/f${String(i).padStart(4, "0")}.txt`, "x"); + const all = await client.list("/s1/big"); + expect(all.length).toBe(1203); + const lists = mock.requests.filter((r) => r.path === "/api/fs/list"); + expect(lists.map((r) => r.body!.page)).toEqual([1, 2, 3]); + expect(lists.every((r) => r.body!.per_page === 500 && r.body!.refresh === false)).toBe(true); + await client.list("/s1/big", { refresh: true }); + expect(mock.requests.filter((r) => r.path === "/api/fs/list").at(-3)!.body!.refresh).toBe(true); + expect(client.stats.refreshes).toBe(1); + }); + + it("路径密码:list / get 都带上", async () => { + const { mock, client } = setup({ token: "t", pathPassword: "pw1" }, { pathPassword: "pw1" }); + mock.put("/s1/a.txt", "hi"); + expect((await client.list("/s1")).map((o) => o.name)).toEqual(["a.txt"]); + expect(mock.requests.at(-1)!.body!.password).toBe("pw1"); + const noPw = new AlistClient({ + baseUrl: mock.base, + auth: { kind: "token", token: "t" }, + fetch: mock.fetch, + qps: 0, + }); + expect((await caught(noPw.list("/s1"))).kind).toBe("auth"); + }); + + it("get 带 raw_url / sign / hash_info;rawUrl 优先 raw_url", async () => { + const { mock, client } = setup(); + mock.put("/s1/a b.mkv", "data", { hashInfo: { md5: "abc" } }); + const o = await client.get("/s1/a b.mkv"); + expect(o.size).toBe(4); + expect(o.hashInfo).toEqual({ md5: "abc" }); + expect(o.sign).not.toBe(""); + expect(await client.rawUrl("/s1/a b.mkv")).toContain("/d/s1/a%20b.mkv?sign="); + }); + + it("容量:有 mount_details 显示,没有返回 null 不报错", async () => { + const { mock, client } = setup({ + token: "t", + mountDetails: { "/s1": { total_space: 1000, free_space: 400, used_space: 600 } }, + }); + mock.mkdirp("/s1/x"); + mock.mkdirp("/s2/y"); + expect(await client.usage("/s1/x")).toEqual({ total: 1000, free: 400, used: 600 }); + expect(await client.usage("/s2/y")).toBeNull(); + mock.failNext(1); + expect(await client.usage("/s1")).toBeNull(); + }); + + it("readFile 读小文件", async () => { + const { mock, client } = setup(); + mock.put("/s1/movie.nfo", ""); + expect(new TextDecoder().decode(await client.readFile("/s1/movie.nfo"))).toBe(""); + }); +}); + +describe("写接口与异步任务", () => { + it("mkdir / rename / remove;改名到已存在 → conflict", async () => { + const { mock, client } = setup(); + await client.mkdir("/s1/a/b"); + expect(mock.exists("/s1/a/b")).toBe(true); + mock.put("/s1/a/x.txt", "1"); + mock.put("/s1/a/y.txt", "2"); + expect((await caught(client.rename("/s1/a/x.txt", "y.txt"))).kind).toBe("conflict"); + await client.rename("/s1/a/x.txt", "z.txt"); + expect(mock.exists("/s1/a/z.txt")).toBe(true); + await client.removePath("/s1/a/z.txt"); + expect(mock.exists("/s1/a/z.txt")).toBe(false); + }); + + it("复制:轮询任务到完成(不是固定 sleep),成功 = 任务成功 + 目标存在", async () => { + const { mock, client } = setup({ token: "t", taskPolls: 3 }); + mock.put("/s1/src/a.mkv", bytes(1000)); + const o = await client.copyFile("/s1/src/a.mkv", "/s2/dst/a.mkv"); + expect(o.size).toBe(1000); + expect(mock.exists("/s1/src/a.mkv")).toBe(true); + const polls = mock.requests.filter((r) => r.path.startsWith("/api/task/copy/info")); + expect(polls.length).toBe(4); + }); + + it("任务失败:拿到任务的错误信息", async () => { + const { mock, client } = setup(); + mock.put("/s1/src/a.mkv", "x"); + mock.failNextTask("upstream quota exceeded"); + const e = await caught(client.copyFile("/s1/src/a.mkv", "/s2/dst/a.mkv")); + expect(e.kind).toBe("task_failed"); + expect(e.message).toContain("upstream quota exceeded"); + }); + + it("任务超时可配", async () => { + let now = 0; + const { mock, client } = setup( + { token: "t", taskPolls: 1000 }, + { + now: () => now, + sleep: async (ms) => { + now += ms; + }, + taskTimeoutMs: 5000, + }, + ); + mock.put("/s1/src/a.mkv", "x"); + expect((await caught(client.copyFile("/s1/src/a.mkv", "/s2/dst/a.mkv"))).kind).toBe("timeout"); + }); + + it("老版本没有任务接口:改为等目标出现", async () => { + const { mock, client } = setup({ token: "t", noTaskApi: true, noTaskIds: true, taskPolls: 2 }); + mock.put("/s1/src/a.mkv", bytes(10)); + const o = await client.copyFile("/s1/src/a.mkv", "/s1/dst/a.mkv"); + expect(o.size).toBe(10); + }); + + it("目标已存在:冲突,不覆盖", async () => { + const { mock, client } = setup(); + mock.put("/s1/src/a.mkv", "new"); + mock.put("/s2/dst/a.mkv", "old"); + expect((await caught(client.copyFile("/s1/src/a.mkv", "/s2/dst/a.mkv"))).kind).toBe("conflict"); + expect(new TextDecoder().decode(mock.read("/s2/dst/a.mkv")!)).toBe("old"); + }); + + it("改名复制走暂存目录:目标目录里已有与源同名的文件也不受影响,暂存清理干净", async () => { + const { mock, client } = setup(); + mock.put("/s1/src/raw.mkv", "new"); + mock.put("/s1/lib/raw.mkv", "unrelated"); + await client.copyFile("/s1/src/raw.mkv", "/s1/lib/Movie (2020).mkv"); + expect(new TextDecoder().decode(mock.read("/s1/lib/Movie (2020).mkv")!)).toBe("new"); + expect(new TextDecoder().decode(mock.read("/s1/lib/raw.mkv")!)).toBe("unrelated"); + expect(mock.listNames("/s1/lib").some((n) => n.startsWith(".umt-stage-"))).toBe(false); + }); + + it("改名复制失败:清理暂存", async () => { + const { mock, client } = setup(); + mock.put("/s1/src/raw.mkv", "new"); + mock.failNextTask("disk full"); + await caught(client.copyFile("/s1/src/raw.mkv", "/s2/lib/Movie.mkv")); + expect(mock.listNames("/s2/lib")).toEqual([]); + }); + + it("移动:同目录只改名;同驱动立即完成;跨驱动等任务;改名移动经暂存", async () => { + const { mock, client } = setup(); + mock.put("/s1/a/x.mkv", "1"); + await client.moveFile("/s1/a/x.mkv", "/s1/a/y.mkv"); + expect(mock.exists("/s1/a/y.mkv")).toBe(true); + await client.moveFile("/s1/a/y.mkv", "/s1/b/y.mkv"); + expect(mock.exists("/s1/b/y.mkv") && !mock.exists("/s1/a/y.mkv")).toBe(true); + await client.moveFile("/s1/b/y.mkv", "/s2/c/Final.mkv"); + expect(mock.exists("/s2/c/Final.mkv")).toBe(true); + expect(mock.exists("/s1/b/y.mkv")).toBe(false); + expect(mock.listNames("/s2/c")).toEqual(["Final.mkv"]); + }); + + it("移动失败:源搬回原处", async () => { + const { mock, client } = setup(); + mock.put("/s1/a/x.mkv", "1"); + mock.failNextTask("network"); + await caught(client.moveFile("/s1/a/x.mkv", "/s2/c/Final.mkv")); + expect(mock.exists("/s1/a/x.mkv")).toBe(true); + expect(mock.listNames("/s2/c")).toEqual([]); + }); +}); + +describe("上传", () => { + it("三种哈希一遍读出,作秒传头;先传临时名再改名;进度回调", async () => { + const { mock, client } = setup(); + const f = join(dir, "a.mkv"); + const data = bytes(3 * 1024 * 1024 + 5); + await writeFile(f, data); + const progress: number[] = []; + const r = await client.upload(f, "/s1/lib/A (2020)/a.mkv", { onProgress: (d) => progress.push(d) }); + expect(r.obj.size).toBe(data.length); + expect(progress.at(-1)).toBe(data.length); + expect(progress.length).toBeGreaterThan(2); + const put = mock.requests.find((r) => r.path === "/api/fs/put")!; + expect(put.headers["x-file-sha256"]).toBe(r.hashes.sha256); + expect(put.headers["x-file-md5"]).toBe(r.hashes.md5); + expect(put.headers["x-file-sha1"]).toBe(r.hashes.sha1); + expect(put.headers["file-path"]).toBe(`/s1/lib/A%20(2020)/a.mkv${PARTIAL_SUFFIX}`); + expect(put.headers["as-task"]).toBe("false"); + expect(put.headers["content-length"]).toBe(String(data.length)); + expect(mock.listNames("/s1/lib/A (2020)")).toEqual(["a.mkv"]); + expect(await hashLocalFile(f)).toEqual(r.hashes); + }); + + it("目标已存在:冲突,不靠 put 覆盖", async () => { + const { mock, client } = setup(); + const f = join(dir, "a.mkv"); + await writeFile(f, "new"); + mock.put("/s1/lib/a.mkv", "old"); + expect((await caught(client.upload(f, "/s1/lib/a.mkv"))).kind).toBe("conflict"); + expect(new TextDecoder().decode(mock.read("/s1/lib/a.mkv")!)).toBe("old"); + expect(mock.requests.some((r) => r.path === "/api/fs/put")).toBe(false); + }); + + it("中断:退避重传;次数用尽时失败且远端不留临时文件", async () => { + const { mock, client } = setup({ token: "t" }, { retries: 1 }); + const f = join(dir, "a.mkv"); + await writeFile(f, bytes(2 * 1024 * 1024)); + mock.breakUploadAfter(1024 * 1024); + const ok = await client.upload(f, "/s1/lib/a.mkv"); + expect(ok.obj.size).toBe(2 * 1024 * 1024); + expect(client.stats.retries).toBe(1); + + const { mock: m2, client: c2 } = setup({ token: "t" }, { retries: 0 }); + m2.breakUploadAfter(1024 * 1024); + const e = await caught(c2.upload(f, "/s1/lib/a.mkv")); + expect(e.kind).toBe("unavailable"); + expect(m2.exists("/s1/lib/a.mkv")).toBe(false); + expect(m2.exists(`/s1/lib/a.mkv${PARTIAL_SUFFIX}`)).toBe(false); + }); + + it("接口成功但目标取不到:失败(调用方据此不删源)", async () => { + let now = 0; + const { mock, client } = setup( + { token: "t" }, + { + now: () => now, + sleep: async (ms) => { + now += ms; + }, + }, + ); + const f = join(dir, "a.mkv"); + await writeFile(f, "x"); + mock.dropNextUpload(); + expect((await caught(client.upload(f, "/s1/lib/a.mkv"))).kind).toBe("not_found"); + }); + + it("As-Task:等上传任务完成", async () => { + const { mock, client } = setup({ token: "t", taskPolls: 2 }, { uploadAsTask: true }); + const f = join(dir, "a.mkv"); + await writeFile(f, "hello"); + const r = await client.upload(f, "/s1/lib/a.mkv"); + expect(r.obj.size).toBe(5); + expect(mock.requests.find((x) => x.path === "/api/fs/put")!.headers["as-task"]).toBe("true"); + expect(mock.requests.filter((x) => x.path.startsWith("/api/task/upload/info")).length).toBe(3); + }); + + it("空闲超时:服务端不读数据时中止", async () => { + const mock = createMockAlist({ token: "t" }); + const stuck: typeof fetch = (async (input: Request | URL | string, init?: RequestInit) => { + const req = input instanceof Request ? input : new Request(String(input), init); + if (new URL(req.url).pathname === "/api/fs/put") + return new Promise((_, reject) => + init?.signal?.addEventListener("abort", () => reject(init.signal!.reason)), + ); + return mock.fetch(input, init); + }) as typeof fetch; + const client = new AlistClient({ + baseUrl: mock.base, + auth: { kind: "token", token: "t" }, + fetch: stuck, + qps: 0, + idleTimeoutMs: 50, + retries: 0, + }); + const f = join(dir, "a.mkv"); + await writeFile(f, "x"); + expect((await caught(client.upload(f, "/s1/a.mkv"))).kind).toBe("timeout"); + }); +}); + +describe("下载", () => { + it("写临时文件成功后改名;带进度;sha256 记事实", async () => { + const { mock, client } = setup(); + const data = bytes(1_500_000); + mock.put("/s1/a.mkv", data); + const out = join(dir, "a.mkv"); + const seen: number[] = []; + const r = await client.download("/s1/a.mkv", out, { onProgress: (d) => seen.push(d) }); + expect(r.bytes).toBe(data.length); + expect(seen.at(-1)).toBe(data.length); + expect(Buffer.compare(await readFile(out), data)).toBe(0); + await expect(stat(`${out}${PARTIAL_SUFFIX}`)).rejects.toThrow(); + }); + + it("中断后 Range 续传", async () => { + const { mock, client } = setup({ token: "t" }, { retries: 2 }); + const data = bytes(1_000_000); + mock.put("/s1/a.mkv", data); + mock.breakDownloadAfter(400_000); + const out = join(dir, "a.mkv"); + await client.download("/s1/a.mkv", out); + expect(Buffer.compare(await readFile(out), data)).toBe(0); + expect(client.stats.retries).toBe(1); + }); + + it("中断且次数用尽:不改名,保留 partial;下次续传完成", async () => { + const { mock, client } = setup({ token: "t" }, { retries: 0 }); + const data = bytes(800_000); + mock.put("/s1/a.mkv", data); + mock.breakDownloadAfter(300_000); + const out = join(dir, "a.mkv"); + expect((await caught(client.download("/s1/a.mkv", out))).kind).toBe("unavailable"); + await expect(stat(out)).rejects.toThrow(); + expect((await stat(`${out}${PARTIAL_SUFFIX}`)).size).toBe(300_000); + await client.download("/s1/a.mkv", out); + expect(Buffer.compare(await readFile(out), data)).toBe(0); + }); + + it("直链过期(401):重新 get 再续", async () => { + const { mock, client } = setup(); + mock.put("/s1/a.mkv", bytes(1000)); + const realFetch = mock.fetch; + let expired = false; + const f: typeof fetch = (async (input: Request | URL | string, init?: RequestInit) => { + const u = String(input instanceof Request ? input.url : input); + if (u.includes("/d/") && !expired) { + expired = true; + mock.expireSigns(); + } + return realFetch(input, init); + }) as typeof fetch; + const c2 = new AlistClient({ baseUrl: mock.base, auth: { kind: "token", token: "perm-token" }, fetch: f, qps: 0 }); + await c2.download("/s1/a.mkv", join(dir, "a.mkv")); + expect((await stat(join(dir, "a.mkv"))).size).toBe(1000); + void client; + }); + + it("本地目标已存在:冲突", async () => { + const { mock, client } = setup(); + mock.put("/s1/a.mkv", "x"); + await writeFile(join(dir, "a.mkv"), "local"); + expect((await caught(client.download("/s1/a.mkv", join(dir, "a.mkv")))).kind).toBe("conflict"); + }); + + it("第三方直链不带令牌", async () => { + const mock = createMockAlist({ token: "t" }); + mock.put("/s1/a.mkv", "x"); + const calls: { url: string; auth: string | null }[] = []; + const f: typeof fetch = (async (input: Request | URL | string, init?: RequestInit) => { + const req = input instanceof Request ? input : new Request(String(input), init); + calls.push({ url: req.url, auth: req.headers.get("authorization") }); + if (req.url.startsWith("https://cdn.example")) return new Response("x"); + const res = await mock.fetch(req); + if (new URL(req.url).pathname === "/api/fs/get") { + const j = await res.json(); + j.data.raw_url = "https://cdn.example/file?sign=zzz"; + return Response.json(j); + } + return res; + }) as typeof fetch; + const c = new AlistClient({ baseUrl: mock.base, auth: { kind: "token", token: "t" }, fetch: f, qps: 0 }); + await c.download("/s1/a.mkv", join(dir, "a.mkv")); + expect(calls.find((x) => x.url.startsWith("https://cdn.example"))!.auth).toBeNull(); + }); +}); + +describe("限速与脱敏", () => { + it("令牌桶:QPS 限制按时钟放行", async () => { + let now = 0; + const waits: number[] = []; + const l = new AlistLimiter( + 2, + 0, + () => now, + async (ms) => { + waits.push(ms); + now += ms; + }, + ); + for (let i = 0; i < 5; i++) await l.run(async () => i); + expect(waits.length).toBe(3); + expect(now).toBeGreaterThanOrEqual(1500); + }); + + it("并发上限", async () => { + const l = new AlistLimiter(0, 2); + let max = 0; + await Promise.all( + Array.from({ length: 6 }, () => + l.run(async () => { + max = Math.max(max, l.inFlight); + await new Promise((r) => setTimeout(r, 5)); + }), + ), + ); + expect(max).toBe(2); + }); + + it("直链 sign、令牌、路径密码不进日志", () => { + expect(redactAlistUrl("http://u:p@h/d/a.mkv?sign=SECRET123:0")).not.toContain("SECRET123"); + const t = redactAlistText("GET http://h/d/a.mkv?sign=abcdef failed token=perm-token-xyz", ["perm-token-xyz"]); + expect(t).not.toContain("abcdef"); + expect(t).not.toContain("perm-token-xyz"); + }); + + it("日志记录里没有令牌", async () => { + const lines: string[] = []; + const logger = { warn: (o: Record, m: string) => void lines.push(JSON.stringify(o) + m) }; + const { mock, client } = setup({ username: "u", password: "secretpw" }, { logger }); + await client.me(); + mock.invalidateSessions(); + await client.me(); + expect(lines.length).toBeGreaterThan(0); + expect(lines.join("\n")).not.toMatch(/secretpw|sess-/); + }); +}); diff --git a/packages/adapters/src/alist/client.ts b/packages/adapters/src/alist/client.ts new file mode 100644 index 00000000..74eac551 --- /dev/null +++ b/packages/adapters/src/alist/client.ts @@ -0,0 +1,951 @@ +// Alist / OpenList v3 接口客户端(docs/design/storage-alist.md)。对照 MP v3 `storages/alist.py`,只借行为。 +// 与 MP 的差异(§9):错误与「不存在」分开、401 自动重登、异步任务轮询到完成、同名冲突检查 + 暂存目录改名、 +// 上传先传临时名、下载断点续传 + 直链过期重取、refresh 节制 + 每实例限速。 +// 网盘方向的成功判定与 MP 一致:接口成功 + 目标存在,不比对大小 / 哈希(用户决定)。 +import { createHash, randomUUID } from "node:crypto"; +import { createReadStream } from "node:fs"; +import { link, open, rm, stat, unlink } from "node:fs/promises"; +import { posix } from "node:path"; +import { AlistError, classifyAlist, isAlistNotFound } from "./errors.ts"; +import { AlistLimiter } from "./limiter.ts"; + +export type AlistAuth = + | { kind: "token"; token: string } + | { kind: "password"; username: string; password: string } + | { kind: "guest" }; + +export interface AlistLogger { + debug?(obj: Record, msg: string): void; + warn?(obj: Record, msg: string): void; +} + +export interface AlistClientOptions { + baseUrl: string; + auth: AlistAuth; + /** 路径密码(Alist「元信息」里给目录设的密码),list / get 时带上 */ + pathPassword?: string | null; + fetch?: typeof fetch; + /** 普通请求超时(ms),缺省 30 s */ + timeoutMs?: number; + /** 数据流无进度空闲超时(ms),缺省 120 s */ + idleTimeoutMs?: number; + /** 可重试错误的重试次数,缺省 3 */ + retries?: number; + /** 每秒请求数上限,缺省 5;0 不限 */ + qps?: number; + /** 并发上限,缺省 4;0 不限 */ + concurrency?: number; + /** 异步任务等待上限(ms),缺省 6 h */ + taskTimeoutMs?: number; + /** 上传用 As-Task(缺省 false,同 MP) */ + uploadAsTask?: boolean; + /** 退避起点(ms),缺省 500 */ + backoffMs?: number; + sleep?: (ms: number) => Promise; + now?: () => number; + logger?: AlistLogger; +} + +export interface AlistObj { + /** Alist 内绝对路径(无尾斜杠) */ + path: string; + name: string; + isDir: boolean; + size: number; + /** 修改时间(ms) */ + modified: number; + created: number; + sign: string; + thumb: string; + hashInfo: Record | null; + rawUrl?: string; + provider?: string; +} + +export interface AlistHashes { + md5: string; + sha1: string; + sha256: string; +} + +export interface AlistTask { + id: string; + name: string; + state: number; + status: string; + progress: number; + error: string; + totalBytes: number; +} + +export interface AlistUsage { + total: number; + free: number; + used: number; +} + +export type ProgressCallback = (done: number, total: number) => void; +export type TaskKind = "copy" | "move" | "upload"; + +/** 任务状态:0 待执行 1 执行中 2 成功 3 取消中 4 已取消 5 出错 6 失败中 7 失败 8 等待重试 9 重试前 */ +export const TASK_SUCCEEDED = 2; +const TASK_CANCELED = new Set([3, 4]); +const TASK_FAILED = new Set([5, 7]); + +/** MP:per_page<=0 会退回后端默认 200,显式指定最大页大小 */ +export const ALIST_PAGE_SIZE = 500; +/** MP:账号密码令牌缓存 2 天、提前 5 分钟更新 */ +export const ALIST_TOKEN_TTL_MS = 2 * 86_400_000 - 5 * 60_000; +export const PARTIAL_SUFFIX = ".uhd-partial"; +const STAGE_PREFIX = ".umt-stage-"; + +/** 规范化 Alist 路径:以 / 开头、无尾斜杠、无空段 */ +export function normAlistPath(p: string): string { + const segs = p.split("/").filter((s) => s.length > 0 && s !== "."); + if (segs.some((s) => s === "..")) throw new AlistError("error", `路径不能含 ..: ${p}`); + return `/${segs.join("/")}`; +} + +/** 按段 URL 编码(保留 /),File-Path 头与 /d 下载链接用 */ +export const encodeAlistPath = (p: string) => normAlistPath(p).split("/").map(encodeURIComponent).join("/"); + +const parseTime = (v: unknown): number => { + const t = typeof v === "string" ? Date.parse(v) : Number.NaN; + return Number.isFinite(t) ? t : 0; +}; + +function parseHashInfo(v: unknown): Record | null { + if (!v || typeof v !== "object") return null; + const out: Record = {}; + for (const [k, val] of Object.entries(v as Record)) if (typeof val === "string" && val) out[k] = val; + return Object.keys(out).length ? out : null; +} + +function toObj(path: string, d: Record): AlistObj { + return { + path: normAlistPath(path), + name: String(d.name ?? posix.basename(path)), + isDir: d.is_dir === true, + size: Number(d.size ?? 0) || 0, + modified: parseTime(d.modified), + created: parseTime(d.created), + sign: String(d.sign ?? ""), + thumb: String(d.thumb ?? ""), + hashInfo: parseHashInfo(d.hash_info), + ...(typeof d.raw_url === "string" && d.raw_url ? { rawUrl: d.raw_url } : {}), + ...(typeof d.provider === "string" ? { provider: d.provider } : {}), + }; +} + +function toTask(d: Record): AlistTask { + return { + id: String(d.id ?? ""), + name: String(d.name ?? ""), + state: Number(d.state ?? 0), + status: String(d.status ?? ""), + progress: Number(d.progress ?? 0), + error: String(d.error ?? ""), + totalBytes: Number(d.total_bytes ?? 0), + }; +} + +/** 一遍读出 md5 / sha1 / sha256(MP 秒传头;§9-5) */ +export async function hashLocalFile( + path: string, + opts: { signal?: AbortSignal; onProgress?: ProgressCallback } = {}, +): Promise { + const md5 = createHash("md5"); + const sha1 = createHash("sha1"); + const sha256 = createHash("sha256"); + const total = (await stat(path)).size; + let done = 0; + for await (const chunk of createReadStream(path, { highWaterMark: 1024 * 1024 })) { + opts.signal?.throwIfAborted(); + const b = chunk as Uint8Array; + md5.update(b); + sha1.update(b); + sha256.update(b); + done += b.length; + opts.onProgress?.(done, total); + } + return { md5: md5.digest("hex"), sha1: sha1.digest("hex"), sha256: sha256.digest("hex") }; +} + +const abortedError = () => new AlistError("aborted", "操作已取消"); + +/** 空闲计时器:每次前进 touch(),超过 ms 没前进就 abort */ +function idleWatch(ms: number, outer?: AbortSignal) { + const ctl = new AbortController(); + let timedOut = false; + let timer: ReturnType | undefined; + const arm = () => { + clearTimeout(timer); + timer = setTimeout(() => { + timedOut = true; + ctl.abort(new AlistError("timeout", `数据流 ${Math.round(ms / 1000)} 秒没有进展`)); + }, ms); + }; + const onOuter = () => ctl.abort(abortedError()); + outer?.addEventListener("abort", onOuter, { once: true }); + arm(); + return { + signal: ctl.signal, + touch: arm, + get timedOut() { + return timedOut; + }, + done() { + clearTimeout(timer); + outer?.removeEventListener("abort", onOuter); + }, + }; +} + +export class AlistClient { + readonly baseUrl: string; + private readonly fetchFn: typeof fetch; + private readonly limiter: AlistLimiter; + private readonly o: Required< + Pick< + AlistClientOptions, + "timeoutMs" | "idleTimeoutMs" | "retries" | "taskTimeoutMs" | "uploadAsTask" | "backoffMs" | "sleep" | "now" + > + >; + private tokenCache: { token: string; exp: number } | null = null; + private loginInflight: Promise | null = null; + /** 观测用:每个 API 请求(路径 + 是否 refresh) */ + readonly stats = { requests: 0, refreshes: 0, relogins: 0, retries: 0 }; + + constructor(private readonly opts: AlistClientOptions) { + this.baseUrl = opts.baseUrl.trim().replace(/\/+$/, ""); + if (!/^https?:\/\//i.test(this.baseUrl)) throw new AlistError("error", "Alist 地址必须以 http:// 或 https:// 开头"); + this.fetchFn = opts.fetch ?? fetch; + const sleep = opts.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms))); + const now = opts.now ?? (() => Date.now()); + this.o = { + timeoutMs: opts.timeoutMs ?? 30_000, + idleTimeoutMs: opts.idleTimeoutMs ?? 120_000, + retries: opts.retries ?? 3, + taskTimeoutMs: opts.taskTimeoutMs ?? 6 * 3600_000, + uploadAsTask: opts.uploadAsTask ?? false, + backoffMs: opts.backoffMs ?? 500, + sleep, + now, + }; + this.limiter = new AlistLimiter(opts.qps ?? 5, opts.concurrency ?? 4, now, sleep); + } + + // ── 令牌(§2.2、§9-2)── + + private async login(): Promise { + const a = this.opts.auth; + if (a.kind !== "password") throw new AlistError("auth", "没有账号密码，无法登录"); + const data = await this.request<{ token?: string }>("POST", "/api/auth/login", { + body: { username: a.username, password: a.password }, + auth: false, + }); + if (!data?.token) throw new AlistError("auth", "登录成功但没有返回令牌"); + this.tokenCache = { token: data.token, exp: this.o.now() + ALIST_TOKEN_TTL_MS }; + this.opts.logger?.debug?.({ base: this.baseUrl }, "Alist 登录换取令牌成功"); + return data.token; + } + + private async token(): Promise { + const a = this.opts.auth; + if (a.kind === "token") return a.token; + if (a.kind === "guest") return null; + if (this.tokenCache && this.tokenCache.exp > this.o.now()) return this.tokenCache.token; + this.loginInflight ??= this.login().finally(() => { + this.loginInflight = null; + }); + return this.loginInflight; + } + + /** 配置变更时调用方重建客户端;这里只供测试 / 运维清缓存 */ + invalidateToken(): void { + this.tokenCache = null; + } + + // ── 请求 ── + + private backoff(attempt: number, err: AlistError): number { + const base = err.kind === "rate_limited" ? Math.max(this.o.backoffMs, 5000) : this.o.backoffMs; + return Math.min(base * 2 ** attempt, 60_000); + } + + private async request( + method: "GET" | "POST", + apiPath: string, + opts: { body?: unknown; auth?: boolean; signal?: AbortSignal; path?: string } = {}, + ): Promise { + const headers: Record = { "Content-Type": "application/json" }; + if (opts.auth !== false) { + const t = await this.token(); + if (t) headers.Authorization = t; + } + const signal = opts.signal + ? AbortSignal.any([opts.signal, AbortSignal.timeout(this.o.timeoutMs)]) + : AbortSignal.timeout(this.o.timeoutMs); + let res: Response; + this.stats.requests++; + try { + res = await this.fetchFn(`${this.baseUrl}${apiPath}`, { + method, + headers, + ...(opts.body !== undefined ? { body: JSON.stringify(opts.body) } : {}), + signal, + }); + } catch (e) { + if (opts.signal?.aborted) throw abortedError(); + if ((e as Error)?.name === "TimeoutError") + throw new AlistError("timeout", `请求超时：${apiPath}`, opts.path ? { path: opts.path } : {}); + throw new AlistError( + "unavailable", + `连不上 Alist：${(e as Error)?.message ?? e}`, + opts.path ? { path: opts.path } : {}, + ); + } + const text = await res.text().catch(() => ""); + let json: { code?: number; message?: string; data?: unknown } | null = null; + try { + json = text ? JSON.parse(text) : null; + } catch { + json = null; + } + if (!res.ok || !json || typeof json.code !== "number") { + throw classifyAlist(res.status, json?.code ?? null, json?.message ?? text.slice(0, 200), opts.path); + } + if (json.code !== 200) throw classifyAlist(res.status, json.code, json.message ?? "", opts.path); + return json.data as T; + } + + /** 带限速、401 重登一次、可重试错误退避的 API 调用 */ + private async api( + apiPath: string, + body: unknown, + opts: { retry?: boolean; signal?: AbortSignal; path?: string; method?: "GET" | "POST" } = {}, + ): Promise { + let attempt = 0; + let relogged = false; + for (;;) { + try { + return await this.limiter.run(() => + this.request(opts.method ?? "POST", apiPath, { + body, + ...(opts.signal ? { signal: opts.signal } : {}), + ...(opts.path ? { path: opts.path } : {}), + }), + ); + } catch (e) { + const err = e instanceof AlistError ? e : new AlistError("error", String(e)); + if (err.kind === "auth" && this.opts.auth.kind === "password" && !relogged) { + relogged = true; + this.stats.relogins++; + this.tokenCache = null; + this.opts.logger?.warn?.({ base: this.baseUrl, api: apiPath }, "Alist 令牌失效，重新登录"); + continue; + } + if (opts.retry !== false && err.retryable && attempt < this.o.retries && !opts.signal?.aborted) { + this.stats.retries++; + await this.o.sleep(this.backoff(attempt, err)); + attempt++; + continue; + } + throw err; + } + } + } + + // ── 基本信息 ── + + async me(): Promise<{ username: string; basePath: string; role: number; permission: number }> { + const d = await this.api>("/api/me", undefined, { method: "GET" }); + return { + username: String(d.username ?? ""), + basePath: String(d.base_path ?? "/"), + role: Number(d.role ?? 0), + permission: Number(d.permission ?? 0), + }; + } + + async version(): Promise { + try { + const d = await this.limiter.run(() => + this.request>("GET", "/api/public/settings", { auth: false }), + ); + return typeof d?.version === "string" ? d.version : null; + } catch (e) { + if (e instanceof AlistError && (e.kind === "unavailable" || e.kind === "timeout")) throw e; + return null; + } + } + + // ── 读 ── + + /** 列目录(自动翻页;refresh 只在用户刷新 / 写后确认时传,§9-8) */ + async list(path: string, opts: { refresh?: boolean; signal?: AbortSignal } = {}): Promise { + const dir = normAlistPath(path); + const out: AlistObj[] = []; + if (opts.refresh) this.stats.refreshes++; + for (let page = 1; ; page++) { + const d = await this.api>( + "/api/fs/list", + { + path: dir, + password: this.opts.pathPassword ?? "", + page, + per_page: ALIST_PAGE_SIZE, + refresh: page === 1 && opts.refresh === true, + }, + { path: dir, ...(opts.signal ? { signal: opts.signal } : {}) }, + ); + const content = (d?.content as Record[] | null) ?? []; + for (const it of content) out.push(toObj(posix.join(dir, String(it.name ?? "")), it)); + const total = Number(d?.filtered_total ?? d?.total ?? 0) || 0; + const pagesTotal = Number(d?.pages_total ?? 0) || 0; + if (content.length === 0 || out.length >= total) return out; + if (d?.has_more === false || (pagesTotal && page >= pagesTotal)) return out; + } + } + + /** 取单个对象;不存在抛 AlistError(not_found),其它失败抛对应类型(§9-1) */ + async get(path: string, opts: { refresh?: boolean; signal?: AbortSignal } = {}): Promise { + const p = normAlistPath(path); + if (opts.refresh) this.stats.refreshes++; + const d = await this.api>( + "/api/fs/get", + { path: p, password: this.opts.pathPassword ?? "", page: 1, per_page: 0, refresh: opts.refresh === true }, + { path: p, ...(opts.signal ? { signal: opts.signal } : {}) }, + ); + return toObj(p, d ?? {}); + } + + /** 不存在 → null;其它失败照抛 */ + async stat(path: string, opts: { refresh?: boolean; signal?: AbortSignal } = {}): Promise { + try { + return await this.get(path, opts); + } catch (e) { + if (isAlistNotFound(e)) return null; + throw e; + } + } + + /** 下载直链:raw_url 优先,否则 /d?sign=(同 MP) */ + async rawUrl(path: string, obj?: AlistObj): Promise { + const o = obj ?? (await this.get(path)); + if (o.isDir) throw new AlistError("error", `是目录，不能下载：${o.path}`); + if (o.rawUrl) return o.rawUrl; + return `${this.baseUrl}/d${encodeAlistPath(o.path)}${o.sign ? `?sign=${encodeURIComponent(o.sign)}` : ""}`; + } + + /** 直链与 Alist 同源时才带令牌(第三方 CDN 直链不能带令牌,MP 会带) */ + private async downloadHeaders(url: string, extra: Record = {}): Promise> { + const h = { ...extra }; + if (new URL(url).origin === new URL(this.baseUrl).origin) { + const t = await this.token(); + if (t) h.Authorization = t; + } + return h; + } + + /** 打开下载流(文件管理下载代理用;支持 Range 透传) */ + async openRead(path: string, opts: { range?: string; signal?: AbortSignal } = {}): Promise { + const obj = await this.get(path); + for (let i = 0; i < 3; i++) { + const url = await this.rawUrl(path, i === 0 ? obj : undefined); + const res = await this.limiter.run( + async () => + this.fetchFn(url, { + headers: await this.downloadHeaders(url, opts.range ? { Range: opts.range } : {}), + ...(opts.signal ? { signal: opts.signal } : {}), + }), + { data: true }, + ); + if ([401, 403, 410].includes(res.status)) { + await res.body?.cancel(); + continue; + } + if (!res.ok) { + await res.body?.cancel(); + throw classifyAlist(res.status, null, `下载失败 HTTP ${res.status}`, obj.path); + } + return res; + } + throw new AlistError("auth", "下载直链多次被拒绝(签名过期或无权限)", { path: obj.path }); + } + + /** 读小文件到内存(NFO 比对等);超过 maxBytes 抛错 */ + async readFile(path: string, maxBytes = 4 * 1024 * 1024): Promise { + const res = await this.openRead(path); + const buf = new Uint8Array(await res.arrayBuffer()); + if (buf.length > maxBytes) throw new AlistError("error", `文件过大(${buf.length} 字节)`, { path }); + return buf; + } + + /** 容量:/api/admin/storage/list 的 mount_details;拿不到返回 null,不报错(§9-10) */ + async usage(path = "/"): Promise { + try { + const d = await this.api<{ content?: Record[] }>("/api/admin/storage/list", undefined, { + method: "GET", + retry: false, + }); + const p = normAlistPath(path); + const hit = (d?.content ?? []) + .filter((s) => { + const m = normAlistPath(String(s.mount_path ?? "/")); + return m === "/" || p === m || p.startsWith(`${m}/`); + }) + .sort((a, b) => String(b.mount_path).length - String(a.mount_path).length)[0]; + const md = hit?.mount_details as Record | undefined; + const total = Number(md?.total_space ?? 0); + if (!md || !(total > 0)) return null; + const free = Number(md.free_space ?? 0); + return { total, free, used: Number(md.used_space ?? total - free) }; + } catch { + return null; + } + } + + // ── 写 ── + + /** 建目录(Alist 递归建,已存在也成功) */ + async mkdir(path: string): Promise { + const p = normAlistPath(path); + if (p === "/") return; + await this.api("/api/fs/mkdir", { path: p }, { path: p }); + } + + /** 改名(只改最后一段);目标已存在抛 conflict */ + async rename(path: string, newName: string): Promise { + if (!newName || newName.includes("/")) throw new AlistError("error", `非法文件名：${newName}`); + const p = normAlistPath(path); + await this.api("/api/fs/rename", { path: p, name: newName }, { path: p, retry: false }); + } + + /** 删除(注意 Alist 删除不存在的名字也返回成功) */ + async remove(dir: string, names: string[]): Promise { + if (!names.length) return; + const d = normAlistPath(dir); + await this.api("/api/fs/remove", { dir: d, names }, { path: d, retry: false }); + } + + async removePath(path: string): Promise { + const p = normAlistPath(path); + if (p === "/") throw new AlistError("error", "不能删除根目录"); + await this.remove(posix.dirname(p), [posix.basename(p)]); + } + + /** 原始 copy / move:返回任务 id(同步完成时为空) */ + async rawTransfer(kind: "copy" | "move", srcDir: string, dstDir: string, names: string[]): Promise { + const d = await this.api<{ tasks?: Record[] } | null>( + `/api/fs/${kind}`, + { src_dir: normAlistPath(srcDir), dst_dir: normAlistPath(dstDir), names }, + { path: srcDir, retry: false }, + ); + return (d?.tasks ?? []).map((t) => String(t.id ?? "")).filter(Boolean); + } + + /** 查任务;接口不支持(老版本)返回 null,任务已不在返回 undefined */ + async taskInfo(kind: TaskKind, id: string): Promise { + for (const prefix of ["/api/task", "/api/admin/task"]) { + try { + const d = await this.api>( + `${prefix}/${kind}/info?tid=${encodeURIComponent(id)}`, + undefined, + {}, + ); + return toTask(d ?? {}); + } catch (e) { + if (!(e instanceof AlistError)) throw e; + // HTTP 404(路由不存在)→ 试下一个前缀;JSON code 404 task not found → 任务不在了 + if (e.detail.status === 404 && e.detail.code === undefined) continue; + if (e.kind === "not_found") return undefined; + throw e; + } + } + return null; + } + + /** + * 等异步任务完成(§9-3):指数退避轮询,成功返回 "succeeded";接口不支持 / 任务已不在返回 "unknown" + * (调用方改为等目标出现);取消 / 失败抛 task_failed(带任务 error);超时抛 timeout。 + */ + async waitTask( + kind: TaskKind, + id: string, + opts: { timeoutMs?: number; signal?: AbortSignal; onProgress?: (t: AlistTask) => void } = {}, + ): Promise<"succeeded" | "unknown"> { + const deadline = this.o.now() + (opts.timeoutMs ?? this.o.taskTimeoutMs); + let delay = 500; + for (;;) { + opts.signal?.throwIfAborted(); + const t = await this.taskInfo(kind, id); + if (t === null || t === undefined) return "unknown"; + opts.onProgress?.(t); + if (t.state === TASK_SUCCEEDED) return "succeeded"; + if (TASK_CANCELED.has(t.state)) throw new AlistError("task_failed", `Alist ${kind} 任务已取消：${t.name}`); + if (TASK_FAILED.has(t.state)) + throw new AlistError("task_failed", `Alist ${kind} 任务失败：${t.error || t.status || t.name}`); + if (this.o.now() >= deadline) throw new AlistError("timeout", `等待 Alist ${kind} 任务超时：${t.name}`); + await this.o.sleep(delay); + delay = Math.min(delay * 2, 10_000); + } + } + + /** 写后等目标出现(refresh,退避);超时抛 not_found */ + async waitExists(path: string, opts: { timeoutMs?: number; signal?: AbortSignal } = {}): Promise { + const deadline = this.o.now() + (opts.timeoutMs ?? 60_000); + let delay = 500; + for (;;) { + opts.signal?.throwIfAborted(); + const o = await this.stat(path, { refresh: true }); + if (o) return o; + if (this.o.now() >= deadline) throw new AlistError("not_found", `写入后目标仍不存在：${normAlistPath(path)}`); + await this.o.sleep(delay); + delay = Math.min(delay * 2, 10_000); + } + } + + private async finishTransfer(kind: "copy" | "move", ids: string[], target: string, signal?: AbortSignal) { + let unknown = ids.length === 0; + for (const id of ids) { + if ((await this.waitTask(kind, id, signal ? { signal } : {})) === "unknown") unknown = true; + } + // 成功判定(同 MP、用户决定):任务成功 + 目标存在;没有任务信息时等目标出现 + return this.waitExists(target, { + timeoutMs: unknown ? this.o.taskTimeoutMs : 60_000, + ...(signal ? { signal } : {}), + }); + } + + /** + * 同实例复制一个文件或目录到 dst(完整目标路径)。目标已存在抛 conflict(§9-4); + * 名字不同或目标目录里已有同名源名时走暂存目录 `.umt-stage-*`:复制 → 改名 → 移入,失败清理暂存。 + */ + async copyFile(src: string, dst: string, opts: { signal?: AbortSignal } = {}): Promise { + return this.transferFile("copy", src, dst, opts); + } + + /** 同实例移动(语义同 copyFile;同目录只改名) */ + async moveFile(src: string, dst: string, opts: { signal?: AbortSignal } = {}): Promise { + return this.transferFile("move", src, dst, opts); + } + + private async transferFile( + kind: "copy" | "move", + srcIn: string, + dstIn: string, + opts: { signal?: AbortSignal }, + ): Promise { + const src = normAlistPath(srcIn); + const dst = normAlistPath(dstIn); + if (src === dst) throw new AlistError("conflict", `源与目标相同：${src}`); + if (dst.startsWith(`${src}/`)) throw new AlistError("error", "不能把目录复制 / 移动到它自己里面"); + await this.get(src); // 源不存在 → not_found + if (await this.stat(dst)) throw new AlistError("conflict", `目标已存在：${dst}`, { path: dst }); + const srcDir = posix.dirname(src); + const srcName = posix.basename(src); + const dstDir = posix.dirname(dst); + const dstName = posix.basename(dst); + await this.mkdir(dstDir); + if (kind === "move" && srcDir === dstDir) { + await this.rename(src, dstName); + return this.waitExists(dst, opts.signal ? { signal: opts.signal } : {}); + } + if (srcName === dstName) { + const ids = await this.rawTransfer(kind, srcDir, dstDir, [srcName]); + return this.finishTransfer(kind, ids, dst, opts.signal); + } + // 暂存目录:不在目标目录里按源名落地,避免与已有同名文件冲突 / 覆盖 + const stageName = `${STAGE_PREFIX}${randomUUID().slice(0, 8)}`; + const stage = posix.join(dstDir, stageName); + let movedToStage = false; + try { + await this.mkdir(stage); + const ids = await this.rawTransfer(kind, srcDir, stage, [srcName]); + movedToStage = kind === "move"; + await this.finishTransfer(kind, ids, posix.join(stage, srcName), opts.signal); + await this.rename(posix.join(stage, srcName), dstName); + // 暂存目录与目标同目录 → 同驱动 move,立即完成 + const ids2 = await this.rawTransfer("move", stage, dstDir, [dstName]); + movedToStage = false; + const obj = await this.finishTransfer("move", ids2, dst, opts.signal); + await this.remove(dstDir, [stageName]).catch(() => undefined); + return obj; + } catch (e) { + if (movedToStage) { + // 移动失败:尽量把源搬回去 + for (const n of [srcName, dstName]) { + const p = posix.join(stage, n); + if (await this.stat(p, { refresh: true }).catch(() => null)) { + await this.rename(p, srcName).catch(() => undefined); + await this.rawTransfer("move", stage, srcDir, [srcName]).catch(() => undefined); + break; + } + } + } + await this.remove(dstDir, [stageName]).catch(() => undefined); + throw e; + } + } + + // ── 上传(§2.6)── + + /** + * 上传本地文件到 dst(完整路径)。目标已存在抛 conflict(Alist 的 put 会静默覆盖,不依赖它); + * 先传 `.uhd-partial` 再改名;成功判定 = 接口成功 + 目标存在(同 MP)。 + */ + async upload( + localPath: string, + dstIn: string, + opts: { + signal?: AbortSignal; + onProgress?: ProgressCallback; + onHashProgress?: ProgressCallback; + hashes?: AlistHashes; + asTask?: boolean; + } = {}, + ): Promise<{ obj: AlistObj; hashes: AlistHashes; bytes: number }> { + const dst = normAlistPath(dstIn); + const st = await stat(localPath); + if (!st.isFile()) throw new AlistError("error", `不是文件：${localPath}`); + const hashes = + opts.hashes ?? + (await hashLocalFile(localPath, { + ...(opts.signal ? { signal: opts.signal } : {}), + ...(opts.onHashProgress ? { onProgress: opts.onHashProgress } : {}), + })); + if (await this.stat(dst)) throw new AlistError("conflict", `目标已存在：${dst}`, { path: dst }); + const tmp = `${dst}${PARTIAL_SUFFIX}`; + await this.mkdir(posix.dirname(dst)); + if (await this.stat(tmp)) await this.removePath(tmp); + let attempt = 0; + for (;;) { + try { + await this.putOnce(localPath, tmp, st.size, st.mtimeMs, hashes, opts); + break; + } catch (e) { + const err = e instanceof AlistError ? e : new AlistError("unavailable", String((e as Error)?.message ?? e)); + await this.removePath(tmp).catch(() => undefined); + if (err.retryable && attempt < this.o.retries && !opts.signal?.aborted) { + this.stats.retries++; + this.opts.logger?.warn?.({ dst, attempt, kind: err.kind }, "Alist 上传失败，重试"); + await this.o.sleep(this.backoff(attempt, err)); + attempt++; + continue; + } + throw err; + } + } + try { + await this.waitExists(tmp, { timeoutMs: 60_000, ...(opts.signal ? { signal: opts.signal } : {}) }); + await this.rename(tmp, posix.basename(dst)); + const obj = await this.waitExists(dst, opts.signal ? { signal: opts.signal } : {}); + return { obj, hashes, bytes: st.size }; + } catch (e) { + await this.removePath(tmp).catch(() => undefined); + throw e; + } + } + + private async putOnce( + localPath: string, + target: string, + size: number, + mtimeMs: number, + hashes: AlistHashes, + opts: { signal?: AbortSignal; onProgress?: ProgressCallback; asTask?: boolean }, + ): Promise { + const asTask = opts.asTask ?? this.o.uploadAsTask; + const t = await this.token(); + const watch = idleWatch(this.o.idleTimeoutMs, opts.signal); + const fh = await open(localPath, "r"); + let sent = 0; + const body = new ReadableStream({ + async pull(ctl) { + const buf = new Uint8Array(1024 * 1024); + const { bytesRead } = await fh.read(buf, 0, buf.length, null); + if (bytesRead === 0) { + ctl.close(); + return; + } + sent += bytesRead; + watch.touch(); + opts.onProgress?.(sent, size); + ctl.enqueue(buf.subarray(0, bytesRead)); + }, + }); + const headers: Record = { + "Content-Type": "application/octet-stream", + "Content-Length": String(size), + "File-Path": encodeAlistPath(target), + "As-Task": String(asTask), + "Last-Modified": String(Math.floor(mtimeMs)), + "X-File-Md5": hashes.md5, + "X-File-Sha1": hashes.sha1, + "X-File-Sha256": hashes.sha256, + }; + if (t) headers.Authorization = t; + let data: { task?: Record } | null; + try { + data = await this.limiter.run( + async () => { + this.stats.requests++; + let res: Response; + try { + res = await this.fetchFn(`${this.baseUrl}/api/fs/put`, { + method: "PUT", + headers, + body, + signal: watch.signal, + duplex: "half", + } as RequestInit); + } catch (e) { + if (opts.signal?.aborted) throw abortedError(); + if (watch.timedOut) + throw new AlistError("timeout", `上传 ${Math.round(this.o.idleTimeoutMs / 1000)} 秒没有进展`); + throw new AlistError("unavailable", `上传中断：${(e as Error)?.message ?? e}`, { path: target }); + } + const text = await res.text().catch(() => ""); + let json: { code?: number; message?: string; data?: unknown } | null = null; + try { + json = text ? JSON.parse(text) : null; + } catch { + json = null; + } + if (!res.ok || !json || json.code !== 200) + throw classifyAlist(res.status, json?.code ?? null, json?.message ?? text.slice(0, 200), target); + return json.data as { task?: Record } | null; + }, + { data: true }, + ); + } finally { + watch.done(); + await fh.close().catch(() => undefined); + } + const taskId = data?.task?.id; + if (asTask && typeof taskId === "string" && taskId) { + await this.waitTask("upload", taskId, opts.signal ? { signal: opts.signal } : {}); + } + } + + // ── 下载(§2.7)── + + /** + * 下载到本地 localPath:写 `.uhd-partial`,支持 Range 续传、直链过期重取; + * 流正常结束即成功(同 MP,不比对大小 / 哈希),之后不覆盖地改名。目标已存在抛 conflict。 + */ + async download( + remote: string, + localPath: string, + opts: { signal?: AbortSignal; onProgress?: ProgressCallback } = {}, + ): Promise<{ bytes: number; sha256: string; obj: AlistObj }> { + if ( + await stat(localPath).then( + () => true, + () => false, + ) + ) + throw new AlistError("conflict", `本地目标已存在：${localPath}`); + let obj = await this.get(remote, opts.signal ? { signal: opts.signal } : {}); + if (obj.isDir) throw new AlistError("error", `是目录，不能下载：${obj.path}`); + const partial = `${localPath}${PARTIAL_SUFFIX}`; + let signFailures = 0; + let attempt = 0; + let url = await this.rawUrl(remote, obj); + for (;;) { + opts.signal?.throwIfAborted(); + const have = await stat(partial).then( + (s) => s.size, + () => 0, + ); + const watch = idleWatch(this.o.idleTimeoutMs, opts.signal); + try { + const res = await this.limiter.run( + async () => + this.fetchFn(url, { + headers: await this.downloadHeaders(url, have > 0 ? { Range: `bytes=${have}-` } : {}), + signal: watch.signal, + }), + { data: true }, + ); + if ([401, 403, 410].includes(res.status)) { + await res.body?.cancel(); + if (++signFailures > 3) + throw new AlistError("auth", `下载直链多次被拒绝(HTTP ${res.status})`, { path: obj.path }); + this.opts.logger?.warn?.({ path: obj.path, status: res.status }, "Alist 直链失效，重新获取"); + obj = await this.get(remote); + url = await this.rawUrl(remote, obj); + continue; + } + if (res.status === 416 && have > 0) { + // 已经下全了(上次在改名前中断) + await res.body?.cancel(); + break; + } + if (!res.ok || !res.body) { + await res.body?.cancel(); + throw classifyAlist(res.status, null, `下载失败 HTTP ${res.status}`, obj.path); + } + const append = have > 0 && res.status === 206; + const total = append + ? have + Number(res.headers.get("content-length") ?? 0) + : Number(res.headers.get("content-length") ?? obj.size) || obj.size; + const fh = await open(partial, append ? "a" : "w"); + let done = append ? have : 0; + try { + const reader = res.body.getReader(); + for (;;) { + const { value, done: end } = await reader.read(); + if (end) break; + watch.touch(); + await fh.write(value); + done += value.length; + opts.onProgress?.(done, total); + } + } finally { + await fh.close(); + } + break; + } catch (e) { + if (opts.signal?.aborted) throw abortedError(); + const err = + e instanceof AlistError + ? e + : watch.timedOut + ? new AlistError("timeout", `下载 ${Math.round(this.o.idleTimeoutMs / 1000)} 秒没有进展`, { + path: obj.path, + }) + : new AlistError("unavailable", `下载中断：${(e as Error)?.message ?? e}`, { path: obj.path }); + if (err.retryable && attempt < this.o.retries) { + this.stats.retries++; + await this.o.sleep(this.backoff(attempt, err)); + attempt++; + continue; + } + throw err; + } finally { + watch.done(); + } + } + const sha = createHash("sha256"); + let bytes = 0; + for await (const chunk of createReadStream(partial, { highWaterMark: 1024 * 1024 })) { + sha.update(chunk as Uint8Array); + bytes += (chunk as Uint8Array).length; + } + try { + await link(partial, localPath); + } catch (e) { + if ((e as { code?: string })?.code === "EEXIST") throw new AlistError("conflict", `本地目标已存在：${localPath}`); + throw e; + } + await unlink(partial).catch(() => undefined); + return { bytes, sha256: sha.digest("hex"), obj }; + } + + /** 清掉本地 partial(调用方放弃续传时) */ + static async dropPartial(localPath: string): Promise { + await rm(`${localPath}${PARTIAL_SUFFIX}`, { force: true }); + } +} diff --git a/packages/adapters/src/alist/errors.ts b/packages/adapters/src/alist/errors.ts new file mode 100644 index 00000000..268a3678 --- /dev/null +++ b/packages/adapters/src/alist/errors.ts @@ -0,0 +1,70 @@ +// Alist / OpenList 错误分类(docs/design/storage-alist.md §2.1、§9-1): +// 「不存在」与连接失败 / 鉴权失败 / 服务端错误分开,调用方不能把后者当成不存在。 + +export type AlistErrorKind = + | "not_found" + | "conflict" + | "auth" + | "rate_limited" + | "unavailable" + | "task_failed" + | "timeout" + | "aborted" + | "error"; + +export class AlistError extends Error { + override name = "AlistError"; + constructor( + readonly kind: AlistErrorKind, + message: string, + readonly detail: { status?: number; code?: number; path?: string } = {}, + ) { + super(message); + } + /** 可重试:连不上、超时、限流 / 风控、5xx */ + get retryable(): boolean { + return this.kind === "unavailable" || this.kind === "rate_limited" || this.kind === "timeout"; + } + /** 对应本地文件系统的错误码(vfs 用;与 FsErrorCode 同名) */ + get fsCode(): string { + switch (this.kind) { + case "not_found": + return "ENOENT"; + case "conflict": + return "EEXIST"; + case "auth": + return "EACCES"; + case "timeout": + case "rate_limited": + return "TIMEOUT"; + case "aborted": + return "ABORTED"; + default: + return "IO"; + } + } +} + +export const isAlistNotFound = (e: unknown): boolean => e instanceof AlistError && e.kind === "not_found"; +export const isAlistConflict = (e: unknown): boolean => e instanceof AlistError && e.kind === "conflict"; + +const NOT_FOUND_RE = /object not found|not exist|no such file|not found/i; +const EXISTS_RE = /\bexists?\b|already exist/i; +const AUTH_RE = /token is (invalidated|expired)|invalid token|unauthorized|password is incorrect|incorrect password/i; +const RATE_RE = /too many|rate limit|频繁|风控|429/i; + +/** HTTP 状态 + 业务 code + message → 错误类型 */ +export function classifyAlist(status: number, code: number | null, message: string, path?: string): AlistError { + const detail = { status, ...(code !== null ? { code } : {}), ...(path ? { path } : {}) }; + const msg = message || `HTTP ${status}`; + if (status === 429 || code === 429 || RATE_RE.test(msg)) return new AlistError("rate_limited", msg, detail); + if (status === 401 || code === 401 || AUTH_RE.test(msg)) return new AlistError("auth", msg, detail); + if (code === 404 && /task/i.test(msg)) return new AlistError("not_found", msg, detail); + if (NOT_FOUND_RE.test(msg)) return new AlistError("not_found", msg, detail); + if (code === 403 && EXISTS_RE.test(msg)) return new AlistError("conflict", msg, detail); + if (status === 403 || code === 403) return new AlistError("auth", msg, detail); + if (status >= 500 && code === null) return new AlistError("unavailable", msg, detail); + if (status >= 500 || (code !== null && code >= 500 && /timeout|connect|EOF|reset|unavailable|bad gateway/i.test(msg))) + return new AlistError("unavailable", msg, detail); + return new AlistError("error", msg, detail); +} diff --git a/packages/adapters/src/alist/index.ts b/packages/adapters/src/alist/index.ts new file mode 100644 index 00000000..2eb7fe89 --- /dev/null +++ b/packages/adapters/src/alist/index.ts @@ -0,0 +1,4 @@ +export * from "./client.ts"; +export * from "./errors.ts"; +export * from "./limiter.ts"; +export * from "./redact.ts"; diff --git a/packages/adapters/src/alist/limiter.ts b/packages/adapters/src/alist/limiter.ts new file mode 100644 index 00000000..2a951962 --- /dev/null +++ b/packages/adapters/src/alist/limiter.ts @@ -0,0 +1,61 @@ +// 每个 Alist 实例一个限速器:令牌桶(QPS)+ 并发上限(docs/design/storage-alist.md §2.3、§9-8)。 +// 数据流(上传 / 下载)占并发但不占 QPS。 + +export class AlistLimiter { + private tokens: number; + private last: number; + private active = 0; + private readonly waiters: (() => void)[] = []; + + constructor( + private readonly qps: number, + private readonly concurrency: number, + private readonly now: () => number = () => Date.now(), + private readonly sleep: (ms: number) => Promise = (ms) => new Promise((r) => setTimeout(r, ms)), + ) { + this.tokens = Math.max(1, qps); + this.last = now(); + } + + private async takeToken(): Promise { + if (this.qps <= 0) return; + for (;;) { + const t = this.now(); + this.tokens = Math.min(Math.max(1, this.qps), this.tokens + ((t - this.last) / 1000) * this.qps); + this.last = t; + if (this.tokens >= 1) { + this.tokens -= 1; + return; + } + await this.sleep(Math.ceil(((1 - this.tokens) / this.qps) * 1000)); + } + } + + private async acquire(): Promise { + if (this.concurrency <= 0 || this.active < this.concurrency) { + this.active++; + return; + } + await new Promise((r) => this.waiters.push(r)); + this.active++; + } + + private release(): void { + this.active--; + this.waiters.shift()?.(); + } + + async run(fn: () => Promise, opts: { data?: boolean } = {}): Promise { + if (!opts.data) await this.takeToken(); + await this.acquire(); + try { + return await fn(); + } finally { + this.release(); + } + } + + get inFlight(): number { + return this.active; + } +} diff --git a/packages/adapters/src/alist/mock.ts b/packages/adapters/src/alist/mock.ts new file mode 100644 index 00000000..40337843 --- /dev/null +++ b/packages/adapters/src/alist/mock.ts @@ -0,0 +1,491 @@ +// 测试用的进程内 Alist / OpenList v4 模拟(docs/design/storage-alist.md §12)。 +// 返回形状照真机(OpenList v4.2.6)核对过:HTTP 200 + {code,message,data};不存在 code 500 "object not found"; +// 令牌无效 code 401;copy 一律异步任务;同挂载 move 立即完成、跨挂载 move 是任务;put 覆盖同名;/d 支持 Range 与 sign。 +// 顶层目录视为「挂载」(驱动),用于区分同驱动 / 跨驱动。 +import { createHash, randomUUID } from "node:crypto"; + +export interface MockFile { + data: Uint8Array; + modified: number; + hashInfo?: Record | null; +} + +export interface MockAlistOptions { + /** 永久令牌 */ + token?: string; + /** 账号密码 */ + username?: string; + password?: string; + /** 目录密码(需要时 list / get 必须带) */ + pathPassword?: string; + /** 任务完成前需要被查询的次数(模拟异步) */ + taskPolls?: number; + /** 不支持任务接口(模拟老 Alist v3) */ + noTaskApi?: boolean; + /** copy / move 不返回任务 id(模拟老 Alist v3 同步返回 data:null,但实际延迟完成) */ + noTaskIds?: boolean; + /** 挂载容量(mount_details) */ + mountDetails?: Record; + version?: string; +} + +interface Task { + id: string; + kind: "copy" | "move" | "upload"; + name: string; + polls: number; + state: number; + error: string; + run: () => void; +} + +export interface MockAlist { + readonly base: string; + setOrigin(origin: string): void; + handler: (req: Request) => Promise; + fetch: typeof fetch; + files: Map; + dirs: Map; + tasks: Map; + requests: { path: string; body: Record | null; headers: Record }[]; + /** 已签发且有效的临时令牌 */ + sessions: Set; + /** 注入:下 n 个 API 请求返回 429 */ + rateLimitNext(n: number): void; + /** 注入:下 n 个 API 请求连接失败(fetch 抛 TypeError) */ + failNext(n: number): void; + /** 注入:上传读到 n 字节后断开 */ + breakUploadAfter(bytes: number | null): void; + /** 注入:下载读到 n 字节后断开(仅一次) */ + breakDownloadAfter(bytes: number | null): void; + /** 注入:使所有 sign 失效一次(下载返回 401) */ + expireSigns(): void; + /** 注入:下一个任务以失败结束 */ + failNextTask(message: string): void; + /** 注入:put 成功但文件不落地(模拟「接口成功目标不存在」) */ + dropNextUpload(): void; + invalidateSessions(): void; + put(path: string, data: Uint8Array | string, opts?: { modified?: number; hashInfo?: Record }): void; + mkdirp(path: string): void; + read(path: string): Uint8Array | null; + exists(path: string): boolean; + listNames(path: string): string[]; +} + +const norm = (p: string) => + `/${p + .split("/") + .filter((s) => s && s !== ".") + .join("/")}`; +const parent = (p: string) => norm(p.split("/").slice(0, -1).join("/")); +const base = (p: string) => p.split("/").filter(Boolean).pop() ?? ""; +const mountOf = (p: string) => p.split("/").filter(Boolean)[0] ?? ""; +const iso = (ms: number) => new Date(ms).toISOString(); + +export function createMockAlist(opts: MockAlistOptions = {}): MockAlist { + let origin = "http://alist.mock"; + const files = new Map(); + const dirs = new Map([["/", Date.now()]]); + const tasks = new Map(); + const sessions = new Set(); + const requests: MockAlist["requests"] = []; + let signEpoch = 1; + let rateLimit = 0; + let fail = 0; + let uploadBreak: number | null = null; + let downloadBreak: number | null = null; + let nextTaskError: string | null = null; + let dropUpload = false; + + const mkdirp = (p: string) => { + const segs = norm(p).split("/").filter(Boolean); + let cur = ""; + for (const s of segs) { + cur += `/${s}`; + if (files.has(cur)) throw new Error(`not a dir: ${cur}`); + if (!dirs.has(cur)) dirs.set(cur, Date.now()); + } + }; + const touch = (dir: string) => { + if (dirs.has(dir)) dirs.set(dir, Date.now()); + }; + const putFile = (p: string, data: Uint8Array, modified = Date.now(), hashInfo?: Record | null) => { + mkdirp(parent(p)); + files.set(norm(p), { data, modified, ...(hashInfo !== undefined ? { hashInfo } : {}) }); + touch(parent(p)); + }; + const children = (dir: string) => { + const d = norm(dir); + const pre = d === "/" ? "/" : `${d}/`; + const names = new Set(); + for (const k of [...files.keys(), ...dirs.keys()]) { + if (k !== d && k.startsWith(pre)) { + const rest = k.slice(pre.length); + if (rest && !rest.includes("/")) names.add(rest); + } + } + return [...names].sort(); + }; + const exists = (p: string) => files.has(norm(p)) || dirs.has(norm(p)); + const signOf = (p: string) => createHash("sha256").update(`${p}:${signEpoch}`).digest("hex").slice(0, 16); + const objOf = (p: string, withRaw = false) => { + const f = files.get(p); + const isDir = !f; + return { + name: base(p), + size: f ? f.data.length : 0, + is_dir: isDir, + modified: iso(f ? f.modified : (dirs.get(p) ?? 0)), + created: iso(f ? f.modified : (dirs.get(p) ?? 0)), + sign: isDir ? "" : signOf(p), + thumb: "", + type: isDir ? 1 : 0, + hashinfo: "null", + hash_info: f?.hashInfo ?? null, + ...(withRaw + ? { + raw_url: isDir ? "" : `${origin}/d${p.split("/").map(encodeURIComponent).join("/")}?sign=${signOf(p)}`, + provider: "Local", + readme: "", + header: "", + related: null, + } + : {}), + }; + }; + const ok = (data: unknown = null) => Response.json({ code: 200, message: "success", data }); + const err = (code: number, message: string) => Response.json({ code, message, data: null }); + const copyTree = (src: string, dst: string) => { + if (files.has(src)) { + const f = files.get(src)!; + putFile(dst, f.data.slice(), f.modified, f.hashInfo ?? null); + return; + } + mkdirp(dst); + for (const n of children(src)) copyTree(`${src}/${n}`, `${dst}/${n}`); + }; + const removeTree = (p: string) => { + for (const k of [...files.keys()]) if (k === p || k.startsWith(`${p}/`)) files.delete(k); + for (const k of [...dirs.keys()]) if (k === p || k.startsWith(`${p}/`)) dirs.delete(k); + touch(parent(p)); + }; + const newTask = (kind: Task["kind"], name: string, run: () => void): Task => { + const t: Task = { id: randomUUID().slice(0, 21), kind, name, polls: 0, state: 0, error: "", run }; + if (nextTaskError) { + t.error = nextTaskError; + nextTaskError = null; + t.run = () => { + throw new Error(t.error); + }; + } + tasks.set(t.id, t); + return t; + }; + const advanceTask = (t: Task) => { + if (t.state >= 2) return; + t.polls++; + t.state = 1; + if (t.polls > (opts.taskPolls ?? 1)) { + try { + t.run(); + t.state = 2; + } catch (e) { + t.state = 7; + t.error = (e as Error).message; + } + } + }; + const taskJson = (t: Task) => ({ + id: t.id, + name: t.name, + creator: "admin", + creator_role: 2, + state: t.state, + status: t.state === 2 ? "done" : "running", + progress: t.state === 2 ? 100 : 0, + start_time: null, + end_time: null, + total_bytes: 0, + error: t.state === 7 ? t.error : "", + }); + + const authed = (req: Request) => { + const a = req.headers.get("authorization") ?? ""; + if (opts.token && a === opts.token) return true; + if (sessions.has(a)) return true; + return !opts.token && !opts.username; + }; + const pwOk = (body: Record) => !opts.pathPassword || body.password === opts.pathPassword; + + const handler = async (req: Request): Promise => { + const url = new URL(req.url); + const path = url.pathname; + if (path.startsWith("/d/")) { + const p = norm(decodeURIComponent(path.slice(2))); + if (url.searchParams.get("sign") !== signOf(p)) return new Response("sign invalid", { status: 401 }); + const f = files.get(p); + if (!f) return new Response("not found", { status: 404 }); + const range = req.headers.get("range"); + let start = 0; + if (range) { + const m = /bytes=(\d+)-/.exec(range); + start = m ? Number(m[1]) : 0; + if (start >= f.data.length) return new Response(null, { status: 416 }); + } + let chunk = f.data.subarray(start); + const broken = downloadBreak; + if (broken !== null) { + downloadBreak = null; + const cut = Math.max(0, broken - start); + const head = chunk.subarray(0, cut); + let sentHead = false; + const stream = new ReadableStream({ + pull(ctl) { + if (!sentHead && head.length) { + sentHead = true; + ctl.enqueue(head.slice()); + return; + } + ctl.error(new Error("connection reset")); + }, + }); + return new Response(stream, { + status: start ? 206 : 200, + headers: { "content-length": String(chunk.length) }, + }); + } + chunk = chunk.slice(); + return new Response(chunk as Uint8Array, { + status: start ? 206 : 200, + headers: { "content-length": String(chunk.length) }, + }); + } + let body: Record | null = null; + if (path !== "/api/fs/put" && req.method === "POST") body = (await req.json().catch(() => null)) ?? null; + requests.push({ path: path + url.search, body, headers: Object.fromEntries(req.headers) }); + if (fail > 0) { + fail--; + throw new TypeError("error sending request: connection refused"); + } + if (rateLimit > 0) { + rateLimit--; + return new Response("Too Many Requests", { status: 429 }); + } + if (path === "/api/public/settings") return ok({ version: opts.version ?? "v4.2.6 (mock)" }); + if (path === "/api/auth/login") { + if (body?.username === opts.username && body?.password === opts.password) { + const t = `sess-${randomUUID()}`; + sessions.add(t); + return ok({ token: t }); + } + return err(400, "password is incorrect or you have no permission"); + } + if (!authed(req)) return err(401, "token is invalidated"); + const b = body ?? {}; + switch (path) { + case "/api/me": + return ok({ id: 1, username: opts.username ?? "admin", base_path: "/", role: 2, permission: 29183 }); + case "/api/admin/storage/list": + return ok({ + content: [...new Set([...dirs.keys()].map(mountOf).filter(Boolean))].map((m, i) => ({ + id: i + 1, + mount_path: `/${m}`, + driver: "Local", + status: "work", + ...(opts.mountDetails?.[`/${m}`] ? { mount_details: opts.mountDetails[`/${m}`] } : {}), + })), + total: 0, + }); + case "/api/fs/list": { + const p = norm(String(b.path ?? "/")); + if (!pwOk(b)) return err(403, "password is incorrect or you have no permission"); + if (!dirs.has(p)) return err(500, "failed to get dir: failed to get obj: object not found"); + const all = children(p).map((n) => objOf(p === "/" ? `/${n}` : `${p}/${n}`)); + const per = Number(b.per_page ?? 0) || 200; + const page = Number(b.page ?? 1) || 1; + return ok({ + content: all.slice((page - 1) * per, page * per), + total: all.length, + readme: "", + header: "", + write: true, + provider: "Local", + }); + } + case "/api/fs/get": { + const p = norm(String(b.path ?? "/")); + if (!pwOk(b)) return err(403, "password is incorrect or you have no permission"); + if (!exists(p)) return err(500, "failed to get obj: object not found"); + return ok(objOf(p, true)); + } + case "/api/fs/mkdir": + try { + mkdirp(String(b.path ?? "/")); + } catch (e) { + return err(500, (e as Error).message); + } + return ok(); + case "/api/fs/rename": { + const p = norm(String(b.path ?? "")); + if (!exists(p)) return err(500, "failed to get obj: object not found"); + const to = `${parent(p)}/${String(b.name)}`.replace("//", "/"); + if (exists(to)) return err(403, `file [${b.name}] exists`); + copyTree(p, to); + removeTree(p); + return ok(); + } + case "/api/fs/remove": { + const dir = norm(String(b.dir ?? "/")); + for (const n of (b.names as string[]) ?? []) removeTree(`${dir}/${n}`.replace("//", "/")); + return ok(); + } + case "/api/fs/copy": + case "/api/fs/move": { + const kind = path.endsWith("copy") ? "copy" : "move"; + const srcDir = norm(String(b.src_dir)); + const dstDir = norm(String(b.dst_dir)); + if (!dirs.has(dstDir)) return err(500, "failed to get dst dir: failed to get obj: object not found"); + const names = (b.names as string[]) ?? []; + for (const n of names) { + if (!exists(`${srcDir}/${n}`)) return err(500, "failed to get src obj: failed to get obj: object not found"); + if (exists(`${dstDir}/${n}`)) return err(403, `file [${n}] exists`); + } + if (kind === "move" && mountOf(srcDir) === mountOf(dstDir)) { + for (const n of names) { + copyTree(`${srcDir}/${n}`, `${dstDir}/${n}`); + removeTree(`${srcDir}/${n}`); + } + return ok({ message: "Move operations completed immediately" }); + } + const created = names.map((n) => + newTask(kind, `${kind} [${srcDir}](${n}) to [${dstDir}]`, () => { + copyTree(`${srcDir}/${n}`, `${dstDir}/${n}`); + if (kind === "move") removeTree(`${srcDir}/${n}`); + }), + ); + if (opts.noTaskIds) { + // 老版本:同步返回,但内容稍后才出现(由任务表推进) + return ok(null); + } + return ok({ message: `Successfully created ${created.length} ${kind} task(s)`, tasks: created.map(taskJson) }); + } + case "/api/fs/put": { + const p = norm(decodeURIComponent(req.headers.get("file-path") ?? "")); + const asTask = req.headers.get("as-task") === "true"; + const reader = req.body!.getReader(); + const chunks: Uint8Array[] = []; + let got = 0; + for (;;) { + const { value, done } = await reader.read(); + if (done) break; + chunks.push(value); + got += value.length; + if (uploadBreak !== null && got >= uploadBreak) { + uploadBreak = null; + await reader.cancel(); + throw new TypeError("connection reset by peer"); + } + } + const data = new Uint8Array(got); + let off = 0; + for (const c of chunks) { + data.set(c, off); + off += c.length; + } + const mod = Number(req.headers.get("last-modified") ?? Date.now()); + const commit = () => { + if (dropUpload) { + dropUpload = false; + return; + } + putFile(p, data, mod); + }; + if (asTask) { + const t = newTask("upload", `upload ${base(p)} to [${parent(p)}]`, commit); + return ok({ task: taskJson(t) }); + } + commit(); + return ok(); + } + } + const tm = /^\/api\/(admin\/)?task\/(copy|move|upload)\/info$/.exec(path); + if (tm) { + if (opts.noTaskApi) return new Response("404 page not found", { status: 404 }); + const t = tasks.get(url.searchParams.get("tid") ?? ""); + if (!t) return err(404, "task not found"); + advanceTask(t); + return ok(taskJson(t)); + } + return new Response("404 page not found", { status: 404 }); + }; + + const mock: MockAlist = { + get base() { + return origin; + }, + setOrigin(o: string) { + origin = o; + }, + handler, + fetch: (async (input: Request | URL | string, init?: RequestInit) => { + const req = input instanceof Request ? input : new Request(String(input), init); + // 老版本(noTaskIds):没有任务 id 的复制 / 移动,在之后的任意 get 上推进 + if (opts.noTaskIds) for (const t of tasks.values()) advanceTask(t); + return handler(req); + }) as typeof fetch, + files, + dirs, + tasks, + requests, + sessions, + rateLimitNext: (n) => { + rateLimit = n; + }, + failNext: (n) => { + fail = n; + }, + breakUploadAfter: (n) => { + uploadBreak = n; + }, + breakDownloadAfter: (n) => { + downloadBreak = n; + }, + expireSigns: () => { + signEpoch++; + }, + failNextTask: (m) => { + nextTaskError = m; + }, + dropNextUpload: () => { + dropUpload = true; + }, + invalidateSessions: () => sessions.clear(), + put: (p, d, o = {}) => putFile(p, typeof d === "string" ? new TextEncoder().encode(d) : d, o.modified, o.hashInfo), + mkdirp, + read: (p) => files.get(norm(p))?.data ?? null, + exists, + listNames: children, + }; + return mock; +} + +/** 起一个真 HTTP 监听(ffprobe / 下载代理等需要真实 URL 的测试用);返回 base 与关闭函数 */ +export function serveMockAlist(mock: MockAlist): { base: string; close: () => Promise } { + const server = Deno.serve({ port: 0, hostname: "127.0.0.1", onListen: () => {} }, (req) => { + const u = new URL(req.url); + return mock + .handler(new Request(`${mock.base}${u.pathname}${u.search}`, req)) + .catch(() => new Response("connection reset", { status: 502 })); + }); + const addr = server.addr as Deno.NetAddr; + const prev = mock.base; + const base = `http://127.0.0.1:${addr.port}`; + mock.setOrigin(base); + return { + base, + close: async () => { + await server.shutdown(); + mock.setOrigin(prev); + }, + }; +} diff --git a/packages/adapters/src/alist/redact.ts b/packages/adapters/src/alist/redact.ts new file mode 100644 index 00000000..9901c123 --- /dev/null +++ b/packages/adapters/src/alist/redact.ts @@ -0,0 +1,22 @@ +import { redactString, redactUrl } from "@umt/contracts"; + +// Alist 相关文本进日志 / 界面前的脱敏:直链的 sign 参数、令牌、密码(§2.3、§9-11)。 + +/** 遮蔽 URL 查询串里的凭证(sign 等)与 userinfo */ +export function redactAlistUrl(url: string): string { + try { + const u = new URL(url); + u.username = ""; + u.password = ""; + return redactUrl(u.toString()); + } catch { + return redactString(url); + } +} + +/** 遮蔽任意文本里出现的给定秘密值(令牌 / 密码 / 路径密码)与直链签名 */ +export function redactAlistText(text: string, secrets: (string | null | undefined)[] = []): string { + let out = text.replace(/([?&]sign=)[^&\s"']+/gi, "$1***"); + for (const s of secrets) if (s && s.length >= 4) out = out.split(s).join("***"); + return redactString(out); +} diff --git a/packages/adapters/src/index.ts b/packages/adapters/src/index.ts index 4b83bba7..8ec0329e 100644 --- a/packages/adapters/src/index.ts +++ b/packages/adapters/src/index.ts @@ -1,5 +1,6 @@ // @umt/adapters 公开入口;W1 起在此导出实现 export const PACKAGE_NAME = "@umt/adapters"; +export * from "./alist/index.ts"; export * from "./browser/index.ts"; export * from "./emby/index.ts"; export * from "./fs/index.ts"; From c2bf8a071a4607e8c4aa50649d0c7ff333e8587c Mon Sep 17 00:00:00 2001 From: root Date: Tue, 6 Oct 2026 23:52:20 +0800 Subject: [PATCH 03/11] =?UTF-8?q?feat(storage):=20=E5=AD=98=E5=82=A8?= =?UTF-8?q?=E5=AE=9E=E4=BE=8B=E4=B8=8E=20Alist=20=E8=99=9A=E6=8B=9F?= =?UTF-8?q?=E8=B7=AF=E5=BE=84(=E8=BF=81=E7=A7=BB=200090=E3=80=81=E8=AE=BE?= =?UTF-8?q?=E7=BD=AE=E6=8E=A5=E5=8F=A3=E3=80=81=E6=B3=A8=E5=86=8C=E8=A1=A8?= =?UTF-8?q?=E3=80=81vfs=E3=80=81=E5=81=A5=E5=BA=B7=E6=A3=80=E6=9F=A5)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - storage_instance 表 + secret 用途 storage_credential;令牌 / 密码 / 路径密码只写不读 - /storage-instances CRUD 与测试连接(版本、用户、根路径、容量);被库根引用不能删 - 远端路径 /@alist//<路径>,vfs 统一本地 / Alist 的 stat、列目录、建目录、读写、移动、删除 - 整理矩阵中涉及 Alist 的传输(storage/transfer.ts)与远端 ffprobe(直链只读头) - 媒体库目录可登记在 Alist 上(不写哨兵,目录可访问即在线);健康检查列出存储实例 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei --- apps/server/src/context.ts | 2 + .../server/src/modules/auth-settings/index.ts | 2 + .../modules/auth-settings/instances-routes.ts | 17 +- .../modules/auth-settings/storage-routes.ts | 264 +++++++++++++ apps/server/src/modules/system/health.ts | 47 ++- apps/server/src/preflight.ts | 5 +- apps/server/src/storage/probe.ts | 23 ++ apps/server/src/storage/registry.ts | 170 +++++++++ apps/server/src/storage/transfer.ts | 278 ++++++++++++++ apps/server/src/storage/vfs.ts | 356 ++++++++++++++++++ apps/server/src/storage/vpath.ts | 45 +++ .../auth-settings/storage-instances.test.ts | 176 +++++++++ packages/adapters/src/fs/ffprobe.ts | 4 +- packages/contracts/src/api/instances.ts | 22 ++ packages/contracts/src/auth.ts | 4 + packages/contracts/src/index.ts | 1 + packages/contracts/src/ops.ts | 4 +- packages/contracts/src/storage.ts | 121 ++++++ .../db/migrations/0090_storage_instance.ts | 62 +++ packages/db/src/types.ts | 30 ++ 20 files changed, 1625 insertions(+), 8 deletions(-) create mode 100644 apps/server/src/modules/auth-settings/storage-routes.ts create mode 100644 apps/server/src/storage/probe.ts create mode 100644 apps/server/src/storage/registry.ts create mode 100644 apps/server/src/storage/transfer.ts create mode 100644 apps/server/src/storage/vfs.ts create mode 100644 apps/server/src/storage/vpath.ts create mode 100644 apps/server/test/auth-settings/storage-instances.test.ts create mode 100644 packages/contracts/src/storage.ts create mode 100644 packages/db/migrations/0090_storage_instance.ts diff --git a/apps/server/src/context.ts b/apps/server/src/context.ts index bd184e0f..5de771e2 100644 --- a/apps/server/src/context.ts +++ b/apps/server/src/context.ts @@ -17,6 +17,7 @@ import type { EventBusPort } from "./http/sse.ts"; import { assertCapabilityCount, createCapabilityRegistry } from "./modules/agent/capability-registry.ts"; import { createSecretStore, createSettingsReader } from "./modules/auth-settings/provided.ts"; import type { OutboxPort, ServerContext, ServerModule, ServerRole } from "./modules/types.ts"; +import { configureStorage } from "./storage/registry.ts"; // 进程装配:ServerContext、任务注册表、outbox 路由。api / worker / all 三种角色共用。 @@ -200,6 +201,7 @@ export async function bootstrap(opts: BootstrapOptions): Promise { // 授权(automation_grant / approval + 自动化总开关)、审计(audit_event)、幂等键与操作账本在这里注入 capabilities: createCapabilityRegistry({ logger, db, settings }), }; + configureStorage(ctx); const tasks = buildTaskRegistry(opts.modules, ctx); for (const m of opts.modules) m.registerCapabilities?.(ctx.capabilities, ctx); ctx.hooks = buildHookRegistry(opts.modules, ctx); diff --git a/apps/server/src/modules/auth-settings/index.ts b/apps/server/src/modules/auth-settings/index.ts index a20f3fdb..584bb9c9 100644 --- a/apps/server/src/modules/auth-settings/index.ts +++ b/apps/server/src/modules/auth-settings/index.ts @@ -7,6 +7,7 @@ import { registerEngineRoutes } from "./engine-routes.ts"; import { registerInstanceRoutes } from "./instances-routes.ts"; import { registerKeyRoutes } from "./keys-routes.ts"; import { registerSettingsRoutes } from "./settings-routes.ts"; +import { registerStorageRoutes } from "./storage-routes.ts"; /** * W2-A 认证/设置/实例/引擎面板:管理员登录门禁、settings、api_key、secret 加密服务、 @@ -38,6 +39,7 @@ export function createAuthSettingsModule(opts: AuthSettingsModuleOptions = {}): registerSettingsRoutes(app, ctx); registerKeyRoutes(app, ctx); registerInstanceRoutes(app, ctx); + registerStorageRoutes(app, ctx); registerEngineRoutes(app, ctx, opts.engineSchema ? { schema: opts.engineSchema } : {}); if (opts.setupTokenOnStart !== false && ctx.db) { ensureSetupToken(ctx).catch((err) => ctx.logger.error({ err }, "生成 setup token 失败")); diff --git a/apps/server/src/modules/auth-settings/instances-routes.ts b/apps/server/src/modules/auth-settings/instances-routes.ts index 4d0e6249..c3eb1447 100644 --- a/apps/server/src/modules/auth-settings/instances-routes.ts +++ b/apps/server/src/modules/auth-settings/instances-routes.ts @@ -25,6 +25,8 @@ import { import { z } from "zod"; import { ApiException } from "../../http/errors.ts"; import { redact, redactString } from "../../logger.ts"; +import { isRemotePath, vcheckRoot } from "../../storage/vfs.ts"; +import { parseVPath } from "../../storage/vpath.ts"; import type { ModuleApp, ServerContext } from "../types.ts"; import { afterCursor, @@ -184,6 +186,17 @@ export function registerInstanceRoutes(app: ModuleApp, ctx: ServerContext) { // ── library_root ── const sentinel = async (localPath: string, file: string | null | undefined) => { + // Alist 库根(/@alist/<存储 id>/<路径>):不写哨兵(同 MP),只确认目录存在且存储可连 + if (isRemotePath(localPath)) { + if (!parseVPath(localPath)) throw new ApiException("validation_failed", "Alist 路径格式不对"); + const h = await vcheckRoot(localPath); + if (h.kind !== "online") + throw new ApiException( + "precondition_failed", + `Alist 目录不可用（${h.reason}），先在 Alist 里建好目录并测试存储连接`, + ); + return null; + } const name = file || DEFAULT_ROOT_SENTINEL; try { const st = await stat(localPath); @@ -229,7 +242,9 @@ export function registerInstanceRoutes(app: ModuleApp, ctx: ServerContext) { test("/library-roots/{id}/test", "检查媒体库目录在线(目录 + 哨兵文件)", "library-roots"), async ({ params, actor }) => { const row = await getRow("library_root", params.id); - const h = await checkRoot(String(row.local_path), (row.sentinel_file as string | null) ?? DEFAULT_ROOT_SENTINEL); + const h = isRemotePath(String(row.local_path)) + ? await vcheckRoot(String(row.local_path)) + : await checkRoot(String(row.local_path), (row.sentinel_file as string | null) ?? DEFAULT_ROOT_SENTINEL); const ok = h.kind === "online"; await update("library_root", row.id, { health_status: ok ? "healthy" : "offline", diff --git a/apps/server/src/modules/auth-settings/storage-routes.ts b/apps/server/src/modules/auth-settings/storage-routes.ts new file mode 100644 index 00000000..6f3d72de --- /dev/null +++ b/apps/server/src/modules/auth-settings/storage-routes.ts @@ -0,0 +1,264 @@ +import { AlistError, normAlistPath, redactAlistText } from "@umt/adapters"; +import { + Api, + StorageInstance, + StorageInstanceCreate, + StorageInstanceUpdate, + type StorageTestResult, + secretInput, +} from "@umt/contracts"; +import { ApiException } from "../../http/errors.ts"; +import { buildClient, clearStorageCache, configureStorage, resolvePlain } from "../../storage/registry.ts"; +import type { ModuleApp, ServerContext } from "../types.ts"; +import { + afterCursor, + audit, + crudDefs, + cursorCol, + decodeCursor, + endpoint, + pageOf, + pgConflict, + toApi, + toDb, +} from "./endpoint.ts"; + +// 存储实例(Alist / OpenList)CRUD + 测试连接(docs/design/storage-alist.md §3、§4)。 +// 凭据(令牌 / 密码)与路径密码只写不读:省略或空串 = 不变,null = 清除;明文只在 secret 表里加密保存。 + +const JSON_COLS = ["options", "health_detail", "version_info"] as const; +const tags = ["storage-instances"]; + +type Row = Record & { + id: string; + secret_ref: string | null; + path_password_ref: string | null; + auth_kind: "token" | "password" | "guest"; + username: string | null; +}; + +export function registerStorageRoutes(app: ModuleApp, ctx: ServerContext) { + configureStorage(ctx); + const S = { + ...crudDefs("/storage-instances", "存储实例", { + row: StorageInstance, + create: StorageInstanceCreate, + update: StorageInstanceUpdate, + }), + test: Api.storageInstances.test, + remove: Api.storageInstances.remove, + }; + const db = () => ctx.db; + + const getRow = async (id: string): Promise => { + const r = await db().selectFrom("storage_instance").selectAll().where("id", "=", id).executeTakeFirst(); + if (!r) throw new ApiException("not_found", "存储实例不存在"); + return r as unknown as Row; + }; + const normalize = (b: Partial) => { + const out: Record = { ...b }; + if (b.baseUrl !== undefined) out.baseUrl = b.baseUrl.trim().replace(/\/+$/, ""); + if (b.rootPath !== undefined) { + try { + out.rootPath = normAlistPath(b.rootPath); + } catch { + throw new ApiException("validation_failed", "根路径不合法"); + } + } + return out; + }; + const checkAuth = (authKind: string, username: string | null | undefined, hasCredential: boolean) => { + if (authKind === "token" && !hasCredential) throw new ApiException("validation_failed", "令牌认证需要填写令牌"); + if (authKind === "password" && (!username || !hasCredential)) + throw new ApiException("validation_failed", "账号密码认证需要填写用户名和密码"); + }; + const putSecret = async (oldRef: string | null, plaintext: string) => { + if (oldRef) { + await ctx.secrets.replace(oldRef, plaintext); + return oldRef; + } + return ctx.secrets.create({ purpose: "storage_credential", plaintext }); + }; + const view = (r: object) => toApi(r); + + endpoint(app, { ...S.list, tags }, async ({ query }) => { + const q0 = query as { cursor?: string; limit: number }; + const cur = decodeCursor(q0.cursor); + let q = db().selectFrom("storage_instance").selectAll().select(cursorCol("created_at")); + if (cur) q = q.where(afterCursor("created_at", cur)); + const rows = await q + .orderBy("created_at", "desc") + .orderBy("id", "desc") + .limit(q0.limit + 1) + .execute(); + const page = pageOf(rows as unknown as { cursor_us: number; id: string }[], q0.limit); + return { items: page.items.map(view), nextCursor: page.nextCursor } as never; + }); + endpoint(app, { ...S.get, tags }, async ({ params }) => view(await getRow(params.id)) as never); + + endpoint(app, { ...S.create, tags }, async ({ body, actor }) => { + const { credential: rawCredential, pathPassword: rawPw, ...rest } = body; + const credential = secretInput(rawCredential) ?? undefined; + const pathPassword = secretInput(rawPw) ?? undefined; + checkAuth(rest.authKind ?? "token", rest.username, !!credential); + const refs: string[] = []; + try { + const secretRef = + credential && (rest.authKind ?? "token") !== "guest" + ? await ctx.secrets.create({ purpose: "storage_credential", plaintext: credential }) + : null; + if (secretRef) refs.push(secretRef); + const pwRef = pathPassword + ? await ctx.secrets.create({ purpose: "storage_credential", plaintext: pathPassword }) + : null; + if (pwRef) refs.push(pwRef); + const row = await db() + .insertInto("storage_instance") + .values({ + ...(toDb(normalize(rest), JSON_COLS) as { name: string; base_url: string }), + secret_ref: secretRef, + path_password_ref: pwRef, + }) + .returningAll() + .executeTakeFirstOrThrow() + .catch((e: unknown) => pgConflict(e, "存储实例名称重复")); + await audit( + ctx, + actor, + "storage_instance.create", + { type: "storage_instance", id: row.id }, + { name: row.name, authKind: row.auth_kind }, + ); + return view(row) as never; + } catch (e) { + for (const r of refs) await ctx.secrets.remove(r).catch(() => {}); + throw e; + } + }); + + endpoint(app, { ...S.update, tags }, async ({ params, body, actor }) => { + const { credential: rawCredential, pathPassword: rawPw, ...rest } = body; + const credential = secretInput(rawCredential); + const pathPassword = secretInput(rawPw); + const cur = await getRow(params.id); + const authKind = rest.authKind ?? cur.auth_kind; + const username = rest.username !== undefined ? rest.username : cur.username; + const hasCred = credential === null ? false : credential ? true : !!cur.secret_ref; + checkAuth(authKind, username, hasCred); + const values = toDb(normalize(rest), JSON_COLS); + const drop: string[] = []; + if (authKind === "guest" || credential === null) { + if (cur.secret_ref) { + values.secret_ref = null; + drop.push(cur.secret_ref); + } + } else if (credential) values.secret_ref = await putSecret(cur.secret_ref, credential); + if (pathPassword === null) { + if (cur.path_password_ref) { + values.path_password_ref = null; + drop.push(cur.path_password_ref); + } + } else if (pathPassword) values.path_password_ref = await putSecret(cur.path_password_ref, pathPassword); + // 改了凭据也要让注册表重建客户端:updated_at 由触发器刷新;无字段变化时补一次 touch + const row = await db() + .updateTable("storage_instance") + .set({ ...(values as Record), updated_at: new Date() }) + .where("id", "=", params.id) + .returningAll() + .executeTakeFirst() + .catch((e: unknown) => pgConflict(e, "存储实例名称重复")); + if (!row) throw new ApiException("not_found", "存储实例不存在"); + for (const r of drop) await ctx.secrets.remove(r).catch(() => {}); + clearStorageCache(row.id); + await audit( + ctx, + actor, + "storage_instance.update", + { type: "storage_instance", id: row.id }, + { + fields: Object.keys(body).map((k) => + k === "credential" || k === "pathPassword" ? `${k}(${body[k] === null ? "cleared" : "set"})` : k, + ), + }, + ); + return view(row) as never; + }); + + endpoint(app, { ...S.remove, tags }, async ({ params, actor }) => { + const row = await getRow(params.id); + const prefix = `/@alist/${row.id}`; + const used = await db() + .selectFrom("library_root") + .select("name") + .where((eb) => eb.or([eb("local_path", "=", prefix), eb("local_path", "like", `${prefix}/%`)])) + .execute(); + if (used.length) + throw new ApiException( + "conflict", + `存储实例仍被媒体库目录使用：${used.map((u) => u.name).join("、")}，先删除或改用其它存储`, + ); + await db().deleteFrom("storage_instance").where("id", "=", row.id).execute(); + for (const r of [row.secret_ref, row.path_password_ref]) if (r) await ctx.secrets.remove(r).catch(() => {}); + clearStorageCache(row.id); + await audit(ctx, actor, "storage_instance.delete", { type: "storage_instance", id: row.id }); + return { ok: true as const }; + }); + + endpoint(app, { ...S.test, tags }, async ({ params, actor }) => { + const row = await getRow(params.id); + const result = await testStorage(ctx, row); + await db() + .updateTable("storage_instance") + .set({ + health_status: result.healthStatus, + last_health_at: new Date(), + health_detail: JSON.stringify({ message: result.message, rootOk: result.rootOk, usage: result.usage }), + ...(result.version ? { version_info: JSON.stringify({ version: result.version, user: result.user }) } : {}), + }) + .where("id", "=", row.id) + .execute(); + clearStorageCache(row.id); + await audit(ctx, actor, "storage_instance.test", { type: "storage_instance", id: row.id }, { ok: result.ok }); + return result; + }); +} + +/** 测试连接(设置页「测试连接」与健康检查共用) */ +export async function testStorage( + ctx: Pick, + row: Row & { base_url?: unknown; root_path?: unknown; options?: unknown }, +): Promise { + let plain: { credential: string | null; pathPassword: string | null } = { credential: null, pathPassword: null }; + const fail = (healthStatus: StorageTestResult["healthStatus"], message: string): StorageTestResult => ({ + ok: false, + healthStatus, + version: null, + user: null, + rootOk: false, + usage: null, + message: redactAlistText(message, [plain.credential, plain.pathPassword]).slice(0, 500), + }); + try { + plain = await resolvePlain(ctx.secrets, row); + const client = buildClient(row as never, plain); + const version = await client.version(); + const user = row.auth_kind === "guest" ? "guest" : (await client.me()).username; + const root = await client.stat(String(row.root_path ?? "/")); + const usage = await client.usage(String(row.root_path ?? "/")); + if (!root?.isDir) return { ...fail("degraded", `根路径不存在或不是目录：${row.root_path}`), version, user, usage }; + return { ok: true, healthStatus: "healthy", version, user, rootOk: true, usage, message: null }; + } catch (e) { + if (e instanceof AlistError) { + const label = + e.kind === "auth" + ? row.auth_kind === "token" + ? "令牌失效或无权限" + : "登录失败（用户名或密码不对）" + : e.kind === "unavailable" || e.kind === "timeout" + ? "连不上" + : "接口出错"; + return fail("offline", `${label}：${e.message}`); + } + return fail("offline", String((e as Error)?.message ?? e)); + } +} diff --git a/apps/server/src/modules/system/health.ts b/apps/server/src/modules/system/health.ts index 207e5639..1cbc209e 100644 --- a/apps/server/src/modules/system/health.ts +++ b/apps/server/src/modules/system/health.ts @@ -10,6 +10,9 @@ import { SystemWorkerLane, } from "@umt/contracts"; import { type DB, type Kysely, type Selectable, sql } from "@umt/db"; +import { storageEntry } from "../../storage/registry.ts"; +import { isRemotePath, vcheckRoot } from "../../storage/vfs.ts"; +import { toVPath } from "../../storage/vpath.ts"; import { CURRENT_HOST_WINDOW_MS, HEARTBEAT_ROW_RETENTION_MS, HEARTBEAT_STALE_MS } from "./heartbeat.ts"; // GET /system/health 各段的取数(W9-I)。每段独立 try/catch:一段失败只把该段标 unknown,不让整页 500。 @@ -214,7 +217,14 @@ export async function loadPathTargets(db: Kysely) { .where("path_mapping.view_kind", "=", "downloader") .orderBy("path_mapping.local_prefix") .execute(); - return { roots, maps }; + const storages = await db + .selectFrom("storage_instance") + .select(["id", "name", "root_path"]) + .where("enabled", "=", true) + .orderBy("name") + .execute() + .catch(() => [] as { id: string; name: string; root_path: string }[]); + return { roots, maps, storages }; } export function diskLevel(ratio: number, t: HealthDeps["thresholds"]): HealthLevel { @@ -226,11 +236,14 @@ export async function loadDisksAndMounts(d: HealthDeps): Promise<{ disks: System const usage = d.diskUsage ?? diskUsage; const check = d.checkRoot ?? checkRoot; const ms = d.ioTimeoutMs ?? 3000; - const { roots, maps } = await loadPathTargets(d.db); + const { roots, maps, storages } = await loadPathTargets(d.db); const targets: PathTarget[] = [ { scope: "data_dir", id: null, name: "DATA_DIR", path: d.dataDir }, - ...roots.map((r) => ({ scope: "library_root" as const, id: r.id, name: r.name, path: r.local_path })), + // Alist 库根没有本机磁盘可统计(容量在文件管理按 mount_details 显示) + ...roots + .filter((r) => !isRemotePath(r.local_path)) + .map((r) => ({ scope: "library_root" as const, id: r.id, name: r.name, path: r.local_path })), ...maps.map((m) => ({ scope: "download_dir" as const, id: m.id, @@ -280,7 +293,12 @@ export async function loadDisksAndMounts(d: HealthDeps): Promise<{ disks: System ...roots.map(async (r): Promise => { const base = { kind: "library_root" as const, id: r.id, name: r.name, path: r.local_path, sentinelChecked: true }; try { - const h = await withTimeout(check(r.local_path, r.sentinel_file ?? DEFAULT_ROOT_SENTINEL), ms); + const h = await withTimeout( + isRemotePath(r.local_path) + ? vcheckRoot(r.local_path) + : check(r.local_path, r.sentinel_file ?? DEFAULT_ROOT_SENTINEL), + isRemotePath(r.local_path) ? Math.max(ms, 15_000) : ms, + ); return h.kind === "online" ? { ...base, status: "online", reason: null } : { ...base, status: "offline", reason: h.reason }; @@ -316,6 +334,27 @@ export async function loadDisksAndMounts(d: HealthDeps): Promise<{ disks: System return { ...base, status: "offline", reason }; } }), + // Alist / OpenList 实例:连通 + 根路径可访问(令牌失效、连不上都在这里显示) + ...storages.map(async (s): Promise => { + const path = toVPath(s.id, s.root_path); + const base = { + kind: "storage_instance" as const, + id: s.id, + name: s.name, + path: s.root_path, + sentinelChecked: false, + }; + try { + await storageEntry(s.id); + const h = await withTimeout(vcheckRoot(path), Math.max(ms, 15_000)); + if (h.kind === "online") return { ...base, status: "online", reason: null }; + return { ...base, status: h.reason === "timeout" ? "timeout" : "offline", reason: h.reason }; + } catch (err) { + return err instanceof Timeout + ? { ...base, status: "timeout", reason: "timeout" } + : { ...base, status: "offline", reason: "error" }; + } + }), ]); return { disks, mounts }; } diff --git a/apps/server/src/preflight.ts b/apps/server/src/preflight.ts index a554bd06..281041a0 100644 --- a/apps/server/src/preflight.ts +++ b/apps/server/src/preflight.ts @@ -85,7 +85,10 @@ export async function loadMountedPaths(db: Kysely): Promise { .where("view_kind", "=", "downloader") .execute(); return [ - ...roots.map((r) => ({ kind: "library_root" as const, id: r.id, name: r.name, path: r.local_path })), + // Alist 库根(/@alist/…)不是本机挂载,由健康检查按实例连通性判断 + ...roots + .filter((r) => !r.local_path.startsWith("/@alist/")) + .map((r) => ({ kind: "library_root" as const, id: r.id, name: r.name, path: r.local_path })), ...maps.map((m) => ({ kind: "path_mapping" as const, id: m.id, name: m.remote_prefix, path: m.local_prefix })), ]; } diff --git a/apps/server/src/storage/probe.ts b/apps/server/src/storage/probe.ts new file mode 100644 index 00000000..4eb6df96 --- /dev/null +++ b/apps/server/src/storage/probe.ts @@ -0,0 +1,23 @@ +import { ffprobe, type ProbeResult, redactAlistText } from "@umt/adapters"; +import { vrawUrl } from "./vfs.ts"; +import { parseVPath } from "./vpath.ts"; + +// 远端文件的 ffprobe(docs/design/storage-alist.md §7):不下载,直接对签名直链探测,ffprobe 只读容器头与少量包(HTTP Range)。 +// 拿不到直链或探测失败返回 unknown(不阻塞,同本地 ffprobe 失败的口径)。直链进错误信息前脱敏。 + +export async function probeAny( + path: string, + local: (p: string) => Promise = (p) => ffprobe(p), +): Promise { + if (!parseVPath(path)) return local(path); + let url: string; + try { + url = await vrawUrl(path); + } catch (e) { + return { status: "unknown", reason: "probe_failed", message: redactAlistText(String((e as Error)?.message ?? e)) }; + } + const r = await ffprobe(url, { inputArgs: ["-rw_timeout", "30000000"], timeoutMs: 60_000 }); + if (r.status === "unknown" && r.message) + return { ...r, message: redactAlistText(r.message.split(url).join("<直链>")) }; + return r; +} diff --git a/apps/server/src/storage/registry.ts b/apps/server/src/storage/registry.ts new file mode 100644 index 00000000..92e6df15 --- /dev/null +++ b/apps/server/src/storage/registry.ts @@ -0,0 +1,170 @@ +import { type AlistAuth, AlistClient, AlistError } from "@umt/adapters"; +import { STORAGE_OPTION_DEFAULTS, type StorageOptions } from "@umt/contracts"; +import type { DB, Kysely, StorageInstanceRow } from "@umt/db"; +import type { ServerContext } from "../modules/types.ts"; + +// 存储实例注册表(docs/design/storage-alist.md §3):按 id 取实例 + 解密凭据 → AlistClient。 +// 进程级:模块装配时 configureStorage(ctx);实例行按 updated_at 缓存,配置一改客户端重建(令牌缓存随之作废,同 MP init_storage)。 + +export interface StorageEntry { + id: string; + name: string; + enabled: boolean; + /** 本系统可用的 Alist 根路径 */ + rootPath: string; + options: Required; + client: AlistClient; + /** 日志脱敏要遮的秘密值 */ + secrets: string[]; +} + +interface Source { + dataDir?: string | undefined; + db: Kysely; + secrets: ServerContext["secrets"]; + logger?: ServerContext["logger"] | undefined; +} + +let source: Source | null = null; +const cache = new Map(); +const overrides = new Map(); +const RECHECK_MS = 5_000; + +/** 模块装配时调用(幂等;最后一次为准) */ +export function configureStorage( + ctx: Pick & { + logger?: ServerContext["logger"]; + config?: Partial>; + }, +) { + if (source?.db !== ctx.db) cache.clear(); + source = { db: ctx.db, secrets: ctx.secrets, logger: ctx.logger, dataDir: ctx.config?.DATA_DIR }; +} + +/** 数据目录(跨存储中转、上传暂存放在 /cache/storage-relay) */ +export const storageDataDir = (): string | undefined => source?.dataDir; + +/** 测试用:直接给某个存储 id 指定客户端(mock Alist) */ +export function overrideStorage( + id: string, + e: { client: AlistClient; name?: string; rootPath?: string; options?: StorageOptions } | null, +): void { + if (!e) { + overrides.delete(id); + return; + } + overrides.set(id, { + id, + name: e.name ?? "Alist", + enabled: true, + rootPath: e.rootPath ?? "/", + options: { ...STORAGE_OPTION_DEFAULTS, ...(e.options ?? {}) }, + client: e.client, + secrets: [], + }); +} + +export function clearStorageCache(id?: string): void { + if (id) cache.delete(id); + else cache.clear(); +} + +export const storageOptions = (o: unknown): Required => ({ + ...STORAGE_OPTION_DEFAULTS, + ...((o && typeof o === "object" ? o : {}) as StorageOptions), +}); + +/** 由实例行 + 明文凭据建客户端(测试连接、注册表共用) */ +export function buildClient( + row: Pick, + plain: { credential: string | null; pathPassword: string | null }, + logger?: ServerContext["logger"], +): AlistClient { + const o = storageOptions(row.options); + let auth: AlistAuth; + if (row.auth_kind === "token") { + if (!plain.credential) throw new AlistError("auth", "没有配置令牌"); + auth = { kind: "token", token: plain.credential }; + } else if (row.auth_kind === "password") { + if (!row.username || !plain.credential) throw new AlistError("auth", "账号密码认证需要用户名和密码"); + auth = { kind: "password", username: row.username, password: plain.credential }; + } else auth = { kind: "guest" }; + return new AlistClient({ + baseUrl: row.base_url, + auth, + pathPassword: plain.pathPassword, + qps: o.qps, + concurrency: o.concurrency, + timeoutMs: o.timeoutSec * 1000, + idleTimeoutMs: o.idleTimeoutSec * 1000, + taskTimeoutMs: o.taskTimeoutMin * 60_000, + retries: o.retries, + uploadAsTask: o.uploadAsTask, + ...(logger + ? { + logger: { + debug: (obj, msg) => logger.debug(obj, msg), + warn: (obj, msg) => logger.warn(obj, msg), + }, + } + : {}), + }); +} + +export async function resolvePlain( + secrets: Source["secrets"], + row: Pick, +) { + return { + credential: row.secret_ref ? await secrets.resolve(row.secret_ref) : null, + pathPassword: row.path_password_ref ? await secrets.resolve(row.path_password_ref) : null, + }; +} + +/** 取存储实例;不存在 / 停用 / 未装配抛 AlistError(unavailable,按「存储离线」处理,不当成文件不存在) */ +export async function storageEntry(id: string, opts: { allowDisabled?: boolean } = {}): Promise { + const o = overrides.get(id); + if (o) return o; + if (!source) throw new AlistError("unavailable", "存储注册表未初始化"); + const hit = cache.get(id); + if (hit && Date.now() - hit.checkedAt < RECHECK_MS) return check(hit.entry, opts); + const row = await source.db.selectFrom("storage_instance").selectAll().where("id", "=", id).executeTakeFirst(); + if (!row) throw new AlistError("unavailable", `存储实例不存在：${id}`); + const stamp = new Date(row.updated_at).toISOString(); + if (hit && hit.stamp === stamp) { + hit.checkedAt = Date.now(); + return check(hit.entry, opts); + } + const plain = await resolvePlain(source.secrets, row); + const entry: StorageEntry = { + id: row.id, + name: row.name, + enabled: row.enabled, + rootPath: row.root_path, + options: storageOptions(row.options), + client: buildClient(row, plain, source.logger), + secrets: [plain.credential, plain.pathPassword].filter((s): s is string => !!s), + }; + cache.set(id, { stamp, entry, checkedAt: Date.now() }); + return check(entry, opts); +} + +function check(e: StorageEntry, opts: { allowDisabled?: boolean }): StorageEntry { + if (!e.enabled && !opts.allowDisabled) throw new AlistError("unavailable", `存储实例已停用：${e.name}`); + return e; +} + +/** 启用的实例(文件管理根、健康检查) */ +export async function listStorages(db?: Kysely): Promise { + const d = db ?? source?.db; + if (!d) return []; + return d.selectFrom("storage_instance").selectAll().orderBy("name").execute(); +} + +/** 存储名表(展示「存储名:/路径」) */ +export async function storageNames(db?: Kysely): Promise> { + const m = new Map(); + for (const [id, e] of overrides) m.set(id, e.name); + for (const r of await listStorages(db)) m.set(r.id, r.name); + return m; +} diff --git a/apps/server/src/storage/transfer.ts b/apps/server/src/storage/transfer.ts new file mode 100644 index 00000000..021e29ab --- /dev/null +++ b/apps/server/src/storage/transfer.ts @@ -0,0 +1,278 @@ +import { randomUUID } from "node:crypto"; +import { mkdir, rename, rm, stat, unlink } from "node:fs/promises"; +import { basename, dirname, join, posix } from "node:path"; +import { AlistError, type ProgressCallback } from "@umt/adapters"; +import { invalidateRemote, relayDir, remote, remoteError } from "./vfs.ts"; +import { parseVPath, sameStorage } from "./vpath.ts"; + +// 整理矩阵中涉及 Alist 的方向(docs/design/storage-alist.md §5): +// 本地→Alist 上传、Alist→本地 下载、同实例 Alist→Alist API copy / move、跨实例 下载→上传。 +// 成功判定与 MP 一致(用户决定):接口成功 + 目标存在,不比对大小 / 哈希;移动在判定通过后删源,删源失败 = attention。 +// 本地 ↔ 本地不经这里(仍走 fsproxy 与现有核验)。 + +export interface RemoteTransferInput { + kind: "copy" | "move"; + src: string; + dst: string; + /** 目标已存在:conflict 失败;replace 先把旧文件移到 replaceTrashDir(同存储),成功后旧文件留在那里,失败移回 */ + onExisting: "conflict" | "replace"; + replaceTrashDir?: string; + /** 本次是重试(上次可能已完成外部动作) */ + resumed?: boolean; + signal?: AbortSignal; + onProgress?: ProgressCallback; +} + +export interface RemoteTransferResult { + status: "succeeded" | "failed" | "attention"; + bytes?: number; + /** 顺带得到的本地文件 sha256(上传的源 / 下载的结果),只是事实记录 */ + checksum?: string; + resumed?: boolean; + error?: { code: string; message: string }; + replaced?: { disposition: "trash"; path: string; size: number; mtimeMs: number }; + steps?: { step: string; status: string; at: string; detail?: string; error?: { code: string; message: string } }[]; + via: "upload" | "download" | "alist_copy" | "alist_move" | "relay"; +} + +/** Alist / 本地错误 → 整理账本错误码(可重试的 IO/TIMEOUT 走任务重试) */ +export function transferErrorCode(e: unknown, role: "源" | "目标" | null = null): { code: string; message: string } { + if (e instanceof AlistError) { + const code = + e.kind === "not_found" + ? role === "源" + ? "SOURCE_MISSING" + : "IO" + : e.kind === "conflict" + ? "EEXIST" + : e.kind === "auth" + ? "EACCES" + : e.kind === "timeout" || e.kind === "rate_limited" + ? "TIMEOUT" + : e.kind === "aborted" + ? "ABORTED" + : "IO"; + return { code, message: `Alist：${e.message}` }; + } + const c = (e as { code?: string } | null)?.code; + const message = (e as Error)?.message ?? String(e); + if (c === "ENOENT") return { code: role === "源" ? "SOURCE_MISSING" : "IO", message }; + if (c === "EEXIST") return { code: "EEXIST", message }; + if (c === "EACCES" || c === "EPERM") return { code: "EACCES", message }; + if (c === "ENOSPC") return { code: "ENOSPC", message }; + if (c === "ETIMEDOUT") return { code: "TIMEOUT", message }; + if (c === "ABORT_ERR") return { code: "ABORTED", message }; + return { code: "IO", message }; +} + +const now = () => new Date().toISOString(); + +async function localExists(p: string): Promise<{ size: number; mtimeMs: number } | null> { + try { + const s = await stat(p); + return { size: s.size, mtimeMs: s.mtimeMs }; + } catch (e) { + if ((e as { code?: string }).code === "ENOENT") return null; + throw e; + } +} + +/** 任一端是远端路径? */ +export const involvesRemote = (src: string, dst: string) => !!parseVPath(src) || !!parseVPath(dst); + +/** 执行一次涉及远端的复制 / 移动 */ +export async function remoteTransfer(x: RemoteTransferInput): Promise { + const rs = parseVPath(x.src); + const rd = parseVPath(x.dst); + if (rs) invalidateRemote(x.src); + if (rd) invalidateRemote(x.dst); + const via: RemoteTransferResult["via"] = + rs && rd + ? sameStorage(x.src, x.dst) + ? x.kind === "copy" + ? "alist_copy" + : "alist_move" + : "relay" + : rd + ? "upload" + : "download"; + const fail = (e: unknown, role: "源" | "目标" | null = null): RemoteTransferResult => ({ + status: "failed", + error: transferErrorCode(e, role), + via, + }); + + // ── 源 / 目标状态 ── + let srcSize = 0; + let srcExists = true; + try { + if (rs) { + const { client } = await remote(x.src); + const o = await client.stat(rs.remote); + srcExists = !!o; + srcSize = o?.size ?? 0; + if (o?.isDir) return fail(new AlistError("error", `源是目录：${x.src}`)); + } else { + const s = await localExists(x.src); + srcExists = !!s; + srcSize = s?.size ?? 0; + } + } catch (e) { + return fail(e, "源"); + } + let dstFacts: { size: number; mtimeMs: number } | null; + try { + if (rd) { + const { client } = await remote(x.dst); + const o = await client.stat(rd.remote); + dstFacts = o ? { size: o.size, mtimeMs: o.modified } : null; + } else dstFacts = await localExists(x.dst); + } catch (e) { + return fail(e, "目标"); + } + // 对账:上次尝试已完成外部动作但没留下结果(目标都是「临时名 → 改名」落地的,目标在 = 上次已写完) + if (dstFacts && x.resumed && x.onExisting !== "replace") { + if (x.kind === "move" && srcExists) { + const del = await deleteSource(x.src).then( + () => null, + (e) => transferErrorCode(e), + ); + if (del) + return { + status: "attention", + bytes: dstFacts.size, + resumed: true, + via, + steps: [{ step: "delete_source", status: "failed", at: now(), error: del }], + }; + } + return { status: "succeeded", bytes: dstFacts.size, resumed: true, via }; + } + if (!srcExists) { + if (x.kind === "move" && dstFacts) return { status: "succeeded", bytes: dstFacts.size, resumed: true, via }; + return { status: "failed", error: { code: "SOURCE_MISSING", message: `源文件不存在：${x.src}` }, via }; + } + if (dstFacts && x.onExisting !== "replace") + return { status: "failed", error: { code: "EEXIST", message: `目标已存在：${x.dst}` }, via }; + + // ── 替换:旧目标先移走 ── + let parked: { path: string; size: number; mtimeMs: number } | null = null; + if (dstFacts && x.onExisting === "replace") { + const trashDir = x.replaceTrashDir ?? join(dirname(x.dst), ".umt-trash"); + const to = join(trashDir, `replaced-${Date.now()}-${randomUUID().slice(0, 6)}`, basename(x.dst)); + try { + await parkOld(x.dst, to); + parked = { path: to, size: dstFacts.size, mtimeMs: dstFacts.mtimeMs }; + } catch (e) { + return fail(e, "目标"); + } + } + const restoreParked = async () => { + if (parked) await parkOld(parked.path, x.dst).catch(() => undefined); + }; + + // ── 传输 ── + let bytes = srcSize; + let checksum: string | undefined; + try { + if (rd && !rs) { + const { client } = await remote(x.dst); + const r = await client.upload(x.src, rd.remote, { + ...(x.signal ? { signal: x.signal } : {}), + ...(x.onProgress ? { onProgress: x.onProgress } : {}), + }); + bytes = r.bytes; + checksum = `sha256:${r.hashes.sha256}`; + } else if (rs && !rd) { + await mkdir(dirname(x.dst), { recursive: true }); + const { client } = await remote(x.src); + const r = await client.download(rs.remote, x.dst, { + ...(x.signal ? { signal: x.signal } : {}), + ...(x.onProgress ? { onProgress: x.onProgress } : {}), + }); + bytes = r.bytes; + checksum = `sha256:${r.sha256}`; + } else if (rs && rd && sameStorage(x.src, x.dst)) { + const { client } = await remote(x.src); + const opts = x.signal ? { signal: x.signal } : {}; + const o = + x.kind === "copy" + ? await client.copyFile(rs.remote, rd.remote, opts) + : await client.moveFile(rs.remote, rd.remote, opts); + bytes = o.size || srcSize; + } else if (rs && rd) { + // 跨实例:下载到本机中转 → 上传(MP 不支持,我们补上) + const tmpDir = join(relayDir(), randomUUID()); + await mkdir(tmpDir, { recursive: true }); + const tmp = join(tmpDir, posix.basename(rs.remote)); + try { + const a = await remote(x.src); + await a.client.download(rs.remote, tmp, { + ...(x.signal ? { signal: x.signal } : {}), + ...(x.onProgress ? { onProgress: (d, t) => x.onProgress?.(d / 2, t) } : {}), + }); + const b = await remote(x.dst); + const r = await b.client.upload(tmp, rd.remote, { + ...(x.signal ? { signal: x.signal } : {}), + ...(x.onProgress ? { onProgress: (d, t) => x.onProgress?.(t / 2 + d / 2, t) } : {}), + }); + bytes = r.bytes; + checksum = `sha256:${r.hashes.sha256}`; + } finally { + await rm(tmpDir, { recursive: true, force: true }).catch(() => undefined); + } + } + } catch (e) { + await restoreParked(); + return fail(e); + } + const out: RemoteTransferResult = { + status: "succeeded", + bytes, + ...(checksum ? { checksum } : {}), + via, + ...(parked ? { replaced: { disposition: "trash" as const, ...parked } } : {}), + }; + // 同实例 API move 自己删了源;其它移动在成功后删源 + if (x.kind === "move" && via !== "alist_move") { + const del = await deleteSource(x.src).then( + () => null, + (e) => transferErrorCode(e), + ); + if (del) + return { + ...out, + status: "attention", + steps: [{ step: "delete_source", status: "failed", at: now(), error: del }], + }; + } + return out; +} + +async function deleteSource(src: string): Promise { + const v = parseVPath(src); + if (!v) { + await unlink(src); + return; + } + const { client } = await remote(src); + try { + await client.removePath(v.remote); + } catch (e) { + throw remoteError(e, src); + } +} + +/** 同存储内把 from 移到 to(替换时暂放旧文件 / 失败移回) */ +async function parkOld(from: string, to: string): Promise { + const a = parseVPath(from); + const b = parseVPath(to); + if (!a && !b) { + await mkdir(dirname(to), { recursive: true }); + await rename(from, to); + return; + } + if (!a || !b || a.storageId !== b.storageId) throw new AlistError("error", "替换暂存必须与目标在同一存储"); + const { client } = await remote(from); + await client.moveFile(a.remote, b.remote); +} diff --git a/apps/server/src/storage/vfs.ts b/apps/server/src/storage/vfs.ts new file mode 100644 index 00000000..2221c5ac --- /dev/null +++ b/apps/server/src/storage/vfs.ts @@ -0,0 +1,356 @@ +import { createHash, randomUUID } from "node:crypto"; +import { lstat, mkdir, readdir, readFile, realpath, rename, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, dirname, join, posix } from "node:path"; +import { + type AlistClient, + AlistError, + checkRoot, + DEFAULT_ROOT_SENTINEL, + type FileFacts, + type RootHealth, + type StatResult, + statSafe, +} from "@umt/adapters"; +import { storageDataDir, storageEntry } from "./registry.ts"; +import { parseVPath, toVPath, type VPath } from "./vpath.ts"; + +// 本地 / Alist 统一的文件操作(docs/design/storage-alist.md §3)。本地分支原样调用 node:fs / adapters, +// 远端分支经注册表取 AlistClient。远端错误映射成 errno 风格的 code:不存在 ENOENT、冲突 EEXIST、鉴权 EACCES, +// 连接 / 服务端失败 EHOSTDOWN / ETIMEDOUT(文件管理按「根不可用」503 处理,整理按可重试 IO 处理,绝不当成不存在)。 + +export { isRemotePath, parseVPath, toVPath } from "./vpath.ts"; + +export interface VStats { + size: number; + mtimeMs: number; + ino: number; + dev: number; + nlink: number; + isFile(): boolean; + isDirectory(): boolean; + isSymbolicLink(): boolean; +} + +/** Alist 错误 → 带 errno code 的 Error */ +export function remoteError(e: unknown, path?: string): Error & { code: string } { + if (e instanceof AlistError) { + const code = + e.kind === "not_found" + ? "ENOENT" + : e.kind === "conflict" + ? "EEXIST" + : e.kind === "auth" + ? "EACCES" + : e.kind === "timeout" || e.kind === "rate_limited" + ? "ETIMEDOUT" + : e.kind === "aborted" + ? "ABORT_ERR" + : "EHOSTDOWN"; + return Object.assign(new Error(`Alist：${e.message}${path ? `（${path}）` : ""}`), { code, alist: e.kind }); + } + if (e && typeof e === "object" && "code" in e) return e as Error & { code: string }; + return Object.assign(new Error(String((e as Error)?.message ?? e)), { code: "EIO" }); +} + +/** 远端对象的伪设备号:同一存储实例相同,与本机设备号不冲突 */ +export function remoteDev(storageId: string): number { + return 2 ** 40 + Number.parseInt(createHash("sha1").update(storageId).digest("hex").slice(0, 8), 16); +} + +export async function remote(p: string): Promise<{ v: VPath; client: AlistClient }> { + const v = parseVPath(p); + if (!v) throw Object.assign(new Error(`不是远端路径：${p}`), { code: "EINVAL" }); + try { + const e = await storageEntry(v.storageId); + return { v, client: e.client }; + } catch (err) { + throw remoteError(err, p); + } +} + +const statsOf = (storageId: string, o: { isDir: boolean; size: number; modified: number }): VStats => ({ + size: o.isDir ? 0 : o.size, + mtimeMs: o.modified, + ino: 0, + dev: remoteDev(storageId), + nlink: 1, + isFile: () => !o.isDir, + isDirectory: () => o.isDir, + isSymbolicLink: () => false, +}); + +export const factsOf = (s: VStats): FileFacts => ({ + size: Number(s.size), + mtimeMs: Number(s.mtimeMs), + ino: Number(s.ino), + dev: Number(s.dev), + nlink: Number(s.nlink), + isFile: s.isFile(), + isDirectory: s.isDirectory(), + isSymbolicLink: s.isSymbolicLink(), +}); + +// 远端 stat 短缓存:列目录拿到的子项 stat 在几秒内复用,避免遍历时逐个 get(§9-8 限制请求量);写操作按目录失效 +const STAT_TTL_MS = 3000; +const statCache = new Map(); +function cacheStat(p: string, stats: VStats) { + statCache.set(p, { at: Date.now(), stats }); + if (statCache.size > 20_000) statCache.delete(statCache.keys().next().value!); +} +/** 写后失效:该路径、其子树与父目录 */ +export function invalidateRemote(p: string): void { + const parent = posix.dirname(p); + for (const k of statCache.keys()) if (k === p || k.startsWith(`${p}/`) || k === parent) statCache.delete(k); +} + +export async function vlstat(p: string, opts: { refresh?: boolean } = {}): Promise { + if (!parseVPath(p)) return (await lstat(p)) as unknown as VStats; + if (!opts.refresh) { + const hit = statCache.get(p); + if (hit && Date.now() - hit.at < STAT_TTL_MS) return hit.stats; + } + const { v, client } = await remote(p); + try { + const st = statsOf(v.storageId, await client.get(v.remote, opts)); + cacheStat(p, st); + return st; + } catch (e) { + throw remoteError(e, p); + } +} + +/** 远端没有符号链接:realpath = 规范化后的路径(存在才返回) */ +export async function vrealpath(p: string): Promise { + const v = parseVPath(p); + if (!v) return realpath(p); + await vlstat(p); + return toVPath(v.storageId, v.remote); +} + +export interface VDirent { + name: string; + stats: VStats; +} + +/** 列目录(带每项的 stat;远端一次 list 拿全) */ +export async function vreaddirStats(p: string, opts: { refresh?: boolean } = {}): Promise { + const v = parseVPath(p); + if (!v) { + const names = await readdir(p); + const out: VDirent[] = []; + for (const name of names) { + const st = await lstat(join(p, name)).catch(() => null); + if (st) out.push({ name, stats: st as unknown as VStats }); + } + return out; + } + const { client } = await remote(p); + try { + return (await client.list(v.remote, opts)).map((o) => { + const stats = statsOf(v.storageId, o); + cacheStat(join(p, o.name), stats); + return { name: o.name, stats }; + }); + } catch (e) { + throw remoteError(e, p); + } +} + +export interface VDirentLite { + name: string; + isDirectory(): boolean; + isFile(): boolean; + isSymbolicLink(): boolean; +} + +/** readdir(withFileTypes) 的统一版本;本地照旧 */ +export async function vreaddirents(p: string): Promise { + if (!parseVPath(p)) return readdir(p, { withFileTypes: true }); + return (await vreaddirStats(p)).map((d) => ({ + name: d.name, + isDirectory: () => d.stats.isDirectory(), + isFile: () => d.stats.isFile(), + isSymbolicLink: () => false, + })); +} + +export async function vreaddir(p: string): Promise { + if (!parseVPath(p)) return readdir(p); + return (await vreaddirStats(p)).map((d) => d.name).sort(); +} + +/** 同 adapters statSafe 的口径;远端失败(连不上 / 超时 / 鉴权)返回 error,只有明确不存在才是 missing */ +export async function vstatSafe(p: string, opts: { root?: string; sentinel?: string } = {}): Promise { + if (!parseVPath(p)) return statSafe(p, opts); + if (opts.root) { + const h = await vcheckRoot(opts.root, opts.sentinel); + if (h.kind === "offline") return { kind: "root_offline", root: opts.root, reason: h.reason }; + } + try { + return { kind: "ok", facts: factsOf(await vlstat(p)) }; + } catch (e) { + const err = remoteError(e, p); + if (err.code === "ENOENT") return { kind: "missing" }; + if (err.code === "EACCES") return { kind: "permission_denied" }; + return { kind: "error", code: err.code === "ETIMEDOUT" ? "TIMEOUT" : "IO", message: err.message }; + } +} + +/** 根在线:本地按哨兵;远端 = 根路径存在且是目录(Alist 不用哨兵,同 MP) */ +export async function vcheckRoot(root: string, sentinel: string = DEFAULT_ROOT_SENTINEL): Promise { + const v = parseVPath(root); + if (!v) return checkRoot(root, sentinel); + try { + const st = await vlstat(root); + if (!st.isDirectory()) return { kind: "offline", reason: "root_missing" }; + return { kind: "online", dev: remoteDev(v.storageId) }; + } catch (e) { + const err = remoteError(e, root); + if (err.code === "ENOENT") return { kind: "offline", reason: "root_missing" }; + if (err.code === "EACCES") return { kind: "offline", reason: "permission_denied", code: "EACCES" }; + if (err.code === "ETIMEDOUT") return { kind: "offline", reason: "timeout" }; + return { kind: "offline", reason: "error", code: err.code }; + } +} + +export async function vmkdirp(p: string): Promise { + const v = parseVPath(p); + if (!v) { + await mkdir(p, { recursive: true }); + return; + } + const { client } = await remote(p); + invalidateRemote(p); + await client.mkdir(v.remote).catch((e) => { + throw remoteError(e, p); + }); +} + +export async function vexists(p: string): Promise { + try { + await vlstat(p); + return true; + } catch (e) { + if ((e as { code?: string }).code === "ENOENT") return false; + throw e; + } +} + +export async function vreadFile(p: string, maxBytes = 4 * 1024 * 1024): Promise { + const v = parseVPath(p); + if (!v) return new Uint8Array(await readFile(p)); + const { client } = await remote(p); + try { + return await client.readFile(v.remote, maxBytes); + } catch (e) { + throw remoteError(e, p); + } +} + +/** 本机临时目录(远端写入的本地中转) */ +export function relayDir(): string { + return join(storageDataDir() ?? Deno.env.get("DATA_DIR") ?? tmpdir(), "cache", "storage-relay"); +} + +/** + * 原子写:本地同目录临时文件 → rename;远端本地生成后上传(同 MP 刮削 `_save_file`): + * 上传到旁边的临时名,成功后删旧文件(若有)再改名。 + */ +export async function vwriteFileAtomic(p: string, content: Uint8Array | string): Promise { + const v = parseVPath(p); + if (!v) { + await mkdir(dirname(p), { recursive: true }); + const tmp = `${p}.${randomUUID().slice(0, 8)}.tmp`; + try { + await writeFile(tmp, content); + await rename(tmp, p); + } catch (e) { + await rm(tmp, { force: true }).catch(() => {}); + throw e; + } + return; + } + const { client } = await remote(p); + invalidateRemote(p); + const dir = relayDir(); + await mkdir(dir, { recursive: true }); + const local = join(dir, `${randomUUID()}-${basename(p)}`); + const stage = `${v.remote}.umt-new`; + try { + await writeFile(local, content); + if (await client.stat(stage)) await client.removePath(stage); + await client.upload(local, stage); + if (await client.stat(v.remote)) await client.removePath(v.remote); + await client.rename(stage, posix.basename(v.remote)); + } catch (e) { + await client.removePath(stage).catch(() => undefined); + throw remoteError(e, p); + } finally { + await rm(local, { force: true }).catch(() => {}); + } +} + +/** 删除文件或目录(远端 remove;本地 rm -r) */ +export async function vremove(p: string): Promise { + const v = parseVPath(p); + if (!v) { + await rm(p, { recursive: true, force: true }); + return; + } + const { client } = await remote(p); + invalidateRemote(p); + await client.removePath(v.remote).catch((e) => { + throw remoteError(e, p); + }); +} + +/** 同存储内移动 / 改名(远端用 API move,目标已存在 EEXIST;本地 rename 不覆盖由调用方保证) */ +export async function vmove(src: string, dst: string): Promise { + const a = parseVPath(src); + const b = parseVPath(dst); + if (!a && !b) { + await mkdir(dirname(dst), { recursive: true }); + await rename(src, dst); + return; + } + if (!a || !b || a.storageId !== b.storageId) + throw Object.assign(new Error(`跨存储不能直接移动：${src} → ${dst}`), { code: "EXDEV" }); + const { client } = await remote(src); + invalidateRemote(src); + invalidateRemote(dst); + await client.moveFile(a.remote, b.remote).catch((e) => { + throw remoteError(e, src); + }); +} + +/** 远端直链(ffprobe 用;日志前要脱敏) */ +export async function vrawUrl(p: string): Promise { + const { v, client } = await remote(p); + return client.rawUrl(v.remote).catch((e) => { + throw remoteError(e, p); + }); +} + +/** 同目录不覆盖改名(远端 Alist rename 本身拒绝同名;本地走 adapters moveSafe) */ +export async function vrenameNoClobber(from: string, to: string): Promise<{ status: "moved" | "exists" | "exdev" }> { + const a = parseVPath(from); + if (!a) { + const { moveSafe } = await import("@umt/adapters"); + return moveSafe(from, to); + } + const b = parseVPath(to); + if (!b || b.storageId !== a.storageId) return { status: "exdev" }; + const { client } = await remote(from); + invalidateRemote(from); + invalidateRemote(to); + try { + if (await client.stat(b.remote)) return { status: "exists" }; + if (posix.dirname(a.remote) === posix.dirname(b.remote)) await client.rename(a.remote, posix.basename(b.remote)); + else await client.moveFile(a.remote, b.remote); + return { status: "moved" }; + } catch (e) { + if (e instanceof AlistError && e.kind === "conflict") return { status: "exists" }; + throw remoteError(e, from); + } +} diff --git a/apps/server/src/storage/vpath.ts b/apps/server/src/storage/vpath.ts new file mode 100644 index 00000000..c4a151ec --- /dev/null +++ b/apps/server/src/storage/vpath.ts @@ -0,0 +1,45 @@ +import { posix } from "node:path"; + +// 远端路径的虚拟前缀(docs/design/storage-alist.md §0、§3):/@alist/<存储 id>/。 +// 它是合法的 POSIX 绝对路径,join / dirname / relative 照常可用;本机没有 /@alist, +// 不认识前缀的旧代码对它做本地 IO 只会得到 ENOENT(失败安全)。 + +export const VROOT = "/@alist"; +const RE = /^\/@alist\/([0-9a-f-]{36})(\/.*)?$/i; + +export interface VPath { + storageId: string; + /** Alist 内绝对路径(以 / 开头,无尾斜杠) */ + remote: string; +} + +export const isRemotePath = (p: string | null | undefined): boolean => + !!p && (p === VROOT || p.startsWith(`${VROOT}/`)); + +export function parseVPath(p: string): VPath | null { + if (!isRemotePath(p)) return null; + const m = RE.exec(p.replace(/\/+$/, "")); + if (!m) return null; + const remote = posix.normalize(m[2] ?? "/"); + if (remote.split("/").includes("..")) return null; + return { storageId: m[1]!.toLowerCase(), remote: remote === "/" ? "/" : remote.replace(/\/+$/, "") }; +} + +export function toVPath(storageId: string, remote: string): string { + const r = posix.normalize(`/${remote}`).replace(/\/+$/, ""); + return `${VROOT}/${storageId}${r === "" || r === "/" ? "" : r}`; +} + +/** 同一存储实例? */ +export function sameStorage(a: string, b: string): boolean { + const pa = parseVPath(a); + const pb = parseVPath(b); + return !!pa && !!pb && pa.storageId === pb.storageId; +} + +/** 界面 / 日志展示:「存储名:/路径」;本地原样 */ +export function displayPath(p: string, names: ReadonlyMap = new Map()): string { + const v = parseVPath(p); + if (!v) return p; + return `${names.get(v.storageId) ?? "Alist"}:${v.remote}`; +} diff --git a/apps/server/test/auth-settings/storage-instances.test.ts b/apps/server/test/auth-settings/storage-instances.test.ts new file mode 100644 index 00000000..3e310d9c --- /dev/null +++ b/apps/server/test/auth-settings/storage-instances.test.ts @@ -0,0 +1,176 @@ +import { createMockAlist, serveMockAlist } from "@umt/adapters/alist-mock"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { buildHealth } from "../../src/modules/system/health.ts"; +import { clearStorageCache, storageEntry } from "../../src/storage/registry.ts"; +import { DEV_KEY, json } from "../helpers.ts"; +import { bearer, createHarness, type Harness, pgAvailable } from "./harness.ts"; + +// 存储实例(Alist / OpenList)设置接口:只写凭据、测试连接(版本 / 用户 / 根路径 / 容量)、 +// 令牌失效 → offline、被库根引用不能删、健康检查里的存储实例(docs/design/storage-alist.md §3、§4)。 + +const hasPg = await pgAvailable(); + +describe.skipIf(!hasPg)("存储实例(真实 PG + 模拟 Alist HTTP)", () => { + let h: Harness; + let admin: ReturnType; + const mock = createMockAlist({ + token: "perm-token-xyz", + username: "alice", + password: "pw-alice", + mountDetails: { "/s1": { total_space: 1000, free_space: 250, used_space: 750 } }, + }); + mock.mkdirp("/s1/media"); + let srv: { base: string; close: () => Promise }; + + beforeAll(async () => { + srv = serveMockAlist(mock); + h = await createHarness({ engine: false }); + admin = bearer(h.app, DEV_KEY); + }, 60_000); + afterAll(async () => { + await h?.close(); + await srv?.close(); + }); + + it("新建(令牌):凭据只写不读;测试连接拿到版本、用户、根路径、容量", async () => { + const c = await admin("POST", "/storage-instances", { + name: "openlist-a", + baseUrl: `${srv.base}/`, + rootPath: "/s1/media/", + authKind: "token", + credential: "perm-token-xyz", + pathPassword: "folder-pw", + }); + expect(c.status).toBe(200); + const row = (await json(c)).data; + expect(row.baseUrl).toBe(srv.base); + expect(row.rootPath).toBe("/s1/media"); + expect(row.secretRef).toBeTruthy(); + expect(row.pathPasswordRef).toBeTruthy(); + expect(JSON.stringify(row)).not.toContain("perm-token-xyz"); + expect(JSON.stringify(row)).not.toContain("folder-pw"); + expect(await h.ctx.secrets.resolve(row.secretRef)).toBe("perm-token-xyz"); + + const t = (await json(await admin("POST", `/storage-instances/${row.id}/test`))).data; + expect(t).toMatchObject({ ok: true, healthStatus: "healthy", user: "alice", rootOk: true }); + expect(t.version).toContain("v4"); + expect(t.usage).toEqual({ total: 1000, free: 250, used: 750 }); + const after = (await json(await admin("GET", `/storage-instances/${row.id}`))).data; + expect(after.healthStatus).toBe("healthy"); + }); + + it("账号密码:缺密码 422;改成错密码后测试 = offline 且信息里没有密码", async () => { + const bad = await admin("POST", "/storage-instances", { + name: "pw-missing", + baseUrl: srv.base, + authKind: "password", + username: "alice", + }); + expect(bad.status).toBe(422); + const c = ( + await json( + await admin("POST", "/storage-instances", { + name: "openlist-pw", + baseUrl: srv.base, + authKind: "password", + username: "alice", + credential: "pw-alice", + }), + ) + ).data; + expect((await json(await admin("POST", `/storage-instances/${c.id}/test`))).data.ok).toBe(true); + await admin("PATCH", `/storage-instances/${c.id}`, { credential: "wrong-secret-pw" }); + const t = (await json(await admin("POST", `/storage-instances/${c.id}/test`))).data; + expect(t).toMatchObject({ ok: false, healthStatus: "offline" }); + expect(t.message).toContain("登录失败"); + expect(t.message).not.toContain("wrong-secret-pw"); + }); + + it("永久令牌失效:测试与健康检查都报不可用", async () => { + const c = ( + await json( + await admin("POST", "/storage-instances", { + name: "openlist-badtoken", + baseUrl: srv.base, + rootPath: "/s1", + credential: "revoked-token", + }), + ) + ).data; + const t = (await json(await admin("POST", `/storage-instances/${c.id}/test`))).data; + expect(t.ok).toBe(false); + expect(t.message).toContain("令牌失效"); + clearStorageCache(); + const health = await buildHealth({ + db: h.ctx.db, + engineSchema: "openworkflow", + dataDir: "/tmp", + self: { version: "t", startedAt: new Date(), sseConnections: 0 }, + thresholds: { warnPercent: 85, criticalPercent: 95 }, + }); + const m = health.mounts.find((x) => x.kind === "storage_instance" && x.id === c.id); + expect(m).toMatchObject({ status: "offline", reason: "permission_denied" }); + const ok = health.mounts.find((x) => x.kind === "storage_instance" && x.name === "openlist-a"); + expect(ok?.status).toBe("online"); + }); + + it("改凭据后注册表重建客户端;清除路径密码", async () => { + const row = (await json(await admin("GET", "/storage-instances?limit=50"))).data.items.find( + (x: { name: string }) => x.name === "openlist-a", + ); + const e1 = await storageEntry(row.id); + expect(e1.secrets).toContain("folder-pw"); + const r = (await json(await admin("PATCH", `/storage-instances/${row.id}`, { pathPassword: null }))).data; + expect(r.pathPasswordRef).toBeNull(); + const e2 = await storageEntry(row.id); + expect(e2.client).not.toBe(e1.client); + expect(e2.secrets).not.toContain("folder-pw"); + }); + + it("被媒体库目录引用时不能删;解除后可删,凭据一并删除", async () => { + const c = ( + await json( + await admin("POST", "/storage-instances", { + name: "to-delete", + baseUrl: srv.base, + rootPath: "/s1", + credential: "perm-token-xyz", + }), + ) + ).data; + const lib = await admin("POST", "/library-roots", { + name: "Alist 电影", + localPath: `/@alist/${c.id}/s1/media`, + mediaKind: "movie", + }); + expect(lib.status).toBe(200); + const libRow = (await json(lib)).data; + expect(libRow.sentinelFile).toBeNull(); + const missing = await admin("POST", "/library-roots", { + name: "不存在的目录", + localPath: `/@alist/${c.id}/s1/nope`, + }); + expect(missing.status).toBe(412); + const blocked = await admin("DELETE", `/storage-instances/${c.id}`); + expect(blocked.status).toBe(409); + const test = (await json(await admin("POST", `/library-roots/${libRow.id}/test`))).data; + expect(test.ok).toBe(true); + await admin("DELETE", `/library-roots/${libRow.id}`); + const del = await admin("DELETE", `/storage-instances/${c.id}`); + expect(del.status).toBe(200); + await expect(h.ctx.secrets.resolve(c.secretRef)).rejects.toThrow(); + const audits = await h.ctx.db + .selectFrom("audit_event") + .select("action") + .where("action", "like", "storage_instance.%") + .execute(); + expect(new Set(audits.map((a) => a.action))).toEqual( + new Set([ + "storage_instance.create", + "storage_instance.update", + "storage_instance.test", + "storage_instance.delete", + ]), + ); + }); +}); diff --git a/packages/adapters/src/fs/ffprobe.ts b/packages/adapters/src/fs/ffprobe.ts index 43f299e7..b14f6afd 100644 --- a/packages/adapters/src/fs/ffprobe.ts +++ b/packages/adapters/src/fs/ffprobe.ts @@ -49,6 +49,8 @@ export interface ProbeOptions { /** ffprobe 可执行文件,默认 "ffprobe"(PATH 查找) */ bin?: string; timeoutMs?: number; + /** 输入前的附加参数(远端直链用 -rw_timeout 等) */ + inputArgs?: string[]; } const optNum = (v: unknown): number | undefined => { @@ -78,7 +80,7 @@ export function ffprobe(path: string, opts: ProbeOptions = {}): Promise { execFile( opts.bin ?? "ffprobe", - ["-v", "error", "-print_format", "json", "-show_format", "-show_streams", "--", path], + ["-v", "error", ...(opts.inputArgs ?? []), "-print_format", "json", "-show_format", "-show_streams", "--", path], { timeout: opts.timeoutMs ?? 30_000, maxBuffer: 16 * 1024 * 1024 }, (err, stdout, stderr) => { if (err) { diff --git a/packages/contracts/src/api/instances.ts b/packages/contracts/src/api/instances.ts index d079c192..579649c4 100644 --- a/packages/contracts/src/api/instances.ts +++ b/packages/contracts/src/api/instances.ts @@ -1,6 +1,7 @@ import { z } from "zod"; import { IdParam } from "../common.ts"; import * as instances from "../instances.ts"; +import * as storage from "../storage.ts"; import { crud, del, ep, instanceTest, R } from "./shared.ts"; export const instancesApi = { @@ -54,6 +55,27 @@ export const instancesApi = { test: instanceTest("/media-server-instances/{id}/test", "测试媒体服务器连接"), remove: del("/media-server-instances/{id}", "删除媒体服务器实例"), }, + storageInstances: { + ...crud( + "/storage-instances", + "存储实例", + { + row: storage.StorageInstance, + create: storage.StorageInstanceCreate, + update: storage.StorageInstanceUpdate, + }, + R, + ), + test: ep({ + method: "post", + path: "/storage-instances/{id}/test", + scope: "admin", + summary: "测试 Alist / OpenList 连接（版本、登录用户、根路径、容量）", + params: IdParam, + response: storage.StorageTestResult, + }), + remove: del("/storage-instances/{id}", "删除存储实例"), + }, pathMappings: { ...crud( "/path-mappings", diff --git a/packages/contracts/src/auth.ts b/packages/contracts/src/auth.ts index 4466c91e..2bdbcc3b 100644 --- a/packages/contracts/src/auth.ts +++ b/packages/contracts/src/auth.ts @@ -225,6 +225,10 @@ export const AuditAction = z.enum([ "media_server_instance.update", "media_server_instance.delete", "media_server_instance.test", + "storage_instance.create", + "storage_instance.update", + "storage_instance.delete", + "storage_instance.test", "path_mapping.create", "path_mapping.update", "path_mapping.delete", diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index b0733398..e5e43e13 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -31,5 +31,6 @@ export * from "./redact.ts"; export * from "./scrape.ts"; export * from "./settings.ts"; export * from "./sources.ts"; +export * from "./storage.ts"; export * from "./subtitles.ts"; export * from "./system-resources.ts"; diff --git a/packages/contracts/src/ops.ts b/packages/contracts/src/ops.ts index b703f4dd..b09f7844 100644 --- a/packages/contracts/src/ops.ts +++ b/packages/contracts/src/ops.ts @@ -168,6 +168,7 @@ export const SecretPurpose = z.enum([ "media_server_api_key", "metadata_api_key", "llm_api_key", + "storage_credential", "other", ]); export type SecretPurpose = z.infer; @@ -448,7 +449,8 @@ export const SystemDisk = z.object({ export type SystemDisk = z.infer; export const SystemMount = z.object({ - kind: z.enum(["library_root", "path_mapping"]), + /** storage_instance = Alist / OpenList 实例(连通 + 根路径可访问;令牌失效为 offline/auth) */ + kind: z.enum(["library_root", "path_mapping", "storage_instance"]), id: z.string(), name: z.string(), path: z.string(), diff --git a/packages/contracts/src/storage.ts b/packages/contracts/src/storage.ts new file mode 100644 index 00000000..39950692 --- /dev/null +++ b/packages/contracts/src/storage.ts @@ -0,0 +1,121 @@ +import { z } from "zod"; +import { EntityBase, Id, JsonObject, Timestamp, WriteOnlySecret } from "./common.ts"; +import { InstanceHealth } from "./instances.ts"; + +// Alist / OpenList 存储实例(docs/design/storage-alist.md §3)。凭据(令牌 / 密码)与路径密码只写不读。 + +export const StorageKind = z.enum(["alist"]); +export type StorageKind = z.infer; +/** 只用于展示与版本差异;接口相同(MP 的 AlistGo 是同实现的子类) */ +export const StorageFlavor = z.enum(["alist", "openlist"]); +export type StorageFlavor = z.infer; +/** token 永久令牌(Alist 后台「设置 → 其他 → 令牌」);password 账号密码换临时令牌;guest 访客 */ +export const StorageAuthKind = z.enum(["token", "password", "guest"]); +export type StorageAuthKind = z.infer; + +/** 实例高级选项;不填用缺省(括号内) */ +export const StorageOptions = z + .object({ + /** 每秒请求数(5);0 不限 */ + qps: z.number().int().min(0).max(100), + /** 并发(4);0 不限 */ + concurrency: z.number().int().min(0).max(32), + /** 普通请求超时秒(30) */ + timeoutSec: z.number().int().min(5).max(600), + /** 上传 / 下载无进度空闲超时秒(120) */ + idleTimeoutSec: z.number().int().min(10).max(3600), + /** 等待 Alist 复制 / 移动 / 上传任务的上限分钟(360) */ + taskTimeoutMin: z.number().int().min(1).max(10080), + /** 可重试错误的重试次数(3) */ + retries: z.number().int().min(0).max(10), + /** 上传用 As-Task(false,同 MP) */ + uploadAsTask: z.boolean(), + /** 目录监控最大深度(5,同 MP) */ + snapshotMaxDepth: z.number().int().min(1).max(20), + /** 目录监控按目录修改时间跳过未变子树(true) */ + snapshotCheckFolderModtime: z.boolean(), + }) + .partial(); +export type StorageOptions = z.infer; + +export const STORAGE_OPTION_DEFAULTS: Required = { + qps: 5, + concurrency: 4, + timeoutSec: 30, + idleTimeoutSec: 120, + taskTimeoutMin: 360, + retries: 3, + uploadAsTask: false, + snapshotMaxDepth: 5, + snapshotCheckFolderModtime: true, +}; + +/** Alist 内路径:以 / 开头,不含 .. 段 */ +export const AlistPath = z + .string() + .max(4096) + .refine((p) => p.startsWith("/") && !p.includes("\0") && !p.split("/").some((s) => s === ".." || s === "."), { + message: "Alist 路径必须以 / 开头，不能含 . 或 ..", + }); + +export const StorageInstance = EntityBase.extend({ + name: z.string().min(1).max(100), + kind: StorageKind, + flavor: StorageFlavor, + baseUrl: z.url(), + /** 本系统可浏览 / 使用的 Alist 根路径(缺省 /) */ + rootPath: AlistPath, + authKind: StorageAuthKind, + username: z.string().nullable(), + secretRef: Id.nullable(), + pathPasswordRef: Id.nullable(), + enabled: z.boolean(), + options: StorageOptions, + healthStatus: InstanceHealth, + lastHealthAt: Timestamp.nullable(), + healthDetail: JsonObject, + versionInfo: JsonObject, +}); +export type StorageInstance = z.infer; + +export const StorageInstanceCreate = StorageInstance.pick({ + name: true, + flavor: true, + baseUrl: true, + rootPath: true, + authKind: true, + username: true, + enabled: true, + options: true, +}) + .partial({ flavor: true, rootPath: true, authKind: true, username: true, enabled: true, options: true }) + .extend({ + /** 令牌(authKind=token)或密码(authKind=password);只写 */ + credential: WriteOnlySecret(4096).optional(), + /** 路径密码;只写 */ + pathPassword: WriteOnlySecret(1024).optional(), + }); +export type StorageInstanceCreate = z.infer; +/** 更新:credential / pathPassword 省略或空串 = 不变,null = 清除 */ +export const StorageInstanceUpdate = StorageInstanceCreate.partial().extend({ + credential: WriteOnlySecret(4096).nullable().optional(), + pathPassword: WriteOnlySecret(1024).nullable().optional(), +}); +export type StorageInstanceUpdate = z.infer; + +export const StorageUsage = z.object({ total: z.number(), free: z.number(), used: z.number() }); +export type StorageUsage = z.infer; + +export const StorageTestResult = z.object({ + ok: z.boolean(), + healthStatus: InstanceHealth, + version: z.string().nullable(), + /** 登录用户(访客为 guest) */ + user: z.string().nullable(), + /** 根路径存在且是目录 */ + rootOk: z.boolean(), + /** 容量;驱动不给或令牌不是管理员时为 null(显示「未知」) */ + usage: StorageUsage.nullable(), + message: z.string().nullable(), +}); +export type StorageTestResult = z.infer; diff --git a/packages/db/migrations/0090_storage_instance.ts b/packages/db/migrations/0090_storage_instance.ts new file mode 100644 index 00000000..9f326395 --- /dev/null +++ b/packages/db/migrations/0090_storage_instance.ts @@ -0,0 +1,62 @@ +import type { Kysely } from "kysely"; +import { entityTable, execDdl, oneOf } from "../src/migration-helpers.ts"; + +// Alist / OpenList 存储实例(docs/design/storage-alist.md §3):远端路径用虚拟前缀 /@alist//<路径> 表示, +// 库根 local_path、整理记录、资产位置里都直接存这个路径,不需要新列。凭据走 secret 表(purpose storage_credential)。 + +const PURPOSE_OLD = [ + "site_cookie", + "site_passkey", + "site_api_key", + "site_password", + "feed_url", + "downloader_password", + "media_server_api_key", + "metadata_api_key", + "llm_api_key", + "other", +]; +const PURPOSE_NEW = [...PURPOSE_OLD.slice(0, -1), "storage_credential", "other"]; +const HEALTH = ["unknown", "healthy", "degraded", "offline"]; + +const swapPurpose = (values: readonly string[]) => ` +alter table secret drop constraint if exists secret_purpose_check; +alter table secret add constraint secret_purpose_check ${oneOf("purpose", values)}; +`; + +// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型 +export async function up(db: Kysely): Promise { + await execDdl(db, swapPurpose(PURPOSE_NEW)); + await execDdl( + db, + entityTable( + "storage_instance", + ` + name text not null unique, + kind text not null default 'alist' ${oneOf("kind", ["alist"])}, + flavor text not null default 'openlist' ${oneOf("flavor", ["alist", "openlist"])}, + base_url text not null, + root_path text not null default '/' check (root_path like '/%'), + auth_kind text not null default 'token' ${oneOf("auth_kind", ["token", "password", "guest"])}, + username text, + secret_ref uuid references secret(id) on delete set null, + path_password_ref uuid references secret(id) on delete set null, + enabled boolean not null default true, + options jsonb not null default '{}'::jsonb, + health_status text not null default 'unknown' ${oneOf("health_status", HEALTH)}, + last_health_at timestamptz, + health_detail jsonb not null default '{}'::jsonb, + version_info jsonb not null default '{}'::jsonb`, + ), + ); +} + +// biome-ignore lint/suspicious/noExplicitAny: 同上 +export async function down(db: Kysely): Promise { + await execDdl( + db, + `drop table if exists storage_instance cascade; + delete from secret where purpose = 'storage_credential'; + ${swapPurpose(PURPOSE_OLD)}`, + ); +} diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index 3bbf065e..e6608c7c 100644 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -246,6 +246,7 @@ export type SecretPurpose = | "media_server_api_key" | "metadata_api_key" | "llm_api_key" + | "storage_credential" | "other"; export type ServerMappingItemType = "movie" | "series" | "season" | "episode" | "other"; export type ServerMappingMatchStatus = "matched" | "ambiguous" | "unmatched" | "stale"; @@ -260,6 +261,10 @@ export type SiteSnapshotKind = "periodic" | "backfill" | "live_search" | "manual export type SiteSnapshotStatus = "running" | "ok" | "partial" | "failed"; export type SourceReleaseResolutionStatus = "unresolved" | "resolved" | "ambiguous" | "unidentifiable" | "rejected"; export type SourceReleaseAvailability = "active" | "site_disabled" | "expired" | "unknown"; +export type StorageInstanceKind = "alist"; +export type StorageInstanceFlavor = "alist" | "openlist"; +export type StorageInstanceAuthKind = "token" | "password" | "guest"; +export type StorageInstanceHealthStatus = "unknown" | "healthy" | "degraded" | "offline"; export type SubscriptionMediaKind = "movie" | "series" | "variety"; export type SubscriptionMode = "once" | "continuous"; export type SubscriptionSeasonScope = "current_season" | "following_seasons"; @@ -1635,6 +1640,27 @@ export interface SseEventTable { payload: Generated; } +export interface StorageInstanceTable { + id: Generated; + name: string; + kind: Generated; + flavor: Generated; + base_url: string; + root_path: Generated; + auth_kind: Generated; + username: string | null; + secret_ref: string | null; + path_password_ref: string | null; + enabled: Generated; + options: Generated; + health_status: Generated; + last_health_at: Timestamptz | null; + health_detail: Generated; + version_info: Generated; + created_at: GeneratedTimestamptz; + updated_at: GeneratedTimestamptz; +} + export interface SubscriptionTable { id: Generated; media_entity_id: string | null; @@ -1869,6 +1895,7 @@ export interface DB { slot_unit: SlotUnitTable; source_release: SourceReleaseTable; sse_event: SseEventTable; + storage_instance: StorageInstanceTable; subscription: SubscriptionTable; subscription_history: SubscriptionHistoryTable; subtitle_write: SubtitleWriteTable; @@ -2115,6 +2142,9 @@ export type SourceReleaseUpdate = Updateable; export type SseEventRow = Selectable; export type NewSseEvent = Insertable; export type SseEventUpdate = Updateable; +export type StorageInstanceRow = Selectable; +export type NewStorageInstance = Insertable; +export type StorageInstanceUpdate = Updateable; export type SubscriptionRow = Selectable; export type NewSubscription = Insertable; export type SubscriptionUpdate = Updateable; From 9d95076a0eea99a175a9d8b385fb344e3e1f9784 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 6 Oct 2026 23:52:39 +0800 Subject: [PATCH 04/11] =?UTF-8?q?feat(files):=20=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E7=AE=A1=E7=90=86=E6=8E=A5=E5=85=A5=20Alist=20=E5=AD=98?= =?UTF-8?q?=E5=82=A8(=E6=B5=8F=E8=A7=88=E3=80=81=E5=A2=9E=E5=88=A0?= =?UTF-8?q?=E6=94=B9=E3=80=81=E7=A7=BB=E5=8A=A8=E5=A4=8D=E5=88=B6=E3=80=81?= =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E4=B8=8B=E8=BD=BD=E3=80=81=E5=9B=9E=E6=94=B6?= =?UTF-8?q?=E7=AB=99)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 文件管理根多出 st:<存储 id>(对照 MP 存储切换),Alist 媒体库目录照常出现在媒体库组 - 列目录一次 list,refresh 只在用户刷新时带;连不上返回 503 不当成空目录 - 新建目录 / 改名(同名不覆盖)/ 删除进 Alist 上的 .umt-trash 并可还原 - 存储内移动复制走 Alist API,本地 ↔ Alist 走整理同一套执行与账本 - 新增 PUT /files/upload(同名不覆盖,Alist 经服务端转传);下载 / 预览流式转发直链 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei --- .../services/organize-guard.ts | 15 +- .../services/organize-transfer.ts | 72 ++++-- .../server/src/modules/file-manager/common.ts | 83 +++++- .../src/modules/file-manager/download.ts | 46 ++++ .../src/modules/file-manager/identify.ts | 4 +- apps/server/src/modules/file-manager/index.ts | 3 + apps/server/src/modules/file-manager/list.ts | 111 +++++++- .../server/src/modules/file-manager/mutate.ts | 23 +- .../src/modules/file-manager/organize.ts | 2 +- .../src/modules/file-manager/preview.ts | 15 ++ apps/server/src/modules/file-manager/probe.ts | 5 +- .../src/modules/file-manager/record-delete.ts | 2 +- .../server/src/modules/file-manager/routes.ts | 8 + .../src/modules/file-manager/transfer.ts | 25 +- apps/server/src/modules/file-manager/trash.ts | 97 ++++++- apps/server/src/modules/file-manager/tree.ts | 18 +- .../server/src/modules/file-manager/upload.ts | 83 ++++++ apps/server/test/file-manager/harness.ts | 11 + apps/server/test/storage/alist-files.test.ts | 240 ++++++++++++++++++ apps/server/test/storage/alist.ts | 66 +++++ apps/web/app/components/files/mock.ts | 4 + apps/web/app/components/files/model.ts | 6 +- packages/contracts/src/api/organize.ts | 8 + packages/contracts/src/files.ts | 26 +- 24 files changed, 901 insertions(+), 72 deletions(-) create mode 100644 apps/server/src/modules/file-manager/upload.ts create mode 100644 apps/server/test/storage/alist-files.test.ts create mode 100644 apps/server/test/storage/alist.ts diff --git a/apps/server/src/modules/acquisition-organize/services/organize-guard.ts b/apps/server/src/modules/acquisition-organize/services/organize-guard.ts index ad3d0f3b..253c7ca2 100644 --- a/apps/server/src/modules/acquisition-organize/services/organize-guard.ts +++ b/apps/server/src/modules/acquisition-organize/services/organize-guard.ts @@ -1,6 +1,8 @@ import { realpath } from "node:fs/promises"; import { isAbsolute, resolve, sep } from "node:path"; import type { Principal } from "../../../http/auth.ts"; +import { isRemotePath, vrealpath } from "../../../storage/vfs.ts"; +import { toVPath } from "../../../storage/vpath.ts"; import { canonicalJson } from "../util.ts"; import { type AoDeps, type Exec, hostPathOf, loadInstances, upsertDecision } from "./common.ts"; import { hnrSatisfied } from "./retention.ts"; @@ -41,8 +43,17 @@ export async function allowedSourceRoots(exec: Exec): Promise { for (const m of t.mappings) roots.add(m.hostPath); if (t.mappings.length === 0 && t.localSavePath) roots.add(t.localSavePath); } + // Alist 存储:实例根路径(文件管理 st: 根、整理来源可选) + for (const s of await exec + .selectFrom("storage_instance") + .select(["id", "root_path"]) + .where("enabled", "=", true) + .execute()) + roots.add(toVPath(s.id, s.root_path)); const out: string[] = []; - for (const r of roots) if (r && isAbsolute(r)) out.push(await real(r.replace(/\/+$/, "") || "/")); + for (const r of roots) + if (r && isAbsolute(r)) + out.push(isRemotePath(r) ? r.replace(/\/+$/, "") : await real(r.replace(/\/+$/, "") || "/")); // 根目录本身("/")不能作为白名单,否则等于不设防 return [...new Set(out)].filter((r) => r !== "/"); } @@ -62,7 +73,7 @@ export async function sourceGuard(exec: Exec): Promise { if (p.split("/").some((seg) => seg === "..")) throw new PathGuardError(`源路径不允许包含 "..": ${p}`, p); let r: string; try { - r = await realpath(p); + r = isRemotePath(p) ? await vrealpath(p) : await realpath(p); } catch { throw new PathGuardError(`源路径不存在或不可访问：${p}`, p); } diff --git a/apps/server/src/modules/acquisition-organize/services/organize-transfer.ts b/apps/server/src/modules/acquisition-organize/services/organize-transfer.ts index cd5c9318..63219a33 100644 --- a/apps/server/src/modules/acquisition-organize/services/organize-transfer.ts +++ b/apps/server/src/modules/acquisition-organize/services/organize-transfer.ts @@ -1,9 +1,17 @@ import { randomUUID } from "node:crypto"; import { readdir, rmdir } from "node:fs/promises"; import { dirname, join, relative } from "node:path"; -import { checkRoot, DEFAULT_ROOT_SENTINEL, statSafe } from "@umt/adapters"; +import { DEFAULT_ROOT_SENTINEL } from "@umt/adapters"; import { fileKindOf, type OrganizeFileKind, type OrganizePlan, type PlannedFile } from "@umt/domain"; import { organizationStatusFromFiles, organizePlanToFileResults } from "@umt/mapping"; +import { + isRemotePath, + vstatSafe as statSafe, + vcheckRoot, + vreaddir, + vreaddirStats, + vremove, +} from "../../../storage/vfs.ts"; import { hashJson, jsonb } from "../util.ts"; import type { AoDeps, Exec } from "./common.ts"; import { type ExecuteResult, emitRecord, type FrozenPlan, type ImportRequest, type PlanResult } from "./organize.ts"; @@ -27,7 +35,8 @@ export interface TransferItem { export interface TransferSpec { items: TransferItem[]; - targetLibraryRootId: string; + /** 目标库根;目标是 Alist 存储根(文件管理 st:)时为 null */ + targetLibraryRootId: string | null; /** 目标库根 realpath(相对路径以它为基准) */ targetRootPath: string; /** 遍历到的每个文件都经它复核(realpath 在允许的根下) */ @@ -36,7 +45,7 @@ export interface TransferSpec { export interface FrozenTransfer { items: (TransferItem & { isDir: boolean; files: number })[]; - targetLibraryRootId: string; + targetLibraryRootId: string | null; } const KIND_OF = (p: string): OrganizeFileKind => { @@ -50,6 +59,18 @@ async function walkTree( out: { files: string[]; dirs: string[]; links: string[] }, limit: number, ): Promise { + if (isRemotePath(dir)) { + // Alist:一次 list 拿到子项与类型,没有符号链接 + for (const it of (await vreaddirStats(dir)).sort((a, b) => (a.name < b.name ? -1 : 1))) { + if (out.files.length >= limit) return true; + const p = join(dir, it.name); + if (it.stats.isDirectory()) { + out.dirs.push(p); + if (await walkTree(p, out, limit)) return true; + } else out.files.push(p); + } + return false; + } const names = (await readdir(dir)).sort(); for (const n of names) { if (out.files.length >= limit) return true; @@ -71,14 +92,14 @@ export async function planTransfer(d: AoDeps, req: ImportRequest, t: TransferSpe const dry = req.dryRun === true; const action = req.action ?? "copy"; const failed = (reason: string): PlanResult => ({ recordId: null, status: "failed", reason }); - const root = await db - .selectFrom("library_root") - .selectAll() - .where("id", "=", t.targetLibraryRootId) - .executeTakeFirst(); - if (root?.management_mode !== "managed") return failed("target_root_not_managed"); - const sentinel = root.sentinel_file ?? DEFAULT_ROOT_SENTINEL; - const health = await checkRoot(t.targetRootPath, sentinel); + const lib = t.targetLibraryRootId + ? await db.selectFrom("library_root").selectAll().where("id", "=", t.targetLibraryRootId).executeTakeFirst() + : null; + if (t.targetLibraryRootId && lib?.management_mode !== "managed") return failed("target_root_not_managed"); + if (!t.targetLibraryRootId && !isRemotePath(t.targetRootPath)) return failed("target_root_not_managed"); + const root = { id: lib?.id ?? null }; + const sentinel = isRemotePath(t.targetRootPath) ? "" : (lib?.sentinel_file ?? DEFAULT_ROOT_SENTINEL); + const health = await vcheckRoot(t.targetRootPath, sentinel); if (health.kind === "offline") return failed(`library_root_offline:${health.reason}`); const max = d.ports.options.walkMaxFiles; @@ -148,7 +169,7 @@ export async function planTransfer(d: AoDeps, req: ImportRequest, t: TransferSpe organize: plan, entityId: req.mediaEntityId ?? "", mediaType: "movie", - libraryRootId: root.id, + libraryRootId: root.id ?? "", rootPath: t.targetRootPath, sentinel, targetDirs: [...dirs].sort(), @@ -162,7 +183,7 @@ export async function planTransfer(d: AoDeps, req: ImportRequest, t: TransferSpe }; if (dry) return { recordId: null, status: "queued", reason: null, plan: frozen, ...(truncated ? { truncated } : {}) }; - const drafts = organizePlanToFileResults(plan, { targetLibraryRootId: root.id }); + const drafts = organizePlanToFileResults(plan, { targetLibraryRootId: root.id ?? "" }); const now = d.ports.now(); const rel = (abs: string | null | undefined) => (abs ? relative(t.targetRootPath, abs) : null); const heldSrc = new Set(files.filter((f) => f.action === "hold").map((f) => f.src)); @@ -321,6 +342,15 @@ export async function relocate(trx: Exec, input: RelocateInput): Promise { const out: string[] = []; + if (isRemotePath(dir)) { + const walk = async (p: string) => { + const ents = await vreaddirStats(p).catch(() => []); + for (const e of ents) if (e.stats.isDirectory()) await walk(join(p, e.name)); + out.push(p); + }; + await walk(dir); + return out; + } const walk = async (p: string) => { const ents = await readdir(p, { withFileTypes: true }).catch(() => []); for (const e of ents) if (e.isDirectory()) await walk(join(p, e.name)); @@ -330,6 +360,16 @@ async function dirsDeepFirst(dir: string): Promise { return out; } +/** 只删空目录(Alist 的 remove 会递归删,先确认为空) */ +async function removeEmptyDir(dir: string): Promise { + if (!isRemotePath(dir)) { + await rmdir(dir).catch(() => undefined); + return; + } + const left = await vreaddir(dir).catch(() => null); + if (left && left.length === 0) await vremove(dir).catch(() => undefined); +} + /** 原样搬运的终态:记录状态 + move 的位置跟随/绑定修订 + 删源侧空目录;行锁 fencing,重复调用幂等 */ export async function finalizeTransfer(d: AoDeps, recordId: string): Promise { const db = d.ctx.db; @@ -351,7 +391,7 @@ export async function finalizeTransfer(d: AoDeps, recordId: string): Promise x.isDir && allOk(x))) - for (const dir of await dirsDeepFirst(it.src)) await rmdir(dir).catch(() => undefined); + for (const dir of await dirsDeepFirst(it.src)) await removeEmptyDir(dir); return db.transaction().execute(async (trx) => { const rec = await trx @@ -373,9 +413,9 @@ export async function finalizeTransfer(d: AoDeps, recordId: string): Promise under(it, f.source_path)); - const base = { fromLib: it.srcLibraryRootId, toLib: t.targetLibraryRootId, actor, opKind: "move" as const }; + const base = { fromLib: it.srcLibraryRootId, toLib: t.targetLibraryRootId!, actor, opKind: "move" as const }; if (mine.length > 0 && mine.every((f) => f.status === "succeeded")) { // 整项搬完:按前缀跟随(目录含路径绑定修订) const r = await relocate(trx, { ...base, fromRel: it.srcRel, toRel: it.dstRel, isDir: it.isDir }); diff --git a/apps/server/src/modules/file-manager/common.ts b/apps/server/src/modules/file-manager/common.ts index 5cdf6157..177f963a 100644 --- a/apps/server/src/modules/file-manager/common.ts +++ b/apps/server/src/modules/file-manager/common.ts @@ -1,10 +1,13 @@ import { createHash } from "node:crypto"; import { lstat, realpath, stat } from "node:fs/promises"; import { basename, dirname, join, sep } from "node:path"; -import { checkRoot, DEFAULT_ROOT_SENTINEL, type TmdbClient } from "@umt/adapters"; +import { AlistError, checkRoot, DEFAULT_ROOT_SENTINEL, type TmdbClient } from "@umt/adapters"; import type { ApiErrorCode, FileRootId, FilesErrorReason } from "@umt/contracts"; import type { LibraryRootRow } from "@umt/db"; import type { Principal } from "../../http/auth.ts"; +import { listStorages } from "../../storage/registry.ts"; +import { isRemotePath, remoteError, vcheckRoot, vlstat, vrealpath } from "../../storage/vfs.ts"; +import { parseVPath, toVPath } from "../../storage/vpath.ts"; import type { AoDeps, Exec } from "../acquisition-organize/services/common.ts"; import { loadInstances } from "../acquisition-organize/services/common.ts"; import type { SourcesRuntime } from "../sources-identity/runtime.ts"; @@ -122,13 +125,15 @@ export const isOfflineErr = (e: unknown) => OFFLINE_CODES.has(errnoOf(e) ?? ""); * 处理器里漏出的文件系统错误(检查与使用之间目录被删、挂载掉线等竞态)→ 业务错误,而不是 500: * 离线类 → 503 root_unavailable(失败 ≠ 空目录),ENOENT/ENOTDIR → 404,EACCES/EPERM → 403,其余 errno → 400。 */ -export function fsErrorToFiles(e: unknown): FilesError | null { - if (e instanceof FilesError) return e; +export function fsErrorToFiles(e0: unknown): FilesError | null { + if (e0 instanceof FilesError) return e0; + const e = e0 instanceof AlistError ? remoteError(e0) : e0; const code = errnoOf(e); if (typeof code !== "string" || !/^E[A-Z]+$/.test(code)) return null; const msg = e instanceof Error ? e.message : String(e); if (OFFLINE_CODES.has(code)) return new FilesError("root_unavailable", `读取失败(${code})，根可能已离线：${msg}`); if (code === "ENOENT" || code === "ENOTDIR") return new FilesError("not_found", `路径已不存在(${code})`); + if (code === "EEXIST") return new FilesError("exists", `目标已存在：${msg}`); if (code === "EACCES" || code === "EPERM") return new FilesError("forbidden", `没有访问权限(${code})`); return new FilesError("bad_request", `文件系统错误(${code}): ${msg}`); } @@ -137,7 +142,7 @@ export function fsErrorToFiles(e: unknown): FilesError | null { export interface RootDef { id: FileRootId; - kind: "library" | "download"; + kind: "library" | "download" | "storage"; name: string; hostPath: string; mediaKind: LibraryRootRow["media_kind"] | null; @@ -146,6 +151,9 @@ export interface RootDef { libraryRootId: string | null; instanceId: string | null; sentinel: string | null; + /** 所在 Alist 存储实例(本机为 null);hostPath 是虚拟路径 /@alist//… */ + storageId: string | null; + storageName: string | null; } const norm = (p: string) => p.replace(/\/+$/, "") || "/"; @@ -153,10 +161,14 @@ const norm = (p: string) => p.replace(/\/+$/, "") || "/"; /** 已登记的根:启用的库根 + 下载器路径映射本机前缀(无映射 = 同机,取保存路径);"/" 永不出现 */ export async function loadRootDefs(exec: Exec): Promise { const out: RootDef[] = []; + const storages = await listStorages(exec as never).catch(() => []); + const storageName = new Map(storages.map((s) => [s.id, s.name])); const libs = await exec.selectFrom("library_root").selectAll().where("enabled", "=", true).orderBy("name").execute(); for (const r of libs) { const hostPath = norm(r.local_path); if (hostPath === "/" || !hostPath.startsWith("/")) continue; + const v = parseVPath(hostPath); + if (isRemotePath(hostPath) && !v) continue; out.push({ id: `lib:${r.id}`, kind: "library", @@ -166,7 +178,9 @@ export async function loadRootDefs(exec: Exec): Promise { modifiable: r.management_mode === "managed", libraryRootId: r.id, instanceId: null, - sentinel: r.sentinel_file ?? DEFAULT_ROOT_SENTINEL, + sentinel: v ? null : (r.sentinel_file ?? DEFAULT_ROOT_SENTINEL), + storageId: v?.storageId ?? null, + storageName: v ? (storageName.get(v.storageId) ?? null) : null, }); } for (const t of await loadInstances(exec, { enabledOnly: false })) { @@ -184,9 +198,28 @@ export async function loadRootDefs(exec: Exec): Promise { libraryRootId: null, instanceId: t.inst.id, sentinel: null, + storageId: null, + storageName: null, }); }); } + // Alist 存储(对照 MP 存储切换):整个实例从根路径开始可浏览 + for (const s of storages) { + if (!s.enabled) continue; + out.push({ + id: `st:${s.id}`, + kind: "storage", + name: s.name, + hostPath: toVPath(s.id, s.root_path), + mediaKind: null, + modifiable: true, + libraryRootId: null, + instanceId: null, + sentinel: null, + storageId: s.id, + storageName: s.name, + }); + } return out; } @@ -211,6 +244,11 @@ const withTimeout = (p: Promise, ms: number, fallback: T): Promise => export async function probeRoot(def: RootDef, timeoutMs: number): Promise { const checkedAt = new Date(); const run = async (): Promise => { + if (def.storageId) { + const h = await vcheckRoot(def.hostPath); + if (h.kind === "offline") return { health: "offline", reason: h.reason, checkedAt, real: null }; + return { health: "healthy", reason: null, checkedAt, real: def.hostPath }; + } if (def.sentinel) { const h = await checkRoot(def.hostPath, def.sentinel); if (h.kind === "offline") return { health: "offline", reason: h.reason, checkedAt, real: null }; @@ -283,6 +321,7 @@ export async function resolveIn( const def = opts.def ?? (await rootDef(d.ctx.db, rootId)); const rootReal = opts.rootReal ?? (await onlineRoot(d, def)); const rel = checkRel(relIn); + if (def.storageId) return resolveRemote(def, rootReal, rel, opts.mustExist !== false); if (rel === "") { const lst = await lstat(rootReal).catch(() => null); return { def, rootReal, rel, abs: rootReal, lst, isSymlink: false }; @@ -314,8 +353,42 @@ export async function resolveIn( return { def, rootReal, rel: relOf(rootReal, abs), abs, lst, isSymlink }; } +/** 远端根:没有符号链接,路径即 realpath;lst 是 vfs 的 stat(不存在为 null,连不上抛 root_unavailable) */ +async function resolveRemote(def: RootDef, rootReal: string, rel: string, mustExist: boolean): Promise { + const abs = rel === "" ? rootReal : join(rootReal, rel); + let lst: Resolved["lst"] = null; + try { + lst = (await vlstat(abs)) as unknown as Resolved["lst"]; + } catch (e) { + if (errnoOf(e) !== "ENOENT") throw remoteFilesError(e, def); + } + if (!lst && mustExist) throw new FilesError("not_found", `不存在：${rel}`); + return { def, rootReal, rel, abs, lst, isSymlink: false }; +} + +/** 远端错误 → 业务错误(连不上 / 超时 → 503 根不可用;鉴权 → 403;冲突 → 409) */ +export function remoteFilesError(e: unknown, def?: RootDef): FilesError { + if (e instanceof FilesError) return e; + const code = errnoOf(e); + const msg = e instanceof Error ? e.message : String(e); + if (code === "ENOENT") return new FilesError("not_found", msg); + if (code === "EEXIST") return new FilesError("exists", msg); + if (code === "EACCES") return new FilesError("forbidden", msg); + return new FilesError("root_unavailable", `${def ? `存储 ${def.storageName ?? def.name} ` : ""}不可用：${msg}`); +} + /** 执行阶段复核(TOCTOU):再次 realpath,必须仍在根内 */ export async function recheck(abs: string, rootReal: string): Promise { + if (isRemotePath(abs)) { + try { + const r = await vrealpath(abs); + if (!within(r, rootReal)) throw new FilesError("path_escape", `执行时路径指向根外：${abs}`); + return r; + } catch (e) { + if (errnoOf(e) === "ENOENT") throw new FilesError("plan_stale", `源已不存在：${abs}`); + throw remoteFilesError(e); + } + } let r: string; try { r = await realpath(abs); diff --git a/apps/server/src/modules/file-manager/download.ts b/apps/server/src/modules/file-manager/download.ts index 8ba3dbb3..b9b30bf3 100644 --- a/apps/server/src/modules/file-manager/download.ts +++ b/apps/server/src/modules/file-manager/download.ts @@ -1,6 +1,7 @@ import { basename, extname } from "node:path"; import type { FilesDownloadQuery } from "@umt/contracts"; import type { Principal } from "../../http/auth.ts"; +import { remote, remoteError } from "../../storage/vfs.ts"; import { FilesError, type FmDeps, isAdminSession, recheck, resolveIn } from "./common.ts"; import { audit } from "./ledger.ts"; import { kindOfName } from "./list.ts"; @@ -55,6 +56,7 @@ export async function downloadFile( if (!ALLOWED.has(kind) && !pk) throw new FilesError("forbidden", `只能下载 NFO、字幕、图片、文本和 .torrent，此文件类型为 ${kind}`); const real = await recheck(r.abs, r.rootReal); + if (r.def.storageId) return downloadRemote(d, principal, q, r.rel, real, name, kind, pk); const file = await Deno.open(real, { read: true }); let size: number; try { @@ -86,3 +88,47 @@ export async function downloadFile( }, }); } + +/** Alist 文件:经服务端流式转发(不落本地盘,同一下载上限与类型限制;MP 是先下到临时目录再回传) */ +async function downloadRemote( + d: FmDeps, + principal: Principal | undefined, + q: FilesDownloadQuery, + rel: string, + abs: string, + name: string, + kind: string, + pk: ReturnType, +): Promise { + const { v, client } = await remote(abs); + let size: number; + let res: Response; + try { + const o = await client.get(v.remote); + if (o.isDir) throw new FilesError("bad_request", "只能下载文件，不能下载目录"); + size = o.size; + if (size > d.options.downloadMaxBytes) + throw new FilesError( + "too_large", + `文件 ${size} 字节，超过下载上限 ${d.options.downloadMaxBytes}(FILES_DOWNLOAD_MAX_BYTES)`, + ); + res = await client.openRead(v.remote); + } catch (e) { + if (e instanceof FilesError) throw e; + throw remoteError(e, abs); + } + const ext = extname(name).replace(/^\./, "").toLowerCase(); + const inline = q.inline && (kind === "image" || pk === "image") && ext !== "svg"; + await audit(d, principal, "download", { root: q.root, path: rel, size, inline }); + return new Response(res.body, { + status: 200, + headers: { + "content-type": (ext !== "svg" && MIME[ext]) || "application/octet-stream", + ...(size ? { "content-length": String(size) } : {}), + "content-disposition": disposition(inline ? "inline" : "attachment", name), + "x-content-type-options": "nosniff", + "content-security-policy": "default-src 'none'; sandbox", + "cache-control": "private, no-store", + }, + }); +} diff --git a/apps/server/src/modules/file-manager/identify.ts b/apps/server/src/modules/file-manager/identify.ts index c80e25db..186d4baf 100644 --- a/apps/server/src/modules/file-manager/identify.ts +++ b/apps/server/src/modules/file-manager/identify.ts @@ -1,5 +1,4 @@ import { randomUUID } from "node:crypto"; -import { readdir, stat } from "node:fs/promises"; import { join } from "node:path"; import { EventType, @@ -21,6 +20,7 @@ import { } from "@umt/domain"; import { localIdentityCandidate } from "@umt/mapping"; import type { Principal } from "../../http/auth.ts"; +import { vlstat as stat, vreaddirents } from "../../storage/vfs.ts"; import { successfulOrganizeCounts } from "../acquisition-organize/services/organize-history.ts"; import { EXTERNAL_INVOKE_TIMEOUT_MS, sharedInvokeLimiter } from "../agent/invoke-limits.ts"; import { createEntityEpisodeMapper, type EntityEpisodeMapper } from "../sources-identity/service/episode-resolution.ts"; @@ -90,7 +90,7 @@ async function collectVideos(d: FmDeps, body: FilesIdentifyRequest): Promise<{ v const stack = [r.rel]; while (stack.length > 0) { const rel = stack.shift()!; - const names = (await readdir(join(r.rootReal, rel), { withFileTypes: true }).catch(() => [])).sort((a, b) => + const names = (await vreaddirents(join(r.rootReal, rel)).catch(() => [])).sort((a, b) => a.name.localeCompare(b.name), ); for (const n of names) { diff --git a/apps/server/src/modules/file-manager/index.ts b/apps/server/src/modules/file-manager/index.ts index b9b12a0c..45493fd8 100644 --- a/apps/server/src/modules/file-manager/index.ts +++ b/apps/server/src/modules/file-manager/index.ts @@ -3,6 +3,7 @@ import type { TmdbClient } from "@umt/adapters"; import { FilesIdentifyRequest } from "@umt/contracts"; import { defineTask } from "@umt/tasks"; import { z } from "zod"; +import { configureStorage } from "../../storage/registry.ts"; import type { AoOverrides } from "../acquisition-organize/index.ts"; import { createPorts } from "../acquisition-organize/ports.ts"; import { registerScheduledJob } from "../schedules/registry.ts"; @@ -57,10 +58,12 @@ export function createFileManagerModule(overrides: FmOverrides = {}): ServerModu return deps; }, registerRoutes(app, ctx) { + configureStorage(ctx); registerFileRoutes(app, lazy(ctx)); registerOrganizeSourceRoutes(app, lazy(ctx)); }, registerTasks(registry, ctx) { + configureStorage(ctx); registry.register( defineTask({ name: "scanOrganizeSource", diff --git a/apps/server/src/modules/file-manager/list.ts b/apps/server/src/modules/file-manager/list.ts index 3452b90a..ad668c98 100644 --- a/apps/server/src/modules/file-manager/list.ts +++ b/apps/server/src/modules/file-manager/list.ts @@ -17,6 +17,8 @@ import type { } from "@umt/contracts"; import { sql } from "@umt/db"; import { fileKindOf } from "@umt/domain"; +import { storageEntry } from "../../storage/registry.ts"; +import { isRemotePath, vlstat, vreaddir, vreaddirStats, vrealpath } from "../../storage/vfs.ts"; import { hostPathOf, loadInstances } from "../acquisition-organize/services/common.ts"; import { hnrSatisfied } from "../acquisition-organize/services/retention.ts"; import { @@ -29,6 +31,7 @@ import { probeRoot, type RootDef, relOf, + remoteFilesError, resolveIn, within, } from "./common.ts"; @@ -43,7 +46,11 @@ export async function listRoots(d: FmDeps): Promise<{ items: FileRoot[] }> { const h = await probeRoot(def, d.options.healthTimeoutMs); let writable = false; let usage: FileRoot["usage"] = null; - if (h.real) { + if (h.real && def.storageId) { + writable = await isWritableTarget(def, h.real); + const u = await storageUsage(def.storageId, def.hostPath); + if (u) usage = { totalBytes: u.total, freeBytes: u.free }; + } else if (h.real) { writable = await isWritableTarget(def, h.real); const du = await d.ao.ports.diskUsage(h.real).catch(() => null); if (du?.status === "ok") usage = { totalBytes: du.totalBytes, freeBytes: du.availableBytes }; @@ -59,6 +66,13 @@ export async function listRoots(d: FmDeps): Promise<{ items: FileRoot[] }> { checkedAt: h.checkedAt.toISOString(), healthReason: h.reason, usage, + storage: def.storageId + ? { + id: def.storageId, + name: def.storageName ?? def.name, + path: def.hostPath.replace(/^\/@alist\/[0-9a-f-]{36}/i, "") || "/", + } + : null, ...(def.kind === "download" && h.real && h.health !== "offline" ? await pendingOf(d, def.id, h.real) : { pendingCount: null, pendingPartial: false }), @@ -68,6 +82,23 @@ export async function listRoots(d: FmDeps): Promise<{ items: FileRoot[] }> { return { items }; } +/** Alist 容量(拿不到为 null,不报错;缓存 5 分钟,避免每次列根都打 admin 接口) */ +const usageCache = new Map(); +async function storageUsage(storageId: string, hostPath: string) { + const hit = usageCache.get(hostPath); + if (hit && Date.now() - hit.at < 5 * 60_000) return hit.v; + let v: { total: number; free: number } | null = null; + try { + const e = await storageEntry(storageId); + const remote = hostPath.replace(/^\/@alist\/[0-9a-f-]{36}/i, "") || "/"; + v = await e.client.usage(remote); + } catch { + v = null; + } + usageCache.set(hostPath, { at: Date.now(), v }); + return v; +} + // ── G6:下载目录待识别计数(缓存 5 分钟;文件变更事件使对应根失效) ── const PENDING_TTL_MS = 5 * 60_000; @@ -121,6 +152,7 @@ async function canWrite(p: string): Promise { } export async function isWritableTarget(def: RootDef, rootReal: string): Promise { + if (def.storageId) return (def.kind === "library" || def.kind === "storage") && def.modifiable; return def.kind === "library" && def.modifiable && (await canWrite(rootReal)); } @@ -144,7 +176,7 @@ export function kindOfName(name: string): FileKindTag { async function rawOf(abs: string, rel: string, rootReal: string): Promise { let st: Awaited>; try { - st = await lstat(abs); + st = (isRemotePath(abs) ? await vlstat(abs) : await lstat(abs)) as Awaited>; } catch (e) { if (isOfflineErr(e)) throw new FilesError("root_unavailable", `读取失败(${errnoOf(e)})`); return null; // 读取期间被删除 @@ -436,6 +468,7 @@ const matchFilter = (e: FileEntry, f: FileFilter) => { export async function listDir(d: FmDeps, q: FilesListQuery): Promise { const r = await resolveIn(d, q.root, q.path); + if (r.def.storageId) return listRemoteDir(d, q, r); if (r.isSymlink || !r.lst?.isDirectory()) { const st = await stat(r.abs).catch(() => null); if (!st?.isDirectory()) throw new FilesError("bad_request", `不是目录：${r.rel || "/"}`); @@ -463,13 +496,7 @@ export async function listDir(d: FmDeps, q: FilesListQuery): Promise (re ? re.test(n) : n.toLowerCase().includes(lower))); - } + const pool = filterNames(names, q.q); const raws: RawEntry[] = []; for (let i = 0; i < pool.length; i += 64) { const batch = await Promise.all( @@ -477,6 +504,66 @@ export async function listDir(d: FmDeps, q: FilesListQuery): Promise (re ? re.test(n) : n.toLowerCase().includes(lower))); +} + +/** Alist 目录:一次 list 拿全(自动翻页),refresh 只在用户点刷新时带(§9-8);不做递归目录摘要(避免逐目录打网盘接口) */ +async function listRemoteDir( + d: FmDeps, + q: FilesListQuery, + r: Awaited>, +): Promise { + if (!r.lst?.isDirectory()) throw new FilesError("bad_request", `不是目录：${r.rel || "/"}`); + const cursor = decodeCursor(q.cursor); + const dirMtime = Number(r.lst.mtimeMs) || 0; + let items: Awaited>; + try { + items = await vreaddirStats(r.abs, { refresh: q.refresh === true }); + } catch (e) { + throw remoteFilesError(e, r.def); + } + const visible = items.filter((it) => q.hidden || !isHidden(it.name, r.def)); + const truncated = visible.length > d.options.listMaxEntries; + const keep = new Set( + filterNames( + visible.slice(0, d.options.listMaxEntries).map((it) => it.name), + q.q, + ), + ); + const raws: RawEntry[] = visible + .filter((it) => keep.has(it.name)) + .map((it) => { + const abs = join(r.abs, it.name); + const dir = it.stats.isDirectory(); + return { + name: it.name, + rel: relOf(r.rootReal, abs), + abs, + type: dir ? "dir" : "file", + size: dir ? 0 : Number(it.stats.size), + mtimeMs: Number(it.stats.mtimeMs) || null, + escape: false, + } satisfies RawEntry; + }); + return finishList(d, q, r, raws, dirMtime, cursor, truncated); +} + +async function finishList( + d: FmDeps, + q: FilesListQuery, + r: Awaited>, + raws: RawEntry[], + dirMtime: number, + cursor: { o: number; m: number } | null, + truncated: boolean, +): Promise { const downloads = await managedDownloads(d); let entries = await annotate(d, r.def, raws, downloads); if (q.filter?.length) entries = entries.filter((e) => q.filter!.some((f) => matchFilter(e, f))); @@ -512,7 +599,7 @@ export async function listDir(d: FmDeps, q: FilesListQuery): Promise e.type === "dir"); - if (dirs.length === 0) return; + if (dirs.length === 0 || def.storageId) return; const reg = new Map(); if (def.kind === "library") { const likes = dirs.map((e) => `${e.path.replace(/[\\%_]/g, (m) => `\\${m}`)}/%`); @@ -558,7 +645,7 @@ export async function sidecarsOf(abs: string): Promise { const name = basename(abs); const stem = name.slice(0, name.length - extname(name).length); if (!stem) return []; - const names = await readdir(dirname(abs)).catch(() => [] as string[]); + const names = await vreaddir(dirname(abs)).catch(() => [] as string[]); return names.filter((n) => { if (n === name) return false; if (!(n.startsWith(`${stem}.`) || n.startsWith(`${stem}-`))) return false; @@ -602,7 +689,7 @@ export async function statEntry(d: FmDeps, root: string, path: string): Promise< const [entry] = await annotate(d, r.def, [raw]); if (entry!.type === "dir") await fillDirSummaries(d, r.def, r.rootReal, [entry!]); const st = r.lst!; - const real = await realpath(r.abs).catch(() => r.abs); + const real = await vrealpath(r.abs).catch(() => r.abs); const nlink = Number(st.nlink); return { entry: entry!, diff --git a/apps/server/src/modules/file-manager/mutate.ts b/apps/server/src/modules/file-manager/mutate.ts index c764ef05..931d50cd 100644 --- a/apps/server/src/modules/file-manager/mutate.ts +++ b/apps/server/src/modules/file-manager/mutate.ts @@ -1,9 +1,9 @@ import { lstat, mkdir } from "node:fs/promises"; import { basename, dirname, extname, join } from "node:path"; -import { moveSafe } from "@umt/adapters"; import type { FilesMkdirRequest, FilesMkdirResult, FilesRenameRequest, FilesRenameResult } from "@umt/contracts"; import { sanitizeSegment } from "@umt/domain"; import type { Principal } from "../../http/auth.ts"; +import { isRemotePath, vexists, vlstat, vmkdirp, vrenameNoClobber } from "../../storage/vfs.ts"; import { actorOfP, errMessage, errnoOf, FilesError, type FmDeps, recheck, relOf, resolveIn } from "./common.ts"; import { audit, filesChanged } from "./ledger.ts"; import { downloadsFor, isWritableTarget, kindOfName, managedDownloads, sidecarsOf } from "./list.ts"; @@ -31,7 +31,10 @@ export async function mkdirIn( await recheck(r.abs, r.rootReal); const abs = join(r.abs, body.name); try { - await mkdir(abs); + if (r.def.storageId) { + if (await vexists(abs)) throw Object.assign(new Error("exists"), { code: "EEXIST" }); + await vmkdirp(abs); + } else await mkdir(abs); } catch (e) { if (errnoOf(e) === "EEXIST") throw new FilesError("exists", `已存在：${body.name}`); throw new FilesError("bad_request", `创建目录失败：${errMessage(e)}`); @@ -88,7 +91,9 @@ export async function renameEntry( // 不覆盖:任一目标已存在 → 409,原文件不动 const taken: string[] = []; for (const p of pairs) { - const st = await lstat(p.toAbs).catch((e) => (errnoOf(e) === "ENOENT" ? null : e)); + const st = await (isRemotePath(p.toAbs) ? vlstat(p.toAbs) : lstat(p.toAbs)).catch((e) => + errnoOf(e) === "ENOENT" ? null : e, + ); if (st) taken.push(p.to); } if (taken.length > 0) throw new FilesError("exists", `目标已存在，不覆盖：${taken.join(", ")}`); @@ -116,17 +121,19 @@ export async function renameEntry( // 执行:逐个 no-clobber 改名;任一失败回滚已改的 const done: Pair[] = []; const rollback = async () => { - for (const p of done.reverse()) await moveSafe(p.toAbs, p.fromAbs).catch(() => undefined); + for (const p of done.reverse()) await vrenameNoClobber(p.toAbs, p.fromAbs).catch(() => undefined); }; for (const p of pairs) { try { await recheck(p.fromAbs, r.rootReal); - const before = await lstat(p.fromAbs); - const res = await moveSafe(p.fromAbs, p.toAbs); + const remote = isRemotePath(p.fromAbs); + const before = remote ? null : await lstat(p.fromAbs); + const res = await vrenameNoClobber(p.fromAbs, p.toAbs); if (res.status === "exists") throw new FilesError("exists", `目标已存在，不覆盖：${p.to}`); if (res.status !== "moved") throw new FilesError("bad_request", `重命名失败(${res.status}): ${p.from}`); - const after = await lstat(p.toAbs); - if (after.ino !== before.ino || after.dev !== before.dev) + // 远端没有 inode:Alist 改名接口成功即算成功(同 MP) + const after = remote ? null : await lstat(p.toAbs); + if (before && after && (after.ino !== before.ino || after.dev !== before.dev)) throw new FilesError("bad_request", `重命名后 inode 不一致，已中止：${p.to}`); done.push(p); } catch (e) { diff --git a/apps/server/src/modules/file-manager/organize.ts b/apps/server/src/modules/file-manager/organize.ts index c12896cb..6a63248e 100644 --- a/apps/server/src/modules/file-manager/organize.ts +++ b/apps/server/src/modules/file-manager/organize.ts @@ -1,9 +1,9 @@ import { randomUUID } from "node:crypto"; -import { stat } from "node:fs/promises"; import { join, relative } from "node:path"; import type { FilesOrganizeGroup, FilesOrganizeRequest, FilesOrganizeResult } from "@umt/contracts"; import type { Principal } from "../../http/auth.ts"; import { ApiException } from "../../http/errors.ts"; +import { vlstat as stat } from "../../storage/vfs.ts"; import { ensureTaskRow, markTask } from "../acquisition-organize/services/common.ts"; import { type FileOverrideInput, type FrozenPlan, planImport } from "../acquisition-organize/services/organize.ts"; import { diff --git a/apps/server/src/modules/file-manager/preview.ts b/apps/server/src/modules/file-manager/preview.ts index 459640d2..2e084d1c 100644 --- a/apps/server/src/modules/file-manager/preview.ts +++ b/apps/server/src/modules/file-manager/preview.ts @@ -4,6 +4,7 @@ import type { FilePreview, FilesPreviewQuery } from "@umt/contracts"; import type { Principal } from "../../http/auth.ts"; import { runSandbox, SandboxError } from "../../sandbox/client.ts"; import { b64encode } from "../../sandbox/jobs.ts"; +import { isRemotePath, remote, remoteError } from "../../storage/vfs.ts"; import { FilesError, type FmDeps, isAdminSession, recheck, resolveIn } from "./common.ts"; import { BAD_TORRENT } from "./preview-build.ts"; import { extOf, previewKindOf } from "./preview-parse.ts"; @@ -16,6 +17,20 @@ const IMAGE_HEAD = 512 * 1024; const TORRENT_MAX = 4 * 1024 * 1024; async function readHead(abs: string, max: number): Promise<{ buf: Uint8Array; size: number }> { + if (isRemotePath(abs)) { + // Alist:按 Range 只取头部 + const { v, client } = await remote(abs); + try { + const o = await client.get(v.remote); + if (o.isDir) throw new FilesError("bad_request", "不是普通文件"); + const res = await client.openRead(v.remote, { range: `bytes=0-${Math.max(0, max - 1)}` }); + const buf = new Uint8Array(await res.arrayBuffer()).subarray(0, max); + return { buf, size: o.size }; + } catch (e) { + if (e instanceof FilesError) throw e; + throw remoteError(e, abs); + } + } const fh = await open(abs, "r"); try { const st = await fh.stat(); diff --git a/apps/server/src/modules/file-manager/probe.ts b/apps/server/src/modules/file-manager/probe.ts index b9e04bce..35997881 100644 --- a/apps/server/src/modules/file-manager/probe.ts +++ b/apps/server/src/modules/file-manager/probe.ts @@ -1,5 +1,5 @@ import { createHash, randomUUID } from "node:crypto"; -import { mkdir, readFile, stat, writeFile } from "node:fs/promises"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; import { join, resolve } from "node:path"; import type { ProbeResult } from "@umt/adapters"; import type { @@ -11,6 +11,7 @@ import type { } from "@umt/contracts"; import { sql } from "@umt/db"; import type { Principal } from "../../http/auth.ts"; +import { vlstat } from "../../storage/vfs.ts"; import { FilesError, type FmDeps, jsonb, resolveIn } from "./common.ts"; import { audit } from "./ledger.ts"; import { kindOfName, mediaSummary } from "./list.ts"; @@ -54,7 +55,7 @@ export async function probeFiles(d: FmDeps, body: FilesProbeRequest): Promise: ` 开头)。 @@ -120,6 +121,13 @@ export function registerFileRoutes(app: ModuleApp, d: FmDeps) { mount(app, F.probe, async (_c, { body }) => probeFiles(d, body)); mount(app, F.records, async (_c, { query }) => findRecords(d, query)); mount(app, F.download, async (c, { query }) => downloadFile(d, c.get("principal"), query), true); + // 上传:请求体是文件原始字节(不走 JSON 校验),query 照常校验;幂等键防重放 + mount(app, F.upload, async (c, { query }) => { + const { idempotencyKey, ...rest } = query as { idempotencyKey: string } & Record; + return idempotent(d, c.get("principal"), "upload", idempotencyKey, rest, () => + uploadFile(d, c.get("principal"), query, c.req.raw.body, c.req.raw.signal), + ); + }); mount(app, F.preview, async (c, { query }) => previewFile(d, c.get("principal"), query)); // W9-H 文件管理 v2 mount(app, F.entityFromCandidate, async (c, { body }) => entityFromCandidate(d, c.get("principal"), body)); diff --git a/apps/server/src/modules/file-manager/transfer.ts b/apps/server/src/modules/file-manager/transfer.ts index 72816f78..04a8826a 100644 --- a/apps/server/src/modules/file-manager/transfer.ts +++ b/apps/server/src/modules/file-manager/transfer.ts @@ -1,7 +1,7 @@ -import { lstat, readdir, stat } from "node:fs/promises"; import { basename, dirname, join } from "node:path"; import type { FilePlanPair, FilesMoveRequest, FilesTransferResult, OrganizationStatus } from "@umt/contracts"; import type { Principal } from "../../http/auth.ts"; +import { isRemotePath, vlstat as lstat, vreaddir as readdir } from "../../storage/vfs.ts"; import { ensureTaskRow, markTask } from "../acquisition-organize/services/common.ts"; import { planImport } from "../acquisition-organize/services/organize.ts"; import { authorizeMoveEntities, PathGuardError, sourceGuard } from "../acquisition-organize/services/organize-guard.ts"; @@ -94,8 +94,10 @@ export async function transfer( const mode = body.mode; const t = await resolveIn(d, body.target.root, body.target.path); if (!t.lst?.isDirectory() || t.isSymlink) throw new FilesError("bad_request", "目标必须是已存在的目录"); - if (!t.def.libraryRootId || !(await isWritableTarget(t.def, t.rootReal))) - throw new FilesError("not_writable", `目标根 ${t.def.name} 不可写，只能选 可写的媒体库目录`); + // 目标:可写的媒体库目录(本地或 Alist),或 Alist 存储根(文件管理的 st: 根,对照 MP 存储内移动 / 复制) + const storageTarget = t.def.kind === "storage"; + if ((!t.def.libraryRootId && !storageTarget) || !(await isWritableTarget(t.def, t.rootReal))) + throw new FilesError("not_writable", `目标根 ${t.def.name} 不可写，只能选 可写的媒体库目录或 Alist 存储`); const sources: Resolved[] = []; for (const it of body.items) { @@ -118,7 +120,7 @@ export async function transfer( for (const h of downloadsFor(downloads, s.abs)) hnrBlocks.push({ taskId: h.taskId, path: s.rel, hnr: h.hnr, status: h.status }); - const targetDev = Number((await stat(t.abs)).dev); + const targetDev = Number((await lstat(t.abs)).dev); const spec: TransferSpec = { items: sources.map((s, i) => { const dst = join(t.abs, names[i]!); @@ -164,6 +166,11 @@ export async function transfer( ? (await relocatePreview(d.ctx.db, s.def.libraryRootId, s.rel, snaps[i]!.isDir)).locations > 0 : false; const crossDevice = Number((await lstat(s.abs)).dev) !== targetDev; + if (storageTarget && mode === "move" && registered) + throw new FilesError( + "needs_retire", + `${s.rel} 含已整理的文件，只能移动到媒体库目录；要放到 Alist 存储请用复制或先从媒体库移除`, + ); pairs.push({ source: { root: body.items[i]!.root, path: s.rel }, target: { root: body.target.root, path: relOf(t.rootReal, dst) }, @@ -175,7 +182,7 @@ export async function transfer( }); if (!(mode === "move" && !crossDevice)) needBytes += snaps[i]!.size; } - const du = await d.ao.ports.diskUsage(t.abs).catch(() => null); + const du = isRemotePath(t.abs) ? null : await d.ao.ports.diskUsage(t.abs).catch(() => null); const freeBytes = du?.status === "ok" ? du.availableBytes : null; const spaceOk = freeBytes === null ? true : needBytes <= freeBytes; const truncated = snaps.some((s) => s.truncated) || planned.truncated === true; @@ -184,10 +191,14 @@ export async function transfer( const request = { sourcePaths: sources.map((s) => s.abs), action: mode, - targetLibraryRootId: t.def.libraryRootId, + ...(t.def.libraryRootId ? { targetLibraryRootId: t.def.libraryRootId } : {}), }; const allowCross = (await d.ctx.settings.get("organize")).allowCrossVolumeMove; - const crossBlocked = mode === "move" && !allowCross && pairs.some((p) => p.crossDevice); + // 跨卷开关只管本地 ↔ 本地;涉及 Alist 的移动一律「传输成功 → 删源」(同 MP) + const crossBlocked = + mode === "move" && + !allowCross && + pairs.some((p, i) => p.crossDevice && !isRemotePath(sources[i]!.abs) && !isRemotePath(t.abs)); const warnings: string[] = []; if (crossBlocked) warnings.push("未开启跨卷移动，执行会失败。改用复制，或在 设置 → 整理 中开启"); if (conflictSrc.size > 0) warnings.push(`${conflictSrc.size} 个文件在目标已存在，不覆盖，执行时记为失败`); diff --git a/apps/server/src/modules/file-manager/trash.ts b/apps/server/src/modules/file-manager/trash.ts index d71ea7f7..e3d30633 100644 --- a/apps/server/src/modules/file-manager/trash.ts +++ b/apps/server/src/modules/file-manager/trash.ts @@ -16,6 +16,8 @@ import type { } from "@umt/contracts"; import { sql, type TrashItemRow } from "@umt/db"; import type { Principal } from "../../http/auth.ts"; +import { isRemotePath, vlstat, vmkdirp, vreaddirStats, vremove, vrenameNoClobber } from "../../storage/vfs.ts"; +import { parseVPath, toVPath } from "../../storage/vpath.ts"; import { upsertDecision } from "../acquisition-organize/services/common.ts"; import { fsRuntime } from "../acquisition-organize/services/organize.ts"; import { @@ -47,6 +49,7 @@ const likePrefix = (rel: string) => `${rel.replace(/[\\%_]/g, (m) => `\\${m}`)}/ const day = (t: Date) => t.toISOString().slice(0, 10).replace(/-/g, ""); async function sizeOf(abs: string, cap = 200_000): Promise<{ size: number; files: string[] }> { + if (isRemotePath(abs)) return remoteSizeOf(abs, cap); const st = await lstat(abs); if (!st.isDirectory()) return { size: Number(st.size), files: [""] }; let size = 0; @@ -68,6 +71,39 @@ async function sizeOf(abs: string, cap = 200_000): Promise<{ size: number; files return { size, files }; } +/** 远端:按目录 list(每个子目录一次请求) */ +async function remoteSizeOf(abs: string, cap: number): Promise<{ size: number; files: string[] }> { + const st = await vlstat(abs); + if (!st.isDirectory()) return { size: Number(st.size), files: [""] }; + let size = 0; + const files: string[] = []; + const walk = async (dir: string, prefix: string) => { + for (const it of await vreaddirStats(dir)) { + if (files.length >= cap) return; + const rel = prefix ? `${prefix}/${it.name}` : it.name; + if (it.stats.isDirectory()) await walk(join(dir, it.name), rel); + else { + size += Number(it.stats.size); + files.push(rel); + } + } + }; + await walk(abs, ""); + return { size, files }; +} + +/** + * 回收站所在目录:本地 = <根>/.umt-trash;远端同布局,用 Alist 同驱动 move(立即完成)。 + * 远端根是 Alist 的 "/"(各挂载的上一级,不能建目录)时放进该项所在挂载的 .umt-trash。 + */ +export function trashBase(rootReal: string, abs: string): string { + const v = parseVPath(rootReal); + if (v?.remote !== "/") return rootReal; + const a = parseVPath(abs); + const mount = a?.remote.split("/").filter(Boolean)[0]; + return mount ? toVPath(v.storageId, `/${mount}`) : rootReal; +} + interface CheckCtx { abs: string | null; rootReal: string | null; @@ -234,7 +270,8 @@ export async function deleteFiles( await recheck(abs, ctx.rootReal!); let where: string | null = null; if (body.mode === "permanent") { - await rm(abs, { recursive: true, force: false }); + if (isRemotePath(abs)) await vremove(abs); + else await rm(abs, { recursive: true, force: false }); deleted.push(check.path); } else { const row = await moveToTrash(d, principal, check, ctx, expiresAt); @@ -379,6 +416,16 @@ async function moveToTrash( const abs = ctx.abs!; const id = randomUUID(); const stamp = day(d.ao.ports.now()); + if (isRemotePath(abs)) { + // Alist:同存储 move 进 <根>/.umt-trash(MP 无回收站;Alist 驱动自己的回收站不依赖) + const base = trashBase(ctx.rootReal!, abs); + const rel = base === ctx.rootReal ? check.path : relOfRoot(base, abs); + const trashPath = join(base, TRASH_DIR, stamp, id, rel); + await vmkdirp(dirname(trashPath)); + const r = await vrenameNoClobber(abs, trashPath); + if (r.status !== "moved") throw new FilesError("exists", `移入回收站失败(${r.status})：${trashPath}`); + return insertTrashRow(d, principal, check, ctx, { id, rootHost: base, trashPath, expiresAt }); + } let trashPath = join(ctx.rootReal!, TRASH_DIR, stamp, id, check.path); let moved = false; try { @@ -404,13 +451,24 @@ async function moveToTrash( await copyTree(d, abs, trashPath, files); await rm(abs, { recursive: true }); } + return insertTrashRow(d, principal, check, ctx, { id, rootHost: ctx.rootReal!, trashPath, expiresAt }); +} + +function insertTrashRow( + d: FmDeps, + principal: Principal | undefined, + check: FileDeleteCheck, + ctx: CheckCtx, + x: { id: string; rootHost: string; trashPath: string; expiresAt: Date }, +): Promise { + const { id, trashPath, expiresAt } = x; return d.ctx.db .insertInto("trash_item") .values({ id, root_id: check.root, library_root_id: ctx.libraryRootId, - root_host_path: ctx.rootReal!, + root_host_path: x.rootHost, original_path: check.path, trash_path: trashPath, is_dir: check.isDir, @@ -477,6 +535,11 @@ function trashPathSafe(d: FmDeps, row: TrashItemRow): boolean { */ export async function mkdirWithin(rootReal: string, rel: string): Promise { if (rel === "" || rel === ".") return; + if (isRemotePath(rootReal)) { + if (rel.split("/").includes("..")) throw new FilesError("path_escape", `路径含 ..: ${rel}`); + await vmkdirp(join(rootReal, rel)); + return; + } let cur = rootReal; for (const seg of rel.split("/")) { if (seg === "" || seg === ".") continue; @@ -494,6 +557,13 @@ export async function mkdirWithin(rootReal: string, rel: string): Promise } } const relOfRoot = (root: string, p: string) => (p === root ? "" : p.slice(root.length + 1)); +const sameStorageOf = (a: string, b: string) => parseVPath(a)?.storageId === parseVPath(b)?.storageId; +/** <…>/.umt-trash//:回收站路径里 uuid 那一级 */ +function trashUuidDir(row: TrashItemRow): string | null { + const segs = row.trash_path.split("/"); + const i = segs.indexOf(row.id); + return i > 0 ? segs.slice(0, i + 1).join("/") : null; +} export async function restoreTrash( d: FmDeps, @@ -537,8 +607,14 @@ export async function restoreTrash( ); dst = parent.abs; } - if (await lstat(dst).catch(() => null)) throw new FilesError("exists", "原位置已被占用，可选择「还原到…」"); - const r = await moveSafe(row.trash_path, dst); + if (await (isRemotePath(dst) ? vlstat(dst) : lstat(dst)).catch(() => null)) + throw new FilesError("exists", "原位置已被占用，可选择「还原到…」"); + if ( + isRemotePath(dst) !== isRemotePath(row.trash_path) || + (isRemotePath(dst) && !sameStorageOf(dst, row.trash_path)) + ) + throw new FilesError("bad_request", "回收站里的项只能还原到它原来所在的存储"); + const r = await vrenameNoClobber(row.trash_path, dst); if (r.status === "exists") throw new FilesError("exists", "原位置已被占用"); if (r.status === "exdev") { // 跨卷还原:与跨卷进回收站同一上限(FILES_TRASH_CROSS_MAX_BYTES),复制经 fsproxy,核验后删回收站副本 @@ -601,10 +677,17 @@ export async function purgeTrash( for (const row of rows) { try { if (!trashPathSafe(d, row)) throw new FilesError("path_escape", "回收站路径异常，拒绝清除"); - await rm(row.trash_path, { recursive: true, force: true }); // 顺带清掉空的 / 目录 - const uuidDir = row.trash_path.slice(0, row.trash_path.length - row.original_path.length - 1); - await rm(uuidDir, { recursive: true, force: true }).catch(() => undefined); + const uuidDir = trashUuidDir(row); + if (isRemotePath(row.trash_path)) { + await vremove(row.trash_path).catch((e) => { + if (errnoOf(e) !== "ENOENT") throw e; + }); + if (uuidDir) await vremove(uuidDir).catch(() => undefined); + } else { + await rm(row.trash_path, { recursive: true, force: true }); + if (uuidDir) await rm(uuidDir, { recursive: true, force: true }).catch(() => undefined); + } const gone = await d.ctx.db.deleteFrom("trash_item").where("id", "=", row.id).returning("id").executeTakeFirst(); if (gone) purged.push(row.id); } catch (e) { diff --git a/apps/server/src/modules/file-manager/tree.ts b/apps/server/src/modules/file-manager/tree.ts index a521639d..1044ae9e 100644 --- a/apps/server/src/modules/file-manager/tree.ts +++ b/apps/server/src/modules/file-manager/tree.ts @@ -1,4 +1,3 @@ -import { lstat, readdir } from "node:fs/promises"; import { join } from "node:path"; import type { FileRootId, @@ -8,6 +7,7 @@ import type { OrganizationStatus, } from "@umt/contracts"; import { sql } from "@umt/db"; +import { isRemotePath, vlstat as lstat, vreaddirents as readdirents } from "../../storage/vfs.ts"; import { FilesError, type FmDeps, isTempName, loadRootDefs, resolveIn, within } from "./common.ts"; import { kindOfName } from "./list.ts"; @@ -20,7 +20,8 @@ const hiddenName = (n: string) => n.startsWith(".") || isTempName(n); export async function treeOf(d: FmDeps, q: FilesTreeQuery): Promise { const r = await resolveIn(d, q.root, q.path); if (!r.lst?.isDirectory() || r.isSymlink) throw new FilesError("bad_request", `不是目录：${r.rel || "/"}`); - const names = await readdir(r.abs, { withFileTypes: true }); + const remote = isRemotePath(r.abs); + const names = await readdirents(r.abs); const subdirs = names .filter((n) => n.isDirectory() && !hiddenName(n.name)) .map((n) => n.name) @@ -30,7 +31,9 @@ export async function treeOf(d: FmDeps, q: FilesTreeQuery): Promise { const path = r.rel ? `${r.rel}/${name}` : name; - const kids = await readdir(join(r.abs, name), { withFileTypes: true }).catch(() => []); + // Alist:不逐个子目录再列一遍(避免 N 次网盘请求),按「可能有下级」展示,展开时再列 + if (remote) return { name, path, hasChildren: true }; + const kids = await readdirents(join(r.abs, name)).catch(() => []); return { name, path, hasChildren: kids.some((k) => k.isDirectory() && !hiddenName(k.name)) }; }), ); @@ -39,10 +42,15 @@ export async function treeOf(d: FmDeps, q: FilesTreeQuery): Promise n.isFile() && kindOfName(n.name) === "video").length; + partial = names.some((n) => n.isDirectory()); + } + const stack = remote ? [] : [r.abs]; while (stack.length > 0 && !partial) { const dir = stack.pop()!; - for (const n of await readdir(dir, { withFileTypes: true }).catch(() => [])) { + for (const n of await readdirents(dir).catch(() => [])) { if (++seen > d.options.dirSummaryMax) { partial = true; break; diff --git a/apps/server/src/modules/file-manager/upload.ts b/apps/server/src/modules/file-manager/upload.ts new file mode 100644 index 00000000..43f6eefb --- /dev/null +++ b/apps/server/src/modules/file-manager/upload.ts @@ -0,0 +1,83 @@ +import { randomUUID } from "node:crypto"; +import { mkdir, open, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { commitNoClobber, partialPathFor } from "@umt/adapters"; +import type { FilesUploadQuery, FilesUploadResult } from "@umt/contracts"; +import { sanitizeSegment } from "@umt/domain"; +import type { Principal } from "../../http/auth.ts"; +import { relayDir, remote, remoteError, vexists } from "../../storage/vfs.ts"; +import { errnoOf, FilesError, type FmDeps, recheck, relOf, resolveIn } from "./common.ts"; +import { audit, filesChanged } from "./ledger.ts"; +import { isWritableTarget } from "./list.ts"; + +// PUT /files/upload(docs/design/storage-alist.md §4):请求体是文件字节,放进 {root, path} 目录,同名不覆盖。 +// Alist 根:先流式写本机临时文件,再按 §2.6 上传(临时名 → 改名,成功判定同 MP);本地根:同目录 partial → 不覆盖改名。 + +async function spool(body: ReadableStream, file: string, signal?: AbortSignal): Promise { + const fh = await open(file, "w"); + let n = 0; + try { + const reader = body.getReader(); + for (;;) { + signal?.throwIfAborted(); + const { value, done } = await reader.read(); + if (done) break; + await fh.write(value); + n += value.length; + } + } finally { + await fh.close(); + } + return n; +} + +export async function uploadFile( + d: FmDeps, + principal: Principal | undefined, + q: FilesUploadQuery, + body: ReadableStream | null, + signal?: AbortSignal, +): Promise { + if (!body) throw new FilesError("bad_request", "请求体为空"); + const r = await resolveIn(d, q.root, q.path); + if (!r.lst?.isDirectory() || r.isSymlink) throw new FilesError("bad_request", `不是目录：${r.rel || "/"}`); + if (!(await isWritableTarget(r.def, r.rootReal))) throw new FilesError("not_writable", `根 ${r.def.name} 不可写`); + if (sanitizeSegment(q.name) !== q.name) + throw new FilesError("bad_request", `文件名含非法字符或首尾有空白/点，可改为：${sanitizeSegment(q.name)}`); + await recheck(r.abs, r.rootReal); + const dst = join(r.abs, q.name); + const rel = relOf(r.rootReal, dst); + let size: number; + if (r.def.storageId) { + if (await vexists(dst).catch((e) => Promise.reject(remoteError(e, dst)))) + throw new FilesError("exists", `已存在，不覆盖：${q.name}`); + const dir = relayDir(); + await mkdir(dir, { recursive: true }); + const tmp = join(dir, `${randomUUID()}-upload`); + try { + size = await spool(body, tmp, signal); + const { v, client } = await remote(dst); + await client.upload(tmp, v.remote, signal ? { signal } : {}); + } catch (e) { + if (e instanceof FilesError) throw e; + const err = remoteError(e, dst); + if (errnoOf(err) === "EEXIST") throw new FilesError("exists", `已存在，不覆盖：${q.name}`); + throw new FilesError("upstream", `上传到 Alist 失败：${err.message}`); + } finally { + await rm(tmp, { force: true }).catch(() => undefined); + } + } else { + const partial = partialPathFor(dst); + try { + size = await spool(body, partial, signal); + await commitNoClobber(partial, dst); + } catch (e) { + await rm(partial, { force: true }).catch(() => undefined); + if (errnoOf(e) === "EEXIST") throw new FilesError("exists", `已存在，不覆盖：${q.name}`); + throw e; + } + } + await audit(d, principal, "upload", { root: q.root, path: rel, size }); + filesChanged(d, q.root, [r.rel]); + return { path: rel, size }; +} diff --git a/apps/server/test/file-manager/harness.ts b/apps/server/test/file-manager/harness.ts index 54d1cfb8..df0b2d19 100644 --- a/apps/server/test/file-manager/harness.ts +++ b/apps/server/test/file-manager/harness.ts @@ -74,6 +74,8 @@ export interface FmHarness { api(method: string, path: string, body?: unknown, key?: string): Promise<{ status: number; body: Loose }>; /** 原始响应(非 JSON 接口,如 /files/download) */ raw(method: string, path: string, key?: string): Promise; + /** 带原始请求体(如 PUT /files/upload) */ + rawBody(method: string, path: string, body: BodyInit, key?: string): Promise; close(): Promise; } @@ -264,6 +266,15 @@ export async function createFmHarness( raw(method, path, key = DEV_KEY) { return Promise.resolve(app.request(`/api/v1${path}`, { method, headers: authHeaders(key) })); }, + rawBody(method, path, body, key = DEV_KEY) { + return Promise.resolve( + app.request(`/api/v1${path}`, { + method, + headers: { ...authHeaders(key), "content-type": "application/octet-stream" }, + body, + }), + ); + }, async close() { await ts.drop(); await rm(base, { recursive: true, force: true }); diff --git a/apps/server/test/storage/alist-files.test.ts b/apps/server/test/storage/alist-files.test.ts new file mode 100644 index 00000000..1bafe65e --- /dev/null +++ b/apps/server/test/storage/alist-files.test.ts @@ -0,0 +1,240 @@ +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { createFmHarness, type FmHarness, key, pgAvailable, SESSION } from "../file-manager/harness.ts"; +import { addAlist, addAlistLibrary, type TestAlist, text } from "./alist.ts"; + +// 文件管理接入 Alist(对照 MP 存储切换 + Alist 浏览 / 新建目录 / 改名 / 删除 / 移动 / 复制 / 上传 / 下载): +// 根列表多出 st: 根、列目录一次 list(refresh 只在用户刷新时带)、删除进 Alist 上的 .umt-trash 并可还原、 +// 存储内移动 / 复制走 Alist API、上传同名不覆盖、连不上返回 503 而不是「空目录」。 + +const skip = !(await pgAvailable()); + +describe.skipIf(skip)("文件管理:Alist 存储", () => { + let h: FmHarness; + let a: TestAlist; + beforeAll(async () => { + h = await createFmHarness("alist_files"); + a = await addAlist(h.db, { + name: "OpenList", + mockOptions: { token: "t", mountDetails: { "/s1": { total_space: 100, free_space: 40, used_space: 60 } } }, + rootPath: "/s1", + }); + a.mock.put("/s1/movies/A.2020.1080p.mkv", "video-a"); + a.mock.put("/s1/movies/A.2020.1080p.chs.srt", "sub-a"); + a.mock.put("/s1/notes/readme.nfo", ""); + }); + afterAll(async () => h?.close()); + + it("根列表:每个启用的 Alist 实例一个 st: 根,带存储名与容量", async () => { + const r = await h.api("GET", "/files/roots"); + expect(r.status).toBe(200); + const st = r.body.data.items.find((x: { id: string }) => x.id === a.root); + expect(st).toMatchObject({ + kind: "storage", + name: "OpenList", + health: "healthy", + writable: true, + storage: { id: a.id, name: "OpenList", path: "/s1" }, + usage: { totalBytes: 100, freeBytes: 40 }, + }); + }); + + it("列目录:一次 list,不带 refresh;用户刷新才带", async () => { + a.mock.requests.length = 0; + const r = await h.api("GET", `/files?root=${a.root}&path=movies`); + expect(r.status).toBe(200); + expect(r.body.data.entries.map((e: { name: string }) => e.name)).toEqual([ + "A.2020.1080p.chs.srt", + "A.2020.1080p.mkv", + ]); + expect(r.body.data.entries.find((e: { name: string }) => e.name.endsWith(".mkv"))).toMatchObject({ + type: "file", + size: 7, + fileKind: "video", + status: "unidentified", + }); + const lists = a.mock.requests.filter((x) => x.path === "/api/fs/list"); + expect(lists).toHaveLength(1); + expect(lists[0]!.body!.refresh).toBe(false); + await h.api("GET", `/files?root=${a.root}&path=movies&refresh=true`); + expect(a.mock.requests.filter((x) => x.path === "/api/fs/list").at(-1)!.body!.refresh).toBe(true); + }); + + it("新建目录 / 改名(带字幕)/ 同名不覆盖", async () => { + const mk = await h.api("POST", "/files/mkdir", { idempotencyKey: key(), root: a.root, path: "", name: "新目录" }); + expect(mk.status).toBe(200); + expect(a.mock.exists("/s1/新目录")).toBe(true); + const dup = await h.api("POST", "/files/mkdir", { idempotencyKey: key(), root: a.root, path: "", name: "新目录" }); + expect(dup.status).toBe(409); + const rn = await h.api("POST", "/files/rename", { + idempotencyKey: key(), + root: a.root, + path: "movies/A.2020.1080p.mkv", + newName: "A (2020).mkv", + withSidecars: true, + }); + expect(rn.status).toBe(200); + expect(a.mock.listNames("/s1/movies")).toEqual(["A (2020).chs.srt", "A (2020).mkv"]); + a.mock.put("/s1/movies/B.mkv", "b"); + const clash = await h.api("POST", "/files/rename", { + idempotencyKey: key(), + root: a.root, + path: "movies/B.mkv", + newName: "A (2020).mkv", + }); + expect(clash.status).toBe(409); + expect(text(a.mock.read("/s1/movies/A (2020).mkv"))).toBe("video-a"); + }); + + it("上传:流式转传到 Alist,同名不覆盖", async () => { + const r = await h.rawBody( + "PUT", + `/files/upload?root=${a.root}&path=notes&name=${encodeURIComponent("新 文件.txt")}&idempotencyKey=${key()}`, + "hello alist", + ); + expect(r.status).toBe(200); + expect((await r.json()).data).toEqual({ path: "notes/新 文件.txt", size: 11 }); + expect(text(a.mock.read("/s1/notes/新 文件.txt"))).toBe("hello alist"); + const again = await h.rawBody( + "PUT", + `/files/upload?root=${a.root}&path=notes&name=readme.nfo&idempotencyKey=${key()}`, + "overwrite?", + ); + expect(again.status).toBe(409); + expect(text(a.mock.read("/s1/notes/readme.nfo"))).toBe(""); + // 本地根同样可以上传 + const local = await h.rawBody( + "PUT", + `/files/upload?root=${h.libRoot}&path=&name=up.txt&idempotencyKey=${key()}`, + "local", + ); + expect(local.status).toBe(200); + expect(await readFile(join(h.lib, "up.txt"), "utf8")).toBe("local"); + }); + + it("下载与预览(管理员会话):流式转发 Alist 直链", async () => { + const r = await h.raw("GET", `/files/download?root=${a.root}&path=notes/readme.nfo`, SESSION); + expect(r.status).toBe(200); + expect(await r.text()).toBe(""); + const p = await h.api("GET", `/files/preview?root=${a.root}&path=notes/readme.nfo`, undefined, SESSION); + expect(p.status).toBe(200); + expect(JSON.stringify(p.body.data)).toContain("movie"); + }); + + it("删除进 Alist 上的回收站,可还原", async () => { + const del = await h.api("POST", "/files/delete", { + idempotencyKey: key(), + items: [{ root: a.root, path: "movies/B.mkv" }], + dryRun: false, + }); + expect(del.status).toBe(200); + expect(del.body.data.trashed).toHaveLength(1); + expect(a.mock.exists("/s1/movies/B.mkv")).toBe(false); + const trashPath = del.body.data.trashed[0].trashPath as string; + expect(trashPath.startsWith(a.v("/s1/.umt-trash/"))).toBe(true); + const restore = await h.api("POST", "/files/trash/restore", { + idempotencyKey: key(), + ids: [del.body.data.trashed[0].id], + }); + expect(restore.status).toBe(200); + expect(restore.body.data.restored).toHaveLength(1); + expect(text(a.mock.read("/s1/movies/B.mkv"))).toBe("b"); + }); + + it("存储内复制 / 移动走 Alist API(复制源保留,移动源消失)", async () => { + const dry = await h.api("POST", "/files/copy", { + idempotencyKey: key(), + items: [{ root: a.root, path: "notes/readme.nfo" }], + target: { root: a.root, path: "新目录" }, + dryRun: true, + }); + expect(dry.status).toBe(200); + const exec = await h.api("POST", "/files/copy", { + idempotencyKey: key(), + items: [{ root: a.root, path: "notes/readme.nfo" }], + target: { root: a.root, path: "新目录" }, + dryRun: false, + planId: dry.body.data.planId, + }); + expect(exec.status).toBe(200); + expect(exec.body.data.executed).toMatchObject({ succeeded: ["notes/readme.nfo"], failed: [] }); + expect(text(a.mock.read("/s1/新目录/readme.nfo"))).toBe(""); + expect(a.mock.exists("/s1/notes/readme.nfo")).toBe(true); + expect(a.mock.requests.some((x) => x.path === "/api/fs/copy")).toBe(true); + + const mdry = await h.api("POST", "/files/move", { + idempotencyKey: key(), + items: [{ root: a.root, path: "movies/B.mkv" }], + target: { root: a.root, path: "新目录" }, + mode: "move", + dryRun: true, + }); + const mexec = await h.api("POST", "/files/move", { + idempotencyKey: key(), + items: [{ root: a.root, path: "movies/B.mkv" }], + target: { root: a.root, path: "新目录" }, + mode: "move", + dryRun: false, + planId: mdry.body.data.planId, + }); + expect(mexec.status).toBe(200); + expect(mexec.body.data.executed.failed).toEqual([]); + expect(a.mock.exists("/s1/新目录/B.mkv")).toBe(true); + expect(a.mock.exists("/s1/movies/B.mkv")).toBe(false); + }); + + it("复制到本地媒体库(下载到本地)与从本地复制到 Alist 媒体库(上传)", async () => { + const dry = await h.api("POST", "/files/copy", { + idempotencyKey: key(), + items: [{ root: a.root, path: "movies/A (2020).mkv" }], + target: { root: h.libRoot, path: "" }, + dryRun: true, + }); + const exec = await h.api("POST", "/files/copy", { + idempotencyKey: key(), + items: [{ root: a.root, path: "movies/A (2020).mkv" }], + target: { root: h.libRoot, path: "" }, + dryRun: false, + planId: dry.body.data.planId, + }); + expect(exec.body.data.executed.failed).toEqual([]); + expect(await readFile(join(h.lib, "A (2020).mkv"), "utf8")).toBe("video-a"); + + const alib = await addAlistLibrary(h.db, a, "/s1/library"); + await writeFile(join(h.lib, "local.mkv"), "local-video"); + const d2 = await h.api("POST", "/files/copy", { + idempotencyKey: key(), + items: [{ root: h.libRoot, path: "local.mkv" }], + target: { root: alib.root, path: "" }, + dryRun: true, + }); + expect(d2.status).toBe(200); + const e2 = await h.api("POST", "/files/copy", { + idempotencyKey: key(), + items: [{ root: h.libRoot, path: "local.mkv" }], + target: { root: alib.root, path: "" }, + dryRun: false, + planId: d2.body.data.planId, + }); + expect(e2.body.data.executed.failed).toEqual([]); + expect(text(a.mock.read("/s1/library/local.mkv"))).toBe("local-video"); + }); + + it("目录树:Alist 只列当前层,不逐个子目录再请求", async () => { + a.mock.requests.length = 0; + const r = await h.api("GET", `/files/tree?root=${a.root}&path=`); + expect(r.status).toBe(200); + expect(r.body.data.dirs.map((x: { name: string }) => x.name)).toEqual( + expect.arrayContaining(["movies", "notes", "新目录"]), + ); + expect(a.mock.requests.filter((x) => x.path === "/api/fs/list")).toHaveLength(1); + }); + + it("Alist 连不上:列目录 503,不当成空目录", async () => { + a.mock.failNext(5); + const r = await h.api("GET", `/files?root=${a.root}&path=movies`); + expect(r.status).toBe(503); + expect(r.body.error.message).toMatch(/^root_unavailable:/); + }); +}); diff --git a/apps/server/test/storage/alist.ts b/apps/server/test/storage/alist.ts new file mode 100644 index 00000000..e336d9ff --- /dev/null +++ b/apps/server/test/storage/alist.ts @@ -0,0 +1,66 @@ +import { AlistClient, type AlistClientOptions } from "@umt/adapters"; +import { createMockAlist, type MockAlist, type MockAlistOptions } from "@umt/adapters/alist-mock"; +import type { DB, Kysely } from "@umt/db"; +import { overrideStorage } from "../../src/storage/registry.ts"; +import { toVPath } from "../../src/storage/vpath.ts"; + +// Alist 测试装配:登记一个存储实例(storage_instance 行)并把注册表里的客户端换成进程内模拟 Alist。 + +export interface TestAlist { + id: string; + mock: MockAlist; + client: AlistClient; + /** 文件管理根 id(st:) */ + root: string; + /** Alist 内路径 → 虚拟路径 */ + v(p: string): string; +} + +export async function addAlist( + db: Kysely, + o: { + name?: string; + rootPath?: string; + mock?: MockAlist; + mockOptions?: MockAlistOptions; + client?: Partial; + } = {}, +): Promise { + const mock = o.mock ?? createMockAlist(o.mockOptions ?? { token: "t" }); + const rootPath = o.rootPath ?? "/"; + const row = await db + .insertInto("storage_instance") + .values({ name: o.name ?? `alist-${crypto.randomUUID().slice(0, 6)}`, base_url: mock.base, root_path: rootPath }) + .returning("id") + .executeTakeFirstOrThrow(); + // 假时钟:等待 / 退避不真睡,超时按累计的「睡眠」计 + let clock = 0; + const client = new AlistClient({ + baseUrl: mock.base, + auth: { kind: "token", token: "t" }, + fetch: mock.fetch, + qps: 0, + concurrency: 0, + retries: 0, + now: () => clock, + sleep: async (ms) => { + clock += ms; + }, + ...o.client, + }); + overrideStorage(row.id, { client, name: o.name ?? "Alist", rootPath }); + return { id: row.id, mock, client, root: `st:${row.id}`, v: (p) => toVPath(row.id, p) }; +} + +/** 在 Alist 上登记一个纳管媒体库目录 */ +export async function addAlistLibrary(db: Kysely, a: TestAlist, path: string, name = "Alist 媒体库") { + a.mock.mkdirp(path); + const r = await db + .insertInto("library_root") + .values({ name, media_kind: "mixed", local_path: a.v(path), management_mode: "managed", sentinel_file: null }) + .returning("id") + .executeTakeFirstOrThrow(); + return { libraryRootId: r.id, root: `lib:${r.id}` }; +} + +export const text = (b: Uint8Array | null) => (b ? new TextDecoder().decode(b) : null); diff --git a/apps/web/app/components/files/mock.ts b/apps/web/app/components/files/mock.ts index a9284f3c..635ea41d 100644 --- a/apps/web/app/components/files/mock.ts +++ b/apps/web/app/components/files/mock.ts @@ -101,6 +101,7 @@ function makeRoots(): { roots: FileRoot[]; trees: Record } { usage: { totalBytes: 16_000 * GB, freeBytes: 3_420 * GB }, pendingCount: null, pendingPartial: false, + storage: null, }, { id: "lib:tv", @@ -115,6 +116,7 @@ function makeRoots(): { roots: FileRoot[]; trees: Record } { usage: { totalBytes: 16_000 * GB, freeBytes: 3_420 * GB }, pendingCount: null, pendingPartial: false, + storage: null, }, { id: "lib:nas-old", @@ -129,6 +131,7 @@ function makeRoots(): { roots: FileRoot[]; trees: Record } { usage: null, pendingCount: null, pendingPartial: false, + storage: null, }, { id: "dl:qb-main:0", @@ -143,6 +146,7 @@ function makeRoots(): { roots: FileRoot[]; trees: Record } { usage: { totalBytes: 8_000 * GB, freeBytes: 1_210 * GB }, pendingCount: 12, pendingPartial: false, + storage: null, }, ]; const tlouEps = [1, 2, 3, 4, 5, 6, 7].map((ep) => diff --git a/apps/web/app/components/files/model.ts b/apps/web/app/components/files/model.ts index ce7902f6..6c47df57 100644 --- a/apps/web/app/components/files/model.ts +++ b/apps/web/app/components/files/model.ts @@ -480,7 +480,11 @@ export function entrySubline(e: FileEntry): { text: string; tone: "link" | "hint return null; } -export const ROOT_KIND_LABEL: Record = { download: "下载目录", library: "媒体库" }; +export const ROOT_KIND_LABEL: Record = { + download: "下载目录", + library: "媒体库", + storage: "Alist 存储", +}; export type Crumb = { label: string; path: string }; diff --git a/packages/contracts/src/api/organize.ts b/packages/contracts/src/api/organize.ts index b5c0c465..5d04d680 100644 --- a/packages/contracts/src/api/organize.ts +++ b/packages/contracts/src/api/organize.ts @@ -266,6 +266,14 @@ export const organizeApi = { body: files.FilesProbeRequest, response: files.FilesProbeResult, }), + upload: ep({ + method: "put", + path: "/files/upload", + scope: "organize.execute", + summary: "上传文件到目录（请求体为文件字节；同名不覆盖；Alist 根经服务端转传）", + query: files.FilesUploadQuery, + response: files.FilesUploadResult, + }), download: ep({ method: "get", path: "/files/download", diff --git a/packages/contracts/src/files.ts b/packages/contracts/src/files.ts index 301d51d4..fddf5167 100644 --- a/packages/contracts/src/files.ts +++ b/packages/contracts/src/files.ts @@ -13,8 +13,10 @@ import { ScrapeMode, ScrapeRecord } from "./scrape.ts"; // 移动/复制/整理/删除都是两段式:dryRun=true 取得 planId 与前后路径清单,执行时回传 planId; // 源文件 size/mtime 变化或 planId 过期/与请求不符 → 409 plan_stale。 -/** `lib:` 或 `dl::<路径映射序号>` */ -export const FileRootId = z.string().regex(/^(lib:[0-9a-f-]{36}|dl:[0-9a-f-]{36}:\d{1,3})$/i, { message: "非法根 id" }); +/** `lib:`、`dl::<路径映射序号>` 或 `st:`(Alist 存储) */ +export const FileRootId = z + .string() + .regex(/^(lib:[0-9a-f-]{36}|dl:[0-9a-f-]{36}:\d{1,3}|st:[0-9a-f-]{36})$/i, { message: "非法根 id" }); export type FileRootId = z.infer; /** 根内相对路径:不以 / 开头,不含 `..` 段与 NUL;服务端仍会 realpath 复核 */ @@ -41,7 +43,7 @@ export const IdempotencyKey = z.string().min(8).max(200); // ── 根 ── -export const FileRootKind = z.enum(["library", "download"]); +export const FileRootKind = z.enum(["library", "download", "storage"]); export type FileRootKind = z.infer; export const FileRoot = z.object({ @@ -59,6 +61,8 @@ export const FileRoot = z.object({ /** 健康探测失败原因(离线时) */ healthReason: z.string().nullable(), usage: z.object({ totalBytes: NonNegInt, freeBytes: NonNegInt }).nullable(), + /** 根所在存储:本机为 null;Alist 根给实例 id 与名称(界面显示「存储名:/路径」) */ + storage: z.object({ id: z.string(), name: z.string(), path: z.string() }).nullable().default(null), /** G6:下载目录下待识别视频数(与列表"待识别"筛选同口径,递归,缓存 5 分钟);库根与离线根为 null */ pendingCount: NonNegInt.nullable(), /** G6:递归超过扫描上限时为 true,pendingCount 是下限 */ @@ -182,6 +186,8 @@ export const FilesListQuery = z.object({ .optional(), /** 默认 false:隐藏点文件、.uhd-partial、.umt-trash、哨兵 */ hidden: boolQuery.default(false), + /** Alist 根:让 Alist 刷新目录缓存(穿透到网盘接口;只在用户点「刷新」时带) */ + refresh: boolQuery.default(false), }); export type FilesListQuery = z.infer; @@ -797,6 +803,20 @@ export type FilesProbeResult = z.infer; // 下载(W5-A2):GET /files/download 直接返回文件流(非 JSON 信封);仅管理员会话; // 仅 nfo/字幕/图片且 ≤ FILES_DOWNLOAD_MAX_BYTES(默认 20 MiB);inline=true 仅对图片生效(预览)。 +/** + * 上传(PUT /files/upload,请求体是文件原始字节):放进 {root, path} 目录,文件名 name;同名已存在 409(不覆盖)。 + * Alist 根经服务端转传(先写本机临时文件再上传到 Alist,同 MP);本地根直接写临时文件后不覆盖改名。 + */ +export const FilesUploadQuery = z.object({ + root: FileRootId, + path: FileRelPath, + name: FileName, + idempotencyKey: IdempotencyKey, +}); +export type FilesUploadQuery = z.infer; +export const FilesUploadResult = z.object({ path: FileRelPath, size: NonNegInt }); +export type FilesUploadResult = z.infer; + export const FilesDownloadQuery = z.object({ root: FileRootId, path: FileRelPath, inline: boolQuery.default(false) }); export type FilesDownloadQuery = z.infer; /** OpenAPI 描述用:实际响应为二进制流 */ From ccd2cc2c377be9b2e3eec5bdc28d0f6883b4bc1c Mon Sep 17 00:00:00 2001 From: root Date: Tue, 6 Oct 2026 23:52:39 +0800 Subject: [PATCH 05/11] =?UTF-8?q?feat(organize):=20=E6=95=B4=E7=90=86?= =?UTF-8?q?=E7=9F=A9=E9=98=B5=E6=94=AF=E6=8C=81=20Alist(=E4=B8=8A=E4=BC=A0?= =?UTF-8?q?=E3=80=81=E4=B8=8B=E8=BD=BD=E3=80=81Alist=20=E5=86=85=E5=A4=8D?= =?UTF-8?q?=E5=88=B6=E7=A7=BB=E5=8A=A8=E3=80=81=E8=B7=A8=E5=AE=9E=E4=BE=8B?= =?UTF-8?q?=E4=B8=AD=E8=BD=AC)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 本地→Alist 上传、Alist→本地 下载、同实例 API copy/move(等任务完成)、跨实例下载再上传 - 成功判定同 MP:接口成功 + 目标存在;移动在判定通过后删源,删源失败记 attention - 连不上 / 超时 / 限流按可重试 IO 走整理账本,不当成源不存在;上传 / 下载中断不删源 - 计划期的目标冲突、库根在线、洗版替换、非视频隔离、对账均按存储分派;远端文件 ffprobe 读直链头 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei --- .../src/modules/acquisition-organize/index.ts | 3 + .../src/modules/acquisition-organize/ports.ts | 4 +- .../services/organize-exec.ts | 60 +++- .../services/organize-finalize.ts | 10 +- .../services/organize-plan.ts | 14 +- .../services/reconcile.ts | 3 +- .../services/replacement.ts | 122 +++++--- .../acquisition-organize/services/takeover.ts | 2 +- .../storage/alist-organize-matrix.test.ts | 276 ++++++++++++++++++ 9 files changed, 426 insertions(+), 68 deletions(-) create mode 100644 apps/server/test/storage/alist-organize-matrix.test.ts diff --git a/apps/server/src/modules/acquisition-organize/index.ts b/apps/server/src/modules/acquisition-organize/index.ts index ce9d726a..00d54e7f 100644 --- a/apps/server/src/modules/acquisition-organize/index.ts +++ b/apps/server/src/modules/acquisition-organize/index.ts @@ -1,5 +1,6 @@ import { type EventPayload, EventType } from "@umt/contracts"; import { sql } from "@umt/db"; +import { configureStorage } from "../../storage/registry.ts"; import { registerDecisionHook } from "../decision-hooks.ts"; import { registerScheduledJob } from "../schedules/registry.ts"; import type { ServerContext, ServerModule } from "../types.ts"; @@ -69,6 +70,7 @@ export function createAcquisitionOrganizeModule(overrides: AoOverrides = {}): Se return deps; }, registerRoutes(app, ctx) { + configureStorage(ctx); const d = lazy(ctx); // 待处理"处理":接管识别失败 / 目标已存在 两类整理事项在关闭前由本模块执行所选处理 registerDecisionHook(ctx, organizeDecisionHook(d)); @@ -82,6 +84,7 @@ export function createAcquisitionOrganizeModule(overrides: AoOverrides = {}): Se registerClassificationRoutes(app, d); }, registerTasks(registry, ctx) { + configureStorage(ctx); registerAoTasks(registry, lazy(ctx)); }, registerCapabilities(registry, ctx) { diff --git a/apps/server/src/modules/acquisition-organize/ports.ts b/apps/server/src/modules/acquisition-organize/ports.ts index 21c33885..7ab27266 100644 --- a/apps/server/src/modules/acquisition-organize/ports.ts +++ b/apps/server/src/modules/acquisition-organize/ports.ts @@ -9,7 +9,6 @@ import { fetchEpisodeGroup, fetchRichDetail, fetchRichSeason, - ffprobe, getSiteAdapter, getSiteDefinition, groupSeasonAsRichSeason, @@ -27,6 +26,7 @@ import type { DownloaderInstanceRow, MediaServerInstanceRow, SiteRow, SourceRele import type { ParsedRelease } from "@umt/domain"; import { dataPaths } from "../../config.ts"; import { externalFetch } from "../../egress.ts"; +import { probeAny } from "../../storage/probe.ts"; import { ImageCache } from "../cache/images.ts"; import { SourcesRuntime } from "../sources-identity/runtime.ts"; import { @@ -250,7 +250,7 @@ export function createPorts( return fetchRichSeason(client, tvId, season, await metadataLanguages(ctx), await metadataTextOptions(ctx)); }, diskUsage: (p) => diskUsage(p), - probe: (p) => ffprobe(p), + probe: (p) => probeAny(p), async mediaServer(inst) { const settings = (inst.settings ?? {}) as Record; const apiKey = (await resolveSecret(inst.secret_ref)) ?? ""; diff --git a/apps/server/src/modules/acquisition-organize/services/organize-exec.ts b/apps/server/src/modules/acquisition-organize/services/organize-exec.ts index b9f4f5a1..c84ff802 100644 --- a/apps/server/src/modules/acquisition-organize/services/organize-exec.ts +++ b/apps/server/src/modules/acquisition-organize/services/organize-exec.ts @@ -13,6 +13,8 @@ import { import type { OrganizationRecord } from "@umt/contracts"; import { sql } from "@umt/db"; import { withAttemptToken } from "@umt/tasks"; +import { involvesRemote, remoteTransfer } from "../../../storage/transfer.ts"; +import { isRemotePath, vcheckRoot, vmkdirp, vrenameNoClobber, vstatSafe } from "../../../storage/vfs.ts"; import { closeDecisions } from "../../sources-identity/service/common.ts"; import { errMessage, jsonb } from "../util.ts"; import { type AoDeps, upsertDecision } from "./common.ts"; @@ -118,7 +120,9 @@ export async function executeRecord( // 根检查也走执行子进程:NFS/FUSE 挂死 → offline(timeout),按根离线重试,绝不当成文件缺失 // 离线 / 建目录失败不抛错(抛错会耗尽任务重试、记录卡在 running):记录保持 running 并写明原因、开待处理, // 由整理巡检(recoverStalledRecords)在库根恢复后重新入队 - const health = await fsr.proxy.checkRoot(frozen.rootPath, frozen.sentinel); + const health = isRemotePath(frozen.rootPath) + ? await vcheckRoot(frozen.rootPath) + : await fsr.proxy.checkRoot(frozen.rootPath, frozen.sentinel); if (health.kind === "offline") { return holdRecord(d, rec, `${ROOT_OFFLINE}:${health.reason}`, { kind: "other", @@ -133,7 +137,7 @@ export async function executeRecord( } if (rec.action !== "in_place") { try { - for (const dir of frozen.targetDirs) await mkdir(dir, { recursive: true }); + for (const dir of frozen.targetDirs) await vmkdirp(dir); } catch (err) { const code = (err as { code?: string } | null)?.code ?? "EIO"; return holdRecord(d, rec, `${TARGET_DIR_UNAVAILABLE}:${code}`, { @@ -164,6 +168,7 @@ export async function executeRecord( // 写前顺带清本目录残留 partial(对照 MP:只清本目录、>24h;失败吞掉)。copy 路径在 copyWithVerify 内还会再清一次 if (rec.action !== "in_place" && fsr.partialMaxAgeMs > 0) { for (const dir of frozen.targetDirs) { + if (isRemotePath(dir)) continue; await fsr.proxy.sweepStalePartials(dir, fsr.partialMaxAgeMs).catch(() => [] as string[]); } } @@ -250,7 +255,7 @@ export async function executeRecord( : undefined; const stagedOld = selectedOld ? await stagedReplacementBackup(d, selectedOld) : null; if (selectedOld && !stagedOld) { - const current = await fsr.proxy.statSafe(pf.dst, { + const current = await (isRemotePath(pf.dst) ? vstatSafe : fsr.proxy.statSafe.bind(fsr.proxy))(pf.dst, { root: selectedOld.rootPath, sentinel: selectedOld.sentinel, }); @@ -489,18 +494,27 @@ export async function trashReplacedOld( ref: { recordId: string; opId: string }, old: { path?: string; size: number }, ): Promise { - const rootReal = await realpath(frozen.rootPath); - const rel = relative(rootReal, await realpath(dirname(dst)).then((p) => join(p, basename(dst)))); + const remote = isRemotePath(frozen.rootPath); + const rootReal = remote ? frozen.rootPath : await realpath(frozen.rootPath); + const rel = remote + ? relative(rootReal, dst) + : relative(rootReal, await realpath(dirname(dst)).then((p) => join(p, basename(dst)))); if (!rel || rel.startsWith("..") || !old.path) return; const id = randomUUID(); const now = d.ports.now(); const to = join(rootReal, ".umt-trash", now.toISOString().slice(0, 10).replace(/-/g, ""), id, rel); - await mkdir(dirname(to), { recursive: true }); - const m = await fsr.proxy.moveSafe(old.path, to); - if (m.status !== "moved") { - throw new Error(`旧文件移入回收站失败(${m.status}): ${old.path}`); + if (remote) { + await vmkdirp(dirname(to)); + const m = await vrenameNoClobber(old.path, to); + if (m.status !== "moved") throw new Error(`旧文件移入回收站失败(${m.status}): ${old.path}`); + } else { + await mkdir(dirname(to), { recursive: true }); + const m = await fsr.proxy.moveSafe(old.path, to); + if (m.status !== "moved") { + throw new Error(`旧文件移入回收站失败(${m.status}): ${old.path}`); + } + await rmdir(dirname(old.path)).catch(() => undefined); } - await rmdir(dirname(old.path)).catch(() => undefined); const days = Number(Deno.env.get("FILES_TRASH_RETENTION_DAYS") ?? 7) || 7; await d.ctx.db .insertInto("trash_item") @@ -602,7 +616,9 @@ export async function verifyInPlace( path: string, size: number, ): Promise { - const observed = await fsr.proxy.statSafe(path, { root: frozen.rootPath, sentinel: frozen.sentinel }); + const observed = isRemotePath(path) + ? await vstatSafe(path, { root: frozen.rootPath }) + : await fsr.proxy.statSafe(path, { root: frozen.rootPath, sentinel: frozen.sentinel }); if (observed.kind !== "ok" || !observed.facts.isFile || observed.facts.isSymbolicLink) { return { status: "failed", error: { code: "SOURCE_MISSING", message: "原地整理的文件不存在或不可访问" } }; } @@ -645,6 +661,28 @@ export async function transferFile( }, ): Promise { const { kind, src, dst, size, crossDevice, resumed } = x; + // 涉及 Alist 的方向(上传 / 下载 / API copy·move / 跨实例中转):成功判定同 MP(接口成功 + 目标存在),不走本地核验 + if (involvesRemote(src, dst)) { + const r = await remoteTransfer({ + kind, + src, + dst, + onExisting: x.onExisting, + ...(x.replaceOld.trashDir ? { replaceTrashDir: x.replaceOld.trashDir } : {}), + resumed, + ...(x.signal ? { signal: x.signal } : {}), + }); + return { + status: r.status, + bytes: r.bytes ?? size, + ...(r.checksum ? { checksum: r.checksum } : {}), + ...(r.resumed ? { resumed: true } : {}), + ...(r.error ? { error: r.error } : {}), + ...(r.steps ? { steps: r.steps } : {}), + ...(r.replaced ? { replaced: r.replaced } : {}), + hashVia: `alist:${r.via}`, + }; + } const px = fsr.proxy; const call = x.signal ? { signal: x.signal } : {}; const fsErr = (e: unknown) => ({ diff --git a/apps/server/src/modules/acquisition-organize/services/organize-finalize.ts b/apps/server/src/modules/acquisition-organize/services/organize-finalize.ts index a1ea1b65..8da57986 100644 --- a/apps/server/src/modules/acquisition-organize/services/organize-finalize.ts +++ b/apps/server/src/modules/acquisition-organize/services/organize-finalize.ts @@ -1,10 +1,9 @@ -import { readdir } from "node:fs/promises"; import { dirname, join, relative } from "node:path"; -import { statSafe } from "@umt/adapters"; import type { DownloadTaskRow, OrganizationRecordRow } from "@umt/db"; import { sql } from "@umt/db"; import { findOtherVersions } from "@umt/domain"; import { isUnresolvedExclusion, organizationStatusFromFiles } from "@umt/mapping"; +import { isRemotePath, vreaddir as readdir, vstatSafe as statSafe } from "../../../storage/vfs.ts"; import { errMessage, jsonb } from "../util.ts"; import { type AoDeps, type Exec, loadInstances, upsertDecision } from "./common.ts"; import { applySlotEvent } from "./downloads.ts"; @@ -51,7 +50,12 @@ export async function finalizeRecord(d: AoDeps, recordId: string): Promise [] as string[]); for (const o of findOtherVersions( diff --git a/apps/server/src/modules/acquisition-organize/services/organize-plan.ts b/apps/server/src/modules/acquisition-organize/services/organize-plan.ts index 3710dbe2..11752b6b 100644 --- a/apps/server/src/modules/acquisition-organize/services/organize-plan.ts +++ b/apps/server/src/modules/acquisition-organize/services/organize-plan.ts @@ -1,6 +1,5 @@ -import { readdir } from "node:fs/promises"; import { basename, dirname, join, relative } from "node:path"; -import { checkRoot, DEFAULT_ROOT_SENTINEL, FsError, sha256Full, statSafe } from "@umt/adapters"; +import { DEFAULT_ROOT_SENTINEL, FsError, sha256Full } from "@umt/adapters"; import type { ActorType, OrganizationTrigger, PathBinding } from "@umt/contracts"; import { reasonText } from "@umt/contracts"; import type { DownloadTaskRow, LibraryRootRow, MediaEntityRow, SourceReleaseRow } from "@umt/db"; @@ -27,6 +26,12 @@ import { titleMismatches, } from "@umt/domain"; import { organizePlanToFileResults, toDomainPathBinding } from "@umt/mapping"; +import { + vcheckRoot as checkRoot, + isRemotePath, + vreaddir as readdir, + vstatSafe as statSafe, +} from "../../../storage/vfs.ts"; import { createEntityEpisodeMapper, createStructureEpisodeMapper, @@ -588,7 +593,7 @@ export async function planImport(d: AoDeps, req: ImportRequest): Promise x.path === f.src); diff --git a/apps/server/src/modules/acquisition-organize/services/reconcile.ts b/apps/server/src/modules/acquisition-organize/services/reconcile.ts index 52117373..2bd69e30 100644 --- a/apps/server/src/modules/acquisition-organize/services/reconcile.ts +++ b/apps/server/src/modules/acquisition-organize/services/reconcile.ts @@ -1,5 +1,6 @@ import { join } from "node:path"; -import { checkRoot, DEFAULT_ROOT_SENTINEL, statSafe } from "@umt/adapters"; +import { DEFAULT_ROOT_SENTINEL } from "@umt/adapters"; +import { vcheckRoot as checkRoot, vstatSafe as statSafe } from "../../../storage/vfs.ts"; import { errMessage, jsonb } from "../util.ts"; import type { AoDeps } from "./common.ts"; import { applySlotEvent } from "./downloads.ts"; diff --git a/apps/server/src/modules/acquisition-organize/services/replacement.ts b/apps/server/src/modules/acquisition-organize/services/replacement.ts index 05a111e8..2afc5fa1 100644 --- a/apps/server/src/modules/acquisition-organize/services/replacement.ts +++ b/apps/server/src/modules/acquisition-organize/services/replacement.ts @@ -4,10 +4,49 @@ import { basename, dirname, join, relative } from "node:path"; import { DEFAULT_ROOT_SENTINEL, type FileFacts } from "@umt/adapters"; import { sql } from "@umt/db"; import { type ReplacementScope, replacementCovers, replacementScope } from "@umt/domain"; +import { + isRemotePath, + vcheckRoot, + vmkdirp, + vreaddir, + vremove, + vrenameNoClobber, + vstatSafe, +} from "../../../storage/vfs.ts"; import { jsonb } from "../util.ts"; import { type AoDeps, upsertDecision } from "./common.ts"; import { type FrozenPlan, fsRuntime } from "./organize.ts"; +// 本地 / Alist 分派(docs/design/storage-alist.md §6.3):Alist 上的旧文件用同存储 move 暂放 / 进回收站 / 移回。 +// 远端没有 inode 与可靠的修改时间,旧文件「没变」只按大小判断;失败目标不做哈希核对(网盘方向不核验,用户决定)。 +const fsx = (d: AoDeps) => { + const fs = fsRuntime(d); + return { + replaceOld: fs.replaceOld, + checkRoot: (root: string, sentinel: string) => + isRemotePath(root) ? vcheckRoot(root) : fs.proxy.checkRoot(root, sentinel), + statSafe: (p: string, opts: { root?: string; sentinel?: string } = {}) => + isRemotePath(p) ? vstatSafe(p, opts.root ? { root: opts.root } : {}) : fs.proxy.statSafe(p, opts), + realpath: (p: string) => (isRemotePath(p) ? Promise.resolve(p) : realpath(p)), + async move(a: string, b: string): Promise<{ status: string }> { + if (isRemotePath(a) || isRemotePath(b)) { + await vmkdirp(dirname(b)); + return vrenameNoClobber(a, b); + } + await mkdir(dirname(b), { recursive: true }); + return fs.proxy.moveSafe(a, b); + }, + rm: (p: string) => (isRemotePath(p) ? vremove(p) : rm(p, { force: true })), + readdir: (p: string) => (isRemotePath(p) ? vreaddir(p) : readdir(p)), + sha256: (p: string) => fs.proxy.sha256(p), + }; +}; +/** 旧文件事实是否没变:本地比 size/mtime/ino/dev,远端只比 size */ +const sameFacts = (path: string, a: Partial, b: Partial) => + isRemotePath(path) + ? a.size === b.size + : (["size", "mtimeMs", "ino", "dev"] as const).every((k) => a[k] === undefined || a[k] === b[k]); + export interface ReplacementFile { assetId: string; locationId: string; @@ -106,7 +145,7 @@ export async function freezeReplacement( ); for (const file of [loc, ...privateChildren]) { const path = join(loc.local_path, file.relative_path); - const st = await fsRuntime(d).proxy.statSafe(path, { + const st = await fsx(d).statSafe(path, { root: loc.local_path, sentinel: loc.sentinel_file ?? DEFAULT_ROOT_SENTINEL, }); @@ -118,7 +157,7 @@ export async function freezeReplacement( locationId: file.id, rootId: loc.root_id, rootPath: loc.local_path, - sentinel: loc.sentinel_file ?? DEFAULT_ROOT_SENTINEL, + sentinel: isRemotePath(loc.local_path) ? "" : (loc.sentinel_file ?? DEFAULT_ROOT_SENTINEL), path: file.relative_path, facts: { size: st.facts.size, @@ -231,31 +270,26 @@ export async function cleanupReplacement(d: AoDeps, frozen: FrozenPlan, recordId if (location.status === "removed") continue; if (successful.some((f) => frozen.libraryRootId === file.rootId && f.target_path === file.path)) continue; try { - const fs = fsRuntime(d); - if ((await fs.proxy.checkRoot(file.rootPath, file.sentinel)).kind !== "online") { + const fs = fsx(d); + if ((await fs.checkRoot(file.rootPath, file.sentinel)).kind !== "online") { throw new Error("旧版本库根离线"); } - const rootReal = await realpath(file.rootPath); + const rootReal = await fs.realpath(file.rootPath); const abs = join(rootReal, file.path); - const parent = await realpath(dirname(abs)); + const parent = await fs.realpath(dirname(abs)); const rel = relative(rootReal, parent); if (rel.startsWith("..") || file.path.split("/").some((s) => s === "..")) throw new Error("旧文件位置越界"); const trash = join(rootReal, ".umt-trash", file.trashDay, file.trashId, file.path); - const st = await fs.proxy.statSafe(abs, { + const st = await fs.statSafe(abs, { root: rootReal, sentinel: file.sentinel, }); - const moved = await fs.proxy.statSafe(trash); + const moved = await fs.statSafe(trash); const resumed = existing?.status === "started" && st.kind === "missing" && (fs.replaceOld === "delete" || moved.kind === "ok"); - if ( - !resumed && - (st.kind !== "ok" || - st.facts.isSymbolicLink || - Object.entries(file.facts).some(([k, v]) => st.facts[k as keyof typeof file.facts] !== v)) - ) { + if (!resumed && (st.kind !== "ok" || st.facts.isSymbolicLink || !sameFacts(abs, file.facts, st.facts))) { throw new Error("旧文件已变化，保留待核对"); } await db @@ -273,10 +307,9 @@ export async function cleanupReplacement(d: AoDeps, frozen: FrozenPlan, recordId .onConflict((oc) => oc.column("operation_id").doUpdateSet({ status: "started" })) .execute(); if (!resumed) { - if (fs.replaceOld === "delete") await rm(abs); + if (fs.replaceOld === "delete") await fs.rm(abs); else { - await mkdir(dirname(trash), { recursive: true }); - const result = await fs.proxy.moveSafe(abs, trash); + const result = await fs.move(abs, trash); if (result.status !== "moved") { throw new Error(`旧文件回收失败：${result.status}`); } @@ -381,10 +414,10 @@ export async function settleSamePathReplacement(d: AoDeps, frozen: FrozenPlan, r continue; } try { - const fs = fsRuntime(d); + const fs = fsx(d); if (main?.status === "succeeded" && main.media_asset_id) { if (fs.replaceOld === "delete") { - await rm(backup.path, { force: true }); + await fs.rm(backup.path); if (backup.trashItemId) { await d.ctx.db.deleteFrom("trash_item").where("id", "=", backup.trashItemId).execute(); } @@ -402,38 +435,40 @@ export async function settleSamePathReplacement(d: AoDeps, frozen: FrozenPlan, r continue; } if (main?.status !== "failed") continue; - if ((await fs.proxy.checkRoot(old.rootPath, old.sentinel)).kind !== "online") { + if ((await fs.checkRoot(old.rootPath, old.sentinel)).kind !== "online") { throw new Error("旧版本库根离线，保留回收副本"); } + const remote = isRemotePath(pf.dst); const rejected = join(old.rootPath, ".umt-trash", old.trashDay, old.trashId, old.path); - const oldBackup = await fs.proxy.statSafe(backup.path); - const target = await fs.proxy.statSafe(pf.dst); + const oldBackup = await fs.statSafe(backup.path); + const target = await fs.statSafe(pf.dst); const alreadyRestored = - oldBackup.kind === "missing" && - target.kind === "ok" && - Object.entries(old.facts).every(([key, value]) => target.facts[key as keyof typeof old.facts] === value); + oldBackup.kind === "missing" && target.kind === "ok" && sameFacts(pf.dst, old.facts, target.facts); if (!alreadyRestored) { if ( oldBackup.kind !== "ok" || oldBackup.facts.size !== old.facts.size || - oldBackup.facts.ino !== old.facts.ino + (!remote && oldBackup.facts.ino !== old.facts.ino) ) { throw new Error("旧文件回收副本已变化"); } - if (!file.checksum?.startsWith("sha256:")) { + // Alist 目标不做哈希核对(网盘方向不核验,用户决定) + if (!remote && !file.checksum?.startsWith("sha256:")) { throw new Error("失败目标缺少全量校验，旧文件留在回收站"); } if (target.kind === "ok") { - if ((await fs.proxy.sha256(pf.dst)).digest !== file.checksum.slice(7)) { + if (!remote && (await fs.sha256(pf.dst)).digest !== file.checksum!.slice(7)) { throw new Error("失败目标已变化，旧文件留在回收站"); } - await mkdir(dirname(rejected), { recursive: true }); - const parked = await fs.proxy.moveSafe(pf.dst, rejected); + const parked = await fs.move(pf.dst, rejected); if (parked.status !== "moved") throw new Error("失败新文件无法暂存"); - } else if (target.kind !== "missing" || (await fs.proxy.sha256(rejected)).digest !== file.checksum.slice(7)) { + } else if ( + target.kind !== "missing" || + (!remote && (await fs.sha256(rejected)).digest !== file.checksum!.slice(7)) + ) { throw new Error("失败新文件暂存无法核验"); } - const restored = await fs.proxy.moveSafe(backup.path, pf.dst); + const restored = await fs.move(backup.path, pf.dst); if (restored.status !== "moved") { throw new Error("旧文件恢复未完成，副本保留"); } @@ -488,15 +523,11 @@ export async function settleSamePathReplacement(d: AoDeps, frozen: FrozenPlan, r export async function stagedReplacementBackup(d: AoDeps, file: ReplacementFile): Promise { const stage = join(file.rootPath, ".umt-trash", "replacement-staging", file.trashId); - for (const dir of await readdir(stage).catch(() => [] as string[])) { + const fs = fsx(d); + for (const dir of await fs.readdir(stage).catch(() => [] as string[])) { const candidate = join(stage, dir, basename(file.path)); - const st = await fsRuntime(d).proxy.statSafe(candidate); - if ( - st.kind === "ok" && - !st.facts.isSymbolicLink && - Object.entries(file.facts).every(([key, value]) => st.facts[key as keyof typeof file.facts] === value) - ) - return candidate; + const st = await fs.statSafe(candidate); + if (st.kind === "ok" && !st.facts.isSymbolicLink && sameFacts(candidate, file.facts, st.facts)) return candidate; } return null; } @@ -535,7 +566,7 @@ export async function quarantineNotVideo(d: AoDeps, frozen: FrozenPlan, recordId f.skip_reason === "main_failed" && rejectedMains.has(planned.get(f.source_path)?.mainSrc ?? ""))), ); - const fsr = fsRuntime(d); + const fsr = fsx(d); const moved: { path: string; trash: string }[] = []; const left: { path: string; error: string }[] = []; for (const f of targets) { @@ -543,10 +574,10 @@ export async function quarantineNotVideo(d: AoDeps, frozen: FrozenPlan, recordId if (frozen.replacement && pf?.action === "replace") continue; const rel = f.target_path!; try { - if ((await fsr.proxy.checkRoot(frozen.rootPath, frozen.sentinel)).kind !== "online") throw new Error("库根离线"); - const rootReal = await realpath(frozen.rootPath); + if ((await fsr.checkRoot(frozen.rootPath, frozen.sentinel)).kind !== "online") throw new Error("库根离线"); + const rootReal = await fsr.realpath(frozen.rootPath); const abs = join(rootReal, rel); - const st = await fsr.proxy.statSafe(abs, { root: rootReal, sentinel: frozen.sentinel }); + const st = await fsr.statSafe(abs, { root: rootReal, sentinel: frozen.sentinel }); if (st.kind === "missing") continue; if (st.kind !== "ok" || !st.facts.isFile || st.facts.isSymbolicLink) throw new Error("目标不是普通文件"); const registered = await db @@ -562,8 +593,7 @@ export async function quarantineNotVideo(d: AoDeps, frozen: FrozenPlan, recordId const id = randomUUID(); const now = d.ports.now(); const trash = join(rootReal, ".umt-trash", now.toISOString().slice(0, 10).replace(/-/g, ""), id, rel); - await mkdir(dirname(trash), { recursive: true }); - const m = await fsr.proxy.moveSafe(abs, trash); + const m = await fsr.move(abs, trash); if (m.status !== "moved") throw new Error(`移入回收站失败：${m.status}`); const days = Number(Deno.env.get("FILES_TRASH_RETENTION_DAYS") ?? 7) || 7; await db diff --git a/apps/server/src/modules/acquisition-organize/services/takeover.ts b/apps/server/src/modules/acquisition-organize/services/takeover.ts index 64e894d9..dde12628 100644 --- a/apps/server/src/modules/acquisition-organize/services/takeover.ts +++ b/apps/server/src/modules/acquisition-organize/services/takeover.ts @@ -1,6 +1,5 @@ import { randomUUID } from "node:crypto"; import { basename, dirname, extname, join } from "node:path"; -import { statSafe } from "@umt/adapters"; import type { DownloadTaskRow } from "@umt/db"; import { fileKindOf, @@ -10,6 +9,7 @@ import { parseRelease, parseReleasePath, } from "@umt/domain"; +import { vstatSafe as statSafe } from "../../../storage/vfs.ts"; import type { DecisionHook, DecisionHookActor, DecisionResolveBody } from "../../decision-hooks.ts"; import { addAliasIfAbsent } from "../../sources-identity/service/entities.ts"; import type { NameIdentifyResult } from "../ports.ts"; diff --git a/apps/server/test/storage/alist-organize-matrix.test.ts b/apps/server/test/storage/alist-organize-matrix.test.ts new file mode 100644 index 00000000..22e56c2a --- /dev/null +++ b/apps/server/test/storage/alist-organize-matrix.test.ts @@ -0,0 +1,276 @@ +import { access, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { executeRecord } from "../../src/modules/acquisition-organize/services/organize.ts"; +import { createFmHarness, type FmHarness, key, pgAvailable, seedEntity } from "../file-manager/harness.ts"; +import { addAlist, addAlistLibrary, type TestAlist, text } from "./alist.ts"; + +// 整理矩阵(docs/design/storage-alist.md §5):本地→Alist(上传)、Alist→本地(下载)、同实例 Alist→Alist(API copy/move)、 +// 跨实例(下载 → 上传),复制 / 移动各一条;成功判定同 MP(接口成功 + 目标存在),移动成功后删源; +// 失败路径:上传中断 / 上传后目标取不到 / 下载中断都不删源,连不上按可重试处理而不是「源不存在」。 + +const skip = !(await pgAvailable()); +const exists = (p: string) => + access(p).then( + () => true, + () => false, + ); + +describe.skipIf(skip)("整理矩阵:本地 ↔ Alist ↔ Alist", () => { + let h: FmHarness; + let a: TestAlist; + let b: TestAlist; + let alib: { libraryRootId: string; root: string }; + let blib: { libraryRootId: string; root: string }; + let n = 0; + beforeAll(async () => { + h = await createFmHarness("alist_matrix"); + a = await addAlist(h.db, { name: "A", rootPath: "/s1" }); + b = await addAlist(h.db, { name: "B", rootPath: "/" }); + alib = await addAlistLibrary(h.db, a, "/s1/library", "A 媒体库"); + blib = await addAlistLibrary(h.db, b, "/m/library", "B 媒体库"); + a.mock.mkdirp("/s1/incoming"); + }); + afterAll(async () => h?.close()); + + /** 新建一部电影作品 + 源文件,返回作品 id 与源路径(相对源根) */ + async function film(where: "local" | TestAlist) { + n++; + const title = `Matrix Film ${n}`; + const id = await seedEntity(h.db, { kind: "movie", title, year: 2000 + n }); + const name = `Matrix.Film.${n}.${2000 + n}.1080p.WEB-DL.mkv`; + const body = `video-${n}-${"x".repeat(32)}`; + if (where === "local") { + await writeFile(join(h.dl, name), body); + return { id, root: h.dlRoot, path: name, body, title, year: 2000 + n }; + } + where.mock.put(`${where === a ? "/s1/incoming" : "/m/incoming"}/${name}`, body); + return { + id, + root: where.root, + path: where === a ? `incoming/${name}` : `m/incoming/${name}`, + body, + title, + year: 2000 + n, + }; + } + + /** dryRun → 执行 → 跑 executeRecord(模拟 worker);返回记录与逐文件结果 */ + async function organize( + src: { id: string; root: string; path: string }, + action: "copy" | "move", + targetLibraryRootId: string, + ) { + const items = [{ root: src.root, path: src.path, mediaEntityId: src.id }]; + const dry = await h.api("POST", "/files/organize", { + idempotencyKey: key(), + items, + action, + targetLibraryRootId, + dryRun: true, + }); + expect(dry.status, JSON.stringify(dry.body)).toBe(200); + expect(dry.body.data.groups[0].error).toBeNull(); + const exec = await h.api("POST", "/files/organize", { + idempotencyKey: key(), + items, + action, + targetLibraryRootId, + dryRun: false, + planIds: [dry.body.data.groups[0].planId], + }); + expect(exec.status, JSON.stringify(exec.body)).toBe(200); + const recordId = exec.body.data.recordIds[0] as string; + let thrown: unknown = null; + await executeRecord(h.module.deps().ao, recordId).catch((e) => { + thrown = e; + }); + const rec = await h.db + .selectFrom("organization_record") + .selectAll() + .where("id", "=", recordId) + .executeTakeFirstOrThrow(); + const files = await h.db + .selectFrom("organization_file_result") + .selectAll() + .where("organization_record_id", "=", recordId) + .execute(); + return { rec, files, thrown, main: files.find((f) => f.role === "main")! }; + } + + const assetAt = (libraryRootId: string, rel: string) => + h.db + .selectFrom("asset_location") + .select(["id", "status"]) + .where("library_root_id", "=", libraryRootId) + .where("relative_path", "=", rel) + .executeTakeFirst(); + + it("本地 → Alist 复制:上传到媒体库目录,登记资产,源保留,记下本地 sha256", async () => { + const f = await film("local"); + const r = await organize(f, "copy", alib.libraryRootId); + expect(r.rec.status).toBe("succeeded"); + expect(r.main.target_path).toContain(`${f.title} (${f.year})`); + expect(text(a.mock.read(`/s1/library/${r.main.target_path}`))).toBe(f.body); + expect(await exists(join(h.dl, f.path))).toBe(true); + expect(r.main.checksum).toMatch(/^sha256:[0-9a-f]{64}$/); + expect((await assetAt(alib.libraryRootId, r.main.target_path!))?.status).toBe("available"); + expect(a.mock.requests.some((x) => x.path === "/api/fs/put")).toBe(true); + }); + + it("本地 → Alist 移动:上传成功后删本地源", async () => { + const f = await film("local"); + const r = await organize(f, "move", alib.libraryRootId); + expect(r.rec.status).toBe("succeeded"); + expect(text(a.mock.read(`/s1/library/${r.main.target_path}`))).toBe(f.body); + expect(await exists(join(h.dl, f.path))).toBe(false); + }); + + it("Alist → 本地 复制 / 移动:下载到本地媒体库;移动后删远端源", async () => { + const c = await film(a); + const rc = await organize(c, "copy", h.libraryRootId); + expect(rc.rec.status).toBe("succeeded"); + expect(await readFile(join(h.lib, rc.main.target_path!), "utf8")).toBe(c.body); + expect(a.mock.exists(`/s1/${c.path}`)).toBe(true); + expect(rc.main.checksum).toMatch(/^sha256:/); + + const m = await film(a); + const rm = await organize(m, "move", h.libraryRootId); + expect(rm.rec.status).toBe("succeeded"); + expect(await readFile(join(h.lib, rm.main.target_path!), "utf8")).toBe(m.body); + expect(a.mock.exists(`/s1/${m.path}`)).toBe(false); + }); + + it("Alist → Alist 同实例:复制 / 移动走 Alist API(等任务完成)", async () => { + const c = await film(a); + a.mock.requests.length = 0; + const rc = await organize(c, "copy", alib.libraryRootId); + expect(rc.rec.status).toBe("succeeded"); + expect(text(a.mock.read(`/s1/library/${rc.main.target_path}`))).toBe(c.body); + expect(a.mock.exists(`/s1/${c.path}`)).toBe(true); + expect(a.mock.requests.some((x) => x.path === "/api/fs/copy")).toBe(true); + expect(a.mock.requests.some((x) => x.path.startsWith("/api/task/copy/info"))).toBe(true); + expect(a.mock.requests.some((x) => x.path === "/api/fs/put")).toBe(false); + + const m = await film(a); + const rm = await organize(m, "move", alib.libraryRootId); + expect(rm.rec.status).toBe("succeeded"); + expect(text(a.mock.read(`/s1/library/${rm.main.target_path}`))).toBe(m.body); + expect(a.mock.exists(`/s1/${m.path}`)).toBe(false); + }); + + it("Alist → Alist 跨实例:下载到本机中转再上传;移动后删源", async () => { + const c = await film(a); + const rc = await organize(c, "copy", blib.libraryRootId); + expect(rc.rec.status).toBe("succeeded"); + expect(text(b.mock.read(`/m/library/${rc.main.target_path}`))).toBe(c.body); + expect(a.mock.exists(`/s1/${c.path}`)).toBe(true); + + const m = await film(a); + const rm = await organize(m, "move", blib.libraryRootId); + expect(rm.rec.status).toBe("succeeded"); + expect(text(b.mock.read(`/m/library/${rm.main.target_path}`))).toBe(m.body); + expect(a.mock.exists(`/s1/${m.path}`)).toBe(false); + }); + + it("失败:上传中断 → 不删源、远端不留临时文件,按可重试等待", async () => { + const f = await film("local"); + a.mock.breakUploadAfter(8); + const r = await organize(f, "move", alib.libraryRootId); + expect(String(r.thrown)).toContain("可重试失败"); + expect(await exists(join(h.dl, f.path))).toBe(true); + expect(r.main.status).toBe("running"); + const op = await h.db + .selectFrom("asset_operation") + .select("error") + .where("organization_record_id", "=", r.rec.id) + .executeTakeFirstOrThrow(); + expect((op.error as { code: string }).code).toBe("IO"); + const dir = `/s1/library/${f.title} (${f.year})`; + expect((a.mock.listNames(dir) ?? []).filter((x) => x.endsWith(".uhd-partial"))).toEqual([]); + // 重试(网络恢复)后完成并删源 + await executeRecord(h.module.deps().ao, r.rec.id); + const done = await h.db + .selectFrom("organization_record") + .select("status") + .where("id", "=", r.rec.id) + .executeTakeFirstOrThrow(); + expect(done.status).toBe("succeeded"); + expect(await exists(join(h.dl, f.path))).toBe(false); + }); + + it("失败:上传接口成功但目标取不到 → 判失败,不删源", async () => { + const f = await film("local"); + a.mock.dropNextUpload(); + const r = await organize(f, "move", alib.libraryRootId); + expect(await exists(join(h.dl, f.path))).toBe(true); + const op = await h.db + .selectFrom("asset_operation") + .select(["error", "status"]) + .where("organization_record_id", "=", r.rec.id) + .executeTakeFirstOrThrow(); + expect(op.error).toBeTruthy(); + }); + + it("失败:下载中断 → 不生成目标、不删远端源", async () => { + const f = await film(a); + a.mock.breakDownloadAfter(4); + const r = await organize(f, "move", h.libraryRootId); + expect(String(r.thrown)).toContain("可重试失败"); + expect(a.mock.exists(`/s1/${f.path}`)).toBe(true); + const target = join(h.lib, `${f.title} (${f.year})`); + const names = await import("node:fs/promises").then((m) => m.readdir(target).catch(() => [] as string[])); + expect(names.filter((x) => x.endsWith(".mkv"))).toEqual([]); + }); + + it("连不上 Alist:按可重试 IO 处理,不当成「源不存在」", async () => { + const f = await film(a); + const items = [{ root: f.root, path: f.path, mediaEntityId: f.id }]; + const dry = await h.api("POST", "/files/organize", { + idempotencyKey: key(), + items, + action: "copy", + targetLibraryRootId: h.libraryRootId, + dryRun: true, + }); + const exec = await h.api("POST", "/files/organize", { + idempotencyKey: key(), + items, + action: "copy", + targetLibraryRootId: h.libraryRootId, + dryRun: false, + planIds: [dry.body.data.groups[0].planId], + }); + const recordId = exec.body.data.recordIds[0]; + a.mock.failNext(50); + await expect(executeRecord(h.module.deps().ao, recordId)).rejects.toThrow(/可重试失败/); + const op = await h.db + .selectFrom("asset_operation") + .select("error") + .where("organization_record_id", "=", recordId) + .executeTakeFirstOrThrow(); + expect((op.error as { code: string }).code).not.toBe("SOURCE_MISSING"); + a.mock.failNext(0); + }); + + it("撤销:删除整理记录连带媒体库文件 → Alist 上移进回收站(可还原)", async () => { + const f = await film("local"); + const r = await organize(f, "copy", alib.libraryRootId); + expect(r.rec.status).toBe("succeeded"); + const del = await h.api("POST", "/organization-records/delete", { + ids: [r.rec.id], + deleteSource: false, + deleteTarget: true, + dryRun: false, + }); + expect(del.status, JSON.stringify(del.body)).toBe(200); + expect(a.mock.exists(`/s1/library/${r.main.target_path}`)).toBe(false); + const trash = await h.db + .selectFrom("trash_item") + .select(["trash_path", "library_root_id"]) + .where("library_root_id", "=", alib.libraryRootId) + .execute(); + expect(trash.length).toBeGreaterThan(0); + expect(trash[0]!.trash_path.startsWith(a.v("/s1/library/.umt-trash/"))).toBe(true); + }); +}); From c1bc389058fe281be9c06e9598f690eb1223d93c Mon Sep 17 00:00:00 2001 From: root Date: Tue, 6 Oct 2026 23:52:39 +0800 Subject: [PATCH 06/11] =?UTF-8?q?feat(organize):=20=E5=88=AE=E5=89=8A?= =?UTF-8?q?=E5=86=99=E8=BF=9C=E7=AB=AF=E4=B8=8E=20Alist=20=E7=9B=AE?= =?UTF-8?q?=E5=BD=95=E7=9B=91=E6=8E=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 目标在 Alist 时 NFO / 图片(含 clearlogo)本地生成后上传,已存在策略与本地一致 - 整理来源可选 Alist 根(手动 / 目录快照):按 MP 快照语义限深度、按目录修改时间跳过未变子树, 列表失败保留基线,复用两次观察稳定与失败记录 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei --- .../acquisition-organize/services/scrape.ts | 19 +- .../modules/file-manager/organize-sources.ts | 13 +- .../modules/file-manager/source-monitor.ts | 69 +++++- .../test/storage/alist-scrape-monitor.test.ts | 213 ++++++++++++++++++ packages/contracts/src/organize-sources.ts | 2 +- 5 files changed, 305 insertions(+), 11 deletions(-) create mode 100644 apps/server/test/storage/alist-scrape-monitor.test.ts diff --git a/apps/server/src/modules/acquisition-organize/services/scrape.ts b/apps/server/src/modules/acquisition-organize/services/scrape.ts index 69951c1f..7a67f10b 100644 --- a/apps/server/src/modules/acquisition-organize/services/scrape.ts +++ b/apps/server/src/modules/acquisition-organize/services/scrape.ts @@ -1,7 +1,7 @@ import { createHash, randomUUID } from "node:crypto"; -import { lstat, mkdir, readdir, readFile, realpath, rename, rm, writeFile } from "node:fs/promises"; +import { mkdir, rename, rm, writeFile } from "node:fs/promises"; import { dirname, isAbsolute, join, relative, resolve } from "node:path"; -import { checkRoot, DEFAULT_ROOT_SENTINEL } from "@umt/adapters"; +import { DEFAULT_ROOT_SENTINEL } from "@umt/adapters"; import { DEFAULT_SETTINGS, type OrganizeSettings, @@ -24,6 +24,15 @@ import { seasonOfDirName, workNfo, } from "@umt/domain"; +import { + isRemotePath, + vlstat as lstat, + vreadFile as readFile, + vrealpath as realpath, + vcheckRoot, + vreaddirents, + vwriteFileAtomic, +} from "../../../storage/vfs.ts"; import { createEntityEpisodeMapper, type EntityEpisodeMapper, @@ -587,6 +596,8 @@ export async function scrapeAfterImport( /** 原子写:同目录临时文件 → rename;目录不存在时建(只在登记目录内) */ async function atomicWrite(abs: string, content: Uint8Array | string): Promise { + // Alist 目标:本地生成后上传到对应目录(同 MP 刮削 _save_file → upload_file),已存在时上传旁名后替换 + if (isRemotePath(abs)) return vwriteFileAtomic(abs, content); await mkdir(dirname(abs), { recursive: true }); const tmp = join(dirname(abs), `.${randomUUID()}.umt-scrape.tmp`); try { @@ -621,7 +632,7 @@ async function pool(items: readonly T[], n: number, fn: (t: T) => Promise< } async function listDir(rootReal: string, rel: string) { - const names = await readdir(join(rootReal, rel), { withFileTypes: true }).catch(() => []); + const names = await vreaddirents(join(rootReal, rel)).catch(() => []); const visible = names.filter((n) => !n.name.startsWith(".")).sort((a, b) => a.name.localeCompare(b.name)); return { videos: visible @@ -948,7 +959,7 @@ export async function runScrapeRecord( .executeTakeFirst(); if (!lr?.enabled) return await fail("scrape_root_offline"); if (lr.management_mode !== "managed") return await fail("scrape_root_read_only"); - const health = await checkRoot(lr.local_path, lr.sentinel_file ?? DEFAULT_ROOT_SENTINEL); + const health = await vcheckRoot(lr.local_path, lr.sentinel_file ?? DEFAULT_ROOT_SENTINEL); if (health.kind !== "online") return await fail("scrape_root_offline"); } const rootReal = await realpath(rec.root_path).catch(() => null); diff --git a/apps/server/src/modules/file-manager/organize-sources.ts b/apps/server/src/modules/file-manager/organize-sources.ts index 87224f15..4a82467c 100644 --- a/apps/server/src/modules/file-manager/organize-sources.ts +++ b/apps/server/src/modules/file-manager/organize-sources.ts @@ -74,14 +74,19 @@ export async function saveOrganizeSource(d: FmDeps, id: string, input: Omit true, - () => false, - )) + (!target.def.storageId && + !(await access(target.abs, constants.W_OK).then( + () => true, + () => false, + ))) ) throw new FilesError("not_writable", "目标必须为可写的纳管媒体库目录"); + // Alist 来源(对照 MP 远程目录监控):只能手动或目录快照触发,qB 标签对网盘没有意义 + if (source.def.storageId && input.trigger === "qb_tag") + throw new FilesError("bad_request", "Alist 来源只能用手动或目录快照触发"); if (input.trigger === "qb_tag") { if (!input.downloaderInstanceId || !input.tag?.trim()) throw new FilesError("bad_request", "标签触发需要下载器和标签"); diff --git a/apps/server/src/modules/file-manager/source-monitor.ts b/apps/server/src/modules/file-manager/source-monitor.ts index 9f0fee84..a36fa6cc 100644 --- a/apps/server/src/modules/file-manager/source-monitor.ts +++ b/apps/server/src/modules/file-manager/source-monitor.ts @@ -3,6 +3,8 @@ import { basename, join } from "node:path"; import { isDownloadComplete } from "@umt/adapters"; import type { FileRef, OrganizeSourceConfig } from "@umt/contracts"; import { sql } from "@umt/db"; +import { storageEntry } from "../../storage/registry.ts"; +import { vreaddirStats } from "../../storage/vfs.ts"; import { hostPathOf, loadInstances, upsertDecision } from "../acquisition-organize/services/common.ts"; import { planImport } from "../acquisition-organize/services/organize.ts"; import { downloadTasksUnder } from "../acquisition-organize/services/takeover.ts"; @@ -15,6 +17,7 @@ import { jsonb, recheck, relOf, + remoteFilesError, resolveIn, rootDef, within, @@ -46,11 +49,17 @@ export function advanceSnapshot(previous: Snapshot | null, observed: Record>; since: number | null }, +) { if (!within(path, config.source.path) && config.source.path !== "") throw new FilesError("path_escape", "扫描路径必须在来源目录内"); const start = await resolveIn(d, config.source.root, path); if (start.isSymlink || !start.lst?.isDirectory()) throw new FilesError("bad_request", "来源必须为目录"); + if (start.def.storageId) return walkRemoteSource(start, prev); const files: Record> = {}; const stack = [start.abs]; let seen = 0; @@ -78,6 +87,59 @@ export async function walkSource(d: FmDeps, config: OrganizeSourceConfig, path = return { files, truncated, rootReal: start.rootReal }; } +/** + * Alist 来源快照(对照 MP StorageBase.snapshot,docs/design/storage-alist.md §8):BFS 列目录,深度 ≤ snapshotMaxDepth; + * 开了「按目录修改时间跳过」时,子目录修改时间不晚于上次扫描就沿用上次快照里该子树的文件(根目录每轮都列,用于发现删除); + * 子目录列表失败保留旧快照(MP「查询异常时保留基线」);根目录列表失败抛错(记 last_error,保留旧快照)。 + * 请求经实例限速器(QPS / 并发),不会打爆网盘。 + */ +async function walkRemoteSource( + start: Awaited>, + prev?: { files: Record>; since: number | null }, +) { + const opts = (await storageEntry(start.def.storageId!)).options; + const files: Record> = {}; + const carry = (relDir: string) => { + for (const [k, v] of Object.entries(prev?.files ?? {})) + if (k === relDir || k.startsWith(`${relDir}/`)) files[k] = { size: v.size, mtimeMs: v.mtimeMs }; + }; + let seen = 0; + let truncated = false; + const queue: { abs: string; depth: number }[] = [{ abs: start.abs, depth: 0 }]; + while (queue.length && !truncated) { + const { abs, depth } = queue.shift()!; + let items: Awaited>; + try { + items = await vreaddirStats(abs); + } catch (e) { + if (depth === 0) throw remoteFilesError(e, start.def); + carry(relOf(start.rootReal, abs)); + continue; + } + for (const it of items) { + if (++seen > SOURCE_SCAN_LIMIT) { + truncated = true; + break; + } + if (it.name.startsWith(".") || isTempName(it.name)) continue; + const child = join(abs, it.name); + const rel = relOf(start.rootReal, child); + if (it.stats.isDirectory()) { + if (depth + 1 >= opts.snapshotMaxDepth) continue; + const unchanged = + opts.snapshotCheckFolderModtime && + prev?.since != null && + it.stats.mtimeMs > 0 && + it.stats.mtimeMs <= prev.since; + if (unchanged) carry(rel); + else queue.push({ abs: child, depth: depth + 1 }); + } else if (kindOfName(it.name) === "video" && Number(it.stats.size) > 0) + files[rel] = { size: Number(it.stats.size), mtimeMs: Number(it.stats.mtimeMs) }; + } + } + return { files, truncated, rootReal: start.rootReal }; +} + /** Query unfiltered QB state once: an untagged download may never have a DB task row. */ async function activeDownloaderPaths(d: FmDeps, config: OrganizeSourceConfig): Promise { const source = await rootDef(d.ctx.db, config.source.root); @@ -220,7 +282,10 @@ export async function scanOrganizeSource(d: FmDeps, input: { sourceId: string }) const now = new Date(); try { const liveActive = await activeDownloaderPaths(d, config); - const scan = await walkSource(d, config); + const scan = await walkSource(d, config, config.source.path, { + files: row?.baseline_at ? ((row.snapshot as unknown as Snapshot).files ?? {}) : {}, + since: row?.baseline_at && row.last_scan_at ? new Date(row.last_scan_at).getTime() : null, + }); if (scan.truncated) throw new FilesError("too_large", "来源超过单次扫描上限，请选择更小的目录"); const baseline = !row?.baseline_at; const advance = advanceSnapshot(baseline ? null : (row!.snapshot as unknown as Snapshot), scan.files); diff --git a/apps/server/test/storage/alist-scrape-monitor.test.ts b/apps/server/test/storage/alist-scrape-monitor.test.ts new file mode 100644 index 00000000..ecb93653 --- /dev/null +++ b/apps/server/test/storage/alist-scrape-monitor.test.ts @@ -0,0 +1,213 @@ +import { randomUUID } from "node:crypto"; +import type { OrganizeSourceConfig } from "@umt/contracts"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { runScrapeRecord, scrapeEntity } from "../../src/modules/acquisition-organize/services/scrape.ts"; +import { saveOrganizeSource } from "../../src/modules/file-manager/organize-sources.ts"; +import { scanOrganizeSource, scanStock } from "../../src/modules/file-manager/source-monitor.ts"; +import { createFmHarness, type FmHarness, pgAvailable, seedEntity } from "../file-manager/harness.ts"; +import { addAlist, addAlistLibrary, type TestAlist, text } from "./alist.ts"; + +// 刮削写远端(NFO / 海报 / 背景 / clearlogo 本地生成后上传;已存在策略与本地一致) +// 与目录监控发现 Alist 新文件(快照两次稳定才整理、深度上限、按目录修改时间跳过未变子树、列表失败保留基线)。 + +const skip = !(await pgAvailable()); +const JPEG = new Uint8Array([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10, 0x4a, 0x46, 0x49, 0x46, 0x00]); +const PNG = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + +describe.skipIf(skip)("Alist:刮削写远端", () => { + let h: FmHarness; + let a: TestAlist; + let lib: { libraryRootId: string; root: string }; + beforeAll(async () => { + h = await createFmHarness( + "alist_scrape", + { + ao: { + async scrapeImage(p) { + return p.endsWith(".png") ? { content: PNG, mime: "image/png" } : { content: JPEG, mime: "image/jpeg" }; + }, + async scrapeTmdbDetail() { + throw new Error("不应请求 TMDB:本地作品有元数据"); + }, + }, + }, + { runTransfers: false }, + ); + a = await addAlist(h.db, { name: "OL", rootPath: "/s1" }); + lib = await addAlistLibrary(h.db, a, "/s1/movies"); + }); + afterAll(async () => h?.close()); + + const run = async (entity: string, mode: "missing" | "overwrite") => { + const recs = await scrapeEntity(h.module.deps().ao, entity, { trigger: "manual", mode }); + for (const r of recs) await runScrapeRecord(h.module.deps().ao, r.recordId); + return h.db + .selectFrom("scrape_record") + .selectAll() + .where( + "id", + "in", + recs.map((r) => r.recordId), + ) + .execute(); + }; + + it("NFO 与图片(含 clearlogo)写到 Alist 上作品目录;只补缺失跳过已有,全部覆盖改写", async () => { + const film = await seedEntity(h.db, { kind: "movie", title: "星际穿越", year: 2014 }); + await h.db + .updateTable("media_entity") + .set({ + metadata: JSON.stringify({ + overview: { value: "穿越虫洞" }, + posterPath: { value: "/inter.jpg" }, + backdropPath: { value: "/inter-bg.jpg" }, + logoPath: { value: "/inter-logo.png" }, + }), + }) + .where("id", "=", film) + .execute(); + await h.db + .insertInto("external_identity") + .values({ + media_entity_id: film, + provider: "tmdb", + entity_type: "movie", + external_id: "157336", + status: "accepted", + }) + .execute(); + a.mock.put("/s1/movies/星际穿越 (2014)/星际穿越 (2014) - 1080p.mkv", "video"); + const asset = await h.db + .insertInto("media_asset") + .values({ media_entity_id: film, kind: "video", size_bytes: 5 }) + .returning("id") + .executeTakeFirstOrThrow(); + await h.db + .insertInto("asset_location") + .values({ + media_asset_id: asset.id, + location_kind: "library", + library_root_id: lib.libraryRootId, + relative_path: "星际穿越 (2014)/星际穿越 (2014) - 1080p.mkv", + status: "available", + is_primary: true, + }) + .execute(); + + const first = await run(film, "overwrite"); + expect(first[0]!.status, JSON.stringify(first[0])).toBe("succeeded"); + const names = a.mock.listNames("/s1/movies/星际穿越 (2014)"); + expect(names.some((n) => n.endsWith(".nfo"))).toBe(true); + expect(names.some((n) => n.includes("poster"))).toBe(true); + expect(names).toContain("clearlogo.png"); + expect(names.filter((n) => n.includes(".umt-new") || n.endsWith(".uhd-partial"))).toEqual([]); + const nfo = names.find((n) => n.endsWith(".nfo"))!; + expect(text(a.mock.read(`/s1/movies/星际穿越 (2014)/${nfo}`))).toContain("星际穿越"); + + // 用户改过 NFO:只补缺失不动它 + a.mock.put(`/s1/movies/星际穿越 (2014)/${nfo}`, "用户改的"); + const missing = await run(film, "missing"); + expect(missing[0]!.status).toMatch(/succeeded|partial|skipped/); + expect(text(a.mock.read(`/s1/movies/星际穿越 (2014)/${nfo}`))).toBe("用户改的"); + // 全部覆盖:改写回来 + await run(film, "overwrite"); + expect(text(a.mock.read(`/s1/movies/星际穿越 (2014)/${nfo}`))).toContain("星际穿越"); + }); +}); + +describe.skipIf(skip)("Alist:目录监控(远端快照)", () => { + let h: FmHarness; + let a: TestAlist; + let sourceId: string; + beforeAll(async () => { + h = await createFmHarness("alist_monitor", {}, { catalog: true, runTransfers: false }); + a = await addAlist(h.db, { name: "OL", rootPath: "/s1", mockOptions: { token: "t" } }); + a.mock.put("/s1/incoming/Old.Film.2010.1080p.mkv", "old video bytes"); + a.mock.put("/s1/incoming/deep/a/b/c/d/e/Too.Deep.2011.mkv", "too deep"); + const film = await seedEntity(h.db, { kind: "movie", title: "New Film", year: 2021 }); + await h.db + .insertInto("external_identity") + .values({ + media_entity_id: film, + provider: "tmdb", + external_id: "654321", + status: "accepted", + entity_type: "movie", + }) + .execute(); + const config: Omit = { + name: "Alist 来源", + enabled: true, + source: { root: a.root as OrganizeSourceConfig["source"]["root"], path: "incoming" }, + target: { root: h.libRoot as OrganizeSourceConfig["target"]["root"], path: "" }, + mediaType: "all", + category: "欧美电影", + action: "copy", + trigger: "snapshot", + downloaderInstanceId: null, + tag: null, + intervalSec: 60, + }; + sourceId = randomUUID(); + await saveOrganizeSource(h.module.deps(), sourceId, config); + }); + afterAll(async () => h?.close()); + + it("首扫建基线不整理;新文件两次观察稳定后整理;深度超限不进快照", async () => { + const d = h.module.deps(); + const base = await scanOrganizeSource(d, { sourceId }); + expect(base).toMatchObject({ baseline: true, queued: 0 }); + const snap0 = await h.db.selectFrom("organize_source_snapshot").selectAll().executeTakeFirstOrThrow(); + const files0 = Object.keys((snap0.snapshot as { files: Record }).files); + expect(files0).toContain("incoming/Old.Film.2010.1080p.mkv"); + expect(files0.some((f) => f.includes("Too.Deep"))).toBe(false); + + a.mock.put("/s1/incoming/New.Film.2021.tmdb-654321.1080p.WEB-DL.mkv", "new video bytes!!"); + const first = await scanOrganizeSource(d, { sourceId }); + expect(first.eligible).toBe(0); + const second = await scanOrganizeSource(d, { sourceId }); + expect(second.eligible).toBe(1); + expect(second.queued).toBe(1); + const rec = await h.db.selectFrom("organization_record").select(["id", "trigger"]).executeTakeFirstOrThrow(); + expect(rec.trigger).toBe("auto"); + const files = await h.db + .selectFrom("organization_file_result") + .select("source_path") + .where("organization_record_id", "=", rec.id) + .execute(); + expect(files.map((f) => f.source_path)).toContain(a.v("/s1/incoming/New.Film.2021.tmdb-654321.1080p.WEB-DL.mkv")); + expect(h.engine.enqueued.some((e) => e.task === "verifyAndImport")).toBe(true); + }); + + it("按目录修改时间跳过未变子树:子目录没变就不再列", async () => { + const d = h.module.deps(); + a.mock.put("/s1/incoming/sub/Inner.Film.2012.mkv", "inner"); + // sub 目录修改时间早于上次扫描 → 沿用旧快照(新文件要等目录时间变化才会被看到) + a.mock.dirs.set("/s1/incoming/sub", 1); + a.mock.requests.length = 0; + await scanOrganizeSource(d, { sourceId }); + const listed = a.mock.requests.filter((r) => r.path === "/api/fs/list").map((r) => r.body!.path); + expect(listed).toContain("/s1/incoming"); + expect(listed).not.toContain("/s1/incoming/sub"); + // 目录时间更新后再扫就会进去 + a.mock.dirs.set("/s1/incoming/sub", Date.now() + 60_000); + a.mock.requests.length = 0; + await scanOrganizeSource(d, { sourceId }); + expect(a.mock.requests.filter((r) => r.path === "/api/fs/list").map((r) => r.body!.path)).toContain( + "/s1/incoming/sub", + ); + }); + + it("Alist 连不上:记 last_error,保留基线;存量整理也能列出远端文件", async () => { + const d = h.module.deps(); + const before = await h.db.selectFrom("organize_source_snapshot").selectAll().executeTakeFirstOrThrow(); + a.mock.failNext(10); + await scanOrganizeSource(d, { sourceId }); + const after = await h.db.selectFrom("organize_source_snapshot").selectAll().executeTakeFirstOrThrow(); + expect(after.last_error).toBeTruthy(); + expect(after.baseline_at).toEqual(before.baseline_at); + a.mock.failNext(0); + const stock = await scanStock(d, sourceId); + expect(stock.items.map((i) => i.path)).toContain("incoming/Old.Film.2010.1080p.mkv"); + }); +}); diff --git a/packages/contracts/src/organize-sources.ts b/packages/contracts/src/organize-sources.ts index 4e7109a2..154616b8 100644 --- a/packages/contracts/src/organize-sources.ts +++ b/packages/contracts/src/organize-sources.ts @@ -3,7 +3,7 @@ import { Id, JsonObject, Timestamp } from "./common.ts"; // Independent reference shape avoids settings → instances → settings import cycles. const SourceRef = z.object({ - root: z.string().regex(/^(lib:[0-9a-f-]{36}|dl:[0-9a-f-]{36}:\d{1,3})$/i), + root: z.string().regex(/^(lib:[0-9a-f-]{36}|dl:[0-9a-f-]{36}:\d{1,3}|st:[0-9a-f-]{36})$/i), path: z .string() .max(4096) From cf364435eb6639df967ffd9b8ebb4f753ecdc67f Mon Sep 17 00:00:00 2001 From: root Date: Tue, 6 Oct 2026 23:57:33 +0800 Subject: [PATCH 07/11] =?UTF-8?q?feat(web):=20=E5=AD=98=E5=82=A8=E8=AE=BE?= =?UTF-8?q?=E7=BD=AE=E4=B8=8E=E6=96=87=E4=BB=B6=E7=AE=A1=E7=90=86=E7=9A=84?= =?UTF-8?q?=20Alist=20=E5=AD=98=E5=82=A8=E5=88=87=E6=8D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 设置 → 媒体库与目录 → 存储:登记 Alist / OpenList(令牌 / 账号密码 / 访客、路径密码、限速与监控选项), 测试连接显示版本、用户与容量;媒体库目录可选存储 - 文件管理左栏多一组「Alist 存储」,路径显示「存储名:/路径」;移动 / 复制目标可选 Alist 存储; 上传按钮;Alist 根点刷新才让 Alist 刷新目录缓存 - 运行状态的挂载根列出存储实例(令牌失效单独提示) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei --- apps/web/app/api/files.ts | 16 +- apps/web/app/api/instances.ts | 13 + apps/web/app/components/files/dialogs.tsx | 66 ++++- apps/web/app/components/files/dir-picker.tsx | 18 +- apps/web/app/components/files/dir-tree.tsx | 17 +- apps/web/app/components/files/file-detail.tsx | 11 +- apps/web/app/components/files/model.ts | 10 + .../components/files/storage-switch.test.ts | 57 ++++ apps/web/app/components/logs/labels.ts | 1 + .../web/app/components/settings/instances.tsx | 47 ++- apps/web/app/components/settings/library.tsx | 12 +- .../components/settings/settings-page.test.ts | 3 +- .../app/components/settings/storages.test.ts | 80 ++++++ apps/web/app/components/settings/storages.tsx | 268 ++++++++++++++++++ apps/web/app/lib/api.ts | 4 + apps/web/app/routes/console/files.tsx | 13 +- apps/web/app/routes/console/status.tsx | 18 +- packages/contracts/src/settings.ts | 5 +- 18 files changed, 618 insertions(+), 41 deletions(-) create mode 100644 apps/web/app/components/files/storage-switch.test.ts create mode 100644 apps/web/app/components/settings/storages.test.ts create mode 100644 apps/web/app/components/settings/storages.tsx diff --git a/apps/web/app/api/files.ts b/apps/web/app/api/files.ts index 9fdf076b..73535867 100644 --- a/apps/web/app/api/files.ts +++ b/apps/web/app/api/files.ts @@ -50,7 +50,7 @@ import { type TmdbCandidate, } from "@umt/contracts"; import type { z } from "zod"; -import type { FetcherArg } from "~/lib/api.ts"; +import { apiData, type FetcherArg } from "~/lib/api.ts"; import { ApiRequestError, apiParsed } from "./parse.ts"; // 文件管理(W5-A:/files*;说明 docs/runbook/files.md,形状取 contracts/files.ts)。 @@ -402,6 +402,20 @@ function useFilesMutation(fn: (b: B) => Promise, opts: { invalidate?: ( const notDry = (b: { dryRun?: boolean }) => b.dryRun === false; export const useMkdir = () => useFilesMutation((b: FilesMkdirRequest) => post(FilesMkdirResult, "/files/mkdir", b)); + +/** 上传一个文件到目录(请求体是文件字节;同名不覆盖,Alist 根经服务端转传) */ +export const useUpload = () => + useFilesMutation((b: { root: string; path: string; file: File }) => + apiData<{ path: string; size: number }>("/files/upload", { + method: "PUT", + params: { root: b.root, path: b.path, name: b.file.name, idempotencyKey: crypto.randomUUID() }, + body: b.file, + }), + ); + +/** Alist 根「刷新」:让 Alist 刷新目录缓存(穿透到网盘,只在用户点刷新时做) */ +export const refreshRemoteDir = (root: string, path: string) => + apiData("/files", { params: { root, path, refresh: "true", limit: 1 } }); export const useRename = () => useFilesMutation((b: FilesRenameRequest) => post(FilesRenameResult, "/files/rename", b), { invalidate: notDry }); export const useMove = () => diff --git a/apps/web/app/api/instances.ts b/apps/web/app/api/instances.ts index 9742fe42..7890c7dc 100644 --- a/apps/web/app/api/instances.ts +++ b/apps/web/app/api/instances.ts @@ -21,6 +21,10 @@ import type { SiteCredentialsPut, SiteCredentialsResult, SiteUpdate, + StorageInstance, + StorageInstanceCreate, + StorageInstanceUpdate, + StorageTestResult, } from "@umt/contracts"; import { apiData } from "~/lib/api.ts"; import { type QueryArea, qk } from "~/lib/query-keys.ts"; @@ -71,6 +75,15 @@ function actionHook(area: QueryArea, path: string) { }; } +/** Alist / OpenList 存储实例 */ +export const storages = { + ...crudHooks( + "storage-instances", + "/storage-instances", + ), + useTest: actionHook("storage-instances", "/storage-instances/{id}/test"), +}; + export const downloaders = { ...crudHooks( "downloader-instances", diff --git a/apps/web/app/components/files/dialogs.tsx b/apps/web/app/components/files/dialogs.tsx index f7a41863..643bca6b 100644 --- a/apps/web/app/components/files/dialogs.tsx +++ b/apps/web/app/components/files/dialogs.tsx @@ -10,6 +10,7 @@ import { Trash2, TriangleAlert, Undo2, + Upload, Wand2, } from "lucide-react"; import { useEffect, useMemo, useRef, useState } from "react"; @@ -34,6 +35,7 @@ import { useScrapeBatch, useScrapeRecords, useSuggestName, + useUpload, } from "~/api/files.ts"; import { useSettings } from "~/api/settings.ts"; import { FormField } from "~/components/console/form-field.tsx"; @@ -58,6 +60,7 @@ import { DirPicker } from "./dir-picker.tsx"; import { baseName, deleteBlockText, + displayPath, nameProblem, nameTaken, newIdempotencyKey, @@ -115,6 +118,46 @@ function useDebounced(v: T, ms: number): T { return d; } +// ─── 上传 ─────────────────────────────────────────────────────── + +/** + * 上传到当前目录(对照 MP 文件管理的上传;Alist 根经服务端转传):选多个文件逐个上传,同名不覆盖并提示。 + * 只读目录与下载目录不能上传。 + */ +export function UploadButton({ root, path }: { root: FileRoot; path: string }) { + const upload = useUpload(); + const input = useRef(null); + const [busy, setBusy] = useState(null); + const disabled = !root.writable || busy !== null; + const run = async (files: FileList | null) => { + if (!files?.length) return; + let ok = 0; + const failed: string[] = []; + for (const file of Array.from(files)) { + setBusy(file.name); + try { + await upload.mutateAsync({ root: root.id, path, file }); + ok++; + } catch (e) { + failed.push(`${file.name}：${filesErrorReason(e) === "exists" ? "同名已存在，没有覆盖" : filesErrorText(e)}`); + } + } + setBusy(null); + if (input.current) input.current.value = ""; + if (ok) toast.success(`已上传 ${ok} 个文件到 ${displayPath(root, path)}`); + if (failed.length) toast.error(`${failed.length} 个文件没有上传`, { description: failed.join("\n") }); + }; + return ( + + void run(e.currentTarget.files)} /> + + + ); +} + // ─── 新建目录 ─────────────────────────────────────────────────── export function MkdirDialog({ @@ -157,10 +200,7 @@ export function MkdirDialog({
新建目录 - - {root.hostPath} - {path ? `/${path}` : ""}/ - + {displayPath(root, path)}/ { setTarget(loc); runPreview(loc, mode); }} className="max-h-72 md:max-h-96" /> -

只能选可写媒体库目录下的目录。

+

+ 可选可写的媒体库目录或 Alist 存储；本机和 Alist 之间会上传或下载。 +

{target && targetRoot ? (

目标目录

-

- {targetRoot.hostPath} - {target.path ? `/${target.path}` : ""} -

+

{displayPath(targetRoot, target.path)}

) : ( 在左侧选择目标目录。 @@ -669,8 +709,8 @@ export function TransferDialog({ {formatBytes(p.size, 1)}
@@ -947,9 +987,9 @@ export function DeleteDialog({ export function CopyPathDialog({ root, entry, onClose }: { root: FileRoot; entry: FileEntry; onClose: () => void }) { const stat = useFileStat(root.id, entry.path); - const local = `${root.hostPath}${entry.path ? `/${entry.path}` : ""}`; + const local = displayPath(root, entry.path); const rows = [ - { label: "本机路径", value: local }, + { label: root.storage ? "Alist 路径" : "本机路径", value: local }, { label: "真实路径（解析链接后）", value: stat.data?.realPath ?? null }, { label: "登记目录内相对路径", value: entry.path || "（登记目录本身）" }, ]; diff --git a/apps/web/app/components/files/dir-picker.tsx b/apps/web/app/components/files/dir-picker.tsx index f71c67d3..fd3fd2de 100644 --- a/apps/web/app/components/files/dir-picker.tsx +++ b/apps/web/app/components/files/dir-picker.tsx @@ -2,6 +2,16 @@ import type { FileRef, FileRoot } from "~/api/files.ts"; import { cn } from "~/lib/utils.ts"; import { DirTree } from "./dir-tree.tsx"; +/** 可选的目标根:可写、在线的媒体库;移动 / 复制还可以放进 Alist 存储(includeStorage),整理「指定目录」与还原只能选媒体库 */ +export function pickableRoots>( + roots: T[], + includeStorage = false, +): T[] { + return roots.filter( + (r) => (r.kind === "library" || (includeStorage && r.kind === "storage")) && r.writable && r.health !== "offline", + ); +} + /** * 目录选择器(§14 DirPicker):基于 DirTree,只能选可写库根下的目录。 * 移动/复制到…、回收站"还原到…"、整理向导"指定目录"共用。 @@ -11,13 +21,15 @@ export function DirPicker({ value, onChange, className, + includeStorage = false, }: { roots: FileRoot[]; value: FileRef | null; onChange: (ref: FileRef) => void; className?: string; + includeStorage?: boolean; }) { - const writable = roots.filter((r) => r.kind === "library" && r.writable && r.health !== "offline"); + const writable = pickableRoots(roots, includeStorage); return (
{writable.length ? ( @@ -30,7 +42,9 @@ export function DirPicker({ onSelect={onChange} /> ) : ( -

没有可写入的媒体库目录。先在设置里添加。

+

+ 没有可写入的{includeStorage ? "目录" : "媒体库目录"}。先在设置里添加。 +

)}
); diff --git a/apps/web/app/components/files/dir-tree.tsx b/apps/web/app/components/files/dir-tree.tsx index 9f482013..2e35ec99 100644 --- a/apps/web/app/components/files/dir-tree.tsx +++ b/apps/web/app/components/files/dir-tree.tsx @@ -1,11 +1,20 @@ -import { ChevronRight, Download, Folder, FolderOpen, HardDrive, LoaderCircle, TriangleAlert } from "lucide-react"; +import { + ChevronRight, + Cloud, + Download, + Folder, + FolderOpen, + HardDrive, + LoaderCircle, + TriangleAlert, +} from "lucide-react"; import { useEffect, useState } from "react"; import { type FileRoot, useDirChildren } from "~/api/files.ts"; import { useListPagination } from "~/components/console/use-list-pagination.ts"; import { Pagination } from "~/components/ui/pagination.tsx"; import { healthLabel } from "~/lib/labels.ts"; import { cn } from "~/lib/utils.ts"; -import { groupRoots, joinRel, segments } from "./model.ts"; +import { displayPath, groupRoots, joinRel, segments } from "./model.ts"; type Loc = { root: string; path: string }; type DropHandler = (target: Loc, e: React.DragEvent) => void; @@ -115,7 +124,7 @@ function RootNode({ const isOpen = open.has(k) && !offline; const active = current?.root === root.id && current.path === ""; const h = healthLabel(root.health); - const Icon = root.kind === "download" ? Download : HardDrive; + const Icon = root.kind === "download" ? Download : root.kind === "storage" ? Cloud : HardDrive; return (
onDropTo({ root: root.id, path: "" }, e) : undefined} - title={disabled ? (offline ? "目录离线" : "只读目录，不能作为目标") : root.hostPath} + title={disabled ? (offline ? "目录离线" : "只读目录，不能作为目标") : displayPath(root)} > diff --git a/apps/web/app/components/files/file-detail.tsx b/apps/web/app/components/files/file-detail.tsx index 7916a78e..2815b966 100644 --- a/apps/web/app/components/files/file-detail.tsx +++ b/apps/web/app/components/files/file-detail.tsx @@ -16,7 +16,7 @@ import { formatBytes, formatDateTime } from "~/lib/utils.ts"; import { EntryStatus } from "./file-list.tsx"; import { FilePreviewSection } from "./file-preview.tsx"; import { IdentifyPanel } from "./identify-panel.tsx"; -import { baseName, previewKindOf } from "./model.ts"; +import { baseName, displayPath, previewKindOf } from "./model.ts"; const ACTION_LABEL: Record = { copy: "复制", @@ -122,8 +122,13 @@ export function FileDetail({ ? `${stat.data.nlink}${stat.data.sharedInode ? "（共享 inode，不能原地改写）" : ""}` : null, }, - { label: "本机路径", value: `${root.hostPath}${e.path ? `/${e.path}` : ""}`, mono: true, wide: true }, - ...(stat.data && stat.data.realPath !== `${root.hostPath}/${e.path}` + { + label: root.storage ? "Alist 路径" : "本机路径", + value: displayPath(root, e.path), + mono: true, + wide: true, + }, + ...(stat.data && !root.storage && stat.data.realPath !== `${root.hostPath}/${e.path}` ? [{ label: "真实路径", value: stat.data.realPath, mono: true, wide: true }] : []), ]} diff --git a/apps/web/app/components/files/model.ts b/apps/web/app/components/files/model.ts index 6c47df57..81e44dd6 100644 --- a/apps/web/app/components/files/model.ts +++ b/apps/web/app/components/files/model.ts @@ -524,9 +524,19 @@ export function groupRoots>( return [ { title: "下载目录", kind: "download", items: roots.filter((r) => r.kind === "download") }, { title: "媒体库", kind: "library", items: roots.filter((r) => r.kind === "library") }, + { title: "Alist 存储", kind: "storage", items: roots.filter((r) => r.kind === "storage") }, ].filter((g) => g.items.length > 0); } +/** 根 + 相对路径的显示:本机给本机路径;Alist 给「存储名:/Alist 内路径」(不暴露内部 /@alist 前缀) */ +export function displayPath(root: Pick, rel = ""): string { + const base = root.storage + ? `${root.storage.name}:${root.storage.path === "/" ? "" : root.storage.path}` + : root.hostPath; + const joined = `${base}${rel ? `/${rel}` : ""}`; + return root.storage && !rel && root.storage.path === "/" ? `${root.storage.name}:/` : joined; +} + /** 目标名在当前目录已存在(忽略自身;不区分大小写只作提示,服务端仍按字节判断) */ export function nameTaken(name: string, self: string, siblings: string[]): boolean { const n = name.trim(); diff --git a/apps/web/app/components/files/storage-switch.test.ts b/apps/web/app/components/files/storage-switch.test.ts new file mode 100644 index 00000000..b0fcb1b1 --- /dev/null +++ b/apps/web/app/components/files/storage-switch.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it } from "vitest"; +import { pickableRoots } from "./dir-picker.tsx"; +import { breadcrumbItems, displayPath, groupRoots, initialLocation } from "./model.ts"; + +// 文件管理的存储切换(对照 MP 存储切换菜单):Alist 实例是单独一组根,路径显示「存储名:/路径」, +// 移动 / 复制的目标可以选 Alist 存储,整理「指定目录」与还原仍只能选媒体库。 + +const root = (id: string, kind: "library" | "download" | "storage", extra: Record = {}) => ({ + id, + kind, + name: id, + hostPath: kind === "storage" ? `/@alist/00000000-0000-4000-8000-000000000001${extra.path ?? ""}` : `/data/${id}`, + writable: kind !== "download", + health: "healthy" as const, + storage: + kind === "storage" + ? { id: "00000000-0000-4000-8000-000000000001", name: "OpenList", path: (extra.path as string) ?? "/" } + : null, + ...extra, +}); + +describe("文件管理:存储切换", () => { + it("分组顺序:下载目录 → 媒体库 → Alist 存储;没有的组不出现", () => { + const g = groupRoots([root("s", "storage"), root("lib", "library"), root("dl", "download")]); + expect(g.map((x) => x.title)).toEqual(["下载目录", "媒体库", "Alist 存储"]); + expect(groupRoots([root("lib", "library")]).map((x) => x.kind)).toEqual(["library"]); + }); + + it("路径显示:本机原样;Alist 显示存储名与 Alist 内路径,不露出内部前缀", () => { + expect(displayPath(root("lib", "library"), "电影/A")).toBe("/data/lib/电影/A"); + expect(displayPath(root("s", "storage", { path: "/115/影视" }), "电影")).toBe("OpenList:/115/影视/电影"); + expect(displayPath(root("s", "storage"), "")).toBe("OpenList:/"); + expect(displayPath(root("s", "storage"), "a/b")).toBe("OpenList:/a/b"); + expect(displayPath(root("s", "storage"), "a")).not.toContain("@alist"); + }); + + it("面包屑的组名:Alist 存储", () => { + expect(breadcrumbItems(root("s", "storage"), "a/b").group).toBe("Alist 存储"); + }); + + it("目标选择:移动 / 复制可选 Alist 存储;整理指定目录与还原只能选媒体库;离线与只读不可选", () => { + const roots = [ + root("lib", "library"), + root("ro", "library", { writable: false }), + root("off", "library", { health: "offline" }), + root("s", "storage"), + root("dl", "download"), + ]; + expect(pickableRoots(roots).map((r) => r.id)).toEqual(["lib"]); + expect(pickableRoots(roots, true).map((r) => r.id)).toEqual(["lib", "s"]); + }); + + it("初始位置:没有下载目录与媒体库时落到 Alist 存储", () => { + expect(initialLocation([root("s", "storage")], null)).toEqual({ root: "s", path: "" }); + expect(initialLocation([root("s", "storage"), root("lib", "library")], null)).toEqual({ root: "lib", path: "" }); + }); +}); diff --git a/apps/web/app/components/logs/labels.ts b/apps/web/app/components/logs/labels.ts index eeb2ac08..3adc2970 100644 --- a/apps/web/app/components/logs/labels.ts +++ b/apps/web/app/components/logs/labels.ts @@ -96,6 +96,7 @@ const NOUN: Record = { downloader_instance: "下载器", media_server_instance: "媒体服务器", path_mapping: "路径映射", + storage_instance: "存储", pi_upstream: "模型服务", schedule: "定时任务", automation_grant: "持续授权", diff --git a/apps/web/app/components/settings/instances.tsx b/apps/web/app/components/settings/instances.tsx index 28d9ca89..202b7086 100644 --- a/apps/web/app/components/settings/instances.tsx +++ b/apps/web/app/components/settings/instances.tsx @@ -11,7 +11,7 @@ import { } from "@umt/contracts"; import { PlugZap } from "lucide-react"; import { toast } from "sonner"; -import { downloaders, libraryRoots, mediaServers } from "~/api/instances.ts"; +import { downloaders, libraryRoots, mediaServers, storages } from "~/api/instances.ts"; import { StatusBadge } from "~/components/console/status-badge.tsx"; import { Button } from "~/components/ui/button.tsx"; import { errorMessage } from "~/lib/api.ts"; @@ -360,7 +360,25 @@ const MEDIA_KIND = [ { value: "other", label: "其它" }, ]; +/** 库根路径 ↔ 存储 + 路径:Alist 上的库根存成 /@alist/<存储 id>/ */ +const ALIST_PATH = /^\/@alist\/([0-9a-f-]{36})(\/.*)?$/i; +export function splitRootPath(p: string): { storage: string; path: string } { + const m = ALIST_PATH.exec(p); + return m ? { storage: m[1]!.toLowerCase(), path: m[2] || "/" } : { storage: "local", path: p }; +} +export function joinRootPath(storage: string, path: string): string { + if (storage === "local") return path; + const p = `/${path.trim()}`.replace(/\/+/g, "/").replace(/\/$/, ""); + return `/@alist/${storage}${p}`; +} + export function LibraryRootsTable() { + const storageList = storages.useList(); + const storageOptions = [ + { value: "local", label: "本机" }, + ...(storageList.data?.items ?? []).map((s) => ({ value: s.id, label: `${s.name}（Alist）` })), + ]; + const storageName = (id: string) => storageList.data?.items.find((s) => s.id === id)?.name ?? "Alist"; const list = libraryRoots.useList(); const create = libraryRoots.useCreate(); const update = libraryRoots.useUpdate(); @@ -371,7 +389,8 @@ export function LibraryRootsTable() { hooks={{ list, create, update, remove } as never} config={{ noun: "媒体库目录", - description: "保存时会在目录里放一个标记文件，用来判断目录是否挂载；目录不存在或不可写时无法保存。", + description: + "本机目录保存时会放一个标记文件，用来判断目录是否挂载；目录不存在或不可写时无法保存。也可以选登记好的 Alist 存储，Alist 上的目录要先建好。", deleteHint: "只删除登记，磁盘上的文件不动。", columns: [ { @@ -380,7 +399,12 @@ export function LibraryRootsTable() { cell: (r) => (

{r.name}

-

{r.localPath}

+

+ {(() => { + const sp = splitRootPath(r.localPath); + return sp.storage === "local" ? r.localPath : `${storageName(sp.storage)}:${sp.path}`; + })()} +

), }, @@ -402,13 +426,20 @@ export function LibraryRootsTable() { fields: [ { name: "name", label: "名称", kind: "text", required: true, placeholder: "例如：电影库" }, { name: "mediaKind", label: "内容类型", kind: "select", options: MEDIA_KIND }, + { + name: "storage", + label: "存储", + kind: "select", + options: storageOptions, + hint: "本机目录，或在「存储」里登记的 Alist / OpenList", + }, { name: "localPath", - label: "本地路径", + label: "路径", kind: "text", required: true, placeholder: "/media/movies", - hint: "服务进程能访问的绝对路径", + hint: "本机：服务进程能访问的绝对路径；Alist：Alist 里的路径，如 /115/电影", wide: true, }, { @@ -428,6 +459,7 @@ export function LibraryRootsTable() { initial: { name: "", mediaKind: "movie", + storage: "local", localPath: "", managementMode: "register_only", enabled: true, @@ -436,7 +468,8 @@ export function LibraryRootsTable() { fromRow: (r) => ({ name: r.name, mediaKind: r.mediaKind, - localPath: r.localPath, + storage: splitRootPath(r.localPath).storage, + localPath: splitRootPath(r.localPath).path, managementMode: r.managementMode, enabled: r.enabled, notes: r.notes ?? "", @@ -444,7 +477,7 @@ export function LibraryRootsTable() { toBody: (v) => ({ name: str(v.name), mediaKind: str(v.mediaKind), - localPath: str(v.localPath), + localPath: joinRootPath(str(v.storage) || "local", str(v.localPath)), managementMode: str(v.managementMode), enabled: v.enabled === true, notes: nullStr(v.notes), diff --git a/apps/web/app/components/settings/library.tsx b/apps/web/app/components/settings/library.tsx index fd9ac570..df6f32ac 100644 --- a/apps/web/app/components/settings/library.tsx +++ b/apps/web/app/components/settings/library.tsx @@ -2,13 +2,23 @@ import { LibraryRootsTable } from "./instances.tsx"; import { NamingSection } from "./naming.tsx"; import { ClassificationSection, OrganizeSourcesSection } from "./organize-sources.tsx"; import { SettingsBlock } from "./settings-block.tsx"; +import { StoragesTable } from "./storages.tsx"; -// 媒体库与目录:媒体库目录(含标记文件检查)、整理来源、分类规则、命名模板(全局 + 目录覆盖)。 +// 媒体库与目录:存储(Alist / OpenList)、媒体库目录(含标记文件检查)、整理来源、分类规则、命名模板(全局 + 目录覆盖)。 // 整理方式与刮削在「整理与刮削」。 export function LibrarySection() { return (
+ + + { const first = (id: string) => SECTIONS.find((s) => s.id === id)?.anchors[0]?.id; expect(first("downloaders")).toBe("instances"); expect(first("media-servers")).toBe("instances"); - expect(first("library")).toBe("roots"); + // 媒体库与目录:第一节是存储(Alist / OpenList 实例,媒体库目录可以登记在它上面),也是对象列表 + expect(first("library")).toBe("storages"); expect(first("agent")).toBe("upstream"); }); }); diff --git a/apps/web/app/components/settings/storages.test.ts b/apps/web/app/components/settings/storages.test.ts new file mode 100644 index 00000000..8e094039 --- /dev/null +++ b/apps/web/app/components/settings/storages.test.ts @@ -0,0 +1,80 @@ +import { StorageInstanceCreate, StorageInstanceUpdate } from "@umt/contracts"; +import { describe, expect, it } from "vitest"; +import { SECRET_CLEAR } from "~/lib/secret.ts"; +import { joinRootPath, splitRootPath } from "./instances.tsx"; +import { STORAGE_FIELDS, STORAGE_INITIAL, storageFromRow, storageToBody } from "./storages.tsx"; + +// 设置 → 存储(Alist / OpenList):认证方式决定显示哪个凭据框;凭据只写(留空不改、清除传 null); +// 高级选项留空用缺省;媒体库目录选 Alist 存储时路径存成 /@alist/<存储 id>/。 + +const visible = (values: Record) => + STORAGE_FIELDS.filter((f) => !f.visible || f.visible(values as never)).map((f) => f.name); + +const ROW = { + id: "00000000-0000-4000-8000-0000000000aa", + createdAt: "2026-10-06T00:00:00.000Z", + updatedAt: "2026-10-06T00:00:00.000Z", + name: "家里的 OpenList", + kind: "alist" as const, + flavor: "openlist" as const, + baseUrl: "http://10.0.0.2:5244", + rootPath: "/115", + authKind: "password" as const, + username: "alice", + secretRef: "00000000-0000-4000-8000-0000000000bb", + pathPasswordRef: null, + enabled: true, + options: { qps: 2, uploadAsTask: true }, + healthStatus: "healthy" as const, + lastHealthAt: null, + healthDetail: {}, + versionInfo: {}, +}; + +describe("设置:存储", () => { + it("认证方式决定凭据框:令牌 / 用户名和密码 / 访客", () => { + expect(visible({ authKind: "token" })).toContain("token"); + expect(visible({ authKind: "token" })).not.toContain("password"); + expect(visible({ authKind: "password" })).toEqual(expect.arrayContaining(["username", "password"])); + expect(visible({ authKind: "guest" })).not.toEqual(expect.arrayContaining(["token"])); + expect(visible({ authKind: "guest" })).not.toContain("username"); + expect(visible({ authKind: "guest" })).toContain("pathPassword"); + }); + + it("新建:令牌写进 credential,高级选项留空不传,请求通过契约校验", () => { + const body = storageToBody({ + ...STORAGE_INITIAL, + name: "OL", + baseUrl: "http://10.0.0.2:5244", + token: " tok-123 ", + qps: "", + retries: "5", + }); + expect(body).toMatchObject({ credential: "tok-123", rootPath: "/", username: null }); + expect(body.options).toEqual({ retries: 5, uploadAsTask: false, snapshotCheckFolderModtime: true }); + expect(StorageInstanceCreate.safeParse(body).success).toBe(true); + }); + + it("编辑:凭据留空不改;清除路径密码传 null;改认证方式后已存凭据不算数", () => { + const v = storageFromRow(ROW); + expect(v).toMatchObject({ authKind: "password", username: "alice", qps: "2", uploadAsTask: true }); + const keep = storageToBody(v); + expect(keep.credential).toBeUndefined(); + expect(keep.pathPassword).toBeUndefined(); + expect(StorageInstanceUpdate.safeParse(keep).success).toBe(true); + expect(storageToBody({ ...v, pathPassword: SECRET_CLEAR }).pathPassword).toBeNull(); + const pw = STORAGE_FIELDS.find((f) => f.name === "password")!; + expect(pw.secret!.set(v)).toBe(true); + expect(pw.secret!.set({ ...v, authKind: "token" })).toBe(false); + }); + + it("媒体库目录的存储:本机路径原样,Alist 路径编码成 /@alist//…", () => { + const id = "00000000-0000-4000-8000-0000000000aa"; + expect(splitRootPath("/media/movies")).toEqual({ storage: "local", path: "/media/movies" }); + expect(splitRootPath(`/@alist/${id}/115/电影`)).toEqual({ storage: id, path: "/115/电影" }); + expect(splitRootPath(`/@alist/${id}`)).toEqual({ storage: id, path: "/" }); + expect(joinRootPath(id, "115/电影/")).toBe(`/@alist/${id}/115/电影`); + expect(joinRootPath(id, "/")).toBe(`/@alist/${id}`); + expect(joinRootPath("local", "/media/movies")).toBe("/media/movies"); + }); +}); diff --git a/apps/web/app/components/settings/storages.tsx b/apps/web/app/components/settings/storages.tsx new file mode 100644 index 00000000..cdfabfda --- /dev/null +++ b/apps/web/app/components/settings/storages.tsx @@ -0,0 +1,268 @@ +import { + STORAGE_OPTION_DEFAULTS, + type StorageInstance, + StorageInstanceCreate, + StorageInstanceUpdate, + type StorageOptions, +} from "@umt/contracts"; +import { storages } from "~/api/instances.ts"; +import { StatusBadge } from "~/components/console/status-badge.tsx"; +import { secretPatch } from "~/lib/secret.ts"; +import { formatBytes } from "~/lib/utils.ts"; +import { Health, TestButton } from "./instances.tsx"; +import { type FieldSpec, type FormValues, nullStr, ResourceSection, str } from "./resource-section.tsx"; + +// 存储(Alist / OpenList,对照 MP 存储设置):地址、根路径、认证(永久令牌 / 账号密码 / 访客)、路径密码与限速。 +// 令牌、密码、路径密码只写不读;测试连接显示版本、登录用户与容量(拿不到容量显示「未知」)。 + +const FLAVORS = [ + { value: "openlist", label: "OpenList" }, + { value: "alist", label: "Alist" }, +]; +const AUTH_KINDS = [ + { value: "token", label: "令牌" }, + { value: "password", label: "用户名和密码" }, + { value: "guest", label: "访客（只读公开目录）" }, +]; + +/** 高级选项:表单里是字符串,空 = 缺省值 */ +const NUM_OPTIONS: { name: keyof StorageOptions; label: string; hint: string }[] = [ + { name: "qps", label: "每秒请求数", hint: `缺省 ${STORAGE_OPTION_DEFAULTS.qps}；挂了 115 等网盘时调低，0 不限` }, + { name: "concurrency", label: "并发数", hint: `缺省 ${STORAGE_OPTION_DEFAULTS.concurrency}；上传下载也占并发` }, + { name: "timeoutSec", label: "请求超时（秒）", hint: `缺省 ${STORAGE_OPTION_DEFAULTS.timeoutSec}` }, + { + name: "idleTimeoutSec", + label: "传输无进度超时（秒）", + hint: `缺省 ${STORAGE_OPTION_DEFAULTS.idleTimeoutSec}；上传下载这么久没有进展就中止重试`, + }, + { + name: "taskTimeoutMin", + label: "等待 Alist 任务（分钟）", + hint: `缺省 ${STORAGE_OPTION_DEFAULTS.taskTimeoutMin}；跨网盘复制、移动是 Alist 后台任务`, + }, + { name: "retries", label: "失败重试次数", hint: `缺省 ${STORAGE_OPTION_DEFAULTS.retries}` }, + { + name: "snapshotMaxDepth", + label: "目录监控深度", + hint: `缺省 ${STORAGE_OPTION_DEFAULTS.snapshotMaxDepth}；整理来源选 Alist 目录时往下看几层`, + }, +]; + +function optionsForm(o: StorageOptions): FormValues { + return { + ...Object.fromEntries(NUM_OPTIONS.map((f) => [f.name, typeof o[f.name] === "number" ? String(o[f.name]) : ""])), + uploadAsTask: o.uploadAsTask ?? STORAGE_OPTION_DEFAULTS.uploadAsTask, + snapshotCheckFolderModtime: o.snapshotCheckFolderModtime ?? STORAGE_OPTION_DEFAULTS.snapshotCheckFolderModtime, + }; +} +function optionsBody(v: FormValues): StorageOptions { + const out: Record = {}; + for (const f of NUM_OPTIONS) { + const raw = String(v[f.name] ?? "").trim(); + if (raw !== "" && Number.isFinite(Number(raw))) out[f.name] = Number(raw); + } + out.uploadAsTask = v.uploadAsTask === true; + out.snapshotCheckFolderModtime = v.snapshotCheckFolderModtime === true; + return out as StorageOptions; +} + +/** 已存凭据只对保存时的认证方式有效(改了认证方式就要重新填) */ +const credSet = (v: FormValues) => v["secret:set"] === true && v["secret:authKind"] === v.authKind; + +export const STORAGE_FIELDS: FieldSpec[] = [ + { name: "name", label: "名称", kind: "text", required: true, placeholder: "例如：家里的 OpenList" }, + { name: "flavor", label: "类型", kind: "select", options: FLAVORS, hint: "接口相同，只用于显示" }, + { + name: "baseUrl", + label: "地址", + kind: "url", + required: true, + placeholder: "http://192.168.1.10:5244", + hint: "Alist / OpenList 的访问地址，带端口和子路径", + }, + { + name: "rootPath", + label: "根路径", + kind: "text", + placeholder: "/", + hint: "本工具只用这个路径下的内容，如 /115/影视", + }, + { name: "authKind", label: "认证方式", kind: "select", options: AUTH_KINDS }, + { + name: "token", + label: "令牌", + kind: "secret", + secret: { set: credSet }, + hint: "Alist 后台「设置 → 其他 → 令牌」；令牌失效会在运行状态里提示", + visible: (v) => v.authKind === "token", + }, + { name: "username", label: "用户名", kind: "text", visible: (v) => v.authKind === "password" }, + { + name: "password", + label: "密码", + kind: "secret", + secret: { set: credSet }, + hint: "用来换取临时令牌，约两天自动续一次", + visible: (v) => v.authKind === "password", + }, + { + name: "pathPassword", + label: "路径密码", + kind: "secret", + secret: { set: (v) => v["secret:pathPassword"] === true, clearable: true }, + hint: "Alist 里给目录设了访问密码时填写", + }, + ...NUM_OPTIONS.map((f) => ({ + name: f.name, + label: f.label, + kind: "number" as const, + placeholder: "留空用缺省值", + hint: f.hint, + })), + { + name: "uploadAsTask", + label: "上传交给 Alist 后台任务", + kind: "switch", + hint: "开启后上传由 Alist 排队完成，本工具等任务结束再继续；缺省关闭（同 MP）", + }, + { + name: "snapshotCheckFolderModtime", + label: "目录监控按修改时间跳过", + kind: "switch", + hint: "子目录修改时间没变就不再列，能少请求网盘；网盘不更新目录时间时关闭", + }, + { name: "enabled", label: "启用", kind: "switch" }, +]; + +export const STORAGE_INITIAL: FormValues = { + name: "", + flavor: "openlist", + baseUrl: "", + rootPath: "/", + authKind: "token", + token: "", + username: "", + password: "", + pathPassword: "", + ...optionsForm({}), + enabled: true, +}; + +export function storageFromRow(r: StorageInstance): FormValues { + return { + name: r.name, + flavor: r.flavor, + baseUrl: r.baseUrl, + rootPath: r.rootPath, + authKind: r.authKind, + token: "", + username: r.username ?? "", + password: "", + pathPassword: "", + "secret:set": !!r.secretRef, + "secret:authKind": r.authKind, + "secret:pathPassword": !!r.pathPasswordRef, + ...optionsForm(r.options), + enabled: r.enabled, + }; +} + +export function storageToBody(v: FormValues) { + return { + name: str(v.name), + flavor: str(v.flavor), + baseUrl: str(v.baseUrl), + rootPath: str(v.rootPath) || "/", + authKind: str(v.authKind), + username: v.authKind === "password" ? nullStr(v.username) : null, + credential: + v.authKind === "token" ? secretPatch(v.token) : v.authKind === "password" ? secretPatch(v.password) : undefined, + pathPassword: secretPatch(v.pathPassword), + options: optionsBody(v), + enabled: v.enabled === true, + }; +} + +export function StoragesTable() { + const list = storages.useList(); + const create = storages.useCreate(); + const update = storages.useUpdate(); + const remove = storages.useRemove(); + const test = storages.useTest(); + return ( + + hooks={{ list, create, update, remove } as never} + config={{ + noun: "存储", + description: + "登记 Alist / OpenList 后，媒体库目录、整理来源和文件管理都能用它。网盘方向的上传、下载、复制和移动以接口成功、目标存在为准，不比对大小和哈希。", + emptyHint: "还没有登记 Alist / OpenList。", + deleteHint: "只删除登记和保存的凭据；被媒体库目录使用时不能删除。", + columns: [ + { + key: "name", + header: "名称", + cell: (r) => ( +
+

+ {r.name} + + {r.flavor === "alist" ? "Alist" : "OpenList"} + +

+

+ {r.baseUrl} + {r.rootPath !== "/" ? ` · ${r.rootPath}` : ""} +

+
+ ), + }, + { + key: "auth", + header: "认证", + hideBelow: "md", + cell: (r) => AUTH_KINDS.find((k) => k.value === r.authKind)?.label ?? r.authKind, + }, + { + key: "usage", + header: "容量", + hideBelow: "lg", + cell: (r) => { + const u = (r.healthDetail as { usage?: { total: number; free: number } | null }).usage; + return u ? `剩余 ${formatBytes(u.free)} / ${formatBytes(u.total)}` : "未知"; + }, + }, + { + key: "enabled", + header: "状态", + hideBelow: "sm", + cell: (r) => ( + {r.enabled ? "启用" : "停用"} + ), + }, + { key: "health", header: "健康", cell: (r) => }, + ], + rowActions: (r) => ( + { + const t = await test.mutateAsync(r.id); + const usage = t.usage ? `，剩余 ${formatBytes(t.usage.free)}` : "，容量未知"; + return { + ok: t.ok, + message: t.message, + version: t.ok ? `${t.version ?? "版本未知"}，用户 ${t.user ?? "—"}${usage}` : null, + }; + }} + /> + ), + fields: STORAGE_FIELDS, + initial: STORAGE_INITIAL, + fromRow: storageFromRow, + toBody: storageToBody, + createSchema: StorageInstanceCreate, + updateSchema: StorageInstanceUpdate, + }} + /> + ); +} diff --git a/apps/web/app/lib/api.ts b/apps/web/app/lib/api.ts index f17faf8a..335433e8 100644 --- a/apps/web/app/lib/api.ts +++ b/apps/web/app/lib/api.ts @@ -112,6 +112,10 @@ export async function apiFetch(path: string, arg: FetcherArg = {}): let reqBody: BodyInit | undefined; if (body instanceof FormData) { reqBody = body; + } else if (typeof Blob !== "undefined" && body instanceof Blob) { + // 原始字节(如 PUT /files/upload) + reqHeaders["Content-Type"] ??= "application/octet-stream"; + reqBody = body; } else if (body !== undefined && body !== null) { reqHeaders["Content-Type"] ??= "application/json"; reqBody = JSON.stringify(body); diff --git a/apps/web/app/routes/console/files.tsx b/apps/web/app/routes/console/files.tsx index 76ef9bda..802b685e 100644 --- a/apps/web/app/routes/console/files.tsx +++ b/apps/web/app/routes/console/files.tsx @@ -5,6 +5,7 @@ import { ArrowUp, ArrowUpAZ, ClipboardCopy, + Cloud, Copy, Download, Eye, @@ -47,6 +48,7 @@ import { isRootOffline, type ListQuery, refKey, + refreshRemoteDir, useFileList, useFileRoots, useProbe, @@ -64,11 +66,13 @@ import { RenameDialog, ScrapeDialog, TransferDialog, + UploadButton, } from "~/components/files/dialogs.tsx"; import { DirTree } from "~/components/files/dir-tree.tsx"; import { FileDetail } from "~/components/files/file-detail.tsx"; import { DRAG_MIME, FileList } from "~/components/files/file-list.tsx"; import { + displayPath, emptySelection, FILTER_CHIPS, groupRoots, @@ -221,7 +225,7 @@ function TreeRail({ {groupRoots(roots).flatMap((g) => g.items.map((r) => { - const Icon = r.kind === "download" ? Download : HardDrive; + const Icon = r.kind === "download" ? Download : r.kind === "storage" ? Cloud : HardDrive; return ( + {root ? : null}
{searchBox} diff --git a/apps/web/app/routes/console/status.tsx b/apps/web/app/routes/console/status.tsx index 7dc1d24b..74db75b6 100644 --- a/apps/web/app/routes/console/status.tsx +++ b/apps/web/app/routes/console/status.tsx @@ -473,7 +473,7 @@ function Mounts({ d }: { d: SystemHealth }) { return ( {d.mounts.length === 0 ? ( -

没有启用的媒体库目录或下载目录映射。

+

没有启用的媒体库目录、下载目录映射或 Alist 存储。

) : (
    {d.mounts.map((m) => ( @@ -481,18 +481,28 @@ function Mounts({ d }: { d: SystemHealth }) {

    - {m.kind === "library_root" ? "媒体库目录" : "下载目录"} + {m.kind === "library_root" + ? "媒体库目录" + : m.kind === "storage_instance" + ? "Alist 存储" + : "下载目录"} {m.name}

    {m.path} - {m.sentinelChecked ? "" : " · 只查目录可读"} + {m.kind === "storage_instance" ? " · 连接与根路径" : m.sentinelChecked ? "" : " · 只查目录可读"}

    {m.status === "online" ? "在线" : m.status === "timeout" ? "超时" : "离线"} - {m.reason && m.status !== "timeout" ? ` · ${MOUNT_REASON[m.reason] ?? m.reason}` : ""} + {m.reason && m.status !== "timeout" + ? ` · ${ + m.kind === "storage_instance" && m.reason === "permission_denied" + ? "令牌失效或没有权限" + : (MOUNT_REASON[m.reason] ?? m.reason) + }` + : ""} ))} diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 1aa346ab..d9d6a85e 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -1085,9 +1085,10 @@ export const SETTINGS_SECTIONS: readonly SettingsSectionDef[] = [ id: "library", group: "flow", label: "媒体库与目录", - description: "媒体库目录、整理来源、分类规则、命名模板", + description: "存储、媒体库目录、整理来源、分类规则、命名模板", legacyTabs: [{ tab: "library-roots", anchor: "roots" }], anchors: [ + { id: "storages", label: "存储", keywords: ["Alist", "OpenList", "网盘", "令牌"] }, { id: "roots", label: "媒体库目录", keywords: ["标记文件", "检查目录"] }, { id: "sources", label: "整理来源" }, { id: "classification", label: "分类规则", keywords: ["二级分类"] }, @@ -1102,7 +1103,7 @@ export const SETTINGS_SECTIONS: readonly SettingsSectionDef[] = [ "naming.variety", "naming.lockFileNameForBound", ], - resources: ["library-roots"], + resources: ["storage-instances", "library-roots"], }, { id: "organize", From fed2202e745b87be376c1051975e38e077ed40db Mon Sep 17 00:00:00 2001 From: root Date: Wed, 7 Oct 2026 00:01:21 +0800 Subject: [PATCH 08/11] =?UTF-8?q?test(storage):=20=E7=9C=9F=E5=AE=9E=20Ope?= =?UTF-8?q?nList=20=E5=BE=80=E8=BF=94=E6=B5=8B=E8=AF=95=E4=B8=8E=20alist-e?= =?UTF-8?q?2e=20=E9=AA=8C=E6=94=B6=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - packages/adapters alist.real.test.ts:上传 / 冲突 / 同驱动与跨驱动复制移动 / 下载比对(环境变量开启) - apps/server alist-organize.real.test.ts:本地 → OpenList(整理复制)→ 本地(移动)→ OpenList 内整理 - scripts/alist-e2e.ts:对任意地址 + 令牌或账号密码跑往返,退出码表示是否全部通过 - 写后目标暂不可见时刷新父目录列表(OpenList 的 get 不认 refresh) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei --- .../test/storage/alist-organize.real.test.ts | 158 ++++++++++++++++++ .../adapters/src/alist/alist.real.test.ts | 102 +++++++++++ packages/adapters/src/alist/client.ts | 10 +- scripts/alist-e2e.ts | 140 ++++++++++++++++ 4 files changed, 408 insertions(+), 2 deletions(-) create mode 100644 apps/server/test/storage/alist-organize.real.test.ts create mode 100644 packages/adapters/src/alist/alist.real.test.ts create mode 100644 scripts/alist-e2e.ts diff --git a/apps/server/test/storage/alist-organize.real.test.ts b/apps/server/test/storage/alist-organize.real.test.ts new file mode 100644 index 00000000..ba706598 --- /dev/null +++ b/apps/server/test/storage/alist-organize.real.test.ts @@ -0,0 +1,158 @@ +import { randomBytes } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { type AlistAuth, AlistClient } from "@umt/adapters"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { executeRecord } from "../../src/modules/acquisition-organize/services/organize.ts"; +import { overrideStorage } from "../../src/storage/registry.ts"; +import { toVPath } from "../../src/storage/vpath.ts"; +import { createFmHarness, type FmHarness, key, pgAvailable, seedEntity } from "../file-manager/harness.ts"; + +// 真实 OpenList 上跑一次整理往返(环境变量开启,缺省跳过;变量同 packages/adapters/src/alist/alist.real.test.ts): +// 本地下载目录 → 整理(复制)到 OpenList 上的媒体库 → 从 OpenList 整理(移动)回本地媒体库 → OpenList 内移动。 + +const URL_ = Deno.env.get("UMT_ALIST_REAL_URL"); +const ROOT = Deno.env.get("UMT_ALIST_REAL_ROOT"); +const auth: AlistAuth | null = Deno.env.get("UMT_ALIST_REAL_TOKEN") + ? { kind: "token", token: Deno.env.get("UMT_ALIST_REAL_TOKEN")! } + : Deno.env.get("UMT_ALIST_REAL_USER") + ? { + kind: "password", + username: Deno.env.get("UMT_ALIST_REAL_USER")!, + password: Deno.env.get("UMT_ALIST_REAL_PASSWORD") ?? "", + } + : null; +const enabled = !!URL_ && !!ROOT && !!auth && (await pgAvailable()); + +describe.skipIf(!enabled)("真实 OpenList:整理往返 本地 → OpenList → 本地", () => { + let h: FmHarness; + let client: AlistClient; + let storageId: string; + let base: string; + let libId: string; + const body = `real-video-${randomBytes(64).toString("hex")}`; + beforeAll(async () => { + h = await createFmHarness("alist_real", {}, { runTransfers: false }); + client = new AlistClient({ baseUrl: URL_!, auth: auth! }); + base = `${ROOT!.replace(/\/+$/, "")}/umt-org-${randomBytes(3).toString("hex")}`; + await client.mkdir(`${base}/library`); + await client.mkdir(`${base}/incoming`); + const row = await h.db + .insertInto("storage_instance") + .values({ name: "真实 OpenList", base_url: URL_!, root_path: base }) + .returning("id") + .executeTakeFirstOrThrow(); + storageId = row.id; + overrideStorage(storageId, { client, name: "真实 OpenList", rootPath: base }); + libId = ( + await h.db + .insertInto("library_root") + .values({ + name: "OpenList 媒体库", + media_kind: "mixed", + local_path: toVPath(storageId, `${base}/library`), + management_mode: "managed", + sentinel_file: null, + }) + .returning("id") + .executeTakeFirstOrThrow() + ).id; + }, 60_000); + afterAll(async () => { + await client?.removePath(base).catch(() => undefined); + await h?.close(); + }); + + async function organize(items: unknown[], action: "copy" | "move", targetLibraryRootId: string) { + const dry = await h.api("POST", "/files/organize", { + idempotencyKey: key(), + items, + action, + targetLibraryRootId, + dryRun: true, + }); + expect(dry.body.data.groups[0].error, JSON.stringify(dry.body)).toBeNull(); + const exec = await h.api("POST", "/files/organize", { + idempotencyKey: key(), + items, + action, + targetLibraryRootId, + dryRun: false, + planIds: [dry.body.data.groups[0].planId], + }); + const recordId = exec.body.data.recordIds[0] as string; + await executeRecord(h.module.deps().ao, recordId); + const rec = await h.db + .selectFrom("organization_record") + .select(["status"]) + .where("id", "=", recordId) + .executeTakeFirstOrThrow(); + const main = await h.db + .selectFrom("organization_file_result") + .select(["target_path", "checksum", "status"]) + .where("organization_record_id", "=", recordId) + .where("role", "=", "main") + .executeTakeFirstOrThrow(); + return { status: rec.status, main }; + } + + it("本地 → OpenList(复制)→ 本地(移动)→ OpenList 内移动", async () => { + const film = await seedEntity(h.db, { kind: "movie", title: "Real Round Trip", year: 2024 }); + const name = "Real.Round.Trip.2024.1080p.WEB-DL.mkv"; + await writeFile(join(h.dl, name), body); + + const up = await organize([{ root: h.dlRoot, path: name, mediaEntityId: film }], "copy", libId); + console.log(`[real] 本地 → OpenList 复制：${up.status}，目标 ${up.main.target_path}，${up.main.checksum}`); + expect(up.status).toBe("succeeded"); + const remoteFile = `${base}/library/${up.main.target_path}`; + expect((await client.get(remoteFile, { refresh: true })).size).toBe(body.length); + + // 从 OpenList 移回本地媒体库(文件管理「移动到」:下载成功后删远端源,已登记位置跟随到本地) + const rel = `library/${up.main.target_path}`; + const src = { root: `st:${storageId}`, path: rel }; + const dryMv = await h.api("POST", "/files/move", { + idempotencyKey: key(), + items: [src], + target: { root: h.libRoot, path: "" }, + mode: "move", + dryRun: true, + }); + expect(dryMv.status, JSON.stringify(dryMv.body)).toBe(200); + const mv = await h.api("POST", "/files/move", { + idempotencyKey: key(), + items: [src], + target: { root: h.libRoot, path: "" }, + mode: "move", + dryRun: false, + planId: dryMv.body.data.planId, + }); + expect(mv.status, JSON.stringify(mv.body)).toBe(200); + const recordId = mv.body.data.executed.recordId as string; + await executeRecord(h.module.deps().ao, recordId); + const back = await h.db + .selectFrom("organization_record") + .select("status") + .where("id", "=", recordId) + .executeTakeFirstOrThrow(); + const localName = up.main.target_path!.split("/").pop()!; + console.log(`[real] OpenList → 本地 移动：${back.status}，目标 ${localName}`); + expect(back.status).toBe("succeeded"); + expect(await readFile(join(h.lib, localName), "utf8")).toBe(body); + expect(await client.stat(remoteFile, { refresh: true })).toBeNull(); + + // OpenList 内整理(同实例 API move):另一部作品放在 OpenList 的 incoming + const film2 = await seedEntity(h.db, { kind: "movie", title: "Real Inside Move", year: 2023 }); + const name2 = "Real.Inside.Move.2023.1080p.WEB-DL.mkv"; + await writeFile(join(h.dl, name2), body); + await client.upload(join(h.dl, name2), `${base}/incoming/${name2}`); + const inside = await organize( + [{ root: `st:${storageId}`, path: `incoming/${name2}`, mediaEntityId: film2 }], + "move", + libId, + ); + console.log(`[real] OpenList 内移动：${inside.status}，目标 ${inside.main.target_path}`); + expect(inside.status).toBe("succeeded"); + expect(await client.stat(`${base}/incoming/${name2}`, { refresh: true })).toBeNull(); + expect((await client.get(`${base}/library/${inside.main.target_path}`, { refresh: true })).size).toBe(body.length); + }, 120_000); +}); diff --git a/packages/adapters/src/alist/alist.real.test.ts b/packages/adapters/src/alist/alist.real.test.ts new file mode 100644 index 00000000..5e670932 --- /dev/null +++ b/packages/adapters/src/alist/alist.real.test.ts @@ -0,0 +1,102 @@ +import { createHash, randomBytes } from "node:crypto"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { type AlistAuth, AlistClient } from "./client.ts"; + +// 真实 Alist / OpenList(环境变量开启,缺省跳过): +// UMT_ALIST_REAL_URL 地址,如 http://127.0.0.1:55244 +// UMT_ALIST_REAL_TOKEN 永久令牌;或 UMT_ALIST_REAL_USER + UMT_ALIST_REAL_PASSWORD +// UMT_ALIST_REAL_ROOT 可写测试目录(会在里面建 umt-it-<随机>/,结束删除),如 /s1 +// UMT_ALIST_REAL_ROOT2 可选:另一个挂载(驱动)下的可写目录,测跨驱动复制 / 移动(异步任务) +// 跑法:docs/runbook/storage-alist.md「真机验收」。 + +const URL_ = Deno.env.get("UMT_ALIST_REAL_URL"); +const ROOT = Deno.env.get("UMT_ALIST_REAL_ROOT"); +const ROOT2 = Deno.env.get("UMT_ALIST_REAL_ROOT2"); +const auth: AlistAuth | null = Deno.env.get("UMT_ALIST_REAL_TOKEN") + ? { kind: "token", token: Deno.env.get("UMT_ALIST_REAL_TOKEN")! } + : Deno.env.get("UMT_ALIST_REAL_USER") + ? { + kind: "password", + username: Deno.env.get("UMT_ALIST_REAL_USER")!, + password: Deno.env.get("UMT_ALIST_REAL_PASSWORD") ?? "", + } + : null; +const enabled = !!URL_ && !!ROOT && !!auth; +const sha = (b: Uint8Array) => createHash("sha256").update(b).digest("hex"); + +describe.skipIf(!enabled)("真实 Alist / OpenList:本地 → Alist → 本地往返", () => { + let c: AlistClient; + let dir: string; + let remote: string; + let remote2: string | null = null; + const data = randomBytes(3 * 1024 * 1024 + 123); + beforeAll(async () => { + c = new AlistClient({ baseUrl: URL_!, auth: auth!, qps: 10, concurrency: 4 }); + dir = await mkdtemp(join(tmpdir(), "umt-alist-real-")); + remote = `${ROOT!.replace(/\/+$/, "")}/umt-it-${randomBytes(3).toString("hex")}`; + if (ROOT2) remote2 = `${ROOT2.replace(/\/+$/, "")}/umt-it-${randomBytes(3).toString("hex")}`; + await writeFile(join(dir, "source.bin"), data); + }); + afterAll(async () => { + await c?.removePath(remote).catch(() => undefined); + if (remote2) await c?.removePath(remote2).catch(() => undefined); + await rm(dir, { recursive: true, force: true }); + }); + + it("连接:版本 / 用户 / 根目录", async () => { + const v = await c.version(); + const me = await c.me(); + console.log(`[real] Alist 版本 ${v}，用户 ${me.username}`); + expect((await c.get(ROOT!)).isDir).toBe(true); + }); + + it("上传(临时名 → 改名、带秒传头)→ 同名冲突不覆盖", async () => { + let last = 0; + const r = await c.upload(join(dir, "source.bin"), `${remote}/电影 A/source.bin`, { onProgress: (d) => (last = d) }); + console.log(`[real] 上传 ${r.bytes} 字节，sha256 ${r.hashes.sha256.slice(0, 12)}…，进度回调到 ${last}`); + expect(r.obj.size).toBe(data.length); + expect(last).toBe(data.length); + const names = (await c.list(`${remote}/电影 A`, { refresh: true })).map((o) => o.name); + expect(names).toEqual(["source.bin"]); + await expect(c.upload(join(dir, "source.bin"), `${remote}/电影 A/source.bin`)).rejects.toMatchObject({ + kind: "conflict", + }); + }); + + it("同驱动复制(改名走暂存目录)/ 移动 / 改名", async () => { + const cp = await c.copyFile(`${remote}/电影 A/source.bin`, `${remote}/library/Movie (2020).bin`); + expect(cp.size).toBe(data.length); + const mv = await c.moveFile(`${remote}/library/Movie (2020).bin`, `${remote}/library2/Moved.bin`); + expect(mv.path).toBe(`${remote}/library2/Moved.bin`); + expect(await c.stat(`${remote}/library/Movie (2020).bin`)).toBeNull(); + const lib = (await c.list(`${remote}/library`, { refresh: true })).map((o) => o.name); + expect(lib.filter((n) => n.startsWith(".umt-stage-"))).toEqual([]); + console.log(`[real] Alist 内复制、移动完成，暂存目录已清理`); + }); + + it.skipIf(!ROOT2)("跨驱动复制 / 移动:轮询 Alist 任务到完成", async () => { + const cp = await c.copyFile(`${remote}/library2/Moved.bin`, `${remote2}/cross/Copied.bin`); + expect(cp.size).toBe(data.length); + const mv = await c.moveFile(`${remote2}/cross/Copied.bin`, `${remote}/back/Back.bin`); + expect(mv.size).toBe(data.length); + console.log(`[real] 跨驱动复制、移动完成(任务轮询)`); + }); + + it("下载回本地:内容与源一致;本地已存在冲突", async () => { + const out = join(dir, "back.bin"); + const r = await c.download(`${remote}/library2/Moved.bin`, out); + const got = new Uint8Array(await readFile(out)); + console.log(`[real] 下载 ${r.bytes} 字节，与源 sha256 一致：${sha(got) === sha(data)}`); + expect(sha(got)).toBe(sha(data)); + await expect(c.download(`${remote}/library2/Moved.bin`, out)).rejects.toMatchObject({ kind: "conflict" }); + }); + + it("不存在 ≠ 失败;容量拿不到返回 null 不报错", async () => { + await expect(c.get(`${remote}/nope.bin`)).rejects.toMatchObject({ kind: "not_found" }); + const u = await c.usage(ROOT!); + console.log(`[real] 容量：${u ? `${u.free}/${u.total}` : "未知"}`); + }); +}); diff --git a/packages/adapters/src/alist/client.ts b/packages/adapters/src/alist/client.ts index 74eac551..edcabceb 100644 --- a/packages/adapters/src/alist/client.ts +++ b/packages/adapters/src/alist/client.ts @@ -596,11 +596,17 @@ export class AlistClient { async waitExists(path: string, opts: { timeoutMs?: number; signal?: AbortSignal } = {}): Promise { const deadline = this.o.now() + (opts.timeoutMs ?? 60_000); let delay = 500; + const p = normAlistPath(path); for (;;) { opts.signal?.throwIfAborted(); - const o = await this.stat(path, { refresh: true }); + const o = await this.stat(p, { refresh: true }); if (o) return o; - if (this.o.now() >= deadline) throw new AlistError("not_found", `写入后目标仍不存在：${normAlistPath(path)}`); + // OpenList 的 get 不认 refresh:刷新父目录列表让缓存失效,再按列表结果判断 + const hit = (await this.list(posix.dirname(p), { refresh: true }).catch(() => [])).find( + (x) => x.name === posix.basename(p), + ); + if (hit) return hit; + if (this.o.now() >= deadline) throw new AlistError("not_found", `写入后目标仍不存在：${p}`); await this.o.sleep(delay); delay = Math.min(delay * 2, 10_000); } diff --git a/scripts/alist-e2e.ts b/scripts/alist-e2e.ts new file mode 100644 index 00000000..30e9af9c --- /dev/null +++ b/scripts/alist-e2e.ts @@ -0,0 +1,140 @@ +// Alist / OpenList 往返验收脚本(docs/runbook/storage-alist.md「真机验收」):对任意地址 + 令牌(或账号密码) +// 用本项目的适配器跑一遍:连接 → 建目录 → 上传(秒传头、临时名改名、进度)→ 同名冲突不覆盖 → 同驱动复制(改名) +// → 移动 → 可选跨驱动复制 / 移动(Alist 异步任务)→ 下载回本地比对 sha256 → 不存在 ≠ 失败 → 清理。 +// +// 用法: +// deno run -A scripts/alist-e2e.ts --url http://127.0.0.1:5244 --token <令牌> --root /s1 [--root2 /s2] [--size-mb 8] +// deno run -A scripts/alist-e2e.ts --url ... --user admin --password <密码> --root /s1 +// 也可用环境变量 UMT_ALIST_REAL_URL / _TOKEN / _USER / _PASSWORD / _ROOT / _ROOT2。退出码 0 = 全部通过。 +import { createHash, randomBytes } from "node:crypto"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { parseArgs } from "node:util"; +import { type AlistAuth, AlistClient, AlistError, redactAlistText } from "../packages/adapters/src/alist/index.ts"; + +const { values: a } = parseArgs({ + args: Deno.args, + options: { + url: { type: "string" }, + token: { type: "string" }, + user: { type: "string" }, + password: { type: "string" }, + root: { type: "string" }, + root2: { type: "string" }, + "size-mb": { type: "string" }, + keep: { type: "boolean" }, + }, +}); +const env = (k: string) => Deno.env.get(`UMT_ALIST_REAL_${k}`); +const url = a.url ?? env("URL"); +const token = a.token ?? env("TOKEN"); +const user = a.user ?? env("USER"); +const password = a.password ?? env("PASSWORD"); +const root = (a.root ?? env("ROOT") ?? "").replace(/\/+$/, ""); +const root2 = (a.root2 ?? env("ROOT2") ?? "").replace(/\/+$/, ""); +if (!url || !root || (!token && !user)) { + console.error( + "用法：--url <地址> (--token <令牌> | --user <用户> --password <密码>) --root <可写目录> [--root2 <另一驱动目录>]", + ); + Deno.exit(2); +} +const auth: AlistAuth = token + ? { kind: "token", token } + : { kind: "password", username: user!, password: password ?? "" }; +const secrets = [token, password]; +const c = new AlistClient({ baseUrl: url, auth }); +const tag = `umt-e2e-${randomBytes(3).toString("hex")}`; +const base = `${root}/${tag}`; +const base2 = root2 ? `${root2}/${tag}` : null; +const size = Math.max(1, Number(a["size-mb"] ?? 8)) * 1024 * 1024 + 7; +const sha = (b: Uint8Array) => createHash("sha256").update(b).digest("hex"); + +let failed = 0; +async function step(name: string, fn: () => Promise) { + const t0 = performance.now(); + try { + const note = await fn(); + console.log(`✓ ${name}（${Math.round(performance.now() - t0)} ms）${note ? `：${note}` : ""}`); + } catch (e) { + failed++; + const msg = e instanceof AlistError ? `${e.kind}：${e.message}` : String((e as Error)?.message ?? e); + console.log(`✗ ${name}（${Math.round(performance.now() - t0)} ms）：${redactAlistText(msg, secrets)}`); + } +} +const expectKind = async (p: Promise, kind: string) => { + try { + await p; + } catch (e) { + if (e instanceof AlistError && e.kind === kind) return; + throw e; + } + throw new Error(`应当失败（${kind}），却成功了`); +}; + +const dir = await mkdtemp(join(tmpdir(), "umt-alist-e2e-")); +const data = randomBytes(size); +await writeFile(join(dir, "source.bin"), data); +console.log( + `Alist 往返验收：${url} 测试目录 ${base}${base2 ? `、${base2}` : ""} 文件 ${(size / 1048576).toFixed(1)} MiB`, +); + +await step("连接（版本、用户、根目录）", async () => { + const v = await c.version(); + const me = await c.me(); + if (!(await c.get(root)).isDir) throw new Error(`${root} 不是目录`); + return `${v ?? "版本未知"}，用户 ${me.username}`; +}); +await step("建目录", async () => { + await c.mkdir(`${base}/电影 A`); +}); +await step("上传（秒传头、临时名 → 改名、进度回调）", async () => { + let last = 0; + const r = await c.upload(join(dir, "source.bin"), `${base}/电影 A/source.bin`, { onProgress: (d) => (last = d) }); + if (last !== size) throw new Error(`进度只到 ${last}/${size}`); + return `${r.bytes} 字节，sha256 ${r.hashes.sha256.slice(0, 12)}…`; +}); +await step("同名不覆盖（上传 / 复制都报冲突）", async () => { + await expectKind(c.upload(join(dir, "source.bin"), `${base}/电影 A/source.bin`), "conflict"); + await c.mkdir(`${base}/dup`); + await c.copyFile(`${base}/电影 A/source.bin`, `${base}/dup/source.bin`); + await expectKind(c.copyFile(`${base}/电影 A/source.bin`, `${base}/dup/source.bin`), "conflict"); +}); +await step("同驱动复制并改名（暂存目录）", async () => { + const o = await c.copyFile(`${base}/电影 A/source.bin`, `${base}/library/Movie (2020).bin`); + const left = (await c.list(`${base}/library`, { refresh: true })).filter((x) => x.name.startsWith(".umt-stage-")); + if (left.length) throw new Error("暂存目录没清理"); + return `${o.size} 字节`; +}); +await step("同驱动移动", async () => { + await c.moveFile(`${base}/library/Movie (2020).bin`, `${base}/moved/Moved.bin`); + if (await c.stat(`${base}/library/Movie (2020).bin`, { refresh: true })) throw new Error("源还在"); +}); +if (base2) { + await step("跨驱动复制 / 移动（等 Alist 任务完成）", async () => { + await c.copyFile(`${base}/moved/Moved.bin`, `${base2}/cross/Copied.bin`); + await c.moveFile(`${base2}/cross/Copied.bin`, `${base}/back/Back.bin`); + return "完成"; + }); +} +await step("下载回本地并比对", async () => { + const out = join(dir, "back.bin"); + const r = await c.download(`${base}/moved/Moved.bin`, out); + const ok = sha(new Uint8Array(await readFile(out))) === sha(data); + if (!ok) throw new Error("内容与源不一致"); + return `${r.bytes} 字节，sha256 一致`; +}); +await step("不存在 ≠ 失败；容量拿不到不报错", async () => { + await expectKind(c.get(`${base}/nope.bin`), "not_found"); + const u = await c.usage(root); + return u ? `剩余 ${(u.free / 1073741824).toFixed(1)} GiB / ${(u.total / 1073741824).toFixed(1)} GiB` : "容量未知"; +}); +if (!a.keep) { + await step("清理测试目录", async () => { + await c.removePath(base); + if (base2) await c.removePath(base2); + }); +} +await rm(dir, { recursive: true, force: true }); +console.log(failed ? `\n${failed} 步失败` : "\n全部通过"); +Deno.exit(failed ? 1 : 0); From adbb9b59a0f0a2d1d441e27396b36771b7aba931 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 7 Oct 2026 00:04:19 +0800 Subject: [PATCH 09/11] =?UTF-8?q?docs(storage):=20Alist=20=E4=BD=BF?= =?UTF-8?q?=E7=94=A8=E4=B8=8E=E7=9C=9F=E6=9C=BA=E9=AA=8C=E6=94=B6=20runboo?= =?UTF-8?q?k,=E6=95=B4=E7=90=86=20/=20=E6=96=87=E4=BB=B6=E7=AE=A1=E7=90=86?= =?UTF-8?q?=20/=20MP=20=E5=AF=B9=E7=85=A7=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增 docs/runbook/storage-alist.md:登记、行为要点、临时 OpenList 容器真机验收步骤、排查 - organize.md 加「Alist / OpenList 存储」;moviepilot-filemanager.md 末节改为已做; v3 对照保留 rclone 不照抄、网盘改为 Alist 已接;file-manager-v2 存储切换与目的存储两行更新 - 设计文档补实现落点与偏离;矩阵测试补原地整理与目标已存在替换 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei --- .../storage/alist-organize-matrix.test.ts | 63 +++++++++++ docs/design/file-manager-v2.md | 4 +- docs/design/storage-alist.md | 28 ++++- docs/porting/moviepilot-filemanager.md | 16 ++- docs/porting/moviepilot-v3-comparison.md | 2 +- docs/runbook/organize.md | 13 ++- docs/runbook/storage-alist.md | 107 ++++++++++++++++++ 7 files changed, 225 insertions(+), 8 deletions(-) create mode 100644 docs/runbook/storage-alist.md diff --git a/apps/server/test/storage/alist-organize-matrix.test.ts b/apps/server/test/storage/alist-organize-matrix.test.ts index 22e56c2a..d6196702 100644 --- a/apps/server/test/storage/alist-organize-matrix.test.ts +++ b/apps/server/test/storage/alist-organize-matrix.test.ts @@ -253,6 +253,69 @@ describe.skipIf(skip)("整理矩阵:本地 ↔ Alist ↔ Alist", () => { a.mock.failNext(0); }); + it("目标已存在选替换:旧文件移进 Alist 上的回收站,新文件上传到位", async () => { + const f = await film("local"); + const first = await organize(f, "copy", alib.libraryRootId); + const target = `/s1/library/${first.main.target_path}`; + a.mock.put(target, "old-version-content"); + await writeFile(join(h.dl, f.path), `${f.body}-v2`); + const items = [{ root: f.root, path: f.path, mediaEntityId: f.id }]; + const base = { items, action: "copy", targetLibraryRootId: alib.libraryRootId, onExisting: "replace_with_verify" }; + const dry = await h.api("POST", "/files/organize", { ...base, idempotencyKey: key(), dryRun: true }); + expect(dry.body.data.groups[0].files[0]).toMatchObject({ existing: "replace", action: "replace" }); + const exec = await h.api("POST", "/files/organize", { + ...base, + idempotencyKey: key(), + dryRun: false, + planIds: [dry.body.data.groups[0].planId], + }); + const recordId = exec.body.data.recordIds[0] as string; + await executeRecord(h.module.deps().ao, recordId); + const rec = await h.db + .selectFrom("organization_record") + .select("status") + .where("id", "=", recordId) + .executeTakeFirstOrThrow(); + expect(rec.status).toBe("succeeded"); + expect(text(a.mock.read(target))).toBe(`${f.body}-v2`); + const trash = await h.db + .selectFrom("trash_item") + .select(["trash_path", "original_path"]) + .where("original_path", "=", first.main.target_path!) + .executeTakeFirstOrThrow(); + expect(trash.trash_path.startsWith(a.v("/s1/library/.umt-trash/"))).toBe(true); + const parked = trash.trash_path.replace(a.v(""), ""); + expect(text(a.mock.read(parked))).toBe("old-version-content"); + }); + + it("原地整理(Alist 媒体库里的文件):只登记,不动文件", async () => { + n++; + const id = await seedEntity(h.db, { kind: "movie", title: "In Place Film", year: 2015 }); + a.mock.put("/s1/library/In Place Film (2015)/In.Place.Film.2015.1080p.mkv", "in-place-video"); + const items = [{ root: alib.root, path: "In Place Film (2015)/In.Place.Film.2015.1080p.mkv", mediaEntityId: id }]; + const base = { items, action: "in_place", minSizeBytes: 0, scrapeAfterImport: false }; + const dry = await h.api("POST", "/files/organize", { ...base, idempotencyKey: key(), dryRun: true }); + expect(dry.body.data.groups[0].error, JSON.stringify(dry.body)).toBeNull(); + const exec = await h.api("POST", "/files/organize", { + ...base, + idempotencyKey: key(), + dryRun: false, + planIds: [dry.body.data.groups[0].planId], + }); + const recordId = exec.body.data.recordIds[0] as string; + await executeRecord(h.module.deps().ao, recordId); + const rec = await h.db + .selectFrom("organization_record") + .select(["status", "action"]) + .where("id", "=", recordId) + .executeTakeFirstOrThrow(); + expect(rec).toEqual({ status: "succeeded", action: "in_place" }); + expect(text(a.mock.read("/s1/library/In Place Film (2015)/In.Place.Film.2015.1080p.mkv"))).toBe("in-place-video"); + expect((await assetAt(alib.libraryRootId, "In Place Film (2015)/In.Place.Film.2015.1080p.mkv"))?.status).toBe( + "available", + ); + }); + it("撤销:删除整理记录连带媒体库文件 → Alist 上移进回收站(可还原)", async () => { const f = await film("local"); const r = await organize(f, "copy", alib.libraryRootId); diff --git a/docs/design/file-manager-v2.md b/docs/design/file-manager-v2.md index acca7d41..9fa68557 100644 --- a/docs/design/file-manager-v2.md +++ b/docs/design/file-manager-v2.md @@ -32,7 +32,7 @@ | # | MP 功能点 | 我们 | v2 决定 | | --- | --- | --- | --- | -| 1 | 存储切换菜单(local/115/阿里/Alist/SMB/rclone) | ≠ 左栏按"媒体库/下载目录"列登记目录 | 改:登记目录 = 库根 + 下载器路径映射,不接网盘(REQ §7.1.1) | +| 1 | 存储切换菜单(local/115/阿里/Alist/SMB/rclone) | ≠ 左栏按"媒体库/下载目录"列登记目录 | 改:登记目录 = 库根 + 下载器路径映射;2026-10-07 起多一组「Alist 存储」(见 `storage-alist.md`),115 / SMB / rclone 不接 | | 2 | 初始位置 = 下载目录公共前缀,记住上次存储 | ≠ 默认第一个在线库根 | 改:上次位置(localStorage)> 第一个下载目录 > 第一个库根。整理从下载目录开始 | | 3 | 面包屑(桌面全段,移动端只显示存储名) | ✓ 可拖放 | 照抄 + 移动端折叠成"…/父/当前" | | 4 | "上一级"按钮 | ✗ 只有 Backspace | 照抄:工具栏加"上一级" | @@ -63,7 +63,7 @@ | 29 | 递归重命名目录内媒体(一个失败即中止) | ✗ | 改:放进向导"原地按模板改名"模式,先预览(阶段 3) | | 30 | 删除确认"是否确认删除{name}?" | ✓+ 回收站 + 引用检查 | 保留 | | 31 | 新建目录对话框 | ✓+ 名称校验 | 保留 | -| 32 | 整理:目的存储 | — | 不做:只有本机路径 | +| 32 | 整理:目的存储 | ✓ | 媒体库目录可登记在 Alist 上,选目标库根即选存储(见 `storage-alist.md`) | | 33 | 整理方式 copy/move/link/softlink | ≠ 复制/移动 | 保留两项;硬链接灰显"W5" | | 34 | 目的路径(自动 / 目录配置下拉 / 手输) | ≠ 目标库根(契约有 targetDir,界面没有) | 改:自动 / 指定库根 / 指定目录(§6.2) | | 35 | 按源路径推荐目标(`/manual/target-path`) | ≠ 按作品绑定或类型 | 保留我们的:已绑定作品沿用原目录 | diff --git a/docs/design/storage-alist.md b/docs/design/storage-alist.md index 2942cf97..22fbe262 100644 --- a/docs/design/storage-alist.md +++ b/docs/design/storage-alist.md @@ -119,7 +119,7 @@ new AlistClient({ baseUrl, auth: {kind:"token",token} | {kind:"password",usernam ## 3. 存储抽象（服务端） - 表 `storage_instance`（迁移 0090）：`id, name, kind('alist'), flavor('alist'|'openlist'), base_url, root_path, auth_kind('token'|'password'|'guest'), username, secret_ref, path_password_ref, enabled, options jsonb, health_status, last_health_at, health_detail, version_info, created_at, updated_at`。凭据走现有 `secret` 表（`SecretPurpose` 加 `storage_credential`），API 只写不读（与下载器 / 媒体服务器同语义：省略或空串不变、null 清除）。 -- `options`：`qps`(5)、`concurrency`(4)、`timeoutMs`(30000)、`idleTimeoutMs`(120000)、`taskTimeoutMs`(21600000)、`retries`(3)、`uploadAsTask`(false)、`snapshotMaxDepth`(5)、`snapshotCheckFolderModtime`(true，MP 本地缺省 true；OpenList 在 MP 里是开关、缺省关——我们缺省开并可关)。 +- `options`：`qps`(5)、`concurrency`(4)、`timeoutMs`(30000)、`idleTimeoutMs`(120000)、`taskTimeoutMs`(21600000)、`retries`(3)、`uploadAsTask`(false)、`snapshotMaxDepth`(5)、`snapshotCheckFolderModtime`(true，同 MP `OPENLIST_SNAPSHOT_CHECK_FOLDER_MODTIME` 缺省开，可关)。 - `apps/server/src/storage/`： - `registry.ts`：按 id 取实例 + 解密凭据 → `AlistClient`（按 `updated_at` 缓存，配置一改就重建）。 - `vpath.ts`：`/@alist//` 编解码；`displayPath()` 给界面「存储名:/路径」。 @@ -184,7 +184,7 @@ ffprobe（§7）判定没有视频流时，远端新文件同样 move 进 `.umt- ## 8. 目录监控（远端快照） -- 来源配置选 Alist 根 + 触发方式「目录快照」：`walkSource` 对远端走 `storage/snapshot.ts`：BFS 列目录，深度 ≤ `snapshotMaxDepth`（MP 5）；`snapshotCheckFolderModtime` 开时，子目录 `modified` 不晚于上次扫描时间就沿用上次快照里该子树的文件（根目录每轮都列，用于发现删除，同 MP）；列表失败的子树保留旧快照（MP「查询异常时保留基线」）。 +- 来源配置选 Alist 根 + 触发方式「目录快照」：`walkSource` 对远端走 `walkRemoteSource`（`file-manager/source-monitor.ts`）：BFS 列目录，深度 ≤ `snapshotMaxDepth`（MP 5）；`snapshotCheckFolderModtime` 开时，子目录 `modified` 不晚于上次扫描时间就沿用上次快照里该子树的文件（根目录每轮都列，用于发现删除，同 MP）；列表失败的子树保留旧快照（MP「查询异常时保留基线」）。 - 之后完全复用现有语义：首扫建基线不整理；新文件 / 变化文件需连续两次观察大小与修改时间一致才整理；结果按现有批次识别上限分批；失败写 `organize_source_snapshot.last_error`，保留旧快照。 - 扫描间隔沿用来源配置 `intervalSec`（MP 按文件数自适应 `adjust_interval`；我们已有逐来源间隔，不再自适应）。 - 按实例限速：快照的 list 走同一个令牌桶，不会打爆网盘。 @@ -228,3 +228,27 @@ ffprobe（§7）判定没有视频流时，远端新文件同样 move 进 `.umt- - 前端：存储设置表单、文件管理存储切换。 - 真机：`*.real.test.ts`，`UMT_ALIST_REAL_URL` / `UMT_ALIST_REAL_TOKEN` 等环境变量开启，缺省跳过；跑本地 → OpenList → 本地往返。 - 真机验收（用户不提供真实 Alist，协调者自起 OpenList 容器）：步骤见 `docs/runbook/storage-alist.md`；脚本 `scripts/alist-e2e.ts` 可对任意 base URL + 令牌跑往返与 Alist 内移动。 + +## 13. 实现落点（2026-10-07） + +| 部分 | 文件 | +| --- | --- | +| 适配器 | `packages/adapters/src/alist/{client,errors,limiter,redact}.ts`；测试用模拟 `mock.ts`（导出 `@umt/adapters/alist-mock`） | +| 存储注册表 / 虚拟路径 / vfs / 传输 / 远端探测 | `apps/server/src/storage/{registry,vpath,vfs,transfer,probe}.ts` | +| 设置接口 | `apps/server/src/modules/auth-settings/storage-routes.ts`；契约 `packages/contracts/src/storage.ts` | +| 文件管理 | `file-manager/common.ts`（`st:` 根、`resolveIn` 远端分支）、`list.ts`、`mutate.ts`、`trash.ts`、`transfer.ts`、`tree.ts`、`download.ts`、`preview.ts`、`upload.ts` | +| 整理 | `organize-plan.ts`、`organize-exec.ts`（`transferFile` 远端分派）、`organize-transfer.ts`、`organize-finalize.ts`、`replacement.ts`、`organize-guard.ts`、`reconcile.ts` | +| 刮削 / 目录监控 | `acquisition-organize/services/scrape.ts`（`atomicWrite` 远端上传）、`file-manager/source-monitor.ts`（`walkRemoteSource`） | +| 健康 | `system/health.ts`（`storage_instance` 挂载项） | +| 前端 | `components/settings/storages.tsx`、`instances.tsx`（库根选存储）、`components/files/*`（存储组、上传、显示路径） | + +## 14. 与设计的偏离 + +- 远端目录监控放在 `source-monitor.ts` 的 `walkRemoteSource`，没有单独的 `storage/snapshot.ts`。 +- 文件管理「下载」到浏览器仍守本地同一规则（NFO / 字幕 / 图片 / 文本 ≤ 20 MiB）；大文件「下载到本地」用「复制到」本机媒体库（走整理执行与账本）。MP 允许浏览器下载任意文件。 +- Alist 根的目录树与列表不做递归目录摘要（视频数），目录树子目录一律显示可展开，避免逐目录请求网盘。 +- 存储根是 Alist 的 `/`（各挂载的上一级）时，回收站放在被删项所在挂载的 `.umt-trash`。 +- 已整理的文件不能「移动到」非媒体库的 Alist 存储根（会脱离媒体库），只能复制或先从媒体库移除。 +- 补字幕（`subtitles`）只写本机媒体库；Alist 媒体库上的字幕写入不在本次范围（MP 的 Alist 能力里没有）。 +- 库存对账（`reconcile`）对 Alist 库根逐个位置 `get`，受实例限速；没有按目录批量列。 +- 迁移只用了 0090；0091 未使用。 diff --git a/docs/porting/moviepilot-filemanager.md b/docs/porting/moviepilot-filemanager.md index 012a3df4..54e3ed7c 100644 --- a/docs/porting/moviepilot-filemanager.md +++ b/docs/porting/moviepilot-filemanager.md @@ -124,9 +124,21 @@ filemanager.vue - 规格中的交互(左树右表、面包屑、行内操作)是通用文件管理器范式,不构成对 MP 表达的复制。 - 若日后参考 MP 的集号定位正则模板(`episode_format`)语法,需单独登记来源,并以 domain 层自有实现替代。 -## 网盘来源（115 / rclone）：暂不做（2026-10-06） +## 网盘：Alist / OpenList 已做（2026-10-07） -用户评估后决定暂不做；不走挂载。将来若做，只做「网盘作为整理来源」，整理目标仍是本地库。MP 做法备查： +2026-10-06 先定的「网盘来源暂不做」被用户推翻：按 MP 做 Alist / OpenList（MP `storages/alist.py`、`alistgo.py`），不做 115 直连与 rclone（115 等网盘挂在 Alist 下用）。设计与 MP 能力对照见 `docs/design/storage-alist.md`，使用与真机验收见 `docs/runbook/storage-alist.md`。 + +| MP | 我们 | +| --- | --- | +| 存储切换菜单（local / Alist / AlistGo …） | 文件管理左栏「Alist 存储」组，每个实例一个根；Alist 上的媒体库目录在「媒体库」组 | +| Alist 浏览 / 新建目录 / 改名 / 删除 / 下载 | 有；删除进 Alist 上的 `.umt-trash`（MP 直接删），下载走服务端流式转发 | +| 上传 | 有（`PUT /files/upload`，同名不覆盖） | +| 整理 本地 ↔ Alist ↔ Alist（同存储） | 有；另补跨实例（下载再上传，MP 不支持）；成功判定同 MP（接口成功 + 目标存在） | +| 刮削写远端（本地生成后 upload） | 有，已存在策略与本地一致 | +| 远程目录监控（快照比对） | 有，复用整理来源的两次观察稳定规则 | +| 存储用量 usage | MP 未实现；我们读 `mount_details`，拿不到显示「未知」 | + +旧结论里的 115 / rclone 做法备查： - 115（`storages/u115.py`）：官方开放平台 `proapi.115.com/open/*`；PKCE 设备码扫码登录（`authDeviceCode` → `qrcodeapi` 轮询 → `deviceCodeToToken`，`refreshToken` 续期）；`ufile/files` 分页列目录（带 sha1、pickcode）、`folder/get_info`、`ufile/move|copy|update|delete`、`ufile/downurl` 换直链下载、`upload/init` 秒传 + OSS 分片上传；限流 API 3 qps、下载 1 qps，风控暂停 1 小时。AppID 须自行申请（MP 用自家应用 `100197847`）。 - rclone（`storages/rclone.py`）：每个操作调一次命令行（`lsjson` / `copyto` / `moveto`，远程名固定 `MP:`），只支持复制 / 移动，目录监控靠快照比对。 diff --git a/docs/porting/moviepilot-v3-comparison.md b/docs/porting/moviepilot-v3-comparison.md index c996e9c5..8d7d16e3 100644 --- a/docs/porting/moviepilot-v3-comparison.md +++ b/docs/porting/moviepilot-v3-comparison.md @@ -267,7 +267,7 @@ MP 手动添加入口识别失败时返回 `requires_confirmation`；前端 `Add ### 6.3 保留与最小验证 -保留仅复制/移动、同分类多个来源、固定/自动分类、已绑定作品沿用原路径、执行前预览、跨卷核验后删源、H&R、来源监控不回填存量，以及 Agent/TMDB/Local 共用整理。不要照抄 MP 的软/硬链接、rclone、音乐或额外分类发布系统。 +保留仅复制/移动、同分类多个来源、固定/自动分类、已绑定作品沿用原路径、执行前预览、跨卷核验后删源、H&R、来源监控不回填存量，以及 Agent/TMDB/Local 共用整理。不要照抄 MP 的软/硬链接、rclone、音乐或额外分类发布系统。网盘已按 MP 接入 Alist / OpenList（`docs/design/storage-alist.md`），115 等网盘挂在 Alist 下使用。 建议优先处理：① 文件/手动/下载/历史入口统一来源策略展示；② 成功移动历史改用目标重整；③ 向导明确批量修改选择与实际整理范围；④ 无来源空状态加现有设置入口。完成这些后再讨论成功记录跳过和本次刮削，不把它们提前扩成新业务系统。 diff --git a/docs/runbook/organize.md b/docs/runbook/organize.md index 10fa4237..41d75c99 100644 --- a/docs/runbook/organize.md +++ b/docs/runbook/organize.md @@ -49,6 +49,17 @@ 不做:切换结构后自动改名已整理的文件(仍用「重新整理」)、剧集组季的独立海报。测试:`apps/server/test/file-manager/organize-episode-structure.test.ts`、`apps/web/app/components/organize-wizard/model.test.ts`「季集结构」。 +## Alist / OpenList 存储 + +媒体库目录、整理来源和文件管理都可以用登记在 设置 → 媒体库与目录 →「存储」里的 Alist / OpenList（设计 `docs/design/storage-alist.md`，使用与真机验收 `docs/runbook/storage-alist.md`）。 + +- 整理矩阵：本地 → Alist 上传、Alist → 本地 下载、同实例 Alist → Alist 用 Alist 的复制 / 移动（等后台任务完成）、跨实例下载再上传。 +- 网盘方向不比对大小和哈希（用户决定）：接口成功 + 目标存在即算成功，移动在此之后删源；本地 ↔ 本地仍走跨卷全量 sha256 核验。 +- 上传 / 下载中断、上传后目标取不到都不删源；连不上、超时、限流按可重试 IO 走整理账本的退避重试，不会被当成「源不存在」。 +- 整理记录、失败重试、智能体识别、H&R 与下载器引用检查照旧；删除整理记录连带的 Alist 文件进 Alist 上的 `.umt-trash`，回收站可还原。本地 → Alist 的移动源已删，撤不回本地源。 +- 刮削目标在 Alist：NFO 与图片本地生成后上传；整理来源选 Alist 目录时按快照（限深度、按目录修改时间跳过未变子树）发现新文件。 +- 记录里的远端路径形如 `/@alist/<存储 id>/`，界面显示「存储名:/路径」。 + ## 来源、分类与存量整理 设置 → 媒体库与目录中维护分类 YAML 和整理来源。新功能已于 2026-10-01 部署为 v0.3.9，迁移 `0059_organize_source_snapshot` 已执行，见 发布记录（已删除，见 git `ceb66f8`）。 @@ -104,7 +115,7 @@ MP 对照源:`MoviePilot-Frontend/src/views/reorganize/TransferHistoryView.vue` | 标题 / 年份 / 季集 | 已有 | `作品 (年) S01E02`;未识别的显示「未识别 · 源文件名」 | | 二级分类(华语电影等) | 已接入 | 分类规则在设置编辑 YAML；电影 / 剧集为母分类，综艺属于剧集 | | 源路径 → 目标路径 | 新增 | 列表加「源 → 目标」列(只显示末两段,悬停看全路径);详情「整理前 / 整理后」已有 | -| 存储标签(本地 / SMB…) | 不做 | 只有本地存储,标签不提供信息 | +| 存储标签(本地 / SMB…) | 不做 | 记录里的路径已带存储(Alist 显示「存储名:/路径」),不另加标签 | | 整理方式 复制 / 移动 | 已有 | 列表「方式」列;整理页聚焦复制／移动结果 | | 软链接、Rclone 复制 / 移动 | 不做 | 整理服务不支持这几种方式 | | 大小 | 新增 | 逐文件源大小之和(`sizeBytes`) | diff --git a/docs/runbook/storage-alist.md b/docs/runbook/storage-alist.md new file mode 100644 index 00000000..c0aec00e --- /dev/null +++ b/docs/runbook/storage-alist.md @@ -0,0 +1,107 @@ +# Alist / OpenList 存储 + +设计与 MP 对照：`docs/design/storage-alist.md`。整理侧口径：`docs/runbook/organize.md`「Alist / OpenList 存储」。 + +## 登记 + +设置 → 媒体库与目录 →「存储」→ 新增： + +| 字段 | 说明 | +| --- | --- | +| 地址 | Alist / OpenList 的访问地址（带端口、子路径），本工具进程要能访问到；容器部署时用容器网络里的地址 | +| 根路径 | 本工具只用这个路径下的内容（如 `/115/影视`），缺省 `/` | +| 认证方式 | 令牌（Alist 后台「设置 → 其他 → 令牌」，推荐）／用户名和密码（换临时令牌，约两天续一次，失效自动重登）／访客 | +| 路径密码 | Alist「元信息」里给目录设了访问密码时填写 | +| 高级 | 每秒请求数（缺省 5）、并发（4）、请求超时、传输无进度超时、等待 Alist 任务上限、重试次数、上传交给 Alist 后台任务（缺省关，同 MP）、目录监控深度（5）与按目录修改时间跳过（开） | + +保存后点「测试连接」：显示版本、登录用户、根路径是否可访问、容量（驱动不给或令牌不是管理员时显示「未知」）。凭据加密保存在 `secret` 表（用途 `storage_credential`），接口与日志里都不出现明文；直链的 `sign` 参数进日志前脱敏。 + +之后： + +- **媒体库目录**：新增时「存储」选这个 Alist，路径填 Alist 内路径（目录要先在 Alist 里建好）。Alist 上不写标记文件，目录可访问即在线。 +- **整理来源**：来源目录可以选文件管理里的「Alist 存储」根下的目录，触发方式只能手动或目录快照。 +- **文件管理**：左栏「Alist 存储」组，每个启用的实例一个根。浏览、新建目录、改名、删除（进 Alist 上的 `.umt-trash`，回收站可还原）、移动 / 复制（Alist 内走 Alist 接口，本机 ↔ Alist 上传 / 下载）、上传、下载小文件、整理、刮削。「刷新」才让 Alist 刷新目录缓存（会穿透到网盘接口）。 + +运行状态 →「挂载根」里列出每个存储实例的连通性；令牌失效显示「令牌失效或没有权限」。 + +## 行为要点 + +- 成功判定（用户决定，同 MP）：网盘方向的上传、下载、复制、移动只看接口成功 + 目标存在，不比对大小和哈希；移动在此之后删源。本地 ↔ 本地仍走现有核验。 +- 不覆盖：上传先查同名再传临时名 `<名>.uhd-partial` 后改名（Alist 的 put 会静默覆盖同名）；复制 / 移动目标已存在报冲突；名字不同的复制 / 移动经 `.umt-stage-*` 暂存目录改名，失败清理。 +- 等待：Alist 的复制（v4 一律）与跨驱动移动是后台任务，按任务状态轮询到完成（失败带任务的错误信息），老版本没有任务接口时等目标出现。 +- 错误：「不存在」与连不上 / 鉴权 / 服务端错误分开；整理遇到后者按可重试 IO 退避，不当成源不存在。限流 / 风控（429、「too many」）自动退避。 +- 下载：写 `<目标>.uhd-partial`，中断后 Range 续传，直链签名过期自动重新获取，成功后不覆盖地改名。 +- ffprobe：远端文件不下载，对直链只读文件头；探测不到不阻塞整理。 + +## 真机验收（临时 OpenList 容器） + +用户不提供真实 Alist 时，按下面起一个 OpenList（本地存储驱动）验收。容器名带 `umt-alist-` 前缀，用完删掉。 + +```sh +W=$PWD/.local/openlist # 工作区的 .local(不进 git) +mkdir -p $W/{data,store1,store2} && chmod -R 777 $W +docker run -d --name umt-alist-it -p 127.0.0.1:55244:5244 \ + -v $W/data:/opt/openlist/data -v $W/store1:/mnt/s1 -v $W/store2:/mnt/s2 openlistteam/openlist:latest +docker exec umt-alist-it ./openlist admin set umtAdmin123 # 设管理员密码 +B=http://127.0.0.1:55244 +TOK=$(curl -s -X POST $B/api/auth/login -H 'Content-Type: application/json' \ + -d '{"username":"admin","password":"umtAdmin123"}' | python3 -c 'import sys,json;print(json.load(sys.stdin)["data"]["token"])') +# 两个本地存储驱动(/s1、/s2),用来测同驱动与跨驱动 +for n in 1 2; do curl -s -X POST $B/api/admin/storage/create -H "Authorization: $TOK" -H 'Content-Type: application/json' \ + -d "{\"mount_path\":\"/s$n\",\"driver\":\"Local\",\"addition\":\"{\\\"root_folder_path\\\":\\\"/mnt/s$n\\\",\\\"mkdir_perm\\\":\\\"777\\\"}\"}"; done +# 永久令牌(设置里登记用) +PERM=$(curl -s "$B/api/admin/setting/get?key=token" -H "Authorization: $TOK" | python3 -c 'import sys,json;print(json.load(sys.stdin)["data"]["value"])') +``` + +部署环境里本工具跑在容器中时，把 OpenList 接到同一个 docker 网络（`docker network connect umt-alist-it`），设置里的地址填 `http://umt-alist-it:5244`。 + +### 1. 适配器往返（脚本，可对任意地址跑） + +```sh +deno run -A scripts/alist-e2e.ts --url $B --token "$PERM" --root /s1 --root2 /s2 +# 或 --user admin --password umtAdmin123;退出码 0 = 全部通过 +``` + +步骤：连接 → 建目录 → 上传（秒传头、临时名改名、进度）→ 同名不覆盖 → 同驱动复制并改名 → 移动 → 跨驱动复制 / 移动（任务轮询）→ 下载比对 sha256 → 不存在 ≠ 失败、容量 → 清理。 + +### 2. 自动化真机测试 + +```sh +export UMT_ALIST_REAL_URL=$B UMT_ALIST_REAL_TOKEN="$PERM" UMT_ALIST_REAL_ROOT=/s1 UMT_ALIST_REAL_ROOT2=/s2 +(cd packages/adapters && deno run -A npm:vitest run src/alist/alist.real.test.ts) +(cd apps/server && DATABASE_URL=... deno run -A npm:vitest run test/storage/alist-organize.real.test.ts) +``` + +后者在测试库上跑整理往返：本地下载目录 → 整理（复制）到 OpenList 上的媒体库 → 文件管理「移动到」本地媒体库（下载后删远端）→ OpenList 内整理（同实例 API 移动）。 + +### 3. 界面验收（部署环境） + +1. 设置 → 媒体库与目录 → 存储：新增「OpenList」，地址、令牌 `$PERM`，测试连接显示版本与用户。 +2. 在 OpenList 里建好 `/s1/media/movies`；媒体库目录新增「OpenList 电影」，存储选 OpenList，路径 `/s1/media/movies`，管理方式「自动维护」。 +3. 文件管理左栏出现「Alist 存储 → OpenList」：新建目录、上传一个小文件、改名、删除后到回收站还原、移动到另一个目录、复制到本机媒体库（下载）。 +4. 整理矩阵（文件管理选文件 →「整理」，目标选对应媒体库目录，各做一次复制、一次移动）： + - 本机下载目录的视频 → OpenList 电影（上传；移动后本机源消失）； + - OpenList `/s1/incoming` 下的视频 → 本机媒体库（下载；移动后 OpenList 源消失）； + - OpenList `/s1/incoming` → OpenList 电影（Alist 内复制 / 移动）。 + 整理记录里每条为「成功」，目标路径显示「OpenList:/…」。 +5. 刮削写远端：OpenList 电影里已整理的作品点「刷新元数据」，OpenList 目录里出现 `.nfo`、`poster`、`clearlogo.png`。 +6. 目录监控：设置 → 整理来源新增，来源选「OpenList」根下的 `incoming`，目标 OpenList 电影，触发「目录快照」，间隔 60 秒；首扫建基线，之后往 `/mnt/s1/incoming`（或经 OpenList 上传）放一个新视频，两次扫描后出现整理记录。 +7. 运行状态 → 挂载根里 OpenList 在线；在 OpenList 后台重置令牌后再看，显示「令牌失效或没有权限」。 + +### 收尾 + +```sh +docker rm -f umt-alist-it && rm -rf .local/openlist +``` + +## 排查 + +| 现象 | 原因 / 处理 | +| --- | --- | +| 测试连接「令牌失效或无权限」 | 令牌被重置或用户没权限；重新复制令牌，或改用账号密码 | +| 测试连接「连不上」 | 地址不对、容器网络不通；在 app 容器里 `curl <地址>/api/public/settings` | +| 根路径不存在 | 根路径要用 Alist 里的完整路径（以挂载路径开头，如 `/115/影视`） | +| 整理卡在「文件操作可重试失败，等待重试」 | Alist 连不上或限流；恢复后任务按退避自动重试，源文件不会被删 | +| 跨网盘复制很慢 | Alist 后台任务在搬数据；在 Alist「任务」页看进度，等待上限在实例高级选项 | +| 目录监控看不到深层新文件 | 超过监控深度，或网盘不更新目录修改时间：调大深度或关掉「按修改时间跳过」 | +| 容量显示「未知」 | 驱动不给容量或令牌不是管理员（`/api/admin/storage/list`），不影响使用 | From a54905e571a34bc67811a52c4e6b20f19f76d14d Mon Sep 17 00:00:00 2001 From: root Date: Wed, 7 Oct 2026 00:16:14 +0800 Subject: [PATCH 10/11] =?UTF-8?q?test(db):=20=E8=A1=A8=E7=BB=93=E6=9E=84?= =?UTF-8?q?=E4=B8=8E=E5=A5=91=E7=BA=A6=E5=90=8C=E6=AD=A5=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E7=99=BB=E8=AE=B0=20storage=5Finstance?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei --- packages/contracts/test/db-sync.test.ts | 5 +++++ packages/db/test/schema.test.ts | 1 + 2 files changed, 6 insertions(+) diff --git a/packages/contracts/test/db-sync.test.ts b/packages/contracts/test/db-sync.test.ts index 28462f3c..b0d12482 100644 --- a/packages/contracts/test/db-sync.test.ts +++ b/packages/contracts/test/db-sync.test.ts @@ -65,6 +65,7 @@ const ROWS: Record MediaAsset: { row: C.MediaAsset }, MediaEntity: { row: C.MediaEntity }, MediaServerInstance: { row: C.MediaServerInstance }, + StorageInstance: { row: C.StorageInstance }, Observation: { row: C.Observation }, OrganizationFileResult: { row: C.OrganizationFileResult }, OrganizeSourceSnapshot: { row: C.OrganizeSourceSnapshotRow }, @@ -224,6 +225,10 @@ const ENUMS: Record = { MediaEntityStatus: C.MediaEntityStatus, MediaServerInstanceKind: C.MediaServerKind, MediaServerInstanceHealthStatus: C.InstanceHealth, + StorageInstanceKind: C.StorageKind, + StorageInstanceFlavor: C.StorageFlavor, + StorageInstanceAuthKind: C.StorageAuthKind, + StorageInstanceHealthStatus: C.InstanceHealth, ObservationStatus: C.ObservationStatus, OrganizationFileResultRole: C.FileRole, OrganizationFileResultMode: C.OrganizationAction, diff --git a/packages/db/test/schema.test.ts b/packages/db/test/schema.test.ts index dae595e2..3b51fd46 100644 --- a/packages/db/test/schema.test.ts +++ b/packages/db/test/schema.test.ts @@ -24,6 +24,7 @@ const EXPECTED_TABLES = [ "downloader_instance", "media_server_instance", "path_mapping", + "storage_instance", // 内容 / 身份 "media_entity", "content_unit", From 521c86d730d63d25b5eb94595da1fdb0a5884d01 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 7 Oct 2026 00:35:41 +0800 Subject: [PATCH 11/11] =?UTF-8?q?fix(storage):=20=E6=9C=AC=E5=9C=B0?= =?UTF-8?q?=E5=86=99=E5=85=A5=E5=8F=A3=E6=8B=92=E7=BB=9D=20Alist=20?= =?UTF-8?q?=E8=99=9A=E6=8B=9F=E8=B7=AF=E5=BE=84,=E8=A1=A5=E5=AD=97?= =?UTF-8?q?=E5=B9=95=E5=9C=A8=20Alist=20=E5=BA=93=E6=A0=B9=E4=B8=8A?= =?UTF-8?q?=E6=98=8E=E7=A1=AE=E5=A4=B1=E8=B4=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 虚拟前缀 /@alist/… 读到本地是 ENOENT,但 mkdir -p、复制 / 移动目标、写文件、回收站落点会在本机真建出 /@alist 并写入文件。统一守卫:任何本地写类操作参数是虚拟路径即抛 REMOTE_PATH_ON_LOCAL_FS (remote_path_on_local_fs),不落盘。 - adapters:fs/remote-guard.ts,接在 runFsOp、FsProxy.run(父进程拒绝不起子进程)、copyWithVerify、 crossDeviceMove、moveSafe、commitNoClobber、sweepStalePartials、writeRootSentinel - server:storage/local-fs.ts 带守卫的写函数,整理 / 替换 / 刮削 / 回收站 / 上传 / 字幕 / vfs 本地分支改用它 - 补字幕目标在 Alist 媒体库上返回失败原因,撤销同样拒绝 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei --- .../services/organize-exec.ts | 2 +- .../services/organize-transfer.ts | 2 +- .../services/replacement.ts | 2 +- .../acquisition-organize/services/scrape.ts | 2 +- .../server/src/modules/file-manager/mutate.ts | 2 +- apps/server/src/modules/file-manager/probe.ts | 2 +- apps/server/src/modules/file-manager/trash.ts | 2 +- .../server/src/modules/file-manager/upload.ts | 2 +- apps/server/src/modules/subtitles/service.ts | 10 +- apps/server/src/storage/local-fs.ts | 61 +++++++++ apps/server/src/storage/transfer.ts | 2 +- apps/server/src/storage/vfs.ts | 2 +- .../test/storage/remote-path-guard.test.ts | 128 ++++++++++++++++++ docs/design/storage-alist.md | 12 +- packages/adapters/src/fs/copy.ts | 4 + packages/adapters/src/fs/errors.ts | 2 + packages/adapters/src/fs/fsops.ts | 19 +++ packages/adapters/src/fs/fsproxy.ts | 4 + packages/adapters/src/fs/index.ts | 1 + packages/adapters/src/fs/move.ts | 3 + packages/adapters/src/fs/remote-guard.test.ts | 97 +++++++++++++ packages/adapters/src/fs/remote-guard.ts | 21 +++ packages/adapters/src/fs/stat.ts | 2 + 23 files changed, 371 insertions(+), 13 deletions(-) create mode 100644 apps/server/src/storage/local-fs.ts create mode 100644 apps/server/test/storage/remote-path-guard.test.ts create mode 100644 packages/adapters/src/fs/remote-guard.test.ts create mode 100644 packages/adapters/src/fs/remote-guard.ts diff --git a/apps/server/src/modules/acquisition-organize/services/organize-exec.ts b/apps/server/src/modules/acquisition-organize/services/organize-exec.ts index c84ff802..153c7c30 100644 --- a/apps/server/src/modules/acquisition-organize/services/organize-exec.ts +++ b/apps/server/src/modules/acquisition-organize/services/organize-exec.ts @@ -1,5 +1,4 @@ import { randomUUID } from "node:crypto"; -import { mkdir, realpath, rmdir } from "node:fs/promises"; import { basename, dirname, join, relative } from "node:path"; import { type CopyReceipt, @@ -13,6 +12,7 @@ import { import type { OrganizationRecord } from "@umt/contracts"; import { sql } from "@umt/db"; import { withAttemptToken } from "@umt/tasks"; +import { mkdir, realpath, rmdir } from "../../../storage/local-fs.ts"; import { involvesRemote, remoteTransfer } from "../../../storage/transfer.ts"; import { isRemotePath, vcheckRoot, vmkdirp, vrenameNoClobber, vstatSafe } from "../../../storage/vfs.ts"; import { closeDecisions } from "../../sources-identity/service/common.ts"; diff --git a/apps/server/src/modules/acquisition-organize/services/organize-transfer.ts b/apps/server/src/modules/acquisition-organize/services/organize-transfer.ts index 63219a33..a4593e11 100644 --- a/apps/server/src/modules/acquisition-organize/services/organize-transfer.ts +++ b/apps/server/src/modules/acquisition-organize/services/organize-transfer.ts @@ -1,9 +1,9 @@ import { randomUUID } from "node:crypto"; -import { readdir, rmdir } from "node:fs/promises"; import { dirname, join, relative } from "node:path"; import { DEFAULT_ROOT_SENTINEL } from "@umt/adapters"; import { fileKindOf, type OrganizeFileKind, type OrganizePlan, type PlannedFile } from "@umt/domain"; import { organizationStatusFromFiles, organizePlanToFileResults } from "@umt/mapping"; +import { readdir, rmdir } from "../../../storage/local-fs.ts"; import { isRemotePath, vstatSafe as statSafe, diff --git a/apps/server/src/modules/acquisition-organize/services/replacement.ts b/apps/server/src/modules/acquisition-organize/services/replacement.ts index 2afc5fa1..1d16263f 100644 --- a/apps/server/src/modules/acquisition-organize/services/replacement.ts +++ b/apps/server/src/modules/acquisition-organize/services/replacement.ts @@ -1,9 +1,9 @@ import { randomUUID } from "node:crypto"; -import { mkdir, readdir, realpath, rm } from "node:fs/promises"; import { basename, dirname, join, relative } from "node:path"; import { DEFAULT_ROOT_SENTINEL, type FileFacts } from "@umt/adapters"; import { sql } from "@umt/db"; import { type ReplacementScope, replacementCovers, replacementScope } from "@umt/domain"; +import { mkdir, readdir, realpath, rm } from "../../../storage/local-fs.ts"; import { isRemotePath, vcheckRoot, diff --git a/apps/server/src/modules/acquisition-organize/services/scrape.ts b/apps/server/src/modules/acquisition-organize/services/scrape.ts index 7a67f10b..feb4b43d 100644 --- a/apps/server/src/modules/acquisition-organize/services/scrape.ts +++ b/apps/server/src/modules/acquisition-organize/services/scrape.ts @@ -1,5 +1,4 @@ import { createHash, randomUUID } from "node:crypto"; -import { mkdir, rename, rm, writeFile } from "node:fs/promises"; import { dirname, isAbsolute, join, relative, resolve } from "node:path"; import { DEFAULT_ROOT_SENTINEL } from "@umt/adapters"; import { @@ -24,6 +23,7 @@ import { seasonOfDirName, workNfo, } from "@umt/domain"; +import { mkdir, rename, rm, writeFile } from "../../../storage/local-fs.ts"; import { isRemotePath, vlstat as lstat, diff --git a/apps/server/src/modules/file-manager/mutate.ts b/apps/server/src/modules/file-manager/mutate.ts index 931d50cd..9eaeb5f8 100644 --- a/apps/server/src/modules/file-manager/mutate.ts +++ b/apps/server/src/modules/file-manager/mutate.ts @@ -1,8 +1,8 @@ -import { lstat, mkdir } from "node:fs/promises"; import { basename, dirname, extname, join } from "node:path"; import type { FilesMkdirRequest, FilesMkdirResult, FilesRenameRequest, FilesRenameResult } from "@umt/contracts"; import { sanitizeSegment } from "@umt/domain"; import type { Principal } from "../../http/auth.ts"; +import { lstat, mkdir } from "../../storage/local-fs.ts"; import { isRemotePath, vexists, vlstat, vmkdirp, vrenameNoClobber } from "../../storage/vfs.ts"; import { actorOfP, errMessage, errnoOf, FilesError, type FmDeps, recheck, relOf, resolveIn } from "./common.ts"; import { audit, filesChanged } from "./ledger.ts"; diff --git a/apps/server/src/modules/file-manager/probe.ts b/apps/server/src/modules/file-manager/probe.ts index 35997881..432aecae 100644 --- a/apps/server/src/modules/file-manager/probe.ts +++ b/apps/server/src/modules/file-manager/probe.ts @@ -1,5 +1,4 @@ import { createHash, randomUUID } from "node:crypto"; -import { mkdir, readFile, writeFile } from "node:fs/promises"; import { join, resolve } from "node:path"; import type { ProbeResult } from "@umt/adapters"; import type { @@ -11,6 +10,7 @@ import type { } from "@umt/contracts"; import { sql } from "@umt/db"; import type { Principal } from "../../http/auth.ts"; +import { mkdir, readFile, writeFile } from "../../storage/local-fs.ts"; import { vlstat } from "../../storage/vfs.ts"; import { FilesError, type FmDeps, jsonb, resolveIn } from "./common.ts"; import { audit } from "./ledger.ts"; diff --git a/apps/server/src/modules/file-manager/trash.ts b/apps/server/src/modules/file-manager/trash.ts index e3d30633..2c043854 100644 --- a/apps/server/src/modules/file-manager/trash.ts +++ b/apps/server/src/modules/file-manager/trash.ts @@ -1,5 +1,4 @@ import { randomUUID } from "node:crypto"; -import { lstat, mkdir, readdir, realpath, rm } from "node:fs/promises"; import { basename, dirname, join, resolve } from "node:path"; import { moveSafe } from "@umt/adapters"; import type { @@ -16,6 +15,7 @@ import type { } from "@umt/contracts"; import { sql, type TrashItemRow } from "@umt/db"; import type { Principal } from "../../http/auth.ts"; +import { lstat, mkdir, readdir, realpath, rm } from "../../storage/local-fs.ts"; import { isRemotePath, vlstat, vmkdirp, vreaddirStats, vremove, vrenameNoClobber } from "../../storage/vfs.ts"; import { parseVPath, toVPath } from "../../storage/vpath.ts"; import { upsertDecision } from "../acquisition-organize/services/common.ts"; diff --git a/apps/server/src/modules/file-manager/upload.ts b/apps/server/src/modules/file-manager/upload.ts index 43f6eefb..2e112111 100644 --- a/apps/server/src/modules/file-manager/upload.ts +++ b/apps/server/src/modules/file-manager/upload.ts @@ -1,10 +1,10 @@ import { randomUUID } from "node:crypto"; -import { mkdir, open, rm } from "node:fs/promises"; import { join } from "node:path"; import { commitNoClobber, partialPathFor } from "@umt/adapters"; import type { FilesUploadQuery, FilesUploadResult } from "@umt/contracts"; import { sanitizeSegment } from "@umt/domain"; import type { Principal } from "../../http/auth.ts"; +import { mkdir, open, rm } from "../../storage/local-fs.ts"; import { relayDir, remote, remoteError, vexists } from "../../storage/vfs.ts"; import { errnoOf, FilesError, type FmDeps, recheck, relOf, resolveIn } from "./common.ts"; import { audit, filesChanged } from "./ledger.ts"; diff --git a/apps/server/src/modules/subtitles/service.ts b/apps/server/src/modules/subtitles/service.ts index 5f963f91..87c00e64 100644 --- a/apps/server/src/modules/subtitles/service.ts +++ b/apps/server/src/modules/subtitles/service.ts @@ -1,4 +1,3 @@ -import { copyFile, mkdir, readdir, rename, rm, stat, writeFile } from "node:fs/promises"; import { dirname, join } from "node:path"; import { lifecycleKey, @@ -26,6 +25,8 @@ import type { Selectable } from "kysely"; import { dataPaths } from "../../config.ts"; import { ApiException } from "../../http/errors.ts"; import { runSandbox, SandboxError } from "../../sandbox/client.ts"; +import { copyFile, mkdir, readdir, rename, rm, stat, writeFile } from "../../storage/local-fs.ts"; +import { isRemotePath } from "../../storage/vpath.ts"; import type { ServerContext } from "../types.ts"; // 补字幕(docs/plan/subtitles.md;对照 MP 插件「海拉鲁字幕大师」upload / matching / subtitle_writer): @@ -199,6 +200,11 @@ export async function applySubtitles( failed.push({ name: it.name, reason: "目标视频不在这部作品的媒体库里" }); continue; } + // Alist 上的媒体库:补字幕暂不支持(MP 的 Alist 能力里也没有),明确失败,不往本机写 + if (isRemotePath(t.rootPath)) { + failed.push({ name: it.name, reason: "目标视频在 Alist 存储上，暂不支持补字幕" }); + continue; + } const name = subtitleFileName(t.relativePath, it.language, it.ext as SubtitleExt, defaultLang); const rel = join(dirname(t.relativePath), name); if (planned.has(rel)) { @@ -322,6 +328,8 @@ export async function undoSubtitleWrite(ctx: ServerContext, id: string): Promise .select("local_path") .where("id", "=", row.library_root_id) .executeTakeFirstOrThrow(); + if (isRemotePath(root.local_path)) + throw new ApiException("precondition_failed", "字幕在 Alist 存储上，不能在这里撤销"); const abs = join(root.local_path, row.relative_path); if (row.backup_path && (await exists(row.backup_path))) { await copyFile(row.backup_path, `${abs}.umt-restore.tmp`); diff --git a/apps/server/src/storage/local-fs.ts b/apps/server/src/storage/local-fs.ts new file mode 100644 index 00000000..8e897fa8 --- /dev/null +++ b/apps/server/src/storage/local-fs.ts @@ -0,0 +1,61 @@ +import * as fsp from "node:fs/promises"; +import { assertLocalPaths } from "@umt/adapters"; + +// 本地写入口的远端路径守卫(server 侧):整理 / 刮削 / 字幕 / 回收站 / 上传里直接调 node:fs 写文件的地方都从这里取函数。 +// 参数是 Alist 虚拟路径(/@alist/…)就抛 FsError("REMOTE_PATH_ON_LOCAL_FS"),不在本机建出 /@alist 目录或写进文件。 +// 读类函数(stat / readdir / readFile …)原样转出:对虚拟路径只会 ENOENT。 + +export * from "node:fs/promises"; + +type P = Parameters[0]; +const s = (p: unknown) => (typeof p === "string" ? p : p instanceof URL ? p.pathname : undefined); + +export const mkdir: typeof fsp.mkdir = (async (p: P, o?: Parameters[1]) => { + assertLocalPaths("mkdir", s(p)); + return fsp.mkdir(p, o); +}) as typeof fsp.mkdir; + +export const writeFile: typeof fsp.writeFile = async (p, data, o) => { + assertLocalPaths("writeFile", s(p)); + return fsp.writeFile(p, data, o); +}; + +export const rename: typeof fsp.rename = async (a, b) => { + assertLocalPaths("rename", s(a), s(b)); + return fsp.rename(a, b); +}; + +export const copyFile: typeof fsp.copyFile = async (a, b, mode) => { + assertLocalPaths("copyFile", s(a), s(b)); + return fsp.copyFile(a, b, mode); +}; + +export const rm: typeof fsp.rm = async (p, o) => { + assertLocalPaths("rm", s(p)); + return fsp.rm(p, o); +}; + +export const rmdir: typeof fsp.rmdir = async (p, o) => { + assertLocalPaths("rmdir", s(p)); + return fsp.rmdir(p, o); +}; + +export const unlink: typeof fsp.unlink = async (p) => { + assertLocalPaths("unlink", s(p)); + return fsp.unlink(p); +}; + +export const link: typeof fsp.link = async (a, b) => { + assertLocalPaths("link", s(a), s(b)); + return fsp.link(a, b); +}; + +/** open:写 / 追加 / 新建模式才拦;只读打开放行(虚拟路径只会 ENOENT) */ +export const open: typeof fsp.open = (async ( + p: Parameters[0], + flags?: string | number, + mode?: number, +) => { + if (flags !== undefined && flags !== "r") assertLocalPaths("open", s(p)); + return fsp.open(p, flags, mode); +}) as typeof fsp.open; diff --git a/apps/server/src/storage/transfer.ts b/apps/server/src/storage/transfer.ts index 021e29ab..c7762430 100644 --- a/apps/server/src/storage/transfer.ts +++ b/apps/server/src/storage/transfer.ts @@ -1,7 +1,7 @@ import { randomUUID } from "node:crypto"; -import { mkdir, rename, rm, stat, unlink } from "node:fs/promises"; import { basename, dirname, join, posix } from "node:path"; import { AlistError, type ProgressCallback } from "@umt/adapters"; +import { mkdir, rename, rm, stat, unlink } from "./local-fs.ts"; import { invalidateRemote, relayDir, remote, remoteError } from "./vfs.ts"; import { parseVPath, sameStorage } from "./vpath.ts"; diff --git a/apps/server/src/storage/vfs.ts b/apps/server/src/storage/vfs.ts index 2221c5ac..3df8ab45 100644 --- a/apps/server/src/storage/vfs.ts +++ b/apps/server/src/storage/vfs.ts @@ -1,5 +1,4 @@ import { createHash, randomUUID } from "node:crypto"; -import { lstat, mkdir, readdir, readFile, realpath, rename, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { basename, dirname, join, posix } from "node:path"; import { @@ -12,6 +11,7 @@ import { type StatResult, statSafe, } from "@umt/adapters"; +import { lstat, mkdir, readdir, readFile, realpath, rename, rm, writeFile } from "./local-fs.ts"; import { storageDataDir, storageEntry } from "./registry.ts"; import { parseVPath, toVPath, type VPath } from "./vpath.ts"; diff --git a/apps/server/test/storage/remote-path-guard.test.ts b/apps/server/test/storage/remote-path-guard.test.ts new file mode 100644 index 00000000..889b8c9d --- /dev/null +++ b/apps/server/test/storage/remote-path-guard.test.ts @@ -0,0 +1,128 @@ +import { existsSync } from "node:fs"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { createSubtitlesModule } from "../../src/modules/subtitles/index.ts"; +import * as lfs from "../../src/storage/local-fs.ts"; +import { createHarness, type Harness, pgAvailable } from "../auth-settings/harness.ts"; +import { DEV_KEY } from "../helpers.ts"; + +// server 侧本地写入口(整理 / 刮削 / 字幕 / 回收站 / 上传共用的 storage/local-fs.ts)对 Alist 虚拟路径一律拒绝, +// 本机不出现 /@alist;补字幕的目标在 Alist 媒体库上时明确失败(不支持),不往本机写。 + +const V = "/@alist/00000000-0000-4000-8000-0000000000ee/s1/lib"; +const code = async (p: Promise) => { + try { + await p; + } catch (e) { + return (e as { code?: string }).code; + } + return "no-error"; +}; + +describe("local-fs 写入口守卫", () => { + let dir: string; + beforeAll(async () => { + dir = await mkdtemp(join(tmpdir(), "umt-lfs-")); + await lfs.writeFile(join(dir, "a.txt"), "x"); + }); + afterAll(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + it("mkdir / writeFile / rename / copyFile / rm / rmdir / unlink / link / open(写)都拒绝虚拟路径", async () => { + const R = "REMOTE_PATH_ON_LOCAL_FS"; + expect(await code(lfs.mkdir(`${V}/d`, { recursive: true }))).toBe(R); + expect(await code(lfs.writeFile(`${V}/a.nfo`, ""))).toBe(R); + expect(await code(lfs.rename(join(dir, "a.txt"), `${V}/a.txt`))).toBe(R); + expect(await code(lfs.copyFile(join(dir, "a.txt"), `${V}/a.txt`))).toBe(R); + expect(await code(lfs.rm(`${V}/a.txt`, { force: true }))).toBe(R); + expect(await code(lfs.rmdir(V))).toBe(R); + expect(await code(lfs.unlink(`${V}/a.txt`))).toBe(R); + expect(await code(lfs.link(join(dir, "a.txt"), `${V}/a.txt`))).toBe(R); + expect(await code(lfs.open(`${V}/a.txt`, "w"))).toBe(R); + expect(existsSync("/@alist")).toBe(false); + expect(existsSync(join(dir, "a.txt"))).toBe(true); + }); + + it("本地路径与只读打开照常", async () => { + await lfs.mkdir(join(dir, "sub"), { recursive: true }); + await lfs.rename(join(dir, "a.txt"), join(dir, "sub", "a.txt")); + expect(await lfs.readFile(join(dir, "sub", "a.txt"), "utf8")).toBe("x"); + expect(await code(lfs.open(`${V}/a.txt`, "r"))).toBe("ENOENT"); + }); +}); + +const hasPg = await pgAvailable(); +describe.skipIf(!hasPg)("补字幕:目标在 Alist 媒体库上明确失败", () => { + let h: Harness; + let entityId: string; + let locationId: string; + beforeAll(async () => { + h = await createHarness({ extraModules: [createSubtitlesModule()] }); + const root = await h.db + .insertInto("library_root") + .values({ + name: "Alist 剧集", + media_kind: "mixed", + local_path: V, + management_mode: "managed", + sentinel_file: null, + }) + .returning("id") + .executeTakeFirstOrThrow(); + entityId = ( + await h.db + .insertInto("media_entity") + .values({ kind: "series", title: "远端剧", year: 2024 } as never) + .returning("id") + .executeTakeFirstOrThrow() + ).id; + const a = await h.db + .insertInto("media_asset") + .values({ media_entity_id: entityId, kind: "video", status: "active" }) + .returning("id") + .executeTakeFirstOrThrow(); + locationId = ( + await h.db + .insertInto("asset_location") + .values({ + media_asset_id: a.id, + location_kind: "library", + library_root_id: root.id, + relative_path: "远端剧 (2024)/Season 1/远端剧 - S01E01.mkv", + status: "available", + }) + .returning("id") + .executeTakeFirstOrThrow() + ).id; + }, 120_000); + afterAll(async () => h?.close()); + + it("写入返回失败原因,不写任何文件、不建资产", async () => { + const res = await h.app.request("/api/v1/subtitles/apply", { + method: "POST", + headers: { authorization: `Bearer ${DEV_KEY}`, "content-type": "application/json" }, + body: JSON.stringify({ + entityId, + items: [ + { + name: "a.srt", + ext: "srt", + language: "zh-Hans", + content: btoa("1\n00:00:01,000 --> 00:00:02,000\nhi\n"), + targetLocationId: locationId, + }, + ], + }), + }); + const body = await res.json(); + expect(res.status, JSON.stringify(body)).toBe(200); + expect(body.data.written).toEqual([]); + expect(body.data.failed).toEqual([{ name: "a.srt", reason: "目标视频在 Alist 存储上，暂不支持补字幕" }]); + expect(existsSync("/@alist")).toBe(false); + const subs = await h.db.selectFrom("media_asset").select("id").where("kind", "=", "subtitle").execute(); + expect(subs).toEqual([]); + }); +}); diff --git a/docs/design/storage-alist.md b/docs/design/storage-alist.md index 22fbe262..f9d5c012 100644 --- a/docs/design/storage-alist.md +++ b/docs/design/storage-alist.md @@ -242,13 +242,21 @@ ffprobe（§7）判定没有视频流时，远端新文件同样 move 进 `.umt- | 健康 | `system/health.ts`（`storage_instance` 挂载项） | | 前端 | `components/settings/storages.tsx`、`instances.tsx`（库根选存储）、`components/files/*`（存储组、上传、显示路径） | -## 14. 与设计的偏离 +## 14. 本地写入守卫 + +虚拟前缀读到本地只会 ENOENT，但写类操作会在本机真建出 `/@alist/…`。所以本地写入口统一拒绝虚拟路径（`FsError("REMOTE_PATH_ON_LOCAL_FS")`，消息以 `remote_path_on_local_fs:` 开头，不落盘）： + +- adapters：`fs/remote-guard.ts`，接在 `runFsOp`、`FsProxy.run`（父进程就拒绝，不起子进程）、`copyWithVerify`、`crossDeviceMove`、`moveSafe`、`commitNoClobber`、`sweepStalePartials`、`writeRootSentinel`。读类（stat / checkRoot / sha256 / match）不拦。 +- server：`storage/local-fs.ts`（`node:fs/promises` 的写函数换成带守卫的版本），整理执行、替换、刮削、原样搬运、回收站、改名 / 新建目录、上传、探测缓存、补字幕、vfs / transfer 的本地分支都从这里取。 +- 测试：`packages/adapters/src/fs/remote-guard.test.ts`、`apps/server/test/storage/remote-path-guard.test.ts`。 + +## 15. 与设计的偏离 - 远端目录监控放在 `source-monitor.ts` 的 `walkRemoteSource`，没有单独的 `storage/snapshot.ts`。 - 文件管理「下载」到浏览器仍守本地同一规则（NFO / 字幕 / 图片 / 文本 ≤ 20 MiB）；大文件「下载到本地」用「复制到」本机媒体库（走整理执行与账本）。MP 允许浏览器下载任意文件。 - Alist 根的目录树与列表不做递归目录摘要（视频数），目录树子目录一律显示可展开，避免逐目录请求网盘。 - 存储根是 Alist 的 `/`（各挂载的上一级）时，回收站放在被删项所在挂载的 `.umt-trash`。 - 已整理的文件不能「移动到」非媒体库的 Alist 存储根（会脱离媒体库），只能复制或先从媒体库移除。 -- 补字幕（`subtitles`）只写本机媒体库；Alist 媒体库上的字幕写入不在本次范围（MP 的 Alist 能力里没有）。 +- 补字幕（`subtitles`）只写本机媒体库；目标在 Alist 媒体库上时明确失败（「目标视频在 Alist 存储上，暂不支持补字幕」），不往本机写（MP 的 Alist 能力里没有）。 - 库存对账（`reconcile`）对 Alist 库根逐个位置 `get`，受实例限速；没有按目录批量列。 - 迁移只用了 0090；0091 未使用。 diff --git a/packages/adapters/src/fs/copy.ts b/packages/adapters/src/fs/copy.ts index c6ec44e0..7dbfdc2c 100644 --- a/packages/adapters/src/fs/copy.ts +++ b/packages/adapters/src/fs/copy.ts @@ -7,6 +7,7 @@ import { basename, dirname, join } from "node:path"; import { Transform } from "node:stream"; import { pipeline } from "node:stream/promises"; import { errnoOf, FsError, toFsError } from "./errors.ts"; +import { assertLocalPaths } from "./remote-guard.ts"; export type VerifyMethod = "size" | "sha256" | "sample"; @@ -231,6 +232,7 @@ const LINK_UNSUPPORTED = ["EPERM", "ENOTSUP", "EOPNOTSUPP", "ENOSYS", "EMLINK"]; /** no-clobber 提交:link(tmp,dst) 在目标存在时原子失败;文件系统不支持硬链接时退回 lstat+rename。 */ export async function commitNoClobber(tmp: string, dst: string): Promise { + assertLocalPaths("commitNoClobber", tmp, dst); try { await link(tmp, dst); await unlink(tmp); @@ -334,6 +336,7 @@ async function replaceCommit( * 用源文件权限覆盖会清掉继承 ACL(与 MP _copy_with_target_permissions 同理)。不要补 chmod/chown。 */ export async function copyWithVerify(src: string, dst: string, opts: CopyOptions = {}): Promise { + assertLocalPaths("copy", src, dst, opts.replaceOld?.trashDir); const started = Date.now(); const method = opts.hash ?? "size"; const onExisting = opts.onExisting ?? "conflict"; @@ -503,6 +506,7 @@ export async function filesMatch( /** 清理目录下超过 olderThanMs 的残留临时文件,返回被删除的路径。 */ export async function sweepStalePartials(dir: string, olderThanMs: number, suffix = ".uhd-partial"): Promise { + assertLocalPaths("sweep", dir); const removed: string[] = []; const now = Date.now(); for (const name of await readdir(dir)) { diff --git a/packages/adapters/src/fs/errors.ts b/packages/adapters/src/fs/errors.ts index 5f8ccb80..a8e6deb9 100644 --- a/packages/adapters/src/fs/errors.ts +++ b/packages/adapters/src/fs/errors.ts @@ -10,6 +10,8 @@ export type FsErrorCode = | "ABORTED" /** 文件操作挂死(进度停滞/快操作超时),执行子进程已被回收;与 IO 同属可退避重试,不代表文件缺失 */ | "TIMEOUT" + /** 本地文件操作收到 Alist 虚拟路径(/@alist/…):调用方没按存储分派,拒绝而不是写错位置 */ + | "REMOTE_PATH_ON_LOCAL_FS" | "IO"; /** diff --git a/packages/adapters/src/fs/fsops.ts b/packages/adapters/src/fs/fsops.ts index 249c4dba..42088d8a 100644 --- a/packages/adapters/src/fs/fsops.ts +++ b/packages/adapters/src/fs/fsops.ts @@ -5,6 +5,7 @@ import { unlink } from "node:fs/promises"; import { type CopyOptions, copyWithVerify, filesMatchSha256, sha256Full, sweepStalePartials } from "./copy.ts"; import { FsError, toFsError } from "./errors.ts"; import { type CrossMoveOptions, crossDeviceMove, moveSafe } from "./move.ts"; +import { assertLocalPaths } from "./remote-guard.ts"; import { checkRoot, statSafe } from "./stat.ts"; /** 可序列化的复制选项(去掉信号与回调) */ @@ -45,7 +46,25 @@ export interface FsOpContext { progress: (p: FsProgress) => void; } +/** 请求里会被写入 / 删除的路径(远端虚拟路径守卫用;读类操作对它只会 ENOENT,不拦) */ +export function writePathsOf(req: FsOpRequest): string[] { + switch (req.op) { + case "copy": + return [req.src, req.dst, ...(req.opts?.replaceOld?.trashDir ? [req.opts.replaceOld.trashDir] : [])]; + case "crossMove": + case "move": + return [req.src, req.dst]; + case "sweep": + return [req.dir]; + case "unlink": + return [req.path]; + default: + return []; + } +} + export async function runFsOp(req: FsOpRequest, ctx: FsOpContext): Promise { + assertLocalPaths(req.op, ...writePathsOf(req)); const { signal, progress } = ctx; const at = (phase: string) => (bytes: number, total: number) => progress({ phase, bytes, total }); switch (req.op) { diff --git a/packages/adapters/src/fs/fsproxy.ts b/packages/adapters/src/fs/fsproxy.ts index 6b428371..882483cd 100644 --- a/packages/adapters/src/fs/fsproxy.ts +++ b/packages/adapters/src/fs/fsproxy.ts @@ -23,8 +23,10 @@ import { runFsOp, type WireCopyOptions, type WireCrossMoveOptions, + writePathsOf, } from "./fsops.ts"; import type { CrossMoveResult, MoveResult } from "./move.ts"; +import { assertLocalPaths } from "./remote-guard.ts"; import type { RootHealth, StatResult } from "./stat.ts"; export type { FsProgress } from "./fsops.ts"; @@ -243,6 +245,8 @@ export class FsProxy { async run(req: FsOpRequest, call: FsCallOptions = {}): Promise { call.signal?.throwIfAborted(); + // 远端虚拟路径在父进程就拒绝,不起执行子进程 + assertLocalPaths(req.op, ...writePathsOf(req)); if (this.#mode === "subprocess") { try { return await this.#runSubprocess(req, call); diff --git a/packages/adapters/src/fs/index.ts b/packages/adapters/src/fs/index.ts index b97a0c30..2b7c97de 100644 --- a/packages/adapters/src/fs/index.ts +++ b/packages/adapters/src/fs/index.ts @@ -4,5 +4,6 @@ export * from "./ffprobe.ts"; export * from "./fsops.ts"; export * from "./fsproxy.ts"; export * from "./move.ts"; +export * from "./remote-guard.ts"; export * from "./scan.ts"; export * from "./stat.ts"; diff --git a/packages/adapters/src/fs/move.ts b/packages/adapters/src/fs/move.ts index f8cc348b..f1bf307d 100644 --- a/packages/adapters/src/fs/move.ts +++ b/packages/adapters/src/fs/move.ts @@ -4,6 +4,7 @@ import { link, lstat, rename, stat, unlink } from "node:fs/promises"; import { dirname } from "node:path"; import { type CopyOptions, copyWithVerify, filesMatch, type VerifyMethod } from "./copy.ts"; import { errnoOf, FsError, toFsError } from "./errors.ts"; +import { assertLocalPaths } from "./remote-guard.ts"; export type MoveResult = { status: "moved" } | { status: "exdev" } | { status: "exists" }; @@ -13,6 +14,7 @@ export type MoveResult = { status: "moved" } | { status: "exdev" } | { status: " * 其他错误抛 FsError。 */ export async function moveSafe(src: string, dst: string): Promise { + assertLocalPaths("move", src, dst); let s: Awaited>; try { s = await lstat(src); @@ -102,6 +104,7 @@ async function exists(p: string): Promise { * - 删源失败 → attention(不能报成功) */ export async function crossDeviceMove(src: string, dst: string, opts: CrossMoveOptions = {}): Promise { + assertLocalPaths("crossMove", src, dst); const method = opts.hash ?? "size"; const io = { ...(opts.signal ? { signal: opts.signal } : {}), diff --git a/packages/adapters/src/fs/remote-guard.test.ts b/packages/adapters/src/fs/remote-guard.test.ts new file mode 100644 index 00000000..17b74e9f --- /dev/null +++ b/packages/adapters/src/fs/remote-guard.test.ts @@ -0,0 +1,97 @@ +import { existsSync } from "node:fs"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { commitNoClobber, copyWithVerify, sweepStalePartials } from "./copy.ts"; +import { FsError, fsErrorClass } from "./errors.ts"; +import { type FsOpRequest, runFsOp } from "./fsops.ts"; +import { FsProxy } from "./fsproxy.ts"; +import { crossDeviceMove, moveSafe } from "./move.ts"; +import { assertLocalPaths, isVirtualRemotePath } from "./remote-guard.ts"; +import { writeRootSentinel } from "./stat.ts"; + +// 远端虚拟路径守卫:Alist 路径(/@alist/…)传到本地写类操作一律抛 REMOTE_PATH_ON_LOCAL_FS,本机不能出现 /@alist。 + +const V = "/@alist/00000000-0000-4000-8000-0000000000ff/s1/lib"; +let dir: string; +let src: string; +beforeAll(async () => { + dir = await mkdtemp(join(tmpdir(), "umt-guard-")); + src = join(dir, "a.mkv"); + await writeFile(src, "video"); +}); +afterAll(async () => { + await rm(dir, { recursive: true, force: true }); +}); + +const rejects = async (p: Promise | (() => unknown)) => { + let err: unknown = null; + try { + await (typeof p === "function" ? p() : p); + } catch (e) { + err = e; + } + expect(err).toBeInstanceOf(FsError); + expect((err as FsError).code).toBe("REMOTE_PATH_ON_LOCAL_FS"); + expect((err as Error).message).toContain("remote_path_on_local_fs"); + expect(existsSync("/@alist")).toBe(false); + expect(existsSync(src)).toBe(true); +}; + +describe("远端虚拟路径守卫", () => { + it("识别与分类", () => { + expect(isVirtualRemotePath(V)).toBe(true); + expect(isVirtualRemotePath("/@alist")).toBe(true); + expect(isVirtualRemotePath("/@alistx/a")).toBe(false); + expect(isVirtualRemotePath("/media/@alist/a")).toBe(false); + expect(fsErrorClass("REMOTE_PATH_ON_LOCAL_FS")).toBe("manual"); + expect(() => assertLocalPaths("x", "/tmp/a", null, undefined)).not.toThrow(); + }); + + it("各写类函数:目标或源是虚拟路径都拒绝,不落盘", async () => { + await rejects(copyWithVerify(src, `${V}/a.mkv`)); + await rejects(copyWithVerify(`${V}/a.mkv`, join(dir, "b.mkv"))); + await rejects( + copyWithVerify(src, join(dir, "c.mkv"), { + onExisting: "replace", + replaceOld: { mode: "trash", trashDir: `${V}/.umt-trash` }, + }), + ); + await rejects(crossDeviceMove(src, `${V}/a.mkv`)); + await rejects(moveSafe(src, `${V}/a.mkv`)); + await rejects(commitNoClobber(src, `${V}/a.mkv`)); + await rejects(sweepStalePartials(V, 0)); + await rejects(writeRootSentinel(V)); + expect(existsSync(join(dir, "b.mkv"))).toBe(false); + }); + + const ctx = { signal: new AbortController().signal, progress: () => {} }; + const reqs: FsOpRequest[] = [ + { op: "copy", src: "/tmp/x", dst: `${V}/a.mkv` }, + { op: "crossMove", src: "/tmp/x", dst: `${V}/a.mkv` }, + { op: "move", src: `${V}/a.mkv`, dst: "/tmp/y" }, + { op: "sweep", dir: V, olderThanMs: 0 }, + { op: "unlink", path: `${V}/a.mkv` }, + ]; + it.each(reqs.map((r) => [r.op, r] as const))("runFsOp %s:拒绝", async (_op, req) => { + await rejects(runFsOp(req, ctx)); + }); + + it("FsProxy.run(子进程与进程内两种模式):在父进程就拒绝,不起子进程", async () => { + for (const mode of ["subprocess", "inprocess"] as const) { + const px = new FsProxy({ mode }); + for (const req of reqs) await rejects(px.run(req)); + await rejects(px.copy(src, `${V}/a.mkv`)); + await rejects(px.moveSafe(src, `${V}/a.mkv`)); + await rejects(px.sweepStalePartials(V, 0)); + expect(px.stats().spawned).toBe(0); + } + }); + + it("读类操作不拦(虚拟路径只会是不存在)", async () => { + const px = new FsProxy({ mode: "inprocess" }); + expect(await px.statSafe(`${V}/a.mkv`)).toEqual({ kind: "missing" }); + expect(existsSync("/@alist")).toBe(false); + }); +}); diff --git a/packages/adapters/src/fs/remote-guard.ts b/packages/adapters/src/fs/remote-guard.ts new file mode 100644 index 00000000..52efac04 --- /dev/null +++ b/packages/adapters/src/fs/remote-guard.ts @@ -0,0 +1,21 @@ +import { FsError } from "./errors.ts"; + +// 远端虚拟路径守卫(docs/design/storage-alist.md §0):Alist 路径在本系统里写成 /@alist/<存储 id>/<路径>。 +// 本地 fs 读它只会 ENOENT,但写类操作(mkdir -p、复制 / 移动目标、写文件、回收站落点)会在本机真建出 +// /@alist/... 并把文件写进去——静默写错位置。所有本地写入口先过这里:参数是远端路径就抛 +// REMOTE_PATH_ON_LOCAL_FS,不落盘。 + +export const REMOTE_PATH_PREFIX = "/@alist"; +export const isVirtualRemotePath = (p: unknown): boolean => + typeof p === "string" && (p === REMOTE_PATH_PREFIX || p.startsWith(`${REMOTE_PATH_PREFIX}/`)); + +/** 任一路径是远端虚拟路径 → 抛 FsError("REMOTE_PATH_ON_LOCAL_FS") */ +export function assertLocalPaths(op: string, ...paths: (string | null | undefined)[]): void { + const hit = paths.find(isVirtualRemotePath); + if (hit) + throw new FsError( + "REMOTE_PATH_ON_LOCAL_FS", + `remote_path_on_local_fs: 本地文件操作 ${op} 收到 Alist 路径，已拒绝：${hit}`, + { path: hit }, + ); +} diff --git a/packages/adapters/src/fs/stat.ts b/packages/adapters/src/fs/stat.ts index aca62781..5add1372 100644 --- a/packages/adapters/src/fs/stat.ts +++ b/packages/adapters/src/fs/stat.ts @@ -2,6 +2,7 @@ import { lstat, stat, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { errnoOf } from "./errors.ts"; +import { assertLocalPaths } from "./remote-guard.ts"; export const DEFAULT_ROOT_SENTINEL = ".uhd-root"; @@ -57,6 +58,7 @@ export async function checkRoot(root: string, sentinel: string = DEFAULT_ROOT_SE /** 在根目录写入哨兵文件(首次登记库根时调用)。 */ export async function writeRootSentinel(root: string, sentinel: string = DEFAULT_ROOT_SENTINEL): Promise { + assertLocalPaths("writeRootSentinel", root); await writeFile(join(root, sentinel), `uhd-media-tool root sentinel\ncreated=${new Date().toISOString()}\n`, { flag: "wx", }).catch((e) => {