diff --git a/.dockerignore b/.dockerignore index 724195b7..c8ddc61e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -15,3 +15,6 @@ docs *.md !apps/**/README.md data +.local +.browser-verify-out +docker-compose.override.yml diff --git a/Dockerfile b/Dockerfile index a148972c..714dd74f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,10 +2,13 @@ # PostgreSQL 不在本镜像内,见 docker-compose.yml。 # 构建:docker build -t uhd-media-tool . 运行时版本锁定 Deno 2.9.7(ADR-004) # ffprobe 取自静态构建镜像(单个 static-pie 二进制,约 135 MB;Debian 的 ffmpeg 包要拉 200+ 个依赖、约 450 MB) -ARG FFPROBE_IMAGE=mwader/static-ffmpeg:9.0.2 +ARG FFPROBE_IMAGE=mwader/static-ffmpeg:9.0.2@sha256:7d9bdaaf887f7e6ce6151f67325c344074b5ff1fb75316011c3376503e449a7b FROM ${FFPROBE_IMAGE} AS ffprobe -FROM denoland/deno:2.9.7 AS build +# 与 PG 18 服务端同主版本的备份工具,避免 Debian 默认 17 客户端拒绝导出 PG 18。 +FROM postgres:18.6-trixie@sha256:5a5a84b19854a9ffaa54082c166ff4ec27473a361e496e5ea167f298f2da9722 AS postgres-tools + +FROM denoland/deno:2.9.7@sha256:fa335acdf6b72106eda2cb6a8cb5f4187e7630e357467489db4b2e7352d5e432 AS build WORKDIR /src COPY . . RUN deno install --frozen @@ -19,19 +22,23 @@ RUN deno task check:tokens && cd apps/web && deno task build # 编译后立即自检:空环境跑 --role preflight,模块图缺件(ERR_MODULE_NOT_FOUND)在构建期就失败,不留到容器启动 RUN cd apps/server && deno task compile && cd /src && sh scripts/check-compiled-binary.sh -FROM debian:trixie-slim -# gosu:root 修完 DATA_DIR 属主后降权;postgresql-client:备份/恢复脚本用 pg_dump/pg_restore/psql -# (trixie 自带 17 版客户端,可备份/恢复 16 版服务端)。Deno 自带根证书,不装 ca-certificates;健康探测用二进制自身的 fetch +FROM debian:trixie-slim@sha256:a99cfc517144bc59b1978475ec53b46ecabec7e43635402ee5b77cc54cd1b20a +# gosu 用于降权;PG 18 客户端来自固定的官方同发行版镜像,这里只安装动态库。 +# Deno 自带根证书;健康探测用二进制自身的 fetch。 # 国内网络可 --build-arg APT_MIRROR=mirrors.ustc.edu.cn 换源 ARG APT_MIRROR="" RUN if [ -n "$APT_MIRROR" ]; then sed -i "s|deb.debian.org|$APT_MIRROR|g" /etc/apt/sources.list.d/debian.sources; fi \ && apt-get update \ - && apt-get install -y --no-install-recommends gosu postgresql-client \ + && apt-get upgrade -y --no-install-recommends \ + && apt-get install -y --no-install-recommends gosu libssl3t64 libgssapi-krb5-2 libldap2 libzstd1 liblz4-1 libreadline8t64 \ && rm -rf /var/lib/apt/lists/* \ && groupadd --system --gid 10001 umt \ && useradd --system --uid 10001 --gid 10001 --home-dir /data --no-create-home --shell /usr/sbin/nologin umt \ && mkdir -p /data && chown 10001:10001 /data WORKDIR /app +COPY --from=postgres-tools /usr/lib/postgresql/18/bin/pg_dump /usr/lib/postgresql/18/bin/pg_restore /usr/lib/postgresql/18/bin/psql /usr/local/bin/ +COPY --from=postgres-tools /usr/lib/*/libpq.so.5.18 /usr/local/lib/libpq.so.5 +RUN ldconfig && pg_dump --version && pg_restore --version && psql --version # /app 全部 root 属主、只读:运行用户(缺省 10001,可用 PUID/PGID 覆盖)改不了二进制与静态产物 COPY --from=ffprobe --chown=root:root --chmod=0555 /ffprobe /usr/local/bin/ffprobe COPY --from=build --chown=root:root --chmod=0555 /src/dist/umt-server /app/umt-server @@ -46,6 +53,10 @@ ENV NODE_ENV=production \ DATA_DIR=/data \ PUID=10001 \ PGID=10001 +ARG SOURCE_REVISION="" +ARG APP_VERSION="" +LABEL org.opencontainers.image.version=${APP_VERSION} \ + org.opencontainers.image.revision=${SOURCE_REVISION} # 以 root 启动只为修 /data 属主,entrypoint 随即 gosu 降权;api/worker/备份脚本都不以 root 运行 VOLUME ["/data"] EXPOSE 4380 diff --git a/README.md b/README.md index d59b4563..10c53a1b 100644 --- a/README.md +++ b/README.md @@ -2,16 +2,27 @@ 内部使用、Agent 自动化优先的媒体整理入库平台。需求基线见 [media_automation_requirements_architecture_v0.7.md](media_automation_requirements_architecture_v0.7.md),v0.1 工程任务书见 [docs/plan/00-first-version-plan.md](docs/plan/00-first-version-plan.md)。 +当前部署为 v0.3.13,媒体整理与手动下载入口见 [发布记录](docs/plan/18-organize-release.md)。依赖与容器升级见 [完整升级记录](docs/plan/17-full-upgrade.md);[开发情况基线](docs/plan/13-development-baseline.md) 保留升级前 v0.3.7 的证据,早期计划和验收表不作为当前完成度统计。恢复与验收调整见 [工程检查](docs/plan/14-engineering-review.md)。 + +逐项业务核对见 [MoviePilot v3 对照](docs/plan/15-moviepilot-v3-comparison.md):固定参考版本,按用户确认的需求实施。新一批独立手动下载、串流外详情页下载、TMDB/Local 元数据与一次性洗版已于 2026-10-01 部署为 v0.3.10;实现与验收见该文第 5.6 节,关联迁移 `0060`。 + +依赖、容器与运行程序的版本检查见 [依赖与程序版本报告](docs/plan/16-dependency-version-review.md):包含完整依赖清单、安全公告适用性、PG 18 迁移和 Pi 原生 MCP 的评估建议;本轮没有执行升级。 + +v0.3.13 将手动、文件管理、下载详情与历史重整统一为 MP 风格整理弹窗;支持电影/剧集、TMDB 搜索与 ID 定位、复制/移动、目标路径及本次刮削选项,仅加入后台整理队列。整理页分队列与记录,成功历史默认不跳过,成功移动后从目标重整;同时统一分页条数选择器并调整通知数字角标。 + +v0.3.14 已补充原地入库代码与迁移,保留路径/文件名,可选就地刮削;当前会话受权限限制尚未部署,生产仍为上述 v0.3.13。见 [发布记录](docs/plan/18-organize-release.md)。 + ## 功能 -已实现(全部只在 mock 服务上验证过,没有连过真实 PT、qB、Emby、camofox 与模型上游;逐条对照见 [docs/plan/11-requirements-coverage.md](docs/plan/11-requirements-coverage.md)): +已实现(有自动化回归及部分真实环境 / 脱敏样本记录,四站与真实下载、媒体服务器的完整验收尚未完成;验证边界见 [开发情况基线](docs/plan/13-development-baseline.md)): - 来源:多 RSS(条件请求、重叠窗口、多层去重、补漏)、站点中心(观众 / 馒头 / HHClub / CHD 等 23 个站点定义,凭证、账号数据、健康检查)、站点快照与实时补查、全局搜索。 - 识别:MP 对齐的标题解析、TMDB + 本地别名召回、本地实体优先、临时季集单元、待识别池、人工纠错与字段锁。 - 订阅与规则:单次 / 持续追更、多版本目标(Slot)、总集数账本、覆盖矩阵、日历;规则组(兼容 MP 规则字符串)、偏好标签与抑制记录;新作品 / 追更 / 升级三种收录模式。 - 下载:多 qB、管理标签接管、缓存卷硬阈值与恢复水位、无进展只记录不换源、缓存清理(入库已验证 + 保留期 + H&R)。 -- 整理:Copy / Move(跨卷可恢复、fsproxy 看门狗)、路径绑定固定、命名模板、文件管理 v2 与整理向导、回收站、公开整理记录增量接口。 -- Agent:Pi 内核、27 个能力、服务端授权(持续授权 / 一次性审批)、步骤清单 / 方案选择 / 内联审批、子会话、后台自动调查与授权内恢复、41 个上游 provider 与订阅登录。 +- 整理:Copy / Move(跨卷可恢复、fsproxy 看门狗)、路径绑定固定、命名模板、文件管理 v2 与统一整理弹窗、回收站、公开整理记录增量接口。 +- 刮削:任意视频 / 目录就地识别并写 NFO 与图片、候选选择、覆盖策略、入库后自动刮削、每次刮削一条记录。 +- Agent:Pi 内核、能力目录(当前源码 42 项)、服务端授权(持续授权 / 一次性审批)、步骤清单 / 方案选择 / 内联审批、子会话、后台自动调查与授权内恢复、42 个内置上游 provider 与订阅登录(未逐家联网实测)。 - 接口:REST + OpenAPI、MCP 服务端(API Key 鉴权)、SSE 带序号补读。 - 浏览器:camofox BrowserProvider、原站视图(截图 + 坐标输入)、接管租约、下载捕获走统一提交。 - 运维:管理员登录(TOTP / Passkey)、API Key 分级、主密钥轮换、定时任务看板、统一任务页、运行状态页、审计、备份与只读对账恢复。 @@ -24,13 +35,13 @@ ```sh cp .env.example .env -deno task db:up # postgres:16-alpine,端口 55433 +deno task db:up # postgres:18.6-alpine,端口 55433 deno install # 按 deno.lock 安装依赖到 node_modules deno task db:migrate deno task dev # API http://localhost:4380 控制台 http://localhost:5180 ``` -提交前:`deno task ci`(typecheck:tsc 7.0.2 + deno check → biome → vitest → check:tokens → web 构建 + server 编译)。 +提交前:`deno task ci`(typecheck:tsc 7.0.2 + deno check → biome → vitest → check:tokens → check:copy → web 构建 + server 编译 → check:binary)。 ### 前端冒烟(`deno task test:web:smoke`) @@ -45,13 +56,15 @@ SMOKE_BASE_URL=http://127.0.0.1:5180 SMOKE_USER=<管理员> SMOKE_PASSWORD=<密 ## 部署 -除 PostgreSQL 外全部在一个容器里(api + worker + worker:agent 三个进程 + web 静态,单端口 4380);compose 只有 `app` + `postgres`。 +核心应用在一个容器里(api + worker + worker:agent 三个进程 + web 静态,单端口 4380);compose 默认服务为 `app` + `postgres` + `camofox`,可选 `tls` profile 的 Caddy 提供 HTTPS。 + +首次部署默认浏览器前,先按 [浏览器镜像构建说明](docs/runbook/browser.md#11-部署自带缺省) 构建 `camofox-browser:1.18.0` 基础镜像,compose 再据此构建桌面镜像。 ```sh -cp docker/compose.env.example .env # 填 SECRET_MASTER_KEY=$(openssl rand -hex 32)、POSTGRES_PASSWORD -docker compose up -d --build -docker compose logs app | grep 'setup token' # 首次初始化管理员 -docker compose exec app umt backup # 备份到 /data/backups/<时间戳>/ +cp docker/compose.env.example .env.deploy # 填 SECRET_MASTER_KEY=$(openssl rand -hex 32)、POSTGRES_PASSWORD +docker compose --env-file .env.deploy up -d --build +docker compose --env-file .env.deploy logs app | grep 'setup token' # 首次初始化管理员 +docker compose --env-file .env.deploy exec app umt backup # 备份到 /data/backups/<时间戳>/ ``` - 应用数据卷 `/data`(Agent 会话、备份、导出);媒体库根与下载目录自行 bind mount,需对 `PUID:PGID`(缺省 10001)可读写。 @@ -64,7 +77,7 @@ docker compose exec app umt backup # 备份到 /data/backups/<时间 | 路径 | 职责 | |---|---| -| `apps/web` | React Router 7 SPA 控制台,沿用 uhd_web 设计系统 | +| `apps/web` | React Router 8 SPA 控制台,沿用 uhd_web 设计系统 | | `apps/server` | Hono API(`/api/v1`)+ SSE + worker 入口;业务按 `src/modules//` 分模块(见 apps/server/README.md) | | `packages/contracts` | zod schema:API/事件/能力的唯一定义 | | `packages/db` | Kysely 客户端、表类型与迁移 | @@ -82,4 +95,8 @@ docker compose exec app umt backup # 备份到 /data/backups/<时间 | `scripts/` | 备份 / 恢复脚本(镜像内 `/app/scripts/`) | | `spikes/` | 独立兼容性验证(不属于 workspace) | -技术选型:Deno 2.9.7(锁定)+ TypeScript 7.0.2 strict,deno.json workspace(`nodeModulesDir: auto`,npm 依赖版本统一登记在根 `deno.json` 的 `imports`,`deno.lock` 锁定)。前端 Vite 7 + React Router 7 由 Deno 执行。决策见 [ADR-004](docs/adr/ADR-004-deno-runtime.md)(取代 ADR-001)。 +技术选型:Deno 2.9.7(锁定)+ TypeScript 7.0.2 strict,deno.json workspace(`nodeModulesDir: auto`,npm 依赖版本统一登记在根 `deno.json` 的 `imports`,`deno.lock` 锁定)。前端 Vite 8 + React Router 8 由 Deno 执行。决策见 [ADR-004](docs/adr/ADR-004-deno-runtime.md)(取代 ADR-001)。 + +首页现展示存储和下载器概况,记录列表统一支持每页条数和翻页,搜索页与快捷搜索共用历史,右上角通知与控制台样式一致,见 [v0.3.12 发布记录](docs/plan/18-organize-release.md#v0312-列表分页搜索历史与首页概况)。 + +源站视图上下黑边与搜索资源详情入口已在 v0.3.11 发布中修复:浏览器窗口自动铺满桌面,搜索结果标题打开同一站点详情抽屉,见 [发布记录](docs/plan/18-organize-release.md#v0311-搜索详情与桌面黑边修复)。 diff --git a/apps/server/deno.json b/apps/server/deno.json index 66154c91..cb9ad937 100644 --- a/apps/server/deno.json +++ b/apps/server/deno.json @@ -1,6 +1,6 @@ { "name": "@umt/server", - "version": "0.3.7", + "version": "0.3.14", "exports": { ".": "./src/main.ts" }, "tasks": { "dev": "deno run -A --watch --env-file=../../.env src/main.ts", diff --git a/apps/server/src/modules/acquisition-organize/__tests__/classification.test.ts b/apps/server/src/modules/acquisition-organize/__tests__/classification.test.ts new file mode 100644 index 00000000..80f60e17 --- /dev/null +++ b/apps/server/src/modules/acquisition-organize/__tests__/classification.test.ts @@ -0,0 +1,142 @@ +import { OpenAPIHono } from "@hono/zod-openapi"; +import type { TmdbClient } from "@umt/adapters"; +import { DEFAULT_CLASSIFICATION_YAML } from "@umt/domain"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import type { AuthEnv } from "../../../http/auth.ts"; +import { createSettingsReader } from "../../auth-settings/provided.ts"; +import { registerSettingsRoutes } from "../../auth-settings/settings-routes.ts"; +import { SourcesRuntime } from "../../sources-identity/runtime.ts"; +import { classifyEntity } from "../services/classification.ts"; +import { createHarness, type Harness, pgAvailable, seedEntity } from "./harness.ts"; + +const hasPg = await pgAvailable(); +describe.skipIf(!hasPg)("TMDB classification API and accepted identities (real PG)", () => { + let h: Harness; + beforeAll(async () => { + h = await createHarness("classification"); + const settingsApp = new OpenAPIHono(); + registerSettingsRoutes(settingsApp, h.ctx); + h.app.route("/api/v1", settingsApp); + }, 60_000); + afterEach(() => vi.restoreAllMocks()); + afterAll(async () => await h?.close()); + + it("legacy stored organize settings receive default YAML without losing preferences", async () => { + await h.setSettings("organize", { minSizeBytes: 456789, conflictDefault: "skip" }); + const reader = createSettingsReader({ db: h.db, logger: h.ctx.logger }); + expect(await reader.get("organize")).toMatchObject({ + minSizeBytes: 456789, + classificationYaml: DEFAULT_CLASSIFICATION_YAML, + }); + }); + + it("ordered categories and raw preview do not write settings", async () => { + const view = await h.api("GET", "/classification"); + expect(view.status).toBe(200); + expect(view.body.data.yaml).toBe(DEFAULT_CLASSIFICATION_YAML); + expect(view.body.data.categories.movie).toEqual(["动画电影", "演唱会", "华语电影", "亚洲电影", "欧美电影"]); + const preview = await h.api("POST", "/classification/preview", { + yaml: "movie:\n 日本片:\n production_countries: JP\ntv: {}\n", + mediaType: "movie", + tmdb: { production_countries: [{ iso_3166_1: "JP" }] }, + }); + expect(preview.status).toBe(200); + expect(preview.body.data).toEqual({ category: "日本片", matchedFields: ["production_countries"], fallback: false }); + expect((await h.ctx.settings.get("organize")).classificationYaml).toBe(DEFAULT_CLASSIFICATION_YAML); + }); + + it("PUT preserves organize preferences, audits and emits settings change; invalid writes reject", async () => { + await h.setSettings("organize", { ...(await h.ctx.settings.get("organize")), minSizeBytes: 765432 }); + const publish = vi.spyOn(h.ctx.events, "publish"); + const yaml = "movie:\n 新分类:\n original_language: zh\n 其它: null\ntv: {}\n"; + const saved = await h.api("PUT", "/classification", { yaml }); + expect(saved.status).toBe(200); + expect(saved.body.data.categories.movie).toEqual(["新分类", "其它"]); + expect(await h.ctx.settings.get("organize")).toMatchObject({ classificationYaml: yaml, minSizeBytes: 765432 }); + expect(publish).toHaveBeenCalledWith("settings.changed", { keys: ["organize"] }); + expect( + (await h.db.selectFrom("audit_event").selectAll().where("action", "=", "settings.update").execute()).length, + ).toBeGreaterThan(0); + for (const invalid of ["movie:\n ../escape: null\ntv: {}", "movie:\n 同名: null\n 同名: null\ntv: {}"]) { + expect((await h.api("PUT", "/classification", { yaml: invalid })).status).toBe(422); + expect((await h.api("PATCH", "/settings", { organize: { classificationYaml: invalid } })).status).toBe(422); + } + expect((await h.ctx.settings.get("organize")).classificationYaml).toBe(yaml); + await h.api("PUT", "/classification", { yaml: DEFAULT_CLASSIFICATION_YAML }); + }); + + it("PUT merges current row after waiting rather than overwriting a newer source edit", async () => { + const source = { + id: "33333333-3333-4333-8333-333333333333", + name: "Concurrent source", + enabled: true, + source: { root: "lib:11111111-1111-4111-8111-111111111111", path: "" }, + target: { root: "lib:22222222-2222-4222-8222-222222222222", path: "" }, + mediaType: "all", + category: null, + action: "copy", + trigger: "manual", + downloaderInstanceId: null, + tag: null, + intervalSec: 300, + }; + const originalAll = h.ctx.settings.all.bind(h.ctx.settings); + vi.spyOn(h.ctx.settings, "all").mockImplementationOnce(async () => { + const stale = await originalAll(); + await h.setSettings("organize", { ...stale.organize, minSizeBytes: 654321, sources: [source] }); + return stale; + }); + expect((await h.api("PUT", "/classification", { yaml: DEFAULT_CLASSIFICATION_YAML })).status).toBe(200); + const stored = await h.db + .selectFrom("settings") + .select("value") + .where("key", "=", "organize") + .executeTakeFirstOrThrow(); + expect(stored.value).toMatchObject({ + minSizeBytes: 654321, + sources: [source], + classificationYaml: DEFAULT_CLASSIFICATION_YAML, + }); + }); + + it("requires accepted movie/tv identity, uses raw detail, and fails without fabricating fallback", async () => { + await h.setSettings("organize", { + ...(await h.ctx.settings.get("organize")), + classificationYaml: DEFAULT_CLASSIFICATION_YAML, + }); + const id = await seedEntity(h.db, { kind: "movie", title: "测试片", year: 2020 }); + const entity = await h.db.selectFrom("media_entity").selectAll().where("id", "=", id).executeTakeFirstOrThrow(); + const getJson = vi.fn().mockResolvedValue({ genres: [{ id: 16 }], original_language: "zh" }); + const factory = vi.spyOn(SourcesRuntime.prototype, "tmdb").mockResolvedValue({ getJson } as unknown as TmdbClient); + expect((await classifyEntity(h.d, entity)).category).toBeNull(); + expect(factory).not.toHaveBeenCalled(); + const identity = await h.db + .insertInto("external_identity") + .values({ media_entity_id: id, provider: "tmdb", entity_type: "movie", external_id: "123", status: "proposed" }) + .returning("id") + .executeTakeFirstOrThrow(); + expect((await classifyEntity(h.d, entity)).category).toBeNull(); + await h.db.updateTable("external_identity").set({ status: "accepted" }).where("id", "=", identity.id).execute(); + expect(await classifyEntity(h.d, entity)).toEqual({ + category: "动画电影", + matchedFields: ["genre_ids"], + fallback: false, + }); + expect(getJson).toHaveBeenCalledWith("/movie/123"); + getJson.mockRejectedValue(new Error("offline")); + expect(await classifyEntity(h.d, entity)).toEqual({ category: null, matchedFields: [], fallback: false }); + getJson.mockResolvedValue({}); + expect((await classifyEntity(h.d, entity)).category).toBeNull(); + getJson.mockResolvedValue({ genres: [{ id: 10764 }], origin_country: ["CN"] }); + const tvId = await seedEntity(h.db, { kind: "series", title: "测试剧", year: 2020 }); + const tv = await h.db.selectFrom("media_entity").selectAll().where("id", "=", tvId).executeTakeFirstOrThrow(); + await h.db + .insertInto("external_identity") + .values({ media_entity_id: tvId, provider: "tmdb", entity_type: "tv", external_id: "456", status: "accepted" }) + .execute(); + expect((await classifyEntity(h.d, tv)).category).toBe("综艺"); + expect(getJson).toHaveBeenCalledWith("/tv/456"); + factory.mockResolvedValue(null); + expect((await classifyEntity(h.d, entity)).category).toBeNull(); + }); +}); diff --git a/apps/server/src/modules/acquisition-organize/__tests__/enqueue-recovery.test.ts b/apps/server/src/modules/acquisition-organize/__tests__/enqueue-recovery.test.ts new file mode 100644 index 00000000..d91f3f40 --- /dev/null +++ b/apps/server/src/modules/acquisition-organize/__tests__/enqueue-recovery.test.ts @@ -0,0 +1,207 @@ +import type { TaskRunView } from "@umt/tasks"; +import { createTempTree, type TempTree } from "@umt/testkit"; +import { sql } from "kysely"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { enqueueSweep, enqueueTracked } from "../services/common.ts"; +import { type QueueScrapeInput, queueScrape, runScrapeRecord } from "../services/scrape.ts"; +import { createHarness, type Harness, pgAvailable, seedBase, seedEntity } from "./harness.ts"; + +const hasPg = await pgAvailable(); +const runView = (runId: string, input: unknown, status: TaskRunView["status"] = "pending"): TaskRunView => ({ + runId, + taskName: "scrapeMedia", + version: 2, + status, + attempts: 0, + idempotencyKey: null, + traceId: null, + priority: 0, + input, + output: null, + error: null, + cancelRequestedAt: null, + cancelReason: null, + availableAt: null, + createdAt: new Date(), + startedAt: null, + finishedAt: null, +}); + +describe.skipIf(!hasPg)("入队关联与巡检恢复", () => { + let h: Harness; + let lib: TempTree; + let rootId: string; + let entityId: string; + + beforeAll(async () => { + lib = await createTempTree({}, { sentinel: true }); + h = await createHarness("enqueue_recovery"); + ({ rootId } = await seedBase(h.db, { libraryPath: lib.root, cachePath: lib.root, qbUrl: "http://127.0.0.1:9" })); + entityId = await seedEntity(h.db, { kind: "movie", title: "恢复测试", year: 2020 }); + await sql`create function reject_association() returns trigger language plpgsql as $$ + begin + if new.engine_run_id is not null and new.engine_run_id is distinct from old.engine_run_id + then raise exception 'injected association failure'; end if; + return new; + end $$`.execute(h.db); + }); + + async function rejectAssociation(table: "task" | "scrape_record") { + await sql + .raw(`create trigger reject_association before update on ${table} + for each row execute function reject_association()`) + .execute(h.db); + } + + async function allowAssociation() { + for (const table of ["task", "scrape_record"]) + await sql.raw(`drop trigger if exists reject_association on ${table}`).execute(h.db); + } + + afterEach(async () => { + vi.restoreAllMocks(); + await allowAssociation(); + }); + afterAll(async () => { + await h?.close(); + await lib?.cleanup(); + }); + + const task = (key: string) => ({ kind: "reconcileLibrary" as const, idempotencyKey: key, input: {} }); + const scrape = (path: string, idempotencyKey?: string): QueueScrapeInput => ({ + rootId: `lib:${rootId}`, + rootPath: lib.root, + libraryRootId: rootId, + path, + match: { + mediaEntityId: entityId, + provider: null, + externalId: null, + kind: "movie", + title: "恢复测试", + year: 2020, + source: "chosen", + }, + trigger: "manual", + mode: "missing", + ...(idempotencyKey ? { idempotencyKey } : {}), + }); + + it("引擎查询异常跳过本轮,下一轮仍查原任务", async () => { + const t = task("query_error"); + const q = await enqueueTracked(h.d, t); + const enqueue = vi.spyOn(h.engine, "enqueue"); + const get = vi + .spyOn(h.engine, "get") + .mockRejectedValueOnce(new Error("temporary engine outage")) + .mockResolvedValue(runView(q.runId, {}, "running")); + expect(await enqueueSweep(h.d, t, "next")).toBeNull(); + expect(await enqueueSweep(h.d, t, "later")).toBeNull(); + expect(enqueue).not.toHaveBeenCalled(); + expect(get).toHaveBeenNthCalledWith(2, q.runId); + expect( + (await h.db.selectFrom("task").select("engine_run_id").where("id", "=", q.taskId).executeTakeFirstOrThrow()) + .engine_run_id, + ).toBe(q.runId); + }); + + it.each(["missing", "completed"])("明确 %s 时允许补投", async (state) => { + const t = task(`retry_${state}`); + const q = await enqueueTracked(h.d, t); + vi.spyOn(h.engine, "get").mockResolvedValue(state === "missing" ? null : runView(q.runId, {}, "completed")); + const enqueue = vi.spyOn(h.engine, "enqueue"); + expect(await enqueueSweep(h.d, t, "next")).not.toBeNull(); + expect(enqueue).toHaveBeenCalledOnce(); + expect(enqueue.mock.calls[0]?.[2].idempotencyKey).toBe(`${t.kind}:${t.idempotencyKey}:next`); + }); + + it.each([false, true])("关联写回失败后巡检按原 taskId 找回已接受的 run (旧关联=%s)", async (oldAssociation) => { + const t = task(`association_${oldAssociation}`); + if (oldAssociation) await enqueueTracked(h.d, t); + await rejectAssociation("task"); + const q = await enqueueTracked(h.d, t, { engineKey: "new-slot" }); + await h.db + .insertInto("tasks_run_meta") + .values({ run_id: q.runId, task_name: t.kind, task_version: 1, idempotency_key: "new-slot", trace_id: q.taskId }) + .execute(); + await allowAssociation(); + vi.spyOn(h.engine, "get").mockResolvedValue(runView(q.runId, {}, "running")); + const enqueue = vi.spyOn(h.engine, "enqueue"); + expect(await enqueueSweep(h.d, t, "later")).toBeNull(); + expect(enqueue).not.toHaveBeenCalled(); + expect( + (await h.db.selectFrom("task").select("engine_run_id").where("id", "=", q.taskId).executeTakeFirstOrThrow()) + .engine_run_id, + ).toBe(q.runId); + }); + + it("刮削已接受而关联失败时保持 queued,重试复用记录与原幂等键", async () => { + const input = scrape("association-retry"); + await rejectAssociation("scrape_record"); + const q = await queueScrape(h.d, input); + expect( + await h.db + .selectFrom("scrape_record") + .select(["status", "reason", "engine_run_id"]) + .where("id", "=", q.recordId) + .executeTakeFirstOrThrow(), + ).toMatchObject({ status: "queued", reason: null, engine_run_id: null }); + const key = h.engine.enqueued.at(-1)?.key; + await allowAssociation(); + const enqueue = vi.spyOn(h.engine, "enqueue").mockResolvedValue({ runId: q.taskId!, deduplicated: true }); + vi.spyOn(h.engine, "get").mockResolvedValue(runView(q.taskId!, { recordId: q.recordId })); + expect(await queueScrape(h.d, input)).toEqual({ recordId: q.recordId, taskId: q.taskId, deduplicated: true }); + expect(enqueue.mock.calls[0]?.[2].idempotencyKey).toBe(key); + expect((await h.db.selectFrom("scrape_record").select("id").where("path", "=", input.path).execute()).length).toBe( + 1, + ); + }); + + it("刮削处理器开始时补关联,不会把已接受的工作误标为入队失败", async () => { + await rejectAssociation("scrape_record"); + const q = await queueScrape(h.d, scrape("worker-backfill")); + await allowAssociation(); + await h.db.updateTable("library_root").set({ management_mode: "register_only" }).where("id", "=", rootId).execute(); + const r = await runScrapeRecord(h.d, q.recordId, undefined, q.taskId!); + expect(r.reason).toBe("scrape_root_read_only"); + expect( + ( + await h.db + .selectFrom("scrape_record") + .select("engine_run_id") + .where("id", "=", q.recordId) + .executeTakeFirstOrThrow() + ).engine_run_id, + ).toBe(q.taskId); + }); + + it("确实入队失败的新刮削记录标失败", async () => { + vi.spyOn(h.engine, "enqueue").mockRejectedValue(new Error("enqueue rejected")); + const input = scrape("enqueue-failed"); + await expect(queueScrape(h.d, input)).rejects.toThrow("enqueue rejected"); + expect( + await h.db + .selectFrom("scrape_record") + .select(["status", "reason"]) + .where("path", "=", input.path) + .executeTakeFirstOrThrow(), + ).toMatchObject({ status: "failed", reason: "task_enqueue_failed" }); + }); + + it("自动重放从原 run 输入找回未关联记录,只删除本次多建的记录", async () => { + const input = scrape("auto-replay", "auto-stable-key"); + await rejectAssociation("scrape_record"); + const q = await queueScrape(h.d, input); + await allowAssociation(); + await h.db.updateTable("scrape_record").set({ status: "succeeded" }).where("id", "=", q.recordId).execute(); + vi.spyOn(h.engine, "enqueue").mockResolvedValue({ runId: q.taskId!, deduplicated: true }); + vi.spyOn(h.engine, "get").mockResolvedValue(runView(q.taskId!, { recordId: q.recordId }, "completed")); + expect(await queueScrape(h.d, input)).toEqual({ recordId: q.recordId, taskId: q.taskId, deduplicated: true }); + const recs = await h.db + .selectFrom("scrape_record") + .select(["id", "status", "engine_run_id"]) + .where("path", "=", input.path) + .execute(); + expect(recs).toEqual([{ id: q.recordId, status: "succeeded", engine_run_id: q.taskId }]); + }); +}); diff --git a/apps/server/src/modules/acquisition-organize/__tests__/library-versions.test.ts b/apps/server/src/modules/acquisition-organize/__tests__/library-versions.test.ts index 6372e2fa..1c799f93 100644 --- a/apps/server/src/modules/acquisition-organize/__tests__/library-versions.test.ts +++ b/apps/server/src/modules/acquisition-organize/__tests__/library-versions.test.ts @@ -2,7 +2,7 @@ import type { DB } from "@umt/db"; import type { Kysely } from "kysely"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { backfillVersionSpecs } from "../services/version-backfill.ts"; -import { createHarness, type Harness, pgAvailable, seedEntity, seedSlot } from "./harness.ts"; +import { createHarness, type Harness, pgAvailable, seedEntity, seedRelease, seedSlot } from "./harness.ts"; // LIBV-1:入库版本 API(每集多个版本、季汇总、单目标比对)与规格回填(dry-run / apply)。 @@ -272,4 +272,68 @@ describe.skipIf(!hasPg)("LIBV-1 入库版本", () => { const dv = r.body.data.seasons[0].episodes[0].versions.find((x: { assetId: string }) => x.assetId === ids.e1dv); expect(dv).toMatchObject({ specStored: true, hdr: ["DV", "HDR10"], releaseGroup: "FLUX" }); }); + + it("作品详情保留跨过旧条数上限的整理、下载和失败计划供分页查看", async () => { + const entityId = await seedEntity(h.db, { kind: "movie", title: "历史分页测试", year: 2026 }); + const instance = await h.db + .insertInto("downloader_instance") + .values({ name: "历史下载器", base_url: "http://127.0.0.1:1" }) + .returning("id") + .executeTakeFirstOrThrow(); + await h.db + .insertInto("download_task") + .values( + Array.from({ length: 105 }, (_, i) => ({ + instance_id: instance.id, + media_entity_id: entityId, + torrent_hash: String(i + 1).padStart(40, "0"), + name: `历史下载 ${i + 1}`, + })), + ) + .execute(); + const slotId = await seedSlot(h.db, { entityId, name: "历史版本" }); + const site = await h.db + .insertInto("site") + .values({ key: "history-source", name: "历史来源", base_url: "http://127.0.0.1:1", adapter: "nexusphp" }) + .returning("id") + .executeTakeFirstOrThrow(); + const releaseId = await seedRelease(h.db, { + siteId: site.id, + entityId, + title: "历史分页测试", + size: 1000, + hash: "a".repeat(40), + }); + await h.db + .insertInto("acquisition_plan") + .values( + Array.from({ length: 51 }, (_, i) => ({ + slot_id: slotId, + source_release_id: releaseId, + status: "failed" as const, + payload_hash: `history-${i}`, + idempotency_key: `history:${i}`, + })), + ) + .execute(); + await h.db + .insertInto("organization_record") + .values( + Array.from({ length: 105 }, (_, i) => ({ + media_entity_id: entityId, + trigger: "manual" as const, + actor_type: "user" as const, + action: "copy" as const, + status: "succeeded" as const, + source_path: `/history/${i}.mkv`, + target_path: `/library/history/${i}.mkv`, + })), + ) + .execute(); + const response = await h.api("GET", `/library/${entityId}`); + expect(response.status).toBe(200); + expect(response.body.data.organizationRecords).toHaveLength(105); + expect(response.body.data.downloads).toHaveLength(105); + expect(response.body.data.droppedPlans).toHaveLength(51); + }); }); diff --git a/apps/server/src/modules/acquisition-organize/__tests__/manual-acquisition.test.ts b/apps/server/src/modules/acquisition-organize/__tests__/manual-acquisition.test.ts new file mode 100644 index 00000000..bcbe23fc --- /dev/null +++ b/apps/server/src/modules/acquisition-organize/__tests__/manual-acquisition.test.ts @@ -0,0 +1,153 @@ +import { ManualAcquisitionRequest } from "@umt/contracts"; +import { createMockQb, createTempTree, makeTorrent } from "@umt/testkit"; +import { describe, expect, it } from "vitest"; +import { evaluateAcquisition } from "../services/acquisition.ts"; +import { submitDownload } from "../services/downloads.ts"; +import { planManualAcquisition } from "../services/manual-acquisition.ts"; +import { createHarness, fakeDisk, pgAvailable, seedBase, seedEntity, seedRelease, seedSlot } from "./harness.ts"; + +const hasPg = await pgAvailable(); +describe.skipIf(!hasPg)("独立手动下载", () => { + it("无订阅下载,未知资源确认,同hash沿用原路径", async () => { + const qb = await createMockQb({ addVisibleAfterPolls: 0 }); + const cache = await createTempTree({}); + const lib = await createTempTree({}, { sentinel: true }); + const torrents = new Map(); + const disk = fakeDisk(10_000_000); + disk.setPct(20); + const h = await createHarness("manual_acquisition", { + diskUsage: disk.fn, + torrentSource: async ({ release }) => ({ kind: "file", bytes: torrents.get(release.id)! }), + }); + try { + const base = await seedBase(h.db, { libraryPath: lib.root, cachePath: cache.root, qbUrl: qb.url }); + const entityId = await seedEntity(h.db, { kind: "movie", title: "Manual Movie", year: 2026 }); + const torrent = makeTorrent({ name: "Manual.Movie.2026.1080p.mkv", length: 300_000 }); + const relId = await seedRelease(h.db, { + siteId: base.siteId, + entityId, + title: torrent.name, + size: 300_000, + hash: torrent.hash, + }); + torrents.set(relId, torrent.bytes); + const request = (id: string, opts = {}) => ManualAcquisitionRequest.parse({ sourceReleaseId: id, ...opts }); + const first = await planManualAcquisition(h.ctx, request(relId)); + expect(first).toMatchObject({ submitted: true, slotId: null }); + expect(await h.db.selectFrom("subscription").selectAll().execute()).toHaveLength(0); + expect(await h.db.selectFrom("acquisition_slot").selectAll().execute()).toHaveLength(0); + const sub = await submitDownload(h.d, first.plan!.id); + expect(sub.status).toBe("submitted"); + const rel2 = await seedRelease(h.db, { + siteId: base.siteId, + entityId, + title: "Other.Release.2160p.mkv", + size: 300_000, + hash: torrent.hash, + }); + torrents.set(rel2, torrent.bytes); + const second = await planManualAcquisition(h.ctx, request(rel2, { savePath: `${cache.root}/other` })); + const reused = await submitDownload(h.d, second.plan!.id); + expect(reused).toMatchObject({ reason: "already_exists", downloadTaskId: sub.downloadTaskId }); + const dt = await h.db + .selectFrom("download_task") + .selectAll() + .where("id", "=", sub.downloadTaskId!) + .executeTakeFirstOrThrow(); + expect(dt.save_path).toBe(cache.root); + expect(dt.media_entity_id).toBe(entityId); + const unknown = makeTorrent({ name: "Unknown.1080p.mkv", length: 300_000, pieceSeed: 42 }); + const uid = await seedRelease(h.db, { + siteId: base.siteId, + entityId, + title: unknown.name, + size: 300_000, + hash: unknown.hash, + }); + await h.db.updateTable("source_release").set({ media_entity_id: null }).where("id", "=", uid).execute(); + torrents.set(uid, unknown.bytes); + expect(await planManualAcquisition(h.ctx, request(uid))).toMatchObject({ + submitted: false, + requiresConfirmation: true, + }); + const confirmed = await planManualAcquisition(h.ctx, request(uid, { allowUnrecognized: true })); + const submitted = await submitDownload(h.d, confirmed.plan!.id); + expect(submitted.status).toBe("submitted"); + expect( + ( + await h.db + .selectFrom("download_task") + .select("media_entity_id") + .where("id", "=", submitted.downloadTaskId!) + .executeTakeFirstOrThrow() + ).media_entity_id, + ).toBeNull(); + const autoEntity = await seedEntity(h.db, { kind: "movie", title: "Auto Movie", year: 2026 }); + const slotId = await seedSlot(h.db, { entityId: autoEntity }); + const autoTorrent = makeTorrent({ + name: "Auto.Movie.2026.1080p.BluRay.x264-GRP.mkv", + length: 300_000, + pieceSeed: 100, + }); + const autoRelease = await seedRelease(h.db, { + siteId: base.siteId, + entityId: autoEntity, + title: autoTorrent.name, + size: 300_000, + hash: autoTorrent.hash, + }); + torrents.set(autoRelease, autoTorrent.bytes); + const auto = await evaluateAcquisition(h.d, { sourceReleaseId: autoRelease }); + expect(auto.plans).toHaveLength(1); + const autoTask = await submitDownload(h.d, auto.plans[0]!); + const oldAsset = await h.db + .insertInto("media_asset") + .values({ media_entity_id: autoEntity, kind: "video", size_bytes: 300_000, status: "active" }) + .returning("id") + .executeTakeFirstOrThrow(); + await h.db + .updateTable("download_task") + .set({ status: "imported" }) + .where("id", "=", autoTask.downloadTaskId!) + .execute(); + const unchanged = await planManualAcquisition(h.ctx, request(autoRelease)); + expect(unchanged.plan!.id).toBe(auto.plans[0]); + const wash = await planManualAcquisition( + h.ctx, + request(autoRelease, { importMode: "replace", replacementAssetIds: [oldAsset.id] }), + ); + const washed = await submitDownload(h.d, wash.plan!.id); + expect(washed.downloadTaskId).toBe(autoTask.downloadTaskId); + const linked = await h.db + .selectFrom("download_task") + .selectAll() + .where("id", "=", autoTask.downloadTaskId!) + .executeTakeFirstOrThrow(); + expect(linked).toMatchObject({ + acquisition_plan_id: auto.plans[0], + import_plan_id: wash.plan!.id, + status: "completed", + }); + expect( + ( + await h.db + .selectFrom("acquisition_plan") + .select("slot_id") + .where("id", "=", auto.plans[0]!) + .executeTakeFirstOrThrow() + ).slot_id, + ).toBe(slotId); + expect(h.engine.enqueued).toContainEqual( + expect.objectContaining({ + task: "verifyAndImport", + input: expect.objectContaining({ downloadTaskId: autoTask.downloadTaskId }), + }), + ); + } finally { + await h.close(); + await qb.close(); + await cache.cleanup(); + await lib.cleanup(); + } + }); +}); diff --git a/apps/server/src/modules/acquisition-organize/__tests__/organize-sources.test.ts b/apps/server/src/modules/acquisition-organize/__tests__/organize-sources.test.ts new file mode 100644 index 00000000..7adf1147 --- /dev/null +++ b/apps/server/src/modules/acquisition-organize/__tests__/organize-sources.test.ts @@ -0,0 +1,248 @@ +import { randomUUID } from "node:crypto"; +import { join } from "node:path"; +import type { TmdbClient } from "@umt/adapters"; +import { DEFAULT_SETTINGS, type OrganizeSourceConfig } from "@umt/contracts"; +import { createMockQb, createTempTree, type MockQb, type TempTree } from "@umt/testkit"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { SourcesRuntime } from "../../sources-identity/runtime.ts"; +import { syncDownloads } from "../services/downloads.ts"; +import { executeRecord, planImport } from "../services/organize.ts"; +import { createHarness, type Harness, pgAvailable, seedBase, seedEntity } from "./harness.ts"; + +const hasPg = await pgAvailable(); +const FILE = "Example.2020.1080p.BluRay.x264-GRP.mkv"; +describe.skipIf(!hasPg)("configured organize sources (real PG, files, mock qB)", () => { + let h: Harness; + let lib: TempTree; + let cache: TempTree; + let qb: MockQb; + let otherQb: MockQb; + let base: Awaited>; + let otherInstance: string; + beforeAll(async () => { + lib = await createTempTree({}, { sentinel: true }); + cache = await createTempTree({ [`a/${FILE}`]: 200_000, [`b/${FILE}`]: 200_000, [`tagged/${FILE}`]: 200_000 }); + qb = await createMockQb(); + otherQb = await createMockQb(); + h = await createHarness("source_config"); + base = await seedBase(h.db, { libraryPath: lib.root, cachePath: cache.root, qbUrl: qb.url }); + otherInstance = ( + await h.db + .insertInto("downloader_instance") + .values({ + name: "other-qb", + base_url: otherQb.url, + username: "admin", + settings: JSON.stringify({ savePath: cache.root }), + }) + .returning("id") + .executeTakeFirstOrThrow() + ).id; + }, 60_000); + afterEach(() => vi.restoreAllMocks()); + afterAll(async () => { + await h?.close(); + await qb?.close(); + await otherQb?.close(); + await lib?.cleanup(); + await cache?.cleanup(); + }); + const profile = (patch: Partial = {}): OrganizeSourceConfig => ({ + id: randomUUID(), + name: "整理来源", + enabled: true, + source: { root: `dl:${base.instanceId}:0`, path: "a" }, + target: { root: `lib:${base.rootId}`, path: "华语电影" }, + mediaType: "movie", + category: "华语电影", + action: "copy", + trigger: "manual", + downloaderInstanceId: null, + tag: null, + intervalSec: 300, + ...patch, + }); + const settings = ( + sources: OrganizeSourceConfig[], + classificationYaml = DEFAULT_SETTINGS.organize.classificationYaml, + ) => h.setSettings("organize", { ...DEFAULT_SETTINGS.organize, minSizeBytes: 100_000, sources, classificationYaml }); + const request = (entityId: string, source: OrganizeSourceConfig, folder = "a") => ({ + sourcePaths: [cache.path(`${folder}/${FILE}`)], + mediaEntityId: entityId, + sourceConfigId: source.id, + trigger: "manual" as const, + actorType: "user" as const, + dryRun: true, + }); + + it("fixed category adds no duplicate; explicit same-category source selects own target and rejects wrong source", async () => { + const a = profile(); + const b = profile({ + source: { root: `dl:${base.instanceId}:0`, path: "b" }, + target: { root: `lib:${base.rootId}`, path: "另一个目录/华语电影" }, + }); + await settings([a, b]); + const movie = await seedEntity(h.db, { kind: "movie", title: "固定片", year: 2020 }); + const first = await planImport(h.d, request(movie, a)); + expect(first.plan?.sourceConfigId).toBe(a.id); + expect(first.plan?.newBindings[0]?.relativePath).toBe("华语电影/固定片 (2020)"); + const explicit = await planImport(h.d, { ...request(movie, a), targetDir: join(lib.root, "华语电影") }); + expect(explicit.plan?.newBindings).toEqual([]); + const explicitDst = explicit.plan?.organize.files.find((f) => f.role === "main")?.dst; + expect(explicitDst?.startsWith(`${join(lib.root, "华语电影")}/`)).toBe(true); + expect(explicitDst?.includes("华语电影/华语电影")).toBe(false); + const second = await planImport(h.d, request(movie, b, "b")); + expect(second.plan?.sourceConfigId).toBe(b.id); + expect(second.plan?.newBindings[0]?.relativePath).toBe("另一个目录/华语电影/固定片 (2020)"); + const mismatch = await planImport(h.d, request(movie, a, "b")); + expect(mismatch.reason).toBe("source_config_mismatch"); + expect(mismatch.plan).toBeUndefined(); + }); + + it("automatic category prefixes new binding; changed YAML/target preserves existing binding", async () => { + const automatic = profile({ category: null, target: { root: `lib:${base.rootId}`, path: "自动" } }); + await settings([automatic]); + const movie = await seedEntity(h.db, { kind: "movie", title: "动画片", year: 2020 }); + await h.db + .insertInto("external_identity") + .values({ + media_entity_id: movie, + provider: "tmdb", + entity_type: "movie", + external_id: "987", + status: "accepted", + }) + .execute(); + const getJson = vi.fn().mockResolvedValue({ genres: [{ id: 16 }], original_language: "zh" }); + vi.spyOn(SourcesRuntime.prototype, "tmdb").mockResolvedValue({ getJson } as unknown as TmdbClient); + const plan = await planImport(h.d, { ...request(movie, automatic), dryRun: false }); + expect(plan.status).toBe("running"); + const record = await h.db + .selectFrom("organization_record") + .select("plan") + .where("id", "=", plan.recordId!) + .executeTakeFirstOrThrow(); + expect(record.plan).toMatchObject({ + classification: { category: "动画电影" }, + newBindings: [{ relativePath: "自动/动画电影/动画片 (2020)" }], + }); + expect((await executeRecord(h.d, plan.recordId!)).status).toBe("succeeded"); + await settings( + [{ ...automatic, target: { ...automatic.target, path: "新目录" } }], + "movie:\n 新分类: null\ntv: {}\n", + ); + const updated = await planImport(h.d, request(movie, automatic)); + expect(updated.plan?.classification?.category).toBe("新分类"); + expect(updated.plan?.newBindings).toEqual([]); + expect(updated.plan?.organize.files.find((f) => f.role === "main")?.dst).toContain( + join(lib.root, "自动", "动画电影", "动画片 (2020)"), + ); + expect( + ( + await h.db + .selectFrom("path_binding") + .select("relative_path") + .where("media_entity_id", "=", movie) + .executeTakeFirstOrThrow() + ).relative_path, + ).toBe("自动/动画电影/动画片 (2020)"); + }); + + it("explicit root creates an unbound path there and refuses to relocate an existing binding", async () => { + const alternate = await createTempTree({}, { sentinel: true }); + const alternateRoot = await h.db + .insertInto("library_root") + .values({ name: "alternate", media_kind: "mixed", local_path: alternate.root, management_mode: "managed" }) + .returning("id") + .executeTakeFirstOrThrow(); + try { + const source = profile(); + await settings([source]); + const fresh = await seedEntity(h.db, { kind: "movie", title: "新根片", year: 2020 }); + const override = await planImport(h.d, { ...request(fresh, source), targetLibraryRootId: alternateRoot.id }); + expect(override.plan?.libraryRootId).toBe(alternateRoot.id); + expect(override.plan?.newBindings[0]?.relativePath).toBe("新根片 (2020)"); + const existing = await seedEntity(h.db, { kind: "movie", title: "已绑定片", year: 2020 }); + const first = await planImport(h.d, { ...request(existing, source), dryRun: false }); + expect((await executeRecord(h.d, first.recordId!)).status).toBe("succeeded"); + const attempted = await planImport(h.d, { ...request(existing, source), targetLibraryRootId: alternateRoot.id }); + expect(attempted.reason).toBe("binding_on_other_root"); + expect( + await h.db + .selectFrom("path_binding") + .select(["library_root_id", "relative_path"]) + .where("media_entity_id", "=", existing) + .executeTakeFirstOrThrow(), + ).toMatchObject({ library_root_id: base.rootId, relative_path: "华语电影/已绑定片 (2020)" }); + } finally { + await h.db.updateTable("library_root").set({ enabled: false }).where("id", "=", alternateRoot.id).execute(); + await alternate.cleanup(); + } + }); + + it("tag and source path adopt only matching instance; removing tag stops management", async () => { + const tag = "source-only-tag"; + await settings([ + profile({ + source: { root: `dl:${base.instanceId}:0`, path: "tagged" }, + trigger: "qb_tag", + downloaderInstanceId: base.instanceId, + tag, + }), + ]); + const hash = (n: number) => n.toString(16).padStart(40, "0"); + qb.seed({ hash: hash(1), name: FILE, save_path: cache.path("tagged"), tags: [tag] }); + qb.seed({ hash: hash(2), name: "No.Tag.mkv", save_path: cache.path("tagged"), tags: [] }); + qb.seed({ hash: hash(3), name: "Wrong.Path.mkv", save_path: cache.path("b"), tags: [tag] }); + otherQb.seed({ hash: hash(1), name: FILE, save_path: cache.path("tagged"), tags: [tag] }); + const result = await syncDownloads(h.d); + expect(result.adopted).toBe(1); + const rows = await h.db.selectFrom("download_task").selectAll().execute(); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ instance_id: base.instanceId, torrent_hash: hash(1), managed: true }); + expect(rows.some((r) => r.instance_id === otherInstance)).toBe(false); + qb.get(hash(1))!.tags = []; + await syncDownloads(h.d); + expect( + (await h.db.selectFrom("download_task").selectAll().where("id", "=", rows[0]!.id).executeTakeFirstOrThrow()) + .managed, + ).toBe(false); + qb.get(hash(1))!.tags = [tag]; + await syncDownloads(h.d); + expect( + (await h.db.selectFrom("download_task").selectAll().where("id", "=", rows[0]!.id).executeTakeFirstOrThrow()) + .managed, + ).toBe(true); + }); + it("automatic source with movie-only media type refuses a TV entity instead of generic organization", async () => { + const tag = "movie-source"; + await settings([ + profile({ + source: { root: `dl:${base.instanceId}:0`, path: "tagged" }, + trigger: "qb_tag", + downloaderInstanceId: base.instanceId, + tag, + }), + ]); + const hash = "4".padStart(40, "0"); + const name = "Only.TV.S01E01.1080p.WEB-DL.x264-GRP.mkv"; + qb.seed({ hash, name, save_path: cache.path("tagged"), tags: [tag] }); + await qb.complete(hash, { files: [{ path: name, size: 200_000 }] }); + await syncDownloads(h.d); + const row = await h.db + .selectFrom("download_task") + .selectAll() + .where("torrent_hash", "=", hash) + .executeTakeFirstOrThrow(); + const entity = await seedEntity(h.db, { kind: "series", title: "Only TV", year: 2020 }); + await h.db.updateTable("download_task").set({ media_entity_id: entity }).where("id", "=", row.id).execute(); + const planned = await planImport(h.d, { + downloadTaskId: row.id, + trigger: "auto", + actorType: "system", + dryRun: true, + }); + expect(planned.reason).toBe("source_config_mismatch"); + expect(planned.plan).toBeUndefined(); + }); +}); diff --git a/apps/server/src/modules/acquisition-organize/__tests__/records-filter.test.ts b/apps/server/src/modules/acquisition-organize/__tests__/records-filter.test.ts index 92c0c207..07172472 100644 --- a/apps/server/src/modules/acquisition-organize/__tests__/records-filter.test.ts +++ b/apps/server/src/modules/acquisition-organize/__tests__/records-filter.test.ts @@ -108,6 +108,33 @@ describe.skipIf(skip)("整理记录筛选与统计", () => { expect(movies.body.data).toMatchObject({ total: 1, succeeded: 1, failed: 0 }); }); + it("历史分页和统计排除排队与运行记录,显式active仍可查询", async () => { + const inserted = await h.db + .insertInto("organization_record") + .values([ + { trigger: "manual", actor_type: "user", action: "copy", status: "queued", source_path: "/queued.mkv" }, + { trigger: "manual", actor_type: "user", action: "move", status: "running", source_path: "/running.mkv" }, + ]) + .returning("id") + .execute(); + try { + expect(await list("phase=history")).toHaveLength(4); + expect(idsOf(await list("phase=active"))).toEqual(inserted.map((item) => item.id).sort()); + expect(await list("phase=history&status=queued")).toHaveLength(0); + const stats = await h.api("GET", "/organization-records/stats?phase=history"); + expect(stats.body.data).toMatchObject({ total: 4, running: 0 }); + } finally { + await h.db + .deleteFrom("organization_record") + .where( + "id", + "in", + inserted.map((item) => item.id), + ) + .execute(); + } + }); + it("列表项带海报、大小、下载器名称与 hash", async () => { const [film] = await list("q=Arrival.2016"); expect(film).toMatchObject({ diff --git a/apps/server/src/modules/acquisition-organize/__tests__/replacement.test.ts b/apps/server/src/modules/acquisition-organize/__tests__/replacement.test.ts new file mode 100644 index 00000000..4f7d5566 --- /dev/null +++ b/apps/server/src/modules/acquisition-organize/__tests__/replacement.test.ts @@ -0,0 +1,440 @@ +import { randomUUID } from "node:crypto"; +import { existsSync } from "node:fs"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { dirname } from "node:path"; +import { DEFAULT_SETTINGS } from "@umt/contracts"; +import { createMockQb, createTempTree, type MockQb, type TempTree } from "@umt/testkit"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { executeRecord, planImport } from "../services/organize.ts"; +import { createHarness, type Harness, pgAvailable, seedBase, seedEntity } from "./harness.ts"; + +const hasPg = await pgAvailable(); +describe.skipIf(!hasPg)("one-time replacement (real PG and library files)", () => { + let h: Harness; + let lib: TempTree; + let src: TempTree; + let rootId: string; + let instanceId: string; + let siteId: string; + let qb: MockQb; + beforeAll(async () => { + lib = await createTempTree({}, { sentinel: true }); + src = await createTempTree({ + "Movie.2020.1080p.BluRay.x264-GRP.mkv": 200_000, + "Same.2020.1080p.BluRay.x264-GRP.mkv": 200_000, + ...Object.fromEntries( + Array.from({ length: 6 }, (_, i) => [ + `Show.S01E${String(i + 1).padStart(2, "0")}.1080p.WEB-DL.x264-GRP.mkv`, + 200_000, + ]), + ), + }); + qb = await createMockQb(); + h = await createHarness("replacement"); + ({ rootId, instanceId, siteId } = await seedBase(h.db, { + libraryPath: lib.root, + cachePath: src.root, + qbUrl: qb.url, + })); + await h.setSettings("naming", { + tv: "{{title}} ({{year}})/Season {{season}}/{{title}} - {{season_episode}} - {{videoFormat}}{{fileExt}}", + }); + }, 60_000); + afterAll(async () => { + await h?.close(); + await qb?.close(); + await lib?.cleanup(); + await src?.cleanup(); + }); + const old = async (entityId: string, path: string, parent?: string) => { + await mkdir(dirname(lib.path(path)), { recursive: true }); + await writeFile(lib.path(path), Buffer.alloc(200_001, 7)); + const asset = await h.db + .insertInto("media_asset") + .values({ + media_entity_id: entityId, + kind: parent ? "subtitle" : "video", + parent_asset_id: parent ?? null, + status: "active", + size_bytes: 200_001, + }) + .returning("id") + .executeTakeFirstOrThrow(); + await h.db + .insertInto("asset_location") + .values({ + media_asset_id: asset.id, + location_kind: "library", + library_root_id: rootId, + relative_path: path, + status: "available", + is_primary: true, + }) + .execute(); + return asset.id; + }; + const request = (entityId: string, paths: string[], assetIds: string[] = []) => ({ + mediaEntityId: entityId, + sourcePaths: paths.map((p) => src.path(p)), + trigger: "manual" as const, + actorType: "user" as const, + action: "copy" as const, + importMode: "replace" as const, + replacementAssetIds: assetIds, + }); + const status = async (id: string) => + (await h.db.selectFrom("media_asset").select("status").where("id", "=", id).executeTakeFirstOrThrow()).status; + it("new movie registers first, retires only selected old and its private subtitle, preserving unselected/shared/cache", async () => { + const e = await seedEntity(h.db, { + kind: "movie", + title: "Movie", + year: 2020, + }); + const path = "Movie (2020)/Movie (2020) - 720p.mkv"; + const selected = await old(e, path); + const subtitle = await old(e, path.replace(".mkv", ".en.srt"), selected); + const kept = await old(e, "Movie (2020)/Movie (2020) - 2160p.mkv"); + await writeFile(lib.path("Movie (2020)/movie.nfo"), "shared"); + const planned = await planImport(h.d, request(e, ["Movie.2020.1080p.BluRay.x264-GRP.mkv"], [selected])); + expect(planned.status).toBe("running"); + expect(existsSync(lib.path(path))).toBe(true); + expect((await executeRecord(h.d, planned.recordId!)).status).toBe("succeeded"); + expect(await status(selected)).toBe("retired"); + expect(await status(subtitle)).toBe("retired"); + expect(await status(kept)).toBe("active"); + expect(existsSync(lib.path(path))).toBe(false); + expect(existsSync(lib.path("Movie (2020)/movie.nfo"))).toBe(true); + expect(existsSync(src.path("Movie.2020.1080p.BluRay.x264-GRP.mkv"))).toBe(true); + expect(await h.db.selectFrom("trash_item").selectAll().execute()).toHaveLength(2); + await executeRecord(h.d, planned.recordId!); + expect(await h.db.selectFrom("trash_item").selectAll().execute()).toHaveLength(2); + }); + it("same-path replaces selected old asset with a fresh asset, but normal/unselected cannot overwrite", async () => { + const e = await seedEntity(h.db, { + kind: "movie", + title: "Same", + year: 2020, + }); + const selected = await old(e, "Same (2020)/Same (2020) - 1080p.mkv"); + const normal = await planImport(h.d, { + ...request(e, ["Same.2020.1080p.BluRay.x264-GRP.mkv"]), + importMode: "normal", + dryRun: true, + }); + expect(normal.plan?.organize.files[0]?.action).toBe("hold"); + const planned = await planImport(h.d, request(e, ["Same.2020.1080p.BluRay.x264-GRP.mkv"], [selected])); + expect(planned.status).toBe("running"); + expect((await executeRecord(h.d, planned.recordId!)).status).toBe("succeeded"); + expect(await status(selected)).toBe("retired"); + const active = await h.db + .selectFrom("media_asset") + .select("id") + .where("media_entity_id", "=", e) + .where("status", "=", "active") + .execute(); + expect(active).toHaveLength(1); + expect(active[0]!.id).not.toBe(selected); + }); + it("selected grants cannot overwrite an unselected path, and changed old files survive cleanup", async () => { + const e = await seedEntity(h.db, { + kind: "movie", + title: "Protected", + year: 2020, + }); + const lower = await old(e, "Protected (2020)/Protected (2020) - 720p.mkv"); + const occupied = await old(e, "Protected (2020)/Protected (2020) - 1080p.mkv"); + const blocked = await planImport(h.d, { + ...request(e, ["Movie.2020.1080p.BluRay.x264-GRP.mkv"], [lower]), + dryRun: true, + }); + expect(blocked.plan?.organize.files[0]?.action).toBe("hold"); + expect(await status(occupied)).toBe("active"); + const other = await seedEntity(h.db, { + kind: "movie", + title: "Stale", + year: 2020, + }); + const oldPath = "Stale (2020)/Stale (2020) - 720p.mkv"; + const stale = await old(other, oldPath); + const planned = await planImport(h.d, request(other, ["Movie.2020.1080p.BluRay.x264-GRP.mkv"], [stale])); + await writeFile(lib.path(oldPath), "changed-after-plan"); + expect((await executeRecord(h.d, planned.recordId!)).status).toBe("succeeded"); + expect(await status(stale)).toBe("active"); + expect((await readFile(lib.path(oldPath))).toString()).toBe("changed-after-plan"); + }); + + it("same-path failed final probe restores old bytes and asset, even with delete policy", async () => { + const e = await seedEntity(h.db, { + kind: "movie", + title: "Same", + year: 2021, + }); + const path = "Same (2021)/Same (2021) - 1080p.mkv"; + const selected = await old(e, path); + const savedProbe = h.d.ports.probe; + const savedConfig = h.ctx.config; + h.ctx.config = { ...savedConfig, FS_REPLACE_OLD: "delete" }; + h.d.ports.probe = async () => ({ + status: "ok", + video: [], + audio: [], + subtitles: [], + }); + try { + const planned = await planImport(h.d, request(e, ["Same.2020.1080p.BluRay.x264-GRP.mkv"], [selected])); + expect(planned.status).toBe("running"); + expect((await executeRecord(h.d, planned.recordId!)).status).toBe("failed"); + expect((await readFile(lib.path(path))).equals(Buffer.alloc(200_001, 7))).toBe(true); + expect(await status(selected)).toBe("active"); + expect( + ( + await h.db + .selectFrom("asset_location") + .select("status") + .where("media_asset_id", "=", selected) + .executeTakeFirstOrThrow() + ).status, + ).toBe("available"); + await executeRecord(h.d, planned.recordId!); + expect((await readFile(lib.path(path))).length).toBe(200_001); + } finally { + h.d.ports.probe = savedProbe; + h.ctx.config = savedConfig; + } + }); + + it("old E01-E10 only loses successful new E01-E06; failed E04 and old merged range survive", async () => { + const e = await seedEntity(h.db, { + kind: "series", + title: "Show", + year: 2020, + }); + const ids: string[] = []; + for (let i = 1; i <= 10; i++) { + ids.push(await old(e, `Show (2020)/Season 1/Show - S01E${String(i).padStart(2, "0")} - 720p.mkv`)); + } + const merged = await old(e, "Show (2020)/Season 1/Show - S01E01-E10 - 480p.mkv"); + const saved = h.d.ports.probe; + h.d.ports.probe = async (path) => + path.includes("E04") + ? { status: "ok", video: [], audio: [], subtitles: [] } + : { status: "unknown", reason: "ffprobe_unavailable" }; + try { + const paths = Array.from( + { length: 6 }, + (_, i) => `Show.S01E${String(i + 1).padStart(2, "0")}.1080p.WEB-DL.x264-GRP.mkv`, + ); + const planned = await planImport(h.d, request(e, paths, [...ids, merged])); + expect(planned.status).toBe("running"); + expect((await executeRecord(h.d, planned.recordId!)).status).toBe("partial"); + for (let i = 1; i <= 10; i++) { + expect(await status(ids[i - 1]!)).toBe(i <= 6 && i !== 4 ? "retired" : "active"); + } + expect(await status(merged)).toBe("active"); + } finally { + h.d.ports.probe = saved; + } + }); + it("unknown manual task resolves Local, classifies and registers in the same import, then explicit reused-hash replacement uses its new plan", async () => { + const name = "Unknown.2022.1080p.WEB-DL.x264-GRP.mkv"; + const hash = "a".repeat(40); + qb.seed({ hash, name, tags: ["umt"], save_path: src.root }); + await qb.complete(hash, { + writeFiles: true, + files: [{ path: name, size: 200_000 }], + }); + const release = await h.db + .insertInto("source_release") + .values({ + site_id: siteId, + site_release_id: randomUUID(), + title: name, + size_bytes: 200_000, + raw: JSON.stringify({}), + resolution_status: "unresolved", + }) + .returning("id") + .executeTakeFirstOrThrow(); + const makePlan = async (evaluation: object) => + ( + await h.db + .insertInto("acquisition_plan") + .values({ + slot_id: null, + source_release_id: release.id, + status: "submitted", + payload_hash: randomUUID(), + idempotency_key: randomUUID(), + evaluation: JSON.stringify(evaluation), + }) + .returning("id") + .executeTakeFirstOrThrow() + ).id; + const initialPlan = await makePlan({ + manual: true, + mediaEntityId: null, + importMode: "normal", + replacementAssetIds: [], + }); + const dt = await h.db + .insertInto("download_task") + .values({ + instance_id: instanceId, + torrent_hash: hash, + name, + save_path: src.root, + content_path: src.path(name), + managed: true, + tags: ["umt"], + status: "completed", + source_release_id: release.id, + acquisition_plan_id: initialPlan, + import_plan_id: initialPlan, + }) + .returning("id") + .executeTakeFirstOrThrow(); + const local = await seedEntity(h.db, { + kind: "movie", + title: "本地作品", + year: 2022, + }); + await h.db + .updateTable("media_entity") + .set({ + metadata: JSON.stringify({ + title: { + value: "本地作品", + source: "web", + evidence: { sourceUrl: "https://example.test/detail" }, + }, + originalLanguage: { value: "zh", source: "web" }, + genres: { value: ["剧情"], source: "web" }, + }), + }) + .where("id", "=", local) + .execute(); + const sourceId = randomUUID(); + await h.setSettings("organize", { + ...DEFAULT_SETTINGS.organize, + minSizeBytes: 100_000, + scrapeAfterImport: false, + sources: [ + { + id: sourceId, + name: "Local来源", + enabled: true, + source: { root: `dl:${instanceId}:0`, path: "" }, + target: { root: `lib:${rootId}`, path: "" }, + mediaType: "movie", + category: null, + action: "copy", + trigger: "manual", + downloaderInstanceId: null, + tag: null, + intervalSec: 300, + }, + ], + }); + const has = vi.spyOn(h.ctx.capabilities, "has").mockReturnValue(true); + const invoke = vi.spyOn(h.ctx.capabilities, "invoke").mockResolvedValue({ + mediaEntityId: local, + mode: "local", + enrichmentId: randomUUID(), + reason: null, + }); + try { + const planned = await planImport(h.d, { + downloadTaskId: dt.id, + sourceConfigId: sourceId, + trigger: "auto", + actorType: "system", + }); + expect(planned.status).toBe("running"); + expect( + ( + await h.db + .selectFrom("organization_record") + .select("plan") + .where("id", "=", planned.recordId!) + .executeTakeFirstOrThrow() + ).plan, + ).toMatchObject({ metadataMode: "local" }); + expect(invoke).toHaveBeenCalledWith( + "metadata.resolve_organize", + { sourceReleaseId: release.id }, + expect.objectContaining({ source: "task" }), + ); + expect((await executeRecord(h.d, planned.recordId!)).status).toBe("succeeded"); + expect( + await h.db + .selectFrom("media_asset") + .select("id") + .where("media_entity_id", "=", local) + .where("status", "=", "active") + .execute(), + ).toHaveLength(1); + expect( + ( + await h.db + .selectFrom("path_binding") + .select("relative_path") + .where("media_entity_id", "=", local) + .executeTakeFirstOrThrow() + ).relative_path, + ).toBe("华语电影/本地作品 (2022)"); + expect(h.engine.enqueued.some((j) => j.task === "scrapeMedia")).toBe(true); + const selected = await old(local, "华语电影/本地作品 (2022)/本地作品 (2022) - 720p.mkv"); + const secondPlan = await makePlan({ + manual: true, + mediaEntityId: local, + importMode: "replace", + replacementAssetIds: [selected], + }); + await h.db + .updateTable("download_task") + .set({ import_plan_id: secondPlan, status: "completed" }) + .where("id", "=", dt.id) + .execute(); + const rerun = await planImport(h.d, { + downloadTaskId: dt.id, + sourceConfigId: sourceId, + trigger: "auto", + actorType: "system", + }); + expect(rerun.status).toBe("running"); + expect( + ( + await h.db + .selectFrom("organization_record") + .select("plan") + .where("id", "=", rerun.recordId!) + .executeTakeFirstOrThrow() + ).plan, + ).toMatchObject({ metadataMode: "local" }); + expect(invoke).toHaveBeenCalledTimes(1); + await executeRecord(h.d, rerun.recordId!); + expect(await status(selected)).toBe("retired"); + expect( + ( + await h.db + .selectFrom("acquisition_plan") + .select("status") + .where("id", "=", secondPlan) + .executeTakeFirstOrThrow() + ).status, + ).toBe("completed"); + expect( + ( + await h.db + .selectFrom("download_task") + .select("acquisition_plan_id") + .where("id", "=", dt.id) + .executeTakeFirstOrThrow() + ).acquisition_plan_id, + ).toBe(initialPlan); + } finally { + has.mockRestore(); + invoke.mockRestore(); + } + }); +}); diff --git a/apps/server/src/modules/acquisition-organize/__tests__/scrape.test.ts b/apps/server/src/modules/acquisition-organize/__tests__/scrape.test.ts index a959308a..5d60f1ea 100644 --- a/apps/server/src/modules/acquisition-organize/__tests__/scrape.test.ts +++ b/apps/server/src/modules/acquisition-organize/__tests__/scrape.test.ts @@ -3,7 +3,7 @@ import { join, relative } from "node:path"; import { createTempTree, type TempTree } from "@umt/testkit"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { executeRecord, planImport } from "../services/organize.ts"; -import { runScrapeRecord, SCRAPE_TASK, scrapeEntity } from "../services/scrape.ts"; +import { entityScrapeLocations, runScrapeRecord, SCRAPE_TASK, scrapeEntity } from "../services/scrape.ts"; import { createHarness, type Harness, pgAvailable, seedBase, seedEntity } from "./harness.ts"; // 刮削(SCR-1 / SCR-2):入库成功后自动对作品目录排 scrapeMedia → 写 NFO 与图片(临时库根);只补缺失 / 全部覆盖; @@ -299,6 +299,21 @@ describe.skipIf(!hasPg)("刮削:NFO 与图片写进媒体库目录", () => { expect(nfo).toContain("169"); }, 60_000); + it("蓝光原盘资产:原盘目录本身就是刮削目录,不取上一级", async () => { + const disc = await seedEntity(h.db, { kind: "movie", title: "原盘", year: 2020 }); + const root = await h.db.selectFrom("library_root").select("id").executeTakeFirstOrThrow(); + const asset = await h.db + .insertInto("media_asset") + .values({ media_entity_id: disc, kind: "bluray_dir" }) + .returning("id") + .executeTakeFirstOrThrow(); + await h.db + .insertInto("asset_location") + .values({ media_asset_id: asset.id, library_root_id: root.id, relative_path: "Movies/原盘 (2020)" }) + .execute(); + expect((await entityScrapeLocations(h.d, disc, "movie")).map((l) => l.path)).toEqual(["Movies/原盘 (2020)"]); + }); + it("写入失败记原因;只登记库根不写文件;没有入库文件的作品 412", async () => { const files = await tree(lib.root); const video = files.find((p) => p.includes("星际穿越") && p.endsWith(".mkv"))!; diff --git a/apps/server/src/modules/acquisition-organize/__tests__/subscription.test.ts b/apps/server/src/modules/acquisition-organize/__tests__/subscription.test.ts index 2d477e59..78277b5a 100644 --- a/apps/server/src/modules/acquisition-organize/__tests__/subscription.test.ts +++ b/apps/server/src/modules/acquisition-organize/__tests__/subscription.test.ts @@ -249,16 +249,19 @@ describe.skipIf(!hasPg)("订阅增强(真实 PG + mock qB + 临时目录)", () = .transaction() .execute((trx) => applyEpisodeTotal(trx, { entityId, season: 1, fresh: 3, now: new Date() })); const r2 = await release(entityId, "Unknown.Total.S01E02.1080p.WEB-DL.x264-GRP"); - await completeAndImport(r2.hash, await submit(await planFor(r2.id))); + const acceptedTask = await submit(await planFor(r2.id)); + expect(await subRow(subId)).toMatchObject({ status: "completed", status_reason: "download_accepted" }); + expect((await slotRow(slotId)).status).toBe("downloading"); + await completeAndImport(r2.hash, acceptedTask); expect((await slotRow(slotId)).status).toBe("satisfied"); - expect(await subRow(subId)).toMatchObject({ status: "completed", status_reason: "slot_satisfied" }); + expect(await subRow(subId)).toMatchObject({ status: "completed", status_reason: "download_accepted" }); const hist = await db() .selectFrom("subscription_history") .selectAll() .where("subscription_id", "=", subId) .execute(); expect(hist).toHaveLength(1); - expect(hist[0]).toMatchObject({ reason: "slot_satisfied", media_entity_id: entityId, mode: "once" }); + expect(hist[0]).toMatchObject({ reason: "download_accepted", media_entity_id: entityId, mode: "once" }); expect((hist[0]!.progress as { seasons: { complete: boolean }[] }).seasons[0]!.complete).toBe(true); // 历史 API + 重新订阅(改回 active) const api = await h.api("GET", `/subscription-history?subscriptionId=${subId}`); @@ -267,6 +270,32 @@ describe.skipIf(!hasPg)("订阅增强(真实 PG + mock qB + 临时目录)", () = expect(re.body.data.status).toBe("active"); }, 90_000); + it("多个目标版本按各自已接收单元完成,单版本入库不提前完成订阅", async () => { + const { entityId, subId } = await seriesSlot("双版本剧", { mode: "once" }); + await db() + .insertInto("acquisition_slot") + .values({ + media_entity_id: entityId, + subscription_id: subId, + name: "720p", + hard_conditions: JSON.stringify({ resolutions: ["720p"] }), + }) + .execute(); + await db() + .transaction() + .execute((trx) => applyEpisodeTotal(trx, { entityId, season: 1, fresh: 1, now: new Date() })); + const first = await release(entityId, "Two.Versions.S01E01.1080p.WEB-DL.x264-GRP"); + const firstTask = await submit(await planFor(first.id)); + expect((await subRow(subId)).status).toBe("active"); + await completeAndImport(first.hash, firstTask); + expect((await subRow(subId)).status).toBe("active"); + const second = await release(entityId, "Two.Versions.S01E01.720p.WEB-DL.x264-GRP"); + await submit(await planFor(second.id)); + expect(await subRow(subId)).toMatchObject({ status: "completed", status_reason: "download_accepted" }); + await qb.complete(second.hash, { writeFiles: true }); + await syncDownloads(h.d); + }, 60_000); + it("pending(TMDB 未就绪)照常下载但不完成;暂停的订阅不评估", async () => { const { entityId, slotId, subId } = await seriesSlot("待定剧", { mode: "once", status: "pending" }); await db() diff --git a/apps/server/src/modules/acquisition-organize/capabilities.ts b/apps/server/src/modules/acquisition-organize/capabilities.ts index 0125a1c9..a3a997aa 100644 --- a/apps/server/src/modules/acquisition-organize/capabilities.ts +++ b/apps/server/src/modules/acquisition-organize/capabilities.ts @@ -532,7 +532,8 @@ export function registerAoCapabilities(registry: CapabilityRegistry, deps: () => if (!dt) throw new CapabilityError("not_found", "下载任务不存在"); if (!["completed", "importing", "imported"].includes(dt.status)) throw new CapabilityError("precondition", `下载任务尚未完成(${dt.status})`); - const action: "copy" | "move" = input.action === "move" ? "move" : "copy"; + const action: "copy" | "move" | "in_place" = + input.action === "in_place" ? "in_place" : input.action === "move" ? "move" : "copy"; const request = { downloadTaskId: input.downloadTaskId, action, @@ -603,7 +604,7 @@ export function registerAoCapabilities(registry: CapabilityRegistry, deps: () => if (!art) throw new CapabilityError("not_found", "整理计划不存在"); if (art.content_hash !== input.payloadHash) throw new CapabilityError("conflict", "payloadHash 与计划不一致"); const content = art.content as { - request: { downloadTaskId: string; action: "copy" | "move"; targetLibraryRootId?: string }; + request: { downloadTaskId: string; action: "copy" | "move" | "in_place"; targetLibraryRootId?: string }; }; const req = content.request; // 已执行过(同计划)→ 返回已有记录 diff --git a/apps/server/src/modules/acquisition-organize/handlers.ts b/apps/server/src/modules/acquisition-organize/handlers.ts index 87ba68e7..625e7fd3 100644 --- a/apps/server/src/modules/acquisition-organize/handlers.ts +++ b/apps/server/src/modules/acquisition-organize/handlers.ts @@ -45,13 +45,7 @@ async function taskRow( runId, ...(subject ? { subjectType: subject.type, subjectId: subject.id } : {}), })); - if (given) - await d.ctx.db - .updateTable("task") - .set({ engine_run_id: runId }) - .where("id", "=", given) - .where("engine_run_id", "is", null) - .execute(); + if (given) await d.ctx.db.updateTable("task").set({ engine_run_id: runId }).where("id", "=", given).execute(); await markTask(d.ctx.db, taskId, "running"); return taskId; } @@ -197,7 +191,7 @@ export function registerAoTasks(registry: TaskRegistry, d: AoDeps) { retryPolicy: { maximumAttempts: 1 }, handler: async (t) => t.step.run("scrape", async () => { - const r = await runScrapeRecord(d, t.input.recordId, t.abortSignal); + const r = await runScrapeRecord(d, t.input.recordId, t.abortSignal, t.runId); return { status: r.status, written: r.written, skipped: r.skipped, failed: r.failed, reason: r.reason }; }), }), diff --git a/apps/server/src/modules/acquisition-organize/index.ts b/apps/server/src/modules/acquisition-organize/index.ts index 0b9e23bd..b061a5c3 100644 --- a/apps/server/src/modules/acquisition-organize/index.ts +++ b/apps/server/src/modules/acquisition-organize/index.ts @@ -7,6 +7,7 @@ import { registerAoCapabilities } from "./capabilities.ts"; import { registerAoTasks } from "./handlers.ts"; import { type AoOptions, type AoPorts, createPorts } from "./ports.ts"; import { registerAssetRoutes } from "./routes/assets.ts"; +import { registerClassificationRoutes } from "./routes/classification.ts"; import { registerDashboardRoutes } from "./routes/dashboard.ts"; import { registerDownloadRoutes } from "./routes/downloads.ts"; import { registerLibraryRoutes } from "./routes/library.ts"; @@ -77,6 +78,7 @@ export function createAcquisitionOrganizeModule(overrides: AoOverrides = {}): Se registerDashboardRoutes(app, d); registerAssetRoutes(app, d); registerNamingRoutes(app, d); + registerClassificationRoutes(app, d); }, registerTasks(registry, ctx) { registerAoTasks(registry, lazy(ctx)); diff --git a/apps/server/src/modules/acquisition-organize/ports.ts b/apps/server/src/modules/acquisition-organize/ports.ts index f55f19f7..444ed111 100644 --- a/apps/server/src/modules/acquisition-organize/ports.ts +++ b/apps/server/src/modules/acquisition-organize/ports.ts @@ -31,6 +31,7 @@ import { createEntityFromTmdbCandidate, enqueueNewEntityRefresh, } from "../sources-identity/service/entity-from-candidate.ts"; +import { fetchPublicImage } from "../sources-identity/service/image-proxy.ts"; import { metadataLanguages, metadataTextOptions } from "../sources-identity/service/metadata-rich.ts"; import { identifyParsed, type NameIdentifyResult } from "../sources-identity/service/resolve.ts"; import type { ServerContext } from "../types.ts"; @@ -200,6 +201,11 @@ export function createPorts( return { kind: "url", url: t.url }; }, async scrapeImage(tmdbPath) { + if (/^https?:\/\//i.test(tmdbPath)) { + const got = await fetchPublicImage(externalFetch, tmdbPath, { maxBytes: 8 * 1024 * 1024, label: "封面" }); + if (!got.ok) throw new Error(got.message); + return { content: got.bytes, mime: got.mime }; + } images ??= new ImageCache({ dir: join(dataPaths(ctx.config).dataDir, "cache", "images"), fetch: externalFetch, diff --git a/apps/server/src/modules/acquisition-organize/routes/classification.ts b/apps/server/src/modules/acquisition-organize/routes/classification.ts new file mode 100644 index 00000000..5aa4bec4 --- /dev/null +++ b/apps/server/src/modules/acquisition-organize/routes/classification.ts @@ -0,0 +1,63 @@ +import { Api, type ClassificationConfigView } from "@umt/contracts"; +import { sql } from "@umt/db"; +import { classifyTmdb, parseClassificationYaml } from "@umt/domain"; +import { ApiException } from "../../../http/errors.ts"; +import { audit, endpoint } from "../../auth-settings/endpoint.ts"; +import { type CachedSettingsReader, SETTINGS_CHANGED_EVENT } from "../../auth-settings/provided.ts"; +import type { ModuleApp } from "../../types.ts"; +import type { AoDeps } from "../services/common.ts"; + +const isCached = (s: unknown): s is CachedSettingsReader => + typeof (s as CachedSettingsReader)?.invalidate === "function"; + +export function classificationView(yaml: string): ClassificationConfigView { + try { + const config = parseClassificationYaml(yaml); + return { yaml, categories: { movie: config.movie.map((r) => r.category), tv: config.tv.map((r) => r.category) } }; + } catch (err) { + throw new ApiException("validation_failed", err instanceof Error ? err.message : String(err)); + } +} + +export function registerClassificationRoutes(app: ModuleApp, d: AoDeps) { + const settings = async () => + isCached(d.ctx.settings) ? (await d.ctx.settings.fresh()).organize : await d.ctx.settings.get("organize"); + endpoint(app, { ...Api.classification.get, tags: ["classification"] }, async () => + classificationView((await settings()).classificationYaml), + ); + endpoint(app, { ...Api.classification.put, tags: ["classification"] }, async ({ body, actor }) => { + const view = classificationView(body.yaml); + const defaults = await settings(); + await d.ctx.db.transaction().execute(async (trx) => { + await sql`select pg_advisory_xact_lock(hashtext('organize.sources'))`.execute(trx); + const row = await trx + .selectFrom("settings") + .select("value") + .where("key", "=", "organize") + .forUpdate() + .executeTakeFirst(); + const current = row?.value && typeof row.value === "object" && !Array.isArray(row.value) ? row.value : {}; + const next = { ...defaults, ...current, classificationYaml: body.yaml }; + await trx + .insertInto("settings") + .values({ key: "organize", value: JSON.stringify(next), updated_at: new Date() }) + .onConflict((oc) => oc.column("key").doUpdateSet({ value: JSON.stringify(next), updated_at: new Date() })) + .execute(); + }); + if (isCached(d.ctx.settings)) d.ctx.settings.invalidate(); + void d.ctx.events.publish(SETTINGS_CHANGED_EVENT, { keys: ["organize"] }); + await audit( + d.ctx, + actor, + "settings.update", + { type: "settings", id: "organize" }, + { patch: { organize: { classificationYaml: body.yaml } } }, + ); + return view; + }); + endpoint(app, { ...Api.classification.preview, tags: ["classification"] }, async ({ body }) => { + const yaml = body.yaml ?? (await settings()).classificationYaml; + classificationView(yaml); + return classifyTmdb(parseClassificationYaml(yaml), body.mediaType, body.tmdb); + }); +} diff --git a/apps/server/src/modules/acquisition-organize/routes/library.ts b/apps/server/src/modules/acquisition-organize/routes/library.ts index 6f4257a6..6b938747 100644 --- a/apps/server/src/modules/acquisition-organize/routes/library.ts +++ b/apps/server/src/modules/acquisition-organize/routes/library.ts @@ -302,7 +302,6 @@ export function registerLibraryRoutes(app: ModuleApp, d: AoDeps) { ]), ) .orderBy("dt.created_at", "desc") - .limit(100) .execute(); const entityRecs = await db() .selectFrom("organization_record") @@ -321,7 +320,6 @@ export function registerLibraryRoutes(app: ModuleApp, d: AoDeps) { ]) .where("media_entity_id", "=", id) .orderBy("created_at", "desc") - .limit(100) .execute(); const planIds = dts.map((t) => t.acquisition_plan_id).filter((x): x is string => !!x); const plans = planIds.length @@ -356,7 +354,6 @@ export function registerLibraryRoutes(app: ModuleApp, d: AoDeps) { ), ) .orderBy("p.updated_at", "desc") - .limit(50) .execute() : []; const rootPath = new Map(roots.map((r) => [r.id, r.local_path])); diff --git a/apps/server/src/modules/acquisition-organize/routes/organization.ts b/apps/server/src/modules/acquisition-organize/routes/organization.ts index 6a471ee7..5f8daeb5 100644 --- a/apps/server/src/modules/acquisition-organize/routes/organization.ts +++ b/apps/server/src/modules/acquisition-organize/routes/organization.ts @@ -266,11 +266,13 @@ function recordFilters=", new Date(query.since)); - if (query.organizeOnly) q = q.where("action", "in", ["copy", "move", "hardlink"]); + if (query.organizeOnly) q = q.where("action", "in", ["copy", "move", "in_place", "hardlink"]); if (query.kind === "none") q = q.where("media_entity_id", "is", null); else if (query.kind) q = q.where("media_entity_id", "in", (eb: ExpressionBuilder) => diff --git a/apps/server/src/modules/acquisition-organize/services/classification.ts b/apps/server/src/modules/acquisition-organize/services/classification.ts new file mode 100644 index 00000000..1955fead --- /dev/null +++ b/apps/server/src/modules/acquisition-organize/services/classification.ts @@ -0,0 +1,105 @@ +import type { ClassificationPreviewResult } from "@umt/contracts"; +import type { MediaEntityRow } from "@umt/db"; +import { classifyTmdb, DEFAULT_CLASSIFICATION_YAML, parseClassificationYaml } from "@umt/domain"; +import { SourcesRuntime } from "../../sources-identity/runtime.ts"; +import type { AoDeps } from "./common.ts"; + +/** Use accepted TMDB identity and the existing shared raw-detail cache. */ +export async function classifyEntity(d: AoDeps, entity: MediaEntityRow): Promise { + const none: ClassificationPreviewResult = { + category: null, + matchedFields: [], + fallback: false, + }; + const config = parseClassificationYaml( + (await d.ctx.settings.get("organize")).classificationYaml ?? DEFAULT_CLASSIFICATION_YAML, + ); + const mediaType = entity.kind === "movie" ? "movie" : "tv"; + if (!config[mediaType].length) return none; + const identity = await d.ctx.db + .selectFrom("external_identity") + .select("external_id") + .where("media_entity_id", "=", entity.id) + .where("provider", "=", "tmdb") + .where("entity_type", "=", mediaType) + .where("content_unit_id", "is", null) + .where("status", "=", "accepted") + .orderBy("id") + .executeTakeFirst(); + if (!identity) { + const metadata = entity.metadata as Record; + const value = (field: string) => { + const entry = metadata[field]; + return entry && typeof entry === "object" && "value" in entry ? (entry as { value: unknown }).value : undefined; + }; + if (!Object.values(metadata).some((v) => v && typeof v === "object" && "value" in v)) return none; + const genres = value("genres"); + const country = value("originCountry"); + const year = value("year") ?? entity.year; + const genreIds: Record = { + 动画: 16, + 动漫: 16, + animation: 16, + 儿童: 10762, + kids: 10762, + 纪录: 99, + 纪录片: 99, + documentary: 99, + 综艺: 10764, + reality: 10764, + talk: 10767, + 音乐: 10402, + music: 10402, + 动作: 28, + action: 28, + 冒险: 12, + adventure: 12, + 喜剧: 35, + comedy: 35, + 犯罪: 80, + crime: 80, + 剧情: 18, + drama: 18, + 家庭: 10751, + family: 10751, + 奇幻: 14, + fantasy: 14, + 历史: 36, + history: 36, + 恐怖: 27, + horror: 27, + 悬疑: 9648, + mystery: 9648, + 爱情: 10749, + romance: 10749, + 科幻: 878, + "science fiction": 878, + 惊悚: 53, + thriller: 53, + 战争: 10752, + war: 10752, + 西部: 37, + western: 37, + }; + const raw = { + original_language: value("originalLanguage"), + origin_country: country, + production_countries: country, + genre_ids: (Array.isArray(genres) ? genres : genres ? [genres] : []).map( + (g) => genreIds[String(g).toLowerCase()] ?? g, + ), + ...(year ? { release_date: String(year), first_air_date: String(year) } : {}), + }; + return classifyTmdb(config, mediaType, raw); + } + if (!/^\d+$/.test(identity.external_id)) return none; + try { + const tmdb = await new SourcesRuntime(d.ctx).tmdb(); + if (!tmdb) return none; + const raw = await tmdb.getJson>(`/${mediaType}/${identity.external_id}`); + return classifyTmdb(config, mediaType, raw); + } catch (err) { + d.ctx.logger.warn({ mediaEntityId: entity.id, err }, "TMDB 分类数据读取失败"); + return none; + } +} diff --git a/apps/server/src/modules/acquisition-organize/services/common.ts b/apps/server/src/modules/acquisition-organize/services/common.ts index a676f45a..756facd7 100644 --- a/apps/server/src/modules/acquisition-organize/services/common.ts +++ b/apps/server/src/modules/acquisition-organize/services/common.ts @@ -3,7 +3,7 @@ import { type DB, type DownloaderInstanceRow, type Kysely, type LibraryRootRow, import { type PathMapping, toDownloaderPath, toHostPath } from "@umt/domain"; import type { ServerContext } from "../../types.ts"; import type { AoPorts } from "../ports.ts"; -import { hashJson, jsonb } from "../util.ts"; +import { errMessage, hashJson, jsonb } from "../util.ts"; // 本模块各服务共用:决策(待处理)、候选评估记录、业务任务行、实例拓扑(路径映射/缓存卷)。 @@ -132,7 +132,7 @@ export async function markTask( * 巡检补投等需要"同一业务行、按时间槽重投"的场景传 engineKey。 */ export async function enqueueTracked( - d: AoDeps, + d: Pick, t: Omit, opts: { engineKey?: string } = {}, ): Promise<{ taskId: string; runId: string }> { @@ -145,7 +145,12 @@ export async function enqueueTracked( traceId: taskId, }, ); - await d.ctx.db.updateTable("task").set({ engine_run_id: runId }).where("id", "=", taskId).execute(); + try { + await d.ctx.db.updateTable("task").set({ engine_run_id: runId }).where("id", "=", taskId).execute(); + } catch (err) { + // run 已被接受;处理器和下一轮巡检会按 taskId 补回关联。 + d.ctx.logger.warn({ taskId, runId, err: { message: errMessage(err) } }, "任务已入队,业务关联暂未写回"); + } return { taskId, runId }; } @@ -163,13 +168,34 @@ export async function enqueueSweep( ): Promise<{ taskId: string; runId: string } | null> { const prev = await d.ctx.db .selectFrom("task") - .select(["engine_run_id"]) + .select(["id", "engine_run_id"]) .where("kind", "=", t.kind) .where("idempotency_key", "=", t.idempotencyKey) .executeTakeFirst(); - if (prev?.engine_run_id) { - const run = await d.ctx.engine.get(prev.engine_run_id).catch(() => null); - if (run && ACTIVE_RUN.has(run.status)) return null; + if (prev) { + try { + // 关联写回失败时,用入队已有的 traceId 找回 run,不随时间槽生成另一个任务。 + const runId = + ( + await d.ctx.db + .selectFrom("tasks_run_meta") + .select("run_id") + .where("task_name", "=", t.kind) + .where("trace_id", "=", prev.id) + .orderBy("created_at", "desc") + .executeTakeFirst() + )?.run_id ?? prev.engine_run_id; + if (runId) { + const run = await d.ctx.engine.get(runId); + if (prev.engine_run_id !== runId && run) { + await d.ctx.db.updateTable("task").set({ engine_run_id: runId }).where("id", "=", prev.id).execute(); + } + if (run && ACTIVE_RUN.has(run.status)) return null; + } + } catch (err) { + d.ctx.logger.warn({ taskId: prev.id, err: { message: errMessage(err) } }, "任务状态查询失败,跳过本轮补投"); + return null; + } } return enqueueTracked(d, t, { engineKey: `${t.kind}:${t.idempotencyKey}:${slot}` }); } diff --git a/apps/server/src/modules/acquisition-organize/services/coverage.ts b/apps/server/src/modules/acquisition-organize/services/coverage.ts index 2deb3b8d..8d1bbbd7 100644 --- a/apps/server/src/modules/acquisition-organize/services/coverage.ts +++ b/apps/server/src/modules/acquisition-organize/services/coverage.ts @@ -250,6 +250,7 @@ export async function buildSubscriptionModels( : r.dt_status === "completed" || r.dt_status === "importing" ? "verifying" : "downloading"; + if (!r.slot_id) continue; const list = inflightBySlot.get(r.slot_id) ?? []; const prev = list.find((x) => x.id === r.id); if (!prev) list.push({ id: r.id, stage, unitKeys: r.unit_keys }); diff --git a/apps/server/src/modules/acquisition-organize/services/download-board.ts b/apps/server/src/modules/acquisition-organize/services/download-board.ts index 9be2bde7..c18bbbdc 100644 --- a/apps/server/src/modules/acquisition-organize/services/download-board.ts +++ b/apps/server/src/modules/acquisition-organize/services/download-board.ts @@ -167,14 +167,14 @@ export function planWaitOf(status: string, deferred: string | null): PlanWait { } /** 等待中的计划 → 面板行 */ -async function planItems(db: Exec, rows: { id: string; slot_id: string; source_release_id: string }[]) { +async function planItems(db: Exec, rows: { id: string; slot_id: string | null; source_release_id: string }[]) { if (!rows.length) return new Map>(); const ids = rows.map((r) => r.id); const [extra, approvals, tasks] = await Promise.all([ db .selectFrom("acquisition_plan as p") - .innerJoin("acquisition_slot as s", "s.id", "p.slot_id") + .leftJoin("acquisition_slot as s", "s.id", "p.slot_id") .leftJoin("media_entity as e", "e.id", "s.media_entity_id") .leftJoin("source_release as r", "r.id", "p.source_release_id") .select([ diff --git a/apps/server/src/modules/acquisition-organize/services/downloads.ts b/apps/server/src/modules/acquisition-organize/services/downloads.ts index 803ade94..ff620999 100644 --- a/apps/server/src/modules/acquisition-organize/services/downloads.ts +++ b/apps/server/src/modules/acquisition-organize/services/downloads.ts @@ -39,6 +39,7 @@ import { downloaderPathOf, type Exec, enqueueSweep, + enqueueTracked, hostPathOf, type InstanceTopology, loadInstances, @@ -46,8 +47,10 @@ import { timeSlot, upsertDecision, } from "./common.ts"; +import { matchesQbSource, qbSourceTags } from "./organize-config.ts"; import { hnrSatisfied, retentionElapsed } from "./retention.ts"; import { + afterDownloadAccepted, afterSlotVerified, type SlotCompletion, slotComplete, @@ -107,10 +110,11 @@ async function verifiedContext( export async function applySlotEvent( d: AoDeps, trx: Exec, - slotId: string, + slotId: string | null, event: SlotEvent, extra: { currentAssetId?: string | null; reason?: string; planId?: string | null } = {}, ): Promise { + if (!slotId) return false; const slot = await trx .selectFrom("acquisition_slot") .selectAll() @@ -217,10 +221,21 @@ export async function submitDownload(d: AoDeps, planId: string, opts: { runId?: const db = d.ctx.db; const plan = await db.selectFrom("acquisition_plan").selectAll().where("id", "=", planId).executeTakeFirst(); if (!plan) return { status: "skipped", reason: "plan_not_found" }; - if (plan.status !== "reserved") return { status: "skipped", reason: `plan_${plan.status}` }; + if (plan.status !== "reserved") { + if (["submitted", "completed"].includes(plan.status) && plan.status_reason !== "accepted_unconfirmed") + await afterDownloadAccepted(d, plan.id); + return { status: "skipped", reason: `plan_${plan.status}` }; + } const blocked = await automationBlock(d); if (blocked) return { status: "deferred", reason: blocked }; - const ev = (plan.evaluation ?? {}) as { instanceId?: string; volumeKey?: string; savePath?: string }; + const ev = (plan.evaluation ?? {}) as { + instanceId?: string; + volumeKey?: string; + savePath?: string; + mediaEntityId?: string | null; + manual?: boolean; + importMode?: string; + }; const topo = (await loadInstances(db)).find((t) => t.inst.id === ev.instanceId); if (!topo?.savePath) return failPlan(d, plan.id, plan.slot_id, "downloader_unavailable"); @@ -313,16 +328,17 @@ export async function submitDownload(d: AoDeps, planId: string, opts: { runId?: } const r = res as Extract; const ins = await sql<{ id: string; inserted: boolean }>` - insert into download_task (instance_id, torrent_hash, infohash_v1, infohash_v2, acquisition_plan_id, source_release_id, + insert into download_task (instance_id, torrent_hash, infohash_v1, infohash_v2, acquisition_plan_id, import_plan_id, source_release_id, media_entity_id, name, save_path, content_path, category, tags, managed, origin, status, size_bytes, next_check_at) values (${topo.inst.id}, ${r.hash}, ${r.hash.length === 40 ? r.hash : null}, ${r.hash.length === 64 ? r.hash : null}, - ${plan.id}, ${rel.id}, (select media_entity_id from acquisition_slot where id = ${plan.slot_id}), + ${plan.id}, ${ev.manual ? plan.id : null}, ${rel.id}, coalesce((select media_entity_id from acquisition_slot where id = ${plan.slot_id}), ${ev.mediaEntityId ?? rel.media_entity_id}::uuid), ${r.name}, ${r.savePath}, ${r.contentPath}, ${topo.category}, ${r.tags}, true, ${r.kind === "already_exists" ? "adopted" : "submitted"}, ${r.kind === "accepted_unconfirmed" ? "submitting" : "queued"}, ${rel.size_bytes}, now()) on conflict (instance_id, torrent_hash) do update set acquisition_plan_id = coalesce(download_task.acquisition_plan_id, excluded.acquisition_plan_id), + import_plan_id = coalesce(excluded.import_plan_id, download_task.import_plan_id), source_release_id = coalesce(download_task.source_release_id, excluded.source_release_id), media_entity_id = coalesce(download_task.media_entity_id, excluded.media_entity_id), managed = true, tags = excluded.tags, updated_at = now() @@ -357,6 +373,7 @@ export async function submitDownload(d: AoDeps, planId: string, opts: { runId?: d.ctx.logger.warn({ planId: plan.id, err: { message: errMessage(err) } }, "季包选文件失败,按整包下载"), ); } + if (result.kind !== "accepted_unconfirmed") await afterDownloadAccepted(d, plan.id); if (!dtId) { // 账本已有结果(重放):按 hash 找回任务行 dtId = ( @@ -368,6 +385,20 @@ export async function submitDownload(d: AoDeps, planId: string, opts: { runId?: .executeTakeFirst() )?.id; } + if (result.kind === "already_exists" && dtId && ev.manual && ev.importMode === "replace") { + const existing = await db.selectFrom("download_task").select("status").where("id", "=", dtId).executeTakeFirst(); + if (existing?.status === "imported") { + await db.updateTable("download_task").set({ status: "completed" }).where("id", "=", dtId).execute(); + await enqueueTracked(d, { + kind: "verifyAndImport", + idempotencyKey: `manual-import:${plan.id}`, + input: { downloadTaskId: dtId }, + subjectType: "download_task", + subjectId: dtId, + triggeredBy: "user", + }); + } + } return { status: "submitted", reason: result.kind, ...(dtId ? { downloadTaskId: dtId } : {}) }; } @@ -385,6 +416,7 @@ async function selectPackFiles( plan: AcquisitionPlanRow, rel: SourceReleaseRow, ): Promise { + if (!plan.slot_id) return; const parsed = parsedOf(rel); const isPack = !parsed.episodes && !parsed.airDate; const multi = !!parsed.episodes && parsed.episodes.end > parsed.episodes.start; @@ -465,7 +497,7 @@ async function selectPackFiles( ); } -async function failPlanTx(d: AoDeps, trx: Exec, planId: string, slotId: string, reason: string) { +async function failPlanTx(d: AoDeps, trx: Exec, planId: string, slotId: string | null, reason: string) { await trx .updateTable("acquisition_plan") .set({ status: "failed", status_reason: reason.slice(0, 500) }) @@ -502,7 +534,7 @@ async function releaseRemovedPlan(d: AoDeps, trx: Exec, planId: string) { ); } -async function failPlan(d: AoDeps, planId: string, slotId: string, reason: string): Promise { +async function failPlan(d: AoDeps, planId: string, slotId: string | null, reason: string): Promise { await d.ctx.db.transaction().execute((trx) => failPlanTx(d, trx, planId, slotId, reason)); return { status: "failed", reason }; } @@ -593,9 +625,14 @@ async function syncInstance(d: AoDeps, topo: InstanceTopology, stats: SyncStats) const db = d.ctx.db; const now = d.ports.now(); const client = await d.ports.downloader(topo.inst); + const sourceTags = await qbSourceTags(d, topo); let list: QbTorrent[]; try { - list = await client.list({ tag: topo.inst.managed_tag }); + const tags = [...new Set([topo.inst.managed_tag, ...sourceTags.map((s) => s.config.tag!)])]; + const batches = await Promise.all(tags.map((tag) => client.list({ tag }))); + list = [...new Map(batches.flat().map((t) => [t.hash, t])).values()].filter( + (t) => t.tags.includes(topo.inst.managed_tag) || matchesQbSource(t, topo, sourceTags), + ); } catch (err) { // 下载器失联:不改任何任务状态(失败 ≠ 不存在) stats.unreachable.push(topo.inst.id); @@ -722,6 +759,7 @@ async function observeLocked( .set({ status: merged.status, pause_reason: merged.status === "paused" ? merged.pauseReason : null, + managed: true, raw_state: t.rawState, progress: Math.min(1, Math.max(0, t.progress)), size_bytes: sizeOf(t), diff --git a/apps/server/src/modules/acquisition-organize/services/manual-acquisition.ts b/apps/server/src/modules/acquisition-organize/services/manual-acquisition.ts new file mode 100644 index 00000000..5ed7de12 --- /dev/null +++ b/apps/server/src/modules/acquisition-organize/services/manual-acquisition.ts @@ -0,0 +1,149 @@ +import type { ManualAcquisitionRequest, ManualAcquisitionResult } from "@umt/contracts"; +import { sql } from "@umt/db"; +import { ApiException } from "../../../http/errors.ts"; +import type { ServerContext } from "../../types.ts"; +import { hashJson, jsonb, toApi } from "../util.ts"; +import { subscriptionSavePath } from "./acquisition.ts"; +import { enqueueTracked, loadInstances } from "./common.ts"; + +/** Manual choice is a one-off plan, independent of subscription filters and slots. */ +export async function planManualAcquisition( + ctx: ServerContext, + body: ManualAcquisitionRequest, +): Promise { + const db = ctx.db; + const rel = await db + .selectFrom("source_release") + .selectAll() + .where("id", "=", body.sourceReleaseId) + .executeTakeFirst(); + if (!rel) throw new ApiException("not_found", "资源不存在"); + const entityId = body.entityId ?? rel.media_entity_id; + if (!entityId && !body.allowUnrecognized) + return { + submitted: false, + requiresConfirmation: true, + plan: null, + slotId: null, + taskId: null, + rejected: [ + { slotId: null, code: "unrecognized", message: "资源尚未识别作品,确认后可以先下载,整理时补全元数据" }, + ], + }; + if (entityId && !(await db.selectFrom("media_entity").select("id").where("id", "=", entityId).executeTakeFirst())) + throw new ApiException("not_found", "作品不存在"); + const instances = await loadInstances(db, body.downloaderInstanceId ? { ids: [body.downloaderInstanceId] } : {}); + const topo = instances.find((i) => i.savePath); + if (!topo?.savePath) throw new ApiException("precondition_failed", "没有可用的下载器或保存目录"); + const replacementIds = body.importMode === "replace" ? [...new Set(body.replacementAssetIds)] : []; + if (body.importMode === "replace" && (!entityId || replacementIds.length === 0)) + throw new ApiException("bad_request", "洗版入库需要明确作品及要替换的已有版本"); + if (replacementIds.length) { + const assets = await db + .selectFrom("media_asset") + .select(["id", "media_entity_id"]) + .where("id", "in", replacementIds) + .execute(); + if (assets.length !== replacementIds.length || assets.some((a) => a.media_entity_id !== entityId)) + throw new ApiException("bad_request", "要替换的版本必须属于本次下载的作品"); + } + const knownHash = rel.infohash_v1 ?? rel.infohash_v2; + if (knownHash && body.importMode !== "replace") { + const existing = await db + .selectFrom("download_task") + .select(["id", "acquisition_plan_id", "import_plan_id"]) + .where("instance_id", "=", topo.inst.id) + .where("torrent_hash", "=", knownHash) + .where("status", "=", "imported") + .where("removed_at", "is", null) + .executeTakeFirst(); + if (existing) { + const id = existing.import_plan_id ?? existing.acquisition_plan_id; + const p = id + ? await db.selectFrom("acquisition_plan").selectAll().where("id", "=", id).executeTakeFirst() + : undefined; + return { + submitted: true, + requiresConfirmation: false, + plan: p ? (toApi(p) as ManualAcquisitionResult["plan"]) : null, + slotId: p?.slot_id ?? null, + taskId: p?.task_id ?? null, + rejected: [], + }; + } + } + const savePath = body.savePath ? subscriptionSavePath(topo, body.savePath) : topo.savePath; + if (!savePath) throw new ApiException("bad_request", "保存目录不在该下载器的路径映射范围内"); + const evaluation = { + manual: true, + mediaEntityId: entityId, + instanceId: topo.inst.id, + volumeKey: topo.volumeKey, + savePath, + importMode: body.importMode, + replacementAssetIds: replacementIds, + reasons: ["manual_selection"], + note: body.note ?? null, + }; + // Pending requests converge. Completed/failed plans do not prevent another explicit choice. + const key = `manual:${hashJson({ release: rel.id, ...evaluation })}`; + const plan = await db.transaction().execute(async (trx) => { + await sql`select pg_advisory_xact_lock(hashtextextended(${key}, 0))`.execute(trx); + const existing = await trx + .selectFrom("acquisition_plan") + .selectAll() + .where("source_release_id", "=", rel.id) + .where("slot_id", "is", null) + .where("status", "in", ["reserved", "submitted"]) + .where("payload_hash", "=", hashJson(evaluation)) + .where(sql`evaluation->>'instanceId' = ${topo.inst.id}`) + .orderBy("created_at", "desc") + .executeTakeFirst(); + if (existing) return existing; + return trx + .insertInto("acquisition_plan") + .values({ + slot_id: null, + source_release_id: rel.id, + task_id: null, + purpose: body.importMode === "replace" ? "upgrade" : "fill", + status: "reserved", + priority: "immediate", + evaluation: jsonb(evaluation), + reserved_bytes: rel.size_bytes ?? 0, + budget_reservation: jsonb({}), + payload_hash: hashJson(evaluation), + idempotency_key: `${key}:${crypto.randomUUID()}`, + expires_at: null, + status_reason: "manual_selection", + file_selection: null, + }) + .returningAll() + .executeTakeFirstOrThrow(); + }); + const { taskId } = await enqueueTracked( + { ctx }, + { + kind: "submitDownload", + idempotencyKey: `plan:${plan.id}`, + input: { acquisitionPlanId: plan.id }, + subjectType: "acquisition_plan", + subjectId: plan.id, + triggeredBy: "user", + }, + ); + const updated = await db + .updateTable("acquisition_plan") + .set({ task_id: taskId }) + .where("id", "=", plan.id) + .returningAll() + .executeTakeFirstOrThrow(); + return { + submitted: true, + requiresConfirmation: false, + plan: toApi(updated) as ManualAcquisitionResult["plan"], + slotId: null, + taskId, + rejected: [], + }; +} diff --git a/apps/server/src/modules/acquisition-organize/services/organize-config.ts b/apps/server/src/modules/acquisition-organize/services/organize-config.ts new file mode 100644 index 00000000..a15b270d --- /dev/null +++ b/apps/server/src/modules/acquisition-organize/services/organize-config.ts @@ -0,0 +1,74 @@ +import { join } from "node:path"; +import type { OrganizeSourceConfig } from "@umt/contracts"; +import { loadRootDefs, within } from "../../file-manager/common.ts"; +import type { AoDeps, InstanceTopology } from "./common.ts"; +import { hostPathOf } from "./common.ts"; + +export async function configuredSources(d: AoDeps) { + const settings = await d.ctx.settings.get("organize"); + if (!settings.sources?.length) return []; + const roots = await loadRootDefs(d.ctx.db); + return (settings.sources ?? []).flatMap((config) => { + const root = roots.find((r) => r.id === config.source.root); + return root ? [{ config, sourcePath: join(root.hostPath, config.source.path) }] : []; + }); +} + +export async function resolveOrganizeSource( + d: AoDeps, + input: { + id?: string | undefined; + paths: string[]; + mediaType: "movie" | "tv"; + instanceId?: string | undefined; + tags?: string[] | undefined; + }, +): Promise { + const sources = await configuredSources(d); + const compatible = sources.filter( + ({ config, sourcePath }) => + (config.mediaType === "all" || config.mediaType === input.mediaType) && + input.paths.length > 0 && + input.paths.every((p) => within(p, sourcePath)), + ); + if (input.id) { + const selected = compatible.find(({ config }) => config.id === input.id); + if (!selected) throw new Error("整理来源不存在,或文件/媒体类型不属于该来源"); + return selected.config; + } + // qB automation selects only its configured tag; manual rows require explicit selection. + const selected = sources + .filter( + ({ config, sourcePath }) => + config.enabled && + config.trigger === "qb_tag" && + config.downloaderInstanceId === input.instanceId && + !!config.tag && + input.tags?.includes(config.tag) && + input.paths.length > 0 && + input.paths.every((p) => within(p, sourcePath)), + ) + .sort((a, b) => b.sourcePath.length - a.sourcePath.length)[0]?.config; + if (selected && selected.mediaType !== "all" && selected.mediaType !== input.mediaType) + throw new Error("媒体类型与 qB 标签来源配置不匹配"); + return selected ?? null; +} + +export async function qbSourceTags(d: AoDeps, topology: InstanceTopology) { + return (await configuredSources(d)).filter( + ({ config }) => + config.enabled && config.trigger === "qb_tag" && config.downloaderInstanceId === topology.inst.id && !!config.tag, + ); +} + +export function matchesQbSource( + t: { tags: string[]; contentPath: string | null; savePath: string; name: string }, + topology: InstanceTopology, + sources: Awaited>, +) { + const path = hostPathOf(topology, t.contentPath ?? join(t.savePath, t.name)); + return ( + !!path && + sources.some(({ config, sourcePath }) => !!config.tag && t.tags.includes(config.tag) && within(path, sourcePath)) + ); +} diff --git a/apps/server/src/modules/acquisition-organize/services/organize-guard.ts b/apps/server/src/modules/acquisition-organize/services/organize-guard.ts index 7402621d..ad3d0f3b 100644 --- a/apps/server/src/modules/acquisition-organize/services/organize-guard.ts +++ b/apps/server/src/modules/acquisition-organize/services/organize-guard.ts @@ -135,7 +135,7 @@ export function requestKey(body: { sourcePaths?: string[] | undefined; mediaEntityId?: string | undefined; contentUnitId?: string | undefined; - action: "copy" | "move"; + action: "copy" | "move" | "in_place"; targetLibraryRootId?: string | undefined; }): string { return canonicalJson({ diff --git a/apps/server/src/modules/acquisition-organize/services/organize-history.ts b/apps/server/src/modules/acquisition-organize/services/organize-history.ts new file mode 100644 index 00000000..a1bc6530 --- /dev/null +++ b/apps/server/src/modules/acquisition-organize/services/organize-history.ts @@ -0,0 +1,20 @@ +import type { AoDeps } from "./common.ts"; + +/** Count complete successful histories for each local main-video source. */ +export async function successfulOrganizeCounts(db: AoDeps["ctx"]["db"], paths: string[]) { + const counts = new Map(); + if (!paths.length) return counts; + const rows = await db + .selectFrom("organization_file_result as f") + .innerJoin("organization_record as r", "r.id", "f.organization_record_id") + .select(["f.source_path", "r.id"]) + .distinct() + .where("r.status", "=", "succeeded") + .where("r.source_path_view", "=", "local") + .where("f.role", "=", "main") + .where("f.status", "=", "succeeded") + .where("f.source_path", "in", paths) + .execute(); + for (const row of rows) counts.set(row.source_path, (counts.get(row.source_path) ?? 0) + 1); + return counts; +} diff --git a/apps/server/src/modules/acquisition-organize/services/organize.ts b/apps/server/src/modules/acquisition-organize/services/organize.ts index e0e88d56..e4bef97d 100644 --- a/apps/server/src/modules/acquisition-organize/services/organize.ts +++ b/apps/server/src/modules/acquisition-organize/services/organize.ts @@ -32,6 +32,8 @@ import { parseReleasePath, planOrganize, type QualityFacts, + replacementCovers, + replacementScope, resolveTargetDirectory, type SourceFile, sidecarStem, @@ -40,12 +42,22 @@ import { organizationStatusFromFiles, organizePlanToFileResults, toDomainPathBin import { withAttemptToken } from "@umt/tasks"; import { errMessage, hashJson, jsonb, toApi } from "../util.ts"; import { publishedAcquisitionPolicies, unitKeysOf } from "./acquisition.ts"; +import { classifyEntity } from "./classification.ts"; import { type AoDeps, type Exec, hostPathOf, loadInstances, upsertDecision } from "./common.ts"; import { applySlotEvent } from "./downloads.ts"; import { deliveryForAsset } from "./library-versions.ts"; import { effectiveNaming, ensureNamingVersion, loadNamingEntity, namingByVersion } from "./naming.ts"; +import { resolveOrganizeSource } from "./organize-config.ts"; import { PathGuardError } from "./organize-guard.ts"; +import { successfulOrganizeCounts } from "./organize-history.ts"; import { type FrozenTransfer, finalizeTransfer, planTransfer, type TransferSpec } from "./organize-transfer.ts"; +import { + cleanupReplacement, + type FrozenReplacement, + freezeReplacement, + settleSamePathReplacement, + stagedReplacementBackup, +} from "./replacement.ts"; import { hnrSatisfied } from "./retention.ts"; import { scrapeAfterImport } from "./scrape.ts"; import { slotComplete as sharedSlotComplete, subscriptionLibraryRoot } from "./subscriptions.ts"; @@ -64,12 +76,18 @@ import { // 资产/位置/路径绑定同事务登记 → Slot 推进 → Move 后移除 qB 任务记录(不删数据)。 export interface ImportRequest { + skipSuccessfulRecords?: boolean | undefined; + scrapeAfterImport?: boolean | undefined; + importMode?: "normal" | "replace"; + replacementAssetIds?: string[]; + sourceConfigId?: string | undefined; + idempotencyKey?: string | undefined; downloadTaskId?: string | undefined; /** 手动:执行进程可访问的源路径(文件或目录) */ sourcePaths?: string[] | undefined; mediaEntityId?: string | undefined; contentUnitId?: string | undefined; - action?: "copy" | "move" | undefined; + action?: "copy" | "move" | "in_place" | undefined; targetLibraryRootId?: string | undefined; trigger: OrganizationTrigger; actorType: ActorType; @@ -122,6 +140,19 @@ export interface FileOverrideInput { /** organization_record.plan 的冻结内容 */ export interface FrozenPlan { + scrapeAfterImport?: boolean; + skippedSuccessful?: OrganizePlan["files"]; + replacement?: FrozenReplacement; + importPlanId?: string; + metadataMode?: "local"; + sourceFacts?: Record; + sourceConfigId?: string; + idempotencyKey?: string; + classification?: { + category: string | null; + matchedFields: string[]; + fallback: boolean; + }; version: 1; organize: OrganizePlan; entityId: string; @@ -182,7 +213,11 @@ function segmentsWithin(path: string, root: string | null) { const file = within.pop()!; const parent = within.pop(); const grandParent = within.pop(); - return { file, ...(parent !== undefined ? { parent } : {}), ...(grandParent !== undefined ? { grandParent } : {}) }; + return { + file, + ...(parent !== undefined ? { parent } : {}), + ...(grandParent !== undefined ? { grandParent } : {}), + }; } /** 递归列出文件(跳过隐藏项),最多 limit 个;调用方传 上限+1 判断是否截断 */ @@ -193,7 +228,11 @@ async function walk( guard?: ((p: string) => Promise) | undefined, ): Promise { const st = await statSafe(p); - if (st.kind !== "ok") throw new FsError("ENOENT", `源路径不可访问:${p} (${st.kind})`, { path: p }); + if (st.kind !== "ok") { + throw new FsError("ENOENT", `源路径不可访问:${p} (${st.kind})`, { + path: p, + }); + } if (st.facts.isFile) { // 目录里的符号链接可能指向根外:逐个文件复核 realpath if (guard) await guard(p); @@ -283,6 +322,21 @@ async function chooseRoot( export async function planImport(d: AoDeps, req: ImportRequest): Promise { if (req.transfer) return planTransfer(d, req, req.transfer); const db = d.ctx.db; + if (req.idempotencyKey && !req.dryRun) { + const existing = await db + .selectFrom("organization_record") + .select(["id", "status", "reason"]) + .where(sql`plan->>'idempotencyKey'`, "=", req.idempotencyKey) + .where("status", "in", ["queued", "running", "succeeded", "partial"]) + .executeTakeFirst(); + if (existing) { + return { + recordId: existing.id, + status: existing.status, + reason: existing.reason, + }; + } + } let dt: DownloadTaskRow | undefined; if (req.downloadTaskId) { dt = await db.selectFrom("download_task").selectAll().where("id", "=", req.downloadTaskId).executeTakeFirst(); @@ -294,7 +348,9 @@ export async function planImport(d: AoDeps, req: ImportRequest): Promise => dry - ? { recordId: null, status: "failed", reason, ...(truncated ? { truncated } : {}) } + ? { + recordId: null, + status: "failed", + reason, + ...(truncated ? { truncated } : {}), + } : writeFailedRecord(d, { ...base, action: req.action ?? "copy", @@ -325,6 +386,7 @@ export async function planImport(d: AoDeps, req: ImportRequest): Promise = {}; let sourceRoot: string | null = null; const topo = dt ? (await loadInstances(db, { enabledOnly: false, ids: [dt.instance_id] }))[0] : undefined; let qbFiles: QbFileLike[] = []; @@ -332,7 +394,12 @@ export async function planImport(d: AoDeps, req: ImportRequest): Promise ({ @@ -351,9 +418,17 @@ export async function planImport(d: AoDeps, req: ImportRequest): Promise).some( + (field) => + field && + typeof field === "object" && + "value" in field && + ["web", "agent"].includes(String((field as { source?: unknown }).source)), + ) + ) { + const tmdb = await db + .selectFrom("external_identity") + .select("id") + .where("media_entity_id", "=", entity.id) + .where("provider", "=", "tmdb") + .where("status", "=", "accepted") + .executeTakeFirst(); + if (!tmdb) metadataMode = "local"; + } const mediaType: "movie" | "tv" = entity.kind === "movie" ? "movie" : "tv"; + const inPlace = req.action === "in_place"; + const importMode = req.importMode ?? evaluation?.importMode ?? "normal"; + if (inPlace && importMode === "replace") return fail("in_place_cannot_replace", {}, entity.id); + let replacement: FrozenReplacement | undefined; + if (importMode === "replace") { + try { + replacement = await freezeReplacement( + d, + entity.id, + mediaType, + req.replacementAssetIds ?? evaluation?.replacementAssetIds ?? [], + ); + } catch (error) { + return fail("replacement_invalid", { message: errMessage(error) }, entity.id); + } + } + // ── 操作方式 ── const orgSettings = await d.ctx.settings.get("organize"); + let sourceConfig: Awaited> = null; + try { + sourceConfig = inPlace + ? null + : await resolveOrganizeSource(d, { + id: req.sourceConfigId, + paths: files.map((f) => f.path), + mediaType, + instanceId: dt?.instance_id, + tags: dt?.tags ?? undefined, + }); + } catch (error) { + return fail("source_config_mismatch", { message: errMessage(error) }, entity.id); + } + const classification = sourceConfig?.category + ? { category: sourceConfig.category, matchedFields: [], fallback: false } + : sourceConfig + ? await classifyEntity(d, entity) + : undefined; + // Existing bindings remain usable even if metadata cannot currently be fetched. + const hasBinding = await db + .selectFrom("path_binding") + .select("id") + .where("media_entity_id", "=", entity.id) + .where("status", "=", "active") + .executeTakeFirst(); + if (sourceConfig && !classification?.category && !hasBinding && !req.targetDir) { + await decide_({ + kind: "other", + title: `暂时无法分类:${entity.title}`, + summary: "检查 TMDB 连接或选择固定分类;目录扫描会在后续轮次重试,也可手动整理存量", + subjectType: "organize_source", + subjectId: sourceConfig.id, + dedupeKey: `source_classification:${sourceConfig.id}:${entity.id}`, + evidence: { + sourceConfigId: sourceConfig.id, + mediaEntityId: entity.id, + sourcePaths: req.sourcePaths ?? [], + }, + }); + return { + recordId: null, + status: "failed", + reason: "classification_unavailable", + }; + } const policy = (await publishedAcquisitionPolicies(db))[0]; - let action: "copy" | "move" = req.action ?? policy?.rules.organizeMode ?? orgSettings.defaultMode; + let action: "copy" | "move" | "in_place" = + req.action ?? sourceConfig?.action ?? policy?.rules.organizeMode ?? orgSettings.defaultMode; const site = rel ? await db.selectFrom("site").selectAll().where("id", "=", rel.site_id).executeTakeFirst() : undefined; @@ -472,19 +669,35 @@ export async function planImport(d: AoDeps, req: ImportRequest): Promise toDomainPathBinding(toApi(r))); - const root = await chooseRoot( - db, - entity, - bindings, - req.targetLibraryRootId, - await subscriptionLibraryRoot(db, { slotId, entityId: entity.id }), - ); - if (!root) return fail("no_library_root", {}, entity.id); - const targetDir = req.targetDir?.replace(/\/+$/, ""); + const root = inPlace + ? (await db.selectFrom("library_root").selectAll().where("enabled", "=", true).execute()) + .filter( + (r) => + (!req.targetLibraryRootId || r.id === req.targetLibraryRootId) && + files.every((f) => f.path.startsWith(`${r.local_path.replace(/\/+$/, "")}/`)), + ) + .sort((a, b) => b.local_path.length - a.local_path.length)[0] + : await chooseRoot( + db, + entity, + bindings, + req.targetLibraryRootId ?? (sourceConfig && !hasBinding ? sourceConfig.target.root.slice(4) : undefined), + await subscriptionLibraryRoot(db, { slotId, entityId: entity.id }), + ); + if (!root) return fail(inPlace ? "in_place_outside_library" : "no_library_root", {}, entity.id); + const targetDir = inPlace ? undefined : req.targetDir?.replace(/\/+$/, ""); if (targetDir !== undefined) { const rp = root.local_path.replace(/\/+$/, ""); - if (!(targetDir === rp || targetDir.startsWith(`${rp}/`)) || targetDir.split("/").includes("..")) - return fail("target_dir_outside_root", { targetDir, libraryRootId: root.id }, entity.id); + if (!(targetDir === rp || targetDir.startsWith(`${rp}/`)) || targetDir.split("/").includes("..")) { + return fail( + "target_dir_outside_root", + { + targetDir, + libraryRootId: root.id, + }, + entity.id, + ); + } } if (req.slotId) { const slot = await db @@ -492,13 +705,20 @@ export async function planImport(d: AoDeps, req: ImportRequest): Promise 库根 > 全局 回落并登记版本;目录只在首次绑定时渲染,文件名每次按当前模板 ── const namingSettings = await d.ctx.settings.get("naming").catch(() => undefined); - const namingEntity = await loadNamingEntity(db, entity, { libraryRoot: root.name, slotId: req.slotId }); - const templates = await ensureNamingVersion(db, effectiveNaming(namingSettings, root, entity).templates); + const namingEntity = await loadNamingEntity(db, entity, { + libraryRoot: root.name, + slotId: req.slotId, + }); + if (classification?.category) namingEntity.category = classification.category; + const effective = effectiveNaming(namingSettings, root, entity).templates; + let configuredTemplates = effective; + if (sourceConfig && !req.targetDir && sourceConfig.target.root === `lib:${root.id}`) { + const prefix = [sourceConfig.target.path, ...(sourceConfig.category ? [] : [classification?.category])] + .filter(Boolean) + .join("/"); + if (prefix) { + const literal = `${prefix + .split("/") + .filter(Boolean) + .map((part) => `{{ ${JSON.stringify(part)} }}`) + .join("/")}/`; + configuredTemplates = { + movie: literal + effective.movie, + tv: literal + effective.tv, + variety: literal + effective.variety, + }; + } + } + const templates = await ensureNamingVersion(db, configuredTemplates); const boundVersion = bindings.find((b) => b.scope === "entity")?.templateVersion; const fileTemplates = namingSettings?.lockFileNameForBound && boundVersion @@ -528,13 +771,18 @@ export async function planImport(d: AoDeps, req: ImportRequest): Promise [s, targetDir])); + else { + target.seasonDirs = Object.fromEntries([...seasons].map((s) => [s, targetDir])); + } } else if (mediaType === "movie") { const r = await handle(decide()); if (typeof r !== "string") return r; @@ -613,6 +866,26 @@ export async function planImport(d: AoDeps, req: ImportRequest): Promise + replacementScope( + path, + mediaType, + f.episodes + ? { + season: f.season, + episodes: Array.from( + { + length: f.episodes.end - f.episodes.start + 1, + }, + (_, i) => f.episodes!.start + i, + ), + } + : undefined, + ), + ); + const replacementPaths = replacement?.assets + .filter((a) => replacementCovers(a.scope, intendedScopes)) + .flatMap((a) => a.files.map((f) => join(f.rootPath, f.path))); const baseInput = { operation: action, mediaType, @@ -620,7 +893,10 @@ export async function planImport(d: AoDeps, req: ImportRequest): Promise f.path), + ); + for (const f of draft.files) { + if (successful.has(f.mainSrc ?? f.src)) skippedSuccessful.push(f); + } + const excluded = new Set(skippedSuccessful.map((f) => f.src)); + for (let i = files.length - 1; i >= 0; i--) { + if (excluded.has(files[i]!.path)) files.splice(i, 1); + } + } + const skippedPaths = new Set(skippedSuccessful.map((f) => f.src)); const existing: ExistingTarget[] = []; for (const f of draft.files) { + if (inPlace || skippedPaths.has(f.src)) continue; // keep:按保留原名后的真实目标观察存在性,仍以模板目标为键交给域层判定策略,之后统一改名 const probe = keepNames ? keptDst(f) : f.dst; const st = await statSafe(probe); @@ -653,10 +954,53 @@ export async function planImport(d: AoDeps, req: ImportRequest): Promise fileKindOf(f.path) === "video")) { + plan.conflicts = plan.conflicts.filter((c) => c.kind !== "no_main_video"); + plan.requiresApproval = plan.conflicts.some((c) => c.blocking); + } + if (inPlace && plan.files.length) { + const registered = await db + .selectFrom("asset_location as l") + .innerJoin("media_asset as a", "a.id", "l.media_asset_id") + .select(["l.relative_path", "a.media_entity_id"]) + .where("l.library_root_id", "=", root.id) + .where( + "l.relative_path", + "in", + plan.files.map((f) => relative(root.local_path, f.src)), + ) + .where("l.status", "!=", "removed") + .execute(); + for (const file of plan.files) + if ( + registered.some( + (r) => r.relative_path === relative(root.local_path, file.src) && r.media_entity_id !== entity.id, + ) + ) { + file.action = "hold"; + plan.conflicts.push({ + kind: "target_exists", + blocking: true, + src: file.src, + dst: file.dst, + detail: "该文件已登记为其它作品", + }); + plan.requiresApproval = true; + } + } + if (keepNames && !inPlace) keepOriginalNames(plan); + if (req.keepBoth && !inPlace) await keepBothTargets(plan); const frozen: FrozenPlan = { version: 1, + ...(req.scrapeAfterImport !== undefined ? { scrapeAfterImport: req.scrapeAfterImport } : {}), + ...(skippedSuccessful.length ? { skippedSuccessful } : {}), + ...(replacement ? { replacement } : {}), + ...(importPlanId ? { importPlanId } : {}), + ...(metadataMode ? { metadataMode } : {}), + sourceFacts, + ...(sourceConfig ? { sourceConfigId: sourceConfig.id } : {}), + ...(classification ? { classification } : {}), + ...(req.idempotencyKey ? { idempotencyKey: req.idempotencyKey } : {}), organize: plan, entityId: entity.id, mediaType, @@ -670,15 +1014,17 @@ export async function planImport(d: AoDeps, req: ImportRequest): Promise 库根 > 全局 回落;保留原文件名 = 本次覆写 templateSource: req.namingSource ?? - (keepNames ? "override" : effectiveNaming(namingSettings, root, entity).sources[namingEntity.kind!]), + (inPlace || keepNames ? "override" : effectiveNaming(namingSettings, root, entity).sources[namingEntity.kind!]), templateVersion: fileTemplates.version, }, }; - const drafts = organizePlanToFileResults(plan, { targetLibraryRootId: root.id }); + const drafts = organizePlanToFileResults(plan, { + targetLibraryRootId: root.id, + }); const blocking = plan.conflicts.filter((c) => c.blocking); const reason = blocking.length ? `blocked:${[...new Set(blocking.map((c) => c.kind))].join(",")}` @@ -687,7 +1033,7 @@ export async function planImport(d: AoDeps, req: ImportRequest): Promise (abs ? relative(root.local_path, abs) : null); - if (dry) + if (dry) { return { recordId: null, status: blocking.length ? "failed" : "queued", @@ -695,6 +1041,7 @@ export async function planImport(d: AoDeps, req: ImportRequest): Promise { const rec = await trx @@ -771,7 +1118,11 @@ export async function planImport(d: AoDeps, req: ImportRequest): Promise(); - for (const f of plan.files) byDst.set(f.dst, [...(byDst.get(f.dst) ?? []), f]); + for (const f of plan.files) { + byDst.set(f.dst, [...(byDst.get(f.dst) ?? []), f]); + } for (const [dst, fs] of byDst) { if (fs.length < 2 || plan.conflicts.some((c) => c.kind === "duplicate_target" && c.dst === dst)) continue; for (const f of fs) f.action = "hold"; - plan.conflicts.push({ kind: "duplicate_target", blocking: true, dst, detail: fs.map((f) => f.src).join(" | ") }); + plan.conflicts.push({ + kind: "duplicate_target", + blocking: true, + dst, + detail: fs.map((f) => f.src).join(" | "), + }); } plan.requiresApproval = plan.conflicts.some((c) => c.blocking); } @@ -819,7 +1189,7 @@ async function writeFailedRecord( actorId: string | null; taskId: string | null; downloadTaskId: string | null; - action: "copy" | "move"; + action: "copy" | "move" | "in_place"; reason: string; evidence: Record; entityId: string | null; @@ -868,7 +1238,13 @@ export async function emitRecord(d: AoDeps, exec: Exec, recordId: string): Promi // ── 执行 ── -type StepLog = { step: string; status: string; at: string; detail?: string; error?: { code: string; message: string } }; +type StepLog = { + step: string; + status: string; + at: string; + detail?: string; + error?: { code: string; message: string }; +}; type FileOutcome = { status: "succeeded" | "failed" | "attention"; bytes?: number; @@ -884,7 +1260,13 @@ type FileOutcome = { /** 失败时的附加证据(如对账阶段比对失败) */ evidence?: { phase: string; src: string; dst: string }; /** replace:旧文件去向 */ - replaced?: { disposition: string; size: number; mtimeMs: number; path?: string; error?: string }; + replaced?: { + disposition: string; + size: number; + mtimeMs: number; + path?: string; + error?: string; + }; /** 写前清走的残留 partial */ sweptPartials?: string[]; }; @@ -909,18 +1291,39 @@ export async function executeRecord( .where("id", "=", recordId) .executeTakeFirstOrThrow(); if (rec.status !== "running" && rec.status !== "queued") { - return { recordId, status: rec.status, succeeded: rec.file_succeeded, failed: rec.file_failed }; + await settleSamePathReplacement(d, rec.plan as unknown as FrozenPlan, recordId); + await cleanupReplacement(d, rec.plan as unknown as FrozenPlan, recordId); + const frozen = rec.plan as unknown as FrozenPlan; + if ( + (frozen.metadataMode === "local" || frozen.scrapeAfterImport !== undefined) && + ["succeeded", "partial"].includes(rec.status) + ) { + await scrapeAfterImport(d, frozen.entityId, recordId, { + force: frozen.metadataMode === "local", + enabled: frozen.scrapeAfterImport, + }); + } + return { + recordId, + status: rec.status, + succeeded: rec.file_succeeded, + failed: rec.file_failed, + }; } const frozen = rec.plan as unknown as FrozenPlan; const fsr = fsRuntime(d); // 根检查也走执行子进程:NFS/FUSE 挂死 → offline(timeout),按根离线重试,绝不当成文件缺失 const health = await fsr.proxy.checkRoot(frozen.rootPath, frozen.sentinel); - if (health.kind === "offline") throw new Error(`媒体库目录离线(${health.reason}),稍后重试`); - for (const dir of frozen.targetDirs) await mkdir(dir, { recursive: true }); + if (health.kind === "offline") { + throw new Error(`媒体库目录离线(${health.reason}),稍后重试`); + } + if (rec.action !== "in_place") for (const dir of frozen.targetDirs) await mkdir(dir, { recursive: true }); // 写前顺带清本目录残留 partial(对照 MP:只清本目录、>24h;失败吞掉)。copy 路径在 copyWithVerify 内还会再清一次 - if (fsr.partialMaxAgeMs > 0) - for (const dir of frozen.targetDirs) + if (rec.action !== "in_place" && fsr.partialMaxAgeMs > 0) { + for (const dir of frozen.targetDirs) { await fsr.proxy.sweepStalePartials(dir, fsr.partialMaxAgeMs).catch(() => [] as string[]); + } + } const orgSettings = await d.ctx.settings.get("organize"); const files = await db @@ -932,7 +1335,12 @@ export async function executeRecord( const planned = new Map(frozen.organize.files.map((f) => [f.src, f])); const failedMains = new Set(); /** 本轮 TIMEOUT/IO 类失败、留待任务重试的文件(结果不入账本,文件结果保持 running) */ - const deferred: { src: string; code: string; message: string; attempt: number }[] = []; + const deferred: { + src: string; + code: string; + message: string; + attempt: number; + }[] = []; for (const fr of files) { if (fr.status !== "queued" && fr.status !== "running") continue; const pf = planned.get(fr.source_path); @@ -942,14 +1350,18 @@ export async function executeRecord( if (pf.role === "sidecar" && pf.mainSrc && failedMains.has(pf.mainSrc)) { await db .updateTable("organization_file_result") - .set({ status: "skipped", skip_reason: "main_failed", finished_at: d.ports.now() }) + .set({ + status: "skipped", + skip_reason: "main_failed", + finished_at: d.ports.now(), + }) .where("id", "=", fr.id) .execute(); continue; } opts.abortSignal?.throwIfAborted(); const opId = `org:${recordId}:${fr.ordinal}`; - const kind = rec.action === "move" ? "move" : "copy"; + const kind = rec.action === "in_place" ? "register" : rec.action === "move" ? "move" : "copy"; await db .insertInto("asset_operation") .values({ @@ -958,15 +1370,23 @@ export async function executeRecord( organization_record_id: recordId, plan_fingerprint: rec.plan_fingerprint, ordinal: fr.ordinal, - phase: "transfer", + phase: kind === "register" ? "verify" : "transfer", kind, status: "started", - intent: jsonb({ src: pf.src, dst: pf.dst, size: pf.size, crossDevice: pf.crossDevice ?? null }), + intent: jsonb({ + src: pf.src, + dst: pf.dst, + size: pf.size, + crossDevice: pf.crossDevice ?? null, + }), download_task_id: rec.download_task_id, started_at: d.ports.now(), }) .onConflict((oc) => - oc.column("operation_id").doUpdateSet({ attempts: sql`asset_operation.attempts + 1`, status: "started" }), + oc.column("operation_id").doUpdateSet({ + attempts: sql`asset_operation.attempts + 1`, + status: "started", + }), ) .execute(); await db @@ -981,19 +1401,63 @@ export async function executeRecord( opId, async (info) => { const t0 = performance.now(); - const r = await transferFile(d, fsr, { - kind, - src: pf.src, - dst: pf.dst, - size: pf.size, - crossDevice: pf.crossDevice, - resumed: info.resumed, - allowCrossVolume: orgSettings.allowCrossVolumeMove, - // 计划冻结的目标存在策略:replace = replace_with_verify 已获授权;其余目标此刻若已存在一律冲突 - onExisting: pf.action === "replace" ? "replace" : "conflict", - replaceOld: { mode: fsr.replaceOld, trashDir: join(frozen.rootPath, ".umt-trash") }, - ...(opts.abortSignal ? { signal: opts.abortSignal } : {}), - }); + const selectedOld = + pf.action === "replace" + ? frozen.replacement?.assets.flatMap((a) => a.files).find((f) => join(f.rootPath, f.path) === pf.dst) + : undefined; + const stagedOld = selectedOld ? await stagedReplacementBackup(d, selectedOld) : null; + if (selectedOld && !stagedOld) { + const current = await fsr.proxy.statSafe(pf.dst, { + root: selectedOld.rootPath, + sentinel: selectedOld.sentinel, + }); + if ( + current.kind !== "ok" || + current.facts.isSymbolicLink || + Object.entries(selectedOld.facts).some( + ([key, value]) => current.facts[key as keyof typeof selectedOld.facts] !== value, + ) + ) { + return { + status: "failed", + error: { + code: "STALE_REPLACEMENT", + message: "选定旧文件在计划后发生变化,保留待核对", + }, + durationMs: 0, + }; + } + } + const r: FileOutcome = + kind === "register" + ? await verifyInPlace(d, fsr, frozen, pf.src, pf.size) + : await transferFile(d, fsr, { + kind, + src: pf.src, + dst: pf.dst, + size: pf.size, + crossDevice: pf.crossDevice, + resumed: info.resumed, + allowCrossVolume: orgSettings.allowCrossVolumeMove, + // 计划冻结的目标存在策略:replace = replace_with_verify 已获授权;其余目标此刻若已存在一律冲突 + onExisting: pf.action === "replace" ? "replace" : "conflict", + replaceOld: { + mode: frozen.replacement ? "trash" : fsr.replaceOld, + trashDir: selectedOld + ? join(selectedOld.rootPath, ".umt-trash", "replacement-staging", selectedOld.trashId) + : join(frozen.rootPath, ".umt-trash"), + }, + forceFullVerify: !!frozen.replacement && pf.action === "replace", + ...(opts.abortSignal ? { signal: opts.abortSignal } : {}), + }); + if (r.status === "succeeded" && stagedOld && selectedOld && !r.replaced) { + r.replaced = { + disposition: "trash", + path: stagedOld, + size: selectedOld.facts.size, + mtimeMs: selectedOld.facts.mtimeMs, + }; + } r.durationMs = Math.round(performance.now() - t0); r.attempt = info.attempt; // TIMEOUT/IO 类:未到上限 → 不入账本、抛出等任务重试;到上限 → 按失败入账并建待处理 @@ -1002,8 +1466,9 @@ export async function executeRecord( r.error && fsErrorClass(r.error.code) === "retry" && info.attempt < fsr.maxAttempts - ) + ) { throw new RetryableTransfer(r.error, info.attempt); + } // 打点:每个文件的耗时/字节/哈希方式,便于判断大文件入库是否拖慢 worker d.ctx.logger.info( { @@ -1052,7 +1517,10 @@ export async function executeRecord( error: r.error ? jsonb(r.error) : r.status === "attention" - ? jsonb({ code: "source_not_removed", message: "目标已就绪,源文件未能删除" }) + ? jsonb({ + code: "source_not_removed", + message: "目标已就绪,源文件未能删除", + }) : null, finished_at: now, }) @@ -1063,32 +1531,49 @@ export async function executeRecord( ); } catch (e) { if (!(e instanceof RetryableTransfer)) throw e; - deferred.push({ src: pf.src, code: e.error.code, message: e.error.message, attempt: e.attempt }); + deferred.push({ + src: pf.src, + code: e.error.code, + message: e.error.message, + attempt: e.attempt, + }); await db .updateTable("asset_operation") - .set({ error: jsonb({ ...e.error, attempt: e.attempt, retrying: true }) }) + .set({ + error: jsonb({ ...e.error, attempt: e.attempt, retrying: true }), + }) .where("operation_id", "=", opId) .execute(); continue; } - if (outcome.status === "failed" && pf.role === "main") failedMains.add(pf.src); + if (outcome.status === "failed" && pf.role === "main") { + failedMains.add(pf.src); + } if (outcome.status === "failed" && outcome.error && fsErrorClass(outcome.error.code) === "retry") { await upsertDecision(d, db, { kind: "other", title: outcome.error.code === "TIMEOUT" ? "文件操作挂死,已达重试上限" : "文件读写失败,已达重试上限", - summary: `${pf.src} → ${pf.dst}:${outcome.error.message}(已尝试 ${outcome.attempt ?? fsr.maxAttempts} 次;记录与下载已保留,不会重新下载)`, + summary: `${pf.src} → ${pf.dst}:${outcome.error.message}(已尝试 ${ + outcome.attempt ?? fsr.maxAttempts + } 次;记录与下载已保留,不会重新下载)`, subjectType: "organization_record", subjectId: recordId, dedupeKey: `fs_retry_exhausted:${opId}`, - evidence: { src: pf.src, dst: pf.dst, error: outcome.error, attempts: outcome.attempt ?? null }, + evidence: { + src: pf.src, + dst: pf.dst, + error: outcome.error, + attempts: outcome.attempt ?? null, + }, options: [{ key: "retry", label: "检查挂载后重试" }], }); } // 被替换的旧文件登记进文件管理回收站(可在回收站视图还原/清除);登记失败只告警,旧文件留在 .umt-trash - if (outcome.replaced?.disposition === "trash" && outcome.replaced.path) + if (outcome.replaced?.disposition === "trash" && outcome.replaced.path) { await trashReplacedOld(d, fsr, frozen, pf.dst, { recordId, opId }, outcome.replaced).catch((err) => d.ctx.logger.warn({ err, recordId, dst: pf.dst }, "替换旧文件登记回收站失败"), ); + } if (outcome.replaced?.disposition === "left") { await upsertDecision(d, db, { kind: "other", @@ -1142,7 +1627,9 @@ async function trashReplacedOld( const to = join(rootReal, ".umt-trash", now.toISOString().slice(0, 10).replace(/-/g, ""), id, rel); await mkdir(dirname(to), { recursive: true }); const m = await fsr.proxy.moveSafe(old.path, to); - if (m.status !== "moved") throw new Error(`旧文件移入回收站失败(${m.status}): ${old.path}`); + if (m.status !== "moved") { + throw new Error(`旧文件移入回收站失败(${m.status}): ${old.path}`); + } await rmdir(dirname(old.path)).catch(() => undefined); const days = Number(Deno.env.get("FILES_TRASH_RETENTION_DAYS") ?? 7) || 7; await d.ctx.db @@ -1159,7 +1646,11 @@ async function trashReplacedOld( actor: "system:organize", deleted_at: now, expires_at: new Date(now.getTime() + days * 86_400_000), - asset_snapshot: jsonb({ source: "organize_replace", recordId: ref.recordId, dst }), + asset_snapshot: jsonb({ + source: "organize_replace", + recordId: ref.recordId, + dst, + }), }) .execute(); // 账本里的旧文件去向跟着改到回收站路径 @@ -1218,7 +1709,10 @@ function statFailure(st: StatResult, path: string, role: "源" | "目标"): { co case "permission_denied": return { code: "EACCES", message: `${role}无权限访问:${path}` }; case "root_offline": - return { code: "ROOT_OFFLINE", message: `${role}所在根离线(${st.reason}): ${path}` }; + return { + code: "ROOT_OFFLINE", + message: `${role}所在根离线(${st.reason}): ${path}`, + }; case "error": return { code: st.code === "TIMEOUT" ? "TIMEOUT" : "IO", @@ -1227,6 +1721,38 @@ function statFailure(st: StatResult, path: string, role: "源" | "目标"): { co } } +export async function verifyInPlace( + d: AoDeps, + fsr: FsRuntime, + frozen: FrozenPlan, + path: string, + size: number, +): Promise { + const observed = await fsr.proxy.statSafe(path, { root: frozen.rootPath, sentinel: frozen.sentinel }); + if (observed.kind !== "ok" || !observed.facts.isFile || observed.facts.isSymbolicLink) { + return { status: "failed", error: { code: "SOURCE_MISSING", message: "原地入库的文件不存在或不可访问" } }; + } + const planned = frozen.sourceFacts?.[path]; + if (observed.facts.size !== size || (planned && observed.facts.mtimeMs !== planned.mtimeMs)) { + return { status: "failed", error: { code: "STALE_SOURCE", message: "文件在计划后发生变化,请重新预览" } }; + } + const location = await d.ctx.db + .selectFrom("asset_location as l") + .innerJoin("media_asset as a", "a.id", "l.media_asset_id") + .select("a.media_entity_id") + .where("l.library_root_id", "=", frozen.libraryRootId) + .where("l.relative_path", "=", relative(frozen.rootPath, path)) + .where("l.status", "!=", "removed") + .executeTakeFirst(); + if (location && location.media_entity_id !== frozen.entityId) { + return { + status: "failed", + error: { code: "ASSET_ENTITY_CONFLICT", message: "该文件已登记为其它作品,请先核对作品归属" }, + }; + } + return { status: "succeeded", bytes: 0 }; +} + async function transferFile( d: AoDeps, fsr: FsRuntime, @@ -1241,12 +1767,16 @@ async function transferFile( onExisting: "conflict" | "replace"; replaceOld: ReplaceOldPolicy; signal?: AbortSignal; + forceFullVerify?: boolean; }, ): Promise { const { kind, src, dst, size, crossDevice, resumed } = x; const px = fsr.proxy; const call = x.signal ? { signal: x.signal } : {}; - const fsErr = (e: unknown) => ({ code: e instanceof FsError ? e.code : "IO", message: errMessage(e) }); + const fsErr = (e: unknown) => ({ + code: e instanceof FsError ? e.code : "IO", + message: errMessage(e), + }); const copyReceipt = (r: CopyReceipt): FileOutcome => ({ status: "succeeded", bytes: r.bytes, @@ -1256,7 +1786,7 @@ async function transferFile( ...(r.sweptPartials?.length ? { sweptPartials: r.sweptPartials } : {}), }); const copyOpts = { - hash: d.ports.options.verifyHash, + hash: x.forceFullVerify ? ("sha256" as const) : d.ports.options.verifyHash, onExisting: x.onExisting, replaceOld: x.replaceOld, sweepPartialsOlderThanMs: fsr.partialMaxAgeMs > 0 ? fsr.partialMaxAgeMs : null, @@ -1271,7 +1801,10 @@ async function transferFile( if (s.kind === "missing" && kind === "move") { return t.facts.size === size ? { status: "succeeded", bytes: size, resumed: true } - : { status: "failed", error: { code: "CONFLICT", message: "源已不存在且目标大小不符" } }; + : { + status: "failed", + error: { code: "CONFLICT", message: "源已不存在且目标大小不符" }, + }; } if (s.kind === "ok") { // 比对读失败/超时 → 按 IO/TIMEOUT 失败并留证据,不得冒充"内容不同"(EEXIST) @@ -1287,8 +1820,14 @@ async function transferFile( }; } if (same.match) { - if (kind === "copy") - return { status: "succeeded", bytes: size, resumed, ...(same.via ? { hashVia: same.via } : {}) }; + if (kind === "copy") { + return { + status: "succeeded", + bytes: size, + resumed, + ...(same.via ? { hashVia: same.via } : {}), + }; + } if (crossDevice === true || resumed || x.onExisting === "replace") { const r = await px.crossDeviceMove(src, dst, { hash: "sha256" }, call); return { @@ -1298,8 +1837,12 @@ async function transferFile( resumed: true, }; } - } else if (x.onExisting !== "replace") - return { status: "failed", error: { code: "EEXIST", message: `目标已存在且内容不同:${dst}` } }; + } else if (x.onExisting !== "replace") { + return { + status: "failed", + error: { code: "EEXIST", message: `目标已存在且内容不同:${dst}` }, + }; + } } } const sFail = statFailure(s, src, "源"); @@ -1314,28 +1857,54 @@ async function transferFile( () => null, (e) => fsErr(e), ); - if (del) + if (del) { return { ...out, status: "attention", - steps: [{ step: "delete_source", status: "failed", at: new Date().toISOString(), error: del }], + steps: [ + { + step: "delete_source", + status: "failed", + at: new Date().toISOString(), + error: del, + }, + ], }; + } } return out; } - if (kind === "copy") return copyReceipt(await px.copy(src, dst, copyOpts, call)); + if (kind === "copy") { + return copyReceipt(await px.copy(src, dst, copyOpts, call)); + } if (crossDevice !== true) { const m = await px.moveSafe(src, dst, call); if (m.status === "moved") return { status: "succeeded", bytes: size }; - if (m.status === "exists") return { status: "failed", error: { code: "EEXIST", message: `目标已存在:${dst}` } }; + if (m.status === "exists") { + return { + status: "failed", + error: { code: "EEXIST", message: `目标已存在:${dst}` }, + }; + } // EXDEV:不静默降级;按设置显式走可恢复三步 } - if (!x.allowCrossVolume) + if (!x.allowCrossVolume) { return { status: "failed", - error: { code: "EXDEV", message: "未启用跨卷移动(settings.organize.allowCrossVolumeMove)" }, + error: { + code: "EXDEV", + message: "未启用跨卷移动(settings.organize.allowCrossVolumeMove)", + }, }; - const r = await px.crossDeviceMove(src, dst, { hash: d.ports.options.verifyHash }, call); + } + const r = await px.crossDeviceMove( + src, + dst, + { + hash: d.ports.options.verifyHash, + }, + call, + ); const failedStep = r.steps.find((st) => st.status === "failed"); return { status: r.status === "completed" ? "succeeded" : r.status, @@ -1378,8 +1947,9 @@ export async function finalizeRecord(d: AoDeps, recordId: string): Promise join(dirname(abs), n)), - )) + )) { otherVersions.push(relative(frozenPre.rootPath, o)); + } } } @@ -1415,7 +1985,13 @@ export async function finalizeRecord(d: AoDeps, recordId: string): Promise a.files.some((old) => old.locationId === hit!.id)) + ) { + await trx.updateTable("asset_location").set({ status: "removed" }).where("id", "=", hit.id).execute(); + await sql`update media_asset set status='retired' where id=${hit.media_asset_id} and not exists(select 1 from asset_location where media_asset_id=${hit.media_asset_id} and status <> 'removed')`.execute( + trx, + ); + hit = undefined; + } let assetId: string; let locationId: string; const st = await statSafe(join(frozen.rootPath, f.target_path)); @@ -1493,12 +2082,18 @@ export async function finalizeRecord(d: AoDeps, recordId: string): Promise code - ? { code: code.split(":")[0]!, message: conflictText(err) || reasonSentence(code, code) } + ? { + code: code.split(":")[0]!, + message: conflictText(err) || reasonSentence(code, code), + } : err ? { code: (err as { code?: string }).code ?? "error", @@ -1916,7 +2562,11 @@ export async function loadRecordsPublic(exec: Exec, rows: OrganizationRecordRow[ episodeNumber: u?.episode_number ?? null, externalIds: idents .filter((i) => i.media_entity_id === e.id) - .map((i) => ({ provider: i.provider, entityType: i.entity_type, externalId: i.external_id })), + .map((i) => ({ + provider: i.provider, + entityType: i.entity_type, + externalId: i.external_id, + })), posterPath: posterOf(e.metadata), } : null, @@ -1933,14 +2583,26 @@ export async function loadRecordsPublic(exec: Exec, rows: OrganizationRecordRow[ }, target: targetRef, reason: reason(r.reason, r.error && r.status === "failed" ? r.error : null), - counts: { total: r.file_total, succeeded: r.file_succeeded, failed: r.file_failed, skipped: r.file_skipped }, + counts: { + total: r.file_total, + succeeded: r.file_succeeded, + failed: r.file_failed, + skipped: r.file_skipped, + }, sizeBytes: (() => { const b = sizes.find((x) => x.organization_record_id === r.id)?.bytes; return b == null ? null : Number(b); })(), download: (() => { const t = dls.find((x) => x.id === r.download_task_id); - return t ? { taskId: t.id, name: t.name, hash: t.torrent_hash, downloaderName: t.downloader_name } : null; + return t + ? { + taskId: t.id, + name: t.name, + hash: t.torrent_hash, + downloaderName: t.downloader_name, + } + : null; })(), files: includeFiles ? files diff --git a/apps/server/src/modules/acquisition-organize/services/plan-why.ts b/apps/server/src/modules/acquisition-organize/services/plan-why.ts index 4f9f8134..f956d6de 100644 --- a/apps/server/src/modules/acquisition-organize/services/plan-why.ts +++ b/apps/server/src/modules/acquisition-organize/services/plan-why.ts @@ -30,7 +30,7 @@ export async function planWhys(db: Exec, planIds: readonly string[]): Promise 40 ? `${goal.slice(0, 40)}…` : goal }; - } else if (inv.length) source = { kind: "manual" }; + } else if (inv.length || evOf(p).manual === true) source = { kind: "manual" }; else if (sub) { const title = sub.title ?? sub.title_query ?? ""; source = { diff --git a/apps/server/src/modules/acquisition-organize/services/replacement.ts b/apps/server/src/modules/acquisition-organize/services/replacement.ts new file mode 100644 index 00000000..f4b37f7e --- /dev/null +++ b/apps/server/src/modules/acquisition-organize/services/replacement.ts @@ -0,0 +1,502 @@ +import { randomUUID } from "node:crypto"; +import { mkdir, readdir, realpath, rm } from "node:fs/promises"; +import { basename, dirname, join, relative } from "node:path"; +import { DEFAULT_ROOT_SENTINEL, type FileFacts } from "@umt/adapters"; +import { sql } from "@umt/db"; +import { type ReplacementScope, replacementCovers, replacementScope } from "@umt/domain"; +import { jsonb } from "../util.ts"; +import { type AoDeps, upsertDecision } from "./common.ts"; +import { type FrozenPlan, fsRuntime } from "./organize.ts"; + +export interface ReplacementFile { + assetId: string; + locationId: string; + rootId: string; + rootPath: string; + sentinel: string; + path: string; + facts: Pick; + trashId: string; + trashDay: string; +} +export interface ReplacementAsset { + assetId: string; + scope: ReplacementScope; + files: ReplacementFile[]; +} +export interface FrozenReplacement { + mode: "replace"; + assets: ReplacementAsset[]; +} + +export async function freezeReplacement( + d: AoDeps, + entityId: string, + mediaType: "movie" | "tv", + ids: string[], +): Promise { + const selected = [...new Set(ids)]; + if (!selected.length) throw new Error("洗版入库必须选择旧版本"); + const assets = await d.ctx.db.selectFrom("media_asset").selectAll().where("id", "in", selected).execute(); + if ( + assets.length !== selected.length || + assets.some( + (a) => a.media_entity_id !== entityId || a.status !== "active" || !["video", "bluray_dir"].includes(a.kind), + ) + ) { + throw new Error("旧版本必须是当前作品的有效库内视频资产"); + } + const out: ReplacementAsset[] = []; + for (const asset of assets) { + const locations = await d.ctx.db + .selectFrom("asset_location as l") + .innerJoin("library_root as r", "r.id", "l.library_root_id") + .select([ + "l.id", + "l.media_asset_id", + "l.relative_path", + "r.id as root_id", + "r.local_path", + "r.sentinel_file", + "r.management_mode", + ]) + .where("l.media_asset_id", "=", asset.id) + .where("l.location_kind", "=", "library") + .where("l.status", "=", "available") + .execute(); + if (!locations.length || locations.some((l) => l.management_mode !== "managed")) { + throw new Error("旧版本必须有可写的库内位置"); + } + const units = await d.ctx.db + .selectFrom("asset_unit_coverage as c") + .innerJoin("content_unit as u", "u.id", "c.content_unit_id") + .select(["u.season_number", "u.episode_number"]) + .where("c.media_asset_id", "=", asset.id) + .where("c.coverage", "=", "full") + .execute(); + const seasons = new Set(units.map((u) => u.season_number).filter((s) => s !== null)); + const explicit = + seasons.size === 1 + ? { + season: [...seasons][0]!, + episodes: units.flatMap((u) => (u.episode_number === null ? [] : [u.episode_number])), + } + : undefined; + const scope = replacementScope( + locations[0]!.relative_path, + mediaType, + explicit?.episodes.length ? explicit : undefined, + ); + if (seasons.size > 1) scope.episodes = []; + const files: ReplacementFile[] = []; + for (const loc of locations) { + const children = await d.ctx.db + .selectFrom("media_asset as a") + .innerJoin("asset_location as l", "l.media_asset_id", "a.id") + .select(["a.id as media_asset_id", "l.id", "l.relative_path"]) + .where("a.parent_asset_id", "=", asset.id) + .where("a.status", "=", "active") + .where("l.library_root_id", "=", loc.root_id) + .where("l.status", "=", "available") + .execute(); + const stem = basename(loc.relative_path).replace(/\.[^.]+$/, ""); + const privateChildren = children.filter( + (c) => + dirname(c.relative_path) === dirname(loc.relative_path) && basename(c.relative_path).startsWith(`${stem}.`), + ); + for (const file of [loc, ...privateChildren]) { + const path = join(loc.local_path, file.relative_path); + const st = await fsRuntime(d).proxy.statSafe(path, { + root: loc.local_path, + sentinel: loc.sentinel_file ?? DEFAULT_ROOT_SENTINEL, + }); + if (st.kind !== "ok" || !st.facts.isFile || st.facts.isSymbolicLink) { + throw new Error(`旧版本文件不可核验:${file.relative_path}`); + } + files.push({ + assetId: file.media_asset_id, + locationId: file.id, + rootId: loc.root_id, + rootPath: loc.local_path, + sentinel: loc.sentinel_file ?? DEFAULT_ROOT_SENTINEL, + path: file.relative_path, + facts: { + size: st.facts.size, + mtimeMs: st.facts.mtimeMs, + ino: st.facts.ino, + dev: st.facts.dev, + }, + trashId: randomUUID(), + trashDay: d.ports.now().toISOString().slice(0, 10).replace(/-/g, ""), + }); + } + } + out.push({ assetId: asset.id, scope, files }); + } + return { mode: "replace", assets: out }; +} + +/** Called after successful new assets commit; only library locations and private sidecars are eligible. */ +export async function cleanupReplacement(d: AoDeps, frozen: FrozenPlan, recordId: string): Promise { + if (!frozen.replacement) return; + const db = d.ctx.db; + const successful = await db + .selectFrom("organization_file_result as f") + .innerJoin("media_asset as a", "a.id", "f.media_asset_id") + .select(["f.source_path", "f.target_path", "f.media_asset_id"]) + .where("f.organization_record_id", "=", recordId) + .where("f.role", "=", "main") + .where((eb) => + eb.or([ + eb("f.status", "=", "succeeded"), + eb.and([eb("f.status", "=", "skipped"), eb("f.skip_reason", "=", "already_in_library")]), + ]), + ) + .where("a.status", "=", "active") + .execute(); + const registeredUnits = successful.length + ? await db + .selectFrom("asset_unit_coverage as c") + .innerJoin("content_unit as u", "u.id", "c.content_unit_id") + .select(["c.media_asset_id", "u.season_number", "u.episode_number"]) + .where( + "c.media_asset_id", + "in", + successful.map((f) => f.media_asset_id!), + ) + .where("c.coverage", "=", "full") + .execute() + : []; + const scopes = successful.flatMap((f) => { + const units = registeredUnits.filter( + (u) => u.media_asset_id === f.media_asset_id && u.season_number !== null && u.episode_number !== null, + ); + if (units.length) { + return [...new Set(units.map((u) => u.season_number!))].map((season) => + replacementScope(f.source_path, frozen.mediaType, { + season, + episodes: units.filter((u) => u.season_number === season).map((u) => u.episode_number!), + }), + ); + } + const facts = frozen.facts[f.source_path]; + return [ + replacementScope( + f.source_path, + frozen.mediaType, + facts?.episodes + ? { + season: facts.season, + episodes: Array.from( + { length: facts.episodes.end - facts.episodes.start + 1 }, + (_, i) => facts.episodes!.start + i, + ), + } + : undefined, + ), + ]; + }); + for (const old of frozen.replacement.assets) { + if (successful.some((f) => f.media_asset_id === old.assetId)) continue; + if (!replacementCovers(old.scope, scopes)) { + if (successful.length) { + await upsertDecision(d, db, { + kind: "other", + severity: "info", + title: "旧版本未完整覆盖,保留文件", + summary: "本次成功文件未明确覆盖旧文件的全部季集或 Part;旧文件保留,可核对范围后处理", + subjectType: "media_asset", + subjectId: old.assetId, + dedupeKey: `replacement_scope:${recordId}:${old.assetId}`, + evidence: { recordId, oldScope: old.scope, successfulScopes: scopes }, + }); + } + continue; + } + for (const file of old.files) { + const opId = `replacement:${recordId}:${file.locationId}`; + const existing = await db + .selectFrom("asset_operation") + .select(["status", "result"]) + .where("operation_id", "=", opId) + .executeTakeFirst(); + if (existing?.status === "succeeded") continue; + const location = await db + .selectFrom("asset_location") + .select(["status", "media_asset_id", "relative_path"]) + .where("id", "=", file.locationId) + .executeTakeFirst(); + if (!location || location.media_asset_id !== file.assetId || location.relative_path !== file.path) continue; + // A selected same-path replacement has already been quarantined by copyWithVerify and its old location retired. + if (location.status === "removed") continue; + if (successful.some((f) => frozen.libraryRootId === file.rootId && f.target_path === file.path)) continue; + try { + const fs = fsRuntime(d); + if ((await fs.proxy.checkRoot(file.rootPath, file.sentinel)).kind !== "online") { + throw new Error("旧版本库根离线"); + } + const rootReal = await realpath(file.rootPath); + const abs = join(rootReal, file.path); + const parent = await realpath(dirname(abs)); + const rel = relative(rootReal, parent); + if (rel.startsWith("..") || file.path.split("/").some((s) => s === "..")) throw new Error("旧文件位置越界"); + const trash = join(rootReal, ".umt-trash", file.trashDay, file.trashId, file.path); + const st = await fs.proxy.statSafe(abs, { + root: rootReal, + sentinel: file.sentinel, + }); + const moved = await fs.proxy.statSafe(trash); + const resumed = + existing?.status === "started" && + st.kind === "missing" && + (fs.replaceOld === "delete" || moved.kind === "ok"); + if ( + !resumed && + (st.kind !== "ok" || + st.facts.isSymbolicLink || + Object.entries(file.facts).some(([k, v]) => st.facts[k as keyof typeof file.facts] !== v)) + ) { + throw new Error("旧文件已变化,保留待核对"); + } + await db + .insertInto("asset_operation") + .values({ + operation_id: opId, + phase: "cleanup", + kind: "delete_target", + status: "started", + organization_record_id: recordId, + media_asset_id: file.assetId, + intent: jsonb({ file, mode: fs.replaceOld }), + started_at: d.ports.now(), + }) + .onConflict((oc) => oc.column("operation_id").doUpdateSet({ status: "started" })) + .execute(); + if (!resumed) { + if (fs.replaceOld === "delete") await rm(abs); + else { + await mkdir(dirname(trash), { recursive: true }); + const result = await fs.proxy.moveSafe(abs, trash); + if (result.status !== "moved") { + throw new Error(`旧文件回收失败:${result.status}`); + } + } + } + await db.transaction().execute(async (trx) => { + if (fs.replaceOld === "trash") { + await trx + .insertInto("trash_item") + .values({ + id: file.trashId, + root_id: `lib:${file.rootId}`, + library_root_id: file.rootId, + root_host_path: rootReal, + original_path: file.path, + trash_path: trash, + is_dir: false, + size_bytes: file.facts.size, + actor: "system:organize", + deleted_at: d.ports.now(), + expires_at: new Date(d.ports.now().getTime() + 7 * 86_400_000), + asset_snapshot: jsonb({ + source: "one_time_replacement", + recordId, + assets: [ + { + assetId: file.assetId, + locationId: file.locationId, + }, + ], + }), + }) + .onConflict((oc) => oc.column("id").doNothing()) + .execute(); + } + await trx + .updateTable("asset_location") + .set({ status: "removed" }) + .where("id", "=", file.locationId) + .execute(); + await sql`update media_asset set status='retired' where id=${file.assetId} and not exists(select 1 from asset_location where media_asset_id=${file.assetId} and status <> 'removed')`.execute( + trx, + ); + await trx + .updateTable("asset_operation") + .set({ + status: "succeeded", + result: jsonb({ + disposition: fs.replaceOld, + path: fs.replaceOld === "trash" ? trash : null, + }), + finished_at: d.ports.now(), + }) + .where("operation_id", "=", opId) + .execute(); + }); + } catch (error) { + await upsertDecision(d, db, { + kind: "other", + title: "旧版本清理未完成", + summary: error instanceof Error ? error.message : String(error), + subjectType: "media_asset", + subjectId: file.assetId, + dedupeKey: `replacement_cleanup:${recordId}:${file.locationId}`, + evidence: { recordId, file }, + }); + } + } + } +} + +/** Keep the old same-path file until the new asset has passed final validation. */ +export async function settleSamePathReplacement(d: AoDeps, frozen: FrozenPlan, recordId: string): Promise { + if (!frozen.replacement) return; + const results = await d.ctx.db + .selectFrom("organization_file_result") + .selectAll() + .where("organization_record_id", "=", recordId) + .execute(); + const bySource = new Map(results.map((f) => [f.source_path, f])); + for (const pf of frozen.organize.files) { + if (pf.action !== "replace") continue; + const old = frozen.replacement.assets.flatMap((a) => a.files).find((f) => join(f.rootPath, f.path) === pf.dst); + if (!old) continue; + const main = bySource.get(pf.mainSrc ?? pf.src); + const file = bySource.get(pf.src); + if (!file) continue; + const opId = `org:${recordId}:${file.ordinal}`; + const op = await d.ctx.db + .selectFrom("asset_operation") + .select("result") + .where("operation_id", "=", opId) + .executeTakeFirst(); + const result = op?.result as + | { + replaced?: { disposition: string; path?: string; trashItemId?: string }; + rollback?: boolean; + } + | undefined; + const backup = result?.replaced; + if (!backup?.path || backup.disposition !== "trash" || result?.rollback) { + continue; + } + try { + const fs = fsRuntime(d); + if (main?.status === "succeeded" && main.media_asset_id) { + if (fs.replaceOld === "delete") { + await rm(backup.path, { force: true }); + if (backup.trashItemId) { + await d.ctx.db.deleteFrom("trash_item").where("id", "=", backup.trashItemId).execute(); + } + await d.ctx.db + .updateTable("asset_operation") + .set({ + result: jsonb({ + ...result, + replaced: { ...backup, disposition: "deleted" }, + }), + }) + .where("operation_id", "=", opId) + .execute(); + } + continue; + } + if (main?.status !== "failed") continue; + if ((await fs.proxy.checkRoot(old.rootPath, old.sentinel)).kind !== "online") { + throw new Error("旧版本库根离线,保留回收副本"); + } + const rejected = join(old.rootPath, ".umt-trash", old.trashDay, old.trashId, old.path); + const oldBackup = await fs.proxy.statSafe(backup.path); + const target = await fs.proxy.statSafe(pf.dst); + const alreadyRestored = + oldBackup.kind === "missing" && + target.kind === "ok" && + Object.entries(old.facts).every(([key, value]) => target.facts[key as keyof typeof old.facts] === value); + if (!alreadyRestored) { + if ( + oldBackup.kind !== "ok" || + oldBackup.facts.size !== old.facts.size || + oldBackup.facts.ino !== old.facts.ino + ) { + throw new Error("旧文件回收副本已变化"); + } + if (!file.checksum?.startsWith("sha256:")) { + throw new Error("失败目标缺少全量校验,旧文件留在回收站"); + } + if (target.kind === "ok") { + if ((await fs.proxy.sha256(pf.dst)).digest !== file.checksum.slice(7)) { + throw new Error("失败目标已变化,旧文件留在回收站"); + } + await mkdir(dirname(rejected), { recursive: true }); + const parked = await fs.proxy.moveSafe(pf.dst, rejected); + if (parked.status !== "moved") throw new Error("失败新文件无法暂存"); + } else if (target.kind !== "missing" || (await fs.proxy.sha256(rejected)).digest !== file.checksum.slice(7)) { + throw new Error("失败新文件暂存无法核验"); + } + const restored = await fs.proxy.moveSafe(backup.path, pf.dst); + if (restored.status !== "moved") { + throw new Error("旧文件恢复未完成,副本保留"); + } + } + await d.ctx.db.transaction().execute(async (trx) => { + if (backup.trashItemId) { + await trx.deleteFrom("trash_item").where("id", "=", backup.trashItemId).execute(); + } + await trx + .insertInto("trash_item") + .values({ + id: old.trashId, + root_id: `lib:${old.rootId}`, + library_root_id: old.rootId, + root_host_path: old.rootPath, + original_path: old.path, + trash_path: rejected, + is_dir: false, + size_bytes: pf.size, + actor: "system:organize", + deleted_at: d.ports.now(), + expires_at: new Date(d.ports.now().getTime() + 7 * 86_400_000), + asset_snapshot: jsonb({ source: "rejected_replacement", recordId }), + }) + .onConflict((oc) => oc.column("id").doNothing()) + .execute(); + await trx + .updateTable("asset_operation") + .set({ + result: jsonb({ + ...result, + rollback: true, + replaced: { ...backup, disposition: "restored" }, + }), + }) + .where("operation_id", "=", opId) + .execute(); + }); + } catch (error) { + await upsertDecision(d, d.ctx.db, { + kind: "other", + title: "洗版旧文件恢复待核对", + summary: error instanceof Error ? error.message : String(error), + subjectType: "media_asset", + subjectId: old.assetId, + dedupeKey: `replacement_restore:${recordId}:${old.locationId}`, + evidence: { recordId, old, backup }, + }); + } + } +} + +export async function stagedReplacementBackup(d: AoDeps, file: ReplacementFile): Promise { + const stage = join(file.rootPath, ".umt-trash", "replacement-staging", file.trashId); + for (const dir of await readdir(stage).catch(() => [] as string[])) { + const candidate = join(stage, dir, basename(file.path)); + const st = await fsRuntime(d).proxy.statSafe(candidate); + if ( + st.kind === "ok" && + !st.facts.isSymbolicLink && + Object.entries(file.facts).every(([key, value]) => st.facts[key as keyof typeof file.facts] === value) + ) + return candidate; + } + return null; +} diff --git a/apps/server/src/modules/acquisition-organize/services/scrape-input.ts b/apps/server/src/modules/acquisition-organize/services/scrape-input.ts index abe9ba14..84857c27 100644 --- a/apps/server/src/modules/acquisition-organize/services/scrape-input.ts +++ b/apps/server/src/modules/acquisition-organize/services/scrape-input.ts @@ -23,7 +23,13 @@ const credits = (v: unknown): Credit[] => (Array.isArray(v) ? (v as Credit[]) : const person = (c: Credit): NfoPerson | null => { const name = str(c.name); if (!name) return null; - return { tmdbId: numOf(c.id), name, role: str(c.role), profilePath: str(c.profilePath) }; + const id = numOf(c.id); + return { + tmdbId: id !== null && Number.isInteger(id) && id > 0 ? id : null, + name, + role: str(c.role), + profilePath: str(c.profilePath), + }; }; /** 制片国:ISO 3166-1 代码 → 中文名(Intl);认不出的保留原码 */ @@ -132,7 +138,10 @@ export function nfoEpisode(season: number, episode: number, u: UnitForNfo | unde /** 图片路径:作品海报 / 背景图,季海报,集剧照 */ export function imagePathsOf(entityMeta: unknown) { - return { posterPath: str(val(entityMeta, "posterPath")), backdropPath: str(val(entityMeta, "backdropPath")) }; + return { + posterPath: str(val(entityMeta, "posterPath")) ?? str(val(entityMeta, "posterUrl")), + backdropPath: str(val(entityMeta, "backdropPath")), + }; } export const unitPosterPath = (m: unknown) => str(val(m, "posterPath")); export const unitStillPath = (m: unknown) => str(val(m, "stillPath")); diff --git a/apps/server/src/modules/acquisition-organize/services/scrape.ts b/apps/server/src/modules/acquisition-organize/services/scrape.ts index 6db75878..8a00a7ee 100644 --- a/apps/server/src/modules/acquisition-organize/services/scrape.ts +++ b/apps/server/src/modules/acquisition-organize/services/scrape.ts @@ -170,51 +170,75 @@ export async function queueScrape( (cur.media_entity_id ?? null) === input.match.mediaEntityId && ((cur.match as ScrapeMatch | null)?.externalId ?? null) === input.match.externalId; if (cur && sameMatch && cur.engine_run_id) return { recordId: cur.id, taskId: cur.engine_run_id, deduplicated: true }; - const rec = await db - .insertInto("scrape_record") - .values({ - root_id: input.rootId, - root_path: input.rootPath, - library_root_id: input.libraryRootId, - path: input.path, - media_entity_id: input.match.mediaEntityId, - match: jsonb(input.match), - status: "queued", - trigger: input.trigger, - mode, - organization_record_id: input.organizationRecordId ?? null, - actor_type: input.actor?.type ?? (input.trigger === "auto" ? "system" : null), - actor_id: input.actor?.id ?? null, - }) - .returning("id") - .executeTakeFirstOrThrow(); + const reused = !!cur && sameMatch; + const rec = reused + ? cur + : await db + .insertInto("scrape_record") + .values({ + root_id: input.rootId, + root_path: input.rootPath, + library_root_id: input.libraryRootId, + path: input.path, + media_entity_id: input.match.mediaEntityId, + match: jsonb(input.match), + status: "queued", + trigger: input.trigger, + mode, + organization_record_id: input.organizationRecordId ?? null, + actor_type: input.actor?.type ?? (input.trigger === "auto" ? "system" : null), + actor_id: input.actor?.id ?? null, + }) + .returning("id") + .executeTakeFirstOrThrow(); + let queued: { runId: string; deduplicated: boolean }; try { - const { runId, deduplicated } = await d.ctx.engine.enqueue( + queued = await d.ctx.engine.enqueue( SCRAPE_TASK, { recordId: rec.id }, { idempotencyKey: input.idempotencyKey ?? `${SCRAPE_TASK}:${rec.id}`, traceId: randomUUID() }, ); - if (deduplicated) { - // 同一幂等键已排过(自动刮削重放):这条记录作废,返回已有的 - await db.deleteFrom("scrape_record").where("id", "=", rec.id).execute(); - const prev = await db - .selectFrom("scrape_record") - .select(["id"]) - .where("engine_run_id", "=", runId) - .executeTakeFirst(); - return { recordId: prev?.id ?? rec.id, taskId: runId, deduplicated: true }; - } - await db.updateTable("scrape_record").set({ engine_run_id: runId }).where("id", "=", rec.id).execute(); - return { recordId: rec.id, taskId: runId, deduplicated: false }; } catch (err) { - await db - .updateTable("scrape_record") - .set({ status: "failed", reason: "task_enqueue_failed", finished_at: new Date() }) - .where("id", "=", rec.id) - .execute() - .catch(() => {}); + // 只有新记录的入队失败才标失败;复用的记录可能已有接受但未关联的 run。 + if (!reused) + await db + .updateTable("scrape_record") + .set({ status: "failed", reason: "task_enqueue_failed", finished_at: new Date() }) + .where("id", "=", rec.id) + .where("status", "=", "queued") + .execute() + .catch(() => {}); throw err; } + const { runId, deduplicated } = queued; + let recordId = rec.id; + if (deduplicated) { + // 自动刮削重放时,返回引擎真正持有的记录;确认之前不能删除新记录。 + const prev = await db + .selectFrom("scrape_record") + .select("id") + .where("engine_run_id", "=", runId) + .executeTakeFirst(); + if (prev) recordId = prev.id; + else { + const run = await d.ctx.engine.get(runId); + const original = (run?.input as { recordId?: unknown } | undefined)?.recordId; + if (typeof original !== "string") throw new Error(`刮削任务缺少记录:${runId}`); + const existing = await db.selectFrom("scrape_record").select("id").where("id", "=", original).executeTakeFirst(); + if (!existing) throw new Error(`刮削记录不存在:${original}`); + recordId = existing.id; + } + } + try { + await db.transaction().execute(async (trx) => { + await trx.updateTable("scrape_record").set({ engine_run_id: runId }).where("id", "=", recordId).execute(); + if (recordId !== rec.id) await trx.deleteFrom("scrape_record").where("id", "=", rec.id).execute(); + }); + } catch (err) { + // run 已被接受,保持 queued;处理器开始时补回关联,重试也复用原记录/幂等键。 + d.ctx.logger.warn({ recordId, runId, err: { message: errMessage(err) } }, "刮削已入队,记录关联暂未写回"); + } + return { recordId, taskId: runId, deduplicated }; } // ── 作品 → 目录(作品详情「重新刮削」、入库后自动刮削) ── @@ -267,7 +291,7 @@ export async function entityScrapeLocations( .selectFrom("media_asset as a") .innerJoin("asset_location as l", "l.media_asset_id", "a.id") .innerJoin("library_root as r", "r.id", "l.library_root_id") - .select(["l.relative_path", "r.id as root_id", "r.local_path"]) + .select(["a.kind", "l.relative_path", "r.id as root_id", "r.local_path"]) .where("a.media_entity_id", "=", entityId) .where("a.status", "=", "active") .where("a.kind", "in", ["video", "bluray_dir"]) @@ -291,7 +315,10 @@ export async function entityScrapeLocations( !!b.relative_path && (l.relative_path === b.relative_path || l.relative_path.startsWith(`${b.relative_path}/`)), ); - const path = bound?.relative_path ?? inferWorkDir(kind, l.relative_path) ?? l.relative_path; + // 原盘目录本身就是作品目录(MP:BDMV 只对原盘根目录刮削) + const path = + bound?.relative_path ?? + (l.kind === "bluray_dir" ? l.relative_path : (inferWorkDir(kind, l.relative_path) ?? l.relative_path)); const key = `${l.root_id}\u0000${path}`; if (!out.has(key)) out.set(key, { rootId: `lib:${l.root_id}`, rootPath: l.local_path, libraryRootId: l.root_id, path }); @@ -335,12 +362,53 @@ export async function scrapeEntity( return out; } -/** 整理入库成功后:设置打开时对作品目录自动刮削(失败只记日志,不影响整理结果) */ -export async function scrapeAfterImport(d: AoDeps, entityId: string | null, recordId: string): Promise { +/** 整理入库成功后:设置打开或本次 Local 补全要求写本地元数据时,复用作品目录刮削任务。 */ +export async function scrapeAfterImport( + d: AoDeps, + entityId: string | null, + recordId: string, + opts: { force?: boolean; enabled?: boolean | undefined } = {}, +): Promise { if (!entityId) return; + if (opts.enabled === false) return; try { const s = await scrapeSettings(d); - if (!s.scrapeAfterImport) return; + if (!s.scrapeAfterImport && !opts.force && opts.enabled !== true) return; + const organization = await d.ctx.db + .selectFrom("organization_record") + .select(["action", "target_library_root_id"]) + .where("id", "=", recordId) + .executeTakeFirst(); + if (organization?.action === "in_place" && organization.target_library_root_id) { + const match = await entityScrapeMatch(d, entityId, "library"); + const root = await d.ctx.db + .selectFrom("library_root") + .selectAll() + .where("id", "=", organization.target_library_root_id) + .executeTakeFirst(); + if (!match || !root || root.management_mode !== "managed") return; + const files = await d.ctx.db + .selectFrom("organization_file_result") + .select("target_path") + .where("organization_record_id", "=", recordId) + .where("role", "=", "main") + .where("status", "=", "succeeded") + .orderBy("ordinal") + .execute(); + for (const file of files) + if (file.target_path) + await queueScrape(d, { + rootId: `lib:${root.id}`, + rootPath: root.local_path, + libraryRootId: root.id, + path: file.target_path, + match, + trigger: "auto", + organizationRecordId: recordId, + idempotencyKey: `${SCRAPE_TASK}:record:${recordId}:${file.target_path}`, + }); + return; + } await scrapeEntity(d, entityId, { trigger: "auto", organizationRecordId: recordId }); } catch (err) { d.ctx.logger.warn({ entityId, recordId, err: { message: errMessage(err) } }, "入库后自动刮削没排上"); @@ -469,7 +537,12 @@ interface Fields { episodeUnit: Map; } -export async function runScrapeRecord(d: AoDeps, recordId: string, signal?: AbortSignal): Promise { +export async function runScrapeRecord( + d: AoDeps, + recordId: string, + signal?: AbortSignal, + runId?: string, +): Promise { const db = d.ctx.db; const rec = await db.selectFrom("scrape_record").selectAll().where("id", "=", recordId).executeTakeFirst(); if (!rec) throw new Error(`刮削记录不存在:${recordId}`); @@ -477,7 +550,7 @@ export async function runScrapeRecord(d: AoDeps, recordId: string, signal?: Abor const now = () => d.ports.now(); await db .updateTable("scrape_record") - .set({ status: "running", started_at: now(), reason: null }) + .set({ status: "running", started_at: now(), reason: null, ...(runId ? { engine_run_id: runId } : {}) }) .where("id", "=", recordId) .execute(); const settings = await scrapeSettings(d); diff --git a/apps/server/src/modules/acquisition-organize/services/subscriptions.ts b/apps/server/src/modules/acquisition-organize/services/subscriptions.ts index 74ee39a2..25577a09 100644 --- a/apps/server/src/modules/acquisition-organize/services/subscriptions.ts +++ b/apps/server/src/modules/acquisition-organize/services/subscriptions.ts @@ -5,6 +5,7 @@ import { episodeKey, movieCompletion, type ParsedRelease, + parseRelease, parseUnitKey, priorityLadder, type QualityFacts, @@ -319,8 +320,19 @@ export async function slotComplete( >, mediaType: "movie" | "tv", newAssets: readonly string[], + opts: { includeSubmitted?: boolean } = {}, ): Promise { const completion = (slot.completion ?? {}) as { kind?: string; cuts?: string[] }; + const accepted = opts.includeSubmitted + ? await exec + .selectFrom("acquisition_plan as p") + .innerJoin("source_release as r", "r.id", "p.source_release_id") + .select(["p.id as plan_id", "p.unit_keys", "r.parsed", "r.title"]) + .where("p.slot_id", "=", slot.id) + .where("p.status", "in", ["submitted", "completed"]) + .where("p.status_reason", "!=", "accepted_unconfirmed") + .execute() + : []; if (mediaType === "movie") { const assets = newAssets.length ? await exec @@ -329,14 +341,20 @@ export async function slotComplete( .where("id", "in", [...newAssets]) .execute() : []; - const complete = movieCompletion( - completion.cuts as never, - assets.map((a) => ({ + const complete = movieCompletion(completion.cuts as never, [ + ...assets.map((a) => ({ assetId: a.id, verified: true, ...(a.content_edition ? { cut: a.content_edition as never } : {}), })), - ).complete; + ...accepted.map((p) => ({ + assetId: p.plan_id, + verified: true, + ...(((p.parsed as { cut?: string } | null)?.cut ?? parseRelease(p.title).cut) + ? { cut: ((p.parsed as { cut?: string } | null)?.cut ?? parseRelease(p.title).cut) as never } + : {}), + })), + ]).complete; return { complete, seasons: [], suspectedMissing: [] }; } const sub = slot.subscription_id @@ -357,6 +375,12 @@ export async function slotComplete( const u = parseUnitKey(k); if (u?.episode !== undefined) inflightEps.set(u.season, [...(inflightEps.get(u.season) ?? []), u.episode]); } + for (const p of accepted) + for (const key of p.unit_keys) { + const unit = parseUnitKey(key); + if (unit?.episode !== undefined) + covered.episodes.set(unit.season, new Set([...(covered.episodes.get(unit.season) ?? []), unit.episode])); + } const ledgers = await seasonLedgers(exec, slot.media_entity_id); const seasons = await scopeSeasons(exec, slot, sub, covered.episodes, ledgers); const range = rangeOf(sub); @@ -490,7 +514,25 @@ export async function afterSlotVerified( .executeTakeFirst(); const seasonDone = sub?.media_kind !== "movie" && sub?.season_scope === "current_season"; if (sub && sub.status === "active" && (sub.mode === "once" || seasonDone)) { - await completeSubscription(d, trx, sub, "slot_satisfied", { seasons: input.completion.seasons }); + const siblings = await trx + .selectFrom("acquisition_slot") + .selectAll() + .where("subscription_id", "=", sub.id) + .where("id", "!=", slot.id) + .execute(); + const others = await Promise.all( + siblings.map((s) => + slotComplete( + trx, + s, + sub.media_kind === "movie" ? "movie" : "tv", + s.current_asset_id ? [s.current_asset_id] : [], + { includeSubmitted: true }, + ), + ), + ); + if (others.every((p) => p.complete)) + await completeSubscription(d, trx, sub, "slot_satisfied", { seasons: input.completion.seasons }); } } // 计划结束:Slot 到期重评(忙时延后的发布、连续更新的下一集) @@ -856,3 +898,46 @@ export async function subscriptionProgress(exec: Exec, subId: string): Promise 0 && seasons.every((s) => s.totalKnown))); return { subscriptionId: sub.id, status: sub.status, completable, seasons, slots: slotOut }; } + +/** MP ordinary subscription progress is settled on confirmed downloader acceptance. + * Reserved/uncertain tasks and season wildcard claims never count as covered episodes. + */ +export async function afterDownloadAccepted(d: AoDeps, planId: string): Promise { + await d.ctx.db.transaction().execute(async (trx) => { + const plan = await trx + .selectFrom("acquisition_plan") + .select(["slot_id"]) + .where("id", "=", planId) + .executeTakeFirst(); + if (!plan?.slot_id) return; + const slot = await trx.selectFrom("acquisition_slot").selectAll().where("id", "=", plan.slot_id).executeTakeFirst(); + if (!slot?.subscription_id) return; + const sub = await trx + .selectFrom("subscription") + .selectAll() + .where("id", "=", slot.subscription_id) + .forUpdate() + .executeTakeFirst(); + if (sub?.status !== "active") return; + await touchProgress(trx, { subscriptionId: sub.id }, d.ports.now()); + if (sub.mode !== "once" && sub.season_scope !== "current_season") return; + const slots = await trx.selectFrom("acquisition_slot").selectAll().where("subscription_id", "=", sub.id).execute(); + if (!slots.length) return; + const progress = await Promise.all( + slots.map((s) => + slotComplete( + trx, + s, + sub.media_kind === "movie" ? "movie" : "tv", + s.current_asset_id ? [s.current_asset_id] : [], + { includeSubmitted: true }, + ), + ), + ); + if (progress.every((p) => p.complete)) + await completeSubscription(d, trx, sub, "download_accepted", { + slots: progress, + seasons: progress.flatMap((p) => p.seasons), + }); + }); +} diff --git a/apps/server/src/modules/agent/authorize.ts b/apps/server/src/modules/agent/authorize.ts index c9a7b374..1dc8f1c9 100644 --- a/apps/server/src/modules/agent/authorize.ts +++ b/apps/server/src/modules/agent/authorize.ts @@ -55,8 +55,10 @@ export async function planBinding(db: Kysely, planType: string, planId: stri if (planType === "acquisition_plan") { const r = await db .selectFrom("acquisition_plan as p") - .innerJoin("acquisition_slot as s", "s.id", "p.slot_id") - .innerJoin("media_entity as e", "e.id", "s.media_entity_id") + .leftJoin("acquisition_slot as s", "s.id", "p.slot_id") + .leftJoin("media_entity as e", (j) => + j.onRef("e.id", "=", sql`coalesce(s.media_entity_id, (p.evaluation->>'mediaEntityId')::uuid)`), + ) .select(["p.payload_hash", "p.expected_revision", "e.id as entity_id", "e.kind", "p.reserved_bytes"]) .where("p.id", "=", planId) .executeTakeFirst(); diff --git a/apps/server/src/modules/agent/capabilities-download.ts b/apps/server/src/modules/agent/capabilities-download.ts index edcc0ba5..05cb0f33 100644 --- a/apps/server/src/modules/agent/capabilities-download.ts +++ b/apps/server/src/modules/agent/capabilities-download.ts @@ -3,6 +3,7 @@ import { CapabilityError, type CapabilityRegistry } from "@umt/agent"; import { capabilitySpec, type DownloadStatus } from "@umt/contracts"; import type { DownloaderInstanceRow } from "@umt/db"; import type { ServerContext } from "../types.ts"; +import { planBinding } from "./authorize.ts"; import type { AgentModuleOptions } from "./runner.ts"; // download.recheck(REQ §8.4.2 / §10.7 授权内恢复):对已纳管的下载任务发起 qB 强制重新校验或恢复暂停。 @@ -29,15 +30,8 @@ async function checkGrantScope(ctx: ServerContext, grantId: string, planId: stri const g = await ctx.db.selectFrom("automation_grant").select("scope").where("id", "=", grantId).executeTakeFirst(); const scope = (g?.scope ?? {}) as { mediaEntityIds?: string[]; mediaKinds?: string[] }; if (!scope.mediaEntityIds?.length && !scope.mediaKinds?.length) return; - const e = planId - ? await ctx.db - .selectFrom("acquisition_plan as p") - .innerJoin("acquisition_slot as s", "s.id", "p.slot_id") - .innerJoin("media_entity as e", "e.id", "s.media_entity_id") - .select(["e.id", "e.kind"]) - .where("p.id", "=", planId) - .executeTakeFirst() - : undefined; + const binding = planId ? await planBinding(ctx.db, "acquisition_plan", planId) : null; + const e = binding?.mediaEntityId && binding.mediaKind ? { id: binding.mediaEntityId, kind: binding.mediaKind } : null; if (!e) throw new CapabilityError("unauthorized", `授权 ${grantId} 限定了对象范围,但下载任务没有关联作品`); if (scope.mediaEntityIds?.length && !scope.mediaEntityIds.includes(e.id)) throw new CapabilityError("unauthorized", `授权 ${grantId} 的对象范围不含目标作品`); diff --git a/apps/server/src/modules/agent/capabilities-metadata.ts b/apps/server/src/modules/agent/capabilities-metadata.ts index f384970e..5230bfb6 100644 --- a/apps/server/src/modules/agent/capabilities-metadata.ts +++ b/apps/server/src/modules/agent/capabilities-metadata.ts @@ -16,6 +16,7 @@ import { } from "@umt/contracts"; import { pgCacheFor } from "@umt/db"; import { resolveRunUpstream } from "../pi/index.ts"; +import { SourcesRuntime } from "../sources-identity/runtime.ts"; import { AUTO_ENRICH_ACTOR, type EnrichDeps, @@ -24,6 +25,7 @@ import { suggestEnrichment, } from "../sources-identity/service/enrich.ts"; import { robotsAllows } from "../sources-identity/service/enrich-sources.ts"; +import { resolveOrganizeMetadata } from "../sources-identity/service/organize-metadata.ts"; import type { ServerContext } from "../types.ts"; import { agentSettings } from "./runner.ts"; @@ -129,6 +131,37 @@ export function registerMetadataCapabilities( ctx: ServerContext, opts: MetadataCapabilityOptions = {}, ): void { + registry.register(capabilitySpec("metadata.resolve_organize"), async (input, c) => { + const sub = (scope: string) => ({ + actorId: scope, + taskId: c.taskId, + scopeIds: [], + signal: c.signal, + source: "task" as const, + }); + const deps: EnrichDeps = { + async search(query, scope) { + const out = (await registry.invoke("web.search", { query }, sub(scope))) as WebSearchOutput; + return out.results.map((r) => ({ title: r.title || null, url: r.url, snippet: r.snippet || null })); + }, + async fetch(url, scope) { + const p = (await registry.invoke("web.fetch", { url }, sub(scope))) as WebFetchOutput; + return { url: p.url, finalUrl: p.finalUrl, title: p.title, text: p.text, image: p.image ?? null }; + }, + complete: opts.complete ?? ((system, user, signal) => upstreamComplete(ctx, system, user, signal)), + robotsAllowed: (url, signal) => robotsAllowedFor(ctx, url, { ...opts, ...(signal ? { signal } : {}) }), + }; + try { + return await resolveOrganizeMetadata(new SourcesRuntime(ctx), input.sourceReleaseId, deps, { + actor: { type: "system", id: c.actorId }, + signal: c.signal, + }); + } catch (e) { + if (c.signal.aborted) throw e; + // Budget/provider errors leave the normal identity decision pending for manual resolution. + return { mediaEntityId: null, mode: null, enrichmentId: null, reason: (e as Error).message.slice(0, 500) }; + } + }); registry.register(capabilitySpec("metadata.enrich"), async (input, c) => { // 网页搜索没开就不占预算 const agent = await ctx.settings.get("agent").catch(() => null); diff --git a/apps/server/src/modules/agent/interaction.ts b/apps/server/src/modules/agent/interaction.ts index 03f1441e..e282b74b 100644 --- a/apps/server/src/modules/agent/interaction.ts +++ b/apps/server/src/modules/agent/interaction.ts @@ -258,14 +258,16 @@ async function actionDetail(db: Kysely, planType: string, planId: string): P if (planType === "acquisition_plan") { const r = await db .selectFrom("acquisition_plan as p") - .innerJoin("acquisition_slot as s", "s.id", "p.slot_id") - .innerJoin("media_entity as e", "e.id", "s.media_entity_id") + .leftJoin("acquisition_slot as s", "s.id", "p.slot_id") + .leftJoin("media_entity as e", (j) => + j.onRef("e.id", "=", sql`coalesce(s.media_entity_id, (p.evaluation->>'mediaEntityId')::uuid)`), + ) .select(["e.title as entity", "p.reserved_bytes"]) .where("p.id", "=", planId) .executeTakeFirst(); if (!r) return null; const gb = Number(r.reserved_bytes ?? 0) / 1024 ** 3; - return safe(`《${r.entity}》${gb > 0 ? `· ${gb.toFixed(1)} GB` : ""}`, 200); + return safe(`《${r.entity ?? "未识别资源"}》${gb > 0 ? `· ${gb.toFixed(1)} GB` : ""}`, 200); } if (planType === "organize_plan") { const r = await db.selectFrom("artifact").select("content").where("id", "=", planId).executeTakeFirst(); @@ -471,15 +473,20 @@ async function planSummary(db: Kysely, planType: string, planId: string): Pr if (planType === "acquisition_plan") { const r = await db .selectFrom("acquisition_plan as p") - .innerJoin("acquisition_slot as s", "s.id", "p.slot_id") - .innerJoin("media_entity as e", "e.id", "s.media_entity_id") + .leftJoin("acquisition_slot as s", "s.id", "p.slot_id") + .leftJoin("media_entity as e", (j) => + j.onRef("e.id", "=", sql`coalesce(s.media_entity_id, (p.evaluation->>'mediaEntityId')::uuid)`), + ) .leftJoin("source_release as r", "r.id", "p.source_release_id") .select(["e.title as entity", "r.title as release", "p.reserved_bytes"]) .where("p.id", "=", planId) .executeTakeFirst(); if (r) { const gb = Number(r.reserved_bytes ?? 0) / 1024 ** 3; - return safe(`下载《${r.entity}》:${r.release ?? "资源"}${gb > 0 ? `(${gb.toFixed(1)} GB)` : ""}`, 300); + return safe( + `下载《${r.entity ?? "未识别资源"}》:${r.release ?? "资源"}${gb > 0 ? `(${gb.toFixed(1)} GB)` : ""}`, + 300, + ); } } if (planType === "organize_plan") { diff --git a/apps/server/src/modules/agent/routes.ts b/apps/server/src/modules/agent/routes.ts index f1962864..8d1c6341 100644 --- a/apps/server/src/modules/agent/routes.ts +++ b/apps/server/src/modules/agent/routes.ts @@ -4,12 +4,12 @@ import { Api, CAPABILITY_CATALOG, type CapabilityDescriptor, - type DownloadPlanOutput, type KnownCapabilityId, } from "@umt/contracts"; import { type AgentRunRow, sql } from "@umt/db"; import { hasScope } from "../../http/auth.ts"; import { ApiException } from "../../http/errors.ts"; +import { planManualAcquisition } from "../acquisition-organize/services/manual-acquisition.ts"; import type { ModuleApp, ServerContext } from "../types.ts"; import { planBinding } from "./authorize.ts"; import { cancelTask, retryTask, taskToApi } from "./capabilities.ts"; @@ -426,73 +426,48 @@ export function registerAgentRoutes( } }); - // ── 手动提交下载:同一条 download.plan → download.submit 能力路径(授权/审计/幂等与 Agent 一致)── + // 手动资源选择不依赖订阅,提交后仍走共同下载及整理任务。 route(app, Api.acquisitionPlans.create, async ({ body, actorId, c }) => { const rel = await db() .selectFrom("source_release") - .select(["id", "media_entity_id"]) + .select("media_entity_id") .where("id", "=", body.sourceReleaseId) .executeTakeFirst(); - if (!rel) throw new ApiException("not_found", "资源不存在"); - let slotIds: string[]; - if (body.slotId) { - const s = await db().selectFrom("acquisition_slot").select("id").where("id", "=", body.slotId).executeTakeFirst(); - if (!s) throw new ApiException("not_found", "收录版本不存在"); - slotIds = [s.id]; - } else { - const entityId = body.entityId ?? rel.media_entity_id; - if (!entityId) - throw new ApiException("precondition_failed", "资源还没有关联本地作品,先在「身份与元数据」指定作品"); - slotIds = ( - await db() - .selectFrom("acquisition_slot") - .select("id") - .where("media_entity_id", "=", entityId) - .orderBy("created_at") - .execute() - ).map((r) => r.id); - } - const rejected: { slotId: string | null; code: string; message: string }[] = []; - if (slotIds.length === 0) - rejected.push({ slotId: null, code: "no_slot", message: "这个作品没有收录版本,先在订阅里新建" }); - const invokeCtx = { - actorId, - taskId: `http:${c.get("requestId") ?? "req"}`, - scopeIds: [], - signal: c.req.raw.signal, - source: "web" as const, - }; - try { - for (const slotId of slotIds) { - const out = (await ctx.capabilities.invoke( - "download.plan", - { slotId, sourceReleaseId: rel.id }, - invokeCtx, - )) as DownloadPlanOutput; - if (!out.plan) { - for (const r of out.rejected) rejected.push({ slotId, ...r }); - continue; - } - const sub = (await ctx.capabilities.invoke("download.submit", { planId: out.plan.id }, invokeCtx)) as { - taskId: string; - }; - const plan = await db() - .selectFrom("acquisition_plan") - .selectAll() - .where("id", "=", out.plan.id) - .executeTakeFirstOrThrow(); - await audit(ctx, actorId, "acquisition_plan.manual_submit", plan.id, { - sourceReleaseId: rel.id, - slotId, - taskId: sub.taskId, - note: body.note ?? null, - }); - return { submitted: true, plan: camel(plan), slotId, taskId: sub.taskId, rejected }; + if ( + rel && + !rel.media_entity_id && + !body.entityId && + !body.allowUnrecognized && + ctx.capabilities.get("identity.resolve") + ) { + try { + await ctx.capabilities.invoke( + "identity.resolve", + { sourceReleaseId: body.sourceReleaseId }, + { + actorId, + taskId: `http:${c.get("requestId") ?? "req"}`, + scopeIds: [], + signal: c.req.raw.signal, + source: "web", + }, + ); + } catch (err) { + ctx.logger.warn( + { sourceReleaseId: body.sourceReleaseId, error: err instanceof Error ? err.message : String(err) }, + "手动下载识别未完成,允许确认后下载", + ); } - } catch (err) { - capabilityHttpError(err); } - return { submitted: false, plan: null, slotId: null, taskId: null, rejected }; + const result = await planManualAcquisition(ctx, body); + if (result.plan) + await audit(ctx, actorId, "acquisition_plan.manual_submit", result.plan.id, { + sourceReleaseId: body.sourceReleaseId, + taskId: result.taskId, + importMode: body.importMode, + note: body.note ?? null, + }); + return result; }); // ── 产物(整理计划 / 汇报 / 证据):只读 ── diff --git a/apps/server/src/modules/auth-settings/engine-routes.ts b/apps/server/src/modules/auth-settings/engine-routes.ts index 5d994a63..d0130460 100644 --- a/apps/server/src/modules/auth-settings/engine-routes.ts +++ b/apps/server/src/modules/auth-settings/engine-routes.ts @@ -128,7 +128,13 @@ export function engineQueries(db: Kysely, schema = "openworkflow"): EngineQu const cur: Cursor | null = decodeCursor(q.cursor); const conds = [sql`namespace_id = ${NAMESPACE}`]; if (q.status?.length) conds.push(sql`status = any(${q.status}::text[])`); - if (q.workflowName) conds.push(sql`workflow_name = ${q.workflowName}`); + if (q.workflowName) { + const workflows = q.workflowName + .split(",") + .map((name) => name.trim()) + .filter(Boolean); + conds.push(sql`workflow_name = any(${workflows}::text[])`); + } if (q.from) conds.push(sql`created_at >= ${new Date(q.from)}`); if (q.to) conds.push(sql`created_at < ${new Date(q.to)}`); conds.push(...businessConds(q)); diff --git a/apps/server/src/modules/auth-settings/passkey.ts b/apps/server/src/modules/auth-settings/passkey.ts index 0d8f29fe..cee474e3 100644 --- a/apps/server/src/modules/auth-settings/passkey.ts +++ b/apps/server/src/modules/auth-settings/passkey.ts @@ -1,6 +1,6 @@ import { type AuthenticationResponseJSON, - type AuthenticatorTransportFuture, + type AuthenticatorTransport, generateAuthenticationOptions, generateRegistrationOptions, type RegistrationResponseJSON, @@ -108,8 +108,8 @@ async function takeChallenge( // ── 登记的 Passkey ── -const transportsOf = (v: unknown): AuthenticatorTransportFuture[] => - Array.isArray(v) ? (v.filter((x) => typeof x === "string") as AuthenticatorTransportFuture[]) : []; +const transportsOf = (v: unknown): AuthenticatorTransport[] => + Array.isArray(v) ? (v.filter((x) => typeof x === "string") as AuthenticatorTransport[]) : []; export function toPasskeyInfo(r: AdminPasskeyRow): PasskeyInfo { return { diff --git a/apps/server/src/modules/auth-settings/settings-routes.ts b/apps/server/src/modules/auth-settings/settings-routes.ts index b4a0e9cd..6253d5c4 100644 --- a/apps/server/src/modules/auth-settings/settings-routes.ts +++ b/apps/server/src/modules/auth-settings/settings-routes.ts @@ -7,6 +7,8 @@ import { type SettingsKey, secretInput, } from "@umt/contracts"; +import { sql } from "@umt/db"; +import { parseClassificationYaml } from "@umt/domain"; import { ApiException } from "../../http/errors.ts"; import type { ModuleApp, ServerContext } from "../types.ts"; import { audit, endpoint } from "./endpoint.ts"; @@ -39,6 +41,13 @@ export function registerSettingsRoutes(app: ModuleApp, ctx: ServerContext) { throw new ApiException("validation_failed", msg); } const next = parsed.data; + if (patch.organize?.classificationYaml !== undefined) { + try { + parseClassificationYaml(next.organize.classificationYaml); + } catch (err) { + throw new ApiException("validation_failed", err instanceof Error ? err.message : String(err)); + } + } const created: string[] = []; const dropped: string[] = []; for (const sl of slots) { @@ -58,6 +67,20 @@ export function registerSettingsRoutes(app: ModuleApp, ctx: ServerContext) { try { await ctx.db.transaction().execute(async (trx) => { for (const key of keys) { + if (key === "organize") { + await sql`select pg_advisory_xact_lock(hashtext('organize.sources'))`.execute(trx); + const stored = await trx + .selectFrom("settings") + .select("value") + .where("key", "=", key) + .forUpdate() + .executeTakeFirst(); + next.organize = { + ...next.organize, + ...((stored?.value as Partial) ?? {}), + ...patch.organize, + }; + } await trx .insertInto("settings") .values({ key, value: JSON.stringify(next[key]), updated_at: new Date() }) diff --git a/apps/server/src/modules/browser/__tests__/browser-api.test.ts b/apps/server/src/modules/browser/__tests__/browser-api.test.ts index 813cc6bc..00d78ac5 100644 --- a/apps/server/src/modules/browser/__tests__/browser-api.test.ts +++ b/apps/server/src/modules/browser/__tests__/browser-api.test.ts @@ -142,14 +142,13 @@ describe.skipIf(!up)("浏览器 HTTP 端点", () => { expect(d.artifact.sourceUrl).not.toContain(PASSKEY); expect(d.verification).toMatchObject({ valid: true, infohashV1: h.torrent.infoHash, private: true }); expect(d.sourceReleaseId).toBe(h.releaseId); - // 资源未识别:与 POST /acquisition-plans 相同的拒绝,产物照常返回 - expect(d.submission).toBeNull(); - expect(d.note).toContain("提交未通过"); + // 资源未识别:与 POST /acquisition-plans 相同的确认结果,产物照常返回 + expect(d.submission).toMatchObject({ submitted: false, requiresConfirmation: true }); const files = await readdir(h.stagingDir); expect(files).toHaveLength(1); expect(new Uint8Array(await readFile(join(h.stagingDir, files[0]!)))).toEqual(h.torrent.bytes); - // 识别到作品后再下载:走同一流程(无收录版本 → no_slot);同一文件第二次捕获标 duplicate、不提交 + // 识别到作品后再下载:同一文件第二次捕获标 duplicate、不提交 const entity = await h.db .insertInto("media_entity") .values({ kind: "movie", title: "奥本海默", year: 2023, status: "active" }) diff --git a/apps/server/src/modules/browser/__tests__/browser-resource.test.ts b/apps/server/src/modules/browser/__tests__/browser-resource.test.ts new file mode 100644 index 00000000..64fdc6e1 --- /dev/null +++ b/apps/server/src/modules/browser/__tests__/browser-resource.test.ts @@ -0,0 +1,74 @@ +import { getSiteDefinition } from "@umt/adapters"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { resourceIdFromUrl } from "../resource.ts"; +import { type BrowserHarness, createBrowserHarness, ORIGIN, pgAvailable, READ_KEY } from "./harness.ts"; + +const def = getSiteDefinition("audiences")!; +describe("原站资源详情定位", () => { + it("仅按本站详情路径定位,不把列表、登录或别站同 id 当作资源", () => { + expect(resourceIdFromUrl(`${ORIGIN}/details.php?id=73`, `${ORIGIN}/`, def)).toBe("73"); + expect(resourceIdFromUrl(`${ORIGIN}/torrents.php?id=73`, `${ORIGIN}/`, def)).toBeNull(); + expect(resourceIdFromUrl(`${ORIGIN}/login.php?id=73`, `${ORIGIN}/`, def)).toBeNull(); + expect(resourceIdFromUrl("https://else.example/details.php?id=73", `${ORIGIN}/`, def)).toBeNull(); + }); +}); + +const up = await pgAvailable(); +describe.skipIf(!up)("原站详情页交接(隔离 PG 与浏览器 fixture)", () => { + let h: BrowserHarness; + let sid: string; + beforeAll(async () => { + h = await createBrowserHarness("resource", { + html: () => `资源详情 +

Example.Movie.2024.2160p.WEB-DL.DV

+ +
副标题测试译名与简介
基本信息大小: 2 GB 类型: 电影
+
详情页提供的影片简介 IMDb
+ `, + }); + h.mock.loggedIn.add(`umt-site-${h.siteId}-acct-${h.accountId}`); + const session = await h.req("/browser/sessions", { + body: { siteId: h.siteId, url: `${ORIGIN}/details.php?id=73` }, + }); + expect(session.status).toBe(200); + sid = session.body.data.id; + }, 120_000); + afterAll(async () => { + await h?.close(); + }); + + it("未在快照/搜索中的详情可直接登记,缓存PT内容供元数据兜底,不点击原站下载按钮", async () => { + const before = h.mock.actions.length; + const r = await h.req(`/browser/sessions/${sid}/resource`, { body: {} }); + expect(r.status, JSON.stringify(r.body)).toBe(200); + expect(r.body.data.release).toMatchObject({ + siteId: h.siteId, + siteReleaseId: "73", + title: "Example.Movie.2024.2160p.WEB-DL.DV", + }); + expect(r.body.data.detailUrl).toBe(`${ORIGIN}/details.php?id=73`); + const row = await h.db + .selectFrom("source_release") + .selectAll() + .where("id", "=", r.body.data.release.id) + .executeTakeFirstOrThrow(); + expect(row.detail_fetched_at).not.toBeNull(); + expect(row.raw).toMatchObject({ _detail: { imdbId: "tt1234567", sourceUrl: `${ORIGIN}/details.php?id=73` } }); + expect(h.mock.actions.slice(before).some((a) => a.action === "click")).toBe(false); + const again = await h.req(`/browser/sessions/${sid}/resource`, { body: {} }); + expect(again.body.data.release.id).toBe(row.id); + }); + + it("使用当前页面而非旧会话地址;列表页明确提示进入详情", async () => { + const tab = [...h.mock.tabs.values()][0]!; + tab.url = `${ORIGIN}/torrents.php`; + const r = await h.req(`/browser/sessions/${sid}/resource`, { body: {} }); + expect(r.status).toBe(400); + expect(r.body.error.message).toContain("详情页"); + }); + + it("只读账号不能通过该入口登记下载资源", async () => { + const r = await h.req(`/browser/sessions/${sid}/resource`, { body: {}, key: READ_KEY }); + expect(r.status).toBe(403); + }); +}); diff --git a/apps/server/src/modules/browser/resource.ts b/apps/server/src/modules/browser/resource.ts new file mode 100644 index 00000000..118d7340 --- /dev/null +++ b/apps/server/src/modules/browser/resource.ts @@ -0,0 +1,85 @@ +import { + getSiteDefinition, + type IndexerDefinition, + parseDetailPage, + type SiteDefinition, + type SiteReleaseDetail, +} from "@umt/adapters"; +import type { BrowserResourceResult } from "@umt/contracts"; +import { sql } from "@umt/db"; +import { saveDetailPage } from "../sources-identity/service/detail-page.ts"; +import { mergeDetailOverList, upsertSiteRelease } from "../sources-identity/service/snapshot.ts"; +import { releaseDetail } from "../sources-identity/service/workbench.ts"; +import { type Actor, type BrowserService, BrowserServiceError } from "./service.ts"; + +/** 详情页定位遵循站点定义;列表、登录等页面不能作为资源提交。 */ +export function resourceIdFromUrl(value: string, baseUrl: string, def: SiteDefinition): string | null { + const url = new URL(value); + const base = new URL(baseUrl); + if (url.origin !== base.origin) return null; + if (def.umt.kind === "nexusphp") { + const expected = new URL(def.umt.detailPath, base); + if (url.pathname !== expected.pathname) return null; + const id = url.searchParams.get("id"); + return id && /^\d{1,20}$/.test(id) ? id : null; + } + const match = /^\/detail\/(\d{1,20})\/?$/.exec(url.pathname); + return match?.[1] ?? null; +} + +export async function currentResource( + service: BrowserService, + sessionId: string, + actor: Actor, + tabId?: string, +): Promise { + const runtime = service.opts.runtime?.(service.ctx); + if (!runtime) throw new BrowserServiceError("precondition", "站点详情解析不可用"); + const current = await service.run( + sessionId, + actor, + async (provider, ref, context) => { + const page = provider.pageContent ? await provider.pageContent(ref, { currentVisible: true }) : null; + if (!page?.url) throw new BrowserServiceError("precondition", "无法读取当前页面,请重新连接浏览器"); + const def = getSiteDefinition(context.site.key); + if (!def) throw new BrowserServiceError("precondition", "这个站点还没有详情页解析配置"); + const resourceId = resourceIdFromUrl(page.url, context.site.base_url, def); + if (!resourceId) throw new BrowserServiceError("bad_request", "请先进入本站资源详情页,再点击下载"); + let detail: SiteReleaseDetail; + if (def.umt.kind === "nexusphp") { + const parsed = parseDetailPage(page.html, def as IndexerDefinition, resourceId, service.now().toISOString()); + if (parsed.kind === "login") throw new BrowserServiceError("precondition", "请先在原站登录"); + if (!parsed.detail) + throw new BrowserServiceError("precondition", "当前详情页未能解析出资源,请检查站点详情配置"); + detail = parsed.detail; + } else { + detail = await runtime.siteAdapter(context.site).detail(resourceId); + } + return { site: context.site, detail, url: page.url }; + }, + { write: false, ...(tabId ? { tabId } : {}) }, + ); + const old = await runtime.ctx.db + .selectFrom("source_release") + .selectAll() + .where("site_id", "=", current.site.id) + .where("site_release_id", "=", current.detail.siteReleaseId) + .executeTakeFirst(); + const { page, descriptionText: _description, ...plain } = current.detail; + const detail = old ? mergeDetailOverList(plain, old) : plain; + const { detailCacheSec } = await runtime.siteAccess(current.site.key); + const id = await runtime.ctx.db.transaction().execute(async (trx) => { + const row = await upsertSiteRelease(runtime, trx, current.site, detail); + await trx + .updateTable("source_release") + .set({ detail_fetched_at: runtime.now() }) + .where("id", "=", row.id) + .execute(); + if (page) await saveDetailPage(runtime, trx, row.id, page, detailCacheSec); + await sql`update source_release set raw = jsonb_set(raw, '{_detail}', + coalesce(raw->'_detail', '{}'::jsonb) || jsonb_build_object('sourceUrl', ${current.url}::text)) + where id = ${row.id}`.execute(trx); + return row.id; + }); + return { release: (await releaseDetail(runtime, current.site.id, id, false)).release, detailUrl: current.url }; +} diff --git a/apps/server/src/modules/browser/routes.ts b/apps/server/src/modules/browser/routes.ts index d1fe5e14..15b798e9 100644 --- a/apps/server/src/modules/browser/routes.ts +++ b/apps/server/src/modules/browser/routes.ts @@ -15,6 +15,7 @@ import { type AuthEnv, requireScope } from "../../http/auth.ts"; import { ApiException } from "../../http/errors.ts"; import { InternalDispatcher } from "../search/dispatch.ts"; import type { ModuleApp } from "../types.ts"; +import { currentResource } from "./resource.ts"; import { type Actor, type BrowserService, BrowserServiceError, imageSize } from "./service.ts"; // C1-E 浏览器 HTTP 端点:原站视图与外部程序共用。读(会话/快照/截图)任何时候可用; @@ -85,6 +86,8 @@ export function registerBrowserRoutes(app: ModuleApp, svc: () => BrowserService) const dispatch = new InternalDispatcher(app); const B = Api.browser; + route(app, B.resource, ({ params, body, c }) => currentResource(svc(), params.id, human(c), body.tabId)); + route(app, B.providers, async ({ query }) => ({ items: await svc().listProviders(query.probe === "true") })); route(app, B.createProvider, async ({ body }) => svc().createProvider(body)); route(app, B.updateProvider, async ({ params, body }) => svc().updateProvider(params.id, body)); diff --git a/apps/server/src/modules/file-manager/identify.ts b/apps/server/src/modules/file-manager/identify.ts index 6c2c5dfc..d669c73e 100644 --- a/apps/server/src/modules/file-manager/identify.ts +++ b/apps/server/src/modules/file-manager/identify.ts @@ -21,6 +21,7 @@ import { } from "@umt/domain"; import { localIdentityCandidate } from "@umt/mapping"; import type { Principal } from "../../http/auth.ts"; +import { successfulOrganizeCounts } from "../acquisition-organize/services/organize-history.ts"; import { EXTERNAL_INVOKE_TIMEOUT_MS, sharedInvokeLimiter } from "../agent/invoke-limits.ts"; import { actorOfP, errMessage, FilesError, type FmDeps, resolveIn } from "./common.ts"; import { kindOfName } from "./list.ts"; @@ -218,6 +219,10 @@ export async function identifyCore( order by f.source_path, r.created_at desc`.execute(db) : { rows: [] }; const recordsBy = new Map(recordRows.rows.map((r) => [r.source_path, r])); + const successfulCounts = await successfulOrganizeCounts( + db, + videos.map((v) => v.abs), + ); const localCache = new Map(); const remoteCache = new Map(); const entityKinds = new Map(); @@ -371,6 +376,7 @@ export async function identifyCore( count: rec?.n ?? 0, lastRecordId: rec?.last_id ?? null, lastStatus: (rec?.last_status ?? null) as OrganizationStatus | null, + successfulCount: successfulCounts.get(v.abs) ?? 0, }; const parsed0 = parseReleasePath(segments(v.rel)); const parsed: ParsedRelease = { diff --git a/apps/server/src/modules/file-manager/index.ts b/apps/server/src/modules/file-manager/index.ts index e8d416bf..0f99f2dd 100644 --- a/apps/server/src/modules/file-manager/index.ts +++ b/apps/server/src/modules/file-manager/index.ts @@ -10,7 +10,9 @@ import { SourcesRuntime } from "../sources-identity/runtime.ts"; import type { ServerContext, ServerModule } from "../types.ts"; import { defaultOptions, type FmDeps, type FmOptions } from "./common.ts"; import { IDENTIFY_REMOTE_TASK, runIdentifyRemote } from "./identify.ts"; +import { registerOrganizeSourceRoutes } from "./routes/organize-sources.ts"; import { registerFileRoutes } from "./routes.ts"; +import { queueDueSourceScans, scanOrganizeSource } from "./source-monitor.ts"; import { purgeTrash } from "./trash.ts"; /** @@ -52,9 +54,18 @@ export function createFileManagerModule(overrides: FmOverrides = {}): ServerModu }, registerRoutes(app, ctx) { registerFileRoutes(app, lazy(ctx)); + registerOrganizeSourceRoutes(app, lazy(ctx)); }, registerTasks(registry, ctx) { registry.register( + defineTask({ + name: "scanOrganizeSource", + version: 1, + lane: "maintenance", + inputSchema: z.object({ sourceId: z.string().uuid() }), + retryPolicy: { maximumAttempts: 3, initialInterval: "5s" }, + handler: async (t) => t.step.run("scan", () => scanOrganizeSource(lazy(ctx), t.input)), + }), defineTask({ name: "purgeTrash", version: 1, @@ -84,7 +95,18 @@ export function createFileManagerModule(overrides: FmOverrides = {}): ServerModu }, start(ctx) { const d = lazy(ctx); - if (!d.options.backgroundLoops || d.options.purgeIntervalMs <= 0) return; + if (!d.options.backgroundLoops) return; + registerScheduledJob(ctx, { + key: "organize.sources", + defaults: { + name: "整理来源扫描", + description: "检查来源目录的新文件", + group: "maintain", + schedule: { type: "interval", everyMs: 60_000 }, + }, + run: async (c) => queueDueSourceScans(d, c.slot), + }); + if (d.options.purgeIntervalMs <= 0) return; registerScheduledJob(ctx, { key: "trash.purge", defaults: { diff --git a/apps/server/src/modules/file-manager/organize-sources.ts b/apps/server/src/modules/file-manager/organize-sources.ts new file mode 100644 index 00000000..87224f15 --- /dev/null +++ b/apps/server/src/modules/file-manager/organize-sources.ts @@ -0,0 +1,112 @@ +import { randomUUID } from "node:crypto"; +import { constants } from "node:fs"; +import { access } from "node:fs/promises"; +import type { OrganizeSourceConfig } from "@umt/contracts"; +import { sql } from "@umt/db"; +import { SETTINGS_CHANGED_EVENT } from "../auth-settings/provided.ts"; +import { FilesError, type FmDeps, jsonb, resolveIn } from "./common.ts"; +import { isWritableTarget } from "./list.ts"; + +export async function sourceConfigs(d: FmDeps): Promise { + const org = await d.ctx.settings.get("organize"); + return (org as { sources?: OrganizeSourceConfig[] }).sources ?? []; +} +export async function sourceConfig(d: FmDeps, id: string): Promise { + const found = (await sourceConfigs(d)).find((s) => s.id === id); + if (!found) throw new FilesError("not_found", "整理来源不存在"); + return found; +} +const iso = (v: Date | string | null | undefined) => (v ? new Date(v).toISOString() : null); +export async function listOrganizeSources(d: FmDeps) { + const rows = await d.ctx.db.selectFrom("organize_source_snapshot").selectAll().execute(); + return { + items: (await sourceConfigs(d)).map((s) => { + const row = rows.find((r) => r.source_id === s.id); + const files = (row?.snapshot as { files?: Record } | undefined)?.files ?? {}; + return { + ...s, + baselineAt: iso(row?.baseline_at), + lastScanAt: iso(row?.last_scan_at), + lastError: row?.last_error ?? null, + fileCount: Object.keys(files).length, + }; + }), + }; +} + +async function editSources( + d: FmDeps, + edit: (items: OrganizeSourceConfig[]) => OrganizeSourceConfig[], + resetId?: string, + lockId = resetId, +) { + const defaults = await d.ctx.settings.get("organize"); + await d.ctx.db.transaction().execute(async (trx) => { + if (lockId) await sql`select pg_advisory_xact_lock(hashtext(${`organize-source:${lockId}`}))`.execute(trx); + // Serialize even the first insert, where no settings row exists to lock. + await sql`select pg_advisory_xact_lock(hashtext('organize.sources'))`.execute(trx); + const row = await trx + .selectFrom("settings") + .select("value") + .where("key", "=", "organize") + .forUpdate() + .executeTakeFirst(); + const org = (row?.value ?? defaults) as typeof defaults & { sources?: OrganizeSourceConfig[] }; + await trx + .insertInto("settings") + .values({ key: "organize", value: jsonb({ ...org, sources: edit(org.sources ?? []) }), updated_at: new Date() }) + .onConflict((oc) => + oc + .column("key") + .doUpdateSet({ value: jsonb({ ...org, sources: edit(org.sources ?? []) }), updated_at: new Date() }), + ) + .execute(); + if (resetId) await trx.deleteFrom("organize_source_snapshot").where("source_id", "=", resetId).execute(); + }); + const settings = d.ctx.settings as typeof d.ctx.settings & { invalidate?: () => void }; + settings.invalidate?.(); + await d.ctx.events.publish(SETTINGS_CHANGED_EVENT, { keys: ["organize"] }); +} + +export async function saveOrganizeSource(d: FmDeps, id: string, input: Omit) { + const source = await resolveIn(d, input.source.root, input.source.path); + if (source.isSymlink || !source.lst?.isDirectory()) throw new FilesError("bad_request", "来源必须为可访问的目录"); + const target = await resolveIn(d, input.target.root, input.target.path); + if ( + target.isSymlink || + !target.lst?.isDirectory() || + !(await isWritableTarget(target.def, target.rootReal)) || + !(await access(target.abs, constants.W_OK).then( + () => true, + () => false, + )) + ) + throw new FilesError("not_writable", "目标必须为可写的纳管媒体库目录"); + if (input.trigger === "qb_tag") { + if (!input.downloaderInstanceId || !input.tag?.trim()) + throw new FilesError("bad_request", "标签触发需要下载器和标签"); + const instance = await d.ctx.db + .selectFrom("downloader_instance") + .select("id") + .where("id", "=", input.downloaderInstanceId) + .executeTakeFirst(); + if (!instance || (source.def.instanceId && source.def.instanceId !== instance.id)) + throw new FilesError("bad_request", "来源和下载器不匹配"); + } + const previous = (await sourceConfigs(d)).find((s) => s.id === id); + const reset = !previous || previous.source.root !== input.source.root || previous.source.path !== input.source.path; + const config = { ...input, id }; + await editSources(d, (items) => [...items.filter((s) => s.id !== id), config], reset ? id : undefined, id); + if (reset) + await d.ctx.engine.enqueue( + "scanOrganizeSource", + { sourceId: id }, + { idempotencyKey: `source-baseline:${id}:${randomUUID()}`, traceId: randomUUID() }, + ); + return (await listOrganizeSources(d)).items.find((s) => s.id === id)!; +} +export async function deleteOrganizeSource(d: FmDeps, id: string) { + const deleted = (await sourceConfigs(d)).some((s) => s.id === id); + await editSources(d, (items) => items.filter((s) => s.id !== id), id); + return { deleted }; +} diff --git a/apps/server/src/modules/file-manager/organize.ts b/apps/server/src/modules/file-manager/organize.ts index 0975f712..3d126bd1 100644 --- a/apps/server/src/modules/file-manager/organize.ts +++ b/apps/server/src/modules/file-manager/organize.ts @@ -29,7 +29,7 @@ type Req = { sourcePaths: string[]; mediaEntityId: string; contentUnitId?: string | undefined; - action: "copy" | "move"; + action: "copy" | "move" | "in_place"; targetLibraryRootId?: string | undefined; }; @@ -45,7 +45,7 @@ interface Frozen { interface Group { entity: string; unit: string | null; - action: "copy" | "move"; + action: "copy" | "move" | "in_place"; targetDir: string | null; targetLibraryRootId: string | null; slotId: string | null; @@ -67,7 +67,7 @@ function filesOf(plan: FrozenPlan | undefined): GroupFile[] { const heldByExisting = new Set( plan.organize.conflicts.filter((c) => c.kind === "target_exists" && c.blocking).map((c) => c.src), ); - return plan.organize.files.map((f) => { + const active = plan.organize.files.map((f) => { const existing: GroupFile["existing"] = f.action === "replace" ? "replace" @@ -88,16 +88,36 @@ function filesOf(plan: FrozenPlan | undefined): GroupFile[] { skipReason: f.skipReason ?? null, }; }); + return [ + ...active, + ...(plan.skippedSuccessful ?? []).map((f) => ({ + source: f.src, + target: f.dst ?? null, + kind: f.kind, + size: f.size, + action: "skip" as const, + existing: null, + skipReason: "successful_record", + })), + ]; } /** 被排除的文件(G2:too_small → below_min_size) */ function excludedOf(plan: FrozenPlan | undefined): NonNullable { - return (plan?.organize.excluded ?? []).map((x) => ({ - source: x.src, - kind: x.kind, - size: x.reason === "too_small" && x.detail && /^\d+$/.test(x.detail) ? Number(x.detail) : null, - reason: x.reason === "too_small" ? "below_min_size" : x.reason, - })); + return [ + ...(plan?.organize.excluded ?? []).map((x) => ({ + source: x.src, + kind: x.kind, + size: x.reason === "too_small" && x.detail && /^\d+$/.test(x.detail) ? Number(x.detail) : null, + reason: x.reason === "too_small" ? "below_min_size" : x.reason, + })), + ...(plan?.skippedSuccessful ?? []).map((f) => ({ + source: f.src, + kind: f.kind, + size: f.size, + reason: "successful_record", + })), + ]; } /** 目标目录分段(相对库根;G4):剧集末段为季目录;bound = 沿用已有路径绑定(本次不新建) */ @@ -105,7 +125,8 @@ function segmentsOf(plan: FrozenPlan | undefined, overrideDir: boolean): FilesOr const dir = plan?.targetDirs[0]; if (!plan || !dir) return []; const parts = relative(plan.rootPath, dir).split("/").filter(Boolean); - if (overrideDir) return parts.map((text) => ({ text, role: "dir" as const, bound: false })); + if (overrideDir || plan.organize.operation === "in_place") + return parts.map((text) => ({ text, role: "dir" as const, bound: false })); const newEntity = plan.newBindings.some((b) => b.scope === "entity"); const newSeason = plan.newBindings.some((b) => b.scope === "season"); const rootIdx = plan.mediaType === "tv" ? parts.length - 2 : parts.length - 1; @@ -152,11 +173,30 @@ export async function organizeFiles( const actorType = principal?.kind === "api_key" ? "api_key" : "user"; const actor = actorOfP(principal); - // G1:外部候选 → 本地作品(就地建,幂等);每项解析出 mediaEntityId - const created = new Map(); + // 先核验整批人工集号,避免无效输入创建外部作品或冻结计划。数字范围由请求 schema 校验。 + const localIds = [...new Set(body.items.flatMap((it) => (it.mediaEntityId ? [it.mediaEntityId] : [])))]; + const localKinds = new Map( + localIds.length + ? (await db.selectFrom("media_entity").select(["id", "kind"]).where("id", "in", localIds).execute()).map((e) => [ + e.id, + e.kind, + ]) + : [], + ); for (const it of body.items) { if (!!it.mediaEntityId === !!it.external) throw new FilesError("bad_request", `整理项 ${it.path || "/"} 需要且只能指定 mediaEntityId 或 external 之一`); + const kind = it.external?.kind ?? localKinds.get(it.mediaEntityId!); + if (!it.episodes?.length || kind === undefined || kind === "movie") continue; + if (!episodeRange(it.episodes)) + throw new FilesError("bad_request", `${it.path}:集号 ${it.episodes.join(",")} 不连续,只支持连续区间`); + const r = await resolveIn(d, it.root, it.path); + if (r.lst?.isDirectory()) throw new FilesError("bad_request", `${r.rel}:目录项不能指定集号,请展开后逐个视频指定`); + } + + // G1:外部候选 → 本地作品(就地建,幂等);每项解析出 mediaEntityId + const created = new Map(); + for (const it of body.items) { if (!it.external) continue; const k = externalKey(it.external); if (created.has(k)) continue; @@ -228,7 +268,13 @@ export async function organizeFiles( // 目标目录覆写:writable 库根下已存在的目录 let targetDir: string | null = null; let targetLib: string | null = body.targetLibraryRootId ?? null; - if (it.targetDir) { + if (action === "in_place") { + // 本次保留实际库根与路径,不使用来源配置的搬运目标。 + if (!r.def.libraryRootId) + throw new FilesError("bad_request", "原地入库需要文件位于已登记的媒体库目录;请先在设置中登记该目录"); + targetLib = r.def.libraryRootId; + } + if (action !== "in_place" && it.targetDir) { const t = await resolveIn(d, it.targetDir.root, it.targetDir.path); if (!t.lst?.isDirectory() || t.isSymlink || !t.def.libraryRootId) throw new FilesError("bad_request", `目标目录必须是媒体库目录下已存在的目录:${it.targetDir.path || "/"}`); @@ -244,9 +290,9 @@ export async function organizeFiles( if (it.season !== undefined) ov.season = it.season; if (it.episodes?.length) { const range = episodeRange(it.episodes); - if (!range) warnings.push(`${r.rel}:集号 ${it.episodes.join(",")} 不连续,未应用(只支持连续区间)`); - else if (isDir) warnings.push(`${r.rel}:目录项不能指定集号(会套到其下所有视频),未应用`); - else ov.episodes = range; + if (!range) throw new FilesError("bad_request", `${r.rel}:集号不连续,只支持连续区间`); + if (isDir) throw new FilesError("bad_request", `${r.rel}:目录项不能指定集号,请展开后逐个视频指定`); + ov.episodes = range; } } } @@ -339,6 +385,7 @@ export async function organizeFiles( const hasOverrides = Object.keys(g.overrides).length > 0; const reqHash = hashJson({ key: requestKey(req), + ...(body.sourceConfigId && g.action !== "in_place" ? { sourceConfigId: body.sourceConfigId } : {}), ...(hasOverrides ? { overrides: g.overrides } : {}), ...(g.targetDir ? { targetDir: g.targetDir } : {}), ...(g.slotId ? { slotId: g.slotId } : {}), @@ -346,6 +393,8 @@ export async function organizeFiles( ...(minSizeBytes !== undefined ? { minSizeBytes } : {}), ...(onExisting !== "conflict" ? { onExisting } : {}), ...(namingMode !== "template" ? { naming: namingMode } : {}), + ...(body.skipSuccessfulRecords ? { skipSuccessfulRecords: true } : {}), + ...(body.scrapeAfterImport !== undefined ? { scrapeAfterImport: body.scrapeAfterImport } : {}), }); const ext = fromExternal.get(g.entity); const base = { @@ -363,6 +412,7 @@ export async function organizeFiles( }; const importReq = (dryRun: boolean, taskId?: string) => ({ ...req, + ...(body.sourceConfigId && g.action !== "in_place" ? { sourceConfigId: body.sourceConfigId } : {}), ...(hasOverrides ? { fileOverrides: g.overrides } : {}), ...(g.targetDir ? { targetDir: g.targetDir } : {}), ...(g.slotId ? { slotId: g.slotId } : {}), @@ -371,6 +421,8 @@ export async function organizeFiles( // 向导里显式选择"替换"即为人工授权;替换走 replace_with_verify(新文件核验通过后旧文件进回收站) ...(onExisting === "replace_with_verify" ? { grantReplace: true } : {}), namingMode, + skipSuccessfulRecords: body.skipSuccessfulRecords, + scrapeAfterImport: body.scrapeAfterImport, trigger: "manual" as const, actorType: actorType as "api_key" | "user", actorId: actor, @@ -421,6 +473,20 @@ export async function organizeFiles( const nowPairs = pairsOf(again).map(({ src, dst }) => ({ src, dst })); if (hashJson(nowPairs) !== hashJson(frozen.pairs)) throw new FilesError("plan_stale", "目标目录或绑定已变化,与预览不一致。重新预览"); + if (again.plan?.skippedSuccessful?.length && !again.plan.organize.files.length) { + await retirePlan(d, planId); + out.push({ + ...base, + ...planExtras(again.plan, !!g.targetDir), + planId, + files: filesOf(again.plan), + truncated: false, + recordId: null, + taskId: null, + error: null, + }); + continue; + } if (again.status === "failed") throw new FilesError("bad_request", `计划不可执行:${again.reason ?? "blocked"}`); if (g.action === "move") { diff --git a/apps/server/src/modules/file-manager/routes.ts b/apps/server/src/modules/file-manager/routes.ts index c106c5f7..b487c1e4 100644 --- a/apps/server/src/modules/file-manager/routes.ts +++ b/apps/server/src/modules/file-manager/routes.ts @@ -116,7 +116,7 @@ export function registerFileRoutes(app: ModuleApp, d: FmDeps) { mount(app, F.scrape, async (c, { body }) => idem(d, c, "scrape", body, () => scrapeFiles(d, c.get("principal"), body)), ); - mount(app, F.scrapeRecords, async (_c, { query }) => scrapeRecords(d, query.ids)); + mount(app, F.scrapeRecords, async (_c, { query }) => scrapeRecords(d, query.ids, query.organizationRecordId)); mount(app, F.probe, async (_c, { body }) => probeFiles(d, body)); mount(app, F.records, async (_c, { query }) => findRecords(d, query)); mount(app, F.download, async (c, { query }) => downloadFile(d, c.get("principal"), query), true); diff --git a/apps/server/src/modules/file-manager/routes/organize-sources.ts b/apps/server/src/modules/file-manager/routes/organize-sources.ts new file mode 100644 index 00000000..c589cedd --- /dev/null +++ b/apps/server/src/modules/file-manager/routes/organize-sources.ts @@ -0,0 +1,24 @@ +import { Api } from "@umt/contracts"; +import { ApiException } from "../../../http/errors.ts"; +import { endpoint } from "../../auth-settings/endpoint.ts"; +import type { ModuleApp } from "../../types.ts"; +import { type FmDeps, fsErrorToFiles } from "../common.ts"; +import { deleteOrganizeSource, listOrganizeSources, saveOrganizeSource } from "../organize-sources.ts"; +import { scanStock } from "../source-monitor.ts"; + +async function guarded(fn: () => Promise): Promise { + try { + return await fn(); + } catch (error) { + const converted = fsErrorToFiles(error); + if (converted) throw new ApiException(converted.code, `${converted.reason}: ${converted.message}`); + throw error; + } +} +export function registerOrganizeSourceRoutes(app: ModuleApp, d: FmDeps) { + const defs = Api.organizeSources; + endpoint(app, defs.list, () => guarded(() => listOrganizeSources(d))); + endpoint(app, defs.save, ({ params, body }) => guarded(() => saveOrganizeSource(d, params.id, body))); + endpoint(app, defs.remove, ({ params }) => guarded(() => deleteOrganizeSource(d, params.id))); + endpoint(app, defs.stock, ({ params, body }) => guarded(() => scanStock(d, params.id, body))); +} diff --git a/apps/server/src/modules/file-manager/scrape.ts b/apps/server/src/modules/file-manager/scrape.ts index 80ca1b90..c67a8622 100644 --- a/apps/server/src/modules/file-manager/scrape.ts +++ b/apps/server/src/modules/file-manager/scrape.ts @@ -204,8 +204,23 @@ export async function scrapeFiles( }; } -export async function scrapeRecords(d: FmDeps, idsParam: string) { - const ids = [...new Set(idsParam.split(",").map((s) => s.trim()))].filter((s) => /^[0-9a-f-]{36}$/i.test(s)); - if (!ids.length || ids.length > 100) throw new FilesError("bad_request", "ids 需要 1–100 个刮削记录 id"); +export async function scrapeRecords(d: FmDeps, idsParam?: string, organizationRecordId?: string) { + let ids = + idsParam === undefined + ? [] + : [...new Set(idsParam.split(",").map((s) => s.trim()))].filter((s) => /^[0-9a-f-]{36}$/i.test(s)); + if (idsParam !== undefined && (!ids.length || ids.length > 100)) + throw new FilesError("bad_request", "ids 需要 1–100 个刮削记录 id"); + if (!idsParam && !organizationRecordId) throw new FilesError("bad_request", "需要 ids 或 organizationRecordId"); + if (organizationRecordId) { + let query = d.ctx.db + .selectFrom("scrape_record") + .select("id") + .where("organization_record_id", "=", organizationRecordId); + if (idsParam !== undefined) query = query.where("id", "in", ids); + const matched = await query.orderBy("requested_at").orderBy("id").execute(); + const allowed = new Set(matched.map((r) => r.id)); + ids = idsParam === undefined ? matched.map((r) => r.id) : ids.filter((id) => allowed.has(id)); + } return { records: await loadScrapeRecords(d.ao, ids) }; } diff --git a/apps/server/src/modules/file-manager/source-monitor.ts b/apps/server/src/modules/file-manager/source-monitor.ts new file mode 100644 index 00000000..9f0fee84 --- /dev/null +++ b/apps/server/src/modules/file-manager/source-monitor.ts @@ -0,0 +1,412 @@ +import { lstat, opendir, realpath } from "node:fs/promises"; +import { basename, join } from "node:path"; +import { isDownloadComplete } from "@umt/adapters"; +import type { FileRef, OrganizeSourceConfig } from "@umt/contracts"; +import { sql } from "@umt/db"; +import { hostPathOf, loadInstances, upsertDecision } from "../acquisition-organize/services/common.ts"; +import { planImport } from "../acquisition-organize/services/organize.ts"; +import { downloadTasksUnder } from "../acquisition-organize/services/takeover.ts"; +import { entityFromCandidate } from "./candidate.ts"; +import { + FilesError, + type FmDeps, + hashJson, + isTempName, + jsonb, + recheck, + relOf, + resolveIn, + rootDef, + within, +} from "./common.ts"; +import { candidateKey, identifyCore } from "./identify.ts"; +import { kindOfName } from "./list.ts"; +import { sourceConfig, sourceConfigs } from "./organize-sources.ts"; + +export interface SnapshotFile { + size: number; + mtimeMs: number; + accepted: boolean; +} +interface Snapshot { + files: Record; +} +export const SOURCE_SCAN_LIMIT = 5000; +/** A changed file becomes eligible only after two observations of the same size and mtime. */ +export function advanceSnapshot(previous: Snapshot | null, observed: Record>) { + const files: Snapshot["files"] = {}; + const eligible: string[] = []; + for (const [path, fact] of Object.entries(observed)) { + const old = previous?.files[path]; + const same = old?.size === fact.size && old.mtimeMs === fact.mtimeMs; + const accepted = previous === null || (same && old.accepted); + files[path] = { ...fact, accepted }; + if (same && !accepted) eligible.push(path); + } + return { snapshot: { files }, eligible }; +} + +export async function walkSource(d: FmDeps, config: OrganizeSourceConfig, path = config.source.path) { + if (!within(path, config.source.path) && config.source.path !== "") + throw new FilesError("path_escape", "扫描路径必须在来源目录内"); + const start = await resolveIn(d, config.source.root, path); + if (start.isSymlink || !start.lst?.isDirectory()) throw new FilesError("bad_request", "来源必须为目录"); + const files: Record> = {}; + const stack = [start.abs]; + let seen = 0; + let truncated = false; + while (stack.length && !truncated) { + const directory = await opendir(stack.pop()!); + for await (const entry of directory) { + if (++seen > SOURCE_SCAN_LIMIT) { + truncated = true; + break; + } + if (entry.name.startsWith(".") || isTempName(entry.name) || entry.isSymbolicLink()) continue; + const abs = join(directory.path, entry.name); + // Re-resolve parents before traversing to keep directory replacement races inside the root guard. + const resolved = await resolveIn(d, config.source.root, relOf(start.rootReal, abs)); + if (resolved.isSymlink) continue; + if (resolved.lst?.isDirectory()) stack.push(resolved.abs); + else if (resolved.lst?.isFile() && kindOfName(entry.name) === "video") { + const fact = await lstat(resolved.abs); + if (!fact.isFile() || fact.isSymbolicLink() || fact.size <= 0) continue; + files[relOf(start.rootReal, resolved.abs)] = { size: Number(fact.size), mtimeMs: Number(fact.mtimeMs) }; + } + } + } + return { files, truncated, rootReal: start.rootReal }; +} + +/** Query unfiltered QB state once: an untagged download may never have a DB task row. */ +async function activeDownloaderPaths(d: FmDeps, config: OrganizeSourceConfig): Promise { + const source = await rootDef(d.ctx.db, config.source.root); + const instanceIds = [...new Set([config.downloaderInstanceId, source.instanceId].filter((id): id is string => !!id))]; + if (!instanceIds.length) return []; + const topology = await loadInstances(d.ctx.db, { enabledOnly: false }); + const paths: string[] = []; + for (const instanceId of instanceIds) { + const instance = topology.find((t) => t.inst.id === instanceId); + if (!instance) throw new FilesError("upstream", "来源下载器不存在,无法确认文件是否下载完成"); + try { + const torrents = await (await d.ao.ports.downloader(instance.inst)).list(); + for (const torrent of torrents) { + if (isDownloadComplete({ progress: torrent.progress, amountLeft: torrent.amountLeft, state: torrent.rawState })) + continue; + const content = torrent.contentPath || join(torrent.savePath, torrent.name); + const host = hostPathOf(instance, content); + if (!host) throw new Error("下载器内容路径未映射,无法确认源文件是否下载完成"); + paths.push(await realpath(host).catch(() => host)); + } + } catch (error) { + throw new FilesError( + "upstream", + `无法查询来源下载器,暂停本次整理:${error instanceof Error ? error.message : String(error)}`, + ); + } + } + return paths; +} + +async function organizedPaths( + d: FmDeps, + paths: string[], + scan: { files: Record>; rootReal: string }, +) { + if (!paths.length) return new Set(); + const rows = await d.ctx.db + .selectFrom("organization_file_result as f") + .innerJoin("organization_record as r", "r.id", "f.organization_record_id") + .select([ + "f.source_path", + "f.source_size_bytes", + "f.status as file_status", + "f.skip_reason", + "r.status as record_status", + "r.plan", + ]) + .where("f.source_path", "in", paths) + .where("r.status", "in", ["running", "queued", "succeeded", "partial"]) + .execute(); + const done = new Set(); + for (const row of rows) { + if (["running", "queued"].includes(row.record_status)) { + done.add(row.source_path); + continue; + } + if (row.file_status !== "succeeded" && !(row.file_status === "skipped" && row.skip_reason === "already_in_library")) + continue; + const observed = scan.files[relOf(scan.rootReal, row.source_path)]; + if (!observed) continue; + const recorded = (row.plan as { sourceFacts?: Record })?.sourceFacts?.[ + row.source_path + ]; + if ( + recorded + ? recorded.size === observed.size && recorded.mtimeMs === observed.mtimeMs + : row.source_size_bytes === null || Number(row.source_size_bytes) === observed.size + ) + done.add(row.source_path); + } + return done; +} + +export async function scanStock(d: FmDeps, id: string, input: { path?: string } = {}) { + const config = await sourceConfig(d, id); + const liveActive = await activeDownloaderPaths(d, config); + const scan = await walkSource(d, config, input.path ?? config.source.path); + const paths = Object.keys(scan.files); + const done = await organizedPaths( + d, + paths.map((p) => join(scan.rootReal, p)), + scan, + ); + const items: FileRef[] = []; + let skipped = 0; + let unavailable = 0; + let truncated = scan.truncated; + const stockLimit = Math.min(200, d.options.identifyMaxVideos); + const known = await downloadTasksUnder( + d.ctx.db, + paths.map((p) => join(scan.rootReal, p)), + ); + const topology = await loadInstances(d.ctx.db, { enabledOnly: false }); + const active = known + .filter((t) => !["completed", "seeding", "removed"].includes(t.status)) + .flatMap((t) => { + const instance = topology.find((i) => i.inst.id === t.instance_id); + const host = instance && t.content_path ? hostPathOf(instance, t.content_path) : null; + return host ? [host] : []; + }); + active.push(...liveActive); + for (const path of paths) { + const abs = join(scan.rootReal, path); + if (done.has(abs)) { + skipped++; + continue; + } + if (active.some((p) => within(abs, p))) { + unavailable++; + continue; + } + if (items.length >= stockLimit) { + truncated = true; + continue; + } + items.push({ root: config.source.root as FileRef["root"], path }); + } + return { items, skipped, unavailable, truncated }; +} + +export async function scanOrganizeSource(d: FmDeps, input: { sourceId: string }) { + const config = await sourceConfig(d, input.sourceId).catch(() => null); + if (!config) return { baseline: false, eligible: 0, queued: 0 }; + // Cross-worker lock covers observations and planning. A duplicate scan never counts as a second stable observation. + return d.ctx.db.transaction().execute(async (trx) => { + const lock = await sql<{ + locked: boolean; + }>`select pg_try_advisory_xact_lock(hashtext(${`organize-source:${config.id}`})) as locked`.execute(trx); + if (!lock.rows[0]?.locked) return { baseline: false, eligible: 0, queued: 0 }; + const stored = await trx.selectFrom("settings").select("value").where("key", "=", "organize").executeTakeFirst(); + const freshConfig = ( + (stored?.value as { sources?: OrganizeSourceConfig[] })?.sources ?? (await sourceConfigs(d)) + ).find((s) => s.id === config.id); + if (!freshConfig || hashJson(freshConfig) !== hashJson(config)) return { baseline: false, eligible: 0, queued: 0 }; + const row = await trx + .selectFrom("organize_source_snapshot") + .selectAll() + .where("source_id", "=", config.id) + .executeTakeFirst(); + const now = new Date(); + try { + const liveActive = await activeDownloaderPaths(d, config); + const scan = await walkSource(d, config); + if (scan.truncated) throw new FilesError("too_large", "来源超过单次扫描上限,请选择更小的目录"); + const baseline = !row?.baseline_at; + const advance = advanceSnapshot(baseline ? null : (row!.snapshot as unknown as Snapshot), scan.files); + let queued = 0; + const automation = await d.ctx.settings.get("automation"); + if ( + !baseline && + config.enabled && + config.trigger === "snapshot" && + automation.enabled && + !automation.readOnlyReconcile + ) { + const done = await organizedPaths( + d, + advance.eligible.map((p) => join(scan.rootReal, p)), + scan, + ); + for (const path of advance.eligible) { + const abs = join(scan.rootReal, path); + if (done.has(abs)) { + advance.snapshot.files[path]!.accepted = true; + continue; + } + if (liveActive.some((p) => within(abs, p))) continue; + // Any known QB task stays under QB's completion/tag policy, including unmanaged and untagged tasks. + const known = await downloadTasksUnder(d.ctx.db, [abs]); + if (known.length) continue; + const identified = await identifyCore( + d, + { + items: [{ root: config.source.root as FileRef["root"], path }], + hints: config.mediaType === "all" ? {} : { mediaKind: config.mediaType === "tv" ? "series" : "movie" }, + }, + { mode: "remote", actorId: "system:source-monitor" }, + ); + const key = identified.autoPicks.get(`${config.source.root}\u0000${path}`); + const candidate = identified.result.items[0]?.candidates.find((c) => candidateKey(c) === key); + if (!candidate) { + const dedupeKey = `source_identity:${config.id}:${hashJson(path)}`; + const existing = await trx + .selectFrom("decision") + .select("id") + .where("dedupe_key", "=", dedupeKey) + .where("status", "in", ["open", "investigating", "awaiting_user"]) + .executeTakeFirst(); + if (!existing) + await upsertDecision(d.ao, trx, { + kind: "identity_ambiguous", + title: `整理来源待识别:${basename(path)}`, + subjectType: "organize_source", + subjectId: config.id, + dedupeKey, + evidence: { sourceConfigId: config.id, root: config.source.root, path }, + options: [{ key: "dismiss", label: "忽略" }], + }); + advance.snapshot.files[path]!.accepted = true; + continue; + } + const entity = + candidate.mediaEntityId ?? + ( + await entityFromCandidate(d, undefined, { + provider: "tmdb", + externalId: candidate.externalId!, + kind: candidate.kind === "movie" ? "movie" : "tv", + }) + ).mediaEntityId; + const organizeSettings = await d.ctx.settings.get("organize"); + const fingerprint = hashJson({ + config, + classificationYaml: organizeSettings.classificationYaml, + path, + ...scan.files[path], + }); + const result = await planImport(d.ao, { + sourcePaths: [abs], + pathGuard: (p) => recheck(p, scan.rootReal), + mediaEntityId: entity, + action: config.action, + trigger: "auto", + actorType: "system", + sourceConfigId: config.id, + idempotencyKey: `source:${fingerprint}`, + }); + if (result.recordId && ["running", "queued"].includes(result.status)) { + await d.ctx.engine.enqueue( + "verifyAndImport", + { recordId: result.recordId }, + { idempotencyKey: `verifyAndImport:source:${fingerprint}`, traceId: config.id }, + ); + queued++; + } + if (result.status === "failed" && result.reason !== "classification_unavailable") { + const dedupeKey = `source_blocked:${config.id}:${hashJson(path)}`; + const existing = await trx + .selectFrom("decision") + .select("id") + .where("dedupe_key", "=", dedupeKey) + .where("status", "in", ["open", "investigating", "awaiting_user"]) + .executeTakeFirst(); + if (!existing) + await upsertDecision(d.ao, trx, { + kind: "other", + title: `来源整理未完成:${basename(path)}`, + summary: `原因:${result.reason ?? "整理计划失败"}。可从整理来源的存量整理向导处理这个文件。`, + subjectType: "organize_source", + subjectId: config.id, + dedupeKey, + evidence: { + sourceConfigId: config.id, + root: config.source.root, + path, + recordId: result.recordId, + reason: result.reason, + fingerprint, + }, + options: [{ key: "dismiss", label: "忽略" }], + }); + } + advance.snapshot.files[path]!.accepted = result.reason !== "classification_unavailable"; + } + } + // A source edit/delete while scanning invalidates the observation instead of restoring the old baseline. + const settings = await trx + .selectFrom("settings") + .select("value") + .where("key", "=", "organize") + .executeTakeFirst(); + const current = ( + (settings?.value as { sources?: OrganizeSourceConfig[] })?.sources ?? (await sourceConfigs(d)) + ).find((s) => s.id === config.id); + if (!current || hashJson(current.source) !== hashJson(config.source)) + return { baseline: false, eligible: 0, queued }; + await trx + .insertInto("organize_source_snapshot") + .values({ + source_id: config.id, + snapshot: jsonb(advance.snapshot), + baseline_at: row?.baseline_at ?? now, + last_scan_at: now, + last_error: null, + }) + .onConflict((oc) => + oc.column("source_id").doUpdateSet({ + snapshot: jsonb(advance.snapshot), + baseline_at: row?.baseline_at ?? now, + last_scan_at: now, + last_error: null, + }), + ) + .execute(); + return { baseline, eligible: advance.eligible.length, queued }; + } catch (error) { + await trx + .insertInto("organize_source_snapshot") + .values({ + source_id: config.id, + snapshot: jsonb({ files: {} }), + baseline_at: null, + last_scan_at: now, + last_error: error instanceof Error ? error.message : String(error), + }) + .onConflict((oc) => + oc + .column("source_id") + .doUpdateSet({ last_scan_at: now, last_error: error instanceof Error ? error.message : String(error) }), + ) + .execute(); + return { baseline: false, eligible: 0, queued: 0 }; + } + }); +} + +export async function queueDueSourceScans(d: FmDeps, slot: string | number) { + const rows = await d.ctx.db.selectFrom("organize_source_snapshot").select(["source_id", "last_scan_at"]).execute(); + let queued = 0; + for (const config of await sourceConfigs(d)) { + if (!config.enabled || config.trigger !== "snapshot") continue; + const last = rows.find((r) => r.source_id === config.id)?.last_scan_at; + if (last && Date.now() - new Date(last).getTime() < config.intervalSec * 1000) continue; + await d.ctx.engine.enqueue( + "scanOrganizeSource", + { sourceId: config.id }, + { idempotencyKey: `source-scan:${config.id}:${slot}`, traceId: config.id }, + ); + queued++; + } + return { summary: `已入队 ${queued} 个来源扫描` }; +} diff --git a/apps/server/src/modules/file-manager/tree.ts b/apps/server/src/modules/file-manager/tree.ts index 8d4846b6..a521639d 100644 --- a/apps/server/src/modules/file-manager/tree.ts +++ b/apps/server/src/modules/file-manager/tree.ts @@ -76,13 +76,23 @@ export async function recordSources(d: FmDeps, recordId: string): Promise m.source_path); - const dir = rec.source_path; + // Successful moves have removed their inputs: reorganize the existing library files. + const moved = rec.action === "move" && rec.status === "succeeded"; + const targetOf = (rootId: string | null, path: string | null) => { + const root = defs.find((d) => d.libraryRootId === rootId); + return root && path && within(join(root.hostPath, path), root.hostPath) ? join(root.hostPath, path) : null; + }; + let sources = moved + ? mains.map((m) => targetOf(m.target_library_root_id, m.target_path)).filter((p): p is string => !!p) + : mains.map((m) => m.source_path); + const dir = moved ? targetOf(rec.target_library_root_id, rec.target_path) : rec.source_path; if ( dir && - mains.length > 1 && - mains.every((m) => within(m.source_path, dir)) && + sources.length > 1 && + sources.every((p) => within(p, dir)) && !defs.some((x) => x.hostPath === dir.replace(/\/+$/, "")) ) sources = [dir]; @@ -109,7 +126,7 @@ export async function recordSources(d: FmDeps, recordId: string): Promise true, @@ -126,5 +143,11 @@ export async function recordSources(d: FmDeps, recordId: string): Promise `site:${d}`) .join(" OR "); - const queries = [`${baseTitle} 短剧`, `${baseTitle} 短剧${siteFilter ? ` ${siteFilter}` : ""}`].map((x) => + const term = input.kind ? (input.kind === "movie" ? "电影" : "电视剧") : "短剧"; + const queries = [`${baseTitle} ${term}`, `${baseTitle} ${term}${siteFilter ? ` ${siteFilter}` : ""}`].map((x) => x.slice(0, 400), ); const sources: EnrichSource[] = []; @@ -583,7 +594,7 @@ export async function runEnrichPipeline( /** 交给模型、参与核对的搜索摘要(白名单网站相关的 + 摘要模式用上的) */ const snippets: EnrichSearchHit[] = []; const snippetEv: EvidenceSource[] = []; - const cands: Candidate[] = []; + const cands: Candidate[] = [...(input.seedCandidates ?? [])]; const results: Partial> = {}; const fetchedKinds: Partial> = {}; const kindOf = new Map(); @@ -598,7 +609,11 @@ export async function runEnrichPipeline( const scope = `system:enrich:${id}:${i}`; used.push(query); usage.searchCalls++; - const hits = await deps.search(query, scope, signal); + const hits = await deps.search(query, scope, signal).catch((e) => { + if (signal?.aborted || !input.seedPages?.length) throw e; + logger.warn({ enrichmentId: id }, "网页搜索不可用,使用已获取的 PT 详情继续补全"); + return []; + }); for (const h of hits) { const host = hostOf(h.url); if (!host || sources.some((x) => x.url === h.url)) continue; @@ -651,8 +666,37 @@ export async function runEnrichPipeline( } } } - const pages: EnrichPage[] = []; - const pageEv: EvidenceSource[] = []; + const pages: EnrichPage[] = [...(input.seedPages ?? [])]; + const pageEv: EvidenceSource[] = pages.map((p) => ({ + url: p.finalUrl, + host: hostOf(p.finalUrl) ?? "", + kind: "other", + page: true, + text: p.text, + image: p.image, + })); + for (const p of pages) { + sources.push({ + url: p.finalUrl, + title: p.title, + domain: hostOf(p.finalUrl) ?? "", + allowed: true, + fetched: true, + skipped: null, + error: null, + kind: "other", + }); + cands.push( + ...extractFromPage(q.title, { + url: p.finalUrl, + host: hostOf(p.finalUrl) ?? "", + kind: "other", + title: p.title, + text: p.text, + image: p.image, + }), + ); + } // 先读标题完全对得上的,再按白名单顺序(百科、平台靠前) const rank = (x: EnrichSource) => settings.domains.findIndex((d) => x.domain === d || x.domain.endsWith(`.${d}`)); const toFetch = sources @@ -753,7 +797,7 @@ export async function runEnrichPipeline( suggestion: { fields, // 补全面向短剧 / 网剧:缺省按剧集建档,应用时可改为电影 - kind: "series", + kind: input.kind ?? "series", queries: used, sources, extractor, @@ -846,6 +890,14 @@ export async function applyEnrichment( const same = await sameTitleEntities(trx, [title, q.title ?? ""]); const hit = same.find((e) => year === null || e.year === null || e.year === year); entityId = hit?.mediaEntityId ?? null; + if (entityId && actor.type === "system") { + const existing = await trx + .selectFrom("media_entity") + .select("kind") + .where("id", "=", entityId) + .executeTakeFirst(); + if (existing?.kind !== kind) entityId = null; + } } if (!entityId) { const e = await trx @@ -919,7 +971,7 @@ export async function applyEnrichment( } let units = 0; const episodes = want.has("episodeCount") ? (f.episodeCount?.value ?? null) : null; - if (kind === "series") { + if (kind === "series" && actor.type !== "system") { const parsed = { season: 1, ...(episodes ? { episodes: { start: 1, end: Math.min(episodes, 200) } } : {}), diff --git a/apps/server/src/modules/sources-identity/service/organize-metadata.ts b/apps/server/src/modules/sources-identity/service/organize-metadata.ts new file mode 100644 index 00000000..b9462f6a --- /dev/null +++ b/apps/server/src/modules/sources-identity/service/organize-metadata.ts @@ -0,0 +1,332 @@ +import { recallTmdbCandidates } from "@umt/adapters"; +import type { OrganizeMetadataOutput } from "@umt/contracts"; +import { parseRelease, resolveIdentity } from "@umt/domain"; +import { jsonb } from "../rows.ts"; +import type { SourcesRuntime } from "../runtime.ts"; +import { type Actor, audit } from "./common.ts"; +import { detailScreenshotUrl, loadDetailPage } from "./detail-page.ts"; +import { applyEnrichment, type EnrichDeps, type EnrichPage, suggestEnrichment } from "./enrich.ts"; +import type { Candidate } from "./enrich-sources.ts"; +import { assignReleaseIdentity } from "./identity-admin.ts"; +import { type EnrichmentFieldWrite, project, writeEnrichmentFields } from "./metadata.ts"; +import { resolveRelease } from "./resolve.ts"; +import { tmdbToCandidate } from "./tmdb-candidate.ts"; + +/** Only values explicitly present in a fetched PT page are considered facts. */ +export function ptDetailFacts(text: string, url: string) { + const lines = text + .replace(/◎/g, "\n") + .split(/\r?\n/) + .map((s) => s.trim()) + .filter(Boolean); + const label = (re: RegExp) => { + const line = lines.find((s) => re.test(s)); + return ( + line + ?.replace(re, "") + .replace(/^\s*[::]\s*/, "") + .trim() || null + ); + }; + const split = (s: string | null) => (s ? s.split(/\s*[/、|,,]\s*/).filter(Boolean) : []); + const title = label(/^(?:译\s*名|中文名|中文片名|片\s*名)\s*/); + const original = label(/^(?:原\s*名|原片名|外文名)\s*/); + const names = [ + ...new Set([title, original, ...split(label(/^(?:又\s*名|别\s*名)\s*/))].filter((s): s is string => !!s)), + ]; + const yearText = label(/^(?:年\s*代|年\s*份|上映年份)\s*/); + const year = yearText?.match(/\b(19\d{2}|20\d{2})\b/)?.[0]; + const synopsisAt = lines.findIndex((s) => /^(?:简\s*介|剧情简介|内容简介)/.test(s)); + const synopsis = + synopsisAt >= 0 + ? lines + .slice(synopsisAt) + .join("\n") + .replace(/^(?:简\s*介|剧情简介|内容简介)\s*[::]?\s*/, "") + .slice(0, 2000) + : null; + const base = { + sourceUrl: url, + host: new URL(url).hostname, + kind: "other" as const, + confidence: 0.8, + verbatim: true, + by: "rule" as const, + }; + const candidates: Candidate[] = []; + const add = (field: Candidate["field"], value: string | number | null) => { + if (value !== null && value !== "") candidates.push({ ...base, field, value }); + }; + add("title", title); + for (const name of names) add("aliases", name); + add("year", year ? Number(year) : null); + const episodeCount = label(/^(?:集\s*数|总集数)\s*/)?.match(/\d+/)?.[0]; + if (episodeCount && Number(episodeCount) > 0 && Number(episodeCount) <= 2000) + add("episodeCount", Number(episodeCount)); + add("synopsis", synopsis); + for (const g of split(label(/^(?:类\s*别|类\s*型)\s*/))) add("genre", g); + for (const name of split(label(/^(?:主\s*演|演员)\s*/))) add("cast", name); + const countries = split(label(/^(?:产\s*地|制片国家\/地区|国家\/地区|地\s*区)\s*/)); + const language = label(/^(?:语\s*言|对白语言)\s*/); + const langCodes: Record = { + 汉语普通话: "zh", + 普通话: "zh", + 中文: "zh", + 粤语: "zh", + 英语: "en", + 日语: "ja", + 韩语: "ko", + 法语: "fr", + 德语: "de", + 西班牙语: "es", + }; + const countryCodes: Record = { + 中国大陆: "CN", + 中国: "CN", + 中国香港: "HK", + 香港: "HK", + 中国台湾: "TW", + 台湾: "TW", + 美国: "US", + 英国: "GB", + 日本: "JP", + 韩国: "KR", + 法国: "FR", + 德国: "DE", + }; + const extras: { field: EnrichmentFieldWrite["field"]; value: unknown }[] = []; + if (original) extras.push({ field: "originalTitle", value: original }); + if (countries.length) extras.push({ field: "originCountry", value: countries.map((s) => countryCodes[s] ?? s) }); + const languageName = split(language)[0]; + if (languageName && langCodes[languageName]) + extras.push({ field: "originalLanguage", value: langCodes[languageName] }); + const directors = split(label(/^(?:导\s*演)\s*/)); + if (directors.length) + extras.push({ + field: "crew", + value: directors.map((name, i) => ({ + id: -(i + 1), + name, + originalName: null, + role: "Director", + profilePath: null, + })), + }); + return { candidates, extras, names, title, year: year ? Number(year) : null }; +} + +/** Reuses identity matching, audited enrichment, field projection and the existing local entity identity. */ +export async function resolveOrganizeMetadata( + rt: SourcesRuntime, + sourceReleaseId: string, + deps: EnrichDeps, + opts: { actor: Actor; signal?: AbortSignal }, +): Promise { + const { ctx } = rt; + const result = ( + mediaEntityId: string | null, + mode: "tmdb" | "local" | null, + reason: string | null, + enrichmentId: string | null = null, + ): OrganizeMetadataOutput => ({ mediaEntityId, mode, reason, enrichmentId }); + const modeOf = async (id: string) => + (await ctx.db + .selectFrom("external_identity") + .select("id") + .where("media_entity_id", "=", id) + .where("provider", "=", "tmdb") + .where("status", "=", "accepted") + .executeTakeFirst()) + ? ("tmdb" as const) + : ("local" as const); + const release = await ctx.db + .selectFrom("source_release") + .selectAll() + .where("id", "=", sourceReleaseId) + .executeTakeFirst(); + if (!release) return result(null, null, "release_not_found"); + if (release.media_entity_id) return result(release.media_entity_id, await modeOf(release.media_entity_id), null); + // Failure to query TMDB is not evidence that no TMDB record exists. + const normal = await resolveRelease(rt, { sourceReleaseId, force: true, trigger: "manual", escalate: true }); + if (normal.mediaEntityId) return result(normal.mediaEntityId, await modeOf(normal.mediaEntityId), null); + if (normal.outcome === "lookup_failed" || /error|failed|unavailable/.test(normal.reason ?? "")) + return result(null, null, normal.reason); + const pool = await ctx.db + .selectFrom("identity_candidate") + .select("id") + .where("source_release_id", "=", sourceReleaseId) + .executeTakeFirst(); + if (!pool) return result(null, null, "identity_pool_missing"); + const page = await loadDetailPage(rt, sourceReleaseId); + const site = await ctx.db.selectFrom("site").select("base_url").where("id", "=", release.site_id).executeTakeFirst(); + const raw = release.raw as { _site?: { detailUrl?: string }; _detail?: { sourceUrl?: string } }; + const candidateUrl = raw?._detail?.sourceUrl ?? raw?._site?.detailUrl; + let seedUrl: string | null = null; + if (site?.base_url) { + const u = new URL( + candidateUrl || `details.php?id=${encodeURIComponent(release.site_release_id)}`, + `${site.base_url.replace(/\/$/, "")}/`, + ); + if (u.origin === new URL(site.base_url).origin) { + u.username = ""; + u.password = ""; + u.hash = ""; + for (const key of [...u.searchParams.keys()]) + if (/passkey|token|cookie|secret|api.?key|auth/i.test(key)) u.searchParams.delete(key); + seedUrl = u.toString(); + } + } + const facts = page?.descriptionText && seedUrl ? ptDetailFacts(page.descriptionText, seedUrl) : null; + const seedPages: EnrichPage[] = + page?.descriptionText && seedUrl + ? [ + { + url: seedUrl, + finalUrl: seedUrl, + title: facts?.title ?? release.title, + text: page.descriptionText, + image: await detailScreenshotUrl(rt, sourceReleaseId, 0), + }, + ] + : []; + if (facts?.title && seedPages[0]?.image && seedUrl) + facts.candidates.push({ + field: "posterUrl", + value: seedPages[0].image, + sourceUrl: seedUrl, + host: new URL(seedUrl).hostname, + kind: "other", + confidence: 0.6, + verbatim: true, + by: "rule", + }); + const parsed = parseRelease(release.title, release.subtitle ? { subtitle: release.subtitle } : {}); + const kind = + parsed.mediaTypeHint === "tv" || + parsed.season !== undefined || + /剧集|电视剧|连续剧/.test(release.category_raw ?? "") || + facts?.candidates.some((c) => c.field === "episodeCount") + ? ("series" as const) + : ("movie" as const); + const out = await suggestEnrichment( + ctx, + deps, + { poolEntryId: pool.id, fresh: true }, + { + trigger: "auto", + createdBy: "system:organize", + ...(opts.signal ? { signal: opts.signal } : {}), + seedPages, + seedCandidates: facts?.candidates ?? [], + kind, + }, + ); + if (out.status === "failed" || !out.suggestion) + return result(null, null, out.error ?? "metadata_no_evidence", out.enrichmentId); + const fields = out.suggestion.fields; + for (const key of ["aliases", "genre", "cast"] as const) fields[key] = fields[key].filter((v) => v.verbatim); + await ctx.db + .updateTable("metadata_enrichment") + .set({ suggestion: jsonb(out.suggestion) }) + .where("id", "=", out.enrichmentId) + .execute(); + const names = [ + ...new Set( + [ + fields.title?.value, + ...fields.aliases.filter((v) => v.verbatim).map((v) => v.value), + ...(facts?.names ?? []), + ].filter((s): s is string => !!s), + ), + ]; + const tmdb = await rt.tmdb(); + if (tmdb && names.length) { + const year = fields.year?.verbatim ? fields.year.value : (facts?.year ?? parsed.year); + const candidates = await recallTmdbCandidates(tmdb, { + names, + type: kind === "movie" ? "movie" : "tv", + ...(year ? { year } : {}), + maxNames: 4, + }); + const matched = new Map>(); + let ambiguous = false; + for (const name of names) { + const query = { ...parsed, title: name, titleCn: undefined, titleEn: undefined, ...(year ? { year } : {}) }; + const r = resolveIdentity(query, candidates.map(tmdbToCandidate)); + if (r.status === "ambiguous") ambiguous = true; + if (r.status === "auto" && r.best) { + const c = candidates.find( + (c) => c.externalId === r.best!.candidate.externalId && c.entityType === r.best!.candidate.entityType, + ); + if (c) matched.set(`${c.entityType}:${c.externalId}`, c); + } + } + if (matched.size === 1 && !ambiguous) { + const c = [...matched.values()][0]!; + const assigned = await assignReleaseIdentity( + rt, + sourceReleaseId, + { + external: { provider: "tmdb", entityType: c.entityType, externalId: c.externalId }, + lock: false, + reason: `整理 Agent 别名补全 ${out.enrichmentId}`, + }, + opts.actor, + ); + return result(assigned.mediaEntityId, "tmdb", null, out.enrichmentId); + } + if (matched.size > 1 || ambiguous) return result(null, null, "metadata_tmdb_ambiguous", out.enrichmentId); + } + if (normal.outcome === "ambiguous" || normal.outcome === "proposed") + return result(null, null, "metadata_tmdb_ambiguous", out.enrichmentId); + // No invented placeholder entity: require a supported title and at least one additional factual field. + if ( + !fields.title?.verbatim || + !( + fields.year?.verbatim || + fields.synopsis?.verbatim || + fields.cast.some((v) => v.verbatim) || + fields.genre.some((v) => v.verbatim) + ) + ) + return result(null, null, "metadata_insufficient_evidence", out.enrichmentId); + const allowedFields = Object.entries(fields) + .filter(([, v]) => (Array.isArray(v) ? v.some((x) => x.verbatim) : v?.verbatim)) + .map(([k]) => k) as NonNullable[2]["fields"]>; + const applied = await applyEnrichment( + rt, + pool.id, + { enrichmentId: out.enrichmentId, kind, fields: allowedFields, lock: false }, + opts.actor, + ); + if (facts?.extras.length) { + const writes: EnrichmentFieldWrite[] = facts.extras.map((f) => ({ + ...f, + source: "web", + evidence: { + sourceUrl: seedUrl, + enrichmentId: out.enrichmentId, + evidenceLevel: "rule_confirmed", + extractor: "rule", + }, + })); + await ctx.db.transaction().execute(async (trx) => { + await writeEnrichmentFields(trx, applied.mediaEntityId, writes, `enrich:${out.enrichmentId}:pt`, rt.now()); + await project(trx, applied.mediaEntityId, rt.now()); + await audit( + trx, + opts.actor, + "metadata.organize_local", + { type: "media_entity", id: applied.mediaEntityId }, + { sourceReleaseId, enrichmentId: out.enrichmentId }, + ); + }); + } + // Automatic fallback must remain revisitable by future identity review. + await ctx.db + .updateTable("source_release") + .set({ resolution_locked: false }) + .where("id", "=", sourceReleaseId) + .execute(); + return result(applied.mediaEntityId, await modeOf(applied.mediaEntityId), null, out.enrichmentId); +} diff --git a/apps/server/src/modules/sources-identity/service/resolve.ts b/apps/server/src/modules/sources-identity/service/resolve.ts index ec39c0f8..0ca46895 100644 --- a/apps/server/src/modules/sources-identity/service/resolve.ts +++ b/apps/server/src/modules/sources-identity/service/resolve.ts @@ -261,6 +261,11 @@ function escalationFetchPolicy( return "no"; } +const RESOLUTION_RANK: Record = { no_match: 0, ambiguous: 1, auto: 2 }; +/** 升级链的结果只在更确定时采用,原有歧义候选不被更差的结果冲掉 */ +const betterResolution = (next: IdentityResolution, cur: IdentityResolution) => + RESOLUTION_RANK[next.status] > RESOLUTION_RANK[cur.status]; + interface EscalationOutcome { evidence: EscalationEvidence; resolution?: IdentityResolution; @@ -305,9 +310,11 @@ async function escalateIdentity( const remote = await tmdbByExplicitIds(tmdb, after); if (remote.length) { const res = resolveIdentity(p, [...local, ...remote.map(tmdbToCandidate)], trustedOf(after)); - best = { evidence: ev, resolution: res, remote, parsed: p }; - ev.via = "site_id"; - if (res.status === "auto") return best; + if (res.status !== "no_match") { + best = { evidence: ev, resolution: res, remote, parsed: p }; + ev.via = "site_id"; + if (res.status === "auto") return best; + } } } // 2) 豆瓣 ID → 缓存的豆瓣条目(标题 + 年份)→ TMDB @@ -317,7 +324,8 @@ async function escalateIdentity( if (d) { ev.douban = { id: d.subject.id, title: d.subject.title, year: d.subject.year, source: d.subject.source }; const res = resolveIdentity(d.parsed, [...local, ...d.remote.map(tmdbToCandidate)]); - if (res.status === "auto" || !best || (best.resolution?.status === "no_match" && res.status !== "no_match")) { + // 与 MP 识别兜底一致:无匹配的结果不顶替任何已有结果 + if (res.status === "auto" || (!best && res.status !== "no_match")) { ev.via = "douban"; best = { evidence: ev, resolution: res, remote: d.remote, parsed: d.parsed }; } @@ -445,7 +453,7 @@ export async function resolveRelease(rt: SourcesRuntime, input: ResolveInput): P try { const up = await escalateIdentity(rt, release, parsed, local, siteIds, policy); escalation = up.evidence; - if (up.resolution && up.remote && up.parsed) { + if (up.resolution && up.remote && up.parsed && betterResolution(up.resolution, resolution)) { resolution = up.resolution; remote = up.remote; scored = up.parsed; diff --git a/apps/server/test/auth-settings/engine-retry.test.ts b/apps/server/test/auth-settings/engine-retry.test.ts index 9af8a82c..03a7a2aa 100644 --- a/apps/server/test/auth-settings/engine-retry.test.ts +++ b/apps/server/test/auth-settings/engine-retry.test.ts @@ -37,6 +37,24 @@ describe.skipIf(!hasPg)("POST /engine/runs/{id}/retry", () => { await h?.close(); }); + it("工作流多值筛选保留排队核验与独立刮削,单值筛选兼容", async () => { + const table = sql.raw(`"${h.engineSchema}".workflow_runs`); + await sql`insert into ${table} (namespace_id, id, workflow_name, version, status, config, context, input, attempts, created_at, updated_at) + values ('default', 'scrape-queue', 'scrapeMedia', 'v2', 'pending', '{}'::jsonb, null, '{"input":{}}'::jsonb, 0, now(), now())`.execute( + h.db, + ); + try { + const both = await json( + await admin("GET", "/engine/runs?workflowName=verifyAndImport,scrapeMedia&status=pending"), + ); + expect(both.data.items.map((run: { id: string }) => run.id).sort()).toEqual(["p1", "scrape-queue"]); + const single = await json(await admin("GET", "/engine/runs?workflowName=verifyAndImport&status=pending")); + expect(single.data.items.map((run: { id: string }) => run.id)).toEqual(["p1"]); + } finally { + await sql`delete from ${table} where id = 'scrape-queue'`.execute(h.db); + } + }); + it("失败的 run:按原任务名与原始输入重新入队,序号递增,写审计", async () => { const r1 = await json(await admin("POST", "/engine/runs/f1/retry")); expect(r1.data).toEqual({ runId: "new-1", retryOf: "f1", attempt: 1 }); diff --git a/apps/server/test/e2e/e10-w4a-endpoints.test.ts b/apps/server/test/e2e/e10-w4a-endpoints.test.ts index cc2ebfc8..bef43506 100644 --- a/apps/server/test/e2e/e10-w4a-endpoints.test.ts +++ b/apps/server/test/e2e/e10-w4a-endpoints.test.ts @@ -10,7 +10,7 @@ import { waitFor, } from "./harness.ts"; -// E10(原编号与 e6-auto-recover 重复,W6-B 改名) W4-A 前后端集成端点:手动提交下载(与自动同流程)、候选评估去重、库存叠加补字段、 +// E10(原编号与 e6-auto-recover 重复,W6-B 改名) W4-A 前后端集成端点:独立手动下载与自动订阅下载分别验证、候选评估去重、库存叠加补字段、 // 整理记录 downloadTaskId、只读资产/产物端点、偏好标签修订历史、列表带作品标题、事件 JSON 补读。 const hasPg = await pgAvailable(); @@ -29,13 +29,17 @@ describe.skipIf(!hasPg)("E10 W4-A 集成端点", () => { await e?.close(); }, 60_000); - it("POST /acquisition-plans:硬条件不满足 → 不提交并给出原因;重复评估只留一条候选记录", async () => { + it("download.plan:自动订阅硬条件不满足 → 无计划并给出原因;重复评估只留一条候选记录", async () => { for (let i = 0; i < 2; i++) { - const r = await e.api("POST", "/acquisition-plans", { sourceReleaseId: target.r720, slotId: target.slotId }); + const r = await e.api("POST", "/capabilities/download.plan/invoke", { + input: { sourceReleaseId: target.r720, slotId: target.slotId }, + }); expect(r.status).toBe(200); - expect(r.body.data).toMatchObject({ submitted: false, plan: null, taskId: null }); - expect(r.body.data.rejected.length).toBeGreaterThan(0); - expect(r.body.data.rejected[0].slotId).toBe(target.slotId); + const out = r.body.data.output ?? r.body.data; + expect(out.plan).toBeNull(); + expect(out.slot.id).toBe(target.slotId); + expect(out.rejected.length).toBeGreaterThan(0); + expect(await e.db.selectFrom("download_task").select("id").execute()).toHaveLength(0); } const n = await sql<{ n: number }>`select count(*)::int as n from artifact where title = 'candidate_evaluation' and subject_id = ${target.slotId} @@ -53,23 +57,30 @@ describe.skipIf(!hasPg)("E10 W4-A 集成端点", () => { expect(c.reasons.length).toBeGreaterThan(0); }); - it("POST /acquisition-plans:按作品选 Slot,走 download.plan → download.submit,qB 收到种子;整理记录带 downloadTaskId", async () => { - const r = await e.api("POST", "/acquisition-plans", { - sourceReleaseId: target.r1080, - entityId: target.entityId, - note: "工作台手动", + it("自动订阅 download.plan → download.submit 推进 Slot,qB 收到种子;整理记录带 downloadTaskId", async () => { + const planned = await e.api("POST", "/capabilities/download.plan/invoke", { + input: { sourceReleaseId: target.r1080, slotId: target.slotId }, }); - expect(r.status).toBe(200); - expect(r.body.data).toMatchObject({ submitted: true, slotId: target.slotId }); - expect(r.body.data.plan.status).toMatch(/reserved|submitted/); - expect(r.body.data.taskId).toBeTruthy(); + expect(planned.status).toBe(200); + const out = planned.body.data.output ?? planned.body.data; + expect(out.plan).toMatchObject({ slotId: target.slotId, sourceReleaseId: target.r1080 }); + const grant = await e.api("POST", "/automation-grants", { + action: "download.submit", + scope: { mediaEntityIds: [target.entityId] }, + reason: "e10 自动订阅收录验证", + }); + expect(grant.status).toBe(201); + const submitted = await e.api("POST", "/capabilities/download.submit/invoke", { input: { planId: out.plan.id } }); + expect(submitted.status).toBe(200); + const accepted = submitted.body.data.output ?? submitted.body.data; + expect(accepted.planId).toBe(out.plan.id); + expect(accepted.taskId).toBeTruthy(); const audits = await e.db .selectFrom("audit_event") .select(["action", "target_id"]) - .where("action", "in", ["capability.invoke", "acquisition_plan.manual_submit"]) + .where("action", "=", "capability.invoke") .execute(); expect(audits.filter((a) => a.target_id === "download.submit")).toHaveLength(1); - expect(audits.some((a) => a.action === "acquisition_plan.manual_submit")).toBe(true); await completeDownload(e, "Dune.Part.Two.2024.1080p.BluRay.x264-GRP"); const rec = await waitFor( @@ -104,6 +115,80 @@ describe.skipIf(!hasPg)("E10 W4-A 集成端点", () => { expect(slots.body.data.items[0].currentAsset).toMatchObject({ resolution: "1080p" }); }); + it("POST /acquisition-plans:没有订阅和 Slot 的作品也能独立手动下载", async () => { + const standalone = await e.db + .insertInto("media_entity") + .values({ kind: "movie", title: "独立手动电影", year: 2025, status: "active" }) + .returning("id") + .executeTakeFirstOrThrow(); + const site = await e.db + .selectFrom("source_release") + .select("site_id") + .where("id", "=", target.r720) + .executeTakeFirstOrThrow(); + const release = await e.db + .insertInto("source_release") + .values({ + site_id: site.site_id, + site_release_id: "880003", + title: "Manual.Standalone.2025.720p.WEB-DL.x264-GRP", + size_bytes: 300000, + media_entity_id: standalone.id, + resolution_status: "resolved", + raw: JSON.stringify({ _site: { hr: false } }), + }) + .returning("id") + .executeTakeFirstOrThrow(); + const before = await e.db.selectFrom("subscription").select("id").execute(); + const slotBefore = await e.db + .selectFrom("acquisition_slot") + .select(["current_asset_id", "revision"]) + .where("id", "=", target.slotId) + .executeTakeFirstOrThrow(); + const r = await e.api("POST", "/acquisition-plans", { + sourceReleaseId: release.id, + entityId: standalone.id, + note: "工作台独立手动", + }); + expect(r.status).toBe(200); + expect(r.body.data).toMatchObject({ + submitted: true, + slotId: null, + rejected: [], + plan: { slotId: null, evaluation: { manual: true, mediaEntityId: standalone.id } }, + }); + expect(r.body.data.taskId).toBeTruthy(); + await waitFor( + async () => + e.db + .selectFrom("download_task") + .selectAll() + .where("import_plan_id", "=", r.body.data.plan.id) + .executeTakeFirst(), + 60000, + "独立手动下载已提交", + ); + expect(await e.db.selectFrom("subscription").select("id").execute()).toEqual(before); + expect( + await e.db.selectFrom("acquisition_slot").select("id").where("media_entity_id", "=", standalone.id).execute(), + ).toHaveLength(0); + expect( + await e.db + .selectFrom("acquisition_slot") + .select(["current_asset_id", "revision"]) + .where("id", "=", target.slotId) + .executeTakeFirstOrThrow(), + ).toEqual(slotBefore); + expect( + await e.db + .selectFrom("audit_event") + .select("id") + .where("action", "=", "acquisition_plan.manual_submit") + .where("target_id", "=", r.body.data.plan.id) + .executeTakeFirst(), + ).toBeTruthy(); + }, 90000); + it("GET /artifacts:列表与详情", async () => { const list = await e.api("GET", "/artifacts?kind=evidence"); expect(list.status).toBe(200); diff --git a/apps/server/test/e2e/e2-agent-delegate.test.ts b/apps/server/test/e2e/e2-agent-delegate.test.ts index e7f87c4c..058c3928 100644 --- a/apps/server/test/e2e/e2-agent-delegate.test.ts +++ b/apps/server/test/e2e/e2-agent-delegate.test.ts @@ -1,3 +1,4 @@ +import { CAPABILITY_IDS } from "@umt/contracts"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { delegateScript } from "./fixtures/scripts.ts"; import { @@ -45,8 +46,16 @@ describe.skipIf(!hasPg)("E2 Agent 委托闭环(fake 模型 + 真实能力/授权 return r.body.data.id as string; }; - it("启动断言:41 个能力全部注册,并在能力目录里可见", async () => { - expect(e.booted.ctx.capabilities.list()).toHaveLength(41); + it("启动断言:合同中的能力全部注册,整理兜底不可重放,并在能力目录里可见", async () => { + expect( + e.booted.ctx.capabilities + .list() + .map((c) => c.descriptor.id) + .sort(), + ).toEqual([...CAPABILITY_IDS].sort()); + expect( + e.booted.ctx.capabilities.list().find((c) => c.descriptor.id === "metadata.resolve_organize")?.descriptor, + ).toMatchObject({ kind: "execute", replay: "never" }); const r = await e.api("GET", "/capabilities"); expect(r.body.data.items.map((c: { id: string }) => c.id).sort()).toEqual( e.booted.ctx.capabilities diff --git a/apps/server/test/e2e/e8-tag-removal.test.ts b/apps/server/test/e2e/e8-tag-removal.test.ts index 2ecd70fd..2be080c8 100644 --- a/apps/server/test/e2e/e8-tag-removal.test.ts +++ b/apps/server/test/e2e/e8-tag-removal.test.ts @@ -1,6 +1,7 @@ import { createQbClient } from "@umt/adapters"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import type { createAcquisitionOrganizeModule } from "../../src/modules/acquisition-organize/index.ts"; +import { evaluateAcquisition } from "../../src/modules/acquisition-organize/services/acquisition.ts"; import { syncDownloads } from "../../src/modules/acquisition-organize/services/downloads.ts"; import { torrentFor } from "./fixtures/modules.ts"; import { createE2E, type E2E, pgAvailable, seedBase, waitFor } from "./harness.ts"; @@ -81,9 +82,11 @@ describe.skipIf(!hasPg)("E8 AT24 标签移除 / 任务消失", () => { }, 60_000); const submit = async (k: K) => { - const r = await e.api("POST", "/acquisition-plans", { sourceReleaseId: ids[k].releaseId }); - expect(r.status, JSON.stringify(r.body)).toBe(200); - expect(r.body.data.submitted).toBe(true); + const planned = await evaluateAcquisition(deps(), { + sourceReleaseId: ids[k].releaseId, + acquisitionSlotId: ids[k].slotId, + }); + expect(planned.plans, JSON.stringify(planned)).toHaveLength(1); await waitFor(async () => e.qb.get(ids[k].hash), 60_000, `qB 收到 ${k}`); const row = await waitFor( async () => diff --git a/apps/server/test/e2e/e9-continuous-series.test.ts b/apps/server/test/e2e/e9-continuous-series.test.ts index 2d8f7ff2..7a38ac7b 100644 --- a/apps/server/test/e2e/e9-continuous-series.test.ts +++ b/apps/server/test/e2e/e9-continuous-series.test.ts @@ -254,7 +254,7 @@ describe.skipIf(!hasPg)("E9 连续更新剧集:Slot 忙时到达的集全部入 30_000, "订阅完成", ); - expect(sub.status_reason).toBe("slot_satisfied"); + expect(sub.status_reason).toBe("download_accepted"); const slot = await e.db .selectFrom("acquisition_slot") .selectAll() @@ -263,7 +263,7 @@ describe.skipIf(!hasPg)("E9 连续更新剧集:Slot 忙时到达的集全部入 expect(slot.status).toBe("satisfied"); const hist = await e.api("GET", `/subscription-history?subscriptionId=${subId}`); expect(hist.body.data.items).toHaveLength(1); - expect(hist.body.data.items[0]).toMatchObject({ reason: "slot_satisfied", mediaEntityId: entityId }); + expect(hist.body.data.items[0]).toMatchObject({ reason: "download_accepted", mediaEntityId: entityId }); // 进度 API:三集已覆盖,总集数已知 const prog = await e.api("GET", `/subscriptions/${subId}/progress`); expect(prog.body.data.seasons[0]).toMatchObject({ episodeTotal: 3, totalKnown: true, complete: true }); diff --git a/apps/server/test/file-manager/organize-in-place.test.ts b/apps/server/test/file-manager/organize-in-place.test.ts new file mode 100644 index 00000000..3fe90a5e --- /dev/null +++ b/apps/server/test/file-manager/organize-in-place.test.ts @@ -0,0 +1,175 @@ +import { randomUUID } from "node:crypto"; +import { mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { executeRecord } from "../../src/modules/acquisition-organize/services/organize.ts"; +import { createFmHarness, type FmHarness, key, pgAvailable, seedEntity } from "./harness.ts"; + +const skip = !(await pgAvailable()); +describe.skipIf(skip)("file manager in-place import", () => { + let h: FmHarness; + beforeAll(async () => { + h = await createFmHarness("in_place", {}, { runTransfers: false }); + }); + afterAll(async () => { + await h?.close(); + }); + async function source(label: string, kind: "movie" | "series" = "movie") { + const title = `${label}-${randomUUID().slice(0, 8)}`; + const entity = await seedEntity(h.db, { title, kind, year: 2024 }); + await mkdir(join(h.lib, title), { recursive: true }); + const path = `${title}/${kind === "movie" ? "original.name.mkv" : "original.name.S03E04.mkv"}`; + await writeFile(join(h.lib, path), "untouched video"); + return { entity, path, items: [{ root: h.libRoot, path, mediaEntityId: entity }] }; + } + async function preview(items: unknown[], extra: Record = {}) { + const payload = { items, action: "in_place", minSizeBytes: 0, scrapeAfterImport: false, ...extra }; + const r = await h.api("POST", "/files/organize", { ...payload, idempotencyKey: key(), dryRun: true }); + expect(r.status).toBe(200); + return { payload, data: r.body.data }; + } + async function queue(items: unknown[], extra: Record = {}) { + const r = await preview(items, extra); + expect(r.data.groups[0].error).toBeNull(); + const accepted = await h.api("POST", "/files/organize", { + ...r.payload, + idempotencyKey: key(), + dryRun: false, + planIds: r.data.groups.map((g: { planId: string }) => g.planId), + }); + expect(accepted.status).toBe(200); + expect(accepted.body.data.groups[0].error).toBeNull(); + return { ...r, recordId: accepted.body.data.recordIds[0] as string }; + } + it("preserves movie and matched subtitle bytes, paths and mtimes; registers once and never transfers", async () => { + const s = await source("Movie"); + const subtitle = s.path.replace(/\.mkv$/, ".chs.srt"); + await writeFile(join(h.lib, subtitle), "untouched subtitle"); + const before = await Promise.all([s.path, subtitle].map((p) => stat(join(h.lib, p)))); + const r = await queue([{ root: h.libRoot, path: s.path.split("/")[0]!, mediaEntityId: s.entity }], { + naming: "template", + onExisting: "replace_with_verify", + }); + expect(r.data.groups[0].files).toHaveLength(2); + expect(r.data.groups[0].files.every((f: { source: string; target: string }) => f.source === f.target)).toBe(true); + expect((await executeRecord(h.module.deps().ao, r.recordId)).status).toBe("succeeded"); + expect((await executeRecord(h.module.deps().ao, r.recordId)).status).toBe("succeeded"); + expect(await readFile(join(h.lib, s.path), "utf8")).toBe("untouched video"); + expect(await readFile(join(h.lib, subtitle), "utf8")).toBe("untouched subtitle"); + const after = await Promise.all([s.path, subtitle].map((p) => stat(join(h.lib, p)))); + expect(after.map((f) => [f.size, f.mtimeMs, f.ino])).toEqual(before.map((f) => [f.size, f.mtimeMs, f.ino])); + expect(await readdir(join(h.lib, s.path.split("/")[0]!))).toEqual(["original.name.chs.srt", "original.name.mkv"]); + const assets = await h.db.selectFrom("media_asset").selectAll().where("media_entity_id", "=", s.entity).execute(); + expect(assets).toHaveLength(2); + expect(assets.find((a) => a.kind === "subtitle")?.parent_asset_id).toBe(assets.find((a) => a.kind === "video")?.id); + const ops = await h.db + .selectFrom("asset_operation") + .select(["kind", "phase"]) + .where("organization_record_id", "=", r.recordId) + .orderBy("ordinal") + .execute(); + expect(ops).toEqual([ + { kind: "register", phase: "verify" }, + { kind: "register", phase: "verify" }, + ]); + expect( + await h.db.selectFrom("path_binding").select("id").where("media_entity_id", "=", s.entity).execute(), + ).toEqual([]); + const again = await queue(s.items); + await executeRecord(h.module.deps().ao, again.recordId); + expect( + await h.db.selectFrom("media_asset").select("id").where("media_entity_id", "=", s.entity).execute(), + ).toHaveLength(2); + // 文件管理装配不挂整理记录路由:直接查记录表 + const history = await h.db + .selectFrom("organization_record") + .select("action") + .where("media_entity_id", "=", s.entity) + .execute(); + expect(history.some((item) => item.action === "in_place")).toBe(true); + }); + it("registers TV episodes in their existing directory and honors manual season/episode", async () => { + const s = await source("TV", "series"); + const r = await queue(s.items.map((i) => ({ ...i, season: 2, episodes: [7] }))); + expect(r.data.groups[0].files[0].target).toBe(join(h.lib, s.path)); + expect((await executeRecord(h.module.deps().ao, r.recordId)).status).toBe("succeeded"); + const units = await h.db + .selectFrom("content_unit") + .select(["season_number", "episode_number"]) + .where("media_entity_id", "=", s.entity) + .execute(); + expect(units.some((u) => u.season_number === 2 && u.episode_number === 7)).toBe(true); + }); + it("scrapes only the newly registered file, retaining its organization association on resume", async () => { + const s = await source("Scrape"); + await h.db + .insertInto("external_identity") + .values({ + media_entity_id: s.entity, + provider: "tmdb", + entity_type: "movie", + external_id: "123456", + status: "accepted", + }) + .execute(); + await writeFile(join(h.lib, s.path.split("/")[0]!, "Unrelated.mkv"), "unrelated video"); + const r = await queue(s.items, { scrapeAfterImport: true }); + await executeRecord(h.module.deps().ao, r.recordId); + await executeRecord(h.module.deps().ao, r.recordId); + const scrapes = await h.db + .selectFrom("scrape_record") + .select(["path", "organization_record_id"]) + .where("organization_record_id", "=", r.recordId) + .execute(); + expect(scrapes).toEqual([{ path: s.path, organization_record_id: r.recordId }]); + expect(await readFile(join(h.lib, s.path.split("/")[0]!, "Unrelated.mkv"), "utf8")).toBe("unrelated video"); + }); + it("rejects an unregistered download directory without submitting a task", async () => { + const entity = await seedEntity(h.db, { kind: "movie", title: "Outside", year: 2024 }); + await writeFile(join(h.dl, "outside.mkv"), "video"); + const r = await h.api("POST", "/files/organize", { + items: [{ root: h.dlRoot, path: "outside.mkv", mediaEntityId: entity }], + action: "in_place", + minSizeBytes: 0, + idempotencyKey: key(), + dryRun: true, + }); + expect(r.status).toBe(400); + expect(await h.db.selectFrom("media_asset").select("id").where("media_entity_id", "=", entity).execute()).toEqual( + [], + ); + }); + it("does not bind a file to another work or import changed files after queue acceptance", async () => { + const s = await source("Changed"); + const r = await queue(s.items); + await writeFile(join(h.lib, s.path), "changed video longer"); + expect((await executeRecord(h.module.deps().ao, r.recordId)).status).toBe("failed"); + expect(await h.db.selectFrom("media_asset").select("id").where("media_entity_id", "=", s.entity).execute()).toEqual( + [], + ); + const original = await source("Owned"); + const a = await queue(original.items); + await executeRecord(h.module.deps().ao, a.recordId); + const other = await seedEntity(h.db, { kind: "movie", title: "Other", year: 2024 }); + const conflict = await preview([{ root: h.libRoot, path: original.path, mediaEntityId: other }]); + expect(conflict.data.groups[0].error).not.toBeNull(); + expect(await h.db.selectFrom("media_asset").select("id").where("media_entity_id", "=", other).execute()).toEqual( + [], + ); + }); + it("registers read-only library roots without writing metadata", async () => { + const s = await source("ReadOnly"); + const rootId = h.libRoot.slice(4); + await h.db.updateTable("library_root").set({ management_mode: "register_only" }).where("id", "=", rootId).execute(); + try { + const r = await queue(s.items, { scrapeAfterImport: true }); + expect((await executeRecord(h.module.deps().ao, r.recordId)).status).toBe("succeeded"); + expect( + await h.db.selectFrom("scrape_record").select("id").where("organization_record_id", "=", r.recordId).execute(), + ).toEqual([]); + expect(await readdir(join(h.lib, s.path.split("/")[0]!))).toEqual(["original.name.mkv"]); + } finally { + await h.db.updateTable("library_root").set({ management_mode: "managed" }).where("id", "=", rootId).execute(); + } + }); +}); diff --git a/apps/server/test/file-manager/organize-manual-options.test.ts b/apps/server/test/file-manager/organize-manual-options.test.ts new file mode 100644 index 00000000..9e05ebeb --- /dev/null +++ b/apps/server/test/file-manager/organize-manual-options.test.ts @@ -0,0 +1,277 @@ +import { randomUUID } from "node:crypto"; +import { access, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { executeRecord } from "../../src/modules/acquisition-organize/services/organize.ts"; +import { createFmHarness, type FmHarness, key, pgAvailable, seedEntity } from "./harness.ts"; + +const skip = !(await pgAvailable()); +describe.skipIf(skip)("manual organize history and per-import scraping", () => { + let h: FmHarness; + beforeAll(async () => { + h = await createFmHarness("manual_options", {}, { runTransfers: false }); + }); + afterAll(async () => { + await h?.close(); + }); + + async function source(label: string, local = false) { + const title = `${label} ${randomUUID().slice(0, 8)}`; + const entity = await seedEntity(h.db, { kind: "movie", title, year: 2024 }); + if (local) { + await h.db + .updateTable("media_entity") + .set({ + metadata: JSON.stringify({ + overview: { value: "Local synopsis", source: "web", sources: [], locked: false }, + }), + }) + .where("id", "=", entity) + .execute(); + } else { + await h.db + .insertInto("external_identity") + .values({ + media_entity_id: entity, + provider: "tmdb", + entity_type: "movie", + external_id: String(Math.floor(Math.random() * 1_000_000)), + status: "accepted", + }) + .execute(); + } + const path = `${title.replaceAll(" ", ".")}.2024.1080p.mkv`; + await writeFile(join(h.dl, path), `video-${title}`); + return { entity, path, items: [{ root: h.dlRoot, path, mediaEntityId: entity }] }; + } + async function organize(items: unknown[], extra: Record = {}) { + const payload = { action: "copy", items, minSizeBytes: 0, ...extra }; + const dry = await h.api("POST", "/files/organize", { ...payload, idempotencyKey: key(), dryRun: true }); + expect(dry.status).toBe(200); + expect(dry.body.data.groups[0].error).toBeNull(); + const exec = await h.api("POST", "/files/organize", { + ...payload, + idempotencyKey: key(), + dryRun: false, + planIds: dry.body.data.groups.map((g: { planId: string }) => g.planId), + }); + expect(exec.status).toBe(200); + expect(exec.body.data.groups[0].error).toBeNull(); + return { dry: dry.body.data, exec: exec.body.data }; + } + + it("successful move reorganizes the target; failed move and copy retain their source", async () => { + const s = await source("Moved"); + const r = await organize(s.items, { action: "move", scrapeAfterImport: false }); + const rid = r.exec.recordIds[0]; + expect((await executeRecord(h.module.deps().ao, rid)).status).toBe("succeeded"); + await expect(access(join(h.dl, s.path))).rejects.toThrow(); + const refs = await h.api("GET", `/files/record-sources?recordId=${rid}`); + expect(refs.status).toBe(200); + expect(refs.body.data.unresolved).toEqual([]); + expect(refs.body.data.items).toEqual([ + { root: h.libRoot, path: r.dry.groups[0].files[0].target.slice(h.lib.length + 1) }, + ]); + expect(await readFile(r.dry.groups[0].files[0].target, "utf8")).toContain("video-Moved"); + const c = await source("Copied"); + const copy = await organize(c.items, { scrapeAfterImport: false }); + const crid = copy.exec.recordIds[0]; + await executeRecord(h.module.deps().ao, crid); + expect((await h.api("GET", `/files/record-sources?recordId=${crid}`)).body.data.items).toEqual([ + { root: h.dlRoot, path: c.path }, + ]); + await h.db + .updateTable("organization_record") + .set({ action: "move", status: "failed" }) + .where("id", "=", crid) + .execute(); + expect((await h.api("GET", `/files/record-sources?recordId=${crid}`)).body.data.items).toEqual([ + { root: h.dlRoot, path: c.path }, + ]); + }); + + it("successful-history skip includes matched sidecars and does not enqueue work; latest failure does not hide success", async () => { + const s = await source("History"); + const sidecar = s.path.replace(/\.mkv$/, ".srt"); + await writeFile(join(h.dl, sidecar), "subtitle"); + const first = await organize(s.items, { scrapeAfterImport: false }); + await executeRecord(h.module.deps().ao, first.exec.recordIds[0]); + const old = await h.db + .selectFrom("organization_record") + .selectAll() + .where("id", "=", first.exec.recordIds[0]) + .executeTakeFirstOrThrow(); + const laterId = randomUUID(); + await h.db + .insertInto("organization_record") + .values({ + ...old, + id: laterId, + change_seq: undefined, + status: "failed", + reason: "later_failure", + created_at: new Date(Date.now() + 1000), + }) + .execute(); + const main = await h.db + .selectFrom("organization_file_result") + .selectAll() + .where("organization_record_id", "=", old.id) + .where("role", "=", "main") + .executeTakeFirstOrThrow(); + await h.db + .insertInto("organization_file_result") + .values({ ...main, id: randomUUID(), organization_record_id: laterId, status: "failed" }) + .execute(); + const identify = await h.api("POST", "/files/identify", { + items: [{ root: h.dlRoot, path: s.path }], + minSizeBytes: 0, + }); + expect(identify.body.data.items[0].records).toMatchObject({ successfulCount: 1, lastStatus: "failed" }); + const before = h.engine.enqueued.filter((x) => x.task === "verifyAndImport").length; + const second = await organize(s.items, { skipSuccessfulRecords: true, scrapeAfterImport: false }); + expect(second.dry.groups[0].files).toHaveLength(2); + expect( + second.dry.groups[0].files.every( + (f: { action: string; skipReason: string }) => f.action === "skip" && f.skipReason === "successful_record", + ), + ).toBe(true); + expect(second.exec.recordIds).toEqual([]); + expect(h.engine.enqueued.filter((x) => x.task === "verifyAndImport")).toHaveLength(before); + const normal = await h.api("POST", "/files/organize", { + idempotencyKey: key(), + action: "copy", + items: s.items, + minSizeBytes: 0, + dryRun: true, + }); + expect( + normal.body.data.groups[0].files.some((f: { skipReason: string }) => f.skipReason === "successful_record"), + ).toBe(false); + }); + + it.each(["failed", "partial"])("%s records do not count as successful histories", async (status) => { + const s = await source(status); + const first = await organize(s.items, { scrapeAfterImport: false }); + await executeRecord(h.module.deps().ao, first.exec.recordIds[0]); + await h.db + .updateTable("organization_record") + .set({ status: status as "failed" | "partial" }) + .where("id", "=", first.exec.recordIds[0]) + .execute(); + const dry = await h.api("POST", "/files/organize", { + idempotencyKey: key(), + action: "copy", + items: s.items, + minSizeBytes: 0, + dryRun: true, + skipSuccessfulRecords: true, + }); + expect( + dry.body.data.groups[0].files.some((f: { skipReason: string }) => f.skipReason === "successful_record"), + ).toBe(false); + }); + + it("mixed selection skips successful files while organizing the remaining video", async () => { + const s = await source("Mixed"); + const first = await organize(s.items, { scrapeAfterImport: false }); + await executeRecord(h.module.deps().ao, first.exec.recordIds[0]); + const newPath = s.path.replace("1080p", "2160p"); + await writeFile(join(h.dl, newPath), "different new video version"); + const result = await organize([...s.items, { root: h.dlRoot, path: newPath, mediaEntityId: s.entity }], { + skipSuccessfulRecords: true, + scrapeAfterImport: false, + }); + expect( + result.dry.groups[0].files.filter((f: { skipReason: string }) => f.skipReason === "successful_record"), + ).toHaveLength(1); + expect(result.exec.recordIds).toHaveLength(1); + expect((await executeRecord(h.module.deps().ao, result.exec.recordIds[0])).status).toBe("succeeded"); + const files = await h.db + .selectFrom("organization_file_result") + .select("source_path") + .where("organization_record_id", "=", result.exec.recordIds[0]) + .execute(); + expect(files).toEqual([{ source_path: join(h.dl, newPath) }]); + }); + + it("a succeeded record with an unsuccessful main file does not count as successful", async () => { + const s = await source("MainFailed"); + const first = await organize(s.items, { scrapeAfterImport: false }); + await executeRecord(h.module.deps().ao, first.exec.recordIds[0]); + await h.db + .updateTable("organization_file_result") + .set({ status: "failed" }) + .where("organization_record_id", "=", first.exec.recordIds[0]) + .where("role", "=", "main") + .execute(); + const dry = await h.api("POST", "/files/organize", { + idempotencyKey: key(), + action: "copy", + items: s.items, + minSizeBytes: 0, + dryRun: true, + skipSuccessfulRecords: true, + }); + expect( + dry.body.data.groups[0].files.some((f: { skipReason: string }) => f.skipReason === "successful_record"), + ).toBe(false); + }); + + it.each([false, true])( + "TMDB per-import scrape=%s overrides the opposite global setting and survives replay", + async (enabled) => { + await h.db + .insertInto("settings") + .values({ key: "organize", value: JSON.stringify({ scrapeAfterImport: !enabled }) }) + .onConflict((oc) => oc.column("key").doUpdateSet({ value: JSON.stringify({ scrapeAfterImport: !enabled }) })) + .execute(); + const s = await source("TMDB"); + const r = await organize(s.items, { scrapeAfterImport: enabled }); + const rid = r.exec.recordIds[0]; + expect((await executeRecord(h.module.deps().ao, rid)).status).toBe("succeeded"); + const records = () => + h.db.selectFrom("scrape_record").selectAll().where("media_entity_id", "=", s.entity).execute(); + const scraped = await records(); + expect(scraped).toHaveLength(enabled ? 1 : 0); + const byJob = await h.api("GET", `/files/scrape-records?organizationRecordId=${rid}`); + expect(byJob.status).toBe(200); + expect(byJob.body.data.records.map((r: { id: string }) => r.id)).toEqual(scraped.map((r) => r.id)); + if (enabled) { + const id = scraped[0]!.id; + const oldQuery = await h.api("GET", `/files/scrape-records?ids=${id}`); + expect(oldQuery.body.data.records[0].id).toBe(id); + const both = await h.api("GET", `/files/scrape-records?ids=${id}&organizationRecordId=${rid}`); + expect(both.body.data.records[0].id).toBe(id); + const differentJob = await h.api("GET", `/files/scrape-records?ids=${id}&organizationRecordId=${randomUUID()}`); + expect(differentJob.body.data.records).toEqual([]); + } + await executeRecord(h.module.deps().ao, rid); + expect(await records()).toHaveLength(enabled ? 1 : 0); + }, + ); + + it.each([false, true])("Local per-import scrape=%s is respected", async (enabled) => { + await h.db + .insertInto("settings") + .values({ key: "organize", value: JSON.stringify({ scrapeAfterImport: false }) }) + .onConflict((oc) => oc.column("key").doUpdateSet({ value: JSON.stringify({ scrapeAfterImport: false }) })) + .execute(); + const s = await source("Local", true); + const r = await organize(s.items, { scrapeAfterImport: enabled }); + const rid = r.exec.recordIds[0]; + const record = await h.db + .selectFrom("organization_record") + .select("plan") + .where("id", "=", rid) + .executeTakeFirstOrThrow(); + expect(record.plan).toMatchObject({ metadataMode: "local", scrapeAfterImport: enabled }); + await executeRecord(h.module.deps().ao, rid); + expect( + await h.db.selectFrom("scrape_record").select("id").where("media_entity_id", "=", s.entity).execute(), + ).toHaveLength(enabled ? 1 : 0); + }); + it("scrape-records requires an ID list or an organization record", async () => { + expect((await h.api("GET", "/files/scrape-records")).status).toBe(422); + }); +}); diff --git a/apps/server/test/file-manager/organize-overrides.test.ts b/apps/server/test/file-manager/organize-overrides.test.ts new file mode 100644 index 00000000..f9dea4ca --- /dev/null +++ b/apps/server/test/file-manager/organize-overrides.test.ts @@ -0,0 +1,121 @@ +import { mkdir, readdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { executeRecord } from "../../src/modules/acquisition-organize/services/organize.ts"; +import { createFmHarness, type FmHarness, key, pgAvailable, seedEntity } from "./harness.ts"; + +const skip = !(await pgAvailable()); + +describe.skipIf(skip)("files 整理人工集号校验", () => { + let h: FmHarness; + let show: string; + let film: string; + const path = "Some.Show.S01E02.mkv"; + const bytes = "x".repeat(64); + + beforeAll(async () => { + h = await createFmHarness("overrides"); + show = await seedEntity(h.db, { kind: "series", title: "Some Show", year: 2022 }); + film = await seedEntity(h.db, { kind: "movie", title: "Film", year: 2021 }); + await writeFile(join(h.dl, path), bytes); + await mkdir(join(h.dl, "Season")); + await writeFile(join(h.dl, "Season", path), bytes); + }); + afterAll(async () => h?.close()); + + const facts = async () => ({ + entities: await h.db.selectFrom("media_entity").select("id").orderBy("id").execute(), + identities: await h.db.selectFrom("external_identity").select("id").orderBy("id").execute(), + plans: await h.db.selectFrom("artifact").select("id").where("title", "=", "files_plan").orderBy("id").execute(), + records: await h.db.selectFrom("organization_record").select("id").orderBy("id").execute(), + tasks: await h.db.selectFrom("task").select("id").orderBy("id").execute(), + jobs: h.engine.enqueued.length, + library: await readdir(h.lib, { recursive: true }), + }); + + it.each([true, false])("非连续集号在 dryRun=%s 时整批拒绝,不创建作品、计划、记录或文件", async (dryRun) => { + const before = await facts(); + const r = await h.api("POST", "/files/organize", { + idempotencyKey: key(), + action: "copy", + dryRun, + items: [ + // 排在坏输入前的远端候选也不能创建作品或发起 TMDB 请求。 + { root: h.dlRoot, path, external: { provider: "tmdb", externalId: "136315", kind: "tv" } }, + { root: h.dlRoot, path, mediaEntityId: show, season: 2, episodes: [5, 7] }, + ], + }); + expect(r.status).toBe(400); + expect(r.body.error.message).toMatch(/bad_request:.*集号 5,7 不连续/); + expect(await facts()).toEqual(before); + expect(await readFile(join(h.dl, path), "utf8")).toBe(bytes); + }); + + it("远端剧集候选的不连续集号也在建档前拒绝", async () => { + const before = await facts(); + const r = await h.api("POST", "/files/organize", { + idempotencyKey: key(), + action: "copy", + dryRun: true, + items: [ + { root: h.dlRoot, path, external: { provider: "tmdb", externalId: "136315", kind: "tv" }, episodes: [2, 4] }, + ], + }); + expect(r.status).toBe(400); + expect(r.body.error.message).toContain("不连续"); + expect(await facts()).toEqual(before); + }); + + it.each(["local", "external"])("%s 剧集目录指定集号时明确拒绝,不保存计划", async (source) => { + const before = await facts(); + const entity = + source === "local" + ? { mediaEntityId: show } + : { external: { provider: "tmdb", externalId: "136315", kind: "tv" } }; + const r = await h.api("POST", "/files/organize", { + idempotencyKey: key(), + action: "copy", + dryRun: true, + items: [{ root: h.dlRoot, path: "Season", ...entity, season: 1, episodes: [2] }], + }); + expect(r.status).toBe(400); + expect(r.body.error.message).toContain("目录项不能指定集号"); + expect(await facts()).toEqual(before); + }); + + it("连续人工季集覆盖原解析,预览与执行路径一致", async () => { + const items = [{ root: h.dlRoot, path, mediaEntityId: show, season: 2, episodes: [4, 3, 4] }]; + const dry = await h.api("POST", "/files/organize", { idempotencyKey: key(), action: "copy", dryRun: true, items }); + expect(dry.status).toBe(200); + const group = dry.body.data.groups[0]; + expect(group.error).toBeNull(); + expect(group.warnings).toEqual([]); + expect(group.files).toHaveLength(1); + expect(group.files[0].target).toMatch(/Season 2\/Some Show - S02E03-E04 - 第 3-4 集\.mkv$/); + const exec = await h.api("POST", "/files/organize", { + idempotencyKey: key(), + action: "copy", + dryRun: false, + items, + planIds: [group.planId], + }); + expect(exec.status).toBe(200); + expect(exec.body.data.groups[0].error).toBeNull(); + expect(exec.body.data.groups[0].files).toEqual(group.files); + expect(exec.body.data.recordIds).toHaveLength(1); + const result = await executeRecord(h.module.deps().ao, exec.body.data.recordIds[0]); + expect(result.status).toBe("succeeded"); + expect(await readFile(group.files[0].target, "utf8")).toBe(bytes); + }); + + it("电影仍忽略季集覆写并提示", async () => { + const r = await h.api("POST", "/files/organize", { + idempotencyKey: key(), + action: "copy", + dryRun: true, + items: [{ root: h.dlRoot, path, mediaEntityId: film, season: 1, episodes: [2, 4] }], + }); + expect(r.status).toBe(200); + expect(r.body.data.groups[0].warnings).toEqual([expect.stringMatching(/电影不适用季\/集覆写/)]); + }); +}); diff --git a/apps/server/test/file-manager/source-monitor.test.ts b/apps/server/test/file-manager/source-monitor.test.ts new file mode 100644 index 00000000..744d0c91 --- /dev/null +++ b/apps/server/test/file-manager/source-monitor.test.ts @@ -0,0 +1,321 @@ +import { randomUUID } from "node:crypto"; +import { mkdir, rename, stat, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { createQbClient } from "@umt/adapters"; +import type { OrganizeSourceConfig } from "@umt/contracts"; +import { createMockQb, type MockQb } from "@umt/testkit"; +import { afterEach, describe, expect, it } from "vitest"; +import { + deleteOrganizeSource, + listOrganizeSources, + saveOrganizeSource, +} from "../../src/modules/file-manager/organize-sources.ts"; +import { advanceSnapshot, scanOrganizeSource, scanStock } from "../../src/modules/file-manager/source-monitor.ts"; +import { createFmHarness, type FmHarness, pgAvailable, seedDownload, seedEntity } from "./harness.ts"; + +const fact = { size: 100, mtimeMs: 1000 }; +describe("source snapshot observations", () => { + it("baselines historical files and requires two stable observations for new and changed files", () => { + const baseline = advanceSnapshot(null, { "old.mkv": fact }); + expect(baseline.eligible).toEqual([]); + const added = advanceSnapshot(baseline.snapshot, { "old.mkv": fact, "new.mkv": fact }); + expect(added.eligible).toEqual([]); + const stable = advanceSnapshot(added.snapshot, { "old.mkv": fact, "new.mkv": fact }); + expect(stable.eligible).toEqual(["new.mkv"]); + const changed = advanceSnapshot(stable.snapshot, { "old.mkv": { ...fact, size: 200 }, "new.mkv": fact }); + expect(changed.eligible).not.toContain("old.mkv"); + expect(advanceSnapshot(changed.snapshot, { "old.mkv": { ...fact, size: 200 } }).eligible).toEqual(["old.mkv"]); + }); +}); + +const skip = !(await pgAvailable()); +describe.skipIf(skip)("persistent source monitoring", () => { + let h: FmHarness | undefined; + let qb: MockQb | undefined; + afterEach(async () => { + await qb?.close(); + qb = undefined; + await h?.close(); + h = undefined; + }); + const setup = async () => { + qb = await createMockQb(); + h = await createFmHarness("source_monitor", { + ao: { + downloader: async (inst) => + createQbClient({ instanceId: inst.id, baseUrl: qb!.url, username: "admin", password: "adminadmin" }), + }, + }); + const config: Omit = { + name: "来源", + enabled: true, + source: { root: h.dlRoot, path: "" }, + target: { root: h.libRoot, path: "" }, + mediaType: "all", + category: null, + action: "copy", + trigger: "snapshot", + downloaderInstanceId: null, + tag: null, + intervalSec: 60, + }; + const id = randomUUID(); + await saveOrganizeSource(h.module.deps(), id, config); + return { h, id, config, qb }; + }; + it("first scan persists stock baseline without records; stock never changes baseline", async () => { + const { h, id } = await setup(); + await writeFile(join(h.dl, "old.mkv"), "old video content"); + const result = await scanOrganizeSource(h.module.deps(), { sourceId: id }); + expect(result).toMatchObject({ baseline: true, queued: 0 }); + const before = await h.db.selectFrom("organize_source_snapshot").selectAll().executeTakeFirstOrThrow(); + expect(before.baseline_at).not.toBeNull(); + await writeFile(join(h.dl, "new.mkv"), "new video content"); + const stock = await scanStock(h.module.deps(), id); + expect(stock.items.map((f) => f.path).sort()).toEqual(["new.mkv", "old.mkv"]); + const after = await h.db.selectFrom("organize_source_snapshot").selectAll().executeTakeFirstOrThrow(); + expect(after).toEqual(before); + expect(await h.db.selectFrom("organization_record").select("id").execute()).toEqual([]); + expect((await listOrganizeSources(h.module.deps())).items[0]?.fileCount).toBe(1); + }); + it("unavailable source preserves baseline and retry does not import historical files", async () => { + const { h, id } = await setup(); + await writeFile(join(h.dl, "old.mkv"), "old video content"); + await scanOrganizeSource(h.module.deps(), { sourceId: id }); + const before = await h.db.selectFrom("organize_source_snapshot").selectAll().executeTakeFirstOrThrow(); + await rename(h.dl, `${h.dl}-offline`); + await scanOrganizeSource(h.module.deps(), { sourceId: id }); + const failed = await h.db.selectFrom("organize_source_snapshot").selectAll().executeTakeFirstOrThrow(); + expect(failed.snapshot).toEqual(before.snapshot); + expect(failed.baseline_at).toEqual(before.baseline_at); + expect(failed.last_error).not.toBeNull(); + await rename(`${h.dl}-offline`, h.dl); + expect(await scanOrganizeSource(h.module.deps(), { sourceId: id })).toMatchObject({ + baseline: false, + eligible: 0, + queued: 0, + }); + }); + it("an identified stable new file plans and queues import once", async () => { + const { h, id, config } = await setup(); + await saveOrganizeSource(h.module.deps(), id, { ...config, category: "欧美电影" }); + const entityId = await seedEntity(h.db, { kind: "movie", title: "Snapshot Film", year: 2024 }); + await h.db + .insertInto("external_identity") + .values({ + media_entity_id: entityId, + provider: "tmdb", + external_id: "123456", + status: "accepted", + entity_type: "movie", + }) + .execute(); + await scanOrganizeSource(h.module.deps(), { sourceId: id }); + await writeFile(join(h.dl, "Snapshot.Film.2024.tmdb-123456.1080p.BluRay.mkv"), "video content for snapshot import"); + expect(await scanOrganizeSource(h.module.deps(), { sourceId: id })).toMatchObject({ eligible: 0, queued: 0 }); + const stableResult = await scanOrganizeSource(h.module.deps(), { sourceId: id }); + expect(stableResult).toMatchObject({ eligible: 1, queued: 1 }); + const imports = h.engine.enqueued.filter((e) => e.task === "verifyAndImport"); + expect(imports).toHaveLength(1); + expect(imports[0]?.input).toMatchObject({ recordId: expect.any(String) }); + await scanOrganizeSource(h.module.deps(), { sourceId: id }); + expect(h.engine.enqueued.filter((e) => e.task === "verifyAndImport")).toHaveLength(1); + }); + it("new stable files create one scoped decision and never reprocess baseline stock", async () => { + const { h, id } = await setup(); + await writeFile(join(h.dl, "historical.mkv"), "historical video content"); + await scanOrganizeSource(h.module.deps(), { sourceId: id }); + await writeFile(join(h.dl, "unidentified.mkv"), "unidentified video content"); + expect(await scanOrganizeSource(h.module.deps(), { sourceId: id })).toMatchObject({ eligible: 0, queued: 0 }); + expect(await scanOrganizeSource(h.module.deps(), { sourceId: id })).toMatchObject({ eligible: 1, queued: 0 }); + const decisions = await h.db.selectFrom("decision").selectAll().execute(); + expect(decisions).toHaveLength(1); + expect(decisions[0]?.evidence).toMatchObject({ sourceConfigId: id, root: h.dlRoot, path: "unidentified.mkv" }); + await scanOrganizeSource(h.module.deps(), { sourceId: id }); + expect(await h.db.selectFrom("decision").select("id").execute()).toHaveLength(1); + }); + it("active and completed QB tasks remain under the downloader tag policy", async () => { + const { h, id } = await setup(); + await scanOrganizeSource(h.module.deps(), { sourceId: id }); + await writeFile(join(h.dl, "active.mkv"), "active video content"); + await writeFile(join(h.dl, "completed.mkv"), "completed video content"); + await seedDownload(h.db, h, { name: "active.mkv", status: "downloading" }); + await seedDownload(h.db, h, { name: "completed.mkv", status: "completed" }); + await scanOrganizeSource(h.module.deps(), { sourceId: id }); + expect(await scanOrganizeSource(h.module.deps(), { sourceId: id })).toMatchObject({ eligible: 2, queued: 0 }); + expect(await h.db.selectFrom("organization_record").select("id").execute()).toEqual([]); + expect(await scanStock(h.module.deps(), id)).toMatchObject({ + unavailable: 1, + items: [{ root: h.dlRoot, path: "completed.mkv" }], + }); + }); + it("untagged paused QB downloads without DB rows wait for completion before snapshot import", async () => { + const { h, id, qb, config } = await setup(); + await saveOrganizeSource(h.module.deps(), id, { ...config, category: "欧美电影" }); + const entityId = await seedEntity(h.db, { kind: "movie", title: "Snapshot Film", year: 2024 }); + await h.db + .insertInto("external_identity") + .values({ + media_entity_id: entityId, + provider: "tmdb", + external_id: "123456", + status: "accepted", + entity_type: "movie", + }) + .execute(); + await scanOrganizeSource(h.module.deps(), { sourceId: id }); + const name = "Snapshot.Film.2024.tmdb-123456.1080p.BluRay.mkv"; + await writeFile(join(h.dl, name), "stable incomplete video content"); + qb.seed({ + hash: "a".repeat(40), + name, + tags: [], + state: "pausedDL", + progress: 0.5, + amount_left: 50, + save_path: h.dl, + content_path: join(h.dl, name), + }); + expect(await h.db.selectFrom("download_task").select("id").execute()).toEqual([]); + await scanOrganizeSource(h.module.deps(), { sourceId: id }); + expect(await scanOrganizeSource(h.module.deps(), { sourceId: id })).toMatchObject({ eligible: 1, queued: 0 }); + expect(await scanStock(h.module.deps(), id)).toMatchObject({ items: [], unavailable: 1 }); + expect(await h.db.selectFrom("decision").select("id").execute()).toEqual([]); + expect(h.engine.enqueued.filter((e) => e.task === "verifyAndImport")).toEqual([]); + await qb.complete("a".repeat(40)); + expect(await scanOrganizeSource(h.module.deps(), { sourceId: id })).toMatchObject({ eligible: 1, queued: 1 }); + expect(await h.db.selectFrom("download_task").select("id").execute()).toEqual([]); + }); + it("unreachable QB preserves the previous baseline and rejects stock", async () => { + const { h, id, qb } = await setup(); + await writeFile(join(h.dl, "baseline.mkv"), "baseline video content"); + await scanOrganizeSource(h.module.deps(), { sourceId: id }); + const before = await h.db.selectFrom("organize_source_snapshot").selectAll().executeTakeFirstOrThrow(); + qb.fault("torrents/info", { status: 503, times: 10 }); + await writeFile(join(h.dl, "later.mkv"), "later video content"); + expect(await scanOrganizeSource(h.module.deps(), { sourceId: id })).toMatchObject({ queued: 0 }); + const after = await h.db.selectFrom("organize_source_snapshot").selectAll().executeTakeFirstOrThrow(); + expect(after.snapshot).toEqual(before.snapshot); + expect(after.baseline_at).toEqual(before.baseline_at); + expect(after.last_error).toContain("无法查询来源下载器"); + await expect(scanStock(h.module.deps(), id)).rejects.toThrow("无法查询来源下载器"); + }); + it("partial records exclude successful files while leaving failed files available", async () => { + const { h, id } = await setup(); + await writeFile(join(h.dl, "success.mkv"), "successful video content"); + await writeFile(join(h.dl, "failure.mkv"), "failed video content"); + const record = await h.db + .insertInto("organization_record") + .values({ trigger: "manual", actor_type: "system", action: "copy", status: "partial" }) + .returning("id") + .executeTakeFirstOrThrow(); + await h.db + .insertInto("organization_file_result") + .values([ + { + organization_record_id: record.id, + ordinal: 0, + mode: "copy", + source_path: join(h.dl, "success.mkv"), + status: "succeeded", + }, + { + organization_record_id: record.id, + ordinal: 1, + mode: "copy", + source_path: join(h.dl, "failure.mkv"), + status: "failed", + }, + ]) + .execute(); + expect(await scanStock(h.module.deps(), id)).toMatchObject({ + items: [{ root: h.dlRoot, path: "failure.mkv" }], + skipped: 1, + }); + }); + it("historical success excludes only matching source facts, so replacement files remain candidates", async () => { + const { h, id } = await setup(); + const names = ["same.mkv", "size-changed.mkv", "mtime-changed.mkv"]; + await Promise.all(names.map((n) => writeFile(join(h.dl, n), "video content"))); + const facts = Object.fromEntries( + await Promise.all( + names.map(async (n) => { + const st = await stat(join(h.dl, n)); + return [join(h.dl, n), { size: st.size, mtimeMs: st.mtimeMs }]; + }), + ), + ); + facts[join(h.dl, "size-changed.mkv")]!.size -= 1; + facts[join(h.dl, "mtime-changed.mkv")]!.mtimeMs -= 1; + const record = await h.db + .insertInto("organization_record") + .values({ + trigger: "manual", + actor_type: "system", + action: "copy", + status: "succeeded", + plan: JSON.stringify({ sourceFacts: facts }), + }) + .returning("id") + .executeTakeFirstOrThrow(); + await h.db + .insertInto("organization_file_result") + .values( + names.map((n, ordinal) => ({ + organization_record_id: record.id, + ordinal, + mode: "copy" as const, + source_path: join(h.dl, n), + source_size_bytes: facts[join(h.dl, n)]!.size, + status: "succeeded" as const, + })), + ) + .execute(); + const stock = await scanStock(h.module.deps(), id); + expect(stock.skipped).toBe(1); + expect(stock.items.map((f) => f.path).sort()).toEqual(["mtime-changed.mkv", "size-changed.mkv"]); + }); + it("stock batches stay within the identify wizard limit", async () => { + const { h, id } = await setup(); + await Promise.all(Array.from({ length: 201 }, (_, n) => writeFile(join(h.dl, `stock-${n}.mkv`), "video content"))); + const stock = await scanStock(h.module.deps(), id); + expect(stock.items).toHaveLength(200); + expect(stock.truncated).toBe(true); + expect(await h.db.selectFrom("organize_source_snapshot").selectAll().execute()).toEqual([]); + }); + it("stock omits successfully organized files", async () => { + const { h, id } = await setup(); + const path = join(h.dl, "organized.mkv"); + await writeFile(path, "organized video content"); + const record = await h.db + .insertInto("organization_record") + .values({ trigger: "manual", actor_type: "system", action: "copy", status: "succeeded" }) + .returning("id") + .executeTakeFirstOrThrow(); + await h.db + .insertInto("organization_file_result") + .values({ organization_record_id: record.id, mode: "copy", source_path: path, status: "succeeded" }) + .execute(); + expect(await scanStock(h.module.deps(), id)).toMatchObject({ items: [], skipped: 1 }); + }); + it("source edits reset baseline; identical categories coexist; deleting removes state", async () => { + const { h, id, config } = await setup(); + await mkdir(join(h.dl, "other")); + await scanOrganizeSource(h.module.deps(), { sourceId: id }); + await saveOrganizeSource(h.module.deps(), randomUUID(), { ...config, category: "电影" }); + await saveOrganizeSource(h.module.deps(), id, { + ...config, + category: "电影", + source: { ...config.source, path: "other" }, + }); + expect((await listOrganizeSources(h.module.deps())).items).toHaveLength(2); + expect( + await h.db.selectFrom("organize_source_snapshot").selectAll().where("source_id", "=", id).executeTakeFirst(), + ).toBeUndefined(); + await scanOrganizeSource(h.module.deps(), { sourceId: id }); + await deleteOrganizeSource(h.module.deps(), id); + expect( + await h.db.selectFrom("organize_source_snapshot").selectAll().where("source_id", "=", id).executeTakeFirst(), + ).toBeUndefined(); + }); +}); diff --git a/apps/server/test/file-manager/w5a2.test.ts b/apps/server/test/file-manager/w5a2.test.ts index 63b88544..9ded14e2 100644 --- a/apps/server/test/file-manager/w5a2.test.ts +++ b/apps/server/test/file-manager/w5a2.test.ts @@ -224,7 +224,7 @@ describe.skipIf(skip)("files W5-A2", () => { dryRun: true, items: [ { root: h.dlRoot, path: "weird-a.mkv", mediaEntityId: series, season: 2, episodes: [3, 4] }, - { root: h.dlRoot, path: "weird-b.mkv", mediaEntityId: series, season: 2, episodes: [5, 7] }, + { root: h.dlRoot, path: "weird-b.mkv", mediaEntityId: series, season: 2, episodes: [5, 6] }, { root: h.dlRoot, path: "Film.2019.1080p.mkv", @@ -243,8 +243,8 @@ describe.skipIf(skip)("files W5-A2", () => { const mv = groups.find((g: { mediaEntityId: string }) => g.mediaEntityId === film); const tvTargets = tv.files.map((f: { target: string | null }) => f.target); expect(tvTargets.some((t: string | null) => t?.endsWith("Season 2/剧 - S02E03-E04 - 第 3-4 集.mkv"))).toBe(true); - // 非连续集号不应用(只按季覆写,文件名无集号 → 计划排除),其余覆写不进 warnings - expect(tv.warnings).toEqual([expect.stringMatching(/weird-b\.mkv:集号 5,7 不连续/)]); + expect(tvTargets.some((t: string | null) => t?.endsWith("Season 2/剧 - S02E05-E06 - 第 5-6 集.mkv"))).toBe(true); + expect(tv.warnings).toEqual([]); expect(mv.warnings).toEqual([expect.stringMatching(/电影不适用季\/集覆写/)]); expect(mv).toMatchObject({ action: "copy", slotId: slot.id, targetDir: join(h.lib, "picked") }); expect(mv.files[0].target).toBe(join(h.lib, "picked", "Film (2019) - 1080p.mkv")); diff --git a/apps/server/test/organize-in-place-verify.test.ts b/apps/server/test/organize-in-place-verify.test.ts new file mode 100644 index 00000000..acd64836 --- /dev/null +++ b/apps/server/test/organize-in-place-verify.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it, vi } from "vitest"; + +// 不挂载 HTTP 路由;隔离 Deno 静态文件适配器,以便本单元测试在 Node 下执行。 +vi.mock("hono/deno", () => ({ serveStatic: vi.fn() })); + +import { verifyInPlace } from "../src/modules/acquisition-organize/services/organize.ts"; + +const path = "/media/unchanged.mkv"; +const frozen = { + rootPath: "/media", + sentinel: ".uhd-root", + libraryRootId: "root", + entityId: "work", + sourceFacts: { [path]: { size: 100, mtimeMs: 200 } }, +} as unknown as Parameters[2]; +async function verify(observed: unknown, owner: string | null = null) { + const executeTakeFirst = vi.fn().mockResolvedValue(owner ? { media_entity_id: owner } : undefined); + const query = { innerJoin: () => query, select: () => query, where: () => query, executeTakeFirst }; + const d = { ctx: { db: { selectFrom: () => query } } } as unknown as Parameters[0]; + const statSafe = vi.fn().mockResolvedValue(observed); + // 仅提供 statSafe;若意外调用复制、移动、清理或写入,本测试立即失败。 + const fsr = { proxy: { statSafe } } as unknown as Parameters[1]; + return { result: await verifyInPlace(d, fsr, frozen, path, 100), statSafe, executeTakeFirst }; +} +const observed = (extra = {}) => ({ + kind: "ok", + facts: { isFile: true, isSymbolicLink: false, size: 100, mtimeMs: 200, ...extra }, +}); +describe("原地登记执行核验", () => { + it("只读核验,不复制或移动文件;同作品重复登记可继续", async () => { + const r = await verify(observed(), "work"); + expect(r.result).toEqual({ status: "succeeded", bytes: 0 }); + expect(r.statSafe).toHaveBeenCalledWith(path, { root: "/media", sentinel: ".uhd-root" }); + }); + it("文件消失、链接、变为目录或计划后的大小/时间变化均不登记", async () => { + for (const value of [ + { kind: "missing" }, + observed({ isSymbolicLink: true }), + observed({ isFile: false }), + observed({ size: 101 }), + observed({ mtimeMs: 201 }), + ]) { + const r = await verify(value); + expect(r.result.status).toBe("failed"); + expect(r.executeTakeFirst).not.toHaveBeenCalled(); + } + }); + it("不会把已有视频悄悄登记为另一个作品", async () => { + expect((await verify(observed(), "other")).result).toMatchObject({ + status: "failed", + error: { code: "ASSET_ENTITY_CONFLICT" }, + }); + }); +}); diff --git a/apps/server/test/pi/upstreams.test.ts b/apps/server/test/pi/upstreams.test.ts index 027c3c69..8f34aeef 100644 --- a/apps/server/test/pi/upstreams.test.ts +++ b/apps/server/test/pi/upstreams.test.ts @@ -102,7 +102,7 @@ describe.skipIf(!hasPg)("Pi 上游(真实 PG)", () => { .plaintextKey as string; expect((await bearer(h.app, readKey)("GET", "/pi/upstreams")).status).toBe(403); const cat = await json(await key("GET", "/pi/providers?models=false")); - expect(cat.data.providers).toHaveLength(43); + expect(cat.data.providers).toHaveLength(44); const anthropic = cat.data.providers.find((p: { id: string }) => p.id === "anthropic"); expect(anthropic.termsNote).toMatch(/封禁/); }); diff --git a/apps/server/test/search/search.test.ts b/apps/server/test/search/search.test.ts index b75f2c72..45c449ef 100644 --- a/apps/server/test/search/search.test.ts +++ b/apps/server/test/search/search.test.ts @@ -15,6 +15,7 @@ import { createOutboxPort } from "../../src/context.ts"; import { createApp } from "../../src/http/app.ts"; import { EventBus } from "../../src/http/sse.ts"; import { createAcquisitionOrganizeModule } from "../../src/modules/acquisition-organize/index.ts"; +import { createCapabilityRegistry } from "../../src/modules/agent/capability-registry.ts"; import { createAgentModule } from "../../src/modules/agent/index.ts"; import { createSearchModule } from "../../src/modules/search/index.ts"; import { type BaseRelease, filterReleases, quickFilter } from "../../src/modules/search/releases.ts"; @@ -149,7 +150,7 @@ describe.skipIf(!up)("全局搜索", () => { outbox: createOutboxPort(), secrets, settings: plainSettings(() => db), - capabilities: {} as never, + capabilities: createCapabilityRegistry({ logger }), } as ServerContext; // 挂起的站点:请求一直不返回,直到被中止(单站超时) const fetchImpl: typeof fetch = (input, init) => { @@ -208,6 +209,28 @@ describe.skipIf(!up)("全局搜索", () => { await addAccount(sites.mteam, "api_key", mteam.apiKey); sites.slow = await addSite("hdsky", "天空", "nexusphp", `http://${SLOW_HOST}/`); await addAccount(sites.slow, "cookie", "uid=1; pass=x"); + const downloader = await db + .insertInto("downloader_instance") + .values({ + name: "search-download", + base_url: "http://downloader.invalid", + cache_volume_key: "search-cache", + settings: JSON.stringify({ savePath: "/downloads" }), + }) + .returning("id") + .executeTakeFirstOrThrow(); + await db + .insertInto("path_mapping") + .values({ + downloader_instance_id: downloader.id, + view_kind: "downloader", + remote_prefix: "/downloads", + local_prefix: "/tmp/umt-search-cache", + media_server_instance_id: null, + library_root_id: null, + verified_at: null, + }) + .execute(); // 已关闭的站点不参与搜索 await db .insertInto("site") @@ -561,12 +584,16 @@ describe.skipIf(!up)("全局搜索", () => { }); it("从搜索结果下载:转调 /acquisition-plans(同一鉴权、同一拒绝原因)", async () => { - // 未识别的资源 → 与 POST /acquisition-plans 相同的 412 + // 未识别资源先返回一次结构化确认,不阻断后续手动下载 const unresolved = await req(`/search/releases/${sites.nexus}/900001/plan`, { body: {} }); - expect(unresolved.status).toBe(412); - expect(unresolved.body.error.code).toBe("precondition_failed"); + expect(unresolved.status, JSON.stringify(unresolved.body)).toBe(200); + expect(unresolved.body.data).toMatchObject({ + submitted: false, + requiresConfirmation: true, + rejected: [{ code: "unrecognized" }], + }); - // 已识别但作品没有收录版本 → submitted=false + no_slot + // 已识别资源无需订阅或收录版本,独立计划提交 const other = ( await db .insertInto("media_entity") @@ -582,7 +609,13 @@ describe.skipIf(!up)("全局搜索", () => { .execute(); const ok = await req(`/search/releases/${sites.mteam}/912234/plan`, { body: { note: "from search" } }); expect(ok.status).toBe(200); - expect(ok.body.data).toMatchObject({ submitted: false, plan: null, rejected: [{ code: "no_slot" }] }); + expect(ok.body.data).toMatchObject({ submitted: true, slotId: null, plan: { slotId: null }, rejected: [] }); + expect(await db.selectFrom("subscription").selectAll().where("media_entity_id", "=", other).execute()).toHaveLength( + 0, + ); + expect( + await db.selectFrom("acquisition_slot").selectAll().where("media_entity_id", "=", other).execute(), + ).toHaveLength(0); // 不在搜索结果里 const missing = await req(`/search/releases/${sites.mteam}/nope/plan`, { body: {} }); diff --git a/apps/server/test/sources-identity/id-escalation.test.ts b/apps/server/test/sources-identity/id-escalation.test.ts index 23e8a712..49e20a77 100644 --- a/apps/server/test/sources-identity/id-escalation.test.ts +++ b/apps/server/test/sources-identity/id-escalation.test.ts @@ -316,6 +316,28 @@ describe.skipIf(!pg)("ID-3 识别升级链:详情页 ID", () => { expect(siteRequests.some((x) => x.includes("douban"))).toBe(false); }); + it("升级链没认出(豆瓣召回无匹配)时保留原有歧义:候选不丢,照常开有歧义的事项", async () => { + await saveDoubanSubject(h.ctx, { + id: "36000097", + title: "查无此片", + aliases: [], + // 年份对不上:按豆瓣条目召回 TMDB 拿不到候选 + year: 1990, + source: "web_snippet", + ref: "x", + at: new Date().toISOString(), + }); + const r = await release(audiences, "725859", "Rose 2026 2160p WEB-DL H265-ADWeb", { + _detail: { doubanId: "36000097" }, + }); + await h.db.updateTable("source_release").set({ detail_fetched_at: new Date() }).where("id", "=", r.id).execute(); + const res = await resolveRelease(h.rt, { sourceReleaseId: r.id, trigger: "manual" }); + expect(res.outcome).toBe("ambiguous"); + expect(res.decisionId).not.toBeNull(); + const d = await h.db.selectFrom("decision").select("kind").where("id", "=", res.decisionId!).executeTakeFirst(); + expect(d?.kind).toBe("identity_ambiguous"); + }); + it("池条目补全前先走升级链:详情页给了 ID 就直接认出,返回 409,不做网页补全", async () => { const r = await release(hhan, "91007", "Example Show S01 2026 2160p YK WEB-DL H.265 DV HQ AAC2.0-HHWEB"); const first = await resolveRelease(h.rt, { sourceReleaseId: r.id, trigger: "manual", escalate: false }); diff --git a/apps/server/test/sources-identity/organize-metadata.test.ts b/apps/server/test/sources-identity/organize-metadata.test.ts new file mode 100644 index 00000000..3968fc07 --- /dev/null +++ b/apps/server/test/sources-identity/organize-metadata.test.ts @@ -0,0 +1,223 @@ +import { workNfo } from "@umt/domain"; +import { createMockTmdb } from "@umt/testkit"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { imagePathsOf, nfoWork } from "../../src/modules/acquisition-organize/services/scrape-input.ts"; +import { saveDetailPage } from "../../src/modules/sources-identity/service/detail-page.ts"; +import { + ptDetailFacts, + resolveOrganizeMetadata, +} from "../../src/modules/sources-identity/service/organize-metadata.ts"; +import { createHarness, type Harness, pgAvailable, seedSite } from "./harness.ts"; + +const ptText = + "◎译 名 未收录的故事\n◎原 名 Unlisted Story\n◎年 代 2025\n◎产 地 中国大陆\n◎语 言 汉语普通话\n◎类 别 剧情\n◎导 演 张三\n◎主 演 李四 / 王五\n◎简 介 一位主人公在旅途中发现了一段不为人知的故事,并开始寻找失踪的朋友。"; +const up = await pgAvailable(); +describe("PT detail extraction", () => { + it("reads evidenced labels and leaves missing fields empty", () => { + const facts = ptDetailFacts(ptText, "https://pt.example/details.php?id=1"); + expect(facts.names).toEqual(["未收录的故事", "Unlisted Story"]); + expect(facts.year).toBe(2025); + expect(facts.extras).toContainEqual({ field: "originCountry", value: ["CN"] }); + expect(facts.extras).toContainEqual({ field: "originalLanguage", value: "zh" }); + expect(ptDetailFacts("Unknown.2025.1080p", "https://pt.example/").candidates).toEqual([]); + }); + it("local final fields reuse NFO input and external poster", () => { + const metadata = { + overview: { value: "简介" }, + posterUrl: { value: "https://img.example/cover.jpg" }, + originCountry: { value: ["CN"] }, + cast: { value: [{ id: -1, name: "本地主演", role: null, profilePath: null }] }, + crew: { value: [{ id: -2, name: "本地导演", role: "Director", profilePath: null }] }, + }; + expect(imagePathsOf(metadata).posterPath).toBe("https://img.example/cover.jpg"); + const nfo = nfoWork({ kind: "movie", title: "本地作品", year: 2025, original_title: null, metadata }, {}); + expect(nfo.ids.tmdb).toBeNull(); + expect(nfo.overview).toBe("简介"); + expect(nfo.countries).toEqual(["中国"]); + expect(nfo.actors?.[0]?.tmdbId).toBeNull(); + expect(nfo.directors?.[0]?.tmdbId).toBeNull(); + const xml = workNfo(nfo); + expect(xml).toContain("本地主演"); + expect(xml).toContain("本地导演"); + expect(xml).not.toContain("-1"); + expect(xml).not.toContain("-2"); + }); +}); +describe.skipIf(!up)("organize metadata local recovery (isolated PG)", () => { + let h: Harness; + let siteId: string; + beforeAll(async () => { + h = await createHarness("local_metadata", { tmdb: () => null }); + siteId = (await seedSite(h, { baseUrl: "https://pt.example/" })).site.id; + }); + afterAll(async () => { + await h?.close(); + }); + const add = async (n: string, text: string, title?: string) => { + const release = await h.db + .insertInto("source_release") + .values({ + site_id: siteId, + site_release_id: n, + title: title ?? `Unlisted Story ${n} 2025 1080p`, + detail_fetched_at: new Date(), + }) + .returning("id") + .executeTakeFirstOrThrow(); + await saveDetailPage( + h.rt, + h.db, + release.id, + { + descriptionText: text, + mediaInfo: null, + screenshots: ["https://img.example/cover.jpg"], + imdbId: null, + doubanId: null, + tmdb: null, + files: [], + tags: [], + uploader: null, + probe: { labels: [], hasDescription: true, imageCount: 1 }, + }, + 3600, + ); + return release.id; + }; + const deps = { + search: async () => [], + fetch: async () => { + throw new Error("unexpected fetch"); + }, + complete: async () => null, + }; + it("creates Local with sourced fields, remains revisitable and reuses identity", async () => { + const id = await add("1", ptText); + const out = await resolveOrganizeMetadata(h.rt, id, deps, { actor: { type: "system", id: "system:organize" } }); + expect(out.mode).toBe("local"); + expect(out.mediaEntityId).toBeTruthy(); + const entity = await h.db + .selectFrom("media_entity") + .selectAll() + .where("id", "=", out.mediaEntityId!) + .executeTakeFirstOrThrow(); + expect(entity.status).toBe("provisional"); + expect(entity.title).toBe("未收录的故事"); + expect((entity.metadata as Record).originCountry?.value).toEqual(["CN"]); + expect((entity.metadata as Record).posterUrl?.value).toBe( + "https://img.example/cover.jpg", + ); + const release = await h.db.selectFrom("source_release").selectAll().where("id", "=", id).executeTakeFirstOrThrow(); + expect(release.resolution_locked).toBe(false); + const again = await resolveOrganizeMetadata(h.rt, id, deps, { actor: { type: "system", id: "system:organize" } }); + expect(again.mediaEntityId).toBe(out.mediaEntityId); + }); + it("series fallback preserves actual season and episode instead of fabricating season one", async () => { + const id = await add( + "season", + "◎译 名 独立剧集\n◎年 代 2025\n◎集 数 12\n◎类 别 剧情", + "Distinct Local Series S02E03 2025 1080p", + ); + const out = await resolveOrganizeMetadata(h.rt, id, deps, { actor: { type: "system", id: "system:organize" } }); + expect(out.mode).toBe("local"); + const units = await h.db + .selectFrom("content_unit") + .select(["season_number", "episode_number"]) + .where("media_entity_id", "=", out.mediaEntityId!) + .execute(); + expect(units.some((u) => u.season_number === 1)).toBe(false); + expect(units.some((u) => u.season_number === 2 && u.episode_number === 3)).toBe(true); + }); + it("does not create a filename placeholder when facts are insufficient", async () => { + const id = await add("2", "no metadata available"); + const out = await resolveOrganizeMetadata(h.rt, id, deps, { actor: { type: "system", id: "system:organize" } }); + expect(out.mediaEntityId).toBeNull(); + }); +}); + +describe.skipIf(!up)("organize metadata translated name recovery", () => { + it("retries TMDB using evidenced PT aliases after normal title lookup fails", async () => { + const tmdb = await createMockTmdb({ + fixtures: [ + { + type: "movie", + id: 990087, + original_title: "True Original", + original_language: "en", + origin_country: ["US"], + date: "2025-01-01", + titles: { "zh-CN": "正确译名", "en-US": "True Original" }, + popularity: 10, + alternative_titles: [], + external_ids: { imdb_id: null }, + }, + ], + }); + const h = await createHarness("local_alias", { tmdbUrl: tmdb.url }); + try { + const siteId = (await seedSite(h, { baseUrl: "https://pt.example/" })).site.id; + const release = await h.db + .insertInto("source_release") + .values({ + site_id: siteId, + site_release_id: "alias", + title: "Unknown Translation 2025 1080p", + detail_fetched_at: new Date(), + }) + .returning("id") + .executeTakeFirstOrThrow(); + await saveDetailPage( + h.rt, + h.db, + release.id, + { + descriptionText: "◎译 名 另类译名\n◎原 名 True Original\n◎年 代 2025\n◎类 别 剧情", + mediaInfo: null, + screenshots: [], + imdbId: null, + doubanId: null, + tmdb: null, + files: [], + tags: [], + uploader: null, + probe: { labels: [], hasDescription: true, imageCount: 0 }, + }, + 3600, + ); + let completions = 0; + const out = await resolveOrganizeMetadata( + h.rt, + release.id, + { + search: async () => [], + fetch: async () => { + throw new Error("unexpected fetch"); + }, + complete: async () => { + completions++; + return { text: "{}", model: "fixture-agent", inputTokens: 100, outputTokens: 5 }; + }, + }, + { actor: { type: "system", id: "system:organize" } }, + ); + expect(completions).toBe(1); + expect(out.mode).toBe("tmdb"); + const identity = await h.db + .selectFrom("external_identity") + .selectAll() + .where("media_entity_id", "=", out.mediaEntityId!) + .where("provider", "=", "tmdb") + .executeTakeFirstOrThrow(); + expect(identity.external_id).toBe("990087"); + const enrichment = await h.db + .selectFrom("metadata_enrichment") + .selectAll() + .where("id", "=", out.enrichmentId!) + .executeTakeFirstOrThrow(); + expect((enrichment.usage as { inputTokens: number }).inputTokens).toBe(100); + } finally { + await h.close(); + await tmdb.close(); + } + }); +}); diff --git a/apps/server/test/sources-identity/site-capabilities.test.ts b/apps/server/test/sources-identity/site-capabilities.test.ts index dc3ce896..e6bba7ad 100644 --- a/apps/server/test/sources-identity/site-capabilities.test.ts +++ b/apps/server/test/sources-identity/site-capabilities.test.ts @@ -15,8 +15,9 @@ const TiB = 1024 ** 4; const SITE_IDS = ["site.list", "site.health_check", "site.refresh_login", "site.user_data"] as const; describe("站点能力规格", () => { - it("进入首批能力目录(17 + 5 只读摘要 + 4 站点 + 10 浏览器 + 2 会话交互 + 2 网页搜索 + 1 元数据补全 = 41);类别 / 风险 / 重放按约定", () => { - expect(CAPABILITY_IDS).toHaveLength(41); + it("进入能力目录(41 个已有能力 + 整理元数据补全);类别 / 风险 / 重放按约定", () => { + expect(CAPABILITY_IDS).toHaveLength(42); + expect(capabilitySpec("metadata.resolve_organize")).toMatchObject({ kind: "execute", replay: "never" }); for (const id of SITE_IDS) expect(CAPABILITY_IDS).toContain(id); expect(capabilitySpec("site.list")).toMatchObject({ kind: "read", riskLevel: "low", replay: "safe" }); expect(capabilitySpec("site.user_data")).toMatchObject({ kind: "read", riskLevel: "low", replay: "safe" }); diff --git a/apps/server/vitest.config.ts b/apps/server/vitest.config.ts index 0b07902d..eb79250c 100644 --- a/apps/server/vitest.config.ts +++ b/apps/server/vitest.config.ts @@ -8,6 +8,7 @@ import { defineConfig } from "vitest/config"; // 新测试文件缺省归入 pg(串行,安全);确认不碰 PG 的才加进 UNIT。 const UNIT = [ "test/api-registry.test.ts", + "test/organize-in-place-verify.test.ts", "test/file-manager/preview-parse.test.ts", "test/auth.test.ts", "test/health.test.ts", diff --git a/apps/web/app/api/api-keys.ts b/apps/web/app/api/api-keys.ts index 9493740d..2efd1055 100644 --- a/apps/web/app/api/api-keys.ts +++ b/apps/web/app/api/api-keys.ts @@ -2,11 +2,15 @@ import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import type { ApiKey, ApiKeyCreate, ApiKeyCreated, ApiKeyUpdate, CursorPage } from "@umt/contracts"; import { apiData } from "~/lib/api.ts"; import { qk } from "~/lib/query-keys.ts"; +import { fetchAllCursorPages } from "./parse.ts"; export function useApiKeys() { return useQuery({ queryKey: qk.list("api-keys"), - queryFn: ({ signal }) => apiData>("/api-keys", { params: { limit: 200 }, signal }), + queryFn: ({ signal }) => + fetchAllCursorPages((cursor) => + apiData>("/api-keys", { params: { limit: 200, cursor }, signal }), + ), }); } diff --git a/apps/web/app/api/dashboard.ts b/apps/web/app/api/dashboard.ts index f044f62f..08d740e1 100644 --- a/apps/web/app/api/dashboard.ts +++ b/apps/web/app/api/dashboard.ts @@ -2,7 +2,7 @@ import { useQuery } from "@tanstack/react-query"; import { cursorPage, type Dashboard, SiteListItem } from "@umt/contracts"; import { apiData } from "~/lib/api.ts"; import { qk } from "~/lib/query-keys.ts"; -import { apiParsed } from "./parse.ts"; +import { apiParsed, fetchAllCursorPages } from "./parse.ts"; /** 首页聚合;SSE 事件触发 invalidate,另有 60 秒兜底轮询 */ export function useDashboard() { @@ -20,7 +20,9 @@ export function useSitesWithAccounts() { return useQuery({ queryKey: [...qk.dashboard, "sites-accounts"], queryFn: ({ signal }) => - apiParsed(SitePage, "/api/v1/sites", { params: { limit: 200, include: "accounts" }, signal }), + fetchAllCursorPages((cursor) => + apiParsed(SitePage, "/api/v1/sites", { params: { limit: 200, cursor, include: "accounts" }, signal }), + ), refetchInterval: 60_000, }); } diff --git a/apps/web/app/api/downloads.ts b/apps/web/app/api/downloads.ts index af8dec7c..143e7200 100644 --- a/apps/web/app/api/downloads.ts +++ b/apps/web/app/api/downloads.ts @@ -13,7 +13,7 @@ import { } from "@umt/contracts"; import { z } from "zod"; import { TTL_SECOND } from "~/lib/api.ts"; -import { apiMutate, apiParsed, cleanParams } from "./parse.ts"; +import { apiMutate, apiParsed, cleanParams, fetchAllCursorPages } from "./parse.ts"; // 下载任务、计划(界面统称"下载")、缓存卷(W2-C:/download-tasks、/acquisition-plans、/download-cache/cleanup、/dashboard) @@ -181,7 +181,12 @@ export function useDownloaderNames() { queryKey: ["downloads", "instances"], staleTime: 5 * 60 * TTL_SECOND, queryFn: ({ signal }) => - apiParsed(cursorPage(DownloaderInstance), "/api/v1/downloader-instances", { params: { limit: 200 }, signal }), + fetchAllCursorPages((cursor) => + apiParsed(cursorPage(DownloaderInstance), "/api/v1/downloader-instances", { + params: { limit: 200, cursor }, + signal, + }), + ), select: (d) => new Map(d.items.map((i) => [i.id, i.name])), }); } diff --git a/apps/web/app/api/engine.ts b/apps/web/app/api/engine.ts index 8f2c0c5e..76293a38 100644 --- a/apps/web/app/api/engine.ts +++ b/apps/web/app/api/engine.ts @@ -9,6 +9,7 @@ export type EngineRunFilters = { status?: string; /** 触发来源,单值或逗号分隔多值(user,api_key) */ triggeredBy?: string; + /** 单值或逗号分隔多个工作流名称 */ workflowName?: string; businessKind?: EngineBusinessKind; businessRefId?: string; diff --git a/apps/web/app/api/feeds.ts b/apps/web/app/api/feeds.ts index 9f8e78f7..15156b1e 100644 --- a/apps/web/app/api/feeds.ts +++ b/apps/web/app/api/feeds.ts @@ -10,7 +10,7 @@ import { } from "@umt/contracts"; import { z } from "zod"; import { TTL_SECOND } from "~/lib/api.ts"; -import { apiMutate, apiParsed, cleanParams } from "./parse.ts"; +import { apiMutate, apiParsed, cleanParams, fetchAllCursorPages } from "./parse.ts"; // Feed 来源(W2-B:/feed-sources) @@ -25,7 +25,9 @@ export function useFeeds(f: { siteId?: string; health?: FeedHealth } = {}) { return useQuery({ queryKey: feedKeys.list(f), queryFn: ({ signal }) => - apiParsed(FeedPage, "/api/v1/feed-sources", { params: cleanParams({ ...f, limit: 200 }), signal }), + fetchAllCursorPages((cursor) => + apiParsed(FeedPage, "/api/v1/feed-sources", { params: cleanParams({ ...f, limit: 200, cursor }), signal }), + ), refetchInterval: 30 * TTL_SECOND, }); } diff --git a/apps/web/app/api/instances.ts b/apps/web/app/api/instances.ts index ceb5f052..9742fe42 100644 --- a/apps/web/app/api/instances.ts +++ b/apps/web/app/api/instances.ts @@ -24,14 +24,16 @@ import type { } from "@umt/contracts"; import { apiData } from "~/lib/api.ts"; import { type QueryArea, qk } from "~/lib/query-keys.ts"; +import { fetchAllCursorPages } from "./parse.ts"; -/** 实例类资源的标准 CRUD 钩子(列表取前 200 条,首版实例数量很少) */ +/** 实例类资源的标准 CRUD 钩子(列表按游标取齐配置) */ function crudHooks(area: QueryArea, base: string) { return { useList: () => useQuery({ queryKey: qk.list(area), - queryFn: ({ signal }) => apiData>(base, { params: { limit: 200 }, signal }), + queryFn: ({ signal }) => + fetchAllCursorPages((cursor) => apiData>(base, { params: { limit: 200, cursor }, signal })), }), useCreate: () => { const qc = useQueryClient(); @@ -100,14 +102,18 @@ export const pathMappings = { export function useSiteOptions() { return useQuery({ queryKey: qk.list("sites", { limit: 200 }), - queryFn: ({ signal }) => apiData>("/sites", { params: { limit: 200 }, signal }), + queryFn: ({ signal }) => + fetchAllCursorPages((cursor) => apiData>("/sites", { params: { limit: 200, cursor }, signal })), }); } export function useSiteAccounts() { return useQuery({ queryKey: qk.list("site-accounts"), - queryFn: ({ signal }) => apiData>("/site-accounts", { params: { limit: 200 }, signal }), + queryFn: ({ signal }) => + fetchAllCursorPages((cursor) => + apiData>("/site-accounts", { params: { limit: 200, cursor }, signal }), + ), }); } diff --git a/apps/web/app/api/library.ts b/apps/web/app/api/library.ts index ad48ff74..96682f10 100644 --- a/apps/web/app/api/library.ts +++ b/apps/web/app/api/library.ts @@ -16,6 +16,7 @@ import { } from "@umt/contracts"; import type { z } from "zod"; import { ApiRequestError, api, type QueryParams } from "~/lib/api.ts"; +import { fetchAllCursorPages } from "./parse.ts"; // 媒体库:列表 / 详情 / 作品详情(内容单元、别名、字段来源)与作品级操作。 // 所有响应都按 contracts 的 zod schema 校验,结构不符按错误处理(不静默渲染半截数据)。 @@ -106,7 +107,8 @@ const RootPage = cursorPage(LibraryRoot); export function useLibraryRoots() { return useQuery({ queryKey: libraryKeys.roots, - queryFn: ({ signal }) => getParsed(RootPage, "/library-roots", { limit: 200 }, signal), + queryFn: ({ signal }) => + fetchAllCursorPages((cursor) => getParsed(RootPage, "/library-roots", { limit: 200, cursor }, signal)), staleTime: 5 * 60_000, }); } diff --git a/apps/web/app/api/organize-sources.ts b/apps/web/app/api/organize-sources.ts new file mode 100644 index 00000000..29fc8e59 --- /dev/null +++ b/apps/web/app/api/organize-sources.ts @@ -0,0 +1,54 @@ +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { + ClassificationConfigView, + type OrganizeSourceInput, + OrganizeSourceList, + OrganizeSourceView, + OrganizeStockResult, +} from "@umt/contracts"; +import { z } from "zod"; +import { apiParsed } from "./parse.ts"; + +export function useOrganizeSources() { + return useQuery({ + queryKey: ["organize-sources"], + queryFn: ({ signal }) => apiParsed(OrganizeSourceList, "/organize-sources", { signal }), + refetchInterval: 10_000, + }); +} +export function useSaveOrganizeSource() { + const qc = useQueryClient(); + return useMutation({ + mutationFn: ({ id, body }: { id: string; body: OrganizeSourceInput }) => + apiParsed(OrganizeSourceView, `/organize-sources/${id}`, { method: "PUT", body }), + onSuccess: () => qc.invalidateQueries({ queryKey: ["organize-sources"] }), + }); +} +export function useRemoveOrganizeSource() { + const qc = useQueryClient(); + return useMutation({ + mutationFn: (id: string) => + apiParsed(z.object({ deleted: z.boolean() }), `/organize-sources/${id}`, { method: "DELETE" }), + onSuccess: () => qc.invalidateQueries({ queryKey: ["organize-sources"] }), + }); +} +export function useOrganizeStock() { + return useMutation({ + mutationFn: ({ id, path }: { id: string; path?: string }) => + apiParsed(OrganizeStockResult, `/organize-sources/${id}/stock`, { method: "POST", body: { path } }), + }); +} +export function useClassification() { + return useQuery({ + queryKey: ["classification"], + queryFn: ({ signal }) => apiParsed(ClassificationConfigView, "/classification", { signal }), + }); +} +export function useSaveClassification() { + const qc = useQueryClient(); + return useMutation({ + mutationFn: (yaml: string) => + apiParsed(ClassificationConfigView, "/classification", { method: "PUT", body: { yaml } }), + onSuccess: (data) => qc.setQueryData(["classification"], data), + }); +} diff --git a/apps/web/app/api/parse.test.ts b/apps/web/app/api/parse.test.ts new file mode 100644 index 00000000..78dcd6d9 --- /dev/null +++ b/apps/web/app/api/parse.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it, vi } from "vitest"; +import { fetchAllCursorPages } from "./parse.ts"; + +describe("fetchAllCursorPages", () => { + it("keeps configuration rows beyond the first cursor page", async () => { + const fetchPage = vi.fn(async (cursor?: string) => + cursor + ? { items: [201, 202], nextCursor: null, total: 202 } + : { items: Array.from({ length: 200 }, (_, i) => i + 1), nextCursor: "page-2", total: 202 }, + ); + const result = await fetchAllCursorPages(fetchPage); + expect(result.items).toHaveLength(202); + expect(result.items.slice(-2)).toEqual([201, 202]); + expect(result.total).toBe(202); + expect(result.nextCursor).toBeNull(); + expect(fetchPage.mock.calls).toEqual([[], ["page-2"]]); + }); + + it("rejects a repeated cursor instead of looping indefinitely", async () => { + const fetchPage = vi.fn(async () => ({ items: [1], nextCursor: "same" })); + await expect(fetchAllCursorPages(fetchPage)).rejects.toThrow("分页游标重复"); + expect(fetchPage).toHaveBeenCalledTimes(2); + }); + + it("propagates a later page failure instead of returning a partial configuration", async () => { + const failure = new Error("aborted"); + await expect( + fetchAllCursorPages(async (cursor) => { + if (cursor) throw failure; + return { items: [1], nextCursor: "next" }; + }), + ).rejects.toBe(failure); + }); +}); diff --git a/apps/web/app/api/parse.ts b/apps/web/app/api/parse.ts index 84210f4e..850057f1 100644 --- a/apps/web/app/api/parse.ts +++ b/apps/web/app/api/parse.ts @@ -37,3 +37,21 @@ export function cleanParams(p: Record( + fetchPage: (cursor?: string) => Promise

, +): Promise

{ + const first = await fetchPage(); + const items = [...first.items]; + const seen = new Set(); + let cursor = first.nextCursor; + while (cursor) { + if (seen.has(cursor)) throw new ApiRequestError("internal", "分页游标重复,请重试"); + seen.add(cursor); + const page = await fetchPage(cursor); + items.push(...page.items); + cursor = page.nextCursor; + } + return { ...first, items, nextCursor: null }; +} diff --git a/apps/web/app/api/policies.ts b/apps/web/app/api/policies.ts index a6850ae1..b7dd2fad 100644 --- a/apps/web/app/api/policies.ts +++ b/apps/web/app/api/policies.ts @@ -9,6 +9,7 @@ import { } from "@umt/contracts"; import { api } from "~/lib/api.ts"; import { getParsed, parseOrThrow } from "./library.ts"; +import { fetchAllCursorPages } from "./parse.ts"; // 策略版本:草案 → 预览 → 授权发布 → 版本化 / 回滚(REQ §6.4) @@ -24,7 +25,8 @@ const Page = cursorPage(PolicyRevision); export function usePolicyRevisions(filters: PolicyFilters = {}) { return useQuery({ queryKey: policyKeys.list(filters), - queryFn: ({ signal }) => getParsed(Page, "/policy-revisions", { ...filters, limit: 200 }, signal), + queryFn: ({ signal }) => + fetchAllCursorPages((cursor) => getParsed(Page, "/policy-revisions", { ...filters, limit: 200, cursor }, signal)), }); } diff --git a/apps/web/app/api/preferences.ts b/apps/web/app/api/preferences.ts index bbdb09f7..0af11bf2 100644 --- a/apps/web/app/api/preferences.ts +++ b/apps/web/app/api/preferences.ts @@ -11,6 +11,7 @@ import { } from "@umt/contracts"; import { api } from "~/lib/api.ts"; import { getParsed, parseOrThrow } from "./library.ts"; +import { fetchAllCursorPages } from "./parse.ts"; // 偏好标签与抑制记录(REQ §6.7):停用/删除/语义变更都会留抑制记录,Agent 不得据旧证据自动加回。 @@ -29,7 +30,10 @@ const SupPage = cursorPage(PreferenceSuppression); export function usePreferenceTags(filters: TagFilters = {}) { return useQuery({ queryKey: preferenceKeys.tags(filters), - queryFn: ({ signal }) => getParsed(TagPage, "/preference-tags", { ...filters, limit: 200 }, signal), + queryFn: ({ signal }) => + fetchAllCursorPages((cursor) => + getParsed(TagPage, "/preference-tags", { ...filters, limit: 200, cursor }, signal), + ), }); } @@ -37,11 +41,13 @@ export function useSuppressions(active?: boolean) { return useQuery({ queryKey: preferenceKeys.suppressions(active), queryFn: ({ signal }) => - getParsed( - SupPage, - "/preference-suppressions", - { active: active === undefined ? undefined : String(active), limit: 200 }, - signal, + fetchAllCursorPages((cursor) => + getParsed( + SupPage, + "/preference-suppressions", + { active: active === undefined ? undefined : String(active), limit: 200, cursor }, + signal, + ), ), }); } diff --git a/apps/web/app/api/records.ts b/apps/web/app/api/records.ts index 046368ce..7a7b62a9 100644 --- a/apps/web/app/api/records.ts +++ b/apps/web/app/api/records.ts @@ -15,7 +15,7 @@ import { PublicOrganizationRecord, } from "@umt/contracts"; import { TTL_SECOND } from "~/lib/api.ts"; -import { apiMutate, apiParsed, cleanParams } from "./parse.ts"; +import { apiMutate, apiParsed, cleanParams, fetchAllCursorPages } from "./parse.ts"; // 整理记录(W2-C:/organization-records;读 organize.read,发起 organize.execute) @@ -23,6 +23,7 @@ const RecordPage = cursorPage(PublicOrganizationRecord); export type RecordFilters = { status?: OrganizationStatus; + phase?: "active" | "history"; trigger?: OrganizationTrigger; mediaEntityId?: string; q?: string; @@ -145,6 +146,7 @@ export const TRIGGER_LABEL: Record = { auto: "自 /** 整理方式 / 文件操作(记录的 action) */ export const ACTION_LABEL = { copy: "复制", + in_place: "原地入库", move: "移动", rename: "重命名", relocate: "目录调整", @@ -174,7 +176,9 @@ export function useLibraryRoots() { queryKey: ["records", "library-roots"], staleTime: 5 * 60 * TTL_SECOND, queryFn: ({ signal }) => - apiParsed(cursorPage(LibraryRoot), "/api/v1/library-roots", { params: { limit: 200 }, signal }), + fetchAllCursorPages((cursor) => + apiParsed(cursorPage(LibraryRoot), "/api/v1/library-roots", { params: { limit: 200, cursor }, signal }), + ), }); } diff --git a/apps/web/app/api/releases.ts b/apps/web/app/api/releases.ts index 3352abfc..159ffda0 100644 --- a/apps/web/app/api/releases.ts +++ b/apps/web/app/api/releases.ts @@ -1,5 +1,6 @@ import { keepPreviousData, useInfiniteQuery, useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { + type ManualAcquisitionRequest, ManualAcquisitionResult, type ResolutionStatus, WorkbenchReleaseDetail, @@ -81,7 +82,7 @@ export function useRefreshReleaseDetail() { export function useSubmitRelease() { const qc = useQueryClient(); return useMutation({ - mutationFn: (body: { sourceReleaseId: string; slotId?: string; entityId?: string; note?: string }) => + mutationFn: (body: ManualAcquisitionRequest) => apiMutate("/api/v1/acquisition-plans", { body }).then((d) => ManualAcquisitionResult.parse(d)), onSuccess: (r) => { if (r.submitted) void qc.invalidateQueries({ queryKey: ["downloads"] }); diff --git a/apps/web/app/api/rules.ts b/apps/web/app/api/rules.ts index f38d8e6c..80251fb8 100644 --- a/apps/web/app/api/rules.ts +++ b/apps/web/app/api/rules.ts @@ -22,6 +22,7 @@ import { } from "@umt/contracts"; import { api } from "~/lib/api.ts"; import { getParsed, parseOrThrow } from "./library.ts"; +import { fetchAllCursorPages } from "./parse.ts"; // 规则(W9-D,docs/runbook/rules.md):规则组 = 收录过滤条件 + 偏好标签合一;自定义条件;三套默认链;抑制记录。 // 写入给 ruleString(优先)或 layers;PATCH 带 expectedRevision,不匹配 409。 @@ -53,7 +54,8 @@ const SupPage = cursorPage(PreferenceSuppression); export function useRuleGroups(filters: RuleGroupFilters = {}) { return useQuery({ queryKey: ruleKeys.groups(filters), - queryFn: ({ signal }) => getParsed(GroupPage, "/rules", { ...filters, limit: 200 }, signal), + queryFn: ({ signal }) => + fetchAllCursorPages((cursor) => getParsed(GroupPage, "/rules", { ...filters, limit: 200, cursor }, signal)), placeholderData: keepPreviousData, }); } @@ -78,7 +80,8 @@ export function useRuleKeywords() { export function useRuleAtoms() { return useQuery({ queryKey: ruleKeys.atoms, - queryFn: ({ signal }) => getParsed(AtomPage, "/rules/atoms", { limit: 200 }, signal), + queryFn: ({ signal }) => + fetchAllCursorPages((cursor) => getParsed(AtomPage, "/rules/atoms", { limit: 200, cursor }, signal)), }); } @@ -92,7 +95,8 @@ export function useRuleDefaults() { export function useRuleSuppressions() { return useQuery({ queryKey: ruleKeys.suppressions, - queryFn: ({ signal }) => getParsed(SupPage, "/rules/suppressions", { limit: 200 }, signal), + queryFn: ({ signal }) => + fetchAllCursorPages((cursor) => getParsed(SupPage, "/rules/suppressions", { limit: 200, cursor }, signal)), }); } diff --git a/apps/web/app/api/sites.ts b/apps/web/app/api/sites.ts index 38e0aae5..7a29b6f4 100644 --- a/apps/web/app/api/sites.ts +++ b/apps/web/app/api/sites.ts @@ -19,7 +19,7 @@ import { } from "@umt/contracts"; import { z } from "zod"; import { TTL_SECOND } from "~/lib/api.ts"; -import { apiMutate, apiParsed } from "./parse.ts"; +import { apiMutate, apiParsed, fetchAllCursorPages } from "./parse.ts"; // 站点、站点账号、站点快照(W2-B:/sites、/site-accounts、/sites/{id}/snapshot(s)) // 站点中心(W9-A):include=accounts、凭证 PUT、健康检测、刷新登录、用户数据、删除 @@ -41,12 +41,14 @@ export const siteKeys = { loginHistory: (accountId: string) => ["sites", "login-history", accountId] as const, }; -/** 站点列表(附账号与整体账号状态,站点页一次请求) */ +/** 站点列表(附账号与整体账号状态,站点页按游标取齐配置) */ export function useSites() { return useQuery({ queryKey: siteKeys.list(), queryFn: ({ signal }) => - apiParsed(SitePage, "/api/v1/sites", { params: { limit: 200, include: "accounts" }, signal }), + fetchAllCursorPages((cursor) => + apiParsed(SitePage, "/api/v1/sites", { params: { limit: 200, include: "accounts", cursor }, signal }), + ), }); } @@ -62,7 +64,9 @@ export function useSiteAccounts(siteId?: string) { return useQuery({ queryKey: siteKeys.accounts(siteId), queryFn: ({ signal }) => - apiParsed(AccountPage, "/api/v1/site-accounts", { params: { limit: 200, siteId }, signal }), + fetchAllCursorPages((cursor) => + apiParsed(AccountPage, "/api/v1/site-accounts", { params: { limit: 200, siteId, cursor }, signal }), + ), }); } diff --git a/apps/web/app/api/slots.ts b/apps/web/app/api/slots.ts index c4069d8b..1c32e321 100644 --- a/apps/web/app/api/slots.ts +++ b/apps/web/app/api/slots.ts @@ -8,6 +8,7 @@ import { } from "@umt/contracts"; import { api } from "~/lib/api.ts"; import { getParsed, parseOrThrow } from "./library.ts"; +import { fetchAllCursorPages } from "./parse.ts"; // 收录目标 Slot:硬条件 / 软偏好 / 回退 / 升级阈值 / 完成条件(REQ §6.1) @@ -24,7 +25,10 @@ export function useSlots(filters: SlotFilters, enabled = true) { return useQuery({ queryKey: slotKeys.list(filters), enabled, - queryFn: ({ signal }) => getParsed(Page, "/acquisition-slots", { ...filters, limit: 200 }, signal), + queryFn: ({ signal }) => + fetchAllCursorPages((cursor) => + getParsed(Page, "/acquisition-slots", { ...filters, limit: 200, cursor }, signal), + ), }); } diff --git a/apps/web/app/components/agent/agent-session-list.tsx b/apps/web/app/components/agent/agent-session-list.tsx index 2350fca7..434e92e7 100644 --- a/apps/web/app/components/agent/agent-session-list.tsx +++ b/apps/web/app/components/agent/agent-session-list.tsx @@ -1,11 +1,12 @@ import { Bot, ChevronRight } from "lucide-react"; -import { type ReactNode, useEffect, useLayoutEffect, useRef, useState } from "react"; +import { type ReactNode, useLayoutEffect, useRef, useState } from "react"; import { Link } from "react-router"; import { type AgentRunView, TRIGGER_LABEL, useAgentRuns } from "~/api/agent.ts"; import { CHEVRON_CLASS } from "~/components/agentui/disclosure.tsx"; import { EmptyState, ErrorState, LoadingState } from "~/components/console/empty-state.tsx"; -import { Button } from "~/components/ui/button.tsx"; +import { useListPagination } from "~/components/console/use-list-pagination.ts"; import { Input } from "~/components/ui/input.tsx"; +import { Pagination } from "~/components/ui/pagination.tsx"; import { relTime } from "~/components/workbench/release-fields.ts"; import { cn } from "~/lib/utils.ts"; import { contextLabel, RunStatusBadge } from "./agent-conversation.tsx"; @@ -176,10 +177,10 @@ function Row({ } /** - * 会话列表:搜索 + 全部 / 等你处理 / 失败 + 游标加载(滚到底自动加载)。 + * 会话列表:搜索 + 全部 / 等你处理 / 失败 + 按需加载的统一分页。 * - hrefFor:会话页左栏,点一条跳到该会话页。 * - onOpen:抽屉历史弹层,点一条在抽屉内打开;remember 时筛选与滚动位置下次打开时恢复。 - * - limit:历史弹层只列最近 N 条(置顶的算在内),不自动翻页。 + * - limit:历史弹层的默认每页条数,可调节并翻页查看更早的会话。 */ export function AgentSessionList({ activeId, @@ -208,32 +209,29 @@ export function AgentSessionList({ const waiting = useAgentRuns("waiting"); const loaded = runs.data?.pages.flatMap((p) => p.items) ?? []; const pending = waiting.data?.pages.flatMap((p) => p.items) ?? []; - const { pinned, rows } = arrangeHistory([...pending.filter(awaitingUser), ...loaded], f, limit); - const canLoadMore = !limit && !!runs.hasNextPage; + const arranged = arrangeHistory([...pending.filter(awaitingUser), ...loaded], f); + const loadWaiting = f.view !== "failed" && !!waiting.hasNextPage; + const canLoadMore = loadWaiting || !!runs.hasNextPage; + const paged = useListPagination([...arranged.pinned, ...arranged.rows], { + hasMore: canLoadMore, + onLoadMore: () => (loadWaiting ? waiting.fetchNextPage() : runs.fetchNextPage()), + loadingMore: waiting.isFetchingNextPage || runs.isFetchingNextPage, + error: runs.error ?? (loadWaiting ? waiting.error : undefined), + resetKey: JSON.stringify(f), + defaultPageSize: limit, + }); + const visible = paged.items ?? []; + const pinned = f.view === "all" ? visible.filter((row) => awaitingUser(row.run)) : []; + const rows = f.view === "all" ? visible.filter((row) => !awaitingUser(row.run)) : visible; const filtered = !!f.q.trim() || f.view !== "all"; const scroller = useRef(null); - const sentinel = useRef(null); const restored = useRef(!remember); useLayoutEffect(() => { if (restored.current || !rows.length || !scroller.current) return; restored.current = true; scroller.current.scrollTop = drawerMemory.scroll; }); - const { fetchNextPage, isFetchingNextPage } = runs; - useEffect(() => { - const el = sentinel.current; - if (!el || !canLoadMore || typeof IntersectionObserver === "undefined") return; - const io = new IntersectionObserver( - (es) => { - if (es.some((e) => e.isIntersecting) && !isFetchingNextPage) void fetchNextPage(); - }, - { root: scroller.current, rootMargin: "120px" }, - ); - io.observe(el); - return () => io.disconnect(); - }, [canLoadMore, isFetchingNextPage, fetchNextPage]); - const rowProps = { activeId, hrefFor, onOpen }; return (

); } diff --git a/apps/web/app/components/browser/desktop-view.tsx b/apps/web/app/components/browser/desktop-view.tsx index 42ece030..1a1ef096 100644 --- a/apps/web/app/components/browser/desktop-view.tsx +++ b/apps/web/app/components/browser/desktop-view.tsx @@ -16,6 +16,7 @@ import { apiData, errorMessage } from "~/lib/api.ts"; import { qk } from "~/lib/query-keys.ts"; import { agentActive, controllerLabel, credentialBadge, holderName, waitingText } from "./coords.ts"; import { ReconnectPanel, ReloadNotice, useAutoReconnect } from "./reconnect.tsx"; +import { ResourceDownload } from "./resource-download.tsx"; import { SiteOriginView } from "./site-origin-view.tsx"; // 原站视图桌面模式(docs/design/webtop-remote-browser.md §5.7):iframe 嵌浏览器桌面(Selkies), @@ -208,6 +209,7 @@ function DesktopView({ return (
+
{conn.text} diff --git a/apps/web/app/components/browser/resource-download.tsx b/apps/web/app/components/browser/resource-download.tsx new file mode 100644 index 00000000..05fca16c --- /dev/null +++ b/apps/web/app/components/browser/resource-download.tsx @@ -0,0 +1,88 @@ +import type { BrowserResourceResult, BrowserSessionView } from "@umt/contracts"; +import { Download, Settings2 } from "lucide-react"; +import { useState } from "react"; +import { Link } from "react-router"; +import { toast } from "sonner"; +import { useSubmitRelease } from "~/api/releases.ts"; +import { ManualDownloadDialog } from "~/components/downloads/manual-download-dialog.tsx"; +import { Button } from "~/components/ui/button.tsx"; +import { apiData, errorMessage } from "~/lib/api.ts"; + +/** 控件位于 iframe/截图外;直接解析当前详情页并交给共同手动下载入口。 */ +export function ResourceDownload({ session }: { session: BrowserSessionView }) { + const [busy, setBusy] = useState(false); + const [resource, setResource] = useState(null); + const [open, setOpen] = useState(false); + const [unrecognized, setUnrecognized] = useState(false); + const [planId, setPlanId] = useState(null); + const submit = useSubmitRelease(); + const prepare = async (options: boolean) => { + setBusy(true); + try { + const r = await apiData("/browser/sessions/{id}/resource", { + method: "POST", + pathParams: { id: session.id }, + body: {}, + }); + setResource(r); + setUnrecognized(false); + if (options) setOpen(true); + else { + const result = await submit.mutateAsync({ + sourceReleaseId: r.release.id, + importMode: "normal", + replacementAssetIds: [], + }); + if (result.requiresConfirmation) { + setUnrecognized(true); + setOpen(true); + } else if (result.submitted) { + setPlanId(result.plan?.id ?? null); + toast.success("已提交下载"); + } else toast.warning(result.rejected.map((v) => v.message).join(";") || "没有提交下载"); + } + } catch (e) { + toast.error(errorMessage(e)); + } finally { + setBusy(false); + } + }; + return ( + <> +
+ + + {planId ? ( + + 查看下载任务 + + ) : null} +
+ {resource ? ( + submit.mutateAsync({ ...options, sourceReleaseId: resource.release.id })} + /> + ) : null} + + ); +} diff --git a/apps/web/app/components/browser/site-origin-view.tsx b/apps/web/app/components/browser/site-origin-view.tsx index 8dd3a7b2..69adc528 100644 --- a/apps/web/app/components/browser/site-origin-view.tsx +++ b/apps/web/app/components/browser/site-origin-view.tsx @@ -36,6 +36,7 @@ import { Input } from "~/components/ui/input.tsx"; import { errorMessage } from "~/lib/api.ts"; import { controllerLabel, credentialBadge, holderName, toViewport, waitingText } from "./coords.ts"; import { isBrowserGone, ReconnectPanel, ReloadNotice, useAutoReconnect } from "./reconnect.tsx"; +import { ResourceDownload } from "./resource-download.tsx"; // 原站视图(REQ §4.7.1、§4.7.3):同一个 camofox 会话的定时截图 + 人工输入映射。 // 人工接管前服务端先停掉该会话的自动化输入;交还后 Agent 必须重新读取页面。 @@ -153,6 +154,8 @@ export function SiteOriginView({ site }: { site: SiteListItem }) {

验证码等需本人操作在此完成。

+ +
{conn.text} diff --git a/apps/web/app/components/console/data-table.test.ts b/apps/web/app/components/console/data-table.test.ts index 29011569..2aabb010 100644 --- a/apps/web/app/components/console/data-table.test.ts +++ b/apps/web/app/components/console/data-table.test.ts @@ -85,6 +85,6 @@ describe("DataTable groupBy:渲染", () => { rowKey: (r: Row) => r.id, }), ); - expect(html).not.toContain("aria-expanded"); + expect(html).not.toContain('aria-expanded="true"'); }); }); diff --git a/apps/web/app/components/console/data-table.tsx b/apps/web/app/components/console/data-table.tsx index b105edb3..a5aa1be5 100644 --- a/apps/web/app/components/console/data-table.tsx +++ b/apps/web/app/components/console/data-table.tsx @@ -1,9 +1,10 @@ import { ChevronRight, type LucideIcon } from "lucide-react"; import { Fragment, type ReactNode, useEffect, useMemo, useState } from "react"; -import { Button } from "~/components/ui/button.tsx"; +import { Pagination, type PaginationProps } from "~/components/ui/pagination.tsx"; import { cn } from "~/lib/utils.ts"; import { groupRows, readCollapsed, toggleCollapsed, writeCollapsed } from "./data-table-groups.ts"; import { EmptyState, ErrorState, LoadingState } from "./empty-state.tsx"; +import { useListPagination } from "./use-list-pagination.ts"; export type DataColumn = { /** 列 key(React key 用) */ @@ -33,7 +34,7 @@ const HIDE: Record["hideBelow"]>, string> = { }; /** - * 数据表:内置 loading / error / empty 三态与"加载更多"游标翻页。 + * 数据表:内置 loading / error / empty 三态与统一分页。 * 外层自带 pg-slab 卡片(plain 时不带);表格在卡片内横向滚动,页面本身不溢出。 * * r.id} @@ -43,7 +44,7 @@ const HIDE: Record["hideBelow"]>, string> = { */ export function DataTable({ columns, - rows, + rows: allRows, rowKey, loading = false, error, @@ -57,6 +58,8 @@ export function DataTable({ plain = false, groupBy, className, + pagination = true, + paginationKey, }: { columns: DataColumn[]; rows: T[] | undefined; @@ -73,7 +76,17 @@ export function DataTable({ plain?: boolean; groupBy?: DataGroupBy; className?: string; + pagination?: boolean | PaginationProps; + paginationKey?: string; }) { + const paged = useListPagination(allRows, { + hasMore: pagination === true && hasMore, + onLoadMore, + loadingMore, + error, + resetKey: paginationKey ?? (allRows?.[0] ? rowKey(allRows[0]) : ""), + }); + const rows = pagination === true ? paged.items : allRows; const storageKey = groupBy?.storageKey; const [collapsed, setCollapsed] = useState>(() => new Set()); // 折叠态在挂载后读(服务端渲染时没有 localStorage,首屏一律展开) @@ -217,17 +230,11 @@ export function DataTable({ return (
{body} - {rows && rows.length > 0 && (hasMore || error) ? ( -
- {error ? ( - {String((error as Error).message ?? error)} - ) : null} - {hasMore && onLoadMore ? ( - - ) : null} -
+ {error && allRows?.length ? ( +
{String((error as Error).message ?? error)}
+ ) : null} + {pagination && allRows ? ( + ) : null}
); diff --git a/apps/web/app/components/console/inbox-bell.tsx b/apps/web/app/components/console/inbox-bell.tsx index f03a951c..fb4b70a5 100644 --- a/apps/web/app/components/console/inbox-bell.tsx +++ b/apps/web/app/components/console/inbox-bell.tsx @@ -4,6 +4,7 @@ import { ArrowRight, Bell, CheckCheck } from "lucide-react"; import { useMemo, useRef, useState } from "react"; import { Link, useNavigate } from "react-router"; import { decisionKeys, useDecisionSummary, useDecisions } from "~/api/decisions.ts"; +import { EmptyState, ErrorState, LoadingState } from "~/components/console/empty-state.tsx"; import { SEVERITY } from "~/components/library/labels.ts"; import { Button } from "~/components/ui/button.tsx"; import { Popover, PopoverContent, PopoverTrigger } from "~/components/ui/popover.tsx"; @@ -59,55 +60,54 @@ export function InboxBell() { - -
+ +

待处理 - {total ? {total} : null} + {total ? {total} : null}

- + +
{list.isPending ? ( -

加载中…

+ ) : list.isError ? ( -

加载失败,稍后再试

+ ) : groups.length === 0 ? ( -
- - 没有需要处理的事项 -
+ ) : ( -
+
{groups.map((g) => ( -
+
    {g.items.map((d) => ( @@ -115,14 +115,12 @@ export function InboxBell() { setOpen(false)} - className="flex items-start gap-2.5 px-4 py-2 hover:bg-foreground/[0.04]" + className="block rounded-lg px-2 py-2 outline-none transition-colors hover:bg-foreground/[0.06] focus-visible:bg-foreground/[0.06] focus-visible:ring-2 focus-visible:ring-ring/50" > - - - {d.title} - - {SEVERITY[d.severity].label} · {formatRelative(d.createdAt)} - + {d.title} + + + {SEVERITY[d.severity].label} · {formatRelative(d.createdAt)} diff --git a/apps/web/app/components/console/list-pagination.test.ts b/apps/web/app/components/console/list-pagination.test.ts new file mode 100644 index 00000000..215a75fe --- /dev/null +++ b/apps/web/app/components/console/list-pagination.test.ts @@ -0,0 +1,47 @@ +import { createElement } from "react"; +import { renderToStaticMarkup } from "react-dom/server"; +import { describe, expect, it } from "vitest"; +import { Pagination, pageNumbers } from "~/components/ui/pagination.tsx"; +import { DataTable } from "./data-table.tsx"; +import { pageWindow } from "./use-list-pagination.ts"; + +describe("list pagination", () => { + it("keeps empty and short final pages in range", () => { + expect(pageWindow(0, 20, 8)).toEqual({ page: 1, pages: 1, start: 0, end: 0 }); + expect(pageWindow(135, 100, 8)).toEqual({ page: 2, pages: 2, start: 100, end: 135 }); + expect(pageWindow(135, 50, 0)).toEqual({ page: 1, pages: 3, start: 0, end: 50 }); + }); + it("shows nearby pages and endpoints without a button for every record page", () => { + expect(pageNumbers(50, 100)).toEqual([1, "…", 49, 50, 51, "…", 100]); + expect(pageNumbers(1, 2)).toEqual([1, 2]); + }); + it("does not invent a total or last page for cursor results", () => { + const html = renderToStaticMarkup( + createElement(Pagination, { + page: 1, + pageSize: 50, + total: 50, + hasMore: true, + onPageChange: () => {}, + }), + ); + expect(html).toContain("已加载 50 条"); + expect(html).toContain('aria-label="第 2 页"'); + expect(html).not.toContain('aria-label="最后一页"'); + }); + it("limits table rows and shares the page-size control even for empty lists", () => { + const table = (rows: { id: string }[]) => + renderToStaticMarkup( + createElement(DataTable<{ id: string }>, { + rows, + columns: [{ key: "id", header: "ID", cell: (r: { id: string }) => r.id }], + rowKey: (r: { id: string }) => r.id, + }), + ); + const html = table(Array.from({ length: 135 }, (_, i) => ({ id: `item-${i}` }))); + expect(html.split("")[1]?.split("")[0]?.match(/"); + expect(table([])).toContain('aria-label="每页条数"'); + }); +}); diff --git a/apps/web/app/components/console/paginated-list.tsx b/apps/web/app/components/console/paginated-list.tsx new file mode 100644 index 00000000..e5b63150 --- /dev/null +++ b/apps/web/app/components/console/paginated-list.tsx @@ -0,0 +1,33 @@ +import type { ReactNode } from "react"; +import { useListPagination } from "~/components/console/use-list-pagination.ts"; +import { Pagination } from "~/components/ui/pagination.tsx"; + +/** Shared paging for custom record lists, keeping row markup and actions with the caller. */ +export function PaginatedList({ + rows, + children, + label, + resetKey, + className, + as = "div", + compact = false, +}: { + rows: T[]; + children: (row: T, index: number) => ReactNode; + label: string; + resetKey?: string; + className?: string; + as?: "div" | "ul" | "fieldset"; + compact?: boolean; +}) { + const { items, pagination } = useListPagination(rows, { resetKey }); + const Container = as; + return ( +
    + + {items?.map((row, index) => children(row, (pagination.page - 1) * pagination.pageSize + index))} + + {!compact || rows.length > 10 ? : null} +
    + ); +} diff --git a/apps/web/app/components/console/use-list-pagination.ts b/apps/web/app/components/console/use-list-pagination.ts new file mode 100644 index 00000000..25b147d2 --- /dev/null +++ b/apps/web/app/components/console/use-list-pagination.ts @@ -0,0 +1,100 @@ +import { useCallback, useEffect, useRef, useState, useSyncExternalStore } from "react"; +import type { PaginationProps } from "~/components/ui/pagination.tsx"; + +export const PAGE_SIZES = [10, 20, 50, 100] as const; +const SIZE_KEY = "umt.list.page-size"; +const SIZE_EVENT = "umt:list-page-size"; +function readSize() { + try { + const size = Number(localStorage.getItem(SIZE_KEY)); + return PAGE_SIZES.some((n) => n === size) ? size : 0; + } catch { + return 0; + } +} +function subscribeSize(notify: () => void) { + window.addEventListener("storage", notify); + window.addEventListener(SIZE_EVENT, notify); + return () => { + window.removeEventListener("storage", notify); + window.removeEventListener(SIZE_EVENT, notify); + }; +} +export function pageWindow(total: number, size: number, requested: number) { + const pages = Math.max(1, Math.ceil(total / size)); + const page = Math.max(1, Math.min(requested, pages)); + return { page, pages, start: (page - 1) * size, end: Math.min(page * size, total) }; +} +/** Present cached cursor results as pages; fetch only enough to fill the requested page. */ +export function useListPagination( + rows: T[] | undefined, + { + hasMore = false, + onLoadMore, + loadingMore = false, + error, + resetKey = "", + defaultPageSize = 50, + }: { + hasMore?: boolean; + onLoadMore?: () => unknown; + loadingMore?: boolean; + error?: unknown; + resetKey?: string; + defaultPageSize?: number; + } = {}, +) { + const storedSize = useSyncExternalStore(subscribeSize, readSize, () => 0); + const [localSize, setLocalSize] = useState(null); + const pageSize = localSize ?? (storedSize || defaultPageSize); + const scope = `${resetKey}|${pageSize}`; + const [position, setPosition] = useState({ scope, page: 1 }); + useEffect(() => { + setPosition({ scope, page: 1 }); + }, [scope]); + const requested = position.scope === scope ? position.page : 1; + const total = rows?.length ?? 0; + const range = pageWindow(total, pageSize, requested); + const needsMore = !!rows && hasMore && total < requested * pageSize; + const fetching = useRef(false); + const load = useCallback(() => { + if (!onLoadMore || fetching.current || loadingMore) return; + fetching.current = true; + Promise.resolve(onLoadMore()) + .catch(() => {}) + .finally(() => { + fetching.current = false; + }); + }, [onLoadMore, loadingMore]); + // biome-ignore lint/correctness/useExhaustiveDependencies: every new cursor chunk must trigger filling the requested page + useEffect(() => { + if (needsMore && !error) load(); + }, [needsMore, total, error, load]); + const onPageChange = (page: number) => { + const next = Math.max(1, Math.min(page, range.pages + (hasMore ? 1 : 0))); + setPosition({ scope, page: next }); + if (hasMore && next * pageSize > total) load(); + }; + const onPageSizeChange = (size: number) => { + if (!PAGE_SIZES.some((n) => n === size)) return; + setLocalSize(size); + setPosition({ scope: `${resetKey}|${size}`, page: 1 }); + try { + localStorage.setItem(SIZE_KEY, String(size)); + window.dispatchEvent(new Event(SIZE_EVENT)); + setLocalSize(null); + } catch { + /* selected size still applies to this list */ + } + }; + const pagination: PaginationProps = { + page: range.page, + pageSize, + total, + hasMore, + busy: loadingMore || (needsMore && !error), + onPageChange, + onPageSizeChange, + }; + return { items: rows?.slice(range.start, range.end), pagination }; +} diff --git a/apps/web/app/components/dashboard/home-runtime.tsx b/apps/web/app/components/dashboard/home-runtime.tsx new file mode 100644 index 00000000..3e5d4a6a --- /dev/null +++ b/apps/web/app/components/dashboard/home-runtime.tsx @@ -0,0 +1,209 @@ +import { useQuery } from "@tanstack/react-query"; +import { type Dashboard, DownloadBoard, type HealthLevel } from "@umt/contracts"; +import { ArrowRight, Download, HardDrive, type LucideIcon } from "lucide-react"; +import type { ReactNode } from "react"; +import { Link } from "react-router"; +import { apiParsed } from "~/api/parse.ts"; +import { useSystemHealth } from "~/api/system.ts"; +import { EmptyState, ErrorState, LoadingState } from "~/components/console/empty-state.tsx"; +import { consolePath } from "~/components/console/nav.ts"; +import { StatusBadge, type StatusTone } from "~/components/console/status-badge.tsx"; +import { formatRelative } from "~/lib/labels.ts"; +import { cn, formatBytes } from "~/lib/utils.ts"; + +const LEVEL: Record = { + ok: { label: "正常", tone: "success" }, + warn: { label: "需要关注", tone: "warning" }, + critical: { label: "需要处理", tone: "destructive" }, + unknown: { label: "用量未知", tone: "neutral" }, +}; +const SCOPE = { data_dir: "系统数据", library_root: "媒体库", download_dir: "下载目录" }; + +function OverviewCard({ + title, + icon: Icon, + to, + children, +}: { + title: string; + icon: LucideIcon; + to: string; + children: ReactNode; +}) { + return ( +
    +
    + +

    {title}

    + + 详情 + + +
    + {children} +
    + ); +} + +function StorageOverview() { + const q = useSystemHealth(60_000); + const mediaDisks = q.data?.disks.filter((d) => d.scope !== "data_dir") ?? []; + const disks = mediaDisks.length ? mediaDisks : (q.data?.disks ?? []); + return ( + + {q.isPending ? ( + + ) : q.isError ? ( + q.refetch()} className="py-6" /> + ) : !disks.length ? ( + + ) : ( +
      + {disks.map((d) => { + const pct = d.usedRatio === null ? null : d.usedRatio * 100; + const state = LEVEL[d.level]; + return ( +
    • +
      +

      + {d.name !== SCOPE[d.scope] ? ( + {SCOPE[d.scope]} + ) : null} + {d.name} +

      + + {state.label} + {pct === null ? "" : ` · ${Math.round(pct)}%`} + +
      + {pct !== null ? ( +
      +
      +
      + ) : null} +

      + + 剩余{" "} + + {d.availableBytes === null ? "未知" : formatBytes(d.availableBytes, 1)} + + + 已用 {d.usedBytes === null ? "未知" : formatBytes(d.usedBytes, 1)} + 总量 {d.totalBytes === null ? "未知" : formatBytes(d.totalBytes, 1)} +

      +

      + {d.path} +

      + {d.error ?

      无法读取磁盘用量

      : null} +
    • + ); + })} +
    + )} + {q.data && !q.isError ? ( +

    所在磁盘用量 · 检查于 {formatRelative(q.data.checkedAt)}

    + ) : null} +
    + ); +} + +function DownloaderOverview({ downloaders }: { downloaders?: Dashboard["sources"]["downloaders"] }) { + // 首页仅取一条面板记录,速度和计数仍是完整汇总;无需拉取下载任务列表。 + const q = useQuery({ + queryKey: ["downloads", "home-overview"], + queryFn: ({ signal }) => + apiParsed(DownloadBoard, "/api/v1/download-tasks/board", { params: { state: "active", limit: 1 }, signal }), + refetchInterval: 15_000, + refetchIntervalInBackground: false, + }); + const board = q.data; + const instances = board?.transfer.instances ?? []; + const sources = + downloaders ?? + instances.map((d) => ({ id: d.instanceId, name: d.name, enabled: true, health: "unknown", lastSeenAt: null })); + return ( + + {q.isPending ? ( + + ) : q.isError ? ( + q.refetch()} className="py-6" /> + ) : !sources.length ? ( + + ) : ( + <> +

    + {board?.counts.active} + 项进行中 +

    +
      + {sources.map((d) => { + const live = instances.find((i) => i.instanceId === d.id); + const state = !d.enabled + ? { label: "已停用", tone: "neutral" as const } + : !live + ? { label: "状态未知", tone: "neutral" as const } + : live.reachable + ? { label: "在线", tone: "success" as const } + : { label: "连接失败", tone: "warning" as const }; + const readable = d.enabled && live?.reachable; + return ( +
    • +
      + + {d.name} + + {state.label} +
      + {d.enabled ? ( +

      + + 下载{" "} + + {readable ? `${formatBytes(live.dlSpeed, 1)}/s` : "未知"} + + + + 上传{" "} + + {readable ? `${formatBytes(live.upSpeed, 1)}/s` : "未知"} + + +

      + ) : null} +
    • + ); + })} +
    +

    + 进行中含等待计划、暂停和异常任务 · 速度含下载器全部任务 +

    + {board?.liveAt ? ( +

    速度采样于 {formatRelative(board.liveAt)}

    + ) : null} + + )} +
    + ); +} + +export function HomeRuntime({ downloaders }: { downloaders?: Dashboard["sources"]["downloaders"] }) { + return ( +
    + + +
    + ); +} diff --git a/apps/web/app/components/downloads/business-tasks.tsx b/apps/web/app/components/downloads/business-tasks.tsx index 6a517eb4..3b02d5fc 100644 --- a/apps/web/app/components/downloads/business-tasks.tsx +++ b/apps/web/app/components/downloads/business-tasks.tsx @@ -231,6 +231,7 @@ export function BusinessTasks() { /> t.id} diff --git a/apps/web/app/components/downloads/download-detail.tsx b/apps/web/app/components/downloads/download-detail.tsx index dbd177e4..098a64e6 100644 --- a/apps/web/app/components/downloads/download-detail.tsx +++ b/apps/web/app/components/downloads/download-detail.tsx @@ -237,6 +237,7 @@ export function DownloadDetail({ task, live }: { task: DownloadTask; live?: Down instanceId: task.instanceId, contentPath: task.contentPath, name: task.name, + tags: task.tags, }} onClose={() => setOrganizing(false)} /> diff --git a/apps/web/app/components/downloads/manual-download-dialog.tsx b/apps/web/app/components/downloads/manual-download-dialog.tsx new file mode 100644 index 00000000..9b943a23 --- /dev/null +++ b/apps/web/app/components/downloads/manual-download-dialog.tsx @@ -0,0 +1,226 @@ +import type { AssetVersion, ManualAcquisitionRequest, ManualAcquisitionResult } from "@umt/contracts"; +import { Bot, Download } from "lucide-react"; +import { useEffect, useId, useState } from "react"; +import { Link } from "react-router"; +import { toast } from "sonner"; +import { downloaders } from "~/api/instances.ts"; +import { useLibraryVersions } from "~/api/library-versions.ts"; +import { FilterSelect } from "~/components/console/filter-bar.tsx"; +import { Button } from "~/components/ui/button.tsx"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "~/components/ui/dialog.tsx"; +import { Input } from "~/components/ui/input.tsx"; +import { errorMessage } from "~/lib/api.ts"; +import { formatBytes } from "~/lib/utils.ts"; + +export type ManualDownloadOptions = Omit; +export interface ManualDownloadResource { + title: string; + sizeBytes: number | null; + entityId?: string; +} + +/** 搜索、工作台和原站视图共用的一次性下载选项。 */ +export function ManualDownloadDialog({ + resource, + open, + onOpenChange, + submit, + initialUnrecognized = false, + onDelegate, +}: { + resource: ManualDownloadResource; + open: boolean; + onOpenChange: (open: boolean) => void; + submit: (options: ManualDownloadOptions) => Promise; + initialUnrecognized?: boolean; + onDelegate?: () => void; +}) { + const instances = downloaders.useList(); + const formId = useId(); + const inventory = useLibraryVersions(open ? resource.entityId : undefined); + const [instanceId, setInstanceId] = useState(""); + const [savePath, setSavePath] = useState(""); + const [mode, setMode] = useState<"normal" | "replace">("normal"); + const [selected, setSelected] = useState([]); + const [unrecognized, setUnrecognized] = useState(initialUnrecognized); + const [busy, setBusy] = useState(false); + const [result, setResult] = useState(null); + const versions = new Map(); + for (const v of inventory.data?.movie?.versions ?? []) versions.set(v.assetId, v); + for (const season of inventory.data?.seasons ?? []) + for (const episode of season.episodes) for (const v of episode.versions) versions.set(v.assetId, v); + for (const v of inventory.data?.unplaced ?? []) versions.set(v.assetId, v); + const existing = [...versions.values()]; + const single = existing.length === 1 ? existing[0]!.assetId : null; + useEffect(() => { + if (mode === "replace" && single) setSelected([single]); + }, [mode, single]); + useEffect(() => { + if (open) { + setResult(null); + setUnrecognized(initialUnrecognized); + } + }, [open, initialUnrecognized]); + const go = async () => { + setBusy(true); + try { + const r = await submit({ + ...(resource.entityId ? { entityId: resource.entityId } : {}), + ...(instanceId ? { downloaderInstanceId: instanceId } : {}), + ...(savePath.trim() ? { savePath: savePath.trim() } : {}), + ...(unrecognized ? { allowUnrecognized: true } : {}), + importMode: mode, + replacementAssetIds: mode === "replace" ? selected : [], + }); + setResult(r); + if (r.requiresConfirmation) setUnrecognized(true); + if (r.submitted) toast.success("已提交下载"); + } catch (e) { + toast.error(errorMessage(e)); + } finally { + setBusy(false); + } + }; + return ( + + + + 下载这个资源 + + {resource.title} + {resource.sizeBytes !== null ? ` · ${formatBytes(resource.sizeBytes, 1)}` : ""} + + +
    + v.enabled) + .map((v) => ({ value: v.id, label: v.name }))} + /> + + setMode(v === "replace" ? "replace" : "normal")} + allLabel="正常入库,保留已有版本" + options={[{ value: "replace", label: "一次性洗版入库" }]} + /> + {resource.entityId && inventory.isPending ? ( +

    正在读取已有版本…

    + ) : null} + {inventory.error ? ( +

    已有版本读取失败,请重试后选择洗版。

    + ) : null} + {existing.length ? ( +
    + + {mode === "replace" ? "选择要替换的已有文件" : "媒体库已有版本"} + + {existing.map((v) => ( +
    + {mode === "replace" ? ( + + setSelected((ids) => + e.target.checked ? [...ids, v.assetId] : ids.filter((id) => id !== v.assetId), + ) + } + /> + ) : null} + {mode === "replace" ? ( + + ) : ( + + )} +
    + ))} +
    + ) : mode === "replace" ? ( +

    没有可替换的已有文件,请选择正常入库。

    + ) : null} + {mode === "replace" ? ( +

    + 本次新文件成功入库后替换所选旧文件。剧集只替换成功对应的季集,其余保留。 +

    + ) : null} + {unrecognized ? ( +

    + 暂未识别到作品。可以继续下载;整理时会尝试补全识别和本地元数据。 +

    + ) : null} + {result && !result.submitted && !result.requiresConfirmation ? ( +

    {result.rejected.map((r) => r.message).join(";")}

    + ) : null} + {result?.submitted ? ( + + 查看下载任务 + + ) : null} +
    + + {onDelegate ? ( + + ) : null} + + +
    +
    + ); +} + +function VersionDescription({ version }: { version: AssetVersion }) { + return ( + + {version.fileName} + + {[version.resolution, version.source, version.videoCodec, ...version.hdr, version.edition] + .filter(Boolean) + .join(" · ")} + + + ); +} diff --git a/apps/web/app/components/files/dir-picker.tsx b/apps/web/app/components/files/dir-picker.tsx index 2d18c0a3..f71c67d3 100644 --- a/apps/web/app/components/files/dir-picker.tsx +++ b/apps/web/app/components/files/dir-picker.tsx @@ -17,7 +17,7 @@ export function DirPicker({ onChange: (ref: FileRef) => void; className?: string; }) { - const writable = roots.filter((r) => r.writable && r.health !== "offline"); + const writable = roots.filter((r) => r.kind === "library" && r.writable && r.health !== "offline"); return (
    {writable.length ? ( @@ -25,6 +25,7 @@ export function DirPicker({ roots={writable} current={value} pickOnly + paginate defaultOpen={value ? [value.root] : writable.slice(0, 1).map((r) => r.id)} onSelect={onChange} /> diff --git a/apps/web/app/components/files/dir-tree.tsx b/apps/web/app/components/files/dir-tree.tsx index b9bc7174..9f482013 100644 --- a/apps/web/app/components/files/dir-tree.tsx +++ b/apps/web/app/components/files/dir-tree.tsx @@ -1,6 +1,8 @@ import { ChevronRight, Download, Folder, FolderOpen, HardDrive, LoaderCircle, TriangleAlert } from "lucide-react"; import { useEffect, useState } from "react"; import { type FileRoot, useDirChildren } from "~/api/files.ts"; +import { useListPagination } from "~/components/console/use-list-pagination.ts"; +import { Pagination } from "~/components/ui/pagination.tsx"; import { healthLabel } from "~/lib/labels.ts"; import { cn } from "~/lib/utils.ts"; import { groupRoots, joinRel, segments } from "./model.ts"; @@ -21,6 +23,7 @@ export function DirTree({ onSelect, onDropTo, pickOnly = false, + paginate = false, defaultOpen = [], className, }: { @@ -29,6 +32,8 @@ export function DirTree({ onSelect: (loc: Loc) => void; onDropTo?: DropHandler; pickOnly?: boolean; + /** 目录选择小窗按页展示子目录;主文件树保持懒展开。 */ + paginate?: boolean; /** 初始展开的根 id(目标选择器里默认展开可写根) */ defaultOpen?: string[]; className?: string; @@ -75,6 +80,7 @@ export function DirTree({ onSelect={onSelect} onDropTo={onDropTo} pickOnly={pickOnly} + paginate={paginate} /> ))}
    @@ -92,6 +98,7 @@ function RootNode({ onSelect, onDropTo, pickOnly, + paginate, }: { root: FileRoot; current: Loc | null; @@ -100,6 +107,7 @@ function RootNode({ onSelect: (loc: Loc) => void; onDropTo?: DropHandler; pickOnly: boolean; + paginate: boolean; }) { const k = nodeKey(root.id, ""); const offline = root.health === "offline"; @@ -162,6 +170,7 @@ function RootNode({ onSelect={onSelect} onDropTo={onDropTo} pickOnly={pickOnly} + paginate={paginate} /> ) : null}
@@ -178,6 +187,7 @@ function Children({ onSelect, onDropTo, pickOnly, + paginate, }: { root: FileRoot; path: string; @@ -188,8 +198,23 @@ function Children({ onSelect: (loc: Loc) => void; onDropTo?: DropHandler; pickOnly: boolean; + paginate: boolean; }) { const q = useDirChildren(root.id, path, true); + const dirs = q.data?.dirs ?? []; + const paged = useListPagination(dirs, { resetKey: JSON.stringify({ root: root.id, path }) }); + const selectedIndex = + current?.root === root.id + ? dirs.findIndex((dir) => { + const childPath = dir.path || joinRel(path, dir.name); + return current.path === childPath || current.path.startsWith(`${childPath}/`); + }) + : -1; + // biome-ignore lint/correctness/useExhaustiveDependencies: 选中目录改变时定位;手动翻页不改变已有选择 + useEffect(() => { + if (paginate && selectedIndex >= 0) + paged.pagination.onPageChange(Math.floor(selectedIndex / paged.pagination.pageSize) + 1); + }, [paginate, root.id, path, selectedIndex, paged.pagination.pageSize]); if (q.isPending) return (
); - const dirs = q.data?.dirs ?? []; if (!dirs.length) return (

@@ -214,7 +238,7 @@ function Children({ ); return (

- {dirs.map((d) => { + {(paginate ? paged.items : dirs)?.map((d) => { const p = d.path || joinRel(path, d.name); const k = nodeKey(root.id, p); const isOpen = open.has(k); @@ -251,11 +275,13 @@ function Children({ onSelect={onSelect} onDropTo={onDropTo} pickOnly={pickOnly} + paginate={paginate} /> ) : null}
); })} + {paginate && dirs.length > 0 ? : null} {q.data?.truncated ? (

子目录过多,只列出一部分 diff --git a/apps/web/app/components/files/directory-dialog.test.ts b/apps/web/app/components/files/directory-dialog.test.ts new file mode 100644 index 00000000..3fc29703 --- /dev/null +++ b/apps/web/app/components/files/directory-dialog.test.ts @@ -0,0 +1,13 @@ +import { describe, expect, it } from "vitest"; +import { directorySelectionAllowed } from "./directory-dialog.tsx"; + +describe("目录选择根保护", () => { + it("配置来源与目标默认允许根目录,手动整理只允许子目录", () => { + const root = { root: "dl:instance:0", path: "" }; + expect(directorySelectionAllowed(root)).toBe(true); + expect(directorySelectionAllowed(root, false)).toBe(false); + expect(directorySelectionAllowed({ ...root, path: "completed/movie" }, false)).toBe(true); + expect(directorySelectionAllowed(null)).toBe(false); + expect(directorySelectionAllowed({ root: "", path: "movie" })).toBe(false); + }); +}); diff --git a/apps/web/app/components/files/directory-dialog.tsx b/apps/web/app/components/files/directory-dialog.tsx new file mode 100644 index 00000000..779269a7 --- /dev/null +++ b/apps/web/app/components/files/directory-dialog.tsx @@ -0,0 +1,138 @@ +import { ArrowUp, File, Folder } from "lucide-react"; +import { useState } from "react"; +import { type FileRef, filesErrorText, useFileList, useFileRoots } from "~/api/files.ts"; +import { useListPagination } from "~/components/console/use-list-pagination.ts"; +import { Button } from "~/components/ui/button.tsx"; +import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from "~/components/ui/dialog.tsx"; +import { Pagination } from "~/components/ui/pagination.tsx"; +import { DirTree } from "./dir-tree.tsx"; + +export const directorySelectionAllowed = (ref: FileRef | null, allowRoot = true): boolean => + !!ref?.root && (allowRoot || ref.path !== ""); + +/** Source selection intentionally permits readable download roots; targets only permit writable libraries. */ +export function DirectoryDialog({ + mode, + value, + onSelect, + onClose, + allowRoot = true, +}: { + mode: "source" | "target"; + value?: FileRef | null; + allowRoot?: boolean; + onSelect: (value: FileRef) => void; + onClose: () => void; +}) { + const roots = useFileRoots(); + const [selected, setSelected] = useState(value ?? null); + const available = (roots.data ?? []).filter( + (r) => r.health !== "offline" && (mode === "source" || (r.kind === "library" && r.writable)), + ); + const root = available.find((r) => r.id === selected?.root); + const listing = useFileList(root && selected ? { root: selected.root, path: selected.path } : null); + const entries = listing.data?.pages.flatMap((p) => p.entries) ?? []; + const paged = useListPagination(entries, { + hasMore: listing.hasNextPage, + onLoadMore: () => listing.fetchNextPage(), + loadingMore: listing.isFetchingNextPage, + error: listing.error, + resetKey: JSON.stringify(selected), + }); + return ( +

!open && onClose()}> + + + {mode === "source" ? "选择来源目录" : "选择媒体库目标目录"} + 展开登记目录,选择一个目录后确认。 + +
+
+ + {roots.isPending ? ( +

读取目录中…

+ ) : !available.length ? ( +

没有可用的登记目录。

+ ) : null} +
+
+ {selected?.path ? ( + + ) : null} + {!selected ? ( +

在左侧选择登记目录,查看内容。

+ ) : listing.isPending ? ( +

读取目录中…

+ ) : listing.error ? ( +

+ {filesErrorText(listing.error, "目录不可读取")} +

+ ) : !entries.length ? ( +

目录为空。

+ ) : ( + paged.items?.map((entry) => + entry.type === "dir" ? ( + + ) : ( +

+ + {entry.name} +

+ ), + ) + )} + {entries.length > 0 || listing.hasNextPage ? : null} +
+
+ {roots.error ? ( +

+ 读取目录失败,请关闭后重试。 +

+ ) : null} +

+ {root ? `${root.name} / ${selected?.path || "(根目录)"}` : "尚未选择"} +

+ {!allowRoot && selected && !selected.path ? ( +

+ 请选择子目录,或先新增来源配置,再整理根目录存量。 +

+ ) : null} +
+ + +
+
+
+ ); +} diff --git a/apps/web/app/components/files/mock.ts b/apps/web/app/components/files/mock.ts index d83a6d04..d8e520eb 100644 --- a/apps/web/app/components/files/mock.ts +++ b/apps/web/app/components/files/mock.ts @@ -854,7 +854,7 @@ function organize(body: { season?: number; episodes?: number[]; }[]; - action: "copy" | "move"; + action: "copy" | "move" | "in_place"; dryRun: boolean; onExisting?: "skip" | "conflict" | "replace_with_verify"; naming?: "template" | "keep"; @@ -881,12 +881,15 @@ function organize(body: { const e = it.episodes?.length ? `E${String(it.episodes[0]).padStart(2, "0")}` : ""; const orig = it.path.split("/").pop() ?? ""; const name = body.naming === "keep" ? orig : tv ? `${dirName} - ${s}${e}.${ext}` : `${dirName}.${ext}`; - const dst = tv - ? `${libRoot}/${dirName}/Season ${String(it.season ?? 1).padStart(2, "0")}/${name}` - : `${libRoot}/${dirName}/${name}`; + const dst = + body.action === "in_place" + ? `${state.roots.find((x) => x.id === it.root)?.hostPath}/${it.path}` + : tv + ? `${libRoot}/${dirName}/Season ${String(it.season ?? 1).padStart(2, "0")}/${name}` + : `${libRoot}/${dirName}/${name}`; const r = state.roots.find((x) => x.id === it.root); const { node, parent } = find(it.root, it.path); - const exists = id === "e-dune"; + const exists = body.action !== "in_place" && id === "e-dune"; if (exists && onExisting === "conflict") conflict = true; const main = { source: `${r?.hostPath}/${it.path}`, @@ -915,7 +918,10 @@ function organize(body: { .filter((c) => c !== node && !c.dir && c.name.startsWith(stem.slice(0, 16)) && /\.(srt|ass)$/.test(c.name)) .map((c) => ({ source: `${r?.hostPath}/${join(dirPath, c.name)}`, - target: dst.replace(/\.[^.]+$/, `.zh.${c.name.split(".").pop()}`), + target: + body.action === "in_place" + ? `${r?.hostPath}/${join(dirPath, c.name)}` + : dst.replace(/\.[^.]+$/, `.zh.${c.name.split(".").pop()}`), kind: "subtitle", size: c.size, action: "transfer" as const, diff --git a/apps/web/app/components/library/identity-aliases.tsx b/apps/web/app/components/library/identity-aliases.tsx index 963c9361..1ed8959d 100644 --- a/apps/web/app/components/library/identity-aliases.tsx +++ b/apps/web/app/components/library/identity-aliases.tsx @@ -198,6 +198,7 @@ export function AliasesPanel() { {entity ? : null} a.id} diff --git a/apps/web/app/components/library/identity-views.tsx b/apps/web/app/components/library/identity-views.tsx index b80e042b..b7cd9969 100644 --- a/apps/web/app/components/library/identity-views.tsx +++ b/apps/web/app/components/library/identity-views.tsx @@ -138,6 +138,7 @@ export function PoolTab() { r.id} @@ -238,6 +239,7 @@ export function ExternalTab() { r.id} diff --git a/apps/web/app/components/library/versions.tsx b/apps/web/app/components/library/versions.tsx index 38ed06e8..9c293093 100644 --- a/apps/web/app/components/library/versions.tsx +++ b/apps/web/app/components/library/versions.tsx @@ -11,6 +11,7 @@ import { Link } from "react-router"; import { useLibraryVersions } from "~/api/library-versions.ts"; import { type KvItem, KvList } from "~/components/console/kv-list.tsx"; import { consolePath } from "~/components/console/nav.ts"; +import { PaginatedList } from "~/components/console/paginated-list.tsx"; import { TechInfo } from "~/components/console/tech-info.tsx"; import { Panel } from "~/components/library/parts.tsx"; import { cn, formatBytes, formatDateTime } from "~/lib/utils.ts"; @@ -175,6 +176,7 @@ export function VersionList({ match, initial, onCollapse, + compact = false, }: { versions: AssetVersion[]; match?: UnitMatch | undefined; @@ -182,12 +184,20 @@ export function VersionList({ initial?: string | null; /** 收起选中版本时回调(剧集用来收回到一行) */ onCollapse?: () => void; + compact?: boolean; }) { const [selected, setSelected] = useState(initial ?? null); const tagBest = match?.state === "satisfied" && versions.length > 1; return ( -
    - {versions.map((v) => { + + {(v) => { const on = selected === v.assetId; return (
  • @@ -214,8 +224,8 @@ export function VersionList({ {on ? : null}
  • ); - })} -
+ }} + ); } @@ -241,7 +251,13 @@ export function EpisodeVersionList({ return (
{open ? ( - setOpen(false)} /> + setOpen(false)} + /> ) : ( -
- ); - } else if (!rows.length) { - body =

选中的项里没有视频文件。

; - } else { - body = ( -
- {src.unresolved.length ? ( -

- {src.unresolved.length} 个源路径不在登记目录里或已不存在,已跳过。 -

- ) : null} - setRows((rs) => setRowEntity(rs, k, e))} - onPatch={(k, p) => setRows((rs) => rs.map((r) => (r.key === k ? { ...r, ...p } : r)))} - onApplyGroup={(g) => setRows((rs) => applyGroupEntity(rs, g))} - onBulkEntity={(e) => setRows((rs) => applyEntityTo(rs, selected, e))} - onBulkSeason={(s) => setRows((rs) => applySeasonTo(rs, selected, s))} - onBulkOffset={(n) => { - const r = applyOffsetTo(rows, selected, n); - setRows(r.rows); - if (r.failed) toast.warning(`${r.failed} 项偏移后集号小于 1,没有改`); - }} - minSizeMb={minSizeMb} - onMinSizeMb={onMinSizeMb} - truncated={truncated} - mobile={mobile} - rootName={rootName} + ), + }, + { + key: "season", + header: "季", + cell: (r) => + r.entity?.kind === "movie" ? ( + "—" + ) : ( + setRows((rs) => rs.map((x) => (x.key === r.key ? { ...x, season: e.target.value } : x)))} /> -
- ); - } - } else if (step === 1) { - body = ( - - ); - } else { - const groups = preview?.groups ?? []; - body = ( -
- {notice ? ( -

- {notice} -

- ) : null} - {results.length ? ( -
-

执行结果

- {results.map((g, i) => ( - - ))} - {executed ? ( -

- 整理在后台执行,状态每 3 秒刷新,文件列表会自动更新。 -

- ) : null} -
- ) : null} - {dry.isPending ? ( -

- 生成整理计划… -

- ) : preview ? ( -
- - -
- ) : !results.length ? ( -

{filesErrorText(dry.error, "生成预览失败")}

- ) : null} -
- ); - } - + ), + }, + { + key: "episodes", + header: "集", + cell: (r) => + r.entity?.kind === "movie" ? ( + "—" + ) : ( + setRows((rs) => rs.map((x) => (x.key === r.key ? { ...x, episodes: e.target.value } : x)))} + /> + ), + }, + { + key: "status", + header: "范围", + cell: (r) => + r.excluded === "successful_record" ? ( + 成功历史 · 跳过 + ) : ( + + ), + }, + ]; + const loading = src.pending || (!started && !src.error && !!src.items.length && !ambiguous) || identify.isPending; return ( - !o && onClose()}> - - - 整理 - {src.label} -
    - {STEPS.map((s, i) => ( -
  1. - {i > 0 ? : null} - + !o && !submitting.current && onClose()}> + + + 整理 + {src.label} · 本次设置作用于整个所选范围 + +
    +
    + {source.kind === "files" && source.scanSummary ? ( +

    {source.scanSummary}

    + ) : null} + ({ + value: p.id, + label: `${p.name} · ${p.action === "copy" ? "复制" : "移动"} · ${p.category ?? "自动分类"}`, + })), + ...(profile && !matches.some((p) => p.id === profile.id) + ? [{ value: profile.id, label: profile.name }] + : []), + ]} + disabled={exec.isPending || inPlace} + /> + {profile && !inPlace ? ( +

    + 目标:{roots.find((r) => r.id === profile.target.root)?.hostPath ?? profile.target.root} + {profile.target.path ? `/${profile.target.path}` : ""} · {profile.category ?? "自动分类"} +

    + ) : null} + {profiles.error ? ( +

    + 来源配置读取失败,可手动指定选项。 + +

    + ) : null} + {!profiles.isPending && !profiles.data?.items.length ? ( + - {i < step ? : {i + 1}} - {s} - -
  2. - ))} -
-
- -
{body}
- -
-
- {step === 1 ? ( - - ) : null} - {step === 2 && !executed ? ( - - ) : null} -
-
- {step === 0 ? ( - <> - {rows.length && !canNext ? ( - - {stats.pending ? `${stats.pending} 项待定:选作品或排除` : "没有要整理的项"} - + 去设置新增整理来源 + + ) : null} +
+
+ { + cancelLookup(); + setMediaType(v as typeof mediaType); + setManual(null); + if ( + v !== "auto" && + rows.some((r) => !r.excluded && (!r.entity || (v === "movie") !== (r.entity.kind === "movie"))) + ) { + reidentify.mutate( + { + items: rows.filter((r) => !r.excluded).map((r) => ({ root: r.root, path: r.path })), + minSizeBytes: options.minSizeBytes, + hints: { mediaKind: v === "movie" ? "movie" : "series" }, + }, + { + onSuccess: (r) => { + setRows((rs) => mergeIdentified(rs, r.items)); + expectRemote(r.remoteTaskId); + }, + onError: (e) => toast.error(filesErrorText(e, "按媒体类型识别失败,可手动指定作品")), + }, + ); + } + }} + choices={[ + { value: "auto", label: "自动识别" }, + { value: "movie", label: "电影" }, + { value: "tv", label: "剧集" }, + ]} + disabled={identify.isPending || reidentify.isPending} + /> +
+ 指定作品 + { + cancelLookup(); + pickEntity(e); + }} + placeholder="自动识别 · 或搜索 TMDB 选择作品" + className="h-9" + /> +
+
+
+ + +
+ {manual ? ( +

+ 本批统一使用:{entityLabel(manual)} + {manual.external ? ` · TMDB ${manual.external.externalId}` : ""}。清除作品选择可恢复逐文件自动识别。 +

) : null} - - - ) : null} - {step === 1 ? ( - + ) : null} + {hnrCount && !inPlace ? ( +

{hnrCount} 个文件仍有做种义务,本次使用复制。

+ ) : null} + {!inPlace ? ( +
+ set({ onExisting: v as WizardOptions["onExisting"] })} + choices={[ + { value: "conflict", label: "交给待处理" }, + { value: "skip", label: "跳过已存在的文件" }, + { value: "replace_with_verify", label: "核验新文件后替换" }, + ]} + /> + set({ naming: v as WizardOptions["naming"] })} + choices={[ + { value: "template", label: "按命名模板" }, + { value: "keep", label: "保留原文件名" }, + ]} + /> +
+ ) : null} +
+ + 整理后刮削 + + 成功整理的文件进入刮削任务,生成 NFO 和封面 + + + set({ scrapeAfterImport: v })} + disabled={inPlace && !canScrapeInPlace} + aria-label="整理后刮削" + /> +
+ {!isHistory && successfulCount > 0 ? ( +
+ + 跳过成功记录 + + {successfulCount} 个文件有成功历史,仅对本次生效 + + + set({ skipSuccessfulRecords: v })} + aria-label="跳过成功记录" + /> +
+ ) : null} +
+ 更多选项 +
+ + {!minSizeValid ? ( +

+ 最小体积需要填写有效的非负数。 +

+ ) : null} +
+ 刷新作品资料 + set({ refreshMetadata: v })} + aria-label="刷新作品资料" + /> +
+

+ 同名字幕、外挂音轨随视频整理;NFO 与图片由刮削生成。 + + 修改命名模板 + +

+
+
+
+ {loading ? ( +

+ + 定位源文件并识别作品… +

+ ) : src.error || (!src.items.length && !src.pending) ? ( +

+ {src.error + ? filesErrorText(src.error, "读取来源失败") + : "源文件不存在或不在登记目录中,请在文件管理器重新选择。"} +

+ ) : identify.error ? ( +
+

{filesErrorText(identify.error, "识别失败")}

+ +
+ ) : ( +
+

+ 本次整理 {active.length} 个视频 · 跳过 {stats.excluded} 个 + {stats.pending ? ` · ${stats.pending} 个待确认` : ""} +

+ {!rows.length && started ? ( +

所选范围没有视频文件。

+ ) : null} + +
+ )} + {truncated ? ( +

本批达到 200 个视频上限,请进入子目录分批整理。

+ ) : null} + {src.unresolved.length ? ( +

以下路径不可用:{src.unresolved.join("、")}

+ ) : null} + {filesOpen && rows.length ? ( + r.key} + paginationKey={src.label} + plain + /> + ) : null} +
+ {previewOpen ? ( +
- 生成预览 - - - ) : null} - {step === 2 && preview ? ( - - ) : null} - {step === 2 && executed ? ( - <> - - - +

整理预览

+ {notice ?

{notice}

: null} + {dry.isPending ? ( +

+ + 生成整理计划… +

+ ) : preview ? ( + <> + + + + ) : ( +

+ {dry.error ? filesErrorText(dry.error, "预览失败") : "请生成预览,查看最终路径和处理结果。"} +

+ )} +
) : null}
-
- - +
+ +
+ + +
+
+ + + {targetPicker ? ( + setTargetPicker(false)} + onSelect={(ref) => { + set({ target: { mode: "dir", ref } }); + setTargetPicker(false); + }} + /> + ) : null} + ); } diff --git a/apps/web/app/components/organize-wizard/step-options.tsx b/apps/web/app/components/organize-wizard/step-options.tsx index 897afe25..46a34949 100644 --- a/apps/web/app/components/organize-wizard/step-options.tsx +++ b/apps/web/app/components/organize-wizard/step-options.tsx @@ -70,6 +70,7 @@ export function StepOptions({ roots, hnrCount, kinds, + profileLabel, }: { options: WizardOptions; onChange: (o: WizardOptions) => void; @@ -78,6 +79,7 @@ export function StepOptions({ hnrCount: number; /** 本次涉及的作品类型(过滤库根) */ kinds: Set; + profileLabel?: string; }) { const set = (p: Partial) => onChange({ ...options, ...p }); const libRoots = roots.filter((r) => r.kind === "library" && r.writable && r.health !== "offline"); @@ -109,7 +111,6 @@ export function StepOptions({ title="移动" hint="入库后删源;跨卷时按复制 → 核验 → 删源" /> - {}} disabled="还不支持硬链接" title="硬链接" />
@@ -144,7 +145,11 @@ export function StepOptions({ checked={t.mode === "auto"} onSelect={() => set({ target: { mode: "auto" } })} title="自动" - hint="已绑定的作品沿用原目录,否则按作品类型选媒体库目录" + hint={ + profileLabel + ? `按来源「${profileLabel}」的目标目录与分类规则整理` + : "已绑定的作品沿用原目录,否则按作品类型选媒体库目录" + } /> {libRoots.map((r) => (
-

同名字幕、NFO、图片随主文件一起处理,预览里列出。

+

+ 同目录同名字幕和外挂音轨会随视频整理,预览里列出。现有 NFO、图片不搬入媒体库,可在整理后刮削生成。 +

diff --git a/apps/web/app/components/organize-wizard/step-preview.tsx b/apps/web/app/components/organize-wizard/step-preview.tsx index 786c6365..0ec681c4 100644 --- a/apps/web/app/components/organize-wizard/step-preview.tsx +++ b/apps/web/app/components/organize-wizard/step-preview.tsx @@ -4,8 +4,10 @@ import { Link } from "react-router"; import type { FileRoot, FilesOrganizeGroup } from "~/api/files.ts"; import { RECORD_STATUS_LABEL, RECORD_STATUS_TONE } from "~/api/records.ts"; import { StatusBadge } from "~/components/console/status-badge.tsx"; +import { useListPagination } from "~/components/console/use-list-pagination.ts"; import { baseName, parentRel } from "~/components/files/model.ts"; -import { Button } from "~/components/ui/button.tsx"; +import { recordFileProgress } from "~/components/organize/record-parts.ts"; +import { Pagination } from "~/components/ui/pagination.tsx"; import { Switch } from "~/components/ui/switch.tsx"; import { cn, formatBytes } from "~/lib/utils.ts"; import { @@ -18,7 +20,6 @@ import { groupHasProblem, nestSidecars, type PreviewStats, - pageOf, SKIP_REASON, TEMPLATE_SOURCE, } from "./model.ts"; @@ -108,6 +109,8 @@ function FileLine({ f, roots, nested }: { f: GroupFile; roots: FileRoot[]; neste 跳过{f.skipReason ? ` · ${SKIP_REASON[f.skipReason] ?? f.skipReason}` : ""} ) : f.action === "replace" ? ( 替换 + ) : f.target === f.source ? ( + 原地登记 ) : !f.target ? ( 不入库 ) : null; @@ -202,6 +205,12 @@ export function PreviewGroupCard({ {errorText(g.error)}

) : null} + {result && record && recordFileProgress(record) ? ( +

+ 已完成 {recordFileProgress(record)?.completed}/{recordFileProgress(record)?.total} 个文件 + {recordFileProgress(record)?.current ? ` · 当前:${recordFileProgress(record)?.current}` : ""} +

+ ) : null} {record?.reason && result ?

{record.reason.message}

: null} {g.warnings.map((w) => (

@@ -245,25 +254,21 @@ export function PreviewList({ titleOf, roots, onlyProblems, - page, - onPage, }: { groups: FilesOrganizeGroup[]; titleOf: GroupTitle; roots: FileRoot[]; onlyProblems: boolean; - page: number; - onPage: (p: number) => void; }) { const shown = onlyProblems ? groups.filter(groupHasProblem) : groups; - const pg = pageOf(shown, page); + const { items, pagination } = useListPagination(shown, { resetKey: String(onlyProblems) }); if (!shown.length) return (

没有问题项。关闭「只看问题」查看全部。

); return (
- {pg.items.map((g, i) => ( + {items?.map((g, i) => ( ))} - {pg.pages > 1 ? ( -
- - - {pg.page + 1} / {pg.pages} - - -
- ) : null} +
); } diff --git a/apps/web/app/components/organize/file-source-selection.tsx b/apps/web/app/components/organize/file-source-selection.tsx new file mode 100644 index 00000000..a51925a1 --- /dev/null +++ b/apps/web/app/components/organize/file-source-selection.tsx @@ -0,0 +1,83 @@ +import { useEffect } from "react"; +import { filesErrorText } from "~/api/files.ts"; +import { useOrganizeSources } from "~/api/organize-sources.ts"; +import type { OrganizeSource } from "~/components/organize-wizard/organize-wizard.tsx"; +import { Button } from "~/components/ui/button.tsx"; +import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from "~/components/ui/dialog.tsx"; +import { matchingOrganizeSources } from "./source-match.ts"; + +type FileSource = Extract; +/** Wait for profiles even on the first click; ambiguous sources require an explicit choice. */ +export function FileSourceSelection({ + source, + onSelect, + onClose, +}: { + source: FileSource; + onSelect: (source: FileSource) => void; + onClose: () => void; +}) { + const profiles = useOrganizeSources(); + const matches = matchingOrganizeSources(source.items, profiles.data?.items ?? []); + useEffect(() => { + if (!profiles.isSuccess || matches.length > 1) return; + const profile = matches[0]; + onSelect( + profile + ? { ...source, sourceConfigId: profile.id, defaultAction: profile.action, label: `来源 · ${profile.name}` } + : source, + ); + }, [profiles.isSuccess, matches, source, onSelect]); + return ( + !open && onClose()}> + + + {profiles.isPending ? "读取整理来源…" : "选择整理策略"} + + 匹配来源配置后,使用其整理方式、目标目录与分类规则。媒体类型将在整理预览时核对。 + + + {profiles.isPending ? ( +

+ 正在检查所选文件对应的来源配置… +

+ ) : profiles.error ? ( + <> +

+ {filesErrorText(profiles.error, "读取整理来源失败")} +

+ + + ) : matches.length > 1 ? ( +
+ {matches.map((profile) => ( + + ))} +
+ ) : null} + {!profiles.isPending && (profiles.error || matches.length > 1) ? ( + + ) : null} +
+
+ ); +} diff --git a/apps/web/app/components/organize/organize-queue.tsx b/apps/web/app/components/organize/organize-queue.tsx new file mode 100644 index 00000000..fc677041 --- /dev/null +++ b/apps/web/app/components/organize/organize-queue.tsx @@ -0,0 +1,189 @@ +import { useQueryClient } from "@tanstack/react-query"; +import type { EngineRun } from "@umt/contracts"; +import { FileStack, RefreshCw } from "lucide-react"; +import { Link } from "react-router"; +import { useDownloadTask } from "~/api/downloads.ts"; +import { useEngineRun, useEngineRuns } from "~/api/engine.ts"; +import { useScrapeRecords } from "~/api/files.ts"; +import { ACTION_LABEL, useRecord } from "~/api/records.ts"; +import { useRecordsForDownload } from "~/api/tasks.ts"; +import { EmptyState, ErrorState, LoadingState } from "~/components/console/empty-state.tsx"; +import { StatusBadge } from "~/components/console/status-badge.tsx"; +import { useListPagination } from "~/components/console/use-list-pagination.ts"; +import { relTime } from "~/components/library/parts.tsx"; +import { Button } from "~/components/ui/button.tsx"; +import { Pagination } from "~/components/ui/pagination.tsx"; +import { qk } from "~/lib/query-keys.ts"; +import { useSse } from "~/lib/sse.ts"; +import { organizeQueueStage, organizeRunInput, scrapeQueueStage } from "./queue-model.ts"; +import { recordFileProgress, recordTitle } from "./record-parts.ts"; + +function ScrapeQueueItem({ run }: { run: EngineRun }) { + const detail = useEngineRun(run.id); + const input = organizeRunInput(detail.data?.input); + const scrapeId = input.scrapeRecordId ?? input.recordId; + const scrape = useScrapeRecords(scrapeId ? [scrapeId] : []); + const record = scrape.data?.records[0]; + const status = run.status === "pending" ? "排队" : run.status === "sleeping" ? "等待" : "执行中"; + return ( +
  • +
    + {record?.match?.title ?? "媒体库刮削"} + 刮削 + {status} + {relTime(run.createdAt)} +
    +
    + 目标目录 / 文件 +

    + {record ? `${record.root} / ${record.path || "(根目录)"}` : "读取刮削目标中…"} +

    +
    +
    + {scrapeQueueStage(run)} + {record ? ( + + 已写入 {record.written} · 跳过 {record.skipped} · 失败 {record.failed} + + ) : null} + + 任务详情 + + {record?.organizationRecordId ? ( + + 关联整理记录 + + ) : null} +
    + {detail.error || scrape.error ? ( +

    刮削详细信息暂时无法读取,任务状态仍会自动刷新。

    + ) : null} +
  • + ); +} + +function QueueItem({ run }: { run: EngineRun }) { + const detail = useEngineRun(run.id); + const input = organizeRunInput(detail.data?.input); + const recordId = run.business?.kind === "organization" ? run.business.refId : input.recordId; + const downloadId = run.business?.kind === "download" ? run.business.refId : input.downloadTaskId; + const related = useRecordsForDownload(downloadId); + const matchingRecord = + related.data?.items.find((item) => item.status === "queued" || item.status === "running") ?? + related.data?.items.find((item) => item.createdAt >= run.createdAt); + const recordQuery = useRecord(recordId ?? matchingRecord?.id); + const download = useDownloadTask(downloadId); + const record = recordQuery.data ?? matchingRecord; + const progress = record ? recordFileProgress(record) : null; + const title = record ? recordTitle(record) : (download.data?.name ?? run.business?.title ?? "核验入库任务"); + const source = record?.source.path ?? download.data?.contentPath ?? download.data?.savePath; + const status = run.status === "pending" ? "排队" : run.status === "sleeping" ? "等待" : "执行中"; + return ( +
  • +
    + {title} + {status} + {relTime(run.createdAt)} +
    +
    +
    + 来源 +

    + {source ?? + (detail.isFetching || download.isFetching || recordQuery.isFetching + ? "读取来源信息中…" + : "来源信息暂不可用")} +

    +
    +
    + 目标 +

    {record?.target.path ?? "核验与生成整理计划后确定"}

    +
    +
    +
    + {organizeQueueStage(run, record)} + {record ? {ACTION_LABEL[record.action]} : null} + {progress ? ( + + 已完成 {progress.completed}/{progress.total} 个文件 + {progress.current ? ` · 当前:${progress.current.split("/").pop()}` : ""} + + ) : null} + {run.attempts > 1 ? 第 {run.attempts} 次尝试 : null} + + 任务详情 + + {record ? ( + + 整理详情 + + ) : null} +
    + {detail.error || recordQuery.error || related.error || download.error ? ( +

    部分详细信息暂时无法读取,任务状态仍会自动刷新。

    + ) : null} +
  • + ); +} + +/** Existing engine workflows are authoritative, including jobs that have not created a record yet. */ +export function OrganizeQueue() { + const qc = useQueryClient(); + const runs = useEngineRuns( + { workflowName: "verifyAndImport,scrapeMedia", status: "pending,running,sleeping" }, + true, + 5_000, + ); + useSse(["task.created", "task.status_changed", "organization.changed", "download.status_changed"], () => { + void qc.invalidateQueries({ queryKey: qk.all("engine") }); + void qc.invalidateQueries({ queryKey: ["records"] }); + }); + const rows = runs.data?.pages.flatMap((page) => page.items); + const paged = useListPagination(rows, { + hasMore: runs.hasNextPage, + onLoadMore: () => runs.fetchNextPage(), + loadingMore: runs.isFetchingNextPage, + error: runs.error, + resetKey: "organize-queue", + }); + return ( +
    +
    +

    + 下载核验、手动与自动整理、整理后的 NFO / 图片刮削在此排队。整理完成后进入记录,刮削继续显示到任务结束。 +

    + +
    + {runs.isPending ? ( + + ) : runs.error && !rows ? ( + runs.refetch()} /> + ) : !rows?.length ? ( + + ) : ( +
      + {paged.items?.map((run) => + run.workflowName === "scrapeMedia" ? ( + + ) : ( + + ), + )} +
    + )} + {rows ? : null} +
    + ); +} diff --git a/apps/web/app/components/organize/organize-scrape-history.tsx b/apps/web/app/components/organize/organize-scrape-history.tsx new file mode 100644 index 00000000..d589043a --- /dev/null +++ b/apps/web/app/components/organize/organize-scrape-history.tsx @@ -0,0 +1,78 @@ +import { useQuery } from "@tanstack/react-query"; +import { FilesScrapeRecordsResult } from "@umt/contracts"; +import { Link } from "react-router"; +import { apiParsed } from "~/api/parse.ts"; +import { ErrorState, LoadingState } from "~/components/console/empty-state.tsx"; +import { PaginatedList } from "~/components/console/paginated-list.tsx"; +import { StatusBadge } from "~/components/console/status-badge.tsx"; +import { SCRAPE_STATUS, scrapeCounts } from "~/components/library/scrape-panel.tsx"; +import { reasonLine } from "~/lib/reasons.ts"; +import { formatDateTime } from "~/lib/utils.ts"; + +/** 整理与刮削分别展示结果,避免将刮削失败当作文件搬运失败。 */ +export function OrganizeScrapeHistory({ recordId }: { recordId: string }) { + const query = useQuery({ + queryKey: ["files", "scrape-records", "organization", recordId], + queryFn: ({ signal }) => + apiParsed(FilesScrapeRecordsResult, "/files/scrape-records", { + params: { organizationRecordId: recordId }, + signal, + }), + refetchInterval: 5_000, + }); + return ( +
    +

    整理后刮削

    + {query.isPending ? ( + + ) : query.error ? ( + query.refetch()} /> + ) : query.data?.records.length ? ( + + {(record) => ( +
  • +
    + {SCRAPE_STATUS[record.status].label} + NFO 与图片 + {formatDateTime(record.requestedAt)} +
    +

    + {record.match?.title ?? record.path} · {record.path || "/"} +

    +

    {scrapeCounts(record) || "等待写入 NFO 与图片"}

    + {record.reason ?

    {reasonLine(record.reason)}

    : null} + {record.files.some((file) => file.action === "failed") ? ( +
    + 失败文件 + file.action === "failed")} + label="刮削失败文件" + resetKey={record.id} + as="ul" + className="mt-2 space-y-1" + > + {(file) => ( +
  • + {file.path} · {reasonLine(file.reason) || "写入失败"} +
  • + )} +
    +
    + ) : null} + {record.taskId ? ( + + 查看刮削任务 + + ) : null} +
  • + )} +
    + ) : ( +

    没有关联的刮削记录。

    + )} +
    + ); +} diff --git a/apps/web/app/components/organize/queue-model.test.ts b/apps/web/app/components/organize/queue-model.test.ts new file mode 100644 index 00000000..2f9a2faa --- /dev/null +++ b/apps/web/app/components/organize/queue-model.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from "vitest"; +import { organizeQueueStage, organizeRunInput, scrapeQueueStage } from "./queue-model.ts"; + +describe("organize queue", () => { + it("reads pending workflow references before a record exists", () => { + expect(organizeRunInput({ input: { downloadTaskId: "download", taskId: "task" } })).toEqual({ + downloadTaskId: "download", + recordId: undefined, + }); + expect(organizeQueueStage({ status: "pending" })).toBe("等待执行"); + expect(organizeQueueStage({ status: "running" })).toBe("核验文件与生成整理计划"); + }); + it("distinguishes file transfer, plan waiting, and workflow waiting", () => { + expect(organizeRunInput({ recordId: "record" }).recordId).toBe("record"); + expect(organizeQueueStage({ status: "running" }, { status: "queued" })).toContain("等待搬运"); + expect(organizeQueueStage({ status: "running" }, { status: "running" })).toContain("搬运文件"); + expect(organizeQueueStage({ status: "sleeping" }, { status: "running" })).toBe("等待继续执行"); + }); + it("keeps the independent scrape stage distinct after importing finishes", () => { + expect(organizeRunInput({ input: { scrapeRecordId: "scrape" } }).scrapeRecordId).toBe("scrape"); + expect(scrapeQueueStage({ status: "running" })).toBe("刮削 NFO / 图片"); + expect(scrapeQueueStage({ status: "pending" })).toContain("等待刮削"); + }); + it("原地入库显示核验与登记,不显示搬运", () => { + expect(organizeQueueStage({ status: "running" }, { status: "running", action: "in_place" })).toBe( + "核验原文件与登记媒体库", + ); + expect(organizeQueueStage({ status: "running" }, { status: "queued", action: "in_place" })).toBe("等待原地登记"); + }); + it("does not describe a finished business record as transferring", () => { + expect(organizeQueueStage({ status: "running" }, { status: "succeeded" })).toContain("收尾"); + expect(organizeRunInput(null)).toEqual({}); + expect(organizeRunInput({ input: { recordId: 5 } })).toEqual({ recordId: undefined, downloadTaskId: undefined }); + }); +}); diff --git a/apps/web/app/components/organize/queue-model.ts b/apps/web/app/components/organize/queue-model.ts new file mode 100644 index 00000000..fa1c94ff --- /dev/null +++ b/apps/web/app/components/organize/queue-model.ts @@ -0,0 +1,37 @@ +import type { EngineRun, PublicOrganizationRecord } from "@umt/contracts"; + +export function organizeRunInput(value: unknown): { + recordId?: string; + downloadTaskId?: string; + scrapeRecordId?: string; +} { + if (!value || typeof value !== "object") return {}; + const envelope = value as Record; + const input = + envelope.input && typeof envelope.input === "object" ? (envelope.input as Record) : envelope; + return { + recordId: typeof input.recordId === "string" ? input.recordId : undefined, + downloadTaskId: typeof input.downloadTaskId === "string" ? input.downloadTaskId : undefined, + ...(typeof input.scrapeRecordId === "string" ? { scrapeRecordId: input.scrapeRecordId } : {}), + }; +} + +export function organizeQueueStage( + run: Pick, + record?: Pick & Partial>, +) { + if (run.status === "sleeping") return "等待继续执行"; + if (run.status === "pending") return "等待执行"; + if (!record) return "核验文件与生成整理计划"; + if (record.action === "in_place" && (record.status === "queued" || record.status === "running")) + return record.status === "queued" ? "等待原地登记" : "核验原文件与登记媒体库"; + if (record.status === "queued") return "整理计划已生成,等待搬运"; + if (record.status === "running") return "搬运文件与登记媒体库"; + return "整理已结束,完成任务收尾"; +} + +export function scrapeQueueStage(run: Pick) { + if (run.status === "pending") return "等待刮削 NFO / 图片"; + if (run.status === "sleeping") return "等待继续刮削 NFO / 图片"; + return "刮削 NFO / 图片"; +} diff --git a/apps/web/app/components/organize/record-parts.test.ts b/apps/web/app/components/organize/record-parts.test.ts index d1a9ffbb..1b4116d7 100644 --- a/apps/web/app/components/organize/record-parts.test.ts +++ b/apps/web/app/components/organize/record-parts.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { canReorganize, episodeTag, hasFiles, isRunning, recordTitle } from "./record-parts.ts"; +import { canReorganize, episodeTag, hasFiles, isRunning, recordFileProgress, recordTitle } from "./record-parts.ts"; // HIS-1 整理记录展示小件:标题 / 季集、能否重新整理、能否连文件删除 @@ -52,3 +52,16 @@ describe("整理记录展示", () => { expect(hasFiles({ action: "move" })).toBe(true); }); }); + +describe("文件级进度", () => { + it("没有明细时不推算进度;已结束包含成功、失败和跳过", () => { + expect(recordFileProgress({ files: null })).toBeNull(); + expect(recordFileProgress({ files: [] })).toBeNull(); + const files = ["succeeded", "failed", "skipped", "running", "queued"].map((status, i) => ({ + status, + source: path(`/source/${i}.mkv`), + target: path(null), + })) as unknown as NonNullable[0]["files"]>; + expect(recordFileProgress({ files })).toEqual({ completed: 3, total: 5, current: "/source/3.mkv" }); + }); +}); diff --git a/apps/web/app/components/organize/record-parts.ts b/apps/web/app/components/organize/record-parts.ts index 7892d66e..20260ed1 100644 --- a/apps/web/app/components/organize/record-parts.ts +++ b/apps/web/app/components/organize/record-parts.ts @@ -23,13 +23,27 @@ export function recordTitle(r: Rec): string { /** 能重新整理的记录:复制 / 移动 / 硬链接,且已结束 */ export const canReorganize = (r: Pick) => - (r.action === "copy" || r.action === "move" || r.action === "hardlink") && + (r.action === "copy" || r.action === "move" || r.action === "in_place" || r.action === "hardlink") && r.status !== "queued" && r.status !== "running"; /** 有源文件 / 媒体库文件可删的记录(改名、删除、还原类只删记录本身) */ export const hasFiles = (r: Pick) => - r.action === "copy" || r.action === "move" || r.action === "hardlink"; + r.action === "copy" || r.action === "move" || r.action === "in_place" || r.action === "hardlink"; export const isRunning = (r: Pick) => r.status === "queued" || r.status === "running"; + +/** File counts come from record detail, never from elapsed time or transferred bytes. */ +export function recordFileProgress( + r: Pick, +): { completed: number; total: number; current: string | null } | null { + if (!r.files?.length) return null; + const files = r.files; + const current = files.find((f) => f.status === "running"); + return { + completed: files.filter((f) => !["queued", "running"].includes(f.status)).length, + total: files.length, + current: current?.source.path ?? current?.target.path ?? null, + }; +} diff --git a/apps/web/app/components/organize/source-match.test.ts b/apps/web/app/components/organize/source-match.test.ts new file mode 100644 index 00000000..f0984051 --- /dev/null +++ b/apps/web/app/components/organize/source-match.test.ts @@ -0,0 +1,50 @@ +import type { OrganizeSourceConfig } from "@umt/contracts"; +import { describe, expect, it } from "vitest"; +import { matchingOrganizeSources } from "./source-match.ts"; + +const profile = (id: string, path: string, root = "dl:one:0"): OrganizeSourceConfig => ({ + id, + name: id, + enabled: true, + source: { root, path }, + target: { root: "lib:target", path: "" }, + mediaType: "all", + category: null, + action: "copy", + trigger: "manual", + downloaderInstanceId: null, + tag: null, + intervalSec: 300, +}); +const ref = (path: string, root = "dl:one:0") => ({ root, path }); + +describe("文件管理整理来源匹配", () => { + it("根目录来源覆盖根内所选文件,但不跨根", () => { + const root = profile("root", ""); + expect(matchingOrganizeSources([ref(""), ref("movie/a.mkv")], [root])).toEqual([root]); + expect(matchingOrganizeSources([ref("movie/a.mkv", "dl:two:0")], [root])).toEqual([]); + }); + it("目录匹配保持分段边界,也可选来源目录本身", () => { + const movie = profile("movie", "completed/movie"); + expect(matchingOrganizeSources([ref("completed/movie"), ref("completed/movie/a.mkv")], [movie])).toEqual([movie]); + expect(matchingOrganizeSources([ref("completed/movies/a.mkv")], [movie])).toEqual([]); + expect(matchingOrganizeSources([ref("completed")], [movie])).toEqual([]); + }); + it("所有所选文件必须归属同一配置,空选择不匹配", () => { + const movie = profile("movie", "movie"); + expect(matchingOrganizeSources([ref("movie/a.mkv"), ref("tv/b.mkv")], [movie])).toEqual([]); + expect(matchingOrganizeSources([ref("movie/a.mkv"), ref("movie/b.mkv", "dl:two:0")], [movie])).toEqual([]); + expect(matchingOrganizeSources([], [movie])).toEqual([]); + }); + it("优先最长来源路径,同长度的多个配置全部保留供用户选择", () => { + const all = profile("all", ""); + const completed = profile("completed", "completed"); + const a = profile("a", "completed/movie"); + const b = profile("b", "completed/movie"); + expect(matchingOrganizeSources([ref("completed/movie/a.mkv")], [all, completed, a])).toEqual([a]); + expect(matchingOrganizeSources([ref("completed/movie/a.mkv")], [all, a, b])).toEqual([a, b]); + expect( + matchingOrganizeSources([ref("completed/movie/a.mkv"), ref("completed/tv/b.mkv")], [all, completed, a]), + ).toEqual([completed]); + }); +}); diff --git a/apps/web/app/components/organize/source-match.ts b/apps/web/app/components/organize/source-match.ts new file mode 100644 index 00000000..756ab491 --- /dev/null +++ b/apps/web/app/components/organize/source-match.ts @@ -0,0 +1,15 @@ +import type { FileRef, OrganizeSourceConfig } from "@umt/contracts"; + +/** Match every selected item at directory boundaries, then prefer the most specific source. */ +export function matchingOrganizeSources(items: FileRef[], profiles: T[]): T[] { + if (!items.length) return []; + const matches = profiles.filter((p) => + items.every( + (item) => + item.root === p.source.root && + (p.source.path === "" || item.path === p.source.path || item.path.startsWith(`${p.source.path}/`)), + ), + ); + const longest = Math.max(-1, ...matches.map((p) => p.source.path.length)); + return matches.filter((p) => p.source.path.length === longest); +} diff --git a/apps/web/app/components/organize/source-selection.tsx b/apps/web/app/components/organize/source-selection.tsx new file mode 100644 index 00000000..51757404 --- /dev/null +++ b/apps/web/app/components/organize/source-selection.tsx @@ -0,0 +1,106 @@ +import type { OrganizeSourceView, OrganizeStockResult } from "@umt/contracts"; +import { useState } from "react"; +import { Link } from "react-router"; +import { toast } from "sonner"; +import { filesErrorText } from "~/api/files.ts"; +import { useOrganizeSources, useOrganizeStock } from "~/api/organize-sources.ts"; +import { PaginatedList } from "~/components/console/paginated-list.tsx"; +import { DirectoryDialog } from "~/components/files/directory-dialog.tsx"; +import type { OrganizeSource } from "~/components/organize-wizard/organize-wizard.tsx"; +import { Button } from "~/components/ui/button.tsx"; +import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from "~/components/ui/dialog.tsx"; + +export function stockWizardSource(profile: OrganizeSourceView, stock: OrganizeStockResult): OrganizeSource { + return { + kind: "files", + items: stock.items, + sourceConfigId: profile.id, + defaultAction: profile.action, + label: `整理存量 · ${profile.name}`, + scanSummary: `找到 ${stock.items.length} 个文件,跳过 ${stock.skipped} 个,不可读取 ${stock.unavailable} 个${stock.truncated ? ";扫描已达到上限,仅显示本批结果" : ""}。本次扫描不改变自动整理基线,请先预览再执行。`, + }; +} +export function SourceSelection({ + onClose, + onSelect, +}: { + onClose: () => void; + onSelect: (source: OrganizeSource) => void; +}) { + const sources = useOrganizeSources(); + const stock = useOrganizeStock(); + const [manual, setManual] = useState(false); + if (manual) + return ( + setManual(false)} + onSelect={(ref) => onSelect({ kind: "files", items: [ref], label: "手动选择来源目录" })} + /> + ); + return ( + !open && onClose()}> + + + 选择整理来源 + 扫描来源存量后,进入文件识别、预览与执行。扫描不会改变自动整理基线。 + +
    + {sources.isPending ?

    读取来源中…

    : null} + {sources.error ? ( +

    + 来源配置读取失败,仍可手动选择目录。 +

    + ) : null} + {sources.data?.items.length ? ( + + {(p) => ( + + )} + + ) : null} +
    + + {sources.isSuccess && !sources.data.items.length ? ( + + 去设置新增整理来源 + + ) : null} +
    +
    + ); +} diff --git a/apps/web/app/components/rules/group-list.tsx b/apps/web/app/components/rules/group-list.tsx index fbcc3d16..e0707a08 100644 --- a/apps/web/app/components/rules/group-list.tsx +++ b/apps/web/app/components/rules/group-list.tsx @@ -1,13 +1,16 @@ import type { RuleGroup } from "@umt/contracts"; import { ListChecks, ShieldOff, Tags, Trash2 } from "lucide-react"; +import { useSearchParams } from "react-router"; import { toast } from "sonner"; import { type RuleRefs, useDeleteRuleGroup, useUpdateRuleGroup } from "~/api/rules.ts"; import { ConfirmDialog } from "~/components/console/confirm-dialog.tsx"; import { type DataColumn, DataTable } from "~/components/console/data-table.tsx"; import { EmptyState } from "~/components/console/empty-state.tsx"; import { StatusBadge } from "~/components/console/status-badge.tsx"; +import { useListPagination } from "~/components/console/use-list-pagination.ts"; import { Chip, Panel, relTime } from "~/components/library/parts.tsx"; import { Button } from "~/components/ui/button.tsx"; +import { Pagination } from "~/components/ui/pagination.tsx"; import { Switch } from "~/components/ui/switch.tsx"; import { errorMessage } from "~/lib/api.ts"; import { cn } from "~/lib/utils.ts"; @@ -65,6 +68,7 @@ export function RuleGroupTable({ onOpen: (g: RuleGroup) => void; onCreate: () => void; }) { + const [params] = useSearchParams(); const { toggle, pending } = useToggle(); const columns: DataColumn[] = [ { @@ -132,6 +136,7 @@ export function RuleGroupTable({ g.id} loading={loading} error={error} @@ -234,6 +239,8 @@ function RuleTag({ g, onOpen }: { g: RuleGroup; onOpen: () => void }) { } export function RuleTags({ rows, onOpen }: { rows: RuleGroup[]; onOpen: (g: RuleGroup) => void }) { + const [params] = useSearchParams(); + const paged = useListPagination(rows, { resetKey: params.toString() }); return ( ) : (
      - {rows.map((g) => ( + {paged.items?.map((g) => ( onOpen(g)} /> ))}
    )} + {rows.length > 0 ? : null}
    ); } diff --git a/apps/web/app/components/schedules/auto-task-list.tsx b/apps/web/app/components/schedules/auto-task-list.tsx index df50122e..46aa313e 100644 --- a/apps/web/app/components/schedules/auto-task-list.tsx +++ b/apps/web/app/components/schedules/auto-task-list.tsx @@ -5,7 +5,9 @@ import { Link } from "react-router"; import { toast } from "sonner"; import { useRunScheduleNow, useSkipSchedule, useUpdateSchedule } from "~/api/schedules.ts"; import { StatusBadge } from "~/components/console/status-badge.tsx"; +import { useListPagination } from "~/components/console/use-list-pagination.ts"; import { Button } from "~/components/ui/button.tsx"; +import { Pagination } from "~/components/ui/pagination.tsx"; import { Switch } from "~/components/ui/switch.tsx"; import { errorMessage } from "~/lib/api.ts"; import { formatRelative } from "~/lib/labels.ts"; @@ -228,7 +230,16 @@ export function AutoTaskList({ if (list) list.push(j); else byGroup.set(j.group, [j]); } - const groups = TASK_GROUPS.filter((g) => byGroup.get(g)?.length); + const ordered = TASK_GROUPS.flatMap((g) => byGroup.get(g) ?? []); + const paged = useListPagination(ordered, { resetKey: "auto-tasks" }); + const visibleIds = new Set(paged.items?.map((job) => job.id)); + const groups = TASK_GROUPS.filter((g) => byGroup.get(g)?.some((job) => visibleIds.has(job.id))); + const highlightedIndex = ordered.findIndex((job) => job.id === highlightJob); + // biome-ignore lint/correctness/useExhaustiveDependencies: 仅在深链目标或每页数量变化时定位所在页 + useEffect(() => { + if (highlightedIndex >= 0) + paged.pagination.onPageChange(Math.floor(highlightedIndex / paged.pagination.pageSize) + 1); + }, [highlightJob, highlightedIndex, paged.pagination.pageSize]); // 折叠:有失败 / 运行中 / 被高亮的组默认展开,其余收起;点过的按用户的来并记在本机 const [saved, setSaved] = useState>(() => readCollapseState(COLLAPSE_KEY) ?? {}); const expanded = (g: ScheduleGroup, list: ScheduledJob[]) => @@ -284,14 +295,17 @@ export function AutoTaskList({ {open ? (
      - {list.map((j) => ( - - ))} + {list + .filter((job) => visibleIds.has(job.id)) + .map((j) => ( + + ))}
    ) : null} ); })} + {ordered.length > 0 ? : null}
    ); } diff --git a/apps/web/app/components/schedules/runs-sheet.tsx b/apps/web/app/components/schedules/runs-sheet.tsx index 5965f858..8c0617c5 100644 --- a/apps/web/app/components/schedules/runs-sheet.tsx +++ b/apps/web/app/components/schedules/runs-sheet.tsx @@ -6,6 +6,8 @@ import { DetailSheet } from "~/components/console/detail-sheet.tsx"; import { EmptyState, ErrorState, LoadingState } from "~/components/console/empty-state.tsx"; import { consolePath } from "~/components/console/nav.ts"; import { StatusBadge } from "~/components/console/status-badge.tsx"; +import { useListPagination } from "~/components/console/use-list-pagination.ts"; +import { Pagination } from "~/components/ui/pagination.tsx"; import { agentRunLabel } from "~/lib/labels.ts"; import { describeSchedule, formatClock, formatMs, RUN_STATUS, TRIGGER_LABEL } from "./labels.ts"; @@ -13,6 +15,7 @@ import { describeSchedule, formatClock, formatMs, RUN_STATUS, TRIGGER_LABEL } fr export function RunsSheet({ job, onClose }: { job: ScheduledJob | null; onClose: () => void }) { const runs = useScheduleRuns(job?.id ?? null); const items = runs.data?.items ?? []; + const paged = useListPagination(items, { resetKey: job?.id }); return ( ) : (
      - {items.map((r) => { + {paged.items?.map((r) => { const s = RUN_STATUS[r.status]; return (
    1. @@ -60,6 +63,7 @@ export function RunsSheet({ job, onClose }: { job: ScheduledJob | null; onClose: })}
    )} + {items.length > 0 ? : null}
    ); } diff --git a/apps/web/app/components/schedules/schedule-board.tsx b/apps/web/app/components/schedules/schedule-board.tsx index 73073481..e7c6195b 100644 --- a/apps/web/app/components/schedules/schedule-board.tsx +++ b/apps/web/app/components/schedules/schedule-board.tsx @@ -6,7 +6,9 @@ import { useLocation } from "react-router"; import { usePiUpstreamModels, usePiUpstreams } from "~/api/pi.ts"; import { useScheduleBoard } from "~/api/schedules.ts"; import { EmptyState, ErrorState, LoadingState } from "~/components/console/empty-state.tsx"; +import { useListPagination } from "~/components/console/use-list-pagination.ts"; import { Button } from "~/components/ui/button.tsx"; +import { Pagination } from "~/components/ui/pagination.tsx"; import { qk } from "~/lib/query-keys.ts"; import { useSse } from "~/lib/sse.ts"; import { AgentCheckCard, type DefaultUpstream } from "./agent-check-card.tsx"; @@ -105,6 +107,13 @@ export function PlanBoard({ highlightJob }: { highlightJob?: string }) { const items = data?.items ?? []; const autoJobs = items.filter((j) => j.kind === "task"); const checks = items.filter((j) => j.kind === "agent_check"); + const pagedChecks = useListPagination(checks, { resetKey: "agent-checks" }); + const highlightedIndex = checks.findIndex((job) => job.id === highlightJob); + // biome-ignore lint/correctness/useExhaustiveDependencies: 仅在深链目标或每页数量变化时定位所在页 + useEffect(() => { + if (highlightedIndex >= 0) + pagedChecks.pagination.onPageChange(Math.floor(highlightedIndex / pagedChecks.pagination.pageSize) + 1); + }, [highlightJob, highlightedIndex, pagedChecks.pagination.pageSize]); const def = upstreams.data?.items.find((u) => u.isDefault); // 上游没填默认模型时,运行时用模型目录的第一个(packages/agent pi/upstream.ts pickModel),这里同样取目录首个 const catalog = usePiUpstreamModels(def && !def.defaultModel ? def.id : null); @@ -170,7 +179,7 @@ export function PlanBoard({ highlightJob }: { highlightJob?: string }) { ) : (
    - {checks.map((j) => ( + {pagedChecks.items?.map((j) => ( )} + {checks.length > 0 ? : null}
    setSearchPaletteOpen(false); const go = (to: string) => { + if (q.trim()) history.add(q); close(); navigate(to); }; @@ -183,9 +186,17 @@ function PaletteBody({ initial }: { initial: string }) { className="min-h-0 flex-1 overflow-y-auto p-2 sm:max-h-[60vh]" > {!term ? ( -

    - 输入作品名、别名或页面名称。选中作品打开详情页。 -

    +
    + { + history.add(term); + go(searchPagePath(term)); + }} + /> +

    + 输入作品名、别名或页面名称。选中作品打开详情页。 +

    +
    ) : ( <> {items.map((it, i) => { diff --git a/apps/web/app/components/search/release-results.tsx b/apps/web/app/components/search/release-results.tsx index d9bd1592..dbb26476 100644 --- a/apps/web/app/components/search/release-results.tsx +++ b/apps/web/app/components/search/release-results.tsx @@ -19,9 +19,12 @@ import { type ReleaseSearchFilters, useRefreshReleaseSearch, useReleaseSearch } import { EmptyState, ErrorState, LoadingState } from "~/components/console/empty-state.tsx"; import { StatusBadge } from "~/components/console/status-badge.tsx"; import { TechInfo } from "~/components/console/tech-info.tsx"; +import { useListPagination } from "~/components/console/use-list-pagination.ts"; import { Button } from "~/components/ui/button.tsx"; import { Input } from "~/components/ui/input.tsx"; +import { Pagination } from "~/components/ui/pagination.tsx"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "~/components/ui/select.tsx"; +import { ReleaseDetailSheet } from "~/components/workbench/release-detail-sheet.tsx"; import { relTime } from "~/components/workbench/release-fields.ts"; import { errorMessage } from "~/lib/api.ts"; import { cn, formatBytes } from "~/lib/utils.ts"; @@ -119,7 +122,15 @@ export function ReleaseResults({ const pages = res.data?.pages ?? []; const first = pages[0]; const rows = useMemo(() => pages.flatMap((p) => p.items), [pages]); + const paged = useListPagination(rows, { + hasMore: res.hasNextPage, + onLoadMore: () => res.fetchNextPage(), + loadingMore: res.isFetchingNextPage, + error: res.error, + resetKey: JSON.stringify(api), + }); const [dl, setDl] = useState(null); + const [selected, setSelected] = useState(null); const filtered = selectedCount(quick) > 0 || RELEASE_FILTER_KEYS.some((k) => k !== "sort" && k !== "order" && !!filters[k]); // 旧链接里的免费 / H&R 参数并进快速筛选 @@ -227,18 +238,12 @@ export function ReleaseResults({ /> ) : (
      - {rows.map((r) => ( - setDl(r)} /> + {paged.items?.map((r) => ( + setSelected(r)} onDownload={() => setDl(r)} /> ))}
    )} - {rows.length > 0 && res.hasNextPage ? ( -
    - -
    - ) : null} + {rows.length > 0 || res.hasNextPage ? : null}
    {first ? ( @@ -254,6 +259,17 @@ export function ReleaseResults({ ) : null} {dl ? !o && setDl(null)} /> : null} + {selected?.sourceReleaseId ? ( + setSelected(null)} + /> + ) : null} ); } @@ -404,7 +420,7 @@ function SizeSortControls({ ); } -function ReleaseRow({ r, onDownload }: { r: SearchRelease; onDownload: () => void }) { +function ReleaseRow({ r, onOpen, onDownload }: { r: SearchRelease; onOpen: () => void; onDownload: () => void }) { const promo = promoLabel(r.promotion.type); const inv = r.inventory ? INVENTORY_STATE[r.inventory.state] : null; const p = r.parsed; @@ -412,7 +428,14 @@ function ReleaseRow({ r, onDownload }: { r: SearchRelease; onDownload: () => voi return (
  • - {r.title} + {promo ? {promo} : null} {r.hr === true ? H&R : null} diff --git a/apps/web/app/components/search/search-download-dialog.tsx b/apps/web/app/components/search/search-download-dialog.tsx index 945c396f..89d062a7 100644 --- a/apps/web/app/components/search/search-download-dialog.tsx +++ b/apps/web/app/components/search/search-download-dialog.tsx @@ -1,44 +1,15 @@ -import type { ManualAcquisitionResult, SearchRelease } from "@umt/contracts"; -import { Bot, Download } from "lucide-react"; -import { useState } from "react"; -import { Link } from "react-router"; -import { toast } from "sonner"; +import type { SearchRelease } from "@umt/contracts"; import { usePlanFromSearch } from "~/api/search.ts"; -import { useSlots } from "~/api/slots.ts"; import { openAgentDrawer } from "~/components/agent/agent-store.ts"; -import { FilterSelect } from "~/components/console/filter-bar.tsx"; -import { SLOT_STATUS } from "~/components/library/labels.ts"; -import { Button } from "~/components/ui/button.tsx"; -import { - Dialog, - DialogContent, - DialogDescription, - DialogFooter, - DialogHeader, - DialogTitle, -} from "~/components/ui/dialog.tsx"; -import { errorMessage } from "~/lib/api.ts"; -import { formatBytes } from "~/lib/utils.ts"; +import { ManualDownloadDialog } from "~/components/downloads/manual-download-dialog.tsx"; -// 与站点工作台同一套下载说明;请求走 POST /search/releases/{siteId}/{siteReleaseId}/plan(同 /acquisition-plans 流程) - -const STEPS = [ - "预留收录版本和预算;每个收录版本同时只有一个在途的下载", - "已有相同资源或相同 hash 的任务时直接接管,不重复提交", - "缓存卷超过硬阈值或读不到磁盘状态时无法提交", - "提交前复核免费、做种等易变字段", -]; - -/** 委托 Agent:和工作台一样带上资源、作品、站点 */ -export function delegateToAgent(r: SearchRelease) { +export function delegateToAgent(release: SearchRelease) { openAgentDrawer({ - label: `资源:${r.title}`, - releaseId: r.sourceReleaseId ?? undefined, - entityId: r.entity?.id, - siteId: r.siteId, - goal: r.sourceReleaseId - ? "判断这个资源是否值得下载;合适就按正常流程提交" - : `判断站点 ${r.siteName} 的资源「${r.title}」是否值得下载;合适就按正常流程提交`, + label: `资源:${release.title}`, + releaseId: release.sourceReleaseId ?? undefined, + entityId: release.entity?.id, + siteId: release.siteId, + goal: `判断资源「${release.title}」是否值得下载;合适就按手动下载流程提交`, }); } @@ -49,124 +20,23 @@ export function SearchDownloadDialog({ }: { release: SearchRelease; open: boolean; - onOpenChange: (o: boolean) => void; + onOpenChange: (open: boolean) => void; }) { - const plan = usePlanFromSearch(); - const entityId = release.entity?.id; - const slots = useSlots({ mediaEntityId: entityId }, open && !!entityId); - const slotList = slots.data?.items ?? []; - const [slotId, setSlotId] = useState(""); - const [rejected, setRejected] = useState(null); - const [done, setDone] = useState(null); - const slotName = (id: string | null) => slotList.find((s) => s.id === id)?.name ?? null; - - const go = () => - plan.mutate( - { - siteId: release.siteId, - siteReleaseId: release.siteReleaseId, - ...(slotId ? { slotId } : entityId ? { entityId } : {}), - }, - { - onSuccess: (r) => { - if (r.submitted && r.plan) { - setRejected(null); - setDone(r.plan.id); - toast.success(`已提交${slotName(r.slotId) ? `(${slotName(r.slotId)})` : ""},进度见「下载面板」`); - } else { - setRejected(r.rejected); - toast.warning("没有下载"); - } - }, - onError: (e) => toast.error(errorMessage(e)), - }, - ); - + const submit = usePlanFromSearch(); return ( - { - if (!o) { - setRejected(null); - setDone(null); - } - onOpenChange(o); + onOpenChange={onOpenChange} + resource={{ + title: release.title, + sizeBytes: release.sizeBytes, + entityId: release.entityMatch === "resolved" ? release.entity?.id : undefined, }} - > - - - 下载这个资源 - - {release.siteName} · {release.title} - {release.sizeBytes !== null ? ` · ${formatBytes(release.sizeBytes, 1)}` : ""} - - -
      - {STEPS.map((s) => ( -
    1. {s}
    2. - ))} -
    - {!entityId ? ( -

    资源未关联本地作品。先在「身份与元数据」指定。

    - ) : slots.isPending ? ( -

    正在读取收录版本…

    - ) : slotList.length ? ( -
    - ({ - value: s.id, - label: `${s.name}(${SLOT_STATUS[s.status]?.label ?? s.status})`, - }))} - /> -
    - ) : ( -

    这个作品没有收录版本。先在订阅里新建。

    - )} - {release.entityMatch === "alias" ? ( -

    作品关联来自标题别名,尚未确认。

    - ) : null} - {rejected ? ( -
    -

    没有下载:

    -
      - {rejected.map((r, i) => ( -
    • - {slotName(r.slotId) ? `${slotName(r.slotId)}:` : ""} - {r.message} -
    • - ))} -
    -
    - ) : null} - {done ? ( -

    - 已加入下载。 - - 查看 - -

    - ) : null} - - - - -
    -
    + onDelegate={() => delegateToAgent(release)} + submit={(options) => + submit.mutateAsync({ ...options, siteId: release.siteId, siteReleaseId: release.siteReleaseId }) + } + /> ); } diff --git a/apps/web/app/components/search/search-history.test.ts b/apps/web/app/components/search/search-history.test.ts new file mode 100644 index 00000000..d7ddaad6 --- /dev/null +++ b/apps/web/app/components/search/search-history.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it } from "vitest"; +import { addHistory, HISTORY_LIMIT, parseHistory } from "./search-history.tsx"; + +describe("search history", () => { + it("ignores malformed storage and non-text entries", () => { + expect(parseHistory("invalid json")).toEqual([]); + expect(parseHistory('[null,8," ","剧名","剧名"]')).toEqual(["剧名"]); + }); + it("moves repeat searches to the front without saving intermediate typing", () => { + expect(addHistory(["Movie", "剧集"], " movie ")).toEqual(["movie", "剧集"]); + expect(addHistory(["剧集"], " ")).toEqual(["剧集"]); + }); + it("retains only the latest searches", () => { + const history = Array.from({ length: HISTORY_LIMIT }, (_, i) => `query-${i}`); + const next = addHistory(history, "新搜索"); + expect(next).toHaveLength(HISTORY_LIMIT); + expect(next[0]).toBe("新搜索"); + expect(next).not.toContain(history.at(-1)); + }); +}); diff --git a/apps/web/app/components/search/search-history.tsx b/apps/web/app/components/search/search-history.tsx new file mode 100644 index 00000000..f3b97db8 --- /dev/null +++ b/apps/web/app/components/search/search-history.tsx @@ -0,0 +1,114 @@ +import { History, X } from "lucide-react"; +import { useCallback, useMemo, useSyncExternalStore } from "react"; +import { useSession } from "~/api/auth.ts"; +import { Button } from "~/components/ui/button.tsx"; + +const EVENT = "umt:search-history"; +export const HISTORY_LIMIT = 30; + +export function parseHistory(raw: string): string[] { + try { + const value: unknown = JSON.parse(raw); + if (!Array.isArray(value)) return []; + return [ + ...new Set(value.filter((s): s is string => typeof s === "string" && !!s.trim()).map((s) => s.trim())), + ].slice(0, HISTORY_LIMIT); + } catch { + return []; + } +} +export function addHistory(history: string[], term: string) { + const q = term.trim(); + return q + ? [q, ...history.filter((s) => s.toLocaleLowerCase() !== q.toLocaleLowerCase())].slice(0, HISTORY_LIMIT) + : history; +} +function subscribe(notify: () => void) { + window.addEventListener(EVENT, notify); + window.addEventListener("storage", notify); + return () => { + window.removeEventListener(EVENT, notify); + window.removeEventListener("storage", notify); + }; +} +const memory = new Map(); +function read(key: string) { + try { + return localStorage.getItem(key) ?? ""; + } catch { + return memory.get(key) ?? ""; + } +} +export function useSearchHistory() { + const session = useSession(); + const actor = session.data?.actor; + const key = actor ? `umt.search-history.${actor.type}.${actor.id}` : ""; + const raw = useSyncExternalStore( + subscribe, + () => (key ? read(key) : ""), + () => "", + ); + const entries = useMemo(() => parseHistory(raw), [raw]); + const write = useCallback( + (next: string[]) => { + if (!key) return; + const value = JSON.stringify(next); + memory.set(key, value); + try { + localStorage.setItem(key, value); + } catch { + /* session-only fallback */ + } + window.dispatchEvent(new Event(EVENT)); + }, + [key], + ); + return { + entries, + add: useCallback((term: string) => write(addHistory(parseHistory(key ? read(key) : ""), term)), [key, write]), + remove: (term: string) => write(entries.filter((s) => s !== term)), + clear: () => write([]), + }; +} + +export function SearchHistory({ onSearch }: { onSearch: (term: string) => void }) { + const history = useSearchHistory(); + if (!history.entries.length) return null; + return ( +
    +
    +

    + + 搜索历史 +

    + +
    +
    + {history.entries.map((term) => ( +
    + + +
    + ))} +
    +
    + ); +} diff --git a/apps/web/app/components/search/search-sections.tsx b/apps/web/app/components/search/search-sections.tsx index 9f59befa..eee06ecf 100644 --- a/apps/web/app/components/search/search-sections.tsx +++ b/apps/web/app/components/search/search-sections.tsx @@ -6,7 +6,9 @@ import { mediaServers } from "~/api/instances.ts"; import { useSearch } from "~/api/search.ts"; import { EmptyState, ErrorState, LoadingState } from "~/components/console/empty-state.tsx"; import { settingsPath } from "~/components/console/nav.ts"; +import { useListPagination } from "~/components/console/use-list-pagination.ts"; import { Button } from "~/components/ui/button.tsx"; +import { Pagination } from "~/components/ui/pagination.tsx"; import { actionIcon } from "./search-labels.ts"; import { SitePickerDialog } from "./site-picker-dialog.tsx"; import { releaseSearchPath, WorkCard, workRef } from "./work-card.tsx"; @@ -40,11 +42,12 @@ export function useReleaseSearchLauncher() { } export function WorksSection({ q }: { q: string }) { - const res = useSearch(q, "media", 20); + const res = useSearch(q, "media", 50); const ms = mediaServers.useList(); const emby = (ms.data?.items ?? []).some((i) => (i as { enabled?: boolean }).enabled !== false); const launcher = useReleaseSearchLauncher(); const works = res.data?.media?.works ?? []; + const paged = useListPagination(works, { resetKey: q }); const tmdb = res.data?.media?.tmdb; const searchReleases = (w: SearchWork) => { const ref = workRef(w); @@ -77,11 +80,12 @@ export function WorksSection({ q }: { q: string }) {
    ) : (
    - {works.map((w) => ( + {paged.items?.map((w) => ( ))}
    )} + {res.data ? : null} {tmdb && !tmdb.configured ? (

    TMDB 未配置,只搜了本地作品。 diff --git a/apps/web/app/components/settings/account-passkeys.tsx b/apps/web/app/components/settings/account-passkeys.tsx index cb8aae6b..6a6f5c8f 100644 --- a/apps/web/app/components/settings/account-passkeys.tsx +++ b/apps/web/app/components/settings/account-passkeys.tsx @@ -11,8 +11,10 @@ import { useRenamePasskey, } from "~/api/auth.ts"; import { ConfirmDialog } from "~/components/console/confirm-dialog.tsx"; +import { useListPagination } from "~/components/console/use-list-pagination.ts"; import { Button } from "~/components/ui/button.tsx"; import { Input } from "~/components/ui/input.tsx"; +import { Pagination } from "~/components/ui/pagination.tsx"; import { errorMessage } from "~/lib/api.ts"; import { formatDateTime } from "~/lib/utils.ts"; import { SettingsBlock } from "./settings-block.tsx"; @@ -124,6 +126,7 @@ export function PasskeyBlock({ disabled }: { disabled: boolean }) { const [name, setName] = useState(""); const supported = passkeySupported(); const items = list.data?.items ?? []; + const paged = useListPagination(items, { resetKey: String(disabled) }); const add = async (e: React.FormEvent) => { e.preventDefault(); @@ -147,13 +150,14 @@ export function PasskeyBlock({ disabled }: { disabled: boolean }) {

    读取中…

    ) : items.length ? (
      - {items.map((p) => ( + {paged.items?.map((p) => ( ))}
    ) : (

    还没有登记 Passkey。

    )} + {items.length > 0 ? : null} {!supported ? (

    这个浏览器用不了 Passkey:需要 HTTPS 或 localhost 访问。

    ) : adding ? ( diff --git a/apps/web/app/components/settings/library.tsx b/apps/web/app/components/settings/library.tsx index aca6bd84..b3db685b 100644 --- a/apps/web/app/components/settings/library.tsx +++ b/apps/web/app/components/settings/library.tsx @@ -1,5 +1,6 @@ import { LibraryRootsTable } from "./instances.tsx"; import { NamingSection } from "./naming.tsx"; +import { ClassificationSection, OrganizeSourcesSection } from "./organize-sources.tsx"; import { str } from "./resource-section.tsx"; import { SettingsBlock } from "./settings-block.tsx"; import { type CardSpec, SettingsCard, SettingsGate } from "./settings-card.tsx"; @@ -98,6 +99,8 @@ export function LibrarySection() { {(s) => } {(s) => } + + ); diff --git a/apps/web/app/components/settings/organize-sources.tsx b/apps/web/app/components/settings/organize-sources.tsx new file mode 100644 index 00000000..38656809 --- /dev/null +++ b/apps/web/app/components/settings/organize-sources.tsx @@ -0,0 +1,470 @@ +import type { FileRef, OrganizeSourceInput, OrganizeSourceView } from "@umt/contracts"; +import { cloneElement, type ReactElement, useId, useState } from "react"; +import { toast } from "sonner"; +import { filesErrorText, useFileRoots } from "~/api/files.ts"; +import { downloaders } from "~/api/instances.ts"; +import { + useClassification, + useOrganizeSources, + useOrganizeStock, + useRemoveOrganizeSource, + useSaveClassification, + useSaveOrganizeSource, +} from "~/api/organize-sources.ts"; +import { openAgentDrawer } from "~/components/agent/agent-store.ts"; +import { useListPagination } from "~/components/console/use-list-pagination.ts"; +import { DirectoryDialog } from "~/components/files/directory-dialog.tsx"; +import { stockWizardSource } from "~/components/organize/source-selection.tsx"; +import { type OrganizeSource, OrganizeWizard } from "~/components/organize-wizard/organize-wizard.tsx"; +import { Button } from "~/components/ui/button.tsx"; +import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from "~/components/ui/dialog.tsx"; +import { Input } from "~/components/ui/input.tsx"; +import { Pagination } from "~/components/ui/pagination.tsx"; +import { Switch } from "~/components/ui/switch.tsx"; +import { Textarea } from "~/components/ui/textarea.tsx"; +import { formatDateTime } from "~/lib/utils.ts"; +import { SaveBar, SettingsBlock } from "./settings-block.tsx"; + +const selectClass = "w-full rounded-lg border border-border bg-background px-3 py-2 text-caption"; +function Field({ label, children }: { label: string; children: ReactElement<{ id?: string }> }) { + const id = useId(); + return ( + + ); +} +export function ClassificationSection() { + const config = useClassification(); + const save = useSaveClassification(); + const [draft, setDraft] = useState(null); + const yaml = draft ?? config.data?.yaml ?? ""; + const dirty = draft !== null && draft !== config.data?.yaml; + return ( + + openAgentDrawer({ + label: "媒体分类规则", + goal: `协助检查和修改媒体分类 YAML,说明调整后的匹配结果。当前规则:\n${yaml}`, + }) + } + > + 让智能体协助 + + } + > + {config.error ? ( +

    + {filesErrorText(config.error, "读取分类规则失败")} +

    + ) : null} +