From 829ca4f7aceebe82ecc7b00b2bec90c156b0de0b Mon Sep 17 00:00:00 2001 From: root Date: Sun, 27 Sep 2026 20:08:47 +0800 Subject: [PATCH] feat(server): W2-C rules, downloads, import, organization records, library and dashboard - Rules API: subscriptions, acquisition slots (revision lock), policy revisions (draft -> publish = automation_grant, rollback as new revision), preference tags with suppression records (agent cannot re-add suppressed fingerprints). - Tasks: evaluateAcquisition (mode/evaluate/budget/cache volume -> frozen plan with slot reservation, candidate reasons as evidence artifacts), submitDownload (withAttemptToken fencing, already_exists adoption), verifyAndImport (frozen organization_record + file results, resumable copy/move/cross-volume steps, assets/locations/bindings in one tx), cleanupDownloadCache (verified+retention+H&R only, assertSafeToDeleteData), reconcileLibrary (root sentinel, suspected -> confirmed missing). - Worker loop: qB sync with tag adoption, no-progress evidence, hard threshold pause / resume-watermark resume, sweepers for deferred plans and imports. - Public organization records API incl. changes cursor (change_xid guard), media library list/detail, inventory overlay, Emby inventory, dashboard. - Contracts: SlotTemplate, AcquisitionPolicyRules, CandidateEvaluation, library/inventory/dashboard read models. Co-Authored-By: Claude Opus 5.5 --- .../acquisition-organize/__tests__/harness.ts | 347 +++++ .../__tests__/organize-move.test.ts | 246 ++++ .../__tests__/rules-api.test.ts | 252 ++++ .../__tests__/vertical.test.ts | 462 ++++++ .../modules/acquisition-organize/handlers.ts | 197 +++ .../src/modules/acquisition-organize/index.ts | 144 +- .../src/modules/acquisition-organize/ports.ts | 144 ++ .../acquisition-organize/routes/dashboard.ts | 288 ++++ .../acquisition-organize/routes/downloads.ts | 115 ++ .../acquisition-organize/routes/helpers.ts | 93 ++ .../acquisition-organize/routes/library.ts | 430 ++++++ .../routes/organization.ts | 170 +++ .../acquisition-organize/routes/rules.ts | 730 ++++++++++ .../services/acquisition.ts | 655 +++++++++ .../acquisition-organize/services/common.ts | 237 +++ .../services/downloads.ts | 917 ++++++++++++ .../acquisition-organize/services/organize.ts | 1292 +++++++++++++++++ .../services/reconcile.ts | 228 +++ .../services/retention.ts | 36 + .../acquisition-organize/services/volumes.ts | 121 ++ .../src/modules/acquisition-organize/util.ts | 89 ++ apps/server/test/modules.test.ts | 7 +- packages/contracts/src/organization.ts | 246 +++- packages/contracts/src/policy.ts | 51 + 24 files changed, 7488 insertions(+), 9 deletions(-) create mode 100644 apps/server/src/modules/acquisition-organize/__tests__/harness.ts create mode 100644 apps/server/src/modules/acquisition-organize/__tests__/organize-move.test.ts create mode 100644 apps/server/src/modules/acquisition-organize/__tests__/rules-api.test.ts create mode 100644 apps/server/src/modules/acquisition-organize/__tests__/vertical.test.ts create mode 100644 apps/server/src/modules/acquisition-organize/handlers.ts create mode 100644 apps/server/src/modules/acquisition-organize/ports.ts create mode 100644 apps/server/src/modules/acquisition-organize/routes/dashboard.ts create mode 100644 apps/server/src/modules/acquisition-organize/routes/downloads.ts create mode 100644 apps/server/src/modules/acquisition-organize/routes/helpers.ts create mode 100644 apps/server/src/modules/acquisition-organize/routes/library.ts create mode 100644 apps/server/src/modules/acquisition-organize/routes/organization.ts create mode 100644 apps/server/src/modules/acquisition-organize/routes/rules.ts create mode 100644 apps/server/src/modules/acquisition-organize/services/acquisition.ts create mode 100644 apps/server/src/modules/acquisition-organize/services/common.ts create mode 100644 apps/server/src/modules/acquisition-organize/services/downloads.ts create mode 100644 apps/server/src/modules/acquisition-organize/services/organize.ts create mode 100644 apps/server/src/modules/acquisition-organize/services/reconcile.ts create mode 100644 apps/server/src/modules/acquisition-organize/services/retention.ts create mode 100644 apps/server/src/modules/acquisition-organize/services/volumes.ts create mode 100644 apps/server/src/modules/acquisition-organize/util.ts diff --git a/apps/server/src/modules/acquisition-organize/__tests__/harness.ts b/apps/server/src/modules/acquisition-organize/__tests__/harness.ts new file mode 100644 index 00000000..376a57b3 --- /dev/null +++ b/apps/server/src/modules/acquisition-organize/__tests__/harness.ts @@ -0,0 +1,347 @@ +import { readdirSync, statSync } from "node:fs"; +import { join } from "node:path"; +import type { DiskUsage } from "@umt/adapters"; +import { createQbClient } from "@umt/adapters"; +import { DEFAULT_SETTINGS, GlobalSettings, type SettingsKey } from "@umt/contracts"; +import { type DB, migrateToLatest } from "@umt/db"; +import type { TaskEngine } from "@umt/tasks"; +import { Kysely, PostgresDialect, sql } from "kysely"; +import pg from "pg"; +import { ConfigSchema } from "../../../config.ts"; +import { createOutboxPort } from "../../../context.ts"; +import { createApp } from "../../../http/app.ts"; +import { hashApiKey } from "../../../http/auth.ts"; +import { EventBus } from "../../../http/sse.ts"; +import { createLogger } from "../../../logger.ts"; +import { createCapabilityRegistry } from "../../agent/capability-registry.ts"; +import type { ServerContext } from "../../types.ts"; +import { type AoOverrides, createAcquisitionOrganizeModule } from "../index.ts"; +import type { AoDeps } from "../services/common.ts"; + +// 真实 PG(独立 schema)+ mock qB + 临时目录 的测试装配;引擎用记录入队的假实现,测试直接驱动服务函数。 + +export const DB_URL = process.env.DATABASE_URL ?? "postgres://umt:umt@localhost:55433/umt"; +export const DEV_KEY = "dev-admin-ao-test-key"; +/** 只读整理记录 / 可发起整理 的 API Key */ +export const READ_KEY = "ao-organize-read-key"; +export const EXEC_KEY = "ao-organize-exec-key"; +const KEYS = [ + { key: READ_KEY, scopes: ["organize.read"] }, + { key: EXEC_KEY, scopes: ["organize.execute"] }, +]; + +export async function pgAvailable() { + if (process.env.UMT_SKIP_DB_TESTS === "1") return false; + const c = new pg.Client({ connectionString: DB_URL, connectionTimeoutMillis: 2000 }); + try { + await c.connect(); + await c.end(); + return true; + } catch { + return false; + } +} + +export interface Enqueued { + task: string; + input: unknown; + key: string; +} + +/** 记录入队、不执行的引擎(处理器由测试直接调服务函数) */ +export function fakeEngine(): TaskEngine & { enqueued: Enqueued[] } { + const enqueued: Enqueued[] = []; + let n = 0; + return { + enqueued, + async enqueue(task, input, opts) { + enqueued.push({ task, input, key: opts.idempotencyKey }); + n++; + return { runId: `run-${n}`, deduplicated: false }; + }, + async signal() { + return { delivered: 0 }; + }, + async cancel() { + return false; + }, + async get() { + return null; + }, + async list() { + return []; + }, + async startWorker() {}, + async stop() {}, + }; +} + +/** 可调的磁盘用量:used = base + 目录下文件总大小(清理删文件后自然下降) */ +export function fakeDisk(total: number) { + const state = { total, base: 0, watch: [] as string[], fail: false }; + const du = (dir: string): number => { + let s = 0; + try { + for (const n of readdirSync(dir)) { + const p = join(dir, n); + const st = statSync(p); + s += st.isDirectory() ? du(p) : st.size; + } + } catch { + // 不存在 + } + return s; + }; + const fn = async (path: string): Promise => { + if (state.fail) return { status: "unknown", path, code: "EIO", message: "fake" }; + const used = state.base + state.watch.reduce((a, d) => a + du(d), 0); + return { + status: "ok", + path, + totalBytes: state.total, + freeBytes: state.total - used, + availableBytes: state.total - used, + usedBytes: used, + usedRatio: used / state.total, + }; + }; + /** 把使用率设为 pct(按当前目录内容反推 base) */ + const setPct = (pct: number) => { + const files = state.watch.reduce((a, d) => a + du(d), 0); + state.base = Math.round((pct / 100) * state.total) - files; + }; + return { state, fn, setPct }; +} + +// biome-ignore lint/suspicious/noExplicitAny: 测试断言直接读响应/事件 JSON +type Loose = any; + +export interface Harness { + db: Kysely; + ctx: ServerContext; + d: AoDeps; + engine: ReturnType; + app: ReturnType; + api(method: string, path: string, body?: unknown, key?: string): Promise<{ status: number; body: Loose }>; + setSettings(key: string, value: Record): Promise; + outbox(type?: string): Promise<{ type: string; payload: Loose; idempotency_key: string | null }[]>; + close(): Promise; +} + +export async function createHarness(name: string, overrides: AoOverrides = {}): Promise { + const schema = `ao_test_${name}_${process.pid.toString(36)}${Math.random().toString(36).slice(2, 7)}`; + const admin = new pg.Client({ connectionString: DB_URL }); + await admin.connect(); + await admin.query(`create schema ${schema}`); + await admin.end(); + const db = new Kysely({ + dialect: new PostgresDialect({ + pool: new pg.Pool({ connectionString: DB_URL, max: 6, options: `-c search_path=${schema},public` }), + }), + }); + await migrateToLatest(db, { migrationTableSchema: schema }); + const logger = createLogger({ level: "silent" }); + const engine = fakeEngine(); + const config = ConfigSchema.parse({ + DATABASE_URL: DB_URL, + SECRET_MASTER_KEY: "test-master-key-0123456789", + DEV_ADMIN_KEY: DEV_KEY, + }); + const outbox = createOutboxPort(); + const bus = new EventBus(50); + const ctx: ServerContext = { + role: "all", + db, + config, + logger, + events: bus, + engine, + outbox, + secrets: { + resolve: async () => "adminadmin", + create: async () => "x", + replace: async () => {}, + remove: async () => {}, + }, + // 不缓存的设置读取(测试里改 settings 行立即生效) + settings: { + async all() { + const rows = await db.selectFrom("settings").select(["key", "value"]).execute(); + const stored = new Map(rows.map((r) => [r.key, r.value])); + const out: Record = {}; + for (const k of Object.keys(DEFAULT_SETTINGS) as SettingsKey[]) { + const v = stored.get(k); + out[k] = GlobalSettings.shape[k].parse({ ...DEFAULT_SETTINGS[k], ...((v as object) ?? {}) }); + } + return out as GlobalSettings; + }, + async get(k) { + return (await this.all())[k]; + }, + }, + capabilities: createCapabilityRegistry({ logger }), + }; + const module = createAcquisitionOrganizeModule({ + downloader: async (inst) => + createQbClient({ + instanceId: inst.id, + baseUrl: inst.base_url, + username: "admin", + password: "adminadmin", + lookup: { attempts: 3, intervalMs: 50 }, + }), + probe: async () => ({ status: "unknown", reason: "ffprobe_unavailable" }), + ...overrides, + options: { backgroundLoops: false, minVideoBytes: 100_000, verifyHash: "sha256", ...(overrides.options ?? {}) }, + }); + const app = createApp({ + config: { NODE_ENV: "development", DEV_ADMIN_KEY: DEV_KEY }, + logger, + bus, + role: "all", + version: "test", + db, + modules: [module], + ctx, + apiKeyStore: { + async findByHash(hash) { + const k = KEYS.find((x) => hashApiKey(x.key) === hash); + return k ? { id: k.key, name: k.key, scopes: k.scopes, revokedAt: null } : null; + }, + async touch() {}, + }, + }); + const d = module.deps(); + return { + db, + ctx, + d, + engine, + app, + async api(method, path, body, key = DEV_KEY) { + const res = await app.request(`/api/v1${path}`, { + method, + headers: { + authorization: `Bearer ${key}`, + ...(body !== undefined ? { "content-type": "application/json" } : {}), + }, + ...(body !== undefined ? { body: JSON.stringify(body) } : {}), + }); + return { status: res.status, body: await res.json() }; + }, + async setSettings(key, value) { + await sql`insert into settings (key, value) values (${key}, ${JSON.stringify(value)}::jsonb) + on conflict (key) do update set value = excluded.value`.execute(db); + }, + async outbox(type) { + const r = await sql<{ type: string; payload: Loose; idempotency_key: string | null }>` + select type, payload, idempotency_key from outbox_event ${type ? sql`where type = ${type}` : sql``} order by at, id`.execute( + db, + ); + return r.rows; + }, + async close() { + await db.destroy(); + const c = new pg.Client({ connectionString: DB_URL }); + await c.connect(); + await c.query(`drop schema if exists ${schema} cascade`); + await c.end(); + }, + }; +} + +// ── 种子数据 ── + +export async function seedBase( + db: Kysely, + opts: { libraryPath: string; cachePath: string; qbUrl: string; mediaKind?: "movie" | "series" }, +) { + const root = await db + .insertInto("library_root") + .values({ + name: `lib-${Math.random().toString(36).slice(2, 6)}`, + media_kind: "mixed", + local_path: opts.libraryPath, + management_mode: "managed", + }) + .returning("id") + .executeTakeFirstOrThrow(); + const inst = await db + .insertInto("downloader_instance") + .values({ + name: `qb-${Math.random().toString(36).slice(2, 6)}`, + base_url: opts.qbUrl, + username: "admin", + settings: JSON.stringify({ savePath: opts.cachePath }), + }) + .returning("id") + .executeTakeFirstOrThrow(); + const site = await db + .insertInto("site") + .values({ + key: `site${Math.random().toString(36).slice(2, 6)}`, + name: "测试站", + adapter: "nexusphp", + base_url: "https://pt.example.test", + config: JSON.stringify({ hitAndRun: { minSeedingSec: 86_400 } }), + }) + .returning("id") + .executeTakeFirstOrThrow(); + return { rootId: root.id, instanceId: inst.id, siteId: site.id }; +} + +export async function seedEntity(db: Kysely, e: { kind: "movie" | "series"; title: string; year: number }) { + return ( + await db + .insertInto("media_entity") + .values({ kind: e.kind, title: e.title, year: e.year, status: "active" }) + .returning("id") + .executeTakeFirstOrThrow() + ).id; +} + +export async function seedRelease( + db: Kysely, + r: { siteId: string; entityId: string; title: string; size: number; hash: string; hr?: boolean | "unknown" }, +) { + return ( + await db + .insertInto("source_release") + .values({ + site_id: r.siteId, + site_release_id: `${Math.floor(Math.random() * 1e9)}`, + title: r.title, + size_bytes: r.size, + infohash_v1: r.hash, + media_entity_id: r.entityId, + resolution_status: "resolved", + raw: JSON.stringify({ _site: { hr: r.hr ?? false } }), + }) + .returning("id") + .executeTakeFirstOrThrow() + ).id; +} + +export async function seedSlot( + db: Kysely, + s: { entityId: string; name?: string; hard?: Record; upgrade?: Record }, +) { + const sub = await db + .insertInto("subscription") + .values({ media_entity_id: s.entityId, media_kind: "movie", mode: "continuous" }) + .returning("id") + .executeTakeFirstOrThrow(); + return ( + await db + .insertInto("acquisition_slot") + .values({ + media_entity_id: s.entityId, + subscription_id: sub.id, + name: s.name ?? "main", + hard_conditions: JSON.stringify(s.hard ?? {}), + upgrade_threshold: JSON.stringify(s.upgrade ?? {}), + }) + .returning("id") + .executeTakeFirstOrThrow() + ).id; +} diff --git a/apps/server/src/modules/acquisition-organize/__tests__/organize-move.test.ts b/apps/server/src/modules/acquisition-organize/__tests__/organize-move.test.ts new file mode 100644 index 00000000..d2d2e0af --- /dev/null +++ b/apps/server/src/modules/acquisition-organize/__tests__/organize-move.test.ts @@ -0,0 +1,246 @@ +import { randomUUID } from "node:crypto"; +import { existsSync } from "node:fs"; +import { copyFile, mkdir } from "node:fs/promises"; +import { join } from "node:path"; +import { sql } from "@umt/db"; +import { + createMockQb, + createTempTree, + findCrossDeviceBase, + type MockQb, + makeTorrent, + type TempTree, +} from "@umt/testkit"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { evaluateAcquisition } from "../services/acquisition.ts"; +import { submitDownload, syncDownloads } from "../services/downloads.ts"; +import { executeRecord, type FrozenPlan, planImport } from "../services/organize.ts"; +import { + createHarness, + EXEC_KEY, + fakeDisk, + type Harness, + pgAvailable, + READ_KEY, + seedBase, + seedEntity, + seedRelease, + seedSlot, +} from "./harness.ts"; + +// 跨卷 Move 分步可恢复(kill 后 resumed 先对账目标)、路径绑定不因译名变化改变、手动整理 API 与 scope。 + +const hasPg = await pgAvailable(); +const xdev = findCrossDeviceBase(); +const SIZE = 250_000; + +describe.skipIf(!hasPg)("整理:跨卷 Move 恢复 / 路径绑定固定 / 手动整理", () => { + let h: Harness; + let qb: MockQb; + let lib: TempTree; + let cache: TempTree; + let src: TempTree; + const disk = fakeDisk(100_000_000); + const torrents = new Map(); + let base: Awaited>; + + beforeAll(async () => { + qb = await createMockQb({ addVisibleAfterPolls: 0 }); + // 库根放到另一个设备(/dev/shm)上,缓存在 tmpdir:入库 Move 必然跨卷 + lib = await createTempTree({}, { sentinel: true, prefix: "umt-xlib-", ...(xdev ? { base: xdev } : {}) }); + cache = await createTempTree({}, { prefix: "umt-xcache-" }); + src = await createTempTree( + { + "Show.S01E01.1080p.WEB-DL.x264-GRP.mkv": SIZE, + "Show.S01E02.1080p.WEB-DL.x264-GRP.mkv": SIZE, + "Show.S02E01.1080p.WEB-DL.x264-GRP.mkv": SIZE, + "Show.S01E01.1080p.WEB-DL.x264-GRP.chs.srt": 2_000, + }, + { prefix: "umt-src-" }, + ); + disk.state.watch = [cache.root]; + disk.setPct(30); + h = await createHarness("move", { + diskUsage: disk.fn, + torrentSource: async ({ release }) => ({ kind: "file", bytes: torrents.get(release.id)! }), + }); + base = await seedBase(h.db, { libraryPath: lib.root, cachePath: cache.root, qbUrl: qb.url }); + }, 60_000); + + afterAll(async () => { + await h?.close(); + await qb?.close(); + for (const t of [lib, cache, src]) await t?.cleanup(); + }); + + it.skipIf(!xdev)( + "跨卷 Move:copy 已完成后进程被 kill → 续跑时 resumed 先对账目标,不重复复制,核验后删源并移除 qB 任务记录", + async () => { + await h.setSettings("organize", { defaultMode: "move", allowCrossVolumeMove: true }); + const e = await seedEntity(h.db, { kind: "movie", title: "跨卷片", year: 2021 }); + await seedSlot(h.db, { entityId: e }); + const t = makeTorrent({ name: "Cross.Volume.2021.1080p.BluRay-XV.mkv", length: SIZE, pieceSeed: 7 }); + const relId = await seedRelease(h.db, { + siteId: base.siteId, + entityId: e, + title: "Cross.Volume.2021.1080p.BluRay-XV", + size: SIZE, + hash: t.hash, + hr: false, + }); + torrents.set(relId, t.bytes); + const ev = await evaluateAcquisition(h.d, { sourceReleaseId: relId }); + const sub = await submitDownload(h.d, ev.plans[0]!); + await qb.complete(t.hash, { writeFiles: true }); + await syncDownloads(h.d); + + const plan = await planImport(h.d, { downloadTaskId: sub.downloadTaskId, trigger: "auto", actorType: "system" }); + expect(plan.status).toBe("running"); + const rec = await h.db + .selectFrom("organization_record") + .selectAll() + .where("id", "=", plan.recordId!) + .executeTakeFirstOrThrow(); + const frozen = rec.plan as unknown as FrozenPlan; + const f = frozen.organize.files[0]!; + expect(f.crossDevice).toBe(true); + + // 模拟上一次 attempt:copy 步骤已把目标写完并校验,删源前 worker 被 kill(账本停在 started) + await mkdir(join(lib.root, "跨卷片 (2021)"), { recursive: true }); + await copyFile(f.src, f.dst); + await sql`insert into tasks_op_ledger (op_key, attempt_token, attempts, status) values (${`org:${plan.recordId}:0`}, ${randomUUID()}, 1, 'started')`.execute( + h.db, + ); + + const ex = await executeRecord(h.d, plan.recordId!); + expect(ex.status).toBe("succeeded"); + expect(existsSync(f.src)).toBe(false); + expect(existsSync(f.dst)).toBe(true); + const op = await h.db + .selectFrom("asset_operation") + .selectAll() + .where("operation_id", "=", `org:${plan.recordId}:0`) + .executeTakeFirstOrThrow(); + const result = op.result as { resumed?: boolean; steps?: { step: string; status: string }[] }; + expect(result.resumed).toBe(true); + expect(result.steps?.find((s) => s.step === "copy")?.status).toBe("skipped"); + expect(result.steps?.find((s) => s.step === "delete_source")?.status).toBe("done"); + // Move 成功:移除 qB 任务记录(deleteFiles=false),结束下载侧管理 + expect(qb.get(t.hash)).toBeUndefined(); + const del = qb.callsTo("torrents/delete").at(-1)!; + expect(del.form.deleteFiles).toBe("false"); + const dt = await h.db + .selectFrom("download_task") + .selectAll() + .where("id", "=", sub.downloadTaskId!) + .executeTakeFirstOrThrow(); + expect(dt.removed_at).not.toBeNull(); + expect(dt.status).toBe("imported"); + await h.setSettings("organize", { defaultMode: "copy" }); + }, + 60_000, + ); + + it("路径绑定固定:译名变化后新集仍进旧目录;新季在旧作品目录下建季绑定;手动整理与 scope", async () => { + const e = await seedEntity(h.db, { kind: "series", title: "旧译名", year: 2020 }); + // 只读 key 不能发起整理 + const denied = await h.api( + "POST", + "/organization-records", + { sourcePaths: [src.path("Show.S01E01.1080p.WEB-DL.x264-GRP.mkv")], mediaEntityId: e, action: "copy" }, + READ_KEY, + ); + expect(denied.status).toBe(403); + + // dryRun:只出计划,不写记录 + const dry = await h.api( + "POST", + "/organization-records", + { + sourcePaths: [src.path("Show.S01E01.1080p.WEB-DL.x264-GRP.mkv")], + mediaEntityId: e, + action: "copy", + dryRun: true, + }, + EXEC_KEY, + ); + expect(dry.status).toBe(201); + expect(dry.body.data).toMatchObject({ recordId: null }); + expect(dry.body.data.planId).toBeTruthy(); + expect( + await h.db.selectFrom("organization_record").select("id").where("media_entity_id", "=", e).execute(), + ).toHaveLength(0); + + const r1 = await h.api( + "POST", + "/organization-records", + { + sourcePaths: [ + src.path("Show.S01E01.1080p.WEB-DL.x264-GRP.mkv"), + src.path("Show.S01E01.1080p.WEB-DL.x264-GRP.chs.srt"), + ], + mediaEntityId: e, + action: "copy", + }, + EXEC_KEY, + ); + expect(r1.status).toBe(201); + expect(h.engine.enqueued.at(-1)).toMatchObject({ + task: "verifyAndImport", + input: { recordId: r1.body.data.recordId }, + }); + await executeRecord(h.d, r1.body.data.recordId); + const ep1 = join(lib.root, "旧译名 (2020)", "Season 1", "旧译名 - S01E01.mkv"); + expect(existsSync(ep1)).toBe(true); + expect(existsSync(join(lib.root, "旧译名 (2020)", "Season 1", "旧译名 - S01E01.chi.zh-cn.srt"))).toBe(true); + const rec1 = await h.api("GET", `/organization-records/${r1.body.data.recordId}`, undefined, READ_KEY); + expect(rec1.body.data).toMatchObject({ origin: "manual", status: "succeeded", counts: { total: 2, succeeded: 2 } }); + expect(rec1.body.data.actor.type).toBe("api_key"); + + // TMDB 改名:只改展示,不改目录 + await h.db.updateTable("media_entity").set({ title: "新译名" }).where("id", "=", e).execute(); + const r2 = await h.api( + "POST", + "/organization-records", + { sourcePaths: [src.path("Show.S01E02.1080p.WEB-DL.x264-GRP.mkv")], mediaEntityId: e, action: "copy" }, + EXEC_KEY, + ); + await executeRecord(h.d, r2.body.data.recordId); + expect(existsSync(join(lib.root, "旧译名 (2020)", "Season 1", "新译名 - S01E02.mkv"))).toBe(true); + expect(existsSync(join(lib.root, "新译名 (2020)"))).toBe(false); + + const r3 = await h.api( + "POST", + "/organization-records", + { sourcePaths: [src.path("Show.S02E01.1080p.WEB-DL.x264-GRP.mkv")], mediaEntityId: e, action: "copy" }, + EXEC_KEY, + ); + await executeRecord(h.d, r3.body.data.recordId); + expect(existsSync(join(lib.root, "旧译名 (2020)", "Season 2", "新译名 - S02E01.mkv"))).toBe(true); + + const bindings = await h.db + .selectFrom("path_binding") + .select(["binding_kind", "season_number", "relative_path", "status"]) + .where("media_entity_id", "=", e) + .execute(); + bindings.sort((a, b) => (a.season_number ?? -1) - (b.season_number ?? -1)); + expect(bindings).toEqual([ + { binding_kind: "entity", season_number: null, relative_path: "旧译名 (2020)", status: "active" }, + { binding_kind: "season", season_number: 1, relative_path: "旧译名 (2020)/Season 1", status: "active" }, + { binding_kind: "season", season_number: 2, relative_path: "旧译名 (2020)/Season 2", status: "active" }, + ]); + // 同目标已存在且内容相同:重复整理判 already_in_library 跳过,不覆盖 + const r4 = await h.api( + "POST", + "/organization-records", + { sourcePaths: [src.path("Show.S01E02.1080p.WEB-DL.x264-GRP.mkv")], mediaEntityId: e, action: "copy" }, + EXEC_KEY, + ); + const rec4 = await h.db + .selectFrom("organization_file_result") + .selectAll() + .where("organization_record_id", "=", r4.body.data.recordId) + .executeTakeFirstOrThrow(); + expect(rec4).toMatchObject({ status: "skipped", skip_reason: "already_in_library" }); + }, 60_000); +}); diff --git a/apps/server/src/modules/acquisition-organize/__tests__/rules-api.test.ts b/apps/server/src/modules/acquisition-organize/__tests__/rules-api.test.ts new file mode 100644 index 00000000..d482de68 --- /dev/null +++ b/apps/server/src/modules/acquisition-organize/__tests__/rules-api.test.ts @@ -0,0 +1,252 @@ +import { makeTorrent } from "@umt/testkit"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { createAcquisitionOrganizeModule } from "../index.ts"; +import { evaluateAcquisition } from "../services/acquisition.ts"; +import { createHarness, fakeDisk, type Harness, pgAvailable, seedBase, seedEntity, seedRelease } from "./harness.ts"; + +// 规则 API:订阅/Slot/策略版本(发布=持续授权、回滚)/偏好标签抑制;新作品收录走已发布策略;outbox 路由。 + +const hasPg = await pgAvailable(); + +describe("outbox 路由", () => { + const routes = createAcquisitionOrganizeModule().outboxRoutes!({} as never); + it("release.resolved → evaluateAcquisition;只有下载完成才建 verifyAndImport", () => { + const ev = (type: string, payload: unknown) => ({ id: "1", type, payload, at: new Date() }); + expect( + routes["release.resolved"]!(ev("release.resolved", { sourceReleaseId: "a", contentUnitIds: ["u"] })), + ).toEqual({ + task: "evaluateAcquisition", + input: { sourceReleaseId: "a", contentUnitIds: ["u"] }, + }); + expect( + routes["download.status_changed"]!(ev("download.status_changed", { downloadTaskId: "d", to: "completed" })), + ).toEqual({ + task: "verifyAndImport", + input: { downloadTaskId: "d" }, + }); + expect( + routes["download.status_changed"]!(ev("download.status_changed", { downloadTaskId: "d", to: "paused" })), + ).toBeNull(); + }); +}); + +describe.skipIf(!hasPg)("规则 API(真实 PG)", () => { + let h: Harness; + const disk = fakeDisk(1_000_000_000); + + beforeAll(async () => { + disk.setPct(10); + h = await createHarness("rules", { diskUsage: disk.fn }); + }, 60_000); + afterAll(async () => { + await h?.close(); + }); + + it("订阅与 Slot:缺省季范围、同名冲突、revision 乐观锁", async () => { + const e = await seedEntity(h.db, { kind: "series", title: "追更剧", year: 2025 }); + const sub = await h.api("POST", "/subscriptions", { mediaEntityId: e, mediaKind: "series" }); + expect(sub.status).toBe(201); + expect(sub.body.data).toMatchObject({ + mode: "continuous", + seasonScope: "following_seasons", + origin: "user", + status: "active", + }); + const once = await h.api("PATCH", `/subscriptions/${sub.body.data.id}`, { + seasonScope: "current_season", + params: { stopAfter: "2030-01-01T00:00:00Z" }, + }); + expect(once.body.data.seasonScope).toBe("current_season"); + expect((await h.api("POST", "/subscriptions", { mediaKind: "movie" })).status).toBe(422); + + const s = await h.api("POST", "/acquisition-slots", { + mediaEntityId: e, + subscriptionId: sub.body.data.id, + name: "1080p", + hardConditions: { resolutions: ["1080p"] }, + }); + expect(s.status).toBe(201); + expect(s.body.data).toMatchObject({ status: "empty", revision: 1 }); + expect((await h.api("POST", "/acquisition-slots", { mediaEntityId: e, name: "1080p" })).status).toBe(409); + expect( + ( + await h.api("PATCH", `/acquisition-slots/${s.body.data.id}`, { + expectedRevision: 9, + softPreferences: { hdr: ["DV"] }, + }) + ).status, + ).toBe(409); + const ok = await h.api("PATCH", `/acquisition-slots/${s.body.data.id}`, { + expectedRevision: 1, + softPreferences: { hdr: ["DV"] }, + }); + expect(ok.body.data).toMatchObject({ revision: 2, softPreferences: { hdr: ["DV"] } }); + const list = await h.api("GET", `/acquisition-slots?mediaEntityId=${e}`); + expect(list.body.data.items).toHaveLength(1); + expect(h.engine.enqueued.some((q) => q.task === "evaluateAcquisition")).toBe(true); + }); + + it("策略版本:草案 → 发布(持续授权)→ 新版本发布(旧授权吊销)→ 回滚(新版本号,内容同目标)", async () => { + const bad = await h.api("POST", "/policy-revisions", { + policyKey: "acq", + kind: "acquisition", + title: "x", + rules: { newTitle: { enabled: "yes" } }, + }); + expect(bad.status).toBe(422); + const rules = { + defaultSlots: [{ name: "1080p", hardConditions: { resolutions: ["1080p"] } }], + newTitle: { + enabled: true, + windowDays: 60, + mediaKinds: ["movie"], + slots: [{ name: "新片", hardConditions: { resolutions: ["1080p", "2160p"] } }], + }, + }; + const r1 = await h.api("POST", "/policy-revisions", { + policyKey: "acq", + kind: "acquisition", + title: "默认收录", + rules, + }); + expect(r1.body.data).toMatchObject({ revision: 1, status: "draft" }); + expect((await h.api("POST", `/policy-revisions/${r1.body.data.id}/preview`)).body.data.preview).toMatchObject({ + valid: true, + templates: 2, + }); + expect((await h.api("POST", `/policy-revisions/${r1.body.data.id}/publish`, { expectedRevision: 2 })).status).toBe( + 409, + ); + const p1 = await h.api("POST", `/policy-revisions/${r1.body.data.id}/publish`, { expectedRevision: 1 }); + expect(p1.body.data.status).toBe("published"); + expect((await h.api("PATCH", `/policy-revisions/${r1.body.data.id}`, { title: "改" })).status).toBe(409); + const g1 = await h.db + .selectFrom("automation_grant") + .selectAll() + .where("policy_revision_id", "=", r1.body.data.id) + .executeTakeFirstOrThrow(); + expect(g1).toMatchObject({ status: "active", action: "policy.acquisition" }); + + const r2 = await h.api("POST", "/policy-revisions", { + policyKey: "acq", + kind: "acquisition", + title: "只要 4K", + rules: { defaultSlots: [{ name: "4k", hardConditions: { resolutions: ["2160p"] } }] }, + }); + expect(r2.body.data.revision).toBe(2); + await h.api("POST", `/policy-revisions/${r2.body.data.id}/publish`, { expectedRevision: 2 }); + expect((await h.api("GET", `/policy-revisions/${r1.body.data.id}`)).body.data.status).toBe("superseded"); + expect( + (await h.db.selectFrom("automation_grant").select("status").where("id", "=", g1.id).executeTakeFirstOrThrow()) + .status, + ).toBe("revoked"); + + const rb = await h.api("POST", `/policy-revisions/${r1.body.data.id}/rollback`); + expect(rb.body.data).toMatchObject({ revision: 3, status: "published", basedOnRevisionId: r1.body.data.id, rules }); + expect((await h.api("GET", `/policy-revisions/${r2.body.data.id}`)).body.data.status).toBe("rolled_back"); + const active = await h.db + .selectFrom("automation_grant") + .select("policy_revision_id") + .where("status", "=", "active") + .execute(); + expect(active.map((a) => a.policy_revision_id)).toEqual([rb.body.data.id]); + + // 订阅没有 Slot 时套用已发布策略的缺省 Slot + const e = await seedEntity(h.db, { kind: "movie", title: "套模板", year: 2025 }); + await h.api("POST", "/subscriptions", { mediaEntityId: e, mediaKind: "movie" }); + const slots = await h.db.selectFrom("acquisition_slot").selectAll().where("media_entity_id", "=", e).execute(); + expect(slots.map((s) => s.name)).toEqual(["1080p"]); + }); + + it("新作品收录:库里没有、首映在窗口内 → 按已发布策略建订阅与 Slot 并预留计划;旧片不算新片", async () => { + const base = await seedBase(h.db, { + libraryPath: "/tmp/nonexistent-lib", + cachePath: "/tmp", + qbUrl: "http://127.0.0.1:9", + }); + const recent = new Date(Date.now() - 5 * 86_400_000).toISOString().slice(0, 10); + const fresh = ( + await h.db + .insertInto("media_entity") + .values({ + kind: "movie", + title: "新片", + year: 2026, + status: "active", + metadata: JSON.stringify({ releaseDate: recent }), + }) + .returning("id") + .executeTakeFirstOrThrow() + ).id; + const t = makeTorrent({ name: "Fresh.2026.2160p.WEB-DL.mkv", length: 1000, pieceSeed: 3 }); + const rel = await seedRelease(h.db, { + siteId: base.siteId, + entityId: fresh, + title: "Fresh.2026.2160p.WEB-DL.H265-NEW", + size: 1000, + hash: t.hash, + }); + const ev = await evaluateAcquisition(h.d, { sourceReleaseId: rel }); + expect(ev.plans).toHaveLength(1); + const sub = await h.db + .selectFrom("subscription") + .selectAll() + .where("media_entity_id", "=", fresh) + .executeTakeFirstOrThrow(); + expect(sub).toMatchObject({ origin: "policy", mode: "once", priority: "new_work" }); + const plan = await h.db + .selectFrom("acquisition_plan") + .selectAll() + .where("id", "=", ev.plans[0]!) + .executeTakeFirstOrThrow(); + expect(plan).toMatchObject({ status: "reserved", priority: "new_work", purpose: "fill" }); + expect((plan.evaluation as { mode: string }).mode).toBe("new_title"); + + const old = await seedEntity(h.db, { kind: "movie", title: "老片", year: 1999 }); + const t2 = makeTorrent({ name: "Old.1999.1080p.mkv", length: 1000, pieceSeed: 4 }); + const rel2 = await seedRelease(h.db, { + siteId: base.siteId, + entityId: old, + title: "Old.1999.1080p.BluRay-OLD", + size: 1000, + hash: t2.hash, + }); + const ev2 = await evaluateAcquisition(h.d, { sourceReleaseId: rel2 }); + expect(ev2.plans).toHaveLength(0); + expect(ev2.skipped[0]?.reason).toBe("new_title:not_new_release"); + + // 总开关关闭:不建计划 + await h.setSettings("automation", { enabled: false }); + const t3 = makeTorrent({ name: "Fresh2.mkv", length: 1000, pieceSeed: 5 }); + const rel3 = await seedRelease(h.db, { + siteId: base.siteId, + entityId: fresh, + title: "Fresh.2026.1080p.WEB-DL.H264-NEW", + size: 1000, + hash: t3.hash, + }); + const ev3 = await evaluateAcquisition(h.d, { sourceReleaseId: rel3 }); + expect(ev3.plans).toHaveLength(0); + await h.setSettings("automation", { enabled: true }); + }); + + it("偏好标签:删除/停用写抑制记录;Agent 不能据旧证据加回;用户重新添加解除抑制", async () => { + const tag = { label: "不收恐怖片", kind: "hard", scope: { genres: ["horror"] }, rule: { exclude: true } }; + const c = await h.api("POST", "/preference-tags", tag); + expect(c.status).toBe(201); + expect((await h.api("POST", "/preference-tags", tag)).status).toBe(409); + expect((await h.api("DELETE", `/preference-tags/${c.body.data.id}`)).status).toBe(200); + const sup = await h.api("GET", "/preference-suppressions?active=true"); + expect(sup.body.data.items).toHaveLength(1); + expect(sup.body.data.items[0].fingerprint).toBe(c.body.data.fingerprint); + expect((await h.api("POST", "/preference-tags", { ...tag, origin: "agent_inferred" })).status).toBe(409); + const again = await h.api("POST", "/preference-tags", { ...tag, origin: "manual" }); + expect(again.status).toBe(201); + expect((await h.api("GET", "/preference-suppressions?active=true")).body.data.items).toHaveLength(0); + const off = await h.api("PATCH", `/preference-tags/${again.body.data.id}`, { status: "disabled" }); + expect(off.body.data).toMatchObject({ status: "disabled", revision: 2 }); + expect((await h.api("GET", "/preference-suppressions?active=true")).body.data.items).toHaveLength(1); + const list = await h.api("GET", "/preference-tags"); + expect(list.body.data.items.map((t: { id: string }) => t.id)).toEqual([again.body.data.id]); + }); +}); diff --git a/apps/server/src/modules/acquisition-organize/__tests__/vertical.test.ts b/apps/server/src/modules/acquisition-organize/__tests__/vertical.test.ts new file mode 100644 index 00000000..b177bc35 --- /dev/null +++ b/apps/server/src/modules/acquisition-organize/__tests__/vertical.test.ts @@ -0,0 +1,462 @@ +import { existsSync, readFileSync } from "node:fs"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { sql } from "@umt/db"; +import { createMockQb, createTempTree, type MockQb, makeTorrent, type TempTree } from "@umt/testkit"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { evaluateAcquisition } from "../services/acquisition.ts"; +import { cleanupDownloadCache, submitDownload, syncDownloads } from "../services/downloads.ts"; +import { executeRecord, planImport } from "../services/organize.ts"; +import { reconcileLibrary } from "../services/reconcile.ts"; +import { + createHarness, + fakeDisk, + type Harness, + pgAvailable, + seedBase, + seedEntity, + seedRelease, + seedSlot, +} from "./harness.ts"; + +// 纵切:release.resolved → 计划 → 提交 mock qB → 完成(落盘)→ Copy 入库 → 整理记录 API(含 changes 游标); +// 以及硬阈值暂停/恢复、already_exists 接管、缓存清理安全边界、资产对账。 + +const hasPg = await pgAvailable(); +const SIZE = 300_000; + +describe.skipIf(!hasPg)("acquisition-organize 纵切(真实 PG + mock qB + 临时目录)", () => { + let h: Harness; + let qb: MockQb; + let lib: TempTree; + let cache: TempTree; + const disk = fakeDisk(10_000_000); + const torrents = new Map(); + let base: Awaited>; + + beforeAll(async () => { + qb = await createMockQb({ addVisibleAfterPolls: 0 }); + lib = await createTempTree({}, { sentinel: true, prefix: "umt-lib-" }); + cache = await createTempTree({}, { prefix: "umt-cache-" }); + disk.state.watch = [cache.root]; + disk.setPct(40); + h = await createHarness("vertical", { + diskUsage: disk.fn, + torrentSource: async ({ release }) => ({ kind: "file", bytes: torrents.get(release.id)! }), + }); + base = await seedBase(h.db, { libraryPath: lib.root, cachePath: cache.root, qbUrl: qb.url }); + await h.setSettings("retention", { seedRetentionDays: 0 }); + }, 60_000); + + afterAll(async () => { + await h?.close(); + await qb?.close(); + await lib?.cleanup(); + await cache?.cleanup(); + }); + + async function release(entityId: string, title: string, opts: { hr?: boolean } = {}) { + const t = makeTorrent({ name: `${title}.mkv`, length: SIZE, pieceSeed: Math.floor(Math.random() * 1e6) }); + const id = await seedRelease(h.db, { + siteId: base.siteId, + entityId, + title, + size: SIZE, + hash: t.hash, + ...(opts.hr !== undefined ? { hr: opts.hr } : {}), + }); + torrents.set(id, t.bytes); + return { id, hash: t.hash, name: t.name }; + } + + async function runToCompleted(entityId: string, title: string, opts: { hr?: boolean } = {}) { + const rel = await release(entityId, title, opts); + const ev = await evaluateAcquisition(h.d, { sourceReleaseId: rel.id }); + expect(ev.plans).toHaveLength(1); + const sub = await submitDownload(h.d, ev.plans[0]!); + expect(sub.status).toBe("submitted"); + await qb.complete(rel.hash, { writeFiles: true }); + const sync = await syncDownloads(h.d); + expect(sync.completed).toContain(sub.downloadTaskId); + return { rel, planId: ev.plans[0]!, dtId: sub.downloadTaskId! }; + } + + let movieId: string; + let slotId: string; + let firstRecord: string; + + it("release → 计划(预留 Slot)→ 提交 qB(管理标签)→ 完成 → Copy 入库 → 登记资产与绑定", async () => { + movieId = await seedEntity(h.db, { kind: "movie", title: "沙丘2", year: 2024 }); + slotId = await seedSlot(h.db, { entityId: movieId, hard: { resolutions: ["1080p"] } }); + // 不满足硬条件的候选:写拒绝理由,不建计划 + const bad = await release(movieId, "Dune.Part.Two.2024.720p.WEB-DL.x264-AAA"); + const r0 = await evaluateAcquisition(h.d, { sourceReleaseId: bad.id }); + expect(r0.plans).toHaveLength(0); + + const rel = await release(movieId, "Dune.Part.Two.2024.1080p.BluRay.x264-GRP"); + const ev = await evaluateAcquisition(h.d, { sourceReleaseId: rel.id }); + expect(ev.plans).toHaveLength(1); + const slot1 = await h.db + .selectFrom("acquisition_slot") + .selectAll() + .where("id", "=", slotId) + .executeTakeFirstOrThrow(); + expect(slot1.status).toBe("reserved"); + expect(h.engine.enqueued.map((e) => e.task)).toContain("submitDownload"); + + const sub = await submitDownload(h.d, ev.plans[0]!); + expect(sub).toMatchObject({ status: "submitted", reason: "added" }); + const t = qb.get(rel.hash)!; + expect(t.tags).toContain("umt"); + expect(t.save_path).toBe(cache.root); + // 重放:账本已有结果,不重复提交 + expect((await submitDownload(h.d, ev.plans[0]!)).status).toBe("skipped"); + expect(qb.callsTo("torrents/add")).toHaveLength(1); + expect((await h.outbox("download.submitted")).length).toBe(1); + expect( + (await h.db.selectFrom("acquisition_slot").select("status").where("id", "=", slotId).executeTakeFirstOrThrow()) + .status, + ).toBe("downloading"); + + // 未完成:同步只更新进度 + qb.setProgress(rel.hash, 0.5); + const s1 = await syncDownloads(h.d); + expect(s1.completed).toHaveLength(0); + await qb.complete(rel.hash, { writeFiles: true }); + const s2 = await syncDownloads(h.d); + expect(s2.completed).toEqual([sub.downloadTaskId]); + const done = await h.outbox("download.status_changed"); + expect(done.filter((e) => e.payload.to === "completed")).toHaveLength(1); + expect( + (await h.db.selectFrom("acquisition_slot").select("status").where("id", "=", slotId).executeTakeFirstOrThrow()) + .status, + ).toBe("verifying"); + + const plan = await planImport(h.d, { downloadTaskId: sub.downloadTaskId, trigger: "auto", actorType: "system" }); + expect(plan.status).toBe("running"); + firstRecord = plan.recordId!; + const ex = await executeRecord(h.d, firstRecord); + expect(ex).toMatchObject({ status: "succeeded", succeeded: 1, failed: 0 }); + + const target = join(lib.root, "沙丘2 (2024)", "沙丘2 (2024) - 1080p.mkv"); + expect(existsSync(target)).toBe(true); + // Copy:源继续做种 + expect(existsSync(join(cache.root, `${rel.name}`))).toBe(true); + const slot = await h.db + .selectFrom("acquisition_slot") + .selectAll() + .where("id", "=", slotId) + .executeTakeFirstOrThrow(); + expect(slot.status).toBe("satisfied"); + expect(slot.current_asset_id).not.toBeNull(); + const binding = await h.db.selectFrom("path_binding").selectAll().where("media_entity_id", "=", movieId).execute(); + expect(binding.map((b) => b.relative_path)).toEqual(["沙丘2 (2024)"]); + const loc = await h.db + .selectFrom("asset_location") + .selectAll() + .where("media_asset_id", "=", slot.current_asset_id!) + .executeTakeFirstOrThrow(); + expect(loc.relative_path).toBe("沙丘2 (2024)/沙丘2 (2024) - 1080p.mkv"); + const dt = await h.db + .selectFrom("download_task") + .selectAll() + .where("id", "=", sub.downloadTaskId!) + .executeTakeFirstOrThrow(); + expect(dt.status).toBe("imported"); + expect( + ( + await h.db + .selectFrom("acquisition_plan") + .select("status") + .where("id", "=", ev.plans[0]!) + .executeTakeFirstOrThrow() + ).status, + ).toBe("completed"); + + // 重跑执行:幂等,不重复登记 + await executeRecord(h.d, firstRecord); + expect( + await h.db.selectFrom("media_asset").select("id").where("media_entity_id", "=", movieId).execute(), + ).toHaveLength(1); + }, 60_000); + + it("整理记录 API:详情含文件明细;changes 游标增量(含既有记录状态更新);changes 先于 {id}", async () => { + const det = await h.api("GET", `/organization-records/${firstRecord}`); + expect(det.status).toBe(200); + expect(det.body.data).toMatchObject({ + id: firstRecord, + status: "succeeded", + action: "copy", + origin: "auto", + counts: { total: 1, succeeded: 1 }, + }); + expect(det.body.data.files[0]).toMatchObject({ + role: "main", + status: "succeeded", + target: { path: "沙丘2 (2024)/沙丘2 (2024) - 1080p.mkv", view: "library" }, + }); + expect(det.body.data.media).toMatchObject({ entityId: movieId, title: "沙丘2" }); + + const poll = async (after: string) => { + for (let i = 0; i < 50; i++) { + const r = await h.api("GET", `/organization-records/changes?after=${after}&limit=100`); + if (r.body.data.items.length > 0) return r; + await new Promise((res) => setTimeout(res, 100)); + } + return h.api("GET", `/organization-records/changes?after=${after}&limit=100`); + }; + const first = await poll("0"); + expect(first.status).toBe(200); + expect(first.body.data.items.map((i: { id: string }) => i.id)).toContain(firstRecord); + const cursor = first.body.data.nextCursor as string; + const again = await h.api("GET", `/organization-records/changes?after=${cursor}`); + expect(again.body.data).toMatchObject({ items: [], nextCursor: cursor, hasMore: false }); + + // 既有记录状态更新 → 以更大的 changeSeq 再次出现 + await sql`update organization_record set reason = 'touched' where id = ${firstRecord}`.execute(h.db); + const upd = await poll(cursor); + expect(upd.body.data.items.map((i: { id: string }) => i.id)).toEqual([firstRecord]); + expect(BigInt(upd.body.data.nextCursor)).toBeGreaterThan(BigInt(cursor)); + + const list = await h.api("GET", "/organization-records?limit=10"); + expect(list.body.data.items[0].files).toBeNull(); + expect((await h.api("GET", "/organization-records/changes?after=abc")).status).toBe(422); + }); + + it("硬阈值:达到时暂停纳管下载中任务并阻止新计划;低于恢复水位只恢复阈值暂停项", async () => { + const e = await seedEntity(h.db, { kind: "movie", title: "阈值片", year: 2024 }); + await seedSlot(h.db, { entityId: e, hard: {} }); + // 一个下载中的纳管任务 + 一个用户手动暂停的纳管任务(贴标签接管) + const a = makeTorrent({ name: "busy.mkv", length: SIZE, pieceSeed: 11 }); + const b = makeTorrent({ name: "user-paused.mkv", length: SIZE, pieceSeed: 12 }); + qb.seed({ hash: a.hash, name: "busy.mkv", tags: ["umt"], progress: 0.2, save_path: cache.root, size: SIZE }); + qb.seed({ + hash: b.hash, + name: "user-paused.mkv", + tags: ["umt"], + progress: 0.2, + save_path: cache.root, + size: SIZE, + state: "pausedDL", + }); + const s0 = await syncDownloads(h.d); + expect(s0.adopted).toBe(2); + const rowB = await h.db + .selectFrom("download_task") + .selectAll() + .where("torrent_hash", "=", b.hash) + .executeTakeFirstOrThrow(); + expect(rowB).toMatchObject({ status: "paused", pause_reason: "user", origin: "adopted" }); + + disk.setPct(95); + const s1 = await syncDownloads(h.d); + expect(s1.thresholdPaused).toBe(1); + expect(qb.get(a.hash)!.state).toMatch(/^paused|^stopped/i); + expect( + (await h.db.selectFrom("download_task").selectAll().where("torrent_hash", "=", a.hash).executeTakeFirstOrThrow()) + .pause_reason, + ).toBe("threshold"); + const tripped = await h.outbox("download.threshold"); + expect(tripped.at(-1)?.payload.state).toBe("tripped"); + + const rel = await release(e, "Threshold.Movie.2024.1080p.WEB-DL-X"); + const ev = await evaluateAcquisition(h.d, { sourceReleaseId: rel.id }); + expect(ev.plans).toHaveLength(0); + expect(ev.skipped[0]?.reason).toMatch(/^cache:/); + // 手动恢复阈值暂停项被拒绝(不绕过阈值) + const rowA = await h.db + .selectFrom("download_task") + .select("id") + .where("torrent_hash", "=", a.hash) + .executeTakeFirstOrThrow(); + expect((await h.api("POST", `/download-tasks/${rowA.id}/resume`)).status).toBe(412); + + // 下一轮观察:暂停中不计入无进展,原因保持 threshold + await syncDownloads(h.d); + expect( + (await h.db.selectFrom("download_task").selectAll().where("id", "=", rowA.id).executeTakeFirstOrThrow()) + .pause_reason, + ).toBe("threshold"); + + disk.setPct(88); // 高于恢复水位:保持暂停 + expect((await syncDownloads(h.d)).thresholdResumed).toBe(0); + disk.setPct(70); + const s3 = await syncDownloads(h.d); + expect(s3.thresholdResumed).toBe(1); + expect(qb.get(a.hash)!.state).not.toMatch(/^paused|^stopped/i); + expect(qb.get(b.hash)!.state).toMatch(/^paused|^stopped/i); + expect( + (await h.db.selectFrom("download_task").selectAll().where("torrent_hash", "=", b.hash).executeTakeFirstOrThrow()) + .pause_reason, + ).toBe("user"); + // 读不到磁盘:保守阻止 + disk.state.fail = true; + const ev2 = await evaluateAcquisition(h.d, { sourceReleaseId: rel.id }); + expect(ev2.plans).toHaveLength(0); + disk.state.fail = false; + disk.setPct(40); + qb.setProgress(a.hash, 0); + }, 60_000); + + it("already_exists:种子原本在 qB 里 → 接管,只补管理标签,不改保存路径和其它标签", async () => { + const e = await seedEntity(h.db, { kind: "movie", title: "旧种", year: 2023 }); + await seedSlot(h.db, { entityId: e }); + const rel = await release(e, "Old.Seed.2023.1080p.BluRay-OLD"); + qb.seed({ hash: rel.hash, name: rel.name, tags: ["keep"], save_path: "/elsewhere", progress: 0.1, size: SIZE }); + const ev = await evaluateAcquisition(h.d, { sourceReleaseId: rel.id }); + const sub = await submitDownload(h.d, ev.plans[0]!); + expect(sub).toMatchObject({ status: "submitted", reason: "already_exists" }); + const t = qb.get(rel.hash)!; + expect(t.save_path).toBe("/elsewhere"); + expect(t.tags.sort()).toEqual(["keep", "umt"]); + const row = await h.db + .selectFrom("download_task") + .selectAll() + .where("id", "=", sub.downloadTaskId!) + .executeTakeFirstOrThrow(); + expect(row).toMatchObject({ origin: "adopted", managed: true, acquisition_plan_id: ev.plans[0] }); + expect(qb.callsTo("torrents/add").length).toBe(1); + }, 30_000); + + it("H&R 未满足时 Move 变冲突:记录失败 + 待处理,不搬源、不静默改 Copy", async () => { + await h.setSettings("organize", { defaultMode: "move" }); + const e = await seedEntity(h.db, { kind: "movie", title: "做种片", year: 2022 }); + const slot = await seedSlot(h.db, { entityId: e }); + const { dtId, rel } = await runToCompleted(e, "Seeding.Movie.2022.1080p.BluRay-HR", { hr: true }); + const plan = await planImport(h.d, { downloadTaskId: dtId, trigger: "auto", actorType: "system" }); + expect(plan.status).toBe("failed"); + expect(plan.reason).toContain("hnr_unsatisfied"); + const rec = await h.db + .selectFrom("organization_record") + .selectAll() + .where("id", "=", plan.recordId!) + .executeTakeFirstOrThrow(); + expect(rec.action).toBe("move"); + expect(existsSync(join(cache.root, rel.name))).toBe(true); + expect(existsSync(join(lib.root, "做种片 (2022)"))).toBe(false); + const dec = await h.db + .selectFrom("decision") + .selectAll() + .where("subject_id", "=", plan.recordId!) + .executeTakeFirstOrThrow(); + expect(dec).toMatchObject({ kind: "approval_required", status: "open" }); + expect( + (await h.db.selectFrom("acquisition_slot").select("status").where("id", "=", slot).executeTakeFirstOrThrow()) + .status, + ).toBe("attention"); + await h.setSettings("organize", { defaultMode: "copy" }); + }, 30_000); + + it("缓存清理:低于阈值不动;超阈值只删已验证入库且满足保留/H&R 的任务,删数据必经安全检查", async () => { + // 已有:第一部片(已入库、hr=false、保留 0 天)→ 可清理;做种片(hr 未满足,未入库)→ 锁定 + disk.setPct(50); + const r0 = await cleanupDownloadCache(h.d); + expect(r0.volumes[0]!.action).toBe("none"); + const firstDt = await h.db.selectFrom("download_task").selectAll().where("status", "=", "imported").execute(); + expect(firstDt.length).toBeGreaterThanOrEqual(1); + // 另造一个"已入库"但 content_path 落在媒体库内的任务:安全检查必须拒绝删数据 + const trap = await h.db + .insertInto("download_task") + .values({ + instance_id: base.instanceId, + torrent_hash: "f".repeat(40), + name: "trap", + source_release_id: firstDt[0]!.source_release_id, + save_path: lib.root, + content_path: join(lib.root, "沙丘2 (2024)"), + status: "imported", + managed: true, + completed_at: new Date(0), + size_bytes: 1, + }) + .returning("id") + .executeTakeFirstOrThrow(); + await h.db + .insertInto("organization_record") + .values({ trigger: "auto", actor_type: "system", action: "copy", status: "succeeded", download_task_id: trap.id }) + .execute(); + qb.seed({ hash: "f".repeat(40), name: "trap", tags: ["umt"], progress: 1, save_path: lib.root, size: 1 }); + + disk.setPct(95); + const r = await cleanupDownloadCache(h.d); + const v = r.volumes[0]!; + expect(v.removed).toContain(firstDt[0]!.id); + expect(v.removed).not.toContain(trap.id); + expect(v.skipped.find((s) => s.downloadTaskId === trap.id)?.reason).toMatch(/拒绝删除|媒体库/); + expect(qb.get(firstDt[0]!.torrent_hash)).toBeUndefined(); + // 库内副本不受影响 + expect(existsSync(join(lib.root, "沙丘2 (2024)", "沙丘2 (2024) - 1080p.mkv"))).toBe(true); + const locked = v.skipped.map((s) => s.reason); + expect(locked.some((x) => x === "hnr_unsatisfied" || x === "import_not_verified")).toBe(false); // 做种片未入库,不在候选里 + const op = await h.db + .selectFrom("asset_operation") + .selectAll() + .where("operation_id", "=", `cleanup:${firstDt[0]!.id}`) + .executeTakeFirstOrThrow(); + expect(op).toMatchObject({ kind: "qb_remove", status: "succeeded" }); + disk.setPct(40); + }, 30_000); + + it("资产对账:库根离线只标 unavailable;缺失按 suspected → confirmed 推进,不删记录不补下载", async () => { + const file = join(lib.root, "沙丘2 (2024)", "沙丘2 (2024) - 1080p.mkv"); + const content = readFileSync(file); + const loc = await h.db + .selectFrom("asset_location") + .selectAll() + .where("relative_path", "=", "沙丘2 (2024)/沙丘2 (2024) - 1080p.mkv") + .executeTakeFirstOrThrow(); + const { rm } = await import("node:fs/promises"); + await rm(join(lib.root, ".uhd-root")); + const off = await reconcileLibrary(h.d); + expect(off.stats.rootsOffline).toHaveLength(1); + expect( + (await h.db.selectFrom("asset_location").select("status").where("id", "=", loc.id).executeTakeFirstOrThrow()) + .status, + ).toBe("unavailable"); + await writeFile(join(lib.root, ".uhd-root"), "x"); + await rm(file); + for (const want of ["suspected_missing", "suspected_missing", "confirmed_missing"]) { + await reconcileLibrary(h.d); + expect( + (await h.db.selectFrom("asset_location").select("status").where("id", "=", loc.id).executeTakeFirstOrThrow()) + .status, + ).toBe(want); + } + expect( + (await h.db.selectFrom("acquisition_slot").select("status").where("id", "=", slotId).executeTakeFirstOrThrow()) + .status, + ).toBe("attention"); + expect( + await h.db.selectFrom("media_asset").select("id").where("id", "=", loc.media_asset_id).executeTakeFirst(), + ).toBeDefined(); + await writeFile(file, content); + await reconcileLibrary(h.d); + expect( + (await h.db.selectFrom("asset_location").select("status").where("id", "=", loc.id).executeTakeFirstOrThrow()) + .status, + ).toBe("available"); + const runs = await h.db.selectFrom("reconcile_run").select(["kind", "status"]).execute(); + expect(runs.every((r) => r.kind === "asset")).toBe(true); + }, 30_000); + + it("首页、媒体库、库存叠加 API", async () => { + const dash = await h.api("GET", "/dashboard"); + expect(dash.status).toBe(200); + expect(dash.body.data.recentOutput.length).toBeGreaterThan(0); + expect(dash.body.data.cache.volumes[0]).toMatchObject({ hardThresholdPercent: 90, resumePercent: 85 }); + expect(dash.body.data.needsAttention.total).toBeGreaterThan(0); + const libList = await h.api("GET", "/library?q=沙丘"); + expect(libList.body.data.items[0]).toMatchObject({ entity: { id: movieId }, assetCount: 1 }); + const det = await h.api("GET", `/library/${movieId}`); + expect(det.status).toBe(200); + expect(det.body.data.assets[0].locations[0].absolutePath).toBe( + join(lib.root, "沙丘2 (2024)", "沙丘2 (2024) - 1080p.mkv"), + ); + expect(det.body.data.candidates.some((c: { decision: string }) => c.decision === "reject")).toBe(true); + expect(det.body.data.downloads[0].organizationRecordIds.length).toBeGreaterThan(0); + const inv = await h.api("GET", `/inventory?entityIds=${movieId}`); + expect(inv.body.data.items[0].versions[0]).toMatchObject({ resolution: "1080p", available: true }); + const ms = await h.api("GET", `/inventory/media-servers?mediaEntityId=${movieId}`); + expect(ms.body.data).toEqual({ mediaEntityId: movieId, instances: [] }); + }); +}); diff --git a/apps/server/src/modules/acquisition-organize/handlers.ts b/apps/server/src/modules/acquisition-organize/handlers.ts new file mode 100644 index 00000000..0d82d13a --- /dev/null +++ b/apps/server/src/modules/acquisition-organize/handlers.ts @@ -0,0 +1,197 @@ +import { defineTask, type TaskRegistry } from "@umt/tasks"; +import { z } from "zod"; +import { evaluateAcquisition } from "./services/acquisition.ts"; +import { type AoDeps, automationBlock, ensureTaskRow, markTask } from "./services/common.ts"; +import { cleanupDownloadCache, submitDownload } from "./services/downloads.ts"; +import { executeRecord, planImport } from "./services/organize.ts"; +import { reconcileLibrary } from "./services/reconcile.ts"; +import { errMessage } from "./util.ts"; + +// 持久任务处理器。输入只放业务 id;业务状态写业务表;副作用经 withAttemptToken(在 services 里)。 +// 步骤体幂等:worker 被 kill 后正在执行的步骤会在新 worker 上从头重跑。 + +const id = z.string().min(1); +const NETWORK_RETRY = { + maximumAttempts: 5, + initialInterval: "1s", + backoffCoefficient: 2, + maximumInterval: "60s", +} as const; +const FILE_RETRY = { + maximumAttempts: 3, + initialInterval: "2s", + backoffCoefficient: 2, + maximumInterval: "60s", +} as const; +const DB_RETRY = { maximumAttempts: 3, initialInterval: "1s", backoffCoefficient: 2, maximumInterval: "30s" } as const; + +/** 业务任务行:手动发起的带 taskId,事件触发的按语义幂等键补建 */ +async function taskRow( + d: AoDeps, + kind: "submitDownload" | "verifyAndImport" | "cleanupDownloadCache" | "reconcileLibrary", + given: string | undefined, + key: string, + input: Record, + runId: string, + subject?: { type: string; id: string }, +) { + const taskId = + given ?? + (await ensureTaskRow(d.ctx.db, { + kind, + idempotencyKey: key, + input, + runId, + ...(subject ? { subjectType: subject.type, subjectId: subject.id } : {}), + })); + if (given) + await d.ctx.db + .updateTable("task") + .set({ engine_run_id: runId }) + .where("id", "=", given) + .where("engine_run_id", "is", null) + .execute(); + await markTask(d.ctx.db, taskId, "running"); + return taskId; +} + +async function tracked( + d: AoDeps, + taskId: string, + fn: () => Promise, + done: (r: T) => "succeeded" | "failed" | "waiting" = () => "succeeded", +): Promise { + try { + const r = await fn(); + await markTask(d.ctx.db, taskId, done(r), { output: r }); + return r; + } catch (err) { + await markTask(d.ctx.db, taskId, "failed", { error: errMessage(err) }).catch(() => {}); + throw err; + } +} + +export function registerAoTasks(registry: TaskRegistry, d: AoDeps) { + registry.register( + defineTask({ + name: "evaluateAcquisition", + version: 1, + inputSchema: z.object({ + subscriptionId: id.optional(), + acquisitionSlotId: id.optional(), + sourceReleaseId: id.optional(), + contentUnitIds: z.array(id).optional(), + }), + retryPolicy: DB_RETRY, + handler: async (t) => + t.step.run("evaluate", async () => JSON.parse(JSON.stringify(await evaluateAcquisition(d, t.input)))), + }), + defineTask({ + name: "submitDownload", + version: 1, + inputSchema: z.object({ acquisitionPlanId: id, taskId: id.optional() }), + retryPolicy: NETWORK_RETRY, + handler: async (t) => { + const taskId = await taskRow( + d, + "submitDownload", + t.input.taskId, + `plan:${t.input.acquisitionPlanId}`, + t.input, + t.runId, + { type: "acquisition_plan", id: t.input.acquisitionPlanId }, + ); + return tracked( + d, + taskId, + () => + t.step.run("submit", async () => ({ + ...(await submitDownload(d, t.input.acquisitionPlanId, { runId: t.runId })), + })), + (r) => (r.status === "failed" ? "failed" : r.status === "deferred" ? "waiting" : "succeeded"), + ); + }, + }), + defineTask({ + name: "verifyAndImport", + version: 1, + inputSchema: z.object({ downloadTaskId: id.optional(), recordId: id.optional(), taskId: id.optional() }), + retryPolicy: FILE_RETRY, + handler: async (t) => { + const key = t.input.recordId ? `record:${t.input.recordId}` : `import:${t.input.downloadTaskId ?? t.runId}`; + const taskId = await taskRow( + d, + "verifyAndImport", + t.input.taskId, + key, + t.input, + t.runId, + t.input.downloadTaskId ? { type: "download_task", id: t.input.downloadTaskId } : undefined, + ); + return tracked( + d, + taskId, + async () => { + const planned = await t.step.run("plan", async () => { + if (t.input.recordId) + return { + recordId: t.input.recordId as string | null, + status: "running", + reason: null as string | null, + }; + if (!t.input.downloadTaskId) return { recordId: null, status: "skipped", reason: "no_input" }; + const dt = await d.ctx.db + .selectFrom("download_task") + .select(["status", "managed"]) + .where("id", "=", t.input.downloadTaskId) + .executeTakeFirst(); + if (!dt || !["completed", "importing"].includes(dt.status)) + return { recordId: null, status: "skipped", reason: `download_${dt?.status ?? "missing"}` }; + if (!dt.managed) return { recordId: null, status: "skipped", reason: "unmanaged" }; + const blocked = await automationBlock(d); + if (blocked) return { recordId: null, status: "deferred", reason: blocked }; + const r = await planImport(d, { + downloadTaskId: t.input.downloadTaskId, + trigger: "auto", + actorType: "system", + actorId: null, + taskId, + }); + return { recordId: r.recordId, status: r.status as string, reason: r.reason }; + }); + if (!planned.recordId || (planned.status !== "running" && planned.status !== "queued")) return planned; + const recordId = planned.recordId; + return t.step.run("execute", async () => ({ + ...(await executeRecord(d, recordId, { runId: t.runId, abortSignal: t.abortSignal })), + })); + }, + (r) => (r.status === "failed" ? "failed" : r.status === "deferred" ? "waiting" : "succeeded"), + ); + }, + }), + defineTask({ + name: "cleanupDownloadCache", + version: 1, + inputSchema: z.object({ downloaderInstanceId: id.optional(), taskId: id.optional() }), + retryPolicy: FILE_RETRY, + handler: async (t) => { + const taskId = await taskRow(d, "cleanupDownloadCache", t.input.taskId, `cleanup:${t.runId}`, t.input, t.runId); + return tracked(d, taskId, () => + t.step.run("cleanup", async () => JSON.parse(JSON.stringify(await cleanupDownloadCache(d, t.input)))), + ); + }, + }), + defineTask({ + name: "reconcileLibrary", + version: 1, + inputSchema: z.object({ libraryRootId: id.optional(), reconcileRunId: id.optional(), taskId: id.optional() }), + retryPolicy: FILE_RETRY, + handler: async (t) => { + const taskId = await taskRow(d, "reconcileLibrary", t.input.taskId, `reconcile:${t.runId}`, t.input, t.runId); + return tracked(d, taskId, () => + t.step.run("reconcile", async () => JSON.parse(JSON.stringify(await reconcileLibrary(d, t.input)))), + ); + }, + }), + ); +} diff --git a/apps/server/src/modules/acquisition-organize/index.ts b/apps/server/src/modules/acquisition-organize/index.ts index 289d8cf4..47549661 100644 --- a/apps/server/src/modules/acquisition-organize/index.ts +++ b/apps/server/src/modules/acquisition-organize/index.ts @@ -1,4 +1,18 @@ -import type { ServerModule } from "../types.ts"; +import { type EventPayload, EventType } from "@umt/contracts"; +import type { ServerContext, ServerModule } from "../types.ts"; +import { registerAoTasks } from "./handlers.ts"; +import { type AoOptions, type AoPorts, createPorts } from "./ports.ts"; +import { registerDashboardRoutes } from "./routes/dashboard.ts"; +import { registerDownloadRoutes } from "./routes/downloads.ts"; +import { registerLibraryRoutes } from "./routes/library.ts"; +import { registerOrganizationRoutes } from "./routes/organization.ts"; +import { registerRuleRoutes } from "./routes/rules.ts"; +import { type AoDeps, automationBlock } from "./services/common.ts"; +import { cleanupDownloadCache, sweepReservedPlans, syncDownloads } from "./services/downloads.ts"; +import { executeRecord, planImport } from "./services/organize.ts"; +import { reconcileLibrary } from "./services/reconcile.ts"; +import { loadVolumes, usedPercent } from "./services/volumes.ts"; +import { errMessage } from "./util.ts"; /** * W2-C 规则/下载/入库/整理记录:subscription、acquisition_slot、policy_revision、preference_tag; @@ -6,8 +20,126 @@ import type { ServerModule } from "../types.ts"; * qB 同步与缓存阈值、Copy/Move、path_binding、organization_record 与公开增量 API、媒体库与首页聚合。 * 消费 release.resolved(outboxRoutes);删种带数据一律经 adapters 的 assertSafeToDeleteData。 */ -export const acquisitionOrganizeModule: ServerModule = { - name: "acquisition-organize", - registerRoutes() {}, - registerTasks() {}, -}; + +export type AoOverrides = Partial> & { options?: Partial }; + +/** 模块工厂:测试注入 mock qB / 固定磁盘用量 / 假种子来源;正式装配用 acquisitionOrganizeModule */ +export function createAcquisitionOrganizeModule(overrides: AoOverrides = {}): ServerModule & { deps(): AoDeps } { + let deps: AoDeps | null = null; + let loop: ReturnType | null = null; + let stopped = false; + let lastReconcile = 0; + // register* 阶段只保存 ctx;端口在首次使用时组装(ctx 其它字段此时才完整) + const lazy = (ctx: ServerContext): AoDeps => { + deps ??= { ctx, ports: createPorts(ctx, overrides) }; + return deps; + }; + + /** worker 后台一轮:同步 qB → 阈值 → 必要时清理 → 补提交 → 补入库 → 低频资产对账 */ + const tick = async (d: AoDeps) => { + await syncDownloads(d); + const blocked = await automationBlock(d); + if (!blocked) { + const cache = await d.ctx.settings.get("cache"); + const over = (await loadVolumes(d)).some((v) => (usedPercent(v) ?? 0) >= cache.hardThresholdPercent); + if (over) await cleanupDownloadCache(d); + await sweepReservedPlans(d); + await sweepPendingImports(d); + } + const every = d.ports.options.reconcileIntervalMs; + if (every > 0 && Date.now() - lastReconcile >= every) { + lastReconcile = Date.now(); + await reconcileLibrary(d); + } + }; + + const module: ServerModule & { deps(): AoDeps } = { + name: "acquisition-organize", + deps() { + if (!deps) throw new Error("acquisition-organize 尚未装配"); + return deps; + }, + registerRoutes(app, ctx) { + const d = lazy(ctx); + registerRuleRoutes(app, d); + registerDownloadRoutes(app, d); + registerOrganizationRoutes(app, d); + registerLibraryRoutes(app, d); + registerDashboardRoutes(app, d); + }, + registerTasks(registry, ctx) { + registerAoTasks(registry, lazy(ctx)); + }, + outboxRoutes: () => ({ + [EventType.releaseResolved]: (ev) => { + const p = ev.payload as EventPayload<"release.resolved">; + return { + task: "evaluateAcquisition", + input: { sourceReleaseId: p.sourceReleaseId, contentUnitIds: p.contentUnitIds }, + }; + }, + // 下载完成 → 核验入库(其余状态变化只给 SSE/时间线,不建任务) + [EventType.downloadStatusChanged]: (ev) => { + const p = ev.payload as EventPayload<"download.status_changed">; + return p.to === "completed" ? { task: "verifyAndImport", input: { downloadTaskId: p.downloadTaskId } } : null; + }, + }), + async start(ctx) { + const d = lazy(ctx); + if (!d.ports.options.backgroundLoops) return; + stopped = false; + const schedule = async () => { + if (stopped) return; + const cache = await ctx.settings.get("cache").catch(() => null); + const ms = d.ports.options.syncIntervalMs ?? (cache?.checkIntervalSec ?? 60) * 1000; + loop = setTimeout(async () => { + try { + await tick(d); + } catch (err) { + ctx.logger.error({ err: { message: errMessage(err) } }, "acquisition-organize 后台轮询失败"); + } + void schedule(); + }, ms); + }; + void schedule(); + }, + stop() { + stopped = true; + if (loop) clearTimeout(loop); + }, + }; + return module; +} + +/** 下载已完成、没有整理记录(入库事件被总开关延后等)的纳管任务:直接补做入库 */ +async function sweepPendingImports(d: AoDeps): Promise { + const rows = await d.ctx.db + .selectFrom("download_task as dt") + .select("dt.id") + .where("dt.status", "=", "completed") + .where("dt.managed", "=", true) + .where("dt.removed_at", "is", null) + .where("dt.updated_at", "<", new Date(d.ports.now().getTime() - 5 * 60_000)) + .where((eb) => + eb.not( + eb.exists( + eb.selectFrom("organization_record as r").select("r.id").whereRef("r.download_task_id", "=", "dt.id"), + ), + ), + ) + .limit(10) + .execute(); + let n = 0; + for (const r of rows) { + try { + const p = await planImport(d, { downloadTaskId: r.id, trigger: "auto", actorType: "system", actorId: null }); + if (p.recordId && (p.status === "running" || p.status === "queued")) await executeRecord(d, p.recordId); + n++; + } catch (err) { + d.ctx.logger.warn({ downloadTaskId: r.id, err: { message: errMessage(err) } }, "补入库失败"); + } + } + return n; +} + +export const acquisitionOrganizeModule = createAcquisitionOrganizeModule(); diff --git a/apps/server/src/modules/acquisition-organize/ports.ts b/apps/server/src/modules/acquisition-organize/ports.ts new file mode 100644 index 00000000..7fba8b29 --- /dev/null +++ b/apps/server/src/modules/acquisition-organize/ports.ts @@ -0,0 +1,144 @@ +import { + createEmbyClient, + createQbClient, + type DiskUsage, + type DownloaderClient, + diskUsage, + type EmbyClient, + ffprobe, + getSiteAdapter, + getSiteDefinition, + type ProbeResult, + type SecretResolver, + type TorrentSource, + type VerifyMethod, +} from "@umt/adapters"; +import type { DownloaderInstanceRow, MediaServerInstanceRow, SiteRow, SourceReleaseRow } from "@umt/db"; +import type { ServerContext } from "../types.ts"; + +// 外部依赖端口:默认实现接真实 qB / 站点 / 文件系统 / Emby;测试注入 fake(mock qB、临时目录、固定磁盘用量)。 + +export type { DownloaderInstanceRow, MediaServerInstanceRow, SiteRow, SourceReleaseRow }; + +export interface AoOptions { + /** 下载同步/阈值巡检周期;缺省按 settings.cache.checkIntervalSec */ + syncIntervalMs?: number; + /** worker 启动时是否跑后台循环(测试里手动调用) */ + backgroundLoops: boolean; + /** 下载中且字节不增长超过该时长 → 记证据 + 待处理(不换源) */ + noProgressMs: number; + /** 主视频最小体积(低于视为样例) */ + minVideoBytes: number; + /** 入库复制的校验方式 */ + verifyHash: VerifyMethod; + /** 新作品窗口(天),策略未给时的缺省 */ + newTitleWindowDays: number; + /** 资产对账:连续多少次观察缺失才从 suspected_missing 进 confirmed_missing */ + missingConfirmAfter: number; + /** 后台资产对账周期(0 = 不自动跑) */ + reconcileIntervalMs: number; +} + +export interface AoPorts { + now(): Date; + /** 按实例取 qB 客户端(凭证现场经 secrets 解出) */ + downloader(inst: DownloaderInstanceRow): Promise; + /** 取种子来源(站点适配器现场取 .torrent / 磁力;不入库、不打日志) */ + torrentSource(input: { release: SourceReleaseRow; site: SiteRow | null }): Promise; + diskUsage(path: string): Promise; + probe(path: string): Promise; + mediaServer(inst: MediaServerInstanceRow): Promise; + options: AoOptions; +} + +export const DEFAULT_OPTIONS: AoOptions = { + backgroundLoops: true, + noProgressMs: 6 * 3600_000, + minVideoBytes: 50 * 1024 * 1024, + verifyHash: "sample", + newTitleWindowDays: 30, + missingConfirmAfter: 3, + reconcileIntervalMs: 6 * 3600_000, +}; + +const str = (v: unknown) => (typeof v === "string" && v.length > 0 ? v : undefined); + +/** 用 ctx 组装默认端口;overrides 逐项覆盖(测试) */ +export function createPorts( + ctx: ServerContext, + overrides: Partial> & { options?: Partial } = {}, +): AoPorts { + const clients = new Map(); + const resolveSecret = async (ref: string | null): Promise => { + if (!ref) return undefined; + try { + return await ctx.secrets.resolve(ref); + } catch (err) { + ctx.logger.warn({ err: { message: (err as Error).message } }, "凭证解析失败"); + return undefined; + } + }; + + const ports: AoPorts = { + now: () => new Date(), + async downloader(inst) { + const stamp = `${inst.updated_at.toISOString()}|${inst.base_url}`; + const hit = clients.get(inst.id); + if (hit && hit.stamp === stamp) return hit.client; + const settings = (inst.settings ?? {}) as Record; + const secret = await resolveSecret(inst.secret_ref); + const useApiKey = settings.authKind === "api_key"; + const client = createQbClient({ + instanceId: inst.id, + baseUrl: inst.base_url, + ...(inst.username ? { username: inst.username } : {}), + ...(secret ? (useApiKey ? { apiKey: secret } : { password: secret }) : {}), + }); + clients.set(inst.id, { stamp, client }); + return client; + }, + async torrentSource({ release, site }) { + if (!site) throw new Error("来源站点不存在,无法取种子"); + const raw = (release.raw ?? {}) as Record; + const siteMeta = (raw._site ?? {}) as Record; + const defId = [str(siteMeta.adapterSiteId), site.key].find((k) => k && getSiteDefinition(k)); + if (!defId) throw new Error(`站点 ${site.key} 没有可用的适配器定义`); + const accounts = await ctx.db + .selectFrom("site_account") + .select(["auth_kind", "secret_ref"]) + .where("site_id", "=", site.id) + .where("enabled", "=", true) + .execute(); + const secrets: SecretResolver = { + async resolve({ kind }) { + const want = kind === "apiKey" ? "api_key" : kind; + const acc = accounts.find((a) => a.auth_kind === want); + return (await resolveSecret(acc?.secret_ref ?? null)) ?? null; + }, + }; + const adapter = getSiteAdapter(defId, { secrets, baseUrl: site.base_url }); + const t = await adapter.downloadTarget(release.site_release_id); + if (t.kind === "torrent") + return { kind: "file", bytes: t.bytes, ...(t.fileName ? { filename: t.fileName } : {}) }; + if (t.kind === "magnet") return { kind: "magnet", uri: t.uri }; + return { kind: "url", url: t.url }; + }, + diskUsage: (p) => diskUsage(p), + probe: (p) => ffprobe(p), + async mediaServer(inst) { + const settings = (inst.settings ?? {}) as Record; + const apiKey = (await resolveSecret(inst.secret_ref)) ?? ""; + return createEmbyClient({ + instanceId: inst.id, + baseUrl: inst.base_url, + apiKey, + flavor: inst.kind, + ...(inst.user_id ? { userId: inst.user_id } : {}), + ...(str(settings.username) ? { username: str(settings.username)! } : {}), + }); + }, + options: { ...DEFAULT_OPTIONS }, + }; + const { options, ...rest } = overrides; + return { ...ports, ...rest, options: { ...ports.options, ...options } }; +} diff --git a/apps/server/src/modules/acquisition-organize/routes/dashboard.ts b/apps/server/src/modules/acquisition-organize/routes/dashboard.ts new file mode 100644 index 00000000..d8968314 --- /dev/null +++ b/apps/server/src/modules/acquisition-organize/routes/dashboard.ts @@ -0,0 +1,288 @@ +import { Dashboard, type DecisionKind, type SlotStatusCounts } from "@umt/contracts"; +import { sql } from "@umt/db"; +import type { ModuleApp } from "../../types.ts"; +import type { AoDeps } from "../services/common.ts"; +import { cleanupCandidates } from "../services/downloads.ts"; +import { decideVolume, loadVolumes, usedPercent } from "../services/volumes.ts"; +import { startOfDay } from "../util.ts"; +import { mount } from "./helpers.ts"; + +// 首页六区块(REQ §13.4.3):只读投影,统计未知时给 null 并在 notes 说明。 + +const iso = (v: Date | null | undefined) => (v ? v.toISOString() : null); + +export function registerDashboardRoutes(app: ModuleApp, d: AoDeps) { + mount( + app, + { + method: "get", + path: "/dashboard", + scope: "read", + summary: "首页聚合:自动化概览/当前工作/需要我处理/缓存与保留/来源健康/最近产出", + response: Dashboard, + }, + async () => { + const db = d.ctx.db; + const now = d.ports.now(); + const day = startOfDay(now); + const [automation, cache] = await Promise.all([d.ctx.settings.get("automation"), d.ctx.settings.get("cache")]); + + const today = ( + await sql<{ + plans: number; + submitted: number; + files: number; + ok: number; + bad: number; + upgrades: number; + subs: number; + }>` + select + (select count(*) from acquisition_plan where created_at >= ${day})::int as plans, + (select count(*) from download_task where created_at >= ${day})::int as submitted, + (select count(*) from organization_file_result f join organization_record r on r.id = f.organization_record_id + where f.status = 'succeeded' and f.role = 'main' and f.finished_at >= ${day})::int as files, + (select count(*) from organization_record where status = 'succeeded' and finished_at >= ${day})::int as ok, + (select count(*) from organization_record where status in ('failed','partial') and finished_at >= ${day})::int as bad, + (select count(*) from acquisition_plan where purpose = 'upgrade' and status = 'completed' and updated_at >= ${day})::int as upgrades, + (select count(*) from subscription where status = 'active')::int as subs`.execute(db) + ).rows[0]!; + const slotRows = await db + .selectFrom("acquisition_slot") + .select(["status"]) + .select(sql`count(*)::int`.as("n")) + .groupBy("status") + .execute(); + const slotCounts: SlotStatusCounts = { + empty: 0, + reserved: 0, + downloading: 0, + verifying: 0, + satisfied: 0, + upgrade_pending: 0, + attention: 0, + }; + for (const r of slotRows) slotCounts[r.status] = r.n; + + const dlRows = await db + .selectFrom("download_task") + .select(["id", "name", "status", "progress", "pause_reason", "no_progress_since"]) + .where("removed_at", "is", null) + .where("status", "in", ["queued", "submitting", "downloading", "stalled", "paused", "importing"]) + .orderBy("updated_at", "desc") + .limit(200) + .execute(); + const count = (s: string) => dlRows.filter((r) => r.status === s).length; + const reservedPlans = Number( + ( + await db + .selectFrom("acquisition_plan") + .select(sql`count(*)::int`.as("n")) + .where("status", "=", "reserved") + .executeTakeFirst() + )?.n ?? 0, + ); + const runningRecords = Number( + ( + await db + .selectFrom("organization_record") + .select(sql`count(*)::int`.as("n")) + .where("status", "in", ["queued", "running"]) + .executeTakeFirst() + )?.n ?? 0, + ); + const agentRuns = await db + .selectFrom("agent_run") + .select(["id", "goal", "status"]) + .where("status", "in", ["queued", "running", "waiting"]) + .orderBy("created_at", "desc") + .limit(10) + .execute(); + + const decisions = await db + .selectFrom("decision") + .select(["kind"]) + .select(sql`count(*)::int`.as("n")) + .where("status", "in", ["open", "investigating", "awaiting_user"]) + .groupBy("kind") + .execute(); + + const thr = await db + .selectFrom("download_task") + .select(["instance_id"]) + .select(sql`count(*)::int`.as("n")) + .where("status", "=", "paused") + .where("pause_reason", "=", "threshold") + .where("removed_at", "is", null) + .groupBy("instance_id") + .execute(); + const volumes: Dashboard["cache"]["volumes"] = []; + for (const v of await loadVolumes(d)) { + const dv = decideVolume(v, cache); + const cands = await cleanupCandidates(d, v); + const notes: string[] = []; + if (!v.snapshot) notes.push(v.path ? "磁盘用量读取失败,已保守阻止新增下载" : "实例未配置保存路径,无法统计"); + if (dv.unknownSizeTaskIds.length) notes.push(`${dv.unknownSizeTaskIds.length} 个在途任务体积未知`); + if (dv.unmanagedGrowthRisk) notes.push("存在未纳管任务仍在增长,暂停纳管任务后磁盘仍可能增长"); + const pct = usedPercent(v); + volumes.push({ + volumeKey: v.volumeKey, + instanceIds: v.instances.map((t) => t.inst.id), + usedPercent: pct === null ? null : Math.min(100, Math.max(0, pct)), + totalBytes: v.snapshot?.totalBytes ?? null, + usedBytes: v.snapshot?.usedBytes ?? null, + pendingBytes: dv.pendingBytes, + reservedBytes: dv.reservedBytes, + hardThresholdPercent: cache.hardThresholdPercent, + resumePercent: cache.resumePercent, + pausedByThreshold: thr + .filter((t) => v.instances.some((i) => i.inst.id === t.instance_id)) + .reduce((s, t) => s + t.n, 0), + cleanableBytes: cands.filter((c) => c.cleanable).reduce((s, c) => s + c.bytes, 0), + retentionLockedBytes: cands.filter((c) => !c.cleanable).reduce((s, c) => s + c.bytes, 0), + notes, + }); + } + + const [sites, feeds, downloaders, servers] = await Promise.all([ + db + .selectFrom("site") + .select(["id", "name", "enabled", "health_status", "last_snapshot_at"]) + .orderBy("name") + .execute(), + db + .selectFrom("feed_source") + .select(["id", "name", "enabled", "health_status", "last_success_at", "consecutive_failures", "last_error"]) + .orderBy("name") + .execute(), + db + .selectFrom("downloader_instance") + .select(["id", "name", "enabled", "health_status", "last_seen_at"]) + .orderBy("name") + .execute(), + db + .selectFrom("media_server_instance") + .select(["id", "name", "enabled", "health_status", "last_seen_at"]) + .orderBy("name") + .execute(), + ]); + + const recent = await db + .selectFrom("organization_record as r") + .leftJoin("media_entity as e", "e.id", "r.media_entity_id") + .select([ + "r.id", + "r.media_entity_id", + "e.title", + "r.action", + "r.status", + "r.target_path", + "r.file_total", + "r.file_succeeded", + "r.file_failed", + "r.file_skipped", + "r.finished_at", + ]) + .where("r.status", "not in", ["queued", "running"]) + .orderBy("r.change_seq", "desc") + .limit(10) + .execute(); + + const out: Dashboard = { + generatedAt: now.toISOString(), + automation: { + enabled: automation.enabled, + readOnlyReconcile: automation.readOnlyReconcile, + today: { + plansCreated: today.plans, + downloadsSubmitted: today.submitted, + filesImported: today.files, + recordsSucceeded: today.ok, + recordsFailed: today.bad, + upgrades: today.upgrades, + }, + activeSubscriptions: today.subs, + slotCounts, + }, + currentWork: { + downloading: count("downloading"), + paused: count("paused"), + stalled: count("stalled"), + importing: count("importing"), + reservedPlans, + runningRecords, + agentRuns: agentRuns.map((r) => ({ id: r.id, goal: r.goal, status: r.status })), + items: dlRows.slice(0, 20).map((r) => ({ + downloadTaskId: r.id, + name: r.name, + status: r.status, + progress: r.progress, + waitReason: + r.status === "paused" + ? `paused:${r.pause_reason ?? "user"}` + : r.status === "stalled" + ? "no_progress" + : r.status === "queued" + ? "queued_in_downloader" + : null, + })), + }, + needsAttention: { + total: decisions.reduce((s, r) => s + r.n, 0), + byKind: decisions.map((r) => ({ kind: r.kind as DecisionKind, count: r.n })), + }, + cache: { volumes }, + sources: { + sites: sites.map((s) => ({ + id: s.id, + name: s.name, + enabled: s.enabled, + health: s.health_status, + lastSnapshotAt: iso(s.last_snapshot_at), + })), + feeds: feeds.map((f) => ({ + id: f.id, + name: f.name, + enabled: f.enabled, + health: f.health_status, + lastSuccessAt: iso(f.last_success_at), + consecutiveFailures: f.consecutive_failures, + lastError: f.last_error + ? String( + (f.last_error as { message?: unknown }).message ?? + (f.last_error as { kind?: unknown }).kind ?? + "error", + ) + : null, + })), + downloaders: downloaders.map((x) => ({ + id: x.id, + name: x.name, + enabled: x.enabled, + health: x.health_status, + lastSeenAt: iso(x.last_seen_at), + })), + mediaServers: servers.map((x) => ({ + id: x.id, + name: x.name, + enabled: x.enabled, + health: x.health_status, + lastSeenAt: iso(x.last_seen_at), + })), + }, + recentOutput: recent.map((r) => ({ + recordId: r.id, + mediaEntityId: r.media_entity_id, + title: r.title, + action: r.action, + status: r.status, + targetPath: r.target_path, + counts: { total: r.file_total, succeeded: r.file_succeeded, failed: r.file_failed, skipped: r.file_skipped }, + finishedAt: iso(r.finished_at), + })), + }; + return out; + }, + { tag: "dashboard" }, + ); +} diff --git a/apps/server/src/modules/acquisition-organize/routes/downloads.ts b/apps/server/src/modules/acquisition-organize/routes/downloads.ts new file mode 100644 index 00000000..e87fea6c --- /dev/null +++ b/apps/server/src/modules/acquisition-organize/routes/downloads.ts @@ -0,0 +1,115 @@ +import { type AcquisitionPlan, Api, type DownloadTask } from "@umt/contracts"; +import { z } from "zod"; +import { ApiException } from "../../../http/errors.ts"; +import type { ModuleApp } from "../../types.ts"; +import type { AoDeps } from "../services/common.ts"; +import { cleanupDownloadCache, userPause, userResume } from "../services/downloads.ts"; +import { reconcileLibrary } from "../services/reconcile.ts"; +import { notFound, toApi, toApiList } from "../util.ts"; +import { mount, paginate } from "./helpers.ts"; + +// 下载计划 / 下载任务(含用户暂停与恢复)/ 手动触发缓存清理与资产对账。 + +const TAG = "downloads"; + +export function registerDownloadRoutes(app: ModuleApp, d: AoDeps) { + const db = () => d.ctx.db; + + mount( + app, + Api.acquisitionPlans.list, + async (_c, { query }) => { + let q = db().selectFrom("acquisition_plan").selectAll(); + if (query.slotId) q = q.where("slot_id", "=", query.slotId); + if (query.status) q = q.where("status", "=", query.status); + const p = await paginate(q, query); + return { items: toApiList(p.rows), nextCursor: p.nextCursor }; + }, + { tag: TAG }, + ); + mount( + app, + Api.acquisitionPlans.get, + async (_c, { params }) => { + const r = await db().selectFrom("acquisition_plan").selectAll().where("id", "=", params.id).executeTakeFirst(); + return toApi(r ?? notFound("下载计划")); + }, + { tag: TAG }, + ); + + mount( + app, + Api.downloadTasks.list, + async (_c, { query }) => { + let q = db().selectFrom("download_task").selectAll(); + if (query.instanceId) q = q.where("instance_id", "=", query.instanceId); + if (query.status) q = q.where("status", "=", query.status); + if (query.managed) q = q.where("managed", "=", query.managed === "true"); + const p = await paginate(q, query); + return { items: toApiList(p.rows), nextCursor: p.nextCursor }; + }, + { tag: TAG }, + ); + mount( + app, + Api.downloadTasks.get, + async (_c, { params }) => { + const r = await db().selectFrom("download_task").selectAll().where("id", "=", params.id).executeTakeFirst(); + return toApi(r ?? notFound("下载任务")); + }, + { tag: TAG }, + ); + mount( + app, + Api.downloadTasks.pause, + async (_c, { params }) => { + const exists = await db().selectFrom("download_task").select("id").where("id", "=", params.id).executeTakeFirst(); + if (!exists) notFound("下载任务"); + return toApi(await userPause(d, params.id)); + }, + { tag: TAG }, + ); + mount( + app, + Api.downloadTasks.resume, + async (_c, { params }) => { + const exists = await db().selectFrom("download_task").select("id").where("id", "=", params.id).executeTakeFirst(); + if (!exists) notFound("下载任务"); + const r = await userResume(d, params.id); + if (r === "threshold_active") + throw new ApiException("precondition_failed", "缓存卷仍高于恢复水位,阈值暂停不能被手动绕过"); + return toApi(r); + }, + { tag: TAG }, + ); + + mount( + app, + { + method: "post", + path: "/download-cache/cleanup", + scope: "admin", + summary: "立即按阈值清理下载缓存(只清已验证入库且满足保留/H&R 的纳管任务)", + body: z.object({ downloaderInstanceId: z.uuid().optional() }), + response: z.object({ report: z.record(z.string(), z.unknown()) }), + }, + async (_c, { body }) => ({ report: await cleanupDownloadCache(d, body) }), + { tag: TAG, status: 200 }, + ); + mount( + app, + { + method: "post", + path: "/library/reconcile", + scope: "admin", + summary: "立即做一轮资产对账(库根哨兵 + 增量 stat;不删记录、不补下载)", + body: z.object({ libraryRootId: z.uuid().optional() }), + response: z.object({ runId: z.uuid(), stats: z.record(z.string(), z.unknown()) }), + }, + async (_c, { body }) => { + const r = await reconcileLibrary(d, body); + return { runId: r.runId, stats: r.stats }; + }, + { tag: TAG, status: 200 }, + ); +} diff --git a/apps/server/src/modules/acquisition-organize/routes/helpers.ts b/apps/server/src/modules/acquisition-organize/routes/helpers.ts new file mode 100644 index 00000000..a45183f5 --- /dev/null +++ b/apps/server/src/modules/acquisition-organize/routes/helpers.ts @@ -0,0 +1,93 @@ +import { createRoute } from "@hono/zod-openapi"; +import { type ApiScope, apiResponse } from "@umt/contracts"; +import { sql } from "@umt/db"; +import type { Context } from "hono"; +import type { SelectQueryBuilder } from "kysely"; +import type { z } from "zod"; +import { type AuthEnv, requireScope } from "../../../http/auth.ts"; +import type { ModuleApp } from "../../types.ts"; +import { decodeCursor, encodeCursor } from "../util.ts"; + +// 路由注册:直接吃 contracts 的 EndpointDef(Api.xxx),scope 走 requireScope 中间件,入参按 schema 校验。 + +export interface RouteDef { + method: "get" | "post" | "patch" | "put" | "delete"; + path: string; + scope: ApiScope; + summary: string; + params?: z.ZodObject; + query?: z.ZodType; + body?: z.ZodType; + response: z.ZodType; +} + +export interface RouteInput { + // biome-ignore lint/suspicious/noExplicitAny: 入参已按各自 schema 校验,处理器按需收窄 + params: any; + // biome-ignore lint/suspicious/noExplicitAny: 同上 + query: any; + // biome-ignore lint/suspicious/noExplicitAny: 同上 + body: any; +} + +export type Handler = (c: Context, input: RouteInput) => Promise; + +export function mount( + app: ModuleApp, + maybe: RouteDef | undefined, + handler: Handler, + opts: { tag?: string; status?: 200 | 201 } = {}, +) { + if (!maybe) throw new Error("contracts 未定义该端点"); + const def = maybe; + const status = opts.status ?? (def.method === "post" && !def.path.includes("}/") ? 201 : 200); + const route = createRoute({ + method: def.method, + path: def.path, + tags: [opts.tag ?? "acquisition-organize"], + summary: def.summary, + middleware: [requireScope(def.scope)] as const, + request: { + ...(def.params ? { params: def.params } : {}), + ...(def.query ? { query: def.query as z.ZodObject } : {}), + ...(def.body ? { body: { content: { "application/json": { schema: def.body } }, required: true } } : {}), + }, + responses: { + [status]: { description: "成功", content: { "application/json": { schema: apiResponse(def.response) } } }, + }, + }); + app.openapi(route, (async (c: Context) => { + const v = c.req as unknown as { valid(t: string): unknown }; + const input: RouteInput = { + params: def.params ? v.valid("param") : {}, + query: def.query ? v.valid("query") : {}, + body: def.body ? v.valid("json") : undefined, + }; + const data = await handler(c, input); + return c.json({ ok: true as const, data }, status); + }) as never); +} + +/** (created_at desc, id desc) 游标翻页;col 前缀用于带别名的查询 */ +export async function paginate( + // biome-ignore lint/suspicious/noExplicitAny: 通用翻页,表类型由调用方决定 + qb: SelectQueryBuilder, + page: { cursor?: string | undefined; limit: number }, + prefix = "", +): Promise<{ rows: R[]; nextCursor: string | null }> { + const c = decodeCursor(page.cursor); + let q = qb; + if (c) + q = q.where( + sql`(${sql.ref(`${prefix}created_at`)}, ${sql.ref(`${prefix}id`)}) < (${c.at}::timestamptz, ${c.id}::uuid)`, + ); + const rows = await q + .orderBy(sql.ref(`${prefix}created_at`), "desc") + .orderBy(sql.ref(`${prefix}id`), "desc") + .limit(page.limit + 1) + .execute(); + const more = rows.length > page.limit; + const items = rows.slice(0, page.limit); + const last = items[items.length - 1]; + return { rows: items, nextCursor: more && last ? encodeCursor(last.created_at, last.id) : null }; +} diff --git a/apps/server/src/modules/acquisition-organize/routes/library.ts b/apps/server/src/modules/acquisition-organize/routes/library.ts new file mode 100644 index 00000000..23623905 --- /dev/null +++ b/apps/server/src/modules/acquisition-organize/routes/library.ts @@ -0,0 +1,430 @@ +import { join } from "node:path"; +import { + type AcquisitionPlan, + type AcquisitionSlot, + type AssetLocation, + type CandidateEvaluation, + cursorPage, + type DownloadTask, + type ExternalIdentity, + IdParam, + InventoryQuery, + InventoryResponse, + LibraryDetail, + type LibraryItem, + LibraryItem as LibraryItemSchema, + LibraryListQuery, + type MediaAsset, + type MediaEntity, + MediaServerInventory, + MediaServerInventoryQuery, + type PathBinding, + type SlotStatusCounts, +} from "@umt/contracts"; +import { sql } from "@umt/db"; +import { ApiException } from "../../../http/errors.ts"; +import type { ModuleApp } from "../../types.ts"; +import { type AoDeps, CANDIDATE_ARTIFACT_TITLE } from "../services/common.ts"; +import { notFound, toApi, toApiList } from "../util.ts"; +import { mount, paginate } from "./helpers.ts"; + +// 媒体库(列表/详情)、站点工作台库存叠加、Emby 兼容库存查询(多实例,未知 ≠ 缺失)。 + +const TAG = "library"; +const ACTIVE_DL = ["queued", "submitting", "downloading", "stalled", "paused", "completed", "importing"] as const; + +const emptyCounts = (): SlotStatusCounts => ({ + empty: 0, + reserved: 0, + downloading: 0, + verifying: 0, + satisfied: 0, + upgrade_pending: 0, + attention: 0, +}); + +export function registerLibraryRoutes(app: ModuleApp, d: AoDeps) { + const db = () => d.ctx.db; + + mount( + app, + { + method: "get", + path: "/library", + scope: "read", + summary: "媒体库列表(已登记资产或正在追踪的作品)", + query: LibraryListQuery, + response: cursorPage(LibraryItemSchema), + }, + async (_c, { query }) => { + let q = db() + .selectFrom("media_entity as e") + .select(["e.id", "e.kind", "e.title", "e.original_title", "e.year", "e.status", "e.created_at"]) + .select( + sql`(select count(*) from media_asset a where a.media_entity_id = e.id and a.status = 'active' and a.parent_asset_id is null)::int`.as( + "asset_count", + ), + ) + .select( + sql`(select count(*) from download_task dt left join acquisition_plan p on p.id = dt.acquisition_plan_id + left join acquisition_slot s on s.id = p.slot_id left join source_release sr on sr.id = dt.source_release_id + where (s.media_entity_id = e.id or sr.media_entity_id = e.id) and dt.removed_at is null + and dt.status in ('queued','submitting','downloading','stalled','paused'))::int`.as("active_downloads"), + ) + .select( + sql`(select max(r.finished_at) from organization_record r where r.media_entity_id = e.id and r.status in ('succeeded','partial'))`.as( + "last_org", + ), + ) + .select( + sql< + Record + >`(select coalesce(json_object_agg(x.status, x.n), '{}'::json) from (select status, count(*)::int n from acquisition_slot where media_entity_id = e.id group by status) x)`.as( + "slot_counts", + ), + ) + .where("e.status", "!=", "merged") + .where((eb) => + eb.or([ + eb.exists(eb.selectFrom("media_asset as a").select("a.id").whereRef("a.media_entity_id", "=", "e.id")), + eb.exists(eb.selectFrom("acquisition_slot as s").select("s.id").whereRef("s.media_entity_id", "=", "e.id")), + eb.exists( + eb.selectFrom("subscription as sub").select("sub.id").whereRef("sub.media_entity_id", "=", "e.id"), + ), + ]), + ); + if (query.kind) q = q.where("e.kind", "=", query.kind); + if (query.q) { + const like = `%${String(query.q).replace(/[\\%_]/g, (m) => `\\${m}`)}%`; + q = q.where((eb) => + eb.or([ + eb("e.title", "ilike", like), + eb("e.original_title", "ilike", like), + eb.exists( + eb + .selectFrom("title_alias as t") + .select("t.id") + .whereRef("t.media_entity_id", "=", "e.id") + .where("t.alias", "ilike", like), + ), + ]), + ); + } + if (query.filter === "gap") + q = q.where( + sql`exists (select 1 from acquisition_slot s where s.media_entity_id = e.id and s.status in ('empty','attention','upgrade_pending'))`, + ); + if (query.filter === "attention") + q = q.where( + sql`exists (select 1 from acquisition_slot s where s.media_entity_id = e.id and s.status = 'attention')`, + ); + if (query.filter === "satisfied") + q = q.where( + sql`exists (select 1 from acquisition_slot s where s.media_entity_id = e.id) and not exists (select 1 from acquisition_slot s where s.media_entity_id = e.id and s.status <> 'satisfied')`, + ); + if (query.filter === "downloading") + q = q.where( + sql`exists (select 1 from download_task dt left join acquisition_plan p on p.id = dt.acquisition_plan_id left join acquisition_slot s on s.id = p.slot_id + where s.media_entity_id = e.id and dt.removed_at is null and dt.status in ('queued','submitting','downloading','stalled','paused'))`, + ); + const p = await paginate(q, query, "e."); + const items: LibraryItem[] = p.rows.map((r) => ({ + entity: { + id: r.id, + kind: r.kind, + title: r.title, + originalTitle: r.original_title, + year: r.year, + status: r.status, + }, + assetCount: r.asset_count, + slotCounts: { ...emptyCounts(), ...(r.slot_counts ?? {}) }, + activeDownloads: r.active_downloads, + lastOrganizedAt: r.last_org ? new Date(r.last_org).toISOString() : null, + })); + return { items, nextCursor: p.nextCursor }; + }, + { tag: TAG }, + ); + + mount( + app, + { + method: "get", + path: "/library/{id}", + scope: "read", + summary: "作品详情:实际路径、身份映射、元数据来源、各 Slot、候选拒绝理由、下载历史与校验结果", + params: IdParam, + response: LibraryDetail, + }, + async (_c, { params }) => { + const e = await db().selectFrom("media_entity").selectAll().where("id", "=", params.id).executeTakeFirst(); + if (!e) notFound("作品"); + const id = e!.id; + const [identities, bindings, assets, slots, roots] = await Promise.all([ + db() + .selectFrom("external_identity") + .selectAll() + .where("media_entity_id", "=", id) + .orderBy("created_at") + .execute(), + db().selectFrom("path_binding").selectAll().where("media_entity_id", "=", id).orderBy("bound_at").execute(), + db().selectFrom("media_asset").selectAll().where("media_entity_id", "=", id).orderBy("created_at").execute(), + db() + .selectFrom("acquisition_slot") + .selectAll() + .where("media_entity_id", "=", id) + .orderBy("created_at") + .execute(), + db().selectFrom("library_root").select(["id", "local_path"]).execute(), + ]); + const assetIds = assets.map((a) => a.id); + const [locs, cov] = assetIds.length + ? await Promise.all([ + db().selectFrom("asset_location").selectAll().where("media_asset_id", "in", assetIds).execute(), + db() + .selectFrom("asset_unit_coverage") + .select(["media_asset_id", "content_unit_id"]) + .where("media_asset_id", "in", assetIds) + .execute(), + ]) + : [[], []]; + const slotIds = slots.map((s) => s.id); + const cands = slotIds.length + ? await db() + .selectFrom("artifact") + .select(["content"]) + .where("subject_type", "=", "acquisition_slot") + .where("subject_id", "in", slotIds) + .where("title", "=", CANDIDATE_ARTIFACT_TITLE) + .orderBy("updated_at", "desc") + .limit(100) + .execute() + : []; + const dts = await db() + .selectFrom("download_task as dt") + .leftJoin("acquisition_plan as p", "p.id", "dt.acquisition_plan_id") + .leftJoin("acquisition_slot as s", "s.id", "p.slot_id") + .leftJoin("source_release as sr", "sr.id", "dt.source_release_id") + .selectAll("dt") + .where((eb) => eb.or([eb("s.media_entity_id", "=", id), eb("sr.media_entity_id", "=", id)])) + .orderBy("dt.created_at", "desc") + .limit(100) + .execute(); + const planIds = dts.map((t) => t.acquisition_plan_id).filter((x): x is string => !!x); + const plans = planIds.length + ? await db().selectFrom("acquisition_plan").selectAll().where("id", "in", planIds).execute() + : []; + const recs = dts.length + ? await db() + .selectFrom("organization_record") + .select(["id", "download_task_id", "status", "created_at"]) + .where( + "download_task_id", + "in", + dts.map((t) => t.id), + ) + .orderBy("created_at") + .execute() + : []; + const rootPath = new Map(roots.map((r) => [r.id, r.local_path])); + const detail: LibraryDetail = { + entity: toApi(e!), + identities: toApiList(identities), + metadataSources: { fields: (e!.metadata ?? {}) as Record, fieldLocks: e!.field_locks }, + pathBindings: toApiList(bindings), + assets: assets.map((a) => ({ + asset: toApi(a), + locations: locs + .filter((l) => l.media_asset_id === a.id) + .map((l) => ({ + ...toApi(l), + absolutePath: + l.library_root_id && rootPath.has(l.library_root_id) + ? join(rootPath.get(l.library_root_id)!, l.relative_path) + : null, + })), + contentUnitIds: cov.filter((c) => c.media_asset_id === a.id).map((c) => c.content_unit_id), + })), + slots: toApiList(slots), + candidates: cands.map((c) => c.content as CandidateEvaluation), + downloads: dts.map((t) => { + const mine = recs.filter((r) => r.download_task_id === t.id); + const plan = plans.find((p) => p.id === t.acquisition_plan_id); + return { + downloadTask: toApi(t), + plan: plan ? toApi(plan) : null, + organizationRecordIds: mine.map((r) => r.id), + lastOrganizationStatus: mine[mine.length - 1]?.status ?? null, + }; + }), + }; + return detail; + }, + { tag: TAG }, + ); + + mount( + app, + { + method: "get", + path: "/inventory", + scope: "read", + summary: "库存叠加(站点工作台):按作品返回已有版本 / Slot 缺口 / 下载中", + query: InventoryQuery, + response: InventoryResponse, + }, + async (_c, { query }) => { + const ids = [ + ...new Set( + String(query.entityIds) + .split(",") + .map((s) => s.trim()) + .filter(Boolean), + ), + ]; + if (ids.length > 200 || ids.some((i) => !/^[0-9a-f-]{36}$/i.test(i))) + throw new ApiException("validation_failed", "entityIds 必须是最多 200 个 UUID"); + if (ids.length === 0) return { items: [] }; + const [assets, slots, dts] = await Promise.all([ + db() + .selectFrom("media_asset as a") + .select(["a.id", "a.media_entity_id", "a.delivery", "a.content_edition", "a.size_bytes"]) + .select( + sql`exists (select 1 from asset_location l where l.media_asset_id = a.id and l.status = 'available')`.as( + "available", + ), + ) + .where("a.media_entity_id", "in", ids) + .where("a.status", "=", "active") + .where("a.parent_asset_id", "is", null) + .execute(), + db() + .selectFrom("acquisition_slot") + .select(["id", "media_entity_id", "name", "status"]) + .where("media_entity_id", "in", ids) + .where("status", "in", ["empty", "attention", "upgrade_pending"]) + .execute(), + db() + .selectFrom("download_task as dt") + .leftJoin("acquisition_plan as p", "p.id", "dt.acquisition_plan_id") + .leftJoin("acquisition_slot as s", "s.id", "p.slot_id") + .leftJoin("source_release as sr", "sr.id", "dt.source_release_id") + .select(["dt.id", "dt.name", "dt.status", "dt.progress"]) + .select(sql`coalesce(s.media_entity_id, sr.media_entity_id)`.as("entity_id")) + .where((eb) => eb.or([eb("s.media_entity_id", "in", ids), eb("sr.media_entity_id", "in", ids)])) + .where("dt.removed_at", "is", null) + .where("dt.status", "in", [...ACTIVE_DL]) + .execute(), + ]); + return { + items: ids.map((entityId) => ({ + entityId, + versions: assets + .filter((a) => a.media_entity_id === entityId) + .map((a) => { + const dv = (a.delivery ?? {}) as { resolution?: string; sourceType?: string; releaseGroup?: string }; + return { + assetId: a.id, + resolution: dv.resolution ?? null, + sourceType: dv.sourceType ?? null, + releaseGroup: dv.releaseGroup ?? null, + contentEdition: a.content_edition, + sizeBytes: a.size_bytes, + available: a.available, + }; + }), + gaps: slots + .filter((s) => s.media_entity_id === entityId) + .map((s) => ({ slotId: s.id, name: s.name, status: s.status })), + downloading: dts + .filter((t) => t.entity_id === entityId) + .map((t) => ({ downloadTaskId: t.id, name: t.name, status: t.status, progress: t.progress })), + })), + }; + }, + { tag: TAG }, + ); + + mount( + app, + { + method: "get", + path: "/inventory/media-servers", + scope: "read", + summary: "Emby 兼容库存查询(多实例;接口失败/截断 = unknown,不等于缺失)", + query: MediaServerInventoryQuery, + response: MediaServerInventory, + }, + async (_c, { query }) => { + const e = await db() + .selectFrom("media_entity") + .select(["id", "kind", "title", "year"]) + .where("id", "=", query.mediaEntityId) + .executeTakeFirst(); + if (!e) notFound("作品"); + const idents = await db() + .selectFrom("external_identity") + .select(["provider", "entity_type", "external_id"]) + .where("media_entity_id", "=", e!.id) + .where("status", "=", "accepted") + .execute(); + const insts = await db() + .selectFrom("media_server_instance") + .selectAll() + .where("enabled", "=", true) + .orderBy("name") + .execute(); + const type = e!.kind === "movie" ? "Movie" : "Series"; + const KEY: Record = { tmdb: "Tmdb", imdb: "Imdb", tvdb: "Tvdb", douban: "Douban" }; + const out: MediaServerInventory = { mediaEntityId: e!.id, instances: [] }; + for (const inst of insts) { + const observedAt = new Date().toISOString(); + const usable = idents.filter((i) => KEY[i.provider]); + if (usable.length === 0) { + out.instances.push({ + instanceId: inst.id, + name: inst.name, + status: "unknown", + reason: "no_external_id", + items: [], + observedAt, + }); + continue; + } + let status: "present" | "not_found" | "unknown" = "not_found"; + let reason: string | null = null; + const items: MediaServerInventory["instances"][number]["items"] = []; + try { + const client = await d.ports.mediaServer(inst); + for (const i of usable) { + const r = await client.findByProviderId(KEY[i.provider]!, i.external_id, type); + if (r.status === "present") { + status = "present"; + for (const it of r.items) { + if (items.some((x) => x.itemId === it.itemId)) continue; + items.push({ + itemId: it.itemId, + type: it.type, + name: it.name, + year: it.year ?? null, + path: it.path ?? null, + versions: it.versions.map((v) => JSON.parse(JSON.stringify(v)) as Record), + }); + } + } else if (r.status === "unknown" && status !== "present") { + status = "unknown"; + reason = r.reason; + } + } + } catch (err) { + status = "unknown"; + reason = (err as Error).message; + } + out.instances.push({ instanceId: inst.id, name: inst.name, status, reason, items, observedAt }); + } + return out; + }, + { tag: TAG }, + ); +} diff --git a/apps/server/src/modules/acquisition-organize/routes/organization.ts b/apps/server/src/modules/acquisition-organize/routes/organization.ts new file mode 100644 index 00000000..48bc1b44 --- /dev/null +++ b/apps/server/src/modules/acquisition-organize/routes/organization.ts @@ -0,0 +1,170 @@ +import { randomUUID } from "node:crypto"; +import { Api, type OrganizationChangesResponse } from "@umt/contracts"; +import { sql } from "@umt/db"; +import { ApiException } from "../../../http/errors.ts"; +import type { ModuleApp } from "../../types.ts"; +import { type AoDeps, ensureTaskRow, markTask } from "../services/common.ts"; +import { loadRecordsPublic, planImport } from "../services/organize.ts"; +import { actorOf, errMessage, jsonb, notFound } from "../util.ts"; +import { mount, paginate } from "./helpers.ts"; + +// 整理记录公开 API(organize.read / organize.execute)。changes 必须先于 {id} 注册。 + +const TAG = "organization"; + +export function registerOrganizationRoutes(app: ModuleApp, d: AoDeps) { + const db = () => d.ctx.db; + + mount( + app, + Api.organization.list, + async (_c, { query }) => { + let q = db().selectFrom("organization_record").selectAll(); + if (query.status) q = q.where("status", "=", query.status); + if (query.trigger) q = q.where("trigger", "=", query.trigger); + if (query.mediaEntityId) q = q.where("media_entity_id", "=", query.mediaEntityId); + if (query.q) { + const like = `%${String(query.q).replace(/[\\%_]/g, (m) => `\\${m}`)}%`; + q = q.where((eb) => + eb.or([ + eb("source_path", "ilike", like), + eb("target_path", "ilike", like), + eb("media_entity_id", "in", eb.selectFrom("media_entity").select("id").where("title", "ilike", like)), + ]), + ); + } + const p = await paginate(q, query); + return { items: await loadRecordsPublic(db(), p.rows, false), nextCursor: p.nextCursor }; + }, + { tag: TAG }, + ); + + // 增量读取:change_xid 早于当前快照最小活跃事务,避免"序号小但提交晚"被跳过 + mount( + app, + Api.organization.changes, + async (_c, { query }): Promise => { + const after = Number(query.after ?? "0"); + let q = db() + .selectFrom("organization_record") + .selectAll() + .where("change_seq", ">", after) + .where(sql`change_xid < pg_snapshot_xmin(pg_current_snapshot())`); + if (query.status) q = q.where("status", "=", query.status); + if (query.mediaEntityId) q = q.where("media_entity_id", "=", query.mediaEntityId); + const rows = await q + .orderBy("change_seq") + .limit(query.limit + 1) + .execute(); + const hasMore = rows.length > query.limit; + const page = rows.slice(0, query.limit); + const last = page[page.length - 1]; + return { + items: await loadRecordsPublic(db(), page, query.includeFiles), + nextCursor: last ? String(last.change_seq) : String(after), + hasMore, + }; + }, + { tag: TAG }, + ); + + mount( + app, + Api.organization.get, + async (_c, { params }) => { + const r = await db().selectFrom("organization_record").selectAll().where("id", "=", params.id).executeTakeFirst(); + if (!r) notFound("整理记录"); + return (await loadRecordsPublic(db(), [r!], true))[0]!; + }, + { tag: TAG }, + ); + + // 手动发起整理:与自动整理走同一冻结计划流程;返回 record/task ID,由任务页追踪 + mount( + app, + Api.organization.create, + async (c, { body }) => { + const principal = c.get("principal"); + const actorType = principal?.kind === "api_key" ? "api_key" : "user"; + const idem = + body.downloadTaskId && !body.dryRun + ? `manual:${body.downloadTaskId}:${randomUUID()}` + : `manual:${randomUUID()}`; + const taskId = await ensureTaskRow(db(), { + kind: "verifyAndImport", + idempotencyKey: idem, + input: { ...body }, + subjectType: body.downloadTaskId ? "download_task" : "organize_request", + subjectId: body.downloadTaskId ?? null, + triggeredBy: actorOf(c), + }); + if (body.downloadTaskId) { + const dt = await db() + .selectFrom("download_task") + .select(["status"]) + .where("id", "=", body.downloadTaskId) + .executeTakeFirst(); + if (!dt) throw new ApiException("not_found", "下载任务不存在"); + if (!["completed", "importing", "imported"].includes(dt.status)) + throw new ApiException("precondition_failed", `下载任务尚未完成(${dt.status})`); + } + let plan: Awaited>; + try { + plan = await planImport(d, { + downloadTaskId: body.downloadTaskId, + sourcePaths: body.sourcePaths, + mediaEntityId: body.mediaEntityId, + contentUnitId: body.contentUnitId, + action: body.action, + targetLibraryRootId: body.targetLibraryRootId, + trigger: "manual", + actorType, + actorId: actorOf(c), + taskId, + dryRun: body.dryRun, + }); + } catch (err) { + await markTask(db(), taskId, "failed", { error: errMessage(err) }); + throw new ApiException("bad_request", errMessage(err)); + } + if (body.dryRun) { + // 只生成计划:冻结为 artifact(kind=plan),planId 供 organize.execute 使用 + const art = await db() + .insertInto("artifact") + .values({ + kind: "plan", + task_id: taskId, + subject_type: "organize_request", + title: "organize_plan", + content: jsonb({ request: body, plan: plan.plan ?? null, reason: plan.reason, status: plan.status }), + content_hash: randomUUID(), + }) + .returning("id") + .executeTakeFirstOrThrow(); + await markTask(db(), taskId, "succeeded", { + output: { planId: art.id, status: plan.status, reason: plan.reason }, + }); + return { recordId: null, taskId, planId: art.id }; + } + if (plan.status === "running" || plan.status === "queued") { + const { runId } = await d.ctx.engine.enqueue( + "verifyAndImport", + { recordId: plan.recordId, downloadTaskId: body.downloadTaskId, taskId }, + { idempotencyKey: `verifyAndImport:${idem}`, traceId: taskId }, + ); + await db().updateTable("task").set({ engine_run_id: runId }).where("id", "=", taskId).execute(); + if (plan.recordId) + await db() + .updateTable("organization_record") + .set({ task_id: taskId }) + .where("id", "=", plan.recordId) + .where("task_id", "is", null) + .execute(); + } else { + await markTask(db(), taskId, "failed", { error: plan.reason ?? plan.status }); + } + return { recordId: plan.recordId, taskId, planId: null }; + }, + { tag: TAG, status: 201 }, + ); +} diff --git a/apps/server/src/modules/acquisition-organize/routes/rules.ts b/apps/server/src/modules/acquisition-organize/routes/rules.ts new file mode 100644 index 00000000..c07454e8 --- /dev/null +++ b/apps/server/src/modules/acquisition-organize/routes/rules.ts @@ -0,0 +1,730 @@ +import { + AcquisitionPolicyRules, + type AcquisitionSlot, + Api, + type PolicyRevision, + type PreferenceSuppression, + type PreferenceTag, + type Subscription, +} from "@umt/contracts"; +import { type DB, sql, type Transaction } from "@umt/db"; +import { evaluateCandidate, parseRelease } from "@umt/domain"; +import { slotSpecFromRow } from "@umt/mapping"; +import { ApiException } from "../../../http/errors.ts"; +import type { ModuleApp } from "../../types.ts"; +import { createSlotsFromTemplates, publishedAcquisitionPolicies } from "../services/acquisition.ts"; +import type { AoDeps } from "../services/common.ts"; +import { actorOf, conflict, errMessage, hashJson, jsonb, notFound, toApi, toApiList } from "../util.ts"; +import { mount, paginate } from "./helpers.ts"; + +// 规则 API:订阅、收录 Slot、策略版本(草案→发布→回滚,发布即持续授权)、偏好标签与抑制记录。 + +const TAG = "rules"; + +/** 规则变化后按需重评(引擎未就绪/入队失败不影响写入) */ +async function kickEvaluate(d: AoDeps, input: { subscriptionId?: string; acquisitionSlotId?: string }) { + try { + const key = input.acquisitionSlotId ?? input.subscriptionId ?? "x"; + await d.ctx.engine.enqueue("evaluateAcquisition", input, { + idempotencyKey: `reeval:${key}:${Date.now()}`, + traceId: key, + }); + } catch (err) { + d.ctx.logger.debug({ err: { message: errMessage(err) } }, "重评入队失败"); + } +} + +export function registerRuleRoutes(app: ModuleApp, d: AoDeps) { + const db = () => d.ctx.db; + + // ── 订阅 ── + mount( + app, + Api.subscriptions.list, + async (_c, { query }) => { + let q = db().selectFrom("subscription").selectAll(); + if (query.status) q = q.where("status", "=", query.status); + if (query.mediaEntityId) q = q.where("media_entity_id", "=", query.mediaEntityId); + const p = await paginate(q, query); + return { items: toApiList(p.rows), nextCursor: p.nextCursor }; + }, + { tag: TAG }, + ); + mount( + app, + Api.subscriptions.get, + async (_c, { params }) => { + const r = await db().selectFrom("subscription").selectAll().where("id", "=", params.id).executeTakeFirst(); + return toApi(r ?? notFound("订阅")); + }, + { tag: TAG }, + ); + mount( + app, + Api.subscriptions.create, + async (c, { body }) => { + if (body.mediaEntityId) { + const e = await db() + .selectFrom("media_entity") + .select("id") + .where("id", "=", body.mediaEntityId) + .executeTakeFirst(); + if (!e) throw new ApiException("bad_request", "mediaEntityId 不存在"); + } + const row = await db() + .transaction() + .execute(async (trx) => { + const r = await trx + .insertInto("subscription") + .values({ + media_entity_id: body.mediaEntityId ?? null, + season_unit_id: body.seasonUnitId ?? null, + title_query: body.titleQuery ?? null, + media_kind: body.mediaKind, + mode: body.mode ?? "continuous", + // 用户主动添加的剧集默认持续追踪并接续后续季;电影无季概念 + season_scope: body.seasonScope ?? (body.mediaKind === "movie" ? "current_season" : "following_seasons"), + origin: body.origin ?? "user", + priority: body.priority ?? "tracking", + policy_revision_id: body.policyRevisionId ?? null, + params: jsonb(body.params), + created_by: actorOf(c), + }) + .returningAll() + .executeTakeFirstOrThrow(); + if (r.media_entity_id) { + const has = await trx + .selectFrom("acquisition_slot") + .select("id") + .where("media_entity_id", "=", r.media_entity_id) + .executeTakeFirst(); + if (!has) { + const pols = await publishedAcquisitionPolicies(trx); + const pol = + (body.policyRevisionId ? pols.find((p) => p.row.id === body.policyRevisionId) : undefined) ?? + pols.find((p) => p.rules.defaultSlots?.length); + if (pol?.rules.defaultSlots?.length) { + await createSlotsFromTemplates(trx, { + entityId: r.media_entity_id, + subscriptionId: r.id, + policyRevisionId: pol.row.id, + templates: pol.rules.defaultSlots, + }); + } + } + } + return r; + }); + if (row.media_entity_id) await kickEvaluate(d, { subscriptionId: row.id }); + return toApi(row); + }, + { tag: TAG }, + ); + mount( + app, + Api.subscriptions.update, + async (_c, { params, body }) => { + const r = await db() + .updateTable("subscription") + .set({ + ...(body.mode !== undefined ? { mode: body.mode } : {}), + ...(body.seasonScope !== undefined ? { season_scope: body.seasonScope } : {}), + ...(body.priority !== undefined ? { priority: body.priority } : {}), + ...(body.status !== undefined ? { status: body.status } : {}), + ...(body.policyRevisionId !== undefined ? { policy_revision_id: body.policyRevisionId } : {}), + ...(body.params !== undefined ? { params: jsonb(body.params) } : {}), + ...(body.statusReason !== undefined ? { status_reason: body.statusReason } : {}), + updated_at: new Date(), + }) + .where("id", "=", params.id) + .returningAll() + .executeTakeFirst(); + return toApi(r ?? notFound("订阅")); + }, + { tag: TAG }, + ); + + // ── 收录 Slot ── + mount( + app, + Api.acquisitionSlots.list, + async (_c, { query }) => { + let q = db().selectFrom("acquisition_slot").selectAll(); + if (query.mediaEntityId) q = q.where("media_entity_id", "=", query.mediaEntityId); + if (query.status) q = q.where("status", "=", query.status); + const p = await paginate(q, query); + return { items: toApiList(p.rows), nextCursor: p.nextCursor }; + }, + { tag: TAG }, + ); + mount( + app, + Api.acquisitionSlots.get, + async (_c, { params }) => { + const r = await db().selectFrom("acquisition_slot").selectAll().where("id", "=", params.id).executeTakeFirst(); + return toApi(r ?? notFound("收录目标")); + }, + { tag: TAG }, + ); + mount( + app, + Api.acquisitionSlots.create, + async (_c, { body }) => { + const e = await db() + .selectFrom("media_entity") + .select("id") + .where("id", "=", body.mediaEntityId) + .executeTakeFirst(); + if (!e) throw new ApiException("bad_request", "mediaEntityId 不存在"); + const r = await db() + .insertInto("acquisition_slot") + .values({ + media_entity_id: body.mediaEntityId, + content_unit_id: body.contentUnitId ?? null, + subscription_id: body.subscriptionId ?? null, + policy_revision_id: body.policyRevisionId ?? null, + name: body.name, + hard_conditions: jsonb(body.hardConditions), + soft_preferences: jsonb(body.softPreferences), + fallback: jsonb(body.fallback), + upgrade_threshold: jsonb(body.upgradeThreshold), + completion: jsonb(body.completion), + distinct_group: body.distinctGroup ?? null, + require_distinct_asset: body.requireDistinctAsset ?? false, + }) + .onConflict((oc) => oc.doNothing()) + .returningAll() + .executeTakeFirst(); + if (!r) conflict("同一作品/内容单元下已有同名 Slot"); + await kickEvaluate(d, { acquisitionSlotId: r!.id }); + return toApi(r!); + }, + { tag: TAG }, + ); + mount( + app, + Api.acquisitionSlots.update, + async (_c, { params, body }) => { + const { expectedRevision, ...b } = body; + const r = await db() + .updateTable("acquisition_slot") + .set({ + ...(b.subscriptionId !== undefined ? { subscription_id: b.subscriptionId } : {}), + ...(b.policyRevisionId !== undefined ? { policy_revision_id: b.policyRevisionId } : {}), + ...(b.hardConditions !== undefined ? { hard_conditions: jsonb(b.hardConditions) } : {}), + ...(b.softPreferences !== undefined ? { soft_preferences: jsonb(b.softPreferences) } : {}), + ...(b.fallback !== undefined ? { fallback: jsonb(b.fallback) } : {}), + ...(b.upgradeThreshold !== undefined ? { upgrade_threshold: jsonb(b.upgradeThreshold) } : {}), + ...(b.completion !== undefined ? { completion: jsonb(b.completion) } : {}), + ...(b.distinctGroup !== undefined ? { distinct_group: b.distinctGroup } : {}), + ...(b.requireDistinctAsset !== undefined ? { require_distinct_asset: b.requireDistinctAsset } : {}), + revision: sql`revision + 1`, + }) + .where("id", "=", params.id) + .where("revision", "=", expectedRevision) + .returningAll() + .executeTakeFirst(); + if (!r) { + const exists = await db() + .selectFrom("acquisition_slot") + .select("revision") + .where("id", "=", params.id) + .executeTakeFirst(); + if (!exists) notFound("收录目标"); + conflict(`revision 不匹配(当前 ${exists!.revision})`); + } + await kickEvaluate(d, { acquisitionSlotId: params.id }); + return toApi(r!); + }, + { tag: TAG }, + ); + + // ── 策略版本 ── + mount( + app, + Api.policyRevisions.list, + async (_c, { query }) => { + let q = db().selectFrom("policy_revision").selectAll(); + if (query.policyKey) q = q.where("policy_key", "=", query.policyKey); + if (query.status) q = q.where("status", "=", query.status); + const p = await paginate(q, query); + return { items: toApiList(p.rows), nextCursor: p.nextCursor }; + }, + { tag: TAG }, + ); + mount( + app, + Api.policyRevisions.get, + async (_c, { params }) => { + const r = await db().selectFrom("policy_revision").selectAll().where("id", "=", params.id).executeTakeFirst(); + return toApi(r ?? notFound("策略版本")); + }, + { tag: TAG }, + ); + const checkRules = (kind: string, rules: unknown) => { + if (kind !== "acquisition") return; + const r = AcquisitionPolicyRules.safeParse(rules); + if (!r.success) + throw new ApiException( + "validation_failed", + `rules 不符合收录策略结构: ${r.error.issues.map((i) => `${i.path.join(".")} ${i.message}`).join("; ")}`, + ); + }; + mount( + app, + Api.policyRevisions.create, + async (c, { body }) => { + checkRules(body.kind, body.rules); + const r = await db() + .transaction() + .execute(async (trx) => { + await sql`select pg_advisory_xact_lock(hashtext(${`policy:${body.policyKey}`}))`.execute(trx); + const prev = await trx + .selectFrom("policy_revision") + .select(["kind"]) + .select(sql`max(revision)`.as("max")) + .where("policy_key", "=", body.policyKey) + .groupBy("kind") + .executeTakeFirst(); + if (prev && prev.kind !== body.kind) conflict(`policyKey ${body.policyKey} 已属于 ${prev.kind}`); + return trx + .insertInto("policy_revision") + .values({ + policy_key: body.policyKey, + revision: (prev?.max ?? 0) + 1, + kind: body.kind, + status: "draft", + title: body.title, + summary: body.summary ?? null, + scope: jsonb(body.scope), + rules: jsonb(body.rules), + based_on_revision_id: body.basedOnRevisionId ?? null, + created_by: actorOf(c), + }) + .returningAll() + .executeTakeFirstOrThrow(); + }); + return toApi(r); + }, + { tag: TAG }, + ); + mount( + app, + Api.policyRevisions.update, + async (_c, { params, body }) => { + const cur = await db().selectFrom("policy_revision").selectAll().where("id", "=", params.id).executeTakeFirst(); + if (!cur) notFound("策略版本"); + if (cur!.status !== "draft") conflict("只有草案可以修改;已发布版本请新建草案"); + if (body.rules !== undefined) checkRules(cur!.kind, body.rules); + const r = await db() + .updateTable("policy_revision") + .set({ + ...(body.title !== undefined ? { title: body.title } : {}), + ...(body.summary !== undefined ? { summary: body.summary } : {}), + ...(body.scope !== undefined ? { scope: jsonb(body.scope) } : {}), + ...(body.rules !== undefined ? { rules: jsonb(body.rules) } : {}), + ...(body.basedOnRevisionId !== undefined ? { based_on_revision_id: body.basedOnRevisionId } : {}), + preview: null, + }) + .where("id", "=", params.id) + .where("status", "=", "draft") + .returningAll() + .executeTakeFirstOrThrow(); + return toApi(r); + }, + { tag: TAG }, + ); + mount( + app, + Api.policyRevisions.preview, + async (_c, { params }) => { + const cur = await db().selectFrom("policy_revision").selectAll().where("id", "=", params.id).executeTakeFirst(); + if (!cur) notFound("策略版本"); + const preview = await previewPolicy(d, cur!.rules); + const r = await db() + .updateTable("policy_revision") + .set({ preview: jsonb(preview) }) + .where("id", "=", params.id) + .returningAll() + .executeTakeFirstOrThrow(); + return toApi(r); + }, + { tag: TAG }, + ); + mount( + app, + Api.policyRevisions.publish, + async (c, { params, body }) => { + const r = await db() + .transaction() + .execute(async (trx) => { + const cur = await trx + .selectFrom("policy_revision") + .selectAll() + .where("id", "=", params.id) + .forUpdate() + .executeTakeFirst(); + if (!cur) notFound("策略版本"); + if (cur!.revision !== body.expectedRevision) conflict(`revision 不匹配(当前 ${cur!.revision})`); + if (cur!.status !== "draft") conflict(`只能发布草案(当前 ${cur!.status})`); + checkRules(cur!.kind, cur!.rules); + return publishRevision(trx, cur!.id, cur!.policy_key, actorOf(c), "superseded"); + }); + return toApi(r); + }, + { tag: TAG }, + ); + mount( + app, + Api.policyRevisions.rollback, + async (c, { params }) => { + const r = await db() + .transaction() + .execute(async (trx) => { + const target = await trx + .selectFrom("policy_revision") + .selectAll() + .where("id", "=", params.id) + .executeTakeFirst(); + if (!target) notFound("策略版本"); + if (target!.status === "published") conflict("该版本已是当前发布版本"); + if (target!.status === "draft") conflict("草案不能作为回滚目标"); + await sql`select pg_advisory_xact_lock(hashtext(${`policy:${target!.policy_key}`}))`.execute(trx); + const max = await trx + .selectFrom("policy_revision") + .select(sql`max(revision)`.as("m")) + .where("policy_key", "=", target!.policy_key) + .executeTakeFirstOrThrow(); + // 回滚 = 以目标版本内容发布一个新版本(历史单调递增,可审计) + const clone = await trx + .insertInto("policy_revision") + .values({ + policy_key: target!.policy_key, + revision: max.m + 1, + kind: target!.kind, + status: "draft", + title: target!.title, + summary: `回滚到 r${target!.revision}`, + scope: jsonb(target!.scope), + rules: jsonb(target!.rules), + based_on_revision_id: target!.id, + created_by: actorOf(c), + }) + .returning("id") + .executeTakeFirstOrThrow(); + return publishRevision(trx, clone.id, target!.policy_key, actorOf(c), "rolled_back"); + }); + return toApi(r); + }, + { tag: TAG }, + ); + + // ── 偏好标签 ── + const fingerprint = (b: { kind: string; scope?: unknown; rule?: unknown }) => + hashJson({ kind: b.kind, scope: b.scope ?? {}, rule: b.rule ?? {} }); + mount( + app, + Api.preferenceTags.list, + async (_c, { query }) => { + let q = db().selectFrom("preference_tag").selectAll(); + if (query.status) q = q.where("status", "=", query.status); + else q = q.where("status", "!=", "deleted"); + if (query.kind) q = q.where("kind", "=", query.kind); + const p = await paginate(q, query); + return { items: toApiList(p.rows), nextCursor: p.nextCursor }; + }, + { tag: TAG }, + ); + mount( + app, + Api.preferenceTags.get, + async (_c, { params }) => { + const r = await db().selectFrom("preference_tag").selectAll().where("id", "=", params.id).executeTakeFirst(); + return toApi(r ?? notFound("偏好标签")); + }, + { tag: TAG }, + ); + mount( + app, + Api.preferenceTags.create, + async (c, { body }) => { + const fp = fingerprint(body); + const origin = body.origin ?? "manual"; + const r = await db() + .transaction() + .execute(async (trx) => { + const sup = await trx + .selectFrom("preference_suppression") + .selectAll() + .where("fingerprint", "=", fp) + .where("active", "=", true) + .executeTakeFirst(); + if (sup) { + // Agent 不得据旧证据自动加回;用户手动(或对话确认)重新添加即解除抑制 + if (origin === "agent_inferred") conflict("该偏好已被用户取消(存在抑制记录),Agent 不能自动加回"); + await trx + .updateTable("preference_suppression") + .set({ active: false, lifted_at: new Date(), lifted_by: actorOf(c) }) + .where("id", "=", sup.id) + .execute(); + } + const ins = await trx + .insertInto("preference_tag") + .values({ + label: body.label, + kind: body.kind, + scope: jsonb(body.scope), + rule: jsonb(body.rule), + fingerprint: fp, + origin, + description: body.description ?? null, + evidence: jsonb(body.evidence), + }) + .onConflict((oc) => oc.doNothing()) + .returningAll() + .executeTakeFirst(); + if (!ins) conflict("已存在同一语义的偏好标签"); + return ins!; + }); + return toApi(r); + }, + { tag: TAG }, + ); + mount( + app, + Api.preferenceTags.update, + async (c, { params, body }) => { + const r = await db() + .transaction() + .execute(async (trx) => { + const cur = await trx + .selectFrom("preference_tag") + .selectAll() + .where("id", "=", params.id) + .forUpdate() + .executeTakeFirst(); + if (!cur || cur.status === "deleted") notFound("偏好标签"); + const next = { kind: cur!.kind, scope: body.scope ?? cur!.scope, rule: body.rule ?? cur!.rule }; + const fp = fingerprint(next); + const status = body.status ?? cur!.status; + if (status !== "active" && cur!.status === "active") + await suppress(trx, cur!.id, cur!.fingerprint, actorOf(c), `status:${status}`); + if (status === "active" && cur!.status !== "active") { + await trx + .updateTable("preference_suppression") + .set({ active: false, lifted_at: new Date(), lifted_by: actorOf(c) }) + .where("fingerprint", "=", fp) + .where("active", "=", true) + .execute(); + } + if (fp !== cur!.fingerprint && cur!.status === "active") + await suppress(trx, cur!.id, cur!.fingerprint, actorOf(c), "edited"); + return trx + .updateTable("preference_tag") + .set({ + ...(body.label !== undefined ? { label: body.label } : {}), + ...(body.kind !== undefined ? { kind: body.kind } : {}), + ...(body.scope !== undefined ? { scope: jsonb(body.scope) } : {}), + ...(body.rule !== undefined ? { rule: jsonb(body.rule) } : {}), + ...(body.description !== undefined ? { description: body.description } : {}), + ...(body.evidence !== undefined ? { evidence: jsonb(body.evidence) } : {}), + fingerprint: fp, + status, + revision: cur!.revision + 1, + }) + .where("id", "=", params.id) + .returningAll() + .executeTakeFirstOrThrow(); + }); + return toApi(r); + }, + { tag: TAG }, + ); + mount( + app, + Api.preferenceTags.remove, + async (c, { params }) => { + await db() + .transaction() + .execute(async (trx) => { + const cur = await trx + .selectFrom("preference_tag") + .selectAll() + .where("id", "=", params.id) + .forUpdate() + .executeTakeFirst(); + if (!cur || cur.status === "deleted") notFound("偏好标签"); + await trx + .updateTable("preference_tag") + .set({ status: "deleted", revision: cur!.revision + 1 }) + .where("id", "=", params.id) + .execute(); + await suppress(trx, cur!.id, cur!.fingerprint, actorOf(c), "deleted"); + }); + return { ok: true as const }; + }, + { tag: TAG, status: 200 }, + ); + + // ── 抑制记录 ── + mount( + app, + Api.preferenceSuppressions.list, + async (_c, { query }) => { + let q = db().selectFrom("preference_suppression").selectAll(); + if (query.active) q = q.where("active", "=", query.active === "true"); + const p = await paginate(q, query); + return { items: toApiList(p.rows), nextCursor: p.nextCursor }; + }, + { tag: TAG }, + ); + mount( + app, + Api.preferenceSuppressions.get, + async (_c, { params }) => { + const r = await db() + .selectFrom("preference_suppression") + .selectAll() + .where("id", "=", params.id) + .executeTakeFirst(); + return toApi(r ?? notFound("抑制记录")); + }, + { tag: TAG }, + ); + mount( + app, + Api.preferenceSuppressions.create, + async (c, { body }) => { + const r = await db() + .insertInto("preference_suppression") + .values({ + preference_tag_id: body.preferenceTagId ?? null, + fingerprint: body.fingerprint, + reason: body.reason ?? null, + evidence: jsonb(body.evidence), + suppressed_by: actorOf(c), + }) + .returningAll() + .executeTakeFirstOrThrow(); + return toApi(r); + }, + { tag: TAG }, + ); + mount( + app, + Api.preferenceSuppressions.update, + async (c, { params, body }) => { + const r = await db() + .updateTable("preference_suppression") + .set({ + active: false, + lifted_at: new Date(), + lifted_by: actorOf(c), + ...(body.reason ? { reason: body.reason } : {}), + }) + .where("id", "=", params.id) + .returningAll() + .executeTakeFirst(); + return toApi(r ?? notFound("抑制记录")); + }, + { tag: TAG }, + ); +} + +async function suppress(trx: Transaction, tagId: string, fingerprint: string, by: string, reason: string) { + await trx + .insertInto("preference_suppression") + .values({ preference_tag_id: tagId, fingerprint, reason, evidence: jsonb({ tagId }), suppressed_by: by }) + .execute(); +} + +/** 发布:旧 published → superseded/rolled_back;吊销旧授权;新版本生成持续授权 automation_grant */ +async function publishRevision( + trx: Transaction, + id: string, + policyKey: string, + by: string, + oldStatus: "superseded" | "rolled_back", +) { + const old = await trx + .selectFrom("policy_revision") + .select(["id"]) + .where("policy_key", "=", policyKey) + .where("status", "=", "published") + .execute(); + const now = new Date(); + for (const o of old as { id: string }[]) { + await trx.updateTable("policy_revision").set({ status: oldStatus }).where("id", "=", o.id).execute(); + await trx + .updateTable("automation_grant") + .set({ status: "revoked", revoked_at: now, revoked_by: by }) + .where("policy_revision_id", "=", o.id) + .where("status", "=", "active") + .execute(); + } + const row = await trx + .updateTable("policy_revision") + .set({ status: "published", published_at: now, published_by: by }) + .where("id", "=", id) + .returningAll() + .executeTakeFirstOrThrow(); + const rules = (row.rules ?? {}) as { budget?: unknown }; + await trx + .insertInto("automation_grant") + .values({ + action: `policy.${row.kind}`, + policy_revision_id: row.id, + scope: jsonb(row.scope), + budget: jsonb(rules.budget ?? {}), + granted_by: by, + reason: `发布策略 ${policyKey} r${row.revision}`, + }) + .execute(); + return row; +} + +/** 预览:用规则里的 Slot 模板评估最近已识别的发布(确定性,不产生副作用) */ +async function previewPolicy(d: AoDeps, rules: unknown) { + const r = AcquisitionPolicyRules.safeParse(rules); + const templates = r.success ? [...(r.data.defaultSlots ?? []), ...(r.data.newTitle?.slots ?? [])] : []; + const rels = await d.ctx.db + .selectFrom("source_release") + .select(["id", "title", "subtitle", "size_bytes"]) + .where("resolution_status", "=", "resolved") + .orderBy("first_seen_at", "desc") + .limit(200) + .execute(); + let matched = 0; + let excluded = 0; + const samples: { sourceReleaseId: string; template: string; decision: string }[] = []; + for (const rel of rels) { + const parsed = { + ...parseRelease(rel.title, rel.subtitle ? { subtitle: rel.subtitle } : {}), + ...(rel.size_bytes !== null ? { sizeBytes: rel.size_bytes } : {}), + }; + for (const t of templates) { + const spec = slotSpecFromRow({ + id: "preview", + name: t.name, + hardConditions: t.hardConditions ?? {}, + softPreferences: t.softPreferences ?? {}, + upgradeThreshold: t.upgradeThreshold ?? {}, + completion: t.completion ?? {}, + fallback: t.fallback ?? {}, + }); + const ev = evaluateCandidate(spec, parsed, null); + if (ev.decision === "accept") matched++; + else excluded++; + if (samples.length < 20) samples.push({ sourceReleaseId: rel.id, template: t.name, decision: ev.decision }); + } + } + return { + evaluatedAt: new Date().toISOString(), + sampleSize: rels.length, + templates: templates.length, + matched, + excluded, + samples, + valid: r.success, + }; +} diff --git a/apps/server/src/modules/acquisition-organize/services/acquisition.ts b/apps/server/src/modules/acquisition-organize/services/acquisition.ts new file mode 100644 index 00000000..3c68be9b --- /dev/null +++ b/apps/server/src/modules/acquisition-organize/services/acquisition.ts @@ -0,0 +1,655 @@ +import type { + AcquisitionPolicyRules, + AcquisitionPriority, + AcquisitionSlot, + CandidateEvaluation, + SlotTemplate, +} from "@umt/contracts"; +import type { AcquisitionSlotRow, MediaEntityRow, SourceReleaseRow, SubscriptionRow } from "@umt/db"; +import { sql } from "@umt/db"; +import { + type AcquisitionMode, + checkBudget, + determineMode, + evaluateCandidate, + type ParsedRelease, + parseRelease, + priorityOf, + type QualityFacts, + type SlotEvaluation, + type SlotState, + transition, +} from "@umt/domain"; +import { fromParsedColumns, slotSpecFromRow, toContractSlotEvaluation } from "@umt/mapping"; +import { hashJson, jsonb, startOfDay, toApi } from "../util.ts"; +import { type AoDeps, automationBlock, type Exec, enqueueTracked, loadInstances, recordCandidate } from "./common.ts"; +import { decideVolume, loadVolumes } from "./volumes.ts"; + +// evaluateAcquisition:release.resolved → 匹配订阅/Slot → 模式 → 确定性评估 → 预算/缓存卷 → 冻结计划并预留。 +// 拒绝与被挡住的原因都写成候选评估记录(artifact),供作品详情"为什么选/不选"。 + +export interface EvaluateInput { + sourceReleaseId?: string | undefined; + subscriptionId?: string | undefined; + acquisitionSlotId?: string | undefined; + /** release.resolved 带来的内容单元 */ + contentUnitIds?: string[] | undefined; +} + +export interface EvaluateOutcome { + evaluated: number; + plans: string[]; + skipped: { sourceReleaseId: string; reason: string }[]; +} + +const PRIORITY: Record, AcquisitionPriority> = { + immediate: "immediate", + catch_up: "tracking", + new_title: "new_work", + upgrade: "upgrade", +}; + +const BUSY: readonly string[] = ["reserved", "downloading", "verifying"]; + +/** source_release → ParsedRelease(解析版本过期时按原标题重解析;站点体积覆盖解析体积) */ +export function parsedOf(r: SourceReleaseRow): ParsedRelease { + const p = + fromParsedColumns({ parsed: (r.parsed ?? {}) as Record, parserVersion: r.parser_version }) ?? + parseRelease(r.title, r.subtitle ? { subtitle: r.subtitle } : {}); + return r.size_bytes !== null ? { ...p, sizeBytes: r.size_bytes } : p; +} + +/** 当前资产的交付特征:入库时记在 media_info._umt.quality */ +export async function currentFacts(exec: Exec, assetId: string | null): Promise { + if (!assetId) return null; + const a = await exec + .selectFrom("media_asset") + .select(["media_info", "delivery", "size_bytes"]) + .where("id", "=", assetId) + .executeTakeFirst(); + if (!a) return null; + const q = ((a.media_info as { _umt?: { quality?: QualityFacts } })?._umt?.quality ?? {}) as QualityFacts; + return { ...q, ...(a.size_bytes !== null ? { sizeBytes: a.size_bytes } : {}) }; +} + +/** 单元键:有 content_unit 用其 id,没有时用 s{季}e{集} 合成键(追更判定两侧同口径) */ +export function unitKeysOf(p: Pick): string[] { + if (p.airDate) return [`d${p.airDate}`]; + if (!p.episodes) return []; + const s = p.season ?? 1; + const out: string[] = []; + for (let e = p.episodes.start; e <= p.episodes.end && out.length < 500; e++) out.push(`s${s}e${e}`); + return out; +} + +async function releaseUnits( + exec: Exec, + entityId: string, + parsed: ParsedRelease, + fromEvent: readonly string[], +): Promise { + const keys = unitKeysOf(parsed); + const ids = new Set(fromEvent); + if (parsed.episodes) { + const rows = await exec + .selectFrom("content_unit") + .select(["id", "season_number", "episode_number"]) + .where("media_entity_id", "=", entityId) + .where("kind", "=", "episode") + .where("season_number", "=", parsed.season ?? 1) + .where("episode_number", ">=", parsed.episodes.start) + .where("episode_number", "<=", parsed.episodes.end) + .execute(); + for (const r of rows) ids.add(r.id); + } + return [...ids, ...keys]; +} + +/** Slot 已满足的单元:该 Slot 的计划带来的资产 + 当前资产的覆盖(content_unit id 与合成键) */ +async function satisfiedUnits(exec: Exec, slot: AcquisitionSlotRow): Promise { + const assets = await sql<{ id: string; media_info: unknown }>` + select distinct a.id, a.media_info from media_asset a + left join download_task dt on dt.id = a.download_task_id + left join acquisition_plan p on p.id = dt.acquisition_plan_id + where a.status = 'active' and a.parent_asset_id is null + and (p.slot_id = ${slot.id} or a.id = ${slot.current_asset_id})`.execute(exec); + if (assets.rows.length === 0) return []; + const ids = assets.rows.map((a) => a.id); + const cov = await exec + .selectFrom("asset_unit_coverage") + .select("content_unit_id") + .where("media_asset_id", "in", ids) + .execute(); + const keys = assets.rows.flatMap((a) => (a.media_info as { _umt?: { unitKeys?: string[] } })?._umt?.unitKeys ?? []); + return [...cov.map((c) => c.content_unit_id), ...keys]; +} + +async function resolveEntity(exec: Exec, id: string): Promise { + let cur = await exec.selectFrom("media_entity").selectAll().where("id", "=", id).executeTakeFirst(); + for (let i = 0; cur?.merged_into_id && i < 10; i++) { + cur = await exec.selectFrom("media_entity").selectAll().where("id", "=", cur.merged_into_id).executeTakeFirst(); + } + return cur ?? null; +} + +const firstReleaseDate = (e: MediaEntityRow): string | undefined => { + const m = (e.metadata ?? {}) as Record; + for (const k of ["releaseDate", "firstAirDate", "release_date", "first_air_date"]) { + const v = m[k]; + const s = + typeof v === "string" + ? v + : typeof (v as { value?: unknown })?.value === "string" + ? (v as { value: string }).value + : undefined; + if (s && /^\d{4}-\d{2}-\d{2}/.test(s)) return s.slice(0, 10); + } + return e.year ? `${e.year}-01-01` : undefined; +}; + +/** 已发布的收录策略(kind=acquisition) */ +export async function publishedAcquisitionPolicies(exec: Exec) { + const rows = await exec + .selectFrom("policy_revision") + .selectAll() + .where("kind", "=", "acquisition") + .where("status", "=", "published") + .execute(); + return rows.map((r) => ({ + row: r, + rules: (r.rules ?? {}) as AcquisitionPolicyRules, + scope: (r.scope ?? {}) as { mediaKinds?: string[] }, + })); +} + +const subKindOf = (k: MediaEntityRow["kind"]): "movie" | "series" | "variety" => + k === "movie" ? "movie" : k === "variety" ? "variety" : "series"; + +export async function createSlotsFromTemplates( + exec: Exec, + input: { + entityId: string; + subscriptionId: string | null; + policyRevisionId: string | null; + templates: readonly SlotTemplate[]; + }, +): Promise { + for (const t of input.templates) { + await exec + .insertInto("acquisition_slot") + .values({ + media_entity_id: input.entityId, + subscription_id: input.subscriptionId, + policy_revision_id: input.policyRevisionId, + name: t.name, + hard_conditions: jsonb(t.hardConditions), + soft_preferences: jsonb(t.softPreferences), + fallback: jsonb(t.fallback), + upgrade_threshold: jsonb(t.upgradeThreshold), + completion: jsonb(t.completion), + }) + .onConflict((oc) => oc.doNothing()) + .execute(); + } +} + +async function budgetUsage(exec: Exec, now: Date) { + const day = startOfDay(now); + const r = await sql<{ new_titles: number; bytes: number; reserved: number; active: number; reserved_count: number }>` + select + (select count(*) from acquisition_plan where created_at >= ${day} and status not in ('cancelled','failed','expired') + and evaluation->>'mode' = 'new_title')::int as new_titles, + (select coalesce(sum(reserved_bytes),0) from acquisition_plan where created_at >= ${day} and status in ('submitted','completed'))::bigint as bytes, + (select coalesce(sum(reserved_bytes),0) from acquisition_plan where created_at >= ${day} and status = 'reserved')::bigint as reserved, + (select count(*) from download_task where managed and removed_at is null and status in ('queued','submitting','downloading','stalled'))::int as active, + (select count(*) from acquisition_plan where status = 'reserved')::int as reserved_count`.execute(exec); + const row = r.rows[0]!; + return { + newTitlesToday: row.new_titles, + bytesToday: Number(row.bytes), + reservedBytesToday: Number(row.reserved), + activeDownloads: row.active + row.reserved_count, + }; +} + +interface Ctx { + d: AoDeps; + now: Date; + out: EvaluateOutcome; +} + +async function note(c: Ctx, base: Omit) { + await recordCandidate(c.d.ctx.db, { ...base, evaluatedAt: c.now.toISOString() }); +} + +export async function evaluateAcquisition(d: AoDeps, input: EvaluateInput): Promise { + const c: Ctx = { d, now: d.ports.now(), out: { evaluated: 0, plans: [], skipped: [] } }; + const db = d.ctx.db; + if (input.sourceReleaseId) { + const rel = await db + .selectFrom("source_release") + .selectAll() + .where("id", "=", input.sourceReleaseId) + .executeTakeFirst(); + if (!rel) { + c.out.skipped.push({ sourceReleaseId: input.sourceReleaseId, reason: "release_not_found" }); + return c.out; + } + await evaluateRelease(c, rel, input.contentUnitIds ?? [], input.acquisitionSlotId); + return c.out; + } + // 按 Slot / 订阅重评:取该作品最近已识别的发布 + let entityId: string | null = null; + if (input.acquisitionSlotId) { + entityId = + ( + await db + .selectFrom("acquisition_slot") + .select("media_entity_id") + .where("id", "=", input.acquisitionSlotId) + .executeTakeFirst() + )?.media_entity_id ?? null; + } else if (input.subscriptionId) { + entityId = + ( + await db + .selectFrom("subscription") + .select("media_entity_id") + .where("id", "=", input.subscriptionId) + .executeTakeFirst() + )?.media_entity_id ?? null; + } + if (!entityId) return c.out; + const rels = await db + .selectFrom("source_release") + .selectAll() + .where("media_entity_id", "=", entityId) + .where("resolution_status", "=", "resolved") + .where("availability", "=", "active") + .orderBy("first_seen_at", "desc") + .limit(50) + .execute(); + for (const r of rels) await evaluateRelease(c, r, [], input.acquisitionSlotId); + return c.out; +} + +async function evaluateRelease(c: Ctx, rel: SourceReleaseRow, eventUnits: readonly string[], onlySlotId?: string) { + const { d, now } = c; + const db = d.ctx.db; + const skip = (reason: string) => c.out.skipped.push({ sourceReleaseId: rel.id, reason }); + if (!rel.media_entity_id || rel.resolution_status !== "resolved") return skip("release_unresolved"); + if (rel.availability !== "active") return skip(`release_${rel.availability}`); + const entity = await resolveEntity(db, rel.media_entity_id); + if (!entity || entity.status === "archived") return skip("entity_unavailable"); + const parsed = parsedOf(rel); + const units = await releaseUnits( + db, + entity.id, + parsed, + rel.content_unit_id ? [...eventUnits, rel.content_unit_id] : eventUnits, + ); + const isMovie = entity.kind === "movie"; + + let subs = await db.selectFrom("subscription").selectAll().where("media_entity_id", "=", entity.id).execute(); + let slots = await db + .selectFrom("acquisition_slot") + .selectAll() + .where("media_entity_id", "=", entity.id) + .orderBy("created_at") + .execute(); + let forcedMode: AcquisitionMode | null = null; + + // 新作品收录:库里没有(无订阅、无 Slot、无资产)且策略开启 + if (subs.length === 0 && slots.length === 0) { + const hasAsset = await db + .selectFrom("media_asset") + .select("id") + .where("media_entity_id", "=", entity.id) + .where("status", "=", "active") + .executeTakeFirst(); + if (hasAsset) return skip("not_tracked"); + const pol = (await publishedAcquisitionPolicies(db)).find( + (p) => + p.rules.newTitle?.enabled && + (!p.rules.newTitle.mediaKinds?.length || p.rules.newTitle.mediaKinds.includes(subKindOf(entity.kind))), + ); + if (!pol?.rules.newTitle) return skip("not_tracked"); + const m = determineMode({ + entity: null, + releaseUnits: units, + satisfiedUnits: [], + slotSatisfied: false, + ...(firstReleaseDate(entity) ? { firstReleaseDate: firstReleaseDate(entity)! } : {}), + now: now.toISOString(), + newTitleWindowDays: pol.rules.newTitle.windowDays ?? d.ports.options.newTitleWindowDays, + }); + if (m.mode !== "new_title") return skip(`new_title:${m.reason}`); + const grant = await db + .selectFrom("automation_grant") + .select("id") + .where("policy_revision_id", "=", pol.row.id) + .where("status", "=", "active") + .executeTakeFirst(); + if (!grant) return skip("no_automation_grant"); + await db.transaction().execute(async (trx) => { + const sub = await trx + .insertInto("subscription") + .values({ + media_entity_id: entity.id, + media_kind: subKindOf(entity.kind), + mode: "once", + season_scope: "current_season", + origin: "policy", + priority: "new_work", + policy_revision_id: pol.row.id, + created_by: `policy:${pol.row.policy_key}`, + }) + .returning("id") + .executeTakeFirstOrThrow(); + await createSlotsFromTemplates(trx, { + entityId: entity.id, + subscriptionId: sub.id, + policyRevisionId: pol.row.id, + templates: pol.rules.newTitle!.slots, + }); + }); + subs = await db.selectFrom("subscription").selectAll().where("media_entity_id", "=", entity.id).execute(); + slots = await db + .selectFrom("acquisition_slot") + .selectAll() + .where("media_entity_id", "=", entity.id) + .orderBy("created_at") + .execute(); + forcedMode = "new_title"; + } + + // 订阅停止条件 + for (const s of subs) { + const stopAfter = (s.params as { stopAfter?: string } | null)?.stopAfter; + if (s.status === "active" && stopAfter && Date.parse(stopAfter) < now.getTime()) { + await db + .updateTable("subscription") + .set({ status: "completed", status_reason: "stop_after_reached" }) + .where("id", "=", s.id) + .execute(); + s.status = "completed"; + } + } + const activeSubs = new Map(subs.filter((s) => s.status === "active").map((s) => [s.id, s])); + const tracked = activeSubs.size > 0 || slots.some((s) => s.subscription_id === null); + const releaseSeason = parsed.season ?? (parsed.episodes ? 1 : undefined); + + let planned: string | null = null; + for (const slot of slots) { + if (onlySlotId && slot.id !== onlySlotId) continue; + const sub = slot.subscription_id ? activeSubs.get(slot.subscription_id) : undefined; + if (slot.subscription_id && !sub) continue; // 订阅已暂停/完成/取消 + if (slot.content_unit_id && !units.includes(slot.content_unit_id)) { + // Slot 限定某季:季单元是否是本发布单元的父级 + const seasonOk = await db + .selectFrom("content_unit") + .select("id") + .where("parent_unit_id", "=", slot.content_unit_id) + .where( + "id", + "in", + units.filter((u) => /^[0-9a-f-]{36}$/.test(u)).length + ? units.filter((u) => /^[0-9a-f-]{36}$/.test(u)) + : ["00000000-0000-0000-0000-000000000000"], + ) + .executeTakeFirst(); + if (!seasonOk) continue; + } + // 仅本季:发布季号与订阅季不一致则跳过 + if (sub?.season_scope === "current_season" && sub.season_unit_id && releaseSeason !== undefined) { + const su = await db + .selectFrom("content_unit") + .select("season_number") + .where("id", "=", sub.season_unit_id) + .executeTakeFirst(); + if (su?.season_number !== null && su?.season_number !== undefined && su.season_number !== releaseSeason) continue; + } + c.out.evaluated++; + const base = { slotId: slot.id, sourceReleaseId: rel.id, title: rel.title, planId: null as string | null }; + if (BUSY.includes(slot.status)) { + await note(c, { ...base, decision: "skipped", mode: null, rejected: [], reasons: [`slot_busy:${slot.status}`] }); + continue; + } + const spec = slotSpecFromRow(toApi(slot)); + const satisfied = await satisfiedUnits(db, slot); + const slotSatisfied = slot.status === "satisfied" || slot.status === "upgrade_pending"; + const gaps = units.filter((u) => !satisfied.includes(u)); + const current = await currentFacts(db, slot.current_asset_id); + const fill = isMovie ? !slotSatisfied : gaps.length > 0 || !current; + const ev: SlotEvaluation = evaluateCandidate(spec, parsed, fill ? null : current); + const flat = toContractSlotEvaluation(ev); + if (!flat.acceptable) { + await note(c, { + ...base, + decision: ev.decision as "reject" | "unknown_field", + mode: null, + rejected: flat.rejected, + reasons: flat.reasons, + }); + continue; + } + const mode: AcquisitionMode | null = + forcedMode ?? + determineMode({ + entity: { tracked, kind: isMovie ? "movie" : "tv" }, + releaseUnits: units, + satisfiedUnits: satisfied, + slotSatisfied, + now: now.toISOString(), + newTitleWindowDays: d.ports.options.newTitleWindowDays, + upgradeCandidate: ev.decision === "upgrade", + }).mode; + if (!mode) { + await note(c, { + ...base, + decision: "skipped", + mode: null, + rejected: [], + reasons: ["mode:none", ...flat.reasons], + }); + continue; + } + if (planned) { + await note(c, { ...base, decision: "skipped", mode, rejected: [], reasons: [`covered_by_plan:${planned}`] }); + continue; + } + const gate = await gates(c, rel, mode, sub); + if (!gate.ok) { + await note(c, { ...base, decision: "skipped", mode, rejected: gate.rejected, reasons: gate.reasons }); + skip(gate.reasons[0] ?? "gated"); + continue; + } + const planId = await reserve(c, { + slot, + rel, + mode, + ev, + sub, + instanceId: gate.instanceId, + volumeKey: gate.volumeKey, + savePath: gate.savePath, + }); + if (!planId) { + await note(c, { ...base, decision: "skipped", mode, rejected: [], reasons: ["slot_changed_or_reserved"] }); + continue; + } + planned = planId; + c.out.plans.push(planId); + await note(c, { + ...base, + planId, + decision: ev.decision as "accept" | "upgrade", + mode, + rejected: [], + reasons: flat.reasons, + }); + await enqueueTracked(d, { + kind: "submitDownload", + idempotencyKey: `plan:${planId}`, + input: { acquisitionPlanId: planId }, + subjectType: "acquisition_plan", + subjectId: planId, + }).then( + (taskId) => db.updateTable("acquisition_plan").set({ task_id: taskId }).where("id", "=", planId).execute(), + (err) => + d.ctx.logger.warn({ err: { message: (err as Error).message }, planId }, "submitDownload 入队失败,留给巡检补投"), + ); + } +} + +interface GateOk { + ok: true; + instanceId: string; + volumeKey: string; + savePath: string; +} +interface GateNo { + ok: false; + reasons: string[]; + rejected: { code: string; message: string }[]; +} + +/** 自动化总开关 → 下载器 → 预算 → 缓存卷阈值;任一不过都不建计划 */ +async function gates( + c: Ctx, + rel: SourceReleaseRow, + mode: AcquisitionMode, + sub: SubscriptionRow | undefined, +): Promise { + const { d } = c; + const blocked = await automationBlock(d); + if (blocked) return { ok: false, reasons: [blocked], rejected: [] }; + const wanted = (sub?.params as { downloaderInstanceId?: string } | null)?.downloaderInstanceId; + const insts = (await loadInstances(d.ctx.db)).filter((t) => t.savePath && (!wanted || t.inst.id === wanted)); + const inst = insts[0]; + if (!inst?.savePath) return { ok: false, reasons: ["no_downloader_instance"], rejected: [] }; + + const budget = await d.ctx.settings.get("budget"); + const b = checkBudget( + { + maxNewTitlesPerDay: budget.maxNewTitlesPerDay, + maxBytesPerDay: budget.maxDownloadBytesPerDay, + maxConcurrent: budget.maxConcurrentDownloads, + }, + await budgetUsage(d.ctx.db, c.now), + { countsAsNewTitle: mode === "new_title", ...(rel.size_bytes !== null ? { bytes: rel.size_bytes } : {}) }, + ); + if (b.decision !== "allow") { + return { + ok: false, + reasons: [b.decision === "unknown" ? "budget_unknown_size" : "budget_exceeded"], + rejected: b.violations.map((v) => ({ + code: `budget.${v.kind}`, + message: `${v.kind}: 上限 ${v.limit},当前 ${v.current},请求 ${v.requested}`, + })), + }; + } + const cache = await d.ctx.settings.get("cache"); + const vol = (await loadVolumes(d, d.ctx.db, await loadInstances(d.ctx.db))).find( + (v) => v.volumeKey === inst.volumeKey, + ); + if (!vol) return { ok: false, reasons: ["volume_state_unknown"], rejected: [] }; + const dv = decideVolume(vol, cache, rel.size_bytes !== null ? { bytes: rel.size_bytes } : {}); + if (dv.decision !== "allow") { + return { + ok: false, + reasons: dv.reasons.length ? dv.reasons.map((r) => `cache:${r}`) : ["cache:blocked"], + rejected: [ + { + code: `cache.${dv.decision}`, + message: `缓存卷 ${vol.volumeKey} 使用率 ${dv.usedPct?.toFixed(1) ?? "未知"}%,预测 ${dv.projectedPct?.toFixed(1) ?? "未知"}%`, + }, + ], + }; + } + return { ok: true, instanceId: inst.inst.id, volumeKey: inst.volumeKey, savePath: inst.savePath }; +} + +async function reserve( + c: Ctx, + p: { + slot: AcquisitionSlotRow; + rel: SourceReleaseRow; + mode: AcquisitionMode; + ev: SlotEvaluation; + sub: SubscriptionRow | undefined; + instanceId: string; + volumeKey: string; + savePath: string; + }, +): Promise { + const { d, now } = c; + const immediate = p.sub?.priority === "immediate"; + const priority = PRIORITY[priorityOf(p.mode, immediate)]; + const payload = { + slotId: p.slot.id, + sourceReleaseId: p.rel.id, + instanceId: p.instanceId, + savePath: p.savePath, + mode: p.mode, + }; + try { + return await d.ctx.db.transaction().execute(async (trx) => { + const slot = await trx + .selectFrom("acquisition_slot") + .selectAll() + .where("id", "=", p.slot.id) + .forUpdate() + .executeTakeFirst(); + if (!slot || slot.revision !== p.slot.revision || BUSY.includes(slot.status)) return null; + let st: SlotState = { status: slot.status, hasAsset: slot.current_asset_id !== null }; + if (st.status === "satisfied") st = transition(st, { type: "upgrade_found" }); + st = transition(st, { type: "reserve" }); + const plan = await trx + .insertInto("acquisition_plan") + .values({ + slot_id: slot.id, + source_release_id: p.rel.id, + purpose: p.mode === "upgrade" ? "upgrade" : "fill", + status: "reserved", + priority, + evaluation: jsonb({ + ...toContractSlotEvaluation(p.ev), + mode: p.mode, + instanceId: p.instanceId, + volumeKey: p.volumeKey, + savePath: p.savePath, + }), + reserved_bytes: p.rel.size_bytes ?? 0, + budget_reservation: jsonb({ + bytes: p.rel.size_bytes, + countsAsNewTitle: p.mode === "new_title", + day: startOfDay(now).toISOString().slice(0, 10), + }), + payload_hash: hashJson(payload), + expected_revision: slot.revision + 1, + idempotency_key: `acq:${slot.id}:${p.rel.id}`, + }) + .onConflict((oc) => oc.column("idempotency_key").doNothing()) + .returning("id") + .executeTakeFirst(); + if (!plan) return null; + await trx + .updateTable("acquisition_slot") + .set({ + status: st.status, + reserved_plan_id: plan.id, + revision: slot.revision + 1, + last_evaluated_at: now, + status_reason: `reserved:${p.mode}`, + }) + .where("id", "=", slot.id) + .execute(); + return plan.id; + }); + } catch (err) { + // 部分唯一索引:同一 Slot 同时只能有一个有效预留 + if ((err as { code?: string }).code === "23505") return null; + throw err; + } +} diff --git a/apps/server/src/modules/acquisition-organize/services/common.ts b/apps/server/src/modules/acquisition-organize/services/common.ts new file mode 100644 index 00000000..3fd81bd7 --- /dev/null +++ b/apps/server/src/modules/acquisition-organize/services/common.ts @@ -0,0 +1,237 @@ +import type { CandidateEvaluation, DecisionKind, Severity, TaskKind } from "@umt/contracts"; +import { type DB, type DownloaderInstanceRow, type Kysely, type LibraryRootRow, sql, type Transaction } from "@umt/db"; +import { type PathMapping, toDownloaderPath, toHostPath } from "@umt/domain"; +import type { ServerContext } from "../../types.ts"; +import type { AoPorts } from "../ports.ts"; +import { hashJson, jsonb } from "../util.ts"; + +// 本模块各服务共用:决策(待处理)、候选评估记录、业务任务行、实例拓扑(路径映射/缓存卷)。 + +export type Exec = Kysely | Transaction; + +export interface AoDeps { + ctx: ServerContext; + ports: AoPorts; +} + +// ── 待处理中心 ── + +export interface DecisionInput { + kind: DecisionKind; + severity?: Severity; + title: string; + summary?: string; + subjectType?: string; + subjectId?: string | null; + /** 同一 dedupeKey 同时只有一个未关闭条目;重复写入只累加 attempts、刷新证据 */ + dedupeKey: string; + evidence?: Record; + options?: { key: string; label: string; description?: string }[]; + taskId?: string | null; +} + +export async function upsertDecision(d: AoDeps, exec: Exec, input: DecisionInput): Promise { + const r = await sql<{ id: string; inserted: boolean }>` + insert into decision (kind, severity, title, summary, subject_type, subject_id, dedupe_key, evidence, options, task_id) + values (${input.kind}, ${input.severity ?? "warning"}, ${input.title}, ${input.summary ?? null}, + ${input.subjectType ?? null}, ${input.subjectId ?? null}, ${input.dedupeKey}, + ${jsonb(input.evidence)}::jsonb, ${JSON.stringify(input.options ?? [])}::jsonb, ${input.taskId ?? null}) + on conflict (dedupe_key) where status in ('open', 'investigating', 'awaiting_user') + do update set attempts = decision.attempts + 1, evidence = excluded.evidence, summary = excluded.summary, + updated_at = now() + returning id, (xmax = 0) as inserted`.execute(exec); + const row = r.rows[0]!; + if (row.inserted) { + await d.ctx.outbox.emit( + exec, + "decision.created", + { decisionId: row.id, kind: input.kind, severity: input.severity ?? "warning", title: input.title }, + { idempotencyKey: `decision.created:${row.id}` }, + ); + } + return row.id; +} + +// ── 候选评估记录(artifact kind=evidence,subject=acquisition_slot) ── + +export const CANDIDATE_ARTIFACT_TITLE = "candidate_evaluation"; + +export async function recordCandidate(exec: Exec, ev: CandidateEvaluation): Promise { + const content = { ...ev }; + const contentHash = hashJson({ + slotId: ev.slotId, + sourceReleaseId: ev.sourceReleaseId, + decision: ev.decision, + rejected: ev.rejected, + reasons: ev.reasons, + planId: ev.planId, + }); + const dup = await exec + .selectFrom("artifact") + .select("id") + .where("subject_type", "=", "acquisition_slot") + .where("subject_id", "=", ev.slotId) + .where("content_hash", "=", contentHash) + .executeTakeFirst(); + if (dup) { + await exec + .updateTable("artifact") + .set({ content: jsonb(content), updated_at: new Date() }) + .where("id", "=", dup.id) + .execute(); + return; + } + await exec + .insertInto("artifact") + .values({ + kind: "evidence", + subject_type: "acquisition_slot", + subject_id: ev.slotId, + title: CANDIDATE_ARTIFACT_TITLE, + content: jsonb(content), + content_hash: contentHash, + }) + .execute(); +} + +// ── 业务任务行(task 表:时间线投影;引擎只管推进) ── + +export interface TaskRowInput { + kind: TaskKind; + idempotencyKey: string; + input: Record; + subjectType?: string; + subjectId?: string | null; + triggeredBy?: string; + runId?: string; +} + +/** 按 (kind, idempotency_key) 幂等建行,返回 id;给了 runId 时回填 engine_run_id */ +export async function ensureTaskRow(exec: Exec, t: TaskRowInput): Promise { + const r = await sql<{ id: string }>` + insert into task (kind, idempotency_key, input, subject_type, subject_id, triggered_by, engine_run_id) + values (${t.kind}, ${t.idempotencyKey}, ${jsonb(t.input)}::jsonb, ${t.subjectType ?? null}, ${t.subjectId ?? null}, + ${t.triggeredBy ?? "system"}, ${t.runId ?? null}) + on conflict (kind, idempotency_key) + do update set engine_run_id = coalesce(excluded.engine_run_id, task.engine_run_id), updated_at = now() + returning id`.execute(exec); + return r.rows[0]!.id; +} + +export async function markTask( + exec: Exec, + taskId: string, + status: "running" | "succeeded" | "failed" | "waiting", + extra: { output?: unknown; error?: string; waitReason?: string } = {}, +): Promise { + const now = new Date(); + await exec + .updateTable("task") + .set({ + status, + ...(status === "running" ? { started_at: now, attempts: sql`attempts + 1`, last_error: null } : {}), + ...(status === "succeeded" || status === "failed" ? { finished_at: now } : {}), + ...(extra.output !== undefined ? { output: jsonb(extra.output) } : {}), + ...(extra.error !== undefined ? { last_error: jsonb({ message: extra.error }) } : {}), + ...(extra.waitReason !== undefined ? { wait_reason: extra.waitReason } : {}), + }) + .where("id", "=", taskId) + .execute(); +} + +/** 建业务任务行 + 引擎入队(幂等键一致);返回 task.id */ +export async function enqueueTracked(d: AoDeps, t: Omit): Promise { + const taskId = await ensureTaskRow(d.ctx.db, t); + const { runId } = await d.ctx.engine.enqueue( + t.kind, + { ...t.input, taskId }, + { + idempotencyKey: `${t.kind}:${t.idempotencyKey}`, + traceId: taskId, + }, + ); + await d.ctx.db.updateTable("task").set({ engine_run_id: runId }).where("id", "=", taskId).execute(); + return taskId; +} + +// ── 实例拓扑:路径映射、保存路径、缓存卷 ── + +export interface InstanceTopology { + inst: DownloaderInstanceRow; + mappings: PathMapping[]; + /** 下载器视角的保存路径(settings.savePath) */ + savePath: string | null; + /** 本机视角的保存路径(用于 statfs / 文件访问) */ + localSavePath: string | null; + volumeKey: string; + category: string | null; +} + +export async function loadInstances( + exec: Exec, + opts: { enabledOnly?: boolean; ids?: string[] } = {}, +): Promise { + let q = exec.selectFrom("downloader_instance").selectAll(); + if (opts.enabledOnly !== false) q = q.where("enabled", "=", true); + if (opts.ids?.length) q = q.where("id", "in", opts.ids); + const insts = await q.orderBy("name").execute(); + if (insts.length === 0) return []; + const maps = await exec + .selectFrom("path_mapping") + .select(["downloader_instance_id", "remote_prefix", "local_prefix"]) + .where("view_kind", "=", "downloader") + .where("enabled", "=", true) + .where( + "downloader_instance_id", + "in", + insts.map((i) => i.id), + ) + .execute(); + return insts.map((inst) => { + const mappings = maps + .filter((m) => m.downloader_instance_id === inst.id) + .map((m) => ({ instanceId: inst.id, downloaderPath: m.remote_prefix, hostPath: m.local_prefix })); + const settings = (inst.settings ?? {}) as Record; + const savePath = + typeof settings.savePath === "string" && settings.savePath + ? settings.savePath + : (mappings[0]?.downloaderPath ?? null); + const topo: InstanceTopology = { + inst, + mappings, + savePath, + localSavePath: null, + volumeKey: inst.cache_volume_key ?? `instance:${inst.id}`, + category: typeof settings.category === "string" && settings.category ? settings.category : null, + }; + topo.localSavePath = savePath ? hostPathOf(topo, savePath) : null; + return topo; + }); +} + +/** 下载器路径 → 本机路径;实例没有配置任何映射时视为同机同路径,配置了但不匹配返回 null(不猜测) */ +export function hostPathOf(t: Pick, downloaderPath: string): string | null { + if (t.mappings.length === 0) return downloaderPath; + const r = toHostPath(downloaderPath, t.inst.id, t.mappings); + return r.ok ? r.path : null; +} + +export function downloaderPathOf(t: Pick, hostPath: string): string | null { + if (t.mappings.length === 0) return hostPath; + const r = toDownloaderPath(hostPath, t.inst.id, t.mappings); + return r.ok ? r.path : null; +} + +export async function loadLibraryRoots(exec: Exec, enabledOnly = false): Promise { + let q = exec.selectFrom("library_root").selectAll(); + if (enabledOnly) q = q.where("enabled", "=", true); + return q.orderBy("name").execute(); +} + +/** 自动化总开关 / 只读对账:返回阻止原因,null 表示可写 */ +export async function automationBlock(d: AoDeps): Promise { + const a = await d.ctx.settings.get("automation"); + if (!a.enabled) return "automation_disabled"; + if (a.readOnlyReconcile) return "read_only_reconcile"; + return null; +} diff --git a/apps/server/src/modules/acquisition-organize/services/downloads.ts b/apps/server/src/modules/acquisition-organize/services/downloads.ts new file mode 100644 index 00000000..eafbbbc8 --- /dev/null +++ b/apps/server/src/modules/acquisition-organize/services/downloads.ts @@ -0,0 +1,917 @@ +import { assertSafeToDeleteData, DownloaderError, type QbTorrent, SiteError, type TorrentSource } from "@umt/adapters"; +import type { DownloadStatus, PauseReason } from "@umt/contracts"; +import { type DB, type DownloadTaskRow, sql, type Transaction } from "@umt/db"; +import { mapQbState, type SlotEvent, transition } from "@umt/domain"; +import { mergeObservedDownloadState, toDomainDownloadStatus } from "@umt/mapping"; +import { withAttemptToken } from "@umt/tasks"; +import { errMessage, jsonb } from "../util.ts"; +import { + type AoDeps, + automationBlock, + downloaderPathOf, + type Exec, + hostPathOf, + type InstanceTopology, + loadInstances, + loadLibraryRoots, + upsertDecision, +} from "./common.ts"; +import { hnrSatisfied, retentionElapsed } from "./retention.ts"; +import { decideVolume, loadVolumes, usedPercent, type VolumeState } from "./volumes.ts"; + +// 下载:提交(幂等 + fencing)、qB 同步与标签接管、无进展证据、缓存卷硬阈值、缓存清理。 + +/** Slot 状态推进(非法迁移只记日志不抛:并发下 Slot 可能已被人工处理) */ +export async function applySlotEvent( + d: AoDeps, + trx: Exec, + slotId: string, + event: SlotEvent, + extra: { currentAssetId?: string | null; reason?: string } = {}, +): Promise { + const slot = await trx + .selectFrom("acquisition_slot") + .selectAll() + .where("id", "=", slotId) + .forUpdate() + .executeTakeFirst(); + if (!slot) return false; + try { + const next = transition({ status: slot.status, hasAsset: slot.current_asset_id !== null }, event); + await trx + .updateTable("acquisition_slot") + .set({ + status: next.status, + revision: slot.revision + 1, + ...(extra.currentAssetId !== undefined ? { current_asset_id: extra.currentAssetId } : {}), + ...(next.status !== "reserved" && next.status !== "downloading" && next.status !== "verifying" + ? { reserved_plan_id: null } + : {}), + status_reason: extra.reason ?? event.type, + }) + .where("id", "=", slotId) + .execute(); + return true; + } catch (err) { + d.ctx.logger.warn({ slotId, event: event.type, err: { message: errMessage(err) } }, "Slot 状态迁移被拒绝"); + return false; + } +} + +async function emitStatus( + d: AoDeps, + exec: Exec, + id: string, + from: DownloadStatus | null, + to: DownloadStatus, + pauseReason: PauseReason | null, +) { + await d.ctx.outbox.emit( + exec, + "download.status_changed", + { downloadTaskId: id, from, to, pauseReason }, + // 完成事件幂等:只触发一次入库;其它状态变化不去重 + to === "completed" ? { idempotencyKey: `download.completed:${id}` } : {}, + ); +} + +// ── 提交 ── + +type SubmitResult = + | { + kind: "added" | "already_exists" | "accepted_unconfirmed"; + hash: string; + name: string; + savePath: string | null; + contentPath: string | null; + tags: string[]; + } + | { kind: "rejected" | "source_failed"; reason: string }; + +export interface SubmitOutcome { + status: "submitted" | "deferred" | "failed" | "skipped"; + reason?: string; + downloadTaskId?: string; +} + +export async function submitDownload(d: AoDeps, planId: string, opts: { runId?: string } = {}): Promise { + const db = d.ctx.db; + const plan = await db.selectFrom("acquisition_plan").selectAll().where("id", "=", planId).executeTakeFirst(); + if (!plan) return { status: "skipped", reason: "plan_not_found" }; + if (plan.status !== "reserved") return { status: "skipped", reason: `plan_${plan.status}` }; + const blocked = await automationBlock(d); + if (blocked) return { status: "deferred", reason: blocked }; + const ev = (plan.evaluation ?? {}) as { instanceId?: string; volumeKey?: string }; + const topo = (await loadInstances(db)).find((t) => t.inst.id === ev.instanceId); + if (!topo?.savePath) return failPlan(d, plan.id, plan.slot_id, "downloader_unavailable"); + + // 阈值复核:计划自身已作为预留计入,这里不再追加请求量 + const cache = await d.ctx.settings.get("cache"); + const vol = (await loadVolumes(d, db, [topo]))[0]; + if (!vol) return { status: "deferred", reason: "volume_state_unknown" }; + const dv = decideVolume(vol, cache); + if (dv.decision !== "allow") return { status: "deferred", reason: `cache:${dv.reasons.join(",") || dv.decision}` }; + + const rel = await db + .selectFrom("source_release") + .selectAll() + .where("id", "=", plan.source_release_id) + .executeTakeFirstOrThrow(); + const site = await db.selectFrom("site").selectAll().where("id", "=", rel.site_id).executeTakeFirst(); + const client = await d.ports.downloader(topo.inst); + const tag = topo.inst.managed_tag; + const savePath = topo.savePath; + + const view = (t: QbTorrent) => ({ + hash: t.hash, + name: t.name, + savePath: t.savePath, + contentPath: t.contentPath, + tags: t.tags, + }); + let dtId: string | undefined; + const result = await withAttemptToken( + db, + `submit:${plan.id}`, + async (info) => { + const known = rel.infohash_v1 ?? rel.infohash_v2; + if (info.resumed && known) { + // 上次 attempt 没留下结果:先按 hash 对账,已在 qB 里就不重提 + const existing = await client.findByHash(known); + if (existing) return { kind: "added", ...view(existing) }; + } + let source: TorrentSource; + try { + source = await d.ports.torrentSource({ release: rel, site: site ?? null }); + } catch (err) { + if (err instanceof SiteError && (err.isAuthError || err.kind === "not_found" || err.kind === "unsupported")) { + return { kind: "source_failed", reason: `${err.kind}: ${err.message}` }; + } + throw err; + } + const out = await client.submit({ + source, + savePath, + tags: [tag], + ...(topo.category ? { category: topo.category } : {}), + }); + switch (out.kind) { + case "added": { + const t = await client.findByHash(out.hash).catch(() => null); + return t + ? { kind: "added", ...view(t) } + : { kind: "added", hash: out.hash, name: rel.title, savePath, contentPath: null, tags: [tag] }; + } + case "already_exists": { + // 原本已存在:接管,只补管理标签,不改保存路径/其它标签/文件选择 + if (!out.existing.tags.includes(tag)) await client.addTags([out.hash], [tag]); + return { kind: "already_exists", ...view(out.existing), tags: [...new Set([...out.existing.tags, tag])] }; + } + case "accepted_unconfirmed": + if (!out.hash) throw new Error("qB 已收单但未能确认任务(无 hash),稍后按探针标签对账"); + return { + kind: "accepted_unconfirmed", + hash: out.hash, + name: rel.title, + savePath, + contentPath: null, + tags: [tag], + }; + case "rejected": + return { kind: "rejected", reason: out.reason }; + case "unreachable": + throw out.error; + } + }, + { + ...(opts.runId ? { runId: opts.runId } : {}), + commit: async (trx, res) => { + if (res.kind === "rejected" || res.kind === "source_failed") { + const r = res; + await failPlanTx(d, trx, plan.id, plan.slot_id, `${r.kind}: ${r.reason}`); + return; + } + const r = res as Extract; + const ins = await sql<{ id: string; inserted: boolean }>` + insert into download_task (instance_id, torrent_hash, infohash_v1, infohash_v2, acquisition_plan_id, source_release_id, name, + save_path, content_path, category, tags, managed, origin, status, size_bytes, next_check_at) + values (${topo.inst.id}, ${r.hash}, ${r.hash.length === 40 ? r.hash : null}, ${r.hash.length === 64 ? r.hash : null}, + ${plan.id}, ${rel.id}, ${r.name}, ${r.savePath}, ${r.contentPath}, ${topo.category}, + ${r.tags}, true, ${r.kind === "already_exists" ? "adopted" : "submitted"}, + ${r.kind === "accepted_unconfirmed" ? "submitting" : "queued"}, ${rel.size_bytes}, now()) + on conflict (instance_id, torrent_hash) do update + set acquisition_plan_id = coalesce(download_task.acquisition_plan_id, excluded.acquisition_plan_id), + source_release_id = coalesce(download_task.source_release_id, excluded.source_release_id), + managed = true, tags = excluded.tags, updated_at = now() + returning id, (xmax = 0) as inserted`.execute(trx); + dtId = ins.rows[0]!.id; + await trx + .updateTable("acquisition_plan") + .set({ status: "submitted", status_reason: r.kind }) + .where("id", "=", plan.id) + .execute(); + await applySlotEvent(d, trx, plan.slot_id, { type: "download_started" }, { reason: `download:${r.kind}` }); + await d.ctx.outbox.emit( + trx, + "download.submitted", + { downloadTaskId: dtId, instanceId: topo.inst.id, torrentHash: r.hash, planId: plan.id, name: r.name }, + { idempotencyKey: `download.submitted:${plan.id}` }, + ); + }, + }, + ); + if (result.kind === "rejected" || result.kind === "source_failed") return { status: "failed", reason: result.reason }; + const hash = (result as Extract).hash; + if (!dtId) { + // 账本已有结果(重放):按 hash 找回任务行 + dtId = ( + await db + .selectFrom("download_task") + .select("id") + .where("instance_id", "=", topo.inst.id) + .where("torrent_hash", "=", hash) + .executeTakeFirst() + )?.id; + } + return { status: "submitted", reason: result.kind, ...(dtId ? { downloadTaskId: dtId } : {}) }; +} + +async function failPlanTx(d: AoDeps, trx: Exec, planId: string, slotId: string, reason: string) { + await trx + .updateTable("acquisition_plan") + .set({ status: "failed", status_reason: reason.slice(0, 500) }) + .where("id", "=", planId) + .execute(); + await applySlotEvent(d, trx, slotId, { type: "release" }, { reason: `submit_failed` }); + await upsertDecision(d, trx, { + kind: "download_failed", + title: "下载提交失败", + summary: reason.slice(0, 500), + subjectType: "acquisition_plan", + subjectId: planId, + dedupeKey: `submit_failed:${planId}`, + evidence: { planId, reason }, + }); +} + +async function failPlan(d: AoDeps, planId: string, slotId: string, reason: string): Promise { + await d.ctx.db.transaction().execute((trx) => failPlanTx(d, trx, planId, slotId, reason)); + return { status: "failed", reason }; +} + +/** 巡检:reserved 但没有下载任务的计划(入队失败、被阈值/总开关延后)→ 条件满足时直接补提交 */ +export async function sweepReservedPlans(d: AoDeps): Promise { + if (await automationBlock(d)) return 0; + const stale = await d.ctx.db + .selectFrom("acquisition_plan as p") + .leftJoin("download_task as dt", "dt.acquisition_plan_id", "p.id") + .select("p.id") + .where("p.status", "=", "reserved") + .where("dt.id", "is", null) + .where("p.updated_at", "<", new Date(d.ports.now().getTime() - 60_000)) + .limit(20) + .execute(); + let n = 0; + for (const p of stale) { + const r = await submitDownload(d, p.id).catch((err) => { + d.ctx.logger.warn({ planId: p.id, err: { message: errMessage(err) } }, "补提交失败"); + return null; + }); + if (r?.status === "submitted") n++; + } + return n; +} + +// ── 同步 ── + +export interface SyncStats { + instances: number; + unreachable: string[]; + updated: number; + adopted: number; + completed: string[]; + removed: number; + thresholdPaused: number; + thresholdResumed: number; +} + +export async function syncDownloads(d: AoDeps): Promise { + const stats: SyncStats = { + instances: 0, + unreachable: [], + updated: 0, + adopted: 0, + completed: [], + removed: 0, + thresholdPaused: 0, + thresholdResumed: 0, + }; + const insts = await loadInstances(d.ctx.db); + for (const topo of insts) { + stats.instances++; + await syncInstance(d, topo, stats); + } + await enforceThresholds(d, insts, stats); + return stats; +} + +async function syncInstance(d: AoDeps, topo: InstanceTopology, stats: SyncStats) { + const db = d.ctx.db; + const now = d.ports.now(); + const client = await d.ports.downloader(topo.inst); + let list: QbTorrent[]; + try { + list = await client.list({ tag: topo.inst.managed_tag }); + } catch (err) { + // 下载器失联:不改任何任务状态(失败 ≠ 不存在) + stats.unreachable.push(topo.inst.id); + await db + .updateTable("downloader_instance") + .set({ health_status: "offline" }) + .where("id", "=", topo.inst.id) + .execute(); + d.ctx.logger.warn({ instanceId: topo.inst.id, err: { message: errMessage(err) } }, "qB 同步失败"); + return; + } + await db + .updateTable("downloader_instance") + .set({ health_status: "healthy", last_seen_at: now }) + .where("id", "=", topo.inst.id) + .execute(); + const rows = await db + .selectFrom("download_task") + .selectAll() + .where("instance_id", "=", topo.inst.id) + .where("removed_at", "is", null) + .where("status", "!=", "removed") + .execute(); + const byHash = new Map(rows.map((r) => [r.torrent_hash, r])); + const seen = new Set(); + for (const t of list) { + seen.add(t.hash); + const row = byHash.get(t.hash); + if (row) { + if (await observe(d, row, t, now, stats)) stats.updated++; + } else { + await adopt(d, topo, t, now, stats); + } + } + for (const row of rows) { + if (seen.has(row.torrent_hash) || !row.managed) continue; + let still: QbTorrent | null; + try { + still = await client.findByHash(row.torrent_hash); + } catch { + continue; + } + await db.transaction().execute(async (trx) => { + if (still) { + // 管理标签被移除:停止后续自动动作,保留历史与资产 + await trx + .updateTable("download_task") + .set({ managed: false, tags: still.tags, last_synced_at: now }) + .where("id", "=", row.id) + .execute(); + } else { + await trx + .updateTable("download_task") + .set({ status: "removed", removed_at: now, last_synced_at: now }) + .where("id", "=", row.id) + .execute(); + await emitStatus(d, trx, row.id, row.status, "removed", null); + stats.removed++; + } + }); + } +} + +const sizeOf = (t: QbTorrent) => (t.totalSize ?? t.size) || null; +const downloadedOf = (t: QbTorrent) => Math.max(0, Math.round((sizeOf(t) ?? 0) - t.amountLeft)); + +async function observe(d: AoDeps, row: DownloadTaskRow, t: QbTorrent, now: Date, stats: SyncStats): Promise { + const cur = { status: row.status, pauseReason: row.pause_reason }; + const obs = mapQbState( + { state: t.rawState, progress: t.progress, amountLeft: t.amountLeft }, + { + pausedByThreshold: row.pause_reason === "threshold", + previous: toDomainDownloadStatus(row.status, row.pause_reason), + }, + ); + const merged = mergeObservedDownloadState(cur, obs.status); + const downloaded = downloadedOf(t); + let noProgressSince = row.no_progress_since; + let lastProgressAt = row.last_progress_at; + let stalledNow = false; + if (obs.status === "downloading") { + if (downloaded > row.downloaded_bytes || !lastProgressAt) { + lastProgressAt = now; + noProgressSince = null; + if (merged.status === "stalled") merged.status = "downloading"; + } else { + noProgressSince ??= now; + if (now.getTime() - noProgressSince.getTime() >= d.ports.options.noProgressMs && merged.status !== "stalled") { + merged.status = "stalled"; + stalledNow = true; + } + } + } else { + // 暂停/排队/完成不计入无进展 + noProgressSince = null; + } + const becameComplete = merged.status === "completed" && !["completed", "importing", "imported"].includes(row.status); + const changed = merged.status !== row.status || merged.pauseReason !== row.pause_reason; + await d.ctx.db.transaction().execute(async (trx) => { + await trx + .updateTable("download_task") + .set({ + status: merged.status, + pause_reason: merged.status === "paused" ? merged.pauseReason : null, + raw_state: t.rawState, + progress: Math.min(1, Math.max(0, t.progress)), + size_bytes: sizeOf(t), + downloaded_bytes: downloaded, + remaining_bytes: Math.max(0, Math.round(t.amountLeft)), + ratio: t.ratio, + seeding_time_sec: t.seedingTimeSec, + save_path: t.savePath, + content_path: t.contentPath, + name: t.name, + tags: t.tags, + last_progress_at: lastProgressAt, + no_progress_since: noProgressSince, + last_synced_at: now, + ...(becameComplete ? { completed_at: now } : {}), + }) + .where("id", "=", row.id) + .execute(); + if (changed) + await emitStatus( + d, + trx, + row.id, + row.status, + merged.status, + merged.status === "paused" ? merged.pauseReason : null, + ); + if (becameComplete) { + stats.completed.push(row.id); + const plan = row.acquisition_plan_id + ? await trx + .selectFrom("acquisition_plan") + .select(["slot_id"]) + .where("id", "=", row.acquisition_plan_id) + .executeTakeFirst() + : null; + if (plan) await applySlotEvent(d, trx, plan.slot_id, { type: "download_completed" }); + } + if (stalledNow) { + // 无进展:只记证据 + 待处理,保留原任务,不换源(REQ §8.4.2) + await upsertDecision(d, trx, { + kind: "download_stalled", + title: `下载无进展:${t.name}`, + summary: `已 ${Math.round((now.getTime() - (noProgressSince?.getTime() ?? now.getTime())) / 60000)} 分钟没有新增字节`, + subjectType: "download_task", + subjectId: row.id, + dedupeKey: `download_stalled:${row.id}`, + evidence: { + rawState: t.rawState, + progress: t.progress, + downloadedBytes: downloaded, + noProgressSince: noProgressSince?.toISOString(), + action: "record_evidence", + }, + }); + } + if ((merged.status === "failed" || merged.status === "missing") && changed) { + await upsertDecision(d, trx, { + kind: "download_failed", + title: `下载异常:${t.name}`, + summary: `qB 状态 ${t.rawState}`, + subjectType: "download_task", + subjectId: row.id, + dedupeKey: `download_failed:${row.id}`, + evidence: { rawState: t.rawState, progress: t.progress }, + }); + } + }); + return changed; +} + +/** 用户给已有任务贴了管理标签:登记接管,不改保存路径与其它标签 */ +async function adopt(d: AoDeps, topo: InstanceTopology, t: QbTorrent, now: Date, stats: SyncStats) { + const db = d.ctx.db; + const rel = await db + .selectFrom("source_release") + .select("id") + .where((eb) => eb.or([eb("infohash_v1", "=", t.hash), eb("infohash_v2", "=", t.hash)])) + .executeTakeFirst(); + const obs = mapQbState( + { state: t.rawState, progress: t.progress, amountLeft: t.amountLeft }, + { pausedByThreshold: false }, + ); + const st = mergeObservedDownloadState({ status: "queued", pauseReason: null }, obs.status); + await db.transaction().execute(async (trx) => { + const ins = await trx + .insertInto("download_task") + .values({ + instance_id: topo.inst.id, + torrent_hash: t.hash, + infohash_v1: t.infohashV1, + infohash_v2: t.infohashV2, + source_release_id: rel?.id ?? null, + name: t.name, + save_path: t.savePath, + content_path: t.contentPath, + category: t.category || null, + tags: t.tags, + managed: true, + origin: "adopted", + status: st.status, + pause_reason: st.status === "paused" ? st.pauseReason : null, + raw_state: t.rawState, + progress: Math.min(1, Math.max(0, t.progress)), + size_bytes: sizeOf(t), + downloaded_bytes: downloadedOf(t), + remaining_bytes: Math.max(0, Math.round(t.amountLeft)), + ratio: t.ratio, + seeding_time_sec: t.seedingTimeSec, + last_synced_at: now, + last_progress_at: now, + ...(st.status === "completed" ? { completed_at: now } : {}), + }) + .onConflict((oc) => oc.columns(["instance_id", "torrent_hash"]).doNothing()) + .returning("id") + .executeTakeFirst(); + if (!ins) return; + stats.adopted++; + await emitStatus(d, trx, ins.id, null, st.status, st.status === "paused" ? st.pauseReason : null); + if (st.status === "completed") stats.completed.push(ins.id); + }); +} + +// ── 缓存卷硬阈值 ── + +async function enforceThresholds(d: AoDeps, insts: InstanceTopology[], stats: SyncStats) { + if (insts.length === 0) return; + // 只读对账模式禁止自动写入(含对 qB 的暂停/恢复) + if ((await d.ctx.settings.get("automation")).readOnlyReconcile) return; + const cache = await d.ctx.settings.get("cache"); + const vols = await loadVolumes(d, d.ctx.db, insts); + for (const v of vols) { + const dv = decideVolume(v, cache); + if (dv.decision === "pause_managed" && dv.pauseTaskIds.length > 0) { + const n = await pauseForThreshold(d, v, dv.pauseTaskIds); + stats.thresholdPaused += n; + await d.ctx.outbox.emit(d.ctx.db, "download.threshold", { + volumeKey: v.volumeKey, + state: "tripped", + usedPercent: dv.usedPct === undefined ? null : Math.min(100, dv.usedPct), + pausedTaskIds: dv.pauseTaskIds, + }); + } else if (dv.resumeThresholdPaused) { + const ids = await resumeThresholdPaused(d, v); + stats.thresholdResumed += ids.length; + await d.ctx.outbox.emit(d.ctx.db, "download.threshold", { + volumeKey: v.volumeKey, + state: "recovered", + usedPercent: dv.usedPct === undefined ? null : Math.min(100, dv.usedPct), + pausedTaskIds: ids, + }); + } + } +} + +async function pauseForThreshold(d: AoDeps, v: VolumeState, taskIds: string[]): Promise { + const rows = await d.ctx.db + .selectFrom("download_task") + .select(["id", "instance_id", "torrent_hash", "status"]) + .where("id", "in", taskIds) + .execute(); + let n = 0; + for (const topo of v.instances) { + const mine = rows.filter((r) => r.instance_id === topo.inst.id); + if (mine.length === 0) continue; + const client = await d.ports.downloader(topo.inst); + try { + await client.pause(mine.map((r) => r.torrent_hash)); + } catch (err) { + d.ctx.logger.error({ instanceId: topo.inst.id, err: { message: errMessage(err) } }, "阈值暂停失败"); + continue; + } + await d.ctx.db.transaction().execute(async (trx) => { + for (const r of mine) { + await trx + .updateTable("download_task") + .set({ status: "paused", pause_reason: "threshold", no_progress_since: null }) + .where("id", "=", r.id) + .execute(); + await emitStatus(d, trx, r.id, r.status, "paused", "threshold"); + n++; + } + }); + } + return n; +} + +/** 恢复水位:只恢复因本阈值暂停的纳管任务,不动用户手动暂停项 */ +async function resumeThresholdPaused(d: AoDeps, v: VolumeState): Promise { + const out: string[] = []; + for (const topo of v.instances) { + const rows = await d.ctx.db + .selectFrom("download_task") + .select(["id", "torrent_hash"]) + .where("instance_id", "=", topo.inst.id) + .where("status", "=", "paused") + .where("pause_reason", "=", "threshold") + .where("managed", "=", true) + .where("removed_at", "is", null) + .execute(); + if (rows.length === 0) continue; + const client = await d.ports.downloader(topo.inst); + try { + await client.resume(rows.map((r) => r.torrent_hash)); + } catch (err) { + d.ctx.logger.error({ instanceId: topo.inst.id, err: { message: errMessage(err) } }, "阈值恢复失败"); + continue; + } + await d.ctx.db.transaction().execute(async (trx) => { + for (const r of rows) { + await trx + .updateTable("download_task") + .set({ status: "downloading", pause_reason: null }) + .where("id", "=", r.id) + .execute(); + await emitStatus(d, trx, r.id, "paused", "downloading", null); + out.push(r.id); + } + }); + } + return out; +} + +// ── 用户暂停/恢复 ── + +export async function userPause(d: AoDeps, id: string): Promise { + const row = await d.ctx.db.selectFrom("download_task").selectAll().where("id", "=", id).executeTakeFirst(); + if (!row) throw new DownloaderError("notfound", "下载任务不存在"); + const [topo] = await loadInstances(d.ctx.db, { enabledOnly: false, ids: [row.instance_id] }); + const client = await d.ports.downloader(topo!.inst); + await client.pause([row.torrent_hash]); + return d.ctx.db.transaction().execute(async (trx) => { + const r = await trx + .updateTable("download_task") + .set({ status: "paused", pause_reason: "user", no_progress_since: null }) + .where("id", "=", id) + .returningAll() + .executeTakeFirstOrThrow(); + if (row.status !== "paused" || row.pause_reason !== "user") + await emitStatus(d, trx, id, row.status, "paused", "user"); + return r; + }); +} + +export async function userResume(d: AoDeps, id: string): Promise { + const row = await d.ctx.db.selectFrom("download_task").selectAll().where("id", "=", id).executeTakeFirst(); + if (!row) throw new DownloaderError("notfound", "下载任务不存在"); + const [topo] = await loadInstances(d.ctx.db, { enabledOnly: false, ids: [row.instance_id] }); + if (row.pause_reason === "threshold") { + // 阈值不被手动操作绕过:卷仍在阈值之上时拒绝 + const cache = await d.ctx.settings.get("cache"); + const v = (await loadVolumes(d, d.ctx.db, [topo!]))[0]; + const pct = v ? usedPercent(v) : null; + if (pct === null || pct > cache.resumePercent) return "threshold_active"; + } + const client = await d.ports.downloader(topo!.inst); + await client.resume([row.torrent_hash]); + return d.ctx.db.transaction().execute(async (trx) => { + const r = await trx + .updateTable("download_task") + .set({ status: "downloading", pause_reason: null }) + .where("id", "=", id) + .returningAll() + .executeTakeFirstOrThrow(); + await emitStatus(d, trx, id, row.status, "downloading", null); + return r; + }); +} + +// ── 缓存清理 ── + +export interface CleanupCandidate { + row: DownloadTaskRow; + topo: InstanceTopology; + bytes: number; + /** 可清理;否则 lockReason 说明为什么锁定 */ + cleanable: boolean; + lockReason: string | null; +} + +/** 挑选已验证入库、满足用户保留天数与 H&R、无其它引用的纳管任务(按完成时间先旧后新) */ +export async function cleanupCandidates(d: AoDeps, v: VolumeState): Promise { + const db = d.ctx.db; + const retention = await d.ctx.settings.get("retention"); + const now = d.ports.now(); + const out: CleanupCandidate[] = []; + for (const topo of v.instances) { + const rows = await db + .selectFrom("download_task") + .selectAll() + .where("instance_id", "=", topo.inst.id) + .where("managed", "=", true) + .where("removed_at", "is", null) + .where("status", "=", "imported") + .orderBy("completed_at", "asc") + .execute(); + for (const row of rows) { + const rel = row.source_release_id + ? await db + .selectFrom("source_release") + .select(["raw", "site_id"]) + .where("id", "=", row.source_release_id) + .executeTakeFirst() + : null; + const site = rel + ? await db.selectFrom("site").select("config").where("id", "=", rel.site_id).executeTakeFirst() + : null; + const lock = await lockReason(d, row, rel ?? null, site ?? null, retention.seedRetentionDays, now); + out.push({ row, topo, bytes: row.size_bytes ?? 0, cleanable: lock === null, lockReason: lock }); + } + } + return out; +} + +async function lockReason( + d: AoDeps, + row: DownloadTaskRow, + rel: { raw: unknown } | null, + site: { config: unknown } | null, + days: number, + now: Date, +): Promise { + if (!retentionElapsed(row, days, now)) return "retention_days"; + const hnr = hnrSatisfied(row, rel as never, site as never); + if (hnr !== true) return hnr === false ? "hnr_unsatisfied" : "hnr_unknown"; + // 已验证入库:最近一次整理记录成功,且没有失败/排队的文件(部分入库不能整种清理) + const rec = await d.ctx.db + .selectFrom("organization_record") + .select(["id", "status"]) + .where("download_task_id", "=", row.id) + .orderBy("created_at", "desc") + .executeTakeFirst(); + if (rec?.status !== "succeeded") return "import_not_verified"; + const bad = await d.ctx.db + .selectFrom("organization_file_result") + .select("id") + .where("organization_record_id", "=", rec.id) + .where("status", "in", ["failed", "queued", "running", "cancelled"]) + .executeTakeFirst(); + if (bad) return "import_partial"; + // 仍被资产位置引用(缓存内直接做种/硬链接) + const ref = await d.ctx.db + .selectFrom("asset_location as l") + .innerJoin("media_asset as a", "a.id", "l.media_asset_id") + .select("l.id") + .where("a.download_task_id", "=", row.id) + .where("l.location_kind", "in", ["download_cache", "hardlink"]) + .where("l.status", "!=", "removed") + .executeTakeFirst(); + if (ref) return "referenced_by_asset"; + return null; +} + +export interface CleanupReport { + volumes: { + volumeKey: string; + action: "none" | "cleaned" | "insufficient" | "unknown"; + usedPercentBefore: number | null; + usedPercentAfter: number | null; + removed: string[]; + skipped: { downloadTaskId: string; reason: string }[]; + }[]; +} + +export async function cleanupDownloadCache( + d: AoDeps, + input: { downloaderInstanceId?: string | undefined } = {}, +): Promise { + const db = d.ctx.db; + const cache = await d.ctx.settings.get("cache"); + const insts = await loadInstances(db); + const vols = (await loadVolumes(d, db, insts)).filter( + (v) => !input.downloaderInstanceId || v.instances.some((t) => t.inst.id === input.downloaderInstanceId), + ); + const roots = await loadLibraryRoots(db); + const report: CleanupReport = { volumes: [] }; + for (const v of vols) { + const before = usedPercent(v); + const entry: CleanupReport["volumes"][number] = { + volumeKey: v.volumeKey, + action: "none", + usedPercentBefore: before, + usedPercentAfter: before, + removed: [], + skipped: [], + }; + report.volumes.push(entry); + if (before === null) { + // 读不到磁盘状态:不清理(不能把"未知"当"需要清理") + entry.action = "unknown"; + continue; + } + if (before < cache.hardThresholdPercent) continue; + let pct = before; + const cands = (await cleanupCandidates(d, v)).filter((c) => { + if (!c.cleanable) entry.skipped.push({ downloadTaskId: c.row.id, reason: c.lockReason ?? "locked" }); + return c.cleanable; + }); + for (const cand of cands) { + if (pct <= cache.cleanupTargetPercent) break; + const ok = await removeWithData( + d, + cand, + roots.map((r) => r.local_path), + ).catch((err) => { + entry.skipped.push({ downloadTaskId: cand.row.id, reason: errMessage(err) }); + return false; + }); + if (!ok) continue; + entry.removed.push(cand.row.id); + const u = v.path ? await d.ports.diskUsage(v.path) : null; + pct = u?.status === "ok" ? (u.usedBytes / (u.usedBytes + u.availableBytes)) * 100 : pct; + } + entry.usedPercentAfter = pct; + entry.action = entry.removed.length > 0 ? "cleaned" : "none"; + if (pct > cache.cleanupTargetPercent && pct >= cache.hardThresholdPercent) { + entry.action = entry.removed.length > 0 ? "cleaned" : "insufficient"; + // 没有可清理对象:待处理 + 阈值继续阻止新下载(不突破保留要求) + await upsertDecision(d, db, { + kind: "space_insufficient", + severity: "critical", + title: `下载缓存卷 ${v.volumeKey} 空间不足`, + summary: `使用率 ${pct.toFixed(1)}%,可清理任务已用尽;新下载已暂停`, + subjectType: "cache_volume", + dedupeKey: `space_insufficient:${v.volumeKey}`, + evidence: { usedPercent: pct, locked: entry.skipped }, + }); + } + } + return report; +} + +/** qB 删种 + 删数据:必经 assertSafeToDeleteData(本机视图与下载器视图各查一次) */ +async function removeWithData(d: AoDeps, cand: CleanupCandidate, libraryRootsLocal: string[]): Promise { + const { row, topo } = cand; + if (!row.content_path) throw new Error("content_path 未知,不能删数据"); + const localContent = hostPathOf(topo, row.content_path); + if (!localContent) throw new Error("content_path 无法映射到本机路径"); + assertSafeToDeleteData( + { contentPath: localContent, libraryRoots: libraryRootsLocal, importVerified: true }, + topo.inst.id, + ); + const rootsDl = libraryRootsLocal.map((r) => downloaderPathOf(topo, r) ?? r); + const client = await d.ports.downloader(topo.inst); + const opId = `cleanup:${row.id}`; + await withAttemptToken( + d.ctx.db, + opId, + async () => { + await client.delete([row.torrent_hash], { + deleteFiles: true, + safety: { [row.torrent_hash]: { contentPath: row.content_path!, libraryRoots: rootsDl, importVerified: true } }, + }); + return { deleted: true }; + }, + { + commit: async (trx: Transaction) => { + const now = d.ports.now(); + await trx + .updateTable("download_task") + .set({ status: "removed", removed_at: now, managed: false }) + .where("id", "=", row.id) + .execute(); + await trx + .insertInto("asset_operation") + .values({ + operation_id: opId, + phase: "cleanup", + kind: "qb_remove", + status: "succeeded", + intent: jsonb({ + torrentHash: row.torrent_hash, + instanceId: topo.inst.id, + deleteFiles: true, + contentPath: row.content_path, + }), + download_task_id: row.id, + started_at: now, + finished_at: now, + attempts: 1, + }) + .onConflict((oc) => oc.column("operation_id").doNothing()) + .execute(); + await emitStatus(d, trx, row.id, row.status, "removed", null); + }, + }, + ); + return true; +} diff --git a/apps/server/src/modules/acquisition-organize/services/organize.ts b/apps/server/src/modules/acquisition-organize/services/organize.ts new file mode 100644 index 00000000..cbe1ffa3 --- /dev/null +++ b/apps/server/src/modules/acquisition-organize/services/organize.ts @@ -0,0 +1,1292 @@ +import { mkdir, readdir } from "node:fs/promises"; +import { dirname, join, relative } from "node:path"; +import { + type CrossMoveStepLog, + checkRoot, + copyWithVerify, + crossDeviceMove, + DEFAULT_ROOT_SENTINEL, + FsError, + filesMatch, + moveSafe, + sha256File, + statSafe, +} from "@umt/adapters"; +import type { ActorType, OrganizationRecord, OrganizationTrigger, PathBinding } from "@umt/contracts"; +import type { DownloadTaskRow, LibraryRootRow, MediaEntityRow, OrganizationRecordRow } from "@umt/db"; +import { sql } from "@umt/db"; +import { + buildNamingContext, + DEFAULT_TEMPLATES, + type PathBinding as DomainBinding, + type ExistingTarget, + movieCompletion, + type NewBinding, + type OrganizePlan, + type ParsedRelease, + parseReleasePath, + planOrganize, + type QualityFacts, + resolveTargetDirectory, + type SourceFile, + unitCompletion, +} from "@umt/domain"; +import { organizationStatusFromFiles, organizePlanToFileResults, toDomainPathBinding } from "@umt/mapping"; +import { withAttemptToken } from "@umt/tasks"; +import { errMessage, hashJson, jsonb, toApi } from "../util.ts"; +import { publishedAcquisitionPolicies, unitKeysOf } from "./acquisition.ts"; +import { type AoDeps, type Exec, hostPathOf, loadInstances, upsertDecision } from "./common.ts"; +import { applySlotEvent } from "./downloads.ts"; +import { hnrSatisfied } from "./retention.ts"; + +// 入库:冻结整理计划(organization_record + 文件明细)→ 逐文件可恢复执行(操作账本 fencing)→ +// 资产/位置/路径绑定同事务登记 → Slot 推进 → Move 后移除 qB 任务记录(不删数据)。 + +export interface ImportRequest { + downloadTaskId?: string | undefined; + /** 手动:执行进程可访问的源路径(文件或目录) */ + sourcePaths?: string[] | undefined; + mediaEntityId?: string | undefined; + contentUnitId?: string | undefined; + action?: "copy" | "move" | undefined; + targetLibraryRootId?: string | undefined; + trigger: OrganizationTrigger; + actorType: ActorType; + actorId?: string | null; + taskId?: string | null; + /** 只生成计划,不写记录、不建待处理 */ + dryRun?: boolean; +} + +/** organization_record.plan 的冻结内容 */ +export interface FrozenPlan { + version: 1; + organize: OrganizePlan; + entityId: string; + mediaType: "movie" | "tv"; + libraryRootId: string; + rootPath: string; + sentinel: string; + targetDirs: string[]; + newBindings: NewBinding[]; + sourceRoot: string | null; + /** 主视频 src → 解析出的质量特征与单元键 */ + facts: Record< + string, + { quality: QualityFacts; unitKeys: string[]; season?: number; episodes?: { start: number; end: number } } + >; + downgradedFromMove?: string; +} + +export interface PlanResult { + /** dryRun 时为 null */ + recordId: string | null; + status: OrganizationRecord["status"]; + reason: string | null; + plan?: FrozenPlan; +} + +const qualityOf = (p: ParsedRelease): QualityFacts => ({ + ...(p.resolution ? { resolution: p.resolution } : {}), + ...(p.source ? { source: p.source } : {}), + hdr: p.hdr, + ...(p.releaseGroup ? { releaseGroup: p.releaseGroup } : {}), + isEncode: p.isEncode, + ...(p.cut ? { cut: p.cut } : {}), +}); + +function segmentsWithin(path: string, root: string | null) { + if (!root) return path; + const r = root.replace(/\/+$/, ""); + // 单文件种子:内容根就是文件本身,不拿缓存目录名参与解析 + if (path === r) return { file: r.split("/").pop() ?? r }; + if (!path.startsWith(`${r}/`)) return path; + const within = [r.split("/").pop() ?? "", ...path.slice(r.length + 1).split("/")].filter(Boolean); + const file = within.pop()!; + const parent = within.pop(); + const grandParent = within.pop(); + return { file, ...(parent !== undefined ? { parent } : {}), ...(grandParent !== undefined ? { grandParent } : {}) }; +} + +async function walk(p: string, out: string[], limit = 5000): Promise { + const st = await statSafe(p); + if (st.kind !== "ok") throw new FsError("ENOENT", `源路径不可访问: ${p} (${st.kind})`, { path: p }); + if (st.facts.isFile) { + out.push(p); + return; + } + if (!st.facts.isDirectory) return; + const names = (await readdir(p)).sort(); + for (const n of names) { + if (out.length >= limit) return; + if (n.startsWith(".")) continue; + await walk(join(p, n), out, limit); + } +} + +function libraryKinds(kind: MediaEntityRow["kind"]): string[] { + if (kind === "movie") return ["movie", "mixed"]; + if (kind === "variety") return ["variety", "series", "mixed"]; + return ["series", "mixed"]; +} + +async function chooseRoot( + exec: Exec, + entity: MediaEntityRow, + bindings: DomainBinding[], + wanted?: string, +): Promise { + const roots = await exec.selectFrom("library_root").selectAll().where("enabled", "=", true).execute(); + if (wanted) return roots.find((r) => r.id === wanted) ?? null; + const bound = bindings.find((b) => b.active !== false); + if (bound) return roots.find((r) => r.id === bound.libraryRootId) ?? null; + const kinds = libraryKinds(entity.kind); + return ( + roots + .filter((r) => r.management_mode === "managed" && kinds.includes(r.media_kind)) + .sort((a, b) => kinds.indexOf(a.media_kind) - kinds.indexOf(b.media_kind))[0] ?? null + ); +} + +/** 生成并冻结整理计划;阻塞冲突 → 记录失败 + 待处理,不执行 */ +export async function planImport(d: AoDeps, req: ImportRequest): Promise { + const db = d.ctx.db; + let dt: DownloadTaskRow | undefined; + if (req.downloadTaskId) { + dt = await db.selectFrom("download_task").selectAll().where("id", "=", req.downloadTaskId).executeTakeFirst(); + if (!dt) throw new Error(`下载任务 ${req.downloadTaskId} 不存在`); + // 同一下载任务已有未结束的记录:复用(步骤重放幂等) + const open = await db + .selectFrom("organization_record") + .select(["id", "status", "reason"]) + .where("download_task_id", "=", dt.id) + .where("status", "in", ["queued", "running"]) + .executeTakeFirst(); + if (open) return { recordId: open.id, status: open.status, reason: open.reason }; + } + const base = { + trigger: req.trigger, + actorType: req.actorType, + actorId: req.actorId ?? null, + taskId: req.taskId ?? null, + downloadTaskId: dt?.id ?? null, + }; + const dry = req.dryRun === true; + const decide_ = (input: Parameters[2]) => + dry ? Promise.resolve("") : upsertDecision(d, db, input); + const fail = async ( + reason: string, + extra: Record = {}, + entityId: string | null = null, + ): Promise => + dry + ? { recordId: null, status: "failed", reason } + : writeFailedRecord(d, { + ...base, + action: req.action ?? "copy", + reason, + evidence: extra, + entityId, + sourceInstanceId: dt?.instance_id ?? null, + }); + + // ── 源文件 ── + const files: SourceFile[] = []; + let sourceRoot: string | null = null; + const topo = dt ? (await loadInstances(db, { enabledOnly: false, ids: [dt.instance_id] }))[0] : undefined; + if (dt) { + if (!topo || !dt.save_path) return fail("source_path_unknown"); + const base = hostPathOf(topo, dt.save_path); + if (!base) return fail("path_mapping_missing", { savePath: dt.save_path }); + let list: { name: string; size: number; priority?: number; index?: number }[] = []; + try { + const client = await d.ports.downloader(topo.inst); + list = (await client.files(dt.torrent_hash)).map((f) => ({ + name: f.name, + size: f.size, + priority: f.priority, + index: f.index, + })); + } catch { + list = ((dt.file_list ?? []) as { path: string; size: number }[]).map((f) => ({ name: f.path, size: f.size })); + } + if (list.length === 0) return fail("file_list_unavailable"); + sourceRoot = dt.content_path ? hostPathOf(topo, dt.content_path) : null; + for (const f of list) { + if (f.priority === 0) continue; + const abs = join(base, f.name); + const st = await statSafe(abs); + if (st.kind !== "ok") return fail("source_missing", { path: abs, stat: st.kind }); + if (st.facts.size !== f.size) + return fail("source_size_mismatch", { path: abs, expected: f.size, actual: st.facts.size }); + files.push({ + path: abs, + size: st.facts.size, + device: st.facts.dev, + ...(f.index !== undefined ? { index: f.index } : {}), + }); + } + } else if (req.sourcePaths?.length) { + const paths: string[] = []; + try { + for (const p of req.sourcePaths) await walk(p, paths); + } catch (err) { + return fail("source_missing", { message: errMessage(err) }); + } + for (const p of paths) { + const st = await statSafe(p); + if (st.kind === "ok") files.push({ path: p, size: st.facts.size, device: st.facts.dev }); + } + sourceRoot = req.sourcePaths.length === 1 ? req.sourcePaths[0]!.replace(/\/+$/, "") : null; + if (sourceRoot && files.length === 1 && files[0]!.path === sourceRoot) sourceRoot = dirname(sourceRoot); + } else { + return fail("no_source"); + } + + // ── 作品 ── + let entityId = req.mediaEntityId ?? null; + let slotId: string | null = null; + if (dt?.acquisition_plan_id) { + const p = await db + .selectFrom("acquisition_plan as p") + .innerJoin("acquisition_slot as s", "s.id", "p.slot_id") + .select(["s.id as slot_id", "s.media_entity_id"]) + .where("p.id", "=", dt.acquisition_plan_id) + .executeTakeFirst(); + slotId = p?.slot_id ?? null; + entityId ??= p?.media_entity_id ?? null; + } + const rel = dt?.source_release_id + ? await db.selectFrom("source_release").selectAll().where("id", "=", dt.source_release_id).executeTakeFirst() + : undefined; + entityId ??= rel?.media_entity_id ?? null; + if (!entityId) { + const rid = await fail("identity_unknown"); + await decide_({ + kind: "identity_ambiguous", + title: `无法确定作品:${dt?.name ?? req.sourcePaths?.[0] ?? "未知来源"}`, + summary: "下载内容没有关联本地作品,需要先识别/指定作品再整理", + subjectType: "organization_record", + subjectId: rid.recordId, + dedupeKey: `organize_identity:${dt?.id ?? rid.recordId ?? "x"}`, + evidence: { downloadTaskId: dt?.id ?? null }, + }); + return rid; + } + const entity = await db.selectFrom("media_entity").selectAll().where("id", "=", entityId).executeTakeFirst(); + if (!entity) return fail("entity_not_found", {}, null); + const mediaType: "movie" | "tv" = entity.kind === "movie" ? "movie" : "tv"; + + // ── 操作方式 ── + const orgSettings = await d.ctx.settings.get("organize"); + const policy = (await publishedAcquisitionPolicies(db))[0]; + let action: "copy" | "move" = req.action ?? policy?.rules.organizeMode ?? orgSettings.defaultMode; + const site = rel + ? await db.selectFrom("site").selectAll().where("id", "=", rel.site_id).executeTakeFirst() + : undefined; + const hnr = dt ? hnrSatisfied(dt, rel, site) : true; + let downgradedFromMove: string | undefined; + if (action === "move" && hnr !== true && policy?.rules.allowCopyWhenHnrPending) { + // 策略预先授权:H&R 未满足时改用 Copy(显式记录,不是静默降级) + action = "copy"; + downgradedFromMove = hnr === false ? "hnr_unsatisfied" : "hnr_unknown"; + } + + // ── 路径绑定与库根 ── + const bindingRows = await db + .selectFrom("path_binding") + .selectAll() + .where("media_entity_id", "=", entity.id) + .where("status", "=", "active") + .execute(); + const bindings = bindingRows.map((r) => toDomainPathBinding(toApi(r))); + const root = await chooseRoot(db, entity, bindings, req.targetLibraryRootId); + if (!root) return fail("no_library_root", {}, entity.id); + const sentinel = root.sentinel_file ?? DEFAULT_ROOT_SENTINEL; + const health = await checkRoot(root.local_path, sentinel); + if (health.kind === "offline") { + const r = await fail(`library_root_offline:${health.reason}`, { libraryRootId: root.id }, entity.id); + await decide_({ + kind: "other", + severity: "critical", + title: `媒体库根离线:${root.name}`, + summary: `哨兵检查失败(${health.reason}),入库暂停;不会因此判定文件缺失`, + subjectType: "library_root", + subjectId: root.id, + dedupeKey: `library_root_offline:${root.id}`, + evidence: { reason: health.reason }, + }); + return r; + } + + const naming = buildNamingContext({ + title: entity.title, + ...(entity.original_title ? { originalTitle: entity.original_title } : {}), + ...(entity.year ? { year: entity.year } : {}), + }); + const facts: FrozenPlan["facts"] = {}; + const seasons = new Set(); + for (const f of files) { + const parsed = parseReleasePath(segmentsWithin(f.path, sourceRoot)); + if (parsed.fileKind !== "video") continue; + const season = parsed.season ?? (parsed.episodes || parsed.airDate ? 1 : undefined); + if (mediaType === "tv" && season !== undefined) seasons.add(season); + facts[f.path] = { + quality: qualityOf(parsed), + unitKeys: unitKeysOf({ ...parsed, ...(season !== undefined ? { season } : {}) }), + ...(season !== undefined ? { season } : {}), + ...(parsed.episodes ? { episodes: parsed.episodes } : {}), + }; + } + const libRoot = { id: root.id, hostPath: root.local_path }; + const newBindings: NewBinding[] = []; + const target: { movieDir?: string; seasonDirs?: Record } = {}; + const working = [...bindings]; + const decide = (season?: number) => + resolveTargetDirectory({ + entity: { id: entity.id, kind: mediaType }, + naming, + ...(season !== undefined ? { season } : {}), + bindings: working, + libraryRoot: libRoot, + templates: DEFAULT_TEMPLATES, + }); + const handle = async (dec: ReturnType, season?: number): Promise => { + if (dec.kind === "conflict" || dec.kind === "error") { + const r = await fail(`path_binding_${dec.kind}:${dec.reason}`, { season }, entity.id); + if (dec.kind === "conflict") { + await decide_({ + kind: "path_conflict", + title: `作品存在多个历史目录:${entity.title}`, + summary: "不按最新标题自动合并,请指定保留的目录", + subjectType: "media_entity", + subjectId: entity.id, + dedupeKey: `path_binding_conflict:${entity.id}:${season ?? "entity"}`, + evidence: { + bindings: dec.bindings.map((b) => ({ + id: b.id, + libraryRootId: b.libraryRootId, + relativePath: b.relativePath, + })), + }, + }); + } + return r; + } + if (dec.kind === "new") { + for (const nb of dec.newBindings) { + if (!newBindings.some((x) => x.scope === nb.scope && x.season === nb.season)) newBindings.push(nb); + working.push({ + id: `pending:${nb.scope}:${nb.season ?? ""}`, + entityId: entity.id, + scope: nb.scope, + ...(nb.season !== undefined ? { season: nb.season } : {}), + libraryRootId: nb.libraryRootId, + relativePath: nb.relativePath, + templateVersion: nb.templateVersion, + active: true, + }); + } + } + if (dec.libraryRootId !== root.id) + return fail("binding_on_other_root", { libraryRootId: dec.libraryRootId }, entity.id); + return join(root.local_path, dec.relativePath); + }; + if (mediaType === "movie") { + const r = await handle(decide()); + if (typeof r !== "string") return r; + target.movieDir = r; + } else { + target.seasonDirs = {}; + for (const s of [...seasons].sort((a, b) => a - b)) { + const r = await handle(decide(s), s); + if (typeof r !== "string") return r; + target.seasonDirs[s] = r; + } + } + + // ── 计划:先算目标,再观察目标是否已存在(失败 ≠ 不存在:stat 出错按存在处理) ── + const baseInput = { + operation: action, + mediaType, + ...(sourceRoot ? { sourceRoot } : {}), + files, + entity: { + title: entity.title, + ...(entity.original_title ? { originalTitle: entity.original_title } : {}), + ...(entity.year ? { year: entity.year } : {}), + }, + target: { ...target, ...(health.kind === "online" ? { device: health.dev } : {}) }, + retention: { hnrSatisfied: hnr }, + minVideoBytes: d.ports.options.minVideoBytes, + } as const; + const draft = planOrganize({ ...baseInput, existingTargets: [] }); + const existing: ExistingTarget[] = []; + for (const f of draft.files) { + const st = await statSafe(f.dst); + if (st.kind === "missing") continue; + if (st.kind !== "ok") { + existing.push({ path: f.dst, size: -1 }); + continue; + } + const e: ExistingTarget = { path: f.dst, size: st.facts.size }; + if (st.facts.size === f.size) { + e.sha256 = await sha256File(f.dst); + const src = files.find((x) => x.path === f.src); + if (src) src.sha256 = await sha256File(src.path); + } + existing.push(e); + } + const plan = planOrganize({ ...baseInput, files, existingTargets: existing }); + const frozen: FrozenPlan = { + version: 1, + organize: plan, + entityId: entity.id, + mediaType, + libraryRootId: root.id, + rootPath: root.local_path, + sentinel, + targetDirs: [...new Set(plan.files.map((f) => dirname(f.dst)))], + newBindings, + sourceRoot, + facts, + ...(downgradedFromMove ? { downgradedFromMove } : {}), + }; + const drafts = organizePlanToFileResults(plan, { targetLibraryRootId: root.id }); + const blocking = plan.conflicts.filter((c) => c.blocking); + const reason = blocking.length + ? `blocked:${[...new Set(blocking.map((c) => c.kind))].join(",")}` + : downgradedFromMove + ? `move_downgraded_to_copy:${downgradedFromMove}` + : null; + const now = d.ports.now(); + const targetRel = (abs: string | null | undefined) => (abs ? relative(root.local_path, abs) : null); + if (dry) return { recordId: null, status: blocking.length ? "failed" : "queued", reason, plan: frozen }; + + const result = await db.transaction().execute(async (trx) => { + const rec = await trx + .insertInto("organization_record") + .values({ + task_id: req.taskId ?? null, + trigger: req.trigger, + actor_type: req.actorType, + actor_id: req.actorId ?? null, + media_entity_id: entity.id, + content_unit_id: req.contentUnitId ?? null, + download_task_id: dt?.id ?? null, + source_downloader_instance_id: dt?.instance_id ?? null, + source_path_view: "local", + source_path: sourceRoot ?? files[0]?.path ?? null, + target_library_root_id: root.id, + target_path: targetRel(frozen.targetDirs[0]), + action, + status: blocking.length ? "failed" : "running", + reason, + plan: jsonb(frozen), + plan_fingerprint: hashJson(frozen), + file_total: drafts.length, + file_skipped: drafts.filter((f) => f.status === "skipped" || blocking.length > 0).length, + started_at: now, + ...(blocking.length ? { finished_at: now, error: jsonb({ conflicts: blocking }) } : {}), + }) + .returning(["id", "status", "change_seq"]) + .executeTakeFirstOrThrow(); + for (const f of drafts) { + await trx + .insertInto("organization_file_result") + .values({ + organization_record_id: rec.id, + ordinal: f.ordinal ?? 0, + role: f.role ?? "main", + mode: f.mode, + source_path: f.sourcePath, + source_size_bytes: f.sourceSizeBytes ?? null, + target_library_root_id: f.targetPath ? root.id : null, + target_path: targetRel(f.targetPath), + status: blocking.length ? "skipped" : f.status, + skip_reason: blocking.length && f.status !== "skipped" ? "blocked_by_conflict" : f.skipReason, + }) + .execute(); + } + if (dt) { + await trx + .updateTable("download_task") + .set({ status: blocking.length ? "completed" : "importing" }) + .where("id", "=", dt.id) + .where("status", "in", ["completed", "importing"]) + .execute(); + } + if (blocking.length) { + const hnrConflict = blocking.find((c) => c.kind === "hnr_unsatisfied" || c.kind === "hnr_unknown"); + await upsertDecision(d, trx, { + kind: hnrConflict ? "approval_required" : "path_conflict", + title: hnrConflict ? `Move 入库被做种义务阻止:${entity.title}` : `整理冲突:${entity.title}`, + summary: blocking + .map((c) => c.detail ?? c.kind) + .join(";") + .slice(0, 500), + subjectType: "organization_record", + subjectId: rec.id, + dedupeKey: `organize_blocked:${dt?.id ?? rec.id}`, + evidence: { conflicts: blocking, action }, + options: hnrConflict + ? [ + { key: "wait", label: "等待做种义务满足后再 Move" }, + { key: "copy", label: "授权改用 Copy 入库" }, + ] + : [{ key: "review", label: "查看冲突并指定处理方式" }], + }); + if (slotId) await applySlotEvent(d, trx, slotId, { type: "verify_failed" }, { reason: reason ?? "blocked" }); + } + await emitRecord(d, trx, rec.id); + return { recordId: rec.id, status: rec.status, reason }; + }); + return result; +} + +async function writeFailedRecord( + d: AoDeps, + r: { + trigger: OrganizationTrigger; + actorType: ActorType; + actorId: string | null; + taskId: string | null; + downloadTaskId: string | null; + action: "copy" | "move"; + reason: string; + evidence: Record; + entityId: string | null; + sourceInstanceId: string | null; + }, +): Promise { + const now = d.ports.now(); + return d.ctx.db.transaction().execute(async (trx) => { + const rec = await trx + .insertInto("organization_record") + .values({ + task_id: r.taskId, + trigger: r.trigger, + actor_type: r.actorType, + actor_id: r.actorId, + media_entity_id: r.entityId, + download_task_id: r.downloadTaskId, + source_downloader_instance_id: r.sourceInstanceId, + action: r.action, + status: "failed", + reason: r.reason, + error: jsonb(r.evidence), + started_at: now, + finished_at: now, + }) + .returning(["id"]) + .executeTakeFirstOrThrow(); + await emitRecord(d, trx, rec.id); + return { recordId: rec.id, status: "failed" as const, reason: r.reason }; + }); +} + +export async function emitRecord(d: AoDeps, exec: Exec, recordId: string): Promise { + const r = await exec + .selectFrom("organization_record") + .select(["status", "change_seq", "media_entity_id"]) + .where("id", "=", recordId) + .executeTakeFirstOrThrow(); + await d.ctx.outbox.emit(exec, "organization.changed", { + recordId, + status: r.status, + changeSeq: String(r.change_seq), + mediaEntityId: r.media_entity_id, + }); +} + +// ── 执行 ── + +type StepLog = { step: string; status: string; at: string; detail?: string; error?: { code: string; message: string } }; +type FileOutcome = { + status: "succeeded" | "failed" | "attention"; + bytes?: number; + checksum?: string; + steps?: StepLog[]; + error?: { code: string; message: string }; + resumed?: boolean; +}; +const stepsOf = (s: CrossMoveStepLog[]): StepLog[] => s.map((x) => ({ ...x })); + +export interface ExecuteResult { + recordId: string; + status: OrganizationRecord["status"]; + succeeded: number; + failed: number; +} + +export async function executeRecord( + d: AoDeps, + recordId: string, + opts: { runId?: string; abortSignal?: AbortSignal } = {}, +): Promise { + const db = d.ctx.db; + const rec = await db + .selectFrom("organization_record") + .selectAll() + .where("id", "=", recordId) + .executeTakeFirstOrThrow(); + if (rec.status !== "running" && rec.status !== "queued") { + return { recordId, status: rec.status, succeeded: rec.file_succeeded, failed: rec.file_failed }; + } + const frozen = rec.plan as unknown as FrozenPlan; + const health = await checkRoot(frozen.rootPath, frozen.sentinel); + if (health.kind === "offline") throw new Error(`媒体库根离线(${health.reason}),稍后重试`); + for (const dir of frozen.targetDirs) await mkdir(dir, { recursive: true }); + const orgSettings = await d.ctx.settings.get("organize"); + + const files = await db + .selectFrom("organization_file_result") + .selectAll() + .where("organization_record_id", "=", recordId) + .orderBy("ordinal") + .execute(); + const planned = new Map(frozen.organize.files.map((f) => [f.src, f])); + const failedMains = new Set(); + for (const fr of files) { + if (fr.status !== "queued" && fr.status !== "running") continue; + const pf = planned.get(fr.source_path); + if (!pf) continue; + if (pf.role === "sidecar" && pf.mainSrc && failedMains.has(pf.mainSrc)) { + await db + .updateTable("organization_file_result") + .set({ status: "skipped", skip_reason: "main_failed", finished_at: d.ports.now() }) + .where("id", "=", fr.id) + .execute(); + continue; + } + opts.abortSignal?.throwIfAborted(); + const opId = `org:${recordId}:${fr.ordinal}`; + const kind = rec.action === "move" ? "move" : "copy"; + await db + .insertInto("asset_operation") + .values({ + operation_id: opId, + task_id: rec.task_id, + organization_record_id: recordId, + plan_fingerprint: rec.plan_fingerprint, + ordinal: fr.ordinal, + phase: "transfer", + kind, + status: "started", + intent: jsonb({ src: pf.src, dst: pf.dst, size: pf.size, crossDevice: pf.crossDevice ?? null }), + download_task_id: rec.download_task_id, + started_at: d.ports.now(), + }) + .onConflict((oc) => + oc.column("operation_id").doUpdateSet({ attempts: sql`asset_operation.attempts + 1`, status: "started" }), + ) + .execute(); + await db + .updateTable("organization_file_result") + .set({ status: "running", started_at: fr.started_at ?? d.ports.now() }) + .where("id", "=", fr.id) + .execute(); + const outcome = await withAttemptToken( + db, + opId, + async (info) => + transferFile( + kind, + pf.src, + pf.dst, + pf.size, + pf.crossDevice, + info.resumed, + d, + orgSettings.allowCrossVolumeMove, + opts.abortSignal, + ), + { + ...(opts.runId ? { runId: opts.runId } : {}), + commit: async (trx, r) => { + const now = d.ports.now(); + const ok = r.status !== "failed"; + await trx + .updateTable("asset_operation") + .set({ + status: ok ? "succeeded" : "failed", + result: jsonb(r), + ...(r.error ? { error: jsonb(r.error) } : {}), + finished_at: now, + }) + .where("operation_id", "=", opId) + .execute(); + const op = await trx + .selectFrom("asset_operation") + .select("id") + .where("operation_id", "=", opId) + .executeTakeFirstOrThrow(); + await trx + .updateTable("organization_file_result") + .set({ + status: ok ? "succeeded" : "failed", + asset_operation_id: op.id, + checksum: r.checksum ?? null, + error: r.error + ? jsonb(r.error) + : r.status === "attention" + ? jsonb({ code: "source_not_removed", message: "目标已就绪,源文件未能删除" }) + : null, + finished_at: now, + }) + .where("id", "=", fr.id) + .execute(); + }, + }, + ); + if (outcome.status === "failed" && pf.role === "main") failedMains.add(pf.src); + if (outcome.status === "attention") { + await upsertDecision(d, db, { + kind: "other", + title: "跨卷搬运后删源失败", + summary: `${pf.src} 已复制到目标并校验,源文件未删除`, + subjectType: "organization_record", + subjectId: recordId, + dedupeKey: `move_source_left:${opId}`, + evidence: { steps: outcome.steps ?? [] }, + }); + } + } + return finalizeRecord(d, recordId); +} + +async function transferFile( + kind: "copy" | "move", + src: string, + dst: string, + size: number, + crossDevice: boolean | undefined, + resumed: boolean, + d: AoDeps, + allowCrossVolume: boolean, + signal?: AbortSignal, +): Promise { + const fsErr = (e: unknown) => ({ code: e instanceof FsError ? e.code : "IO", message: errMessage(e) }); + const [s, t] = await Promise.all([statSafe(src), statSafe(dst)]); + // 对账:上次 attempt 已完成外部动作但没留下结果 + if (t.kind === "ok") { + if (s.kind === "missing" && kind === "move") { + return t.facts.size === size + ? { status: "succeeded", bytes: size, resumed: true } + : { status: "failed", error: { code: "CONFLICT", message: "源已不存在且目标大小不符" } }; + } + if (s.kind === "ok") { + const same = await filesMatch(src, dst, "sha256").catch(() => false); + if (!same) return { status: "failed", error: { code: "EEXIST", message: `目标已存在且内容不同: ${dst}` } }; + if (kind === "copy") return { status: "succeeded", bytes: size, resumed }; + if (crossDevice === true || resumed) { + const r = await crossDeviceMove(src, dst, { hash: "sha256" }); + return { + status: r.status === "completed" ? "succeeded" : r.status, + bytes: size, + steps: stepsOf(r.steps), + resumed: true, + }; + } + } + } + if (s.kind !== "ok") + return { status: "failed", error: { code: "SOURCE_MISSING", message: `源文件不可访问(${s.kind}): ${src}` } }; + try { + if (kind === "copy") { + const r = await copyWithVerify(src, dst, { hash: d.ports.options.verifyHash, ...(signal ? { signal } : {}) }); + return { + status: "succeeded", + bytes: r.bytes, + ...(r.sha256 + ? { checksum: `sha256:${r.sha256}` } + : r.sampleDigest + ? { checksum: `sample:${r.sampleDigest}` } + : {}), + }; + } + if (crossDevice !== true) { + const m = await moveSafe(src, dst); + if (m.status === "moved") return { status: "succeeded", bytes: size }; + if (m.status === "exists") return { status: "failed", error: { code: "EEXIST", message: `目标已存在: ${dst}` } }; + // EXDEV:不静默降级;按设置显式走可恢复三步 + } + if (!allowCrossVolume) + return { + status: "failed", + error: { code: "EXDEV", message: "跨卷 Move 未启用(settings.organize.allowCrossVolumeMove)" }, + }; + const r = await crossDeviceMove(src, dst, { hash: d.ports.options.verifyHash, ...(signal ? { signal } : {}) }); + return { status: r.status === "completed" ? "succeeded" : r.status, bytes: size, steps: stepsOf(r.steps) }; + } catch (e) { + return { status: "failed", error: fsErr(e) }; + } +} + +/** 登记资产/位置/绑定 + Slot/计划/下载任务 + 记录终态,同一事务;记录行锁作 fencing */ +export async function finalizeRecord(d: AoDeps, recordId: string): Promise { + const db = d.ctx.db; + const pre = await db + .selectFrom("organization_record") + .selectAll() + .where("id", "=", recordId) + .executeTakeFirstOrThrow(); + const frozenPre = pre.plan as unknown as FrozenPlan; + // ffprobe 在事务外做(可能较慢);未知不阻塞 + const probes = new Map(); + const filesPre = await db + .selectFrom("organization_file_result") + .selectAll() + .where("organization_record_id", "=", recordId) + .execute(); + for (const f of filesPre) { + if (f.status === "succeeded" && f.role === "main" && f.target_path) { + const r = await d.ports.probe(join(frozenPre.rootPath, f.target_path)); + probes.set(f.id, r); + } + } + + const result = await db.transaction().execute(async (trx) => { + const rec = await trx + .selectFrom("organization_record") + .selectAll() + .where("id", "=", recordId) + .forUpdate() + .executeTakeFirstOrThrow(); + if (rec.status !== "running" && rec.status !== "queued") { + return { + recordId, + status: rec.status, + succeeded: rec.file_succeeded, + failed: rec.file_failed, + move: false, + dt: null as DownloadTaskRow | null, + }; + } + const frozen = rec.plan as unknown as FrozenPlan; + const now = d.ports.now(); + const files = await trx + .selectFrom("organization_file_result") + .selectAll() + .where("organization_record_id", "=", recordId) + .orderBy("ordinal") + .execute(); + + // 探测结果明确"不是视频" → 该文件核验失败 + for (const f of files) { + const p = probes.get(f.id) as { status?: string; video?: unknown[] } | undefined; + if (p?.status === "ok" && Array.isArray(p.video) && p.video.length === 0) { + await trx + .updateTable("organization_file_result") + .set({ status: "failed", error: jsonb({ code: "NOT_VIDEO", message: "ffprobe 未发现视频流" }) }) + .where("id", "=", f.id) + .execute(); + f.status = "failed"; + } + } + + // 新路径绑定与资产登记同事务 + for (const nb of frozen.newBindings) { + await sql`insert into path_binding (media_entity_id, season_number, binding_kind, library_root_id, relative_path, template_version, bound_by) + values (${frozen.entityId}, ${nb.scope === "season" ? (nb.season ?? null) : null}, ${nb.scope}, ${nb.libraryRootId}, ${nb.relativePath}, + ${String(nb.templateVersion)}, ${`organize:${recordId}`}) + on conflict do nothing`.execute(trx); + } + const dt = rec.download_task_id + ? await trx.selectFrom("download_task").selectAll().where("id", "=", rec.download_task_id).executeTakeFirst() + : undefined; + const planFiles = new Map(frozen.organize.files.map((f) => [f.src, f])); + const assetBySrc = new Map(); + const mainAssets: string[] = []; + for (const f of files) { + if (f.status !== "succeeded" || !f.target_path) continue; + const pf = planFiles.get(f.source_path); + const isMain = f.role === "main"; + const parent = pf?.mainSrc ? assetBySrc.get(pf.mainSrc) : undefined; + if (!isMain && !parent) continue; + const hit = await trx + .selectFrom("asset_location") + .select(["id", "media_asset_id"]) + .where("library_root_id", "=", frozen.libraryRootId) + .where("relative_path", "=", f.target_path) + .where("status", "!=", "removed") + .executeTakeFirst(); + let assetId: string; + let locationId: string; + const st = await statSafe(join(frozen.rootPath, f.target_path)); + const factsOf = st.kind === "ok" ? st.facts : null; + if (hit) { + assetId = hit.media_asset_id; + locationId = hit.id; + } else { + const fx = frozen.facts[f.source_path]; + const probe = probes.get(f.id); + const asset = await trx + .insertInto("media_asset") + .values({ + media_entity_id: frozen.entityId, + kind: isMain ? "video" : f.role === "audio" ? "audio" : f.role === "subtitle" ? "subtitle" : "other", + parent_asset_id: parent?.assetId ?? null, + size_bytes: factsOf?.size ?? f.source_size_bytes, + sample_digest: f.checksum?.startsWith("sample:") ? f.checksum.slice(7) : null, + content_hash: f.checksum?.startsWith("sha256:") ? f.checksum.slice(7) : null, + hash_algorithm: f.checksum?.startsWith("sha256:") ? "sha256" : null, + media_info: jsonb({ + ...(probe ? { probe } : {}), + _umt: { quality: fx?.quality ?? {}, unitKeys: fx?.unitKeys ?? [] }, + }), + delivery: jsonb( + fx + ? { + ...(fx.quality.resolution ? { resolution: fx.quality.resolution } : {}), + ...(fx.quality.source ? { sourceType: fx.quality.source } : {}), + ...(fx.quality.releaseGroup ? { releaseGroup: fx.quality.releaseGroup.name } : {}), + ...(fx.quality.hdr?.length ? { hdr: fx.quality.hdr } : {}), + } + : {}, + ), + content_edition: fx?.quality.cut ?? null, + source_release_id: dt?.source_release_id ?? null, + download_task_id: dt?.id ?? null, + status: "active", + verified_at: now, + }) + .returning("id") + .executeTakeFirstOrThrow(); + assetId = asset.id; + const loc = await trx + .insertInto("asset_location") + .values({ + media_asset_id: assetId, + location_kind: "library", + library_root_id: frozen.libraryRootId, + relative_path: f.target_path, + status: "available", + is_primary: true, + size_bytes: factsOf?.size ?? null, + inode: factsOf?.ino ?? null, + device: factsOf?.dev ?? null, + mtime: factsOf ? new Date(factsOf.mtimeMs) : null, + sidecar_of_location_id: parent?.locationId ?? null, + last_seen_at: now, + last_checked_at: now, + }) + .returning("id") + .executeTakeFirstOrThrow(); + locationId = loc.id; + if (isMain && fx?.episodes) { + const units = await trx + .selectFrom("content_unit") + .select("id") + .where("media_entity_id", "=", frozen.entityId) + .where("kind", "=", "episode") + .where("season_number", "=", fx.season ?? 1) + .where("episode_number", ">=", fx.episodes.start) + .where("episode_number", "<=", fx.episodes.end) + .execute(); + for (const u of units) { + await trx + .insertInto("asset_unit_coverage") + .values({ + media_asset_id: assetId, + content_unit_id: u.id, + coverage: "full", + evidence: jsonb({ from: "file_name", recordId }), + }) + .onConflict((oc) => oc.doNothing()) + .execute(); + } + } + } + assetBySrc.set(f.source_path, { assetId, locationId }); + if (isMain) mainAssets.push(assetId); + await trx + .updateTable("organization_file_result") + .set({ media_asset_id: assetId }) + .where("id", "=", f.id) + .execute(); + } + + const statuses = files.map((f) => f.status); + const status = organizationStatusFromFiles(statuses); + const counts = { + file_succeeded: statuses.filter((s) => s === "succeeded").length, + file_failed: statuses.filter((s) => s === "failed").length, + file_skipped: statuses.filter((s) => s === "skipped").length, + }; + const okAny = mainAssets.length > 0; + await trx + .updateTable("organization_record") + .set({ + status, + ...counts, + finished_at: now, + ...(status === "failed" && !rec.reason ? { reason: "transfer_failed" } : {}), + }) + .where("id", "=", recordId) + .execute(); + + // 下载任务 / 计划 / Slot + const move = rec.action === "move" && status === "succeeded"; + if (dt) { + await trx + .updateTable("download_task") + .set({ status: okAny ? "imported" : "completed" }) + .where("id", "=", dt.id) + .execute(); + if (dt.acquisition_plan_id) { + const plan = await trx + .selectFrom("acquisition_plan") + .select(["id", "slot_id", "purpose"]) + .where("id", "=", dt.acquisition_plan_id) + .executeTakeFirst(); + if (plan) { + await trx + .updateTable("acquisition_plan") + .set({ status: okAny ? "completed" : "failed", status_reason: `organize:${status}` }) + .where("id", "=", plan.id) + .execute(); + const slot = await trx + .selectFrom("acquisition_slot") + .selectAll() + .where("id", "=", plan.slot_id) + .executeTakeFirst(); + if (slot && okAny) { + const complete = await slotComplete(trx, slot, frozen, mainAssets); + const old = slot.current_asset_id; + await applySlotEvent( + d, + trx, + slot.id, + { type: "verify_passed", complete }, + { currentAssetId: mainAssets[0]!, reason: complete ? "verified" : "verified_partial" }, + ); + if (old && old !== mainAssets[0] && plan.purpose === "upgrade") { + // 升级:旧版本保留,处置只记待处理,不删 + await upsertDecision(d, trx, { + kind: "other", + severity: "info", + title: "旧版本待处置", + summary: "新版本已核验入库;旧资产保留,按媒体保留策略另行处理", + subjectType: "media_asset", + subjectId: old, + dedupeKey: `old_version:${old}`, + evidence: { slotId: slot.id, oldAssetId: old, newAssetId: mainAssets[0] }, + }); + } + if (complete && slot.subscription_id) { + await trx + .updateTable("subscription") + .set({ status: "completed", status_reason: "slot_satisfied" }) + .where("id", "=", slot.subscription_id) + .where("mode", "=", "once") + .execute(); + } + } else if (slot && !okAny) { + await applySlotEvent(d, trx, slot.id, { type: "verify_failed" }, { reason: `organize:${status}` }); + } + } + } + } + await emitRecord(d, trx, recordId); + return { recordId, status, succeeded: counts.file_succeeded, failed: counts.file_failed, move, dt: dt ?? null }; + }); + + // Move 完成:移除 qB 任务记录,结束下载侧管理(不删数据) + if (result.move && result.dt) { + const topo = (await loadInstances(db, { enabledOnly: false, ids: [result.dt.instance_id] }))[0]; + if (topo) { + try { + const client = await d.ports.downloader(topo.inst); + await client.delete([result.dt.torrent_hash], { deleteFiles: false }); + await db + .updateTable("download_task") + .set({ removed_at: d.ports.now(), managed: false }) + .where("id", "=", result.dt.id) + .execute(); + } catch (err) { + d.ctx.logger.warn( + { downloadTaskId: result.dt.id, err: { message: errMessage(err) } }, + "Move 后移除 qB 任务失败,下次同步重试", + ); + } + } + } + return { recordId: result.recordId, status: result.status, succeeded: result.succeeded, failed: result.failed }; +} + +async function slotComplete( + exec: Exec, + slot: { id: string; media_entity_id: string; content_unit_id: string | null; completion: unknown }, + frozen: FrozenPlan, + newAssets: string[], +): Promise { + const completion = (slot.completion ?? {}) as { kind?: string; cuts?: string[] }; + if (frozen.mediaType === "movie") { + const assets = await exec + .selectFrom("media_asset") + .select(["id", "content_edition"]) + .where("id", "in", newAssets) + .execute(); + return movieCompletion( + completion.cuts as never, + assets.map((a) => ({ + assetId: a.id, + verified: true, + ...(a.content_edition ? { cut: a.content_edition as never } : {}), + })), + ).complete; + } + // 剧集/综艺:按内容单元覆盖;单元未建档(总集数未知)时继续追踪,不声称完整 + let q = exec + .selectFrom("content_unit") + .select("id") + .where("media_entity_id", "=", slot.media_entity_id) + .where("kind", "=", "episode") + .where("content_type", "=", "main"); + if (slot.content_unit_id) q = q.where("parent_unit_id", "=", slot.content_unit_id); + const required = (await q.execute()).map((u) => u.id); + if (required.length === 0) return false; + const cov = await sql<{ asset_id: string; unit_id: string }>` + select c.media_asset_id as asset_id, c.content_unit_id as unit_id from asset_unit_coverage c + join media_asset a on a.id = c.media_asset_id + where a.media_entity_id = ${slot.media_entity_id} and a.status = 'active'`.execute(exec); + const byAsset = new Map(); + for (const r of cov.rows) byAsset.set(r.asset_id, [...(byAsset.get(r.asset_id) ?? []), r.unit_id]); + return unitCompletion( + required, + [...byAsset].map(([assetId, fileUnits]) => ({ assetId, verified: true, fileUnits })), + { totalKnown: true }, + ).complete; +} + +// ── 公开记录映射 ── + +export async function loadRecordsPublic(exec: Exec, rows: OrganizationRecordRow[], includeFiles: boolean) { + if (rows.length === 0) return []; + const ids = rows.map((r) => r.id); + const entityIds = [...new Set(rows.map((r) => r.media_entity_id).filter((x): x is string => !!x))]; + const [files, ops, entities, idents, units] = await Promise.all([ + includeFiles + ? exec + .selectFrom("organization_file_result") + .selectAll() + .where("organization_record_id", "in", ids) + .orderBy("ordinal") + .execute() + : Promise.resolve([]), + exec + .selectFrom("asset_operation") + .select(["id", "organization_record_id"]) + .where("organization_record_id", "in", ids) + .execute(), + entityIds.length + ? exec.selectFrom("media_entity").selectAll().where("id", "in", entityIds).execute() + : Promise.resolve([]), + entityIds.length + ? exec + .selectFrom("external_identity") + .select(["media_entity_id", "provider", "entity_type", "external_id"]) + .where("media_entity_id", "in", entityIds) + .where("status", "=", "accepted") + .execute() + : Promise.resolve([]), + exec + .selectFrom("content_unit") + .select(["id", "season_number", "episode_number"]) + .where("id", "in", [ + ...new Set(rows.map((r) => r.content_unit_id).filter((x): x is string => !!x)), + "00000000-0000-0000-0000-000000000000", + ]) + .execute(), + ]); + const iso = (v: Date | null) => (v ? v.toISOString() : null); + const reason = (code: string | null, err: unknown) => + code + ? { code: code.split(":")[0]!, message: code } + : err + ? { + code: (err as { code?: string }).code ?? "error", + message: (err as { message?: string }).message ?? JSON.stringify(err).slice(0, 300), + } + : null; + return rows.map((r) => { + const e = entities.find((x) => x.id === r.media_entity_id); + const u = units.find((x) => x.id === r.content_unit_id); + const targetRef = { + path: r.target_path, + view: "library" as const, + downloaderInstanceId: null, + libraryRootId: r.target_library_root_id, + }; + return { + id: r.id, + taskId: r.task_id, + operationIds: ops.filter((o) => o.organization_record_id === r.id).map((o) => o.id), + media: e + ? { + entityId: e.id, + kind: e.kind, + title: e.title, + year: e.year, + contentUnitId: r.content_unit_id, + seasonNumber: u?.season_number ?? null, + episodeNumber: u?.episode_number ?? null, + externalIds: idents + .filter((i) => i.media_entity_id === e.id) + .map((i) => ({ provider: i.provider, entityType: i.entity_type, externalId: i.external_id })), + } + : null, + trigger: r.trigger, + origin: r.trigger === "manual" ? ("manual" as const) : ("auto" as const), + actor: { type: r.actor_type, id: r.actor_id }, + action: r.action, + status: r.status, + source: { + path: r.source_path, + view: r.source_path_view, + downloaderInstanceId: r.source_downloader_instance_id, + libraryRootId: null, + }, + target: targetRef, + reason: reason(r.reason, r.error && r.status === "failed" ? r.error : null), + counts: { total: r.file_total, succeeded: r.file_succeeded, failed: r.file_failed, skipped: r.file_skipped }, + files: includeFiles + ? files + .filter((f) => f.organization_record_id === r.id) + .map((f) => ({ + id: f.id, + ordinal: f.ordinal, + role: f.role, + mode: f.mode, + status: f.status, + source: { + path: f.source_path, + view: r.source_path_view, + downloaderInstanceId: r.source_downloader_instance_id, + libraryRootId: null, + }, + target: { + path: f.target_path, + view: "library" as const, + downloaderInstanceId: null, + libraryRootId: f.target_library_root_id, + }, + sizeBytes: f.source_size_bytes, + contentUnitId: f.content_unit_id, + mediaAssetId: f.media_asset_id, + operationId: f.asset_operation_id, + skipReason: f.skip_reason ? { code: f.skip_reason.split(":")[0]!, message: f.skip_reason } : null, + failure: f.status === "failed" && f.error ? reason(null, f.error) : null, + startedAt: iso(f.started_at), + finishedAt: iso(f.finished_at), + })) + : null, + createdAt: r.created_at.toISOString(), + updatedAt: r.updated_at.toISOString(), + startedAt: iso(r.started_at), + finishedAt: iso(r.finished_at), + changeSeq: String(r.change_seq), + }; + }); +} diff --git a/apps/server/src/modules/acquisition-organize/services/reconcile.ts b/apps/server/src/modules/acquisition-organize/services/reconcile.ts new file mode 100644 index 00000000..52117373 --- /dev/null +++ b/apps/server/src/modules/acquisition-organize/services/reconcile.ts @@ -0,0 +1,228 @@ +import { join } from "node:path"; +import { checkRoot, DEFAULT_ROOT_SENTINEL, statSafe } from "@umt/adapters"; +import { errMessage, jsonb } from "../util.ts"; +import type { AoDeps } from "./common.ts"; +import { applySlotEvent } from "./downloads.ts"; + +// 资产对账(Asset Reconciler,REQ §8.1/§8.2):先查库根哨兵再增量 stat; +// 状态按 available → unavailable / suspected_missing → confirmed_missing 推进,不因单次失败删记录或补下载。 + +export interface ReconcileInput { + libraryRootId?: string | undefined; + reconcileRunId?: string | undefined; + /** 每个库根每轮最多检查的位置数(续扫靠 resume_cursor) */ + batch?: number; +} + +export interface ReconcileStats { + checked: number; + available: number; + unavailable: number; + suspectedMissing: number; + confirmedMissing: number; + rootsOffline: string[]; +} + +export async function reconcileLibrary( + d: AoDeps, + input: ReconcileInput = {}, +): Promise<{ runId: string; stats: ReconcileStats }> { + const db = d.ctx.db; + const automation = await d.ctx.settings.get("automation"); + const readOnly = automation.readOnlyReconcile; + const now = d.ports.now(); + let run = input.reconcileRunId + ? await db.selectFrom("reconcile_run").selectAll().where("id", "=", input.reconcileRunId).executeTakeFirst() + : undefined; + if (!run) { + run = await db + .insertInto("reconcile_run") + .values({ + kind: "asset", + mode: readOnly ? "read_only" : "normal", + status: "running", + scope: jsonb({ libraryRootId: input.libraryRootId ?? null }), + started_at: now, + }) + .returningAll() + .executeTakeFirstOrThrow(); + } else { + await db + .updateTable("reconcile_run") + .set({ status: "running", started_at: run.started_at ?? now }) + .where("id", "=", run.id) + .execute(); + } + const cursor = (run.resume_cursor ?? {}) as Record; + const stats: ReconcileStats = { + checked: 0, + available: 0, + unavailable: 0, + suspectedMissing: 0, + confirmedMissing: 0, + rootsOffline: [], + }; + const batch = input.batch ?? 2000; + const confirmAfter = d.ports.options.missingConfirmAfter; + let partial = false; + try { + let rq = db.selectFrom("library_root").selectAll().where("enabled", "=", true); + const scopeRoot = input.libraryRootId ?? ((run.scope ?? {}) as { libraryRootId?: string }).libraryRootId; + if (scopeRoot) rq = rq.where("id", "=", scopeRoot); + for (const root of await rq.execute()) { + const health = await checkRoot(root.local_path, root.sentinel_file ?? DEFAULT_ROOT_SENTINEL); + await db + .updateTable("library_root") + .set({ + health_status: health.kind === "online" ? "healthy" : "offline", + last_health_at: now, + health_detail: jsonb(health), + }) + .where("id", "=", root.id) + .execute(); + if (health.kind === "offline") { + // 根离线:只标记不可用,绝不据此判定缺失 + stats.rootsOffline.push(root.id); + const r = await db + .updateTable("asset_location") + .set({ + status: "unavailable", + last_checked_at: now, + check_detail: jsonb({ reason: `root_offline:${health.reason}` }), + }) + .where("library_root_id", "=", root.id) + .where("status", "=", "available") + .executeTakeFirst(); + stats.unavailable += Number(r.numUpdatedRows); + continue; + } + let q = db + .selectFrom("asset_location") + .selectAll() + .where("library_root_id", "=", root.id) + .where("status", "!=", "removed") + .orderBy("id") + .limit(batch + 1); + if (cursor[root.id]) q = q.where("id", ">", cursor[root.id]!); + const locs = await q.execute(); + const more = locs.length > batch; + for (const loc of locs.slice(0, batch)) { + stats.checked++; + const st = await statSafe(join(root.local_path, loc.relative_path), { + root: root.local_path, + sentinel: root.sentinel_file ?? DEFAULT_ROOT_SENTINEL, + }); + const detail = (loc.check_detail ?? {}) as { missCount?: number }; + if (st.kind === "ok") { + stats.available++; + await db + .updateTable("asset_location") + .set({ + status: "available", + last_seen_at: now, + last_checked_at: now, + check_detail: jsonb({ + missCount: 0, + ...(loc.size_bytes !== null && loc.size_bytes !== st.facts.size + ? { sizeChanged: { from: loc.size_bytes, to: st.facts.size } } + : {}), + }), + }) + .where("id", "=", loc.id) + .execute(); + // 文件回来了:资产恢复 active(Slot 的 attention 仍留给人工确认) + await db + .updateTable("media_asset") + .set({ status: "active" }) + .where("id", "=", loc.media_asset_id) + .where("status", "=", "missing") + .execute(); + } else if (st.kind === "missing") { + const missCount = (detail.missCount ?? 0) + 1; + const confirmed = missCount >= confirmAfter; + if (confirmed) stats.confirmedMissing++; + else stats.suspectedMissing++; + await db.transaction().execute(async (trx) => { + await trx + .updateTable("asset_location") + .set({ + status: confirmed ? "confirmed_missing" : "suspected_missing", + last_checked_at: now, + check_detail: jsonb({ missCount }), + }) + .where("id", "=", loc.id) + .execute(); + if (confirmed && loc.status !== "confirmed_missing" && !readOnly) { + const other = await trx + .selectFrom("asset_location") + .select("id") + .where("media_asset_id", "=", loc.media_asset_id) + .where("id", "!=", loc.id) + .where("status", "=", "available") + .executeTakeFirst(); + if (!other) { + await trx + .updateTable("media_asset") + .set({ status: "missing" }) + .where("id", "=", loc.media_asset_id) + .where("status", "=", "active") + .execute(); + const slots = await trx + .selectFrom("acquisition_slot") + .select("id") + .where("current_asset_id", "=", loc.media_asset_id) + .execute(); + // 只推进到 attention 等人处理;不自动补下载(确认缺失仍需遵循恢复策略) + for (const s of slots) + await applySlotEvent( + d, + trx, + s.id, + { type: "asset_missing_confirmed" }, + { reason: "asset_confirmed_missing" }, + ); + } + } + }); + } else { + stats.unavailable++; + await db + .updateTable("asset_location") + .set({ status: "unavailable", last_checked_at: now, check_detail: jsonb({ ...detail, reason: st.kind }) }) + .where("id", "=", loc.id) + .execute(); + } + } + if (more) { + partial = true; + cursor[root.id] = locs[batch - 1]!.id; + } else { + delete cursor[root.id]; + } + } + await db + .updateTable("reconcile_run") + .set({ + status: partial ? "partial" : "succeeded", + stats: jsonb(stats), + resume_cursor: jsonb(cursor), + finished_at: d.ports.now(), + }) + .where("id", "=", run.id) + .execute(); + } catch (err) { + await db + .updateTable("reconcile_run") + .set({ + status: "failed", + stats: jsonb(stats), + resume_cursor: jsonb(cursor), + error: jsonb({ message: errMessage(err) }), + finished_at: d.ports.now(), + }) + .where("id", "=", run.id) + .execute(); + throw err; + } + return { runId: run.id, stats }; +} diff --git a/apps/server/src/modules/acquisition-organize/services/retention.ts b/apps/server/src/modules/acquisition-organize/services/retention.ts new file mode 100644 index 00000000..57e612ed --- /dev/null +++ b/apps/server/src/modules/acquisition-organize/services/retention.ts @@ -0,0 +1,36 @@ +import type { DownloadTaskRow, SiteRow, SourceReleaseRow } from "@umt/db"; + +// H&R / 做种保留判断:与用户保留天数分开(REQ §8.3)。未知一律按"未满足"对待,不据此删除或 Move。 + +export interface HnrRequirement { + /** 站点 config.hitAndRun:{ minSeedingSec?, minRatio? };任一满足即解除 */ + minSeedingSec?: number; + minRatio?: number; +} + +export function releaseHnr(rel: Pick | null | undefined): boolean | "unknown" { + const site = ((rel?.raw ?? {}) as { _site?: { hr?: unknown } })._site; + const hr = site?.hr; + return hr === true ? true : hr === false ? false : "unknown"; +} + +export function hnrSatisfied( + task: Pick, + rel: Pick | null | undefined, + site: Pick | null | undefined, +): boolean | "unknown" { + const hr = releaseHnr(rel); + if (hr === false) return true; + if (hr === "unknown") return "unknown"; + const req = ((site?.config ?? {}) as { hitAndRun?: HnrRequirement }).hitAndRun; + if (!req || (req.minSeedingSec === undefined && req.minRatio === undefined)) return "unknown"; + const bySeed = req.minSeedingSec !== undefined && (task.seeding_time_sec ?? 0) >= req.minSeedingSec; + const byRatio = req.minRatio !== undefined && (task.ratio ?? 0) >= req.minRatio; + return bySeed || byRatio; +} + +/** 用户保留天数:从下载完成起算 */ +export function retentionElapsed(task: Pick, days: number, now: Date): boolean { + if (!task.completed_at) return false; + return now.getTime() - task.completed_at.getTime() >= days * 86_400_000; +} diff --git a/apps/server/src/modules/acquisition-organize/services/volumes.ts b/apps/server/src/modules/acquisition-organize/services/volumes.ts new file mode 100644 index 00000000..fc251dbe --- /dev/null +++ b/apps/server/src/modules/acquisition-organize/services/volumes.ts @@ -0,0 +1,121 @@ +import type { DiskUsage } from "@umt/adapters"; +import type { CacheSettings } from "@umt/contracts"; +import { + type CacheVolumeDecision, + evaluateCacheVolume, + type InflightTask, + type Reservation, + type VolumeSnapshot, +} from "@umt/domain"; +import type { AoDeps, Exec, InstanceTopology } from "./common.ts"; +import { loadInstances } from "./common.ts"; + +// 缓存卷:多个 qB 共用同一卷时按 cache_volume_key 汇总在途需求与预留(REQ §8.4.1)。 + +export interface VolumeState { + volumeKey: string; + instances: InstanceTopology[]; + /** statfs 用的本机路径(第一个有保存路径的实例) */ + path: string | null; + usage: DiskUsage | null; + snapshot: VolumeSnapshot | null; + inflight: InflightTask[]; + reservations: Reservation[]; + /** 该卷上存在因阈值暂停的纳管任务 */ + pausedByThreshold: boolean; +} + +const INFLIGHT_STATUS: Record = { + queued: "queued", + submitting: "queued", + downloading: "downloading", + stalled: "downloading", + paused: "paused", + completed: "completed", + importing: "completed", + imported: "completed", +}; + +export async function loadVolumes( + d: AoDeps, + exec: Exec = d.ctx.db, + instances?: InstanceTopology[], +): Promise { + const insts = instances ?? (await loadInstances(exec)); + const byKey = new Map(); + for (const t of insts) byKey.set(t.volumeKey, [...(byKey.get(t.volumeKey) ?? []), t]); + if (byKey.size === 0) return []; + + const tasks = await exec + .selectFrom("download_task") + .select([ + "id", + "instance_id", + "acquisition_plan_id", + "size_bytes", + "downloaded_bytes", + "managed", + "status", + "pause_reason", + ]) + .where( + "instance_id", + "in", + insts.map((i) => i.inst.id), + ) + .where("removed_at", "is", null) + .where("status", "not in", ["removed", "failed", "missing"]) + .execute(); + const plans = await exec + .selectFrom("acquisition_plan") + .select(["id", "reserved_bytes", "evaluation"]) + .where("status", "=", "reserved") + .execute(); + + const out: VolumeState[] = []; + for (const [volumeKey, members] of byKey) { + const ids = new Set(members.map((m) => m.inst.id)); + const mine = tasks.filter((t) => ids.has(t.instance_id)); + const path = members.find((m) => m.localSavePath)?.localSavePath ?? null; + const usage = path ? await d.ports.diskUsage(path) : null; + const snapshot: VolumeSnapshot | null = + usage?.status === "ok" + ? { volumeId: volumeKey, totalBytes: usage.usedBytes + usage.availableBytes, usedBytes: usage.usedBytes } + : null; + out.push({ + volumeKey, + instances: members, + path, + usage, + snapshot, + inflight: mine.map((t) => ({ + taskId: t.id, + ...(t.acquisition_plan_id ? { planId: t.acquisition_plan_id } : {}), + volumeId: volumeKey, + ...(t.size_bytes !== null ? { sizeBytes: t.size_bytes } : {}), + onDiskBytes: t.downloaded_bytes, + managed: t.managed, + status: INFLIGHT_STATUS[t.status] ?? "other", + })), + reservations: plans + .filter((p) => (p.evaluation as { volumeKey?: string } | null)?.volumeKey === volumeKey) + .map((p) => ({ planId: p.id, volumeId: volumeKey, bytes: p.reserved_bytes })), + pausedByThreshold: mine.some((t) => t.managed && t.status === "paused" && t.pause_reason === "threshold"), + }); + } + return out; +} + +export function decideVolume(v: VolumeState, cache: CacheSettings, request?: { bytes?: number }): CacheVolumeDecision { + return evaluateCacheVolume({ + volume: v.snapshot, + config: { hardPct: cache.hardThresholdPercent, resumePct: cache.resumePercent }, + inflight: v.inflight, + reservations: v.reservations, + ...(request ? { request } : {}), + pausedByThreshold: v.pausedByThreshold, + }); +} + +export const usedPercent = (v: VolumeState): number | null => + v.snapshot ? (v.snapshot.usedBytes / v.snapshot.totalBytes) * 100 : null; diff --git a/apps/server/src/modules/acquisition-organize/util.ts b/apps/server/src/modules/acquisition-organize/util.ts new file mode 100644 index 00000000..fcef2bed --- /dev/null +++ b/apps/server/src/modules/acquisition-organize/util.ts @@ -0,0 +1,89 @@ +import { createHash } from "node:crypto"; +import type { Context } from "hono"; +import type { AuthEnv } from "../../http/auth.ts"; +import { ApiException } from "../../http/errors.ts"; + +// 模块内通用小工具:行转换、哈希、游标、错误。 + +const camel = (s: string) => s.replace(/_([a-z0-9])/g, (_, c: string) => c.toUpperCase()); + +/** + * db 行(snake_case、Date)→ contracts 形状(camelCase、ISO 串)。 + * contracts 的 Row schema 与表列一一对应(db-sync 测试保证),因此可以通用转换。 + */ +export function toApi(row: object): T { + const out: Record = {}; + for (const [k, v] of Object.entries(row)) out[camel(k)] = v instanceof Date ? v.toISOString() : v; + return out as T; +} + +export const toApiList = (rows: readonly object[]): T[] => rows.map((r) => toApi(r)); + +/** 规范化 JSON(键排序)后 sha256,用于幂等键 / payloadHash / 指纹 */ +export function canonicalJson(v: unknown): string { + if (Array.isArray(v)) return `[${v.map(canonicalJson).join(",")}]`; + if (v && typeof v === "object" && !(v instanceof Date)) { + const o = v as Record; + return `{${Object.keys(o) + .filter((k) => o[k] !== undefined) + .sort() + .map((k) => `${JSON.stringify(k)}:${canonicalJson(o[k])}`) + .join(",")}}`; + } + return JSON.stringify(v instanceof Date ? v.toISOString() : v) ?? "null"; +} + +export const sha256 = (s: string) => createHash("sha256").update(s, "utf8").digest("hex"); +export const hashJson = (v: unknown) => sha256(canonicalJson(v)); + +/** jsonb 写入:一律 stringify(pg 会把 JS 数组当 PG 数组) */ +export const jsonb = (v: unknown) => JSON.stringify(v ?? {}); + +/** 按 (created_at desc, id desc) 翻页的不透明游标 */ +export function encodeCursor(at: Date | string, id: string): string { + const t = at instanceof Date ? at.toISOString() : at; + return Buffer.from(`${t}|${id}`, "utf8").toString("base64url"); +} + +export function decodeCursor(cursor: string | undefined): { at: string; id: string } | null { + if (!cursor) return null; + const raw = Buffer.from(cursor, "base64url").toString("utf8"); + const i = raw.lastIndexOf("|"); + const at = raw.slice(0, i); + const id = raw.slice(i + 1); + if (i <= 0 || Number.isNaN(Date.parse(at)) || !/^[0-9a-f-]{36}$/i.test(id)) { + throw new ApiException("bad_request", "cursor 无效"); + } + return { at, id }; +} + +/** 统一成功信封 */ +export function ok(c: Context, data: T, status: 200 | 201 = 200) { + return c.json({ ok: true as const, data }, status); +} + +export const notFound = (what: string): never => { + throw new ApiException("not_found", `${what}不存在`); +}; +export const conflict = (msg: string): never => { + throw new ApiException("conflict", msg); +}; + +/** 调用者标识(写 created_by / granted_by 等) */ +export function actorOf(c: Context): string { + const p = c.get("principal"); + return p ? `${p.kind}:${p.id}` : "system"; +} + +export const errMessage = (e: unknown) => (e instanceof Error ? e.message : String(e)); + +/** 当天 00:00(UTC)起点,用于每日预算与今日统计 */ +export function startOfDay(now: Date): Date { + return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate())); +} + +export const posixJoin = (...parts: string[]) => + parts + .filter((p) => p.length > 0) + .join("/") + .replace(/\/{2,}/g, "/"); diff --git a/apps/server/test/modules.test.ts b/apps/server/test/modules.test.ts index bffa739b..d1ddd0f5 100644 --- a/apps/server/test/modules.test.ts +++ b/apps/server/test/modules.test.ts @@ -38,7 +38,8 @@ const pingModule: ServerModule = { }), ); }, - outboxRoutes: () => ({ "release.resolved": () => ({ task: "evaluateAcquisition", input: {} }) }), + // 测试专用事件类型:真实事件(如 release.resolved)已由业务模块认领 + outboxRoutes: () => ({ "test.ping": () => ({ task: "evaluateAcquisition", input: {} }) }), }; describe("ServerModule 骨架", () => { @@ -81,7 +82,9 @@ describe("ServerModule 骨架", () => { }); it("outbox 路由合并;同一事件被两个模块认领报错", () => { - expect(Object.keys(collectOutboxRoutes([...SERVER_MODULES, pingModule], fakeCtx))).toEqual(["release.resolved"]); + const merged = Object.keys(collectOutboxRoutes([...SERVER_MODULES, pingModule], fakeCtx)); + expect(merged).toContain("test.ping"); + expect(merged).toContain("release.resolved"); expect(() => collectOutboxRoutes([pingModule, { ...pingModule, name: "dup" }], fakeCtx)).toThrow(/重复认领/); }); diff --git a/packages/contracts/src/organization.ts b/packages/contracts/src/organization.ts index c8d8d8fc..42d265c2 100644 --- a/packages/contracts/src/organization.ts +++ b/packages/contracts/src/organization.ts @@ -1,6 +1,11 @@ import { z } from "zod"; +import { AgentRunStatus, DecisionKind } from "./agent.ts"; +import { AssetLocation, MediaAsset, PathBinding } from "./assets.ts"; import { ActorType, EntityBase, Id, JsonObject, NonNegInt, Timestamp } from "./common.ts"; -import { MediaKind } from "./content.ts"; +import { MediaEntity, MediaKind } from "./content.ts"; +import { AcquisitionPlan, DownloadStatus, DownloadTask } from "./execution.ts"; +import { ExternalIdentity } from "./identity.ts"; +import { AcquisitionSlot, CandidateEvaluation, SlotStatus } from "./policy.ts"; // 整理域:整理记录(批次)+ 逐文件结果,以及对外开放的整理记录 API 合同(REQ §12.4) // 自动与手动整理产生同格式记录;读记录需 organize.read,发起整理需 organize.execute。 @@ -286,3 +291,242 @@ export const OrganizeAccepted = z.object({ planId: Id.nullable(), }); export type OrganizeAccepted = z.infer; + +// ── 媒体库 / 库存叠加 / 首页聚合(W2-C 读模型,均为只读投影) ── + +/** 各 Slot 状态计数(不用 record:键集固定,缺省 0) */ +export const SlotStatusCounts = z.object({ + empty: NonNegInt, + reserved: NonNegInt, + downloading: NonNegInt, + verifying: NonNegInt, + satisfied: NonNegInt, + upgrade_pending: NonNegInt, + attention: NonNegInt, +}); +export type SlotStatusCounts = z.infer; + +export const LibraryListQuery = z.object({ + cursor: z.string().min(1).optional(), + limit: z.coerce.number().int().min(1).max(200).default(50), + kind: MediaKind.optional(), + /** 标题/原名/别名模糊搜索 */ + q: z.string().max(200).optional(), + /** gap = 有未满足 Slot;downloading = 有下载中任务;attention = 有 Slot 需处理 */ + filter: z.enum(["gap", "downloading", "attention", "satisfied"]).optional(), +}); +export type LibraryListQuery = z.infer; + +export const LibraryItem = z.object({ + entity: MediaEntity.pick({ id: true, kind: true, title: true, originalTitle: true, year: true, status: true }), + assetCount: NonNegInt, + slotCounts: SlotStatusCounts, + activeDownloads: NonNegInt, + lastOrganizedAt: Timestamp.nullable(), +}); +export type LibraryItem = z.infer; + +/** 资产位置 + 执行进程可访问的实际路径(库根离线时仍返回拼接路径,状态看 location.status) */ +export const LibraryAssetView = z.object({ + asset: MediaAsset, + locations: z.array(AssetLocation.extend({ absolutePath: z.string().nullable() })), + /** 覆盖的内容单元 */ + contentUnitIds: z.array(Id), +}); +export type LibraryAssetView = z.infer; + +export const LibraryDownloadView = z.object({ + downloadTask: DownloadTask, + plan: AcquisitionPlan.nullable(), + /** 该下载任务的整理记录(含文件级核验结果) */ + organizationRecordIds: z.array(Id), + lastOrganizationStatus: OrganizationStatus.nullable(), +}); +export type LibraryDownloadView = z.infer; + +export const LibraryDetail = z.object({ + entity: MediaEntity, + /** 身份映射状态(含 proposed/conflicted,前端区分展示) */ + identities: z.array(ExternalIdentity), + /** 元数据来源:metadata 中各字段的来源标注 + 人工锁 */ + metadataSources: z.object({ fields: JsonObject, fieldLocks: z.array(z.string()) }), + pathBindings: z.array(PathBinding), + assets: z.array(LibraryAssetView), + slots: z.array(AcquisitionSlot), + candidates: z.array(CandidateEvaluation), + downloads: z.array(LibraryDownloadView), +}); +export type LibraryDetail = z.infer; + +/** 站点工作台叠加:按作品返回已有版本 / Slot 缺口 / 下载中 */ +export const InventoryQuery = z.object({ + /** 逗号分隔的作品 UUID,最多 200 个 */ + entityIds: z.string().min(1).max(8000), +}); +export type InventoryQuery = z.infer; + +export const InventoryEntry = z.object({ + entityId: Id, + /** 已入库版本(主视频资产的交付特征) */ + versions: z.array( + z.object({ + assetId: Id, + resolution: z.string().nullable(), + sourceType: z.string().nullable(), + releaseGroup: z.string().nullable(), + contentEdition: z.string().nullable(), + sizeBytes: NonNegInt.nullable(), + available: z.boolean(), + }), + ), + /** 未满足的 Slot(empty/attention/upgrade_pending) */ + gaps: z.array(z.object({ slotId: Id, name: z.string(), status: SlotStatus })), + downloading: z.array( + z.object({ downloadTaskId: Id, name: z.string(), status: DownloadStatus, progress: z.number().min(0).max(1) }), + ), +}); +export type InventoryEntry = z.infer; + +export const InventoryResponse = z.object({ items: z.array(InventoryEntry) }); +export type InventoryResponse = z.infer; + +/** Emby 兼容库存:按实例分别给出 present / not_found / unknown(未知 ≠ 缺失) */ +export const MediaServerInventoryQuery = z.object({ mediaEntityId: Id }); +export type MediaServerInventoryQuery = z.infer; + +export const MediaServerInventory = z.object({ + mediaEntityId: Id, + instances: z.array( + z.object({ + instanceId: Id, + name: z.string(), + status: z.enum(["present", "not_found", "unknown"]), + reason: z.string().nullable(), + items: z.array( + z.object({ + itemId: z.string(), + type: z.string(), + name: z.string(), + year: z.number().int().nullable(), + path: z.string().nullable(), + versions: z.array(JsonObject), + }), + ), + observedAt: Timestamp, + }), + ), +}); +export type MediaServerInventory = z.infer; + +/** 首页六区块(REQ §13.4.3);统计未知时对应字段为 null 并在 notes 说明 */ +export const Dashboard = z.object({ + generatedAt: Timestamp, + automation: z.object({ + enabled: z.boolean(), + readOnlyReconcile: z.boolean(), + today: z.object({ + plansCreated: NonNegInt, + downloadsSubmitted: NonNegInt, + filesImported: NonNegInt, + recordsSucceeded: NonNegInt, + recordsFailed: NonNegInt, + upgrades: NonNegInt, + }), + activeSubscriptions: NonNegInt, + slotCounts: SlotStatusCounts, + }), + currentWork: z.object({ + downloading: NonNegInt, + paused: NonNegInt, + stalled: NonNegInt, + importing: NonNegInt, + reservedPlans: NonNegInt, + runningRecords: NonNegInt, + agentRuns: z.array(z.object({ id: Id, goal: z.string(), status: AgentRunStatus })), + items: z.array( + z.object({ + downloadTaskId: Id, + name: z.string(), + status: DownloadStatus, + progress: z.number().min(0).max(1), + waitReason: z.string().nullable(), + }), + ), + }), + needsAttention: z.object({ + total: NonNegInt, + byKind: z.array(z.object({ kind: DecisionKind, count: NonNegInt })), + }), + cache: z.object({ + volumes: z.array( + z.object({ + volumeKey: z.string(), + instanceIds: z.array(Id), + usedPercent: z.number().min(0).max(100).nullable(), + totalBytes: NonNegInt.nullable(), + usedBytes: NonNegInt.nullable(), + pendingBytes: NonNegInt, + reservedBytes: NonNegInt, + hardThresholdPercent: z.number(), + resumePercent: z.number(), + pausedByThreshold: NonNegInt, + cleanableBytes: NonNegInt, + retentionLockedBytes: NonNegInt, + notes: z.array(z.string()), + }), + ), + }), + sources: z.object({ + sites: z.array( + z.object({ + id: Id, + name: z.string(), + enabled: z.boolean(), + health: z.string(), + lastSnapshotAt: Timestamp.nullable(), + }), + ), + feeds: z.array( + z.object({ + id: Id, + name: z.string(), + enabled: z.boolean(), + health: z.string(), + lastSuccessAt: Timestamp.nullable(), + consecutiveFailures: NonNegInt, + lastError: z.string().nullable(), + }), + ), + downloaders: z.array( + z.object({ + id: Id, + name: z.string(), + enabled: z.boolean(), + health: z.string(), + lastSeenAt: Timestamp.nullable(), + }), + ), + mediaServers: z.array( + z.object({ + id: Id, + name: z.string(), + enabled: z.boolean(), + health: z.string(), + lastSeenAt: Timestamp.nullable(), + }), + ), + }), + recentOutput: z.array( + z.object({ + recordId: Id, + mediaEntityId: Id.nullable(), + title: z.string().nullable(), + action: OrganizationAction, + status: OrganizationStatus, + targetPath: z.string().nullable(), + counts: z.object({ total: NonNegInt, succeeded: NonNegInt, failed: NonNegInt, skipped: NonNegInt }), + finishedAt: Timestamp.nullable(), + }), + ), +}); +export type Dashboard = z.infer; diff --git a/packages/contracts/src/policy.ts b/packages/contracts/src/policy.ts index 0884246d..a55753d1 100644 --- a/packages/contracts/src/policy.ts +++ b/packages/contracts/src/policy.ts @@ -245,3 +245,54 @@ export type PreferenceSuppressionCreate = z.infer; + +// ── 收录策略规则与候选评估(W2-C) ── + +/** Slot 模板:策略规则里用于自动建 Slot(新作品收录 / 订阅缺省 Slot);字段同 AcquisitionSlot 规则列 */ +export const SlotTemplate = z.object({ + name: z.string().min(1).max(64), + hardConditions: JsonObject.optional(), + softPreferences: JsonObject.optional(), + fallback: JsonObject.optional(), + upgradeThreshold: JsonObject.optional(), + completion: JsonObject.optional(), +}); +export type SlotTemplate = z.infer; + +/** + * kind=acquisition 的 policy_revision.rules 结构(结构化数据,不是可执行脚本)。 + * 发布即对 scope 范围内的正常下载/追更/整理生成持续授权 automation_grant(REQ §6.4)。 + */ +export const AcquisitionPolicyRules = z.object({ + /** 新作品收录:发行/首播在窗口内且库里没有的作品,按模板建订阅与 Slot */ + newTitle: z + .object({ + enabled: z.boolean(), + windowDays: z.number().int().min(1).max(3650).optional(), + mediaKinds: z.array(SubscriptionMediaKind).optional(), + slots: z.array(SlotTemplate), + }) + .optional(), + /** 订阅没有 Slot 时套用的缺省 Slot */ + defaultSlots: z.array(SlotTemplate).optional(), + /** 整理方式覆盖(缺省读 settings.organize.defaultMode) */ + organizeMode: z.enum(["copy", "move"]).optional(), + /** Move 在 H&R 未满足时经授权改用 Copy */ + allowCopyWhenHnrPending: z.boolean().optional(), +}); +export type AcquisitionPolicyRules = z.infer; + +/** 候选对某个 Slot 的确定性评估(供作品详情"为什么选/不选") */ +export const CandidateEvaluation = z.object({ + slotId: Id, + sourceReleaseId: Id, + title: z.string(), + /** accept / upgrade / reject / unknown_field / skipped(被模式、预算、阈值、总开关等挡住) */ + decision: z.enum(["accept", "upgrade", "reject", "unknown_field", "skipped"]), + mode: z.enum(["new_title", "catch_up", "upgrade"]).nullable(), + rejected: z.array(z.object({ code: z.string(), message: z.string() })), + reasons: z.array(z.string()), + planId: Id.nullable(), + evaluatedAt: Timestamp, +}); +export type CandidateEvaluation = z.infer;