diff --git a/.env.example b/.env.example index f7bbd79a..16644b25 100644 --- a/.env.example +++ b/.env.example @@ -7,3 +7,5 @@ LOG_LEVEL=info NODE_ENV=development # 开发模式下可直接用此 Key 以 admin scope 访问 API;生产环境忽略 DEV_ADMIN_KEY=umt-dev-admin-key +# Agent(Pi)JSONL 会话目录:api 与 worker 进程需共享同一目录(崩溃恢复点);缺省 ./data/agent-sessions +# AGENT_SESSION_DIR=./data/agent-sessions diff --git a/.gitignore b/.gitignore index 1a1ffd42..5192453e 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ coverage/ .vite/ types/deno.d.ts apps/web/test/screenshots/ +data/agent-sessions/ diff --git a/apps/server/README.md b/apps/server/README.md index dbb6d3b5..bf5fd780 100644 --- a/apps/server/README.md +++ b/apps/server/README.md @@ -16,7 +16,8 @@ src/ auth-settings/ W2-A(provided.ts 提供 secrets / settings 实现) sources-identity/ W2-B acquisition-organize/ W2-C - agent/ W2-D(capability-registry.ts 负责 authorize / audit 注入) + agent/ W2-D:capability-registry.ts(authorize / audit / 幂等键 / 操作账本)、会话 API、runAgent、resolveDecision + (见 docs/runbook/agent.md;agent 模块可以 import 其它模块的 service 导出,反向不行) ``` **分工规则:各 W2 子任务只改自己的 `modules//` 目录,以及 `packages/contracts`、`packages/db` 中本域的文件**(新迁移按编号顺延,开工前在群里认领编号)。`modules/types.ts`、`modules/index.ts`、`context.ts`、`http/app.ts`、`worker/index.ts`、`main.ts` 是共享装配,确需改动时单独提交并通知其他子任务;`http/auth.ts`、`http/sse.ts` 归 W2-A。跨模块只通过 DB 表 + `outbox_event`(事件类型在 contracts/events.ts)+ contracts 类型耦合,不互相 import 对方模块。 diff --git a/apps/server/src/context.ts b/apps/server/src/context.ts index 948a6a6e..c91c4f62 100644 --- a/apps/server/src/context.ts +++ b/apps/server/src/context.ts @@ -12,7 +12,7 @@ import { import type { Logger } from "pino"; import type { Config } from "./config.ts"; import type { EventBusPort } from "./http/sse.ts"; -import { createCapabilityRegistry } from "./modules/agent/capability-registry.ts"; +import { assertCapabilityCount, createCapabilityRegistry } from "./modules/agent/capability-registry.ts"; import { createSecretStore, createSettingsReader } from "./modules/auth-settings/provided.ts"; import type { OutboxPort, ServerContext, ServerModule, ServerRole } from "./modules/types.ts"; @@ -138,6 +138,7 @@ export async function bootstrap(opts: BootstrapOptions): Promise { const { config, logger, db } = opts; const engine = new DeferredTaskEngine(); const outbox = createOutboxPort(); + const settings = createSettingsReader({ db, logger, events: opts.events }); const ctx: ServerContext = { role: opts.role, db, @@ -147,11 +148,14 @@ export async function bootstrap(opts: BootstrapOptions): Promise { engine, outbox, secrets: createSecretStore({ db, masterKey: config.SECRET_MASTER_KEY }), - settings: createSettingsReader({ db, logger, events: opts.events }), - capabilities: createCapabilityRegistry({ logger }), + settings, + // 授权(automation_grant / approval + 自动化总开关)、审计(audit_event)、幂等键与操作账本在这里注入 + capabilities: createCapabilityRegistry({ logger, db, settings }), }; const tasks = buildTaskRegistry(opts.modules, ctx); for (const m of opts.modules) m.registerCapabilities?.(ctx.capabilities, ctx); + // 装配了 agent 模块(即完整模块清单)时,首批 14 个能力必须全部挂上处理器 + if (opts.modules.some((m) => m.name === "agent")) assertCapabilityCount(ctx.capabilities); engine.attach( opts.engine ?? (await OpenWorkflowEngine.connect({ diff --git a/apps/server/src/modules/acquisition-organize/capabilities.ts b/apps/server/src/modules/acquisition-organize/capabilities.ts new file mode 100644 index 00000000..de5fa904 --- /dev/null +++ b/apps/server/src/modules/acquisition-organize/capabilities.ts @@ -0,0 +1,415 @@ +import { CapabilityError, type CapabilityRegistry } from "@umt/agent"; +import { capabilitySpec } from "@umt/contracts"; +import { sql } from "@umt/db"; +import { normalizeTitle, transition } from "@umt/domain"; +import { previewPolicy, publishRevision } from "./routes/rules.ts"; +import { evaluateAcquisition, publishedAcquisitionPolicies } from "./services/acquisition.ts"; +import { type AoDeps, enqueueTracked, ensureTaskRow, markTask } from "./services/common.ts"; +import { type FrozenPlan, planImport } from "./services/organize.ts"; +import { errMessage, hashJson, jsonb, toApi, toApiList } from "./util.ts"; + +// 本模块领域的能力处理器(规格一律取 contracts): +// inventory.query / download.plan / download.submit / organize.plan / organize.execute / policy.preview / policy.apply +// 授权(grant/approval)、幂等键与操作账本由注册中心统一处理(modules/agent/capability-registry.ts); +// 这里的处理器只调本模块服务,并保证自身幂等(崩溃后同幂等键重跑用于对账)。 + +const BUSY = ["reserved", "downloading", "verifying"]; + +export function registerAoCapabilities(registry: CapabilityRegistry, deps: () => AoDeps): void { + registry.register(capabilitySpec("inventory.query"), async (input) => { + const d = deps(); + const db = d.ctx.db; + let ids: string[] = []; + if (input.mediaEntityId) ids = [input.mediaEntityId]; + else if (input.query) { + const n = normalizeTitle(input.query); + const like = `%${input.query.replace(/[\\%_]/g, (m) => `\\${m}`)}%`; + const rows = await sql<{ id: string }>` + select e.id, max(greatest(coalesce(similarity(a.normalized, ${n}), 0), + case when a.normalized = ${n} then 1 else 0 end)) as score + from media_entity e left join title_alias a on a.media_entity_id = e.id + where e.status <> 'merged' + and (e.title ilike ${like} or e.original_title ilike ${like} or a.normalized = ${n} + or similarity(a.normalized, ${n}) >= 0.3) + group by e.id order by score desc, e.id limit ${input.limit}`.execute(db); + ids = rows.rows.map((r) => r.id); + } else { + ids = ( + await db + .selectFrom("media_entity") + .select("id") + .where("status", "<>", "merged") + .orderBy("updated_at", "desc") + .limit(input.limit) + .execute() + ).map((r) => r.id); + } + const items = []; + for (const id of ids) { + const e = await db.selectFrom("media_entity").selectAll().where("id", "=", id).executeTakeFirst(); + if (!e) continue; + const [assets, slots, identities, server] = await Promise.all([ + db.selectFrom("media_asset").selectAll().where("media_entity_id", "=", id).execute(), + db.selectFrom("acquisition_slot").selectAll().where("media_entity_id", "=", id).orderBy("created_at").execute(), + db + .selectFrom("external_identity") + .select(["provider", "entity_type", "external_id", "status"]) + .where("media_entity_id", "=", id) + .execute(), + input.includeServerInventory + ? db.selectFrom("server_mapping").selectAll().where("media_entity_id", "=", id).execute() + : Promise.resolve([]), + ]); + items.push({ + entity: toApi(e), + assets: toApiList(assets), + slots: toApiList(slots), + identities: identities.map((i) => ({ + provider: i.provider, + entityType: i.entity_type, + externalId: i.external_id, + status: i.status, + })), + server: toApiList(server), + }); + } + return { items } as never; + }); + + registry.register(capabilitySpec("download.plan"), async (input) => { + const d = deps(); + const db = d.ctx.db; + const slot = await db.selectFrom("acquisition_slot").selectAll().where("id", "=", input.slotId).executeTakeFirst(); + if (!slot) throw new CapabilityError("not_found", "收录目标(Slot)不存在"); + const rel = await db + .selectFrom("source_release") + .select(["id", "media_entity_id"]) + .where("id", "=", input.sourceReleaseId) + .executeTakeFirst(); + if (!rel) throw new CapabilityError("not_found", "来源资源不存在"); + const out = await evaluateAcquisition(d, { + sourceReleaseId: input.sourceReleaseId, + acquisitionSlotId: input.slotId, + draft: true, + }); + const planId = out.plans[0] ?? null; + const plan = planId + ? await db.selectFrom("acquisition_plan").selectAll().where("id", "=", planId).executeTakeFirst() + : undefined; + const rejected: { code: string; message: string }[] = []; + if (!plan) { + for (const s of out.skipped) rejected.push({ code: s.reason.split(":")[0] ?? s.reason, message: s.reason }); + // 候选评估记录(硬条件拒绝等) + const ev = await db + .selectFrom("artifact") + .select(["content"]) + .where("subject_type", "=", "acquisition_slot") + .where("subject_id", "=", input.slotId) + .where(sql`content->>'sourceReleaseId' = ${input.sourceReleaseId}`) + .orderBy("updated_at", "desc") + .executeTakeFirst(); + const c = (ev?.content ?? {}) as { rejected?: { code: string; message: string }[]; reasons?: string[] }; + for (const r of c.rejected ?? []) rejected.push({ code: r.code, message: r.message }); + for (const r of c.reasons ?? []) rejected.push({ code: r.split(":")[0] ?? r, message: r }); + if (rejected.length === 0) rejected.push({ code: "not_planned", message: "未生成计划(Slot 忙或条件不满足)" }); + } + const fresh = await db + .selectFrom("acquisition_slot") + .selectAll() + .where("id", "=", input.slotId) + .executeTakeFirstOrThrow(); + const grant = await db + .selectFrom("automation_grant") + .select("id") + .where("status", "=", "active") + .where("action", "in", ["download.submit", "policy.acquisition", "*"]) + .executeTakeFirst(); + return { + plan: plan ? (toApi(plan) as never) : null, + slot: toApi(fresh) as never, + rejected, + requiresApproval: !grant, + }; + }); + + registry.register(capabilitySpec("download.submit"), async (input) => { + const d = deps(); + const db = d.ctx.db; + const plan = await db.selectFrom("acquisition_plan").selectAll().where("id", "=", input.planId).executeTakeFirst(); + if (!plan) throw new CapabilityError("not_found", "下载计划不存在"); + if (plan.status === "draft") { + // 草案 → reserved:Slot 必须与生成计划时一致(revision 未变、没有其它预留) + const ok = await db + .transaction() + .execute(async (trx) => { + const slot = await trx + .selectFrom("acquisition_slot") + .selectAll() + .where("id", "=", plan.slot_id) + .forUpdate() + .executeTakeFirstOrThrow(); + if (slot.revision + 1 !== plan.expected_revision) return "Slot 已变化,请重新生成计划"; + if (BUSY.includes(slot.status)) return `Slot 正忙(${slot.status})`; + let st = { status: slot.status, hasAsset: slot.current_asset_id !== null }; + if (st.status === "satisfied") st = transition(st, { type: "upgrade_found" }); + st = transition(st, { type: "reserve" }); + await trx + .updateTable("acquisition_plan") + .set({ status: "reserved", status_reason: "authorized" }) + .where("id", "=", plan.id) + .where("status", "=", "draft") + .execute(); + await trx + .updateTable("acquisition_slot") + .set({ + status: st.status, + reserved_plan_id: plan.id, + revision: slot.revision + 1, + status_reason: "reserved:agent", + }) + .where("id", "=", slot.id) + .execute(); + return null; + }) + .catch((err) => { + if ((err as { code?: string }).code === "23505") return "Slot 已有其它有效预留"; + throw err; + }); + if (ok) throw new CapabilityError("conflict", ok); + } else if (!["reserved", "submitted", "completed"].includes(plan.status)) { + throw new CapabilityError("precondition", `计划状态为 ${plan.status},不能提交`); + } + // 与自动路径同一幂等键:同一计划只会有一个 submitDownload 任务 + const taskId = await enqueueTracked(d, { + kind: "submitDownload", + idempotencyKey: `plan:${plan.id}`, + input: { acquisitionPlanId: plan.id }, + subjectType: "acquisition_plan", + subjectId: plan.id, + triggeredBy: "agent", + }); + await db + .updateTable("acquisition_plan") + .set({ task_id: taskId }) + .where("id", "=", plan.id) + .where("task_id", "is", null) + .execute(); + const dt = await db + .selectFrom("download_task") + .selectAll() + .where("acquisition_plan_id", "=", plan.id) + .executeTakeFirst(); + return { taskId, planId: plan.id, downloadTask: dt ? (toApi(dt) as never) : null }; + }); + + registry.register(capabilitySpec("organize.plan"), async (input, ctx) => { + const d = deps(); + if (!input.downloadTaskId) + throw new CapabilityError("precondition", "v0.1 只支持按下载任务(downloadTaskId)生成整理计划"); + const dt = await d.ctx.db + .selectFrom("download_task") + .select(["status"]) + .where("id", "=", input.downloadTaskId) + .executeTakeFirst(); + if (!dt) throw new CapabilityError("not_found", "下载任务不存在"); + if (!["completed", "importing", "imported"].includes(dt.status)) + throw new CapabilityError("precondition", `下载任务尚未完成(${dt.status})`); + const action: "copy" | "move" = input.action === "move" ? "move" : "copy"; + const request = { + downloadTaskId: input.downloadTaskId, + action, + ...(input.targetLibraryRootId ? { targetLibraryRootId: input.targetLibraryRootId } : {}), + }; + const res = await planImport(d, { + ...request, + trigger: "agent", + actorType: "agent", + actorId: ctx.actorId, + dryRun: true, + }); + const frozen = res.plan; + if (!frozen) { + throw new CapabilityError( + "precondition", + res.recordId + ? `该下载任务已有进行中的整理记录 ${res.recordId}` + : `无法生成整理计划:${res.reason ?? res.status}`, + ); + } + const entity = await d.ctx.db + .selectFrom("media_entity") + .select(["kind"]) + .where("id", "=", frozen.entityId) + .executeTakeFirst(); + const payloadHash = planHash(request, frozen); + const art = await d.ctx.db + .insertInto("artifact") + .values({ + kind: "plan", + agent_run_id: ctx.agentRunId && isUuid(ctx.agentRunId) ? ctx.agentRunId : null, + subject_type: "download_task", + subject_id: input.downloadTaskId, + title: "organize_plan", + content: jsonb({ request, plan: frozen, mediaEntityId: frozen.entityId, mediaKind: entity?.kind ?? null }), + content_hash: payloadHash, + }) + .returning("id") + .executeTakeFirstOrThrow(); + const org = frozen.organize; + return { + planId: art.id, + payloadHash, + files: org.files.map((f) => ({ + sourcePath: f.src, + targetPath: f.action === "skip" || f.action === "hold" ? null : f.dst, + action, + skipReason: f.skipReason ?? (f.action === "hold" ? "hold" : null), + })), + conflicts: org.conflicts.map((c) => ({ + code: c.kind, + message: c.detail ?? c.kind, + path: c.dst ?? c.src ?? null, + })), + requiresApproval: org.requiresApproval, + }; + }); + + registry.register(capabilitySpec("organize.execute"), async (input, ctx) => { + const d = deps(); + const db = d.ctx.db; + const art = await db + .selectFrom("artifact") + .selectAll() + .where("id", "=", input.planId) + .where("kind", "=", "plan") + .where("title", "=", "organize_plan") + .executeTakeFirst(); + if (!art) throw new CapabilityError("not_found", "整理计划不存在"); + if (art.content_hash !== input.payloadHash) throw new CapabilityError("conflict", "payloadHash 与计划不一致"); + const content = art.content as { + request: { downloadTaskId: string; action: "copy" | "move"; targetLibraryRootId?: string }; + }; + const req = content.request; + // 已执行过(同计划)→ 返回已有记录 + const taskKey = `organize-plan:${art.id}`; + const existing = await db + .selectFrom("task") + .select(["id"]) + .where("kind", "=", "verifyAndImport") + .where("idempotency_key", "=", taskKey) + .executeTakeFirst(); + if (existing) { + const rec = await db + .selectFrom("organization_record") + .selectAll() + .where("task_id", "=", existing.id) + .executeTakeFirst(); + return { taskId: existing.id, planId: art.id, record: rec ? (toApi(rec) as never) : null }; + } + // 执行前复核:重新冻结一次,与审批/授权时看到的计划一致才执行 + const again = await planImport(d, { + ...req, + trigger: "agent", + actorType: "agent", + actorId: ctx.actorId, + dryRun: true, + }); + if (!again.plan || planHash(req, again.plan) !== art.content_hash) + throw new CapabilityError("conflict", "源文件或目标已变化,计划失效,请重新生成整理计划"); + const taskId = await ensureTaskRow(db, { + kind: "verifyAndImport", + idempotencyKey: taskKey, + input: { ...req, planId: art.id }, + subjectType: "download_task", + subjectId: req.downloadTaskId, + triggeredBy: "agent", + }); + let res: Awaited>; + try { + res = await planImport(d, { ...req, trigger: "agent", actorType: "agent", actorId: ctx.actorId, taskId }); + } catch (err) { + await markTask(db, taskId, "failed", { error: errMessage(err) }); + throw new CapabilityError("precondition", errMessage(err)); + } + if (res.recordId && (res.status === "running" || res.status === "queued")) { + const { runId } = await d.ctx.engine.enqueue( + "verifyAndImport", + { recordId: res.recordId, downloadTaskId: req.downloadTaskId, taskId }, + { idempotencyKey: `verifyAndImport:${taskKey}`, traceId: taskId }, + ); + await db.updateTable("task").set({ engine_run_id: runId }).where("id", "=", taskId).execute(); + await db + .updateTable("organization_record") + .set({ task_id: taskId }) + .where("id", "=", res.recordId) + .where("task_id", "is", null) + .execute(); + } else { + await markTask(db, taskId, "failed", { error: res.reason ?? res.status }); + } + const rec = res.recordId + ? await db.selectFrom("organization_record").selectAll().where("id", "=", res.recordId).executeTakeFirst() + : undefined; + return { taskId, planId: art.id, record: rec ? (toApi(rec) as never) : null }; + }); + + registry.register(capabilitySpec("policy.preview"), async (input) => { + const d = deps(); + const cur = await d.ctx.db + .selectFrom("policy_revision") + .selectAll() + .where("id", "=", input.policyRevisionId) + .executeTakeFirst(); + if (!cur) throw new CapabilityError("not_found", "策略版本不存在"); + const draft = await previewPolicy(d, cur.rules); + const published = (await publishedAcquisitionPolicies(d.ctx.db)).find( + (p) => p.row.policy_key === cur.policy_key && p.row.id !== cur.id, + ); + const base = published ? await previewPolicy(d, published.row.rules) : null; + const before = new Map((base?.samples ?? []).map((s) => [`${s.sourceReleaseId}`, s.decision])); + return { + newlyMatched: Math.max(0, draft.matched - (base?.matched ?? 0)), + newlyExcluded: Math.max(0, draft.excluded - (base?.excluded ?? 0)), + samples: draft.samples.slice(0, input.sampleLimit).map((s) => ({ + sourceReleaseId: s.sourceReleaseId, + before: before.get(s.sourceReleaseId) ?? "none", + after: s.decision, + reason: `template:${s.template}`, + })), + }; + }); + + registry.register(capabilitySpec("policy.apply"), async (input, ctx) => { + const d = deps(); + const row = await d.ctx.db.transaction().execute(async (trx) => { + const cur = await trx + .selectFrom("policy_revision") + .selectAll() + .where("id", "=", input.policyRevisionId) + .forUpdate() + .executeTakeFirst(); + if (!cur) throw new CapabilityError("not_found", "策略版本不存在"); + if (cur.status === "published") return cur; // 幂等:已发布 + if (cur.revision !== input.expectedRevision) + throw new CapabilityError("conflict", `revision 不匹配(当前 ${cur.revision})`); + if (cur.status !== "draft") throw new CapabilityError("precondition", `只能发布草案(当前 ${cur.status})`); + return publishRevision(trx, cur.id, cur.policy_key, ctx.actorId, "superseded"); + }); + return { policy: toApi(row) as never }; + }); +} + +/** 整理计划指纹:请求 + 冻结的文件动作/目标/前置条件(源大小变化、目标出现都会改变指纹) */ +function planHash(request: object, frozen: FrozenPlan): string { + const o = frozen.organize; + return hashJson({ + request, + entityId: frozen.entityId, + root: frozen.libraryRootId, + files: o.files.map((f) => [f.src, f.dst, f.size, f.action, f.skipReason ?? null]), + conflicts: o.conflicts.map((c) => [c.kind, c.src ?? null, c.dst ?? null, c.blocking]), + preconditions: o.preconditions, + }); +} + +const isUuid = (s: string) => /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(s); diff --git a/apps/server/src/modules/acquisition-organize/index.ts b/apps/server/src/modules/acquisition-organize/index.ts index 47549661..7924c389 100644 --- a/apps/server/src/modules/acquisition-organize/index.ts +++ b/apps/server/src/modules/acquisition-organize/index.ts @@ -1,5 +1,6 @@ import { type EventPayload, EventType } from "@umt/contracts"; import type { ServerContext, ServerModule } from "../types.ts"; +import { registerAoCapabilities } from "./capabilities.ts"; import { registerAoTasks } from "./handlers.ts"; import { type AoOptions, type AoPorts, createPorts } from "./ports.ts"; import { registerDashboardRoutes } from "./routes/dashboard.ts"; @@ -70,6 +71,9 @@ export function createAcquisitionOrganizeModule(overrides: AoOverrides = {}): Se registerTasks(registry, ctx) { registerAoTasks(registry, lazy(ctx)); }, + registerCapabilities(registry, ctx) { + registerAoCapabilities(registry, () => lazy(ctx)); + }, outboxRoutes: () => ({ [EventType.releaseResolved]: (ev) => { const p = ev.payload as EventPayload<"release.resolved">; diff --git a/apps/server/src/modules/acquisition-organize/routes/rules.ts b/apps/server/src/modules/acquisition-organize/routes/rules.ts index c07454e8..77131441 100644 --- a/apps/server/src/modules/acquisition-organize/routes/rules.ts +++ b/apps/server/src/modules/acquisition-organize/routes/rules.ts @@ -639,7 +639,7 @@ async function suppress(trx: Transaction, tagId: string, fingerprint: string } /** 发布:旧 published → superseded/rolled_back;吊销旧授权;新版本生成持续授权 automation_grant */ -async function publishRevision( +export async function publishRevision( trx: Transaction, id: string, policyKey: string, @@ -684,7 +684,7 @@ async function publishRevision( } /** 预览:用规则里的 Slot 模板评估最近已识别的发布(确定性,不产生副作用) */ -async function previewPolicy(d: AoDeps, rules: unknown) { +export async function previewPolicy(d: AoDeps, rules: unknown) { const r = AcquisitionPolicyRules.safeParse(rules); const templates = r.success ? [...(r.data.defaultSlots ?? []), ...(r.data.newTitle?.slots ?? [])] : []; const rels = await d.ctx.db diff --git a/apps/server/src/modules/acquisition-organize/services/acquisition.ts b/apps/server/src/modules/acquisition-organize/services/acquisition.ts index 3c68be9b..993cc832 100644 --- a/apps/server/src/modules/acquisition-organize/services/acquisition.ts +++ b/apps/server/src/modules/acquisition-organize/services/acquisition.ts @@ -34,6 +34,11 @@ export interface EvaluateInput { acquisitionSlotId?: string | undefined; /** release.resolved 带来的内容单元 */ contentUnitIds?: string[] | undefined; + /** + * 只生成草案计划(download.plan 能力):status=draft,不预留 Slot、不计入预算、不入队提交; + * 由 download.submit 在授权通过后转为 reserved 再提交。 + */ + draft?: boolean | undefined; } export interface EvaluateOutcome { @@ -216,6 +221,7 @@ interface Ctx { d: AoDeps; now: Date; out: EvaluateOutcome; + draft: boolean; } async function note(c: Ctx, base: Omit) { @@ -223,7 +229,7 @@ async function note(c: Ctx, base: Omit) { } export async function evaluateAcquisition(d: AoDeps, input: EvaluateInput): Promise { - const c: Ctx = { d, now: d.ports.now(), out: { evaluated: 0, plans: [], skipped: [] } }; + const c: Ctx = { d, now: d.ports.now(), out: { evaluated: 0, plans: [], skipped: [] }, draft: input.draft === true }; const db = d.ctx.db; if (input.sourceReleaseId) { const rel = await db @@ -488,6 +494,7 @@ async function evaluateRelease(c: Ctx, rel: SourceReleaseRow, eventUnits: readon rejected: [], reasons: flat.reasons, }); + if (c.draft) continue; await enqueueTracked(d, { kind: "submitDownload", idempotencyKey: `plan:${planId}`, @@ -593,6 +600,54 @@ async function reserve( savePath: p.savePath, mode: p.mode, }; + const evaluation = jsonb({ + ...toContractSlotEvaluation(p.ev), + mode: p.mode, + instanceId: p.instanceId, + volumeKey: p.volumeKey, + savePath: p.savePath, + }); + if (c.draft) { + // 草案:同一 Slot+发布复用未提交的草案;Slot 状态与预算不动,expected_revision 记下当前 Slot 版本 +1 + const key = `draft:${p.slot.id}:${p.rel.id}`; + const hit = await d.ctx.db + .selectFrom("acquisition_plan") + .select(["id", "status"]) + .where("idempotency_key", "=", key) + .executeTakeFirst(); + if (hit) { + if (hit.status !== "draft") return null; + await d.ctx.db + .updateTable("acquisition_plan") + .set({ evaluation, payload_hash: hashJson(payload), expected_revision: p.slot.revision + 1, priority }) + .where("id", "=", hit.id) + .execute(); + return hit.id; + } + const row = await d.ctx.db + .insertInto("acquisition_plan") + .values({ + slot_id: p.slot.id, + source_release_id: p.rel.id, + purpose: p.mode === "upgrade" ? "upgrade" : "fill", + status: "draft", + priority, + evaluation, + reserved_bytes: p.rel.size_bytes ?? 0, + budget_reservation: jsonb({ + bytes: p.rel.size_bytes, + countsAsNewTitle: p.mode === "new_title", + day: startOfDay(now).toISOString().slice(0, 10), + }), + payload_hash: hashJson(payload), + expected_revision: p.slot.revision + 1, + idempotency_key: key, + }) + .onConflict((oc) => oc.column("idempotency_key").doNothing()) + .returning("id") + .executeTakeFirst(); + return row?.id ?? null; + } try { return await d.ctx.db.transaction().execute(async (trx) => { const slot = await trx @@ -613,13 +668,7 @@ async function reserve( purpose: p.mode === "upgrade" ? "upgrade" : "fill", status: "reserved", priority, - evaluation: jsonb({ - ...toContractSlotEvaluation(p.ev), - mode: p.mode, - instanceId: p.instanceId, - volumeKey: p.volumeKey, - savePath: p.savePath, - }), + evaluation, reserved_bytes: p.rel.size_bytes ?? 0, budget_reservation: jsonb({ bytes: p.rel.size_bytes, diff --git a/apps/server/src/modules/agent/authorize.ts b/apps/server/src/modules/agent/authorize.ts new file mode 100644 index 00000000..7ebf67a8 --- /dev/null +++ b/apps/server/src/modules/agent/authorize.ts @@ -0,0 +1,210 @@ +import { createHash } from "node:crypto"; +import type { AuthorizationChecker, AuthorizationDecision } from "@umt/agent"; +import type { CapabilitySpec } from "@umt/contracts"; +import { type DB, type Kysely, sql } from "@umt/db"; +import type { SettingsReader } from "../types.ts"; + +// 服务端授权(REQ §10.2、§10.3):propose/approve/execute 类能力必须命中 +// - 单次审批 approval:status=approved、绑定 plan 类型 + plan id + payload hash + revision、未过期(用后 consumed);或 +// - 持续授权 automation_grant:action 覆盖本能力、active 未过期、绑定的策略版本仍是 published、 +// 对象范围(mediaEntityIds / mediaKinds)包含目标、预算(每日次数 / 字节)未用尽。 +// 模型参数里的"用户已同意"不会被读取;输入里的 approvalId 也只是指针,必须在库里核对。 +// 控制台管理员从 Web 手动调用(source=web 且 actor=user:*)本身就是人工授权;API Key / Agent / 后台任务一律要授权记录。 + +/** 能力 → 可覆盖它的授权动作(策略发布生成 policy. 持续授权;"*" 全部) */ +export const GRANT_ACTIONS: Record = { + "identity.resolve": ["identity.resolve", "policy.identity"], + "download.submit": ["download.submit", "policy.acquisition"], + "organize.execute": ["organize.execute", "policy.acquisition", "policy.organize"], + "task.retry": ["task.retry", "task.*"], + "task.cancel": ["task.cancel", "task.*"], + "policy.apply": ["policy.apply"], +}; + +/** 单次审批绑定的计划类型与输入里的计划 ID 字段 */ +const APPROVAL_PLAN: Record = { + "download.submit": { planType: "acquisition_plan", field: "planId" }, + "organize.execute": { planType: "organize_plan", field: "planId" }, + "policy.apply": { planType: "policy_revision", field: "policyRevisionId" }, +}; + +export interface PlanBinding { + payloadHash: string; + revision: number; + mediaEntityId: string | null; + mediaKind: string | null; + bytes: number | null; +} + +const sha = (s: string) => createHash("sha256").update(s, "utf8").digest("hex"); + +/** 读计划当前的 payload hash / revision / 目标作品(审批绑定与范围校验共用) */ +export async function planBinding(db: Kysely, planType: string, planId: string): Promise { + if (planType === "acquisition_plan") { + const r = await db + .selectFrom("acquisition_plan as p") + .innerJoin("acquisition_slot as s", "s.id", "p.slot_id") + .innerJoin("media_entity as e", "e.id", "s.media_entity_id") + .select(["p.payload_hash", "p.expected_revision", "e.id as entity_id", "e.kind", "p.reserved_bytes"]) + .where("p.id", "=", planId) + .executeTakeFirst(); + return r + ? { + payloadHash: r.payload_hash, + revision: r.expected_revision, + mediaEntityId: r.entity_id, + mediaKind: r.kind, + bytes: Number(r.reserved_bytes), + } + : null; + } + if (planType === "organize_plan") { + const r = await db + .selectFrom("artifact") + .select(["content_hash", "content"]) + .where("id", "=", planId) + .where("kind", "=", "plan") + .executeTakeFirst(); + if (!r) return null; + const c = (r.content ?? {}) as { mediaEntityId?: string | null; mediaKind?: string | null }; + return { + payloadHash: r.content_hash, + revision: 1, + mediaEntityId: c.mediaEntityId ?? null, + mediaKind: c.mediaKind ?? null, + bytes: null, + }; + } + if (planType === "policy_revision") { + const r = await db + .selectFrom("policy_revision") + .select(["revision", "rules", "scope"]) + .where("id", "=", planId) + .executeTakeFirst(); + return r + ? { + payloadHash: sha(JSON.stringify({ rules: r.rules, scope: r.scope })), + revision: r.revision, + mediaEntityId: null, + mediaKind: null, + bytes: null, + } + : null; + } + return null; +} + +export function createAuthorizer(deps: { + db: Kysely; + settings: SettingsReader; + now?: (() => Date) | undefined; +}): AuthorizationChecker { + const { db } = deps; + const now = () => deps.now?.() ?? new Date(); + + return async ({ descriptor, input, ctx }): Promise => { + const d = descriptor as CapabilitySpec; + const inp = (input ?? {}) as Record; + // 自动化总开关 / 只读对账:自动来源(agent / 后台任务)一律不执行写操作 + if (ctx.source !== "web" && d.kind === "execute") { + const a = await deps.settings.get("automation"); + if (!a.enabled) return { allowed: false, reason: "自动化总开关已关闭,自动执行被拒绝" }; + if (a.readOnlyReconcile) return { allowed: false, reason: "只读对账模式,自动执行被拒绝" }; + } + if (ctx.source === "web" && /^(user|admin):/.test(ctx.actorId)) return { allowed: true }; + + // 1) 单次审批 + const ap = APPROVAL_PLAN[d.id]; + const approvalId = typeof inp.approvalId === "string" ? inp.approvalId : null; + let target: PlanBinding | null = null; + if (ap) { + const planId = typeof inp[ap.field] === "string" ? (inp[ap.field] as string) : null; + if (planId) target = await planBinding(db, ap.planType, planId); + if (approvalId) { + if (!planId || !target) return { allowed: false, reason: `审批 ${approvalId} 绑定的计划不存在` }; + const a = await db.selectFrom("approval").selectAll().where("id", "=", approvalId).executeTakeFirst(); + if (!a) return { allowed: false, reason: `审批 ${approvalId} 不存在` }; + if (a.status !== "approved") return { allowed: false, reason: `审批 ${approvalId} 状态为 ${a.status}` }; + if (a.plan_type !== ap.planType || a.plan_id !== planId) + return { allowed: false, reason: "审批绑定的计划与本次调用不一致" }; + if (a.payload_hash !== target.payloadHash) return { allowed: false, reason: "计划内容已变化,旧审批失效" }; + if (a.expected_revision !== target.revision) + return { allowed: false, reason: "计划 revision 已变化,旧审批失效" }; + if (a.expires_at && a.expires_at.getTime() <= now().getTime()) return { allowed: false, reason: "审批已过期" }; + return { allowed: true, approvalId: a.id }; + } + } + + // 2) 持续授权 + const actions = [...(GRANT_ACTIONS[d.id] ?? [d.id]), "*"]; + const grants = await db + .selectFrom("automation_grant as g") + .leftJoin("policy_revision as p", "p.id", "g.policy_revision_id") + .select(["g.id", "g.action", "g.scope", "g.budget", "g.policy_revision_id", "p.status as policy_status"]) + .where("g.status", "=", "active") + .where("g.action", "in", actions) + .where((eb) => eb.or([eb("g.expires_at", "is", null), eb("g.expires_at", ">", now())])) + .orderBy("g.created_at", "desc") + .execute(); + const reasons: string[] = []; + for (const g of grants) { + if (g.policy_revision_id && g.policy_status !== "published") { + reasons.push(`授权 ${g.id} 绑定的策略版本已不是发布状态`); + continue; + } + const scope = (g.scope ?? {}) as { mediaEntityIds?: string[]; mediaKinds?: string[]; capabilities?: string[] }; + if (scope.capabilities?.length && !scope.capabilities.includes(d.id)) { + reasons.push(`授权 ${g.id} 不含能力 ${d.id}`); + continue; + } + if ( + scope.mediaEntityIds?.length && + (!target?.mediaEntityId || !scope.mediaEntityIds.includes(target.mediaEntityId)) + ) { + reasons.push(`授权 ${g.id} 的对象范围不含目标作品`); + continue; + } + if (scope.mediaKinds?.length && target?.mediaKind && !kindCovered(scope.mediaKinds, target.mediaKind)) { + reasons.push(`授权 ${g.id} 的媒体类型范围不含 ${target.mediaKind}`); + continue; + } + const budget = (g.budget ?? {}) as { maxInvocationsPerDay?: number; maxBytesPerDay?: number }; + if (budget.maxInvocationsPerDay !== undefined || budget.maxBytesPerDay !== undefined) { + const day = new Date(now()); + day.setHours(0, 0, 0, 0); + const used = await sql<{ n: number }>` + select count(*)::int as n from capability_invocation + where grant_id = ${g.id} and status = 'succeeded' and created_at >= ${day}`.execute(db); + if (budget.maxInvocationsPerDay !== undefined && (used.rows[0]?.n ?? 0) >= budget.maxInvocationsPerDay) { + reasons.push(`授权 ${g.id} 今日次数预算已用尽`); + continue; + } + if (budget.maxBytesPerDay !== undefined && d.id === "download.submit") { + const bytes = await sql<{ b: string }>` + select coalesce(sum(p.reserved_bytes), 0)::bigint as b + from capability_invocation ci join acquisition_plan p on p.id::text = ci.input->>'planId' + where ci.grant_id = ${g.id} and ci.status = 'succeeded' and ci.created_at >= ${day}`.execute(db); + if (Number(bytes.rows[0]?.b ?? 0) + (target?.bytes ?? 0) > budget.maxBytesPerDay) { + reasons.push(`授权 ${g.id} 今日字节预算不足`); + continue; + } + } + } + return { allowed: true, grantId: g.id }; + } + return { + allowed: false, + reason: + reasons.length > 0 + ? `没有可用的授权:${reasons.join(";")}` + : `缺少 ${d.id} 的持续授权(automation_grant)或有效的单次审批(approval)`, + }; + }; +} + +/** 策略范围用订阅口径(movie / series / variety),作品用 media_entity.kind(movie / tv / variety …) */ +function kindCovered(kinds: readonly string[], entityKind: string): boolean { + if (kinds.includes(entityKind)) return true; + if (entityKind !== "movie" && entityKind !== "variety" && kinds.includes("series")) return true; + return false; +} diff --git a/apps/server/src/modules/agent/capabilities.ts b/apps/server/src/modules/agent/capabilities.ts new file mode 100644 index 00000000..d493b7d7 --- /dev/null +++ b/apps/server/src/modules/agent/capabilities.ts @@ -0,0 +1,213 @@ +import { CapabilityError, type CapabilityRegistry } from "@umt/agent"; +import { capabilitySpec, type TaskStatus } from "@umt/contracts"; +import { sql, type TaskRow } from "@umt/db"; +import type { ServerContext } from "../types.ts"; + +// 任务类能力:task.inspect / task.retry / task.cancel。业务任务行(task 表)是时间线投影, +// 引擎只管推进;重试 = 按原输入重新入队(副作用由各处理器的操作账本去重),取消 = 写取消标志 + 引擎取消。 + +/** task.kind → 引擎任务名(agentRun 行对应 runAgent) */ +export const engineTaskName = (kind: string) => (kind === "agentRun" ? "runAgent" : kind); + +const toIso = (d: Date | null) => (d ? d.toISOString() : null); + +export function taskToApi(t: TaskRow) { + return { + id: t.id, + kind: t.kind, + handlerVersion: t.handler_version, + idempotencyKey: t.idempotency_key, + status: t.status, + priority: t.priority, + input: t.input, + output: t.output, + subjectType: t.subject_type, + subjectId: t.subject_id, + parentTaskId: t.parent_task_id, + engineRunId: t.engine_run_id, + attempts: t.attempts, + maxAttempts: t.max_attempts, + nextRunAt: t.next_run_at.toISOString(), + waitReason: t.wait_reason, + cancelRequestedAt: toIso(t.cancel_requested_at), + cancelReason: t.cancel_reason, + lastError: t.last_error, + startedAt: toIso(t.started_at), + finishedAt: toIso(t.finished_at), + triggeredBy: t.triggered_by, + createdAt: t.created_at.toISOString(), + updatedAt: t.updated_at.toISOString(), + }; +} + +/** 业务时间线:任务自身 + 关联业务对象(计划/下载/整理记录/agent 运行)的当前状态 */ +export async function taskTimeline(ctx: ServerContext, t: TaskRow) { + const db = ctx.db; + const out: { at: string; type: string; summary: string; data?: Record }[] = []; + out.push({ at: t.created_at.toISOString(), type: "task.created", summary: `${t.kind} 已创建` }); + if (t.started_at) + out.push({ at: t.started_at.toISOString(), type: "task.started", summary: `第 ${t.attempts} 次执行` }); + if (t.cancel_requested_at) + out.push({ + at: t.cancel_requested_at.toISOString(), + type: "task.cancel_requested", + summary: t.cancel_reason ?? "请求取消", + }); + if (t.finished_at) + out.push({ + at: t.finished_at.toISOString(), + type: `task.${t.status}`, + summary: t.status === "failed" ? `失败:${JSON.stringify(t.last_error ?? {}).slice(0, 200)}` : t.status, + }); + if (t.subject_type === "acquisition_plan" && t.subject_id) { + const p = await db + .selectFrom("acquisition_plan") + .select(["status", "status_reason", "updated_at"]) + .where("id", "=", t.subject_id) + .executeTakeFirst(); + if (p) + out.push({ + at: p.updated_at.toISOString(), + type: "plan.status", + summary: `下载计划 ${p.status}${p.status_reason ? `(${p.status_reason})` : ""}`, + data: { planId: t.subject_id, status: p.status }, + }); + const dts = await db + .selectFrom("download_task") + .select(["id", "status", "progress", "updated_at"]) + .where("acquisition_plan_id", "=", t.subject_id) + .execute(); + for (const dt of dts) + out.push({ + at: dt.updated_at.toISOString(), + type: "download.status", + summary: `下载任务 ${dt.status}(进度 ${Math.round(Number(dt.progress ?? 0) * 100)}%)`, + data: { downloadTaskId: dt.id, status: dt.status }, + }); + const recs = dts.length + ? await db + .selectFrom("organization_record") + .select(["id", "status", "updated_at"]) + .where( + "download_task_id", + "in", + dts.map((d) => d.id), + ) + .execute() + : []; + for (const r of recs) + out.push({ + at: r.updated_at.toISOString(), + type: "organization.status", + summary: `整理记录 ${r.status}`, + data: { recordId: r.id, status: r.status }, + }); + } + if (t.kind === "verifyAndImport") { + const recs = await db + .selectFrom("organization_record") + .select(["id", "status", "updated_at"]) + .where("task_id", "=", t.id) + .execute(); + for (const r of recs) + out.push({ + at: r.updated_at.toISOString(), + type: "organization.status", + summary: `整理记录 ${r.status}`, + data: { recordId: r.id, status: r.status }, + }); + } + return out.sort((a, b) => a.at.localeCompare(b.at)); +} + +export async function retryTask(ctx: ServerContext, taskId: string, by: string, reason?: string) { + const t = await ctx.db.selectFrom("task").selectAll().where("id", "=", taskId).executeTakeFirst(); + if (!t) throw new CapabilityError("not_found", "任务不存在"); + if (!["failed", "dead", "cancelled"].includes(t.status)) + throw new CapabilityError("precondition", `只能重试失败/死信/已取消的任务(当前 ${t.status})`); + const input = + t.kind === "agentRun" ? { agentRunId: t.subject_id } : { ...((t.input ?? {}) as Record), taskId }; + const { runId } = await ctx.engine.enqueue(engineTaskName(t.kind), input, { + idempotencyKey: `${t.kind}:${t.idempotency_key}:retry:${t.attempts}:${t.updated_at.getTime()}`, + traceId: t.id, + }); + const row = await ctx.db + .updateTable("task") + .set({ + status: "pending", + engine_run_id: runId, + cancel_requested_at: null, + cancel_reason: null, + finished_at: null, + next_run_at: new Date(), + triggered_by: t.triggered_by ?? by, + last_error: sql`coalesce(last_error, '{}'::jsonb) || ${JSON.stringify({ retriedBy: by, retryReason: reason ?? null })}::jsonb`, + }) + .where("id", "=", taskId) + .returningAll() + .executeTakeFirstOrThrow(); + if (t.kind === "agentRun" && t.subject_id) + await ctx.db + .updateTable("agent_run") + .set({ status: "queued", cancel_requested_at: null, finished_at: null }) + .where("id", "=", t.subject_id) + .execute(); + return row; +} + +export async function cancelTask(ctx: ServerContext, taskId: string, reason: string) { + const t = await ctx.db.selectFrom("task").selectAll().where("id", "=", taskId).executeTakeFirst(); + if (!t) throw new CapabilityError("not_found", "任务不存在"); + if (["succeeded", "failed", "cancelled", "dead"].includes(t.status)) return { row: t, cancelRequested: false }; + const now = new Date(); + await ctx.db + .updateTable("task") + .set({ cancel_requested_at: now, cancel_reason: reason }) + .where("id", "=", taskId) + .execute(); + if (t.kind === "agentRun" && t.subject_id) + await ctx.db + .updateTable("agent_run") + .set({ cancel_requested_at: now }) + .where("id", "=", t.subject_id) + .where("cancel_requested_at", "is", null) + .execute(); + let cancelled = false; + if (t.engine_run_id) cancelled = await ctx.engine.cancel(t.engine_run_id, reason).catch(() => false); + const row = await ctx.db + .updateTable("task") + .set(cancelled || !t.engine_run_id ? { status: "cancelled", finished_at: now } : {}) + .where("id", "=", taskId) + .returningAll() + .executeTakeFirstOrThrow(); + return { row, cancelRequested: true }; +} + +export function registerTaskCapabilities(registry: CapabilityRegistry, ctx: ServerContext): void { + registry.register(capabilitySpec("task.inspect"), async (input) => { + const t = await ctx.db.selectFrom("task").selectAll().where("id", "=", input.taskId).executeTakeFirst(); + if (!t) throw new CapabilityError("not_found", "任务不存在"); + const children = await ctx.db + .selectFrom("task") + .selectAll() + .where("parent_task_id", "=", t.id) + .orderBy("created_at") + .limit(100) + .execute(); + return { + task: taskToApi(t) as never, + children: children.map(taskToApi) as never, + timeline: await taskTimeline(ctx, t), + }; + }); + + registry.register(capabilitySpec("task.retry"), async (input, c) => { + const row = await retryTask(ctx, input.taskId, c.actorId, input.reason); + return { taskId: row.id, status: row.status as TaskStatus }; + }); + + registry.register(capabilitySpec("task.cancel"), async (input) => { + const r = await cancelTask(ctx, input.taskId, input.reason); + return { taskId: r.row.id, status: r.row.status as TaskStatus, cancelRequested: r.cancelRequested }; + }); +} diff --git a/apps/server/src/modules/agent/capability-registry.ts b/apps/server/src/modules/agent/capability-registry.ts index df15c0cf..1cfd31b8 100644 --- a/apps/server/src/modules/agent/capability-registry.ts +++ b/apps/server/src/modules/agent/capability-registry.ts @@ -1,13 +1,200 @@ -import { CapabilityRegistry } from "@umt/agent"; +import { createHash } from "node:crypto"; +import { + type AuthorizationChecker, + type CapabilityAuditRecord, + type CapabilityHandler, + CapabilityRegistry, + type IdempotencyKeyGenerator, + stableStringify, +} from "@umt/agent"; +import type { ActorType, CapabilityKind, CapabilitySpec } from "@umt/contracts"; +import { type DB, type Kysely, sql } from "@umt/db"; import type { Logger } from "pino"; +import type { z } from "zod"; +import type { SettingsReader } from "../types.ts"; +import { createAuthorizer } from "./authorize.ts"; + +// 能力注册中心工厂:authorize(automation_grant / approval + 自动化总开关)、audit(audit_event)、 +// 服务端幂等键、有副作用能力的操作账本(capability_invocation)。Web / Pi / CLI / MCP 共用这一实例。 + +const SIDE_EFFECT: readonly CapabilityKind[] = ["propose", "approve", "execute"]; + +/** 首批能力总数:启动时断言注册数与此一致(漏挂处理器即启动失败) */ +export const EXPECTED_CAPABILITY_COUNT = 14; /** - * 能力注册中心工厂(W2-D 负责接 authorize:automation_grant / approval + 自动化总开关;audit:写 audit_event)。 - * 当前未注入 authorize:read/preview 可调用,propose/approve/execute 一律拒绝(安全默认)。 + * 幂等键:能力 ID + 版本 + 所属运行(agent run / 任务)+ 规范化输入的 sha256。 + * 同一运行里同参数重复调用 → 同一个键 → 账本直接返回已记录的结果(崩溃恢复后的对账也走这里)。 */ -export function createCapabilityRegistry(deps: { logger: Logger }): CapabilityRegistry { +export const serverIdempotencyKey: IdempotencyKeyGenerator = (d, input, ctx) => + `${d.id}:v${d.version}:${ctx.agentRunId ?? ctx.taskId}:${createHash("sha256") + .update(stableStringify(input)) + .digest("hex") + .slice(0, 32)}`; + +export interface CapabilityRegistryDeps { + logger: Logger; + /** 不给 db 时退化为 W1 安全默认(无授权回调 → propose/approve/execute 一律拒绝;审计只写日志) */ + db?: Kysely; + settings?: SettingsReader; + now?: () => Date; +} + +/** 注册时给有副作用的处理器套上操作账本 */ +class LedgeredCapabilityRegistry extends CapabilityRegistry { + constructor( + private readonly db: Kysely | undefined, + options: ConstructorParameters[0], + ) { + super(options); + } + + override register( + descriptor: CapabilitySpec, + handler: CapabilityHandler, + ): this { + if (!this.db || !SIDE_EFFECT.includes(descriptor.kind)) return super.register(descriptor, handler); + return super.register(descriptor, ledgered(this.db, descriptor as unknown as CapabilitySpec, handler)); + } +} + +function ledgered( + db: Kysely, + d: CapabilitySpec, + handler: CapabilityHandler, +): CapabilityHandler { + return async (input, ctx) => { + const key = ctx.idempotencyKey; + if (!key) return handler(input, ctx); + const prev = await db + .selectFrom("capability_invocation") + .select(["id", "status", "output"]) + .where("idempotency_key", "=", key) + .executeTakeFirst(); + if (prev?.status === "succeeded") return prev.output as z.input; + const auth = ctx.authorization; + if (prev) { + await db + .updateTable("capability_invocation") + .set({ status: "running", attempts: sql`attempts + 1`, error: null }) + .where("id", "=", prev.id) + .execute(); + } else { + await db + .insertInto("capability_invocation") + .values({ + idempotency_key: key, + capability_id: d.id, + capability_version: d.version, + agent_run_id: ctx.agentRunId && isUuid(ctx.agentRunId) ? ctx.agentRunId : null, + actor_id: ctx.actorId, + source: ctx.source ?? "unknown", + input: JSON.stringify(input ?? {}), + grant_id: auth?.grantId ?? null, + approval_id: auth?.approvalId ?? null, + }) + .onConflict((oc) => + oc.column("idempotency_key").doUpdateSet({ attempts: sql`capability_invocation.attempts + 1` }), + ) + .execute(); + } + try { + const out = await handler(input, ctx); + await db + .updateTable("capability_invocation") + .set({ status: "succeeded", output: JSON.stringify(out ?? {}), finished_at: new Date() }) + .where("idempotency_key", "=", key) + .execute(); + return out; + } catch (err) { + const e = err as { code?: string; message?: string }; + await db + .updateTable("capability_invocation") + .set({ + status: "failed", + error: JSON.stringify({ code: e.code ?? "internal", message: String(e.message ?? err).slice(0, 500) }), + finished_at: new Date(), + }) + .where("idempotency_key", "=", key) + .execute() + .catch(() => {}); + throw err; + } + }; +} + +const isUuid = (s: string) => /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(s); + +function actorTypeOf(r: CapabilityAuditRecord): ActorType { + if (r.source === "agent") return "agent"; + if (r.actorId.startsWith("api_key:")) return "api_key"; + if (r.actorId.startsWith("user:") || r.actorId.startsWith("admin:")) return "user"; + return "system"; +} + +export function createCapabilityRegistry(deps: CapabilityRegistryDeps): CapabilityRegistry { const log = deps.logger.child({ component: "capabilities" }); - return new CapabilityRegistry({ - audit: (r) => log.info({ capability: r.capabilityId, outcome: r.outcome, actor: r.actorId }, "capability.invoke"), + const db = deps.db; + let authorize: AuthorizationChecker | undefined; + if (db && deps.settings) authorize = createAuthorizer({ db, settings: deps.settings, now: deps.now }); + return new LedgeredCapabilityRegistry(db, { + ...(authorize ? { authorize } : {}), + idempotencyKey: serverIdempotencyKey, + ...(deps.now ? { now: deps.now } : {}), + audit: async (r) => { + log.info( + { capability: r.capabilityId, outcome: r.outcome, actor: r.actorId, source: r.source }, + "capability.invoke", + ); + if (!db) return; + await db + .insertInto("audit_event") + .values({ + actor_type: actorTypeOf(r), + actor_id: r.actorId, + action: r.outcome === "denied" ? "capability.denied" : "capability.invoke", + target_type: "capability", + target_id: r.capabilityId, + payload: JSON.stringify({ + outcome: r.outcome, + version: r.version, + kind: r.kind, + riskLevel: r.riskLevel, + source: r.source ?? null, + agentRunId: r.agentRunId ?? null, + invocationId: r.invocationId ?? null, + idempotencyKey: r.idempotencyKey ?? null, + grantId: r.grantId ?? null, + approvalId: r.approvalId ?? null, + errorCode: r.errorCode ?? null, + errorMessage: r.errorMessage ?? null, + inputSummary: r.inputSummary, + durationMs: r.durationMs, + }), + trace_id: r.agentRunId ?? r.taskId, + }) + .execute(); + // 单次审批只用一次:成功执行后标记 consumed + if (r.outcome === "ok" && r.approvalId) { + await db + .updateTable("approval") + .set({ status: "consumed" }) + .where("id", "=", r.approvalId) + .where("status", "=", "approved") + .execute(); + } + }, }); } + +/** 启动断言:首批 14 个能力全部挂上处理器 */ +export function assertCapabilityCount(registry: CapabilityRegistry): void { + const n = registry.list().length; + if (n !== EXPECTED_CAPABILITY_COUNT) { + const ids = registry + .list() + .map((c) => c.descriptor.id) + .sort(); + throw new Error(`能力注册数 ${n} ≠ ${EXPECTED_CAPABILITY_COUNT}:${ids.join(", ")}`); + } +} diff --git a/apps/server/src/modules/agent/fake-scripts.ts b/apps/server/src/modules/agent/fake-scripts.ts new file mode 100644 index 00000000..1bbb9e11 --- /dev/null +++ b/apps/server/src/modules/agent/fake-scripts.ts @@ -0,0 +1,85 @@ +import type { FakeScript, FakeStep, FakeToolResult } from "@umt/agent"; +import { z } from "zod"; + +// fake 模式脚本(pi.provider = "fake"):测试直接注入 FakeScript;设置里的 agent.fakeScript 是可序列化子集(JSON): +// { "match": "收进来" | { "regex": "..." }, "steps": [{ "text"?, "toolCalls": [{ "capability", "input" }] }], "finalText": "..." } +// 或它们的数组。input 里的字符串可以写 "{{download.plan.plan.id}}":取该能力最近一次成功结果里的路径值。 + +const JsonStep = z.object({ + thinking: z.string().optional(), + text: z.string().optional(), + toolCalls: z + .array(z.object({ capability: z.string().min(3), input: z.record(z.string(), z.unknown()).default({}) })) + .optional(), +}); +const JsonScript = z.object({ + name: z.string().optional(), + match: z.union([z.string(), z.object({ regex: z.string() })]), + steps: z.array(JsonStep), + finalText: z.string(), +}); +const JsonScripts = z.union([JsonScript, z.array(JsonScript)]); + +const TEMPLATE = /^\{\{\s*([a-z][a-z0-9]*\.[a-z][a-z0-9_]*)((?:\.[A-Za-z0-9_]+)*)\s*\}\}$/; + +function lookup(results: FakeToolResult[], capability: string, path: string[]): unknown { + const hit = [...results].reverse().find((r) => r.capabilityId === capability && !r.isError); + let cur: unknown = hit?.json; + for (const k of path) { + if (cur === null || cur === undefined) return undefined; + cur = Array.isArray(cur) ? cur[Number(k)] : (cur as Record)[k]; + } + return cur; +} + +function render(v: unknown, results: FakeToolResult[]): unknown { + if (typeof v === "string") { + const m = TEMPLATE.exec(v); + if (!m) return v; + return lookup(results, m[1]!, (m[2] ?? "").split(".").filter(Boolean)) ?? null; + } + if (Array.isArray(v)) return v.map((x) => render(x, results)); + if (v && typeof v === "object") { + return Object.fromEntries(Object.entries(v as Record).map(([k, x]) => [k, render(x, results)])); + } + return v; +} + +/** 解析 settings.agent.fakeScript;不合法时抛错(运行失败并给出原因) */ +export function parseFakeScripts(json: string): FakeScript[] { + const parsed = JsonScripts.parse(JSON.parse(json)); + const list = Array.isArray(parsed) ? parsed : [parsed]; + return list.map((s) => ({ + ...(s.name ? { name: s.name } : {}), + match: typeof s.match === "string" ? s.match : new RegExp(s.match.regex), + steps: s.steps.map( + (st): FakeStep => ({ + ...(st.thinking ? { thinking: st.thinking } : {}), + ...(st.text ? { text: st.text } : {}), + ...(st.toolCalls + ? { + toolCalls: st.toolCalls.map((c) => ({ + capability: c.capability, + input: (results: FakeToolResult[]) => render(c.input, results) as never, + })), + } + : {}), + }), + ), + finalText: s.finalText, + })); +} + +/** 内置演示脚本:任何目标都先查库存再如实汇报(不做写操作) */ +export const DEMO_SCRIPTS: FakeScript[] = [ + { + name: "demo", + match: () => true, + steps: [{ text: "[fake] 先查本地库存。", toolCalls: [{ capability: "inventory.query", input: { limit: 5 } }] }], + finalText: (results) => { + const r = results.find((x) => x.capabilityId === "inventory.query"); + const n = (r?.json as { items?: unknown[] } | undefined)?.items?.length ?? 0; + return `[fake] 演示模式:库存查询返回 ${n} 个作品。演示模型不会提交下载;需要真实执行请在设置里配置 Pi 上游。`; + }, + }, +]; diff --git a/apps/server/src/modules/agent/http.ts b/apps/server/src/modules/agent/http.ts new file mode 100644 index 00000000..4d40c256 --- /dev/null +++ b/apps/server/src/modules/agent/http.ts @@ -0,0 +1,68 @@ +import { createRoute, type RouteConfig } from "@hono/zod-openapi"; +import { type Actor, type ApiScope, apiResponse, type EndpointDef } from "@umt/contracts"; +import type { Context } from "hono"; +import type { z } from "zod"; +import { type AuthEnv, requireScope } from "../../http/auth.ts"; +import type { ModuleApp } from "../types.ts"; + +// 本模块路由封装:吃 contracts 的 EndpointDef(Api.xxx),createRoute 进 OpenAPI + requireScope + 入参校验, +// 出参按合同 parse(剥掉多余列)。 + +// crud() 生成的端点 schema 类型被擦成 z.ZodType(输出 unknown):这时退化为 any,处理器按合同自行使用 +// biome-ignore lint/suspicious/noExplicitAny: 见上 +type Loose = unknown extends O ? any : O; +type Infer = T extends z.ZodType ? Loose> : Record; + +export interface Input { + params: Infer; + query: Infer; + body: Infer; + actor: Actor; + /** 能力/审计用的调用者 ID:管理员会话 user:,API Key api_key: */ + actorId: string; + c: Context; +} + +export function actorIdOf(a: Actor | undefined): string { + if (!a) return "system:unknown"; + return `${a.type === "api_key" ? "api_key" : "user"}:${a.id ?? a.name ?? "?"}`; +} + +export function route( + app: ModuleApp, + def: D, + handler: (input: Input) => Promise, + opts: { tag?: string; status?: 200 | 201 } = {}, +) { + const request: RouteConfig["request"] = {}; + if (def.params) request.params = def.params as never; + if (def.query) request.query = def.query as never; + if (def.body) request.body = { content: { "application/json": { schema: def.body as never } }, required: true }; + const status = opts.status ?? 200; + const r = createRoute({ + method: def.method, + path: def.path, + summary: def.summary, + tags: [opts.tag ?? "agent"], + middleware: [requireScope(def.scope as ApiScope)] as never, + request, + responses: { + [status]: { description: def.summary, content: { "application/json": { schema: apiResponse(def.response) } } }, + }, + }); + // biome-ignore lint/suspicious/noExplicitAny: 路由泛型在封装里擦除,入参已由 zod-openapi 校验 + app.openapi(r, (async (c: any) => { + const actor = c.get("actor") as Actor | undefined; + const input = { + params: def.params ? c.req.valid("param") : {}, + query: def.query ? c.req.valid("query") : {}, + body: def.body ? c.req.valid("json") : {}, + actor, + actorId: actorIdOf(actor), + c, + } as Input; + const data = def.response.parse(await handler(input)); + return c.json({ ok: true, data }, status); + // biome-ignore lint/suspicious/noExplicitAny: 同上 + }) as any); +} diff --git a/apps/server/src/modules/agent/index.ts b/apps/server/src/modules/agent/index.ts index 64bfbc67..4611e1d1 100644 --- a/apps/server/src/modules/agent/index.ts +++ b/apps/server/src/modules/agent/index.ts @@ -1,11 +1,41 @@ -import type { ServerModule } from "../types.ts"; +import type { ServerContext, ServerModule } from "../types.ts"; +import { registerTaskCapabilities } from "./capabilities.ts"; +import { registerAgentRoutes } from "./routes.ts"; +import type { AgentEnv, AgentModuleOptions } from "./runner.ts"; +import { agentOutboxRoutes, registerAgentTasks } from "./tasks.ts"; + +export type { AgentModuleOptions } from "./runner.ts"; /** - * W2-D Agent 装配:14 个能力的 authorize/audit 注入(./capability-registry.ts)、Agent 会话 API + SSE、 - * agent_run_event 落库、resolveDecision 处理器、后台触发的有界 agent 任务。 + * W2-D Agent 装配:能力注册中心的 authorize/audit/账本(./capability-registry.ts,在 context.ts 创建)、 + * task.* 能力、Agent 会话 API(/agent/runs)、持续授权 / 单次审批 / 能力调用 / 业务任务 API、 + * runAgent(有界会话 + 等目标落定 + 业务表汇报)与 resolveDecision(自动调查 / 执行处理结果)。 + * 本模块可以 import 其它模块的 service 导出,反向不行。 */ -export const agentModule: ServerModule = { - name: "agent", - registerRoutes() {}, - registerTasks() {}, -}; +export function createAgentModule(opts: AgentModuleOptions = {}): ServerModule & { env(ctx: ServerContext): AgentEnv } { + const envs = new WeakMap(); + const env = (ctx: ServerContext): AgentEnv => { + let e = envs.get(ctx); + if (!e) { + e = { ctx, opts }; + envs.set(ctx, e); + } + return e; + }; + return { + name: "agent", + env, + registerRoutes(app, ctx) { + registerAgentRoutes(app, ctx); + }, + registerTasks(registry, ctx) { + registerAgentTasks(registry, () => env(ctx)); + }, + outboxRoutes: () => agentOutboxRoutes, + registerCapabilities(registry, ctx) { + registerTaskCapabilities(registry, ctx); + }, + }; +} + +export const agentModule = createAgentModule(); diff --git a/apps/server/src/modules/agent/report.ts b/apps/server/src/modules/agent/report.ts new file mode 100644 index 00000000..7f6eead4 --- /dev/null +++ b/apps/server/src/modules/agent/report.ts @@ -0,0 +1,142 @@ +import type { AgentGoalReport } from "@umt/contracts"; +import type { DB, Kysely } from "@umt/db"; + +// 目标完成度汇报(REQ §10.7 "完成标准以可核查结果为准"):从业务表汇总本次运行触达的 +// 下载计划 → 下载任务 → 整理记录。"已提交"只代表进入下载器,只有整理记录 succeeded 才算入库; +// 模型文本只作为参考附在 modelText,不参与判定。 + +const DL_FAILED = ["failed", "removed", "missing"]; +const REC_DONE = ["succeeded", "partial", "failed", "cancelled"]; +const PLAN_FAILED = ["failed", "cancelled", "expired"]; + +export async function buildGoalReport( + db: Kysely, + runId: string, + modelText: string | null, + now = new Date(), +): Promise { + const inv = await db + .selectFrom("capability_invocation") + .select(["capability_id", "input", "output"]) + .where("agent_run_id", "=", runId) + .where("status", "=", "succeeded") + .where("capability_id", "in", ["download.submit", "organize.execute"]) + .orderBy("created_at") + .execute(); + const planIds = new Set(); + const recordIds = new Set(); + for (const i of inv) { + const input = (i.input ?? {}) as { planId?: string }; + const output = (i.output ?? {}) as { record?: { id?: string } | null }; + if (i.capability_id === "download.submit" && input.planId) planIds.add(input.planId); + if (i.capability_id === "organize.execute" && output.record?.id) recordIds.add(output.record.id); + } + const plans = planIds.size + ? await db + .selectFrom("acquisition_plan") + .select(["id", "status", "slot_id", "source_release_id"]) + .where("id", "in", [...planIds]) + .execute() + : []; + const downloads = planIds.size + ? await db + .selectFrom("download_task") + .select(["id", "acquisition_plan_id", "status", "name"]) + .where("acquisition_plan_id", "in", [...planIds]) + .execute() + : []; + const dtIds = downloads.map((d) => d.id); + const records = + dtIds.length || recordIds.size + ? await db + .selectFrom("organization_record") + .select(["id", "status", "download_task_id", "media_entity_id", "target_path", "created_at"]) + .where((eb) => + eb.or([ + ...(dtIds.length ? [eb("download_task_id", "in", dtIds)] : []), + ...(recordIds.size ? [eb("id", "in", [...recordIds])] : []), + ]), + ) + .orderBy("created_at") + .execute() + : []; + + let imported = 0; + let failed = 0; + let inProgress = 0; + const submitted = downloads.length; + // 按下载任务判定;最后一条整理记录为准 + for (const dt of downloads) { + const recs = records.filter((r) => r.download_task_id === dt.id); + const last = recs.at(-1); + if (last?.status === "succeeded" || last?.status === "partial") imported++; + else if (last && REC_DONE.includes(last.status)) failed++; + else if (DL_FAILED.includes(dt.status)) failed++; + else inProgress++; + } + // 还没进入下载器的计划 + for (const p of plans) { + if (downloads.some((d) => d.acquisition_plan_id === p.id)) continue; + if (PLAN_FAILED.includes(p.status)) failed++; + else inProgress++; + } + // 直接执行的整理(不经下载) + for (const r of records) { + if (r.download_task_id && dtIds.includes(r.download_task_id)) continue; + if (r.status === "succeeded" || r.status === "partial") imported++; + else if (REC_DONE.includes(r.status)) failed++; + else inProgress++; + } + + const touched = plans.length + records.length; + const state: AgentGoalReport["state"] = + touched === 0 + ? "no_actions" + : inProgress > 0 + ? "waiting" + : failed === 0 + ? "completed" + : imported > 0 + ? "partial" + : "failed"; + const done = records.filter((r) => r.status === "succeeded" || r.status === "partial"); + const summary = + state === "no_actions" + ? "本次运行没有提交下载或整理;结论见模型汇报(未产生可核查的业务变更)。" + : [ + `已提交下载 ${submitted} 个,已入库 ${imported} 个,进行中 ${inProgress} 个,失败 ${failed} 个。`, + state === "waiting" ? "已提交≠已入库:仍在等待下载完成与核验入库。" : "", + done.length ? `整理记录:${done.map((r) => `${r.id}(${r.status})`).join("、")}。` : "", + ] + .filter(Boolean) + .join(""); + return { + state, + summary, + submitted, + imported, + inProgress, + failed, + plans: plans.map((p) => ({ + planId: p.id, + status: p.status, + slotId: p.slot_id, + sourceReleaseId: p.source_release_id, + })), + downloads: downloads.map((d) => ({ + downloadTaskId: d.id, + planId: d.acquisition_plan_id, + status: d.status, + name: d.name, + })), + organization: records.map((r) => ({ + recordId: r.id, + status: r.status, + downloadTaskId: r.download_task_id, + mediaEntityId: r.media_entity_id, + targetPath: r.target_path, + })), + modelText, + generatedAt: now.toISOString(), + }; +} diff --git a/apps/server/src/modules/agent/routes.ts b/apps/server/src/modules/agent/routes.ts new file mode 100644 index 00000000..7b17ec67 --- /dev/null +++ b/apps/server/src/modules/agent/routes.ts @@ -0,0 +1,428 @@ +import { isCapabilityError } from "@umt/agent"; +import { + type AgentGoalReport, + Api, + CAPABILITY_CATALOG, + type CapabilityDescriptor, + type KnownCapabilityId, +} from "@umt/contracts"; +import { type AgentRunRow, sql } from "@umt/db"; +import { hasScope } from "../../http/auth.ts"; +import { ApiException } from "../../http/errors.ts"; +import type { ModuleApp, ServerContext } from "../types.ts"; +import { planBinding } from "./authorize.ts"; +import { cancelTask, retryTask, taskToApi } from "./capabilities.ts"; +import { route } from "./http.ts"; +import { AgentRunError, abortRun, appendMessage, startRun } from "./service.ts"; + +// Agent 会话 API、持续授权 / 单次审批、能力目录与调用、业务任务。 + +/** crud() 生成的列表 query 类型被擦除,这里按合同收窄 */ +type ListQuery = { + cursor?: string; + limit: number; + status?: string; + planId?: string; + kind?: string; + subjectId?: string; +}; + +const toIso = (d: Date | null) => (d ? d.toISOString() : null); + +export function runToApi(r: AgentRunRow) { + return { + id: r.id, + taskId: r.task_id, + parentRunId: r.parent_run_id, + trigger: r.trigger, + goal: r.goal, + status: r.status, + provider: r.provider, + model: r.model, + thinkingLevel: r.thinking_level, + scope: r.scope, + budget: r.budget, + usage: r.usage, + evidenceVersion: r.evidence_version, + sessionState: null, + summary: r.summary, + toolCallCount: r.tool_call_count, + lastEventSeq: Number(r.last_event_seq), + error: r.error, + cancelRequestedAt: toIso(r.cancel_requested_at), + startedAt: toIso(r.started_at), + finishedAt: toIso(r.finished_at), + createdBy: r.created_by, + mode: r.mode, + dedupeKey: r.dedupe_key, + createdAt: r.created_at.toISOString(), + updatedAt: r.updated_at.toISOString(), + }; +} + +/** snake_case 行 → camelCase(Date → ISO) */ +function camel(row: object): Record { + const out: Record = {}; + for (const [k, v] of Object.entries(row)) + out[k.replace(/_([a-z0-9])/g, (_, c: string) => c.toUpperCase())] = v instanceof Date ? v.toISOString() : v; + return out; +} + +function mapErr(err: unknown): never { + if (err instanceof AgentRunError) + throw new ApiException( + err.code === "budget" ? "rate_limited" : err.code === "bad_request" ? "validation_failed" : err.code, + err.message, + ); + throw err; +} + +const encode = (at: Date, id: string) => Buffer.from(`${at.toISOString()}|${id}`).toString("base64url"); +const decode = (c?: string) => { + if (!c) return null; + const [at, id] = Buffer.from(c, "base64url").toString().split("|"); + return at && id ? { at, id } : null; +}; + +export function registerAgentRoutes(app: ModuleApp, ctx: ServerContext) { + const db = () => ctx.db; + + // ── Agent 会话 ── + route(app, Api.agentRuns.list, async ({ query }) => { + let q = db().selectFrom("agent_run").selectAll(); + if (query.status) q = q.where("status", "=", query.status); + if (query.trigger) q = q.where("trigger", "=", query.trigger); + const cur = decode(query.cursor); + if (cur) q = q.where(sql`(created_at, id) < (${cur.at}::timestamptz, ${cur.id}::uuid)`); + const rows = await q + .orderBy("created_at", "desc") + .orderBy("id", "desc") + .limit(query.limit + 1) + .execute(); + const items = rows.slice(0, query.limit); + const last = items.at(-1); + return { + items: items.map(runToApi), + nextCursor: rows.length > query.limit && last ? encode(last.created_at, last.id) : null, + }; + }); + + route(app, Api.agentRuns.get, async ({ params, query }) => { + const run = await db().selectFrom("agent_run").selectAll().where("id", "=", params.id).executeTakeFirst(); + if (!run) throw new ApiException("not_found", "Agent 运行不存在"); + const events = await db() + .selectFrom("agent_run_event") + .selectAll() + .where("run_id", "=", params.id) + .where("seq", ">", query.afterSeq) + .orderBy("seq") + .limit(query.limit + 1) + .execute(); + const page = events.slice(0, query.limit); + const messages = await db() + .selectFrom("agent_run_message") + .selectAll() + .where("run_id", "=", params.id) + .orderBy("created_at") + .execute(); + const report = await db() + .selectFrom("artifact") + .select("content") + .where("agent_run_id", "=", params.id) + .where("kind", "=", "report") + .orderBy("created_at", "desc") + .executeTakeFirst(); + return { + run: runToApi(run), + events: page.map((e) => ({ + runId: e.run_id, + seq: Number(e.seq), + type: e.type, + at: e.at.toISOString(), + payload: e.payload, + })), + nextAfterSeq: events.length > query.limit ? Number(page.at(-1)!.seq) : null, + messages: messages.map(camel), + report: (report?.content as AgentGoalReport | undefined) ?? null, + }; + }); + + route( + app, + Api.agentRuns.create, + async ({ body, actorId }) => { + const r = await startRun(ctx, { + goal: body.goal, + mode: body.mode, + capabilityAllowlist: body.capabilityAllowlist, + trigger: "user", + createdBy: actorId, + }).catch(mapErr); + return runToApi(r.run); + }, + { status: 201 }, + ); + + route(app, Api.agentRuns.message, async ({ params, body, actorId }) => + runToApi(await appendMessage(ctx, params.id, body.text, actorId).catch(mapErr)), + ); + + route(app, Api.agentRuns.abort, async ({ params, body, actorId }) => + runToApi(await abortRun(ctx, params.id, body.reason, actorId).catch(mapErr)), + ); + + // ── 持续授权 ── + route(app, Api.automationGrants.list, async ({ query: raw }) => { + const query = raw as ListQuery; + let q = db().selectFrom("automation_grant").selectAll(); + if (query.status) q = q.where("status", "=", query.status as never); + const cur = decode(query.cursor); + if (cur) q = q.where(sql`(created_at, id) < (${cur.at}::timestamptz, ${cur.id}::uuid)`); + const rows = await q + .orderBy("created_at", "desc") + .orderBy("id", "desc") + .limit(query.limit + 1) + .execute(); + const items = rows.slice(0, query.limit); + const last = items.at(-1); + return { + items: items.map(camel), + nextCursor: rows.length > query.limit && last ? encode(last.created_at, last.id) : null, + }; + }); + route(app, Api.automationGrants.get, async ({ params }) => { + const r = await db().selectFrom("automation_grant").selectAll().where("id", "=", params.id).executeTakeFirst(); + if (!r) throw new ApiException("not_found", "授权不存在"); + return camel(r); + }); + route( + app, + Api.automationGrants.create!, + async ({ body, actorId }) => { + const r = await db() + .insertInto("automation_grant") + .values({ + action: body.action, + policy_revision_id: body.policyRevisionId ?? null, + scope: JSON.stringify(body.scope ?? {}), + budget: JSON.stringify(body.budget ?? {}), + granted_by: actorId, + reason: body.reason ?? null, + expires_at: body.expiresAt ? new Date(body.expiresAt) : null, + }) + .returningAll() + .executeTakeFirstOrThrow(); + await audit(ctx, actorId, "automation_grant.create", r.id, { action: r.action }); + return camel(r); + }, + { status: 201 }, + ); + route(app, Api.automationGrants.update!, async ({ params, body, actorId }) => { + const cur = await db().selectFrom("automation_grant").selectAll().where("id", "=", params.id).executeTakeFirst(); + if (!cur) throw new ApiException("not_found", "授权不存在"); + if (body.expiresAt && cur.expires_at && new Date(body.expiresAt) > cur.expires_at) + throw new ApiException("validation_failed", "只能缩短有效期;扩大范围请新建授权"); + const r = await db() + .updateTable("automation_grant") + .set({ + ...(body.status === "revoked" ? { status: "revoked", revoked_at: new Date(), revoked_by: actorId } : {}), + ...(body.expiresAt !== undefined ? { expires_at: body.expiresAt ? new Date(body.expiresAt) : null } : {}), + ...(body.reason !== undefined ? { reason: body.reason } : {}), + }) + .where("id", "=", params.id) + .returningAll() + .executeTakeFirstOrThrow(); + await audit(ctx, actorId, "automation_grant.update", r.id, { status: r.status }); + return camel(r); + }); + + // ── 单次审批 ── + route(app, Api.approvals.list, async ({ query: raw }) => { + const query = raw as ListQuery; + let q = db().selectFrom("approval").selectAll(); + if (query.status) q = q.where("status", "=", query.status as never); + if (query.planId) q = q.where("plan_id", "=", query.planId); + const cur = decode(query.cursor); + if (cur) q = q.where(sql`(created_at, id) < (${cur.at}::timestamptz, ${cur.id}::uuid)`); + const rows = await q + .orderBy("created_at", "desc") + .orderBy("id", "desc") + .limit(query.limit + 1) + .execute(); + const items = rows.slice(0, query.limit); + const last = items.at(-1); + return { + items: items.map(camel), + nextCursor: rows.length > query.limit && last ? encode(last.created_at, last.id) : null, + }; + }); + route(app, Api.approvals.get, async ({ params }) => { + const r = await db().selectFrom("approval").selectAll().where("id", "=", params.id).executeTakeFirst(); + if (!r) throw new ApiException("not_found", "审批不存在"); + return camel(r); + }); + route( + app, + Api.approvals.create, + async ({ body, actorId }) => { + const b = await planBinding(db(), body.planType, body.planId); + if (!b) throw new ApiException("not_found", "计划不存在"); + const r = await db() + .insertInto("approval") + .values({ + plan_type: body.planType, + plan_id: body.planId, + payload_hash: b.payloadHash, + expected_revision: b.revision, + requested_by: actorId, + expires_at: new Date(Date.now() + body.expiresInSec * 1000), + reason: body.reason ?? null, + }) + .returningAll() + .executeTakeFirstOrThrow(); + return camel(r); + }, + { status: 201 }, + ); + route(app, Api.approvals.update!, async ({ params, body, actorId }) => { + const r = await db() + .transaction() + .execute(async (trx) => { + const cur = await trx + .selectFrom("approval") + .selectAll() + .where("id", "=", params.id) + .forUpdate() + .executeTakeFirst(); + if (!cur) throw new ApiException("not_found", "审批不存在"); + if (cur.payload_hash !== body.payloadHash) + throw new ApiException("conflict", "payloadHash 不一致:计划已变化,请重新申请审批"); + const allowed = body.status === "revoked" ? ["pending", "approved"] : ["pending"]; + if (!allowed.includes(cur.status)) throw new ApiException("conflict", `审批已是 ${cur.status}`); + return trx + .updateTable("approval") + .set({ status: body.status, decided_by: actorId, decided_at: new Date(), reason: body.reason ?? cur.reason }) + .where("id", "=", params.id) + .returningAll() + .executeTakeFirstOrThrow(); + }); + await audit(ctx, actorId, `approval.${body.status}`, r.id, { planType: r.plan_type, planId: r.plan_id }); + return camel(r); + }); + + // ── 能力目录与调用(Web / MCP 与 Agent 同一条 invoke 路径)── + route(app, Api.capabilities.list, async () => ({ + items: ctx.capabilities.list().map( + (c): CapabilityDescriptor => ({ + ...CAPABILITY_CATALOG[c.descriptor.id as KnownCapabilityId], + }), + ), + })); + route(app, Api.capabilities.invoke, async ({ params, body, actor, actorId, c }) => { + const entry = ctx.capabilities.get(params.capabilityId); + if (!entry) throw new ApiException("not_found", `能力未注册:${params.capabilityId}`); + const scopes = (c.get("principal")?.scopes ?? actor?.scopes ?? []) as string[]; + if (!hasScope(scopes, entry.descriptor.scope)) + throw new ApiException("forbidden", `调用 ${entry.descriptor.id} 需要 scope: ${entry.descriptor.scope}`); + if (body.preview && !entry.descriptor.supportsPreview) + throw new ApiException("bad_request", `${entry.descriptor.id} 不支持预览`); + try { + const output = await ctx.capabilities.invoke(params.capabilityId, body.input, { + actorId, + taskId: `http:${c.get("requestId") ?? "req"}`, + scopeIds: [], + signal: c.req.raw.signal, + source: "web", + }); + const o = (output ?? {}) as { taskId?: unknown; planId?: unknown; plan?: { id?: unknown } | null }; + return { + output, + taskId: typeof o.taskId === "string" ? o.taskId : null, + planId: typeof o.planId === "string" ? o.planId : typeof o.plan?.id === "string" ? o.plan.id : null, + }; + } catch (err) { + if (!isCapabilityError(err)) throw err; + const code = + err.code === "unauthorized" + ? "forbidden" + : err.code === "not_found" + ? "not_found" + : err.code === "conflict" + ? "conflict" + : err.code === "precondition" || err.code === "budget" + ? "precondition_failed" + : err.code === "invalid_input" + ? "validation_failed" + : "internal"; + throw new ApiException(code, err.message); + } + }); + + // ── 业务任务 ── + route(app, Api.tasks.list, async ({ query: raw }) => { + const query = raw as ListQuery; + let q = db().selectFrom("task").selectAll(); + if (query.status) q = q.where("status", "=", query.status as never); + if (query.kind) q = q.where("kind", "=", query.kind); + if (query.subjectId) q = q.where("subject_id", "=", query.subjectId); + const cur = decode(query.cursor); + if (cur) q = q.where(sql`(created_at, id) < (${cur.at}::timestamptz, ${cur.id}::uuid)`); + const rows = await q + .orderBy("created_at", "desc") + .orderBy("id", "desc") + .limit(query.limit + 1) + .execute(); + const items = rows.slice(0, query.limit); + const last = items.at(-1); + return { + items: items.map(taskToApi), + nextCursor: rows.length > query.limit && last ? encode(last.created_at, last.id) : null, + }; + }); + route(app, Api.tasks.get, async ({ params }) => { + const r = await db().selectFrom("task").selectAll().where("id", "=", params.id).executeTakeFirst(); + if (!r) throw new ApiException("not_found", "任务不存在"); + return taskToApi(r); + }); + route(app, Api.tasks.operations, async ({ params }) => { + const rows = await db() + .selectFrom("asset_operation") + .selectAll() + .where("task_id", "=", params.id) + .orderBy("created_at") + .execute(); + return { items: rows.map(camel) }; + }); + route(app, Api.tasks.cancel, async ({ params, body }) => { + try { + return taskToApi((await cancelTask(ctx, params.id, body.reason)).row); + } catch (err) { + if (isCapabilityError(err)) + throw new ApiException(err.code === "not_found" ? "not_found" : "conflict", err.message); + throw err; + } + }); + route(app, Api.tasks.retry, async ({ params, actorId }) => { + try { + return taskToApi(await retryTask(ctx, params.id, actorId)); + } catch (err) { + if (isCapabilityError(err)) + throw new ApiException(err.code === "not_found" ? "not_found" : "conflict", err.message); + throw err; + } + }); +} + +async function audit(ctx: ServerContext, actorId: string, action: string, targetId: string, payload: object) { + await ctx.db + .insertInto("audit_event") + .values({ + actor_type: actorId.startsWith("api_key:") ? "api_key" : "user", + actor_id: actorId, + action, + target_type: action.split(".")[0]!, + target_id: targetId, + payload: JSON.stringify(payload), + }) + .execute(); +} diff --git a/apps/server/src/modules/agent/runner.ts b/apps/server/src/modules/agent/runner.ts new file mode 100644 index 00000000..475e6356 --- /dev/null +++ b/apps/server/src/modules/agent/runner.ts @@ -0,0 +1,434 @@ +import { join, resolve } from "node:path"; +import { + type AgentRunHandle, + type AgentSettings, + buildSystemPrompt, + type CapabilityRegistry, + createFakeRuntime, + type FakeScript, + PiAgentRuntime, + type RunOutcome, +} from "@umt/agent"; +import { + type AgentBudget, + type AgentGoalReport, + type AgentRunEvent, + type AgentRunStatus, + agentRunEventToBusEvent, + agentRunEventToRow, + CAPABILITY_CATALOG, + type CapabilityKind, + type GlobalSettings, + KnownCapabilityId, +} from "@umt/contracts"; +import { type AgentRunRow, sql } from "@umt/db"; +import type { ServerContext } from "../types.ts"; +import { DEMO_SCRIPTS, parseFakeScripts } from "./fake-scripts.ts"; +import { buildGoalReport } from "./report.ts"; + +// runAgent 任务的执行体:从设置装配 Pi(或 fake)运行时 → 事件先写 agent_run_event 再推 SSE → +// 崩溃恢复(pendingOperations>0 时先按幂等键对账未知结果再 resume)→ 消费追加消息 → 汇报以业务表为准。 + +export interface AgentModuleOptions { + /** Pi JSONL 会话目录(崩溃恢复点;新 worker 需能读到同一目录) */ + sessionDir?: string; + /** fake 模式脚本(测试注入);缺省读 settings.agent.fakeScript,再缺省用内置演示脚本 */ + fakeScripts?: FakeScript[]; + /** 覆盖 settings.agent.goalPollSec(测试用小值) */ + goalPollSec?: number; +} + +export interface AgentEnv { + ctx: ServerContext; + opts: AgentModuleOptions; +} + +/** "先给方案"与后台调查只允许这些类别 */ +export const PLAN_ONLY_KINDS: CapabilityKind[] = ["read", "preview", "propose"]; +/** 后台自动调查的能力白名单:只读/预览 + 身份提案 */ +export const INVESTIGATE_CAPABILITIES = [ + "catalog.lookup", + "release.search", + "release.inspect", + "inventory.query", + "task.inspect", + "identity.resolve", +] as const; + +export interface RunScope { + capabilities: string[]; + allowedKinds: CapabilityKind[] | null; + decisionId?: string | null; +} + +/** 按模式确定能力白名单与类别上限;plan_only 下给了 execute 类能力直接报错 */ +export function scopeFor(mode: "execute" | "plan_only", allowlist?: readonly string[]): RunScope { + const all = Object.values(CAPABILITY_CATALOG); + if (mode === "plan_only") { + const bad = (allowlist ?? []).filter((id) => { + const d = CAPABILITY_CATALOG[id as KnownCapabilityId]; + return !d || !PLAN_ONLY_KINDS.includes(d.kind); + }); + if (bad.length) throw new Error(`plan_only 模式只允许 read/preview/propose 类能力,不允许:${bad.join(", ")}`); + return { + capabilities: allowlist?.length + ? [...allowlist] + : all.filter((d) => PLAN_ONLY_KINDS.includes(d.kind)).map((d) => d.id), + allowedKinds: PLAN_ONLY_KINDS, + }; + } + for (const id of allowlist ?? []) KnownCapabilityId.parse(id); + return { capabilities: allowlist?.length ? [...allowlist] : all.map((d) => d.id), allowedKinds: null }; +} + +export function sessionDirOf(env: AgentEnv): string { + return resolve( + env.opts.sessionDir ?? Deno.env.get("AGENT_SESSION_DIR") ?? join(Deno.cwd(), "data", "agent-sessions"), + ); +} + +export interface DriveResult { + status: "completed" | "failed" | "cancelled"; + reason: "budget" | "aborted" | "error" | null; + budget: string | null; + message: string | null; + finalText: string; +} + +/** 设置 → @umt/agent 的运行参数(apiKey 现场经 secrets 解密,不落日志) */ +async function agentSettings( + env: AgentEnv, + s: GlobalSettings, + budget: AgentBudget, + fake: boolean, +): Promise { + const pi = s.pi; + let apiKey = "fake-key-not-used"; + if (!fake) { + if (!pi.apiKeySecretRef) throw new Error("未配置 Pi 上游密钥(设置 → pi.apiKey)"); + apiKey = await env.ctx.secrets.resolve(pi.apiKeySecretRef); + if (!pi.model) throw new Error("未配置模型(设置 → pi.model)"); + } + const maxTokens = budget.maxTokens ?? s.agent.maxTokensPerRun ?? undefined; + const maxCost = budget.maxCostUsd ?? s.agent.maxCostUsdPerRun ?? undefined; + return { + provider: pi.provider === "openai-compatible" ? "openai-compatible" : "anthropic", + ...(pi.baseUrl && !fake ? { baseUrl: pi.baseUrl } : {}), + apiKey, + model: pi.model ?? "fake-model", + thinkingLevel: pi.thinkingLevel, + contextBudgetTokens: pi.contextBudget, + compaction: { + mode: pi.compaction.enabled ? "threshold" : "manual", + threshold: Math.min(0.95, Math.max(0.5, pi.compaction.thresholdRatio)), + }, + subagentsEnabled: pi.subagentsEnabled, + maxToolCalls: budget.maxToolCalls ?? s.agent.maxToolCalls, + maxDurationMs: (budget.maxDurationSec ?? s.agent.maxDurationSec) * 1000, + ...(maxTokens !== undefined || maxCost !== undefined + ? { + costBudget: { + ...(maxTokens !== undefined ? { maxTotalTokens: maxTokens } : {}), + ...(maxCost !== undefined && maxCost > 0 ? { maxCostUsd: maxCost } : {}), + }, + } + : {}), + }; +} + +/** 事件:先落库(run_id, seq 唯一)再推全局 SSE;未授权的工具失败在总线上标为 denied */ +function eventSink(env: AgentEnv) { + const { ctx } = env; + return async (e: AgentRunEvent) => { + const row = agentRunEventToRow(e); + await ctx.db + .insertInto("agent_run_event") + .values({ + run_id: row.runId, + seq: row.seq, + type: row.type, + at: new Date(row.at), + payload: JSON.stringify(row.payload), + }) + .onConflict((oc) => oc.columns(["run_id", "seq"]).doNothing()) + .execute(); + await ctx.db + .updateTable("agent_run") + .set({ + last_event_seq: sql`greatest(last_event_seq, ${e.seq})`, + ...(e.type === "tool_call_start" && !e.recovery ? { tool_call_count: sql`tool_call_count + 1` } : {}), + ...(e.type === "run_completed" || e.type === "run_failed" ? { usage: JSON.stringify(e.usage) } : {}), + }) + .where("id", "=", e.runId) + .execute(); + const bus = agentRunEventToBusEvent(e); + if (!bus) return; + // run_completed/run_failed 只代表模型本轮结束;agent_run 的最终状态由 finalize 按业务结果发布 + if (bus.type === "agent.run_status" && (e.type === "run_completed" || e.type === "run_failed")) return; + if ( + bus.type === "agent.tool_call" && + e.type === "tool_call_end" && + e.isError && + /"code":"unauthorized"/.test(e.outputSummary) + ) { + bus.payload.phase = "denied"; + } + void ctx.events.publish(bus.type, bus.payload); + }; +} + +export async function publishStatus(ctx: ServerContext, runId: string, status: AgentRunStatus, summary: string | null) { + void ctx.events.publish("agent.run_status", { runId, status, summary }); +} + +/** 取下一条待消费的追加消息(原子认领,两个驱动者不会拿到同一条) */ +async function claimMessage(ctx: ServerContext, runId: string): Promise { + const r = await sql<{ text: string }>` + update agent_run_message set status = 'consumed', consumed_at = now() + where id = (select id from agent_run_message where run_id = ${runId} and status = 'pending' + order by created_at, id limit 1 for update skip locked) + returning text`.execute(ctx.db); + return r.rows[0]?.text ?? null; +} + +/** + * 崩溃恢复对账:本 run 在账本里停在 running 的有副作用调用,按同一幂等键重跑处理器 + * (处理器自身幂等:下载按计划状态 + qB hash,整理按记录状态),让账本落到真实结果。 + * 之后 resume 时模型拿到"结果未知",再次调用同参数能力会直接得到账本里的结果。 + */ +export async function reconcilePending( + env: AgentEnv, + registry: CapabilityRegistry, + run: AgentRunRow, + signal: AbortSignal, +) { + const rows = await env.ctx.db + .selectFrom("capability_invocation") + .selectAll() + .where("agent_run_id", "=", run.id) + .where("status", "=", "running") + .orderBy("created_at") + .execute(); + for (const r of rows) { + try { + await registry.invoke(r.capability_id, r.input, { + actorId: r.actor_id, + taskId: run.task_id ?? run.id, + scopeIds: [], + signal, + source: "task", + agentRunId: run.id, + invocationId: `reconcile:${r.id}`, + }); + } catch (err) { + env.ctx.logger.warn( + { runId: run.id, capability: r.capability_id, err: { message: (err as Error).message } }, + "agent 崩溃恢复对账失败", + ); + } + } +} + +function systemPromptFor(run: AgentRunRow, scope: RunScope): string { + const constraints: string[] = [ + `可用能力:${scope.capabilities.join(", ")}`, + "有副作用的能力(提交下载、执行整理、发布策略、重试/取消任务)需要服务端授权;被拒绝时如实说明缺少哪项授权。", + ]; + if (run.mode === "plan_only") + constraints.push("本次为“先给方案”:只调查与规划,不提交下载、不做文件变更、不发布策略。"); + if (run.trigger === "event") + constraints.push("本任务由后台事件触发,是有界的只读调查:收集证据、给出结论与建议,不要反复重试同一操作。"); + return buildSystemPrompt({ + role: run.trigger === "event" ? "媒体库异常调查员" : "媒体库自动化助手", + goal: run.goal, + constraints, + }); +} + +/** 读 agent_run_event 里最后一轮的结束事件(崩溃发生在"模型已结束、业务状态未写"之间时复用) */ +async function lastOutcomeEvent(env: AgentEnv, runId: string) { + const r = await env.ctx.db + .selectFrom("agent_run_event") + .select(["type", "payload", "seq"]) + .where("run_id", "=", runId) + .where("type", "in", ["run_started", "run_completed", "run_failed"]) + .orderBy("seq", "desc") + .executeTakeFirst(); + return r ?? null; +} + +async function dailyBudgetBlock(env: AgentEnv, s: GlobalSettings, run: AgentRunRow): Promise { + if (run.started_at) return null; // 续跑不重复计数 + const day = new Date(); + day.setHours(0, 0, 0, 0); + const r = await sql<{ n: number; cost: number | null }>` + select count(*)::int as n, sum(coalesce((usage->>'costUsd')::float8, 0))::float8 as cost + from agent_run where started_at >= ${day} and id <> ${run.id}`.execute(env.ctx.db); + const n = r.rows[0]?.n ?? 0; + const cost = r.rows[0]?.cost ?? 0; + if (n >= s.budget.agentMaxRunsPerDay) return `今日 Agent 运行次数已达上限 ${s.budget.agentMaxRunsPerDay}`; + if (cost >= s.budget.agentMaxCostUsdPerDay) return `今日 Agent 费用已达上限 $${s.budget.agentMaxCostUsdPerDay}`; + return null; +} + +/** + * 驱动一次模型会话直到本轮结束(含追加消息)。整个函数在一个持久步骤里执行:worker 崩溃后新 worker 从头重跑本函数, + * 打开同一 JSONL 会话,pendingOperations>0 时对账 + resume,已完成的工具调用不会重放。 + */ +export async function drive( + env: AgentEnv, + registry: CapabilityRegistry, + runId: string, + signal: AbortSignal, +): Promise { + const { ctx } = env; + const run = await ctx.db.selectFrom("agent_run").selectAll().where("id", "=", runId).executeTakeFirstOrThrow(); + if (run.cancel_requested_at || signal.aborted) + return { status: "cancelled", reason: "aborted", budget: null, message: "运行已被取消", finalText: "" }; + const s = await ctx.settings.all(); + const blocked = await dailyBudgetBlock(env, s, run); + if (blocked) return { status: "failed", reason: "budget", budget: "daily", message: blocked, finalText: "" }; + + const fake = s.pi.provider === "fake"; + const scope = (run.scope ?? {}) as unknown as RunScope; + let settings: AgentSettings; + try { + settings = await agentSettings(env, s, (run.budget ?? {}) as AgentBudget, fake); + } catch (err) { + return { status: "failed", reason: "error", budget: null, message: (err as Error).message, finalText: "" }; + } + await ctx.db + .updateTable("agent_run") + .set({ + status: "running", + started_at: run.started_at ?? new Date(), + provider: fake ? "fake" : s.pi.provider, + model: fake ? "fake-model" : s.pi.model, + thinking_level: s.pi.thinkingLevel, + }) + .where("id", "=", runId) + .execute(); + void publishStatus(ctx, runId, "running", null); + + const onEvent = eventSink(env); + const onEventError = (err: unknown, e: AgentRunEvent) => + ctx.logger.warn({ runId, seq: e.seq, err: { message: (err as Error).message } }, "agent 事件落库失败"); + let scripts: FakeScript[] = DEMO_SCRIPTS; + if (fake) { + try { + scripts = env.opts.fakeScripts ?? (s.agent.fakeScript ? parseFakeScripts(s.agent.fakeScript) : DEMO_SCRIPTS); + } catch (err) { + return { + status: "failed", + reason: "error", + budget: null, + message: `fakeScript 不合法:${(err as Error).message}`, + finalText: "", + }; + } + } + const runtime = fake + ? createFakeRuntime({ registry, scripts, onEvent, onEventError }).runtime + : new PiAgentRuntime({ registry, onEvent, onEventError }); + const maxSeq = await ctx.db + .selectFrom("agent_run_event") + .select(sql`max(seq)`.as("m")) + .where("run_id", "=", runId) + .executeTakeFirst(); + const actor = { + actorId: run.created_by ?? `agent:${runId}`, + taskId: run.task_id ?? runId, + scopeIds: [] as string[], + ...(scope.allowedKinds ? { allowedKinds: scope.allowedKinds } : {}), + }; + let handle: AgentRunHandle; + try { + handle = await runtime.createRun({ + runId, + goal: run.goal, + systemPrompt: systemPromptFor(run, scope), + capabilities: scope.capabilities?.length ? scope.capabilities : Object.keys(CAPABILITY_CATALOG), + settings, + sessionDir: sessionDirOf(env), + actor, + firstSeq: maxSeq?.m === null || maxSeq?.m === undefined ? 0 : Number(maxSeq.m) + 1, + }); + } catch (err) { + return { status: "failed", reason: "error", budget: null, message: (err as Error).message, finalText: "" }; + } + + // 取消传播:任务取消信号或 agent_run.cancel_requested_at → 中止会话 + let aborting = false; + const abortOnce = () => { + if (aborting) return; + aborting = true; + void handle.abort("用户取消"); + }; + signal.addEventListener("abort", abortOnce, { once: true }); + const watch = setInterval(async () => { + const r = await ctx.db + .selectFrom("agent_run") + .select("cancel_requested_at") + .where("id", "=", runId) + .executeTakeFirst() + .catch(() => null); + if (r?.cancel_requested_at) abortOnce(); + }, 1000); + + try { + let outcome: RunOutcome | null = null; + if (handle.pendingOperations > 0) { + await reconcilePending(env, registry, run, signal); + outcome = await handle.resume(); + } else { + const last = await lastOutcomeEvent(env, runId); + if (!last) outcome = await handle.prompt(); + else if (last.type === "run_started") outcome = await handle.resume(); + else { + const p = last.payload as { finalText?: string; reason?: string; message?: string; budget?: string }; + outcome = + last.type === "run_completed" + ? { status: "completed", finalText: p.finalText ?? "", usage: {} as never } + : { + status: "failed", + reason: (p.reason ?? "error") as RunOutcome["reason"] & string, + message: p.message ?? "", + finalText: "", + usage: {} as never, + ...(p.budget ? { budget: p.budget as never } : {}), + }; + } + } + while (outcome.status === "completed" && !aborting) { + const text = await claimMessage(ctx, runId); + if (text === null) break; + outcome = await handle.prompt(text); + } + if (outcome.status === "completed") + return { status: "completed", reason: null, budget: null, message: null, finalText: outcome.finalText }; + return { + status: outcome.reason === "aborted" ? "cancelled" : "failed", + reason: outcome.reason ?? "error", + budget: outcome.budget ?? null, + message: outcome.message ?? null, + finalText: outcome.finalText, + }; + } finally { + clearInterval(watch); + signal.removeEventListener("abort", abortOnce); + await handle.close(); + } +} + +/** 等待目标关联的下载/入库落定:返回报告(state≠waiting)或 null(继续等) */ +export async function goalSettled(env: AgentEnv, runId: string, modelText: string): Promise { + const r = await buildGoalReport(env.ctx.db, runId, modelText); + if (r.state !== "waiting") return r; + await env.ctx.db + .updateTable("agent_run") + .set({ status: "waiting", summary: r.summary }) + .where("id", "=", runId) + .where("status", "in", ["running", "waiting"]) + .execute(); + return null; +} diff --git a/apps/server/src/modules/agent/service.ts b/apps/server/src/modules/agent/service.ts new file mode 100644 index 00000000..a0743786 --- /dev/null +++ b/apps/server/src/modules/agent/service.ts @@ -0,0 +1,353 @@ +import { createHash } from "node:crypto"; +import type { + AgentBudget, + AgentGoalReport, + AgentRunMode, + AgentRunStatus, + AgentTrigger, + DecisionKind, +} from "@umt/contracts"; +import { type AgentRunRow, sql } from "@umt/db"; +import type { ServerContext } from "../types.ts"; +import { buildGoalReport } from "./report.ts"; +import { type AgentEnv, type DriveResult, INVESTIGATE_CAPABILITIES, publishStatus, scopeFor } from "./runner.ts"; + +// agent_run 生命周期:建行 + 业务任务行 + 入队 runAgent;追加消息 / 中止;终态汇报(业务表核查)与后台调查去重。 + +const TERMINAL: AgentRunStatus[] = ["succeeded", "failed", "cancelled", "budget_exceeded"]; + +export class AgentRunError extends Error { + constructor( + readonly code: "not_found" | "conflict" | "bad_request" | "budget", + message: string, + ) { + super(message); + } +} + +export interface StartRunInput { + goal: string; + mode: AgentRunMode; + capabilityAllowlist?: readonly string[] | undefined; + trigger: AgentTrigger; + createdBy: string; + /** 后台触发按根因去重;同键已存在则返回已有运行 */ + dedupeKey?: string | null; + budget?: AgentBudget; + allowedKinds?: ("read" | "preview" | "propose")[]; +} + +const sha = (s: string) => createHash("sha256").update(s, "utf8").digest("hex"); + +/** 建 agent_run + task 行,入队 runAgent(不在调用方进程里跑模型) */ +export async function startRun(ctx: ServerContext, p: StartRunInput): Promise<{ run: AgentRunRow; created: boolean }> { + let scope: ReturnType; + try { + scope = scopeFor(p.mode, p.capabilityAllowlist); + } catch (err) { + throw new AgentRunError("bad_request", (err as Error).message); + } + if (p.allowedKinds) scope = { ...scope, allowedKinds: p.allowedKinds }; + const ins = await ctx.db.transaction().execute(async (trx) => { + const row = await sql` + insert into agent_run (trigger, goal, status, scope, budget, mode, dedupe_key, created_by) + values (${p.trigger}, ${p.goal}, 'queued', ${JSON.stringify(scope)}::jsonb, ${JSON.stringify(p.budget ?? {})}::jsonb, + ${p.mode}, ${p.dedupeKey ?? null}, ${p.createdBy}) + on conflict (dedupe_key) where dedupe_key is not null do nothing + returning *`.execute(trx); + const run = row.rows[0]; + if (!run) return null; + const task = await trx + .insertInto("task") + .values({ + kind: "agentRun", + idempotency_key: `agent_run:${run.id}`, + input: JSON.stringify({ agentRunId: run.id }), + subject_type: "agent_run", + subject_id: run.id, + triggered_by: p.createdBy, + }) + .returning("id") + .executeTakeFirstOrThrow(); + return trx + .updateTable("agent_run") + .set({ task_id: task.id }) + .where("id", "=", run.id) + .returningAll() + .executeTakeFirstOrThrow(); + }); + if (!ins) { + const existing = await ctx.db + .selectFrom("agent_run") + .selectAll() + .where("dedupe_key", "=", p.dedupeKey!) + .executeTakeFirstOrThrow(); + return { run: existing, created: false }; + } + await enqueueRun(ctx, ins, `runAgent:${ins.id}`); + void publishStatus(ctx, ins.id, "queued", null); + return { run: ins, created: true }; +} + +async function enqueueRun(ctx: ServerContext, run: AgentRunRow, key: string) { + try { + const { runId } = await ctx.engine.enqueue( + "runAgent", + { agentRunId: run.id }, + { + idempotencyKey: key, + traceId: run.task_id ?? run.id, + }, + ); + if (run.task_id) + await ctx.db + .updateTable("task") + .set({ engine_run_id: runId, status: "pending" }) + .where("id", "=", run.task_id) + .execute(); + } catch (err) { + await ctx.db + .updateTable("agent_run") + .set({ status: "failed", error: JSON.stringify({ message: `入队失败:${(err as Error).message}` }) }) + .where("id", "=", run.id) + .execute(); + throw err; + } +} + +/** 追加用户消息:运行中由当前会话在本轮结束后消费;已结束/等待中则唤醒续跑(新的 runAgent run) */ +export async function appendMessage(ctx: ServerContext, runId: string, text: string, by: string): Promise { + const run = await ctx.db.selectFrom("agent_run").selectAll().where("id", "=", runId).executeTakeFirst(); + if (!run) throw new AgentRunError("not_found", "Agent 运行不存在"); + const msg = await ctx.db + .insertInto("agent_run_message") + .values({ run_id: runId, text, created_by: by }) + .returning("id") + .executeTakeFirstOrThrow(); + if (run.status === "queued" || run.status === "running") return run; + const woke = await ctx.db + .updateTable("agent_run") + .set({ status: "queued", cancel_requested_at: null, finished_at: null }) + .where("id", "=", runId) + .returningAll() + .executeTakeFirstOrThrow(); + await enqueueRun(ctx, woke, `runAgent:${runId}:msg:${msg.id}`); + void publishStatus(ctx, runId, "queued", null); + return woke; +} + +/** 中止:只写取消标志,由会话监视器中止模型并走 finalize(不取消引擎 run,保证终态落库) */ +export async function abortRun(ctx: ServerContext, runId: string, reason: string | undefined, by: string) { + const run = await ctx.db.selectFrom("agent_run").selectAll().where("id", "=", runId).executeTakeFirst(); + if (!run) throw new AgentRunError("not_found", "Agent 运行不存在"); + if (TERMINAL.includes(run.status)) return run; + const now = new Date(); + const row = await ctx.db + .updateTable("agent_run") + .set({ + cancel_requested_at: run.cancel_requested_at ?? now, + error: JSON.stringify({ abortReason: reason ?? "用户中止", by }), + }) + .where("id", "=", runId) + .returningAll() + .executeTakeFirstOrThrow(); + if (run.task_id) + await ctx.db + .updateTable("task") + .set({ cancel_requested_at: now, cancel_reason: reason ?? "用户中止" }) + .where("id", "=", run.task_id) + .execute(); + return row; +} + +async function writeDecision( + ctx: ServerContext, + d: { kind: DecisionKind; title: string; summary: string; subjectId: string; dedupeKey: string; evidence: object }, +) { + await ctx.db.transaction().execute(async (trx) => { + const r = await sql<{ id: string; inserted: boolean }>` + insert into decision (kind, severity, title, summary, subject_type, subject_id, dedupe_key, evidence, agent_run_id) + values (${d.kind}, 'warning', ${d.title}, ${d.summary}, 'agent_run', ${d.subjectId}, ${d.dedupeKey}, + ${JSON.stringify(d.evidence)}::jsonb, ${d.subjectId}) + on conflict (dedupe_key) where status in ('open', 'investigating', 'awaiting_user') + do update set attempts = decision.attempts + 1, evidence = excluded.evidence, updated_at = now() + returning id, (xmax = 0) as inserted`.execute(trx); + const row = r.rows[0]!; + if (row.inserted) + await ctx.outbox.emit( + trx, + "decision.created", + { decisionId: row.id, kind: d.kind, severity: "warning", title: d.title }, + { idempotencyKey: `decision.created:${row.id}` }, + ); + }); +} + +/** 终态:按业务表汇总的报告决定状态;预算超限 → budget_exceeded + 待处理;调查运行回写 decision */ +export async function finalize( + env: AgentEnv, + runId: string, + res: DriveResult, + settled: AgentGoalReport | null, +): Promise<{ status: AgentRunStatus; summary: string }> { + const { ctx } = env; + const run = await ctx.db.selectFrom("agent_run").selectAll().where("id", "=", runId).executeTakeFirstOrThrow(); + const report = settled ?? (await buildGoalReport(ctx.db, runId, res.finalText || null)); + let status: AgentRunStatus; + let summary: string; + if (res.status === "cancelled" || run.cancel_requested_at) { + status = "cancelled"; + summary = `已取消(停止未来步骤,不撤销已完成动作)。${report.state === "no_actions" ? "" : report.summary}`; + } else if (res.reason === "budget") { + status = "budget_exceeded"; + summary = `超出预算:${res.message ?? ""}。${report.state === "no_actions" ? "" : report.summary}`; + } else if (res.status === "failed") { + status = "failed"; + summary = `运行失败:${res.message ?? ""}`; + } else if (report.state === "waiting") { + status = "waiting"; + summary = report.summary; + } else { + status = report.state === "failed" ? "failed" : "succeeded"; + summary = report.summary; + } + // 模型文本只作参考:附在业务核查结论之后,明确标注未经核查 + if (res.finalText && res.status !== "failed") summary += `\n模型汇报(未经核查):${res.finalText}`; + await ctx.db + .insertInto("artifact") + .values({ + kind: "report", + agent_run_id: runId, + task_id: run.task_id, + subject_type: "agent_run", + subject_id: runId, + title: "goal_report", + content: JSON.stringify(report), + content_hash: sha(JSON.stringify({ runId, report })), + }) + .execute(); + await ctx.db + .updateTable("agent_run") + .set({ + status, + summary: summary.slice(0, 4000), + ...(status === "waiting" ? {} : { finished_at: new Date() }), + ...(res.status === "failed" + ? { error: JSON.stringify({ reason: res.reason, budget: res.budget, message: res.message }) } + : {}), + }) + .where("id", "=", runId) + .execute(); + if (run.task_id) { + const taskStatus = + status === "cancelled" + ? "cancelled" + : status === "waiting" + ? "waiting" + : status === "succeeded" + ? "succeeded" + : "failed"; + await ctx.db + .updateTable("task") + .set({ + status: taskStatus, + output: JSON.stringify({ + status, + report: { state: report.state, imported: report.imported, submitted: report.submitted }, + }), + ...(taskStatus === "waiting" ? {} : { finished_at: new Date() }), + }) + .where("id", "=", run.task_id) + .execute(); + } + if (status === "budget_exceeded") { + await writeDecision(ctx, { + kind: "other", + title: "Agent 运行超出预算", + summary: res.message ?? "超出预算", + subjectId: runId, + dedupeKey: `agent_budget:${runId}`, + evidence: { runId, budget: res.budget, message: res.message, goal: run.goal }, + }); + } + // 后台调查:结论回写待处理事项,交还人工(仍可被新证据重新唤醒) + if (run.dedupe_key?.startsWith("investigate:")) { + const decisionId = run.dedupe_key.slice("investigate:".length); + await ctx.db.transaction().execute(async (trx) => { + const r = await trx + .updateTable("decision") + .set({ + status: "awaiting_user", + evidence: sql`evidence || ${JSON.stringify({ agentInvestigation: { runId, status, summary: summary.slice(0, 2000) } })}::jsonb`, + }) + .where("id", "=", decisionId) + .where("status", "=", "investigating") + .returning("id") + .executeTakeFirst(); + if (r) + await ctx.outbox.emit( + trx, + "decision.updated", + { decisionId, status: "awaiting_user" }, + { idempotencyKey: `decision.updated:${decisionId}:investigated:${runId}` }, + ); + }); + } + void publishStatus(ctx, runId, status, summary.slice(0, 2000)); + // 结束前刚到的追加消息:再唤醒一次 + const pending = await ctx.db + .selectFrom("agent_run_message") + .select("id") + .where("run_id", "=", runId) + .where("status", "=", "pending") + .executeTakeFirst(); + if (pending && status !== "cancelled") { + const woke = await ctx.db + .updateTable("agent_run") + .set({ status: "queued", finished_at: null }) + .where("id", "=", runId) + .returningAll() + .executeTakeFirstOrThrow(); + await enqueueRun(ctx, woke, `runAgent:${runId}:msg:${pending.id}`); + } + return { status, summary }; +} + +/** 后台触发:识别歧义 / 下载失败诊断 → 有界只读调查(同一 decision 只一个 run) */ +export async function autoInvestigate( + ctx: ServerContext, + decisionId: string, +): Promise<{ runId: string | null; created: boolean }> { + const d = await ctx.db.selectFrom("decision").selectAll().where("id", "=", decisionId).executeTakeFirst(); + if (d?.status !== "open") return { runId: null, created: false }; + if (!["identity_ambiguous", "download_failed", "download_stalled"].includes(d.kind)) + return { runId: null, created: false }; + // Agent 自己引起的事项不再触发 Agent(防止递归自触发) + if (d.subject_type === "agent_run") return { runId: null, created: false }; + const s = await ctx.settings.get("agent"); + if (!s.autoInvestigate) return { runId: null, created: false }; + const goal = [ + `调查待处理事项 ${d.id}(${d.kind}):${d.title}。`, + d.summary ? `摘要:${d.summary}。` : "", + d.subject_type && d.subject_id ? `对象:${d.subject_type} ${d.subject_id}。` : "", + "只做有界的只读调查:查证据、给出结论与处理建议;不要提交下载或变更文件。", + ].join(""); + const { run, created } = await startRun(ctx, { + goal, + mode: "plan_only", + capabilityAllowlist: [...INVESTIGATE_CAPABILITIES], + trigger: "event", + createdBy: "system:auto-investigate", + dedupeKey: `investigate:${d.id}`, + budget: { maxToolCalls: s.investigateMaxToolCalls, maxDurationSec: Math.min(600, s.maxDurationSec) }, + }); + if (created) { + await ctx.db + .updateTable("decision") + .set({ status: "investigating", agent_run_id: run.id }) + .where("id", "=", d.id) + .where("status", "=", "open") + .execute(); + } + return { runId: run.id, created }; +} diff --git a/apps/server/src/modules/agent/tasks.ts b/apps/server/src/modules/agent/tasks.ts new file mode 100644 index 00000000..40435d5a --- /dev/null +++ b/apps/server/src/modules/agent/tasks.ts @@ -0,0 +1,162 @@ +import type { AgentGoalReport, DecisionOption, EventPayload } from "@umt/contracts"; +import { sql } from "@umt/db"; +import { defineTask, type OutboxRouteFn, type TaskRegistry } from "@umt/tasks"; +import { z } from "zod"; +import type { ServerContext } from "../types.ts"; +import { retryTask } from "./capabilities.ts"; +import { buildGoalReport } from "./report.ts"; +import { type AgentEnv, type DriveResult, drive, goalSettled } from "./runner.ts"; +import { autoInvestigate, finalize } from "./service.ts"; + +// 持久任务:runAgent(驱动一次有界 Agent 会话 → 等目标关联的下载/入库落定 → 以业务表汇报) +// 与 resolveDecision(decision.created → 自动调查;decision.updated(resolved)→ 执行所选处理)。 + +const id = z.string().uuid(); +const DB_RETRY = { maximumAttempts: 3, initialInterval: "1s", backoffCoefficient: 2, maximumInterval: "30s" } as const; + +export function registerAgentTasks(registry: TaskRegistry, env: () => AgentEnv) { + registry.register( + defineTask({ + name: "runAgent", + version: 1, + inputSchema: z.object({ agentRunId: id }), + retryPolicy: DB_RETRY, + handler: async (t) => { + const e = env(); + const runId = t.input.agentRunId; + await t.step.run("start", async () => { + await e.ctx.db + .updateTable("task") + .set({ status: "running", started_at: new Date(), attempts: sql`attempts + 1` }) + .where("subject_type", "=", "agent_run") + .where("subject_id", "=", runId) + .execute(); + return null; + }); + const res: DriveResult = await t.step.run("drive", () => drive(e, e.ctx.capabilities, runId, t.abortSignal)); + let settled: AgentGoalReport | null = null; + const run = await e.ctx.db + .selectFrom("agent_run") + .select(["mode"]) + .where("id", "=", runId) + .executeTakeFirstOrThrow(); + if (res.status === "completed" && run.mode === "execute") { + const s = await e.ctx.settings.get("agent"); + const pollSec = e.opts.goalPollSec ?? s.goalPollSec; + const rounds = Math.max(1, Math.min(900, Math.ceil((s.goalWaitHours * 3600) / pollSec))); + settled = await t.step.waitForFact("goal-settled", { + check: async () => { + const c = await e.ctx.db + .selectFrom("agent_run") + .select("cancel_requested_at") + .where("id", "=", runId) + .executeTakeFirst(); + // 已请求取消:不再等待,finalize 按取消收尾 + if (c?.cancel_requested_at) return buildGoalReport(e.ctx.db, runId, res.finalText || null); + return goalSettled(e, runId, res.finalText); + }, + timeout: `${pollSec}s`, + maxRounds: rounds, + }); + } + return t.step.run("finalize", () => finalize(e, runId, res, settled)); + }, + }), + defineTask({ + name: "resolveDecision", + version: 1, + inputSchema: z.object({ decisionId: id, trigger: z.enum(["created", "updated"]).default("updated") }), + retryPolicy: DB_RETRY, + handler: async (t) => { + const e = env(); + if (t.input.trigger === "created") + return t.step.run("investigate", () => autoInvestigate(e.ctx, t.input.decisionId)); + return t.step.run("execute", () => executeResolution(e.ctx, t.input.decisionId)); + }, + }), + ); +} + +/** + * 执行 decision 的处理结果(身份歧义由 sources-identity 在处理端点里直接执行,这里跳过): + * - 选项带 capabilityId → 经注册中心调用(仍走授权/审计/账本); + * - retry → 重试关联任务;reevaluate / accept → 按对象重发 evaluateAcquisition;dismiss / ignore → 无动作。 + */ +export async function executeResolution(ctx: ServerContext, decisionId: string) { + const d = await ctx.db.selectFrom("decision").selectAll().where("id", "=", decisionId).executeTakeFirst(); + if (!d) return { executed: false, reason: "not_found" }; + if (d.status !== "resolved") return { executed: false, reason: `status_${d.status}` }; + if (d.kind === "identity_ambiguous") return { executed: false, reason: "handled_by_sources_identity" }; + const res = (d.resolution ?? {}) as { optionKey?: string | null; by?: string; execution?: unknown }; + if (res.execution) return { executed: false, reason: "already_executed" }; + const key = res.optionKey ?? null; + const opt = ((d.options ?? []) as DecisionOption[]).find((o) => o.key === key); + const by = d.resolved_by ?? res.by ?? "system"; + let execution: Record; + try { + if (opt?.capabilityId) { + const output = await ctx.capabilities.invoke(opt.capabilityId, opt.input ?? {}, { + actorId: by.replace(/^admin:/, "user:"), + taskId: `decision:${d.id}`, + scopeIds: [], + signal: new AbortController().signal, + // 人工在待处理中心选中的处理 = 控制台人工授权;系统/Agent 关闭的事项仍需授权记录 + source: /^(admin|user):/.test(by) ? "web" : "task", + invocationId: `decision:${d.id}`, + }); + execution = { kind: "capability", capabilityId: opt.capabilityId, output }; + } else if (key === "retry" && d.task_id) { + const row = await retryTask(ctx, d.task_id, by, `decision ${d.id}`); + execution = { kind: "retry", taskId: row.id, status: row.status }; + } else if ((key === "reevaluate" || key === "accept") && d.subject_id) { + const input = + d.subject_type === "acquisition_slot" + ? { acquisitionSlotId: d.subject_id } + : d.subject_type === "source_release" + ? { sourceReleaseId: d.subject_id } + : d.subject_type === "acquisition_plan" + ? { + acquisitionSlotId: ( + await ctx.db + .selectFrom("acquisition_plan") + .select("slot_id") + .where("id", "=", d.subject_id) + .executeTakeFirst() + )?.slot_id, + } + : null; + if (!input || Object.values(input).some((v) => !v)) execution = { kind: "none", reason: "subject_not_evaluable" }; + else { + const { runId } = await ctx.engine.enqueue("evaluateAcquisition", input, { + idempotencyKey: `decision:${d.id}:reevaluate`, + traceId: d.id, + }); + execution = { kind: "reevaluate", engineRunId: runId, input }; + } + } else { + execution = { kind: "none", reason: key ? `option_${key}` : "no_option" }; + } + } catch (err) { + execution = { kind: "error", message: (err as Error).message.slice(0, 500) }; + } + await ctx.db + .updateTable("decision") + .set({ resolution: sql`coalesce(resolution, '{}'::jsonb) || ${JSON.stringify({ execution })}::jsonb` }) + .where("id", "=", d.id) + .execute(); + return { executed: execution.kind !== "none" && execution.kind !== "error", execution }; +} + +export const agentOutboxRoutes: Record = { + "decision.created": (ev) => { + const p = ev.payload as EventPayload<"decision.created">; + if (!["identity_ambiguous", "download_failed", "download_stalled"].includes(p.kind)) return null; + return { task: "resolveDecision", input: { decisionId: p.decisionId, trigger: "created" } }; + }, + "decision.updated": (ev) => { + const p = ev.payload as EventPayload<"decision.updated">; + return p.status === "resolved" + ? { task: "resolveDecision", input: { decisionId: p.decisionId, trigger: "updated" } } + : null; + }, +}; diff --git a/apps/server/src/modules/auth-settings/settings-routes.ts b/apps/server/src/modules/auth-settings/settings-routes.ts index 3ebbca85..ae99abeb 100644 --- a/apps/server/src/modules/auth-settings/settings-routes.ts +++ b/apps/server/src/modules/auth-settings/settings-routes.ts @@ -29,6 +29,15 @@ export function registerSettingsRoutes(app: ModuleApp, ctx: ServerContext) { merged.pi.apiKeySecretRef = null; dropRef = oldRef; } + // TMDB 密钥同样只写:存 secret(metadata_api_key),设置里只留 tmdbApiKeySecretRef + const oldTmdbRef = current.metadata.tmdbApiKeySecretRef; + const tmdbKey = patch.metadata?.tmdbApiKey; + let dropTmdbRef: string | null = null; + let newTmdbRef: string | null = null; + if (tmdbKey === null) { + merged.metadata.tmdbApiKeySecretRef = null; + dropTmdbRef = oldTmdbRef; + } const parsed = GlobalSettings.safeParse(merged); if (!parsed.success) { const msg = parsed.error.issues.map((i) => `${i.path.join(".")}: ${i.message}`).join("; "); @@ -44,6 +53,15 @@ export function registerSettingsRoutes(app: ModuleApp, ctx: ServerContext) { next.pi.apiKeySecretRef = newRef; } } + if (typeof tmdbKey === "string") { + if (oldTmdbRef) { + await ctx.secrets.replace(oldTmdbRef, tmdbKey); + next.metadata.tmdbApiKeySecretRef = oldTmdbRef; + } else { + newTmdbRef = await ctx.secrets.create({ purpose: "metadata_api_key", plaintext: tmdbKey, label: "tmdb" }); + next.metadata.tmdbApiKeySecretRef = newTmdbRef; + } + } const keys = (Object.keys(patch) as SettingsKey[]).filter((k) => patch[k] !== undefined); try { await ctx.db.transaction().execute(async (trx) => { @@ -59,14 +77,18 @@ export function registerSettingsRoutes(app: ModuleApp, ctx: ServerContext) { }); } catch (err) { if (newRef) await ctx.secrets.remove(newRef).catch(() => {}); + if (newTmdbRef) await ctx.secrets.remove(newTmdbRef).catch(() => {}); throw err; } if (dropRef) await ctx.secrets.remove(dropRef).catch(() => {}); + if (dropTmdbRef) await ctx.secrets.remove(dropTmdbRef).catch(() => {}); if (isCached(ctx.settings)) ctx.settings.invalidate(); void ctx.events.publish(SETTINGS_CHANGED_EVENT, { keys }); // 审计里不留明文密钥 const logged: Record = { ...patch }; if (patch.pi && "apiKey" in patch.pi) logged.pi = { ...patch.pi, apiKey: apiKey === null ? "(cleared)" : "(set)" }; + if (patch.metadata && "tmdbApiKey" in patch.metadata) + logged.metadata = { ...patch.metadata, tmdbApiKey: tmdbKey === null ? "(cleared)" : "(set)" }; await audit(ctx, actor, "settings.update", { type: "settings", id: keys.join(",") }, { patch: logged }); return next; }); diff --git a/apps/server/src/modules/sources-identity/capabilities.ts b/apps/server/src/modules/sources-identity/capabilities.ts index 718aba9e..f27fdf53 100644 --- a/apps/server/src/modules/sources-identity/capabilities.ts +++ b/apps/server/src/modules/sources-identity/capabilities.ts @@ -52,7 +52,7 @@ export function registerSourcesCapabilities(registry: CapabilityRegistry, rt: () title: string; year: number | null; }[] = []; - const tmdb = rt().tmdb(); + const tmdb = await rt().tmdb(); if (input.includeRemote && input.query && tmdb) { for (const m of (await tmdb.searchAll(input.query, input.year ? { year: input.year } : {})).slice( 0, diff --git a/apps/server/src/modules/sources-identity/runtime.ts b/apps/server/src/modules/sources-identity/runtime.ts index 2525e16e..56fa323b 100644 --- a/apps/server/src/modules/sources-identity/runtime.ts +++ b/apps/server/src/modules/sources-identity/runtime.ts @@ -40,6 +40,7 @@ export class SourcesRuntime { readonly rss: RssFetcher; private readonly adapters = new Map(); private tmdbClient: TmdbClient | null | undefined; + private tmdbSig: string | null = null; constructor( readonly ctx: ServerContext, @@ -53,17 +54,41 @@ export class SourcesRuntime { }); } - /** TMDB:优先注入;否则读环境变量 TMDB_API_KEY / TMDB_ACCESS_TOKEN(W2-A 设置页接管前的过渡) */ - tmdb(): TmdbClient | null { - if (this.tmdbClient !== undefined) return this.tmdbClient; + /** + * TMDB 客户端:优先注入(测试);否则读设置 metadata.tmdbApiKeySecretRef(secret 现场解密)与 tmdbBaseUrl, + * 未配置时回退环境变量 TMDB_API_KEY / TMDB_ACCESS_TOKEN / TMDB_BASE_URL。设置变化(引用/地址)时重建。 + */ + async tmdb(): Promise { if (this.overrides.tmdb) { - this.tmdbClient = this.overrides.tmdb(); + if (this.tmdbClient === undefined) this.tmdbClient = this.overrides.tmdb(); return this.tmdbClient; } + const meta = await this.ctx.settings.get("metadata").catch(() => null); const env = (k: string) => Deno.env.get(k); - const apiKey = env("TMDB_API_KEY"); - const accessToken = env("TMDB_ACCESS_TOKEN"); - const baseUrl = env("TMDB_BASE_URL"); + const ref = meta?.tmdbApiKeySecretRef ?? null; + const baseUrl = meta?.tmdbBaseUrl ?? env("TMDB_BASE_URL"); + const sig = JSON.stringify([ + ref, + baseUrl ?? null, + ref ? null : [!!env("TMDB_API_KEY"), !!env("TMDB_ACCESS_TOKEN")], + ]); + if (this.tmdbClient !== undefined && this.tmdbSig === sig) return this.tmdbClient; + let apiKey: string | undefined; + let accessToken: string | undefined; + if (ref) { + try { + const secret = (await this.ctx.secrets.resolve(ref)).trim(); + // v3 API Key 是 32 位十六进制;其余(v4 read access token,JWT)走 Bearer + if (/^[0-9a-f]{32}$/i.test(secret)) apiKey = secret; + else accessToken = secret; + } catch (err) { + this.ctx.logger.warn({ err: { message: (err as Error).message } }, "TMDB 密钥解密失败,按未配置处理"); + } + } else { + apiKey = env("TMDB_API_KEY"); + accessToken = env("TMDB_ACCESS_TOKEN"); + } + this.tmdbSig = sig; this.tmdbClient = apiKey || accessToken ? createTmdbClient({ diff --git a/apps/server/src/modules/sources-identity/service/identity-admin.ts b/apps/server/src/modules/sources-identity/service/identity-admin.ts index 99f53215..9023cea2 100644 --- a/apps/server/src/modules/sources-identity/service/identity-admin.ts +++ b/apps/server/src/modules/sources-identity/service/identity-admin.ts @@ -35,7 +35,7 @@ async function entityForExternal(rt: SourcesRuntime, exec: Exec, ref: ExternalRe if (hit) return hit; if (ref.provider !== "tmdb") throw new ApiException("bad_request", `未登记的外部身份 ${ref.provider}:${ref.externalId},只支持按 TMDB 建档`); - const tmdb = rt.tmdb(); + const tmdb = await rt.tmdb(); if (!tmdb) throw new ApiException("precondition_failed", "未配置 TMDB,无法按外部身份建档"); let detail: Awaited>; try { diff --git a/apps/server/src/modules/sources-identity/service/metadata.ts b/apps/server/src/modules/sources-identity/service/metadata.ts index 66d4be09..763e1cc5 100644 --- a/apps/server/src/modules/sources-identity/service/metadata.ts +++ b/apps/server/src/modules/sources-identity/service/metadata.ts @@ -47,7 +47,7 @@ export async function refreshMetadata(rt: SourcesRuntime, mediaEntityId: string) .where("provider", "=", "tmdb") .where("status", "=", "accepted") .executeTakeFirst(); - const tmdb = rt.tmdb(); + const tmdb = await rt.tmdb(); if (!ident || !tmdb || (ident.entity_type !== "movie" && ident.entity_type !== "tv")) { return { status: "skipped", diff --git a/apps/server/src/modules/sources-identity/service/resolve.ts b/apps/server/src/modules/sources-identity/service/resolve.ts index c4ea9902..0af3e7ac 100644 --- a/apps/server/src/modules/sources-identity/service/resolve.ts +++ b/apps/server/src/modules/sources-identity/service/resolve.ts @@ -156,7 +156,7 @@ async function localCandidates( } async function tmdbCandidates(rt: SourcesRuntime, parsed: ParsedRelease): Promise { - const tmdb = rt.tmdb(); + const tmdb = await rt.tmdb(); if (!tmdb) return []; const queries = [parsed.titleCn, parsed.titleEn, parsed.title].filter((t): t is string => !!t); const seen = new Map(); @@ -282,7 +282,7 @@ export async function resolveRelease(rt: SourcesRuntime, input: ResolveInput): P }); if (!existing) { try { - detailed = await rt.tmdb()!.details(remoteBest.entityType, remoteBest.externalId); + detailed = await (await rt.tmdb())!.details(remoteBest.entityType, remoteBest.externalId); } catch (e) { if (!(e instanceof TmdbError)) throw e; detailed = remoteBest; diff --git a/apps/server/test/api-registry.test.ts b/apps/server/test/api-registry.test.ts new file mode 100644 index 00000000..0eb3b51b --- /dev/null +++ b/apps/server/test/api-registry.test.ts @@ -0,0 +1,38 @@ +import { API_PREFIX, listEndpoints } from "@umt/contracts"; +import { describe, expect, it } from "vitest"; +import { createApp } from "../src/http/app.ts"; +import { EventBus } from "../src/http/sse.ts"; +import { SERVER_MODULES } from "../src/modules/index.ts"; +import type { ServerContext } from "../src/modules/types.ts"; +import { DEV_KEY, json, memoryLogger } from "./helpers.ts"; + +// OpenAPI 里出现的每条 /api/v1 路由都必须登记在 @umt/contracts 的 Api 注册表(web 与测试都从那里取路径/schema)。 + +describe("Api 注册表覆盖", () => { + it("OpenAPI 文档中的每条路由都在 listEndpoints() 里", async () => { + const app = createApp({ + config: { NODE_ENV: "development", DEV_ADMIN_KEY: DEV_KEY }, + logger: memoryLogger().logger, + bus: new EventBus(5), + role: "all", + version: "0.0.0-test", + modules: SERVER_MODULES, + ctx: { role: "all" } as unknown as ServerContext, + }); + const res = await app.request("/api/openapi.json"); + expect(res.status).toBe(200); + const doc = (await json(res)) as { paths: Record> }; + + const served: string[] = []; + for (const [path, ops] of Object.entries(doc.paths)) { + if (!path.startsWith(`${API_PREFIX}/`)) continue; // /api/openapi.json 等非业务路由 + const rel = path.slice(API_PREFIX.length); + for (const method of Object.keys(ops)) served.push(`${method.toUpperCase()} ${rel}`); + } + expect(served.length).toBeGreaterThan(0); + + const registered = new Set(listEndpoints().map((e) => `${e.method.toUpperCase()} ${e.path}`)); + const missing = served.filter((k) => !registered.has(k)).sort(); + expect(missing, `以下路由未登记到 packages/contracts/src/api.ts:\n${missing.join("\n")}`).toEqual([]); + }); +}); diff --git a/apps/server/test/e2e/e1-rss-to-library.test.ts b/apps/server/test/e2e/e1-rss-to-library.test.ts new file mode 100644 index 00000000..c48124d6 --- /dev/null +++ b/apps/server/test/e2e/e1-rss-to-library.test.ts @@ -0,0 +1,158 @@ +import { createMockRssServer, type MockRssServer } from "@umt/testkit"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { completeDownload, createE2E, type E2E, pgAvailable, seedBase, waitFor } from "./harness.ts"; + +// E1 RSS → 入库(全部经真实任务引擎 + outbox 中继): +// mock RSS(观众风格条目)→ ingestFeed → release.observed → resolveObservation(mock TMDB,密钥来自设置) +// → release.resolved → evaluateAcquisition(已发布策略:新作品 1080p Slot,发布即生成持续授权)→ 计划 + 提交 mock qB +// → 完成(落盘)→ download.status_changed(completed)→ verifyAndImport(Copy)→ /organization-records/changes 文件级记录 +// → 媒体库详情 Slot=satisfied。 + +const hasPg = await pgAvailable(); +const BASE = "https://audiences.example"; +const TITLE = "Dune.Part.Two.2024.1080p.BluRay.x264-GRP"; + +describe.skipIf(!hasPg)("E1 RSS → 入库", () => { + let e: E2E; + let rss: MockRssServer; + let base: Awaited>; + + beforeAll(async () => { + e = await createE2E("e1"); + base = await seedBase(e, { baseUrl: `${BASE}/` }); + rss = await createMockRssServer({ + passkey: "E2EPASSKEY", + items: [ + { + guid: "aud-812345", + title: TITLE, + link: `${BASE}/details.php?id=812345&hit=1`, + enclosureUrl: `${BASE}/download.php?id=812345&passkey=E2EPASSKEY`, + size: 300_000, + pubDate: new Date(), + categories: ["电影"], + }, + { + guid: "aud-812346", + title: "Dune.Part.Two.2024.720p.WEB-DL.x264-AAA", + link: `${BASE}/details.php?id=812346&hit=1`, + enclosureUrl: `${BASE}/download.php?id=812346&passkey=E2EPASSKEY`, + size: 300_000, + pubDate: new Date(), + categories: ["电影"], + }, + ], + }); + }, 120_000); + + afterAll(async () => { + await rss?.close(); + await e?.close(); + }, 60_000); + + it("发布 1080p 收录策略 → 持续授权", async () => { + const created = await e.api("POST", "/policy-revisions", { + policyKey: "default", + kind: "acquisition", + title: "新片 1080p", + scope: { mediaKinds: ["movie"] }, + rules: { + newTitle: { + enabled: true, + windowDays: 3650, + mediaKinds: ["movie"], + slots: [{ name: "1080p", hardConditions: { resolutions: ["1080p"] } }], + }, + }, + }); + expect(created.status).toBe(201); + const pub = await e.api("POST", `/policy-revisions/${created.body.data.id}/publish`, { + expectedRevision: created.body.data.revision, + }); + expect(pub.status).toBe(200); + expect(pub.body.data.status).toBe("published"); + const grants = await e.api("GET", "/automation-grants"); + expect(grants.body.data.items.map((g: { action: string }) => g.action)).toContain("policy.acquisition"); + }); + + it("RSS 抓取 → 识别 → 计划 → 提交 → 完成 → Copy 入库 → 整理记录增量可读 → Slot satisfied", async () => { + const urlRef = await e.booted.ctx.secrets.create({ purpose: "feed_url", plaintext: rss.feedUrl }); + const feed = await e.db + .insertInto("feed_source") + .values({ + site_id: base.siteId, + name: "观众-电影", + purpose: "movie", + url_secret_ref: urlRef, + url_display: rss.feedUrl.replace(/passkey=[^&]+/, "passkey=[REDACTED]"), + poll_interval_sec: 3600, + }) + .returning("id") + .executeTakeFirstOrThrow(); + const fetched = await e.api("POST", `/feed-sources/${feed.id}/fetch`); + expect(fetched.status).toBe(200); + + // 识别:TMDB 客户端来自设置(secret_ref),建本地作品 + const rel = await waitFor( + async () => + e.db + .selectFrom("source_release") + .selectAll() + .where("title", "=", TITLE) + .where("resolution_status", "=", "resolved") + .executeTakeFirst(), + 60_000, + "1080p 资源识别完成", + ); + const entity = await e.db + .selectFrom("media_entity") + .selectAll() + .where("id", "=", rel.media_entity_id!) + .executeTakeFirstOrThrow(); + expect(entity).toMatchObject({ kind: "movie", year: 2024 }); + + // 计划与提交:只有 1080p 被选中 + await completeDownload(e, TITLE); + expect(e.qb.callsTo("torrents/add")).toHaveLength(1); + + const rec = await waitFor( + async () => + e.db + .selectFrom("organization_record") + .selectAll() + .where("media_entity_id", "=", entity.id) + .where("status", "=", "succeeded") + .executeTakeFirst(), + 90_000, + "整理记录 succeeded", + ); + expect(rec).toMatchObject({ action: "copy", trigger: "auto" }); + + // 公开整理记录增量 API:文件级结果(只返回早于所有活跃事务的变更,共享库上其它测试的长事务会让它稍晚可见) + const item = await waitFor( + async () => { + const changes = await e.api("GET", "/organization-records/changes?after=0&includeFiles=true"); + expect(changes.status).toBe(200); + return changes.body.data.items.find( + (x: { id: string; status: string }) => x.id === rec.id && x.status === "succeeded", + ); + }, + 60_000, + "整理记录出现在 changes 增量里", + 500, + ); + expect(item.files[0]).toMatchObject({ role: "main", status: "succeeded" }); + expect(item.files[0].target.path).toMatch(/1080p\.mkv$/); + + // 媒体库详情:Slot satisfied,资产登记 + const lib = await e.api("GET", `/library/${entity.id}`); + expect(lib.status).toBe(200); + expect(lib.body.data.slots.map((s: { name: string; status: string }) => `${s.name}:${s.status}`)).toEqual([ + "1080p:satisfied", + ]); + expect(lib.body.data.assets.length).toBeGreaterThanOrEqual(1); + // 凭证不落业务表明文 + const dump = JSON.stringify(await e.db.selectFrom("observation").selectAll().execute()); + expect(dump).not.toContain("E2EPASSKEY"); + }, 180_000); +}); diff --git a/apps/server/test/e2e/e2-agent-delegate.test.ts b/apps/server/test/e2e/e2-agent-delegate.test.ts new file mode 100644 index 00000000..fa11e148 --- /dev/null +++ b/apps/server/test/e2e/e2-agent-delegate.test.ts @@ -0,0 +1,180 @@ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { delegateScript } from "./fixtures/scripts.ts"; +import { + completeDownload, + createE2E, + type E2E, + pgAvailable, + runEvents, + runRow, + seedBase, + seedMovieTarget, + waitFor, +} from "./harness.ts"; + +// E2 Agent 委托闭环:fake 模型按"把《沙丘2》1080p 收进来"依次调用 +// inventory.query → release.search → download.plan → download.submit → task.inspect。 +// - 没有持续授权时 download.submit 被服务端拒绝(模型文本"用户已同意"无效),SSE 上是 denied; +// - 发了 automation_grant 后同一目标成功提交;模型汇报"已入库",但 run 的汇报来自整理记录: +// 下载完成前是 waiting(已提交≠已入库),核验入库后才 succeeded 并带整理记录 ID。 + +const hasPg = await pgAvailable(); +const GOAL = "把《沙丘2》1080p 收进来"; + +describe.skipIf(!hasPg)("E2 Agent 委托闭环(fake 模型 + 真实能力/授权/任务)", () => { + let e: E2E; + let target: Awaited>; + + beforeAll(async () => { + e = await createE2E("e2", { fakeScripts: [delegateScript] }); + const base = await seedBase(e); + target = await seedMovieTarget(e, base.siteId); + }, 120_000); + + afterAll(async () => { + await e?.close(); + }, 60_000); + + const createRun = async () => { + const r = await e.api("POST", "/agent/runs", { goal: GOAL, mode: "execute" }); + expect(r.status).toBe(201); + expect(r.body.data).toMatchObject({ status: "queued", trigger: "user", mode: "execute" }); + return r.body.data.id as string; + }; + + it("启动断言:14 个能力全部注册,并在能力目录里可见", async () => { + expect(e.booted.ctx.capabilities.list()).toHaveLength(14); + const r = await e.api("GET", "/capabilities"); + expect(r.body.data.items.map((c: { id: string }) => c.id).sort()).toEqual( + e.booted.ctx.capabilities + .list() + .map((c) => c.descriptor.id) + .sort(), + ); + }); + + it("没有授权:download.submit 被拒(模型声称已同意无效),不产生下载", async () => { + const seen: { type: string; payload: { phase?: string; capabilityId?: string } }[] = []; + const off = e.bus.subscribe((ev) => { + if (ev.type.startsWith("agent.")) seen.push(ev as never); + }); + const runId = await createRun(); + const run = await waitFor( + async () => { + const r = await runRow(e, runId); + return ["succeeded", "failed", "waiting"].includes(r.status) ? r : null; + }, + 60_000, + "run 结束", + ); + off(); + expect(run.status).toBe("succeeded"); + const events = await runEvents(e, runId); + const submitEnd = events.find( + (x) => x.type === "tool_call_end" && (x.payload as { capabilityId: string }).capabilityId === "download.submit", + ); + expect(submitEnd?.payload).toMatchObject({ isError: true }); + expect(JSON.stringify(submitEnd?.payload)).toContain("unauthorized"); + // SSE:未授权的调用标为 denied + expect( + seen.some( + (x) => + x.type === "agent.tool_call" && x.payload.capabilityId === "download.submit" && x.payload.phase === "denied", + ), + ).toBe(true); + // 审计:capability.denied + const denied = await e.db + .selectFrom("audit_event") + .selectAll() + .where("action", "=", "capability.denied") + .where("target_id", "=", "download.submit") + .execute(); + expect(denied.length).toBeGreaterThanOrEqual(1); + expect(await e.db.selectFrom("download_task").select("id").execute()).toHaveLength(0); + // 草案计划存在,但没有预留 Slot + const plans = await e.db.selectFrom("acquisition_plan").selectAll().execute(); + expect(plans.map((p) => p.status)).toEqual(["draft"]); + const slot = await e.db + .selectFrom("acquisition_slot") + .select("status") + .where("id", "=", target.slotId) + .executeTakeFirstOrThrow(); + expect(slot.status).toBe("empty"); + // 汇报不采信模型文本:没有可核查的业务变更 + expect(run.summary).toContain("没有提交下载"); + expect(run.summary).toContain("模型汇报(未经核查)"); + }, 90_000); + + it("有持续授权:提交成功 → 已提交≠已入库(waiting)→ 下载完成、核验入库后 succeeded,汇报来自整理记录", async () => { + const g = await e.api("POST", "/automation-grants", { + action: "download.submit", + scope: { mediaEntityIds: [target.entityId] }, + reason: "e2e 授权", + }); + expect(g.status).toBe(201); + + const runId = await createRun(); + // 模型已经说完"已入库",但业务上只是已提交:run 进入 waiting + const waiting = await waitFor( + async () => { + const r = await runRow(e, runId); + return r.status === "waiting" ? r : null; + }, + 60_000, + "run waiting", + ); + expect(waiting.summary).toContain("已提交≠已入库"); + const events = await runEvents(e, runId); + const completed = events.find((x) => x.type === "run_completed"); + expect((completed!.payload as { finalText: string }).finalText).toContain("已入库"); + const calls = events + .filter((x) => x.type === "tool_call_start") + .map((x) => (x.payload as { capabilityId: string }).capabilityId); + expect(calls).toEqual(["inventory.query", "release.search", "download.plan", "download.submit", "task.inspect"]); + const submitEnd = events.find( + (x) => x.type === "tool_call_end" && (x.payload as { capabilityId: string }).capabilityId === "download.submit", + ); + expect(submitEnd?.payload).toMatchObject({ isError: false }); + // 账本:download.submit 记在本 run 名下,带授权 ID + const inv = await e.db + .selectFrom("capability_invocation") + .selectAll() + .where("agent_run_id", "=", runId) + .where("capability_id", "=", "download.submit") + .executeTakeFirstOrThrow(); + expect(inv).toMatchObject({ status: "succeeded", grant_id: g.body.data.id }); + // 草案被转为预留并提交 + await completeDownload(e, "Dune.Part.Two.2024.1080p.BluRay.x264-GRP"); + + const done = await waitFor( + async () => { + const r = await runRow(e, runId); + return r.status === "succeeded" ? r : null; + }, + 90_000, + "run succeeded", + ); + const rec = await e.db + .selectFrom("organization_record") + .selectAll() + .where("status", "=", "succeeded") + .executeTakeFirstOrThrow(); + expect(done.summary).toContain("已入库 1 个"); + expect(done.summary).toContain(rec.id); + const detail = await e.api("GET", `/agent/runs/${runId}?afterSeq=-1&limit=500`); + expect(detail.body.data.report).toMatchObject({ state: "completed", imported: 1, submitted: 1 }); + expect(detail.body.data.report.organization[0]).toMatchObject({ recordId: rec.id, status: "succeeded" }); + // 断线补读:afterSeq 之后的事件 + const tail = await e.api("GET", `/agent/runs/${runId}?afterSeq=3&limit=2`); + expect(tail.body.data.events.map((x: { seq: number }) => x.seq)).toEqual([4, 5]); + expect(tail.body.data.nextAfterSeq).toBe(5); + // 入库成功:Slot 满足 + const slot = await e.db + .selectFrom("acquisition_slot") + .select("status") + .where("id", "=", target.slotId) + .executeTakeFirstOrThrow(); + expect(slot.status).toBe("satisfied"); + expect(e.qb.callsTo("torrents/add")).toHaveLength(1); + }, 180_000); +}); diff --git a/apps/server/test/e2e/e3-background-continues.test.ts b/apps/server/test/e2e/e3-background-continues.test.ts new file mode 100644 index 00000000..ab951a0a --- /dev/null +++ b/apps/server/test/e2e/e3-background-continues.test.ts @@ -0,0 +1,76 @@ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { delegateScript } from "./fixtures/scripts.ts"; +import { + completeDownload, + createE2E, + type E2E, + pgAvailable, + runRow, + seedBase, + seedMovieTarget, + waitFor, +} from "./harness.ts"; + +// E3 关闭聊天不影响后台:只发一次 POST /agent/runs(HTTP 请求里不跑模型,立即返回 queued),之后不再有任何 API 请求 +// 也不订阅 SSE;持久任务独立推进:模型会话 → 提交 → (外部下载器完成)→ 核验入库 → run succeeded。 + +const hasPg = await pgAvailable(); + +describe.skipIf(!hasPg)("E3 关闭聊天不影响后台", () => { + let e: E2E; + let target: Awaited>; + + beforeAll(async () => { + e = await createE2E("e3", { fakeScripts: [delegateScript] }); + const base = await seedBase(e); + target = await seedMovieTarget(e, base.siteId); + await e.db + .insertInto("automation_grant") + .values({ action: "download.submit", scope: JSON.stringify({}), granted_by: "user:e2e" }) + .execute(); + }, 120_000); + + afterAll(async () => { + await e?.close(); + }, 60_000); + + it("创建后不再请求,任务系统独立推进到 completed", async () => { + const t0 = Date.now(); + const r = await e.api("POST", "/agent/runs", { goal: "把《沙丘2》1080p 收进来", mode: "execute" }); + // HTTP 不驱动模型:立即返回 queued(模型会话在 worker 的持久任务里跑) + expect(r.status).toBe(201); + expect(r.body.data.status).toBe("queued"); + expect(Date.now() - t0).toBeLessThan(5_000); + const runId = r.body.data.id as string; + const task = await e.db + .selectFrom("task") + .selectAll() + .where("id", "=", r.body.data.taskId) + .executeTakeFirstOrThrow(); + expect(task).toMatchObject({ kind: "agentRun", subject_type: "agent_run", subject_id: runId }); + expect(task.engine_run_id).toBeTruthy(); + + // "聊天已关闭":从这里开始只有外部世界(下载器)在变化 + await completeDownload(e, "Dune.Part.Two.2024.1080p.BluRay.x264-GRP"); + const run = await waitFor( + async () => { + const x = await runRow(e, runId); + return x.status === "succeeded" ? x : null; + }, + 120_000, + "run succeeded", + ); + expect(run.finished_at).not.toBeNull(); + expect(run.summary).toContain("已入库 1 个"); + const engineRun = await e.booted.ctx.engine.get(task.engine_run_id!); + expect(engineRun?.status).toBe("completed"); + const t = await e.db.selectFrom("task").select("status").where("id", "=", task.id).executeTakeFirstOrThrow(); + expect(t.status).toBe("succeeded"); + const slot = await e.db + .selectFrom("acquisition_slot") + .select("status") + .where("id", "=", target.slotId) + .executeTakeFirstOrThrow(); + expect(slot.status).toBe("satisfied"); + }, 180_000); +}); diff --git a/apps/server/test/e2e/e4-crash-recovery.test.ts b/apps/server/test/e2e/e4-crash-recovery.test.ts new file mode 100644 index 00000000..6111aae0 --- /dev/null +++ b/apps/server/test/e2e/e4-crash-recovery.test.ts @@ -0,0 +1,142 @@ +import { type ChildProcess, spawn } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { torrentFor } from "./fixtures/modules.ts"; +import { crashScript } from "./fixtures/scripts.ts"; +import { + createE2E, + type E2E, + PG_URL, + pgAvailable, + runEvents, + runRow, + seedBase, + seedMovieTarget, + waitFor, +} from "./harness.ts"; + +// E4 崩溃恢复:worker 子进程在 Agent 调用 download.submit 时被 kill -9;新 worker 在租约到期(约 30s)后接手: +// runAgent 的 drive 步骤重跑 → 打开同一 JSONL 会话(pendingOperations>0)→ 按幂等键对账账本里停在 running 的调用 +// → resume(模型拿到"结果未知")→ 同参数再调 download.submit 得到账本结果;mock qB 只收到一次 add。 + +const hasPg = await pgAvailable(); +const WORKER = fileURLToPath(new URL("./fixtures/worker.ts", import.meta.url)); +const SERVER_DIR = fileURLToPath(new URL("../..", import.meta.url)); +const TITLE = "Dune.Part.Two.2024.1080p.BluRay.x264-GRP"; + +describe.skipIf(!hasPg)("E4 崩溃恢复(worker 子进程 kill -9)", () => { + let e: E2E; + const children: ChildProcess[] = []; + + const spawnWorker = (): Promise => { + const child = spawn(process.execPath, ["run", "-A", WORKER], { + cwd: SERVER_DIR, + env: { + ...process.env, + DATABASE_URL: PG_URL, + BIZ_SCHEMA: e.biz, + OW_SCHEMA: e.ow, + SESSION_DIR: e.sessionDir, + }, + stdio: ["ignore", "pipe", "pipe"], + }); + children.push(child); + let err = ""; + child.stderr!.on("data", (d) => { + err += d; + }); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error(`worker 启动超时: ${err}`)), 60_000); + child.stdout!.on("data", (d) => { + if (String(d).includes("worker ready")) { + clearTimeout(timer); + resolve(child); + } + }); + child.on("exit", (code) => { + clearTimeout(timer); + reject(new Error(`worker 提前退出 code=${code}: ${err}`)); + }); + }); + }; + + beforeAll(async () => { + e = await createE2E("e4", { worker: false, fakeScripts: [crashScript] }); + const base = await seedBase(e); + await seedMovieTarget(e, base.siteId); + await e.db + .insertInto("automation_grant") + .values({ action: "download.submit", scope: JSON.stringify({}), granted_by: "user:e2e" }) + .execute(); + }, 120_000); + + afterAll(async () => { + for (const c of children) if (c.exitCode === null && c.signalCode === null) c.kill("SIGKILL"); + await e?.close(); + }, 60_000); + + it("download.submit 途中 kill worker → 新 worker resume,不重复提交,最终入库", async () => { + const w1 = await spawnWorker(); + const r = await e.api("POST", "/agent/runs", { goal: "把《沙丘2》1080p 收进来", mode: "execute" }); + expect(r.status).toBe(201); + const runId = r.body.data.id as string; + + // Agent 调到 download.submit(账本行出现)的瞬间 kill -9 + await waitFor( + async () => + e.db + .selectFrom("capability_invocation") + .select("id") + .where("agent_run_id", "=", runId) + .where("capability_id", "=", "download.submit") + .executeTakeFirst(), + 60_000, + "download.submit 开始", + 10, + ); + w1.kill("SIGKILL"); + await new Promise((res) => w1.once("exit", res)); + const before = await runRow(e, runId); + expect(["running", "queued"]).toContain(before.status); + + await spawnWorker(); + // 新 worker 接手(租约约 30s 到期)后提交进入 qB,外部下载器完成 + const hash = torrentFor(TITLE).hash; + await waitFor(async () => e.qb.get(hash), 120_000, "qB 收到种子", 500); + await e.qb.complete(hash, { writeFiles: true }); + const done = await waitFor( + async () => { + const x = await runRow(e, runId); + return x.status === "succeeded" ? x : null; + }, + 120_000, + "run succeeded", + 500, + ); + expect(done.summary).toContain("已入库 1 个"); + + // 不重复提交:qB 只收到一次 add;download.submit 账本只有一条(同幂等键) + expect(e.qb.callsTo("torrents/add")).toHaveLength(1); + const inv = await e.db + .selectFrom("capability_invocation") + .selectAll() + .where("agent_run_id", "=", runId) + .where("capability_id", "=", "download.submit") + .execute(); + expect(inv).toHaveLength(1); + expect(inv[0]!.status).toBe("succeeded"); + expect(await e.db.selectFrom("download_task").select("id").execute()).toHaveLength(1); + // 事件:续跑标记 + 恢复的工具调用 + const events = await runEvents(e, runId); + expect(events.some((x) => x.type === "run_started" && (x.payload as { resumed: boolean }).resumed)).toBe(true); + const recovered = events.filter((x) => x.type === "tool_call_end" && (x.payload as { recovery: boolean }).recovery); + expect(recovered.map((x) => (x.payload as { capabilityId: string }).capabilityId)).toContain("download.submit"); + // (run_id, seq) 连续不冲突 + const seqs = events.map((x) => Number(x.seq)); + expect(new Set(seqs).size).toBe(seqs.length); + const starts = events + .filter((x) => x.type === "tool_call_start" && !(x.payload as { recovery: boolean }).recovery) + .map((x) => (x.payload as { capabilityId: string }).capabilityId); + expect(starts.slice(-2)).toEqual(["download.submit", "task.inspect"]); + }, 300_000); +}); diff --git a/apps/server/test/e2e/e5-auto-investigate.test.ts b/apps/server/test/e2e/e5-auto-investigate.test.ts new file mode 100644 index 00000000..bdec6ed1 --- /dev/null +++ b/apps/server/test/e2e/e5-auto-investigate.test.ts @@ -0,0 +1,147 @@ +import { TMDB_FIXTURES, type TmdbFixture } from "@umt/testkit"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { INVESTIGATE_CAPABILITIES } from "../../src/modules/agent/runner.ts"; +import { investigateScript } from "./fixtures/scripts.ts"; +import { createE2E, type E2E, pgAvailable, runEvents, seedBase, waitFor } from "./harness.ts"; + +// E5 后台唤醒:识别歧义产生 decision.created → outbox → resolveDecision → autoInvestigate 开启时入队有界 runAgent +// (plan_only、白名单只读 + identity.resolve、小预算、trigger=event);同一 decision 只一个 run(dedupe_key)。 + +const hasPg = await pgAvailable(); + +const twin = (id: number): TmdbFixture => ({ + type: "movie", + id, + original_title: "Twins", + original_language: "en", + origin_country: ["US"], + date: "1988-12-09", + titles: { "zh-CN": "龙兄鼠弟", "en-US": "Twins" }, + popularity: 10 + id / 1000, + alternative_titles: [], + external_ids: {}, +}); + +describe.skipIf(!hasPg)("E5 后台唤醒有界调查", () => { + let e: E2E; + let siteId: string; + + beforeAll(async () => { + e = await createE2E("e5", { + fakeScripts: [investigateScript], + tmdbFixtures: [...TMDB_FIXTURES, twin(111), twin(112)], + }); + siteId = (await seedBase(e)).siteId; + await e.settings({ agent: { autoInvestigate: true, investigateMaxToolCalls: 6 } }); + }, 120_000); + + afterAll(async () => { + await e?.close(); + }, 60_000); + + it("识别歧义 → 自动发起只读调查 → 结论回写待处理事项;重复事件只一个 run", async () => { + const rel = await e.db + .insertInto("source_release") + .values({ site_id: siteId, site_release_id: "990001", title: "Twins.1988.1080p.BluRay.x264-GRP" }) + .returning("id") + .executeTakeFirstOrThrow(); + await e.booted.ctx.engine.enqueue( + "resolveObservation", + { sourceReleaseId: rel.id }, + { idempotencyKey: `e5:resolve:${rel.id}`, traceId: rel.id }, + ); + const decision = await waitFor( + async () => + e.db + .selectFrom("decision") + .selectAll() + .where("kind", "=", "identity_ambiguous") + .where("subject_id", "=", rel.id) + .executeTakeFirst(), + 60_000, + "识别歧义 decision", + ); + const run = await waitFor( + async () => + e.db + .selectFrom("agent_run") + .selectAll() + .where("dedupe_key", "=", `investigate:${decision.id}`) + .executeTakeFirst(), + 60_000, + "自动调查 run", + ); + expect(run).toMatchObject({ trigger: "event", mode: "plan_only", created_by: "system:auto-investigate" }); + expect(run.goal).toContain(decision.id); + expect(run.scope).toMatchObject({ + capabilities: [...INVESTIGATE_CAPABILITIES], + allowedKinds: ["read", "preview", "propose"], + }); + expect(run.budget).toMatchObject({ maxToolCalls: 6 }); + + const finished = await waitFor( + async () => { + const r = await e.db.selectFrom("agent_run").selectAll().where("id", "=", run.id).executeTakeFirstOrThrow(); + return ["succeeded", "failed", "budget_exceeded"].includes(r.status) ? r : null; + }, + 60_000, + "调查 run 结束", + ); + expect(finished.status).toBe("succeeded"); + const events = await runEvents(e, run.id); + const invoked = events + .filter((x) => x.type === "tool_call_start") + .map((x) => (x.payload as { capabilityId: string }).capabilityId); + expect(invoked).toEqual(["catalog.lookup", "inventory.query"]); + // 白名单外的写能力:被拒绝,没有执行 + const rejected = events.filter((x) => x.type === "tool_call_rejected"); + expect(rejected.map((x) => (x.payload as { toolName: string }).toolName)).toEqual(["download__submit"]); + expect( + await e.db.selectFrom("capability_invocation").select("id").where("agent_run_id", "=", run.id).execute(), + ).toHaveLength(0); + // 结论回写:待处理事项交还人工并带调查摘要 + const d2 = await e.db.selectFrom("decision").selectAll().where("id", "=", decision.id).executeTakeFirstOrThrow(); + expect(d2.status).toBe("awaiting_user"); + expect(d2.agent_run_id).toBe(run.id); + expect((d2.evidence as { agentInvestigation?: { runId: string } }).agentInvestigation?.runId).toBe(run.id); + + // 同一根因的重复事件(不同 outbox 键)与事项重新打开后的再次触发:仍只有一个 run + await e.db.updateTable("decision").set({ status: "open" }).where("id", "=", decision.id).execute(); + for (const k of ["dup-1", "dup-2"]) { + await e.booted.ctx.outbox.emit( + e.db, + "decision.created", + { decisionId: decision.id, kind: "identity_ambiguous", severity: "warning", title: decision.title }, + { idempotencyKey: `e5:${k}` }, + ); + } + await waitFor( + async () => { + const rows = await e.db + .selectFrom("outbox_event") + .select("status") + .where("idempotency_key", "in", ["e5:dup-1", "e5:dup-2"]) + .execute(); + return rows.length === 2 && rows.every((r) => r.status === "dispatched"); + }, + 30_000, + "重复事件已投递", + ); + // 等 resolveDecision 跑完 + await waitFor( + async () => { + const runs = await e.booted.ctx.engine.list({ taskName: "resolveDecision" }); + return runs.length >= 3 && runs.every((r) => r.status === "completed") ? runs : null; + }, + 30_000, + "resolveDecision 完成", + ); + const runs = await e.db + .selectFrom("agent_run") + .select("id") + .where("dedupe_key", "=", `investigate:${decision.id}`) + .execute(); + expect(runs).toHaveLength(1); + expect(await e.db.selectFrom("agent_run").select("id").execute()).toHaveLength(1); + }, 180_000); +}); diff --git a/apps/server/test/e2e/fixtures/modules.ts b/apps/server/test/e2e/fixtures/modules.ts new file mode 100644 index 00000000..ea6ca07e --- /dev/null +++ b/apps/server/test/e2e/fixtures/modules.ts @@ -0,0 +1,68 @@ +import { createHash } from "node:crypto"; +import { createQbClient } from "@umt/adapters"; +import type { FakeScript } from "@umt/agent"; +import { makeTorrent } from "@umt/testkit"; +import { createAcquisitionOrganizeModule } from "../../../src/modules/acquisition-organize/index.ts"; +import { createAgentModule } from "../../../src/modules/agent/index.ts"; +import { authSettingsModule } from "../../../src/modules/auth-settings/index.ts"; +import { createSourcesIdentityModule } from "../../../src/modules/sources-identity/index.ts"; +import type { ServerModule } from "../../../src/modules/types.ts"; + +// e2e 模块装配:真实四个模块,外部依赖指向 testkit mock(qB 账号 admin/adminadmin、固定磁盘用量、按标题生成的种子)。 +// harness(API/worker 同进程)与崩溃恢复用的 worker 子进程共用这一份,保证两边行为一致。 + +export const TORRENT_SIZE = 300_000; + +/** 按发布标题确定性地生成种子(同标题 → 同 infohash,父子进程一致) */ +export function torrentFor(title: string) { + const seed = createHash("sha256").update(title).digest().readUInt32BE(0); + return makeTorrent({ name: `${title}.mkv`, length: TORRENT_SIZE, pieceSeed: seed }); +} + +export interface E2EModuleOptions { + sessionDir: string; + fakeScripts?: FakeScript[]; + /** 下载同步周期(毫秒) */ + syncIntervalMs?: number; +} + +export function buildModules(o: E2EModuleOptions): ServerModule[] { + return [ + authSettingsModule, + // TMDB 不注入:走设置 metadata.tmdbApiKeySecretRef / tmdbBaseUrl(W2 收口 b) + createSourcesIdentityModule({ schedulerIntervalMs: 3_600_000 }), + createAcquisitionOrganizeModule({ + downloader: async (inst) => + createQbClient({ + instanceId: inst.id, + baseUrl: inst.base_url, + username: "admin", + password: "adminadmin", + lookup: { attempts: 5, intervalMs: 50 }, + }), + torrentSource: async ({ release }) => ({ kind: "file", bytes: torrentFor(release.title).bytes }), + diskUsage: async (path) => ({ + status: "ok", + path, + totalBytes: 1e12, + freeBytes: 6e11, + availableBytes: 6e11, + usedBytes: 4e11, + usedRatio: 0.4, + }), + probe: async () => ({ status: "unknown", reason: "ffprobe_unavailable" }), + options: { + backgroundLoops: true, + syncIntervalMs: o.syncIntervalMs ?? 300, + minVideoBytes: 100_000, + verifyHash: "sha256", + reconcileIntervalMs: 0, + }, + }), + createAgentModule({ + sessionDir: o.sessionDir, + ...(o.fakeScripts ? { fakeScripts: o.fakeScripts } : {}), + goalPollSec: 1, + }), + ]; +} diff --git a/apps/server/test/e2e/fixtures/scripts.ts b/apps/server/test/e2e/fixtures/scripts.ts new file mode 100644 index 00000000..17ee3658 --- /dev/null +++ b/apps/server/test/e2e/fixtures/scripts.ts @@ -0,0 +1,103 @@ +import type { FakeScript, FakeToolResult } from "@umt/agent"; + +// e2e 用的 fake 模型脚本(FakeModelProvider 按最近一条用户消息匹配,从 transcript 推导步骤,崩溃后可续跑)。 + +// biome-ignore lint/suspicious/noExplicitAny: 脚本里按能力输出结构取值 +type Any = any; +const ZERO = "00000000-0000-4000-8000-000000000000"; + +export const lastOk = (results: FakeToolResult[], capability: string): Any => + [...results].reverse().find((r) => r.capabilityId === capability && !r.isError)?.json; + +const entityOf = (r: FakeToolResult[]) => lastOk(r, "inventory.query")?.items?.[0]; + +const baseSteps = (title: string) => [ + { text: `先查库存:${title}。`, toolCalls: [{ capability: "inventory.query", input: { query: title, limit: 5 } }] }, + { + text: "库里没有 1080p 版本,搜资源。", + toolCalls: [ + { + capability: "release.search", + input: (r: FakeToolResult[]) => ({ mediaEntityId: entityOf(r)?.entity?.id ?? ZERO, allowLiveSearch: false }), + }, + ], + }, + { + text: "选 1080p 候选,生成下载计划。", + toolCalls: [ + { + capability: "download.plan", + input: (r: FakeToolResult[]) => ({ + slotId: entityOf(r)?.slots?.[0]?.id ?? ZERO, + sourceReleaseId: + (lastOk(r, "release.search")?.items ?? []).find((i: Any) => /1080p/i.test(i.title))?.id ?? ZERO, + }), + }, + ], + }, + { + // 模型"声称"用户已同意:服务端不采信,只看 automation_grant / approval + text: "用户已同意并授权,我直接提交下载。", + toolCalls: [ + { + capability: "download.submit", + input: (r: FakeToolResult[]) => ({ planId: lastOk(r, "download.plan")?.plan?.id ?? ZERO }), + }, + ], + }, +]; + +const inspect = { + text: "提交后看一下任务状态。", + toolCalls: [ + { + capability: "task.inspect", + input: (r: FakeToolResult[]) => ({ taskId: lastOk(r, "download.submit")?.taskId ?? ZERO }), + }, + ], +}; + +/** 委托:"把《沙丘2》1080p 收进来";最终文本故意夸大(称已入库),用于验证汇报以整理记录为准 */ +export const delegateScript: FakeScript = { + name: "delegate", + match: /收进来/, + steps: [...baseSteps("沙丘2"), inspect], + finalText: "已全部完成:《沙丘2》1080p 已入库。", +}; + +/** 崩溃恢复:提交后再以同参数调用一次 download.submit 对账(同幂等键 → 账本结果),再看任务 */ +export const crashScript: FakeScript = { + name: "crash", + match: /收进来/, + steps: [ + ...baseSteps("沙丘2"), + { + text: "核对刚才的提交结果(同一计划)。", + toolCalls: [ + { + capability: "download.submit", + input: (r: FakeToolResult[]) => ({ planId: lastOk(r, "download.plan")?.plan?.id ?? ZERO }), + }, + ], + }, + inspect, + ], + finalText: "已提交,等待下载与入库。", +}; + +/** 后台调查(识别歧义):只读取证;试图提交下载会被白名单拒绝 */ +export const investigateScript: FakeScript = { + name: "investigate", + match: /调查待处理事项/, + steps: [ + { + text: "查本地作品与远端候选。", + toolCalls: [{ capability: "catalog.lookup", input: { query: "Twins", limit: 5 } }], + }, + { text: "查库存。", toolCalls: [{ capability: "inventory.query", input: { query: "Twins", limit: 5 } }] }, + { text: "顺手提交下载。", toolCalls: [{ capability: "download.submit", input: { planId: ZERO } }] }, + ], + finalText: "两个同名候选(1988)证据相当,无法自动确定;建议人工在待处理中心选择。", +}; + +export const ALL_SCRIPTS: FakeScript[] = [delegateScript, investigateScript]; diff --git a/apps/server/test/e2e/fixtures/worker.ts b/apps/server/test/e2e/fixtures/worker.ts new file mode 100644 index 00000000..0f23fb5f --- /dev/null +++ b/apps/server/test/e2e/fixtures/worker.ts @@ -0,0 +1,45 @@ +// 崩溃恢复 e2e 的 worker 子进程:与 harness 同一套模块装配(fixtures/modules.ts),连同一业务/引擎 schema。 +// 环境变量:BIZ_SCHEMA / OW_SCHEMA / SESSION_DIR(可选 DATABASE_URL)。就绪后输出 "worker ready";SIGTERM 优雅退出。 +import type { DB } from "@umt/db"; +import { Kysely, PostgresDialect } from "kysely"; +import pg from "pg"; +import { ConfigSchema } from "../../../src/config.ts"; +import { bootstrap } from "../../../src/context.ts"; +import { EventBus } from "../../../src/http/sse.ts"; +import { createLogger } from "../../../src/logger.ts"; +import { startWorker } from "../../../src/worker/index.ts"; +import { buildModules } from "./modules.ts"; +import { crashScript } from "./scripts.ts"; + +const url = Deno.env.get("DATABASE_URL") ?? "postgres://umt:umt@localhost:55433/umt"; +const biz = Deno.env.get("BIZ_SCHEMA")!; +const ow = Deno.env.get("OW_SCHEMA")!; +const sessionDir = Deno.env.get("SESSION_DIR")!; +await import("@umt/db"); // 注册 int8 解析器 +const db = new Kysely({ + dialect: new PostgresDialect({ + pool: new pg.Pool({ connectionString: url, max: 8, options: `-c search_path=${biz},public` }), + }), +}); +const logger = createLogger({ level: Deno.env.get("E2E_LOG_LEVEL") === "debug" ? "debug" : "warn" }); +const config = ConfigSchema.parse({ + DATABASE_URL: url, + SECRET_MASTER_KEY: "e2e-master-key-0123456789", + NODE_ENV: "development", +}); +const bus = new EventBus(1000); +const modules = buildModules({ sessionDir, fakeScripts: [crashScript] }); +const booted = await bootstrap({ role: "worker", config, logger, db, events: bus, modules, engineSchema: ow }); +const worker = await startWorker({ + logger, + heartbeatMs: 60_000, + bus, + runtime: { ctx: booted.ctx, modules, setRelay: booted.outbox.setRelay, outboxIntervalMs: 100, concurrency: 4 }, +}); +console.log(`worker ready pid=${Deno.pid}`); +Deno.addSignalListener("SIGTERM", async () => { + await worker.stop().catch(() => {}); + await booted.engine.stop().catch(() => {}); + await db.destroy().catch(() => {}); + Deno.exit(0); +}); diff --git a/apps/server/test/e2e/harness.ts b/apps/server/test/e2e/harness.ts new file mode 100644 index 00000000..6cfbc305 --- /dev/null +++ b/apps/server/test/e2e/harness.ts @@ -0,0 +1,317 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { FakeScript } from "@umt/agent"; +import { type DB, migrateToLatest } from "@umt/db"; +import { + createMockQb, + createMockTmdb, + createTempTree, + type MockQb, + type MockTmdb, + type TempTree, + TMDB_FIXTURES, + type TmdbFixture, +} from "@umt/testkit"; +import { Kysely, PostgresDialect, sql } from "kysely"; +import pg from "pg"; +import { ConfigSchema } from "../../src/config.ts"; +import { type Bootstrapped, bootstrap } from "../../src/context.ts"; +import { createApp } from "../../src/http/app.ts"; +import { EventBus } from "../../src/http/sse.ts"; +import { createLogger } from "../../src/logger.ts"; +import type { ServerModule } from "../../src/modules/types.ts"; +import { type RunningWorker, startWorker } from "../../src/worker/index.ts"; +import { withSchemaLock } from "../auth-settings/harness.ts"; +import { buildModules } from "./fixtures/modules.ts"; + +// 端到端夹具:真实 PG(每个文件独立业务 schema + 独立引擎 schema)+ 真实四个模块 + OpenWorkflow 引擎/worker/outbox 中继 +// + testkit mock(qB / TMDB / RSS)+ 临时目录(缓存卷、媒体库、Agent 会话)。HTTP 走 createApp(DEV key = admin)。 + +export const PG_URL = process.env.DATABASE_URL ?? "postgres://umt:umt@localhost:55433/umt"; +export const DEV_KEY = "dev-admin-e2e-key"; +export const MASTER_KEY = "e2e-master-key-0123456789"; +/** mock TMDB 的 v3 key(32 位十六进制,经设置 metadata.tmdbApiKey 录入) */ +export const TMDB_KEY = "0123456789abcdef0123456789abcdef"; + +export async function pgAvailable(): Promise { + if (process.env.UMT_SKIP_DB_TESTS === "1") return false; + const c = new pg.Client({ connectionString: PG_URL, connectionTimeoutMillis: 2000 }); + try { + await c.connect(); + await c.end(); + return true; + } catch { + return false; + } +} + +// biome-ignore lint/suspicious/noExplicitAny: 测试断言直接读 JSON +type Loose = any; + +export interface E2E { + name: string; + biz: string; + ow: string; + db: Kysely; + booted: Bootstrapped; + modules: ServerModule[]; + bus: EventBus; + worker: RunningWorker | null; + qb: MockQb; + tmdb: MockTmdb; + lib: TempTree; + cache: TempTree; + sessionDir: string; + api(method: string, path: string, body?: unknown): Promise<{ status: number; body: Loose }>; + /** PATCH /settings 并断言成功 */ + settings(patch: Record): Promise; + startWorker(): Promise; + close(): Promise; +} + +export interface E2EOptions { + fakeScripts?: FakeScript[]; + /** false:只做 API/引擎客户端(崩溃恢复用例由子进程跑 worker) */ + worker?: boolean; + tmdbFixtures?: TmdbFixture[]; +} + +export async function waitFor( + fn: () => Promise, + ms: number, + what: string, + every = 200, +): Promise { + const deadline = Date.now() + ms; + let last: unknown; + while (Date.now() < deadline) { + try { + const v = await fn(); + if (v) return v as T; + } catch (err) { + last = err; + } + await new Promise((r) => setTimeout(r, every)); + } + throw new Error(`等待超时:${what}${last ? `(${(last as Error).message})` : ""}`); +} + +export async function createE2E(name: string, opts: E2EOptions = {}): Promise { + const uniq = `${process.pid.toString(36)}${Math.random().toString(36).slice(2, 7)}`; + const biz = `e2e_${name}_${uniq}`; + const ow = `${biz}_ow`; + const admin = new pg.Client({ connectionString: PG_URL }); + await admin.connect(); + await admin.query(`create schema ${biz}`); + await admin.end(); + const db = new Kysely({ + dialect: new PostgresDialect({ + pool: new pg.Pool({ connectionString: PG_URL, max: 10, options: `-c search_path=${biz},public` }), + }), + }); + await withSchemaLock(() => migrateToLatest(db, { migrationTableSchema: biz })); + + const qb = await createMockQb({ addVisibleAfterPolls: 0 }); + const tmdb = await createMockTmdb({ apiKey: TMDB_KEY, fixtures: opts.tmdbFixtures ?? TMDB_FIXTURES }); + const lib = await createTempTree({}, { sentinel: true, prefix: "umt-e2e-lib-" }); + const cache = await createTempTree({}, { prefix: "umt-e2e-cache-" }); + const sessionDir = await mkdtemp(join(tmpdir(), "umt-e2e-sessions-")); + + const logger = createLogger({ level: process.env.E2E_LOG_LEVEL === "debug" ? "debug" : "silent" }); + const config = ConfigSchema.parse({ + DATABASE_URL: PG_URL, + SECRET_MASTER_KEY: MASTER_KEY, + DEV_ADMIN_KEY: DEV_KEY, + NODE_ENV: "development", + }); + const bus = new EventBus(5000); + const modules = buildModules({ sessionDir, ...(opts.fakeScripts ? { fakeScripts: opts.fakeScripts } : {}) }); + const booted = await bootstrap({ + role: "all", + config, + logger, + db, + events: bus, + modules, + engineSchema: ow, + engineMigrations: true, + }); + const app = createApp({ + config: { NODE_ENV: "development", DEV_ADMIN_KEY: DEV_KEY }, + logger, + bus, + role: "all", + version: "e2e", + db, + modules, + ctx: booted.ctx, + }); + + let worker: RunningWorker | null = null; + const e: E2E = { + name, + biz, + ow, + db, + booted, + modules, + bus, + worker, + qb, + tmdb, + lib, + cache, + sessionDir, + async api(method, path, body) { + const res = await app.request(`/api/v1${path}`, { + method, + headers: { + authorization: `Bearer ${DEV_KEY}`, + ...(body !== undefined ? { "content-type": "application/json" } : {}), + }, + ...(body !== undefined ? { body: JSON.stringify(body) } : {}), + }); + return { status: res.status, body: await res.json() }; + }, + async settings(patch) { + const r = await e.api("PATCH", "/settings", patch); + if (r.status !== 200) throw new Error(`PATCH /settings 失败:${JSON.stringify(r.body)}`); + return r.body.data; + }, + async startWorker() { + worker = await startWorker({ + logger, + heartbeatMs: 60_000, + bus, + runtime: { + ctx: booted.ctx, + modules, + setRelay: booted.outbox.setRelay, + outboxIntervalMs: 100, + concurrency: 4, + }, + }); + e.worker = worker; + }, + async close() { + await worker?.stop().catch(() => {}); + await booted.engine.stop().catch(() => {}); + for (const m of [...modules].reverse()) await m.stop?.(); + await db.destroy(); + await qb.close(); + await tmdb.close(); + await lib.cleanup(); + await cache.cleanup(); + await rm(sessionDir, { recursive: true, force: true }); + await withSchemaLock(async () => { + const c = new pg.Client({ connectionString: PG_URL }); + await c.connect(); + await c.query(`drop schema if exists ${biz} cascade`); + await c.query(`drop schema if exists ${ow} cascade`); + await c.end(); + }); + }, + }; + // 通用设置:fake 模型、TMDB 走设置里的 secret、结算快一点 + await e.settings({ + pi: { provider: "fake", model: "fake-model" }, + metadata: { tmdbApiKey: TMDB_KEY, tmdbBaseUrl: `${tmdb.url}/3` }, + retention: { seedRetentionDays: 0 }, + }); + if (opts.worker !== false) await e.startWorker(); + return e; +} + +/** 媒体库根 + qB 实例(保存路径 = 缓存临时目录)+ 站点 */ +export async function seedBase(e: E2E, site: { key?: string; baseUrl?: string } = {}) { + const root = await e.db + .insertInto("library_root") + .values({ name: `lib-${e.name}`, media_kind: "mixed", local_path: e.lib.root, management_mode: "managed" }) + .returning("id") + .executeTakeFirstOrThrow(); + const inst = await e.db + .insertInto("downloader_instance") + .values({ + name: `qb-${e.name}`, + base_url: e.qb.url, + username: "admin", + settings: JSON.stringify({ savePath: e.cache.root }), + }) + .returning("id") + .executeTakeFirstOrThrow(); + const s = await e.db + .insertInto("site") + .values({ + key: site.key ?? "audiences", + name: "观众", + adapter: "nexusphp", + base_url: site.baseUrl ?? "https://audiences.example/", + rate_limit: JSON.stringify({ minIntervalMs: 0, concurrency: 5 }), + }) + .returning("id") + .executeTakeFirstOrThrow(); + return { rootId: root.id, instanceId: inst.id, siteId: s.id }; +} + +/** 作品 + 订阅 + 1080p Slot + 两个已识别的站点资源(720p / 1080p);不发 release.resolved,只有 Agent 会去规划 */ +export async function seedMovieTarget(e: E2E, siteId: string) { + const entity = await e.db + .insertInto("media_entity") + .values({ kind: "movie", title: "沙丘2", original_title: "Dune: Part Two", year: 2024, status: "active" }) + .returning("id") + .executeTakeFirstOrThrow(); + const sub = await e.db + .insertInto("subscription") + .values({ media_entity_id: entity.id, media_kind: "movie", mode: "once" }) + .returning("id") + .executeTakeFirstOrThrow(); + const slot = await e.db + .insertInto("acquisition_slot") + .values({ + media_entity_id: entity.id, + subscription_id: sub.id, + name: "1080p", + hard_conditions: JSON.stringify({ resolutions: ["1080p"] }), + }) + .returning("id") + .executeTakeFirstOrThrow(); + const rel = async (title: string, n: number) => + ( + await e.db + .insertInto("source_release") + .values({ + site_id: siteId, + site_release_id: String(880000 + n), + title, + size_bytes: 300_000, + media_entity_id: entity.id, + resolution_status: "resolved", + raw: JSON.stringify({ _site: { hr: false } }), + }) + .returning("id") + .executeTakeFirstOrThrow() + ).id; + const r720 = await rel("Dune.Part.Two.2024.720p.WEB-DL.x264-AAA", 1); + const r1080 = await rel("Dune.Part.Two.2024.1080p.BluRay.x264-GRP", 2); + return { entityId: entity.id, slotId: slot.id, r720, r1080 }; +} + +export async function runRow(e: E2E, runId: string) { + return e.db.selectFrom("agent_run").selectAll().where("id", "=", runId).executeTakeFirstOrThrow(); +} + +export async function runEvents(e: E2E, runId: string) { + return e.db.selectFrom("agent_run_event").selectAll().where("run_id", "=", runId).orderBy("seq").execute(); +} + +/** 等 mock qB 收到某个种子后置为完成并落盘 */ +export async function completeDownload(e: E2E, title: string, ms = 60_000) { + const { torrentFor } = await import("./fixtures/modules.ts"); + const hash = torrentFor(title).hash; + await waitFor(async () => e.qb.get(hash), ms, `qB 收到 ${title}`); + await e.qb.complete(hash, { writeFiles: true }); + return hash; +} + +export { sql }; diff --git a/apps/server/vitest.config.ts b/apps/server/vitest.config.ts new file mode 100644 index 00000000..d2e7e872 --- /dev/null +++ b/apps/server/vitest.config.ts @@ -0,0 +1,9 @@ +import { defineConfig } from "vitest/config"; + +// 真实 PG 的集成/e2e 测试共用开发库并行跑:建/删 schema 串行在一把咨询锁上(withSchemaLock), +// 负载高时 beforeAll/afterAll 排队会超过缺省 10s,这里放宽钩子超时。 +export default defineConfig({ + test: { + hookTimeout: 120_000, + }, +}); diff --git a/docs/runbook/agent.md b/docs/runbook/agent.md new file mode 100644 index 00000000..5fce3f35 --- /dev/null +++ b/docs/runbook/agent.md @@ -0,0 +1,141 @@ +# Agent 运行手册(W2-D) + +实现:`apps/server/src/modules/agent/`(会话 API、runAgent / resolveDecision 任务、授权与账本)、 +`apps/server/src/modules/acquisition-organize/capabilities.ts`、`apps/server/src/modules/sources-identity/capabilities.ts`、 +`packages/agent`(Pi 接入、FakeModelProvider)、迁移 `0021_agent_runtime`。 + +## 能力 + +启动时装配 14 个能力(`catalog.lookup`、`release.search`、`release.inspect`、`identity.resolve`、`download.plan`、`download.submit`、`organize.plan`、`organize.execute`、`task.inspect`、`task.retry`、`task.cancel`、`policy.preview`、`policy.apply`、`inventory.query`)。注册数不等于 14 时进程启动失败。Web(`POST /capabilities/{id}/invoke`)、Pi 工具和后台任务都走 `ctx.capabilities.invoke` 这一条路径。 + +- **授权**(`agent/authorize.ts`):propose / approve / execute 类必须满足以下之一: + - 单次审批 `approval`:状态为 approved,绑定计划类型 + 计划 ID + payload hash + revision,且未过期。执行成功后审批标记为 consumed。 + - 持续授权 `automation_grant`:`action` 覆盖该能力(比如 `download.submit`,或策略发布自动生成的 `policy.acquisition`,或 `*`),状态 active 且未过期,绑定的策略版本仍是 published。`scope.mediaEntityIds`、`scope.mediaKinds`、`scope.capabilities` 要包含目标;`budget.maxInvocationsPerDay`、`budget.maxBytesPerDay` 还有余量。 + + 模型参数里写"用户已同意"不起作用。输入里的 `approvalId` 只是一个指针,授权结果以库里的审批记录为准。控制台管理员从 Web 手动调用(actor=`user:*`)算人工授权,API Key、Agent 和后台任务都必须有授权记录。自动化总开关关闭或处于只读对账时,自动来源的 execute 一律拒绝。 +- **审计**:每次调用都写一条 `audit_event`,action 为 `capability.invoke` 或 `capability.denied`,payload 包含 outcome、授权 ID、幂等键和脱敏后的输入摘要。 +- **幂等键与操作账本**:有副作用的调用由服务端生成幂等键,格式为 `id:v版本::sha256(规范化输入)`,并记入 `capability_invocation`。状态为 succeeded 时直接返回记录的结果。状态为 running(进程在调用途中崩溃)时,用同一个键重跑处理器来对账,处理器本身是幂等的。 +- `download.plan` 只生成 **draft** 计划:不预留 Slot,不计入预算,也不会被后台巡检补提交。`download.submit` 在授权通过后先把 draft 转成 reserved(Slot revision 必须与生成计划时一致),再入队 `submitDownload`,和自动路径共用幂等键 `plan:`。 +- `organize.plan` 把冻结的整理计划存成 artifact(kind=plan),`payloadHash` 就是计划指纹。`organize.execute` 执行前会重新冻结一次,指纹不一致就返回 conflict。 + +## 会话 API + +| 端点 | 说明 | +|---|---| +| `POST /agent/runs {goal, mode: execute\|plan_only, capabilityAllowlist?}` | 建 `agent_run` 和业务任务行(kind=agentRun),并入队持久任务 `runAgent`。**HTTP 请求里不跑模型**,返回时状态为 `queued`。`plan_only` 只允许 read / preview / propose 类能力,白名单里放了 execute 类会返回 422 | +| `GET /agent/runs` / `GET /agent/runs/{id}?afterSeq&limit` | 详情包括:`agent_run_event` 按 seq 分页、追加的消息、服务端核查过的目标汇报(`report`) | +| `POST /agent/runs/{id}/messages {text}` | 追加用户消息。运行中时,当前会话在本轮结束后消费;已结束或等待中时,会唤醒一次新的 runAgent 续跑 | +| `POST /agent/runs/{id}/abort {reason?}` | 写取消标志,会话监视器会中止模型并收尾为 cancelled。只停止后续步骤,不撤销已经完成的外部动作 | +| `GET/POST /automation-grants`、`PATCH /automation-grants/{id}` | 持续授权。PATCH 只能撤销或缩短有效期,扩大范围要新建一条 | +| `POST /approvals {planType, planId}`、`PATCH /approvals/{id} {status, payloadHash}` | 单次审批。批准时要回传 payloadHash,计划变化后旧审批失效 | +| `GET /capabilities`、`POST /capabilities/{id}/invoke` | 能力目录与调用,HTTP scope 按能力描述符再校验一次 | +| `GET /tasks[/{id}]`、`POST /tasks/{id}/cancel`、`POST /tasks/{id}/retry` | 业务任务 | + +**增量推送**:事件先写 `agent_run_event`(run_id + seq 唯一),再经 `ctx.events` 推到全局 SSE(`GET /events`)。推送的类型有 `agent.run_status`、`agent.text_delta`、`agent.tool_call`;未授权导致的工具失败,`phase` 为 `denied`。断线后,先用全局 `?after=` 补读总线事件,再用 `GET /agent/runs/{id}?afterSeq=<最后收到的 runSeq>` 补读单个 run 的完整事件。 + +## runAgent 做什么 + +1. `drive` 步骤: + - 从设置读 Pi 参数(provider / baseUrl / model / 思考档位 / 上下文预算 / 压缩)和预算。`pi.apiKeySecretRef` 现场解密后传给 SDK,不落日志。 + - 检查今日次数和费用预算(`budget.agentMaxRunsPerDay` / `agentMaxCostUsdPerDay`)。 + - 打开 `AGENT_SESSION_DIR/.jsonl`。`pendingOperations>0` 说明上次崩溃:先按幂等键对账,再 `resume()`;否则就 `prompt(goal)`。最后逐条消费追加的消息。 +2. `goal-settled` 步骤:只在 execute 模式且会话正常结束时执行。每隔 `agent.goalPollSec` 检查一次,直到本 run 触达的计划、下载和整理记录全部落定,最长等 `agent.goalWaitHours`。等待期间 run 状态为 `waiting`,汇报写的是"已提交≠已入库"。 +3. `finalize` 步骤:从业务表汇总计划、下载任务和整理记录,生成 `AgentGoalReport`(artifact kind=report),并据此决定终态: + - 入库完成 → `succeeded`,汇报带整理记录 ID; + - 仍在等待 → `waiting`; + - 失败 → `failed`; + - 超预算 → `budget_exceeded`,同时开一条待处理事项 `agent_budget:`; + - 取消 → `cancelled`。 + + 模型的最终文本只附在汇报后面,标注"模型汇报(未经核查)"。 + +worker 崩溃后,新 worker 在引擎租约到期后(约 30s)接手,`drive` 步骤会从头重跑。已经完成的工具调用不会重放。副作用调用的结果模型会看到"结果未知",模型需要重新核对;同参数再调用一次,会直接拿到账本里的结果。 + +## 后台触发与 resolveDecision + +- `decision.created` 路由到 `resolveDecision(trigger=created)`。满足以下全部条件时,会发起一次有界调查 run:`agent.autoInvestigate=true`、kind 为 `identity_ambiguous` / `download_failed` / `download_stalled`、事项仍为 open、事项不是由 Agent 自己引起的。这个 run 的参数:`trigger=event`、`mode=plan_only`,白名单为只读能力加 `identity.resolve`,工具调用上限为 `agent.investigateMaxToolCalls`,`dedupe_key=investigate:`。同一事项只会有一个 run,重复事件或重新打开都不会再建。调查结束后,事项转为 `awaiting_user`,`evidence.agentInvestigation` 里带上结论。 +- `decision.updated(status=resolved)` 路由到 `resolveDecision(trigger=updated)`,执行用户选中的处理: + - 选项带 `capabilityId` 时经注册中心调用(照样校验授权); + - `retry` 重试关联任务; + - `reevaluate` / `accept` 按对象重新发起 evaluateAcquisition; + - `dismiss` / `ignore` 不做任何动作。 + + 执行结果写进 `decision.resolution.execution`。身份歧义类由 sources-identity 在处理端点里直接执行。 + +## 配置上游 + +在设置页或 `PATCH /settings` 里配置: + +```jsonc +{ + "pi": { + "provider": "anthropic", // 或 "openai-compatible"(baseUrl 要带 /v1);"fake" = 演示假模型 + "baseUrl": null, // Anthropic 写根地址,不带 /v1 + "model": "<模型 ID>", + "thinkingLevel": "medium", + "contextBudget": 128000, + "compaction": { "enabled": true, "thresholdRatio": 0.8 }, + "apiKey": "<只写,存为 secret>" + }, + "agent": { + "maxToolCalls": 40, "maxDurationSec": 900, // 单次运行预算 + "maxTokensPerRun": null, "maxCostUsdPerRun": null, + "autoInvestigate": false, "investigateMaxToolCalls": 8, + "goalPollSec": 60, "goalWaitHours": 72 + }, + "budget": { "agentMaxRunsPerDay": 50, "agentMaxCostUsdPerDay": 5 } +} +``` + +- 会话目录由环境变量 `AGENT_SESSION_DIR` 指定,缺省为 `/data/agent-sessions`。api 进程和 worker 进程要能读到同一个目录,这是崩溃恢复的前提。 +- TMDB 密钥放在 `metadata.tmdbApiKey`(只写,存为 secret `metadata_api_key`),地址放在 `metadata.tmdbBaseUrl`(可选)。设置里没有时,回退到环境变量 `TMDB_API_KEY` / `TMDB_ACCESS_TOKEN` / `TMDB_BASE_URL`。 + +## fake 模式演示 + +```sh +curl -X PATCH localhost:4380/api/v1/settings -H "Authorization: Bearer umt-dev-admin-key" \ + -H 'content-type: application/json' -d '{"pi":{"provider":"fake","model":"fake-model"}}' +curl -X POST localhost:4380/api/v1/agent/runs -H "Authorization: Bearer umt-dev-admin-key" \ + -H 'content-type: application/json' -d '{"goal":"看看库里有什么","mode":"plan_only"}' +``` + +没有配置 `agent.fakeScript` 时,使用内置演示脚本:查一次库存,然后如实汇报,不做写操作。自定义脚本用 JSON 写,可以是单个对象或数组: + +```json +{ + "match": "收进来", + "steps": [ + { "text": "查库存", "toolCalls": [{ "capability": "inventory.query", "input": { "query": "沙丘2" } }] }, + { "toolCalls": [{ "capability": "download.plan", + "input": { "slotId": "{{inventory.query.items.0.slots.0.id}}", "sourceReleaseId": "" } }] }, + { "toolCalls": [{ "capability": "download.submit", "input": { "planId": "{{download.plan.plan.id}}" } }] } + ], + "finalText": "已提交" +} +``` + +`{{能力ID.路径}}` 取该能力最近一次成功调用结果里对应路径的值。测试直接给 `createAgentModule({ fakeScripts })` 注入 `FakeScript`,输入可以写成函数。 + +## 事件约定 + +没有单独的 `download.completed` / `organization.completed` 事件,用现有事件表达: +- 下载完成 = `download.status_changed` 且 `to=completed`(只发一次); +- 入库完成 = `organization.changed` 且 `status=succeeded`。 + +## e2e + +`apps/server/test/e2e/`:每个文件使用独立的业务 schema 和引擎 schema,跑真实的四个模块、OpenWorkflow worker 和 outbox 中继,外部依赖用 testkit 的 mock(qB、TMDB、RSS),文件放在临时目录。 + +| 文件 | 内容 | +|---|---| +| `e1-rss-to-library` | RSS → 识别 → 策略 → 下载 → Copy 入库 → 整理记录增量 → Slot satisfied | +| `e2-agent-delegate` | 委托闭环:无授权时被拒,有授权时成功;汇报以整理记录为准 | +| `e3-background-continues` | 关闭聊天后台照样推进 | +| `e4-crash-recovery` | worker 子进程在 download.submit 时 kill -9,新 worker resume,qB 只收到一次 add | +| `e5-auto-investigate` | 识别歧义 → 有界调查,去重 | + +## 已知限制 + +- 编译后的二进制(`deno compile`)只授予 `--allow-read=/app`,没有写权限。Agent 会话目录和入库 Copy/Move 在容器里都需要写权限,需要在 W4 统一补 `--allow-write=<数据目录,库根>`,在此之前生产容器里的 runAgent 会失败。 +- 子代理(`subagentsEnabled`)只做记录,没有实现。 +- 追加消息与正在收尾的 run 存在很小的竞态窗口。`finalize` 会检查一次未消费的消息并补唤醒一次 run。 diff --git a/docs/runbook/auth.md b/docs/runbook/auth.md index 357dcb1e..dc8e5cbf 100644 --- a/docs/runbook/auth.md +++ b/docs/runbook/auth.md @@ -68,7 +68,7 @@ update admin_session set revoked_at = now(), revoke_reason = 'reset' where revok - 库根登记/改路径时写哨兵文件 `.uhd-root`(`writeRootSentinel`),目录不存在或不可写 → 412;`POST /library-roots/{id}/test` 查哨兵在线。 - 实例凭证 `credential` 只写 → secret;`POST /downloader-instances/{id}/test`(qB:有用户名走登录,否则当 WebUI API Key)、`POST /media-server-instances/{id}/test`(Emby/Jellyfin ping),结果写回 health。 - `POST /path-mappings/{id}/verify`:本地前缀是否为可达目录。 -- `PUT /sites/{id}/credentials {label?, authKind, credential, username?}`:按 (site, label) 建或更新 site_account,凭证换新 secret(旧的删除),响应不回显。 +- `PUT /sites/{id}/credentials {label?, authKind, credential, username?}`(兼容别名):等价于按 (site, label) 新建或更新一条 site_account,凭证换新 secret(旧的删除),响应不回显。规范接口是 `/site-accounts` 资源 CRUD,新客户端请用 `POST /site-accounts` / `PATCH /site-accounts/{id}`。 - `GET /audit-events?actorType&actorId&action&targetType&targetId&cursor&limit`:新→旧,`action=admin.*` 前缀匹配。动作枚举见 contracts `AuditAction`。 ## 引擎面板(仅 admin,只读直查 `openworkflow` schema) diff --git a/packages/contracts/src/agent.ts b/packages/contracts/src/agent.ts index 360c0021..7f82f888 100644 --- a/packages/contracts/src/agent.ts +++ b/packages/contracts/src/agent.ts @@ -1,4 +1,6 @@ import { z } from "zod"; +import { AgentRunEventRow } from "./agent-events.ts"; +import { KnownCapabilityId } from "./capability.ts"; import { EntityBase, Id, JsonObject, NonNegInt, SERVER_MANAGED, Timestamp } from "./common.ts"; // 人工/AI 域:Agent 运行、证据/计划产物、待处理事项、审批、持续授权(REQ §10、§11.3、§11.4) @@ -15,6 +17,9 @@ export const AgentRunStatus = z.enum([ "budget_exceeded", ]); export type AgentRunStatus = z.infer; +/** execute:授权范围内边执行边汇报;plan_only:"先给方案",只允许 read/preview/propose 类能力 */ +export const AgentRunMode = z.enum(["execute", "plan_only"]); +export type AgentRunMode = z.infer; /** 单次运行预算(调用次数/token/时长/费用,REQ §10.5) */ export const AgentBudget = z.object({ @@ -60,6 +65,9 @@ export const AgentRun = EntityBase.extend({ startedAt: Timestamp.nullable(), finishedAt: Timestamp.nullable(), createdBy: z.string().nullable(), + mode: AgentRunMode, + /** 后台触发按根因去重(如 investigate:);用户委托为 null */ + dedupeKey: z.string().nullable(), }); export type AgentRun = z.infer; export const AgentRunCreate = AgentRun.pick({ @@ -283,3 +291,125 @@ export const AutomationGrantUpdate = z.object({ reason: z.string().max(2000).optional(), }); export type AutomationGrantUpdate = z.infer; + +// ── Agent 会话 API(W2-D) ── + +/** POST /agent/runs:建 agent_run + 入队持久任务 runAgent(不在 HTTP 请求里跑模型) */ +export const AgentRunStartInput = z.object({ + goal: z.string().trim().min(1).max(8000), + mode: AgentRunMode.default("execute"), + /** 能力白名单(缺省按模式给全部可用能力);plan_only 下超出 read/preview/propose 的会被拒绝 */ + capabilityAllowlist: z.array(KnownCapabilityId).min(1).max(50).optional(), +}); +export type AgentRunStartInput = z.infer; + +export const AgentRunMessageInput = z.object({ text: z.string().trim().min(1).max(8000) }); +export type AgentRunMessageInput = z.infer; + +export const AgentRunAbortInput = z.object({ reason: z.string().max(500).optional() }); +export type AgentRunAbortInput = z.infer; + +export const AgentRunListQuery = z.object({ + cursor: z.string().min(1).optional(), + limit: z.coerce.number().int().min(1).max(200).default(50), + status: AgentRunStatus.optional(), + trigger: AgentTrigger.optional(), +}); +export type AgentRunListQuery = z.infer; + +/** 详情里的事件分页:afterSeq(不含)之后按 seq 升序;断线后按最后收到的 runSeq 补读 */ +export const AgentRunDetailQuery = z.object({ + afterSeq: z.coerce.number().int().min(-1).default(-1), + limit: z.coerce.number().int().min(1).max(1000).default(200), +}); +export type AgentRunDetailQuery = z.infer; + +/** + * 目标完成度汇报:由服务端从业务表汇总(下载计划 → 下载任务 → 整理记录),不采信模型文本。 + * "已提交"只代表进入下载器;只有整理记录 succeeded 才算入库。 + */ +export const AgentGoalReport = z.object({ + state: z.enum(["no_actions", "waiting", "completed", "partial", "failed"]), + summary: z.string(), + submitted: NonNegInt, + imported: NonNegInt, + inProgress: NonNegInt, + failed: NonNegInt, + plans: z.array(z.object({ planId: Id, status: z.string(), slotId: Id.nullable(), sourceReleaseId: Id.nullable() })), + downloads: z.array( + z.object({ downloadTaskId: Id, planId: Id.nullable(), status: z.string(), name: z.string().nullable() }), + ), + organization: z.array( + z.object({ + recordId: Id, + status: z.string(), + downloadTaskId: Id.nullable(), + mediaEntityId: Id.nullable(), + targetPath: z.string().nullable(), + }), + ), + /** 模型最终文本(仅供参考,不作为完成依据) */ + modelText: z.string().nullable(), + generatedAt: Timestamp, +}); +export type AgentGoalReport = z.infer; + +export const AgentRunMessageStatus = z.enum(["pending", "consumed"]); +export type AgentRunMessageStatus = z.infer; +/** 追加到会话的用户消息(纠正/补充);runAgent 任务按创建顺序消费 */ +export const AgentRunMessage = EntityBase.extend({ + runId: Id, + text: z.string(), + status: AgentRunMessageStatus, + createdBy: z.string().nullable(), + consumedAt: Timestamp.nullable(), +}); +export type AgentRunMessage = z.infer; + +export const AgentRunDetail = z.object({ + run: AgentRun, + events: z.array(AgentRunEventRow), + /** 下一页起点(传给 afterSeq);null 表示已到最新 */ + nextAfterSeq: z.number().int().nullable(), + messages: z.array(AgentRunMessage), + report: AgentGoalReport.nullable(), +}); +export type AgentRunDetail = z.infer; + +/** 单次审批的创建:服务端从计划读取 payloadHash / expectedRevision,调用方只给计划 */ +export const ApprovalRequest = z.object({ + planType: z.enum(["acquisition_plan", "organize_plan", "policy_revision"]), + planId: Id, + expiresInSec: z + .number() + .int() + .min(60) + .max(30 * 86400) + .default(86400), + reason: z.string().max(2000).optional(), +}); +export type ApprovalRequest = z.infer; + +export const CapabilityInvocationStatus = z.enum(["running", "succeeded", "failed"]); +export type CapabilityInvocationStatus = z.infer; +/** + * 有副作用能力的操作账本(capability_invocation):幂等键由服务端生成且唯一;succeeded 直接返回记录的输出, + * running(调用中崩溃)按同一幂等键重跑处理器对账。 + */ +export const CapabilityInvocation = EntityBase.extend({ + idempotencyKey: z.string().min(1), + capabilityId: z.string().min(1), + capabilityVersion: z.number().int().min(1), + agentRunId: Id.nullable(), + actorId: z.string(), + source: z.string(), + status: CapabilityInvocationStatus, + input: JsonObject, + output: JsonObject.nullable(), + error: JsonObject.nullable(), + grantId: Id.nullable(), + approvalId: Id.nullable(), + attempts: NonNegInt, + finishedAt: Timestamp.nullable(), +}); +export type CapabilityInvocation = z.infer; diff --git a/packages/contracts/src/api.ts b/packages/contracts/src/api.ts index 631267cc..6e538884 100644 --- a/packages/contracts/src/api.ts +++ b/packages/contracts/src/api.ts @@ -1,8 +1,9 @@ import { z } from "zod"; import * as agent from "./agent.ts"; import * as assets from "./assets.ts"; +import * as auth from "./auth.ts"; import { CapabilityDescriptor, KnownCapabilityId } from "./capability.ts"; -import { type ApiScope, CursorQuery, cursorPage, Id, IdParam } from "./common.ts"; +import { type ApiScope, CursorQuery, cursorPage, HealthStatus, Id, IdParam } from "./common.ts"; import * as content from "./content.ts"; import * as execution from "./execution.ts"; import * as identity from "./identity.ts"; @@ -105,6 +106,37 @@ const R = { read: "read", write: "admin" } as const; const del = (path: string, summary: string) => ep({ method: "delete", path, scope: "admin", summary, params: IdParam, response: OkResult }); +/** 实例连通性测试 */ +const instanceTest = (path: string, summary: string) => + ep({ method: "post", path, scope: "admin", summary, params: IdParam, response: instances.InstanceTestResult }); + +/** 入队后台任务的受理结果(taskId = 引擎 run id) */ +const Accepted = z.object({ taskId: Id }); + +/** 人工指定来源资源身份的结果 */ +const ReleaseIdentityResult = z.object({ mediaEntityId: Id, contentUnitIds: z.array(Id) }); + +/** 字段覆盖/取消覆盖的结果 */ +const FieldOverrideResult = z.object({ + mediaEntityId: Id, + changedFields: z.array(z.string()), + lockedFields: z.array(z.string()), +}); +const FieldParam = z.object({ id: Id, field: content.EntityMetadataField }); + +/** 引擎 run id 不是 uuid(openworkflow 自有 id) */ +const EngineRunIdParam = z.object({ id: z.string().min(1).max(200) }); + +/** 待处理事项聚合分组 */ +const DecisionSummaryGroup = z.object({ + kind: agent.DecisionKind, + mediaEntityId: Id.nullable(), + mediaEntityTitle: z.string().nullable(), + count: z.number().int(), + maxSeverity: z.enum(["info", "warning", "critical"]), + latestAt: z.string(), +}); + export const Api = { // ── 内容 / 身份 ── mediaEntities: { @@ -132,6 +164,53 @@ export const Api = { body: z.object({ intoId: Id, reason: z.string().max(2000) }), response: content.MediaEntity, }), + detail: ep({ + method: "get", + path: "/media-entities/{id}/detail", + scope: "read", + summary: "作品详情:外部身份、别名、内容单元、字段来源与覆盖(旧 ID 自动跳转)", + params: IdParam, + response: content.MediaEntityDetail, + }), + setIdentity: ep({ + method: "post", + path: "/media-entities/{id}/identities", + scope: "organize.execute", + summary: "人工纠错外部映射:旧映射退役(留修订),新映射 accepted + 人工锁", + params: IdParam, + body: identity.EntityIdentitySet, + response: z.object({ + mediaEntityId: Id, + changed: z.boolean(), + identityId: Id.optional(), + retired: z.array(Id).optional(), + }), + }), + refresh: ep({ + method: "post", + path: "/media-entities/{id}/refresh", + scope: "organize.execute", + summary: "刷新元数据(字段级;用户锁优先,不动路径)", + params: IdParam, + response: Accepted, + }), + setField: ep({ + method: "put", + path: "/media-entities/{id}/fields/{field}", + scope: "organize.execute", + summary: "覆盖作品字段并锁定(自动刷新不再改写)", + params: FieldParam, + body: content.FieldOverrideInput, + response: FieldOverrideResult, + }), + clearField: ep({ + method: "delete", + path: "/media-entities/{id}/fields/{field}", + scope: "organize.execute", + summary: "取消字段覆盖与锁定", + params: FieldParam, + response: FieldOverrideResult, + }), }, contentUnits: crud( "/content-units", @@ -229,16 +308,20 @@ export const Api = { ), // ── 实例 ── - libraryRoots: crud( - "/library-roots", - "媒体库根", - { - row: instances.LibraryRoot, - create: instances.LibraryRootCreate, - update: instances.LibraryRootUpdate, - }, - R, - ), + libraryRoots: { + ...crud( + "/library-roots", + "媒体库根", + { + row: instances.LibraryRoot, + create: instances.LibraryRootCreate, + update: instances.LibraryRootUpdate, + }, + R, + ), + test: instanceTest("/library-roots/{id}/test", "检查库根在线(目录 + 哨兵文件)"), + remove: del("/library-roots/{id}", "删除媒体库根"), + }, downloaderInstances: { ...crud( "/downloader-instances", @@ -258,17 +341,22 @@ export const Api = { params: IdParam, response: z.object({ ok: z.boolean(), version: z.string().nullable(), message: z.string().nullable() }), }), + remove: del("/downloader-instances/{id}", "删除下载器实例"), + }, + mediaServerInstances: { + ...crud( + "/media-server-instances", + "媒体服务器实例", + { + row: instances.MediaServerInstance, + create: instances.MediaServerInstanceCreate, + update: instances.MediaServerInstanceUpdate, + }, + R, + ), + test: instanceTest("/media-server-instances/{id}/test", "测试媒体服务器连接"), + remove: del("/media-server-instances/{id}", "删除媒体服务器实例"), }, - mediaServerInstances: crud( - "/media-server-instances", - "媒体服务器实例", - { - row: instances.MediaServerInstance, - create: instances.MediaServerInstanceCreate, - update: instances.MediaServerInstanceUpdate, - }, - R, - ), pathMappings: { ...crud( "/path-mappings", @@ -292,7 +380,84 @@ export const Api = { }, // ── 来源 ── - sites: crud("/sites", "站点", { row: sources.Site, create: sources.SiteCreate, update: sources.SiteUpdate }, R), + sites: { + ...crud("/sites", "站点", { row: sources.Site, create: sources.SiteCreate, update: sources.SiteUpdate }, R), + enable: ep({ + method: "post", + path: "/sites/{id}/enable", + scope: "admin", + summary: "开启站点", + params: IdParam, + response: sources.Site, + }), + disable: ep({ + method: "post", + path: "/sites/{id}/disable", + scope: "admin", + summary: "关闭站点(停止网络访问与新候选,历史资源仍可查)", + params: IdParam, + response: sources.Site, + }), + snapshot: ep({ + method: "post", + path: "/sites/{id}/snapshot", + scope: "admin", + summary: "立即同步站点快照(受站点限流)", + params: IdParam, + body: z.object({ maxPages: z.number().int().min(1).max(20).optional() }), + response: Accepted, + }), + snapshots: ep({ + method: "get", + path: "/sites/{id}/snapshots", + scope: "read", + summary: "站点快照/补查记录(覆盖范围与新鲜度)", + params: IdParam, + query: CursorQuery, + response: cursorPage(sources.SiteSnapshot), + }), + /** 兼容别名:规范接口是 /site-accounts 资源 CRUD;这里按 (site,label) 新建或更新 site_account 的凭证 */ + credentials: ep({ + method: "put", + path: "/sites/{id}/credentials", + scope: "admin", + summary: + "兼容别名:按 (站点, label) 新建或更新 site_account 凭证(→ secret_ref,不回显);新客户端请用 POST/PATCH /site-accounts", + params: IdParam, + body: instances.SiteCredentialsPut, + response: instances.SiteCredentialsResult, + }), + }, + siteDefinitions: { + list: ep({ + method: "get", + path: "/site-definitions", + scope: "read", + summary: "随产品发布的站点适配器定义(含是否已登记)", + response: z.object({ items: z.array(sources.SiteDefinitionView) }), + }), + }, + /** 站点工作台(整理视图):库存叠加走 inventory.overlay */ + siteWorkbench: { + releases: ep({ + method: "get", + path: "/sites/{id}/releases", + scope: "read", + summary: "站点资源列表/搜索(快照优先,必要时实时补查;标注来源与更新时间)", + params: IdParam, + query: sources.WorkbenchReleaseQuery, + response: sources.WorkbenchReleasePage, + }), + release: ep({ + method: "get", + path: "/sites/{id}/releases/{rid}", + scope: "read", + summary: "站点资源详情(refresh=true 时实时复核详情;失败保留既有数据)", + params: z.object({ id: Id, rid: Id }), + query: z.object({ refresh: z.enum(["true", "false"]).optional() }), + response: sources.WorkbenchReleaseDetail, + }), + }, siteAccounts: crud( "/site-accounts", "站点账号", @@ -324,21 +489,64 @@ export const Api = { params: IdParam, response: z.object({ taskId: Id }), }), + enable: ep({ + method: "post", + path: "/feed-sources/{id}/enable", + scope: "admin", + summary: "启用 Feed", + params: IdParam, + response: sources.FeedSource, + }), + disable: ep({ + method: "post", + path: "/feed-sources/{id}/disable", + scope: "admin", + summary: "停用 Feed", + params: IdParam, + response: sources.FeedSource, + }), + reset: ep({ + method: "post", + path: "/feed-sources/{id}/reset", + scope: "admin", + summary: "手动恢复:关闭熔断、清空退避(凭证已更新后使用)", + params: IdParam, + response: sources.FeedSource, + }), }, - sourceReleases: crud( - "/source-releases", - "站点资源", - { - row: sources.SourceRelease, - filters: { - siteId: Id.optional(), - mediaEntityId: Id.optional(), - resolutionStatus: sources.ResolutionStatus.optional(), - q: z.string().max(200).optional(), + sourceReleases: { + ...crud( + "/source-releases", + "站点资源", + { + row: sources.SourceRelease, + filters: { + siteId: Id.optional(), + mediaEntityId: Id.optional(), + resolutionStatus: sources.ResolutionStatus.optional(), + q: z.string().max(200).optional(), + }, }, - }, - R, - ), + R, + ), + assignIdentity: ep({ + method: "post", + path: "/source-releases/{id}/identity", + scope: "organize.execute", + summary: "人工指定来源资源的作品/内容单元(加人工锁,自动识别不再改写)", + params: IdParam, + body: identity.ReleaseIdentityAssign, + response: ReleaseIdentityResult, + }), + unlock: ep({ + method: "post", + path: "/source-releases/{id}/unlock", + scope: "organize.execute", + summary: "解除来源资源的人工锁并重新识别", + params: IdParam, + response: Accepted, + }), + }, observations: crud( "/observations", "观察", @@ -539,17 +747,35 @@ export const Api = { body: agent.DecisionResolveInput, response: agent.Decision, }), + summary: ep({ + method: "get", + path: "/decisions/summary", + scope: "read", + summary: "未关闭待处理事项按作品与问题类型聚合", + query: z.object({ status: agent.DecisionStatus.optional() }), + response: z.object({ total: z.number().int(), groups: z.array(DecisionSummaryGroup) }), + }), + }, + approvals: { + ...crud( + "/approvals", + "审批", + { + row: agent.Approval, + update: agent.ApprovalUpdate, + filters: { status: agent.ApprovalStatus.optional(), planId: Id.optional() }, + }, + { read: "read", write: "organize.execute" }, + ), + create: ep({ + method: "post", + path: "/approvals", + scope: "organize.execute", + summary: "为计划申请单次审批(绑定 plan id + payload hash + revision + 有效期)", + body: agent.ApprovalRequest, + response: agent.Approval, + }), }, - approvals: crud( - "/approvals", - "审批", - { - row: agent.Approval, - update: agent.ApprovalUpdate, - filters: { status: agent.ApprovalStatus.optional(), planId: Id.optional() }, - }, - { read: "read", write: "organize.execute" }, - ), automationGrants: crud( "/automation-grants", "持续授权", @@ -561,32 +787,49 @@ export const Api = { }, R, ), + /** Agent 会话(W2-D):建 run 只入队持久任务 runAgent;增量经全局 SSE(agent.*),断线按 runSeq 用 afterSeq 补读 */ agentRuns: { - ...crud( - "/agent-runs", - "Agent 运行", - { - row: agent.AgentRun, - create: agent.AgentRunCreate, - filters: { status: agent.AgentRunStatus.optional(), taskId: Id.optional() }, - }, - { read: "read", write: "organize.execute" }, - ), - cancel: ep({ - method: "post", - path: "/agent-runs/{id}/cancel", - scope: "organize.execute", - summary: "取消 Agent 运行", + list: ep({ + method: "get", + path: "/agent/runs", + scope: "read", + summary: "Agent 运行列表(新→旧)", + query: agent.AgentRunListQuery, + response: cursorPage(agent.AgentRun), + }), + get: ep({ + method: "get", + path: "/agent/runs/{id}", + scope: "read", + summary: "Agent 运行详情:事件按 seq 分页、追加消息、服务端核查的目标汇报", params: IdParam, + query: agent.AgentRunDetailQuery, + response: agent.AgentRunDetail, + }), + create: ep({ + method: "post", + path: "/agent/runs", + scope: "organize.execute", + summary: "委托目标:建 agent_run 并入队 runAgent(execute | plan_only)", + body: agent.AgentRunStartInput, response: agent.AgentRun, }), message: ep({ method: "post", - path: "/agent-runs/{id}/messages", + path: "/agent/runs/{id}/messages", scope: "organize.execute", - summary: "向运行中的会话追加用户消息(纠正/补充)", + summary: "追加用户消息(纠正/补充);运行已结束时唤醒续跑", params: IdParam, - body: z.object({ text: z.string().min(1).max(8000) }), + body: agent.AgentRunMessageInput, + response: agent.AgentRun, + }), + abort: ep({ + method: "post", + path: "/agent/runs/{id}/abort", + scope: "organize.execute", + summary: "中止运行(停止未来步骤,不撤销已完成的外部动作)", + params: IdParam, + body: agent.AgentRunAbortInput, response: agent.AgentRun, }), }, @@ -651,18 +894,15 @@ export const Api = { body: ops.ApiKeyUpdate, response: ops.ApiKey, }), + revoke: del("/api-keys/{id}", "撤销 API Key"), }, auditEvents: { list: ep({ method: "get", path: "/audit-events", scope: "admin", - summary: "审计事件", - query: listQuery({ - targetType: z.string().optional(), - targetId: z.string().optional(), - action: z.string().optional(), - }), + summary: "审计事件(按 actor/action/target 过滤,新→旧)", + query: ops.AuditEventQuery, response: cursorPage(ops.AuditEvent), }), }, @@ -692,6 +932,7 @@ export const Api = { body: ops.SecretUpdate, response: ops.Secret, }), + remove: del("/secrets/{id}", "删除凭证(被引用时 409)"), }, // ── 整理(公开合同) ── @@ -762,6 +1003,208 @@ export const Api = { response: GlobalSettings, }), }, + + // ── 登录门禁 ── + /** /auth/setup、/auth/login、/auth/session 免鉴权;改密与 TOTP 只接受控制台 cookie 会话 */ + auth: { + setup: ep({ + method: "post", + path: "/auth/setup", + scope: "read", + summary: "首次初始化管理员", + body: auth.AdminSetupRequest, + response: auth.SessionInfo, + }), + login: ep({ + method: "post", + path: "/auth/login", + scope: "read", + summary: "管理员登录", + body: auth.LoginRequest, + response: auth.LoginResult, + }), + logout: ep({ method: "post", path: "/auth/logout", scope: "read", summary: "注销当前会话", response: OkResult }), + session: ep({ + method: "get", + path: "/auth/session", + scope: "read", + summary: "当前登录态(免鉴权)", + response: auth.SessionInfo, + }), + password: ep({ + method: "post", + path: "/auth/password", + scope: "admin", + summary: "修改密码(吊销其它会话)", + body: auth.PasswordChangeRequest, + response: auth.PasswordChangeResult, + }), + totpStatus: ep({ + method: "get", + path: "/auth/totp", + scope: "admin", + summary: "TOTP 状态", + response: auth.TotpStatus, + }), + totp: ep({ + method: "post", + path: "/auth/totp", + scope: "admin", + summary: "TOTP 绑定/启用/停用", + body: auth.TotpAction, + response: auth.TotpActionResult, + }), + }, + + // ── 系统 ── + system: { + health: ep({ + method: "get", + path: "/health", + scope: "read", + summary: "服务健康状态(免鉴权)", + response: HealthStatus, + }), + }, + + // ── 引擎面板(仅 admin) ── + engine: { + runs: ep({ + method: "get", + path: "/engine/runs", + scope: "admin", + summary: "引擎 runs(状态/名称/时间筛选,新→旧)", + query: ops.EngineRunQuery, + response: cursorPage(ops.EngineRun), + }), + run: ep({ + method: "get", + path: "/engine/runs/{id}", + scope: "admin", + summary: "run 详情(step attempts + signals)", + params: EngineRunIdParam, + response: ops.EngineRunDetail, + }), + stats: ep({ + method: "get", + path: "/engine/stats", + scope: "admin", + summary: "按状态计数 / 最老待处理 / 活跃 worker 快照", + response: ops.EngineStats, + }), + cancel: ep({ + method: "post", + path: "/engine/runs/{id}/cancel", + scope: "admin", + summary: "取消 run", + params: EngineRunIdParam, + body: ops.EngineCancelRequest, + response: ops.EngineCancelResult, + }), + }, + + // ── 待识别池 ── + identityPool: { + list: ep({ + method: "get", + path: "/identity-pool", + scope: "read", + summary: "待识别池(可按原始标题检索;候选不建正式作品)", + query: identity.IdentityPoolQuery, + response: cursorPage(identity.IdentityPoolItem), + }), + assign: ep({ + method: "post", + path: "/identity-pool/{id}/assign", + scope: "organize.execute", + summary: "为待识别资源手动指定候选 / 本地作品 / 外部身份", + params: IdParam, + body: identity.ReleaseIdentityAssign, + response: ReleaseIdentityResult, + }), + dismiss: ep({ + method: "post", + path: "/identity-pool/{id}/dismiss", + scope: "organize.execute", + summary: "忽略待识别资源(不再自动复查)", + params: IdParam, + body: z.object({ note: z.string().max(2000).optional() }), + response: identity.IdentityCandidate, + }), + retry: ep({ + method: "post", + path: "/identity-pool/{id}/retry", + scope: "organize.execute", + summary: "立即重新识别", + params: IdParam, + response: Accepted, + }), + }, + + // ── 媒体库 / 库存 / 首页 ── + library: { + list: ep({ + method: "get", + path: "/library", + scope: "read", + summary: "媒体库列表(已登记资产或正在追踪的作品)", + query: organization.LibraryListQuery, + response: cursorPage(organization.LibraryItem), + }), + get: ep({ + method: "get", + path: "/library/{id}", + scope: "read", + summary: "作品详情:实际路径、身份映射、元数据来源、各 Slot、候选拒绝理由、下载历史与校验结果", + params: IdParam, + response: organization.LibraryDetail, + }), + reconcile: ep({ + method: "post", + path: "/library/reconcile", + scope: "admin", + summary: "立即做一轮资产对账(库根哨兵 + 增量 stat;不删记录、不补下载)", + body: z.object({ libraryRootId: z.uuid().optional() }), + response: z.object({ runId: z.uuid(), stats: z.record(z.string(), z.unknown()) }), + }), + }, + inventory: { + overlay: ep({ + method: "get", + path: "/inventory", + scope: "read", + summary: "库存叠加(站点工作台):按作品返回已有版本 / Slot 缺口 / 下载中", + query: organization.InventoryQuery, + response: organization.InventoryResponse, + }), + mediaServers: ep({ + method: "get", + path: "/inventory/media-servers", + scope: "read", + summary: "Emby 兼容库存查询(多实例;接口失败/截断 = unknown,不等于缺失)", + query: organization.MediaServerInventoryQuery, + response: organization.MediaServerInventory, + }), + }, + dashboard: { + get: ep({ + method: "get", + path: "/dashboard", + scope: "read", + summary: "首页聚合:自动化概览/当前工作/需要我处理/缓存与保留/来源健康/最近产出", + response: organization.Dashboard, + }), + }, + downloadCache: { + cleanup: ep({ + method: "post", + path: "/download-cache/cleanup", + scope: "admin", + summary: "立即按阈值清理下载缓存(只清已验证入库且满足保留/H&R 的纳管任务)", + body: z.object({ downloaderInstanceId: z.uuid().optional() }), + response: z.object({ report: z.record(z.string(), z.unknown()) }), + }), + }, } as const; /** 用参数填充 `{name}` 路径占位,返回带 API_PREFIX 的完整路径 */ diff --git a/packages/contracts/src/events.ts b/packages/contracts/src/events.ts index 8c6eb61d..91076af7 100644 --- a/packages/contracts/src/events.ts +++ b/packages/contracts/src/events.ts @@ -20,7 +20,14 @@ export type EventEnvelope

= { payload: P; }; -/** 已知事件类型:`域.动作`。新增事件先在这里加类型与 payload schema */ +/** + * 已知事件类型:`域.动作`。新增事件先在这里加类型与 payload schema。 + * + * 没有单独的 download.completed / organization.completed 事件,约定用现有事件表达(W2 收口): + * - "下载完成" = `download.status_changed` 且 `to === "completed"`(outbox 幂等键 `download.completed:`,只发一次; + * acquisition-organize 据此入队 verifyAndImport); + * - "入库完成" = `organization.changed` 且 `status === "succeeded"`(或 partial);`changeSeq` 与公开增量 API 游标一致。 + */ export const EventType = { heartbeat: "system.heartbeat", resync: "system.resync", diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 5db7c7bd..177f10e4 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -68,6 +68,47 @@ export const PiSettings = z.object({ }); export type PiSettings = z.infer; +/** + * Agent 运行参数(REQ §10.5、§10.7):单次运行预算、后台自动调查、fake 演示脚本。 + * 上游/模型/思考档位在 pi 分组;pi.provider = "fake" 时使用脚本化假模型(不联网,演示与 e2e 用)。 + */ +export const AgentRuntimeSettings = z.object({ + /** 识别歧义 / 下载失败诊断等待处理事项创建时,自动发起有界只读调查 */ + autoInvestigate: z.boolean(), + /** 单次运行:工具调用次数上限 */ + maxToolCalls: z.number().int().min(1).max(500), + /** 单次运行:时长上限(秒) */ + maxDurationSec: z + .number() + .int() + .min(10) + .max(24 * 3600), + /** 单次运行:token 上限(null = 不限) */ + maxTokensPerRun: z.number().int().positive().nullable(), + /** 单次运行:费用上限(美元,null = 不限) */ + maxCostUsdPerRun: z.number().positive().nullable(), + /** 后台自动调查的工具调用上限(比用户委托更小) */ + investigateMaxToolCalls: z.number().int().min(1).max(100), + /** 等待目标关联的下载/入库结束时的轮询间隔(秒) */ + goalPollSec: z.number().int().min(1).max(3600), + /** 等待目标完成的最长时间(小时),超时后以"仍在等待"汇报结束 */ + goalWaitHours: z + .number() + .min(0) + .max(24 * 30), + /** fake 模式脚本(JSON,FakeScript 的可序列化子集);null 用内置演示脚本 */ + fakeScript: z.string().max(100_000).nullable(), +}); +export type AgentRuntimeSettings = z.infer; + +/** 元数据源(TMDB):密钥只存 secret 引用;未配置时回退环境变量 TMDB_API_KEY / TMDB_ACCESS_TOKEN */ +export const MetadataSettings = z.object({ + tmdbApiKeySecretRef: Id.nullable(), + /** 自建反代或测试 mock;null 用官方地址 */ + tmdbBaseUrl: z.url().nullable(), +}); +export type MetadataSettings = z.infer; + /** 每日预算与并发(REQ §6.4、§10.5) */ export const BudgetSettings = z.object({ maxNewTitlesPerDay: z.number().int().min(0), @@ -85,6 +126,8 @@ export const GlobalSettings = z.object({ retention: RetentionSettings, pi: PiSettings, budget: BudgetSettings, + agent: AgentRuntimeSettings, + metadata: MetadataSettings, }); export type GlobalSettings = z.infer; @@ -115,6 +158,18 @@ export const DEFAULT_SETTINGS: GlobalSettings = { agentMaxRunsPerDay: 50, agentMaxCostUsdPerDay: 5, }, + agent: { + autoInvestigate: false, + maxToolCalls: 40, + maxDurationSec: 900, + maxTokensPerRun: null, + maxCostUsdPerRun: null, + investigateMaxToolCalls: 8, + goalPollSec: 60, + goalWaitHours: 72, + fakeScript: null, + }, + metadata: { tmdbApiKeySecretRef: null, tmdbBaseUrl: null }, }; /** PATCH /settings:按分组部分更新;服务端与现值合并后用 GlobalSettings 整体再校验 */ @@ -132,12 +187,21 @@ export const GlobalSettingsUpdate = z.object({ }) .optional(), budget: BudgetSettings.partial().optional(), + agent: AgentRuntimeSettings.partial().optional(), + metadata: MetadataSettings.omit({ tmdbApiKeySecretRef: true }) + .partial() + .extend({ + /** 只写:TMDB API Key(v3 key 或 v4 read token),服务端存为 secret 并回填 tmdbApiKeySecretRef;null 清除 */ + tmdbApiKey: z.string().min(1).max(4096).nullable().optional(), + }) + .optional(), }); export type GlobalSettingsUpdate = z.infer; /** 合并部分更新(纯函数,web 乐观更新与 server 共用) */ export function mergeSettings(current: GlobalSettings, patch: GlobalSettingsUpdate): GlobalSettings { const { apiKey: _apiKey, compaction, ...pi } = patch.pi ?? {}; + const { tmdbApiKey: _tmdbApiKey, ...metadata } = patch.metadata ?? {}; return { automation: { ...current.automation, ...patch.automation }, cache: { ...current.cache, ...patch.cache }, @@ -145,6 +209,8 @@ export function mergeSettings(current: GlobalSettings, patch: GlobalSettingsUpda retention: { ...current.retention, ...patch.retention }, pi: { ...current.pi, ...pi, compaction: { ...current.pi.compaction, ...compaction } }, budget: { ...current.budget, ...patch.budget }, + agent: { ...current.agent, ...patch.agent }, + metadata: { ...current.metadata, ...metadata }, }; } diff --git a/packages/contracts/test/db-sync.test.ts b/packages/contracts/test/db-sync.test.ts index 30fad541..40b13b3a 100644 --- a/packages/contracts/test/db-sync.test.ts +++ b/packages/contracts/test/db-sync.test.ts @@ -41,6 +41,8 @@ const ROWS: Record AcquisitionSlot: { row: C.AcquisitionSlot }, AgentRun: { row: C.AgentRun }, AgentRunEvent: { row: C.AgentRunEventRow }, + AgentRunMessage: { row: C.AgentRunMessage }, + CapabilityInvocation: { row: C.CapabilityInvocation }, ApiKey: { row: C.ApiKey, dbOnly: ["keyHash"] }, Approval: { row: C.Approval }, Artifact: { row: C.Artifact }, @@ -110,6 +112,9 @@ const ENUMS: Record = { AgentRunTrigger: C.AgentTrigger, AgentRunStatus: C.AgentRunStatus, AgentRunEventType: C.AgentRunEventType, + AgentRunMode: C.AgentRunMode, + AgentRunMessageStatus: C.AgentRunMessageStatus, + CapabilityInvocationStatus: C.CapabilityInvocationStatus, ApprovalStatus: C.ApprovalStatus, ArtifactKind: C.ArtifactKind, AssetLocationLocationKind: C.LocationKind, diff --git a/packages/db/migrations/0021_agent_runtime.ts b/packages/db/migrations/0021_agent_runtime.ts new file mode 100644 index 00000000..96a820e5 --- /dev/null +++ b/packages/db/migrations/0021_agent_runtime.ts @@ -0,0 +1,65 @@ +import type { Kysely } from "kysely"; +import { dropTables, entityTable, execDdl, oneOf } from "../src/migration-helpers.ts"; + +// W2-D Agent 装配(REQ §10.3、§10.6、§10.7、§11): +// - capability_invocation:有副作用能力的操作账本。幂等键由服务端按能力 ID/版本/运行/规范化输入生成(唯一); +// succeeded 行直接返回已记录的输出,running 行(进程在调用中崩溃)按同一幂等键重跑处理器对账—— +// 处理器自身必须幂等(下载提交按计划状态 + qB hash、整理按记录状态)。 +// - agent_run_message:会话追加的用户消息(纠正/补充),runAgent 任务按顺序消费; +// - agent_run 增加 mode(execute | plan_only)与 dedupe_key(后台触发按根因去重)。 + +// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型 +export async function up(db: Kysely): Promise { + await execDdl( + db, + ` +alter table agent_run add column mode text not null default 'execute' ${oneOf("mode", ["execute", "plan_only"])}; +alter table agent_run add column dedupe_key text; +create unique index agent_run_dedupe_key on agent_run (dedupe_key) where dedupe_key is not null; + +${entityTable( + "capability_invocation", + ` + idempotency_key text not null, + capability_id text not null, + capability_version integer not null, + agent_run_id uuid references agent_run (id) on delete set null, + actor_id text not null, + source text not null, + status text not null default 'running' ${oneOf("status", ["running", "succeeded", "failed"])}, + input jsonb not null default '{}'::jsonb, + output jsonb, + error jsonb, + grant_id uuid, + approval_id uuid, + attempts integer not null default 1, + finished_at timestamptz, + constraint capability_invocation_idempotency_key unique (idempotency_key)`, +)} +create index capability_invocation_run_idx on capability_invocation (agent_run_id, created_at) where agent_run_id is not null; +create index capability_invocation_running_idx on capability_invocation (created_at) where status = 'running'; + +${entityTable( + "agent_run_message", + ` + run_id uuid not null references agent_run (id) on delete cascade, + text text not null, + status text not null default 'pending' ${oneOf("status", ["pending", "consumed"])}, + created_by text, + consumed_at timestamptz`, +)} +create index agent_run_message_pending_idx on agent_run_message (run_id, created_at) where status = 'pending'; +`, + ); +} + +// biome-ignore lint/suspicious/noExplicitAny: 同上 +export async function down(db: Kysely): Promise { + await execDdl( + db, + `${dropTables(["agent_run_message", "capability_invocation"])} +drop index if exists agent_run_dedupe_key; +alter table agent_run drop column if exists dedupe_key; +alter table agent_run drop column if exists mode;`, + ); +} diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index 91565a3a..ca2d4d1f 100644 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -37,6 +37,7 @@ export type AgentRunStatus = | "failed" | "cancelled" | "budget_exceeded"; +export type AgentRunMode = "execute" | "plan_only"; export type AgentRunEventType = | "run_started" | "text_delta" @@ -47,6 +48,7 @@ export type AgentRunEventType = | "compaction" | "run_completed" | "run_failed"; +export type AgentRunMessageStatus = "pending" | "consumed"; export type ApprovalStatus = "pending" | "approved" | "rejected" | "revoked" | "expired" | "consumed"; export type ArtifactKind = "evidence" | "plan" | "proposal" | "report" | "snapshot" | "log"; export type AssetLocationLocationKind = "library" | "download_cache" | "hardlink" | "external"; @@ -71,6 +73,7 @@ export type AssetOperationStatus = "planned" | "started" | "succeeded" | "failed export type AssetUnitCoverageCoverage = "full" | "partial" | "segment" | "unknown"; export type AssetUnitCoverageSegment = "full" | "upper" | "middle" | "lower" | "unknown"; export type AutomationGrantStatus = "active" | "revoked" | "expired"; +export type CapabilityInvocationStatus = "running" | "succeeded" | "failed"; export type ContentUnitKind = "season" | "episode" | "special" | "part"; export type ContentUnitContentType = | "main" @@ -305,6 +308,8 @@ export interface AgentRunTable { created_by: string | null; created_at: GeneratedTimestamptz; updated_at: GeneratedTimestamptz; + mode: Generated; + dedupe_key: string | null; } export interface AgentRunEventTable { @@ -315,6 +320,17 @@ export interface AgentRunEventTable { payload: Generated; } +export interface AgentRunMessageTable { + id: Generated; + run_id: string; + text: string; + status: Generated; + created_by: string | null; + consumed_at: Timestamptz | null; + created_at: GeneratedTimestamptz; + updated_at: GeneratedTimestamptz; +} + export interface ApiKeyTable { id: Generated; name: string; @@ -444,6 +460,26 @@ export interface AutomationGrantTable { updated_at: GeneratedTimestamptz; } +export interface CapabilityInvocationTable { + id: Generated; + idempotency_key: string; + capability_id: string; + capability_version: number; + agent_run_id: string | null; + actor_id: string; + source: string; + status: Generated; + input: Generated; + output: Json | null; + error: Json | null; + grant_id: string | null; + approval_id: string | null; + attempts: Generated; + finished_at: Timestamptz | null; + created_at: GeneratedTimestamptz; + updated_at: GeneratedTimestamptz; +} + export interface ContentUnitTable { id: Generated; media_entity_id: string; @@ -1180,6 +1216,7 @@ export interface DB { admin_user: AdminUserTable; agent_run: AgentRunTable; agent_run_event: AgentRunEventTable; + agent_run_message: AgentRunMessageTable; api_key: ApiKeyTable; approval: ApprovalTable; artifact: ArtifactTable; @@ -1188,6 +1225,7 @@ export interface DB { asset_unit_coverage: AssetUnitCoverageTable; audit_event: AuditEventTable; automation_grant: AutomationGrantTable; + capability_invocation: CapabilityInvocationTable; content_unit: ContentUnitTable; decision: DecisionTable; download_task: DownloadTaskTable; @@ -1249,6 +1287,9 @@ export type AgentRunUpdate = Updateable; export type AgentRunEventRow = Selectable; export type NewAgentRunEvent = Insertable; export type AgentRunEventUpdate = Updateable; +export type AgentRunMessageRow = Selectable; +export type NewAgentRunMessage = Insertable; +export type AgentRunMessageUpdate = Updateable; export type ApiKeyRow = Selectable; export type NewApiKey = Insertable; export type ApiKeyUpdate = Updateable; @@ -1273,6 +1314,9 @@ export type AuditEventUpdate = Updateable; export type AutomationGrantRow = Selectable; export type NewAutomationGrant = Insertable; export type AutomationGrantUpdate = Updateable; +export type CapabilityInvocationRow = Selectable; +export type NewCapabilityInvocation = Insertable; +export type CapabilityInvocationUpdate = Updateable; export type ContentUnitRow = Selectable; export type NewContentUnit = Insertable; export type ContentUnitUpdate = Updateable; diff --git a/packages/db/test/schema.test.ts b/packages/db/test/schema.test.ts index ff5da3d3..bd9afaad 100644 --- a/packages/db/test/schema.test.ts +++ b/packages/db/test/schema.test.ts @@ -75,6 +75,9 @@ const EXPECTED_TABLES = [ "identity_redirect", "metadata_field", "field_override", + // W2-D Agent 装配(0021) + "capability_invocation", + "agent_run_message", // 登录门禁(0015)与 SSE 多进程总线(0016) "admin_user", "admin_session",