feat(server): W2-D agent assembly, session API, runAgent/resolveDecision and backend e2e

- Capabilities: all 14 registered (AO: inventory.query, download.plan/submit, organize.plan/execute, policy.preview/apply; agent: task.inspect/retry/cancel); startup asserts count == 14. Registry injects server-side authorize (automation_grant by action/scope/policy version/budget, one-shot approval bound to plan id + payload hash + revision + expiry; model claims ignored; automation switch), audit_event, server idempotency keys and a capability_invocation op ledger (migration 0021) used for crash reconciliation. download.plan creates draft plans only; download.submit promotes draft -> reserved after authorization. - Agent session API: POST/GET /agent/runs, GET /agent/runs/{id} (events by seq, messages, report), POST .../messages (wake), POST .../abort; automation-grants, approvals, capabilities invoke, tasks. - runAgent task: Pi or fake provider from settings (apiKey via secrets), events persisted to agent_run_event then published, resume + ledger reconciliation when pendingOperations > 0, waits for goal-linked downloads/imports and reports from business tables (submitted != imported). - Background triggers: decision.created -> bounded plan_only investigation (dedupe per decision); decision.updated(resolved) -> resolveDecision executes chosen option. - W2 closure: all OpenAPI routes registered in contracts api.ts (+ subset test); TMDB key from settings secret_ref with env fallback; PUT /sites/{id}/credentials documented as alias of /site-accounts; completion events documented as existing status events. - e2e (real PG + OpenWorkflow + testkit mocks): RSS->library, agent delegation with grant/deny, background continuation, worker kill -9 recovery (single qB add), auto investigation dedupe. Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-09-27 20:52:12 +08:00
1 parent 0dec0854df
commit 6280d52ec3
@@ -0,0 +1,142 @@
+import { type ChildProcess, spawn } from "node:child_process";
+import { fileURLToPath } from "node:url";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { torrentFor } from "./fixtures/modules.ts";
+import { crashScript } from "./fixtures/scripts.ts";
+import {
+  createE2E,
+  type E2E,
+  PG_URL,
+  pgAvailable,
+  runEvents,
+  runRow,
+  seedBase,
+  seedMovieTarget,
+  waitFor,
+} from "./harness.ts";
+
+// E4 崩溃恢复:worker 子进程在 Agent 调用 download.submit 时被 kill -9;新 worker 在租约到期(约 30s)后接手:
+// runAgent 的 drive 步骤重跑 → 打开同一 JSONL 会话(pendingOperations>0)→ 按幂等键对账账本里停在 running 的调用
+// → resume(模型拿到"结果未知")→ 同参数再调 download.submit 得到账本结果;mock qB 只收到一次 add。
+
+const hasPg = await pgAvailable();
+const WORKER = fileURLToPath(new URL("./fixtures/worker.ts", import.meta.url));
+const SERVER_DIR = fileURLToPath(new URL("../..", import.meta.url));
+const TITLE = "Dune.Part.Two.2024.1080p.BluRay.x264-GRP";
+
+describe.skipIf(!hasPg)("E4 崩溃恢复(worker 子进程 kill -9)", () => {
+  let e: E2E;
+  const children: ChildProcess[] = [];
+
+  const spawnWorker = (): Promise<ChildProcess> => {
+    const child = spawn(process.execPath, ["run", "-A", WORKER], {
+      cwd: SERVER_DIR,
+      env: {
+        ...process.env,
+        DATABASE_URL: PG_URL,
+        BIZ_SCHEMA: e.biz,
+        OW_SCHEMA: e.ow,
+        SESSION_DIR: e.sessionDir,
+      },
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+    children.push(child);
+    let err = "";
+    child.stderr!.on("data", (d) => {
+      err += d;
+    });
+    return new Promise((resolve, reject) => {
+      const timer = setTimeout(() => reject(new Error(`worker 启动超时: ${err}`)), 60_000);
+      child.stdout!.on("data", (d) => {
+        if (String(d).includes("worker ready")) {
+          clearTimeout(timer);
+          resolve(child);
+        }
+      });
+      child.on("exit", (code) => {
+        clearTimeout(timer);
+        reject(new Error(`worker 提前退出 code=${code}: ${err}`));
+      });
+    });
+  };
+
+  beforeAll(async () => {
+    e = await createE2E("e4", { worker: false, fakeScripts: [crashScript] });
+    const base = await seedBase(e);
+    await seedMovieTarget(e, base.siteId);
+    await e.db
+      .insertInto("automation_grant")
+      .values({ action: "download.submit", scope: JSON.stringify({}), granted_by: "user:e2e" })
+      .execute();
+  }, 120_000);
+
+  afterAll(async () => {
+    for (const c of children) if (c.exitCode === null && c.signalCode === null) c.kill("SIGKILL");
+    await e?.close();
+  }, 60_000);
+
+  it("download.submit 途中 kill worker → 新 worker resume,不重复提交,最终入库", async () => {
+    const w1 = await spawnWorker();
+    const r = await e.api("POST", "/agent/runs", { goal: "把《沙丘2》1080p 收进来", mode: "execute" });
+    expect(r.status).toBe(201);
+    const runId = r.body.data.id as string;
+
+    // Agent 调到 download.submit(账本行出现)的瞬间 kill -9
+    await waitFor(
+      async () =>
+        e.db
+          .selectFrom("capability_invocation")
+          .select("id")
+          .where("agent_run_id", "=", runId)
+          .where("capability_id", "=", "download.submit")
+          .executeTakeFirst(),
+      60_000,
+      "download.submit 开始",
+      10,
+    );
+    w1.kill("SIGKILL");
+    await new Promise((res) => w1.once("exit", res));
+    const before = await runRow(e, runId);
+    expect(["running", "queued"]).toContain(before.status);
+
+    await spawnWorker();
+    // 新 worker 接手(租约约 30s 到期)后提交进入 qB,外部下载器完成
+    const hash = torrentFor(TITLE).hash;
+    await waitFor(async () => e.qb.get(hash), 120_000, "qB 收到种子", 500);
+    await e.qb.complete(hash, { writeFiles: true });
+    const done = await waitFor(
+      async () => {
+        const x = await runRow(e, runId);
+        return x.status === "succeeded" ? x : null;
+      },
+      120_000,
+      "run succeeded",
+      500,
+    );
+    expect(done.summary).toContain("已入库 1 个");
+
+    // 不重复提交:qB 只收到一次 add;download.submit 账本只有一条(同幂等键)
+    expect(e.qb.callsTo("torrents/add")).toHaveLength(1);
+    const inv = await e.db
+      .selectFrom("capability_invocation")
+      .selectAll()
+      .where("agent_run_id", "=", runId)
+      .where("capability_id", "=", "download.submit")
+      .execute();
+    expect(inv).toHaveLength(1);
+    expect(inv[0]!.status).toBe("succeeded");
+    expect(await e.db.selectFrom("download_task").select("id").execute()).toHaveLength(1);
+    // 事件:续跑标记 + 恢复的工具调用
+    const events = await runEvents(e, runId);
+    expect(events.some((x) => x.type === "run_started" && (x.payload as { resumed: boolean }).resumed)).toBe(true);
+    const recovered = events.filter((x) => x.type === "tool_call_end" && (x.payload as { recovery: boolean }).recovery);
+    expect(recovered.map((x) => (x.payload as { capabilityId: string }).capabilityId)).toContain("download.submit");
+    // (run_id, seq) 连续不冲突
+    const seqs = events.map((x) => Number(x.seq));
+    expect(new Set(seqs).size).toBe(seqs.length);
+    const starts = events
+      .filter((x) => x.type === "tool_call_start" && !(x.payload as { recovery: boolean }).recovery)
+      .map((x) => (x.payload as { capabilityId: string }).capabilityId);
+    expect(starts.slice(-2)).toEqual(["download.submit", "task.inspect"]);
+  }, 300_000);
+});