test(agent): prompt eval with fake model and real-model comparison harness

Co-Authored-By: Claude Opus 5.5 <[email protected]>

apps/server/test/e2e/e16-agent-prompt-eval.real.test.ts

@@ -0,0 +1,186 @@
+import { readFileSync, writeFileSync } from "node:fs";
+import { lintReport } from "@umt/agent";
+import { CAPABILITY_IDS } from "@umt/contracts";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import type { RunPrompt } from "../../src/modules/agent/runner.ts";
+import { setDefaultUpstream } from "../../src/modules/pi/store.ts";
+import { evalCases, seedEvalWorld } from "./fixtures/agent-eval.ts";
+import { createE2E, type E2E, pgAvailable, runEvents, runRow, waitFor } from "./harness.ts";
+
+// E16 真实模型评测(AG-5,手动跑,不进 CI):同一套测试库数据与提示,分别用旧提示(main 的 system-prompt.ts)
+// 和分层提示跑真实模型,记录能力调用、汇报原文与 lintReport 结果,供 docs/design/agent-prompt-eval.md 对比。
+// 智能体只接触测试库数据与 mock qB / mock TMDB;站点地址是本机关闭的端口,不会请求真实站点。
+//
+// 运行(凭证只经环境变量传入,不写进代码与报告):
+//   UMT_EVAL_REAL=1 UMT_EVAL_API_KEY=… UMT_EVAL_PROVIDER=opencode-go UMT_EVAL_MODEL=deepseek-v4.1-flash \
+//   UMT_EVAL_LEGACY_PROMPT=/tmp/legacy-system-prompt.ts UMT_EVAL_CASES=health,acquire,why UMT_EVAL_OUT=/tmp/eval.json \
+//   DATABASE_URL=… deno run -A npm:vitest run test/e2e/e16-agent-prompt-eval.real.test.ts
+// UMT_EVAL_LEGACY_PROMPT 用 `git show main:packages/agent/src/system-prompt.ts > /tmp/legacy-system-prompt.ts` 生成;
+// UMT_EVAL_PROMPTS=legacy,layered 控制跑哪几组(缺省两组都跑)。
+
+// biome-ignore lint/suspicious/noExplicitAny: 评测按输出结构取值
+type Any = any;
+
+const enabled = process.env.UMT_EVAL_REAL === "1" && (await pgAvailable());
+
+function apiKey(): string {
+  const direct = process.env.UMT_EVAL_API_KEY;
+  if (direct) return direct.trim();
+  const file = process.env.UMT_EVAL_API_KEY_FILE;
+  if (!file) throw new Error("缺少 UMT_EVAL_API_KEY 或 UMT_EVAL_API_KEY_FILE");
+  // 凭证文件:取带 opencode 字样那一行里最长的 token(不回显内容)
+  const line = readFileSync(file, "utf8")
+    .split("\n")
+    .find((l) => /opencode/i.test(l));
+  const token = line?.match(/[A-Za-z0-9_\-.]{24,}/g)?.sort((a, b) => b.length - a.length)[0];
+  if (!token) throw new Error("凭证文件里没找到 OpenCode 的 key");
+  return token;
+}
+
+interface EvalRecord {
+  prompt: "legacy" | "layered";
+  case: string;
+  goal: string;
+  status: string;
+  calls: { capability: string; ok: boolean }[];
+  finalText: string;
+  lint: string[];
+  lines: number;
+  usage: unknown;
+  systemPromptChars: number;
+}
+
+describe.skipIf(!enabled)("E16 真实模型评测", () => {
+  let e: E2E;
+  const legacyPath = process.env.UMT_EVAL_LEGACY_PROMPT;
+  const which = (process.env.UMT_EVAL_PROMPTS ?? "legacy,layered").split(",") as ("legacy" | "layered")[];
+  const caseIds = (process.env.UMT_EVAL_CASES ?? "health,acquire,why").split(",");
+  const records: EvalRecord[] = [];
+  let current: "legacy" | "layered" = "layered";
+  const sizes = new Map<string, number>();
+  // biome-ignore lint/suspicious/noExplicitAny: 旧提示模块按路径动态导入
+  let legacy: any = null;
+
+  beforeAll(async () => {
+    if (which.includes("legacy")) {
+      if (!legacyPath) throw new Error("跑旧提示需要 UMT_EVAL_LEGACY_PROMPT");
+      legacy = await import(legacyPath);
+    }
+    e = await createE2E("e16real", {
+      agent: {
+        systemPrompt: (p: RunPrompt, run, capabilities) => {
+          const text = current === "legacy" ? legacyPrompt(run.goal, capabilities) : p.systemPrompt;
+          sizes.set(`${current}:${run.goal}`, text.length);
+          return text;
+        },
+      },
+    });
+    await seedEvalWorld(e);
+    const ref = await e.booted.ctx.secrets.create({ purpose: "llm_api_key", plaintext: apiKey() });
+    const up = await e.db
+      .insertInto("pi_upstream")
+      .values({
+        name: "eval",
+        provider: process.env.UMT_EVAL_PROVIDER ?? "opencode-go",
+        auth_kind: "api_key",
+        secret_ref: ref,
+        default_model: process.env.UMT_EVAL_MODEL ?? "deepseek-v4.1-flash",
+        thinking_level: "low",
+        extra: JSON.stringify({}),
+        enabled: true,
+        status: "ok",
+      })
+      .returning("id")
+      .executeTakeFirstOrThrow();
+    await setDefaultUpstream(e.booted.ctx, up.id);
+    await e.settings({
+      agent: { maxToolCalls: 16, maxDurationSec: 240, optionsTimeoutSec: 20, optionsTimeoutAction: "stop" },
+    });
+    await waitFor(
+      async () =>
+        (await e.db.selectFrom("downloader_instance").select("health_status").executeTakeFirst())?.health_status ===
+        "healthy",
+      20_000,
+      "下载器同步",
+    );
+  }, 180_000);
+
+  afterAll(async () => {
+    if (process.env.UMT_EVAL_OUT) writeFileSync(process.env.UMT_EVAL_OUT, JSON.stringify(records, null, 2));
+    await e?.close();
+  }, 60_000);
+
+  /** main 上 runner.systemPromptFor 的旧装配(旧 buildSystemPrompt + 旧约束) */
+  function legacyPrompt(goal: string, capabilities: string[]): string {
+    return legacy.buildSystemPrompt({
+      role: "媒体库自动化智能体",
+      goal,
+      constraints: [
+        `可用能力：${capabilities.join(", ")}`,
+        "有副作用的能力（提交下载、执行整理、发布策略、重试/取消任务）需要服务端授权；被拒时说明缺少哪项授权。",
+      ],
+      audience: "user",
+      interaction: {
+        plan: capabilities.includes("agent.plan"),
+        options: capabilities.includes("agent.propose_options"),
+        proposeFirst: false,
+        approvals: false,
+      },
+    });
+  }
+
+  it("跑固定提示并记录", async () => {
+    const cases = evalCases({ siteId: "", instanceId: "", entityId: "", slot4k: "", r2160: "" }).filter((c) =>
+      caseIds.includes(c.id),
+    );
+    for (const p of which) {
+      for (const c of cases) {
+        current = p;
+        // 旧提示对应 main 的能力集合(没有 download.list)
+        const allow = p === "legacy" ? CAPABILITY_IDS.filter((id) => id !== "download.list") : undefined;
+        const r = await e.api("POST", "/agent/runs", {
+          goal: c.goal,
+          ...(allow ? { capabilityAllowlist: allow } : {}),
+        });
+        expect(r.status).toBe(201);
+        const runId = r.body.data.id as string;
+        const run = await waitFor(
+          async () => {
+            const x = await runRow(e, runId);
+            return ["succeeded", "failed", "waiting", "budget_exceeded", "cancelled"].includes(x.status) ? x : null;
+          },
+          300_000,
+          `run 结束:${p}:${c.goal}`,
+          1000,
+        );
+        const events = await runEvents(e, runId);
+        const ends = new Map(
+          events
+            .filter((x) => x.type === "tool_call_end")
+            .map((x) => [(x.payload as Any).toolCallId, !(x.payload as Any).isError]),
+        );
+        const calls = events
+          .filter((x) => x.type === "tool_call_start")
+          .map((x) => ({
+            capability: (x.payload as Any).capabilityId as string,
+            ok: ends.get((x.payload as Any).toolCallId) ?? false,
+          }));
+        const done = [...events].reverse().find((x) => x.type === "run_completed");
+        const finalText = ((done?.payload as Any)?.finalText as string | undefined) ?? run.summary ?? "";
+        records.push({
+          prompt: p,
+          case: c.id,
+          goal: c.goal,
+          status: run.status,
+          calls,
+          finalText,
+          lint: lintReport(finalText).map((i) => `${i.rule}:${i.detail}`),
+          lines: finalText.split("\n").filter((l) => l.trim()).length,
+          usage: run.usage,
+          systemPromptChars: sizes.get(`${p}:${c.goal}`) ?? 0,
+        });
+      }
+    }
+    expect(records.length).toBe(which.length * cases.length);
+  }, 1_800_000);
+});

apps/server/test/e2e/e16-agent-prompt-eval.test.ts

@@ -0,0 +1,148 @@
+import { type FakeScript, type FakeToolResult, lintReport } from "@umt/agent";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import type { RunPrompt } from "../../src/modules/agent/runner.ts";
+import { type EvalCase, type EvalWorld, evalCases, evalScripts, seedEvalWorld } from "./fixtures/agent-eval.ts";
+import { createE2E, type E2E, pgAvailable, runEvents, runRow, waitFor } from "./harness.ts";
+
+// E16 智能体提示评测(AG-5,fake 模型 + 真实能力 + 测试库数据):
+// - 固定提示集(8 条,含反例"检查站点、下载器和媒体服务器是否正常"):按意图载入的 Skill、当前环境层的事实;
+// - fake 模型按"合格模型"的路线调用能力:断言调用顺序、只读调用都成功、需要批准的被拒绝(不向用户要 ID:
+//   download.inspect 的 ID 来自 download.list);
+// - 汇报格式约束:合格汇报(只用工具结果里的数据写)通过 lintReport(无 UUID / 原始字节 / 内部字段,≤8 行,结论先行)。
+// 真实模型的前后对比见 e16-agent-prompt-eval.real.test.ts 与 docs/design/agent-prompt-eval.md。
+
+const hasPg = await pgAvailable();
+
+describe.skipIf(!hasPg)("E16 智能体提示评测(fake 模型)", () => {
+  const scripts: FakeScript[] = [];
+  const prompts = new Map<string, RunPrompt>();
+  const done = new Map<string, { results: FakeToolResult[]; text: string }>();
+  let e: E2E;
+  let world: EvalWorld;
+  let cases: EvalCase[];
+
+  beforeAll(async () => {
+    e = await createE2E("e16", {
+      fakeScripts: scripts,
+      agent: {
+        systemPrompt: (p, run) => {
+          prompts.set(run.goal, p);
+          return p.systemPrompt;
+        },
+      },
+    });
+    world = await seedEvalWorld(e);
+    cases = evalCases(world);
+    scripts.push(...evalScripts(cases, (id, results, text) => done.set(id, { results, text })));
+    // 等下载同步把 mock qB 的状态写回(下载器连通状态、任务状态)
+    await waitFor(
+      async () =>
+        (await e.db.selectFrom("downloader_instance").select("health_status").executeTakeFirst())?.health_status ===
+        "healthy",
+      20_000,
+      "下载器同步",
+    );
+  }, 120_000);
+
+  afterAll(async () => {
+    await e?.close();
+  }, 60_000);
+
+  const runToEnd = async (goal: string) => {
+    const r = await e.api("POST", "/agent/runs", { goal });
+    expect(r.status).toBe(201);
+    const runId = r.body.data.id as string;
+    const run = await waitFor(
+      async () => {
+        const x = await runRow(e, runId);
+        return ["succeeded", "failed", "waiting", "budget_exceeded", "cancelled"].includes(x.status) ? x : null;
+      },
+      60_000,
+      `run 结束:${goal}`,
+    );
+    return { runId, run };
+  };
+
+  it("固定提示集至少 8 条,含反例那条", () => {
+    expect(cases.length).toBeGreaterThanOrEqual(8);
+    expect(cases.map((c) => c.goal)).toContain("检查站点、下载器和媒体服务器是否正常");
+  });
+
+  it.each(["health", "acquire", "why", "subs", "failed", "storage", "decisions", "site-login"])(
+    "%s:Skill、环境层、调用顺序与汇报格式",
+    async (id) => {
+      const c = cases.find((x) => x.id === id)!;
+      const { runId, run } = await runToEnd(c.goal);
+      expect(["succeeded", "waiting"], `${run.status}: ${run.summary}`).toContain(run.status);
+
+      // 系统提示:按意图载入的 Skill + 当前环境
+      const p = prompts.get(c.goal)!;
+      expect(p.kind).toBe("chat");
+      expect(p.skills[0]?.name ?? null).toBe(c.skill);
+      expect(p.systemPrompt).toContain("下载器 1 个：「家里的 qB」qBittorrent");
+      expect(p.systemPrompt).toContain("媒体服务器：还没有接入");
+      expect(p.systemPrompt).toContain("「憨憨」PT 站，");
+      expect(p.systemPrompt).not.toMatch(/127\.0\.0\.1|hhanclub/);
+
+      // 调用顺序:按事件流里的 tool_call_start
+      const events = await runEvents(e, runId);
+      const calls = events
+        .filter((x) => x.type === "tool_call_start")
+        .map((x) => (x.payload as { capabilityId?: string }).capabilityId);
+      expect(calls).toEqual(c.calls);
+
+      // 只读调用都成功;需要批准的被拒绝
+      const r = done.get(id)!;
+      for (const x of r.results) {
+        const denied = c.denied?.includes(x.capabilityId ?? "");
+        expect(x.isError, `${x.capabilityId}: ${x.text.slice(0, 200)}`).toBe(!!denied);
+        if (denied) expect(x.text).toMatch(/unauthorized|授权|批准/);
+      }
+
+      // 汇报格式
+      expect(lintReport(r.text), r.text).toEqual([]);
+    },
+    90_000,
+  );
+
+  it("download.list:人话大小 / 进度 / 相对时间,不含 hash 与原始字节数", async () => {
+    const r = done.get("storage")!.results.find((x) => x.capabilityId === "download.list")!;
+    expect(r.json).toMatchObject({
+      total: 1,
+      byStatus: [{ status: "imported", label: "已入库", count: 1 }],
+      items: [{ name: "Poor.Things.2023.2160p.WEB-DL", size: "18.6 GB", progress: "100%", completed: "12 天前" }],
+    });
+    expect(r.text).not.toMatch(/c3c3c3|20000000000/);
+  });
+
+  it("巡检会话固定载入「巡检」Skill,汇报层要求按巡检模板写", async () => {
+    const { buildRunPrompt, scopeFor } = await import("../../src/modules/agent/runner.ts");
+    const row = {
+      ...(await e.db.selectFrom("agent_run").selectAll().limit(1).executeTakeFirstOrThrow()),
+      trigger: "schedule" as const,
+      goal: "巡检待处理(2026-09-30)：有 1 条未关闭的待处理事项。",
+      mode: "plan_only" as const,
+    };
+    const scope = scopeFor("plan_only", ["decision.list", "downloader.list", "site.list"]);
+    const p = await buildRunPrompt({ ctx: e.booted.ctx, opts: {} }, row, scope, {
+      canSpawn: false,
+      maxConcurrent: 1,
+      maxDepth: 1,
+    });
+    expect(p.kind).toBe("patrol");
+    expect(p.skills[0]?.name).toBe("patrol");
+    expect(p.systemPrompt).toContain("### Skill「巡检」");
+    expect(p.systemPrompt).toContain("按 Skill「巡检」的汇报模板写");
+    expect(p.systemPrompt).toContain("本次只读");
+  });
+
+  it("追加消息换了意图:新命中的 Skill 用 Pi 的 skill 块附在消息前,已载入的不重复", async () => {
+    const { followUpWithSkills } = await import("../../src/modules/agent/runner.ts");
+    const p = prompts.get("检查站点、下载器和媒体服务器是否正常")!;
+    const copy = { ...p, skills: [...p.skills] };
+    const msg = followUpWithSkills(copy, "那订阅追更情况呢");
+    expect(msg).toMatch(/^<skill name="subscription-check"/);
+    expect(followUpWithSkills(copy, "订阅再看一遍")).toBe("订阅再看一遍");
+    expect(followUpWithSkills(copy, "站点正常吗")).toBe("站点正常吗");
+  });
+});

apps/server/test/e2e/fixtures/agent-eval.ts

@@ -0,0 +1,363 @@
+import { type FakeScript, type FakeStep, type FakeToolResult, healthName, humanRelative, kindName } from "@umt/agent";
+import { type E2E, seedBase, seedMovieTarget } from "../harness.ts";
+
+// AG-5 智能体提示评测的共用夹具:测试库里的一套"用户环境"(对照 docs/design/agent-report-sample-bad.md 的现场)
+// + 固定提示集。e16-agent-prompt-eval(fake 模型,CI)与 e16-agent-prompt-eval.real(真实模型,手动)共用。
+// 站点地址指向本机关闭的端口:智能体即使实时补查也不会请求真实站点。
+
+// biome-ignore lint/suspicious/noExplicitAny: 评测按输出结构取值
+type Any = any;
+
+export const HASHES = {
+  dune: "a1".repeat(20),
+  failed: "b2".repeat(20),
+  seeding: "c3".repeat(20),
+} as const;
+
+export interface EvalWorld {
+  siteId: string;
+  instanceId: string;
+  entityId: string;
+  slot4k: string;
+  r2160: string;
+}
+
+/** 下载器(mock qB,3 个任务:下载中 / 失败 / 已入库做种)+ 站点憨憨(上次检查正常、账号有用户数据)+ 沙丘2 + 5 条订阅 */
+export async function seedEvalWorld(e: E2E): Promise<EvalWorld> {
+  const base = await seedBase(e, { key: "hhanclub", baseUrl: "http://127.0.0.1:9/" });
+  await e.db
+    .updateTable("site")
+    .set({
+      name: "憨憨",
+      health_status: "healthy",
+      last_health_at: new Date(Date.now() - 20 * 3600_000),
+      last_health: JSON.stringify({ ok: true, status: "ok", latencyMs: 1201, message: null }),
+    })
+    .where("id", "=", base.siteId)
+    .execute();
+  await e.db.updateTable("downloader_instance").set({ name: "家里的 qB" }).where("id", "=", base.instanceId).execute();
+  await e.db
+    .insertInto("site_account")
+    .values({
+      site_id: base.siteId,
+      label: "主账号",
+      auth_kind: "cookie",
+      status: "active",
+      user_data: JSON.stringify({
+        schema: "nexusphp",
+        userId: "1",
+        username: "u",
+        userLevel: "Power User",
+        vip: false,
+        joinAt: null,
+        upload: 17008345370067,
+        download: 0,
+        ratio: null,
+        bonus: 414799,
+        seeding: 24,
+        seedingSize: 79407286843,
+        leeching: 0,
+        leechingSize: 0,
+        messageUnread: 0,
+      }),
+      user_data_at: new Date(Date.now() - 3600_000),
+    })
+    .execute();
+  const target = await seedMovieTarget(e, base.siteId);
+  const slot4k = (
+    await e.db
+      .insertInto("acquisition_slot")
+      .values({
+        media_entity_id: target.entityId,
+        name: "2160p",
+        hard_conditions: JSON.stringify({ resolutions: ["2160p"] }),
+      })
+      .returning("id")
+      .executeTakeFirstOrThrow()
+  ).id;
+  const r2160 = (
+    await e.db
+      .insertInto("source_release")
+      .values({
+        site_id: base.siteId,
+        site_release_id: "889001",
+        title: "Dune.Part.Two.2024.2160p.UHD.BluRay.x265-GRP",
+        size_bytes: 62_490_000_000,
+        media_entity_id: target.entityId,
+        resolution_status: "resolved",
+        raw: JSON.stringify({ _site: { hr: false } }),
+      })
+      .returning("id")
+      .executeTakeFirstOrThrow()
+  ).id;
+  // 4 条已取消的订阅 + seedMovieTarget 的 1 条追更中
+  for (const t of ["繁花", "三体", "漫长的季节", "狂飙"])
+    await e.db
+      .insertInto("subscription")
+      .values({ title_query: t, media_kind: "series", status: "cancelled" })
+      .execute();
+  // 种子先进 mock qB,同步会把它们接管成任务行;之后按本用例需要覆盖状态与关联(upsert,和接管谁先谁后都一样)
+  const qb = [
+    [HASHES.dune, "Dune.Part.Two.2024.1080p.BluRay.x264-GRP", "downloading", 0.63],
+    [HASHES.failed, "Oppenheimer.2023.1080p.BluRay.x264", "missingFiles", 0.4],
+    [HASHES.seeding, "Poor.Things.2023.2160p.WEB-DL", "uploading", 1],
+  ] as const;
+  for (const [hash, name, state, progress] of qb)
+    e.qb.seed({ hash, name, state, progress, size: 20_000_000_000, tags: ["umt"], category: "umt" });
+  const task = async (hash: string, name: string, status: string, progress: number, extra: Record<string, unknown>) => {
+    const values = {
+      instance_id: base.instanceId,
+      torrent_hash: hash,
+      infohash_v1: hash,
+      name,
+      status: status as never,
+      progress,
+      size_bytes: 20_000_000_000,
+      tags: ["umt"],
+      managed: true,
+      origin: "submitted" as const,
+      media_entity_id: target.entityId,
+      last_progress_at: new Date(Date.now() - 5 * 60_000),
+      ...extra,
+    } as never;
+    await e.db
+      .insertInto("download_task")
+      .values(values)
+      .onConflict((oc) => oc.columns(["instance_id", "torrent_hash"]).doUpdateSet(values))
+      .execute();
+  };
+  await task(HASHES.dune, "Dune.Part.Two.2024.1080p.BluRay.x264-GRP", "downloading", 0.63, {
+    source_release_id: target.r1080,
+  });
+  await task(HASHES.failed, "Oppenheimer.2023.1080p.BluRay.x264", "failed", 0.4, {
+    media_entity_id: null,
+    last_error: JSON.stringify({ message: "下载器找不到数据文件" }),
+  });
+  await task(HASHES.seeding, "Poor.Things.2023.2160p.WEB-DL", "imported", 1, {
+    media_entity_id: null,
+    completed_at: new Date(Date.now() - 12 * 86400_000),
+  });
+  await e.db
+    .insertInto("decision")
+    .values({
+      kind: "download_failed",
+      title: "下载异常：Oppenheimer.2023",
+      subject_type: "download_task",
+      subject_id: (
+        await e.db
+          .selectFrom("download_task")
+          .select("id")
+          .where("torrent_hash", "=", HASHES.failed)
+          .executeTakeFirstOrThrow()
+      ).id,
+      dedupe_key: `download_failed:eval`,
+      options: JSON.stringify([{ key: "retry", label: "重试" }]),
+    })
+    .execute();
+  return { siteId: base.siteId, instanceId: base.instanceId, entityId: target.entityId, slot4k, r2160 };
+}
+
+const last = (r: FakeToolResult[], cap: string): Any => [...r].reverse().find((x) => x.capabilityId === cap)?.json;
+const lastResult = (r: FakeToolResult[], cap: string) => [...r].reverse().find((x) => x.capabilityId === cap);
+
+export interface EvalCase {
+  id: string;
+  goal: string;
+  /** 期望按意图载入的 Skill(null = 不载入) */
+  skill: string | null;
+  /** fake 模型的调用脚本(按"合格模型"会走的路线写) */
+  steps: FakeStep[];
+  /** 期望的能力调用顺序 */
+  calls: string[];
+  /** 允许失败(被拒绝)的能力 */
+  denied?: string[];
+  /** 合格模型按工具结果写出的汇报(只用结果里的数据) */
+  report: (r: FakeToolResult[]) => string;
+}
+
+const now = () => new Date();
+
+export function evalCases(w: EvalWorld): EvalCase[] {
+  return [
+    {
+      id: "health",
+      goal: "检查站点、下载器和媒体服务器是否正常",
+      skill: "patrol",
+      steps: [
+        { toolCalls: [{ capability: "downloader.list", input: {} }] },
+        { toolCalls: [{ capability: "site.list", input: {} }] },
+        { toolCalls: [{ capability: "site.health_check", input: { siteId: w.siteId } }] },
+        { toolCalls: [{ capability: "decision.list", input: {} }] },
+      ],
+      calls: ["downloader.list", "site.list", "site.health_check", "decision.list"],
+      denied: ["site.health_check"],
+      report: (r) => {
+        const d = last(r, "downloader.list");
+        const q = d.downloaders[0];
+        const site = last(r, "site.list").items[0];
+        const dec = last(r, "decision.list");
+        const at = humanRelative(site.lastHealthAt, now());
+        return [
+          `下载器「${q.name}」${kindName(q.kind)} ${healthName(q.healthStatus)}，管理 ${q.tasks.total} 个任务，${q.tasks.failed} 个失败；媒体服务器还没接入。`,
+          `站点「${site.name}」上次检查${healthName(site.healthStatus)}（${at}），没做：实时检测，需要你批准。`,
+          `待处理 ${dec.total} 条：${dec.items[0].title}。`,
+        ].join("\n");
+      },
+    },
+    {
+      id: "acquire",
+      goal: "把《沙丘 2》4K 收进来",
+      skill: "acquire-title",
+      steps: [
+        { toolCalls: [{ capability: "catalog.lookup", input: { query: "沙丘 2" } }] },
+        { toolCalls: [{ capability: "inventory.query", input: { mediaEntityId: w.entityId } }] },
+        { toolCalls: [{ capability: "download.list", input: { mediaEntityId: w.entityId } }] },
+        { toolCalls: [{ capability: "release.search", input: { mediaEntityId: w.entityId } }] },
+        {
+          toolCalls: [
+            {
+              capability: "download.plan",
+              input: (r) => ({
+                slotId: w.slot4k,
+                sourceReleaseId: last(r, "release.search").items.find((x: Any) => /2160p/.test(x.title)).id,
+              }),
+            },
+          ],
+        },
+        {
+          toolCalls: [
+            { capability: "download.submit", input: (r) => ({ planId: last(r, "download.plan").plan?.id ?? w.r2160 }) },
+          ],
+        },
+      ],
+      calls: [
+        "catalog.lookup",
+        "inventory.query",
+        "download.list",
+        "release.search",
+        "download.plan",
+        "download.submit",
+      ],
+      denied: ["download.submit"],
+      report: (r) => {
+        const rel = last(r, "release.search").items.find((x: Any) => /2160p/.test(x.title));
+        const gb = (rel.sizeBytes / 1024 ** 3).toFixed(1);
+        const dl = last(r, "download.list").items[0];
+        return [
+          `《沙丘2》4K 找到了，还没提交：${rel.title}（${gb} GB，来自憨憨），没做：提交下载，需要你批准。`,
+          `库里现有 1080p 正在下载（${dl.progress}）。`,
+        ].join("\n");
+      },
+    },
+    {
+      id: "why",
+      goal: "为什么沙丘2没入库",
+      skill: "import-troubleshoot",
+      steps: [
+        { toolCalls: [{ capability: "inventory.query", input: { query: "沙丘2" } }] },
+        { toolCalls: [{ capability: "download.list", input: { query: "沙丘2" } }] },
+        {
+          toolCalls: [
+            {
+              capability: "download.inspect",
+              input: (r) => ({ downloadTaskId: last(r, "download.list").items[0].id }),
+            },
+          ],
+        },
+      ],
+      calls: ["inventory.query", "download.list", "download.inspect"],
+      report: (r) => {
+        const it = last(r, "download.list").items[0];
+        return `《沙丘2》还没入库：还在${it.statusLabel}，进度 ${it.progress}，最近进展 ${it.lastProgress}；下完会自动整理入库。`;
+      },
+    },
+    {
+      id: "subs",
+      goal: "订阅追更情况",
+      skill: "subscription-check",
+      steps: [{ toolCalls: [{ capability: "subscription.list", input: { groupBy: "status" } }] }],
+      calls: ["subscription.list"],
+      report: (r) => {
+        const s = last(r, "subscription.list");
+        const n = (k: string) => s.groups.find((g: Any) => g.key === k)?.count ?? 0;
+        return `${s.total} 个订阅：${n("active")} 个在追更（沙丘2，收录版本还没满足），${n("cancelled")} 个已取消。`;
+      },
+    },
+    {
+      id: "failed",
+      goal: "有没有失败的下载任务",
+      skill: "download-diagnose",
+      steps: [
+        { toolCalls: [{ capability: "download.list", input: { status: ["failed", "missing", "stalled"] } }] },
+        {
+          toolCalls: [
+            {
+              capability: "download.inspect",
+              input: (r) => ({ downloadTaskId: last(r, "download.list").items[0].id }),
+            },
+          ],
+        },
+      ],
+      calls: ["download.list", "download.inspect"],
+      report: (r) => {
+        const it = last(r, "download.list").items[0];
+        return `有 1 个下载失败：${it.name}（${it.statusLabel}，${it.error ?? "原因未确认"}）。其余下载正常。`;
+      },
+    },
+    {
+      id: "storage",
+      goal: "空间不够了，看看能清理什么",
+      skill: "storage-cleanup",
+      steps: [
+        { toolCalls: [{ capability: "downloader.list", input: {} }] },
+        { toolCalls: [{ capability: "download.list", input: { status: ["imported"] } }] },
+      ],
+      calls: ["downloader.list", "download.list"],
+      report: (r) => {
+        const l = last(r, "download.list");
+        const it = l.items[0];
+        return `下载缓存里有 ${l.total} 个已入库还在做种的种子可以清理：${it.name}（${it.size}，${it.completed}完成）。在下载页点「清理缓存」会按保种规则只清可清理的。`;
+      },
+    },
+    {
+      id: "decisions",
+      goal: "待处理里有什么要我处理的",
+      skill: null,
+      steps: [{ toolCalls: [{ capability: "decision.list", input: {} }] }],
+      calls: ["decision.list"],
+      report: (r) => {
+        const d = last(r, "decision.list");
+        return `有 ${d.total} 条待处理：${d.items[0].title}，可以在待处理中心选「重试」。`;
+      },
+    },
+    {
+      id: "site-login",
+      goal: "站点憨憨的登录是不是掉了",
+      skill: "patrol",
+      steps: [{ toolCalls: [{ capability: "site.list", input: {} }] }],
+      calls: ["site.list"],
+      report: (r) => {
+        const s = last(r, "site.list").items[0];
+        return `站点「${s.name}」登录正常，上次检查${healthName(s.healthStatus)}（${humanRelative(s.lastHealthAt, now())}）。`;
+      },
+    },
+  ];
+}
+
+/** 评测用 fake 脚本:按 goal 精确匹配;finalText 用合格模型的汇报,并把工具结果交给 onDone */
+export function evalScripts(cases: EvalCase[], onDone: (id: string, r: FakeToolResult[], text: string) => void) {
+  return cases.map(
+    (c): FakeScript => ({
+      name: `eval:${c.id}`,
+      match: (goal) => goal === c.goal,
+      steps: c.steps,
+      finalText: (r) => {
+        const text = c.report(r);
+        onDone(c.id, r, text);
+        return text;
+      },
+    }),
+  );
+}
+
+export { lastResult };

apps/server/test/e2e/fixtures/modules.ts

@@ -4,6 +4,7 @@ import type { FakeScript } from "@umt/agent";
 import { makeTorrent } from "@umt/testkit";
 import { createAcquisitionOrganizeModule } from "../../../src/modules/acquisition-organize/index.ts";
 import { createAgentModule } from "../../../src/modules/agent/index.ts";
+import type { AgentModuleOptions } from "../../../src/modules/agent/runner.ts";
 import { authSettingsModule } from "../../../src/modules/auth-settings/index.ts";
 import { createBrowserModule } from "../../../src/modules/browser/index.ts";
 import type { BrowserModuleOptions } from "../../../src/modules/browser/service.ts";
@@ -32,6 +33,8 @@ export interface E2EModuleOptions {
   webSearch?: WebSearchModuleOptions;
   /** 浏览器模块注入(E14 视觉自动化:证据目录、动作后等待) */
   browser?: BrowserModuleOptions;
+  /** 智能体模块的额外参数(E16:替换 / 捕获系统提示) */
+  agent?: Pick<AgentModuleOptions, "systemPrompt">;
 }
 
 export function buildModules(o: E2EModuleOptions): ServerModule[] {
@@ -71,6 +74,7 @@ export function buildModules(o: E2EModuleOptions): ServerModule[] {
       sessionDir: o.sessionDir,
       ...(o.fakeScripts ? { fakeScripts: o.fakeScripts } : {}),
       goalPollSec: 1,
+      ...(o.agent ?? {}),
     }),
     // 浏览器能力(browser.*)随首批能力一起注册;浏览器服务由用例自己登记(E14 指向 mock camofox)
     createBrowserModule(o.browser ?? {}),

apps/server/test/e2e/harness.ts

@@ -20,6 +20,7 @@ import { type Bootstrapped, bootstrap } from "../../src/context.ts";
 import { createApp } from "../../src/http/app.ts";
 import { EventBus } from "../../src/http/sse.ts";
 import { createLogger } from "../../src/logger.ts";
+import type { AgentModuleOptions } from "../../src/modules/agent/runner.ts";
 import type { BrowserModuleOptions } from "../../src/modules/browser/service.ts";
 import { setDefaultUpstream } from "../../src/modules/pi/store.ts";
 import type { ServerModule } from "../../src/modules/types.ts";
@@ -84,6 +85,8 @@ export interface E2EOptions {
   webSearch?: WebSearchModuleOptions;
   /** 浏览器模块注入(E14) */
   browser?: BrowserModuleOptions;
+  /** 智能体模块的额外参数(E16:替换 / 捕获系统提示) */
+  agent?: Pick<AgentModuleOptions, "systemPrompt">;
 }
 
 export async function waitFor<T>(
@@ -140,6 +143,7 @@ export async function createE2E(name: string, opts: E2EOptions = {}): Promise<E2
     ...(opts.fakeScripts ? { fakeScripts: opts.fakeScripts } : {}),
     ...(opts.webSearch ? { webSearch: opts.webSearch } : {}),
     ...(opts.browser ? { browser: opts.browser } : {}),
+    ...(opts.agent ? { agent: opts.agent } : {}),
   });
   const booted = await bootstrap({
     role: "all",