UHD_Media_Tool/apps/server/test/e2e/agent-prompt-eval.test.ts

import { type FakeScript, type FakeToolResult, lintReport } from "@umt/agent";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import type { RunPrompt } from "../../src/modules/agent/runner.ts";
import { type EvalCase, type EvalWorld, evalCases, evalScripts, seedEvalWorld } from "./fixtures/agent-eval.ts";
import { createE2E, type E2E, pgAvailable, runEvents, runRow, waitFor } from "./harness.ts";

// E16 智能体提示评测(AG-5,fake 模型 + 真实能力 + 测试库数据):
// - 固定提示集(8 条,含反例"检查站点、下载器和媒体服务器是否正常"):按意图载入的 Skill、当前环境层的事实;
// - fake 模型按"合格模型"的路线调用能力:断言调用顺序、只读调用都成功、需要批准的被拒绝(不向用户要 ID:
//   download.inspect 的 ID 来自 download.list);
// - 汇报格式约束:合格汇报(只用工具结果里的数据写)通过 lintReport(无 UUID / 原始字节 / 内部字段,≤8 行,结论先行)。
// 真实模型的前后对比见 agent-prompt-eval.real.test.ts 与 docs/design/agent-prompt-eval.md。

const hasPg = await pgAvailable();

describe.skipIf(!hasPg)("E16 智能体提示评测(fake 模型)", () => {
  const scripts: FakeScript[] = [];
  const prompts = new Map<string, RunPrompt>();
  const done = new Map<string, { results: FakeToolResult[]; text: string }>();
  let e: E2E;
  let world: EvalWorld;
  let cases: EvalCase[];

  beforeAll(async () => {
    e = await createE2E("e16", {
      fakeScripts: scripts,
      agent: {
        systemPrompt: (p, run) => {
          prompts.set(run.goal, p);
          return p.systemPrompt;
        },
      },
    });
    world = await seedEvalWorld(e);
    cases = evalCases(world);
    scripts.push(...evalScripts(cases, (id, results, text) => done.set(id, { results, text })));
    // 等下载同步把 mock qB 的状态写回(下载器连通状态、任务状态)
    await waitFor(
      async () =>
        (await e.db.selectFrom("downloader_instance").select("health_status").executeTakeFirst())?.health_status ===
        "healthy",
      20_000,
      "下载器同步",
    );
  }, 120_000);

  afterAll(async () => {
    await e?.close();
  }, 60_000);

  const runToEnd = async (goal: string) => {
    const r = await e.api("POST", "/agent/runs", { goal });
    expect(r.status).toBe(201);
    const runId = r.body.data.id as string;
    const run = await waitFor(
      async () => {
        const x = await runRow(e, runId);
        return ["succeeded", "failed", "waiting", "budget_exceeded", "cancelled"].includes(x.status) ? x : null;
      },
      60_000,
      `run 结束:${goal}`,
    );
    return { runId, run };
  };

  it("固定提示集至少 8 条,含反例那条", () => {
    expect(cases.length).toBeGreaterThanOrEqual(8);
    expect(cases.map((c) => c.goal)).toContain("检查站点、下载器和媒体服务器是否正常");
  });

  it.each(["health", "acquire", "why", "subs", "failed", "storage", "decisions", "site-login"])(
    "%s:Skill、环境层、调用顺序与汇报格式",
    async (id) => {
      const c = cases.find((x) => x.id === id)!;
      const { runId, run } = await runToEnd(c.goal);
      expect(["succeeded", "waiting"], `${run.status}: ${run.summary}`).toContain(run.status);

      // 系统提示:按意图载入的 Skill + 当前环境
      const p = prompts.get(c.goal)!;
      expect(p.kind).toBe("chat");
      expect(p.skills[0]?.name ?? null).toBe(c.skill);
      expect(p.systemPrompt).toContain("下载器 1 个：「家里的 qB」qBittorrent");
      expect(p.systemPrompt).toContain("媒体服务器：还没有接入");
      expect(p.systemPrompt).toContain("「憨憨」PT 站，");
      expect(p.systemPrompt).not.toMatch(/127\.0\.0\.1|hhanclub/);

      // 调用顺序:按事件流里的 tool_call_start
      const events = await runEvents(e, runId);
      const calls = events
        .filter((x) => x.type === "tool_call_start")
        .map((x) => (x.payload as { capabilityId?: string }).capabilityId);
      expect(calls).toEqual(c.calls);
      // 巡检那条:需要批准的检测实际调用一次(界面才有审批卡 /「批准并重试」),汇报里写「没做：…，需要你批准」
      if (id === "health") {
        expect(calls.filter((x) => x === "site.health_check")).toHaveLength(1);
        expect(done.get(id)!.text).toMatch(/没做：.*需要你批准/);
      }

      // 只读调用都成功;需要批准的被拒绝
      const r = done.get(id)!;
      for (const x of r.results) {
        const denied = c.denied?.includes(x.capabilityId ?? "");
        expect(x.isError, `${x.capabilityId}: ${x.text.slice(0, 200)}`).toBe(!!denied);
        if (denied) expect(x.text).toMatch(/unauthorized|授权|批准/);
      }

      // 汇报格式
      expect(lintReport(r.text), r.text).toEqual([]);
    },
    90_000,
  );

  it("download.list:人话大小 / 进度 / 相对时间,不含 hash 与原始字节数", async () => {
    const r = done.get("storage")!.results.find((x) => x.capabilityId === "download.list")!;
    expect(r.json).toMatchObject({
      total: 1,
      byStatus: [{ status: "imported", label: "已入库", count: 1 }],
      items: [{ name: "Poor.Things.2023.2160p.WEB-DL", size: "18.6 GB", progress: "100%", completed: "12 天前" }],
    });
    expect(r.text).not.toMatch(/c3c3c3|20000000000/);
  });

  it("巡检会话固定载入「巡检」Skill,汇报层要求按巡检模板写", async () => {
    const { buildRunPrompt, scopeFor } = await import("../../src/modules/agent/runner.ts");
    const row = {
      ...(await e.db.selectFrom("agent_run").selectAll().limit(1).executeTakeFirstOrThrow()),
      trigger: "schedule" as const,
      goal: "巡检待处理(2026-09-30)：有 1 条未关闭的待处理事项。",
      mode: "plan_only" as const,
    };
    const scope = scopeFor("plan_only", ["decision.list", "downloader.list", "site.list"]);
    const p = await buildRunPrompt({ ctx: e.booted.ctx, opts: {} }, row, scope, {
      canSpawn: false,
      maxConcurrent: 1,
      maxDepth: 1,
    });
    expect(p.kind).toBe("patrol");
    expect(p.skills[0]?.name).toBe("patrol");
    expect(p.systemPrompt).toContain("### Skill「巡检」");
    expect(p.systemPrompt).toContain("按 Skill「巡检」的汇报模板写");
    expect(p.systemPrompt).toContain("本次只读");
  });

  it("追加消息换了意图:新命中的 Skill 用 Pi 的 skill 块附在消息前,已载入的不重复", async () => {
    const { followUpWithSkills } = await import("../../src/modules/agent/runner.ts");
    const p = prompts.get("检查站点、下载器和媒体服务器是否正常")!;
    const copy = { ...p, skills: [...p.skills] };
    const msg = followUpWithSkills(copy, "那订阅追更情况呢");
    expect(msg).toMatch(/^<skill name="subscription-check"/);
    expect(followUpWithSkills(copy, "订阅再看一遍")).toBe("订阅再看一遍");
    expect(followUpWithSkills(copy, "站点正常吗")).toBe("站点正常吗");
  });
});