UHD_Media_Tool/apps/server/test/e2e/agent-prompt-eval.real.test.ts

import { readFileSync, writeFileSync } from "node:fs";
import { lintReport } from "@umt/agent";
import { CAPABILITY_IDS } from "@umt/contracts";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import type { RunPrompt } from "../../src/modules/agent/runner.ts";
import { setDefaultUpstream } from "../../src/modules/pi/store.ts";
import { evalCases, seedEvalWorld } from "./fixtures/agent-eval.ts";
import { createE2E, type E2E, pgAvailable, runEvents, runRow, waitFor } from "./harness.ts";

// E16 真实模型评测(AG-5,手动跑,不进 CI):同一套测试库数据与提示,分别用旧提示(main 的 system-prompt.ts)
// 和分层提示跑真实模型,记录能力调用、汇报原文与 lintReport 结果,供 docs/design/agent-prompt-eval.md 对比。
// 智能体只接触测试库数据与 mock qB / mock TMDB;站点地址是本机关闭的端口,不会请求真实站点。
//
// 运行(凭证只经环境变量传入,不写进代码与报告):
//   UMT_EVAL_REAL=1 UMT_EVAL_API_KEY=… UMT_EVAL_PROVIDER=opencode-go UMT_EVAL_MODEL=deepseek-v4.1-flash \
//   UMT_EVAL_LEGACY_PROMPT=/tmp/legacy-system-prompt.ts UMT_EVAL_CASES=health,acquire,why UMT_EVAL_OUT=/tmp/eval.json \
//   DATABASE_URL=… deno run -A npm:vitest run test/e2e/agent-prompt-eval.real.test.ts
// UMT_EVAL_LEGACY_PROMPT 用 `git show main:packages/agent/src/system-prompt.ts > /tmp/legacy-system-prompt.ts` 生成;
// UMT_EVAL_PROMPTS=legacy,layered 控制跑哪几组(缺省两组都跑)。

// biome-ignore lint/suspicious/noExplicitAny: 评测按输出结构取值
type Any = any;

const enabled = process.env.UMT_EVAL_REAL === "1" && (await pgAvailable());

function apiKey(): string {
  const direct = process.env.UMT_EVAL_API_KEY;
  if (direct) return direct.trim();
  const file = process.env.UMT_EVAL_API_KEY_FILE;
  if (!file) throw new Error("缺少 UMT_EVAL_API_KEY 或 UMT_EVAL_API_KEY_FILE");
  // 凭证文件:取带 opencode 字样那一行里最长的 token(不回显内容)
  const line = readFileSync(file, "utf8")
    .split("\n")
    .find((l) => /opencode/i.test(l));
  const token = line?.match(/[A-Za-z0-9_\-.]{24,}/g)?.sort((a, b) => b.length - a.length)[0];
  if (!token) throw new Error("凭证文件里没找到 OpenCode 的 key");
  return token;
}

interface EvalRecord {
  prompt: "legacy" | "layered";
  case: string;
  goal: string;
  status: string;
  calls: { capability: string; ok: boolean }[];
  finalText: string;
  lint: string[];
  lines: number;
  usage: unknown;
  systemPromptChars: number;
}

describe.skipIf(!enabled)("E16 真实模型评测", () => {
  let e: E2E;
  const legacyPath = process.env.UMT_EVAL_LEGACY_PROMPT;
  const which = (process.env.UMT_EVAL_PROMPTS ?? "legacy,layered").split(",") as ("legacy" | "layered")[];
  const caseIds = (process.env.UMT_EVAL_CASES ?? "health,acquire,why").split(",");
  const records: EvalRecord[] = [];
  let current: "legacy" | "layered" = "layered";
  const sizes = new Map<string, number>();
  // biome-ignore lint/suspicious/noExplicitAny: 旧提示模块按路径动态导入
  let legacy: any = null;

  beforeAll(async () => {
    if (which.includes("legacy")) {
      if (!legacyPath) throw new Error("跑旧提示需要 UMT_EVAL_LEGACY_PROMPT");
      legacy = await import(legacyPath);
    }
    e = await createE2E("e16real", {
      agent: {
        systemPrompt: (p: RunPrompt, run, capabilities) => {
          const text = current === "legacy" ? legacyPrompt(run.goal, capabilities) : p.systemPrompt;
          sizes.set(`${current}:${run.goal}`, text.length);
          return text;
        },
      },
    });
    await seedEvalWorld(e);
    const ref = await e.booted.ctx.secrets.create({ purpose: "llm_api_key", plaintext: apiKey() });
    const up = await e.db
      .insertInto("pi_upstream")
      .values({
        name: "eval",
        provider: process.env.UMT_EVAL_PROVIDER ?? "opencode-go",
        auth_kind: "api_key",
        secret_ref: ref,
        default_model: process.env.UMT_EVAL_MODEL ?? "deepseek-v4.1-flash",
        thinking_level: "low",
        extra: JSON.stringify({}),
        enabled: true,
        status: "ok",
      })
      .returning("id")
      .executeTakeFirstOrThrow();
    await setDefaultUpstream(e.booted.ctx, up.id);
    await e.settings({
      agent: { maxToolCalls: 16, maxDurationSec: 240, optionsTimeoutSec: 20, optionsTimeoutAction: "stop" },
    });
    await waitFor(
      async () =>
        (await e.db.selectFrom("downloader_instance").select("health_status").executeTakeFirst())?.health_status ===
        "healthy",
      20_000,
      "下载器同步",
    );
  }, 180_000);

  afterAll(async () => {
    if (process.env.UMT_EVAL_OUT) writeFileSync(process.env.UMT_EVAL_OUT, JSON.stringify(records, null, 2));
    await e?.close();
  }, 60_000);

  /** main 上 runner.systemPromptFor 的旧装配(旧 buildSystemPrompt + 旧约束) */
  function legacyPrompt(goal: string, capabilities: string[]): string {
    return legacy.buildSystemPrompt({
      role: "媒体库自动化智能体",
      goal,
      constraints: [
        `可用能力：${capabilities.join(", ")}`,
        "有副作用的能力（提交下载、执行整理、发布策略、重试/取消任务）需要服务端授权；被拒时说明缺少哪项授权。",
      ],
      audience: "user",
      interaction: {
        plan: capabilities.includes("agent.plan"),
        options: capabilities.includes("agent.propose_options"),
        proposeFirst: false,
        approvals: false,
      },
    });
  }

  it("跑固定提示并记录", async () => {
    const cases = evalCases({ siteId: "", instanceId: "", entityId: "", slot4k: "", r2160: "" }).filter((c) =>
      caseIds.includes(c.id),
    );
    for (const p of which) {
      for (const c of cases) {
        current = p;
        // 旧提示对应 main 的能力集合(没有 download.list)
        const allow = p === "legacy" ? CAPABILITY_IDS.filter((id) => id !== "download.list") : undefined;
        const r = await e.api("POST", "/agent/runs", {
          goal: c.goal,
          ...(allow ? { capabilityAllowlist: allow } : {}),
        });
        expect(r.status).toBe(201);
        const runId = r.body.data.id as string;
        const run = await waitFor(
          async () => {
            const x = await runRow(e, runId);
            return ["succeeded", "failed", "waiting", "budget_exceeded", "cancelled"].includes(x.status) ? x : null;
          },
          300_000,
          `run 结束:${p}:${c.goal}`,
          1000,
        );
        const events = await runEvents(e, runId);
        const ends = new Map(
          events
            .filter((x) => x.type === "tool_call_end")
            .map((x) => [(x.payload as Any).toolCallId, !(x.payload as Any).isError]),
        );
        const calls = events
          .filter((x) => x.type === "tool_call_start")
          .map((x) => ({
            capability: (x.payload as Any).capabilityId as string,
            ok: ends.get((x.payload as Any).toolCallId) ?? false,
          }));
        const done = [...events].reverse().find((x) => x.type === "run_completed");
        const finalText = ((done?.payload as Any)?.finalText as string | undefined) ?? run.summary ?? "";
        records.push({
          prompt: p,
          case: c.id,
          goal: c.goal,
          status: run.status,
          calls,
          finalText,
          lint: lintReport(finalText).map((i) => `${i.rule}:${i.detail}`),
          lines: finalText.split("\n").filter((l) => l.trim()).length,
          usage: run.usage,
          systemPromptChars: sizes.get(`${p}:${c.goal}`) ?? 0,
        });
      }
    }
    expect(records.length).toBe(which.length * cases.length);
  }, 1_800_000);
});