test: 测试按行为命名、server 测试统一到 test/、PG 测试分组并行
- server 的 src/modules/*/__tests__ 挪到 test/<模块>/;e2e 去掉 eN- 前缀, w5a2 / v2 / search2 / *-dh1 / *-md1 / *-md2、web 的 agui3 / search2、domain 的 id2 / id3 改为按行为命名 - vitest 分三组:unit、pg(真实 PG + 假引擎,文件并行,最多 6 个)、serial(真实引擎 / worker / e2e,串行); pg 组用例超时放宽到 60s(迁移类用例并行时排队等 schema 锁)。server 测试 615s → 约 350s - CLAUDE.md 补测试约定;待办补 harness 去重与 continuous-series 偶发超时(疑似 Slot 评估竞争) - ADR-004 补充:维持 Deno,记录实际只用到 --allow-run 白名单 Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
b343dddbf6
commit
b61335eaf6
75 files changed
+159
-115
No files matched your search
@@ -0,0 +1,186 @@
|
||||
import { readFileSync, writeFileSync } from "node:fs";
|
||||
import { lintReport } from "@umt/agent";
|
||||
import { CAPABILITY_IDS } from "@umt/contracts";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import type { RunPrompt } from "../../src/modules/agent/runner.ts";
|
||||
import { setDefaultUpstream } from "../../src/modules/pi/store.ts";
|
||||
import { evalCases, seedEvalWorld } from "./fixtures/agent-eval.ts";
|
||||
import { createE2E, type E2E, pgAvailable, runEvents, runRow, waitFor } from "./harness.ts";
|
||||
|
||||
// E16 真实模型评测(AG-5,手动跑,不进 CI):同一套测试库数据与提示,分别用旧提示(main 的 system-prompt.ts)
|
||||
// 和分层提示跑真实模型,记录能力调用、汇报原文与 lintReport 结果,供 docs/design/agent-prompt-eval.md 对比。
|
||||
// 智能体只接触测试库数据与 mock qB / mock TMDB;站点地址是本机关闭的端口,不会请求真实站点。
|
||||
//
|
||||
// 运行(凭证只经环境变量传入,不写进代码与报告):
|
||||
// UMT_EVAL_REAL=1 UMT_EVAL_API_KEY=… UMT_EVAL_PROVIDER=opencode-go UMT_EVAL_MODEL=deepseek-v4.1-flash \
|
||||
// UMT_EVAL_LEGACY_PROMPT=/tmp/legacy-system-prompt.ts UMT_EVAL_CASES=health,acquire,why UMT_EVAL_OUT=/tmp/eval.json \
|
||||
// DATABASE_URL=… deno run -A npm:vitest run test/e2e/agent-prompt-eval.real.test.ts
|
||||
// UMT_EVAL_LEGACY_PROMPT 用 `git show main:packages/agent/src/system-prompt.ts > /tmp/legacy-system-prompt.ts` 生成;
|
||||
// UMT_EVAL_PROMPTS=legacy,layered 控制跑哪几组(缺省两组都跑)。
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 评测按输出结构取值
|
||||
type Any = any;
|
||||
|
||||
const enabled = process.env.UMT_EVAL_REAL === "1" && (await pgAvailable());
|
||||
|
||||
function apiKey(): string {
|
||||
const direct = process.env.UMT_EVAL_API_KEY;
|
||||
if (direct) return direct.trim();
|
||||
const file = process.env.UMT_EVAL_API_KEY_FILE;
|
||||
if (!file) throw new Error("缺少 UMT_EVAL_API_KEY 或 UMT_EVAL_API_KEY_FILE");
|
||||
// 凭证文件:取带 opencode 字样那一行里最长的 token(不回显内容)
|
||||
const line = readFileSync(file, "utf8")
|
||||
.split("\n")
|
||||
.find((l) => /opencode/i.test(l));
|
||||
const token = line?.match(/[A-Za-z0-9_\-.]{24,}/g)?.sort((a, b) => b.length - a.length)[0];
|
||||
if (!token) throw new Error("凭证文件里没找到 OpenCode 的 key");
|
||||
return token;
|
||||
}
|
||||
|
||||
interface EvalRecord {
|
||||
prompt: "legacy" | "layered";
|
||||
case: string;
|
||||
goal: string;
|
||||
status: string;
|
||||
calls: { capability: string; ok: boolean }[];
|
||||
finalText: string;
|
||||
lint: string[];
|
||||
lines: number;
|
||||
usage: unknown;
|
||||
systemPromptChars: number;
|
||||
}
|
||||
|
||||
describe.skipIf(!enabled)("E16 真实模型评测", () => {
|
||||
let e: E2E;
|
||||
const legacyPath = process.env.UMT_EVAL_LEGACY_PROMPT;
|
||||
const which = (process.env.UMT_EVAL_PROMPTS ?? "legacy,layered").split(",") as ("legacy" | "layered")[];
|
||||
const caseIds = (process.env.UMT_EVAL_CASES ?? "health,acquire,why").split(",");
|
||||
const records: EvalRecord[] = [];
|
||||
let current: "legacy" | "layered" = "layered";
|
||||
const sizes = new Map<string, number>();
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 旧提示模块按路径动态导入
|
||||
let legacy: any = null;
|
||||
|
||||
beforeAll(async () => {
|
||||
if (which.includes("legacy")) {
|
||||
if (!legacyPath) throw new Error("跑旧提示需要 UMT_EVAL_LEGACY_PROMPT");
|
||||
legacy = await import(legacyPath);
|
||||
}
|
||||
e = await createE2E("e16real", {
|
||||
agent: {
|
||||
systemPrompt: (p: RunPrompt, run, capabilities) => {
|
||||
const text = current === "legacy" ? legacyPrompt(run.goal, capabilities) : p.systemPrompt;
|
||||
sizes.set(`${current}:${run.goal}`, text.length);
|
||||
return text;
|
||||
},
|
||||
},
|
||||
});
|
||||
await seedEvalWorld(e);
|
||||
const ref = await e.booted.ctx.secrets.create({ purpose: "llm_api_key", plaintext: apiKey() });
|
||||
const up = await e.db
|
||||
.insertInto("pi_upstream")
|
||||
.values({
|
||||
name: "eval",
|
||||
provider: process.env.UMT_EVAL_PROVIDER ?? "opencode-go",
|
||||
auth_kind: "api_key",
|
||||
secret_ref: ref,
|
||||
default_model: process.env.UMT_EVAL_MODEL ?? "deepseek-v4.1-flash",
|
||||
thinking_level: "low",
|
||||
extra: JSON.stringify({}),
|
||||
enabled: true,
|
||||
status: "ok",
|
||||
})
|
||||
.returning("id")
|
||||
.executeTakeFirstOrThrow();
|
||||
await setDefaultUpstream(e.booted.ctx, up.id);
|
||||
await e.settings({
|
||||
agent: { maxToolCalls: 16, maxDurationSec: 240, optionsTimeoutSec: 20, optionsTimeoutAction: "stop" },
|
||||
});
|
||||
await waitFor(
|
||||
async () =>
|
||||
(await e.db.selectFrom("downloader_instance").select("health_status").executeTakeFirst())?.health_status ===
|
||||
"healthy",
|
||||
20_000,
|
||||
"下载器同步",
|
||||
);
|
||||
}, 180_000);
|
||||
|
||||
afterAll(async () => {
|
||||
if (process.env.UMT_EVAL_OUT) writeFileSync(process.env.UMT_EVAL_OUT, JSON.stringify(records, null, 2));
|
||||
await e?.close();
|
||||
}, 60_000);
|
||||
|
||||
/** main 上 runner.systemPromptFor 的旧装配(旧 buildSystemPrompt + 旧约束) */
|
||||
function legacyPrompt(goal: string, capabilities: string[]): string {
|
||||
return legacy.buildSystemPrompt({
|
||||
role: "媒体库自动化智能体",
|
||||
goal,
|
||||
constraints: [
|
||||
`可用能力：${capabilities.join(", ")}`,
|
||||
"有副作用的能力（提交下载、执行整理、发布策略、重试/取消任务）需要服务端授权；被拒时说明缺少哪项授权。",
|
||||
],
|
||||
audience: "user",
|
||||
interaction: {
|
||||
plan: capabilities.includes("agent.plan"),
|
||||
options: capabilities.includes("agent.propose_options"),
|
||||
proposeFirst: false,
|
||||
approvals: false,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
it("跑固定提示并记录", async () => {
|
||||
const cases = evalCases({ siteId: "", instanceId: "", entityId: "", slot4k: "", r2160: "" }).filter((c) =>
|
||||
caseIds.includes(c.id),
|
||||
);
|
||||
for (const p of which) {
|
||||
for (const c of cases) {
|
||||
current = p;
|
||||
// 旧提示对应 main 的能力集合(没有 download.list)
|
||||
const allow = p === "legacy" ? CAPABILITY_IDS.filter((id) => id !== "download.list") : undefined;
|
||||
const r = await e.api("POST", "/agent/runs", {
|
||||
goal: c.goal,
|
||||
...(allow ? { capabilityAllowlist: allow } : {}),
|
||||
});
|
||||
expect(r.status).toBe(201);
|
||||
const runId = r.body.data.id as string;
|
||||
const run = await waitFor(
|
||||
async () => {
|
||||
const x = await runRow(e, runId);
|
||||
return ["succeeded", "failed", "waiting", "budget_exceeded", "cancelled"].includes(x.status) ? x : null;
|
||||
},
|
||||
300_000,
|
||||
`run 结束:${p}:${c.goal}`,
|
||||
1000,
|
||||
);
|
||||
const events = await runEvents(e, runId);
|
||||
const ends = new Map(
|
||||
events
|
||||
.filter((x) => x.type === "tool_call_end")
|
||||
.map((x) => [(x.payload as Any).toolCallId, !(x.payload as Any).isError]),
|
||||
);
|
||||
const calls = events
|
||||
.filter((x) => x.type === "tool_call_start")
|
||||
.map((x) => ({
|
||||
capability: (x.payload as Any).capabilityId as string,
|
||||
ok: ends.get((x.payload as Any).toolCallId) ?? false,
|
||||
}));
|
||||
const done = [...events].reverse().find((x) => x.type === "run_completed");
|
||||
const finalText = ((done?.payload as Any)?.finalText as string | undefined) ?? run.summary ?? "";
|
||||
records.push({
|
||||
prompt: p,
|
||||
case: c.id,
|
||||
goal: c.goal,
|
||||
status: run.status,
|
||||
calls,
|
||||
finalText,
|
||||
lint: lintReport(finalText).map((i) => `${i.rule}:${i.detail}`),
|
||||
lines: finalText.split("\n").filter((l) => l.trim()).length,
|
||||
usage: run.usage,
|
||||
systemPromptChars: sizes.get(`${p}:${c.goal}`) ?? 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
expect(records.length).toBe(which.length * cases.length);
|
||||
}, 1_800_000);
|
||||
});
|
||||
Reference in new issue
Block a user