test(agent): prompt eval with fake model and real-model comparison harness
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
21b7e8ff23
commit
425415b3ba
5 files changed
+705
No files matched your search
@@ -0,0 +1,363 @@
|
||||
import { type FakeScript, type FakeStep, type FakeToolResult, healthName, humanRelative, kindName } from "@umt/agent";
|
||||
import { type E2E, seedBase, seedMovieTarget } from "../harness.ts";
|
||||
|
||||
// AG-5 智能体提示评测的共用夹具:测试库里的一套"用户环境"(对照 docs/design/agent-report-sample-bad.md 的现场)
|
||||
// + 固定提示集。e16-agent-prompt-eval(fake 模型,CI)与 e16-agent-prompt-eval.real(真实模型,手动)共用。
|
||||
// 站点地址指向本机关闭的端口:智能体即使实时补查也不会请求真实站点。
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 评测按输出结构取值
|
||||
type Any = any;
|
||||
|
||||
export const HASHES = {
|
||||
dune: "a1".repeat(20),
|
||||
failed: "b2".repeat(20),
|
||||
seeding: "c3".repeat(20),
|
||||
} as const;
|
||||
|
||||
export interface EvalWorld {
|
||||
siteId: string;
|
||||
instanceId: string;
|
||||
entityId: string;
|
||||
slot4k: string;
|
||||
r2160: string;
|
||||
}
|
||||
|
||||
/** 下载器(mock qB,3 个任务:下载中 / 失败 / 已入库做种)+ 站点憨憨(上次检查正常、账号有用户数据)+ 沙丘2 + 5 条订阅 */
|
||||
export async function seedEvalWorld(e: E2E): Promise<EvalWorld> {
|
||||
const base = await seedBase(e, { key: "hhanclub", baseUrl: "http://127.0.0.1:9/" });
|
||||
await e.db
|
||||
.updateTable("site")
|
||||
.set({
|
||||
name: "憨憨",
|
||||
health_status: "healthy",
|
||||
last_health_at: new Date(Date.now() - 20 * 3600_000),
|
||||
last_health: JSON.stringify({ ok: true, status: "ok", latencyMs: 1201, message: null }),
|
||||
})
|
||||
.where("id", "=", base.siteId)
|
||||
.execute();
|
||||
await e.db.updateTable("downloader_instance").set({ name: "家里的 qB" }).where("id", "=", base.instanceId).execute();
|
||||
await e.db
|
||||
.insertInto("site_account")
|
||||
.values({
|
||||
site_id: base.siteId,
|
||||
label: "主账号",
|
||||
auth_kind: "cookie",
|
||||
status: "active",
|
||||
user_data: JSON.stringify({
|
||||
schema: "nexusphp",
|
||||
userId: "1",
|
||||
username: "u",
|
||||
userLevel: "Power User",
|
||||
vip: false,
|
||||
joinAt: null,
|
||||
upload: 17008345370067,
|
||||
download: 0,
|
||||
ratio: null,
|
||||
bonus: 414799,
|
||||
seeding: 24,
|
||||
seedingSize: 79407286843,
|
||||
leeching: 0,
|
||||
leechingSize: 0,
|
||||
messageUnread: 0,
|
||||
}),
|
||||
user_data_at: new Date(Date.now() - 3600_000),
|
||||
})
|
||||
.execute();
|
||||
const target = await seedMovieTarget(e, base.siteId);
|
||||
const slot4k = (
|
||||
await e.db
|
||||
.insertInto("acquisition_slot")
|
||||
.values({
|
||||
media_entity_id: target.entityId,
|
||||
name: "2160p",
|
||||
hard_conditions: JSON.stringify({ resolutions: ["2160p"] }),
|
||||
})
|
||||
.returning("id")
|
||||
.executeTakeFirstOrThrow()
|
||||
).id;
|
||||
const r2160 = (
|
||||
await e.db
|
||||
.insertInto("source_release")
|
||||
.values({
|
||||
site_id: base.siteId,
|
||||
site_release_id: "889001",
|
||||
title: "Dune.Part.Two.2024.2160p.UHD.BluRay.x265-GRP",
|
||||
size_bytes: 62_490_000_000,
|
||||
media_entity_id: target.entityId,
|
||||
resolution_status: "resolved",
|
||||
raw: JSON.stringify({ _site: { hr: false } }),
|
||||
})
|
||||
.returning("id")
|
||||
.executeTakeFirstOrThrow()
|
||||
).id;
|
||||
// 4 条已取消的订阅 + seedMovieTarget 的 1 条追更中
|
||||
for (const t of ["繁花", "三体", "漫长的季节", "狂飙"])
|
||||
await e.db
|
||||
.insertInto("subscription")
|
||||
.values({ title_query: t, media_kind: "series", status: "cancelled" })
|
||||
.execute();
|
||||
// 种子先进 mock qB,同步会把它们接管成任务行;之后按本用例需要覆盖状态与关联(upsert,和接管谁先谁后都一样)
|
||||
const qb = [
|
||||
[HASHES.dune, "Dune.Part.Two.2024.1080p.BluRay.x264-GRP", "downloading", 0.63],
|
||||
[HASHES.failed, "Oppenheimer.2023.1080p.BluRay.x264", "missingFiles", 0.4],
|
||||
[HASHES.seeding, "Poor.Things.2023.2160p.WEB-DL", "uploading", 1],
|
||||
] as const;
|
||||
for (const [hash, name, state, progress] of qb)
|
||||
e.qb.seed({ hash, name, state, progress, size: 20_000_000_000, tags: ["umt"], category: "umt" });
|
||||
const task = async (hash: string, name: string, status: string, progress: number, extra: Record<string, unknown>) => {
|
||||
const values = {
|
||||
instance_id: base.instanceId,
|
||||
torrent_hash: hash,
|
||||
infohash_v1: hash,
|
||||
name,
|
||||
status: status as never,
|
||||
progress,
|
||||
size_bytes: 20_000_000_000,
|
||||
tags: ["umt"],
|
||||
managed: true,
|
||||
origin: "submitted" as const,
|
||||
media_entity_id: target.entityId,
|
||||
last_progress_at: new Date(Date.now() - 5 * 60_000),
|
||||
...extra,
|
||||
} as never;
|
||||
await e.db
|
||||
.insertInto("download_task")
|
||||
.values(values)
|
||||
.onConflict((oc) => oc.columns(["instance_id", "torrent_hash"]).doUpdateSet(values))
|
||||
.execute();
|
||||
};
|
||||
await task(HASHES.dune, "Dune.Part.Two.2024.1080p.BluRay.x264-GRP", "downloading", 0.63, {
|
||||
source_release_id: target.r1080,
|
||||
});
|
||||
await task(HASHES.failed, "Oppenheimer.2023.1080p.BluRay.x264", "failed", 0.4, {
|
||||
media_entity_id: null,
|
||||
last_error: JSON.stringify({ message: "下载器找不到数据文件" }),
|
||||
});
|
||||
await task(HASHES.seeding, "Poor.Things.2023.2160p.WEB-DL", "imported", 1, {
|
||||
media_entity_id: null,
|
||||
completed_at: new Date(Date.now() - 12 * 86400_000),
|
||||
});
|
||||
await e.db
|
||||
.insertInto("decision")
|
||||
.values({
|
||||
kind: "download_failed",
|
||||
title: "下载异常：Oppenheimer.2023",
|
||||
subject_type: "download_task",
|
||||
subject_id: (
|
||||
await e.db
|
||||
.selectFrom("download_task")
|
||||
.select("id")
|
||||
.where("torrent_hash", "=", HASHES.failed)
|
||||
.executeTakeFirstOrThrow()
|
||||
).id,
|
||||
dedupe_key: `download_failed:eval`,
|
||||
options: JSON.stringify([{ key: "retry", label: "重试" }]),
|
||||
})
|
||||
.execute();
|
||||
return { siteId: base.siteId, instanceId: base.instanceId, entityId: target.entityId, slot4k, r2160 };
|
||||
}
|
||||
|
||||
const last = (r: FakeToolResult[], cap: string): Any => [...r].reverse().find((x) => x.capabilityId === cap)?.json;
|
||||
const lastResult = (r: FakeToolResult[], cap: string) => [...r].reverse().find((x) => x.capabilityId === cap);
|
||||
|
||||
export interface EvalCase {
|
||||
id: string;
|
||||
goal: string;
|
||||
/** 期望按意图载入的 Skill(null = 不载入) */
|
||||
skill: string | null;
|
||||
/** fake 模型的调用脚本(按"合格模型"会走的路线写) */
|
||||
steps: FakeStep[];
|
||||
/** 期望的能力调用顺序 */
|
||||
calls: string[];
|
||||
/** 允许失败(被拒绝)的能力 */
|
||||
denied?: string[];
|
||||
/** 合格模型按工具结果写出的汇报(只用结果里的数据) */
|
||||
report: (r: FakeToolResult[]) => string;
|
||||
}
|
||||
|
||||
const now = () => new Date();
|
||||
|
||||
export function evalCases(w: EvalWorld): EvalCase[] {
|
||||
return [
|
||||
{
|
||||
id: "health",
|
||||
goal: "检查站点、下载器和媒体服务器是否正常",
|
||||
skill: "patrol",
|
||||
steps: [
|
||||
{ toolCalls: [{ capability: "downloader.list", input: {} }] },
|
||||
{ toolCalls: [{ capability: "site.list", input: {} }] },
|
||||
{ toolCalls: [{ capability: "site.health_check", input: { siteId: w.siteId } }] },
|
||||
{ toolCalls: [{ capability: "decision.list", input: {} }] },
|
||||
],
|
||||
calls: ["downloader.list", "site.list", "site.health_check", "decision.list"],
|
||||
denied: ["site.health_check"],
|
||||
report: (r) => {
|
||||
const d = last(r, "downloader.list");
|
||||
const q = d.downloaders[0];
|
||||
const site = last(r, "site.list").items[0];
|
||||
const dec = last(r, "decision.list");
|
||||
const at = humanRelative(site.lastHealthAt, now());
|
||||
return [
|
||||
`下载器「${q.name}」${kindName(q.kind)} ${healthName(q.healthStatus)}，管理 ${q.tasks.total} 个任务，${q.tasks.failed} 个失败；媒体服务器还没接入。`,
|
||||
`站点「${site.name}」上次检查${healthName(site.healthStatus)}（${at}），没做：实时检测，需要你批准。`,
|
||||
`待处理 ${dec.total} 条：${dec.items[0].title}。`,
|
||||
].join("\n");
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "acquire",
|
||||
goal: "把《沙丘 2》4K 收进来",
|
||||
skill: "acquire-title",
|
||||
steps: [
|
||||
{ toolCalls: [{ capability: "catalog.lookup", input: { query: "沙丘 2" } }] },
|
||||
{ toolCalls: [{ capability: "inventory.query", input: { mediaEntityId: w.entityId } }] },
|
||||
{ toolCalls: [{ capability: "download.list", input: { mediaEntityId: w.entityId } }] },
|
||||
{ toolCalls: [{ capability: "release.search", input: { mediaEntityId: w.entityId } }] },
|
||||
{
|
||||
toolCalls: [
|
||||
{
|
||||
capability: "download.plan",
|
||||
input: (r) => ({
|
||||
slotId: w.slot4k,
|
||||
sourceReleaseId: last(r, "release.search").items.find((x: Any) => /2160p/.test(x.title)).id,
|
||||
}),
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
toolCalls: [
|
||||
{ capability: "download.submit", input: (r) => ({ planId: last(r, "download.plan").plan?.id ?? w.r2160 }) },
|
||||
],
|
||||
},
|
||||
],
|
||||
calls: [
|
||||
"catalog.lookup",
|
||||
"inventory.query",
|
||||
"download.list",
|
||||
"release.search",
|
||||
"download.plan",
|
||||
"download.submit",
|
||||
],
|
||||
denied: ["download.submit"],
|
||||
report: (r) => {
|
||||
const rel = last(r, "release.search").items.find((x: Any) => /2160p/.test(x.title));
|
||||
const gb = (rel.sizeBytes / 1024 ** 3).toFixed(1);
|
||||
const dl = last(r, "download.list").items[0];
|
||||
return [
|
||||
`《沙丘2》4K 找到了，还没提交：${rel.title}（${gb} GB，来自憨憨），没做：提交下载，需要你批准。`,
|
||||
`库里现有 1080p 正在下载（${dl.progress}）。`,
|
||||
].join("\n");
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "why",
|
||||
goal: "为什么沙丘2没入库",
|
||||
skill: "import-troubleshoot",
|
||||
steps: [
|
||||
{ toolCalls: [{ capability: "inventory.query", input: { query: "沙丘2" } }] },
|
||||
{ toolCalls: [{ capability: "download.list", input: { query: "沙丘2" } }] },
|
||||
{
|
||||
toolCalls: [
|
||||
{
|
||||
capability: "download.inspect",
|
||||
input: (r) => ({ downloadTaskId: last(r, "download.list").items[0].id }),
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
calls: ["inventory.query", "download.list", "download.inspect"],
|
||||
report: (r) => {
|
||||
const it = last(r, "download.list").items[0];
|
||||
return `《沙丘2》还没入库：还在${it.statusLabel}，进度 ${it.progress}，最近进展 ${it.lastProgress}；下完会自动整理入库。`;
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "subs",
|
||||
goal: "订阅追更情况",
|
||||
skill: "subscription-check",
|
||||
steps: [{ toolCalls: [{ capability: "subscription.list", input: { groupBy: "status" } }] }],
|
||||
calls: ["subscription.list"],
|
||||
report: (r) => {
|
||||
const s = last(r, "subscription.list");
|
||||
const n = (k: string) => s.groups.find((g: Any) => g.key === k)?.count ?? 0;
|
||||
return `${s.total} 个订阅：${n("active")} 个在追更（沙丘2，收录版本还没满足），${n("cancelled")} 个已取消。`;
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "failed",
|
||||
goal: "有没有失败的下载任务",
|
||||
skill: "download-diagnose",
|
||||
steps: [
|
||||
{ toolCalls: [{ capability: "download.list", input: { status: ["failed", "missing", "stalled"] } }] },
|
||||
{
|
||||
toolCalls: [
|
||||
{
|
||||
capability: "download.inspect",
|
||||
input: (r) => ({ downloadTaskId: last(r, "download.list").items[0].id }),
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
calls: ["download.list", "download.inspect"],
|
||||
report: (r) => {
|
||||
const it = last(r, "download.list").items[0];
|
||||
return `有 1 个下载失败：${it.name}（${it.statusLabel}，${it.error ?? "原因未确认"}）。其余下载正常。`;
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "storage",
|
||||
goal: "空间不够了，看看能清理什么",
|
||||
skill: "storage-cleanup",
|
||||
steps: [
|
||||
{ toolCalls: [{ capability: "downloader.list", input: {} }] },
|
||||
{ toolCalls: [{ capability: "download.list", input: { status: ["imported"] } }] },
|
||||
],
|
||||
calls: ["downloader.list", "download.list"],
|
||||
report: (r) => {
|
||||
const l = last(r, "download.list");
|
||||
const it = l.items[0];
|
||||
return `下载缓存里有 ${l.total} 个已入库还在做种的种子可以清理：${it.name}（${it.size}，${it.completed}完成）。在下载页点「清理缓存」会按保种规则只清可清理的。`;
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "decisions",
|
||||
goal: "待处理里有什么要我处理的",
|
||||
skill: null,
|
||||
steps: [{ toolCalls: [{ capability: "decision.list", input: {} }] }],
|
||||
calls: ["decision.list"],
|
||||
report: (r) => {
|
||||
const d = last(r, "decision.list");
|
||||
return `有 ${d.total} 条待处理：${d.items[0].title}，可以在待处理中心选「重试」。`;
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "site-login",
|
||||
goal: "站点憨憨的登录是不是掉了",
|
||||
skill: "patrol",
|
||||
steps: [{ toolCalls: [{ capability: "site.list", input: {} }] }],
|
||||
calls: ["site.list"],
|
||||
report: (r) => {
|
||||
const s = last(r, "site.list").items[0];
|
||||
return `站点「${s.name}」登录正常，上次检查${healthName(s.healthStatus)}（${humanRelative(s.lastHealthAt, now())}）。`;
|
||||
},
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
/** 评测用 fake 脚本:按 goal 精确匹配;finalText 用合格模型的汇报,并把工具结果交给 onDone */
|
||||
export function evalScripts(cases: EvalCase[], onDone: (id: string, r: FakeToolResult[], text: string) => void) {
|
||||
return cases.map(
|
||||
(c): FakeScript => ({
|
||||
name: `eval:${c.id}`,
|
||||
match: (goal) => goal === c.goal,
|
||||
steps: c.steps,
|
||||
finalText: (r) => {
|
||||
const text = c.report(r);
|
||||
onDone(c.id, r, text);
|
||||
return text;
|
||||
},
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export { lastResult };
|
||||
@@ -4,6 +4,7 @@ import type { FakeScript } from "@umt/agent";
|
||||
import { makeTorrent } from "@umt/testkit";
|
||||
import { createAcquisitionOrganizeModule } from "../../../src/modules/acquisition-organize/index.ts";
|
||||
import { createAgentModule } from "../../../src/modules/agent/index.ts";
|
||||
import type { AgentModuleOptions } from "../../../src/modules/agent/runner.ts";
|
||||
import { authSettingsModule } from "../../../src/modules/auth-settings/index.ts";
|
||||
import { createBrowserModule } from "../../../src/modules/browser/index.ts";
|
||||
import type { BrowserModuleOptions } from "../../../src/modules/browser/service.ts";
|
||||
@@ -32,6 +33,8 @@ export interface E2EModuleOptions {
|
||||
webSearch?: WebSearchModuleOptions;
|
||||
/** 浏览器模块注入(E14 视觉自动化:证据目录、动作后等待) */
|
||||
browser?: BrowserModuleOptions;
|
||||
/** 智能体模块的额外参数(E16:替换 / 捕获系统提示) */
|
||||
agent?: Pick<AgentModuleOptions, "systemPrompt">;
|
||||
}
|
||||
|
||||
export function buildModules(o: E2EModuleOptions): ServerModule[] {
|
||||
@@ -71,6 +74,7 @@ export function buildModules(o: E2EModuleOptions): ServerModule[] {
|
||||
sessionDir: o.sessionDir,
|
||||
...(o.fakeScripts ? { fakeScripts: o.fakeScripts } : {}),
|
||||
goalPollSec: 1,
|
||||
...(o.agent ?? {}),
|
||||
}),
|
||||
// 浏览器能力(browser.*)随首批能力一起注册;浏览器服务由用例自己登记(E14 指向 mock camofox)
|
||||
createBrowserModule(o.browser ?? {}),
|
||||
|
||||
Reference in new issue
Block a user