diff --git a/apps/server/test/e2e/e16-agent-prompt-eval.real.test.ts b/apps/server/test/e2e/e16-agent-prompt-eval.real.test.ts new file mode 100644 index 00000000..100883ef --- /dev/null +++ b/apps/server/test/e2e/e16-agent-prompt-eval.real.test.ts @@ -0,0 +1,186 @@ +import { readFileSync, writeFileSync } from "node:fs"; +import { lintReport } from "@umt/agent"; +import { CAPABILITY_IDS } from "@umt/contracts"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import type { RunPrompt } from "../../src/modules/agent/runner.ts"; +import { setDefaultUpstream } from "../../src/modules/pi/store.ts"; +import { evalCases, seedEvalWorld } from "./fixtures/agent-eval.ts"; +import { createE2E, type E2E, pgAvailable, runEvents, runRow, waitFor } from "./harness.ts"; + +// E16 真实模型评测(AG-5,手动跑,不进 CI):同一套测试库数据与提示,分别用旧提示(main 的 system-prompt.ts) +// 和分层提示跑真实模型,记录能力调用、汇报原文与 lintReport 结果,供 docs/design/agent-prompt-eval.md 对比。 +// 智能体只接触测试库数据与 mock qB / mock TMDB;站点地址是本机关闭的端口,不会请求真实站点。 +// +// 运行(凭证只经环境变量传入,不写进代码与报告): +// UMT_EVAL_REAL=1 UMT_EVAL_API_KEY=… UMT_EVAL_PROVIDER=opencode-go UMT_EVAL_MODEL=deepseek-v4.1-flash \ +// UMT_EVAL_LEGACY_PROMPT=/tmp/legacy-system-prompt.ts UMT_EVAL_CASES=health,acquire,why UMT_EVAL_OUT=/tmp/eval.json \ +// DATABASE_URL=… deno run -A npm:vitest run test/e2e/e16-agent-prompt-eval.real.test.ts +// UMT_EVAL_LEGACY_PROMPT 用 `git show main:packages/agent/src/system-prompt.ts > /tmp/legacy-system-prompt.ts` 生成; +// UMT_EVAL_PROMPTS=legacy,layered 控制跑哪几组(缺省两组都跑)。 + +// biome-ignore lint/suspicious/noExplicitAny: 评测按输出结构取值 +type Any = any; + +const enabled = process.env.UMT_EVAL_REAL === "1" && (await pgAvailable()); + +function apiKey(): string { + const direct = process.env.UMT_EVAL_API_KEY; + if (direct) return direct.trim(); + const file = process.env.UMT_EVAL_API_KEY_FILE; + if (!file) throw new Error("缺少 UMT_EVAL_API_KEY 或 UMT_EVAL_API_KEY_FILE"); + // 凭证文件:取带 opencode 字样那一行里最长的 token(不回显内容) + const line = readFileSync(file, "utf8") + .split("\n") + .find((l) => /opencode/i.test(l)); + const token = line?.match(/[A-Za-z0-9_\-.]{24,}/g)?.sort((a, b) => b.length - a.length)[0]; + if (!token) throw new Error("凭证文件里没找到 OpenCode 的 key"); + return token; +} + +interface EvalRecord { + prompt: "legacy" | "layered"; + case: string; + goal: string; + status: string; + calls: { capability: string; ok: boolean }[]; + finalText: string; + lint: string[]; + lines: number; + usage: unknown; + systemPromptChars: number; +} + +describe.skipIf(!enabled)("E16 真实模型评测", () => { + let e: E2E; + const legacyPath = process.env.UMT_EVAL_LEGACY_PROMPT; + const which = (process.env.UMT_EVAL_PROMPTS ?? "legacy,layered").split(",") as ("legacy" | "layered")[]; + const caseIds = (process.env.UMT_EVAL_CASES ?? "health,acquire,why").split(","); + const records: EvalRecord[] = []; + let current: "legacy" | "layered" = "layered"; + const sizes = new Map(); + // biome-ignore lint/suspicious/noExplicitAny: 旧提示模块按路径动态导入 + let legacy: any = null; + + beforeAll(async () => { + if (which.includes("legacy")) { + if (!legacyPath) throw new Error("跑旧提示需要 UMT_EVAL_LEGACY_PROMPT"); + legacy = await import(legacyPath); + } + e = await createE2E("e16real", { + agent: { + systemPrompt: (p: RunPrompt, run, capabilities) => { + const text = current === "legacy" ? legacyPrompt(run.goal, capabilities) : p.systemPrompt; + sizes.set(`${current}:${run.goal}`, text.length); + return text; + }, + }, + }); + await seedEvalWorld(e); + const ref = await e.booted.ctx.secrets.create({ purpose: "llm_api_key", plaintext: apiKey() }); + const up = await e.db + .insertInto("pi_upstream") + .values({ + name: "eval", + provider: process.env.UMT_EVAL_PROVIDER ?? "opencode-go", + auth_kind: "api_key", + secret_ref: ref, + default_model: process.env.UMT_EVAL_MODEL ?? "deepseek-v4.1-flash", + thinking_level: "low", + extra: JSON.stringify({}), + enabled: true, + status: "ok", + }) + .returning("id") + .executeTakeFirstOrThrow(); + await setDefaultUpstream(e.booted.ctx, up.id); + await e.settings({ + agent: { maxToolCalls: 16, maxDurationSec: 240, optionsTimeoutSec: 20, optionsTimeoutAction: "stop" }, + }); + await waitFor( + async () => + (await e.db.selectFrom("downloader_instance").select("health_status").executeTakeFirst())?.health_status === + "healthy", + 20_000, + "下载器同步", + ); + }, 180_000); + + afterAll(async () => { + if (process.env.UMT_EVAL_OUT) writeFileSync(process.env.UMT_EVAL_OUT, JSON.stringify(records, null, 2)); + await e?.close(); + }, 60_000); + + /** main 上 runner.systemPromptFor 的旧装配(旧 buildSystemPrompt + 旧约束) */ + function legacyPrompt(goal: string, capabilities: string[]): string { + return legacy.buildSystemPrompt({ + role: "媒体库自动化智能体", + goal, + constraints: [ + `可用能力：${capabilities.join(", ")}`, + "有副作用的能力（提交下载、执行整理、发布策略、重试/取消任务）需要服务端授权；被拒时说明缺少哪项授权。", + ], + audience: "user", + interaction: { + plan: capabilities.includes("agent.plan"), + options: capabilities.includes("agent.propose_options"), + proposeFirst: false, + approvals: false, + }, + }); + } + + it("跑固定提示并记录", async () => { + const cases = evalCases({ siteId: "", instanceId: "", entityId: "", slot4k: "", r2160: "" }).filter((c) => + caseIds.includes(c.id), + ); + for (const p of which) { + for (const c of cases) { + current = p; + // 旧提示对应 main 的能力集合(没有 download.list) + const allow = p === "legacy" ? CAPABILITY_IDS.filter((id) => id !== "download.list") : undefined; + const r = await e.api("POST", "/agent/runs", { + goal: c.goal, + ...(allow ? { capabilityAllowlist: allow } : {}), + }); + expect(r.status).toBe(201); + const runId = r.body.data.id as string; + const run = await waitFor( + async () => { + const x = await runRow(e, runId); + return ["succeeded", "failed", "waiting", "budget_exceeded", "cancelled"].includes(x.status) ? x : null; + }, + 300_000, + `run 结束:${p}:${c.goal}`, + 1000, + ); + const events = await runEvents(e, runId); + const ends = new Map( + events + .filter((x) => x.type === "tool_call_end") + .map((x) => [(x.payload as Any).toolCallId, !(x.payload as Any).isError]), + ); + const calls = events + .filter((x) => x.type === "tool_call_start") + .map((x) => ({ + capability: (x.payload as Any).capabilityId as string, + ok: ends.get((x.payload as Any).toolCallId) ?? false, + })); + const done = [...events].reverse().find((x) => x.type === "run_completed"); + const finalText = ((done?.payload as Any)?.finalText as string | undefined) ?? run.summary ?? ""; + records.push({ + prompt: p, + case: c.id, + goal: c.goal, + status: run.status, + calls, + finalText, + lint: lintReport(finalText).map((i) => `${i.rule}:${i.detail}`), + lines: finalText.split("\n").filter((l) => l.trim()).length, + usage: run.usage, + systemPromptChars: sizes.get(`${p}:${c.goal}`) ?? 0, + }); + } + } + expect(records.length).toBe(which.length * cases.length); + }, 1_800_000); +}); diff --git a/apps/server/test/e2e/e16-agent-prompt-eval.test.ts b/apps/server/test/e2e/e16-agent-prompt-eval.test.ts new file mode 100644 index 00000000..b7d8619e --- /dev/null +++ b/apps/server/test/e2e/e16-agent-prompt-eval.test.ts @@ -0,0 +1,148 @@ +import { type FakeScript, type FakeToolResult, lintReport } from "@umt/agent"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import type { RunPrompt } from "../../src/modules/agent/runner.ts"; +import { type EvalCase, type EvalWorld, evalCases, evalScripts, seedEvalWorld } from "./fixtures/agent-eval.ts"; +import { createE2E, type E2E, pgAvailable, runEvents, runRow, waitFor } from "./harness.ts"; + +// E16 智能体提示评测(AG-5,fake 模型 + 真实能力 + 测试库数据): +// - 固定提示集(8 条,含反例"检查站点、下载器和媒体服务器是否正常"):按意图载入的 Skill、当前环境层的事实; +// - fake 模型按"合格模型"的路线调用能力:断言调用顺序、只读调用都成功、需要批准的被拒绝(不向用户要 ID: +// download.inspect 的 ID 来自 download.list); +// - 汇报格式约束:合格汇报(只用工具结果里的数据写)通过 lintReport(无 UUID / 原始字节 / 内部字段,≤8 行,结论先行)。 +// 真实模型的前后对比见 e16-agent-prompt-eval.real.test.ts 与 docs/design/agent-prompt-eval.md。 + +const hasPg = await pgAvailable(); + +describe.skipIf(!hasPg)("E16 智能体提示评测(fake 模型)", () => { + const scripts: FakeScript[] = []; + const prompts = new Map(); + const done = new Map(); + let e: E2E; + let world: EvalWorld; + let cases: EvalCase[]; + + beforeAll(async () => { + e = await createE2E("e16", { + fakeScripts: scripts, + agent: { + systemPrompt: (p, run) => { + prompts.set(run.goal, p); + return p.systemPrompt; + }, + }, + }); + world = await seedEvalWorld(e); + cases = evalCases(world); + scripts.push(...evalScripts(cases, (id, results, text) => done.set(id, { results, text }))); + // 等下载同步把 mock qB 的状态写回(下载器连通状态、任务状态) + await waitFor( + async () => + (await e.db.selectFrom("downloader_instance").select("health_status").executeTakeFirst())?.health_status === + "healthy", + 20_000, + "下载器同步", + ); + }, 120_000); + + afterAll(async () => { + await e?.close(); + }, 60_000); + + const runToEnd = async (goal: string) => { + const r = await e.api("POST", "/agent/runs", { goal }); + expect(r.status).toBe(201); + const runId = r.body.data.id as string; + const run = await waitFor( + async () => { + const x = await runRow(e, runId); + return ["succeeded", "failed", "waiting", "budget_exceeded", "cancelled"].includes(x.status) ? x : null; + }, + 60_000, + `run 结束:${goal}`, + ); + return { runId, run }; + }; + + it("固定提示集至少 8 条,含反例那条", () => { + expect(cases.length).toBeGreaterThanOrEqual(8); + expect(cases.map((c) => c.goal)).toContain("检查站点、下载器和媒体服务器是否正常"); + }); + + it.each(["health", "acquire", "why", "subs", "failed", "storage", "decisions", "site-login"])( + "%s:Skill、环境层、调用顺序与汇报格式", + async (id) => { + const c = cases.find((x) => x.id === id)!; + const { runId, run } = await runToEnd(c.goal); + expect(["succeeded", "waiting"], `${run.status}: ${run.summary}`).toContain(run.status); + + // 系统提示:按意图载入的 Skill + 当前环境 + const p = prompts.get(c.goal)!; + expect(p.kind).toBe("chat"); + expect(p.skills[0]?.name ?? null).toBe(c.skill); + expect(p.systemPrompt).toContain("下载器 1 个：「家里的 qB」qBittorrent"); + expect(p.systemPrompt).toContain("媒体服务器：还没有接入"); + expect(p.systemPrompt).toContain("「憨憨」PT 站，"); + expect(p.systemPrompt).not.toMatch(/127\.0\.0\.1|hhanclub/); + + // 调用顺序:按事件流里的 tool_call_start + const events = await runEvents(e, runId); + const calls = events + .filter((x) => x.type === "tool_call_start") + .map((x) => (x.payload as { capabilityId?: string }).capabilityId); + expect(calls).toEqual(c.calls); + + // 只读调用都成功;需要批准的被拒绝 + const r = done.get(id)!; + for (const x of r.results) { + const denied = c.denied?.includes(x.capabilityId ?? ""); + expect(x.isError, `${x.capabilityId}: ${x.text.slice(0, 200)}`).toBe(!!denied); + if (denied) expect(x.text).toMatch(/unauthorized|授权|批准/); + } + + // 汇报格式 + expect(lintReport(r.text), r.text).toEqual([]); + }, + 90_000, + ); + + it("download.list:人话大小 / 进度 / 相对时间,不含 hash 与原始字节数", async () => { + const r = done.get("storage")!.results.find((x) => x.capabilityId === "download.list")!; + expect(r.json).toMatchObject({ + total: 1, + byStatus: [{ status: "imported", label: "已入库", count: 1 }], + items: [{ name: "Poor.Things.2023.2160p.WEB-DL", size: "18.6 GB", progress: "100%", completed: "12 天前" }], + }); + expect(r.text).not.toMatch(/c3c3c3|20000000000/); + }); + + it("巡检会话固定载入「巡检」Skill,汇报层要求按巡检模板写", async () => { + const { buildRunPrompt, scopeFor } = await import("../../src/modules/agent/runner.ts"); + const row = { + ...(await e.db.selectFrom("agent_run").selectAll().limit(1).executeTakeFirstOrThrow()), + trigger: "schedule" as const, + goal: "巡检待处理(2026-09-30)：有 1 条未关闭的待处理事项。", + mode: "plan_only" as const, + }; + const scope = scopeFor("plan_only", ["decision.list", "downloader.list", "site.list"]); + const p = await buildRunPrompt({ ctx: e.booted.ctx, opts: {} }, row, scope, { + canSpawn: false, + maxConcurrent: 1, + maxDepth: 1, + }); + expect(p.kind).toBe("patrol"); + expect(p.skills[0]?.name).toBe("patrol"); + expect(p.systemPrompt).toContain("### Skill「巡检」"); + expect(p.systemPrompt).toContain("按 Skill「巡检」的汇报模板写"); + expect(p.systemPrompt).toContain("本次只读"); + }); + + it("追加消息换了意图:新命中的 Skill 用 Pi 的 skill 块附在消息前,已载入的不重复", async () => { + const { followUpWithSkills } = await import("../../src/modules/agent/runner.ts"); + const p = prompts.get("检查站点、下载器和媒体服务器是否正常")!; + const copy = { ...p, skills: [...p.skills] }; + const msg = followUpWithSkills(copy, "那订阅追更情况呢"); + expect(msg).toMatch(/^ { + const base = await seedBase(e, { key: "hhanclub", baseUrl: "http://127.0.0.1:9/" }); + await e.db + .updateTable("site") + .set({ + name: "憨憨", + health_status: "healthy", + last_health_at: new Date(Date.now() - 20 * 3600_000), + last_health: JSON.stringify({ ok: true, status: "ok", latencyMs: 1201, message: null }), + }) + .where("id", "=", base.siteId) + .execute(); + await e.db.updateTable("downloader_instance").set({ name: "家里的 qB" }).where("id", "=", base.instanceId).execute(); + await e.db + .insertInto("site_account") + .values({ + site_id: base.siteId, + label: "主账号", + auth_kind: "cookie", + status: "active", + user_data: JSON.stringify({ + schema: "nexusphp", + userId: "1", + username: "u", + userLevel: "Power User", + vip: false, + joinAt: null, + upload: 17008345370067, + download: 0, + ratio: null, + bonus: 414799, + seeding: 24, + seedingSize: 79407286843, + leeching: 0, + leechingSize: 0, + messageUnread: 0, + }), + user_data_at: new Date(Date.now() - 3600_000), + }) + .execute(); + const target = await seedMovieTarget(e, base.siteId); + const slot4k = ( + await e.db + .insertInto("acquisition_slot") + .values({ + media_entity_id: target.entityId, + name: "2160p", + hard_conditions: JSON.stringify({ resolutions: ["2160p"] }), + }) + .returning("id") + .executeTakeFirstOrThrow() + ).id; + const r2160 = ( + await e.db + .insertInto("source_release") + .values({ + site_id: base.siteId, + site_release_id: "889001", + title: "Dune.Part.Two.2024.2160p.UHD.BluRay.x265-GRP", + size_bytes: 62_490_000_000, + media_entity_id: target.entityId, + resolution_status: "resolved", + raw: JSON.stringify({ _site: { hr: false } }), + }) + .returning("id") + .executeTakeFirstOrThrow() + ).id; + // 4 条已取消的订阅 + seedMovieTarget 的 1 条追更中 + for (const t of ["繁花", "三体", "漫长的季节", "狂飙"]) + await e.db + .insertInto("subscription") + .values({ title_query: t, media_kind: "series", status: "cancelled" }) + .execute(); + // 种子先进 mock qB,同步会把它们接管成任务行;之后按本用例需要覆盖状态与关联(upsert,和接管谁先谁后都一样) + const qb = [ + [HASHES.dune, "Dune.Part.Two.2024.1080p.BluRay.x264-GRP", "downloading", 0.63], + [HASHES.failed, "Oppenheimer.2023.1080p.BluRay.x264", "missingFiles", 0.4], + [HASHES.seeding, "Poor.Things.2023.2160p.WEB-DL", "uploading", 1], + ] as const; + for (const [hash, name, state, progress] of qb) + e.qb.seed({ hash, name, state, progress, size: 20_000_000_000, tags: ["umt"], category: "umt" }); + const task = async (hash: string, name: string, status: string, progress: number, extra: Record) => { + const values = { + instance_id: base.instanceId, + torrent_hash: hash, + infohash_v1: hash, + name, + status: status as never, + progress, + size_bytes: 20_000_000_000, + tags: ["umt"], + managed: true, + origin: "submitted" as const, + media_entity_id: target.entityId, + last_progress_at: new Date(Date.now() - 5 * 60_000), + ...extra, + } as never; + await e.db + .insertInto("download_task") + .values(values) + .onConflict((oc) => oc.columns(["instance_id", "torrent_hash"]).doUpdateSet(values)) + .execute(); + }; + await task(HASHES.dune, "Dune.Part.Two.2024.1080p.BluRay.x264-GRP", "downloading", 0.63, { + source_release_id: target.r1080, + }); + await task(HASHES.failed, "Oppenheimer.2023.1080p.BluRay.x264", "failed", 0.4, { + media_entity_id: null, + last_error: JSON.stringify({ message: "下载器找不到数据文件" }), + }); + await task(HASHES.seeding, "Poor.Things.2023.2160p.WEB-DL", "imported", 1, { + media_entity_id: null, + completed_at: new Date(Date.now() - 12 * 86400_000), + }); + await e.db + .insertInto("decision") + .values({ + kind: "download_failed", + title: "下载异常：Oppenheimer.2023", + subject_type: "download_task", + subject_id: ( + await e.db + .selectFrom("download_task") + .select("id") + .where("torrent_hash", "=", HASHES.failed) + .executeTakeFirstOrThrow() + ).id, + dedupe_key: `download_failed:eval`, + options: JSON.stringify([{ key: "retry", label: "重试" }]), + }) + .execute(); + return { siteId: base.siteId, instanceId: base.instanceId, entityId: target.entityId, slot4k, r2160 }; +} + +const last = (r: FakeToolResult[], cap: string): Any => [...r].reverse().find((x) => x.capabilityId === cap)?.json; +const lastResult = (r: FakeToolResult[], cap: string) => [...r].reverse().find((x) => x.capabilityId === cap); + +export interface EvalCase { + id: string; + goal: string; + /** 期望按意图载入的 Skill(null = 不载入) */ + skill: string | null; + /** fake 模型的调用脚本(按"合格模型"会走的路线写) */ + steps: FakeStep[]; + /** 期望的能力调用顺序 */ + calls: string[]; + /** 允许失败(被拒绝)的能力 */ + denied?: string[]; + /** 合格模型按工具结果写出的汇报(只用结果里的数据) */ + report: (r: FakeToolResult[]) => string; +} + +const now = () => new Date(); + +export function evalCases(w: EvalWorld): EvalCase[] { + return [ + { + id: "health", + goal: "检查站点、下载器和媒体服务器是否正常", + skill: "patrol", + steps: [ + { toolCalls: [{ capability: "downloader.list", input: {} }] }, + { toolCalls: [{ capability: "site.list", input: {} }] }, + { toolCalls: [{ capability: "site.health_check", input: { siteId: w.siteId } }] }, + { toolCalls: [{ capability: "decision.list", input: {} }] }, + ], + calls: ["downloader.list", "site.list", "site.health_check", "decision.list"], + denied: ["site.health_check"], + report: (r) => { + const d = last(r, "downloader.list"); + const q = d.downloaders[0]; + const site = last(r, "site.list").items[0]; + const dec = last(r, "decision.list"); + const at = humanRelative(site.lastHealthAt, now()); + return [ + `下载器「${q.name}」${kindName(q.kind)} ${healthName(q.healthStatus)}，管理 ${q.tasks.total} 个任务，${q.tasks.failed} 个失败；媒体服务器还没接入。`, + `站点「${site.name}」上次检查${healthName(site.healthStatus)}（${at}），没做：实时检测，需要你批准。`, + `待处理 ${dec.total} 条：${dec.items[0].title}。`, + ].join("\n"); + }, + }, + { + id: "acquire", + goal: "把《沙丘 2》4K 收进来", + skill: "acquire-title", + steps: [ + { toolCalls: [{ capability: "catalog.lookup", input: { query: "沙丘 2" } }] }, + { toolCalls: [{ capability: "inventory.query", input: { mediaEntityId: w.entityId } }] }, + { toolCalls: [{ capability: "download.list", input: { mediaEntityId: w.entityId } }] }, + { toolCalls: [{ capability: "release.search", input: { mediaEntityId: w.entityId } }] }, + { + toolCalls: [ + { + capability: "download.plan", + input: (r) => ({ + slotId: w.slot4k, + sourceReleaseId: last(r, "release.search").items.find((x: Any) => /2160p/.test(x.title)).id, + }), + }, + ], + }, + { + toolCalls: [ + { capability: "download.submit", input: (r) => ({ planId: last(r, "download.plan").plan?.id ?? w.r2160 }) }, + ], + }, + ], + calls: [ + "catalog.lookup", + "inventory.query", + "download.list", + "release.search", + "download.plan", + "download.submit", + ], + denied: ["download.submit"], + report: (r) => { + const rel = last(r, "release.search").items.find((x: Any) => /2160p/.test(x.title)); + const gb = (rel.sizeBytes / 1024 ** 3).toFixed(1); + const dl = last(r, "download.list").items[0]; + return [ + `《沙丘2》4K 找到了，还没提交：${rel.title}（${gb} GB，来自憨憨），没做：提交下载，需要你批准。`, + `库里现有 1080p 正在下载（${dl.progress}）。`, + ].join("\n"); + }, + }, + { + id: "why", + goal: "为什么沙丘2没入库", + skill: "import-troubleshoot", + steps: [ + { toolCalls: [{ capability: "inventory.query", input: { query: "沙丘2" } }] }, + { toolCalls: [{ capability: "download.list", input: { query: "沙丘2" } }] }, + { + toolCalls: [ + { + capability: "download.inspect", + input: (r) => ({ downloadTaskId: last(r, "download.list").items[0].id }), + }, + ], + }, + ], + calls: ["inventory.query", "download.list", "download.inspect"], + report: (r) => { + const it = last(r, "download.list").items[0]; + return `《沙丘2》还没入库：还在${it.statusLabel}，进度 ${it.progress}，最近进展 ${it.lastProgress}；下完会自动整理入库。`; + }, + }, + { + id: "subs", + goal: "订阅追更情况", + skill: "subscription-check", + steps: [{ toolCalls: [{ capability: "subscription.list", input: { groupBy: "status" } }] }], + calls: ["subscription.list"], + report: (r) => { + const s = last(r, "subscription.list"); + const n = (k: string) => s.groups.find((g: Any) => g.key === k)?.count ?? 0; + return `${s.total} 个订阅：${n("active")} 个在追更（沙丘2，收录版本还没满足），${n("cancelled")} 个已取消。`; + }, + }, + { + id: "failed", + goal: "有没有失败的下载任务", + skill: "download-diagnose", + steps: [ + { toolCalls: [{ capability: "download.list", input: { status: ["failed", "missing", "stalled"] } }] }, + { + toolCalls: [ + { + capability: "download.inspect", + input: (r) => ({ downloadTaskId: last(r, "download.list").items[0].id }), + }, + ], + }, + ], + calls: ["download.list", "download.inspect"], + report: (r) => { + const it = last(r, "download.list").items[0]; + return `有 1 个下载失败：${it.name}（${it.statusLabel}，${it.error ?? "原因未确认"}）。其余下载正常。`; + }, + }, + { + id: "storage", + goal: "空间不够了，看看能清理什么", + skill: "storage-cleanup", + steps: [ + { toolCalls: [{ capability: "downloader.list", input: {} }] }, + { toolCalls: [{ capability: "download.list", input: { status: ["imported"] } }] }, + ], + calls: ["downloader.list", "download.list"], + report: (r) => { + const l = last(r, "download.list"); + const it = l.items[0]; + return `下载缓存里有 ${l.total} 个已入库还在做种的种子可以清理：${it.name}（${it.size}，${it.completed}完成）。在下载页点「清理缓存」会按保种规则只清可清理的。`; + }, + }, + { + id: "decisions", + goal: "待处理里有什么要我处理的", + skill: null, + steps: [{ toolCalls: [{ capability: "decision.list", input: {} }] }], + calls: ["decision.list"], + report: (r) => { + const d = last(r, "decision.list"); + return `有 ${d.total} 条待处理：${d.items[0].title}，可以在待处理中心选「重试」。`; + }, + }, + { + id: "site-login", + goal: "站点憨憨的登录是不是掉了", + skill: "patrol", + steps: [{ toolCalls: [{ capability: "site.list", input: {} }] }], + calls: ["site.list"], + report: (r) => { + const s = last(r, "site.list").items[0]; + return `站点「${s.name}」登录正常，上次检查${healthName(s.healthStatus)}（${humanRelative(s.lastHealthAt, now())}）。`; + }, + }, + ]; +} + +/** 评测用 fake 脚本:按 goal 精确匹配;finalText 用合格模型的汇报,并把工具结果交给 onDone */ +export function evalScripts(cases: EvalCase[], onDone: (id: string, r: FakeToolResult[], text: string) => void) { + return cases.map( + (c): FakeScript => ({ + name: `eval:${c.id}`, + match: (goal) => goal === c.goal, + steps: c.steps, + finalText: (r) => { + const text = c.report(r); + onDone(c.id, r, text); + return text; + }, + }), + ); +} + +export { lastResult }; diff --git a/apps/server/test/e2e/fixtures/modules.ts b/apps/server/test/e2e/fixtures/modules.ts index af379a5a..fdfd800f 100644 --- a/apps/server/test/e2e/fixtures/modules.ts +++ b/apps/server/test/e2e/fixtures/modules.ts @@ -4,6 +4,7 @@ import type { FakeScript } from "@umt/agent"; import { makeTorrent } from "@umt/testkit"; import { createAcquisitionOrganizeModule } from "../../../src/modules/acquisition-organize/index.ts"; import { createAgentModule } from "../../../src/modules/agent/index.ts"; +import type { AgentModuleOptions } from "../../../src/modules/agent/runner.ts"; import { authSettingsModule } from "../../../src/modules/auth-settings/index.ts"; import { createBrowserModule } from "../../../src/modules/browser/index.ts"; import type { BrowserModuleOptions } from "../../../src/modules/browser/service.ts"; @@ -32,6 +33,8 @@ export interface E2EModuleOptions { webSearch?: WebSearchModuleOptions; /** 浏览器模块注入(E14 视觉自动化:证据目录、动作后等待) */ browser?: BrowserModuleOptions; + /** 智能体模块的额外参数(E16:替换 / 捕获系统提示) */ + agent?: Pick; } export function buildModules(o: E2EModuleOptions): ServerModule[] { @@ -71,6 +74,7 @@ export function buildModules(o: E2EModuleOptions): ServerModule[] { sessionDir: o.sessionDir, ...(o.fakeScripts ? { fakeScripts: o.fakeScripts } : {}), goalPollSec: 1, + ...(o.agent ?? {}), }), // 浏览器能力(browser.*)随首批能力一起注册;浏览器服务由用例自己登记(E14 指向 mock camofox) createBrowserModule(o.browser ?? {}), diff --git a/apps/server/test/e2e/harness.ts b/apps/server/test/e2e/harness.ts index 448a7a65..1bacec14 100644 --- a/apps/server/test/e2e/harness.ts +++ b/apps/server/test/e2e/harness.ts @@ -20,6 +20,7 @@ import { type Bootstrapped, bootstrap } from "../../src/context.ts"; import { createApp } from "../../src/http/app.ts"; import { EventBus } from "../../src/http/sse.ts"; import { createLogger } from "../../src/logger.ts"; +import type { AgentModuleOptions } from "../../src/modules/agent/runner.ts"; import type { BrowserModuleOptions } from "../../src/modules/browser/service.ts"; import { setDefaultUpstream } from "../../src/modules/pi/store.ts"; import type { ServerModule } from "../../src/modules/types.ts"; @@ -84,6 +85,8 @@ export interface E2EOptions { webSearch?: WebSearchModuleOptions; /** 浏览器模块注入(E14) */ browser?: BrowserModuleOptions; + /** 智能体模块的额外参数(E16:替换 / 捕获系统提示) */ + agent?: Pick; } export async function waitFor( @@ -140,6 +143,7 @@ export async function createE2E(name: string, opts: E2EOptions = {}): Promise