import { type FakeScript, type FakeStep, type FakeToolResult, healthName, humanRelative, kindName } from "@umt/agent"; import { type E2E, seedBase, seedMovieTarget } from "../harness.ts"; // AG-5 智能体提示评测的共用夹具:测试库里的一套"用户环境"(对照 docs/design/agent-report-sample-bad.md 的现场) // + 固定提示集。e16-agent-prompt-eval(fake 模型,CI)与 e16-agent-prompt-eval.real(真实模型,手动)共用。 // 站点地址指向本机关闭的端口:智能体即使实时补查也不会请求真实站点。 // biome-ignore lint/suspicious/noExplicitAny: 评测按输出结构取值 type Any = any; export const HASHES = { dune: "a1".repeat(20), failed: "b2".repeat(20), seeding: "c3".repeat(20), } as const; export interface EvalWorld { siteId: string; instanceId: string; entityId: string; slot4k: string; r2160: string; } /** 下载器(mock qB,3 个任务:下载中 / 失败 / 已入库做种)+ 站点憨憨(上次检查正常、账号有用户数据)+ 沙丘2 + 5 条订阅 */ export async function seedEvalWorld(e: E2E): Promise { const base = await seedBase(e, { key: "hhanclub", baseUrl: "http://127.0.0.1:9/" }); await e.db .updateTable("site") .set({ name: "憨憨", health_status: "healthy", last_health_at: new Date(Date.now() - 20 * 3600_000), last_health: JSON.stringify({ ok: true, status: "ok", latencyMs: 1201, message: null }), }) .where("id", "=", base.siteId) .execute(); await e.db.updateTable("downloader_instance").set({ name: "家里的 qB" }).where("id", "=", base.instanceId).execute(); await e.db .insertInto("site_account") .values({ site_id: base.siteId, label: "主账号", auth_kind: "cookie", status: "active", user_data: JSON.stringify({ schema: "nexusphp", userId: "1", username: "u", userLevel: "Power User", vip: false, joinAt: null, upload: 17008345370067, download: 0, ratio: null, bonus: 414799, seeding: 24, seedingSize: 79407286843, leeching: 0, leechingSize: 0, messageUnread: 0, }), user_data_at: new Date(Date.now() - 3600_000), }) .execute(); const target = await seedMovieTarget(e, base.siteId); const slot4k = ( await e.db .insertInto("acquisition_slot") .values({ media_entity_id: target.entityId, name: "2160p", hard_conditions: JSON.stringify({ resolutions: ["2160p"] }), }) .returning("id") .executeTakeFirstOrThrow() ).id; const r2160 = ( await e.db .insertInto("source_release") .values({ site_id: base.siteId, site_release_id: "889001", title: "Dune.Part.Two.2024.2160p.UHD.BluRay.x265-GRP", size_bytes: 62_490_000_000, media_entity_id: target.entityId, resolution_status: "resolved", raw: JSON.stringify({ _site: { hr: false } }), }) .returning("id") .executeTakeFirstOrThrow() ).id; // 4 条已取消的订阅 + seedMovieTarget 的 1 条追更中 for (const t of ["繁花", "三体", "漫长的季节", "狂飙"]) await e.db .insertInto("subscription") .values({ title_query: t, media_kind: "series", status: "cancelled" }) .execute(); // 种子先进 mock qB,同步会把它们接管成任务行;之后按本用例需要覆盖状态与关联(upsert,和接管谁先谁后都一样) const qb = [ [HASHES.dune, "Dune.Part.Two.2024.1080p.BluRay.x264-GRP", "downloading", 0.63], [HASHES.failed, "Oppenheimer.2023.1080p.BluRay.x264", "missingFiles", 0.4], [HASHES.seeding, "Poor.Things.2023.2160p.WEB-DL", "uploading", 1], ] as const; for (const [hash, name, state, progress] of qb) e.qb.seed({ hash, name, state, progress, size: 20_000_000_000, tags: ["umt"], category: "umt" }); const task = async (hash: string, name: string, status: string, progress: number, extra: Record) => { const values = { instance_id: base.instanceId, torrent_hash: hash, infohash_v1: hash, name, status: status as never, progress, size_bytes: 20_000_000_000, tags: ["umt"], managed: true, origin: "submitted" as const, media_entity_id: target.entityId, last_progress_at: new Date(Date.now() - 5 * 60_000), ...extra, } as never; await e.db .insertInto("download_task") .values(values) .onConflict((oc) => oc.columns(["instance_id", "torrent_hash"]).doUpdateSet(values)) .execute(); }; await task(HASHES.dune, "Dune.Part.Two.2024.1080p.BluRay.x264-GRP", "downloading", 0.63, { source_release_id: target.r1080, }); await task(HASHES.failed, "Oppenheimer.2023.1080p.BluRay.x264", "failed", 0.4, { media_entity_id: null, last_error: JSON.stringify({ message: "下载器找不到数据文件" }), }); await task(HASHES.seeding, "Poor.Things.2023.2160p.WEB-DL", "imported", 1, { media_entity_id: null, completed_at: new Date(Date.now() - 12 * 86400_000), }); await e.db .insertInto("decision") .values({ kind: "download_failed", title: "下载异常:Oppenheimer.2023", subject_type: "download_task", subject_id: ( await e.db .selectFrom("download_task") .select("id") .where("torrent_hash", "=", HASHES.failed) .executeTakeFirstOrThrow() ).id, dedupe_key: `download_failed:eval`, options: JSON.stringify([{ key: "retry", label: "重试" }]), }) .execute(); return { siteId: base.siteId, instanceId: base.instanceId, entityId: target.entityId, slot4k, r2160 }; } const last = (r: FakeToolResult[], cap: string): Any => [...r].reverse().find((x) => x.capabilityId === cap)?.json; const lastResult = (r: FakeToolResult[], cap: string) => [...r].reverse().find((x) => x.capabilityId === cap); export interface EvalCase { id: string; goal: string; /** 期望按意图载入的 Skill(null = 不载入) */ skill: string | null; /** fake 模型的调用脚本(按"合格模型"会走的路线写) */ steps: FakeStep[]; /** 期望的能力调用顺序 */ calls: string[]; /** 允许失败(被拒绝)的能力 */ denied?: string[]; /** 合格模型按工具结果写出的汇报(只用结果里的数据) */ report: (r: FakeToolResult[]) => string; } const now = () => new Date(); export function evalCases(w: EvalWorld): EvalCase[] { return [ { id: "health", goal: "检查站点、下载器和媒体服务器是否正常", skill: "patrol", steps: [ { toolCalls: [{ capability: "downloader.list", input: {} }] }, { toolCalls: [{ capability: "site.list", input: {} }] }, { toolCalls: [{ capability: "site.health_check", input: { siteId: w.siteId } }] }, { toolCalls: [{ capability: "decision.list", input: {} }] }, ], calls: ["downloader.list", "site.list", "site.health_check", "decision.list"], denied: ["site.health_check"], report: (r) => { const d = last(r, "downloader.list"); const q = d.downloaders[0]; const site = last(r, "site.list").items[0]; const dec = last(r, "decision.list"); const at = humanRelative(site.lastHealthAt, now()); return [ `下载器「${q.name}」${kindName(q.kind)} ${healthName(q.healthStatus)},管理 ${q.tasks.total} 个任务,${q.tasks.failed} 个失败;媒体服务器还没接入。`, `站点「${site.name}」上次检查${healthName(site.healthStatus)}(${at}),没做:实时检测,需要你批准。`, `待处理 ${dec.total} 条:${dec.items[0].title}。`, ].join("\n"); }, }, { id: "acquire", goal: "把《沙丘 2》4K 收进来", skill: "acquire-title", steps: [ { toolCalls: [{ capability: "catalog.lookup", input: { query: "沙丘 2" } }] }, { toolCalls: [{ capability: "inventory.query", input: { mediaEntityId: w.entityId } }] }, { toolCalls: [{ capability: "download.list", input: { mediaEntityId: w.entityId } }] }, { toolCalls: [{ capability: "release.search", input: { mediaEntityId: w.entityId } }] }, { toolCalls: [ { capability: "download.plan", input: (r) => ({ slotId: w.slot4k, sourceReleaseId: last(r, "release.search").items.find((x: Any) => /2160p/.test(x.title)).id, }), }, ], }, { toolCalls: [ { capability: "download.submit", input: (r) => ({ planId: last(r, "download.plan").plan?.id ?? w.r2160 }) }, ], }, ], calls: [ "catalog.lookup", "inventory.query", "download.list", "release.search", "download.plan", "download.submit", ], denied: ["download.submit"], report: (r) => { const rel = last(r, "release.search").items.find((x: Any) => /2160p/.test(x.title)); const gb = (rel.sizeBytes / 1024 ** 3).toFixed(1); const dl = last(r, "download.list").items[0]; return [ `《沙丘2》4K 找到了,还没提交:${rel.title}(${gb} GB,来自憨憨),没做:提交下载,需要你批准。`, `库里现有 1080p 正在下载(${dl.progress})。`, ].join("\n"); }, }, { id: "why", goal: "为什么沙丘2没入库", skill: "import-troubleshoot", steps: [ { toolCalls: [{ capability: "inventory.query", input: { query: "沙丘2" } }] }, { toolCalls: [{ capability: "download.list", input: { query: "沙丘2" } }] }, { toolCalls: [ { capability: "download.inspect", input: (r) => ({ downloadTaskId: last(r, "download.list").items[0].id }), }, ], }, ], calls: ["inventory.query", "download.list", "download.inspect"], report: (r) => { const it = last(r, "download.list").items[0]; return `《沙丘2》还没入库:还在${it.statusLabel},进度 ${it.progress},最近进展 ${it.lastProgress};下完会自动整理入库。`; }, }, { id: "subs", goal: "订阅追更情况", skill: "subscription-check", steps: [{ toolCalls: [{ capability: "subscription.list", input: { groupBy: "status" } }] }], calls: ["subscription.list"], report: (r) => { const s = last(r, "subscription.list"); const n = (k: string) => s.groups.find((g: Any) => g.key === k)?.count ?? 0; return `${s.total} 个订阅:${n("active")} 个在追更(沙丘2,收录版本还没满足),${n("cancelled")} 个已取消。`; }, }, { id: "failed", goal: "有没有失败的下载任务", skill: "download-diagnose", steps: [ { toolCalls: [{ capability: "download.list", input: { status: ["failed", "missing", "stalled"] } }] }, { toolCalls: [ { capability: "download.inspect", input: (r) => ({ downloadTaskId: last(r, "download.list").items[0].id }), }, ], }, ], calls: ["download.list", "download.inspect"], report: (r) => { const it = last(r, "download.list").items[0]; return `有 1 个下载失败:${it.name}(${it.statusLabel},${it.error ?? "原因未确认"})。其余下载正常。`; }, }, { id: "storage", goal: "空间不够了,看看能清理什么", skill: "storage-cleanup", steps: [ { toolCalls: [{ capability: "downloader.list", input: {} }] }, { toolCalls: [{ capability: "download.list", input: { status: ["imported"] } }] }, ], calls: ["downloader.list", "download.list"], report: (r) => { const l = last(r, "download.list"); const it = l.items[0]; return `下载缓存里有 ${l.total} 个已入库还在做种的种子可以清理:${it.name}(${it.size},${it.completed}完成)。在下载页点「清理缓存」会按保种规则只清可清理的。`; }, }, { id: "decisions", goal: "待处理里有什么要我处理的", skill: null, steps: [{ toolCalls: [{ capability: "decision.list", input: {} }] }], calls: ["decision.list"], report: (r) => { const d = last(r, "decision.list"); return `有 ${d.total} 条待处理:${d.items[0].title},可以在待处理中心选「重试」。`; }, }, { id: "site-login", goal: "站点憨憨的登录是不是掉了", skill: "patrol", steps: [{ toolCalls: [{ capability: "site.list", input: {} }] }], calls: ["site.list"], report: (r) => { const s = last(r, "site.list").items[0]; return `站点「${s.name}」登录正常,上次检查${healthName(s.healthStatus)}(${humanRelative(s.lastHealthAt, now())})。`; }, }, ]; } /** 评测用 fake 脚本:按 goal 精确匹配;finalText 用合格模型的汇报,并把工具结果交给 onDone */ export function evalScripts(cases: EvalCase[], onDone: (id: string, r: FakeToolResult[], text: string) => void) { return cases.map( (c): FakeScript => ({ name: `eval:${c.id}`, match: (goal) => goal === c.goal, steps: c.steps, finalText: (r) => { const text = c.report(r); onDone(c.id, r, text); return text; }, }), ); } export { lastResult };