364 lines
13 KiB
TypeScript
364 lines
13 KiB
TypeScript
import { type FakeScript, type FakeStep, type FakeToolResult, healthName, humanRelative, kindName } from "@umt/agent";
|
||
import { type E2E, seedBase, seedMovieTarget } from "../harness.ts";
|
||
|
||
// AG-5 智能体提示评测的共用夹具:测试库里的一套"用户环境"(对照 docs/design/agent-report-sample-bad.md 的现场)
|
||
// + 固定提示集。e16-agent-prompt-eval(fake 模型,CI)与 e16-agent-prompt-eval.real(真实模型,手动)共用。
|
||
// 站点地址指向本机关闭的端口:智能体即使实时补查也不会请求真实站点。
|
||
|
||
// biome-ignore lint/suspicious/noExplicitAny: 评测按输出结构取值
|
||
type Any = any;
|
||
|
||
export const HASHES = {
|
||
dune: "a1".repeat(20),
|
||
failed: "b2".repeat(20),
|
||
seeding: "c3".repeat(20),
|
||
} as const;
|
||
|
||
export interface EvalWorld {
|
||
siteId: string;
|
||
instanceId: string;
|
||
entityId: string;
|
||
slot4k: string;
|
||
r2160: string;
|
||
}
|
||
|
||
/** 下载器(mock qB,3 个任务:下载中 / 失败 / 已入库做种)+ 站点憨憨(上次检查正常、账号有用户数据)+ 沙丘2 + 5 条订阅 */
|
||
export async function seedEvalWorld(e: E2E): Promise<EvalWorld> {
|
||
const base = await seedBase(e, { key: "hhanclub", baseUrl: "http://127.0.0.1:9/" });
|
||
await e.db
|
||
.updateTable("site")
|
||
.set({
|
||
name: "憨憨",
|
||
health_status: "healthy",
|
||
last_health_at: new Date(Date.now() - 20 * 3600_000),
|
||
last_health: JSON.stringify({ ok: true, status: "ok", latencyMs: 1201, message: null }),
|
||
})
|
||
.where("id", "=", base.siteId)
|
||
.execute();
|
||
await e.db.updateTable("downloader_instance").set({ name: "家里的 qB" }).where("id", "=", base.instanceId).execute();
|
||
await e.db
|
||
.insertInto("site_account")
|
||
.values({
|
||
site_id: base.siteId,
|
||
label: "主账号",
|
||
auth_kind: "cookie",
|
||
status: "active",
|
||
user_data: JSON.stringify({
|
||
schema: "nexusphp",
|
||
userId: "1",
|
||
username: "u",
|
||
userLevel: "Power User",
|
||
vip: false,
|
||
joinAt: null,
|
||
upload: 17008345370067,
|
||
download: 0,
|
||
ratio: null,
|
||
bonus: 414799,
|
||
seeding: 24,
|
||
seedingSize: 79407286843,
|
||
leeching: 0,
|
||
leechingSize: 0,
|
||
messageUnread: 0,
|
||
}),
|
||
user_data_at: new Date(Date.now() - 3600_000),
|
||
})
|
||
.execute();
|
||
const target = await seedMovieTarget(e, base.siteId);
|
||
const slot4k = (
|
||
await e.db
|
||
.insertInto("acquisition_slot")
|
||
.values({
|
||
media_entity_id: target.entityId,
|
||
name: "2160p",
|
||
hard_conditions: JSON.stringify({ resolutions: ["2160p"] }),
|
||
})
|
||
.returning("id")
|
||
.executeTakeFirstOrThrow()
|
||
).id;
|
||
const r2160 = (
|
||
await e.db
|
||
.insertInto("source_release")
|
||
.values({
|
||
site_id: base.siteId,
|
||
site_release_id: "889001",
|
||
title: "Dune.Part.Two.2024.2160p.UHD.BluRay.x265-GRP",
|
||
size_bytes: 62_490_000_000,
|
||
media_entity_id: target.entityId,
|
||
resolution_status: "resolved",
|
||
raw: JSON.stringify({ _site: { hr: false } }),
|
||
})
|
||
.returning("id")
|
||
.executeTakeFirstOrThrow()
|
||
).id;
|
||
// 4 条已取消的订阅 + seedMovieTarget 的 1 条追更中
|
||
for (const t of ["繁花", "三体", "漫长的季节", "狂飙"])
|
||
await e.db
|
||
.insertInto("subscription")
|
||
.values({ title_query: t, media_kind: "series", status: "cancelled" })
|
||
.execute();
|
||
// 种子先进 mock qB,同步会把它们接管成任务行;之后按本用例需要覆盖状态与关联(upsert,和接管谁先谁后都一样)
|
||
const qb = [
|
||
[HASHES.dune, "Dune.Part.Two.2024.1080p.BluRay.x264-GRP", "downloading", 0.63],
|
||
[HASHES.failed, "Oppenheimer.2023.1080p.BluRay.x264", "missingFiles", 0.4],
|
||
[HASHES.seeding, "Poor.Things.2023.2160p.WEB-DL", "uploading", 1],
|
||
] as const;
|
||
for (const [hash, name, state, progress] of qb)
|
||
e.qb.seed({ hash, name, state, progress, size: 20_000_000_000, tags: ["umt"], category: "umt" });
|
||
const task = async (hash: string, name: string, status: string, progress: number, extra: Record<string, unknown>) => {
|
||
const values = {
|
||
instance_id: base.instanceId,
|
||
torrent_hash: hash,
|
||
infohash_v1: hash,
|
||
name,
|
||
status: status as never,
|
||
progress,
|
||
size_bytes: 20_000_000_000,
|
||
tags: ["umt"],
|
||
managed: true,
|
||
origin: "submitted" as const,
|
||
media_entity_id: target.entityId,
|
||
last_progress_at: new Date(Date.now() - 5 * 60_000),
|
||
...extra,
|
||
} as never;
|
||
await e.db
|
||
.insertInto("download_task")
|
||
.values(values)
|
||
.onConflict((oc) => oc.columns(["instance_id", "torrent_hash"]).doUpdateSet(values))
|
||
.execute();
|
||
};
|
||
await task(HASHES.dune, "Dune.Part.Two.2024.1080p.BluRay.x264-GRP", "downloading", 0.63, {
|
||
source_release_id: target.r1080,
|
||
});
|
||
await task(HASHES.failed, "Oppenheimer.2023.1080p.BluRay.x264", "failed", 0.4, {
|
||
media_entity_id: null,
|
||
last_error: JSON.stringify({ message: "下载器找不到数据文件" }),
|
||
});
|
||
await task(HASHES.seeding, "Poor.Things.2023.2160p.WEB-DL", "imported", 1, {
|
||
media_entity_id: null,
|
||
completed_at: new Date(Date.now() - 12 * 86400_000),
|
||
});
|
||
await e.db
|
||
.insertInto("decision")
|
||
.values({
|
||
kind: "download_failed",
|
||
title: "下载异常：Oppenheimer.2023",
|
||
subject_type: "download_task",
|
||
subject_id: (
|
||
await e.db
|
||
.selectFrom("download_task")
|
||
.select("id")
|
||
.where("torrent_hash", "=", HASHES.failed)
|
||
.executeTakeFirstOrThrow()
|
||
).id,
|
||
dedupe_key: `download_failed:eval`,
|
||
options: JSON.stringify([{ key: "retry", label: "重试" }]),
|
||
})
|
||
.execute();
|
||
return { siteId: base.siteId, instanceId: base.instanceId, entityId: target.entityId, slot4k, r2160 };
|
||
}
|
||
|
||
const last = (r: FakeToolResult[], cap: string): Any => [...r].reverse().find((x) => x.capabilityId === cap)?.json;
|
||
const lastResult = (r: FakeToolResult[], cap: string) => [...r].reverse().find((x) => x.capabilityId === cap);
|
||
|
||
export interface EvalCase {
|
||
id: string;
|
||
goal: string;
|
||
/** 期望按意图载入的 Skill(null = 不载入) */
|
||
skill: string | null;
|
||
/** fake 模型的调用脚本(按"合格模型"会走的路线写) */
|
||
steps: FakeStep[];
|
||
/** 期望的能力调用顺序 */
|
||
calls: string[];
|
||
/** 允许失败(被拒绝)的能力 */
|
||
denied?: string[];
|
||
/** 合格模型按工具结果写出的汇报(只用结果里的数据) */
|
||
report: (r: FakeToolResult[]) => string;
|
||
}
|
||
|
||
const now = () => new Date();
|
||
|
||
export function evalCases(w: EvalWorld): EvalCase[] {
|
||
return [
|
||
{
|
||
id: "health",
|
||
goal: "检查站点、下载器和媒体服务器是否正常",
|
||
skill: "patrol",
|
||
steps: [
|
||
{ toolCalls: [{ capability: "downloader.list", input: {} }] },
|
||
{ toolCalls: [{ capability: "site.list", input: {} }] },
|
||
{ toolCalls: [{ capability: "site.health_check", input: { siteId: w.siteId } }] },
|
||
{ toolCalls: [{ capability: "decision.list", input: {} }] },
|
||
],
|
||
calls: ["downloader.list", "site.list", "site.health_check", "decision.list"],
|
||
denied: ["site.health_check"],
|
||
report: (r) => {
|
||
const d = last(r, "downloader.list");
|
||
const q = d.downloaders[0];
|
||
const site = last(r, "site.list").items[0];
|
||
const dec = last(r, "decision.list");
|
||
const at = humanRelative(site.lastHealthAt, now());
|
||
return [
|
||
`下载器「${q.name}」${kindName(q.kind)} ${healthName(q.healthStatus)}，管理 ${q.tasks.total} 个任务，${q.tasks.failed} 个失败；媒体服务器还没接入。`,
|
||
`站点「${site.name}」上次检查${healthName(site.healthStatus)}（${at}），没做：实时检测，需要你批准。`,
|
||
`待处理 ${dec.total} 条：${dec.items[0].title}。`,
|
||
].join("\n");
|
||
},
|
||
},
|
||
{
|
||
id: "acquire",
|
||
goal: "把《沙丘 2》4K 收进来",
|
||
skill: "acquire-title",
|
||
steps: [
|
||
{ toolCalls: [{ capability: "catalog.lookup", input: { query: "沙丘 2" } }] },
|
||
{ toolCalls: [{ capability: "inventory.query", input: { mediaEntityId: w.entityId } }] },
|
||
{ toolCalls: [{ capability: "download.list", input: { mediaEntityId: w.entityId } }] },
|
||
{ toolCalls: [{ capability: "release.search", input: { mediaEntityId: w.entityId } }] },
|
||
{
|
||
toolCalls: [
|
||
{
|
||
capability: "download.plan",
|
||
input: (r) => ({
|
||
slotId: w.slot4k,
|
||
sourceReleaseId: last(r, "release.search").items.find((x: Any) => /2160p/.test(x.title)).id,
|
||
}),
|
||
},
|
||
],
|
||
},
|
||
{
|
||
toolCalls: [
|
||
{ capability: "download.submit", input: (r) => ({ planId: last(r, "download.plan").plan?.id ?? w.r2160 }) },
|
||
],
|
||
},
|
||
],
|
||
calls: [
|
||
"catalog.lookup",
|
||
"inventory.query",
|
||
"download.list",
|
||
"release.search",
|
||
"download.plan",
|
||
"download.submit",
|
||
],
|
||
denied: ["download.submit"],
|
||
report: (r) => {
|
||
const rel = last(r, "release.search").items.find((x: Any) => /2160p/.test(x.title));
|
||
const gb = (rel.sizeBytes / 1024 ** 3).toFixed(1);
|
||
const dl = last(r, "download.list").items[0];
|
||
return [
|
||
`《沙丘2》4K 找到了，还没提交：${rel.title}（${gb} GB，来自憨憨），没做：提交下载，需要你批准。`,
|
||
`库里现有 1080p 正在下载（${dl.progress}）。`,
|
||
].join("\n");
|
||
},
|
||
},
|
||
{
|
||
id: "why",
|
||
goal: "为什么沙丘2没入库",
|
||
skill: "import-troubleshoot",
|
||
steps: [
|
||
{ toolCalls: [{ capability: "inventory.query", input: { query: "沙丘2" } }] },
|
||
{ toolCalls: [{ capability: "download.list", input: { query: "沙丘2" } }] },
|
||
{
|
||
toolCalls: [
|
||
{
|
||
capability: "download.inspect",
|
||
input: (r) => ({ downloadTaskId: last(r, "download.list").items[0].id }),
|
||
},
|
||
],
|
||
},
|
||
],
|
||
calls: ["inventory.query", "download.list", "download.inspect"],
|
||
report: (r) => {
|
||
const it = last(r, "download.list").items[0];
|
||
return `《沙丘2》还没入库：还在${it.statusLabel}，进度 ${it.progress}，最近进展 ${it.lastProgress}；下完会自动整理入库。`;
|
||
},
|
||
},
|
||
{
|
||
id: "subs",
|
||
goal: "订阅追更情况",
|
||
skill: "subscription-check",
|
||
steps: [{ toolCalls: [{ capability: "subscription.list", input: { groupBy: "status" } }] }],
|
||
calls: ["subscription.list"],
|
||
report: (r) => {
|
||
const s = last(r, "subscription.list");
|
||
const n = (k: string) => s.groups.find((g: Any) => g.key === k)?.count ?? 0;
|
||
return `${s.total} 个订阅：${n("active")} 个在追更（沙丘2，收录版本还没满足），${n("cancelled")} 个已取消。`;
|
||
},
|
||
},
|
||
{
|
||
id: "failed",
|
||
goal: "有没有失败的下载任务",
|
||
skill: "download-diagnose",
|
||
steps: [
|
||
{ toolCalls: [{ capability: "download.list", input: { status: ["failed", "missing", "stalled"] } }] },
|
||
{
|
||
toolCalls: [
|
||
{
|
||
capability: "download.inspect",
|
||
input: (r) => ({ downloadTaskId: last(r, "download.list").items[0].id }),
|
||
},
|
||
],
|
||
},
|
||
],
|
||
calls: ["download.list", "download.inspect"],
|
||
report: (r) => {
|
||
const it = last(r, "download.list").items[0];
|
||
return `有 1 个下载失败：${it.name}（${it.statusLabel}，${it.error ?? "原因未确认"}）。其余下载正常。`;
|
||
},
|
||
},
|
||
{
|
||
id: "storage",
|
||
goal: "空间不够了，看看能清理什么",
|
||
skill: "storage-cleanup",
|
||
steps: [
|
||
{ toolCalls: [{ capability: "downloader.list", input: {} }] },
|
||
{ toolCalls: [{ capability: "download.list", input: { status: ["imported"] } }] },
|
||
],
|
||
calls: ["downloader.list", "download.list"],
|
||
report: (r) => {
|
||
const l = last(r, "download.list");
|
||
const it = l.items[0];
|
||
return `下载缓存里有 ${l.total} 个已入库还在做种的种子可以清理：${it.name}（${it.size}，${it.completed}完成）。在下载页点「清理缓存」会按保种规则只清可清理的。`;
|
||
},
|
||
},
|
||
{
|
||
id: "decisions",
|
||
goal: "待处理里有什么要我处理的",
|
||
skill: null,
|
||
steps: [{ toolCalls: [{ capability: "decision.list", input: {} }] }],
|
||
calls: ["decision.list"],
|
||
report: (r) => {
|
||
const d = last(r, "decision.list");
|
||
return `有 ${d.total} 条待处理：${d.items[0].title}，可以在待处理中心选「重试」。`;
|
||
},
|
||
},
|
||
{
|
||
id: "site-login",
|
||
goal: "站点憨憨的登录是不是掉了",
|
||
skill: "patrol",
|
||
steps: [{ toolCalls: [{ capability: "site.list", input: {} }] }],
|
||
calls: ["site.list"],
|
||
report: (r) => {
|
||
const s = last(r, "site.list").items[0];
|
||
return `站点「${s.name}」登录正常，上次检查${healthName(s.healthStatus)}（${humanRelative(s.lastHealthAt, now())}）。`;
|
||
},
|
||
},
|
||
];
|
||
}
|
||
|
||
/** 评测用 fake 脚本:按 goal 精确匹配;finalText 用合格模型的汇报,并把工具结果交给 onDone */
|
||
export function evalScripts(cases: EvalCase[], onDone: (id: string, r: FakeToolResult[], text: string) => void) {
|
||
return cases.map(
|
||
(c): FakeScript => ({
|
||
name: `eval:${c.id}`,
|
||
match: (goal) => goal === c.goal,
|
||
steps: c.steps,
|
||
finalText: (r) => {
|
||
const text = c.report(r);
|
||
onDone(c.id, r, text);
|
||
return text;
|
||
},
|
||
}),
|
||
);
|
||
}
|
||
|
||
export { lastResult };
|