197 lines
8.7 KiB
TypeScript
197 lines
8.7 KiB
TypeScript
import type { FakeScript, FakeToolResult } from "@umt/agent";
|
|
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
|
import { resolveBusiness } from "../../src/modules/auth-settings/engine-business.ts";
|
|
import { createE2E, type E2E, pgAvailable, runEvents, runRow, seedBase, seedMovieTarget, waitFor } from "./harness.ts";
|
|
|
|
// E15 Agent 质量(AG-3,来自 AGENT-REAL 真实上游验证):
|
|
// - 只读摘要能力 subscription.list / decision.list / download.inspect(按 id 或 hash);task.inspect 收到下载任务 ID
|
|
// 时明确报错并指向 download.inspect;
|
|
// - download.plan 的 requiresApproval 按授权范围判定(覆盖本作品的持续授权 → false;范围外 / 没有 → true);
|
|
// - 超预算:run = budget_exceeded,任务页(引擎 run 的 business.outcome)显示超出预算而不是完成;
|
|
// 被预算截断时中间过程文字不作为汇报;同一目标模板的超预算合并到同一条待处理事项。
|
|
|
|
// biome-ignore lint/suspicious/noExplicitAny: 断言里按输出结构取值
|
|
type Any = any;
|
|
const hasPg = await pgAvailable();
|
|
const HASH = "ab".repeat(20);
|
|
const SURVEY = "盘点订阅、待处理与下载";
|
|
const LOOP = (at: string) => `巡检库存(${at})：逐个作品查看版本目标`;
|
|
|
|
describe.skipIf(!hasPg)("E15 Agent 质量(fake 模型 + 真实能力)", () => {
|
|
const scripts: FakeScript[] = [];
|
|
const results = new Map<string, FakeToolResult[]>();
|
|
let e: E2E;
|
|
let target: Awaited<ReturnType<typeof seedMovieTarget>>;
|
|
let downloadTaskId: string;
|
|
|
|
beforeAll(async () => {
|
|
e = await createE2E("e15", { fakeScripts: scripts });
|
|
const base = await seedBase(e);
|
|
target = await seedMovieTarget(e, base.siteId);
|
|
downloadTaskId = (
|
|
await e.db
|
|
.insertInto("download_task")
|
|
.values({
|
|
instance_id: base.instanceId,
|
|
torrent_hash: HASH,
|
|
infohash_v1: HASH,
|
|
source_release_id: target.r1080,
|
|
name: "Dune.Part.Two.2024.1080p.BluRay.x264-GRP",
|
|
status: "failed",
|
|
last_error: JSON.stringify({ message: "qB 状态 missingFiles" }),
|
|
})
|
|
.returning("id")
|
|
.executeTakeFirstOrThrow()
|
|
).id;
|
|
await e.db
|
|
.insertInto("decision")
|
|
.values({
|
|
kind: "download_failed",
|
|
title: "下载异常：Dune.Part.Two",
|
|
subject_type: "download_task",
|
|
subject_id: downloadTaskId,
|
|
dedupe_key: `download_failed:${downloadTaskId}`,
|
|
options: JSON.stringify([{ key: "retry", label: "重试" }]),
|
|
})
|
|
.execute();
|
|
scripts.push(
|
|
{
|
|
match: SURVEY,
|
|
steps: [
|
|
{ text: "先列订阅。", toolCalls: [{ capability: "subscription.list", input: { groupBy: "status" } }] },
|
|
{ text: "再看待处理。", toolCalls: [{ capability: "decision.list", input: {} }] },
|
|
{
|
|
text: "按 hash 查下载。",
|
|
toolCalls: [{ capability: "download.inspect", input: { hash: HASH.toUpperCase() } }],
|
|
},
|
|
// 常见误用:把下载任务 ID 当业务任务 ID
|
|
{ text: "看任务时间线。", toolCalls: [{ capability: "task.inspect", input: { taskId: downloadTaskId } }] },
|
|
],
|
|
finalText: (r) => {
|
|
results.set(SURVEY, r);
|
|
return "## 盘点\n\n- 订阅 1 个\n- 待处理 1 条\n\n| 下载 | 状态 |\n|---|---|\n| Dune | failed |";
|
|
},
|
|
},
|
|
{
|
|
match: /^巡检库存/,
|
|
steps: Array.from({ length: 6 }, () => ({
|
|
text: "Let me check the inventory once more.",
|
|
toolCalls: [{ capability: "inventory.query", input: { query: "沙丘2" } }],
|
|
})),
|
|
finalText: "不该到这里",
|
|
},
|
|
);
|
|
}, 120_000);
|
|
|
|
afterAll(async () => {
|
|
await e?.close();
|
|
}, 60_000);
|
|
|
|
const runToEnd = async (goal: string) => {
|
|
const r = await e.api("POST", "/agent/runs", { goal });
|
|
expect(r.status).toBe(201);
|
|
const runId = r.body.data.id as string;
|
|
const run = await waitFor(
|
|
async () => {
|
|
const x = await runRow(e, runId);
|
|
return ["succeeded", "failed", "waiting", "budget_exceeded", "cancelled"].includes(x.status) ? x : null;
|
|
},
|
|
60_000,
|
|
`run 结束:${goal}`,
|
|
);
|
|
return { runId, run };
|
|
};
|
|
|
|
it("只读摘要能力:订阅分组、待处理含 options、下载按 hash;task.inspect 收到下载任务 ID 时指向 download.inspect", async () => {
|
|
const { run } = await runToEnd(SURVEY);
|
|
expect(run.status).toBe("succeeded");
|
|
const r = results.get(SURVEY)!;
|
|
const byCap = (id: string) => r.find((x) => x.capabilityId === id)!;
|
|
expect(byCap("subscription.list")).toMatchObject({ isError: false });
|
|
expect(byCap("subscription.list").json).toMatchObject({
|
|
total: 1,
|
|
groups: [{ key: "active", count: 1 }],
|
|
items: [{ title: "沙丘2", mediaEntityId: target.entityId, slots: { total: 1, satisfied: 0 } }],
|
|
});
|
|
expect(byCap("decision.list").json).toMatchObject({
|
|
total: 1,
|
|
byKind: [{ kind: "download_failed", count: 1 }],
|
|
items: [{ kind: "download_failed", subjectId: downloadTaskId, options: [{ key: "retry", label: "重试" }] }],
|
|
});
|
|
expect(byCap("download.inspect").json).toMatchObject({
|
|
downloadTask: { id: downloadTaskId, torrentHash: HASH },
|
|
release: { id: target.r1080 },
|
|
decisions: [{ kind: "download_failed" }],
|
|
otherMatches: [],
|
|
});
|
|
expect(byCap("task.inspect")).toMatchObject({ isError: true });
|
|
expect(byCap("task.inspect").text).toContain("download.inspect");
|
|
|
|
// HTTP 同一路径:按 id 查;不存在的下载任务 404 类错误
|
|
const inv = await e.api("POST", "/capabilities/download.inspect/invoke", { input: { downloadTaskId } });
|
|
expect(inv.status).toBe(200);
|
|
expect(JSON.stringify(inv.body)).toContain(HASH);
|
|
const bad = await e.api("POST", "/capabilities/release.search/invoke", { input: { query: "a" } });
|
|
expect(bad.status).toBeGreaterThanOrEqual(400);
|
|
expect(JSON.stringify(bad.body)).toContain("检索词太短");
|
|
}, 90_000);
|
|
|
|
it("download.plan 的 requiresApproval 按授权范围判定", async () => {
|
|
const plan = async () => {
|
|
const r = await e.api("POST", "/capabilities/download.plan/invoke", {
|
|
input: { slotId: target.slotId, sourceReleaseId: target.r1080 },
|
|
});
|
|
expect(r.status).toBe(200);
|
|
return r.body.data.output ?? r.body.data;
|
|
};
|
|
expect((await plan()).requiresApproval).toBe(true);
|
|
// 范围外的持续授权不算覆盖
|
|
const other = await e.api("POST", "/automation-grants", {
|
|
action: "download.submit",
|
|
scope: { mediaEntityIds: ["00000000-0000-4000-8000-000000000001"] },
|
|
reason: "e2e 范围外授权",
|
|
});
|
|
expect(other.status).toBe(201);
|
|
expect((await plan()).requiresApproval).toBe(true);
|
|
const g = await e.api("POST", "/automation-grants", {
|
|
action: "download.submit",
|
|
scope: { mediaEntityIds: [target.entityId] },
|
|
reason: "e2e 授权",
|
|
});
|
|
expect(g.status).toBe(201);
|
|
expect((await plan()).requiresApproval).toBe(false);
|
|
}, 60_000);
|
|
|
|
it("超预算:状态为超出预算(任务页同样),中间过程不当汇报,同一目标模板只一条待处理事项", async () => {
|
|
await e.settings({ agent: { maxToolCalls: 3 } });
|
|
const a = await runToEnd(LOOP("2026-09-29T01:00:00.000Z"));
|
|
expect(a.run.status).toBe("budget_exceeded");
|
|
expect(a.run.summary).not.toContain("Let me check");
|
|
expect(a.run.summary).not.toContain("模型汇报");
|
|
// 预算将尽提醒进了工具结果
|
|
const ends = (await runEvents(e, a.runId)).filter((x) => x.type === "tool_call_end");
|
|
expect(ends.length).toBeGreaterThan(0);
|
|
|
|
// 引擎 run 正常跑完,业务结果是超出预算:任务页按 business.outcome 显示「超出预算」而不是「完成」
|
|
const task = await e.db
|
|
.selectFrom("task")
|
|
.select(["engine_run_id", "status", "output"])
|
|
.where("id", "=", a.run.task_id!)
|
|
.executeTakeFirstOrThrow();
|
|
expect(task.status).toBe("failed");
|
|
expect((task.output as Any).status).toBe("budget_exceeded");
|
|
const biz = await resolveBusiness(e.db, e.ow, [task.engine_run_id!]);
|
|
const row = { business: biz.get(task.engine_run_id!) };
|
|
expect(row?.business).toMatchObject({ kind: "agent", outcome: "budget_exceeded" });
|
|
|
|
const b = await runToEnd(LOOP("2026-09-29T02:00:00.000Z"));
|
|
expect(b.run.status).toBe("budget_exceeded");
|
|
const ds = await e.db.selectFrom("decision").selectAll().where("dedupe_key", "like", "agent_budget:%").execute();
|
|
expect(ds).toHaveLength(1);
|
|
expect(ds[0]!.dedupe_key).toMatch(/^agent_budget:goal:/);
|
|
expect(ds[0]!.agent_run_id).toBe(b.runId);
|
|
expect((ds[0]!.evidence as Any).runIds.sort()).toEqual([a.runId, b.runId].sort());
|
|
expect(ds[0]!.attempts).toBeGreaterThanOrEqual(1);
|
|
}, 120_000);
|
|
});
|