Files
UHD_Media_Tool/apps/server/test/e2e/e2-agent-delegate.test.ts
T
rootandClaude Opus 5.5 c437c6bff6 feat(browser): vision automation loop with screenshots, coordinate actions and bounded challenge solving
- browser.screenshot returns the page as an image content block (≤1280px wide, JPEG quality
  configurable) for vision-capable upstreams; non-vision upstreams get a redacted text snapshot.
  Login pages / password fields are marked sensitive and withheld unless a
  browser.sensitive_screenshot grant exists and allowSensitive is passed.
- browser.click_at / type_at / scroll / wait act in screenshot pixels; the server converts via
  viewport/DPR, requires automation control and an allowlisted host, and writes a thumbnail
  evidence file (DATA_DIR/tmp/evidence) referenced from agent_run_event.
- browser.solve_challenge runs a bounded child agent_run (vision caps only, maxSteps ≤ 12,
  budget), judges success from page facts, stops immediately on 2FA / SMS / device confirmation,
  and otherwise moves the session to waiting with evidence and a takeover link.
- PiAgentRuntime passes modelImage outputs as pi-ai image content when model.input has image;
  the ledger never stores image data; coordinate actions use per-invocation idempotency keys.
- Tests: image codec, runtime image content, E14 e2e (login solved, 2FA stop, step limit,
  sensitive withheld). Real camofox 1.17.0 check on example.com recorded in the runbook.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-09-29 01:27:48 +08:00

184 lines
7.5 KiB
TypeScript

import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { delegateScript } from "./fixtures/scripts.ts";
import {
completeDownload,
createE2E,
type E2E,
pgAvailable,
runEvents,
runRow,
seedBase,
seedMovieTarget,
waitFor,
} from "./harness.ts";
// E2 Agent 委托闭环:fake 模型按"把《沙丘2》1080p 收进来"依次调用
// inventory.query → release.search → download.plan → download.submit → task.inspect。
// - 没有持续授权时 download.submit 被服务端拒绝(模型文本"用户已同意"无效),SSE 上是 denied;
// - 发了 automation_grant 后同一目标成功提交;模型汇报"已入库",但 run 的汇报来自整理记录:
// 下载完成前是 waiting(已提交≠已入库),核验入库后才 succeeded 并带整理记录 ID。
const hasPg = await pgAvailable();
const GOAL = "把《沙丘2》1080p 收进来";
describe.skipIf(!hasPg)("E2 Agent 委托闭环(fake 模型 + 真实能力/授权/任务)", () => {
let e: E2E;
let target: Awaited<ReturnType<typeof seedMovieTarget>>;
beforeAll(async () => {
e = await createE2E("e2", { fakeScripts: [delegateScript] });
const base = await seedBase(e);
target = await seedMovieTarget(e, base.siteId);
}, 120_000);
afterAll(async () => {
await e?.close();
}, 60_000);
const createRun = async () => {
const r = await e.api("POST", "/agent/runs", { goal: GOAL });
expect(r.status).toBe(201);
expect(r.body.data).toMatchObject({ status: "queued", trigger: "user" });
// 运行模式已取消(AG-2):API 不再返回 mode;API Key / 开发密钥发起的会话不出会话内审批卡
expect(r.body.data.mode).toBeUndefined();
expect(r.body.data.scope).not.toHaveProperty("interactive");
return r.body.data.id as string;
};
it("启动断言:35 个能力全部注册,并在能力目录里可见", async () => {
expect(e.booted.ctx.capabilities.list()).toHaveLength(35);
const r = await e.api("GET", "/capabilities");
expect(r.body.data.items.map((c: { id: string }) => c.id).sort()).toEqual(
e.booted.ctx.capabilities
.list()
.map((c) => c.descriptor.id)
.sort(),
);
});
it("没有授权:download.submit 被拒(模型声称已同意无效),不产生下载", async () => {
const seen: { type: string; payload: { phase?: string; capabilityId?: string } }[] = [];
const off = e.bus.subscribe((ev) => {
if (ev.type.startsWith("agent.")) seen.push(ev as never);
});
const runId = await createRun();
const run = await waitFor(
async () => {
const r = await runRow(e, runId);
return ["succeeded", "failed", "waiting"].includes(r.status) ? r : null;
},
60_000,
"run 结束",
);
off();
expect(run.status).toBe("succeeded");
const events = await runEvents(e, runId);
const submitEnd = events.find(
(x) => x.type === "tool_call_end" && (x.payload as { capabilityId: string }).capabilityId === "download.submit",
);
expect(submitEnd?.payload).toMatchObject({ isError: true });
expect(JSON.stringify(submitEnd?.payload)).toContain("unauthorized");
// SSE:未授权的调用标为 denied
expect(
seen.some(
(x) =>
x.type === "agent.tool_call" && x.payload.capabilityId === "download.submit" && x.payload.phase === "denied",
),
).toBe(true);
// 审计:capability.denied
const denied = await e.db
.selectFrom("audit_event")
.selectAll()
.where("action", "=", "capability.denied")
.where("target_id", "=", "download.submit")
.execute();
expect(denied.length).toBeGreaterThanOrEqual(1);
expect(await e.db.selectFrom("download_task").select("id").execute()).toHaveLength(0);
// 草案计划存在,但没有预留 Slot
const plans = await e.db.selectFrom("acquisition_plan").selectAll().execute();
expect(plans.map((p) => p.status)).toEqual(["draft"]);
const slot = await e.db
.selectFrom("acquisition_slot")
.select("status")
.where("id", "=", target.slotId)
.executeTakeFirstOrThrow();
expect(slot.status).toBe("empty");
// 汇报不采信模型文本:没有可核查的业务变更
expect(run.summary).toContain("没有提交下载");
expect(run.summary).toContain("模型汇报(未经核查)");
}, 90_000);
it("有持续授权:提交成功 → 已提交≠已入库(waiting)→ 下载完成、核验入库后 succeeded,汇报来自整理记录", async () => {
const g = await e.api("POST", "/automation-grants", {
action: "download.submit",
scope: { mediaEntityIds: [target.entityId] },
reason: "e2e 授权",
});
expect(g.status).toBe(201);
const runId = await createRun();
// 模型已经说完"已入库",但业务上只是已提交:run 进入 waiting
const waiting = await waitFor(
async () => {
const r = await runRow(e, runId);
return r.status === "waiting" ? r : null;
},
60_000,
"run waiting",
);
expect(waiting.summary).toContain("已提交≠已入库");
const events = await runEvents(e, runId);
const completed = events.find((x) => x.type === "run_completed");
expect((completed!.payload as { finalText: string }).finalText).toContain("已入库");
const calls = events
.filter((x) => x.type === "tool_call_start")
.map((x) => (x.payload as { capabilityId: string }).capabilityId);
expect(calls).toEqual(["inventory.query", "release.search", "download.plan", "download.submit", "task.inspect"]);
const submitEnd = events.find(
(x) => x.type === "tool_call_end" && (x.payload as { capabilityId: string }).capabilityId === "download.submit",
);
expect(submitEnd?.payload).toMatchObject({ isError: false });
// 账本:download.submit 记在本 run 名下,带授权 ID
const inv = await e.db
.selectFrom("capability_invocation")
.selectAll()
.where("agent_run_id", "=", runId)
.where("capability_id", "=", "download.submit")
.executeTakeFirstOrThrow();
expect(inv).toMatchObject({ status: "succeeded", grant_id: g.body.data.id });
// 草案被转为预留并提交
await completeDownload(e, "Dune.Part.Two.2024.1080p.BluRay.x264-GRP");
const done = await waitFor(
async () => {
const r = await runRow(e, runId);
return r.status === "succeeded" ? r : null;
},
90_000,
"run succeeded",
);
const rec = await e.db
.selectFrom("organization_record")
.selectAll()
.where("status", "=", "succeeded")
.executeTakeFirstOrThrow();
expect(done.summary).toContain("已入库 1 个");
expect(done.summary).toContain(rec.id);
const detail = await e.api("GET", `/agent/runs/${runId}?afterSeq=-1&limit=500`);
expect(detail.body.data.report).toMatchObject({ state: "completed", imported: 1, submitted: 1 });
expect(detail.body.data.report.organization[0]).toMatchObject({ recordId: rec.id, status: "succeeded" });
// 断线补读:afterSeq 之后的事件
const tail = await e.api("GET", `/agent/runs/${runId}?afterSeq=3&limit=2`);
expect(tail.body.data.events.map((x: { seq: number }) => x.seq)).toEqual([4, 5]);
expect(tail.body.data.nextAfterSeq).toBe(5);
// 入库成功:Slot 满足
const slot = await e.db
.selectFrom("acquisition_slot")
.select("status")
.where("id", "=", target.slotId)
.executeTakeFirstOrThrow();
expect(slot.status).toBe("satisfied");
expect(e.qb.callsTo("torrents/add")).toHaveLength(1);
}, 180_000);
});