rootandClaude Opus 5.5
c437c6bff6
- browser.screenshot returns the page as an image content block (≤1280px wide, JPEG quality configurable) for vision-capable upstreams; non-vision upstreams get a redacted text snapshot. Login pages / password fields are marked sensitive and withheld unless a browser.sensitive_screenshot grant exists and allowSensitive is passed. - browser.click_at / type_at / scroll / wait act in screenshot pixels; the server converts via viewport/DPR, requires automation control and an allowlisted host, and writes a thumbnail evidence file (DATA_DIR/tmp/evidence) referenced from agent_run_event. - browser.solve_challenge runs a bounded child agent_run (vision caps only, maxSteps ≤ 12, budget), judges success from page facts, stops immediately on 2FA / SMS / device confirmation, and otherwise moves the session to waiting with evidence and a takeover link. - PiAgentRuntime passes modelImage outputs as pi-ai image content when model.input has image; the ledger never stores image data; coordinate actions use per-invocation idempotency keys. - Tests: image codec, runtime image content, E14 e2e (login solved, 2FA stop, step limit, sensitive withheld). Real camofox 1.17.0 check on example.com recorded in the runbook. Co-Authored-By: Claude Opus 5.5 <[email protected]>
357 lines
15 KiB
TypeScript
357 lines
15 KiB
TypeScript
import { existsSync } from "node:fs";
|
||
import { mkdtemp, rm } from "node:fs/promises";
|
||
import { tmpdir } from "node:os";
|
||
import { join } from "node:path";
|
||
import type { FakeScript, FakeToolResult, InvokeContext } from "@umt/agent";
|
||
import { createMockCamofox, type MockCamofox } from "@umt/testkit";
|
||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||
import { browserServiceFor } from "../../src/modules/browser/index.ts";
|
||
import type { BrowserService } from "../../src/modules/browser/service.ts";
|
||
import { createE2E, type E2E, pgAvailable, runEvents, runRow, waitFor } from "./harness.ts";
|
||
|
||
// E14 视觉自动化闭环(VIS-1):fake 视觉模型(脚本返回固定坐标)+ testkit mock camofox(视口 1600×1000、DPR 1,
|
||
// 截图缩到 1280×800 交给模型,坐标按 0.8 换算回页面)。
|
||
// - 主会话调用 browser.solve_challenge:子流程 截图 → 点用户名框输入 → 点密码框输入并回车 → 成功条件(地址含 /torrents.php)满足;
|
||
// - 二次验证页:服务端识别后会话立即转等待处理,子流程后续动作被拒,不输入短信验证码;
|
||
// - 步数超限:超出 maxSteps 转等待处理(challenge_unsolved),待处理事项附证据截图;
|
||
// - 敏感截图(登录页 / 密码框)默认不交给模型,改给已脱敏的文字快照。
|
||
|
||
const hasPg = await pgAvailable();
|
||
// biome-ignore lint/suspicious/noExplicitAny: 测试断言直接读 JSON
|
||
type Any = any;
|
||
|
||
const ORIGIN = "https://audiences.example";
|
||
const ORIGIN_2FA = "https://secure.example";
|
||
const USER = "fixture-user";
|
||
const PASSWORD = "e14-login-pw";
|
||
const ACCESS_KEY = "camofox-e14-key";
|
||
|
||
/** 登录页各行在 1280×800 截图上的坐标(页面 CSS 行中心 y = 40 + 40i,x = 300;×0.8) */
|
||
const AT = { username: { x: 240, y: 64 }, password: { x: 240, y: 96 } };
|
||
|
||
let sid = "";
|
||
const childResults = new Map<string, FakeToolResult[]>();
|
||
let mainResults: FakeToolResult[] = [];
|
||
|
||
const typeUser = () => ({ sessionId: sid, ...AT.username, text: USER, allowSensitive: true });
|
||
const typePassword = () => ({ sessionId: sid, ...AT.password, text: PASSWORD, pressEnter: true, allowSensitive: true });
|
||
|
||
const scripts: FakeScript[] = [
|
||
{
|
||
name: "e14-main",
|
||
match: "E14 主会话登录",
|
||
steps: [
|
||
{
|
||
toolCalls: [
|
||
{
|
||
capability: "browser.solve_challenge",
|
||
input: () => ({
|
||
sessionId: sid,
|
||
goal: "E14-登录 用测试账号登录站点",
|
||
success: { urlContains: "/torrents.php" },
|
||
maxSteps: 6,
|
||
allowSensitive: true,
|
||
}),
|
||
},
|
||
],
|
||
},
|
||
],
|
||
finalText: (r) => {
|
||
mainResults = r;
|
||
return `挑战处理结果:${(r[0]?.json as Any)?.status ?? r[0]?.text}`;
|
||
},
|
||
},
|
||
{
|
||
name: "e14-login",
|
||
match: "E14-登录",
|
||
steps: [
|
||
{ toolCalls: [{ capability: "browser.screenshot", input: () => ({ sessionId: sid, allowSensitive: true }) }] },
|
||
{ toolCalls: [{ capability: "browser.type_at", input: typeUser }] },
|
||
{ toolCalls: [{ capability: "browser.type_at", input: typePassword }] },
|
||
],
|
||
finalText: (r) => {
|
||
childResults.set("login", r);
|
||
return "已提交登录表单";
|
||
},
|
||
},
|
||
{
|
||
name: "e14-2fa",
|
||
match: "E14-二次验证",
|
||
steps: [
|
||
{ toolCalls: [{ capability: "browser.screenshot", input: () => ({ sessionId: sid, allowSensitive: true }) }] },
|
||
{ toolCalls: [{ capability: "browser.type_at", input: typeUser }] },
|
||
{ toolCalls: [{ capability: "browser.type_at", input: typePassword }] },
|
||
// 模型不听话继续输短信验证码:服务端已转等待处理,必须被拒
|
||
{
|
||
toolCalls: [
|
||
{ capability: "browser.type_at", input: () => ({ sessionId: sid, x: 240, y: 96, text: "123456" }) },
|
||
],
|
||
},
|
||
],
|
||
finalText: (r) => {
|
||
childResults.set("2fa", r);
|
||
return "NEEDS_HUMAN: 需要短信验证码";
|
||
},
|
||
},
|
||
{
|
||
name: "e14-steps",
|
||
match: "E14-步数",
|
||
steps: [
|
||
{ toolCalls: [{ capability: "browser.screenshot", input: () => ({ sessionId: sid }) }] },
|
||
{ toolCalls: [{ capability: "browser.scroll", input: () => ({ sessionId: sid, dy: 300 }) }] },
|
||
{ toolCalls: [{ capability: "browser.wait", input: () => ({ sessionId: sid, ms: 50 }) }] },
|
||
{ toolCalls: [{ capability: "browser.wait", input: () => ({ sessionId: sid, ms: 50 }) }] },
|
||
],
|
||
finalText: (r) => {
|
||
childResults.set("steps", r);
|
||
return "还没完成";
|
||
},
|
||
},
|
||
{
|
||
name: "e14-sensitive",
|
||
match: "E14-敏感",
|
||
steps: [{ toolCalls: [{ capability: "browser.screenshot", input: () => ({ sessionId: sid }) }] }],
|
||
finalText: (r) => {
|
||
childResults.set("sensitive", r);
|
||
return "NEEDS_HUMAN: 截图不可见，登录页需要人工";
|
||
},
|
||
},
|
||
];
|
||
|
||
describe.skipIf(!hasPg)("E14 视觉自动化闭环", () => {
|
||
let e: E2E;
|
||
let mock: MockCamofox;
|
||
let mock2fa: MockCamofox;
|
||
let evidenceDir: string;
|
||
let svc: BrowserService;
|
||
const ids = { site: "", site2fa: "", provider: "", provider2fa: "" };
|
||
|
||
const agentCtx = (): InvokeContext => ({
|
||
actorId: "agent:e14",
|
||
taskId: crypto.randomUUID(),
|
||
scopeIds: [],
|
||
signal: new AbortController().signal,
|
||
source: "agent",
|
||
invocationId: crypto.randomUUID(),
|
||
});
|
||
const solve = (input: Record<string, unknown>) =>
|
||
e.booted.ctx.capabilities.invoke("browser.solve_challenge", input, agentCtx()) as Promise<Any>;
|
||
const open = async (siteId: string, providerId: string, url: string) => {
|
||
for (const s of await svc.list({ siteId })) if (s.controller !== "closed") await svc.close(s.id);
|
||
sid = await svc.open({ siteId, providerId, url });
|
||
return sid;
|
||
};
|
||
|
||
beforeAll(async () => {
|
||
const site = (origin: string) => ({ origin, username: USER, password: PASSWORD });
|
||
mock = await createMockCamofox({
|
||
accessKey: ACCESS_KEY,
|
||
site: site(ORIGIN),
|
||
viewport: { width: 1600, height: 1000 },
|
||
});
|
||
mock2fa = await createMockCamofox({
|
||
accessKey: ACCESS_KEY,
|
||
site: { ...site(ORIGIN_2FA), twoFactor: true },
|
||
viewport: { width: 1600, height: 1000 },
|
||
});
|
||
evidenceDir = await mkdtemp(join(tmpdir(), "umt-e14-evidence-"));
|
||
e = await createE2E("e14", { fakeScripts: scripts, browser: { evidenceDir, settleMs: 0 } });
|
||
svc = browserServiceFor(e.booted.ctx)!;
|
||
const ctx = e.booted.ctx;
|
||
const keyRef = await ctx.secrets.create({ purpose: "other", plaintext: ACCESS_KEY });
|
||
const provider = async (name: string, url: string) =>
|
||
(
|
||
await e.db
|
||
.insertInto("browser_provider")
|
||
.values({
|
||
name,
|
||
kind: "camofox",
|
||
base_url: url,
|
||
secret_ref: keyRef,
|
||
enabled: true,
|
||
config: JSON.stringify({ navigationTimeoutMs: 2_000, maxRetries: 0 }),
|
||
})
|
||
.returning("id")
|
||
.executeTakeFirstOrThrow()
|
||
).id;
|
||
ids.provider = await provider("camofox-e14", mock.url);
|
||
ids.provider2fa = await provider("camofox-e14-2fa", mock2fa.url);
|
||
const addSite = async (key: string, origin: string) =>
|
||
(
|
||
await e.db
|
||
.insertInto("site")
|
||
.values({
|
||
key,
|
||
name: key,
|
||
adapter: "nexusphp",
|
||
base_url: `${origin}/`,
|
||
rate_limit: JSON.stringify({ minIntervalMs: 0, concurrency: 5 }),
|
||
})
|
||
.returning("id")
|
||
.executeTakeFirstOrThrow()
|
||
).id;
|
||
ids.site = await addSite("e14site", ORIGIN);
|
||
ids.site2fa = await addSite("e14secure", ORIGIN_2FA);
|
||
// 坐标操作与视觉子流程:browser.interact 持续授权覆盖(敏感截图授权在主会话用例前再发)
|
||
await e.db.insertInto("automation_grant").values({ action: "browser.interact", granted_by: "e14" }).execute();
|
||
}, 120_000);
|
||
|
||
afterAll(async () => {
|
||
await e?.close();
|
||
await mock?.close();
|
||
await mock2fa?.close();
|
||
if (evidenceDir) await rm(evidenceDir, { recursive: true, force: true });
|
||
});
|
||
|
||
it("二次验证页:服务端识别后立即转等待处理,不输入短信验证码", async () => {
|
||
await open(ids.site2fa, ids.provider2fa, `${ORIGIN_2FA}/torrents.php`);
|
||
const out = await solve({
|
||
sessionId: sid,
|
||
goal: "E14-二次验证 登录站点",
|
||
success: { urlContains: "/torrents.php" },
|
||
maxSteps: 8,
|
||
allowSensitive: true,
|
||
});
|
||
expect(out).toMatchObject({ status: "needs_human", controller: "waiting" });
|
||
expect(out.reason).toContain("二次验证");
|
||
expect(out.takeover).toEqual({
|
||
sessionId: sid,
|
||
siteId: ids.site2fa,
|
||
path: `/console/sites/${ids.site2fa}?tab=origin`,
|
||
});
|
||
const r = childResults.get("2fa")!;
|
||
// 输完密码回车后的截图:识别到二次验证,不给图片、会话已转等待
|
||
expect(r[2]?.json).toMatchObject({ controller: "waiting", waitingReason: "two_factor", imageForModel: false });
|
||
expect(r[2]?.images).toBe(0);
|
||
// 第 4 步(输短信验证码)被拒
|
||
expect(r[3]?.isError).toBe(true);
|
||
expect(r[3]?.text).toContain("等待人工处理");
|
||
expect(mock2fa.actions.some((a) => a.action === "type" && a.body.text === "123456")).toBe(false);
|
||
const view = await svc.view(sid);
|
||
expect(view).toMatchObject({ controller: "waiting", waitingReason: "two_factor" });
|
||
const d = await e.db.selectFrom("decision").selectAll().where("id", "=", out.decisionId).executeTakeFirstOrThrow();
|
||
expect(d).toMatchObject({ status: "awaiting_user", subject_type: "browser_session", subject_id: sid });
|
||
expect((d.evidence as Any).challenge.screenshots.length).toBeGreaterThanOrEqual(3);
|
||
expect((d.evidence as Any).challenge.childRunId).toBe(out.childRunId);
|
||
for (const ev of out.evidence) expect(existsSync(join(evidenceDir, `${ev.id}.jpg`))).toBe(true);
|
||
// 二次验证等待不能被视觉子流程接手
|
||
await expect(solve({ sessionId: sid, goal: "E14-二次验证 再试", success: { urlContains: "/x" } })).rejects.toThrow(
|
||
/等待人工处理/,
|
||
);
|
||
}, 60_000);
|
||
|
||
it("步数超限:转等待处理(challenge_unsolved),待处理事项附证据截图", async () => {
|
||
await open(ids.site, ids.provider, `${ORIGIN}/torrents.php`);
|
||
const out = await solve({
|
||
sessionId: sid,
|
||
goal: "E14-步数 处理页面",
|
||
success: { textPresent: "种子列表" },
|
||
maxSteps: 3,
|
||
});
|
||
expect(out).toMatchObject({ status: "exhausted", steps: 3, controller: "waiting" });
|
||
expect(out.reason).toContain("工具调用次数超出预算");
|
||
const child = await runRow(e, out.childRunId);
|
||
expect(child).toMatchObject({
|
||
status: "budget_exceeded",
|
||
spawn_reason: "browser.solve_challenge",
|
||
parent_run_id: null,
|
||
});
|
||
const events = await runEvents(e, out.childRunId);
|
||
const ends = events.filter((x) => x.type === "tool_call_end");
|
||
expect(ends.length).toBe(3);
|
||
for (const x of ends) expect((x.payload as Any).outputSummary).toMatch(/evidence\/[0-9a-f-]{36}\.jpg/);
|
||
expect(events.some((x) => x.type === "tool_call_rejected")).toBe(true);
|
||
const view = await svc.view(sid);
|
||
expect(view).toMatchObject({ controller: "waiting", waitingReason: "challenge_unsolved" });
|
||
const d = await e.db.selectFrom("decision").selectAll().where("id", "=", out.decisionId).executeTakeFirstOrThrow();
|
||
expect((d.evidence as Any).challenge.screenshots.length).toBe(3);
|
||
}, 60_000);
|
||
|
||
it("敏感截图默认不交给模型:登录页改给已脱敏文字快照;未解决的挑战可以再次接手", async () => {
|
||
// 上一用例留下的 challenge_unsolved 等待:视觉子流程可以接手
|
||
const out = await solve({
|
||
sessionId: sid,
|
||
goal: "E14-敏感 看看登录页",
|
||
success: { urlContains: "/torrents.php" },
|
||
maxSteps: 3,
|
||
});
|
||
expect(out).toMatchObject({ status: "needs_human", controller: "waiting" });
|
||
expect(out.reason).toContain("模型判定需要人工");
|
||
const shot = childResults.get("sensitive")![0]!;
|
||
expect(shot.images).toBe(0);
|
||
expect(shot.json).toMatchObject({ sensitive: true, imageForModel: false });
|
||
// modelImage 由运行时摘出,不出现在 JSON 文本里
|
||
expect("modelImage" in (shot.json as Any)).toBe(false);
|
||
expect((shot.json as Any).note).toContain("敏感");
|
||
expect((shot.json as Any).textSnapshot).toContain("用户名");
|
||
expect((shot.json as Any).evidence.ref).toMatch(/^evidence\//);
|
||
}, 60_000);
|
||
|
||
it("主会话 → solve_challenge:截图 → 点输入框 → 输入 → 回车提交 → 成功条件满足", async () => {
|
||
await e.db
|
||
.insertInto("automation_grant")
|
||
.values({ action: "browser.sensitive_screenshot", granted_by: "e14" })
|
||
.execute();
|
||
await open(ids.site, ids.provider, `${ORIGIN}/torrents.php`);
|
||
const created = await e.api("POST", "/agent/runs", { goal: "E14 主会话登录 观众站点" });
|
||
expect(created.status).toBe(201);
|
||
const runId = created.body.data.id as string;
|
||
const done = await waitFor(
|
||
async () => {
|
||
const r = await runRow(e, runId);
|
||
return ["succeeded", "failed"].includes(r.status) ? r : null;
|
||
},
|
||
60_000,
|
||
"主会话结束",
|
||
);
|
||
expect(done.summary).toContain("挑战处理结果:solved");
|
||
const out = mainResults[0]?.json as Any;
|
||
expect(out).toMatchObject({
|
||
status: "solved",
|
||
steps: 3,
|
||
controller: "automation",
|
||
decisionId: null,
|
||
takeover: null,
|
||
});
|
||
expect(out.url).toContain("/torrents.php");
|
||
// 子流程:父会话 = 主会话;每步截图都作为图片交给了模型(已授权敏感截图)
|
||
const child = await runRow(e, out.childRunId);
|
||
expect(child).toMatchObject({ parent_run_id: runId, trigger: "subagent", status: "succeeded" });
|
||
const r = childResults.get("login")!;
|
||
expect(r.map((x) => x.images)).toEqual([1, 1, 1]);
|
||
expect(r[0]?.json).toMatchObject({
|
||
sensitive: true,
|
||
imageForModel: true,
|
||
coordinates: {
|
||
imageWidth: 1280,
|
||
imageHeight: 800,
|
||
viewportWidth: 1600,
|
||
viewportHeight: 1000,
|
||
devicePixelRatio: 1,
|
||
},
|
||
});
|
||
// 模型图片数据不进 JSON 文本
|
||
expect(r[0]?.text).not.toContain('"data"');
|
||
// 坐标换算:截图 (240, 64) → 页面 (300, 80);(240, 96) → (300, 120)
|
||
expect(r[1]?.json).toMatchObject({ pageX: 300, pageY: 80 });
|
||
expect(r[2]?.json).toMatchObject({ pageX: 300, pageY: 120 });
|
||
const evals = mock.actions.filter((a) => a.action === "evaluate").map((a) => String(a.body.expression));
|
||
expect(evals.some((x) => x.includes("elementFromPoint(300, 80)"))).toBe(true);
|
||
expect(evals.some((x) => x.includes("elementFromPoint(300, 120)"))).toBe(true);
|
||
expect(mock.loggedIn.size).toBe(1);
|
||
// 父会话事件流里有子流程
|
||
const events = await runEvents(e, runId);
|
||
expect(events.some((x) => x.type === "subagent_spawned" && (x.payload as Any).childRunId === out.childRunId)).toBe(
|
||
true,
|
||
);
|
||
// 账本不存图片
|
||
const ledger = await e.db
|
||
.selectFrom("capability_invocation")
|
||
.select(["capability_id", "output"])
|
||
.where("agent_run_id", "=", out.childRunId)
|
||
.execute();
|
||
expect(ledger.length).toBe(2);
|
||
for (const l of ledger) expect((l.output as Any).modelImage).toBeNull();
|
||
}, 90_000);
|
||
});
|