feat(browser): vision automation loop with screenshots, coordinate actions and bounded challenge solving

- browser.screenshot returns the page as an image content block (≤1280px wide, JPEG quality
  configurable) for vision-capable upstreams; non-vision upstreams get a redacted text snapshot.
  Login pages / password fields are marked sensitive and withheld unless a
  browser.sensitive_screenshot grant exists and allowSensitive is passed.
- browser.click_at / type_at / scroll / wait act in screenshot pixels; the server converts via
  viewport/DPR, requires automation control and an allowlisted host, and writes a thumbnail
  evidence file (DATA_DIR/tmp/evidence) referenced from agent_run_event.
- browser.solve_challenge runs a bounded child agent_run (vision caps only, maxSteps ≤ 12,
  budget), judges success from page facts, stops immediately on 2FA / SMS / device confirmation,
  and otherwise moves the session to waiting with evidence and a takeover link.
- PiAgentRuntime passes modelImage outputs as pi-ai image content when model.input has image;
  the ledger never stores image data; coordinate actions use per-invocation idempotency keys.
- Tests: image codec, runtime image content, E14 e2e (login solved, 2FA stop, step limit,
  sensitive withheld). Real camofox 1.17.0 check on example.com recorded in the runbook.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed 2026-09-29 01:27:48 +08:00
1 parent 0765264a47
commit c437c6bff6
34 files changed
+2194 -35

No files matched your search

+356
View File
@@ -0,0 +1,356 @@
import { existsSync } from "node:fs";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { FakeScript, FakeToolResult, InvokeContext } from "@umt/agent";
import { createMockCamofox, type MockCamofox } from "@umt/testkit";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { browserServiceFor } from "../../src/modules/browser/index.ts";
import type { BrowserService } from "../../src/modules/browser/service.ts";
import { createE2E, type E2E, pgAvailable, runEvents, runRow, waitFor } from "./harness.ts";
// E14 视觉自动化闭环(VIS-1):fake 视觉模型(脚本返回固定坐标)+ testkit mock camofox(视口 1600×1000、DPR 1,
// 截图缩到 1280×800 交给模型,坐标按 0.8 换算回页面)。
// - 主会话调用 browser.solve_challenge:子流程 截图 → 点用户名框输入 → 点密码框输入并回车 → 成功条件(地址含 /torrents.php)满足;
// - 二次验证页:服务端识别后会话立即转等待处理,子流程后续动作被拒,不输入短信验证码;
// - 步数超限:超出 maxSteps 转等待处理(challenge_unsolved),待处理事项附证据截图;
// - 敏感截图(登录页 / 密码框)默认不交给模型,改给已脱敏的文字快照。
const hasPg = await pgAvailable();
// biome-ignore lint/suspicious/noExplicitAny: 测试断言直接读 JSON
type Any = any;
const ORIGIN = "https://audiences.example";
const ORIGIN_2FA = "https://secure.example";
const USER = "fixture-user";
const PASSWORD = "e14-login-pw";
const ACCESS_KEY = "camofox-e14-key";
/** 登录页各行在 1280×800 截图上的坐标(页面 CSS 行中心 y = 40 + 40i,x = 300;×0.8) */
const AT = { username: { x: 240, y: 64 }, password: { x: 240, y: 96 } };
let sid = "";
const childResults = new Map<string, FakeToolResult[]>();
let mainResults: FakeToolResult[] = [];
const typeUser = () => ({ sessionId: sid, ...AT.username, text: USER, allowSensitive: true });
const typePassword = () => ({ sessionId: sid, ...AT.password, text: PASSWORD, pressEnter: true, allowSensitive: true });
const scripts: FakeScript[] = [
{
name: "e14-main",
match: "E14 主会话登录",
steps: [
{
toolCalls: [
{
capability: "browser.solve_challenge",
input: () => ({
sessionId: sid,
goal: "E14-登录 用测试账号登录站点",
success: { urlContains: "/torrents.php" },
maxSteps: 6,
allowSensitive: true,
}),
},
],
},
],
finalText: (r) => {
mainResults = r;
return `挑战处理结果:${(r[0]?.json as Any)?.status ?? r[0]?.text}`;
},
},
{
name: "e14-login",
match: "E14-登录",
steps: [
{ toolCalls: [{ capability: "browser.screenshot", input: () => ({ sessionId: sid, allowSensitive: true }) }] },
{ toolCalls: [{ capability: "browser.type_at", input: typeUser }] },
{ toolCalls: [{ capability: "browser.type_at", input: typePassword }] },
],
finalText: (r) => {
childResults.set("login", r);
return "已提交登录表单";
},
},
{
name: "e14-2fa",
match: "E14-二次验证",
steps: [
{ toolCalls: [{ capability: "browser.screenshot", input: () => ({ sessionId: sid, allowSensitive: true }) }] },
{ toolCalls: [{ capability: "browser.type_at", input: typeUser }] },
{ toolCalls: [{ capability: "browser.type_at", input: typePassword }] },
// 模型不听话继续输短信验证码:服务端已转等待处理,必须被拒
{
toolCalls: [
{ capability: "browser.type_at", input: () => ({ sessionId: sid, x: 240, y: 96, text: "123456" }) },
],
},
],
finalText: (r) => {
childResults.set("2fa", r);
return "NEEDS_HUMAN: 需要短信验证码";
},
},
{
name: "e14-steps",
match: "E14-步数",
steps: [
{ toolCalls: [{ capability: "browser.screenshot", input: () => ({ sessionId: sid }) }] },
{ toolCalls: [{ capability: "browser.scroll", input: () => ({ sessionId: sid, dy: 300 }) }] },
{ toolCalls: [{ capability: "browser.wait", input: () => ({ sessionId: sid, ms: 50 }) }] },
{ toolCalls: [{ capability: "browser.wait", input: () => ({ sessionId: sid, ms: 50 }) }] },
],
finalText: (r) => {
childResults.set("steps", r);
return "还没完成";
},
},
{
name: "e14-sensitive",
match: "E14-敏感",
steps: [{ toolCalls: [{ capability: "browser.screenshot", input: () => ({ sessionId: sid }) }] }],
finalText: (r) => {
childResults.set("sensitive", r);
return "NEEDS_HUMAN: 截图不可见,登录页需要人工";
},
},
];
describe.skipIf(!hasPg)("E14 视觉自动化闭环", () => {
let e: E2E;
let mock: MockCamofox;
let mock2fa: MockCamofox;
let evidenceDir: string;
let svc: BrowserService;
const ids = { site: "", site2fa: "", provider: "", provider2fa: "" };
const agentCtx = (): InvokeContext => ({
actorId: "agent:e14",
taskId: crypto.randomUUID(),
scopeIds: [],
signal: new AbortController().signal,
source: "agent",
invocationId: crypto.randomUUID(),
});
const solve = (input: Record<string, unknown>) =>
e.booted.ctx.capabilities.invoke("browser.solve_challenge", input, agentCtx()) as Promise<Any>;
const open = async (siteId: string, providerId: string, url: string) => {
for (const s of await svc.list({ siteId })) if (s.controller !== "closed") await svc.close(s.id);
sid = await svc.open({ siteId, providerId, url });
return sid;
};
beforeAll(async () => {
const site = (origin: string) => ({ origin, username: USER, password: PASSWORD });
mock = await createMockCamofox({
accessKey: ACCESS_KEY,
site: site(ORIGIN),
viewport: { width: 1600, height: 1000 },
});
mock2fa = await createMockCamofox({
accessKey: ACCESS_KEY,
site: { ...site(ORIGIN_2FA), twoFactor: true },
viewport: { width: 1600, height: 1000 },
});
evidenceDir = await mkdtemp(join(tmpdir(), "umt-e14-evidence-"));
e = await createE2E("e14", { fakeScripts: scripts, browser: { evidenceDir, settleMs: 0 } });
svc = browserServiceFor(e.booted.ctx)!;
const ctx = e.booted.ctx;
const keyRef = await ctx.secrets.create({ purpose: "other", plaintext: ACCESS_KEY });
const provider = async (name: string, url: string) =>
(
await e.db
.insertInto("browser_provider")
.values({
name,
kind: "camofox",
base_url: url,
secret_ref: keyRef,
enabled: true,
config: JSON.stringify({ navigationTimeoutMs: 2_000, maxRetries: 0 }),
})
.returning("id")
.executeTakeFirstOrThrow()
).id;
ids.provider = await provider("camofox-e14", mock.url);
ids.provider2fa = await provider("camofox-e14-2fa", mock2fa.url);
const addSite = async (key: string, origin: string) =>
(
await e.db
.insertInto("site")
.values({
key,
name: key,
adapter: "nexusphp",
base_url: `${origin}/`,
rate_limit: JSON.stringify({ minIntervalMs: 0, concurrency: 5 }),
})
.returning("id")
.executeTakeFirstOrThrow()
).id;
ids.site = await addSite("e14site", ORIGIN);
ids.site2fa = await addSite("e14secure", ORIGIN_2FA);
// 坐标操作与视觉子流程:browser.interact 持续授权覆盖(敏感截图授权在主会话用例前再发)
await e.db.insertInto("automation_grant").values({ action: "browser.interact", granted_by: "e14" }).execute();
}, 120_000);
afterAll(async () => {
await e?.close();
await mock?.close();
await mock2fa?.close();
if (evidenceDir) await rm(evidenceDir, { recursive: true, force: true });
});
it("二次验证页:服务端识别后立即转等待处理,不输入短信验证码", async () => {
await open(ids.site2fa, ids.provider2fa, `${ORIGIN_2FA}/torrents.php`);
const out = await solve({
sessionId: sid,
goal: "E14-二次验证 登录站点",
success: { urlContains: "/torrents.php" },
maxSteps: 8,
allowSensitive: true,
});
expect(out).toMatchObject({ status: "needs_human", controller: "waiting" });
expect(out.reason).toContain("二次验证");
expect(out.takeover).toEqual({
sessionId: sid,
siteId: ids.site2fa,
path: `/console/sites/${ids.site2fa}?tab=origin`,
});
const r = childResults.get("2fa")!;
// 输完密码回车后的截图:识别到二次验证,不给图片、会话已转等待
expect(r[2]?.json).toMatchObject({ controller: "waiting", waitingReason: "two_factor", imageForModel: false });
expect(r[2]?.images).toBe(0);
// 第 4 步(输短信验证码)被拒
expect(r[3]?.isError).toBe(true);
expect(r[3]?.text).toContain("等待人工处理");
expect(mock2fa.actions.some((a) => a.action === "type" && a.body.text === "123456")).toBe(false);
const view = await svc.view(sid);
expect(view).toMatchObject({ controller: "waiting", waitingReason: "two_factor" });
const d = await e.db.selectFrom("decision").selectAll().where("id", "=", out.decisionId).executeTakeFirstOrThrow();
expect(d).toMatchObject({ status: "awaiting_user", subject_type: "browser_session", subject_id: sid });
expect((d.evidence as Any).challenge.screenshots.length).toBeGreaterThanOrEqual(3);
expect((d.evidence as Any).challenge.childRunId).toBe(out.childRunId);
for (const ev of out.evidence) expect(existsSync(join(evidenceDir, `${ev.id}.jpg`))).toBe(true);
// 二次验证等待不能被视觉子流程接手
await expect(solve({ sessionId: sid, goal: "E14-二次验证 再试", success: { urlContains: "/x" } })).rejects.toThrow(
/等待人工处理/,
);
}, 60_000);
it("步数超限:转等待处理(challenge_unsolved),待处理事项附证据截图", async () => {
await open(ids.site, ids.provider, `${ORIGIN}/torrents.php`);
const out = await solve({
sessionId: sid,
goal: "E14-步数 处理页面",
success: { textPresent: "种子列表" },
maxSteps: 3,
});
expect(out).toMatchObject({ status: "exhausted", steps: 3, controller: "waiting" });
expect(out.reason).toContain("工具调用次数超出预算");
const child = await runRow(e, out.childRunId);
expect(child).toMatchObject({
status: "budget_exceeded",
spawn_reason: "browser.solve_challenge",
parent_run_id: null,
});
const events = await runEvents(e, out.childRunId);
const ends = events.filter((x) => x.type === "tool_call_end");
expect(ends.length).toBe(3);
for (const x of ends) expect((x.payload as Any).outputSummary).toMatch(/evidence\/[0-9a-f-]{36}\.jpg/);
expect(events.some((x) => x.type === "tool_call_rejected")).toBe(true);
const view = await svc.view(sid);
expect(view).toMatchObject({ controller: "waiting", waitingReason: "challenge_unsolved" });
const d = await e.db.selectFrom("decision").selectAll().where("id", "=", out.decisionId).executeTakeFirstOrThrow();
expect((d.evidence as Any).challenge.screenshots.length).toBe(3);
}, 60_000);
it("敏感截图默认不交给模型:登录页改给已脱敏文字快照;未解决的挑战可以再次接手", async () => {
// 上一用例留下的 challenge_unsolved 等待:视觉子流程可以接手
const out = await solve({
sessionId: sid,
goal: "E14-敏感 看看登录页",
success: { urlContains: "/torrents.php" },
maxSteps: 3,
});
expect(out).toMatchObject({ status: "needs_human", controller: "waiting" });
expect(out.reason).toContain("模型判定需要人工");
const shot = childResults.get("sensitive")![0]!;
expect(shot.images).toBe(0);
expect(shot.json).toMatchObject({ sensitive: true, imageForModel: false });
// modelImage 由运行时摘出,不出现在 JSON 文本里
expect("modelImage" in (shot.json as Any)).toBe(false);
expect((shot.json as Any).note).toContain("敏感");
expect((shot.json as Any).textSnapshot).toContain("用户名");
expect((shot.json as Any).evidence.ref).toMatch(/^evidence\//);
}, 60_000);
it("主会话 → solve_challenge:截图 → 点输入框 → 输入 → 回车提交 → 成功条件满足", async () => {
await e.db
.insertInto("automation_grant")
.values({ action: "browser.sensitive_screenshot", granted_by: "e14" })
.execute();
await open(ids.site, ids.provider, `${ORIGIN}/torrents.php`);
const created = await e.api("POST", "/agent/runs", { goal: "E14 主会话登录 观众站点" });
expect(created.status).toBe(201);
const runId = created.body.data.id as string;
const done = await waitFor(
async () => {
const r = await runRow(e, runId);
return ["succeeded", "failed"].includes(r.status) ? r : null;
},
60_000,
"主会话结束",
);
expect(done.summary).toContain("挑战处理结果:solved");
const out = mainResults[0]?.json as Any;
expect(out).toMatchObject({
status: "solved",
steps: 3,
controller: "automation",
decisionId: null,
takeover: null,
});
expect(out.url).toContain("/torrents.php");
// 子流程:父会话 = 主会话;每步截图都作为图片交给了模型(已授权敏感截图)
const child = await runRow(e, out.childRunId);
expect(child).toMatchObject({ parent_run_id: runId, trigger: "subagent", status: "succeeded" });
const r = childResults.get("login")!;
expect(r.map((x) => x.images)).toEqual([1, 1, 1]);
expect(r[0]?.json).toMatchObject({
sensitive: true,
imageForModel: true,
coordinates: {
imageWidth: 1280,
imageHeight: 800,
viewportWidth: 1600,
viewportHeight: 1000,
devicePixelRatio: 1,
},
});
// 模型图片数据不进 JSON 文本
expect(r[0]?.text).not.toContain('"data"');
// 坐标换算:截图 (240, 64) → 页面 (300, 80);(240, 96) → (300, 120)
expect(r[1]?.json).toMatchObject({ pageX: 300, pageY: 80 });
expect(r[2]?.json).toMatchObject({ pageX: 300, pageY: 120 });
const evals = mock.actions.filter((a) => a.action === "evaluate").map((a) => String(a.body.expression));
expect(evals.some((x) => x.includes("elementFromPoint(300, 80)"))).toBe(true);
expect(evals.some((x) => x.includes("elementFromPoint(300, 120)"))).toBe(true);
expect(mock.loggedIn.size).toBe(1);
// 父会话事件流里有子流程
const events = await runEvents(e, runId);
expect(events.some((x) => x.type === "subagent_spawned" && (x.payload as Any).childRunId === out.childRunId)).toBe(
true,
);
// 账本不存图片
const ledger = await e.db
.selectFrom("capability_invocation")
.select(["capability_id", "output"])
.where("agent_run_id", "=", out.childRunId)
.execute();
expect(ledger.length).toBe(2);
for (const l of ledger) expect((l.output as Any).modelImage).toBeNull();
}, 90_000);
});
@@ -45,8 +45,8 @@ describe.skipIf(!hasPg)("E2 Agent 委托闭环(fake 模型 + 真实能力/授权
return r.body.data.id as string;
};
it("启动断言:29 个能力全部注册,并在能力目录里可见", async () => {
expect(e.booted.ctx.capabilities.list()).toHaveLength(29);
it("启动断言:35 个能力全部注册,并在能力目录里可见", async () => {
expect(e.booted.ctx.capabilities.list()).toHaveLength(35);
const r = await e.api("GET", "/capabilities");
expect(r.body.data.items.map((c: { id: string }) => c.id).sort()).toEqual(
e.booted.ctx.capabilities
+5 -2
View File
@@ -6,6 +6,7 @@ import { createAcquisitionOrganizeModule } from "../../../src/modules/acquisitio
import { createAgentModule } from "../../../src/modules/agent/index.ts";
import { authSettingsModule } from "../../../src/modules/auth-settings/index.ts";
import { createBrowserModule } from "../../../src/modules/browser/index.ts";
import type { BrowserModuleOptions } from "../../../src/modules/browser/service.ts";
import { createSchedulesModule } from "../../../src/modules/schedules/index.ts";
import { createSourcesIdentityModule } from "../../../src/modules/sources-identity/index.ts";
import type { ServerModule } from "../../../src/modules/types.ts";
@@ -29,6 +30,8 @@ export interface E2EModuleOptions {
syncIntervalMs?: number;
/** 网页搜索的 fetch / DNS 注入(E13);缺省不联网也能注册能力 */
webSearch?: WebSearchModuleOptions;
/** 浏览器模块注入(E14 视觉自动化:证据目录、动作后等待) */
browser?: BrowserModuleOptions;
}
export function buildModules(o: E2EModuleOptions): ServerModule[] {
@@ -69,8 +72,8 @@ export function buildModules(o: E2EModuleOptions): ServerModule[] {
...(o.fakeScripts ? { fakeScripts: o.fakeScripts } : {}),
goalPollSec: 1,
}),
// 浏览器能力(browser.*)随首批能力一起注册;e2e 不配置浏览器服务
createBrowserModule(),
// 浏览器能力(browser.*)随首批能力一起注册;浏览器服务由用例自己登记(E14 指向 mock camofox)
createBrowserModule(o.browser ?? {}),
// 网页搜索(web.search / web.fetch);e2e 注入 mock 搜索服务与网页
createWebSearchModule(o.webSearch ?? {}),
// 定时任务调度器(驱动上面模块登记的周期任务;须在末尾)
+4
View File
@@ -20,6 +20,7 @@ import { type Bootstrapped, bootstrap } from "../../src/context.ts";
import { createApp } from "../../src/http/app.ts";
import { EventBus } from "../../src/http/sse.ts";
import { createLogger } from "../../src/logger.ts";
import type { BrowserModuleOptions } from "../../src/modules/browser/service.ts";
import { setDefaultUpstream } from "../../src/modules/pi/store.ts";
import type { ServerModule } from "../../src/modules/types.ts";
import type { WebSearchModuleOptions } from "../../src/modules/websearch/index.ts";
@@ -81,6 +82,8 @@ export interface E2EOptions {
tmdbFixtures?: TmdbFixture[];
/** 网页搜索模块注入(E13) */
webSearch?: WebSearchModuleOptions;
/** 浏览器模块注入(E14) */
browser?: BrowserModuleOptions;
}
export async function waitFor<T>(
@@ -136,6 +139,7 @@ export async function createE2E(name: string, opts: E2EOptions = {}): Promise<E2
sessionDir,
...(opts.fakeScripts ? { fakeScripts: opts.fakeScripts } : {}),
...(opts.webSearch ? { webSearch: opts.webSearch } : {}),
...(opts.browser ? { browser: opts.browser } : {}),
});
const booted = await bootstrap({
role: "all",
@@ -15,8 +15,8 @@ const TiB = 1024 ** 4;
const SITE_IDS = ["site.list", "site.health_check", "site.refresh_login", "site.user_data"] as const;
describe("站点能力规格", () => {
it("进入首批能力目录(17 + 4 站点 + 4 浏览器 + 2 会话交互 + 2 网页搜索 = 29);类别 / 风险 / 重放按约定", () => {
expect(CAPABILITY_IDS).toHaveLength(29);
it("进入首批能力目录(17 + 4 站点 + 10 浏览器 + 2 会话交互 + 2 网页搜索 = 35);类别 / 风险 / 重放按约定", () => {
expect(CAPABILITY_IDS).toHaveLength(35);
for (const id of SITE_IDS) expect(CAPABILITY_IDS).toContain(id);
expect(capabilitySpec("site.list")).toMatchObject({ kind: "read", riskLevel: "low", replay: "safe" });
expect(capabilitySpec("site.user_data")).toMatchObject({ kind: "read", riskLevel: "low", replay: "safe" });