feat(browser): vision automation loop with screenshots, coordinate actions and bounded challenge solving

- browser.screenshot returns the page as an image content block (≤1280px wide, JPEG quality
  configurable) for vision-capable upstreams; non-vision upstreams get a redacted text snapshot.
  Login pages / password fields are marked sensitive and withheld unless a
  browser.sensitive_screenshot grant exists and allowSensitive is passed.
- browser.click_at / type_at / scroll / wait act in screenshot pixels; the server converts via
  viewport/DPR, requires automation control and an allowlisted host, and writes a thumbnail
  evidence file (DATA_DIR/tmp/evidence) referenced from agent_run_event.
- browser.solve_challenge runs a bounded child agent_run (vision caps only, maxSteps ≤ 12,
  budget), judges success from page facts, stops immediately on 2FA / SMS / device confirmation,
  and otherwise moves the session to waiting with evidence and a takeover link.
- PiAgentRuntime passes modelImage outputs as pi-ai image content when model.input has image;
  the ledger never stores image data; coordinate actions use per-invocation idempotency keys.
- Tests: image codec, runtime image content, E14 e2e (login solved, 2FA stop, step limit,
  sensitive withheld). Real camofox 1.17.0 check on example.com recorded in the runbook.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

packages/agent/src/capability/types.ts

@@ -18,6 +18,11 @@ export interface InvokeContext extends CapabilityContext {
   /** agent 调用时的 tool call id / invocation id,用于审计对账 */
   invocationId?: string;
   agentRunId?: string;
+  /**
+   * 本次调用的结果会不会以图片交给模型(Agent 调用时 = 上游模型支持图片输入)。
+   * false:能力不要产出图片(browser.screenshot 改给文字快照);undefined:非 Agent 调用方,按能力缺省。
+   */
+  modelImages?: boolean;
 }
 
 /** handler 实际收到的上下文:注册中心补上服务端生成的幂等键与授权结果 */

packages/agent/src/index.ts

@@ -36,6 +36,7 @@ export {
   DEFAULT_COMPACTION_INSTRUCTIONS,
   PiAgentRuntime,
   type PiAgentRuntimeOptions,
+  toolContent,
 } from "./pi/runtime.ts";
 export { sessionPathFor } from "./pi/session.ts";
 export {

packages/agent/src/pi/fake.ts

@@ -25,6 +25,8 @@ export interface FakeToolResult {
   text: string;
   /** text 能解析成 JSON 时的值 */
   json?: unknown;
+  /** 工具结果里的图片内容块数(视觉自动化:截图真的交给了模型) */
+  images: number;
 }
 
 export interface FakeToolCall {
@@ -54,6 +56,8 @@ export interface FakeModelProviderOptions {
   maxTokens?: number;
   /** 压缩请求时返回的摘要文本 */
   summaryText?: string;
+  /** 模拟支持图片输入的模型(model.input 含 image);缺省 false */
+  vision?: boolean;
 }
 
 const SUMMARY_MARKERS = ["context summarization assistant", "structured context checkpoint summary"];
@@ -70,6 +74,8 @@ export class FakeModelProvider {
   calls = 0;
   /** 最近一次请求看到的系统提示与工具名(测试断言用) */
   lastRequest: { transcript: string } | undefined;
+  /** 历次请求里收到的图片内容块(测试断言截图确实交给了模型) */
+  images: { mimeType: string; bytes: number }[] = [];
 
   constructor(scripts: FakeScript[], options: FakeModelProviderOptions = {}) {
     this.scripts = scripts;
@@ -82,6 +88,7 @@ export class FakeModelProvider {
           contextWindow: options.contextWindow ?? 64_000,
           maxTokens: options.maxTokens ?? 4096,
           reasoning: true,
+          input: options.vision ? ["text", "image"] : ["text"],
         },
       ],
       tokensPerSecond: 0,
@@ -105,6 +112,10 @@ export class FakeModelProvider {
     const messages = context.messages as Message[];
     const transcript = messages.map(messageText).join("\n");
     this.lastRequest = { transcript };
+    const last = messages[messages.length - 1];
+    if (last?.role === "toolResult")
+      for (const c of last.content as Array<{ type: string; data?: string; mimeType?: string }>)
+        if (c.type === "image") this.images.push({ mimeType: c.mimeType ?? "", bytes: (c.data ?? "").length });
     const head = messages[0] ? messageText(messages[0]) : "";
     if (SUMMARY_MARKERS.some((m) => head.includes(m) || lastUserText(messages).includes(m))) {
       return fauxAssistantMessage(this.summaryText);
@@ -203,6 +214,7 @@ function toolResults(messages: Message[]): FakeToolResult[] {
       isError: m.isError,
       text,
       ...(json !== undefined ? { json } : {}),
+      images: (m.content as Array<{ type: string }>).filter((c) => c.type === "image").length,
     });
   }
   return out;

packages/agent/src/pi/runtime.ts

@@ -10,7 +10,13 @@ import {
   type RunResult,
 } from "@earendil-works/pi-agent-core";
 import type { Usage } from "@earendil-works/pi-ai";
-import type { AgentBudgetKind, AgentRunEvent, AgentRunEventBody, AgentUsageTotals } from "@umt/contracts";
+import {
+  type AgentBudgetKind,
+  type AgentRunEvent,
+  type AgentRunEventBody,
+  type AgentUsageTotals,
+  MODEL_IMAGE_KEY,
+} from "@umt/contracts";
 import type { TSchema } from "typebox";
 import { isCapabilityError } from "../capability/errors.ts";
 import type { CapabilityRegistry, RegisteredCapability } from "../capability/registry.ts";
@@ -81,6 +87,7 @@ export class PiAgentRuntime implements AgentRuntimePort {
     const opened = await openRunSession(spec.sessionDir, spec.runId, ctx);
 
     const run = new PiAgentRun(spec, settings, capabilities, this.options, opened.path, opened.repo);
+    run.supportsImages = Array.isArray(model.input) && model.input.includes("image");
     try {
       const { harness, open } = await AgentHarness.create(
         {
@@ -115,6 +122,8 @@ class PiAgentRun implements AgentRunHandle {
   readonly runId: string;
   readonly sessionPath: string;
   pendingOperations = 0;
+  /** 上游模型声明支持图片输入(model.input 含 image) */
+  supportsImages = false;
 
   private readonly channel = new EventChannel<AgentRunEvent>();
   private readonly allowed = new Map<string, string>();
@@ -190,11 +199,9 @@ class PiAgentRun implements AgentRunHandle {
             source: "agent",
             invocationId,
             agentRunId: this.runId,
+            modelImages: this.supportsImages,
           });
-          return {
-            content: [{ type: "text", text: JSON.stringify(redactForModel(output) ?? null) }],
-            details: undefined,
-          };
+          return { content: toolContent(output, this.supportsImages), details: undefined };
         } catch (error) {
           if (isCapabilityError(error)) {
             throw new Error(JSON.stringify({ error: { code: error.code, message: redactString(error.message) } }));
@@ -475,6 +482,36 @@ class PiAgentRun implements AgentRunHandle {
   }
 }
 
+/**
+ * 能力输出 → 工具结果内容块。输出里的 modelImage(见 @umt/contracts MODEL_IMAGE_KEY)摘出来:
+ * 上游支持图片 → image content;不支持 → 丢弃并在文本里提示。图片数据不进 JSON 文本、不经文本脱敏。
+ */
+export function toolContent(
+  output: unknown,
+  vision: boolean,
+): Array<{ type: "text"; text: string } | { type: "image"; data: string; mimeType: string }> {
+  let rest = output;
+  let image: { data: string; mimeType: string } | null = null;
+  if (output && typeof output === "object" && !Array.isArray(output) && MODEL_IMAGE_KEY in output) {
+    const { [MODEL_IMAGE_KEY]: img, ...others } = output as Record<string, unknown>;
+    rest = others;
+    const m = img as { data?: unknown; mimeType?: unknown } | null | undefined;
+    if (m && typeof m.data === "string" && typeof m.mimeType === "string")
+      image = { data: m.data, mimeType: m.mimeType };
+  }
+  const text = JSON.stringify(redactForModel(rest) ?? null);
+  if (!image) return [{ type: "text", text }];
+  if (!vision)
+    return [
+      { type: "text", text },
+      { type: "text", text: "(截图未发送:当前上游模型不支持图片输入;用 browser.snapshot 读取页面文字)" },
+    ];
+  return [
+    { type: "text", text },
+    { type: "image", data: image.data, mimeType: image.mimeType },
+  ];
+}
+
 /** before_tool 白名单判定(纯函数,便于单测) */
 export function checkToolCall(
   toolName: string,

packages/agent/src/runtime/port.ts

@@ -95,6 +95,8 @@ export interface AgentRunHandle {
   readonly sessionPath: string;
   /** 打开已有会话时发现的未完成操作数;>0 时应调用 resume() */
   readonly pendingOperations: number;
+  /** 上游模型支持图片输入(视觉自动化需要);fake 模型按创建参数 */
+  readonly supportsImages?: boolean;
   /** 从 fromSeq(含)开始回放并持续推送事件;close() 后结束 */
   events(fromSeq?: number): AsyncIterable<AgentRunEvent>;
   /** 本进程已产生的全部事件快照 */

packages/agent/test/vision.test.ts

@@ -0,0 +1,109 @@
+import { afterEach, describe, expect, it } from "vitest";
+import { z } from "zod";
+import {
+  type AgentRunHandle,
+  CapabilityRegistry,
+  createFakeRuntime,
+  type FakeScript,
+  toolContent,
+} from "../src/index.ts";
+import { settings, tempDir } from "./helpers.ts";
+
+// VIS-1:能力输出里的 modelImage 以 image content 交给支持视觉的模型;不支持时降级为文字提示,图片数据不进 JSON 文本。
+
+const PNG_1X1 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==";
+
+const opened: AgentRunHandle[] = [];
+afterEach(async () => {
+  await Promise.all(opened.splice(0).map((r) => r.close()));
+});
+
+function registry(seen: Array<boolean | undefined>) {
+  const r = new CapabilityRegistry();
+  r.register(
+    {
+      id: "browser.screenshot",
+      version: 1,
+      title: "截图",
+      description: "截取页面",
+      inputSchema: z.object({ sessionId: z.string() }),
+      outputSchema: z.object({
+        url: z.string(),
+        modelImage: z
+          .object({ mimeType: z.string(), data: z.string(), width: z.number(), height: z.number() })
+          .nullable(),
+      }),
+      kind: "read",
+      scope: "read",
+      riskLevel: "low",
+      idempotent: true,
+      supportsPreview: false,
+      replay: "safe",
+    },
+    (_input, ctx) => {
+      seen.push(ctx.modelImages);
+      return {
+        url: "https://site.example/login",
+        modelImage: { mimeType: "image/png", data: PNG_1X1, width: 1, height: 1 },
+      };
+    },
+  );
+  return r;
+}
+
+const script: FakeScript = {
+  match: /看页面/,
+  steps: [{ toolCalls: [{ capability: "browser.screenshot", input: { sessionId: "s1" } }] }],
+  finalText: (r) => `images=${r[0]?.images};text=${r[0]?.text}`,
+};
+
+async function run(vision: boolean) {
+  const seen: Array<boolean | undefined> = [];
+  const { runtime, fake } = createFakeRuntime({ registry: registry(seen), scripts: [script], fake: { vision } });
+  const handle = await runtime.createRun({
+    runId: `vision-${vision}`,
+    goal: "看页面",
+    systemPrompt: "测试",
+    capabilities: ["browser.screenshot"],
+    settings: settings(),
+    sessionDir: await tempDir(),
+    actor: { actorId: "user:1", taskId: "task:1", scopeIds: [] },
+  });
+  opened.push(handle);
+  const outcome = await handle.prompt();
+  return { handle, fake, outcome, seen };
+}
+
+describe("VIS-1 图片内容块", () => {
+  it("支持视觉的模型:截图作为 image content 交给模型,JSON 文本里没有图片数据", async () => {
+    const { handle, fake, outcome, seen } = await run(true);
+    expect(handle.supportsImages).toBe(true);
+    expect(seen).toEqual([true]);
+    expect(fake.images).toEqual([{ mimeType: "image/png", bytes: PNG_1X1.length }]);
+    expect(outcome.finalText).toContain("images=1");
+    expect(outcome.finalText).not.toContain(PNG_1X1);
+  });
+
+  it("不支持视觉的模型:能力收到 modelImages=false,图片不发送并提示", async () => {
+    const { handle, fake, outcome, seen } = await run(false);
+    expect(handle.supportsImages).toBe(false);
+    expect(seen).toEqual([false]);
+    expect(fake.images).toEqual([]);
+    expect(outcome.finalText).toContain("images=0");
+    expect(outcome.finalText).not.toContain(PNG_1X1);
+  });
+
+  it("toolContent:摘出 modelImage;null 时只有文本", () => {
+    const img = toolContent({ a: 1, modelImage: { mimeType: "image/jpeg", data: "QUJD" } }, true);
+    expect(img).toEqual([
+      { type: "text", text: '{"a":1}' },
+      { type: "image", data: "QUJD", mimeType: "image/jpeg" },
+    ]);
+    const none = toolContent({ a: 1, modelImage: null }, true);
+    expect(none).toEqual([{ type: "text", text: '{"a":1}' }]);
+    const text = toolContent({ a: 1, modelImage: { mimeType: "image/jpeg", data: "QUJD" } }, false);
+    expect(text).toHaveLength(2);
+    expect(text[1]).toMatchObject({ type: "text" });
+    expect(JSON.stringify(text)).not.toContain("QUJD");
+  });
+});