feat(browser): vision automation loop with screenshots, coordinate actions and bounded challenge solving

- browser.screenshot returns the page as an image content block (≤1280px wide, JPEG quality
  configurable) for vision-capable upstreams; non-vision upstreams get a redacted text snapshot.
  Login pages / password fields are marked sensitive and withheld unless a
  browser.sensitive_screenshot grant exists and allowSensitive is passed.
- browser.click_at / type_at / scroll / wait act in screenshot pixels; the server converts via
  viewport/DPR, requires automation control and an allowlisted host, and writes a thumbnail
  evidence file (DATA_DIR/tmp/evidence) referenced from agent_run_event.
- browser.solve_challenge runs a bounded child agent_run (vision caps only, maxSteps ≤ 12,
  budget), judges success from page facts, stops immediately on 2FA / SMS / device confirmation,
  and otherwise moves the session to waiting with evidence and a takeover link.
- PiAgentRuntime passes modelImage outputs as pi-ai image content when model.input has image;
  the ledger never stores image data; coordinate actions use per-invocation idempotency keys.
- Tests: image codec, runtime image content, E14 e2e (login solved, 2FA stop, step limit,
  sensitive withheld). Real camofox 1.17.0 check on example.com recorded in the runbook.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

packages/agent/src/capability/types.ts

@@ -18,6 +18,11 @@ export interface InvokeContext extends CapabilityContext {
   /** agent 调用时的 tool call id / invocation id,用于审计对账 */
   invocationId?: string;
   agentRunId?: string;
+  /**
+   * 本次调用的结果会不会以图片交给模型(Agent 调用时 = 上游模型支持图片输入)。
+   * false:能力不要产出图片(browser.screenshot 改给文字快照);undefined:非 Agent 调用方,按能力缺省。
+   */
+  modelImages?: boolean;
 }
 
 /** handler 实际收到的上下文:注册中心补上服务端生成的幂等键与授权结果 */

packages/agent/src/index.ts

@@ -36,6 +36,7 @@ export {
   DEFAULT_COMPACTION_INSTRUCTIONS,
   PiAgentRuntime,
   type PiAgentRuntimeOptions,
+  toolContent,
 } from "./pi/runtime.ts";
 export { sessionPathFor } from "./pi/session.ts";
 export {

packages/agent/src/pi/fake.ts

@@ -25,6 +25,8 @@ export interface FakeToolResult {
   text: string;
   /** text 能解析成 JSON 时的值 */
   json?: unknown;
+  /** 工具结果里的图片内容块数(视觉自动化:截图真的交给了模型) */
+  images: number;
 }
 
 export interface FakeToolCall {
@@ -54,6 +56,8 @@ export interface FakeModelProviderOptions {
   maxTokens?: number;
   /** 压缩请求时返回的摘要文本 */
   summaryText?: string;
+  /** 模拟支持图片输入的模型(model.input 含 image);缺省 false */
+  vision?: boolean;
 }
 
 const SUMMARY_MARKERS = ["context summarization assistant", "structured context checkpoint summary"];
@@ -70,6 +74,8 @@ export class FakeModelProvider {
   calls = 0;
   /** 最近一次请求看到的系统提示与工具名(测试断言用) */
   lastRequest: { transcript: string } | undefined;
+  /** 历次请求里收到的图片内容块(测试断言截图确实交给了模型) */
+  images: { mimeType: string; bytes: number }[] = [];
 
   constructor(scripts: FakeScript[], options: FakeModelProviderOptions = {}) {
     this.scripts = scripts;
@@ -82,6 +88,7 @@ export class FakeModelProvider {
           contextWindow: options.contextWindow ?? 64_000,
           maxTokens: options.maxTokens ?? 4096,
           reasoning: true,
+          input: options.vision ? ["text", "image"] : ["text"],
         },
       ],
       tokensPerSecond: 0,
@@ -105,6 +112,10 @@ export class FakeModelProvider {
     const messages = context.messages as Message[];
     const transcript = messages.map(messageText).join("\n");
     this.lastRequest = { transcript };
+    const last = messages[messages.length - 1];
+    if (last?.role === "toolResult")
+      for (const c of last.content as Array<{ type: string; data?: string; mimeType?: string }>)
+        if (c.type === "image") this.images.push({ mimeType: c.mimeType ?? "", bytes: (c.data ?? "").length });
     const head = messages[0] ? messageText(messages[0]) : "";
     if (SUMMARY_MARKERS.some((m) => head.includes(m) || lastUserText(messages).includes(m))) {
       return fauxAssistantMessage(this.summaryText);
@@ -203,6 +214,7 @@ function toolResults(messages: Message[]): FakeToolResult[] {
       isError: m.isError,
       text,
       ...(json !== undefined ? { json } : {}),
+      images: (m.content as Array<{ type: string }>).filter((c) => c.type === "image").length,
     });
   }
   return out;

packages/agent/src/pi/runtime.ts

@@ -10,7 +10,13 @@ import {
   type RunResult,
 } from "@earendil-works/pi-agent-core";
 import type { Usage } from "@earendil-works/pi-ai";
-import type { AgentBudgetKind, AgentRunEvent, AgentRunEventBody, AgentUsageTotals } from "@umt/contracts";
+import {
+  type AgentBudgetKind,
+  type AgentRunEvent,
+  type AgentRunEventBody,
+  type AgentUsageTotals,
+  MODEL_IMAGE_KEY,
+} from "@umt/contracts";
 import type { TSchema } from "typebox";
 import { isCapabilityError } from "../capability/errors.ts";
 import type { CapabilityRegistry, RegisteredCapability } from "../capability/registry.ts";
@@ -81,6 +87,7 @@ export class PiAgentRuntime implements AgentRuntimePort {
     const opened = await openRunSession(spec.sessionDir, spec.runId, ctx);
 
     const run = new PiAgentRun(spec, settings, capabilities, this.options, opened.path, opened.repo);
+    run.supportsImages = Array.isArray(model.input) && model.input.includes("image");
     try {
       const { harness, open } = await AgentHarness.create(
         {
@@ -115,6 +122,8 @@ class PiAgentRun implements AgentRunHandle {
   readonly runId: string;
   readonly sessionPath: string;
   pendingOperations = 0;
+  /** 上游模型声明支持图片输入(model.input 含 image) */
+  supportsImages = false;
 
   private readonly channel = new EventChannel<AgentRunEvent>();
   private readonly allowed = new Map<string, string>();
@@ -190,11 +199,9 @@ class PiAgentRun implements AgentRunHandle {
             source: "agent",
             invocationId,
             agentRunId: this.runId,
+            modelImages: this.supportsImages,
           });
-          return {
-            content: [{ type: "text", text: JSON.stringify(redactForModel(output) ?? null) }],
-            details: undefined,
-          };
+          return { content: toolContent(output, this.supportsImages), details: undefined };
         } catch (error) {
           if (isCapabilityError(error)) {
             throw new Error(JSON.stringify({ error: { code: error.code, message: redactString(error.message) } }));
@@ -475,6 +482,36 @@ class PiAgentRun implements AgentRunHandle {
   }
 }
 
+/**
+ * 能力输出 → 工具结果内容块。输出里的 modelImage(见 @umt/contracts MODEL_IMAGE_KEY)摘出来:
+ * 上游支持图片 → image content;不支持 → 丢弃并在文本里提示。图片数据不进 JSON 文本、不经文本脱敏。
+ */
+export function toolContent(
+  output: unknown,
+  vision: boolean,
+): Array<{ type: "text"; text: string } | { type: "image"; data: string; mimeType: string }> {
+  let rest = output;
+  let image: { data: string; mimeType: string } | null = null;
+  if (output && typeof output === "object" && !Array.isArray(output) && MODEL_IMAGE_KEY in output) {
+    const { [MODEL_IMAGE_KEY]: img, ...others } = output as Record<string, unknown>;
+    rest = others;
+    const m = img as { data?: unknown; mimeType?: unknown } | null | undefined;
+    if (m && typeof m.data === "string" && typeof m.mimeType === "string")
+      image = { data: m.data, mimeType: m.mimeType };
+  }
+  const text = JSON.stringify(redactForModel(rest) ?? null);
+  if (!image) return [{ type: "text", text }];
+  if (!vision)
+    return [
+      { type: "text", text },
+      { type: "text", text: "(截图未发送:当前上游模型不支持图片输入;用 browser.snapshot 读取页面文字)" },
+    ];
+  return [
+    { type: "text", text },
+    { type: "image", data: image.data, mimeType: image.mimeType },
+  ];
+}
+
 /** before_tool 白名单判定(纯函数,便于单测) */
 export function checkToolCall(
   toolName: string,

packages/agent/src/runtime/port.ts

@@ -95,6 +95,8 @@ export interface AgentRunHandle {
   readonly sessionPath: string;
   /** 打开已有会话时发现的未完成操作数;>0 时应调用 resume() */
   readonly pendingOperations: number;
+  /** 上游模型支持图片输入(视觉自动化需要);fake 模型按创建参数 */
+  readonly supportsImages?: boolean;
   /** 从 fromSeq(含)开始回放并持续推送事件;close() 后结束 */
   events(fromSeq?: number): AsyncIterable<AgentRunEvent>;
   /** 本进程已产生的全部事件快照 */