feat(browser): vision automation loop with screenshots, coordinate actions and bounded challenge solving

- browser.screenshot returns the page as an image content block (≤1280px wide, JPEG quality
  configurable) for vision-capable upstreams; non-vision upstreams get a redacted text snapshot.
  Login pages / password fields are marked sensitive and withheld unless a
  browser.sensitive_screenshot grant exists and allowSensitive is passed.
- browser.click_at / type_at / scroll / wait act in screenshot pixels; the server converts via
  viewport/DPR, requires automation control and an allowlisted host, and writes a thumbnail
  evidence file (DATA_DIR/tmp/evidence) referenced from agent_run_event.
- browser.solve_challenge runs a bounded child agent_run (vision caps only, maxSteps ≤ 12,
  budget), judges success from page facts, stops immediately on 2FA / SMS / device confirmation,
  and otherwise moves the session to waiting with evidence and a takeover link.
- PiAgentRuntime passes modelImage outputs as pi-ai image content when model.input has image;
  the ledger never stores image data; coordinate actions use per-invocation idempotency keys.
- Tests: image codec, runtime image content, E14 e2e (login solved, 2FA stop, step limit,
  sensitive withheld). Real camofox 1.17.0 check on example.com recorded in the runbook.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

packages/agent/src/pi/runtime.ts

@@ -10,7 +10,13 @@ import {
   type RunResult,
 } from "@earendil-works/pi-agent-core";
 import type { Usage } from "@earendil-works/pi-ai";
-import type { AgentBudgetKind, AgentRunEvent, AgentRunEventBody, AgentUsageTotals } from "@umt/contracts";
+import {
+  type AgentBudgetKind,
+  type AgentRunEvent,
+  type AgentRunEventBody,
+  type AgentUsageTotals,
+  MODEL_IMAGE_KEY,
+} from "@umt/contracts";
 import type { TSchema } from "typebox";
 import { isCapabilityError } from "../capability/errors.ts";
 import type { CapabilityRegistry, RegisteredCapability } from "../capability/registry.ts";
@@ -81,6 +87,7 @@ export class PiAgentRuntime implements AgentRuntimePort {
     const opened = await openRunSession(spec.sessionDir, spec.runId, ctx);
 
     const run = new PiAgentRun(spec, settings, capabilities, this.options, opened.path, opened.repo);
+    run.supportsImages = Array.isArray(model.input) && model.input.includes("image");
     try {
       const { harness, open } = await AgentHarness.create(
         {
@@ -115,6 +122,8 @@ class PiAgentRun implements AgentRunHandle {
   readonly runId: string;
   readonly sessionPath: string;
   pendingOperations = 0;
+  /** 上游模型声明支持图片输入(model.input 含 image) */
+  supportsImages = false;
 
   private readonly channel = new EventChannel<AgentRunEvent>();
   private readonly allowed = new Map<string, string>();
@@ -190,11 +199,9 @@ class PiAgentRun implements AgentRunHandle {
             source: "agent",
             invocationId,
             agentRunId: this.runId,
+            modelImages: this.supportsImages,
           });
-          return {
-            content: [{ type: "text", text: JSON.stringify(redactForModel(output) ?? null) }],
-            details: undefined,
-          };
+          return { content: toolContent(output, this.supportsImages), details: undefined };
         } catch (error) {
           if (isCapabilityError(error)) {
             throw new Error(JSON.stringify({ error: { code: error.code, message: redactString(error.message) } }));
@@ -475,6 +482,36 @@ class PiAgentRun implements AgentRunHandle {
   }
 }
 
+/**
+ * 能力输出 → 工具结果内容块。输出里的 modelImage(见 @umt/contracts MODEL_IMAGE_KEY)摘出来:
+ * 上游支持图片 → image content;不支持 → 丢弃并在文本里提示。图片数据不进 JSON 文本、不经文本脱敏。
+ */
+export function toolContent(
+  output: unknown,
+  vision: boolean,
+): Array<{ type: "text"; text: string } | { type: "image"; data: string; mimeType: string }> {
+  let rest = output;
+  let image: { data: string; mimeType: string } | null = null;
+  if (output && typeof output === "object" && !Array.isArray(output) && MODEL_IMAGE_KEY in output) {
+    const { [MODEL_IMAGE_KEY]: img, ...others } = output as Record<string, unknown>;
+    rest = others;
+    const m = img as { data?: unknown; mimeType?: unknown } | null | undefined;
+    if (m && typeof m.data === "string" && typeof m.mimeType === "string")
+      image = { data: m.data, mimeType: m.mimeType };
+  }
+  const text = JSON.stringify(redactForModel(rest) ?? null);
+  if (!image) return [{ type: "text", text }];
+  if (!vision)
+    return [
+      { type: "text", text },
+      { type: "text", text: "(截图未发送:当前上游模型不支持图片输入;用 browser.snapshot 读取页面文字)" },
+    ];
+  return [
+    { type: "text", text },
+    { type: "image", data: image.data, mimeType: image.mimeType },
+  ];
+}
+
 /** before_tool 白名单判定(纯函数,便于单测) */
 export function checkToolCall(
   toolName: string,