feat(browser): vision automation loop with screenshots, coordinate actions and bounded challenge solving

- browser.screenshot returns the page as an image content block (≤1280px wide, JPEG quality
  configurable) for vision-capable upstreams; non-vision upstreams get a redacted text snapshot.
  Login pages / password fields are marked sensitive and withheld unless a
  browser.sensitive_screenshot grant exists and allowSensitive is passed.
- browser.click_at / type_at / scroll / wait act in screenshot pixels; the server converts via
  viewport/DPR, requires automation control and an allowlisted host, and writes a thumbnail
  evidence file (DATA_DIR/tmp/evidence) referenced from agent_run_event.
- browser.solve_challenge runs a bounded child agent_run (vision caps only, maxSteps ≤ 12,
  budget), judges success from page facts, stops immediately on 2FA / SMS / device confirmation,
  and otherwise moves the session to waiting with evidence and a takeover link.
- PiAgentRuntime passes modelImage outputs as pi-ai image content when model.input has image;
  the ledger never stores image data; coordinate actions use per-invocation idempotency keys.
- Tests: image codec, runtime image content, E14 e2e (login solved, 2FA stop, step limit,
  sensitive withheld). Real camofox 1.17.0 check on example.com recorded in the runbook.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

scripts/browser-verify/vision-real.ts

@@ -0,0 +1,80 @@
+/**
+ * VIS-1 真机核验:对真实 camofox 打开 https://example.com/,截图 → 缩放成交给模型的图 → 在图上取链接坐标 →
+ * 按服务端同一套换算(coordinateSpace / screenToPage)点回页面 → 核对跳转。记录视口 / DPR / 截图尺寸。
+ * 用法(只读用现有容器,不改部署):
+ *   CAMOFOX_URL=http://<camofox 容器 IP>:9377 CAMOFOX_ACCESS_KEY=... deno run -A scripts/browser-verify/vision-real.ts [out-dir]
+ * 访问密钥只从环境变量读,不写进文件。
+ */
+import { mkdir, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { createCamofoxBrowserProvider, imageDimensions, prepareImage, staticBrowserSecrets } from "@umt/adapters";
+import type { NavigationBudget } from "@umt/contracts";
+import { coordinateSpace, screenToPage } from "../../apps/server/src/modules/browser/service.ts";
+
+const baseUrl = Deno.env.get("CAMOFOX_URL") ?? "http://camofox:9377";
+const key = Deno.env.get("CAMOFOX_ACCESS_KEY") ?? "";
+const out = Deno.args[0] ?? ".browser-verify-out/vision";
+const PROVIDER = "00000000-0000-4000-8000-00000000f15a";
+const budget: NavigationBudget = {
+  domainAllowlist: ["example.com", "iana.org"],
+  timeoutMs: 30_000,
+  maxRetries: 1,
+  maxBytes: 512 * 1024,
+};
+
+const p = createCamofoxBrowserProvider({
+  providerId: PROVIDER,
+  baseUrl,
+  secrets: staticBrowserSecrets(key ? { [`provider:${PROVIDER}`]: key } : {}),
+});
+const report: Record<string, unknown> = { baseUrl, at: new Date().toISOString() };
+const opened = await p.openSession({
+  providerId: PROVIDER,
+  siteId: "vis1-real",
+  accountId: null,
+  profileKey: `umt-vis1-real-${Date.now()}`,
+  initialUrl: "https://example.com/",
+  budget,
+});
+const ref = { sessionId: opened.session.id };
+try {
+  const shot = await p.screenshot(ref, { maxBytes: 8 * 1024 * 1024 });
+  const metrics = await p.pageMetrics(ref);
+  const src = imageDimensions(shot.data)!;
+  const img = prepareImage(shot.data, { maxWidth: 1280, format: "jpeg", quality: 70 });
+  const space = coordinateSpace(src, img, metrics);
+  await mkdir(out, { recursive: true });
+  await writeFile(join(out, "example-model.jpg"), img.data);
+  // 链接在页面上的位置(CSS 像素)→ 模拟视觉模型在缩放图上给出的坐标
+  const link = (await (
+    await fetch(`${baseUrl}/tabs/${opened.tab.upstreamTabId}/evaluate`, {
+      method: "POST",
+      headers: { "content-type": "application/json", ...(key ? { authorization: `Bearer ${key}` } : {}) },
+      body: JSON.stringify({
+        userId: opened.session.profileKey,
+        expression:
+          "(() => { const r = document.querySelector('a').getBoundingClientRect(); return { x: r.x + r.width / 2, y: r.y + r.height / 2, text: document.querySelector('a').textContent }; })()",
+      }),
+    })
+  ).json()) as { result: { x: number; y: number; text: string } };
+  const imgX = Math.round((link.result.x * space.imageWidth) / space.viewportWidth);
+  const imgY = Math.round((link.result.y * space.imageHeight) / space.viewportHeight);
+  const page = screenToPage(space, imgX, imgY);
+  Object.assign(report, {
+    screenshot: { mimeType: shot.mimeType, ...src, bytes: shot.data.byteLength, modelSafe: shot.modelSafe },
+    metrics: metrics && { ...metrics, textSample: `${metrics.textSample.slice(0, 40)}…` },
+    modelImage: { mimeType: img.mimeType, width: img.width, height: img.height, bytes: img.data.byteLength },
+    space,
+    link: { css: link.result, image: { x: imgX, y: imgY }, page },
+  });
+  await p.click(ref, { x: page.pageX, y: page.pageY });
+  await new Promise((r) => setTimeout(r, 2500));
+  await p.snapshot(ref).catch(() => null);
+  const after = p.sessionInfo(opened.session.id)?.session.currentUrl ?? null;
+  report.afterClickUrl = after;
+  report.pass = !!after && /iana\.org/.test(after);
+} finally {
+  await p.closeSession(opened.session.id).catch(() => {});
+}
+console.log(JSON.stringify(report, null, 2));
+if (!report.pass) Deno.exit(1);