feat(browser): vision automation loop with screenshots, coordinate actions and bounded challenge solving

- browser.screenshot returns the page as an image content block (≤1280px wide, JPEG quality
  configurable) for vision-capable upstreams; non-vision upstreams get a redacted text snapshot.
  Login pages / password fields are marked sensitive and withheld unless a
  browser.sensitive_screenshot grant exists and allowSensitive is passed.
- browser.click_at / type_at / scroll / wait act in screenshot pixels; the server converts via
  viewport/DPR, requires automation control and an allowlisted host, and writes a thumbnail
  evidence file (DATA_DIR/tmp/evidence) referenced from agent_run_event.
- browser.solve_challenge runs a bounded child agent_run (vision caps only, maxSteps ≤ 12,
  budget), judges success from page facts, stops immediately on 2FA / SMS / device confirmation,
  and otherwise moves the session to waiting with evidence and a takeover link.
- PiAgentRuntime passes modelImage outputs as pi-ai image content when model.input has image;
  the ledger never stores image data; coordinate actions use per-invocation idempotency keys.
- Tests: image codec, runtime image content, E14 e2e (login solved, 2FA stop, step limit,
  sensitive withheld). Real camofox 1.17.0 check on example.com recorded in the runbook.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

apps/server/src/modules/browser/capabilities.ts

@@ -1,5 +1,5 @@
-import { isLoginUrl } from "@umt/adapters";
-import { CapabilityError, type CapabilityRegistry } from "@umt/agent";
+import { type BrowserProvider, isLoginUrl, type TabRef } from "@umt/adapters";
+import { CapabilityError, type CapabilityRegistry, type HandlerContext } from "@umt/agent";
 import { capabilitySpec } from "@umt/contracts";
 import type { Actor, BrowserService } from "./service.ts";
 import { BrowserServiceError } from "./service.ts";
@@ -28,6 +28,24 @@ async function guard<T>(fn: () => Promise<T>): Promise<T> {
 
 const agent = (actorId: string): Actor => ({ kind: "automation", holder: actorId });
 
+/** 敏感截图(登录页 / 密码框)交给模型的授权:有效的 browser.sensitive_screenshot(或 *)持续授权 */
+export const SENSITIVE_SCREENSHOT_GRANT = "browser.sensitive_screenshot";
+
+async function sensitiveAllowed(svc: BrowserService, want: boolean | undefined): Promise<boolean> {
+  if (!want) return false;
+  const g = await svc.ctx.db
+    .selectFrom("automation_grant")
+    .select("id")
+    .where("status", "=", "active")
+    .where("action", "in", [SENSITIVE_SCREENSHOT_GRANT, "*"])
+    .where((eb) => eb.or([eb("expires_at", "is", null), eb("expires_at", ">", new Date())]))
+    .executeTakeFirst();
+  return !!g;
+}
+
+/** 调用方能不能收图片:Agent 调用按上游是否支持图片输入;其它调用方(HTTP / MCP)缺省给 */
+const wantImage = (ctx: HandlerContext) => ctx.modelImages !== false;
+
 export function registerBrowserCapabilities(registry: CapabilityRegistry, svc: () => BrowserService): void {
   /** sessionId 优先;否则取站点的未关闭会话 */
   const resolveSession = async (input: { sessionId?: string | undefined; siteId?: string | undefined }) => {
@@ -196,4 +214,76 @@ export function registerBrowserCapabilities(registry: CapabilityRegistry, svc: (
       };
     }),
   );
+
+  // ── VIS-1 视觉自动化:截图(图片内容块)与按截图坐标操作 ──
+
+  registry.register(capabilitySpec("browser.screenshot"), (input, ctx) =>
+    guard(async () => {
+      const s = svc();
+      return s.capture(input.sessionId, agent(ctx.actorId), {
+        ...(input.tabId ? { tabId: input.tabId } : {}),
+        wantImage: wantImage(ctx),
+        sensitiveAllowed: await sensitiveAllowed(s, input.allowSensitive),
+      });
+    }),
+  );
+
+  const act = async (
+    ctx: HandlerContext,
+    input: { sessionId: string; allowSensitive?: boolean | undefined },
+    point: { x: number; y: number } | null,
+    fn: (p: BrowserProvider, ref: TabRef, at: { x: number; y: number } | null) => Promise<void>,
+  ) => {
+    const s = svc();
+    const page = point ? s.toPage(input.sessionId, point.x, point.y) : null;
+    const at = page ? { x: page.pageX, y: page.pageY } : null;
+    const shot = await s.visionAct(input.sessionId, agent(ctx.actorId), (p, ref) => fn(p, ref, at), {
+      wantImage: wantImage(ctx),
+      sensitiveAllowed: await sensitiveAllowed(s, input.allowSensitive),
+    });
+    return { ok: true as const, pageX: at?.x ?? null, pageY: at?.y ?? null, ...shot };
+  };
+
+  registry.register(capabilitySpec("browser.click_at"), (input, ctx) =>
+    guard(async () => {
+      if (input.button && input.button !== "left")
+        throw new BrowserServiceError("bad_request", "浏览器服务(camofox 1.17.0)只支持左键点击");
+      return act(ctx, input, { x: input.x, y: input.y }, (p, ref, at) => p.click(ref, at!));
+    }),
+  );
+
+  registry.register(capabilitySpec("browser.type_at"), (input, ctx) =>
+    guard(async () => {
+      if ((input.x === undefined) !== (input.y === undefined))
+        throw new BrowserServiceError("bad_request", "x 与 y 要一起给（都不给 = 输入到当前焦点）");
+      const point = input.x !== undefined && input.y !== undefined ? { x: input.x, y: input.y } : null;
+      return act(ctx, input, point, async (p, ref, at) => {
+        if (at) await p.click(ref, at);
+        await p.type(ref, null, { kind: "text", text: input.text });
+        if (input.pressEnter) await p.press(ref, "Enter");
+      });
+    }),
+  );
+
+  registry.register(capabilitySpec("browser.scroll"), (input, ctx) =>
+    guard(async () => {
+      if (!input.dx && !input.dy) throw new BrowserServiceError("bad_request", "dx 与 dy 至少一个不为 0");
+      return act(ctx, input, null, async (p, ref) => {
+        if (input.dy) await p.scroll(ref, { direction: input.dy > 0 ? "down" : "up", amount: Math.abs(input.dy) });
+        if (input.dx) await p.scroll(ref, { direction: input.dx > 0 ? "right" : "left", amount: Math.abs(input.dx) });
+      });
+    }),
+  );
+
+  registry.register(capabilitySpec("browser.wait"), (input, ctx) =>
+    guard(async () => {
+      const s = svc();
+      await new Promise((r) => setTimeout(r, input.ms));
+      const shot = await s.capture(input.sessionId, agent(ctx.actorId), {
+        wantImage: wantImage(ctx),
+        sensitiveAllowed: await sensitiveAllowed(s, input.allowSensitive),
+      });
+      return { ok: true as const, pageX: null, pageY: null, ...shot };
+    }),
+  );
 }