feat(browser): vision automation loop with screenshots, coordinate actions and bounded challenge solving

- browser.screenshot returns the page as an image content block (≤1280px wide, JPEG quality
  configurable) for vision-capable upstreams; non-vision upstreams get a redacted text snapshot.
  Login pages / password fields are marked sensitive and withheld unless a
  browser.sensitive_screenshot grant exists and allowSensitive is passed.
- browser.click_at / type_at / scroll / wait act in screenshot pixels; the server converts via
  viewport/DPR, requires automation control and an allowlisted host, and writes a thumbnail
  evidence file (DATA_DIR/tmp/evidence) referenced from agent_run_event.
- browser.solve_challenge runs a bounded child agent_run (vision caps only, maxSteps ≤ 12,
  budget), judges success from page facts, stops immediately on 2FA / SMS / device confirmation,
  and otherwise moves the session to waiting with evidence and a takeover link.
- PiAgentRuntime passes modelImage outputs as pi-ai image content when model.input has image;
  the ledger never stores image data; coordinate actions use per-invocation idempotency keys.
- Tests: image codec, runtime image content, E14 e2e (login solved, 2FA stop, step limit,
  sensitive withheld). Real camofox 1.17.0 check on example.com recorded in the runbook.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

apps/server/src/modules/browser/service.ts

@@ -8,16 +8,23 @@ import {
   createCamofoxBrowserProvider,
   type DownloadSink,
   type ElementTarget,
+  hostAllowed,
+  imageDimensions,
+  type PageMetrics,
   parseTorrent,
+  prepareImage,
   profileKeyFor,
   type TabRef,
+  toBase64,
   totpCode,
 } from "@umt/adapters";
 import type {
   BrowserCapabilities,
   BrowserControlLease,
   BrowserController,
+  BrowserCoordinateSpace,
   BrowserDownloadVerification,
+  BrowserEvidenceRef,
   BrowserHealth,
   BrowserProviderView,
   BrowserSession,
@@ -26,7 +33,14 @@ import type {
   DownloadArtifact,
   NavigationBudget,
 } from "@umt/contracts";
-import type { BrowserControlLeaseRow, BrowserProviderRow, BrowserSessionRow, BrowserTabRow, SiteRow } from "@umt/db";
+import {
+  type BrowserControlLeaseRow,
+  type BrowserProviderRow,
+  type BrowserSessionRow,
+  type BrowserTabRow,
+  type SiteRow,
+  sql,
+} from "@umt/db";
 import type { SourcesRuntime } from "../sources-identity/runtime.ts";
 import type { ServerContext } from "../types.ts";
 
@@ -45,6 +59,10 @@ export interface BrowserModuleOptions {
   /** 下载产物暂存目录;缺省 DATA_DIR/tmp/browser-downloads */
   stagingDir?: string;
   downloadPollMs?: number;
+  /** 视觉自动化每步证据缩略图目录;缺省 DATA_DIR/tmp/evidence */
+  evidenceDir?: string;
+  /** 坐标动作之后等页面稳定再截图的毫秒数;缺省 400 */
+  settleMs?: number;
   /** 测试替换 provider 实现 */
   providerFactory?: (row: BrowserProviderRow, deps: ProviderDeps) => BrowserProvider;
 }
@@ -101,6 +119,9 @@ const HEALTH: Record<BrowserHealth["status"], "healthy" | "degraded" | "offline"
 /** 验证码 / 二次认证 / Cloudflare 挑战:自动化不能完成,转等待处理 */
 const HUMAN_CHALLENGE =
   /验证码|captcha|二次验证|两步验证|二步验证|2fa|two-?step|one-time code|cf-challenge|just a moment|checking your browser/i;
+/** 二次验证 / 短信 / 本人设备确认:任何情况下都不自动处理(视觉子流程里也一样),立即转人工 */
+export const TWO_FACTOR =
+  /二次验证|两步验证|二步验证|双重验证|短信验证码|验证码已发送|本人设备|设备确认|身份验证器|2fa|two-?step|two-?factor|one-time (?:code|password)|authenticator app|verification code (?:has been |was )?sent|confirm (?:it'?s|this is) you|approve (?:the )?sign.?in/i;
 
 const iso = (d: Date | string | null | undefined) => (d ? new Date(d).toISOString() : null);
 type Json = Record<string, unknown>;
@@ -987,11 +1008,16 @@ export class BrowserService {
     await this.rateLimit(site);
   }
 
-  /** 快照后判断是否需要人工:只在自动化控制下、页面出现验证码/二次认证/挑战时转等待处理 */
+  /**
+   * 快照后判断是否需要人工:只在自动化控制下。二次验证 / 短信 / 本人设备确认一律转等待处理;
+   * 验证码 / 挑战页在视觉子流程(browser.solve_challenge)进行中不转,由子流程处理,失败再转。
+   */
   async afterAutomationSnapshot(id: string, text: string, loginPage: boolean): Promise<void> {
     const row = await this.sessionRow(id);
     if (row.controller !== "automation") return;
-    if (HUMAN_CHALLENGE.test(text)) await this.enterWaiting(row, loginPage ? "captcha" : "challenge");
+    if (TWO_FACTOR.test(text)) await this.enterWaiting(row, "two_factor");
+    else if (HUMAN_CHALLENGE.test(text) && !this.challenges.has(id))
+      await this.enterWaiting(row, loginPage ? "captcha" : "challenge");
   }
 
   async requestHuman(id: string, reason: string): Promise<void> {
@@ -1001,6 +1027,241 @@ export class BrowserService {
     await this.enterWaiting(row, reason || "agent_request");
   }
 
+  // ---------- VIS-1 视觉自动化 ----------
+
+  /** 每个会话最近一次截图的坐标系(截图像素 → 页面 CSS 像素);只在进程内 */
+  private readonly coordSpaces = new Map<string, BrowserCoordinateSpace>();
+  /** 进行中的视觉子流程:会话 → 已写的证据 */
+  private readonly challenges = new Map<string, { evidence: BrowserEvidenceRef[] }>();
+
+  private evidenceDir(): string {
+    if (this.opts.evidenceDir) return this.opts.evidenceDir;
+    const dataDir = (this.ctx.config as { DATA_DIR?: string }).DATA_DIR ?? "./data";
+    return join(resolve(dataDir), "tmp", "evidence");
+  }
+
+  /** 证据缩略图(≤480 宽 JPEG)落 DATA_DIR/tmp/evidence(0600),超过 7 天的顺手清掉;返回引用 */
+  async writeEvidence(sessionId: string, shot: Uint8Array): Promise<BrowserEvidenceRef | null> {
+    try {
+      const thumb = prepareImage(shot, { maxWidth: 480, format: "jpeg", quality: 60 });
+      const dir = this.evidenceDir();
+      await mkdir(dir, { recursive: true, mode: 0o700 });
+      if (Math.random() < 0.05) await this.cleanupDir(dir, 7 * 24 * 3_600_000);
+      const id = crypto.randomUUID();
+      await writeFile(join(dir, `${id}.jpg`), thumb.data, { mode: 0o600 });
+      const ref: BrowserEvidenceRef = { id, ref: `evidence/${id}.jpg`, capturedAt: this.now().toISOString() };
+      this.challenges.get(sessionId)?.evidence.push(ref);
+      return ref;
+    } catch (err) {
+      this.ctx.logger.warn({ sessionId, err: { message: (err as Error).message } }, "截图证据写入失败");
+      return null;
+    }
+  }
+
+  evidencePath(ref: string): string {
+    return join(this.evidenceDir(), ref.replace(/^evidence\//, "").replace(/[/\\]/g, "_"));
+  }
+
+  private async cleanupDir(dir: string, maxAgeMs: number): Promise<void> {
+    const cutoff = Date.now() - maxAgeMs;
+    for (const f of await readdir(dir).catch(() => [] as string[])) {
+      const p = join(dir, f);
+      const st = await stat(p).catch(() => null);
+      if (st && st.mtimeMs < cutoff) await rm(p, { force: true }).catch(() => {});
+    }
+  }
+
+  /** 交给模型的截图参数:browser_provider.config 的 screenshotMaxWidth(≤1280)/ screenshotFormat / screenshotQuality */
+  private imageOptions(prow: BrowserProviderRow | null): {
+    maxWidth: number;
+    format: "png" | "jpeg";
+    quality: number;
+  } {
+    const cfg = obj(prow?.config);
+    const w = typeof cfg.screenshotMaxWidth === "number" ? cfg.screenshotMaxWidth : 1280;
+    const q = typeof cfg.screenshotQuality === "number" ? cfg.screenshotQuality : 70;
+    return {
+      maxWidth: Math.max(320, Math.min(1280, Math.round(w))),
+      format: cfg.screenshotFormat === "png" ? "png" : "jpeg",
+      quality: Math.max(20, Math.min(95, Math.round(q))),
+    };
+  }
+
+  /**
+   * 截图 + 页面度量 + 证据:缩放后的截图作为模型坐标系登记在会话上;判定敏感(登录页 / 密码框 / 地址带凭据);
+   * 页面文字出现二次验证 / 短信 / 本人设备确认时,自动化控制下的会话立即转等待处理。
+   * wantImage=false(上游不支持图片)或敏感且未授权时不给图片,改给已脱敏的文字快照。
+   */
+  async capture(
+    id: string,
+    actor: Actor,
+    opts: { tabId?: string; wantImage: boolean; sensitiveAllowed: boolean },
+  ): Promise<VisionShot> {
+    const got = await this.run(
+      id,
+      actor,
+      async (p, ref, x) => {
+        const shot = await p.screenshot(ref, { maxBytes: 8 * 1024 * 1024 });
+        const metrics = await p.pageMetrics(ref).catch(() => null);
+        const prow = await this.providerRow(x.row.provider_id).catch(() => null);
+        return { shot, metrics, prow };
+      },
+      { write: false, persist: false, ...(opts.tabId ? { tabId: opts.tabId } : {}) },
+    );
+    const { shot, metrics } = got;
+    const src = imageDimensions(shot.data);
+    const img = prepareImage(shot.data, this.imageOptions(got.prow));
+    const space = coordinateSpace(src ?? { width: img.width, height: img.height }, img, metrics);
+    this.coordSpaces.set(id, space);
+    const sensitive = !shot.modelSafe || metrics?.passwordField === true;
+    let row = await this.sessionRow(id);
+    if (row.controller === "automation" && metrics && TWO_FACTOR.test(metrics.textSample))
+      row = await this.enterWaiting(row, "two_factor");
+    const waitingReason =
+      row.controller === "waiting" ? ((obj(row.recovery_state) as RecoveryState).waitingReason ?? null) : null;
+    const evidence = await this.writeEvidence(id, shot.data);
+    const blocked = waitingReason === "two_factor";
+    const give = opts.wantImage && !blocked && (!sensitive || opts.sensitiveAllowed);
+    let note: string | null = null;
+    if (blocked) note = "页面要求二次验证 / 短信 / 本人设备确认：不自动处理，会话已转等待处理，由人在原站视图接管";
+    else if (!opts.wantImage) note = "当前上游模型不支持图片输入，截图没有交给模型；下面是页面的文字快照";
+    else if (sensitive && !opts.sensitiveAllowed)
+      note =
+        "登录页 / 含密码框的截图属于敏感截图，默认不交给模型（需要 browser.sensitive_screenshot 持续授权并传 allowSensitive）；下面是已脱敏的文字快照";
+    let textSnapshot: string | null = null;
+    if (!give && !blocked) {
+      const snap = await this.run(id, actor, (p, ref) => p.snapshot(ref), {
+        write: false,
+        persist: false,
+        ...(opts.tabId ? { tabId: opts.tabId } : {}),
+      }).catch(() => null);
+      textSnapshot = snap ? snap.text.slice(0, 6000) : null;
+    }
+    return {
+      evidence,
+      url: shot.url,
+      controller: row.controller,
+      sensitive,
+      imageForModel: give,
+      note,
+      textSnapshot,
+      coordinates: space,
+      waitingReason,
+      modelImage: give
+        ? { mimeType: img.mimeType, data: toBase64(img.data), width: img.width, height: img.height }
+        : null,
+    };
+  }
+
+  /** 截图像素 → 页面 CSS 像素;没有截图过(或坐标越界)报错 */
+  toPage(id: string, x: number, y: number): { pageX: number; pageY: number } {
+    const sp = this.coordSpaces.get(id);
+    if (!sp) throw new BrowserServiceError("precondition", "先用 browser.screenshot 截图，坐标以截图像素为准");
+    if (x < 0 || y < 0 || x >= sp.imageWidth || y >= sp.imageHeight)
+      throw new BrowserServiceError("bad_request", `坐标 (${x}, ${y}) 超出截图范围 ${sp.imageWidth}×${sp.imageHeight}`);
+    return screenToPage(sp, x, y);
+  }
+
+  /**
+   * 坐标类动作:自动化控制权 + 当前页面在站点域名内;动作之后旧坐标系作废(页面可能已变化),
+   * 等 settleMs 再截一张作为新坐标系与证据。
+   */
+  async visionAct(
+    id: string,
+    actor: Actor,
+    act: (p: BrowserProvider, ref: TabRef) => Promise<void>,
+    opts: { wantImage: boolean; sensitiveAllowed: boolean },
+  ): Promise<VisionShot> {
+    await this.run(id, actor, async (p, ref, x) => {
+      const url = p.sessionInfo(id)?.session.currentUrl ?? null;
+      const host = url ? safeUrl(url)?.hostname : undefined;
+      if (!host || !hostAllowed(host, x.budget.domainAllowlist))
+        throw new BrowserServiceError("precondition", "当前页面不在站点域名内，先用 browser.navigate 回到站点页面");
+      await act(p, ref);
+    });
+    this.coordSpaces.delete(id);
+    await new Promise((r) => setTimeout(r, this.opts.settleMs ?? 400));
+    return this.capture(id, actor, opts);
+  }
+
+  /**
+   * 开始视觉子流程:会话须在自动化控制下,或因验证码 / 页面挑战在等待处理(此时收回自动化控制,待处理事项保留,
+   * 成功后关闭);人工接管中、二次验证、租约过期等其它等待原因一律拒绝。
+   */
+  async beginChallenge(id: string): Promise<{ row: BrowserSessionRow; url: string | null }> {
+    if (this.challenges.has(id)) throw new BrowserServiceError("conflict", "这个会话已有进行中的挑战处理");
+    let row = await this.expireLeases(await this.sessionRow(id));
+    const rs = obj(row.recovery_state) as RecoveryState;
+    if (row.controller === "waiting" && SOLVABLE.includes(rs.waitingReason ?? "")) {
+      const { provider } = await this.loaded(row);
+      provider.setController(id, "automation");
+      row = await this.ctx.db
+        .updateTable("browser_session")
+        .set({
+          controller: "automation",
+          recovery_state: JSON.stringify({ ...rs, waitingReason: null, needsSnapshot: true }),
+          updated_at: this.now(),
+        })
+        .where("id", "=", id)
+        .where("controller", "=", "waiting")
+        .returningAll()
+        .executeTakeFirstOrThrow();
+    } else if (row.controller !== "automation") {
+      throw new BrowserServiceError(
+        "conflict",
+        row.controller === "human"
+          ? "会话正由人工接管，不能自动处理"
+          : row.controller === "closed"
+            ? "会话已关闭"
+            : `会话在等待人工处理(${REASON_TEXT[rs.waitingReason ?? ""] ?? rs.waitingReason ?? "未知原因"})，不能自动处理`,
+      );
+    }
+    this.challenges.set(id, { evidence: [] });
+    return { row, url: row.current_url };
+  }
+
+  challengeEvidence(id: string): BrowserEvidenceRef[] {
+    return [...(this.challenges.get(id)?.evidence ?? [])];
+  }
+
+  /**
+   * 结束视觉子流程:成功 → 关闭之前的等待处理事项;否则会话转等待处理(原因 two_factor / challenge_unsolved),
+   * 待处理事项附上每步证据与子流程运行,供人在原站视图接管。
+   */
+  async endChallenge(
+    id: string,
+    out: { solved: boolean; reason: string; summary: string; childRunId: string | null },
+  ): Promise<{ decisionId: string | null; controller: BrowserController }> {
+    const evidence = this.challengeEvidence(id);
+    this.challenges.delete(id);
+    let row = await this.sessionRow(id);
+    if (out.solved) {
+      if (row.controller === "automation") {
+        await this.resolveWaiting(row, "system:browser.solve_challenge");
+        await this.patchRecovery(row, { waitingDecisionId: null });
+      }
+      return { decisionId: null, controller: row.controller };
+    }
+    if (row.controller === "automation") row = await this.enterWaiting(row, out.reason);
+    const decisionId = (obj(row.recovery_state) as RecoveryState).waitingDecisionId ?? null;
+    if (decisionId)
+      await this.ctx.db
+        .updateTable("decision")
+        .set({
+          evidence: sql`evidence || ${JSON.stringify({
+            challenge: {
+              reason: out.reason,
+              summary: out.summary.slice(0, 1000),
+              childRunId: out.childRunId,
+              screenshots: evidence.slice(-12),
+            },
+          })}::jsonb`,
+        })
+        .where("id", "=", decisionId)
+        .execute();
+    return { decisionId, controller: row.controller };
+  }
+
   // ---------- 下载 ----------
 
   /** 暂存产物核验:解析种子、按 infohash / 站点资源 ID 匹配本地资源、会话内去重 */
@@ -1081,6 +1342,8 @@ export class BrowserService {
 const REASON_TEXT: Record<string, string> = {
   captcha: "登录页需要验证码",
   challenge: "页面需要人工验证",
+  two_factor: "需要二次验证或本人设备确认",
+  challenge_unsolved: "自动处理页面挑战未成功",
   login_required: "登录已失效",
   lease_expired: "人工接管超时未交还",
   agent_request: "Agent 请求人工处理",
@@ -1150,6 +1413,49 @@ export function toLease(r: BrowserControlLeaseRow): BrowserControlLease {
   };
 }
 
+/** 视觉子流程可以接手的等待原因(验证码 / 页面挑战 / 上次没处理成功) */
+const SOLVABLE = ["captcha", "challenge", "challenge_unsolved"];
+
+/** capture / visionAct 的结果(能力输出的公共部分) */
+export interface VisionShot {
+  evidence: BrowserEvidenceRef | null;
+  url: string | null;
+  controller: BrowserController;
+  sensitive: boolean;
+  imageForModel: boolean;
+  note: string | null;
+  textSnapshot: string | null;
+  coordinates: BrowserCoordinateSpace | null;
+  waitingReason: string | null;
+  modelImage: { mimeType: "image/png" | "image/jpeg"; data: string; width: number; height: number } | null;
+}
+
+/** 模型坐标系:缩放后的截图 ↔ 页面视口;取不到页面度量时按原始截图尺寸、DPR 1 推定 */
+export function coordinateSpace(
+  src: { width: number; height: number },
+  img: { width: number; height: number },
+  m: PageMetrics | null,
+): BrowserCoordinateSpace {
+  return {
+    imageWidth: img.width,
+    imageHeight: img.height,
+    sourceWidth: src.width,
+    sourceHeight: src.height,
+    viewportWidth: m?.viewportWidth ?? src.width,
+    viewportHeight: m?.viewportHeight ?? src.height,
+    devicePixelRatio: m?.devicePixelRatio ?? 1,
+    assumed: !m,
+  };
+}
+
+/** 截图像素(交给模型的缩放图)→ 页面 CSS 像素:按像素中心换算,四舍五入 */
+export function screenToPage(sp: BrowserCoordinateSpace, x: number, y: number): { pageX: number; pageY: number } {
+  return {
+    pageX: Math.round(((x + 0.5) * sp.viewportWidth) / sp.imageWidth - 0.5),
+    pageY: Math.round(((y + 0.5) * sp.viewportHeight) / sp.imageHeight - 0.5),
+  };
+}
+
 /** 截图像素尺寸:PNG IHDR / JPEG SOF */
 export function imageSize(data: Uint8Array): { width: number | null; height: number | null } {
   if (data.length > 24 && data[0] === 0x89 && data[1] === 0x50) {