rootandClaude Opus 5.5
c437c6bff6
- browser.screenshot returns the page as an image content block (≤1280px wide, JPEG quality configurable) for vision-capable upstreams; non-vision upstreams get a redacted text snapshot. Login pages / password fields are marked sensitive and withheld unless a browser.sensitive_screenshot grant exists and allowSensitive is passed. - browser.click_at / type_at / scroll / wait act in screenshot pixels; the server converts via viewport/DPR, requires automation control and an allowlisted host, and writes a thumbnail evidence file (DATA_DIR/tmp/evidence) referenced from agent_run_event. - browser.solve_challenge runs a bounded child agent_run (vision caps only, maxSteps ≤ 12, budget), judges success from page facts, stops immediately on 2FA / SMS / device confirmation, and otherwise moves the session to waiting with evidence and a takeover link. - PiAgentRuntime passes modelImage outputs as pi-ai image content when model.input has image; the ledger never stores image data; coordinate actions use per-invocation idempotency keys. - Tests: image codec, runtime image content, E14 e2e (login solved, 2FA stop, step limit, sensitive withheld). Real camofox 1.17.0 check on example.com recorded in the runbook. Co-Authored-By: Claude Opus 5.5 <[email protected]>
81 lines
3.9 KiB
TypeScript
81 lines
3.9 KiB
TypeScript
/**
|
|
* VIS-1 真机核验:对真实 camofox 打开 https://example.com/,截图 → 缩放成交给模型的图 → 在图上取链接坐标 →
|
|
* 按服务端同一套换算(coordinateSpace / screenToPage)点回页面 → 核对跳转。记录视口 / DPR / 截图尺寸。
|
|
* 用法(只读用现有容器,不改部署):
|
|
* CAMOFOX_URL=http://<camofox 容器 IP>:9377 CAMOFOX_ACCESS_KEY=... deno run -A scripts/browser-verify/vision-real.ts [out-dir]
|
|
* 访问密钥只从环境变量读,不写进文件。
|
|
*/
|
|
import { mkdir, writeFile } from "node:fs/promises";
|
|
import { join } from "node:path";
|
|
import { createCamofoxBrowserProvider, imageDimensions, prepareImage, staticBrowserSecrets } from "@umt/adapters";
|
|
import type { NavigationBudget } from "@umt/contracts";
|
|
import { coordinateSpace, screenToPage } from "../../apps/server/src/modules/browser/service.ts";
|
|
|
|
const baseUrl = Deno.env.get("CAMOFOX_URL") ?? "http://camofox:9377";
|
|
const key = Deno.env.get("CAMOFOX_ACCESS_KEY") ?? "";
|
|
const out = Deno.args[0] ?? ".browser-verify-out/vision";
|
|
const PROVIDER = "00000000-0000-4000-8000-00000000f15a";
|
|
const budget: NavigationBudget = {
|
|
domainAllowlist: ["example.com", "iana.org"],
|
|
timeoutMs: 30_000,
|
|
maxRetries: 1,
|
|
maxBytes: 512 * 1024,
|
|
};
|
|
|
|
const p = createCamofoxBrowserProvider({
|
|
providerId: PROVIDER,
|
|
baseUrl,
|
|
secrets: staticBrowserSecrets(key ? { [`provider:${PROVIDER}`]: key } : {}),
|
|
});
|
|
const report: Record<string, unknown> = { baseUrl, at: new Date().toISOString() };
|
|
const opened = await p.openSession({
|
|
providerId: PROVIDER,
|
|
siteId: "vis1-real",
|
|
accountId: null,
|
|
profileKey: `umt-vis1-real-${Date.now()}`,
|
|
initialUrl: "https://example.com/",
|
|
budget,
|
|
});
|
|
const ref = { sessionId: opened.session.id };
|
|
try {
|
|
const shot = await p.screenshot(ref, { maxBytes: 8 * 1024 * 1024 });
|
|
const metrics = await p.pageMetrics(ref);
|
|
const src = imageDimensions(shot.data)!;
|
|
const img = prepareImage(shot.data, { maxWidth: 1280, format: "jpeg", quality: 70 });
|
|
const space = coordinateSpace(src, img, metrics);
|
|
await mkdir(out, { recursive: true });
|
|
await writeFile(join(out, "example-model.jpg"), img.data);
|
|
// 链接在页面上的位置(CSS 像素)→ 模拟视觉模型在缩放图上给出的坐标
|
|
const link = (await (
|
|
await fetch(`${baseUrl}/tabs/${opened.tab.upstreamTabId}/evaluate`, {
|
|
method: "POST",
|
|
headers: { "content-type": "application/json", ...(key ? { authorization: `Bearer ${key}` } : {}) },
|
|
body: JSON.stringify({
|
|
userId: opened.session.profileKey,
|
|
expression:
|
|
"(() => { const r = document.querySelector('a').getBoundingClientRect(); return { x: r.x + r.width / 2, y: r.y + r.height / 2, text: document.querySelector('a').textContent }; })()",
|
|
}),
|
|
})
|
|
).json()) as { result: { x: number; y: number; text: string } };
|
|
const imgX = Math.round((link.result.x * space.imageWidth) / space.viewportWidth);
|
|
const imgY = Math.round((link.result.y * space.imageHeight) / space.viewportHeight);
|
|
const page = screenToPage(space, imgX, imgY);
|
|
Object.assign(report, {
|
|
screenshot: { mimeType: shot.mimeType, ...src, bytes: shot.data.byteLength, modelSafe: shot.modelSafe },
|
|
metrics: metrics && { ...metrics, textSample: `${metrics.textSample.slice(0, 40)}…` },
|
|
modelImage: { mimeType: img.mimeType, width: img.width, height: img.height, bytes: img.data.byteLength },
|
|
space,
|
|
link: { css: link.result, image: { x: imgX, y: imgY }, page },
|
|
});
|
|
await p.click(ref, { x: page.pageX, y: page.pageY });
|
|
await new Promise((r) => setTimeout(r, 2500));
|
|
await p.snapshot(ref).catch(() => null);
|
|
const after = p.sessionInfo(opened.session.id)?.session.currentUrl ?? null;
|
|
report.afterClickUrl = after;
|
|
report.pass = !!after && /iana\.org/.test(after);
|
|
} finally {
|
|
await p.closeSession(opened.session.id).catch(() => {});
|
|
}
|
|
console.log(JSON.stringify(report, null, 2));
|
|
if (!report.pass) Deno.exit(1);
|