/** * VIS-1 真机核验:对真实 camofox 打开 https://example.com/,截图 → 缩放成交给模型的图 → 在图上取链接坐标 → * 按服务端同一套换算(coordinateSpace / screenToPage)点回页面 → 核对跳转。记录视口 / DPR / 截图尺寸。 * 用法(只读用现有容器,不改部署): * CAMOFOX_URL=http://:9377 CAMOFOX_ACCESS_KEY=... deno run -A scripts/browser-verify/vision-real.ts [out-dir] * 访问密钥只从环境变量读,不写进文件。 */ import { mkdir, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { createCamofoxBrowserProvider, imageDimensions, prepareImage, staticBrowserSecrets } from "@umt/adapters"; import type { NavigationBudget } from "@umt/contracts"; import { coordinateSpace, screenToPage } from "../../apps/server/src/modules/browser/service.ts"; const baseUrl = Deno.env.get("CAMOFOX_URL") ?? "http://camofox:9377"; const key = Deno.env.get("CAMOFOX_ACCESS_KEY") ?? ""; const out = Deno.args[0] ?? ".browser-verify-out/vision"; const PROVIDER = "00000000-0000-4000-8000-00000000f15a"; const budget: NavigationBudget = { domainAllowlist: ["example.com", "iana.org"], timeoutMs: 30_000, maxRetries: 1, maxBytes: 512 * 1024, }; const p = createCamofoxBrowserProvider({ providerId: PROVIDER, baseUrl, secrets: staticBrowserSecrets(key ? { [`provider:${PROVIDER}`]: key } : {}), }); const report: Record = { baseUrl, at: new Date().toISOString() }; const opened = await p.openSession({ providerId: PROVIDER, siteId: "vis1-real", accountId: null, profileKey: `umt-vis1-real-${Date.now()}`, initialUrl: "https://example.com/", budget, }); const ref = { sessionId: opened.session.id }; try { const shot = await p.screenshot(ref, { maxBytes: 8 * 1024 * 1024 }); const metrics = await p.pageMetrics(ref); const src = imageDimensions(shot.data)!; const img = prepareImage(shot.data, { maxWidth: 1280, format: "jpeg", quality: 70 }); const space = coordinateSpace(src, img, metrics); await mkdir(out, { recursive: true }); await writeFile(join(out, "example-model.jpg"), img.data); // 链接在页面上的位置(CSS 像素)→ 模拟视觉模型在缩放图上给出的坐标 const link = (await ( await fetch(`${baseUrl}/tabs/${opened.tab.upstreamTabId}/evaluate`, { method: "POST", headers: { "content-type": "application/json", ...(key ? { authorization: `Bearer ${key}` } : {}) }, body: JSON.stringify({ userId: opened.session.profileKey, expression: "(() => { const r = document.querySelector('a').getBoundingClientRect(); return { x: r.x + r.width / 2, y: r.y + r.height / 2, text: document.querySelector('a').textContent }; })()", }), }) ).json()) as { result: { x: number; y: number; text: string } }; const imgX = Math.round((link.result.x * space.imageWidth) / space.viewportWidth); const imgY = Math.round((link.result.y * space.imageHeight) / space.viewportHeight); const page = screenToPage(space, imgX, imgY); Object.assign(report, { screenshot: { mimeType: shot.mimeType, ...src, bytes: shot.data.byteLength, modelSafe: shot.modelSafe }, metrics: metrics && { ...metrics, textSample: `${metrics.textSample.slice(0, 40)}…` }, modelImage: { mimeType: img.mimeType, width: img.width, height: img.height, bytes: img.data.byteLength }, space, link: { css: link.result, image: { x: imgX, y: imgY }, page }, }); await p.click(ref, { x: page.pageX, y: page.pageY }); await new Promise((r) => setTimeout(r, 2500)); await p.snapshot(ref).catch(() => null); const after = p.sessionInfo(opened.session.id)?.session.currentUrl ?? null; report.afterClickUrl = after; report.pass = !!after && /iana\.org/.test(after); } finally { await p.closeSession(opened.session.id).catch(() => {}); } console.log(JSON.stringify(report, null, 2)); if (!report.pass) Deno.exit(1);