feat(browser): vision automation loop with screenshots, coordinate actions and bounded challenge solving
- browser.screenshot returns the page as an image content block (≤1280px wide, JPEG quality configurable) for vision-capable upstreams; non-vision upstreams get a redacted text snapshot. Login pages / password fields are marked sensitive and withheld unless a browser.sensitive_screenshot grant exists and allowSensitive is passed. - browser.click_at / type_at / scroll / wait act in screenshot pixels; the server converts via viewport/DPR, requires automation control and an allowlisted host, and writes a thumbnail evidence file (DATA_DIR/tmp/evidence) referenced from agent_run_event. - browser.solve_challenge runs a bounded child agent_run (vision caps only, maxSteps ≤ 12, budget), judges success from page facts, stops immediately on 2FA / SMS / device confirmation, and otherwise moves the session to waiting with evidence and a takeover link. - PiAgentRuntime passes modelImage outputs as pi-ai image content when model.input has image; the ledger never stores image data; coordinate actions use per-invocation idempotency keys. - Tests: image codec, runtime image content, E14 e2e (login solved, 2FA stop, step limit, sensitive withheld). Real camofox 1.17.0 check on example.com recorded in the runbook. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
0765264a47
commit
c437c6bff6
34 files changed
+2194
-35
No files matched your search
@@ -0,0 +1,275 @@
|
||||
/**
|
||||
* 截图的解码、缩放与编码(VIS-1):交给视觉模型的截图缩放到不超过 maxWidth(缺省 1280)像素宽,
|
||||
* 证据缩略图更小。PNG 自己解码 / 编码(node:zlib),JPEG 用 jpeg-js(纯 JS,无原生依赖,deno compile 可嵌入)。
|
||||
* 只支持浏览器截图实际会出现的格式:8 位 PNG(灰度 / RGB / 调色板 / 带 alpha,非隔行)与基线 JPEG。
|
||||
*/
|
||||
import { deflateSync, inflateSync } from "node:zlib";
|
||||
import { decode as decodeJpeg, encode as encodeJpeg } from "jpeg-js";
|
||||
|
||||
export interface RgbaImage {
|
||||
width: number;
|
||||
height: number;
|
||||
/** RGBA,每像素 4 字节 */
|
||||
data: Uint8Array;
|
||||
}
|
||||
|
||||
export interface EncodedImage {
|
||||
mimeType: "image/png" | "image/jpeg";
|
||||
data: Uint8Array;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export interface PrepareImageOptions {
|
||||
/** 最大宽度(像素);原图更窄时不放大 */
|
||||
maxWidth?: number;
|
||||
/** 输出格式;缺省 jpeg */
|
||||
format?: "png" | "jpeg";
|
||||
/** JPEG 质量 1–100,缺省 70 */
|
||||
quality?: number;
|
||||
}
|
||||
|
||||
/** PNG IHDR / JPEG SOF 里的像素尺寸;不认识的格式返回 null */
|
||||
export function imageDimensions(data: Uint8Array): { width: number; height: number } | null {
|
||||
if (data.length > 24 && data[0] === 0x89 && data[1] === 0x50 && data[2] === 0x4e && data[3] === 0x47) {
|
||||
const v = new DataView(data.buffer, data.byteOffset, data.byteLength);
|
||||
return { width: v.getUint32(16), height: v.getUint32(20) };
|
||||
}
|
||||
if (data[0] === 0xff && data[1] === 0xd8) {
|
||||
let i = 2;
|
||||
while (i + 9 < data.length) {
|
||||
if (data[i] !== 0xff) break;
|
||||
const marker = data[i + 1]!;
|
||||
const len = (data[i + 2]! << 8) | data[i + 3]!;
|
||||
if (marker >= 0xc0 && marker <= 0xc3)
|
||||
return { width: (data[i + 7]! << 8) | data[i + 8]!, height: (data[i + 5]! << 8) | data[i + 6]! };
|
||||
i += 2 + len;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function decodeImage(data: Uint8Array): RgbaImage {
|
||||
if (data[0] === 0x89 && data[1] === 0x50) return decodePng(data);
|
||||
if (data[0] === 0xff && data[1] === 0xd8) {
|
||||
const img = decodeJpeg(data, { useTArray: true, formatAsRGBA: true, maxMemoryUsageInMB: 256 });
|
||||
return { width: img.width, height: img.height, data: img.data };
|
||||
}
|
||||
throw new Error("不支持的图片格式");
|
||||
}
|
||||
|
||||
/** 解码 8 位、非隔行 PNG(颜色类型 0/2/3/4/6) */
|
||||
export function decodePng(bytes: Uint8Array): RgbaImage {
|
||||
const v = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
||||
let pos = 8;
|
||||
let width = 0;
|
||||
let height = 0;
|
||||
let depth = 0;
|
||||
let color = 0;
|
||||
let interlace = 0;
|
||||
let palette: Uint8Array | null = null;
|
||||
let trns: Uint8Array | null = null;
|
||||
const idat: Uint8Array[] = [];
|
||||
while (pos + 8 <= bytes.length) {
|
||||
const len = v.getUint32(pos);
|
||||
const type = String.fromCharCode(bytes[pos + 4]!, bytes[pos + 5]!, bytes[pos + 6]!, bytes[pos + 7]!);
|
||||
const body = bytes.subarray(pos + 8, pos + 8 + len);
|
||||
if (type === "IHDR") {
|
||||
width = v.getUint32(pos + 8);
|
||||
height = v.getUint32(pos + 12);
|
||||
depth = bytes[pos + 16]!;
|
||||
color = bytes[pos + 17]!;
|
||||
interlace = bytes[pos + 20]!;
|
||||
} else if (type === "PLTE") palette = body;
|
||||
else if (type === "tRNS") trns = body;
|
||||
else if (type === "IDAT") idat.push(body);
|
||||
else if (type === "IEND") break;
|
||||
pos += 12 + len;
|
||||
}
|
||||
if (depth !== 8 || interlace !== 0) throw new Error(`不支持的 PNG(位深 ${depth},隔行 ${interlace})`);
|
||||
const channels = ({ 0: 1, 2: 3, 3: 1, 4: 2, 6: 4 } as Record<number, number>)[color];
|
||||
if (!channels) throw new Error(`不支持的 PNG 颜色类型 ${color}`);
|
||||
if (width <= 0 || height <= 0 || width * height > 40_000_000) throw new Error("PNG 尺寸异常");
|
||||
const raw = new Uint8Array(inflateSync(concat(idat)));
|
||||
const stride = width * channels;
|
||||
const out = new Uint8Array(width * height * 4);
|
||||
let prev = new Uint8Array(stride);
|
||||
let cur = new Uint8Array(stride);
|
||||
for (let y = 0; y < height; y++) {
|
||||
const off = y * (stride + 1);
|
||||
const filter = raw[off]!;
|
||||
for (let i = 0; i < stride; i++) {
|
||||
const x = raw[off + 1 + i]!;
|
||||
const a = i >= channels ? cur[i - channels]! : 0;
|
||||
const b = prev[i]!;
|
||||
const c = i >= channels ? prev[i - channels]! : 0;
|
||||
let val: number;
|
||||
switch (filter) {
|
||||
case 0:
|
||||
val = x;
|
||||
break;
|
||||
case 1:
|
||||
val = x + a;
|
||||
break;
|
||||
case 2:
|
||||
val = x + b;
|
||||
break;
|
||||
case 3:
|
||||
val = x + ((a + b) >> 1);
|
||||
break;
|
||||
case 4: {
|
||||
const p = a + b - c;
|
||||
const pa = Math.abs(p - a);
|
||||
const pb = Math.abs(p - b);
|
||||
const pc = Math.abs(p - c);
|
||||
val = x + (pa <= pb && pa <= pc ? a : pb <= pc ? b : c);
|
||||
break;
|
||||
}
|
||||
default:
|
||||
throw new Error(`PNG 过滤类型 ${filter} 非法`);
|
||||
}
|
||||
cur[i] = val & 0xff;
|
||||
}
|
||||
for (let px = 0; px < width; px++) {
|
||||
const o = (y * width + px) * 4;
|
||||
const s = px * channels;
|
||||
if (color === 6) {
|
||||
out[o] = cur[s]!;
|
||||
out[o + 1] = cur[s + 1]!;
|
||||
out[o + 2] = cur[s + 2]!;
|
||||
out[o + 3] = cur[s + 3]!;
|
||||
} else if (color === 2) {
|
||||
out[o] = cur[s]!;
|
||||
out[o + 1] = cur[s + 1]!;
|
||||
out[o + 2] = cur[s + 2]!;
|
||||
out[o + 3] = 255;
|
||||
} else if (color === 0 || color === 4) {
|
||||
out[o] = out[o + 1] = out[o + 2] = cur[s]!;
|
||||
out[o + 3] = color === 4 ? cur[s + 1]! : 255;
|
||||
} else {
|
||||
const idx = cur[s]!;
|
||||
out[o] = palette?.[idx * 3] ?? 0;
|
||||
out[o + 1] = palette?.[idx * 3 + 1] ?? 0;
|
||||
out[o + 2] = palette?.[idx * 3 + 2] ?? 0;
|
||||
out[o + 3] = trns?.[idx] ?? 255;
|
||||
}
|
||||
}
|
||||
[prev, cur] = [cur, prev];
|
||||
}
|
||||
return { width, height, data: out };
|
||||
}
|
||||
|
||||
/** 编码为 8 位 RGBA PNG(每行 filter 0,deflate 压缩) */
|
||||
export function encodePng(img: RgbaImage): Uint8Array {
|
||||
const stride = img.width * 4;
|
||||
const raw = new Uint8Array((stride + 1) * img.height);
|
||||
for (let y = 0; y < img.height; y++) raw.set(img.data.subarray(y * stride, (y + 1) * stride), y * (stride + 1) + 1);
|
||||
const ihdr = new Uint8Array(13);
|
||||
const hv = new DataView(ihdr.buffer);
|
||||
hv.setUint32(0, img.width);
|
||||
hv.setUint32(4, img.height);
|
||||
ihdr[8] = 8;
|
||||
ihdr[9] = 6;
|
||||
return concat([
|
||||
new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]),
|
||||
chunk("IHDR", ihdr),
|
||||
chunk("IDAT", new Uint8Array(deflateSync(raw))),
|
||||
chunk("IEND", new Uint8Array()),
|
||||
]);
|
||||
}
|
||||
|
||||
/** 按面积平均缩小到 maxWidth(等比);不放大 */
|
||||
export function downscale(img: RgbaImage, maxWidth: number): RgbaImage {
|
||||
if (img.width <= maxWidth) return img;
|
||||
const w = Math.max(1, Math.round(maxWidth));
|
||||
const h = Math.max(1, Math.round((img.height * w) / img.width));
|
||||
const sx = img.width / w;
|
||||
const sy = img.height / h;
|
||||
const out = new Uint8Array(w * h * 4);
|
||||
for (let y = 0; y < h; y++) {
|
||||
const y0 = Math.floor(y * sy);
|
||||
const y1 = Math.max(y0 + 1, Math.min(img.height, Math.floor((y + 1) * sy)));
|
||||
for (let x = 0; x < w; x++) {
|
||||
const x0 = Math.floor(x * sx);
|
||||
const x1 = Math.max(x0 + 1, Math.min(img.width, Math.floor((x + 1) * sx)));
|
||||
let r = 0;
|
||||
let g = 0;
|
||||
let b = 0;
|
||||
let a = 0;
|
||||
for (let yy = y0; yy < y1; yy++) {
|
||||
let i = (yy * img.width + x0) * 4;
|
||||
for (let xx = x0; xx < x1; xx++, i += 4) {
|
||||
r += img.data[i]!;
|
||||
g += img.data[i + 1]!;
|
||||
b += img.data[i + 2]!;
|
||||
a += img.data[i + 3]!;
|
||||
}
|
||||
}
|
||||
const n = (y1 - y0) * (x1 - x0);
|
||||
const o = (y * w + x) * 4;
|
||||
out[o] = r / n;
|
||||
out[o + 1] = g / n;
|
||||
out[o + 2] = b / n;
|
||||
out[o + 3] = a / n;
|
||||
}
|
||||
}
|
||||
return { width: w, height: h, data: out };
|
||||
}
|
||||
|
||||
/** 截图 → 交给模型 / 证据用的图片:缩放到 maxWidth 以内,按 format 编码 */
|
||||
export function prepareImage(data: Uint8Array, opts: PrepareImageOptions = {}): EncodedImage {
|
||||
const maxWidth = opts.maxWidth ?? 1280;
|
||||
const format = opts.format ?? "jpeg";
|
||||
const dim = imageDimensions(data);
|
||||
const isPng = data[0] === 0x89 && data[1] === 0x50;
|
||||
// 已经够小且格式一致:原样返回,不重新编码
|
||||
if (dim && dim.width <= maxWidth && ((format === "png" && isPng) || (format === "jpeg" && !isPng)))
|
||||
return { mimeType: format === "png" ? "image/png" : "image/jpeg", data, ...dim };
|
||||
const img = downscale(decodeImage(data), maxWidth);
|
||||
if (format === "png") return { mimeType: "image/png", data: encodePng(img), width: img.width, height: img.height };
|
||||
const quality = Math.min(100, Math.max(1, Math.round(opts.quality ?? 70)));
|
||||
const jpeg = encodeJpeg({ width: img.width, height: img.height, data: img.data }, quality);
|
||||
return { mimeType: "image/jpeg", data: new Uint8Array(jpeg.data), width: img.width, height: img.height };
|
||||
}
|
||||
|
||||
export function toBase64(data: Uint8Array): string {
|
||||
let s = "";
|
||||
for (let i = 0; i < data.length; i += 0x8000) s += String.fromCharCode(...data.subarray(i, i + 0x8000));
|
||||
return btoa(s);
|
||||
}
|
||||
|
||||
function concat(parts: Uint8Array[]): Uint8Array {
|
||||
const out = new Uint8Array(parts.reduce((n, p) => n + p.length, 0));
|
||||
let o = 0;
|
||||
for (const p of parts) {
|
||||
out.set(p, o);
|
||||
o += p.length;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const CRC_TABLE = (() => {
|
||||
const t = new Uint32Array(256);
|
||||
for (let n = 0; n < 256; n++) {
|
||||
let c = n;
|
||||
for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
|
||||
t[n] = c >>> 0;
|
||||
}
|
||||
return t;
|
||||
})();
|
||||
|
||||
function crc32(buf: Uint8Array): number {
|
||||
let c = 0xffffffff;
|
||||
for (const b of buf) c = CRC_TABLE[(c ^ b) & 0xff]! ^ (c >>> 8);
|
||||
return (c ^ 0xffffffff) >>> 0;
|
||||
}
|
||||
|
||||
function chunk(type: string, data: Uint8Array): Uint8Array {
|
||||
const out = new Uint8Array(12 + data.length);
|
||||
const v = new DataView(out.buffer);
|
||||
v.setUint32(0, data.length);
|
||||
for (let i = 0; i < 4; i++) out[4 + i] = type.charCodeAt(i);
|
||||
out.set(data, 8);
|
||||
v.setUint32(8 + data.length, crc32(out.subarray(4, 8 + data.length)));
|
||||
return out;
|
||||
}
|
||||
Reference in new issue
Block a user