rootandClaude Opus 5.5
c437c6bff6
- browser.screenshot returns the page as an image content block (≤1280px wide, JPEG quality configurable) for vision-capable upstreams; non-vision upstreams get a redacted text snapshot. Login pages / password fields are marked sensitive and withheld unless a browser.sensitive_screenshot grant exists and allowSensitive is passed. - browser.click_at / type_at / scroll / wait act in screenshot pixels; the server converts via viewport/DPR, requires automation control and an allowlisted host, and writes a thumbnail evidence file (DATA_DIR/tmp/evidence) referenced from agent_run_event. - browser.solve_challenge runs a bounded child agent_run (vision caps only, maxSteps ≤ 12, budget), judges success from page facts, stops immediately on 2FA / SMS / device confirmation, and otherwise moves the session to waiting with evidence and a takeover link. - PiAgentRuntime passes modelImage outputs as pi-ai image content when model.input has image; the ledger never stores image data; coordinate actions use per-invocation idempotency keys. - Tests: image codec, runtime image content, E14 e2e (login solved, 2FA stop, step limit, sensitive withheld). Real camofox 1.17.0 check on example.com recorded in the runbook. Co-Authored-By: Claude Opus 5.5 <[email protected]>
58 lines
2.9 KiB
TypeScript
58 lines
2.9 KiB
TypeScript
import { MOCK_ROW_H, MOCK_ROW_Y0, solidPngBase64 } from "@umt/testkit";
|
|
import { decode as decodeJpeg } from "jpeg-js";
|
|
import { describe, expect, it } from "vitest";
|
|
import { decodeImage, decodePng, downscale, encodePng, imageDimensions, prepareImage, toBase64 } from "./image.ts";
|
|
|
|
// VIS-1:截图解码 / 缩放 / 编码(交给模型的截图 ≤1280 宽,证据缩略图更小)。
|
|
|
|
const b64 = (s: string) => Uint8Array.from(atob(s), (c) => c.charCodeAt(0));
|
|
|
|
describe("截图图片处理", () => {
|
|
it("PNG 解码(RGB,mock 截图)与尺寸读取", () => {
|
|
const png = b64(solidPngBase64(200, 120, [10, 20, 30], [0]));
|
|
expect(imageDimensions(png)).toEqual({ width: 200, height: 120 });
|
|
const img = decodePng(png);
|
|
expect(img.width).toBe(200);
|
|
expect(Array.from(img.data.subarray(0, 4))).toEqual([10, 20, 30, 255]);
|
|
// 第 0 行元素横条:y ∈ [MOCK_ROW_Y0+8, MOCK_ROW_Y0+MOCK_ROW_H-8),x ∈ [40, 640)
|
|
const y = MOCK_ROW_Y0 + Math.floor(MOCK_ROW_H / 2);
|
|
const o = (y * 200 + 50) * 4;
|
|
expect(Array.from(img.data.subarray(o, o + 3))).toEqual([90, 110, 140]);
|
|
});
|
|
|
|
it("RGBA 编码 → 解码往返;过滤器 1–4 的行由 zlib 往返覆盖", () => {
|
|
const w = 37;
|
|
const h = 11;
|
|
const data = new Uint8Array(w * h * 4);
|
|
for (let i = 0; i < data.length; i++) data[i] = (i * 7 + (i >> 5)) & 0xff;
|
|
const back = decodeImage(encodePng({ width: w, height: h, data }));
|
|
expect(back.width).toBe(w);
|
|
expect(back.height).toBe(h);
|
|
expect(Buffer.from(back.data).equals(Buffer.from(data))).toBe(true);
|
|
});
|
|
|
|
it("缩放:1600 宽 → 1280 宽等比,面积平均;窄图不放大", () => {
|
|
const big = decodePng(b64(solidPngBase64(1600, 1000, [200, 100, 50])));
|
|
const small = downscale(big, 1280);
|
|
expect([small.width, small.height]).toEqual([1280, 800]);
|
|
expect(Array.from(small.data.subarray(0, 4))).toEqual([200, 100, 50, 255]);
|
|
expect(downscale(small, 2000)).toBe(small);
|
|
});
|
|
|
|
it("prepareImage:缩到 maxWidth 并编码 JPEG(质量可配);已够小的同格式原样返回", () => {
|
|
const png = b64(solidPngBase64(1600, 1000, [240, 240, 240], [0, 1, 2]));
|
|
const jpg = prepareImage(png, { maxWidth: 1280, format: "jpeg", quality: 60 });
|
|
expect(jpg).toMatchObject({ mimeType: "image/jpeg", width: 1280, height: 800 });
|
|
const dec = decodeJpeg(jpg.data, { useTArray: true });
|
|
expect([dec.width, dec.height]).toEqual([1280, 800]);
|
|
expect(imageDimensions(jpg.data)).toEqual({ width: 1280, height: 800 });
|
|
const hi = prepareImage(png, { maxWidth: 1280, format: "jpeg", quality: 95 });
|
|
expect(hi.data.length).toBeGreaterThan(jpg.data.length);
|
|
const tiny = b64(solidPngBase64(300, 200, [1, 2, 3]));
|
|
expect(prepareImage(tiny, { format: "png" }).data).toBe(tiny);
|
|
const thumb = prepareImage(png, { maxWidth: 480 });
|
|
expect(thumb.width).toBe(480);
|
|
expect(toBase64(tiny)).toBe(solidPngBase64(300, 200, [1, 2, 3]));
|
|
});
|
|
});
|