feat(browser): vision automation loop with screenshots, coordinate actions and bounded challenge solving
- browser.screenshot returns the page as an image content block (≤1280px wide, JPEG quality configurable) for vision-capable upstreams; non-vision upstreams get a redacted text snapshot. Login pages / password fields are marked sensitive and withheld unless a browser.sensitive_screenshot grant exists and allowSensitive is passed. - browser.click_at / type_at / scroll / wait act in screenshot pixels; the server converts via viewport/DPR, requires automation control and an allowlisted host, and writes a thumbnail evidence file (DATA_DIR/tmp/evidence) referenced from agent_run_event. - browser.solve_challenge runs a bounded child agent_run (vision caps only, maxSteps ≤ 12, budget), judges success from page facts, stops immediately on 2FA / SMS / device confirmation, and otherwise moves the session to waiting with evidence and a takeover link. - PiAgentRuntime passes modelImage outputs as pi-ai image content when model.input has image; the ledger never stores image data; coordinate actions use per-invocation idempotency keys. - Tests: image codec, runtime image content, E14 e2e (login solved, 2FA stop, step limit, sensitive withheld). Real camofox 1.17.0 check on example.com recorded in the runbook. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
0765264a47
commit
c437c6bff6
34 files changed
+2194
-35
No files matched your search
@@ -40,6 +40,7 @@ import {
|
||||
type NavigationResult,
|
||||
type OpenSessionRequest,
|
||||
type OpenSessionResult,
|
||||
type PageMetrics,
|
||||
profileKeyFor,
|
||||
type RestoreSessionState,
|
||||
type SnapshotOptions,
|
||||
@@ -132,6 +133,11 @@ const TEXTBOX_WITH_VALUE = /-\s*(?:textbox|searchbox)\b[^\n]*\[e\d+\][^\n]*?:\s*
|
||||
/** 兜底:遮蔽全部输入框的值(只替换冒号后的值部分) */
|
||||
const TEXTBOX_VALUE_ANY = /(?<=-\s*(?:textbox|searchbox)\b[^\n]*\[e\d+\][^\n]*?:\s*)\S[^\n]*/g;
|
||||
|
||||
/** 页面度量表达式(mock 按 umt-metrics 标记识别) */
|
||||
const PAGE_METRICS_EXPR = `/* umt-metrics */ (() => ({ w: innerWidth, h: innerHeight, dpr: devicePixelRatio,
|
||||
pw: !!document.querySelector("input[type=password]"),
|
||||
text: String((document.body && document.body.innerText) || "").slice(0, 4000) }))()`;
|
||||
|
||||
const LOGIN_PATH = /(?:^|\/)(?:login|takelogin|signin|sign_in|auth)(?:\.php)?(?:\/|$|\?)/i;
|
||||
|
||||
export class CamofoxBrowserProvider implements BrowserProvider {
|
||||
@@ -743,6 +749,36 @@ export class CamofoxBrowserProvider implements BrowserProvider {
|
||||
});
|
||||
}
|
||||
|
||||
/** 视口 / DPR / 密码框 / 文字样本:一次 evaluate;失败(旧版本、页面禁止脚本)返回 null */
|
||||
async pageMetrics(ref: TabRef): Promise<PageMetrics | null> {
|
||||
const s = this.state(ref.sessionId);
|
||||
return this.withProfile(s, async () => {
|
||||
const tab = this.tabOf(s, ref);
|
||||
const path = `/tabs/${encodeURIComponent(tab.upstreamTabId)}/evaluate`;
|
||||
try {
|
||||
const res = await this.request("POST", path, {
|
||||
body: { userId: s.session.profileKey, expression: PAGE_METRICS_EXPR },
|
||||
timeoutMs: s.budget.timeoutMs,
|
||||
});
|
||||
const out = await this.json(res, path);
|
||||
const r = (out.result && typeof out.result === "object" ? out.result : null) as Json | null;
|
||||
const w = typeof r?.w === "number" ? r.w : NaN;
|
||||
const h = typeof r?.h === "number" ? r.h : NaN;
|
||||
if (!(w > 0) || !(h > 0)) return null;
|
||||
return {
|
||||
viewportWidth: w,
|
||||
viewportHeight: h,
|
||||
devicePixelRatio: typeof r?.dpr === "number" && r.dpr > 0 ? r.dpr : 1,
|
||||
passwordField: r?.pw === true,
|
||||
textSample: typeof r?.text === "string" ? r.text.slice(0, 4000) : "",
|
||||
};
|
||||
} catch (e) {
|
||||
if (e instanceof BrowserError && e.kind === "not_found") throw e;
|
||||
return null;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// ---------- 交互 ----------
|
||||
|
||||
/** 交互动作的公共路径:控制者校验 → 串行锁 → POST /tabs/{id}/<action> */
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
import { MOCK_ROW_H, MOCK_ROW_Y0, solidPngBase64 } from "@umt/testkit";
|
||||
import { decode as decodeJpeg } from "jpeg-js";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { decodeImage, decodePng, downscale, encodePng, imageDimensions, prepareImage, toBase64 } from "./image.ts";
|
||||
|
||||
// VIS-1:截图解码 / 缩放 / 编码(交给模型的截图 ≤1280 宽,证据缩略图更小)。
|
||||
|
||||
const b64 = (s: string) => Uint8Array.from(atob(s), (c) => c.charCodeAt(0));
|
||||
|
||||
describe("截图图片处理", () => {
|
||||
it("PNG 解码(RGB,mock 截图)与尺寸读取", () => {
|
||||
const png = b64(solidPngBase64(200, 120, [10, 20, 30], [0]));
|
||||
expect(imageDimensions(png)).toEqual({ width: 200, height: 120 });
|
||||
const img = decodePng(png);
|
||||
expect(img.width).toBe(200);
|
||||
expect(Array.from(img.data.subarray(0, 4))).toEqual([10, 20, 30, 255]);
|
||||
// 第 0 行元素横条:y ∈ [MOCK_ROW_Y0+8, MOCK_ROW_Y0+MOCK_ROW_H-8),x ∈ [40, 640)
|
||||
const y = MOCK_ROW_Y0 + Math.floor(MOCK_ROW_H / 2);
|
||||
const o = (y * 200 + 50) * 4;
|
||||
expect(Array.from(img.data.subarray(o, o + 3))).toEqual([90, 110, 140]);
|
||||
});
|
||||
|
||||
it("RGBA 编码 → 解码往返;过滤器 1–4 的行由 zlib 往返覆盖", () => {
|
||||
const w = 37;
|
||||
const h = 11;
|
||||
const data = new Uint8Array(w * h * 4);
|
||||
for (let i = 0; i < data.length; i++) data[i] = (i * 7 + (i >> 5)) & 0xff;
|
||||
const back = decodeImage(encodePng({ width: w, height: h, data }));
|
||||
expect(back.width).toBe(w);
|
||||
expect(back.height).toBe(h);
|
||||
expect(Buffer.from(back.data).equals(Buffer.from(data))).toBe(true);
|
||||
});
|
||||
|
||||
it("缩放:1600 宽 → 1280 宽等比,面积平均;窄图不放大", () => {
|
||||
const big = decodePng(b64(solidPngBase64(1600, 1000, [200, 100, 50])));
|
||||
const small = downscale(big, 1280);
|
||||
expect([small.width, small.height]).toEqual([1280, 800]);
|
||||
expect(Array.from(small.data.subarray(0, 4))).toEqual([200, 100, 50, 255]);
|
||||
expect(downscale(small, 2000)).toBe(small);
|
||||
});
|
||||
|
||||
it("prepareImage:缩到 maxWidth 并编码 JPEG(质量可配);已够小的同格式原样返回", () => {
|
||||
const png = b64(solidPngBase64(1600, 1000, [240, 240, 240], [0, 1, 2]));
|
||||
const jpg = prepareImage(png, { maxWidth: 1280, format: "jpeg", quality: 60 });
|
||||
expect(jpg).toMatchObject({ mimeType: "image/jpeg", width: 1280, height: 800 });
|
||||
const dec = decodeJpeg(jpg.data, { useTArray: true });
|
||||
expect([dec.width, dec.height]).toEqual([1280, 800]);
|
||||
expect(imageDimensions(jpg.data)).toEqual({ width: 1280, height: 800 });
|
||||
const hi = prepareImage(png, { maxWidth: 1280, format: "jpeg", quality: 95 });
|
||||
expect(hi.data.length).toBeGreaterThan(jpg.data.length);
|
||||
const tiny = b64(solidPngBase64(300, 200, [1, 2, 3]));
|
||||
expect(prepareImage(tiny, { format: "png" }).data).toBe(tiny);
|
||||
const thumb = prepareImage(png, { maxWidth: 480 });
|
||||
expect(thumb.width).toBe(480);
|
||||
expect(toBase64(tiny)).toBe(solidPngBase64(300, 200, [1, 2, 3]));
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,275 @@
|
||||
/**
|
||||
* 截图的解码、缩放与编码(VIS-1):交给视觉模型的截图缩放到不超过 maxWidth(缺省 1280)像素宽,
|
||||
* 证据缩略图更小。PNG 自己解码 / 编码(node:zlib),JPEG 用 jpeg-js(纯 JS,无原生依赖,deno compile 可嵌入)。
|
||||
* 只支持浏览器截图实际会出现的格式:8 位 PNG(灰度 / RGB / 调色板 / 带 alpha,非隔行)与基线 JPEG。
|
||||
*/
|
||||
import { deflateSync, inflateSync } from "node:zlib";
|
||||
import { decode as decodeJpeg, encode as encodeJpeg } from "jpeg-js";
|
||||
|
||||
export interface RgbaImage {
|
||||
width: number;
|
||||
height: number;
|
||||
/** RGBA,每像素 4 字节 */
|
||||
data: Uint8Array;
|
||||
}
|
||||
|
||||
export interface EncodedImage {
|
||||
mimeType: "image/png" | "image/jpeg";
|
||||
data: Uint8Array;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export interface PrepareImageOptions {
|
||||
/** 最大宽度(像素);原图更窄时不放大 */
|
||||
maxWidth?: number;
|
||||
/** 输出格式;缺省 jpeg */
|
||||
format?: "png" | "jpeg";
|
||||
/** JPEG 质量 1–100,缺省 70 */
|
||||
quality?: number;
|
||||
}
|
||||
|
||||
/** PNG IHDR / JPEG SOF 里的像素尺寸;不认识的格式返回 null */
|
||||
export function imageDimensions(data: Uint8Array): { width: number; height: number } | null {
|
||||
if (data.length > 24 && data[0] === 0x89 && data[1] === 0x50 && data[2] === 0x4e && data[3] === 0x47) {
|
||||
const v = new DataView(data.buffer, data.byteOffset, data.byteLength);
|
||||
return { width: v.getUint32(16), height: v.getUint32(20) };
|
||||
}
|
||||
if (data[0] === 0xff && data[1] === 0xd8) {
|
||||
let i = 2;
|
||||
while (i + 9 < data.length) {
|
||||
if (data[i] !== 0xff) break;
|
||||
const marker = data[i + 1]!;
|
||||
const len = (data[i + 2]! << 8) | data[i + 3]!;
|
||||
if (marker >= 0xc0 && marker <= 0xc3)
|
||||
return { width: (data[i + 7]! << 8) | data[i + 8]!, height: (data[i + 5]! << 8) | data[i + 6]! };
|
||||
i += 2 + len;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function decodeImage(data: Uint8Array): RgbaImage {
|
||||
if (data[0] === 0x89 && data[1] === 0x50) return decodePng(data);
|
||||
if (data[0] === 0xff && data[1] === 0xd8) {
|
||||
const img = decodeJpeg(data, { useTArray: true, formatAsRGBA: true, maxMemoryUsageInMB: 256 });
|
||||
return { width: img.width, height: img.height, data: img.data };
|
||||
}
|
||||
throw new Error("不支持的图片格式");
|
||||
}
|
||||
|
||||
/** 解码 8 位、非隔行 PNG(颜色类型 0/2/3/4/6) */
|
||||
export function decodePng(bytes: Uint8Array): RgbaImage {
|
||||
const v = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
||||
let pos = 8;
|
||||
let width = 0;
|
||||
let height = 0;
|
||||
let depth = 0;
|
||||
let color = 0;
|
||||
let interlace = 0;
|
||||
let palette: Uint8Array | null = null;
|
||||
let trns: Uint8Array | null = null;
|
||||
const idat: Uint8Array[] = [];
|
||||
while (pos + 8 <= bytes.length) {
|
||||
const len = v.getUint32(pos);
|
||||
const type = String.fromCharCode(bytes[pos + 4]!, bytes[pos + 5]!, bytes[pos + 6]!, bytes[pos + 7]!);
|
||||
const body = bytes.subarray(pos + 8, pos + 8 + len);
|
||||
if (type === "IHDR") {
|
||||
width = v.getUint32(pos + 8);
|
||||
height = v.getUint32(pos + 12);
|
||||
depth = bytes[pos + 16]!;
|
||||
color = bytes[pos + 17]!;
|
||||
interlace = bytes[pos + 20]!;
|
||||
} else if (type === "PLTE") palette = body;
|
||||
else if (type === "tRNS") trns = body;
|
||||
else if (type === "IDAT") idat.push(body);
|
||||
else if (type === "IEND") break;
|
||||
pos += 12 + len;
|
||||
}
|
||||
if (depth !== 8 || interlace !== 0) throw new Error(`不支持的 PNG(位深 ${depth},隔行 ${interlace})`);
|
||||
const channels = ({ 0: 1, 2: 3, 3: 1, 4: 2, 6: 4 } as Record<number, number>)[color];
|
||||
if (!channels) throw new Error(`不支持的 PNG 颜色类型 ${color}`);
|
||||
if (width <= 0 || height <= 0 || width * height > 40_000_000) throw new Error("PNG 尺寸异常");
|
||||
const raw = new Uint8Array(inflateSync(concat(idat)));
|
||||
const stride = width * channels;
|
||||
const out = new Uint8Array(width * height * 4);
|
||||
let prev = new Uint8Array(stride);
|
||||
let cur = new Uint8Array(stride);
|
||||
for (let y = 0; y < height; y++) {
|
||||
const off = y * (stride + 1);
|
||||
const filter = raw[off]!;
|
||||
for (let i = 0; i < stride; i++) {
|
||||
const x = raw[off + 1 + i]!;
|
||||
const a = i >= channels ? cur[i - channels]! : 0;
|
||||
const b = prev[i]!;
|
||||
const c = i >= channels ? prev[i - channels]! : 0;
|
||||
let val: number;
|
||||
switch (filter) {
|
||||
case 0:
|
||||
val = x;
|
||||
break;
|
||||
case 1:
|
||||
val = x + a;
|
||||
break;
|
||||
case 2:
|
||||
val = x + b;
|
||||
break;
|
||||
case 3:
|
||||
val = x + ((a + b) >> 1);
|
||||
break;
|
||||
case 4: {
|
||||
const p = a + b - c;
|
||||
const pa = Math.abs(p - a);
|
||||
const pb = Math.abs(p - b);
|
||||
const pc = Math.abs(p - c);
|
||||
val = x + (pa <= pb && pa <= pc ? a : pb <= pc ? b : c);
|
||||
break;
|
||||
}
|
||||
default:
|
||||
throw new Error(`PNG 过滤类型 ${filter} 非法`);
|
||||
}
|
||||
cur[i] = val & 0xff;
|
||||
}
|
||||
for (let px = 0; px < width; px++) {
|
||||
const o = (y * width + px) * 4;
|
||||
const s = px * channels;
|
||||
if (color === 6) {
|
||||
out[o] = cur[s]!;
|
||||
out[o + 1] = cur[s + 1]!;
|
||||
out[o + 2] = cur[s + 2]!;
|
||||
out[o + 3] = cur[s + 3]!;
|
||||
} else if (color === 2) {
|
||||
out[o] = cur[s]!;
|
||||
out[o + 1] = cur[s + 1]!;
|
||||
out[o + 2] = cur[s + 2]!;
|
||||
out[o + 3] = 255;
|
||||
} else if (color === 0 || color === 4) {
|
||||
out[o] = out[o + 1] = out[o + 2] = cur[s]!;
|
||||
out[o + 3] = color === 4 ? cur[s + 1]! : 255;
|
||||
} else {
|
||||
const idx = cur[s]!;
|
||||
out[o] = palette?.[idx * 3] ?? 0;
|
||||
out[o + 1] = palette?.[idx * 3 + 1] ?? 0;
|
||||
out[o + 2] = palette?.[idx * 3 + 2] ?? 0;
|
||||
out[o + 3] = trns?.[idx] ?? 255;
|
||||
}
|
||||
}
|
||||
[prev, cur] = [cur, prev];
|
||||
}
|
||||
return { width, height, data: out };
|
||||
}
|
||||
|
||||
/** 编码为 8 位 RGBA PNG(每行 filter 0,deflate 压缩) */
|
||||
export function encodePng(img: RgbaImage): Uint8Array {
|
||||
const stride = img.width * 4;
|
||||
const raw = new Uint8Array((stride + 1) * img.height);
|
||||
for (let y = 0; y < img.height; y++) raw.set(img.data.subarray(y * stride, (y + 1) * stride), y * (stride + 1) + 1);
|
||||
const ihdr = new Uint8Array(13);
|
||||
const hv = new DataView(ihdr.buffer);
|
||||
hv.setUint32(0, img.width);
|
||||
hv.setUint32(4, img.height);
|
||||
ihdr[8] = 8;
|
||||
ihdr[9] = 6;
|
||||
return concat([
|
||||
new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]),
|
||||
chunk("IHDR", ihdr),
|
||||
chunk("IDAT", new Uint8Array(deflateSync(raw))),
|
||||
chunk("IEND", new Uint8Array()),
|
||||
]);
|
||||
}
|
||||
|
||||
/** 按面积平均缩小到 maxWidth(等比);不放大 */
|
||||
export function downscale(img: RgbaImage, maxWidth: number): RgbaImage {
|
||||
if (img.width <= maxWidth) return img;
|
||||
const w = Math.max(1, Math.round(maxWidth));
|
||||
const h = Math.max(1, Math.round((img.height * w) / img.width));
|
||||
const sx = img.width / w;
|
||||
const sy = img.height / h;
|
||||
const out = new Uint8Array(w * h * 4);
|
||||
for (let y = 0; y < h; y++) {
|
||||
const y0 = Math.floor(y * sy);
|
||||
const y1 = Math.max(y0 + 1, Math.min(img.height, Math.floor((y + 1) * sy)));
|
||||
for (let x = 0; x < w; x++) {
|
||||
const x0 = Math.floor(x * sx);
|
||||
const x1 = Math.max(x0 + 1, Math.min(img.width, Math.floor((x + 1) * sx)));
|
||||
let r = 0;
|
||||
let g = 0;
|
||||
let b = 0;
|
||||
let a = 0;
|
||||
for (let yy = y0; yy < y1; yy++) {
|
||||
let i = (yy * img.width + x0) * 4;
|
||||
for (let xx = x0; xx < x1; xx++, i += 4) {
|
||||
r += img.data[i]!;
|
||||
g += img.data[i + 1]!;
|
||||
b += img.data[i + 2]!;
|
||||
a += img.data[i + 3]!;
|
||||
}
|
||||
}
|
||||
const n = (y1 - y0) * (x1 - x0);
|
||||
const o = (y * w + x) * 4;
|
||||
out[o] = r / n;
|
||||
out[o + 1] = g / n;
|
||||
out[o + 2] = b / n;
|
||||
out[o + 3] = a / n;
|
||||
}
|
||||
}
|
||||
return { width: w, height: h, data: out };
|
||||
}
|
||||
|
||||
/** 截图 → 交给模型 / 证据用的图片:缩放到 maxWidth 以内,按 format 编码 */
|
||||
export function prepareImage(data: Uint8Array, opts: PrepareImageOptions = {}): EncodedImage {
|
||||
const maxWidth = opts.maxWidth ?? 1280;
|
||||
const format = opts.format ?? "jpeg";
|
||||
const dim = imageDimensions(data);
|
||||
const isPng = data[0] === 0x89 && data[1] === 0x50;
|
||||
// 已经够小且格式一致:原样返回,不重新编码
|
||||
if (dim && dim.width <= maxWidth && ((format === "png" && isPng) || (format === "jpeg" && !isPng)))
|
||||
return { mimeType: format === "png" ? "image/png" : "image/jpeg", data, ...dim };
|
||||
const img = downscale(decodeImage(data), maxWidth);
|
||||
if (format === "png") return { mimeType: "image/png", data: encodePng(img), width: img.width, height: img.height };
|
||||
const quality = Math.min(100, Math.max(1, Math.round(opts.quality ?? 70)));
|
||||
const jpeg = encodeJpeg({ width: img.width, height: img.height, data: img.data }, quality);
|
||||
return { mimeType: "image/jpeg", data: new Uint8Array(jpeg.data), width: img.width, height: img.height };
|
||||
}
|
||||
|
||||
export function toBase64(data: Uint8Array): string {
|
||||
let s = "";
|
||||
for (let i = 0; i < data.length; i += 0x8000) s += String.fromCharCode(...data.subarray(i, i + 0x8000));
|
||||
return btoa(s);
|
||||
}
|
||||
|
||||
function concat(parts: Uint8Array[]): Uint8Array {
|
||||
const out = new Uint8Array(parts.reduce((n, p) => n + p.length, 0));
|
||||
let o = 0;
|
||||
for (const p of parts) {
|
||||
out.set(p, o);
|
||||
o += p.length;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const CRC_TABLE = (() => {
|
||||
const t = new Uint32Array(256);
|
||||
for (let n = 0; n < 256; n++) {
|
||||
let c = n;
|
||||
for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
|
||||
t[n] = c >>> 0;
|
||||
}
|
||||
return t;
|
||||
})();
|
||||
|
||||
function crc32(buf: Uint8Array): number {
|
||||
let c = 0xffffffff;
|
||||
for (const b of buf) c = CRC_TABLE[(c ^ b) & 0xff]! ^ (c >>> 8);
|
||||
return (c ^ 0xffffffff) >>> 0;
|
||||
}
|
||||
|
||||
function chunk(type: string, data: Uint8Array): Uint8Array {
|
||||
const out = new Uint8Array(12 + data.length);
|
||||
const v = new DataView(out.buffer);
|
||||
v.setUint32(0, data.length);
|
||||
for (let i = 0; i < 4; i++) out[4 + i] = type.charCodeAt(i);
|
||||
out.set(data, 8);
|
||||
v.setUint32(8 + data.length, crc32(out.subarray(4, 8 + data.length)));
|
||||
return out;
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
export * from "./budget.ts";
|
||||
export * from "./camofox/index.ts";
|
||||
export * from "./errors.ts";
|
||||
export * from "./image.ts";
|
||||
export * from "./none.ts";
|
||||
export * from "./profile-lock.ts";
|
||||
export * from "./provider.ts";
|
||||
|
||||
@@ -57,5 +57,6 @@ export function createNoneBrowserProvider(): BrowserProvider {
|
||||
reconcileTabs: unavailable,
|
||||
sessionInfo: () => null,
|
||||
drain: async () => {},
|
||||
pageMetrics: async () => null,
|
||||
};
|
||||
}
|
||||
@@ -60,6 +60,21 @@ export interface BrowserProvider {
|
||||
sessionInfo(sessionId: string): BrowserSessionInfo | null;
|
||||
/** 等待进行中的操作收尾(接管前调用) */
|
||||
drain(sessionId: string): Promise<void>;
|
||||
/**
|
||||
* 页面度量(VIS-1 坐标换算与敏感判定):视口 CSS 尺寸、DPR、是否有密码框、页面文字样本(只在服务端判定用,不外传)。
|
||||
* 只读,不要求自动化控制权;上游不支持 evaluate 时返回 null。
|
||||
*/
|
||||
pageMetrics(ref: TabRef): Promise<PageMetrics | null>;
|
||||
}
|
||||
|
||||
export interface PageMetrics {
|
||||
viewportWidth: number;
|
||||
viewportHeight: number;
|
||||
devicePixelRatio: number;
|
||||
/** 页面上有 input[type=password] */
|
||||
passwordField: boolean;
|
||||
/** document.body.innerText 前 4000 字(判定二次验证等,不进模型、不落日志) */
|
||||
textSample: string;
|
||||
}
|
||||
|
||||
/** 动作发起方:自动化(Agent/任务)或持有接管租约的人 */
|
||||
|
||||
@@ -18,6 +18,11 @@ export interface InvokeContext extends CapabilityContext {
|
||||
/** agent 调用时的 tool call id / invocation id,用于审计对账 */
|
||||
invocationId?: string;
|
||||
agentRunId?: string;
|
||||
/**
|
||||
* 本次调用的结果会不会以图片交给模型(Agent 调用时 = 上游模型支持图片输入)。
|
||||
* false:能力不要产出图片(browser.screenshot 改给文字快照);undefined:非 Agent 调用方,按能力缺省。
|
||||
*/
|
||||
modelImages?: boolean;
|
||||
}
|
||||
|
||||
/** handler 实际收到的上下文:注册中心补上服务端生成的幂等键与授权结果 */
|
||||
|
||||
@@ -36,6 +36,7 @@ export {
|
||||
DEFAULT_COMPACTION_INSTRUCTIONS,
|
||||
PiAgentRuntime,
|
||||
type PiAgentRuntimeOptions,
|
||||
toolContent,
|
||||
} from "./pi/runtime.ts";
|
||||
export { sessionPathFor } from "./pi/session.ts";
|
||||
export {
|
||||
|
||||
@@ -25,6 +25,8 @@ export interface FakeToolResult {
|
||||
text: string;
|
||||
/** text 能解析成 JSON 时的值 */
|
||||
json?: unknown;
|
||||
/** 工具结果里的图片内容块数(视觉自动化:截图真的交给了模型) */
|
||||
images: number;
|
||||
}
|
||||
|
||||
export interface FakeToolCall {
|
||||
@@ -54,6 +56,8 @@ export interface FakeModelProviderOptions {
|
||||
maxTokens?: number;
|
||||
/** 压缩请求时返回的摘要文本 */
|
||||
summaryText?: string;
|
||||
/** 模拟支持图片输入的模型(model.input 含 image);缺省 false */
|
||||
vision?: boolean;
|
||||
}
|
||||
|
||||
const SUMMARY_MARKERS = ["context summarization assistant", "structured context checkpoint summary"];
|
||||
@@ -70,6 +74,8 @@ export class FakeModelProvider {
|
||||
calls = 0;
|
||||
/** 最近一次请求看到的系统提示与工具名(测试断言用) */
|
||||
lastRequest: { transcript: string } | undefined;
|
||||
/** 历次请求里收到的图片内容块(测试断言截图确实交给了模型) */
|
||||
images: { mimeType: string; bytes: number }[] = [];
|
||||
|
||||
constructor(scripts: FakeScript[], options: FakeModelProviderOptions = {}) {
|
||||
this.scripts = scripts;
|
||||
@@ -82,6 +88,7 @@ export class FakeModelProvider {
|
||||
contextWindow: options.contextWindow ?? 64_000,
|
||||
maxTokens: options.maxTokens ?? 4096,
|
||||
reasoning: true,
|
||||
input: options.vision ? ["text", "image"] : ["text"],
|
||||
},
|
||||
],
|
||||
tokensPerSecond: 0,
|
||||
@@ -105,6 +112,10 @@ export class FakeModelProvider {
|
||||
const messages = context.messages as Message[];
|
||||
const transcript = messages.map(messageText).join("\n");
|
||||
this.lastRequest = { transcript };
|
||||
const last = messages[messages.length - 1];
|
||||
if (last?.role === "toolResult")
|
||||
for (const c of last.content as Array<{ type: string; data?: string; mimeType?: string }>)
|
||||
if (c.type === "image") this.images.push({ mimeType: c.mimeType ?? "", bytes: (c.data ?? "").length });
|
||||
const head = messages[0] ? messageText(messages[0]) : "";
|
||||
if (SUMMARY_MARKERS.some((m) => head.includes(m) || lastUserText(messages).includes(m))) {
|
||||
return fauxAssistantMessage(this.summaryText);
|
||||
@@ -203,6 +214,7 @@ function toolResults(messages: Message[]): FakeToolResult[] {
|
||||
isError: m.isError,
|
||||
text,
|
||||
...(json !== undefined ? { json } : {}),
|
||||
images: (m.content as Array<{ type: string }>).filter((c) => c.type === "image").length,
|
||||
});
|
||||
}
|
||||
return out;
|
||||
|
||||
@@ -10,7 +10,13 @@ import {
|
||||
type RunResult,
|
||||
} from "@earendil-works/pi-agent-core";
|
||||
import type { Usage } from "@earendil-works/pi-ai";
|
||||
import type { AgentBudgetKind, AgentRunEvent, AgentRunEventBody, AgentUsageTotals } from "@umt/contracts";
|
||||
import {
|
||||
type AgentBudgetKind,
|
||||
type AgentRunEvent,
|
||||
type AgentRunEventBody,
|
||||
type AgentUsageTotals,
|
||||
MODEL_IMAGE_KEY,
|
||||
} from "@umt/contracts";
|
||||
import type { TSchema } from "typebox";
|
||||
import { isCapabilityError } from "../capability/errors.ts";
|
||||
import type { CapabilityRegistry, RegisteredCapability } from "../capability/registry.ts";
|
||||
@@ -81,6 +87,7 @@ export class PiAgentRuntime implements AgentRuntimePort {
|
||||
const opened = await openRunSession(spec.sessionDir, spec.runId, ctx);
|
||||
|
||||
const run = new PiAgentRun(spec, settings, capabilities, this.options, opened.path, opened.repo);
|
||||
run.supportsImages = Array.isArray(model.input) && model.input.includes("image");
|
||||
try {
|
||||
const { harness, open } = await AgentHarness.create(
|
||||
{
|
||||
@@ -115,6 +122,8 @@ class PiAgentRun implements AgentRunHandle {
|
||||
readonly runId: string;
|
||||
readonly sessionPath: string;
|
||||
pendingOperations = 0;
|
||||
/** 上游模型声明支持图片输入(model.input 含 image) */
|
||||
supportsImages = false;
|
||||
|
||||
private readonly channel = new EventChannel<AgentRunEvent>();
|
||||
private readonly allowed = new Map<string, string>();
|
||||
@@ -190,11 +199,9 @@ class PiAgentRun implements AgentRunHandle {
|
||||
source: "agent",
|
||||
invocationId,
|
||||
agentRunId: this.runId,
|
||||
modelImages: this.supportsImages,
|
||||
});
|
||||
return {
|
||||
content: [{ type: "text", text: JSON.stringify(redactForModel(output) ?? null) }],
|
||||
details: undefined,
|
||||
};
|
||||
return { content: toolContent(output, this.supportsImages), details: undefined };
|
||||
} catch (error) {
|
||||
if (isCapabilityError(error)) {
|
||||
throw new Error(JSON.stringify({ error: { code: error.code, message: redactString(error.message) } }));
|
||||
@@ -475,6 +482,36 @@ class PiAgentRun implements AgentRunHandle {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 能力输出 → 工具结果内容块。输出里的 modelImage(见 @umt/contracts MODEL_IMAGE_KEY)摘出来:
|
||||
* 上游支持图片 → image content;不支持 → 丢弃并在文本里提示。图片数据不进 JSON 文本、不经文本脱敏。
|
||||
*/
|
||||
export function toolContent(
|
||||
output: unknown,
|
||||
vision: boolean,
|
||||
): Array<{ type: "text"; text: string } | { type: "image"; data: string; mimeType: string }> {
|
||||
let rest = output;
|
||||
let image: { data: string; mimeType: string } | null = null;
|
||||
if (output && typeof output === "object" && !Array.isArray(output) && MODEL_IMAGE_KEY in output) {
|
||||
const { [MODEL_IMAGE_KEY]: img, ...others } = output as Record<string, unknown>;
|
||||
rest = others;
|
||||
const m = img as { data?: unknown; mimeType?: unknown } | null | undefined;
|
||||
if (m && typeof m.data === "string" && typeof m.mimeType === "string")
|
||||
image = { data: m.data, mimeType: m.mimeType };
|
||||
}
|
||||
const text = JSON.stringify(redactForModel(rest) ?? null);
|
||||
if (!image) return [{ type: "text", text }];
|
||||
if (!vision)
|
||||
return [
|
||||
{ type: "text", text },
|
||||
{ type: "text", text: "(截图未发送:当前上游模型不支持图片输入;用 browser.snapshot 读取页面文字)" },
|
||||
];
|
||||
return [
|
||||
{ type: "text", text },
|
||||
{ type: "image", data: image.data, mimeType: image.mimeType },
|
||||
];
|
||||
}
|
||||
|
||||
/** before_tool 白名单判定(纯函数,便于单测) */
|
||||
export function checkToolCall(
|
||||
toolName: string,
|
||||
|
||||
@@ -95,6 +95,8 @@ export interface AgentRunHandle {
|
||||
readonly sessionPath: string;
|
||||
/** 打开已有会话时发现的未完成操作数;>0 时应调用 resume() */
|
||||
readonly pendingOperations: number;
|
||||
/** 上游模型支持图片输入(视觉自动化需要);fake 模型按创建参数 */
|
||||
readonly supportsImages?: boolean;
|
||||
/** 从 fromSeq(含)开始回放并持续推送事件;close() 后结束 */
|
||||
events(fromSeq?: number): AsyncIterable<AgentRunEvent>;
|
||||
/** 本进程已产生的全部事件快照 */
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { z } from "zod";
|
||||
import {
|
||||
type AgentRunHandle,
|
||||
CapabilityRegistry,
|
||||
createFakeRuntime,
|
||||
type FakeScript,
|
||||
toolContent,
|
||||
} from "../src/index.ts";
|
||||
import { settings, tempDir } from "./helpers.ts";
|
||||
|
||||
// VIS-1:能力输出里的 modelImage 以 image content 交给支持视觉的模型;不支持时降级为文字提示,图片数据不进 JSON 文本。
|
||||
|
||||
const PNG_1X1 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==";
|
||||
|
||||
const opened: AgentRunHandle[] = [];
|
||||
afterEach(async () => {
|
||||
await Promise.all(opened.splice(0).map((r) => r.close()));
|
||||
});
|
||||
|
||||
function registry(seen: Array<boolean | undefined>) {
|
||||
const r = new CapabilityRegistry();
|
||||
r.register(
|
||||
{
|
||||
id: "browser.screenshot",
|
||||
version: 1,
|
||||
title: "截图",
|
||||
description: "截取页面",
|
||||
inputSchema: z.object({ sessionId: z.string() }),
|
||||
outputSchema: z.object({
|
||||
url: z.string(),
|
||||
modelImage: z
|
||||
.object({ mimeType: z.string(), data: z.string(), width: z.number(), height: z.number() })
|
||||
.nullable(),
|
||||
}),
|
||||
kind: "read",
|
||||
scope: "read",
|
||||
riskLevel: "low",
|
||||
idempotent: true,
|
||||
supportsPreview: false,
|
||||
replay: "safe",
|
||||
},
|
||||
(_input, ctx) => {
|
||||
seen.push(ctx.modelImages);
|
||||
return {
|
||||
url: "https://site.example/login",
|
||||
modelImage: { mimeType: "image/png", data: PNG_1X1, width: 1, height: 1 },
|
||||
};
|
||||
},
|
||||
);
|
||||
return r;
|
||||
}
|
||||
|
||||
const script: FakeScript = {
|
||||
match: /看页面/,
|
||||
steps: [{ toolCalls: [{ capability: "browser.screenshot", input: { sessionId: "s1" } }] }],
|
||||
finalText: (r) => `images=${r[0]?.images};text=${r[0]?.text}`,
|
||||
};
|
||||
|
||||
async function run(vision: boolean) {
|
||||
const seen: Array<boolean | undefined> = [];
|
||||
const { runtime, fake } = createFakeRuntime({ registry: registry(seen), scripts: [script], fake: { vision } });
|
||||
const handle = await runtime.createRun({
|
||||
runId: `vision-${vision}`,
|
||||
goal: "看页面",
|
||||
systemPrompt: "测试",
|
||||
capabilities: ["browser.screenshot"],
|
||||
settings: settings(),
|
||||
sessionDir: await tempDir(),
|
||||
actor: { actorId: "user:1", taskId: "task:1", scopeIds: [] },
|
||||
});
|
||||
opened.push(handle);
|
||||
const outcome = await handle.prompt();
|
||||
return { handle, fake, outcome, seen };
|
||||
}
|
||||
|
||||
describe("VIS-1 图片内容块", () => {
|
||||
it("支持视觉的模型:截图作为 image content 交给模型,JSON 文本里没有图片数据", async () => {
|
||||
const { handle, fake, outcome, seen } = await run(true);
|
||||
expect(handle.supportsImages).toBe(true);
|
||||
expect(seen).toEqual([true]);
|
||||
expect(fake.images).toEqual([{ mimeType: "image/png", bytes: PNG_1X1.length }]);
|
||||
expect(outcome.finalText).toContain("images=1");
|
||||
expect(outcome.finalText).not.toContain(PNG_1X1);
|
||||
});
|
||||
|
||||
it("不支持视觉的模型:能力收到 modelImages=false,图片不发送并提示", async () => {
|
||||
const { handle, fake, outcome, seen } = await run(false);
|
||||
expect(handle.supportsImages).toBe(false);
|
||||
expect(seen).toEqual([false]);
|
||||
expect(fake.images).toEqual([]);
|
||||
expect(outcome.finalText).toContain("images=0");
|
||||
expect(outcome.finalText).not.toContain(PNG_1X1);
|
||||
});
|
||||
|
||||
it("toolContent:摘出 modelImage;null 时只有文本", () => {
|
||||
const img = toolContent({ a: 1, modelImage: { mimeType: "image/jpeg", data: "QUJD" } }, true);
|
||||
expect(img).toEqual([
|
||||
{ type: "text", text: '{"a":1}' },
|
||||
{ type: "image", data: "QUJD", mimeType: "image/jpeg" },
|
||||
]);
|
||||
const none = toolContent({ a: 1, modelImage: null }, true);
|
||||
expect(none).toEqual([{ type: "text", text: '{"a":1}' }]);
|
||||
const text = toolContent({ a: 1, modelImage: { mimeType: "image/jpeg", data: "QUJD" } }, false);
|
||||
expect(text).toHaveLength(2);
|
||||
expect(text[1]).toMatchObject({ type: "text" });
|
||||
expect(JSON.stringify(text)).not.toContain("QUJD");
|
||||
});
|
||||
});
|
||||
@@ -445,6 +445,164 @@ export const BrowserDownloadOutput = z.object({
|
||||
note: z.string().nullable(),
|
||||
});
|
||||
|
||||
// ── VIS-1 视觉自动化闭环 ──
|
||||
// 截图以图片内容块交给模型:能力输出里的 modelImage 由 Agent 运行时摘出,按上游是否支持图片输入转成
|
||||
// image content(不支持时降级为文字快照并提示);JSON 文本里不再带图片数据。
|
||||
|
||||
/** 能力输出里交给模型的图片(运行时摘出为 image content,不进 JSON 文本、不进操作账本) */
|
||||
export const ModelImage = z.object({
|
||||
mimeType: z.enum(["image/png", "image/jpeg"]),
|
||||
/** base64 */
|
||||
data: z.string(),
|
||||
width: z.number().int().positive(),
|
||||
height: z.number().int().positive(),
|
||||
});
|
||||
export type ModelImage = z.infer<typeof ModelImage>;
|
||||
/** 运行时识别图片的键名 */
|
||||
export const MODEL_IMAGE_KEY = "modelImage" as const;
|
||||
|
||||
/** 每步证据:截图缩略图存 DATA_DIR/tmp/evidence,这里只给引用 */
|
||||
export const BrowserEvidenceRef = z.object({
|
||||
id: Id,
|
||||
/** 相对 DATA_DIR/tmp 的路径,如 evidence/<id>.jpg */
|
||||
ref: z.string(),
|
||||
capturedAt: z.string(),
|
||||
});
|
||||
export type BrowserEvidenceRef = z.infer<typeof BrowserEvidenceRef>;
|
||||
|
||||
/** 截图像素 ↔ 页面 CSS 像素的换算依据 */
|
||||
export const BrowserCoordinateSpace = z.object({
|
||||
/** 交给模型的截图尺寸(坐标以它为准) */
|
||||
imageWidth: z.number().int().positive(),
|
||||
imageHeight: z.number().int().positive(),
|
||||
/** 浏览器原始截图尺寸 */
|
||||
sourceWidth: z.number().int().positive(),
|
||||
sourceHeight: z.number().int().positive(),
|
||||
/** 页面视口(CSS 像素)与设备像素比;取不到时按原始截图尺寸、DPR 1 推定 */
|
||||
viewportWidth: z.number().positive(),
|
||||
viewportHeight: z.number().positive(),
|
||||
devicePixelRatio: z.number().positive(),
|
||||
assumed: z.boolean(),
|
||||
});
|
||||
export type BrowserCoordinateSpace = z.infer<typeof BrowserCoordinateSpace>;
|
||||
|
||||
const BrowserShotFields = {
|
||||
evidence: BrowserEvidenceRef.nullable(),
|
||||
/** 已脱敏的页面地址 */
|
||||
url: z.string().nullable(),
|
||||
controller: BrowserController,
|
||||
/** 登录页 / 含密码框 / 地址带凭据:截图敏感,默认不交给模型 */
|
||||
sensitive: z.boolean(),
|
||||
/** 本次截图是否作为图片交给了模型 */
|
||||
imageForModel: z.boolean(),
|
||||
/** 没交给模型时的原因与替代:上游不支持图片输入时附文字快照 */
|
||||
note: z.string().nullable(),
|
||||
textSnapshot: z.string().nullable(),
|
||||
coordinates: BrowserCoordinateSpace.nullable(),
|
||||
/** 会话转为等待处理的原因(二次验证等);非 null 时自动化动作已停止 */
|
||||
waitingReason: z.string().nullable(),
|
||||
modelImage: ModelImage.nullable(),
|
||||
};
|
||||
|
||||
export const BrowserScreenshotInput = z.object({
|
||||
sessionId: Id,
|
||||
tabId: z.string().min(1).optional(),
|
||||
/** 敏感截图也交给模型:需要 browser.sensitive_screenshot 持续授权,否则忽略 */
|
||||
allowSensitive: z.boolean().optional(),
|
||||
});
|
||||
export const BrowserScreenshotOutput = z.object(BrowserShotFields);
|
||||
|
||||
const Coord = z.number().min(0).max(10_000);
|
||||
|
||||
export const BrowserClickAtInput = z.object({
|
||||
sessionId: Id,
|
||||
x: Coord,
|
||||
y: Coord,
|
||||
/** camofox 1.17.0 只支持左键 */
|
||||
button: z.enum(["left", "right", "middle"]).optional(),
|
||||
allowSensitive: z.boolean().optional(),
|
||||
});
|
||||
export const BrowserTypeAtInput = z.object({
|
||||
sessionId: Id,
|
||||
/** 不给坐标 = 输入到当前焦点 */
|
||||
x: Coord.optional(),
|
||||
y: Coord.optional(),
|
||||
text: z.string().min(1).max(500),
|
||||
pressEnter: z.boolean().optional(),
|
||||
allowSensitive: z.boolean().optional(),
|
||||
});
|
||||
export const BrowserScrollInput = z.object({
|
||||
sessionId: Id,
|
||||
dx: z.number().int().min(-5000).max(5000).default(0),
|
||||
dy: z.number().int().min(-5000).max(5000).default(0),
|
||||
allowSensitive: z.boolean().optional(),
|
||||
});
|
||||
export const BrowserWaitInput = z.object({
|
||||
sessionId: Id,
|
||||
ms: z.number().int().min(0).max(5000),
|
||||
allowSensitive: z.boolean().optional(),
|
||||
});
|
||||
const BrowserActionOutput = z.object({
|
||||
ok: z.literal(true),
|
||||
/** 换算后实际下发给浏览器的页面坐标(CSS 像素) */
|
||||
pageX: z.number().nullable(),
|
||||
pageY: z.number().nullable(),
|
||||
...BrowserShotFields,
|
||||
});
|
||||
export const BrowserClickAtOutput = BrowserActionOutput;
|
||||
export const BrowserTypeAtOutput = BrowserActionOutput;
|
||||
export const BrowserScrollOutput = BrowserActionOutput;
|
||||
export const BrowserWaitOutput = BrowserActionOutput;
|
||||
|
||||
/** 成功条件(调用方给出,服务端按页面事实判定;至少给一项,全部满足才算成功) */
|
||||
export const BrowserChallengeSuccess = z
|
||||
.object({
|
||||
/** 地址包含 */
|
||||
urlContains: z.string().min(1).max(500).optional(),
|
||||
/** 地址与开始时不同 */
|
||||
urlChanged: z.boolean().optional(),
|
||||
/** 页面文字包含 */
|
||||
textPresent: z.string().min(1).max(200).optional(),
|
||||
/** 页面文字不再包含 */
|
||||
textAbsent: z.string().min(1).max(200).optional(),
|
||||
})
|
||||
.refine((v) => !!v.urlContains || v.urlChanged === true || !!v.textPresent || !!v.textAbsent, {
|
||||
message: "成功条件至少给一项",
|
||||
});
|
||||
export type BrowserChallengeSuccess = z.infer<typeof BrowserChallengeSuccess>;
|
||||
|
||||
export const BrowserSolveChallengeInput = z.object({
|
||||
sessionId: Id,
|
||||
/** 要达成的目标(给子流程模型看),如"通过 Cloudflare 人机验证进入种子列表" */
|
||||
goal: z.string().trim().min(1).max(500),
|
||||
success: BrowserChallengeSuccess,
|
||||
/** 子流程的工具调用步数上限(一步 = 一次截图 / 点击 / 输入 / 滚动 / 等待) */
|
||||
maxSteps: z.number().int().min(1).max(12).default(8),
|
||||
budget: z
|
||||
.object({
|
||||
maxTokens: z.number().int().min(1000).max(2_000_000).optional(),
|
||||
maxCostUsd: z.number().min(0).max(50).optional(),
|
||||
maxDurationSec: z.number().int().min(10).max(600).optional(),
|
||||
})
|
||||
.default({}),
|
||||
/** 敏感截图(登录页 / 含密码框)也交给模型:仍需 browser.sensitive_screenshot 持续授权 */
|
||||
allowSensitive: z.boolean().optional(),
|
||||
});
|
||||
export const BrowserSolveChallengeOutput = z.object({
|
||||
/** solved 成功条件已满足;needs_human 需要人工(二次验证、模型判定、上游不支持图片);exhausted 步数 / 预算用尽;failed 出错 */
|
||||
status: z.enum(["solved", "needs_human", "exhausted", "failed"]),
|
||||
reason: z.string(),
|
||||
steps: NonNegInt,
|
||||
/** 子流程的 Agent 运行(事件与每步证据在它的事件流里) */
|
||||
childRunId: Id.nullable(),
|
||||
url: z.string().nullable(),
|
||||
controller: BrowserController,
|
||||
evidence: z.array(BrowserEvidenceRef),
|
||||
/** 转人工时的待处理事项与原站视图接管入口 */
|
||||
decisionId: Id.nullable(),
|
||||
takeover: z.object({ sessionId: Id, siteId: Id, path: z.string() }).nullable(),
|
||||
});
|
||||
|
||||
// ── WEB-1 网页搜索(REQ §10.2、§10.5):结果与正文一律标为不可信数据 ──
|
||||
|
||||
/** 给模型看的不可信标注(工具输出里原样带上) */
|
||||
@@ -540,6 +698,12 @@ export const CAPABILITY_IO = {
|
||||
"browser.navigate": { input: BrowserNavigateInput, output: BrowserNavigateOutput },
|
||||
"browser.interact": { input: BrowserInteractInput, output: BrowserInteractOutput },
|
||||
"browser.download": { input: BrowserDownloadInput, output: BrowserDownloadOutput },
|
||||
"browser.screenshot": { input: BrowserScreenshotInput, output: BrowserScreenshotOutput },
|
||||
"browser.click_at": { input: BrowserClickAtInput, output: BrowserClickAtOutput },
|
||||
"browser.type_at": { input: BrowserTypeAtInput, output: BrowserTypeAtOutput },
|
||||
"browser.scroll": { input: BrowserScrollInput, output: BrowserScrollOutput },
|
||||
"browser.wait": { input: BrowserWaitInput, output: BrowserWaitOutput },
|
||||
"browser.solve_challenge": { input: BrowserSolveChallengeInput, output: BrowserSolveChallengeOutput },
|
||||
"agent.plan": { input: AgentPlanInput, output: AgentPlanOutput },
|
||||
"agent.propose_options": { input: AgentProposeOptionsInput, output: AgentProposeOptionsOutput },
|
||||
"web.search": { input: WebSearchInput, output: WebSearchOutput },
|
||||
|
||||
@@ -76,6 +76,12 @@ export const CAPABILITY_IDS = [
|
||||
"browser.navigate",
|
||||
"browser.interact",
|
||||
"browser.download",
|
||||
"browser.screenshot",
|
||||
"browser.click_at",
|
||||
"browser.type_at",
|
||||
"browser.scroll",
|
||||
"browser.wait",
|
||||
"browser.solve_challenge",
|
||||
"agent.plan",
|
||||
"agent.propose_options",
|
||||
"web.search",
|
||||
@@ -402,6 +408,75 @@ export const CAPABILITY_CATALOG: Record<KnownCapabilityId, CapabilityDescriptor>
|
||||
false,
|
||||
false,
|
||||
),
|
||||
// VIS-1 视觉自动化闭环:截图(图片内容块)→ 视觉模型 → 坐标操作 → 再截图。坐标以最近一次截图的像素为准,
|
||||
// 服务端按视口 / DPR 换算;动作只在自动化控制权下、站点域名内执行,每步写证据缩略图;
|
||||
// 二次验证 / 短信 / 本人设备确认不自动处理,立即转人工
|
||||
"browser.screenshot": d(
|
||||
"browser.screenshot",
|
||||
"截取原站页面",
|
||||
"截取站点浏览器会话当前页面（缩放到不超过 1280 像素宽，作为图片交给支持视觉的模型）；后续 click_at / type_at 的坐标以这张截图的像素为准。登录页或含密码框的截图标为敏感，默认不交给模型",
|
||||
"read",
|
||||
"BrowserScreenshot",
|
||||
"read",
|
||||
"low",
|
||||
true,
|
||||
false,
|
||||
),
|
||||
"browser.click_at": d(
|
||||
"browser.click_at",
|
||||
"按坐标点击",
|
||||
"在最近一次 browser.screenshot 的像素坐标处点击，返回点击后的新截图；只在自动化控制权下、站点域名内执行",
|
||||
"execute",
|
||||
"BrowserClickAt",
|
||||
"admin",
|
||||
"medium",
|
||||
false,
|
||||
false,
|
||||
),
|
||||
"browser.type_at": d(
|
||||
"browser.type_at",
|
||||
"按坐标输入",
|
||||
"先点击截图坐标处的输入框（不给坐标则输入到当前焦点），再输入文本，可选回车提交；返回输入后的新截图。账号密码不要用它输入，用 browser.interact 的 credential",
|
||||
"execute",
|
||||
"BrowserTypeAt",
|
||||
"admin",
|
||||
"medium",
|
||||
false,
|
||||
false,
|
||||
),
|
||||
"browser.scroll": d(
|
||||
"browser.scroll",
|
||||
"滚动页面",
|
||||
"按像素滚动当前页面（dy 正数向下、dx 正数向右），返回滚动后的新截图",
|
||||
"execute",
|
||||
"BrowserScroll",
|
||||
"admin",
|
||||
"low",
|
||||
false,
|
||||
false,
|
||||
),
|
||||
"browser.wait": d(
|
||||
"browser.wait",
|
||||
"等待页面",
|
||||
"等待页面加载或动画完成（最多 5 秒），返回等待后的新截图",
|
||||
"read",
|
||||
"BrowserWait",
|
||||
"read",
|
||||
"low",
|
||||
true,
|
||||
false,
|
||||
),
|
||||
"browser.solve_challenge": d(
|
||||
"browser.solve_challenge",
|
||||
"处理页面挑战",
|
||||
"起一个有界的视觉子流程（只能截图、按坐标点击 / 输入、滚动、等待、读页面），识别并处理页面上的挑战，直到调用方给出的成功条件满足；超出步数或预算、遇到二次验证 / 短信 / 本人设备确认时停止，会话转等待处理并附证据截图，由人在原站视图接管",
|
||||
"execute",
|
||||
"BrowserSolveChallenge",
|
||||
"admin",
|
||||
"medium",
|
||||
false,
|
||||
false,
|
||||
),
|
||||
// AG-2 自动化优先的会话交互:步骤清单与方案选择。只在 Agent 会话里可用(HTTP / MCP 调用直接报错)
|
||||
"agent.plan": d(
|
||||
"agent.plan",
|
||||
|
||||
@@ -28,6 +28,8 @@ export interface MockCamofoxSite {
|
||||
/** 登录页带验证码:只有填对 captchaAnswer 才能登录(模拟需本人操作) */
|
||||
captcha?: boolean;
|
||||
captchaAnswer?: string;
|
||||
/** 登录成功后先到二次验证页(短信验证码,需本人操作;自动化不能完成) */
|
||||
twoFactor?: boolean;
|
||||
releases?: { id: string; title: string }[];
|
||||
/** 下载内容;缺省返回固定字节 */
|
||||
torrent?: (id: string) => Uint8Array;
|
||||
@@ -47,6 +49,9 @@ export interface MockCamofoxOptions {
|
||||
omitPaths?: string[];
|
||||
version?: string;
|
||||
port?: number;
|
||||
/** 视口(CSS 像素)与设备像素比:截图尺寸 = 视口 × DPR;元素行坐标按 CSS 像素。缺省 MOCK_VIEWPORT、DPR 1 */
|
||||
viewport?: { width: number; height: number };
|
||||
devicePixelRatio?: number;
|
||||
}
|
||||
|
||||
export interface MockCamofoxTab {
|
||||
@@ -239,16 +244,26 @@ function pngChunk(type: string, data: Uint8Array): Buffer {
|
||||
}
|
||||
const pngCache = new Map<string, string>();
|
||||
/** 纯色 PNG(base64),可选若干横条表示元素行 */
|
||||
export function solidPngBase64(w: number, h: number, rgb: [number, number, number], rows: number[] = []): string {
|
||||
const key = `${w}x${h}:${rgb.join(",")}:${rows.join(",")}`;
|
||||
export function solidPngBase64(
|
||||
w: number,
|
||||
h: number,
|
||||
rgb: [number, number, number],
|
||||
rows: number[] = [],
|
||||
/** 行坐标按 CSS 像素,乘以 DPR 画到截图上 */
|
||||
scale = 1,
|
||||
): string {
|
||||
const key = `${w}x${h}:${rgb.join(",")}:${rows.join(",")}:${scale}`;
|
||||
const hit = pngCache.get(key);
|
||||
if (hit) return hit;
|
||||
const raw = Buffer.alloc((w * 3 + 1) * h);
|
||||
for (let y = 0; y < h; y++) {
|
||||
const off = y * (w * 3 + 1);
|
||||
const inRow = rows.some((r) => y >= MOCK_ROW_Y0 + r * MOCK_ROW_H + 8 && y < MOCK_ROW_Y0 + (r + 1) * MOCK_ROW_H - 8);
|
||||
const cy = y / scale;
|
||||
const inRow = rows.some(
|
||||
(r) => cy >= MOCK_ROW_Y0 + r * MOCK_ROW_H + 8 && cy < MOCK_ROW_Y0 + (r + 1) * MOCK_ROW_H - 8,
|
||||
);
|
||||
for (let x = 0; x < w; x++) {
|
||||
const bar = inRow && x >= 40 && x < 640;
|
||||
const bar = inRow && x >= 40 * scale && x < 640 * scale;
|
||||
raw[off + 1 + x * 3] = bar ? 90 : rgb[0];
|
||||
raw[off + 2 + x * 3] = bar ? 110 : rgb[1];
|
||||
raw[off + 3 + x * 3] = bar ? 140 : rgb[2];
|
||||
@@ -284,6 +299,8 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise<
|
||||
const actions: MockCamofoxAction[] = [];
|
||||
const loggedIn = new Set<string>();
|
||||
const site = opts.site;
|
||||
const viewport = opts.viewport ?? MOCK_VIEWPORT;
|
||||
const dpr = opts.devicePixelRatio ?? 1;
|
||||
const loginPath = site?.loginPath ?? "/login.php";
|
||||
const protectedPaths = site?.protectedPaths ?? /^\/(?:torrents|details|download|index)\.php/;
|
||||
let seq = 0;
|
||||
@@ -328,8 +345,12 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise<
|
||||
site.captcha && t.fields.get("e5") !== (site.captchaAnswer ?? "4321") ? "验证码错误" : "用户名或密码错误";
|
||||
return;
|
||||
}
|
||||
loggedIn.add(t.userId);
|
||||
const back = new URL(t.url).searchParams.get("returnto") ?? "/torrents.php";
|
||||
if (site.twoFactor) {
|
||||
go(t, `${site.origin}/2fa.php?returnto=${encodeURIComponent(back)}`);
|
||||
return;
|
||||
}
|
||||
loggedIn.add(t.userId);
|
||||
go(t, `${site.origin}${back}`);
|
||||
};
|
||||
|
||||
@@ -354,6 +375,14 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise<
|
||||
if (t.error) lines.push({ ref: "e7", role: "alert", text: t.error });
|
||||
return lines;
|
||||
}
|
||||
if (u.pathname === "/2fa.php") {
|
||||
return [
|
||||
{ ref: "e1", role: "heading", text: "二次验证" },
|
||||
{ ref: "e2", role: "text", text: "验证码已发送到尾号 0042 的手机，请输入短信验证码" },
|
||||
{ ref: "e3", role: "textbox", text: `短信验证码: ${field("e3")}` },
|
||||
{ ref: "e4", role: "button", text: "确认" },
|
||||
];
|
||||
}
|
||||
if (u.pathname === "/torrents.php") {
|
||||
const lines: Line[] = [{ ref: "e1", role: "heading", text: "种子列表" }];
|
||||
releases.forEach((r, i) => {
|
||||
@@ -566,10 +595,11 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise<
|
||||
const login = new URL(t.url).pathname === loginPath;
|
||||
const rows = pageLines(t).map((_l, i) => i);
|
||||
const data = solidPngBase64(
|
||||
MOCK_VIEWPORT.width,
|
||||
MOCK_VIEWPORT.height,
|
||||
Math.round(viewport.width * dpr),
|
||||
Math.round(viewport.height * dpr),
|
||||
login ? [236, 232, 224] : [248, 248, 250],
|
||||
rows,
|
||||
dpr,
|
||||
);
|
||||
return { json: { screenshot: { data, mimeType: "image/png" } } };
|
||||
},
|
||||
@@ -605,6 +635,22 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise<
|
||||
action("evaluate", (t, b) => {
|
||||
// 只模拟适配器的坐标定位表达式:elementFromPoint(x, y) + data-umt-pt 标记
|
||||
const expr = typeof b.expression === "string" ? b.expression : "";
|
||||
// 适配器的页面度量表达式(VIS-1)
|
||||
if (expr.includes("umt-metrics"))
|
||||
return {
|
||||
json: {
|
||||
ok: true,
|
||||
result: {
|
||||
w: viewport.width,
|
||||
h: viewport.height,
|
||||
dpr,
|
||||
pw: pageLines(t).some((l) => l.role.includes("password")),
|
||||
text: pageLines(t)
|
||||
.map((l) => l.text)
|
||||
.join("\n"),
|
||||
},
|
||||
},
|
||||
};
|
||||
const pt = /elementFromPoint\((\d+), (\d+)\)/.exec(expr);
|
||||
const nonce = /setAttribute\("data-umt-pt", "([^"]+)"\)/.exec(expr)?.[1];
|
||||
if (!pt || !nonce) return { json: { ok: true, result: null } };
|
||||
|
||||
Reference in new issue
Block a user