feat(browser): vision automation loop with screenshots, coordinate actions and bounded challenge solving
- browser.screenshot returns the page as an image content block (≤1280px wide, JPEG quality configurable) for vision-capable upstreams; non-vision upstreams get a redacted text snapshot. Login pages / password fields are marked sensitive and withheld unless a browser.sensitive_screenshot grant exists and allowSensitive is passed. - browser.click_at / type_at / scroll / wait act in screenshot pixels; the server converts via viewport/DPR, requires automation control and an allowlisted host, and writes a thumbnail evidence file (DATA_DIR/tmp/evidence) referenced from agent_run_event. - browser.solve_challenge runs a bounded child agent_run (vision caps only, maxSteps ≤ 12, budget), judges success from page facts, stops immediately on 2FA / SMS / device confirmation, and otherwise moves the session to waiting with evidence and a takeover link. - PiAgentRuntime passes modelImage outputs as pi-ai image content when model.input has image; the ledger never stores image data; coordinate actions use per-invocation idempotency keys. - Tests: image codec, runtime image content, E14 e2e (login solved, 2FA stop, step limit, sensitive withheld). Real camofox 1.17.0 check on example.com recorded in the runbook. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
0765264a47
commit
c437c6bff6
34 files changed
+2194
-35
No files matched your search
@@ -28,6 +28,8 @@ export interface MockCamofoxSite {
|
||||
/** 登录页带验证码:只有填对 captchaAnswer 才能登录(模拟需本人操作) */
|
||||
captcha?: boolean;
|
||||
captchaAnswer?: string;
|
||||
/** 登录成功后先到二次验证页(短信验证码,需本人操作;自动化不能完成) */
|
||||
twoFactor?: boolean;
|
||||
releases?: { id: string; title: string }[];
|
||||
/** 下载内容;缺省返回固定字节 */
|
||||
torrent?: (id: string) => Uint8Array;
|
||||
@@ -47,6 +49,9 @@ export interface MockCamofoxOptions {
|
||||
omitPaths?: string[];
|
||||
version?: string;
|
||||
port?: number;
|
||||
/** 视口(CSS 像素)与设备像素比:截图尺寸 = 视口 × DPR;元素行坐标按 CSS 像素。缺省 MOCK_VIEWPORT、DPR 1 */
|
||||
viewport?: { width: number; height: number };
|
||||
devicePixelRatio?: number;
|
||||
}
|
||||
|
||||
export interface MockCamofoxTab {
|
||||
@@ -239,16 +244,26 @@ function pngChunk(type: string, data: Uint8Array): Buffer {
|
||||
}
|
||||
const pngCache = new Map<string, string>();
|
||||
/** 纯色 PNG(base64),可选若干横条表示元素行 */
|
||||
export function solidPngBase64(w: number, h: number, rgb: [number, number, number], rows: number[] = []): string {
|
||||
const key = `${w}x${h}:${rgb.join(",")}:${rows.join(",")}`;
|
||||
export function solidPngBase64(
|
||||
w: number,
|
||||
h: number,
|
||||
rgb: [number, number, number],
|
||||
rows: number[] = [],
|
||||
/** 行坐标按 CSS 像素,乘以 DPR 画到截图上 */
|
||||
scale = 1,
|
||||
): string {
|
||||
const key = `${w}x${h}:${rgb.join(",")}:${rows.join(",")}:${scale}`;
|
||||
const hit = pngCache.get(key);
|
||||
if (hit) return hit;
|
||||
const raw = Buffer.alloc((w * 3 + 1) * h);
|
||||
for (let y = 0; y < h; y++) {
|
||||
const off = y * (w * 3 + 1);
|
||||
const inRow = rows.some((r) => y >= MOCK_ROW_Y0 + r * MOCK_ROW_H + 8 && y < MOCK_ROW_Y0 + (r + 1) * MOCK_ROW_H - 8);
|
||||
const cy = y / scale;
|
||||
const inRow = rows.some(
|
||||
(r) => cy >= MOCK_ROW_Y0 + r * MOCK_ROW_H + 8 && cy < MOCK_ROW_Y0 + (r + 1) * MOCK_ROW_H - 8,
|
||||
);
|
||||
for (let x = 0; x < w; x++) {
|
||||
const bar = inRow && x >= 40 && x < 640;
|
||||
const bar = inRow && x >= 40 * scale && x < 640 * scale;
|
||||
raw[off + 1 + x * 3] = bar ? 90 : rgb[0];
|
||||
raw[off + 2 + x * 3] = bar ? 110 : rgb[1];
|
||||
raw[off + 3 + x * 3] = bar ? 140 : rgb[2];
|
||||
@@ -284,6 +299,8 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise<
|
||||
const actions: MockCamofoxAction[] = [];
|
||||
const loggedIn = new Set<string>();
|
||||
const site = opts.site;
|
||||
const viewport = opts.viewport ?? MOCK_VIEWPORT;
|
||||
const dpr = opts.devicePixelRatio ?? 1;
|
||||
const loginPath = site?.loginPath ?? "/login.php";
|
||||
const protectedPaths = site?.protectedPaths ?? /^\/(?:torrents|details|download|index)\.php/;
|
||||
let seq = 0;
|
||||
@@ -328,8 +345,12 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise<
|
||||
site.captcha && t.fields.get("e5") !== (site.captchaAnswer ?? "4321") ? "验证码错误" : "用户名或密码错误";
|
||||
return;
|
||||
}
|
||||
loggedIn.add(t.userId);
|
||||
const back = new URL(t.url).searchParams.get("returnto") ?? "/torrents.php";
|
||||
if (site.twoFactor) {
|
||||
go(t, `${site.origin}/2fa.php?returnto=${encodeURIComponent(back)}`);
|
||||
return;
|
||||
}
|
||||
loggedIn.add(t.userId);
|
||||
go(t, `${site.origin}${back}`);
|
||||
};
|
||||
|
||||
@@ -354,6 +375,14 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise<
|
||||
if (t.error) lines.push({ ref: "e7", role: "alert", text: t.error });
|
||||
return lines;
|
||||
}
|
||||
if (u.pathname === "/2fa.php") {
|
||||
return [
|
||||
{ ref: "e1", role: "heading", text: "二次验证" },
|
||||
{ ref: "e2", role: "text", text: "验证码已发送到尾号 0042 的手机，请输入短信验证码" },
|
||||
{ ref: "e3", role: "textbox", text: `短信验证码: ${field("e3")}` },
|
||||
{ ref: "e4", role: "button", text: "确认" },
|
||||
];
|
||||
}
|
||||
if (u.pathname === "/torrents.php") {
|
||||
const lines: Line[] = [{ ref: "e1", role: "heading", text: "种子列表" }];
|
||||
releases.forEach((r, i) => {
|
||||
@@ -566,10 +595,11 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise<
|
||||
const login = new URL(t.url).pathname === loginPath;
|
||||
const rows = pageLines(t).map((_l, i) => i);
|
||||
const data = solidPngBase64(
|
||||
MOCK_VIEWPORT.width,
|
||||
MOCK_VIEWPORT.height,
|
||||
Math.round(viewport.width * dpr),
|
||||
Math.round(viewport.height * dpr),
|
||||
login ? [236, 232, 224] : [248, 248, 250],
|
||||
rows,
|
||||
dpr,
|
||||
);
|
||||
return { json: { screenshot: { data, mimeType: "image/png" } } };
|
||||
},
|
||||
@@ -605,6 +635,22 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise<
|
||||
action("evaluate", (t, b) => {
|
||||
// 只模拟适配器的坐标定位表达式:elementFromPoint(x, y) + data-umt-pt 标记
|
||||
const expr = typeof b.expression === "string" ? b.expression : "";
|
||||
// 适配器的页面度量表达式(VIS-1)
|
||||
if (expr.includes("umt-metrics"))
|
||||
return {
|
||||
json: {
|
||||
ok: true,
|
||||
result: {
|
||||
w: viewport.width,
|
||||
h: viewport.height,
|
||||
dpr,
|
||||
pw: pageLines(t).some((l) => l.role.includes("password")),
|
||||
text: pageLines(t)
|
||||
.map((l) => l.text)
|
||||
.join("\n"),
|
||||
},
|
||||
},
|
||||
};
|
||||
const pt = /elementFromPoint\((\d+), (\d+)\)/.exec(expr);
|
||||
const nonce = /setAttribute\("data-umt-pt", "([^"]+)"\)/.exec(expr)?.[1];
|
||||
if (!pt || !nonce) return { json: { ok: true, result: null } };
|
||||
|
||||
Reference in new issue
Block a user