feat(browser): vision automation loop with screenshots, coordinate actions and bounded challenge solving
- browser.screenshot returns the page as an image content block (≤1280px wide, JPEG quality configurable) for vision-capable upstreams; non-vision upstreams get a redacted text snapshot. Login pages / password fields are marked sensitive and withheld unless a browser.sensitive_screenshot grant exists and allowSensitive is passed. - browser.click_at / type_at / scroll / wait act in screenshot pixels; the server converts via viewport/DPR, requires automation control and an allowlisted host, and writes a thumbnail evidence file (DATA_DIR/tmp/evidence) referenced from agent_run_event. - browser.solve_challenge runs a bounded child agent_run (vision caps only, maxSteps ≤ 12, budget), judges success from page facts, stops immediately on 2FA / SMS / device confirmation, and otherwise moves the session to waiting with evidence and a takeover link. - PiAgentRuntime passes modelImage outputs as pi-ai image content when model.input has image; the ledger never stores image data; coordinate actions use per-invocation idempotency keys. - Tests: image codec, runtime image content, E14 e2e (login solved, 2FA stop, step limit, sensitive withheld). Real camofox 1.17.0 check on example.com recorded in the runbook. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
0765264a47
commit
c437c6bff6
34 files changed
+2194
-35
No files matched your search
@@ -20,6 +20,7 @@ import { type Bootstrapped, bootstrap } from "../../src/context.ts";
|
||||
import { createApp } from "../../src/http/app.ts";
|
||||
import { EventBus } from "../../src/http/sse.ts";
|
||||
import { createLogger } from "../../src/logger.ts";
|
||||
import type { BrowserModuleOptions } from "../../src/modules/browser/service.ts";
|
||||
import { setDefaultUpstream } from "../../src/modules/pi/store.ts";
|
||||
import type { ServerModule } from "../../src/modules/types.ts";
|
||||
import type { WebSearchModuleOptions } from "../../src/modules/websearch/index.ts";
|
||||
@@ -81,6 +82,8 @@ export interface E2EOptions {
|
||||
tmdbFixtures?: TmdbFixture[];
|
||||
/** 网页搜索模块注入(E13) */
|
||||
webSearch?: WebSearchModuleOptions;
|
||||
/** 浏览器模块注入(E14) */
|
||||
browser?: BrowserModuleOptions;
|
||||
}
|
||||
|
||||
export async function waitFor<T>(
|
||||
@@ -136,6 +139,7 @@ export async function createE2E(name: string, opts: E2EOptions = {}): Promise<E2
|
||||
sessionDir,
|
||||
...(opts.fakeScripts ? { fakeScripts: opts.fakeScripts } : {}),
|
||||
...(opts.webSearch ? { webSearch: opts.webSearch } : {}),
|
||||
...(opts.browser ? { browser: opts.browser } : {}),
|
||||
});
|
||||
const booted = await bootstrap({
|
||||
role: "all",
|
||||
|
||||
Reference in new issue
Block a user