feat(browser): vision automation loop with screenshots, coordinate actions and bounded challenge solving
- browser.screenshot returns the page as an image content block (≤1280px wide, JPEG quality configurable) for vision-capable upstreams; non-vision upstreams get a redacted text snapshot. Login pages / password fields are marked sensitive and withheld unless a browser.sensitive_screenshot grant exists and allowSensitive is passed. - browser.click_at / type_at / scroll / wait act in screenshot pixels; the server converts via viewport/DPR, requires automation control and an allowlisted host, and writes a thumbnail evidence file (DATA_DIR/tmp/evidence) referenced from agent_run_event. - browser.solve_challenge runs a bounded child agent_run (vision caps only, maxSteps ≤ 12, budget), judges success from page facts, stops immediately on 2FA / SMS / device confirmation, and otherwise moves the session to waiting with evidence and a takeover link. - PiAgentRuntime passes modelImage outputs as pi-ai image content when model.input has image; the ledger never stores image data; coordinate actions use per-invocation idempotency keys. - Tests: image codec, runtime image content, E14 e2e (login solved, 2FA stop, step limit, sensitive withheld). Real camofox 1.17.0 check on example.com recorded in the runbook. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
0765264a47
commit
c437c6bff6
34 files changed
+2194
-35
No files matched your search
@@ -445,6 +445,164 @@ export const BrowserDownloadOutput = z.object({
|
||||
note: z.string().nullable(),
|
||||
});
|
||||
|
||||
// ── VIS-1 视觉自动化闭环 ──
|
||||
// 截图以图片内容块交给模型:能力输出里的 modelImage 由 Agent 运行时摘出,按上游是否支持图片输入转成
|
||||
// image content(不支持时降级为文字快照并提示);JSON 文本里不再带图片数据。
|
||||
|
||||
/** 能力输出里交给模型的图片(运行时摘出为 image content,不进 JSON 文本、不进操作账本) */
|
||||
export const ModelImage = z.object({
|
||||
mimeType: z.enum(["image/png", "image/jpeg"]),
|
||||
/** base64 */
|
||||
data: z.string(),
|
||||
width: z.number().int().positive(),
|
||||
height: z.number().int().positive(),
|
||||
});
|
||||
export type ModelImage = z.infer<typeof ModelImage>;
|
||||
/** 运行时识别图片的键名 */
|
||||
export const MODEL_IMAGE_KEY = "modelImage" as const;
|
||||
|
||||
/** 每步证据:截图缩略图存 DATA_DIR/tmp/evidence,这里只给引用 */
|
||||
export const BrowserEvidenceRef = z.object({
|
||||
id: Id,
|
||||
/** 相对 DATA_DIR/tmp 的路径,如 evidence/<id>.jpg */
|
||||
ref: z.string(),
|
||||
capturedAt: z.string(),
|
||||
});
|
||||
export type BrowserEvidenceRef = z.infer<typeof BrowserEvidenceRef>;
|
||||
|
||||
/** 截图像素 ↔ 页面 CSS 像素的换算依据 */
|
||||
export const BrowserCoordinateSpace = z.object({
|
||||
/** 交给模型的截图尺寸(坐标以它为准) */
|
||||
imageWidth: z.number().int().positive(),
|
||||
imageHeight: z.number().int().positive(),
|
||||
/** 浏览器原始截图尺寸 */
|
||||
sourceWidth: z.number().int().positive(),
|
||||
sourceHeight: z.number().int().positive(),
|
||||
/** 页面视口(CSS 像素)与设备像素比;取不到时按原始截图尺寸、DPR 1 推定 */
|
||||
viewportWidth: z.number().positive(),
|
||||
viewportHeight: z.number().positive(),
|
||||
devicePixelRatio: z.number().positive(),
|
||||
assumed: z.boolean(),
|
||||
});
|
||||
export type BrowserCoordinateSpace = z.infer<typeof BrowserCoordinateSpace>;
|
||||
|
||||
const BrowserShotFields = {
|
||||
evidence: BrowserEvidenceRef.nullable(),
|
||||
/** 已脱敏的页面地址 */
|
||||
url: z.string().nullable(),
|
||||
controller: BrowserController,
|
||||
/** 登录页 / 含密码框 / 地址带凭据:截图敏感,默认不交给模型 */
|
||||
sensitive: z.boolean(),
|
||||
/** 本次截图是否作为图片交给了模型 */
|
||||
imageForModel: z.boolean(),
|
||||
/** 没交给模型时的原因与替代:上游不支持图片输入时附文字快照 */
|
||||
note: z.string().nullable(),
|
||||
textSnapshot: z.string().nullable(),
|
||||
coordinates: BrowserCoordinateSpace.nullable(),
|
||||
/** 会话转为等待处理的原因(二次验证等);非 null 时自动化动作已停止 */
|
||||
waitingReason: z.string().nullable(),
|
||||
modelImage: ModelImage.nullable(),
|
||||
};
|
||||
|
||||
export const BrowserScreenshotInput = z.object({
|
||||
sessionId: Id,
|
||||
tabId: z.string().min(1).optional(),
|
||||
/** 敏感截图也交给模型:需要 browser.sensitive_screenshot 持续授权,否则忽略 */
|
||||
allowSensitive: z.boolean().optional(),
|
||||
});
|
||||
export const BrowserScreenshotOutput = z.object(BrowserShotFields);
|
||||
|
||||
const Coord = z.number().min(0).max(10_000);
|
||||
|
||||
export const BrowserClickAtInput = z.object({
|
||||
sessionId: Id,
|
||||
x: Coord,
|
||||
y: Coord,
|
||||
/** camofox 1.17.0 只支持左键 */
|
||||
button: z.enum(["left", "right", "middle"]).optional(),
|
||||
allowSensitive: z.boolean().optional(),
|
||||
});
|
||||
export const BrowserTypeAtInput = z.object({
|
||||
sessionId: Id,
|
||||
/** 不给坐标 = 输入到当前焦点 */
|
||||
x: Coord.optional(),
|
||||
y: Coord.optional(),
|
||||
text: z.string().min(1).max(500),
|
||||
pressEnter: z.boolean().optional(),
|
||||
allowSensitive: z.boolean().optional(),
|
||||
});
|
||||
export const BrowserScrollInput = z.object({
|
||||
sessionId: Id,
|
||||
dx: z.number().int().min(-5000).max(5000).default(0),
|
||||
dy: z.number().int().min(-5000).max(5000).default(0),
|
||||
allowSensitive: z.boolean().optional(),
|
||||
});
|
||||
export const BrowserWaitInput = z.object({
|
||||
sessionId: Id,
|
||||
ms: z.number().int().min(0).max(5000),
|
||||
allowSensitive: z.boolean().optional(),
|
||||
});
|
||||
const BrowserActionOutput = z.object({
|
||||
ok: z.literal(true),
|
||||
/** 换算后实际下发给浏览器的页面坐标(CSS 像素) */
|
||||
pageX: z.number().nullable(),
|
||||
pageY: z.number().nullable(),
|
||||
...BrowserShotFields,
|
||||
});
|
||||
export const BrowserClickAtOutput = BrowserActionOutput;
|
||||
export const BrowserTypeAtOutput = BrowserActionOutput;
|
||||
export const BrowserScrollOutput = BrowserActionOutput;
|
||||
export const BrowserWaitOutput = BrowserActionOutput;
|
||||
|
||||
/** 成功条件(调用方给出,服务端按页面事实判定;至少给一项,全部满足才算成功) */
|
||||
export const BrowserChallengeSuccess = z
|
||||
.object({
|
||||
/** 地址包含 */
|
||||
urlContains: z.string().min(1).max(500).optional(),
|
||||
/** 地址与开始时不同 */
|
||||
urlChanged: z.boolean().optional(),
|
||||
/** 页面文字包含 */
|
||||
textPresent: z.string().min(1).max(200).optional(),
|
||||
/** 页面文字不再包含 */
|
||||
textAbsent: z.string().min(1).max(200).optional(),
|
||||
})
|
||||
.refine((v) => !!v.urlContains || v.urlChanged === true || !!v.textPresent || !!v.textAbsent, {
|
||||
message: "成功条件至少给一项",
|
||||
});
|
||||
export type BrowserChallengeSuccess = z.infer<typeof BrowserChallengeSuccess>;
|
||||
|
||||
export const BrowserSolveChallengeInput = z.object({
|
||||
sessionId: Id,
|
||||
/** 要达成的目标(给子流程模型看),如"通过 Cloudflare 人机验证进入种子列表" */
|
||||
goal: z.string().trim().min(1).max(500),
|
||||
success: BrowserChallengeSuccess,
|
||||
/** 子流程的工具调用步数上限(一步 = 一次截图 / 点击 / 输入 / 滚动 / 等待) */
|
||||
maxSteps: z.number().int().min(1).max(12).default(8),
|
||||
budget: z
|
||||
.object({
|
||||
maxTokens: z.number().int().min(1000).max(2_000_000).optional(),
|
||||
maxCostUsd: z.number().min(0).max(50).optional(),
|
||||
maxDurationSec: z.number().int().min(10).max(600).optional(),
|
||||
})
|
||||
.default({}),
|
||||
/** 敏感截图(登录页 / 含密码框)也交给模型:仍需 browser.sensitive_screenshot 持续授权 */
|
||||
allowSensitive: z.boolean().optional(),
|
||||
});
|
||||
export const BrowserSolveChallengeOutput = z.object({
|
||||
/** solved 成功条件已满足;needs_human 需要人工(二次验证、模型判定、上游不支持图片);exhausted 步数 / 预算用尽;failed 出错 */
|
||||
status: z.enum(["solved", "needs_human", "exhausted", "failed"]),
|
||||
reason: z.string(),
|
||||
steps: NonNegInt,
|
||||
/** 子流程的 Agent 运行(事件与每步证据在它的事件流里) */
|
||||
childRunId: Id.nullable(),
|
||||
url: z.string().nullable(),
|
||||
controller: BrowserController,
|
||||
evidence: z.array(BrowserEvidenceRef),
|
||||
/** 转人工时的待处理事项与原站视图接管入口 */
|
||||
decisionId: Id.nullable(),
|
||||
takeover: z.object({ sessionId: Id, siteId: Id, path: z.string() }).nullable(),
|
||||
});
|
||||
|
||||
// ── WEB-1 网页搜索(REQ §10.2、§10.5):结果与正文一律标为不可信数据 ──
|
||||
|
||||
/** 给模型看的不可信标注(工具输出里原样带上) */
|
||||
@@ -540,6 +698,12 @@ export const CAPABILITY_IO = {
|
||||
"browser.navigate": { input: BrowserNavigateInput, output: BrowserNavigateOutput },
|
||||
"browser.interact": { input: BrowserInteractInput, output: BrowserInteractOutput },
|
||||
"browser.download": { input: BrowserDownloadInput, output: BrowserDownloadOutput },
|
||||
"browser.screenshot": { input: BrowserScreenshotInput, output: BrowserScreenshotOutput },
|
||||
"browser.click_at": { input: BrowserClickAtInput, output: BrowserClickAtOutput },
|
||||
"browser.type_at": { input: BrowserTypeAtInput, output: BrowserTypeAtOutput },
|
||||
"browser.scroll": { input: BrowserScrollInput, output: BrowserScrollOutput },
|
||||
"browser.wait": { input: BrowserWaitInput, output: BrowserWaitOutput },
|
||||
"browser.solve_challenge": { input: BrowserSolveChallengeInput, output: BrowserSolveChallengeOutput },
|
||||
"agent.plan": { input: AgentPlanInput, output: AgentPlanOutput },
|
||||
"agent.propose_options": { input: AgentProposeOptionsInput, output: AgentProposeOptionsOutput },
|
||||
"web.search": { input: WebSearchInput, output: WebSearchOutput },
|
||||
|
||||
@@ -76,6 +76,12 @@ export const CAPABILITY_IDS = [
|
||||
"browser.navigate",
|
||||
"browser.interact",
|
||||
"browser.download",
|
||||
"browser.screenshot",
|
||||
"browser.click_at",
|
||||
"browser.type_at",
|
||||
"browser.scroll",
|
||||
"browser.wait",
|
||||
"browser.solve_challenge",
|
||||
"agent.plan",
|
||||
"agent.propose_options",
|
||||
"web.search",
|
||||
@@ -402,6 +408,75 @@ export const CAPABILITY_CATALOG: Record<KnownCapabilityId, CapabilityDescriptor>
|
||||
false,
|
||||
false,
|
||||
),
|
||||
// VIS-1 视觉自动化闭环:截图(图片内容块)→ 视觉模型 → 坐标操作 → 再截图。坐标以最近一次截图的像素为准,
|
||||
// 服务端按视口 / DPR 换算;动作只在自动化控制权下、站点域名内执行,每步写证据缩略图;
|
||||
// 二次验证 / 短信 / 本人设备确认不自动处理,立即转人工
|
||||
"browser.screenshot": d(
|
||||
"browser.screenshot",
|
||||
"截取原站页面",
|
||||
"截取站点浏览器会话当前页面（缩放到不超过 1280 像素宽，作为图片交给支持视觉的模型）；后续 click_at / type_at 的坐标以这张截图的像素为准。登录页或含密码框的截图标为敏感，默认不交给模型",
|
||||
"read",
|
||||
"BrowserScreenshot",
|
||||
"read",
|
||||
"low",
|
||||
true,
|
||||
false,
|
||||
),
|
||||
"browser.click_at": d(
|
||||
"browser.click_at",
|
||||
"按坐标点击",
|
||||
"在最近一次 browser.screenshot 的像素坐标处点击，返回点击后的新截图；只在自动化控制权下、站点域名内执行",
|
||||
"execute",
|
||||
"BrowserClickAt",
|
||||
"admin",
|
||||
"medium",
|
||||
false,
|
||||
false,
|
||||
),
|
||||
"browser.type_at": d(
|
||||
"browser.type_at",
|
||||
"按坐标输入",
|
||||
"先点击截图坐标处的输入框（不给坐标则输入到当前焦点），再输入文本，可选回车提交；返回输入后的新截图。账号密码不要用它输入，用 browser.interact 的 credential",
|
||||
"execute",
|
||||
"BrowserTypeAt",
|
||||
"admin",
|
||||
"medium",
|
||||
false,
|
||||
false,
|
||||
),
|
||||
"browser.scroll": d(
|
||||
"browser.scroll",
|
||||
"滚动页面",
|
||||
"按像素滚动当前页面（dy 正数向下、dx 正数向右），返回滚动后的新截图",
|
||||
"execute",
|
||||
"BrowserScroll",
|
||||
"admin",
|
||||
"low",
|
||||
false,
|
||||
false,
|
||||
),
|
||||
"browser.wait": d(
|
||||
"browser.wait",
|
||||
"等待页面",
|
||||
"等待页面加载或动画完成（最多 5 秒），返回等待后的新截图",
|
||||
"read",
|
||||
"BrowserWait",
|
||||
"read",
|
||||
"low",
|
||||
true,
|
||||
false,
|
||||
),
|
||||
"browser.solve_challenge": d(
|
||||
"browser.solve_challenge",
|
||||
"处理页面挑战",
|
||||
"起一个有界的视觉子流程（只能截图、按坐标点击 / 输入、滚动、等待、读页面），识别并处理页面上的挑战，直到调用方给出的成功条件满足；超出步数或预算、遇到二次验证 / 短信 / 本人设备确认时停止，会话转等待处理并附证据截图，由人在原站视图接管",
|
||||
"execute",
|
||||
"BrowserSolveChallenge",
|
||||
"admin",
|
||||
"medium",
|
||||
false,
|
||||
false,
|
||||
),
|
||||
// AG-2 自动化优先的会话交互:步骤清单与方案选择。只在 Agent 会话里可用(HTTP / MCP 调用直接报错)
|
||||
"agent.plan": d(
|
||||
"agent.plan",
|
||||
|
||||
Reference in new issue
Block a user