feat(browser): complete camofox BrowserProvider with tabs, interaction, download capture and control lease
Contracts for browser API and browser.* agent capabilities; adapter implements close/list/restore sessions, tabs + reconcile, history, click/type/select/scroll/press, extract, waitForDownload (staged via DownloadSink, sha256), stale-ref guard and automation/human actor check. testkit mock camofox gains an interactive site (login redirect, captcha, click-driven page changes, downloads, PNG screenshots). Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
62290d4477
commit
c17d4b90bd
12 files changed
+1832
-101
No files matched your search
@@ -1,5 +1,6 @@
|
||||
import { z } from "zod";
|
||||
import { MediaAsset } from "./assets.ts";
|
||||
import { BrowserController, BrowserDownloadVerification, BrowserSnapshot } from "./browser.ts";
|
||||
import { CAPABILITY_CATALOG, type CapabilitySpec, type KnownCapabilityId } from "./capability.ts";
|
||||
import { Id, JsonObject, NonNegInt } from "./common.ts";
|
||||
import { MediaEntity, MediaKind } from "./content.ts";
|
||||
@@ -318,6 +319,93 @@ export const SiteUserDataOutput = z.object({
|
||||
history: z.array(SiteUserDataPoint),
|
||||
});
|
||||
|
||||
// C1-E 浏览器能力:会话按站点/账号隔离;元素引用只来自最近一次 browser.snapshot,
|
||||
// 控制权交还或导航后旧引用失效;凭证用 credential 字段由服务端注入,不进入模型。
|
||||
const BrowserAgentTarget = z.union([
|
||||
z.object({ ref: z.string().min(1).max(40) }),
|
||||
z.object({ selector: z.string().min(1).max(500) }),
|
||||
]);
|
||||
|
||||
export const BrowserSnapshotInput = z
|
||||
.object({
|
||||
/** 会话 ID;缺省用 siteId 对应的未关闭会话 */
|
||||
sessionId: Id.optional(),
|
||||
siteId: Id.optional(),
|
||||
tabId: z.string().min(1).optional(),
|
||||
offset: z.number().int().min(0).optional(),
|
||||
})
|
||||
.refine((v) => !!v.sessionId || !!v.siteId, { message: "sessionId 与 siteId 至少给一个" });
|
||||
export const BrowserSnapshotOutput = z.object({
|
||||
snapshot: BrowserSnapshot,
|
||||
controller: BrowserController,
|
||||
/** 会话在等待处理(验证码 / 二次认证 / 登录失效)时的原因;此时自动化动作被拒,需人工在原站视图处理 */
|
||||
waitingReason: z.string().nullable(),
|
||||
loginRequired: z.boolean(),
|
||||
});
|
||||
|
||||
export const BrowserNavigateInput = z
|
||||
.object({
|
||||
sessionId: Id.optional(),
|
||||
/** 无会话时按站点(和账号)打开一个 */
|
||||
siteId: Id.optional(),
|
||||
accountId: Id.optional(),
|
||||
tabId: z.string().min(1).optional(),
|
||||
/** 必须在站点域名内 */
|
||||
url: z.url().max(4000),
|
||||
})
|
||||
.refine((v) => !!v.sessionId || !!v.siteId, { message: "sessionId 与 siteId 至少给一个" });
|
||||
export const BrowserNavigateOutput = z.object({
|
||||
sessionId: Id,
|
||||
url: z.string(),
|
||||
httpStatus: z.number().int().nullable(),
|
||||
ok: z.boolean(),
|
||||
attempts: z.number().int(),
|
||||
loginRequired: z.boolean(),
|
||||
controller: BrowserController,
|
||||
});
|
||||
|
||||
export const BrowserInteractInput = z.object({
|
||||
sessionId: Id,
|
||||
tabId: z.string().min(1).optional(),
|
||||
/** request_human:转"等待处理",由人在原站视图接管(验证码、二次认证等) */
|
||||
action: z.enum(["click", "type", "select", "scroll", "press", "request_human"]),
|
||||
target: BrowserAgentTarget.optional(),
|
||||
text: z.string().max(4000).optional(),
|
||||
/** 站点账号凭证字段:服务端解析后直接下发到输入框 */
|
||||
credential: z.enum(["username", "password", "totp"]).optional(),
|
||||
key: z.string().min(1).max(40).optional(),
|
||||
value: z.string().max(500).optional(),
|
||||
direction: z.enum(["up", "down", "left", "right"]).optional(),
|
||||
amount: z.number().int().min(1).max(10_000).optional(),
|
||||
submit: z.boolean().optional(),
|
||||
reason: z.string().max(200).optional(),
|
||||
});
|
||||
export const BrowserInteractOutput = z.object({
|
||||
ok: z.literal(true),
|
||||
controller: BrowserController,
|
||||
/** true = 页面可能已变化,下一步先 browser.snapshot 再用新引用 */
|
||||
needsSnapshot: z.boolean(),
|
||||
});
|
||||
|
||||
export const BrowserDownloadInput = z.object({
|
||||
sessionId: Id,
|
||||
tabId: z.string().min(1).optional(),
|
||||
/** 触发下载的元素(快照引用);缺省只等待已触发的下载 */
|
||||
target: BrowserAgentTarget.optional(),
|
||||
timeoutMs: z.number().int().min(500).max(120_000).default(15_000),
|
||||
});
|
||||
export const BrowserDownloadOutput = z.object({
|
||||
artifactId: Id,
|
||||
filename: z.string(),
|
||||
sizeBytes: NonNegInt,
|
||||
sha256: z.string(),
|
||||
verification: BrowserDownloadVerification,
|
||||
/** 匹配到的本地资源;下一步用 download.plan / download.submit 走同一提交流程 */
|
||||
sourceReleaseId: Id.nullable(),
|
||||
duplicate: z.boolean(),
|
||||
note: z.string().nullable(),
|
||||
});
|
||||
|
||||
/** 能力 ID → 输入/输出 schema(server 注册处理器、agent 生成工具定义时共用) */
|
||||
export const CAPABILITY_IO = {
|
||||
"catalog.lookup": { input: CatalogLookupInput, output: CatalogLookupOutput },
|
||||
@@ -341,6 +429,10 @@ export const CAPABILITY_IO = {
|
||||
"site.health_check": { input: SiteHealthCheckInput, output: SiteHealthCheckOutput },
|
||||
"site.refresh_login": { input: SiteLoginRefreshInput, output: SiteLoginRefreshOutput },
|
||||
"site.user_data": { input: SiteUserDataInput, output: SiteUserDataOutput },
|
||||
"browser.snapshot": { input: BrowserSnapshotInput, output: BrowserSnapshotOutput },
|
||||
"browser.navigate": { input: BrowserNavigateInput, output: BrowserNavigateOutput },
|
||||
"browser.interact": { input: BrowserInteractInput, output: BrowserInteractOutput },
|
||||
"browser.download": { input: BrowserDownloadInput, output: BrowserDownloadOutput },
|
||||
} as const satisfies Record<KnownCapabilityId, { input: z.ZodType; output: z.ZodType }>;
|
||||
|
||||
type CapabilityIo = typeof CAPABILITY_IO;
|
||||
@@ -404,3 +496,11 @@ export type SiteLoginRefreshOutput = z.infer<typeof SiteLoginRefreshOutput>;
|
||||
export type SiteUserDataInput = z.infer<typeof SiteUserDataInput>;
|
||||
export type SiteUserDataOutput = z.infer<typeof SiteUserDataOutput>;
|
||||
export type SubagentBudget = z.infer<typeof SubagentBudget>;
|
||||
export type BrowserSnapshotInput = z.infer<typeof BrowserSnapshotInput>;
|
||||
export type BrowserSnapshotOutput = z.infer<typeof BrowserSnapshotOutput>;
|
||||
export type BrowserNavigateInput = z.infer<typeof BrowserNavigateInput>;
|
||||
export type BrowserNavigateOutput = z.infer<typeof BrowserNavigateOutput>;
|
||||
export type BrowserInteractInput = z.infer<typeof BrowserInteractInput>;
|
||||
export type BrowserInteractOutput = z.infer<typeof BrowserInteractOutput>;
|
||||
export type BrowserDownloadInput = z.infer<typeof BrowserDownloadInput>;
|
||||
export type BrowserDownloadOutput = z.infer<typeof BrowserDownloadOutput>;
|
||||
Reference in new issue
Block a user