feat(browser): complete camofox BrowserProvider with tabs, interaction, download capture and control lease

Contracts for browser API and browser.* agent capabilities; adapter implements
close/list/restore sessions, tabs + reconcile, history, click/type/select/scroll/press,
extract, waitForDownload (staged via DownloadSink, sha256), stale-ref guard and
automation/human actor check. testkit mock camofox gains an interactive site
(login redirect, captcha, click-driven page changes, downloads, PNG screenshots).

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed 2026-09-28 20:17:46 +08:00
1 parent 62290d4477
commit c17d4b90bd
12 files changed
+1832 -101

No files matched your search

+100
View File
@@ -1,5 +1,6 @@
import { z } from "zod";
import { MediaAsset } from "./assets.ts";
import { BrowserController, BrowserDownloadVerification, BrowserSnapshot } from "./browser.ts";
import { CAPABILITY_CATALOG, type CapabilitySpec, type KnownCapabilityId } from "./capability.ts";
import { Id, JsonObject, NonNegInt } from "./common.ts";
import { MediaEntity, MediaKind } from "./content.ts";
@@ -318,6 +319,93 @@ export const SiteUserDataOutput = z.object({
history: z.array(SiteUserDataPoint),
});
// C1-E 浏览器能力:会话按站点/账号隔离;元素引用只来自最近一次 browser.snapshot,
// 控制权交还或导航后旧引用失效;凭证用 credential 字段由服务端注入,不进入模型。
const BrowserAgentTarget = z.union([
z.object({ ref: z.string().min(1).max(40) }),
z.object({ selector: z.string().min(1).max(500) }),
]);
export const BrowserSnapshotInput = z
.object({
/** 会话 ID;缺省用 siteId 对应的未关闭会话 */
sessionId: Id.optional(),
siteId: Id.optional(),
tabId: z.string().min(1).optional(),
offset: z.number().int().min(0).optional(),
})
.refine((v) => !!v.sessionId || !!v.siteId, { message: "sessionId 与 siteId 至少给一个" });
export const BrowserSnapshotOutput = z.object({
snapshot: BrowserSnapshot,
controller: BrowserController,
/** 会话在等待处理(验证码 / 二次认证 / 登录失效)时的原因;此时自动化动作被拒,需人工在原站视图处理 */
waitingReason: z.string().nullable(),
loginRequired: z.boolean(),
});
export const BrowserNavigateInput = z
.object({
sessionId: Id.optional(),
/** 无会话时按站点(和账号)打开一个 */
siteId: Id.optional(),
accountId: Id.optional(),
tabId: z.string().min(1).optional(),
/** 必须在站点域名内 */
url: z.url().max(4000),
})
.refine((v) => !!v.sessionId || !!v.siteId, { message: "sessionId 与 siteId 至少给一个" });
export const BrowserNavigateOutput = z.object({
sessionId: Id,
url: z.string(),
httpStatus: z.number().int().nullable(),
ok: z.boolean(),
attempts: z.number().int(),
loginRequired: z.boolean(),
controller: BrowserController,
});
export const BrowserInteractInput = z.object({
sessionId: Id,
tabId: z.string().min(1).optional(),
/** request_human:转"等待处理",由人在原站视图接管(验证码、二次认证等) */
action: z.enum(["click", "type", "select", "scroll", "press", "request_human"]),
target: BrowserAgentTarget.optional(),
text: z.string().max(4000).optional(),
/** 站点账号凭证字段:服务端解析后直接下发到输入框 */
credential: z.enum(["username", "password", "totp"]).optional(),
key: z.string().min(1).max(40).optional(),
value: z.string().max(500).optional(),
direction: z.enum(["up", "down", "left", "right"]).optional(),
amount: z.number().int().min(1).max(10_000).optional(),
submit: z.boolean().optional(),
reason: z.string().max(200).optional(),
});
export const BrowserInteractOutput = z.object({
ok: z.literal(true),
controller: BrowserController,
/** true = 页面可能已变化,下一步先 browser.snapshot 再用新引用 */
needsSnapshot: z.boolean(),
});
export const BrowserDownloadInput = z.object({
sessionId: Id,
tabId: z.string().min(1).optional(),
/** 触发下载的元素(快照引用);缺省只等待已触发的下载 */
target: BrowserAgentTarget.optional(),
timeoutMs: z.number().int().min(500).max(120_000).default(15_000),
});
export const BrowserDownloadOutput = z.object({
artifactId: Id,
filename: z.string(),
sizeBytes: NonNegInt,
sha256: z.string(),
verification: BrowserDownloadVerification,
/** 匹配到的本地资源;下一步用 download.plan / download.submit 走同一提交流程 */
sourceReleaseId: Id.nullable(),
duplicate: z.boolean(),
note: z.string().nullable(),
});
/** 能力 ID → 输入/输出 schema(server 注册处理器、agent 生成工具定义时共用) */
export const CAPABILITY_IO = {
"catalog.lookup": { input: CatalogLookupInput, output: CatalogLookupOutput },
@@ -341,6 +429,10 @@ export const CAPABILITY_IO = {
"site.health_check": { input: SiteHealthCheckInput, output: SiteHealthCheckOutput },
"site.refresh_login": { input: SiteLoginRefreshInput, output: SiteLoginRefreshOutput },
"site.user_data": { input: SiteUserDataInput, output: SiteUserDataOutput },
"browser.snapshot": { input: BrowserSnapshotInput, output: BrowserSnapshotOutput },
"browser.navigate": { input: BrowserNavigateInput, output: BrowserNavigateOutput },
"browser.interact": { input: BrowserInteractInput, output: BrowserInteractOutput },
"browser.download": { input: BrowserDownloadInput, output: BrowserDownloadOutput },
} as const satisfies Record<KnownCapabilityId, { input: z.ZodType; output: z.ZodType }>;
type CapabilityIo = typeof CAPABILITY_IO;
@@ -404,3 +496,11 @@ export type SiteLoginRefreshOutput = z.infer<typeof SiteLoginRefreshOutput>;
export type SiteUserDataInput = z.infer<typeof SiteUserDataInput>;
export type SiteUserDataOutput = z.infer<typeof SiteUserDataOutput>;
export type SubagentBudget = z.infer<typeof SubagentBudget>;
export type BrowserSnapshotInput = z.infer<typeof BrowserSnapshotInput>;
export type BrowserSnapshotOutput = z.infer<typeof BrowserSnapshotOutput>;
export type BrowserNavigateInput = z.infer<typeof BrowserNavigateInput>;
export type BrowserNavigateOutput = z.infer<typeof BrowserNavigateOutput>;
export type BrowserInteractInput = z.infer<typeof BrowserInteractInput>;
export type BrowserInteractOutput = z.infer<typeof BrowserInteractOutput>;
export type BrowserDownloadInput = z.infer<typeof BrowserDownloadInput>;
export type BrowserDownloadOutput = z.infer<typeof BrowserDownloadOutput>;