diff --git a/docs/design/browser-provider.md b/docs/design/browser-provider.md new file mode 100644 index 00000000..09d336d0 --- /dev/null +++ b/docs/design/browser-provider.md @@ -0,0 +1,96 @@ +# BrowserProvider 合同与 camofox-browser 适配 + +对应 REQ v0.7 §4.4、§4.7.2–4.7.4、§12.1 浏览器实体、附录 S18–S20;计划 `docs/plan/00-first-version-plan.md` §3.2:v0.1 只落接口,浏览器接入与原站视图在 M2。 + +代码位置: + +| 层 | 路径 | +| --- | --- | +| 合同(zod) | `packages/contracts/src/browser.ts` | +| Provider 接口与实现 | `packages/adapters/src/browser/**`(`provider.ts`、`camofox/`、`none.ts`、`redact.ts`、`profile-lock.ts`、`budget.ts`) | +| mock camofox | `packages/testkit/src/browser/mock-camofox.ts` | +| 表结构 | `packages/db/migrations/0023_browser.ts` | + +## 1. Playwright 不是产品浏览器能力 + +产品内所有浏览器动作(站点排障、原站视图、Agent 受控交互、下载捕获)只通过 `BrowserProvider` 访问浏览器服务,首个实现对接用户已经运行的 **camofox-browser**。仓库里的 Playwright 只用于前端截图/视觉测试,不作为产品能力、不在服务端启动、不作为 camofox 不可用时的回退。不得把"另起一个浏览器完成了操作"标为同会话协作(REQ §4.7.2)。 + +## 2. 合同要点 + +- **BrowserProviderKind**:`camofox | none`。`none` 为默认,能力全关,动作报 `unavailable`,界面据此显示"浏览器不可用"。 +- **BrowserCapabilities**:`sessions / tabs / navigate / snapshot / screenshot / interact / extract / downloadCapture / takeover / persistentProfile`,每项是 `{ supported, verified, source, note? }`: + - `supported`:本适配器当前对该上游能提供(上游有端点但适配器尚未实现的也记 false,并注明 M2); + - `verified`:仅在真实实例原型验证通过后由配置注入为 true,探测永远不会把它设为 true; + - `source`:`static`(按文档声明)或 `probed`(从上游 `/openapi.json` 推导); + - 另有 `upstreamVersion`、`probedAt`。界面只把 `supported && verified` 视为可用(REQ §4.7.4)。 +- **BrowserSession**:`providerId, siteId, accountId, profileKey, controller, currentUrl, lastActivityAt, upstreamRef, recoveryState, closedAt, closeReason`。`controller ∈ automation | human | waiting | closed`;非 `automation` 时适配器拒绝自动化动作(`control_conflict`)。 +- **BrowserTab**:上游标签页映射(`upstreamTabId`、`url`、`status open|closed|lost`、`lastSnapshotAt`)。 +- **BrowserControlLease**:`sessionId, holderKind(automation|human), holder, reason, status, acquiredAt, expiresAt, releasedAt, releaseReason`;每会话至多一条 `active`(DB 部分唯一索引)。 +- **NavigationBudget**:`domainAllowlist[]`(含子域;空 = 全拒)、`timeoutMs`、`maxRetries`(只重试超时/网络/5xx/503)、`maxBytes`(快照截断、截图超限报 `budget_exceeded`)。导航前检查目标,导航后检查落地地址(重定向出白名单报 `domain_blocked`)。 +- **BrowserSnapshot**:可直接交给模型的结构 —— 脱敏后的可访问性文本(元素以 `eN` 引用)、脱敏 URL、表单/表格摘要、`redactions` 计数、分页 `nextOffset`。不含原始 HTML/脚本。 +- **DownloadArtifact**:`id, sessionId, filename, mimeType, sizeBytes, bytes: BlobRef{kind: blob|secret, ref}, sha256, capturedAt`。文件内容只以引用出现(种子可能含 passkey),交后端暂存核验后再走原有提交链路。 +- **BrowserErrorKind**:`not_implemented, unavailable, auth, timeout, network, not_found, rate_limited, bad_request, upstream, protocol, domain_blocked, budget_exceeded, control_conflict, lock_timeout`。 + +### 安全约束 + +- 访问凭据(camofox `CAMOFOX_ACCESS_KEY`)只以 `browser_provider.secret_ref` 保存,每次请求经 `BrowserSecretResolver` 解出;日志只记方法/路径(不含 query)/状态/耗时;错误信息不含凭据。 +- 站点登录凭据经 `TypeInput { kind: 'secret' }` 由服务端解析后直接下发到输入框,不进入模型(M2)。 +- `redactSnapshot`:遮蔽密码框值(a11y 文本、HTML、structure JSON 三种形态)、Cookie 值(`Cookie:` 行与常见站点会话 Cookie 名)、URL 凭据参数(passkey/authkey/rsskey/token/…)、路径形式 passkey 与 URL userinfo。camofox 快照中密码框的真实呈现方式未核验,规则宁可多遮。 +- 截图暂不做像素级遮蔽:登录页或含凭据地址的截图标 `modelSafe: false`,只给人看不喂模型。 +- 同一 `profileKey` 的操作串行(`ProfileLock`,当前进程内实现;多进程部署需换 PG advisory lock 或基于 `browser_control_lease` 的持久化实现)。 +- 浏览器没有媒体库写权限;人工接管不是对其他站点、宿主终端或文件系统的授权。 + +## 3. camofox-browser OpenAPI 对照 + +来源:`https://raw.githubusercontent.com/jo-inc/camofox-browser/master/openapi.json`(`info.version` 1.17.0)与同仓 README,2026-09-27 取得,**仅读文档,未连接真实实例**;用户实际部署的仓库/fork/版本尚未确认。 + +上游模型:一个 Camoufox 实例 → `userId` 对应一个隔离 BrowserContext(cookies/storage)→ `sessionKey` 为标签组 → 标签页。本项目以 `profileKey`(站点/账号)作 `userId`,以 `browser_session.id` 作 `sessionKey`。认证:设置 `CAMOFOX_ACCESS_KEY` 后,除 `/health`、cookie 导入、`/stop` 外全部路由要求 `Authorization: Bearer `;`CAMOFOX_API_KEY` 另管 cookie 导入与 traces。 + +| 合同方法 | camofox 端点 | 状态 | +| --- | --- | --- | +| `health` | `GET /health`(200 `{ok, engine, browserConnected, browserRunning, activeTabs, activeSessions, consecutiveFailures}`;503 `{ok:false, recovering}`) | 已实现(mock 验证) | +| `capabilities` | `GET /openapi.json`(按 paths 推导) | 已实现(mock 验证) | +| `openSession` | `POST /tabs {userId, sessionKey, url?}` → `{tabId, url, httpStatus, navigationOk}`;上游无显式建会话端点 | 已实现(mock 验证) | +| `navigate` | `POST /tabs/{tabId}/navigate {userId, url}` → `{ok, tabId, url, httpStatus, navigationOk}` | 已实现(mock 验证) | +| `snapshot` | `GET /tabs/{tabId}/snapshot?userId&format=text&offset` → `{url, snapshot, structure?, refsCount, truncated, hasMore, nextOffset}` | 已实现(mock 验证) | +| `screenshot` | `GET /tabs/{tabId}/screenshot?userId` → `{screenshot:{data(base64), mimeType}}`;兼容直接返回 `image/*` | 已实现(mock 验证) | +| `closeSession` | `DELETE /sessions/{userId}`(关闭该 userId 全部标签);或 `DELETE /tabs/group/{listItemId}` | M2 | +| `listSessions` | `GET /tabs?userId`(只有标签页) | M2,以 DB 为准 | +| `click` / `type` | `POST /tabs/{tabId}/click {ref|selector}`、`/type {ref|selector, text, clear, submit}`、`/select`、`/press` | M2 | +| `extract` | `POST /tabs/{tabId}/extract {schema}`(属性带 `x-ref`,需先 snapshot) | M2 | +| `waitForDownload` | `GET /tabs/{tabId}/downloads?includeData&consume&maxBytes`(README;OpenAPI 未列这些 query) | M2 | +| `acquireControl` / `releaseControl` | 无;VNC 插件(`ENABLE_VNC`、noVNC 6080)提供整块虚拟显示,不在 OpenAPI 中 | M2,原型验证 | + +### 与设计的差异 / 待核验 + +1. 上游**没有会话概念的建立端点**:会话在首个 `POST /tabs` 时隐式创建,由 `SESSION_TIMEOUT_MS`(默认 10 分钟,README 架构节写 30 分钟,自相矛盾)空闲回收;标签页 `TAB_INACTIVITY_MS` 默认 5 分钟回收,且达到上限时"最旧标签被自动回收"。本项目的会话/标签状态必须以 DB 为准并定期对账,不能假定上游会话长期存在。 +2. `userId` 允许字符集、长度未在 OpenAPI 定义(`profileKeyFor` 生成 `umt-site--acct-`),待核验。 +3. 快照是可访问性树文本(`[button e1] Submit`),**密码框的值是否出现、以什么形式出现未核验**;`format=json` 与 `structure` 的具体字段也未核验。 +4. 截图 OpenAPI 为 JSON base64;是否有版本直接返回二进制未核验(适配器两者兼容)。 +5. README 的 type 示例用 `pressEnter`,OpenAPI 用 `submit`。 +6. 接管:上游没有 API;VNC 插件是整块显示,能否定位到同一会话/标签、画面认证能否接产品认证,需原型;`CAMOFOX_INTERACTIVE=desktop` 只适用于本机桌面。 +7. 持久化:持久化插件默认保存 cookies + localStorage 到 `~/.camofox/profiles`,sessionStorage 不保存,IndexedDB 需显式开启;"重启后仍登录"要逐站验证(REQ §4.4)。 +8. 下载捕获的 `includeData` 返回 base64,大文件内存占用与 `maxBytes` 行为待验证。 + +## 4. 表结构(0023) + +- `browser_provider`:`name, kind, base_url, secret_ref, enabled, capabilities jsonb, verified_at, health_status, last_health_at, config`;`kind <> 'none'` 时必须有 `base_url`。 +- `browser_session`:`provider_id, site_id, account_id, profile_key, controller, current_url(脱敏), last_activity_at, upstream_ref, recovery_state, closed_at, close_reason`;`(provider_id, profile_key)` 在未关闭会话中唯一;`controller = 'closed'` 与 `closed_at` 一致。 +- `browser_tab`:`session_id, upstream_tab_id, url, title, status, last_snapshot_at, closed_at`;`(session_id, upstream_tab_id)` 唯一。 +- `browser_control_lease`:`session_id, holder_kind, holder, reason, status, acquired_at, expires_at, released_at, release_reason`;`session_id` 在 `status = 'active'` 中唯一,`expires_at > acquired_at`。 + +## 5. M2 原型验证清单(REQ §4.7.4) + +先固定用户现有 camofox-browser 的**仓库、版本(`/openapi.json` 的 info.version)、启动方式与认证配置**,再逐项验证,结果写入 `browser_provider.capabilities` 的 `verified` 与 `verified_at`。未通过项在界面标为不可用,不得用另一浏览器窗口代替后宣称通过。 + +| # | 项目 | 验证方法 | 通过标准 | +| --- | --- | --- | --- | +| 1 | 登录保持 | 观众/馒头/HHClub/CHD 各一账号:导入 Cookie 或经接管登录后,闲置超过 `SESSION_TIMEOUT_MS` 再访问 | 无需重新登录;失败时准确识别"登录过期"而非空列表 | +| 2 | 结构化快照 | 列表页/详情页/登录页 `snapshot` | 元素引用稳定可点;密码框值、Cookie、passkey 经 `redactSnapshot` 后不可见;大页面分页正确 | +| 3 | 截图 | 列表/详情/登录页 | 返回格式与大小;登录页 `modelSafe=false` | +| 4 | 列表/详情提取 | 快照 + `extract`(x-ref)对比现有 HTTP 适配器结果 | 标题/分类/副标题/促销/做种字段一致,原始字段不丢 | +| 5 | 下载捕获 | 详情页点击下载 .torrent | `downloads` 能拿到文件与文件名;sha256 与直接 HTTP 下载一致;进入后端暂存核验,不直写媒体库 | +| 6 | 重启后会话 | 重启 camofox 容器/浏览器(`BROWSER_IDLE_TIMEOUT_MS` 触发与强制重启各一次) | 持久化 profile 恢复登录;旧 tabId 失效被识别为 `lost`,不重放旧元素引用 | +| 7 | 人机同标签页连续操作 | Agent 打开详情页 → 转人工(VNC)在**同一标签页**操作 → 交还 Agent 重新 snapshot | 人工看到的是同一 Camoufox 会话与页面;交还后 Agent 读到人工操作后的状态;不重复提交下载 | +| 8 | 异常路径 | 登录过期、页面结构变化、需人工验证(验证码/2FA)、服务不可用(503/断连) | 分别落为 `auth` / 提取失败标过期 / `waiting` 待处理 / `unavailable`,均不无限重试 | +| 9 | 性能与资源 | 四站并发 1/2/4 会话 | 记录启动延迟、实时查询延迟、内存、失败恢复时间,作为默认并发与空闲释放参数依据 | diff --git a/packages/adapters/src/browser/README.md b/packages/adapters/src/browser/README.md new file mode 100644 index 00000000..bd143962 --- /dev/null +++ b/packages/adapters/src/browser/README.md @@ -0,0 +1,26 @@ +# browser + +`BrowserProvider` 合同(`provider.ts`)及其实现:业务侧访问浏览器服务只走这套接口。产品浏览器能力对接用户已有的 camofox-browser;**Playwright 只用于前端截图测试,不是产品浏览器能力**。设计说明见 `docs/design/browser-provider.md`。 + +- `camofox/`:camofox-browser 适配器。按 jo-inc/camofox-browser master `openapi.json`(1.17.0)与 README 实现,`Authorization: Bearer `,凭据每次请求经 `BrowserSecretResolver` 解出,不缓存、不入日志。camofox `userId` = 本项目 `profileKey`(站点/账号隔离),`sessionKey` = `browser_session.id`。 +- `none.ts`:默认实现(未配置浏览器),能力全关,动作报 `unavailable`。 +- `redact.ts`:`redactSnapshot` / `redactBrowserUrl`,快照与地址进入模型前遮蔽密码框值、Cookie 值、含 passkey/token 的 URL。 +- `profile-lock.ts`:同一 profile 串行锁(进程内内存实现;多进程部署换持久化实现,接口不变)。 +- `budget.ts`:导航域名白名单(含子域),非 http(s) 拒绝。 + +## v0.1 已实现 + +`health`(`/health`,免认证,503 → degraded)、`capabilities`(静态声明;`probe: true` 读上游 `/openapi.json` 推导,缺端点即降级)、`openSession`(`POST /tabs`)、`navigate`(`POST /tabs/{id}/navigate`,预算内重试 5xx/超时/网络错误)、`snapshot`(`GET /tabs/{id}/snapshot?format=text`,字节预算截断 + 脱敏)、`screenshot`(`GET /tabs/{id}/screenshot`,JSON base64 或二进制图片;登录页/含凭据地址标 `modelSafe: false`)。 + +## M2(当前抛 `BrowserError('not_implemented')`) + +| 方法 | 上游候选端点 | 待办 | +| --- | --- | --- | +| `closeSession` | `DELETE /sessions/{userId}` | 与 browser_session 持久化、租约释放一起落地 | +| `listSessions` | `GET /tabs?userId=` | 上游只列标签页,会话清单以 DB 为准,需对账 | +| `click` / `type` | `POST /tabs/{id}/click` / `type` / `select` | 接凭证代理(`TypeInput.kind = 'secret'`);README 示例 `pressEnter` 与 OpenAPI `submit` 字段名不一致,需核验 | +| `extract` | `POST /tabs/{id}/extract`(JSON Schema + `x-ref`) | 需先 snapshot 建 ref 表 | +| `waitForDownload` | `GET /tabs/{id}/downloads?includeData&consume&maxBytes` | 产物落 blob 暂存 + sha256,交后端核验 | +| `acquireControl` / `releaseControl` | 无(上游无接管 API) | 服务层以 `browser_control_lease` 持久化;画面/输入通道待 VNC 原型 | + +仅对 mock 与上游文档验证,未连接真实实例。 diff --git a/packages/adapters/src/browser/browser.test.ts b/packages/adapters/src/browser/browser.test.ts new file mode 100644 index 00000000..b05aee90 --- /dev/null +++ b/packages/adapters/src/browser/browser.test.ts @@ -0,0 +1,348 @@ +import type { NavigationBudget } from "@umt/contracts"; +import { + CAMOFOX_FIXTURE_COOKIE, + CAMOFOX_FIXTURE_PASSKEY, + CAMOFOX_FIXTURE_PASSWORD, + createMockCamofox, + type MockCamofox, +} from "@umt/testkit"; +import { afterEach, describe, expect, it } from "vitest"; +import { + BrowserError, + type BrowserLogger, + type CamofoxConfig, + camofoxStaticCapabilities, + classifyHttpStatus, + createCamofoxBrowserProvider, + createNoneBrowserProvider, + hostAllowed, + InMemoryProfileLock, + profileKeyFor, + redactBrowserUrl, + redactSnapshot, + staticBrowserSecrets, +} from "./index.ts"; + +const ACCESS_KEY = "camofox-access-key-fixture"; +const PROVIDER = "11111111-1111-4111-8111-111111111111"; +const SITE = "22222222-2222-4222-8222-222222222222"; +const ACCOUNT = "33333333-3333-4333-8333-333333333333"; + +const budget = (over: Partial = {}): NavigationBudget => ({ + domainAllowlist: ["audiences.example"], + timeoutMs: 500, + maxRetries: 1, + maxBytes: 1_000_000, + ...over, +}); + +let mocks: MockCamofox[] = []; +afterEach(async () => { + await Promise.all(mocks.map((m) => m.close())); + mocks = []; +}); + +async function setup( + opts: { accessKey?: string; key?: string | null; omitPaths?: string[] } = {}, + extra: Partial = {}, +) { + const mock = await createMockCamofox({ + ...(opts.accessKey ? { accessKey: opts.accessKey } : {}), + ...(opts.omitPaths ? { omitPaths: opts.omitPaths } : {}), + }); + mocks.push(mock); + const logs: Parameters[0][] = []; + const key = opts.key === undefined ? opts.accessKey : opts.key; + const provider = createCamofoxBrowserProvider({ + providerId: PROVIDER, + baseUrl: `${mock.url}/`, + secrets: staticBrowserSecrets(key ? { [`provider:${PROVIDER}`]: key } : {}), + timeoutMs: 500, + logger: (e) => logs.push(e), + ...extra, + }); + return { mock, provider, logs }; +} + +const open = (p: Awaited>["provider"], url?: string) => + p.openSession({ + providerId: PROVIDER, + siteId: SITE, + accountId: ACCOUNT, + budget: budget(), + ...(url ? { initialUrl: url } : {}), + }); + +async function rejectKind(p: Promise): Promise { + try { + await p; + } catch (e) { + expect(e).toBeInstanceOf(BrowserError); + return (e as BrowserError).kind; + } + throw new Error("预期抛错"); +} + +describe("能力声明", () => { + it("静态声明:已实现能力 supported、M2 能力 false 并注明,默认全部未验证", () => { + const c = camofoxStaticCapabilities({ snapshot: true, interact: true }); + expect(c.navigate).toMatchObject({ supported: true, verified: false, source: "static" }); + expect(c.snapshot).toMatchObject({ supported: true, verified: true }); + // 未实现的能力即使配置了 verified 也不会被标为已验证 + expect(c.interact).toMatchObject({ supported: false, verified: false }); + expect(c.interact.note).toContain("M2"); + expect(c.takeover.supported).toBe(false); + expect(c.upstreamVersion).toBeNull(); + }); + + it("探测 /openapi.json:读取上游版本,缺端点的能力降级", async () => { + const { provider } = await setup({ omitPaths: ["get /tabs/{tabId}/snapshot"] }); + const c = await provider.capabilities({ probe: true }); + expect(c.upstreamVersion).toBe("1.17.0"); + expect(c.probedAt).not.toBeNull(); + expect(c.navigate).toMatchObject({ supported: true, source: "probed", verified: false }); + expect(c.snapshot).toMatchObject({ supported: false, source: "probed" }); + expect(c.snapshot.note).toContain("/tabs/{tabId}/snapshot"); + }); + + it("none 实现:能力全关,动作报 unavailable", async () => { + const none = createNoneBrowserProvider(); + const c = await none.capabilities(); + expect(Object.values(c).filter((v) => v && typeof v === "object" && "supported" in v && v.supported)).toEqual([]); + expect((await none.health()).status).toBe("disabled"); + expect(await rejectKind(none.navigate({ sessionId: "x" }, "https://a.example", budget()))).toBe("unavailable"); + }); +}); + +describe("认证与健康", () => { + it("每个请求带 Bearer 访问凭据;凭据不进日志与错误信息", async () => { + const { mock, provider, logs } = await setup({ accessKey: ACCESS_KEY }); + await open(provider, "https://audiences.example/torrents.php"); + const call = mock.callsTo("/tabs")[0]!; + expect(call.headers.authorization).toBe(`Bearer ${ACCESS_KEY}`); + expect(JSON.stringify(logs)).not.toContain(ACCESS_KEY); + expect(logs.every((l) => !l.path?.includes("?"))).toBe(true); + }); + + it("凭据缺失/错误 → auth,不重试", async () => { + const { mock, provider } = await setup({ accessKey: ACCESS_KEY, key: "wrong-key" }); + try { + await open(provider); + throw new Error("预期抛错"); + } catch (e) { + expect(e).toMatchObject({ kind: "auth", httpStatus: 401 }); + expect(String((e as Error).message)).not.toContain("wrong-key"); + } + expect(mock.callsTo("/tabs")).toHaveLength(1); + }); + + it("health:/health 免认证;503 → degraded;连不上 → down", async () => { + const { mock, provider } = await setup({ accessKey: ACCESS_KEY, key: null }); + const ok = await provider.health(); + expect(ok.status).toBe("ok"); + expect(ok.detail).toMatchObject({ engine: "camoufox", browserConnected: true }); + mock.fault("/health", { status: 503, body: { ok: false, recovering: true } }); + expect((await provider.health()).status).toBe("degraded"); + await mock.close(); + mocks = []; + expect((await provider.health()).status).toBe("down"); + }); +}); + +describe("会话、导航与快照", () => { + it("openSession 以 profileKey 作 camofox userId,会话 id 作 sessionKey", async () => { + const { mock, provider } = await setup(); + const r = await open(provider, "https://audiences.example/torrents.php"); + expect(r.session.profileKey).toBe(profileKeyFor(SITE, ACCOUNT)); + expect(r.session.controller).toBe("automation"); + expect(r.tab.upstreamTabId).toBe("tab-1"); + expect(r.navigation).toMatchObject({ httpStatus: 200, ok: true }); + expect([...mock.tabs.values()][0]).toMatchObject({ userId: r.session.profileKey, sessionKey: r.session.id }); + // 同一 profile 不能并存两个未关闭会话 + expect(await rejectKind(open(provider))).toBe("control_conflict"); + }); + + it("快照进入模型前脱敏:密码值、Cookie 值、passkey URL 均被遮蔽", async () => { + const { provider } = await setup(); + const { session } = await open(provider, "https://audiences.example/torrents.php"); + await provider.navigate( + { sessionId: session.id }, + `https://audiences.example/details.php?id=42&passkey=${CAMOFOX_FIXTURE_PASSKEY}`, + budget(), + ); + const snap = await provider.snapshot({ sessionId: session.id }); + const dump = JSON.stringify(snap); + for (const secret of [CAMOFOX_FIXTURE_PASSWORD, CAMOFOX_FIXTURE_COOKIE, CAMOFOX_FIXTURE_PASSKEY]) + expect(dump).not.toContain(secret); + expect(snap.text).toContain("奥本海默"); + expect(snap.text).toContain("[button e6] 登录"); + expect(snap.url).toBe("https://audiences.example/details.php?id=42&passkey=[REDACTED]"); + const kinds = Object.fromEntries(snap.redactions.map((r) => [r.kind, r.count])); + expect(kinds.password_field).toBeGreaterThanOrEqual(2); + expect(kinds.cookie).toBeGreaterThanOrEqual(1); + expect(kinds.credential_url).toBeGreaterThanOrEqual(2); + expect(snap.refsCount).toBe(7); + }); + + it("redactSnapshot:HTML 密码框、URL userinfo、路径 passkey、YAML 形式;普通文本不动", () => { + const r = redactSnapshot({ + text: [ + '', + "https://user:pw123@tracker.example/announce/passkey/abcdef0123456789", + '- textbox "Password": yaml-secret', + "做种 128 · 大小 58.2 GB", + ].join("\n"), + url: "https://audiences.example/rss.php?rsskey=deadbeefdeadbeef&cat=401", + }); + expect(r.text).not.toMatch(/s3cret-html|pw123|abcdef0123456789|yaml-secret/); + expect(r.text).toContain("做种 128 · 大小 58.2 GB"); + expect(r.url).toBe("https://audiences.example/rss.php?rsskey=[REDACTED]&cat=401"); + expect(redactBrowserUrl("https://a.example/x?id=1")).toBe("https://a.example/x?id=1"); + // 幂等:已脱敏的文本再过一遍不变 + expect(redactSnapshot({ text: r.text, url: r.url }).text).toBe(r.text); + }); + + it("导航预算:白名单外直接拒绝不发请求;重定向出白名单报 domain_blocked", async () => { + const { mock, provider } = await setup(); + const { session } = await open(provider); + expect(await rejectKind(provider.navigate({ sessionId: session.id }, "https://evil.example/", budget()))).toBe( + "domain_blocked", + ); + expect(await rejectKind(provider.navigate({ sessionId: session.id }, "file:///etc/passwd", budget()))).toBe( + "domain_blocked", + ); + expect(mock.callsTo(/navigate$/)).toHaveLength(0); + const redirect = `https://audiences.example/jump?redirect=${encodeURIComponent("https://other.example/login")}`; + expect(await rejectKind(provider.navigate({ sessionId: session.id }, redirect, budget()))).toBe("domain_blocked"); + expect(hostAllowed("cdn.audiences.example", ["audiences.example"])).toBe(true); + expect(hostAllowed("audiences.example.evil", ["audiences.example"])).toBe(false); + expect(hostAllowed("a.example", [])).toBe(false); + }); + + it("截图:解码 1x1 PNG;登录页不可进模型;超字节预算报 budget_exceeded", async () => { + const { provider } = await setup(); + const { session } = await open(provider, "https://audiences.example/torrents.php"); + const shot = await provider.screenshot({ sessionId: session.id }); + expect(shot.mimeType).toBe("image/png"); + expect([...shot.data.slice(1, 4)].map((c) => String.fromCharCode(c)).join("")).toBe("PNG"); + expect(shot.modelSafe).toBe(true); + await provider.navigate({ sessionId: session.id }, "https://audiences.example/login.php", budget()); + expect((await provider.screenshot({ sessionId: session.id })).modelSafe).toBe(false); + expect(await rejectKind(provider.screenshot({ sessionId: session.id }, { maxBytes: 10 }))).toBe("budget_exceeded"); + }); + + it("人工接管期间拒绝自动化导航", async () => { + const { provider } = await setup(); + const { session } = await open(provider); + provider.setController(session.id, "human"); + expect(await rejectKind(provider.navigate({ sessionId: session.id }, "https://audiences.example/", budget()))).toBe( + "control_conflict", + ); + }); +}); + +describe("串行锁", () => { + it("同一 profile 严格串行、不同 profile 并行;等待超时报 lock_timeout", async () => { + const lock = new InMemoryProfileLock(); + const order: string[] = []; + const task = (key: string, name: string, ms: number) => + lock.run(key, async () => { + order.push(`${name}:start`); + await new Promise((r) => setTimeout(r, ms)); + order.push(`${name}:end`); + }); + await Promise.all([task("p1", "a", 20), task("p1", "b", 1), task("p2", "c", 1)]); + expect(order.indexOf("a:end")).toBeLessThan(order.indexOf("b:start")); + expect(order.indexOf("c:start")).toBeLessThan(order.indexOf("a:end")); + expect(lock.size()).toBe(0); + + const slow = lock.run("p1", () => new Promise((r) => setTimeout(r, 60))); + try { + await lock.run("p1", async () => "never", { waitMs: 10 }); + throw new Error("预期超时"); + } catch (e) { + expect(e).toMatchObject({ kind: "lock_timeout" }); + } + await slow; + expect(lock.size()).toBe(0); + }); + + it("适配器对同一会话的并发请求逐个发往上游", async () => { + const { mock, provider } = await setup(); + const { session } = await open(provider, "https://audiences.example/torrents.php"); + mock.fault(/snapshot$/, { delayMs: 40 }); + const started = Date.now(); + await Promise.all([provider.snapshot({ sessionId: session.id }), provider.screenshot({ sessionId: session.id })]); + const shotCall = mock.callsTo(/screenshot$/)[0]!; + expect(shotCall.status).toBe(200); + expect(Date.now() - started).toBeGreaterThanOrEqual(40); + }); +}); + +describe("M2 未实现分支", () => { + it("click/type/extract/waitForDownload/接管/关闭会话均抛 not_implemented", async () => { + const { provider } = await setup(); + const ref = { sessionId: "x" }; + const calls: Promise[] = [ + provider.click(ref, { ref: "e1" }), + provider.type(ref, { ref: "e1" }, { kind: "text", text: "x" }), + provider.extract(ref, {}), + provider.waitForDownload(ref, { timeoutMs: 10, maxBytes: 10 }), + provider.acquireControl({ + sessionId: "x", + holderKind: "human", + holder: "admin", + reason: "登录验证", + ttlMs: 1000, + }), + provider.releaseControl("lease"), + provider.closeSession("x"), + provider.listSessions(), + ]; + for (const c of calls) expect(await rejectKind(c)).toBe("not_implemented"); + }); +}); + +describe("错误分类", () => { + it("HTTP 状态映射", () => { + expect(classifyHttpStatus(401)).toBe("auth"); + expect(classifyHttpStatus(403)).toBe("auth"); + expect(classifyHttpStatus(404)).toBe("not_found"); + expect(classifyHttpStatus(409)).toBe("bad_request"); + expect(classifyHttpStatus(429)).toBe("rate_limited"); + expect(classifyHttpStatus(500)).toBe("upstream"); + expect(classifyHttpStatus(503)).toBe("unavailable"); + expect(new BrowserError("upstream").retryable).toBe(true); + expect(new BrowserError("auth").retryable).toBe(false); + }); + + it("上游 5xx 在 maxRetries 内重试;超时分类为 timeout;429 不重试", async () => { + const { mock, provider } = await setup(); + const { session } = await open(provider); + const ref = { sessionId: session.id }; + mock.fault(/navigate$/, { status: 502, times: 1 }); + const r = await provider.navigate(ref, "https://audiences.example/torrents.php", budget({ maxRetries: 1 })); + expect(r.attempts).toBe(2); + + mock.fault(/navigate$/, { delayMs: 200, times: 5 }); + expect( + await rejectKind(provider.navigate(ref, "https://audiences.example/", budget({ timeoutMs: 50, maxRetries: 0 }))), + ).toBe("timeout"); + mock.clearFaults(); + + mock.fault(/navigate$/, { status: 429, times: 5 }); + const before = mock.callsTo(/navigate$/).length; + expect(await rejectKind(provider.navigate(ref, "https://audiences.example/", budget({ maxRetries: 3 })))).toBe( + "rate_limited", + ); + expect(mock.callsTo(/navigate$/).length - before).toBe(1); + }); + + it("未知会话 → not_found;上游返回非 JSON → protocol", async () => { + const { mock, provider } = await setup(); + expect(await rejectKind(provider.snapshot({ sessionId: "nope" }))).toBe("not_found"); + const { session } = await open(provider); + mock.fault(/snapshot$/, { status: 200, body: undefined }); + expect(await rejectKind(provider.snapshot({ sessionId: session.id }))).toBe("protocol"); + }); +}); diff --git a/packages/adapters/src/browser/budget.ts b/packages/adapters/src/browser/budget.ts new file mode 100644 index 00000000..b826ec26 --- /dev/null +++ b/packages/adapters/src/browser/budget.ts @@ -0,0 +1,27 @@ +import type { NavigationBudget } from "@umt/contracts"; +import { BrowserError } from "./errors.ts"; +import { redactBrowserUrl } from "./redact.ts"; + +/** 主机是否在白名单内:精确匹配或为白名单项的子域。空白名单 = 全部拒绝。 */ +export function hostAllowed(host: string, allowlist: readonly string[]): boolean { + const h = host.toLowerCase().replace(/\.$/, ""); + return allowlist.some((raw) => { + const d = raw.toLowerCase().replace(/^\*\./, "").replace(/\.$/, ""); + return d !== "" && (h === d || h.endsWith(`.${d}`)); + }); +} + +/** 导航前检查:只允许 http(s) 且主机在白名单内;违规抛 domain_blocked(信息里只含脱敏 URL) */ +export function assertNavigable(url: string, budget: NavigationBudget): URL { + let u: URL; + try { + u = new URL(url); + } catch { + throw new BrowserError("bad_request", "非法 URL"); + } + if (u.protocol !== "http:" && u.protocol !== "https:") + throw new BrowserError("domain_blocked", `不允许的协议 ${u.protocol}`); + if (!hostAllowed(u.hostname, budget.domainAllowlist)) + throw new BrowserError("domain_blocked", `域名不在白名单:${redactBrowserUrl(u.origin)}`); + return u; +} diff --git a/packages/adapters/src/browser/camofox/capabilities.ts b/packages/adapters/src/browser/camofox/capabilities.ts new file mode 100644 index 00000000..ed6cb187 --- /dev/null +++ b/packages/adapters/src/browser/camofox/capabilities.ts @@ -0,0 +1,95 @@ +import type { BrowserCapabilities, BrowserCapabilityFlag, BrowserCapabilityKey } from "@umt/contracts"; + +/** + * camofox-browser 能力声明。依据:jo-inc/camofox-browser master 的 openapi.json(info.version 1.17.0)与 README, + * 2026-09-27 取得;用户实际部署可能是其他 fork/版本,运行时以 /openapi.json 探测结果为准(REQ §4.7.4)。 + * + * supported = 本适配器当前能对该上游提供的能力(上游有端点但适配器尚未实现的记 false 并注明 M2); + * verified 只能由真实实例原型验证结果置 true(配置注入),探测永远不会把它设为 true。 + */ + +/** 每项能力依赖的上游端点(OpenAPI paths 键,小写方法) */ +export const CAMOFOX_ENDPOINTS: Record = { + sessions: ["post /tabs", "delete /sessions/{userId}"], + tabs: ["post /tabs", "get /tabs", "delete /tabs/{tabId}"], + navigate: ["post /tabs/{tabId}/navigate"], + snapshot: ["get /tabs/{tabId}/snapshot"], + screenshot: ["get /tabs/{tabId}/screenshot"], + interact: ["post /tabs/{tabId}/click", "post /tabs/{tabId}/type", "post /tabs/{tabId}/select"], + extract: ["post /tabs/{tabId}/extract"], + downloadCapture: ["get /tabs/{tabId}/downloads"], + // 上游 OpenAPI 无接管端点;VNC 插件(ENABLE_VNC,noVNC 独立端口)是整块虚拟显示,需原型验证 + takeover: [], + // 持久化插件默认开启,无需端点;storage_state 端点属插件,不在 OpenAPI 中 + persistentProfile: [], +}; + +/** 适配器已实现的能力;其余上游虽有端点,适配器在 M2 实现 */ +const ADAPTER_IMPLEMENTED: ReadonlySet = new Set([ + "sessions", + "tabs", + "navigate", + "snapshot", + "screenshot", + "persistentProfile", +]); + +const STATIC_NOTES: Partial> = { + sessions: "上游无显式建会话端点:userId 即会话(BrowserContext),首个 POST /tabs 隐式创建;关闭会话适配器 M2", + tabs: "仅建首个标签页;列举/关闭适配器 M2", + snapshot: "可访问性树文本 + eN 元素引用,支持 offset 分页", + interact: "上游有 click/type/select;适配器 M2", + extract: "上游 POST /extract 需先 snapshot 建 ref 表(x-ref);适配器 M2", + downloadCapture: "上游 GET /downloads 支持 includeData/consume/maxBytes;适配器 M2", + takeover: "上游无接管 API;VNC 插件画面/输入与同会话连续操作待原型验证", + persistentProfile: "上游持久化插件默认保存 cookies+localStorage(~/.camofox/profiles),重启恢复待验证", +}; + +export type VerifiedMap = Partial>; + +const KEYS = Object.keys(CAMOFOX_ENDPOINTS) as BrowserCapabilityKey[]; + +function flag( + key: BrowserCapabilityKey, + supported: boolean, + source: "static" | "probed", + verified: VerifiedMap, + note?: string, +): BrowserCapabilityFlag { + const n = note ?? STATIC_NOTES[key]; + return { supported, verified: supported && verified[key] === true, source, ...(n ? { note: n } : {}) }; +} + +export function camofoxStaticCapabilities(verified: VerifiedMap = {}): BrowserCapabilities { + const flags = Object.fromEntries( + KEYS.map((k) => [k, flag(k, ADAPTER_IMPLEMENTED.has(k), "static", verified)]), + ) as Record; + return { ...flags, upstreamVersion: null, probedAt: null }; +} + +/** 从上游 openapi.json 文档推导能力:缺端点的能力降为 false 并注明 */ +export function camofoxCapabilitiesFromOpenApi( + doc: unknown, + verified: VerifiedMap = {}, + now = new Date(), +): BrowserCapabilities { + const d = (doc ?? {}) as { info?: { version?: unknown }; paths?: Record> }; + const have = new Set(); + for (const [p, ops] of Object.entries(d.paths ?? {})) + for (const m of Object.keys(ops ?? {})) have.add(`${m.toLowerCase()} ${p}`); + // 路径参数名不同的 fork 也视为同一端点 + const norm = (s: string) => s.replace(/\{[^}]+\}/g, "{}"); + const haveNorm = new Set([...have].map(norm)); + const flags = Object.fromEntries( + KEYS.map((k) => { + const missing = CAMOFOX_ENDPOINTS[k].filter((e) => !haveNorm.has(norm(e))); + if (missing.length > 0) return [k, flag(k, false, "probed", verified, `上游缺少端点:${missing.join(", ")}`)]; + return [k, flag(k, ADAPTER_IMPLEMENTED.has(k), CAMOFOX_ENDPOINTS[k].length > 0 ? "probed" : "static", verified)]; + }), + ) as Record; + return { + ...flags, + upstreamVersion: typeof d.info?.version === "string" ? d.info.version : null, + probedAt: now.toISOString(), + }; +} diff --git a/packages/adapters/src/browser/camofox/client.ts b/packages/adapters/src/browser/camofox/client.ts new file mode 100644 index 00000000..65b6b17a --- /dev/null +++ b/packages/adapters/src/browser/camofox/client.ts @@ -0,0 +1,509 @@ +/** + * camofox-browser 适配器(BrowserProvider 首个实现)。 + * 对照:jo-inc/camofox-browser master openapi.json(info.version 1.17.0)与 README,仅读文档,未连接真实实例。 + * + * 上游模型与本项目合同的映射: + * - camofox userId = 一个隔离 BrowserContext(cookies/storage),本项目用 profileKey(站点/账号)充当; + * - camofox sessionKey = 标签组,本项目用 browser_session.id; + * - 上游没有"建会话"端点,首个 POST /tabs 隐式创建;会话由本适配器在内存登记,持久化由服务层写 browser_session。 + * 认证:README/OpenAPI 为 `Authorization: Bearer `(设置后除 /health 外全部路由要求); + * 凭据每次请求经 BrowserSecretResolver 解出,不缓存、不入日志、不进错误信息。 + * + * v0.1 实现:health、capabilities(静态 + /openapi.json 探测)、openSession、navigate、snapshot、screenshot; + * 其余方法抛 BrowserError('not_implemented'),见 README 的 M2 清单。 + */ +import type { + BrowserCapabilities, + BrowserControlLease, + BrowserHealth, + BrowserSession, + BrowserSnapshot, + BrowserTab, + DownloadArtifact, + NavigationBudget, +} from "@umt/contracts"; +import { assertNavigable, hostAllowed } from "../budget.ts"; +import { BrowserError, classifyHttpStatus, notImplemented } from "../errors.ts"; +import { InMemoryProfileLock, type ProfileLock } from "../profile-lock.ts"; +import { + type AcquireControlRequest, + type BrowserLogger, + type BrowserProvider, + type BrowserScreenshot, + type BrowserSecretResolver, + type ElementTarget, + type NavigationResult, + type OpenSessionRequest, + type OpenSessionResult, + profileKeyFor, + type SnapshotOptions, + type TabRef, + type TypeInput, +} from "../provider.ts"; +import { redactBrowserUrl, redactSnapshot, urlHasCredential } from "../redact.ts"; +import { camofoxCapabilitiesFromOpenApi, camofoxStaticCapabilities, type VerifiedMap } from "./capabilities.ts"; + +export interface CamofoxConfig { + /** browser_provider.id */ + providerId: string; + /** 例 http://camofox:9377 */ + baseUrl: string; + secrets: BrowserSecretResolver; + /** 单次请求默认超时;导航用 budget.timeoutMs */ + timeoutMs?: number; + /** 同一 profile 串行锁;缺省进程内内存锁 */ + lock?: ProfileLock; + lockWaitMs?: number; + /** 已对真实实例验证通过的能力(来自原型验证记录);缺省全部未验证 */ + verified?: VerifiedMap; + /** 截图/快照默认字节上限(未传 budget 时) */ + defaultMaxBytes?: number; + fetch?: typeof fetch; + logger?: BrowserLogger; + now?: () => Date; + newId?: () => string; +} + +interface SessionState { + session: BrowserSession; + tabs: Map; + currentTabId: string; + /** 未脱敏地址只留在内存,用于判断截图能否进入模型;不对外返回 */ + rawUrl: string | null; + budget: NavigationBudget; +} + +type Json = Record; +const str = (v: unknown) => (typeof v === "string" ? v : undefined); +const int = (v: unknown) => (typeof v === "number" && Number.isInteger(v) ? v : undefined); + +const LOGIN_PATH = /(?:^|\/)(?:login|takelogin|signin|sign_in|auth)(?:\.php)?(?:\/|$|\?)/i; + +export class CamofoxBrowserProvider implements BrowserProvider { + readonly kind = "camofox" as const; + private readonly base: string; + private readonly fetchImpl: typeof fetch; + private readonly lock: ProfileLock; + private readonly sessions = new Map(); + private readonly now: () => Date; + private readonly newId: () => string; + + constructor(private readonly cfg: CamofoxConfig) { + this.base = cfg.baseUrl.replace(/\/+$/, ""); + this.fetchImpl = cfg.fetch ?? fetch; + this.lock = cfg.lock ?? new InMemoryProfileLock(cfg.lockWaitMs); + this.now = cfg.now ?? (() => new Date()); + this.newId = cfg.newId ?? (() => crypto.randomUUID()); + } + + // ---------- HTTP ---------- + + private async authHeader(): Promise> { + const key = await this.cfg.secrets.resolve({ kind: "provider_access_key", providerId: this.cfg.providerId }); + return key ? { authorization: `Bearer ${key}` } : {}; + } + + private async request( + method: "GET" | "POST" | "DELETE", + path: string, + opts: { query?: Record; body?: Json; timeoutMs?: number; auth?: boolean } = {}, + ): Promise { + const u = new URL(this.base + path); + for (const [k, v] of Object.entries(opts.query ?? {})) if (v !== undefined) u.searchParams.set(k, v); + const headers: Record = { + accept: "application/json", + ...(opts.auth === false ? {} : await this.authHeader()), + }; + if (opts.body) headers["content-type"] = "application/json"; + const timeoutMs = opts.timeoutMs ?? this.cfg.timeoutMs ?? 30_000; + const started = Date.now(); + let res: Response; + try { + res = await this.fetchImpl(u, { + method, + headers, + ...(opts.body ? { body: JSON.stringify(opts.body) } : {}), + signal: AbortSignal.timeout(timeoutMs), + }); + } catch (e) { + const name = (e as { name?: string })?.name; + const kind = name === "TimeoutError" || name === "AbortError" ? "timeout" : "network"; + this.cfg.logger?.({ level: "warn", msg: `camofox ${kind}`, method, path, ms: Date.now() - started }); + throw new BrowserError( + kind, + `camofox ${method} ${path} ${kind === "timeout" ? `超时(${timeoutMs}ms)` : "网络错误"}`, + ); + } + this.cfg.logger?.({ level: "debug", msg: "camofox", method, path, status: res.status, ms: Date.now() - started }); + if (!res.ok) { + await res.body?.cancel(); + throw new BrowserError(classifyHttpStatus(res.status), `camofox ${method} ${path} → HTTP ${res.status}`, { + httpStatus: res.status, + }); + } + return res; + } + + private async json(res: Response, path: string): Promise { + try { + const v = (await res.json()) as unknown; + if (v && typeof v === "object" && !Array.isArray(v)) return v as Json; + } catch { + /* fallthrough */ + } + throw new BrowserError("protocol", `camofox ${path} 返回非 JSON 对象`); + } + + // ---------- 状态 ---------- + + private state(sessionId: string): SessionState { + const s = this.sessions.get(sessionId); + if (!s) throw new BrowserError("not_found", "会话不存在或已关闭"); + return s; + } + + private tabOf(s: SessionState, ref: TabRef): BrowserTab { + const t = s.tabs.get(ref.tabId ?? s.currentTabId); + if (t?.status !== "open") throw new BrowserError("not_found", "标签页不存在或已关闭"); + return t; + } + + /** 接管/等待/关闭中的会话拒绝自动化动作(REQ §4.7.3) */ + private assertAutomation(s: SessionState): void { + if (s.session.controller !== "automation") + throw new BrowserError("control_conflict", `会话当前由 ${s.session.controller} 控制,自动化动作已停止`); + } + + private touch(s: SessionState, rawUrl?: string | null): void { + const at = this.now().toISOString(); + if (rawUrl !== undefined) { + s.rawUrl = rawUrl; + s.session.currentUrl = rawUrl === null ? null : redactBrowserUrl(rawUrl); + } + s.session.lastActivityAt = at; + s.session.updatedAt = at; + } + + private withProfile(s: SessionState | string, fn: () => Promise): Promise { + const key = typeof s === "string" ? s : s.session.profileKey; + return this.lock.run(key, fn, this.cfg.lockWaitMs === undefined ? {} : { waitMs: this.cfg.lockWaitMs }); + } + + /** 服务层把会话控制权变更(接管/交还/等待)同步给适配器;租约本身由服务层持久化 */ + setController(sessionId: string, controller: BrowserSession["controller"]): void { + const s = this.state(sessionId); + s.session.controller = controller; + this.touch(s); + } + + // ---------- 健康与能力 ---------- + + async health(): Promise { + const checkedAt = this.now().toISOString(); + const u = `${this.base}/health`; + try { + // /health 在 CAMOFOX_ACCESS_KEY 下也免认证(README);503 表示恢复中 + const res = await this.fetchImpl(u, { signal: AbortSignal.timeout(this.cfg.timeoutMs ?? 10_000) }); + const body = (await res.json().catch(() => ({}))) as Json; + const detail: Json = { + httpStatus: res.status, + engine: str(body.engine) ?? null, + browserConnected: body.browserConnected ?? null, + browserRunning: body.browserRunning ?? null, + activeTabs: int(body.activeTabs) ?? null, + activeSessions: int(body.activeSessions) ?? null, + consecutiveFailures: int(body.consecutiveFailures) ?? null, + recovering: body.recovering ?? null, + }; + if (res.status === 503) return { status: "degraded", kind: "camofox", detail, checkedAt }; + if (!res.ok || body.ok === false) return { status: "down", kind: "camofox", detail, checkedAt }; + return { status: "ok", kind: "camofox", detail, checkedAt }; + } catch (e) { + const name = (e as { name?: string })?.name; + return { + status: "down", + kind: "camofox", + detail: { error: name === "TimeoutError" ? "timeout" : "network" }, + checkedAt, + }; + } + } + + /** probe=true 时读取上游 /openapi.json 推导;失败回退静态声明(note 标注探测失败) */ + async capabilities(opts: { probe?: boolean } = {}): Promise { + const verified = this.cfg.verified ?? {}; + if (!opts.probe) return camofoxStaticCapabilities(verified); + try { + const res = await this.request("GET", "/openapi.json"); + return camofoxCapabilitiesFromOpenApi(await this.json(res, "/openapi.json"), verified, this.now()); + } catch (e) { + if (e instanceof BrowserError && e.kind === "auth") throw e; + return camofoxStaticCapabilities(verified); + } + } + + // ---------- 会话 ---------- + + async openSession(req: OpenSessionRequest): Promise { + const profileKey = req.profileKey ?? profileKeyFor(req.siteId, req.accountId); + if (req.initialUrl) assertNavigable(req.initialUrl, req.budget); + return this.withProfile(profileKey, async () => { + for (const s of this.sessions.values()) + if (s.session.profileKey === profileKey && s.session.controller !== "closed") + throw new BrowserError("control_conflict", "该站点/账号已有未关闭的浏览器会话"); + const sessionId = this.newId(); + const res = await this.request("POST", "/tabs", { + body: { userId: profileKey, sessionKey: sessionId, ...(req.initialUrl ? { url: req.initialUrl } : {}) }, + timeoutMs: req.budget.timeoutMs, + }); + const body = await this.json(res, "/tabs"); + const tabId = str(body.tabId); + if (!tabId) throw new BrowserError("protocol", "camofox POST /tabs 未返回 tabId"); + const rawUrl = str(body.url) ?? req.initialUrl ?? null; + const at = this.now().toISOString(); + const session: BrowserSession = { + id: sessionId, + providerId: this.cfg.providerId, + siteId: req.siteId, + accountId: req.accountId, + profileKey, + controller: "automation", + currentUrl: rawUrl === null ? null : redactBrowserUrl(rawUrl), + lastActivityAt: at, + upstreamRef: { userId: profileKey, sessionKey: sessionId }, + recoveryState: {}, + closedAt: null, + closeReason: null, + createdAt: at, + updatedAt: at, + }; + const tab: BrowserTab = { + id: this.newId(), + sessionId, + upstreamTabId: tabId, + url: session.currentUrl, + title: null, + status: "open", + lastSnapshotAt: null, + closedAt: null, + createdAt: at, + updatedAt: at, + }; + const st: SessionState = { + session, + tabs: new Map([[tabId, tab]]), + currentTabId: tabId, + rawUrl, + budget: req.budget, + }; + this.sessions.set(sessionId, st); + let navigation: NavigationResult | null = null; + if (req.initialUrl) { + navigation = { + url: session.currentUrl ?? "", + httpStatus: int(body.httpStatus) ?? null, + ok: body.navigationOk !== false, + attempts: 1, + }; + this.assertLanded(rawUrl, req.budget); + } + return { session: { ...session }, tab: { ...tab }, navigation }; + }); + } + + /** 重定向落到白名单外:报 domain_blocked(页面已打开,调用方应停止后续动作并转人工) */ + private assertLanded(rawUrl: string | null, budget: NavigationBudget): void { + if (!rawUrl) return; + let host: string; + try { + host = new URL(rawUrl).hostname; + } catch { + return; + } + if (!hostAllowed(host, budget.domainAllowlist)) + throw new BrowserError("domain_blocked", `导航被重定向到白名单外:${redactBrowserUrl(new URL(rawUrl).origin)}`); + } + + async closeSession(_sessionId: string, _reason?: string): Promise { + throw notImplemented("closeSession"); + } + + async listSessions(): Promise { + throw notImplemented("listSessions"); + } + + // ---------- 页面 ---------- + + async navigate(ref: TabRef, url: string, budget: NavigationBudget): Promise { + const s = this.state(ref.sessionId); + this.assertAutomation(s); + assertNavigable(url, budget); + return this.withProfile(s, async () => { + this.assertAutomation(s); + const tab = this.tabOf(s, ref); + const path = `/tabs/${encodeURIComponent(tab.upstreamTabId)}/navigate`; + let attempts = 0; + for (;;) { + attempts++; + try { + const res = await this.request("POST", path, { + body: { userId: s.session.profileKey, url }, + timeoutMs: budget.timeoutMs, + }); + const body = await this.json(res, path); + const rawUrl = str(body.url) ?? url; + this.touch(s, rawUrl); + tab.url = s.session.currentUrl; + tab.updatedAt = s.session.updatedAt; + s.budget = budget; + this.assertLanded(rawUrl, budget); + return { + url: redactBrowserUrl(rawUrl), + httpStatus: int(body.httpStatus) ?? null, + ok: body.ok !== false && body.navigationOk !== false, + attempts, + }; + } catch (e) { + if (e instanceof BrowserError && e.retryable && attempts <= budget.maxRetries) continue; + throw e; + } + } + }); + } + + async snapshot(ref: TabRef, opts: SnapshotOptions = {}): Promise { + const s = this.state(ref.sessionId); + return this.withProfile(s, async () => { + const tab = this.tabOf(s, ref); + const path = `/tabs/${encodeURIComponent(tab.upstreamTabId)}/snapshot`; + const res = await this.request("GET", path, { + query: { + userId: s.session.profileKey, + format: "text", + ...(opts.offset !== undefined ? { offset: String(opts.offset) } : {}), + }, + timeoutMs: s.budget.timeoutMs, + }); + const body = await this.json(res, path); + let text = str(body.snapshot) ?? ""; + let truncated = body.truncated === true || body.hasMore === true; + const maxBytes = opts.maxBytes ?? s.budget.maxBytes; + const enc = new TextEncoder(); + if (enc.encode(text).byteLength > maxBytes) { + // 按字节预算截断;在字符边界上截,避免半个 UTF-8 字符 + let lo = 0; + let hi = text.length; + while (lo < hi) { + const mid = Math.ceil((lo + hi) / 2); + if (enc.encode(text.slice(0, mid)).byteLength <= maxBytes) lo = mid; + else hi = mid - 1; + } + text = text.slice(0, lo); + truncated = true; + } + const rawUrl = str(body.url) ?? s.rawUrl ?? ""; + const structure = + body.structure && typeof body.structure === "object" && !Array.isArray(body.structure) + ? (body.structure as Json) + : null; + const red = redactSnapshot({ text, url: rawUrl, structure }, { extra: opts.extraRedactions ?? [] }); + this.touch(s, rawUrl || s.rawUrl); + const at = s.session.lastActivityAt; + tab.lastSnapshotAt = at; + tab.url = s.session.currentUrl; + const refsCount = int(body.refsCount) ?? new Set(text.match(/\be\d+\b/g) ?? []).size; + return { + sessionId: s.session.id, + tabId: tab.upstreamTabId, + url: red.url, + format: "a11y_text", + text: red.text, + structure: red.structure, + refsCount, + truncated, + nextOffset: truncated ? (int(body.nextOffset) ?? null) : null, + redactions: red.redactions, + capturedAt: at, + }; + }); + } + + async screenshot(ref: TabRef, opts: { maxBytes?: number } = {}): Promise { + const s = this.state(ref.sessionId); + return this.withProfile(s, async () => { + const tab = this.tabOf(s, ref); + const path = `/tabs/${encodeURIComponent(tab.upstreamTabId)}/screenshot`; + const res = await this.request("GET", path, { + query: { userId: s.session.profileKey }, + timeoutMs: s.budget.timeoutMs, + }); + const maxBytes = opts.maxBytes ?? s.budget.maxBytes ?? this.cfg.defaultMaxBytes ?? 8 * 1024 * 1024; + const declared = Number(res.headers.get("content-length") ?? "NaN"); + if (Number.isFinite(declared) && declared > maxBytes * 1.4) { + await res.body?.cancel(); + throw new BrowserError("budget_exceeded", `截图超出字节预算(${maxBytes})`); + } + const ctype = res.headers.get("content-type") ?? ""; + let data: Uint8Array; + let mimeType: string; + if (ctype.startsWith("image/")) { + // 部分版本/fork 直接返回二进制图片(待核验) + data = new Uint8Array(await res.arrayBuffer()); + mimeType = ctype.split(";")[0]!.trim(); + } else { + // OpenAPI 1.17.0:{ screenshot: { data: base64, mimeType } } + const body = await this.json(res, path); + const shot = (body.screenshot && typeof body.screenshot === "object" ? body.screenshot : body) as Json; + const b64 = str(shot.data); + if (!b64) throw new BrowserError("protocol", "camofox 截图响应缺少 data"); + try { + data = Uint8Array.from(atob(b64.replace(/^data:[^,]*,/, "")), (c) => c.charCodeAt(0)); + } catch { + throw new BrowserError("protocol", "camofox 截图不是合法 base64"); + } + mimeType = str(shot.mimeType) ?? "image/png"; + } + if (data.byteLength > maxBytes) throw new BrowserError("budget_exceeded", `截图超出字节预算(${maxBytes})`); + this.touch(s); + const raw = s.rawUrl; + let loginPage = false; + try { + loginPage = raw !== null && LOGIN_PATH.test(new URL(raw).pathname); + } catch { + /* ignore */ + } + return { + mimeType, + data, + url: raw === null ? null : redactBrowserUrl(raw), + modelSafe: raw !== null && !urlHasCredential(raw) && !loginPage, + capturedAt: s.session.lastActivityAt, + }; + }); + } + + // ---------- M2 ---------- + + async click(_ref: TabRef, _target: ElementTarget): Promise { + throw notImplemented("click"); + } + async type(_ref: TabRef, _target: ElementTarget, _text: TypeInput): Promise { + throw notImplemented("type"); + } + async extract(_ref: TabRef, _schema: Record): Promise> { + throw notImplemented("extract"); + } + async waitForDownload(_ref: TabRef, _opts: { timeoutMs: number; maxBytes: number }): Promise { + throw notImplemented("waitForDownload"); + } + async acquireControl(_req: AcquireControlRequest): Promise { + throw notImplemented("acquireControl"); + } + async releaseControl(_leaseId: string, _reason?: string): Promise { + throw notImplemented("releaseControl"); + } +} + +export function createCamofoxBrowserProvider(cfg: CamofoxConfig): CamofoxBrowserProvider { + return new CamofoxBrowserProvider(cfg); +} diff --git a/packages/adapters/src/browser/camofox/index.ts b/packages/adapters/src/browser/camofox/index.ts new file mode 100644 index 00000000..3c5f04fc --- /dev/null +++ b/packages/adapters/src/browser/camofox/index.ts @@ -0,0 +1,2 @@ +export * from "./capabilities.ts"; +export * from "./client.ts"; diff --git a/packages/adapters/src/browser/errors.ts b/packages/adapters/src/browser/errors.ts new file mode 100644 index 00000000..bd50541d --- /dev/null +++ b/packages/adapters/src/browser/errors.ts @@ -0,0 +1,36 @@ +import type { BrowserErrorKind } from "@umt/contracts"; + +/** + * 浏览器适配层统一错误。message 只含方法、路径与状态,不含访问凭据、Cookie、页面正文。 + * retryable:同一预算内是否值得重试(超时/网络/上游 5xx/不可用);认证、预算、控制权冲突一律不重试。 + */ +export class BrowserError extends Error { + readonly kind: BrowserErrorKind; + readonly httpStatus?: number; + constructor(kind: BrowserErrorKind, message?: string, opts: { httpStatus?: number; cause?: unknown } = {}) { + super(message ?? kind, opts.cause === undefined ? undefined : { cause: opts.cause }); + this.name = "BrowserError"; + this.kind = kind; + if (opts.httpStatus !== undefined) this.httpStatus = opts.httpStatus; + } + get retryable(): boolean { + return ( + this.kind === "timeout" || this.kind === "network" || this.kind === "upstream" || this.kind === "unavailable" + ); + } +} + +export const notImplemented = (method: string): BrowserError => + new BrowserError("not_implemented", `${method} 未实现(M2 原型验证后启用)`); + +/** HTTP 状态 → 错误种类(camofox OpenAPI 1.17.0 的响应码:400/401/403/404/409/422/429/500/503) */ +export function classifyHttpStatus(status: number): BrowserErrorKind { + if (status === 401 || status === 403) return "auth"; + if (status === 404) return "not_found"; + if (status === 408) return "timeout"; + if (status === 429) return "rate_limited"; + if (status === 503) return "unavailable"; + if (status >= 500) return "upstream"; + if (status >= 400) return "bad_request"; + return "protocol"; +} diff --git a/packages/adapters/src/browser/index.ts b/packages/adapters/src/browser/index.ts new file mode 100644 index 00000000..b4d06656 --- /dev/null +++ b/packages/adapters/src/browser/index.ts @@ -0,0 +1,7 @@ +export * from "./budget.ts"; +export * from "./camofox/index.ts"; +export * from "./errors.ts"; +export * from "./none.ts"; +export * from "./profile-lock.ts"; +export * from "./provider.ts"; +export * from "./redact.ts"; diff --git a/packages/adapters/src/browser/none.ts b/packages/adapters/src/browser/none.ts new file mode 100644 index 00000000..7ca4b3b7 --- /dev/null +++ b/packages/adapters/src/browser/none.ts @@ -0,0 +1,46 @@ +import type { BrowserCapabilities, BrowserCapabilityFlag, BrowserHealth } from "@umt/contracts"; +import { BROWSER_CAPABILITY_KEYS } from "@umt/contracts"; +import { BrowserError } from "./errors.ts"; +import type { BrowserProvider } from "./provider.ts"; + +const off: BrowserCapabilityFlag = { supported: false, verified: false, source: "static", note: "未配置浏览器服务" }; + +export function noneCapabilities(): BrowserCapabilities { + const flags = Object.fromEntries(BROWSER_CAPABILITY_KEYS.map((k) => [k, { ...off }])) as Record< + (typeof BROWSER_CAPABILITY_KEYS)[number], + BrowserCapabilityFlag + >; + return { ...flags, upstreamVersion: null, probedAt: null }; +} + +const unavailable = () => Promise.reject(new BrowserError("unavailable", "未配置浏览器服务")); + +/** + * 默认实现:未配置浏览器服务。所有动作返回 unavailable,能力全部 false; + * 业务侧据此在界面标注"浏览器不可用",不回退到其他浏览器。 + */ +export function createNoneBrowserProvider(): BrowserProvider { + return { + kind: "none", + async health(): Promise { + return { status: "disabled", kind: "none", detail: {}, checkedAt: new Date().toISOString() }; + }, + async capabilities() { + return noneCapabilities(); + }, + openSession: unavailable, + closeSession: unavailable, + async listSessions() { + return []; + }, + navigate: unavailable, + snapshot: unavailable, + screenshot: unavailable, + click: unavailable, + type: unavailable, + extract: unavailable, + waitForDownload: unavailable, + acquireControl: unavailable, + releaseControl: unavailable, + }; +} diff --git a/packages/adapters/src/browser/profile-lock.ts b/packages/adapters/src/browser/profile-lock.ts new file mode 100644 index 00000000..b9fde1b1 --- /dev/null +++ b/packages/adapters/src/browser/profile-lock.ts @@ -0,0 +1,66 @@ +import { BrowserError } from "./errors.ts"; + +/** + * 同一 profile(站点/账号)的浏览器操作串行(REQ §4.4)。 + * 内存实现只在单进程内有效;多进程/多实例部署需换成持久化实现(PG advisory lock 或 + * browser_control_lease 表),接口保持不变。 + */ +export interface ProfileLock { + /** 在持有 key 锁期间执行 fn;等待超过 waitMs 抛 BrowserError('lock_timeout') */ + run(key: string, fn: () => Promise, opts?: { waitMs?: number }): Promise; + /** 当前持有或排队中的 key 数量(诊断用) */ + size(): number; +} + +interface Waiter { + resolve: () => void; + reject: (e: unknown) => void; + timer?: ReturnType; +} + +export class InMemoryProfileLock implements ProfileLock { + private readonly held = new Map(); + constructor(private readonly defaultWaitMs = 30_000) {} + + size(): number { + return this.held.size; + } + + async run(key: string, fn: () => Promise, opts: { waitMs?: number } = {}): Promise { + await this.acquire(key, opts.waitMs ?? this.defaultWaitMs); + try { + return await fn(); + } finally { + this.release(key); + } + } + + private acquire(key: string, waitMs: number): Promise { + const queue = this.held.get(key); + if (!queue) { + this.held.set(key, []); + return Promise.resolve(); + } + return new Promise((resolve, reject) => { + const w: Waiter = { resolve, reject }; + w.timer = setTimeout(() => { + const i = queue.indexOf(w); + if (i >= 0) queue.splice(i, 1); + reject(new BrowserError("lock_timeout", `profile 锁等待超时(${waitMs}ms)`)); + }, waitMs); + queue.push(w); + }); + } + + private release(key: string): void { + const queue = this.held.get(key); + if (!queue) return; + const next = queue.shift(); + if (!next) { + this.held.delete(key); + return; + } + if (next.timer) clearTimeout(next.timer); + next.resolve(); + } +} diff --git a/packages/adapters/src/browser/provider.ts b/packages/adapters/src/browser/provider.ts new file mode 100644 index 00000000..2dced4e7 --- /dev/null +++ b/packages/adapters/src/browser/provider.ts @@ -0,0 +1,147 @@ +import type { + BrowserCapabilities, + BrowserControlLease, + BrowserHealth, + BrowserLeaseHolderKind, + BrowserProviderKind, + BrowserSession, + BrowserSnapshot, + BrowserTab, + DownloadArtifact, + NavigationBudget, +} from "@umt/contracts"; + +/** + * BrowserProvider:业务侧访问浏览器服务的唯一合同(REQ §4.7.2)。 + * 这是本项目接口;上游(camofox-browser 等)的同名能力由各实现映射,不假定上游具备同名端点。 + * 约束: + * - 同一 profileKey(站点/账号)的操作串行(ProfileLock); + * - 导航受 NavigationBudget 约束(域名白名单、超时、重试、字节上限); + * - 进入模型的快照/截图先经脱敏钩子;凭证经 BrowserSecretResolver 注入,不落日志、不进模型; + * - controller 不是 automation 的会话拒绝自动化输入(接管期间,REQ §4.7.3); + * - 浏览器没有媒体库写权限,下载产物只交后端暂存核验。 + */ +export interface BrowserProvider { + readonly kind: BrowserProviderKind; + health(): Promise; + capabilities(opts?: { probe?: boolean }): Promise; + + openSession(req: OpenSessionRequest): Promise; + closeSession(sessionId: string, reason?: string): Promise; + listSessions(): Promise; + + navigate(ref: TabRef, url: string, budget: NavigationBudget): Promise; + snapshot(ref: TabRef, opts?: SnapshotOptions): Promise; + screenshot(ref: TabRef, opts?: { maxBytes?: number }): Promise; + click(ref: TabRef, target: ElementTarget): Promise; + /** text 为明文或凭证引用;凭证引用由服务端解析后直接下发,不进入模型与日志 */ + type(ref: TabRef, target: ElementTarget, text: TypeInput): Promise; + extract(ref: TabRef, schema: Record): Promise>; + waitForDownload(ref: TabRef, opts: { timeoutMs: number; maxBytes: number }): Promise; + + acquireControl(req: AcquireControlRequest): Promise; + releaseControl(leaseId: string, reason?: string): Promise; +} + +export interface OpenSessionRequest { + providerId: string; + siteId: string; + accountId: string | null; + /** 缺省按 siteId/accountId 生成(profileKeyFor) */ + profileKey?: string; + /** 可选初始地址,同样受 budget 约束 */ + initialUrl?: string; + budget: NavigationBudget; +} + +export interface OpenSessionResult { + session: BrowserSession; + tab: BrowserTab; + navigation: NavigationResult | null; +} + +/** 会话内标签页定位;tabId 缺省 = 会话当前标签页 */ +export interface TabRef { + sessionId: string; + tabId?: string; +} + +export interface NavigationResult { + /** 已脱敏的最终地址 */ + url: string; + httpStatus: number | null; + /** 上游报告的文档状态 < 400 */ + ok: boolean; + attempts: number; +} + +export interface SnapshotOptions { + offset?: number; + maxBytes?: number; + /** 追加脱敏规则(站点特定字段) */ + extraRedactions?: RegExp[]; +} + +export interface BrowserScreenshot { + mimeType: string; + data: Uint8Array; + /** 已脱敏的页面地址 */ + url: string | null; + /** + * 是否可直接进入模型。像素级遮蔽未实现(M2),当前按页面地址/已知密码框保守判断: + * 含凭据的地址或登录页一律 false,调用方只能给人看、不喂模型。 + */ + modelSafe: boolean; + capturedAt: string; +} + +export type ElementTarget = { ref: string } | { selector: string }; + +export type TypeInput = + | { kind: "text"; text: string } + | { kind: "secret"; siteId: string; accountId: string | null; field: "username" | "password" | "totp" }; + +export interface AcquireControlRequest { + sessionId: string; + holderKind: BrowserLeaseHolderKind; + holder: string; + reason: string; + ttlMs: number; +} + +/** 凭证注入:服务端从 secret store 按需解出明文;适配器不持久化、不打印、不放进错误信息 */ +export type BrowserSecretRequest = + | { kind: "provider_access_key"; providerId: string } + | { kind: "site_login"; siteId: string; accountId: string | null; field: "username" | "password" | "totp" }; + +export interface BrowserSecretResolver { + resolve(req: BrowserSecretRequest): Promise; +} + +/** 测试/脚本用固定凭证解析器;键为 `provider:` 或 `site:::` */ +export function staticBrowserSecrets(table: Record): BrowserSecretResolver { + return { + async resolve(req) { + const key = + req.kind === "provider_access_key" + ? `provider:${req.providerId}` + : `site:${req.siteId}:${req.accountId ?? "-"}:${req.field}`; + return table[key] ?? null; + }, + }; +} + +/** 站点/账号维度的 profile 键;同时用作 camofox userId(登录状态隔离单元) */ +export function profileKeyFor(siteId: string, accountId: string | null): string { + return `umt-site-${siteId}-acct-${accountId ?? "anon"}`; +} + +/** 只记录方法/路径/状态;实现不得把请求头、正文或快照写进日志 */ +export type BrowserLogger = (entry: { + level: "debug" | "warn"; + msg: string; + method?: string; + path?: string; + status?: number; + ms?: number; +}) => void; diff --git a/packages/adapters/src/browser/redact.ts b/packages/adapters/src/browser/redact.ts new file mode 100644 index 00000000..31d0b873 --- /dev/null +++ b/packages/adapters/src/browser/redact.ts @@ -0,0 +1,181 @@ +import type { SnapshotRedactionKind } from "@umt/contracts"; +import { REDACTED } from "../rss/redact.ts"; + +/** + * 快照/截图进入模型前的脱敏钩子(REQ §4.4、§4.7.3)。 + * 覆盖:密码框的值、Cookie 值、URL 中的 passkey/token 等凭据参数。 + * camofox 可访问性快照的精确文本格式("[role eN] name ...")未在真实实例核验, + * 这里同时处理 a11y 文本、HTML 片段与 structure JSON,宁可多遮不可漏。 + */ + +/** URL 查询参数/路径段里的凭据键(PT 站 passkey、RSS key、下载签名等) */ +const CREDENTIAL_PARAM = + /([?&;](?:passkey|authkey|torrent_pass|rsskey|downhash|credential|token|access_token|api_key|apikey|secret|sign|signature|key)=)([^&#\s"'<>)\]]+)/gi; +/** 路径形式的 passkey:/passkey/ 或 /rss/<32+ hex> */ +const CREDENTIAL_PATH = /(\/(?:passkey|rsskey|torrent_pass)\/)([A-Za-z0-9]{8,})/gi; + +/** 常见站点登录/会话 Cookie 名 */ +const COOKIE_PAIR = + /\b(c_secure_[a-z_]+|PHPSESSID|cf_clearance|__cf_bm|sessionid|session_id|laravel_session|XSRF-TOKEN|auth_token|remember_[a-z0-9_]+|tp|uid|pass)=([^;\s"'<>]+)/gi; +/** "Cookie: ..." / "set-cookie: ..." 整行 */ +const COOKIE_HEADER = /\b((?:set-)?cookie\s*[:=]\s*)([^\n]+)/gi; + +/** HTML:type=password 的 input 上的 value */ +const HTML_PASSWORD_INPUT = /]*\btype\s*=\s*["']?password["']?[^>]*>/gi; +const HTML_VALUE_ATTR = /(\bvalue\s*=\s*)("[^"]*"|'[^']*'|[^\s>]+)/i; + +/** + * a11y 文本中名称像密码/凭据的元素,遮蔽其后的取值。 + * 形如 `[textbox e3] Password: hunter2`、`[textbox e3 password] 密码 "hunter2"`、`- textbox "Password": hunter2`。 + */ +const A11Y_SECRET_FIELD = + /(\[[a-z]+ e\d+[^\]]*\][^\n[]*?(?:password|passwd|密码|passkey|2fa|totp|验证码|otp)[^\n[:"]*?(?::\s*|\s+))("[^"\n]*"|[^\s[\n][^[\n]*?)(?=\s{2,}|\s*\[|\n|$)/gi; +const A11Y_TYPED_PASSWORD = + /(\[[a-z]+ e\d+[^\]]*\bpassword\b[^\]]*\](?:[^\n[:"]*:)?\s*)("[^"\n]*"|[^\s[\n][^[\n]*?)(?=\s{2,}|\s*\[|\n|$)/gi; +/(\[[a-z]+ e\d+[^\]]*\bpassword\b[^\]]*\][^\n[]*?(?::\s*|\s+))("[^"\n]*"|[^\s[\n][^[\n]*?)(?=\s{2,}|\s*\[|\n|$)/gi; +const YAML_SECRET_FIELD = /(-\s*textbox\s+"[^"\n]*(?:password|passwd|密码|passkey)[^"\n]*"\s*:\s*)(\S[^\n]*)/gi; + +export interface RedactionReport { + kind: SnapshotRedactionKind; + count: number; +} + +class Counter { + private readonly m = new Map(); + add(kind: SnapshotRedactionKind, n = 1) { + this.m.set(kind, (this.m.get(kind) ?? 0) + n); + } + list(): RedactionReport[] { + return [...this.m].filter(([, n]) => n > 0).map(([kind, count]) => ({ kind, count })); + } +} + +function replaceCounting(s: string, re: RegExp, fn: (...m: string[]) => string, onHit: () => void): string { + return s.replace(re, (...args: unknown[]) => { + const groups = args.filter((a): a is string => typeof a === "string"); + const out = fn(...groups); + if (out !== groups[0]) onHit(); + return out; + }); +} + +const USERINFO = /\b([a-z][a-z0-9+.-]*:\/\/)([^/\s@:]+(?::[^/\s@]*)?)@/gi; + +function redactUrlsIn(s: string, c: Counter): string { + let out = replaceCounting( + s, + USERINFO, + (_a, scheme) => scheme, + () => c.add("credential_url"), + ); + out = replaceCounting( + out, + CREDENTIAL_PARAM, + (all, k, v) => (v.startsWith("[REDACTED") ? all : `${k}${REDACTED}`), + () => c.add("credential_url"), + ); + out = replaceCounting( + out, + CREDENTIAL_PATH, + (_all, k) => `${k}${REDACTED}`, + () => c.add("credential_url"), + ); + return out; +} + +/** 单独脱敏一个 URL(日志、导航结果、会话 currentUrl 都用它) */ +export function redactBrowserUrl(url: string): string { + return redactUrlsIn(url, new Counter()); +} + +/** URL 是否带凭据(用于判定截图能否进入模型) */ +export function urlHasCredential(url: string): boolean { + return redactBrowserUrl(url) !== url; +} + +function redactText(s: string, c: Counter, extra: RegExp[]): string { + let out = redactUrlsIn(s, c); + out = replaceCounting( + out, + COOKIE_HEADER, + (_a, k) => `${k}${REDACTED}`, + () => c.add("cookie"), + ); + out = replaceCounting( + out, + COOKIE_PAIR, + (a, k, v) => (v === REDACTED ? a : `${k}=${REDACTED}`), + () => c.add("cookie"), + ); + out = out.replace(HTML_PASSWORD_INPUT, (tag) => { + if (!HTML_VALUE_ATTR.test(tag)) return tag; + c.add("password_field"); + return tag.replace(HTML_VALUE_ATTR, (_m, k) => `${k}"${REDACTED}"`); + }); + for (const re of [A11Y_TYPED_PASSWORD, A11Y_SECRET_FIELD, YAML_SECRET_FIELD]) { + out = replaceCounting( + out, + re, + (a, k, v) => (v === REDACTED || v === `"${REDACTED}"` ? a : `${k}${REDACTED}`), + () => c.add("password_field"), + ); + } + for (const re of extra) { + out = replaceCounting( + out, + re, + () => REDACTED, + () => c.add("secret_value"), + ); + } + return out; +} + +const SECRET_KEY = /password|passwd|passkey|cookie|token|secret|authkey|credential/i; + +function redactJson(v: unknown, c: Counter, extra: RegExp[], parentIsPassword = false): unknown { + if (typeof v === "string") return redactText(v, c, extra); + if (Array.isArray(v)) return v.map((x) => redactJson(x, c, extra, parentIsPassword)); + if (v && typeof v === "object") { + const o = v as Record; + const isPassword = + parentIsPassword || + (typeof o.type === "string" && o.type.toLowerCase() === "password") || + (typeof o.inputType === "string" && o.inputType.toLowerCase() === "password"); + const out: Record = {}; + for (const [k, val] of Object.entries(o)) { + if ((isPassword && k === "value") || (SECRET_KEY.test(k) && typeof val === "string" && val !== "")) { + c.add(k.toLowerCase().includes("cookie") ? "cookie" : "password_field"); + out[k] = REDACTED; + } else out[k] = redactJson(val, c, extra, false); + } + return out; + } + return v; +} + +export interface SnapshotInput { + text: string; + url: string; + structure?: Record | null; +} + +export interface RedactedSnapshot { + text: string; + url: string; + structure: Record | null; + redactions: RedactionReport[]; +} + +/** + * 快照脱敏:隐藏密码框值、Cookie 值、含 passkey 等凭据的 URL。 + * 输出可直接交给模型;调用方仍不得把未经此函数的上游快照写进 agent 上下文或日志。 + */ +export function redactSnapshot(input: SnapshotInput, opts: { extra?: RegExp[] } = {}): RedactedSnapshot { + const c = new Counter(); + const extra = (opts.extra ?? []).map((r) => new RegExp(r.source, r.flags.includes("g") ? r.flags : `${r.flags}g`)); + const text = redactText(input.text, c, extra); + const url = redactUrlsIn(input.url, c); + const structure = input.structure ? (redactJson(input.structure, c, extra) as Record) : null; + return { text, url, structure, redactions: c.list() }; +} diff --git a/packages/adapters/src/index.ts b/packages/adapters/src/index.ts index 715d2433..d3bb65f7 100644 --- a/packages/adapters/src/index.ts +++ b/packages/adapters/src/index.ts @@ -1,5 +1,6 @@ // @umt/adapters 公开入口;W1 起在此导出实现 export const PACKAGE_NAME = "@umt/adapters"; +export * from "./browser/index.ts"; export * from "./emby/index.ts"; export * from "./fs/index.ts"; export * from "./qbittorrent/index.ts"; diff --git a/packages/contracts/src/browser.ts b/packages/contracts/src/browser.ts new file mode 100644 index 00000000..b6bd699e --- /dev/null +++ b/packages/contracts/src/browser.ts @@ -0,0 +1,223 @@ +import { z } from "zod"; +import { EntityBase, Id, JsonObject, NonNegInt, Timestamp } from "./common.ts"; +import { InstanceHealth } from "./instances.ts"; + +// 浏览器能力合同(REQ §4.4、§4.7.2–4.7.4、§12 浏览器实体)。 +// 这是本项目自有接口,不等于上游同名接口:上游对接由 @umt/adapters 的 BrowserProvider 实现 +// (首个实现为用户已有的 camofox-browser;Playwright 只用于前端截图测试,不是产品浏览器能力)。 +// v0.1 只落合同与骨架,会话/接管/下载捕获在 M2 原型验证后启用(docs/design/browser-provider.md)。 + +/** 浏览器服务实现种类;none = 未配置(默认),所有浏览器动作返回不可用 */ +export const BrowserProviderKind = z.enum(["camofox", "none"]); +export type BrowserProviderKind = z.infer; + +/** + * 单项能力声明。supported = 实现/上游声明具备;verified = 已对用户真实实例做过原型验证(REQ §4.7.4)。 + * 界面只把 supported && verified 的能力当作可用,未验证的明确标注"未验证",不得用其他浏览器替代后宣称可用。 + */ +export const BrowserCapabilityFlag = z.object({ + supported: z.boolean(), + verified: z.boolean(), + /** static = 适配器按文档静态声明;probed = 从服务端 OpenAPI/健康端点探测到 */ + source: z.enum(["static", "probed"]), + note: z.string().optional(), +}); +export type BrowserCapabilityFlag = z.infer; + +export const BROWSER_CAPABILITY_KEYS = [ + "sessions", + "tabs", + "navigate", + "snapshot", + "screenshot", + "interact", + "extract", + "downloadCapture", + "takeover", + "persistentProfile", +] as const; +export type BrowserCapabilityKey = (typeof BROWSER_CAPABILITY_KEYS)[number]; + +export const BrowserCapabilities = z.object({ + /** 会话(站点/账号隔离的浏览器上下文) */ + sessions: BrowserCapabilityFlag, + /** 标签页列举/新建/关闭 */ + tabs: BrowserCapabilityFlag, + navigate: BrowserCapabilityFlag, + /** 结构化快照(可访问性树 + 元素引用),非原始 HTML */ + snapshot: BrowserCapabilityFlag, + screenshot: BrowserCapabilityFlag, + /** click / type / select */ + interact: BrowserCapabilityFlag, + /** 结构化提取(按 schema 取字段) */ + extract: BrowserCapabilityFlag, + /** 捕获浏览器下载产物(.torrent 等)交后端暂存核验 */ + downloadCapture: BrowserCapabilityFlag, + /** 人工接管:同一会话的画面与输入通道 */ + takeover: BrowserCapabilityFlag, + /** 登录状态随 profile 持久化,重启后恢复 */ + persistentProfile: BrowserCapabilityFlag, + /** 上游版本(来自 OpenAPI info.version 等);未取得为 null */ + upstreamVersion: z.string().nullable(), + probedAt: Timestamp.nullable(), +}); +export type BrowserCapabilities = z.infer; + +/** 会话控制权:自动化 / 人工接管 / 等待处理(验证码、2FA 等)/ 已关闭(REQ §4.7.3) */ +export const BrowserController = z.enum(["automation", "human", "waiting", "closed"]); +export type BrowserController = z.infer; + +export const BrowserProviderRow = EntityBase.extend({ + name: z.string().min(1), + kind: BrowserProviderKind, + baseUrl: z.string().nullable(), + /** 访问凭据(如 camofox CAMOFOX_ACCESS_KEY)只以 secret 引用保存 */ + secretRef: Id.nullable(), + enabled: z.boolean(), + /** 最近一次声明/探测到的 BrowserCapabilities 快照 */ + capabilities: JsonObject, + /** 最近一次对真实实例完成原型验证的时间;null = 从未验证 */ + verifiedAt: Timestamp.nullable(), + healthStatus: InstanceHealth, + lastHealthAt: Timestamp.nullable(), + config: JsonObject, +}); +export type BrowserProviderRow = z.infer; + +/** 浏览器会话:按站点/账号绑定;同一 profileKey 同时至多一个未关闭会话 */ +export const BrowserSession = EntityBase.extend({ + providerId: Id, + siteId: Id, + accountId: Id.nullable(), + /** 站点/账号维度的 profile 键,决定上游登录状态隔离与串行锁 */ + profileKey: z.string().min(1), + controller: BrowserController, + currentUrl: z.string().nullable(), + lastActivityAt: Timestamp, + /** 上游会话标识(camofox: userId/sessionKey);不含凭据 */ + upstreamRef: JsonObject, + /** 接管/断线/重启时的可恢复状态;恢复后必须重新读取页面,不重放旧元素引用 */ + recoveryState: JsonObject, + closedAt: Timestamp.nullable(), + closeReason: z.string().nullable(), +}); +export type BrowserSession = z.infer; + +export const BrowserTabStatus = z.enum(["open", "closed", "lost"]); +export type BrowserTabStatus = z.infer; + +export const BrowserTab = EntityBase.extend({ + sessionId: Id, + /** 上游标签页 ID(camofox tabId) */ + upstreamTabId: z.string().min(1), + url: z.string().nullable(), + title: z.string().nullable(), + status: BrowserTabStatus, + lastSnapshotAt: Timestamp.nullable(), + closedAt: Timestamp.nullable(), +}); +export type BrowserTab = z.infer; + +export const BrowserLeaseHolderKind = z.enum(["automation", "human"]); +export type BrowserLeaseHolderKind = z.infer; + +export const BrowserLeaseStatus = z.enum(["active", "released", "expired", "revoked"]); +export type BrowserLeaseStatus = z.infer; + +/** 会话控制权租约:每个会话同时至多一个 active(DB 部分唯一索引保证) */ +export const BrowserControlLease = EntityBase.extend({ + sessionId: Id, + holderKind: BrowserLeaseHolderKind, + /** 持有者标识:agent_run id / admin 用户名等 */ + holder: z.string().min(1), + reason: z.string().min(1), + status: BrowserLeaseStatus, + acquiredAt: Timestamp, + expiresAt: Timestamp, + releasedAt: Timestamp.nullable(), + releaseReason: z.string().nullable(), +}); +export type BrowserControlLease = z.infer; + +/** 导航预算(REQ §4.4):域名范围、超时、重试与资源上限;空白名单 = 拒绝全部导航 */ +export const NavigationBudget = z.object({ + /** 允许的主机名;"example.org" 同时放行其子域 */ + domainAllowlist: z.array(z.string().min(1)), + timeoutMs: z.number().int().min(1), + maxRetries: z.number().int().min(0).max(10), + /** 单次响应(快照文本/截图)字节上限 */ + maxBytes: z.number().int().min(1), +}); +export type NavigationBudget = z.infer; + +export const SnapshotRedactionKind = z.enum(["password_field", "cookie", "credential_url", "secret_value"]); +export type SnapshotRedactionKind = z.infer; + +/** + * 可交给模型的页面快照:只含经 redactSnapshot 脱敏后的结构化文本与元素引用, + * 密码框值、Cookie 值、含 passkey 等凭据的 URL 已遮蔽;不含原始 HTML/脚本。 + */ +export const BrowserSnapshot = z.object({ + sessionId: Id, + tabId: z.string().min(1), + /** 已脱敏的当前地址 */ + url: z.string(), + format: z.enum(["a11y_text"]), + /** 已脱敏的可访问性快照文本,元素以 eN 引用 */ + text: z.string(), + /** 表单/表格摘要(已脱敏) */ + structure: JsonObject.nullable(), + refsCount: NonNegInt, + truncated: z.boolean(), + nextOffset: NonNegInt.nullable(), + redactions: z.array(z.object({ kind: SnapshotRedactionKind, count: NonNegInt })), + capturedAt: Timestamp, +}); +export type BrowserSnapshot = z.infer; + +/** 二进制内容的引用:blob 暂存路径或 secret(下载产物可能含 passkey,不内联进合同/日志) */ +export const BlobRef = z.object({ + kind: z.enum(["blob", "secret"]), + ref: z.string().min(1), +}); +export type BlobRef = z.infer; + +/** 浏览器捕获的下载产物:交后端受控暂存与核验后再走原有提交链路(REQ §4.7.1) */ +export const DownloadArtifact = z.object({ + id: Id, + sessionId: Id, + filename: z.string().min(1), + mimeType: z.string().nullable(), + sizeBytes: NonNegInt, + bytes: BlobRef, + /** 十六进制 sha256 */ + sha256: z.string().min(1), + capturedAt: Timestamp, +}); +export type DownloadArtifact = z.infer; + +export const BrowserHealth = z.object({ + status: z.enum(["ok", "degraded", "down", "disabled"]), + kind: BrowserProviderKind, + detail: JsonObject, + checkedAt: Timestamp, +}); +export type BrowserHealth = z.infer; + +export const BrowserErrorKind = z.enum([ + "not_implemented", + "unavailable", + "auth", + "timeout", + "network", + "not_found", + "rate_limited", + "bad_request", + "upstream", + "protocol", + "domain_blocked", + "budget_exceeded", + "control_conflict", + "lock_timeout", +]); +export type BrowserErrorKind = z.infer; diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index 72ea350c..92f4612a 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -3,6 +3,7 @@ export * from "./agent-events.ts"; export * from "./api.ts"; export * from "./assets.ts"; export * from "./auth.ts"; +export * from "./browser.ts"; export * from "./capabilities-io.ts"; export * from "./capability.ts"; export * from "./common.ts"; diff --git a/packages/contracts/test/db-sync.test.ts b/packages/contracts/test/db-sync.test.ts index 25547c3c..253b6860 100644 --- a/packages/contracts/test/db-sync.test.ts +++ b/packages/contracts/test/db-sync.test.ts @@ -88,6 +88,11 @@ const ROWS: Record MetadataField: { row: C.MetadataField }, ResolutionAttempt: { row: C.ResolutionAttempt }, SiteSnapshot: { row: C.SiteSnapshot }, + // W4-C 浏览器实体(0023) + BrowserProvider: { row: C.BrowserProviderRow }, + BrowserSession: { row: C.BrowserSession }, + BrowserTab: { row: C.BrowserTab }, + BrowserControlLease: { row: C.BrowserControlLease }, }; /** 引擎自有表(@umt/tasks 内部使用,不经 API 暴露),不要求 Row schema */ @@ -209,6 +214,13 @@ const ENUMS: Record = { ResolutionAttemptActorType: C.ActorType, SiteSnapshotKind: C.SiteSnapshotKind, SiteSnapshotStatus: C.SiteSnapshotStatus, + // W4-C(0023) + BrowserProviderKind: C.BrowserProviderKind, + BrowserProviderHealthStatus: C.InstanceHealth, + BrowserSessionController: C.BrowserController, + BrowserTabStatus: C.BrowserTabStatus, + BrowserControlLeaseHolderKind: C.BrowserLeaseHolderKind, + BrowserControlLeaseStatus: C.BrowserLeaseStatus, }; describe("contracts ⇔ db 表结构同步", () => { diff --git a/packages/db/migrations/0023_browser.ts b/packages/db/migrations/0023_browser.ts new file mode 100644 index 00000000..62fa3114 --- /dev/null +++ b/packages/db/migrations/0023_browser.ts @@ -0,0 +1,93 @@ +import type { Kysely } from "kysely"; +import { dropTables, entityTable, execDdl, oneOf } from "../src/migration-helpers.ts"; + +// W4-C 浏览器实体(REQ §12.1 浏览器域、§4.7.3 会话控制与交接): +// - browser_provider:浏览器服务(首个为用户已有的 camofox-browser);访问凭据只存 secret_ref; +// capabilities 保存声明/探测结果,verified_at 记录最近一次真实实例原型验证; +// - browser_session:按站点/账号(profile_key)隔离;同一 provider+profile 至多一个未关闭会话; +// recovery_state 保存接管/断线/重启后的可恢复状态(恢复时重新读页面,不重放旧元素引用); +// - browser_tab:上游标签页映射; +// - browser_control_lease:控制权租约,每个会话至多一条 active(部分唯一索引)。 + +const HEALTH = ["unknown", "healthy", "degraded", "offline"] as const; + +// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型 +export async function up(db: Kysely): Promise { + await execDdl( + db, + ` +${entityTable( + "browser_provider", + ` + name text not null, + kind text not null ${oneOf("kind", ["camofox", "none"])}, + base_url text, + secret_ref uuid references secret (id) on delete restrict, + enabled boolean not null default false, + capabilities jsonb not null default '{}'::jsonb, + verified_at timestamptz, + health_status text not null default 'unknown' ${oneOf("health_status", HEALTH)}, + last_health_at timestamptz, + config jsonb not null default '{}'::jsonb, + constraint browser_provider_name_key unique (name), + constraint browser_provider_base_url_required check (kind = 'none' or base_url is not null)`, +)} + +${entityTable( + "browser_session", + ` + provider_id uuid not null references browser_provider (id) on delete cascade, + site_id uuid not null references site (id) on delete cascade, + account_id uuid references site_account (id) on delete set null, + profile_key text not null, + controller text not null default 'automation' ${oneOf("controller", ["automation", "human", "waiting", "closed"])}, + current_url text, + last_activity_at timestamptz not null default now(), + upstream_ref jsonb not null default '{}'::jsonb, + recovery_state jsonb not null default '{}'::jsonb, + closed_at timestamptz, + close_reason text, + constraint browser_session_closed_consistent check ((controller = 'closed') = (closed_at is not null))`, +)} +create unique index browser_session_live_profile_key + on browser_session (provider_id, profile_key) where controller <> 'closed'; +create index browser_session_site_idx on browser_session (site_id, account_id, last_activity_at desc); + +${entityTable( + "browser_tab", + ` + session_id uuid not null references browser_session (id) on delete cascade, + upstream_tab_id text not null, + url text, + title text, + status text not null default 'open' ${oneOf("status", ["open", "closed", "lost"])}, + last_snapshot_at timestamptz, + closed_at timestamptz, + constraint browser_tab_upstream_key unique (session_id, upstream_tab_id)`, +)} + +${entityTable( + "browser_control_lease", + ` + session_id uuid not null references browser_session (id) on delete cascade, + holder_kind text not null ${oneOf("holder_kind", ["automation", "human"])}, + holder text not null, + reason text not null, + status text not null default 'active' ${oneOf("status", ["active", "released", "expired", "revoked"])}, + acquired_at timestamptz not null default now(), + expires_at timestamptz not null, + released_at timestamptz, + release_reason text, + constraint browser_control_lease_window check (expires_at > acquired_at)`, +)} +create unique index browser_control_lease_one_active + on browser_control_lease (session_id) where status = 'active'; +create index browser_control_lease_session_idx on browser_control_lease (session_id, acquired_at desc); +`, + ); +} + +// biome-ignore lint/suspicious/noExplicitAny: 同上 +export async function down(db: Kysely): Promise { + await execDdl(db, dropTables(["browser_control_lease", "browser_tab", "browser_session", "browser_provider"])); +} diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index ca2d4d1f..f5f52816 100644 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -73,6 +73,12 @@ export type AssetOperationStatus = "planned" | "started" | "succeeded" | "failed export type AssetUnitCoverageCoverage = "full" | "partial" | "segment" | "unknown"; export type AssetUnitCoverageSegment = "full" | "upper" | "middle" | "lower" | "unknown"; export type AutomationGrantStatus = "active" | "revoked" | "expired"; +export type BrowserControlLeaseHolderKind = "automation" | "human"; +export type BrowserControlLeaseStatus = "active" | "released" | "expired" | "revoked"; +export type BrowserProviderKind = "camofox" | "none"; +export type BrowserProviderHealthStatus = "unknown" | "healthy" | "degraded" | "offline"; +export type BrowserSessionController = "automation" | "human" | "waiting" | "closed"; +export type BrowserTabStatus = "open" | "closed" | "lost"; export type CapabilityInvocationStatus = "running" | "succeeded" | "failed"; export type ContentUnitKind = "season" | "episode" | "special" | "part"; export type ContentUnitContentType = @@ -460,6 +466,67 @@ export interface AutomationGrantTable { updated_at: GeneratedTimestamptz; } +export interface BrowserControlLeaseTable { + id: Generated; + session_id: string; + holder_kind: BrowserControlLeaseHolderKind; + holder: string; + reason: string; + status: Generated; + acquired_at: GeneratedTimestamptz; + expires_at: Timestamptz; + released_at: Timestamptz | null; + release_reason: string | null; + created_at: GeneratedTimestamptz; + updated_at: GeneratedTimestamptz; +} + +export interface BrowserProviderTable { + id: Generated; + name: string; + kind: BrowserProviderKind; + base_url: string | null; + secret_ref: string | null; + enabled: Generated; + capabilities: Generated; + verified_at: Timestamptz | null; + health_status: Generated; + last_health_at: Timestamptz | null; + config: Generated; + created_at: GeneratedTimestamptz; + updated_at: GeneratedTimestamptz; +} + +export interface BrowserSessionTable { + id: Generated; + provider_id: string; + site_id: string; + account_id: string | null; + profile_key: string; + controller: Generated; + current_url: string | null; + last_activity_at: GeneratedTimestamptz; + upstream_ref: Generated; + recovery_state: Generated; + closed_at: Timestamptz | null; + close_reason: string | null; + created_at: GeneratedTimestamptz; + updated_at: GeneratedTimestamptz; +} + +export interface BrowserTabTable { + id: Generated; + session_id: string; + upstream_tab_id: string; + url: string | null; + title: string | null; + status: Generated; + last_snapshot_at: Timestamptz | null; + closed_at: Timestamptz | null; + created_at: GeneratedTimestamptz; + updated_at: GeneratedTimestamptz; +} + export interface CapabilityInvocationTable { id: Generated; idempotency_key: string; @@ -1225,6 +1292,10 @@ export interface DB { asset_unit_coverage: AssetUnitCoverageTable; audit_event: AuditEventTable; automation_grant: AutomationGrantTable; + browser_control_lease: BrowserControlLeaseTable; + browser_provider: BrowserProviderTable; + browser_session: BrowserSessionTable; + browser_tab: BrowserTabTable; capability_invocation: CapabilityInvocationTable; content_unit: ContentUnitTable; decision: DecisionTable; @@ -1314,6 +1385,18 @@ export type AuditEventUpdate = Updateable; export type AutomationGrantRow = Selectable; export type NewAutomationGrant = Insertable; export type AutomationGrantUpdate = Updateable; +export type BrowserControlLeaseRow = Selectable; +export type NewBrowserControlLease = Insertable; +export type BrowserControlLeaseUpdate = Updateable; +export type BrowserProviderRow = Selectable; +export type NewBrowserProvider = Insertable; +export type BrowserProviderUpdate = Updateable; +export type BrowserSessionRow = Selectable; +export type NewBrowserSession = Insertable; +export type BrowserSessionUpdate = Updateable; +export type BrowserTabRow = Selectable; +export type NewBrowserTab = Insertable; +export type BrowserTabUpdate = Updateable; export type CapabilityInvocationRow = Selectable; export type NewCapabilityInvocation = Insertable; export type CapabilityInvocationUpdate = Updateable; diff --git a/packages/db/test/schema.test.ts b/packages/db/test/schema.test.ts index bd9afaad..bf6c9cb6 100644 --- a/packages/db/test/schema.test.ts +++ b/packages/db/test/schema.test.ts @@ -83,6 +83,11 @@ const EXPECTED_TABLES = [ "admin_session", "login_attempt", "sse_event", + // W4-C 浏览器实体(0023) + "browser_provider", + "browser_session", + "browser_tab", + "browser_control_lease", ].sort(); /** 不走 entityTable() 的表:追加型/引擎自有表,没有 uuid id + created_at/updated_at 三件套 */ diff --git a/packages/testkit/src/browser/index.ts b/packages/testkit/src/browser/index.ts new file mode 100644 index 00000000..5db967c0 --- /dev/null +++ b/packages/testkit/src/browser/index.ts @@ -0,0 +1 @@ +export * from "./mock-camofox.ts"; diff --git a/packages/testkit/src/browser/mock-camofox.ts b/packages/testkit/src/browser/mock-camofox.ts new file mode 100644 index 00000000..6acfae84 --- /dev/null +++ b/packages/testkit/src/browser/mock-camofox.ts @@ -0,0 +1,284 @@ +import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; +import type { AddressInfo } from "node:net"; +import type { MockCall, MockFault, MockHttp, MockReply } from "../tmdb/mock-http.ts"; + +/** + * mock camofox-browser(形状按 jo-inc/camofox-browser openapi.json 1.17.0,取值虚构,未对真实实例核验)。 + * 路由:GET /health、GET /openapi.json、POST /tabs、GET /tabs、POST /tabs/{id}/navigate、 + * GET /tabs/{id}/snapshot(固定可访问性快照 + 表单 structure,含密码值/Cookie/passkey 供脱敏测试)、 + * GET /tabs/{id}/screenshot(1x1 PNG,base64)、DELETE /sessions/{userId}。 + * accessKey 设置时模拟 CAMOFOX_ACCESS_KEY:除 /health 外要求 `Authorization: Bearer `,否则 401。 + * 故障注入(401/超时/断连)用 MockHttp.fault。 + */ +export interface MockCamofoxOptions { + accessKey?: string; + /** 覆盖快照文本;{url} 替换为当前地址 */ + snapshotText?: string; + /** 让 openapi.json 缺少某些端点(模拟旧版本/fork) */ + omitPaths?: string[]; + version?: string; + port?: number; +} + +export interface MockCamofoxTab { + tabId: string; + userId: string; + sessionKey: string; + url: string; +} + +export type MockCamofox = MockHttp & { tabs: Map }; + +/** 1x1 透明 PNG */ +export const PNG_1X1_BASE64 = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=="; + +export const CAMOFOX_FIXTURE_PASSWORD = "hunter2-fixture-password"; +export const CAMOFOX_FIXTURE_COOKIE = "c0ffee-fixture-cookie"; +export const CAMOFOX_FIXTURE_PASSKEY = "0123456789abcdef0123456789abcdef"; + +export const CAMOFOX_FIXTURE_SNAPSHOT = [ + "[heading e1] 观众 · 种子列表", + `[link e2] 下载 https://audiences.example/download.php?id=42&passkey=${CAMOFOX_FIXTURE_PASSKEY}`, + "[textbox e3] 用户名: fixture-user", + `[textbox e4 password] 密码: ${CAMOFOX_FIXTURE_PASSWORD}`, + `[text e5] Cookie: c_secure_pass=${CAMOFOX_FIXTURE_COOKIE}; c_secure_uid=1001`, + "[button e6] 登录", + "[row e7] 奥本海默 Oppenheimer 2023 2160p UHD BluRay 做种 128", +].join("\n"); + +const OPENAPI_PATHS: Record = { + "/health": ["get"], + "/tabs": ["post", "get"], + "/tabs/{tabId}": ["delete"], + "/tabs/{tabId}/navigate": ["post"], + "/tabs/{tabId}/snapshot": ["get"], + "/tabs/{tabId}/screenshot": ["get"], + "/tabs/{tabId}/click": ["post"], + "/tabs/{tabId}/type": ["post"], + "/tabs/{tabId}/select": ["post"], + "/tabs/{tabId}/extract": ["post"], + "/tabs/{tabId}/downloads": ["get"], + "/sessions/{userId}": ["delete"], + "/sessions/{userId}/cookies": ["post"], +}; + +type Body = Record; +type Route = (m: RegExpMatchArray, q: Record, req: IncomingMessage, body: Body) => MockReply; + +async function readJson(req: IncomingMessage): Promise { + const chunks: Buffer[] = []; + for await (const c of req) chunks.push(c as Buffer); + if (chunks.length === 0) return {}; + try { + const v = JSON.parse(Buffer.concat(chunks).toString("utf8")) as unknown; + return v && typeof v === "object" ? (v as Body) : {}; + } catch { + return {}; + } +} + +const matches = (p: string | RegExp, path: string) => (typeof p === "string" ? path.startsWith(p) : p.test(path)); + +/** 与 tmdb/mock-http 同形的 MockHttp,但路由能拿到 JSON 请求体(camofox 动作都是 POST JSON) */ +async function startJsonMock( + routes: [RegExp, Route][], + opts: { port?: number | undefined; guard?: (req: IncomingMessage) => MockReply | null }, +): Promise { + const calls: MockCall[] = []; + const faults: { pattern: string | RegExp; f: MockFault }[] = []; + const server = createServer(async (req: IncomingMessage, res: ServerResponse) => { + const u = new URL(req.url ?? "/", "http://mock"); + const q = Object.fromEntries(u.searchParams); + const call: MockCall = { method: req.method ?? "GET", path: u.pathname, query: q, headers: req.headers, status: 0 }; + calls.push(call); + const body = await readJson(req); + const reply = (r: MockReply) => { + call.status = r.status ?? 200; + if (res.destroyed) return; + res.writeHead(call.status, { "content-type": "application/json", ...r.headers }); + res.end(r.json === undefined ? "" : JSON.stringify(r.json)); + }; + const fi = faults.findIndex((x) => matches(x.pattern, u.pathname)); + const fault = fi >= 0 ? faults[fi]!.f : undefined; + if (fault) { + fault.times = (fault.times ?? 1) - 1; + if (fault.times <= 0) faults.splice(fi, 1); + if (fault.delayMs) await new Promise((ok) => setTimeout(ok, fault.delayMs)); + if (fault.drop) { + call.status = -1; + req.socket.destroy(); + return; + } + if (fault.status) + return reply({ status: fault.status, json: fault.body, ...(fault.headers ? { headers: fault.headers } : {}) }); + } + const denied = opts.guard?.(req); + if (denied) return reply(denied); + for (const [re, h] of routes) { + const m = re.exec(u.pathname); + if (m) { + try { + return reply(h(m, q, req, body)); + } catch (e) { + return reply({ status: 500, json: { error: String(e) } }); + } + } + } + reply({ status: 404, json: { error: "no route" } }); + }); + await new Promise((ok) => server.listen(opts.port ?? 0, "127.0.0.1", ok)); + const { port } = server.address() as AddressInfo; + return { + url: `http://127.0.0.1:${port}`, + calls, + close: () => + new Promise((ok) => { + server.closeAllConnections(); + server.close(() => ok()); + }), + fault: (pattern, f) => faults.push({ pattern, f: { ...f } }), + clearFaults: () => { + faults.length = 0; + }, + callsTo: (p) => calls.filter((c) => matches(p, c.path)), + }; +} + +export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise { + const tabs = new Map(); + let seq = 0; + const err = (status: number, error: string): MockReply => ({ status, json: { error } }); + const tabFor = (id: string, userId: string | undefined) => { + const t = tabs.get(decodeURIComponent(id)); + return t && t.userId === userId ? t : undefined; + }; + + const mock = await startJsonMock( + [ + [ + /^\/health$/, + () => ({ + json: { + ok: true, + engine: "camoufox", + browserConnected: true, + browserRunning: true, + activeTabs: tabs.size, + activeSessions: new Set([...tabs.values()].map((t) => t.userId)).size, + consecutiveFailures: 0, + }, + }), + ], + [ + /^\/openapi\.json$/, + () => { + const paths: Record> = {}; + for (const [p, ms] of Object.entries(OPENAPI_PATHS)) + for (const m of ms) + if (!opts.omitPaths?.includes(`${m} ${p}`)) { + const ops = paths[p] ?? {}; + ops[m] = { summary: `${m} ${p}` }; + paths[p] = ops; + } + return { + json: { openapi: "3.0.3", info: { title: "camofox-browser", version: opts.version ?? "1.17.0" }, paths }, + }; + }, + ], + [ + /^\/tabs$/, + (_m, q, req, b) => { + if (req.method === "GET") + return { + json: { + running: true, + tabs: [...tabs.values()] + .filter((t) => !q.userId || t.userId === q.userId) + .map((t) => ({ tabId: t.tabId, targetId: t.tabId, url: t.url, title: "", listItemId: t.sessionKey })), + }, + }; + if (typeof b.userId !== "string" || typeof b.sessionKey !== "string") + return err(400, "userId and sessionKey required"); + const tabId = `tab-${++seq}`; + const url = typeof b.url === "string" ? b.url : "about:blank"; + tabs.set(tabId, { tabId, userId: b.userId, sessionKey: b.sessionKey, url }); + return { json: { tabId, url, httpStatus: b.url ? 200 : null, navigationOk: true } }; + }, + ], + [ + /^\/tabs\/([^/]+)\/navigate$/, + (m, _q, _req, b) => { + const t = tabFor(m[1]!, b.userId as string | undefined); + if (!t) return err(404, "Tab not found"); + if (typeof b.url !== "string") return err(400, "url required"); + // 模拟站点重定向:?redirect= 落到另一地址 + const target = new URL(b.url).searchParams.get("redirect") ?? b.url; + t.url = target; + const httpStatus = target.includes("/missing") ? 404 : 200; + return { json: { ok: true, tabId: t.tabId, url: target, httpStatus, navigationOk: httpStatus < 400 } }; + }, + ], + [ + /^\/tabs\/([^/]+)\/snapshot$/, + (m, q) => { + const t = tabFor(m[1]!, q.userId); + if (!t) return err(404, "Tab not found"); + const snapshot = (opts.snapshotText ?? CAMOFOX_FIXTURE_SNAPSHOT).replaceAll("{url}", t.url); + const offset = Number(q.offset ?? 0); + return { + json: { + url: t.url, + snapshot: snapshot.slice(offset), + structure: { + forms: [ + { + action: "takelogin.php", + fields: [ + { name: "username", type: "text", value: "fixture-user" }, + { name: "password", type: "password", value: CAMOFOX_FIXTURE_PASSWORD }, + ], + }, + ], + }, + refsCount: new Set(snapshot.match(/\[[a-z]+ (e\d+)/g) ?? []).size, + truncated: false, + totalChars: snapshot.length, + hasMore: false, + }, + }; + }, + ], + [ + /^\/tabs\/([^/]+)\/screenshot$/, + (m, q) => { + const t = tabFor(m[1]!, q.userId); + if (!t) return err(404, "Tab not found"); + return { json: { screenshot: { data: PNG_1X1_BASE64, mimeType: "image/png" } } }; + }, + ], + [ + /^\/sessions\/([^/]+)$/, + (m) => { + const userId = decodeURIComponent(m[1]!); + let closed = 0; + for (const [id, t] of tabs) + if (t.userId === userId) { + tabs.delete(id); + closed++; + } + return { json: { ok: true, closed } }; + }, + ], + ], + { + port: opts.port, + guard: (req) => { + const path = new URL(req.url ?? "/", "http://mock").pathname; + if (!opts.accessKey || path === "/health") return null; + return req.headers.authorization === `Bearer ${opts.accessKey}` ? null : err(401, "Unauthorized"); + }, + }, + ); + return Object.assign(mock, { tabs }); +} diff --git a/packages/testkit/src/index.ts b/packages/testkit/src/index.ts index 520ba70b..aaaeefcb 100644 --- a/packages/testkit/src/index.ts +++ b/packages/testkit/src/index.ts @@ -1,5 +1,6 @@ // @umt/testkit 公开入口;W1 起在此导出实现 export const PACKAGE_NAME = "@umt/testkit"; +export * from "./browser/index.ts"; export * from "./emby/index.ts"; export * from "./fs/index.ts"; export * from "./qbittorrent/index.ts";