import { existsSync } from "node:fs"; import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { FakeScript, FakeToolResult, InvokeContext } from "@umt/agent"; import { createMockCamofox, type MockCamofox } from "@umt/testkit"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { browserServiceFor } from "../../src/modules/browser/index.ts"; import { BrowserService } from "../../src/modules/browser/service.ts"; import { createE2E, type E2E, pgAvailable, runEvents, runRow, waitFor } from "./harness.ts"; // E14 视觉自动化闭环(VIS-1):fake 视觉模型(脚本返回固定坐标)+ testkit mock camofox(视口 1600×1000、DPR 1, // 截图缩到 1280×800 交给模型,坐标按 0.8 换算回页面)。 // - 主会话调用 browser.solve_challenge:子流程 截图 → 点用户名框输入 → 点密码框输入并回车 → 成功条件(地址含 /torrents.php)满足; // - 二次验证页:服务端识别后会话立即转等待处理,子流程后续动作被拒,不输入短信验证码; // - 步数超限:超出 maxSteps 转等待处理(challenge_unsolved),待处理事项附证据截图; // - 敏感截图(登录页 / 密码框)默认不交给模型,改给已脱敏的文字快照。 const hasPg = await pgAvailable(); // biome-ignore lint/suspicious/noExplicitAny: 测试断言直接读 JSON type Any = any; const ORIGIN = "https://audiences.example"; const ORIGIN_2FA = "https://secure.example"; const USER = "fixture-user"; const PASSWORD = "e14-login-pw"; const ACCESS_KEY = "camofox-e14-key"; /** 登录页各行在 1280×800 截图上的坐标(页面 CSS 行中心 y = 40 + 40i,x = 300;×0.8) */ const AT = { username: { x: 240, y: 64 }, password: { x: 240, y: 96 } }; let sid = ""; const childResults = new Map(); let mainResults: FakeToolResult[] = []; const typeUser = () => ({ sessionId: sid, ...AT.username, text: USER, allowSensitive: true }); const typePassword = () => ({ sessionId: sid, ...AT.password, text: PASSWORD, pressEnter: true, allowSensitive: true }); const scripts: FakeScript[] = [ { name: "e14-main", match: "E14 主会话登录", steps: [ { toolCalls: [ { capability: "browser.solve_challenge", input: () => ({ sessionId: sid, goal: "E14-登录 用测试账号登录站点", success: { urlContains: "/torrents.php" }, maxSteps: 6, allowSensitive: true, }), }, ], }, ], finalText: (r) => { mainResults = r; return `挑战处理结果:${(r[0]?.json as Any)?.status ?? r[0]?.text}`; }, }, { name: "e14-login", match: "E14-登录", steps: [ { toolCalls: [{ capability: "browser.screenshot", input: () => ({ sessionId: sid, allowSensitive: true }) }] }, { toolCalls: [{ capability: "browser.type_at", input: typeUser }] }, { toolCalls: [{ capability: "browser.type_at", input: typePassword }] }, ], finalText: (r) => { childResults.set("login", r); return "已提交登录表单"; }, }, { name: "e14-2fa", match: "E14-二次验证", steps: [ { toolCalls: [{ capability: "browser.screenshot", input: () => ({ sessionId: sid, allowSensitive: true }) }] }, { toolCalls: [{ capability: "browser.type_at", input: typeUser }] }, { toolCalls: [{ capability: "browser.type_at", input: typePassword }] }, // 模型不听话继续输短信验证码:服务端已转等待处理,必须被拒 { toolCalls: [ { capability: "browser.type_at", input: () => ({ sessionId: sid, x: 240, y: 96, text: "123456" }) }, ], }, ], finalText: (r) => { childResults.set("2fa", r); return "NEEDS_HUMAN: 需要短信验证码"; }, }, { name: "e14-steps", match: "E14-步数", steps: [ { toolCalls: [{ capability: "browser.screenshot", input: () => ({ sessionId: sid }) }] }, { toolCalls: [{ capability: "browser.scroll", input: () => ({ sessionId: sid, dy: 300 }) }] }, { toolCalls: [{ capability: "browser.wait", input: () => ({ sessionId: sid, ms: 50 }) }] }, { toolCalls: [{ capability: "browser.wait", input: () => ({ sessionId: sid, ms: 50 }) }] }, ], finalText: (r) => { childResults.set("steps", r); return "还没完成"; }, }, { name: "e14-sensitive", match: "E14-敏感", steps: [{ toolCalls: [{ capability: "browser.screenshot", input: () => ({ sessionId: sid }) }] }], finalText: (r) => { childResults.set("sensitive", r); return "NEEDS_HUMAN: 截图不可见,登录页需要人工"; }, }, ]; describe.skipIf(!hasPg)("E14 视觉自动化闭环", () => { let e: E2E; let mock: MockCamofox; let mock2fa: MockCamofox; let evidenceDir: string; let svc: BrowserService; const ids = { site: "", site2fa: "", provider: "", provider2fa: "" }; const agentCtx = (): InvokeContext => ({ actorId: "agent:e14", taskId: crypto.randomUUID(), scopeIds: [], signal: new AbortController().signal, source: "agent", invocationId: crypto.randomUUID(), }); const solve = (input: Record) => e.booted.ctx.capabilities.invoke("browser.solve_challenge", input, agentCtx()) as Promise; const open = async (siteId: string, providerId: string, url: string) => { for (const s of await svc.list({ siteId })) if (s.controller !== "closed") await svc.close(s.id); sid = await svc.open({ siteId, providerId, url }); return sid; }; beforeAll(async () => { const site = (origin: string) => ({ origin, username: USER, password: PASSWORD }); mock = await createMockCamofox({ accessKey: ACCESS_KEY, site: site(ORIGIN), viewport: { width: 1600, height: 1000 }, }); mock2fa = await createMockCamofox({ accessKey: ACCESS_KEY, site: { ...site(ORIGIN_2FA), twoFactor: true }, viewport: { width: 1600, height: 1000 }, }); evidenceDir = await mkdtemp(join(tmpdir(), "umt-e14-evidence-")); e = await createE2E("e14", { fakeScripts: scripts, browser: { evidenceDir, settleMs: 0 } }); svc = browserServiceFor(e.booted.ctx)!; const ctx = e.booted.ctx; const keyRef = await ctx.secrets.create({ purpose: "other", plaintext: ACCESS_KEY }); const provider = async (name: string, url: string) => ( await e.db .insertInto("browser_provider") .values({ name, kind: "camofox", base_url: url, secret_ref: keyRef, enabled: true, config: JSON.stringify({ navigationTimeoutMs: 2_000, maxRetries: 0 }), }) .returning("id") .executeTakeFirstOrThrow() ).id; ids.provider = await provider("camofox-e14", mock.url); ids.provider2fa = await provider("camofox-e14-2fa", mock2fa.url); const addSite = async (key: string, origin: string) => ( await e.db .insertInto("site") .values({ key, name: key, adapter: "nexusphp", base_url: `${origin}/`, rate_limit: JSON.stringify({ minIntervalMs: 0, concurrency: 5 }), }) .returning("id") .executeTakeFirstOrThrow() ).id; ids.site = await addSite("e14site", ORIGIN); ids.site2fa = await addSite("e14secure", ORIGIN_2FA); // 坐标操作与视觉子流程:browser.interact 持续授权覆盖(敏感截图授权在主会话用例前再发) await e.db.insertInto("automation_grant").values({ action: "browser.interact", granted_by: "e14" }).execute(); }, 120_000); afterAll(async () => { await e?.close(); await mock?.close(); await mock2fa?.close(); if (evidenceDir) await rm(evidenceDir, { recursive: true, force: true }); }); it("二次验证页:服务端识别后立即转等待处理,不输入短信验证码", async () => { await open(ids.site2fa, ids.provider2fa, `${ORIGIN_2FA}/torrents.php`); const out = await solve({ sessionId: sid, goal: "E14-二次验证 登录站点", success: { urlContains: "/torrents.php" }, maxSteps: 8, allowSensitive: true, }); expect(out).toMatchObject({ status: "needs_human", controller: "waiting" }); expect(out.reason).toContain("二次验证"); expect(out.takeover).toEqual({ sessionId: sid, siteId: ids.site2fa, path: `/console/sites/${ids.site2fa}?tab=origin`, }); const r = childResults.get("2fa")!; // 输完密码回车后的截图:识别到二次验证,不给图片、会话已转等待 expect(r[2]?.json).toMatchObject({ controller: "waiting", waitingReason: "two_factor", imageForModel: false }); expect(r[2]?.images).toBe(0); // 第 4 步(输短信验证码)被拒 expect(r[3]?.isError).toBe(true); expect(r[3]?.text).toContain("等待人工处理"); expect(mock2fa.actions.some((a) => a.action === "type" && a.body.text === "123456")).toBe(false); const view = await svc.view(sid); expect(view).toMatchObject({ controller: "waiting", waitingReason: "two_factor" }); const d = await e.db.selectFrom("decision").selectAll().where("id", "=", out.decisionId).executeTakeFirstOrThrow(); expect(d).toMatchObject({ status: "awaiting_user", subject_type: "browser_session", subject_id: sid }); expect((d.evidence as Any).challenge.screenshots.length).toBeGreaterThanOrEqual(3); expect((d.evidence as Any).challenge.childRunId).toBe(out.childRunId); for (const ev of out.evidence) expect(existsSync(join(evidenceDir, `${ev.id}.jpg`))).toBe(true); // 二次验证等待不能被视觉子流程接手 await expect(solve({ sessionId: sid, goal: "E14-二次验证 再试", success: { urlContains: "/x" } })).rejects.toThrow( /等待人工处理/, ); }, 60_000); it("步数超限:转等待处理(challenge_unsolved),待处理事项附证据截图", async () => { await open(ids.site, ids.provider, `${ORIGIN}/torrents.php`); const out = await solve({ sessionId: sid, goal: "E14-步数 处理页面", success: { textPresent: "种子列表" }, maxSteps: 3, }); expect(out).toMatchObject({ status: "exhausted", steps: 3, controller: "waiting" }); expect(out.reason).toContain("工具调用次数超出预算"); const child = await runRow(e, out.childRunId); expect(child).toMatchObject({ status: "budget_exceeded", spawn_reason: "browser.solve_challenge", parent_run_id: null, }); const events = await runEvents(e, out.childRunId); const ends = events.filter((x) => x.type === "tool_call_end"); expect(ends.length).toBe(3); for (const x of ends) expect((x.payload as Any).outputSummary).toMatch(/evidence\/[0-9a-f-]{36}\.jpg/); expect(events.some((x) => x.type === "tool_call_rejected")).toBe(true); const view = await svc.view(sid); expect(view).toMatchObject({ controller: "waiting", waitingReason: "challenge_unsolved" }); const d = await e.db.selectFrom("decision").selectAll().where("id", "=", out.decisionId).executeTakeFirstOrThrow(); expect((d.evidence as Any).challenge.screenshots.length).toBe(3); }, 60_000); it("敏感截图默认不交给模型:登录页改给已脱敏文字快照;未解决的挑战可以再次接手", async () => { // 上一用例留下的 challenge_unsolved 等待:视觉子流程可以接手 const out = await solve({ sessionId: sid, goal: "E14-敏感 看看登录页", success: { urlContains: "/torrents.php" }, maxSteps: 3, }); expect(out).toMatchObject({ status: "needs_human", controller: "waiting" }); expect(out.reason).toContain("模型判定需要人工"); const shot = childResults.get("sensitive")![0]!; expect(shot.images).toBe(0); expect(shot.json).toMatchObject({ sensitive: true, imageForModel: false }); // modelImage 由运行时摘出,不出现在 JSON 文本里 expect("modelImage" in (shot.json as Any)).toBe(false); expect((shot.json as Any).note).toContain("敏感"); expect((shot.json as Any).textSnapshot).toContain("用户名"); expect((shot.json as Any).evidence.ref).toMatch(/^evidence\//); }, 60_000); it("主会话 → solve_challenge:截图 → 点输入框 → 输入 → 回车提交 → 成功条件满足", async () => { await e.db .insertInto("automation_grant") .values({ action: "browser.sensitive_screenshot", granted_by: "e14" }) .execute(); await open(ids.site, ids.provider, `${ORIGIN}/torrents.php`); const created = await e.api("POST", "/agent/runs", { goal: "E14 主会话登录 观众站点" }); expect(created.status).toBe(201); const runId = created.body.data.id as string; const done = await waitFor( async () => { const r = await runRow(e, runId); return ["succeeded", "failed"].includes(r.status) ? r : null; }, 60_000, "主会话结束", ); expect(done.summary).toContain("挑战处理结果:solved"); const out = mainResults[0]?.json as Any; expect(out).toMatchObject({ status: "solved", steps: 3, controller: "automation", decisionId: null, takeover: null, }); expect(out.url).toContain("/torrents.php"); // 子流程:父会话 = 主会话;每步截图都作为图片交给了模型(已授权敏感截图) const child = await runRow(e, out.childRunId); expect(child).toMatchObject({ parent_run_id: runId, trigger: "subagent", status: "succeeded" }); const r = childResults.get("login")!; expect(r.map((x) => x.images)).toEqual([1, 1, 1]); expect(r[0]?.json).toMatchObject({ sensitive: true, imageForModel: true, coordinates: { imageWidth: 1280, imageHeight: 800, viewportWidth: 1600, viewportHeight: 1000, devicePixelRatio: 1, }, }); // 模型图片数据不进 JSON 文本 expect(r[0]?.text).not.toContain('"data"'); // 坐标换算:截图 (240, 64) → 页面 (300, 80);(240, 96) → (300, 120) expect(r[1]?.json).toMatchObject({ pageX: 300, pageY: 80 }); expect(r[2]?.json).toMatchObject({ pageX: 300, pageY: 120 }); const evals = mock.actions.filter((a) => a.action === "evaluate").map((a) => String(a.body.expression)); expect(evals.some((x) => x.includes("elementFromPoint(300, 80)"))).toBe(true); expect(evals.some((x) => x.includes("elementFromPoint(300, 120)"))).toBe(true); expect(mock.loggedIn.size).toBe(1); // 父会话事件流里有子流程 const events = await runEvents(e, runId); expect(events.some((x) => x.type === "subagent_spawned" && (x.payload as Any).childRunId === out.childRunId)).toBe( true, ); // 账本不存图片 const ledger = await e.db .selectFrom("capability_invocation") .select(["capability_id", "output"]) .where("agent_run_id", "=", out.childRunId) .execute(); expect(ledger.length).toBe(2); for (const l of ledger) expect((l.output as Any).modelImage).toBeNull(); }, 90_000); it("多进程:api 自愈重开被回收的标签页后 worker 换到新标签页;api 关闭会话后 worker 能重新打开", async () => { await open(ids.site, ids.provider, `${ORIGIN}/torrents.php`); const worker = new BrowserService(e.booted.ctx, svc.opts); const auto = { kind: "automation", holder: "agent:e14" } as const; const snap = (s: BrowserService) => s.run(sid, auto, (p, ref) => p.snapshot(ref), { write: false, persist: false }); await snap(worker); const old = (await svc.view(sid)).currentTabId!; mock.dropTab(old); // BR-3:api 发现标签页被回收 → 同一会话新开标签页回到原地址并重试 await expect(snap(svc)).resolves.toMatchObject({ url: expect.stringContaining("/torrents.php") }); expect((await svc.view(sid)).currentTabId).not.toBe(old); await expect(snap(worker)).resolves.toMatchObject({ url: expect.stringContaining("/torrents.php") }); // api 关闭会话后,worker 能按站点重新打开(不被内存里的旧会话挡住) await svc.close(sid); const again = await worker.open({ siteId: ids.site, providerId: ids.provider, url: `${ORIGIN}/torrents.php` }); expect(again).not.toBe(sid); await worker.close(again); }, 60_000); });