feat(browser): 插件真实坐标点击与页面内容读取(CF-2)
- umt 插件 POST /umt/tabs/{id}/mouse-click:page.mouse.click,跨域 iframe 的 Turnstile 也点得到;click 坐标优先走它,旧镜像退回 elementFromPoint
- 适配器 pageContent(地址 / 标题 / outerHTML / 正文 / navigator.userAgent);setEgress 返回插件出口模式
- mock camofox 增加 Cloudflare 模式(auto / interactive / captcha)与插件鼠标点击
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
a11ef16e40
commit
6b14b0da45
7 files changed
+402
-10
No files matched your search
@@ -22,6 +22,7 @@ type Any = any;
|
||||
|
||||
const ORIGIN = "https://audiences.example";
|
||||
const ORIGIN_2FA = "https://secure.example";
|
||||
const ORIGIN_CF = "https://cf.example";
|
||||
const USER = "fixture-user";
|
||||
const PASSWORD = "e14-login-pw";
|
||||
const ACCESS_KEY = "camofox-e14-key";
|
||||
@@ -30,6 +31,7 @@ const ACCESS_KEY = "camofox-e14-key";
|
||||
const AT = { username: { x: 240, y: 64 }, password: { x: 240, y: 96 } };
|
||||
|
||||
let sid = "";
|
||||
let cfSid = "";
|
||||
const childResults = new Map<string, FakeToolResult[]>();
|
||||
let mainResults: FakeToolResult[] = [];
|
||||
|
||||
@@ -37,6 +39,16 @@ const typeUser = () => ({ sessionId: sid, ...AT.username, text: USER, allowSensi
|
||||
const typePassword = () => ({ sessionId: sid, ...AT.password, text: PASSWORD, pressEnter: true, allowSensitive: true });
|
||||
|
||||
const scripts: FakeScript[] = [
|
||||
{
|
||||
// CF-2 系统发起的视觉子流程(没有父会话):截图 → 点 Turnstile 复选框(第 2 行,截图坐标 (240, 64))
|
||||
name: "e14-cf",
|
||||
match: "Cloudflare 人机验证页",
|
||||
steps: [
|
||||
{ toolCalls: [{ capability: "browser.screenshot", input: () => ({ sessionId: cfSid }) }] },
|
||||
{ toolCalls: [{ capability: "browser.click_at", input: () => ({ sessionId: cfSid, x: 240, y: 64 }) }] },
|
||||
],
|
||||
finalText: () => "已点复选框",
|
||||
},
|
||||
{
|
||||
name: "e14-main",
|
||||
match: "E14 主会话登录",
|
||||
@@ -122,6 +134,7 @@ describe.skipIf(!hasPg)("E14 视觉自动化闭环", () => {
|
||||
let e: E2E;
|
||||
let mock: MockCamofox;
|
||||
let mock2fa: MockCamofox;
|
||||
let mockCf: MockCamofox;
|
||||
let evidenceDir: string;
|
||||
let svc: BrowserService;
|
||||
const ids = { site: "", site2fa: "", provider: "", provider2fa: "" };
|
||||
@@ -154,8 +167,16 @@ describe.skipIf(!hasPg)("E14 视觉自动化闭环", () => {
|
||||
site: { ...site(ORIGIN_2FA), twoFactor: true },
|
||||
viewport: { width: 1600, height: 1000 },
|
||||
});
|
||||
mockCf = await createMockCamofox({
|
||||
accessKey: ACCESS_KEY,
|
||||
site: { ...site(ORIGIN_CF), protectedPaths: /^$/, cloudflare: { mode: "interactive", clearance: "clr-e14" } },
|
||||
viewport: { width: 1600, height: 1000 },
|
||||
});
|
||||
evidenceDir = await mkdtemp(join(tmpdir(), "umt-e14-evidence-"));
|
||||
e = await createE2E("e14", { fakeScripts: scripts, browser: { evidenceDir, settleMs: 0 } });
|
||||
e = await createE2E("e14", {
|
||||
fakeScripts: scripts,
|
||||
browser: { evidenceDir, settleMs: 0, challengePollMs: 10, challengeAutoWaitMs: 200 },
|
||||
});
|
||||
svc = browserServiceFor(e.booted.ctx)!;
|
||||
const ctx = e.booted.ctx;
|
||||
const keyRef = await ctx.secrets.create({ purpose: "other", plaintext: ACCESS_KEY });
|
||||
@@ -200,6 +221,7 @@ describe.skipIf(!hasPg)("E14 视觉自动化闭环", () => {
|
||||
await e?.close();
|
||||
await mock?.close();
|
||||
await mock2fa?.close();
|
||||
await mockCf?.close();
|
||||
if (evidenceDir) await rm(evidenceDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
@@ -335,9 +357,14 @@ describe.skipIf(!hasPg)("E14 视觉自动化闭环", () => {
|
||||
// 坐标换算:截图 (240, 64) → 页面 (300, 80);(240, 96) → (300, 120)
|
||||
expect(r[1]?.json).toMatchObject({ pageX: 300, pageY: 80 });
|
||||
expect(r[2]?.json).toMatchObject({ pageX: 300, pageY: 120 });
|
||||
const evals = mock.actions.filter((a) => a.action === "evaluate").map((a) => String(a.body.expression));
|
||||
expect(evals.some((x) => x.includes("elementFromPoint(300, 80)"))).toBe(true);
|
||||
expect(evals.some((x) => x.includes("elementFromPoint(300, 120)"))).toBe(true);
|
||||
// CF-2:坐标点击走 umt 插件的真实鼠标事件(按页面 CSS 像素)
|
||||
const clicks = mock.actions.filter((a) => a.action === "mouse-click").map((a) => [a.body.x, a.body.y]);
|
||||
expect(clicks).toEqual(
|
||||
expect.arrayContaining([
|
||||
[300, 80],
|
||||
[300, 120],
|
||||
]),
|
||||
);
|
||||
expect(mock.loggedIn.size).toBe(1);
|
||||
// 父会话事件流里有子流程
|
||||
const events = await runEvents(e, runId);
|
||||
@@ -372,4 +399,72 @@ describe.skipIf(!hasPg)("E14 视觉自动化闭环", () => {
|
||||
expect(again).not.toBe(sid);
|
||||
await worker.close(again);
|
||||
}, 60_000);
|
||||
|
||||
it("CF-2 站点请求遇 Turnstile:系统发起视觉子流程(没有父会话,预算取设置)点复选框,浏览器读回 cf_clearance", async () => {
|
||||
// 只留 Cloudflare 这个浏览器服务(站点请求一侧不指定服务,取启用的那个)
|
||||
await e.db.updateTable("browser_provider").set({ enabled: false }).execute();
|
||||
const providerCf = (
|
||||
await e.db
|
||||
.insertInto("browser_provider")
|
||||
.values({
|
||||
name: "camofox-e14-cf",
|
||||
kind: "camofox",
|
||||
base_url: mockCf.url,
|
||||
secret_ref: (
|
||||
await e.db.selectFrom("browser_provider").select("secret_ref").executeTakeFirstOrThrow()
|
||||
).secret_ref,
|
||||
enabled: true,
|
||||
config: JSON.stringify({ navigationTimeoutMs: 2_000, maxRetries: 0 }),
|
||||
})
|
||||
.returning("id")
|
||||
.executeTakeFirstOrThrow()
|
||||
).id;
|
||||
const siteCf = (
|
||||
await e.db
|
||||
.insertInto("site")
|
||||
.values({
|
||||
key: "audiences",
|
||||
name: "观众",
|
||||
adapter: "nexusphp",
|
||||
base_url: `${ORIGIN_CF}/`,
|
||||
rate_limit: JSON.stringify({ minIntervalMs: 0, concurrency: 5 }),
|
||||
})
|
||||
.returning("id")
|
||||
.executeTakeFirstOrThrow()
|
||||
).id;
|
||||
const ref = await e.booted.ctx.secrets.create({ purpose: "site_cookie", plaintext: "c_secure_pass=e14" });
|
||||
const accountId = (
|
||||
await e.db
|
||||
.insertInto("site_account")
|
||||
.values({ site_id: siteCf, label: "main", auth_kind: "cookie", secret_ref: ref, status: "active" })
|
||||
.returning("id")
|
||||
.executeTakeFirstOrThrow()
|
||||
).id;
|
||||
cfSid = `umt-site-${siteCf}-acct-${accountId}`; // 先占位,打开后换成会话 ID
|
||||
const pending = svc.solveSiteChallenge({ siteId: siteCf, accountId, url: `${ORIGIN_CF}/torrents.php` });
|
||||
// 会话打开后脚本才知道 sessionId
|
||||
await waitFor(
|
||||
async () => {
|
||||
const live = await svc.liveSessionFor(siteCf, accountId);
|
||||
if (live) cfSid = live.id;
|
||||
return !!live;
|
||||
},
|
||||
10_000,
|
||||
"Cloudflare 会话打开",
|
||||
10,
|
||||
);
|
||||
const out = await pending;
|
||||
expect(out).toMatchObject({ status: "passed", by: "vision", clearance: "clr-e14" });
|
||||
expect(mockCf.actions.some((a) => a.action === "mouse-click" && a.body.x === 300 && a.body.y === 80)).toBe(true);
|
||||
const child = await e.db
|
||||
.selectFrom("agent_run")
|
||||
.selectAll()
|
||||
.where("spawn_reason", "=", "browser.solve_challenge")
|
||||
.where("created_by", "=", "system:cloudflare")
|
||||
.executeTakeFirstOrThrow();
|
||||
expect(child).toMatchObject({ trigger: "event", parent_run_id: null, status: "succeeded" });
|
||||
expect((child.budget as Any).maxToolCalls).toBe(4);
|
||||
expect((child.budget as Any).maxDurationSec).toBeLessThanOrEqual(120);
|
||||
expect(providerCf).toBeTruthy();
|
||||
}, 60_000);
|
||||
});
|
||||
Reference in new issue
Block a user