UHD_Media_Tool/apps/server/test/e2e/vision.test.ts

import { existsSync } from "node:fs";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { FakeScript, FakeToolResult, InvokeContext } from "@umt/agent";
import { createMockCamofox, type MockCamofox } from "@umt/testkit";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { browserServiceFor } from "../../src/modules/browser/index.ts";
import { BrowserService } from "../../src/modules/browser/service.ts";
import { createE2E, type E2E, pgAvailable, runEvents, runRow, waitFor } from "./harness.ts";

// E14 视觉自动化闭环(VIS-1):fake 视觉模型(脚本返回固定坐标)+ testkit mock camofox(视口 1600×1000、DPR 1,
// 截图缩到 1280×800 交给模型,坐标按 0.8 换算回页面)。
// - 主会话调用 browser.solve_challenge:子流程 截图 → 点用户名框输入 → 点密码框输入并回车 → 成功条件(地址含 /torrents.php)满足;
// - 二次验证页:服务端识别后会话立即转等待处理,子流程后续动作被拒,不输入短信验证码;
// - 步数超限:超出 maxSteps 转等待处理(challenge_unsolved),待处理事项附证据截图;
// - 敏感截图(登录页 / 密码框)默认不交给模型,改给已脱敏的文字快照。

const hasPg = await pgAvailable();
// biome-ignore lint/suspicious/noExplicitAny: 测试断言直接读 JSON
type Any = any;

const ORIGIN = "https://audiences.example";
const ORIGIN_2FA = "https://secure.example";
const ORIGIN_CF = "https://cf.example";
const USER = "fixture-user";
const PASSWORD = "e14-login-pw";
const ACCESS_KEY = "camofox-e14-key";

/** 登录页各行在 1280×800 截图上的坐标(页面 CSS 行中心 y = 40 + 40i,x = 300;×0.8) */
const AT = { username: { x: 240, y: 64 }, password: { x: 240, y: 96 } };

let sid = "";
let cfSid = "";
const childResults = new Map<string, FakeToolResult[]>();
let mainResults: FakeToolResult[] = [];

const typeUser = () => ({ sessionId: sid, ...AT.username, text: USER, allowSensitive: true });
const typePassword = () => ({ sessionId: sid, ...AT.password, text: PASSWORD, pressEnter: true, allowSensitive: true });

const scripts: FakeScript[] = [
  {
    // CF-2 系统发起的视觉子流程(没有父会话):截图 → 点 Turnstile 复选框(第 2 行,截图坐标 (240, 64))
    name: "e14-cf",
    match: "Cloudflare 人机验证页",
    steps: [
      { toolCalls: [{ capability: "browser.screenshot", input: () => ({ sessionId: cfSid }) }] },
      { toolCalls: [{ capability: "browser.click_at", input: () => ({ sessionId: cfSid, x: 240, y: 64 }) }] },
    ],
    finalText: () => "已点复选框",
  },
  {
    name: "e14-main",
    match: "E14 主会话登录",
    steps: [
      {
        toolCalls: [
          {
            capability: "browser.solve_challenge",
            input: () => ({
              sessionId: sid,
              goal: "E14-登录 用测试账号登录站点",
              success: { urlContains: "/torrents.php" },
              maxSteps: 6,
              allowSensitive: true,
            }),
          },
        ],
      },
    ],
    finalText: (r) => {
      mainResults = r;
      return `挑战处理结果:${(r[0]?.json as Any)?.status ?? r[0]?.text}`;
    },
  },
  {
    name: "e14-login",
    match: "E14-登录",
    steps: [
      { toolCalls: [{ capability: "browser.screenshot", input: () => ({ sessionId: sid, allowSensitive: true }) }] },
      { toolCalls: [{ capability: "browser.type_at", input: typeUser }] },
      { toolCalls: [{ capability: "browser.type_at", input: typePassword }] },
    ],
    finalText: (r) => {
      childResults.set("login", r);
      return "已提交登录表单";
    },
  },
  {
    name: "e14-2fa",
    match: "E14-二次验证",
    steps: [
      { toolCalls: [{ capability: "browser.screenshot", input: () => ({ sessionId: sid, allowSensitive: true }) }] },
      { toolCalls: [{ capability: "browser.type_at", input: typeUser }] },
      { toolCalls: [{ capability: "browser.type_at", input: typePassword }] },
      // 模型不听话继续输短信验证码:服务端已转等待处理,必须被拒
      {
        toolCalls: [
          { capability: "browser.type_at", input: () => ({ sessionId: sid, x: 240, y: 96, text: "123456" }) },
        ],
      },
    ],
    finalText: (r) => {
      childResults.set("2fa", r);
      return "NEEDS_HUMAN: 需要短信验证码";
    },
  },
  {
    name: "e14-steps",
    match: "E14-步数",
    steps: [
      { toolCalls: [{ capability: "browser.screenshot", input: () => ({ sessionId: sid }) }] },
      { toolCalls: [{ capability: "browser.scroll", input: () => ({ sessionId: sid, dy: 300 }) }] },
      { toolCalls: [{ capability: "browser.wait", input: () => ({ sessionId: sid, ms: 50 }) }] },
      { toolCalls: [{ capability: "browser.wait", input: () => ({ sessionId: sid, ms: 50 }) }] },
    ],
    finalText: (r) => {
      childResults.set("steps", r);
      return "还没完成";
    },
  },
  {
    name: "e14-sensitive",
    match: "E14-敏感",
    steps: [{ toolCalls: [{ capability: "browser.screenshot", input: () => ({ sessionId: sid }) }] }],
    finalText: (r) => {
      childResults.set("sensitive", r);
      return "NEEDS_HUMAN: 截图不可见，登录页需要人工";
    },
  },
];

describe.skipIf(!hasPg)("E14 视觉自动化闭环", () => {
  let e: E2E;
  let mock: MockCamofox;
  let mock2fa: MockCamofox;
  let mockCf: MockCamofox;
  let evidenceDir: string;
  let svc: BrowserService;
  const ids = { site: "", site2fa: "", provider: "", provider2fa: "" };

  const agentCtx = (): InvokeContext => ({
    actorId: "agent:e14",
    taskId: crypto.randomUUID(),
    scopeIds: [],
    signal: new AbortController().signal,
    source: "agent",
    invocationId: crypto.randomUUID(),
  });
  const solve = (input: Record<string, unknown>) =>
    e.booted.ctx.capabilities.invoke("browser.solve_challenge", input, agentCtx()) as Promise<Any>;
  const open = async (siteId: string, providerId: string, url: string) => {
    for (const s of await svc.list({ siteId })) if (s.controller !== "closed") await svc.close(s.id);
    sid = await svc.open({ siteId, providerId, url });
    return sid;
  };

  beforeAll(async () => {
    const site = (origin: string) => ({ origin, username: USER, password: PASSWORD });
    mock = await createMockCamofox({
      accessKey: ACCESS_KEY,
      site: site(ORIGIN),
      viewport: { width: 1600, height: 1000 },
    });
    mock2fa = await createMockCamofox({
      accessKey: ACCESS_KEY,
      site: { ...site(ORIGIN_2FA), twoFactor: true },
      viewport: { width: 1600, height: 1000 },
    });
    mockCf = await createMockCamofox({
      accessKey: ACCESS_KEY,
      site: { ...site(ORIGIN_CF), protectedPaths: /^$/, cloudflare: { mode: "interactive", clearance: "clr-e14" } },
      viewport: { width: 1600, height: 1000 },
    });
    evidenceDir = await mkdtemp(join(tmpdir(), "umt-e14-evidence-"));
    e = await createE2E("e14", {
      fakeScripts: scripts,
      browser: { evidenceDir, settleMs: 0, challengePollMs: 10, challengeAutoWaitMs: 200 },
    });
    svc = browserServiceFor(e.booted.ctx)!;
    const ctx = e.booted.ctx;
    const keyRef = await ctx.secrets.create({ purpose: "other", plaintext: ACCESS_KEY });
    const provider = async (name: string, url: string) =>
      (
        await e.db
          .insertInto("browser_provider")
          .values({
            name,
            kind: "camofox",
            base_url: url,
            secret_ref: keyRef,
            enabled: true,
            config: JSON.stringify({ navigationTimeoutMs: 2_000, maxRetries: 0 }),
          })
          .returning("id")
          .executeTakeFirstOrThrow()
      ).id;
    ids.provider = await provider("camofox-e14", mock.url);
    ids.provider2fa = await provider("camofox-e14-2fa", mock2fa.url);
    const addSite = async (key: string, origin: string) =>
      (
        await e.db
          .insertInto("site")
          .values({
            key,
            name: key,
            adapter: "nexusphp",
            base_url: `${origin}/`,
            rate_limit: JSON.stringify({ minIntervalMs: 0, concurrency: 5 }),
          })
          .returning("id")
          .executeTakeFirstOrThrow()
      ).id;
    ids.site = await addSite("e14site", ORIGIN);
    ids.site2fa = await addSite("e14secure", ORIGIN_2FA);
    // 坐标操作与视觉子流程:browser.interact 持续授权覆盖(敏感截图授权在主会话用例前再发)
    await e.db.insertInto("automation_grant").values({ action: "browser.interact", granted_by: "e14" }).execute();
  }, 120_000);

  afterAll(async () => {
    await e?.close();
    await mock?.close();
    await mock2fa?.close();
    await mockCf?.close();
    if (evidenceDir) await rm(evidenceDir, { recursive: true, force: true });
  });

  it("二次验证页:服务端识别后立即转等待处理,不输入短信验证码", async () => {
    await open(ids.site2fa, ids.provider2fa, `${ORIGIN_2FA}/torrents.php`);
    const out = await solve({
      sessionId: sid,
      goal: "E14-二次验证 登录站点",
      success: { urlContains: "/torrents.php" },
      maxSteps: 8,
      allowSensitive: true,
    });
    expect(out).toMatchObject({ status: "needs_human", controller: "waiting" });
    expect(out.reason).toContain("二次验证");
    expect(out.takeover).toEqual({
      sessionId: sid,
      siteId: ids.site2fa,
      path: `/console/sites/${ids.site2fa}?tab=origin`,
    });
    const r = childResults.get("2fa")!;
    // 输完密码回车后的截图:识别到二次验证,不给图片、会话已转等待
    expect(r[2]?.json).toMatchObject({ controller: "waiting", waitingReason: "two_factor", imageForModel: false });
    expect(r[2]?.images).toBe(0);
    // 第 4 步(输短信验证码)被拒
    expect(r[3]?.isError).toBe(true);
    expect(r[3]?.text).toContain("等待人工处理");
    expect(mock2fa.actions.some((a) => a.action === "type" && a.body.text === "123456")).toBe(false);
    const view = await svc.view(sid);
    expect(view).toMatchObject({ controller: "waiting", waitingReason: "two_factor" });
    const d = await e.db.selectFrom("decision").selectAll().where("id", "=", out.decisionId).executeTakeFirstOrThrow();
    expect(d).toMatchObject({ status: "awaiting_user", subject_type: "browser_session", subject_id: sid });
    expect((d.evidence as Any).challenge.screenshots.length).toBeGreaterThanOrEqual(3);
    expect((d.evidence as Any).challenge.childRunId).toBe(out.childRunId);
    for (const ev of out.evidence) expect(existsSync(join(evidenceDir, `${ev.id}.jpg`))).toBe(true);
    // 二次验证等待不能被视觉子流程接手
    await expect(solve({ sessionId: sid, goal: "E14-二次验证 再试", success: { urlContains: "/x" } })).rejects.toThrow(
      /等待人工处理/,
    );
  }, 60_000);

  it("步数超限:转等待处理(challenge_unsolved),待处理事项附证据截图", async () => {
    await open(ids.site, ids.provider, `${ORIGIN}/torrents.php`);
    const out = await solve({
      sessionId: sid,
      goal: "E14-步数 处理页面",
      success: { textPresent: "种子列表" },
      maxSteps: 3,
    });
    expect(out).toMatchObject({ status: "exhausted", steps: 3, controller: "waiting" });
    expect(out.reason).toContain("工具调用次数超出预算");
    const child = await runRow(e, out.childRunId);
    expect(child).toMatchObject({
      status: "budget_exceeded",
      spawn_reason: "browser.solve_challenge",
      parent_run_id: null,
    });
    const events = await runEvents(e, out.childRunId);
    const ends = events.filter((x) => x.type === "tool_call_end");
    expect(ends.length).toBe(3);
    for (const x of ends) expect((x.payload as Any).outputSummary).toMatch(/evidence\/[0-9a-f-]{36}\.jpg/);
    expect(events.some((x) => x.type === "tool_call_rejected")).toBe(true);
    const view = await svc.view(sid);
    expect(view).toMatchObject({ controller: "waiting", waitingReason: "challenge_unsolved" });
    const d = await e.db.selectFrom("decision").selectAll().where("id", "=", out.decisionId).executeTakeFirstOrThrow();
    expect((d.evidence as Any).challenge.screenshots.length).toBe(3);
  }, 60_000);

  it("敏感截图默认不交给模型:登录页改给已脱敏文字快照;未解决的挑战可以再次接手", async () => {
    // 上一用例留下的 challenge_unsolved 等待:视觉子流程可以接手
    const out = await solve({
      sessionId: sid,
      goal: "E14-敏感 看看登录页",
      success: { urlContains: "/torrents.php" },
      maxSteps: 3,
    });
    expect(out).toMatchObject({ status: "needs_human", controller: "waiting" });
    expect(out.reason).toContain("模型判定需要人工");
    const shot = childResults.get("sensitive")![0]!;
    expect(shot.images).toBe(0);
    expect(shot.json).toMatchObject({ sensitive: true, imageForModel: false });
    // modelImage 由运行时摘出,不出现在 JSON 文本里
    expect("modelImage" in (shot.json as Any)).toBe(false);
    expect((shot.json as Any).note).toContain("敏感");
    expect((shot.json as Any).textSnapshot).toContain("用户名");
    expect((shot.json as Any).evidence.ref).toMatch(/^evidence\//);
  }, 60_000);

  it("主会话 → solve_challenge:截图 → 点输入框 → 输入 → 回车提交 → 成功条件满足", async () => {
    await e.db
      .insertInto("automation_grant")
      .values({ action: "browser.sensitive_screenshot", granted_by: "e14" })
      .execute();
    await open(ids.site, ids.provider, `${ORIGIN}/torrents.php`);
    const created = await e.api("POST", "/agent/runs", { goal: "E14 主会话登录 观众站点" });
    expect(created.status).toBe(201);
    const runId = created.body.data.id as string;
    const done = await waitFor(
      async () => {
        const r = await runRow(e, runId);
        return ["succeeded", "failed"].includes(r.status) ? r : null;
      },
      60_000,
      "主会话结束",
    );
    expect(done.summary).toContain("挑战处理结果:solved");
    const out = mainResults[0]?.json as Any;
    expect(out).toMatchObject({
      status: "solved",
      steps: 3,
      controller: "automation",
      decisionId: null,
      takeover: null,
    });
    expect(out.url).toContain("/torrents.php");
    // 子流程:父会话 = 主会话;每步截图都作为图片交给了模型(已授权敏感截图)
    const child = await runRow(e, out.childRunId);
    expect(child).toMatchObject({ parent_run_id: runId, trigger: "subagent", status: "succeeded" });
    const r = childResults.get("login")!;
    expect(r.map((x) => x.images)).toEqual([1, 1, 1]);
    expect(r[0]?.json).toMatchObject({
      sensitive: true,
      imageForModel: true,
      coordinates: {
        imageWidth: 1280,
        imageHeight: 800,
        viewportWidth: 1600,
        viewportHeight: 1000,
        devicePixelRatio: 1,
      },
    });
    // 模型图片数据不进 JSON 文本
    expect(r[0]?.text).not.toContain('"data"');
    // 坐标换算:截图 (240, 64) → 页面 (300, 80);(240, 96) → (300, 120)
    expect(r[1]?.json).toMatchObject({ pageX: 300, pageY: 80 });
    expect(r[2]?.json).toMatchObject({ pageX: 300, pageY: 120 });
    // CF-2:坐标点击走 umt 插件的真实鼠标事件(按页面 CSS 像素)
    const clicks = mock.actions.filter((a) => a.action === "mouse-click").map((a) => [a.body.x, a.body.y]);
    expect(clicks).toEqual(
      expect.arrayContaining([
        [300, 80],
        [300, 120],
      ]),
    );
    expect(mock.loggedIn.size).toBe(1);
    // 父会话事件流里有子流程
    const events = await runEvents(e, runId);
    expect(events.some((x) => x.type === "subagent_spawned" && (x.payload as Any).childRunId === out.childRunId)).toBe(
      true,
    );
    // 账本不存图片
    const ledger = await e.db
      .selectFrom("capability_invocation")
      .select(["capability_id", "output"])
      .where("agent_run_id", "=", out.childRunId)
      .execute();
    expect(ledger.length).toBe(2);
    for (const l of ledger) expect((l.output as Any).modelImage).toBeNull();
  }, 90_000);

  it("多进程:api 自愈重开被回收的标签页后 worker 换到新标签页;api 关闭会话后 worker 能重新打开", async () => {
    await open(ids.site, ids.provider, `${ORIGIN}/torrents.php`);
    const worker = new BrowserService(e.booted.ctx, svc.opts);
    const auto = { kind: "automation", holder: "agent:e14" } as const;
    const snap = (s: BrowserService) => s.run(sid, auto, (p, ref) => p.snapshot(ref), { write: false, persist: false });
    await snap(worker);
    const old = (await svc.view(sid)).currentTabId!;
    mock.dropTab(old);
    // BR-3:api 发现标签页被回收 → 同一会话新开标签页回到原地址并重试
    await expect(snap(svc)).resolves.toMatchObject({ url: expect.stringContaining("/torrents.php") });
    expect((await svc.view(sid)).currentTabId).not.toBe(old);
    await expect(snap(worker)).resolves.toMatchObject({ url: expect.stringContaining("/torrents.php") });
    // api 关闭会话后,worker 能按站点重新打开(不被内存里的旧会话挡住)
    await svc.close(sid);
    const again = await worker.open({ siteId: ids.site, providerId: ids.provider, url: `${ORIGIN}/torrents.php` });
    expect(again).not.toBe(sid);
    await worker.close(again);
  }, 60_000);

  it("CF-2 站点请求遇 Turnstile:系统发起视觉子流程(没有父会话,预算取设置)点复选框,浏览器读回 cf_clearance", async () => {
    // 只留 Cloudflare 这个浏览器服务(站点请求一侧不指定服务,取启用的那个)
    await e.db.updateTable("browser_provider").set({ enabled: false }).execute();
    const providerCf = (
      await e.db
        .insertInto("browser_provider")
        .values({
          name: "camofox-e14-cf",
          kind: "camofox",
          base_url: mockCf.url,
          secret_ref: (
            await e.db.selectFrom("browser_provider").select("secret_ref").executeTakeFirstOrThrow()
          ).secret_ref,
          enabled: true,
          config: JSON.stringify({ navigationTimeoutMs: 2_000, maxRetries: 0 }),
        })
        .returning("id")
        .executeTakeFirstOrThrow()
    ).id;
    const siteCf = (
      await e.db
        .insertInto("site")
        .values({
          key: "audiences",
          name: "观众",
          adapter: "nexusphp",
          base_url: `${ORIGIN_CF}/`,
          rate_limit: JSON.stringify({ minIntervalMs: 0, concurrency: 5 }),
        })
        .returning("id")
        .executeTakeFirstOrThrow()
    ).id;
    const ref = await e.booted.ctx.secrets.create({ purpose: "site_cookie", plaintext: "c_secure_pass=e14" });
    const accountId = (
      await e.db
        .insertInto("site_account")
        .values({ site_id: siteCf, label: "main", auth_kind: "cookie", secret_ref: ref, status: "active" })
        .returning("id")
        .executeTakeFirstOrThrow()
    ).id;
    cfSid = `umt-site-${siteCf}-acct-${accountId}`; // 先占位,打开后换成会话 ID
    const pending = svc.solveSiteChallenge({ siteId: siteCf, accountId, url: `${ORIGIN_CF}/torrents.php` });
    // 会话打开后脚本才知道 sessionId
    await waitFor(
      async () => {
        const live = await svc.liveSessionFor(siteCf, accountId);
        if (live) cfSid = live.id;
        return !!live;
      },
      10_000,
      "Cloudflare 会话打开",
      10,
    );
    const out = await pending;
    expect(out).toMatchObject({ status: "passed", by: "vision", clearance: "clr-e14" });
    expect(mockCf.actions.some((a) => a.action === "mouse-click" && a.body.x === 300 && a.body.y === 80)).toBe(true);
    const child = await e.db
      .selectFrom("agent_run")
      .selectAll()
      .where("spawn_reason", "=", "browser.solve_challenge")
      .where("created_by", "=", "system:cloudflare")
      .executeTakeFirstOrThrow();
    expect(child).toMatchObject({ trigger: "event", parent_run_id: null, status: "succeeded" });
    expect((child.budget as Any).maxToolCalls).toBe(4);
    expect((child.budget as Any).maxDurationSec).toBeLessThanOrEqual(120);
    expect(providerCf).toBeTruthy();
  }, 60_000);
});