UHD_Media_Tool/apps/server/src/modules/browser/capabilities.ts

import { type BrowserProvider, isLoginUrl, type TabRef } from "@umt/adapters";
import { CapabilityError, type CapabilityRegistry, type HandlerContext } from "@umt/agent";
import { capabilitySpec } from "@umt/contracts";
import type { Actor, BrowserService } from "./service.ts";
import { BrowserServiceError } from "./service.ts";

// C1-E Agent 浏览器能力:browser.snapshot(read)/ browser.navigate / browser.interact / browser.download(execute)。
// 一律以自动化身份执行:会话被人工接管或在等待处理时拒绝;元素引用只来自最近一次 snapshot;
// 凭证用 credential 字段由服务端注入;下载只暂存核验并匹配本地资源,提交仍走 download.plan / download.submit。

const CODE = {
  not_found: "not_found",
  conflict: "conflict",
  precondition: "precondition",
  bad_request: "precondition",
  upstream: "precondition",
  rate_limited: "budget",
  reconnect_failed: "precondition",
} as const;

async function guard<T>(fn: () => Promise<T>): Promise<T> {
  try {
    return await fn();
  } catch (e) {
    if (e instanceof BrowserServiceError) throw new CapabilityError(CODE[e.code], e.message);
    throw e;
  }
}

const agent = (actorId: string): Actor => ({ kind: "automation", holder: actorId });

/** 敏感截图(登录页 / 密码框)交给模型的授权:有效的 browser.sensitive_screenshot(或 *)持续授权 */
export const SENSITIVE_SCREENSHOT_GRANT = "browser.sensitive_screenshot";

async function sensitiveAllowed(svc: BrowserService, want: boolean | undefined): Promise<boolean> {
  if (!want) return false;
  const g = await svc.ctx.db
    .selectFrom("automation_grant")
    .select("id")
    .where("status", "=", "active")
    .where("action", "in", [SENSITIVE_SCREENSHOT_GRANT, "*"])
    .where((eb) => eb.or([eb("expires_at", "is", null), eb("expires_at", ">", new Date())]))
    .executeTakeFirst();
  return !!g;
}

/** 调用方能不能收图片:Agent 调用按上游是否支持图片输入;其它调用方(HTTP / MCP)缺省给 */
const wantImage = (ctx: HandlerContext) => ctx.modelImages !== false;

export function registerBrowserCapabilities(registry: CapabilityRegistry, svc: () => BrowserService): void {
  /** sessionId 优先;否则取站点的未关闭会话 */
  const resolveSession = async (input: { sessionId?: string | undefined; siteId?: string | undefined }) => {
    if (input.sessionId) return input.sessionId;
    const row = input.siteId ? await svc().liveSessionFor(input.siteId) : undefined;
    if (!row) throw new BrowserServiceError("not_found", "该站点没有浏览器会话，先用 browser.navigate 打开页面");
    return row.id;
  };

  registry.register(capabilitySpec("browser.snapshot"), (input, ctx) =>
    guard(async () => {
      const id = await resolveSession(input);
      const snap = await svc().run(
        id,
        agent(ctx.actorId),
        (p, ref) => p.snapshot(ref, input.offset !== undefined ? { offset: input.offset } : {}),
        { write: false, patch: () => ({ needsSnapshot: false }), ...(input.tabId ? { tabId: input.tabId } : {}) },
      );
      const s = svc();
      const loginPage = isLoginUrl(snap.url || null);
      await s.afterAutomationSnapshot(id, snap.text, loginPage);
      const view = await s.view(id);
      return {
        snapshot: snap,
        controller: view.controller,
        waitingReason: view.waitingReason,
        loginRequired: loginPage,
      };
    }),
  );

  registry.register(capabilitySpec("browser.navigate"), (input, ctx) =>
    guard(async () => {
      let id = input.sessionId;
      if (!id) {
        const live = await svc().liveSessionFor(input.siteId!, input.accountId);
        id =
          live?.id ??
          (await svc().open({
            siteId: input.siteId!,
            ...(input.accountId ? { accountId: input.accountId } : {}),
            url: input.url,
          }));
        if (!live) {
          const v = await svc().view(id);
          const info = v.currentUrl ?? "";
          return {
            sessionId: id,
            url: info,
            httpStatus: null,
            ok: true,
            attempts: 1,
            loginRequired: isLoginUrl(info || null),
            controller: v.controller,
          };
        }
      }
      const sid = id;
      const nav = await svc().run(
        sid,
        agent(ctx.actorId),
        async (p, ref, x) => {
          await svc().limited(x.site);
          // 标签页全被上游回收(重启 / 空闲超时,已对账为 lost):在同一会话(同一 profile)里开新标签页,不重放旧引用
          if (!input.tabId && !p.sessionInfo(sid)?.tabs.some((t) => t.status === "open")) {
            const tab = await p.openTab(sid, { url: input.url, budget: x.budget, as: "automation" });
            return {
              url: tab.url ?? "",
              httpStatus: null,
              ok: true,
              attempts: 1,
              loginRequired: p.sessionInfo(sid)?.loginPage ?? false,
            };
          }
          const r = await p.navigate(ref, input.url, x.budget);
          return { ...r, loginRequired: p.sessionInfo(sid)?.loginPage ?? false };
        },
        input.tabId ? { tabId: input.tabId } : {},
      );
      const v = await svc().view(sid);
      return { sessionId: sid, ...nav, controller: v.controller };
    }),
  );

  registry.register(capabilitySpec("browser.interact"), (input, ctx) =>
    guard(async () => {
      const s = svc();
      if (input.action === "request_human") {
        await s.requestHuman(input.sessionId, input.reason ?? "agent_request");
        const v = await s.view(input.sessionId);
        return { ok: true as const, controller: v.controller, needsSnapshot: true };
      }
      const needTarget = input.action === "click" || input.action === "select";
      if (needTarget && !input.target) throw new BrowserServiceError("bad_request", `${input.action} 需要 target`);
      await s.run(
        input.sessionId,
        agent(ctx.actorId),
        async (p, ref, x) => {
          switch (input.action) {
            case "click":
              return p.click(ref, input.target!);
            case "select":
              return p.select(ref, input.target!, input.value ?? "");
            case "scroll":
              return p.scroll(ref, { direction: input.direction ?? "down", amount: input.amount ?? 600 });
            case "press":
              return p.press(ref, input.key ?? "Enter");
            case "type": {
              const extra = input.submit !== undefined ? { submit: input.submit } : {};
              if (input.credential)
                return p.type(ref, input.target ?? null, {
                  kind: "secret",
                  siteId: x.row.site_id,
                  accountId: x.row.account_id,
                  field: input.credential,
                  ...extra,
                });
              if (input.text === undefined)
                throw new BrowserServiceError("bad_request", "type 需要 text 或 credential");
              return p.type(ref, input.target ?? null, { kind: "text", text: input.text, ...extra });
            }
          }
        },
        input.tabId ? { tabId: input.tabId } : {},
      );
      const info = (await s.view(input.sessionId)).controller;
      const changes = input.action === "click" || input.action === "press" || !!input.submit;
      return { ok: true as const, controller: info, needsSnapshot: changes };
    }),
  );

  registry.register(capabilitySpec("browser.download"), (input, ctx) =>
    guard(async () => {
      const s = svc();
      const artifact = await s.run(
        input.sessionId,
        agent(ctx.actorId),
        async (p, ref) => {
          if (input.target) await p.click(ref, input.target);
          return p.waitForDownload(ref, { timeoutMs: input.timeoutMs, maxBytes: 10 * 1024 * 1024 });
        },
        input.tabId ? { tabId: input.tabId } : {},
      );
      const v = await s.verifyDownload(input.sessionId, artifact);
      await s.recordDownload(input.sessionId, {
        sha256: artifact.sha256,
        artifactId: artifact.id,
        sourceReleaseId: v.sourceReleaseId,
        planId: v.duplicate?.planId ?? null,
      });
      return {
        artifactId: artifact.id,
        filename: artifact.filename,
        sizeBytes: artifact.sizeBytes,
        sha256: artifact.sha256,
        verification: v.verification,
        sourceReleaseId: v.sourceReleaseId,
        duplicate: !!v.duplicate,
        note: v.duplicate
          ? "同一会话已捕获过这个文件；先查已有下载计划，不要重复提交"
          : !v.verification.valid
            ? "不是有效的种子文件"
            : v.sourceReleaseId
              ? "已暂存核验；用 download.plan 提交这个 sourceReleaseId"
              : "资源不在本地索引；先 release.search 该站点后再提交",
      };
    }),
  );

  // ── VIS-1 视觉自动化:截图(图片内容块)与按截图坐标操作 ──

  registry.register(capabilitySpec("browser.screenshot"), (input, ctx) =>
    guard(async () => {
      const s = svc();
      return s.capture(input.sessionId, agent(ctx.actorId), {
        ...(input.tabId ? { tabId: input.tabId } : {}),
        wantImage: wantImage(ctx),
        sensitiveAllowed: await sensitiveAllowed(s, input.allowSensitive),
      });
    }),
  );

  const act = async (
    ctx: HandlerContext,
    input: { sessionId: string; allowSensitive?: boolean | undefined },
    point: { x: number; y: number } | null,
    fn: (p: BrowserProvider, ref: TabRef, at: { x: number; y: number } | null) => Promise<void>,
  ) => {
    const s = svc();
    const page = point ? s.toPage(input.sessionId, point.x, point.y) : null;
    const at = page ? { x: page.pageX, y: page.pageY } : null;
    const shot = await s.visionAct(input.sessionId, agent(ctx.actorId), (p, ref) => fn(p, ref, at), {
      wantImage: wantImage(ctx),
      sensitiveAllowed: await sensitiveAllowed(s, input.allowSensitive),
    });
    return { ok: true as const, pageX: at?.x ?? null, pageY: at?.y ?? null, ...shot };
  };

  registry.register(capabilitySpec("browser.click_at"), (input, ctx) =>
    guard(async () => {
      if (input.button && input.button !== "left")
        throw new BrowserServiceError("bad_request", "浏览器服务(camofox 1.17.0)只支持左键点击");
      return act(ctx, input, { x: input.x, y: input.y }, (p, ref, at) => p.click(ref, at!));
    }),
  );

  registry.register(capabilitySpec("browser.type_at"), (input, ctx) =>
    guard(async () => {
      if ((input.x === undefined) !== (input.y === undefined))
        throw new BrowserServiceError("bad_request", "x 与 y 要一起给（都不给 = 输入到当前焦点）");
      const point = input.x !== undefined && input.y !== undefined ? { x: input.x, y: input.y } : null;
      return act(ctx, input, point, async (p, ref, at) => {
        if (at) await p.click(ref, at);
        await p.type(ref, null, { kind: "text", text: input.text });
        if (input.pressEnter) await p.press(ref, "Enter");
      });
    }),
  );

  registry.register(capabilitySpec("browser.scroll"), (input, ctx) =>
    guard(async () => {
      if (!input.dx && !input.dy) throw new BrowserServiceError("bad_request", "dx 与 dy 至少一个不为 0");
      return act(ctx, input, null, async (p, ref) => {
        if (input.dy) await p.scroll(ref, { direction: input.dy > 0 ? "down" : "up", amount: Math.abs(input.dy) });
        if (input.dx) await p.scroll(ref, { direction: input.dx > 0 ? "right" : "left", amount: Math.abs(input.dx) });
      });
    }),
  );

  registry.register(capabilitySpec("browser.wait"), (input, ctx) =>
    guard(async () => {
      const s = svc();
      await new Promise((r) => setTimeout(r, input.ms));
      const shot = await s.capture(input.sessionId, agent(ctx.actorId), {
        wantImage: wantImage(ctx),
        sensitiveAllowed: await sensitiveAllowed(s, input.allowSensitive),
      });
      return { ok: true as const, pageX: null, pageY: null, ...shot };
    }),
  );
}