feat(web): site origin view tab with polling screenshot, coordinate input and takeover

Co-Authored-By: Claude Opus 5.5 <[email protected]>

apps/web/app/components/browser/coords.test.ts

@@ -0,0 +1,31 @@
+import { describe, expect, it } from "vitest";
+import { holderName, toViewport, waitingText } from "./coords.ts";
+
+describe("原站视图坐标换算", () => {
+  const natural = { width: 1280, height: 800 };
+  it("缩放显示时按比例换算到视口像素", () => {
+    // 手机上 320px 宽显示 1280 宽的截图:放大 4 倍
+    expect(toViewport({ x: 110, y: 70 }, { left: 10, top: 20, width: 320, height: 200 }, natural)).toEqual({
+      x: 400,
+      y: 200,
+    });
+    // 原尺寸显示
+    expect(toViewport({ x: 640, y: 400 }, { left: 0, top: 0, width: 1280, height: 800 }, natural)).toEqual({
+      x: 640,
+      y: 400,
+    });
+  });
+  it("画面外或尺寸未知返回 null;右下边缘夹在画面内", () => {
+    expect(toViewport({ x: 5, y: 5 }, { left: 10, top: 10, width: 100, height: 100 }, natural)).toBeNull();
+    expect(toViewport({ x: 5, y: 5 }, { left: 0, top: 0, width: 0, height: 0 }, natural)).toBeNull();
+    expect(toViewport({ x: 100, y: 100 }, { left: 0, top: 0, width: 100, height: 100 }, natural)).toEqual({
+      x: 1279,
+      y: 799,
+    });
+  });
+  it("等待原因与持有者显示", () => {
+    expect(waitingText("captcha")).toBe("登录页需要验证码");
+    expect(waitingText(null)).toBeNull();
+    expect(holderName("admin_session:alice")).toBe("alice");
+  });
+});

apps/web/app/components/browser/coords.ts

@@ -0,0 +1,44 @@
+import type { BrowserController } from "@umt/contracts";
+
+/** 截图上的点击位置 → 浏览器视口像素坐标(按图片实际像素与显示尺寸的比例换算,四舍五入并夹在画面内) */
+export function toViewport(
+  client: { x: number; y: number },
+  rect: { left: number; top: number; width: number; height: number },
+  natural: { width: number; height: number },
+): { x: number; y: number } | null {
+  if (rect.width <= 0 || rect.height <= 0 || natural.width <= 0 || natural.height <= 0) return null;
+  const rx = (client.x - rect.left) / rect.width;
+  const ry = (client.y - rect.top) / rect.height;
+  if (rx < 0 || ry < 0 || rx > 1 || ry > 1) return null;
+  return {
+    x: Math.min(natural.width - 1, Math.round(rx * natural.width)),
+    y: Math.min(natural.height - 1, Math.round(ry * natural.height)),
+  };
+}
+
+export const CONTROLLER_LABEL: Record<BrowserController, string> = {
+  automation: "Agent",
+  human: "人工接管",
+  waiting: "等待处理",
+  closed: "已关闭",
+};
+
+const WAITING_TEXT: Record<string, string> = {
+  captcha: "登录页需要验证码",
+  challenge: "页面需要人工验证",
+  login_required: "登录已失效",
+  lease_expired: "接管超时未交还",
+  agent_request: "Agent 请求人工处理",
+  human_released_waiting: "交还时选择继续等待",
+};
+
+export function waitingText(reason: string | null): string | null {
+  if (!reason) return null;
+  return WAITING_TEXT[reason] ?? reason;
+}
+
+/** 租约持有者 "admin_session:alice" → "alice" */
+export function holderName(holder: string): string {
+  const i = holder.indexOf(":");
+  return i >= 0 ? holder.slice(i + 1) : holder;
+}

apps/web/app/components/browser/site-origin-view.tsx

@@ -0,0 +1,329 @@
+import type { SiteListItem } from "@umt/contracts";
+import {
+  ArrowDown,
+  ArrowLeft,
+  ArrowRight,
+  ArrowUp,
+  CornerDownLeft,
+  Globe,
+  Hand,
+  KeyRound,
+  LoaderCircle,
+  Pause,
+  Play,
+  Power,
+  RotateCw,
+  Send,
+  Undo2,
+  UserRound,
+} from "lucide-react";
+import { type FormEvent, type MouseEvent, useEffect, useState } from "react";
+import { toast } from "sonner";
+import {
+  type BrowserInput,
+  useBrowserControl,
+  useBrowserInput,
+  useBrowserProviders,
+  useBrowserScreenshot,
+  useCloseBrowserSession,
+  useOpenBrowserSession,
+  useSiteBrowserSession,
+} from "~/api/browser.ts";
+import { EmptyState, ErrorState, LoadingState } from "~/components/console/empty-state.tsx";
+import { StatusBadge, type StatusTone } from "~/components/console/status-badge.tsx";
+import { Button } from "~/components/ui/button.tsx";
+import { Input } from "~/components/ui/input.tsx";
+import { errorMessage } from "~/lib/api.ts";
+import { CONTROLLER_LABEL, holderName, toViewport, waitingText } from "./coords.ts";
+
+// 原站视图(REQ §4.7.1、§4.7.3):同一个 camofox 会话的定时截图 + 人工输入映射。
+// 人工接管前服务端先停掉该会话的自动化输入;交还后 Agent 必须重新读取页面。
+
+const CONTROLLER_TONE: Record<string, StatusTone> = {
+  automation: "neutral",
+  human: "success",
+  waiting: "warning",
+  closed: "neutral",
+};
+
+export function SiteOriginView({ site }: { site: SiteListItem }) {
+  const providers = useBrowserProviders();
+  const session = useSiteBrowserSession(site.id);
+  const s = session.data ?? null;
+  const [paused, setPaused] = useState(false);
+  const shot = useBrowserScreenshot(s && s.controller !== "closed" ? s.id : null, { paused });
+  const open = useOpenBrowserSession(site.id);
+  const close = useCloseBrowserSession(site.id);
+  const control = useBrowserControl(site.id);
+  const input = useBrowserInput(site.id, s?.id ?? null);
+  const [url, setUrl] = useState("");
+  const [text, setText] = useState("");
+  const [editingUrl, setEditingUrl] = useState(false);
+
+  const currentUrl = s?.currentUrl ?? "";
+  useEffect(() => {
+    if (!editingUrl) setUrl(currentUrl);
+  }, [currentUrl, editingUrl]);
+
+  const provider = providers.data?.items.find((p) => p.enabled && p.kind !== "none");
+  const human = s?.controller === "human";
+  const busy = input.isPending || control.acquire.isPending || control.release.isPending;
+
+  const send = (i: BrowserInput) =>
+    input.mutate(i, { onError: (e) => toast.error(errorMessage(e, "操作没有送达浏览器")) });
+
+  if (providers.isPending || session.isPending) return <LoadingState rows={4} />;
+  if (providers.error) return <ErrorState error={providers.error} onRetry={() => void providers.refetch()} />;
+  if (!provider)
+    return (
+      <EmptyState
+        icon={Globe}
+        title="没有启用的浏览器服务"
+        description="登记 camofox 地址与访问凭证后,这里显示站点的真实页面。"
+      />
+    );
+  if (!site.enabled) return <EmptyState icon={Globe} title="站点已停用" description="启用站点后才能打开原站页面。" />;
+  if (!s)
+    return (
+      <EmptyState
+        icon={Globe}
+        title="还没有打开这个站点的浏览器会话"
+        description="打开后显示站点首页;验证码等需本人操作在此完成。"
+        action={
+          <Button
+            size="sm"
+            disabled={open.isPending}
+            onClick={() => open.mutate(undefined, { onError: (e) => toast.error(errorMessage(e, "打开失败")) })}
+          >
+            {open.isPending ? <LoaderCircle className="animate-spin" /> : <Globe />}
+            打开会话
+          </Button>
+        }
+      />
+    );
+
+  const caps = provider.capabilities;
+  const unverified = !caps?.takeover.verified || !caps?.screenshot.verified;
+  const conn: { tone: StatusTone; text: string } = paused
+    ? { tone: "neutral", text: "已暂停刷新" }
+    : shot.isError
+      ? { tone: "destructive", text: "连接中断" }
+      : shot.data
+        ? { tone: "success", text: "已连接" }
+        : { tone: "neutral", text: "连接中" };
+  const waiting = waitingText(s.waitingReason);
+  const leaseHolder = s.lease ? holderName(s.lease.holder) : null;
+
+  const onScreenClick = (e: MouseEvent<HTMLImageElement>) => {
+    if (!human || !shot.data || busy) return;
+    const img = e.currentTarget;
+    const pt = toViewport({ x: e.clientX, y: e.clientY }, img.getBoundingClientRect(), {
+      width: shot.data.width ?? img.naturalWidth,
+      height: shot.data.height ?? img.naturalHeight,
+    });
+    if (pt) send({ kind: "click", target: pt });
+  };
+
+  const onUrlSubmit = (e: FormEvent) => {
+    e.preventDefault();
+    setEditingUrl(false);
+    if (url.trim() && url !== currentUrl) send({ kind: "navigate", url: url.trim() });
+  };
+  const onTextSubmit = (e: FormEvent) => {
+    e.preventDefault();
+    if (!text) return;
+    send({ kind: "text", text });
+    setText("");
+  };
+
+  return (
+    <div className="space-y-3">
+      <p className="text-caption text-muted-foreground">验证码等需本人操作在此完成。</p>
+
+      <div className="flex flex-wrap items-center gap-2">
+        <StatusBadge tone={conn.tone}>{conn.text}</StatusBadge>
+        <StatusBadge tone={CONTROLLER_TONE[s.controller] ?? "neutral"}>
+          当前控制:{CONTROLLER_LABEL[s.controller]}
+          {human && leaseHolder ? `(${leaseHolder})` : ""}
+        </StatusBadge>
+        {waiting ? <StatusBadge tone="warning">{waiting}</StatusBadge> : null}
+        {unverified ? <StatusBadge tone="warning">画面与输入未对真实实例核验</StatusBadge> : null}
+        <div className="ml-auto flex flex-wrap items-center gap-2">
+          {human ? (
+            <Button
+              size="sm"
+              disabled={busy}
+              onClick={() =>
+                control.release.mutate(s.id, {
+                  onSuccess: () => toast.success("已交还,Agent 会重新读取页面"),
+                  onError: (e) => toast.error(errorMessage(e)),
+                })
+              }
+            >
+              <Undo2 />
+              交还
+            </Button>
+          ) : (
+            <Button
+              size="sm"
+              disabled={busy || s.controller === "closed"}
+              onClick={() => control.acquire.mutate(s.id, { onError: (e) => toast.error(errorMessage(e)) })}
+            >
+              <Hand />
+              接管
+            </Button>
+          )}
+          <Button variant="outline" size="sm" onClick={() => setPaused((p) => !p)}>
+            {paused ? <Play /> : <Pause />}
+            {paused ? "继续刷新" : "暂停刷新"}
+          </Button>
+          <Button
+            variant="ghost"
+            size="sm"
+            disabled={close.isPending}
+            onClick={() => close.mutate(s.id, { onError: (e) => toast.error(errorMessage(e)) })}
+          >
+            <Power />
+            <span className="sr-only sm:not-sr-only">关闭会话</span>
+          </Button>
+        </div>
+      </div>
+
+      <form onSubmit={onUrlSubmit} className="flex min-w-0 items-center gap-1.5">
+        <Button
+          type="button"
+          variant="ghost"
+          size="icon-sm"
+          aria-label="后退"
+          disabled={!human || busy}
+          onClick={() => send({ kind: "history", action: "back" })}
+        >
+          <ArrowLeft />
+        </Button>
+        <Button
+          type="button"
+          variant="ghost"
+          size="icon-sm"
+          aria-label="前进"
+          disabled={!human || busy}
+          onClick={() => send({ kind: "history", action: "forward" })}
+        >
+          <ArrowRight />
+        </Button>
+        <Button
+          type="button"
+          variant="ghost"
+          size="icon-sm"
+          aria-label="刷新"
+          disabled={!human || busy}
+          onClick={() => send({ kind: "history", action: "reload" })}
+        >
+          <RotateCw />
+        </Button>
+        <Input
+          aria-label="地址"
+          value={url}
+          readOnly={!human}
+          onFocus={() => setEditingUrl(true)}
+          onBlur={() => setEditingUrl(false)}
+          onChange={(e) => setUrl(e.target.value)}
+          className="h-8 min-w-0 flex-1 font-mono text-meta"
+        />
+        <Button type="submit" variant="outline" size="sm" disabled={!human || busy}>
+          前往
+        </Button>
+      </form>
+
+      <div className="pg-slab relative min-w-0 overflow-hidden rounded-xl">
+        {shot.data ? (
+          // biome-ignore lint/a11y/useKeyWithClickEvents: 画面点击映射到浏览器;键盘输入走下方输入栏
+          <img
+            src={`data:${shot.data.mimeType};base64,${shot.data.dataBase64}`}
+            alt="站点页面画面"
+            draggable={false}
+            onClick={onScreenClick}
+            className={human ? "block h-auto w-full cursor-crosshair select-none" : "block h-auto w-full select-none"}
+          />
+        ) : shot.isError ? (
+          <ErrorState plain title="读取画面失败" error={shot.error} onRetry={() => void shot.refetch()} />
+        ) : (
+          <LoadingState rows={6} className="p-4" />
+        )}
+        {shot.data && !human ? (
+          <p className="absolute right-2 bottom-2 rounded-md bg-background/85 px-2 py-1 text-meta text-muted-foreground">
+            接管后可点击和输入
+          </p>
+        ) : null}
+      </div>
+
+      {human ? (
+        <div className="space-y-2">
+          <form onSubmit={onTextSubmit} className="flex min-w-0 items-center gap-1.5">
+            <Input
+              aria-label="输入到页面"
+              placeholder="先点页面上的输入框,再在这里输入"
+              value={text}
+              onChange={(e) => setText(e.target.value)}
+              className="h-8 min-w-0 flex-1"
+              autoComplete="off"
+            />
+            <Button type="submit" size="sm" disabled={!text || busy}>
+              <Send />
+              发送
+            </Button>
+          </form>
+          <div className="flex flex-wrap items-center gap-1.5">
+            <Button variant="outline" size="xs" disabled={busy} onClick={() => send({ kind: "key", key: "Enter" })}>
+              <CornerDownLeft />
+              回车
+            </Button>
+            <Button variant="outline" size="xs" disabled={busy} onClick={() => send({ kind: "key", key: "Tab" })}>
+              Tab
+            </Button>
+            <Button variant="outline" size="xs" disabled={busy} onClick={() => send({ kind: "key", key: "Backspace" })}>
+              退格
+            </Button>
+            <Button
+              variant="outline"
+              size="xs"
+              disabled={busy}
+              onClick={() => send({ kind: "scroll", direction: "up" })}
+            >
+              <ArrowUp />
+              上滚
+            </Button>
+            <Button
+              variant="outline"
+              size="xs"
+              disabled={busy}
+              onClick={() => send({ kind: "scroll", direction: "down" })}
+            >
+              <ArrowDown />
+              下滚
+            </Button>
+            <Button
+              variant="outline"
+              size="xs"
+              disabled={busy}
+              title="用站点账号的用户名填当前输入框"
+              onClick={() => send({ kind: "credential", credential: "username" })}
+            >
+              <UserRound />
+              填用户名
+            </Button>
+            <Button
+              variant="outline"
+              size="xs"
+              disabled={busy}
+              title="用站点账号的密码填当前输入框;密码不经过这个页面"
+              onClick={() => send({ kind: "credential", credential: "password" })}
+            >
+              <KeyRound />
+              填密码
+            </Button>
+          </div>
+        </div>
+      ) : null}
+    </div>
+  );
+}