diff --git a/apps/web/app/api/browser.ts b/apps/web/app/api/browser.ts new file mode 100644 index 00000000..9632c5e2 --- /dev/null +++ b/apps/web/app/api/browser.ts @@ -0,0 +1,144 @@ +import { keepPreviousData, useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import type { + BrowserActionResult, + BrowserControlResult, + BrowserNavigation, + BrowserProviderView, + BrowserScreenshotView, + BrowserSessionView, + BrowserTarget, +} from "@umt/contracts"; +import { apiData } from "~/lib/api.ts"; +import { qk } from "~/lib/query-keys.ts"; + +// C1-E 原站视图:浏览器服务、站点会话、定时截图与人工输入。输入类动作要先接管(服务端校验租约)。 + +const AREA = "browser"; + +export function useBrowserProviders() { + return useQuery({ + queryKey: qk.list(AREA, { kind: "providers" }), + queryFn: ({ signal }) => apiData<{ items: BrowserProviderView[] }>("/browser/providers", { signal }), + staleTime: 60_000, + }); +} + +/** 站点的未关闭会话(最新的一个) */ +export function useSiteBrowserSession(siteId: string, enabled = true) { + return useQuery({ + queryKey: qk.sub(AREA, "site", siteId), + queryFn: async ({ signal }) => { + const r = await apiData<{ items: BrowserSessionView[] }>("/browser/sessions", { params: { siteId }, signal }); + return r.items[0] ?? null; + }, + enabled: !!siteId && enabled, + refetchInterval: 5_000, + }); +} + +/** + * 定时截图(1.5 秒,可暂停)。上一张保留到新图到达,避免闪烁;失败时保留旧图并在界面标"连接中断"。 + */ +export function useBrowserScreenshot(sessionId: string | null, opts: { paused: boolean; intervalMs?: number }) { + return useQuery({ + queryKey: qk.sub(AREA, "screenshot", sessionId ?? ""), + queryFn: ({ signal }) => + apiData("/browser/sessions/{id}/screenshot", { + pathParams: { id: sessionId ?? "" }, + signal, + }), + enabled: !!sessionId, + refetchInterval: opts.paused ? false : (opts.intervalMs ?? 1_500), + refetchIntervalInBackground: false, + placeholderData: keepPreviousData, + retry: false, + gcTime: 0, + }); +} + +function useRefresh(siteId: string) { + const qc = useQueryClient(); + return (session?: BrowserSessionView | null) => { + if (session !== undefined) qc.setQueryData(qk.sub(AREA, "site", siteId), session); + void qc.invalidateQueries({ queryKey: qk.sub(AREA, "site", siteId) }); + void qc.invalidateQueries({ queryKey: qk.sub(AREA, "screenshot") }); + }; +} + +export function useOpenBrowserSession(siteId: string) { + const refresh = useRefresh(siteId); + return useMutation({ + mutationFn: () => apiData("/browser/sessions", { method: "POST", body: { siteId } }), + onSuccess: (s) => refresh(s), + }); +} + +export function useCloseBrowserSession(siteId: string) { + const refresh = useRefresh(siteId); + return useMutation({ + mutationFn: (id: string) => + apiData("/browser/sessions/{id}", { method: "DELETE", pathParams: { id } }), + onSuccess: () => refresh(null), + }); +} + +export function useBrowserControl(siteId: string) { + const refresh = useRefresh(siteId); + const acquire = useMutation({ + mutationFn: (id: string) => + apiData("/browser/sessions/{id}/control/acquire", { + method: "POST", + pathParams: { id }, + body: { reason: "原站视图接管" }, + }), + onSuccess: (r) => refresh(r.session), + }); + const release = useMutation({ + mutationFn: (id: string) => + apiData("/browser/sessions/{id}/control/release", { + method: "POST", + pathParams: { id }, + body: { reason: "原站视图交还" }, + }), + onSuccess: (r) => refresh(r.session), + }); + return { acquire, release }; +} + +export type BrowserInput = + | { kind: "click"; target: BrowserTarget } + | { kind: "text"; text: string; submit?: boolean } + | { kind: "credential"; credential: "username" | "password" | "totp" } + | { kind: "key"; key: string } + | { kind: "scroll"; direction: "up" | "down" } + | { kind: "navigate"; url: string } + | { kind: "history"; action: "back" | "forward" | "reload" }; + +/** 人工输入:点击 / 文本 / 凭证 / 按键 / 滚动 / 导航;成功后立即刷新截图 */ +export function useBrowserInput(siteId: string, sessionId: string | null) { + const refresh = useRefresh(siteId); + return useMutation({ + mutationFn: async (input: BrowserInput) => { + const id = sessionId ?? ""; + const post = (path: string, body: unknown) => + apiData(`/browser/sessions/{id}/${path}`, { method: "POST", pathParams: { id }, body }); + switch (input.kind) { + case "click": + return (await post("click", { target: input.target })).session; + case "text": + return (await post("type", { text: input.text, submit: input.submit })).session; + case "credential": + return (await post("type", { credential: input.credential })).session; + case "key": + return (await post("type", { key: input.key })).session; + case "scroll": + return (await post("scroll", { direction: input.direction, amount: 600 })).session; + case "navigate": + return (await post("navigate", { url: input.url })).session; + case "history": + return (await post("navigate", { action: input.action })).session; + } + }, + onSuccess: (s) => refresh(s), + }); +} diff --git a/apps/web/app/components/browser/coords.test.ts b/apps/web/app/components/browser/coords.test.ts new file mode 100644 index 00000000..ef5ab875 --- /dev/null +++ b/apps/web/app/components/browser/coords.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from "vitest"; +import { holderName, toViewport, waitingText } from "./coords.ts"; + +describe("原站视图坐标换算", () => { + const natural = { width: 1280, height: 800 }; + it("缩放显示时按比例换算到视口像素", () => { + // 手机上 320px 宽显示 1280 宽的截图:放大 4 倍 + expect(toViewport({ x: 110, y: 70 }, { left: 10, top: 20, width: 320, height: 200 }, natural)).toEqual({ + x: 400, + y: 200, + }); + // 原尺寸显示 + expect(toViewport({ x: 640, y: 400 }, { left: 0, top: 0, width: 1280, height: 800 }, natural)).toEqual({ + x: 640, + y: 400, + }); + }); + it("画面外或尺寸未知返回 null;右下边缘夹在画面内", () => { + expect(toViewport({ x: 5, y: 5 }, { left: 10, top: 10, width: 100, height: 100 }, natural)).toBeNull(); + expect(toViewport({ x: 5, y: 5 }, { left: 0, top: 0, width: 0, height: 0 }, natural)).toBeNull(); + expect(toViewport({ x: 100, y: 100 }, { left: 0, top: 0, width: 100, height: 100 }, natural)).toEqual({ + x: 1279, + y: 799, + }); + }); + it("等待原因与持有者显示", () => { + expect(waitingText("captcha")).toBe("登录页需要验证码"); + expect(waitingText(null)).toBeNull(); + expect(holderName("admin_session:alice")).toBe("alice"); + }); +}); diff --git a/apps/web/app/components/browser/coords.ts b/apps/web/app/components/browser/coords.ts new file mode 100644 index 00000000..223fda21 --- /dev/null +++ b/apps/web/app/components/browser/coords.ts @@ -0,0 +1,44 @@ +import type { BrowserController } from "@umt/contracts"; + +/** 截图上的点击位置 → 浏览器视口像素坐标(按图片实际像素与显示尺寸的比例换算,四舍五入并夹在画面内) */ +export function toViewport( + client: { x: number; y: number }, + rect: { left: number; top: number; width: number; height: number }, + natural: { width: number; height: number }, +): { x: number; y: number } | null { + if (rect.width <= 0 || rect.height <= 0 || natural.width <= 0 || natural.height <= 0) return null; + const rx = (client.x - rect.left) / rect.width; + const ry = (client.y - rect.top) / rect.height; + if (rx < 0 || ry < 0 || rx > 1 || ry > 1) return null; + return { + x: Math.min(natural.width - 1, Math.round(rx * natural.width)), + y: Math.min(natural.height - 1, Math.round(ry * natural.height)), + }; +} + +export const CONTROLLER_LABEL: Record = { + automation: "Agent", + human: "人工接管", + waiting: "等待处理", + closed: "已关闭", +}; + +const WAITING_TEXT: Record = { + captcha: "登录页需要验证码", + challenge: "页面需要人工验证", + login_required: "登录已失效", + lease_expired: "接管超时未交还", + agent_request: "Agent 请求人工处理", + human_released_waiting: "交还时选择继续等待", +}; + +export function waitingText(reason: string | null): string | null { + if (!reason) return null; + return WAITING_TEXT[reason] ?? reason; +} + +/** 租约持有者 "admin_session:alice" → "alice" */ +export function holderName(holder: string): string { + const i = holder.indexOf(":"); + return i >= 0 ? holder.slice(i + 1) : holder; +} diff --git a/apps/web/app/components/browser/site-origin-view.tsx b/apps/web/app/components/browser/site-origin-view.tsx new file mode 100644 index 00000000..f8466c45 --- /dev/null +++ b/apps/web/app/components/browser/site-origin-view.tsx @@ -0,0 +1,329 @@ +import type { SiteListItem } from "@umt/contracts"; +import { + ArrowDown, + ArrowLeft, + ArrowRight, + ArrowUp, + CornerDownLeft, + Globe, + Hand, + KeyRound, + LoaderCircle, + Pause, + Play, + Power, + RotateCw, + Send, + Undo2, + UserRound, +} from "lucide-react"; +import { type FormEvent, type MouseEvent, useEffect, useState } from "react"; +import { toast } from "sonner"; +import { + type BrowserInput, + useBrowserControl, + useBrowserInput, + useBrowserProviders, + useBrowserScreenshot, + useCloseBrowserSession, + useOpenBrowserSession, + useSiteBrowserSession, +} from "~/api/browser.ts"; +import { EmptyState, ErrorState, LoadingState } from "~/components/console/empty-state.tsx"; +import { StatusBadge, type StatusTone } from "~/components/console/status-badge.tsx"; +import { Button } from "~/components/ui/button.tsx"; +import { Input } from "~/components/ui/input.tsx"; +import { errorMessage } from "~/lib/api.ts"; +import { CONTROLLER_LABEL, holderName, toViewport, waitingText } from "./coords.ts"; + +// 原站视图(REQ §4.7.1、§4.7.3):同一个 camofox 会话的定时截图 + 人工输入映射。 +// 人工接管前服务端先停掉该会话的自动化输入;交还后 Agent 必须重新读取页面。 + +const CONTROLLER_TONE: Record = { + automation: "neutral", + human: "success", + waiting: "warning", + closed: "neutral", +}; + +export function SiteOriginView({ site }: { site: SiteListItem }) { + const providers = useBrowserProviders(); + const session = useSiteBrowserSession(site.id); + const s = session.data ?? null; + const [paused, setPaused] = useState(false); + const shot = useBrowserScreenshot(s && s.controller !== "closed" ? s.id : null, { paused }); + const open = useOpenBrowserSession(site.id); + const close = useCloseBrowserSession(site.id); + const control = useBrowserControl(site.id); + const input = useBrowserInput(site.id, s?.id ?? null); + const [url, setUrl] = useState(""); + const [text, setText] = useState(""); + const [editingUrl, setEditingUrl] = useState(false); + + const currentUrl = s?.currentUrl ?? ""; + useEffect(() => { + if (!editingUrl) setUrl(currentUrl); + }, [currentUrl, editingUrl]); + + const provider = providers.data?.items.find((p) => p.enabled && p.kind !== "none"); + const human = s?.controller === "human"; + const busy = input.isPending || control.acquire.isPending || control.release.isPending; + + const send = (i: BrowserInput) => + input.mutate(i, { onError: (e) => toast.error(errorMessage(e, "操作没有送达浏览器")) }); + + if (providers.isPending || session.isPending) return ; + if (providers.error) return void providers.refetch()} />; + if (!provider) + return ( + + ); + if (!site.enabled) return ; + if (!s) + return ( + open.mutate(undefined, { onError: (e) => toast.error(errorMessage(e, "打开失败")) })} + > + {open.isPending ? : } + 打开会话 + + } + /> + ); + + const caps = provider.capabilities; + const unverified = !caps?.takeover.verified || !caps?.screenshot.verified; + const conn: { tone: StatusTone; text: string } = paused + ? { tone: "neutral", text: "已暂停刷新" } + : shot.isError + ? { tone: "destructive", text: "连接中断" } + : shot.data + ? { tone: "success", text: "已连接" } + : { tone: "neutral", text: "连接中" }; + const waiting = waitingText(s.waitingReason); + const leaseHolder = s.lease ? holderName(s.lease.holder) : null; + + const onScreenClick = (e: MouseEvent) => { + if (!human || !shot.data || busy) return; + const img = e.currentTarget; + const pt = toViewport({ x: e.clientX, y: e.clientY }, img.getBoundingClientRect(), { + width: shot.data.width ?? img.naturalWidth, + height: shot.data.height ?? img.naturalHeight, + }); + if (pt) send({ kind: "click", target: pt }); + }; + + const onUrlSubmit = (e: FormEvent) => { + e.preventDefault(); + setEditingUrl(false); + if (url.trim() && url !== currentUrl) send({ kind: "navigate", url: url.trim() }); + }; + const onTextSubmit = (e: FormEvent) => { + e.preventDefault(); + if (!text) return; + send({ kind: "text", text }); + setText(""); + }; + + return ( +
+

验证码等需本人操作在此完成。

+ +
+ {conn.text} + + 当前控制:{CONTROLLER_LABEL[s.controller]} + {human && leaseHolder ? `(${leaseHolder})` : ""} + + {waiting ? {waiting} : null} + {unverified ? 画面与输入未对真实实例核验 : null} +
+ {human ? ( + + ) : ( + + )} + + +
+
+ +
+ + + + setEditingUrl(true)} + onBlur={() => setEditingUrl(false)} + onChange={(e) => setUrl(e.target.value)} + className="h-8 min-w-0 flex-1 font-mono text-meta" + /> + +
+ +
+ {shot.data ? ( + // biome-ignore lint/a11y/useKeyWithClickEvents: 画面点击映射到浏览器;键盘输入走下方输入栏 + 站点页面画面 + ) : shot.isError ? ( + void shot.refetch()} /> + ) : ( + + )} + {shot.data && !human ? ( +

+ 接管后可点击和输入 +

+ ) : null} +
+ + {human ? ( +
+
+ setText(e.target.value)} + className="h-8 min-w-0 flex-1" + autoComplete="off" + /> + +
+
+ + + + + + + +
+
+ ) : null} +
+ ); +} diff --git a/apps/web/app/routes/console/sites.$siteId.tsx b/apps/web/app/routes/console/sites.$siteId.tsx index a75a76b3..0c362f48 100644 --- a/apps/web/app/routes/console/sites.$siteId.tsx +++ b/apps/web/app/routes/console/sites.$siteId.tsx @@ -19,6 +19,7 @@ import { useInventory } from "~/api/inventory.ts"; import { type ReleaseFilters, useSiteReleases } from "~/api/releases.ts"; import { snapshotStale, useSites, useSnapshotSite } from "~/api/sites.ts"; import { openAgentDrawer } from "~/components/agent/agent-store.ts"; +import { SiteOriginView } from "~/components/browser/site-origin-view.tsx"; import { DetailSection } from "~/components/console/detail-sheet.tsx"; import { EmptyState, ErrorState, LoadingState } from "~/components/console/empty-state.tsx"; import { FilterBar, FilterSelect } from "~/components/console/filter-bar.tsx"; @@ -97,9 +98,9 @@ function useDebounced(value: T, ms = 400) { return v; } -const TABS = ["overview", "resources", "snapshots", "accounts"] as const; -/** 旧页签(整理视图 / 原站视图)落到"资源" */ -const LEGACY_TAB: Record = { organized: "resources", origin: "resources" }; +const TABS = ["overview", "resources", "origin", "snapshots", "accounts"] as const; +/** 旧页签(整理视图)落到"资源" */ +const LEGACY_TAB: Record = { organized: "resources" }; export const meta = () => [{ title: "站点 · UHD 媒体自动化控制台" }]; @@ -274,6 +275,7 @@ export default function SiteWorkbenchPage() { 概览 资源 + 原站视图 快照与 Feed 账号 @@ -419,6 +421,8 @@ export default function SiteWorkbenchPage() { + {s ? : } +
{s ? : }