import { useQuery, useQueryClient } from "@tanstack/react-query"; import type { SiteListItem } from "@umt/contracts"; import { Hand, Image as ImageIcon, LoaderCircle, Monitor, Power, RotateCw, Undo2 } from "lucide-react"; import { useEffect, useRef, useState } from "react"; import { toast } from "sonner"; import { useBrowserControl, useBrowserReconnect, useCloseBrowserSession, useSiteBrowserSession, } from "~/api/browser.ts"; import { LoadingState } from "~/components/console/empty-state.tsx"; import { StatusBadge, type StatusTone } from "~/components/console/status-badge.tsx"; import { SubtitleFromPage } from "~/components/subtitles/subtitle-from-page.tsx"; import { Button } from "~/components/ui/button.tsx"; import { apiData, errorMessage } from "~/lib/api.ts"; import { qk } from "~/lib/query-keys.ts"; import { agentActive, controllerLabel, credentialBadge, holderName, waitingText } from "./coords.ts"; import { ReconnectPanel, ReloadNotice, useAutoReconnect } from "./reconnect.tsx"; import { ResourceDownload } from "./resource-download.tsx"; import { SiteOriginView } from "./site-origin-view.tsx"; // 原站视图桌面模式(docs/design/webtop-remote-browser.md §5.7):iframe 嵌浏览器桌面(Selkies), // 画面就是 Agent 正在驱动的同一个浏览器窗口。键鼠权限由服务端按人工租约决定:未接管只能看,接管后可直接点和输入。 // 桌面不可用(旧镜像 / 未配置)或当前不是安全上下文(HTTP)时回退截图模式,并说明原因。 interface DesktopStatus { available: boolean; reason: string | null; role: "controller" | "viewer"; leaseExpiresAt: string | null; path: string; requiresSecureContext: true; } function useDesktopStatus(sessionId: string | null) { return useQuery({ queryKey: qk.sub("browser", "desktop", sessionId ?? ""), queryFn: ({ signal }) => apiData("/browser/desktop/{id}/_status", { pathParams: { id: sessionId ?? "" }, signal }), enabled: !!sessionId, refetchInterval: 10_000, retry: false, }); } const CONTROLLER_TONE: Record = { automation: "neutral", human: "success", waiting: "warning", closed: "neutral", }; type Mode = "auto" | "screenshot"; /** 画面加载后多久还没出图算黑屏 */ const BLANK_TIMEOUT_MS = 5_000; /** 同源时看 Selkies 的视频流有没有出帧;跨源 / 拿不到文档时不判黑屏(只按 onLoad 判断) */ function desktopBlank(frame: HTMLIFrameElement | null): boolean { try { const video = frame?.contentDocument?.querySelector("video"); return !!video && (video.readyState < 2 || video.videoWidth === 0); } catch { return false; } } /** 原站视图 Tab:能用桌面模式就用,否则截图模式 */ export function SiteOriginPanel({ site }: { site: SiteListItem }) { const session = useSiteBrowserSession(site.id); const s = session.data ?? null; const status = useDesktopStatus(s && s.controller !== "closed" ? s.id : null); const [mode, setMode] = useState("auto"); const [secure, setSecure] = useState(true); useEffect(() => setSecure(window.isSecureContext), []); const desktop = status.data; const canDesktop = !!desktop?.available && secure; if (s && status.isPending && s.controller !== "closed") return ; if (!s || !canDesktop || mode === "screenshot") { let note: string | null = null; if (s && desktop && !desktop.available && desktop.reason) note = `桌面模式不可用:${desktop.reason}。`; else if (s && desktop?.available && !secure) note = "桌面模式需要用 HTTPS 打开控制台,现在是截图模式。"; return (
{note || (canDesktop && mode === "screenshot") ? (
{note ? {note} : null} {canDesktop ? ( ) : null}
) : null}
); } return setMode("screenshot")} />; } function DesktopView({ site, status, onScreenshotMode, }: { site: SiteListItem; status: DesktopStatus; onScreenshotMode: () => void; }) { const session = useSiteBrowserSession(site.id); const s = session.data ?? null; const control = useBrowserControl(site.id); const close = useCloseBrowserSession(site.id); const qc = useQueryClient(); const refreshRole = () => void qc.invalidateQueries({ queryKey: qk.sub("browser", "desktop") }); const [reloadKey, setReloadKey] = useState(0); const [loaded, setLoaded] = useState(false); const frame = useRef(null); /** 打开时确认失败 / 黑屏超时:交给 useAutoReconnect 自动重连一次 */ const [stuck, setStuck] = useState(false); const reloadDesktop = () => { setLoaded(false); setReloadKey((k) => k + 1); }; // BR-3:浏览器服务(同一容器里的 Selkies)重启过 → 自动重连会话一次,成功后重建桌面连接 const rc = useAutoReconnect(site.id, s, stuck, () => { setStuck(false); reloadDesktop(); }); // INT-3(VER-3b):桌面 iframe 直连 Selkies,不经过截图 / 操作接口,不会触发服务端的被动重连 → // 打开桌面时先调一次幂等的 reconnect(上游上下文还在时服务端什么都不做),失败再走自动重连。 // FIX-5(VER-4):先等这次确认结束再连桌面画面,不再先连上黑屏、等 5 秒黑屏检测才重连 const ensure = useBrowserReconnect(site.id); const { mutate: ensureMutate } = ensure; const sid = s && s.controller !== "closed" ? s.id : null; const ensured = useRef(null); const [ensuredSid, setEnsuredSid] = useState(null); useEffect(() => { if (!sid || ensured.current === sid) return; ensured.current = sid; ensureMutate(sid, { onError: () => setStuck(true), onSettled: () => setEnsuredSid(sid), }); }, [sid, ensureMutate]); /** 会话已确认(或已关闭):可以连桌面画面 */ const ready = !sid || ensuredSid === sid; // 画面加载失败或黑屏超过 5 秒:每个会话自动重连一次(之后靠「重新连接」按钮) const blankTried = useRef(null); // biome-ignore lint/correctness/useExhaustiveDependencies: reloadKey 变化 = 新一轮加载,重新计时 useEffect(() => { if (!sid || !ready || blankTried.current === sid) return; const t = setTimeout(() => { if (loaded && !desktopBlank(frame.current)) return; blankTried.current = sid; console.debug("[browser] 桌面画面 5 秒未出图,自动重新连接一次", sid); setStuck(true); }, BLANK_TIMEOUT_MS); return () => clearTimeout(t); }, [sid, ready, loaded, reloadKey]); // 别处(Agent 操作、截图)触发的重连成功后也重建桌面连接 const reconnectedAt = s?.reconnect?.status === "ok" ? s.reconnect.at : null; const prevReconnect = useRef(reconnectedAt); useEffect(() => { if (prevReconnect.current !== reconnectedAt) { prevReconnect.current = reconnectedAt; if (reconnectedAt) { setLoaded(false); setReloadKey((k) => k + 1); } } }, [reconnectedAt]); // 控制权变化(接管 / 交还 / 过期)后重连,Selkies 按新口令认证;服务端也会主动断开旧连接 const role = status.role; const prevRole = useRef(role); useEffect(() => { if (prevRole.current !== role) { prevRole.current = role; setLoaded(false); setReloadKey((k) => k + 1); } }, [role]); if (!s) return ; const human = s.controller === "human"; const mine = role === "controller"; const busy = control.acquire.isPending || control.release.isPending; const waiting = waitingText(s.waitingReason); const cred = credentialBadge(s.credential); const leaseHolder = s.lease ? holderName(s.lease.holder) : null; const conn: { tone: StatusTone; text: string } = !ready ? { tone: "neutral", text: "正在确认浏览器会话" } : rc.state.phase === "reconnecting" ? { tone: "warning", text: "正在重新连接" } : rc.state.phase === "failed" ? { tone: "destructive", text: "连接中断" } : loaded ? mine ? { tone: "success", text: "已连接,可操作" } : { tone: "success", text: "已连接,只读" } : { tone: "neutral", text: "连接中" }; return (
{site.kind === "subtitle" ? : }
{conn.text} 当前控制:{controllerLabel(s)} {human && leaseHolder ? `(${leaseHolder})` : ""} {waiting ? {waiting} : null} {cred ? ( {cred.text} ) : null}
{human && mine ? ( ) : ( )}

{mine ? "直接在画面里点击和输入。中文、日文用本机输入法输入;复制粘贴用 Ctrl+C / Ctrl+V。" : agentActive(s) ? "只读观看,智能体正在操作。需要自己点或输入时先接管,智能体会暂停。" : "只读观看,目前没有智能体在操作。需要自己点或输入时先接管。"}

{rc.state.phase !== "idle" ? (
) : null} {ready ? (