feat(browser): self-heal sessions after camofox restarts (BR-3)

- Any session operation that hits 404/410 (or finds no open tab) reconciles
  tabs; if none survive it re-injects the account cookies (BR-2), opens a new
  tab under the same userId/sessionKey, returns to the recorded URL (site home
  otherwise), marks old tabs lost and retries the operation once. Failure
  returns 503 browser_reconnect_failed with the reason, with a 10s backoff.
- Restart detection: umt plugin GET /umt/instance (fallback /health fields);
  a changed instance id marks every open session needsReconnect.
- POST /browser/sessions/{id}/reconnect (idempotent); session view exposes
  needsReconnect and reconnect {status, trigger, at, reason, humanNotice}.
- Controller and lease are kept; reconnecting during takeover asks the human
  to double-check the reloaded page.
- Web: screenshot and desktop modes show "reconnecting", auto-reconnect once,
  pause polling, offer a reconnect button with the reason on failure; desktop
  mode rebuilds the Selkies connection after a reconnect.
- Mock camofox restart(), tests, smoke step and runbook section 10.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed 2026-09-29 17:29:11 +08:00
1 parent da24380870
commit 838aea3ebd
19 files changed
+977 -45

No files matched your search

@@ -43,6 +43,8 @@ import {
type OpenSessionResult,
type PageMetrics,
profileKeyFor,
type ReopenSessionRequest,
type ReopenSessionResult,
type RestoreSessionState,
type SnapshotOptions,
type TabRef,
@@ -153,6 +155,8 @@ export class CamofoxBrowserProvider implements BrowserProvider {
private readonly newId: () => string;
/** Cookie 接口:plugin = umt-cookies 插件(读写);upstream = 上游导入端点(只写,要求 CAMOFOX_API_KEY 与访问凭据相同);none = 都没有 */
private cookieApi: "plugin" | "upstream" | "none" | null = null;
/** 实例标识接口:plugin = umt 插件 GET /umt/instance;health = 只能从 /health 里找;null = 未探测 */
private instanceApi: "plugin" | "health" | null = null;
constructor(private readonly cfg: CamofoxConfig) {
this.base = cfg.baseUrl.replace(/\/+$/, "");
@@ -361,6 +365,82 @@ export class CamofoxBrowserProvider implements BrowserProvider {
}
}
/**
* 实例标识(BR-3 重启检测):umt 插件 `GET /umt/instance` 的 instanceId(插件加载时生成,进程重启即变);
* 插件不在(404)退回 `/health` 里的 instanceId / bootId / startedAt;都没有返回 null。
*/
async instanceId(): Promise<string | null> {
if (this.instanceApi !== "health") {
try {
const body = await this.json(await this.request("GET", "/umt/instance", { timeoutMs: 5_000 }), "/umt/instance");
this.instanceApi = "plugin";
return str(body.instanceId) ?? str(body.startedAt) ?? null;
} catch (e) {
if (!(e instanceof BrowserError && e.httpStatus === 404)) throw e;
this.instanceApi = "health";
}
}
const res = await this.fetchImpl(`${this.base}/health`, { signal: AbortSignal.timeout(5_000) });
const body = (await res.json().catch(() => ({}))) as Json;
const v = body.instanceId ?? body.bootId ?? body.startedAt;
return typeof v === "string" || typeof v === "number" ? String(v) : null;
}
/**
* 重建会话的上游上下文(BR-3):上游重启 / 回收后旧标签页都不在了。先(可选)注入 Cookie,再用同一 userId / sessionKey
* 开新标签页并导航到 url;旧的打开中标签页标 lost,旧元素引用作废。不检查控制者:接管中的会话重连后仍归原控制者。
*/
async reopenSession(sessionId: string, req: ReopenSessionRequest): Promise<ReopenSessionResult> {
const s = this.state(sessionId);
assertNavigable(req.url, req.budget);
return this.withProfile(s, async () => {
let cookies: ReopenSessionResult["cookies"];
if (req.cookies?.length) {
try {
cookies = { status: "injected", count: (await this.putCookies(s.session.profileKey, req.cookies)).count };
} catch (e) {
if (e instanceof BrowserError && e.kind === "not_implemented") cookies = { status: "unsupported", count: 0 };
else if (e instanceof BrowserError && e.kind !== "auth") cookies = { status: "failed", count: 0 };
else throw e;
}
}
const res = await this.request("POST", "/tabs", {
body: { userId: s.session.profileKey, sessionKey: s.session.id, url: req.url },
timeoutMs: req.budget.timeoutMs,
});
const body = await this.json(res, "/tabs");
const tabId = str(body.tabId);
if (!tabId) throw new BrowserError("protocol", "camofox POST /tabs 未返回 tabId");
const rawUrl = str(body.url) ?? req.url;
this.touch(s, rawUrl);
const at = s.session.lastActivityAt;
const lost: string[] = [];
for (const t of s.tabs.values())
if (t.status === "open") {
t.status = "lost";
t.updatedAt = at;
lost.push(t.upstreamTabId);
}
const tab: BrowserTab = {
id: this.newId(),
sessionId,
upstreamTabId: tabId,
url: s.session.currentUrl,
title: null,
status: "open",
lastSnapshotAt: null,
closedAt: null,
createdAt: at,
updatedAt: at,
};
s.tabs.set(tabId, tab);
s.currentTabId = tabId;
s.refsValid = false;
s.budget = req.budget;
return { tab: { ...tab }, lost, ...(cookies ? { cookies } : {}) };
});
}
// ---------- 会话 ----------
async openSession(req: OpenSessionRequest): Promise<OpenSessionResult> {
+25
View File
@@ -78,6 +78,31 @@ export interface BrowserProvider {
* 上游没有接口时报 BrowserError("not_implemented")。
*/
getCookies?(profileKey: string, urls: string[]): Promise<BrowserCookie[]>;
// ── BR-3 浏览器服务重启后自愈 ──
/**
* 浏览器服务实例标识:进程重启后会变(camofox 走 umt 插件 `GET /umt/instance`,否则取 `/health` 里的
* instanceId / bootId / startedAt)。取不到(旧镜像)时返回 null,调用方只靠 404/410 被动发现。
*/
instanceId?(): Promise<string | null>;
/**
* 重建会话的上游上下文:(可选)先把 Cookie 写进该 profile,再在同一会话(同一 userId / sessionKey)开新标签页并导航到 url;
* 原有打开中的标签页一律标 lost。不检查控制者(接管中的会话也能重连,控制权不变)。
*/
reopenSession?(sessionId: string, req: ReopenSessionRequest): Promise<ReopenSessionResult>;
}
export interface ReopenSessionRequest {
url: string;
budget: NavigationBudget;
cookies?: BrowserCookie[];
}
export interface ReopenSessionResult {
tab: BrowserTab;
/** 被标 lost 的旧标签页(上游 tabId) */
lost: string[];
cookies?: { status: "injected" | "unsupported" | "failed"; count: number };
}
export interface PageMetrics {