feat(browser): self-heal sessions after camofox restarts (BR-3)
- Any session operation that hits 404/410 (or finds no open tab) reconciles
tabs; if none survive it re-injects the account cookies (BR-2), opens a new
tab under the same userId/sessionKey, returns to the recorded URL (site home
otherwise), marks old tabs lost and retries the operation once. Failure
returns 503 browser_reconnect_failed with the reason, with a 10s backoff.
- Restart detection: umt plugin GET /umt/instance (fallback /health fields);
a changed instance id marks every open session needsReconnect.
- POST /browser/sessions/{id}/reconnect (idempotent); session view exposes
needsReconnect and reconnect {status, trigger, at, reason, humanNotice}.
- Controller and lease are kept; reconnecting during takeover asks the human
to double-check the reloaded page.
- Web: screenshot and desktop modes show "reconnecting", auto-reconnect once,
pause polling, offer a reconnect button with the reason on failure; desktop
mode rebuilds the Selkies connection after a reconnect.
- Mock camofox restart(), tests, smoke step and runbook section 10.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
da24380870
commit
838aea3ebd
19 files changed
+977
-45
No files matched your search
@@ -43,6 +43,8 @@ import {
|
||||
type OpenSessionResult,
|
||||
type PageMetrics,
|
||||
profileKeyFor,
|
||||
type ReopenSessionRequest,
|
||||
type ReopenSessionResult,
|
||||
type RestoreSessionState,
|
||||
type SnapshotOptions,
|
||||
type TabRef,
|
||||
@@ -153,6 +155,8 @@ export class CamofoxBrowserProvider implements BrowserProvider {
|
||||
private readonly newId: () => string;
|
||||
/** Cookie 接口:plugin = umt-cookies 插件(读写);upstream = 上游导入端点(只写,要求 CAMOFOX_API_KEY 与访问凭据相同);none = 都没有 */
|
||||
private cookieApi: "plugin" | "upstream" | "none" | null = null;
|
||||
/** 实例标识接口:plugin = umt 插件 GET /umt/instance;health = 只能从 /health 里找;null = 未探测 */
|
||||
private instanceApi: "plugin" | "health" | null = null;
|
||||
|
||||
constructor(private readonly cfg: CamofoxConfig) {
|
||||
this.base = cfg.baseUrl.replace(/\/+$/, "");
|
||||
@@ -361,6 +365,82 @@ export class CamofoxBrowserProvider implements BrowserProvider {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 实例标识(BR-3 重启检测):umt 插件 `GET /umt/instance` 的 instanceId(插件加载时生成,进程重启即变);
|
||||
* 插件不在(404)退回 `/health` 里的 instanceId / bootId / startedAt;都没有返回 null。
|
||||
*/
|
||||
async instanceId(): Promise<string | null> {
|
||||
if (this.instanceApi !== "health") {
|
||||
try {
|
||||
const body = await this.json(await this.request("GET", "/umt/instance", { timeoutMs: 5_000 }), "/umt/instance");
|
||||
this.instanceApi = "plugin";
|
||||
return str(body.instanceId) ?? str(body.startedAt) ?? null;
|
||||
} catch (e) {
|
||||
if (!(e instanceof BrowserError && e.httpStatus === 404)) throw e;
|
||||
this.instanceApi = "health";
|
||||
}
|
||||
}
|
||||
const res = await this.fetchImpl(`${this.base}/health`, { signal: AbortSignal.timeout(5_000) });
|
||||
const body = (await res.json().catch(() => ({}))) as Json;
|
||||
const v = body.instanceId ?? body.bootId ?? body.startedAt;
|
||||
return typeof v === "string" || typeof v === "number" ? String(v) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 重建会话的上游上下文(BR-3):上游重启 / 回收后旧标签页都不在了。先(可选)注入 Cookie,再用同一 userId / sessionKey
|
||||
* 开新标签页并导航到 url;旧的打开中标签页标 lost,旧元素引用作废。不检查控制者:接管中的会话重连后仍归原控制者。
|
||||
*/
|
||||
async reopenSession(sessionId: string, req: ReopenSessionRequest): Promise<ReopenSessionResult> {
|
||||
const s = this.state(sessionId);
|
||||
assertNavigable(req.url, req.budget);
|
||||
return this.withProfile(s, async () => {
|
||||
let cookies: ReopenSessionResult["cookies"];
|
||||
if (req.cookies?.length) {
|
||||
try {
|
||||
cookies = { status: "injected", count: (await this.putCookies(s.session.profileKey, req.cookies)).count };
|
||||
} catch (e) {
|
||||
if (e instanceof BrowserError && e.kind === "not_implemented") cookies = { status: "unsupported", count: 0 };
|
||||
else if (e instanceof BrowserError && e.kind !== "auth") cookies = { status: "failed", count: 0 };
|
||||
else throw e;
|
||||
}
|
||||
}
|
||||
const res = await this.request("POST", "/tabs", {
|
||||
body: { userId: s.session.profileKey, sessionKey: s.session.id, url: req.url },
|
||||
timeoutMs: req.budget.timeoutMs,
|
||||
});
|
||||
const body = await this.json(res, "/tabs");
|
||||
const tabId = str(body.tabId);
|
||||
if (!tabId) throw new BrowserError("protocol", "camofox POST /tabs 未返回 tabId");
|
||||
const rawUrl = str(body.url) ?? req.url;
|
||||
this.touch(s, rawUrl);
|
||||
const at = s.session.lastActivityAt;
|
||||
const lost: string[] = [];
|
||||
for (const t of s.tabs.values())
|
||||
if (t.status === "open") {
|
||||
t.status = "lost";
|
||||
t.updatedAt = at;
|
||||
lost.push(t.upstreamTabId);
|
||||
}
|
||||
const tab: BrowserTab = {
|
||||
id: this.newId(),
|
||||
sessionId,
|
||||
upstreamTabId: tabId,
|
||||
url: s.session.currentUrl,
|
||||
title: null,
|
||||
status: "open",
|
||||
lastSnapshotAt: null,
|
||||
closedAt: null,
|
||||
createdAt: at,
|
||||
updatedAt: at,
|
||||
};
|
||||
s.tabs.set(tabId, tab);
|
||||
s.currentTabId = tabId;
|
||||
s.refsValid = false;
|
||||
s.budget = req.budget;
|
||||
return { tab: { ...tab }, lost, ...(cookies ? { cookies } : {}) };
|
||||
});
|
||||
}
|
||||
|
||||
// ---------- 会话 ----------
|
||||
|
||||
async openSession(req: OpenSessionRequest): Promise<OpenSessionResult> {
|
||||
|
||||
@@ -78,6 +78,31 @@ export interface BrowserProvider {
|
||||
* 上游没有接口时报 BrowserError("not_implemented")。
|
||||
*/
|
||||
getCookies?(profileKey: string, urls: string[]): Promise<BrowserCookie[]>;
|
||||
|
||||
// ── BR-3 浏览器服务重启后自愈 ──
|
||||
/**
|
||||
* 浏览器服务实例标识:进程重启后会变(camofox 走 umt 插件 `GET /umt/instance`,否则取 `/health` 里的
|
||||
* instanceId / bootId / startedAt)。取不到(旧镜像)时返回 null,调用方只靠 404/410 被动发现。
|
||||
*/
|
||||
instanceId?(): Promise<string | null>;
|
||||
/**
|
||||
* 重建会话的上游上下文:(可选)先把 Cookie 写进该 profile,再在同一会话(同一 userId / sessionKey)开新标签页并导航到 url;
|
||||
* 原有打开中的标签页一律标 lost。不检查控制者(接管中的会话也能重连,控制权不变)。
|
||||
*/
|
||||
reopenSession?(sessionId: string, req: ReopenSessionRequest): Promise<ReopenSessionResult>;
|
||||
}
|
||||
|
||||
export interface ReopenSessionRequest {
|
||||
url: string;
|
||||
budget: NavigationBudget;
|
||||
cookies?: BrowserCookie[];
|
||||
}
|
||||
|
||||
export interface ReopenSessionResult {
|
||||
tab: BrowserTab;
|
||||
/** 被标 lost 的旧标签页(上游 tabId) */
|
||||
lost: string[];
|
||||
cookies?: { status: "injected" | "unsupported" | "failed"; count: number };
|
||||
}
|
||||
|
||||
export interface PageMetrics {
|
||||
|
||||
Reference in new issue
Block a user