feat(browser): self-heal sessions after camofox restarts (BR-3)

- Any session operation that hits 404/410 (or finds no open tab) reconciles
  tabs; if none survive it re-injects the account cookies (BR-2), opens a new
  tab under the same userId/sessionKey, returns to the recorded URL (site home
  otherwise), marks old tabs lost and retries the operation once. Failure
  returns 503 browser_reconnect_failed with the reason, with a 10s backoff.
- Restart detection: umt plugin GET /umt/instance (fallback /health fields);
  a changed instance id marks every open session needsReconnect.
- POST /browser/sessions/{id}/reconnect (idempotent); session view exposes
  needsReconnect and reconnect {status, trigger, at, reason, humanNotice}.
- Controller and lease are kept; reconnecting during takeover asks the human
  to double-check the reloaded page.
- Web: screenshot and desktop modes show "reconnecting", auto-reconnect once,
  pause polling, offer a reconnect button with the reason on failure; desktop
  mode rebuilds the Selkies connection after a reconnect.
- Mock camofox restart(), tests, smoke step and runbook section 10.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

packages/adapters/src/browser/camofox/client.ts

@@ -43,6 +43,8 @@ import {
   type OpenSessionResult,
   type PageMetrics,
   profileKeyFor,
+  type ReopenSessionRequest,
+  type ReopenSessionResult,
   type RestoreSessionState,
   type SnapshotOptions,
   type TabRef,
@@ -153,6 +155,8 @@ export class CamofoxBrowserProvider implements BrowserProvider {
   private readonly newId: () => string;
   /** Cookie 接口:plugin = umt-cookies 插件(读写);upstream = 上游导入端点(只写,要求 CAMOFOX_API_KEY 与访问凭据相同);none = 都没有 */
   private cookieApi: "plugin" | "upstream" | "none" | null = null;
+  /** 实例标识接口:plugin = umt 插件 GET /umt/instance;health = 只能从 /health 里找;null = 未探测 */
+  private instanceApi: "plugin" | "health" | null = null;
 
   constructor(private readonly cfg: CamofoxConfig) {
     this.base = cfg.baseUrl.replace(/\/+$/, "");
@@ -361,6 +365,82 @@ export class CamofoxBrowserProvider implements BrowserProvider {
     }
   }
 
+  /**
+   * 实例标识(BR-3 重启检测):umt 插件 `GET /umt/instance` 的 instanceId(插件加载时生成,进程重启即变);
+   * 插件不在(404)退回 `/health` 里的 instanceId / bootId / startedAt;都没有返回 null。
+   */
+  async instanceId(): Promise<string | null> {
+    if (this.instanceApi !== "health") {
+      try {
+        const body = await this.json(await this.request("GET", "/umt/instance", { timeoutMs: 5_000 }), "/umt/instance");
+        this.instanceApi = "plugin";
+        return str(body.instanceId) ?? str(body.startedAt) ?? null;
+      } catch (e) {
+        if (!(e instanceof BrowserError && e.httpStatus === 404)) throw e;
+        this.instanceApi = "health";
+      }
+    }
+    const res = await this.fetchImpl(`${this.base}/health`, { signal: AbortSignal.timeout(5_000) });
+    const body = (await res.json().catch(() => ({}))) as Json;
+    const v = body.instanceId ?? body.bootId ?? body.startedAt;
+    return typeof v === "string" || typeof v === "number" ? String(v) : null;
+  }
+
+  /**
+   * 重建会话的上游上下文(BR-3):上游重启 / 回收后旧标签页都不在了。先(可选)注入 Cookie,再用同一 userId / sessionKey
+   * 开新标签页并导航到 url;旧的打开中标签页标 lost,旧元素引用作废。不检查控制者:接管中的会话重连后仍归原控制者。
+   */
+  async reopenSession(sessionId: string, req: ReopenSessionRequest): Promise<ReopenSessionResult> {
+    const s = this.state(sessionId);
+    assertNavigable(req.url, req.budget);
+    return this.withProfile(s, async () => {
+      let cookies: ReopenSessionResult["cookies"];
+      if (req.cookies?.length) {
+        try {
+          cookies = { status: "injected", count: (await this.putCookies(s.session.profileKey, req.cookies)).count };
+        } catch (e) {
+          if (e instanceof BrowserError && e.kind === "not_implemented") cookies = { status: "unsupported", count: 0 };
+          else if (e instanceof BrowserError && e.kind !== "auth") cookies = { status: "failed", count: 0 };
+          else throw e;
+        }
+      }
+      const res = await this.request("POST", "/tabs", {
+        body: { userId: s.session.profileKey, sessionKey: s.session.id, url: req.url },
+        timeoutMs: req.budget.timeoutMs,
+      });
+      const body = await this.json(res, "/tabs");
+      const tabId = str(body.tabId);
+      if (!tabId) throw new BrowserError("protocol", "camofox POST /tabs 未返回 tabId");
+      const rawUrl = str(body.url) ?? req.url;
+      this.touch(s, rawUrl);
+      const at = s.session.lastActivityAt;
+      const lost: string[] = [];
+      for (const t of s.tabs.values())
+        if (t.status === "open") {
+          t.status = "lost";
+          t.updatedAt = at;
+          lost.push(t.upstreamTabId);
+        }
+      const tab: BrowserTab = {
+        id: this.newId(),
+        sessionId,
+        upstreamTabId: tabId,
+        url: s.session.currentUrl,
+        title: null,
+        status: "open",
+        lastSnapshotAt: null,
+        closedAt: null,
+        createdAt: at,
+        updatedAt: at,
+      };
+      s.tabs.set(tabId, tab);
+      s.currentTabId = tabId;
+      s.refsValid = false;
+      s.budget = req.budget;
+      return { tab: { ...tab }, lost, ...(cookies ? { cookies } : {}) };
+    });
+  }
+
   // ---------- 会话 ----------
 
   async openSession(req: OpenSessionRequest): Promise<OpenSessionResult> {

packages/adapters/src/browser/provider.ts

@@ -78,6 +78,31 @@ export interface BrowserProvider {
    * 上游没有接口时报 BrowserError("not_implemented")。
    */
   getCookies?(profileKey: string, urls: string[]): Promise<BrowserCookie[]>;
+
+  // ── BR-3 浏览器服务重启后自愈 ──
+  /**
+   * 浏览器服务实例标识:进程重启后会变(camofox 走 umt 插件 `GET /umt/instance`,否则取 `/health` 里的
+   * instanceId / bootId / startedAt)。取不到(旧镜像)时返回 null,调用方只靠 404/410 被动发现。
+   */
+  instanceId?(): Promise<string | null>;
+  /**
+   * 重建会话的上游上下文:(可选)先把 Cookie 写进该 profile,再在同一会话(同一 userId / sessionKey)开新标签页并导航到 url;
+   * 原有打开中的标签页一律标 lost。不检查控制者(接管中的会话也能重连,控制权不变)。
+   */
+  reopenSession?(sessionId: string, req: ReopenSessionRequest): Promise<ReopenSessionResult>;
+}
+
+export interface ReopenSessionRequest {
+  url: string;
+  budget: NavigationBudget;
+  cookies?: BrowserCookie[];
+}
+
+export interface ReopenSessionResult {
+  tab: BrowserTab;
+  /** 被标 lost 的旧标签页(上游 tabId) */
+  lost: string[];
+  cookies?: { status: "injected" | "unsupported" | "failed"; count: number };
 }
 
 export interface PageMetrics {

packages/contracts/src/api.ts

@@ -2400,6 +2400,15 @@ export const Api = {
       body: browser.BrowserDownload,
       response: browser.BrowserDownloadResult,
     }),
+    reconnect: ep({
+      method: "post",
+      path: "/browser/sessions/{id}/reconnect",
+      scope: "admin",
+      summary:
+        "浏览器服务重启后重新连接（幂等）：上游上下文还在就什么都不做；否则注入账号 Cookie、新开标签页回到原地址，控制权不变；失败 503 browser_reconnect_failed",
+      params: browser.BrowserSessionParams,
+      response: browser.BrowserSessionView,
+    }),
     acquire: ep({
       method: "post",
       path: "/browser/sessions/{id}/control/acquire",

packages/contracts/src/browser.ts

@@ -327,6 +327,21 @@ export const BrowserCredentialState = z.object({
 });
 export type BrowserCredentialState = z.infer<typeof BrowserCredentialState>;
 
+/**
+ * 浏览器服务重启 / 回收后的自动重连(BR-3,存 recovery_state.reconnect):
+ * trigger = tab_gone(操作时发现标签页不在)/ provider_restarted(实例标识变了)/ manual(原站视图调用 reconnect);
+ * humanNotice = 重连时会话正由人工接管(页面已重新加载,请核对)。
+ */
+export const BrowserReconnectState = z.object({
+  status: z.enum(["ok", "failed"]),
+  trigger: z.enum(["tab_gone", "provider_restarted", "manual"]),
+  at: Timestamp,
+  /** 失败原因(不含凭据) */
+  reason: z.string().nullable(),
+  humanNotice: z.boolean(),
+});
+export type BrowserReconnectState = z.infer<typeof BrowserReconnectState>;
+
 export const BrowserSessionView = BrowserSession.extend({
   tabs: z.array(BrowserTab),
   currentTabId: z.string().nullable(),
@@ -337,6 +352,10 @@ export const BrowserSessionView = BrowserSession.extend({
   waitingReason: z.string().nullable(),
   /** 打开会话时的凭据注入与回写状态(BR-2);旧会话没有 */
   credential: BrowserCredentialState.nullable().optional(),
+  /** 浏览器服务重启过、上游上下文已不在:下一次操作(或 POST .../reconnect)自动重建(BR-3) */
+  needsReconnect: z.boolean().optional(),
+  /** 最近一次自动重连的结果 */
+  reconnect: BrowserReconnectState.nullable().optional(),
 });
 export type BrowserSessionView = z.infer<typeof BrowserSessionView>;
 

packages/contracts/src/common.ts

@@ -35,6 +35,7 @@ export const ApiErrorCode = z.enum([
   "rate_limited",
   "automation_disabled",
   "upstream_unavailable",
+  "browser_reconnect_failed",
   "internal",
 ]);
 export type ApiErrorCode = z.infer<typeof ApiErrorCode>;
@@ -51,6 +52,8 @@ export const API_ERROR_STATUS = {
   rate_limited: 429,
   automation_disabled: 423,
   upstream_unavailable: 502,
+  /** 浏览器服务重启后自动重连失败(BR-3):浏览器服务暂不可用,稍后重试 */
+  browser_reconnect_failed: 503,
   internal: 500,
 } as const satisfies Record<ApiErrorCode, number>;
 

packages/testkit/src/browser/mock-camofox.ts

@@ -119,6 +119,13 @@ export type MockCamofox = MockHttp & {
   expireLogin(userId?: string): void;
   /** 模拟上游回收标签页(空闲超时 / 重启) */
   dropTab(tabId: string): void;
+  /**
+   * 模拟 camofox 容器重启(BR-3):全部标签页消失,旧 tabId 之后返回 410 browser_restarted(与 1.17.0 实测一致),
+   * 实例标识(`GET /umt/instance`)换新。loseProfiles = true 时连 profile 的 Cookie / 登录态一起丢(SIGKILL 未落盘)。
+   */
+  restart(opts?: { loseProfiles?: boolean }): void;
+  /** 当前实例标识 */
+  readonly instanceId: string;
 };
 
 /** 1x1 透明 PNG */
@@ -325,6 +332,9 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise<
   const loggedIn = new Set<string>();
   const cookieJars = new Map<string, MockCamofoxCookie[]>();
   const cookieApi = opts.cookieApi ?? "plugin";
+  let instanceId = `inst-${crypto.randomUUID()}`;
+  /** 重启前存在过的 tabId:之后访问返回 410 */
+  const restartedTabs = new Set<string>();
   const site = opts.site;
   const putCookie = (userId: string, c: MockCamofoxCookie) => {
     const jar = (cookieJars.get(userId) ?? []).filter(
@@ -784,6 +794,7 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise<
           return { json: { downloads: list } };
         },
       ],
+      [/^\/umt\/instance$/, () => (cookieApi === "plugin" ? { json: { instanceId } } : err(404, "no route"))],
       [
         /^\/umt\/sessions\/([^/]+)\/cookies$/,
         (m, q, req, b) => {
@@ -829,12 +840,20 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise<
       guard: (req) => {
         const path = new URL(req.url ?? "/", "http://mock").pathname;
         // 与上游一致:导入端点有自己的口令(CAMOFOX_API_KEY),不走访问凭据
-        if (!opts.accessKey || path === "/health" || /^\/sessions\/[^/]+\/cookies$/.test(path)) return null;
-        return req.headers.authorization === `Bearer ${opts.accessKey}` ? null : err(401, "Unauthorized");
+        const gone = /^\/tabs\/([^/]+)/.exec(path)?.[1];
+        const restarted =
+          gone && restartedTabs.has(decodeURIComponent(gone))
+            ? {
+                status: 410,
+                json: { error: "Tab no longer exists (browser was restarted).", code: "browser_restarted" },
+              }
+            : null;
+        if (!opts.accessKey || path === "/health" || /^\/sessions\/[^/]+\/cookies$/.test(path)) return restarted;
+        return req.headers.authorization === `Bearer ${opts.accessKey}` ? restarted : err(401, "Unauthorized");
       },
     },
   );
-  return Object.assign(mock, {
+  const out = Object.assign(mock, {
     tabs,
     actions,
     loggedIn,
@@ -846,5 +865,17 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise<
     dropTab(tabId: string) {
       tabs.delete(tabId);
     },
+    restart(o: { loseProfiles?: boolean } = {}) {
+      for (const id of tabs.keys()) restartedTabs.add(id);
+      tabs.clear();
+      instanceId = `inst-${crypto.randomUUID()}`;
+      if (o.loseProfiles) {
+        loggedIn.clear();
+        cookieJars.clear();
+      }
+    },
   });
+  // Object.assign 会把 getter 取值成常量:单独定义
+  Object.defineProperty(out, "instanceId", { get: () => instanceId, enumerable: true });
+  return out as MockCamofox;
 }