feat(browser): self-heal sessions after camofox restarts (BR-3)

- Any session operation that hits 404/410 (or finds no open tab) reconciles
  tabs; if none survive it re-injects the account cookies (BR-2), opens a new
  tab under the same userId/sessionKey, returns to the recorded URL (site home
  otherwise), marks old tabs lost and retries the operation once. Failure
  returns 503 browser_reconnect_failed with the reason, with a 10s backoff.
- Restart detection: umt plugin GET /umt/instance (fallback /health fields);
  a changed instance id marks every open session needsReconnect.
- POST /browser/sessions/{id}/reconnect (idempotent); session view exposes
  needsReconnect and reconnect {status, trigger, at, reason, humanNotice}.
- Controller and lease are kept; reconnecting during takeover asks the human
  to double-check the reloaded page.
- Web: screenshot and desktop modes show "reconnecting", auto-reconnect once,
  pause polling, offer a reconnect button with the reason on failure; desktop
  mode rebuilds the Selkies connection after a reconnect.
- Mock camofox restart(), tests, smoke step and runbook section 10.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed 2026-09-29 17:29:11 +08:00
1 parent da24380870
commit 838aea3ebd
19 files changed
+977 -45

No files matched your search

@@ -204,7 +204,7 @@ describe.skipIf(!up)("浏览器 HTTP 端点", () => {
expect(d2.status).toBe("resolved");
});
it("服务重启:按数据库恢复会话,旧引用失效;上游回收的标签页标 lost", async () => {
it("服务重启:按数据库恢复会话,旧引用失效;上游回收的标签页标 lost 并自动重开", async () => {
// 新模块实例 = 进程重启(适配器内存为空)
const fresh = createBrowserModule({ stagingDir: h.stagingDir });
const reg = (await import("../../agent/capability-registry.ts")).createCapabilityRegistry({
@@ -225,9 +225,10 @@ describe.skipIf(!up)("浏览器 HTTP 端点", () => {
await h.db.selectFrom("browser_tab").selectAll().where("session_id", "=", sid).executeTakeFirstOrThrow()
).upstream_tab_id;
h.mock.dropTab(tab);
await expect(reg.invoke("browser.snapshot", { sessionId: sid }, h.agentCtx())).rejects.toMatchObject({
code: "conflict",
});
// BR-3:上游回收后自愈 —— 同一会话新开标签页回到原地址,重试一次
const healed = (await reg.invoke("browser.snapshot", { sessionId: sid }, h.agentCtx())) as Loose;
expect(healed.snapshot.text).toContain("种子列表");
expect(healed.snapshot.tabId).not.toBe(tab);
const t = await h.db
.selectFrom("browser_tab")
.select("status")
@@ -0,0 +1,143 @@
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { type BrowserHarness, createBrowserHarness, type Loose, ORIGIN, pgAvailable } from "./harness.ts";
// BR-3 浏览器服务重启后自愈(对 testkit mock camofox):容器重启 → 旧标签页 410、GET /tabs 为空、实例标识换新、
// profile 登录态丢失(SIGKILL 未落盘)。任一会话操作 → 注入账号 Cookie → 同一会话新开标签页回到原地址 → 重试原操作;
// 失败 503 browser_reconnect_failed;POST .../reconnect 幂等;人工接管中重连控制权不变并提示核对。
const up = await pgAvailable();
const GOOD = "good-session-value-br3";
describe.skipIf(!up)("BR-3 浏览器服务重启后自动重连", () => {
let h: BrowserHarness;
const setCookieAccount = async (cookie: string) => {
const ref = await h.secrets.create({ purpose: "site_cookie", plaintext: cookie });
await h.db
.updateTable("site_account")
.set({ auth_kind: "cookie", secret_ref: ref, session_secret_ref: null, status: "active", user_info: "{}" })
.where("id", "=", h.accountId)
.execute();
};
const open = async (path: string) =>
(await h.req("/browser/sessions", { body: { siteId: h.siteId, accountId: h.accountId, url: `${ORIGIN}${path}` } }))
.body.data as Loose;
const closeAll = async () => {
for (const s of (await h.req("/browser/sessions")).body.data.items as Loose[])
await h.req(`/browser/sessions/${s.id}`, { method: "DELETE" });
};
const tabRow = (upstream: string) =>
h.db.selectFrom("browser_tab").selectAll().where("upstream_tab_id", "=", upstream).executeTakeFirstOrThrow();
const cookiePosts = (profileKey: string) =>
h.mock.callsTo(`/umt/sessions/${encodeURIComponent(profileKey)}/cookies`).filter((c) => c.method === "POST").length;
beforeAll(async () => {
h = await createBrowserHarness("br3", {
sessionCookie: { name: "c_secure_pass", valid: [GOOD], fresh: "fresh-br3" },
});
await setCookieAccount(`c_secure_uid=1001; c_secure_pass=${GOOD}`);
}, 120_000);
afterAll(async () => {
await h?.close();
});
it("重启后截图:标签页 410 → 注入 Cookie、新开标签页回到原地址、重试成功;旧标签页标 lost", async () => {
const s = await open("/details.php?id=42");
expect(s.currentUrl).toBe(`${ORIGIN}/details.php?id=42`);
const old = s.currentTabId as string;
const injected = cookiePosts(s.profileKey);
h.mock.restart({ loseProfiles: true });
const r = await h.req(`/browser/sessions/${s.id}/screenshot`);
expect(r.status).toBe(200);
expect(r.body.data.url).toBe(`${ORIGIN}/details.php?id=42`);
// Cookie 注入先于新标签页;profile 丢了登录态也能回到受保护页(不是登录页)
expect(cookiePosts(s.profileKey)).toBe(injected + 1);
const calls = h.mock.calls.map((c) => `${c.method} ${c.path}`);
expect(calls.lastIndexOf(`POST /umt/sessions/${encodeURIComponent(s.profileKey)}/cookies`)).toBeLessThan(
calls.lastIndexOf("POST /tabs"),
);
const tab = [...h.mock.tabs.values()].find((t) => t.userId === s.profileKey)!;
expect(tab.sessionKey).toBe(s.id);
expect(tab.url).toBe(`${ORIGIN}/details.php?id=42`);
expect(h.mock.loggedIn.has(s.profileKey)).toBe(true);
const v = (await h.req(`/browser/sessions/${s.id}`)).body.data as Loose;
expect(v.currentTabId).toBe(tab.tabId);
expect(v.currentTabId).not.toBe(old);
expect(v).toMatchObject({ controller: "automation", needsReconnect: false, needsSnapshot: true });
expect(v.reconnect).toMatchObject({ status: "ok", humanNotice: false, reason: null });
expect((await tabRow(old)).status).toBe("lost");
expect(JSON.stringify(v)).not.toContain(GOOD);
await closeAll();
});
it("重启检测:实例标识变了 → 会话标需要重连;POST .../reconnect 幂等", async () => {
const s = await open("/torrents.php");
h.mock.restart();
// 会话详情轮询发现重启(实例标识缓存 5 秒:用显式 reconnect 强制刷新)
const r1 = await h.req(`/browser/sessions/${s.id}/reconnect`, { body: {} });
expect(r1.status).toBe(200);
expect(r1.body.data).toMatchObject({
needsReconnect: false,
reconnect: { status: "ok", trigger: "provider_restarted" },
});
expect(r1.body.data.currentUrl).toBe(`${ORIGIN}/torrents.php`);
const opened = h.mock.callsTo(/^\/tabs$/).filter((c) => c.method === "POST").length;
const r2 = await h.req(`/browser/sessions/${s.id}/reconnect`, { body: {} });
expect(r2.status).toBe(200);
expect(r2.body.data.currentTabId).toBe(r1.body.data.currentTabId);
expect(h.mock.callsTo(/^\/tabs$/).filter((c) => c.method === "POST").length).toBe(opened);
// Agent 快照也走同一条自愈
h.mock.restart();
const snap = await h.req(`/browser/sessions/${s.id}/snapshot`);
expect(snap.status).toBe(200);
expect(snap.body.data.url).toBe(`${ORIGIN}/torrents.php`);
await closeAll();
});
it("人工接管中重连:控制者与租约不变,提示页面已重新加载", async () => {
const s = await open("/torrents.php");
const acq = await h.req(`/browser/sessions/${s.id}/control/acquire`, { body: { reason: "核对" } });
const leaseId = acq.body.data.lease.id;
h.mock.restart();
const r = await h.req(`/browser/sessions/${s.id}/screenshot`);
expect(r.status).toBe(200);
expect(r.body.data.controller).toBe("human");
const v = (await h.req(`/browser/sessions/${s.id}`)).body.data as Loose;
expect(v.lease).toMatchObject({ id: leaseId, status: "active" });
expect(v.reconnect).toMatchObject({ status: "ok", humanNotice: true });
// 接管者的输入照常可用
const nav = await h.req(`/browser/sessions/${s.id}/navigate`, { body: { url: `${ORIGIN}/index.php` } });
expect(nav.status).toBe(200);
await closeAll();
});
it("重连失败:503 browser_reconnect_failed 带原因;短时间内不反复重连;恢复后手动重连成功", async () => {
const s = await open("/torrents.php");
h.mock.restart();
h.mock.fault(/^\/tabs$/, { status: 500, times: 10 });
const r = await h.req(`/browser/sessions/${s.id}/screenshot`);
expect(r.status).toBe(503);
expect(r.body.error.code).toBe("browser_reconnect_failed");
expect(r.body.error.message).toContain("HTTP 500");
const v = (await h.req(`/browser/sessions/${s.id}`)).body.data as Loose;
expect(v).toMatchObject({ needsReconnect: true, reconnect: { status: "failed", trigger: "tab_gone" } });
// 退避期内的截图直接报上次原因,不再打上游
const before = h.mock.calls.length;
const again = await h.req(`/browser/sessions/${s.id}/screenshot`);
expect(again.status).toBe(503);
expect(h.mock.calls.slice(before).some((c) => c.path === "/tabs")).toBe(false);
h.mock.clearFaults();
const ok = await h.req(`/browser/sessions/${s.id}/reconnect`, { body: {} });
expect(ok.status).toBe(200);
expect(ok.body.data).toMatchObject({ needsReconnect: false, reconnect: { status: "ok" } });
expect((await h.req(`/browser/sessions/${s.id}/screenshot`)).status).toBe(200);
await closeAll();
});
});
@@ -15,6 +15,7 @@ const CODE = {
bad_request: "precondition",
upstream: "precondition",
rate_limited: "budget",
reconnect_failed: "precondition",
} as const;
async function guard<T>(fn: () => Promise<T>): Promise<T> {
@@ -35,6 +35,7 @@ const CODE = {
bad_request: "bad_request",
upstream: "upstream_unavailable",
rate_limited: "rate_limited",
reconnect_failed: "browser_reconnect_failed",
} as const;
function route<D extends EndpointDef>(app: ModuleApp, def: D, handler: (input: Input<D>) => Promise<unknown>) {
@@ -284,6 +285,12 @@ export function registerBrowserRoutes(app: ModuleApp, svc: () => BrowserService)
};
});
// BR-3:浏览器服务重启后重新连接(幂等;原站视图收到 404/410/503 时调用一次)。不要求接管:控制权不变
route(app, B.reconnect, async ({ params }) => {
await svc().reconnectIfNeeded(params.id);
return svc().view(params.id);
});
route(app, B.acquire, async ({ params, body, c }) => {
const lease = await svc().acquire(params.id, human(c).holder, body.reason, body.ttlMs);
return { lease, session: await svc().view(params.id) };
+259 -9
View File
@@ -32,6 +32,7 @@ import type {
BrowserEvidenceRef,
BrowserHealth,
BrowserProviderView,
BrowserReconnectState,
BrowserSession,
BrowserSessionView,
BrowserTab,
@@ -88,7 +89,14 @@ export interface ProviderDeps {
/** 业务错误(路由层转 ApiException,能力层转 CapabilityError) */
export class BrowserServiceError extends Error {
constructor(
readonly code: "not_found" | "conflict" | "precondition" | "bad_request" | "upstream" | "rate_limited",
readonly code:
| "not_found"
| "conflict"
| "precondition"
| "bad_request"
| "upstream"
| "rate_limited"
| "reconnect_failed",
message: string,
) {
super(message);
@@ -147,6 +155,26 @@ export interface RecoveryState {
downloads?: { sha256: string; artifactId: string; sourceReleaseId: string | null; planId: string | null }[];
/** BR-2:打开时的凭据注入结果与最近一次回写检查 */
credential?: BrowserCredentialState;
/** BR-3:上游上下文已不在(浏览器服务重启),下一次操作前先重连 */
needsReconnect?: boolean;
reconnectTrigger?: BrowserReconnectState["trigger"];
/** 打开 / 重连时浏览器服务的实例标识;变了 = 浏览器服务重启过 */
instanceId?: string | null;
reconnect?: BrowserReconnectState;
}
/** 同一浏览器服务的实例标识多久查一次 */
const INSTANCE_CHECK_MS = 5_000;
/** 自动重连失败后,操作触发的重连至少间隔多久 */
const RECONNECT_BACKOFF_MS = 10_000;
/** 上游说标签页 / 会话不在了:404 / 410,或适配器内存里已没有打开的标签页 */
function isGone(e: unknown): boolean {
return (
e instanceof BrowserError &&
e.kind === "not_found" &&
(e.httpStatus === undefined || e.httpStatus === 404 || e.httpStatus === 410)
);
}
/** 有活动的会话每隔多久从浏览器读回一次 Cookie */
@@ -631,6 +659,9 @@ export class BrowserService {
async view(id: string): Promise<BrowserSessionView> {
let row = await this.sessionRow(id);
row = await this.expireLeases(row);
// 原站视图轮询会话详情:顺带发现浏览器服务重启(节流,失败不影响详情)
if (row.controller !== "closed" && (await this.checkInstance(row.provider_id).catch(() => false)))
row = await this.sessionRow(id);
const [tabs, lease] = await Promise.all([this.tabRows(id), this.activeLease(id)]);
const rs = obj(row.recovery_state) as RecoveryState;
return {
@@ -641,6 +672,8 @@ export class BrowserService {
needsSnapshot: rs.needsSnapshot === true,
waitingReason: row.controller === "waiting" || row.controller === "human" ? (rs.waitingReason ?? null) : null,
credential: rs.credential ?? null,
needsReconnect: rs.needsReconnect === true,
reconnect: rs.reconnect ?? null,
};
}
@@ -716,6 +749,7 @@ export class BrowserService {
throw fromBrowserError(e);
}
const s = res.session;
const instanceId = await this.currentInstance(prow, true).catch(() => null);
const credential: BrowserCredentialState = { ...prep.state };
if (res.cookies) {
credential.status = res.cookies.status;
@@ -745,7 +779,12 @@ export class BrowserService {
current_url: s.currentUrl,
last_activity_at: new Date(s.lastActivityAt),
upstream_ref: JSON.stringify(s.upstreamRef),
recovery_state: JSON.stringify({ currentTabId: res.tab.upstreamTabId, needsSnapshot: true, credential }),
recovery_state: JSON.stringify({
currentTabId: res.tab.upstreamTabId,
needsSnapshot: true,
credential,
instanceId,
}),
})
.execute();
await trx
@@ -1173,7 +1212,18 @@ export class BrowserService {
);
}
}
const loaded = await this.loaded(row);
// BR-3:浏览器服务重启过(实例标识变了)→ 先重建上游上下文再操作
if (await this.checkInstance(row.provider_id).catch(() => false)) row = await this.sessionRow(id);
const pending = obj(row.recovery_state) as RecoveryState;
if (pending.needsReconnect) {
// 刚失败过:短时间内不反复重连(原站视图 1.5 秒一次截图),直接报上次的原因;显式 POST .../reconnect 不受限
const last = pending.reconnect;
if (last?.status === "failed" && this.now().getTime() - new Date(last.at).getTime() < RECONNECT_BACKOFF_MS)
throw new BrowserServiceError("reconnect_failed", `浏览器已重启,重新连接失败:${last.reason ?? "未知原因"}`);
await this.reconnect(id, pending.reconnectTrigger ?? "provider_restarted");
row = await this.sessionRow(id);
}
let loaded = await this.loaded(row);
row = loaded.row;
const prow = await this.providerRow(row.provider_id).catch(() => null);
const site = await this.liveSite(row.site_id);
@@ -1185,19 +1235,211 @@ export class BrowserService {
try {
out = await fn(loaded.provider, ref, { row, site, budget });
} catch (e) {
if (e instanceof BrowserError && e.kind === "not_found" && (e.httpStatus === 404 || e.httpStatus === 410)) {
const r = await loaded.provider.reconcileTabs(id).catch(() => null);
await this.persist(id, loaded.provider, { needsSnapshot: true });
if (r?.lost.length) throw new BrowserServiceError("conflict", "标签页已被浏览器服务回收,重新打开页面");
if (!isGone(e)) {
await this.persist(id, loaded.provider).catch(() => {});
throw fromBrowserError(e);
}
// 上游说标签页 / 会话不在了:先对账。当前标签页还活着 → 指定的旧标签页没了照旧报错,否则在当前标签页重试一次;
// 全没了(浏览器服务重启 / 回收)→ 注入 Cookie、新开标签页回到原地址,再重试一次(失败 503 browser_reconnect_failed)
const r = await loaded.provider.reconcileTabs(id).catch(() => null);
await this.persist(id, loaded.provider, { needsSnapshot: true }).catch(() => {});
const alive = r ? (loaded.provider.sessionInfo(id)?.currentTabId ?? null) : null;
if (alive && opts.tabId && opts.tabId !== alive) {
if (r?.lost.includes(opts.tabId))
throw new BrowserServiceError("conflict", "标签页已被浏览器服务回收,重新打开页面");
throw fromBrowserError(e);
}
if (!alive) {
await this.markReconnect(id, "tab_gone");
await this.reconnect(id, "tab_gone");
loaded = await this.loaded(await this.sessionRow(id));
row = loaded.row;
}
const retry: TabRef = { sessionId: id, as: actor.kind };
try {
out = await fn(loaded.provider, retry, { row, site, budget });
} catch (e2) {
await this.persist(id, loaded.provider).catch(() => {});
throw fromBrowserError(e2);
}
await this.persist(id, loaded.provider).catch(() => {});
throw fromBrowserError(e);
}
if (opts.persist !== false) await this.persist(id, loaded.provider, opts.patch?.(out) ?? {});
this.maybeSyncCookies(row);
return out;
}
// ---------- BR-3 浏览器服务重启后自愈 ----------
/** 各浏览器服务最近一次读到的实例标识(节流缓存;只在进程内) */
private readonly instances = new Map<string, { at: number; id: string | null; pending?: Promise<string | null> }>();
/** 各浏览器服务已按哪个实例标识对过会话(同一标识不重复写库) */
private readonly applied = new Map<string, string>();
/** 进行中的重连:同一会话并发的操作共用一次 */
private readonly reconnecting = new Map<string, Promise<void>>();
/** 浏览器服务当前实例标识:INSTANCE_CHECK_MS 内复用上次的结果;取不到(旧镜像)为 null */
private async currentInstance(prow: BrowserProviderRow, force = false): Promise<string | null> {
const provider = this.providerFor(prow);
if (!provider.instanceId) return null;
const hit = this.instances.get(prow.id);
if (hit?.pending) return hit.pending;
if (hit && !force && Date.now() - hit.at < INSTANCE_CHECK_MS) return hit.id;
const pending = provider.instanceId().then(
(id) => {
this.instances.set(prow.id, { at: Date.now(), id });
return id;
},
(e) => {
// 连不上:沿用旧值,不当成重启(真重启时标签页 404/410 会被动发现)
this.instances.set(prow.id, { at: Date.now(), id: hit?.id ?? null });
throw e;
},
);
this.instances.set(prow.id, { at: hit?.at ?? 0, id: hit?.id ?? null, pending });
return pending;
}
/**
* 重启检测:浏览器服务实例标识变了 → 该服务下全部未关闭会话标 needsReconnect(下一次操作 / 原站视图触发重连)。
* 没记录实例标识的旧会话补记当前值。返回本次是否有会话被标记。
*/
async checkInstance(providerId: string, force = false): Promise<boolean> {
const prow = await this.ctx.db
.selectFrom("browser_provider")
.selectAll()
.where("id", "=", providerId)
.executeTakeFirst();
if (!prow?.enabled || prow.kind === "none") return false;
const id = await this.currentInstance(prow, force);
if (!id || (!force && this.applied.get(providerId) === id)) return false;
const db = this.ctx.db;
await db
.updateTable("browser_session")
.set({ recovery_state: sql`recovery_state || ${JSON.stringify({ instanceId: id })}::jsonb` })
.where("provider_id", "=", providerId)
.where("controller", "<>", "closed")
.where(sql<boolean>`recovery_state->>'instanceId' is null`)
.execute();
const r = await db
.updateTable("browser_session")
.set({
recovery_state: sql`recovery_state || ${JSON.stringify({ needsReconnect: true, reconnectTrigger: "provider_restarted" })}::jsonb`,
updated_at: this.now(),
})
.where("provider_id", "=", providerId)
.where("controller", "<>", "closed")
.where(sql<boolean>`recovery_state->>'instanceId' <> ${id}`)
.where(sql<boolean>`coalesce(recovery_state->>'needsReconnect', 'false') <> 'true'`)
.executeTakeFirst();
this.applied.set(providerId, id);
const n = Number(r.numUpdatedRows ?? 0);
if (n > 0)
this.ctx.logger.info(
{ component: "browser", providerId, sessions: n },
"浏览器服务重启过,未关闭的会话标为需要重新连接",
);
return n > 0;
}
private async markReconnect(id: string, trigger: BrowserReconnectState["trigger"]): Promise<void> {
await this.patchRecovery(await this.sessionRow(id), { needsReconnect: true, reconnectTrigger: trigger });
}
/**
* POST /browser/sessions/{id}/reconnect(幂等):先查实例标识;没标需要重连时按上游对账,当前标签页还活着就什么都不做;
* 否则重建。失败抛 reconnect_failed(503)。
*/
async reconnectIfNeeded(id: string): Promise<void> {
let row = await this.sessionRow(id);
if (row.controller === "closed") throw new BrowserServiceError("conflict", "会话已关闭");
if (await this.checkInstance(row.provider_id, true).catch(() => false)) row = await this.sessionRow(id);
const rs = obj(row.recovery_state) as RecoveryState;
if (!rs.needsReconnect) {
const { provider } = await this.loaded(row);
try {
await provider.reconcileTabs(id);
} catch (e) {
throw new BrowserServiceError(
"reconnect_failed",
`浏览器服务不可用:${e instanceof Error ? e.message : String(e)}`,
);
}
await this.persist(id, provider).catch(() => {});
if (provider.sessionInfo(id)?.currentTabId) return;
}
await this.reconnect(id, rs.needsReconnect ? (rs.reconnectTrigger ?? "provider_restarted") : "manual");
}
/** 重建会话的上游上下文(同一会话并发调用只做一次) */
reconnect(id: string, trigger: BrowserReconnectState["trigger"]): Promise<void> {
const running = this.reconnecting.get(id);
if (running) return running;
const p = this.doReconnect(id, trigger).finally(() => this.reconnecting.delete(id));
this.reconnecting.set(id, p);
return p;
}
/**
* 按会话的站点 / 账号:注入账号 Cookie(BR-2 同一套准备)→ 同一 userId 开新标签页 → 导航到会话记录的地址(没有或不在站点域名内
* 则站点首页)→ 旧标签页标 lost、换当前标签页。控制者与租约不变;人工接管中重连记 humanNotice(页面已重新加载,请核对)。
*/
private async doReconnect(id: string, trigger: BrowserReconnectState["trigger"]): Promise<void> {
let row = await this.sessionRow(id);
if (row.controller === "closed") throw new BrowserServiceError("conflict", "会话已关闭");
const loaded = await this.loaded(row);
row = loaded.row;
const provider = loaded.provider;
const prow = await this.providerRow(row.provider_id);
const site = await this.liveSite(row.site_id);
const budget = this.budgetFor(site, prow);
const at = this.now().toISOString();
const fail = async (reason: string) => {
const cur = await this.sessionRow(id);
await this.patchRecovery(cur, {
needsReconnect: true,
reconnectTrigger: trigger,
reconnect: { status: "failed", trigger, at, reason, humanNotice: false },
});
this.ctx.logger.warn({ component: "browser", sessionId: id, trigger, reason }, "浏览器会话重新连接失败");
return new BrowserServiceError("reconnect_failed", `浏览器已重启,重新连接失败:${reason}`);
};
if (!provider.reopenSession) throw await fail("浏览器服务不支持重新连接");
const url = restoreUrl(row.current_url, budget) ?? site.base_url;
const prep = await this.prepareCookies(site, row.account_id).catch(() => null);
let res: Awaited<ReturnType<NonNullable<BrowserProvider["reopenSession"]>>>;
try {
await this.rateLimit(site);
res = await provider.reopenSession(id, {
url,
budget,
...(prep?.cookies.length ? { cookies: prep.cookies } : {}),
});
} catch (e) {
throw await fail(e instanceof Error ? e.message : String(e));
}
const instanceId = await this.currentInstance(prow, true).catch(() => null);
const rs = obj((await this.sessionRow(id)).recovery_state) as RecoveryState;
await this.persist(id, provider, {
needsReconnect: false,
needsSnapshot: true,
instanceId: instanceId ?? rs.instanceId ?? null,
reconnect: { status: "ok", trigger, at, reason: null, humanNotice: row.controller === "human" },
});
this.coordSpaces.delete(id);
this.ctx.logger.info(
{
component: "browser",
sessionId: id,
trigger,
lostTabs: res.lost.length,
cookies: res.cookies?.status ?? "none",
cookieCount: res.cookies?.count ?? 0,
controller: row.controller,
},
"浏览器会话已重新连接",
);
}
/** 有活动时每 COOKIE_SYNC_INTERVAL_MS 读回一次登录态(后台,不拖慢动作;同一会话同时只跑一个) */
private readonly syncing = new Set<string>();
private maybeSyncCookies(row: BrowserSessionRow): void {
@@ -1557,6 +1799,14 @@ const REASON_TEXT: Record<string, string> = {
human_released_waiting: "人工交还时选择继续等待",
};
/** 重连回到的地址:会话记录的地址是 http(s)、在站点域名内、没有被脱敏过才用;否则 null(调用方退回站点首页) */
function restoreUrl(url: string | null, budget: NavigationBudget): string | null {
const u = url ? safeUrl(url) : null;
if (!u || !/^https?:$/.test(u.protocol) || !hostAllowed(u.hostname, budget.domainAllowlist)) return null;
if (url!.includes("[REDACTED")) return null;
return url;
}
function safeUrl(s: string): URL | null {
try {
return new URL(s);
+3 -5
View File
@@ -354,7 +354,7 @@ describe.skipIf(!hasPg)("E14 视觉自动化闭环", () => {
for (const l of ledger) expect((l.output as Any).modelImage).toBeNull();
}, 90_000);
it("多进程:api 重开被回收的标签页后 worker 换到新标签页;api 关闭会话后 worker 能重新打开", async () => {
it("多进程:api 自愈重开被回收的标签页后 worker 换到新标签页;api 关闭会话后 worker 能重新打开", async () => {
await open(ids.site, ids.provider, `${ORIGIN}/torrents.php`);
const worker = new BrowserService(e.booted.ctx, svc.opts);
const auto = { kind: "automation", holder: "agent:e14" } as const;
@@ -362,10 +362,8 @@ describe.skipIf(!hasPg)("E14 视觉自动化闭环", () => {
await snap(worker);
const old = (await svc.view(sid)).currentTabId!;
mock.dropTab(old);
await expect(snap(svc)).rejects.toThrow(/回收/);
await svc.run(sid, auto, (p, _ref, x) =>
p.openTab(sid, { url: `${ORIGIN}/torrents.php`, budget: x.budget, as: "automation" }),
);
// BR-3:api 发现标签页被回收 → 同一会话新开标签页回到原地址并重试
await expect(snap(svc)).resolves.toMatchObject({ url: expect.stringContaining("/torrents.php") });
expect((await svc.view(sid)).currentTabId).not.toBe(old);
await expect(snap(worker)).resolves.toMatchObject({ url: expect.stringContaining("/torrents.php") });
// api 关闭会话后,worker 能按站点重新打开(不被内存里的旧会话挡住)
+10
View File
@@ -128,6 +128,16 @@ export function useCloseBrowserSession(siteId: string) {
});
}
/** BR-3:浏览器服务重启后重新连接(幂等;上游上下文还在时服务端什么都不做) */
export function useBrowserReconnect(siteId: string) {
const refresh = useRefresh(siteId);
return useMutation({
mutationFn: (id: string) =>
apiData<BrowserSessionView>("/browser/sessions/{id}/reconnect", { method: "POST", pathParams: { id }, body: {} }),
onSuccess: (s) => refresh(s),
});
}
export function useBrowserControl(siteId: string) {
const refresh = useRefresh(siteId);
const acquire = useMutation({
@@ -10,6 +10,7 @@ import { Button } from "~/components/ui/button.tsx";
import { apiData, errorMessage } from "~/lib/api.ts";
import { qk } from "~/lib/query-keys.ts";
import { CONTROLLER_LABEL, credentialBadge, holderName, waitingText } from "./coords.ts";
import { ReconnectPanel, ReloadNotice, useAutoReconnect } from "./reconnect.tsx";
import { SiteOriginView } from "./site-origin-view.tsx";
// 原站视图桌面模式(docs/design/webtop-remote-browser.md §5.7):iframe 嵌浏览器桌面(Selkies),
@@ -99,6 +100,25 @@ function DesktopView({
const [reloadKey, setReloadKey] = useState(0);
const [loaded, setLoaded] = useState(false);
const reloadDesktop = () => {
setLoaded(false);
setReloadKey((k) => k + 1);
};
// BR-3:浏览器服务(同一容器里的 Selkies)重启过 → 自动重连会话一次,成功后重建桌面连接
const rc = useAutoReconnect(site.id, s, false, reloadDesktop);
// 别处(Agent 操作、截图)触发的重连成功后也重建桌面连接
const reconnectedAt = s?.reconnect?.status === "ok" ? s.reconnect.at : null;
const prevReconnect = useRef(reconnectedAt);
useEffect(() => {
if (prevReconnect.current !== reconnectedAt) {
prevReconnect.current = reconnectedAt;
if (reconnectedAt) {
setLoaded(false);
setReloadKey((k) => k + 1);
}
}
}, [reconnectedAt]);
// 控制权变化(接管 / 交还 / 过期)后重连,Selkies 按新口令认证;服务端也会主动断开旧连接
const role = status.role;
const prevRole = useRef(role);
@@ -117,11 +137,16 @@ function DesktopView({
const waiting = waitingText(s.waitingReason);
const cred = credentialBadge(s.credential);
const leaseHolder = s.lease ? holderName(s.lease.holder) : null;
const conn: { tone: StatusTone; text: string } = loaded
? mine
? { tone: "success", text: "已连接,可操作" }
: { tone: "success", text: "已连接,只读" }
: { tone: "neutral", text: "连接中" };
const conn: { tone: StatusTone; text: string } =
rc.state.phase === "reconnecting"
? { tone: "warning", text: "正在重新连接" }
: rc.state.phase === "failed"
? { tone: "destructive", text: "连接中断" }
: loaded
? mine
? { tone: "success", text: "已连接,可操作" }
: { tone: "success", text: "已连接,只读" }
: { tone: "neutral", text: "连接中" };
return (
<div className="space-y-3">
@@ -167,14 +192,7 @@ function DesktopView({
接管
</Button>
)}
<Button
variant="outline"
size="sm"
onClick={() => {
setLoaded(false);
setReloadKey((k) => k + 1);
}}
>
<Button variant="outline" size="sm" disabled={rc.busy} onClick={reloadDesktop}>
<RotateCw />
重新连接
</Button>
@@ -194,6 +212,8 @@ function DesktopView({
</div>
</div>
<ReloadNotice session={s} />
<p className="text-caption text-muted-foreground">
{mine
? "直接在画面里点击和输入。中文、日文用本机输入法输入;复制粘贴用 Ctrl+C / Ctrl+V。"
@@ -201,6 +221,11 @@ function DesktopView({
</p>
<div className="pg-slab relative min-w-0 overflow-hidden rounded-xl">
{rc.state.phase !== "idle" ? (
<div className="absolute inset-0 z-10 grid place-items-center bg-background/85">
<ReconnectPanel state={rc.state} onRetry={rc.retry} />
</div>
) : null}
<iframe
key={reloadKey}
title="浏览器桌面"
@@ -0,0 +1,118 @@
import type { BrowserSessionView } from "@umt/contracts";
import { LoaderCircle, RotateCw } from "lucide-react";
import { useCallback, useEffect, useRef, useState } from "react";
import { useBrowserReconnect } from "~/api/browser.ts";
import { StatusBadge } from "~/components/console/status-badge.tsx";
import { Button } from "~/components/ui/button.tsx";
import { ApiRequestError, errorMessage } from "~/lib/api.ts";
// BR-3 浏览器服务(camofox)重启后的自动重连:截图模式与桌面模式共用。
// 收到"标签页 / 会话不在了"(404)、"已被回收"(409)或自动重连失败(503),或会话详情标了 needsReconnect 时,
// 自动调用一次 POST /browser/sessions/{id}/reconnect;成功后继续轮询,失败显示原因与"重新连接"按钮,不再反复刷错误。
/** 这个错误是否说明浏览器服务重启过(上游标签页 / 会话不在了、自动重连失败) */
export function isBrowserGone(err: unknown): boolean {
if (!(err instanceof ApiRequestError)) return false;
if (err.code === "not_found" || err.code === "browser_reconnect_failed") return true;
return err.code === "conflict" && /回收/.test(err.message);
}
export type ReconnectPhase = { phase: "idle" } | { phase: "reconnecting" } | { phase: "failed"; reason: string };
/**
* gone = 当前视图收到了"浏览器不在了"类错误。每次出问题只自动重连一次(问题消失后复位);
* 失败后等人点"重新连接"(retry)。onReconnected:重连成功(桌面模式据此重建 Selkies 连接)。
*/
export function useAutoReconnect(
siteId: string,
session: BrowserSessionView | null,
gone: boolean,
onReconnected?: () => void,
) {
const mutation = useBrowserReconnect(siteId);
const [state, setState] = useState<ReconnectPhase>({ phase: "idle" });
const tried = useRef<string | null>(null);
const logged = useRef(false);
const done = useRef(onReconnected);
done.current = onReconnected;
const id = session && session.controller !== "closed" ? session.id : null;
const trigger = !!id && (gone || session?.needsReconnect === true);
const { mutate } = mutation;
const run = useCallback(() => {
if (!id) return;
setState({ phase: "reconnecting" });
mutate(id, {
onSuccess: () => {
setState({ phase: "idle" });
done.current?.();
},
onError: (e) => setState({ phase: "failed", reason: errorMessage(e, "浏览器服务不可用") }),
});
}, [id, mutate]);
useEffect(() => {
if (!trigger) {
// 问题消失(截图恢复 / 会话不再需要重连):下次再出问题时还能自动重连一次
if (tried.current === id) tried.current = null;
return;
}
if (tried.current === id) return;
tried.current = id;
if (!logged.current) {
logged.current = true;
console.debug("[browser] 浏览器服务可能重启过,自动重新连接一次", id);
}
run();
}, [trigger, id, run]);
return { state, retry: run, busy: state.phase === "reconnecting" };
}
/** 画面区域:重连中 / 重连失败(原因 + 重新连接按钮) */
export function ReconnectPanel({
state,
error,
onRetry,
}: {
state: ReconnectPhase;
/** idle 但仍拿到"浏览器不在了"类错误时显示它的消息 */
error?: unknown;
onRetry: () => void;
}) {
if (state.phase === "reconnecting")
return (
<div role="status" className="flex flex-col items-center justify-center gap-3 px-6 py-12 text-center">
<LoaderCircle className="h-5 w-5 animate-spin text-muted-foreground" />
<p className="text-lead font-medium text-foreground">浏览器已重启,正在重新连接…</p>
<p className="text-caption text-muted-foreground">重新注入账号登录态,并回到原来的页面。</p>
</div>
);
const reason = state.phase === "failed" ? state.reason : errorMessage(error, "浏览器服务不可用");
return (
<div role="alert" className="flex flex-col items-center justify-center gap-3 px-6 py-12 text-center">
<p className="text-lead font-medium text-foreground">浏览器连接已断开</p>
<p className="max-w-md text-caption break-words text-muted-foreground">{reason}</p>
<Button variant="outline" size="sm" onClick={onRetry}>
<RotateCw />
重新连接
</Button>
</div>
);
}
/** 人工接管中浏览器重连过:页面已重新加载(表单输入、滚动位置都没了),提示核对;点"知道了"收起 */
export function ReloadNotice({ session }: { session: BrowserSessionView }) {
const r = session.reconnect;
const at = r?.status === "ok" && r.humanNotice && session.controller === "human" ? r.at : null;
const [dismissed, setDismissed] = useState<string | null>(null);
if (!at || dismissed === at) return null;
return (
<div className="flex flex-wrap items-center gap-2">
<StatusBadge tone="warning">浏览器重启过,页面已重新加载,请核对</StatusBadge>
<Button variant="ghost" size="xs" onClick={() => setDismissed(at)}>
知道了
</Button>
</div>
);
}
@@ -35,6 +35,7 @@ import { Button } from "~/components/ui/button.tsx";
import { Input } from "~/components/ui/input.tsx";
import { errorMessage } from "~/lib/api.ts";
import { CONTROLLER_LABEL, credentialBadge, holderName, toViewport, waitingText } from "./coords.ts";
import { isBrowserGone, ReconnectPanel, ReloadNotice, useAutoReconnect } from "./reconnect.tsx";
// 原站视图(REQ §4.7.1、§4.7.3):同一个 camofox 会话的定时截图 + 人工输入映射。
// 人工接管前服务端先停掉该会话的自动化输入;交还后 Agent 必须重新读取页面。
@@ -51,7 +52,13 @@ export function SiteOriginView({ site }: { site: SiteListItem }) {
const session = useSiteBrowserSession(site.id);
const s = session.data ?? null;
const [paused, setPaused] = useState(false);
const shot = useBrowserScreenshot(s && s.controller !== "closed" ? s.id : null, { paused });
const [holdShots, setHoldShots] = useState(false);
const shot = useBrowserScreenshot(s && s.controller !== "closed" ? s.id : null, { paused: paused || holdShots });
// BR-3:浏览器服务重启 → 自动重连一次;重连中 / 失败时停掉截图轮询,不反复刷错误
const gone = shot.isError && isBrowserGone(shot.error);
const rc = useAutoReconnect(site.id, s, gone);
const reconnectView = rc.state.phase !== "idle" || gone;
useEffect(() => setHoldShots(rc.state.phase !== "idle"), [rc.state.phase]);
const open = useOpenBrowserSession(site.id);
const close = useCloseBrowserSession(site.id);
const control = useBrowserControl(site.id);
@@ -108,11 +115,13 @@ export function SiteOriginView({ site }: { site: SiteListItem }) {
const unverified = !unsupported && (!caps?.takeover.verified || !caps?.screenshot.verified);
const conn: { tone: StatusTone; text: string } = paused
? { tone: "neutral", text: "已暂停刷新" }
: shot.isError
? { tone: "destructive", text: "连接中断" }
: shot.data
? { tone: "success", text: "已连接" }
: { tone: "neutral", text: "连接中" };
: rc.state.phase === "reconnecting"
? { tone: "warning", text: "正在重新连接" }
: shot.isError
? { tone: "destructive", text: "连接中断" }
: shot.data
? { tone: "success", text: "已连接" }
: { tone: "neutral", text: "连接中" };
const waiting = waitingText(s.waitingReason);
const cred = credentialBadge(s.credential);
const leaseHolder = s.lease ? holderName(s.lease.holder) : null;
@@ -142,6 +151,7 @@ export function SiteOriginView({ site }: { site: SiteListItem }) {
return (
<div className="space-y-3">
<p className="text-caption text-muted-foreground">验证码等需本人操作在此完成。</p>
<ReloadNotice session={s} />
<div className="flex flex-wrap items-center gap-2">
<StatusBadge tone={conn.tone}>{conn.text}</StatusBadge>
@@ -244,7 +254,9 @@ export function SiteOriginView({ site }: { site: SiteListItem }) {
</form>
<div className="pg-slab relative min-w-0 overflow-hidden rounded-xl">
{shot.data ? (
{reconnectView ? (
<ReconnectPanel state={rc.state} error={shot.error} onRetry={rc.retry} />
) : shot.data ? (
// biome-ignore lint/a11y/useKeyWithClickEvents: 画面点击映射到浏览器;键盘输入走下方输入栏
<img
src={`data:${shot.data.mimeType};base64,${shot.data.dataBase64}`}
@@ -258,7 +270,7 @@ export function SiteOriginView({ site }: { site: SiteListItem }) {
) : (
<LoadingState rows={6} className="p-4" />
)}
{shot.data && !human ? (
{shot.data && !human && !reconnectView ? (
<p className="absolute right-2 bottom-2 rounded-md bg-background/85 px-2 py-1 text-meta text-muted-foreground">
接管后可点击和输入
</p>
+139
View File
@@ -280,6 +280,145 @@ async function main() {
results.push({ path: label, ms: Date.now() - t0, problems: [...new Set(problems)] });
}
// ── 交互:原站视图在浏览器服务重启后自动重连(BR-3;浏览器 API 全部 mock,不连真实 camofox)──
// 截图先 404(标签页不在了)→ 显示"正在重新连接" → 自动调一次 POST .../reconnect → 截图恢复
{
const label = "原站视图 → 浏览器重启后自动重连(mock)";
const sitesRes = await page.request.get(`${BASE}/api/v1/sites?limit=50`);
const site = sitesRes.ok()
? ((await sitesRes.json().catch(() => null))?.data?.items ?? []).find((x) => x.enabled)
: null;
if (!site) skipped.push(`${label}(没有启用的站点)`);
else {
problems = [];
const t0 = Date.now();
const sid = "00000000-0000-4000-8000-00000000b3b3";
const at = new Date().toISOString();
const session = {
id: sid,
providerId: "00000000-0000-4000-8000-00000000b3b4",
siteId: site.id,
accountId: null,
profileKey: `umt-site-${site.id}-acct-anon`,
controller: "automation",
currentUrl: "https://example.invalid/torrents.php",
lastActivityAt: at,
upstreamRef: {},
recoveryState: {},
closedAt: null,
closeReason: null,
createdAt: at,
updatedAt: at,
tabs: [],
currentTabId: "tab-mock",
lease: null,
needsSnapshot: false,
waitingReason: null,
credential: null,
needsReconnect: false,
reconnect: null,
};
const png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==";
const ok = (data) => ({ status: 200, contentType: "application/json", body: JSON.stringify({ ok: true, data }) });
let reconnects = 0;
let shots = 0;
await page.route("**/api/v1/browser/**", async (route) => {
const u = new URL(route.request().url());
const p = u.pathname.replace(/^\/api\/v1/, "");
if (p === "/browser/providers")
return route.fulfill(
ok({
items: [
{
id: session.providerId,
name: "camofox-mock",
kind: "camofox",
baseUrl: "http://camofox.invalid:9377",
secretRef: null,
managedBy: null,
hasAccessKey: true,
enabled: true,
capabilities: null,
verifiedAt: null,
healthStatus: "healthy",
lastHealthAt: at,
health: null,
},
],
}),
);
if (p === "/browser/sessions" && route.request().method() === "GET")
return route.fulfill(ok({ items: [session] }));
if (p === `/browser/desktop/${sid}/_status`)
return route.fulfill(
ok({
available: false,
reason: "冒烟 mock",
role: "viewer",
leaseExpiresAt: null,
path: `/api/v1/browser/desktop/${sid}/`,
requiresSecureContext: true,
}),
);
if (p === `/browser/sessions/${sid}/screenshot`) {
shots++;
if (reconnects === 0)
return route.fulfill({
status: 404,
contentType: "application/json",
body: JSON.stringify({ ok: false, error: { code: "not_found", message: "标签页不存在或已关闭" } }),
});
return route.fulfill(
ok({
mimeType: "image/png",
dataBase64: png,
width: 1,
height: 1,
url: session.currentUrl,
modelSafe: true,
capturedAt: new Date().toISOString(),
controller: "automation",
}),
);
}
if (p === `/browser/sessions/${sid}/reconnect`) {
reconnects++;
await new Promise((r) => setTimeout(r, 800));
return route.fulfill(
ok({ ...session, reconnect: { status: "ok", trigger: "tab_gone", at, reason: null, humanNotice: false } }),
);
}
return route.fulfill({
status: 404,
contentType: "application/json",
body: JSON.stringify({ ok: false, error: { code: "not_found", message: `冒烟 mock 没有 ${p}` } }),
});
});
try {
await page.goto("about:blank");
await page.goto(`${BASE}/console/sites/${encodeURIComponent(site.id)}?tab=origin`, {
waitUntil: "domcontentloaded",
timeout: 30_000,
});
await page.getByText("浏览器已重启,正在重新连接…").waitFor({ timeout: 15_000 });
await page.locator('img[alt="站点页面画面"]').waitFor({ timeout: 15_000 });
await page.waitForTimeout(2_000);
if (reconnects !== 1) problems.push(`自动重连应只调用 1 次,实际 ${reconnects} 次`);
if (shots < 2) problems.push("重连后没有继续轮询截图");
if (await page.getByText("读取画面失败").count()) problems.push("仍显示读取画面失败");
} catch (err) {
problems.push(`交互失败:${String(err?.message ?? err).split("\n")[0]}`);
}
if (SHOTS) await page.screenshot({ path: join(OUT, "site_origin_reconnect.png") }).catch(() => {});
await page.unroute("**/api/v1/browser/**");
// mock 的首个截图 404 是预期内的
const kept = problems.filter(
(x) => !(/HTTP 404/.test(x) && x.includes("/screenshot")) && !/status of 404/.test(x),
);
results.push({ path: label, ms: Date.now() - t0, problems: [...new Set(kept)] });
}
}
// ── 旧链接重定向 ──
for (const r of REDIRECT_ROUTES) {
let from = r.from;