fix(system): retired heartbeat rows no longer alarm on status page

- GET /system/health splits processes into current instances (hosts with a beat in the last 10 min, newest per host+kind, not stopped) and retiredProcesses[]; 45s stale and level only consider current ones - heartbeat writer marks stopped_at on graceful exit (migration 0043), upsert clears it; pure api role now stops its writer on SIGTERM - scheduled job system.heartbeat_cleanup (maintain, 1h) deletes rows without a beat for 24h and samples older than 7 days - status page: per-role alarms ("io worker 45 秒没有心跳"), collapsible retired instances; glossary/runbook updated - bump all members to 0.2.2, CHANGELOG [0.2.2] Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-09-28 23:31:21 +08:00
1 parent fa261eb6cd
commit b4c9b1ead2
@@ -22,6 +22,16 @@
 - 没有调查整个 qB 实例(含未纳管任务)的能力。
 - AT18(恶意标题)、AT29(部分入库不清理)、AT42(未纳管占用)、AT73(多下载器同 hash)与流程版本升级没有测试。

+## [0.2.2] - 2026-09-28
+
+### 修复
+
+- 编译产物启动即 `ERR_MODULE_NOT_FOUND`:web 的 `shadcn` 开发依赖带进第二份 `@modelcontextprotocol/[email protected]`(zod@3),`deno compile --exclude-unused-npm` 嵌入的目录与裁剪后快照解析的路径不一致。去掉 `shadcn` 依赖,只把它的 `tailwind.css` 拷成 `apps/web/app/shadcn-tailwind.css`,MCP SDK 只剩一份。
+- 编译产物自检:`scripts/check-compiled-binary.sh` 用空环境跑 `--role preflight`,必须加载完整模块图并在配置校验处以退出码 2 结束;已接入 `deno task ci`(`check:binary`)与 Dockerfile 构建阶段。
+- 运行状态页把容器重建前留下的旧心跳行当成"N 个进程超过 45 秒没有心跳"报警。`GET /system/health` 现在把进程分成当前实例(最近 10 分钟有心跳的 host 上、同 host 同 kind 最新启动且未退出的进程)与已下线实例,45 秒 stale 判定与告警只看当前实例;已下线实例放在新字段 `retiredProcesses[]`,不计入 `level`,页面折叠显示。横幅按角色报警,如「io worker 45 秒没有心跳」。
+- 进程正常退出(SIGTERM / SIGINT)时把自己的心跳行标 `stopped_at`(迁移 0043),退出后立即不再当活进程;重启后 upsert 清回 null。纯 api 进程退出时也会停心跳写入器。
+- 心跳清理改为定时任务 `system.heartbeat_cleanup`(维护组,1 小时):删 24 小时没有心跳的进程行和 7 天前的采样;写入器不再自己裁剪。
+
 ## [0.2.0] - 2026-09-28

 Agent 上游全覆盖与订阅登录、控制台信息架构重排、定时任务看板、全站文案第二轮审校;W9 与前端波补上站点中心、订阅重做、统一规则、命名模板、全局搜索、文件管理 v2 与统一任务页。基线见 `docs/plan/08-v0.2-closeout.md`(W8-A 与"W10 集成记录")、`docs/plan/09-visual-qa.md`、`docs/design/{console-ia,site-center,subscriptions,naming-template,file-manager-v2,pi-providers,agent-ui-reference}.md`、`docs/runbook/{schedules,agent,sites,rules}.md`。
@@ -1,6 +1,6 @@
 {
  "name": "@umt/server",
-  "version": "0.2.0",
+  "version": "0.2.2",
  "exports": { ".": "./src/main.ts" },
  "tasks": {
    "dev": "deno run -A --watch --env-file=../../.env src/main.ts",
@@ -1,6 +1,9 @@
 import { join, resolve } from "node:path";
 import { z } from "zod";

+/** compose 里 `${X:-}` 展开成空串:按未设置处理 */
+const emptyToUndef = (v: unknown) => (v === "" ? undefined : v);
+
 const LogLevel = z.enum(["fatal", "error", "warn", "info", "debug", "trace", "silent"]);

 export const ConfigSchema = z.object({
@@ -66,6 +69,22 @@ export const ConfigSchema = z.object({
  ORGANIZE_ON_EXISTING: z.enum(["conflict", "skip", "replace_with_verify"]).default("conflict"),
  /** 随行字幕中此语言加 .default 标记(对照 MP DEFAULT_SUB);不设 = 不加 */
  ORGANIZE_DEFAULT_SUBTITLE_LANG: z.enum(["zh-cn", "zh-tw", "en", "ja", "ko"]).optional(),
+  // ── 浏览器服务(camofox-browser,docs/runbook/browser.md)──
+  /**
+   * 部署自带的浏览器服务地址(compose 缺省 http://camofox:9377;接自有实例时改成它的地址)。
+   * 设置后启动时登记/更新名为 camofox-env 的浏览器服务;不设则只用设置页登记的。
+   */
+  BROWSER_PROVIDER_URL: z.preprocess(emptyToUndef, z.string().url().optional()),
+  /** 该服务的访问凭据(camofox CAMOFOX_ACCESS_KEY);只在进程内使用,不入库 */
+  BROWSER_ACCESS_KEY: z.preprocess(emptyToUndef, z.string().min(1).optional()),
+  /** false = 停用 camofox-env(容器可照常运行,界面标"浏览器不可用") */
+  BROWSER_ENABLED: z.preprocess(
+    emptyToUndef,
+    z
+      .enum(["true", "false", "1", "0"])
+      .default("true")
+      .transform((v) => v === "true" || v === "1"),
+  ),
 });
 export type Config = z.infer<typeof ConfigSchema>;

@@ -237,6 +237,11 @@ if (role === "api" || role === "all") {
    ac.abort();
    await server.finished;
  });
+  if (role === "api") {
+    // 纯 api 进程不走 worker 的模块 stop:这里停 system 模块的心跳写入器,把本进程的心跳行标 stopped_at
+    const sys = SERVER_MODULES.find((m) => m.name === "system");
+    if (sys?.stop) stops.push(async () => await sys.stop?.());
+  }
 }
 if (role === "worker" || role === "worker:agent" || role === "all") {
  const worker = await startWorker({
@@ -85,6 +85,13 @@ export function registerBrowserRoutes(app: ModuleApp, svc: () => BrowserService)
  const B = Api.browser;

  route(app, B.providers, async ({ query }) => ({ items: await svc().listProviders(query.probe === "true") }));
+  route(app, B.createProvider, async ({ body }) => svc().createProvider(body));
+  route(app, B.updateProvider, async ({ params, body }) => svc().updateProvider(params.id, body));
+  route(app, B.probeProvider, async ({ params }) => {
+    const [v] = await svc().listProviders(true, params.id);
+    if (!v) throw new BrowserServiceError("not_found", "浏览器服务不存在");
+    return v;
+  });

  route(app, B.sessions, async ({ query }) => ({
    items: await svc().list({
@@ -114,8 +114,13 @@ export interface RecoveryState {
  downloads?: { sha256: string; artifactId: string; sourceReleaseId: string | null; planId: string | null }[];
 }

+/** 环境变量(BROWSER_PROVIDER_URL)登记的浏览器服务 */
+export const ENV_PROVIDER_NAME = "camofox-env";
+const ENV_MANAGED = "env";
+
 export class BrowserService {
  private readonly providers = new Map<string, { sig: string; provider: BrowserProvider }>();
+  private envSync: Promise<void> | undefined;
  readonly now: () => Date;

  constructor(
@@ -163,7 +168,11 @@ export class BrowserService {
    const { db, secrets } = this.ctx;
    return {
      resolve: async (req) => {
-        if (req.kind === "provider_access_key") return row.secret_ref ? secrets.resolve(row.secret_ref) : null;
+        if (req.kind === "provider_access_key") {
+          if (row.secret_ref) return secrets.resolve(row.secret_ref);
+          // 环境变量登记的服务:凭据只在进程内(BROWSER_ACCESS_KEY),不入库
+          return obj(row.config).managedBy === ENV_MANAGED ? (this.ctx.config?.BROWSER_ACCESS_KEY ?? null) : null;
+        }
        let q = db
          .selectFrom("site_account")
          .selectAll()
@@ -210,7 +219,126 @@ export class BrowserService {
    return provider;
  }

+  /**
+   * BROWSER_PROVIDER_URL / BROWSER_ACCESS_KEY / BROWSER_ENABLED → 名为 camofox-env 的浏览器服务(每个进程同步一次)。
+   * 地址与启用由环境变量决定;config 里的其它键(verified、超时等)保留。没设地址时已有的 camofox-env 置为停用。
+   */
+  syncEnvProvider(): Promise<void> {
+    this.envSync ??= this.doSyncEnvProvider().catch((e) => {
+      this.envSync = undefined;
+      this.ctx.logger.warn({ component: "browser", err: String(e) }, "登记环境变量浏览器服务失败");
+    });
+    return this.envSync;
+  }
+
+  private async doSyncEnvProvider(): Promise<void> {
+    const c = this.ctx.config;
+    const url = c?.BROWSER_PROVIDER_URL;
+    const enabled = !!url && c.BROWSER_ENABLED !== false;
+    const db = this.ctx.db;
+    const row = await db
+      .selectFrom("browser_provider")
+      .selectAll()
+      .where("name", "=", ENV_PROVIDER_NAME)
+      .executeTakeFirst();
+    if (!url) {
+      if (row?.enabled)
+        await db.updateTable("browser_provider").set({ enabled: false }).where("id", "=", row.id).execute();
+      return;
+    }
+    const config = JSON.stringify({ ...obj(row?.config), managedBy: ENV_MANAGED });
+    if (row)
+      await db
+        .updateTable("browser_provider")
+        .set({ base_url: url, enabled, kind: "camofox", secret_ref: null, config })
+        .where("id", "=", row.id)
+        .execute();
+    else
+      await db
+        .insertInto("browser_provider")
+        .values({ name: ENV_PROVIDER_NAME, kind: "camofox", base_url: url, enabled, config })
+        .onConflict((oc) => oc.column("name").doNothing())
+        .execute();
+  }
+
+  /** 设置页登记浏览器服务:访问凭据加密存 secret;名称 camofox-env 保留给环境变量 */
+  async createProvider(body: {
+    name: string;
+    baseUrl: string;
+    accessKey?: string | undefined;
+    enabled?: boolean | undefined;
+  }): Promise<BrowserProviderView> {
+    if (body.name === ENV_PROVIDER_NAME)
+      throw new BrowserServiceError("bad_request", `名称 ${ENV_PROVIDER_NAME} 保留给环境变量`);
+    const exists = await this.ctx.db
+      .selectFrom("browser_provider")
+      .select("id")
+      .where("name", "=", body.name)
+      .executeTakeFirst();
+    if (exists) throw new BrowserServiceError("conflict", `浏览器服务「${body.name}」已存在`);
+    const secretRef = body.accessKey
+      ? await this.ctx.secrets.create({ purpose: "other", plaintext: body.accessKey, label: `browser:${body.name}` })
+      : null;
+    const row = await this.ctx.db
+      .insertInto("browser_provider")
+      .values({
+        name: body.name,
+        kind: "camofox",
+        base_url: body.baseUrl,
+        secret_ref: secretRef,
+        enabled: body.enabled ?? true,
+      })
+      .returning("id")
+      .executeTakeFirstOrThrow();
+    return this.providerView(row.id);
+  }
+
+  /** 改地址 / 凭据 / 启用;accessKey = null 清除凭据。camofox-env 的地址与凭据归环境变量管,只能改启用 */
+  async updateProvider(
+    id: string,
+    body: { baseUrl?: string | undefined; accessKey?: string | null | undefined; enabled?: boolean | undefined },
+  ): Promise<BrowserProviderView> {
+    const row = await this.ctx.db.selectFrom("browser_provider").selectAll().where("id", "=", id).executeTakeFirst();
+    if (!row) throw new BrowserServiceError("not_found", "浏览器服务不存在");
+    if (obj(row.config).managedBy === ENV_MANAGED && (body.baseUrl !== undefined || body.accessKey !== undefined))
+      throw new BrowserServiceError(
+        "conflict",
+        "该浏览器服务由环境变量 BROWSER_PROVIDER_URL / BROWSER_ACCESS_KEY 管理，改部署配置后重启",
+      );
+    let secretRef = row.secret_ref;
+    if (body.accessKey === null) secretRef = null;
+    else if (body.accessKey) {
+      if (secretRef) await this.ctx.secrets.replace(secretRef, body.accessKey);
+      else
+        secretRef = await this.ctx.secrets.create({
+          purpose: "other",
+          plaintext: body.accessKey,
+          label: `browser:${row.name}`,
+        });
+    }
+    await this.ctx.db
+      .updateTable("browser_provider")
+      .set({
+        ...(body.baseUrl !== undefined ? { base_url: body.baseUrl } : {}),
+        ...(body.enabled !== undefined ? { enabled: body.enabled } : {}),
+        secret_ref: secretRef,
+      })
+      .where("id", "=", id)
+      .execute();
+    if (row.secret_ref && secretRef === null) await this.ctx.secrets.remove(row.secret_ref).catch(() => {});
+    // 凭据换了但 secret_ref 不变:签名不变,显式丢弃缓存的适配器
+    this.providers.delete(id);
+    return this.providerView(id);
+  }
+
+  private async providerView(id: string): Promise<BrowserProviderView> {
+    const v = (await this.listProviders(false, id))[0];
+    if (!v) throw new BrowserServiceError("not_found", "浏览器服务不存在");
+    return v;
+  }
+
  async providerRow(id?: string): Promise<BrowserProviderRow> {
+    await this.syncEnvProvider();
    let q = this.ctx.db
      .selectFrom("browser_provider")
      .selectAll()
@@ -223,12 +351,14 @@ export class BrowserService {
    return row;
  }

-  async listProviders(probe: boolean): Promise<BrowserProviderView[]> {
-    const rows = await this.ctx.db.selectFrom("browser_provider").selectAll().orderBy("created_at").execute();
+  async listProviders(probe: boolean, onlyId?: string): Promise<BrowserProviderView[]> {
+    await this.syncEnvProvider();
+    let rows = await this.ctx.db.selectFrom("browser_provider").selectAll().orderBy("created_at").execute();
+    if (onlyId) rows = rows.filter((r) => r.id === onlyId);
    const out: BrowserProviderView[] = [];
    for (let row of rows) {
      let health: BrowserHealth | null = null;
-      if (probe && row.enabled && row.kind !== "none") {
+      if (probe && (row.enabled || onlyId) && row.kind !== "none") {
        const p = this.providerFor(row);
        health = await p.health();
        let caps: BrowserCapabilities | null = null;
@@ -255,6 +385,9 @@ export class BrowserService {
        kind: row.kind as "camofox" | "none",
        baseUrl: row.base_url,
        secretRef: row.secret_ref,
+        managedBy: obj(row.config).managedBy === ENV_MANAGED ? "env" : null,
+        hasAccessKey:
+          !!row.secret_ref || (obj(row.config).managedBy === ENV_MANAGED && !!this.ctx.config?.BROWSER_ACCESS_KEY),
        enabled: row.enabled,
        capabilities: Object.keys(caps).length ? (caps as unknown as BrowserCapabilities) : null,
        verifiedAt: iso(row.verified_at),
@@ -8,8 +8,8 @@ import type {
  SystemMount,
  SystemProcess,
 } from "@umt/contracts";
-import { type DB, type Kysely, sql } from "@umt/db";
-import { HEARTBEAT_STALE_MS } from "./heartbeat.ts";
+import { type DB, type Kysely, type Selectable, sql } from "@umt/db";
+import { CURRENT_HOST_WINDOW_MS, HEARTBEAT_ROW_RETENTION_MS, HEARTBEAT_STALE_MS } from "./heartbeat.ts";

 // GET /system/health 各段的取数(W9-I)。每段独立 try/catch:一段失败只把该段标 unknown,不让整页 500。
 // 慢 IO(statfs / 哨兵检查)都带超时:挂载卡死时只标 timeout,不拖住请求。
@@ -53,15 +53,9 @@ const num = (v: unknown) => (v == null ? null : Number(v));

 // ── 进程 ──

-export async function loadProcesses(db: Kysely<DB>, now: Date): Promise<SystemProcess[]> {
-  const rows = await db
-    .selectFrom("process_heartbeat")
-    .selectAll()
-    .where("last_beat_at", ">", new Date(now.getTime() - 24 * 3600_000))
-    .orderBy("process_kind")
-    .orderBy("instance_id")
-    .execute();
-  return rows.map((r) => {
+type HeartbeatRow = Selectable<DB["process_heartbeat"]>;
+
+function toProcess(r: HeartbeatRow, now: Date): SystemProcess {
  const beat = toDate(r.last_beat_at);
  const age = Math.max(0, now.getTime() - beat.getTime());
  const extra = (r.extra ?? {}) as Record<string, unknown>;
@@ -77,6 +71,7 @@ export async function loadProcesses(db: Kysely<DB>, now: Date): Promise<SystemPr
    lastBeatAt: beat.toISOString(),
    beatAgeSec: Math.round(age / 100) / 10,
    stale: age > HEARTBEAT_STALE_MS,
+    stoppedAt: r.stopped_at ? toDate(r.stopped_at).toISOString() : null,
    eventLoopLagMs: r.event_loop_lag_ms,
    eventLoopLagMaxMs: r.event_loop_lag_max_ms,
    concurrency: {
@@ -87,10 +82,55 @@ export async function loadProcesses(db: Kysely<DB>, now: Date): Promise<SystemPr
    sseConnections: typeof sse === "number" ? sse : null,
    extra,
  };
-  });
 }

-/** 进程段的档位:有 stale 或延迟 ≥ 5s 为 critical;延迟 ≥ 1s 为 warn;没有任何 worker 心跳为 warn */
+/**
+ * 把心跳行分成当前实例与已下线实例:
+ * - 当前 host = 最近 10 分钟内有心跳且没标 stopped_at 的行所在 host(容器重建后 host 会变);
+ * - 当前实例 = 当前 host 上、没标 stopped_at、且是同 host 同 kind 里最新启动的那一行;
+ * - 其余(已正常退出、host 已不在、被同 host 新进程替换)都算已下线实例,不参与 stale 判定与告警。
+ */
+export function splitProcesses(
+  ps: SystemProcess[],
+  now: Date,
+): { processes: SystemProcess[]; retiredProcesses: SystemProcess[] } {
+  const alive = ps.filter((p) => !p.stoppedAt);
+  const hosts = new Set(
+    alive.filter((p) => now.getTime() - Date.parse(p.lastBeatAt) <= CURRENT_HOST_WINDOW_MS).map((p) => p.host),
+  );
+  const newest = new Map<string, SystemProcess>();
+  for (const p of alive) {
+    if (!hosts.has(p.host)) continue;
+    const k = `${p.host}\u0000${p.processKind}`;
+    const cur = newest.get(k);
+    if (!cur || Date.parse(p.startedAt) > Date.parse(cur.startedAt)) newest.set(k, p);
+  }
+  const current = new Set([...newest.values()].map((p) => p.instanceId));
+  const processes = ps.filter((p) => current.has(p.instanceId));
+  const retiredProcesses = ps
+    .filter((p) => !current.has(p.instanceId))
+    .sort((a, b) => Date.parse(b.lastBeatAt) - Date.parse(a.lastBeatAt));
+  return { processes, retiredProcesses };
+}
+
+export async function loadProcesses(
+  db: Kysely<DB>,
+  now: Date,
+): Promise<{ processes: SystemProcess[]; retiredProcesses: SystemProcess[] }> {
+  const rows = await db
+    .selectFrom("process_heartbeat")
+    .selectAll()
+    .where("last_beat_at", ">", new Date(now.getTime() - HEARTBEAT_ROW_RETENTION_MS))
+    .orderBy("process_kind")
+    .orderBy("instance_id")
+    .execute();
+  return splitProcesses(
+    rows.map((r) => toProcess(r, now)),
+    now,
+  );
+}
+
+/** 进程段的档位(只看当前实例):有 stale 或延迟 ≥ 1s 为 warn;延迟 ≥ 5s 为 critical;没有任何 worker 心跳为 warn */
 export function processLevel(ps: SystemProcess[]): HealthLevel {
  const live = ps.filter((p) => !p.stale);
  const levels: HealthLevel[] = [];
@@ -344,8 +384,11 @@ export async function loadSchedules(db: Kysely<DB>, now: Date): Promise<SystemHe

 export async function buildHealth(d: HealthDeps): Promise<SystemHealth> {
  const now = d.now?.() ?? new Date();
-  const [processes, db, paths, engine, schedules] = await Promise.all([
-    loadProcesses(d.db, now).catch(() => [] as SystemProcess[]),
+  const [{ processes, retiredProcesses }, db, paths, engine, schedules] = await Promise.all([
+    loadProcesses(d.db, now).catch(() => ({
+      processes: [] as SystemProcess[],
+      retiredProcesses: [] as SystemProcess[],
+    })),
    loadDb(d.db),
    loadDisksAndMounts(d).catch(() => ({ disks: [] as SystemDisk[], mounts: [] as SystemMount[] })),
    loadEngine(d.db, d.engineSchema, now).catch((): SystemHealth["engine"] => ({
@@ -388,6 +431,7 @@ export async function buildHealth(d: HealthDeps): Promise<SystemHealth> {
      uptimeSec: Math.max(0, Math.round((now.getTime() - d.self.startedAt.getTime()) / 1000)),
    },
    processes,
+    retiredProcesses,
    sse: { connections: sse },
    db,
    disks: paths.disks,
@@ -6,12 +6,17 @@ import serverDeno from "../../../deno.json" with { type: "json" };

 // 进程心跳写入(W9-I):每个常驻进程每 15s upsert process_heartbeat 一行 + 追加一条采样。
 // 写失败只记 debug(库暂时不可达不影响业务;读取方按 last_beat_at 判 stale)。
+// 正常退出(stop)时把本行标 stopped_at,读取方立即把它归入已下线实例;裁剪由定时任务 system.heartbeat_cleanup 负责。

 export const HEARTBEAT_WRITE_MS = 15_000;
 /** 超过这个时长没有心跳即 stale(3 个写周期) */
 export const HEARTBEAT_STALE_MS = 45_000;
 /** 采样保留天数 */
 export const SAMPLE_RETENTION_DAYS = 7;
+/** 进程行保留:超过这个时长没心跳的行删除 */
+export const HEARTBEAT_ROW_RETENTION_MS = 24 * 3600_000;
+/** 当前实例判定窗口:最近这段时间内有心跳的 host 视为当前在跑的容器 */
+export const CURRENT_HOST_WINDOW_MS = 10 * 60_000;

 export const SERVER_VERSION: string = serverDeno.version;

@@ -109,7 +114,7 @@ export async function writeHeartbeat(
      host = excluded.host, version = excluded.version, started_at = excluded.started_at,
      last_beat_at = excluded.last_beat_at, event_loop_lag_ms = excluded.event_loop_lag_ms,
      event_loop_lag_max_ms = excluded.event_loop_lag_max_ms, concurrency = excluded.concurrency,
-      running_runs = excluded.running_runs, extra = excluded.extra`.execute(db);
+      running_runs = excluded.running_runs, extra = excluded.extra, stopped_at = null`.execute(db);
  await sql`
    insert into process_heartbeat_sample (instance_id, process_kind, at, event_loop_lag_ms, event_loop_lag_max_ms,
      running_runs, sse_connections)
@@ -117,10 +122,22 @@ export async function writeHeartbeat(
      ${s.sseConnections ?? null})`.execute(db);
 }

-/** 裁剪过期采样与 7 天没心跳的进程行 */
-export async function pruneHeartbeats(db: Kysely<DB>, days = SAMPLE_RETENTION_DAYS): Promise<void> {
-  await sql`delete from process_heartbeat_sample where at < now() - make_interval(days => ${days})`.execute(db);
-  await sql`delete from process_heartbeat where last_beat_at < now() - make_interval(days => ${days})`.execute(db);
+/** 正常退出:把本实例的行标 stopped_at(之后读取方不再当活进程) */
+export async function markHeartbeatStopped(db: Kysely<DB>, instanceId: string, at = new Date()): Promise<void> {
+  await sql`update process_heartbeat set stopped_at = ${at} where instance_id = ${instanceId}`.execute(db);
+}
+
+/** 裁剪:超过 24 小时没心跳的进程行、超过 7 天的采样;返回删除行数 */
+export async function pruneHeartbeats(
+  db: Kysely<DB>,
+  o: { now?: Date; rowRetentionMs?: number; sampleDays?: number } = {},
+): Promise<{ rows: number; samples: number }> {
+  const now = o.now ?? new Date();
+  const rowCutoff = new Date(now.getTime() - (o.rowRetentionMs ?? HEARTBEAT_ROW_RETENTION_MS));
+  const sampleCutoff = new Date(now.getTime() - (o.sampleDays ?? SAMPLE_RETENTION_DAYS) * 24 * 3600_000);
+  const rows = await db.deleteFrom("process_heartbeat").where("last_beat_at", "<", rowCutoff).executeTakeFirst();
+  const samples = await db.deleteFrom("process_heartbeat_sample").where("at", "<", sampleCutoff).executeTakeFirst();
+  return { rows: Number(rows.numDeletedRows), samples: Number(samples.numDeletedRows) };
 }

 export interface HeartbeatWriter {
@@ -130,7 +147,7 @@ export interface HeartbeatWriter {
  stop(): Promise<void>;
 }

-/** 起一个周期写入器:立即写一次,之后每 intervalMs 写一次;每约 1 小时裁剪一次 */
+/** 起一个周期写入器:立即写一次,之后每 intervalMs 写一次;stop 时把本行标 stopped_at */
 export function startHeartbeatWriter(o: HeartbeatWriterOptions): HeartbeatWriter {
  const host = o.host ?? safeHostname();
  const pid = o.pid ?? Deno.pid;
@@ -140,8 +157,6 @@ export function startHeartbeatWriter(o: HeartbeatWriterOptions): HeartbeatWriter
  const intervalMs = o.intervalMs ?? HEARTBEAT_WRITE_MS;
  const lag = o.lag ?? createLagSampler();
  const log = o.logger.child({ component: "heartbeat", kind: o.kind });
-  const pruneEvery = Math.max(1, Math.round(3_600_000 / intervalMs));
-  let n = 0;
  let inflight: Promise<void> | null = null;
  let stopped = false;

@@ -165,8 +180,6 @@ export function startHeartbeatWriter(o: HeartbeatWriterOptions): HeartbeatWriter
        lagMaxMs: l.windowMaxMs,
        sample: { ...sample, extra: { ...memory(), lifetimeMaxLagMs: l.lifetimeMaxMs, ...(sample.extra ?? {}) } },
      });
-      if (n % pruneEvery === 0) await pruneHeartbeats(o.db);
-      n += 1;
    } catch (err) {
      log.debug({ err }, "心跳写入失败");
    }
@@ -194,6 +207,11 @@ export function startHeartbeatWriter(o: HeartbeatWriterOptions): HeartbeatWriter
      clearInterval(timer);
      lag.stop();
      await inflight?.catch(() => {});
+      try {
+        await markHeartbeatStopped(o.db, instanceId);
+      } catch (err) {
+        log.debug({ err }, "心跳退出标记失败");
+      }
    },
  };
 }
@@ -4,9 +4,17 @@ import type { z } from "zod";
 import { dataPaths } from "../../config.ts";
 import { requireScope } from "../../http/auth.ts";
 import { sseConnectionCount } from "../../http/sse.ts";
+import { registerScheduledJob } from "../schedules/registry.ts";
 import type { ModuleApp, ServerContext, ServerModule } from "../types.ts";
 import { buildHealth, type HealthDeps, loadHistory } from "./health.ts";
-import { type HeartbeatWriter, instanceIdOf, SERVER_VERSION, safeHostname, startHeartbeatWriter } from "./heartbeat.ts";
+import {
+  type HeartbeatWriter,
+  instanceIdOf,
+  pruneHeartbeats,
+  SERVER_VERSION,
+  safeHostname,
+  startHeartbeatWriter,
+} from "./heartbeat.ts";

 export { HEARTBEAT_STALE_MS, HEARTBEAT_WRITE_MS, startHeartbeatWriter } from "./heartbeat.ts";

@@ -15,6 +23,7 @@ export { HEARTBEAT_STALE_MS, HEARTBEAT_WRITE_MS, startHeartbeatWriter } from "./
 * - GET /system/health:进程心跳 / 事件循环延迟 / SSE 连接 / DB / 磁盘 / 挂载根 / 引擎 / 定时任务,仅 admin;
 * - GET /system/health/history?hours=:心跳采样时间序列。
 * 心跳写入:worker 进程在 worker/index.ts 里起写入器;纯 api 进程(role=api)在注册路由时起(role=all 由 worker 写 kind=all)。
+ * 清理:定时任务 system.heartbeat_cleanup(维护组,1 小时)删 24 小时没心跳的进程行与 7 天前的采样。
 */
 export interface SystemModuleOptions {
  /** 引擎 schema(测试覆盖);缺省 openworkflow */
@@ -23,6 +32,8 @@ export interface SystemModuleOptions {
  healthDeps?: Partial<Pick<HealthDeps, "diskUsage" | "checkRoot" | "now" | "ioTimeoutMs" | "dataDir">>;
  /** 缺省 true;false 时 role=api 也不起心跳写入器(测试) */
  apiHeartbeat?: boolean;
+  /** 定时任务 system.heartbeat_cleanup 的默认周期,默认 1 小时;<=0 不登记 */
+  cleanupIntervalMs?: number;
 }

 type Infer<T> = T extends z.ZodType ? z.infer<T> : Record<string, never>;
@@ -93,6 +104,23 @@ export function createSystemModule(opts: SystemModuleOptions = {}): ServerModule
      route(app, Api.system.healthHistory, async ({ query }) => await loadHistory(ctx.db, query.hours));
    },
    registerTasks() {},
+    start(ctx) {
+      const everyMs = opts.cleanupIntervalMs ?? 3600_000;
+      if (everyMs <= 0) return;
+      registerScheduledJob(ctx, {
+        key: "system.heartbeat_cleanup",
+        defaults: {
+          name: "进程心跳清理",
+          description: "删除 24 小时没有心跳的进程行和 7 天前的心跳采样",
+          group: "maintain",
+          schedule: { type: "interval", everyMs },
+        },
+        run: async ({ now }) => {
+          const r = await pruneHeartbeats(ctx.db, { now });
+          return { summary: `删除 ${r.rows} 个进程行、${r.samples} 条采样` };
+        },
+      });
+    },
    async stop() {
      await writer?.stop();
      writer = null;
@@ -6,9 +6,17 @@ import { SystemHealth, SystemHealthHistory } from "@umt/contracts";
 import { sql } from "@umt/db";
 import { afterAll, beforeAll, describe, expect, it } from "vitest";
 import { inferTriggeredBy, triggeredByFromActor } from "../../src/modules/auth-settings/engine-business.ts";
-import { diskLevel, processLevel, worst } from "../../src/modules/system/health.ts";
-import { HEARTBEAT_STALE_MS, startHeartbeatWriter, writeHeartbeat } from "../../src/modules/system/heartbeat.ts";
+import { scheduleRegistry } from "../../src/modules/schedules/registry.ts";
+import { diskLevel, processLevel, splitProcesses, worst } from "../../src/modules/system/health.ts";
+import {
+  HEARTBEAT_STALE_MS,
+  markHeartbeatStopped,
+  pruneHeartbeats,
+  startHeartbeatWriter,
+  writeHeartbeat,
+} from "../../src/modules/system/heartbeat.ts";
 import { createSystemModule } from "../../src/modules/system/index.ts";
+import type { ServerContext } from "../../src/modules/types.ts";
 import { bearer, createHarness, type Harness, pgAvailable } from "../auth-settings/harness.ts";
 import { DEV_KEY, json } from "../helpers.ts";

@@ -39,6 +47,7 @@ describe("纯函数", () => {
        lastBeatAt: new Date().toISOString(),
        beatAgeSec: 0,
        stale: o.stale ?? false,
+        stoppedAt: null,
        eventLoopLagMs: 0,
        eventLoopLagMaxMs: o.lag ?? 0,
        concurrency: {},
@@ -53,6 +62,82 @@ describe("纯函数", () => {
    expect(processLevel([p("api"), p("worker", { lag: 6000 })])).toBe("critical");
  });

+  it("当前实例 / 已下线实例:host 10 分钟无心跳、已退出、被同 host 新进程替换的都算下线", () => {
+    const now = new Date("2026-09-28T12:00:00Z");
+    const ago = (sec: number) => new Date(now.getTime() - sec * 1000).toISOString();
+    const p = (id: string, kind: "api" | "worker" | "worker:agent", host: string, beatAgo: number, o = {}) => ({
+      instanceId: id,
+      processKind: kind,
+      pid: 1,
+      host,
+      version: "t",
+      startedAt: ago(beatAgo + 600),
+      lastBeatAt: ago(beatAgo),
+      beatAgeSec: beatAgo,
+      stale: beatAgo > 45,
+      stoppedAt: null as string | null,
+      eventLoopLagMs: 0,
+      eventLoopLagMaxMs: 0,
+      concurrency: {},
+      runningRuns: null,
+      sseConnections: null,
+      extra: {},
+      ...o,
+    });
+    const rows = [
+      // 当前容器 3 个进程,心跳正常
+      p("api:new-a:1", "api", "new-a", 5),
+      p("worker:new-w:1", "worker", "new-w", 5),
+      p("worker:agent:new-g:1", "worker:agent", "new-g", 5),
+      // 容器重建前的 3 行,最后心跳 40 分钟前
+      p("api:old-a:1", "api", "old-a", 2400),
+      p("worker:old-w:1", "worker", "old-w", 2400),
+      p("worker:agent:old-g:1", "worker:agent", "old-g", 2400),
+      // 同容器进程重启:旧 pid 行 2 分钟没心跳、启动更早
+      p("worker:new-w:0", "worker", "new-w", 120, { startedAt: ago(7200) }),
+      // 正常退出
+      p("api:new-a:0", "api", "new-a", 20, { startedAt: ago(9000), stoppedAt: ago(20) }),
+    ];
+    const r = splitProcesses(rows, now);
+    expect(r.processes.map((x) => x.instanceId).sort()).toEqual([
+      "api:new-a:1",
+      "worker:agent:new-g:1",
+      "worker:new-w:1",
+    ]);
+    expect(r.retiredProcesses.map((x) => x.instanceId)).toEqual([
+      "api:new-a:0",
+      "worker:new-w:0",
+      "api:old-a:1",
+      "worker:old-w:1",
+      "worker:agent:old-g:1",
+    ]);
+    expect(processLevel(r.processes)).toBe("ok");
+
+    // 当前容器里的 worker 停了心跳(1 分钟):仍是当前实例,stale 告警
+    const hung = splitProcesses([p("api:a:1", "api", "a", 5), p("worker:w:1", "worker", "w", 60)], now);
+    expect(hung.processes).toHaveLength(2);
+    expect(processLevel(hung.processes)).toBe("warn");
+    // 退出后立即不算活进程
+    const gone = splitProcesses(
+      [p("api:a:1", "api", "a", 5), p("worker:w:1", "worker", "w", 3, { stoppedAt: ago(1) })],
+      now,
+    );
+    expect(gone.processes.map((x) => x.instanceId)).toEqual(["api:a:1"]);
+    expect(gone.retiredProcesses.map((x) => x.instanceId)).toEqual(["worker:w:1"]);
+  });
+
+  it("system.heartbeat_cleanup:1 小时,维护组", async () => {
+    const ctx = {} as ServerContext;
+    await createSystemModule({ apiHeartbeat: false }).start?.(ctx);
+    expect(scheduleRegistry(ctx).get("system.heartbeat_cleanup")?.defaults).toMatchObject({
+      group: "maintain",
+      schedule: { type: "interval", everyMs: 3_600_000 },
+    });
+    const off = {} as ServerContext;
+    await createSystemModule({ apiHeartbeat: false, cleanupIntervalMs: 0 }).start?.(off);
+    expect(scheduleRegistry(off).get("system.heartbeat_cleanup")).toBeUndefined();
+  });
+
  it("触发来源推断", () => {
    expect(triggeredByFromActor("api_key:abc")).toBe("api_key");
    expect(triggeredByFromActor("admin_session:u1")).toBe("user");
@@ -156,10 +241,15 @@ describe.skipIf(!hasPg)("W9-I 进程与健康(真实 PG)", () => {
    });
    await w.beat();
    await w.beat();
+    expect(
+      (await h.db.selectFrom("process_heartbeat").select("stopped_at").executeTakeFirstOrThrow()).stopped_at,
+    ).toBeNull();
    await w.stop();
    expect(w.instanceId).toBe("worker:test-host:4242");
    const rows = await h.db.selectFrom("process_heartbeat").selectAll().execute();
    expect(rows).toHaveLength(1);
+    // 正常退出:标 stopped_at
+    expect(rows[0]!.stopped_at).toBeInstanceOf(Date);
    expect(rows[0]).toMatchObject({
      process_kind: "worker",
      pid: 4242,
@@ -190,8 +280,15 @@ describe.skipIf(!hasPg)("W9-I 进程与健康(真实 PG)", () => {
    expect(res.status).toBe(200);
    const d = SystemHealth.parse((await json(res)).data);
    const byId = new Map(d.processes.map((p) => [p.instanceId, p]));
-    expect(byId.get("worker:test-host:4242")).toMatchObject({ stale: false, runningRuns: 2, concurrency: { io: 4 } });
-    expect(byId.get("api:old-host:1")).toMatchObject({ stale: true, sseConnections: 3 });
+    // 已退出的 worker 立即归入已下线实例,不再当活进程
+    expect(byId.has("worker:test-host:4242")).toBe(false);
+    expect(d.retiredProcesses.find((p) => p.instanceId === "worker:test-host:4242")).toMatchObject({
+      runningRuns: 2,
+      concurrency: { io: 4 },
+      stoppedAt: expect.any(String),
+    });
+    // 60s 前的心跳,host 仍在 10 分钟窗口内 → 当前实例,stale
+    expect(byId.get("api:old-host:1")).toMatchObject({ stale: true, sseConnections: 3, stoppedAt: null });
    // stale 进程的 SSE 不计入
    expect(d.sse.connections).toBe(0);
  });
@@ -379,4 +476,61 @@ describe.skipIf(!hasPg)("W9-I 进程与健康(真实 PG)", () => {
    expect(detail.business).toMatchObject({ kind: "agent", refId: agentId });
    expect(detail.stepAttempts).toEqual([]);
  });
+  it("容器重建留下的旧心跳行不计入告警;24 小时的行由清理任务删除;重启后 upsert 清掉 stopped_at", async () => {
+    await sql`delete from process_heartbeat`.execute(h.db);
+    const now = Date.now();
+    const beat = (id: string, kind: "api" | "worker" | "worker:agent", host: string, agoMs: number) =>
+      writeHeartbeat(h.db, {
+        instanceId: id,
+        kind,
+        pid: 1,
+        host,
+        version: "0.2.0",
+        startedAt: new Date(now - agoMs - 3600_000),
+        lagMs: 0,
+        lagMaxMs: 0,
+        sample: {},
+        at: new Date(now - agoMs),
+      });
+    await beat("api:c-api:1", "api", "c-api", 2_000);
+    await beat("worker:c-io:1", "worker", "c-io", 2_000);
+    await beat("worker:agent:c-ag:1", "worker:agent", "c-ag", 2_000);
+    await beat("api:o-api:1", "api", "o-api", 40 * 60_000);
+    await beat("worker:o-io:1", "worker", "o-io", 40 * 60_000);
+    await beat("worker:agent:o-ag:1", "worker:agent", "o-ag", 40 * 60_000);
+    await beat("worker:ancient:1", "worker", "ancient", 25 * 3600_000);
+
+    const d = SystemHealth.parse((await json(await admin("GET", "/system/health"))).data);
+    expect(d.processes.map((p) => p.instanceId).sort()).toEqual([
+      "api:c-api:1",
+      "worker:agent:c-ag:1",
+      "worker:c-io:1",
+    ]);
+    expect(d.processes.every((p) => !p.stale)).toBe(true);
+    // 超过 24 小时的行不再返回
+    expect(d.retiredProcesses.map((p) => p.instanceId).sort()).toEqual([
+      "api:o-api:1",
+      "worker:agent:o-ag:1",
+      "worker:o-io:1",
+    ]);
+
+    // 清理任务:删 24 小时没心跳的行
+    const r = await pruneHeartbeats(h.db);
+    expect(r.rows).toBe(1);
+    const left = await h.db.selectFrom("process_heartbeat").select("instance_id").execute();
+    expect(left.map((x) => x.instance_id)).not.toContain("worker:ancient:1");
+    expect(left).toHaveLength(6);
+
+    // 退出标记后再次心跳(同 instance_id 重启):stopped_at 清回 null
+    await markHeartbeatStopped(h.db, "worker:c-io:1");
+    const d2 = SystemHealth.parse((await json(await admin("GET", "/system/health"))).data);
+    expect(d2.processes.map((p) => p.instanceId)).not.toContain("worker:c-io:1");
+    await beat("worker:c-io:1", "worker", "c-io", 0);
+    const row = await h.db
+      .selectFrom("process_heartbeat")
+      .select("stopped_at")
+      .where("instance_id", "=", "worker:c-io:1")
+      .executeTakeFirstOrThrow();
+    expect(row.stopped_at).toBeNull();
+  });
 });
@@ -1,6 +1,17 @@
 import { useQueryClient } from "@tanstack/react-query";
 import type { HealthLevel, SystemHealth, SystemProcess } from "@umt/contracts";
-import { AlertTriangle, CircleCheck, Cpu, Database, HardDrive, ListTodo, RotateCw, Timer, XCircle } from "lucide-react";
+import {
+  AlertTriangle,
+  ChevronDown,
+  CircleCheck,
+  Cpu,
+  Database,
+  HardDrive,
+  ListTodo,
+  RotateCw,
+  Timer,
+  XCircle,
+} from "lucide-react";
 import { type ReactNode, useState } from "react";
 import { Link } from "react-router";
 import { useSystemHealth, useSystemHealthHistory } from "~/api/system.ts";
@@ -35,6 +46,14 @@ const KIND_LABEL: Record<string, string> = {
  all: "单进程(all)",
 };

+/** 横幅告警里的进程称呼 */
+const KIND_NAME: Record<string, string> = {
+  api: "API",
+  worker: "io worker",
+  "worker:agent": "Agent worker",
+  all: "单进程",
+};
+
 const MOUNT_REASON: Record<string, string> = {
  root_missing: "目录不存在",
  sentinel_missing: "哨兵文件缺失",
@@ -74,8 +93,11 @@ function Card({
 /** 顶部横幅:整体档位 + 需要看的几条 */
 function problems(d: SystemHealth): string[] {
  const out: string[] = [];
-  const stale = d.processes.filter((p) => p.stale);
-  if (stale.length) out.push(`${stale.length} 个进程超过 45 秒没有心跳`);
+  // 只看当前实例;已下线实例不告警
+  for (const p of d.processes.filter((x) => x.stale)) {
+    const line = `${KIND_NAME[p.processKind] ?? p.processKind} 45 秒没有心跳`;
+    if (!out.includes(line)) out.push(line);
+  }
  if (!d.processes.some((p) => !p.stale && p.processKind !== "api")) out.push("没有在线的 worker，任务不会被执行");
  const lag = d.processes.filter((p) => !p.stale && p.eventLoopLagMaxMs >= 1000);
  if (lag.length) out.push(`${lag.length} 个进程事件循环延迟超过 1 秒`);
@@ -136,7 +158,7 @@ function Processes({ d }: { d: SystemHealth }) {
    <DataTable<SystemProcess>
      rows={d.processes}
      rowKey={(p) => p.instanceId}
-      empty={{ title: "24 小时内没有进程心跳", description: "进程启动后每 15 秒写一次心跳。" }}
+      empty={{ title: "没有在线的进程", description: "进程启动后每 15 秒写一次心跳。" }}
      columns={[
        {
          key: "kind",
@@ -236,6 +258,77 @@ function Processes({ d }: { d: SystemHealth }) {
  );
 }

+/** 已下线实例:已退出、容器已重建或被同容器新进程替换的旧心跳行,默认折叠,不参与告警 */
+function RetiredProcesses({ rows }: { rows: SystemProcess[] }) {
+  const [open, setOpen] = useState(false);
+  if (!rows.length) return null;
+  return (
+    <div className="mt-3">
+      <button
+        type="button"
+        aria-expanded={open}
+        onClick={() => setOpen((v) => !v)}
+        className="flex cursor-pointer items-center gap-1.5 text-caption text-muted-foreground hover:text-foreground"
+      >
+        <ChevronDown className={cn("h-3.5 w-3.5 transition-transform", !open && "-rotate-90")} />
+        <span>已下线实例 {rows.length} 个</span>
+        <span className="text-meta">不计入告警，24 小时后清理</span>
+      </button>
+      {open ? (
+        <div className="mt-2">
+          <DataTable<SystemProcess>
+            rows={rows}
+            rowKey={(p) => p.instanceId}
+            empty={{ title: "没有已下线实例" }}
+            columns={[
+              {
+                key: "id",
+                header: "实例",
+                cell: (p) => (
+                  <div className="min-w-0">
+                    <p className="flex items-center gap-2 font-medium text-muted-foreground">
+                      {KIND_LABEL[p.processKind] ?? p.processKind}
+                      {p.stoppedAt ? <StatusBadge tone="neutral">已退出</StatusBadge> : null}
+                    </p>
+                    <p className="truncate font-mono text-meta text-muted-foreground" title={p.instanceId}>
+                      {p.instanceId}
+                    </p>
+                  </div>
+                ),
+              },
+              {
+                key: "ver",
+                header: "版本",
+                hideBelow: "md",
+                cell: (p) => <span className="font-mono text-meta">{p.version}</span>,
+              },
+              {
+                key: "start",
+                header: "启动",
+                hideBelow: "md",
+                cell: (p) => (
+                  <span className="whitespace-nowrap text-muted-foreground" title={formatDateTime(p.startedAt)}>
+                    {formatRelative(p.startedAt)}
+                  </span>
+                ),
+              },
+              {
+                key: "beat",
+                header: "最后心跳",
+                cell: (p) => (
+                  <span className="whitespace-nowrap text-muted-foreground" title={formatDateTime(p.lastBeatAt)}>
+                    {formatRelative(p.lastBeatAt)}
+                  </span>
+                ),
+              },
+            ]}
+          />
+        </div>
+      ) : null}
+    </div>
+  );
+}
+
 function History() {
  const [hours, setHours] = useState(6);
  const h = useSystemHealthHistory(hours);
@@ -397,6 +490,7 @@ export default function StatusPage() {
              <span className="text-meta font-normal text-muted-foreground">SSE 连接合计 {d.sse.connections}</span>
            </h2>
            <Processes d={d} />
+            <RetiredProcesses rows={d.retiredProcesses} />
          </section>
          <History />
          <div className="grid gap-4 md:grid-cols-2 xl:grid-cols-3">
@@ -1,6 +1,6 @@
 {
  "name": "@umt/web",
-  "version": "0.2.0",
+  "version": "0.2.2",
  "private": true,
  "type": "module",
  "scripts": {
@@ -1,5 +1,5 @@
 {
-  "version": "0.2.0",
+  "version": "0.2.2",
  "workspace": [
    "./packages/adapters",
    "./packages/agent",
@@ -42,6 +42,8 @@
 | agent.propose_options | 方案;动作叫"选定",都不要叫"退回" | 选项、提案 |
 | scope.proposeFirst | 先看方案再执行 | 只要方案、只给方案、plan_only 模式 |
 | 工作流引擎的一次执行 | 运行 | run |
+| 正在写心跳的进程(process_heartbeat 当前行) | 当前实例;横幅按角色称呼:API / io worker / Agent worker / 单进程 | 活进程、在线实例 |
+| 已退出、容器已重建或被同容器新进程替换的心跳行 | 已下线实例;正常退出的标"已退出" | 历史实例、僵尸进程、死进程 |

 ## 文案规则

@@ -13,7 +13,8 @@

 - 每 15 秒 upsert `process_heartbeat` 一行,同时往 `process_heartbeat_sample` 追加一条采样。
 - 行主键 `instance_id = <kind>:<host>:<pid>`,容器重启覆盖同一行。
- 写入方每小时裁剪一次:采样保留 7 天,7 天没心跳的进程行删除。
+- 正常退出(SIGTERM / SIGINT)时写入器把本行标 `stopped_at`,读取方立即归入已下线实例;重启后同一行的 upsert 把它清回 null。
+- 裁剪由定时任务 `system.heartbeat_cleanup`(维护组,1 小时)负责:24 小时没心跳的进程行删除,采样保留 7 天。
 - 写库失败只记 debug 日志,不影响业务。

 ## GET /system/health 字段
@@ -22,8 +23,10 @@
 | --- | --- |
 | `level` | 各段最差档:`ok` / `warn` / `critical`。取不到数据的段是 `unknown`,不参与汇总 |
 | `api` | 处理本次请求的 API 进程:版本、启动时间、运行秒数 |
-| `processes[]` | 最近 24 小时有心跳的进程 |
-| `processes[].stale` | 超过 45 秒没心跳 |
+| `processes[]` | 当前实例:最近 10 分钟内有心跳的 host 上、同 host 同 kind 最新启动且没标 `stopped_at` 的进程 |
+| `processes[].stale` | 超过 45 秒没心跳。只有当前实例参与告警 |
+| `processes[].stoppedAt` | 正常退出时间;非 null 即已下线 |
+| `retiredProcesses[]` | 已下线实例:正常退出的、所在 host 10 分钟没有心跳(容器已重建)的、被同 host 新进程替换的。不计入 `level`,页面折叠显示 |
 | `processes[].eventLoopLagMs` | 最近一次采样的事件循环延迟(1 秒采一次) |
 | `processes[].eventLoopLagMaxMs` | 最近一个心跳周期(15 秒)内的最大延迟 |
 | `processes[].concurrency` | 引擎 worker 并发 `{io, agent}`;api 进程为 `{}` |
@@ -48,7 +51,7 @@

 ## 告警阈值建议

- 进程 `stale`:warn。连续 2 分钟仍 stale 就去查容器状态和日志。
+- 当前实例 `stale`:warn。连续 2 分钟仍 stale 就去查容器状态和日志。容器整个没了 10 分钟后它的进程转入已下线实例,此时靠下一条"没有存活的 worker"兜底。
 - 没有任何存活的 worker / all 进程:warn。任务不会被执行。
 - 事件循环最大延迟 ≥ 1 秒记 warn,≥ 5 秒记 critical。延迟接近 15 秒时,引擎租约有丢失风险。
 - DB 延迟 ≥ 100 ms 或连接数 ≥ 75% `max_connections` 记 warn;≥ 500 ms 或 ≥ 90% 记 critical。
@@ -30,6 +30,7 @@
 | `trash.purge` | 回收站清理 | 维护 | 24 小时 | 入队 `purgeTrash` |
 | `pi.token_refresh` | 上游登录刷新 | 维护 | 10 分钟 | 入队 `refreshPiTokens`,幂等键 `refreshPiTokens:<时间槽>`;对 30 分钟内过期的订阅登录上游刷新 token |
 | `auth.cleanup` | 登录会话清理 | 维护 | 1 小时 | 直接执行 `cleanupAuthTables`:删 7 天前的登录尝试、30 天前过期或吊销的会话 |
+| `system.heartbeat_cleanup` | 进程心跳清理 | 维护 | 1 小时 | 删 24 小时没有心跳的进程行、7 天前的心跳采样 |
 | `maintain.backup` | 备份 | 维护 | 每天 03:30(占位) | 未配置。备份目前由宿主机上的 `scripts/backup.sh` 执行 |
 | `check.pending` | 待处理巡检 | Agent | 6 小时，默认关闭 | 只读。有未关闭事项时才会运行 |
 | `check.downloads` | 下载诊断 | Agent | 2 小时，默认关闭 | 默认只读。打开 `allowRecover` 后，在授权范围内重试或重新校验 |
@@ -1,6 +1,6 @@
 {
  "name": "@umt/adapters",
-  "version": "0.2.0",
+  "version": "0.2.2",
  "exports": { ".": "./src/index.ts", "./fsworker": "./src/fs/fsworker.ts" },
  "tasks": {
    "test": "deno run -A npm:vitest run",
@@ -1,6 +1,7 @@
 /**
 * camofox-browser 适配器(BrowserProvider 首个实现)。
- * 对照:jo-inc/camofox-browser master openapi.json(info.version 1.17.0)与 README,仅读文档,未连接真实实例。
+ * 对照:jo-inc/camofox-browser v1.17.0(commit 389c996)openapi.json、README 与 server.js;2026-09-28 对真实容器核验,
+ * 结果见 docs/runbook/browser.md「真机核验」。
 *
 * 上游模型与本项目合同的映射:
 * - camofox userId = 一个隔离 BrowserContext(cookies/storage),本项目用 profileKey(站点/账号)充当;
@@ -9,10 +10,10 @@
 * 认证:README/OpenAPI 为 `Authorization: Bearer <CAMOFOX_ACCESS_KEY>`(设置后除 /health 外全部路由要求);
 * 凭据每次请求经 BrowserSecretResolver 解出,不缓存、不入日志、不进错误信息。
 *
- * 已实现(仅对 mock 与上游文档验证):health、capabilities、会话开/关/列举、标签页增删与对账、navigate(白名单/超时/重试)、
+ * 已实现:health、capabilities、会话开/关/列举、标签页增删与对账、navigate(白名单/超时/重试)、
 * 后退/前进/刷新、snapshot(脱敏)、screenshot、click/type/select/scroll/press、extract、waitForDownload、
 * 内存版控制权租约与串行锁。会话/标签/租约的持久化与对账由服务层(apps/server modules/browser)以数据库为准。
- * 未核验的上游行为(坐标点击、无目标输入、downloads 字段形状)见 docs/runbook/browser.md。
+ * 与文档不同的实测行为(截图直接返回 image/png、click 不接受坐标、焦点输入需 mode=keyboard、downloads 字段名)已按实测适配。
 */
 import type {
  BrowserCapabilities,
@@ -718,6 +719,13 @@ export class CamofoxBrowserProvider implements BrowserProvider {
    await this.withProfile(s, async () => {
      this.assertActor(s, ref.as);
      const tab = this.tabOf(s, ref);
+      // 上游 click/type 只认 ref/selector(1.17.0 实测不接受 {x, y}):坐标先经 evaluate 标记该点元素,再按选择器操作,
+      // 这样仍走上游完整的鼠标事件序列与下载守卫,而不是页面内合成 click
+      if (typeof payload.x === "number" && typeof payload.y === "number") {
+        payload.selector = await this.pointSelector(s, tab, payload.x, payload.y);
+        delete payload.x;
+        delete payload.y;
+      }
      const path = `/tabs/${encodeURIComponent(tab.upstreamTabId)}/${action}`;
      const res = await this.request("POST", path, {
        body: { userId: s.session.profileKey, ...payload },
@@ -732,6 +740,22 @@ export class CamofoxBrowserProvider implements BrowserProvider {
    });
  }

+  /** 视口坐标 → 唯一属性选择器(POST /tabs/{id}/evaluate + elementFromPoint);坐标处无元素报 bad_request */
+  private async pointSelector(s: SessionState, tab: BrowserTab, x: number, y: number): Promise<string> {
+    const nonce = this.newId().replace(/[^a-z0-9]/gi, "").slice(0, 16);
+    const expression = `(() => { const e = document.elementFromPoint(${x}, ${y}); if (!e) return false;
+      for (const o of document.querySelectorAll("[data-umt-pt]")) o.removeAttribute("data-umt-pt");
+      e.setAttribute("data-umt-pt", "${nonce}"); return true; })()`;
+    const path = `/tabs/${encodeURIComponent(tab.upstreamTabId)}/evaluate`;
+    const res = await this.request("POST", path, {
+      body: { userId: s.session.profileKey, expression },
+      timeoutMs: s.budget.timeoutMs,
+    });
+    const out = await this.json(res, path);
+    if (out.result !== true) throw new BrowserError("bad_request", "该坐标处没有可操作的元素");
+    return `[data-umt-pt="${nonce}"]`;
+  }
+
  /** 点击可能跳转或改动页面:之后旧引用作废 */
  async click(ref: TabRef, target: ElementTarget): Promise<void> {
    await this.act(
@@ -765,7 +789,8 @@ export class CamofoxBrowserProvider implements BrowserProvider {
      ref,
      "type",
      (s) => ({
-        ...(target ? this.assertTarget(s, target) : {}),
+        // 无目标 = 输入到当前焦点:上游 mode=fill 必须带 ref/selector,焦点输入要 mode=keyboard(1.17.0 实测)
+        ...(target ? this.assertTarget(s, target) : { mode: "keyboard" }),
        text,
        ...(input.clear !== undefined ? { clear: input.clear } : {}),
        ...(input.submit !== undefined ? { submit: input.submit } : {}),
@@ -875,7 +900,13 @@ export class CamofoxBrowserProvider implements BrowserProvider {
          if (s.seenDownloads.has(key)) continue;
          const status = str(d.status) ?? str(d.state);
          if (status && !["completed", "complete", "done", "finished"].includes(status)) continue;
-          const b64 = str(d.data) ?? str(d.base64) ?? str(d.content);
+          // 1.17.0 实测字段:{ id, url, suggestedFilename, mimeType, bytes, createdAt, failure, dataBase64 | dataSkipped | readError }
+          if (d.failure) {
+            s.seenDownloads.add(key);
+            throw new BrowserError("upstream", "浏览器下载失败");
+          }
+          if (d.dataSkipped) throw new BrowserError("budget_exceeded", `下载产物超出字节预算(${opts.maxBytes})`);
+          const b64 = str(d.dataBase64) ?? str(d.data) ?? str(d.base64) ?? str(d.content);
          const size = int(d.size) ?? int(d.bytes) ?? null;
          if (size !== null && size > opts.maxBytes)
            throw new BrowserError("budget_exceeded", `下载产物超出字节预算(${opts.maxBytes})`);
@@ -1,6 +1,6 @@
 {
  "name": "@umt/agent",
-  "version": "0.2.0",
+  "version": "0.2.2",
  "exports": { ".": "./src/index.ts" },
  "tasks": {
    "test": "deno run -A npm:vitest run",
@@ -1,6 +1,6 @@
 {
  "name": "@umt/contracts",
-  "version": "0.2.0",
+  "version": "0.2.2",
  "exports": { ".": "./src/index.ts" },
  "tasks": {
    "test": "deno run -A npm:vitest run",
@@ -2167,6 +2167,31 @@ export const Api = {
      query: browser.BrowserProvidersQuery,
      response: z.object({ items: z.array(browser.BrowserProviderView) }),
    }),
+    createProvider: ep({
+      method: "post",
+      path: "/browser/providers",
+      scope: "admin",
+      summary: "登记浏览器服务（camofox 地址、访问凭据存凭证库、启用）",
+      body: browser.BrowserProviderCreate,
+      response: browser.BrowserProviderView,
+    }),
+    updateProvider: ep({
+      method: "patch",
+      path: "/browser/providers/{id}",
+      scope: "admin",
+      summary: "修改浏览器服务地址、访问凭据或启用（环境变量登记的只能改启用）",
+      params: browser.BrowserProviderParams,
+      body: browser.BrowserProviderUpdate,
+      response: browser.BrowserProviderView,
+    }),
+    probeProvider: ep({
+      method: "post",
+      path: "/browser/providers/{id}/probe",
+      scope: "admin",
+      summary: "测试连通：探测该浏览器服务的 /health 与 /openapi.json 并写回能力",
+      params: browser.BrowserProviderParams,
+      response: browser.BrowserProviderView,
+    }),
    sessions: ep({
      method: "get",
      path: "/browser/sessions",
@@ -240,8 +240,12 @@ export const BrowserProviderView = z.object({
  baseUrl: z.string().nullable(),
  /** 访问凭据的 secret 引用(明文永不出 API) */
  secretRef: Id.nullable(),
+  /** env = 由 BROWSER_PROVIDER_URL / BROWSER_ACCESS_KEY 登记(地址与凭据只能改部署配置);null = 设置页登记 */
+  managedBy: z.enum(["env"]).nullable(),
+  /** 是否配置了访问凭据(secret 或环境变量) */
+  hasAccessKey: z.boolean(),
  enabled: z.boolean(),
-  /** 最近一次声明/探测到的能力;verified 只由人工核验写入 config.verified */
+  /** 最近一次声明/探测到的能力;verified 只由人工核验写入(config.verified 或适配器内置的已核验版本表) */
  capabilities: BrowserCapabilities.nullable(),
  verifiedAt: Timestamp.nullable(),
  healthStatus: InstanceHealth,
@@ -251,6 +255,30 @@ export const BrowserProviderView = z.object({
 });
 export type BrowserProviderView = z.infer<typeof BrowserProviderView>;

+const HttpUrl = z
+  .string()
+  .url()
+  .refine((u) => /^https?:\/\//i.test(u), "只支持 http(s) 地址");
+
+/** 设置页登记浏览器服务(camofox);accessKey 明文只在请求里出现一次,落库为 secret */
+export const BrowserProviderCreate = z.object({
+  name: z.string().min(1).max(64),
+  baseUrl: HttpUrl,
+  accessKey: z.string().min(1).max(512).optional(),
+  enabled: z.boolean().optional(),
+});
+export type BrowserProviderCreate = z.infer<typeof BrowserProviderCreate>;
+
+/** 改地址 / 凭据 / 启用;accessKey = null 清除凭据 */
+export const BrowserProviderUpdate = z.object({
+  baseUrl: HttpUrl.optional(),
+  accessKey: z.string().min(1).max(512).nullable().optional(),
+  enabled: z.boolean().optional(),
+});
+export type BrowserProviderUpdate = z.infer<typeof BrowserProviderUpdate>;
+
+export const BrowserProviderParams = z.object({ id: Id });
+
 export const BrowserProvidersQuery = z.object({
  /** true = 现场探测 /health 与 /openapi.json 并写回 */
  probe: QueryBool,
@@ -381,8 +381,10 @@ export const SystemProcess = z.object({
  lastBeatAt: Timestamp,
  /** 距上次心跳秒数 */
  beatAgeSec: z.number().nonnegative(),
-  /** 超过 45s 没心跳 */
+  /** 超过 45s 没心跳(只对当前实例告警) */
  stale: z.boolean(),
+  /** 进程正常退出时写入;非 null 即已下线 */
+  stoppedAt: Timestamp.nullable(),
  /** 最近一次采样的事件循环延迟 */
  eventLoopLagMs: NonNegInt,
  /** 最近一个心跳周期(15s)内的最大延迟 */
@@ -434,7 +436,10 @@ export const SystemHealth = z.object({
  checkedAt: Timestamp,
  /** 处理本请求的 api 进程 */
  api: z.object({ version: z.string(), startedAt: Timestamp, uptimeSec: z.number().nonnegative() }),
+  /** 当前实例:最近 10 分钟内有心跳的 host 上、同 host 同 kind 最新启动且未退出的进程;stale 与告警只看这里 */
  processes: z.array(SystemProcess),
+  /** 已下线实例:已正常退出、所在 host 10 分钟没有心跳(容器已重建)、或被同 host 新进程替换;不计入 level */
+  retiredProcesses: z.array(SystemProcess),
  sse: z.object({ connections: NonNegInt }),
  db: z.object({
    level: HealthLevel,
@@ -1,6 +1,6 @@
 {
  "name": "@umt/db",
-  "version": "0.2.0",
+  "version": "0.2.2",
  "exports": { ".": "./src/index.ts" },
  "tasks": {
    "test": "deno run -A npm:vitest run",
@@ -0,0 +1,16 @@
+import type { Kysely } from "kysely";
+import { execDdl } from "../src/migration-helpers.ts";
+
+// 心跳残留修复:进程正常退出(SIGTERM / SIGINT)时把自己的行标 stopped_at,读取方立即把它当已下线实例,
+// 不再等 45s 判 stale。重启后同一 instance_id 的 upsert 把 stopped_at 清回 null。
+// 超过 24 小时没心跳的行由定时任务 system.heartbeat_cleanup 删除。
+
+// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型
+export async function up(db: Kysely<any>): Promise<void> {
+  await execDdl(db, `alter table process_heartbeat add column stopped_at timestamptz;`);
+}
+
+// biome-ignore lint/suspicious/noExplicitAny: 同上
+export async function down(db: Kysely<any>): Promise<void> {
+  await execDdl(db, `alter table process_heartbeat drop column if exists stopped_at;`);
+}
@@ -1,5 +1,5 @@
 // 统一从本包取 Kysely 与 sql,避免各包各装一份 kysely
-export { type Kysely, sql, type Transaction } from "kysely";
+export { type Kysely, type Selectable, sql, type Transaction } from "kysely";
 export {
  type CachedOptions,
  cachedCall,
@@ -1152,6 +1152,7 @@ export interface ProcessHeartbeatTable {
  concurrency: Generated<Json>;
  running_runs: number | null;
  extra: Generated<Json>;
+  stopped_at: Timestamptz | null;
 }

 export interface ProcessHeartbeatSampleTable {
@@ -1,6 +1,6 @@
 {
  "name": "@umt/domain",
-  "version": "0.2.0",
+  "version": "0.2.2",
  "exports": { ".": "./src/index.ts" },
  "tasks": {
    "test": "deno run -A npm:vitest run",
@@ -1,6 +1,6 @@
 {
  "name": "@umt/mapping",
-  "version": "0.2.0",
+  "version": "0.2.2",
  "exports": { ".": "./src/index.ts" },
  "tasks": {
    "test": "deno run -A npm:vitest run",
@@ -1,6 +1,6 @@
 {
  "name": "@umt/tasks",
-  "version": "0.2.0",
+  "version": "0.2.2",
  "exports": { ".": "./src/index.ts" },
  "tasks": {
    "test": "deno run -A npm:vitest run",
@@ -1,6 +1,6 @@
 {
  "name": "@umt/testkit",
-  "version": "0.2.0",
+  "version": "0.2.2",
  "exports": { ".": "./src/index.ts" },
  "tasks": {
    "test": "deno run -A npm:vitest run",
@@ -4,14 +4,15 @@ import { deflateSync } from "node:zlib";
 import type { MockCall, MockFault, MockHttp, MockReply } from "../tmdb/mock-http.ts";

 /**
- * mock camofox-browser(形状按 jo-inc/camofox-browser openapi.json 1.17.0,取值虚构,未对真实实例核验)。
+ * mock camofox-browser(形状按 jo-inc/camofox-browser v1.17.0,2026-09-28 与真实容器对照过请求/响应字段;取值虚构)。
 * 路由:GET /health、GET /openapi.json、POST /tabs、GET /tabs、DELETE /tabs/{id}、POST /tabs/{id}/navigate、
 * POST /tabs/{id}/{back,forward,refresh}、GET /tabs/{id}/snapshot、GET /tabs/{id}/screenshot(base64 PNG)、
- * POST /tabs/{id}/{click,type,select,scroll,press,extract}、GET /tabs/{id}/downloads、DELETE /sessions/{userId}。
+ * POST /tabs/{id}/{click,type,select,scroll,press,extract,evaluate}、GET /tabs/{id}/downloads、DELETE /sessions/{userId}。
 * 不给 site:快照为固定文本(含密码值/Cookie/passkey 供脱敏测试)。
 * 给 site:模拟一个 NexusPHP 风格站点——未登录访问受保护页重定向登录页,登录页可带验证码(需人工),
 * 点击/输入改变页面,点"下载"把 .torrent 放进该标签页的下载列表;登录状态按 userId(profile)保存。
- * 坐标点击:页面元素按快照行排布,第 i 行(从 0 起)占 y ∈ [MOCK_ROW_Y0 + i·MOCK_ROW_H, +MOCK_ROW_H)。
+ * 坐标:页面元素按快照行排布,第 i 行(从 0 起)占 y ∈ [MOCK_ROW_Y0 + i·MOCK_ROW_H, +MOCK_ROW_H);与真实上游一样
+ * click 不接受 {x, y},适配器先 evaluate(elementFromPoint)打 data-umt-pt 标记再按选择器点击。
 * accessKey 设置时模拟 CAMOFOX_ACCESS_KEY:除 /health 外要求 `Authorization: Bearer <key>`,否则 401。
 * 故障注入(401/超时/断连)用 MockHttp.fault。
 */
@@ -58,6 +59,8 @@ export interface MockCamofoxTab {
  /** 元素引用 → 已输入的值 */
  fields: Map<string, string>;
  focus: string | null;
+  /** evaluate(elementFromPoint)打的 data-umt-pt 标记 → 元素引用 */
+  marks: Map<string, string>;
  downloads: MockCamofoxDownload[];
  error: string | null;
 }
@@ -121,6 +124,7 @@ const OPENAPI_PATHS: Record<string, string[]> = {
  "/tabs/{tabId}/click": ["post"],
  "/tabs/{tabId}/type": ["post"],
  "/tabs/{tabId}/select": ["post"],
+  "/tabs/{tabId}/evaluate": ["post"],
  "/tabs/{tabId}/extract": ["post"],
  "/tabs/{tabId}/downloads": ["get"],
  "/sessions/{userId}": ["delete"],
@@ -403,8 +407,11 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise<
  const findLine = (t: MockCamofoxTab, b: Body): Line | undefined => {
    const lines = pageLines(t);
    if (typeof b.ref === "string") return lines.find((l) => l.ref === b.ref);
-    if (typeof b.y === "number") return lines[Math.floor((b.y - MOCK_ROW_Y0) / MOCK_ROW_H)];
-    if (typeof b.selector === "string") return lines.find((l) => l.role.startsWith("button"));
+    if (typeof b.selector === "string") {
+      const mark = /^\[data-umt-pt="([^"]+)"\]$/.exec(b.selector)?.[1];
+      if (mark) return lines.find((l) => l.ref === t.marks.get(mark));
+      return lines.find((l) => l.role.startsWith("button"));
+    }
    return undefined;
  };

@@ -473,6 +480,7 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise<
            cursor: -1,
            fields: new Map(),
            focus: null,
+            marks: new Map(),
            downloads: [],
            error: null,
          };
@@ -565,6 +573,9 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise<
      [
        /^\/tabs\/([^/]+)\/click$/,
        action("click", (t, b) => {
+          // 1.17.0 实测:只认 ref / selector,坐标点击由适配器先 evaluate 打标记
+          if (typeof b.ref !== "string" && typeof b.selector !== "string")
+            return err(400, "ref or selector required");
          const line = findLine(t, b);
          if (!line) return typeof b.ref === "string" ? err(422, "Unknown ref") : undefined;
          if (line.role.startsWith("textbox")) t.focus = line.ref;
@@ -576,6 +587,8 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise<
        /^\/tabs\/([^/]+)\/type$/,
        action("type", (t, b) => {
          if (typeof b.text !== "string") return err(400, "text required");
+          if (typeof b.ref !== "string" && typeof b.selector !== "string" && b.mode !== "keyboard")
+            return err(400, "ref or selector required for mode=fill");
          const ref = typeof b.ref === "string" ? b.ref : (findLine(t, b)?.ref ?? t.focus);
          if (!ref) return err(422, "No focused element");
          t.fields.set(ref, b.clear === false ? (t.fields.get(ref) ?? "") + b.text : b.text);
@@ -584,6 +597,21 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise<
          return undefined;
        }),
      ],
+      [
+        /^\/tabs\/([^/]+)\/evaluate$/,
+        action("evaluate", (t, b) => {
+          // 只模拟适配器的坐标定位表达式:elementFromPoint(x, y) + data-umt-pt 标记
+          const expr = typeof b.expression === "string" ? b.expression : "";
+          const pt = /elementFromPoint\((\d+), (\d+)\)/.exec(expr);
+          const nonce = /setAttribute\("data-umt-pt", "([^"]+)"\)/.exec(expr)?.[1];
+          if (!pt || !nonce) return { json: { ok: true, result: null } };
+          const line = pageLines(t)[Math.floor((Number(pt[2]) - MOCK_ROW_Y0) / MOCK_ROW_H)];
+          if (!line) return { json: { ok: true, result: false } };
+          t.marks.clear();
+          t.marks.set(nonce, line.ref);
+          return { json: { ok: true, result: true } };
+        }),
+      ],
      [/^\/tabs\/([^/]+)\/select$/, action("select", () => undefined)],
      [/^\/tabs\/([^/]+)\/scroll$/, action("scroll", () => undefined)],
      [
@@ -617,16 +645,20 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise<
          const t = tabFor(m[1]!, q.userId);
          if (!t) return err(404, "Tab not found");
          const max = q.maxBytes ? Number(q.maxBytes) : Number.POSITIVE_INFINITY;
+          // 字段形状按 1.17.0 实测(lib/downloads.js getDownloadsList)
          const list = t.downloads.map((d) => ({
            id: d.id,
-            filename: d.filename,
            url: d.url,
+            suggestedFilename: d.filename,
            mimeType: d.mimeType,
-            size: d.bytes.byteLength,
-            status: "completed",
-            ...(q.includeData === "true" && d.bytes.byteLength <= max
-              ? { data: Buffer.from(d.bytes).toString("base64") }
-              : { truncated: d.bytes.byteLength > max }),
+            bytes: d.bytes.byteLength,
+            createdAt: Date.now(),
+            failure: null,
+            ...(q.includeData === "true"
+              ? d.bytes.byteLength <= max
+                ? { dataBase64: Buffer.from(d.bytes).toString("base64") }
+                : { dataSkipped: "max_bytes_exceeded" }
+              : {}),
          }));
          if (q.consume === "true") t.downloads = [];
          return { json: { downloads: list } };