fix(system): retired heartbeat rows no longer alarm on status page

- GET /system/health splits processes into current instances (hosts with
  a beat in the last 10 min, newest per host+kind, not stopped) and
  retiredProcesses[]; 45s stale and level only consider current ones
- heartbeat writer marks stopped_at on graceful exit (migration 0043),
  upsert clears it; pure api role now stops its writer on SIGTERM
- scheduled job system.heartbeat_cleanup (maintain, 1h) deletes rows
  without a beat for 24h and samples older than 7 days
- status page: per-role alarms ("io worker 45 秒没有心跳"), collapsible
  retired instances; glossary/runbook updated
- bump all members to 0.2.2, CHANGELOG [0.2.2]

Co-Authored-By: Claude Opus 5.5 <[email protected]>

apps/server/src/modules/system/health.ts

@@ -8,8 +8,8 @@ import type {
   SystemMount,
   SystemProcess,
 } from "@umt/contracts";
-import { type DB, type Kysely, sql } from "@umt/db";
-import { HEARTBEAT_STALE_MS } from "./heartbeat.ts";
+import { type DB, type Kysely, type Selectable, sql } from "@umt/db";
+import { CURRENT_HOST_WINDOW_MS, HEARTBEAT_ROW_RETENTION_MS, HEARTBEAT_STALE_MS } from "./heartbeat.ts";
 
 // GET /system/health 各段的取数(W9-I)。每段独立 try/catch:一段失败只把该段标 unknown,不让整页 500。
 // 慢 IO(statfs / 哨兵检查)都带超时:挂载卡死时只标 timeout,不拖住请求。
@@ -53,44 +53,84 @@ const num = (v: unknown) => (v == null ? null : Number(v));
 
 // ── 进程 ──
 
-export async function loadProcesses(db: Kysely<DB>, now: Date): Promise<SystemProcess[]> {
+type HeartbeatRow = Selectable<DB["process_heartbeat"]>;
+
+function toProcess(r: HeartbeatRow, now: Date): SystemProcess {
+  const beat = toDate(r.last_beat_at);
+  const age = Math.max(0, now.getTime() - beat.getTime());
+  const extra = (r.extra ?? {}) as Record<string, unknown>;
+  const conc = (r.concurrency ?? {}) as { io?: number; agent?: number };
+  const sse = extra.sseConnections;
+  return {
+    instanceId: r.instance_id,
+    processKind: r.process_kind,
+    pid: r.pid,
+    host: r.host,
+    version: r.version,
+    startedAt: toDate(r.started_at).toISOString(),
+    lastBeatAt: beat.toISOString(),
+    beatAgeSec: Math.round(age / 100) / 10,
+    stale: age > HEARTBEAT_STALE_MS,
+    stoppedAt: r.stopped_at ? toDate(r.stopped_at).toISOString() : null,
+    eventLoopLagMs: r.event_loop_lag_ms,
+    eventLoopLagMaxMs: r.event_loop_lag_max_ms,
+    concurrency: {
+      ...(typeof conc.io === "number" ? { io: conc.io } : {}),
+      ...(typeof conc.agent === "number" ? { agent: conc.agent } : {}),
+    },
+    runningRuns: r.running_runs,
+    sseConnections: typeof sse === "number" ? sse : null,
+    extra,
+  };
+}
+
+/**
+ * 把心跳行分成当前实例与已下线实例:
+ * - 当前 host = 最近 10 分钟内有心跳且没标 stopped_at 的行所在 host(容器重建后 host 会变);
+ * - 当前实例 = 当前 host 上、没标 stopped_at、且是同 host 同 kind 里最新启动的那一行;
+ * - 其余(已正常退出、host 已不在、被同 host 新进程替换)都算已下线实例,不参与 stale 判定与告警。
+ */
+export function splitProcesses(
+  ps: SystemProcess[],
+  now: Date,
+): { processes: SystemProcess[]; retiredProcesses: SystemProcess[] } {
+  const alive = ps.filter((p) => !p.stoppedAt);
+  const hosts = new Set(
+    alive.filter((p) => now.getTime() - Date.parse(p.lastBeatAt) <= CURRENT_HOST_WINDOW_MS).map((p) => p.host),
+  );
+  const newest = new Map<string, SystemProcess>();
+  for (const p of alive) {
+    if (!hosts.has(p.host)) continue;
+    const k = `${p.host}\u0000${p.processKind}`;
+    const cur = newest.get(k);
+    if (!cur || Date.parse(p.startedAt) > Date.parse(cur.startedAt)) newest.set(k, p);
+  }
+  const current = new Set([...newest.values()].map((p) => p.instanceId));
+  const processes = ps.filter((p) => current.has(p.instanceId));
+  const retiredProcesses = ps
+    .filter((p) => !current.has(p.instanceId))
+    .sort((a, b) => Date.parse(b.lastBeatAt) - Date.parse(a.lastBeatAt));
+  return { processes, retiredProcesses };
+}
+
+export async function loadProcesses(
+  db: Kysely<DB>,
+  now: Date,
+): Promise<{ processes: SystemProcess[]; retiredProcesses: SystemProcess[] }> {
   const rows = await db
     .selectFrom("process_heartbeat")
     .selectAll()
-    .where("last_beat_at", ">", new Date(now.getTime() - 24 * 3600_000))
+    .where("last_beat_at", ">", new Date(now.getTime() - HEARTBEAT_ROW_RETENTION_MS))
     .orderBy("process_kind")
     .orderBy("instance_id")
     .execute();
-  return rows.map((r) => {
-    const beat = toDate(r.last_beat_at);
-    const age = Math.max(0, now.getTime() - beat.getTime());
-    const extra = (r.extra ?? {}) as Record<string, unknown>;
-    const conc = (r.concurrency ?? {}) as { io?: number; agent?: number };
-    const sse = extra.sseConnections;
-    return {
-      instanceId: r.instance_id,
-      processKind: r.process_kind,
-      pid: r.pid,
-      host: r.host,
-      version: r.version,
-      startedAt: toDate(r.started_at).toISOString(),
-      lastBeatAt: beat.toISOString(),
-      beatAgeSec: Math.round(age / 100) / 10,
-      stale: age > HEARTBEAT_STALE_MS,
-      eventLoopLagMs: r.event_loop_lag_ms,
-      eventLoopLagMaxMs: r.event_loop_lag_max_ms,
-      concurrency: {
-        ...(typeof conc.io === "number" ? { io: conc.io } : {}),
-        ...(typeof conc.agent === "number" ? { agent: conc.agent } : {}),
-      },
-      runningRuns: r.running_runs,
-      sseConnections: typeof sse === "number" ? sse : null,
-      extra,
-    };
-  });
+  return splitProcesses(
+    rows.map((r) => toProcess(r, now)),
+    now,
+  );
 }
 
-/** 进程段的档位:有 stale 或延迟 ≥ 5s 为 critical;延迟 ≥ 1s 为 warn;没有任何 worker 心跳为 warn */
+/** 进程段的档位(只看当前实例):有 stale 或延迟 ≥ 1s 为 warn;延迟 ≥ 5s 为 critical;没有任何 worker 心跳为 warn */
 export function processLevel(ps: SystemProcess[]): HealthLevel {
   const live = ps.filter((p) => !p.stale);
   const levels: HealthLevel[] = [];
@@ -344,8 +384,11 @@ export async function loadSchedules(db: Kysely<DB>, now: Date): Promise<SystemHe
 
 export async function buildHealth(d: HealthDeps): Promise<SystemHealth> {
   const now = d.now?.() ?? new Date();
-  const [processes, db, paths, engine, schedules] = await Promise.all([
-    loadProcesses(d.db, now).catch(() => [] as SystemProcess[]),
+  const [{ processes, retiredProcesses }, db, paths, engine, schedules] = await Promise.all([
+    loadProcesses(d.db, now).catch(() => ({
+      processes: [] as SystemProcess[],
+      retiredProcesses: [] as SystemProcess[],
+    })),
     loadDb(d.db),
     loadDisksAndMounts(d).catch(() => ({ disks: [] as SystemDisk[], mounts: [] as SystemMount[] })),
     loadEngine(d.db, d.engineSchema, now).catch((): SystemHealth["engine"] => ({
@@ -388,6 +431,7 @@ export async function buildHealth(d: HealthDeps): Promise<SystemHealth> {
       uptimeSec: Math.max(0, Math.round((now.getTime() - d.self.startedAt.getTime()) / 1000)),
     },
     processes,
+    retiredProcesses,
     sse: { connections: sse },
     db,
     disks: paths.disks,