fix(system): retired heartbeat rows no longer alarm on status page

- GET /system/health splits processes into current instances (hosts with
  a beat in the last 10 min, newest per host+kind, not stopped) and
  retiredProcesses[]; 45s stale and level only consider current ones
- heartbeat writer marks stopped_at on graceful exit (migration 0043),
  upsert clears it; pure api role now stops its writer on SIGTERM
- scheduled job system.heartbeat_cleanup (maintain, 1h) deletes rows
  without a beat for 24h and samples older than 7 days
- status page: per-role alarms ("io worker 45 秒没有心跳"), collapsible
  retired instances; glossary/runbook updated
- bump all members to 0.2.2, CHANGELOG [0.2.2]

Co-Authored-By: Claude Opus 5.5 <[email protected]>

apps/server/src/modules/system/health.ts

@@ -8,8 +8,8 @@ import type {
   SystemMount,
   SystemProcess,
 } from "@umt/contracts";
-import { type DB, type Kysely, sql } from "@umt/db";
-import { HEARTBEAT_STALE_MS } from "./heartbeat.ts";
+import { type DB, type Kysely, type Selectable, sql } from "@umt/db";
+import { CURRENT_HOST_WINDOW_MS, HEARTBEAT_ROW_RETENTION_MS, HEARTBEAT_STALE_MS } from "./heartbeat.ts";
 
 // GET /system/health 各段的取数(W9-I)。每段独立 try/catch:一段失败只把该段标 unknown,不让整页 500。
 // 慢 IO(statfs / 哨兵检查)都带超时:挂载卡死时只标 timeout,不拖住请求。
@@ -53,44 +53,84 @@ const num = (v: unknown) => (v == null ? null : Number(v));
 
 // ── 进程 ──
 
-export async function loadProcesses(db: Kysely<DB>, now: Date): Promise<SystemProcess[]> {
+type HeartbeatRow = Selectable<DB["process_heartbeat"]>;
+
+function toProcess(r: HeartbeatRow, now: Date): SystemProcess {
+  const beat = toDate(r.last_beat_at);
+  const age = Math.max(0, now.getTime() - beat.getTime());
+  const extra = (r.extra ?? {}) as Record<string, unknown>;
+  const conc = (r.concurrency ?? {}) as { io?: number; agent?: number };
+  const sse = extra.sseConnections;
+  return {
+    instanceId: r.instance_id,
+    processKind: r.process_kind,
+    pid: r.pid,
+    host: r.host,
+    version: r.version,
+    startedAt: toDate(r.started_at).toISOString(),
+    lastBeatAt: beat.toISOString(),
+    beatAgeSec: Math.round(age / 100) / 10,
+    stale: age > HEARTBEAT_STALE_MS,
+    stoppedAt: r.stopped_at ? toDate(r.stopped_at).toISOString() : null,
+    eventLoopLagMs: r.event_loop_lag_ms,
+    eventLoopLagMaxMs: r.event_loop_lag_max_ms,
+    concurrency: {
+      ...(typeof conc.io === "number" ? { io: conc.io } : {}),
+      ...(typeof conc.agent === "number" ? { agent: conc.agent } : {}),
+    },
+    runningRuns: r.running_runs,
+    sseConnections: typeof sse === "number" ? sse : null,
+    extra,
+  };
+}
+
+/**
+ * 把心跳行分成当前实例与已下线实例:
+ * - 当前 host = 最近 10 分钟内有心跳且没标 stopped_at 的行所在 host(容器重建后 host 会变);
+ * - 当前实例 = 当前 host 上、没标 stopped_at、且是同 host 同 kind 里最新启动的那一行;
+ * - 其余(已正常退出、host 已不在、被同 host 新进程替换)都算已下线实例,不参与 stale 判定与告警。
+ */
+export function splitProcesses(
+  ps: SystemProcess[],
+  now: Date,
+): { processes: SystemProcess[]; retiredProcesses: SystemProcess[] } {
+  const alive = ps.filter((p) => !p.stoppedAt);
+  const hosts = new Set(
+    alive.filter((p) => now.getTime() - Date.parse(p.lastBeatAt) <= CURRENT_HOST_WINDOW_MS).map((p) => p.host),
+  );
+  const newest = new Map<string, SystemProcess>();
+  for (const p of alive) {
+    if (!hosts.has(p.host)) continue;
+    const k = `${p.host}\u0000${p.processKind}`;
+    const cur = newest.get(k);
+    if (!cur || Date.parse(p.startedAt) > Date.parse(cur.startedAt)) newest.set(k, p);
+  }
+  const current = new Set([...newest.values()].map((p) => p.instanceId));
+  const processes = ps.filter((p) => current.has(p.instanceId));
+  const retiredProcesses = ps
+    .filter((p) => !current.has(p.instanceId))
+    .sort((a, b) => Date.parse(b.lastBeatAt) - Date.parse(a.lastBeatAt));
+  return { processes, retiredProcesses };
+}
+
+export async function loadProcesses(
+  db: Kysely<DB>,
+  now: Date,
+): Promise<{ processes: SystemProcess[]; retiredProcesses: SystemProcess[] }> {
   const rows = await db
     .selectFrom("process_heartbeat")
     .selectAll()
-    .where("last_beat_at", ">", new Date(now.getTime() - 24 * 3600_000))
+    .where("last_beat_at", ">", new Date(now.getTime() - HEARTBEAT_ROW_RETENTION_MS))
     .orderBy("process_kind")
     .orderBy("instance_id")
     .execute();
-  return rows.map((r) => {
-    const beat = toDate(r.last_beat_at);
-    const age = Math.max(0, now.getTime() - beat.getTime());
-    const extra = (r.extra ?? {}) as Record<string, unknown>;
-    const conc = (r.concurrency ?? {}) as { io?: number; agent?: number };
-    const sse = extra.sseConnections;
-    return {
-      instanceId: r.instance_id,
-      processKind: r.process_kind,
-      pid: r.pid,
-      host: r.host,
-      version: r.version,
-      startedAt: toDate(r.started_at).toISOString(),
-      lastBeatAt: beat.toISOString(),
-      beatAgeSec: Math.round(age / 100) / 10,
-      stale: age > HEARTBEAT_STALE_MS,
-      eventLoopLagMs: r.event_loop_lag_ms,
-      eventLoopLagMaxMs: r.event_loop_lag_max_ms,
-      concurrency: {
-        ...(typeof conc.io === "number" ? { io: conc.io } : {}),
-        ...(typeof conc.agent === "number" ? { agent: conc.agent } : {}),
-      },
-      runningRuns: r.running_runs,
-      sseConnections: typeof sse === "number" ? sse : null,
-      extra,
-    };
-  });
+  return splitProcesses(
+    rows.map((r) => toProcess(r, now)),
+    now,
+  );
 }
 
-/** 进程段的档位:有 stale 或延迟 ≥ 5s 为 critical;延迟 ≥ 1s 为 warn;没有任何 worker 心跳为 warn */
+/** 进程段的档位(只看当前实例):有 stale 或延迟 ≥ 1s 为 warn;延迟 ≥ 5s 为 critical;没有任何 worker 心跳为 warn */
 export function processLevel(ps: SystemProcess[]): HealthLevel {
   const live = ps.filter((p) => !p.stale);
   const levels: HealthLevel[] = [];
@@ -344,8 +384,11 @@ export async function loadSchedules(db: Kysely<DB>, now: Date): Promise<SystemHe
 
 export async function buildHealth(d: HealthDeps): Promise<SystemHealth> {
   const now = d.now?.() ?? new Date();
-  const [processes, db, paths, engine, schedules] = await Promise.all([
-    loadProcesses(d.db, now).catch(() => [] as SystemProcess[]),
+  const [{ processes, retiredProcesses }, db, paths, engine, schedules] = await Promise.all([
+    loadProcesses(d.db, now).catch(() => ({
+      processes: [] as SystemProcess[],
+      retiredProcesses: [] as SystemProcess[],
+    })),
     loadDb(d.db),
     loadDisksAndMounts(d).catch(() => ({ disks: [] as SystemDisk[], mounts: [] as SystemMount[] })),
     loadEngine(d.db, d.engineSchema, now).catch((): SystemHealth["engine"] => ({
@@ -388,6 +431,7 @@ export async function buildHealth(d: HealthDeps): Promise<SystemHealth> {
       uptimeSec: Math.max(0, Math.round((now.getTime() - d.self.startedAt.getTime()) / 1000)),
     },
     processes,
+    retiredProcesses,
     sse: { connections: sse },
     db,
     disks: paths.disks,

apps/server/src/modules/system/heartbeat.ts

@@ -6,12 +6,17 @@ import serverDeno from "../../../deno.json" with { type: "json" };
 
 // 进程心跳写入(W9-I):每个常驻进程每 15s upsert process_heartbeat 一行 + 追加一条采样。
 // 写失败只记 debug(库暂时不可达不影响业务;读取方按 last_beat_at 判 stale)。
+// 正常退出(stop)时把本行标 stopped_at,读取方立即把它归入已下线实例;裁剪由定时任务 system.heartbeat_cleanup 负责。
 
 export const HEARTBEAT_WRITE_MS = 15_000;
 /** 超过这个时长没有心跳即 stale(3 个写周期) */
 export const HEARTBEAT_STALE_MS = 45_000;
 /** 采样保留天数 */
 export const SAMPLE_RETENTION_DAYS = 7;
+/** 进程行保留:超过这个时长没心跳的行删除 */
+export const HEARTBEAT_ROW_RETENTION_MS = 24 * 3600_000;
+/** 当前实例判定窗口:最近这段时间内有心跳的 host 视为当前在跑的容器 */
+export const CURRENT_HOST_WINDOW_MS = 10 * 60_000;
 
 export const SERVER_VERSION: string = serverDeno.version;
 
@@ -109,7 +114,7 @@ export async function writeHeartbeat(
       host = excluded.host, version = excluded.version, started_at = excluded.started_at,
       last_beat_at = excluded.last_beat_at, event_loop_lag_ms = excluded.event_loop_lag_ms,
       event_loop_lag_max_ms = excluded.event_loop_lag_max_ms, concurrency = excluded.concurrency,
-      running_runs = excluded.running_runs, extra = excluded.extra`.execute(db);
+      running_runs = excluded.running_runs, extra = excluded.extra, stopped_at = null`.execute(db);
   await sql`
     insert into process_heartbeat_sample (instance_id, process_kind, at, event_loop_lag_ms, event_loop_lag_max_ms,
       running_runs, sse_connections)
@@ -117,10 +122,22 @@ export async function writeHeartbeat(
       ${s.sseConnections ?? null})`.execute(db);
 }
 
-/** 裁剪过期采样与 7 天没心跳的进程行 */
-export async function pruneHeartbeats(db: Kysely<DB>, days = SAMPLE_RETENTION_DAYS): Promise<void> {
-  await sql`delete from process_heartbeat_sample where at < now() - make_interval(days => ${days})`.execute(db);
-  await sql`delete from process_heartbeat where last_beat_at < now() - make_interval(days => ${days})`.execute(db);
+/** 正常退出:把本实例的行标 stopped_at(之后读取方不再当活进程) */
+export async function markHeartbeatStopped(db: Kysely<DB>, instanceId: string, at = new Date()): Promise<void> {
+  await sql`update process_heartbeat set stopped_at = ${at} where instance_id = ${instanceId}`.execute(db);
+}
+
+/** 裁剪:超过 24 小时没心跳的进程行、超过 7 天的采样;返回删除行数 */
+export async function pruneHeartbeats(
+  db: Kysely<DB>,
+  o: { now?: Date; rowRetentionMs?: number; sampleDays?: number } = {},
+): Promise<{ rows: number; samples: number }> {
+  const now = o.now ?? new Date();
+  const rowCutoff = new Date(now.getTime() - (o.rowRetentionMs ?? HEARTBEAT_ROW_RETENTION_MS));
+  const sampleCutoff = new Date(now.getTime() - (o.sampleDays ?? SAMPLE_RETENTION_DAYS) * 24 * 3600_000);
+  const rows = await db.deleteFrom("process_heartbeat").where("last_beat_at", "<", rowCutoff).executeTakeFirst();
+  const samples = await db.deleteFrom("process_heartbeat_sample").where("at", "<", sampleCutoff).executeTakeFirst();
+  return { rows: Number(rows.numDeletedRows), samples: Number(samples.numDeletedRows) };
 }
 
 export interface HeartbeatWriter {
@@ -130,7 +147,7 @@ export interface HeartbeatWriter {
   stop(): Promise<void>;
 }
 
-/** 起一个周期写入器:立即写一次,之后每 intervalMs 写一次;每约 1 小时裁剪一次 */
+/** 起一个周期写入器:立即写一次,之后每 intervalMs 写一次;stop 时把本行标 stopped_at */
 export function startHeartbeatWriter(o: HeartbeatWriterOptions): HeartbeatWriter {
   const host = o.host ?? safeHostname();
   const pid = o.pid ?? Deno.pid;
@@ -140,8 +157,6 @@ export function startHeartbeatWriter(o: HeartbeatWriterOptions): HeartbeatWriter
   const intervalMs = o.intervalMs ?? HEARTBEAT_WRITE_MS;
   const lag = o.lag ?? createLagSampler();
   const log = o.logger.child({ component: "heartbeat", kind: o.kind });
-  const pruneEvery = Math.max(1, Math.round(3_600_000 / intervalMs));
-  let n = 0;
   let inflight: Promise<void> | null = null;
   let stopped = false;
 
@@ -165,8 +180,6 @@ export function startHeartbeatWriter(o: HeartbeatWriterOptions): HeartbeatWriter
         lagMaxMs: l.windowMaxMs,
         sample: { ...sample, extra: { ...memory(), lifetimeMaxLagMs: l.lifetimeMaxMs, ...(sample.extra ?? {}) } },
       });
-      if (n % pruneEvery === 0) await pruneHeartbeats(o.db);
-      n += 1;
     } catch (err) {
       log.debug({ err }, "心跳写入失败");
     }
@@ -194,6 +207,11 @@ export function startHeartbeatWriter(o: HeartbeatWriterOptions): HeartbeatWriter
       clearInterval(timer);
       lag.stop();
       await inflight?.catch(() => {});
+      try {
+        await markHeartbeatStopped(o.db, instanceId);
+      } catch (err) {
+        log.debug({ err }, "心跳退出标记失败");
+      }
     },
   };
 }

apps/server/src/modules/system/index.ts

@@ -4,9 +4,17 @@ import type { z } from "zod";
 import { dataPaths } from "../../config.ts";
 import { requireScope } from "../../http/auth.ts";
 import { sseConnectionCount } from "../../http/sse.ts";
+import { registerScheduledJob } from "../schedules/registry.ts";
 import type { ModuleApp, ServerContext, ServerModule } from "../types.ts";
 import { buildHealth, type HealthDeps, loadHistory } from "./health.ts";
-import { type HeartbeatWriter, instanceIdOf, SERVER_VERSION, safeHostname, startHeartbeatWriter } from "./heartbeat.ts";
+import {
+  type HeartbeatWriter,
+  instanceIdOf,
+  pruneHeartbeats,
+  SERVER_VERSION,
+  safeHostname,
+  startHeartbeatWriter,
+} from "./heartbeat.ts";
 
 export { HEARTBEAT_STALE_MS, HEARTBEAT_WRITE_MS, startHeartbeatWriter } from "./heartbeat.ts";
 
@@ -15,6 +23,7 @@ export { HEARTBEAT_STALE_MS, HEARTBEAT_WRITE_MS, startHeartbeatWriter } from "./
  * - GET /system/health:进程心跳 / 事件循环延迟 / SSE 连接 / DB / 磁盘 / 挂载根 / 引擎 / 定时任务,仅 admin;
  * - GET /system/health/history?hours=:心跳采样时间序列。
  * 心跳写入:worker 进程在 worker/index.ts 里起写入器;纯 api 进程(role=api)在注册路由时起(role=all 由 worker 写 kind=all)。
+ * 清理:定时任务 system.heartbeat_cleanup(维护组,1 小时)删 24 小时没心跳的进程行与 7 天前的采样。
  */
 export interface SystemModuleOptions {
   /** 引擎 schema(测试覆盖);缺省 openworkflow */
@@ -23,6 +32,8 @@ export interface SystemModuleOptions {
   healthDeps?: Partial<Pick<HealthDeps, "diskUsage" | "checkRoot" | "now" | "ioTimeoutMs" | "dataDir">>;
   /** 缺省 true;false 时 role=api 也不起心跳写入器(测试) */
   apiHeartbeat?: boolean;
+  /** 定时任务 system.heartbeat_cleanup 的默认周期,默认 1 小时;<=0 不登记 */
+  cleanupIntervalMs?: number;
 }
 
 type Infer<T> = T extends z.ZodType ? z.infer<T> : Record<string, never>;
@@ -93,6 +104,23 @@ export function createSystemModule(opts: SystemModuleOptions = {}): ServerModule
       route(app, Api.system.healthHistory, async ({ query }) => await loadHistory(ctx.db, query.hours));
     },
     registerTasks() {},
+    start(ctx) {
+      const everyMs = opts.cleanupIntervalMs ?? 3600_000;
+      if (everyMs <= 0) return;
+      registerScheduledJob(ctx, {
+        key: "system.heartbeat_cleanup",
+        defaults: {
+          name: "进程心跳清理",
+          description: "删除 24 小时没有心跳的进程行和 7 天前的心跳采样",
+          group: "maintain",
+          schedule: { type: "interval", everyMs },
+        },
+        run: async ({ now }) => {
+          const r = await pruneHeartbeats(ctx.db, { now });
+          return { summary: `删除 ${r.rows} 个进程行、${r.samples} 条采样` };
+        },
+      });
+    },
     async stop() {
       await writer?.stop();
       writer = null;