fix(system): retired heartbeat rows no longer alarm on status page

- GET /system/health splits processes into current instances (hosts with
  a beat in the last 10 min, newest per host+kind, not stopped) and
  retiredProcesses[]; 45s stale and level only consider current ones
- heartbeat writer marks stopped_at on graceful exit (migration 0043),
  upsert clears it; pure api role now stops its writer on SIGTERM
- scheduled job system.heartbeat_cleanup (maintain, 1h) deletes rows
  without a beat for 24h and samples older than 7 days
- status page: per-role alarms ("io worker 45 秒没有心跳"), collapsible
  retired instances; glossary/runbook updated
- bump all members to 0.2.2, CHANGELOG [0.2.2]

Co-Authored-By: Claude Opus 5.5 <[email protected]>

apps/server/test/system/system.test.ts

@@ -6,9 +6,17 @@ import { SystemHealth, SystemHealthHistory } from "@umt/contracts";
 import { sql } from "@umt/db";
 import { afterAll, beforeAll, describe, expect, it } from "vitest";
 import { inferTriggeredBy, triggeredByFromActor } from "../../src/modules/auth-settings/engine-business.ts";
-import { diskLevel, processLevel, worst } from "../../src/modules/system/health.ts";
-import { HEARTBEAT_STALE_MS, startHeartbeatWriter, writeHeartbeat } from "../../src/modules/system/heartbeat.ts";
+import { scheduleRegistry } from "../../src/modules/schedules/registry.ts";
+import { diskLevel, processLevel, splitProcesses, worst } from "../../src/modules/system/health.ts";
+import {
+  HEARTBEAT_STALE_MS,
+  markHeartbeatStopped,
+  pruneHeartbeats,
+  startHeartbeatWriter,
+  writeHeartbeat,
+} from "../../src/modules/system/heartbeat.ts";
 import { createSystemModule } from "../../src/modules/system/index.ts";
+import type { ServerContext } from "../../src/modules/types.ts";
 import { bearer, createHarness, type Harness, pgAvailable } from "../auth-settings/harness.ts";
 import { DEV_KEY, json } from "../helpers.ts";
 
@@ -39,6 +47,7 @@ describe("纯函数", () => {
         lastBeatAt: new Date().toISOString(),
         beatAgeSec: 0,
         stale: o.stale ?? false,
+        stoppedAt: null,
         eventLoopLagMs: 0,
         eventLoopLagMaxMs: o.lag ?? 0,
         concurrency: {},
@@ -53,6 +62,82 @@ describe("纯函数", () => {
     expect(processLevel([p("api"), p("worker", { lag: 6000 })])).toBe("critical");
   });
 
+  it("当前实例 / 已下线实例:host 10 分钟无心跳、已退出、被同 host 新进程替换的都算下线", () => {
+    const now = new Date("2026-09-28T12:00:00Z");
+    const ago = (sec: number) => new Date(now.getTime() - sec * 1000).toISOString();
+    const p = (id: string, kind: "api" | "worker" | "worker:agent", host: string, beatAgo: number, o = {}) => ({
+      instanceId: id,
+      processKind: kind,
+      pid: 1,
+      host,
+      version: "t",
+      startedAt: ago(beatAgo + 600),
+      lastBeatAt: ago(beatAgo),
+      beatAgeSec: beatAgo,
+      stale: beatAgo > 45,
+      stoppedAt: null as string | null,
+      eventLoopLagMs: 0,
+      eventLoopLagMaxMs: 0,
+      concurrency: {},
+      runningRuns: null,
+      sseConnections: null,
+      extra: {},
+      ...o,
+    });
+    const rows = [
+      // 当前容器 3 个进程,心跳正常
+      p("api:new-a:1", "api", "new-a", 5),
+      p("worker:new-w:1", "worker", "new-w", 5),
+      p("worker:agent:new-g:1", "worker:agent", "new-g", 5),
+      // 容器重建前的 3 行,最后心跳 40 分钟前
+      p("api:old-a:1", "api", "old-a", 2400),
+      p("worker:old-w:1", "worker", "old-w", 2400),
+      p("worker:agent:old-g:1", "worker:agent", "old-g", 2400),
+      // 同容器进程重启:旧 pid 行 2 分钟没心跳、启动更早
+      p("worker:new-w:0", "worker", "new-w", 120, { startedAt: ago(7200) }),
+      // 正常退出
+      p("api:new-a:0", "api", "new-a", 20, { startedAt: ago(9000), stoppedAt: ago(20) }),
+    ];
+    const r = splitProcesses(rows, now);
+    expect(r.processes.map((x) => x.instanceId).sort()).toEqual([
+      "api:new-a:1",
+      "worker:agent:new-g:1",
+      "worker:new-w:1",
+    ]);
+    expect(r.retiredProcesses.map((x) => x.instanceId)).toEqual([
+      "api:new-a:0",
+      "worker:new-w:0",
+      "api:old-a:1",
+      "worker:old-w:1",
+      "worker:agent:old-g:1",
+    ]);
+    expect(processLevel(r.processes)).toBe("ok");
+
+    // 当前容器里的 worker 停了心跳(1 分钟):仍是当前实例,stale 告警
+    const hung = splitProcesses([p("api:a:1", "api", "a", 5), p("worker:w:1", "worker", "w", 60)], now);
+    expect(hung.processes).toHaveLength(2);
+    expect(processLevel(hung.processes)).toBe("warn");
+    // 退出后立即不算活进程
+    const gone = splitProcesses(
+      [p("api:a:1", "api", "a", 5), p("worker:w:1", "worker", "w", 3, { stoppedAt: ago(1) })],
+      now,
+    );
+    expect(gone.processes.map((x) => x.instanceId)).toEqual(["api:a:1"]);
+    expect(gone.retiredProcesses.map((x) => x.instanceId)).toEqual(["worker:w:1"]);
+  });
+
+  it("system.heartbeat_cleanup:1 小时,维护组", async () => {
+    const ctx = {} as ServerContext;
+    await createSystemModule({ apiHeartbeat: false }).start?.(ctx);
+    expect(scheduleRegistry(ctx).get("system.heartbeat_cleanup")?.defaults).toMatchObject({
+      group: "maintain",
+      schedule: { type: "interval", everyMs: 3_600_000 },
+    });
+    const off = {} as ServerContext;
+    await createSystemModule({ apiHeartbeat: false, cleanupIntervalMs: 0 }).start?.(off);
+    expect(scheduleRegistry(off).get("system.heartbeat_cleanup")).toBeUndefined();
+  });
+
   it("触发来源推断", () => {
     expect(triggeredByFromActor("api_key:abc")).toBe("api_key");
     expect(triggeredByFromActor("admin_session:u1")).toBe("user");
@@ -156,10 +241,15 @@ describe.skipIf(!hasPg)("W9-I 进程与健康(真实 PG)", () => {
     });
     await w.beat();
     await w.beat();
+    expect(
+      (await h.db.selectFrom("process_heartbeat").select("stopped_at").executeTakeFirstOrThrow()).stopped_at,
+    ).toBeNull();
     await w.stop();
     expect(w.instanceId).toBe("worker:test-host:4242");
     const rows = await h.db.selectFrom("process_heartbeat").selectAll().execute();
     expect(rows).toHaveLength(1);
+    // 正常退出:标 stopped_at
+    expect(rows[0]!.stopped_at).toBeInstanceOf(Date);
     expect(rows[0]).toMatchObject({
       process_kind: "worker",
       pid: 4242,
@@ -190,8 +280,15 @@ describe.skipIf(!hasPg)("W9-I 进程与健康(真实 PG)", () => {
     expect(res.status).toBe(200);
     const d = SystemHealth.parse((await json(res)).data);
     const byId = new Map(d.processes.map((p) => [p.instanceId, p]));
-    expect(byId.get("worker:test-host:4242")).toMatchObject({ stale: false, runningRuns: 2, concurrency: { io: 4 } });
-    expect(byId.get("api:old-host:1")).toMatchObject({ stale: true, sseConnections: 3 });
+    // 已退出的 worker 立即归入已下线实例,不再当活进程
+    expect(byId.has("worker:test-host:4242")).toBe(false);
+    expect(d.retiredProcesses.find((p) => p.instanceId === "worker:test-host:4242")).toMatchObject({
+      runningRuns: 2,
+      concurrency: { io: 4 },
+      stoppedAt: expect.any(String),
+    });
+    // 60s 前的心跳,host 仍在 10 分钟窗口内 → 当前实例,stale
+    expect(byId.get("api:old-host:1")).toMatchObject({ stale: true, sseConnections: 3, stoppedAt: null });
     // stale 进程的 SSE 不计入
     expect(d.sse.connections).toBe(0);
   });
@@ -379,4 +476,61 @@ describe.skipIf(!hasPg)("W9-I 进程与健康(真实 PG)", () => {
     expect(detail.business).toMatchObject({ kind: "agent", refId: agentId });
     expect(detail.stepAttempts).toEqual([]);
   });
+  it("容器重建留下的旧心跳行不计入告警;24 小时的行由清理任务删除;重启后 upsert 清掉 stopped_at", async () => {
+    await sql`delete from process_heartbeat`.execute(h.db);
+    const now = Date.now();
+    const beat = (id: string, kind: "api" | "worker" | "worker:agent", host: string, agoMs: number) =>
+      writeHeartbeat(h.db, {
+        instanceId: id,
+        kind,
+        pid: 1,
+        host,
+        version: "0.2.0",
+        startedAt: new Date(now - agoMs - 3600_000),
+        lagMs: 0,
+        lagMaxMs: 0,
+        sample: {},
+        at: new Date(now - agoMs),
+      });
+    await beat("api:c-api:1", "api", "c-api", 2_000);
+    await beat("worker:c-io:1", "worker", "c-io", 2_000);
+    await beat("worker:agent:c-ag:1", "worker:agent", "c-ag", 2_000);
+    await beat("api:o-api:1", "api", "o-api", 40 * 60_000);
+    await beat("worker:o-io:1", "worker", "o-io", 40 * 60_000);
+    await beat("worker:agent:o-ag:1", "worker:agent", "o-ag", 40 * 60_000);
+    await beat("worker:ancient:1", "worker", "ancient", 25 * 3600_000);
+
+    const d = SystemHealth.parse((await json(await admin("GET", "/system/health"))).data);
+    expect(d.processes.map((p) => p.instanceId).sort()).toEqual([
+      "api:c-api:1",
+      "worker:agent:c-ag:1",
+      "worker:c-io:1",
+    ]);
+    expect(d.processes.every((p) => !p.stale)).toBe(true);
+    // 超过 24 小时的行不再返回
+    expect(d.retiredProcesses.map((p) => p.instanceId).sort()).toEqual([
+      "api:o-api:1",
+      "worker:agent:o-ag:1",
+      "worker:o-io:1",
+    ]);
+
+    // 清理任务:删 24 小时没心跳的行
+    const r = await pruneHeartbeats(h.db);
+    expect(r.rows).toBe(1);
+    const left = await h.db.selectFrom("process_heartbeat").select("instance_id").execute();
+    expect(left.map((x) => x.instance_id)).not.toContain("worker:ancient:1");
+    expect(left).toHaveLength(6);
+
+    // 退出标记后再次心跳(同 instance_id 重启):stopped_at 清回 null
+    await markHeartbeatStopped(h.db, "worker:c-io:1");
+    const d2 = SystemHealth.parse((await json(await admin("GET", "/system/health"))).data);
+    expect(d2.processes.map((p) => p.instanceId)).not.toContain("worker:c-io:1");
+    await beat("worker:c-io:1", "worker", "c-io", 0);
+    const row = await h.db
+      .selectFrom("process_heartbeat")
+      .select("stopped_at")
+      .where("instance_id", "=", "worker:c-io:1")
+      .executeTakeFirstOrThrow();
+    expect(row.stopped_at).toBeNull();
+  });
 });