fix(system): retired heartbeat rows no longer alarm on status page

- GET /system/health splits processes into current instances (hosts with
  a beat in the last 10 min, newest per host+kind, not stopped) and
  retiredProcesses[]; 45s stale and level only consider current ones
- heartbeat writer marks stopped_at on graceful exit (migration 0043),
  upsert clears it; pure api role now stops its writer on SIGTERM
- scheduled job system.heartbeat_cleanup (maintain, 1h) deletes rows
  without a beat for 24h and samples older than 7 days
- status page: per-role alarms ("io worker 45 秒没有心跳"), collapsible
  retired instances; glossary/runbook updated
- bump all members to 0.2.2, CHANGELOG [0.2.2]

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed 2026-09-28 23:31:21 +08:00
1 parent fa261eb6cd
commit b4c9b1ead2
32 files changed
+747 -91

No files matched your search

+29 -1
View File
@@ -4,9 +4,17 @@ import type { z } from "zod";
import { dataPaths } from "../../config.ts";
import { requireScope } from "../../http/auth.ts";
import { sseConnectionCount } from "../../http/sse.ts";
import { registerScheduledJob } from "../schedules/registry.ts";
import type { ModuleApp, ServerContext, ServerModule } from "../types.ts";
import { buildHealth, type HealthDeps, loadHistory } from "./health.ts";
import { type HeartbeatWriter, instanceIdOf, SERVER_VERSION, safeHostname, startHeartbeatWriter } from "./heartbeat.ts";
import {
type HeartbeatWriter,
instanceIdOf,
pruneHeartbeats,
SERVER_VERSION,
safeHostname,
startHeartbeatWriter,
} from "./heartbeat.ts";
export { HEARTBEAT_STALE_MS, HEARTBEAT_WRITE_MS, startHeartbeatWriter } from "./heartbeat.ts";
@@ -15,6 +23,7 @@ export { HEARTBEAT_STALE_MS, HEARTBEAT_WRITE_MS, startHeartbeatWriter } from "./
* - GET /system/health:进程心跳 / 事件循环延迟 / SSE 连接 / DB / 磁盘 / 挂载根 / 引擎 / 定时任务,仅 admin;
* - GET /system/health/history?hours=:心跳采样时间序列。
* 心跳写入:worker 进程在 worker/index.ts 里起写入器;纯 api 进程(role=api)在注册路由时起(role=all 由 worker 写 kind=all)。
* 清理:定时任务 system.heartbeat_cleanup(维护组,1 小时)删 24 小时没心跳的进程行与 7 天前的采样。
*/
export interface SystemModuleOptions {
/** 引擎 schema(测试覆盖);缺省 openworkflow */
@@ -23,6 +32,8 @@ export interface SystemModuleOptions {
healthDeps?: Partial<Pick<HealthDeps, "diskUsage" | "checkRoot" | "now" | "ioTimeoutMs" | "dataDir">>;
/** 缺省 true;false 时 role=api 也不起心跳写入器(测试) */
apiHeartbeat?: boolean;
/** 定时任务 system.heartbeat_cleanup 的默认周期,默认 1 小时;<=0 不登记 */
cleanupIntervalMs?: number;
}
type Infer<T> = T extends z.ZodType ? z.infer<T> : Record<string, never>;
@@ -93,6 +104,23 @@ export function createSystemModule(opts: SystemModuleOptions = {}): ServerModule
route(app, Api.system.healthHistory, async ({ query }) => await loadHistory(ctx.db, query.hours));
},
registerTasks() {},
start(ctx) {
const everyMs = opts.cleanupIntervalMs ?? 3600_000;
if (everyMs <= 0) return;
registerScheduledJob(ctx, {
key: "system.heartbeat_cleanup",
defaults: {
name: "进程心跳清理",
description: "删除 24 小时没有心跳的进程行和 7 天前的心跳采样",
group: "maintain",
schedule: { type: "interval", everyMs },
},
run: async ({ now }) => {
const r = await pruneHeartbeats(ctx.db, { now });
return { summary: `删除 ${r.rows} 个进程行、${r.samples} 条采样` };
},
});
},
async stop() {
await writer?.stop();
writer = null;