fix(system): retired heartbeat rows no longer alarm on status page
- GET /system/health splits processes into current instances (hosts with
a beat in the last 10 min, newest per host+kind, not stopped) and
retiredProcesses[]; 45s stale and level only consider current ones
- heartbeat writer marks stopped_at on graceful exit (migration 0043),
upsert clears it; pure api role now stops its writer on SIGTERM
- scheduled job system.heartbeat_cleanup (maintain, 1h) deletes rows
without a beat for 24h and samples older than 7 days
- status page: per-role alarms ("io worker 45 秒没有心跳"), collapsible
retired instances; glossary/runbook updated
- bump all members to 0.2.2, CHANGELOG [0.2.2]
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
fa261eb6cd
commit
b4c9b1ead2
32 files changed
+747
-91
No files matched your search
@@ -4,9 +4,17 @@ import type { z } from "zod";
|
||||
import { dataPaths } from "../../config.ts";
|
||||
import { requireScope } from "../../http/auth.ts";
|
||||
import { sseConnectionCount } from "../../http/sse.ts";
|
||||
import { registerScheduledJob } from "../schedules/registry.ts";
|
||||
import type { ModuleApp, ServerContext, ServerModule } from "../types.ts";
|
||||
import { buildHealth, type HealthDeps, loadHistory } from "./health.ts";
|
||||
import { type HeartbeatWriter, instanceIdOf, SERVER_VERSION, safeHostname, startHeartbeatWriter } from "./heartbeat.ts";
|
||||
import {
|
||||
type HeartbeatWriter,
|
||||
instanceIdOf,
|
||||
pruneHeartbeats,
|
||||
SERVER_VERSION,
|
||||
safeHostname,
|
||||
startHeartbeatWriter,
|
||||
} from "./heartbeat.ts";
|
||||
|
||||
export { HEARTBEAT_STALE_MS, HEARTBEAT_WRITE_MS, startHeartbeatWriter } from "./heartbeat.ts";
|
||||
|
||||
@@ -15,6 +23,7 @@ export { HEARTBEAT_STALE_MS, HEARTBEAT_WRITE_MS, startHeartbeatWriter } from "./
|
||||
* - GET /system/health:进程心跳 / 事件循环延迟 / SSE 连接 / DB / 磁盘 / 挂载根 / 引擎 / 定时任务,仅 admin;
|
||||
* - GET /system/health/history?hours=:心跳采样时间序列。
|
||||
* 心跳写入:worker 进程在 worker/index.ts 里起写入器;纯 api 进程(role=api)在注册路由时起(role=all 由 worker 写 kind=all)。
|
||||
* 清理:定时任务 system.heartbeat_cleanup(维护组,1 小时)删 24 小时没心跳的进程行与 7 天前的采样。
|
||||
*/
|
||||
export interface SystemModuleOptions {
|
||||
/** 引擎 schema(测试覆盖);缺省 openworkflow */
|
||||
@@ -23,6 +32,8 @@ export interface SystemModuleOptions {
|
||||
healthDeps?: Partial<Pick<HealthDeps, "diskUsage" | "checkRoot" | "now" | "ioTimeoutMs" | "dataDir">>;
|
||||
/** 缺省 true;false 时 role=api 也不起心跳写入器(测试) */
|
||||
apiHeartbeat?: boolean;
|
||||
/** 定时任务 system.heartbeat_cleanup 的默认周期,默认 1 小时;<=0 不登记 */
|
||||
cleanupIntervalMs?: number;
|
||||
}
|
||||
|
||||
type Infer<T> = T extends z.ZodType ? z.infer<T> : Record<string, never>;
|
||||
@@ -93,6 +104,23 @@ export function createSystemModule(opts: SystemModuleOptions = {}): ServerModule
|
||||
route(app, Api.system.healthHistory, async ({ query }) => await loadHistory(ctx.db, query.hours));
|
||||
},
|
||||
registerTasks() {},
|
||||
start(ctx) {
|
||||
const everyMs = opts.cleanupIntervalMs ?? 3600_000;
|
||||
if (everyMs <= 0) return;
|
||||
registerScheduledJob(ctx, {
|
||||
key: "system.heartbeat_cleanup",
|
||||
defaults: {
|
||||
name: "进程心跳清理",
|
||||
description: "删除 24 小时没有心跳的进程行和 7 天前的心跳采样",
|
||||
group: "maintain",
|
||||
schedule: { type: "interval", everyMs },
|
||||
},
|
||||
run: async ({ now }) => {
|
||||
const r = await pruneHeartbeats(ctx.db, { now });
|
||||
return { summary: `删除 ${r.rows} 个进程行、${r.samples} 条采样` };
|
||||
},
|
||||
});
|
||||
},
|
||||
async stop() {
|
||||
await writer?.stop();
|
||||
writer = null;
|
||||
|
||||
Reference in new issue
Block a user