feat(system): process heartbeat + /system/health, engine runs business fields (W9-I)
- process_heartbeat / process_heartbeat_sample (0034): api / worker / worker:agent / all
upsert every 15s (event loop lag last/max, concurrency, running runs, SSE connections, memory)
- GET /system/health (admin): processes with 45s stale flag, SSE, DB latency/size/connections,
disks (DATA_DIR / library roots / download dirs, cache thresholds), mount sentinel checks
with IO timeout, engine pending/oldest/24h failure rate, schedules
- GET /system/health/history?hours=: bucketed lag / running / SSE series
- /engine/runs and /engine/runs/{id}: business {kind, refId, title, triggeredBy} joined from
run input / task / agent_run / tasks_run_meta / outbox; businessKind / businessRefId filters
- docs/runbook/ops.md: field meanings and alert thresholds
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
46d37d87fe
commit
e8745115f4
17 files changed
+1789
-4
No files matched your search
@@ -2,7 +2,8 @@ import { EventType } from "@umt/contracts";
|
||||
import { OutboxRelay } from "@umt/tasks";
|
||||
import type { Logger } from "pino";
|
||||
import { collectOutboxRoutes, tasksLogSink } from "../context.ts";
|
||||
import type { EventBusPort } from "../http/sse.ts";
|
||||
import { type EventBusPort, sseConnectionCount } from "../http/sse.ts";
|
||||
import { type HeartbeatWriter, startHeartbeatWriter } from "../modules/system/heartbeat.ts";
|
||||
import type { ServerContext, ServerModule } from "../modules/types.ts";
|
||||
|
||||
/** agent 组任务:整段会话占一槽数分钟,与文件复制等 io 任务分池 */
|
||||
@@ -155,6 +156,34 @@ export async function startWorker({ logger, heartbeatMs, bus, runtime, lagProbe
|
||||
}
|
||||
}
|
||||
|
||||
// 进程与健康页:每 15s 把心跳/延迟/并发/运行中 run 数写进 process_heartbeat(写失败不影响 worker)
|
||||
let hbWriter: HeartbeatWriter | null = null;
|
||||
if (runtime) {
|
||||
const { ctx } = runtime;
|
||||
const groupTasks = new Set(plan.flatMap((p) => p.versions ?? []));
|
||||
const countAll = plan.some((p) => !p.versions);
|
||||
hbWriter = startHeartbeatWriter({
|
||||
db: ctx.db,
|
||||
logger,
|
||||
kind: group === "split" ? "all" : group === "agent" ? "worker:agent" : "worker",
|
||||
sample: async () => {
|
||||
const running = await ctx.engine.list({ status: "running", limit: 500 }).catch(() => null);
|
||||
const s = probe.stats();
|
||||
return {
|
||||
concurrency: {
|
||||
...(plan.some((p) => p.label !== "agent") ? { io: runtime.concurrency ?? 4 } : {}),
|
||||
...(plan.some((p) => p.label === "agent") ? { agent: runtime.agentConcurrency ?? 4 } : {}),
|
||||
},
|
||||
runningRuns: running
|
||||
? running.filter((r) => r.status === "running" && (countAll || groupTasks.has(r.taskName))).length
|
||||
: null,
|
||||
sseConnections: group === "split" ? sseConnectionCount() : null,
|
||||
extra: { group, lagWarnings: s.loopLagWarnings, probeMaxLagMs: s.maxLoopLagMs },
|
||||
};
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
log.info(
|
||||
{
|
||||
heartbeatMs,
|
||||
@@ -170,6 +199,7 @@ export async function startWorker({ logger, heartbeatMs, bus, runtime, lagProbe
|
||||
async stop() {
|
||||
clearInterval(timer);
|
||||
probe.stop();
|
||||
await hbWriter?.stop();
|
||||
for (const m of started.reverse()) {
|
||||
try {
|
||||
await m.stop?.();
|
||||
|
||||
Reference in new issue
Block a user