fix(system): retired heartbeat rows no longer alarm on status page
- GET /system/health splits processes into current instances (hosts with
a beat in the last 10 min, newest per host+kind, not stopped) and
retiredProcesses[]; 45s stale and level only consider current ones
- heartbeat writer marks stopped_at on graceful exit (migration 0043),
upsert clears it; pure api role now stops its writer on SIGTERM
- scheduled job system.heartbeat_cleanup (maintain, 1h) deletes rows
without a beat for 24h and samples older than 7 days
- status page: per-role alarms ("io worker 45 秒没有心跳"), collapsible
retired instances; glossary/runbook updated
- bump all members to 0.2.2, CHANGELOG [0.2.2]
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
fa261eb6cd
commit
b4c9b1ead2
32 files changed
+747
-91
No files matched your search
@@ -1,6 +1,17 @@
|
||||
import { useQueryClient } from "@tanstack/react-query";
|
||||
import type { HealthLevel, SystemHealth, SystemProcess } from "@umt/contracts";
|
||||
import { AlertTriangle, CircleCheck, Cpu, Database, HardDrive, ListTodo, RotateCw, Timer, XCircle } from "lucide-react";
|
||||
import {
|
||||
AlertTriangle,
|
||||
ChevronDown,
|
||||
CircleCheck,
|
||||
Cpu,
|
||||
Database,
|
||||
HardDrive,
|
||||
ListTodo,
|
||||
RotateCw,
|
||||
Timer,
|
||||
XCircle,
|
||||
} from "lucide-react";
|
||||
import { type ReactNode, useState } from "react";
|
||||
import { Link } from "react-router";
|
||||
import { useSystemHealth, useSystemHealthHistory } from "~/api/system.ts";
|
||||
@@ -35,6 +46,14 @@ const KIND_LABEL: Record<string, string> = {
|
||||
all: "单进程(all)",
|
||||
};
|
||||
|
||||
/** 横幅告警里的进程称呼 */
|
||||
const KIND_NAME: Record<string, string> = {
|
||||
api: "API",
|
||||
worker: "io worker",
|
||||
"worker:agent": "Agent worker",
|
||||
all: "单进程",
|
||||
};
|
||||
|
||||
const MOUNT_REASON: Record<string, string> = {
|
||||
root_missing: "目录不存在",
|
||||
sentinel_missing: "哨兵文件缺失",
|
||||
@@ -74,8 +93,11 @@ function Card({
|
||||
/** 顶部横幅:整体档位 + 需要看的几条 */
|
||||
function problems(d: SystemHealth): string[] {
|
||||
const out: string[] = [];
|
||||
const stale = d.processes.filter((p) => p.stale);
|
||||
if (stale.length) out.push(`${stale.length} 个进程超过 45 秒没有心跳`);
|
||||
// 只看当前实例;已下线实例不告警
|
||||
for (const p of d.processes.filter((x) => x.stale)) {
|
||||
const line = `${KIND_NAME[p.processKind] ?? p.processKind} 45 秒没有心跳`;
|
||||
if (!out.includes(line)) out.push(line);
|
||||
}
|
||||
if (!d.processes.some((p) => !p.stale && p.processKind !== "api")) out.push("没有在线的 worker，任务不会被执行");
|
||||
const lag = d.processes.filter((p) => !p.stale && p.eventLoopLagMaxMs >= 1000);
|
||||
if (lag.length) out.push(`${lag.length} 个进程事件循环延迟超过 1 秒`);
|
||||
@@ -136,7 +158,7 @@ function Processes({ d }: { d: SystemHealth }) {
|
||||
<DataTable<SystemProcess>
|
||||
rows={d.processes}
|
||||
rowKey={(p) => p.instanceId}
|
||||
empty={{ title: "24 小时内没有进程心跳", description: "进程启动后每 15 秒写一次心跳。" }}
|
||||
empty={{ title: "没有在线的进程", description: "进程启动后每 15 秒写一次心跳。" }}
|
||||
columns={[
|
||||
{
|
||||
key: "kind",
|
||||
@@ -236,6 +258,77 @@ function Processes({ d }: { d: SystemHealth }) {
|
||||
);
|
||||
}
|
||||
|
||||
/** 已下线实例:已退出、容器已重建或被同容器新进程替换的旧心跳行,默认折叠,不参与告警 */
|
||||
function RetiredProcesses({ rows }: { rows: SystemProcess[] }) {
|
||||
const [open, setOpen] = useState(false);
|
||||
if (!rows.length) return null;
|
||||
return (
|
||||
<div className="mt-3">
|
||||
<button
|
||||
type="button"
|
||||
aria-expanded={open}
|
||||
onClick={() => setOpen((v) => !v)}
|
||||
className="flex cursor-pointer items-center gap-1.5 text-caption text-muted-foreground hover:text-foreground"
|
||||
>
|
||||
<ChevronDown className={cn("h-3.5 w-3.5 transition-transform", !open && "-rotate-90")} />
|
||||
<span>已下线实例 {rows.length} 个</span>
|
||||
<span className="text-meta">不计入告警，24 小时后清理</span>
|
||||
</button>
|
||||
{open ? (
|
||||
<div className="mt-2">
|
||||
<DataTable<SystemProcess>
|
||||
rows={rows}
|
||||
rowKey={(p) => p.instanceId}
|
||||
empty={{ title: "没有已下线实例" }}
|
||||
columns={[
|
||||
{
|
||||
key: "id",
|
||||
header: "实例",
|
||||
cell: (p) => (
|
||||
<div className="min-w-0">
|
||||
<p className="flex items-center gap-2 font-medium text-muted-foreground">
|
||||
{KIND_LABEL[p.processKind] ?? p.processKind}
|
||||
{p.stoppedAt ? <StatusBadge tone="neutral">已退出</StatusBadge> : null}
|
||||
</p>
|
||||
<p className="truncate font-mono text-meta text-muted-foreground" title={p.instanceId}>
|
||||
{p.instanceId}
|
||||
</p>
|
||||
</div>
|
||||
),
|
||||
},
|
||||
{
|
||||
key: "ver",
|
||||
header: "版本",
|
||||
hideBelow: "md",
|
||||
cell: (p) => <span className="font-mono text-meta">{p.version}</span>,
|
||||
},
|
||||
{
|
||||
key: "start",
|
||||
header: "启动",
|
||||
hideBelow: "md",
|
||||
cell: (p) => (
|
||||
<span className="whitespace-nowrap text-muted-foreground" title={formatDateTime(p.startedAt)}>
|
||||
{formatRelative(p.startedAt)}
|
||||
</span>
|
||||
),
|
||||
},
|
||||
{
|
||||
key: "beat",
|
||||
header: "最后心跳",
|
||||
cell: (p) => (
|
||||
<span className="whitespace-nowrap text-muted-foreground" title={formatDateTime(p.lastBeatAt)}>
|
||||
{formatRelative(p.lastBeatAt)}
|
||||
</span>
|
||||
),
|
||||
},
|
||||
]}
|
||||
/>
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function History() {
|
||||
const [hours, setHours] = useState(6);
|
||||
const h = useSystemHealthHistory(hours);
|
||||
@@ -397,6 +490,7 @@ export default function StatusPage() {
|
||||
<span className="text-meta font-normal text-muted-foreground">SSE 连接合计 {d.sse.connections}</span>
|
||||
</h2>
|
||||
<Processes d={d} />
|
||||
<RetiredProcesses rows={d.retiredProcesses} />
|
||||
</section>
|
||||
<History />
|
||||
<div className="grid gap-4 md:grid-cols-2 xl:grid-cols-3">
|
||||
|
||||
Reference in new issue
Block a user