feat(server): 资源监控——/proc 与 cgroup 采样、/system/resources 趋势，心跳带进程 CPU
Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
744252b969
commit
2ce1841628
6 files changed
+712
-2
No files matched your search
@@ -0,0 +1,206 @@
|
||||
import { SystemResources } from "@umt/contracts";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
bucketSamples,
|
||||
computePoint,
|
||||
createProcessCpuMeter,
|
||||
createResourceSampler,
|
||||
parseCgroupBytes,
|
||||
parseCgroupCpuMax,
|
||||
parseCgroupCpuUsage,
|
||||
parseDiskstats,
|
||||
parseIoStat,
|
||||
parseLoadavg,
|
||||
parseMeminfo,
|
||||
parseNetDev,
|
||||
parseProcStat,
|
||||
type ReadText,
|
||||
readResources,
|
||||
} from "../src/modules/system/resources.ts";
|
||||
|
||||
// 资源监控采集:/proc 与 cgroup v2 的解析、速率计算、降级与采样器(不碰 PG;读文件用注入的假实现)。
|
||||
|
||||
const STAT = (user: number, idle: number) => `cpu ${user} 0 100 ${idle} 50 0 0 0 0 0\ncpu0 1 2 3 4 5 6 7 8 0 0\n`;
|
||||
const MEMINFO = `MemTotal: 16000000 kB
|
||||
MemFree: 2000000 kB
|
||||
MemAvailable: 8000000 kB
|
||||
SwapTotal: 1000000 kB
|
||||
SwapFree: 600000 kB
|
||||
`;
|
||||
const NETDEV = (rx: number, tx: number) => `Inter-| Receive | Transmit
|
||||
face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed
|
||||
lo: 999999 10 0 0 0 0 0 0 999999 10 0 0 0 0 0 0
|
||||
eth0: ${rx} 100 0 0 0 0 0 0 ${tx} 80 0 0 0 0 0 0
|
||||
`;
|
||||
const IOSTAT = (r: number, w: number) =>
|
||||
`8:0 rbytes=${r} wbytes=${w} rios=10 wios=5 dbytes=0 dios=0\n259:0 rbytes=0 wbytes=0\n`;
|
||||
|
||||
/** 假文件系统:按路径给内容,没给的读不到 */
|
||||
function fakeFs(files: Record<string, string>): ReadText {
|
||||
return (path) => Promise.resolve(files[path] ?? null);
|
||||
}
|
||||
|
||||
describe("解析", () => {
|
||||
it("/proc 各文件", () => {
|
||||
expect(parseProcStat(STAT(1000, 8000))).toEqual({ total: 1000 + 100 + 8000 + 50, idle: 8050 });
|
||||
expect(parseProcStat("garbage")).toBeNull();
|
||||
expect(parseMeminfo(MEMINFO)).toEqual({
|
||||
totalBytes: 16000000 * 1024,
|
||||
availableBytes: 8000000 * 1024,
|
||||
swapUsedBytes: 400000 * 1024,
|
||||
});
|
||||
expect(parseMeminfo(null)).toBeNull();
|
||||
expect(parseLoadavg("0.52 0.40 0.31 2/345 6789")).toEqual([0.52, 0.4, 0.31]);
|
||||
expect(parseLoadavg("")).toBeNull();
|
||||
// lo 不计
|
||||
expect(parseNetDev(NETDEV(5000, 3000))).toEqual({ rx: 5000, tx: 3000 });
|
||||
// 整盘才计(分区、loop 不计)
|
||||
expect(
|
||||
parseDiskstats(
|
||||
" 8 0 sda 100 0 2000 0 50 0 4000 0 0 0 0\n 8 1 sda1 100 0 2000 0 50 0 4000 0 0 0 0\n 7 0 loop0 1 0 8 0 0 0 0 0 0 0 0\n",
|
||||
),
|
||||
).toEqual({ read: 2000 * 512, write: 4000 * 512 });
|
||||
});
|
||||
|
||||
it("cgroup v2", () => {
|
||||
expect(parseCgroupCpuUsage("usage_usec 123456\nuser_usec 1\n")).toBe(123456);
|
||||
expect(parseCgroupCpuMax("max 100000")).toBeNull();
|
||||
expect(parseCgroupCpuMax("200000 100000\n")).toBe(2);
|
||||
expect(parseCgroupBytes("max\n")).toBeNull();
|
||||
expect(parseCgroupBytes("1048576\n")).toBe(1048576);
|
||||
expect(parseIoStat(IOSTAT(4096, 8192))).toEqual({ read: 4096, write: 8192 });
|
||||
expect(parseIoStat("")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("读数与速率", () => {
|
||||
it("容器:CPU 按 cgroup 用量 / 核数,内存扣掉 inactive_file,磁盘读写取 io.stat", async () => {
|
||||
const files = (usage: number, user: number, idle: number, rx: number, io: number) =>
|
||||
fakeFs({
|
||||
"/proc/stat": STAT(user, idle),
|
||||
"/sys/fs/cgroup/cpu.stat": `usage_usec ${usage}\n`,
|
||||
"/sys/fs/cgroup/cpu.max": "200000 100000",
|
||||
"/proc/loadavg": "1.00 0.50 0.25 1/2 3",
|
||||
"/proc/meminfo": MEMINFO,
|
||||
"/sys/fs/cgroup/memory.current": String(600 * 1024 * 1024),
|
||||
"/sys/fs/cgroup/memory.max": String(1024 * 1024 * 1024),
|
||||
"/sys/fs/cgroup/memory.stat": `anon 1\ninactive_file ${100 * 1024 * 1024}\n`,
|
||||
"/proc/net/dev": NETDEV(rx, rx / 2),
|
||||
"/sys/fs/cgroup/io.stat": IOSTAT(io, io * 2),
|
||||
"/proc/uptime": "3600.5 7000.1",
|
||||
});
|
||||
const a = await readResources(files(0, 1000, 9000, 0, 0), 0);
|
||||
const b = await readResources(files(5_000_000, 2000, 10000, 10_000_000, 1_000_000), 5000);
|
||||
const first = computePoint(null, a);
|
||||
expect(first.sample.cpuPercent).toBeNull();
|
||||
expect(first.sample.netRxBps).toBeNull();
|
||||
const p = computePoint(a, b);
|
||||
// 5 秒用了 5 秒 CPU,限额 2 核 → 50%
|
||||
expect(p.current.cpu).toMatchObject({ scope: "cgroup", percent: 50, cores: 2, load: [1, 0.5, 0.25] });
|
||||
// 整机:total 增 2000、idle 增 1000 → 50%
|
||||
expect(p.current.cpu.hostPercent).toBe(50);
|
||||
expect(p.current.memory).toMatchObject({
|
||||
scope: "cgroup",
|
||||
usedBytes: 500 * 1024 * 1024,
|
||||
limitBytes: 1024 * 1024 * 1024,
|
||||
percent: 48.8,
|
||||
});
|
||||
expect(p.sample).toMatchObject({ netRxBps: 2_000_000, netTxBps: 1_000_000, diskReadBps: 200_000 });
|
||||
expect(p.current.diskIo).toMatchObject({ scope: "cgroup", writeBps: 400_000 });
|
||||
expect(p.current.hostUptimeSec).toBe(3600.5);
|
||||
});
|
||||
|
||||
it("宿主(没有 cgroup 文件):退回 /proc 整机口径", async () => {
|
||||
const files = (user: number, idle: number) =>
|
||||
fakeFs({ "/proc/stat": STAT(user, idle), "/proc/meminfo": MEMINFO, "/proc/net/dev": NETDEV(0, 0) });
|
||||
const p = computePoint(await readResources(files(1000, 9000), 0), await readResources(files(1500, 9500), 1000));
|
||||
expect(p.current.cpu).toMatchObject({ scope: "host", percent: 50 });
|
||||
expect(p.current.memory).toMatchObject({ scope: "host", usedBytes: 8000000 * 1024, percent: 50 });
|
||||
expect(p.current.diskIo).toMatchObject({ scope: "none", readBps: null });
|
||||
expect(p.current.network.scope).toBe("host");
|
||||
});
|
||||
|
||||
it("什么都读不到:全部为 null,不抛错", async () => {
|
||||
const none = fakeFs({});
|
||||
const p = computePoint(await readResources(none, 0), await readResources(none, 1000));
|
||||
expect(p.current.cpu).toMatchObject({ scope: "none", percent: null, hostPercent: null, load: null });
|
||||
expect(p.current.memory).toMatchObject({ scope: "none", usedBytes: null, percent: null });
|
||||
expect(p.current.network).toMatchObject({ scope: "none", rxBps: null });
|
||||
});
|
||||
|
||||
it("计数器回绕 / 重置时速率为 null,不出负数", async () => {
|
||||
const f = (rx: number) => fakeFs({ "/proc/net/dev": NETDEV(rx, rx) });
|
||||
const p = computePoint(await readResources(f(1000), 0), await readResources(f(10), 1000));
|
||||
expect(p.sample.netRxBps).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("采样器", () => {
|
||||
it("环形保留、按桶平均、快照符合契约", async () => {
|
||||
let now = Date.parse("2026-10-06T00:00:00Z");
|
||||
let usage = 0;
|
||||
const read: ReadText = (path) => {
|
||||
if (path === "/sys/fs/cgroup/cpu.stat") return Promise.resolve(`usage_usec ${usage}\n`);
|
||||
if (path === "/sys/fs/cgroup/cpu.max") return Promise.resolve("100000 100000");
|
||||
if (path === "/proc/meminfo") return Promise.resolve(MEMINFO);
|
||||
return Promise.resolve(null);
|
||||
};
|
||||
const s = createResourceSampler({ read, autoStart: false, intervalMs: 5000, retentionMs: 60_000, now: () => now });
|
||||
for (let i = 0; i < 30; i++) {
|
||||
// 每 5 秒用 2.5 秒 CPU(1 核)→ 50%
|
||||
usage += 2_500_000;
|
||||
now += 5000;
|
||||
await s.tick();
|
||||
}
|
||||
const snap = SystemResources.parse(await s.snapshot(5));
|
||||
// 保留 60 秒(含边界)→ 13 个点;第一次读数没有速率,不进趋势
|
||||
expect(snap.samples).toHaveLength(13);
|
||||
expect(snap.samples.every((x) => x.cpuPercent === 50)).toBe(true);
|
||||
expect(snap.cpu).toMatchObject({ scope: "cgroup", percent: 50, cores: 1 });
|
||||
expect(snap.intervalSec).toBe(5);
|
||||
s.stop();
|
||||
|
||||
// 长窗口:按桶取平均,点数不超过 360
|
||||
const many = Array.from({ length: 4000 }, (_, i) => ({
|
||||
at: new Date(Date.parse("2026-10-06T00:00:00Z") + i * 5000).toISOString(),
|
||||
cpuPercent: i % 2 ? 40 : 60,
|
||||
memoryUsedBytes: 100,
|
||||
memoryPercent: 10,
|
||||
netRxBps: null,
|
||||
netTxBps: null,
|
||||
diskReadBps: null,
|
||||
diskWriteBps: null,
|
||||
}));
|
||||
const b = bucketSamples(many, 5, 360);
|
||||
expect(b.bucketSec).toBe(60);
|
||||
expect(b.points.length).toBeLessThanOrEqual(360);
|
||||
expect(b.points[0]).toMatchObject({ cpuPercent: 50, memoryUsedBytes: 100, netRxBps: null });
|
||||
});
|
||||
|
||||
it("刚启动没有趋势时,快照现采两次给出当前值", async () => {
|
||||
let usage = 0;
|
||||
const read: ReadText = (path) => {
|
||||
if (path === "/sys/fs/cgroup/cpu.stat") {
|
||||
usage += 100_000;
|
||||
return Promise.resolve(`usage_usec ${usage}\n`);
|
||||
}
|
||||
return Promise.resolve(null);
|
||||
};
|
||||
const s = createResourceSampler({ read, autoStart: false });
|
||||
const snap = await s.snapshot(60);
|
||||
expect(snap.cpu.scope).toBe("cgroup");
|
||||
expect(snap.cpu.percent).not.toBeNull();
|
||||
expect(snap.samples.length).toBeGreaterThanOrEqual(1);
|
||||
});
|
||||
|
||||
it("进程 CPU:首次为 null,之后为非负数", () => {
|
||||
const m = createProcessCpuMeter();
|
||||
expect(m()).toBeNull();
|
||||
let x = 0;
|
||||
for (let i = 0; i < 1e5; i++) x += Math.sqrt(i);
|
||||
expect(x).toBeGreaterThan(0);
|
||||
const v = m();
|
||||
expect(v).not.toBeNull();
|
||||
expect(v!).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
});
|
||||
@@ -2,7 +2,7 @@ import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { DiskUsage } from "@umt/adapters";
|
||||
import { SystemHealth, SystemHealthHistory } from "@umt/contracts";
|
||||
import { SystemHealth, SystemHealthHistory, SystemResources } from "@umt/contracts";
|
||||
import { sql } from "@umt/db";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { inferTriggeredBy, triggeredByFromActor } from "../../src/modules/auth-settings/engine-business.ts";
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
writeHeartbeat,
|
||||
} from "../../src/modules/system/heartbeat.ts";
|
||||
import { createSystemModule } from "../../src/modules/system/index.ts";
|
||||
import { createResourceSampler } from "../../src/modules/system/resources.ts";
|
||||
import type { ServerContext } from "../../src/modules/types.ts";
|
||||
import { bearer, createHarness, type Harness, pgAvailable } from "../auth-settings/harness.ts";
|
||||
import { DEV_KEY, json } from "../helpers.ts";
|
||||
@@ -217,6 +218,16 @@ describe.skipIf(!hasPg)("W9-I 进程与健康(真实 PG)", () => {
|
||||
engineSchema: h.engineSchema,
|
||||
apiHeartbeat: false,
|
||||
healthDeps: { dataDir: dir, diskUsage: usage, ioTimeoutMs: 200 },
|
||||
// 资源采样:只给 meminfo 与 cgroup cpu.stat,其余读不到 → 降级为 null
|
||||
resourceSampler: createResourceSampler({
|
||||
autoStart: false,
|
||||
read: (path) => {
|
||||
if (path === "/proc/meminfo") return Promise.resolve("MemTotal: 1000 kB\nMemAvailable: 250 kB\n");
|
||||
if (path === "/sys/fs/cgroup/cpu.stat")
|
||||
return Promise.resolve(`usage_usec ${Math.round(performance.now() * 500)}\n`);
|
||||
return Promise.resolve(null);
|
||||
},
|
||||
}),
|
||||
}),
|
||||
],
|
||||
ctx: h.ctx,
|
||||
@@ -295,6 +306,43 @@ describe.skipIf(!hasPg)("W9-I 进程与健康(真实 PG)", () => {
|
||||
expect(d.sse.connections).toBe(0);
|
||||
});
|
||||
|
||||
it("资源监控:GET /system/resources 给当前值与趋势,读不到的字段为 null", async () => {
|
||||
const res = await admin("GET", "/system/resources?minutes=30");
|
||||
expect(res.status).toBe(200);
|
||||
const d = SystemResources.parse((await json(res)).data);
|
||||
expect(d.memory).toMatchObject({ scope: "host", usedBytes: 750 * 1024, percent: 75 });
|
||||
expect(d.cpu.scope).toBe("cgroup");
|
||||
expect(d.cpu.percent).not.toBeNull();
|
||||
expect(d.network).toMatchObject({ scope: "none", rxBps: null });
|
||||
expect(d.diskIo.scope).toBe("none");
|
||||
expect(d.samples.length).toBeGreaterThanOrEqual(1);
|
||||
expect((await admin("GET", "/system/resources?minutes=1")).status).toBe(422);
|
||||
});
|
||||
|
||||
it("心跳带上本进程 CPU 占用", async () => {
|
||||
const w = startHeartbeatWriter({
|
||||
db: h.db,
|
||||
logger: h.ctx.logger,
|
||||
kind: "worker:agent",
|
||||
host: "cpu-host",
|
||||
pid: 7,
|
||||
intervalMs: 60_000,
|
||||
lag: { take: () => ({ lastMs: 0, windowMaxMs: 0, lifetimeMaxMs: 0 }), stop() {} },
|
||||
});
|
||||
await w.beat();
|
||||
await new Promise((r) => setTimeout(r, 20));
|
||||
await w.beat();
|
||||
const row = await h.db
|
||||
.selectFrom("process_heartbeat")
|
||||
.select("extra")
|
||||
.where("instance_id", "=", w.instanceId)
|
||||
.executeTakeFirstOrThrow();
|
||||
expect(row.extra).toMatchObject({ cpuPercent: expect.any(Number), rssBytes: expect.any(Number) });
|
||||
await w.stop();
|
||||
await sql`delete from process_heartbeat where instance_id = ${w.instanceId}`.execute(h.db);
|
||||
await sql`delete from process_heartbeat_sample where instance_id = ${w.instanceId}`.execute(h.db);
|
||||
});
|
||||
|
||||
it("health 各段形状:DB / 磁盘阈值 / 挂载根哨兵 / 引擎 / 定时任务", async () => {
|
||||
await sql`insert into library_root (name, local_path, sentinel_file) values
|
||||
('电影', ${libOk}, '.umt-root'), ('剧集(掉线)', ${join(dir, "missing")}, null)`.execute(h.db);
|
||||
|
||||
Reference in new issue
Block a user