feat(system): process heartbeat + /system/health, engine runs business fields (W9-I)

- process_heartbeat / process_heartbeat_sample (0034): api / worker / worker:agent / all
  upsert every 15s (event loop lag last/max, concurrency, running runs, SSE connections, memory)
- GET /system/health (admin): processes with 45s stale flag, SSE, DB latency/size/connections,
  disks (DATA_DIR / library roots / download dirs, cache thresholds), mount sentinel checks
  with IO timeout, engine pending/oldest/24h failure rate, schedules
- GET /system/health/history?hours=: bucketed lag / running / SSE series
- /engine/runs and /engine/runs/{id}: business {kind, refId, title, triggeredBy} joined from
  run input / task / agent_run / tasks_run_meta / outbox; businessKind / businessRefId filters
- docs/runbook/ops.md: field meanings and alert thresholds

Co-Authored-By: Claude Opus 5.5 <[email protected]>

apps/server/src/modules/system/index.ts

@@ -0,0 +1,103 @@
+import { createRoute, type RouteConfig } from "@hono/zod-openapi";
+import { Api, apiResponse, DEFAULT_SETTINGS, type EndpointDef } from "@umt/contracts";
+import type { z } from "zod";
+import { dataPaths } from "../../config.ts";
+import { requireScope } from "../../http/auth.ts";
+import { sseConnectionCount } from "../../http/sse.ts";
+import type { ModuleApp, ServerContext, ServerModule } from "../types.ts";
+import { buildHealth, type HealthDeps, loadHistory } from "./health.ts";
+import { type HeartbeatWriter, instanceIdOf, SERVER_VERSION, safeHostname, startHeartbeatWriter } from "./heartbeat.ts";
+
+export { HEARTBEAT_STALE_MS, HEARTBEAT_WRITE_MS, startHeartbeatWriter } from "./heartbeat.ts";
+
+/**
+ * W9-I 进程与健康(docs/runbook/ops.md):
+ * - GET /system/health:进程心跳 / 事件循环延迟 / SSE 连接 / DB / 磁盘 / 挂载根 / 引擎 / 定时任务,仅 admin;
+ * - GET /system/health/history?hours=:心跳采样时间序列。
+ * 心跳写入:worker 进程在 worker/index.ts 里起写入器;纯 api 进程(role=api)在注册路由时起(role=all 由 worker 写 kind=all)。
+ */
+export interface SystemModuleOptions {
+  /** 引擎 schema(测试覆盖);缺省 openworkflow */
+  engineSchema?: string;
+  /** 测试注入:替换 health 取数依赖 */
+  healthDeps?: Partial<Pick<HealthDeps, "diskUsage" | "checkRoot" | "now" | "ioTimeoutMs" | "dataDir">>;
+  /** 缺省 true;false 时 role=api 也不起心跳写入器(测试) */
+  apiHeartbeat?: boolean;
+}
+
+type Infer<T> = T extends z.ZodType ? z.infer<T> : Record<string, never>;
+
+function route<D extends EndpointDef>(
+  app: ModuleApp,
+  def: D,
+  handler: (i: { query: Infer<D["query"]> }) => Promise<unknown>,
+) {
+  const request: RouteConfig["request"] = {};
+  if (def.query) request.query = def.query as never;
+  const r = createRoute({
+    method: def.method,
+    path: def.path,
+    summary: def.summary,
+    tags: ["system"],
+    middleware: [requireScope(def.scope)] as never,
+    request,
+    responses: {
+      200: { description: def.summary, content: { "application/json": { schema: apiResponse(def.response) } } },
+    },
+  });
+  // biome-ignore lint/suspicious/noExplicitAny: 路由泛型在封装里擦除,入参已由 zod-openapi 校验
+  app.openapi(r, (async (c: any) => {
+    const data = def.response.parse(await handler({ query: def.query ? c.req.valid("query") : {} } as never));
+    return c.json({ ok: true, data }, 200);
+    // biome-ignore lint/suspicious/noExplicitAny: 同上
+  }) as any);
+}
+
+export function createSystemModule(opts: SystemModuleOptions = {}): ServerModule & {
+  apiHeartbeat(): HeartbeatWriter | null;
+} {
+  const startedAt = new Date();
+  let writer: HeartbeatWriter | null = null;
+  return {
+    name: "system",
+    apiHeartbeat: () => writer,
+    registerRoutes(app, ctx: ServerContext) {
+      if (ctx.role === "api" && opts.apiHeartbeat !== false && !writer) {
+        writer = startHeartbeatWriter({
+          db: ctx.db,
+          logger: ctx.logger,
+          kind: "api",
+          startedAt,
+          sample: () => ({ sseConnections: sseConnectionCount() }),
+        });
+      }
+      const selfKind = ctx.role === "all" ? "all" : ctx.role === "api" ? "api" : null;
+      const selfId = selfKind ? instanceIdOf(selfKind, safeHostname(), Deno.pid) : undefined;
+
+      route(app, Api.system.healthDetail, async () => {
+        const cache = await ctx.settings.get("cache").catch(() => DEFAULT_SETTINGS.cache);
+        return await buildHealth({
+          db: ctx.db,
+          engineSchema: opts.engineSchema ?? "openworkflow",
+          dataDir: opts.healthDeps?.dataDir ?? dataPaths(ctx.config).dataDir,
+          self: {
+            version: SERVER_VERSION,
+            startedAt,
+            sseConnections: sseConnectionCount(),
+            ...(selfId ? { instanceId: selfId } : {}),
+          },
+          thresholds: { warnPercent: cache.resumePercent, criticalPercent: cache.hardThresholdPercent },
+          ...(opts.healthDeps ?? {}),
+        });
+      });
+      route(app, Api.system.healthHistory, async ({ query }) => await loadHistory(ctx.db, query.hours));
+    },
+    registerTasks() {},
+    async stop() {
+      await writer?.stop();
+      writer = null;
+    },
+  };
+}
+
+export const systemModule = createSystemModule();