feat(system): process heartbeat + /system/health, engine runs business fields (W9-I)

- process_heartbeat / process_heartbeat_sample (0034): api / worker / worker:agent / all upsert every 15s (event loop lag last/max, concurrency, running runs, SSE connections, memory) - GET /system/health (admin): processes with 45s stale flag, SSE, DB latency/size/connections, disks (DATA_DIR / library roots / download dirs, cache thresholds), mount sentinel checks with IO timeout, engine pending/oldest/24h failure rate, schedules - GET /system/health/history?hours=: bucketed lag / running / SSE series - /engine/runs and /engine/runs/{id}: business {kind, refId, title, triggeredBy} joined from run input / task / agent_run / tasks_run_meta / outbox; businessKind / businessRefId filters - docs/runbook/ops.md: field meanings and alert thresholds Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-09-28 15:23:37 +08:00
1 parent 46d37d87fe
commit e8745115f4
@@ -1654,6 +1654,21 @@ export const Api = {
      summary: "服务健康状态(免鉴权)",
      response: HealthStatus,
    }),
+    healthDetail: ep({
+      method: "get",
+      path: "/system/health",
+      scope: "admin",
+      summary: "进程与健康:进程心跳 / 事件循环延迟 / SSE 连接 / DB / 磁盘 / 挂载根 / 引擎 / 定时任务",
+      response: ops.SystemHealth,
+    }),
+    healthHistory: ep({
+      method: "get",
+      path: "/system/health/history",
+      scope: "admin",
+      summary: "心跳采样时间序列(事件循环延迟、运行中 run、SSE 连接;按桶聚合)",
+      query: ops.SystemHealthHistoryQuery,
+      response: ops.SystemHealthHistory,
+    }),
  },

  // ── 引擎面板(仅 admin) ──
@@ -225,6 +225,32 @@ export type EngineStepKind = z.infer<typeof EngineStepKind>;
 /** 引擎 ID 是 openworkflow 自己生成的文本 ID(非本地 UUID) */
 const EngineId = z.string().min(1).max(200);

+/** run 关联的业务对象(W9-I):由 run 输入 / 业务任务行 / agent_run / tasks_run_meta / outbox 事件推断,推断不出为 null */
+export const EngineBusinessKind = z.enum([
+  "subscription",
+  "download",
+  "organization",
+  "agent",
+  "schedule",
+  "feed",
+  "site",
+  "decision",
+  "media",
+]);
+export type EngineBusinessKind = z.infer<typeof EngineBusinessKind>;
+/** 触发来源:user=控制台会话 / 手动按钮 / 重跑;api_key=API Key 调用;schedule=定时任务;agent=Agent 或子会话;event=outbox 事件链 */
+export const EngineTriggeredBy = z.enum(["user", "api_key", "schedule", "agent", "event"]);
+export type EngineTriggeredBy = z.infer<typeof EngineTriggeredBy>;
+export const EngineRunBusiness = z.object({
+  kind: EngineBusinessKind,
+  /** 业务对象 id(schedule 为定时任务 key,如 trash.purge) */
+  refId: z.string(),
+  /** 展示名(订阅/作品标题、下载任务名、Feed/站点名、会话目标等);对象已删除时为 null */
+  title: z.string().nullable(),
+  triggeredBy: EngineTriggeredBy.nullable(),
+});
+export type EngineRunBusiness = z.infer<typeof EngineRunBusiness>;
+
 export const EngineRun = z.object({
  id: EngineId,
  workflowName: z.string(),
@@ -239,6 +265,8 @@ export const EngineRun = z.object({
  finishedAt: Timestamp.nullable(),
  createdAt: Timestamp,
  updatedAt: Timestamp,
+  /** 关联业务对象;推断不出为 null(旧客户端可忽略) */
+  business: EngineRunBusiness.nullable().optional(),
 });
 export type EngineRun = z.infer<typeof EngineRun>;

@@ -251,6 +279,10 @@ export const EngineRunQuery = z.object({
  from: Timestamp.optional(),
  /** created_at 上界(不含) */
  to: Timestamp.optional(),
+  /** 按关联业务对象类型筛选 */
+  businessKind: EngineBusinessKind.optional(),
+  /** 按关联业务对象 id 筛选(可与 businessKind 同用) */
+  businessRefId: z.string().min(1).max(200).optional(),
 });
 export type EngineRunQuery = z.infer<typeof EngineRunQuery>;

@@ -304,3 +336,146 @@ export type EngineStats = z.infer<typeof EngineStats>;

 export const EngineCancelRequest = z.object({ reason: z.string().max(500).optional() });
 export const EngineCancelResult = z.object({ canceled: z.boolean() });
+
+// ── 进程与健康(W9-I;GET /system/health、/system/health/history,仅 admin) ──
+
+export const ProcessKind = z.enum(["api", "worker", "worker:agent", "all"]);
+export type ProcessKind = z.infer<typeof ProcessKind>;
+/** 三档状态:ok / warn(需要关注)/ critical(需要处理);unknown = 取不到数据 */
+export const HealthLevel = z.enum(["ok", "warn", "critical", "unknown"]);
+export type HealthLevel = z.infer<typeof HealthLevel>;
+
+export const SystemProcess = z.object({
+  /** <kind>:<host>:<pid>;同一容器重启覆盖同一行 */
+  instanceId: z.string(),
+  processKind: ProcessKind,
+  pid: z.number().int(),
+  host: z.string(),
+  version: z.string(),
+  startedAt: Timestamp,
+  lastBeatAt: Timestamp,
+  /** 距上次心跳秒数 */
+  beatAgeSec: z.number().nonnegative(),
+  /** 超过 45s 没心跳 */
+  stale: z.boolean(),
+  /** 最近一次采样的事件循环延迟 */
+  eventLoopLagMs: NonNegInt,
+  /** 最近一个心跳周期(15s)内的最大延迟 */
+  eventLoopLagMaxMs: NonNegInt,
+  /** 引擎 worker 并发;api 进程为 {} */
+  concurrency: z.object({ io: NonNegInt.optional(), agent: NonNegInt.optional() }),
+  /** 本组任务正在执行的 run 数(同组多实例时是组合计);api 进程为 null */
+  runningRuns: NonNegInt.nullable(),
+  /** 本进程 SSE 长连接数;不托管 HTTP 的进程为 null */
+  sseConnections: NonNegInt.nullable(),
+  /** 其它采样:rssBytes/heapUsedBytes/lagWarnings/lifetimeMaxLagMs/group 等 */
+  extra: JsonObject,
+});
+export type SystemProcess = z.infer<typeof SystemProcess>;
+
+export const SystemDisk = z.object({
+  scope: z.enum(["data_dir", "library_root", "download_dir"]),
+  /** data_dir 为 null */
+  id: z.string().nullable(),
+  name: z.string(),
+  path: z.string(),
+  level: HealthLevel,
+  totalBytes: NonNegInt.nullable(),
+  usedBytes: NonNegInt.nullable(),
+  availableBytes: NonNegInt.nullable(),
+  usedRatio: z.number().min(0).max(1).nullable(),
+  /** 取不到用量时的错误码/说明 */
+  error: z.string().nullable(),
+});
+export type SystemDisk = z.infer<typeof SystemDisk>;
+
+export const SystemMount = z.object({
+  kind: z.enum(["library_root", "path_mapping"]),
+  id: z.string(),
+  name: z.string(),
+  path: z.string(),
+  /** online=目录与哨兵都在;offline=目录缺失/无权限/哨兵缺失;timeout=检查超时(挂载可能卡死) */
+  status: z.enum(["online", "offline", "timeout"]),
+  /** root_missing / sentinel_missing / permission_denied / error / timeout;online 为 null */
+  reason: z.string().nullable(),
+  /** 是否做了哨兵检查(下载目录映射没有哨兵,只查目录可读) */
+  sentinelChecked: z.boolean(),
+});
+export type SystemMount = z.infer<typeof SystemMount>;
+
+export const SystemHealth = z.object({
+  /** 各段最差档(unknown 不参与) */
+  level: HealthLevel,
+  checkedAt: Timestamp,
+  /** 处理本请求的 api 进程 */
+  api: z.object({ version: z.string(), startedAt: Timestamp, uptimeSec: z.number().nonnegative() }),
+  processes: z.array(SystemProcess),
+  sse: z.object({ connections: NonNegInt }),
+  db: z.object({
+    level: HealthLevel,
+    latencyMs: z.number().nonnegative().nullable(),
+    sizeBytes: NonNegInt.nullable(),
+    /** 本库连接数(pg_stat_activity),按状态分 */
+    connections: z
+      .object({ total: NonNegInt, active: NonNegInt, idle: NonNegInt, idleInTransaction: NonNegInt, max: NonNegInt })
+      .nullable(),
+    error: z.string().nullable(),
+  }),
+  disks: z.array(SystemDisk),
+  /** 磁盘阈值(取自设置 cache.resumePercent / cache.hardThresholdPercent) */
+  diskThresholds: z.object({ warnPercent: z.number(), criticalPercent: z.number() }),
+  mounts: z.array(SystemMount),
+  engine: z.object({
+    level: HealthLevel,
+    pending: NonNegInt,
+    running: NonNegInt,
+    oldestPendingAgeSec: z.number().nonnegative().nullable(),
+    finished24h: NonNegInt,
+    failed24h: NonNegInt,
+    /** failed / (completed + failed),24h 内无结束 run 为 null */
+    failureRate24h: z.number().min(0).max(1).nullable(),
+  }),
+  schedules: z.object({
+    level: HealthLevel,
+    total: NonNegInt,
+    enabled: NonNegInt,
+    running: NonNegInt,
+    /** 最近一次运行失败的已启用定时任务 */
+    failing: z.array(
+      z.object({ id: z.string(), name: z.string(), lastRunAt: Timestamp.nullable(), lastError: z.string().nullable() }),
+    ),
+    failedRuns24h: NonNegInt,
+  }),
+});
+export type SystemHealth = z.infer<typeof SystemHealth>;
+
+export const SystemHealthHistoryQuery = z.object({
+  /** 回看小时数,1–168,缺省 6 */
+  hours: z.coerce.number().int().min(1).max(168).default(6),
+});
+export type SystemHealthHistoryQuery = z.infer<typeof SystemHealthHistoryQuery>;
+
+export const SystemHealthHistory = z.object({
+  from: Timestamp,
+  to: Timestamp,
+  /** 聚合桶宽(秒);点数约 ≤ 360 */
+  bucketSec: z.number().int().positive(),
+  series: z.array(
+    z.object({
+      instanceId: z.string(),
+      processKind: z.string(),
+      points: z.array(
+        z.object({
+          at: Timestamp,
+          /** 桶内平均 */
+          lagMs: z.number().nonnegative(),
+          /** 桶内最大 */
+          lagMaxMs: z.number().nonnegative(),
+          runningRuns: z.number().nonnegative().nullable(),
+          sseConnections: z.number().nonnegative().nullable(),
+        }),
+      ),
+    }),
+  ),
+});
+export type SystemHealthHistory = z.infer<typeof SystemHealthHistory>;