feat(system): process heartbeat + /system/health, engine runs business fields (W9-I)

- process_heartbeat / process_heartbeat_sample (0034): api / worker / worker:agent / all
  upsert every 15s (event loop lag last/max, concurrency, running runs, SSE connections, memory)
- GET /system/health (admin): processes with 45s stale flag, SSE, DB latency/size/connections,
  disks (DATA_DIR / library roots / download dirs, cache thresholds), mount sentinel checks
  with IO timeout, engine pending/oldest/24h failure rate, schedules
- GET /system/health/history?hours=: bucketed lag / running / SSE series
- /engine/runs and /engine/runs/{id}: business {kind, refId, title, triggeredBy} joined from
  run input / task / agent_run / tasks_run_meta / outbox; businessKind / businessRefId filters
- docs/runbook/ops.md: field meanings and alert thresholds

Co-Authored-By: Claude Opus 5.5 <[email protected]>

packages/contracts/src/api.ts

@@ -1654,6 +1654,21 @@ export const Api = {
       summary: "服务健康状态(免鉴权)",
       response: HealthStatus,
     }),
+    healthDetail: ep({
+      method: "get",
+      path: "/system/health",
+      scope: "admin",
+      summary: "进程与健康:进程心跳 / 事件循环延迟 / SSE 连接 / DB / 磁盘 / 挂载根 / 引擎 / 定时任务",
+      response: ops.SystemHealth,
+    }),
+    healthHistory: ep({
+      method: "get",
+      path: "/system/health/history",
+      scope: "admin",
+      summary: "心跳采样时间序列(事件循环延迟、运行中 run、SSE 连接;按桶聚合)",
+      query: ops.SystemHealthHistoryQuery,
+      response: ops.SystemHealthHistory,
+    }),
   },
 
   // ── 引擎面板(仅 admin) ──

packages/contracts/src/ops.ts

@@ -225,6 +225,32 @@ export type EngineStepKind = z.infer<typeof EngineStepKind>;
 /** 引擎 ID 是 openworkflow 自己生成的文本 ID(非本地 UUID) */
 const EngineId = z.string().min(1).max(200);
 
+/** run 关联的业务对象(W9-I):由 run 输入 / 业务任务行 / agent_run / tasks_run_meta / outbox 事件推断,推断不出为 null */
+export const EngineBusinessKind = z.enum([
+  "subscription",
+  "download",
+  "organization",
+  "agent",
+  "schedule",
+  "feed",
+  "site",
+  "decision",
+  "media",
+]);
+export type EngineBusinessKind = z.infer<typeof EngineBusinessKind>;
+/** 触发来源:user=控制台会话 / 手动按钮 / 重跑;api_key=API Key 调用;schedule=定时任务;agent=Agent 或子会话;event=outbox 事件链 */
+export const EngineTriggeredBy = z.enum(["user", "api_key", "schedule", "agent", "event"]);
+export type EngineTriggeredBy = z.infer<typeof EngineTriggeredBy>;
+export const EngineRunBusiness = z.object({
+  kind: EngineBusinessKind,
+  /** 业务对象 id(schedule 为定时任务 key,如 trash.purge) */
+  refId: z.string(),
+  /** 展示名(订阅/作品标题、下载任务名、Feed/站点名、会话目标等);对象已删除时为 null */
+  title: z.string().nullable(),
+  triggeredBy: EngineTriggeredBy.nullable(),
+});
+export type EngineRunBusiness = z.infer<typeof EngineRunBusiness>;
+
 export const EngineRun = z.object({
   id: EngineId,
   workflowName: z.string(),
@@ -239,6 +265,8 @@ export const EngineRun = z.object({
   finishedAt: Timestamp.nullable(),
   createdAt: Timestamp,
   updatedAt: Timestamp,
+  /** 关联业务对象;推断不出为 null(旧客户端可忽略) */
+  business: EngineRunBusiness.nullable().optional(),
 });
 export type EngineRun = z.infer<typeof EngineRun>;
 
@@ -251,6 +279,10 @@ export const EngineRunQuery = z.object({
   from: Timestamp.optional(),
   /** created_at 上界(不含) */
   to: Timestamp.optional(),
+  /** 按关联业务对象类型筛选 */
+  businessKind: EngineBusinessKind.optional(),
+  /** 按关联业务对象 id 筛选(可与 businessKind 同用) */
+  businessRefId: z.string().min(1).max(200).optional(),
 });
 export type EngineRunQuery = z.infer<typeof EngineRunQuery>;
 
@@ -304,3 +336,146 @@ export type EngineStats = z.infer<typeof EngineStats>;
 
 export const EngineCancelRequest = z.object({ reason: z.string().max(500).optional() });
 export const EngineCancelResult = z.object({ canceled: z.boolean() });
+
+// ── 进程与健康(W9-I;GET /system/health、/system/health/history,仅 admin) ──
+
+export const ProcessKind = z.enum(["api", "worker", "worker:agent", "all"]);
+export type ProcessKind = z.infer<typeof ProcessKind>;
+/** 三档状态:ok / warn(需要关注)/ critical(需要处理);unknown = 取不到数据 */
+export const HealthLevel = z.enum(["ok", "warn", "critical", "unknown"]);
+export type HealthLevel = z.infer<typeof HealthLevel>;
+
+export const SystemProcess = z.object({
+  /** <kind>:<host>:<pid>;同一容器重启覆盖同一行 */
+  instanceId: z.string(),
+  processKind: ProcessKind,
+  pid: z.number().int(),
+  host: z.string(),
+  version: z.string(),
+  startedAt: Timestamp,
+  lastBeatAt: Timestamp,
+  /** 距上次心跳秒数 */
+  beatAgeSec: z.number().nonnegative(),
+  /** 超过 45s 没心跳 */
+  stale: z.boolean(),
+  /** 最近一次采样的事件循环延迟 */
+  eventLoopLagMs: NonNegInt,
+  /** 最近一个心跳周期(15s)内的最大延迟 */
+  eventLoopLagMaxMs: NonNegInt,
+  /** 引擎 worker 并发;api 进程为 {} */
+  concurrency: z.object({ io: NonNegInt.optional(), agent: NonNegInt.optional() }),
+  /** 本组任务正在执行的 run 数(同组多实例时是组合计);api 进程为 null */
+  runningRuns: NonNegInt.nullable(),
+  /** 本进程 SSE 长连接数;不托管 HTTP 的进程为 null */
+  sseConnections: NonNegInt.nullable(),
+  /** 其它采样:rssBytes/heapUsedBytes/lagWarnings/lifetimeMaxLagMs/group 等 */
+  extra: JsonObject,
+});
+export type SystemProcess = z.infer<typeof SystemProcess>;
+
+export const SystemDisk = z.object({
+  scope: z.enum(["data_dir", "library_root", "download_dir"]),
+  /** data_dir 为 null */
+  id: z.string().nullable(),
+  name: z.string(),
+  path: z.string(),
+  level: HealthLevel,
+  totalBytes: NonNegInt.nullable(),
+  usedBytes: NonNegInt.nullable(),
+  availableBytes: NonNegInt.nullable(),
+  usedRatio: z.number().min(0).max(1).nullable(),
+  /** 取不到用量时的错误码/说明 */
+  error: z.string().nullable(),
+});
+export type SystemDisk = z.infer<typeof SystemDisk>;
+
+export const SystemMount = z.object({
+  kind: z.enum(["library_root", "path_mapping"]),
+  id: z.string(),
+  name: z.string(),
+  path: z.string(),
+  /** online=目录与哨兵都在;offline=目录缺失/无权限/哨兵缺失;timeout=检查超时(挂载可能卡死) */
+  status: z.enum(["online", "offline", "timeout"]),
+  /** root_missing / sentinel_missing / permission_denied / error / timeout;online 为 null */
+  reason: z.string().nullable(),
+  /** 是否做了哨兵检查(下载目录映射没有哨兵,只查目录可读) */
+  sentinelChecked: z.boolean(),
+});
+export type SystemMount = z.infer<typeof SystemMount>;
+
+export const SystemHealth = z.object({
+  /** 各段最差档(unknown 不参与) */
+  level: HealthLevel,
+  checkedAt: Timestamp,
+  /** 处理本请求的 api 进程 */
+  api: z.object({ version: z.string(), startedAt: Timestamp, uptimeSec: z.number().nonnegative() }),
+  processes: z.array(SystemProcess),
+  sse: z.object({ connections: NonNegInt }),
+  db: z.object({
+    level: HealthLevel,
+    latencyMs: z.number().nonnegative().nullable(),
+    sizeBytes: NonNegInt.nullable(),
+    /** 本库连接数(pg_stat_activity),按状态分 */
+    connections: z
+      .object({ total: NonNegInt, active: NonNegInt, idle: NonNegInt, idleInTransaction: NonNegInt, max: NonNegInt })
+      .nullable(),
+    error: z.string().nullable(),
+  }),
+  disks: z.array(SystemDisk),
+  /** 磁盘阈值(取自设置 cache.resumePercent / cache.hardThresholdPercent) */
+  diskThresholds: z.object({ warnPercent: z.number(), criticalPercent: z.number() }),
+  mounts: z.array(SystemMount),
+  engine: z.object({
+    level: HealthLevel,
+    pending: NonNegInt,
+    running: NonNegInt,
+    oldestPendingAgeSec: z.number().nonnegative().nullable(),
+    finished24h: NonNegInt,
+    failed24h: NonNegInt,
+    /** failed / (completed + failed),24h 内无结束 run 为 null */
+    failureRate24h: z.number().min(0).max(1).nullable(),
+  }),
+  schedules: z.object({
+    level: HealthLevel,
+    total: NonNegInt,
+    enabled: NonNegInt,
+    running: NonNegInt,
+    /** 最近一次运行失败的已启用定时任务 */
+    failing: z.array(
+      z.object({ id: z.string(), name: z.string(), lastRunAt: Timestamp.nullable(), lastError: z.string().nullable() }),
+    ),
+    failedRuns24h: NonNegInt,
+  }),
+});
+export type SystemHealth = z.infer<typeof SystemHealth>;
+
+export const SystemHealthHistoryQuery = z.object({
+  /** 回看小时数,1–168,缺省 6 */
+  hours: z.coerce.number().int().min(1).max(168).default(6),
+});
+export type SystemHealthHistoryQuery = z.infer<typeof SystemHealthHistoryQuery>;
+
+export const SystemHealthHistory = z.object({
+  from: Timestamp,
+  to: Timestamp,
+  /** 聚合桶宽(秒);点数约 ≤ 360 */
+  bucketSec: z.number().int().positive(),
+  series: z.array(
+    z.object({
+      instanceId: z.string(),
+      processKind: z.string(),
+      points: z.array(
+        z.object({
+          at: Timestamp,
+          /** 桶内平均 */
+          lagMs: z.number().nonnegative(),
+          /** 桶内最大 */
+          lagMaxMs: z.number().nonnegative(),
+          runningRuns: z.number().nonnegative().nullable(),
+          sseConnections: z.number().nonnegative().nullable(),
+        }),
+      ),
+    }),
+  ),
+});
+export type SystemHealthHistory = z.infer<typeof SystemHealthHistory>;

packages/contracts/test/db-sync.test.ts

@@ -127,6 +127,9 @@ const INTERNAL_TABLES = [
   "AdminSession",
   "LoginAttempt",
   "SseEvent",
+  // W9-I 进程心跳(0034):服务端内部,经 /system/health 的专用 DTO 暴露
+  "ProcessHeartbeat",
+  "ProcessHeartbeatSample",
 ];
 const INTERNAL_ENUMS = ["TasksOpLedgerStatus"];
 
@@ -209,6 +212,7 @@ const ENUMS: Record<string, z.ZodEnum> = {
   ReconcileRunMode: C.ReconcileMode,
   ReconcileRunStatus: C.ReconcileStatus,
   SecretPurpose: C.SecretPurpose,
+  ProcessHeartbeatProcessKind: C.ProcessKind,
   ScheduledJobBoardGroup: C.ScheduleGroup,
   ScheduledJobKind: C.ScheduleKind,
   ScheduledJobLastStatus: C.ScheduleRunStatus,

packages/db/migrations/0034_process_heartbeat.ts

@@ -0,0 +1,54 @@
+import type { Kysely } from "kysely";
+import { dropTables, execDdl, oneOf } from "../src/migration-helpers.ts";
+
+// W9-I 进程与健康:各常驻进程(api / worker / worker:agent / all)每 15s upsert 一行心跳,
+// 同时追加一条采样到 process_heartbeat_sample 供 GET /system/health/history 画图(保留 7 天,写入方顺手裁剪)。
+// instance_id = <kind>:<host>:<pid>:同一容器重启覆盖旧行,不会越积越多;超过 45s 没心跳由读取方标 stale。
+// 另给 task.engine_run_id 补索引:引擎 runs 列表按业务字段筛选时按 run id 反查业务任务行。
+
+// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型
+export async function up(db: Kysely<any>): Promise<void> {
+  await execDdl(
+    db,
+    `
+create table process_heartbeat (
+  instance_id text primary key,
+  process_kind text not null ${oneOf("process_kind", ["api", "worker", "worker:agent", "all"])},
+  pid integer not null,
+  host text not null,
+  version text not null,
+  started_at timestamptz not null,
+  last_beat_at timestamptz not null default now(),
+  event_loop_lag_ms integer not null default 0 check (event_loop_lag_ms >= 0),
+  event_loop_lag_max_ms integer not null default 0 check (event_loop_lag_max_ms >= 0),
+  concurrency jsonb not null default '{}'::jsonb,
+  running_runs integer check (running_runs is null or running_runs >= 0),
+  extra jsonb not null default '{}'::jsonb
+);
+create index process_heartbeat_beat_idx on process_heartbeat (last_beat_at);
+
+create table process_heartbeat_sample (
+  id bigserial primary key,
+  instance_id text not null,
+  process_kind text not null,
+  at timestamptz not null default now(),
+  event_loop_lag_ms integer not null default 0,
+  event_loop_lag_max_ms integer not null default 0,
+  running_runs integer,
+  sse_connections integer
+);
+create index process_heartbeat_sample_at_idx on process_heartbeat_sample (at, instance_id);
+
+create index if not exists task_engine_run_idx on task (engine_run_id) where engine_run_id is not null;
+`,
+  );
+}
+
+// biome-ignore lint/suspicious/noExplicitAny: 同上
+export async function down(db: Kysely<any>): Promise<void> {
+  await execDdl(
+    db,
+    `drop index if exists task_engine_run_idx;
+${dropTables(["process_heartbeat_sample", "process_heartbeat"])}`,
+  );
+}

packages/db/src/types.ts

@@ -194,6 +194,7 @@ export type PolicyRevisionStatus = "draft" | "published" | "superseded" | "rolle
 export type PreferenceTagKind = "soft" | "hard" | "version" | "target";
 export type PreferenceTagOrigin = "manual" | "conversation" | "agent_inferred";
 export type PreferenceTagStatus = "active" | "disabled" | "deleted";
+export type ProcessHeartbeatProcessKind = "api" | "worker" | "worker:agent" | "all";
 export type ReconcileRunKind = "identity" | "metadata" | "asset" | "acquisition" | "server";
 export type ReconcileRunMode = "normal" | "read_only";
 export type ReconcileRunStatus = "queued" | "running" | "succeeded" | "partial" | "failed" | "cancelled";
@@ -1079,6 +1080,32 @@ export interface PreferenceTagTable {
   updated_at: GeneratedTimestamptz;
 }
 
+export interface ProcessHeartbeatTable {
+  instance_id: string;
+  process_kind: ProcessHeartbeatProcessKind;
+  pid: number;
+  host: string;
+  version: string;
+  started_at: Timestamptz;
+  last_beat_at: GeneratedTimestamptz;
+  event_loop_lag_ms: Generated<number>;
+  event_loop_lag_max_ms: Generated<number>;
+  concurrency: Generated<Json>;
+  running_runs: number | null;
+  extra: Generated<Json>;
+}
+
+export interface ProcessHeartbeatSampleTable {
+  id: Generated<number>;
+  instance_id: string;
+  process_kind: string;
+  at: GeneratedTimestamptz;
+  event_loop_lag_ms: Generated<number>;
+  event_loop_lag_max_ms: Generated<number>;
+  running_runs: number | null;
+  sse_connections: number | null;
+}
+
 export interface ReconcileRunTable {
   id: Generated<string>;
   kind: ReconcileRunKind;
@@ -1558,6 +1585,8 @@ export interface DB {
   policy_revision: PolicyRevisionTable;
   preference_suppression: PreferenceSuppressionTable;
   preference_tag: PreferenceTagTable;
+  process_heartbeat: ProcessHeartbeatTable;
+  process_heartbeat_sample: ProcessHeartbeatSampleTable;
   reconcile_run: ReconcileRunTable;
   resolution_attempt: ResolutionAttemptTable;
   rule_atom: RuleAtomTable;
@@ -1723,6 +1752,12 @@ export type PreferenceSuppressionUpdate = Updateable<PreferenceSuppressionTable>
 export type PreferenceTagRow = Selectable<PreferenceTagTable>;
 export type NewPreferenceTag = Insertable<PreferenceTagTable>;
 export type PreferenceTagUpdate = Updateable<PreferenceTagTable>;
+export type ProcessHeartbeatRow = Selectable<ProcessHeartbeatTable>;
+export type NewProcessHeartbeat = Insertable<ProcessHeartbeatTable>;
+export type ProcessHeartbeatUpdate = Updateable<ProcessHeartbeatTable>;
+export type ProcessHeartbeatSampleRow = Selectable<ProcessHeartbeatSampleTable>;
+export type NewProcessHeartbeatSample = Insertable<ProcessHeartbeatSampleTable>;
+export type ProcessHeartbeatSampleUpdate = Updateable<ProcessHeartbeatSampleTable>;
 export type ReconcileRunRow = Selectable<ReconcileRunTable>;
 export type NewReconcileRun = Insertable<ReconcileRunTable>;
 export type ReconcileRunUpdate = Updateable<ReconcileRunTable>;

packages/db/test/schema.test.ts

@@ -102,6 +102,9 @@ const EXPECTED_TABLES = [
   // W9-D 规则组(0033)
   "rule_group",
   "rule_atom",
+  // W9-I 进程心跳(0034)
+  "process_heartbeat",
+  "process_heartbeat_sample",
 ].sort();
 
 /** 不走 entityTable() 的表:追加型/引擎自有表,没有 uuid id + created_at/updated_at 三件套 */
@@ -118,6 +121,9 @@ const NON_ENTITY_TABLES = [
   // 稳定 key 作主键 / 只追加的运行历史
   "scheduled_job",
   "scheduled_job_run",
+  // 进程心跳:instance_id 主键 / 只追加的采样
+  "process_heartbeat",
+  "process_heartbeat_sample",
 ];
 
 describe.skipIf(skip)("schema migrations (real PG)", () => {