feat(system): process heartbeat + /system/health, engine runs business fields (W9-I)

- process_heartbeat / process_heartbeat_sample (0034): api / worker / worker:agent / all
  upsert every 15s (event loop lag last/max, concurrency, running runs, SSE connections, memory)
- GET /system/health (admin): processes with 45s stale flag, SSE, DB latency/size/connections,
  disks (DATA_DIR / library roots / download dirs, cache thresholds), mount sentinel checks
  with IO timeout, engine pending/oldest/24h failure rate, schedules
- GET /system/health/history?hours=: bucketed lag / running / SSE series
- /engine/runs and /engine/runs/{id}: business {kind, refId, title, triggeredBy} joined from
  run input / task / agent_run / tasks_run_meta / outbox; businessKind / businessRefId filters
- docs/runbook/ops.md: field meanings and alert thresholds

Co-Authored-By: Claude Opus 5.5 <[email protected]>

packages/db/migrations/0034_process_heartbeat.ts

@@ -0,0 +1,54 @@
+import type { Kysely } from "kysely";
+import { dropTables, execDdl, oneOf } from "../src/migration-helpers.ts";
+
+// W9-I 进程与健康:各常驻进程(api / worker / worker:agent / all)每 15s upsert 一行心跳,
+// 同时追加一条采样到 process_heartbeat_sample 供 GET /system/health/history 画图(保留 7 天,写入方顺手裁剪)。
+// instance_id = <kind>:<host>:<pid>:同一容器重启覆盖旧行,不会越积越多;超过 45s 没心跳由读取方标 stale。
+// 另给 task.engine_run_id 补索引:引擎 runs 列表按业务字段筛选时按 run id 反查业务任务行。
+
+// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型
+export async function up(db: Kysely<any>): Promise<void> {
+  await execDdl(
+    db,
+    `
+create table process_heartbeat (
+  instance_id text primary key,
+  process_kind text not null ${oneOf("process_kind", ["api", "worker", "worker:agent", "all"])},
+  pid integer not null,
+  host text not null,
+  version text not null,
+  started_at timestamptz not null,
+  last_beat_at timestamptz not null default now(),
+  event_loop_lag_ms integer not null default 0 check (event_loop_lag_ms >= 0),
+  event_loop_lag_max_ms integer not null default 0 check (event_loop_lag_max_ms >= 0),
+  concurrency jsonb not null default '{}'::jsonb,
+  running_runs integer check (running_runs is null or running_runs >= 0),
+  extra jsonb not null default '{}'::jsonb
+);
+create index process_heartbeat_beat_idx on process_heartbeat (last_beat_at);
+
+create table process_heartbeat_sample (
+  id bigserial primary key,
+  instance_id text not null,
+  process_kind text not null,
+  at timestamptz not null default now(),
+  event_loop_lag_ms integer not null default 0,
+  event_loop_lag_max_ms integer not null default 0,
+  running_runs integer,
+  sse_connections integer
+);
+create index process_heartbeat_sample_at_idx on process_heartbeat_sample (at, instance_id);
+
+create index if not exists task_engine_run_idx on task (engine_run_id) where engine_run_id is not null;
+`,
+  );
+}
+
+// biome-ignore lint/suspicious/noExplicitAny: 同上
+export async function down(db: Kysely<any>): Promise<void> {
+  await execDdl(
+    db,
+    `drop index if exists task_engine_run_idx;
+${dropTables(["process_heartbeat_sample", "process_heartbeat"])}`,
+  );
+}

packages/db/src/types.ts

@@ -194,6 +194,7 @@ export type PolicyRevisionStatus = "draft" | "published" | "superseded" | "rolle
 export type PreferenceTagKind = "soft" | "hard" | "version" | "target";
 export type PreferenceTagOrigin = "manual" | "conversation" | "agent_inferred";
 export type PreferenceTagStatus = "active" | "disabled" | "deleted";
+export type ProcessHeartbeatProcessKind = "api" | "worker" | "worker:agent" | "all";
 export type ReconcileRunKind = "identity" | "metadata" | "asset" | "acquisition" | "server";
 export type ReconcileRunMode = "normal" | "read_only";
 export type ReconcileRunStatus = "queued" | "running" | "succeeded" | "partial" | "failed" | "cancelled";
@@ -1079,6 +1080,32 @@ export interface PreferenceTagTable {
   updated_at: GeneratedTimestamptz;
 }
 
+export interface ProcessHeartbeatTable {
+  instance_id: string;
+  process_kind: ProcessHeartbeatProcessKind;
+  pid: number;
+  host: string;
+  version: string;
+  started_at: Timestamptz;
+  last_beat_at: GeneratedTimestamptz;
+  event_loop_lag_ms: Generated<number>;
+  event_loop_lag_max_ms: Generated<number>;
+  concurrency: Generated<Json>;
+  running_runs: number | null;
+  extra: Generated<Json>;
+}
+
+export interface ProcessHeartbeatSampleTable {
+  id: Generated<number>;
+  instance_id: string;
+  process_kind: string;
+  at: GeneratedTimestamptz;
+  event_loop_lag_ms: Generated<number>;
+  event_loop_lag_max_ms: Generated<number>;
+  running_runs: number | null;
+  sse_connections: number | null;
+}
+
 export interface ReconcileRunTable {
   id: Generated<string>;
   kind: ReconcileRunKind;
@@ -1558,6 +1585,8 @@ export interface DB {
   policy_revision: PolicyRevisionTable;
   preference_suppression: PreferenceSuppressionTable;
   preference_tag: PreferenceTagTable;
+  process_heartbeat: ProcessHeartbeatTable;
+  process_heartbeat_sample: ProcessHeartbeatSampleTable;
   reconcile_run: ReconcileRunTable;
   resolution_attempt: ResolutionAttemptTable;
   rule_atom: RuleAtomTable;
@@ -1723,6 +1752,12 @@ export type PreferenceSuppressionUpdate = Updateable<PreferenceSuppressionTable>
 export type PreferenceTagRow = Selectable<PreferenceTagTable>;
 export type NewPreferenceTag = Insertable<PreferenceTagTable>;
 export type PreferenceTagUpdate = Updateable<PreferenceTagTable>;
+export type ProcessHeartbeatRow = Selectable<ProcessHeartbeatTable>;
+export type NewProcessHeartbeat = Insertable<ProcessHeartbeatTable>;
+export type ProcessHeartbeatUpdate = Updateable<ProcessHeartbeatTable>;
+export type ProcessHeartbeatSampleRow = Selectable<ProcessHeartbeatSampleTable>;
+export type NewProcessHeartbeatSample = Insertable<ProcessHeartbeatSampleTable>;
+export type ProcessHeartbeatSampleUpdate = Updateable<ProcessHeartbeatSampleTable>;
 export type ReconcileRunRow = Selectable<ReconcileRunTable>;
 export type NewReconcileRun = Insertable<ReconcileRunTable>;
 export type ReconcileRunUpdate = Updateable<ReconcileRunTable>;

packages/db/test/schema.test.ts

@@ -102,6 +102,9 @@ const EXPECTED_TABLES = [
   // W9-D 规则组(0033)
   "rule_group",
   "rule_atom",
+  // W9-I 进程心跳(0034)
+  "process_heartbeat",
+  "process_heartbeat_sample",
 ].sort();
 
 /** 不走 entityTable() 的表:追加型/引擎自有表,没有 uuid id + created_at/updated_at 三件套 */
@@ -118,6 +121,9 @@ const NON_ENTITY_TABLES = [
   // 稳定 key 作主键 / 只追加的运行历史
   "scheduled_job",
   "scheduled_job_run",
+  // 进程心跳:instance_id 主键 / 只追加的采样
+  "process_heartbeat",
+  "process_heartbeat_sample",
 ];
 
 describe.skipIf(skip)("schema migrations (real PG)", () => {