feat(system): process heartbeat + /system/health, engine runs business fields (W9-I)
- process_heartbeat / process_heartbeat_sample (0034): api / worker / worker:agent / all
upsert every 15s (event loop lag last/max, concurrency, running runs, SSE connections, memory)
- GET /system/health (admin): processes with 45s stale flag, SSE, DB latency/size/connections,
disks (DATA_DIR / library roots / download dirs, cache thresholds), mount sentinel checks
with IO timeout, engine pending/oldest/24h failure rate, schedules
- GET /system/health/history?hours=: bucketed lag / running / SSE series
- /engine/runs and /engine/runs/{id}: business {kind, refId, title, triggeredBy} joined from
run input / task / agent_run / tasks_run_meta / outbox; businessKind / businessRefId filters
- docs/runbook/ops.md: field meanings and alert thresholds
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
46d37d87fe
commit
e8745115f4
17 files changed
+1789
-4
No files matched your search
@@ -194,6 +194,7 @@ export type PolicyRevisionStatus = "draft" | "published" | "superseded" | "rolle
|
||||
export type PreferenceTagKind = "soft" | "hard" | "version" | "target";
|
||||
export type PreferenceTagOrigin = "manual" | "conversation" | "agent_inferred";
|
||||
export type PreferenceTagStatus = "active" | "disabled" | "deleted";
|
||||
export type ProcessHeartbeatProcessKind = "api" | "worker" | "worker:agent" | "all";
|
||||
export type ReconcileRunKind = "identity" | "metadata" | "asset" | "acquisition" | "server";
|
||||
export type ReconcileRunMode = "normal" | "read_only";
|
||||
export type ReconcileRunStatus = "queued" | "running" | "succeeded" | "partial" | "failed" | "cancelled";
|
||||
@@ -1079,6 +1080,32 @@ export interface PreferenceTagTable {
|
||||
updated_at: GeneratedTimestamptz;
|
||||
}
|
||||
|
||||
export interface ProcessHeartbeatTable {
|
||||
instance_id: string;
|
||||
process_kind: ProcessHeartbeatProcessKind;
|
||||
pid: number;
|
||||
host: string;
|
||||
version: string;
|
||||
started_at: Timestamptz;
|
||||
last_beat_at: GeneratedTimestamptz;
|
||||
event_loop_lag_ms: Generated<number>;
|
||||
event_loop_lag_max_ms: Generated<number>;
|
||||
concurrency: Generated<Json>;
|
||||
running_runs: number | null;
|
||||
extra: Generated<Json>;
|
||||
}
|
||||
|
||||
export interface ProcessHeartbeatSampleTable {
|
||||
id: Generated<number>;
|
||||
instance_id: string;
|
||||
process_kind: string;
|
||||
at: GeneratedTimestamptz;
|
||||
event_loop_lag_ms: Generated<number>;
|
||||
event_loop_lag_max_ms: Generated<number>;
|
||||
running_runs: number | null;
|
||||
sse_connections: number | null;
|
||||
}
|
||||
|
||||
export interface ReconcileRunTable {
|
||||
id: Generated<string>;
|
||||
kind: ReconcileRunKind;
|
||||
@@ -1558,6 +1585,8 @@ export interface DB {
|
||||
policy_revision: PolicyRevisionTable;
|
||||
preference_suppression: PreferenceSuppressionTable;
|
||||
preference_tag: PreferenceTagTable;
|
||||
process_heartbeat: ProcessHeartbeatTable;
|
||||
process_heartbeat_sample: ProcessHeartbeatSampleTable;
|
||||
reconcile_run: ReconcileRunTable;
|
||||
resolution_attempt: ResolutionAttemptTable;
|
||||
rule_atom: RuleAtomTable;
|
||||
@@ -1723,6 +1752,12 @@ export type PreferenceSuppressionUpdate = Updateable<PreferenceSuppressionTable>
|
||||
export type PreferenceTagRow = Selectable<PreferenceTagTable>;
|
||||
export type NewPreferenceTag = Insertable<PreferenceTagTable>;
|
||||
export type PreferenceTagUpdate = Updateable<PreferenceTagTable>;
|
||||
export type ProcessHeartbeatRow = Selectable<ProcessHeartbeatTable>;
|
||||
export type NewProcessHeartbeat = Insertable<ProcessHeartbeatTable>;
|
||||
export type ProcessHeartbeatUpdate = Updateable<ProcessHeartbeatTable>;
|
||||
export type ProcessHeartbeatSampleRow = Selectable<ProcessHeartbeatSampleTable>;
|
||||
export type NewProcessHeartbeatSample = Insertable<ProcessHeartbeatSampleTable>;
|
||||
export type ProcessHeartbeatSampleUpdate = Updateable<ProcessHeartbeatSampleTable>;
|
||||
export type ReconcileRunRow = Selectable<ReconcileRunTable>;
|
||||
export type NewReconcileRun = Insertable<ReconcileRunTable>;
|
||||
export type ReconcileRunUpdate = Updateable<ReconcileRunTable>;
|
||||
|
||||
Reference in new issue
Block a user