rootandClaude Opus 5.5
e8745115f4
- process_heartbeat / process_heartbeat_sample (0034): api / worker / worker:agent / all
upsert every 15s (event loop lag last/max, concurrency, running runs, SSE connections, memory)
- GET /system/health (admin): processes with 45s stale flag, SSE, DB latency/size/connections,
disks (DATA_DIR / library roots / download dirs, cache thresholds), mount sentinel checks
with IO timeout, engine pending/oldest/24h failure rate, schedules
- GET /system/health/history?hours=: bucketed lag / running / SSE series
- /engine/runs and /engine/runs/{id}: business {kind, refId, title, triggeredBy} joined from
run input / task / agent_run / tasks_run_meta / outbox; businessKind / businessRefId filters
- docs/runbook/ops.md: field meanings and alert thresholds
Co-Authored-By: Claude Opus 5.5 <[email protected]>
482 lines
17 KiB
TypeScript
482 lines
17 KiB
TypeScript
import { z } from "zod";
|
||
import { ActorType, ApiScope, EntityBase, Id, JsonObject, NonNegInt, SERVER_MANAGED, Timestamp } from "./common.ts";
|
||
|
||
// 运维域:校验运行、媒体服务器库存映射、API Key、审计、outbox、凭证(REQ §8.1、§7.4、§13.2)
|
||
|
||
export const ReconcileKind = z.enum(["identity", "metadata", "asset", "acquisition", "server"]);
|
||
export type ReconcileKind = z.infer<typeof ReconcileKind>;
|
||
/** read_only:恢复后默认只读对账模式(REQ §8.5) */
|
||
export const ReconcileMode = z.enum(["normal", "read_only"]);
|
||
export type ReconcileMode = z.infer<typeof ReconcileMode>;
|
||
export const ReconcileStatus = z.enum(["queued", "running", "succeeded", "partial", "failed", "cancelled"]);
|
||
export type ReconcileStatus = z.infer<typeof ReconcileStatus>;
|
||
|
||
export const ReconcileRun = EntityBase.extend({
|
||
kind: ReconcileKind,
|
||
mode: ReconcileMode,
|
||
status: ReconcileStatus,
|
||
scope: JsonObject,
|
||
/** 中断续扫位置 */
|
||
resumeCursor: JsonObject,
|
||
stats: JsonObject,
|
||
taskId: Id.nullable(),
|
||
error: JsonObject.nullable(),
|
||
startedAt: Timestamp.nullable(),
|
||
finishedAt: Timestamp.nullable(),
|
||
});
|
||
export type ReconcileRun = z.infer<typeof ReconcileRun>;
|
||
export const ReconcileRunCreate = ReconcileRun.pick({ kind: true, mode: true, scope: true }).partial({
|
||
mode: true,
|
||
scope: true,
|
||
});
|
||
export type ReconcileRunCreate = z.infer<typeof ReconcileRunCreate>;
|
||
export const ReconcileRunUpdate = ReconcileRun.pick({
|
||
status: true,
|
||
resumeCursor: true,
|
||
stats: true,
|
||
taskId: true,
|
||
error: true,
|
||
startedAt: true,
|
||
finishedAt: true,
|
||
}).partial();
|
||
export type ReconcileRunUpdate = z.infer<typeof ReconcileRunUpdate>;
|
||
|
||
export const ServerItemType = z.enum(["movie", "series", "season", "episode", "other"]);
|
||
export type ServerItemType = z.infer<typeof ServerItemType>;
|
||
/** 未知 ≠ 缺失:stale/unmatched 不能推出"没有资源" */
|
||
export const ServerMatchStatus = z.enum(["matched", "ambiguous", "unmatched", "stale"]);
|
||
export type ServerMatchStatus = z.infer<typeof ServerMatchStatus>;
|
||
|
||
/** Emby 兼容库存条目与本地对象的映射(辅助证据,不构成文件纳管授权) */
|
||
export const ServerMapping = EntityBase.extend({
|
||
mediaServerInstanceId: Id,
|
||
serverItemId: z.string().min(1),
|
||
itemType: ServerItemType,
|
||
mediaEntityId: Id.nullable(),
|
||
contentUnitId: Id.nullable(),
|
||
mediaAssetId: Id.nullable(),
|
||
serverPath: z.string().nullable(),
|
||
pathMappingId: Id.nullable(),
|
||
matchStatus: ServerMatchStatus,
|
||
versionInfo: JsonObject,
|
||
evidence: JsonObject,
|
||
observedAt: Timestamp,
|
||
});
|
||
export type ServerMapping = z.infer<typeof ServerMapping>;
|
||
export const ServerMappingCreate = ServerMapping.omit(SERVER_MANAGED).partial({
|
||
mediaEntityId: true,
|
||
contentUnitId: true,
|
||
mediaAssetId: true,
|
||
serverPath: true,
|
||
pathMappingId: true,
|
||
matchStatus: true,
|
||
versionInfo: true,
|
||
evidence: true,
|
||
observedAt: true,
|
||
});
|
||
export type ServerMappingCreate = z.infer<typeof ServerMappingCreate>;
|
||
export const ServerMappingUpdate = ServerMappingCreate.omit({
|
||
mediaServerInstanceId: true,
|
||
serverItemId: true,
|
||
}).partial();
|
||
export type ServerMappingUpdate = z.infer<typeof ServerMappingUpdate>;
|
||
|
||
/** API Key(不含 key_hash) */
|
||
export const ApiKey = EntityBase.extend({
|
||
name: z.string().min(1),
|
||
scopes: z.array(ApiScope),
|
||
revokedAt: Timestamp.nullable(),
|
||
lastUsedAt: Timestamp.nullable(),
|
||
});
|
||
export type ApiKey = z.infer<typeof ApiKey>;
|
||
export const ApiKeyCreate = z.object({
|
||
name: z.string().min(1).max(80),
|
||
scopes: z.array(ApiScope).min(1),
|
||
});
|
||
export type ApiKeyCreate = z.infer<typeof ApiKeyCreate>;
|
||
/** 创建响应:明文 key 只在此返回一次 */
|
||
export const ApiKeyCreated = z.object({ apiKey: ApiKey, plaintextKey: z.string().min(16) });
|
||
export type ApiKeyCreated = z.infer<typeof ApiKeyCreated>;
|
||
export const ApiKeyUpdate = z.object({
|
||
name: z.string().min(1).max(80).optional(),
|
||
scopes: z.array(ApiScope).min(1).optional(),
|
||
revoked: z.literal(true).optional(),
|
||
});
|
||
export type ApiKeyUpdate = z.infer<typeof ApiKeyUpdate>;
|
||
|
||
/** 审计事件(只追加) */
|
||
export const AuditEvent = z.object({
|
||
id: Id,
|
||
at: Timestamp,
|
||
actorType: z.string(),
|
||
actorId: z.string().nullable(),
|
||
action: z.string().min(1),
|
||
targetType: z.string().nullable(),
|
||
targetId: z.string().nullable(),
|
||
payload: JsonObject,
|
||
traceId: z.string().nullable(),
|
||
});
|
||
export type AuditEvent = z.infer<typeof AuditEvent>;
|
||
export const AuditEventCreate = AuditEvent.omit({ id: true, at: true })
|
||
.partial({ actorId: true, targetType: true, targetId: true, payload: true, traceId: true })
|
||
.extend({ actorType: ActorType });
|
||
export type AuditEventCreate = z.infer<typeof AuditEventCreate>;
|
||
|
||
export const OutboxStatus = z.enum(["pending", "dispatched", "failed", "dead"]);
|
||
export type OutboxStatus = z.infer<typeof OutboxStatus>;
|
||
/** outbox:业务事务内写入,分发器轮询 pending 并以 idempotencyKey 入队(ADR-002 §2.1) */
|
||
export const OutboxEvent = z.object({
|
||
id: Id,
|
||
at: Timestamp,
|
||
type: z.string().min(1),
|
||
payload: JsonObject,
|
||
publishedAt: Timestamp.nullable(),
|
||
idempotencyKey: z.string().nullable(),
|
||
status: OutboxStatus,
|
||
attempts: NonNegInt,
|
||
nextAttemptAt: Timestamp,
|
||
lastError: JsonObject.nullable(),
|
||
});
|
||
export type OutboxEvent = z.infer<typeof OutboxEvent>;
|
||
export const OutboxEventCreate = OutboxEvent.pick({
|
||
type: true,
|
||
payload: true,
|
||
idempotencyKey: true,
|
||
nextAttemptAt: true,
|
||
}).partial({
|
||
payload: true,
|
||
idempotencyKey: true,
|
||
nextAttemptAt: true,
|
||
});
|
||
export type OutboxEventCreate = z.infer<typeof OutboxEventCreate>;
|
||
export const OutboxEventUpdate = OutboxEvent.pick({
|
||
status: true,
|
||
attempts: true,
|
||
nextAttemptAt: true,
|
||
lastError: true,
|
||
publishedAt: true,
|
||
}).partial();
|
||
export type OutboxEventUpdate = z.infer<typeof OutboxEventUpdate>;
|
||
|
||
export const SecretPurpose = z.enum([
|
||
"site_cookie",
|
||
"site_passkey",
|
||
"site_api_key",
|
||
"site_password",
|
||
"feed_url",
|
||
"downloader_password",
|
||
"media_server_api_key",
|
||
"metadata_api_key",
|
||
"llm_api_key",
|
||
"other",
|
||
]);
|
||
export type SecretPurpose = z.infer<typeof SecretPurpose>;
|
||
|
||
/** 加密凭证的元数据视图:密文/nonce 永不出 API */
|
||
export const Secret = EntityBase.extend({
|
||
name: z.string().min(1),
|
||
purpose: SecretPurpose,
|
||
algorithm: z.string(),
|
||
kdf: z.string(),
|
||
kdfParams: JsonObject,
|
||
keyVersion: z.number().int().min(1),
|
||
rotatedAt: Timestamp.nullable(),
|
||
});
|
||
export type Secret = z.infer<typeof Secret>;
|
||
/** plaintext 只写;服务端用 SECRET_MASTER_KEY 派生密钥加密后存 ciphertext/nonce */
|
||
export const SecretCreate = z.object({
|
||
name: z.string().min(1).max(120),
|
||
purpose: SecretPurpose,
|
||
plaintext: z.string().min(1).max(16384),
|
||
});
|
||
export type SecretCreate = z.infer<typeof SecretCreate>;
|
||
export const SecretUpdate = z.object({ plaintext: z.string().min(1).max(16384) });
|
||
export type SecretUpdate = z.infer<typeof SecretUpdate>;
|
||
|
||
/** GET /audit-events 过滤:按 actor / action / target;游标分页(新 → 旧) */
|
||
export const AuditEventQuery = z.object({
|
||
cursor: z.string().min(1).optional(),
|
||
limit: z.coerce.number().int().min(1).max(200).default(50),
|
||
actorType: z.string().max(40).optional(),
|
||
actorId: z.string().max(200).optional(),
|
||
/** 精确匹配;以 `.*` 结尾时按前缀匹配(如 `admin.*`) */
|
||
action: z.string().max(80).optional(),
|
||
targetType: z.string().max(80).optional(),
|
||
targetId: z.string().max(200).optional(),
|
||
});
|
||
export type AuditEventQuery = z.infer<typeof AuditEventQuery>;
|
||
|
||
// ── 引擎面板(只读直查 openworkflow schema,仅 admin;docs/design/openworkflow-dashboard-reference.md) ──
|
||
|
||
/** 引擎 run 状态(含 legacy sleeping/succeeded) */
|
||
export const EngineRunStatus = z.enum([
|
||
"pending",
|
||
"running",
|
||
"sleeping",
|
||
"succeeded",
|
||
"completed",
|
||
"failed",
|
||
"canceled",
|
||
]);
|
||
export type EngineRunStatus = z.infer<typeof EngineRunStatus>;
|
||
export const EngineStepKind = z.enum(["function", "sleep", "workflow", "signal-send", "signal-wait"]);
|
||
export type EngineStepKind = z.infer<typeof EngineStepKind>;
|
||
|
||
/** 引擎 ID 是 openworkflow 自己生成的文本 ID(非本地 UUID) */
|
||
const EngineId = z.string().min(1).max(200);
|
||
|
||
/** run 关联的业务对象(W9-I):由 run 输入 / 业务任务行 / agent_run / tasks_run_meta / outbox 事件推断,推断不出为 null */
|
||
export const EngineBusinessKind = z.enum([
|
||
"subscription",
|
||
"download",
|
||
"organization",
|
||
"agent",
|
||
"schedule",
|
||
"feed",
|
||
"site",
|
||
"decision",
|
||
"media",
|
||
]);
|
||
export type EngineBusinessKind = z.infer<typeof EngineBusinessKind>;
|
||
/** 触发来源:user=控制台会话 / 手动按钮 / 重跑;api_key=API Key 调用;schedule=定时任务;agent=Agent 或子会话;event=outbox 事件链 */
|
||
export const EngineTriggeredBy = z.enum(["user", "api_key", "schedule", "agent", "event"]);
|
||
export type EngineTriggeredBy = z.infer<typeof EngineTriggeredBy>;
|
||
export const EngineRunBusiness = z.object({
|
||
kind: EngineBusinessKind,
|
||
/** 业务对象 id(schedule 为定时任务 key,如 trash.purge) */
|
||
refId: z.string(),
|
||
/** 展示名(订阅/作品标题、下载任务名、Feed/站点名、会话目标等);对象已删除时为 null */
|
||
title: z.string().nullable(),
|
||
triggeredBy: EngineTriggeredBy.nullable(),
|
||
});
|
||
export type EngineRunBusiness = z.infer<typeof EngineRunBusiness>;
|
||
|
||
export const EngineRun = z.object({
|
||
id: EngineId,
|
||
workflowName: z.string(),
|
||
version: z.string().nullable(),
|
||
status: z.string(),
|
||
attempts: z.number().int(),
|
||
workerId: z.string().nullable(),
|
||
parentStepAttemptId: z.string().nullable(),
|
||
availableAt: Timestamp.nullable(),
|
||
deadlineAt: Timestamp.nullable(),
|
||
startedAt: Timestamp.nullable(),
|
||
finishedAt: Timestamp.nullable(),
|
||
createdAt: Timestamp,
|
||
updatedAt: Timestamp,
|
||
/** 关联业务对象;推断不出为 null(旧客户端可忽略) */
|
||
business: EngineRunBusiness.nullable().optional(),
|
||
});
|
||
export type EngineRun = z.infer<typeof EngineRun>;
|
||
|
||
export const EngineRunQuery = z.object({
|
||
cursor: z.string().min(1).optional(),
|
||
limit: z.coerce.number().int().min(1).max(200).default(50),
|
||
status: EngineRunStatus.optional(),
|
||
workflowName: z.string().max(200).optional(),
|
||
/** created_at 下界(含) */
|
||
from: Timestamp.optional(),
|
||
/** created_at 上界(不含) */
|
||
to: Timestamp.optional(),
|
||
/** 按关联业务对象类型筛选 */
|
||
businessKind: EngineBusinessKind.optional(),
|
||
/** 按关联业务对象 id 筛选(可与 businessKind 同用) */
|
||
businessRefId: z.string().min(1).max(200).optional(),
|
||
});
|
||
export type EngineRunQuery = z.infer<typeof EngineRunQuery>;
|
||
|
||
/** step attempt:不含 config;context 只露 signal-wait 的信号名与超时(排查卡等待用) */
|
||
export const EngineStepAttempt = z.object({
|
||
id: EngineId,
|
||
stepName: z.string(),
|
||
kind: z.string(),
|
||
status: z.string(),
|
||
output: z.unknown(),
|
||
error: z.unknown(),
|
||
waitSignal: z.string().nullable(),
|
||
timeoutAt: z.string().nullable(),
|
||
childWorkflowRunId: z.string().nullable(),
|
||
startedAt: Timestamp.nullable(),
|
||
finishedAt: Timestamp.nullable(),
|
||
createdAt: Timestamp,
|
||
});
|
||
export type EngineStepAttempt = z.infer<typeof EngineStepAttempt>;
|
||
|
||
export const EngineSignal = z.object({
|
||
id: EngineId,
|
||
signal: z.string(),
|
||
data: z.unknown(),
|
||
stepAttemptId: z.string(),
|
||
createdAt: Timestamp,
|
||
});
|
||
export type EngineSignal = z.infer<typeof EngineSignal>;
|
||
|
||
/** run 详情:input/output/error 经敏感字段脱敏;不返回 config/context */
|
||
export const EngineRunDetail = EngineRun.extend({
|
||
input: z.unknown(),
|
||
output: z.unknown(),
|
||
error: z.unknown(),
|
||
stepAttempts: z.array(EngineStepAttempt),
|
||
signals: z.array(EngineSignal),
|
||
});
|
||
export type EngineRunDetail = z.infer<typeof EngineRunDetail>;
|
||
|
||
export const EngineStats = z.object({
|
||
/** 全表按状态计数(不受列表筛选影响) */
|
||
byStatus: z.record(z.string(), z.number().int().nonnegative()),
|
||
/** 最老的待处理 run(pending,按 available_at/created_at) */
|
||
oldestPending: z
|
||
.object({ id: EngineId, workflowName: z.string(), since: Timestamp, ageSec: z.number().nonnegative() })
|
||
.nullable(),
|
||
/** 当前持有租约的 worker 快照(非权威注册表:引擎没有 worker 表) */
|
||
activeWorkers: z.array(z.object({ workerId: z.string(), running: z.number().int(), since: Timestamp.nullable() })),
|
||
});
|
||
export type EngineStats = z.infer<typeof EngineStats>;
|
||
|
||
export const EngineCancelRequest = z.object({ reason: z.string().max(500).optional() });
|
||
export const EngineCancelResult = z.object({ canceled: z.boolean() });
|
||
|
||
// ── 进程与健康(W9-I;GET /system/health、/system/health/history,仅 admin) ──
|
||
|
||
export const ProcessKind = z.enum(["api", "worker", "worker:agent", "all"]);
|
||
export type ProcessKind = z.infer<typeof ProcessKind>;
|
||
/** 三档状态:ok / warn(需要关注)/ critical(需要处理);unknown = 取不到数据 */
|
||
export const HealthLevel = z.enum(["ok", "warn", "critical", "unknown"]);
|
||
export type HealthLevel = z.infer<typeof HealthLevel>;
|
||
|
||
export const SystemProcess = z.object({
|
||
/** <kind>:<host>:<pid>;同一容器重启覆盖同一行 */
|
||
instanceId: z.string(),
|
||
processKind: ProcessKind,
|
||
pid: z.number().int(),
|
||
host: z.string(),
|
||
version: z.string(),
|
||
startedAt: Timestamp,
|
||
lastBeatAt: Timestamp,
|
||
/** 距上次心跳秒数 */
|
||
beatAgeSec: z.number().nonnegative(),
|
||
/** 超过 45s 没心跳 */
|
||
stale: z.boolean(),
|
||
/** 最近一次采样的事件循环延迟 */
|
||
eventLoopLagMs: NonNegInt,
|
||
/** 最近一个心跳周期(15s)内的最大延迟 */
|
||
eventLoopLagMaxMs: NonNegInt,
|
||
/** 引擎 worker 并发;api 进程为 {} */
|
||
concurrency: z.object({ io: NonNegInt.optional(), agent: NonNegInt.optional() }),
|
||
/** 本组任务正在执行的 run 数(同组多实例时是组合计);api 进程为 null */
|
||
runningRuns: NonNegInt.nullable(),
|
||
/** 本进程 SSE 长连接数;不托管 HTTP 的进程为 null */
|
||
sseConnections: NonNegInt.nullable(),
|
||
/** 其它采样:rssBytes/heapUsedBytes/lagWarnings/lifetimeMaxLagMs/group 等 */
|
||
extra: JsonObject,
|
||
});
|
||
export type SystemProcess = z.infer<typeof SystemProcess>;
|
||
|
||
export const SystemDisk = z.object({
|
||
scope: z.enum(["data_dir", "library_root", "download_dir"]),
|
||
/** data_dir 为 null */
|
||
id: z.string().nullable(),
|
||
name: z.string(),
|
||
path: z.string(),
|
||
level: HealthLevel,
|
||
totalBytes: NonNegInt.nullable(),
|
||
usedBytes: NonNegInt.nullable(),
|
||
availableBytes: NonNegInt.nullable(),
|
||
usedRatio: z.number().min(0).max(1).nullable(),
|
||
/** 取不到用量时的错误码/说明 */
|
||
error: z.string().nullable(),
|
||
});
|
||
export type SystemDisk = z.infer<typeof SystemDisk>;
|
||
|
||
export const SystemMount = z.object({
|
||
kind: z.enum(["library_root", "path_mapping"]),
|
||
id: z.string(),
|
||
name: z.string(),
|
||
path: z.string(),
|
||
/** online=目录与哨兵都在;offline=目录缺失/无权限/哨兵缺失;timeout=检查超时(挂载可能卡死) */
|
||
status: z.enum(["online", "offline", "timeout"]),
|
||
/** root_missing / sentinel_missing / permission_denied / error / timeout;online 为 null */
|
||
reason: z.string().nullable(),
|
||
/** 是否做了哨兵检查(下载目录映射没有哨兵,只查目录可读) */
|
||
sentinelChecked: z.boolean(),
|
||
});
|
||
export type SystemMount = z.infer<typeof SystemMount>;
|
||
|
||
export const SystemHealth = z.object({
|
||
/** 各段最差档(unknown 不参与) */
|
||
level: HealthLevel,
|
||
checkedAt: Timestamp,
|
||
/** 处理本请求的 api 进程 */
|
||
api: z.object({ version: z.string(), startedAt: Timestamp, uptimeSec: z.number().nonnegative() }),
|
||
processes: z.array(SystemProcess),
|
||
sse: z.object({ connections: NonNegInt }),
|
||
db: z.object({
|
||
level: HealthLevel,
|
||
latencyMs: z.number().nonnegative().nullable(),
|
||
sizeBytes: NonNegInt.nullable(),
|
||
/** 本库连接数(pg_stat_activity),按状态分 */
|
||
connections: z
|
||
.object({ total: NonNegInt, active: NonNegInt, idle: NonNegInt, idleInTransaction: NonNegInt, max: NonNegInt })
|
||
.nullable(),
|
||
error: z.string().nullable(),
|
||
}),
|
||
disks: z.array(SystemDisk),
|
||
/** 磁盘阈值(取自设置 cache.resumePercent / cache.hardThresholdPercent) */
|
||
diskThresholds: z.object({ warnPercent: z.number(), criticalPercent: z.number() }),
|
||
mounts: z.array(SystemMount),
|
||
engine: z.object({
|
||
level: HealthLevel,
|
||
pending: NonNegInt,
|
||
running: NonNegInt,
|
||
oldestPendingAgeSec: z.number().nonnegative().nullable(),
|
||
finished24h: NonNegInt,
|
||
failed24h: NonNegInt,
|
||
/** failed / (completed + failed),24h 内无结束 run 为 null */
|
||
failureRate24h: z.number().min(0).max(1).nullable(),
|
||
}),
|
||
schedules: z.object({
|
||
level: HealthLevel,
|
||
total: NonNegInt,
|
||
enabled: NonNegInt,
|
||
running: NonNegInt,
|
||
/** 最近一次运行失败的已启用定时任务 */
|
||
failing: z.array(
|
||
z.object({ id: z.string(), name: z.string(), lastRunAt: Timestamp.nullable(), lastError: z.string().nullable() }),
|
||
),
|
||
failedRuns24h: NonNegInt,
|
||
}),
|
||
});
|
||
export type SystemHealth = z.infer<typeof SystemHealth>;
|
||
|
||
export const SystemHealthHistoryQuery = z.object({
|
||
/** 回看小时数,1–168,缺省 6 */
|
||
hours: z.coerce.number().int().min(1).max(168).default(6),
|
||
});
|
||
export type SystemHealthHistoryQuery = z.infer<typeof SystemHealthHistoryQuery>;
|
||
|
||
export const SystemHealthHistory = z.object({
|
||
from: Timestamp,
|
||
to: Timestamp,
|
||
/** 聚合桶宽(秒);点数约 ≤ 360 */
|
||
bucketSec: z.number().int().positive(),
|
||
series: z.array(
|
||
z.object({
|
||
instanceId: z.string(),
|
||
processKind: z.string(),
|
||
points: z.array(
|
||
z.object({
|
||
at: Timestamp,
|
||
/** 桶内平均 */
|
||
lagMs: z.number().nonnegative(),
|
||
/** 桶内最大 */
|
||
lagMaxMs: z.number().nonnegative(),
|
||
runningRuns: z.number().nonnegative().nullable(),
|
||
sseConnections: z.number().nonnegative().nullable(),
|
||
}),
|
||
),
|
||
}),
|
||
),
|
||
});
|
||
export type SystemHealthHistory = z.infer<typeof SystemHealthHistory>;
|