feat(system): retention cleanup jobs, storage panel API and health storage level (CACHE-2)
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
d13c18b4b1
commit
05b163ce32
9 files changed
+1254
-9
No files matched your search
@@ -2017,6 +2017,21 @@ export const Api = {
|
||||
query: ops.SystemHealthHistoryQuery,
|
||||
response: ops.SystemHealthHistory,
|
||||
}),
|
||||
storage: ep({
|
||||
method: "get",
|
||||
path: "/system/storage",
|
||||
scope: "admin",
|
||||
summary: "存储：表体积前 15、库总体积、各缓存用量与预算、备份占用、DATA_DIR 分目录用量、清理任务状态",
|
||||
response: ops.SystemStorage,
|
||||
}),
|
||||
storageCleanup: ep({
|
||||
method: "post",
|
||||
path: "/system/storage/cleanup/{target}",
|
||||
scope: "admin",
|
||||
summary: "立即执行某个存储清理任务（排入对应定时任务，已在排队或运行中则合并）",
|
||||
params: ops.StorageCleanupParam,
|
||||
response: ops.StorageCleanupResult,
|
||||
}),
|
||||
},
|
||||
|
||||
// ── 引擎面板(仅 admin) ──
|
||||
|
||||
@@ -481,9 +481,110 @@ export const SystemHealth = z.object({
|
||||
),
|
||||
failedRuns24h: NonNegInt,
|
||||
}),
|
||||
/** 存储预算(CACHE-2):PG 库体积、图片缓存、备份、会话与证据对照 settings.storage.budgets;明细见 GET /system/storage */
|
||||
storage: z.object({
|
||||
level: HealthLevel,
|
||||
/** 超出告警线 / 预算的项(人读文案) */
|
||||
alerts: z.array(z.string()),
|
||||
}),
|
||||
});
|
||||
export type SystemHealth = z.infer<typeof SystemHealth>;
|
||||
|
||||
// ── 存储预算与保留(CACHE-2;GET /system/storage、POST /system/storage/cleanup/{target},仅 admin) ──
|
||||
|
||||
/** 手动触发的清理目标;每个对应一个维护组定时任务 */
|
||||
export const StorageCleanupTarget = z.enum([
|
||||
"observation",
|
||||
"sse_event",
|
||||
"heartbeat",
|
||||
"task_run",
|
||||
"agent_event",
|
||||
"audit",
|
||||
"capability_stats",
|
||||
"identity_pool",
|
||||
"analyze",
|
||||
"images",
|
||||
"kv_cache",
|
||||
]);
|
||||
export type StorageCleanupTarget = z.infer<typeof StorageCleanupTarget>;
|
||||
export const StorageCleanupParam = z.object({ target: StorageCleanupTarget });
|
||||
export const StorageCleanupResult = z.object({
|
||||
target: StorageCleanupTarget,
|
||||
/** 对应的定时任务 key */
|
||||
jobId: z.string(),
|
||||
/** false = 已在排队或运行中(请求被合并) */
|
||||
queued: z.boolean(),
|
||||
});
|
||||
export type StorageCleanupResult = z.infer<typeof StorageCleanupResult>;
|
||||
|
||||
/** 一项占用对照预算 */
|
||||
export const StorageBudgetUsage = z.object({
|
||||
usedBytes: NonNegInt.nullable(),
|
||||
/** 预算(critical 线);无预算为 null */
|
||||
budgetBytes: NonNegInt.nullable(),
|
||||
/** 告警线;无单独告警线时为预算的 80% */
|
||||
warnBytes: NonNegInt.nullable(),
|
||||
level: HealthLevel,
|
||||
error: z.string().nullable(),
|
||||
});
|
||||
export type StorageBudgetUsage = z.infer<typeof StorageBudgetUsage>;
|
||||
|
||||
export const SystemStorage = z.object({
|
||||
/** 各项最差档(unknown 不参与) */
|
||||
level: HealthLevel,
|
||||
checkedAt: Timestamp,
|
||||
/** 目录用量的统计时间(目录遍历结果缓存 5 分钟) */
|
||||
dirsScannedAt: Timestamp.nullable(),
|
||||
db: StorageBudgetUsage.extend({
|
||||
/** 体积前 15 的表(含索引与 TOAST) */
|
||||
tables: z.array(
|
||||
z.object({
|
||||
schema: z.string(),
|
||||
name: z.string(),
|
||||
totalBytes: NonNegInt,
|
||||
tableBytes: NonNegInt,
|
||||
indexBytes: NonNegInt,
|
||||
/** 估算行数(pg_class.reltuples);未 ANALYZE 为 null */
|
||||
rowsEstimate: NonNegInt.nullable(),
|
||||
deadRows: NonNegInt.nullable(),
|
||||
lastAutovacuumAt: Timestamp.nullable(),
|
||||
lastAnalyzeAt: Timestamp.nullable(),
|
||||
}),
|
||||
),
|
||||
}),
|
||||
/** PG kv_cache 各 region(条数含未清理的过期行) */
|
||||
kvCache: z.array(z.object({ region: z.string(), entries: NonNegInt, expired: NonNegInt, bytes: NonNegInt })),
|
||||
images: StorageBudgetUsage.extend({ files: NonNegInt.nullable() }),
|
||||
backups: StorageBudgetUsage.extend({ count: NonNegInt.nullable() }),
|
||||
sessionsEvidence: StorageBudgetUsage,
|
||||
/** DATA_DIR 下各一级目录(cache 再细分一层)的用量 */
|
||||
dataDir: z.array(
|
||||
z.object({
|
||||
name: z.string(),
|
||||
path: z.string(),
|
||||
usedBytes: NonNegInt,
|
||||
files: NonNegInt,
|
||||
error: z.string().nullable(),
|
||||
}),
|
||||
),
|
||||
/** 各清理任务的当前设置与最近运行 */
|
||||
cleanup: z.array(
|
||||
z.object({
|
||||
target: StorageCleanupTarget,
|
||||
jobId: z.string(),
|
||||
/** 保留期说明(如 "90 天"、"24 小时"、"容量 20 GB") */
|
||||
retention: z.string().nullable(),
|
||||
/** 定时任务不存在(未登记 / 调度器未同步)时为 null */
|
||||
enabled: z.boolean().nullable(),
|
||||
lastRunAt: Timestamp.nullable(),
|
||||
lastStatus: z.string().nullable(),
|
||||
lastSummary: z.string().nullable(),
|
||||
nextRunAt: Timestamp.nullable(),
|
||||
}),
|
||||
),
|
||||
});
|
||||
export type SystemStorage = z.infer<typeof SystemStorage>;
|
||||
|
||||
export const SystemHealthHistoryQuery = z.object({
|
||||
/** 回看小时数,1–168,缺省 6 */
|
||||
hours: z.coerce.number().int().min(1).max(168).default(6),
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
import type { Kysely } from "kysely";
|
||||
import { execDdl } from "../src/migration-helpers.ts";
|
||||
|
||||
// CACHE-2 存储保留(docs/runbook/storage.md):
|
||||
// - 保留清理按时间列分批删除,给还没有时间索引的高流水表补上(已有:task_created_idx、agent_run_event_at_idx、
|
||||
// process_heartbeat_sample_at_idx);
|
||||
// - 按表设置 autovacuum:批量删除 / 追加为主的大表降低触发比例,频繁原地更新的小表更激进,避免死元组堆积。
|
||||
// 引擎 schema(openworkflow)的表由引擎自己建,这里不动。
|
||||
|
||||
const INDEXES: [name: string, ddl: string][] = [
|
||||
["observation_created_idx", "observation (created_at)"],
|
||||
["sse_event_at_idx", "sse_event (at)"],
|
||||
["audit_event_at_idx", "audit_event (at)"],
|
||||
["capability_invocation_created_idx", "capability_invocation (created_at)"],
|
||||
["agent_run_message_created_idx", "agent_run_message (created_at)"],
|
||||
["tasks_fact_created_idx", "tasks_fact (created_at)"],
|
||||
["tasks_run_meta_created_idx", "tasks_run_meta (created_at)"],
|
||||
["tasks_op_ledger_started_idx", "tasks_op_ledger (started_at) where status = 'succeeded'"],
|
||||
];
|
||||
|
||||
/** 追加 + 批量删除为主的大表 */
|
||||
const APPEND_TABLES = [
|
||||
"observation",
|
||||
"sse_event",
|
||||
"audit_event",
|
||||
"agent_run_event",
|
||||
"process_heartbeat_sample",
|
||||
"capability_invocation",
|
||||
"task",
|
||||
"tasks_fact",
|
||||
"tasks_run_meta",
|
||||
];
|
||||
const APPEND_OPTS = {
|
||||
autovacuum_vacuum_scale_factor: "0.05",
|
||||
autovacuum_analyze_scale_factor: "0.02",
|
||||
autovacuum_vacuum_insert_scale_factor: "0.1",
|
||||
};
|
||||
|
||||
/** 频繁原地更新的小表(心跳 15 秒一次、令牌桶、调度状态、缓存命中刷新 accessed_at) */
|
||||
const HOT_TABLES = ["process_heartbeat", "rate_limit_bucket", "scheduled_job", "kv_cache", "outbox_event"];
|
||||
const HOT_OPTS = {
|
||||
autovacuum_vacuum_scale_factor: "0.02",
|
||||
autovacuum_vacuum_threshold: "100",
|
||||
autovacuum_analyze_scale_factor: "0.05",
|
||||
};
|
||||
|
||||
const setOpts = (tables: string[], o: Record<string, string>) =>
|
||||
tables
|
||||
.map(
|
||||
(t) =>
|
||||
`alter table ${t} set (${Object.entries(o)
|
||||
.map(([k, v]) => `${k} = ${v}`)
|
||||
.join(", ")});`,
|
||||
)
|
||||
.join("\n");
|
||||
const resetOpts = (tables: string[], o: Record<string, string>) =>
|
||||
tables.map((t) => `alter table ${t} reset (${Object.keys(o).join(", ")});`).join("\n");
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型
|
||||
export async function up(db: Kysely<any>): Promise<void> {
|
||||
await execDdl(
|
||||
db,
|
||||
`${INDEXES.map(([n, d]) => `create index if not exists ${n} on ${d};`).join("\n")}
|
||||
${setOpts(APPEND_TABLES, APPEND_OPTS)}
|
||||
${setOpts(HOT_TABLES, HOT_OPTS)}
|
||||
`,
|
||||
);
|
||||
}
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 同上
|
||||
export async function down(db: Kysely<any>): Promise<void> {
|
||||
await execDdl(
|
||||
db,
|
||||
`${resetOpts(HOT_TABLES, HOT_OPTS)}
|
||||
${resetOpts(APPEND_TABLES, APPEND_OPTS)}
|
||||
${INDEXES.map(([n]) => `drop index if exists ${n};`).join("\n")}
|
||||
`,
|
||||
);
|
||||
}
|
||||
Reference in new issue
Block a user