feat(system): retention cleanup jobs, storage panel API and health storage level (CACHE-2)

Co-Authored-By: Claude Opus 5.5 <[email protected]>

apps/server/src/modules/system/health.ts

@@ -28,6 +28,8 @@ export interface HealthDeps {
   checkRoot?: typeof checkRoot;
   /** 单个路径 IO 检查超时(ms),缺省 3000 */
   ioTimeoutMs?: number;
+  /** 存储段(CACHE-2);不给时该段为 unknown */
+  storage?: () => Promise<{ level: HealthLevel; alerts: string[] }>;
 }
 
 const LEVEL_ORDER: HealthLevel[] = ["ok", "warn", "critical"];
@@ -384,7 +386,7 @@ export async function loadSchedules(db: Kysely<DB>, now: Date): Promise<SystemHe
 
 export async function buildHealth(d: HealthDeps): Promise<SystemHealth> {
   const now = d.now?.() ?? new Date();
-  const [{ processes, retiredProcesses }, db, paths, engine, schedules] = await Promise.all([
+  const [{ processes, retiredProcesses }, db, paths, engine, schedules, storage] = await Promise.all([
     loadProcesses(d.db, now).catch(() => ({
       processes: [] as SystemProcess[],
       retiredProcesses: [] as SystemProcess[],
@@ -408,6 +410,10 @@ export async function buildHealth(d: HealthDeps): Promise<SystemHealth> {
       failing: [],
       failedRuns24h: 0,
     })),
+    (d.storage?.() ?? Promise.resolve({ level: "unknown" as HealthLevel, alerts: [] })).catch(() => ({
+      level: "unknown" as HealthLevel,
+      alerts: [] as string[],
+    })),
   ]);
   // SSE:存活的 api/all 进程上报的连接数合计;本进程若还没写过心跳,用实时值
   const selfCounted = processes.some((p) => !p.stale && p.instanceId === d.self.instanceId && p.sseConnections != null);
@@ -423,6 +429,7 @@ export async function buildHealth(d: HealthDeps): Promise<SystemHealth> {
       mountLevel,
       engine.level,
       schedules.level,
+      storage.level,
     ]),
     checkedAt: now.toISOString(),
     api: {
@@ -439,6 +446,7 @@ export async function buildHealth(d: HealthDeps): Promise<SystemHealth> {
     mounts: paths.mounts,
     engine,
     schedules,
+    storage,
   };
 }
 

apps/server/src/modules/system/heartbeat.ts

@@ -130,11 +130,13 @@ export async function markHeartbeatStopped(db: Kysely<DB>, instanceId: string, a
 /** 裁剪:超过 24 小时没心跳的进程行、超过 7 天的采样;返回删除行数 */
 export async function pruneHeartbeats(
   db: Kysely<DB>,
-  o: { now?: Date; rowRetentionMs?: number; sampleDays?: number } = {},
+  o: { now?: Date; rowRetentionMs?: number; sampleDays?: number; sampleHours?: number } = {},
 ): Promise<{ rows: number; samples: number }> {
   const now = o.now ?? new Date();
   const rowCutoff = new Date(now.getTime() - (o.rowRetentionMs ?? HEARTBEAT_ROW_RETENTION_MS));
-  const sampleCutoff = new Date(now.getTime() - (o.sampleDays ?? SAMPLE_RETENTION_DAYS) * 24 * 3600_000);
+  // CACHE-2:采样保留取 storage.retentionDays.heartbeatHours(调用方传 sampleHours);sampleDays 为旧参数
+  const sampleHours = o.sampleHours ?? (o.sampleDays ?? SAMPLE_RETENTION_DAYS) * 24;
+  const sampleCutoff = new Date(now.getTime() - sampleHours * 3600_000);
   const rows = await db.deleteFrom("process_heartbeat").where("last_beat_at", "<", rowCutoff).executeTakeFirst();
   const samples = await db.deleteFrom("process_heartbeat_sample").where("at", "<", sampleCutoff).executeTakeFirst();
   return { rows: Number(rows.numDeletedRows), samples: Number(samples.numDeletedRows) };

apps/server/src/modules/system/index.ts

@@ -1,9 +1,19 @@
 import { createRoute, type RouteConfig } from "@hono/zod-openapi";
-import { Api, apiResponse, DEFAULT_SETTINGS, type EndpointDef } from "@umt/contracts";
+import {
+  type Actor,
+  Api,
+  apiResponse,
+  DEFAULT_SETTINGS,
+  type EndpointDef,
+  type StorageCleanupTarget,
+} from "@umt/contracts";
+import { sql } from "@umt/db";
 import type { z } from "zod";
 import { dataPaths } from "../../config.ts";
 import { requireScope } from "../../http/auth.ts";
+import { ApiException } from "../../http/errors.ts";
 import { sseConnectionCount } from "../../http/sse.ts";
+import { storageSettings } from "../../shared-cache.ts";
 import { registerScheduledJob } from "../schedules/registry.ts";
 import type { ModuleApp, ServerContext, ServerModule } from "../types.ts";
 import { buildHealth, type HealthDeps, loadHistory } from "./health.ts";
@@ -15,6 +25,8 @@ import {
   safeHostname,
   startHeartbeatWriter,
 } from "./heartbeat.ts";
+import { type BatchOptions, CLEANUP_JOB_IDS, RETENTION_JOBS } from "./retention.ts";
+import { buildStorage, type StoragePaths, storageHealth } from "./storage.ts";
 
 export { HEARTBEAT_STALE_MS, HEARTBEAT_WRITE_MS, startHeartbeatWriter } from "./heartbeat.ts";
 
@@ -23,7 +35,13 @@ export { HEARTBEAT_STALE_MS, HEARTBEAT_WRITE_MS, startHeartbeatWriter } from "./
  * - GET /system/health:进程心跳 / 事件循环延迟 / SSE 连接 / DB / 磁盘 / 挂载根 / 引擎 / 定时任务,仅 admin;
  * - GET /system/health/history?hours=:心跳采样时间序列。
  * 心跳写入:worker 进程在 worker/index.ts 里起写入器;纯 api 进程(role=api)在注册路由时起(role=all 由 worker 写 kind=all)。
- * 清理:定时任务 system.heartbeat_cleanup(维护组,1 小时)删 24 小时没心跳的进程行与 7 天前的采样。
+ * 清理:定时任务 system.heartbeat_cleanup(维护组,1 小时)删 24 小时没心跳的进程行与超过
+ * storage.retentionDays.heartbeatHours 的采样。
+ *
+ * CACHE-2 存储(docs/runbook/storage.md):
+ * - GET /system/storage:表体积 top 15、库体积、kv_cache 各 region、图片 / 备份 / 会话与证据对照预算、DATA_DIR 分目录、清理任务;
+ * - POST /system/storage/cleanup/{target}:把对应的维护任务排入立即执行(同 schedules run_now 语义);
+ * - 保留清理任务 storage.retention.* 与每周 storage.analyze(维护组,各自开关与周期在定时任务看板改)。
  */
 export interface SystemModuleOptions {
   /** 引擎 schema(测试覆盖);缺省 openworkflow */
@@ -34,6 +52,12 @@ export interface SystemModuleOptions {
   apiHeartbeat?: boolean;
   /** 定时任务 system.heartbeat_cleanup 的默认周期,默认 1 小时;<=0 不登记 */
   cleanupIntervalMs?: number;
+  /** 存储面板路径(测试覆盖);缺省按 DATA_DIR / AGENT_SESSION_DIR */
+  storagePaths?: StoragePaths;
+  /** false 时不登记保留清理任务(测试);缺省 true */
+  retentionJobs?: boolean;
+  /** 保留清理的分批参数(测试) */
+  retentionBatch?: BatchOptions;
 }
 
 type Infer<T> = T extends z.ZodType ? z.infer<T> : Record<string, never>;
@@ -41,10 +65,11 @@ type Infer<T> = T extends z.ZodType ? z.infer<T> : Record<string, never>;
 function route<D extends EndpointDef>(
   app: ModuleApp,
   def: D,
-  handler: (i: { query: Infer<D["query"]> }) => Promise<unknown>,
+  handler: (i: { query: Infer<D["query"]>; params: Infer<D["params"]>; actor: Actor | undefined }) => Promise<unknown>,
 ) {
   const request: RouteConfig["request"] = {};
   if (def.query) request.query = def.query as never;
+  if (def.params) request.params = def.params as never;
   const r = createRoute({
     method: def.method,
     path: def.path,
@@ -58,7 +83,13 @@ function route<D extends EndpointDef>(
   });
   // biome-ignore lint/suspicious/noExplicitAny: 路由泛型在封装里擦除,入参已由 zod-openapi 校验
   app.openapi(r, (async (c: any) => {
-    const data = def.response.parse(await handler({ query: def.query ? c.req.valid("query") : {} } as never));
+    const data = def.response.parse(
+      await handler({
+        query: def.query ? c.req.valid("query") : {},
+        params: def.params ? c.req.valid("param") : {},
+        actor: c.get("actor"),
+      } as never),
+    );
     return c.json({ ok: true, data }, 200);
     // biome-ignore lint/suspicious/noExplicitAny: 同上
   }) as any);
@@ -84,6 +115,11 @@ export function createSystemModule(opts: SystemModuleOptions = {}): ServerModule
       }
       const selfKind = ctx.role === "all" ? "all" : ctx.role === "api" ? "api" : null;
       const selfId = selfKind ? instanceIdOf(selfKind, safeHostname(), Deno.pid) : undefined;
+      const paths = (): StoragePaths => {
+        if (opts.storagePaths) return opts.storagePaths;
+        const p = dataPaths(ctx.config);
+        return { dataDir: opts.healthDeps?.dataDir ?? p.dataDir, sessionDir: p.sessionDir };
+      };
 
       route(app, Api.system.healthDetail, async () => {
         const cache = await ctx.settings.get("cache").catch(() => DEFAULT_SETTINGS.cache);
@@ -98,25 +134,89 @@ export function createSystemModule(opts: SystemModuleOptions = {}): ServerModule
             ...(selfId ? { instanceId: selfId } : {}),
           },
           thresholds: { warnPercent: cache.resumePercent, criticalPercent: cache.hardThresholdPercent },
+          storage: async () =>
+            await storageHealth({ db: ctx.db, paths: paths(), settings: await storageSettings(ctx) }),
           ...(opts.healthDeps ?? {}),
         });
       });
       route(app, Api.system.healthHistory, async ({ query }) => await loadHistory(ctx.db, query.hours));
+      route(app, Api.system.storage, async () =>
+        buildStorage({
+          db: ctx.db,
+          paths: paths(),
+          settings: await storageSettings(ctx),
+          engineSchema: opts.engineSchema ?? "openworkflow",
+        }),
+      );
+      route(app, Api.system.storageCleanup, async ({ params, actor }) => {
+        const target = params.target as StorageCleanupTarget;
+        const jobId = CLEANUP_JOB_IDS[target];
+        const job = await ctx.db
+          .selectFrom("scheduled_job")
+          .select(["id", "available"])
+          .where("id", "=", jobId)
+          .executeTakeFirst();
+        if (!job) throw new ApiException("not_found", `清理任务 ${jobId} 还没登记（调度器未同步）`);
+        if (!job.available) throw new ApiException("conflict", "任务未配置，不能执行");
+        // 同 schedules 的立即执行:只置 run_requested_at,已在排队或运行中则合并
+        const r = await sql`update scheduled_job set run_requested_at = now()
+          where id = ${jobId} and run_requested_at is null and running_since is null`.execute(ctx.db);
+        const queued = Number(r.numAffectedRows ?? 0) > 0;
+        if (queued) {
+          await ctx.db
+            .insertInto("audit_event")
+            .values({
+              actor_type: actor?.type === "api_key" ? "api_key" : actor ? "user" : "system",
+              actor_id: actor?.id ?? null,
+              action: "storage.cleanup",
+              target_type: "scheduled_job",
+              target_id: jobId,
+              payload: JSON.stringify({ target }),
+            })
+            .execute()
+            .catch((err) => ctx.logger.warn({ err }, "审计写入失败"));
+        }
+        return { target, jobId, queued };
+      });
     },
     registerTasks() {},
     start(ctx) {
+      if (opts.retentionJobs !== false) {
+        for (const j of RETENTION_JOBS) {
+          registerScheduledJob(ctx, {
+            key: j.jobId,
+            defaults: {
+              name: j.name,
+              description: j.description,
+              group: "maintain",
+              schedule: { type: "interval", everyMs: j.everyMs },
+              firstRun: "after_interval",
+            },
+            run: async ({ now }) => ({
+              summary: await j.run({
+                db: ctx.db,
+                engineSchema: opts.engineSchema ?? "openworkflow",
+                settings: await storageSettings(ctx),
+                now,
+                ...(opts.retentionBatch ? { batch: opts.retentionBatch } : {}),
+              }),
+            }),
+          });
+        }
+      }
       const everyMs = opts.cleanupIntervalMs ?? 3600_000;
       if (everyMs <= 0) return;
       registerScheduledJob(ctx, {
         key: "system.heartbeat_cleanup",
         defaults: {
           name: "进程心跳清理",
-          description: "删除 24 小时没有心跳的进程行和 7 天前的心跳采样",
+          description: "删除 24 小时没有心跳的进程行和超过保留期（存储设置，缺省 24 小时）的心跳采样",
           group: "maintain",
           schedule: { type: "interval", everyMs },
         },
         run: async ({ now }) => {
-          const r = await pruneHeartbeats(ctx.db, { now });
+          const sampleHours = (await storageSettings(ctx)).retentionDays.heartbeatHours;
+          const r = await pruneHeartbeats(ctx.db, { now, sampleHours });
           return { summary: `删除 ${r.rows} 个进程行、${r.samples} 条采样` };
         },
       });

apps/server/src/modules/system/retention.ts

@@ -0,0 +1,273 @@
+import type { StorageCleanupTarget, StorageSettings } from "@umt/contracts";
+import { type DB, type Kysely, type RawBuilder, sql } from "@umt/db";
+
+// CACHE-2 保留策略清理(docs/runbook/storage.md):按表按时间列分批删除流水数据。
+// - 每批:单独事务,`set local statement_timeout / lock_timeout` 限时,按 ctid 取一批再删,不做整表长事务;
+// - 每次运行有总时限,到点就停,剩下的留给下次;批与批之间让出一小段时间给业务写入;
+// - 只删已结束的数据(任务 / Agent 会话 / 能力调用看状态),进行中的不动。
+// 待识别池不删:unmatched 条目到期归档(与 sources-identity archiveStalePool 同一条件)。
+
+export interface BatchOptions {
+  /** 每批行数,缺省 5000 */
+  batchSize?: number;
+  /** 每批语句超时(ms),缺省 15000 */
+  statementTimeoutMs?: number;
+  /** 等锁超时(ms),缺省 3000 */
+  lockTimeoutMs?: number;
+  /** 本次运行总时限(ms),缺省 5 分钟 */
+  budgetMs?: number;
+  /** 批间停顿(ms),缺省 50 */
+  pauseMs?: number;
+  /** 测试注入 */
+  now?: () => number;
+}
+
+export interface BatchResult {
+  deleted: number;
+  batches: number;
+  /** 到总时限还没删完 */
+  truncated: boolean;
+}
+
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+/**
+ * 分批删除 `table` 中满足 `where` 的行:每批 `delete … where ctid = any(array(select ctid … limit n))`,
+ * 单独事务 + 语句 / 锁超时。某批超时(57014)或等锁超时(55P03)时停止本次运行,不抛错。
+ */
+export async function deleteInBatches(
+  db: Kysely<DB>,
+  table: RawBuilder<unknown>,
+  where: RawBuilder<unknown>,
+  o: BatchOptions = {},
+): Promise<BatchResult> {
+  const size = o.batchSize ?? 5000;
+  const now = o.now ?? Date.now;
+  const deadline = now() + (o.budgetMs ?? 5 * 60_000);
+  let deleted = 0;
+  let batches = 0;
+  for (;;) {
+    let n: number;
+    try {
+      n = await db.transaction().execute(async (trx) => {
+        await sql`select set_config('statement_timeout', ${String(o.statementTimeoutMs ?? 15_000)}, true),
+                         set_config('lock_timeout', ${String(o.lockTimeoutMs ?? 3000)}, true)`.execute(trx);
+        const r = await sql`delete from ${table} where ctid = any(array(
+            select ctid from ${table} where ${where} limit ${size}))`.execute(trx);
+        return Number(r.numAffectedRows ?? 0);
+      });
+    } catch (e) {
+      const code = (e as { code?: string }).code;
+      if (code === "57014" || code === "55P03") return { deleted, batches, truncated: true };
+      throw e;
+    }
+    batches++;
+    deleted += n;
+    if (n < size) return { deleted, batches, truncated: false };
+    if (now() >= deadline) return { deleted, batches, truncated: true };
+    if ((o.pauseMs ?? 50) > 0) await sleep(o.pauseMs ?? 50);
+  }
+}
+
+export interface RetentionJob {
+  target: StorageCleanupTarget;
+  /** scheduled_job key */
+  jobId: string;
+  name: string;
+  description: string;
+  everyMs: number;
+  /** 保留期说明(存储面板) */
+  retention(s: StorageSettings): string | null;
+  run(c: RetentionRunContext): Promise<string>;
+}
+
+export interface RetentionRunContext {
+  db: Kysely<DB>;
+  engineSchema: string;
+  settings: StorageSettings;
+  now: Date;
+  batch?: BatchOptions;
+}
+
+const DAY_MS = 86_400_000;
+const cutoffOf = (now: Date, days: number) => new Date(now.getTime() - days * DAY_MS);
+const t = (name: string) => sql.raw(`"${name}"`);
+const summaryOf = (label: string, r: BatchResult) =>
+  `${label}删除 ${r.deleted} 行（${r.batches} 批）${r.truncated ? "，未删完，下次继续" : ""}`;
+
+const HOUR = 3600_000;
+
+/**
+ * 保留清理任务(维护组;heartbeat 目标由 system.heartbeat_cleanup 承担,images / kv_cache 由 cache 模块承担)。
+ * 顺序即存储面板的展示顺序。
+ */
+export const RETENTION_JOBS: RetentionJob[] = [
+  {
+    target: "observation",
+    jobId: "storage.retention.observation",
+    name: "观察记录清理",
+    description: "删除超过保留期的 Feed 观察记录（observation，按 created_at）",
+    everyMs: 6 * HOUR,
+    retention: (s) => `${s.retentionDays.observation} 天`,
+    async run(c) {
+      const cutoff = cutoffOf(c.now, c.settings.retentionDays.observation);
+      const r = await deleteInBatches(c.db, t("observation"), sql`created_at < ${cutoff}`, c.batch);
+      return summaryOf("观察记录", r);
+    },
+  },
+  {
+    target: "sse_event",
+    jobId: "storage.retention.sse_event",
+    name: "SSE 事件清理",
+    description: "删除超过保留期的 SSE 补读事件（sse_event，按 at；发布方另按条数裁剪）",
+    everyMs: HOUR,
+    retention: (s) => `${s.retentionDays.sseEvent} 天`,
+    async run(c) {
+      const cutoff = cutoffOf(c.now, c.settings.retentionDays.sseEvent);
+      const r = await deleteInBatches(c.db, t("sse_event"), sql`at < ${cutoff}`, c.batch);
+      return summaryOf("SSE 事件", r);
+    },
+  },
+  {
+    target: "task_run",
+    jobId: "storage.retention.task_run",
+    name: "任务运行清理",
+    description: "删除超过保留期、已结束的业务任务（task）与引擎 run（含步骤、运行元数据）",
+    everyMs: 6 * HOUR,
+    retention: (s) => `${s.retentionDays.taskRun} 天`,
+    async run(c) {
+      const cutoff = cutoffOf(c.now, c.settings.retentionDays.taskRun);
+      const e = (n: string) => sql.raw(`"${c.engineSchema}"."${n}"`);
+      const task = await deleteInBatches(
+        c.db,
+        t("task"),
+        sql`created_at < ${cutoff} and status in ('succeeded', 'failed', 'cancelled', 'dead')`,
+        c.batch,
+      );
+      // 引擎 run:step_attempts 随 run 级联删除
+      const runs = await deleteInBatches(
+        c.db,
+        e("workflow_runs"),
+        sql`created_at < ${cutoff} and status in ('completed', 'succeeded', 'failed', 'canceled')`,
+        c.batch,
+      );
+      // 运行元数据与 run 同时写入:早于保留期的 run 已结束(上一步删掉了),元数据按时间直接删
+      const meta = await deleteInBatches(c.db, t("tasks_run_meta"), sql`created_at < ${cutoff}`, c.batch);
+      const facts = await deleteInBatches(c.db, t("tasks_fact"), sql`created_at < ${cutoff}`, c.batch);
+      const ledger = await deleteInBatches(
+        c.db,
+        t("tasks_op_ledger"),
+        sql`status = 'succeeded' and started_at < ${cutoff}`,
+        c.batch,
+      );
+      const truncated = [task, runs, meta, facts, ledger].some((x) => x.truncated);
+      return `业务任务 ${task.deleted}、引擎 run ${runs.deleted}、运行元数据 ${meta.deleted + facts.deleted + ledger.deleted} 行已删除${truncated ? "，未删完，下次继续" : ""}`;
+    },
+  },
+  {
+    target: "agent_event",
+    jobId: "storage.retention.agent_event",
+    name: "Agent 事件清理",
+    description: "删除超过保留期、会话已结束的 Agent 事件流与消息（agent_run_event / agent_run_message；会话本身保留）",
+    everyMs: 6 * HOUR,
+    retention: (s) => `${s.retentionDays.agentEvent} 天`,
+    async run(c) {
+      const cutoff = cutoffOf(c.now, c.settings.retentionDays.agentEvent);
+      const ended = sql`exists (select 1 from agent_run r where r.id = run_id
+        and r.status in ('succeeded', 'failed', 'cancelled', 'budget_exceeded'))`;
+      const ev = await deleteInBatches(c.db, t("agent_run_event"), sql`at < ${cutoff} and ${ended}`, c.batch);
+      const msg = await deleteInBatches(
+        c.db,
+        t("agent_run_message"),
+        sql`created_at < ${cutoff} and status = 'consumed' and ${ended}`,
+        c.batch,
+      );
+      return `Agent 事件 ${ev.deleted}、消息 ${msg.deleted} 行已删除${ev.truncated || msg.truncated ? "，未删完，下次继续" : ""}`;
+    },
+  },
+  {
+    target: "audit",
+    jobId: "storage.retention.audit",
+    name: "审计清理",
+    description: "删除超过保留期的审计事件（audit_event，能力调用审计按能力统计的保留期另算）",
+    everyMs: 24 * HOUR,
+    retention: (s) => `${s.retentionDays.audit} 天`,
+    async run(c) {
+      const cutoff = cutoffOf(c.now, c.settings.retentionDays.audit);
+      const r = await deleteInBatches(
+        c.db,
+        t("audit_event"),
+        sql`at < ${cutoff} and target_type is distinct from 'capability'`,
+        c.batch,
+      );
+      return summaryOf("审计事件", r);
+    },
+  },
+  {
+    target: "capability_stats",
+    jobId: "storage.retention.capability_stats",
+    name: "能力统计清理",
+    description: "删除超过保留期的能力调用审计（audit_event target_type=capability）与已结束的能力调用记录",
+    everyMs: 24 * HOUR,
+    retention: (s) => `${s.retentionDays.capabilityStats} 天`,
+    async run(c) {
+      const cutoff = cutoffOf(c.now, c.settings.retentionDays.capabilityStats);
+      const a = await deleteInBatches(
+        c.db,
+        t("audit_event"),
+        sql`at < ${cutoff} and target_type = 'capability'`,
+        c.batch,
+      );
+      const inv = await deleteInBatches(
+        c.db,
+        t("capability_invocation"),
+        sql`created_at < ${cutoff} and status <> 'running'`,
+        c.batch,
+      );
+      return `能力审计 ${a.deleted}、调用记录 ${inv.deleted} 行已删除${a.truncated || inv.truncated ? "，未删完，下次继续" : ""}`;
+    },
+  },
+  {
+    target: "identity_pool",
+    jobId: "storage.retention.identity_pool",
+    name: "待识别池归档",
+    description: "待识别池里 unmatched 条目超过保留期没有新证据就归档（不删除，仍可按状态筛出）",
+    everyMs: 6 * HOUR,
+    retention: (s) => `${s.retentionDays.identityPoolArchive} 天`,
+    async run(c) {
+      const cutoff = cutoffOf(c.now, c.settings.retentionDays.identityPoolArchive);
+      let archived = 0;
+      const size = c.batch?.batchSize ?? 5000;
+      for (;;) {
+        const r = await sql`update identity_candidate set status = 'archived', next_review_at = null
+          where id in (select id from identity_candidate where status = 'pending' and reason = 'unmatched'
+                         and coalesce(last_attempt_at, created_at) < ${cutoff} limit ${size})`.execute(c.db);
+        const n = Number(r.numAffectedRows ?? 0);
+        archived += n;
+        if (n < size) break;
+      }
+      return `归档 ${archived} 条待识别条目`;
+    },
+  },
+  {
+    target: "analyze",
+    jobId: "storage.analyze",
+    name: "统计信息更新",
+    description: "对全库执行 ANALYZE，刷新查询规划用的统计信息（每周）",
+    everyMs: 7 * 24 * HOUR,
+    retention: () => null,
+    async run(c) {
+      const started = Date.now();
+      await sql`analyze`.execute(c.db);
+      return `ANALYZE 完成，用时 ${Math.round((Date.now() - started) / 100) / 10} 秒`;
+    },
+  },
+];
+
+/** 清理目标 → 定时任务 key(heartbeat / images / kv_cache 由其它任务承担) */
+export const CLEANUP_JOB_IDS: Record<StorageCleanupTarget, string> = {
+  ...(Object.fromEntries(RETENTION_JOBS.map((j) => [j.target, j.jobId])) as Record<string, string>),
+  heartbeat: "system.heartbeat_cleanup",
+  images: "cache.images_lru",
+  kv_cache: "cache.sweep",
+} as Record<StorageCleanupTarget, string>;

apps/server/src/modules/system/storage.ts

@@ -0,0 +1,347 @@
+import { promises as fs } from "node:fs";
+import { join, relative } from "node:path";
+import type { HealthLevel, StorageBudgetUsage, StorageSettings, SystemStorage } from "@umt/contracts";
+import { type DB, type Kysely, sql } from "@umt/db";
+import { worst } from "./health.ts";
+import { CLEANUP_JOB_IDS, RETENTION_JOBS } from "./retention.ts";
+
+// CACHE-2 存储面板(GET /system/storage)与 health 的存储档位。
+// 目录遍历可能很慢(图片缓存几十万个文件),结果按目录集合缓存 5 分钟;health 只用缓存值,过期时后台刷新,不等。
+
+const GB = 1024 ** 3;
+const DIR_TTL_MS = 5 * 60_000;
+
+export interface StoragePaths {
+  dataDir: string;
+  /** Agent 会话目录(可能在 DATA_DIR 之外) */
+  sessionDir: string;
+  /** 图片缓存目录;缺省 DATA_DIR/cache/images */
+  imagesDir?: string;
+}
+
+export interface DirUsage {
+  name: string;
+  path: string;
+  usedBytes: number;
+  files: number;
+  error: string | null;
+}
+
+interface DirScan {
+  at: Date;
+  dirs: DirUsage[];
+  images: DirUsage;
+  sessions: DirUsage;
+  evidence: DirUsage;
+  backups: DirUsage & { count: number };
+}
+
+async function walk(path: string): Promise<{ bytes: number; files: number; error: string | null }> {
+  let bytes = 0;
+  let files = 0;
+  const go = async (d: string, top: boolean): Promise<string | null> => {
+    let names: string[];
+    try {
+      names = await fs.readdir(d);
+    } catch (e) {
+      const code = (e as { code?: string }).code;
+      // 目录不存在按 0 计
+      return top && code !== "ENOENT" ? (code ?? (e as Error).message) : null;
+    }
+    for (const n of names) {
+      const p = join(d, n);
+      const st = await fs.lstat(p).catch(() => null);
+      if (!st) continue;
+      if (st.isDirectory()) await go(p, false);
+      else {
+        bytes += st.size;
+        files++;
+      }
+    }
+    return null;
+  };
+  const error = await go(path, true);
+  return { bytes, files, error };
+}
+
+const usageOf = async (name: string, path: string): Promise<DirUsage> => {
+  const w = await walk(path);
+  return { name, path, usedBytes: w.bytes, files: w.files, error: w.error };
+};
+
+/** DATA_DIR 一级目录(cache / tmp 再细分一层)+ 会话目录 + 备份份数 */
+export async function scanDirs(p: StoragePaths): Promise<DirScan> {
+  const dirs: DirUsage[] = [];
+  const top = await fs.readdir(p.dataDir, { withFileTypes: true }).catch(() => []);
+  for (const e of top.sort((a, b) => a.name.localeCompare(b.name))) {
+    if (!e.isDirectory()) continue;
+    const path = join(p.dataDir, e.name);
+    if (e.name === "cache" || e.name === "tmp") {
+      const sub = await fs.readdir(path, { withFileTypes: true }).catch(() => []);
+      for (const s of sub.sort((a, b) => a.name.localeCompare(b.name))) {
+        if (s.isDirectory()) dirs.push(await usageOf(`${e.name}/${s.name}`, join(path, s.name)));
+      }
+      continue;
+    }
+    dirs.push(await usageOf(e.name, path));
+  }
+  const find = async (path: string, name: string) => dirs.find((d) => d.path === path) ?? (await usageOf(name, path));
+  const imagesDir = p.imagesDir ?? join(p.dataDir, "cache", "images");
+  const backupsDir = join(p.dataDir, "backups");
+  const backupNames = await fs.readdir(backupsDir, { withFileTypes: true }).catch(() => []);
+  const sessionsInData = !relative(p.dataDir, p.sessionDir).startsWith("..");
+  const sessions = await find(p.sessionDir, "agent-sessions");
+  if (!sessionsInData) dirs.push(sessions);
+  return {
+    at: new Date(),
+    dirs,
+    images: await find(imagesDir, "cache/images"),
+    sessions,
+    evidence: await find(join(p.dataDir, "tmp", "evidence"), "tmp/evidence"),
+    backups: {
+      ...(await find(backupsDir, "backups")),
+      count: backupNames.filter((b) => b.isDirectory() && !b.name.startsWith(".")).length,
+    },
+  };
+}
+
+const scans = new Map<string, { at: number; value: Promise<DirScan>; last: DirScan | null }>();
+
+/**
+ * 目录用量(缓存 5 分钟)。wait=false 时不等遍历:有旧值返回旧值(过期则后台刷新),没有返回 null。
+ */
+export async function dirScan(p: StoragePaths, o: { wait: boolean; force?: boolean }): Promise<DirScan | null> {
+  const key = `${p.dataDir}\u0000${p.sessionDir}\u0000${p.imagesDir ?? ""}`;
+  const cur = scans.get(key);
+  if (!cur || o.force || Date.now() - cur.at > DIR_TTL_MS) {
+    const value = scanDirs(p);
+    const entry = { at: Date.now(), value, last: cur?.last ?? null };
+    scans.set(key, entry);
+    value.then(
+      (v) => {
+        entry.last = v;
+      },
+      () => {
+        if (scans.get(key) === entry) scans.delete(key);
+      },
+    );
+    if (o.wait) return await value;
+    return entry.last;
+  }
+  if (o.wait) return await cur.value;
+  return cur.last;
+}
+
+/** 测试用 */
+export function resetDirScans(): void {
+  scans.clear();
+}
+
+/** 用量对照预算:≥ budget critical,≥ warn warn */
+export function budgetLevel(used: number | null, budget: number | null, warn: number | null): HealthLevel {
+  if (used === null) return "unknown";
+  if (budget !== null && used >= budget) return "critical";
+  if (warn !== null && used >= warn) return "warn";
+  return "ok";
+}
+
+const gb = (v: number) => Math.round(v * GB);
+
+function usage(used: number | null, budgetGb: number, warnGb: number, error: string | null = null): StorageBudgetUsage {
+  const budgetBytes = gb(budgetGb);
+  const warnBytes = gb(warnGb);
+  return {
+    usedBytes: used,
+    budgetBytes,
+    warnBytes,
+    level: error ? "unknown" : budgetLevel(used, budgetBytes, warnBytes),
+    error,
+  };
+}
+
+/**
+ * 图片缓存有容量 LRU(到预算就按最久未访问删),接近预算是常态:超过预算 10% 才告警(说明 LRU 没跟上),不给 critical。
+ */
+function imagesUsage(used: number | null, budgetGb: number, error: string | null): StorageBudgetUsage {
+  const budgetBytes = gb(budgetGb);
+  const warnBytes = Math.round(budgetBytes * 1.1);
+  return {
+    usedBytes: used,
+    budgetBytes,
+    warnBytes,
+    level: error ? "unknown" : used === null ? "unknown" : used >= warnBytes ? "warn" : "ok",
+    error,
+  };
+}
+
+const fmtGb = (b: number | null) => (b === null ? "?" : `${Math.round((b / GB) * 10) / 10} GB`);
+
+export interface StorageDeps {
+  db: Kysely<DB>;
+  paths: StoragePaths;
+  settings: StorageSettings;
+}
+
+async function dbSize(db: Kysely<DB>): Promise<number> {
+  const r = await sql<{ n: string }>`select pg_database_size(current_database())::text as n`.execute(db);
+  return Number(r.rows[0]?.n ?? 0);
+}
+
+/** 汇总档位 + 告警文案(health 与存储面板共用) */
+function levelsOf(
+  s: StorageSettings,
+  dbBytes: number | null,
+  scan: DirScan | null,
+): {
+  db: StorageBudgetUsage;
+  images: StorageBudgetUsage;
+  backups: StorageBudgetUsage;
+  sessionsEvidence: StorageBudgetUsage;
+  level: HealthLevel;
+  alerts: string[];
+} {
+  const b = s.budgets;
+  const db = usage(dbBytes, b.pgDataGb, b.pgDataWarnGb, dbBytes === null ? "db_unavailable" : null);
+  const images = imagesUsage(scan ? scan.images.usedBytes : null, b.imagesGb, scan?.images.error ?? null);
+  const backups = usage(
+    scan ? scan.backups.usedBytes : null,
+    b.backupsGb,
+    b.backupsGb * 0.8,
+    scan?.backups.error ?? null,
+  );
+  const se = scan ? scan.sessions.usedBytes + scan.evidence.usedBytes : null;
+  const sessionsEvidence = usage(
+    se,
+    b.sessionsEvidenceGb,
+    b.sessionsEvidenceGb * 0.8,
+    scan?.sessions.error ?? scan?.evidence.error ?? null,
+  );
+  const alerts: string[] = [];
+  const say = (label: string, u: StorageBudgetUsage) => {
+    if (u.level === "warn" || u.level === "critical") {
+      alerts.push(
+        `${label} ${fmtGb(u.usedBytes)}，${u.level === "critical" ? "已超过预算" : "超过告警线"} ${fmtGb(u.level === "critical" ? u.budgetBytes : u.warnBytes)}`,
+      );
+    }
+  };
+  say("数据库", db);
+  say("图片缓存", images);
+  say("备份", backups);
+  say("会话与证据", sessionsEvidence);
+  return {
+    db,
+    images,
+    backups,
+    sessionsEvidence,
+    level: worst([db.level, images.level, backups.level, sessionsEvidence.level]),
+    alerts,
+  };
+}
+
+/** health 的存储段:库体积实时取,目录用量只用缓存(不等遍历) */
+export async function storageHealth(d: StorageDeps): Promise<{ level: HealthLevel; alerts: string[] }> {
+  const [bytes, scan] = await Promise.all([
+    dbSize(d.db).catch(() => null),
+    dirScan(d.paths, { wait: false }).catch(() => null),
+  ]);
+  const l = levelsOf(d.settings, bytes, scan);
+  return { level: l.level, alerts: l.alerts };
+}
+
+export async function buildStorage(
+  d: StorageDeps & { engineSchema: string; refresh?: boolean },
+): Promise<SystemStorage> {
+  const [bytes, scan, tables, kv, jobs] = await Promise.all([
+    dbSize(d.db).catch(() => null),
+    dirScan(d.paths, { wait: true, force: d.refresh ?? false }).catch(() => null),
+    sql<{
+      schema: string;
+      name: string;
+      total: string;
+      tbl: string;
+      idx: string;
+      rows: number | null;
+      dead: number | null;
+      vac: Date | null;
+      ana: Date | null;
+    }>`select n.nspname as schema, c.relname as name,
+          pg_total_relation_size(c.oid)::text as total, pg_relation_size(c.oid)::text as tbl,
+          pg_indexes_size(c.oid)::text as idx,
+          case when c.reltuples < 0 then null else c.reltuples::bigint end as rows,
+          s.n_dead_tup as dead, s.last_autovacuum as vac,
+          greatest(s.last_analyze, s.last_autoanalyze) as ana
+        from pg_class c join pg_namespace n on n.oid = c.relnamespace
+        left join pg_stat_all_tables s on s.relid = c.oid
+       where c.relkind in ('r', 'm', 'p') and n.nspname not in ('pg_catalog', 'information_schema')
+         and n.nspname not like 'pg_toast%'
+       order by pg_total_relation_size(c.oid) desc limit 15`.execute(d.db),
+    sql<{ region: string; entries: number; expired: number; bytes: string }>`
+      select region, count(*)::int as entries, count(*) filter (where expires_at <= now())::int as expired,
+             coalesce(sum(pg_column_size(value) + pg_column_size(key)), 0)::text as bytes
+        from kv_cache group by region order by region`
+      .execute(d.db)
+      .catch(() => ({ rows: [] })),
+    d.db
+      .selectFrom("scheduled_job")
+      .select(["id", "enabled", "last_run_at", "last_status", "next_run_at"])
+      .where("id", "in", Object.values(CLEANUP_JOB_IDS))
+      .execute()
+      .catch(() => []),
+  ]);
+  const lastSummaries = new Map<string, string | null>();
+  if (jobs.length) {
+    const r = await sql<{ job_id: string; summary: string | null; error: string | null }>`
+      select distinct on (job_id) job_id, summary, error from scheduled_job_run
+       where job_id in (${sql.join(jobs.map((j) => j.id))}) and status <> 'running'
+       order by job_id, started_at desc`
+      .execute(d.db)
+      .catch(() => ({ rows: [] as { job_id: string; summary: string | null; error: string | null }[] }));
+    for (const x of r.rows) lastSummaries.set(x.job_id, x.summary ?? x.error);
+  }
+  const l = levelsOf(d.settings, bytes, scan);
+  const s = d.settings;
+  const retentionOf: Record<string, string | null> = {
+    ...Object.fromEntries(RETENTION_JOBS.map((j) => [j.target, j.retention(s)])),
+    heartbeat: `${s.retentionDays.heartbeatHours} 小时`,
+    images: `容量 ${s.budgets.imagesGb} GB`,
+    kv_cache: null,
+  };
+  const iso = (v: Date | string | null) => (v ? new Date(v).toISOString() : null);
+  return {
+    level: l.level,
+    checkedAt: new Date().toISOString(),
+    dirsScannedAt: scan ? scan.at.toISOString() : null,
+    db: {
+      ...l.db,
+      tables: tables.rows.map((t) => ({
+        schema: t.schema,
+        name: t.name,
+        totalBytes: Number(t.total),
+        tableBytes: Number(t.tbl),
+        indexBytes: Number(t.idx),
+        rowsEstimate: t.rows === null ? null : Number(t.rows),
+        deadRows: t.dead === null ? null : Number(t.dead),
+        lastAutovacuumAt: iso(t.vac),
+        lastAnalyzeAt: iso(t.ana),
+      })),
+    },
+    kvCache: kv.rows.map((r) => ({ region: r.region, entries: r.entries, expired: r.expired, bytes: Number(r.bytes) })),
+    images: { ...l.images, files: scan ? scan.images.files : null },
+    backups: { ...l.backups, count: scan ? scan.backups.count : null },
+    sessionsEvidence: l.sessionsEvidence,
+    dataDir: scan?.dirs ?? [],
+    cleanup: (Object.entries(CLEANUP_JOB_IDS) as [keyof typeof CLEANUP_JOB_IDS, string][]).map(([target, jobId]) => {
+      const j = jobs.find((x) => x.id === jobId);
+      return {
+        target,
+        jobId,
+        retention: retentionOf[target] ?? null,
+        enabled: j ? j.enabled : null,
+        lastRunAt: iso(j?.last_run_at ?? null),
+        lastStatus: j?.last_status ?? null,
+        lastSummary: lastSummaries.get(jobId) ?? null,
+        nextRunAt: iso(j?.next_run_at ?? null),
+      };
+    }),
+  };
+}