Merge branch 'cache2'

packages/adapters/src/tmdb/client.ts

@@ -98,8 +98,11 @@ export interface TmdbConfig {
   maxRetryAfterMs?: number;
   maxConcurrent?: number;
   minIntervalMs?: number;
-  /** 默认 max 1024、ttl 24h;空结果(搜索 0 条)用 emptyTtlMs,默认 30min(对照 MP EMPTY_RESULT_CACHE_TTL) */
-  cache?: { max?: number; ttlMs?: number; emptyTtlMs?: number };
+  /**
+   * 默认 max 1024、ttl 24h;空结果(搜索 0 条)用 emptyTtlMs,默认 30min(对照 MP EMPTY_RESULT_CACHE_TTL);
+   * searchTtlMs:/search/、/find/ 请求的 TTL(缺省同 ttlMs),其余(详情 / 季)用 ttlMs。
+   */
+  cache?: { max?: number; ttlMs?: number; searchTtlMs?: number; emptyTtlMs?: number };
   /**
    * 跨进程共享的请求缓存(服务端传 PG 缓存);给了就不再用进程内 LRU。
    * 语义对照 MP @cached(skip_none、业务失败不缓存、results 为空按 emptyTtlMs)。
@@ -164,6 +167,8 @@ export class TmdbClient {
     for (const k of Object.keys(params).sort()) if (params[k] !== undefined) qs.set(k, params[k]!);
     const key = `${path}?${qs}`;
     const emptyTtl = this.cfg.cache?.emptyTtlMs ?? 30 * 60_000;
+    const baseTtl = this.cfg.cache?.ttlMs ?? 24 * 3600_000;
+    const ttl = /^\/(search|find)\//.test(path) ? (this.cfg.cache?.searchTtlMs ?? baseTtl) : baseTtl;
     if (this.cfg.sharedCache) {
       const shared = this.cfg.sharedCache;
       const hit = await shared.get(key);
@@ -171,14 +176,14 @@ export class TmdbClient {
       // 网络错误 / 业务失败在 fetchWithRetry 抛出,不会走到写缓存(MP skip_none / skip_if)
       const v = await this.limiter.run(() => this.fetchWithRetry(path, qs));
       if (v !== null && v !== undefined) {
-        await shared.set(key, v, isEmptyResult(v) ? emptyTtl : (this.cfg.cache?.ttlMs ?? 24 * 3600_000));
+        await shared.set(key, v, isEmptyResult(v) ? emptyTtl : ttl);
       }
       return v as T;
     }
     return this.cache.getOrLoad(
       key,
       () => this.limiter.run(() => this.fetchWithRetry(path, qs)),
-      (v) => (isEmptyResult(v) ? emptyTtl : undefined),
+      (v) => (isEmptyResult(v) ? emptyTtl : ttl === baseTtl ? undefined : ttl),
     ) as Promise<T>;
   }
 

packages/contracts/src/api.ts

@@ -2017,6 +2017,21 @@ export const Api = {
       query: ops.SystemHealthHistoryQuery,
       response: ops.SystemHealthHistory,
     }),
+    storage: ep({
+      method: "get",
+      path: "/system/storage",
+      scope: "admin",
+      summary: "存储：表体积前 15、库总体积、各缓存用量与预算、备份占用、DATA_DIR 分目录用量、清理任务状态",
+      response: ops.SystemStorage,
+    }),
+    storageCleanup: ep({
+      method: "post",
+      path: "/system/storage/cleanup/{target}",
+      scope: "admin",
+      summary: "立即执行某个存储清理任务（排入对应定时任务，已在排队或运行中则合并）",
+      params: ops.StorageCleanupParam,
+      response: ops.StorageCleanupResult,
+    }),
   },
 
   // ── 引擎面板(仅 admin) ──

packages/contracts/src/ops.ts

@@ -481,9 +481,110 @@ export const SystemHealth = z.object({
     ),
     failedRuns24h: NonNegInt,
   }),
+  /** 存储预算(CACHE-2):PG 库体积、图片缓存、备份、会话与证据对照 settings.storage.budgets;明细见 GET /system/storage */
+  storage: z.object({
+    level: HealthLevel,
+    /** 超出告警线 / 预算的项(人读文案) */
+    alerts: z.array(z.string()),
+  }),
 });
 export type SystemHealth = z.infer<typeof SystemHealth>;
 
+// ── 存储预算与保留(CACHE-2;GET /system/storage、POST /system/storage/cleanup/{target},仅 admin) ──
+
+/** 手动触发的清理目标;每个对应一个维护组定时任务 */
+export const StorageCleanupTarget = z.enum([
+  "observation",
+  "sse_event",
+  "heartbeat",
+  "task_run",
+  "agent_event",
+  "audit",
+  "capability_stats",
+  "identity_pool",
+  "analyze",
+  "images",
+  "kv_cache",
+]);
+export type StorageCleanupTarget = z.infer<typeof StorageCleanupTarget>;
+export const StorageCleanupParam = z.object({ target: StorageCleanupTarget });
+export const StorageCleanupResult = z.object({
+  target: StorageCleanupTarget,
+  /** 对应的定时任务 key */
+  jobId: z.string(),
+  /** false = 已在排队或运行中(请求被合并) */
+  queued: z.boolean(),
+});
+export type StorageCleanupResult = z.infer<typeof StorageCleanupResult>;
+
+/** 一项占用对照预算 */
+export const StorageBudgetUsage = z.object({
+  usedBytes: NonNegInt.nullable(),
+  /** 预算(critical 线);无预算为 null */
+  budgetBytes: NonNegInt.nullable(),
+  /** 告警线;无单独告警线时为预算的 80% */
+  warnBytes: NonNegInt.nullable(),
+  level: HealthLevel,
+  error: z.string().nullable(),
+});
+export type StorageBudgetUsage = z.infer<typeof StorageBudgetUsage>;
+
+export const SystemStorage = z.object({
+  /** 各项最差档(unknown 不参与) */
+  level: HealthLevel,
+  checkedAt: Timestamp,
+  /** 目录用量的统计时间(目录遍历结果缓存 5 分钟) */
+  dirsScannedAt: Timestamp.nullable(),
+  db: StorageBudgetUsage.extend({
+    /** 体积前 15 的表(含索引与 TOAST) */
+    tables: z.array(
+      z.object({
+        schema: z.string(),
+        name: z.string(),
+        totalBytes: NonNegInt,
+        tableBytes: NonNegInt,
+        indexBytes: NonNegInt,
+        /** 估算行数(pg_class.reltuples);未 ANALYZE 为 null */
+        rowsEstimate: NonNegInt.nullable(),
+        deadRows: NonNegInt.nullable(),
+        lastAutovacuumAt: Timestamp.nullable(),
+        lastAnalyzeAt: Timestamp.nullable(),
+      }),
+    ),
+  }),
+  /** PG kv_cache 各 region(条数含未清理的过期行) */
+  kvCache: z.array(z.object({ region: z.string(), entries: NonNegInt, expired: NonNegInt, bytes: NonNegInt })),
+  images: StorageBudgetUsage.extend({ files: NonNegInt.nullable() }),
+  backups: StorageBudgetUsage.extend({ count: NonNegInt.nullable() }),
+  sessionsEvidence: StorageBudgetUsage,
+  /** DATA_DIR 下各一级目录(cache 再细分一层)的用量 */
+  dataDir: z.array(
+    z.object({
+      name: z.string(),
+      path: z.string(),
+      usedBytes: NonNegInt,
+      files: NonNegInt,
+      error: z.string().nullable(),
+    }),
+  ),
+  /** 各清理任务的当前设置与最近运行 */
+  cleanup: z.array(
+    z.object({
+      target: StorageCleanupTarget,
+      jobId: z.string(),
+      /** 保留期说明(如 "90 天"、"24 小时"、"容量 20 GB") */
+      retention: z.string().nullable(),
+      /** 定时任务不存在(未登记 / 调度器未同步)时为 null */
+      enabled: z.boolean().nullable(),
+      lastRunAt: Timestamp.nullable(),
+      lastStatus: z.string().nullable(),
+      lastSummary: z.string().nullable(),
+      nextRunAt: Timestamp.nullable(),
+    }),
+  ),
+});
+export type SystemStorage = z.infer<typeof SystemStorage>;
+
 export const SystemHealthHistoryQuery = z.object({
   /** 回看小时数,1–168,缺省 6 */
   hours: z.coerce.number().int().min(1).max(168).default(6),

packages/contracts/src/settings.ts

@@ -24,11 +24,14 @@ const CacheShape = z.object({
   checkIntervalSec: z.number().int().min(10).max(3600),
   /** 缓存清理的目标使用率(低水位) */
   cleanupTargetPercent: Percent,
-  /** TMDB 缓存(请求响应 + 识别结果)保留小时数(对照 MP META_CACHE_EXPIRE,缺省 24) */
+  /**
+   * @deprecated CACHE-2 起由 storage.cacheTtl(tmdbSearchDays / tmdbDetailDays)取代,服务端不再读取;
+   * 保留在 schema 里只为兼容已存的 JSON 与旧设置页。
+   */
   tmdbTtlHours: z.number().int().min(1).max(720),
   /** TMDB 缓存每类条数上限,超出删最旧(对照 MP CONF.tmdb,缺省 256;大内存模式 1024) */
   tmdbMaxSize: z.number().int().min(16).max(100_000),
-  /** 空结果 / 未识别的短缓存秒数(对照 MP EMPTY_RESULT_CACHE_TTL,缺省 1800) */
+  /** @deprecated CACHE-2 起由 storage.cacheTtl.tmdbMissDays 取代,服务端不再读取 */
   emptyResultTtlSec: z.number().int().min(0).max(86_400),
   /** 图片代理落盘缓存(对照 MP GLOBAL_IMAGE_CACHE,缺省关闭;关闭时仍代理,只是不落盘) */
   globalImageCache: z.boolean(),
@@ -410,7 +413,7 @@ export const DEFAULT_ENRICH_DOMAINS: string[] = [
 
 export const IdentitySettings = z.object({
   eagerTmdbLookup: z.boolean(),
-  /** 待识别池里 unmatched 条目多少天没有新证据就归档(仍可按状态筛出、可手动指定) */
+  /** @deprecated CACHE-2 起由 storage.retentionDays.identityPoolArchive 取代,服务端不再读取 */
   poolArchiveDays: z.number().int().min(1).max(3650),
   /**
    * META-1:订阅别名 / 新作品收录规则命中、TMDB 却查不到时自动尝试一次元数据补全(只出建议,不建作品)。
@@ -432,6 +435,98 @@ export const IdentitySettings = z.object({
 });
 export type IdentitySettings = z.infer<typeof IdentitySettings>;
 
+// ── 存储预算与保留(CACHE-2,docs/runbook/storage.md)──
+
+const Gb = z.number().min(0.01).max(10_000);
+const Days = z.number().int().min(1).max(3650);
+
+export const StorageBudgets = z
+  .object({
+    /** PG 数据目录预算(GB):超过 pgDataWarnGb 告警,超过本值 critical */
+    pgDataGb: Gb,
+    pgDataWarnGb: Gb,
+    /** 图片代理缓存容量(GB):超出按最久未访问删除(LRU) */
+    imagesGb: Gb,
+    /** DATA_DIR/backups 占用预算(GB) */
+    backupsGb: Gb,
+    /** Agent 会话 + 视觉证据(DATA_DIR/agent-sessions、tmp/evidence)预算(GB) */
+    sessionsEvidenceGb: Gb,
+  })
+  .refine((v) => v.pgDataWarnGb <= v.pgDataGb, { message: "PG 告警线不得高于预算" });
+export type StorageBudgets = z.infer<typeof StorageBudgets>;
+
+/** 各类流水数据保留期(到期由 storage.retention.* 定时任务分批删除) */
+export const StorageRetentionDays = z.object({
+  /** observation(Feed 观察) */
+  observation: Days,
+  /** sse_event(SSE 补读) */
+  sseEvent: Days,
+  /** process_heartbeat_sample(小时) */
+  heartbeatHours: z
+    .number()
+    .int()
+    .min(1)
+    .max(24 * 90),
+  /** 已结束的业务任务 task 与引擎 run */
+  taskRun: Days,
+  /** 已结束 Agent 会话的 agent_run_event / agent_run_message */
+  agentEvent: Days,
+  /** audit_event(能力调用审计除外) */
+  audit: Days,
+  /** 能力调用统计:audit_event(target_type=capability)与已结束的 capability_invocation */
+  capabilityStats: Days,
+  /** 待识别池 unmatched 条目无新证据多少天后归档 */
+  identityPoolArchive: Days,
+});
+export type StorageRetentionDays = z.infer<typeof StorageRetentionDays>;
+
+/** 缓存有效期(天):只改 TTL 的取值来源,缓存语义仍对照 MP(docs/porting/moviepilot-tmdb-cache.md) */
+export const StorageCacheTtl = z.object({
+  /** TMDB 搜索 / find 响应与识别结果 */
+  tmdbSearchDays: Days,
+  /** TMDB 详情(/movie/{id}、/tv/{id}、季);追踪中的作品命中时续期 */
+  tmdbDetailDays: Days,
+  /** TMDB 空结果与未识别负缓存(MP EMPTY_RESULT_CACHE_TTL) */
+  tmdbMissDays: Days,
+  /** 站点详情页扩展内容(kv_cache site.detail) */
+  siteDetailDays: Days,
+  /** 网页搜索结果 */
+  webSearchDays: Days,
+});
+export type StorageCacheTtl = z.infer<typeof StorageCacheTtl>;
+
+export const StorageSettings = z.object({
+  budgets: StorageBudgets,
+  retentionDays: StorageRetentionDays,
+  cacheTtl: StorageCacheTtl,
+});
+export type StorageSettings = z.infer<typeof StorageSettings>;
+
+export const DEFAULT_STORAGE_SETTINGS: StorageSettings = {
+  budgets: { pgDataGb: 25, pgDataWarnGb: 20, imagesGb: 20, backupsGb: 20, sessionsEvidenceGb: 10 },
+  retentionDays: {
+    observation: 90,
+    sseEvent: 7,
+    heartbeatHours: 24,
+    taskRun: 60,
+    agentEvent: 90,
+    audit: 180,
+    capabilityStats: 90,
+    identityPoolArchive: 30,
+  },
+  cacheTtl: { tmdbSearchDays: 7, tmdbDetailDays: 30, tmdbMissDays: 1, siteDetailDays: 7, webSearchDays: 1 },
+};
+
+/** 存储设置与缺省逐层合并(旧库里缺子字段时补齐) */
+export function withStorageDefaults(v: unknown): StorageSettings {
+  const o = (v && typeof v === "object" ? v : {}) as Partial<Record<keyof StorageSettings, object>>;
+  return {
+    budgets: { ...DEFAULT_STORAGE_SETTINGS.budgets, ...(o.budgets ?? {}) },
+    retentionDays: { ...DEFAULT_STORAGE_SETTINGS.retentionDays, ...(o.retentionDays ?? {}) },
+    cacheTtl: { ...DEFAULT_STORAGE_SETTINGS.cacheTtl, ...(o.cacheTtl ?? {}) },
+  } as StorageSettings;
+}
+
 export const GlobalSettings = z.object({
   automation: AutomationSettings,
   cache: CacheSettings,
@@ -445,6 +540,7 @@ export const GlobalSettings = z.object({
   naming: NamingSettings,
   identity: IdentitySettings,
   search: SearchSettings,
+  storage: StorageSettings,
 });
 export type GlobalSettings = z.infer<typeof GlobalSettings>;
 
@@ -526,6 +622,7 @@ export const DEFAULT_SETTINGS: GlobalSettings = {
     enrichDomains: DEFAULT_ENRICH_DOMAINS,
   },
   search: { defaultSiteMode: "default", siteIds: [] },
+  storage: DEFAULT_STORAGE_SETTINGS,
 };
 
 /** PATCH /settings:按分组部分更新;服务端与现值合并后用 GlobalSettings 整体再校验 */
@@ -576,6 +673,13 @@ export const GlobalSettingsUpdate = z.object({
   naming: NamingSettings.partial().optional(),
   identity: IdentitySettings.partial().optional(),
   search: SearchSettings.partial().optional(),
+  storage: z
+    .object({
+      budgets: z.object(StorageBudgets.shape).partial().optional(),
+      retentionDays: StorageRetentionDays.partial().optional(),
+      cacheTtl: StorageCacheTtl.partial().optional(),
+    })
+    .optional(),
 });
 export type GlobalSettingsUpdate = z.infer<typeof GlobalSettingsUpdate>;
 
@@ -602,6 +706,16 @@ export function mergeSettings(current: GlobalSettings, patch: GlobalSettingsUpda
     naming: { ...(current.naming ?? DEFAULT_SETTINGS.naming), ...patch.naming },
     identity: { ...(current.identity ?? DEFAULT_SETTINGS.identity), ...patch.identity },
     search: { ...(current.search ?? DEFAULT_SETTINGS.search), ...patch.search },
+    storage: mergeStorage(current.storage, patch.storage),
+  };
+}
+
+function mergeStorage(current: StorageSettings | undefined, patch: GlobalSettingsUpdate["storage"]): StorageSettings {
+  const base = withStorageDefaults(current);
+  return {
+    budgets: { ...base.budgets, ...patch?.budgets },
+    retentionDays: { ...base.retentionDays, ...patch?.retentionDays },
+    cacheTtl: { ...base.cacheTtl, ...patch?.cacheTtl },
   };
 }
 
@@ -638,6 +752,10 @@ export const DEPRECATED_SETTINGS_FIELDS = [
   "pi.model",
   "pi.thinkingLevel",
   "pi.apiKeySecretRef",
+  // CACHE-2:由 storage.cacheTtl / storage.retentionDays 取代
+  "cache.tmdbTtlHours",
+  "cache.emptyResultTtlSec",
+  "identity.poolArchiveDays",
 ] as const;
 
 // ── 设置页分区(docs/design/console-ia.md §3.4) ──
@@ -682,6 +800,7 @@ export const SETTINGS_SECTIONS: readonly SettingsSectionDef[] = [
       { id: "cache", label: "缓存水位" },
       { id: "budget", label: "每日预算" },
       { id: "retention", label: "保留" },
+      { id: "storage", label: "存储与保留" },
     ],
     fields: [
       "automation.enabled",
@@ -695,6 +814,9 @@ export const SETTINGS_SECTIONS: readonly SettingsSectionDef[] = [
       "budget.maxConcurrentDownloads",
       "retention.seedRetentionDays",
       "retention.logRetentionDays",
+      "storage.budgets",
+      "storage.retentionDays",
+      "storage.cacheTtl",
     ],
     resources: [],
   },
@@ -774,11 +896,8 @@ export const SETTINGS_SECTIONS: readonly SettingsSectionDef[] = [
     fields: [
       "metadata.tmdbApiKeySecretRef",
       "metadata.tmdbBaseUrl",
-      "cache.tmdbTtlHours",
       "cache.tmdbMaxSize",
-      "cache.emptyResultTtlSec",
       "identity.eagerTmdbLookup",
-      "identity.poolArchiveDays",
       "identity.autoEnrich",
       "identity.enrichDailyBudget",
       "identity.enrichDomains",

packages/db/migrations/0050_storage_retention.ts

@@ -0,0 +1,79 @@
+import type { Kysely } from "kysely";
+import { execDdl } from "../src/migration-helpers.ts";
+
+// CACHE-2 存储保留(docs/runbook/storage.md):
+// - 保留清理按时间列分批删除,给还没有时间索引的高流水表补上(已有:task_created_idx、agent_run_event_at_idx、
+//   process_heartbeat_sample_at_idx);
+// - 按表设置 autovacuum:批量删除 / 追加为主的大表降低触发比例,频繁原地更新的小表更激进,避免死元组堆积。
+// 引擎 schema(openworkflow)的表由引擎自己建,这里不动。
+
+const INDEXES: [name: string, ddl: string][] = [
+  ["observation_created_idx", "observation (created_at)"],
+  ["sse_event_at_idx", "sse_event (at)"],
+  ["audit_event_at_idx", "audit_event (at)"],
+  ["capability_invocation_created_idx", "capability_invocation (created_at)"],
+  ["agent_run_message_created_idx", "agent_run_message (created_at)"],
+  ["tasks_fact_created_idx", "tasks_fact (created_at)"],
+  ["tasks_run_meta_created_idx", "tasks_run_meta (created_at)"],
+  ["tasks_op_ledger_started_idx", "tasks_op_ledger (started_at) where status = 'succeeded'"],
+];
+
+/** 追加 + 批量删除为主的大表 */
+const APPEND_TABLES = [
+  "observation",
+  "sse_event",
+  "audit_event",
+  "agent_run_event",
+  "process_heartbeat_sample",
+  "capability_invocation",
+  "task",
+  "tasks_fact",
+  "tasks_run_meta",
+];
+const APPEND_OPTS = {
+  autovacuum_vacuum_scale_factor: "0.05",
+  autovacuum_analyze_scale_factor: "0.02",
+  autovacuum_vacuum_insert_scale_factor: "0.1",
+};
+
+/** 频繁原地更新的小表(心跳 15 秒一次、令牌桶、调度状态、缓存命中刷新 accessed_at) */
+const HOT_TABLES = ["process_heartbeat", "rate_limit_bucket", "scheduled_job", "kv_cache", "outbox_event"];
+const HOT_OPTS = {
+  autovacuum_vacuum_scale_factor: "0.02",
+  autovacuum_vacuum_threshold: "100",
+  autovacuum_analyze_scale_factor: "0.05",
+};
+
+const setOpts = (tables: string[], o: Record<string, string>) =>
+  tables
+    .map(
+      (t) =>
+        `alter table ${t} set (${Object.entries(o)
+          .map(([k, v]) => `${k} = ${v}`)
+          .join(", ")});`,
+    )
+    .join("\n");
+const resetOpts = (tables: string[], o: Record<string, string>) =>
+  tables.map((t) => `alter table ${t} reset (${Object.keys(o).join(", ")});`).join("\n");
+
+// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型
+export async function up(db: Kysely<any>): Promise<void> {
+  await execDdl(
+    db,
+    `${INDEXES.map(([n, d]) => `create index if not exists ${n} on ${d};`).join("\n")}
+${setOpts(APPEND_TABLES, APPEND_OPTS)}
+${setOpts(HOT_TABLES, HOT_OPTS)}
+`,
+  );
+}
+
+// biome-ignore lint/suspicious/noExplicitAny: 同上
+export async function down(db: Kysely<any>): Promise<void> {
+  await execDdl(
+    db,
+    `${resetOpts(HOT_TABLES, HOT_OPTS)}
+${resetOpts(APPEND_TABLES, APPEND_OPTS)}
+${INDEXES.map(([n]) => `drop index if exists ${n};`).join("\n")}
+`,
+  );
+}

packages/db/src/cache.ts

@@ -58,6 +58,14 @@ export class PgCache {
     return r.rows.length ? (r.rows[0]!.value as T) : null;
   }
 
+  /** 续期:未过期的条目把 expires_at 推到 now + ttl(只延长不缩短);返回是否有行被续期 */
+  async renew(region: string, key: string, ttl: number): Promise<boolean> {
+    if (ttl <= 0) return false;
+    const r = await sql`update kv_cache set expires_at = greatest(expires_at, now() + make_interval(secs => ${ttl}))
+      where region = ${region} and key = ${key} and expires_at > now()`.execute(this.db);
+    return Number(r.numAffectedRows ?? 0) > 0;
+  }
+
   async exists(region: string, key: string): Promise<boolean> {
     const r =
       await sql`select 1 from kv_cache where region = ${region} and key = ${key} and expires_at > now()`.execute(

packages/db/src/index.ts

@@ -1,5 +1,5 @@
 // 统一从本包取 Kysely 与 sql,避免各包各装一份 kysely
-export { type Kysely, type Selectable, sql, type Transaction } from "kysely";
+export { type Kysely, type RawBuilder, type Selectable, sql, type Transaction } from "kysely";
 export {
   type CachedOptions,
   cachedCall,