diff --git a/apps/web/app/components/settings/automation.tsx b/apps/web/app/components/settings/automation.tsx index 3b4a98d0..c05a9402 100644 --- a/apps/web/app/components/settings/automation.tsx +++ b/apps/web/app/components/settings/automation.tsx @@ -5,11 +5,11 @@ import { useUpdateSettings } from "~/api/settings.ts"; import { Switch } from "~/components/ui/switch.tsx"; import { errorMessage } from "~/lib/api.ts"; import { cn } from "~/lib/utils.ts"; -import { num } from "./resource-section.tsx"; +import { type FormValues, num } from "./resource-section.tsx"; import { SettingsBlock } from "./settings-block.tsx"; import { type CardSpec, n, SettingsCard, SettingsGate } from "./settings-card.tsx"; -// 自动化分区:总开关、只读对账、缓存水位、每日下载预算、保留天数 +// 自动化分区:总开关、只读对账、缓存水位、每日下载预算、保留天数、存储预算与保留(CACHE-2) const GiB = 1024 ** 3; @@ -89,6 +89,88 @@ const RETENTION_CARD: CardSpec = { }), }; +// CACHE-2:存储预算、流水保留期、缓存有效期(settings.storage,docs/runbook/storage.md) +const gbField = (k: string, label: string, hint?: string) => ({ + name: `storage.budgets.${k}`, + label, + kind: "number" as const, + hint, +}); +const dayField = (group: "retentionDays" | "cacheTtl", k: string, label: string, hint?: string) => ({ + name: `storage.${group}.${k}`, + label, + kind: "number" as const, + hint, +}); +const pick = (v: FormValues, prefix: string, keys: readonly string[]) => + Object.fromEntries(keys.map((k) => [k, num(v[`${prefix}.${k}`])])); + +const BUDGET_KEYS = ["pgDataGb", "pgDataWarnGb", "imagesGb", "backupsGb", "sessionsEvidenceGb"] as const; +const STORAGE_BUDGET_CARD: CardSpec = { + anchor: "storage", + title: "存储预算", + description: "超过预算时健康页告警；图片缓存超出容量按最久未访问删除。单位 GB。", + fields: [ + gbField("pgDataGb", "数据库预算", "超过为严重"), + gbField("pgDataWarnGb", "数据库告警线", "不高于数据库预算"), + gbField("imagesGb", "图片缓存容量"), + gbField("backupsGb", "备份目录预算"), + gbField("sessionsEvidenceGb", "Agent 会话与截图证据预算"), + ], + fromSettings: (s) => Object.fromEntries(BUDGET_KEYS.map((k) => [`storage.budgets.${k}`, n(s.storage.budgets[k])])), + toPatch: (v) => ({ storage: { budgets: pick(v, "storage.budgets", BUDGET_KEYS) } }), +}; + +const RETENTION_KEYS = [ + "observation", + "sseEvent", + "heartbeatHours", + "taskRun", + "agentEvent", + "audit", + "capabilityStats", + "identityPoolArchive", +] as const; +const STORAGE_RETENTION_CARD: CardSpec = { + anchor: "storage-retention", + title: "流水数据保留", + description: "到期由定时清理任务分批删除。单位天，心跳为小时。", + fields: [ + dayField("retentionDays", "observation", "Feed 观察"), + dayField("retentionDays", "sseEvent", "实时事件"), + dayField("retentionDays", "heartbeatHours", "进程心跳（小时）"), + dayField("retentionDays", "taskRun", "任务运行记录"), + dayField("retentionDays", "agentEvent", "Agent 会话事件", "只删已结束的会话"), + dayField("retentionDays", "audit", "审计日志", "能力调用审计除外"), + dayField("retentionDays", "capabilityStats", "能力调用统计"), + dayField( + "retentionDays", + "identityPoolArchive", + "待识别池归档", + "未命中的条目这么久没有新证据就归档，仍可手动指定", + ), + ], + fromSettings: (s) => + Object.fromEntries(RETENTION_KEYS.map((k) => [`storage.retentionDays.${k}`, n(s.storage.retentionDays[k])])), + toPatch: (v) => ({ storage: { retentionDays: pick(v, "storage.retentionDays", RETENTION_KEYS) } }), +}; + +const TTL_KEYS = ["tmdbSearchDays", "tmdbDetailDays", "tmdbMissDays", "siteDetailDays", "webSearchDays"] as const; +const STORAGE_TTL_CARD: CardSpec = { + anchor: "storage-ttl", + title: "缓存有效期", + description: "单位天。追踪中的作品命中 TMDB 详情缓存时自动续期。", + fields: [ + dayField("cacheTtl", "tmdbSearchDays", "TMDB 搜索与识别结果"), + dayField("cacheTtl", "tmdbDetailDays", "TMDB 详情"), + dayField("cacheTtl", "tmdbMissDays", "TMDB 空结果", "搜索无结果、未识别时的负缓存"), + dayField("cacheTtl", "siteDetailDays", "站点详情页内容"), + dayField("cacheTtl", "webSearchDays", "网页搜索结果"), + ], + fromSettings: (s) => Object.fromEntries(TTL_KEYS.map((k) => [`storage.cacheTtl.${k}`, n(s.storage.cacheTtl[k])])), + toPatch: (v) => ({ storage: { cacheTtl: pick(v, "storage.cacheTtl", TTL_KEYS) } }), +}; + function ToggleRow({ id, icon, @@ -181,6 +263,9 @@ export function AutomationSection() { + + + )} diff --git a/apps/web/app/components/settings/metadata.tsx b/apps/web/app/components/settings/metadata.tsx index a5fc7381..f253d33d 100644 --- a/apps/web/app/components/settings/metadata.tsx +++ b/apps/web/app/components/settings/metadata.tsx @@ -102,27 +102,15 @@ function TmdbKeyStatus({ saved }: { saved: boolean }) { const TMDB_CACHE_CARD: CardSpec = { anchor: "tmdb-cache", title: "TMDB 缓存", - description: "请求响应与识别结果存在数据库里，三个进程共用。超过条数上限时删最久未用的条目。", - fields: [ - { name: "cache.tmdbTtlHours", label: "有效期（小时）", kind: "number" }, - { name: "cache.tmdbMaxSize", label: "条数上限", kind: "number", hint: "请求缓存和识别缓存各自计数" }, - { - name: "cache.emptyResultTtlSec", - label: "空结果有效期（秒）", - kind: "number", - hint: "搜索无结果、未识别时的短缓存", - }, - ], + description: + "请求响应与识别结果存在数据库里，三个进程共用。超过条数上限时删最久未用的条目；有效期在「自动化 → 存储与保留」里设。", + fields: [{ name: "cache.tmdbMaxSize", label: "条数上限", kind: "number", hint: "请求缓存和识别缓存各自计数" }], fromSettings: (s) => ({ - "cache.tmdbTtlHours": n(s.cache.tmdbTtlHours), "cache.tmdbMaxSize": n(s.cache.tmdbMaxSize), - "cache.emptyResultTtlSec": n(s.cache.emptyResultTtlSec), }), toPatch: (v) => ({ cache: { - tmdbTtlHours: Number(v["cache.tmdbTtlHours"]), tmdbMaxSize: Number(v["cache.tmdbMaxSize"]), - emptyResultTtlSec: Number(v["cache.emptyResultTtlSec"]), }, }), extra: () => , @@ -224,12 +212,6 @@ const IDENTITY_CARD: CardSpec = { hint: "旧行为。每条资源约 1–2 次搜索 + 命中时 1 次详情（识别缓存命中为 0）；每小时 100 条的 RSS 约每天 2400–7200 次请求，站点独有的短剧 / 综艺多半查不到", wide: true, }, - { - name: "identity.poolArchiveDays", - label: "待识别池归档（天）", - kind: "number", - hint: "未命中订阅的条目超过这么多天没有新证据就归档，仍可筛出并手动指定", - }, { name: "identity.autoEnrich", label: "TMDB 查不到时自动补全元数据", @@ -253,7 +235,6 @@ const IDENTITY_CARD: CardSpec = { ], fromSettings: (s) => ({ "identity.eagerTmdbLookup": s.identity.eagerTmdbLookup, - "identity.poolArchiveDays": n(s.identity.poolArchiveDays), "identity.autoEnrich": s.identity.autoEnrich, "identity.enrichDailyBudget": n(s.identity.enrichDailyBudget), "identity.enrichDomains": s.identity.enrichDomains.join("\n"), @@ -261,7 +242,6 @@ const IDENTITY_CARD: CardSpec = { toPatch: (v) => ({ identity: { eagerTmdbLookup: v["identity.eagerTmdbLookup"] === true, - poolArchiveDays: Number(v["identity.poolArchiveDays"]), autoEnrich: v["identity.autoEnrich"] === true, enrichDailyBudget: Number(v["identity.enrichDailyBudget"]), enrichDomains: String(v["identity.enrichDomains"] ?? "") diff --git a/docs/porting/moviepilot-tmdb-cache.md b/docs/porting/moviepilot-tmdb-cache.md index f649ecd3..56f4f1a1 100644 --- a/docs/porting/moviepilot-tmdb-cache.md +++ b/docs/porting/moviepilot-tmdb-cache.md @@ -23,7 +23,7 @@ PG 读写失败时 TMDB 路径按未命中处理(内存缓存不会失败)。 | 上游 | 我们 | |---|---| -| `maxsize = CONF.tmdb`(256;大内存 1024) | 设置 `cache.tmdbMaxSize`(缺省 256) | +| `maxsize = CONF.tmdb`(256;大内存 1024) | 设置 `cache.tmdbMaxSize`(缺省 5000,存 PG 不受进程内存约束) | | `ttl = CONF.meta = META_CACHE_EXPIRE 小时`(缺省 24) | CACHE-2 起按请求类别取设置:`/search/`、`/find/` 用 `storage.cacheTtl.tmdbSearchDays`(7 天),其余用 `tmdbDetailDays`(30 天) | | `empty_ttl = EMPTY_RESULT_CACHE_TTL`(1800 秒),`empty_if = results 为空列表` | `storage.cacheTtl.tmdbMissDays`(1 天);`isEmptyResult` 同判定 | | `skip_none`(连接失败不缓存)、`skip_if = success:false`(业务失败不缓存) | 这两类在 `fetchWithRetry` 抛错,不会写缓存 | diff --git a/docs/runbook/cache.md b/docs/runbook/cache.md index c6f2c610..176bd935 100644 --- a/docs/runbook/cache.md +++ b/docs/runbook/cache.md @@ -31,7 +31,7 @@ TTL 设置(CACHE-2 起):`settings.storage.cacheTtl`,见 [storage.md](storage.md)。MP 的 `META_CACHE_EXPIRE` 只有一个值, 我们按请求类别拆成搜索 / 详情两档,`EMPTY_RESULT_CACHE_TTL` 对应 `tmdbMissDays`;缓存语义不变。 -条数上限仍是 `cache.tmdbMaxSize`(缺省 256,MP `CONF.tmdb`)。旧字段 `cache.tmdbTtlHours`、`cache.emptyResultTtlSec` 不再读取。 +条数上限仍是 `cache.tmdbMaxSize`(缺省 5000;MP `CONF.tmdb` 为 256,本项目缓存在 PG 里不占进程内存)。旧字段 `cache.tmdbTtlHours`、`cache.emptyResultTtlSec` 不再读取。 API:`GET /api/v1/cache/tmdb?kind=recognize|request&limit=` 列出条目(键 / 过期 / 大小,识别条目带 tmdbId / 标题 / 海报); `POST /api/v1/cache/tmdb/clear` 清空两类(仅 admin)。 @@ -63,5 +63,5 @@ API:`GET /api/v1/cache/tmdb?kind=recognize|request&limit=` 列出条目(键 / - TMDB 缓存不生效:确认 `select region, count(*) from kv_cache group by 1` 有 `tmdb.request`;PG 读写失败时按未命中处理并记 `TMDB 缓存读写失败，按未命中处理`。 - 改了 TMDB 缓存设置:新写入的条目按新 TTL / 上限;已有条目保持原过期时间,必要时清空。 -- TTL 拉长后条目更早被条数上限挤掉:`cache.tmdbMaxSize` 缺省只有 256,追踪作品多时调大。 +- TTL 拉长后条目更早被条数上限挤掉:`cache.tmdbMaxSize` 缺省 5000,追踪作品很多时再调大。 - 数据库重启后缓存为空:UNLOGGED 表的正常行为。 diff --git a/docs/runbook/storage.md b/docs/runbook/storage.md index fa79de6f..5b0ca3a7 100644 --- a/docs/runbook/storage.md +++ b/docs/runbook/storage.md @@ -34,7 +34,7 @@ PATCH /api/v1/settings {"storage":{"retentionDays":{"observation":60}}} 被取代的旧字段(保留在 schema 里兼容已存 JSON,服务端不再读取,已列入 `DEPRECATED_SETTINGS_FIELDS`): `cache.tmdbTtlHours` → `cacheTtl.tmdbSearchDays / tmdbDetailDays`,`cache.emptyResultTtlSec` → `cacheTtl.tmdbMissDays`, `identity.poolArchiveDays` → `retentionDays.identityPoolArchive`。设置页「元数据」里这三项接 UI 时要换成新字段。 -`cache.tmdbMaxSize`(条数上限,缺省 256)仍然有效。TTL 拉长后条数上限会先起作用,需要时调大。 +`cache.tmdbMaxSize`(条数上限,缺省 5000)仍然有效。TTL 拉长后条数上限会先起作用,需要时调大。 ## 缓存有效期 diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 1d71866a..173e034f 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -29,7 +29,7 @@ const CacheShape = z.object({ * 保留在 schema 里只为兼容已存的 JSON 与旧设置页。 */ tmdbTtlHours: z.number().int().min(1).max(720), - /** TMDB 缓存每类条数上限,超出删最旧(对照 MP CONF.tmdb,缺省 256;大内存模式 1024) */ + /** TMDB 缓存每类条数上限,超出删最旧(MP CONF.tmdb 为 256 / 大内存 1024;本项目存 PG 不占进程内存,缺省 5000) */ tmdbMaxSize: z.number().int().min(16).max(100_000), /** @deprecated CACHE-2 起由 storage.cacheTtl.tmdbMissDays 取代,服务端不再读取 */ emptyResultTtlSec: z.number().int().min(0).max(86_400), @@ -566,7 +566,7 @@ export const DEFAULT_SETTINGS: GlobalSettings = { checkIntervalSec: 60, cleanupTargetPercent: 80, tmdbTtlHours: 24, - tmdbMaxSize: 256, + tmdbMaxSize: 5000, emptyResultTtlSec: 1800, globalImageCache: false, globalImageCacheDays: 7, diff --git a/packages/db/migrations/0051_tmdb_cache_size.ts b/packages/db/migrations/0051_tmdb_cache_size.ts new file mode 100644 index 00000000..5bb52f83 --- /dev/null +++ b/packages/db/migrations/0051_tmdb_cache_size.ts @@ -0,0 +1,23 @@ +import type { Kysely } from "kysely"; +import { execDdl } from "../src/migration-helpers.ts"; + +// INT-3:cache.tmdbMaxSize 缺省 256 → 5000(CACHE-2 把 TTL 拉长到天级后,256 条先被挤掉)。 +// settings 表里整段存 cache 对象,已保存过设置的库仍是旧缺省 256:只改仍等于旧缺省的值,用户改过的不动。 + +// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型 +export async function up(db: Kysely): Promise { + await execDdl( + db, + `update settings set value = jsonb_set(value, '{tmdbMaxSize}', '5000'::jsonb), updated_at = now() + where key = 'cache' and value->'tmdbMaxSize' = '256'::jsonb;`, + ); +} + +// biome-ignore lint/suspicious/noExplicitAny: 同上 +export async function down(db: Kysely): Promise { + await execDdl( + db, + `update settings set value = jsonb_set(value, '{tmdbMaxSize}', '256'::jsonb), updated_at = now() + where key = 'cache' and value->'tmdbMaxSize' = '5000'::jsonb;`, + ); +}