diff --git a/apps/server/src/modules/cache/images.ts b/apps/server/src/modules/cache/images.ts index d3219010..dcd961b9 100644 --- a/apps/server/src/modules/cache/images.ts +++ b/apps/server/src/modules/cache/images.ts @@ -9,6 +9,8 @@ import { TMDB_IMAGE_SIZES } from "@umt/contracts"; // - 读缓存只在 use_cache(GLOBAL_IMAGE_CACHE)开启时;拉到的合法图片总是写盘(与 MP 一致); // - 合法图片按内容识别 MIME,拒绝非图片与 SVG;ETag = md5(内容),Cache-Control: public, max-age=7 天,If-None-Match 相同回 304; // - 清理:删除 mtime 早于 GLOBAL_IMAGE_CACHE_DAYS 天的文件与空目录(MP clear_temp → SystemUtils.clear)。 +// - CACHE-2 容量 LRU(我们加的):读盘命中时把 atime 显式写成当前时间(不依赖挂载的 relatime), +// 超过 storage.budgets.imagesGb 时按 atime 从旧到新删到预算的 90%;按天清理保留作兜底。 // 我们只代理 TMDB 图片域(MP SECURITY_IMAGE_DOMAINS 的 TMDB 项),上游失败回 502。 export const TMDB_IMAGE_HOST = "image.tmdb.org"; @@ -51,6 +53,13 @@ export function imageMimeType(b: Uint8Array): string | null { return null; } +interface ImageFile { + path: string; + size: number; + mtimeMs: number; + atimeMs: number; +} + export interface ImageCacheOptions { /** DATA_DIR/cache/images */ dir: string; @@ -104,7 +113,10 @@ export class ImageCache { if (useCache) { const cached = await fs.readFile(path).catch(() => null); const mime = cached?.length ? imageMimeType(cached) : null; - if (cached && mime) return { content: cached, mime }; + if (cached && mime) { + void this.touch(path); + return { content: cached, mime }; + } } let res: Response; try { @@ -130,8 +142,18 @@ export class ImageCache { return { content, mime }; } - private async files(): Promise<{ path: string; size: number; mtimeMs: number }[]> { - const out: { path: string; size: number; mtimeMs: number }[] = []; + /** 记一次访问:atime = 现在,mtime 不变(按天清理仍按写入时间) */ + private async touch(path: string): Promise { + try { + const st = await fs.stat(path); + await fs.utimes(path, new Date(), st.mtime); + } catch { + // 文件刚被清理等:忽略 + } + } + + private async files(): Promise { + const out: ImageFile[] = []; const walk = async (d: string) => { let names: string[]; try { @@ -144,7 +166,7 @@ export class ImageCache { const st = await fs.stat(p).catch(() => null); if (!st) continue; if (st.isDirectory()) await walk(p); - else out.push({ path: p, size: st.size, mtimeMs: st.mtimeMs }); + else if (!n.endsWith(".tmp")) out.push({ path: p, size: st.size, mtimeMs: st.mtimeMs, atimeMs: st.atimeMs }); } }; await walk(this.opts.dir); @@ -156,6 +178,31 @@ export class ImageCache { return { bytes: f.reduce((a, x) => a + x.size, 0), files: f.length }; } + /** + * 容量 LRU:总量超过 maxBytes 时按最近访问(atime)从旧到新删除,直到不超过 maxBytes × lowWater; + * 没超不动。返回删除数、释放字节与清理后用量。 + */ + async enforceBudget( + maxBytes: number, + lowWater = 0.9, + ): Promise<{ deleted: number; freedBytes: number; usedBytes: number }> { + const files = await this.files(); + let used = files.reduce((a, f) => a + f.size, 0); + if (used <= maxBytes) return { deleted: 0, freedBytes: 0, usedBytes: used }; + const target = Math.floor(maxBytes * lowWater); + files.sort((a, b) => a.atimeMs - b.atimeMs || a.mtimeMs - b.mtimeMs); + let deleted = 0; + let freed = 0; + for (const f of files) { + if (used <= target) break; + await fs.rm(f.path, { force: true }); + used -= f.size; + freed += f.size; + deleted++; + } + return { deleted, freedBytes: freed, usedBytes: used }; + } + /** SystemUtils.clear(images, days):删 mtime 早于 days 天的文件,再删空目录;返回删除数 */ async clean(days: number, now = Date.now()): Promise { let deleted = 0; diff --git a/apps/server/src/modules/cache/index.ts b/apps/server/src/modules/cache/index.ts index 59233861..dd0b53d8 100644 --- a/apps/server/src/modules/cache/index.ts +++ b/apps/server/src/modules/cache/index.ts @@ -12,7 +12,7 @@ import type { z } from "zod"; import { dataPaths } from "../../config.ts"; import { requireScope } from "../../http/auth.ts"; import { errorBody } from "../../http/errors.ts"; -import { CACHE_REGIONS, sharedCache, tmdbCachePolicy } from "../../shared-cache.ts"; +import { CACHE_REGIONS, sharedCache, storageSettings, tmdbCachePolicy } from "../../shared-cache.ts"; import { registerScheduledJob } from "../schedules/registry.ts"; import type { ModuleApp, ServerContext, ServerModule } from "../types.ts"; import { ImageCache, ImageProxyError } from "./images.ts"; @@ -22,7 +22,8 @@ import { ImageCache, ImageProxyError } from "./images.ts"; * - PG 缓存 kv_cache(UNLOGGED,移植 MP TTLCache)的过期行物理清理任务 cache.sweep(每 10 分钟); * - TMDB 缓存管理(对照 MP tmdb_cache_items / tmdb_cache_clear):GET /cache/tmdb、POST /cache/tmdb/clear; * - 图片代理 GET /img/tmdb/{size}/{file}(会话或 API Key read,移植 MP ImageHelper / fetch_image)+ DATA_DIR/cache/images 落盘, - * 用量 / 清空 GET /cache/images、POST /cache/images/clear,按保留天数清理的任务 cache.images_clean(每天)。 + * 用量 / 清空 GET /cache/images、POST /cache/images/clear,按保留天数清理的任务 cache.images_clean(每天); + * - CACHE-2:图片缓存容量 LRU 任务 cache.images_lru(30 分钟,预算 storage.budgets.imagesGb,超出删最久未访问)。 */ export interface CacheModuleOptions { /** 图片上游 fetch(测试注入) */ @@ -203,6 +204,26 @@ export function createCacheModule(opts: CacheModuleOptions = {}): ServerModule & return { summary: `删除 ${n} 张过期图片` }; }, }); + registerScheduledJob(ctx, { + key: "cache.images_lru", + defaults: { + name: "图片缓存容量控制", + description: "图片代理缓存超过容量预算(存储设置,缺省 20 GB)时,按最久未访问删除到预算的 90%", + group: "maintain", + schedule: { type: "interval", everyMs: 30 * 60_000 }, + firstRun: "after_interval", + }, + run: async () => { + const { imagesGb } = (await storageSettings(ctx)).budgets; + const r = await images(ctx).enforceBudget(Math.round(imagesGb * 1024 ** 3)); + const gb = (b: number) => `${Math.round((b / 1024 ** 3) * 100) / 100} GB`; + return { + summary: r.deleted + ? `删除 ${r.deleted} 张最久未访问的图片,释放 ${gb(r.freedBytes)},现用 ${gb(r.usedBytes)}` + : `现用 ${gb(r.usedBytes)},未超过预算 ${imagesGb} GB`, + }; + }, + }); }, }; } diff --git a/apps/server/test/cache/cache.test.ts b/apps/server/test/cache/cache.test.ts index fc8fac34..5736b683 100644 --- a/apps/server/test/cache/cache.test.ts +++ b/apps/server/test/cache/cache.test.ts @@ -1,14 +1,14 @@ -import { mkdtempSync, rmSync, utimesSync } from "node:fs"; +import { existsSync, mkdtempSync, rmSync, statSync, utimesSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { PgCache, sql } from "@umt/db"; import { createMockTmdb, type MockTmdb } from "@umt/testkit"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import { imageCachePath } from "../../src/modules/cache/images.ts"; +import { ImageCache, imageCachePath } from "../../src/modules/cache/images.ts"; import { createCacheModule } from "../../src/modules/cache/index.ts"; import { scheduleRegistry } from "../../src/modules/schedules/registry.ts"; import { SourcesRuntime } from "../../src/modules/sources-identity/runtime.ts"; -import { sharedCache } from "../../src/shared-cache.ts"; +import { resetTrackedTmdbRefs, sharedCache } from "../../src/shared-cache.ts"; import { bearer, createHarness, type Harness, pgAvailable } from "../auth-settings/harness.ts"; import { DEV_KEY, json } from "../helpers.ts"; @@ -53,31 +53,77 @@ describe.skipIf(!hasPg)("C2-A 缓存(真实 PG)", () => { rmSync(dir, { recursive: true, force: true }); }); - it("TMDB 请求响应进 kv_cache(tmdb.request,TTL = cache.tmdbTtlHours,条数上限 = cache.tmdbMaxSize),另一个进程直接命中", async () => { + it("TMDB 请求响应进 kv_cache(tmdb.request,TTL 取 storage.cacheTtl:详情 / 搜索 / 未命中,条数上限 = cache.tmdbMaxSize),另一个进程直接命中", async () => { const r = await admin("PATCH", "/settings", { metadata: { tmdbApiKey: "0123456789abcdef0123456789abcdef", tmdbBaseUrl: `${tmdb.url}/3` }, - cache: { tmdbTtlHours: 6, tmdbMaxSize: 16 }, + cache: { tmdbMaxSize: 16 }, + storage: { cacheTtl: { tmdbDetailDays: 20, tmdbSearchDays: 3, tmdbMissDays: 2 } }, }); expect(r.status).toBe(200); + // 部分更新只改给出的子字段,其余保持缺省 + const st = (await json(await admin("GET", "/settings"))).data.storage; + expect(st.cacheTtl).toMatchObject({ tmdbDetailDays: 20, tmdbSearchDays: 3, tmdbMissDays: 2, siteDetailDays: 7 }); + expect(st.budgets.imagesGb).toBe(20); await (await new SourcesRuntime(h.ctx).tmdb())!.details("movie", 872585); const calls = tmdb.calls.length; const rows = ( - await sql<{ key: string; hours: number }>`select key, extract(epoch from expires_at - now()) / 3600 as hours + await sql<{ key: string; days: number }>`select key, extract(epoch from expires_at - now()) / 86400 as days from kv_cache where region = 'tmdb.request'`.execute(h.db) ).rows; - const key = rows.find((x) => x.key.startsWith("/movie/872585?"))!.key; + const detail = rows.find((x) => x.key.startsWith("/movie/872585?"))!; expect(rows.every((x) => !x.key.includes("0123456789abcdef"))).toBe(true); - expect(Number(rows[0]!.hours)).toBeGreaterThan(5.9); - expect(await new PgCache(h.db).get("tmdb.request", key)).toMatchObject({ id: 872585 }); + expect(Number(detail.days)).toBeGreaterThan(19.9); + expect(Number(detail.days)).toBeLessThanOrEqual(20); + expect(await new PgCache(h.db).get("tmdb.request", detail.key)).toMatchObject({ id: 872585 }); await (await new SourcesRuntime(h.ctx).tmdb())!.details("movie", 872585); expect(tmdb.calls.length).toBe(calls); - // 空搜索结果按 EMPTY_RESULT_CACHE_TTL(30 分钟) + const days = async (like: string) => + Number( + ( + await sql<{ d: number }>`select extract(epoch from expires_at - now()) / 86400 as d + from kv_cache where region = 'tmdb.request' and key like ${like}`.execute(h.db) + ).rows[0]!.d, + ); + // 有结果的搜索按 tmdbSearchDays + await (await new SourcesRuntime(h.ctx).tmdb())!.search("Oppenheimer", { type: "movie" }); + const hit = await days("/search/movie?%Oppenheimer%"); + expect(hit).toBeGreaterThan(2.9); + expect(hit).toBeLessThanOrEqual(3); + // 空搜索结果按 EMPTY_RESULT_CACHE_TTL 语义,取 tmdbMissDays await (await new SourcesRuntime(h.ctx).tmdb())!.search("完全不存在的片名c2a", { type: "movie" }); - const empty = ( - await sql<{ m: number }>`select extract(epoch from expires_at - now()) / 60 as m - from kv_cache where region = 'tmdb.request' and key like '/search/movie?%'`.execute(h.db) - ).rows[0]; - expect(Number(empty!.m)).toBeLessThanOrEqual(30); + const empty = await days("/search/movie?%c2a%"); + expect(empty).toBeGreaterThan(1.9); + expect(empty).toBeLessThanOrEqual(2); + }); + + it("追踪中的作品:详情缓存命中时续期到 now + 详情 TTL;未追踪的不动", async () => { + const ent = ( + await sql<{ + id: string; + }>`insert into media_entity (kind, title) values ('movie', '奥本海默') returning id`.execute(h.db) + ).rows[0]!.id; + await sql`insert into external_identity (media_entity_id, provider, entity_type, external_id, status) + values (${ent}, 'tmdb', 'movie', '872585', 'accepted')`.execute(h.db); + await sql`insert into subscription (media_entity_id, media_kind, status) values (${ent}, 'movie', 'active')`.execute( + h.db, + ); + resetTrackedTmdbRefs(h.ctx); + const client = (await new SourcesRuntime(h.ctx).tmdb())!; + await client.details("movie", 872585); + await client.details("movie", 693134); + await sql`update kv_cache set expires_at = now() + interval '1 day' + where region = 'tmdb.request' and (key like '/movie/872585?%' or key like '/movie/693134?%')`.execute(h.db); + await client.details("movie", 872585); + await client.details("movie", 693134); + const left = async (like: string) => + Number( + ( + await sql<{ d: number }>`select extract(epoch from expires_at - now()) / 86400 as d + from kv_cache where region = 'tmdb.request' and key like ${like}`.execute(h.db) + ).rows[0]!.d, + ); + expect(await left("/movie/872585?%")).toBeGreaterThan(19.9); + expect(await left("/movie/693134?%")).toBeLessThanOrEqual(1); }); it("GET /cache/tmdb 列出条目(键 / 过期 / 大小)与上限;POST /cache/tmdb/clear 清空", async () => { @@ -88,7 +134,7 @@ describe.skipIf(!hasPg)("C2-A 缓存(真实 PG)", () => { { ttl: 3600 }, ); const list = await json(await admin("GET", "/cache/tmdb")); - expect(list.data).toMatchObject({ maxSize: 16, ttlHours: 6, emptyResultTtlSec: 1800 }); + expect(list.data).toMatchObject({ maxSize: 16, ttlHours: 20 * 24, emptyResultTtlSec: 2 * 86400 }); expect(list.data.counts.request).toBeGreaterThan(0); const rec = list.data.items.find((i: { kind: string }) => i.kind === "recognize"); expect(rec).toMatchObject({ tmdbId: 872585, title: "奥本海默", mediaType: "movie", posterPath: "/p.jpg" }); @@ -143,4 +189,54 @@ describe.skipIf(!hasPg)("C2-A 缓存(真实 PG)", () => { expect((await json(await admin("GET", "/cache/images"))).data.files).toBe(0); expect((await json(await admin("POST", "/cache/images/clear"))).data.deleted).toBe(0); }); + + it("图片容量 LRU:超过预算按最久未访问(atime)删到预算的 90%;读盘命中刷新 atime、不动 mtime", async () => { + const lruDir = mkdtempSync(join(tmpdir(), "umt-c2-lru-")); + try { + const cache = new ImageCache({ + dir: lruDir, + upstreamBase: "https://image.tmdb.org/t/p", + fetch: async () => new Response(new Uint8Array([...JPEG, ...new Uint8Array(92)])), + }); + // 4 张 100 字节的图,访问时间从旧到新 a < b < c < d + for (const f of ["a", "b", "c", "d"]) await cache.serve("w92", `${f}.jpg`, false, null); + const p = (f: string) => join(lruDir, "t/p/w92", `${f}.jpg`); + const base = Date.now() - 10 * 86_400_000; + ["a", "b", "c", "d"].forEach((f, i) => { + utimesSync(p(f), new Date(base + i * 3600_000), new Date(base)); + }); + // 读盘命中 a:atime 变成现在(最近使用),mtime 仍是写入时间 + await cache.serve("w92", "a.jpg", true, null); + await new Promise((r) => setTimeout(r, 50)); + const st = statSync(p("a")); + expect(st.atimeMs).toBeGreaterThan(Date.now() - 60_000); + expect(Math.abs(st.mtimeMs - base)).toBeLessThan(1000); + // 没超预算不动 + expect(await cache.enforceBudget(1000)).toMatchObject({ deleted: 0, usedBytes: 400 }); + // 预算 300 → 删到 ≤ 270:最久未访问的 b、c 被删,a(刚读过)与 d 保留 + const r = await cache.enforceBudget(300); + expect(r).toMatchObject({ deleted: 2, freedBytes: 200, usedBytes: 200 }); + expect(existsSync(p("a"))).toBe(true); + expect(existsSync(p("b"))).toBe(false); + expect(existsSync(p("c"))).toBe(false); + expect(existsSync(p("d"))).toBe(true); + } finally { + rmSync(lruDir, { recursive: true, force: true }); + } + }); + + it("cache.images_lru 登记为维护任务,预算取 storage.budgets.imagesGb", async () => { + await mod.start?.(h.ctx); + const job = scheduleRegistry(h.ctx).get("cache.images_lru"); + expect(job?.defaults).toMatchObject({ group: "maintain", schedule: { everyMs: 30 * 60_000 } }); + const out = await job!.run({ + jobId: "cache.images_lru", + runId: "r", + trigger: "manual", + now: new Date(), + config: {}, + slot: "1", + }); + expect(out?.summary).toMatch(/未超过预算 20 GB/); + }); });