feat(cache): capacity LRU for image cache (CACHE-2)
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
85b352b8eb
commit
d13c18b4b1
3 files changed
+186
-22
No files matched your search
+51
-4
@@ -9,6 +9,8 @@ import { TMDB_IMAGE_SIZES } from "@umt/contracts";
|
||||
// - 读缓存只在 use_cache(GLOBAL_IMAGE_CACHE)开启时;拉到的合法图片总是写盘(与 MP 一致);
|
||||
// - 合法图片按内容识别 MIME,拒绝非图片与 SVG;ETag = md5(内容),Cache-Control: public, max-age=7 天,If-None-Match 相同回 304;
|
||||
// - 清理:删除 mtime 早于 GLOBAL_IMAGE_CACHE_DAYS 天的文件与空目录(MP clear_temp → SystemUtils.clear)。
|
||||
// - CACHE-2 容量 LRU(我们加的):读盘命中时把 atime 显式写成当前时间(不依赖挂载的 relatime),
|
||||
// 超过 storage.budgets.imagesGb 时按 atime 从旧到新删到预算的 90%;按天清理保留作兜底。
|
||||
// 我们只代理 TMDB 图片域(MP SECURITY_IMAGE_DOMAINS 的 TMDB 项),上游失败回 502。
|
||||
|
||||
export const TMDB_IMAGE_HOST = "image.tmdb.org";
|
||||
@@ -51,6 +53,13 @@ export function imageMimeType(b: Uint8Array): string | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
interface ImageFile {
|
||||
path: string;
|
||||
size: number;
|
||||
mtimeMs: number;
|
||||
atimeMs: number;
|
||||
}
|
||||
|
||||
export interface ImageCacheOptions {
|
||||
/** DATA_DIR/cache/images */
|
||||
dir: string;
|
||||
@@ -104,7 +113,10 @@ export class ImageCache {
|
||||
if (useCache) {
|
||||
const cached = await fs.readFile(path).catch(() => null);
|
||||
const mime = cached?.length ? imageMimeType(cached) : null;
|
||||
if (cached && mime) return { content: cached, mime };
|
||||
if (cached && mime) {
|
||||
void this.touch(path);
|
||||
return { content: cached, mime };
|
||||
}
|
||||
}
|
||||
let res: Response;
|
||||
try {
|
||||
@@ -130,8 +142,18 @@ export class ImageCache {
|
||||
return { content, mime };
|
||||
}
|
||||
|
||||
private async files(): Promise<{ path: string; size: number; mtimeMs: number }[]> {
|
||||
const out: { path: string; size: number; mtimeMs: number }[] = [];
|
||||
/** 记一次访问:atime = 现在,mtime 不变(按天清理仍按写入时间) */
|
||||
private async touch(path: string): Promise<void> {
|
||||
try {
|
||||
const st = await fs.stat(path);
|
||||
await fs.utimes(path, new Date(), st.mtime);
|
||||
} catch {
|
||||
// 文件刚被清理等:忽略
|
||||
}
|
||||
}
|
||||
|
||||
private async files(): Promise<ImageFile[]> {
|
||||
const out: ImageFile[] = [];
|
||||
const walk = async (d: string) => {
|
||||
let names: string[];
|
||||
try {
|
||||
@@ -144,7 +166,7 @@ export class ImageCache {
|
||||
const st = await fs.stat(p).catch(() => null);
|
||||
if (!st) continue;
|
||||
if (st.isDirectory()) await walk(p);
|
||||
else out.push({ path: p, size: st.size, mtimeMs: st.mtimeMs });
|
||||
else if (!n.endsWith(".tmp")) out.push({ path: p, size: st.size, mtimeMs: st.mtimeMs, atimeMs: st.atimeMs });
|
||||
}
|
||||
};
|
||||
await walk(this.opts.dir);
|
||||
@@ -156,6 +178,31 @@ export class ImageCache {
|
||||
return { bytes: f.reduce((a, x) => a + x.size, 0), files: f.length };
|
||||
}
|
||||
|
||||
/**
|
||||
* 容量 LRU:总量超过 maxBytes 时按最近访问(atime)从旧到新删除,直到不超过 maxBytes × lowWater;
|
||||
* 没超不动。返回删除数、释放字节与清理后用量。
|
||||
*/
|
||||
async enforceBudget(
|
||||
maxBytes: number,
|
||||
lowWater = 0.9,
|
||||
): Promise<{ deleted: number; freedBytes: number; usedBytes: number }> {
|
||||
const files = await this.files();
|
||||
let used = files.reduce((a, f) => a + f.size, 0);
|
||||
if (used <= maxBytes) return { deleted: 0, freedBytes: 0, usedBytes: used };
|
||||
const target = Math.floor(maxBytes * lowWater);
|
||||
files.sort((a, b) => a.atimeMs - b.atimeMs || a.mtimeMs - b.mtimeMs);
|
||||
let deleted = 0;
|
||||
let freed = 0;
|
||||
for (const f of files) {
|
||||
if (used <= target) break;
|
||||
await fs.rm(f.path, { force: true });
|
||||
used -= f.size;
|
||||
freed += f.size;
|
||||
deleted++;
|
||||
}
|
||||
return { deleted, freedBytes: freed, usedBytes: used };
|
||||
}
|
||||
|
||||
/** SystemUtils.clear(images, days):删 mtime 早于 days 天的文件,再删空目录;返回删除数 */
|
||||
async clean(days: number, now = Date.now()): Promise<number> {
|
||||
let deleted = 0;
|
||||
|
||||
+23
-2
@@ -12,7 +12,7 @@ import type { z } from "zod";
|
||||
import { dataPaths } from "../../config.ts";
|
||||
import { requireScope } from "../../http/auth.ts";
|
||||
import { errorBody } from "../../http/errors.ts";
|
||||
import { CACHE_REGIONS, sharedCache, tmdbCachePolicy } from "../../shared-cache.ts";
|
||||
import { CACHE_REGIONS, sharedCache, storageSettings, tmdbCachePolicy } from "../../shared-cache.ts";
|
||||
import { registerScheduledJob } from "../schedules/registry.ts";
|
||||
import type { ModuleApp, ServerContext, ServerModule } from "../types.ts";
|
||||
import { ImageCache, ImageProxyError } from "./images.ts";
|
||||
@@ -22,7 +22,8 @@ import { ImageCache, ImageProxyError } from "./images.ts";
|
||||
* - PG 缓存 kv_cache(UNLOGGED,移植 MP TTLCache)的过期行物理清理任务 cache.sweep(每 10 分钟);
|
||||
* - TMDB 缓存管理(对照 MP tmdb_cache_items / tmdb_cache_clear):GET /cache/tmdb、POST /cache/tmdb/clear;
|
||||
* - 图片代理 GET /img/tmdb/{size}/{file}(会话或 API Key read,移植 MP ImageHelper / fetch_image)+ DATA_DIR/cache/images 落盘,
|
||||
* 用量 / 清空 GET /cache/images、POST /cache/images/clear,按保留天数清理的任务 cache.images_clean(每天)。
|
||||
* 用量 / 清空 GET /cache/images、POST /cache/images/clear,按保留天数清理的任务 cache.images_clean(每天);
|
||||
* - CACHE-2:图片缓存容量 LRU 任务 cache.images_lru(30 分钟,预算 storage.budgets.imagesGb,超出删最久未访问)。
|
||||
*/
|
||||
export interface CacheModuleOptions {
|
||||
/** 图片上游 fetch(测试注入) */
|
||||
@@ -203,6 +204,26 @@ export function createCacheModule(opts: CacheModuleOptions = {}): ServerModule &
|
||||
return { summary: `删除 ${n} 张过期图片` };
|
||||
},
|
||||
});
|
||||
registerScheduledJob(ctx, {
|
||||
key: "cache.images_lru",
|
||||
defaults: {
|
||||
name: "图片缓存容量控制",
|
||||
description: "图片代理缓存超过容量预算(存储设置,缺省 20 GB)时,按最久未访问删除到预算的 90%",
|
||||
group: "maintain",
|
||||
schedule: { type: "interval", everyMs: 30 * 60_000 },
|
||||
firstRun: "after_interval",
|
||||
},
|
||||
run: async () => {
|
||||
const { imagesGb } = (await storageSettings(ctx)).budgets;
|
||||
const r = await images(ctx).enforceBudget(Math.round(imagesGb * 1024 ** 3));
|
||||
const gb = (b: number) => `${Math.round((b / 1024 ** 3) * 100) / 100} GB`;
|
||||
return {
|
||||
summary: r.deleted
|
||||
? `删除 ${r.deleted} 张最久未访问的图片,释放 ${gb(r.freedBytes)},现用 ${gb(r.usedBytes)}`
|
||||
: `现用 ${gb(r.usedBytes)},未超过预算 ${imagesGb} GB`,
|
||||
};
|
||||
},
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user