feat(cache): TMDB image proxy, posters, cache admin API and settings cards
GET /img/tmdb/{size}/{file} ported from MP ImageHelper / fetch_image
(sanitized cache path under DATA_DIR/cache/images, GLOBAL_IMAGE_CACHE
read switch, md5 ETag / 304, day-based cleanup job). Cache module adds
cache.sweep / cache.images_clean jobs and /cache/tmdb, /cache/images
list and clear endpoints. Library list/detail and the search palette
show posters with the neutral placeholder as fallback; metadata settings
get TMDB cache and image cache cards. Runbook docs/runbook/cache.md and
porting notes docs/porting/moviepilot-tmdb-cache.md.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
946fca5e55
commit
7862573944
16 files changed
+1028
-9
No files matched your search
@@ -150,6 +150,7 @@ export function registerLibraryRoutes(app: ModuleApp, d: AoDeps) {
|
||||
where (s.media_entity_id = e.id or sr.media_entity_id = e.id) and dt.removed_at is null
|
||||
and dt.status in ('queued','submitting','downloading','stalled','paused'))::int`.as("active_downloads"),
|
||||
)
|
||||
.select(sql<string | null>`e.metadata #>> '{posterPath,value}'`.as("poster_path"))
|
||||
.select(
|
||||
sql<Date | null>`(select max(r.finished_at) from organization_record r where r.media_entity_id = e.id and r.status in ('succeeded','partial'))`.as(
|
||||
"last_org",
|
||||
@@ -220,6 +221,7 @@ export function registerLibraryRoutes(app: ModuleApp, d: AoDeps) {
|
||||
slotCounts: { ...emptyCounts(), ...(r.slot_counts ?? {}) },
|
||||
activeDownloads: r.active_downloads,
|
||||
lastOrganizedAt: r.last_org ? new Date(r.last_org).toISOString() : null,
|
||||
posterPath: r.poster_path ?? null,
|
||||
}));
|
||||
return { items, nextCursor: p.nextCursor };
|
||||
},
|
||||
|
||||
+189
@@ -0,0 +1,189 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { promises as fs } from "node:fs";
|
||||
import { dirname, extname, join } from "node:path";
|
||||
import { TMDB_IMAGE_SIZES } from "@umt/contracts";
|
||||
|
||||
// TMDB 图片代理与文件缓存,移植 MP app/application/image.py(ImageHelper)+ api/endpoints/system.py(fetch_image):
|
||||
// - 缓存路径 = SecurityUtils.sanitize_url_path(url):URL 路径去掉开头 "/" 后百分号编码,超过 120 字符换成
|
||||
// compressed_<sha256 前 16 位><扩展名>;没有扩展名补 .jpg;落在 DATA_DIR/cache/images/ 下(MP CACHE_PATH/images);
|
||||
// - 读缓存只在 use_cache(GLOBAL_IMAGE_CACHE)开启时;拉到的合法图片总是写盘(与 MP 一致);
|
||||
// - 合法图片按内容识别 MIME,拒绝非图片与 SVG;ETag = md5(内容),Cache-Control: public, max-age=7 天,If-None-Match 相同回 304;
|
||||
// - 清理:删除 mtime 早于 GLOBAL_IMAGE_CACHE_DAYS 天的文件与空目录(MP clear_temp → SystemUtils.clear)。
|
||||
// 我们只代理 TMDB 图片域(MP SECURITY_IMAGE_DOMAINS 的 TMDB 项),上游失败回 502。
|
||||
|
||||
export const TMDB_IMAGE_HOST = "image.tmdb.org";
|
||||
const FILE_RE = /^[A-Za-z0-9_-]+\.(jpg|jpeg|png|webp)$/;
|
||||
|
||||
export class ImageProxyError extends Error {
|
||||
constructor(
|
||||
readonly status: 400 | 502,
|
||||
message: string,
|
||||
) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
/** SecurityUtils.sanitize_url_path + ImageHelper._prepare_cache_path */
|
||||
export function imageCachePath(url: string, maxLength = 120): string {
|
||||
const path = new URL(url).pathname.replace(/^\/+/, "");
|
||||
let safe = path
|
||||
.split("/")
|
||||
.map((s) => encodeURIComponent(decodeURIComponent(s)))
|
||||
.join("/");
|
||||
if (safe.length > maxLength) {
|
||||
const hash = createHash("sha256").update(safe).digest("hex").slice(0, 16);
|
||||
safe = `compressed_${hash}${extname(safe).toLowerCase()}`;
|
||||
}
|
||||
return extname(safe) ? safe : `${safe}.jpg`;
|
||||
}
|
||||
|
||||
/** ImageHelper.get_image_mime_type(按文件头识别;非图片 / SVG 返回 null) */
|
||||
export function imageMimeType(b: Uint8Array): string | null {
|
||||
if (b.length >= 3 && b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff) return "image/jpeg";
|
||||
if (b.length >= 8 && b[0] === 0x89 && b[1] === 0x50 && b[2] === 0x4e && b[3] === 0x47) return "image/png";
|
||||
if (
|
||||
b.length >= 12 &&
|
||||
String.fromCharCode(...b.slice(0, 4)) === "RIFF" &&
|
||||
String.fromCharCode(...b.slice(8, 12)) === "WEBP"
|
||||
)
|
||||
return "image/webp";
|
||||
if (b.length >= 6 && String.fromCharCode(...b.slice(0, 4)) === "GIF8") return "image/gif";
|
||||
return null;
|
||||
}
|
||||
|
||||
export interface ImageCacheOptions {
|
||||
/** DATA_DIR/cache/images */
|
||||
dir: string;
|
||||
fetch?: typeof globalThis.fetch;
|
||||
timeoutMs?: number;
|
||||
/** 上游基址(测试指向 mock);缺省 https://image.tmdb.org/t/p */
|
||||
upstreamBase?: string;
|
||||
}
|
||||
|
||||
export class ImageCache {
|
||||
private readonly fetchImpl: typeof globalThis.fetch;
|
||||
|
||||
constructor(private readonly opts: ImageCacheOptions) {
|
||||
this.fetchImpl = opts.fetch ?? globalThis.fetch;
|
||||
}
|
||||
|
||||
get dir(): string {
|
||||
return this.opts.dir;
|
||||
}
|
||||
|
||||
/** 尺寸白名单 + 文件名校验后拼上游 URL(只允许 TMDB 图片域) */
|
||||
upstream(size: string, file: string): string {
|
||||
if (!(TMDB_IMAGE_SIZES as readonly string[]).includes(size)) throw new ImageProxyError(400, "不支持的图片尺寸");
|
||||
if (!FILE_RE.test(file)) throw new ImageProxyError(400, "图片路径不合法");
|
||||
const base = (this.opts.upstreamBase ?? `https://${TMDB_IMAGE_HOST}/t/p`).replace(/\/+$/, "");
|
||||
const url = `${base}/${size}/${file}`;
|
||||
if (!this.opts.upstreamBase && new URL(url).hostname !== TMDB_IMAGE_HOST) {
|
||||
throw new ImageProxyError(400, "只代理 TMDB 图片");
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
/** fetch_image:取图(可读缓存)→ ETag / 304 */
|
||||
async serve(size: string, file: string, useCache: boolean, ifNoneMatch: string | null): Promise<Response> {
|
||||
const url = this.upstream(size, file);
|
||||
const { content, mime } = await this.fetchImage(url, useCache);
|
||||
const etag = createHash("md5").update(content).digest("hex");
|
||||
const headers = new Headers({
|
||||
etag,
|
||||
"cache-control": `public, max-age=${86400 * 7}`,
|
||||
"content-type": mime,
|
||||
"x-content-type-options": "nosniff",
|
||||
});
|
||||
if (ifNoneMatch === etag) return new Response(null, { status: 304, headers });
|
||||
return new Response(content as BodyInit, { status: 200, headers });
|
||||
}
|
||||
|
||||
/** ImageHelper.fetch_image_with_mime_type */
|
||||
private async fetchImage(url: string, useCache: boolean): Promise<{ content: Uint8Array; mime: string }> {
|
||||
const path = join(this.opts.dir, imageCachePath(url));
|
||||
if (useCache) {
|
||||
const cached = await fs.readFile(path).catch(() => null);
|
||||
const mime = cached?.length ? imageMimeType(cached) : null;
|
||||
if (cached && mime) return { content: cached, mime };
|
||||
}
|
||||
let res: Response;
|
||||
try {
|
||||
res = await this.fetchImpl(url, {
|
||||
signal: AbortSignal.timeout(this.opts.timeoutMs ?? 20_000),
|
||||
headers: { accept: "image/avif,image/webp,image/apng,*/*" },
|
||||
});
|
||||
} catch (e) {
|
||||
const reason = (e as Error).name === "TimeoutError" ? "超时" : (e as Error).message;
|
||||
throw new ImageProxyError(502, `拉取 TMDB 图片失败：${reason}`);
|
||||
}
|
||||
if (res.status !== 200) {
|
||||
await res.body?.cancel().catch(() => {});
|
||||
throw new ImageProxyError(502, `TMDB 图片返回 HTTP ${res.status}`);
|
||||
}
|
||||
const content = new Uint8Array(await res.arrayBuffer());
|
||||
const mime = imageMimeType(content);
|
||||
if (!mime) throw new ImageProxyError(502, "TMDB 返回的不是图片");
|
||||
await fs.mkdir(dirname(path), { recursive: true });
|
||||
const tmp = `${path}.${process.pid}.tmp`;
|
||||
await fs.writeFile(tmp, content);
|
||||
await fs.rename(tmp, path);
|
||||
return { content, mime };
|
||||
}
|
||||
|
||||
private async files(): Promise<{ path: string; size: number; mtimeMs: number }[]> {
|
||||
const out: { path: string; size: number; mtimeMs: number }[] = [];
|
||||
const walk = async (d: string) => {
|
||||
let names: string[];
|
||||
try {
|
||||
names = await fs.readdir(d);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
for (const n of names) {
|
||||
const p = join(d, n);
|
||||
const st = await fs.stat(p).catch(() => null);
|
||||
if (!st) continue;
|
||||
if (st.isDirectory()) await walk(p);
|
||||
else out.push({ path: p, size: st.size, mtimeMs: st.mtimeMs });
|
||||
}
|
||||
};
|
||||
await walk(this.opts.dir);
|
||||
return out;
|
||||
}
|
||||
|
||||
async usage(): Promise<{ bytes: number; files: number }> {
|
||||
const f = await this.files();
|
||||
return { bytes: f.reduce((a, x) => a + x.size, 0), files: f.length };
|
||||
}
|
||||
|
||||
/** SystemUtils.clear(images, days):删 mtime 早于 days 天的文件,再删空目录;返回删除数 */
|
||||
async clean(days: number, now = Date.now()): Promise<number> {
|
||||
let deleted = 0;
|
||||
for (const f of await this.files()) {
|
||||
if (Math.floor((now - f.mtimeMs) / 86_400_000) > days) {
|
||||
await fs.rm(f.path, { force: true });
|
||||
deleted++;
|
||||
}
|
||||
}
|
||||
const prune = async (d: string): Promise<boolean> => {
|
||||
const names = await fs.readdir(d).catch(() => null);
|
||||
if (!names) return false;
|
||||
let empty = true;
|
||||
for (const n of names) {
|
||||
const p = join(d, n);
|
||||
const st = await fs.stat(p).catch(() => null);
|
||||
if (st?.isDirectory() && (await prune(p))) await fs.rmdir(p).catch(() => {});
|
||||
else empty = false;
|
||||
}
|
||||
return empty;
|
||||
};
|
||||
await prune(this.opts.dir);
|
||||
return deleted;
|
||||
}
|
||||
|
||||
async clear(): Promise<number> {
|
||||
const n = (await this.files()).length;
|
||||
await fs.rm(this.opts.dir, { recursive: true, force: true });
|
||||
return n;
|
||||
}
|
||||
}
|
||||
+210
@@ -0,0 +1,210 @@
|
||||
import { join } from "node:path";
|
||||
import { createRoute, type RouteConfig } from "@hono/zod-openapi";
|
||||
import {
|
||||
Api,
|
||||
apiResponse,
|
||||
DEFAULT_SETTINGS,
|
||||
type EndpointDef,
|
||||
type TmdbCacheItem,
|
||||
type TmdbCacheList,
|
||||
} from "@umt/contracts";
|
||||
import type { z } from "zod";
|
||||
import { dataPaths } from "../../config.ts";
|
||||
import { requireScope } from "../../http/auth.ts";
|
||||
import { errorBody } from "../../http/errors.ts";
|
||||
import { CACHE_REGIONS, sharedCache, tmdbCachePolicy } from "../../shared-cache.ts";
|
||||
import { registerScheduledJob } from "../schedules/registry.ts";
|
||||
import type { ModuleApp, ServerContext, ServerModule } from "../types.ts";
|
||||
import { ImageCache, ImageProxyError } from "./images.ts";
|
||||
|
||||
/**
|
||||
* C2-A 缓存(docs/runbook/cache.md):
|
||||
* - PG 缓存 kv_cache(UNLOGGED,移植 MP TTLCache)的过期行物理清理任务 cache.sweep(每 10 分钟);
|
||||
* - TMDB 缓存管理(对照 MP tmdb_cache_items / tmdb_cache_clear):GET /cache/tmdb、POST /cache/tmdb/clear;
|
||||
* - 图片代理 GET /img/tmdb/{size}/{file}(会话或 API Key read,移植 MP ImageHelper / fetch_image)+ DATA_DIR/cache/images 落盘,
|
||||
* 用量 / 清空 GET /cache/images、POST /cache/images/clear,按保留天数清理的任务 cache.images_clean(每天)。
|
||||
*/
|
||||
export interface CacheModuleOptions {
|
||||
/** 图片上游 fetch(测试注入) */
|
||||
imageFetch?: typeof globalThis.fetch;
|
||||
/** 图片上游基址(测试指向 mock) */
|
||||
imageUpstreamBase?: string;
|
||||
/** 图片缓存目录;缺省 DATA_DIR/cache/images */
|
||||
imageDir?: string;
|
||||
}
|
||||
|
||||
type Infer<T> = T extends z.ZodType ? z.infer<T> : Record<string, never>;
|
||||
|
||||
function route<D extends EndpointDef>(
|
||||
app: ModuleApp,
|
||||
def: D,
|
||||
handler: (i: { query: Infer<D["query"]> }) => Promise<unknown>,
|
||||
) {
|
||||
const request: RouteConfig["request"] = {};
|
||||
if (def.query) request.query = def.query as never;
|
||||
const r = createRoute({
|
||||
method: def.method,
|
||||
path: def.path,
|
||||
summary: def.summary,
|
||||
tags: ["cache"],
|
||||
middleware: [requireScope(def.scope)] as never,
|
||||
request,
|
||||
responses: {
|
||||
200: { description: def.summary, content: { "application/json": { schema: apiResponse(def.response) } } },
|
||||
},
|
||||
});
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 路由泛型在封装里擦除,入参已由 zod-openapi 校验
|
||||
app.openapi(r, (async (c: any) => {
|
||||
const data = def.response.parse(await handler({ query: def.query ? c.req.valid("query") : {} } as never));
|
||||
return c.json({ ok: true, data }, 200);
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 同上
|
||||
}) as any);
|
||||
}
|
||||
|
||||
const str = (v: unknown) => (typeof v === "string" && v ? v : null);
|
||||
|
||||
export function createCacheModule(opts: CacheModuleOptions = {}): ServerModule & {
|
||||
images(ctx: ServerContext): ImageCache;
|
||||
} {
|
||||
const imageCaches = new WeakMap<object, ImageCache>();
|
||||
const images = (ctx: ServerContext) => {
|
||||
let c = imageCaches.get(ctx);
|
||||
if (!c) {
|
||||
c = new ImageCache({
|
||||
dir: opts.imageDir ?? join(dataPaths(ctx.config).dataDir, "cache", "images"),
|
||||
...(opts.imageFetch ? { fetch: opts.imageFetch } : {}),
|
||||
...(opts.imageUpstreamBase ? { upstreamBase: opts.imageUpstreamBase } : {}),
|
||||
});
|
||||
imageCaches.set(ctx, c);
|
||||
}
|
||||
return c;
|
||||
};
|
||||
const cacheSettings = (ctx: ServerContext) => ctx.settings.get("cache").catch(() => DEFAULT_SETTINGS.cache);
|
||||
|
||||
return {
|
||||
name: "cache",
|
||||
images,
|
||||
registerRoutes(app, ctx) {
|
||||
// 图片代理:不进 OpenAPI(二进制响应);<img> 带 cookie 会话即可,API Key 需 read
|
||||
app.get("/img/tmdb/:size/:file", requireScope("read"), async (c) => {
|
||||
const s = await cacheSettings(ctx);
|
||||
try {
|
||||
// MP cache_img:GLOBAL_IMAGE_CACHE 开启才读磁盘缓存
|
||||
return await images(ctx).serve(
|
||||
c.req.param("size"),
|
||||
c.req.param("file"),
|
||||
s.globalImageCache ?? false,
|
||||
c.req.header("if-none-match") ?? null,
|
||||
);
|
||||
} catch (e) {
|
||||
if (e instanceof ImageProxyError) {
|
||||
return c.json(errorBody(e.status === 400 ? "bad_request" : "upstream_unavailable", e.message), e.status);
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
});
|
||||
|
||||
route(app, Api.cache.tmdb, async ({ query }) => {
|
||||
const cache = sharedCache(ctx);
|
||||
const policy = await tmdbCachePolicy(ctx);
|
||||
const want = (k: "request" | "recognize") => !query.kind || query.kind === k;
|
||||
const items: TmdbCacheItem[] = [];
|
||||
if (want("recognize")) {
|
||||
for (const r of await cache.items<Record<string, unknown>>(CACHE_REGIONS.tmdbRecognize, query.limit)) {
|
||||
const v = r.value ?? {};
|
||||
const t = v.type === "movie" || v.type === "tv" ? v.type : "unknown";
|
||||
items.push({
|
||||
kind: "recognize",
|
||||
key: r.key,
|
||||
expiresAt: r.expiresAt.toISOString(),
|
||||
updatedAt: r.updatedAt.toISOString(),
|
||||
bytes: r.bytes,
|
||||
tmdbId: typeof v.id === "number" ? v.id : Number(v.id) || 0,
|
||||
title: str(v.title),
|
||||
year: str(v.year),
|
||||
mediaType: t,
|
||||
posterPath: str(v.poster_path),
|
||||
});
|
||||
}
|
||||
}
|
||||
if (want("request")) {
|
||||
for (const r of await cache.items(CACHE_REGIONS.tmdbRequest, query.limit)) {
|
||||
items.push({
|
||||
kind: "request",
|
||||
key: r.key,
|
||||
expiresAt: r.expiresAt.toISOString(),
|
||||
updatedAt: r.updatedAt.toISOString(),
|
||||
bytes: r.bytes,
|
||||
tmdbId: null,
|
||||
title: null,
|
||||
year: null,
|
||||
mediaType: null,
|
||||
posterPath: null,
|
||||
});
|
||||
}
|
||||
}
|
||||
const out: TmdbCacheList = {
|
||||
items,
|
||||
counts: {
|
||||
request: await cache.count(CACHE_REGIONS.tmdbRequest),
|
||||
recognize: await cache.count(CACHE_REGIONS.tmdbRecognize),
|
||||
},
|
||||
maxSize: policy.maxsize,
|
||||
ttlHours: Math.round(policy.ttl / 3600),
|
||||
emptyResultTtlSec: policy.emptyTtl,
|
||||
};
|
||||
return out;
|
||||
});
|
||||
route(app, Api.cache.tmdbClear, async () => {
|
||||
const cache = sharedCache(ctx);
|
||||
const deleted =
|
||||
(await cache.clear(CACHE_REGIONS.tmdbRecognize)) + (await cache.clear(CACHE_REGIONS.tmdbRequest));
|
||||
ctx.logger.info({ deleted }, "TMDB 缓存已清空");
|
||||
return { deleted };
|
||||
});
|
||||
route(app, Api.cache.images, async () => {
|
||||
const s = await cacheSettings(ctx);
|
||||
const img = images(ctx);
|
||||
const u = await img.usage();
|
||||
return {
|
||||
enabled: s.globalImageCache ?? false,
|
||||
days: s.globalImageCacheDays ?? DEFAULT_SETTINGS.cache.globalImageCacheDays,
|
||||
usedBytes: u.bytes,
|
||||
files: u.files,
|
||||
dir: img.dir,
|
||||
};
|
||||
});
|
||||
route(app, Api.cache.imagesClear, async () => ({ deleted: await images(ctx).clear() }));
|
||||
},
|
||||
registerTasks() {},
|
||||
start(ctx: ServerContext) {
|
||||
registerScheduledJob(ctx, {
|
||||
key: "cache.sweep",
|
||||
defaults: {
|
||||
name: "缓存清理",
|
||||
description: "删除 PG 缓存里已过期的条目（TMDB 请求与识别缓存）",
|
||||
group: "maintain",
|
||||
schedule: { type: "interval", everyMs: 10 * 60_000 },
|
||||
firstRun: "after_interval",
|
||||
},
|
||||
run: async () => ({ summary: `删除 ${await sharedCache(ctx).sweep()} 条过期缓存` }),
|
||||
});
|
||||
registerScheduledJob(ctx, {
|
||||
key: "cache.images_clean",
|
||||
defaults: {
|
||||
name: "图片缓存清理",
|
||||
description: "删除超过保留天数的图片代理缓存文件（对照 MP 启动时的 clear_temp）",
|
||||
group: "maintain",
|
||||
schedule: { type: "interval", everyMs: 24 * 3_600_000 },
|
||||
},
|
||||
run: async () => {
|
||||
const s = await cacheSettings(ctx);
|
||||
const n = await images(ctx).clean(s.globalImageCacheDays ?? DEFAULT_SETTINGS.cache.globalImageCacheDays);
|
||||
return { summary: `删除 ${n} 张过期图片` };
|
||||
},
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export const cacheModule = createCacheModule();
|
||||
Vendored
+146
@@ -0,0 +1,146 @@
|
||||
import { mkdtempSync, rmSync, utimesSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { PgCache, sql } from "@umt/db";
|
||||
import { createMockTmdb, type MockTmdb } from "@umt/testkit";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { imageCachePath } from "../../src/modules/cache/images.ts";
|
||||
import { createCacheModule } from "../../src/modules/cache/index.ts";
|
||||
import { scheduleRegistry } from "../../src/modules/schedules/registry.ts";
|
||||
import { SourcesRuntime } from "../../src/modules/sources-identity/runtime.ts";
|
||||
import { sharedCache } from "../../src/shared-cache.ts";
|
||||
import { bearer, createHarness, type Harness, pgAvailable } from "../auth-settings/harness.ts";
|
||||
import { DEV_KEY, json } from "../helpers.ts";
|
||||
|
||||
// C2-A 缓存:TMDB 请求缓存进 kv_cache(TTL / maxsize 取设置,对照 MP)、/cache/tmdb 列表与清空、cache.sweep、
|
||||
// 图片代理(对照 MP ImageHelper:缓存路径、GLOBAL_IMAGE_CACHE 才读盘、md5 ETag / 304、按天清理)。
|
||||
|
||||
const hasPg = await pgAvailable();
|
||||
const JPEG = new Uint8Array([0xff, 0xd8, 0xff, 0xe0, 1, 2, 3, 4]);
|
||||
|
||||
describe("图片缓存路径(MP sanitize_url_path)", () => {
|
||||
it("URL 路径去掉开头 /;过长换 compressed_<sha16>;无扩展名补 .jpg", () => {
|
||||
expect(imageCachePath("https://image.tmdb.org/t/p/w342/abc.jpg")).toBe("t/p/w342/abc.jpg");
|
||||
expect(imageCachePath(`https://x.org/${"a".repeat(130)}.png`)).toMatch(/^compressed_[0-9a-f]{16}\.png$/);
|
||||
expect(imageCachePath("https://x.org/t/p/w92/noext")).toBe("t/p/w92/noext.jpg");
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!hasPg)("C2-A 缓存(真实 PG)", () => {
|
||||
let h: Harness;
|
||||
let admin: ReturnType<typeof bearer>;
|
||||
let tmdb: MockTmdb;
|
||||
const dir = mkdtempSync(join(tmpdir(), "umt-c2a-img-"));
|
||||
let upstreamHits = 0;
|
||||
const mod = createCacheModule({
|
||||
imageDir: dir,
|
||||
imageUpstreamBase: "https://image.tmdb.org/t/p",
|
||||
imageFetch: async (input) => {
|
||||
upstreamHits++;
|
||||
if (String(input).includes("broken")) return new Response("<html>", { headers: { "content-type": "text/html" } });
|
||||
return new Response(JPEG, { headers: { "content-type": "image/jpeg" } });
|
||||
},
|
||||
});
|
||||
|
||||
beforeAll(async () => {
|
||||
h = await createHarness({ extraModules: [mod] });
|
||||
admin = bearer(h.app, DEV_KEY);
|
||||
tmdb = await createMockTmdb({ apiKey: "0123456789abcdef0123456789abcdef" });
|
||||
});
|
||||
afterAll(async () => {
|
||||
await tmdb?.close();
|
||||
await h?.close();
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("TMDB 请求响应进 kv_cache(tmdb.request,TTL = cache.tmdbTtlHours,条数上限 = cache.tmdbMaxSize),另一个进程直接命中", async () => {
|
||||
const r = await admin("PATCH", "/settings", {
|
||||
metadata: { tmdbApiKey: "0123456789abcdef0123456789abcdef", tmdbBaseUrl: `${tmdb.url}/3` },
|
||||
cache: { tmdbTtlHours: 6, tmdbMaxSize: 16 },
|
||||
});
|
||||
expect(r.status).toBe(200);
|
||||
await (await new SourcesRuntime(h.ctx).tmdb())!.details("movie", 872585);
|
||||
const calls = tmdb.calls.length;
|
||||
const rows = (
|
||||
await sql<{ key: string; hours: number }>`select key, extract(epoch from expires_at - now()) / 3600 as hours
|
||||
from kv_cache where region = 'tmdb.request'`.execute(h.db)
|
||||
).rows;
|
||||
const key = rows.find((x) => x.key.startsWith("/movie/872585?"))!.key;
|
||||
expect(rows.every((x) => !x.key.includes("0123456789abcdef"))).toBe(true);
|
||||
expect(Number(rows[0]!.hours)).toBeGreaterThan(5.9);
|
||||
expect(await new PgCache(h.db).get("tmdb.request", key)).toMatchObject({ id: 872585 });
|
||||
await (await new SourcesRuntime(h.ctx).tmdb())!.details("movie", 872585);
|
||||
expect(tmdb.calls.length).toBe(calls);
|
||||
// 空搜索结果按 EMPTY_RESULT_CACHE_TTL(30 分钟)
|
||||
await (await new SourcesRuntime(h.ctx).tmdb())!.search("完全不存在的片名c2a", { type: "movie" });
|
||||
const empty = (
|
||||
await sql<{ m: number }>`select extract(epoch from expires_at - now()) / 60 as m
|
||||
from kv_cache where region = 'tmdb.request' and key like '/search/movie?%'`.execute(h.db)
|
||||
).rows[0];
|
||||
expect(Number(empty!.m)).toBeLessThanOrEqual(30);
|
||||
});
|
||||
|
||||
it("GET /cache/tmdb 列出条目(键 / 过期 / 大小)与上限;POST /cache/tmdb/clear 清空", async () => {
|
||||
await sharedCache(h.ctx).set(
|
||||
"tmdb.recognize",
|
||||
"[未知][zh-CN]Oppenheimer-2023-None",
|
||||
{ id: 872585, type: "movie", year: "2023", title: "奥本海默", poster_path: "/p.jpg" },
|
||||
{ ttl: 3600 },
|
||||
);
|
||||
const list = await json(await admin("GET", "/cache/tmdb"));
|
||||
expect(list.data).toMatchObject({ maxSize: 16, ttlHours: 6, emptyResultTtlSec: 1800 });
|
||||
expect(list.data.counts.request).toBeGreaterThan(0);
|
||||
const rec = list.data.items.find((i: { kind: string }) => i.kind === "recognize");
|
||||
expect(rec).toMatchObject({ tmdbId: 872585, title: "奥本海默", mediaType: "movie", posterPath: "/p.jpg" });
|
||||
expect(rec.bytes).toBeGreaterThan(0);
|
||||
const cleared = await json(await admin("POST", "/cache/tmdb/clear"));
|
||||
expect(cleared.data.deleted).toBeGreaterThan(1);
|
||||
expect((await json(await admin("GET", "/cache/tmdb"))).data.items).toEqual([]);
|
||||
});
|
||||
|
||||
it("cache.sweep / cache.images_clean 登记为维护任务;sweep 删除过期行", async () => {
|
||||
await mod.start?.(h.ctx);
|
||||
const reg = scheduleRegistry(h.ctx);
|
||||
expect(reg.get("cache.sweep")?.defaults).toMatchObject({ group: "maintain", schedule: { everyMs: 600_000 } });
|
||||
expect(reg.get("cache.images_clean")?.defaults.group).toBe("maintain");
|
||||
await sharedCache(h.ctx).set("t", "gone", 1, { ttl: 60 });
|
||||
await sql`update kv_cache set expires_at = now() - interval '1 second' where region = 't'`.execute(h.db);
|
||||
const run = {
|
||||
jobId: "cache.sweep",
|
||||
runId: "r",
|
||||
trigger: "manual" as const,
|
||||
now: new Date(),
|
||||
config: {},
|
||||
slot: "1",
|
||||
};
|
||||
expect((await reg.get("cache.sweep")!.run(run))?.summary).toMatch(/删除 [1-9]\d* 条过期缓存/);
|
||||
});
|
||||
|
||||
it("图片代理:只收 TMDB 尺寸 / 文件名;GLOBAL_IMAGE_CACHE 关闭时每次回源(仍落盘),开启后读盘;md5 ETag / 304;非图片 502;按天清理", async () => {
|
||||
const get = (p: string, headers: Record<string, string> = {}) =>
|
||||
h.app.request(`/api/v1${p}`, { headers: { authorization: `Bearer ${DEV_KEY}`, ...headers } });
|
||||
expect((await get("/img/tmdb/w9999/a.jpg")).status).toBe(400);
|
||||
expect((await get("/img/tmdb/w342/..%2Fetc.jpg")).status).toBe(400);
|
||||
expect((await h.app.request("/api/v1/img/tmdb/w342/a.jpg")).status).toBe(401);
|
||||
const r1 = await get("/img/tmdb/w342/a.jpg");
|
||||
expect(r1.status).toBe(200);
|
||||
expect(r1.headers.get("content-type")).toBe("image/jpeg");
|
||||
expect(r1.headers.get("cache-control")).toBe("public, max-age=604800");
|
||||
const etag = r1.headers.get("etag")!;
|
||||
expect(etag).toMatch(/^[0-9a-f]{32}$/);
|
||||
await get("/img/tmdb/w342/a.jpg");
|
||||
expect(upstreamHits).toBe(2);
|
||||
await admin("PATCH", "/settings", { cache: { globalImageCache: true } });
|
||||
const r3 = await get("/img/tmdb/w342/a.jpg", { "if-none-match": etag });
|
||||
expect(r3.status).toBe(304);
|
||||
expect(upstreamHits).toBe(2);
|
||||
expect((await get("/img/tmdb/w342/broken.jpg")).status).toBe(502);
|
||||
const usage = await json(await admin("GET", "/cache/images"));
|
||||
expect(usage.data).toMatchObject({ enabled: true, days: 7, files: 1 });
|
||||
const old = new Date(Date.now() - 9 * 86_400_000);
|
||||
utimesSync(join(dir, "t/p/w342/a.jpg"), old, old);
|
||||
expect(await mod.images(h.ctx).clean(7)).toBe(1);
|
||||
expect((await json(await admin("GET", "/cache/images"))).data.files).toBe(0);
|
||||
expect((await json(await admin("POST", "/cache/images/clear"))).data.deleted).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,37 @@
|
||||
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import type { CacheClearResult, ImageCacheStatus, TmdbCacheList } from "@umt/contracts";
|
||||
import { apiData } from "~/lib/api.ts";
|
||||
|
||||
// C2-A 缓存管理:TMDB 缓存(条数 / 上限 / TTL / 清空)、图片代理缓存(用量 / 清空)
|
||||
|
||||
const keys = { tmdb: ["cache", "tmdb"] as const, images: ["cache", "images"] as const };
|
||||
|
||||
export function useTmdbCache() {
|
||||
return useQuery({
|
||||
queryKey: keys.tmdb,
|
||||
queryFn: ({ signal }) => apiData<TmdbCacheList>("/cache/tmdb?limit=1", { signal }),
|
||||
});
|
||||
}
|
||||
|
||||
export function useClearTmdbCache() {
|
||||
const qc = useQueryClient();
|
||||
return useMutation({
|
||||
mutationFn: () => apiData<CacheClearResult>("/cache/tmdb/clear", { method: "POST" }),
|
||||
onSuccess: () => qc.invalidateQueries({ queryKey: keys.tmdb }),
|
||||
});
|
||||
}
|
||||
|
||||
export function useImageCache() {
|
||||
return useQuery({
|
||||
queryKey: keys.images,
|
||||
queryFn: ({ signal }) => apiData<ImageCacheStatus>("/cache/images", { signal }),
|
||||
});
|
||||
}
|
||||
|
||||
export function useClearImageCache() {
|
||||
const qc = useQueryClient();
|
||||
return useMutation({
|
||||
mutationFn: () => apiData<CacheClearResult>("/cache/images/clear", { method: "POST" }),
|
||||
onSuccess: () => qc.invalidateQueries({ queryKey: keys.images }),
|
||||
});
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
import { tmdbImageUrl } from "@umt/contracts";
|
||||
import type { LucideIcon } from "lucide-react";
|
||||
import { Clapperboard, Film, Layers, Tv } from "lucide-react";
|
||||
import type { ReactNode } from "react";
|
||||
import { type ReactNode, useState } from "react";
|
||||
import { StatusBadge, type StatusTone } from "~/components/console/status-badge.tsx";
|
||||
import { cn } from "~/lib/utils.ts";
|
||||
|
||||
@@ -117,19 +118,43 @@ export function ChipToggleGroup<T extends string>({
|
||||
|
||||
const KIND_ICON: Record<string, LucideIcon> = { movie: Film, series: Tv, variety: Clapperboard, collection: Layers };
|
||||
|
||||
/** 封面占位:无彩材质 + 类型图标 + 首字(不引入海报色彩) */
|
||||
/** media_entity.metadata 里的最终海报路径(metadata.posterPath.value) */
|
||||
export function posterPathOf(metadata: unknown): string | null {
|
||||
const v = (metadata as { posterPath?: { value?: unknown } } | null)?.posterPath?.value;
|
||||
return typeof v === "string" && v ? v : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 封面:有 TMDB 海报路径时经图片代理显示海报,加载失败或没有海报时回退无彩占位(类型图标 + 首字)。
|
||||
*/
|
||||
export function CoverPlaceholder({
|
||||
title,
|
||||
kind,
|
||||
small = false,
|
||||
className,
|
||||
posterPath,
|
||||
}: {
|
||||
title: string;
|
||||
kind: string;
|
||||
/** 列表缩略:只显示首字,不叠类型图标 */
|
||||
small?: boolean;
|
||||
className?: string;
|
||||
/** TMDB poster_path */
|
||||
posterPath?: string | null | undefined;
|
||||
}) {
|
||||
const [failed, setFailed] = useState(false);
|
||||
const src = failed ? null : tmdbImageUrl(posterPath, small ? "w154" : "w342");
|
||||
if (src) {
|
||||
return (
|
||||
<img
|
||||
src={src}
|
||||
alt=""
|
||||
loading="lazy"
|
||||
onError={() => setFailed(true)}
|
||||
className={cn("shrink-0 rounded-lg border border-border bg-foreground/[0.05] object-cover", className)}
|
||||
/>
|
||||
);
|
||||
}
|
||||
const Icon = KIND_ICON[kind] ?? Film;
|
||||
const glyph = title.replace(/^w32-/, "").trim().charAt(0) || "?";
|
||||
return (
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import type { LocalMediaHit, SearchAction, TmdbCandidate } from "@umt/contracts";
|
||||
import { type LocalMediaHit, type SearchAction, type TmdbCandidate, tmdbImageUrl } from "@umt/contracts";
|
||||
import { ArrowRight, CornerDownLeft, Film, Globe, LoaderCircle, type LucideIcon, Plus, Search } from "lucide-react";
|
||||
import { type ReactNode, useEffect, useId, useMemo, useRef, useState } from "react";
|
||||
import { Link, useNavigate } from "react-router";
|
||||
@@ -33,6 +33,8 @@ type Item = {
|
||||
key: string;
|
||||
group: string;
|
||||
icon: LucideIcon;
|
||||
/** TMDB 海报路径:有则以小海报代替图标(失败回退图标) */
|
||||
posterPath?: string | null | undefined;
|
||||
title: ReactNode;
|
||||
meta?: ReactNode;
|
||||
/** 右侧动作提示 */
|
||||
@@ -216,7 +218,7 @@ function PaletteBody({ initial }: { initial: string }) {
|
||||
i === active ? "bg-foreground/[0.06]" : "",
|
||||
)}
|
||||
>
|
||||
<it.icon className="h-4 w-4 shrink-0 text-muted-foreground" />
|
||||
<ItemIcon it={it} />
|
||||
<div className="min-w-0 flex-1">
|
||||
<div className="truncate text-sm text-foreground">{it.title}</div>
|
||||
{it.meta ? <div className="truncate text-meta text-muted-foreground">{it.meta}</div> : null}
|
||||
@@ -287,12 +289,30 @@ function localItem(h: LocalMediaHit, run: () => void): Item {
|
||||
};
|
||||
}
|
||||
|
||||
function ItemIcon({ it }: { it: Item }) {
|
||||
const [failed, setFailed] = useState(false);
|
||||
const src = failed ? null : tmdbImageUrl(it.posterPath, "w92");
|
||||
if (src) {
|
||||
return (
|
||||
<img
|
||||
src={src}
|
||||
alt=""
|
||||
loading="lazy"
|
||||
onError={() => setFailed(true)}
|
||||
className="h-9 w-6 shrink-0 rounded-tight border border-border object-cover"
|
||||
/>
|
||||
);
|
||||
}
|
||||
return <it.icon className="h-4 w-4 shrink-0 text-muted-foreground" />;
|
||||
}
|
||||
|
||||
function tmdbItem(c: TmdbCandidate, view: () => void, subscribe: (c: TmdbCandidate) => void): Item {
|
||||
const local = !!c.localEntityId;
|
||||
return {
|
||||
key: `tmdb-${c.entityType}-${c.externalId}`,
|
||||
group: "TMDB",
|
||||
icon: local ? Globe : Plus,
|
||||
posterPath: c.posterPath,
|
||||
title: (
|
||||
<span>
|
||||
{c.title}
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
import { toast } from "sonner";
|
||||
import { useClearImageCache, useClearTmdbCache, useImageCache, useTmdbCache } from "~/api/cache.ts";
|
||||
import { useUpdateSettings } from "~/api/settings.ts";
|
||||
import { StatusBadge } from "~/components/console/status-badge.tsx";
|
||||
import { Button } from "~/components/ui/button.tsx";
|
||||
import { errorMessage } from "~/lib/api.ts";
|
||||
import { str } from "./resource-section.tsx";
|
||||
import { type CardSpec, SettingsCard, SettingsGate } from "./settings-card.tsx";
|
||||
import { type CardSpec, n, SettingsCard, SettingsGate } from "./settings-card.tsx";
|
||||
|
||||
// 元数据:TMDB API Key(只写,存 secret)与 Base URL
|
||||
|
||||
@@ -67,6 +68,128 @@ function TmdbKeyStatus({ saved }: { saved: boolean }) {
|
||||
);
|
||||
}
|
||||
|
||||
// TMDB 缓存(对照 MP:META_CACHE_EXPIRE / CONF.tmdb / EMPTY_RESULT_CACHE_TTL)
|
||||
const TMDB_CACHE_CARD: CardSpec = {
|
||||
anchor: "tmdb-cache",
|
||||
title: "TMDB 缓存",
|
||||
description: "请求响应与识别结果存在数据库里，三个进程共用。超过条数上限时删最久未用的条目。",
|
||||
fields: [
|
||||
{ name: "cache.tmdbTtlHours", label: "有效期（小时）", kind: "number" },
|
||||
{ name: "cache.tmdbMaxSize", label: "条数上限", kind: "number", hint: "请求缓存和识别缓存各自计数" },
|
||||
{
|
||||
name: "cache.emptyResultTtlSec",
|
||||
label: "空结果有效期（秒）",
|
||||
kind: "number",
|
||||
hint: "搜索无结果、未识别时的短缓存",
|
||||
},
|
||||
],
|
||||
fromSettings: (s) => ({
|
||||
"cache.tmdbTtlHours": n(s.cache.tmdbTtlHours),
|
||||
"cache.tmdbMaxSize": n(s.cache.tmdbMaxSize),
|
||||
"cache.emptyResultTtlSec": n(s.cache.emptyResultTtlSec),
|
||||
}),
|
||||
toPatch: (v) => ({
|
||||
cache: {
|
||||
tmdbTtlHours: Number(v["cache.tmdbTtlHours"]),
|
||||
tmdbMaxSize: Number(v["cache.tmdbMaxSize"]),
|
||||
emptyResultTtlSec: Number(v["cache.emptyResultTtlSec"]),
|
||||
},
|
||||
}),
|
||||
extra: () => <TmdbCacheStatus />,
|
||||
};
|
||||
|
||||
function TmdbCacheStatus() {
|
||||
const q = useTmdbCache();
|
||||
const clear = useClearTmdbCache();
|
||||
const c = q.data?.counts;
|
||||
return (
|
||||
<div className="mb-2 flex flex-wrap items-center gap-2 text-meta text-muted-foreground">
|
||||
<span className="tabular-nums">
|
||||
{c ? `识别 ${c.recognize} 条 · 请求 ${c.request} 条 · 上限各 ${q.data?.maxSize} 条` : "读取中…"}
|
||||
</span>
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="xs"
|
||||
className="hover:text-destructive"
|
||||
disabled={clear.isPending}
|
||||
onClick={async () => {
|
||||
try {
|
||||
const r = await clear.mutateAsync();
|
||||
toast.success(`已清空 ${r.deleted} 条 TMDB 缓存`);
|
||||
} catch (err) {
|
||||
toast.error(errorMessage(err, "清空失败"));
|
||||
}
|
||||
}}
|
||||
>
|
||||
清空
|
||||
</Button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// 图片代理与缓存(对照 MP GLOBAL_IMAGE_CACHE / GLOBAL_IMAGE_CACHE_DAYS)
|
||||
const IMAGES_CARD: CardSpec = {
|
||||
anchor: "images",
|
||||
title: "图片代理与缓存",
|
||||
description: "海报经服务端从 TMDB 取回。开启缓存后保存到数据目录，下次直接读取。",
|
||||
fields: [
|
||||
{ name: "cache.globalImageCache", label: "缓存图片", kind: "switch" },
|
||||
{ name: "cache.globalImageCacheDays", label: "保留天数", kind: "number", hint: "超过天数的图片每天清理一次" },
|
||||
],
|
||||
fromSettings: (s) => ({
|
||||
"cache.globalImageCache": s.cache.globalImageCache,
|
||||
"cache.globalImageCacheDays": n(s.cache.globalImageCacheDays),
|
||||
}),
|
||||
toPatch: (v) => ({
|
||||
cache: {
|
||||
globalImageCache: v["cache.globalImageCache"] === true,
|
||||
globalImageCacheDays: Number(v["cache.globalImageCacheDays"]),
|
||||
},
|
||||
}),
|
||||
extra: () => <ImageCacheStatus />,
|
||||
};
|
||||
|
||||
function ImageCacheStatus() {
|
||||
const q = useImageCache();
|
||||
const clear = useClearImageCache();
|
||||
const d = q.data;
|
||||
return (
|
||||
<div className="mb-2 flex flex-wrap items-center gap-2 text-meta text-muted-foreground">
|
||||
<span className="tabular-nums">
|
||||
{d ? `${d.files} 张 · ${(d.usedBytes / 1024 / 1024).toFixed(1)} MiB` : "读取中…"}
|
||||
</span>
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="xs"
|
||||
className="hover:text-destructive"
|
||||
disabled={clear.isPending}
|
||||
onClick={async () => {
|
||||
try {
|
||||
const r = await clear.mutateAsync();
|
||||
toast.success(`已删除 ${r.deleted} 张缓存图片`);
|
||||
} catch (err) {
|
||||
toast.error(errorMessage(err, "清空失败"));
|
||||
}
|
||||
}}
|
||||
>
|
||||
清空
|
||||
</Button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function MetadataSection() {
|
||||
return <SettingsGate>{(s) => <SettingsCard section="metadata" spec={TMDB_CARD} settings={s} />}</SettingsGate>;
|
||||
return (
|
||||
<SettingsGate>
|
||||
{(s) => (
|
||||
<>
|
||||
<SettingsCard section="metadata" spec={TMDB_CARD} settings={s} />
|
||||
<SettingsCard section="metadata" spec={TMDB_CACHE_CARD} settings={s} />
|
||||
<SettingsCard section="metadata" spec={IMAGES_CARD} settings={s} />
|
||||
</>
|
||||
)}
|
||||
</SettingsGate>
|
||||
);
|
||||
}
|
||||
@@ -51,7 +51,16 @@ import {
|
||||
UNIT_KIND,
|
||||
VERIFY_STATUS,
|
||||
} from "~/components/library/labels.ts";
|
||||
import { Chip, ChipList, CoverPlaceholder, Panel, ProgressLine, relTime, Tone } from "~/components/library/parts.tsx";
|
||||
import {
|
||||
Chip,
|
||||
ChipList,
|
||||
CoverPlaceholder,
|
||||
Panel,
|
||||
ProgressLine,
|
||||
posterPathOf,
|
||||
relTime,
|
||||
Tone,
|
||||
} from "~/components/library/parts.tsx";
|
||||
import { SlotCard } from "~/components/library/slot-card.tsx";
|
||||
import { EntityNamingButton } from "~/components/naming/entity-naming-dialog.tsx";
|
||||
import { Button } from "~/components/ui/button.tsx";
|
||||
@@ -617,7 +626,12 @@ export default function LibraryDetailPage() {
|
||||
<p className="mb-4 text-meta text-muted-foreground">旧 ID {redirected} 已跳转到当前作品。</p>
|
||||
) : null}
|
||||
<header className="mb-6 flex flex-wrap items-start gap-4">
|
||||
<CoverPlaceholder title={e.title} kind={e.kind} className="h-28 w-20 sm:h-36 sm:w-24" />
|
||||
<CoverPlaceholder
|
||||
title={e.title}
|
||||
kind={e.kind}
|
||||
posterPath={posterPathOf(e.metadata)}
|
||||
className="h-28 w-20 sm:h-36 sm:w-24"
|
||||
/>
|
||||
<div className="min-w-0 flex-1 basis-56">
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<h1 className="min-w-0 text-title-lg font-semibold tracking-tight break-words text-foreground">
|
||||
|
||||
@@ -128,7 +128,13 @@ function WorksView() {
|
||||
header: "作品",
|
||||
cell: (r) => (
|
||||
<div className="flex min-w-0 items-center gap-3">
|
||||
<CoverPlaceholder title={r.entity.title} kind={r.entity.kind} small className="h-14 w-10" />
|
||||
<CoverPlaceholder
|
||||
title={r.entity.title}
|
||||
kind={r.entity.kind}
|
||||
posterPath={r.posterPath}
|
||||
small
|
||||
className="h-14 w-10"
|
||||
/>
|
||||
<div className="min-w-0">
|
||||
<Link
|
||||
to={consolePath("library", r.entity.id)}
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
# MoviePilot 缓存移植对照(TMDB 缓存 / 通用缓存 / 图片缓存)
|
||||
|
||||
上游:`/opt/moviepilot-plus/upstream/MoviePilot/app`。原则:接口与行为照搬,只把存储从进程内存 / 文件换成 PG。
|
||||
|
||||
## 通用缓存层:`app/runtime/cache.py` → `packages/db/src/cache.ts`
|
||||
|
||||
| 上游 | 我们 | 说明 |
|
||||
|---|---|---|
|
||||
| `MemoryBackend` + `_MemoryTLRUCache`(按 region 分区的 cachetools `TLRUCache`) | `PgCache`(表 `kv_cache`,迁移 0042) | region 分区、每 key 独立 TTL |
|
||||
| `set(key, value, ttl, region, maxsize)`;`ttl <= 0` 删除 key | `PgCache.set(region, key, value, {ttl, maxsize})` | 同;TTL 单位秒 |
|
||||
| TLRU 淘汰:写入时先 `expire()` 过期项,再按最近使用淘汰到 maxsize | `evict()`:删 region 内过期行,再按 `accessed_at` 从旧到新删到 maxsize | get 命中刷新 `accessed_at`(TLRU 的"使用") |
|
||||
| `get` 未命中返回 None | `get` 返回 null | |
|
||||
| `exists / delete / clear(region) / items(region)` | 同名方法 | `items` 额外带过期时间与字节数(管理界面) |
|
||||
| `DEFAULT_CACHE_TTL = 365 天`、`DEFAULT_CACHE_SIZE = 1024` | 同名常量 | |
|
||||
| `@cached(region, maxsize, ttl, skip_none, skip_empty, skip_if, empty_ttl, empty_if)` | `cachedCall(cache, key, opts, fn)` | should_cache / get_cache_ttl 逻辑逐条对应 |
|
||||
| `fresh()` 上下文 | `opts.fresh` | 跳过读、照常写 |
|
||||
|
||||
偏离(仅存储介质):值用 JSON(jsonb)代替 pickle;过期行需要物理删除,加定时任务 `cache.sweep`(每 10 分钟);
|
||||
PG 读写失败时 TMDB 路径按未命中处理(内存缓存不会失败)。
|
||||
|
||||
## TMDB 请求缓存:`tmdbv3api/tmdb.py TMDb.request` 的 `@cached` → `packages/adapters/src/tmdb/client.ts`
|
||||
|
||||
| 上游 | 我们 |
|
||||
|---|---|
|
||||
| `maxsize = CONF.tmdb`(256;大内存 1024) | 设置 `cache.tmdbMaxSize`(缺省 256) |
|
||||
| `ttl = CONF.meta = META_CACHE_EXPIRE 小时`(缺省 24) | 设置 `cache.tmdbTtlHours`(缺省 24) |
|
||||
| `empty_ttl = EMPTY_RESULT_CACHE_TTL`(1800 秒),`empty_if = results 为空列表` | 设置 `cache.emptyResultTtlSec`(缺省 1800);`isEmptyResult` 同判定 |
|
||||
| `skip_none`(连接失败不缓存)、`skip_if = success:false`(业务失败不缓存) | 这两类在 `fetchWithRetry` 抛错,不会写缓存 |
|
||||
| 键 = 函数名 + (method, url, params) | 键 = `path?query`(参数排序,不含 api_key / token) |
|
||||
| region 自动生成 | region `tmdb.request` |
|
||||
|
||||
## TMDB 识别缓存:`modules/themoviedb/cache.py TmdbCache` → `apps/server/src/modules/sources-identity/service/tmdb-recognize-cache.ts`
|
||||
|
||||
| 上游 | 我们 |
|
||||
|---|---|
|
||||
| `__get_key`:`[{meta.type.value 或 '未知'}][{TMDB_LOCALE}]{media_id 或 meta.name}-{meta.year}-{meta.begin_season}` | `recognizeKey`:同格式;类型用 MP 的中文值(电影 / 电视剧 / 未知),语言取 TMDB 客户端首选语言,None 写作 `None` |
|
||||
| `meta.name`:中文名全是汉字优先,否则英文名,再退中文名 | `recognizeMetaOf` 同规则;有季或集即"电视剧",否则"未知"(MetaVideo) |
|
||||
| 值 `{id, type, year, title, poster_path, backdrop_path}` | 同字段(type 用 movie / tv) |
|
||||
| `update(meta, info)`:info 非空写正常条目(默认 ttl);`{}` 写负缓存 `{"id": 0}`(`EMPTY_RESULT_CACHE_TTL`);None 不写 | `saveRecognizeCache(meta, info)`:info / null / undefined 三态一一对应 |
|
||||
| `get(meta)` 读;类型冲突(元数据电视剧、缓存电影)删除后按未命中 | `loadRecognizeCache` 同 |
|
||||
| `update` 时类型冲突拒绝写入 | 同 |
|
||||
| `maxsize = CONF.tmdb`、`ttl = CONF.meta`,每条独立 `expires_at` | 同设置;`kv_cache.expires_at` 每条独立 |
|
||||
| `_restore / save`:pickle 文件持久化与恢复 | 不需要:表本身就是持久化(UNLOGGED 在 PG 崩溃时清空,等价于缓存丢失) |
|
||||
| `list_items / delete / clear` | `GET /cache/tmdb`、`POST /cache/tmdb/clear`(清空时同时清请求缓存,对应 MP `clear_cache()`) |
|
||||
| `is_redis` 分支 | 忽略 |
|
||||
|
||||
调用点(`modules/themoviedb/__init__.py`):
|
||||
|
||||
| 上游 | 我们(`service/resolve.ts`、`service/identity-admin.ts`) |
|
||||
|---|---|
|
||||
| `_recognize_steps`:LOAD_CACHE(`cache=True` 时)→ 命中且有 title 则 LOAD_DETAILS(按缓存 id / type 取详情);命中负缓存 → 未识别 | `tmdbCandidates`:命中且有标题 → `details(type, id)` 作唯一远端候选;命中 `{id:0}` → 远端候选为空 |
|
||||
| 未命中:按名称搜索(中文名、简体、英文名依次)→ SAVE_CACHE(info) | 未命中:按解析名搜索 → 识别结论为 TMDB 条目写正常条目,`no_match` 写负缓存,TMDB 出错 / 有歧义不写 |
|
||||
| `recognize_media(cache=False)` 不读缓存 | 强制复查(`force`)不读缓存,照常回写 |
|
||||
| `update_recognize_cache(meta, mediainfo)`:共享识别后回填 | 人工认定资源身份后 `backfillRecognizeCache` 回填 |
|
||||
|
||||
偏离:我们的识别有本地作品优先与歧义(需人工确认)两个 MP 没有的结论。歧义时不写缓存(MP 没有这一态);
|
||||
本地作品胜出时不写(MP 无本地召回)。
|
||||
|
||||
## 图片缓存:`application/image.py ImageHelper` + `api/endpoints/system.py fetch_image` → `apps/server/src/modules/cache/images.ts`
|
||||
|
||||
| 上游 | 我们 |
|
||||
|---|---|
|
||||
| 缓存路径 `SecurityUtils.sanitize_url_path(url)`:URL 路径编码,超 120 字符换 `compressed_<sha256 前 16><ext>`,无后缀补 `.jpg`;放在 `CACHE_PATH/images/` | `imageCachePath`,放在 `DATA_DIR/cache/images/` |
|
||||
| `use_cache` 才读缓存;拉到的合法图片总会 `file_cache.set` | 同 |
|
||||
| `/cache/image`:`use_cache = GLOBAL_IMAGE_CACHE`(缺省 False) | 设置 `cache.globalImageCache`(缺省关闭) |
|
||||
| `get_image_mime_type`:Pillow 识别,拒绝非图片与 SVG | 按文件头识别 jpeg / png / webp / gif,其余拒绝 |
|
||||
| `ETag = md5(content)`,`Cache-Control: public, max-age=604800`,If-None-Match 相同回 304 | 同 |
|
||||
| `SECURITY_IMAGE_DOMAINS` 白名单 | 只代理 `image.tmdb.org`,尺寸与文件名走白名单 |
|
||||
| `clear_temp`:删除 `CACHE_PATH/images` 下 mtime 超过 `GLOBAL_IMAGE_CACHE_DAYS`(7)天的文件与空目录 | 设置 `cache.globalImageCacheDays`(缺省 7);定时任务 `cache.images_clean` 每天执行 |
|
||||
|
||||
偏离:MP 失败时返回空响应,我们回 502(前端回退占位);MP 按配置走代理服务器,我们直连(TMDB API 也未走代理);
|
||||
MP 只在启动时清理,我们改为每天一次的定时任务。
|
||||
@@ -0,0 +1,61 @@
|
||||
# 缓存与限流运行手册(C2-A)
|
||||
|
||||
不引入 Redis / Deno KV。跨进程共享的状态都放 PG:站点限流令牌桶、TMDB 缓存。缓存语义照搬 MoviePilot,
|
||||
对照与偏离见 [docs/porting/moviepilot-tmdb-cache.md](../porting/moviepilot-tmdb-cache.md)。
|
||||
|
||||
## 跨进程站点限流
|
||||
|
||||
- 表 `rate_limit_bucket`(迁移 `0041_rate_limit`):key `site:<站点 id>`,tokens / capacity / refill_per_sec / blocked_until。
|
||||
- `@umt/db` 的 `PgTokenBucket.acquire(key, cost, {capacity, refillPerSec, waitMs})`:事务里 `select … for update`,
|
||||
按 DB 时钟补令牌后扣减;拿不到按需要等待的时间加抖动轮询,超过 waitMs 返回 `ok:false`(站点限流器会重新排队)。
|
||||
- 站点适配器用 `createSharedSiteRateLimiter`:进程内先串行排队,再到 PG 扣令牌;429 的 Retry-After 写 `blocked_until`。
|
||||
站点限流间隔为 0(不限流)时只走本地;PG 出错时本次退回进程内令牌桶并记 warn。
|
||||
- 容量 = 站点并发(burst),速率 = 1000 / 最小间隔毫秒。改站点限流设置即时生效(每次扣令牌带上最新参数)。
|
||||
|
||||
排查:`select * from rate_limit_bucket where key = 'site:<id>'`;`blocked_until` 不为空说明站点要求暂停。
|
||||
|
||||
## PG 缓存(kv_cache)
|
||||
|
||||
- 表 `kv_cache`(迁移 `0042_cache`,`UNLOGGED`:不写 WAL,崩溃后清空):(region, key) 主键,value jsonb,
|
||||
expires_at 每条独立,accessed_at 为最近读写时间。
|
||||
- `@umt/db` 的 `PgCache`(对照 MP `TTLCache`):set 带 ttl(秒,≤ 0 删除)与 maxsize(按 region 计条数,超出先删过期再删最久未用);
|
||||
get 命中刷新最近使用;`cachedCall` 对照 MP `@cached`(skip_none / skip_if / empty_ttl / fresh)。
|
||||
- 定时任务 `cache.sweep`(维护组,每 10 分钟)物理删除过期行。
|
||||
|
||||
| region | 内容 | 键 | TTL | 条数上限 |
|
||||
|---|---|---|---|---|
|
||||
| `tmdb.request` | TMDB 请求响应(MP `TMDb.request`) | `path?query`(不含凭证) | `cache.tmdbTtlHours`;`results` 为空按 `cache.emptyResultTtlSec` | `cache.tmdbMaxSize` |
|
||||
| `tmdb.recognize` | TMDB 识别结果(MP `TmdbCache`) | `[类型][语言]{tmdbid 或名称}-{年份}-{季}` | 同上;未识别 `{id:0}` 按空结果 TTL | `cache.tmdbMaxSize` |
|
||||
|
||||
设置(设置 → 元数据 → TMDB 缓存):`cache.tmdbTtlHours`(缺省 24,MP `META_CACHE_EXPIRE`)、`cache.tmdbMaxSize`(缺省 256,MP `CONF.tmdb`)、
|
||||
`cache.emptyResultTtlSec`(缺省 1800,MP `EMPTY_RESULT_CACHE_TTL`)。
|
||||
|
||||
API:`GET /api/v1/cache/tmdb?kind=recognize|request&limit=` 列出条目(键 / 过期 / 大小,识别条目带 tmdbId / 标题 / 海报);
|
||||
`POST /api/v1/cache/tmdb/clear` 清空两类(仅 admin)。
|
||||
|
||||
识别缓存何时读写(`sources-identity/service/tmdb-recognize-cache.ts`):
|
||||
- 识别资源时先读:命中且有标题 → 按 id 取 TMDB 详情作为唯一远端候选;命中 `{id:0}` → 不再搜索;强制复查(force)不读。
|
||||
- 写:自动识别到 TMDB 条目写正常条目;确认无匹配写 `{id:0}`;TMDB 出错或有歧义不写。
|
||||
- 人工认定资源身份后回填(MP `update_recognize_cache`),覆盖名称负缓存。
|
||||
- "元数据是剧集、缓存是电影"视为类型冲突:读时丢弃,写时拒绝。
|
||||
|
||||
## 图片代理与海报
|
||||
|
||||
- `GET /api/v1/img/tmdb/{size}/{file}`(会话或 API Key read):只代理 `image.tmdb.org`,size 走白名单(w92…original),
|
||||
file 只允许 `[A-Za-z0-9_-]+.(jpg|jpeg|png|webp)`。
|
||||
- 缓存路径对照 MP `sanitize_url_path`:`DATA_DIR/cache/images/t/p/<size>/<file>`。
|
||||
- `cache.globalImageCache`(缺省关闭,MP `GLOBAL_IMAGE_CACHE`)开启才读磁盘缓存;拉到的合法图片总会写盘。
|
||||
- 按文件头识别图片类型,非图片 / SVG 拒绝;ETag = md5(内容),`Cache-Control: public, max-age=604800`,If-None-Match 相同回 304。
|
||||
- 上游超时(20s)/ 非 200 / 非图片 → 502,前端回退无彩占位。
|
||||
- 清理:定时任务 `cache.images_clean`(每天)删除 mtime 早于 `cache.globalImageCacheDays`(缺省 7)天的文件。
|
||||
- 用量 / 清空:`GET /api/v1/cache/images`、`POST /api/v1/cache/images/clear`(仅 admin);设置 → 元数据 → 图片代理与缓存。
|
||||
|
||||
前端:作品库列表、作品详情、搜索面板的 TMDB 候选在有 `posterPath`(metadata.posterPath / TMDB 候选)时显示海报,
|
||||
加载失败回退原来的无彩占位。TMDB 详情刷新会把 posterPath / backdropPath 写进 metadata_field。
|
||||
|
||||
## 常见问题
|
||||
|
||||
- TMDB 缓存不生效:确认 `select region, count(*) from kv_cache group by 1` 有 `tmdb.request`;PG 读写失败时按未命中处理并记
|
||||
`TMDB 缓存读写失败，按未命中处理`。
|
||||
- 改了 TMDB 缓存设置:新写入的条目按新 TTL / 上限;已有条目保持原过期时间,必要时清空。
|
||||
- 数据库重启后缓存为空:UNLOGGED 表的正常行为。
|
||||
@@ -3,6 +3,7 @@ import * as agent from "./agent.ts";
|
||||
import * as assets from "./assets.ts";
|
||||
import * as auth from "./auth.ts";
|
||||
import * as browser from "./browser.ts";
|
||||
import * as cache from "./cache.ts";
|
||||
import { CapabilityDescriptor, KnownCapabilityId } from "./capability.ts";
|
||||
import { type ApiScope, CursorQuery, cursorPage, HealthStatus, Id, IdParam } from "./common.ts";
|
||||
import * as content from "./content.ts";
|
||||
@@ -1611,6 +1612,39 @@ export const Api = {
|
||||
}),
|
||||
},
|
||||
|
||||
// ── 缓存(C2-A,docs/runbook/cache.md) ──
|
||||
cache: {
|
||||
tmdb: ep({
|
||||
method: "get",
|
||||
path: "/cache/tmdb",
|
||||
scope: "admin",
|
||||
summary: "TMDB 缓存条目（请求响应与识别结果：键、过期时间、大小）与上限 / TTL",
|
||||
query: cache.TmdbCacheQuery,
|
||||
response: cache.TmdbCacheList,
|
||||
}),
|
||||
tmdbClear: ep({
|
||||
method: "post",
|
||||
path: "/cache/tmdb/clear",
|
||||
scope: "admin",
|
||||
summary: "清空 TMDB 缓存（请求响应与识别结果）",
|
||||
response: cache.CacheClearResult,
|
||||
}),
|
||||
images: ep({
|
||||
method: "get",
|
||||
path: "/cache/images",
|
||||
scope: "admin",
|
||||
summary: "图片代理缓存用量",
|
||||
response: cache.ImageCacheStatus,
|
||||
}),
|
||||
imagesClear: ep({
|
||||
method: "post",
|
||||
path: "/cache/images/clear",
|
||||
scope: "admin",
|
||||
summary: "清空图片代理缓存",
|
||||
response: cache.CacheClearResult,
|
||||
}),
|
||||
},
|
||||
|
||||
// ── 设置 ──
|
||||
settings: {
|
||||
get: ep({ method: "get", path: "/settings", scope: "read", summary: "全局设置", response: GlobalSettings }),
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
import { z } from "zod";
|
||||
import { NonNegInt, Timestamp } from "./common.ts";
|
||||
|
||||
// C2-A 缓存管理(docs/runbook/cache.md):TMDB 缓存(请求响应 + 识别结果,对照 MP TmdbCache)、图片代理缓存。
|
||||
|
||||
export const TmdbCacheKind = z.enum(["request", "recognize"]);
|
||||
export type TmdbCacheKind = z.infer<typeof TmdbCacheKind>;
|
||||
|
||||
/** 一条 TMDB 缓存(对照 MP list_items:识别条目带 tmdbId / 标题 / 年份 / 类型 / 海报) */
|
||||
export const TmdbCacheItem = z.object({
|
||||
kind: TmdbCacheKind,
|
||||
key: z.string(),
|
||||
expiresAt: Timestamp,
|
||||
updatedAt: Timestamp,
|
||||
/** jsonb 存储字节数 */
|
||||
bytes: NonNegInt,
|
||||
/** 识别条目:TMDB id,0 = 负缓存(未识别) */
|
||||
tmdbId: z.number().int().nullable(),
|
||||
title: z.string().nullable(),
|
||||
year: z.string().nullable(),
|
||||
mediaType: z.enum(["movie", "tv", "unknown"]).nullable(),
|
||||
posterPath: z.string().nullable(),
|
||||
});
|
||||
export type TmdbCacheItem = z.infer<typeof TmdbCacheItem>;
|
||||
|
||||
export const TmdbCacheQuery = z.object({
|
||||
kind: TmdbCacheKind.optional(),
|
||||
limit: z.coerce.number().int().min(1).max(5000).default(500),
|
||||
});
|
||||
|
||||
export const TmdbCacheList = z.object({
|
||||
items: z.array(TmdbCacheItem),
|
||||
counts: z.object({ request: NonNegInt, recognize: NonNegInt }),
|
||||
/** 每类条数上限(cache.tmdbMaxSize) */
|
||||
maxSize: NonNegInt,
|
||||
ttlHours: NonNegInt,
|
||||
emptyResultTtlSec: NonNegInt,
|
||||
});
|
||||
export type TmdbCacheList = z.infer<typeof TmdbCacheList>;
|
||||
|
||||
export const CacheClearResult = z.object({ deleted: NonNegInt });
|
||||
export type CacheClearResult = z.infer<typeof CacheClearResult>;
|
||||
|
||||
/** 图片代理缓存用量 */
|
||||
export const ImageCacheStatus = z.object({
|
||||
/** cache.globalImageCache */
|
||||
enabled: z.boolean(),
|
||||
/** cache.globalImageCacheDays */
|
||||
days: NonNegInt,
|
||||
usedBytes: NonNegInt,
|
||||
files: NonNegInt,
|
||||
dir: z.string(),
|
||||
});
|
||||
export type ImageCacheStatus = z.infer<typeof ImageCacheStatus>;
|
||||
|
||||
/** TMDB 图片尺寸白名单(与 TMDB configuration.images 一致) */
|
||||
export const TMDB_IMAGE_SIZES = [
|
||||
"w92",
|
||||
"w154",
|
||||
"w185",
|
||||
"w300",
|
||||
"w342",
|
||||
"w500",
|
||||
"w780",
|
||||
"w1280",
|
||||
"h632",
|
||||
"original",
|
||||
] as const;
|
||||
export type TmdbImageSize = (typeof TMDB_IMAGE_SIZES)[number];
|
||||
|
||||
/** 海报 / 背景图路径 → 图片代理 URL(相对 API_PREFIX 之外的完整路径,前端直接用作 img src) */
|
||||
export function tmdbImageUrl(path: string | null | undefined, size: TmdbImageSize = "w342"): string | null {
|
||||
if (!path) return null;
|
||||
const p = path.startsWith("/") ? path.slice(1) : path;
|
||||
if (!/^[A-Za-z0-9_-]+\.(jpg|jpeg|png|webp)$/.test(p)) return null;
|
||||
return `/api/v1/img/tmdb/${size}/${p}`;
|
||||
}
|
||||
@@ -4,6 +4,7 @@ export * from "./api.ts";
|
||||
export * from "./assets.ts";
|
||||
export * from "./auth.ts";
|
||||
export * from "./browser.ts";
|
||||
export * from "./cache.ts";
|
||||
export * from "./capabilities-io.ts";
|
||||
export * from "./capability.ts";
|
||||
export * from "./common.ts";
|
||||
|
||||
@@ -340,6 +340,8 @@ export const LibraryItem = z.object({
|
||||
slotCounts: SlotStatusCounts,
|
||||
activeDownloads: NonNegInt,
|
||||
lastOrganizedAt: Timestamp.nullable(),
|
||||
/** TMDB 海报路径(metadata.posterPath);前端经图片代理显示,没有时显示无彩占位 */
|
||||
posterPath: z.string().nullable().optional(),
|
||||
});
|
||||
export type LibraryItem = z.infer<typeof LibraryItem>;
|
||||
|
||||
|
||||
Reference in new issue
Block a user