feat(cache): TMDB image proxy, posters, cache admin API and settings cards

GET /img/tmdb/{size}/{file} ported from MP ImageHelper / fetch_image (sanitized cache path under DATA_DIR/cache/images, GLOBAL_IMAGE_CACHE read switch, md5 ETag / 304, day-based cleanup job). Cache module adds cache.sweep / cache.images_clean jobs and /cache/tmdb, /cache/images list and clear endpoints. Library list/detail and the search palette show posters with the neutral placeholder as fallback; metadata settings get TMDB cache and image cache cards. Runbook docs/runbook/cache.md and porting notes docs/porting/moviepilot-tmdb-cache.md. Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-09-28 22:26:29 +08:00
1 parent 946fca5e55
commit 7862573944
@@ -150,6 +150,7 @@ export function registerLibraryRoutes(app: ModuleApp, d: AoDeps) {
            where (s.media_entity_id = e.id or sr.media_entity_id = e.id) and dt.removed_at is null
              and dt.status in ('queued','submitting','downloading','stalled','paused'))::int`.as("active_downloads"),
        )
+        .select(sql<string | null>`e.metadata #>> '{posterPath,value}'`.as("poster_path"))
        .select(
          sql<Date | null>`(select max(r.finished_at) from organization_record r where r.media_entity_id = e.id and r.status in ('succeeded','partial'))`.as(
            "last_org",
@@ -220,6 +221,7 @@ export function registerLibraryRoutes(app: ModuleApp, d: AoDeps) {
        slotCounts: { ...emptyCounts(), ...(r.slot_counts ?? {}) },
        activeDownloads: r.active_downloads,
        lastOrganizedAt: r.last_org ? new Date(r.last_org).toISOString() : null,
+        posterPath: r.poster_path ?? null,
      }));
      return { items, nextCursor: p.nextCursor };
    },
@@ -0,0 +1,189 @@
+import { createHash } from "node:crypto";
+import { promises as fs } from "node:fs";
+import { dirname, extname, join } from "node:path";
+import { TMDB_IMAGE_SIZES } from "@umt/contracts";
+
+// TMDB 图片代理与文件缓存,移植 MP app/application/image.py(ImageHelper)+ api/endpoints/system.py(fetch_image):
+// - 缓存路径 = SecurityUtils.sanitize_url_path(url):URL 路径去掉开头 "/" 后百分号编码,超过 120 字符换成
+//   compressed_<sha256 前 16 位><扩展名>;没有扩展名补 .jpg;落在 DATA_DIR/cache/images/ 下(MP CACHE_PATH/images);
+// - 读缓存只在 use_cache(GLOBAL_IMAGE_CACHE)开启时;拉到的合法图片总是写盘(与 MP 一致);
+// - 合法图片按内容识别 MIME,拒绝非图片与 SVG;ETag = md5(内容),Cache-Control: public, max-age=7 天,If-None-Match 相同回 304;
+// - 清理:删除 mtime 早于 GLOBAL_IMAGE_CACHE_DAYS 天的文件与空目录(MP clear_temp → SystemUtils.clear)。
+// 我们只代理 TMDB 图片域(MP SECURITY_IMAGE_DOMAINS 的 TMDB 项),上游失败回 502。
+
+export const TMDB_IMAGE_HOST = "image.tmdb.org";
+const FILE_RE = /^[A-Za-z0-9_-]+\.(jpg|jpeg|png|webp)$/;
+
+export class ImageProxyError extends Error {
+  constructor(
+    readonly status: 400 | 502,
+    message: string,
+  ) {
+    super(message);
+  }
+}
+
+/** SecurityUtils.sanitize_url_path + ImageHelper._prepare_cache_path */
+export function imageCachePath(url: string, maxLength = 120): string {
+  const path = new URL(url).pathname.replace(/^\/+/, "");
+  let safe = path
+    .split("/")
+    .map((s) => encodeURIComponent(decodeURIComponent(s)))
+    .join("/");
+  if (safe.length > maxLength) {
+    const hash = createHash("sha256").update(safe).digest("hex").slice(0, 16);
+    safe = `compressed_${hash}${extname(safe).toLowerCase()}`;
+  }
+  return extname(safe) ? safe : `${safe}.jpg`;
+}
+
+/** ImageHelper.get_image_mime_type(按文件头识别;非图片 / SVG 返回 null) */
+export function imageMimeType(b: Uint8Array): string | null {
+  if (b.length >= 3 && b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff) return "image/jpeg";
+  if (b.length >= 8 && b[0] === 0x89 && b[1] === 0x50 && b[2] === 0x4e && b[3] === 0x47) return "image/png";
+  if (
+    b.length >= 12 &&
+    String.fromCharCode(...b.slice(0, 4)) === "RIFF" &&
+    String.fromCharCode(...b.slice(8, 12)) === "WEBP"
+  )
+    return "image/webp";
+  if (b.length >= 6 && String.fromCharCode(...b.slice(0, 4)) === "GIF8") return "image/gif";
+  return null;
+}
+
+export interface ImageCacheOptions {
+  /** DATA_DIR/cache/images */
+  dir: string;
+  fetch?: typeof globalThis.fetch;
+  timeoutMs?: number;
+  /** 上游基址(测试指向 mock);缺省 https://image.tmdb.org/t/p */
+  upstreamBase?: string;
+}
+
+export class ImageCache {
+  private readonly fetchImpl: typeof globalThis.fetch;
+
+  constructor(private readonly opts: ImageCacheOptions) {
+    this.fetchImpl = opts.fetch ?? globalThis.fetch;
+  }
+
+  get dir(): string {
+    return this.opts.dir;
+  }
+
+  /** 尺寸白名单 + 文件名校验后拼上游 URL(只允许 TMDB 图片域) */
+  upstream(size: string, file: string): string {
+    if (!(TMDB_IMAGE_SIZES as readonly string[]).includes(size)) throw new ImageProxyError(400, "不支持的图片尺寸");
+    if (!FILE_RE.test(file)) throw new ImageProxyError(400, "图片路径不合法");
+    const base = (this.opts.upstreamBase ?? `https://${TMDB_IMAGE_HOST}/t/p`).replace(/\/+$/, "");
+    const url = `${base}/${size}/${file}`;
+    if (!this.opts.upstreamBase && new URL(url).hostname !== TMDB_IMAGE_HOST) {
+      throw new ImageProxyError(400, "只代理 TMDB 图片");
+    }
+    return url;
+  }
+
+  /** fetch_image:取图(可读缓存)→ ETag / 304 */
+  async serve(size: string, file: string, useCache: boolean, ifNoneMatch: string | null): Promise<Response> {
+    const url = this.upstream(size, file);
+    const { content, mime } = await this.fetchImage(url, useCache);
+    const etag = createHash("md5").update(content).digest("hex");
+    const headers = new Headers({
+      etag,
+      "cache-control": `public, max-age=${86400 * 7}`,
+      "content-type": mime,
+      "x-content-type-options": "nosniff",
+    });
+    if (ifNoneMatch === etag) return new Response(null, { status: 304, headers });
+    return new Response(content as BodyInit, { status: 200, headers });
+  }
+
+  /** ImageHelper.fetch_image_with_mime_type */
+  private async fetchImage(url: string, useCache: boolean): Promise<{ content: Uint8Array; mime: string }> {
+    const path = join(this.opts.dir, imageCachePath(url));
+    if (useCache) {
+      const cached = await fs.readFile(path).catch(() => null);
+      const mime = cached?.length ? imageMimeType(cached) : null;
+      if (cached && mime) return { content: cached, mime };
+    }
+    let res: Response;
+    try {
+      res = await this.fetchImpl(url, {
+        signal: AbortSignal.timeout(this.opts.timeoutMs ?? 20_000),
+        headers: { accept: "image/avif,image/webp,image/apng,*/*" },
+      });
+    } catch (e) {
+      const reason = (e as Error).name === "TimeoutError" ? "超时" : (e as Error).message;
+      throw new ImageProxyError(502, `拉取 TMDB 图片失败：${reason}`);
+    }
+    if (res.status !== 200) {
+      await res.body?.cancel().catch(() => {});
+      throw new ImageProxyError(502, `TMDB 图片返回 HTTP ${res.status}`);
+    }
+    const content = new Uint8Array(await res.arrayBuffer());
+    const mime = imageMimeType(content);
+    if (!mime) throw new ImageProxyError(502, "TMDB 返回的不是图片");
+    await fs.mkdir(dirname(path), { recursive: true });
+    const tmp = `${path}.${process.pid}.tmp`;
+    await fs.writeFile(tmp, content);
+    await fs.rename(tmp, path);
+    return { content, mime };
+  }
+
+  private async files(): Promise<{ path: string; size: number; mtimeMs: number }[]> {
+    const out: { path: string; size: number; mtimeMs: number }[] = [];
+    const walk = async (d: string) => {
+      let names: string[];
+      try {
+        names = await fs.readdir(d);
+      } catch {
+        return;
+      }
+      for (const n of names) {
+        const p = join(d, n);
+        const st = await fs.stat(p).catch(() => null);
+        if (!st) continue;
+        if (st.isDirectory()) await walk(p);
+        else out.push({ path: p, size: st.size, mtimeMs: st.mtimeMs });
+      }
+    };
+    await walk(this.opts.dir);
+    return out;
+  }
+
+  async usage(): Promise<{ bytes: number; files: number }> {
+    const f = await this.files();
+    return { bytes: f.reduce((a, x) => a + x.size, 0), files: f.length };
+  }
+
+  /** SystemUtils.clear(images, days):删 mtime 早于 days 天的文件,再删空目录;返回删除数 */
+  async clean(days: number, now = Date.now()): Promise<number> {
+    let deleted = 0;
+    for (const f of await this.files()) {
+      if (Math.floor((now - f.mtimeMs) / 86_400_000) > days) {
+        await fs.rm(f.path, { force: true });
+        deleted++;
+      }
+    }
+    const prune = async (d: string): Promise<boolean> => {
+      const names = await fs.readdir(d).catch(() => null);
+      if (!names) return false;
+      let empty = true;
+      for (const n of names) {
+        const p = join(d, n);
+        const st = await fs.stat(p).catch(() => null);
+        if (st?.isDirectory() && (await prune(p))) await fs.rmdir(p).catch(() => {});
+        else empty = false;
+      }
+      return empty;
+    };
+    await prune(this.opts.dir);
+    return deleted;
+  }
+
+  async clear(): Promise<number> {
+    const n = (await this.files()).length;
+    await fs.rm(this.opts.dir, { recursive: true, force: true });
+    return n;
+  }
+}
@@ -0,0 +1,210 @@
+import { join } from "node:path";
+import { createRoute, type RouteConfig } from "@hono/zod-openapi";
+import {
+  Api,
+  apiResponse,
+  DEFAULT_SETTINGS,
+  type EndpointDef,
+  type TmdbCacheItem,
+  type TmdbCacheList,
+} from "@umt/contracts";
+import type { z } from "zod";
+import { dataPaths } from "../../config.ts";
+import { requireScope } from "../../http/auth.ts";
+import { errorBody } from "../../http/errors.ts";
+import { CACHE_REGIONS, sharedCache, tmdbCachePolicy } from "../../shared-cache.ts";
+import { registerScheduledJob } from "../schedules/registry.ts";
+import type { ModuleApp, ServerContext, ServerModule } from "../types.ts";
+import { ImageCache, ImageProxyError } from "./images.ts";
+
+/**
+ * C2-A 缓存(docs/runbook/cache.md):
+ * - PG 缓存 kv_cache(UNLOGGED,移植 MP TTLCache)的过期行物理清理任务 cache.sweep(每 10 分钟);
+ * - TMDB 缓存管理(对照 MP tmdb_cache_items / tmdb_cache_clear):GET /cache/tmdb、POST /cache/tmdb/clear;
+ * - 图片代理 GET /img/tmdb/{size}/{file}(会话或 API Key read,移植 MP ImageHelper / fetch_image)+ DATA_DIR/cache/images 落盘,
+ *   用量 / 清空 GET /cache/images、POST /cache/images/clear,按保留天数清理的任务 cache.images_clean(每天)。
+ */
+export interface CacheModuleOptions {
+  /** 图片上游 fetch(测试注入) */
+  imageFetch?: typeof globalThis.fetch;
+  /** 图片上游基址(测试指向 mock) */
+  imageUpstreamBase?: string;
+  /** 图片缓存目录;缺省 DATA_DIR/cache/images */
+  imageDir?: string;
+}
+
+type Infer<T> = T extends z.ZodType ? z.infer<T> : Record<string, never>;
+
+function route<D extends EndpointDef>(
+  app: ModuleApp,
+  def: D,
+  handler: (i: { query: Infer<D["query"]> }) => Promise<unknown>,
+) {
+  const request: RouteConfig["request"] = {};
+  if (def.query) request.query = def.query as never;
+  const r = createRoute({
+    method: def.method,
+    path: def.path,
+    summary: def.summary,
+    tags: ["cache"],
+    middleware: [requireScope(def.scope)] as never,
+    request,
+    responses: {
+      200: { description: def.summary, content: { "application/json": { schema: apiResponse(def.response) } } },
+    },
+  });
+  // biome-ignore lint/suspicious/noExplicitAny: 路由泛型在封装里擦除,入参已由 zod-openapi 校验
+  app.openapi(r, (async (c: any) => {
+    const data = def.response.parse(await handler({ query: def.query ? c.req.valid("query") : {} } as never));
+    return c.json({ ok: true, data }, 200);
+    // biome-ignore lint/suspicious/noExplicitAny: 同上
+  }) as any);
+}
+
+const str = (v: unknown) => (typeof v === "string" && v ? v : null);
+
+export function createCacheModule(opts: CacheModuleOptions = {}): ServerModule & {
+  images(ctx: ServerContext): ImageCache;
+} {
+  const imageCaches = new WeakMap<object, ImageCache>();
+  const images = (ctx: ServerContext) => {
+    let c = imageCaches.get(ctx);
+    if (!c) {
+      c = new ImageCache({
+        dir: opts.imageDir ?? join(dataPaths(ctx.config).dataDir, "cache", "images"),
+        ...(opts.imageFetch ? { fetch: opts.imageFetch } : {}),
+        ...(opts.imageUpstreamBase ? { upstreamBase: opts.imageUpstreamBase } : {}),
+      });
+      imageCaches.set(ctx, c);
+    }
+    return c;
+  };
+  const cacheSettings = (ctx: ServerContext) => ctx.settings.get("cache").catch(() => DEFAULT_SETTINGS.cache);
+
+  return {
+    name: "cache",
+    images,
+    registerRoutes(app, ctx) {
+      // 图片代理:不进 OpenAPI(二进制响应);<img> 带 cookie 会话即可,API Key 需 read
+      app.get("/img/tmdb/:size/:file", requireScope("read"), async (c) => {
+        const s = await cacheSettings(ctx);
+        try {
+          // MP cache_img:GLOBAL_IMAGE_CACHE 开启才读磁盘缓存
+          return await images(ctx).serve(
+            c.req.param("size"),
+            c.req.param("file"),
+            s.globalImageCache ?? false,
+            c.req.header("if-none-match") ?? null,
+          );
+        } catch (e) {
+          if (e instanceof ImageProxyError) {
+            return c.json(errorBody(e.status === 400 ? "bad_request" : "upstream_unavailable", e.message), e.status);
+          }
+          throw e;
+        }
+      });
+
+      route(app, Api.cache.tmdb, async ({ query }) => {
+        const cache = sharedCache(ctx);
+        const policy = await tmdbCachePolicy(ctx);
+        const want = (k: "request" | "recognize") => !query.kind || query.kind === k;
+        const items: TmdbCacheItem[] = [];
+        if (want("recognize")) {
+          for (const r of await cache.items<Record<string, unknown>>(CACHE_REGIONS.tmdbRecognize, query.limit)) {
+            const v = r.value ?? {};
+            const t = v.type === "movie" || v.type === "tv" ? v.type : "unknown";
+            items.push({
+              kind: "recognize",
+              key: r.key,
+              expiresAt: r.expiresAt.toISOString(),
+              updatedAt: r.updatedAt.toISOString(),
+              bytes: r.bytes,
+              tmdbId: typeof v.id === "number" ? v.id : Number(v.id) || 0,
+              title: str(v.title),
+              year: str(v.year),
+              mediaType: t,
+              posterPath: str(v.poster_path),
+            });
+          }
+        }
+        if (want("request")) {
+          for (const r of await cache.items(CACHE_REGIONS.tmdbRequest, query.limit)) {
+            items.push({
+              kind: "request",
+              key: r.key,
+              expiresAt: r.expiresAt.toISOString(),
+              updatedAt: r.updatedAt.toISOString(),
+              bytes: r.bytes,
+              tmdbId: null,
+              title: null,
+              year: null,
+              mediaType: null,
+              posterPath: null,
+            });
+          }
+        }
+        const out: TmdbCacheList = {
+          items,
+          counts: {
+            request: await cache.count(CACHE_REGIONS.tmdbRequest),
+            recognize: await cache.count(CACHE_REGIONS.tmdbRecognize),
+          },
+          maxSize: policy.maxsize,
+          ttlHours: Math.round(policy.ttl / 3600),
+          emptyResultTtlSec: policy.emptyTtl,
+        };
+        return out;
+      });
+      route(app, Api.cache.tmdbClear, async () => {
+        const cache = sharedCache(ctx);
+        const deleted =
+          (await cache.clear(CACHE_REGIONS.tmdbRecognize)) + (await cache.clear(CACHE_REGIONS.tmdbRequest));
+        ctx.logger.info({ deleted }, "TMDB 缓存已清空");
+        return { deleted };
+      });
+      route(app, Api.cache.images, async () => {
+        const s = await cacheSettings(ctx);
+        const img = images(ctx);
+        const u = await img.usage();
+        return {
+          enabled: s.globalImageCache ?? false,
+          days: s.globalImageCacheDays ?? DEFAULT_SETTINGS.cache.globalImageCacheDays,
+          usedBytes: u.bytes,
+          files: u.files,
+          dir: img.dir,
+        };
+      });
+      route(app, Api.cache.imagesClear, async () => ({ deleted: await images(ctx).clear() }));
+    },
+    registerTasks() {},
+    start(ctx: ServerContext) {
+      registerScheduledJob(ctx, {
+        key: "cache.sweep",
+        defaults: {
+          name: "缓存清理",
+          description: "删除 PG 缓存里已过期的条目（TMDB 请求与识别缓存）",
+          group: "maintain",
+          schedule: { type: "interval", everyMs: 10 * 60_000 },
+          firstRun: "after_interval",
+        },
+        run: async () => ({ summary: `删除 ${await sharedCache(ctx).sweep()} 条过期缓存` }),
+      });
+      registerScheduledJob(ctx, {
+        key: "cache.images_clean",
+        defaults: {
+          name: "图片缓存清理",
+          description: "删除超过保留天数的图片代理缓存文件（对照 MP 启动时的 clear_temp）",
+          group: "maintain",
+          schedule: { type: "interval", everyMs: 24 * 3_600_000 },
+        },
+        run: async () => {
+          const s = await cacheSettings(ctx);
+          const n = await images(ctx).clean(s.globalImageCacheDays ?? DEFAULT_SETTINGS.cache.globalImageCacheDays);
+          return { summary: `删除 ${n} 张过期图片` };
+        },
+      });
+    },
+  };
+}
+
+export const cacheModule = createCacheModule();