243 lines
12 KiB
TypeScript
243 lines
12 KiB
TypeScript
import { existsSync, mkdtempSync, rmSync, statSync, utimesSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import { PgCache, sql } from "@umt/db";
|
|
import { createMockTmdb, type MockTmdb } from "@umt/testkit";
|
|
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
|
import { ImageCache, imageCachePath } from "../../src/modules/cache/images.ts";
|
|
import { createCacheModule } from "../../src/modules/cache/index.ts";
|
|
import { scheduleRegistry } from "../../src/modules/schedules/registry.ts";
|
|
import { SourcesRuntime } from "../../src/modules/sources-identity/runtime.ts";
|
|
import { resetTrackedTmdbRefs, sharedCache } from "../../src/shared-cache.ts";
|
|
import { bearer, createHarness, type Harness, pgAvailable } from "../auth-settings/harness.ts";
|
|
import { DEV_KEY, json } from "../helpers.ts";
|
|
|
|
// C2-A 缓存:TMDB 请求缓存进 kv_cache(TTL / maxsize 取设置,对照 MP)、/cache/tmdb 列表与清空、cache.sweep、
|
|
// 图片代理(对照 MP ImageHelper:缓存路径、GLOBAL_IMAGE_CACHE 才读盘、md5 ETag / 304、按天清理)。
|
|
|
|
const hasPg = await pgAvailable();
|
|
const JPEG = new Uint8Array([0xff, 0xd8, 0xff, 0xe0, 1, 2, 3, 4]);
|
|
|
|
describe("图片缓存路径(MP sanitize_url_path)", () => {
|
|
it("URL 路径去掉开头 /;过长换 compressed_<sha16>;无扩展名补 .jpg", () => {
|
|
expect(imageCachePath("https://image.tmdb.org/t/p/w342/abc.jpg")).toBe("t/p/w342/abc.jpg");
|
|
expect(imageCachePath(`https://x.org/${"a".repeat(130)}.png`)).toMatch(/^compressed_[0-9a-f]{16}\.png$/);
|
|
expect(imageCachePath("https://x.org/t/p/w92/noext")).toBe("t/p/w92/noext.jpg");
|
|
});
|
|
});
|
|
|
|
describe.skipIf(!hasPg)("C2-A 缓存(真实 PG)", () => {
|
|
let h: Harness;
|
|
let admin: ReturnType<typeof bearer>;
|
|
let tmdb: MockTmdb;
|
|
const dir = mkdtempSync(join(tmpdir(), "umt-c2a-img-"));
|
|
let upstreamHits = 0;
|
|
const mod = createCacheModule({
|
|
imageDir: dir,
|
|
imageUpstreamBase: "https://image.tmdb.org/t/p",
|
|
imageFetch: async (input) => {
|
|
upstreamHits++;
|
|
if (String(input).includes("broken")) return new Response("<html>", { headers: { "content-type": "text/html" } });
|
|
return new Response(JPEG, { headers: { "content-type": "image/jpeg" } });
|
|
},
|
|
});
|
|
|
|
beforeAll(async () => {
|
|
h = await createHarness({ extraModules: [mod] });
|
|
admin = bearer(h.app, DEV_KEY);
|
|
tmdb = await createMockTmdb({ apiKey: "0123456789abcdef0123456789abcdef" });
|
|
});
|
|
afterAll(async () => {
|
|
await tmdb?.close();
|
|
await h?.close();
|
|
rmSync(dir, { recursive: true, force: true });
|
|
});
|
|
|
|
it("TMDB 请求响应进 kv_cache(tmdb.request,TTL 取 storage.cacheTtl:详情 / 搜索 / 未命中,条数上限 = cache.tmdbMaxSize),另一个进程直接命中", async () => {
|
|
const r = await admin("PATCH", "/settings", {
|
|
metadata: { tmdbApiKey: "0123456789abcdef0123456789abcdef", tmdbBaseUrl: `${tmdb.url}/3` },
|
|
cache: { tmdbMaxSize: 16 },
|
|
storage: { cacheTtl: { tmdbDetailDays: 20, tmdbSearchDays: 3, tmdbMissDays: 2 } },
|
|
});
|
|
expect(r.status).toBe(200);
|
|
// 部分更新只改给出的子字段,其余保持缺省
|
|
const st = (await json(await admin("GET", "/settings"))).data.storage;
|
|
expect(st.cacheTtl).toMatchObject({ tmdbDetailDays: 20, tmdbSearchDays: 3, tmdbMissDays: 2, siteDetailDays: 7 });
|
|
expect(st.budgets.imagesGb).toBe(20);
|
|
await (await new SourcesRuntime(h.ctx).tmdb())!.details("movie", 872585);
|
|
const calls = tmdb.calls.length;
|
|
const rows = (
|
|
await sql<{ key: string; days: number }>`select key, extract(epoch from expires_at - now()) / 86400 as days
|
|
from kv_cache where region = 'tmdb.request'`.execute(h.db)
|
|
).rows;
|
|
const detail = rows.find((x) => x.key.startsWith("/movie/872585?"))!;
|
|
expect(rows.every((x) => !x.key.includes("0123456789abcdef"))).toBe(true);
|
|
expect(Number(detail.days)).toBeGreaterThan(19.9);
|
|
expect(Number(detail.days)).toBeLessThanOrEqual(20);
|
|
expect(await new PgCache(h.db).get("tmdb.request", detail.key)).toMatchObject({ id: 872585 });
|
|
await (await new SourcesRuntime(h.ctx).tmdb())!.details("movie", 872585);
|
|
expect(tmdb.calls.length).toBe(calls);
|
|
const days = async (like: string) =>
|
|
Number(
|
|
(
|
|
await sql<{ d: number }>`select extract(epoch from expires_at - now()) / 86400 as d
|
|
from kv_cache where region = 'tmdb.request' and key like ${like}`.execute(h.db)
|
|
).rows[0]!.d,
|
|
);
|
|
// 有结果的搜索按 tmdbSearchDays
|
|
await (await new SourcesRuntime(h.ctx).tmdb())!.search("Oppenheimer", { type: "movie" });
|
|
const hit = await days("/search/movie?%Oppenheimer%");
|
|
expect(hit).toBeGreaterThan(2.9);
|
|
expect(hit).toBeLessThanOrEqual(3);
|
|
// 空搜索结果按 EMPTY_RESULT_CACHE_TTL 语义,取 tmdbMissDays
|
|
await (await new SourcesRuntime(h.ctx).tmdb())!.search("完全不存在的片名c2a", { type: "movie" });
|
|
const empty = await days("/search/movie?%c2a%");
|
|
expect(empty).toBeGreaterThan(1.9);
|
|
expect(empty).toBeLessThanOrEqual(2);
|
|
});
|
|
|
|
it("追踪中的作品:详情缓存命中时续期到 now + 详情 TTL;未追踪的不动", async () => {
|
|
const ent = (
|
|
await sql<{
|
|
id: string;
|
|
}>`insert into media_entity (kind, title) values ('movie', '奥本海默') returning id`.execute(h.db)
|
|
).rows[0]!.id;
|
|
await sql`insert into external_identity (media_entity_id, provider, entity_type, external_id, status)
|
|
values (${ent}, 'tmdb', 'movie', '872585', 'accepted')`.execute(h.db);
|
|
await sql`insert into subscription (media_entity_id, media_kind, status) values (${ent}, 'movie', 'active')`.execute(
|
|
h.db,
|
|
);
|
|
resetTrackedTmdbRefs(h.ctx);
|
|
const client = (await new SourcesRuntime(h.ctx).tmdb())!;
|
|
await client.details("movie", 872585);
|
|
await client.details("movie", 693134);
|
|
await sql`update kv_cache set expires_at = now() + interval '1 day'
|
|
where region = 'tmdb.request' and (key like '/movie/872585?%' or key like '/movie/693134?%')`.execute(h.db);
|
|
await client.details("movie", 872585);
|
|
await client.details("movie", 693134);
|
|
const left = async (like: string) =>
|
|
Number(
|
|
(
|
|
await sql<{ d: number }>`select extract(epoch from expires_at - now()) / 86400 as d
|
|
from kv_cache where region = 'tmdb.request' and key like ${like}`.execute(h.db)
|
|
).rows[0]!.d,
|
|
);
|
|
expect(await left("/movie/872585?%")).toBeGreaterThan(19.9);
|
|
expect(await left("/movie/693134?%")).toBeLessThanOrEqual(1);
|
|
});
|
|
|
|
it("GET /cache/tmdb 列出条目(键 / 过期 / 大小)与上限;POST /cache/tmdb/clear 清空", async () => {
|
|
await sharedCache(h.ctx).set(
|
|
"tmdb.recognize",
|
|
"[未知][zh-CN]Oppenheimer-2023-None",
|
|
{ id: 872585, type: "movie", year: "2023", title: "奥本海默", poster_path: "/p.jpg" },
|
|
{ ttl: 3600 },
|
|
);
|
|
const list = await json(await admin("GET", "/cache/tmdb"));
|
|
expect(list.data).toMatchObject({ maxSize: 16, ttlHours: 20 * 24, emptyResultTtlSec: 2 * 86400 });
|
|
expect(list.data.counts.request).toBeGreaterThan(0);
|
|
const rec = list.data.items.find((i: { kind: string }) => i.kind === "recognize");
|
|
expect(rec).toMatchObject({ tmdbId: 872585, title: "奥本海默", mediaType: "movie", posterPath: "/p.jpg" });
|
|
expect(rec.bytes).toBeGreaterThan(0);
|
|
const cleared = await json(await admin("POST", "/cache/tmdb/clear"));
|
|
expect(cleared.data.deleted).toBeGreaterThan(1);
|
|
expect((await json(await admin("GET", "/cache/tmdb"))).data.items).toEqual([]);
|
|
});
|
|
|
|
it("cache.sweep / cache.images_clean 登记为维护任务;sweep 删除过期行", async () => {
|
|
await mod.start?.(h.ctx);
|
|
const reg = scheduleRegistry(h.ctx);
|
|
expect(reg.get("cache.sweep")?.defaults).toMatchObject({ group: "maintain", schedule: { everyMs: 600_000 } });
|
|
expect(reg.get("cache.images_clean")?.defaults.group).toBe("maintain");
|
|
await sharedCache(h.ctx).set("t", "gone", 1, { ttl: 60 });
|
|
await sql`update kv_cache set expires_at = now() - interval '1 second' where region = 't'`.execute(h.db);
|
|
const run = {
|
|
jobId: "cache.sweep",
|
|
runId: "r",
|
|
trigger: "manual" as const,
|
|
now: new Date(),
|
|
config: {},
|
|
slot: "1",
|
|
};
|
|
expect((await reg.get("cache.sweep")!.run(run))?.summary).toMatch(/删除 [1-9]\d* 条过期缓存/);
|
|
});
|
|
|
|
it("图片代理:只收 TMDB 尺寸 / 文件名;GLOBAL_IMAGE_CACHE 关闭时每次回源(仍落盘),开启后读盘;md5 ETag / 304;非图片 502;按天清理", async () => {
|
|
const get = (p: string, headers: Record<string, string> = {}) =>
|
|
h.app.request(`/api/v1${p}`, { headers: { authorization: `Bearer ${DEV_KEY}`, ...headers } });
|
|
expect((await get("/img/tmdb/w9999/a.jpg")).status).toBe(400);
|
|
expect((await get("/img/tmdb/w342/..%2Fetc.jpg")).status).toBe(400);
|
|
expect((await h.app.request("/api/v1/img/tmdb/w342/a.jpg")).status).toBe(401);
|
|
const r1 = await get("/img/tmdb/w342/a.jpg");
|
|
expect(r1.status).toBe(200);
|
|
expect(r1.headers.get("content-type")).toBe("image/jpeg");
|
|
expect(r1.headers.get("cache-control")).toBe("public, max-age=604800");
|
|
const etag = r1.headers.get("etag")!;
|
|
expect(etag).toMatch(/^[0-9a-f]{32}$/);
|
|
await get("/img/tmdb/w342/a.jpg");
|
|
expect(upstreamHits).toBe(2);
|
|
await admin("PATCH", "/settings", { cache: { globalImageCache: true } });
|
|
const r3 = await get("/img/tmdb/w342/a.jpg", { "if-none-match": etag });
|
|
expect(r3.status).toBe(304);
|
|
expect(upstreamHits).toBe(2);
|
|
expect((await get("/img/tmdb/w342/broken.jpg")).status).toBe(502);
|
|
const usage = await json(await admin("GET", "/cache/images"));
|
|
expect(usage.data).toMatchObject({ enabled: true, days: 7, files: 1 });
|
|
const old = new Date(Date.now() - 9 * 86_400_000);
|
|
utimesSync(join(dir, "t/p/w342/a.jpg"), old, old);
|
|
expect(await mod.images(h.ctx).clean(7)).toBe(1);
|
|
expect((await json(await admin("GET", "/cache/images"))).data.files).toBe(0);
|
|
expect((await json(await admin("POST", "/cache/images/clear"))).data.deleted).toBe(0);
|
|
});
|
|
|
|
it("图片容量 LRU:超过预算按最久未访问(atime)删到预算的 90%;读盘命中刷新 atime、不动 mtime", async () => {
|
|
const lruDir = mkdtempSync(join(tmpdir(), "umt-c2-lru-"));
|
|
try {
|
|
const cache = new ImageCache({
|
|
dir: lruDir,
|
|
upstreamBase: "https://image.tmdb.org/t/p",
|
|
fetch: async () => new Response(new Uint8Array([...JPEG, ...new Uint8Array(92)])),
|
|
});
|
|
// 4 张 100 字节的图,访问时间从旧到新 a < b < c < d
|
|
for (const f of ["a", "b", "c", "d"]) await cache.serve("w92", `${f}.jpg`, false, null);
|
|
const p = (f: string) => join(lruDir, "t/p/w92", `${f}.jpg`);
|
|
const base = Date.now() - 10 * 86_400_000;
|
|
["a", "b", "c", "d"].forEach((f, i) => {
|
|
utimesSync(p(f), new Date(base + i * 3600_000), new Date(base));
|
|
});
|
|
// 读盘命中 a:atime 变成现在(最近使用),mtime 仍是写入时间
|
|
await cache.serve("w92", "a.jpg", true, null);
|
|
await new Promise((r) => setTimeout(r, 50));
|
|
const st = statSync(p("a"));
|
|
expect(st.atimeMs).toBeGreaterThan(Date.now() - 60_000);
|
|
expect(Math.abs(st.mtimeMs - base)).toBeLessThan(1000);
|
|
// 没超预算不动
|
|
expect(await cache.enforceBudget(1000)).toMatchObject({ deleted: 0, usedBytes: 400 });
|
|
// 预算 300 → 删到 ≤ 270:最久未访问的 b、c 被删,a(刚读过)与 d 保留
|
|
const r = await cache.enforceBudget(300);
|
|
expect(r).toMatchObject({ deleted: 2, freedBytes: 200, usedBytes: 200 });
|
|
expect(existsSync(p("a"))).toBe(true);
|
|
expect(existsSync(p("b"))).toBe(false);
|
|
expect(existsSync(p("c"))).toBe(false);
|
|
expect(existsSync(p("d"))).toBe(true);
|
|
} finally {
|
|
rmSync(lruDir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
it("cache.images_lru 登记为维护任务,预算取 storage.budgets.imagesGb", async () => {
|
|
await mod.start?.(h.ctx);
|
|
const job = scheduleRegistry(h.ctx).get("cache.images_lru");
|
|
expect(job?.defaults).toMatchObject({ group: "maintain", schedule: { everyMs: 30 * 60_000 } });
|
|
const out = await job!.run({
|
|
jobId: "cache.images_lru",
|
|
runId: "r",
|
|
trigger: "manual",
|
|
now: new Date(),
|
|
config: {},
|
|
slot: "1",
|
|
});
|
|
expect(out?.summary).toMatch(/未超过预算 20 GB/);
|
|
});
|
|
});
|