feat(cache): TMDB / 豆瓣 / Bangumi 接入统一元数据缓存与策略表
- 策略表「来源 × 资源类型」(metaCachePolicy):搜索 / 识别 7 天(空 1 天)、详情 / 季 / 剧集组 30 天(空 1 天)、 榜单 / 热门 / 每日放送 6 小时(空 1 小时)、人物 / 图片 90 天;取值来自 storage.cacheTtl, 追加 tmdbListHours / tmdbListEmptyHours / imageDays / metaDetailDays / metaMissDays,metaSearchDays 缺省改 7 天 - TMDB 客户端共享缓存端口加 fetch(带请求类别与空结果判定),服务端整条读取交给 MetadataCache; 识别缓存、douban.request / bangumi.request、douban.subject / bangumi.subject、meta.subject_miss、discover.match 都走这一层 - 豆瓣 / bgm 条目只在「不存在」时记查无,网络失败 / 被拦不缓存(本进程冷却 30 分钟) - 关注中作品(isEntityWatched 口径,含选中的剧集组)的 TMDB 条目写入 pinned、命中续期 - 各进程启动 LISTEN 失效通知并每分钟把命中计数写 cache_stat;GET /cache/regions 给元数据 region 带近 7 天命中统计 Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
ad472a1a19
commit
1dca95f883
16 files changed
+687
-137
No files matched your search
@@ -105,7 +105,15 @@ export interface TmdbConfig {
|
||||
* listTtlMs:流行趋势 / 热门 / 正在上映等列表与 /discover/ 的 TTL(缺省 6 小时且不超过 ttlMs;列表会变,不能按详情缓存 30 天),
|
||||
* 这类请求的空结果 TTL 也不超过它。
|
||||
*/
|
||||
cache?: { max?: number; ttlMs?: number; searchTtlMs?: number; emptyTtlMs?: number; listTtlMs?: number };
|
||||
cache?: {
|
||||
max?: number;
|
||||
ttlMs?: number;
|
||||
searchTtlMs?: number;
|
||||
emptyTtlMs?: number;
|
||||
listTtlMs?: number;
|
||||
/** 人物 / 图片列表(/person/*、/images);缺省同 ttlMs */
|
||||
imageTtlMs?: number;
|
||||
};
|
||||
/**
|
||||
* 跨进程共享的请求缓存(服务端传 PG 缓存);给了就不再用进程内 LRU。
|
||||
* 语义对照 MP @cached(skip_none、业务失败不缓存、results 为空按 emptyTtlMs)。
|
||||
@@ -124,6 +132,27 @@ type Json = Record<string, unknown>;
|
||||
export interface TmdbSharedCache {
|
||||
get(key: string): Promise<unknown | null>;
|
||||
set(key: string, value: unknown, ttlMs: number): Promise<void>;
|
||||
/**
|
||||
* 给了就由它负责整条读取路径(服务端统一元数据缓存:两级、过期先用旧值、同请求合并、统计、关注作品续期),
|
||||
* TTL 按 req.kind 取服务端策略表;load 抛错(网络、鉴权、限流、5xx)不缓存。没给时走 get / set。
|
||||
*/
|
||||
fetch?(
|
||||
req: { key: string; kind: TmdbRequestKind; isEmpty: (v: unknown) => boolean },
|
||||
load: () => Promise<unknown>,
|
||||
): Promise<unknown>;
|
||||
}
|
||||
|
||||
/** 请求类别(缓存 TTL 的依据):搜索 / find、列表、人物 / 图片、其余为详情(含季、剧集组) */
|
||||
export type TmdbRequestKind = "search" | "list" | "image" | "detail";
|
||||
|
||||
/** 人物 / 图片类请求:/person/*、…/images */
|
||||
export const isTmdbImagePath = (path: string) => /^\/person\//.test(path) || /\/images$/.test(path);
|
||||
|
||||
export function tmdbRequestKind(path: string): TmdbRequestKind {
|
||||
if (/^\/(search|find)\//.test(path)) return "search";
|
||||
if (isTmdbListPath(path)) return "list";
|
||||
if (isTmdbImagePath(path)) return "image";
|
||||
return "detail";
|
||||
}
|
||||
|
||||
function requestQuery(params: Record<string, string | undefined>): URLSearchParams {
|
||||
@@ -186,10 +215,16 @@ export class TmdbClient {
|
||||
const key = `${path}?${qs}`;
|
||||
let emptyTtl = this.cfg.cache?.emptyTtlMs ?? 30 * 60_000;
|
||||
const baseTtl = this.cfg.cache?.ttlMs ?? 24 * 3600_000;
|
||||
let ttl = /^\/(search|find)\//.test(path) ? (this.cfg.cache?.searchTtlMs ?? baseTtl) : baseTtl;
|
||||
if (isTmdbListPath(path)) {
|
||||
const kind = tmdbRequestKind(path);
|
||||
let ttl = kind === "search" ? (this.cfg.cache?.searchTtlMs ?? baseTtl) : baseTtl;
|
||||
if (kind === "list") {
|
||||
ttl = this.cfg.cache?.listTtlMs ?? Math.min(baseTtl, 6 * 3600_000);
|
||||
emptyTtl = Math.min(emptyTtl, ttl);
|
||||
} else if (kind === "image") ttl = this.cfg.cache?.imageTtlMs ?? baseTtl;
|
||||
if (this.cfg.sharedCache?.fetch) {
|
||||
return (await this.cfg.sharedCache.fetch({ key, kind, isEmpty: isEmptyResult }, () =>
|
||||
this.limiter.run(() => this.fetchWithRetry(path, qs)),
|
||||
)) as T;
|
||||
}
|
||||
if (this.cfg.sharedCache) {
|
||||
const shared = this.cfg.sharedCache;
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
matchTmdbMulti,
|
||||
type TmdbConfig,
|
||||
TmdbError,
|
||||
tmdbRequestKind,
|
||||
yearRange,
|
||||
} from "./index.ts";
|
||||
|
||||
@@ -135,6 +136,44 @@ describe("TmdbClient 缓存/限流/错误分类", () => {
|
||||
expect(tmdb.cacheStats().size).toBe(0);
|
||||
});
|
||||
|
||||
it("共享缓存给了 fetch:整条读取交给它,带请求类别(搜索 / 详情 / 列表 / 图片)与空结果判定;失败不进缓存", async () => {
|
||||
const seen: { key: string; kind: string; empty: boolean }[] = [];
|
||||
const store = new Map<string, unknown>();
|
||||
const { mock, tmdb } = await setup({
|
||||
sharedCache: {
|
||||
get: async () => null,
|
||||
set: async () => {},
|
||||
fetch: async (req, load) => {
|
||||
if (store.has(req.key)) return store.get(req.key);
|
||||
const v = await load();
|
||||
seen.push({ key: req.key, kind: req.kind, empty: req.isEmpty(v) });
|
||||
store.set(req.key, v);
|
||||
return v;
|
||||
},
|
||||
},
|
||||
});
|
||||
await tmdb.details("movie", 872585);
|
||||
await tmdb.details("movie", 872585);
|
||||
await tmdb.search("不存在的片名xyz", { type: "movie" });
|
||||
await tmdb.getJson("/movie/872585/images", {});
|
||||
await tmdb.getJson("/trending/all/week", {});
|
||||
await expect(tmdb.getJson("/movie/999999999", {})).rejects.toThrow();
|
||||
const byPath = new Map(seen.map((x) => [x.key.split("?")[0], [x.kind, x.empty]]));
|
||||
expect(Object.fromEntries(byPath)).toEqual({
|
||||
"/movie/872585": ["detail", false],
|
||||
"/search/movie": ["search", true],
|
||||
"/movie/872585/images": ["image", false],
|
||||
"/trending/all/week": ["list", false],
|
||||
});
|
||||
const detailCalls = mock.callsTo("/3/movie/872585").length;
|
||||
await tmdb.details("movie", 872585);
|
||||
expect(mock.callsTo("/3/movie/872585")).toHaveLength(detailCalls);
|
||||
expect(seen.some((x) => x.key.startsWith("/movie/999999999"))).toBe(false);
|
||||
expect(tmdbRequestKind("/person/1")).toBe("image");
|
||||
expect(tmdbRequestKind("/tv/1/season/2")).toBe("detail");
|
||||
expect(tmdbRequestKind("/find/tt1")).toBe("search");
|
||||
});
|
||||
|
||||
it("并发合并:同一请求并发只发一次", async () => {
|
||||
const { mock, tmdb } = await setup();
|
||||
await Promise.all(Array.from({ length: 5 }, () => tmdb.season(100088, 1)));
|
||||
|
||||
Reference in new issue
Block a user