feat(cache): TMDB / 豆瓣 / Bangumi 接入统一元数据缓存与策略表
- 策略表「来源 × 资源类型」(metaCachePolicy):搜索 / 识别 7 天(空 1 天)、详情 / 季 / 剧集组 30 天(空 1 天)、 榜单 / 热门 / 每日放送 6 小时(空 1 小时)、人物 / 图片 90 天;取值来自 storage.cacheTtl, 追加 tmdbListHours / tmdbListEmptyHours / imageDays / metaDetailDays / metaMissDays,metaSearchDays 缺省改 7 天 - TMDB 客户端共享缓存端口加 fetch(带请求类别与空结果判定),服务端整条读取交给 MetadataCache; 识别缓存、douban.request / bangumi.request、douban.subject / bangumi.subject、meta.subject_miss、discover.match 都走这一层 - 豆瓣 / bgm 条目只在「不存在」时记查无,网络失败 / 被拦不缓存(本进程冷却 30 分钟) - 关注中作品(isEntityWatched 口径,含选中的剧集组)的 TMDB 条目写入 pinned、命中续期 - 各进程启动 LISTEN 失效通知并每分钟把命中计数写 cache_stat;GET /cache/regions 给元数据 region 带近 7 天命中统计 Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
ad472a1a19
commit
1dca95f883
16 files changed
+687
-137
No files matched your search
@@ -105,7 +105,15 @@ export interface TmdbConfig {
|
||||
* listTtlMs:流行趋势 / 热门 / 正在上映等列表与 /discover/ 的 TTL(缺省 6 小时且不超过 ttlMs;列表会变,不能按详情缓存 30 天),
|
||||
* 这类请求的空结果 TTL 也不超过它。
|
||||
*/
|
||||
cache?: { max?: number; ttlMs?: number; searchTtlMs?: number; emptyTtlMs?: number; listTtlMs?: number };
|
||||
cache?: {
|
||||
max?: number;
|
||||
ttlMs?: number;
|
||||
searchTtlMs?: number;
|
||||
emptyTtlMs?: number;
|
||||
listTtlMs?: number;
|
||||
/** 人物 / 图片列表(/person/*、/images);缺省同 ttlMs */
|
||||
imageTtlMs?: number;
|
||||
};
|
||||
/**
|
||||
* 跨进程共享的请求缓存(服务端传 PG 缓存);给了就不再用进程内 LRU。
|
||||
* 语义对照 MP @cached(skip_none、业务失败不缓存、results 为空按 emptyTtlMs)。
|
||||
@@ -124,6 +132,27 @@ type Json = Record<string, unknown>;
|
||||
export interface TmdbSharedCache {
|
||||
get(key: string): Promise<unknown | null>;
|
||||
set(key: string, value: unknown, ttlMs: number): Promise<void>;
|
||||
/**
|
||||
* 给了就由它负责整条读取路径(服务端统一元数据缓存:两级、过期先用旧值、同请求合并、统计、关注作品续期),
|
||||
* TTL 按 req.kind 取服务端策略表;load 抛错(网络、鉴权、限流、5xx)不缓存。没给时走 get / set。
|
||||
*/
|
||||
fetch?(
|
||||
req: { key: string; kind: TmdbRequestKind; isEmpty: (v: unknown) => boolean },
|
||||
load: () => Promise<unknown>,
|
||||
): Promise<unknown>;
|
||||
}
|
||||
|
||||
/** 请求类别(缓存 TTL 的依据):搜索 / find、列表、人物 / 图片、其余为详情(含季、剧集组) */
|
||||
export type TmdbRequestKind = "search" | "list" | "image" | "detail";
|
||||
|
||||
/** 人物 / 图片类请求:/person/*、…/images */
|
||||
export const isTmdbImagePath = (path: string) => /^\/person\//.test(path) || /\/images$/.test(path);
|
||||
|
||||
export function tmdbRequestKind(path: string): TmdbRequestKind {
|
||||
if (/^\/(search|find)\//.test(path)) return "search";
|
||||
if (isTmdbListPath(path)) return "list";
|
||||
if (isTmdbImagePath(path)) return "image";
|
||||
return "detail";
|
||||
}
|
||||
|
||||
function requestQuery(params: Record<string, string | undefined>): URLSearchParams {
|
||||
@@ -186,10 +215,16 @@ export class TmdbClient {
|
||||
const key = `${path}?${qs}`;
|
||||
let emptyTtl = this.cfg.cache?.emptyTtlMs ?? 30 * 60_000;
|
||||
const baseTtl = this.cfg.cache?.ttlMs ?? 24 * 3600_000;
|
||||
let ttl = /^\/(search|find)\//.test(path) ? (this.cfg.cache?.searchTtlMs ?? baseTtl) : baseTtl;
|
||||
if (isTmdbListPath(path)) {
|
||||
const kind = tmdbRequestKind(path);
|
||||
let ttl = kind === "search" ? (this.cfg.cache?.searchTtlMs ?? baseTtl) : baseTtl;
|
||||
if (kind === "list") {
|
||||
ttl = this.cfg.cache?.listTtlMs ?? Math.min(baseTtl, 6 * 3600_000);
|
||||
emptyTtl = Math.min(emptyTtl, ttl);
|
||||
} else if (kind === "image") ttl = this.cfg.cache?.imageTtlMs ?? baseTtl;
|
||||
if (this.cfg.sharedCache?.fetch) {
|
||||
return (await this.cfg.sharedCache.fetch({ key, kind, isEmpty: isEmptyResult }, () =>
|
||||
this.limiter.run(() => this.fetchWithRetry(path, qs)),
|
||||
)) as T;
|
||||
}
|
||||
if (this.cfg.sharedCache) {
|
||||
const shared = this.cfg.sharedCache;
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
matchTmdbMulti,
|
||||
type TmdbConfig,
|
||||
TmdbError,
|
||||
tmdbRequestKind,
|
||||
yearRange,
|
||||
} from "./index.ts";
|
||||
|
||||
@@ -135,6 +136,44 @@ describe("TmdbClient 缓存/限流/错误分类", () => {
|
||||
expect(tmdb.cacheStats().size).toBe(0);
|
||||
});
|
||||
|
||||
it("共享缓存给了 fetch:整条读取交给它,带请求类别(搜索 / 详情 / 列表 / 图片)与空结果判定;失败不进缓存", async () => {
|
||||
const seen: { key: string; kind: string; empty: boolean }[] = [];
|
||||
const store = new Map<string, unknown>();
|
||||
const { mock, tmdb } = await setup({
|
||||
sharedCache: {
|
||||
get: async () => null,
|
||||
set: async () => {},
|
||||
fetch: async (req, load) => {
|
||||
if (store.has(req.key)) return store.get(req.key);
|
||||
const v = await load();
|
||||
seen.push({ key: req.key, kind: req.kind, empty: req.isEmpty(v) });
|
||||
store.set(req.key, v);
|
||||
return v;
|
||||
},
|
||||
},
|
||||
});
|
||||
await tmdb.details("movie", 872585);
|
||||
await tmdb.details("movie", 872585);
|
||||
await tmdb.search("不存在的片名xyz", { type: "movie" });
|
||||
await tmdb.getJson("/movie/872585/images", {});
|
||||
await tmdb.getJson("/trending/all/week", {});
|
||||
await expect(tmdb.getJson("/movie/999999999", {})).rejects.toThrow();
|
||||
const byPath = new Map(seen.map((x) => [x.key.split("?")[0], [x.kind, x.empty]]));
|
||||
expect(Object.fromEntries(byPath)).toEqual({
|
||||
"/movie/872585": ["detail", false],
|
||||
"/search/movie": ["search", true],
|
||||
"/movie/872585/images": ["image", false],
|
||||
"/trending/all/week": ["list", false],
|
||||
});
|
||||
const detailCalls = mock.callsTo("/3/movie/872585").length;
|
||||
await tmdb.details("movie", 872585);
|
||||
expect(mock.callsTo("/3/movie/872585")).toHaveLength(detailCalls);
|
||||
expect(seen.some((x) => x.key.startsWith("/movie/999999999"))).toBe(false);
|
||||
expect(tmdbRequestKind("/person/1")).toBe("image");
|
||||
expect(tmdbRequestKind("/tv/1/season/2")).toBe("detail");
|
||||
expect(tmdbRequestKind("/find/tt1")).toBe("search");
|
||||
});
|
||||
|
||||
it("并发合并:同一请求并发只发一次", async () => {
|
||||
const { mock, tmdb } = await setup();
|
||||
await Promise.all(Array.from({ length: 5 }, () => tmdb.season(100088, 1)));
|
||||
|
||||
@@ -82,6 +82,27 @@ export function tmdbImageUrl(path: string | null | undefined, size: TmdbImageSiz
|
||||
export const CacheRegionName = z.string().regex(/^[a-z0-9][a-z0-9._-]{0,63}$/, "缓存区名不合法");
|
||||
export const CacheRegionParam = z.object({ region: CacheRegionName });
|
||||
|
||||
/**
|
||||
* 命中统计口径(docs/runbook/cache.md):各进程在内存里计数,每分钟把增量加进 cache_stat(按天 × region),
|
||||
* 接口返回最近 7 天各进程之和 + 当前进程还没写入的部分(别的进程最近不到一分钟的计数不在内)。
|
||||
* hitRate = (l1Hits + l2Hits + staleHits) / (l1Hits + l2Hits + staleHits + misses);没有读取时为 null。
|
||||
*/
|
||||
export const CacheHitStats = z.object({
|
||||
/** 进程内存(一级)命中 */
|
||||
l1Hits: NonNegInt,
|
||||
/** PG(二级)命中 */
|
||||
l2Hits: NonNegInt,
|
||||
/** 过期在宽限期内、先返回旧值的次数 */
|
||||
staleHits: NonNegInt,
|
||||
misses: NonNegInt,
|
||||
/** 回源失败(网络、被拦、5xx) */
|
||||
failures: NonNegInt,
|
||||
/** 后台刷新次数 */
|
||||
refreshes: NonNegInt,
|
||||
hitRate: z.number().min(0).max(1).nullable(),
|
||||
});
|
||||
export type CacheHitStats = z.infer<typeof CacheHitStats>;
|
||||
|
||||
export const CacheRegionStat = z.object({
|
||||
region: z.string(),
|
||||
/** 未过期条目数 */
|
||||
@@ -92,6 +113,8 @@ export const CacheRegionStat = z.object({
|
||||
bytes: NonNegInt,
|
||||
/** 最近一次写入 */
|
||||
lastWriteAt: Timestamp.nullable(),
|
||||
/** 命中统计(统一元数据缓存的 region 才有):最近 7 天,见 CacheHitStats */
|
||||
hits: CacheHitStats.optional(),
|
||||
});
|
||||
export type CacheRegionStat = z.infer<typeof CacheRegionStat>;
|
||||
|
||||
|
||||
@@ -578,8 +578,19 @@ export const StorageCacheTtl = z.object({
|
||||
metaSearchDays: Days,
|
||||
/** 豆瓣 / bgm 榜单、热门、每日放送等列表请求(小时) */
|
||||
metaListHours: z.number().int().min(1).max(720),
|
||||
/** 豆瓣 / bgm 空结果(小时,对照 MP empty_ttl) */
|
||||
/** 豆瓣 / bgm 列表空结果(小时,对照 MP empty_ttl);搜索空结果改按 metaMissDays */
|
||||
metaEmptyHours: z.number().int().min(1).max(720),
|
||||
// ── 统一元数据缓存(2026-10-06,docs/runbook/cache.md 策略表);旧库缺这些字段时按缺省补 ──
|
||||
/** 豆瓣 / bgm 条目详情(douban.subject / bangumi.subject)与发现页条目 → TMDB 匹配(discover.match) */
|
||||
metaDetailDays: Days,
|
||||
/** 豆瓣 / bgm 查无结果:搜索空结果、条目不存在(meta.subject_miss) */
|
||||
metaMissDays: Days,
|
||||
/** TMDB 榜单 / 热门 / 趋势 / 按条件筛选(小时) */
|
||||
tmdbListHours: z.number().int().min(1).max(720),
|
||||
/** TMDB 列表空结果(小时) */
|
||||
tmdbListEmptyHours: z.number().int().min(1).max(720),
|
||||
/** TMDB 人物 / 图片列表(/person/*、/images) */
|
||||
imageDays: Days,
|
||||
});
|
||||
export type StorageCacheTtl = z.infer<typeof StorageCacheTtl>;
|
||||
|
||||
@@ -608,9 +619,14 @@ export const DEFAULT_STORAGE_SETTINGS: StorageSettings = {
|
||||
tmdbMissDays: 1,
|
||||
siteDetailDays: 7,
|
||||
webSearchDays: 1,
|
||||
metaSearchDays: 1,
|
||||
metaSearchDays: 7,
|
||||
metaListHours: 6,
|
||||
metaEmptyHours: 1,
|
||||
metaDetailDays: 30,
|
||||
metaMissDays: 1,
|
||||
tmdbListHours: 6,
|
||||
tmdbListEmptyHours: 1,
|
||||
imageDays: 90,
|
||||
},
|
||||
};
|
||||
|
||||
|
||||
Reference in new issue
Block a user