merge: feat/y-meta-cache
Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei
This commit is contained in:
rootandClaude Opus 5.5 committed
commit
6d37a337c9
51 files changed
+2407
-200
No files matched your search
@@ -105,7 +105,15 @@ export interface TmdbConfig {
|
||||
* listTtlMs:流行趋势 / 热门 / 正在上映等列表与 /discover/ 的 TTL(缺省 6 小时且不超过 ttlMs;列表会变,不能按详情缓存 30 天),
|
||||
* 这类请求的空结果 TTL 也不超过它。
|
||||
*/
|
||||
cache?: { max?: number; ttlMs?: number; searchTtlMs?: number; emptyTtlMs?: number; listTtlMs?: number };
|
||||
cache?: {
|
||||
max?: number;
|
||||
ttlMs?: number;
|
||||
searchTtlMs?: number;
|
||||
emptyTtlMs?: number;
|
||||
listTtlMs?: number;
|
||||
/** 人物 / 图片列表(/person/*、/images);缺省同 ttlMs */
|
||||
imageTtlMs?: number;
|
||||
};
|
||||
/**
|
||||
* 跨进程共享的请求缓存(服务端传 PG 缓存);给了就不再用进程内 LRU。
|
||||
* 语义对照 MP @cached(skip_none、业务失败不缓存、results 为空按 emptyTtlMs)。
|
||||
@@ -124,6 +132,27 @@ type Json = Record<string, unknown>;
|
||||
export interface TmdbSharedCache {
|
||||
get(key: string): Promise<unknown | null>;
|
||||
set(key: string, value: unknown, ttlMs: number): Promise<void>;
|
||||
/**
|
||||
* 给了就由它负责整条读取路径(服务端统一元数据缓存:两级、过期先用旧值、同请求合并、统计、关注作品续期),
|
||||
* TTL 按 req.kind 取服务端策略表;load 抛错(网络、鉴权、限流、5xx)不缓存。没给时走 get / set。
|
||||
*/
|
||||
fetch?(
|
||||
req: { key: string; kind: TmdbRequestKind; isEmpty: (v: unknown) => boolean },
|
||||
load: () => Promise<unknown>,
|
||||
): Promise<unknown>;
|
||||
}
|
||||
|
||||
/** 请求类别(缓存 TTL 的依据):搜索 / find、列表、人物 / 图片、其余为详情(含季、剧集组) */
|
||||
export type TmdbRequestKind = "search" | "list" | "image" | "detail";
|
||||
|
||||
/** 人物 / 图片类请求:/person/*、…/images */
|
||||
export const isTmdbImagePath = (path: string) => /^\/person\//.test(path) || /\/images$/.test(path);
|
||||
|
||||
export function tmdbRequestKind(path: string): TmdbRequestKind {
|
||||
if (/^\/(search|find)\//.test(path)) return "search";
|
||||
if (isTmdbListPath(path)) return "list";
|
||||
if (isTmdbImagePath(path)) return "image";
|
||||
return "detail";
|
||||
}
|
||||
|
||||
function requestQuery(params: Record<string, string | undefined>): URLSearchParams {
|
||||
@@ -186,10 +215,16 @@ export class TmdbClient {
|
||||
const key = `${path}?${qs}`;
|
||||
let emptyTtl = this.cfg.cache?.emptyTtlMs ?? 30 * 60_000;
|
||||
const baseTtl = this.cfg.cache?.ttlMs ?? 24 * 3600_000;
|
||||
let ttl = /^\/(search|find)\//.test(path) ? (this.cfg.cache?.searchTtlMs ?? baseTtl) : baseTtl;
|
||||
if (isTmdbListPath(path)) {
|
||||
const kind = tmdbRequestKind(path);
|
||||
let ttl = kind === "search" ? (this.cfg.cache?.searchTtlMs ?? baseTtl) : baseTtl;
|
||||
if (kind === "list") {
|
||||
ttl = this.cfg.cache?.listTtlMs ?? Math.min(baseTtl, 6 * 3600_000);
|
||||
emptyTtl = Math.min(emptyTtl, ttl);
|
||||
} else if (kind === "image") ttl = this.cfg.cache?.imageTtlMs ?? baseTtl;
|
||||
if (this.cfg.sharedCache?.fetch) {
|
||||
return (await this.cfg.sharedCache.fetch({ key, kind, isEmpty: isEmptyResult }, () =>
|
||||
this.limiter.run(() => this.fetchWithRetry(path, qs)),
|
||||
)) as T;
|
||||
}
|
||||
if (this.cfg.sharedCache) {
|
||||
const shared = this.cfg.sharedCache;
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
parseAltTitles,
|
||||
parseTranslations,
|
||||
pickChineseTitle,
|
||||
pickLogo,
|
||||
pickText,
|
||||
pickTitle,
|
||||
} from "./index.ts";
|
||||
@@ -223,3 +224,70 @@ describe("fetchRichDetail / fetchRichSeason(mock TMDB)", () => {
|
||||
expect(mock.calls.length).toBe(n);
|
||||
});
|
||||
});
|
||||
|
||||
describe("TMDB 标志(clearlogo)", () => {
|
||||
const logos = (rows: { file_path: string; iso_639_1: string | null; vote_average?: number }[]) => ({ logos: rows });
|
||||
|
||||
it("语言优先 原语言 → 中文 → 英文 → 无文字;同语言按评分;只要 PNG", () => {
|
||||
const all = logos([
|
||||
{ file_path: "/none.png", iso_639_1: null, vote_average: 9 },
|
||||
{ file_path: "/en.png", iso_639_1: "en", vote_average: 5 },
|
||||
{ file_path: "/zh-low.png", iso_639_1: "zh", vote_average: 3 },
|
||||
{ file_path: "/zh-high.png", iso_639_1: "zh", vote_average: 6 },
|
||||
{ file_path: "/ja.svg", iso_639_1: "ja", vote_average: 9 },
|
||||
{ file_path: "/ja.png", iso_639_1: "ja", vote_average: 1 },
|
||||
]);
|
||||
expect(pickLogo([all], "ja")).toBe("/ja.png");
|
||||
expect(pickLogo([all], "ko")).toBe("/zh-high.png");
|
||||
expect(
|
||||
pickLogo(
|
||||
[
|
||||
logos([
|
||||
{ file_path: "/en.png", iso_639_1: "en" },
|
||||
{ file_path: "/n.png", iso_639_1: null },
|
||||
]),
|
||||
],
|
||||
"zh",
|
||||
),
|
||||
).toBe("/en.png");
|
||||
expect(pickLogo([logos([{ file_path: "/n.png", iso_639_1: null }])], "en")).toBe("/n.png");
|
||||
expect(pickLogo([logos([{ file_path: "/x.svg", iso_639_1: "en" }])], "en")).toBeNull();
|
||||
expect(pickLogo([undefined], null)).toBeNull();
|
||||
});
|
||||
|
||||
it("详情请求带 images(zh,en,null);原语言不是中英文时另取原语言标志", async () => {
|
||||
const fx = TMDB_FIXTURES.map((f) =>
|
||||
f.id === 872585
|
||||
? {
|
||||
...f,
|
||||
logos: [
|
||||
{ file_path: "/oppen-en.png", iso_639_1: "en" },
|
||||
{ file_path: "/oppen-zh.png", iso_639_1: "zh" },
|
||||
],
|
||||
}
|
||||
: f.id === 95396
|
||||
? {
|
||||
...f,
|
||||
original_language: "ja",
|
||||
logos: [
|
||||
{ file_path: "/sev-ja.png", iso_639_1: "ja" },
|
||||
{ file_path: "/sev-en.png", iso_639_1: "en" },
|
||||
],
|
||||
}
|
||||
: f,
|
||||
);
|
||||
mock = await createMockTmdb({ fixtures: fx });
|
||||
const tmdb = createTmdbClient({ baseUrl: `${mock.url}/3`, apiKey: "test-tmdb-key", retries: 0, timeoutMs: 1000 });
|
||||
const movie = await fetchRichDetail(tmdb, "movie", 872585, CHAIN);
|
||||
expect(movie.logoPath).toBe("/oppen-en.png");
|
||||
const call = mock.calls.find((c) => c.path === "/3/movie/872585")!;
|
||||
expect(call.query).toMatchObject({ include_image_language: "zh,en,null" });
|
||||
expect(call.query.append_to_response).toContain("images");
|
||||
expect(mock.calls.some((c) => c.path === "/3/movie/872585/images")).toBe(false);
|
||||
const tv = await fetchRichDetail(tmdb, "tv", 95396, CHAIN);
|
||||
expect(tv.logoPath).toBe("/sev-ja.png");
|
||||
expect(mock.calls.find((c) => c.path === "/3/tv/95396/images")?.query).toMatchObject({
|
||||
include_image_language: "ja",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -82,6 +82,8 @@ export interface TmdbRichDetail {
|
||||
originCountry: string[];
|
||||
posterPath: string | null;
|
||||
backdropPath: string | null;
|
||||
/** 标志(透明 PNG,刮削写 clearlogo.png);语言优先 原语言 → 中文 → 英文 → 无文字;没有为 null(旧数据可缺省) */
|
||||
logoPath?: string | null;
|
||||
cast: TmdbCredit[];
|
||||
crew: TmdbCredit[];
|
||||
externalIds: { imdb: string | null; tvdb: string | null; wikidata: string | null };
|
||||
@@ -163,9 +165,48 @@ function credits(raw: unknown, type: MediaEntityType, createdBy: unknown): { cas
|
||||
return { cast, crew: crew.slice(0, 6) };
|
||||
}
|
||||
|
||||
/** 详情请求带回的图片语言:中文、英文、无文字;作品原语言不在其中时另取一次 /images */
|
||||
export const DETAIL_IMAGE_LANGUAGES = "zh,en,null";
|
||||
|
||||
interface LogoRow {
|
||||
path: string;
|
||||
lang: string | null;
|
||||
vote: number;
|
||||
count: number;
|
||||
}
|
||||
|
||||
const logoRows = (images: unknown): LogoRow[] => {
|
||||
const logos = ((images ?? {}) as Json).logos;
|
||||
return ((Array.isArray(logos) ? logos : []) as Json[]).flatMap((x) => {
|
||||
const path = s(x.file_path);
|
||||
// 只要 PNG:Kodi / Emby / Jellyfin 的 clearlogo 都是 png,TMDB 的 svg 标志不用
|
||||
if (!path || !/\.png$/i.test(path)) return [];
|
||||
return [{ path, lang: s(x.iso_639_1), vote: n(x.vote_average) ?? 0, count: n(x.vote_count) ?? 0 }];
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* 富详情:一次请求 `/{type}/{id}?language=链首&append_to_response=alternative_titles,credits,external_ids,translations`,
|
||||
* 标题 / 简介 / 标语按链从 translations(+ 同地区别名)取。不存在抛 TmdbError(notfound)。
|
||||
* 选一张标志:语言优先 作品原语言 → 中文 → 英文 → 无文字;同语言按评分、票数从高到低。没有 PNG 标志返回 null。
|
||||
* images 可以是多份(详情带回的 + 原语言单独取的)。
|
||||
*/
|
||||
export function pickLogo(images: unknown[], originalLanguage: string | null): string | null {
|
||||
const rows = images.flatMap(logoRows);
|
||||
const orig = originalLanguage?.toLowerCase().split("-")[0] ?? null;
|
||||
const order = [orig, "zh", "en", null].filter((x, i, a) => a.indexOf(x) === i && (x !== null || i === a.length - 1));
|
||||
for (const lang of order) {
|
||||
const hit = rows
|
||||
.filter((r) => (r.lang?.toLowerCase() ?? null) === lang)
|
||||
.sort((a, b) => b.vote - a.vote || b.count - a.count)[0];
|
||||
if (hit) return hit.path;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 富详情:一次请求 `/{type}/{id}?language=链首&append_to_response=alternative_titles,credits,external_ids,translations,images`
|
||||
* (`include_image_language=zh,en,null`),标题 / 简介 / 标语按链从 translations(+ 同地区别名)取。
|
||||
* 作品原语言不是中文 / 英文时另取 `/{type}/{id}/images?include_image_language=<原语言>` 选标志(失败忽略)。
|
||||
* 不存在抛 TmdbError(notfound)。
|
||||
*/
|
||||
export async function fetchRichDetail(
|
||||
client: Pick<TmdbClient, "getJson">,
|
||||
@@ -177,7 +218,8 @@ export async function fetchRichDetail(
|
||||
const requested = chainOf(languages);
|
||||
const j = await client.getJson<Json>(`/${type}/${id}`, {
|
||||
language: requested[0],
|
||||
append_to_response: "alternative_titles,credits,external_ids,translations",
|
||||
append_to_response: "alternative_titles,credits,external_ids,translations,images",
|
||||
include_image_language: DETAIL_IMAGE_LANGUAGES,
|
||||
});
|
||||
const titleKey = type === "movie" ? "title" : "name";
|
||||
const originalTitle = s(type === "movie" ? j.original_title : j.original_name);
|
||||
@@ -212,6 +254,15 @@ export async function fetchRichDetail(
|
||||
const ext = (j.external_ids ?? {}) as Json;
|
||||
const runtimes = Array.isArray(j.episode_run_time) ? (j.episode_run_time as unknown[]) : [];
|
||||
const { cast, crew } = credits(j.credits, type, j.created_by);
|
||||
const logoSources: unknown[] = [j.images];
|
||||
const origLang = originalLanguage?.toLowerCase().split("-")[0] ?? null;
|
||||
if (origLang && origLang !== "zh" && origLang !== "en") {
|
||||
try {
|
||||
logoSources.unshift(await client.getJson<Json>(`/${type}/${id}/images`, { include_image_language: origLang }));
|
||||
} catch {
|
||||
// 原语言标志取不到就用中文 / 英文 / 无文字的
|
||||
}
|
||||
}
|
||||
return {
|
||||
entityType: type,
|
||||
externalId: String(id),
|
||||
@@ -240,6 +291,7 @@ export async function fetchRichDetail(
|
||||
originCountry,
|
||||
posterPath: s(j.poster_path),
|
||||
backdropPath: s(j.backdrop_path),
|
||||
logoPath: pickLogo(logoSources, originalLanguage),
|
||||
cast,
|
||||
crew,
|
||||
externalIds: {
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
matchTmdbMulti,
|
||||
type TmdbConfig,
|
||||
TmdbError,
|
||||
tmdbRequestKind,
|
||||
yearRange,
|
||||
} from "./index.ts";
|
||||
|
||||
@@ -135,6 +136,44 @@ describe("TmdbClient 缓存/限流/错误分类", () => {
|
||||
expect(tmdb.cacheStats().size).toBe(0);
|
||||
});
|
||||
|
||||
it("共享缓存给了 fetch:整条读取交给它,带请求类别(搜索 / 详情 / 列表 / 图片)与空结果判定;失败不进缓存", async () => {
|
||||
const seen: { key: string; kind: string; empty: boolean }[] = [];
|
||||
const store = new Map<string, unknown>();
|
||||
const { mock, tmdb } = await setup({
|
||||
sharedCache: {
|
||||
get: async () => null,
|
||||
set: async () => {},
|
||||
fetch: async (req, load) => {
|
||||
if (store.has(req.key)) return store.get(req.key);
|
||||
const v = await load();
|
||||
seen.push({ key: req.key, kind: req.kind, empty: req.isEmpty(v) });
|
||||
store.set(req.key, v);
|
||||
return v;
|
||||
},
|
||||
},
|
||||
});
|
||||
await tmdb.details("movie", 872585);
|
||||
await tmdb.details("movie", 872585);
|
||||
await tmdb.search("不存在的片名xyz", { type: "movie" });
|
||||
await tmdb.getJson("/movie/872585/images", {});
|
||||
await tmdb.getJson("/trending/all/week", {});
|
||||
await expect(tmdb.getJson("/movie/999999999", {})).rejects.toThrow();
|
||||
const byPath = new Map(seen.map((x) => [x.key.split("?")[0], [x.kind, x.empty]]));
|
||||
expect(Object.fromEntries(byPath)).toEqual({
|
||||
"/movie/872585": ["detail", false],
|
||||
"/search/movie": ["search", true],
|
||||
"/movie/872585/images": ["image", false],
|
||||
"/trending/all/week": ["list", false],
|
||||
});
|
||||
const detailCalls = mock.callsTo("/3/movie/872585").length;
|
||||
await tmdb.details("movie", 872585);
|
||||
expect(mock.callsTo("/3/movie/872585")).toHaveLength(detailCalls);
|
||||
expect(seen.some((x) => x.key.startsWith("/movie/999999999"))).toBe(false);
|
||||
expect(tmdbRequestKind("/person/1")).toBe("image");
|
||||
expect(tmdbRequestKind("/tv/1/season/2")).toBe("detail");
|
||||
expect(tmdbRequestKind("/find/tt1")).toBe("search");
|
||||
});
|
||||
|
||||
it("并发合并:同一请求并发只发一次", async () => {
|
||||
const { mock, tmdb } = await setup();
|
||||
await Promise.all(Array.from({ length: 5 }, () => tmdb.season(100088, 1)));
|
||||
|
||||
@@ -82,6 +82,27 @@ export function tmdbImageUrl(path: string | null | undefined, size: TmdbImageSiz
|
||||
export const CacheRegionName = z.string().regex(/^[a-z0-9][a-z0-9._-]{0,63}$/, "缓存区名不合法");
|
||||
export const CacheRegionParam = z.object({ region: CacheRegionName });
|
||||
|
||||
/**
|
||||
* 命中统计口径(docs/runbook/cache.md):各进程在内存里计数,每分钟把增量加进 cache_stat(按天 × region),
|
||||
* 接口返回最近 7 天各进程之和 + 当前进程还没写入的部分(别的进程最近不到一分钟的计数不在内)。
|
||||
* hitRate = (l1Hits + l2Hits + staleHits) / (l1Hits + l2Hits + staleHits + misses);没有读取时为 null。
|
||||
*/
|
||||
export const CacheHitStats = z.object({
|
||||
/** 进程内存(一级)命中 */
|
||||
l1Hits: NonNegInt,
|
||||
/** PG(二级)命中 */
|
||||
l2Hits: NonNegInt,
|
||||
/** 过期在宽限期内、先返回旧值的次数 */
|
||||
staleHits: NonNegInt,
|
||||
misses: NonNegInt,
|
||||
/** 回源失败(网络、被拦、5xx) */
|
||||
failures: NonNegInt,
|
||||
/** 后台刷新次数 */
|
||||
refreshes: NonNegInt,
|
||||
hitRate: z.number().min(0).max(1).nullable(),
|
||||
});
|
||||
export type CacheHitStats = z.infer<typeof CacheHitStats>;
|
||||
|
||||
export const CacheRegionStat = z.object({
|
||||
region: z.string(),
|
||||
/** 未过期条目数 */
|
||||
@@ -92,6 +113,8 @@ export const CacheRegionStat = z.object({
|
||||
bytes: NonNegInt,
|
||||
/** 最近一次写入 */
|
||||
lastWriteAt: Timestamp.nullable(),
|
||||
/** 命中统计(统一元数据缓存的 region 才有):最近 7 天,见 CacheHitStats */
|
||||
hits: CacheHitStats.optional(),
|
||||
});
|
||||
export type CacheRegionStat = z.infer<typeof CacheRegionStat>;
|
||||
|
||||
|
||||
@@ -220,6 +220,8 @@ export const EntityMetadataField = z.enum([
|
||||
"cast",
|
||||
"crew",
|
||||
"externalIds",
|
||||
// 统一元数据缓存一轮:TMDB 标志(刮削写 clearlogo.png)
|
||||
"logoPath",
|
||||
]);
|
||||
export type EntityMetadataField = z.infer<typeof EntityMetadataField>;
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ import { Id, NonNegInt, Timestamp } from "./common.ts";
|
||||
export const ScrapeFileResult = z.object({
|
||||
path: z.string(),
|
||||
kind: z.enum(["nfo", "image"]),
|
||||
/** movie_nfo / tvshow_nfo / season_nfo / episode_nfo / poster / fanart / season_poster / episode_thumb */
|
||||
/** movie_nfo / tvshow_nfo / season_nfo / episode_nfo / poster / fanart / clearlogo / season_poster / episode_thumb */
|
||||
role: z.string(),
|
||||
action: z.enum(["written", "skipped", "failed"]),
|
||||
/** 跳过 / 失败原因码(reasons.ts:scrape_exists / scrape_write_failed / scrape_image_failed …) */
|
||||
|
||||
@@ -578,8 +578,19 @@ export const StorageCacheTtl = z.object({
|
||||
metaSearchDays: Days,
|
||||
/** 豆瓣 / bgm 榜单、热门、每日放送等列表请求(小时) */
|
||||
metaListHours: z.number().int().min(1).max(720),
|
||||
/** 豆瓣 / bgm 空结果(小时,对照 MP empty_ttl) */
|
||||
/** 豆瓣 / bgm 列表空结果(小时,对照 MP empty_ttl);搜索空结果改按 metaMissDays */
|
||||
metaEmptyHours: z.number().int().min(1).max(720),
|
||||
// ── 统一元数据缓存(2026-10-06,docs/runbook/cache.md 策略表);旧库缺这些字段时按缺省补 ──
|
||||
/** 豆瓣 / bgm 条目详情(douban.subject / bangumi.subject)与发现页条目 → TMDB 匹配(discover.match) */
|
||||
metaDetailDays: Days,
|
||||
/** 豆瓣 / bgm 查无结果:搜索空结果、条目不存在(meta.subject_miss) */
|
||||
metaMissDays: Days,
|
||||
/** TMDB 榜单 / 热门 / 趋势 / 按条件筛选(小时) */
|
||||
tmdbListHours: z.number().int().min(1).max(720),
|
||||
/** TMDB 列表空结果(小时) */
|
||||
tmdbListEmptyHours: z.number().int().min(1).max(720),
|
||||
/** TMDB 人物 / 图片列表(/person/*、/images) */
|
||||
imageDays: Days,
|
||||
});
|
||||
export type StorageCacheTtl = z.infer<typeof StorageCacheTtl>;
|
||||
|
||||
@@ -608,9 +619,14 @@ export const DEFAULT_STORAGE_SETTINGS: StorageSettings = {
|
||||
tmdbMissDays: 1,
|
||||
siteDetailDays: 7,
|
||||
webSearchDays: 1,
|
||||
metaSearchDays: 1,
|
||||
metaSearchDays: 7,
|
||||
metaListHours: 6,
|
||||
metaEmptyHours: 1,
|
||||
metaDetailDays: 30,
|
||||
metaMissDays: 1,
|
||||
tmdbListHours: 6,
|
||||
tmdbListEmptyHours: 1,
|
||||
imageDays: 90,
|
||||
},
|
||||
};
|
||||
|
||||
|
||||
@@ -149,6 +149,8 @@ const INTERNAL_TABLES = [
|
||||
// C2-A 跨进程限流桶(0041)与 PG 缓存(0042):服务端内部,缓存经 /cache/* 的专用 DTO 暴露
|
||||
"RateLimitBucket",
|
||||
"KvCache",
|
||||
// 统一元数据缓存命中统计(0086):服务端内部,经 GET /cache/regions 的 hits 字段暴露
|
||||
"CacheStat",
|
||||
// 站点请求计数(0046):服务端内部,经 GET /sites/{id}/request-stats 的专用 DTO 暴露
|
||||
"SiteRequestCount",
|
||||
// W9-I 进程心跳(0034):服务端内部,经 /system/health 的专用 DTO 暴露
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
import type { Kysely } from "kysely";
|
||||
import { dropTables, execDdl } from "../src/migration-helpers.ts";
|
||||
|
||||
// 统一元数据缓存(docs/runbook/cache.md):
|
||||
// - kv_cache.fresh_until:新鲜期截止。过了 fresh_until、还没到 expires_at 的条目是「旧值」:
|
||||
// MetadataCache 先返回旧值再后台刷新(stale-while-revalidate);PgCache.get 只返回新鲜条目。null = 没有宽限期。
|
||||
// - kv_cache.pinned:关注中作品的条目,写入时按条数上限淘汰(LRU)跳过它们;到期仍会过期。
|
||||
// - cache_stat:按天 × region 汇总各进程的命中统计(进程内计数,定期把增量加进来)。
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型
|
||||
export async function up(db: Kysely<any>): Promise<void> {
|
||||
await execDdl(
|
||||
db,
|
||||
`
|
||||
alter table kv_cache add column fresh_until timestamptz;
|
||||
alter table kv_cache add column pinned boolean not null default false;
|
||||
create table cache_stat (
|
||||
day date not null,
|
||||
region text not null,
|
||||
l1_hits bigint not null default 0,
|
||||
l2_hits bigint not null default 0,
|
||||
stale_hits bigint not null default 0,
|
||||
misses bigint not null default 0,
|
||||
failures bigint not null default 0,
|
||||
refreshes bigint not null default 0,
|
||||
updated_at timestamptz not null default now(),
|
||||
primary key (day, region)
|
||||
);
|
||||
`,
|
||||
);
|
||||
}
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 同上
|
||||
export async function down(db: Kysely<any>): Promise<void> {
|
||||
await execDdl(
|
||||
db,
|
||||
`${dropTables(["cache_stat"])}
|
||||
alter table kv_cache drop column if exists pinned;
|
||||
alter table kv_cache drop column if exists fresh_until;`,
|
||||
);
|
||||
}
|
||||
+530
-17
@@ -1,4 +1,5 @@
|
||||
import { type Kysely, sql } from "kysely";
|
||||
import pg from "pg";
|
||||
import type { DB } from "./types.ts";
|
||||
|
||||
/**
|
||||
@@ -24,10 +25,37 @@ export interface SetOptions {
|
||||
ttl?: number;
|
||||
/** region 条数上限;缺省 DEFAULT_CACHE_SIZE */
|
||||
maxsize?: number;
|
||||
/**
|
||||
* 宽限期(秒,0086):> 0 时 fresh_until = now + ttl、expires_at = now + ttl + grace,
|
||||
* 宽限期内的条目由 MetadataCache 先返回旧值再后台刷新;PgCache.get 只认新鲜条目。缺省 0。
|
||||
*/
|
||||
grace?: number;
|
||||
/** 关注中作品的条目:按条数上限淘汰时跳过(到期照样过期)。缺省 false */
|
||||
pinned?: boolean;
|
||||
}
|
||||
|
||||
/** 跨进程 L1 失效通知频道(payload 见 invalidationPayload) */
|
||||
export const CACHE_INVALIDATE_CHANNEL = "umt_cache_invalidate";
|
||||
|
||||
/** 通知内容:`{"r":region,"k":key}`;k 缺省 = 整个 region;r 为 "*" = 全部。超长 key 退化为整 region */
|
||||
function invalidationPayload(region: string, key?: string): string {
|
||||
const full = JSON.stringify(key === undefined ? { r: region } : { r: region, k: key });
|
||||
return full.length > 7000 ? JSON.stringify({ r: region }) : full;
|
||||
}
|
||||
|
||||
/** getEntry 的结果:stale = 已过新鲜期、还在宽限期内 */
|
||||
export interface CacheEntry<T> {
|
||||
value: T;
|
||||
stale: boolean;
|
||||
/** 新鲜期截止(毫秒);没有宽限期时同 expiresAt */
|
||||
freshUntil: number;
|
||||
/** 硬过期(毫秒) */
|
||||
expiresAt: number;
|
||||
pinned: boolean;
|
||||
}
|
||||
|
||||
export class PgCache {
|
||||
constructor(private readonly db: AnyDb) {}
|
||||
constructor(readonly db: AnyDb) {}
|
||||
|
||||
/** MemoryBackend.set */
|
||||
async set(region: string, key: string, value: unknown, opts: SetOptions = {}): Promise<void> {
|
||||
@@ -36,36 +64,83 @@ export class PgCache {
|
||||
await this.delete(region, key);
|
||||
return;
|
||||
}
|
||||
await sql`insert into kv_cache (region, key, value, expires_at, updated_at, accessed_at)
|
||||
values (${region}, ${key}, ${JSON.stringify(value ?? null)}::jsonb, now() + make_interval(secs => ${ttl}), now(), now())
|
||||
const grace = Math.max(0, opts.grace ?? 0);
|
||||
await sql`insert into kv_cache (region, key, value, expires_at, fresh_until, pinned, updated_at, accessed_at)
|
||||
values (${region}, ${key}, ${JSON.stringify(value ?? null)}::jsonb,
|
||||
now() + make_interval(secs => ${ttl + grace}),
|
||||
${grace > 0 ? sql`now() + make_interval(secs => ${ttl})` : sql`null`},
|
||||
${opts.pinned === true}, now(), now())
|
||||
on conflict (region, key) do update set value = excluded.value, expires_at = excluded.expires_at,
|
||||
fresh_until = excluded.fresh_until, pinned = excluded.pinned,
|
||||
updated_at = excluded.updated_at, accessed_at = excluded.accessed_at`.execute(this.db);
|
||||
await this.evict(region, opts.maxsize ?? DEFAULT_CACHE_SIZE);
|
||||
}
|
||||
|
||||
/** TLRUCache 写入时的淘汰:先删已过期,再按最近使用从旧到新删到 maxsize */
|
||||
/**
|
||||
* TLRUCache 写入时的淘汰:先删已过期,再按最近使用从旧到新删到 maxsize。
|
||||
* 0086:pinned(关注中作品)的条目不参与 LRU,但计入条数;刚写入的那条总留着。
|
||||
*/
|
||||
private async evict(region: string, maxsize: number): Promise<void> {
|
||||
await sql`delete from kv_cache where region = ${region} and expires_at <= now()`.execute(this.db);
|
||||
await sql`delete from kv_cache where region = ${region} and key in (
|
||||
select key from kv_cache where region = ${region}
|
||||
order by accessed_at desc, updated_at desc offset ${Math.max(1, Math.floor(maxsize))})`.execute(this.db);
|
||||
await sql`delete from kv_cache where region = ${region} and not pinned and key in (
|
||||
select key from kv_cache where region = ${region} and not pinned
|
||||
order by accessed_at desc, updated_at desc
|
||||
offset greatest(1, ${Math.floor(maxsize)} - (select count(*) from kv_cache where region = ${region} and pinned)))`.execute(
|
||||
this.db,
|
||||
);
|
||||
}
|
||||
|
||||
/** MemoryBackend.get:未命中 / 过期返回 null;命中刷新最近使用时间 */
|
||||
/** MemoryBackend.get:未命中 / 过期 / 已过新鲜期返回 null;命中刷新最近使用时间 */
|
||||
async get<T = unknown>(region: string, key: string): Promise<T | null> {
|
||||
const r = await sql<{ value: T }>`update kv_cache set accessed_at = now()
|
||||
where region = ${region} and key = ${key} and expires_at > now() returning value`.execute(this.db);
|
||||
where region = ${region} and key = ${key} and expires_at > now()
|
||||
and (fresh_until is null or fresh_until > now()) returning value`.execute(this.db);
|
||||
return r.rows.length ? (r.rows[0]!.value as T) : null;
|
||||
}
|
||||
|
||||
/** 续期:未过期的条目把 expires_at 推到 now + ttl(只延长不缩短);返回是否有行被续期 */
|
||||
async renew(region: string, key: string, ttl: number): Promise<boolean> {
|
||||
/** 读一条(含宽限期内的旧值),命中刷新最近使用时间;MetadataCache 的 L2 读 */
|
||||
async getEntry<T = unknown>(region: string, key: string): Promise<CacheEntry<T> | null> {
|
||||
const r = await sql<{ value: T; expires_at: Date; fresh_until: Date | null; pinned: boolean; now: Date }>`
|
||||
update kv_cache set accessed_at = now()
|
||||
where region = ${region} and key = ${key} and expires_at > now()
|
||||
returning value, expires_at, fresh_until, pinned, now() as now`.execute(this.db);
|
||||
const row = r.rows[0];
|
||||
if (!row) return null;
|
||||
const expiresAt = new Date(row.expires_at).getTime();
|
||||
const freshUntil = row.fresh_until ? new Date(row.fresh_until).getTime() : expiresAt;
|
||||
return {
|
||||
value: row.value as T,
|
||||
stale: freshUntil <= new Date(row.now).getTime(),
|
||||
freshUntil,
|
||||
expiresAt,
|
||||
pinned: !!row.pinned,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 续期:新鲜的条目把过期时间推到 now + ttl(只延长不缩短,有宽限期的保留原宽限长度);返回是否有行被续期。
|
||||
* opts.pin = true 时顺带标为 pinned。旧值(已过新鲜期)不续期:该刷新就刷新。
|
||||
*/
|
||||
async renew(region: string, key: string, ttl: number, opts: { pin?: boolean } = {}): Promise<boolean> {
|
||||
if (ttl <= 0) return false;
|
||||
const r = await sql`update kv_cache set expires_at = greatest(expires_at, now() + make_interval(secs => ${ttl}))
|
||||
where region = ${region} and key = ${key} and expires_at > now()`.execute(this.db);
|
||||
const r = await sql`update kv_cache set
|
||||
expires_at = greatest(expires_at,
|
||||
now() + make_interval(secs => ${ttl}) + coalesce(expires_at - fresh_until, interval '0')),
|
||||
fresh_until = case when fresh_until is null then null
|
||||
else greatest(fresh_until, now() + make_interval(secs => ${ttl})) end,
|
||||
pinned = pinned or ${opts.pin === true}
|
||||
where region = ${region} and key = ${key} and expires_at > now()
|
||||
and (fresh_until is null or fresh_until > now())`.execute(this.db);
|
||||
return Number(r.numAffectedRows ?? 0) > 0;
|
||||
}
|
||||
|
||||
/** 通知各进程丢掉 L1(删除 / 清空 / 手动刷新时);失败不抛 */
|
||||
async notifyInvalidate(region: string, key?: string): Promise<void> {
|
||||
await sql`select pg_notify(${CACHE_INVALIDATE_CHANNEL}, ${invalidationPayload(region, key)})`
|
||||
.execute(this.db)
|
||||
.catch(() => {});
|
||||
}
|
||||
|
||||
async exists(region: string, key: string): Promise<boolean> {
|
||||
const r =
|
||||
await sql`select 1 from kv_cache where region = ${region} and key = ${key} and expires_at > now()`.execute(
|
||||
@@ -74,13 +149,16 @@ export class PgCache {
|
||||
return r.rows.length > 0;
|
||||
}
|
||||
|
||||
/** 删除并通知各进程丢掉这条 L1 */
|
||||
async delete(region: string, key: string): Promise<void> {
|
||||
await sql`delete from kv_cache where region = ${region} and key = ${key}`.execute(this.db);
|
||||
await this.notifyInvalidate(region, key);
|
||||
}
|
||||
|
||||
/** 清一个区,返回删除行数 */
|
||||
/** 清一个区,返回删除行数;通知各进程丢掉这个区的 L1 */
|
||||
async clear(region: string): Promise<number> {
|
||||
const r = await sql`delete from kv_cache where region = ${region}`.execute(this.db);
|
||||
await this.notifyInvalidate(region);
|
||||
return Number(r.numAffectedRows ?? 0);
|
||||
}
|
||||
|
||||
@@ -88,14 +166,24 @@ export class PgCache {
|
||||
async items<T = unknown>(
|
||||
region: string,
|
||||
limit = 5000,
|
||||
): Promise<{ key: string; value: T; expiresAt: Date; updatedAt: Date; bytes: number }[]> {
|
||||
const r = await sql<{ key: string; value: T; expires_at: Date; updated_at: Date; bytes: number }>`
|
||||
select key, value, expires_at, updated_at, pg_column_size(value)::int as bytes
|
||||
): Promise<{ key: string; value: T; expiresAt: Date; freshUntil: Date; updatedAt: Date; bytes: number }[]> {
|
||||
const r = await sql<{
|
||||
key: string;
|
||||
value: T;
|
||||
expires_at: Date;
|
||||
fresh_until: Date;
|
||||
updated_at: Date;
|
||||
bytes: number;
|
||||
}>`
|
||||
select key, value, expires_at, coalesce(fresh_until, expires_at) as fresh_until, updated_at,
|
||||
pg_column_size(value)::int as bytes
|
||||
from kv_cache where region = ${region} and expires_at > now() order by key limit ${limit}`.execute(this.db);
|
||||
return r.rows.map((x) => ({
|
||||
key: x.key,
|
||||
value: x.value,
|
||||
expiresAt: x.expires_at,
|
||||
/** 新鲜期截止(有宽限期的条目早于 expiresAt) */
|
||||
freshUntil: x.fresh_until,
|
||||
updatedAt: x.updated_at,
|
||||
bytes: Number(x.bytes),
|
||||
}));
|
||||
@@ -181,3 +269,428 @@ export function pgCacheFor(db: AnyDb): PgCache {
|
||||
}
|
||||
return c;
|
||||
}
|
||||
|
||||
// ── 统一元数据缓存(TMDB / 豆瓣 / Bangumi 共用;docs/runbook/cache.md) ──
|
||||
// 两级:L1 进程内存(按 region 限条数的 LRU,TTL 短)+ L2 PG kv_cache(三进程共享)。读 L1 → L2 → 源,写两级都写。
|
||||
// L1 只在跨进程失效通知(LISTEN umt_cache_invalidate)在线时启用:离线或断线重连时整体清空并停用,
|
||||
// 避免别的进程清空 / 删除后本进程还拿着旧值。
|
||||
|
||||
/** 宽限期:TTL 的 50%,上限 7 天 */
|
||||
export const STALE_GRACE_RATIO = 0.5;
|
||||
export const STALE_GRACE_MAX_SEC = 7 * 86_400;
|
||||
export const defaultGrace = (ttlSec: number) => Math.min(Math.max(0, ttlSec) * STALE_GRACE_RATIO, STALE_GRACE_MAX_SEC);
|
||||
|
||||
export interface FetchPolicy<T> {
|
||||
/** 秒 */
|
||||
ttl: number;
|
||||
/** 空结果 TTL(秒);缺省同 ttl */
|
||||
emptyTtl?: number;
|
||||
/** 空结果判定;缺省按假值(null / 空数组 / 空对象) */
|
||||
isEmpty?: (v: T) => boolean;
|
||||
maxsize?: number;
|
||||
/** 宽限期(秒);缺省 defaultGrace(本条 TTL);0 = 过期就回源 */
|
||||
grace?: number;
|
||||
/** 缺省 true:null / undefined 不缓存 */
|
||||
skipNone?: boolean;
|
||||
/** 返回真值时不缓存(业务失败的响应) */
|
||||
skipIf?: (v: T) => boolean;
|
||||
/** true:跳过读(手动刷新),照常写,并通知各进程丢掉这条 L1 */
|
||||
fresh?: boolean;
|
||||
/**
|
||||
* 关注中作品的条目:写入时标 pinned(LRU 不淘汰),L2 命中时按 ttl 续期。
|
||||
* 只在写入 / L2 命中时调用,失败按 false。
|
||||
*/
|
||||
pin?: () => Promise<boolean>;
|
||||
}
|
||||
|
||||
export interface CacheCounters {
|
||||
l1Hits: number;
|
||||
l2Hits: number;
|
||||
/** 先返回旧值(宽限期内)的次数 */
|
||||
staleHits: number;
|
||||
misses: number;
|
||||
/** 回源失败(网络、被拦、5xx 等抛错) */
|
||||
failures: number;
|
||||
/** 后台刷新次数 */
|
||||
refreshes: number;
|
||||
}
|
||||
|
||||
const zero = (): CacheCounters => ({ l1Hits: 0, l2Hits: 0, staleHits: 0, misses: 0, failures: 0, refreshes: 0 });
|
||||
|
||||
export interface MetadataCacheOptions {
|
||||
/** L1 每个 region 的条数上限,缺省 1000 */
|
||||
l1MaxPerRegion?: number;
|
||||
/** L1 TTL(毫秒),缺省 5 分钟;不超过条目在 L2 的新鲜期 */
|
||||
l1TtlMs?: number;
|
||||
/** 是否立即启用 L1(测试用;服务端由失效监听上线时打开) */
|
||||
l1?: boolean;
|
||||
now?: () => number;
|
||||
logger?: { warn(obj: object, msg: string): void };
|
||||
}
|
||||
|
||||
interface L1Entry {
|
||||
value: unknown;
|
||||
freshUntil: number;
|
||||
/** L1 内的过期(min(写入 + l1Ttl, 硬过期)) */
|
||||
expiresAt: number;
|
||||
}
|
||||
|
||||
const clone = <T>(v: T): T => (v === null || typeof v !== "object" ? v : structuredClone(v));
|
||||
|
||||
export class MetadataCache {
|
||||
private readonly l1 = new Map<string, Map<string, L1Entry>>();
|
||||
private readonly inflight = new Map<string, Promise<unknown>>();
|
||||
private readonly counters = new Map<string, CacheCounters>();
|
||||
private l1On: boolean;
|
||||
private readonly now: () => number;
|
||||
|
||||
constructor(
|
||||
readonly pg: PgCache,
|
||||
private readonly opts: MetadataCacheOptions = {},
|
||||
) {
|
||||
this.l1On = opts.l1 ?? false;
|
||||
this.now = opts.now ?? Date.now;
|
||||
}
|
||||
|
||||
// ── L1 ──
|
||||
|
||||
get l1Enabled(): boolean {
|
||||
return this.l1On;
|
||||
}
|
||||
|
||||
/** 打开 / 关闭 L1;状态变化时都整体清空 */
|
||||
setL1Enabled(on: boolean): void {
|
||||
if (this.l1On !== on) this.l1.clear();
|
||||
this.l1On = on;
|
||||
}
|
||||
|
||||
/** 丢掉本进程 L1:给 key 丢一条,不给丢整个 region,region 为 "*" 全部丢 */
|
||||
dropLocal(region: string, key?: string): void {
|
||||
if (region === "*") this.l1.clear();
|
||||
else if (key === undefined) this.l1.delete(region);
|
||||
else this.l1.get(region)?.delete(key);
|
||||
}
|
||||
|
||||
/** L1 当前条数(测试 / 诊断) */
|
||||
l1Size(region: string): number {
|
||||
return this.l1.get(region)?.size ?? 0;
|
||||
}
|
||||
|
||||
private l1Get(region: string, key: string): L1Entry | null {
|
||||
if (!this.l1On) return null;
|
||||
const m = this.l1.get(region);
|
||||
const e = m?.get(key);
|
||||
if (!m || !e) return null;
|
||||
if (e.expiresAt <= this.now()) {
|
||||
m.delete(key);
|
||||
return null;
|
||||
}
|
||||
m.delete(key);
|
||||
m.set(key, e);
|
||||
return e;
|
||||
}
|
||||
|
||||
private l1Set(region: string, key: string, value: unknown, freshUntil: number, hardExpiresAt: number): void {
|
||||
if (!this.l1On) return;
|
||||
let m = this.l1.get(region);
|
||||
if (!m) {
|
||||
m = new Map();
|
||||
this.l1.set(region, m);
|
||||
}
|
||||
m.delete(key);
|
||||
m.set(key, {
|
||||
value: clone(value),
|
||||
freshUntil,
|
||||
expiresAt: Math.min(this.now() + (this.opts.l1TtlMs ?? 5 * 60_000), hardExpiresAt),
|
||||
});
|
||||
const max = this.opts.l1MaxPerRegion ?? 1000;
|
||||
while (m.size > max) {
|
||||
const oldest = m.keys().next().value;
|
||||
if (oldest === undefined) break;
|
||||
m.delete(oldest);
|
||||
}
|
||||
}
|
||||
|
||||
// ── 统计 ──
|
||||
|
||||
private count(region: string, k: keyof CacheCounters, n = 1): void {
|
||||
let c = this.counters.get(region);
|
||||
if (!c) {
|
||||
c = zero();
|
||||
this.counters.set(region, c);
|
||||
}
|
||||
c[k] += n;
|
||||
}
|
||||
|
||||
/** 本进程还没写进 cache_stat 的计数 */
|
||||
pendingStats(): Map<string, CacheCounters> {
|
||||
return new Map([...this.counters].map(([r, c]) => [r, { ...c }]));
|
||||
}
|
||||
|
||||
/**
|
||||
* 把本进程的计数增量加进 cache_stat(按天 × region),成功后清零;失败保留到下次。
|
||||
* 顺带删掉 30 天前的统计行。返回写入的 region 数。
|
||||
*/
|
||||
async flushStats(): Promise<number> {
|
||||
const batch = [...this.counters];
|
||||
if (!batch.length) return 0;
|
||||
this.counters.clear();
|
||||
let done = 0;
|
||||
try {
|
||||
for (const [region, c] of batch) {
|
||||
await sql`insert into cache_stat (day, region, l1_hits, l2_hits, stale_hits, misses, failures, refreshes, updated_at)
|
||||
values (current_date, ${region}, ${c.l1Hits}, ${c.l2Hits}, ${c.staleHits}, ${c.misses}, ${c.failures}, ${c.refreshes}, now())
|
||||
on conflict (day, region) do update set
|
||||
l1_hits = cache_stat.l1_hits + excluded.l1_hits, l2_hits = cache_stat.l2_hits + excluded.l2_hits,
|
||||
stale_hits = cache_stat.stale_hits + excluded.stale_hits, misses = cache_stat.misses + excluded.misses,
|
||||
failures = cache_stat.failures + excluded.failures, refreshes = cache_stat.refreshes + excluded.refreshes,
|
||||
updated_at = now()`.execute(this.pg.db);
|
||||
done++;
|
||||
}
|
||||
await sql`delete from cache_stat where day < current_date - 30`.execute(this.pg.db);
|
||||
return done;
|
||||
} catch (err) {
|
||||
// 没写进去的加回去
|
||||
for (const [region, c] of batch.slice(done))
|
||||
for (const k of Object.keys(c) as (keyof CacheCounters)[]) this.count(region, k, c[k]);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
private warn(obj: object, msg: string) {
|
||||
this.opts.logger?.warn(obj, msg);
|
||||
}
|
||||
|
||||
// ── 读写 ──
|
||||
|
||||
/** 只读(不回源):L1 → L2 新鲜条目;记命中 / 未命中 */
|
||||
async get<T = unknown>(region: string, key: string): Promise<T | null> {
|
||||
const l1 = this.l1Get(region, key);
|
||||
if (l1 && l1.freshUntil > this.now()) {
|
||||
this.count(region, "l1Hits");
|
||||
return clone(l1.value) as T;
|
||||
}
|
||||
const e = await this.pg.getEntry<T>(region, key).catch((err) => {
|
||||
this.warn({ region, err: { message: (err as Error).message } }, "缓存读取失败，按未命中处理");
|
||||
return null;
|
||||
});
|
||||
if (!e || e.stale || e.value === null || e.value === undefined) {
|
||||
this.count(region, "misses");
|
||||
return null;
|
||||
}
|
||||
this.count(region, "l2Hits");
|
||||
this.l1Set(region, key, e.value, e.freshUntil, e.expiresAt);
|
||||
return e.value;
|
||||
}
|
||||
|
||||
/** 直接写两级(ttl ≤ 0 删除) */
|
||||
async set(region: string, key: string, value: unknown, opts: SetOptions = {}): Promise<void> {
|
||||
await this.pg.set(region, key, value, opts);
|
||||
const ttl = opts.ttl ?? DEFAULT_CACHE_TTL;
|
||||
if (ttl <= 0) {
|
||||
this.dropLocal(region, key);
|
||||
return;
|
||||
}
|
||||
const fresh = this.now() + ttl * 1000;
|
||||
this.l1Set(region, key, value, fresh, fresh + Math.max(0, opts.grace ?? 0) * 1000);
|
||||
}
|
||||
|
||||
async delete(region: string, key: string): Promise<void> {
|
||||
this.dropLocal(region, key);
|
||||
await this.pg.delete(region, key);
|
||||
}
|
||||
|
||||
async clear(region: string): Promise<number> {
|
||||
this.dropLocal(region);
|
||||
return await this.pg.clear(region);
|
||||
}
|
||||
|
||||
/**
|
||||
* 带回源的读(MP @cached 的调用体 + 两级 + 过期先用旧值 + 同请求合并 + 统计):
|
||||
* - L1 / L2 新鲜命中直接返回;L2 命中且 pin() 为真时续期;
|
||||
* - 宽限期内的旧值先返回,后台刷新(同 key 同时只刷一次),刷新失败继续用旧值并记一次警告;
|
||||
* - 未命中:同进程同 key 只发一次 load;load 抛错不写缓存(网络、被拦、5xx);结果按 skipNone / skipIf 决定是否写。
|
||||
* PG 读写失败按未命中 / 不写处理,不让业务请求失败。
|
||||
*/
|
||||
async fetch<T>(region: string, key: string, p: FetchPolicy<T>, load: () => Promise<T>): Promise<T> {
|
||||
if (!p.fresh) {
|
||||
const l1 = this.l1Get(region, key);
|
||||
if (l1) {
|
||||
if (l1.freshUntil > this.now()) {
|
||||
this.count(region, "l1Hits");
|
||||
return clone(l1.value) as T;
|
||||
}
|
||||
this.count(region, "staleHits");
|
||||
this.revalidate(region, key, p, load);
|
||||
return clone(l1.value) as T;
|
||||
}
|
||||
const e = await this.pg.getEntry<T>(region, key).catch((err) => {
|
||||
this.warn({ region, err: { message: (err as Error).message } }, "缓存读取失败，按未命中处理");
|
||||
return null;
|
||||
});
|
||||
if (e && this.shouldCache(p, e.value)) {
|
||||
this.l1Set(region, key, e.value, e.freshUntil, e.expiresAt);
|
||||
if (!e.stale) {
|
||||
this.count(region, "l2Hits");
|
||||
if (p.pin) await this.renewPinned(region, key, p);
|
||||
return e.value;
|
||||
}
|
||||
this.count(region, "staleHits");
|
||||
this.revalidate(region, key, p, load);
|
||||
return e.value;
|
||||
}
|
||||
}
|
||||
this.count(region, "misses");
|
||||
return (await this.loadOnce(region, key, p, load)) as T;
|
||||
}
|
||||
|
||||
private shouldCache<T>(p: FetchPolicy<T>, v: T | null | undefined): v is T {
|
||||
if ((p.skipNone ?? true) && (v === null || v === undefined)) return false;
|
||||
if (p.skipIf && v !== null && v !== undefined && p.skipIf(v)) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
private async renewPinned<T>(region: string, key: string, p: FetchPolicy<T>): Promise<void> {
|
||||
try {
|
||||
if (await p.pin!()) await this.pg.renew(region, key, p.ttl, { pin: true });
|
||||
} catch (err) {
|
||||
this.warn({ region, err: { message: (err as Error).message } }, "缓存续期失败");
|
||||
}
|
||||
}
|
||||
|
||||
private loadOnce<T>(region: string, key: string, p: FetchPolicy<T>, load: () => Promise<T>): Promise<T> {
|
||||
const id = `${region}\u0000${key}`;
|
||||
const pending = this.inflight.get(id);
|
||||
if (pending) return pending as Promise<T>;
|
||||
const run = (async () => {
|
||||
let v: T;
|
||||
try {
|
||||
v = await load();
|
||||
} catch (err) {
|
||||
this.count(region, "failures");
|
||||
throw err;
|
||||
}
|
||||
if (this.shouldCache(p, v)) await this.write(region, key, p, v);
|
||||
return v;
|
||||
})().finally(() => this.inflight.delete(id));
|
||||
this.inflight.set(id, run);
|
||||
return run;
|
||||
}
|
||||
|
||||
/** 宽限期内的后台刷新:同 key 已在刷新 / 加载就不再发 */
|
||||
private revalidate<T>(region: string, key: string, p: FetchPolicy<T>, load: () => Promise<T>): void {
|
||||
if (this.inflight.has(`${region}\u0000${key}`)) return;
|
||||
this.count(region, "refreshes");
|
||||
this.loadOnce(region, key, { ...p, fresh: false }, load).catch((err) =>
|
||||
this.warn({ region, key, err: { message: (err as Error).message } }, "缓存后台刷新失败，继续用旧值"),
|
||||
);
|
||||
}
|
||||
|
||||
private async write<T>(region: string, key: string, p: FetchPolicy<T>, v: T): Promise<void> {
|
||||
const empty = p.isEmpty ? p.isEmpty(v) : isFalsy(v);
|
||||
const ttl = empty && p.emptyTtl !== undefined ? p.emptyTtl : p.ttl;
|
||||
const grace = p.grace ?? defaultGrace(ttl);
|
||||
let pinned = false;
|
||||
if (p.pin) pinned = await p.pin().catch(() => false);
|
||||
try {
|
||||
await this.pg.set(region, key, v, { ttl, grace, pinned, ...(p.maxsize ? { maxsize: p.maxsize } : {}) });
|
||||
} catch (err) {
|
||||
this.warn({ region, err: { message: (err as Error).message } }, "缓存写入失败");
|
||||
}
|
||||
const fresh = this.now() + ttl * 1000;
|
||||
this.l1Set(region, key, v, fresh, fresh + grace * 1000);
|
||||
if (p.fresh) await this.pg.notifyInvalidate(region, key);
|
||||
}
|
||||
}
|
||||
|
||||
const metadataCaches = new WeakMap<object, MetadataCache>();
|
||||
|
||||
/** 同一个 Kysely 实例共用一个 MetadataCache(L2 用 pgCacheFor(db)) */
|
||||
export function metadataCacheFor(db: AnyDb, opts?: MetadataCacheOptions): MetadataCache {
|
||||
let c = metadataCaches.get(db);
|
||||
if (!c) {
|
||||
c = new MetadataCache(pgCacheFor(db), opts);
|
||||
metadataCaches.set(db, c);
|
||||
}
|
||||
return c;
|
||||
}
|
||||
|
||||
export interface InvalidationListenerOptions {
|
||||
connectionString: string;
|
||||
logger?: { warn(obj: object, msg: string): void; info(obj: object, msg: string): void };
|
||||
/** 断线重连退避上限,缺省 10s */
|
||||
maxReconnectMs?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* 跨进程 L1 失效:一条专用连接 LISTEN umt_cache_invalidate,收到通知丢掉对应 L1。
|
||||
* 连上时清空并启用 L1;断开时停用(清空)L1 并退避重连,重连成功再整体清一次后启用。
|
||||
*/
|
||||
export async function listenCacheInvalidation(
|
||||
cache: MetadataCache,
|
||||
opts: InvalidationListenerOptions,
|
||||
): Promise<{ stop(): Promise<void>; connected(): boolean }> {
|
||||
let client: pg.Client | null = null;
|
||||
let stopped = false;
|
||||
let timer: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
const connect = async () => {
|
||||
const c = new pg.Client({ connectionString: opts.connectionString });
|
||||
c.on("notification", (msg) => {
|
||||
if (msg.channel !== CACHE_INVALIDATE_CHANNEL) return;
|
||||
try {
|
||||
const p = JSON.parse(msg.payload ?? "{}") as { r?: string; k?: string };
|
||||
if (typeof p.r === "string") cache.dropLocal(p.r, typeof p.k === "string" ? p.k : undefined);
|
||||
} catch {
|
||||
cache.dropLocal("*");
|
||||
}
|
||||
});
|
||||
const lost = (err?: Error) => {
|
||||
if (client !== c) return;
|
||||
client = null;
|
||||
cache.setL1Enabled(false);
|
||||
c.removeAllListeners("notification");
|
||||
c.end().catch(() => {});
|
||||
if (!stopped) {
|
||||
opts.logger?.warn({ err: err ? { message: err.message } : undefined }, "缓存失效监听断开，L1 停用并准备重连");
|
||||
reconnect(500);
|
||||
}
|
||||
};
|
||||
c.on("error", lost);
|
||||
c.on("end", () => lost());
|
||||
await c.connect();
|
||||
await c.query(`listen ${CACHE_INVALIDATE_CHANNEL}`);
|
||||
client = c;
|
||||
cache.setL1Enabled(false);
|
||||
cache.setL1Enabled(true);
|
||||
};
|
||||
const reconnect = (delay: number) => {
|
||||
if (stopped || timer) return;
|
||||
timer = setTimeout(async () => {
|
||||
timer = null;
|
||||
try {
|
||||
await connect();
|
||||
opts.logger?.info({}, "缓存失效监听已重连");
|
||||
} catch (err) {
|
||||
opts.logger?.warn({ err: { message: (err as Error).message } }, "缓存失效监听重连失败");
|
||||
reconnect(Math.min(delay * 2, opts.maxReconnectMs ?? 10_000));
|
||||
}
|
||||
}, delay);
|
||||
};
|
||||
|
||||
await connect();
|
||||
return {
|
||||
connected: () => client !== null,
|
||||
async stop() {
|
||||
stopped = true;
|
||||
if (timer) clearTimeout(timer);
|
||||
const c = client;
|
||||
client = null;
|
||||
cache.setL1Enabled(false);
|
||||
if (c) {
|
||||
c.removeAllListeners("notification");
|
||||
await c.end().catch(() => {});
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -1,13 +1,24 @@
|
||||
// 统一从本包取 Kysely 与 sql,避免各包各装一份 kysely
|
||||
export { type Kysely, type RawBuilder, type Selectable, sql, type Transaction } from "kysely";
|
||||
export {
|
||||
CACHE_INVALIDATE_CHANNEL,
|
||||
type CacheCounters,
|
||||
type CachedOptions,
|
||||
type CacheEntry,
|
||||
cachedCall,
|
||||
DEFAULT_CACHE_SIZE,
|
||||
DEFAULT_CACHE_TTL,
|
||||
defaultGrace,
|
||||
type FetchPolicy,
|
||||
listenCacheInvalidation,
|
||||
MetadataCache,
|
||||
type MetadataCacheOptions,
|
||||
metadataCacheFor,
|
||||
PgCache,
|
||||
pgCacheFor,
|
||||
type SetOptions as CacheSetOptions,
|
||||
STALE_GRACE_MAX_SEC,
|
||||
STALE_GRACE_RATIO,
|
||||
} from "./cache.ts";
|
||||
export { type CreateDbOptions, createDb } from "./client.ts";
|
||||
export { createMigrator, MIGRATIONS_DIR, type MigratorOptions, migrateToLatest, resetSchema } from "./migrator.ts";
|
||||
|
||||
@@ -617,6 +617,18 @@ export interface BrowserTabTable {
|
||||
updated_at: GeneratedTimestamptz;
|
||||
}
|
||||
|
||||
export interface CacheStatTable {
|
||||
day: string;
|
||||
region: string;
|
||||
l1_hits: Generated<number>;
|
||||
l2_hits: Generated<number>;
|
||||
stale_hits: Generated<number>;
|
||||
misses: Generated<number>;
|
||||
failures: Generated<number>;
|
||||
refreshes: Generated<number>;
|
||||
updated_at: GeneratedTimestamptz;
|
||||
}
|
||||
|
||||
export interface CapabilityInvocationTable {
|
||||
id: Generated<string>;
|
||||
idempotency_key: string;
|
||||
@@ -883,6 +895,8 @@ export interface KvCacheTable {
|
||||
expires_at: Timestamptz;
|
||||
updated_at: GeneratedTimestamptz;
|
||||
accessed_at: GeneratedTimestamptz;
|
||||
fresh_until: Timestamptz | null;
|
||||
pinned: Generated<boolean>;
|
||||
}
|
||||
|
||||
export interface LibraryRootTable {
|
||||
@@ -1798,6 +1812,7 @@ export interface DB {
|
||||
browser_provider: BrowserProviderTable;
|
||||
browser_session: BrowserSessionTable;
|
||||
browser_tab: BrowserTabTable;
|
||||
cache_stat: CacheStatTable;
|
||||
capability_invocation: CapabilityInvocationTable;
|
||||
content_unit: ContentUnitTable;
|
||||
decision: DecisionTable;
|
||||
@@ -1929,6 +1944,9 @@ export type BrowserSessionUpdate = Updateable<BrowserSessionTable>;
|
||||
export type BrowserTabRow = Selectable<BrowserTabTable>;
|
||||
export type NewBrowserTab = Insertable<BrowserTabTable>;
|
||||
export type BrowserTabUpdate = Updateable<BrowserTabTable>;
|
||||
export type CacheStatRow = Selectable<CacheStatTable>;
|
||||
export type NewCacheStat = Insertable<CacheStatTable>;
|
||||
export type CacheStatUpdate = Updateable<CacheStatTable>;
|
||||
export type CapabilityInvocationRow = Selectable<CapabilityInvocationTable>;
|
||||
export type NewCapabilityInvocation = Insertable<CapabilityInvocationTable>;
|
||||
export type CapabilityInvocationUpdate = Updateable<CapabilityInvocationTable>;
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
import { Kysely, PostgresDialect, sql } from "kysely";
|
||||
import pg from "pg";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { type DB, listenCacheInvalidation, MetadataCache, migrateToLatest, PgCache } from "../src/index.ts";
|
||||
|
||||
// 统一元数据缓存(0086):两级读写、跨进程失效、过期先用旧值、同请求合并、失败不缓存、pinned 不被淘汰、命中统计。
|
||||
const url = process.env.DATABASE_URL ?? "postgres://umt:umt@localhost:55433/umt";
|
||||
const skip = process.env.UMT_SKIP_DB_TESTS === "1";
|
||||
const schema = `umt_mcache_${process.pid}_${Date.now().toString(36)}`;
|
||||
|
||||
const until = async (cond: () => boolean | Promise<boolean>, ms = 5000) => {
|
||||
const end = Date.now() + ms;
|
||||
while (!(await cond())) {
|
||||
if (Date.now() > end) throw new Error("等待超时");
|
||||
await new Promise((r) => setTimeout(r, 20));
|
||||
}
|
||||
};
|
||||
|
||||
describe.skipIf(skip)("MetadataCache(真实 PG)", () => {
|
||||
let admin: Kysely<unknown>;
|
||||
let db: Kysely<DB>;
|
||||
/** 另一个「进程」:独立连接池 */
|
||||
let db2: Kysely<DB>;
|
||||
|
||||
const pool = () =>
|
||||
new Kysely<DB>({
|
||||
dialect: new PostgresDialect({
|
||||
pool: new pg.Pool({ connectionString: url, max: 4, options: `-c search_path=${schema},public` }),
|
||||
}),
|
||||
});
|
||||
|
||||
beforeAll(async () => {
|
||||
admin = new Kysely({ dialect: new PostgresDialect({ pool: new pg.Pool({ connectionString: url, max: 1 }) }) });
|
||||
await sql.raw(`create schema ${schema}`).execute(admin);
|
||||
db = pool();
|
||||
db2 = pool();
|
||||
await migrateToLatest(db, { migrationTableSchema: schema });
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await db?.destroy();
|
||||
await db2?.destroy();
|
||||
await sql.raw(`drop schema if exists ${schema} cascade`).execute(admin);
|
||||
await admin?.destroy();
|
||||
});
|
||||
|
||||
it("读 L1 → L2 → 源,写两级;另一进程从 L2 命中;统计各级命中", async () => {
|
||||
const a = new MetadataCache(new PgCache(db), { l1: true });
|
||||
const b = new MetadataCache(new PgCache(db2), { l1: true });
|
||||
let n = 0;
|
||||
const load = async () => ({ v: ++n });
|
||||
const p = { ttl: 600 };
|
||||
expect(await a.fetch("r1", "k", p, load)).toEqual({ v: 1 });
|
||||
expect(await a.fetch("r1", "k", p, load)).toEqual({ v: 1 });
|
||||
expect(await b.fetch("r1", "k", p, load)).toEqual({ v: 1 });
|
||||
expect(await b.fetch("r1", "k", p, load)).toEqual({ v: 1 });
|
||||
expect(n).toBe(1);
|
||||
expect(a.pendingStats().get("r1")).toMatchObject({ misses: 1, l1Hits: 1, l2Hits: 0 });
|
||||
expect(b.pendingStats().get("r1")).toMatchObject({ misses: 0, l2Hits: 1, l1Hits: 1 });
|
||||
// L1 返回副本:调用方改了不污染缓存
|
||||
const got = await a.fetch<{ v: number }>("r1", "k", p, load);
|
||||
got.v = 99;
|
||||
expect(await a.fetch("r1", "k", p, load)).toEqual({ v: 1 });
|
||||
// 有宽限期:fresh_until = now + ttl,expires_at 再加 ttl 的 50%
|
||||
const r = await sql<{ g: number }>`select extract(epoch from expires_at - fresh_until)::int as g
|
||||
from kv_cache where region = 'r1' and key = 'k'`.execute(db);
|
||||
expect(r.rows[0]!.g).toBe(300);
|
||||
// 统计写进 cache_stat 后清零
|
||||
expect(await a.flushStats()).toBe(1);
|
||||
await b.flushStats();
|
||||
expect(a.pendingStats().size).toBe(0);
|
||||
const s = await sql<{ l1: number; l2: number; m: number }>`select l1_hits::int as l1, l2_hits::int as l2,
|
||||
misses::int as m from cache_stat where region = 'r1'`.execute(db);
|
||||
expect(s.rows[0]).toEqual({ l1: 4, l2: 1, m: 1 });
|
||||
});
|
||||
|
||||
it("跨进程失效:删除 / 清空通知其它进程丢 L1;监听断开时 L1 停用,重连后清空再启用", async () => {
|
||||
const a = new MetadataCache(new PgCache(db));
|
||||
const b = new MetadataCache(new PgCache(db2));
|
||||
const la = await listenCacheInvalidation(a, { connectionString: url });
|
||||
const lb = await listenCacheInvalidation(b, { connectionString: url });
|
||||
try {
|
||||
expect(a.l1Enabled && b.l1Enabled).toBe(true);
|
||||
const p = { ttl: 600 };
|
||||
await a.fetch("inv", "x", p, async () => 1);
|
||||
await a.fetch("inv", "y", p, async () => 2);
|
||||
await b.fetch("inv", "x", p, async () => 0);
|
||||
await b.fetch("inv", "y", p, async () => 0);
|
||||
expect(b.l1Size("inv")).toBe(2);
|
||||
// A 删 x:B 的 L1 只丢 x
|
||||
await a.delete("inv", "x");
|
||||
await until(() => b.l1Size("inv") === 1);
|
||||
expect(await b.fetch("inv", "x", p, async () => 10)).toBe(10);
|
||||
// A 清空整个 region
|
||||
await a.clear("inv");
|
||||
await until(() => b.l1Size("inv") === 0);
|
||||
expect(await b.fetch("inv", "y", p, async () => 20)).toBe(20);
|
||||
// 手动刷新(fresh)写入后也通知
|
||||
await a.fetch("inv", "y", { ...p, fresh: true }, async () => 21);
|
||||
await until(() => b.l1Size("inv") === 0);
|
||||
expect(await b.fetch("inv", "y", p, async () => 0)).toBe(21);
|
||||
// 断开 B 的监听连接:L1 停用;自动重连后重新启用且是空的
|
||||
expect(b.l1Size("inv")).toBe(1);
|
||||
await sql`select pg_terminate_backend(pid) from pg_stat_activity
|
||||
where query ilike 'listen umt_cache_invalidate%' and pid <> pg_backend_pid()`.execute(db2);
|
||||
await until(() => !b.l1Enabled || b.l1Size("inv") === 0);
|
||||
expect(b.l1Size("inv")).toBe(0);
|
||||
await until(() => b.l1Enabled && lb.connected(), 10_000);
|
||||
expect(b.l1Size("inv")).toBe(0);
|
||||
} finally {
|
||||
await la.stop();
|
||||
await lb.stop();
|
||||
}
|
||||
expect(a.l1Enabled).toBe(false);
|
||||
});
|
||||
|
||||
it("过期在宽限期内:先返回旧值、后台只刷一次;源失败继续用旧值并记警告", async () => {
|
||||
const warns: string[] = [];
|
||||
const c = new MetadataCache(new PgCache(db), { l1: false, logger: { warn: (_o, m) => warns.push(m) } });
|
||||
let n = 0;
|
||||
let gate: () => void = () => {};
|
||||
const slow = () =>
|
||||
new Promise<number>((ok) => {
|
||||
n++;
|
||||
gate = () => ok(100 + n);
|
||||
});
|
||||
await c.fetch("swr", "k", { ttl: 0.2, grace: 60 }, async () => 1);
|
||||
await new Promise((r) => setTimeout(r, 300));
|
||||
const p = { ttl: 600 };
|
||||
const [x, y] = await Promise.all([c.fetch("swr", "k", p, slow), c.fetch("swr", "k", p, slow)]);
|
||||
expect([x, y]).toEqual([1, 1]);
|
||||
await until(() => n === 1);
|
||||
gate();
|
||||
await until(async () => (await new PgCache(db).get("swr", "k")) === 101);
|
||||
expect(n).toBe(1);
|
||||
expect(c.pendingStats().get("swr")).toMatchObject({ staleHits: 2, refreshes: 1 });
|
||||
// 再次过期,源失败:返回旧值,不删旧值,记一次警告
|
||||
await c.fetch("swr", "k2", { ttl: 0.2, grace: 60 }, async () => "old");
|
||||
await new Promise((r) => setTimeout(r, 300));
|
||||
const failing = async (): Promise<string> => {
|
||||
throw new Error("HTTP 503");
|
||||
};
|
||||
expect(await c.fetch("swr", "k2", p, failing)).toBe("old");
|
||||
await until(() => warns.length === 1);
|
||||
expect(warns[0]).toContain("继续用旧值");
|
||||
expect(await c.fetch("swr", "k2", p, async () => "new")).toBe("old");
|
||||
// 过了宽限期 = 硬过期,回源
|
||||
await c.fetch("swr", "k3", { ttl: 0.1, grace: 0.1 }, async () => "a");
|
||||
await new Promise((r) => setTimeout(r, 300));
|
||||
expect(await c.fetch("swr", "k3", p, async () => "b")).toBe("b");
|
||||
// PgCache.get 只认新鲜条目
|
||||
await c.fetch("swr", "k4", { ttl: 0.1, grace: 60 }, async () => "x");
|
||||
await new Promise((r) => setTimeout(r, 200));
|
||||
expect(await new PgCache(db).get("swr", "k4")).toBeNull();
|
||||
expect(await new PgCache(db).exists("swr", "k4")).toBe(true);
|
||||
});
|
||||
|
||||
it("同请求合并:并发同 key 只回源一次;失败不缓存并计数", async () => {
|
||||
const c = new MetadataCache(new PgCache(db), { l1: true });
|
||||
let n = 0;
|
||||
const load = async () => {
|
||||
n++;
|
||||
await new Promise((r) => setTimeout(r, 50));
|
||||
return { ok: true };
|
||||
};
|
||||
const all = await Promise.all(Array.from({ length: 5 }, () => c.fetch("sf", "k", { ttl: 60 }, load)));
|
||||
expect(n).toBe(1);
|
||||
expect(all.every((v) => v.ok)).toBe(true);
|
||||
let m = 0;
|
||||
const fail = async () => {
|
||||
m++;
|
||||
throw new Error("ECONNRESET");
|
||||
};
|
||||
await expect(c.fetch("sf", "bad", { ttl: 60 }, fail)).rejects.toThrow("ECONNRESET");
|
||||
await expect(c.fetch("sf", "bad", { ttl: 60 }, fail)).rejects.toThrow("ECONNRESET");
|
||||
expect(m).toBe(2);
|
||||
expect(await new PgCache(db).exists("sf", "bad")).toBe(false);
|
||||
expect(c.pendingStats().get("sf")).toMatchObject({ failures: 2 });
|
||||
// skipIf / null 不缓存;空结果按 emptyTtl
|
||||
await c.fetch("sf", "none", { ttl: 60 }, async () => null);
|
||||
expect(await new PgCache(db).exists("sf", "none")).toBe(false);
|
||||
await c.fetch("sf", "empty", { ttl: 3600, emptyTtl: 60, isEmpty: (v: number[]) => !v.length }, async () => []);
|
||||
const t = await sql<{ s: number }>`select extract(epoch from fresh_until - now())::int as s
|
||||
from kv_cache where region = 'sf' and key = 'empty'`.execute(db);
|
||||
expect(t.rows[0]!.s).toBeLessThanOrEqual(60);
|
||||
});
|
||||
|
||||
it("pinned:按条数上限淘汰时跳过;L2 命中时续期", async () => {
|
||||
const pgc = new PgCache(db);
|
||||
const c = new MetadataCache(pgc, { l1: false });
|
||||
const tick = () => new Promise((r) => setTimeout(r, 5));
|
||||
await c.fetch("pin", "watched", { ttl: 100, maxsize: 2, pin: async () => true }, async () => "w");
|
||||
for (const k of ["a", "b", "c", "d"]) {
|
||||
await tick();
|
||||
await c.fetch("pin", k, { ttl: 100, maxsize: 2 }, async () => k);
|
||||
}
|
||||
expect((await pgc.items("pin")).map((i) => i.key).sort()).toEqual(["d", "watched"]);
|
||||
// 续期:新鲜条目命中且 pin 为真 → 过期时间推到 now + ttl(保留宽限长度)
|
||||
await sql`update kv_cache set fresh_until = now() + interval '10 seconds', expires_at = now() + interval '60 seconds'
|
||||
where region = 'pin' and key = 'watched'`.execute(db);
|
||||
expect(await c.fetch("pin", "watched", { ttl: 3600, pin: async () => true }, async () => "x")).toBe("w");
|
||||
const r = await sql<{ f: number; g: number }>`select extract(epoch from fresh_until - now())::int as f,
|
||||
extract(epoch from expires_at - fresh_until)::int as g from kv_cache where region = 'pin' and key = 'watched'`.execute(
|
||||
db,
|
||||
);
|
||||
expect(r.rows[0]!.f).toBeGreaterThan(3500);
|
||||
expect(r.rows[0]!.g).toBe(50);
|
||||
});
|
||||
});
|
||||
@@ -132,6 +132,8 @@ const EXPECTED_TABLES = [
|
||||
"episode_mapping_rule",
|
||||
// 站点请求失败留样(0077)
|
||||
"site_failure_sample",
|
||||
// 缓存命中统计(0086,按天 × region)
|
||||
"cache_stat",
|
||||
].sort();
|
||||
|
||||
/** 不走 entityTable() 的表:追加型/引擎自有表,没有 uuid id + created_at/updated_at 三件套 */
|
||||
@@ -167,6 +169,8 @@ const NON_ENTITY_TABLES = [
|
||||
"site_request_count",
|
||||
// 失败留样:只追加,只有 created_at
|
||||
"site_failure_sample",
|
||||
// 缓存命中统计:(day, region) 主键
|
||||
"cache_stat",
|
||||
];
|
||||
|
||||
describe.skipIf(skip)("schema migrations (real PG)", () => {
|
||||
|
||||
@@ -7,6 +7,8 @@
|
||||
// - 剧集:作品目录 tvshow.nfo、poster.jpg、fanart.jpg;季目录 season.nfo;季海报同时写作品目录
|
||||
// `season01-poster.jpg`(特别篇 `season-specials-poster.jpg`)与季目录 `poster.jpg`(MP 同);
|
||||
// - 每集:`<视频名>.nfo`;剧照 `<视频名>-thumb.jpg`(Kodi / Jellyfin / Emby 都认;MP 写 `<视频名>.jpg`)。
|
||||
// - 标志(TMDB logos,MP 只在接 fanart.tv 时写):电影写在影片目录、剧集写在剧集根目录,文件名 `clearlogo.png`
|
||||
// (Kodi / Emby / Jellyfin 通用);平铺 / 库根的电影用 `<视频名>-clearlogo.png`。
|
||||
// 视频直接在库根(没有作品目录)或电影目录里还有别的作品时,电影图片改用 `<视频名>-poster.jpg` / `<视频名>-fanart.jpg`,
|
||||
// 剧集不写 tvshow / 季文件(不能往库根里放 tvshow.nfo)。
|
||||
|
||||
@@ -42,6 +44,7 @@ export type ScrapeTargetRole =
|
||||
| "episode_nfo"
|
||||
| "poster"
|
||||
| "fanart"
|
||||
| "clearlogo"
|
||||
| "season_poster"
|
||||
| "episode_thumb";
|
||||
|
||||
@@ -61,6 +64,8 @@ export interface ScrapeVideo {
|
||||
export interface ScrapeImages {
|
||||
posterPath?: string | null;
|
||||
backdropPath?: string | null;
|
||||
/** TMDB 标志(PNG) */
|
||||
logoPath?: string | null;
|
||||
/** 季号 → 季海报路径 */
|
||||
seasonPosters?: ReadonlyMap<number, string | null>;
|
||||
/** "季:集" → 剧照路径 */
|
||||
@@ -165,7 +170,7 @@ export function planScrapeTargets(input: ScrapeLayoutInput): ScrapeTarget[] {
|
||||
}
|
||||
if (!input.options.images || input.parts?.work === false) return out;
|
||||
const inWork = workDir && !input.sharedDir && videos.some((v) => isUnder(v.path, workDir));
|
||||
const art = (role: "poster" | "fanart", p: string | null | undefined) => {
|
||||
const art = (role: "poster" | "fanart" | "clearlogo", p: string | null | undefined) => {
|
||||
if (!p) return;
|
||||
if (inWork) add({ path: join(workDir, `${role}${extOf(p)}`), kind: "image", role, imagePath: p });
|
||||
else
|
||||
@@ -182,6 +187,7 @@ export function planScrapeTargets(input: ScrapeLayoutInput): ScrapeTarget[] {
|
||||
};
|
||||
art("poster", img.posterPath);
|
||||
art("fanart", img.backdropPath);
|
||||
art("clearlogo", img.logoPath);
|
||||
return out;
|
||||
}
|
||||
|
||||
@@ -206,6 +212,13 @@ export function planScrapeTargets(input: ScrapeLayoutInput): ScrapeTarget[] {
|
||||
role: "fanart",
|
||||
imagePath: img.backdropPath,
|
||||
});
|
||||
if (img.logoPath)
|
||||
add({
|
||||
path: join(workDir, `clearlogo${extOf(img.logoPath)}`),
|
||||
kind: "image",
|
||||
role: "clearlogo",
|
||||
imagePath: img.logoPath,
|
||||
});
|
||||
}
|
||||
}
|
||||
// 季:按视频所在目录(不等于作品目录的才算季目录);没有作品目录时只写季目录里的 season.nfo / poster
|
||||
|
||||
@@ -199,6 +199,60 @@ describe("刮削目标布局", () => {
|
||||
expect(atRoot.map((x) => x.path)).toEqual(["x.nfo", "x-poster.jpg", "x-fanart.png"]);
|
||||
});
|
||||
|
||||
it("标志 clearlogo.png:电影写影片目录、剧集写剧集根目录;库根 / 平铺的电影用 <视频名>-clearlogo.png;选中季 / 集不写", () => {
|
||||
const withLogo = { ...images, logoPath: "/logo.png" };
|
||||
const movie = planScrapeTargets({
|
||||
kind: "movie",
|
||||
workDir: "星际穿越 (2014)",
|
||||
videos: [{ path: "星际穿越 (2014)/星际穿越 (2014) - 1080p.mkv" }],
|
||||
images: withLogo,
|
||||
options: { images: true, episodeThumbs: true },
|
||||
});
|
||||
expect(movie.filter((x) => x.role === "clearlogo").map((x) => [x.path, x.imagePath])).toEqual([
|
||||
["星际穿越 (2014)/clearlogo.png", "/logo.png"],
|
||||
]);
|
||||
const atRoot = planScrapeTargets({
|
||||
kind: "movie",
|
||||
workDir: null,
|
||||
videos: [{ path: "x.mkv" }],
|
||||
images: withLogo,
|
||||
options: { images: true, episodeThumbs: true },
|
||||
});
|
||||
expect(atRoot.map((x) => x.path)).toEqual(["x.nfo", "x-poster.jpg", "x-fanart.png", "x-clearlogo.png"]);
|
||||
const tv = planScrapeTargets({
|
||||
kind: "tv",
|
||||
workDir: "S",
|
||||
videos: [{ path: "S/Season 1/a.mkv", season: 1, episode: 1 }],
|
||||
images: withLogo,
|
||||
options: { images: true, episodeThumbs: true },
|
||||
});
|
||||
expect(tv.slice(0, 4).map((x) => x.path)).toEqual([
|
||||
"S/tvshow.nfo",
|
||||
"S/poster.jpg",
|
||||
"S/fanart.png",
|
||||
"S/clearlogo.png",
|
||||
]);
|
||||
const seasonOnly = planScrapeTargets({
|
||||
kind: "tv",
|
||||
workDir: "S",
|
||||
videos: [{ path: "S/Season 1/a.mkv", season: 1, episode: 1 }],
|
||||
images: withLogo,
|
||||
options: { images: true, episodeThumbs: true },
|
||||
parts: { work: false },
|
||||
});
|
||||
expect(seasonOnly.some((x) => x.role === "clearlogo")).toBe(false);
|
||||
// 关图片不写;没有标志不写
|
||||
const noImages = planScrapeTargets({
|
||||
kind: "tv",
|
||||
workDir: "S",
|
||||
videos: [{ path: "S/Season 1/a.mkv", season: 1, episode: 1 }],
|
||||
images: withLogo,
|
||||
options: { images: false, episodeThumbs: false },
|
||||
});
|
||||
expect(noImages.some((x) => x.role === "clearlogo")).toBe(false);
|
||||
expect(movie.length).toBe(4);
|
||||
});
|
||||
|
||||
it("剧集视频在库根:不写 tvshow / 季文件,只写集 NFO", () => {
|
||||
const t = planScrapeTargets({
|
||||
kind: "tv",
|
||||
|
||||
@@ -60,6 +60,8 @@ export interface TmdbFixture {
|
||||
genre_ids?: number[];
|
||||
/** 剧集组(`/tv/{id}/episode_groups`、`/tv/episode_group/{id}`);组内集按 TMDB 默认季集号引用 seasons 里的集 */
|
||||
episode_groups?: TmdbFixtureEpisodeGroup[];
|
||||
/** 标志(append_to_response=images 与 `/{type}/{id}/images` 按 include_image_language 过滤);缺省没有 */
|
||||
logos?: { file_path: string; iso_639_1: string | null; vote_average?: number; vote_count?: number }[];
|
||||
}
|
||||
|
||||
export interface TmdbFixtureEpisodeGroup {
|
||||
|
||||
@@ -22,7 +22,7 @@ const notFound: MockReply = {
|
||||
/** 列表接口一页的条数(同 TMDB) */
|
||||
const LIST_PAGE = 20;
|
||||
|
||||
/** mock TMDB v3:/3/trending、/3/discover、电影 / 剧集榜单(按热度,一页 20 条)、/3/search/{movie,tv}、/3/find/:imdbId、/3/{movie,tv}/:id(append_to_response)、/3/tv/:id/season/:n、/3/tv/:id/episode_groups、/3/tv/episode_group/:id */
|
||||
/** mock TMDB v3:/3/trending、/3/discover、电影 / 剧集榜单(按热度,一页 20 条)、/3/search/{movie,tv}、/3/find/:imdbId、/3/{movie,tv}/:id(append_to_response)、/3/{movie,tv}/:id/images、/3/tv/:id/season/:n、/3/tv/:id/episode_groups、/3/tv/episode_group/:id */
|
||||
export async function createMockTmdb(opts: MockTmdbOptions = {}): Promise<MockTmdb> {
|
||||
const fx = opts.fixtures ?? TMDB_FIXTURES;
|
||||
const apiKey = opts.apiKey ?? "test-tmdb-key";
|
||||
@@ -59,6 +59,21 @@ export async function createMockTmdb(opts: MockTmdbOptions = {}): Promise<MockTm
|
||||
};
|
||||
};
|
||||
|
||||
/** images.logos 按 include_image_language(逗号分隔,null = 无文字)过滤;不给参数全给 */
|
||||
const images = (f: TmdbFixture, q: Record<string, string>) => {
|
||||
const langs = q.include_image_language ? q.include_image_language.split(",") : null;
|
||||
const logos = (f.logos ?? [])
|
||||
.filter((l) => !langs || langs.includes(l.iso_639_1 ?? "null"))
|
||||
.map((l) => ({
|
||||
...l,
|
||||
vote_average: l.vote_average ?? 5,
|
||||
vote_count: l.vote_count ?? 1,
|
||||
width: 500,
|
||||
height: 200,
|
||||
}));
|
||||
return { backdrops: [], posters: [], logos };
|
||||
};
|
||||
|
||||
const listItem = (f: TmdbFixture, lang?: string) => ({
|
||||
...brief(f, lang),
|
||||
genre_ids: f.genre_ids ?? [],
|
||||
@@ -222,6 +237,13 @@ export async function createMockTmdb(opts: MockTmdbOptions = {}): Promise<MockTm
|
||||
};
|
||||
},
|
||||
],
|
||||
[
|
||||
/^\/3\/(movie|tv)\/(\d+)\/images$/,
|
||||
(m, q) => {
|
||||
const f = find(m[1]!, m[2]!);
|
||||
return f ? { json: { id: f.id, ...images(f, q) } } : notFound;
|
||||
},
|
||||
],
|
||||
[
|
||||
/^\/3\/tv\/(\d+)\/season\/(\d+)$/,
|
||||
(m, q) => {
|
||||
@@ -298,6 +320,7 @@ export async function createMockTmdb(opts: MockTmdbOptions = {}): Promise<MockTm
|
||||
f.type === "movie" ? { titles: f.alternative_titles } : { results: f.alternative_titles };
|
||||
}
|
||||
if (append.has("external_ids")) out.external_ids = f.external_ids;
|
||||
if (append.has("images")) out.images = images(f, q);
|
||||
if (append.has("translations")) {
|
||||
const key = f.type === "movie" ? "title" : "name";
|
||||
out.translations = {
|
||||
|
||||
Reference in new issue
Block a user