diff --git a/apps/server/src/modules/index.ts b/apps/server/src/modules/index.ts index 7654a110..c199e88c 100644 --- a/apps/server/src/modules/index.ts +++ b/apps/server/src/modules/index.ts @@ -2,6 +2,7 @@ import { acquisitionOrganizeModule } from "./acquisition-organize/index.ts"; import { agentModule } from "./agent/index.ts"; import { authSettingsModule } from "./auth-settings/index.ts"; import { browserModule } from "./browser/index.ts"; +import { cacheModule } from "./cache/index.ts"; import { fileManagerModule } from "./file-manager/index.ts"; import { mcpModule } from "./mcp/index.ts"; import { piModule } from "./pi/index.ts"; @@ -28,5 +29,6 @@ export const SERVER_MODULES: readonly ServerModule[] = [ searchModule, browserModule, mcpModule, + cacheModule, systemModule, ]; diff --git a/apps/server/src/modules/search/media.ts b/apps/server/src/modules/search/media.ts index a9206da3..21e980a6 100644 --- a/apps/server/src/modules/search/media.ts +++ b/apps/server/src/modules/search/media.ts @@ -163,6 +163,7 @@ export class TmdbSearch { year: m.year, overview: m.overview ? (m.overview.length > 300 ? `${m.overview.slice(0, 300)}…` : m.overview) : null, popularity: m.popularity, + ...(m.posterPath ? { posterPath: m.posterPath } : {}), })), ) .then((list) => { diff --git a/apps/server/src/modules/sources-identity/runtime.ts b/apps/server/src/modules/sources-identity/runtime.ts index a62e0021..40d6d8f6 100644 --- a/apps/server/src/modules/sources-identity/runtime.ts +++ b/apps/server/src/modules/sources-identity/runtime.ts @@ -11,9 +11,11 @@ import { type SiteDefinition, type SiteRateLimiter, type TmdbClient, + type TmdbSharedCache, } from "@umt/adapters"; import { DEFAULT_SETTINGS, type SitesSettings } from "@umt/contracts"; import { pgTokenBucketFor, type SiteRow } from "@umt/db"; +import { CACHE_REGIONS, sharedCache, type TmdbCachePolicy, tmdbCachePolicy } from "../../shared-cache.ts"; import type { ServerContext } from "../types.ts"; // 模块运行时依赖:站点适配器与限流器(每站一个实例,令牌在 PG 跨进程共享)、RSS 抓取器、TMDB 客户端、时钟。 @@ -71,12 +73,14 @@ export class SourcesRuntime { return this.tmdbClient; } const meta = await this.ctx.settings.get("metadata").catch(() => null); + const policy = await tmdbCachePolicy(this.ctx); const env = (k: string) => Deno.env.get(k); const ref = meta?.tmdbApiKeySecretRef ?? null; const baseUrl = meta?.tmdbBaseUrl ?? env("TMDB_BASE_URL"); const sig = JSON.stringify([ ref, baseUrl ?? null, + policy, ref ? null : [!!env("TMDB_API_KEY"), !!env("TMDB_ACCESS_TOKEN")], ]); if (this.tmdbClient !== undefined && this.tmdbSig === sig) return this.tmdbClient; @@ -103,6 +107,9 @@ export class SourcesRuntime { ...(accessToken ? { accessToken } : {}), ...(baseUrl ? { baseUrl } : {}), ...(this.overrides.fetch ? { fetch: this.overrides.fetch } : {}), + // 请求响应进 PG 缓存(region tmdb.request),三进程共享;TTL / 条数上限 / 空结果短 TTL 取设置(对照 MP) + cache: { ttlMs: policy.ttl * 1000, emptyTtlMs: policy.emptyTtl * 1000 }, + sharedCache: tmdbSharedCache(this.ctx, policy), }) : null; return this.tmdbClient; @@ -233,6 +240,22 @@ export class SourcesRuntime { } } +/** TMDB 请求响应缓存:PG kv_cache region=tmdb.request;PG 出错按未命中 / 不写处理,不让 TMDB 请求失败 */ +function tmdbSharedCache(ctx: ServerContext, policy: TmdbCachePolicy): TmdbSharedCache { + const cache = sharedCache(ctx); + const warn = (err: unknown) => + ctx.logger.warn({ err: { message: (err as Error).message } }, "TMDB 缓存读写失败，按未命中处理"); + return { + get: (key) => + cache.get(CACHE_REGIONS.tmdbRequest, key).catch((e) => { + warn(e); + return null; + }), + set: (key, value, ttlMs) => + cache.set(CACHE_REGIONS.tmdbRequest, key, value, { ttl: ttlMs / 1000, maxsize: policy.maxsize }).catch(warn), + }; +} + function sameUrl(a: string, b: string): boolean { return a.replace(/\/+$/, "") === b.replace(/\/+$/, ""); } diff --git a/apps/server/src/modules/sources-identity/service/identity-admin.ts b/apps/server/src/modules/sources-identity/service/identity-admin.ts index 1409915b..f555448a 100644 --- a/apps/server/src/modules/sources-identity/service/identity-admin.ts +++ b/apps/server/src/modules/sources-identity/service/identity-admin.ts @@ -23,6 +23,7 @@ import { resolveEntityId, } from "./entities.ts"; import { recordAttempt } from "./resolve.ts"; +import { backfillRecognizeCache } from "./tmdb-recognize-cache.ts"; // 人工纠错:指定来源资源身份(加锁)、改作品外部映射(新增修订、旧映射退役)、合并作品(旧 ID 跳转)。 // 所有变更留 identity_revision / resolution_attempt / audit_event,不直接覆盖历史。 @@ -85,7 +86,7 @@ export async function assignReleaseIdentity( for (let attempt = 0; ; attempt++) { try { - return await ctx.db.transaction().execute(async (trx) => { + const done = await ctx.db.transaction().execute(async (trx) => { let entityId: string; if (localId) { const r = await resolveEntityId(trx, localId); @@ -190,6 +191,8 @@ export async function assignReleaseIdentity( }); return { mediaEntityId: entityId, contentUnitIds: unitIds }; }); + await backfillRecognizeCache(rt, parsed, done.mediaEntityId); + return done; } catch (e) { if (e instanceof IdentityConflictError && attempt < 2) continue; throw e; diff --git a/apps/server/src/modules/sources-identity/service/metadata.ts b/apps/server/src/modules/sources-identity/service/metadata.ts index 5335d7b0..9b355a87 100644 --- a/apps/server/src/modules/sources-identity/service/metadata.ts +++ b/apps/server/src/modules/sources-identity/service/metadata.ts @@ -32,6 +32,9 @@ function tmdbFields(m: MetadataCandidate): Partial { +/** + * TMDB 召回。先查识别缓存(移植 MP TmdbCache,见 tmdb-recognize-cache.ts):命中且有标题 → 按 id 取详情作唯一远端候选; + * 命中负缓存 → 不再搜索(视为 TMDB 无结果)。强制复查(force)不读缓存但照常回写(MP recognize_media 的 cache=False)。 + */ +async function tmdbCandidates( + rt: SourcesRuntime, + parsed: ParsedRelease, + useCache: boolean, +): Promise<{ remote: MetadataCandidate[]; cacheHit: boolean; meta: RecognizeMeta | null }> { const tmdb = await rt.tmdb(); - if (!tmdb) return []; + if (!tmdb) return { remote: [], cacheHit: false, meta: null }; + const meta = recognizeMetaOf(parsed); + if (meta && useCache) { + const hit = await loadRecognizeCache(rt, meta); + if (hit) { + if (hit.id && hit.title && (hit.type === "movie" || hit.type === "tv")) { + return { remote: [await tmdb.details(hit.type, hit.id)], cacheHit: true, meta }; + } + return { remote: [], cacheHit: true, meta }; + } + } const queries = [parsed.titleCn, parsed.titleEn, parsed.title].filter((t): t is string => !!t); const seen = new Map(); for (const q of [...new Set(queries)].slice(0, 2)) { @@ -166,7 +185,7 @@ async function tmdbCandidates(rt: SourcesRuntime, parsed: ParsedRelease): Promis if (!seen.has(k)) seen.set(k, m); } } - return [...seen.values()].slice(0, 20); + return { remote: [...seen.values()].slice(0, 20), cacheHit: false, meta }; } async function loadRelease(rt: SourcesRuntime, input: ResolveInput) { @@ -240,14 +259,27 @@ export async function resolveRelease(rt: SourcesRuntime, input: ResolveInput): P const local = await localCandidates(ctx.db, parsed, release); let remote: MetadataCandidate[] = []; let lookupError: string | null = null; + let recognize: { cacheHit: boolean; meta: RecognizeMeta | null } = { cacheHit: false, meta: null }; try { - remote = await tmdbCandidates(rt, parsed); + const r = await tmdbCandidates(rt, parsed, !input.force); + remote = r.remote; + recognize = r; } catch (e) { if (!(e instanceof TmdbError)) throw e; lookupError = `${e.kind}: ${e.message}`; } const candidates = [...local, ...remote.map(tmdbToCandidate)]; const resolution = resolveIdentity(parsed, candidates); + // 识别缓存回写(MP TmdbCache.update):识别到 TMDB 条目写正常条目,确认无匹配写负缓存;TMDB 出错 / 有歧义不写 + if (recognize.meta && !recognize.cacheHit && !lookupError) { + const top = resolution.status === "auto" ? resolution.best?.candidate : undefined; + const hit = + top?.provider === "tmdb" + ? remote.find((m) => m.externalId === top.externalId && m.entityType === top.entityType) + : undefined; + const info = hit ?? (resolution.status === "no_match" ? null : undefined); + await saveRecognizeCache(rt, recognize.meta, info); + } const version = evidenceVersion(parsed, resolution.ranked); // 已识别且证据未变:幂等短路(强制复查时) diff --git a/apps/server/src/modules/sources-identity/service/tmdb-recognize-cache.ts b/apps/server/src/modules/sources-identity/service/tmdb-recognize-cache.ts new file mode 100644 index 00000000..40a92775 --- /dev/null +++ b/apps/server/src/modules/sources-identity/service/tmdb-recognize-cache.ts @@ -0,0 +1,154 @@ +import type { MetadataCandidate } from "@umt/adapters"; +import type { ParsedRelease } from "@umt/domain"; +import { CACHE_REGIONS, sharedCache, tmdbCachePolicy } from "../../../shared-cache.ts"; +import type { SourcesRuntime } from "../runtime.ts"; + +// TMDB 识别缓存:移植 MP app/modules/themoviedb/cache.py 的 TmdbCache(对照表见 docs/porting/moviepilot-tmdb-cache.md)。 +// 存储从 内存 TTLCache + pickle 文件持久化 换成 PG kv_cache(region tmdb.recognize),每条独立 expires_at, +// 三进程共享;"持久化与恢复"等价于表本身。语义照抄: +// - 键:[类型][语言]{tmdbid 或 名称}-{年份}-{开始季}(Python 的 None 原样写作 "None"); +// - 值:{id, type, year, title, poster_path, backdrop_path};负缓存 {id: 0}; +// - TTL:正常条目 = META_CACHE_EXPIRE(cache.tmdbTtlHours);负缓存 = EMPTY_RESULT_CACHE_TTL(cache.emptyResultTtlSec); +// - 条数上限 = CONF.tmdb(cache.tmdbMaxSize),超出删最旧; +// - 网络错误(None)不写;查无结果({})写负缓存; +// - 读 / 写时"元数据是电视剧、缓存是电影"视为类型冲突:读时丢弃该条,写时拒绝写入。 + +/** MP MediaType 的值(键里的类型段,与 MP 一致用中文) */ +export type MpMediaType = "电影" | "电视剧" | "未知"; + +export interface RecognizeCacheEntry { + /** TMDB id;0 = 负缓存(未识别) */ + id: number; + type?: "movie" | "tv" | null; + year?: string | null; + title?: string | null; + poster_path?: string | null; + backdrop_path?: string | null; +} + +/** 键所需的元数据(对照 MP MetaBase:type / name / year / begin_season / media_source+media_id) */ +export interface RecognizeMeta { + type: MpMediaType; + name: string; + year: number | null; + beginSeason: number | null; + /** 已知 TMDB id 时键用它代替名称(MP:media_source == TMDB 时取 media_id) */ + tmdbId?: string | null; +} + +const isAllChinese = (s: string) => /^[㐀-鿿豈-﫿]+$/.test(s); + +/** 解析结果 → 识别元数据(MP MetaBase.name:中文名全是汉字优先,否则英文名,再退回中文名) */ +export function recognizeMetaOf(p: ParsedRelease): RecognizeMeta | null { + const cn = p.titleCn ?? null; + const en = p.titleEn ?? null; + const name = cn && isAllChinese(cn) ? cn : (en ?? cn ?? p.title ?? ""); + if (!name) return null; + // MP MetaVideo:识别出季或集即为电视剧,否则类型未知 + const tv = p.season !== undefined || p.episodes !== undefined; + return { type: tv ? "电视剧" : "未知", name, year: p.year ?? null, beginSeason: p.season ?? null }; +} + +/** TmdbCache.__get_key */ +export function recognizeKey(meta: RecognizeMeta, locale: string): string { + const py = (v: number | null) => (v === null ? "None" : String(v)); + return `[${meta.type}][${locale}]${meta.tmdbId || meta.name}-${py(meta.year)}-${py(meta.beginSeason)}`; +} + +/** TmdbCache.__is_type_conflicted:只有"元数据是电视剧、结果是电影"算冲突;负缓存不带类型不算 */ +function typeConflicted(meta: RecognizeMeta, type: unknown, id: unknown): boolean { + if (meta.type !== "电视剧" || !id) return false; + return type === "movie"; +} + +function localeOf(tmdb: unknown): string { + return (tmdb as { language?: string } | null)?.language ?? "zh-CN"; +} + +/** TmdbCache.get:未命中返回 null;类型冲突的脏条目删除后按未命中处理 */ +export async function loadRecognizeCache(rt: SourcesRuntime, meta: RecognizeMeta): Promise { + const tmdb = await rt.tmdb(); + const key = recognizeKey(meta, localeOf(tmdb)); + const cache = sharedCache(rt.ctx); + const v = await cache.get(CACHE_REGIONS.tmdbRecognize, key).catch(() => null); + if (!v || typeof v !== "object") return null; + if (typeConflicted(meta, v.type, v.id)) { + rt.ctx.logger.warn({ key, title: v.title, type: v.type }, "识别缓存类型与元数据冲突，已丢弃并重新识别"); + await cache.delete(CACHE_REGIONS.tmdbRecognize, key); + return null; + } + return v; +} + +/** + * TmdbCache.update:info 为识别结果写正常条目;info 为 null(查无此项)写负缓存 {id: 0}; + * info 为 undefined(网络错误 / 没有结论)不写,允许下次重新请求。 + */ +export async function saveRecognizeCache( + rt: SourcesRuntime, + meta: RecognizeMeta, + info: + | Pick + | null + | undefined, +): Promise { + if (info === undefined) return; + const tmdb = await rt.tmdb(); + const key = recognizeKey(meta, localeOf(tmdb)); + const policy = await tmdbCachePolicy(rt.ctx); + const cache = sharedCache(rt.ctx); + if (info) { + if (typeConflicted(meta, info.entityType, info.externalId)) { + rt.ctx.logger.warn({ key, title: info.title, type: info.entityType }, "识别结果类型与元数据冲突，不写入识别缓存"); + return; + } + const entry: RecognizeCacheEntry = { + id: Number(info.externalId), + type: info.entityType, + year: info.year === null ? null : String(info.year), + title: info.title, + poster_path: info.posterPath ?? null, + backdrop_path: info.backdropPath ?? null, + }; + await cache.set(CACHE_REGIONS.tmdbRecognize, key, entry, { ttl: policy.ttl, maxsize: policy.maxsize }); + return; + } + // 负识别缓存用独立短 TTL(EMPTY_RESULT_CACHE_TTL):故障期间的空结果不被固化整个有效期 + await cache.set(CACHE_REGIONS.tmdbRecognize, key, { id: 0 }, { ttl: policy.emptyTtl, maxsize: policy.maxsize }); +} + +/** + * 人工认定后回填识别缓存(MP update_recognize_cache:覆盖名称负缓存,避免同名资源再次回查)。 + * 作品没有已接受的 TMDB 身份时不写。失败只记日志。 + */ +export async function backfillRecognizeCache( + rt: SourcesRuntime, + parsed: ParsedRelease, + entityId: string, +): Promise { + try { + const meta = recognizeMetaOf(parsed); + if (!meta || !(await rt.tmdb())) return; + const row = await rt.ctx.db + .selectFrom("external_identity as i") + .innerJoin("media_entity as e", "e.id", "i.media_entity_id") + .select(["i.entity_type", "i.external_id", "e.title", "e.year", "e.metadata"]) + .where("i.media_entity_id", "=", entityId) + .where("i.provider", "=", "tmdb") + .where("i.status", "=", "accepted") + .executeTakeFirst(); + if (!row || (row.entity_type !== "movie" && row.entity_type !== "tv")) return; + const md = (row.metadata ?? {}) as Record; + const str = (v: unknown) => (typeof v === "string" && v ? v : null); + await saveRecognizeCache(rt, meta, { + externalId: row.external_id, + entityType: row.entity_type, + title: row.title, + year: row.year, + posterPath: str(md.posterPath?.value), + backdropPath: str(md.backdropPath?.value), + }); + } catch (e) { + rt.ctx.logger.warn({ entityId, err: { message: (e as Error).message } }, "识别缓存回填失败"); + } +} diff --git a/apps/server/src/shared-cache.ts b/apps/server/src/shared-cache.ts new file mode 100644 index 00000000..193a9aad --- /dev/null +++ b/apps/server/src/shared-cache.ts @@ -0,0 +1,36 @@ +import { DEFAULT_SETTINGS } from "@umt/contracts"; +import { type PgCache, pgCacheFor } from "@umt/db"; +import type { ServerContext } from "./modules/types.ts"; + +// 进程级共享缓存入口:PG kv_cache(移植 MP app/runtime/cache.py 的 TTLCache,存储换成 PG,三进程共享)。 +// region 与 TTL 规则见 docs/runbook/cache.md 与 docs/porting/moviepilot-tmdb-cache.md。 + +export const CACHE_REGIONS = { + /** TMDB 请求响应(MP TMDb.request 的 @cached):key = path?query */ + tmdbRequest: "tmdb.request", + /** TMDB 识别结果(MP TmdbCache):key = [类型][语言]{tmdbid 或名称}-{年份}-{季} */ + tmdbRecognize: "tmdb.recognize", +} as const; + +export function sharedCache(ctx: Pick): PgCache { + return pgCacheFor(ctx.db); +} + +/** TMDB 缓存策略(MP:CONF.meta = META_CACHE_EXPIRE 小时、CONF.tmdb 条数、EMPTY_RESULT_CACHE_TTL 秒) */ +export interface TmdbCachePolicy { + /** 秒 */ + ttl: number; + maxsize: number; + /** 秒 */ + emptyTtl: number; +} + +export async function tmdbCachePolicy(ctx: Pick): Promise { + const c = await ctx.settings.get("cache").catch(() => DEFAULT_SETTINGS.cache); + const d = DEFAULT_SETTINGS.cache; + return { + ttl: (c.tmdbTtlHours ?? d.tmdbTtlHours) * 3600, + maxsize: c.tmdbMaxSize ?? d.tmdbMaxSize, + emptyTtl: c.emptyResultTtlSec ?? d.emptyResultTtlSec, + }; +} diff --git a/apps/server/test/modules.test.ts b/apps/server/test/modules.test.ts index efb4bb85..c927632c 100644 --- a/apps/server/test/modules.test.ts +++ b/apps/server/test/modules.test.ts @@ -43,7 +43,7 @@ const pingModule: ServerModule = { }; describe("ServerModule 骨架", () => { - it("清单里是四个 W2 模块 + 文件管理 + Pi 上游 + 定时任务 + 全局搜索 + 浏览器 + MCP + 进程与健康(末尾),名字唯一", () => { + it("清单里是四个 W2 模块 + 文件管理 + Pi 上游 + 定时任务 + 全局搜索 + 浏览器 + MCP + 缓存 + 进程与健康(末尾),名字唯一", () => { expect(SERVER_MODULES.map((m) => m.name)).toEqual([ "auth-settings", "sources-identity", @@ -55,6 +55,7 @@ describe("ServerModule 骨架", () => { "search", "browser", "mcp", + "cache", "system", ]); }); diff --git a/apps/server/test/sources-identity/resolve.test.ts b/apps/server/test/sources-identity/resolve.test.ts index 353d5e01..ea6e4109 100644 --- a/apps/server/test/sources-identity/resolve.test.ts +++ b/apps/server/test/sources-identity/resolve.test.ts @@ -1,4 +1,5 @@ import { CapabilityRegistry } from "@umt/agent"; +import { sql } from "@umt/db"; import { createMockTmdb, type MockTmdb, TMDB_FIXTURES, type TmdbFixture } from "@umt/testkit"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { refreshMetadata } from "../../src/modules/sources-identity/service/metadata.ts"; @@ -453,4 +454,39 @@ describe.skipIf(!up)("resolveObservation / 身份与元数据", () => { expect(search.items.length).toBeGreaterThan(0); await expect(reg.invoke("identity.resolve", { sourceReleaseId: twinsRelease }, c)).rejects.toThrow(); }); + it("识别缓存(移植 MP TmdbCache):键 [类型][语言]名称-年份-季;命中不再搜索;查无写 {id:0} 短 TTL;force 不读缓存", async () => { + const rows = async () => + ( + await sql<{ key: string; value: { id: number; type?: string; title?: string }; minutes: number }>` + select key, value, extract(epoch from expires_at - now()) / 60 as minutes + from kv_cache where region = 'tmdb.recognize'`.execute(h.db) + ).rows; + const opp = (await rows()).find((r) => r.key === "[未知][zh-CN]Oppenheimer-2023-None"); + expect(opp?.value).toMatchObject({ id: 872585, type: "movie", title: "奥本海默" }); + expect(Number(opp!.minutes)).toBeGreaterThan(23 * 60); + // 同名同年的另一条资源:命中识别缓存,不再调 TMDB 搜索 + const searches = tmdb.callsTo(/\/3\/search\//).length; + const again = await release("Oppenheimer.2023.1080p.BluRay.x264-OTHER"); + await resolveRelease(h.rt, { sourceReleaseId: again.id }); + expect(tmdb.callsTo(/\/3\/search\//).length).toBe(searches); + // 查无此项 → 负缓存 {id: 0},TTL = 空结果短 TTL(30 分钟) + const none = await release("Zzqxv.Nonexistent.Title.2019.1080p.WEB-DL-NOGRP"); + await resolveRelease(h.rt, { sourceReleaseId: none.id }); + const neg = (await rows()).find((r) => r.key === "[未知][zh-CN]Zzqxv Nonexistent Title-2019-None"); + expect(neg?.value).toEqual({ id: 0 }); + expect(Number(neg!.minutes)).toBeLessThanOrEqual(30); + const s2 = tmdb.callsTo(/\/3\/search\//).length; + await resolveRelease(h.rt, { sourceReleaseId: none.id }); + expect(tmdb.callsTo(/\/3\/search\//).length).toBe(s2); + // force:不读识别缓存(回源结果由请求层缓存兜住),照常回写识别缓存 + const at = async () => + ( + await sql<{ t: Date }>`select updated_at as t from kv_cache + where region = 'tmdb.recognize' and key = ${neg!.key}`.execute(h.db) + ).rows[0]!.t.getTime(); + const before = await at(); + await new Promise((r) => setTimeout(r, 10)); + await resolveRelease(h.rt, { sourceReleaseId: none.id, force: true }); + expect(await at()).toBeGreaterThan(before); + }); }); diff --git a/packages/adapters/src/tmdb/client.ts b/packages/adapters/src/tmdb/client.ts index 6d8f7924..f14ac4f7 100644 --- a/packages/adapters/src/tmdb/client.ts +++ b/packages/adapters/src/tmdb/client.ts @@ -59,6 +59,8 @@ export interface MetadataCandidate { overview: string | null; /** TMDB poster_path(如 `/abc.jpg`),拼图片 CDN 用;没有为 null */ posterPath: string | null; + /** TMDB backdrop_path;没有为 null(旧数据 / 测试构造可缺省) */ + backdropPath?: string | null; seasons?: CandidateSeason[]; externalIds?: { imdb?: string; tvdb?: string }; /** 是否已拉过详情(别名/季信息只在详情中完整) */ @@ -98,12 +100,30 @@ export interface TmdbConfig { minIntervalMs?: number; /** 默认 max 1024、ttl 24h;空结果(搜索 0 条)用 emptyTtlMs,默认 30min(对照 MP EMPTY_RESULT_CACHE_TTL) */ cache?: { max?: number; ttlMs?: number; emptyTtlMs?: number }; + /** + * 跨进程共享的请求缓存(服务端传 PG 缓存);给了就不再用进程内 LRU。 + * 语义对照 MP @cached(skip_none、业务失败不缓存、results 为空按 emptyTtlMs)。 + */ + sharedCache?: TmdbSharedCache; fetch?: typeof fetch; now?: () => number; } type Json = Record; +/** + * 共享请求缓存端口(对照 MP TMDb.request 的 @cached;服务端用 PG 实现):key 为 `path?query`(不含凭证); + * get 未命中返回 null;set 的 ttlMs 由客户端按 ttl / 空结果短 TTL 决定。 + */ +export interface TmdbSharedCache { + get(key: string): Promise; + set(key: string, value: unknown, ttlMs: number): Promise; +} + +/** 空结果(搜索 0 条)用短 TTL(对照 MP EMPTY_RESULT_CACHE_TTL) */ +const isEmptyResult = (v: unknown) => + Array.isArray((v as Json | null)?.results) && ((v as Json).results as unknown[]).length === 0; + const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); const s = (v: unknown): string | null => (typeof v === "string" && v !== "" ? v : null); const n = (v: unknown): number | null => (typeof v === "number" && Number.isFinite(v) ? v : null); @@ -130,6 +150,11 @@ export class TmdbClient { return redactUrl(text, [this.cfg.apiKey, this.cfg.accessToken]); } + /** 首选查询语言(识别缓存键里的语言段,对照 MP TMDB_LOCALE) */ + get language(): string { + return this.langs[0]!; + } + cacheStats() { return { size: this.cache.size, hits: this.cache.hits, misses: this.cache.misses, coalesced: this.cache.coalesced }; } @@ -139,13 +164,21 @@ export class TmdbClient { for (const k of Object.keys(params).sort()) if (params[k] !== undefined) qs.set(k, params[k]!); const key = `${path}?${qs}`; const emptyTtl = this.cfg.cache?.emptyTtlMs ?? 30 * 60_000; + if (this.cfg.sharedCache) { + const shared = this.cfg.sharedCache; + const hit = await shared.get(key); + if (hit !== null && hit !== undefined) return hit as T; + // 网络错误 / 业务失败在 fetchWithRetry 抛出,不会走到写缓存(MP skip_none / skip_if) + const v = await this.limiter.run(() => this.fetchWithRetry(path, qs)); + if (v !== null && v !== undefined) { + await shared.set(key, v, isEmptyResult(v) ? emptyTtl : (this.cfg.cache?.ttlMs ?? 24 * 3600_000)); + } + return v as T; + } return this.cache.getOrLoad( key, () => this.limiter.run(() => this.fetchWithRetry(path, qs)), - (v) => - Array.isArray((v as Json | null)?.results) && ((v as Json).results as unknown[]).length === 0 - ? emptyTtl - : undefined, + (v) => (isEmptyResult(v) ? emptyTtl : undefined), ) as Promise; } @@ -269,6 +302,7 @@ export class TmdbClient { popularity: n(r.popularity), overview: s(r.overview), posterPath: s(r.poster_path), + backdropPath: s(r.backdrop_path), detailed: false, }; } diff --git a/packages/adapters/src/tmdb/tmdb.test.ts b/packages/adapters/src/tmdb/tmdb.test.ts index 210d7597..83fa3cf8 100644 --- a/packages/adapters/src/tmdb/tmdb.test.ts +++ b/packages/adapters/src/tmdb/tmdb.test.ts @@ -110,6 +110,31 @@ describe("TmdbClient 缓存/限流/错误分类", () => { expect(mock.calls.length).toBe(n1 * 2); }); + it("共享响应缓存:给了 sharedCache 就走它,key 不含凭证,空结果 30 分钟,其余按 ttlMs", async () => { + const store = new Map(); + const { mock, tmdb } = await setup({ + cache: { ttlMs: 7 * 3600_000 }, + sharedCache: { + get: async (key) => store.get(key)?.v ?? null, + set: async (key, v, ttl) => { + store.set(key, { v, ttl }); + }, + }, + }); + await tmdb.details("movie", 872585); + const n1 = mock.calls.length; + await tmdb.details("movie", 872585); + expect(mock.calls.length).toBe(n1); + await tmdb.search("不存在的片名xyz", { type: "movie" }); + const keys = [...store.keys()]; + expect(keys.some((k) => k.includes("test-tmdb-key"))).toBe(false); + const detailKey = keys.find((k) => k.startsWith("/movie/872585?"))!; + expect(store.get(detailKey)!.ttl).toBe(7 * 3600_000); + const emptyKey = keys.find((k) => k.startsWith("/search/movie?"))!; + expect(store.get(emptyKey)!.ttl).toBe(30 * 60_000); + expect(tmdb.cacheStats().size).toBe(0); + }); + it("并发合并:同一请求并发只发一次", async () => { const { mock, tmdb } = await setup(); await Promise.all(Array.from({ length: 5 }, () => tmdb.season(100088, 1))); diff --git a/packages/contracts/src/content.ts b/packages/contracts/src/content.ts index 55009e3e..36df07a7 100644 --- a/packages/contracts/src/content.ts +++ b/packages/contracts/src/content.ts @@ -189,6 +189,7 @@ export const EntityMetadataField = z.enum([ "originalLanguage", "originCountry", "posterPath", + "backdropPath", ]); export type EntityMetadataField = z.infer; diff --git a/packages/contracts/src/search.ts b/packages/contracts/src/search.ts index b78b344e..08676531 100644 --- a/packages/contracts/src/search.ts +++ b/packages/contracts/src/search.ts @@ -52,6 +52,8 @@ export const TmdbCandidate = z.object({ year: z.number().int().nullable(), overview: z.string().nullable(), popularity: z.number().nullable(), + /** TMDB poster_path;前端经图片代理显示海报 */ + posterPath: z.string().nullable().optional(), /** 已有本地作品接受了这个 TMDB 身份时给出(跟随合并跳转),前端显示"查看"而不是"订阅" */ localEntityId: Id.nullable(), }); diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 7c362bc9..ea337df3 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -24,6 +24,16 @@ const CacheShape = z.object({ checkIntervalSec: z.number().int().min(10).max(3600), /** 缓存清理的目标使用率(低水位) */ cleanupTargetPercent: Percent, + /** TMDB 缓存(请求响应 + 识别结果)保留小时数(对照 MP META_CACHE_EXPIRE,缺省 24) */ + tmdbTtlHours: z.number().int().min(1).max(720), + /** TMDB 缓存每类条数上限,超出删最旧(对照 MP CONF.tmdb,缺省 256;大内存模式 1024) */ + tmdbMaxSize: z.number().int().min(16).max(100_000), + /** 空结果 / 未识别的短缓存秒数(对照 MP EMPTY_RESULT_CACHE_TTL,缺省 1800) */ + emptyResultTtlSec: z.number().int().min(0).max(86_400), + /** 图片代理落盘缓存(对照 MP GLOBAL_IMAGE_CACHE,缺省关闭;关闭时仍代理,只是不落盘) */ + globalImageCache: z.boolean(), + /** 图片缓存保留天数,超过即清理(对照 MP GLOBAL_IMAGE_CACHE_DAYS,缺省 7) */ + globalImageCacheDays: z.number().int().min(1).max(3650), }); /** 下载缓存卷硬阈值与恢复水位(REQ §8.4.1) */ export const CacheSettings = CacheShape.refine((v) => v.resumePercent < v.hardThresholdPercent, { @@ -330,7 +340,17 @@ export type SettingsKey = z.infer; /** 缺省设置(settings 表缺 key 或缺字段时使用) */ export const DEFAULT_SETTINGS: GlobalSettings = { automation: { enabled: true, readOnlyReconcile: false }, - cache: { hardThresholdPercent: 90, resumePercent: 85, checkIntervalSec: 60, cleanupTargetPercent: 80 }, + cache: { + hardThresholdPercent: 90, + resumePercent: 85, + checkIntervalSec: 60, + cleanupTargetPercent: 80, + tmdbTtlHours: 24, + tmdbMaxSize: 256, + emptyResultTtlSec: 1800, + globalImageCache: false, + globalImageCacheDays: 7, + }, organize: { defaultMode: "copy", allowCrossVolumeMove: true, minSizeBytes: 50 * 1024 * 1024 }, retention: { seedRetentionDays: 7, logRetentionDays: 180 }, pi: { @@ -587,10 +607,22 @@ export const SETTINGS_SECTIONS: readonly SettingsSectionDef[] = [ { id: "metadata", label: "元数据", - description: "TMDB API Key 与地址", + description: "TMDB API Key 与地址、TMDB 缓存、图片代理", legacyTabs: [], - anchors: [{ id: "tmdb", label: "TMDB" }], - fields: ["metadata.tmdbApiKeySecretRef", "metadata.tmdbBaseUrl"], + anchors: [ + { id: "tmdb", label: "TMDB" }, + { id: "tmdb-cache", label: "TMDB 缓存" }, + { id: "images", label: "图片代理与缓存" }, + ], + fields: [ + "metadata.tmdbApiKeySecretRef", + "metadata.tmdbBaseUrl", + "cache.tmdbTtlHours", + "cache.tmdbMaxSize", + "cache.emptyResultTtlSec", + "cache.globalImageCache", + "cache.globalImageCacheDays", + ], resources: [], }, { diff --git a/packages/contracts/test/db-sync.test.ts b/packages/contracts/test/db-sync.test.ts index 324fb3a4..eba5d075 100644 --- a/packages/contracts/test/db-sync.test.ts +++ b/packages/contracts/test/db-sync.test.ts @@ -130,6 +130,9 @@ const INTERNAL_TABLES = [ "SseEvent", // W9-C 命名模板版本登记(0032):服务端内部,经 /naming/* 暴露 "NamingTemplateVersion", + // C2-A 跨进程限流桶(0041)与 PG 缓存(0042):服务端内部,缓存经 /cache/* 的专用 DTO 暴露 + "RateLimitBucket", + "KvCache", // W9-I 进程心跳(0034):服务端内部,经 /system/health 的专用 DTO 暴露 "ProcessHeartbeat", "ProcessHeartbeatSample", diff --git a/packages/db/migrations/0042_cache.ts b/packages/db/migrations/0042_cache.ts new file mode 100644 index 00000000..748c88fd --- /dev/null +++ b/packages/db/migrations/0042_cache.ts @@ -0,0 +1,30 @@ +import type { Kysely } from "kysely"; +import { dropTables, execDdl } from "../src/migration-helpers.ts"; + +// C2-A 缓存:移植 MP app/runtime/cache.py 的 TTLCache(cachetools TLRUCache:按 region 分区、每个 key 独立 TTL、 +// maxsize 超限先删过期再删最久未用),存储从进程内存换成 PG UNLOGGED 表(不写 WAL,崩溃后清空,正合缓存语义),三进程共享。 +// expires_at 过期即视为未命中;accessed_at 为最近读写时间(LRU 依据);定时任务 cache.sweep 每 10 分钟删过期行。 + +// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型 +export async function up(db: Kysely): Promise { + await execDdl( + db, + ` +create unlogged table kv_cache ( + region text not null, + key text not null, + value jsonb not null, + expires_at timestamptz not null, + updated_at timestamptz not null default now(), + accessed_at timestamptz not null default now(), + primary key (region, key) +); +create index kv_cache_expires_idx on kv_cache (expires_at); +`, + ); +} + +// biome-ignore lint/suspicious/noExplicitAny: 同上 +export async function down(db: Kysely): Promise { + await execDdl(db, dropTables(["kv_cache"])); +} diff --git a/packages/db/src/cache.ts b/packages/db/src/cache.ts new file mode 100644 index 00000000..fdcff6a6 --- /dev/null +++ b/packages/db/src/cache.ts @@ -0,0 +1,175 @@ +import { type Kysely, sql } from "kysely"; +import type { DB } from "./types.ts"; + +/** + * PG 缓存后端:移植 MP app/runtime/cache.py 的内存后端(MemoryBackend + _MemoryTLRUCache)与 @cached 装饰器语义, + * 只把存储从进程内存换成 PG UNLOGGED 表 kv_cache(迁移 0042),三进程共享。行为对照(docs/porting/moviepilot-tmdb-cache.md): + * - region 分区;set 时每个 key 独立 TTL(秒),ttl ≤ 0 = 删除该 key;未给 ttl 用后端缺省(DEFAULT_CACHE_TTL 365 天); + * - maxsize 按 region 计条数,超出时先删已过期、再删最久未用(TLRU);get 命中刷新"最近使用"; + * - get 未命中 / 已过期返回 null;clear(region) 清一个区;items(region) 列出未过期条目; + * - cached():skipNone / skipEmpty / skipIf 不写缓存,emptyTtl + emptyIf 空结果短 TTL,fresh 跳过读缓存。 + * 与 MP 的差异只在存储介质:值用 JSON(jsonb)而不是 pickle;过期行由定时任务 sweep() 物理删除。 + */ + +/** MP DEFAULT_CACHE_TTL(秒) */ +export const DEFAULT_CACHE_TTL = 365 * 24 * 60 * 60; +/** MP DEFAULT_CACHE_SIZE */ +export const DEFAULT_CACHE_SIZE = 1024; + +// biome-ignore lint/suspicious/noExplicitAny: 测试会传入隔离 schema 的实例 +type AnyDb = Kysely | Kysely; + +export interface SetOptions { + /** 秒;缺省 DEFAULT_CACHE_TTL;≤ 0 删除该 key */ + ttl?: number; + /** region 条数上限;缺省 DEFAULT_CACHE_SIZE */ + maxsize?: number; +} + +export class PgCache { + constructor(private readonly db: AnyDb) {} + + /** MemoryBackend.set */ + async set(region: string, key: string, value: unknown, opts: SetOptions = {}): Promise { + const ttl = opts.ttl ?? DEFAULT_CACHE_TTL; + if (ttl <= 0) { + await this.delete(region, key); + return; + } + await sql`insert into kv_cache (region, key, value, expires_at, updated_at, accessed_at) + values (${region}, ${key}, ${JSON.stringify(value ?? null)}::jsonb, now() + make_interval(secs => ${ttl}), now(), now()) + on conflict (region, key) do update set value = excluded.value, expires_at = excluded.expires_at, + updated_at = excluded.updated_at, accessed_at = excluded.accessed_at`.execute(this.db); + await this.evict(region, opts.maxsize ?? DEFAULT_CACHE_SIZE); + } + + /** TLRUCache 写入时的淘汰:先删已过期,再按最近使用从旧到新删到 maxsize */ + private async evict(region: string, maxsize: number): Promise { + await sql`delete from kv_cache where region = ${region} and expires_at <= now()`.execute(this.db); + await sql`delete from kv_cache where region = ${region} and key in ( + select key from kv_cache where region = ${region} + order by accessed_at desc, updated_at desc offset ${Math.max(1, Math.floor(maxsize))})`.execute(this.db); + } + + /** MemoryBackend.get:未命中 / 过期返回 null;命中刷新最近使用时间 */ + async get(region: string, key: string): Promise { + const r = await sql<{ value: T }>`update kv_cache set accessed_at = now() + where region = ${region} and key = ${key} and expires_at > now() returning value`.execute(this.db); + return r.rows.length ? (r.rows[0]!.value as T) : null; + } + + async exists(region: string, key: string): Promise { + const r = + await sql`select 1 from kv_cache where region = ${region} and key = ${key} and expires_at > now()`.execute( + this.db, + ); + return r.rows.length > 0; + } + + async delete(region: string, key: string): Promise { + await sql`delete from kv_cache where region = ${region} and key = ${key}`.execute(this.db); + } + + /** 清一个区,返回删除行数 */ + async clear(region: string): Promise { + const r = await sql`delete from kv_cache where region = ${region}`.execute(this.db); + return Number(r.numAffectedRows ?? 0); + } + + /** MemoryBackend.items:未过期条目(附过期时间与大小,管理界面用),按 key 排序 */ + async items( + region: string, + limit = 5000, + ): Promise<{ key: string; value: T; expiresAt: Date; updatedAt: Date; bytes: number }[]> { + const r = await sql<{ key: string; value: T; expires_at: Date; updated_at: Date; bytes: number }>` + select key, value, expires_at, updated_at, pg_column_size(value)::int as bytes + from kv_cache where region = ${region} and expires_at > now() order by key limit ${limit}`.execute(this.db); + return r.rows.map((x) => ({ + key: x.key, + value: x.value, + expiresAt: x.expires_at, + updatedAt: x.updated_at, + bytes: Number(x.bytes), + })); + } + + /** 未过期条目数(len(region_cache)) */ + async count(region: string): Promise { + const r = await sql<{ n: number }>`select count(*)::int as n from kv_cache + where region = ${region} and expires_at > now()`.execute(this.db); + return Number(r.rows[0]?.n ?? 0); + } + + /** 删除全部过期行(存储换成 PG 后需要的物理清理;cache.sweep 定时任务) */ + async sweep(): Promise { + const r = await sql`delete from kv_cache where expires_at <= now()`.execute(this.db); + return Number(r.numAffectedRows ?? 0); + } +} + +/** MP @cached 的参数 */ +export interface CachedOptions { + region: string; + maxsize?: number; + /** 秒 */ + ttl?: number; + /** 缺省 true:结果为 null / undefined 不缓存 */ + skipNone?: boolean; + /** 缺省 false:结果为假值(空数组 / 空对象 / 空串…)不缓存 */ + skipEmpty?: boolean; + /** 返回真值时不缓存(如业务失败的响应) */ + skipIf?: (v: T) => boolean; + /** 空结果的独立 TTL(秒) */ + emptyTtl?: number; + /** 判断空结果;缺省按假值 */ + emptyIf?: (v: T) => boolean; + /** MP fresh():true 时不读缓存(仍写) */ + fresh?: boolean; +} + +const isFalsy = (v: unknown) => + !v || (Array.isArray(v) && v.length === 0) || (typeof v === "object" && Object.keys(v as object).length === 0); + +/** MP @cached 的调用体:命中直接返回;否则执行 fn,按 should_cache / get_cache_ttl 写入 */ +export async function cachedCall( + cache: PgCache, + key: string, + o: CachedOptions, + fn: () => Promise, +): Promise { + const should = (v: T | null | undefined) => { + if ((o.skipNone ?? true) && (v === null || v === undefined)) return false; + if (o.skipEmpty && isFalsy(v)) return false; + if (o.skipIf && v !== null && v !== undefined && o.skipIf(v)) return false; + return true; + }; + if (!o.fresh) { + const hit = await cache.get(o.region, key); + if (should(hit)) return hit as T; + } + const result = await fn(); + if (!should(result)) return result; + let ttl = o.ttl; + if (o.emptyTtl !== undefined) { + if (result === null || result === undefined) ttl = o.emptyTtl; + else if (o.emptyIf) ttl = o.emptyIf(result) ? o.emptyTtl : o.ttl; + else ttl = isFalsy(result) ? o.emptyTtl : o.ttl; + } + await cache.set(o.region, key, result, { + ...(ttl !== undefined ? { ttl } : {}), + ...(o.maxsize ? { maxsize: o.maxsize } : {}), + }); + return result; +} + +const caches = new WeakMap(); + +/** 同一个 Kysely 实例共用一个 PgCache */ +export function pgCacheFor(db: AnyDb): PgCache { + let c = caches.get(db); + if (!c) { + c = new PgCache(db); + caches.set(db, c); + } + return c; +} diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index 05b34b56..b2642d60 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -1,5 +1,14 @@ // 统一从本包取 Kysely 与 sql,避免各包各装一份 kysely export { type Kysely, sql, type Transaction } from "kysely"; +export { + type CachedOptions, + cachedCall, + DEFAULT_CACHE_SIZE, + DEFAULT_CACHE_TTL, + PgCache, + pgCacheFor, + type SetOptions as CacheSetOptions, +} from "./cache.ts"; export { type CreateDbOptions, createDb } from "./client.ts"; export { createMigrator, MIGRATIONS_DIR, type MigratorOptions, migrateToLatest, resetSchema } from "./migrator.ts"; export { diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index f76d35e3..8515f3af 100644 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -827,6 +827,15 @@ export interface IdentityRevisionTable { updated_at: GeneratedTimestamptz; } +export interface KvCacheTable { + region: string; + key: string; + value: Json; + expires_at: Timestamptz; + updated_at: GeneratedTimestamptz; + accessed_at: GeneratedTimestamptz; +} + export interface LibraryRootTable { id: Generated; name: string; @@ -1642,6 +1651,7 @@ export interface DB { identity_candidate: IdentityCandidateTable; identity_redirect: IdentityRedirectTable; identity_revision: IdentityRevisionTable; + kv_cache: KvCacheTable; library_root: LibraryRootTable; login_attempt: LoginAttemptTable; mcp_server_config: McpServerConfigTable; @@ -1787,6 +1797,9 @@ export type IdentityRedirectUpdate = Updateable; export type IdentityRevisionRow = Selectable; export type NewIdentityRevision = Insertable; export type IdentityRevisionUpdate = Updateable; +export type KvCacheRow = Selectable; +export type NewKvCache = Insertable; +export type KvCacheUpdate = Updateable; export type LibraryRootRow = Selectable; export type NewLibraryRoot = Insertable; export type LibraryRootUpdate = Updateable; diff --git a/packages/db/test/cache.test.ts b/packages/db/test/cache.test.ts new file mode 100644 index 00000000..e0a0822a --- /dev/null +++ b/packages/db/test/cache.test.ts @@ -0,0 +1,110 @@ +import { Kysely, PostgresDialect, sql } from "kysely"; +import pg from "pg"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { cachedCall, type DB, migrateToLatest, PgCache } from "../src/index.ts"; + +// PG 缓存后端(0042 kv_cache,UNLOGGED;移植 MP runtime/cache.py 的 TTLCache / @cached 语义)。 +const url = process.env.DATABASE_URL ?? "postgres://umt:umt@localhost:55433/umt"; +const skip = process.env.UMT_SKIP_DB_TESTS === "1"; +const schema = `umt_cache_${process.pid}_${Date.now().toString(36)}`; + +describe.skipIf(skip)("PgCache(真实 PG)", () => { + let admin: Kysely; + let db: Kysely; + + beforeAll(async () => { + admin = new Kysely({ dialect: new PostgresDialect({ pool: new pg.Pool({ connectionString: url, max: 1 }) }) }); + await sql.raw(`create schema ${schema}`).execute(admin); + db = new Kysely({ + dialect: new PostgresDialect({ + pool: new pg.Pool({ connectionString: url, max: 4, options: `-c search_path=${schema},public` }), + }), + }); + await migrateToLatest(db, { migrationTableSchema: schema }); + }); + + afterAll(async () => { + await db?.destroy(); + await sql.raw(`drop schema if exists ${schema} cascade`).execute(admin); + await admin?.destroy(); + }); + + it("kv_cache 是 UNLOGGED 表", async () => { + const r = await sql<{ p: string }>`select relpersistence as p from pg_class c + join pg_namespace n on n.oid = c.relnamespace where n.nspname = ${schema} and c.relname = 'kv_cache'`.execute(db); + expect(r.rows[0]?.p).toBe("u"); + }); + + it("set/get 跨实例共享;每个 key 独立 TTL,过期即未命中;ttl ≤ 0 删除;clear / items / count / sweep", async () => { + const a = new PgCache(db); + const b = new PgCache(db); + await a.set("t", "k1", { x: 1, list: [1, 2] }, { ttl: 60 }); + expect(await b.get("t", "k1")).toEqual({ x: 1, list: [1, 2] }); + expect(await b.get("t", "nope")).toBeNull(); + await a.set("t", "short", "v", { ttl: 0.05 }); + await new Promise((r) => setTimeout(r, 120)); + expect(await b.get("t", "short")).toBeNull(); + expect(await a.exists("t", "k1")).toBe(true); + await a.set("t", "k1", "x", { ttl: 0 }); + expect(await a.exists("t", "k1")).toBe(false); + await a.set("t", "k2", 2, { ttl: 60 }); + expect((await a.items("t")).map((i) => i.key)).toEqual(["k2"]); + expect(await a.count("t")).toBe(1); + await sql`update kv_cache set expires_at = now() - interval '1 second' where region = 't'`.execute(db); + expect(await a.sweep()).toBe(1); + await a.set("t", "k3", 3, { ttl: 60 }); + expect(await a.clear("t")).toBe(1); + }); + + it("maxsize:超限先删过期,再删最久未用(get 命中刷新最近使用,对照 TLRUCache)", async () => { + const c = new PgCache(db); + const tick = () => new Promise((r) => setTimeout(r, 5)); + await c.set("cap", "expired", 0, { ttl: 0.01, maxsize: 3 }); + await tick(); + await tick(); + for (const k of ["a", "b", "c"]) { + await c.set("cap", k, k, { ttl: 60, maxsize: 3 }); + await tick(); + } + await c.get("cap", "a"); // a 最近使用 + await tick(); + await c.set("cap", "d", "d", { ttl: 60, maxsize: 3 }); + expect((await c.items("cap")).map((i) => i.key)).toEqual(["a", "c", "d"]); + const all = await sql<{ n: number }>`select count(*)::int as n from kv_cache where region = 'cap'`.execute(db); + expect(all.rows[0]!.n).toBe(3); + }); + + it("cachedCall(@cached):skipNone / skipIf 不写;空结果按 emptyTtl;fresh 跳过读", async () => { + const c = new PgCache(db); + let n = 0; + const o = { + region: "cc", + ttl: 3600, + emptyTtl: 1800, + emptyIf: (v: { results: unknown[] }) => v.results.length === 0, + }; + const full = () => + cachedCall(c, "full", o, async () => { + n++; + return { results: [1] }; + }); + await full(); + await full(); + expect(n).toBe(1); + await cachedCall(c, "empty", o, async () => ({ results: [] })); + const ttl = await sql<{ k: string; s: number }>`select key as k, extract(epoch from expires_at - now()) as s + from kv_cache where region = 'cc' order by key`.execute(db); + expect(Number(ttl.rows.find((r) => r.k === "empty")!.s)).toBeLessThanOrEqual(1800); + expect(Number(ttl.rows.find((r) => r.k === "full")!.s)).toBeGreaterThan(1800); + await cachedCall(c, "none", o, async () => null as unknown as { results: unknown[] }); + await cachedCall(c, "fail", { ...o, skipIf: () => true }, async () => ({ results: [1] })); + expect(await c.exists("cc", "none")).toBe(false); + expect(await c.exists("cc", "fail")).toBe(false); + await cachedCall(c, "full", { ...o, fresh: true }, async () => { + n++; + return { results: [2] }; + }); + expect(n).toBe(2); + expect(await c.get("cc", "full")).toEqual({ results: [2] }); + }); +}); diff --git a/packages/db/test/schema.test.ts b/packages/db/test/schema.test.ts index 6c733c36..684f653a 100644 --- a/packages/db/test/schema.test.ts +++ b/packages/db/test/schema.test.ts @@ -117,6 +117,8 @@ const EXPECTED_TABLES = [ "webauthn_challenge", // C2-A 跨进程站点限流(0041) "rate_limit_bucket", + // C2-A PG 短期缓存(0042,UNLOGGED) + "kv_cache", ].sort(); /** 不走 entityTable() 的表:追加型/引擎自有表,没有 uuid id + created_at/updated_at 三件套 */ @@ -143,6 +145,8 @@ const NON_ENTITY_TABLES = [ "webauthn_challenge", // 限流桶:key 主键,只有 updated_at "rate_limit_bucket", + // 缓存:(region, key) 主键 + "kv_cache", ]; describe.skipIf(skip)("schema migrations (real PG)", () => {