feat(cache): PG kv_cache ported from MP TTLCache, TMDB request and recognize caches
Migration 0042_cache adds UNLOGGED kv_cache. PgCache mirrors MP
runtime/cache.py (region, per-key TTL, TLRU maxsize eviction, @cached as
cachedCall); only the storage moves from memory to PG. TMDB request
responses (MP TMDb.request @cached) and recognize results (MP TmdbCache:
same key, {id:0} negative cache with EMPTY_RESULT_CACHE_TTL, type-conflict
checks, backfill after manual assignment) live in kv_cache, shared by all
processes. Settings cache.tmdbTtlHours / tmdbMaxSize / emptyResultTtlSec
map to META_CACHE_EXPIRE / CONF.tmdb / EMPTY_RESULT_CACHE_TTL. TMDB
details now also store posterPath / backdropPath.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
61f585a707
commit
946fca5e55
22 files changed
+743
-14
No files matched your search
@@ -2,6 +2,7 @@ import { acquisitionOrganizeModule } from "./acquisition-organize/index.ts";
|
||||
import { agentModule } from "./agent/index.ts";
|
||||
import { authSettingsModule } from "./auth-settings/index.ts";
|
||||
import { browserModule } from "./browser/index.ts";
|
||||
import { cacheModule } from "./cache/index.ts";
|
||||
import { fileManagerModule } from "./file-manager/index.ts";
|
||||
import { mcpModule } from "./mcp/index.ts";
|
||||
import { piModule } from "./pi/index.ts";
|
||||
@@ -28,5 +29,6 @@ export const SERVER_MODULES: readonly ServerModule[] = [
|
||||
searchModule,
|
||||
browserModule,
|
||||
mcpModule,
|
||||
cacheModule,
|
||||
systemModule,
|
||||
];
|
||||
@@ -163,6 +163,7 @@ export class TmdbSearch {
|
||||
year: m.year,
|
||||
overview: m.overview ? (m.overview.length > 300 ? `${m.overview.slice(0, 300)}…` : m.overview) : null,
|
||||
popularity: m.popularity,
|
||||
...(m.posterPath ? { posterPath: m.posterPath } : {}),
|
||||
})),
|
||||
)
|
||||
.then((list) => {
|
||||
|
||||
@@ -11,9 +11,11 @@ import {
|
||||
type SiteDefinition,
|
||||
type SiteRateLimiter,
|
||||
type TmdbClient,
|
||||
type TmdbSharedCache,
|
||||
} from "@umt/adapters";
|
||||
import { DEFAULT_SETTINGS, type SitesSettings } from "@umt/contracts";
|
||||
import { pgTokenBucketFor, type SiteRow } from "@umt/db";
|
||||
import { CACHE_REGIONS, sharedCache, type TmdbCachePolicy, tmdbCachePolicy } from "../../shared-cache.ts";
|
||||
import type { ServerContext } from "../types.ts";
|
||||
|
||||
// 模块运行时依赖:站点适配器与限流器(每站一个实例,令牌在 PG 跨进程共享)、RSS 抓取器、TMDB 客户端、时钟。
|
||||
@@ -71,12 +73,14 @@ export class SourcesRuntime {
|
||||
return this.tmdbClient;
|
||||
}
|
||||
const meta = await this.ctx.settings.get("metadata").catch(() => null);
|
||||
const policy = await tmdbCachePolicy(this.ctx);
|
||||
const env = (k: string) => Deno.env.get(k);
|
||||
const ref = meta?.tmdbApiKeySecretRef ?? null;
|
||||
const baseUrl = meta?.tmdbBaseUrl ?? env("TMDB_BASE_URL");
|
||||
const sig = JSON.stringify([
|
||||
ref,
|
||||
baseUrl ?? null,
|
||||
policy,
|
||||
ref ? null : [!!env("TMDB_API_KEY"), !!env("TMDB_ACCESS_TOKEN")],
|
||||
]);
|
||||
if (this.tmdbClient !== undefined && this.tmdbSig === sig) return this.tmdbClient;
|
||||
@@ -103,6 +107,9 @@ export class SourcesRuntime {
|
||||
...(accessToken ? { accessToken } : {}),
|
||||
...(baseUrl ? { baseUrl } : {}),
|
||||
...(this.overrides.fetch ? { fetch: this.overrides.fetch } : {}),
|
||||
// 请求响应进 PG 缓存(region tmdb.request),三进程共享;TTL / 条数上限 / 空结果短 TTL 取设置(对照 MP)
|
||||
cache: { ttlMs: policy.ttl * 1000, emptyTtlMs: policy.emptyTtl * 1000 },
|
||||
sharedCache: tmdbSharedCache(this.ctx, policy),
|
||||
})
|
||||
: null;
|
||||
return this.tmdbClient;
|
||||
@@ -233,6 +240,22 @@ export class SourcesRuntime {
|
||||
}
|
||||
}
|
||||
|
||||
/** TMDB 请求响应缓存:PG kv_cache region=tmdb.request;PG 出错按未命中 / 不写处理,不让 TMDB 请求失败 */
|
||||
function tmdbSharedCache(ctx: ServerContext, policy: TmdbCachePolicy): TmdbSharedCache {
|
||||
const cache = sharedCache(ctx);
|
||||
const warn = (err: unknown) =>
|
||||
ctx.logger.warn({ err: { message: (err as Error).message } }, "TMDB 缓存读写失败,按未命中处理");
|
||||
return {
|
||||
get: (key) =>
|
||||
cache.get(CACHE_REGIONS.tmdbRequest, key).catch((e) => {
|
||||
warn(e);
|
||||
return null;
|
||||
}),
|
||||
set: (key, value, ttlMs) =>
|
||||
cache.set(CACHE_REGIONS.tmdbRequest, key, value, { ttl: ttlMs / 1000, maxsize: policy.maxsize }).catch(warn),
|
||||
};
|
||||
}
|
||||
|
||||
function sameUrl(a: string, b: string): boolean {
|
||||
return a.replace(/\/+$/, "") === b.replace(/\/+$/, "");
|
||||
}
|
||||
|
||||
@@ -23,6 +23,7 @@ import {
|
||||
resolveEntityId,
|
||||
} from "./entities.ts";
|
||||
import { recordAttempt } from "./resolve.ts";
|
||||
import { backfillRecognizeCache } from "./tmdb-recognize-cache.ts";
|
||||
|
||||
// 人工纠错:指定来源资源身份(加锁)、改作品外部映射(新增修订、旧映射退役)、合并作品(旧 ID 跳转)。
|
||||
// 所有变更留 identity_revision / resolution_attempt / audit_event,不直接覆盖历史。
|
||||
@@ -85,7 +86,7 @@ export async function assignReleaseIdentity(
|
||||
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
try {
|
||||
return await ctx.db.transaction().execute(async (trx) => {
|
||||
const done = await ctx.db.transaction().execute(async (trx) => {
|
||||
let entityId: string;
|
||||
if (localId) {
|
||||
const r = await resolveEntityId(trx, localId);
|
||||
@@ -190,6 +191,8 @@ export async function assignReleaseIdentity(
|
||||
});
|
||||
return { mediaEntityId: entityId, contentUnitIds: unitIds };
|
||||
});
|
||||
await backfillRecognizeCache(rt, parsed, done.mediaEntityId);
|
||||
return done;
|
||||
} catch (e) {
|
||||
if (e instanceof IdentityConflictError && attempt < 2) continue;
|
||||
throw e;
|
||||
|
||||
@@ -32,6 +32,9 @@ function tmdbFields(m: MetadataCandidate): Partial<Record<EntityMetadataField, u
|
||||
releaseDate: m.releaseDate,
|
||||
originalLanguage: m.originalLanguage,
|
||||
originCountry: m.originCountry,
|
||||
// 海报 / 背景图路径(前端经图片代理显示);TMDB 没有时不写,保留旧值
|
||||
...(m.posterPath ? { posterPath: m.posterPath } : {}),
|
||||
...(m.backdropPath ? { backdropPath: m.backdropPath } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -32,6 +32,7 @@ import {
|
||||
looksLikeVariety,
|
||||
resolveEntityId,
|
||||
} from "./entities.ts";
|
||||
import { loadRecognizeCache, type RecognizeMeta, recognizeMetaOf, saveRecognizeCache } from "./tmdb-recognize-cache.ts";
|
||||
|
||||
// resolveObservation:parseRelease → 本地召回(别名 pg_trgm + 已有外部身份 + 人工锁)→ TMDB 候选 → domain.resolveIdentity。
|
||||
// auto 才建/关联正式作品;ambiguous / no_match / 查询失败进待识别池(候选只存 JSON),有界退避复查。
|
||||
@@ -155,9 +156,27 @@ async function localCandidates(
|
||||
return out;
|
||||
}
|
||||
|
||||
async function tmdbCandidates(rt: SourcesRuntime, parsed: ParsedRelease): Promise<MetadataCandidate[]> {
|
||||
/**
|
||||
* TMDB 召回。先查识别缓存(移植 MP TmdbCache,见 tmdb-recognize-cache.ts):命中且有标题 → 按 id 取详情作唯一远端候选;
|
||||
* 命中负缓存 → 不再搜索(视为 TMDB 无结果)。强制复查(force)不读缓存但照常回写(MP recognize_media 的 cache=False)。
|
||||
*/
|
||||
async function tmdbCandidates(
|
||||
rt: SourcesRuntime,
|
||||
parsed: ParsedRelease,
|
||||
useCache: boolean,
|
||||
): Promise<{ remote: MetadataCandidate[]; cacheHit: boolean; meta: RecognizeMeta | null }> {
|
||||
const tmdb = await rt.tmdb();
|
||||
if (!tmdb) return [];
|
||||
if (!tmdb) return { remote: [], cacheHit: false, meta: null };
|
||||
const meta = recognizeMetaOf(parsed);
|
||||
if (meta && useCache) {
|
||||
const hit = await loadRecognizeCache(rt, meta);
|
||||
if (hit) {
|
||||
if (hit.id && hit.title && (hit.type === "movie" || hit.type === "tv")) {
|
||||
return { remote: [await tmdb.details(hit.type, hit.id)], cacheHit: true, meta };
|
||||
}
|
||||
return { remote: [], cacheHit: true, meta };
|
||||
}
|
||||
}
|
||||
const queries = [parsed.titleCn, parsed.titleEn, parsed.title].filter((t): t is string => !!t);
|
||||
const seen = new Map<string, MetadataCandidate>();
|
||||
for (const q of [...new Set(queries)].slice(0, 2)) {
|
||||
@@ -166,7 +185,7 @@ async function tmdbCandidates(rt: SourcesRuntime, parsed: ParsedRelease): Promis
|
||||
if (!seen.has(k)) seen.set(k, m);
|
||||
}
|
||||
}
|
||||
return [...seen.values()].slice(0, 20);
|
||||
return { remote: [...seen.values()].slice(0, 20), cacheHit: false, meta };
|
||||
}
|
||||
|
||||
async function loadRelease(rt: SourcesRuntime, input: ResolveInput) {
|
||||
@@ -240,14 +259,27 @@ export async function resolveRelease(rt: SourcesRuntime, input: ResolveInput): P
|
||||
const local = await localCandidates(ctx.db, parsed, release);
|
||||
let remote: MetadataCandidate[] = [];
|
||||
let lookupError: string | null = null;
|
||||
let recognize: { cacheHit: boolean; meta: RecognizeMeta | null } = { cacheHit: false, meta: null };
|
||||
try {
|
||||
remote = await tmdbCandidates(rt, parsed);
|
||||
const r = await tmdbCandidates(rt, parsed, !input.force);
|
||||
remote = r.remote;
|
||||
recognize = r;
|
||||
} catch (e) {
|
||||
if (!(e instanceof TmdbError)) throw e;
|
||||
lookupError = `${e.kind}: ${e.message}`;
|
||||
}
|
||||
const candidates = [...local, ...remote.map(tmdbToCandidate)];
|
||||
const resolution = resolveIdentity(parsed, candidates);
|
||||
// 识别缓存回写(MP TmdbCache.update):识别到 TMDB 条目写正常条目,确认无匹配写负缓存;TMDB 出错 / 有歧义不写
|
||||
if (recognize.meta && !recognize.cacheHit && !lookupError) {
|
||||
const top = resolution.status === "auto" ? resolution.best?.candidate : undefined;
|
||||
const hit =
|
||||
top?.provider === "tmdb"
|
||||
? remote.find((m) => m.externalId === top.externalId && m.entityType === top.entityType)
|
||||
: undefined;
|
||||
const info = hit ?? (resolution.status === "no_match" ? null : undefined);
|
||||
await saveRecognizeCache(rt, recognize.meta, info);
|
||||
}
|
||||
const version = evidenceVersion(parsed, resolution.ranked);
|
||||
|
||||
// 已识别且证据未变:幂等短路(强制复查时)
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
import type { MetadataCandidate } from "@umt/adapters";
|
||||
import type { ParsedRelease } from "@umt/domain";
|
||||
import { CACHE_REGIONS, sharedCache, tmdbCachePolicy } from "../../../shared-cache.ts";
|
||||
import type { SourcesRuntime } from "../runtime.ts";
|
||||
|
||||
// TMDB 识别缓存:移植 MP app/modules/themoviedb/cache.py 的 TmdbCache(对照表见 docs/porting/moviepilot-tmdb-cache.md)。
|
||||
// 存储从 内存 TTLCache + pickle 文件持久化 换成 PG kv_cache(region tmdb.recognize),每条独立 expires_at,
|
||||
// 三进程共享;"持久化与恢复"等价于表本身。语义照抄:
|
||||
// - 键:[类型][语言]{tmdbid 或 名称}-{年份}-{开始季}(Python 的 None 原样写作 "None");
|
||||
// - 值:{id, type, year, title, poster_path, backdrop_path};负缓存 {id: 0};
|
||||
// - TTL:正常条目 = META_CACHE_EXPIRE(cache.tmdbTtlHours);负缓存 = EMPTY_RESULT_CACHE_TTL(cache.emptyResultTtlSec);
|
||||
// - 条数上限 = CONF.tmdb(cache.tmdbMaxSize),超出删最旧;
|
||||
// - 网络错误(None)不写;查无结果({})写负缓存;
|
||||
// - 读 / 写时"元数据是电视剧、缓存是电影"视为类型冲突:读时丢弃该条,写时拒绝写入。
|
||||
|
||||
/** MP MediaType 的值(键里的类型段,与 MP 一致用中文) */
|
||||
export type MpMediaType = "电影" | "电视剧" | "未知";
|
||||
|
||||
export interface RecognizeCacheEntry {
|
||||
/** TMDB id;0 = 负缓存(未识别) */
|
||||
id: number;
|
||||
type?: "movie" | "tv" | null;
|
||||
year?: string | null;
|
||||
title?: string | null;
|
||||
poster_path?: string | null;
|
||||
backdrop_path?: string | null;
|
||||
}
|
||||
|
||||
/** 键所需的元数据(对照 MP MetaBase:type / name / year / begin_season / media_source+media_id) */
|
||||
export interface RecognizeMeta {
|
||||
type: MpMediaType;
|
||||
name: string;
|
||||
year: number | null;
|
||||
beginSeason: number | null;
|
||||
/** 已知 TMDB id 时键用它代替名称(MP:media_source == TMDB 时取 media_id) */
|
||||
tmdbId?: string | null;
|
||||
}
|
||||
|
||||
const isAllChinese = (s: string) => /^[㐀-鿿豈-]+$/.test(s);
|
||||
|
||||
/** 解析结果 → 识别元数据(MP MetaBase.name:中文名全是汉字优先,否则英文名,再退回中文名) */
|
||||
export function recognizeMetaOf(p: ParsedRelease): RecognizeMeta | null {
|
||||
const cn = p.titleCn ?? null;
|
||||
const en = p.titleEn ?? null;
|
||||
const name = cn && isAllChinese(cn) ? cn : (en ?? cn ?? p.title ?? "");
|
||||
if (!name) return null;
|
||||
// MP MetaVideo:识别出季或集即为电视剧,否则类型未知
|
||||
const tv = p.season !== undefined || p.episodes !== undefined;
|
||||
return { type: tv ? "电视剧" : "未知", name, year: p.year ?? null, beginSeason: p.season ?? null };
|
||||
}
|
||||
|
||||
/** TmdbCache.__get_key */
|
||||
export function recognizeKey(meta: RecognizeMeta, locale: string): string {
|
||||
const py = (v: number | null) => (v === null ? "None" : String(v));
|
||||
return `[${meta.type}][${locale}]${meta.tmdbId || meta.name}-${py(meta.year)}-${py(meta.beginSeason)}`;
|
||||
}
|
||||
|
||||
/** TmdbCache.__is_type_conflicted:只有"元数据是电视剧、结果是电影"算冲突;负缓存不带类型不算 */
|
||||
function typeConflicted(meta: RecognizeMeta, type: unknown, id: unknown): boolean {
|
||||
if (meta.type !== "电视剧" || !id) return false;
|
||||
return type === "movie";
|
||||
}
|
||||
|
||||
function localeOf(tmdb: unknown): string {
|
||||
return (tmdb as { language?: string } | null)?.language ?? "zh-CN";
|
||||
}
|
||||
|
||||
/** TmdbCache.get:未命中返回 null;类型冲突的脏条目删除后按未命中处理 */
|
||||
export async function loadRecognizeCache(rt: SourcesRuntime, meta: RecognizeMeta): Promise<RecognizeCacheEntry | null> {
|
||||
const tmdb = await rt.tmdb();
|
||||
const key = recognizeKey(meta, localeOf(tmdb));
|
||||
const cache = sharedCache(rt.ctx);
|
||||
const v = await cache.get<RecognizeCacheEntry>(CACHE_REGIONS.tmdbRecognize, key).catch(() => null);
|
||||
if (!v || typeof v !== "object") return null;
|
||||
if (typeConflicted(meta, v.type, v.id)) {
|
||||
rt.ctx.logger.warn({ key, title: v.title, type: v.type }, "识别缓存类型与元数据冲突,已丢弃并重新识别");
|
||||
await cache.delete(CACHE_REGIONS.tmdbRecognize, key);
|
||||
return null;
|
||||
}
|
||||
return v;
|
||||
}
|
||||
|
||||
/**
|
||||
* TmdbCache.update:info 为识别结果写正常条目;info 为 null(查无此项)写负缓存 {id: 0};
|
||||
* info 为 undefined(网络错误 / 没有结论)不写,允许下次重新请求。
|
||||
*/
|
||||
export async function saveRecognizeCache(
|
||||
rt: SourcesRuntime,
|
||||
meta: RecognizeMeta,
|
||||
info:
|
||||
| Pick<MetadataCandidate, "externalId" | "entityType" | "title" | "year" | "posterPath" | "backdropPath">
|
||||
| null
|
||||
| undefined,
|
||||
): Promise<void> {
|
||||
if (info === undefined) return;
|
||||
const tmdb = await rt.tmdb();
|
||||
const key = recognizeKey(meta, localeOf(tmdb));
|
||||
const policy = await tmdbCachePolicy(rt.ctx);
|
||||
const cache = sharedCache(rt.ctx);
|
||||
if (info) {
|
||||
if (typeConflicted(meta, info.entityType, info.externalId)) {
|
||||
rt.ctx.logger.warn({ key, title: info.title, type: info.entityType }, "识别结果类型与元数据冲突,不写入识别缓存");
|
||||
return;
|
||||
}
|
||||
const entry: RecognizeCacheEntry = {
|
||||
id: Number(info.externalId),
|
||||
type: info.entityType,
|
||||
year: info.year === null ? null : String(info.year),
|
||||
title: info.title,
|
||||
poster_path: info.posterPath ?? null,
|
||||
backdrop_path: info.backdropPath ?? null,
|
||||
};
|
||||
await cache.set(CACHE_REGIONS.tmdbRecognize, key, entry, { ttl: policy.ttl, maxsize: policy.maxsize });
|
||||
return;
|
||||
}
|
||||
// 负识别缓存用独立短 TTL(EMPTY_RESULT_CACHE_TTL):故障期间的空结果不被固化整个有效期
|
||||
await cache.set(CACHE_REGIONS.tmdbRecognize, key, { id: 0 }, { ttl: policy.emptyTtl, maxsize: policy.maxsize });
|
||||
}
|
||||
|
||||
/**
|
||||
* 人工认定后回填识别缓存(MP update_recognize_cache:覆盖名称负缓存,避免同名资源再次回查)。
|
||||
* 作品没有已接受的 TMDB 身份时不写。失败只记日志。
|
||||
*/
|
||||
export async function backfillRecognizeCache(
|
||||
rt: SourcesRuntime,
|
||||
parsed: ParsedRelease,
|
||||
entityId: string,
|
||||
): Promise<void> {
|
||||
try {
|
||||
const meta = recognizeMetaOf(parsed);
|
||||
if (!meta || !(await rt.tmdb())) return;
|
||||
const row = await rt.ctx.db
|
||||
.selectFrom("external_identity as i")
|
||||
.innerJoin("media_entity as e", "e.id", "i.media_entity_id")
|
||||
.select(["i.entity_type", "i.external_id", "e.title", "e.year", "e.metadata"])
|
||||
.where("i.media_entity_id", "=", entityId)
|
||||
.where("i.provider", "=", "tmdb")
|
||||
.where("i.status", "=", "accepted")
|
||||
.executeTakeFirst();
|
||||
if (!row || (row.entity_type !== "movie" && row.entity_type !== "tv")) return;
|
||||
const md = (row.metadata ?? {}) as Record<string, { value?: unknown } | undefined>;
|
||||
const str = (v: unknown) => (typeof v === "string" && v ? v : null);
|
||||
await saveRecognizeCache(rt, meta, {
|
||||
externalId: row.external_id,
|
||||
entityType: row.entity_type,
|
||||
title: row.title,
|
||||
year: row.year,
|
||||
posterPath: str(md.posterPath?.value),
|
||||
backdropPath: str(md.backdropPath?.value),
|
||||
});
|
||||
} catch (e) {
|
||||
rt.ctx.logger.warn({ entityId, err: { message: (e as Error).message } }, "识别缓存回填失败");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
import { DEFAULT_SETTINGS } from "@umt/contracts";
|
||||
import { type PgCache, pgCacheFor } from "@umt/db";
|
||||
import type { ServerContext } from "./modules/types.ts";
|
||||
|
||||
// 进程级共享缓存入口:PG kv_cache(移植 MP app/runtime/cache.py 的 TTLCache,存储换成 PG,三进程共享)。
|
||||
// region 与 TTL 规则见 docs/runbook/cache.md 与 docs/porting/moviepilot-tmdb-cache.md。
|
||||
|
||||
export const CACHE_REGIONS = {
|
||||
/** TMDB 请求响应(MP TMDb.request 的 @cached):key = path?query */
|
||||
tmdbRequest: "tmdb.request",
|
||||
/** TMDB 识别结果(MP TmdbCache):key = [类型][语言]{tmdbid 或名称}-{年份}-{季} */
|
||||
tmdbRecognize: "tmdb.recognize",
|
||||
} as const;
|
||||
|
||||
export function sharedCache(ctx: Pick<ServerContext, "db">): PgCache {
|
||||
return pgCacheFor(ctx.db);
|
||||
}
|
||||
|
||||
/** TMDB 缓存策略(MP:CONF.meta = META_CACHE_EXPIRE 小时、CONF.tmdb 条数、EMPTY_RESULT_CACHE_TTL 秒) */
|
||||
export interface TmdbCachePolicy {
|
||||
/** 秒 */
|
||||
ttl: number;
|
||||
maxsize: number;
|
||||
/** 秒 */
|
||||
emptyTtl: number;
|
||||
}
|
||||
|
||||
export async function tmdbCachePolicy(ctx: Pick<ServerContext, "settings">): Promise<TmdbCachePolicy> {
|
||||
const c = await ctx.settings.get("cache").catch(() => DEFAULT_SETTINGS.cache);
|
||||
const d = DEFAULT_SETTINGS.cache;
|
||||
return {
|
||||
ttl: (c.tmdbTtlHours ?? d.tmdbTtlHours) * 3600,
|
||||
maxsize: c.tmdbMaxSize ?? d.tmdbMaxSize,
|
||||
emptyTtl: c.emptyResultTtlSec ?? d.emptyResultTtlSec,
|
||||
};
|
||||
}
|
||||
@@ -43,7 +43,7 @@ const pingModule: ServerModule = {
|
||||
};
|
||||
|
||||
describe("ServerModule 骨架", () => {
|
||||
it("清单里是四个 W2 模块 + 文件管理 + Pi 上游 + 定时任务 + 全局搜索 + 浏览器 + MCP + 进程与健康(末尾),名字唯一", () => {
|
||||
it("清单里是四个 W2 模块 + 文件管理 + Pi 上游 + 定时任务 + 全局搜索 + 浏览器 + MCP + 缓存 + 进程与健康(末尾),名字唯一", () => {
|
||||
expect(SERVER_MODULES.map((m) => m.name)).toEqual([
|
||||
"auth-settings",
|
||||
"sources-identity",
|
||||
@@ -55,6 +55,7 @@ describe("ServerModule 骨架", () => {
|
||||
"search",
|
||||
"browser",
|
||||
"mcp",
|
||||
"cache",
|
||||
"system",
|
||||
]);
|
||||
});
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { CapabilityRegistry } from "@umt/agent";
|
||||
import { sql } from "@umt/db";
|
||||
import { createMockTmdb, type MockTmdb, TMDB_FIXTURES, type TmdbFixture } from "@umt/testkit";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { refreshMetadata } from "../../src/modules/sources-identity/service/metadata.ts";
|
||||
@@ -453,4 +454,39 @@ describe.skipIf(!up)("resolveObservation / 身份与元数据", () => {
|
||||
expect(search.items.length).toBeGreaterThan(0);
|
||||
await expect(reg.invoke("identity.resolve", { sourceReleaseId: twinsRelease }, c)).rejects.toThrow();
|
||||
});
|
||||
it("识别缓存(移植 MP TmdbCache):键 [类型][语言]名称-年份-季;命中不再搜索;查无写 {id:0} 短 TTL;force 不读缓存", async () => {
|
||||
const rows = async () =>
|
||||
(
|
||||
await sql<{ key: string; value: { id: number; type?: string; title?: string }; minutes: number }>`
|
||||
select key, value, extract(epoch from expires_at - now()) / 60 as minutes
|
||||
from kv_cache where region = 'tmdb.recognize'`.execute(h.db)
|
||||
).rows;
|
||||
const opp = (await rows()).find((r) => r.key === "[未知][zh-CN]Oppenheimer-2023-None");
|
||||
expect(opp?.value).toMatchObject({ id: 872585, type: "movie", title: "奥本海默" });
|
||||
expect(Number(opp!.minutes)).toBeGreaterThan(23 * 60);
|
||||
// 同名同年的另一条资源:命中识别缓存,不再调 TMDB 搜索
|
||||
const searches = tmdb.callsTo(/\/3\/search\//).length;
|
||||
const again = await release("Oppenheimer.2023.1080p.BluRay.x264-OTHER");
|
||||
await resolveRelease(h.rt, { sourceReleaseId: again.id });
|
||||
expect(tmdb.callsTo(/\/3\/search\//).length).toBe(searches);
|
||||
// 查无此项 → 负缓存 {id: 0},TTL = 空结果短 TTL(30 分钟)
|
||||
const none = await release("Zzqxv.Nonexistent.Title.2019.1080p.WEB-DL-NOGRP");
|
||||
await resolveRelease(h.rt, { sourceReleaseId: none.id });
|
||||
const neg = (await rows()).find((r) => r.key === "[未知][zh-CN]Zzqxv Nonexistent Title-2019-None");
|
||||
expect(neg?.value).toEqual({ id: 0 });
|
||||
expect(Number(neg!.minutes)).toBeLessThanOrEqual(30);
|
||||
const s2 = tmdb.callsTo(/\/3\/search\//).length;
|
||||
await resolveRelease(h.rt, { sourceReleaseId: none.id });
|
||||
expect(tmdb.callsTo(/\/3\/search\//).length).toBe(s2);
|
||||
// force:不读识别缓存(回源结果由请求层缓存兜住),照常回写识别缓存
|
||||
const at = async () =>
|
||||
(
|
||||
await sql<{ t: Date }>`select updated_at as t from kv_cache
|
||||
where region = 'tmdb.recognize' and key = ${neg!.key}`.execute(h.db)
|
||||
).rows[0]!.t.getTime();
|
||||
const before = await at();
|
||||
await new Promise((r) => setTimeout(r, 10));
|
||||
await resolveRelease(h.rt, { sourceReleaseId: none.id, force: true });
|
||||
expect(await at()).toBeGreaterThan(before);
|
||||
});
|
||||
});
|
||||
@@ -59,6 +59,8 @@ export interface MetadataCandidate {
|
||||
overview: string | null;
|
||||
/** TMDB poster_path(如 `/abc.jpg`),拼图片 CDN 用;没有为 null */
|
||||
posterPath: string | null;
|
||||
/** TMDB backdrop_path;没有为 null(旧数据 / 测试构造可缺省) */
|
||||
backdropPath?: string | null;
|
||||
seasons?: CandidateSeason[];
|
||||
externalIds?: { imdb?: string; tvdb?: string };
|
||||
/** 是否已拉过详情(别名/季信息只在详情中完整) */
|
||||
@@ -98,12 +100,30 @@ export interface TmdbConfig {
|
||||
minIntervalMs?: number;
|
||||
/** 默认 max 1024、ttl 24h;空结果(搜索 0 条)用 emptyTtlMs,默认 30min(对照 MP EMPTY_RESULT_CACHE_TTL) */
|
||||
cache?: { max?: number; ttlMs?: number; emptyTtlMs?: number };
|
||||
/**
|
||||
* 跨进程共享的请求缓存(服务端传 PG 缓存);给了就不再用进程内 LRU。
|
||||
* 语义对照 MP @cached(skip_none、业务失败不缓存、results 为空按 emptyTtlMs)。
|
||||
*/
|
||||
sharedCache?: TmdbSharedCache;
|
||||
fetch?: typeof fetch;
|
||||
now?: () => number;
|
||||
}
|
||||
|
||||
type Json = Record<string, unknown>;
|
||||
|
||||
/**
|
||||
* 共享请求缓存端口(对照 MP TMDb.request 的 @cached;服务端用 PG 实现):key 为 `path?query`(不含凭证);
|
||||
* get 未命中返回 null;set 的 ttlMs 由客户端按 ttl / 空结果短 TTL 决定。
|
||||
*/
|
||||
export interface TmdbSharedCache {
|
||||
get(key: string): Promise<unknown | null>;
|
||||
set(key: string, value: unknown, ttlMs: number): Promise<void>;
|
||||
}
|
||||
|
||||
/** 空结果(搜索 0 条)用短 TTL(对照 MP EMPTY_RESULT_CACHE_TTL) */
|
||||
const isEmptyResult = (v: unknown) =>
|
||||
Array.isArray((v as Json | null)?.results) && ((v as Json).results as unknown[]).length === 0;
|
||||
|
||||
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
||||
const s = (v: unknown): string | null => (typeof v === "string" && v !== "" ? v : null);
|
||||
const n = (v: unknown): number | null => (typeof v === "number" && Number.isFinite(v) ? v : null);
|
||||
@@ -130,6 +150,11 @@ export class TmdbClient {
|
||||
return redactUrl(text, [this.cfg.apiKey, this.cfg.accessToken]);
|
||||
}
|
||||
|
||||
/** 首选查询语言(识别缓存键里的语言段,对照 MP TMDB_LOCALE) */
|
||||
get language(): string {
|
||||
return this.langs[0]!;
|
||||
}
|
||||
|
||||
cacheStats() {
|
||||
return { size: this.cache.size, hits: this.cache.hits, misses: this.cache.misses, coalesced: this.cache.coalesced };
|
||||
}
|
||||
@@ -139,13 +164,21 @@ export class TmdbClient {
|
||||
for (const k of Object.keys(params).sort()) if (params[k] !== undefined) qs.set(k, params[k]!);
|
||||
const key = `${path}?${qs}`;
|
||||
const emptyTtl = this.cfg.cache?.emptyTtlMs ?? 30 * 60_000;
|
||||
if (this.cfg.sharedCache) {
|
||||
const shared = this.cfg.sharedCache;
|
||||
const hit = await shared.get(key);
|
||||
if (hit !== null && hit !== undefined) return hit as T;
|
||||
// 网络错误 / 业务失败在 fetchWithRetry 抛出,不会走到写缓存(MP skip_none / skip_if)
|
||||
const v = await this.limiter.run(() => this.fetchWithRetry(path, qs));
|
||||
if (v !== null && v !== undefined) {
|
||||
await shared.set(key, v, isEmptyResult(v) ? emptyTtl : (this.cfg.cache?.ttlMs ?? 24 * 3600_000));
|
||||
}
|
||||
return v as T;
|
||||
}
|
||||
return this.cache.getOrLoad(
|
||||
key,
|
||||
() => this.limiter.run(() => this.fetchWithRetry(path, qs)),
|
||||
(v) =>
|
||||
Array.isArray((v as Json | null)?.results) && ((v as Json).results as unknown[]).length === 0
|
||||
? emptyTtl
|
||||
: undefined,
|
||||
(v) => (isEmptyResult(v) ? emptyTtl : undefined),
|
||||
) as Promise<T>;
|
||||
}
|
||||
|
||||
@@ -269,6 +302,7 @@ export class TmdbClient {
|
||||
popularity: n(r.popularity),
|
||||
overview: s(r.overview),
|
||||
posterPath: s(r.poster_path),
|
||||
backdropPath: s(r.backdrop_path),
|
||||
detailed: false,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -110,6 +110,31 @@ describe("TmdbClient 缓存/限流/错误分类", () => {
|
||||
expect(mock.calls.length).toBe(n1 * 2);
|
||||
});
|
||||
|
||||
it("共享响应缓存:给了 sharedCache 就走它,key 不含凭证,空结果 30 分钟,其余按 ttlMs", async () => {
|
||||
const store = new Map<string, { v: unknown; ttl: number }>();
|
||||
const { mock, tmdb } = await setup({
|
||||
cache: { ttlMs: 7 * 3600_000 },
|
||||
sharedCache: {
|
||||
get: async (key) => store.get(key)?.v ?? null,
|
||||
set: async (key, v, ttl) => {
|
||||
store.set(key, { v, ttl });
|
||||
},
|
||||
},
|
||||
});
|
||||
await tmdb.details("movie", 872585);
|
||||
const n1 = mock.calls.length;
|
||||
await tmdb.details("movie", 872585);
|
||||
expect(mock.calls.length).toBe(n1);
|
||||
await tmdb.search("不存在的片名xyz", { type: "movie" });
|
||||
const keys = [...store.keys()];
|
||||
expect(keys.some((k) => k.includes("test-tmdb-key"))).toBe(false);
|
||||
const detailKey = keys.find((k) => k.startsWith("/movie/872585?"))!;
|
||||
expect(store.get(detailKey)!.ttl).toBe(7 * 3600_000);
|
||||
const emptyKey = keys.find((k) => k.startsWith("/search/movie?"))!;
|
||||
expect(store.get(emptyKey)!.ttl).toBe(30 * 60_000);
|
||||
expect(tmdb.cacheStats().size).toBe(0);
|
||||
});
|
||||
|
||||
it("并发合并:同一请求并发只发一次", async () => {
|
||||
const { mock, tmdb } = await setup();
|
||||
await Promise.all(Array.from({ length: 5 }, () => tmdb.season(100088, 1)));
|
||||
|
||||
@@ -189,6 +189,7 @@ export const EntityMetadataField = z.enum([
|
||||
"originalLanguage",
|
||||
"originCountry",
|
||||
"posterPath",
|
||||
"backdropPath",
|
||||
]);
|
||||
export type EntityMetadataField = z.infer<typeof EntityMetadataField>;
|
||||
|
||||
|
||||
@@ -52,6 +52,8 @@ export const TmdbCandidate = z.object({
|
||||
year: z.number().int().nullable(),
|
||||
overview: z.string().nullable(),
|
||||
popularity: z.number().nullable(),
|
||||
/** TMDB poster_path;前端经图片代理显示海报 */
|
||||
posterPath: z.string().nullable().optional(),
|
||||
/** 已有本地作品接受了这个 TMDB 身份时给出(跟随合并跳转),前端显示"查看"而不是"订阅" */
|
||||
localEntityId: Id.nullable(),
|
||||
});
|
||||
|
||||
@@ -24,6 +24,16 @@ const CacheShape = z.object({
|
||||
checkIntervalSec: z.number().int().min(10).max(3600),
|
||||
/** 缓存清理的目标使用率(低水位) */
|
||||
cleanupTargetPercent: Percent,
|
||||
/** TMDB 缓存(请求响应 + 识别结果)保留小时数(对照 MP META_CACHE_EXPIRE,缺省 24) */
|
||||
tmdbTtlHours: z.number().int().min(1).max(720),
|
||||
/** TMDB 缓存每类条数上限,超出删最旧(对照 MP CONF.tmdb,缺省 256;大内存模式 1024) */
|
||||
tmdbMaxSize: z.number().int().min(16).max(100_000),
|
||||
/** 空结果 / 未识别的短缓存秒数(对照 MP EMPTY_RESULT_CACHE_TTL,缺省 1800) */
|
||||
emptyResultTtlSec: z.number().int().min(0).max(86_400),
|
||||
/** 图片代理落盘缓存(对照 MP GLOBAL_IMAGE_CACHE,缺省关闭;关闭时仍代理,只是不落盘) */
|
||||
globalImageCache: z.boolean(),
|
||||
/** 图片缓存保留天数,超过即清理(对照 MP GLOBAL_IMAGE_CACHE_DAYS,缺省 7) */
|
||||
globalImageCacheDays: z.number().int().min(1).max(3650),
|
||||
});
|
||||
/** 下载缓存卷硬阈值与恢复水位(REQ §8.4.1) */
|
||||
export const CacheSettings = CacheShape.refine((v) => v.resumePercent < v.hardThresholdPercent, {
|
||||
@@ -330,7 +340,17 @@ export type SettingsKey = z.infer<typeof SettingsKey>;
|
||||
/** 缺省设置(settings 表缺 key 或缺字段时使用) */
|
||||
export const DEFAULT_SETTINGS: GlobalSettings = {
|
||||
automation: { enabled: true, readOnlyReconcile: false },
|
||||
cache: { hardThresholdPercent: 90, resumePercent: 85, checkIntervalSec: 60, cleanupTargetPercent: 80 },
|
||||
cache: {
|
||||
hardThresholdPercent: 90,
|
||||
resumePercent: 85,
|
||||
checkIntervalSec: 60,
|
||||
cleanupTargetPercent: 80,
|
||||
tmdbTtlHours: 24,
|
||||
tmdbMaxSize: 256,
|
||||
emptyResultTtlSec: 1800,
|
||||
globalImageCache: false,
|
||||
globalImageCacheDays: 7,
|
||||
},
|
||||
organize: { defaultMode: "copy", allowCrossVolumeMove: true, minSizeBytes: 50 * 1024 * 1024 },
|
||||
retention: { seedRetentionDays: 7, logRetentionDays: 180 },
|
||||
pi: {
|
||||
@@ -587,10 +607,22 @@ export const SETTINGS_SECTIONS: readonly SettingsSectionDef[] = [
|
||||
{
|
||||
id: "metadata",
|
||||
label: "元数据",
|
||||
description: "TMDB API Key 与地址",
|
||||
description: "TMDB API Key 与地址、TMDB 缓存、图片代理",
|
||||
legacyTabs: [],
|
||||
anchors: [{ id: "tmdb", label: "TMDB" }],
|
||||
fields: ["metadata.tmdbApiKeySecretRef", "metadata.tmdbBaseUrl"],
|
||||
anchors: [
|
||||
{ id: "tmdb", label: "TMDB" },
|
||||
{ id: "tmdb-cache", label: "TMDB 缓存" },
|
||||
{ id: "images", label: "图片代理与缓存" },
|
||||
],
|
||||
fields: [
|
||||
"metadata.tmdbApiKeySecretRef",
|
||||
"metadata.tmdbBaseUrl",
|
||||
"cache.tmdbTtlHours",
|
||||
"cache.tmdbMaxSize",
|
||||
"cache.emptyResultTtlSec",
|
||||
"cache.globalImageCache",
|
||||
"cache.globalImageCacheDays",
|
||||
],
|
||||
resources: [],
|
||||
},
|
||||
{
|
||||
|
||||
@@ -130,6 +130,9 @@ const INTERNAL_TABLES = [
|
||||
"SseEvent",
|
||||
// W9-C 命名模板版本登记(0032):服务端内部,经 /naming/* 暴露
|
||||
"NamingTemplateVersion",
|
||||
// C2-A 跨进程限流桶(0041)与 PG 缓存(0042):服务端内部,缓存经 /cache/* 的专用 DTO 暴露
|
||||
"RateLimitBucket",
|
||||
"KvCache",
|
||||
// W9-I 进程心跳(0034):服务端内部,经 /system/health 的专用 DTO 暴露
|
||||
"ProcessHeartbeat",
|
||||
"ProcessHeartbeatSample",
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
import type { Kysely } from "kysely";
|
||||
import { dropTables, execDdl } from "../src/migration-helpers.ts";
|
||||
|
||||
// C2-A 缓存:移植 MP app/runtime/cache.py 的 TTLCache(cachetools TLRUCache:按 region 分区、每个 key 独立 TTL、
|
||||
// maxsize 超限先删过期再删最久未用),存储从进程内存换成 PG UNLOGGED 表(不写 WAL,崩溃后清空,正合缓存语义),三进程共享。
|
||||
// expires_at 过期即视为未命中;accessed_at 为最近读写时间(LRU 依据);定时任务 cache.sweep 每 10 分钟删过期行。
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型
|
||||
export async function up(db: Kysely<any>): Promise<void> {
|
||||
await execDdl(
|
||||
db,
|
||||
`
|
||||
create unlogged table kv_cache (
|
||||
region text not null,
|
||||
key text not null,
|
||||
value jsonb not null,
|
||||
expires_at timestamptz not null,
|
||||
updated_at timestamptz not null default now(),
|
||||
accessed_at timestamptz not null default now(),
|
||||
primary key (region, key)
|
||||
);
|
||||
create index kv_cache_expires_idx on kv_cache (expires_at);
|
||||
`,
|
||||
);
|
||||
}
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 同上
|
||||
export async function down(db: Kysely<any>): Promise<void> {
|
||||
await execDdl(db, dropTables(["kv_cache"]));
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
import { type Kysely, sql } from "kysely";
|
||||
import type { DB } from "./types.ts";
|
||||
|
||||
/**
|
||||
* PG 缓存后端:移植 MP app/runtime/cache.py 的内存后端(MemoryBackend + _MemoryTLRUCache)与 @cached 装饰器语义,
|
||||
* 只把存储从进程内存换成 PG UNLOGGED 表 kv_cache(迁移 0042),三进程共享。行为对照(docs/porting/moviepilot-tmdb-cache.md):
|
||||
* - region 分区;set 时每个 key 独立 TTL(秒),ttl ≤ 0 = 删除该 key;未给 ttl 用后端缺省(DEFAULT_CACHE_TTL 365 天);
|
||||
* - maxsize 按 region 计条数,超出时先删已过期、再删最久未用(TLRU);get 命中刷新"最近使用";
|
||||
* - get 未命中 / 已过期返回 null;clear(region) 清一个区;items(region) 列出未过期条目;
|
||||
* - cached():skipNone / skipEmpty / skipIf 不写缓存,emptyTtl + emptyIf 空结果短 TTL,fresh 跳过读缓存。
|
||||
* 与 MP 的差异只在存储介质:值用 JSON(jsonb)而不是 pickle;过期行由定时任务 sweep() 物理删除。
|
||||
*/
|
||||
|
||||
/** MP DEFAULT_CACHE_TTL(秒) */
|
||||
export const DEFAULT_CACHE_TTL = 365 * 24 * 60 * 60;
|
||||
/** MP DEFAULT_CACHE_SIZE */
|
||||
export const DEFAULT_CACHE_SIZE = 1024;
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 测试会传入隔离 schema 的实例
|
||||
type AnyDb = Kysely<DB> | Kysely<any>;
|
||||
|
||||
export interface SetOptions {
|
||||
/** 秒;缺省 DEFAULT_CACHE_TTL;≤ 0 删除该 key */
|
||||
ttl?: number;
|
||||
/** region 条数上限;缺省 DEFAULT_CACHE_SIZE */
|
||||
maxsize?: number;
|
||||
}
|
||||
|
||||
export class PgCache {
|
||||
constructor(private readonly db: AnyDb) {}
|
||||
|
||||
/** MemoryBackend.set */
|
||||
async set(region: string, key: string, value: unknown, opts: SetOptions = {}): Promise<void> {
|
||||
const ttl = opts.ttl ?? DEFAULT_CACHE_TTL;
|
||||
if (ttl <= 0) {
|
||||
await this.delete(region, key);
|
||||
return;
|
||||
}
|
||||
await sql`insert into kv_cache (region, key, value, expires_at, updated_at, accessed_at)
|
||||
values (${region}, ${key}, ${JSON.stringify(value ?? null)}::jsonb, now() + make_interval(secs => ${ttl}), now(), now())
|
||||
on conflict (region, key) do update set value = excluded.value, expires_at = excluded.expires_at,
|
||||
updated_at = excluded.updated_at, accessed_at = excluded.accessed_at`.execute(this.db);
|
||||
await this.evict(region, opts.maxsize ?? DEFAULT_CACHE_SIZE);
|
||||
}
|
||||
|
||||
/** TLRUCache 写入时的淘汰:先删已过期,再按最近使用从旧到新删到 maxsize */
|
||||
private async evict(region: string, maxsize: number): Promise<void> {
|
||||
await sql`delete from kv_cache where region = ${region} and expires_at <= now()`.execute(this.db);
|
||||
await sql`delete from kv_cache where region = ${region} and key in (
|
||||
select key from kv_cache where region = ${region}
|
||||
order by accessed_at desc, updated_at desc offset ${Math.max(1, Math.floor(maxsize))})`.execute(this.db);
|
||||
}
|
||||
|
||||
/** MemoryBackend.get:未命中 / 过期返回 null;命中刷新最近使用时间 */
|
||||
async get<T = unknown>(region: string, key: string): Promise<T | null> {
|
||||
const r = await sql<{ value: T }>`update kv_cache set accessed_at = now()
|
||||
where region = ${region} and key = ${key} and expires_at > now() returning value`.execute(this.db);
|
||||
return r.rows.length ? (r.rows[0]!.value as T) : null;
|
||||
}
|
||||
|
||||
async exists(region: string, key: string): Promise<boolean> {
|
||||
const r =
|
||||
await sql`select 1 from kv_cache where region = ${region} and key = ${key} and expires_at > now()`.execute(
|
||||
this.db,
|
||||
);
|
||||
return r.rows.length > 0;
|
||||
}
|
||||
|
||||
async delete(region: string, key: string): Promise<void> {
|
||||
await sql`delete from kv_cache where region = ${region} and key = ${key}`.execute(this.db);
|
||||
}
|
||||
|
||||
/** 清一个区,返回删除行数 */
|
||||
async clear(region: string): Promise<number> {
|
||||
const r = await sql`delete from kv_cache where region = ${region}`.execute(this.db);
|
||||
return Number(r.numAffectedRows ?? 0);
|
||||
}
|
||||
|
||||
/** MemoryBackend.items:未过期条目(附过期时间与大小,管理界面用),按 key 排序 */
|
||||
async items<T = unknown>(
|
||||
region: string,
|
||||
limit = 5000,
|
||||
): Promise<{ key: string; value: T; expiresAt: Date; updatedAt: Date; bytes: number }[]> {
|
||||
const r = await sql<{ key: string; value: T; expires_at: Date; updated_at: Date; bytes: number }>`
|
||||
select key, value, expires_at, updated_at, pg_column_size(value)::int as bytes
|
||||
from kv_cache where region = ${region} and expires_at > now() order by key limit ${limit}`.execute(this.db);
|
||||
return r.rows.map((x) => ({
|
||||
key: x.key,
|
||||
value: x.value,
|
||||
expiresAt: x.expires_at,
|
||||
updatedAt: x.updated_at,
|
||||
bytes: Number(x.bytes),
|
||||
}));
|
||||
}
|
||||
|
||||
/** 未过期条目数(len(region_cache)) */
|
||||
async count(region: string): Promise<number> {
|
||||
const r = await sql<{ n: number }>`select count(*)::int as n from kv_cache
|
||||
where region = ${region} and expires_at > now()`.execute(this.db);
|
||||
return Number(r.rows[0]?.n ?? 0);
|
||||
}
|
||||
|
||||
/** 删除全部过期行(存储换成 PG 后需要的物理清理;cache.sweep 定时任务) */
|
||||
async sweep(): Promise<number> {
|
||||
const r = await sql`delete from kv_cache where expires_at <= now()`.execute(this.db);
|
||||
return Number(r.numAffectedRows ?? 0);
|
||||
}
|
||||
}
|
||||
|
||||
/** MP @cached 的参数 */
|
||||
export interface CachedOptions<T> {
|
||||
region: string;
|
||||
maxsize?: number;
|
||||
/** 秒 */
|
||||
ttl?: number;
|
||||
/** 缺省 true:结果为 null / undefined 不缓存 */
|
||||
skipNone?: boolean;
|
||||
/** 缺省 false:结果为假值(空数组 / 空对象 / 空串…)不缓存 */
|
||||
skipEmpty?: boolean;
|
||||
/** 返回真值时不缓存(如业务失败的响应) */
|
||||
skipIf?: (v: T) => boolean;
|
||||
/** 空结果的独立 TTL(秒) */
|
||||
emptyTtl?: number;
|
||||
/** 判断空结果;缺省按假值 */
|
||||
emptyIf?: (v: T) => boolean;
|
||||
/** MP fresh():true 时不读缓存(仍写) */
|
||||
fresh?: boolean;
|
||||
}
|
||||
|
||||
const isFalsy = (v: unknown) =>
|
||||
!v || (Array.isArray(v) && v.length === 0) || (typeof v === "object" && Object.keys(v as object).length === 0);
|
||||
|
||||
/** MP @cached 的调用体:命中直接返回;否则执行 fn,按 should_cache / get_cache_ttl 写入 */
|
||||
export async function cachedCall<T>(
|
||||
cache: PgCache,
|
||||
key: string,
|
||||
o: CachedOptions<T>,
|
||||
fn: () => Promise<T>,
|
||||
): Promise<T> {
|
||||
const should = (v: T | null | undefined) => {
|
||||
if ((o.skipNone ?? true) && (v === null || v === undefined)) return false;
|
||||
if (o.skipEmpty && isFalsy(v)) return false;
|
||||
if (o.skipIf && v !== null && v !== undefined && o.skipIf(v)) return false;
|
||||
return true;
|
||||
};
|
||||
if (!o.fresh) {
|
||||
const hit = await cache.get<T>(o.region, key);
|
||||
if (should(hit)) return hit as T;
|
||||
}
|
||||
const result = await fn();
|
||||
if (!should(result)) return result;
|
||||
let ttl = o.ttl;
|
||||
if (o.emptyTtl !== undefined) {
|
||||
if (result === null || result === undefined) ttl = o.emptyTtl;
|
||||
else if (o.emptyIf) ttl = o.emptyIf(result) ? o.emptyTtl : o.ttl;
|
||||
else ttl = isFalsy(result) ? o.emptyTtl : o.ttl;
|
||||
}
|
||||
await cache.set(o.region, key, result, {
|
||||
...(ttl !== undefined ? { ttl } : {}),
|
||||
...(o.maxsize ? { maxsize: o.maxsize } : {}),
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
const caches = new WeakMap<object, PgCache>();
|
||||
|
||||
/** 同一个 Kysely 实例共用一个 PgCache */
|
||||
export function pgCacheFor(db: AnyDb): PgCache {
|
||||
let c = caches.get(db);
|
||||
if (!c) {
|
||||
c = new PgCache(db);
|
||||
caches.set(db, c);
|
||||
}
|
||||
return c;
|
||||
}
|
||||
@@ -1,5 +1,14 @@
|
||||
// 统一从本包取 Kysely 与 sql,避免各包各装一份 kysely
|
||||
export { type Kysely, sql, type Transaction } from "kysely";
|
||||
export {
|
||||
type CachedOptions,
|
||||
cachedCall,
|
||||
DEFAULT_CACHE_SIZE,
|
||||
DEFAULT_CACHE_TTL,
|
||||
PgCache,
|
||||
pgCacheFor,
|
||||
type SetOptions as CacheSetOptions,
|
||||
} from "./cache.ts";
|
||||
export { type CreateDbOptions, createDb } from "./client.ts";
|
||||
export { createMigrator, MIGRATIONS_DIR, type MigratorOptions, migrateToLatest, resetSchema } from "./migrator.ts";
|
||||
export {
|
||||
|
||||
@@ -827,6 +827,15 @@ export interface IdentityRevisionTable {
|
||||
updated_at: GeneratedTimestamptz;
|
||||
}
|
||||
|
||||
export interface KvCacheTable {
|
||||
region: string;
|
||||
key: string;
|
||||
value: Json;
|
||||
expires_at: Timestamptz;
|
||||
updated_at: GeneratedTimestamptz;
|
||||
accessed_at: GeneratedTimestamptz;
|
||||
}
|
||||
|
||||
export interface LibraryRootTable {
|
||||
id: Generated<string>;
|
||||
name: string;
|
||||
@@ -1642,6 +1651,7 @@ export interface DB {
|
||||
identity_candidate: IdentityCandidateTable;
|
||||
identity_redirect: IdentityRedirectTable;
|
||||
identity_revision: IdentityRevisionTable;
|
||||
kv_cache: KvCacheTable;
|
||||
library_root: LibraryRootTable;
|
||||
login_attempt: LoginAttemptTable;
|
||||
mcp_server_config: McpServerConfigTable;
|
||||
@@ -1787,6 +1797,9 @@ export type IdentityRedirectUpdate = Updateable<IdentityRedirectTable>;
|
||||
export type IdentityRevisionRow = Selectable<IdentityRevisionTable>;
|
||||
export type NewIdentityRevision = Insertable<IdentityRevisionTable>;
|
||||
export type IdentityRevisionUpdate = Updateable<IdentityRevisionTable>;
|
||||
export type KvCacheRow = Selectable<KvCacheTable>;
|
||||
export type NewKvCache = Insertable<KvCacheTable>;
|
||||
export type KvCacheUpdate = Updateable<KvCacheTable>;
|
||||
export type LibraryRootRow = Selectable<LibraryRootTable>;
|
||||
export type NewLibraryRoot = Insertable<LibraryRootTable>;
|
||||
export type LibraryRootUpdate = Updateable<LibraryRootTable>;
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
import { Kysely, PostgresDialect, sql } from "kysely";
|
||||
import pg from "pg";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { cachedCall, type DB, migrateToLatest, PgCache } from "../src/index.ts";
|
||||
|
||||
// PG 缓存后端(0042 kv_cache,UNLOGGED;移植 MP runtime/cache.py 的 TTLCache / @cached 语义)。
|
||||
const url = process.env.DATABASE_URL ?? "postgres://umt:umt@localhost:55433/umt";
|
||||
const skip = process.env.UMT_SKIP_DB_TESTS === "1";
|
||||
const schema = `umt_cache_${process.pid}_${Date.now().toString(36)}`;
|
||||
|
||||
describe.skipIf(skip)("PgCache(真实 PG)", () => {
|
||||
let admin: Kysely<unknown>;
|
||||
let db: Kysely<DB>;
|
||||
|
||||
beforeAll(async () => {
|
||||
admin = new Kysely({ dialect: new PostgresDialect({ pool: new pg.Pool({ connectionString: url, max: 1 }) }) });
|
||||
await sql.raw(`create schema ${schema}`).execute(admin);
|
||||
db = new Kysely<DB>({
|
||||
dialect: new PostgresDialect({
|
||||
pool: new pg.Pool({ connectionString: url, max: 4, options: `-c search_path=${schema},public` }),
|
||||
}),
|
||||
});
|
||||
await migrateToLatest(db, { migrationTableSchema: schema });
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await db?.destroy();
|
||||
await sql.raw(`drop schema if exists ${schema} cascade`).execute(admin);
|
||||
await admin?.destroy();
|
||||
});
|
||||
|
||||
it("kv_cache 是 UNLOGGED 表", async () => {
|
||||
const r = await sql<{ p: string }>`select relpersistence as p from pg_class c
|
||||
join pg_namespace n on n.oid = c.relnamespace where n.nspname = ${schema} and c.relname = 'kv_cache'`.execute(db);
|
||||
expect(r.rows[0]?.p).toBe("u");
|
||||
});
|
||||
|
||||
it("set/get 跨实例共享;每个 key 独立 TTL,过期即未命中;ttl ≤ 0 删除;clear / items / count / sweep", async () => {
|
||||
const a = new PgCache(db);
|
||||
const b = new PgCache(db);
|
||||
await a.set("t", "k1", { x: 1, list: [1, 2] }, { ttl: 60 });
|
||||
expect(await b.get("t", "k1")).toEqual({ x: 1, list: [1, 2] });
|
||||
expect(await b.get("t", "nope")).toBeNull();
|
||||
await a.set("t", "short", "v", { ttl: 0.05 });
|
||||
await new Promise((r) => setTimeout(r, 120));
|
||||
expect(await b.get("t", "short")).toBeNull();
|
||||
expect(await a.exists("t", "k1")).toBe(true);
|
||||
await a.set("t", "k1", "x", { ttl: 0 });
|
||||
expect(await a.exists("t", "k1")).toBe(false);
|
||||
await a.set("t", "k2", 2, { ttl: 60 });
|
||||
expect((await a.items("t")).map((i) => i.key)).toEqual(["k2"]);
|
||||
expect(await a.count("t")).toBe(1);
|
||||
await sql`update kv_cache set expires_at = now() - interval '1 second' where region = 't'`.execute(db);
|
||||
expect(await a.sweep()).toBe(1);
|
||||
await a.set("t", "k3", 3, { ttl: 60 });
|
||||
expect(await a.clear("t")).toBe(1);
|
||||
});
|
||||
|
||||
it("maxsize:超限先删过期,再删最久未用(get 命中刷新最近使用,对照 TLRUCache)", async () => {
|
||||
const c = new PgCache(db);
|
||||
const tick = () => new Promise((r) => setTimeout(r, 5));
|
||||
await c.set("cap", "expired", 0, { ttl: 0.01, maxsize: 3 });
|
||||
await tick();
|
||||
await tick();
|
||||
for (const k of ["a", "b", "c"]) {
|
||||
await c.set("cap", k, k, { ttl: 60, maxsize: 3 });
|
||||
await tick();
|
||||
}
|
||||
await c.get("cap", "a"); // a 最近使用
|
||||
await tick();
|
||||
await c.set("cap", "d", "d", { ttl: 60, maxsize: 3 });
|
||||
expect((await c.items("cap")).map((i) => i.key)).toEqual(["a", "c", "d"]);
|
||||
const all = await sql<{ n: number }>`select count(*)::int as n from kv_cache where region = 'cap'`.execute(db);
|
||||
expect(all.rows[0]!.n).toBe(3);
|
||||
});
|
||||
|
||||
it("cachedCall(@cached):skipNone / skipIf 不写;空结果按 emptyTtl;fresh 跳过读", async () => {
|
||||
const c = new PgCache(db);
|
||||
let n = 0;
|
||||
const o = {
|
||||
region: "cc",
|
||||
ttl: 3600,
|
||||
emptyTtl: 1800,
|
||||
emptyIf: (v: { results: unknown[] }) => v.results.length === 0,
|
||||
};
|
||||
const full = () =>
|
||||
cachedCall(c, "full", o, async () => {
|
||||
n++;
|
||||
return { results: [1] };
|
||||
});
|
||||
await full();
|
||||
await full();
|
||||
expect(n).toBe(1);
|
||||
await cachedCall(c, "empty", o, async () => ({ results: [] }));
|
||||
const ttl = await sql<{ k: string; s: number }>`select key as k, extract(epoch from expires_at - now()) as s
|
||||
from kv_cache where region = 'cc' order by key`.execute(db);
|
||||
expect(Number(ttl.rows.find((r) => r.k === "empty")!.s)).toBeLessThanOrEqual(1800);
|
||||
expect(Number(ttl.rows.find((r) => r.k === "full")!.s)).toBeGreaterThan(1800);
|
||||
await cachedCall(c, "none", o, async () => null as unknown as { results: unknown[] });
|
||||
await cachedCall(c, "fail", { ...o, skipIf: () => true }, async () => ({ results: [1] }));
|
||||
expect(await c.exists("cc", "none")).toBe(false);
|
||||
expect(await c.exists("cc", "fail")).toBe(false);
|
||||
await cachedCall(c, "full", { ...o, fresh: true }, async () => {
|
||||
n++;
|
||||
return { results: [2] };
|
||||
});
|
||||
expect(n).toBe(2);
|
||||
expect(await c.get("cc", "full")).toEqual({ results: [2] });
|
||||
});
|
||||
});
|
||||
@@ -117,6 +117,8 @@ const EXPECTED_TABLES = [
|
||||
"webauthn_challenge",
|
||||
// C2-A 跨进程站点限流(0041)
|
||||
"rate_limit_bucket",
|
||||
// C2-A PG 短期缓存(0042,UNLOGGED)
|
||||
"kv_cache",
|
||||
].sort();
|
||||
|
||||
/** 不走 entityTable() 的表:追加型/引擎自有表,没有 uuid id + created_at/updated_at 三件套 */
|
||||
@@ -143,6 +145,8 @@ const NON_ENTITY_TABLES = [
|
||||
"webauthn_challenge",
|
||||
// 限流桶:key 主键,只有 updated_at
|
||||
"rate_limit_bucket",
|
||||
// 缓存:(region, key) 主键
|
||||
"kv_cache",
|
||||
];
|
||||
|
||||
describe.skipIf(skip)("schema migrations (real PG)", () => {
|
||||
|
||||
Reference in new issue
Block a user