merge: feat/y-meta-cache
Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei
This commit is contained in:
rootandClaude Opus 5.5 committed
commit
6d37a337c9
51 files changed
+2402
-195
No files matched your search
@@ -84,6 +84,7 @@ const { createApp } = await import("./http/app.ts");
|
||||
const { PgEventBus } = await import("./http/sse-pg.ts");
|
||||
const { createLogger } = await import("./logger.ts");
|
||||
const { SERVER_MODULES } = await import("./modules/index.ts");
|
||||
const { startMetadataCache } = await import("./shared-cache.ts");
|
||||
const { diffCounts, enableReadOnlyReconcile, exportMappings, writeMappingExport } = await import("./ops.ts");
|
||||
const { auditStorageWarnings, checkStorage, loadMountedPaths, runPreflight } = await import("./preflight.ts");
|
||||
const { startWorker } = await import("./worker/index.ts");
|
||||
@@ -219,6 +220,8 @@ try {
|
||||
Deno.exit(1);
|
||||
}
|
||||
const { ctx, outbox, engine } = booted;
|
||||
// 统一元数据缓存:LISTEN 跨进程失效(在线才启用进程内一级缓存)+ 命中统计定期落库
|
||||
stops.push(await startMetadataCache(ctx, config.DATABASE_URL));
|
||||
|
||||
if (role === "api" || role === "all") {
|
||||
const app = createApp({
|
||||
|
||||
@@ -152,11 +152,12 @@ export function episodeMetadataMissing(u: UnitForNfo | undefined): boolean {
|
||||
return !u || (!filled(u.metadata) && !str(u.title) && !u.air_date);
|
||||
}
|
||||
|
||||
/** 图片路径:作品海报 / 背景图,季海报,集剧照 */
|
||||
/** 图片路径:作品海报 / 背景图 / 标志,季海报,集剧照 */
|
||||
export function imagePathsOf(entityMeta: unknown) {
|
||||
return {
|
||||
posterPath: str(val(entityMeta, "posterPath")) ?? str(val(entityMeta, "posterUrl")),
|
||||
backdropPath: str(val(entityMeta, "backdropPath")),
|
||||
logoPath: str(val(entityMeta, "logoPath")),
|
||||
};
|
||||
}
|
||||
export const unitPosterPath = (m: unknown) => str(val(m, "posterPath"));
|
||||
|
||||
+36
-8
@@ -14,7 +14,16 @@ import { dataPaths } from "../../config.ts";
|
||||
import { externalFetch } from "../../egress.ts";
|
||||
import { requireScope } from "../../http/auth.ts";
|
||||
import { errorBody } from "../../http/errors.ts";
|
||||
import { CACHE_REGIONS, sharedCache, storageSettings, tmdbCachePolicy } from "../../shared-cache.ts";
|
||||
import {
|
||||
CACHE_REGIONS,
|
||||
METADATA_REGIONS,
|
||||
metadataCache,
|
||||
type RegionHitStats,
|
||||
regionHitStats,
|
||||
sharedCache,
|
||||
storageSettings,
|
||||
tmdbCachePolicy,
|
||||
} from "../../shared-cache.ts";
|
||||
import { registerScheduledJob } from "../schedules/registry.ts";
|
||||
import type { ModuleApp, ServerContext, ServerModule } from "../types.ts";
|
||||
import { ImageCache, ImageProxyError } from "./images.ts";
|
||||
@@ -72,6 +81,13 @@ function route<D extends EndpointDef>(
|
||||
|
||||
const str = (v: unknown) => (typeof v === "string" && v ? v : null);
|
||||
|
||||
/** 命中统计 → 接口形状;命中率 = (L1 + L2 + 旧值) / (L1 + L2 + 旧值 + 未命中) */
|
||||
function hitView(h: RegionHitStats | undefined) {
|
||||
const v = h ?? { l1Hits: 0, l2Hits: 0, staleHits: 0, misses: 0, failures: 0, refreshes: 0 };
|
||||
const hit = v.l1Hits + v.l2Hits + v.staleHits;
|
||||
return { ...v, hitRate: hit + v.misses > 0 ? hit / (hit + v.misses) : null };
|
||||
}
|
||||
|
||||
export function createCacheModule(opts: CacheModuleOptions = {}): ServerModule & {
|
||||
images(ctx: ServerContext): ImageCache;
|
||||
} {
|
||||
@@ -166,7 +182,7 @@ export function createCacheModule(opts: CacheModuleOptions = {}): ServerModule &
|
||||
return out;
|
||||
});
|
||||
route(app, Api.cache.tmdbClear, async () => {
|
||||
const cache = sharedCache(ctx);
|
||||
const cache = metadataCache(ctx);
|
||||
const deleted =
|
||||
(await cache.clear(CACHE_REGIONS.tmdbRecognize)) + (await cache.clear(CACHE_REGIONS.tmdbRequest));
|
||||
ctx.logger.info({ deleted }, "TMDB 缓存已清空");
|
||||
@@ -194,18 +210,30 @@ export function createCacheModule(opts: CacheModuleOptions = {}): ServerModule &
|
||||
coalesce(sum(pg_column_size(value) + pg_column_size(key)), 0)::text as bytes,
|
||||
max(updated_at) as last
|
||||
from kv_cache group by region order by region`.execute(ctx.db);
|
||||
return {
|
||||
items: r.rows.map((x) => ({
|
||||
// 统一元数据缓存的 region 带最近 7 天命中统计(口径见 contracts CacheHitStats)
|
||||
const hits = await regionHitStats(ctx).catch(() => new Map<string, RegionHitStats>());
|
||||
const withHits = (region: string) => {
|
||||
const h = hits.get(region);
|
||||
if (!h) return METADATA_REGIONS.includes(region) ? { hits: hitView(undefined) } : {};
|
||||
return { hits: hitView(h) };
|
||||
};
|
||||
const items = r.rows.map((x) => ({
|
||||
region: x.region,
|
||||
entries: Number(x.entries),
|
||||
expired: Number(x.expired),
|
||||
bytes: Number(x.bytes),
|
||||
lastWriteAt: x.last ? new Date(x.last).toISOString() : null,
|
||||
})),
|
||||
};
|
||||
...withHits(x.region),
|
||||
}));
|
||||
// 条目已清空、但最近有读取统计的元数据 region 也列出来
|
||||
for (const region of hits.keys())
|
||||
if (!items.some((i) => i.region === region))
|
||||
items.push({ region, entries: 0, expired: 0, bytes: 0, lastWriteAt: null, ...withHits(region) });
|
||||
items.sort((a, b) => a.region.localeCompare(b.region));
|
||||
return { items };
|
||||
});
|
||||
route(app, Api.cache.regionClear, async ({ params }) => {
|
||||
const deleted = await sharedCache(ctx).clear(params.region);
|
||||
const deleted = await metadataCache(ctx).clear(params.region);
|
||||
ctx.logger.info({ region: params.region, deleted }, "缓存区已清空");
|
||||
return { deleted };
|
||||
});
|
||||
@@ -217,7 +245,7 @@ export function createCacheModule(opts: CacheModuleOptions = {}): ServerModule &
|
||||
lane: "maintenance",
|
||||
defaults: {
|
||||
name: "缓存清理",
|
||||
description: "删除过期的 TMDB 与识别缓存",
|
||||
description: "删除过期的元数据与其它缓存",
|
||||
group: "maintain",
|
||||
schedule: { type: "interval", everyMs: 10 * 60_000 },
|
||||
firstRun: "after_interval",
|
||||
|
||||
@@ -24,10 +24,10 @@ import {
|
||||
type DiscoverTmdbQuery,
|
||||
type InventoryEntry,
|
||||
} from "@umt/contracts";
|
||||
import { pgCacheFor } from "@umt/db";
|
||||
import { type IdentityCandidate, parseRelease, resolveIdentity } from "@umt/domain";
|
||||
import { ApiException } from "../../http/errors.ts";
|
||||
import { runSandbox } from "../../sandbox/client.ts";
|
||||
import { CACHE_REGIONS, metaCachePolicy, metadataCache, storageSettings } from "../../shared-cache.ts";
|
||||
import type { SourcesRuntime } from "../sources-identity/runtime.ts";
|
||||
import { subjectRecall } from "../sources-identity/service/id-escalation.ts";
|
||||
import { metaClients, toCached } from "../sources-identity/service/meta-subjects.ts";
|
||||
@@ -41,9 +41,8 @@ import { localEntities } from "./media.ts";
|
||||
// - 每条结果按 TMDB 身份标本地状态(已订阅 / 媒体库已有 / 下载中);豆瓣 / bgm 条目匹配过 TMDB 的才知道(discover.match)。
|
||||
// 本地状态每次现查,不进缓存。
|
||||
|
||||
/** 豆瓣 / bgm 条目 → TMDB 作品的确定匹配(点开时写入,列表据此标本地状态) */
|
||||
const MATCH_REGION = "discover.match";
|
||||
const MATCH_TTL_SEC = 30 * 86_400;
|
||||
/** 豆瓣 / bgm 条目 → TMDB 作品的确定匹配(点开时写入,列表据此标本地状态);TTL 按「豆瓣 / bgm × 详情」 */
|
||||
const MATCH_REGION = CACHE_REGIONS.discoverMatch;
|
||||
const MATCH_MAX = 20_000;
|
||||
|
||||
export type InventoryLookup = (ids: string[]) => Promise<Map<string, InventoryEntry>>;
|
||||
@@ -108,12 +107,12 @@ const fromTmdb = (t: TmdbListItem): DiscoverItem => ({
|
||||
|
||||
/** 豆瓣 / bgm 条目补上记住的 TMDB 匹配,再按 TMDB 身份查本地作品与库存 */
|
||||
export async function annotate(rt: SourcesRuntime, items: DiscoverItem[], inventory: InventoryLookup) {
|
||||
const cache = pgCacheFor(rt.ctx.db);
|
||||
const cache = metadataCache(rt.ctx);
|
||||
await Promise.all(
|
||||
items
|
||||
.filter((it) => !it.tmdb && it.source !== "tmdb")
|
||||
.map(async (it) => {
|
||||
const hit = await cache.get<DiscoverMatchCandidate>(MATCH_REGION, `${it.source}:${it.id}`).catch(() => null);
|
||||
const hit = await cache.get<DiscoverMatchCandidate>(MATCH_REGION, `${it.source}:${it.id}`);
|
||||
if (hit) it.tmdb = { type: hit.type, id: hit.id };
|
||||
}),
|
||||
);
|
||||
@@ -257,9 +256,9 @@ const view = (m: MetadataCandidate): DiscoverMatchCandidate => ({
|
||||
});
|
||||
|
||||
export async function matchTmdb(rt: SourcesRuntime, req: DiscoverMatchRequest): Promise<DiscoverMatchResult> {
|
||||
const cache = pgCacheFor(rt.ctx.db);
|
||||
const cache = metadataCache(rt.ctx);
|
||||
const memoKey = `${req.source}:${req.id}`;
|
||||
const memo = await cache.get<DiscoverMatchCandidate>(MATCH_REGION, memoKey).catch(() => null);
|
||||
const memo = await cache.get<DiscoverMatchCandidate>(MATCH_REGION, memoKey);
|
||||
if (memo) return { match: memo, candidates: [memo] };
|
||||
const subject = toCached({ ...req, cover: null, rating: null, date: null, episodes: null, url: "" }, rt.now());
|
||||
const base = parseRelease(req.title);
|
||||
@@ -278,6 +277,9 @@ export async function matchTmdb(rt: SourcesRuntime, req: DiscoverMatchRequest):
|
||||
const best = res.status === "auto" && res.best ? byCand.get(res.best.candidate) : undefined;
|
||||
const match = best ? view(best) : null;
|
||||
// 只记确定的匹配;不确定 / 失败不记,下次重算
|
||||
if (match) await cache.set(MATCH_REGION, memoKey, match, { ttl: MATCH_TTL_SEC, maxsize: MATCH_MAX }).catch(() => {});
|
||||
if (match) {
|
||||
const ttl = metaCachePolicy((await storageSettings(rt.ctx)).cacheTtl, req.source, "detail").ttl;
|
||||
await cache.set(MATCH_REGION, memoKey, match, { ttl, maxsize: MATCH_MAX }).catch(() => {});
|
||||
}
|
||||
return DiscoverMatchResult.parse({ match, candidates: ranked.map(view) });
|
||||
}
|
||||
@@ -11,19 +11,13 @@ import {
|
||||
type SiteDefinition,
|
||||
type SiteRateLimiter,
|
||||
type TmdbClient,
|
||||
type TmdbRequestKind,
|
||||
type TmdbSharedCache,
|
||||
} from "@umt/adapters";
|
||||
import { DEFAULT_SETTINGS, type SitesSettings } from "@umt/contracts";
|
||||
import { pgTokenBucketFor, type SiteRow, sql } from "@umt/db";
|
||||
import { egressKey, externalFetch, siteEgress, siteFetchFor } from "../../egress.ts";
|
||||
import {
|
||||
CACHE_REGIONS,
|
||||
renewTrackedTmdbDetail,
|
||||
sharedCache,
|
||||
type TmdbCachePolicy,
|
||||
tmdbCachePolicy,
|
||||
tmdbDetailRef,
|
||||
} from "../../shared-cache.ts";
|
||||
import { CACHE_REGIONS, metadataCache, type TmdbCachePolicy, tmdbCachePolicy, watchedPin } from "../../shared-cache.ts";
|
||||
import type { ServerContext } from "../types.ts";
|
||||
import { CloudflareGate, withClearance } from "./service/cloudflare.ts";
|
||||
import { failureSampleSink } from "./service/failure-sample.ts";
|
||||
@@ -165,6 +159,8 @@ export class SourcesRuntime {
|
||||
ttlMs: policy.ttl * 1000,
|
||||
searchTtlMs: policy.searchTtl * 1000,
|
||||
emptyTtlMs: policy.emptyTtl * 1000,
|
||||
listTtlMs: policy.listTtl * 1000,
|
||||
imageTtlMs: policy.imageTtl * 1000,
|
||||
},
|
||||
sharedCache: tmdbSharedCache(this.ctx, policy),
|
||||
}
|
||||
@@ -420,28 +416,34 @@ export class SourcesRuntime {
|
||||
}
|
||||
|
||||
/**
|
||||
* TMDB 请求响应缓存:PG kv_cache region=tmdb.request;PG 出错按未命中 / 不写处理,不让 TMDB 请求失败。
|
||||
* 详情命中且属于追踪中的作品时续期(CACHE-2),续期失败只记 debug。
|
||||
* TMDB 请求响应缓存:统一元数据缓存 region=tmdb.request(L1 进程内存 + L2 kv_cache,过期先用旧值后台刷新、同请求合并)。
|
||||
* TTL 按请求类别取策略表(搜索 / 详情 / 列表 / 图片);关注中作品的条目写入 pinned、命中续期。
|
||||
* PG 出错按未命中 / 不写处理,不让 TMDB 请求失败。get / set 保留给不走 fetch 的旧调用方。
|
||||
*/
|
||||
function tmdbSharedCache(ctx: ServerContext, policy: TmdbCachePolicy): TmdbSharedCache {
|
||||
const cache = sharedCache(ctx);
|
||||
const warn = (err: unknown) =>
|
||||
ctx.logger.warn({ err: { message: (err as Error).message } }, "TMDB 缓存读写失败，按未命中处理");
|
||||
export function tmdbSharedCache(ctx: ServerContext, policy: TmdbCachePolicy): TmdbSharedCache {
|
||||
const cache = metadataCache(ctx);
|
||||
const region = CACHE_REGIONS.tmdbRequest;
|
||||
const ttlOf = (kind: TmdbRequestKind) =>
|
||||
kind === "search"
|
||||
? { ttl: policy.searchTtl, emptyTtl: policy.emptyTtl }
|
||||
: kind === "list"
|
||||
? { ttl: policy.listTtl, emptyTtl: policy.listEmptyTtl }
|
||||
: kind === "image"
|
||||
? { ttl: policy.imageTtl, emptyTtl: policy.emptyTtl }
|
||||
: { ttl: policy.ttl, emptyTtl: policy.emptyTtl };
|
||||
return {
|
||||
get: async (key) => {
|
||||
const v = await cache.get(CACHE_REGIONS.tmdbRequest, key).catch((e) => {
|
||||
warn(e);
|
||||
return null;
|
||||
});
|
||||
if (v !== null && tmdbDetailRef(key)) {
|
||||
await renewTrackedTmdbDetail(ctx, key, policy.ttl).catch((e) =>
|
||||
ctx.logger.debug({ err: { message: (e as Error).message } }, "TMDB 详情缓存续期失败"),
|
||||
);
|
||||
}
|
||||
return v;
|
||||
},
|
||||
fetch: (req, load) =>
|
||||
cache.fetch(
|
||||
region,
|
||||
req.key,
|
||||
{ ...ttlOf(req.kind), isEmpty: req.isEmpty, maxsize: policy.maxsize, pin: watchedPin(ctx, req.key) },
|
||||
load,
|
||||
),
|
||||
get: (key) => cache.get(region, key),
|
||||
set: (key, value, ttlMs) =>
|
||||
cache.set(CACHE_REGIONS.tmdbRequest, key, value, { ttl: ttlMs / 1000, maxsize: policy.maxsize }).catch(warn),
|
||||
cache
|
||||
.set(region, key, value, { ttl: ttlMs / 1000, maxsize: policy.maxsize })
|
||||
.catch((e) => ctx.logger.warn({ err: { message: (e as Error).message } }, "TMDB 缓存读写失败，按未命中处理")),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { formatDetailPage } from "@umt/adapters";
|
||||
import { pgCacheFor } from "@umt/db";
|
||||
import { DEFAULT_STORAGE_SETTINGS } from "@umt/contracts";
|
||||
import { metaCachePolicy, metadataCache, storageSettings } from "../../../shared-cache.ts";
|
||||
import type { ServerContext } from "../../types.ts";
|
||||
|
||||
// ID-3 豆瓣条目缓存(kv_cache region douban.subject,key = 豆瓣 subject id):只存标题 / 别名 / 年份,供「详情页只有豆瓣 ID」时
|
||||
@@ -10,8 +11,8 @@ import type { ServerContext } from "../../types.ts";
|
||||
// - douban_api:豆瓣移动版接口(识别升级时缓存没有才查)。bgm.tv 条目同形,存 region bangumi.subject。
|
||||
|
||||
export const DOUBAN_SUBJECT_REGION = "douban.subject";
|
||||
const TTL_SEC = 30 * 86_400;
|
||||
const MAX_ENTRIES = 20_000;
|
||||
/** 条目 TTL 取策略表「豆瓣 / bgm × 详情」(storage.cacheTtl.metaDetailDays,缺省 30 天) */
|
||||
export const META_SUBJECT_MAX_ENTRIES = 20_000;
|
||||
|
||||
export interface DoubanSubject {
|
||||
id: string;
|
||||
@@ -92,16 +93,24 @@ export function doubanSubjectFromSnippet(
|
||||
};
|
||||
}
|
||||
|
||||
type Db = Pick<ServerContext, "db">;
|
||||
type Db = Pick<ServerContext, "db"> & Partial<Pick<ServerContext, "settings" | "logger">>;
|
||||
|
||||
/** 写入缓存;站点详情来源优先(已有 site_detail 条目时不用搜索摘要覆盖) */
|
||||
/** 条目 TTL(秒):有设置读设置,否则缺省 */
|
||||
export async function metaSubjectTtl(ctx: Db): Promise<number> {
|
||||
const t = ctx.settings
|
||||
? (await storageSettings({ settings: ctx.settings })).cacheTtl
|
||||
: DEFAULT_STORAGE_SETTINGS.cacheTtl;
|
||||
return metaCachePolicy(t, "douban", "detail").ttl;
|
||||
}
|
||||
|
||||
/** 写入缓存(统一元数据缓存,两级);站点详情来源优先(已有 site_detail 条目时不用搜索摘要覆盖) */
|
||||
export async function saveDoubanSubject(ctx: Db, s: DoubanSubject, region = DOUBAN_SUBJECT_REGION): Promise<void> {
|
||||
const cache = pgCacheFor(ctx.db);
|
||||
const cache = metadataCache(ctx);
|
||||
if (s.source === "web_snippet") {
|
||||
const prev = await cache.get<DoubanSubject>(region, s.id);
|
||||
if (prev?.source === "site_detail") return;
|
||||
}
|
||||
await cache.set(region, s.id, s, { ttl: TTL_SEC, maxsize: MAX_ENTRIES });
|
||||
await cache.set(region, s.id, s, { ttl: await metaSubjectTtl(ctx), maxsize: META_SUBJECT_MAX_ENTRIES });
|
||||
}
|
||||
|
||||
export async function loadDoubanSubject(
|
||||
@@ -110,7 +119,7 @@ export async function loadDoubanSubject(
|
||||
region = DOUBAN_SUBJECT_REGION,
|
||||
): Promise<DoubanSubject | null> {
|
||||
try {
|
||||
const v = await pgCacheFor(ctx.db).get<DoubanSubject>(region, id);
|
||||
const v = await metadataCache(ctx).get<DoubanSubject>(region, id);
|
||||
return v && typeof v.title === "string" && v.title ? v : null;
|
||||
} catch {
|
||||
return null;
|
||||
|
||||
@@ -4,51 +4,56 @@ import {
|
||||
createDoubanClient,
|
||||
type DoubanClient,
|
||||
type MetaRequestCache,
|
||||
MetaSourceError,
|
||||
type MetaSubject,
|
||||
RequestLimiter,
|
||||
} from "@umt/adapters";
|
||||
import { cachedCall, pgCacheFor } from "@umt/db";
|
||||
import { normalizeTitle, toSimplified } from "@umt/domain";
|
||||
import { storageSettings } from "../../../shared-cache.ts";
|
||||
import { CACHE_REGIONS, metaCachePolicy, metadataCache, storageSettings } from "../../../shared-cache.ts";
|
||||
import type { SourcesRuntime } from "../runtime.ts";
|
||||
import { DOUBAN_SUBJECT_REGION, type DoubanSubject, loadDoubanSubject, saveDoubanSubject } from "./douban-subject.ts";
|
||||
import {
|
||||
DOUBAN_SUBJECT_REGION,
|
||||
type DoubanSubject,
|
||||
loadDoubanSubject,
|
||||
META_SUBJECT_MAX_ENTRIES,
|
||||
} from "./douban-subject.ts";
|
||||
|
||||
// 豆瓣 / bgm.tv 作为辅助信息源(对照 MP:豆瓣、Bangumi 条目 → 名称 + 年份 → TMDB 投影;身份仍以 TMDB 为准)。
|
||||
// 条目先查缓存(30 天),没有才在线查一次;查不到 / 网络失败记 6 小时负缓存,不反复打外站。
|
||||
|
||||
export const BANGUMI_SUBJECT_REGION = "bangumi.subject";
|
||||
const MISS_REGION = "meta.subject_miss";
|
||||
const MISS_TTL_SEC = 6 * 3600;
|
||||
export const BANGUMI_SUBJECT_REGION = CACHE_REGIONS.bangumiSubject;
|
||||
const MISS_REGION = CACHE_REGIONS.subjectMiss;
|
||||
/** 网络失败 / 被拦后同一条目在本进程里冷却多久不再在线查(不写缓存,只是不连着打外站) */
|
||||
const FAIL_COOLDOWN_MS = 30 * 60_000;
|
||||
const failedAt = new Map<string, number>();
|
||||
|
||||
export type MetaSource = "douban" | "bangumi";
|
||||
const REGION: Record<MetaSource, string> = { douban: DOUBAN_SUBJECT_REGION, bangumi: BANGUMI_SUBJECT_REGION };
|
||||
|
||||
/** 豆瓣 / bgm 请求缓存 region(对照 MP DoubanApi.__invoke / BangumiApi 的 @cached);key 见 metaRequestKey */
|
||||
export const META_REQUEST_REGIONS = { douban: "douban.request", bangumi: "bangumi.request" } as const;
|
||||
export const META_REQUEST_REGIONS = {
|
||||
douban: CACHE_REGIONS.doubanRequest,
|
||||
bangumi: CACHE_REGIONS.bangumiRequest,
|
||||
} as const;
|
||||
const META_REQUEST_MAXSIZE = 5000;
|
||||
/** 每源限流(进程内):豆瓣风控严,间隔大一些;缓存命中不占额度 */
|
||||
const META_LIMITERS = { douban: new RequestLimiter(2, 300), bangumi: new RequestLimiter(3, 100) };
|
||||
|
||||
/**
|
||||
* 请求缓存:搜索按 storage.cacheTtl.metaSearchDays,列表(榜单 / 热门 / 每日放送 / 趋势页)按 metaListHours,
|
||||
* 空结果按 metaEmptyHours(不超过正常 TTL);失败(网络、403、风控页)在适配器里抛出,cachedCall 不写。
|
||||
* 条目详情不走这里:douban.subject / bangumi.subject 条目级缓存(30 天)已经存了,避免同一份数据存两遍。
|
||||
* 请求缓存(统一元数据缓存):按策略表「来源 × 资源类型」取 TTL——搜索 metaSearchDays / 查无 metaMissDays,
|
||||
* 列表(榜单 / 热门 / 每日放送 / 趋势页)metaListHours / 空 metaEmptyHours;过期先用旧值后台刷新。
|
||||
* 失败(网络、403、风控页)在适配器里抛出,不写。
|
||||
* 条目详情不走这里:douban.subject / bangumi.subject 条目级缓存已经存了,避免同一份数据存两遍。
|
||||
*/
|
||||
export function metaRequestCache(ctx: SourcesRuntime["ctx"]): MetaRequestCache {
|
||||
return {
|
||||
async cached(req, load) {
|
||||
const t = (await storageSettings(ctx)).cacheTtl;
|
||||
const ttl = req.kind === "search" ? t.metaSearchDays * 86_400 : t.metaListHours * 3600;
|
||||
return cachedCall(
|
||||
pgCacheFor(ctx.db),
|
||||
const p = metaCachePolicy(t, req.source, req.kind);
|
||||
return metadataCache(ctx).fetch(
|
||||
META_REQUEST_REGIONS[req.source],
|
||||
req.key,
|
||||
{
|
||||
region: META_REQUEST_REGIONS[req.source],
|
||||
ttl,
|
||||
emptyTtl: Math.min(ttl, t.metaEmptyHours * 3600),
|
||||
emptyIf: req.empty,
|
||||
maxsize: META_REQUEST_MAXSIZE,
|
||||
},
|
||||
{ ...p, isEmpty: req.empty, maxsize: META_REQUEST_MAXSIZE },
|
||||
load,
|
||||
);
|
||||
},
|
||||
@@ -77,23 +82,38 @@ export function toCached(s: MetaSubject, now: Date): DoubanSubject {
|
||||
};
|
||||
}
|
||||
|
||||
/** 条目(缓存 → 在线);拿不到为 null */
|
||||
/**
|
||||
* 条目(缓存 → 在线,统一元数据缓存:过期先用旧值后台刷新);拿不到为 null。
|
||||
* 条目不存在记查无(metaMissDays);网络失败 / 被拦不缓存,同一条目本进程冷却 30 分钟再查。
|
||||
*/
|
||||
export async function loadSubject(rt: SourcesRuntime, source: MetaSource, id: string): Promise<DoubanSubject | null> {
|
||||
const hit = await loadDoubanSubject(rt.ctx, id, REGION[source]);
|
||||
if (hit) return hit;
|
||||
const clients = metaClients(rt);
|
||||
if (!clients) return null;
|
||||
const cache = pgCacheFor(rt.ctx.db);
|
||||
if (!clients) return loadDoubanSubject(rt.ctx, id, REGION[source]);
|
||||
const cache = metadataCache(rt.ctx);
|
||||
const missKey = `${source}:${id}`;
|
||||
if (await cache.get(MISS_REGION, missKey).catch(() => null)) return null;
|
||||
const t = (await storageSettings(rt.ctx)).cacheTtl;
|
||||
const p = metaCachePolicy(t, source, "detail");
|
||||
try {
|
||||
const v = await cache.fetch<DoubanSubject | null>(
|
||||
REGION[source],
|
||||
id,
|
||||
{ ttl: p.ttl, maxsize: META_SUBJECT_MAX_ENTRIES, skipIf: (x) => !x?.title },
|
||||
async () => {
|
||||
if ((failedAt.get(missKey) ?? 0) > Date.now() - FAIL_COOLDOWN_MS) return null;
|
||||
if (await cache.get(MISS_REGION, missKey)) return null;
|
||||
const s = source === "douban" ? await clients.douban.subject(id) : await clients.bangumi.subject(id);
|
||||
const cached = toCached(s, rt.now());
|
||||
await saveDoubanSubject(rt.ctx, cached, REGION[source]);
|
||||
return cached;
|
||||
return toCached(s, rt.now());
|
||||
},
|
||||
);
|
||||
return v && typeof v.title === "string" && v.title ? v : null;
|
||||
} catch (e) {
|
||||
rt.ctx.logger.info({ source, id, err: (e as Error).message }, "辅助信息源条目查询失败");
|
||||
await cache.set(MISS_REGION, missKey, true, { ttl: MISS_TTL_SEC, maxsize: 5000 }).catch(() => {});
|
||||
if (e instanceof MetaSourceError && e.kind === "not_found")
|
||||
await cache.set(MISS_REGION, missKey, true, { ttl: p.emptyTtl, maxsize: 5000 }).catch(() => {});
|
||||
else {
|
||||
if (failedAt.size > 5000) failedAt.clear();
|
||||
failedAt.set(missKey, Date.now());
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
import { join } from "node:path";
|
||||
import { fetchEpisodeGroup, fetchRichDetail, fetchRichSeason, TmdbError } from "@umt/adapters";
|
||||
import { dataPaths } from "../../../config.ts";
|
||||
import { externalFetch } from "../../../egress.ts";
|
||||
import { resetTrackedTmdbRefs } from "../../../shared-cache.ts";
|
||||
import { ImageCache } from "../../cache/images.ts";
|
||||
import type { SourcesRuntime } from "../runtime.ts";
|
||||
import { entityEpisodeGroup } from "./episode-structure.ts";
|
||||
import { metadataIncludeSpecials, metadataLanguages, metadataTextOptions } from "./metadata-rich.ts";
|
||||
|
||||
// 关注作品预热(统一元数据缓存,docs/runbook/cache.md):作品进入关注时(建订阅、用户搜索 / 手动识别、开始下载)
|
||||
// 把之后刷新 / 刮削会用到的 TMDB 条目先拉进缓存:详情(含标志)、各季详情、选中的剧集组,海报 / 背景 / 标志图片进图片缓存。
|
||||
// 请求键与刷新元数据 / 刮削完全相同(同一套 fetchRichDetail / fetchRichSeason / fetchEpisodeGroup,同样的语言链),
|
||||
// 所以后面直接命中;关注中作品的条目写入时 pinned(条数上限淘汰不清掉),命中时续期。
|
||||
|
||||
/** 一部作品最多预热多少季(长寿剧) */
|
||||
const MAX_SEASONS = 60;
|
||||
|
||||
export interface PrewarmResult {
|
||||
status: "warmed" | "skipped";
|
||||
reason?: string;
|
||||
seasons: number;
|
||||
episodeGroup: boolean;
|
||||
images: number;
|
||||
errors: string[];
|
||||
}
|
||||
|
||||
/** 图片预热用的 fetch:测试注入优先;测试进程没注入就不出外网 */
|
||||
function imageFetch(rt: SourcesRuntime): typeof globalThis.fetch | null {
|
||||
if (rt.overrides.fetch) return rt.overrides.fetch;
|
||||
return Deno.env.get("VITEST") ? null : externalFetch;
|
||||
}
|
||||
|
||||
export async function prewarmMetadata(rt: SourcesRuntime, mediaEntityId: string): Promise<PrewarmResult> {
|
||||
const { ctx } = rt;
|
||||
const out: PrewarmResult = { status: "skipped", seasons: 0, episodeGroup: false, images: 0, errors: [] };
|
||||
const ident = await ctx.db
|
||||
.selectFrom("external_identity as x")
|
||||
.innerJoin("media_entity as e", "e.id", "x.media_entity_id")
|
||||
.select(["x.entity_type", "x.external_id", "e.status"])
|
||||
.where("x.media_entity_id", "=", mediaEntityId)
|
||||
.where("x.provider", "=", "tmdb")
|
||||
.where("x.status", "=", "accepted")
|
||||
.executeTakeFirst();
|
||||
if (!ident || ident.status === "merged") return { ...out, reason: ident ? "merged" : "no_tmdb_identity" };
|
||||
if (ident.entity_type !== "movie" && ident.entity_type !== "tv") return { ...out, reason: "unsupported_type" };
|
||||
const tmdb = await rt.tmdb();
|
||||
if (!tmdb) return { ...out, reason: "tmdb_not_configured" };
|
||||
// 刚进入关注:关注集合按进程缓存 5 分钟,清掉让这次写入就标 pinned
|
||||
resetTrackedTmdbRefs(ctx);
|
||||
const type = ident.entity_type;
|
||||
const id = ident.external_id;
|
||||
const langs = await metadataLanguages(ctx);
|
||||
const textOpts = await metadataTextOptions(ctx);
|
||||
const note = (what: string, e: unknown) => out.errors.push(`${what}: ${(e as Error).message}`);
|
||||
|
||||
// 刷新元数据的基础详情与富详情(含标志)
|
||||
await tmdb.details(type, id).catch((e) => note("details", e));
|
||||
// 注入的简化客户端(测试)没有 getJson:只预热基础详情
|
||||
if (typeof tmdb.getJson !== "function") return { ...out, status: "warmed" };
|
||||
let rich: Awaited<ReturnType<typeof fetchRichDetail>> | null = null;
|
||||
try {
|
||||
rich = await fetchRichDetail(tmdb, type, id, langs, textOpts);
|
||||
} catch (e) {
|
||||
// 详情都拿不到(404 / 鉴权 / 网络):网络类交给任务重试,其余记下结束
|
||||
if (e instanceof TmdbError && ["network", "timeout", "rate"].includes(e.kind)) throw e;
|
||||
note("detail", e);
|
||||
return { ...out, reason: "detail_unavailable" };
|
||||
}
|
||||
out.status = "warmed";
|
||||
|
||||
if (type === "tv") {
|
||||
const specials = await metadataIncludeSpecials(ctx);
|
||||
const seasons = rich.seasons
|
||||
.map((s) => s.seasonNumber)
|
||||
.filter((n) => specials || n > 0)
|
||||
.slice(0, MAX_SEASONS);
|
||||
for (const n of seasons) {
|
||||
try {
|
||||
await fetchRichSeason(tmdb, id, n, langs, {
|
||||
now: rt.now(),
|
||||
...textOpts,
|
||||
originalLanguage: rich.originalLanguage,
|
||||
});
|
||||
out.seasons++;
|
||||
} catch (e) {
|
||||
note(`season ${n}`, e);
|
||||
}
|
||||
}
|
||||
const group = await entityEpisodeGroup(ctx.db, mediaEntityId);
|
||||
if (group) {
|
||||
try {
|
||||
await fetchEpisodeGroup(tmdb, group.id, langs[0]);
|
||||
out.episodeGroup = true;
|
||||
} catch (e) {
|
||||
note("episode group", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 图片:刮削取原图(original),这里预先拉进图片缓存目录
|
||||
const f = imageFetch(rt);
|
||||
if (f) {
|
||||
const images = new ImageCache({
|
||||
dir: join(dataPaths(ctx.config).dataDir, "cache", "images"),
|
||||
fetch: f,
|
||||
timeoutMs: 30_000,
|
||||
});
|
||||
for (const p of [rich.posterPath, rich.backdropPath, rich.logoPath]) {
|
||||
if (!p) continue;
|
||||
try {
|
||||
await images.bytes("original", p);
|
||||
out.images++;
|
||||
} catch (e) {
|
||||
note(`image ${p}`, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (out.errors.length)
|
||||
ctx.logger.debug({ entityId: mediaEntityId, errors: out.errors }, "关注作品预热有部分条目没拉到");
|
||||
return out;
|
||||
}
|
||||
@@ -72,6 +72,7 @@ export function richEntityFields(r: TmdbRichDetail): Partial<Record<EntityMetada
|
||||
originCountry: v(r.originCountry),
|
||||
...(r.posterPath ? { posterPath: v(r.posterPath) } : {}),
|
||||
...(r.backdropPath ? { backdropPath: v(r.backdropPath) } : {}),
|
||||
...(r.logoPath ? { logoPath: v(r.logoPath) } : {}),
|
||||
...(r.genres.length ? { genres: v(r.genres) } : {}),
|
||||
...(r.networks.length ? { platform: v(r.networks.join(" / ")) } : {}),
|
||||
...(r.entityType === "tv" && r.episodeCount !== null ? { episodeCount: v(r.episodeCount) } : {}),
|
||||
|
||||
@@ -140,6 +140,7 @@ const ENTITY_NFO_FIELDS: ReadonlySet<string> = new Set([
|
||||
"externalIds",
|
||||
"posterPath",
|
||||
"backdropPath",
|
||||
"logoPath",
|
||||
]);
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { MetadataCandidate } from "@umt/adapters";
|
||||
import type { ParsedRelease } from "@umt/domain";
|
||||
import { CACHE_REGIONS, sharedCache, tmdbCachePolicy } from "../../../shared-cache.ts";
|
||||
import { CACHE_REGIONS, metadataCache, tmdbCachePolicy } from "../../../shared-cache.ts";
|
||||
import type { SourcesRuntime } from "../runtime.ts";
|
||||
|
||||
// TMDB 识别缓存:移植 MP app/modules/themoviedb/cache.py 的 TmdbCache(对照表见 docs/porting/moviepilot-tmdb-cache.md)。
|
||||
@@ -92,7 +92,7 @@ export async function loadRecognizeCache(
|
||||
): Promise<RecognizeCacheEntry | null> {
|
||||
const tmdb = await rt.tmdb();
|
||||
const key = recognizeKey(meta, localeOf(tmdb));
|
||||
const cache = sharedCache(rt.ctx);
|
||||
const cache = metadataCache(rt.ctx);
|
||||
const v = await cache.get<RecognizeCacheEntry>(CACHE_REGIONS.tmdbRecognize, key).catch(() => null);
|
||||
if (!v || typeof v !== "object") return null;
|
||||
if (typeConflicted(meta, v.type, v.id)) {
|
||||
@@ -120,7 +120,7 @@ export async function saveRecognizeCache(
|
||||
const tmdb = await rt.tmdb();
|
||||
const key = recognizeKey(meta, localeOf(tmdb));
|
||||
const policy = await tmdbCachePolicy(rt.ctx);
|
||||
const cache = sharedCache(rt.ctx);
|
||||
const cache = metadataCache(rt.ctx);
|
||||
if (info) {
|
||||
if (typeConflicted(meta, info.entityType, info.externalId)) {
|
||||
rt.ctx.logger.warn({ key, title: info.title, type: info.entityType }, "识别结果类型与元数据冲突，不写入识别缓存");
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { sql } from "@umt/db";
|
||||
import { type RawBuilder, sql } from "@umt/db";
|
||||
import type { ServerContext } from "../../types.ts";
|
||||
import type { Exec } from "./common.ts";
|
||||
|
||||
@@ -17,9 +17,8 @@ import type { Exec } from "./common.ts";
|
||||
const OPEN_SUBSCRIPTION = ["active", "pending"];
|
||||
const DOWNLOAD_DONE = ["imported", "removed"];
|
||||
|
||||
export async function isEntityWatched(exec: Exec, entityId: string): Promise<boolean> {
|
||||
const r = await sql<{ watched: boolean }>`
|
||||
select (
|
||||
/** 作品 e(SQL 片段:作品 id 表达式)是否关注中;isEntityWatched 与 watchedTmdbRefs 共用一个口径 */
|
||||
const watchedExpr = (entityId: RawBuilder<unknown>) => sql<boolean>`(
|
||||
exists (select 1 from subscription s where s.media_entity_id = ${entityId} and s.status = any(${OPEN_SUBSCRIPTION}))
|
||||
or exists (select 1 from media_asset a where a.media_entity_id = ${entityId} and a.status = 'active')
|
||||
or exists (select 1 from external_identity x
|
||||
@@ -27,7 +26,7 @@ export async function isEntityWatched(exec: Exec, entityId: string): Promise<boo
|
||||
or exists (select 1 from source_release r
|
||||
where r.media_entity_id = ${entityId} and r.resolution_evidence->>'lookup' in ('search', 'manual'))
|
||||
or exists (select 1 from acquisition_plan p left join acquisition_slot sl on sl.id = p.slot_id
|
||||
where (sl.media_entity_id = ${entityId} or p.evaluation->>'mediaEntityId' = ${entityId}::text)
|
||||
where (sl.media_entity_id = ${entityId} or p.evaluation->>'mediaEntityId' = (${entityId})::text)
|
||||
and (p.evaluation->>'manual')::boolean is true)
|
||||
or exists (select 1 from download_task t
|
||||
left join source_release r on r.id = t.source_release_id
|
||||
@@ -37,14 +36,38 @@ export async function isEntityWatched(exec: Exec, entityId: string): Promise<boo
|
||||
and ${entityId} in (t.media_entity_id, r.media_entity_id, sl.media_entity_id))
|
||||
or exists (select 1 from organization_record o
|
||||
where o.media_entity_id = ${entityId} and o.status in ('queued', 'running'))
|
||||
) as watched`.execute(exec);
|
||||
)`;
|
||||
|
||||
/**
|
||||
* 关注中作品的 TMDB 条目集合(统一元数据缓存的 pinned / 续期用):`movie:<id>` / `tv:<id>` 与选中的剧集组 `group:<gid>`。
|
||||
* 口径同 isEntityWatched。
|
||||
*/
|
||||
export async function watchedTmdbRefs(exec: Exec): Promise<Set<string>> {
|
||||
const r = await sql<{ ref: string; grp: string | null }>`
|
||||
select x.entity_type || ':' || x.external_id as ref, e.episode_group_id as grp
|
||||
from external_identity x join media_entity e on e.id = x.media_entity_id
|
||||
where x.provider = 'tmdb' and x.status = 'accepted' and x.entity_type in ('movie', 'tv')
|
||||
and e.status <> 'merged' and ${watchedExpr(sql`e.id`)}`.execute(exec);
|
||||
const out = new Set<string>();
|
||||
for (const x of r.rows) {
|
||||
out.add(x.ref);
|
||||
if (x.grp) out.add(`group:${x.grp}`);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function isEntityWatched(exec: Exec, entityId: string): Promise<boolean> {
|
||||
const r = await sql<{ watched: boolean }>`select ${watchedExpr(sql`${entityId}::uuid`)} as watched`.execute(exec);
|
||||
return r.rows[0]?.watched === true;
|
||||
}
|
||||
|
||||
/**
|
||||
* 作品进入关注范围(建订阅、用户搜索、开始下载)时补排一次 full 元数据刷新:只对从没刷新过的作品排
|
||||
* (自动建档时因不在关注范围而没排的),刷新完成会重算待确认 / 滞后放行资源的集数(必要时开集数待确认)。
|
||||
* 入队失败只记日志;同一作品同一原因只排一次(幂等键)。
|
||||
* 作品进入关注范围(建订阅、用户搜索 / 手动识别、开始下载)时:
|
||||
* - 补排一次 full 元数据刷新:只对从没刷新过的作品排(自动建档时因不在关注范围而没排的),
|
||||
* 刷新完成会重算待确认 / 滞后放行资源的集数(必要时开集数待确认);同一作品只排一次(幂等键);
|
||||
* - 每次都排一次缓存预热(prewarmMetadata:详情、各季、剧集组、海报 / 背景 / 标志进缓存),幂等键按作品 + 日期,
|
||||
* 同一天重复进入只跑一次;条目已在缓存里时预热只是命中,不发请求。
|
||||
* 入队失败只记日志。返回是否排了刷新。
|
||||
*/
|
||||
export async function enqueueWatchedRefresh(ctx: ServerContext, entityId: string, reason: string): Promise<boolean> {
|
||||
const e = await ctx.db
|
||||
@@ -52,7 +75,9 @@ export async function enqueueWatchedRefresh(ctx: ServerContext, entityId: string
|
||||
.select(["id", "last_metadata_refresh_at", "status"])
|
||||
.where("id", "=", entityId)
|
||||
.executeTakeFirst();
|
||||
if (!e || e.last_metadata_refresh_at || e.status === "merged") return false;
|
||||
if (!e || e.status === "merged") return false;
|
||||
await enqueuePrewarm(ctx, entityId, reason);
|
||||
if (e.last_metadata_refresh_at) return false;
|
||||
try {
|
||||
await ctx.engine.enqueue(
|
||||
"refreshMetadata",
|
||||
@@ -68,3 +93,22 @@ export async function enqueueWatchedRefresh(ctx: ServerContext, entityId: string
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** 关注作品预热入队(幂等键按作品 + 日期);失败只记日志 */
|
||||
export async function enqueuePrewarm(ctx: ServerContext, entityId: string, reason: string): Promise<boolean> {
|
||||
const day = new Date().toISOString().slice(0, 10);
|
||||
try {
|
||||
const r = await ctx.engine.enqueue(
|
||||
"prewarmMetadata",
|
||||
{ mediaEntityId: entityId },
|
||||
{ idempotencyKey: `prewarmMetadata:${entityId}:${day}`, traceId: entityId },
|
||||
);
|
||||
return !r.deduplicated;
|
||||
} catch (err) {
|
||||
ctx.logger.debug(
|
||||
{ entityId, reason, err: { message: err instanceof Error ? err.message : String(err) } },
|
||||
"关注作品预热入队失败",
|
||||
);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import { AUTO_ENRICH_TASK, runAutoEnrich } from "./service/enrich.ts";
|
||||
import { ingestFeed } from "./service/feeds.ts";
|
||||
import { emitMetadataRefreshed } from "./service/lifecycle-events.ts";
|
||||
import { refreshMetadata } from "./service/metadata.ts";
|
||||
import { prewarmMetadata } from "./service/metadata-prewarm.ts";
|
||||
import { resolveRelease } from "./service/resolve.ts";
|
||||
import { snapshotSite } from "./service/snapshot.ts";
|
||||
import { liveSearch } from "./service/workbench.ts";
|
||||
@@ -89,6 +90,15 @@ export function registerSourcesTasks(registry: TaskRegistry, rt: () => SourcesRu
|
||||
return r;
|
||||
}),
|
||||
}),
|
||||
// 关注作品预热(统一元数据缓存):详情 / 各季 / 剧集组 / 图片先进缓存;网络道,走 TMDB 客户端限流
|
||||
defineTask({
|
||||
name: "prewarmMetadata",
|
||||
version: 1,
|
||||
lane: "network",
|
||||
inputSchema: z.object({ mediaEntityId: id }),
|
||||
retryPolicy: { maximumAttempts: 3, initialInterval: "5s", backoffCoefficient: 2, maximumInterval: "60s" },
|
||||
handler: (t) => t.step.run("prewarm", () => prewarmMetadata(rt(), t.input.mediaEntityId)),
|
||||
}),
|
||||
defineTask({
|
||||
name: "searchSubscription",
|
||||
version: 1,
|
||||
|
||||
+191
-29
@@ -1,33 +1,131 @@
|
||||
import { DEFAULT_SETTINGS, type StorageSettings, withStorageDefaults } from "@umt/contracts";
|
||||
import { type PgCache, pgCacheFor, sql } from "@umt/db";
|
||||
import { DEFAULT_SETTINGS, type StorageCacheTtl, type StorageSettings, withStorageDefaults } from "@umt/contracts";
|
||||
import {
|
||||
type FetchPolicy,
|
||||
listenCacheInvalidation,
|
||||
type MetadataCache,
|
||||
metadataCacheFor,
|
||||
type PgCache,
|
||||
pgCacheFor,
|
||||
sql,
|
||||
} from "@umt/db";
|
||||
import { watchedTmdbRefs } from "./modules/sources-identity/service/watched.ts";
|
||||
import type { ServerContext } from "./modules/types.ts";
|
||||
|
||||
// 进程级共享缓存入口:PG kv_cache(移植 MP app/runtime/cache.py 的 TTLCache,存储换成 PG,三进程共享)。
|
||||
// region 与 TTL 规则见 docs/runbook/cache.md 与 docs/porting/moviepilot-tmdb-cache.md;
|
||||
// CACHE-2 起各 region 的 TTL 取值来源是 settings.storage.cacheTtl(docs/runbook/storage.md),语义不变。
|
||||
// 进程级共享缓存入口(docs/runbook/cache.md):
|
||||
// - PgCache:PG kv_cache(移植 MP app/runtime/cache.py 的 TTLCache,存储换成 PG,三进程共享),非元数据的缓存仍直接用;
|
||||
// - MetadataCache:统一元数据缓存,TMDB / 豆瓣 / Bangumi 共用。L1 进程内存 + L2 kv_cache、过期先用旧值后台刷新、
|
||||
// 同请求合并、按 region 计命中。TTL 按「来源 × 资源类型」取 settings.storage.cacheTtl(metaCachePolicy)。
|
||||
|
||||
export const CACHE_REGIONS = {
|
||||
/** TMDB 请求响应(MP TMDb.request 的 @cached):key = path?query */
|
||||
tmdbRequest: "tmdb.request",
|
||||
/** TMDB 识别结果(MP TmdbCache):key = [类型][语言]{tmdbid 或名称}-{年份}-{季} */
|
||||
tmdbRecognize: "tmdb.recognize",
|
||||
doubanRequest: "douban.request",
|
||||
bangumiRequest: "bangumi.request",
|
||||
doubanSubject: "douban.subject",
|
||||
bangumiSubject: "bangumi.subject",
|
||||
/** 豆瓣 / bgm 条目查无(只记「不存在」,网络失败 / 被拦不记) */
|
||||
subjectMiss: "meta.subject_miss",
|
||||
/** 豆瓣 / bgm 条目 → TMDB 作品的确定匹配 */
|
||||
discoverMatch: "discover.match",
|
||||
} as const;
|
||||
|
||||
/** 命中统计只看这些 region(统一元数据缓存);其它 region 不经 MetadataCache */
|
||||
export const METADATA_REGIONS: readonly string[] = Object.values(CACHE_REGIONS);
|
||||
|
||||
const DAY = 86_400;
|
||||
const HOUR = 3_600;
|
||||
|
||||
export function sharedCache(ctx: Pick<ServerContext, "db">): PgCache {
|
||||
return pgCacheFor(ctx.db);
|
||||
}
|
||||
|
||||
/** 统一元数据缓存(同一个 db 共用一个实例) */
|
||||
export function metadataCache(ctx: Pick<ServerContext, "db"> & Partial<Pick<ServerContext, "logger">>): MetadataCache {
|
||||
return metadataCacheFor(ctx.db, ctx.logger ? { logger: ctx.logger } : {});
|
||||
}
|
||||
|
||||
/**
|
||||
* 进程启动时调用:LISTEN umt_cache_invalidate(在线才启用 L1)+ 每分钟把命中计数写进 cache_stat。
|
||||
* 返回 stop(停监听、最后写一次统计)。监听连不上时只记警告,L1 保持停用,照常走 L2。
|
||||
*/
|
||||
export async function startMetadataCache(
|
||||
ctx: Pick<ServerContext, "db" | "logger">,
|
||||
connectionString: string,
|
||||
opts: { flushMs?: number } = {},
|
||||
): Promise<() => Promise<void>> {
|
||||
const cache = metadataCache(ctx);
|
||||
const log = ctx.logger.child({ component: "metadata-cache" });
|
||||
let listener: Awaited<ReturnType<typeof listenCacheInvalidation>> | null = null;
|
||||
try {
|
||||
listener = await listenCacheInvalidation(cache, { connectionString, logger: log });
|
||||
} catch (err) {
|
||||
log.warn({ err: { message: (err as Error).message } }, "缓存失效监听启动失败，进程内一级缓存停用");
|
||||
}
|
||||
const flush = () =>
|
||||
cache.flushStats().catch((err) => log.debug({ err: { message: (err as Error).message } }, "缓存统计写入失败"));
|
||||
const timer = setInterval(flush, opts.flushMs ?? 60_000);
|
||||
return async () => {
|
||||
clearInterval(timer);
|
||||
await listener?.stop();
|
||||
await flush();
|
||||
};
|
||||
}
|
||||
|
||||
/** settings.storage(读失败用缺省) */
|
||||
export async function storageSettings(ctx: Pick<ServerContext, "settings">): Promise<StorageSettings> {
|
||||
return withStorageDefaults(await ctx.settings.get("storage").catch(() => undefined));
|
||||
}
|
||||
|
||||
// ── 策略表:来源 × 资源类型 → TTL ──
|
||||
|
||||
export type MetaCacheSource = "tmdb" | "douban" | "bangumi";
|
||||
/** search:搜索 / 识别;detail:详情 / 季 / 剧集组 / 条目;list:榜单 / 热门 / 每日放送;image:人物 / 图片 */
|
||||
export type MetaResourceKind = "search" | "detail" | "list" | "image";
|
||||
|
||||
export interface MetaCachePolicy {
|
||||
/** 秒 */
|
||||
ttl: number;
|
||||
/** 空结果 / 查无(秒) */
|
||||
emptyTtl: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* 策略表(docs/runbook/cache.md)。宽限期统一按 TTL 的 50%(上限 7 天,见 @umt/db defaultGrace);失败不缓存。
|
||||
*
|
||||
* | 来源 | search | detail | list | image |
|
||||
* | TMDB | tmdbSearchDays / tmdbMissDays | tmdbDetailDays / tmdbMissDays | tmdbListHours / tmdbListEmptyHours | imageDays / tmdbMissDays |
|
||||
* | 豆瓣 / bgm | metaSearchDays / metaMissDays | metaDetailDays / metaMissDays | metaListHours / metaEmptyHours | (无) |
|
||||
*/
|
||||
export function metaCachePolicy(t: StorageCacheTtl, source: MetaCacheSource, kind: MetaResourceKind): MetaCachePolicy {
|
||||
const cap = (ttl: number, empty: number) => ({ ttl, emptyTtl: Math.min(ttl, empty) });
|
||||
if (source === "tmdb") {
|
||||
switch (kind) {
|
||||
case "search":
|
||||
return cap(t.tmdbSearchDays * DAY, t.tmdbMissDays * DAY);
|
||||
case "list":
|
||||
return cap(t.tmdbListHours * HOUR, t.tmdbListEmptyHours * HOUR);
|
||||
case "image":
|
||||
return cap(t.imageDays * DAY, t.tmdbMissDays * DAY);
|
||||
default:
|
||||
return cap(t.tmdbDetailDays * DAY, t.tmdbMissDays * DAY);
|
||||
}
|
||||
}
|
||||
switch (kind) {
|
||||
case "search":
|
||||
return cap(t.metaSearchDays * DAY, t.metaMissDays * DAY);
|
||||
case "list":
|
||||
return cap(t.metaListHours * HOUR, t.metaEmptyHours * HOUR);
|
||||
default:
|
||||
return cap(t.metaDetailDays * DAY, t.metaMissDays * DAY);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* TMDB 缓存策略(MP:CONF.meta = META_CACHE_EXPIRE、CONF.tmdb 条数、EMPTY_RESULT_CACHE_TTL)。
|
||||
* CACHE-2:MP 只有一个 CONF.meta,这里按请求类别取 storage.cacheTtl:搜索 / find 与识别结果用 searchTtl,
|
||||
* 详情(/movie/{id}、/tv/{id}、季)用 ttl(= tmdbDetailDays),空结果与未识别负缓存用 emptyTtl。
|
||||
* 按请求类别取 storage.cacheTtl:搜索 / find 与识别结果用 searchTtl,详情(/movie/{id}、/tv/{id}、季)用 ttl,
|
||||
* 空结果与未识别负缓存用 emptyTtl;列表 / 图片见 metaCachePolicy。
|
||||
*/
|
||||
export interface TmdbCachePolicy {
|
||||
/** 详情 TTL(秒) */
|
||||
@@ -37,16 +135,25 @@ export interface TmdbCachePolicy {
|
||||
maxsize: number;
|
||||
/** 秒 */
|
||||
emptyTtl: number;
|
||||
/** 列表(秒)与列表空结果(秒) */
|
||||
listTtl: number;
|
||||
listEmptyTtl: number;
|
||||
/** 人物 / 图片(秒) */
|
||||
imageTtl: number;
|
||||
}
|
||||
|
||||
export async function tmdbCachePolicy(ctx: Pick<ServerContext, "settings">): Promise<TmdbCachePolicy> {
|
||||
const c = await ctx.settings.get("cache").catch(() => DEFAULT_SETTINGS.cache);
|
||||
const t = (await storageSettings(ctx)).cacheTtl;
|
||||
const list = metaCachePolicy(t, "tmdb", "list");
|
||||
return {
|
||||
ttl: t.tmdbDetailDays * DAY,
|
||||
searchTtl: t.tmdbSearchDays * DAY,
|
||||
maxsize: c.tmdbMaxSize ?? DEFAULT_SETTINGS.cache.tmdbMaxSize,
|
||||
emptyTtl: t.tmdbMissDays * DAY,
|
||||
listTtl: list.ttl,
|
||||
listEmptyTtl: list.emptyTtl,
|
||||
imageTtl: t.imageDays * DAY,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -55,47 +162,102 @@ export function isTmdbSearchPath(key: string): boolean {
|
||||
return /^\/(search|find)\//.test(key);
|
||||
}
|
||||
|
||||
/** 详情键 → `movie:<id>` / `tv:<id>`;不是详情返回 null */
|
||||
/**
|
||||
* 键 → 所属作品:`movie:<id>` / `tv:<id>`(详情、季、剧集、图片、剧集组列表都算);
|
||||
* 剧集组详情 `/tv/episode_group/<gid>` → `group:<gid>`;都不是返回 null。
|
||||
*/
|
||||
export function tmdbDetailRef(key: string): string | null {
|
||||
const g = /^\/tv\/episode_group\/([^/?]+)/.exec(key);
|
||||
if (g) return `group:${decodeURIComponent(g[1]!)}`;
|
||||
const m = /^\/(movie|tv)\/(\d+)(?:[/?]|$)/.exec(key);
|
||||
return m ? `${m[1]}:${m[2]}` : null;
|
||||
}
|
||||
|
||||
// ── 追踪中的作品:详情缓存命中时续期 ──
|
||||
// 追踪 = 作品有未结束的订阅(active / paused / pending)且有已接受的 TMDB 身份。集合按进程缓存 5 分钟。
|
||||
// ── 关注中的作品:条目 pinned(LRU 不淘汰)、命中续期 ──
|
||||
// 关注口径同 service/watched.ts isEntityWatched;集合 = 关注中作品的 TMDB 身份 + 选中的剧集组,按进程缓存 5 分钟。
|
||||
|
||||
const TRACKED_TTL_MS = 5 * 60_000;
|
||||
const trackedCache = new WeakMap<object, { at: number; set: Promise<Set<string>> }>();
|
||||
const WATCHED_TTL_MS = 5 * 60_000;
|
||||
const watchedCache = new WeakMap<object, { at: number; set: Promise<Set<string>> }>();
|
||||
|
||||
export async function trackedTmdbRefs(ctx: Pick<ServerContext, "db">, now = Date.now()): Promise<Set<string>> {
|
||||
const hit = trackedCache.get(ctx.db);
|
||||
if (hit && now - hit.at < TRACKED_TTL_MS) return hit.set;
|
||||
const set = sql<{ ref: string }>`select distinct x.entity_type || ':' || x.external_id as ref
|
||||
from subscription s join external_identity x on x.media_entity_id = s.media_entity_id
|
||||
where s.status in ('active', 'paused', 'pending') and x.provider = 'tmdb' and x.status = 'accepted'
|
||||
and x.entity_type in ('movie', 'tv')`
|
||||
.execute(ctx.db)
|
||||
.then((r) => new Set(r.rows.map((x) => x.ref)));
|
||||
trackedCache.set(ctx.db, { at: now, set });
|
||||
set.catch(() => trackedCache.delete(ctx.db));
|
||||
const hit = watchedCache.get(ctx.db);
|
||||
if (hit && now - hit.at < WATCHED_TTL_MS) return hit.set;
|
||||
const set = watchedTmdbRefs(ctx.db);
|
||||
watchedCache.set(ctx.db, { at: now, set });
|
||||
set.catch(() => watchedCache.delete(ctx.db));
|
||||
return set;
|
||||
}
|
||||
|
||||
/** 测试用:清掉追踪集合缓存 */
|
||||
/** 清掉关注集合缓存(测试;进入关注时预热前也清,让新关注的作品立即 pinned) */
|
||||
export function resetTrackedTmdbRefs(ctx: Pick<ServerContext, "db">): void {
|
||||
trackedCache.delete(ctx.db);
|
||||
watchedCache.delete(ctx.db);
|
||||
}
|
||||
|
||||
/** 这个 TMDB 请求键属于关注中的作品(详情 / 季 / 剧集组 / 图片) */
|
||||
export async function isWatchedTmdbKey(ctx: Pick<ServerContext, "db">, key: string): Promise<boolean> {
|
||||
const ref = tmdbDetailRef(key);
|
||||
return !!ref && (await trackedTmdbRefs(ctx)).has(ref);
|
||||
}
|
||||
|
||||
/**
|
||||
* TMDB 详情缓存命中后的续期:键属于追踪中的作品时把过期时间推到 now + 详情 TTL(只延长不缩短)。
|
||||
* 返回是否续期。失败不影响命中结果。
|
||||
* TMDB 详情缓存命中后的续期(旧接口,保留给直接用 PgCache 的地方):键属于关注中的作品时把过期时间推到
|
||||
* now + 详情 TTL(只延长不缩短)并标 pinned。返回是否续期。
|
||||
*/
|
||||
export async function renewTrackedTmdbDetail(
|
||||
ctx: Pick<ServerContext, "db">,
|
||||
key: string,
|
||||
detailTtlSec: number,
|
||||
): Promise<boolean> {
|
||||
const ref = tmdbDetailRef(key);
|
||||
if (!ref || !(await trackedTmdbRefs(ctx)).has(ref)) return false;
|
||||
return await sharedCache(ctx).renew(CACHE_REGIONS.tmdbRequest, key, detailTtlSec);
|
||||
if (!(await isWatchedTmdbKey(ctx, key))) return false;
|
||||
return await sharedCache(ctx).renew(CACHE_REGIONS.tmdbRequest, key, detailTtlSec, { pin: true });
|
||||
}
|
||||
|
||||
/** 关注中作品的 TMDB 条目:写入 pinned、命中续期(FetchPolicy.pin) */
|
||||
export function watchedPin(ctx: Pick<ServerContext, "db">, key: string): FetchPolicy<unknown>["pin"] {
|
||||
return tmdbDetailRef(key) ? () => isWatchedTmdbKey(ctx, key) : undefined;
|
||||
}
|
||||
|
||||
// ── 命中统计(GET /cache/regions 用) ──
|
||||
|
||||
export interface RegionHitStats {
|
||||
l1Hits: number;
|
||||
l2Hits: number;
|
||||
staleHits: number;
|
||||
misses: number;
|
||||
failures: number;
|
||||
refreshes: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* 最近 days 天各 region 的命中统计:cache_stat(各进程每分钟写入的增量之和)+ 本进程还没写入的计数。
|
||||
* 其它进程最近不到一分钟的计数不在内。
|
||||
*/
|
||||
export async function regionHitStats(ctx: Pick<ServerContext, "db">, days = 7): Promise<Map<string, RegionHitStats>> {
|
||||
const r = await sql<{
|
||||
region: string;
|
||||
l1: number;
|
||||
l2: number;
|
||||
stale: number;
|
||||
miss: number;
|
||||
fail: number;
|
||||
refresh: number;
|
||||
}>`select region, sum(l1_hits)::int8 as l1, sum(l2_hits)::int8 as l2, sum(stale_hits)::int8 as stale,
|
||||
sum(misses)::int8 as miss, sum(failures)::int8 as fail, sum(refreshes)::int8 as refresh
|
||||
from cache_stat where day > current_date - ${days}::int group by region`.execute(ctx.db);
|
||||
const out = new Map<string, RegionHitStats>();
|
||||
for (const x of r.rows)
|
||||
out.set(x.region, {
|
||||
l1Hits: Number(x.l1),
|
||||
l2Hits: Number(x.l2),
|
||||
staleHits: Number(x.stale),
|
||||
misses: Number(x.miss),
|
||||
failures: Number(x.fail),
|
||||
refreshes: Number(x.refresh),
|
||||
});
|
||||
for (const [region, c] of metadataCache(ctx).pendingStats()) {
|
||||
const o = out.get(region) ?? { l1Hits: 0, l2Hits: 0, staleHits: 0, misses: 0, failures: 0, refreshes: 0 };
|
||||
for (const k of Object.keys(o) as (keyof RegionHitStats)[]) o[k] += c[k];
|
||||
out.set(region, o);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import type { ScrapeFileResult, ScrapeMatch } from "@umt/contracts";
|
||||
import { createTempTree, type TempTree } from "@umt/testkit";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { queueScrape, runScrapeRecord } from "../../src/modules/acquisition-organize/services/scrape.ts";
|
||||
import { createHarness, type Harness, pgAvailable, seedBase } from "./harness.ts";
|
||||
|
||||
// TMDB 标志 → clearlogo.png:电影写影片目录、剧集写剧集根目录;覆盖方式与其它图片一致
|
||||
// (更新:没变不重写、本工具写过且没被改过的按新图覆盖、被人改过的跳过;只补缺失;全部覆盖)。
|
||||
|
||||
const hasPg = await pgAvailable();
|
||||
const at = new Date().toISOString();
|
||||
const f = (value: unknown) => ({ value, source: "tmdb", language: null, at, locked: false });
|
||||
const enc = (s: string) => new TextEncoder().encode(s);
|
||||
|
||||
describe.skipIf(!hasPg)("刮削:TMDB 标志写 clearlogo.png", () => {
|
||||
let h: Harness;
|
||||
let lib: TempTree;
|
||||
let rootId = "";
|
||||
|
||||
beforeAll(async () => {
|
||||
lib = await createTempTree(
|
||||
{
|
||||
"Movies/星际穿越 (2014)/Interstellar.2014.mkv": 100,
|
||||
"TV/人生切割术 (2022)/Season 1/Severance.S01E01.mkv": 100,
|
||||
},
|
||||
{ sentinel: true, prefix: "umt-scrlogo-" },
|
||||
);
|
||||
h = await createHarness("scrape_logo", {
|
||||
// 内容 = 路径,方便看出写的是哪张
|
||||
scrapeImage: async (p) => ({ content: enc(p), mime: p.endsWith(".png") ? "image/png" : "image/jpeg" }),
|
||||
});
|
||||
rootId = (await seedBase(h.db, { libraryPath: lib.root, cachePath: lib.root, qbUrl: "http://127.0.0.1:9" })).rootId;
|
||||
}, 60_000);
|
||||
afterAll(async () => {
|
||||
await h?.close();
|
||||
await lib?.cleanup();
|
||||
});
|
||||
|
||||
async function entity(kind: "movie" | "series", title: string, meta: object) {
|
||||
return (
|
||||
await h.db
|
||||
.insertInto("media_entity")
|
||||
.values({ kind, title, year: 2020, status: "active", metadata: JSON.stringify(meta) })
|
||||
.returning("id")
|
||||
.executeTakeFirstOrThrow()
|
||||
).id;
|
||||
}
|
||||
const setMeta = (id: string, meta: object) =>
|
||||
h.db
|
||||
.updateTable("media_entity")
|
||||
.set({ metadata: JSON.stringify(meta) })
|
||||
.where("id", "=", id)
|
||||
.execute();
|
||||
async function scrape(
|
||||
entityId: string,
|
||||
kind: "movie" | "tv",
|
||||
path: string,
|
||||
mode?: "missing" | "overwrite" | "update",
|
||||
) {
|
||||
const match: ScrapeMatch = {
|
||||
mediaEntityId: entityId,
|
||||
provider: null,
|
||||
externalId: null,
|
||||
kind,
|
||||
title: "x",
|
||||
year: null,
|
||||
source: "library",
|
||||
};
|
||||
const q = await queueScrape(h.d, {
|
||||
rootId: `lib:${rootId}`,
|
||||
rootPath: lib.root,
|
||||
libraryRootId: rootId,
|
||||
path,
|
||||
match,
|
||||
mode,
|
||||
trigger: "manual",
|
||||
});
|
||||
return runScrapeRecord(h.d, q.recordId);
|
||||
}
|
||||
const logo = (files: ScrapeFileResult[]) => files.find((x) => x.role === "clearlogo");
|
||||
const MOVIE_DIR = "Movies/星际穿越 (2014)";
|
||||
const LOGO = `${MOVIE_DIR}/clearlogo.png`;
|
||||
|
||||
it("电影:写在影片目录 clearlogo.png;更新模式内容没变不重写,换了标志按新图覆盖", async () => {
|
||||
const movie = await entity("movie", "星际穿越", { title: f("星际穿越"), logoPath: f("/logo-a.png") });
|
||||
const r = await scrape(movie, "movie", MOVIE_DIR);
|
||||
expect(logo(r.files)).toMatchObject({
|
||||
path: LOGO,
|
||||
action: "written",
|
||||
hash: expect.stringMatching(/^[0-9a-f]{64}$/),
|
||||
});
|
||||
expect(await readFile(lib.path(LOGO), "utf8")).toBe("/logo-a.png");
|
||||
const same = await scrape(movie, "movie", MOVIE_DIR, "update");
|
||||
expect(logo(same.files)).toMatchObject({ action: "skipped", reason: "scrape_unchanged" });
|
||||
await setMeta(movie, { title: f("星际穿越"), logoPath: f("/logo-b.png") });
|
||||
const upd = await scrape(movie, "movie", MOVIE_DIR, "update");
|
||||
expect(logo(upd.files)).toMatchObject({ action: "written" });
|
||||
expect(await readFile(lib.path(LOGO), "utf8")).toBe("/logo-b.png");
|
||||
|
||||
// 被人换掉的标志:更新跳过、只补缺失跳过、全部覆盖改写
|
||||
await writeFile(lib.path(LOGO), "我自己的标志");
|
||||
await setMeta(movie, { title: f("星际穿越"), logoPath: f("/logo-c.png") });
|
||||
expect(logo((await scrape(movie, "movie", MOVIE_DIR, "update")).files)).toMatchObject({
|
||||
action: "skipped",
|
||||
reason: "scrape_user_modified",
|
||||
});
|
||||
expect(logo((await scrape(movie, "movie", MOVIE_DIR, "missing")).files)).toMatchObject({
|
||||
action: "skipped",
|
||||
reason: "scrape_exists",
|
||||
});
|
||||
expect(await readFile(lib.path(LOGO), "utf8")).toBe("我自己的标志");
|
||||
expect(logo((await scrape(movie, "movie", MOVIE_DIR, "overwrite")).files)).toMatchObject({ action: "written" });
|
||||
expect(await readFile(lib.path(LOGO), "utf8")).toBe("/logo-c.png");
|
||||
});
|
||||
|
||||
it("剧集:写在剧集根目录,季目录不写;没有标志不写", async () => {
|
||||
const show = await entity("series", "人生切割术", { title: f("人生切割术"), logoPath: f("/sev.png") });
|
||||
const r = await scrape(show, "tv", "TV/人生切割术 (2022)");
|
||||
expect(logo(r.files)).toMatchObject({ path: "TV/人生切割术 (2022)/clearlogo.png", action: "written" });
|
||||
expect(r.files.filter((x) => x.role === "clearlogo")).toHaveLength(1);
|
||||
await setMeta(show, { title: f("人生切割术") });
|
||||
const none = await scrape(show, "tv", "TV/人生切割术 (2022)", "overwrite");
|
||||
expect(logo(none.files)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
Vendored
+13
-5
@@ -67,7 +67,10 @@ describe.skipIf(!hasPg)("C2-A 缓存(真实 PG)", () => {
|
||||
await (await new SourcesRuntime(h.ctx).tmdb())!.details("movie", 872585);
|
||||
const calls = tmdb.calls.length;
|
||||
const rows = (
|
||||
await sql<{ key: string; days: number }>`select key, extract(epoch from expires_at - now()) / 86400 as days
|
||||
await sql<{
|
||||
key: string;
|
||||
days: number;
|
||||
}>`select key, extract(epoch from coalesce(fresh_until, expires_at) - now()) / 86400 as days
|
||||
from kv_cache where region = 'tmdb.request'`.execute(h.db)
|
||||
).rows;
|
||||
const detail = rows.find((x) => x.key.startsWith("/movie/872585?"))!;
|
||||
@@ -80,7 +83,7 @@ describe.skipIf(!hasPg)("C2-A 缓存(真实 PG)", () => {
|
||||
const days = async (like: string) =>
|
||||
Number(
|
||||
(
|
||||
await sql<{ d: number }>`select extract(epoch from expires_at - now()) / 86400 as d
|
||||
await sql<{ d: number }>`select extract(epoch from coalesce(fresh_until, expires_at) - now()) / 86400 as d
|
||||
from kv_cache where region = 'tmdb.request' and key like ${like}`.execute(h.db)
|
||||
).rows[0]!.d,
|
||||
);
|
||||
@@ -96,7 +99,7 @@ describe.skipIf(!hasPg)("C2-A 缓存(真实 PG)", () => {
|
||||
expect(empty).toBeLessThanOrEqual(2);
|
||||
});
|
||||
|
||||
it("追踪中的作品:详情缓存命中时续期到 now + 详情 TTL;未追踪的不动", async () => {
|
||||
it("关注中的作品:详情缓存命中时续期到 now + 详情 TTL 并标 pinned;未关注的不动", async () => {
|
||||
const ent = (
|
||||
await sql<{
|
||||
id: string;
|
||||
@@ -111,19 +114,24 @@ describe.skipIf(!hasPg)("C2-A 缓存(真实 PG)", () => {
|
||||
const client = (await new SourcesRuntime(h.ctx).tmdb())!;
|
||||
await client.details("movie", 872585);
|
||||
await client.details("movie", 693134);
|
||||
await sql`update kv_cache set expires_at = now() + interval '1 day'
|
||||
await sql`update kv_cache set fresh_until = now() + interval '1 day', expires_at = now() + interval '2 days',
|
||||
pinned = false
|
||||
where region = 'tmdb.request' and (key like '/movie/872585?%' or key like '/movie/693134?%')`.execute(h.db);
|
||||
await client.details("movie", 872585);
|
||||
await client.details("movie", 693134);
|
||||
const left = async (like: string) =>
|
||||
Number(
|
||||
(
|
||||
await sql<{ d: number }>`select extract(epoch from expires_at - now()) / 86400 as d
|
||||
await sql<{ d: number }>`select extract(epoch from fresh_until - now()) / 86400 as d
|
||||
from kv_cache where region = 'tmdb.request' and key like ${like}`.execute(h.db)
|
||||
).rows[0]!.d,
|
||||
);
|
||||
expect(await left("/movie/872585?%")).toBeGreaterThan(19.9);
|
||||
expect(await left("/movie/693134?%")).toBeLessThanOrEqual(1);
|
||||
const pinned = await sql<{ k: string; p: boolean }>`select key as k, pinned as p from kv_cache
|
||||
where region = 'tmdb.request' and (key like '/movie/872585?%' or key like '/movie/693134?%')`.execute(h.db);
|
||||
expect(pinned.rows.filter((r) => r.p).every((r) => r.k.startsWith("/movie/872585?"))).toBe(true);
|
||||
expect(pinned.rows.some((r) => r.p)).toBe(true);
|
||||
});
|
||||
|
||||
it("GET /cache/tmdb 列出条目(键 / 过期 / 大小)与上限;POST /cache/tmdb/clear 清空", async () => {
|
||||
|
||||
+183
@@ -0,0 +1,183 @@
|
||||
import { metadataCacheFor, PgCache, sql } from "@umt/db";
|
||||
import { createMockTmdb, type MockTmdb } from "@umt/testkit";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { createCacheModule } from "../../src/modules/cache/index.ts";
|
||||
import { SourcesRuntime } from "../../src/modules/sources-identity/runtime.ts";
|
||||
import { metaRequestCache } from "../../src/modules/sources-identity/service/meta-subjects.ts";
|
||||
import { metaCachePolicy, regionHitStats, resetTrackedTmdbRefs } from "../../src/shared-cache.ts";
|
||||
import { bearer, createHarness, type Harness, pgAvailable } from "../auth-settings/harness.ts";
|
||||
import { DEV_KEY, json } from "../helpers.ts";
|
||||
|
||||
// 统一元数据缓存接到 TMDB / 豆瓣 / bgm:策略表、过期先用旧值后台刷新、同请求合并、失败不缓存、
|
||||
// 关注中作品 pinned 不被淘汰、按 region 统计命中(GET /cache/regions)。
|
||||
|
||||
const hasPg = await pgAvailable();
|
||||
const KEY = "0123456789abcdef0123456789abcdef";
|
||||
|
||||
const until = async (cond: () => boolean | Promise<boolean>, ms = 5000) => {
|
||||
const end = Date.now() + ms;
|
||||
while (!(await cond())) {
|
||||
if (Date.now() > end) throw new Error("等待超时");
|
||||
await new Promise((r) => setTimeout(r, 20));
|
||||
}
|
||||
};
|
||||
|
||||
describe("策略表(来源 × 资源类型)", () => {
|
||||
it("缺省:搜索 7 天(空 1 天)、详情 30 天(空 1 天)、列表 6 小时(空 1 小时)、图片 90 天", async () => {
|
||||
const { DEFAULT_STORAGE_SETTINGS } = await import("@umt/contracts");
|
||||
const t = DEFAULT_STORAGE_SETTINGS.cacheTtl;
|
||||
const day = 86_400;
|
||||
for (const s of ["tmdb", "douban", "bangumi"] as const) {
|
||||
expect(metaCachePolicy(t, s, "search")).toEqual({ ttl: 7 * day, emptyTtl: day });
|
||||
expect(metaCachePolicy(t, s, "detail")).toEqual({ ttl: 30 * day, emptyTtl: day });
|
||||
expect(metaCachePolicy(t, s, "list")).toEqual({ ttl: 6 * 3600, emptyTtl: 3600 });
|
||||
}
|
||||
expect(metaCachePolicy(t, "tmdb", "image")).toEqual({ ttl: 90 * day, emptyTtl: day });
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!hasPg)("统一元数据缓存(真实 PG)", () => {
|
||||
let h: Harness;
|
||||
let admin: ReturnType<typeof bearer>;
|
||||
let tmdb: MockTmdb;
|
||||
|
||||
beforeAll(async () => {
|
||||
h = await createHarness({ extraModules: [createCacheModule()] });
|
||||
admin = bearer(h.app, DEV_KEY);
|
||||
tmdb = await createMockTmdb({ apiKey: KEY });
|
||||
const r = await admin("PATCH", "/settings", {
|
||||
metadata: { tmdbApiKey: KEY, tmdbBaseUrl: `${tmdb.url}/3` },
|
||||
cache: { tmdbMaxSize: 16 },
|
||||
});
|
||||
expect(r.status).toBe(200);
|
||||
});
|
||||
afterAll(async () => {
|
||||
await tmdb?.close();
|
||||
await h?.close();
|
||||
});
|
||||
|
||||
const client = async () => (await new SourcesRuntime(h.ctx).tmdb())!;
|
||||
const row = async (like: string) =>
|
||||
(
|
||||
await sql<{ key: string; fresh: number; pinned: boolean }>`select key,
|
||||
extract(epoch from fresh_until - now()) / 86400 as fresh, pinned
|
||||
from kv_cache where region = 'tmdb.request' and key like ${like}`.execute(h.db)
|
||||
).rows[0];
|
||||
|
||||
it("TMDB 详情过期在宽限期内:先返回旧值,后台只刷一次;刷新后是新值", async () => {
|
||||
const c = await client();
|
||||
await c.details("movie", 872585);
|
||||
const before = tmdb.callsTo("/3/movie/872585").length;
|
||||
// 改成「已过新鲜期、还在宽限期」并在值里做个记号(热度 12345)
|
||||
await sql`update kv_cache set fresh_until = now() - interval '1 second', expires_at = now() + interval '1 day',
|
||||
value = jsonb_set(value, '{popularity}', '12345')
|
||||
where region = 'tmdb.request' and key like '/movie/872585?%'`.execute(h.db);
|
||||
const keys = (
|
||||
await sql<{
|
||||
key: string;
|
||||
}>`select key from kv_cache where region = 'tmdb.request' and key like '/movie/872585?%'`.execute(h.db)
|
||||
).rows.map((r) => r.key);
|
||||
expect(keys.length).toBeGreaterThan(0);
|
||||
// 回源慢一点,让两个调用方都在刷新完成前读到
|
||||
tmdb.fault("/3/movie/872585", { delayMs: 300, times: 10 });
|
||||
const [a, b] = await Promise.all([c.details("movie", 872585), c.details("movie", 872585)]);
|
||||
expect(a.popularity).toBe(12345);
|
||||
expect(b.popularity).toBe(12345);
|
||||
// 后台刷新:两个调用方同时拿到旧值,每个过期键只回源一次;刷新后新鲜且是新值
|
||||
await until(async () => {
|
||||
const r = await sql<{ n: number }>`select count(*)::int as n from kv_cache where region = 'tmdb.request'
|
||||
and key like '/movie/872585?%' and fresh_until > now() + interval '29 days'`.execute(h.db);
|
||||
return r.rows[0]!.n === keys.length;
|
||||
});
|
||||
expect(tmdb.callsTo("/3/movie/872585").length - before).toBe(keys.length);
|
||||
tmdb.clearFaults();
|
||||
expect((await c.details("movie", 872585)).popularity).not.toBe(12345);
|
||||
});
|
||||
|
||||
it("同请求合并:并发同一详情只回源一次;失败(5xx)不缓存", async () => {
|
||||
const c = await client();
|
||||
const n0 = tmdb.callsTo("/3/tv/95396/season/1").length;
|
||||
await Promise.all(Array.from({ length: 5 }, () => c.getJson("/tv/95396/season/1", { language: "zh-CN" })));
|
||||
expect(tmdb.callsTo("/3/tv/95396/season/1").length - n0).toBe(1);
|
||||
// 5xx 重试用尽后抛错(客户端缺省重试 3 次,退避约 3.5 秒),不写缓存
|
||||
tmdb.fault("/3/tv/100088/season/1", { status: 503, times: 10 });
|
||||
await expect(c.getJson("/tv/100088/season/1", { language: "zh-CN" })).rejects.toThrow();
|
||||
expect(await row("/tv/100088/season/1?%")).toBeUndefined();
|
||||
tmdb.clearFaults();
|
||||
});
|
||||
|
||||
it("关注中的作品:条目 pinned,条数上限淘汰时不被清掉", async () => {
|
||||
const ent = (
|
||||
await sql<{
|
||||
id: string;
|
||||
}>`insert into media_entity (kind, title) values ('series', '人生切割术') returning id`.execute(h.db)
|
||||
).rows[0]!.id;
|
||||
await sql`insert into external_identity (media_entity_id, provider, entity_type, external_id, status)
|
||||
values (${ent}, 'tmdb', 'tv', '95396', 'accepted')`.execute(h.db);
|
||||
await sql`insert into subscription (media_entity_id, media_kind, status) values (${ent}, 'series', 'active')`.execute(
|
||||
h.db,
|
||||
);
|
||||
resetTrackedTmdbRefs(h.ctx);
|
||||
await sql`delete from kv_cache where region = 'tmdb.request'`.execute(h.db);
|
||||
const c = await client();
|
||||
await c.getJson("/tv/95396", { language: "zh-CN" });
|
||||
expect((await row("/tv/95396?%"))?.pinned).toBe(true);
|
||||
// 上限 16:再写一批别的请求,关注作品的条目留着
|
||||
for (let i = 0; i < 24; i++) await c.searchRaw("movie", `不存在${i}`);
|
||||
expect(await row("/tv/95396?%")).toBeDefined();
|
||||
const n = await sql<{ n: number }>`select count(*)::int as n from kv_cache where region = 'tmdb.request'`.execute(
|
||||
h.db,
|
||||
);
|
||||
expect(n.rows[0]!.n).toBeLessThanOrEqual(16);
|
||||
});
|
||||
|
||||
it("豆瓣 / bgm 请求缓存按策略表:搜索 7 天、列表 6 小时、空结果按查无 / 列表空;失败不写", async () => {
|
||||
const cache = metaRequestCache(h.ctx);
|
||||
await cache.cached({ source: "douban", key: "GET s1", kind: "search", empty: () => false }, async () => [1]);
|
||||
await cache.cached({ source: "douban", key: "GET s0", kind: "search", empty: () => true }, async () => []);
|
||||
await cache.cached({ source: "bangumi", key: "GET l1", kind: "list", empty: () => false }, async () => ({
|
||||
data: [1],
|
||||
}));
|
||||
await expect(
|
||||
cache.cached({ source: "bangumi", key: "GET bad", kind: "list", empty: () => false }, async () => {
|
||||
throw new Error("HTTP 403");
|
||||
}),
|
||||
).rejects.toThrow("403");
|
||||
const ttl = async (region: string, key: string) =>
|
||||
Number(
|
||||
(
|
||||
await sql<{ s: number }>`select extract(epoch from fresh_until - now()) as s from kv_cache
|
||||
where region = ${region} and key = ${key}`.execute(h.db)
|
||||
).rows[0]?.s ?? -1,
|
||||
);
|
||||
expect(await ttl("douban.request", "GET s1")).toBeGreaterThan(7 * 86400 - 60);
|
||||
expect(await ttl("douban.request", "GET s0")).toBeLessThanOrEqual(86400);
|
||||
expect(await ttl("bangumi.request", "GET l1")).toBeLessThanOrEqual(6 * 3600);
|
||||
expect(await ttl("bangumi.request", "GET l1")).toBeGreaterThan(6 * 3600 - 60);
|
||||
expect(await ttl("bangumi.request", "GET bad")).toBe(-1);
|
||||
});
|
||||
|
||||
it("命中统计:各 region 计 L1 / L2 / 旧值 / 未命中 / 失败 / 后台刷新;写入 cache_stat 后接口合计并给命中率", async () => {
|
||||
const mc = metadataCacheFor(h.db);
|
||||
const pending = mc.pendingStats().get("tmdb.request");
|
||||
expect(pending?.misses).toBeGreaterThan(0);
|
||||
expect(pending?.staleHits).toBeGreaterThan(0);
|
||||
expect(pending?.refreshes).toBeGreaterThan(0);
|
||||
expect(pending?.failures).toBeGreaterThan(0);
|
||||
await mc.flushStats();
|
||||
expect(mc.pendingStats().size).toBe(0);
|
||||
// 再来一次 L2 命中(未写入 cache_stat 的也算进接口)
|
||||
await (await client()).getJson("/tv/95396", { language: "zh-CN" });
|
||||
const stats = await regionHitStats(h.ctx);
|
||||
expect(stats.get("tmdb.request")!.l2Hits).toBeGreaterThan(0);
|
||||
const list = await json(await admin("GET", "/cache/regions"));
|
||||
const tmdbRow = list.data.items.find((i: { region: string }) => i.region === "tmdb.request");
|
||||
expect(tmdbRow.hits.hitRate).toBeGreaterThan(0);
|
||||
expect(tmdbRow.hits.hitRate).toBeLessThan(1);
|
||||
expect(tmdbRow.hits).toMatchObject({ misses: expect.any(Number), l2Hits: expect.any(Number) });
|
||||
// 非元数据 region 没有命中统计
|
||||
await new PgCache(h.db).set("site.detail", "x", 1, { ttl: 60 });
|
||||
const again = await json(await admin("GET", "/cache/regions"));
|
||||
expect(again.data.items.find((i: { region: string }) => i.region === "site.detail").hits).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -40,6 +40,19 @@ const DEVIATIONS: { id: string; apply: (files: string[], scenario: string) => st
|
||||
? fs
|
||||
: fs.map((f) => (f === "poster.jpg" || f === "fanart.jpg" ? `${LOOSE.replace(/\.mkv$/, "")}-${f}` : f)),
|
||||
},
|
||||
// 4. 作品级标志 clearlogo.png(TMDB logos;MP 只在接 fanart.tv 时写 logo,我们不接 fanart.tv):
|
||||
// 写在作品海报旁边——电影影片目录 / 剧集根目录 clearlogo.png,库根的电影 <视频名>-clearlogo.png;季目录不写
|
||||
{
|
||||
id: "tmdb_clearlogo",
|
||||
apply: (fs) => [
|
||||
...fs,
|
||||
...fs.flatMap((f) => {
|
||||
const m = /^(.*?)(\/|^|-)poster\.jpg$/.exec(f);
|
||||
if (!m || /(^|\/)Season \d+$/.test(m[1]!) || /(^|\/)season(\d+|-specials)$/.test(m[1]!)) return [];
|
||||
return [`${m[1]}${m[2]}clearlogo.png`];
|
||||
}),
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const detail = (entityType: "movie" | "tv", externalId: string): TmdbRichDetail =>
|
||||
@@ -69,6 +82,7 @@ const detail = (entityType: "movie" | "tv", externalId: string): TmdbRichDetail
|
||||
originCountry: [],
|
||||
posterPath: "/poster.jpg",
|
||||
backdropPath: "/backdrop.jpg",
|
||||
logoPath: "/logo.png",
|
||||
cast: [],
|
||||
crew: [],
|
||||
externalIds: { imdb: null, tvdb: null, wikidata: null },
|
||||
@@ -124,7 +138,10 @@ describe.skipIf(skip)("刮削文件布局与 MP 差分", () => {
|
||||
"scrlayout",
|
||||
{
|
||||
ao: {
|
||||
scrapeImage: async () => ({ content: new Uint8Array([0xff, 0xd8, 0xff]), mime: "image/jpeg" }),
|
||||
scrapeImage: async (p) =>
|
||||
p.endsWith(".png")
|
||||
? { content: new Uint8Array([0x89, 0x50, 0x4e, 0x47, 1]), mime: "image/png" }
|
||||
: { content: new Uint8Array([0xff, 0xd8, 0xff]), mime: "image/jpeg" },
|
||||
scrapeTmdbDetail: async (type, id) => detail(type, id),
|
||||
scrapeTmdbSeason: async (_tv, n) => (n === 1 ? season1 : null),
|
||||
},
|
||||
|
||||
@@ -218,13 +218,13 @@ describe.skipIf(!up)("发现页接口", () => {
|
||||
expect(meta.filter((m) => m.includes("movie_showing"))).toHaveLength(3);
|
||||
expect(meta.at(-1)).toContain("start=20&count=20");
|
||||
|
||||
// 空结果:写入但用空结果 TTL(缺省 1 小时),正常 6 小时
|
||||
// 空结果:写入但用空结果 TTL(缺省 1 小时),正常 6 小时(看新鲜期;过期后还有 50% 宽限期先用旧值)
|
||||
doubanReply = () => json({ subject_collection_items: [], total: 0 });
|
||||
await req("/discover/douban?list=tv_korean");
|
||||
const rows = await cache.items(META_REQUEST_REGIONS.douban);
|
||||
const ttlOf = (part: string) => {
|
||||
const row = rows.find((x) => x.key.includes(part))!;
|
||||
return Math.round((new Date(row.expiresAt!).getTime() - Date.now()) / 60_000);
|
||||
return Math.round((new Date(row.freshUntil).getTime() - Date.now()) / 60_000);
|
||||
};
|
||||
expect(ttlOf("tv_korean")).toBeLessThanOrEqual(60);
|
||||
expect(ttlOf("tv_korean")).toBeGreaterThan(55);
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
import { existsSync, mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { createTmdbClient } from "@umt/adapters";
|
||||
import { sql } from "@umt/db";
|
||||
import { createMockTmdb, type MockTmdb, SPLIT_COUR_FIXTURE, TMDB_FIXTURES } from "@umt/testkit";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { tmdbSharedCache } from "../../src/modules/sources-identity/runtime.ts";
|
||||
import { prewarmMetadata } from "../../src/modules/sources-identity/service/metadata-prewarm.ts";
|
||||
import { enqueueWatchedRefresh } from "../../src/modules/sources-identity/service/watched.ts";
|
||||
import { tmdbCachePolicy } from "../../src/shared-cache.ts";
|
||||
import { createHarness, type Harness, pgAvailable } from "./harness.ts";
|
||||
|
||||
// 关注作品预热:进入关注时入队 prewarmMetadata(幂等键按作品 + 日期);执行后详情(含标志)、各季、选中的剧集组
|
||||
// 在 TMDB 请求缓存里且 pinned,海报 / 背景 / 标志进图片缓存;再跑一次全部命中,不发请求。
|
||||
|
||||
const skip = !(await pgAvailable());
|
||||
const PNG = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 1]);
|
||||
const JPEG = new Uint8Array([0xff, 0xd8, 0xff, 0xe0, 1]);
|
||||
|
||||
describe.skipIf(skip)("关注作品预热", () => {
|
||||
let h: Harness;
|
||||
let tmdb: MockTmdb;
|
||||
let entityId: string;
|
||||
const images: string[] = [];
|
||||
const dataDir = mkdtempSync(join(tmpdir(), "umt-prewarm-"));
|
||||
|
||||
beforeAll(async () => {
|
||||
tmdb = await createMockTmdb({
|
||||
fixtures: [
|
||||
...TMDB_FIXTURES,
|
||||
{
|
||||
...SPLIT_COUR_FIXTURE,
|
||||
rich: { ...SPLIT_COUR_FIXTURE.rich, backdrop_path: "/split-bg.jpg" },
|
||||
logos: [
|
||||
{ file_path: "/split-en.png", iso_639_1: "en" },
|
||||
{ file_path: "/split-ja.png", iso_639_1: "ja" },
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
let policy: Awaited<ReturnType<typeof tmdbCachePolicy>> | null = null;
|
||||
h = await createHarness("prewarm", {
|
||||
// 和服务端一样走统一元数据缓存(PG 请求缓存)
|
||||
tmdb: () =>
|
||||
createTmdbClient({
|
||||
apiKey: "test-tmdb-key",
|
||||
baseUrl: `${tmdb.url}/3`,
|
||||
retries: 0,
|
||||
sharedCache: tmdbSharedCache(h.ctx, policy!),
|
||||
}),
|
||||
// 图片上游(image.tmdb.org)
|
||||
fetch: async (input) => {
|
||||
const url = String(input);
|
||||
images.push(url);
|
||||
return new Response(url.endsWith(".png") ? PNG : JPEG);
|
||||
},
|
||||
});
|
||||
policy = await tmdbCachePolicy(h.ctx);
|
||||
(h.ctx.config as { DATA_DIR?: string }).DATA_DIR = dataDir;
|
||||
entityId = (
|
||||
await sql<{ id: string }>`insert into media_entity (kind, title, episode_group_id, episode_group_name)
|
||||
values ('series', '分割番', 'grp-production', '制作季') returning id`.execute(h.db)
|
||||
).rows[0]!.id;
|
||||
await sql`insert into external_identity (media_entity_id, provider, entity_type, external_id, status)
|
||||
values (${entityId}, 'tmdb', 'tv', ${String(SPLIT_COUR_FIXTURE.id)}, 'accepted')`.execute(h.db);
|
||||
});
|
||||
afterAll(async () => {
|
||||
await h?.close();
|
||||
await tmdb?.close();
|
||||
rmSync(dataDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("进入关注(建订阅)时入队预热,幂等键按作品 + 日期;同一天再进入不重复排", async () => {
|
||||
await sql`insert into subscription (media_entity_id, media_kind, status) values (${entityId}, 'series', 'active')`.execute(
|
||||
h.db,
|
||||
);
|
||||
await enqueueWatchedRefresh(h.ctx, entityId, "subscription");
|
||||
await enqueueWatchedRefresh(h.ctx, entityId, "download");
|
||||
const jobs = h.engine.enqueued.filter((e) => e.task === "prewarmMetadata");
|
||||
expect(jobs).toHaveLength(1);
|
||||
expect(jobs[0]).toMatchObject({
|
||||
input: { mediaEntityId: entityId },
|
||||
key: `prewarmMetadata:${entityId}:${new Date().toISOString().slice(0, 10)}`,
|
||||
});
|
||||
});
|
||||
|
||||
it("执行后:详情 / 各季 / 剧集组 / 标志在缓存里且 pinned;海报、背景、标志进图片缓存", async () => {
|
||||
const r = await prewarmMetadata(h.rt, entityId);
|
||||
expect(r).toMatchObject({ status: "warmed", episodeGroup: true, images: 3, errors: [] });
|
||||
expect(r.seasons).toBe(SPLIT_COUR_FIXTURE.seasons!.length);
|
||||
const rows = (
|
||||
await sql<{ key: string; pinned: boolean }>`select key, pinned from kv_cache where region = 'tmdb.request'
|
||||
order by key`.execute(h.db)
|
||||
).rows;
|
||||
const has = (re: RegExp) => rows.filter((x) => re.test(x.key));
|
||||
const id = SPLIT_COUR_FIXTURE.id;
|
||||
expect(has(new RegExp(`^/tv/${id}\\?.*images`)).length).toBeGreaterThan(0);
|
||||
for (const s of SPLIT_COUR_FIXTURE.seasons!)
|
||||
expect(has(new RegExp(`^/tv/${id}/season/${s.season_number}\\?`)).length).toBeGreaterThan(0);
|
||||
expect(has(/^\/tv\/episode_group\/grp-production\?/)).toHaveLength(1);
|
||||
// 原语言是日语:另取了日语标志
|
||||
expect(has(new RegExp(`^/tv/${id}/images\\?include_image_language=ja`))).toHaveLength(1);
|
||||
expect(rows.every((x) => x.pinned)).toBe(true);
|
||||
for (const f of ["split.jpg", "split-bg.jpg", "split-ja.png"])
|
||||
expect(existsSync(join(dataDir, "cache/images/t/p/original", f))).toBe(true);
|
||||
expect(images.every((u) => u.startsWith("https://image.tmdb.org/t/p/original/"))).toBe(true);
|
||||
});
|
||||
|
||||
it("再预热一次全部命中:不发 TMDB 请求、不再下载图片", async () => {
|
||||
const calls = tmdb.calls.length;
|
||||
const fetched = images.length;
|
||||
const r = await prewarmMetadata(h.rt, entityId);
|
||||
expect(r.status).toBe("warmed");
|
||||
expect(tmdb.calls.length).toBe(calls);
|
||||
expect(images.length).toBe(fetched);
|
||||
});
|
||||
|
||||
it("没有 TMDB 身份的作品跳过", async () => {
|
||||
const other = (
|
||||
await sql<{ id: string }>`insert into media_entity (kind, title) values ('movie', '本地') returning id`.execute(
|
||||
h.db,
|
||||
)
|
||||
).rows[0]!.id;
|
||||
expect(await prewarmMetadata(h.rt, other)).toMatchObject({ status: "skipped", reason: "no_tmdb_identity" });
|
||||
});
|
||||
});
|
||||
@@ -69,12 +69,13 @@ describe.skipIf(!up)("M-Team 快照与任务注册", () => {
|
||||
api.setFailMode(null);
|
||||
});
|
||||
|
||||
it("模块注册 7 个任务与 3 条 outbox 路由;任务处理器经 step.run 调服务", async () => {
|
||||
it("模块注册 8 个任务与 3 条 outbox 路由;任务处理器经 step.run 调服务", async () => {
|
||||
const reg = new TaskRegistry();
|
||||
h.module.registerTasks(reg, h.ctx);
|
||||
expect(reg.names()).toEqual([
|
||||
"enrichPoolEntry",
|
||||
"ingestFeed",
|
||||
"prewarmMetadata",
|
||||
"refreshMetadata",
|
||||
"refreshSiteAccount",
|
||||
"resolveObservation",
|
||||
|
||||
@@ -260,6 +260,7 @@ export const ENTITY_FIELD_LABEL: Record<string, string> = {
|
||||
cast: "演员",
|
||||
crew: "导演 / 主创",
|
||||
externalIds: "外部 ID",
|
||||
logoPath: "标志",
|
||||
};
|
||||
|
||||
/** 季 / 集字段(contracts UnitMetadataField) */
|
||||
|
||||
@@ -37,6 +37,7 @@ const ROLE: Record<string, string> = {
|
||||
episode_nfo: "集 NFO",
|
||||
poster: "海报",
|
||||
fanart: "背景图",
|
||||
clearlogo: "标志",
|
||||
season_poster: "季海报",
|
||||
episode_thumb: "剧照",
|
||||
};
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
|
||||
import { type CacheRegionList, DEFAULT_SETTINGS } from "@umt/contracts";
|
||||
import { createElement } from "react";
|
||||
import { renderToString } from "react-dom/server";
|
||||
import { MemoryRouter } from "react-router";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { qk } from "~/lib/query-keys.ts";
|
||||
import { CacheSection, hitDetailText, hitRateText } from "./cache.tsx";
|
||||
|
||||
// 设置 → 缓存:缓存区表显示统一元数据缓存的命中率(最近 7 天、三个进程合计)。
|
||||
|
||||
const hits = { l1Hits: 50, l2Hits: 30, staleHits: 4, misses: 12, failures: 1, refreshes: 4, hitRate: 84 / 96 };
|
||||
|
||||
describe("缓存区命中率", () => {
|
||||
it("命中率按百分比显示一位小数;没有统计的区显示「—」,没读取过显示「暂无读取」", () => {
|
||||
expect(hitRateText(hits)).toBe("87.5%");
|
||||
expect(hitRateText(undefined)).toBe("—");
|
||||
expect(hitRateText({ ...hits, hitRate: null })).toBe("暂无读取");
|
||||
expect(hitDetailText(hits)).toBe("内存 50 · 数据库 30 · 旧值 4 · 未命中 12 · 回源失败 1 · 后台刷新 4");
|
||||
expect(hitDetailText({ ...hits, staleHits: 0, failures: 0, refreshes: 0 })).toBe("内存 50 · 数据库 30 · 未命中 12");
|
||||
});
|
||||
|
||||
it("缓存区表有「命中率」列,元数据区显示命中率与明细", () => {
|
||||
const qc = new QueryClient({ defaultOptions: { queries: { retry: false, enabled: false } } });
|
||||
qc.setQueryData(qk.settings, DEFAULT_SETTINGS);
|
||||
const regions: CacheRegionList = {
|
||||
items: [
|
||||
{ region: "tmdb.request", entries: 120, expired: 0, bytes: 4096, lastWriteAt: null, hits },
|
||||
{ region: "site.detail", entries: 3, expired: 0, bytes: 100, lastWriteAt: null },
|
||||
],
|
||||
};
|
||||
qc.setQueryData(["cache", "regions"], regions);
|
||||
const html = renderToString(
|
||||
createElement(
|
||||
QueryClientProvider,
|
||||
{ client: qc },
|
||||
createElement(MemoryRouter, { initialEntries: ["/console/settings#cache"] }, createElement(CacheSection)),
|
||||
),
|
||||
);
|
||||
expect(html).toContain("命中率");
|
||||
expect(html).toContain("87.5%");
|
||||
expect(html).toContain("内存 50 · 数据库 30");
|
||||
expect(html).toContain("最近 7 天");
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,4 @@
|
||||
import { cacheRegionLabel } from "@umt/contracts";
|
||||
import { type CacheHitStats, cacheRegionLabel } from "@umt/contracts";
|
||||
import { useState } from "react";
|
||||
import { toast } from "sonner";
|
||||
import {
|
||||
@@ -32,6 +32,8 @@ export type CacheRow = {
|
||||
expired: number;
|
||||
bytes: number;
|
||||
lastWriteAt: string | null;
|
||||
/** 统一元数据缓存的区才有:最近 7 天命中统计 */
|
||||
hits?: CacheHitStats | undefined;
|
||||
};
|
||||
|
||||
/** 区名 → 行:有中文名的用中文名,没有的原样显示 region */
|
||||
@@ -39,6 +41,23 @@ export function cacheRow(r: Omit<CacheRow, "label">): CacheRow {
|
||||
return { ...r, label: cacheRegionLabel(r.region) ?? r.region };
|
||||
}
|
||||
|
||||
/** 命中率文字:没有统计的区显示「—」,有统计没读取显示「暂无读取」 */
|
||||
export function hitRateText(h: CacheHitStats | undefined): string {
|
||||
if (!h) return "—";
|
||||
if (h.hitRate === null) return "暂无读取";
|
||||
return `${Math.round(h.hitRate * 1000) / 10}%`;
|
||||
}
|
||||
|
||||
/** 命中明细:内存 / 数据库 / 旧值 / 未命中,有失败与后台刷新时附上 */
|
||||
export function hitDetailText(h: CacheHitStats): string {
|
||||
const parts = [`内存 ${h.l1Hits}`, `数据库 ${h.l2Hits}`];
|
||||
if (h.staleHits) parts.push(`旧值 ${h.staleHits}`);
|
||||
parts.push(`未命中 ${h.misses}`);
|
||||
if (h.failures) parts.push(`回源失败 ${h.failures}`);
|
||||
if (h.refreshes) parts.push(`后台刷新 ${h.refreshes}`);
|
||||
return parts.join(" · ");
|
||||
}
|
||||
|
||||
function CacheRegionsBlock() {
|
||||
const regions = useCacheRegions();
|
||||
const images = useImageCache();
|
||||
@@ -85,6 +104,19 @@ function CacheRegionsBlock() {
|
||||
</div>
|
||||
),
|
||||
},
|
||||
{
|
||||
key: "hitRate",
|
||||
header: "命中率",
|
||||
align: "right",
|
||||
cell: (r) => (
|
||||
<div className="tabular-nums">
|
||||
{hitRateText(r.hits)}
|
||||
{r.hits && r.hits.hitRate !== null ? (
|
||||
<p className="text-meta text-muted-foreground">{hitDetailText(r.hits)}</p>
|
||||
) : null}
|
||||
</div>
|
||||
),
|
||||
},
|
||||
{
|
||||
key: "bytes",
|
||||
header: "占用",
|
||||
@@ -121,7 +153,7 @@ function CacheRegionsBlock() {
|
||||
section="cache"
|
||||
anchor="regions"
|
||||
title="缓存区"
|
||||
description="清空后下次用到时重新请求，不影响作品、资源和文件。已过期的条目由定时任务清理。"
|
||||
description="清空后下次用到时重新请求，不影响作品、资源和文件。已过期的条目由定时任务清理。命中率按最近 7 天、三个进程合计。"
|
||||
plain
|
||||
>
|
||||
<DataTable<CacheRow>
|
||||
@@ -224,7 +256,16 @@ export const IMAGES_CARD: CardSpec = {
|
||||
}),
|
||||
};
|
||||
|
||||
const TTL_KEYS = ["tmdbSearchDays", "tmdbDetailDays", "tmdbMissDays", "siteDetailDays", "webSearchDays"] as const;
|
||||
const TTL_KEYS = [
|
||||
"tmdbSearchDays",
|
||||
"tmdbDetailDays",
|
||||
"tmdbMissDays",
|
||||
"imageDays",
|
||||
"metaSearchDays",
|
||||
"metaDetailDays",
|
||||
"siteDetailDays",
|
||||
"webSearchDays",
|
||||
] as const;
|
||||
const ttlField = (k: string, label: string, hint?: string) => ({
|
||||
name: `storage.cacheTtl.${k}`,
|
||||
label,
|
||||
@@ -234,11 +275,14 @@ const ttlField = (k: string, label: string, hint?: string) => ({
|
||||
export const TTL_CARD: CardSpec = {
|
||||
anchor: "ttl",
|
||||
title: "缓存有效期",
|
||||
description: "单位天。追踪中的作品命中 TMDB 详情缓存时自动续期。",
|
||||
description: "单位天。过期后先用旧值、后台刷新；关注中的作品命中详情缓存时自动续期，不会因条数上限被清掉。",
|
||||
fields: [
|
||||
ttlField("tmdbSearchDays", "TMDB 搜索与识别结果"),
|
||||
ttlField("tmdbDetailDays", "TMDB 详情"),
|
||||
ttlField("tmdbMissDays", "TMDB 空结果", "搜索无结果、未识别时的负缓存"),
|
||||
ttlField("imageDays", "TMDB 人物与图片"),
|
||||
ttlField("metaSearchDays", "豆瓣 / Bangumi 搜索"),
|
||||
ttlField("metaDetailDays", "豆瓣 / Bangumi 条目"),
|
||||
ttlField("siteDetailDays", "站点详情页内容"),
|
||||
ttlField("webSearchDays", "网页搜索结果"),
|
||||
],
|
||||
|
||||
@@ -39,6 +39,7 @@ export const WORKFLOW_LABEL: Record<string, string> = {
|
||||
verifyAndImport: "核验整理",
|
||||
scrapeMedia: "刮削",
|
||||
refreshMetadata: "刷新元数据",
|
||||
prewarmMetadata: "预热元数据缓存",
|
||||
reconcileLibrary: "文件核对",
|
||||
cleanupDownloadCache: "清理缓存",
|
||||
resolveDecision: "处理待处理事项",
|
||||
|
||||
@@ -46,7 +46,7 @@
|
||||
| 上游 | 我们 |
|
||||
| --- | --- |
|
||||
| 每个请求 `@cached(maxsize=CONF.douban, ttl=CONF.meta, skip_none=True)`，键 = 函数 + 参数 | 列表 / 搜索请求经 `MetaRequestCache` 端口 → 服务端 `cachedCall`（region `douban.request` / `bangumi.request`，上限 5000 条）；键 = `方法 + 参数排序后的地址 + 请求体`（`metaRequestKey`） |
|
||||
| TTL 只有一个 `CONF.meta` | 按类别取 `storage.cacheTtl`：搜索 `metaSearchDays`（缺省 1 天）、榜单 / 热门 / 每日放送 / 趋势页 `metaListHours`（缺省 6 小时）、空结果 `metaEmptyHours`（缺省 1 小时，不超过正常 TTL） |
|
||||
| TTL 只有一个 `CONF.meta` | 按类别取 `storage.cacheTtl`（统一元数据缓存策略表，见 runbook/cache.md）：搜索 `metaSearchDays`（缺省 7 天，空结果 `metaMissDays` 1 天）、榜单 / 热门 / 每日放送 / 趋势页 `metaListHours`（缺省 6 小时，空结果 `metaEmptyHours` 1 小时），空结果 TTL 不超过正常 TTL；过期先用旧值后台刷新 |
|
||||
| `skip_none`：请求失败返回 None 不缓存 | 网络失败、非 2xx（含 403）、返回的不是 JSON（风控 / 验证页）、趋势页不是浏览页，适配器都抛错，`cachedCall` 不写 |
|
||||
| 条目详情同样走请求缓存 | 条目详情**不走**请求缓存：已有条目级缓存 `douban.subject` / `bangumi.subject`（30 天，存归一化后的标题 / 别名 / 年份，还收站点详情页与网页搜索得到的条目），再缓存一遍原始响应就是同一份数据存两遍；`meta.subject_miss`（6 小时）保留，是识别升级时防止反复打外站的负缓存，不属于请求缓存 |
|
||||
| 限流：无（靠缓存） | 每源进程内限流（豆瓣 2 并发、起始间隔 300 ms；bgm 3 并发、100 ms），只包真正发出的网络请求，缓存命中不占额度 |
|
||||
|
||||
@@ -72,3 +72,21 @@ PG 读写失败时 TMDB 路径按未命中处理(内存缓存不会失败)。
|
||||
偏离:MP 失败时返回空响应,我们回 502(前端回退占位);CACHE-2 加了容量 LRU(`cache.images_lru`,按 atime,预算 `storage.budgets.imagesGb`),
|
||||
按天清理保留作兜底;MP 按配置走代理服务器,我们直连(TMDB API 也未走代理);
|
||||
MP 只在启动时清理,我们改为每天一次的定时任务。
|
||||
|
||||
## 统一元数据缓存(2026-10-06)与 MP 的差异
|
||||
|
||||
用户定:TMDB / 豆瓣 / Bangumi 共用一套缓存策略,更积极地缓存,不上 Redis(PG `kv_cache` 二级 + 进程内存一级)。
|
||||
键、TTL 语义(正常 / 空结果短 TTL)、失败不缓存与上面各表一致;以下是 MP 没有、我们加的部分(详见 [runbook/cache.md](../runbook/cache.md)):
|
||||
|
||||
| 项 | MP | 我们 | 理由 |
|
||||
|---|---|---|---|
|
||||
| 存储层级 | 单进程内存 `TLRUCache`(可选 Redis) | L1 进程内存(每 region 1000 条、5 分钟)+ L2 PG `kv_cache`;读 L1 → L2 → 源,写两级 | 三个进程共享 L2,热点走 L1 少一次 PG 往返 |
|
||||
| TTL | `META_CACHE_EXPIRE` 一个值 + `EMPTY_RESULT_CACHE_TTL` | 「来源 × 资源类型」策略表:搜索 7 天 / 详情 30 天 / 列表 6 小时 / 人物与图片 90 天,空结果 1 天(列表 1 小时),取 `storage.cacheTtl` | 用户要求更积极地缓存,不同资源变化频率差很多 |
|
||||
| 过期 | 过期即未命中 | 宽限期(TTL 的 50%,最多 7 天)内先返回旧值、后台刷新;刷新失败继续用旧值并记警告 | 元数据过期后通常仍可用,不让请求等外站;外站故障时不降级 |
|
||||
| 并发 | `@cached` 无合并(TMDB 客户端内存 LRU 有) | 同进程同 key 只回源一次(含后台刷新) | 列表页 / 批量识别并发多 |
|
||||
| 淘汰 | TLRU | TLRU,但关注中作品的 TMDB 条目 `pinned` 不参与 LRU,命中续期 | 关注作品的详情 / 季 / 剧集组要一直在 |
|
||||
| 预热 | 无 | 作品进入关注时排 `prewarmMetadata`:详情、各季、剧集组、海报 / 背景 / 标志 | 订阅 / 下载后马上要刷新与刮削 |
|
||||
| 失效 | 进程内,`clear_cache()` | `PgCache.delete / clear` 发 `pg_notify('umt_cache_invalidate')`,各进程丢 L1;监听断开时 L1 停用,重连后整体清空 | 多进程下 L1 不能拿旧值 |
|
||||
| 统计 | 无 | 按 region 计 L1 / L2 / 旧值 / 未命中 / 回源失败 / 后台刷新,每分钟写 `cache_stat`,设置页显示近 7 天命中率 | 看缓存是否起作用 |
|
||||
| 豆瓣 / bgm 条目查询失败 | `DoubanApi` / `BangumiApi` 失败不缓存 | 同;只有「条目不存在」写 `meta.subject_miss`(以前网络失败也记 6 小时负缓存),失败后本进程同一条目冷却 30 分钟 | 策略表「失败不缓存」 |
|
||||
| TMDB 详情请求 | `append_to_response` 不含 images | 富详情加 `images`(`include_image_language=zh,en,null`),原语言不是中英文时另取 `/images`,选标志写 `clearlogo.png` | 用户定:不接 fanart.tv,用 TMDB logos;请求键变化后旧的富详情缓存会各自 miss 一次 |
|
||||
+108
-19
@@ -1,6 +1,6 @@
|
||||
# 缓存与限流运行手册(C2-A)
|
||||
# 缓存与限流运行手册
|
||||
|
||||
不引入 Redis / Deno KV。跨进程共享的状态都放 PG:站点限流令牌桶、TMDB 缓存。缓存语义照搬 MoviePilot,
|
||||
不引入 Redis / Deno KV。跨进程共享的状态都放 PG:站点限流令牌桶、缓存。缓存语义照搬 MoviePilot,
|
||||
对照与偏离见 [docs/porting/moviepilot-tmdb-cache.md](../porting/moviepilot-tmdb-cache.md)。
|
||||
|
||||
## 跨进程站点限流
|
||||
@@ -14,27 +14,113 @@
|
||||
|
||||
排查:`select * from rate_limit_bucket where key = 'site:<id>'`;`blocked_until` 不为空说明站点要求暂停。
|
||||
|
||||
## PG 缓存(kv_cache)
|
||||
## 统一元数据缓存
|
||||
|
||||
- 表 `kv_cache`(迁移 `0042_cache`,`UNLOGGED`:不写 WAL,崩溃后清空):(region, key) 主键,value jsonb,
|
||||
expires_at 每条独立,accessed_at 为最近读写时间。
|
||||
- `@umt/db` 的 `PgCache`(对照 MP `TTLCache`):set 带 ttl(秒,≤ 0 删除)与 maxsize(按 region 计条数,超出先删过期再删最久未用);
|
||||
get 命中刷新最近使用;`cachedCall` 对照 MP `@cached`(skip_none / skip_if / empty_ttl / fresh)。
|
||||
- 定时任务 `cache.sweep`(维护组,每 10 分钟)物理删除过期行。
|
||||
TMDB / 豆瓣 / Bangumi 三个来源共用一套缓存层(`@umt/db` 的 `MetadataCache`,服务端入口 `apps/server/src/shared-cache.ts`)。
|
||||
用户 2026-10-06 定:更积极地缓存,不上 Redis;PG `kv_cache`(UNLOGGED)作二级,进程内存作一级。
|
||||
|
||||
### 两级
|
||||
|
||||
| 级 | 存哪 | 容量 / 有效期 | 说明 |
|
||||
| --- | --- | --- | --- |
|
||||
| L1 | 各进程内存(api / worker / worker:agent 各一份) | 每个 region 最多 1000 条(LRU),5 分钟,且不超过条目在 L2 的有效期 | 只在跨进程失效监听在线时启用,见下 |
|
||||
| L2 | PG `kv_cache`(迁移 0042,0086 加 `fresh_until` / `pinned`) | 按 region 条数上限(TLRU,先删过期再删最久未用),每条独立 TTL | 三进程共享;UNLOGGED,PG 崩溃后清空 |
|
||||
|
||||
读:L1 → L2 → 源;写:两级都写。L1 返回的是副本,调用方改了不影响缓存。PG 读写失败按未命中 / 不写处理,不让业务请求失败。
|
||||
|
||||
### 策略表(来源 × 资源类型)
|
||||
|
||||
TTL 取值来自 `settings.storage.cacheTtl`(见 [storage.md](storage.md);旧库缺新字段时按缺省补),代码在 `metaCachePolicy`。
|
||||
|
||||
| 资源类型 | TMDB | 豆瓣 / Bangumi | 缺省 TTL | 空结果 / 查无 |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| 搜索 / 识别 | `/search/*`、`/find/*`、识别缓存 `tmdb.recognize`:`tmdbSearchDays` | 搜索请求:`metaSearchDays` | 7 天 | TMDB `tmdbMissDays`、豆瓣 / bgm `metaMissDays`,1 天 |
|
||||
| 详情 / 季 / 剧集组 | `/movie/{id}`、`/tv/{id}`、季、`/tv/episode_group/*`:`tmdbDetailDays` | 条目 `douban.subject` / `bangumi.subject`、发现页匹配 `discover.match`:`metaDetailDays` | 30 天 | 1 天(TMDB `tmdbMissDays`;条目不存在 `meta.subject_miss` 用 `metaMissDays`) |
|
||||
| 榜单 / 热门 / 每日放送 | trending / discover / popular 等:`tmdbListHours` | 榜单、标签、筛选、每日放送、趋势页:`metaListHours` | 6 小时 | TMDB `tmdbListEmptyHours`、豆瓣 / bgm `metaEmptyHours`,1 小时 |
|
||||
| 人物 / 图片 | `/person/*`、`…/images`:`imageDays` | — | 90 天 | `tmdbMissDays` |
|
||||
|
||||
- 空结果:TMDB 按 `results` 为空数组(MP `EMPTY_RESULT_CACHE_TTL` 判定);豆瓣 / bgm 按各接口的列表字段为空。空结果 TTL 不超过正常 TTL。
|
||||
- **失败不缓存**:网络错误、超时、鉴权失败、被拦(403、风控页)、限流、5xx 都在适配器里抛错,不写缓存(MP `skip_none` / `skip_if`)。
|
||||
豆瓣 / bgm 条目查询失败(不是「不存在」)时同一条目本进程冷却 30 分钟再在线查,不写缓存。
|
||||
- 条数上限:TMDB 两个 region 用 `cache.tmdbMaxSize`(缺省 5000);豆瓣 / bgm 请求 5000、条目 20000、发现匹配 20000。
|
||||
- `metaSearchDays` 缺省从 1 天改为 7 天(策略表统一);豆瓣 / bgm 搜索的空结果改按 `metaMissDays`(以前按 `metaEmptyHours`)。
|
||||
|
||||
### 过期先用旧值(stale-while-revalidate)
|
||||
|
||||
- 写入时 `fresh_until = now + TTL`、`expires_at = fresh_until + 宽限期`,宽限期 = TTL 的 50%,最多 7 天。
|
||||
- 过了 `fresh_until`、还没到 `expires_at` 的条目是旧值:先返回旧值,后台刷新;同一个 key 同时只刷一次。
|
||||
- 后台刷新失败继续用旧值,记一次警告「缓存后台刷新失败，继续用旧值」;过了宽限期就是普通未命中,同步回源。
|
||||
- `PgCache.get`(非元数据缓存与旧调用方用)只认新鲜条目;`PgCache.getEntry` 带回旧值,给 `MetadataCache` 用。
|
||||
- 识别缓存、豆瓣条目这类「只读不回源」的读取(`MetadataCache.get`)只认新鲜条目。
|
||||
|
||||
### 同请求合并(single-flight)
|
||||
|
||||
同进程同 region + key 并发只发一次请求,其余等同一个结果;后台刷新也走这条,刷新进行中的 key 不再发。跨进程不合并(各进程各自一次)。
|
||||
|
||||
### 关注中作品:pinned 与续期
|
||||
|
||||
- 关注口径同 `service/watched.ts isEntityWatched`(进行中的订阅、用户搜索 / 手动识别 / 人工确认、手动下载、在下载或整理、已在库)。
|
||||
集合 = 关注中作品的 TMDB 身份(`movie:<id>` / `tv:<id>`)+ 选中的剧集组(`group:<gid>`),按进程缓存 5 分钟。
|
||||
- 属于关注作品的 TMDB 请求(详情、季、剧集、图片、剧集组列表、剧集组详情)写入时标 `pinned`:按条数上限淘汰时跳过(仍计入条数,到期照样过期)。
|
||||
- L2 新鲜命中时续期:过期时间推到 now + 本类 TTL(只延长不缩短,保留宽限长度)。旧值不续期,该刷新就刷新。
|
||||
- 作品不再关注后 `pinned` 不会立即清掉,到期自然过期。
|
||||
|
||||
### 进入关注时预热
|
||||
|
||||
建订阅、用户搜索 / 手动识别出资源、提交下载时(`enqueueWatchedRefresh`,原来补排 full 刷新的触发点),每次额外入队一次
|
||||
`prewarmMetadata`(网络道,幂等键 `prewarmMetadata:<作品 id>:<日期>`,同一天重复进入只跑一次):
|
||||
|
||||
- 拉基础详情、富详情(含标志)、各季详情(按 `metadata.includeSpecials` 决定含不含季 0,最多 60 季)、选中的剧集组;
|
||||
请求键和刷新元数据 / 刮削完全相同,之后直接命中;预热前先清关注集合,保证这次写入就 pinned;
|
||||
- 海报、背景、标志的原图拉进图片缓存目录(刮削用的就是原图);已在磁盘上的不再下载;
|
||||
- 走 TMDB 客户端原有的限流与重试;详情都拿不到时网络类错误交给任务重试,其余记下结束。
|
||||
- 代码:`sources-identity/service/metadata-prewarm.ts`。
|
||||
|
||||
### 跨进程失效
|
||||
|
||||
- 每个进程启动时 `startMetadataCache` 开一条专用连接 `LISTEN umt_cache_invalidate`。
|
||||
- `PgCache.delete / clear`(含设置页「清空」、`POST /cache/tmdb/clear`、识别缓存类型冲突删除)和手动刷新(`fresh`)写入后
|
||||
`pg_notify('umt_cache_invalidate', {"r": region, "k": key})`;不带 `k` = 整个 region。各进程收到后丢掉对应 L1。
|
||||
- 普通写入不通知:别的进程 L1 里的旧值最多再用 5 分钟(L1 有效期)。
|
||||
- 监听在线才启用 L1;断开时立即停用并清空 L1,退避重连(最长 10 秒),重连成功后整体清一次再启用。
|
||||
启动时连不上只记警告,该进程只用 L2。脚本、测试等没启动监听的进程 L1 一直关着。
|
||||
|
||||
### 命中统计
|
||||
|
||||
- 每个进程按 region 在内存里计:L1 命中、L2 命中、旧值命中(先返回旧值)、未命中、回源失败、后台刷新。
|
||||
- 每分钟把增量加进表 `cache_stat`(迁移 0086,按「天 × region」一行;30 天前的行顺带删掉),进程退出时再写一次。
|
||||
- 口径:`GET /api/v1/cache/regions` 每个元数据 region 带 `hits`:最近 7 天 `cache_stat` 之和 + 当前进程还没写入的计数;
|
||||
别的进程最近不到一分钟的计数不在内。命中率 = (L1 + L2 + 旧值) / (L1 + L2 + 旧值 + 未命中),没读取过为 null。
|
||||
合并等待(single-flight 的跟随者)按未命中计。
|
||||
- 设置 → 缓存 → 缓存区表的「命中率」列显示百分比与明细(内存 / 数据库 / 旧值 / 未命中,有失败与后台刷新时附上);非元数据区显示「—」。
|
||||
- 排查:`select * from cache_stat where day = current_date order by region;`
|
||||
|
||||
### 接到这一层的 region
|
||||
|
||||
| region | 内容 | 键 | 读写方式 |
|
||||
|---|---|---|---|
|
||||
| `tmdb.request` | TMDB 请求响应(MP `TMDb.request`) | `path?query`(参数排序,不含凭证) | `fetch`:两级 + 旧值 + 合并 + pinned |
|
||||
| `tmdb.recognize` | TMDB 识别结果(MP `TmdbCache`) | `[类型][语言]{tmdbid 或名称}-{年份}-{季}` | `get` / `set`;未识别 `{id:0}` 按空结果 TTL |
|
||||
| `douban.request` / `bangumi.request` | 豆瓣 / bgm 搜索与列表请求 | `方法 地址?排序后的参数 [请求体]` | `fetch` |
|
||||
| `douban.subject` / `bangumi.subject` | 条目(标题 / 别名 / 年份) | 条目 id | 在线查走 `fetch`;站点详情页 / 网页摘要写入走 `set` |
|
||||
| `meta.subject_miss` | 豆瓣 / bgm 条目不存在 | `来源:id` | `set` / `get` |
|
||||
| `discover.match` | 豆瓣 / bgm 条目 → TMDB 作品的确定匹配 | `来源:id` | `get` / `set` |
|
||||
|
||||
其它 `kv_cache` 区(`site.detail`、`websearch`、`enrich*`、`identity.detail_ids`)仍直接用 `PgCache`,没有 L1 与命中统计:
|
||||
|
||||
| region | 内容 | 键 | TTL | 条数上限 |
|
||||
|---|---|---|---|---|
|
||||
| `tmdb.request` | TMDB 请求响应(MP `TMDb.request`) | `path?query`(不含凭证) | 搜索 / find:`storage.cacheTtl.tmdbSearchDays`(7 天);详情 / 季:`tmdbDetailDays`(30 天,追踪中的作品命中续期);`results` 为空:`tmdbMissDays`(1 天) | `cache.tmdbMaxSize` |
|
||||
| `tmdb.recognize` | TMDB 识别结果(MP `TmdbCache`) | `[类型][语言]{tmdbid 或名称}-{年份}-{季}` | `tmdbSearchDays`;未识别 `{id:0}` 按 `tmdbMissDays` | `cache.tmdbMaxSize` |
|
||||
| `site.detail` | 站点详情页扩展内容 | `source_release.id` | `max(站点详情缓存期, storage.cacheTtl.siteDetailDays)`(7 天) | 5000 |
|
||||
| `websearch` | 网页搜索结果 | 参数 sha256 | `storage.cacheTtl.webSearchDays`(1 天) | 1000 |
|
||||
|
||||
TTL 设置(CACHE-2 起):`settings.storage.cacheTtl`,见 [storage.md](storage.md)。MP 的 `META_CACHE_EXPIRE` 只有一个值,
|
||||
我们按请求类别拆成搜索 / 详情两档,`EMPTY_RESULT_CACHE_TTL` 对应 `tmdbMissDays`;缓存语义不变。
|
||||
条数上限仍是 `cache.tmdbMaxSize`(缺省 5000;MP `CONF.tmdb` 为 256,本项目缓存在 PG 里不占进程内存)。旧字段 `cache.tmdbTtlHours`、`cache.emptyResultTtlSec` 不再读取。
|
||||
### 清空与清理
|
||||
|
||||
API:`GET /api/v1/cache/tmdb?kind=recognize|request&limit=` 列出条目(键 / 过期 / 大小,识别条目带 tmdbId / 标题 / 海报);
|
||||
`POST /api/v1/cache/tmdb/clear` 清空两类(仅 admin)。
|
||||
- 设置 → 缓存 → 缓存区:按 region 清空(`POST /api/v1/cache/regions/{region}/clear`,仅 admin),同时通知各进程丢 L1。
|
||||
- `POST /api/v1/cache/tmdb/clear`:清 `tmdb.request` 与 `tmdb.recognize`(对应 MP `clear_cache()`)。
|
||||
- 定时任务 `cache.sweep`(维护组,每 10 分钟)物理删除已过硬过期(`expires_at`)的行;宽限期内的旧值不删。
|
||||
- 改了 TTL:新写入的条目按新值,已有条目保持原有效期,必要时清空。
|
||||
|
||||
API:`GET /api/v1/cache/tmdb?kind=recognize|request&limit=` 列出条目(键 / 过期 / 大小,识别条目带 tmdbId / 标题 / 海报)。
|
||||
|
||||
识别缓存何时读写(`sources-identity/service/tmdb-recognize-cache.ts`):
|
||||
- 识别资源时先读:命中且有标题 → 按 id 取 TMDB 详情作为唯一远端候选;命中 `{id:0}` → 不再搜索;强制复查(force)不读。
|
||||
@@ -48,20 +134,23 @@ API:`GET /api/v1/cache/tmdb?kind=recognize|request&limit=` 列出条目(键 /
|
||||
file 只允许 `[A-Za-z0-9_-]+.(jpg|jpeg|png|webp)`。
|
||||
- 缓存路径对照 MP `sanitize_url_path`:`DATA_DIR/cache/images/t/p/<size>/<file>`。
|
||||
- `cache.globalImageCache`(缺省关闭,MP `GLOBAL_IMAGE_CACHE`)开启才读磁盘缓存;拉到的合法图片总会写盘。
|
||||
刮削与关注作品预热取原图(`original`),总是先读磁盘缓存。
|
||||
- 按文件头识别图片类型,非图片 / SVG 拒绝;ETag = md5(内容),`Cache-Control: public, max-age=604800`,If-None-Match 相同回 304。
|
||||
- 上游超时(20s)/ 非 200 / 非图片 → 502,前端回退无彩占位。
|
||||
- 清理:定时任务 `cache.images_clean`(每天)删除 mtime 早于 `cache.globalImageCacheDays`(缺省 7)天的文件。
|
||||
- 容量 LRU(CACHE-2):`cache.images_lru`(30 分钟)在总量超过 `storage.budgets.imagesGb`(缺省 20 GB)时按最久未访问删到 90%;
|
||||
读盘命中会刷新文件 atime。见 [storage.md](storage.md)。
|
||||
- 用量 / 清空:`GET /api/v1/cache/images`、`POST /api/v1/cache/images/clear`(仅 admin);设置 → 缓存(缓存区表里的「图片」行可清空;「图片缓存」节开关与保留天数)。按区统计与清空通用缓存:`GET /api/v1/cache/regions`、`POST /api/v1/cache/regions/{region}/clear`(仅 admin),设置 → 缓存 → 缓存区。
|
||||
- 用量 / 清空:`GET /api/v1/cache/images`、`POST /api/v1/cache/images/clear`(仅 admin);设置 → 缓存(缓存区表里的「图片」行可清空;「图片缓存」节开关与保留天数)。
|
||||
|
||||
前端:作品库列表、作品详情、搜索面板的 TMDB 候选在有 `posterPath`(metadata.posterPath / TMDB 候选)时显示海报,
|
||||
加载失败回退原来的无彩占位。TMDB 详情刷新会把 posterPath / backdropPath 写进 metadata_field。
|
||||
加载失败回退原来的无彩占位。TMDB 详情刷新会把 posterPath / backdropPath / logoPath 写进 metadata_field。
|
||||
|
||||
## 常见问题
|
||||
|
||||
- TMDB 缓存不生效:确认 `select region, count(*) from kv_cache group by 1` 有 `tmdb.request`;PG 读写失败时按未命中处理并记
|
||||
`TMDB 缓存读写失败，按未命中处理`。
|
||||
`缓存读取失败，按未命中处理` / `缓存写入失败`。
|
||||
- 命中率一直是 0 或没有:`cache_stat` 有没有今天的行;进程日志里有没有 `缓存失效监听启动失败`(只影响 L1,L2 命中照样计)。
|
||||
- 清空后别的进程还拿到旧值:看该进程有没有 `缓存失效监听断开` 日志;断线期间 L1 是停用的,不会拿旧值。
|
||||
- 改了 TMDB 缓存设置:新写入的条目按新 TTL / 上限;已有条目保持原过期时间,必要时清空。
|
||||
- TTL 拉长后条目更早被条数上限挤掉:`cache.tmdbMaxSize` 缺省 5000,追踪作品很多时再调大。
|
||||
- TTL 拉长后条目更早被条数上限挤掉:`cache.tmdbMaxSize` 缺省 5000,关注作品的条目 pinned 不受影响,其余按最久未用淘汰。
|
||||
- 数据库重启后缓存为空:UNLOGGED 表的正常行为。
|
||||
@@ -99,6 +99,7 @@ ci 逐行核对这张表与任务注册表(`apps/server/test/worker-lanes.test.t
|
||||
| `resolveObservation` | network | 标题识别 + TMDB |
|
||||
| `enrichPoolEntry` | network | 未识别资源的元数据补全 |
|
||||
| `refreshMetadata` | network | TMDB 元数据刷新 |
|
||||
| `prewarmMetadata` | network | 关注作品预热:详情 / 各季 / 剧集组 / 海报背景标志进缓存(docs/runbook/cache.md) |
|
||||
| `searchSubscription` | network | 订阅搜索 |
|
||||
| `snapshotSite` | network | 站点快照 |
|
||||
| `refreshSiteAccount` | network | 站点登录 / 用户数据刷新 |
|
||||
|
||||
@@ -108,16 +108,20 @@ TMDB 有这一集但没有译名 / 简介(只有占位名和播出日期)不算
|
||||
|
||||
| 选中的 | 写的文件 |
|
||||
| --- | --- |
|
||||
| 剧集目录 | `tvshow.nfo`、`poster.jpg`、`fanart.jpg`、`seasonNN-poster.jpg`;每个季目录(`Season 1` / `S01` / `Specials` / `SPs` / `第 1 季`)`season.nfo` + `poster.jpg`;每集 `<视频名>.nfo` + `<视频名>-thumb.jpg`。只进季目录一层,其他子目录不管 |
|
||||
| 剧集目录 | `tvshow.nfo`、`poster.jpg`、`fanart.jpg`、`clearlogo.png`(有 TMDB 标志时)、`seasonNN-poster.jpg`;每个季目录(`Season 1` / `S01` / `Specials` / `SPs` / `第 1 季`)`season.nfo` + `poster.jpg`;每集 `<视频名>.nfo` + `<视频名>-thumb.jpg`。只进季目录一层,其他子目录不管 |
|
||||
| 季目录 | `season.nfo` + 季目录 `poster.jpg` + 上级 `seasonNN-poster.jpg`;每集 NFO 与剧照。不写 `tvshow.nfo` |
|
||||
| 单集文件 | 该集 `<视频名>.nfo` + 剧照 |
|
||||
| 电影目录 | 目录里每个视频 `<视频名>.nfo`(不进子目录)+ 目录 `poster.jpg` / `fanart.jpg`;蓝光原盘目录(含 `BDMV` / `VIDEO_TS`)写 `<目录名>.nfo` |
|
||||
| 电影文件 | `<视频名>.nfo` + 同目录 `poster.jpg` / `fanart.jpg`;文件直接在登记目录根时图片改为 `<视频名>-poster.jpg` / `-fanart.jpg` |
|
||||
| 电影目录 | 目录里每个视频 `<视频名>.nfo`(不进子目录)+ 目录 `poster.jpg` / `fanart.jpg` / `clearlogo.png`;蓝光原盘目录(含 `BDMV` / `VIDEO_TS`)写 `<目录名>.nfo` |
|
||||
| 电影文件 | `<视频名>.nfo` + 同目录 `poster.jpg` / `fanart.jpg` / `clearlogo.png`;文件直接在登记目录根时图片改为 `<视频名>-poster.jpg` / `-fanart.jpg` / `-clearlogo.png` |
|
||||
|
||||
- 自定义平铺模板(多部电影放在同一目录):目录里有登记在别的作品下的视频,或只刮部分视频时目录里还有没登记到本作品的视频,
|
||||
图片改为 `<视频名>-poster.jpg` / `-fanart.jpg`,不写整个目录共用的 `poster.jpg`。作品详情「重新刮削」对电影只刮该作品已入库的视频,
|
||||
不给同目录别的电影写本作品的 NFO(同一作品的多个版本照旧共用一张 poster)。
|
||||
- 图片扩展名按下载到的实际类型(jpeg / png / webp),URL 认不出或与内容不符时以内容为准。
|
||||
- 标志(clearlogo,2026-10-06):刷新元数据时 TMDB 详情带 `images`(`include_image_language=zh,en,null`,作品原语言不是中英文时另取原语言),
|
||||
选一张 PNG 标志存为作品字段 `logoPath`,语言优先 作品原语言 → 中文 → 英文 → 无文字,同语言取评分最高;SVG 标志不用。
|
||||
文件名按 Kodi / Emby / Jellyfin 通用约定 `clearlogo.png`:电影写影片目录、剧集写剧集根目录(季目录不写);覆盖方式与 hash 规则同其它图片。
|
||||
标志变化算 NFO 相关字段,刮削过的作品会延后排一次「更新」模式重刮。不接 fanart.tv。
|
||||
|
||||
- 集号:已登记视频按覆盖的集单元;其余按路径解析(季号缺失时取季目录名,再缺按第 1 季;同 MP 对每集重新 `MetaInfoPath`)。
|
||||
认不出集号的视频记 `scrape_no_episode` 跳过,记录算部分完成。
|
||||
@@ -166,6 +170,7 @@ TMDB 有这一集但没有译名 / 简介(只有占位名和播出日期)不算
|
||||
| 作品 / 电影目录背景图 | `backdrop.jpg` + `fanart.jpg` | `fanart.jpg` | 三家都认 fanart.jpg,副本没用 |
|
||||
| 每集剧照 | `<视频名>.jpg` | `<视频名>-thumb.jpg` | Kodi 约定,Jellyfin / Emby 同样识别;不和同名封面混 |
|
||||
| 视频直接在登记目录根 | 根里写 `poster.jpg` / `backdrop.jpg` / `fanart.jpg` | `<视频名>-poster.jpg` / `<视频名>-fanart.jpg` | 根里放一张 poster.jpg 会被当成整个库的封面 |
|
||||
| 作品级标志 | 只在接 fanart.tv 时写 logo | 作品海报旁写 `clearlogo.png`(根下电影 `<视频名>-clearlogo.png`),来源 TMDB logos | 用户定:不接 fanart.tv,TMDB 标志够用(差分登记 `tmdb_clearlogo`) |
|
||||
| 其余(tvshow / season / 集 NFO、季海报两处、电影 NFO、蓝光 `<目录名>.nfo`、季目录只写季级、单集只写该集) | — | 同 MP | — |
|
||||
|
||||
行为上的差异:
|
||||
@@ -177,7 +182,7 @@ TMDB 有这一集但没有译名 / 简介(只有占位名和播出日期)不算
|
||||
| 识别 | 按路径识别,认不出直接失败;对话框可手填 TMDB / 豆瓣 id | 按路径识别,认不准时返回前 5 候选让用户选(也可搜索);已登记的目录直接用作品 | 选一次比手填 id 省事;已登记的不必再识别 |
|
||||
| 执行 | 请求内同步 | 后台任务 `scrapeMedia`,请求内最多等 20 s | 大季包图片多;超时不丢 |
|
||||
| 记录 | 只有日志 | 每次刮削一条记录(目录、识别结果、逐文件结果、原因),作品详情显示最近一条 | 可回看、可排查 |
|
||||
| 图片种类 | poster / backdrop / logo / banner / disc / clearart / thumb(按图片源) | poster / fanart / 季海报 / 剧照 | 本地只有 TMDB 海报、背景图、剧照;其余要 fanart.tv 源 |
|
||||
| 图片种类 | poster / backdrop / logo / banner / disc / clearart / thumb(按图片源) | poster / fanart / clearlogo / 季海报 / 剧照 | 本地只有 TMDB 海报、背景图、标志、剧照;其余要 fanart.tv 源(不接) |
|
||||
| 批量 | 多选共用一个识别条件 | 多选逐项各自识别,各自选候选 | 多选里常是不同作品 |
|
||||
| 音乐 | 支持 | 不做 | 范围外 |
|
||||
|
||||
|
||||
+15
-5
@@ -26,8 +26,16 @@ PATCH /api/v1/settings {"storage":{"retentionDays":{"observation":60}}}
|
||||
| `retentionDays.capabilityStats` | 90 | 能力调用审计(`target_type=capability`)与已结束的 `capability_invocation` |
|
||||
| `retentionDays.identityPoolArchive` | 30 | 待识别池 unmatched 条目无新证据多少天后归档(不删) |
|
||||
| `cacheTtl.tmdbSearchDays` | 7 | TMDB `/search/*`、`/find/*` 响应与识别缓存 |
|
||||
| `cacheTtl.tmdbDetailDays` | 30 | TMDB 详情 / 季。追踪中的作品命中时续期 |
|
||||
| `cacheTtl.tmdbDetailDays` | 30 | TMDB 详情 / 季 / 剧集组。关注中的作品命中时续期 |
|
||||
| `cacheTtl.tmdbMissDays` | 1 | TMDB 空结果与未识别负缓存 |
|
||||
| `cacheTtl.tmdbListHours` | 6 | TMDB 榜单 / 热门 / 趋势 / 按条件筛选(小时) |
|
||||
| `cacheTtl.tmdbListEmptyHours` | 1 | TMDB 列表空结果(小时) |
|
||||
| `cacheTtl.imageDays` | 90 | TMDB 人物 / 图片列表(`/person/*`、`…/images`) |
|
||||
| `cacheTtl.metaSearchDays` | 7 | 豆瓣 / bgm 搜索请求(2026-10-06 起缺省 7 天,以前 1 天) |
|
||||
| `cacheTtl.metaDetailDays` | 30 | 豆瓣 / bgm 条目、发现页条目 → TMDB 匹配 |
|
||||
| `cacheTtl.metaMissDays` | 1 | 豆瓣 / bgm 搜索空结果、条目不存在 |
|
||||
| `cacheTtl.metaListHours` | 6 | 豆瓣 / bgm 榜单、热门、每日放送等列表(小时) |
|
||||
| `cacheTtl.metaEmptyHours` | 1 | 豆瓣 / bgm 列表空结果(小时) |
|
||||
| `cacheTtl.siteDetailDays` | 7 | 站点详情页扩展内容(`kv_cache` `site.detail`) |
|
||||
| `cacheTtl.webSearchDays` | 1 | 网页搜索结果 |
|
||||
|
||||
@@ -39,11 +47,13 @@ PATCH /api/v1/settings {"storage":{"retentionDays":{"observation":60}}}
|
||||
## 缓存有效期
|
||||
|
||||
只改 TTL 的取值来源,缓存语义还是对照 MP(见 [porting/moviepilot-tmdb-cache.md](../porting/moviepilot-tmdb-cache.md))。
|
||||
2026-10-06 起 TMDB / 豆瓣 / Bangumi 按「来源 × 资源类型」统一取值、过期先用旧值(宽限期 TTL 的 50%,最多 7 天),
|
||||
完整策略表、两级缓存与关注作品的 pinned / 续期见 [cache.md](cache.md)「统一元数据缓存」。
|
||||
|
||||
- TMDB 请求缓存:键以 `/search/` 或 `/find/` 开头的用搜索 TTL,其余用详情 TTL。`results` 为空的用未命中 TTL。
|
||||
- TMDB 请求缓存:键以 `/search/` 或 `/find/` 开头的用搜索 TTL;榜单类用列表 TTL;`/person/*`、`…/images` 用图片 TTL;其余用详情 TTL。`results` 为空的用未命中 TTL。
|
||||
- TMDB 识别缓存:正常条目用搜索 TTL(识别结果就是名称到条目的映射),负缓存 `{id:0}` 用未命中 TTL。
|
||||
- 追踪中的作品续期:作品有未结束的订阅(active / paused / pending)且有已接受的 TMDB 身份,
|
||||
它的详情键(`/movie/<id>?…`、`/tv/<id>…`)命中时把过期时间推到「现在 + 详情 TTL」,只延长不缩短。追踪集合每进程缓存 5 分钟。
|
||||
- 关注中的作品续期:口径同 `isEntityWatched`,它的详情 / 季 / 剧集组 / 图片键命中时把过期时间推到「现在 + 本类 TTL」,
|
||||
只延长不缩短,并标 pinned(条数上限淘汰时跳过)。关注集合每进程缓存 5 分钟。
|
||||
- 站点详情:`kv_cache` TTL 取 `max(站点详情缓存期, siteDetailDays)`。是否重抓仍按站点详情缓存期
|
||||
(`sites.defaultDetailCacheSec`,站点定义可更长)判断,下载前复核促销和 H&R 走的是这条。
|
||||
- 改了 TTL:新写入的条目按新值,已有条目保持原过期时间。
|
||||
@@ -71,7 +81,7 @@ PATCH /api/v1/settings {"storage":{"retentionDays":{"observation":60}}}
|
||||
| `storage.retention.identity_pool` | `identity_candidate` | 6 小时 | pending + unmatched,`coalesce(last_attempt_at, created_at)` 早于保留期 → 改为 archived |
|
||||
| `storage.analyze` | 全库 | 7 天 | `ANALYZE` |
|
||||
| `cache.images_lru` | 图片缓存 | 30 分钟 | 见上 |
|
||||
| `cache.sweep` | `kv_cache` | 10 分钟 | 已过期 |
|
||||
| `cache.sweep` | `kv_cache` | 10 分钟 | 已过硬过期(宽限期内的旧值保留) |
|
||||
|
||||
分批删除(`modules/system/retention.ts` 的 `deleteInBatches`):
|
||||
|
||||
|
||||
@@ -105,7 +105,15 @@ export interface TmdbConfig {
|
||||
* listTtlMs:流行趋势 / 热门 / 正在上映等列表与 /discover/ 的 TTL(缺省 6 小时且不超过 ttlMs;列表会变,不能按详情缓存 30 天),
|
||||
* 这类请求的空结果 TTL 也不超过它。
|
||||
*/
|
||||
cache?: { max?: number; ttlMs?: number; searchTtlMs?: number; emptyTtlMs?: number; listTtlMs?: number };
|
||||
cache?: {
|
||||
max?: number;
|
||||
ttlMs?: number;
|
||||
searchTtlMs?: number;
|
||||
emptyTtlMs?: number;
|
||||
listTtlMs?: number;
|
||||
/** 人物 / 图片列表(/person/*、/images);缺省同 ttlMs */
|
||||
imageTtlMs?: number;
|
||||
};
|
||||
/**
|
||||
* 跨进程共享的请求缓存(服务端传 PG 缓存);给了就不再用进程内 LRU。
|
||||
* 语义对照 MP @cached(skip_none、业务失败不缓存、results 为空按 emptyTtlMs)。
|
||||
@@ -124,6 +132,27 @@ type Json = Record<string, unknown>;
|
||||
export interface TmdbSharedCache {
|
||||
get(key: string): Promise<unknown | null>;
|
||||
set(key: string, value: unknown, ttlMs: number): Promise<void>;
|
||||
/**
|
||||
* 给了就由它负责整条读取路径(服务端统一元数据缓存:两级、过期先用旧值、同请求合并、统计、关注作品续期),
|
||||
* TTL 按 req.kind 取服务端策略表;load 抛错(网络、鉴权、限流、5xx)不缓存。没给时走 get / set。
|
||||
*/
|
||||
fetch?(
|
||||
req: { key: string; kind: TmdbRequestKind; isEmpty: (v: unknown) => boolean },
|
||||
load: () => Promise<unknown>,
|
||||
): Promise<unknown>;
|
||||
}
|
||||
|
||||
/** 请求类别(缓存 TTL 的依据):搜索 / find、列表、人物 / 图片、其余为详情(含季、剧集组) */
|
||||
export type TmdbRequestKind = "search" | "list" | "image" | "detail";
|
||||
|
||||
/** 人物 / 图片类请求:/person/*、…/images */
|
||||
export const isTmdbImagePath = (path: string) => /^\/person\//.test(path) || /\/images$/.test(path);
|
||||
|
||||
export function tmdbRequestKind(path: string): TmdbRequestKind {
|
||||
if (/^\/(search|find)\//.test(path)) return "search";
|
||||
if (isTmdbListPath(path)) return "list";
|
||||
if (isTmdbImagePath(path)) return "image";
|
||||
return "detail";
|
||||
}
|
||||
|
||||
function requestQuery(params: Record<string, string | undefined>): URLSearchParams {
|
||||
@@ -186,10 +215,16 @@ export class TmdbClient {
|
||||
const key = `${path}?${qs}`;
|
||||
let emptyTtl = this.cfg.cache?.emptyTtlMs ?? 30 * 60_000;
|
||||
const baseTtl = this.cfg.cache?.ttlMs ?? 24 * 3600_000;
|
||||
let ttl = /^\/(search|find)\//.test(path) ? (this.cfg.cache?.searchTtlMs ?? baseTtl) : baseTtl;
|
||||
if (isTmdbListPath(path)) {
|
||||
const kind = tmdbRequestKind(path);
|
||||
let ttl = kind === "search" ? (this.cfg.cache?.searchTtlMs ?? baseTtl) : baseTtl;
|
||||
if (kind === "list") {
|
||||
ttl = this.cfg.cache?.listTtlMs ?? Math.min(baseTtl, 6 * 3600_000);
|
||||
emptyTtl = Math.min(emptyTtl, ttl);
|
||||
} else if (kind === "image") ttl = this.cfg.cache?.imageTtlMs ?? baseTtl;
|
||||
if (this.cfg.sharedCache?.fetch) {
|
||||
return (await this.cfg.sharedCache.fetch({ key, kind, isEmpty: isEmptyResult }, () =>
|
||||
this.limiter.run(() => this.fetchWithRetry(path, qs)),
|
||||
)) as T;
|
||||
}
|
||||
if (this.cfg.sharedCache) {
|
||||
const shared = this.cfg.sharedCache;
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
parseAltTitles,
|
||||
parseTranslations,
|
||||
pickChineseTitle,
|
||||
pickLogo,
|
||||
pickText,
|
||||
pickTitle,
|
||||
} from "./index.ts";
|
||||
@@ -223,3 +224,70 @@ describe("fetchRichDetail / fetchRichSeason(mock TMDB)", () => {
|
||||
expect(mock.calls.length).toBe(n);
|
||||
});
|
||||
});
|
||||
|
||||
describe("TMDB 标志(clearlogo)", () => {
|
||||
const logos = (rows: { file_path: string; iso_639_1: string | null; vote_average?: number }[]) => ({ logos: rows });
|
||||
|
||||
it("语言优先 原语言 → 中文 → 英文 → 无文字;同语言按评分;只要 PNG", () => {
|
||||
const all = logos([
|
||||
{ file_path: "/none.png", iso_639_1: null, vote_average: 9 },
|
||||
{ file_path: "/en.png", iso_639_1: "en", vote_average: 5 },
|
||||
{ file_path: "/zh-low.png", iso_639_1: "zh", vote_average: 3 },
|
||||
{ file_path: "/zh-high.png", iso_639_1: "zh", vote_average: 6 },
|
||||
{ file_path: "/ja.svg", iso_639_1: "ja", vote_average: 9 },
|
||||
{ file_path: "/ja.png", iso_639_1: "ja", vote_average: 1 },
|
||||
]);
|
||||
expect(pickLogo([all], "ja")).toBe("/ja.png");
|
||||
expect(pickLogo([all], "ko")).toBe("/zh-high.png");
|
||||
expect(
|
||||
pickLogo(
|
||||
[
|
||||
logos([
|
||||
{ file_path: "/en.png", iso_639_1: "en" },
|
||||
{ file_path: "/n.png", iso_639_1: null },
|
||||
]),
|
||||
],
|
||||
"zh",
|
||||
),
|
||||
).toBe("/en.png");
|
||||
expect(pickLogo([logos([{ file_path: "/n.png", iso_639_1: null }])], "en")).toBe("/n.png");
|
||||
expect(pickLogo([logos([{ file_path: "/x.svg", iso_639_1: "en" }])], "en")).toBeNull();
|
||||
expect(pickLogo([undefined], null)).toBeNull();
|
||||
});
|
||||
|
||||
it("详情请求带 images(zh,en,null);原语言不是中英文时另取原语言标志", async () => {
|
||||
const fx = TMDB_FIXTURES.map((f) =>
|
||||
f.id === 872585
|
||||
? {
|
||||
...f,
|
||||
logos: [
|
||||
{ file_path: "/oppen-en.png", iso_639_1: "en" },
|
||||
{ file_path: "/oppen-zh.png", iso_639_1: "zh" },
|
||||
],
|
||||
}
|
||||
: f.id === 95396
|
||||
? {
|
||||
...f,
|
||||
original_language: "ja",
|
||||
logos: [
|
||||
{ file_path: "/sev-ja.png", iso_639_1: "ja" },
|
||||
{ file_path: "/sev-en.png", iso_639_1: "en" },
|
||||
],
|
||||
}
|
||||
: f,
|
||||
);
|
||||
mock = await createMockTmdb({ fixtures: fx });
|
||||
const tmdb = createTmdbClient({ baseUrl: `${mock.url}/3`, apiKey: "test-tmdb-key", retries: 0, timeoutMs: 1000 });
|
||||
const movie = await fetchRichDetail(tmdb, "movie", 872585, CHAIN);
|
||||
expect(movie.logoPath).toBe("/oppen-en.png");
|
||||
const call = mock.calls.find((c) => c.path === "/3/movie/872585")!;
|
||||
expect(call.query).toMatchObject({ include_image_language: "zh,en,null" });
|
||||
expect(call.query.append_to_response).toContain("images");
|
||||
expect(mock.calls.some((c) => c.path === "/3/movie/872585/images")).toBe(false);
|
||||
const tv = await fetchRichDetail(tmdb, "tv", 95396, CHAIN);
|
||||
expect(tv.logoPath).toBe("/sev-ja.png");
|
||||
expect(mock.calls.find((c) => c.path === "/3/tv/95396/images")?.query).toMatchObject({
|
||||
include_image_language: "ja",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -82,6 +82,8 @@ export interface TmdbRichDetail {
|
||||
originCountry: string[];
|
||||
posterPath: string | null;
|
||||
backdropPath: string | null;
|
||||
/** 标志(透明 PNG,刮削写 clearlogo.png);语言优先 原语言 → 中文 → 英文 → 无文字;没有为 null(旧数据可缺省) */
|
||||
logoPath?: string | null;
|
||||
cast: TmdbCredit[];
|
||||
crew: TmdbCredit[];
|
||||
externalIds: { imdb: string | null; tvdb: string | null; wikidata: string | null };
|
||||
@@ -163,9 +165,48 @@ function credits(raw: unknown, type: MediaEntityType, createdBy: unknown): { cas
|
||||
return { cast, crew: crew.slice(0, 6) };
|
||||
}
|
||||
|
||||
/** 详情请求带回的图片语言:中文、英文、无文字;作品原语言不在其中时另取一次 /images */
|
||||
export const DETAIL_IMAGE_LANGUAGES = "zh,en,null";
|
||||
|
||||
interface LogoRow {
|
||||
path: string;
|
||||
lang: string | null;
|
||||
vote: number;
|
||||
count: number;
|
||||
}
|
||||
|
||||
const logoRows = (images: unknown): LogoRow[] => {
|
||||
const logos = ((images ?? {}) as Json).logos;
|
||||
return ((Array.isArray(logos) ? logos : []) as Json[]).flatMap((x) => {
|
||||
const path = s(x.file_path);
|
||||
// 只要 PNG:Kodi / Emby / Jellyfin 的 clearlogo 都是 png,TMDB 的 svg 标志不用
|
||||
if (!path || !/\.png$/i.test(path)) return [];
|
||||
return [{ path, lang: s(x.iso_639_1), vote: n(x.vote_average) ?? 0, count: n(x.vote_count) ?? 0 }];
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* 富详情:一次请求 `/{type}/{id}?language=链首&append_to_response=alternative_titles,credits,external_ids,translations`,
|
||||
* 标题 / 简介 / 标语按链从 translations(+ 同地区别名)取。不存在抛 TmdbError(notfound)。
|
||||
* 选一张标志:语言优先 作品原语言 → 中文 → 英文 → 无文字;同语言按评分、票数从高到低。没有 PNG 标志返回 null。
|
||||
* images 可以是多份(详情带回的 + 原语言单独取的)。
|
||||
*/
|
||||
export function pickLogo(images: unknown[], originalLanguage: string | null): string | null {
|
||||
const rows = images.flatMap(logoRows);
|
||||
const orig = originalLanguage?.toLowerCase().split("-")[0] ?? null;
|
||||
const order = [orig, "zh", "en", null].filter((x, i, a) => a.indexOf(x) === i && (x !== null || i === a.length - 1));
|
||||
for (const lang of order) {
|
||||
const hit = rows
|
||||
.filter((r) => (r.lang?.toLowerCase() ?? null) === lang)
|
||||
.sort((a, b) => b.vote - a.vote || b.count - a.count)[0];
|
||||
if (hit) return hit.path;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 富详情:一次请求 `/{type}/{id}?language=链首&append_to_response=alternative_titles,credits,external_ids,translations,images`
|
||||
* (`include_image_language=zh,en,null`),标题 / 简介 / 标语按链从 translations(+ 同地区别名)取。
|
||||
* 作品原语言不是中文 / 英文时另取 `/{type}/{id}/images?include_image_language=<原语言>` 选标志(失败忽略)。
|
||||
* 不存在抛 TmdbError(notfound)。
|
||||
*/
|
||||
export async function fetchRichDetail(
|
||||
client: Pick<TmdbClient, "getJson">,
|
||||
@@ -177,7 +218,8 @@ export async function fetchRichDetail(
|
||||
const requested = chainOf(languages);
|
||||
const j = await client.getJson<Json>(`/${type}/${id}`, {
|
||||
language: requested[0],
|
||||
append_to_response: "alternative_titles,credits,external_ids,translations",
|
||||
append_to_response: "alternative_titles,credits,external_ids,translations,images",
|
||||
include_image_language: DETAIL_IMAGE_LANGUAGES,
|
||||
});
|
||||
const titleKey = type === "movie" ? "title" : "name";
|
||||
const originalTitle = s(type === "movie" ? j.original_title : j.original_name);
|
||||
@@ -212,6 +254,15 @@ export async function fetchRichDetail(
|
||||
const ext = (j.external_ids ?? {}) as Json;
|
||||
const runtimes = Array.isArray(j.episode_run_time) ? (j.episode_run_time as unknown[]) : [];
|
||||
const { cast, crew } = credits(j.credits, type, j.created_by);
|
||||
const logoSources: unknown[] = [j.images];
|
||||
const origLang = originalLanguage?.toLowerCase().split("-")[0] ?? null;
|
||||
if (origLang && origLang !== "zh" && origLang !== "en") {
|
||||
try {
|
||||
logoSources.unshift(await client.getJson<Json>(`/${type}/${id}/images`, { include_image_language: origLang }));
|
||||
} catch {
|
||||
// 原语言标志取不到就用中文 / 英文 / 无文字的
|
||||
}
|
||||
}
|
||||
return {
|
||||
entityType: type,
|
||||
externalId: String(id),
|
||||
@@ -240,6 +291,7 @@ export async function fetchRichDetail(
|
||||
originCountry,
|
||||
posterPath: s(j.poster_path),
|
||||
backdropPath: s(j.backdrop_path),
|
||||
logoPath: pickLogo(logoSources, originalLanguage),
|
||||
cast,
|
||||
crew,
|
||||
externalIds: {
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
matchTmdbMulti,
|
||||
type TmdbConfig,
|
||||
TmdbError,
|
||||
tmdbRequestKind,
|
||||
yearRange,
|
||||
} from "./index.ts";
|
||||
|
||||
@@ -135,6 +136,44 @@ describe("TmdbClient 缓存/限流/错误分类", () => {
|
||||
expect(tmdb.cacheStats().size).toBe(0);
|
||||
});
|
||||
|
||||
it("共享缓存给了 fetch:整条读取交给它,带请求类别(搜索 / 详情 / 列表 / 图片)与空结果判定;失败不进缓存", async () => {
|
||||
const seen: { key: string; kind: string; empty: boolean }[] = [];
|
||||
const store = new Map<string, unknown>();
|
||||
const { mock, tmdb } = await setup({
|
||||
sharedCache: {
|
||||
get: async () => null,
|
||||
set: async () => {},
|
||||
fetch: async (req, load) => {
|
||||
if (store.has(req.key)) return store.get(req.key);
|
||||
const v = await load();
|
||||
seen.push({ key: req.key, kind: req.kind, empty: req.isEmpty(v) });
|
||||
store.set(req.key, v);
|
||||
return v;
|
||||
},
|
||||
},
|
||||
});
|
||||
await tmdb.details("movie", 872585);
|
||||
await tmdb.details("movie", 872585);
|
||||
await tmdb.search("不存在的片名xyz", { type: "movie" });
|
||||
await tmdb.getJson("/movie/872585/images", {});
|
||||
await tmdb.getJson("/trending/all/week", {});
|
||||
await expect(tmdb.getJson("/movie/999999999", {})).rejects.toThrow();
|
||||
const byPath = new Map(seen.map((x) => [x.key.split("?")[0], [x.kind, x.empty]]));
|
||||
expect(Object.fromEntries(byPath)).toEqual({
|
||||
"/movie/872585": ["detail", false],
|
||||
"/search/movie": ["search", true],
|
||||
"/movie/872585/images": ["image", false],
|
||||
"/trending/all/week": ["list", false],
|
||||
});
|
||||
const detailCalls = mock.callsTo("/3/movie/872585").length;
|
||||
await tmdb.details("movie", 872585);
|
||||
expect(mock.callsTo("/3/movie/872585")).toHaveLength(detailCalls);
|
||||
expect(seen.some((x) => x.key.startsWith("/movie/999999999"))).toBe(false);
|
||||
expect(tmdbRequestKind("/person/1")).toBe("image");
|
||||
expect(tmdbRequestKind("/tv/1/season/2")).toBe("detail");
|
||||
expect(tmdbRequestKind("/find/tt1")).toBe("search");
|
||||
});
|
||||
|
||||
it("并发合并:同一请求并发只发一次", async () => {
|
||||
const { mock, tmdb } = await setup();
|
||||
await Promise.all(Array.from({ length: 5 }, () => tmdb.season(100088, 1)));
|
||||
|
||||
@@ -82,6 +82,27 @@ export function tmdbImageUrl(path: string | null | undefined, size: TmdbImageSiz
|
||||
export const CacheRegionName = z.string().regex(/^[a-z0-9][a-z0-9._-]{0,63}$/, "缓存区名不合法");
|
||||
export const CacheRegionParam = z.object({ region: CacheRegionName });
|
||||
|
||||
/**
|
||||
* 命中统计口径(docs/runbook/cache.md):各进程在内存里计数,每分钟把增量加进 cache_stat(按天 × region),
|
||||
* 接口返回最近 7 天各进程之和 + 当前进程还没写入的部分(别的进程最近不到一分钟的计数不在内)。
|
||||
* hitRate = (l1Hits + l2Hits + staleHits) / (l1Hits + l2Hits + staleHits + misses);没有读取时为 null。
|
||||
*/
|
||||
export const CacheHitStats = z.object({
|
||||
/** 进程内存(一级)命中 */
|
||||
l1Hits: NonNegInt,
|
||||
/** PG(二级)命中 */
|
||||
l2Hits: NonNegInt,
|
||||
/** 过期在宽限期内、先返回旧值的次数 */
|
||||
staleHits: NonNegInt,
|
||||
misses: NonNegInt,
|
||||
/** 回源失败(网络、被拦、5xx) */
|
||||
failures: NonNegInt,
|
||||
/** 后台刷新次数 */
|
||||
refreshes: NonNegInt,
|
||||
hitRate: z.number().min(0).max(1).nullable(),
|
||||
});
|
||||
export type CacheHitStats = z.infer<typeof CacheHitStats>;
|
||||
|
||||
export const CacheRegionStat = z.object({
|
||||
region: z.string(),
|
||||
/** 未过期条目数 */
|
||||
@@ -92,6 +113,8 @@ export const CacheRegionStat = z.object({
|
||||
bytes: NonNegInt,
|
||||
/** 最近一次写入 */
|
||||
lastWriteAt: Timestamp.nullable(),
|
||||
/** 命中统计(统一元数据缓存的 region 才有):最近 7 天,见 CacheHitStats */
|
||||
hits: CacheHitStats.optional(),
|
||||
});
|
||||
export type CacheRegionStat = z.infer<typeof CacheRegionStat>;
|
||||
|
||||
|
||||
@@ -220,6 +220,8 @@ export const EntityMetadataField = z.enum([
|
||||
"cast",
|
||||
"crew",
|
||||
"externalIds",
|
||||
// 统一元数据缓存一轮:TMDB 标志(刮削写 clearlogo.png)
|
||||
"logoPath",
|
||||
]);
|
||||
export type EntityMetadataField = z.infer<typeof EntityMetadataField>;
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ import { Id, NonNegInt, Timestamp } from "./common.ts";
|
||||
export const ScrapeFileResult = z.object({
|
||||
path: z.string(),
|
||||
kind: z.enum(["nfo", "image"]),
|
||||
/** movie_nfo / tvshow_nfo / season_nfo / episode_nfo / poster / fanart / season_poster / episode_thumb */
|
||||
/** movie_nfo / tvshow_nfo / season_nfo / episode_nfo / poster / fanart / clearlogo / season_poster / episode_thumb */
|
||||
role: z.string(),
|
||||
action: z.enum(["written", "skipped", "failed"]),
|
||||
/** 跳过 / 失败原因码(reasons.ts:scrape_exists / scrape_write_failed / scrape_image_failed …) */
|
||||
|
||||
@@ -578,8 +578,19 @@ export const StorageCacheTtl = z.object({
|
||||
metaSearchDays: Days,
|
||||
/** 豆瓣 / bgm 榜单、热门、每日放送等列表请求(小时) */
|
||||
metaListHours: z.number().int().min(1).max(720),
|
||||
/** 豆瓣 / bgm 空结果(小时,对照 MP empty_ttl) */
|
||||
/** 豆瓣 / bgm 列表空结果(小时,对照 MP empty_ttl);搜索空结果改按 metaMissDays */
|
||||
metaEmptyHours: z.number().int().min(1).max(720),
|
||||
// ── 统一元数据缓存(2026-10-06,docs/runbook/cache.md 策略表);旧库缺这些字段时按缺省补 ──
|
||||
/** 豆瓣 / bgm 条目详情(douban.subject / bangumi.subject)与发现页条目 → TMDB 匹配(discover.match) */
|
||||
metaDetailDays: Days,
|
||||
/** 豆瓣 / bgm 查无结果:搜索空结果、条目不存在(meta.subject_miss) */
|
||||
metaMissDays: Days,
|
||||
/** TMDB 榜单 / 热门 / 趋势 / 按条件筛选(小时) */
|
||||
tmdbListHours: z.number().int().min(1).max(720),
|
||||
/** TMDB 列表空结果(小时) */
|
||||
tmdbListEmptyHours: z.number().int().min(1).max(720),
|
||||
/** TMDB 人物 / 图片列表(/person/*、/images) */
|
||||
imageDays: Days,
|
||||
});
|
||||
export type StorageCacheTtl = z.infer<typeof StorageCacheTtl>;
|
||||
|
||||
@@ -608,9 +619,14 @@ export const DEFAULT_STORAGE_SETTINGS: StorageSettings = {
|
||||
tmdbMissDays: 1,
|
||||
siteDetailDays: 7,
|
||||
webSearchDays: 1,
|
||||
metaSearchDays: 1,
|
||||
metaSearchDays: 7,
|
||||
metaListHours: 6,
|
||||
metaEmptyHours: 1,
|
||||
metaDetailDays: 30,
|
||||
metaMissDays: 1,
|
||||
tmdbListHours: 6,
|
||||
tmdbListEmptyHours: 1,
|
||||
imageDays: 90,
|
||||
},
|
||||
};
|
||||
|
||||
|
||||
@@ -149,6 +149,8 @@ const INTERNAL_TABLES = [
|
||||
// C2-A 跨进程限流桶(0041)与 PG 缓存(0042):服务端内部,缓存经 /cache/* 的专用 DTO 暴露
|
||||
"RateLimitBucket",
|
||||
"KvCache",
|
||||
// 统一元数据缓存命中统计(0086):服务端内部,经 GET /cache/regions 的 hits 字段暴露
|
||||
"CacheStat",
|
||||
// 站点请求计数(0046):服务端内部,经 GET /sites/{id}/request-stats 的专用 DTO 暴露
|
||||
"SiteRequestCount",
|
||||
// W9-I 进程心跳(0034):服务端内部,经 /system/health 的专用 DTO 暴露
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
import type { Kysely } from "kysely";
|
||||
import { dropTables, execDdl } from "../src/migration-helpers.ts";
|
||||
|
||||
// 统一元数据缓存(docs/runbook/cache.md):
|
||||
// - kv_cache.fresh_until:新鲜期截止。过了 fresh_until、还没到 expires_at 的条目是「旧值」:
|
||||
// MetadataCache 先返回旧值再后台刷新(stale-while-revalidate);PgCache.get 只返回新鲜条目。null = 没有宽限期。
|
||||
// - kv_cache.pinned:关注中作品的条目,写入时按条数上限淘汰(LRU)跳过它们;到期仍会过期。
|
||||
// - cache_stat:按天 × region 汇总各进程的命中统计(进程内计数,定期把增量加进来)。
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型
|
||||
export async function up(db: Kysely<any>): Promise<void> {
|
||||
await execDdl(
|
||||
db,
|
||||
`
|
||||
alter table kv_cache add column fresh_until timestamptz;
|
||||
alter table kv_cache add column pinned boolean not null default false;
|
||||
create table cache_stat (
|
||||
day date not null,
|
||||
region text not null,
|
||||
l1_hits bigint not null default 0,
|
||||
l2_hits bigint not null default 0,
|
||||
stale_hits bigint not null default 0,
|
||||
misses bigint not null default 0,
|
||||
failures bigint not null default 0,
|
||||
refreshes bigint not null default 0,
|
||||
updated_at timestamptz not null default now(),
|
||||
primary key (day, region)
|
||||
);
|
||||
`,
|
||||
);
|
||||
}
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 同上
|
||||
export async function down(db: Kysely<any>): Promise<void> {
|
||||
await execDdl(
|
||||
db,
|
||||
`${dropTables(["cache_stat"])}
|
||||
alter table kv_cache drop column if exists pinned;
|
||||
alter table kv_cache drop column if exists fresh_until;`,
|
||||
);
|
||||
}
|
||||
+530
-17
@@ -1,4 +1,5 @@
|
||||
import { type Kysely, sql } from "kysely";
|
||||
import pg from "pg";
|
||||
import type { DB } from "./types.ts";
|
||||
|
||||
/**
|
||||
@@ -24,10 +25,37 @@ export interface SetOptions {
|
||||
ttl?: number;
|
||||
/** region 条数上限;缺省 DEFAULT_CACHE_SIZE */
|
||||
maxsize?: number;
|
||||
/**
|
||||
* 宽限期(秒,0086):> 0 时 fresh_until = now + ttl、expires_at = now + ttl + grace,
|
||||
* 宽限期内的条目由 MetadataCache 先返回旧值再后台刷新;PgCache.get 只认新鲜条目。缺省 0。
|
||||
*/
|
||||
grace?: number;
|
||||
/** 关注中作品的条目:按条数上限淘汰时跳过(到期照样过期)。缺省 false */
|
||||
pinned?: boolean;
|
||||
}
|
||||
|
||||
/** 跨进程 L1 失效通知频道(payload 见 invalidationPayload) */
|
||||
export const CACHE_INVALIDATE_CHANNEL = "umt_cache_invalidate";
|
||||
|
||||
/** 通知内容:`{"r":region,"k":key}`;k 缺省 = 整个 region;r 为 "*" = 全部。超长 key 退化为整 region */
|
||||
function invalidationPayload(region: string, key?: string): string {
|
||||
const full = JSON.stringify(key === undefined ? { r: region } : { r: region, k: key });
|
||||
return full.length > 7000 ? JSON.stringify({ r: region }) : full;
|
||||
}
|
||||
|
||||
/** getEntry 的结果:stale = 已过新鲜期、还在宽限期内 */
|
||||
export interface CacheEntry<T> {
|
||||
value: T;
|
||||
stale: boolean;
|
||||
/** 新鲜期截止(毫秒);没有宽限期时同 expiresAt */
|
||||
freshUntil: number;
|
||||
/** 硬过期(毫秒) */
|
||||
expiresAt: number;
|
||||
pinned: boolean;
|
||||
}
|
||||
|
||||
export class PgCache {
|
||||
constructor(private readonly db: AnyDb) {}
|
||||
constructor(readonly db: AnyDb) {}
|
||||
|
||||
/** MemoryBackend.set */
|
||||
async set(region: string, key: string, value: unknown, opts: SetOptions = {}): Promise<void> {
|
||||
@@ -36,36 +64,83 @@ export class PgCache {
|
||||
await this.delete(region, key);
|
||||
return;
|
||||
}
|
||||
await sql`insert into kv_cache (region, key, value, expires_at, updated_at, accessed_at)
|
||||
values (${region}, ${key}, ${JSON.stringify(value ?? null)}::jsonb, now() + make_interval(secs => ${ttl}), now(), now())
|
||||
const grace = Math.max(0, opts.grace ?? 0);
|
||||
await sql`insert into kv_cache (region, key, value, expires_at, fresh_until, pinned, updated_at, accessed_at)
|
||||
values (${region}, ${key}, ${JSON.stringify(value ?? null)}::jsonb,
|
||||
now() + make_interval(secs => ${ttl + grace}),
|
||||
${grace > 0 ? sql`now() + make_interval(secs => ${ttl})` : sql`null`},
|
||||
${opts.pinned === true}, now(), now())
|
||||
on conflict (region, key) do update set value = excluded.value, expires_at = excluded.expires_at,
|
||||
fresh_until = excluded.fresh_until, pinned = excluded.pinned,
|
||||
updated_at = excluded.updated_at, accessed_at = excluded.accessed_at`.execute(this.db);
|
||||
await this.evict(region, opts.maxsize ?? DEFAULT_CACHE_SIZE);
|
||||
}
|
||||
|
||||
/** TLRUCache 写入时的淘汰:先删已过期,再按最近使用从旧到新删到 maxsize */
|
||||
/**
|
||||
* TLRUCache 写入时的淘汰:先删已过期,再按最近使用从旧到新删到 maxsize。
|
||||
* 0086:pinned(关注中作品)的条目不参与 LRU,但计入条数;刚写入的那条总留着。
|
||||
*/
|
||||
private async evict(region: string, maxsize: number): Promise<void> {
|
||||
await sql`delete from kv_cache where region = ${region} and expires_at <= now()`.execute(this.db);
|
||||
await sql`delete from kv_cache where region = ${region} and key in (
|
||||
select key from kv_cache where region = ${region}
|
||||
order by accessed_at desc, updated_at desc offset ${Math.max(1, Math.floor(maxsize))})`.execute(this.db);
|
||||
await sql`delete from kv_cache where region = ${region} and not pinned and key in (
|
||||
select key from kv_cache where region = ${region} and not pinned
|
||||
order by accessed_at desc, updated_at desc
|
||||
offset greatest(1, ${Math.floor(maxsize)} - (select count(*) from kv_cache where region = ${region} and pinned)))`.execute(
|
||||
this.db,
|
||||
);
|
||||
}
|
||||
|
||||
/** MemoryBackend.get:未命中 / 过期返回 null;命中刷新最近使用时间 */
|
||||
/** MemoryBackend.get:未命中 / 过期 / 已过新鲜期返回 null;命中刷新最近使用时间 */
|
||||
async get<T = unknown>(region: string, key: string): Promise<T | null> {
|
||||
const r = await sql<{ value: T }>`update kv_cache set accessed_at = now()
|
||||
where region = ${region} and key = ${key} and expires_at > now() returning value`.execute(this.db);
|
||||
where region = ${region} and key = ${key} and expires_at > now()
|
||||
and (fresh_until is null or fresh_until > now()) returning value`.execute(this.db);
|
||||
return r.rows.length ? (r.rows[0]!.value as T) : null;
|
||||
}
|
||||
|
||||
/** 续期:未过期的条目把 expires_at 推到 now + ttl(只延长不缩短);返回是否有行被续期 */
|
||||
async renew(region: string, key: string, ttl: number): Promise<boolean> {
|
||||
/** 读一条(含宽限期内的旧值),命中刷新最近使用时间;MetadataCache 的 L2 读 */
|
||||
async getEntry<T = unknown>(region: string, key: string): Promise<CacheEntry<T> | null> {
|
||||
const r = await sql<{ value: T; expires_at: Date; fresh_until: Date | null; pinned: boolean; now: Date }>`
|
||||
update kv_cache set accessed_at = now()
|
||||
where region = ${region} and key = ${key} and expires_at > now()
|
||||
returning value, expires_at, fresh_until, pinned, now() as now`.execute(this.db);
|
||||
const row = r.rows[0];
|
||||
if (!row) return null;
|
||||
const expiresAt = new Date(row.expires_at).getTime();
|
||||
const freshUntil = row.fresh_until ? new Date(row.fresh_until).getTime() : expiresAt;
|
||||
return {
|
||||
value: row.value as T,
|
||||
stale: freshUntil <= new Date(row.now).getTime(),
|
||||
freshUntil,
|
||||
expiresAt,
|
||||
pinned: !!row.pinned,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 续期:新鲜的条目把过期时间推到 now + ttl(只延长不缩短,有宽限期的保留原宽限长度);返回是否有行被续期。
|
||||
* opts.pin = true 时顺带标为 pinned。旧值(已过新鲜期)不续期:该刷新就刷新。
|
||||
*/
|
||||
async renew(region: string, key: string, ttl: number, opts: { pin?: boolean } = {}): Promise<boolean> {
|
||||
if (ttl <= 0) return false;
|
||||
const r = await sql`update kv_cache set expires_at = greatest(expires_at, now() + make_interval(secs => ${ttl}))
|
||||
where region = ${region} and key = ${key} and expires_at > now()`.execute(this.db);
|
||||
const r = await sql`update kv_cache set
|
||||
expires_at = greatest(expires_at,
|
||||
now() + make_interval(secs => ${ttl}) + coalesce(expires_at - fresh_until, interval '0')),
|
||||
fresh_until = case when fresh_until is null then null
|
||||
else greatest(fresh_until, now() + make_interval(secs => ${ttl})) end,
|
||||
pinned = pinned or ${opts.pin === true}
|
||||
where region = ${region} and key = ${key} and expires_at > now()
|
||||
and (fresh_until is null or fresh_until > now())`.execute(this.db);
|
||||
return Number(r.numAffectedRows ?? 0) > 0;
|
||||
}
|
||||
|
||||
/** 通知各进程丢掉 L1(删除 / 清空 / 手动刷新时);失败不抛 */
|
||||
async notifyInvalidate(region: string, key?: string): Promise<void> {
|
||||
await sql`select pg_notify(${CACHE_INVALIDATE_CHANNEL}, ${invalidationPayload(region, key)})`
|
||||
.execute(this.db)
|
||||
.catch(() => {});
|
||||
}
|
||||
|
||||
async exists(region: string, key: string): Promise<boolean> {
|
||||
const r =
|
||||
await sql`select 1 from kv_cache where region = ${region} and key = ${key} and expires_at > now()`.execute(
|
||||
@@ -74,13 +149,16 @@ export class PgCache {
|
||||
return r.rows.length > 0;
|
||||
}
|
||||
|
||||
/** 删除并通知各进程丢掉这条 L1 */
|
||||
async delete(region: string, key: string): Promise<void> {
|
||||
await sql`delete from kv_cache where region = ${region} and key = ${key}`.execute(this.db);
|
||||
await this.notifyInvalidate(region, key);
|
||||
}
|
||||
|
||||
/** 清一个区,返回删除行数 */
|
||||
/** 清一个区,返回删除行数;通知各进程丢掉这个区的 L1 */
|
||||
async clear(region: string): Promise<number> {
|
||||
const r = await sql`delete from kv_cache where region = ${region}`.execute(this.db);
|
||||
await this.notifyInvalidate(region);
|
||||
return Number(r.numAffectedRows ?? 0);
|
||||
}
|
||||
|
||||
@@ -88,14 +166,24 @@ export class PgCache {
|
||||
async items<T = unknown>(
|
||||
region: string,
|
||||
limit = 5000,
|
||||
): Promise<{ key: string; value: T; expiresAt: Date; updatedAt: Date; bytes: number }[]> {
|
||||
const r = await sql<{ key: string; value: T; expires_at: Date; updated_at: Date; bytes: number }>`
|
||||
select key, value, expires_at, updated_at, pg_column_size(value)::int as bytes
|
||||
): Promise<{ key: string; value: T; expiresAt: Date; freshUntil: Date; updatedAt: Date; bytes: number }[]> {
|
||||
const r = await sql<{
|
||||
key: string;
|
||||
value: T;
|
||||
expires_at: Date;
|
||||
fresh_until: Date;
|
||||
updated_at: Date;
|
||||
bytes: number;
|
||||
}>`
|
||||
select key, value, expires_at, coalesce(fresh_until, expires_at) as fresh_until, updated_at,
|
||||
pg_column_size(value)::int as bytes
|
||||
from kv_cache where region = ${region} and expires_at > now() order by key limit ${limit}`.execute(this.db);
|
||||
return r.rows.map((x) => ({
|
||||
key: x.key,
|
||||
value: x.value,
|
||||
expiresAt: x.expires_at,
|
||||
/** 新鲜期截止(有宽限期的条目早于 expiresAt) */
|
||||
freshUntil: x.fresh_until,
|
||||
updatedAt: x.updated_at,
|
||||
bytes: Number(x.bytes),
|
||||
}));
|
||||
@@ -181,3 +269,428 @@ export function pgCacheFor(db: AnyDb): PgCache {
|
||||
}
|
||||
return c;
|
||||
}
|
||||
|
||||
// ── 统一元数据缓存(TMDB / 豆瓣 / Bangumi 共用;docs/runbook/cache.md) ──
|
||||
// 两级:L1 进程内存(按 region 限条数的 LRU,TTL 短)+ L2 PG kv_cache(三进程共享)。读 L1 → L2 → 源,写两级都写。
|
||||
// L1 只在跨进程失效通知(LISTEN umt_cache_invalidate)在线时启用:离线或断线重连时整体清空并停用,
|
||||
// 避免别的进程清空 / 删除后本进程还拿着旧值。
|
||||
|
||||
/** 宽限期:TTL 的 50%,上限 7 天 */
|
||||
export const STALE_GRACE_RATIO = 0.5;
|
||||
export const STALE_GRACE_MAX_SEC = 7 * 86_400;
|
||||
export const defaultGrace = (ttlSec: number) => Math.min(Math.max(0, ttlSec) * STALE_GRACE_RATIO, STALE_GRACE_MAX_SEC);
|
||||
|
||||
export interface FetchPolicy<T> {
|
||||
/** 秒 */
|
||||
ttl: number;
|
||||
/** 空结果 TTL(秒);缺省同 ttl */
|
||||
emptyTtl?: number;
|
||||
/** 空结果判定;缺省按假值(null / 空数组 / 空对象) */
|
||||
isEmpty?: (v: T) => boolean;
|
||||
maxsize?: number;
|
||||
/** 宽限期(秒);缺省 defaultGrace(本条 TTL);0 = 过期就回源 */
|
||||
grace?: number;
|
||||
/** 缺省 true:null / undefined 不缓存 */
|
||||
skipNone?: boolean;
|
||||
/** 返回真值时不缓存(业务失败的响应) */
|
||||
skipIf?: (v: T) => boolean;
|
||||
/** true:跳过读(手动刷新),照常写,并通知各进程丢掉这条 L1 */
|
||||
fresh?: boolean;
|
||||
/**
|
||||
* 关注中作品的条目:写入时标 pinned(LRU 不淘汰),L2 命中时按 ttl 续期。
|
||||
* 只在写入 / L2 命中时调用,失败按 false。
|
||||
*/
|
||||
pin?: () => Promise<boolean>;
|
||||
}
|
||||
|
||||
export interface CacheCounters {
|
||||
l1Hits: number;
|
||||
l2Hits: number;
|
||||
/** 先返回旧值(宽限期内)的次数 */
|
||||
staleHits: number;
|
||||
misses: number;
|
||||
/** 回源失败(网络、被拦、5xx 等抛错) */
|
||||
failures: number;
|
||||
/** 后台刷新次数 */
|
||||
refreshes: number;
|
||||
}
|
||||
|
||||
const zero = (): CacheCounters => ({ l1Hits: 0, l2Hits: 0, staleHits: 0, misses: 0, failures: 0, refreshes: 0 });
|
||||
|
||||
export interface MetadataCacheOptions {
|
||||
/** L1 每个 region 的条数上限,缺省 1000 */
|
||||
l1MaxPerRegion?: number;
|
||||
/** L1 TTL(毫秒),缺省 5 分钟;不超过条目在 L2 的新鲜期 */
|
||||
l1TtlMs?: number;
|
||||
/** 是否立即启用 L1(测试用;服务端由失效监听上线时打开) */
|
||||
l1?: boolean;
|
||||
now?: () => number;
|
||||
logger?: { warn(obj: object, msg: string): void };
|
||||
}
|
||||
|
||||
interface L1Entry {
|
||||
value: unknown;
|
||||
freshUntil: number;
|
||||
/** L1 内的过期(min(写入 + l1Ttl, 硬过期)) */
|
||||
expiresAt: number;
|
||||
}
|
||||
|
||||
const clone = <T>(v: T): T => (v === null || typeof v !== "object" ? v : structuredClone(v));
|
||||
|
||||
export class MetadataCache {
|
||||
private readonly l1 = new Map<string, Map<string, L1Entry>>();
|
||||
private readonly inflight = new Map<string, Promise<unknown>>();
|
||||
private readonly counters = new Map<string, CacheCounters>();
|
||||
private l1On: boolean;
|
||||
private readonly now: () => number;
|
||||
|
||||
constructor(
|
||||
readonly pg: PgCache,
|
||||
private readonly opts: MetadataCacheOptions = {},
|
||||
) {
|
||||
this.l1On = opts.l1 ?? false;
|
||||
this.now = opts.now ?? Date.now;
|
||||
}
|
||||
|
||||
// ── L1 ──
|
||||
|
||||
get l1Enabled(): boolean {
|
||||
return this.l1On;
|
||||
}
|
||||
|
||||
/** 打开 / 关闭 L1;状态变化时都整体清空 */
|
||||
setL1Enabled(on: boolean): void {
|
||||
if (this.l1On !== on) this.l1.clear();
|
||||
this.l1On = on;
|
||||
}
|
||||
|
||||
/** 丢掉本进程 L1:给 key 丢一条,不给丢整个 region,region 为 "*" 全部丢 */
|
||||
dropLocal(region: string, key?: string): void {
|
||||
if (region === "*") this.l1.clear();
|
||||
else if (key === undefined) this.l1.delete(region);
|
||||
else this.l1.get(region)?.delete(key);
|
||||
}
|
||||
|
||||
/** L1 当前条数(测试 / 诊断) */
|
||||
l1Size(region: string): number {
|
||||
return this.l1.get(region)?.size ?? 0;
|
||||
}
|
||||
|
||||
private l1Get(region: string, key: string): L1Entry | null {
|
||||
if (!this.l1On) return null;
|
||||
const m = this.l1.get(region);
|
||||
const e = m?.get(key);
|
||||
if (!m || !e) return null;
|
||||
if (e.expiresAt <= this.now()) {
|
||||
m.delete(key);
|
||||
return null;
|
||||
}
|
||||
m.delete(key);
|
||||
m.set(key, e);
|
||||
return e;
|
||||
}
|
||||
|
||||
private l1Set(region: string, key: string, value: unknown, freshUntil: number, hardExpiresAt: number): void {
|
||||
if (!this.l1On) return;
|
||||
let m = this.l1.get(region);
|
||||
if (!m) {
|
||||
m = new Map();
|
||||
this.l1.set(region, m);
|
||||
}
|
||||
m.delete(key);
|
||||
m.set(key, {
|
||||
value: clone(value),
|
||||
freshUntil,
|
||||
expiresAt: Math.min(this.now() + (this.opts.l1TtlMs ?? 5 * 60_000), hardExpiresAt),
|
||||
});
|
||||
const max = this.opts.l1MaxPerRegion ?? 1000;
|
||||
while (m.size > max) {
|
||||
const oldest = m.keys().next().value;
|
||||
if (oldest === undefined) break;
|
||||
m.delete(oldest);
|
||||
}
|
||||
}
|
||||
|
||||
// ── 统计 ──
|
||||
|
||||
private count(region: string, k: keyof CacheCounters, n = 1): void {
|
||||
let c = this.counters.get(region);
|
||||
if (!c) {
|
||||
c = zero();
|
||||
this.counters.set(region, c);
|
||||
}
|
||||
c[k] += n;
|
||||
}
|
||||
|
||||
/** 本进程还没写进 cache_stat 的计数 */
|
||||
pendingStats(): Map<string, CacheCounters> {
|
||||
return new Map([...this.counters].map(([r, c]) => [r, { ...c }]));
|
||||
}
|
||||
|
||||
/**
|
||||
* 把本进程的计数增量加进 cache_stat(按天 × region),成功后清零;失败保留到下次。
|
||||
* 顺带删掉 30 天前的统计行。返回写入的 region 数。
|
||||
*/
|
||||
async flushStats(): Promise<number> {
|
||||
const batch = [...this.counters];
|
||||
if (!batch.length) return 0;
|
||||
this.counters.clear();
|
||||
let done = 0;
|
||||
try {
|
||||
for (const [region, c] of batch) {
|
||||
await sql`insert into cache_stat (day, region, l1_hits, l2_hits, stale_hits, misses, failures, refreshes, updated_at)
|
||||
values (current_date, ${region}, ${c.l1Hits}, ${c.l2Hits}, ${c.staleHits}, ${c.misses}, ${c.failures}, ${c.refreshes}, now())
|
||||
on conflict (day, region) do update set
|
||||
l1_hits = cache_stat.l1_hits + excluded.l1_hits, l2_hits = cache_stat.l2_hits + excluded.l2_hits,
|
||||
stale_hits = cache_stat.stale_hits + excluded.stale_hits, misses = cache_stat.misses + excluded.misses,
|
||||
failures = cache_stat.failures + excluded.failures, refreshes = cache_stat.refreshes + excluded.refreshes,
|
||||
updated_at = now()`.execute(this.pg.db);
|
||||
done++;
|
||||
}
|
||||
await sql`delete from cache_stat where day < current_date - 30`.execute(this.pg.db);
|
||||
return done;
|
||||
} catch (err) {
|
||||
// 没写进去的加回去
|
||||
for (const [region, c] of batch.slice(done))
|
||||
for (const k of Object.keys(c) as (keyof CacheCounters)[]) this.count(region, k, c[k]);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
private warn(obj: object, msg: string) {
|
||||
this.opts.logger?.warn(obj, msg);
|
||||
}
|
||||
|
||||
// ── 读写 ──
|
||||
|
||||
/** 只读(不回源):L1 → L2 新鲜条目;记命中 / 未命中 */
|
||||
async get<T = unknown>(region: string, key: string): Promise<T | null> {
|
||||
const l1 = this.l1Get(region, key);
|
||||
if (l1 && l1.freshUntil > this.now()) {
|
||||
this.count(region, "l1Hits");
|
||||
return clone(l1.value) as T;
|
||||
}
|
||||
const e = await this.pg.getEntry<T>(region, key).catch((err) => {
|
||||
this.warn({ region, err: { message: (err as Error).message } }, "缓存读取失败，按未命中处理");
|
||||
return null;
|
||||
});
|
||||
if (!e || e.stale || e.value === null || e.value === undefined) {
|
||||
this.count(region, "misses");
|
||||
return null;
|
||||
}
|
||||
this.count(region, "l2Hits");
|
||||
this.l1Set(region, key, e.value, e.freshUntil, e.expiresAt);
|
||||
return e.value;
|
||||
}
|
||||
|
||||
/** 直接写两级(ttl ≤ 0 删除) */
|
||||
async set(region: string, key: string, value: unknown, opts: SetOptions = {}): Promise<void> {
|
||||
await this.pg.set(region, key, value, opts);
|
||||
const ttl = opts.ttl ?? DEFAULT_CACHE_TTL;
|
||||
if (ttl <= 0) {
|
||||
this.dropLocal(region, key);
|
||||
return;
|
||||
}
|
||||
const fresh = this.now() + ttl * 1000;
|
||||
this.l1Set(region, key, value, fresh, fresh + Math.max(0, opts.grace ?? 0) * 1000);
|
||||
}
|
||||
|
||||
async delete(region: string, key: string): Promise<void> {
|
||||
this.dropLocal(region, key);
|
||||
await this.pg.delete(region, key);
|
||||
}
|
||||
|
||||
async clear(region: string): Promise<number> {
|
||||
this.dropLocal(region);
|
||||
return await this.pg.clear(region);
|
||||
}
|
||||
|
||||
/**
|
||||
* 带回源的读(MP @cached 的调用体 + 两级 + 过期先用旧值 + 同请求合并 + 统计):
|
||||
* - L1 / L2 新鲜命中直接返回;L2 命中且 pin() 为真时续期;
|
||||
* - 宽限期内的旧值先返回,后台刷新(同 key 同时只刷一次),刷新失败继续用旧值并记一次警告;
|
||||
* - 未命中:同进程同 key 只发一次 load;load 抛错不写缓存(网络、被拦、5xx);结果按 skipNone / skipIf 决定是否写。
|
||||
* PG 读写失败按未命中 / 不写处理,不让业务请求失败。
|
||||
*/
|
||||
async fetch<T>(region: string, key: string, p: FetchPolicy<T>, load: () => Promise<T>): Promise<T> {
|
||||
if (!p.fresh) {
|
||||
const l1 = this.l1Get(region, key);
|
||||
if (l1) {
|
||||
if (l1.freshUntil > this.now()) {
|
||||
this.count(region, "l1Hits");
|
||||
return clone(l1.value) as T;
|
||||
}
|
||||
this.count(region, "staleHits");
|
||||
this.revalidate(region, key, p, load);
|
||||
return clone(l1.value) as T;
|
||||
}
|
||||
const e = await this.pg.getEntry<T>(region, key).catch((err) => {
|
||||
this.warn({ region, err: { message: (err as Error).message } }, "缓存读取失败，按未命中处理");
|
||||
return null;
|
||||
});
|
||||
if (e && this.shouldCache(p, e.value)) {
|
||||
this.l1Set(region, key, e.value, e.freshUntil, e.expiresAt);
|
||||
if (!e.stale) {
|
||||
this.count(region, "l2Hits");
|
||||
if (p.pin) await this.renewPinned(region, key, p);
|
||||
return e.value;
|
||||
}
|
||||
this.count(region, "staleHits");
|
||||
this.revalidate(region, key, p, load);
|
||||
return e.value;
|
||||
}
|
||||
}
|
||||
this.count(region, "misses");
|
||||
return (await this.loadOnce(region, key, p, load)) as T;
|
||||
}
|
||||
|
||||
private shouldCache<T>(p: FetchPolicy<T>, v: T | null | undefined): v is T {
|
||||
if ((p.skipNone ?? true) && (v === null || v === undefined)) return false;
|
||||
if (p.skipIf && v !== null && v !== undefined && p.skipIf(v)) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
private async renewPinned<T>(region: string, key: string, p: FetchPolicy<T>): Promise<void> {
|
||||
try {
|
||||
if (await p.pin!()) await this.pg.renew(region, key, p.ttl, { pin: true });
|
||||
} catch (err) {
|
||||
this.warn({ region, err: { message: (err as Error).message } }, "缓存续期失败");
|
||||
}
|
||||
}
|
||||
|
||||
private loadOnce<T>(region: string, key: string, p: FetchPolicy<T>, load: () => Promise<T>): Promise<T> {
|
||||
const id = `${region}\u0000${key}`;
|
||||
const pending = this.inflight.get(id);
|
||||
if (pending) return pending as Promise<T>;
|
||||
const run = (async () => {
|
||||
let v: T;
|
||||
try {
|
||||
v = await load();
|
||||
} catch (err) {
|
||||
this.count(region, "failures");
|
||||
throw err;
|
||||
}
|
||||
if (this.shouldCache(p, v)) await this.write(region, key, p, v);
|
||||
return v;
|
||||
})().finally(() => this.inflight.delete(id));
|
||||
this.inflight.set(id, run);
|
||||
return run;
|
||||
}
|
||||
|
||||
/** 宽限期内的后台刷新:同 key 已在刷新 / 加载就不再发 */
|
||||
private revalidate<T>(region: string, key: string, p: FetchPolicy<T>, load: () => Promise<T>): void {
|
||||
if (this.inflight.has(`${region}\u0000${key}`)) return;
|
||||
this.count(region, "refreshes");
|
||||
this.loadOnce(region, key, { ...p, fresh: false }, load).catch((err) =>
|
||||
this.warn({ region, key, err: { message: (err as Error).message } }, "缓存后台刷新失败，继续用旧值"),
|
||||
);
|
||||
}
|
||||
|
||||
private async write<T>(region: string, key: string, p: FetchPolicy<T>, v: T): Promise<void> {
|
||||
const empty = p.isEmpty ? p.isEmpty(v) : isFalsy(v);
|
||||
const ttl = empty && p.emptyTtl !== undefined ? p.emptyTtl : p.ttl;
|
||||
const grace = p.grace ?? defaultGrace(ttl);
|
||||
let pinned = false;
|
||||
if (p.pin) pinned = await p.pin().catch(() => false);
|
||||
try {
|
||||
await this.pg.set(region, key, v, { ttl, grace, pinned, ...(p.maxsize ? { maxsize: p.maxsize } : {}) });
|
||||
} catch (err) {
|
||||
this.warn({ region, err: { message: (err as Error).message } }, "缓存写入失败");
|
||||
}
|
||||
const fresh = this.now() + ttl * 1000;
|
||||
this.l1Set(region, key, v, fresh, fresh + grace * 1000);
|
||||
if (p.fresh) await this.pg.notifyInvalidate(region, key);
|
||||
}
|
||||
}
|
||||
|
||||
const metadataCaches = new WeakMap<object, MetadataCache>();
|
||||
|
||||
/** 同一个 Kysely 实例共用一个 MetadataCache(L2 用 pgCacheFor(db)) */
|
||||
export function metadataCacheFor(db: AnyDb, opts?: MetadataCacheOptions): MetadataCache {
|
||||
let c = metadataCaches.get(db);
|
||||
if (!c) {
|
||||
c = new MetadataCache(pgCacheFor(db), opts);
|
||||
metadataCaches.set(db, c);
|
||||
}
|
||||
return c;
|
||||
}
|
||||
|
||||
export interface InvalidationListenerOptions {
|
||||
connectionString: string;
|
||||
logger?: { warn(obj: object, msg: string): void; info(obj: object, msg: string): void };
|
||||
/** 断线重连退避上限,缺省 10s */
|
||||
maxReconnectMs?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* 跨进程 L1 失效:一条专用连接 LISTEN umt_cache_invalidate,收到通知丢掉对应 L1。
|
||||
* 连上时清空并启用 L1;断开时停用(清空)L1 并退避重连,重连成功再整体清一次后启用。
|
||||
*/
|
||||
export async function listenCacheInvalidation(
|
||||
cache: MetadataCache,
|
||||
opts: InvalidationListenerOptions,
|
||||
): Promise<{ stop(): Promise<void>; connected(): boolean }> {
|
||||
let client: pg.Client | null = null;
|
||||
let stopped = false;
|
||||
let timer: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
const connect = async () => {
|
||||
const c = new pg.Client({ connectionString: opts.connectionString });
|
||||
c.on("notification", (msg) => {
|
||||
if (msg.channel !== CACHE_INVALIDATE_CHANNEL) return;
|
||||
try {
|
||||
const p = JSON.parse(msg.payload ?? "{}") as { r?: string; k?: string };
|
||||
if (typeof p.r === "string") cache.dropLocal(p.r, typeof p.k === "string" ? p.k : undefined);
|
||||
} catch {
|
||||
cache.dropLocal("*");
|
||||
}
|
||||
});
|
||||
const lost = (err?: Error) => {
|
||||
if (client !== c) return;
|
||||
client = null;
|
||||
cache.setL1Enabled(false);
|
||||
c.removeAllListeners("notification");
|
||||
c.end().catch(() => {});
|
||||
if (!stopped) {
|
||||
opts.logger?.warn({ err: err ? { message: err.message } : undefined }, "缓存失效监听断开，L1 停用并准备重连");
|
||||
reconnect(500);
|
||||
}
|
||||
};
|
||||
c.on("error", lost);
|
||||
c.on("end", () => lost());
|
||||
await c.connect();
|
||||
await c.query(`listen ${CACHE_INVALIDATE_CHANNEL}`);
|
||||
client = c;
|
||||
cache.setL1Enabled(false);
|
||||
cache.setL1Enabled(true);
|
||||
};
|
||||
const reconnect = (delay: number) => {
|
||||
if (stopped || timer) return;
|
||||
timer = setTimeout(async () => {
|
||||
timer = null;
|
||||
try {
|
||||
await connect();
|
||||
opts.logger?.info({}, "缓存失效监听已重连");
|
||||
} catch (err) {
|
||||
opts.logger?.warn({ err: { message: (err as Error).message } }, "缓存失效监听重连失败");
|
||||
reconnect(Math.min(delay * 2, opts.maxReconnectMs ?? 10_000));
|
||||
}
|
||||
}, delay);
|
||||
};
|
||||
|
||||
await connect();
|
||||
return {
|
||||
connected: () => client !== null,
|
||||
async stop() {
|
||||
stopped = true;
|
||||
if (timer) clearTimeout(timer);
|
||||
const c = client;
|
||||
client = null;
|
||||
cache.setL1Enabled(false);
|
||||
if (c) {
|
||||
c.removeAllListeners("notification");
|
||||
await c.end().catch(() => {});
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -1,13 +1,24 @@
|
||||
// 统一从本包取 Kysely 与 sql,避免各包各装一份 kysely
|
||||
export { type Kysely, type RawBuilder, type Selectable, sql, type Transaction } from "kysely";
|
||||
export {
|
||||
CACHE_INVALIDATE_CHANNEL,
|
||||
type CacheCounters,
|
||||
type CachedOptions,
|
||||
type CacheEntry,
|
||||
cachedCall,
|
||||
DEFAULT_CACHE_SIZE,
|
||||
DEFAULT_CACHE_TTL,
|
||||
defaultGrace,
|
||||
type FetchPolicy,
|
||||
listenCacheInvalidation,
|
||||
MetadataCache,
|
||||
type MetadataCacheOptions,
|
||||
metadataCacheFor,
|
||||
PgCache,
|
||||
pgCacheFor,
|
||||
type SetOptions as CacheSetOptions,
|
||||
STALE_GRACE_MAX_SEC,
|
||||
STALE_GRACE_RATIO,
|
||||
} from "./cache.ts";
|
||||
export { type CreateDbOptions, createDb } from "./client.ts";
|
||||
export { createMigrator, MIGRATIONS_DIR, type MigratorOptions, migrateToLatest, resetSchema } from "./migrator.ts";
|
||||
|
||||
@@ -617,6 +617,18 @@ export interface BrowserTabTable {
|
||||
updated_at: GeneratedTimestamptz;
|
||||
}
|
||||
|
||||
export interface CacheStatTable {
|
||||
day: string;
|
||||
region: string;
|
||||
l1_hits: Generated<number>;
|
||||
l2_hits: Generated<number>;
|
||||
stale_hits: Generated<number>;
|
||||
misses: Generated<number>;
|
||||
failures: Generated<number>;
|
||||
refreshes: Generated<number>;
|
||||
updated_at: GeneratedTimestamptz;
|
||||
}
|
||||
|
||||
export interface CapabilityInvocationTable {
|
||||
id: Generated<string>;
|
||||
idempotency_key: string;
|
||||
@@ -883,6 +895,8 @@ export interface KvCacheTable {
|
||||
expires_at: Timestamptz;
|
||||
updated_at: GeneratedTimestamptz;
|
||||
accessed_at: GeneratedTimestamptz;
|
||||
fresh_until: Timestamptz | null;
|
||||
pinned: Generated<boolean>;
|
||||
}
|
||||
|
||||
export interface LibraryRootTable {
|
||||
@@ -1798,6 +1812,7 @@ export interface DB {
|
||||
browser_provider: BrowserProviderTable;
|
||||
browser_session: BrowserSessionTable;
|
||||
browser_tab: BrowserTabTable;
|
||||
cache_stat: CacheStatTable;
|
||||
capability_invocation: CapabilityInvocationTable;
|
||||
content_unit: ContentUnitTable;
|
||||
decision: DecisionTable;
|
||||
@@ -1929,6 +1944,9 @@ export type BrowserSessionUpdate = Updateable<BrowserSessionTable>;
|
||||
export type BrowserTabRow = Selectable<BrowserTabTable>;
|
||||
export type NewBrowserTab = Insertable<BrowserTabTable>;
|
||||
export type BrowserTabUpdate = Updateable<BrowserTabTable>;
|
||||
export type CacheStatRow = Selectable<CacheStatTable>;
|
||||
export type NewCacheStat = Insertable<CacheStatTable>;
|
||||
export type CacheStatUpdate = Updateable<CacheStatTable>;
|
||||
export type CapabilityInvocationRow = Selectable<CapabilityInvocationTable>;
|
||||
export type NewCapabilityInvocation = Insertable<CapabilityInvocationTable>;
|
||||
export type CapabilityInvocationUpdate = Updateable<CapabilityInvocationTable>;
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
import { Kysely, PostgresDialect, sql } from "kysely";
|
||||
import pg from "pg";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { type DB, listenCacheInvalidation, MetadataCache, migrateToLatest, PgCache } from "../src/index.ts";
|
||||
|
||||
// 统一元数据缓存(0086):两级读写、跨进程失效、过期先用旧值、同请求合并、失败不缓存、pinned 不被淘汰、命中统计。
|
||||
const url = process.env.DATABASE_URL ?? "postgres://umt:umt@localhost:55433/umt";
|
||||
const skip = process.env.UMT_SKIP_DB_TESTS === "1";
|
||||
const schema = `umt_mcache_${process.pid}_${Date.now().toString(36)}`;
|
||||
|
||||
const until = async (cond: () => boolean | Promise<boolean>, ms = 5000) => {
|
||||
const end = Date.now() + ms;
|
||||
while (!(await cond())) {
|
||||
if (Date.now() > end) throw new Error("等待超时");
|
||||
await new Promise((r) => setTimeout(r, 20));
|
||||
}
|
||||
};
|
||||
|
||||
describe.skipIf(skip)("MetadataCache(真实 PG)", () => {
|
||||
let admin: Kysely<unknown>;
|
||||
let db: Kysely<DB>;
|
||||
/** 另一个「进程」:独立连接池 */
|
||||
let db2: Kysely<DB>;
|
||||
|
||||
const pool = () =>
|
||||
new Kysely<DB>({
|
||||
dialect: new PostgresDialect({
|
||||
pool: new pg.Pool({ connectionString: url, max: 4, options: `-c search_path=${schema},public` }),
|
||||
}),
|
||||
});
|
||||
|
||||
beforeAll(async () => {
|
||||
admin = new Kysely({ dialect: new PostgresDialect({ pool: new pg.Pool({ connectionString: url, max: 1 }) }) });
|
||||
await sql.raw(`create schema ${schema}`).execute(admin);
|
||||
db = pool();
|
||||
db2 = pool();
|
||||
await migrateToLatest(db, { migrationTableSchema: schema });
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await db?.destroy();
|
||||
await db2?.destroy();
|
||||
await sql.raw(`drop schema if exists ${schema} cascade`).execute(admin);
|
||||
await admin?.destroy();
|
||||
});
|
||||
|
||||
it("读 L1 → L2 → 源,写两级;另一进程从 L2 命中;统计各级命中", async () => {
|
||||
const a = new MetadataCache(new PgCache(db), { l1: true });
|
||||
const b = new MetadataCache(new PgCache(db2), { l1: true });
|
||||
let n = 0;
|
||||
const load = async () => ({ v: ++n });
|
||||
const p = { ttl: 600 };
|
||||
expect(await a.fetch("r1", "k", p, load)).toEqual({ v: 1 });
|
||||
expect(await a.fetch("r1", "k", p, load)).toEqual({ v: 1 });
|
||||
expect(await b.fetch("r1", "k", p, load)).toEqual({ v: 1 });
|
||||
expect(await b.fetch("r1", "k", p, load)).toEqual({ v: 1 });
|
||||
expect(n).toBe(1);
|
||||
expect(a.pendingStats().get("r1")).toMatchObject({ misses: 1, l1Hits: 1, l2Hits: 0 });
|
||||
expect(b.pendingStats().get("r1")).toMatchObject({ misses: 0, l2Hits: 1, l1Hits: 1 });
|
||||
// L1 返回副本:调用方改了不污染缓存
|
||||
const got = await a.fetch<{ v: number }>("r1", "k", p, load);
|
||||
got.v = 99;
|
||||
expect(await a.fetch("r1", "k", p, load)).toEqual({ v: 1 });
|
||||
// 有宽限期:fresh_until = now + ttl,expires_at 再加 ttl 的 50%
|
||||
const r = await sql<{ g: number }>`select extract(epoch from expires_at - fresh_until)::int as g
|
||||
from kv_cache where region = 'r1' and key = 'k'`.execute(db);
|
||||
expect(r.rows[0]!.g).toBe(300);
|
||||
// 统计写进 cache_stat 后清零
|
||||
expect(await a.flushStats()).toBe(1);
|
||||
await b.flushStats();
|
||||
expect(a.pendingStats().size).toBe(0);
|
||||
const s = await sql<{ l1: number; l2: number; m: number }>`select l1_hits::int as l1, l2_hits::int as l2,
|
||||
misses::int as m from cache_stat where region = 'r1'`.execute(db);
|
||||
expect(s.rows[0]).toEqual({ l1: 4, l2: 1, m: 1 });
|
||||
});
|
||||
|
||||
it("跨进程失效:删除 / 清空通知其它进程丢 L1;监听断开时 L1 停用,重连后清空再启用", async () => {
|
||||
const a = new MetadataCache(new PgCache(db));
|
||||
const b = new MetadataCache(new PgCache(db2));
|
||||
const la = await listenCacheInvalidation(a, { connectionString: url });
|
||||
const lb = await listenCacheInvalidation(b, { connectionString: url });
|
||||
try {
|
||||
expect(a.l1Enabled && b.l1Enabled).toBe(true);
|
||||
const p = { ttl: 600 };
|
||||
await a.fetch("inv", "x", p, async () => 1);
|
||||
await a.fetch("inv", "y", p, async () => 2);
|
||||
await b.fetch("inv", "x", p, async () => 0);
|
||||
await b.fetch("inv", "y", p, async () => 0);
|
||||
expect(b.l1Size("inv")).toBe(2);
|
||||
// A 删 x:B 的 L1 只丢 x
|
||||
await a.delete("inv", "x");
|
||||
await until(() => b.l1Size("inv") === 1);
|
||||
expect(await b.fetch("inv", "x", p, async () => 10)).toBe(10);
|
||||
// A 清空整个 region
|
||||
await a.clear("inv");
|
||||
await until(() => b.l1Size("inv") === 0);
|
||||
expect(await b.fetch("inv", "y", p, async () => 20)).toBe(20);
|
||||
// 手动刷新(fresh)写入后也通知
|
||||
await a.fetch("inv", "y", { ...p, fresh: true }, async () => 21);
|
||||
await until(() => b.l1Size("inv") === 0);
|
||||
expect(await b.fetch("inv", "y", p, async () => 0)).toBe(21);
|
||||
// 断开 B 的监听连接:L1 停用;自动重连后重新启用且是空的
|
||||
expect(b.l1Size("inv")).toBe(1);
|
||||
await sql`select pg_terminate_backend(pid) from pg_stat_activity
|
||||
where query ilike 'listen umt_cache_invalidate%' and pid <> pg_backend_pid()`.execute(db2);
|
||||
await until(() => !b.l1Enabled || b.l1Size("inv") === 0);
|
||||
expect(b.l1Size("inv")).toBe(0);
|
||||
await until(() => b.l1Enabled && lb.connected(), 10_000);
|
||||
expect(b.l1Size("inv")).toBe(0);
|
||||
} finally {
|
||||
await la.stop();
|
||||
await lb.stop();
|
||||
}
|
||||
expect(a.l1Enabled).toBe(false);
|
||||
});
|
||||
|
||||
it("过期在宽限期内:先返回旧值、后台只刷一次;源失败继续用旧值并记警告", async () => {
|
||||
const warns: string[] = [];
|
||||
const c = new MetadataCache(new PgCache(db), { l1: false, logger: { warn: (_o, m) => warns.push(m) } });
|
||||
let n = 0;
|
||||
let gate: () => void = () => {};
|
||||
const slow = () =>
|
||||
new Promise<number>((ok) => {
|
||||
n++;
|
||||
gate = () => ok(100 + n);
|
||||
});
|
||||
await c.fetch("swr", "k", { ttl: 0.2, grace: 60 }, async () => 1);
|
||||
await new Promise((r) => setTimeout(r, 300));
|
||||
const p = { ttl: 600 };
|
||||
const [x, y] = await Promise.all([c.fetch("swr", "k", p, slow), c.fetch("swr", "k", p, slow)]);
|
||||
expect([x, y]).toEqual([1, 1]);
|
||||
await until(() => n === 1);
|
||||
gate();
|
||||
await until(async () => (await new PgCache(db).get("swr", "k")) === 101);
|
||||
expect(n).toBe(1);
|
||||
expect(c.pendingStats().get("swr")).toMatchObject({ staleHits: 2, refreshes: 1 });
|
||||
// 再次过期,源失败:返回旧值,不删旧值,记一次警告
|
||||
await c.fetch("swr", "k2", { ttl: 0.2, grace: 60 }, async () => "old");
|
||||
await new Promise((r) => setTimeout(r, 300));
|
||||
const failing = async (): Promise<string> => {
|
||||
throw new Error("HTTP 503");
|
||||
};
|
||||
expect(await c.fetch("swr", "k2", p, failing)).toBe("old");
|
||||
await until(() => warns.length === 1);
|
||||
expect(warns[0]).toContain("继续用旧值");
|
||||
expect(await c.fetch("swr", "k2", p, async () => "new")).toBe("old");
|
||||
// 过了宽限期 = 硬过期,回源
|
||||
await c.fetch("swr", "k3", { ttl: 0.1, grace: 0.1 }, async () => "a");
|
||||
await new Promise((r) => setTimeout(r, 300));
|
||||
expect(await c.fetch("swr", "k3", p, async () => "b")).toBe("b");
|
||||
// PgCache.get 只认新鲜条目
|
||||
await c.fetch("swr", "k4", { ttl: 0.1, grace: 60 }, async () => "x");
|
||||
await new Promise((r) => setTimeout(r, 200));
|
||||
expect(await new PgCache(db).get("swr", "k4")).toBeNull();
|
||||
expect(await new PgCache(db).exists("swr", "k4")).toBe(true);
|
||||
});
|
||||
|
||||
it("同请求合并:并发同 key 只回源一次;失败不缓存并计数", async () => {
|
||||
const c = new MetadataCache(new PgCache(db), { l1: true });
|
||||
let n = 0;
|
||||
const load = async () => {
|
||||
n++;
|
||||
await new Promise((r) => setTimeout(r, 50));
|
||||
return { ok: true };
|
||||
};
|
||||
const all = await Promise.all(Array.from({ length: 5 }, () => c.fetch("sf", "k", { ttl: 60 }, load)));
|
||||
expect(n).toBe(1);
|
||||
expect(all.every((v) => v.ok)).toBe(true);
|
||||
let m = 0;
|
||||
const fail = async () => {
|
||||
m++;
|
||||
throw new Error("ECONNRESET");
|
||||
};
|
||||
await expect(c.fetch("sf", "bad", { ttl: 60 }, fail)).rejects.toThrow("ECONNRESET");
|
||||
await expect(c.fetch("sf", "bad", { ttl: 60 }, fail)).rejects.toThrow("ECONNRESET");
|
||||
expect(m).toBe(2);
|
||||
expect(await new PgCache(db).exists("sf", "bad")).toBe(false);
|
||||
expect(c.pendingStats().get("sf")).toMatchObject({ failures: 2 });
|
||||
// skipIf / null 不缓存;空结果按 emptyTtl
|
||||
await c.fetch("sf", "none", { ttl: 60 }, async () => null);
|
||||
expect(await new PgCache(db).exists("sf", "none")).toBe(false);
|
||||
await c.fetch("sf", "empty", { ttl: 3600, emptyTtl: 60, isEmpty: (v: number[]) => !v.length }, async () => []);
|
||||
const t = await sql<{ s: number }>`select extract(epoch from fresh_until - now())::int as s
|
||||
from kv_cache where region = 'sf' and key = 'empty'`.execute(db);
|
||||
expect(t.rows[0]!.s).toBeLessThanOrEqual(60);
|
||||
});
|
||||
|
||||
it("pinned:按条数上限淘汰时跳过;L2 命中时续期", async () => {
|
||||
const pgc = new PgCache(db);
|
||||
const c = new MetadataCache(pgc, { l1: false });
|
||||
const tick = () => new Promise((r) => setTimeout(r, 5));
|
||||
await c.fetch("pin", "watched", { ttl: 100, maxsize: 2, pin: async () => true }, async () => "w");
|
||||
for (const k of ["a", "b", "c", "d"]) {
|
||||
await tick();
|
||||
await c.fetch("pin", k, { ttl: 100, maxsize: 2 }, async () => k);
|
||||
}
|
||||
expect((await pgc.items("pin")).map((i) => i.key).sort()).toEqual(["d", "watched"]);
|
||||
// 续期:新鲜条目命中且 pin 为真 → 过期时间推到 now + ttl(保留宽限长度)
|
||||
await sql`update kv_cache set fresh_until = now() + interval '10 seconds', expires_at = now() + interval '60 seconds'
|
||||
where region = 'pin' and key = 'watched'`.execute(db);
|
||||
expect(await c.fetch("pin", "watched", { ttl: 3600, pin: async () => true }, async () => "x")).toBe("w");
|
||||
const r = await sql<{ f: number; g: number }>`select extract(epoch from fresh_until - now())::int as f,
|
||||
extract(epoch from expires_at - fresh_until)::int as g from kv_cache where region = 'pin' and key = 'watched'`.execute(
|
||||
db,
|
||||
);
|
||||
expect(r.rows[0]!.f).toBeGreaterThan(3500);
|
||||
expect(r.rows[0]!.g).toBe(50);
|
||||
});
|
||||
});
|
||||
@@ -132,6 +132,8 @@ const EXPECTED_TABLES = [
|
||||
"episode_mapping_rule",
|
||||
// 站点请求失败留样(0077)
|
||||
"site_failure_sample",
|
||||
// 缓存命中统计(0086,按天 × region)
|
||||
"cache_stat",
|
||||
].sort();
|
||||
|
||||
/** 不走 entityTable() 的表:追加型/引擎自有表,没有 uuid id + created_at/updated_at 三件套 */
|
||||
@@ -167,6 +169,8 @@ const NON_ENTITY_TABLES = [
|
||||
"site_request_count",
|
||||
// 失败留样:只追加,只有 created_at
|
||||
"site_failure_sample",
|
||||
// 缓存命中统计:(day, region) 主键
|
||||
"cache_stat",
|
||||
];
|
||||
|
||||
describe.skipIf(skip)("schema migrations (real PG)", () => {
|
||||
|
||||
@@ -7,6 +7,8 @@
|
||||
// - 剧集:作品目录 tvshow.nfo、poster.jpg、fanart.jpg;季目录 season.nfo;季海报同时写作品目录
|
||||
// `season01-poster.jpg`(特别篇 `season-specials-poster.jpg`)与季目录 `poster.jpg`(MP 同);
|
||||
// - 每集:`<视频名>.nfo`;剧照 `<视频名>-thumb.jpg`(Kodi / Jellyfin / Emby 都认;MP 写 `<视频名>.jpg`)。
|
||||
// - 标志(TMDB logos,MP 只在接 fanart.tv 时写):电影写在影片目录、剧集写在剧集根目录,文件名 `clearlogo.png`
|
||||
// (Kodi / Emby / Jellyfin 通用);平铺 / 库根的电影用 `<视频名>-clearlogo.png`。
|
||||
// 视频直接在库根(没有作品目录)或电影目录里还有别的作品时,电影图片改用 `<视频名>-poster.jpg` / `<视频名>-fanart.jpg`,
|
||||
// 剧集不写 tvshow / 季文件(不能往库根里放 tvshow.nfo)。
|
||||
|
||||
@@ -42,6 +44,7 @@ export type ScrapeTargetRole =
|
||||
| "episode_nfo"
|
||||
| "poster"
|
||||
| "fanart"
|
||||
| "clearlogo"
|
||||
| "season_poster"
|
||||
| "episode_thumb";
|
||||
|
||||
@@ -61,6 +64,8 @@ export interface ScrapeVideo {
|
||||
export interface ScrapeImages {
|
||||
posterPath?: string | null;
|
||||
backdropPath?: string | null;
|
||||
/** TMDB 标志(PNG) */
|
||||
logoPath?: string | null;
|
||||
/** 季号 → 季海报路径 */
|
||||
seasonPosters?: ReadonlyMap<number, string | null>;
|
||||
/** "季:集" → 剧照路径 */
|
||||
@@ -165,7 +170,7 @@ export function planScrapeTargets(input: ScrapeLayoutInput): ScrapeTarget[] {
|
||||
}
|
||||
if (!input.options.images || input.parts?.work === false) return out;
|
||||
const inWork = workDir && !input.sharedDir && videos.some((v) => isUnder(v.path, workDir));
|
||||
const art = (role: "poster" | "fanart", p: string | null | undefined) => {
|
||||
const art = (role: "poster" | "fanart" | "clearlogo", p: string | null | undefined) => {
|
||||
if (!p) return;
|
||||
if (inWork) add({ path: join(workDir, `${role}${extOf(p)}`), kind: "image", role, imagePath: p });
|
||||
else
|
||||
@@ -182,6 +187,7 @@ export function planScrapeTargets(input: ScrapeLayoutInput): ScrapeTarget[] {
|
||||
};
|
||||
art("poster", img.posterPath);
|
||||
art("fanart", img.backdropPath);
|
||||
art("clearlogo", img.logoPath);
|
||||
return out;
|
||||
}
|
||||
|
||||
@@ -206,6 +212,13 @@ export function planScrapeTargets(input: ScrapeLayoutInput): ScrapeTarget[] {
|
||||
role: "fanart",
|
||||
imagePath: img.backdropPath,
|
||||
});
|
||||
if (img.logoPath)
|
||||
add({
|
||||
path: join(workDir, `clearlogo${extOf(img.logoPath)}`),
|
||||
kind: "image",
|
||||
role: "clearlogo",
|
||||
imagePath: img.logoPath,
|
||||
});
|
||||
}
|
||||
}
|
||||
// 季:按视频所在目录(不等于作品目录的才算季目录);没有作品目录时只写季目录里的 season.nfo / poster
|
||||
|
||||
@@ -199,6 +199,60 @@ describe("刮削目标布局", () => {
|
||||
expect(atRoot.map((x) => x.path)).toEqual(["x.nfo", "x-poster.jpg", "x-fanart.png"]);
|
||||
});
|
||||
|
||||
it("标志 clearlogo.png:电影写影片目录、剧集写剧集根目录;库根 / 平铺的电影用 <视频名>-clearlogo.png;选中季 / 集不写", () => {
|
||||
const withLogo = { ...images, logoPath: "/logo.png" };
|
||||
const movie = planScrapeTargets({
|
||||
kind: "movie",
|
||||
workDir: "星际穿越 (2014)",
|
||||
videos: [{ path: "星际穿越 (2014)/星际穿越 (2014) - 1080p.mkv" }],
|
||||
images: withLogo,
|
||||
options: { images: true, episodeThumbs: true },
|
||||
});
|
||||
expect(movie.filter((x) => x.role === "clearlogo").map((x) => [x.path, x.imagePath])).toEqual([
|
||||
["星际穿越 (2014)/clearlogo.png", "/logo.png"],
|
||||
]);
|
||||
const atRoot = planScrapeTargets({
|
||||
kind: "movie",
|
||||
workDir: null,
|
||||
videos: [{ path: "x.mkv" }],
|
||||
images: withLogo,
|
||||
options: { images: true, episodeThumbs: true },
|
||||
});
|
||||
expect(atRoot.map((x) => x.path)).toEqual(["x.nfo", "x-poster.jpg", "x-fanart.png", "x-clearlogo.png"]);
|
||||
const tv = planScrapeTargets({
|
||||
kind: "tv",
|
||||
workDir: "S",
|
||||
videos: [{ path: "S/Season 1/a.mkv", season: 1, episode: 1 }],
|
||||
images: withLogo,
|
||||
options: { images: true, episodeThumbs: true },
|
||||
});
|
||||
expect(tv.slice(0, 4).map((x) => x.path)).toEqual([
|
||||
"S/tvshow.nfo",
|
||||
"S/poster.jpg",
|
||||
"S/fanart.png",
|
||||
"S/clearlogo.png",
|
||||
]);
|
||||
const seasonOnly = planScrapeTargets({
|
||||
kind: "tv",
|
||||
workDir: "S",
|
||||
videos: [{ path: "S/Season 1/a.mkv", season: 1, episode: 1 }],
|
||||
images: withLogo,
|
||||
options: { images: true, episodeThumbs: true },
|
||||
parts: { work: false },
|
||||
});
|
||||
expect(seasonOnly.some((x) => x.role === "clearlogo")).toBe(false);
|
||||
// 关图片不写;没有标志不写
|
||||
const noImages = planScrapeTargets({
|
||||
kind: "tv",
|
||||
workDir: "S",
|
||||
videos: [{ path: "S/Season 1/a.mkv", season: 1, episode: 1 }],
|
||||
images: withLogo,
|
||||
options: { images: false, episodeThumbs: false },
|
||||
});
|
||||
expect(noImages.some((x) => x.role === "clearlogo")).toBe(false);
|
||||
expect(movie.length).toBe(4);
|
||||
});
|
||||
|
||||
it("剧集视频在库根:不写 tvshow / 季文件,只写集 NFO", () => {
|
||||
const t = planScrapeTargets({
|
||||
kind: "tv",
|
||||
|
||||
@@ -60,6 +60,8 @@ export interface TmdbFixture {
|
||||
genre_ids?: number[];
|
||||
/** 剧集组(`/tv/{id}/episode_groups`、`/tv/episode_group/{id}`);组内集按 TMDB 默认季集号引用 seasons 里的集 */
|
||||
episode_groups?: TmdbFixtureEpisodeGroup[];
|
||||
/** 标志(append_to_response=images 与 `/{type}/{id}/images` 按 include_image_language 过滤);缺省没有 */
|
||||
logos?: { file_path: string; iso_639_1: string | null; vote_average?: number; vote_count?: number }[];
|
||||
}
|
||||
|
||||
export interface TmdbFixtureEpisodeGroup {
|
||||
|
||||
@@ -22,7 +22,7 @@ const notFound: MockReply = {
|
||||
/** 列表接口一页的条数(同 TMDB) */
|
||||
const LIST_PAGE = 20;
|
||||
|
||||
/** mock TMDB v3:/3/trending、/3/discover、电影 / 剧集榜单(按热度,一页 20 条)、/3/search/{movie,tv}、/3/find/:imdbId、/3/{movie,tv}/:id(append_to_response)、/3/tv/:id/season/:n、/3/tv/:id/episode_groups、/3/tv/episode_group/:id */
|
||||
/** mock TMDB v3:/3/trending、/3/discover、电影 / 剧集榜单(按热度,一页 20 条)、/3/search/{movie,tv}、/3/find/:imdbId、/3/{movie,tv}/:id(append_to_response)、/3/{movie,tv}/:id/images、/3/tv/:id/season/:n、/3/tv/:id/episode_groups、/3/tv/episode_group/:id */
|
||||
export async function createMockTmdb(opts: MockTmdbOptions = {}): Promise<MockTmdb> {
|
||||
const fx = opts.fixtures ?? TMDB_FIXTURES;
|
||||
const apiKey = opts.apiKey ?? "test-tmdb-key";
|
||||
@@ -59,6 +59,21 @@ export async function createMockTmdb(opts: MockTmdbOptions = {}): Promise<MockTm
|
||||
};
|
||||
};
|
||||
|
||||
/** images.logos 按 include_image_language(逗号分隔,null = 无文字)过滤;不给参数全给 */
|
||||
const images = (f: TmdbFixture, q: Record<string, string>) => {
|
||||
const langs = q.include_image_language ? q.include_image_language.split(",") : null;
|
||||
const logos = (f.logos ?? [])
|
||||
.filter((l) => !langs || langs.includes(l.iso_639_1 ?? "null"))
|
||||
.map((l) => ({
|
||||
...l,
|
||||
vote_average: l.vote_average ?? 5,
|
||||
vote_count: l.vote_count ?? 1,
|
||||
width: 500,
|
||||
height: 200,
|
||||
}));
|
||||
return { backdrops: [], posters: [], logos };
|
||||
};
|
||||
|
||||
const listItem = (f: TmdbFixture, lang?: string) => ({
|
||||
...brief(f, lang),
|
||||
genre_ids: f.genre_ids ?? [],
|
||||
@@ -222,6 +237,13 @@ export async function createMockTmdb(opts: MockTmdbOptions = {}): Promise<MockTm
|
||||
};
|
||||
},
|
||||
],
|
||||
[
|
||||
/^\/3\/(movie|tv)\/(\d+)\/images$/,
|
||||
(m, q) => {
|
||||
const f = find(m[1]!, m[2]!);
|
||||
return f ? { json: { id: f.id, ...images(f, q) } } : notFound;
|
||||
},
|
||||
],
|
||||
[
|
||||
/^\/3\/tv\/(\d+)\/season\/(\d+)$/,
|
||||
(m, q) => {
|
||||
@@ -298,6 +320,7 @@ export async function createMockTmdb(opts: MockTmdbOptions = {}): Promise<MockTm
|
||||
f.type === "movie" ? { titles: f.alternative_titles } : { results: f.alternative_titles };
|
||||
}
|
||||
if (append.has("external_ids")) out.external_ids = f.external_ids;
|
||||
if (append.has("images")) out.images = images(f, q);
|
||||
if (append.has("translations")) {
|
||||
const key = f.type === "movie" ? "title" : "name";
|
||||
out.translations = {
|
||||
|
||||
Reference in new issue
Block a user