diff --git a/apps/server/src/main.ts b/apps/server/src/main.ts index a2b206cb..de1fd89f 100644 --- a/apps/server/src/main.ts +++ b/apps/server/src/main.ts @@ -84,6 +84,7 @@ const { createApp } = await import("./http/app.ts"); const { PgEventBus } = await import("./http/sse-pg.ts"); const { createLogger } = await import("./logger.ts"); const { SERVER_MODULES } = await import("./modules/index.ts"); +const { startMetadataCache } = await import("./shared-cache.ts"); const { diffCounts, enableReadOnlyReconcile, exportMappings, writeMappingExport } = await import("./ops.ts"); const { auditStorageWarnings, checkStorage, loadMountedPaths, runPreflight } = await import("./preflight.ts"); const { startWorker } = await import("./worker/index.ts"); @@ -219,6 +220,8 @@ try { Deno.exit(1); } const { ctx, outbox, engine } = booted; +// 统一元数据缓存:LISTEN 跨进程失效(在线才启用进程内一级缓存)+ 命中统计定期落库 +stops.push(await startMetadataCache(ctx, config.DATABASE_URL)); if (role === "api" || role === "all") { const app = createApp({ diff --git a/apps/server/src/modules/acquisition-organize/services/scrape-input.ts b/apps/server/src/modules/acquisition-organize/services/scrape-input.ts index dd4c0a97..5100965a 100644 --- a/apps/server/src/modules/acquisition-organize/services/scrape-input.ts +++ b/apps/server/src/modules/acquisition-organize/services/scrape-input.ts @@ -152,11 +152,12 @@ export function episodeMetadataMissing(u: UnitForNfo | undefined): boolean { return !u || (!filled(u.metadata) && !str(u.title) && !u.air_date); } -/** 图片路径:作品海报 / 背景图,季海报,集剧照 */ +/** 图片路径:作品海报 / 背景图 / 标志,季海报,集剧照 */ export function imagePathsOf(entityMeta: unknown) { return { posterPath: str(val(entityMeta, "posterPath")) ?? str(val(entityMeta, "posterUrl")), backdropPath: str(val(entityMeta, "backdropPath")), + logoPath: str(val(entityMeta, "logoPath")), }; } export const unitPosterPath = (m: unknown) => str(val(m, "posterPath")); diff --git a/apps/server/src/modules/cache/index.ts b/apps/server/src/modules/cache/index.ts index 097133d9..5f38f5ab 100644 --- a/apps/server/src/modules/cache/index.ts +++ b/apps/server/src/modules/cache/index.ts @@ -14,7 +14,16 @@ import { dataPaths } from "../../config.ts"; import { externalFetch } from "../../egress.ts"; import { requireScope } from "../../http/auth.ts"; import { errorBody } from "../../http/errors.ts"; -import { CACHE_REGIONS, sharedCache, storageSettings, tmdbCachePolicy } from "../../shared-cache.ts"; +import { + CACHE_REGIONS, + METADATA_REGIONS, + metadataCache, + type RegionHitStats, + regionHitStats, + sharedCache, + storageSettings, + tmdbCachePolicy, +} from "../../shared-cache.ts"; import { registerScheduledJob } from "../schedules/registry.ts"; import type { ModuleApp, ServerContext, ServerModule } from "../types.ts"; import { ImageCache, ImageProxyError } from "./images.ts"; @@ -72,6 +81,13 @@ function route( const str = (v: unknown) => (typeof v === "string" && v ? v : null); +/** 命中统计 → 接口形状;命中率 = (L1 + L2 + 旧值) / (L1 + L2 + 旧值 + 未命中) */ +function hitView(h: RegionHitStats | undefined) { + const v = h ?? { l1Hits: 0, l2Hits: 0, staleHits: 0, misses: 0, failures: 0, refreshes: 0 }; + const hit = v.l1Hits + v.l2Hits + v.staleHits; + return { ...v, hitRate: hit + v.misses > 0 ? hit / (hit + v.misses) : null }; +} + export function createCacheModule(opts: CacheModuleOptions = {}): ServerModule & { images(ctx: ServerContext): ImageCache; } { @@ -166,7 +182,7 @@ export function createCacheModule(opts: CacheModuleOptions = {}): ServerModule & return out; }); route(app, Api.cache.tmdbClear, async () => { - const cache = sharedCache(ctx); + const cache = metadataCache(ctx); const deleted = (await cache.clear(CACHE_REGIONS.tmdbRecognize)) + (await cache.clear(CACHE_REGIONS.tmdbRequest)); ctx.logger.info({ deleted }, "TMDB 缓存已清空"); @@ -194,18 +210,30 @@ export function createCacheModule(opts: CacheModuleOptions = {}): ServerModule & coalesce(sum(pg_column_size(value) + pg_column_size(key)), 0)::text as bytes, max(updated_at) as last from kv_cache group by region order by region`.execute(ctx.db); - return { - items: r.rows.map((x) => ({ - region: x.region, - entries: Number(x.entries), - expired: Number(x.expired), - bytes: Number(x.bytes), - lastWriteAt: x.last ? new Date(x.last).toISOString() : null, - })), + // 统一元数据缓存的 region 带最近 7 天命中统计(口径见 contracts CacheHitStats) + const hits = await regionHitStats(ctx).catch(() => new Map()); + const withHits = (region: string) => { + const h = hits.get(region); + if (!h) return METADATA_REGIONS.includes(region) ? { hits: hitView(undefined) } : {}; + return { hits: hitView(h) }; }; + const items = r.rows.map((x) => ({ + region: x.region, + entries: Number(x.entries), + expired: Number(x.expired), + bytes: Number(x.bytes), + lastWriteAt: x.last ? new Date(x.last).toISOString() : null, + ...withHits(x.region), + })); + // 条目已清空、但最近有读取统计的元数据 region 也列出来 + for (const region of hits.keys()) + if (!items.some((i) => i.region === region)) + items.push({ region, entries: 0, expired: 0, bytes: 0, lastWriteAt: null, ...withHits(region) }); + items.sort((a, b) => a.region.localeCompare(b.region)); + return { items }; }); route(app, Api.cache.regionClear, async ({ params }) => { - const deleted = await sharedCache(ctx).clear(params.region); + const deleted = await metadataCache(ctx).clear(params.region); ctx.logger.info({ region: params.region, deleted }, "缓存区已清空"); return { deleted }; }); @@ -217,7 +245,7 @@ export function createCacheModule(opts: CacheModuleOptions = {}): ServerModule & lane: "maintenance", defaults: { name: "缓存清理", - description: "删除过期的 TMDB 与识别缓存", + description: "删除过期的元数据与其它缓存", group: "maintain", schedule: { type: "interval", everyMs: 10 * 60_000 }, firstRun: "after_interval", diff --git a/apps/server/src/modules/search/discover.ts b/apps/server/src/modules/search/discover.ts index 6e72634c..3126b7f4 100644 --- a/apps/server/src/modules/search/discover.ts +++ b/apps/server/src/modules/search/discover.ts @@ -24,10 +24,10 @@ import { type DiscoverTmdbQuery, type InventoryEntry, } from "@umt/contracts"; -import { pgCacheFor } from "@umt/db"; import { type IdentityCandidate, parseRelease, resolveIdentity } from "@umt/domain"; import { ApiException } from "../../http/errors.ts"; import { runSandbox } from "../../sandbox/client.ts"; +import { CACHE_REGIONS, metaCachePolicy, metadataCache, storageSettings } from "../../shared-cache.ts"; import type { SourcesRuntime } from "../sources-identity/runtime.ts"; import { subjectRecall } from "../sources-identity/service/id-escalation.ts"; import { metaClients, toCached } from "../sources-identity/service/meta-subjects.ts"; @@ -41,9 +41,8 @@ import { localEntities } from "./media.ts"; // - 每条结果按 TMDB 身份标本地状态(已订阅 / 媒体库已有 / 下载中);豆瓣 / bgm 条目匹配过 TMDB 的才知道(discover.match)。 // 本地状态每次现查,不进缓存。 -/** 豆瓣 / bgm 条目 → TMDB 作品的确定匹配(点开时写入,列表据此标本地状态) */ -const MATCH_REGION = "discover.match"; -const MATCH_TTL_SEC = 30 * 86_400; +/** 豆瓣 / bgm 条目 → TMDB 作品的确定匹配(点开时写入,列表据此标本地状态);TTL 按「豆瓣 / bgm × 详情」 */ +const MATCH_REGION = CACHE_REGIONS.discoverMatch; const MATCH_MAX = 20_000; export type InventoryLookup = (ids: string[]) => Promise>; @@ -108,12 +107,12 @@ const fromTmdb = (t: TmdbListItem): DiscoverItem => ({ /** 豆瓣 / bgm 条目补上记住的 TMDB 匹配,再按 TMDB 身份查本地作品与库存 */ export async function annotate(rt: SourcesRuntime, items: DiscoverItem[], inventory: InventoryLookup) { - const cache = pgCacheFor(rt.ctx.db); + const cache = metadataCache(rt.ctx); await Promise.all( items .filter((it) => !it.tmdb && it.source !== "tmdb") .map(async (it) => { - const hit = await cache.get(MATCH_REGION, `${it.source}:${it.id}`).catch(() => null); + const hit = await cache.get(MATCH_REGION, `${it.source}:${it.id}`); if (hit) it.tmdb = { type: hit.type, id: hit.id }; }), ); @@ -257,9 +256,9 @@ const view = (m: MetadataCandidate): DiscoverMatchCandidate => ({ }); export async function matchTmdb(rt: SourcesRuntime, req: DiscoverMatchRequest): Promise { - const cache = pgCacheFor(rt.ctx.db); + const cache = metadataCache(rt.ctx); const memoKey = `${req.source}:${req.id}`; - const memo = await cache.get(MATCH_REGION, memoKey).catch(() => null); + const memo = await cache.get(MATCH_REGION, memoKey); if (memo) return { match: memo, candidates: [memo] }; const subject = toCached({ ...req, cover: null, rating: null, date: null, episodes: null, url: "" }, rt.now()); const base = parseRelease(req.title); @@ -278,6 +277,9 @@ export async function matchTmdb(rt: SourcesRuntime, req: DiscoverMatchRequest): const best = res.status === "auto" && res.best ? byCand.get(res.best.candidate) : undefined; const match = best ? view(best) : null; // 只记确定的匹配;不确定 / 失败不记,下次重算 - if (match) await cache.set(MATCH_REGION, memoKey, match, { ttl: MATCH_TTL_SEC, maxsize: MATCH_MAX }).catch(() => {}); + if (match) { + const ttl = metaCachePolicy((await storageSettings(rt.ctx)).cacheTtl, req.source, "detail").ttl; + await cache.set(MATCH_REGION, memoKey, match, { ttl, maxsize: MATCH_MAX }).catch(() => {}); + } return DiscoverMatchResult.parse({ match, candidates: ranked.map(view) }); } diff --git a/apps/server/src/modules/sources-identity/runtime.ts b/apps/server/src/modules/sources-identity/runtime.ts index 64854cec..cef5cb14 100644 --- a/apps/server/src/modules/sources-identity/runtime.ts +++ b/apps/server/src/modules/sources-identity/runtime.ts @@ -11,19 +11,13 @@ import { type SiteDefinition, type SiteRateLimiter, type TmdbClient, + type TmdbRequestKind, type TmdbSharedCache, } from "@umt/adapters"; import { DEFAULT_SETTINGS, type SitesSettings } from "@umt/contracts"; import { pgTokenBucketFor, type SiteRow, sql } from "@umt/db"; import { egressKey, externalFetch, siteEgress, siteFetchFor } from "../../egress.ts"; -import { - CACHE_REGIONS, - renewTrackedTmdbDetail, - sharedCache, - type TmdbCachePolicy, - tmdbCachePolicy, - tmdbDetailRef, -} from "../../shared-cache.ts"; +import { CACHE_REGIONS, metadataCache, type TmdbCachePolicy, tmdbCachePolicy, watchedPin } from "../../shared-cache.ts"; import type { ServerContext } from "../types.ts"; import { CloudflareGate, withClearance } from "./service/cloudflare.ts"; import { failureSampleSink } from "./service/failure-sample.ts"; @@ -165,6 +159,8 @@ export class SourcesRuntime { ttlMs: policy.ttl * 1000, searchTtlMs: policy.searchTtl * 1000, emptyTtlMs: policy.emptyTtl * 1000, + listTtlMs: policy.listTtl * 1000, + imageTtlMs: policy.imageTtl * 1000, }, sharedCache: tmdbSharedCache(this.ctx, policy), } @@ -420,28 +416,34 @@ export class SourcesRuntime { } /** - * TMDB 请求响应缓存:PG kv_cache region=tmdb.request;PG 出错按未命中 / 不写处理,不让 TMDB 请求失败。 - * 详情命中且属于追踪中的作品时续期(CACHE-2),续期失败只记 debug。 + * TMDB 请求响应缓存:统一元数据缓存 region=tmdb.request(L1 进程内存 + L2 kv_cache,过期先用旧值后台刷新、同请求合并)。 + * TTL 按请求类别取策略表(搜索 / 详情 / 列表 / 图片);关注中作品的条目写入 pinned、命中续期。 + * PG 出错按未命中 / 不写处理,不让 TMDB 请求失败。get / set 保留给不走 fetch 的旧调用方。 */ -function tmdbSharedCache(ctx: ServerContext, policy: TmdbCachePolicy): TmdbSharedCache { - const cache = sharedCache(ctx); - const warn = (err: unknown) => - ctx.logger.warn({ err: { message: (err as Error).message } }, "TMDB 缓存读写失败，按未命中处理"); +export function tmdbSharedCache(ctx: ServerContext, policy: TmdbCachePolicy): TmdbSharedCache { + const cache = metadataCache(ctx); + const region = CACHE_REGIONS.tmdbRequest; + const ttlOf = (kind: TmdbRequestKind) => + kind === "search" + ? { ttl: policy.searchTtl, emptyTtl: policy.emptyTtl } + : kind === "list" + ? { ttl: policy.listTtl, emptyTtl: policy.listEmptyTtl } + : kind === "image" + ? { ttl: policy.imageTtl, emptyTtl: policy.emptyTtl } + : { ttl: policy.ttl, emptyTtl: policy.emptyTtl }; return { - get: async (key) => { - const v = await cache.get(CACHE_REGIONS.tmdbRequest, key).catch((e) => { - warn(e); - return null; - }); - if (v !== null && tmdbDetailRef(key)) { - await renewTrackedTmdbDetail(ctx, key, policy.ttl).catch((e) => - ctx.logger.debug({ err: { message: (e as Error).message } }, "TMDB 详情缓存续期失败"), - ); - } - return v; - }, + fetch: (req, load) => + cache.fetch( + region, + req.key, + { ...ttlOf(req.kind), isEmpty: req.isEmpty, maxsize: policy.maxsize, pin: watchedPin(ctx, req.key) }, + load, + ), + get: (key) => cache.get(region, key), set: (key, value, ttlMs) => - cache.set(CACHE_REGIONS.tmdbRequest, key, value, { ttl: ttlMs / 1000, maxsize: policy.maxsize }).catch(warn), + cache + .set(region, key, value, { ttl: ttlMs / 1000, maxsize: policy.maxsize }) + .catch((e) => ctx.logger.warn({ err: { message: (e as Error).message } }, "TMDB 缓存读写失败，按未命中处理")), }; } diff --git a/apps/server/src/modules/sources-identity/service/douban-subject.ts b/apps/server/src/modules/sources-identity/service/douban-subject.ts index 1e12d1b4..56b32153 100644 --- a/apps/server/src/modules/sources-identity/service/douban-subject.ts +++ b/apps/server/src/modules/sources-identity/service/douban-subject.ts @@ -1,5 +1,6 @@ import { formatDetailPage } from "@umt/adapters"; -import { pgCacheFor } from "@umt/db"; +import { DEFAULT_STORAGE_SETTINGS } from "@umt/contracts"; +import { metaCachePolicy, metadataCache, storageSettings } from "../../../shared-cache.ts"; import type { ServerContext } from "../../types.ts"; // ID-3 豆瓣条目缓存(kv_cache region douban.subject,key = 豆瓣 subject id):只存标题 / 别名 / 年份,供「详情页只有豆瓣 ID」时 @@ -10,8 +11,8 @@ import type { ServerContext } from "../../types.ts"; // - douban_api:豆瓣移动版接口(识别升级时缓存没有才查)。bgm.tv 条目同形,存 region bangumi.subject。 export const DOUBAN_SUBJECT_REGION = "douban.subject"; -const TTL_SEC = 30 * 86_400; -const MAX_ENTRIES = 20_000; +/** 条目 TTL 取策略表「豆瓣 / bgm × 详情」(storage.cacheTtl.metaDetailDays,缺省 30 天) */ +export const META_SUBJECT_MAX_ENTRIES = 20_000; export interface DoubanSubject { id: string; @@ -92,16 +93,24 @@ export function doubanSubjectFromSnippet( }; } -type Db = Pick; +type Db = Pick & Partial>; -/** 写入缓存;站点详情来源优先(已有 site_detail 条目时不用搜索摘要覆盖) */ +/** 条目 TTL(秒):有设置读设置,否则缺省 */ +export async function metaSubjectTtl(ctx: Db): Promise { + const t = ctx.settings + ? (await storageSettings({ settings: ctx.settings })).cacheTtl + : DEFAULT_STORAGE_SETTINGS.cacheTtl; + return metaCachePolicy(t, "douban", "detail").ttl; +} + +/** 写入缓存(统一元数据缓存,两级);站点详情来源优先(已有 site_detail 条目时不用搜索摘要覆盖) */ export async function saveDoubanSubject(ctx: Db, s: DoubanSubject, region = DOUBAN_SUBJECT_REGION): Promise { - const cache = pgCacheFor(ctx.db); + const cache = metadataCache(ctx); if (s.source === "web_snippet") { const prev = await cache.get(region, s.id); if (prev?.source === "site_detail") return; } - await cache.set(region, s.id, s, { ttl: TTL_SEC, maxsize: MAX_ENTRIES }); + await cache.set(region, s.id, s, { ttl: await metaSubjectTtl(ctx), maxsize: META_SUBJECT_MAX_ENTRIES }); } export async function loadDoubanSubject( @@ -110,7 +119,7 @@ export async function loadDoubanSubject( region = DOUBAN_SUBJECT_REGION, ): Promise { try { - const v = await pgCacheFor(ctx.db).get(region, id); + const v = await metadataCache(ctx).get(region, id); return v && typeof v.title === "string" && v.title ? v : null; } catch { return null; diff --git a/apps/server/src/modules/sources-identity/service/meta-subjects.ts b/apps/server/src/modules/sources-identity/service/meta-subjects.ts index efe038b1..4f7df5ce 100644 --- a/apps/server/src/modules/sources-identity/service/meta-subjects.ts +++ b/apps/server/src/modules/sources-identity/service/meta-subjects.ts @@ -4,51 +4,56 @@ import { createDoubanClient, type DoubanClient, type MetaRequestCache, + MetaSourceError, type MetaSubject, RequestLimiter, } from "@umt/adapters"; -import { cachedCall, pgCacheFor } from "@umt/db"; import { normalizeTitle, toSimplified } from "@umt/domain"; -import { storageSettings } from "../../../shared-cache.ts"; +import { CACHE_REGIONS, metaCachePolicy, metadataCache, storageSettings } from "../../../shared-cache.ts"; import type { SourcesRuntime } from "../runtime.ts"; -import { DOUBAN_SUBJECT_REGION, type DoubanSubject, loadDoubanSubject, saveDoubanSubject } from "./douban-subject.ts"; +import { + DOUBAN_SUBJECT_REGION, + type DoubanSubject, + loadDoubanSubject, + META_SUBJECT_MAX_ENTRIES, +} from "./douban-subject.ts"; // 豆瓣 / bgm.tv 作为辅助信息源(对照 MP:豆瓣、Bangumi 条目 → 名称 + 年份 → TMDB 投影;身份仍以 TMDB 为准)。 // 条目先查缓存(30 天),没有才在线查一次;查不到 / 网络失败记 6 小时负缓存,不反复打外站。 -export const BANGUMI_SUBJECT_REGION = "bangumi.subject"; -const MISS_REGION = "meta.subject_miss"; -const MISS_TTL_SEC = 6 * 3600; +export const BANGUMI_SUBJECT_REGION = CACHE_REGIONS.bangumiSubject; +const MISS_REGION = CACHE_REGIONS.subjectMiss; +/** 网络失败 / 被拦后同一条目在本进程里冷却多久不再在线查(不写缓存,只是不连着打外站) */ +const FAIL_COOLDOWN_MS = 30 * 60_000; +const failedAt = new Map(); export type MetaSource = "douban" | "bangumi"; const REGION: Record = { douban: DOUBAN_SUBJECT_REGION, bangumi: BANGUMI_SUBJECT_REGION }; /** 豆瓣 / bgm 请求缓存 region(对照 MP DoubanApi.__invoke / BangumiApi 的 @cached);key 见 metaRequestKey */ -export const META_REQUEST_REGIONS = { douban: "douban.request", bangumi: "bangumi.request" } as const; +export const META_REQUEST_REGIONS = { + douban: CACHE_REGIONS.doubanRequest, + bangumi: CACHE_REGIONS.bangumiRequest, +} as const; const META_REQUEST_MAXSIZE = 5000; /** 每源限流(进程内):豆瓣风控严,间隔大一些;缓存命中不占额度 */ const META_LIMITERS = { douban: new RequestLimiter(2, 300), bangumi: new RequestLimiter(3, 100) }; /** - * 请求缓存:搜索按 storage.cacheTtl.metaSearchDays,列表(榜单 / 热门 / 每日放送 / 趋势页)按 metaListHours, - * 空结果按 metaEmptyHours(不超过正常 TTL);失败(网络、403、风控页)在适配器里抛出,cachedCall 不写。 - * 条目详情不走这里:douban.subject / bangumi.subject 条目级缓存(30 天)已经存了,避免同一份数据存两遍。 + * 请求缓存(统一元数据缓存):按策略表「来源 × 资源类型」取 TTL——搜索 metaSearchDays / 查无 metaMissDays, + * 列表(榜单 / 热门 / 每日放送 / 趋势页)metaListHours / 空 metaEmptyHours;过期先用旧值后台刷新。 + * 失败(网络、403、风控页)在适配器里抛出,不写。 + * 条目详情不走这里:douban.subject / bangumi.subject 条目级缓存已经存了,避免同一份数据存两遍。 */ export function metaRequestCache(ctx: SourcesRuntime["ctx"]): MetaRequestCache { return { async cached(req, load) { const t = (await storageSettings(ctx)).cacheTtl; - const ttl = req.kind === "search" ? t.metaSearchDays * 86_400 : t.metaListHours * 3600; - return cachedCall( - pgCacheFor(ctx.db), + const p = metaCachePolicy(t, req.source, req.kind); + return metadataCache(ctx).fetch( + META_REQUEST_REGIONS[req.source], req.key, - { - region: META_REQUEST_REGIONS[req.source], - ttl, - emptyTtl: Math.min(ttl, t.metaEmptyHours * 3600), - emptyIf: req.empty, - maxsize: META_REQUEST_MAXSIZE, - }, + { ...p, isEmpty: req.empty, maxsize: META_REQUEST_MAXSIZE }, load, ); }, @@ -77,23 +82,38 @@ export function toCached(s: MetaSubject, now: Date): DoubanSubject { }; } -/** 条目(缓存 → 在线);拿不到为 null */ +/** + * 条目(缓存 → 在线,统一元数据缓存:过期先用旧值后台刷新);拿不到为 null。 + * 条目不存在记查无(metaMissDays);网络失败 / 被拦不缓存,同一条目本进程冷却 30 分钟再查。 + */ export async function loadSubject(rt: SourcesRuntime, source: MetaSource, id: string): Promise { - const hit = await loadDoubanSubject(rt.ctx, id, REGION[source]); - if (hit) return hit; const clients = metaClients(rt); - if (!clients) return null; - const cache = pgCacheFor(rt.ctx.db); + if (!clients) return loadDoubanSubject(rt.ctx, id, REGION[source]); + const cache = metadataCache(rt.ctx); const missKey = `${source}:${id}`; - if (await cache.get(MISS_REGION, missKey).catch(() => null)) return null; + const t = (await storageSettings(rt.ctx)).cacheTtl; + const p = metaCachePolicy(t, source, "detail"); try { - const s = source === "douban" ? await clients.douban.subject(id) : await clients.bangumi.subject(id); - const cached = toCached(s, rt.now()); - await saveDoubanSubject(rt.ctx, cached, REGION[source]); - return cached; + const v = await cache.fetch( + REGION[source], + id, + { ttl: p.ttl, maxsize: META_SUBJECT_MAX_ENTRIES, skipIf: (x) => !x?.title }, + async () => { + if ((failedAt.get(missKey) ?? 0) > Date.now() - FAIL_COOLDOWN_MS) return null; + if (await cache.get(MISS_REGION, missKey)) return null; + const s = source === "douban" ? await clients.douban.subject(id) : await clients.bangumi.subject(id); + return toCached(s, rt.now()); + }, + ); + return v && typeof v.title === "string" && v.title ? v : null; } catch (e) { rt.ctx.logger.info({ source, id, err: (e as Error).message }, "辅助信息源条目查询失败"); - await cache.set(MISS_REGION, missKey, true, { ttl: MISS_TTL_SEC, maxsize: 5000 }).catch(() => {}); + if (e instanceof MetaSourceError && e.kind === "not_found") + await cache.set(MISS_REGION, missKey, true, { ttl: p.emptyTtl, maxsize: 5000 }).catch(() => {}); + else { + if (failedAt.size > 5000) failedAt.clear(); + failedAt.set(missKey, Date.now()); + } return null; } } diff --git a/apps/server/src/modules/sources-identity/service/metadata-prewarm.ts b/apps/server/src/modules/sources-identity/service/metadata-prewarm.ts new file mode 100644 index 00000000..9d861916 --- /dev/null +++ b/apps/server/src/modules/sources-identity/service/metadata-prewarm.ts @@ -0,0 +1,122 @@ +import { join } from "node:path"; +import { fetchEpisodeGroup, fetchRichDetail, fetchRichSeason, TmdbError } from "@umt/adapters"; +import { dataPaths } from "../../../config.ts"; +import { externalFetch } from "../../../egress.ts"; +import { resetTrackedTmdbRefs } from "../../../shared-cache.ts"; +import { ImageCache } from "../../cache/images.ts"; +import type { SourcesRuntime } from "../runtime.ts"; +import { entityEpisodeGroup } from "./episode-structure.ts"; +import { metadataIncludeSpecials, metadataLanguages, metadataTextOptions } from "./metadata-rich.ts"; + +// 关注作品预热(统一元数据缓存,docs/runbook/cache.md):作品进入关注时(建订阅、用户搜索 / 手动识别、开始下载) +// 把之后刷新 / 刮削会用到的 TMDB 条目先拉进缓存:详情(含标志)、各季详情、选中的剧集组,海报 / 背景 / 标志图片进图片缓存。 +// 请求键与刷新元数据 / 刮削完全相同(同一套 fetchRichDetail / fetchRichSeason / fetchEpisodeGroup,同样的语言链), +// 所以后面直接命中;关注中作品的条目写入时 pinned(条数上限淘汰不清掉),命中时续期。 + +/** 一部作品最多预热多少季(长寿剧) */ +const MAX_SEASONS = 60; + +export interface PrewarmResult { + status: "warmed" | "skipped"; + reason?: string; + seasons: number; + episodeGroup: boolean; + images: number; + errors: string[]; +} + +/** 图片预热用的 fetch:测试注入优先;测试进程没注入就不出外网 */ +function imageFetch(rt: SourcesRuntime): typeof globalThis.fetch | null { + if (rt.overrides.fetch) return rt.overrides.fetch; + return Deno.env.get("VITEST") ? null : externalFetch; +} + +export async function prewarmMetadata(rt: SourcesRuntime, mediaEntityId: string): Promise { + const { ctx } = rt; + const out: PrewarmResult = { status: "skipped", seasons: 0, episodeGroup: false, images: 0, errors: [] }; + const ident = await ctx.db + .selectFrom("external_identity as x") + .innerJoin("media_entity as e", "e.id", "x.media_entity_id") + .select(["x.entity_type", "x.external_id", "e.status"]) + .where("x.media_entity_id", "=", mediaEntityId) + .where("x.provider", "=", "tmdb") + .where("x.status", "=", "accepted") + .executeTakeFirst(); + if (!ident || ident.status === "merged") return { ...out, reason: ident ? "merged" : "no_tmdb_identity" }; + if (ident.entity_type !== "movie" && ident.entity_type !== "tv") return { ...out, reason: "unsupported_type" }; + const tmdb = await rt.tmdb(); + if (!tmdb) return { ...out, reason: "tmdb_not_configured" }; + // 刚进入关注:关注集合按进程缓存 5 分钟,清掉让这次写入就标 pinned + resetTrackedTmdbRefs(ctx); + const type = ident.entity_type; + const id = ident.external_id; + const langs = await metadataLanguages(ctx); + const textOpts = await metadataTextOptions(ctx); + const note = (what: string, e: unknown) => out.errors.push(`${what}: ${(e as Error).message}`); + + // 刷新元数据的基础详情与富详情(含标志) + await tmdb.details(type, id).catch((e) => note("details", e)); + // 注入的简化客户端(测试)没有 getJson:只预热基础详情 + if (typeof tmdb.getJson !== "function") return { ...out, status: "warmed" }; + let rich: Awaited> | null = null; + try { + rich = await fetchRichDetail(tmdb, type, id, langs, textOpts); + } catch (e) { + // 详情都拿不到(404 / 鉴权 / 网络):网络类交给任务重试,其余记下结束 + if (e instanceof TmdbError && ["network", "timeout", "rate"].includes(e.kind)) throw e; + note("detail", e); + return { ...out, reason: "detail_unavailable" }; + } + out.status = "warmed"; + + if (type === "tv") { + const specials = await metadataIncludeSpecials(ctx); + const seasons = rich.seasons + .map((s) => s.seasonNumber) + .filter((n) => specials || n > 0) + .slice(0, MAX_SEASONS); + for (const n of seasons) { + try { + await fetchRichSeason(tmdb, id, n, langs, { + now: rt.now(), + ...textOpts, + originalLanguage: rich.originalLanguage, + }); + out.seasons++; + } catch (e) { + note(`season ${n}`, e); + } + } + const group = await entityEpisodeGroup(ctx.db, mediaEntityId); + if (group) { + try { + await fetchEpisodeGroup(tmdb, group.id, langs[0]); + out.episodeGroup = true; + } catch (e) { + note("episode group", e); + } + } + } + + // 图片:刮削取原图(original),这里预先拉进图片缓存目录 + const f = imageFetch(rt); + if (f) { + const images = new ImageCache({ + dir: join(dataPaths(ctx.config).dataDir, "cache", "images"), + fetch: f, + timeoutMs: 30_000, + }); + for (const p of [rich.posterPath, rich.backdropPath, rich.logoPath]) { + if (!p) continue; + try { + await images.bytes("original", p); + out.images++; + } catch (e) { + note(`image ${p}`, e); + } + } + } + if (out.errors.length) + ctx.logger.debug({ entityId: mediaEntityId, errors: out.errors }, "关注作品预热有部分条目没拉到"); + return out; +} diff --git a/apps/server/src/modules/sources-identity/service/metadata-rich.ts b/apps/server/src/modules/sources-identity/service/metadata-rich.ts index 3bfcd8c7..6a4ae35a 100644 --- a/apps/server/src/modules/sources-identity/service/metadata-rich.ts +++ b/apps/server/src/modules/sources-identity/service/metadata-rich.ts @@ -72,6 +72,7 @@ export function richEntityFields(r: TmdbRichDetail): Partial = new Set([ "externalIds", "posterPath", "backdropPath", + "logoPath", ]); /** diff --git a/apps/server/src/modules/sources-identity/service/tmdb-recognize-cache.ts b/apps/server/src/modules/sources-identity/service/tmdb-recognize-cache.ts index c7f2d18d..c9413e84 100644 --- a/apps/server/src/modules/sources-identity/service/tmdb-recognize-cache.ts +++ b/apps/server/src/modules/sources-identity/service/tmdb-recognize-cache.ts @@ -1,6 +1,6 @@ import type { MetadataCandidate } from "@umt/adapters"; import type { ParsedRelease } from "@umt/domain"; -import { CACHE_REGIONS, sharedCache, tmdbCachePolicy } from "../../../shared-cache.ts"; +import { CACHE_REGIONS, metadataCache, tmdbCachePolicy } from "../../../shared-cache.ts"; import type { SourcesRuntime } from "../runtime.ts"; // TMDB 识别缓存:移植 MP app/modules/themoviedb/cache.py 的 TmdbCache(对照表见 docs/porting/moviepilot-tmdb-cache.md)。 @@ -92,7 +92,7 @@ export async function loadRecognizeCache( ): Promise { const tmdb = await rt.tmdb(); const key = recognizeKey(meta, localeOf(tmdb)); - const cache = sharedCache(rt.ctx); + const cache = metadataCache(rt.ctx); const v = await cache.get(CACHE_REGIONS.tmdbRecognize, key).catch(() => null); if (!v || typeof v !== "object") return null; if (typeConflicted(meta, v.type, v.id)) { @@ -120,7 +120,7 @@ export async function saveRecognizeCache( const tmdb = await rt.tmdb(); const key = recognizeKey(meta, localeOf(tmdb)); const policy = await tmdbCachePolicy(rt.ctx); - const cache = sharedCache(rt.ctx); + const cache = metadataCache(rt.ctx); if (info) { if (typeConflicted(meta, info.entityType, info.externalId)) { rt.ctx.logger.warn({ key, title: info.title, type: info.entityType }, "识别结果类型与元数据冲突，不写入识别缓存"); diff --git a/apps/server/src/modules/sources-identity/service/watched.ts b/apps/server/src/modules/sources-identity/service/watched.ts index 13cb7e67..b39579f8 100644 --- a/apps/server/src/modules/sources-identity/service/watched.ts +++ b/apps/server/src/modules/sources-identity/service/watched.ts @@ -1,4 +1,4 @@ -import { sql } from "@umt/db"; +import { type RawBuilder, sql } from "@umt/db"; import type { ServerContext } from "../../types.ts"; import type { Exec } from "./common.ts"; @@ -17,9 +17,8 @@ import type { Exec } from "./common.ts"; const OPEN_SUBSCRIPTION = ["active", "pending"]; const DOWNLOAD_DONE = ["imported", "removed"]; -export async function isEntityWatched(exec: Exec, entityId: string): Promise { - const r = await sql<{ watched: boolean }>` - select ( +/** 作品 e(SQL 片段:作品 id 表达式)是否关注中;isEntityWatched 与 watchedTmdbRefs 共用一个口径 */ +const watchedExpr = (entityId: RawBuilder) => sql`( exists (select 1 from subscription s where s.media_entity_id = ${entityId} and s.status = any(${OPEN_SUBSCRIPTION})) or exists (select 1 from media_asset a where a.media_entity_id = ${entityId} and a.status = 'active') or exists (select 1 from external_identity x @@ -27,7 +26,7 @@ export async function isEntityWatched(exec: Exec, entityId: string): Promise>'lookup' in ('search', 'manual')) or exists (select 1 from acquisition_plan p left join acquisition_slot sl on sl.id = p.slot_id - where (sl.media_entity_id = ${entityId} or p.evaluation->>'mediaEntityId' = ${entityId}::text) + where (sl.media_entity_id = ${entityId} or p.evaluation->>'mediaEntityId' = (${entityId})::text) and (p.evaluation->>'manual')::boolean is true) or exists (select 1 from download_task t left join source_release r on r.id = t.source_release_id @@ -37,14 +36,38 @@ export async function isEntityWatched(exec: Exec, entityId: string): Promise` / `tv:` 与选中的剧集组 `group:`。 + * 口径同 isEntityWatched。 + */ +export async function watchedTmdbRefs(exec: Exec): Promise> { + const r = await sql<{ ref: string; grp: string | null }>` + select x.entity_type || ':' || x.external_id as ref, e.episode_group_id as grp + from external_identity x join media_entity e on e.id = x.media_entity_id + where x.provider = 'tmdb' and x.status = 'accepted' and x.entity_type in ('movie', 'tv') + and e.status <> 'merged' and ${watchedExpr(sql`e.id`)}`.execute(exec); + const out = new Set(); + for (const x of r.rows) { + out.add(x.ref); + if (x.grp) out.add(`group:${x.grp}`); + } + return out; +} + +export async function isEntityWatched(exec: Exec, entityId: string): Promise { + const r = await sql<{ watched: boolean }>`select ${watchedExpr(sql`${entityId}::uuid`)} as watched`.execute(exec); return r.rows[0]?.watched === true; } /** - * 作品进入关注范围(建订阅、用户搜索、开始下载)时补排一次 full 元数据刷新:只对从没刷新过的作品排 - * (自动建档时因不在关注范围而没排的),刷新完成会重算待确认 / 滞后放行资源的集数(必要时开集数待确认)。 - * 入队失败只记日志;同一作品同一原因只排一次(幂等键)。 + * 作品进入关注范围(建订阅、用户搜索 / 手动识别、开始下载)时: + * - 补排一次 full 元数据刷新:只对从没刷新过的作品排(自动建档时因不在关注范围而没排的), + * 刷新完成会重算待确认 / 滞后放行资源的集数(必要时开集数待确认);同一作品只排一次(幂等键); + * - 每次都排一次缓存预热(prewarmMetadata:详情、各季、剧集组、海报 / 背景 / 标志进缓存),幂等键按作品 + 日期, + * 同一天重复进入只跑一次;条目已在缓存里时预热只是命中,不发请求。 + * 入队失败只记日志。返回是否排了刷新。 */ export async function enqueueWatchedRefresh(ctx: ServerContext, entityId: string, reason: string): Promise { const e = await ctx.db @@ -52,7 +75,9 @@ export async function enqueueWatchedRefresh(ctx: ServerContext, entityId: string .select(["id", "last_metadata_refresh_at", "status"]) .where("id", "=", entityId) .executeTakeFirst(); - if (!e || e.last_metadata_refresh_at || e.status === "merged") return false; + if (!e || e.status === "merged") return false; + await enqueuePrewarm(ctx, entityId, reason); + if (e.last_metadata_refresh_at) return false; try { await ctx.engine.enqueue( "refreshMetadata", @@ -68,3 +93,22 @@ export async function enqueueWatchedRefresh(ctx: ServerContext, entityId: string return false; } } + +/** 关注作品预热入队(幂等键按作品 + 日期);失败只记日志 */ +export async function enqueuePrewarm(ctx: ServerContext, entityId: string, reason: string): Promise { + const day = new Date().toISOString().slice(0, 10); + try { + const r = await ctx.engine.enqueue( + "prewarmMetadata", + { mediaEntityId: entityId }, + { idempotencyKey: `prewarmMetadata:${entityId}:${day}`, traceId: entityId }, + ); + return !r.deduplicated; + } catch (err) { + ctx.logger.debug( + { entityId, reason, err: { message: err instanceof Error ? err.message : String(err) } }, + "关注作品预热入队失败", + ); + return false; + } +} diff --git a/apps/server/src/modules/sources-identity/tasks.ts b/apps/server/src/modules/sources-identity/tasks.ts index 7c943a7d..d9a1943d 100644 --- a/apps/server/src/modules/sources-identity/tasks.ts +++ b/apps/server/src/modules/sources-identity/tasks.ts @@ -7,6 +7,7 @@ import { AUTO_ENRICH_TASK, runAutoEnrich } from "./service/enrich.ts"; import { ingestFeed } from "./service/feeds.ts"; import { emitMetadataRefreshed } from "./service/lifecycle-events.ts"; import { refreshMetadata } from "./service/metadata.ts"; +import { prewarmMetadata } from "./service/metadata-prewarm.ts"; import { resolveRelease } from "./service/resolve.ts"; import { snapshotSite } from "./service/snapshot.ts"; import { liveSearch } from "./service/workbench.ts"; @@ -89,6 +90,15 @@ export function registerSourcesTasks(registry: TaskRegistry, rt: () => SourcesRu return r; }), }), + // 关注作品预热(统一元数据缓存):详情 / 各季 / 剧集组 / 图片先进缓存;网络道,走 TMDB 客户端限流 + defineTask({ + name: "prewarmMetadata", + version: 1, + lane: "network", + inputSchema: z.object({ mediaEntityId: id }), + retryPolicy: { maximumAttempts: 3, initialInterval: "5s", backoffCoefficient: 2, maximumInterval: "60s" }, + handler: (t) => t.step.run("prewarm", () => prewarmMetadata(rt(), t.input.mediaEntityId)), + }), defineTask({ name: "searchSubscription", version: 1, diff --git a/apps/server/src/shared-cache.ts b/apps/server/src/shared-cache.ts index 6cf2099c..5a2425ec 100644 --- a/apps/server/src/shared-cache.ts +++ b/apps/server/src/shared-cache.ts @@ -1,33 +1,131 @@ -import { DEFAULT_SETTINGS, type StorageSettings, withStorageDefaults } from "@umt/contracts"; -import { type PgCache, pgCacheFor, sql } from "@umt/db"; +import { DEFAULT_SETTINGS, type StorageCacheTtl, type StorageSettings, withStorageDefaults } from "@umt/contracts"; +import { + type FetchPolicy, + listenCacheInvalidation, + type MetadataCache, + metadataCacheFor, + type PgCache, + pgCacheFor, + sql, +} from "@umt/db"; +import { watchedTmdbRefs } from "./modules/sources-identity/service/watched.ts"; import type { ServerContext } from "./modules/types.ts"; -// 进程级共享缓存入口:PG kv_cache(移植 MP app/runtime/cache.py 的 TTLCache,存储换成 PG,三进程共享)。 -// region 与 TTL 规则见 docs/runbook/cache.md 与 docs/porting/moviepilot-tmdb-cache.md; -// CACHE-2 起各 region 的 TTL 取值来源是 settings.storage.cacheTtl(docs/runbook/storage.md),语义不变。 +// 进程级共享缓存入口(docs/runbook/cache.md): +// - PgCache:PG kv_cache(移植 MP app/runtime/cache.py 的 TTLCache,存储换成 PG,三进程共享),非元数据的缓存仍直接用; +// - MetadataCache:统一元数据缓存,TMDB / 豆瓣 / Bangumi 共用。L1 进程内存 + L2 kv_cache、过期先用旧值后台刷新、 +// 同请求合并、按 region 计命中。TTL 按「来源 × 资源类型」取 settings.storage.cacheTtl(metaCachePolicy)。 export const CACHE_REGIONS = { /** TMDB 请求响应(MP TMDb.request 的 @cached):key = path?query */ tmdbRequest: "tmdb.request", /** TMDB 识别结果(MP TmdbCache):key = [类型][语言]{tmdbid 或名称}-{年份}-{季} */ tmdbRecognize: "tmdb.recognize", + doubanRequest: "douban.request", + bangumiRequest: "bangumi.request", + doubanSubject: "douban.subject", + bangumiSubject: "bangumi.subject", + /** 豆瓣 / bgm 条目查无(只记「不存在」,网络失败 / 被拦不记) */ + subjectMiss: "meta.subject_miss", + /** 豆瓣 / bgm 条目 → TMDB 作品的确定匹配 */ + discoverMatch: "discover.match", } as const; +/** 命中统计只看这些 region(统一元数据缓存);其它 region 不经 MetadataCache */ +export const METADATA_REGIONS: readonly string[] = Object.values(CACHE_REGIONS); + const DAY = 86_400; +const HOUR = 3_600; export function sharedCache(ctx: Pick): PgCache { return pgCacheFor(ctx.db); } +/** 统一元数据缓存(同一个 db 共用一个实例) */ +export function metadataCache(ctx: Pick & Partial>): MetadataCache { + return metadataCacheFor(ctx.db, ctx.logger ? { logger: ctx.logger } : {}); +} + +/** + * 进程启动时调用:LISTEN umt_cache_invalidate(在线才启用 L1)+ 每分钟把命中计数写进 cache_stat。 + * 返回 stop(停监听、最后写一次统计)。监听连不上时只记警告,L1 保持停用,照常走 L2。 + */ +export async function startMetadataCache( + ctx: Pick, + connectionString: string, + opts: { flushMs?: number } = {}, +): Promise<() => Promise> { + const cache = metadataCache(ctx); + const log = ctx.logger.child({ component: "metadata-cache" }); + let listener: Awaited> | null = null; + try { + listener = await listenCacheInvalidation(cache, { connectionString, logger: log }); + } catch (err) { + log.warn({ err: { message: (err as Error).message } }, "缓存失效监听启动失败，进程内一级缓存停用"); + } + const flush = () => + cache.flushStats().catch((err) => log.debug({ err: { message: (err as Error).message } }, "缓存统计写入失败")); + const timer = setInterval(flush, opts.flushMs ?? 60_000); + return async () => { + clearInterval(timer); + await listener?.stop(); + await flush(); + }; +} + /** settings.storage(读失败用缺省) */ export async function storageSettings(ctx: Pick): Promise { return withStorageDefaults(await ctx.settings.get("storage").catch(() => undefined)); } +// ── 策略表:来源 × 资源类型 → TTL ── + +export type MetaCacheSource = "tmdb" | "douban" | "bangumi"; +/** search:搜索 / 识别;detail:详情 / 季 / 剧集组 / 条目;list:榜单 / 热门 / 每日放送;image:人物 / 图片 */ +export type MetaResourceKind = "search" | "detail" | "list" | "image"; + +export interface MetaCachePolicy { + /** 秒 */ + ttl: number; + /** 空结果 / 查无(秒) */ + emptyTtl: number; +} + +/** + * 策略表(docs/runbook/cache.md)。宽限期统一按 TTL 的 50%(上限 7 天,见 @umt/db defaultGrace);失败不缓存。 + * + * | 来源 | search | detail | list | image | + * | TMDB | tmdbSearchDays / tmdbMissDays | tmdbDetailDays / tmdbMissDays | tmdbListHours / tmdbListEmptyHours | imageDays / tmdbMissDays | + * | 豆瓣 / bgm | metaSearchDays / metaMissDays | metaDetailDays / metaMissDays | metaListHours / metaEmptyHours | (无) | + */ +export function metaCachePolicy(t: StorageCacheTtl, source: MetaCacheSource, kind: MetaResourceKind): MetaCachePolicy { + const cap = (ttl: number, empty: number) => ({ ttl, emptyTtl: Math.min(ttl, empty) }); + if (source === "tmdb") { + switch (kind) { + case "search": + return cap(t.tmdbSearchDays * DAY, t.tmdbMissDays * DAY); + case "list": + return cap(t.tmdbListHours * HOUR, t.tmdbListEmptyHours * HOUR); + case "image": + return cap(t.imageDays * DAY, t.tmdbMissDays * DAY); + default: + return cap(t.tmdbDetailDays * DAY, t.tmdbMissDays * DAY); + } + } + switch (kind) { + case "search": + return cap(t.metaSearchDays * DAY, t.metaMissDays * DAY); + case "list": + return cap(t.metaListHours * HOUR, t.metaEmptyHours * HOUR); + default: + return cap(t.metaDetailDays * DAY, t.metaMissDays * DAY); + } +} + /** * TMDB 缓存策略(MP:CONF.meta = META_CACHE_EXPIRE、CONF.tmdb 条数、EMPTY_RESULT_CACHE_TTL)。 - * CACHE-2:MP 只有一个 CONF.meta,这里按请求类别取 storage.cacheTtl:搜索 / find 与识别结果用 searchTtl, - * 详情(/movie/{id}、/tv/{id}、季)用 ttl(= tmdbDetailDays),空结果与未识别负缓存用 emptyTtl。 + * 按请求类别取 storage.cacheTtl:搜索 / find 与识别结果用 searchTtl,详情(/movie/{id}、/tv/{id}、季)用 ttl, + * 空结果与未识别负缓存用 emptyTtl;列表 / 图片见 metaCachePolicy。 */ export interface TmdbCachePolicy { /** 详情 TTL(秒) */ @@ -37,16 +135,25 @@ export interface TmdbCachePolicy { maxsize: number; /** 秒 */ emptyTtl: number; + /** 列表(秒)与列表空结果(秒) */ + listTtl: number; + listEmptyTtl: number; + /** 人物 / 图片(秒) */ + imageTtl: number; } export async function tmdbCachePolicy(ctx: Pick): Promise { const c = await ctx.settings.get("cache").catch(() => DEFAULT_SETTINGS.cache); const t = (await storageSettings(ctx)).cacheTtl; + const list = metaCachePolicy(t, "tmdb", "list"); return { ttl: t.tmdbDetailDays * DAY, searchTtl: t.tmdbSearchDays * DAY, maxsize: c.tmdbMaxSize ?? DEFAULT_SETTINGS.cache.tmdbMaxSize, emptyTtl: t.tmdbMissDays * DAY, + listTtl: list.ttl, + listEmptyTtl: list.emptyTtl, + imageTtl: t.imageDays * DAY, }; } @@ -55,47 +162,102 @@ export function isTmdbSearchPath(key: string): boolean { return /^\/(search|find)\//.test(key); } -/** 详情键 → `movie:` / `tv:`;不是详情返回 null */ +/** + * 键 → 所属作品:`movie:` / `tv:`(详情、季、剧集、图片、剧集组列表都算); + * 剧集组详情 `/tv/episode_group/` → `group:`;都不是返回 null。 + */ export function tmdbDetailRef(key: string): string | null { + const g = /^\/tv\/episode_group\/([^/?]+)/.exec(key); + if (g) return `group:${decodeURIComponent(g[1]!)}`; const m = /^\/(movie|tv)\/(\d+)(?:[/?]|$)/.exec(key); return m ? `${m[1]}:${m[2]}` : null; } -// ── 追踪中的作品:详情缓存命中时续期 ── -// 追踪 = 作品有未结束的订阅(active / paused / pending)且有已接受的 TMDB 身份。集合按进程缓存 5 分钟。 +// ── 关注中的作品:条目 pinned(LRU 不淘汰)、命中续期 ── +// 关注口径同 service/watched.ts isEntityWatched;集合 = 关注中作品的 TMDB 身份 + 选中的剧集组,按进程缓存 5 分钟。 -const TRACKED_TTL_MS = 5 * 60_000; -const trackedCache = new WeakMap> }>(); +const WATCHED_TTL_MS = 5 * 60_000; +const watchedCache = new WeakMap> }>(); export async function trackedTmdbRefs(ctx: Pick, now = Date.now()): Promise> { - const hit = trackedCache.get(ctx.db); - if (hit && now - hit.at < TRACKED_TTL_MS) return hit.set; - const set = sql<{ ref: string }>`select distinct x.entity_type || ':' || x.external_id as ref - from subscription s join external_identity x on x.media_entity_id = s.media_entity_id - where s.status in ('active', 'paused', 'pending') and x.provider = 'tmdb' and x.status = 'accepted' - and x.entity_type in ('movie', 'tv')` - .execute(ctx.db) - .then((r) => new Set(r.rows.map((x) => x.ref))); - trackedCache.set(ctx.db, { at: now, set }); - set.catch(() => trackedCache.delete(ctx.db)); + const hit = watchedCache.get(ctx.db); + if (hit && now - hit.at < WATCHED_TTL_MS) return hit.set; + const set = watchedTmdbRefs(ctx.db); + watchedCache.set(ctx.db, { at: now, set }); + set.catch(() => watchedCache.delete(ctx.db)); return set; } -/** 测试用:清掉追踪集合缓存 */ +/** 清掉关注集合缓存(测试;进入关注时预热前也清,让新关注的作品立即 pinned) */ export function resetTrackedTmdbRefs(ctx: Pick): void { - trackedCache.delete(ctx.db); + watchedCache.delete(ctx.db); +} + +/** 这个 TMDB 请求键属于关注中的作品(详情 / 季 / 剧集组 / 图片) */ +export async function isWatchedTmdbKey(ctx: Pick, key: string): Promise { + const ref = tmdbDetailRef(key); + return !!ref && (await trackedTmdbRefs(ctx)).has(ref); } /** - * TMDB 详情缓存命中后的续期:键属于追踪中的作品时把过期时间推到 now + 详情 TTL(只延长不缩短)。 - * 返回是否续期。失败不影响命中结果。 + * TMDB 详情缓存命中后的续期(旧接口,保留给直接用 PgCache 的地方):键属于关注中的作品时把过期时间推到 + * now + 详情 TTL(只延长不缩短)并标 pinned。返回是否续期。 */ export async function renewTrackedTmdbDetail( ctx: Pick, key: string, detailTtlSec: number, ): Promise { - const ref = tmdbDetailRef(key); - if (!ref || !(await trackedTmdbRefs(ctx)).has(ref)) return false; - return await sharedCache(ctx).renew(CACHE_REGIONS.tmdbRequest, key, detailTtlSec); + if (!(await isWatchedTmdbKey(ctx, key))) return false; + return await sharedCache(ctx).renew(CACHE_REGIONS.tmdbRequest, key, detailTtlSec, { pin: true }); +} + +/** 关注中作品的 TMDB 条目:写入 pinned、命中续期(FetchPolicy.pin) */ +export function watchedPin(ctx: Pick, key: string): FetchPolicy["pin"] { + return tmdbDetailRef(key) ? () => isWatchedTmdbKey(ctx, key) : undefined; +} + +// ── 命中统计(GET /cache/regions 用) ── + +export interface RegionHitStats { + l1Hits: number; + l2Hits: number; + staleHits: number; + misses: number; + failures: number; + refreshes: number; +} + +/** + * 最近 days 天各 region 的命中统计:cache_stat(各进程每分钟写入的增量之和)+ 本进程还没写入的计数。 + * 其它进程最近不到一分钟的计数不在内。 + */ +export async function regionHitStats(ctx: Pick, days = 7): Promise> { + const r = await sql<{ + region: string; + l1: number; + l2: number; + stale: number; + miss: number; + fail: number; + refresh: number; + }>`select region, sum(l1_hits)::int8 as l1, sum(l2_hits)::int8 as l2, sum(stale_hits)::int8 as stale, + sum(misses)::int8 as miss, sum(failures)::int8 as fail, sum(refreshes)::int8 as refresh + from cache_stat where day > current_date - ${days}::int group by region`.execute(ctx.db); + const out = new Map(); + for (const x of r.rows) + out.set(x.region, { + l1Hits: Number(x.l1), + l2Hits: Number(x.l2), + staleHits: Number(x.stale), + misses: Number(x.miss), + failures: Number(x.fail), + refreshes: Number(x.refresh), + }); + for (const [region, c] of metadataCache(ctx).pendingStats()) { + const o = out.get(region) ?? { l1Hits: 0, l2Hits: 0, staleHits: 0, misses: 0, failures: 0, refreshes: 0 }; + for (const k of Object.keys(o) as (keyof RegionHitStats)[]) o[k] += c[k]; + out.set(region, o); + } + return out; } diff --git a/apps/server/test/acquisition-organize/scrape-clearlogo.test.ts b/apps/server/test/acquisition-organize/scrape-clearlogo.test.ts new file mode 100644 index 00000000..84921226 --- /dev/null +++ b/apps/server/test/acquisition-organize/scrape-clearlogo.test.ts @@ -0,0 +1,126 @@ +import { readFile, writeFile } from "node:fs/promises"; +import type { ScrapeFileResult, ScrapeMatch } from "@umt/contracts"; +import { createTempTree, type TempTree } from "@umt/testkit"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { queueScrape, runScrapeRecord } from "../../src/modules/acquisition-organize/services/scrape.ts"; +import { createHarness, type Harness, pgAvailable, seedBase } from "./harness.ts"; + +// TMDB 标志 → clearlogo.png:电影写影片目录、剧集写剧集根目录;覆盖方式与其它图片一致 +// (更新:没变不重写、本工具写过且没被改过的按新图覆盖、被人改过的跳过;只补缺失;全部覆盖)。 + +const hasPg = await pgAvailable(); +const at = new Date().toISOString(); +const f = (value: unknown) => ({ value, source: "tmdb", language: null, at, locked: false }); +const enc = (s: string) => new TextEncoder().encode(s); + +describe.skipIf(!hasPg)("刮削:TMDB 标志写 clearlogo.png", () => { + let h: Harness; + let lib: TempTree; + let rootId = ""; + + beforeAll(async () => { + lib = await createTempTree( + { + "Movies/星际穿越 (2014)/Interstellar.2014.mkv": 100, + "TV/人生切割术 (2022)/Season 1/Severance.S01E01.mkv": 100, + }, + { sentinel: true, prefix: "umt-scrlogo-" }, + ); + h = await createHarness("scrape_logo", { + // 内容 = 路径,方便看出写的是哪张 + scrapeImage: async (p) => ({ content: enc(p), mime: p.endsWith(".png") ? "image/png" : "image/jpeg" }), + }); + rootId = (await seedBase(h.db, { libraryPath: lib.root, cachePath: lib.root, qbUrl: "http://127.0.0.1:9" })).rootId; + }, 60_000); + afterAll(async () => { + await h?.close(); + await lib?.cleanup(); + }); + + async function entity(kind: "movie" | "series", title: string, meta: object) { + return ( + await h.db + .insertInto("media_entity") + .values({ kind, title, year: 2020, status: "active", metadata: JSON.stringify(meta) }) + .returning("id") + .executeTakeFirstOrThrow() + ).id; + } + const setMeta = (id: string, meta: object) => + h.db + .updateTable("media_entity") + .set({ metadata: JSON.stringify(meta) }) + .where("id", "=", id) + .execute(); + async function scrape( + entityId: string, + kind: "movie" | "tv", + path: string, + mode?: "missing" | "overwrite" | "update", + ) { + const match: ScrapeMatch = { + mediaEntityId: entityId, + provider: null, + externalId: null, + kind, + title: "x", + year: null, + source: "library", + }; + const q = await queueScrape(h.d, { + rootId: `lib:${rootId}`, + rootPath: lib.root, + libraryRootId: rootId, + path, + match, + mode, + trigger: "manual", + }); + return runScrapeRecord(h.d, q.recordId); + } + const logo = (files: ScrapeFileResult[]) => files.find((x) => x.role === "clearlogo"); + const MOVIE_DIR = "Movies/星际穿越 (2014)"; + const LOGO = `${MOVIE_DIR}/clearlogo.png`; + + it("电影:写在影片目录 clearlogo.png;更新模式内容没变不重写,换了标志按新图覆盖", async () => { + const movie = await entity("movie", "星际穿越", { title: f("星际穿越"), logoPath: f("/logo-a.png") }); + const r = await scrape(movie, "movie", MOVIE_DIR); + expect(logo(r.files)).toMatchObject({ + path: LOGO, + action: "written", + hash: expect.stringMatching(/^[0-9a-f]{64}$/), + }); + expect(await readFile(lib.path(LOGO), "utf8")).toBe("/logo-a.png"); + const same = await scrape(movie, "movie", MOVIE_DIR, "update"); + expect(logo(same.files)).toMatchObject({ action: "skipped", reason: "scrape_unchanged" }); + await setMeta(movie, { title: f("星际穿越"), logoPath: f("/logo-b.png") }); + const upd = await scrape(movie, "movie", MOVIE_DIR, "update"); + expect(logo(upd.files)).toMatchObject({ action: "written" }); + expect(await readFile(lib.path(LOGO), "utf8")).toBe("/logo-b.png"); + + // 被人换掉的标志:更新跳过、只补缺失跳过、全部覆盖改写 + await writeFile(lib.path(LOGO), "我自己的标志"); + await setMeta(movie, { title: f("星际穿越"), logoPath: f("/logo-c.png") }); + expect(logo((await scrape(movie, "movie", MOVIE_DIR, "update")).files)).toMatchObject({ + action: "skipped", + reason: "scrape_user_modified", + }); + expect(logo((await scrape(movie, "movie", MOVIE_DIR, "missing")).files)).toMatchObject({ + action: "skipped", + reason: "scrape_exists", + }); + expect(await readFile(lib.path(LOGO), "utf8")).toBe("我自己的标志"); + expect(logo((await scrape(movie, "movie", MOVIE_DIR, "overwrite")).files)).toMatchObject({ action: "written" }); + expect(await readFile(lib.path(LOGO), "utf8")).toBe("/logo-c.png"); + }); + + it("剧集:写在剧集根目录,季目录不写;没有标志不写", async () => { + const show = await entity("series", "人生切割术", { title: f("人生切割术"), logoPath: f("/sev.png") }); + const r = await scrape(show, "tv", "TV/人生切割术 (2022)"); + expect(logo(r.files)).toMatchObject({ path: "TV/人生切割术 (2022)/clearlogo.png", action: "written" }); + expect(r.files.filter((x) => x.role === "clearlogo")).toHaveLength(1); + await setMeta(show, { title: f("人生切割术") }); + const none = await scrape(show, "tv", "TV/人生切割术 (2022)", "overwrite"); + expect(logo(none.files)).toBeUndefined(); + }); +}); diff --git a/apps/server/test/cache/cache.test.ts b/apps/server/test/cache/cache.test.ts index 5736b683..02a4c9ee 100644 --- a/apps/server/test/cache/cache.test.ts +++ b/apps/server/test/cache/cache.test.ts @@ -67,7 +67,10 @@ describe.skipIf(!hasPg)("C2-A 缓存(真实 PG)", () => { await (await new SourcesRuntime(h.ctx).tmdb())!.details("movie", 872585); const calls = tmdb.calls.length; const rows = ( - await sql<{ key: string; days: number }>`select key, extract(epoch from expires_at - now()) / 86400 as days + await sql<{ + key: string; + days: number; + }>`select key, extract(epoch from coalesce(fresh_until, expires_at) - now()) / 86400 as days from kv_cache where region = 'tmdb.request'`.execute(h.db) ).rows; const detail = rows.find((x) => x.key.startsWith("/movie/872585?"))!; @@ -80,7 +83,7 @@ describe.skipIf(!hasPg)("C2-A 缓存(真实 PG)", () => { const days = async (like: string) => Number( ( - await sql<{ d: number }>`select extract(epoch from expires_at - now()) / 86400 as d + await sql<{ d: number }>`select extract(epoch from coalesce(fresh_until, expires_at) - now()) / 86400 as d from kv_cache where region = 'tmdb.request' and key like ${like}`.execute(h.db) ).rows[0]!.d, ); @@ -96,7 +99,7 @@ describe.skipIf(!hasPg)("C2-A 缓存(真实 PG)", () => { expect(empty).toBeLessThanOrEqual(2); }); - it("追踪中的作品:详情缓存命中时续期到 now + 详情 TTL;未追踪的不动", async () => { + it("关注中的作品:详情缓存命中时续期到 now + 详情 TTL 并标 pinned;未关注的不动", async () => { const ent = ( await sql<{ id: string; @@ -111,19 +114,24 @@ describe.skipIf(!hasPg)("C2-A 缓存(真实 PG)", () => { const client = (await new SourcesRuntime(h.ctx).tmdb())!; await client.details("movie", 872585); await client.details("movie", 693134); - await sql`update kv_cache set expires_at = now() + interval '1 day' + await sql`update kv_cache set fresh_until = now() + interval '1 day', expires_at = now() + interval '2 days', + pinned = false where region = 'tmdb.request' and (key like '/movie/872585?%' or key like '/movie/693134?%')`.execute(h.db); await client.details("movie", 872585); await client.details("movie", 693134); const left = async (like: string) => Number( ( - await sql<{ d: number }>`select extract(epoch from expires_at - now()) / 86400 as d + await sql<{ d: number }>`select extract(epoch from fresh_until - now()) / 86400 as d from kv_cache where region = 'tmdb.request' and key like ${like}`.execute(h.db) ).rows[0]!.d, ); expect(await left("/movie/872585?%")).toBeGreaterThan(19.9); expect(await left("/movie/693134?%")).toBeLessThanOrEqual(1); + const pinned = await sql<{ k: string; p: boolean }>`select key as k, pinned as p from kv_cache + where region = 'tmdb.request' and (key like '/movie/872585?%' or key like '/movie/693134?%')`.execute(h.db); + expect(pinned.rows.filter((r) => r.p).every((r) => r.k.startsWith("/movie/872585?"))).toBe(true); + expect(pinned.rows.some((r) => r.p)).toBe(true); }); it("GET /cache/tmdb 列出条目(键 / 过期 / 大小)与上限;POST /cache/tmdb/clear 清空", async () => { diff --git a/apps/server/test/cache/metadata-cache.test.ts b/apps/server/test/cache/metadata-cache.test.ts new file mode 100644 index 00000000..581df51c --- /dev/null +++ b/apps/server/test/cache/metadata-cache.test.ts @@ -0,0 +1,183 @@ +import { metadataCacheFor, PgCache, sql } from "@umt/db"; +import { createMockTmdb, type MockTmdb } from "@umt/testkit"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { createCacheModule } from "../../src/modules/cache/index.ts"; +import { SourcesRuntime } from "../../src/modules/sources-identity/runtime.ts"; +import { metaRequestCache } from "../../src/modules/sources-identity/service/meta-subjects.ts"; +import { metaCachePolicy, regionHitStats, resetTrackedTmdbRefs } from "../../src/shared-cache.ts"; +import { bearer, createHarness, type Harness, pgAvailable } from "../auth-settings/harness.ts"; +import { DEV_KEY, json } from "../helpers.ts"; + +// 统一元数据缓存接到 TMDB / 豆瓣 / bgm:策略表、过期先用旧值后台刷新、同请求合并、失败不缓存、 +// 关注中作品 pinned 不被淘汰、按 region 统计命中(GET /cache/regions)。 + +const hasPg = await pgAvailable(); +const KEY = "0123456789abcdef0123456789abcdef"; + +const until = async (cond: () => boolean | Promise, ms = 5000) => { + const end = Date.now() + ms; + while (!(await cond())) { + if (Date.now() > end) throw new Error("等待超时"); + await new Promise((r) => setTimeout(r, 20)); + } +}; + +describe("策略表(来源 × 资源类型)", () => { + it("缺省:搜索 7 天(空 1 天)、详情 30 天(空 1 天)、列表 6 小时(空 1 小时)、图片 90 天", async () => { + const { DEFAULT_STORAGE_SETTINGS } = await import("@umt/contracts"); + const t = DEFAULT_STORAGE_SETTINGS.cacheTtl; + const day = 86_400; + for (const s of ["tmdb", "douban", "bangumi"] as const) { + expect(metaCachePolicy(t, s, "search")).toEqual({ ttl: 7 * day, emptyTtl: day }); + expect(metaCachePolicy(t, s, "detail")).toEqual({ ttl: 30 * day, emptyTtl: day }); + expect(metaCachePolicy(t, s, "list")).toEqual({ ttl: 6 * 3600, emptyTtl: 3600 }); + } + expect(metaCachePolicy(t, "tmdb", "image")).toEqual({ ttl: 90 * day, emptyTtl: day }); + }); +}); + +describe.skipIf(!hasPg)("统一元数据缓存(真实 PG)", () => { + let h: Harness; + let admin: ReturnType; + let tmdb: MockTmdb; + + beforeAll(async () => { + h = await createHarness({ extraModules: [createCacheModule()] }); + admin = bearer(h.app, DEV_KEY); + tmdb = await createMockTmdb({ apiKey: KEY }); + const r = await admin("PATCH", "/settings", { + metadata: { tmdbApiKey: KEY, tmdbBaseUrl: `${tmdb.url}/3` }, + cache: { tmdbMaxSize: 16 }, + }); + expect(r.status).toBe(200); + }); + afterAll(async () => { + await tmdb?.close(); + await h?.close(); + }); + + const client = async () => (await new SourcesRuntime(h.ctx).tmdb())!; + const row = async (like: string) => + ( + await sql<{ key: string; fresh: number; pinned: boolean }>`select key, + extract(epoch from fresh_until - now()) / 86400 as fresh, pinned + from kv_cache where region = 'tmdb.request' and key like ${like}`.execute(h.db) + ).rows[0]; + + it("TMDB 详情过期在宽限期内:先返回旧值,后台只刷一次;刷新后是新值", async () => { + const c = await client(); + await c.details("movie", 872585); + const before = tmdb.callsTo("/3/movie/872585").length; + // 改成「已过新鲜期、还在宽限期」并在值里做个记号(热度 12345) + await sql`update kv_cache set fresh_until = now() - interval '1 second', expires_at = now() + interval '1 day', + value = jsonb_set(value, '{popularity}', '12345') + where region = 'tmdb.request' and key like '/movie/872585?%'`.execute(h.db); + const keys = ( + await sql<{ + key: string; + }>`select key from kv_cache where region = 'tmdb.request' and key like '/movie/872585?%'`.execute(h.db) + ).rows.map((r) => r.key); + expect(keys.length).toBeGreaterThan(0); + // 回源慢一点,让两个调用方都在刷新完成前读到 + tmdb.fault("/3/movie/872585", { delayMs: 300, times: 10 }); + const [a, b] = await Promise.all([c.details("movie", 872585), c.details("movie", 872585)]); + expect(a.popularity).toBe(12345); + expect(b.popularity).toBe(12345); + // 后台刷新:两个调用方同时拿到旧值,每个过期键只回源一次;刷新后新鲜且是新值 + await until(async () => { + const r = await sql<{ n: number }>`select count(*)::int as n from kv_cache where region = 'tmdb.request' + and key like '/movie/872585?%' and fresh_until > now() + interval '29 days'`.execute(h.db); + return r.rows[0]!.n === keys.length; + }); + expect(tmdb.callsTo("/3/movie/872585").length - before).toBe(keys.length); + tmdb.clearFaults(); + expect((await c.details("movie", 872585)).popularity).not.toBe(12345); + }); + + it("同请求合并:并发同一详情只回源一次;失败(5xx)不缓存", async () => { + const c = await client(); + const n0 = tmdb.callsTo("/3/tv/95396/season/1").length; + await Promise.all(Array.from({ length: 5 }, () => c.getJson("/tv/95396/season/1", { language: "zh-CN" }))); + expect(tmdb.callsTo("/3/tv/95396/season/1").length - n0).toBe(1); + // 5xx 重试用尽后抛错(客户端缺省重试 3 次,退避约 3.5 秒),不写缓存 + tmdb.fault("/3/tv/100088/season/1", { status: 503, times: 10 }); + await expect(c.getJson("/tv/100088/season/1", { language: "zh-CN" })).rejects.toThrow(); + expect(await row("/tv/100088/season/1?%")).toBeUndefined(); + tmdb.clearFaults(); + }); + + it("关注中的作品:条目 pinned,条数上限淘汰时不被清掉", async () => { + const ent = ( + await sql<{ + id: string; + }>`insert into media_entity (kind, title) values ('series', '人生切割术') returning id`.execute(h.db) + ).rows[0]!.id; + await sql`insert into external_identity (media_entity_id, provider, entity_type, external_id, status) + values (${ent}, 'tmdb', 'tv', '95396', 'accepted')`.execute(h.db); + await sql`insert into subscription (media_entity_id, media_kind, status) values (${ent}, 'series', 'active')`.execute( + h.db, + ); + resetTrackedTmdbRefs(h.ctx); + await sql`delete from kv_cache where region = 'tmdb.request'`.execute(h.db); + const c = await client(); + await c.getJson("/tv/95396", { language: "zh-CN" }); + expect((await row("/tv/95396?%"))?.pinned).toBe(true); + // 上限 16:再写一批别的请求,关注作品的条目留着 + for (let i = 0; i < 24; i++) await c.searchRaw("movie", `不存在${i}`); + expect(await row("/tv/95396?%")).toBeDefined(); + const n = await sql<{ n: number }>`select count(*)::int as n from kv_cache where region = 'tmdb.request'`.execute( + h.db, + ); + expect(n.rows[0]!.n).toBeLessThanOrEqual(16); + }); + + it("豆瓣 / bgm 请求缓存按策略表:搜索 7 天、列表 6 小时、空结果按查无 / 列表空;失败不写", async () => { + const cache = metaRequestCache(h.ctx); + await cache.cached({ source: "douban", key: "GET s1", kind: "search", empty: () => false }, async () => [1]); + await cache.cached({ source: "douban", key: "GET s0", kind: "search", empty: () => true }, async () => []); + await cache.cached({ source: "bangumi", key: "GET l1", kind: "list", empty: () => false }, async () => ({ + data: [1], + })); + await expect( + cache.cached({ source: "bangumi", key: "GET bad", kind: "list", empty: () => false }, async () => { + throw new Error("HTTP 403"); + }), + ).rejects.toThrow("403"); + const ttl = async (region: string, key: string) => + Number( + ( + await sql<{ s: number }>`select extract(epoch from fresh_until - now()) as s from kv_cache + where region = ${region} and key = ${key}`.execute(h.db) + ).rows[0]?.s ?? -1, + ); + expect(await ttl("douban.request", "GET s1")).toBeGreaterThan(7 * 86400 - 60); + expect(await ttl("douban.request", "GET s0")).toBeLessThanOrEqual(86400); + expect(await ttl("bangumi.request", "GET l1")).toBeLessThanOrEqual(6 * 3600); + expect(await ttl("bangumi.request", "GET l1")).toBeGreaterThan(6 * 3600 - 60); + expect(await ttl("bangumi.request", "GET bad")).toBe(-1); + }); + + it("命中统计:各 region 计 L1 / L2 / 旧值 / 未命中 / 失败 / 后台刷新;写入 cache_stat 后接口合计并给命中率", async () => { + const mc = metadataCacheFor(h.db); + const pending = mc.pendingStats().get("tmdb.request"); + expect(pending?.misses).toBeGreaterThan(0); + expect(pending?.staleHits).toBeGreaterThan(0); + expect(pending?.refreshes).toBeGreaterThan(0); + expect(pending?.failures).toBeGreaterThan(0); + await mc.flushStats(); + expect(mc.pendingStats().size).toBe(0); + // 再来一次 L2 命中(未写入 cache_stat 的也算进接口) + await (await client()).getJson("/tv/95396", { language: "zh-CN" }); + const stats = await regionHitStats(h.ctx); + expect(stats.get("tmdb.request")!.l2Hits).toBeGreaterThan(0); + const list = await json(await admin("GET", "/cache/regions")); + const tmdbRow = list.data.items.find((i: { region: string }) => i.region === "tmdb.request"); + expect(tmdbRow.hits.hitRate).toBeGreaterThan(0); + expect(tmdbRow.hits.hitRate).toBeLessThan(1); + expect(tmdbRow.hits).toMatchObject({ misses: expect.any(Number), l2Hits: expect.any(Number) }); + // 非元数据 region 没有命中统计 + await new PgCache(h.db).set("site.detail", "x", 1, { ttl: 60 }); + const again = await json(await admin("GET", "/cache/regions")); + expect(again.data.items.find((i: { region: string }) => i.region === "site.detail").hits).toBeUndefined(); + }); +}); diff --git a/apps/server/test/scrape/scrape-layout-mp.test.ts b/apps/server/test/scrape/scrape-layout-mp.test.ts index 37952076..dad8d895 100644 --- a/apps/server/test/scrape/scrape-layout-mp.test.ts +++ b/apps/server/test/scrape/scrape-layout-mp.test.ts @@ -40,6 +40,19 @@ const DEVIATIONS: { id: string; apply: (files: string[], scenario: string) => st ? fs : fs.map((f) => (f === "poster.jpg" || f === "fanart.jpg" ? `${LOOSE.replace(/\.mkv$/, "")}-${f}` : f)), }, + // 4. 作品级标志 clearlogo.png(TMDB logos;MP 只在接 fanart.tv 时写 logo,我们不接 fanart.tv): + // 写在作品海报旁边——电影影片目录 / 剧集根目录 clearlogo.png,库根的电影 <视频名>-clearlogo.png;季目录不写 + { + id: "tmdb_clearlogo", + apply: (fs) => [ + ...fs, + ...fs.flatMap((f) => { + const m = /^(.*?)(\/|^|-)poster\.jpg$/.exec(f); + if (!m || /(^|\/)Season \d+$/.test(m[1]!) || /(^|\/)season(\d+|-specials)$/.test(m[1]!)) return []; + return [`${m[1]}${m[2]}clearlogo.png`]; + }), + ], + }, ]; const detail = (entityType: "movie" | "tv", externalId: string): TmdbRichDetail => @@ -69,6 +82,7 @@ const detail = (entityType: "movie" | "tv", externalId: string): TmdbRichDetail originCountry: [], posterPath: "/poster.jpg", backdropPath: "/backdrop.jpg", + logoPath: "/logo.png", cast: [], crew: [], externalIds: { imdb: null, tvdb: null, wikidata: null }, @@ -124,7 +138,10 @@ describe.skipIf(skip)("刮削文件布局与 MP 差分", () => { "scrlayout", { ao: { - scrapeImage: async () => ({ content: new Uint8Array([0xff, 0xd8, 0xff]), mime: "image/jpeg" }), + scrapeImage: async (p) => + p.endsWith(".png") + ? { content: new Uint8Array([0x89, 0x50, 0x4e, 0x47, 1]), mime: "image/png" } + : { content: new Uint8Array([0xff, 0xd8, 0xff]), mime: "image/jpeg" }, scrapeTmdbDetail: async (type, id) => detail(type, id), scrapeTmdbSeason: async (_tv, n) => (n === 1 ? season1 : null), }, diff --git a/apps/server/test/search/discover.test.ts b/apps/server/test/search/discover.test.ts index 45859a60..cdbaad60 100644 --- a/apps/server/test/search/discover.test.ts +++ b/apps/server/test/search/discover.test.ts @@ -218,13 +218,13 @@ describe.skipIf(!up)("发现页接口", () => { expect(meta.filter((m) => m.includes("movie_showing"))).toHaveLength(3); expect(meta.at(-1)).toContain("start=20&count=20"); - // 空结果:写入但用空结果 TTL(缺省 1 小时),正常 6 小时 + // 空结果:写入但用空结果 TTL(缺省 1 小时),正常 6 小时(看新鲜期;过期后还有 50% 宽限期先用旧值) doubanReply = () => json({ subject_collection_items: [], total: 0 }); await req("/discover/douban?list=tv_korean"); const rows = await cache.items(META_REQUEST_REGIONS.douban); const ttlOf = (part: string) => { const row = rows.find((x) => x.key.includes(part))!; - return Math.round((new Date(row.expiresAt!).getTime() - Date.now()) / 60_000); + return Math.round((new Date(row.freshUntil).getTime() - Date.now()) / 60_000); }; expect(ttlOf("tv_korean")).toBeLessThanOrEqual(60); expect(ttlOf("tv_korean")).toBeGreaterThan(55); diff --git a/apps/server/test/sources-identity/metadata-prewarm.test.ts b/apps/server/test/sources-identity/metadata-prewarm.test.ts new file mode 100644 index 00000000..c378e4a8 --- /dev/null +++ b/apps/server/test/sources-identity/metadata-prewarm.test.ts @@ -0,0 +1,127 @@ +import { existsSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createTmdbClient } from "@umt/adapters"; +import { sql } from "@umt/db"; +import { createMockTmdb, type MockTmdb, SPLIT_COUR_FIXTURE, TMDB_FIXTURES } from "@umt/testkit"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { tmdbSharedCache } from "../../src/modules/sources-identity/runtime.ts"; +import { prewarmMetadata } from "../../src/modules/sources-identity/service/metadata-prewarm.ts"; +import { enqueueWatchedRefresh } from "../../src/modules/sources-identity/service/watched.ts"; +import { tmdbCachePolicy } from "../../src/shared-cache.ts"; +import { createHarness, type Harness, pgAvailable } from "./harness.ts"; + +// 关注作品预热:进入关注时入队 prewarmMetadata(幂等键按作品 + 日期);执行后详情(含标志)、各季、选中的剧集组 +// 在 TMDB 请求缓存里且 pinned,海报 / 背景 / 标志进图片缓存;再跑一次全部命中,不发请求。 + +const skip = !(await pgAvailable()); +const PNG = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 1]); +const JPEG = new Uint8Array([0xff, 0xd8, 0xff, 0xe0, 1]); + +describe.skipIf(skip)("关注作品预热", () => { + let h: Harness; + let tmdb: MockTmdb; + let entityId: string; + const images: string[] = []; + const dataDir = mkdtempSync(join(tmpdir(), "umt-prewarm-")); + + beforeAll(async () => { + tmdb = await createMockTmdb({ + fixtures: [ + ...TMDB_FIXTURES, + { + ...SPLIT_COUR_FIXTURE, + rich: { ...SPLIT_COUR_FIXTURE.rich, backdrop_path: "/split-bg.jpg" }, + logos: [ + { file_path: "/split-en.png", iso_639_1: "en" }, + { file_path: "/split-ja.png", iso_639_1: "ja" }, + ], + }, + ], + }); + let policy: Awaited> | null = null; + h = await createHarness("prewarm", { + // 和服务端一样走统一元数据缓存(PG 请求缓存) + tmdb: () => + createTmdbClient({ + apiKey: "test-tmdb-key", + baseUrl: `${tmdb.url}/3`, + retries: 0, + sharedCache: tmdbSharedCache(h.ctx, policy!), + }), + // 图片上游(image.tmdb.org) + fetch: async (input) => { + const url = String(input); + images.push(url); + return new Response(url.endsWith(".png") ? PNG : JPEG); + }, + }); + policy = await tmdbCachePolicy(h.ctx); + (h.ctx.config as { DATA_DIR?: string }).DATA_DIR = dataDir; + entityId = ( + await sql<{ id: string }>`insert into media_entity (kind, title, episode_group_id, episode_group_name) + values ('series', '分割番', 'grp-production', '制作季') returning id`.execute(h.db) + ).rows[0]!.id; + await sql`insert into external_identity (media_entity_id, provider, entity_type, external_id, status) + values (${entityId}, 'tmdb', 'tv', ${String(SPLIT_COUR_FIXTURE.id)}, 'accepted')`.execute(h.db); + }); + afterAll(async () => { + await h?.close(); + await tmdb?.close(); + rmSync(dataDir, { recursive: true, force: true }); + }); + + it("进入关注(建订阅)时入队预热,幂等键按作品 + 日期;同一天再进入不重复排", async () => { + await sql`insert into subscription (media_entity_id, media_kind, status) values (${entityId}, 'series', 'active')`.execute( + h.db, + ); + await enqueueWatchedRefresh(h.ctx, entityId, "subscription"); + await enqueueWatchedRefresh(h.ctx, entityId, "download"); + const jobs = h.engine.enqueued.filter((e) => e.task === "prewarmMetadata"); + expect(jobs).toHaveLength(1); + expect(jobs[0]).toMatchObject({ + input: { mediaEntityId: entityId }, + key: `prewarmMetadata:${entityId}:${new Date().toISOString().slice(0, 10)}`, + }); + }); + + it("执行后:详情 / 各季 / 剧集组 / 标志在缓存里且 pinned;海报、背景、标志进图片缓存", async () => { + const r = await prewarmMetadata(h.rt, entityId); + expect(r).toMatchObject({ status: "warmed", episodeGroup: true, images: 3, errors: [] }); + expect(r.seasons).toBe(SPLIT_COUR_FIXTURE.seasons!.length); + const rows = ( + await sql<{ key: string; pinned: boolean }>`select key, pinned from kv_cache where region = 'tmdb.request' + order by key`.execute(h.db) + ).rows; + const has = (re: RegExp) => rows.filter((x) => re.test(x.key)); + const id = SPLIT_COUR_FIXTURE.id; + expect(has(new RegExp(`^/tv/${id}\\?.*images`)).length).toBeGreaterThan(0); + for (const s of SPLIT_COUR_FIXTURE.seasons!) + expect(has(new RegExp(`^/tv/${id}/season/${s.season_number}\\?`)).length).toBeGreaterThan(0); + expect(has(/^\/tv\/episode_group\/grp-production\?/)).toHaveLength(1); + // 原语言是日语:另取了日语标志 + expect(has(new RegExp(`^/tv/${id}/images\\?include_image_language=ja`))).toHaveLength(1); + expect(rows.every((x) => x.pinned)).toBe(true); + for (const f of ["split.jpg", "split-bg.jpg", "split-ja.png"]) + expect(existsSync(join(dataDir, "cache/images/t/p/original", f))).toBe(true); + expect(images.every((u) => u.startsWith("https://image.tmdb.org/t/p/original/"))).toBe(true); + }); + + it("再预热一次全部命中:不发 TMDB 请求、不再下载图片", async () => { + const calls = tmdb.calls.length; + const fetched = images.length; + const r = await prewarmMetadata(h.rt, entityId); + expect(r.status).toBe("warmed"); + expect(tmdb.calls.length).toBe(calls); + expect(images.length).toBe(fetched); + }); + + it("没有 TMDB 身份的作品跳过", async () => { + const other = ( + await sql<{ id: string }>`insert into media_entity (kind, title) values ('movie', '本地') returning id`.execute( + h.db, + ) + ).rows[0]!.id; + expect(await prewarmMetadata(h.rt, other)).toMatchObject({ status: "skipped", reason: "no_tmdb_identity" }); + }); +}); diff --git a/apps/server/test/sources-identity/mteam-tasks.test.ts b/apps/server/test/sources-identity/mteam-tasks.test.ts index 4d68d56e..ecb84487 100644 --- a/apps/server/test/sources-identity/mteam-tasks.test.ts +++ b/apps/server/test/sources-identity/mteam-tasks.test.ts @@ -69,12 +69,13 @@ describe.skipIf(!up)("M-Team 快照与任务注册", () => { api.setFailMode(null); }); - it("模块注册 7 个任务与 3 条 outbox 路由;任务处理器经 step.run 调服务", async () => { + it("模块注册 8 个任务与 3 条 outbox 路由;任务处理器经 step.run 调服务", async () => { const reg = new TaskRegistry(); h.module.registerTasks(reg, h.ctx); expect(reg.names()).toEqual([ "enrichPoolEntry", "ingestFeed", + "prewarmMetadata", "refreshMetadata", "refreshSiteAccount", "resolveObservation", diff --git a/apps/web/app/components/library/labels.ts b/apps/web/app/components/library/labels.ts index bc9789e9..958ce731 100644 --- a/apps/web/app/components/library/labels.ts +++ b/apps/web/app/components/library/labels.ts @@ -260,6 +260,7 @@ export const ENTITY_FIELD_LABEL: Record = { cast: "演员", crew: "导演 / 主创", externalIds: "外部 ID", + logoPath: "标志", }; /** 季 / 集字段(contracts UnitMetadataField) */ diff --git a/apps/web/app/components/library/scrape-panel.tsx b/apps/web/app/components/library/scrape-panel.tsx index 6ad70372..2e18a13a 100644 --- a/apps/web/app/components/library/scrape-panel.tsx +++ b/apps/web/app/components/library/scrape-panel.tsx @@ -37,6 +37,7 @@ const ROLE: Record = { episode_nfo: "集 NFO", poster: "海报", fanart: "背景图", + clearlogo: "标志", season_poster: "季海报", episode_thumb: "剧照", }; diff --git a/apps/web/app/components/settings/cache.test.ts b/apps/web/app/components/settings/cache.test.ts new file mode 100644 index 00000000..88439ab0 --- /dev/null +++ b/apps/web/app/components/settings/cache.test.ts @@ -0,0 +1,45 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { type CacheRegionList, DEFAULT_SETTINGS } from "@umt/contracts"; +import { createElement } from "react"; +import { renderToString } from "react-dom/server"; +import { MemoryRouter } from "react-router"; +import { describe, expect, it } from "vitest"; +import { qk } from "~/lib/query-keys.ts"; +import { CacheSection, hitDetailText, hitRateText } from "./cache.tsx"; + +// 设置 → 缓存:缓存区表显示统一元数据缓存的命中率(最近 7 天、三个进程合计)。 + +const hits = { l1Hits: 50, l2Hits: 30, staleHits: 4, misses: 12, failures: 1, refreshes: 4, hitRate: 84 / 96 }; + +describe("缓存区命中率", () => { + it("命中率按百分比显示一位小数;没有统计的区显示「—」,没读取过显示「暂无读取」", () => { + expect(hitRateText(hits)).toBe("87.5%"); + expect(hitRateText(undefined)).toBe("—"); + expect(hitRateText({ ...hits, hitRate: null })).toBe("暂无读取"); + expect(hitDetailText(hits)).toBe("内存 50 · 数据库 30 · 旧值 4 · 未命中 12 · 回源失败 1 · 后台刷新 4"); + expect(hitDetailText({ ...hits, staleHits: 0, failures: 0, refreshes: 0 })).toBe("内存 50 · 数据库 30 · 未命中 12"); + }); + + it("缓存区表有「命中率」列,元数据区显示命中率与明细", () => { + const qc = new QueryClient({ defaultOptions: { queries: { retry: false, enabled: false } } }); + qc.setQueryData(qk.settings, DEFAULT_SETTINGS); + const regions: CacheRegionList = { + items: [ + { region: "tmdb.request", entries: 120, expired: 0, bytes: 4096, lastWriteAt: null, hits }, + { region: "site.detail", entries: 3, expired: 0, bytes: 100, lastWriteAt: null }, + ], + }; + qc.setQueryData(["cache", "regions"], regions); + const html = renderToString( + createElement( + QueryClientProvider, + { client: qc }, + createElement(MemoryRouter, { initialEntries: ["/console/settings#cache"] }, createElement(CacheSection)), + ), + ); + expect(html).toContain("命中率"); + expect(html).toContain("87.5%"); + expect(html).toContain("内存 50 · 数据库 30"); + expect(html).toContain("最近 7 天"); + }); +}); diff --git a/apps/web/app/components/settings/cache.tsx b/apps/web/app/components/settings/cache.tsx index a6c5bf27..abe188a5 100644 --- a/apps/web/app/components/settings/cache.tsx +++ b/apps/web/app/components/settings/cache.tsx @@ -1,4 +1,4 @@ -import { cacheRegionLabel } from "@umt/contracts"; +import { type CacheHitStats, cacheRegionLabel } from "@umt/contracts"; import { useState } from "react"; import { toast } from "sonner"; import { @@ -32,6 +32,8 @@ export type CacheRow = { expired: number; bytes: number; lastWriteAt: string | null; + /** 统一元数据缓存的区才有:最近 7 天命中统计 */ + hits?: CacheHitStats | undefined; }; /** 区名 → 行:有中文名的用中文名,没有的原样显示 region */ @@ -39,6 +41,23 @@ export function cacheRow(r: Omit): CacheRow { return { ...r, label: cacheRegionLabel(r.region) ?? r.region }; } +/** 命中率文字:没有统计的区显示「—」,有统计没读取显示「暂无读取」 */ +export function hitRateText(h: CacheHitStats | undefined): string { + if (!h) return "—"; + if (h.hitRate === null) return "暂无读取"; + return `${Math.round(h.hitRate * 1000) / 10}%`; +} + +/** 命中明细:内存 / 数据库 / 旧值 / 未命中,有失败与后台刷新时附上 */ +export function hitDetailText(h: CacheHitStats): string { + const parts = [`内存 ${h.l1Hits}`, `数据库 ${h.l2Hits}`]; + if (h.staleHits) parts.push(`旧值 ${h.staleHits}`); + parts.push(`未命中 ${h.misses}`); + if (h.failures) parts.push(`回源失败 ${h.failures}`); + if (h.refreshes) parts.push(`后台刷新 ${h.refreshes}`); + return parts.join(" · "); +} + function CacheRegionsBlock() { const regions = useCacheRegions(); const images = useImageCache(); @@ -85,6 +104,19 @@ function CacheRegionsBlock() { ), }, + { + key: "hitRate", + header: "命中率", + align: "right", + cell: (r) => ( +
+ {hitRateText(r.hits)} + {r.hits && r.hits.hitRate !== null ? ( +

{hitDetailText(r.hits)}

+ ) : null} +
+ ), + }, { key: "bytes", header: "占用", @@ -121,7 +153,7 @@ function CacheRegionsBlock() { section="cache" anchor="regions" title="缓存区" - description="清空后下次用到时重新请求，不影响作品、资源和文件。已过期的条目由定时任务清理。" + description="清空后下次用到时重新请求，不影响作品、资源和文件。已过期的条目由定时任务清理。命中率按最近 7 天、三个进程合计。" plain > @@ -224,7 +256,16 @@ export const IMAGES_CARD: CardSpec = { }), }; -const TTL_KEYS = ["tmdbSearchDays", "tmdbDetailDays", "tmdbMissDays", "siteDetailDays", "webSearchDays"] as const; +const TTL_KEYS = [ + "tmdbSearchDays", + "tmdbDetailDays", + "tmdbMissDays", + "imageDays", + "metaSearchDays", + "metaDetailDays", + "siteDetailDays", + "webSearchDays", +] as const; const ttlField = (k: string, label: string, hint?: string) => ({ name: `storage.cacheTtl.${k}`, label, @@ -234,11 +275,14 @@ const ttlField = (k: string, label: string, hint?: string) => ({ export const TTL_CARD: CardSpec = { anchor: "ttl", title: "缓存有效期", - description: "单位天。追踪中的作品命中 TMDB 详情缓存时自动续期。", + description: "单位天。过期后先用旧值、后台刷新；关注中的作品命中详情缓存时自动续期，不会因条数上限被清掉。", fields: [ ttlField("tmdbSearchDays", "TMDB 搜索与识别结果"), ttlField("tmdbDetailDays", "TMDB 详情"), ttlField("tmdbMissDays", "TMDB 空结果", "搜索无结果、未识别时的负缓存"), + ttlField("imageDays", "TMDB 人物与图片"), + ttlField("metaSearchDays", "豆瓣 / Bangumi 搜索"), + ttlField("metaDetailDays", "豆瓣 / Bangumi 条目"), ttlField("siteDetailDays", "站点详情页内容"), ttlField("webSearchDays", "网页搜索结果"), ], diff --git a/apps/web/app/components/tasks/labels.ts b/apps/web/app/components/tasks/labels.ts index 825f2fa1..5d9c1ae1 100644 --- a/apps/web/app/components/tasks/labels.ts +++ b/apps/web/app/components/tasks/labels.ts @@ -39,6 +39,7 @@ export const WORKFLOW_LABEL: Record = { verifyAndImport: "核验整理", scrapeMedia: "刮削", refreshMetadata: "刷新元数据", + prewarmMetadata: "预热元数据缓存", reconcileLibrary: "文件核对", cleanupDownloadCache: "清理缓存", resolveDecision: "处理待处理事项", diff --git a/docs/porting/moviepilot-discover.md b/docs/porting/moviepilot-discover.md index 3fb725fc..b6786d2c 100644 --- a/docs/porting/moviepilot-discover.md +++ b/docs/porting/moviepilot-discover.md @@ -46,7 +46,7 @@ | 上游 | 我们 | | --- | --- | | 每个请求 `@cached(maxsize=CONF.douban, ttl=CONF.meta, skip_none=True)`，键 = 函数 + 参数 | 列表 / 搜索请求经 `MetaRequestCache` 端口 → 服务端 `cachedCall`（region `douban.request` / `bangumi.request`，上限 5000 条）；键 = `方法 + 参数排序后的地址 + 请求体`（`metaRequestKey`） | -| TTL 只有一个 `CONF.meta` | 按类别取 `storage.cacheTtl`：搜索 `metaSearchDays`（缺省 1 天）、榜单 / 热门 / 每日放送 / 趋势页 `metaListHours`（缺省 6 小时）、空结果 `metaEmptyHours`（缺省 1 小时，不超过正常 TTL） | +| TTL 只有一个 `CONF.meta` | 按类别取 `storage.cacheTtl`（统一元数据缓存策略表，见 runbook/cache.md）：搜索 `metaSearchDays`（缺省 7 天，空结果 `metaMissDays` 1 天）、榜单 / 热门 / 每日放送 / 趋势页 `metaListHours`（缺省 6 小时，空结果 `metaEmptyHours` 1 小时），空结果 TTL 不超过正常 TTL；过期先用旧值后台刷新 | | `skip_none`：请求失败返回 None 不缓存 | 网络失败、非 2xx（含 403）、返回的不是 JSON（风控 / 验证页）、趋势页不是浏览页，适配器都抛错，`cachedCall` 不写 | | 条目详情同样走请求缓存 | 条目详情**不走**请求缓存：已有条目级缓存 `douban.subject` / `bangumi.subject`（30 天，存归一化后的标题 / 别名 / 年份，还收站点详情页与网页搜索得到的条目），再缓存一遍原始响应就是同一份数据存两遍；`meta.subject_miss`（6 小时）保留，是识别升级时防止反复打外站的负缓存，不属于请求缓存 | | 限流：无（靠缓存） | 每源进程内限流（豆瓣 2 并发、起始间隔 300 ms；bgm 3 并发、100 ms），只包真正发出的网络请求，缓存命中不占额度 | diff --git a/docs/porting/moviepilot-tmdb-cache.md b/docs/porting/moviepilot-tmdb-cache.md index 56f4f1a1..babe3b4c 100644 --- a/docs/porting/moviepilot-tmdb-cache.md +++ b/docs/porting/moviepilot-tmdb-cache.md @@ -72,3 +72,21 @@ PG 读写失败时 TMDB 路径按未命中处理(内存缓存不会失败)。 偏离:MP 失败时返回空响应,我们回 502(前端回退占位);CACHE-2 加了容量 LRU(`cache.images_lru`,按 atime,预算 `storage.budgets.imagesGb`), 按天清理保留作兜底;MP 按配置走代理服务器,我们直连(TMDB API 也未走代理); MP 只在启动时清理,我们改为每天一次的定时任务。 + +## 统一元数据缓存(2026-10-06)与 MP 的差异 + +用户定:TMDB / 豆瓣 / Bangumi 共用一套缓存策略,更积极地缓存,不上 Redis(PG `kv_cache` 二级 + 进程内存一级)。 +键、TTL 语义(正常 / 空结果短 TTL)、失败不缓存与上面各表一致;以下是 MP 没有、我们加的部分(详见 [runbook/cache.md](../runbook/cache.md)): + +| 项 | MP | 我们 | 理由 | +|---|---|---|---| +| 存储层级 | 单进程内存 `TLRUCache`(可选 Redis) | L1 进程内存(每 region 1000 条、5 分钟)+ L2 PG `kv_cache`;读 L1 → L2 → 源,写两级 | 三个进程共享 L2,热点走 L1 少一次 PG 往返 | +| TTL | `META_CACHE_EXPIRE` 一个值 + `EMPTY_RESULT_CACHE_TTL` | 「来源 × 资源类型」策略表:搜索 7 天 / 详情 30 天 / 列表 6 小时 / 人物与图片 90 天,空结果 1 天(列表 1 小时),取 `storage.cacheTtl` | 用户要求更积极地缓存,不同资源变化频率差很多 | +| 过期 | 过期即未命中 | 宽限期(TTL 的 50%,最多 7 天)内先返回旧值、后台刷新;刷新失败继续用旧值并记警告 | 元数据过期后通常仍可用,不让请求等外站;外站故障时不降级 | +| 并发 | `@cached` 无合并(TMDB 客户端内存 LRU 有) | 同进程同 key 只回源一次(含后台刷新) | 列表页 / 批量识别并发多 | +| 淘汰 | TLRU | TLRU,但关注中作品的 TMDB 条目 `pinned` 不参与 LRU,命中续期 | 关注作品的详情 / 季 / 剧集组要一直在 | +| 预热 | 无 | 作品进入关注时排 `prewarmMetadata`:详情、各季、剧集组、海报 / 背景 / 标志 | 订阅 / 下载后马上要刷新与刮削 | +| 失效 | 进程内,`clear_cache()` | `PgCache.delete / clear` 发 `pg_notify('umt_cache_invalidate')`,各进程丢 L1;监听断开时 L1 停用,重连后整体清空 | 多进程下 L1 不能拿旧值 | +| 统计 | 无 | 按 region 计 L1 / L2 / 旧值 / 未命中 / 回源失败 / 后台刷新,每分钟写 `cache_stat`,设置页显示近 7 天命中率 | 看缓存是否起作用 | +| 豆瓣 / bgm 条目查询失败 | `DoubanApi` / `BangumiApi` 失败不缓存 | 同;只有「条目不存在」写 `meta.subject_miss`(以前网络失败也记 6 小时负缓存),失败后本进程同一条目冷却 30 分钟 | 策略表「失败不缓存」 | +| TMDB 详情请求 | `append_to_response` 不含 images | 富详情加 `images`(`include_image_language=zh,en,null`),原语言不是中英文时另取 `/images`,选标志写 `clearlogo.png` | 用户定:不接 fanart.tv,用 TMDB logos;请求键变化后旧的富详情缓存会各自 miss 一次 | diff --git a/docs/runbook/cache.md b/docs/runbook/cache.md index 0e6b2ab6..11ec1d72 100644 --- a/docs/runbook/cache.md +++ b/docs/runbook/cache.md @@ -1,6 +1,6 @@ -# 缓存与限流运行手册(C2-A) +# 缓存与限流运行手册 -不引入 Redis / Deno KV。跨进程共享的状态都放 PG:站点限流令牌桶、TMDB 缓存。缓存语义照搬 MoviePilot, +不引入 Redis / Deno KV。跨进程共享的状态都放 PG:站点限流令牌桶、缓存。缓存语义照搬 MoviePilot, 对照与偏离见 [docs/porting/moviepilot-tmdb-cache.md](../porting/moviepilot-tmdb-cache.md)。 ## 跨进程站点限流 @@ -14,27 +14,113 @@ 排查:`select * from rate_limit_bucket where key = 'site:'`;`blocked_until` 不为空说明站点要求暂停。 -## PG 缓存(kv_cache) +## 统一元数据缓存 -- 表 `kv_cache`(迁移 `0042_cache`,`UNLOGGED`:不写 WAL,崩溃后清空):(region, key) 主键,value jsonb, - expires_at 每条独立,accessed_at 为最近读写时间。 -- `@umt/db` 的 `PgCache`(对照 MP `TTLCache`):set 带 ttl(秒,≤ 0 删除)与 maxsize(按 region 计条数,超出先删过期再删最久未用); - get 命中刷新最近使用;`cachedCall` 对照 MP `@cached`(skip_none / skip_if / empty_ttl / fresh)。 -- 定时任务 `cache.sweep`(维护组,每 10 分钟)物理删除过期行。 +TMDB / 豆瓣 / Bangumi 三个来源共用一套缓存层(`@umt/db` 的 `MetadataCache`,服务端入口 `apps/server/src/shared-cache.ts`)。 +用户 2026-10-06 定:更积极地缓存,不上 Redis;PG `kv_cache`(UNLOGGED)作二级,进程内存作一级。 + +### 两级 + +| 级 | 存哪 | 容量 / 有效期 | 说明 | +| --- | --- | --- | --- | +| L1 | 各进程内存(api / worker / worker:agent 各一份) | 每个 region 最多 1000 条(LRU),5 分钟,且不超过条目在 L2 的有效期 | 只在跨进程失效监听在线时启用,见下 | +| L2 | PG `kv_cache`(迁移 0042,0086 加 `fresh_until` / `pinned`) | 按 region 条数上限(TLRU,先删过期再删最久未用),每条独立 TTL | 三进程共享;UNLOGGED,PG 崩溃后清空 | + +读:L1 → L2 → 源;写:两级都写。L1 返回的是副本,调用方改了不影响缓存。PG 读写失败按未命中 / 不写处理,不让业务请求失败。 + +### 策略表(来源 × 资源类型) + +TTL 取值来自 `settings.storage.cacheTtl`(见 [storage.md](storage.md);旧库缺新字段时按缺省补),代码在 `metaCachePolicy`。 + +| 资源类型 | TMDB | 豆瓣 / Bangumi | 缺省 TTL | 空结果 / 查无 | +| --- | --- | --- | --- | --- | +| 搜索 / 识别 | `/search/*`、`/find/*`、识别缓存 `tmdb.recognize`:`tmdbSearchDays` | 搜索请求:`metaSearchDays` | 7 天 | TMDB `tmdbMissDays`、豆瓣 / bgm `metaMissDays`,1 天 | +| 详情 / 季 / 剧集组 | `/movie/{id}`、`/tv/{id}`、季、`/tv/episode_group/*`:`tmdbDetailDays` | 条目 `douban.subject` / `bangumi.subject`、发现页匹配 `discover.match`:`metaDetailDays` | 30 天 | 1 天(TMDB `tmdbMissDays`;条目不存在 `meta.subject_miss` 用 `metaMissDays`) | +| 榜单 / 热门 / 每日放送 | trending / discover / popular 等:`tmdbListHours` | 榜单、标签、筛选、每日放送、趋势页:`metaListHours` | 6 小时 | TMDB `tmdbListEmptyHours`、豆瓣 / bgm `metaEmptyHours`,1 小时 | +| 人物 / 图片 | `/person/*`、`…/images`:`imageDays` | — | 90 天 | `tmdbMissDays` | + +- 空结果:TMDB 按 `results` 为空数组(MP `EMPTY_RESULT_CACHE_TTL` 判定);豆瓣 / bgm 按各接口的列表字段为空。空结果 TTL 不超过正常 TTL。 +- **失败不缓存**:网络错误、超时、鉴权失败、被拦(403、风控页)、限流、5xx 都在适配器里抛错,不写缓存(MP `skip_none` / `skip_if`)。 + 豆瓣 / bgm 条目查询失败(不是「不存在」)时同一条目本进程冷却 30 分钟再在线查,不写缓存。 +- 条数上限:TMDB 两个 region 用 `cache.tmdbMaxSize`(缺省 5000);豆瓣 / bgm 请求 5000、条目 20000、发现匹配 20000。 +- `metaSearchDays` 缺省从 1 天改为 7 天(策略表统一);豆瓣 / bgm 搜索的空结果改按 `metaMissDays`(以前按 `metaEmptyHours`)。 + +### 过期先用旧值(stale-while-revalidate) + +- 写入时 `fresh_until = now + TTL`、`expires_at = fresh_until + 宽限期`,宽限期 = TTL 的 50%,最多 7 天。 +- 过了 `fresh_until`、还没到 `expires_at` 的条目是旧值:先返回旧值,后台刷新;同一个 key 同时只刷一次。 +- 后台刷新失败继续用旧值,记一次警告「缓存后台刷新失败，继续用旧值」;过了宽限期就是普通未命中,同步回源。 +- `PgCache.get`(非元数据缓存与旧调用方用)只认新鲜条目;`PgCache.getEntry` 带回旧值,给 `MetadataCache` 用。 +- 识别缓存、豆瓣条目这类「只读不回源」的读取(`MetadataCache.get`)只认新鲜条目。 + +### 同请求合并(single-flight) + +同进程同 region + key 并发只发一次请求,其余等同一个结果;后台刷新也走这条,刷新进行中的 key 不再发。跨进程不合并(各进程各自一次)。 + +### 关注中作品:pinned 与续期 + +- 关注口径同 `service/watched.ts isEntityWatched`(进行中的订阅、用户搜索 / 手动识别 / 人工确认、手动下载、在下载或整理、已在库)。 + 集合 = 关注中作品的 TMDB 身份(`movie:` / `tv:`)+ 选中的剧集组(`group:`),按进程缓存 5 分钟。 +- 属于关注作品的 TMDB 请求(详情、季、剧集、图片、剧集组列表、剧集组详情)写入时标 `pinned`:按条数上限淘汰时跳过(仍计入条数,到期照样过期)。 +- L2 新鲜命中时续期:过期时间推到 now + 本类 TTL(只延长不缩短,保留宽限长度)。旧值不续期,该刷新就刷新。 +- 作品不再关注后 `pinned` 不会立即清掉,到期自然过期。 + +### 进入关注时预热 + +建订阅、用户搜索 / 手动识别出资源、提交下载时(`enqueueWatchedRefresh`,原来补排 full 刷新的触发点),每次额外入队一次 +`prewarmMetadata`(网络道,幂等键 `prewarmMetadata:<作品 id>:<日期>`,同一天重复进入只跑一次): + +- 拉基础详情、富详情(含标志)、各季详情(按 `metadata.includeSpecials` 决定含不含季 0,最多 60 季)、选中的剧集组; + 请求键和刷新元数据 / 刮削完全相同,之后直接命中;预热前先清关注集合,保证这次写入就 pinned; +- 海报、背景、标志的原图拉进图片缓存目录(刮削用的就是原图);已在磁盘上的不再下载; +- 走 TMDB 客户端原有的限流与重试;详情都拿不到时网络类错误交给任务重试,其余记下结束。 +- 代码:`sources-identity/service/metadata-prewarm.ts`。 + +### 跨进程失效 + +- 每个进程启动时 `startMetadataCache` 开一条专用连接 `LISTEN umt_cache_invalidate`。 +- `PgCache.delete / clear`(含设置页「清空」、`POST /cache/tmdb/clear`、识别缓存类型冲突删除)和手动刷新(`fresh`)写入后 + `pg_notify('umt_cache_invalidate', {"r": region, "k": key})`;不带 `k` = 整个 region。各进程收到后丢掉对应 L1。 +- 普通写入不通知:别的进程 L1 里的旧值最多再用 5 分钟(L1 有效期)。 +- 监听在线才启用 L1;断开时立即停用并清空 L1,退避重连(最长 10 秒),重连成功后整体清一次再启用。 + 启动时连不上只记警告,该进程只用 L2。脚本、测试等没启动监听的进程 L1 一直关着。 + +### 命中统计 + +- 每个进程按 region 在内存里计:L1 命中、L2 命中、旧值命中(先返回旧值)、未命中、回源失败、后台刷新。 +- 每分钟把增量加进表 `cache_stat`(迁移 0086,按「天 × region」一行;30 天前的行顺带删掉),进程退出时再写一次。 +- 口径:`GET /api/v1/cache/regions` 每个元数据 region 带 `hits`:最近 7 天 `cache_stat` 之和 + 当前进程还没写入的计数; + 别的进程最近不到一分钟的计数不在内。命中率 = (L1 + L2 + 旧值) / (L1 + L2 + 旧值 + 未命中),没读取过为 null。 + 合并等待(single-flight 的跟随者)按未命中计。 +- 设置 → 缓存 → 缓存区表的「命中率」列显示百分比与明细(内存 / 数据库 / 旧值 / 未命中,有失败与后台刷新时附上);非元数据区显示「—」。 +- 排查:`select * from cache_stat where day = current_date order by region;` + +### 接到这一层的 region + +| region | 内容 | 键 | 读写方式 | +|---|---|---|---| +| `tmdb.request` | TMDB 请求响应(MP `TMDb.request`) | `path?query`(参数排序,不含凭证) | `fetch`:两级 + 旧值 + 合并 + pinned | +| `tmdb.recognize` | TMDB 识别结果(MP `TmdbCache`) | `[类型][语言]{tmdbid 或名称}-{年份}-{季}` | `get` / `set`;未识别 `{id:0}` 按空结果 TTL | +| `douban.request` / `bangumi.request` | 豆瓣 / bgm 搜索与列表请求 | `方法 地址?排序后的参数 [请求体]` | `fetch` | +| `douban.subject` / `bangumi.subject` | 条目(标题 / 别名 / 年份) | 条目 id | 在线查走 `fetch`;站点详情页 / 网页摘要写入走 `set` | +| `meta.subject_miss` | 豆瓣 / bgm 条目不存在 | `来源:id` | `set` / `get` | +| `discover.match` | 豆瓣 / bgm 条目 → TMDB 作品的确定匹配 | `来源:id` | `get` / `set` | + +其它 `kv_cache` 区(`site.detail`、`websearch`、`enrich*`、`identity.detail_ids`)仍直接用 `PgCache`,没有 L1 与命中统计: | region | 内容 | 键 | TTL | 条数上限 | |---|---|---|---|---| -| `tmdb.request` | TMDB 请求响应(MP `TMDb.request`) | `path?query`(不含凭证) | 搜索 / find:`storage.cacheTtl.tmdbSearchDays`(7 天);详情 / 季:`tmdbDetailDays`(30 天,追踪中的作品命中续期);`results` 为空:`tmdbMissDays`(1 天) | `cache.tmdbMaxSize` | -| `tmdb.recognize` | TMDB 识别结果(MP `TmdbCache`) | `[类型][语言]{tmdbid 或名称}-{年份}-{季}` | `tmdbSearchDays`;未识别 `{id:0}` 按 `tmdbMissDays` | `cache.tmdbMaxSize` | | `site.detail` | 站点详情页扩展内容 | `source_release.id` | `max(站点详情缓存期, storage.cacheTtl.siteDetailDays)`(7 天) | 5000 | | `websearch` | 网页搜索结果 | 参数 sha256 | `storage.cacheTtl.webSearchDays`(1 天) | 1000 | -TTL 设置(CACHE-2 起):`settings.storage.cacheTtl`,见 [storage.md](storage.md)。MP 的 `META_CACHE_EXPIRE` 只有一个值, -我们按请求类别拆成搜索 / 详情两档,`EMPTY_RESULT_CACHE_TTL` 对应 `tmdbMissDays`;缓存语义不变。 -条数上限仍是 `cache.tmdbMaxSize`(缺省 5000;MP `CONF.tmdb` 为 256,本项目缓存在 PG 里不占进程内存)。旧字段 `cache.tmdbTtlHours`、`cache.emptyResultTtlSec` 不再读取。 +### 清空与清理 -API:`GET /api/v1/cache/tmdb?kind=recognize|request&limit=` 列出条目(键 / 过期 / 大小,识别条目带 tmdbId / 标题 / 海报); -`POST /api/v1/cache/tmdb/clear` 清空两类(仅 admin)。 +- 设置 → 缓存 → 缓存区:按 region 清空(`POST /api/v1/cache/regions/{region}/clear`,仅 admin),同时通知各进程丢 L1。 +- `POST /api/v1/cache/tmdb/clear`:清 `tmdb.request` 与 `tmdb.recognize`(对应 MP `clear_cache()`)。 +- 定时任务 `cache.sweep`(维护组,每 10 分钟)物理删除已过硬过期(`expires_at`)的行;宽限期内的旧值不删。 +- 改了 TTL:新写入的条目按新值,已有条目保持原有效期,必要时清空。 + +API:`GET /api/v1/cache/tmdb?kind=recognize|request&limit=` 列出条目(键 / 过期 / 大小,识别条目带 tmdbId / 标题 / 海报)。 识别缓存何时读写(`sources-identity/service/tmdb-recognize-cache.ts`): - 识别资源时先读:命中且有标题 → 按 id 取 TMDB 详情作为唯一远端候选;命中 `{id:0}` → 不再搜索;强制复查(force)不读。 @@ -48,20 +134,23 @@ API:`GET /api/v1/cache/tmdb?kind=recognize|request&limit=` 列出条目(键 / file 只允许 `[A-Za-z0-9_-]+.(jpg|jpeg|png|webp)`。 - 缓存路径对照 MP `sanitize_url_path`:`DATA_DIR/cache/images/t/p//`。 - `cache.globalImageCache`(缺省关闭,MP `GLOBAL_IMAGE_CACHE`)开启才读磁盘缓存;拉到的合法图片总会写盘。 + 刮削与关注作品预热取原图(`original`),总是先读磁盘缓存。 - 按文件头识别图片类型,非图片 / SVG 拒绝;ETag = md5(内容),`Cache-Control: public, max-age=604800`,If-None-Match 相同回 304。 - 上游超时(20s)/ 非 200 / 非图片 → 502,前端回退无彩占位。 - 清理:定时任务 `cache.images_clean`(每天)删除 mtime 早于 `cache.globalImageCacheDays`(缺省 7)天的文件。 - 容量 LRU(CACHE-2):`cache.images_lru`(30 分钟)在总量超过 `storage.budgets.imagesGb`(缺省 20 GB)时按最久未访问删到 90%; 读盘命中会刷新文件 atime。见 [storage.md](storage.md)。 -- 用量 / 清空:`GET /api/v1/cache/images`、`POST /api/v1/cache/images/clear`(仅 admin);设置 → 缓存(缓存区表里的「图片」行可清空;「图片缓存」节开关与保留天数)。按区统计与清空通用缓存:`GET /api/v1/cache/regions`、`POST /api/v1/cache/regions/{region}/clear`(仅 admin),设置 → 缓存 → 缓存区。 +- 用量 / 清空:`GET /api/v1/cache/images`、`POST /api/v1/cache/images/clear`(仅 admin);设置 → 缓存(缓存区表里的「图片」行可清空;「图片缓存」节开关与保留天数)。 前端:作品库列表、作品详情、搜索面板的 TMDB 候选在有 `posterPath`(metadata.posterPath / TMDB 候选)时显示海报, -加载失败回退原来的无彩占位。TMDB 详情刷新会把 posterPath / backdropPath 写进 metadata_field。 +加载失败回退原来的无彩占位。TMDB 详情刷新会把 posterPath / backdropPath / logoPath 写进 metadata_field。 ## 常见问题 - TMDB 缓存不生效:确认 `select region, count(*) from kv_cache group by 1` 有 `tmdb.request`;PG 读写失败时按未命中处理并记 - `TMDB 缓存读写失败，按未命中处理`。 + `缓存读取失败，按未命中处理` / `缓存写入失败`。 +- 命中率一直是 0 或没有:`cache_stat` 有没有今天的行;进程日志里有没有 `缓存失效监听启动失败`(只影响 L1,L2 命中照样计)。 +- 清空后别的进程还拿到旧值:看该进程有没有 `缓存失效监听断开` 日志;断线期间 L1 是停用的,不会拿旧值。 - 改了 TMDB 缓存设置:新写入的条目按新 TTL / 上限;已有条目保持原过期时间,必要时清空。 -- TTL 拉长后条目更早被条数上限挤掉:`cache.tmdbMaxSize` 缺省 5000,追踪作品很多时再调大。 +- TTL 拉长后条目更早被条数上限挤掉:`cache.tmdbMaxSize` 缺省 5000,关注作品的条目 pinned 不受影响,其余按最久未用淘汰。 - 数据库重启后缓存为空:UNLOGGED 表的正常行为。 diff --git a/docs/runbook/dev.md b/docs/runbook/dev.md index 21b45ad1..955a6c5f 100644 --- a/docs/runbook/dev.md +++ b/docs/runbook/dev.md @@ -99,6 +99,7 @@ ci 逐行核对这张表与任务注册表(`apps/server/test/worker-lanes.test.t | `resolveObservation` | network | 标题识别 + TMDB | | `enrichPoolEntry` | network | 未识别资源的元数据补全 | | `refreshMetadata` | network | TMDB 元数据刷新 | +| `prewarmMetadata` | network | 关注作品预热:详情 / 各季 / 剧集组 / 海报背景标志进缓存(docs/runbook/cache.md) | | `searchSubscription` | network | 订阅搜索 | | `snapshotSite` | network | 站点快照 | | `refreshSiteAccount` | network | 站点登录 / 用户数据刷新 | diff --git a/docs/runbook/scrape.md b/docs/runbook/scrape.md index 0afa2b30..d2bfe6c8 100644 --- a/docs/runbook/scrape.md +++ b/docs/runbook/scrape.md @@ -108,16 +108,20 @@ TMDB 有这一集但没有译名 / 简介(只有占位名和播出日期)不算 | 选中的 | 写的文件 | | --- | --- | -| 剧集目录 | `tvshow.nfo`、`poster.jpg`、`fanart.jpg`、`seasonNN-poster.jpg`;每个季目录(`Season 1` / `S01` / `Specials` / `SPs` / `第 1 季`)`season.nfo` + `poster.jpg`;每集 `<视频名>.nfo` + `<视频名>-thumb.jpg`。只进季目录一层,其他子目录不管 | +| 剧集目录 | `tvshow.nfo`、`poster.jpg`、`fanart.jpg`、`clearlogo.png`(有 TMDB 标志时)、`seasonNN-poster.jpg`;每个季目录(`Season 1` / `S01` / `Specials` / `SPs` / `第 1 季`)`season.nfo` + `poster.jpg`;每集 `<视频名>.nfo` + `<视频名>-thumb.jpg`。只进季目录一层,其他子目录不管 | | 季目录 | `season.nfo` + 季目录 `poster.jpg` + 上级 `seasonNN-poster.jpg`;每集 NFO 与剧照。不写 `tvshow.nfo` | | 单集文件 | 该集 `<视频名>.nfo` + 剧照 | -| 电影目录 | 目录里每个视频 `<视频名>.nfo`(不进子目录)+ 目录 `poster.jpg` / `fanart.jpg`;蓝光原盘目录(含 `BDMV` / `VIDEO_TS`)写 `<目录名>.nfo` | -| 电影文件 | `<视频名>.nfo` + 同目录 `poster.jpg` / `fanart.jpg`;文件直接在登记目录根时图片改为 `<视频名>-poster.jpg` / `-fanart.jpg` | +| 电影目录 | 目录里每个视频 `<视频名>.nfo`(不进子目录)+ 目录 `poster.jpg` / `fanart.jpg` / `clearlogo.png`;蓝光原盘目录(含 `BDMV` / `VIDEO_TS`)写 `<目录名>.nfo` | +| 电影文件 | `<视频名>.nfo` + 同目录 `poster.jpg` / `fanart.jpg` / `clearlogo.png`;文件直接在登记目录根时图片改为 `<视频名>-poster.jpg` / `-fanart.jpg` / `-clearlogo.png` | - 自定义平铺模板(多部电影放在同一目录):目录里有登记在别的作品下的视频,或只刮部分视频时目录里还有没登记到本作品的视频, 图片改为 `<视频名>-poster.jpg` / `-fanart.jpg`,不写整个目录共用的 `poster.jpg`。作品详情「重新刮削」对电影只刮该作品已入库的视频, 不给同目录别的电影写本作品的 NFO(同一作品的多个版本照旧共用一张 poster)。 - 图片扩展名按下载到的实际类型(jpeg / png / webp),URL 认不出或与内容不符时以内容为准。 +- 标志(clearlogo,2026-10-06):刷新元数据时 TMDB 详情带 `images`(`include_image_language=zh,en,null`,作品原语言不是中英文时另取原语言), + 选一张 PNG 标志存为作品字段 `logoPath`,语言优先 作品原语言 → 中文 → 英文 → 无文字,同语言取评分最高;SVG 标志不用。 + 文件名按 Kodi / Emby / Jellyfin 通用约定 `clearlogo.png`:电影写影片目录、剧集写剧集根目录(季目录不写);覆盖方式与 hash 规则同其它图片。 + 标志变化算 NFO 相关字段,刮削过的作品会延后排一次「更新」模式重刮。不接 fanart.tv。 - 集号:已登记视频按覆盖的集单元;其余按路径解析(季号缺失时取季目录名,再缺按第 1 季;同 MP 对每集重新 `MetaInfoPath`)。 认不出集号的视频记 `scrape_no_episode` 跳过,记录算部分完成。 @@ -166,6 +170,7 @@ TMDB 有这一集但没有译名 / 简介(只有占位名和播出日期)不算 | 作品 / 电影目录背景图 | `backdrop.jpg` + `fanart.jpg` | `fanart.jpg` | 三家都认 fanart.jpg,副本没用 | | 每集剧照 | `<视频名>.jpg` | `<视频名>-thumb.jpg` | Kodi 约定,Jellyfin / Emby 同样识别;不和同名封面混 | | 视频直接在登记目录根 | 根里写 `poster.jpg` / `backdrop.jpg` / `fanart.jpg` | `<视频名>-poster.jpg` / `<视频名>-fanart.jpg` | 根里放一张 poster.jpg 会被当成整个库的封面 | +| 作品级标志 | 只在接 fanart.tv 时写 logo | 作品海报旁写 `clearlogo.png`(根下电影 `<视频名>-clearlogo.png`),来源 TMDB logos | 用户定:不接 fanart.tv,TMDB 标志够用(差分登记 `tmdb_clearlogo`) | | 其余(tvshow / season / 集 NFO、季海报两处、电影 NFO、蓝光 `<目录名>.nfo`、季目录只写季级、单集只写该集) | — | 同 MP | — | 行为上的差异: @@ -177,7 +182,7 @@ TMDB 有这一集但没有译名 / 简介(只有占位名和播出日期)不算 | 识别 | 按路径识别,认不出直接失败;对话框可手填 TMDB / 豆瓣 id | 按路径识别,认不准时返回前 5 候选让用户选(也可搜索);已登记的目录直接用作品 | 选一次比手填 id 省事;已登记的不必再识别 | | 执行 | 请求内同步 | 后台任务 `scrapeMedia`,请求内最多等 20 s | 大季包图片多;超时不丢 | | 记录 | 只有日志 | 每次刮削一条记录(目录、识别结果、逐文件结果、原因),作品详情显示最近一条 | 可回看、可排查 | -| 图片种类 | poster / backdrop / logo / banner / disc / clearart / thumb(按图片源) | poster / fanart / 季海报 / 剧照 | 本地只有 TMDB 海报、背景图、剧照;其余要 fanart.tv 源 | +| 图片种类 | poster / backdrop / logo / banner / disc / clearart / thumb(按图片源) | poster / fanart / clearlogo / 季海报 / 剧照 | 本地只有 TMDB 海报、背景图、标志、剧照;其余要 fanart.tv 源(不接) | | 批量 | 多选共用一个识别条件 | 多选逐项各自识别,各自选候选 | 多选里常是不同作品 | | 音乐 | 支持 | 不做 | 范围外 | diff --git a/docs/runbook/storage.md b/docs/runbook/storage.md index c8b469f2..3b0b74f7 100644 --- a/docs/runbook/storage.md +++ b/docs/runbook/storage.md @@ -26,8 +26,16 @@ PATCH /api/v1/settings {"storage":{"retentionDays":{"observation":60}}} | `retentionDays.capabilityStats` | 90 | 能力调用审计(`target_type=capability`)与已结束的 `capability_invocation` | | `retentionDays.identityPoolArchive` | 30 | 待识别池 unmatched 条目无新证据多少天后归档(不删) | | `cacheTtl.tmdbSearchDays` | 7 | TMDB `/search/*`、`/find/*` 响应与识别缓存 | -| `cacheTtl.tmdbDetailDays` | 30 | TMDB 详情 / 季。追踪中的作品命中时续期 | +| `cacheTtl.tmdbDetailDays` | 30 | TMDB 详情 / 季 / 剧集组。关注中的作品命中时续期 | | `cacheTtl.tmdbMissDays` | 1 | TMDB 空结果与未识别负缓存 | +| `cacheTtl.tmdbListHours` | 6 | TMDB 榜单 / 热门 / 趋势 / 按条件筛选(小时) | +| `cacheTtl.tmdbListEmptyHours` | 1 | TMDB 列表空结果(小时) | +| `cacheTtl.imageDays` | 90 | TMDB 人物 / 图片列表(`/person/*`、`…/images`) | +| `cacheTtl.metaSearchDays` | 7 | 豆瓣 / bgm 搜索请求(2026-10-06 起缺省 7 天,以前 1 天) | +| `cacheTtl.metaDetailDays` | 30 | 豆瓣 / bgm 条目、发现页条目 → TMDB 匹配 | +| `cacheTtl.metaMissDays` | 1 | 豆瓣 / bgm 搜索空结果、条目不存在 | +| `cacheTtl.metaListHours` | 6 | 豆瓣 / bgm 榜单、热门、每日放送等列表(小时) | +| `cacheTtl.metaEmptyHours` | 1 | 豆瓣 / bgm 列表空结果(小时) | | `cacheTtl.siteDetailDays` | 7 | 站点详情页扩展内容(`kv_cache` `site.detail`) | | `cacheTtl.webSearchDays` | 1 | 网页搜索结果 | @@ -39,11 +47,13 @@ PATCH /api/v1/settings {"storage":{"retentionDays":{"observation":60}}} ## 缓存有效期 只改 TTL 的取值来源,缓存语义还是对照 MP(见 [porting/moviepilot-tmdb-cache.md](../porting/moviepilot-tmdb-cache.md))。 +2026-10-06 起 TMDB / 豆瓣 / Bangumi 按「来源 × 资源类型」统一取值、过期先用旧值(宽限期 TTL 的 50%,最多 7 天), +完整策略表、两级缓存与关注作品的 pinned / 续期见 [cache.md](cache.md)「统一元数据缓存」。 -- TMDB 请求缓存:键以 `/search/` 或 `/find/` 开头的用搜索 TTL,其余用详情 TTL。`results` 为空的用未命中 TTL。 +- TMDB 请求缓存:键以 `/search/` 或 `/find/` 开头的用搜索 TTL;榜单类用列表 TTL;`/person/*`、`…/images` 用图片 TTL;其余用详情 TTL。`results` 为空的用未命中 TTL。 - TMDB 识别缓存:正常条目用搜索 TTL(识别结果就是名称到条目的映射),负缓存 `{id:0}` 用未命中 TTL。 -- 追踪中的作品续期:作品有未结束的订阅(active / paused / pending)且有已接受的 TMDB 身份, - 它的详情键(`/movie/?…`、`/tv/…`)命中时把过期时间推到「现在 + 详情 TTL」,只延长不缩短。追踪集合每进程缓存 5 分钟。 +- 关注中的作品续期:口径同 `isEntityWatched`,它的详情 / 季 / 剧集组 / 图片键命中时把过期时间推到「现在 + 本类 TTL」, + 只延长不缩短,并标 pinned(条数上限淘汰时跳过)。关注集合每进程缓存 5 分钟。 - 站点详情:`kv_cache` TTL 取 `max(站点详情缓存期, siteDetailDays)`。是否重抓仍按站点详情缓存期 (`sites.defaultDetailCacheSec`,站点定义可更长)判断,下载前复核促销和 H&R 走的是这条。 - 改了 TTL:新写入的条目按新值,已有条目保持原过期时间。 @@ -71,7 +81,7 @@ PATCH /api/v1/settings {"storage":{"retentionDays":{"observation":60}}} | `storage.retention.identity_pool` | `identity_candidate` | 6 小时 | pending + unmatched,`coalesce(last_attempt_at, created_at)` 早于保留期 → 改为 archived | | `storage.analyze` | 全库 | 7 天 | `ANALYZE` | | `cache.images_lru` | 图片缓存 | 30 分钟 | 见上 | -| `cache.sweep` | `kv_cache` | 10 分钟 | 已过期 | +| `cache.sweep` | `kv_cache` | 10 分钟 | 已过硬过期(宽限期内的旧值保留) | 分批删除(`modules/system/retention.ts` 的 `deleteInBatches`): diff --git a/packages/adapters/src/tmdb/client.ts b/packages/adapters/src/tmdb/client.ts index d94c76fc..1a2b2f47 100644 --- a/packages/adapters/src/tmdb/client.ts +++ b/packages/adapters/src/tmdb/client.ts @@ -105,7 +105,15 @@ export interface TmdbConfig { * listTtlMs:流行趋势 / 热门 / 正在上映等列表与 /discover/ 的 TTL(缺省 6 小时且不超过 ttlMs;列表会变,不能按详情缓存 30 天), * 这类请求的空结果 TTL 也不超过它。 */ - cache?: { max?: number; ttlMs?: number; searchTtlMs?: number; emptyTtlMs?: number; listTtlMs?: number }; + cache?: { + max?: number; + ttlMs?: number; + searchTtlMs?: number; + emptyTtlMs?: number; + listTtlMs?: number; + /** 人物 / 图片列表(/person/*、/images);缺省同 ttlMs */ + imageTtlMs?: number; + }; /** * 跨进程共享的请求缓存(服务端传 PG 缓存);给了就不再用进程内 LRU。 * 语义对照 MP @cached(skip_none、业务失败不缓存、results 为空按 emptyTtlMs)。 @@ -124,6 +132,27 @@ type Json = Record; export interface TmdbSharedCache { get(key: string): Promise; set(key: string, value: unknown, ttlMs: number): Promise; + /** + * 给了就由它负责整条读取路径(服务端统一元数据缓存:两级、过期先用旧值、同请求合并、统计、关注作品续期), + * TTL 按 req.kind 取服务端策略表;load 抛错(网络、鉴权、限流、5xx)不缓存。没给时走 get / set。 + */ + fetch?( + req: { key: string; kind: TmdbRequestKind; isEmpty: (v: unknown) => boolean }, + load: () => Promise, + ): Promise; +} + +/** 请求类别(缓存 TTL 的依据):搜索 / find、列表、人物 / 图片、其余为详情(含季、剧集组) */ +export type TmdbRequestKind = "search" | "list" | "image" | "detail"; + +/** 人物 / 图片类请求:/person/*、…/images */ +export const isTmdbImagePath = (path: string) => /^\/person\//.test(path) || /\/images$/.test(path); + +export function tmdbRequestKind(path: string): TmdbRequestKind { + if (/^\/(search|find)\//.test(path)) return "search"; + if (isTmdbListPath(path)) return "list"; + if (isTmdbImagePath(path)) return "image"; + return "detail"; } function requestQuery(params: Record): URLSearchParams { @@ -186,10 +215,16 @@ export class TmdbClient { const key = `${path}?${qs}`; let emptyTtl = this.cfg.cache?.emptyTtlMs ?? 30 * 60_000; const baseTtl = this.cfg.cache?.ttlMs ?? 24 * 3600_000; - let ttl = /^\/(search|find)\//.test(path) ? (this.cfg.cache?.searchTtlMs ?? baseTtl) : baseTtl; - if (isTmdbListPath(path)) { + const kind = tmdbRequestKind(path); + let ttl = kind === "search" ? (this.cfg.cache?.searchTtlMs ?? baseTtl) : baseTtl; + if (kind === "list") { ttl = this.cfg.cache?.listTtlMs ?? Math.min(baseTtl, 6 * 3600_000); emptyTtl = Math.min(emptyTtl, ttl); + } else if (kind === "image") ttl = this.cfg.cache?.imageTtlMs ?? baseTtl; + if (this.cfg.sharedCache?.fetch) { + return (await this.cfg.sharedCache.fetch({ key, kind, isEmpty: isEmptyResult }, () => + this.limiter.run(() => this.fetchWithRetry(path, qs)), + )) as T; } if (this.cfg.sharedCache) { const shared = this.cfg.sharedCache; diff --git a/packages/adapters/src/tmdb/details.test.ts b/packages/adapters/src/tmdb/details.test.ts index 71551fe8..1dc8518e 100644 --- a/packages/adapters/src/tmdb/details.test.ts +++ b/packages/adapters/src/tmdb/details.test.ts @@ -9,6 +9,7 @@ import { parseAltTitles, parseTranslations, pickChineseTitle, + pickLogo, pickText, pickTitle, } from "./index.ts"; @@ -223,3 +224,70 @@ describe("fetchRichDetail / fetchRichSeason(mock TMDB)", () => { expect(mock.calls.length).toBe(n); }); }); + +describe("TMDB 标志(clearlogo)", () => { + const logos = (rows: { file_path: string; iso_639_1: string | null; vote_average?: number }[]) => ({ logos: rows }); + + it("语言优先 原语言 → 中文 → 英文 → 无文字;同语言按评分;只要 PNG", () => { + const all = logos([ + { file_path: "/none.png", iso_639_1: null, vote_average: 9 }, + { file_path: "/en.png", iso_639_1: "en", vote_average: 5 }, + { file_path: "/zh-low.png", iso_639_1: "zh", vote_average: 3 }, + { file_path: "/zh-high.png", iso_639_1: "zh", vote_average: 6 }, + { file_path: "/ja.svg", iso_639_1: "ja", vote_average: 9 }, + { file_path: "/ja.png", iso_639_1: "ja", vote_average: 1 }, + ]); + expect(pickLogo([all], "ja")).toBe("/ja.png"); + expect(pickLogo([all], "ko")).toBe("/zh-high.png"); + expect( + pickLogo( + [ + logos([ + { file_path: "/en.png", iso_639_1: "en" }, + { file_path: "/n.png", iso_639_1: null }, + ]), + ], + "zh", + ), + ).toBe("/en.png"); + expect(pickLogo([logos([{ file_path: "/n.png", iso_639_1: null }])], "en")).toBe("/n.png"); + expect(pickLogo([logos([{ file_path: "/x.svg", iso_639_1: "en" }])], "en")).toBeNull(); + expect(pickLogo([undefined], null)).toBeNull(); + }); + + it("详情请求带 images(zh,en,null);原语言不是中英文时另取原语言标志", async () => { + const fx = TMDB_FIXTURES.map((f) => + f.id === 872585 + ? { + ...f, + logos: [ + { file_path: "/oppen-en.png", iso_639_1: "en" }, + { file_path: "/oppen-zh.png", iso_639_1: "zh" }, + ], + } + : f.id === 95396 + ? { + ...f, + original_language: "ja", + logos: [ + { file_path: "/sev-ja.png", iso_639_1: "ja" }, + { file_path: "/sev-en.png", iso_639_1: "en" }, + ], + } + : f, + ); + mock = await createMockTmdb({ fixtures: fx }); + const tmdb = createTmdbClient({ baseUrl: `${mock.url}/3`, apiKey: "test-tmdb-key", retries: 0, timeoutMs: 1000 }); + const movie = await fetchRichDetail(tmdb, "movie", 872585, CHAIN); + expect(movie.logoPath).toBe("/oppen-en.png"); + const call = mock.calls.find((c) => c.path === "/3/movie/872585")!; + expect(call.query).toMatchObject({ include_image_language: "zh,en,null" }); + expect(call.query.append_to_response).toContain("images"); + expect(mock.calls.some((c) => c.path === "/3/movie/872585/images")).toBe(false); + const tv = await fetchRichDetail(tmdb, "tv", 95396, CHAIN); + expect(tv.logoPath).toBe("/sev-ja.png"); + expect(mock.calls.find((c) => c.path === "/3/tv/95396/images")?.query).toMatchObject({ + include_image_language: "ja", + }); + }); +}); diff --git a/packages/adapters/src/tmdb/details.ts b/packages/adapters/src/tmdb/details.ts index 332f1995..13f8303e 100644 --- a/packages/adapters/src/tmdb/details.ts +++ b/packages/adapters/src/tmdb/details.ts @@ -82,6 +82,8 @@ export interface TmdbRichDetail { originCountry: string[]; posterPath: string | null; backdropPath: string | null; + /** 标志(透明 PNG,刮削写 clearlogo.png);语言优先 原语言 → 中文 → 英文 → 无文字;没有为 null(旧数据可缺省) */ + logoPath?: string | null; cast: TmdbCredit[]; crew: TmdbCredit[]; externalIds: { imdb: string | null; tvdb: string | null; wikidata: string | null }; @@ -163,9 +165,48 @@ function credits(raw: unknown, type: MediaEntityType, createdBy: unknown): { cas return { cast, crew: crew.slice(0, 6) }; } +/** 详情请求带回的图片语言:中文、英文、无文字;作品原语言不在其中时另取一次 /images */ +export const DETAIL_IMAGE_LANGUAGES = "zh,en,null"; + +interface LogoRow { + path: string; + lang: string | null; + vote: number; + count: number; +} + +const logoRows = (images: unknown): LogoRow[] => { + const logos = ((images ?? {}) as Json).logos; + return ((Array.isArray(logos) ? logos : []) as Json[]).flatMap((x) => { + const path = s(x.file_path); + // 只要 PNG:Kodi / Emby / Jellyfin 的 clearlogo 都是 png,TMDB 的 svg 标志不用 + if (!path || !/\.png$/i.test(path)) return []; + return [{ path, lang: s(x.iso_639_1), vote: n(x.vote_average) ?? 0, count: n(x.vote_count) ?? 0 }]; + }); +}; + /** - * 富详情:一次请求 `/{type}/{id}?language=链首&append_to_response=alternative_titles,credits,external_ids,translations`, - * 标题 / 简介 / 标语按链从 translations(+ 同地区别名)取。不存在抛 TmdbError(notfound)。 + * 选一张标志:语言优先 作品原语言 → 中文 → 英文 → 无文字;同语言按评分、票数从高到低。没有 PNG 标志返回 null。 + * images 可以是多份(详情带回的 + 原语言单独取的)。 + */ +export function pickLogo(images: unknown[], originalLanguage: string | null): string | null { + const rows = images.flatMap(logoRows); + const orig = originalLanguage?.toLowerCase().split("-")[0] ?? null; + const order = [orig, "zh", "en", null].filter((x, i, a) => a.indexOf(x) === i && (x !== null || i === a.length - 1)); + for (const lang of order) { + const hit = rows + .filter((r) => (r.lang?.toLowerCase() ?? null) === lang) + .sort((a, b) => b.vote - a.vote || b.count - a.count)[0]; + if (hit) return hit.path; + } + return null; +} + +/** + * 富详情:一次请求 `/{type}/{id}?language=链首&append_to_response=alternative_titles,credits,external_ids,translations,images` + * (`include_image_language=zh,en,null`),标题 / 简介 / 标语按链从 translations(+ 同地区别名)取。 + * 作品原语言不是中文 / 英文时另取 `/{type}/{id}/images?include_image_language=<原语言>` 选标志(失败忽略)。 + * 不存在抛 TmdbError(notfound)。 */ export async function fetchRichDetail( client: Pick, @@ -177,7 +218,8 @@ export async function fetchRichDetail( const requested = chainOf(languages); const j = await client.getJson(`/${type}/${id}`, { language: requested[0], - append_to_response: "alternative_titles,credits,external_ids,translations", + append_to_response: "alternative_titles,credits,external_ids,translations,images", + include_image_language: DETAIL_IMAGE_LANGUAGES, }); const titleKey = type === "movie" ? "title" : "name"; const originalTitle = s(type === "movie" ? j.original_title : j.original_name); @@ -212,6 +254,15 @@ export async function fetchRichDetail( const ext = (j.external_ids ?? {}) as Json; const runtimes = Array.isArray(j.episode_run_time) ? (j.episode_run_time as unknown[]) : []; const { cast, crew } = credits(j.credits, type, j.created_by); + const logoSources: unknown[] = [j.images]; + const origLang = originalLanguage?.toLowerCase().split("-")[0] ?? null; + if (origLang && origLang !== "zh" && origLang !== "en") { + try { + logoSources.unshift(await client.getJson(`/${type}/${id}/images`, { include_image_language: origLang })); + } catch { + // 原语言标志取不到就用中文 / 英文 / 无文字的 + } + } return { entityType: type, externalId: String(id), @@ -240,6 +291,7 @@ export async function fetchRichDetail( originCountry, posterPath: s(j.poster_path), backdropPath: s(j.backdrop_path), + logoPath: pickLogo(logoSources, originalLanguage), cast, crew, externalIds: { diff --git a/packages/adapters/src/tmdb/tmdb.test.ts b/packages/adapters/src/tmdb/tmdb.test.ts index 83fa3cf8..a976843a 100644 --- a/packages/adapters/src/tmdb/tmdb.test.ts +++ b/packages/adapters/src/tmdb/tmdb.test.ts @@ -7,6 +7,7 @@ import { matchTmdbMulti, type TmdbConfig, TmdbError, + tmdbRequestKind, yearRange, } from "./index.ts"; @@ -135,6 +136,44 @@ describe("TmdbClient 缓存/限流/错误分类", () => { expect(tmdb.cacheStats().size).toBe(0); }); + it("共享缓存给了 fetch:整条读取交给它,带请求类别(搜索 / 详情 / 列表 / 图片)与空结果判定;失败不进缓存", async () => { + const seen: { key: string; kind: string; empty: boolean }[] = []; + const store = new Map(); + const { mock, tmdb } = await setup({ + sharedCache: { + get: async () => null, + set: async () => {}, + fetch: async (req, load) => { + if (store.has(req.key)) return store.get(req.key); + const v = await load(); + seen.push({ key: req.key, kind: req.kind, empty: req.isEmpty(v) }); + store.set(req.key, v); + return v; + }, + }, + }); + await tmdb.details("movie", 872585); + await tmdb.details("movie", 872585); + await tmdb.search("不存在的片名xyz", { type: "movie" }); + await tmdb.getJson("/movie/872585/images", {}); + await tmdb.getJson("/trending/all/week", {}); + await expect(tmdb.getJson("/movie/999999999", {})).rejects.toThrow(); + const byPath = new Map(seen.map((x) => [x.key.split("?")[0], [x.kind, x.empty]])); + expect(Object.fromEntries(byPath)).toEqual({ + "/movie/872585": ["detail", false], + "/search/movie": ["search", true], + "/movie/872585/images": ["image", false], + "/trending/all/week": ["list", false], + }); + const detailCalls = mock.callsTo("/3/movie/872585").length; + await tmdb.details("movie", 872585); + expect(mock.callsTo("/3/movie/872585")).toHaveLength(detailCalls); + expect(seen.some((x) => x.key.startsWith("/movie/999999999"))).toBe(false); + expect(tmdbRequestKind("/person/1")).toBe("image"); + expect(tmdbRequestKind("/tv/1/season/2")).toBe("detail"); + expect(tmdbRequestKind("/find/tt1")).toBe("search"); + }); + it("并发合并:同一请求并发只发一次", async () => { const { mock, tmdb } = await setup(); await Promise.all(Array.from({ length: 5 }, () => tmdb.season(100088, 1))); diff --git a/packages/contracts/src/cache.ts b/packages/contracts/src/cache.ts index 2b748e57..4ff11ce8 100644 --- a/packages/contracts/src/cache.ts +++ b/packages/contracts/src/cache.ts @@ -82,6 +82,27 @@ export function tmdbImageUrl(path: string | null | undefined, size: TmdbImageSiz export const CacheRegionName = z.string().regex(/^[a-z0-9][a-z0-9._-]{0,63}$/, "缓存区名不合法"); export const CacheRegionParam = z.object({ region: CacheRegionName }); +/** + * 命中统计口径(docs/runbook/cache.md):各进程在内存里计数,每分钟把增量加进 cache_stat(按天 × region), + * 接口返回最近 7 天各进程之和 + 当前进程还没写入的部分(别的进程最近不到一分钟的计数不在内)。 + * hitRate = (l1Hits + l2Hits + staleHits) / (l1Hits + l2Hits + staleHits + misses);没有读取时为 null。 + */ +export const CacheHitStats = z.object({ + /** 进程内存(一级)命中 */ + l1Hits: NonNegInt, + /** PG(二级)命中 */ + l2Hits: NonNegInt, + /** 过期在宽限期内、先返回旧值的次数 */ + staleHits: NonNegInt, + misses: NonNegInt, + /** 回源失败(网络、被拦、5xx) */ + failures: NonNegInt, + /** 后台刷新次数 */ + refreshes: NonNegInt, + hitRate: z.number().min(0).max(1).nullable(), +}); +export type CacheHitStats = z.infer; + export const CacheRegionStat = z.object({ region: z.string(), /** 未过期条目数 */ @@ -92,6 +113,8 @@ export const CacheRegionStat = z.object({ bytes: NonNegInt, /** 最近一次写入 */ lastWriteAt: Timestamp.nullable(), + /** 命中统计(统一元数据缓存的 region 才有):最近 7 天,见 CacheHitStats */ + hits: CacheHitStats.optional(), }); export type CacheRegionStat = z.infer; diff --git a/packages/contracts/src/content.ts b/packages/contracts/src/content.ts index 473b985f..b4090ff3 100644 --- a/packages/contracts/src/content.ts +++ b/packages/contracts/src/content.ts @@ -220,6 +220,8 @@ export const EntityMetadataField = z.enum([ "cast", "crew", "externalIds", + // 统一元数据缓存一轮:TMDB 标志(刮削写 clearlogo.png) + "logoPath", ]); export type EntityMetadataField = z.infer; diff --git a/packages/contracts/src/scrape.ts b/packages/contracts/src/scrape.ts index d86078ce..614d02a3 100644 --- a/packages/contracts/src/scrape.ts +++ b/packages/contracts/src/scrape.ts @@ -9,7 +9,7 @@ import { Id, NonNegInt, Timestamp } from "./common.ts"; export const ScrapeFileResult = z.object({ path: z.string(), kind: z.enum(["nfo", "image"]), - /** movie_nfo / tvshow_nfo / season_nfo / episode_nfo / poster / fanart / season_poster / episode_thumb */ + /** movie_nfo / tvshow_nfo / season_nfo / episode_nfo / poster / fanart / clearlogo / season_poster / episode_thumb */ role: z.string(), action: z.enum(["written", "skipped", "failed"]), /** 跳过 / 失败原因码(reasons.ts:scrape_exists / scrape_write_failed / scrape_image_failed …) */ diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index df4c843f..28a91969 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -578,8 +578,19 @@ export const StorageCacheTtl = z.object({ metaSearchDays: Days, /** 豆瓣 / bgm 榜单、热门、每日放送等列表请求(小时) */ metaListHours: z.number().int().min(1).max(720), - /** 豆瓣 / bgm 空结果(小时,对照 MP empty_ttl) */ + /** 豆瓣 / bgm 列表空结果(小时,对照 MP empty_ttl);搜索空结果改按 metaMissDays */ metaEmptyHours: z.number().int().min(1).max(720), + // ── 统一元数据缓存(2026-10-06,docs/runbook/cache.md 策略表);旧库缺这些字段时按缺省补 ── + /** 豆瓣 / bgm 条目详情(douban.subject / bangumi.subject)与发现页条目 → TMDB 匹配(discover.match) */ + metaDetailDays: Days, + /** 豆瓣 / bgm 查无结果:搜索空结果、条目不存在(meta.subject_miss) */ + metaMissDays: Days, + /** TMDB 榜单 / 热门 / 趋势 / 按条件筛选(小时) */ + tmdbListHours: z.number().int().min(1).max(720), + /** TMDB 列表空结果(小时) */ + tmdbListEmptyHours: z.number().int().min(1).max(720), + /** TMDB 人物 / 图片列表(/person/*、/images) */ + imageDays: Days, }); export type StorageCacheTtl = z.infer; @@ -608,9 +619,14 @@ export const DEFAULT_STORAGE_SETTINGS: StorageSettings = { tmdbMissDays: 1, siteDetailDays: 7, webSearchDays: 1, - metaSearchDays: 1, + metaSearchDays: 7, metaListHours: 6, metaEmptyHours: 1, + metaDetailDays: 30, + metaMissDays: 1, + tmdbListHours: 6, + tmdbListEmptyHours: 1, + imageDays: 90, }, }; diff --git a/packages/contracts/test/db-sync.test.ts b/packages/contracts/test/db-sync.test.ts index 026c736f..28462f3c 100644 --- a/packages/contracts/test/db-sync.test.ts +++ b/packages/contracts/test/db-sync.test.ts @@ -149,6 +149,8 @@ const INTERNAL_TABLES = [ // C2-A 跨进程限流桶(0041)与 PG 缓存(0042):服务端内部,缓存经 /cache/* 的专用 DTO 暴露 "RateLimitBucket", "KvCache", + // 统一元数据缓存命中统计(0086):服务端内部,经 GET /cache/regions 的 hits 字段暴露 + "CacheStat", // 站点请求计数(0046):服务端内部,经 GET /sites/{id}/request-stats 的专用 DTO 暴露 "SiteRequestCount", // W9-I 进程心跳(0034):服务端内部,经 /system/health 的专用 DTO 暴露 diff --git a/packages/db/migrations/0086_metadata_cache.ts b/packages/db/migrations/0086_metadata_cache.ts new file mode 100644 index 00000000..3da25625 --- /dev/null +++ b/packages/db/migrations/0086_metadata_cache.ts @@ -0,0 +1,41 @@ +import type { Kysely } from "kysely"; +import { dropTables, execDdl } from "../src/migration-helpers.ts"; + +// 统一元数据缓存(docs/runbook/cache.md): +// - kv_cache.fresh_until:新鲜期截止。过了 fresh_until、还没到 expires_at 的条目是「旧值」: +// MetadataCache 先返回旧值再后台刷新(stale-while-revalidate);PgCache.get 只返回新鲜条目。null = 没有宽限期。 +// - kv_cache.pinned:关注中作品的条目,写入时按条数上限淘汰(LRU)跳过它们;到期仍会过期。 +// - cache_stat:按天 × region 汇总各进程的命中统计(进程内计数,定期把增量加进来)。 + +// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型 +export async function up(db: Kysely): Promise { + await execDdl( + db, + ` +alter table kv_cache add column fresh_until timestamptz; +alter table kv_cache add column pinned boolean not null default false; +create table cache_stat ( + day date not null, + region text not null, + l1_hits bigint not null default 0, + l2_hits bigint not null default 0, + stale_hits bigint not null default 0, + misses bigint not null default 0, + failures bigint not null default 0, + refreshes bigint not null default 0, + updated_at timestamptz not null default now(), + primary key (day, region) +); +`, + ); +} + +// biome-ignore lint/suspicious/noExplicitAny: 同上 +export async function down(db: Kysely): Promise { + await execDdl( + db, + `${dropTables(["cache_stat"])} +alter table kv_cache drop column if exists pinned; +alter table kv_cache drop column if exists fresh_until;`, + ); +} diff --git a/packages/db/src/cache.ts b/packages/db/src/cache.ts index e36d0a60..0e692c3f 100644 --- a/packages/db/src/cache.ts +++ b/packages/db/src/cache.ts @@ -1,4 +1,5 @@ import { type Kysely, sql } from "kysely"; +import pg from "pg"; import type { DB } from "./types.ts"; /** @@ -24,10 +25,37 @@ export interface SetOptions { ttl?: number; /** region 条数上限;缺省 DEFAULT_CACHE_SIZE */ maxsize?: number; + /** + * 宽限期(秒,0086):> 0 时 fresh_until = now + ttl、expires_at = now + ttl + grace, + * 宽限期内的条目由 MetadataCache 先返回旧值再后台刷新;PgCache.get 只认新鲜条目。缺省 0。 + */ + grace?: number; + /** 关注中作品的条目:按条数上限淘汰时跳过(到期照样过期)。缺省 false */ + pinned?: boolean; +} + +/** 跨进程 L1 失效通知频道(payload 见 invalidationPayload) */ +export const CACHE_INVALIDATE_CHANNEL = "umt_cache_invalidate"; + +/** 通知内容:`{"r":region,"k":key}`;k 缺省 = 整个 region;r 为 "*" = 全部。超长 key 退化为整 region */ +function invalidationPayload(region: string, key?: string): string { + const full = JSON.stringify(key === undefined ? { r: region } : { r: region, k: key }); + return full.length > 7000 ? JSON.stringify({ r: region }) : full; +} + +/** getEntry 的结果:stale = 已过新鲜期、还在宽限期内 */ +export interface CacheEntry { + value: T; + stale: boolean; + /** 新鲜期截止(毫秒);没有宽限期时同 expiresAt */ + freshUntil: number; + /** 硬过期(毫秒) */ + expiresAt: number; + pinned: boolean; } export class PgCache { - constructor(private readonly db: AnyDb) {} + constructor(readonly db: AnyDb) {} /** MemoryBackend.set */ async set(region: string, key: string, value: unknown, opts: SetOptions = {}): Promise { @@ -36,36 +64,83 @@ export class PgCache { await this.delete(region, key); return; } - await sql`insert into kv_cache (region, key, value, expires_at, updated_at, accessed_at) - values (${region}, ${key}, ${JSON.stringify(value ?? null)}::jsonb, now() + make_interval(secs => ${ttl}), now(), now()) + const grace = Math.max(0, opts.grace ?? 0); + await sql`insert into kv_cache (region, key, value, expires_at, fresh_until, pinned, updated_at, accessed_at) + values (${region}, ${key}, ${JSON.stringify(value ?? null)}::jsonb, + now() + make_interval(secs => ${ttl + grace}), + ${grace > 0 ? sql`now() + make_interval(secs => ${ttl})` : sql`null`}, + ${opts.pinned === true}, now(), now()) on conflict (region, key) do update set value = excluded.value, expires_at = excluded.expires_at, + fresh_until = excluded.fresh_until, pinned = excluded.pinned, updated_at = excluded.updated_at, accessed_at = excluded.accessed_at`.execute(this.db); await this.evict(region, opts.maxsize ?? DEFAULT_CACHE_SIZE); } - /** TLRUCache 写入时的淘汰:先删已过期,再按最近使用从旧到新删到 maxsize */ + /** + * TLRUCache 写入时的淘汰:先删已过期,再按最近使用从旧到新删到 maxsize。 + * 0086:pinned(关注中作品)的条目不参与 LRU,但计入条数;刚写入的那条总留着。 + */ private async evict(region: string, maxsize: number): Promise { await sql`delete from kv_cache where region = ${region} and expires_at <= now()`.execute(this.db); - await sql`delete from kv_cache where region = ${region} and key in ( - select key from kv_cache where region = ${region} - order by accessed_at desc, updated_at desc offset ${Math.max(1, Math.floor(maxsize))})`.execute(this.db); + await sql`delete from kv_cache where region = ${region} and not pinned and key in ( + select key from kv_cache where region = ${region} and not pinned + order by accessed_at desc, updated_at desc + offset greatest(1, ${Math.floor(maxsize)} - (select count(*) from kv_cache where region = ${region} and pinned)))`.execute( + this.db, + ); } - /** MemoryBackend.get:未命中 / 过期返回 null;命中刷新最近使用时间 */ + /** MemoryBackend.get:未命中 / 过期 / 已过新鲜期返回 null;命中刷新最近使用时间 */ async get(region: string, key: string): Promise { const r = await sql<{ value: T }>`update kv_cache set accessed_at = now() - where region = ${region} and key = ${key} and expires_at > now() returning value`.execute(this.db); + where region = ${region} and key = ${key} and expires_at > now() + and (fresh_until is null or fresh_until > now()) returning value`.execute(this.db); return r.rows.length ? (r.rows[0]!.value as T) : null; } - /** 续期:未过期的条目把 expires_at 推到 now + ttl(只延长不缩短);返回是否有行被续期 */ - async renew(region: string, key: string, ttl: number): Promise { + /** 读一条(含宽限期内的旧值),命中刷新最近使用时间;MetadataCache 的 L2 读 */ + async getEntry(region: string, key: string): Promise | null> { + const r = await sql<{ value: T; expires_at: Date; fresh_until: Date | null; pinned: boolean; now: Date }>` + update kv_cache set accessed_at = now() + where region = ${region} and key = ${key} and expires_at > now() + returning value, expires_at, fresh_until, pinned, now() as now`.execute(this.db); + const row = r.rows[0]; + if (!row) return null; + const expiresAt = new Date(row.expires_at).getTime(); + const freshUntil = row.fresh_until ? new Date(row.fresh_until).getTime() : expiresAt; + return { + value: row.value as T, + stale: freshUntil <= new Date(row.now).getTime(), + freshUntil, + expiresAt, + pinned: !!row.pinned, + }; + } + + /** + * 续期:新鲜的条目把过期时间推到 now + ttl(只延长不缩短,有宽限期的保留原宽限长度);返回是否有行被续期。 + * opts.pin = true 时顺带标为 pinned。旧值(已过新鲜期)不续期:该刷新就刷新。 + */ + async renew(region: string, key: string, ttl: number, opts: { pin?: boolean } = {}): Promise { if (ttl <= 0) return false; - const r = await sql`update kv_cache set expires_at = greatest(expires_at, now() + make_interval(secs => ${ttl})) - where region = ${region} and key = ${key} and expires_at > now()`.execute(this.db); + const r = await sql`update kv_cache set + expires_at = greatest(expires_at, + now() + make_interval(secs => ${ttl}) + coalesce(expires_at - fresh_until, interval '0')), + fresh_until = case when fresh_until is null then null + else greatest(fresh_until, now() + make_interval(secs => ${ttl})) end, + pinned = pinned or ${opts.pin === true} + where region = ${region} and key = ${key} and expires_at > now() + and (fresh_until is null or fresh_until > now())`.execute(this.db); return Number(r.numAffectedRows ?? 0) > 0; } + /** 通知各进程丢掉 L1(删除 / 清空 / 手动刷新时);失败不抛 */ + async notifyInvalidate(region: string, key?: string): Promise { + await sql`select pg_notify(${CACHE_INVALIDATE_CHANNEL}, ${invalidationPayload(region, key)})` + .execute(this.db) + .catch(() => {}); + } + async exists(region: string, key: string): Promise { const r = await sql`select 1 from kv_cache where region = ${region} and key = ${key} and expires_at > now()`.execute( @@ -74,13 +149,16 @@ export class PgCache { return r.rows.length > 0; } + /** 删除并通知各进程丢掉这条 L1 */ async delete(region: string, key: string): Promise { await sql`delete from kv_cache where region = ${region} and key = ${key}`.execute(this.db); + await this.notifyInvalidate(region, key); } - /** 清一个区,返回删除行数 */ + /** 清一个区,返回删除行数;通知各进程丢掉这个区的 L1 */ async clear(region: string): Promise { const r = await sql`delete from kv_cache where region = ${region}`.execute(this.db); + await this.notifyInvalidate(region); return Number(r.numAffectedRows ?? 0); } @@ -88,14 +166,24 @@ export class PgCache { async items( region: string, limit = 5000, - ): Promise<{ key: string; value: T; expiresAt: Date; updatedAt: Date; bytes: number }[]> { - const r = await sql<{ key: string; value: T; expires_at: Date; updated_at: Date; bytes: number }>` - select key, value, expires_at, updated_at, pg_column_size(value)::int as bytes + ): Promise<{ key: string; value: T; expiresAt: Date; freshUntil: Date; updatedAt: Date; bytes: number }[]> { + const r = await sql<{ + key: string; + value: T; + expires_at: Date; + fresh_until: Date; + updated_at: Date; + bytes: number; + }>` + select key, value, expires_at, coalesce(fresh_until, expires_at) as fresh_until, updated_at, + pg_column_size(value)::int as bytes from kv_cache where region = ${region} and expires_at > now() order by key limit ${limit}`.execute(this.db); return r.rows.map((x) => ({ key: x.key, value: x.value, expiresAt: x.expires_at, + /** 新鲜期截止(有宽限期的条目早于 expiresAt) */ + freshUntil: x.fresh_until, updatedAt: x.updated_at, bytes: Number(x.bytes), })); @@ -181,3 +269,428 @@ export function pgCacheFor(db: AnyDb): PgCache { } return c; } + +// ── 统一元数据缓存(TMDB / 豆瓣 / Bangumi 共用;docs/runbook/cache.md) ── +// 两级:L1 进程内存(按 region 限条数的 LRU,TTL 短)+ L2 PG kv_cache(三进程共享)。读 L1 → L2 → 源,写两级都写。 +// L1 只在跨进程失效通知(LISTEN umt_cache_invalidate)在线时启用:离线或断线重连时整体清空并停用, +// 避免别的进程清空 / 删除后本进程还拿着旧值。 + +/** 宽限期:TTL 的 50%,上限 7 天 */ +export const STALE_GRACE_RATIO = 0.5; +export const STALE_GRACE_MAX_SEC = 7 * 86_400; +export const defaultGrace = (ttlSec: number) => Math.min(Math.max(0, ttlSec) * STALE_GRACE_RATIO, STALE_GRACE_MAX_SEC); + +export interface FetchPolicy { + /** 秒 */ + ttl: number; + /** 空结果 TTL(秒);缺省同 ttl */ + emptyTtl?: number; + /** 空结果判定;缺省按假值(null / 空数组 / 空对象) */ + isEmpty?: (v: T) => boolean; + maxsize?: number; + /** 宽限期(秒);缺省 defaultGrace(本条 TTL);0 = 过期就回源 */ + grace?: number; + /** 缺省 true:null / undefined 不缓存 */ + skipNone?: boolean; + /** 返回真值时不缓存(业务失败的响应) */ + skipIf?: (v: T) => boolean; + /** true:跳过读(手动刷新),照常写,并通知各进程丢掉这条 L1 */ + fresh?: boolean; + /** + * 关注中作品的条目:写入时标 pinned(LRU 不淘汰),L2 命中时按 ttl 续期。 + * 只在写入 / L2 命中时调用,失败按 false。 + */ + pin?: () => Promise; +} + +export interface CacheCounters { + l1Hits: number; + l2Hits: number; + /** 先返回旧值(宽限期内)的次数 */ + staleHits: number; + misses: number; + /** 回源失败(网络、被拦、5xx 等抛错) */ + failures: number; + /** 后台刷新次数 */ + refreshes: number; +} + +const zero = (): CacheCounters => ({ l1Hits: 0, l2Hits: 0, staleHits: 0, misses: 0, failures: 0, refreshes: 0 }); + +export interface MetadataCacheOptions { + /** L1 每个 region 的条数上限,缺省 1000 */ + l1MaxPerRegion?: number; + /** L1 TTL(毫秒),缺省 5 分钟;不超过条目在 L2 的新鲜期 */ + l1TtlMs?: number; + /** 是否立即启用 L1(测试用;服务端由失效监听上线时打开) */ + l1?: boolean; + now?: () => number; + logger?: { warn(obj: object, msg: string): void }; +} + +interface L1Entry { + value: unknown; + freshUntil: number; + /** L1 内的过期(min(写入 + l1Ttl, 硬过期)) */ + expiresAt: number; +} + +const clone = (v: T): T => (v === null || typeof v !== "object" ? v : structuredClone(v)); + +export class MetadataCache { + private readonly l1 = new Map>(); + private readonly inflight = new Map>(); + private readonly counters = new Map(); + private l1On: boolean; + private readonly now: () => number; + + constructor( + readonly pg: PgCache, + private readonly opts: MetadataCacheOptions = {}, + ) { + this.l1On = opts.l1 ?? false; + this.now = opts.now ?? Date.now; + } + + // ── L1 ── + + get l1Enabled(): boolean { + return this.l1On; + } + + /** 打开 / 关闭 L1;状态变化时都整体清空 */ + setL1Enabled(on: boolean): void { + if (this.l1On !== on) this.l1.clear(); + this.l1On = on; + } + + /** 丢掉本进程 L1:给 key 丢一条,不给丢整个 region,region 为 "*" 全部丢 */ + dropLocal(region: string, key?: string): void { + if (region === "*") this.l1.clear(); + else if (key === undefined) this.l1.delete(region); + else this.l1.get(region)?.delete(key); + } + + /** L1 当前条数(测试 / 诊断) */ + l1Size(region: string): number { + return this.l1.get(region)?.size ?? 0; + } + + private l1Get(region: string, key: string): L1Entry | null { + if (!this.l1On) return null; + const m = this.l1.get(region); + const e = m?.get(key); + if (!m || !e) return null; + if (e.expiresAt <= this.now()) { + m.delete(key); + return null; + } + m.delete(key); + m.set(key, e); + return e; + } + + private l1Set(region: string, key: string, value: unknown, freshUntil: number, hardExpiresAt: number): void { + if (!this.l1On) return; + let m = this.l1.get(region); + if (!m) { + m = new Map(); + this.l1.set(region, m); + } + m.delete(key); + m.set(key, { + value: clone(value), + freshUntil, + expiresAt: Math.min(this.now() + (this.opts.l1TtlMs ?? 5 * 60_000), hardExpiresAt), + }); + const max = this.opts.l1MaxPerRegion ?? 1000; + while (m.size > max) { + const oldest = m.keys().next().value; + if (oldest === undefined) break; + m.delete(oldest); + } + } + + // ── 统计 ── + + private count(region: string, k: keyof CacheCounters, n = 1): void { + let c = this.counters.get(region); + if (!c) { + c = zero(); + this.counters.set(region, c); + } + c[k] += n; + } + + /** 本进程还没写进 cache_stat 的计数 */ + pendingStats(): Map { + return new Map([...this.counters].map(([r, c]) => [r, { ...c }])); + } + + /** + * 把本进程的计数增量加进 cache_stat(按天 × region),成功后清零;失败保留到下次。 + * 顺带删掉 30 天前的统计行。返回写入的 region 数。 + */ + async flushStats(): Promise { + const batch = [...this.counters]; + if (!batch.length) return 0; + this.counters.clear(); + let done = 0; + try { + for (const [region, c] of batch) { + await sql`insert into cache_stat (day, region, l1_hits, l2_hits, stale_hits, misses, failures, refreshes, updated_at) + values (current_date, ${region}, ${c.l1Hits}, ${c.l2Hits}, ${c.staleHits}, ${c.misses}, ${c.failures}, ${c.refreshes}, now()) + on conflict (day, region) do update set + l1_hits = cache_stat.l1_hits + excluded.l1_hits, l2_hits = cache_stat.l2_hits + excluded.l2_hits, + stale_hits = cache_stat.stale_hits + excluded.stale_hits, misses = cache_stat.misses + excluded.misses, + failures = cache_stat.failures + excluded.failures, refreshes = cache_stat.refreshes + excluded.refreshes, + updated_at = now()`.execute(this.pg.db); + done++; + } + await sql`delete from cache_stat where day < current_date - 30`.execute(this.pg.db); + return done; + } catch (err) { + // 没写进去的加回去 + for (const [region, c] of batch.slice(done)) + for (const k of Object.keys(c) as (keyof CacheCounters)[]) this.count(region, k, c[k]); + throw err; + } + } + + private warn(obj: object, msg: string) { + this.opts.logger?.warn(obj, msg); + } + + // ── 读写 ── + + /** 只读(不回源):L1 → L2 新鲜条目;记命中 / 未命中 */ + async get(region: string, key: string): Promise { + const l1 = this.l1Get(region, key); + if (l1 && l1.freshUntil > this.now()) { + this.count(region, "l1Hits"); + return clone(l1.value) as T; + } + const e = await this.pg.getEntry(region, key).catch((err) => { + this.warn({ region, err: { message: (err as Error).message } }, "缓存读取失败，按未命中处理"); + return null; + }); + if (!e || e.stale || e.value === null || e.value === undefined) { + this.count(region, "misses"); + return null; + } + this.count(region, "l2Hits"); + this.l1Set(region, key, e.value, e.freshUntil, e.expiresAt); + return e.value; + } + + /** 直接写两级(ttl ≤ 0 删除) */ + async set(region: string, key: string, value: unknown, opts: SetOptions = {}): Promise { + await this.pg.set(region, key, value, opts); + const ttl = opts.ttl ?? DEFAULT_CACHE_TTL; + if (ttl <= 0) { + this.dropLocal(region, key); + return; + } + const fresh = this.now() + ttl * 1000; + this.l1Set(region, key, value, fresh, fresh + Math.max(0, opts.grace ?? 0) * 1000); + } + + async delete(region: string, key: string): Promise { + this.dropLocal(region, key); + await this.pg.delete(region, key); + } + + async clear(region: string): Promise { + this.dropLocal(region); + return await this.pg.clear(region); + } + + /** + * 带回源的读(MP @cached 的调用体 + 两级 + 过期先用旧值 + 同请求合并 + 统计): + * - L1 / L2 新鲜命中直接返回;L2 命中且 pin() 为真时续期; + * - 宽限期内的旧值先返回,后台刷新(同 key 同时只刷一次),刷新失败继续用旧值并记一次警告; + * - 未命中:同进程同 key 只发一次 load;load 抛错不写缓存(网络、被拦、5xx);结果按 skipNone / skipIf 决定是否写。 + * PG 读写失败按未命中 / 不写处理,不让业务请求失败。 + */ + async fetch(region: string, key: string, p: FetchPolicy, load: () => Promise): Promise { + if (!p.fresh) { + const l1 = this.l1Get(region, key); + if (l1) { + if (l1.freshUntil > this.now()) { + this.count(region, "l1Hits"); + return clone(l1.value) as T; + } + this.count(region, "staleHits"); + this.revalidate(region, key, p, load); + return clone(l1.value) as T; + } + const e = await this.pg.getEntry(region, key).catch((err) => { + this.warn({ region, err: { message: (err as Error).message } }, "缓存读取失败，按未命中处理"); + return null; + }); + if (e && this.shouldCache(p, e.value)) { + this.l1Set(region, key, e.value, e.freshUntil, e.expiresAt); + if (!e.stale) { + this.count(region, "l2Hits"); + if (p.pin) await this.renewPinned(region, key, p); + return e.value; + } + this.count(region, "staleHits"); + this.revalidate(region, key, p, load); + return e.value; + } + } + this.count(region, "misses"); + return (await this.loadOnce(region, key, p, load)) as T; + } + + private shouldCache(p: FetchPolicy, v: T | null | undefined): v is T { + if ((p.skipNone ?? true) && (v === null || v === undefined)) return false; + if (p.skipIf && v !== null && v !== undefined && p.skipIf(v)) return false; + return true; + } + + private async renewPinned(region: string, key: string, p: FetchPolicy): Promise { + try { + if (await p.pin!()) await this.pg.renew(region, key, p.ttl, { pin: true }); + } catch (err) { + this.warn({ region, err: { message: (err as Error).message } }, "缓存续期失败"); + } + } + + private loadOnce(region: string, key: string, p: FetchPolicy, load: () => Promise): Promise { + const id = `${region}\u0000${key}`; + const pending = this.inflight.get(id); + if (pending) return pending as Promise; + const run = (async () => { + let v: T; + try { + v = await load(); + } catch (err) { + this.count(region, "failures"); + throw err; + } + if (this.shouldCache(p, v)) await this.write(region, key, p, v); + return v; + })().finally(() => this.inflight.delete(id)); + this.inflight.set(id, run); + return run; + } + + /** 宽限期内的后台刷新:同 key 已在刷新 / 加载就不再发 */ + private revalidate(region: string, key: string, p: FetchPolicy, load: () => Promise): void { + if (this.inflight.has(`${region}\u0000${key}`)) return; + this.count(region, "refreshes"); + this.loadOnce(region, key, { ...p, fresh: false }, load).catch((err) => + this.warn({ region, key, err: { message: (err as Error).message } }, "缓存后台刷新失败，继续用旧值"), + ); + } + + private async write(region: string, key: string, p: FetchPolicy, v: T): Promise { + const empty = p.isEmpty ? p.isEmpty(v) : isFalsy(v); + const ttl = empty && p.emptyTtl !== undefined ? p.emptyTtl : p.ttl; + const grace = p.grace ?? defaultGrace(ttl); + let pinned = false; + if (p.pin) pinned = await p.pin().catch(() => false); + try { + await this.pg.set(region, key, v, { ttl, grace, pinned, ...(p.maxsize ? { maxsize: p.maxsize } : {}) }); + } catch (err) { + this.warn({ region, err: { message: (err as Error).message } }, "缓存写入失败"); + } + const fresh = this.now() + ttl * 1000; + this.l1Set(region, key, v, fresh, fresh + grace * 1000); + if (p.fresh) await this.pg.notifyInvalidate(region, key); + } +} + +const metadataCaches = new WeakMap(); + +/** 同一个 Kysely 实例共用一个 MetadataCache(L2 用 pgCacheFor(db)) */ +export function metadataCacheFor(db: AnyDb, opts?: MetadataCacheOptions): MetadataCache { + let c = metadataCaches.get(db); + if (!c) { + c = new MetadataCache(pgCacheFor(db), opts); + metadataCaches.set(db, c); + } + return c; +} + +export interface InvalidationListenerOptions { + connectionString: string; + logger?: { warn(obj: object, msg: string): void; info(obj: object, msg: string): void }; + /** 断线重连退避上限,缺省 10s */ + maxReconnectMs?: number; +} + +/** + * 跨进程 L1 失效:一条专用连接 LISTEN umt_cache_invalidate,收到通知丢掉对应 L1。 + * 连上时清空并启用 L1;断开时停用(清空)L1 并退避重连,重连成功再整体清一次后启用。 + */ +export async function listenCacheInvalidation( + cache: MetadataCache, + opts: InvalidationListenerOptions, +): Promise<{ stop(): Promise; connected(): boolean }> { + let client: pg.Client | null = null; + let stopped = false; + let timer: ReturnType | null = null; + + const connect = async () => { + const c = new pg.Client({ connectionString: opts.connectionString }); + c.on("notification", (msg) => { + if (msg.channel !== CACHE_INVALIDATE_CHANNEL) return; + try { + const p = JSON.parse(msg.payload ?? "{}") as { r?: string; k?: string }; + if (typeof p.r === "string") cache.dropLocal(p.r, typeof p.k === "string" ? p.k : undefined); + } catch { + cache.dropLocal("*"); + } + }); + const lost = (err?: Error) => { + if (client !== c) return; + client = null; + cache.setL1Enabled(false); + c.removeAllListeners("notification"); + c.end().catch(() => {}); + if (!stopped) { + opts.logger?.warn({ err: err ? { message: err.message } : undefined }, "缓存失效监听断开，L1 停用并准备重连"); + reconnect(500); + } + }; + c.on("error", lost); + c.on("end", () => lost()); + await c.connect(); + await c.query(`listen ${CACHE_INVALIDATE_CHANNEL}`); + client = c; + cache.setL1Enabled(false); + cache.setL1Enabled(true); + }; + const reconnect = (delay: number) => { + if (stopped || timer) return; + timer = setTimeout(async () => { + timer = null; + try { + await connect(); + opts.logger?.info({}, "缓存失效监听已重连"); + } catch (err) { + opts.logger?.warn({ err: { message: (err as Error).message } }, "缓存失效监听重连失败"); + reconnect(Math.min(delay * 2, opts.maxReconnectMs ?? 10_000)); + } + }, delay); + }; + + await connect(); + return { + connected: () => client !== null, + async stop() { + stopped = true; + if (timer) clearTimeout(timer); + const c = client; + client = null; + cache.setL1Enabled(false); + if (c) { + c.removeAllListeners("notification"); + await c.end().catch(() => {}); + } + }, + }; +} diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index 031e3928..7c7a2962 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -1,13 +1,24 @@ // 统一从本包取 Kysely 与 sql,避免各包各装一份 kysely export { type Kysely, type RawBuilder, type Selectable, sql, type Transaction } from "kysely"; export { + CACHE_INVALIDATE_CHANNEL, + type CacheCounters, type CachedOptions, + type CacheEntry, cachedCall, DEFAULT_CACHE_SIZE, DEFAULT_CACHE_TTL, + defaultGrace, + type FetchPolicy, + listenCacheInvalidation, + MetadataCache, + type MetadataCacheOptions, + metadataCacheFor, PgCache, pgCacheFor, type SetOptions as CacheSetOptions, + STALE_GRACE_MAX_SEC, + STALE_GRACE_RATIO, } from "./cache.ts"; export { type CreateDbOptions, createDb } from "./client.ts"; export { createMigrator, MIGRATIONS_DIR, type MigratorOptions, migrateToLatest, resetSchema } from "./migrator.ts"; diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index 041d46d2..3bbf065e 100644 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -617,6 +617,18 @@ export interface BrowserTabTable { updated_at: GeneratedTimestamptz; } +export interface CacheStatTable { + day: string; + region: string; + l1_hits: Generated; + l2_hits: Generated; + stale_hits: Generated; + misses: Generated; + failures: Generated; + refreshes: Generated; + updated_at: GeneratedTimestamptz; +} + export interface CapabilityInvocationTable { id: Generated; idempotency_key: string; @@ -883,6 +895,8 @@ export interface KvCacheTable { expires_at: Timestamptz; updated_at: GeneratedTimestamptz; accessed_at: GeneratedTimestamptz; + fresh_until: Timestamptz | null; + pinned: Generated; } export interface LibraryRootTable { @@ -1798,6 +1812,7 @@ export interface DB { browser_provider: BrowserProviderTable; browser_session: BrowserSessionTable; browser_tab: BrowserTabTable; + cache_stat: CacheStatTable; capability_invocation: CapabilityInvocationTable; content_unit: ContentUnitTable; decision: DecisionTable; @@ -1929,6 +1944,9 @@ export type BrowserSessionUpdate = Updateable; export type BrowserTabRow = Selectable; export type NewBrowserTab = Insertable; export type BrowserTabUpdate = Updateable; +export type CacheStatRow = Selectable; +export type NewCacheStat = Insertable; +export type CacheStatUpdate = Updateable; export type CapabilityInvocationRow = Selectable; export type NewCapabilityInvocation = Insertable; export type CapabilityInvocationUpdate = Updateable; diff --git a/packages/db/test/metadata-cache.test.ts b/packages/db/test/metadata-cache.test.ts new file mode 100644 index 00000000..441ce55a --- /dev/null +++ b/packages/db/test/metadata-cache.test.ts @@ -0,0 +1,209 @@ +import { Kysely, PostgresDialect, sql } from "kysely"; +import pg from "pg"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { type DB, listenCacheInvalidation, MetadataCache, migrateToLatest, PgCache } from "../src/index.ts"; + +// 统一元数据缓存(0086):两级读写、跨进程失效、过期先用旧值、同请求合并、失败不缓存、pinned 不被淘汰、命中统计。 +const url = process.env.DATABASE_URL ?? "postgres://umt:umt@localhost:55433/umt"; +const skip = process.env.UMT_SKIP_DB_TESTS === "1"; +const schema = `umt_mcache_${process.pid}_${Date.now().toString(36)}`; + +const until = async (cond: () => boolean | Promise, ms = 5000) => { + const end = Date.now() + ms; + while (!(await cond())) { + if (Date.now() > end) throw new Error("等待超时"); + await new Promise((r) => setTimeout(r, 20)); + } +}; + +describe.skipIf(skip)("MetadataCache(真实 PG)", () => { + let admin: Kysely; + let db: Kysely; + /** 另一个「进程」:独立连接池 */ + let db2: Kysely; + + const pool = () => + new Kysely({ + dialect: new PostgresDialect({ + pool: new pg.Pool({ connectionString: url, max: 4, options: `-c search_path=${schema},public` }), + }), + }); + + beforeAll(async () => { + admin = new Kysely({ dialect: new PostgresDialect({ pool: new pg.Pool({ connectionString: url, max: 1 }) }) }); + await sql.raw(`create schema ${schema}`).execute(admin); + db = pool(); + db2 = pool(); + await migrateToLatest(db, { migrationTableSchema: schema }); + }); + + afterAll(async () => { + await db?.destroy(); + await db2?.destroy(); + await sql.raw(`drop schema if exists ${schema} cascade`).execute(admin); + await admin?.destroy(); + }); + + it("读 L1 → L2 → 源,写两级;另一进程从 L2 命中;统计各级命中", async () => { + const a = new MetadataCache(new PgCache(db), { l1: true }); + const b = new MetadataCache(new PgCache(db2), { l1: true }); + let n = 0; + const load = async () => ({ v: ++n }); + const p = { ttl: 600 }; + expect(await a.fetch("r1", "k", p, load)).toEqual({ v: 1 }); + expect(await a.fetch("r1", "k", p, load)).toEqual({ v: 1 }); + expect(await b.fetch("r1", "k", p, load)).toEqual({ v: 1 }); + expect(await b.fetch("r1", "k", p, load)).toEqual({ v: 1 }); + expect(n).toBe(1); + expect(a.pendingStats().get("r1")).toMatchObject({ misses: 1, l1Hits: 1, l2Hits: 0 }); + expect(b.pendingStats().get("r1")).toMatchObject({ misses: 0, l2Hits: 1, l1Hits: 1 }); + // L1 返回副本:调用方改了不污染缓存 + const got = await a.fetch<{ v: number }>("r1", "k", p, load); + got.v = 99; + expect(await a.fetch("r1", "k", p, load)).toEqual({ v: 1 }); + // 有宽限期:fresh_until = now + ttl,expires_at 再加 ttl 的 50% + const r = await sql<{ g: number }>`select extract(epoch from expires_at - fresh_until)::int as g + from kv_cache where region = 'r1' and key = 'k'`.execute(db); + expect(r.rows[0]!.g).toBe(300); + // 统计写进 cache_stat 后清零 + expect(await a.flushStats()).toBe(1); + await b.flushStats(); + expect(a.pendingStats().size).toBe(0); + const s = await sql<{ l1: number; l2: number; m: number }>`select l1_hits::int as l1, l2_hits::int as l2, + misses::int as m from cache_stat where region = 'r1'`.execute(db); + expect(s.rows[0]).toEqual({ l1: 4, l2: 1, m: 1 }); + }); + + it("跨进程失效:删除 / 清空通知其它进程丢 L1;监听断开时 L1 停用,重连后清空再启用", async () => { + const a = new MetadataCache(new PgCache(db)); + const b = new MetadataCache(new PgCache(db2)); + const la = await listenCacheInvalidation(a, { connectionString: url }); + const lb = await listenCacheInvalidation(b, { connectionString: url }); + try { + expect(a.l1Enabled && b.l1Enabled).toBe(true); + const p = { ttl: 600 }; + await a.fetch("inv", "x", p, async () => 1); + await a.fetch("inv", "y", p, async () => 2); + await b.fetch("inv", "x", p, async () => 0); + await b.fetch("inv", "y", p, async () => 0); + expect(b.l1Size("inv")).toBe(2); + // A 删 x:B 的 L1 只丢 x + await a.delete("inv", "x"); + await until(() => b.l1Size("inv") === 1); + expect(await b.fetch("inv", "x", p, async () => 10)).toBe(10); + // A 清空整个 region + await a.clear("inv"); + await until(() => b.l1Size("inv") === 0); + expect(await b.fetch("inv", "y", p, async () => 20)).toBe(20); + // 手动刷新(fresh)写入后也通知 + await a.fetch("inv", "y", { ...p, fresh: true }, async () => 21); + await until(() => b.l1Size("inv") === 0); + expect(await b.fetch("inv", "y", p, async () => 0)).toBe(21); + // 断开 B 的监听连接:L1 停用;自动重连后重新启用且是空的 + expect(b.l1Size("inv")).toBe(1); + await sql`select pg_terminate_backend(pid) from pg_stat_activity + where query ilike 'listen umt_cache_invalidate%' and pid <> pg_backend_pid()`.execute(db2); + await until(() => !b.l1Enabled || b.l1Size("inv") === 0); + expect(b.l1Size("inv")).toBe(0); + await until(() => b.l1Enabled && lb.connected(), 10_000); + expect(b.l1Size("inv")).toBe(0); + } finally { + await la.stop(); + await lb.stop(); + } + expect(a.l1Enabled).toBe(false); + }); + + it("过期在宽限期内:先返回旧值、后台只刷一次;源失败继续用旧值并记警告", async () => { + const warns: string[] = []; + const c = new MetadataCache(new PgCache(db), { l1: false, logger: { warn: (_o, m) => warns.push(m) } }); + let n = 0; + let gate: () => void = () => {}; + const slow = () => + new Promise((ok) => { + n++; + gate = () => ok(100 + n); + }); + await c.fetch("swr", "k", { ttl: 0.2, grace: 60 }, async () => 1); + await new Promise((r) => setTimeout(r, 300)); + const p = { ttl: 600 }; + const [x, y] = await Promise.all([c.fetch("swr", "k", p, slow), c.fetch("swr", "k", p, slow)]); + expect([x, y]).toEqual([1, 1]); + await until(() => n === 1); + gate(); + await until(async () => (await new PgCache(db).get("swr", "k")) === 101); + expect(n).toBe(1); + expect(c.pendingStats().get("swr")).toMatchObject({ staleHits: 2, refreshes: 1 }); + // 再次过期,源失败:返回旧值,不删旧值,记一次警告 + await c.fetch("swr", "k2", { ttl: 0.2, grace: 60 }, async () => "old"); + await new Promise((r) => setTimeout(r, 300)); + const failing = async (): Promise => { + throw new Error("HTTP 503"); + }; + expect(await c.fetch("swr", "k2", p, failing)).toBe("old"); + await until(() => warns.length === 1); + expect(warns[0]).toContain("继续用旧值"); + expect(await c.fetch("swr", "k2", p, async () => "new")).toBe("old"); + // 过了宽限期 = 硬过期,回源 + await c.fetch("swr", "k3", { ttl: 0.1, grace: 0.1 }, async () => "a"); + await new Promise((r) => setTimeout(r, 300)); + expect(await c.fetch("swr", "k3", p, async () => "b")).toBe("b"); + // PgCache.get 只认新鲜条目 + await c.fetch("swr", "k4", { ttl: 0.1, grace: 60 }, async () => "x"); + await new Promise((r) => setTimeout(r, 200)); + expect(await new PgCache(db).get("swr", "k4")).toBeNull(); + expect(await new PgCache(db).exists("swr", "k4")).toBe(true); + }); + + it("同请求合并:并发同 key 只回源一次;失败不缓存并计数", async () => { + const c = new MetadataCache(new PgCache(db), { l1: true }); + let n = 0; + const load = async () => { + n++; + await new Promise((r) => setTimeout(r, 50)); + return { ok: true }; + }; + const all = await Promise.all(Array.from({ length: 5 }, () => c.fetch("sf", "k", { ttl: 60 }, load))); + expect(n).toBe(1); + expect(all.every((v) => v.ok)).toBe(true); + let m = 0; + const fail = async () => { + m++; + throw new Error("ECONNRESET"); + }; + await expect(c.fetch("sf", "bad", { ttl: 60 }, fail)).rejects.toThrow("ECONNRESET"); + await expect(c.fetch("sf", "bad", { ttl: 60 }, fail)).rejects.toThrow("ECONNRESET"); + expect(m).toBe(2); + expect(await new PgCache(db).exists("sf", "bad")).toBe(false); + expect(c.pendingStats().get("sf")).toMatchObject({ failures: 2 }); + // skipIf / null 不缓存;空结果按 emptyTtl + await c.fetch("sf", "none", { ttl: 60 }, async () => null); + expect(await new PgCache(db).exists("sf", "none")).toBe(false); + await c.fetch("sf", "empty", { ttl: 3600, emptyTtl: 60, isEmpty: (v: number[]) => !v.length }, async () => []); + const t = await sql<{ s: number }>`select extract(epoch from fresh_until - now())::int as s + from kv_cache where region = 'sf' and key = 'empty'`.execute(db); + expect(t.rows[0]!.s).toBeLessThanOrEqual(60); + }); + + it("pinned:按条数上限淘汰时跳过;L2 命中时续期", async () => { + const pgc = new PgCache(db); + const c = new MetadataCache(pgc, { l1: false }); + const tick = () => new Promise((r) => setTimeout(r, 5)); + await c.fetch("pin", "watched", { ttl: 100, maxsize: 2, pin: async () => true }, async () => "w"); + for (const k of ["a", "b", "c", "d"]) { + await tick(); + await c.fetch("pin", k, { ttl: 100, maxsize: 2 }, async () => k); + } + expect((await pgc.items("pin")).map((i) => i.key).sort()).toEqual(["d", "watched"]); + // 续期:新鲜条目命中且 pin 为真 → 过期时间推到 now + ttl(保留宽限长度) + await sql`update kv_cache set fresh_until = now() + interval '10 seconds', expires_at = now() + interval '60 seconds' + where region = 'pin' and key = 'watched'`.execute(db); + expect(await c.fetch("pin", "watched", { ttl: 3600, pin: async () => true }, async () => "x")).toBe("w"); + const r = await sql<{ f: number; g: number }>`select extract(epoch from fresh_until - now())::int as f, + extract(epoch from expires_at - fresh_until)::int as g from kv_cache where region = 'pin' and key = 'watched'`.execute( + db, + ); + expect(r.rows[0]!.f).toBeGreaterThan(3500); + expect(r.rows[0]!.g).toBe(50); + }); +}); diff --git a/packages/db/test/schema.test.ts b/packages/db/test/schema.test.ts index 0b21163b..dae595e2 100644 --- a/packages/db/test/schema.test.ts +++ b/packages/db/test/schema.test.ts @@ -132,6 +132,8 @@ const EXPECTED_TABLES = [ "episode_mapping_rule", // 站点请求失败留样(0077) "site_failure_sample", + // 缓存命中统计(0086,按天 × region) + "cache_stat", ].sort(); /** 不走 entityTable() 的表:追加型/引擎自有表,没有 uuid id + created_at/updated_at 三件套 */ @@ -167,6 +169,8 @@ const NON_ENTITY_TABLES = [ "site_request_count", // 失败留样:只追加,只有 created_at "site_failure_sample", + // 缓存命中统计:(day, region) 主键 + "cache_stat", ]; describe.skipIf(skip)("schema migrations (real PG)", () => { diff --git a/packages/domain/src/scrape/layout.ts b/packages/domain/src/scrape/layout.ts index 1c51ccc5..0d10eb0b 100644 --- a/packages/domain/src/scrape/layout.ts +++ b/packages/domain/src/scrape/layout.ts @@ -7,6 +7,8 @@ // - 剧集:作品目录 tvshow.nfo、poster.jpg、fanart.jpg;季目录 season.nfo;季海报同时写作品目录 // `season01-poster.jpg`(特别篇 `season-specials-poster.jpg`)与季目录 `poster.jpg`(MP 同); // - 每集:`<视频名>.nfo`;剧照 `<视频名>-thumb.jpg`(Kodi / Jellyfin / Emby 都认;MP 写 `<视频名>.jpg`)。 +// - 标志(TMDB logos,MP 只在接 fanart.tv 时写):电影写在影片目录、剧集写在剧集根目录,文件名 `clearlogo.png` +// (Kodi / Emby / Jellyfin 通用);平铺 / 库根的电影用 `<视频名>-clearlogo.png`。 // 视频直接在库根(没有作品目录)或电影目录里还有别的作品时,电影图片改用 `<视频名>-poster.jpg` / `<视频名>-fanart.jpg`, // 剧集不写 tvshow / 季文件(不能往库根里放 tvshow.nfo)。 @@ -42,6 +44,7 @@ export type ScrapeTargetRole = | "episode_nfo" | "poster" | "fanart" + | "clearlogo" | "season_poster" | "episode_thumb"; @@ -61,6 +64,8 @@ export interface ScrapeVideo { export interface ScrapeImages { posterPath?: string | null; backdropPath?: string | null; + /** TMDB 标志(PNG) */ + logoPath?: string | null; /** 季号 → 季海报路径 */ seasonPosters?: ReadonlyMap; /** "季:集" → 剧照路径 */ @@ -165,7 +170,7 @@ export function planScrapeTargets(input: ScrapeLayoutInput): ScrapeTarget[] { } if (!input.options.images || input.parts?.work === false) return out; const inWork = workDir && !input.sharedDir && videos.some((v) => isUnder(v.path, workDir)); - const art = (role: "poster" | "fanart", p: string | null | undefined) => { + const art = (role: "poster" | "fanart" | "clearlogo", p: string | null | undefined) => { if (!p) return; if (inWork) add({ path: join(workDir, `${role}${extOf(p)}`), kind: "image", role, imagePath: p }); else @@ -182,6 +187,7 @@ export function planScrapeTargets(input: ScrapeLayoutInput): ScrapeTarget[] { }; art("poster", img.posterPath); art("fanart", img.backdropPath); + art("clearlogo", img.logoPath); return out; } @@ -206,6 +212,13 @@ export function planScrapeTargets(input: ScrapeLayoutInput): ScrapeTarget[] { role: "fanart", imagePath: img.backdropPath, }); + if (img.logoPath) + add({ + path: join(workDir, `clearlogo${extOf(img.logoPath)}`), + kind: "image", + role: "clearlogo", + imagePath: img.logoPath, + }); } } // 季:按视频所在目录(不等于作品目录的才算季目录);没有作品目录时只写季目录里的 season.nfo / poster diff --git a/packages/domain/test/scrape.test.ts b/packages/domain/test/scrape.test.ts index 417da9cb..7908cae9 100644 --- a/packages/domain/test/scrape.test.ts +++ b/packages/domain/test/scrape.test.ts @@ -199,6 +199,60 @@ describe("刮削目标布局", () => { expect(atRoot.map((x) => x.path)).toEqual(["x.nfo", "x-poster.jpg", "x-fanart.png"]); }); + it("标志 clearlogo.png:电影写影片目录、剧集写剧集根目录;库根 / 平铺的电影用 <视频名>-clearlogo.png;选中季 / 集不写", () => { + const withLogo = { ...images, logoPath: "/logo.png" }; + const movie = planScrapeTargets({ + kind: "movie", + workDir: "星际穿越 (2014)", + videos: [{ path: "星际穿越 (2014)/星际穿越 (2014) - 1080p.mkv" }], + images: withLogo, + options: { images: true, episodeThumbs: true }, + }); + expect(movie.filter((x) => x.role === "clearlogo").map((x) => [x.path, x.imagePath])).toEqual([ + ["星际穿越 (2014)/clearlogo.png", "/logo.png"], + ]); + const atRoot = planScrapeTargets({ + kind: "movie", + workDir: null, + videos: [{ path: "x.mkv" }], + images: withLogo, + options: { images: true, episodeThumbs: true }, + }); + expect(atRoot.map((x) => x.path)).toEqual(["x.nfo", "x-poster.jpg", "x-fanart.png", "x-clearlogo.png"]); + const tv = planScrapeTargets({ + kind: "tv", + workDir: "S", + videos: [{ path: "S/Season 1/a.mkv", season: 1, episode: 1 }], + images: withLogo, + options: { images: true, episodeThumbs: true }, + }); + expect(tv.slice(0, 4).map((x) => x.path)).toEqual([ + "S/tvshow.nfo", + "S/poster.jpg", + "S/fanart.png", + "S/clearlogo.png", + ]); + const seasonOnly = planScrapeTargets({ + kind: "tv", + workDir: "S", + videos: [{ path: "S/Season 1/a.mkv", season: 1, episode: 1 }], + images: withLogo, + options: { images: true, episodeThumbs: true }, + parts: { work: false }, + }); + expect(seasonOnly.some((x) => x.role === "clearlogo")).toBe(false); + // 关图片不写;没有标志不写 + const noImages = planScrapeTargets({ + kind: "tv", + workDir: "S", + videos: [{ path: "S/Season 1/a.mkv", season: 1, episode: 1 }], + images: withLogo, + options: { images: false, episodeThumbs: false }, + }); + expect(noImages.some((x) => x.role === "clearlogo")).toBe(false); + expect(movie.length).toBe(4); + }); + it("剧集视频在库根:不写 tvshow / 季文件,只写集 NFO", () => { const t = planScrapeTargets({ kind: "tv", diff --git a/packages/testkit/src/tmdb/fixtures.ts b/packages/testkit/src/tmdb/fixtures.ts index 515c0983..2939e56e 100644 --- a/packages/testkit/src/tmdb/fixtures.ts +++ b/packages/testkit/src/tmdb/fixtures.ts @@ -60,6 +60,8 @@ export interface TmdbFixture { genre_ids?: number[]; /** 剧集组(`/tv/{id}/episode_groups`、`/tv/episode_group/{id}`);组内集按 TMDB 默认季集号引用 seasons 里的集 */ episode_groups?: TmdbFixtureEpisodeGroup[]; + /** 标志(append_to_response=images 与 `/{type}/{id}/images` 按 include_image_language 过滤);缺省没有 */ + logos?: { file_path: string; iso_639_1: string | null; vote_average?: number; vote_count?: number }[]; } export interface TmdbFixtureEpisodeGroup { diff --git a/packages/testkit/src/tmdb/mock-tmdb.ts b/packages/testkit/src/tmdb/mock-tmdb.ts index f0fb16e6..12026530 100644 --- a/packages/testkit/src/tmdb/mock-tmdb.ts +++ b/packages/testkit/src/tmdb/mock-tmdb.ts @@ -22,7 +22,7 @@ const notFound: MockReply = { /** 列表接口一页的条数(同 TMDB) */ const LIST_PAGE = 20; -/** mock TMDB v3:/3/trending、/3/discover、电影 / 剧集榜单(按热度,一页 20 条)、/3/search/{movie,tv}、/3/find/:imdbId、/3/{movie,tv}/:id(append_to_response)、/3/tv/:id/season/:n、/3/tv/:id/episode_groups、/3/tv/episode_group/:id */ +/** mock TMDB v3:/3/trending、/3/discover、电影 / 剧集榜单(按热度,一页 20 条)、/3/search/{movie,tv}、/3/find/:imdbId、/3/{movie,tv}/:id(append_to_response)、/3/{movie,tv}/:id/images、/3/tv/:id/season/:n、/3/tv/:id/episode_groups、/3/tv/episode_group/:id */ export async function createMockTmdb(opts: MockTmdbOptions = {}): Promise { const fx = opts.fixtures ?? TMDB_FIXTURES; const apiKey = opts.apiKey ?? "test-tmdb-key"; @@ -59,6 +59,21 @@ export async function createMockTmdb(opts: MockTmdbOptions = {}): Promise) => { + const langs = q.include_image_language ? q.include_image_language.split(",") : null; + const logos = (f.logos ?? []) + .filter((l) => !langs || langs.includes(l.iso_639_1 ?? "null")) + .map((l) => ({ + ...l, + vote_average: l.vote_average ?? 5, + vote_count: l.vote_count ?? 1, + width: 500, + height: 200, + })); + return { backdrops: [], posters: [], logos }; + }; + const listItem = (f: TmdbFixture, lang?: string) => ({ ...brief(f, lang), genre_ids: f.genre_ids ?? [], @@ -222,6 +237,13 @@ export async function createMockTmdb(opts: MockTmdbOptions = {}): Promise { + const f = find(m[1]!, m[2]!); + return f ? { json: { id: f.id, ...images(f, q) } } : notFound; + }, + ], [ /^\/3\/tv\/(\d+)\/season\/(\d+)$/, (m, q) => { @@ -298,6 +320,7 @@ export async function createMockTmdb(opts: MockTmdbOptions = {}): Promise