merge: feat/y-meta-cache
Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei
This commit is contained in:
rootandClaude Opus 5.5 committed
commit
6d37a337c9
51 files changed
+2407
-200
No files matched your search
@@ -84,6 +84,7 @@ const { createApp } = await import("./http/app.ts");
|
||||
const { PgEventBus } = await import("./http/sse-pg.ts");
|
||||
const { createLogger } = await import("./logger.ts");
|
||||
const { SERVER_MODULES } = await import("./modules/index.ts");
|
||||
const { startMetadataCache } = await import("./shared-cache.ts");
|
||||
const { diffCounts, enableReadOnlyReconcile, exportMappings, writeMappingExport } = await import("./ops.ts");
|
||||
const { auditStorageWarnings, checkStorage, loadMountedPaths, runPreflight } = await import("./preflight.ts");
|
||||
const { startWorker } = await import("./worker/index.ts");
|
||||
@@ -219,6 +220,8 @@ try {
|
||||
Deno.exit(1);
|
||||
}
|
||||
const { ctx, outbox, engine } = booted;
|
||||
// 统一元数据缓存:LISTEN 跨进程失效(在线才启用进程内一级缓存)+ 命中统计定期落库
|
||||
stops.push(await startMetadataCache(ctx, config.DATABASE_URL));
|
||||
|
||||
if (role === "api" || role === "all") {
|
||||
const app = createApp({
|
||||
|
||||
@@ -152,11 +152,12 @@ export function episodeMetadataMissing(u: UnitForNfo | undefined): boolean {
|
||||
return !u || (!filled(u.metadata) && !str(u.title) && !u.air_date);
|
||||
}
|
||||
|
||||
/** 图片路径:作品海报 / 背景图,季海报,集剧照 */
|
||||
/** 图片路径:作品海报 / 背景图 / 标志,季海报,集剧照 */
|
||||
export function imagePathsOf(entityMeta: unknown) {
|
||||
return {
|
||||
posterPath: str(val(entityMeta, "posterPath")) ?? str(val(entityMeta, "posterUrl")),
|
||||
backdropPath: str(val(entityMeta, "backdropPath")),
|
||||
logoPath: str(val(entityMeta, "logoPath")),
|
||||
};
|
||||
}
|
||||
export const unitPosterPath = (m: unknown) => str(val(m, "posterPath"));
|
||||
|
||||
+40
-12
@@ -14,7 +14,16 @@ import { dataPaths } from "../../config.ts";
|
||||
import { externalFetch } from "../../egress.ts";
|
||||
import { requireScope } from "../../http/auth.ts";
|
||||
import { errorBody } from "../../http/errors.ts";
|
||||
import { CACHE_REGIONS, sharedCache, storageSettings, tmdbCachePolicy } from "../../shared-cache.ts";
|
||||
import {
|
||||
CACHE_REGIONS,
|
||||
METADATA_REGIONS,
|
||||
metadataCache,
|
||||
type RegionHitStats,
|
||||
regionHitStats,
|
||||
sharedCache,
|
||||
storageSettings,
|
||||
tmdbCachePolicy,
|
||||
} from "../../shared-cache.ts";
|
||||
import { registerScheduledJob } from "../schedules/registry.ts";
|
||||
import type { ModuleApp, ServerContext, ServerModule } from "../types.ts";
|
||||
import { ImageCache, ImageProxyError } from "./images.ts";
|
||||
@@ -72,6 +81,13 @@ function route<D extends EndpointDef>(
|
||||
|
||||
const str = (v: unknown) => (typeof v === "string" && v ? v : null);
|
||||
|
||||
/** 命中统计 → 接口形状;命中率 = (L1 + L2 + 旧值) / (L1 + L2 + 旧值 + 未命中) */
|
||||
function hitView(h: RegionHitStats | undefined) {
|
||||
const v = h ?? { l1Hits: 0, l2Hits: 0, staleHits: 0, misses: 0, failures: 0, refreshes: 0 };
|
||||
const hit = v.l1Hits + v.l2Hits + v.staleHits;
|
||||
return { ...v, hitRate: hit + v.misses > 0 ? hit / (hit + v.misses) : null };
|
||||
}
|
||||
|
||||
export function createCacheModule(opts: CacheModuleOptions = {}): ServerModule & {
|
||||
images(ctx: ServerContext): ImageCache;
|
||||
} {
|
||||
@@ -166,7 +182,7 @@ export function createCacheModule(opts: CacheModuleOptions = {}): ServerModule &
|
||||
return out;
|
||||
});
|
||||
route(app, Api.cache.tmdbClear, async () => {
|
||||
const cache = sharedCache(ctx);
|
||||
const cache = metadataCache(ctx);
|
||||
const deleted =
|
||||
(await cache.clear(CACHE_REGIONS.tmdbRecognize)) + (await cache.clear(CACHE_REGIONS.tmdbRequest));
|
||||
ctx.logger.info({ deleted }, "TMDB 缓存已清空");
|
||||
@@ -194,18 +210,30 @@ export function createCacheModule(opts: CacheModuleOptions = {}): ServerModule &
|
||||
coalesce(sum(pg_column_size(value) + pg_column_size(key)), 0)::text as bytes,
|
||||
max(updated_at) as last
|
||||
from kv_cache group by region order by region`.execute(ctx.db);
|
||||
return {
|
||||
items: r.rows.map((x) => ({
|
||||
region: x.region,
|
||||
entries: Number(x.entries),
|
||||
expired: Number(x.expired),
|
||||
bytes: Number(x.bytes),
|
||||
lastWriteAt: x.last ? new Date(x.last).toISOString() : null,
|
||||
})),
|
||||
// 统一元数据缓存的 region 带最近 7 天命中统计(口径见 contracts CacheHitStats)
|
||||
const hits = await regionHitStats(ctx).catch(() => new Map<string, RegionHitStats>());
|
||||
const withHits = (region: string) => {
|
||||
const h = hits.get(region);
|
||||
if (!h) return METADATA_REGIONS.includes(region) ? { hits: hitView(undefined) } : {};
|
||||
return { hits: hitView(h) };
|
||||
};
|
||||
const items = r.rows.map((x) => ({
|
||||
region: x.region,
|
||||
entries: Number(x.entries),
|
||||
expired: Number(x.expired),
|
||||
bytes: Number(x.bytes),
|
||||
lastWriteAt: x.last ? new Date(x.last).toISOString() : null,
|
||||
...withHits(x.region),
|
||||
}));
|
||||
// 条目已清空、但最近有读取统计的元数据 region 也列出来
|
||||
for (const region of hits.keys())
|
||||
if (!items.some((i) => i.region === region))
|
||||
items.push({ region, entries: 0, expired: 0, bytes: 0, lastWriteAt: null, ...withHits(region) });
|
||||
items.sort((a, b) => a.region.localeCompare(b.region));
|
||||
return { items };
|
||||
});
|
||||
route(app, Api.cache.regionClear, async ({ params }) => {
|
||||
const deleted = await sharedCache(ctx).clear(params.region);
|
||||
const deleted = await metadataCache(ctx).clear(params.region);
|
||||
ctx.logger.info({ region: params.region, deleted }, "缓存区已清空");
|
||||
return { deleted };
|
||||
});
|
||||
@@ -217,7 +245,7 @@ export function createCacheModule(opts: CacheModuleOptions = {}): ServerModule &
|
||||
lane: "maintenance",
|
||||
defaults: {
|
||||
name: "缓存清理",
|
||||
description: "删除过期的 TMDB 与识别缓存",
|
||||
description: "删除过期的元数据与其它缓存",
|
||||
group: "maintain",
|
||||
schedule: { type: "interval", everyMs: 10 * 60_000 },
|
||||
firstRun: "after_interval",
|
||||
|
||||
@@ -24,10 +24,10 @@ import {
|
||||
type DiscoverTmdbQuery,
|
||||
type InventoryEntry,
|
||||
} from "@umt/contracts";
|
||||
import { pgCacheFor } from "@umt/db";
|
||||
import { type IdentityCandidate, parseRelease, resolveIdentity } from "@umt/domain";
|
||||
import { ApiException } from "../../http/errors.ts";
|
||||
import { runSandbox } from "../../sandbox/client.ts";
|
||||
import { CACHE_REGIONS, metaCachePolicy, metadataCache, storageSettings } from "../../shared-cache.ts";
|
||||
import type { SourcesRuntime } from "../sources-identity/runtime.ts";
|
||||
import { subjectRecall } from "../sources-identity/service/id-escalation.ts";
|
||||
import { metaClients, toCached } from "../sources-identity/service/meta-subjects.ts";
|
||||
@@ -41,9 +41,8 @@ import { localEntities } from "./media.ts";
|
||||
// - 每条结果按 TMDB 身份标本地状态(已订阅 / 媒体库已有 / 下载中);豆瓣 / bgm 条目匹配过 TMDB 的才知道(discover.match)。
|
||||
// 本地状态每次现查,不进缓存。
|
||||
|
||||
/** 豆瓣 / bgm 条目 → TMDB 作品的确定匹配(点开时写入,列表据此标本地状态) */
|
||||
const MATCH_REGION = "discover.match";
|
||||
const MATCH_TTL_SEC = 30 * 86_400;
|
||||
/** 豆瓣 / bgm 条目 → TMDB 作品的确定匹配(点开时写入,列表据此标本地状态);TTL 按「豆瓣 / bgm × 详情」 */
|
||||
const MATCH_REGION = CACHE_REGIONS.discoverMatch;
|
||||
const MATCH_MAX = 20_000;
|
||||
|
||||
export type InventoryLookup = (ids: string[]) => Promise<Map<string, InventoryEntry>>;
|
||||
@@ -108,12 +107,12 @@ const fromTmdb = (t: TmdbListItem): DiscoverItem => ({
|
||||
|
||||
/** 豆瓣 / bgm 条目补上记住的 TMDB 匹配,再按 TMDB 身份查本地作品与库存 */
|
||||
export async function annotate(rt: SourcesRuntime, items: DiscoverItem[], inventory: InventoryLookup) {
|
||||
const cache = pgCacheFor(rt.ctx.db);
|
||||
const cache = metadataCache(rt.ctx);
|
||||
await Promise.all(
|
||||
items
|
||||
.filter((it) => !it.tmdb && it.source !== "tmdb")
|
||||
.map(async (it) => {
|
||||
const hit = await cache.get<DiscoverMatchCandidate>(MATCH_REGION, `${it.source}:${it.id}`).catch(() => null);
|
||||
const hit = await cache.get<DiscoverMatchCandidate>(MATCH_REGION, `${it.source}:${it.id}`);
|
||||
if (hit) it.tmdb = { type: hit.type, id: hit.id };
|
||||
}),
|
||||
);
|
||||
@@ -257,9 +256,9 @@ const view = (m: MetadataCandidate): DiscoverMatchCandidate => ({
|
||||
});
|
||||
|
||||
export async function matchTmdb(rt: SourcesRuntime, req: DiscoverMatchRequest): Promise<DiscoverMatchResult> {
|
||||
const cache = pgCacheFor(rt.ctx.db);
|
||||
const cache = metadataCache(rt.ctx);
|
||||
const memoKey = `${req.source}:${req.id}`;
|
||||
const memo = await cache.get<DiscoverMatchCandidate>(MATCH_REGION, memoKey).catch(() => null);
|
||||
const memo = await cache.get<DiscoverMatchCandidate>(MATCH_REGION, memoKey);
|
||||
if (memo) return { match: memo, candidates: [memo] };
|
||||
const subject = toCached({ ...req, cover: null, rating: null, date: null, episodes: null, url: "" }, rt.now());
|
||||
const base = parseRelease(req.title);
|
||||
@@ -278,6 +277,9 @@ export async function matchTmdb(rt: SourcesRuntime, req: DiscoverMatchRequest):
|
||||
const best = res.status === "auto" && res.best ? byCand.get(res.best.candidate) : undefined;
|
||||
const match = best ? view(best) : null;
|
||||
// 只记确定的匹配;不确定 / 失败不记,下次重算
|
||||
if (match) await cache.set(MATCH_REGION, memoKey, match, { ttl: MATCH_TTL_SEC, maxsize: MATCH_MAX }).catch(() => {});
|
||||
if (match) {
|
||||
const ttl = metaCachePolicy((await storageSettings(rt.ctx)).cacheTtl, req.source, "detail").ttl;
|
||||
await cache.set(MATCH_REGION, memoKey, match, { ttl, maxsize: MATCH_MAX }).catch(() => {});
|
||||
}
|
||||
return DiscoverMatchResult.parse({ match, candidates: ranked.map(view) });
|
||||
}
|
||||
@@ -11,19 +11,13 @@ import {
|
||||
type SiteDefinition,
|
||||
type SiteRateLimiter,
|
||||
type TmdbClient,
|
||||
type TmdbRequestKind,
|
||||
type TmdbSharedCache,
|
||||
} from "@umt/adapters";
|
||||
import { DEFAULT_SETTINGS, type SitesSettings } from "@umt/contracts";
|
||||
import { pgTokenBucketFor, type SiteRow, sql } from "@umt/db";
|
||||
import { egressKey, externalFetch, siteEgress, siteFetchFor } from "../../egress.ts";
|
||||
import {
|
||||
CACHE_REGIONS,
|
||||
renewTrackedTmdbDetail,
|
||||
sharedCache,
|
||||
type TmdbCachePolicy,
|
||||
tmdbCachePolicy,
|
||||
tmdbDetailRef,
|
||||
} from "../../shared-cache.ts";
|
||||
import { CACHE_REGIONS, metadataCache, type TmdbCachePolicy, tmdbCachePolicy, watchedPin } from "../../shared-cache.ts";
|
||||
import type { ServerContext } from "../types.ts";
|
||||
import { CloudflareGate, withClearance } from "./service/cloudflare.ts";
|
||||
import { failureSampleSink } from "./service/failure-sample.ts";
|
||||
@@ -165,6 +159,8 @@ export class SourcesRuntime {
|
||||
ttlMs: policy.ttl * 1000,
|
||||
searchTtlMs: policy.searchTtl * 1000,
|
||||
emptyTtlMs: policy.emptyTtl * 1000,
|
||||
listTtlMs: policy.listTtl * 1000,
|
||||
imageTtlMs: policy.imageTtl * 1000,
|
||||
},
|
||||
sharedCache: tmdbSharedCache(this.ctx, policy),
|
||||
}
|
||||
@@ -420,28 +416,34 @@ export class SourcesRuntime {
|
||||
}
|
||||
|
||||
/**
|
||||
* TMDB 请求响应缓存:PG kv_cache region=tmdb.request;PG 出错按未命中 / 不写处理,不让 TMDB 请求失败。
|
||||
* 详情命中且属于追踪中的作品时续期(CACHE-2),续期失败只记 debug。
|
||||
* TMDB 请求响应缓存:统一元数据缓存 region=tmdb.request(L1 进程内存 + L2 kv_cache,过期先用旧值后台刷新、同请求合并)。
|
||||
* TTL 按请求类别取策略表(搜索 / 详情 / 列表 / 图片);关注中作品的条目写入 pinned、命中续期。
|
||||
* PG 出错按未命中 / 不写处理,不让 TMDB 请求失败。get / set 保留给不走 fetch 的旧调用方。
|
||||
*/
|
||||
function tmdbSharedCache(ctx: ServerContext, policy: TmdbCachePolicy): TmdbSharedCache {
|
||||
const cache = sharedCache(ctx);
|
||||
const warn = (err: unknown) =>
|
||||
ctx.logger.warn({ err: { message: (err as Error).message } }, "TMDB 缓存读写失败，按未命中处理");
|
||||
export function tmdbSharedCache(ctx: ServerContext, policy: TmdbCachePolicy): TmdbSharedCache {
|
||||
const cache = metadataCache(ctx);
|
||||
const region = CACHE_REGIONS.tmdbRequest;
|
||||
const ttlOf = (kind: TmdbRequestKind) =>
|
||||
kind === "search"
|
||||
? { ttl: policy.searchTtl, emptyTtl: policy.emptyTtl }
|
||||
: kind === "list"
|
||||
? { ttl: policy.listTtl, emptyTtl: policy.listEmptyTtl }
|
||||
: kind === "image"
|
||||
? { ttl: policy.imageTtl, emptyTtl: policy.emptyTtl }
|
||||
: { ttl: policy.ttl, emptyTtl: policy.emptyTtl };
|
||||
return {
|
||||
get: async (key) => {
|
||||
const v = await cache.get(CACHE_REGIONS.tmdbRequest, key).catch((e) => {
|
||||
warn(e);
|
||||
return null;
|
||||
});
|
||||
if (v !== null && tmdbDetailRef(key)) {
|
||||
await renewTrackedTmdbDetail(ctx, key, policy.ttl).catch((e) =>
|
||||
ctx.logger.debug({ err: { message: (e as Error).message } }, "TMDB 详情缓存续期失败"),
|
||||
);
|
||||
}
|
||||
return v;
|
||||
},
|
||||
fetch: (req, load) =>
|
||||
cache.fetch(
|
||||
region,
|
||||
req.key,
|
||||
{ ...ttlOf(req.kind), isEmpty: req.isEmpty, maxsize: policy.maxsize, pin: watchedPin(ctx, req.key) },
|
||||
load,
|
||||
),
|
||||
get: (key) => cache.get(region, key),
|
||||
set: (key, value, ttlMs) =>
|
||||
cache.set(CACHE_REGIONS.tmdbRequest, key, value, { ttl: ttlMs / 1000, maxsize: policy.maxsize }).catch(warn),
|
||||
cache
|
||||
.set(region, key, value, { ttl: ttlMs / 1000, maxsize: policy.maxsize })
|
||||
.catch((e) => ctx.logger.warn({ err: { message: (e as Error).message } }, "TMDB 缓存读写失败，按未命中处理")),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { formatDetailPage } from "@umt/adapters";
|
||||
import { pgCacheFor } from "@umt/db";
|
||||
import { DEFAULT_STORAGE_SETTINGS } from "@umt/contracts";
|
||||
import { metaCachePolicy, metadataCache, storageSettings } from "../../../shared-cache.ts";
|
||||
import type { ServerContext } from "../../types.ts";
|
||||
|
||||
// ID-3 豆瓣条目缓存(kv_cache region douban.subject,key = 豆瓣 subject id):只存标题 / 别名 / 年份,供「详情页只有豆瓣 ID」时
|
||||
@@ -10,8 +11,8 @@ import type { ServerContext } from "../../types.ts";
|
||||
// - douban_api:豆瓣移动版接口(识别升级时缓存没有才查)。bgm.tv 条目同形,存 region bangumi.subject。
|
||||
|
||||
export const DOUBAN_SUBJECT_REGION = "douban.subject";
|
||||
const TTL_SEC = 30 * 86_400;
|
||||
const MAX_ENTRIES = 20_000;
|
||||
/** 条目 TTL 取策略表「豆瓣 / bgm × 详情」(storage.cacheTtl.metaDetailDays,缺省 30 天) */
|
||||
export const META_SUBJECT_MAX_ENTRIES = 20_000;
|
||||
|
||||
export interface DoubanSubject {
|
||||
id: string;
|
||||
@@ -92,16 +93,24 @@ export function doubanSubjectFromSnippet(
|
||||
};
|
||||
}
|
||||
|
||||
type Db = Pick<ServerContext, "db">;
|
||||
type Db = Pick<ServerContext, "db"> & Partial<Pick<ServerContext, "settings" | "logger">>;
|
||||
|
||||
/** 写入缓存;站点详情来源优先(已有 site_detail 条目时不用搜索摘要覆盖) */
|
||||
/** 条目 TTL(秒):有设置读设置,否则缺省 */
|
||||
export async function metaSubjectTtl(ctx: Db): Promise<number> {
|
||||
const t = ctx.settings
|
||||
? (await storageSettings({ settings: ctx.settings })).cacheTtl
|
||||
: DEFAULT_STORAGE_SETTINGS.cacheTtl;
|
||||
return metaCachePolicy(t, "douban", "detail").ttl;
|
||||
}
|
||||
|
||||
/** 写入缓存(统一元数据缓存,两级);站点详情来源优先(已有 site_detail 条目时不用搜索摘要覆盖) */
|
||||
export async function saveDoubanSubject(ctx: Db, s: DoubanSubject, region = DOUBAN_SUBJECT_REGION): Promise<void> {
|
||||
const cache = pgCacheFor(ctx.db);
|
||||
const cache = metadataCache(ctx);
|
||||
if (s.source === "web_snippet") {
|
||||
const prev = await cache.get<DoubanSubject>(region, s.id);
|
||||
if (prev?.source === "site_detail") return;
|
||||
}
|
||||
await cache.set(region, s.id, s, { ttl: TTL_SEC, maxsize: MAX_ENTRIES });
|
||||
await cache.set(region, s.id, s, { ttl: await metaSubjectTtl(ctx), maxsize: META_SUBJECT_MAX_ENTRIES });
|
||||
}
|
||||
|
||||
export async function loadDoubanSubject(
|
||||
@@ -110,7 +119,7 @@ export async function loadDoubanSubject(
|
||||
region = DOUBAN_SUBJECT_REGION,
|
||||
): Promise<DoubanSubject | null> {
|
||||
try {
|
||||
const v = await pgCacheFor(ctx.db).get<DoubanSubject>(region, id);
|
||||
const v = await metadataCache(ctx).get<DoubanSubject>(region, id);
|
||||
return v && typeof v.title === "string" && v.title ? v : null;
|
||||
} catch {
|
||||
return null;
|
||||
|
||||
@@ -4,51 +4,56 @@ import {
|
||||
createDoubanClient,
|
||||
type DoubanClient,
|
||||
type MetaRequestCache,
|
||||
MetaSourceError,
|
||||
type MetaSubject,
|
||||
RequestLimiter,
|
||||
} from "@umt/adapters";
|
||||
import { cachedCall, pgCacheFor } from "@umt/db";
|
||||
import { normalizeTitle, toSimplified } from "@umt/domain";
|
||||
import { storageSettings } from "../../../shared-cache.ts";
|
||||
import { CACHE_REGIONS, metaCachePolicy, metadataCache, storageSettings } from "../../../shared-cache.ts";
|
||||
import type { SourcesRuntime } from "../runtime.ts";
|
||||
import { DOUBAN_SUBJECT_REGION, type DoubanSubject, loadDoubanSubject, saveDoubanSubject } from "./douban-subject.ts";
|
||||
import {
|
||||
DOUBAN_SUBJECT_REGION,
|
||||
type DoubanSubject,
|
||||
loadDoubanSubject,
|
||||
META_SUBJECT_MAX_ENTRIES,
|
||||
} from "./douban-subject.ts";
|
||||
|
||||
// 豆瓣 / bgm.tv 作为辅助信息源(对照 MP:豆瓣、Bangumi 条目 → 名称 + 年份 → TMDB 投影;身份仍以 TMDB 为准)。
|
||||
// 条目先查缓存(30 天),没有才在线查一次;查不到 / 网络失败记 6 小时负缓存,不反复打外站。
|
||||
|
||||
export const BANGUMI_SUBJECT_REGION = "bangumi.subject";
|
||||
const MISS_REGION = "meta.subject_miss";
|
||||
const MISS_TTL_SEC = 6 * 3600;
|
||||
export const BANGUMI_SUBJECT_REGION = CACHE_REGIONS.bangumiSubject;
|
||||
const MISS_REGION = CACHE_REGIONS.subjectMiss;
|
||||
/** 网络失败 / 被拦后同一条目在本进程里冷却多久不再在线查(不写缓存,只是不连着打外站) */
|
||||
const FAIL_COOLDOWN_MS = 30 * 60_000;
|
||||
const failedAt = new Map<string, number>();
|
||||
|
||||
export type MetaSource = "douban" | "bangumi";
|
||||
const REGION: Record<MetaSource, string> = { douban: DOUBAN_SUBJECT_REGION, bangumi: BANGUMI_SUBJECT_REGION };
|
||||
|
||||
/** 豆瓣 / bgm 请求缓存 region(对照 MP DoubanApi.__invoke / BangumiApi 的 @cached);key 见 metaRequestKey */
|
||||
export const META_REQUEST_REGIONS = { douban: "douban.request", bangumi: "bangumi.request" } as const;
|
||||
export const META_REQUEST_REGIONS = {
|
||||
douban: CACHE_REGIONS.doubanRequest,
|
||||
bangumi: CACHE_REGIONS.bangumiRequest,
|
||||
} as const;
|
||||
const META_REQUEST_MAXSIZE = 5000;
|
||||
/** 每源限流(进程内):豆瓣风控严,间隔大一些;缓存命中不占额度 */
|
||||
const META_LIMITERS = { douban: new RequestLimiter(2, 300), bangumi: new RequestLimiter(3, 100) };
|
||||
|
||||
/**
|
||||
* 请求缓存:搜索按 storage.cacheTtl.metaSearchDays,列表(榜单 / 热门 / 每日放送 / 趋势页)按 metaListHours,
|
||||
* 空结果按 metaEmptyHours(不超过正常 TTL);失败(网络、403、风控页)在适配器里抛出,cachedCall 不写。
|
||||
* 条目详情不走这里:douban.subject / bangumi.subject 条目级缓存(30 天)已经存了,避免同一份数据存两遍。
|
||||
* 请求缓存(统一元数据缓存):按策略表「来源 × 资源类型」取 TTL——搜索 metaSearchDays / 查无 metaMissDays,
|
||||
* 列表(榜单 / 热门 / 每日放送 / 趋势页)metaListHours / 空 metaEmptyHours;过期先用旧值后台刷新。
|
||||
* 失败(网络、403、风控页)在适配器里抛出,不写。
|
||||
* 条目详情不走这里:douban.subject / bangumi.subject 条目级缓存已经存了,避免同一份数据存两遍。
|
||||
*/
|
||||
export function metaRequestCache(ctx: SourcesRuntime["ctx"]): MetaRequestCache {
|
||||
return {
|
||||
async cached(req, load) {
|
||||
const t = (await storageSettings(ctx)).cacheTtl;
|
||||
const ttl = req.kind === "search" ? t.metaSearchDays * 86_400 : t.metaListHours * 3600;
|
||||
return cachedCall(
|
||||
pgCacheFor(ctx.db),
|
||||
const p = metaCachePolicy(t, req.source, req.kind);
|
||||
return metadataCache(ctx).fetch(
|
||||
META_REQUEST_REGIONS[req.source],
|
||||
req.key,
|
||||
{
|
||||
region: META_REQUEST_REGIONS[req.source],
|
||||
ttl,
|
||||
emptyTtl: Math.min(ttl, t.metaEmptyHours * 3600),
|
||||
emptyIf: req.empty,
|
||||
maxsize: META_REQUEST_MAXSIZE,
|
||||
},
|
||||
{ ...p, isEmpty: req.empty, maxsize: META_REQUEST_MAXSIZE },
|
||||
load,
|
||||
);
|
||||
},
|
||||
@@ -77,23 +82,38 @@ export function toCached(s: MetaSubject, now: Date): DoubanSubject {
|
||||
};
|
||||
}
|
||||
|
||||
/** 条目(缓存 → 在线);拿不到为 null */
|
||||
/**
|
||||
* 条目(缓存 → 在线,统一元数据缓存:过期先用旧值后台刷新);拿不到为 null。
|
||||
* 条目不存在记查无(metaMissDays);网络失败 / 被拦不缓存,同一条目本进程冷却 30 分钟再查。
|
||||
*/
|
||||
export async function loadSubject(rt: SourcesRuntime, source: MetaSource, id: string): Promise<DoubanSubject | null> {
|
||||
const hit = await loadDoubanSubject(rt.ctx, id, REGION[source]);
|
||||
if (hit) return hit;
|
||||
const clients = metaClients(rt);
|
||||
if (!clients) return null;
|
||||
const cache = pgCacheFor(rt.ctx.db);
|
||||
if (!clients) return loadDoubanSubject(rt.ctx, id, REGION[source]);
|
||||
const cache = metadataCache(rt.ctx);
|
||||
const missKey = `${source}:${id}`;
|
||||
if (await cache.get(MISS_REGION, missKey).catch(() => null)) return null;
|
||||
const t = (await storageSettings(rt.ctx)).cacheTtl;
|
||||
const p = metaCachePolicy(t, source, "detail");
|
||||
try {
|
||||
const s = source === "douban" ? await clients.douban.subject(id) : await clients.bangumi.subject(id);
|
||||
const cached = toCached(s, rt.now());
|
||||
await saveDoubanSubject(rt.ctx, cached, REGION[source]);
|
||||
return cached;
|
||||
const v = await cache.fetch<DoubanSubject | null>(
|
||||
REGION[source],
|
||||
id,
|
||||
{ ttl: p.ttl, maxsize: META_SUBJECT_MAX_ENTRIES, skipIf: (x) => !x?.title },
|
||||
async () => {
|
||||
if ((failedAt.get(missKey) ?? 0) > Date.now() - FAIL_COOLDOWN_MS) return null;
|
||||
if (await cache.get(MISS_REGION, missKey)) return null;
|
||||
const s = source === "douban" ? await clients.douban.subject(id) : await clients.bangumi.subject(id);
|
||||
return toCached(s, rt.now());
|
||||
},
|
||||
);
|
||||
return v && typeof v.title === "string" && v.title ? v : null;
|
||||
} catch (e) {
|
||||
rt.ctx.logger.info({ source, id, err: (e as Error).message }, "辅助信息源条目查询失败");
|
||||
await cache.set(MISS_REGION, missKey, true, { ttl: MISS_TTL_SEC, maxsize: 5000 }).catch(() => {});
|
||||
if (e instanceof MetaSourceError && e.kind === "not_found")
|
||||
await cache.set(MISS_REGION, missKey, true, { ttl: p.emptyTtl, maxsize: 5000 }).catch(() => {});
|
||||
else {
|
||||
if (failedAt.size > 5000) failedAt.clear();
|
||||
failedAt.set(missKey, Date.now());
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
import { join } from "node:path";
|
||||
import { fetchEpisodeGroup, fetchRichDetail, fetchRichSeason, TmdbError } from "@umt/adapters";
|
||||
import { dataPaths } from "../../../config.ts";
|
||||
import { externalFetch } from "../../../egress.ts";
|
||||
import { resetTrackedTmdbRefs } from "../../../shared-cache.ts";
|
||||
import { ImageCache } from "../../cache/images.ts";
|
||||
import type { SourcesRuntime } from "../runtime.ts";
|
||||
import { entityEpisodeGroup } from "./episode-structure.ts";
|
||||
import { metadataIncludeSpecials, metadataLanguages, metadataTextOptions } from "./metadata-rich.ts";
|
||||
|
||||
// 关注作品预热(统一元数据缓存,docs/runbook/cache.md):作品进入关注时(建订阅、用户搜索 / 手动识别、开始下载)
|
||||
// 把之后刷新 / 刮削会用到的 TMDB 条目先拉进缓存:详情(含标志)、各季详情、选中的剧集组,海报 / 背景 / 标志图片进图片缓存。
|
||||
// 请求键与刷新元数据 / 刮削完全相同(同一套 fetchRichDetail / fetchRichSeason / fetchEpisodeGroup,同样的语言链),
|
||||
// 所以后面直接命中;关注中作品的条目写入时 pinned(条数上限淘汰不清掉),命中时续期。
|
||||
|
||||
/** 一部作品最多预热多少季(长寿剧) */
|
||||
const MAX_SEASONS = 60;
|
||||
|
||||
export interface PrewarmResult {
|
||||
status: "warmed" | "skipped";
|
||||
reason?: string;
|
||||
seasons: number;
|
||||
episodeGroup: boolean;
|
||||
images: number;
|
||||
errors: string[];
|
||||
}
|
||||
|
||||
/** 图片预热用的 fetch:测试注入优先;测试进程没注入就不出外网 */
|
||||
function imageFetch(rt: SourcesRuntime): typeof globalThis.fetch | null {
|
||||
if (rt.overrides.fetch) return rt.overrides.fetch;
|
||||
return Deno.env.get("VITEST") ? null : externalFetch;
|
||||
}
|
||||
|
||||
export async function prewarmMetadata(rt: SourcesRuntime, mediaEntityId: string): Promise<PrewarmResult> {
|
||||
const { ctx } = rt;
|
||||
const out: PrewarmResult = { status: "skipped", seasons: 0, episodeGroup: false, images: 0, errors: [] };
|
||||
const ident = await ctx.db
|
||||
.selectFrom("external_identity as x")
|
||||
.innerJoin("media_entity as e", "e.id", "x.media_entity_id")
|
||||
.select(["x.entity_type", "x.external_id", "e.status"])
|
||||
.where("x.media_entity_id", "=", mediaEntityId)
|
||||
.where("x.provider", "=", "tmdb")
|
||||
.where("x.status", "=", "accepted")
|
||||
.executeTakeFirst();
|
||||
if (!ident || ident.status === "merged") return { ...out, reason: ident ? "merged" : "no_tmdb_identity" };
|
||||
if (ident.entity_type !== "movie" && ident.entity_type !== "tv") return { ...out, reason: "unsupported_type" };
|
||||
const tmdb = await rt.tmdb();
|
||||
if (!tmdb) return { ...out, reason: "tmdb_not_configured" };
|
||||
// 刚进入关注:关注集合按进程缓存 5 分钟,清掉让这次写入就标 pinned
|
||||
resetTrackedTmdbRefs(ctx);
|
||||
const type = ident.entity_type;
|
||||
const id = ident.external_id;
|
||||
const langs = await metadataLanguages(ctx);
|
||||
const textOpts = await metadataTextOptions(ctx);
|
||||
const note = (what: string, e: unknown) => out.errors.push(`${what}: ${(e as Error).message}`);
|
||||
|
||||
// 刷新元数据的基础详情与富详情(含标志)
|
||||
await tmdb.details(type, id).catch((e) => note("details", e));
|
||||
// 注入的简化客户端(测试)没有 getJson:只预热基础详情
|
||||
if (typeof tmdb.getJson !== "function") return { ...out, status: "warmed" };
|
||||
let rich: Awaited<ReturnType<typeof fetchRichDetail>> | null = null;
|
||||
try {
|
||||
rich = await fetchRichDetail(tmdb, type, id, langs, textOpts);
|
||||
} catch (e) {
|
||||
// 详情都拿不到(404 / 鉴权 / 网络):网络类交给任务重试,其余记下结束
|
||||
if (e instanceof TmdbError && ["network", "timeout", "rate"].includes(e.kind)) throw e;
|
||||
note("detail", e);
|
||||
return { ...out, reason: "detail_unavailable" };
|
||||
}
|
||||
out.status = "warmed";
|
||||
|
||||
if (type === "tv") {
|
||||
const specials = await metadataIncludeSpecials(ctx);
|
||||
const seasons = rich.seasons
|
||||
.map((s) => s.seasonNumber)
|
||||
.filter((n) => specials || n > 0)
|
||||
.slice(0, MAX_SEASONS);
|
||||
for (const n of seasons) {
|
||||
try {
|
||||
await fetchRichSeason(tmdb, id, n, langs, {
|
||||
now: rt.now(),
|
||||
...textOpts,
|
||||
originalLanguage: rich.originalLanguage,
|
||||
});
|
||||
out.seasons++;
|
||||
} catch (e) {
|
||||
note(`season ${n}`, e);
|
||||
}
|
||||
}
|
||||
const group = await entityEpisodeGroup(ctx.db, mediaEntityId);
|
||||
if (group) {
|
||||
try {
|
||||
await fetchEpisodeGroup(tmdb, group.id, langs[0]);
|
||||
out.episodeGroup = true;
|
||||
} catch (e) {
|
||||
note("episode group", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 图片:刮削取原图(original),这里预先拉进图片缓存目录
|
||||
const f = imageFetch(rt);
|
||||
if (f) {
|
||||
const images = new ImageCache({
|
||||
dir: join(dataPaths(ctx.config).dataDir, "cache", "images"),
|
||||
fetch: f,
|
||||
timeoutMs: 30_000,
|
||||
});
|
||||
for (const p of [rich.posterPath, rich.backdropPath, rich.logoPath]) {
|
||||
if (!p) continue;
|
||||
try {
|
||||
await images.bytes("original", p);
|
||||
out.images++;
|
||||
} catch (e) {
|
||||
note(`image ${p}`, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (out.errors.length)
|
||||
ctx.logger.debug({ entityId: mediaEntityId, errors: out.errors }, "关注作品预热有部分条目没拉到");
|
||||
return out;
|
||||
}
|
||||
@@ -72,6 +72,7 @@ export function richEntityFields(r: TmdbRichDetail): Partial<Record<EntityMetada
|
||||
originCountry: v(r.originCountry),
|
||||
...(r.posterPath ? { posterPath: v(r.posterPath) } : {}),
|
||||
...(r.backdropPath ? { backdropPath: v(r.backdropPath) } : {}),
|
||||
...(r.logoPath ? { logoPath: v(r.logoPath) } : {}),
|
||||
...(r.genres.length ? { genres: v(r.genres) } : {}),
|
||||
...(r.networks.length ? { platform: v(r.networks.join(" / ")) } : {}),
|
||||
...(r.entityType === "tv" && r.episodeCount !== null ? { episodeCount: v(r.episodeCount) } : {}),
|
||||
|
||||
@@ -140,6 +140,7 @@ const ENTITY_NFO_FIELDS: ReadonlySet<string> = new Set([
|
||||
"externalIds",
|
||||
"posterPath",
|
||||
"backdropPath",
|
||||
"logoPath",
|
||||
]);
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { MetadataCandidate } from "@umt/adapters";
|
||||
import type { ParsedRelease } from "@umt/domain";
|
||||
import { CACHE_REGIONS, sharedCache, tmdbCachePolicy } from "../../../shared-cache.ts";
|
||||
import { CACHE_REGIONS, metadataCache, tmdbCachePolicy } from "../../../shared-cache.ts";
|
||||
import type { SourcesRuntime } from "../runtime.ts";
|
||||
|
||||
// TMDB 识别缓存:移植 MP app/modules/themoviedb/cache.py 的 TmdbCache(对照表见 docs/porting/moviepilot-tmdb-cache.md)。
|
||||
@@ -92,7 +92,7 @@ export async function loadRecognizeCache(
|
||||
): Promise<RecognizeCacheEntry | null> {
|
||||
const tmdb = await rt.tmdb();
|
||||
const key = recognizeKey(meta, localeOf(tmdb));
|
||||
const cache = sharedCache(rt.ctx);
|
||||
const cache = metadataCache(rt.ctx);
|
||||
const v = await cache.get<RecognizeCacheEntry>(CACHE_REGIONS.tmdbRecognize, key).catch(() => null);
|
||||
if (!v || typeof v !== "object") return null;
|
||||
if (typeConflicted(meta, v.type, v.id)) {
|
||||
@@ -120,7 +120,7 @@ export async function saveRecognizeCache(
|
||||
const tmdb = await rt.tmdb();
|
||||
const key = recognizeKey(meta, localeOf(tmdb));
|
||||
const policy = await tmdbCachePolicy(rt.ctx);
|
||||
const cache = sharedCache(rt.ctx);
|
||||
const cache = metadataCache(rt.ctx);
|
||||
if (info) {
|
||||
if (typeConflicted(meta, info.entityType, info.externalId)) {
|
||||
rt.ctx.logger.warn({ key, title: info.title, type: info.entityType }, "识别结果类型与元数据冲突，不写入识别缓存");
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { sql } from "@umt/db";
|
||||
import { type RawBuilder, sql } from "@umt/db";
|
||||
import type { ServerContext } from "../../types.ts";
|
||||
import type { Exec } from "./common.ts";
|
||||
|
||||
@@ -17,9 +17,8 @@ import type { Exec } from "./common.ts";
|
||||
const OPEN_SUBSCRIPTION = ["active", "pending"];
|
||||
const DOWNLOAD_DONE = ["imported", "removed"];
|
||||
|
||||
export async function isEntityWatched(exec: Exec, entityId: string): Promise<boolean> {
|
||||
const r = await sql<{ watched: boolean }>`
|
||||
select (
|
||||
/** 作品 e(SQL 片段:作品 id 表达式)是否关注中;isEntityWatched 与 watchedTmdbRefs 共用一个口径 */
|
||||
const watchedExpr = (entityId: RawBuilder<unknown>) => sql<boolean>`(
|
||||
exists (select 1 from subscription s where s.media_entity_id = ${entityId} and s.status = any(${OPEN_SUBSCRIPTION}))
|
||||
or exists (select 1 from media_asset a where a.media_entity_id = ${entityId} and a.status = 'active')
|
||||
or exists (select 1 from external_identity x
|
||||
@@ -27,7 +26,7 @@ export async function isEntityWatched(exec: Exec, entityId: string): Promise<boo
|
||||
or exists (select 1 from source_release r
|
||||
where r.media_entity_id = ${entityId} and r.resolution_evidence->>'lookup' in ('search', 'manual'))
|
||||
or exists (select 1 from acquisition_plan p left join acquisition_slot sl on sl.id = p.slot_id
|
||||
where (sl.media_entity_id = ${entityId} or p.evaluation->>'mediaEntityId' = ${entityId}::text)
|
||||
where (sl.media_entity_id = ${entityId} or p.evaluation->>'mediaEntityId' = (${entityId})::text)
|
||||
and (p.evaluation->>'manual')::boolean is true)
|
||||
or exists (select 1 from download_task t
|
||||
left join source_release r on r.id = t.source_release_id
|
||||
@@ -37,14 +36,38 @@ export async function isEntityWatched(exec: Exec, entityId: string): Promise<boo
|
||||
and ${entityId} in (t.media_entity_id, r.media_entity_id, sl.media_entity_id))
|
||||
or exists (select 1 from organization_record o
|
||||
where o.media_entity_id = ${entityId} and o.status in ('queued', 'running'))
|
||||
) as watched`.execute(exec);
|
||||
)`;
|
||||
|
||||
/**
|
||||
* 关注中作品的 TMDB 条目集合(统一元数据缓存的 pinned / 续期用):`movie:<id>` / `tv:<id>` 与选中的剧集组 `group:<gid>`。
|
||||
* 口径同 isEntityWatched。
|
||||
*/
|
||||
export async function watchedTmdbRefs(exec: Exec): Promise<Set<string>> {
|
||||
const r = await sql<{ ref: string; grp: string | null }>`
|
||||
select x.entity_type || ':' || x.external_id as ref, e.episode_group_id as grp
|
||||
from external_identity x join media_entity e on e.id = x.media_entity_id
|
||||
where x.provider = 'tmdb' and x.status = 'accepted' and x.entity_type in ('movie', 'tv')
|
||||
and e.status <> 'merged' and ${watchedExpr(sql`e.id`)}`.execute(exec);
|
||||
const out = new Set<string>();
|
||||
for (const x of r.rows) {
|
||||
out.add(x.ref);
|
||||
if (x.grp) out.add(`group:${x.grp}`);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function isEntityWatched(exec: Exec, entityId: string): Promise<boolean> {
|
||||
const r = await sql<{ watched: boolean }>`select ${watchedExpr(sql`${entityId}::uuid`)} as watched`.execute(exec);
|
||||
return r.rows[0]?.watched === true;
|
||||
}
|
||||
|
||||
/**
|
||||
* 作品进入关注范围(建订阅、用户搜索、开始下载)时补排一次 full 元数据刷新:只对从没刷新过的作品排
|
||||
* (自动建档时因不在关注范围而没排的),刷新完成会重算待确认 / 滞后放行资源的集数(必要时开集数待确认)。
|
||||
* 入队失败只记日志;同一作品同一原因只排一次(幂等键)。
|
||||
* 作品进入关注范围(建订阅、用户搜索 / 手动识别、开始下载)时:
|
||||
* - 补排一次 full 元数据刷新:只对从没刷新过的作品排(自动建档时因不在关注范围而没排的),
|
||||
* 刷新完成会重算待确认 / 滞后放行资源的集数(必要时开集数待确认);同一作品只排一次(幂等键);
|
||||
* - 每次都排一次缓存预热(prewarmMetadata:详情、各季、剧集组、海报 / 背景 / 标志进缓存),幂等键按作品 + 日期,
|
||||
* 同一天重复进入只跑一次;条目已在缓存里时预热只是命中,不发请求。
|
||||
* 入队失败只记日志。返回是否排了刷新。
|
||||
*/
|
||||
export async function enqueueWatchedRefresh(ctx: ServerContext, entityId: string, reason: string): Promise<boolean> {
|
||||
const e = await ctx.db
|
||||
@@ -52,7 +75,9 @@ export async function enqueueWatchedRefresh(ctx: ServerContext, entityId: string
|
||||
.select(["id", "last_metadata_refresh_at", "status"])
|
||||
.where("id", "=", entityId)
|
||||
.executeTakeFirst();
|
||||
if (!e || e.last_metadata_refresh_at || e.status === "merged") return false;
|
||||
if (!e || e.status === "merged") return false;
|
||||
await enqueuePrewarm(ctx, entityId, reason);
|
||||
if (e.last_metadata_refresh_at) return false;
|
||||
try {
|
||||
await ctx.engine.enqueue(
|
||||
"refreshMetadata",
|
||||
@@ -68,3 +93,22 @@ export async function enqueueWatchedRefresh(ctx: ServerContext, entityId: string
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** 关注作品预热入队(幂等键按作品 + 日期);失败只记日志 */
|
||||
export async function enqueuePrewarm(ctx: ServerContext, entityId: string, reason: string): Promise<boolean> {
|
||||
const day = new Date().toISOString().slice(0, 10);
|
||||
try {
|
||||
const r = await ctx.engine.enqueue(
|
||||
"prewarmMetadata",
|
||||
{ mediaEntityId: entityId },
|
||||
{ idempotencyKey: `prewarmMetadata:${entityId}:${day}`, traceId: entityId },
|
||||
);
|
||||
return !r.deduplicated;
|
||||
} catch (err) {
|
||||
ctx.logger.debug(
|
||||
{ entityId, reason, err: { message: err instanceof Error ? err.message : String(err) } },
|
||||
"关注作品预热入队失败",
|
||||
);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import { AUTO_ENRICH_TASK, runAutoEnrich } from "./service/enrich.ts";
|
||||
import { ingestFeed } from "./service/feeds.ts";
|
||||
import { emitMetadataRefreshed } from "./service/lifecycle-events.ts";
|
||||
import { refreshMetadata } from "./service/metadata.ts";
|
||||
import { prewarmMetadata } from "./service/metadata-prewarm.ts";
|
||||
import { resolveRelease } from "./service/resolve.ts";
|
||||
import { snapshotSite } from "./service/snapshot.ts";
|
||||
import { liveSearch } from "./service/workbench.ts";
|
||||
@@ -89,6 +90,15 @@ export function registerSourcesTasks(registry: TaskRegistry, rt: () => SourcesRu
|
||||
return r;
|
||||
}),
|
||||
}),
|
||||
// 关注作品预热(统一元数据缓存):详情 / 各季 / 剧集组 / 图片先进缓存;网络道,走 TMDB 客户端限流
|
||||
defineTask({
|
||||
name: "prewarmMetadata",
|
||||
version: 1,
|
||||
lane: "network",
|
||||
inputSchema: z.object({ mediaEntityId: id }),
|
||||
retryPolicy: { maximumAttempts: 3, initialInterval: "5s", backoffCoefficient: 2, maximumInterval: "60s" },
|
||||
handler: (t) => t.step.run("prewarm", () => prewarmMetadata(rt(), t.input.mediaEntityId)),
|
||||
}),
|
||||
defineTask({
|
||||
name: "searchSubscription",
|
||||
version: 1,
|
||||
|
||||
+191
-29
@@ -1,33 +1,131 @@
|
||||
import { DEFAULT_SETTINGS, type StorageSettings, withStorageDefaults } from "@umt/contracts";
|
||||
import { type PgCache, pgCacheFor, sql } from "@umt/db";
|
||||
import { DEFAULT_SETTINGS, type StorageCacheTtl, type StorageSettings, withStorageDefaults } from "@umt/contracts";
|
||||
import {
|
||||
type FetchPolicy,
|
||||
listenCacheInvalidation,
|
||||
type MetadataCache,
|
||||
metadataCacheFor,
|
||||
type PgCache,
|
||||
pgCacheFor,
|
||||
sql,
|
||||
} from "@umt/db";
|
||||
import { watchedTmdbRefs } from "./modules/sources-identity/service/watched.ts";
|
||||
import type { ServerContext } from "./modules/types.ts";
|
||||
|
||||
// 进程级共享缓存入口:PG kv_cache(移植 MP app/runtime/cache.py 的 TTLCache,存储换成 PG,三进程共享)。
|
||||
// region 与 TTL 规则见 docs/runbook/cache.md 与 docs/porting/moviepilot-tmdb-cache.md;
|
||||
// CACHE-2 起各 region 的 TTL 取值来源是 settings.storage.cacheTtl(docs/runbook/storage.md),语义不变。
|
||||
// 进程级共享缓存入口(docs/runbook/cache.md):
|
||||
// - PgCache:PG kv_cache(移植 MP app/runtime/cache.py 的 TTLCache,存储换成 PG,三进程共享),非元数据的缓存仍直接用;
|
||||
// - MetadataCache:统一元数据缓存,TMDB / 豆瓣 / Bangumi 共用。L1 进程内存 + L2 kv_cache、过期先用旧值后台刷新、
|
||||
// 同请求合并、按 region 计命中。TTL 按「来源 × 资源类型」取 settings.storage.cacheTtl(metaCachePolicy)。
|
||||
|
||||
export const CACHE_REGIONS = {
|
||||
/** TMDB 请求响应(MP TMDb.request 的 @cached):key = path?query */
|
||||
tmdbRequest: "tmdb.request",
|
||||
/** TMDB 识别结果(MP TmdbCache):key = [类型][语言]{tmdbid 或名称}-{年份}-{季} */
|
||||
tmdbRecognize: "tmdb.recognize",
|
||||
doubanRequest: "douban.request",
|
||||
bangumiRequest: "bangumi.request",
|
||||
doubanSubject: "douban.subject",
|
||||
bangumiSubject: "bangumi.subject",
|
||||
/** 豆瓣 / bgm 条目查无(只记「不存在」,网络失败 / 被拦不记) */
|
||||
subjectMiss: "meta.subject_miss",
|
||||
/** 豆瓣 / bgm 条目 → TMDB 作品的确定匹配 */
|
||||
discoverMatch: "discover.match",
|
||||
} as const;
|
||||
|
||||
/** 命中统计只看这些 region(统一元数据缓存);其它 region 不经 MetadataCache */
|
||||
export const METADATA_REGIONS: readonly string[] = Object.values(CACHE_REGIONS);
|
||||
|
||||
const DAY = 86_400;
|
||||
const HOUR = 3_600;
|
||||
|
||||
export function sharedCache(ctx: Pick<ServerContext, "db">): PgCache {
|
||||
return pgCacheFor(ctx.db);
|
||||
}
|
||||
|
||||
/** 统一元数据缓存(同一个 db 共用一个实例) */
|
||||
export function metadataCache(ctx: Pick<ServerContext, "db"> & Partial<Pick<ServerContext, "logger">>): MetadataCache {
|
||||
return metadataCacheFor(ctx.db, ctx.logger ? { logger: ctx.logger } : {});
|
||||
}
|
||||
|
||||
/**
|
||||
* 进程启动时调用:LISTEN umt_cache_invalidate(在线才启用 L1)+ 每分钟把命中计数写进 cache_stat。
|
||||
* 返回 stop(停监听、最后写一次统计)。监听连不上时只记警告,L1 保持停用,照常走 L2。
|
||||
*/
|
||||
export async function startMetadataCache(
|
||||
ctx: Pick<ServerContext, "db" | "logger">,
|
||||
connectionString: string,
|
||||
opts: { flushMs?: number } = {},
|
||||
): Promise<() => Promise<void>> {
|
||||
const cache = metadataCache(ctx);
|
||||
const log = ctx.logger.child({ component: "metadata-cache" });
|
||||
let listener: Awaited<ReturnType<typeof listenCacheInvalidation>> | null = null;
|
||||
try {
|
||||
listener = await listenCacheInvalidation(cache, { connectionString, logger: log });
|
||||
} catch (err) {
|
||||
log.warn({ err: { message: (err as Error).message } }, "缓存失效监听启动失败，进程内一级缓存停用");
|
||||
}
|
||||
const flush = () =>
|
||||
cache.flushStats().catch((err) => log.debug({ err: { message: (err as Error).message } }, "缓存统计写入失败"));
|
||||
const timer = setInterval(flush, opts.flushMs ?? 60_000);
|
||||
return async () => {
|
||||
clearInterval(timer);
|
||||
await listener?.stop();
|
||||
await flush();
|
||||
};
|
||||
}
|
||||
|
||||
/** settings.storage(读失败用缺省) */
|
||||
export async function storageSettings(ctx: Pick<ServerContext, "settings">): Promise<StorageSettings> {
|
||||
return withStorageDefaults(await ctx.settings.get("storage").catch(() => undefined));
|
||||
}
|
||||
|
||||
// ── 策略表:来源 × 资源类型 → TTL ──
|
||||
|
||||
export type MetaCacheSource = "tmdb" | "douban" | "bangumi";
|
||||
/** search:搜索 / 识别;detail:详情 / 季 / 剧集组 / 条目;list:榜单 / 热门 / 每日放送;image:人物 / 图片 */
|
||||
export type MetaResourceKind = "search" | "detail" | "list" | "image";
|
||||
|
||||
export interface MetaCachePolicy {
|
||||
/** 秒 */
|
||||
ttl: number;
|
||||
/** 空结果 / 查无(秒) */
|
||||
emptyTtl: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* 策略表(docs/runbook/cache.md)。宽限期统一按 TTL 的 50%(上限 7 天,见 @umt/db defaultGrace);失败不缓存。
|
||||
*
|
||||
* | 来源 | search | detail | list | image |
|
||||
* | TMDB | tmdbSearchDays / tmdbMissDays | tmdbDetailDays / tmdbMissDays | tmdbListHours / tmdbListEmptyHours | imageDays / tmdbMissDays |
|
||||
* | 豆瓣 / bgm | metaSearchDays / metaMissDays | metaDetailDays / metaMissDays | metaListHours / metaEmptyHours | (无) |
|
||||
*/
|
||||
export function metaCachePolicy(t: StorageCacheTtl, source: MetaCacheSource, kind: MetaResourceKind): MetaCachePolicy {
|
||||
const cap = (ttl: number, empty: number) => ({ ttl, emptyTtl: Math.min(ttl, empty) });
|
||||
if (source === "tmdb") {
|
||||
switch (kind) {
|
||||
case "search":
|
||||
return cap(t.tmdbSearchDays * DAY, t.tmdbMissDays * DAY);
|
||||
case "list":
|
||||
return cap(t.tmdbListHours * HOUR, t.tmdbListEmptyHours * HOUR);
|
||||
case "image":
|
||||
return cap(t.imageDays * DAY, t.tmdbMissDays * DAY);
|
||||
default:
|
||||
return cap(t.tmdbDetailDays * DAY, t.tmdbMissDays * DAY);
|
||||
}
|
||||
}
|
||||
switch (kind) {
|
||||
case "search":
|
||||
return cap(t.metaSearchDays * DAY, t.metaMissDays * DAY);
|
||||
case "list":
|
||||
return cap(t.metaListHours * HOUR, t.metaEmptyHours * HOUR);
|
||||
default:
|
||||
return cap(t.metaDetailDays * DAY, t.metaMissDays * DAY);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* TMDB 缓存策略(MP:CONF.meta = META_CACHE_EXPIRE、CONF.tmdb 条数、EMPTY_RESULT_CACHE_TTL)。
|
||||
* CACHE-2:MP 只有一个 CONF.meta,这里按请求类别取 storage.cacheTtl:搜索 / find 与识别结果用 searchTtl,
|
||||
* 详情(/movie/{id}、/tv/{id}、季)用 ttl(= tmdbDetailDays),空结果与未识别负缓存用 emptyTtl。
|
||||
* 按请求类别取 storage.cacheTtl:搜索 / find 与识别结果用 searchTtl,详情(/movie/{id}、/tv/{id}、季)用 ttl,
|
||||
* 空结果与未识别负缓存用 emptyTtl;列表 / 图片见 metaCachePolicy。
|
||||
*/
|
||||
export interface TmdbCachePolicy {
|
||||
/** 详情 TTL(秒) */
|
||||
@@ -37,16 +135,25 @@ export interface TmdbCachePolicy {
|
||||
maxsize: number;
|
||||
/** 秒 */
|
||||
emptyTtl: number;
|
||||
/** 列表(秒)与列表空结果(秒) */
|
||||
listTtl: number;
|
||||
listEmptyTtl: number;
|
||||
/** 人物 / 图片(秒) */
|
||||
imageTtl: number;
|
||||
}
|
||||
|
||||
export async function tmdbCachePolicy(ctx: Pick<ServerContext, "settings">): Promise<TmdbCachePolicy> {
|
||||
const c = await ctx.settings.get("cache").catch(() => DEFAULT_SETTINGS.cache);
|
||||
const t = (await storageSettings(ctx)).cacheTtl;
|
||||
const list = metaCachePolicy(t, "tmdb", "list");
|
||||
return {
|
||||
ttl: t.tmdbDetailDays * DAY,
|
||||
searchTtl: t.tmdbSearchDays * DAY,
|
||||
maxsize: c.tmdbMaxSize ?? DEFAULT_SETTINGS.cache.tmdbMaxSize,
|
||||
emptyTtl: t.tmdbMissDays * DAY,
|
||||
listTtl: list.ttl,
|
||||
listEmptyTtl: list.emptyTtl,
|
||||
imageTtl: t.imageDays * DAY,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -55,47 +162,102 @@ export function isTmdbSearchPath(key: string): boolean {
|
||||
return /^\/(search|find)\//.test(key);
|
||||
}
|
||||
|
||||
/** 详情键 → `movie:<id>` / `tv:<id>`;不是详情返回 null */
|
||||
/**
|
||||
* 键 → 所属作品:`movie:<id>` / `tv:<id>`(详情、季、剧集、图片、剧集组列表都算);
|
||||
* 剧集组详情 `/tv/episode_group/<gid>` → `group:<gid>`;都不是返回 null。
|
||||
*/
|
||||
export function tmdbDetailRef(key: string): string | null {
|
||||
const g = /^\/tv\/episode_group\/([^/?]+)/.exec(key);
|
||||
if (g) return `group:${decodeURIComponent(g[1]!)}`;
|
||||
const m = /^\/(movie|tv)\/(\d+)(?:[/?]|$)/.exec(key);
|
||||
return m ? `${m[1]}:${m[2]}` : null;
|
||||
}
|
||||
|
||||
// ── 追踪中的作品:详情缓存命中时续期 ──
|
||||
// 追踪 = 作品有未结束的订阅(active / paused / pending)且有已接受的 TMDB 身份。集合按进程缓存 5 分钟。
|
||||
// ── 关注中的作品:条目 pinned(LRU 不淘汰)、命中续期 ──
|
||||
// 关注口径同 service/watched.ts isEntityWatched;集合 = 关注中作品的 TMDB 身份 + 选中的剧集组,按进程缓存 5 分钟。
|
||||
|
||||
const TRACKED_TTL_MS = 5 * 60_000;
|
||||
const trackedCache = new WeakMap<object, { at: number; set: Promise<Set<string>> }>();
|
||||
const WATCHED_TTL_MS = 5 * 60_000;
|
||||
const watchedCache = new WeakMap<object, { at: number; set: Promise<Set<string>> }>();
|
||||
|
||||
export async function trackedTmdbRefs(ctx: Pick<ServerContext, "db">, now = Date.now()): Promise<Set<string>> {
|
||||
const hit = trackedCache.get(ctx.db);
|
||||
if (hit && now - hit.at < TRACKED_TTL_MS) return hit.set;
|
||||
const set = sql<{ ref: string }>`select distinct x.entity_type || ':' || x.external_id as ref
|
||||
from subscription s join external_identity x on x.media_entity_id = s.media_entity_id
|
||||
where s.status in ('active', 'paused', 'pending') and x.provider = 'tmdb' and x.status = 'accepted'
|
||||
and x.entity_type in ('movie', 'tv')`
|
||||
.execute(ctx.db)
|
||||
.then((r) => new Set(r.rows.map((x) => x.ref)));
|
||||
trackedCache.set(ctx.db, { at: now, set });
|
||||
set.catch(() => trackedCache.delete(ctx.db));
|
||||
const hit = watchedCache.get(ctx.db);
|
||||
if (hit && now - hit.at < WATCHED_TTL_MS) return hit.set;
|
||||
const set = watchedTmdbRefs(ctx.db);
|
||||
watchedCache.set(ctx.db, { at: now, set });
|
||||
set.catch(() => watchedCache.delete(ctx.db));
|
||||
return set;
|
||||
}
|
||||
|
||||
/** 测试用:清掉追踪集合缓存 */
|
||||
/** 清掉关注集合缓存(测试;进入关注时预热前也清,让新关注的作品立即 pinned) */
|
||||
export function resetTrackedTmdbRefs(ctx: Pick<ServerContext, "db">): void {
|
||||
trackedCache.delete(ctx.db);
|
||||
watchedCache.delete(ctx.db);
|
||||
}
|
||||
|
||||
/** 这个 TMDB 请求键属于关注中的作品(详情 / 季 / 剧集组 / 图片) */
|
||||
export async function isWatchedTmdbKey(ctx: Pick<ServerContext, "db">, key: string): Promise<boolean> {
|
||||
const ref = tmdbDetailRef(key);
|
||||
return !!ref && (await trackedTmdbRefs(ctx)).has(ref);
|
||||
}
|
||||
|
||||
/**
|
||||
* TMDB 详情缓存命中后的续期:键属于追踪中的作品时把过期时间推到 now + 详情 TTL(只延长不缩短)。
|
||||
* 返回是否续期。失败不影响命中结果。
|
||||
* TMDB 详情缓存命中后的续期(旧接口,保留给直接用 PgCache 的地方):键属于关注中的作品时把过期时间推到
|
||||
* now + 详情 TTL(只延长不缩短)并标 pinned。返回是否续期。
|
||||
*/
|
||||
export async function renewTrackedTmdbDetail(
|
||||
ctx: Pick<ServerContext, "db">,
|
||||
key: string,
|
||||
detailTtlSec: number,
|
||||
): Promise<boolean> {
|
||||
const ref = tmdbDetailRef(key);
|
||||
if (!ref || !(await trackedTmdbRefs(ctx)).has(ref)) return false;
|
||||
return await sharedCache(ctx).renew(CACHE_REGIONS.tmdbRequest, key, detailTtlSec);
|
||||
if (!(await isWatchedTmdbKey(ctx, key))) return false;
|
||||
return await sharedCache(ctx).renew(CACHE_REGIONS.tmdbRequest, key, detailTtlSec, { pin: true });
|
||||
}
|
||||
|
||||
/** 关注中作品的 TMDB 条目:写入 pinned、命中续期(FetchPolicy.pin) */
|
||||
export function watchedPin(ctx: Pick<ServerContext, "db">, key: string): FetchPolicy<unknown>["pin"] {
|
||||
return tmdbDetailRef(key) ? () => isWatchedTmdbKey(ctx, key) : undefined;
|
||||
}
|
||||
|
||||
// ── 命中统计(GET /cache/regions 用) ──
|
||||
|
||||
export interface RegionHitStats {
|
||||
l1Hits: number;
|
||||
l2Hits: number;
|
||||
staleHits: number;
|
||||
misses: number;
|
||||
failures: number;
|
||||
refreshes: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* 最近 days 天各 region 的命中统计:cache_stat(各进程每分钟写入的增量之和)+ 本进程还没写入的计数。
|
||||
* 其它进程最近不到一分钟的计数不在内。
|
||||
*/
|
||||
export async function regionHitStats(ctx: Pick<ServerContext, "db">, days = 7): Promise<Map<string, RegionHitStats>> {
|
||||
const r = await sql<{
|
||||
region: string;
|
||||
l1: number;
|
||||
l2: number;
|
||||
stale: number;
|
||||
miss: number;
|
||||
fail: number;
|
||||
refresh: number;
|
||||
}>`select region, sum(l1_hits)::int8 as l1, sum(l2_hits)::int8 as l2, sum(stale_hits)::int8 as stale,
|
||||
sum(misses)::int8 as miss, sum(failures)::int8 as fail, sum(refreshes)::int8 as refresh
|
||||
from cache_stat where day > current_date - ${days}::int group by region`.execute(ctx.db);
|
||||
const out = new Map<string, RegionHitStats>();
|
||||
for (const x of r.rows)
|
||||
out.set(x.region, {
|
||||
l1Hits: Number(x.l1),
|
||||
l2Hits: Number(x.l2),
|
||||
staleHits: Number(x.stale),
|
||||
misses: Number(x.miss),
|
||||
failures: Number(x.fail),
|
||||
refreshes: Number(x.refresh),
|
||||
});
|
||||
for (const [region, c] of metadataCache(ctx).pendingStats()) {
|
||||
const o = out.get(region) ?? { l1Hits: 0, l2Hits: 0, staleHits: 0, misses: 0, failures: 0, refreshes: 0 };
|
||||
for (const k of Object.keys(o) as (keyof RegionHitStats)[]) o[k] += c[k];
|
||||
out.set(region, o);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import type { ScrapeFileResult, ScrapeMatch } from "@umt/contracts";
|
||||
import { createTempTree, type TempTree } from "@umt/testkit";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { queueScrape, runScrapeRecord } from "../../src/modules/acquisition-organize/services/scrape.ts";
|
||||
import { createHarness, type Harness, pgAvailable, seedBase } from "./harness.ts";
|
||||
|
||||
// TMDB 标志 → clearlogo.png:电影写影片目录、剧集写剧集根目录;覆盖方式与其它图片一致
|
||||
// (更新:没变不重写、本工具写过且没被改过的按新图覆盖、被人改过的跳过;只补缺失;全部覆盖)。
|
||||
|
||||
const hasPg = await pgAvailable();
|
||||
const at = new Date().toISOString();
|
||||
const f = (value: unknown) => ({ value, source: "tmdb", language: null, at, locked: false });
|
||||
const enc = (s: string) => new TextEncoder().encode(s);
|
||||
|
||||
describe.skipIf(!hasPg)("刮削:TMDB 标志写 clearlogo.png", () => {
|
||||
let h: Harness;
|
||||
let lib: TempTree;
|
||||
let rootId = "";
|
||||
|
||||
beforeAll(async () => {
|
||||
lib = await createTempTree(
|
||||
{
|
||||
"Movies/星际穿越 (2014)/Interstellar.2014.mkv": 100,
|
||||
"TV/人生切割术 (2022)/Season 1/Severance.S01E01.mkv": 100,
|
||||
},
|
||||
{ sentinel: true, prefix: "umt-scrlogo-" },
|
||||
);
|
||||
h = await createHarness("scrape_logo", {
|
||||
// 内容 = 路径,方便看出写的是哪张
|
||||
scrapeImage: async (p) => ({ content: enc(p), mime: p.endsWith(".png") ? "image/png" : "image/jpeg" }),
|
||||
});
|
||||
rootId = (await seedBase(h.db, { libraryPath: lib.root, cachePath: lib.root, qbUrl: "http://127.0.0.1:9" })).rootId;
|
||||
}, 60_000);
|
||||
afterAll(async () => {
|
||||
await h?.close();
|
||||
await lib?.cleanup();
|
||||
});
|
||||
|
||||
async function entity(kind: "movie" | "series", title: string, meta: object) {
|
||||
return (
|
||||
await h.db
|
||||
.insertInto("media_entity")
|
||||
.values({ kind, title, year: 2020, status: "active", metadata: JSON.stringify(meta) })
|
||||
.returning("id")
|
||||
.executeTakeFirstOrThrow()
|
||||
).id;
|
||||
}
|
||||
const setMeta = (id: string, meta: object) =>
|
||||
h.db
|
||||
.updateTable("media_entity")
|
||||
.set({ metadata: JSON.stringify(meta) })
|
||||
.where("id", "=", id)
|
||||
.execute();
|
||||
async function scrape(
|
||||
entityId: string,
|
||||
kind: "movie" | "tv",
|
||||
path: string,
|
||||
mode?: "missing" | "overwrite" | "update",
|
||||
) {
|
||||
const match: ScrapeMatch = {
|
||||
mediaEntityId: entityId,
|
||||
provider: null,
|
||||
externalId: null,
|
||||
kind,
|
||||
title: "x",
|
||||
year: null,
|
||||
source: "library",
|
||||
};
|
||||
const q = await queueScrape(h.d, {
|
||||
rootId: `lib:${rootId}`,
|
||||
rootPath: lib.root,
|
||||
libraryRootId: rootId,
|
||||
path,
|
||||
match,
|
||||
mode,
|
||||
trigger: "manual",
|
||||
});
|
||||
return runScrapeRecord(h.d, q.recordId);
|
||||
}
|
||||
const logo = (files: ScrapeFileResult[]) => files.find((x) => x.role === "clearlogo");
|
||||
const MOVIE_DIR = "Movies/星际穿越 (2014)";
|
||||
const LOGO = `${MOVIE_DIR}/clearlogo.png`;
|
||||
|
||||
it("电影:写在影片目录 clearlogo.png;更新模式内容没变不重写,换了标志按新图覆盖", async () => {
|
||||
const movie = await entity("movie", "星际穿越", { title: f("星际穿越"), logoPath: f("/logo-a.png") });
|
||||
const r = await scrape(movie, "movie", MOVIE_DIR);
|
||||
expect(logo(r.files)).toMatchObject({
|
||||
path: LOGO,
|
||||
action: "written",
|
||||
hash: expect.stringMatching(/^[0-9a-f]{64}$/),
|
||||
});
|
||||
expect(await readFile(lib.path(LOGO), "utf8")).toBe("/logo-a.png");
|
||||
const same = await scrape(movie, "movie", MOVIE_DIR, "update");
|
||||
expect(logo(same.files)).toMatchObject({ action: "skipped", reason: "scrape_unchanged" });
|
||||
await setMeta(movie, { title: f("星际穿越"), logoPath: f("/logo-b.png") });
|
||||
const upd = await scrape(movie, "movie", MOVIE_DIR, "update");
|
||||
expect(logo(upd.files)).toMatchObject({ action: "written" });
|
||||
expect(await readFile(lib.path(LOGO), "utf8")).toBe("/logo-b.png");
|
||||
|
||||
// 被人换掉的标志:更新跳过、只补缺失跳过、全部覆盖改写
|
||||
await writeFile(lib.path(LOGO), "我自己的标志");
|
||||
await setMeta(movie, { title: f("星际穿越"), logoPath: f("/logo-c.png") });
|
||||
expect(logo((await scrape(movie, "movie", MOVIE_DIR, "update")).files)).toMatchObject({
|
||||
action: "skipped",
|
||||
reason: "scrape_user_modified",
|
||||
});
|
||||
expect(logo((await scrape(movie, "movie", MOVIE_DIR, "missing")).files)).toMatchObject({
|
||||
action: "skipped",
|
||||
reason: "scrape_exists",
|
||||
});
|
||||
expect(await readFile(lib.path(LOGO), "utf8")).toBe("我自己的标志");
|
||||
expect(logo((await scrape(movie, "movie", MOVIE_DIR, "overwrite")).files)).toMatchObject({ action: "written" });
|
||||
expect(await readFile(lib.path(LOGO), "utf8")).toBe("/logo-c.png");
|
||||
});
|
||||
|
||||
it("剧集:写在剧集根目录,季目录不写;没有标志不写", async () => {
|
||||
const show = await entity("series", "人生切割术", { title: f("人生切割术"), logoPath: f("/sev.png") });
|
||||
const r = await scrape(show, "tv", "TV/人生切割术 (2022)");
|
||||
expect(logo(r.files)).toMatchObject({ path: "TV/人生切割术 (2022)/clearlogo.png", action: "written" });
|
||||
expect(r.files.filter((x) => x.role === "clearlogo")).toHaveLength(1);
|
||||
await setMeta(show, { title: f("人生切割术") });
|
||||
const none = await scrape(show, "tv", "TV/人生切割术 (2022)", "overwrite");
|
||||
expect(logo(none.files)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
Vendored
+13
-5
@@ -67,7 +67,10 @@ describe.skipIf(!hasPg)("C2-A 缓存(真实 PG)", () => {
|
||||
await (await new SourcesRuntime(h.ctx).tmdb())!.details("movie", 872585);
|
||||
const calls = tmdb.calls.length;
|
||||
const rows = (
|
||||
await sql<{ key: string; days: number }>`select key, extract(epoch from expires_at - now()) / 86400 as days
|
||||
await sql<{
|
||||
key: string;
|
||||
days: number;
|
||||
}>`select key, extract(epoch from coalesce(fresh_until, expires_at) - now()) / 86400 as days
|
||||
from kv_cache where region = 'tmdb.request'`.execute(h.db)
|
||||
).rows;
|
||||
const detail = rows.find((x) => x.key.startsWith("/movie/872585?"))!;
|
||||
@@ -80,7 +83,7 @@ describe.skipIf(!hasPg)("C2-A 缓存(真实 PG)", () => {
|
||||
const days = async (like: string) =>
|
||||
Number(
|
||||
(
|
||||
await sql<{ d: number }>`select extract(epoch from expires_at - now()) / 86400 as d
|
||||
await sql<{ d: number }>`select extract(epoch from coalesce(fresh_until, expires_at) - now()) / 86400 as d
|
||||
from kv_cache where region = 'tmdb.request' and key like ${like}`.execute(h.db)
|
||||
).rows[0]!.d,
|
||||
);
|
||||
@@ -96,7 +99,7 @@ describe.skipIf(!hasPg)("C2-A 缓存(真实 PG)", () => {
|
||||
expect(empty).toBeLessThanOrEqual(2);
|
||||
});
|
||||
|
||||
it("追踪中的作品:详情缓存命中时续期到 now + 详情 TTL;未追踪的不动", async () => {
|
||||
it("关注中的作品:详情缓存命中时续期到 now + 详情 TTL 并标 pinned;未关注的不动", async () => {
|
||||
const ent = (
|
||||
await sql<{
|
||||
id: string;
|
||||
@@ -111,19 +114,24 @@ describe.skipIf(!hasPg)("C2-A 缓存(真实 PG)", () => {
|
||||
const client = (await new SourcesRuntime(h.ctx).tmdb())!;
|
||||
await client.details("movie", 872585);
|
||||
await client.details("movie", 693134);
|
||||
await sql`update kv_cache set expires_at = now() + interval '1 day'
|
||||
await sql`update kv_cache set fresh_until = now() + interval '1 day', expires_at = now() + interval '2 days',
|
||||
pinned = false
|
||||
where region = 'tmdb.request' and (key like '/movie/872585?%' or key like '/movie/693134?%')`.execute(h.db);
|
||||
await client.details("movie", 872585);
|
||||
await client.details("movie", 693134);
|
||||
const left = async (like: string) =>
|
||||
Number(
|
||||
(
|
||||
await sql<{ d: number }>`select extract(epoch from expires_at - now()) / 86400 as d
|
||||
await sql<{ d: number }>`select extract(epoch from fresh_until - now()) / 86400 as d
|
||||
from kv_cache where region = 'tmdb.request' and key like ${like}`.execute(h.db)
|
||||
).rows[0]!.d,
|
||||
);
|
||||
expect(await left("/movie/872585?%")).toBeGreaterThan(19.9);
|
||||
expect(await left("/movie/693134?%")).toBeLessThanOrEqual(1);
|
||||
const pinned = await sql<{ k: string; p: boolean }>`select key as k, pinned as p from kv_cache
|
||||
where region = 'tmdb.request' and (key like '/movie/872585?%' or key like '/movie/693134?%')`.execute(h.db);
|
||||
expect(pinned.rows.filter((r) => r.p).every((r) => r.k.startsWith("/movie/872585?"))).toBe(true);
|
||||
expect(pinned.rows.some((r) => r.p)).toBe(true);
|
||||
});
|
||||
|
||||
it("GET /cache/tmdb 列出条目(键 / 过期 / 大小)与上限;POST /cache/tmdb/clear 清空", async () => {
|
||||
|
||||
+183
@@ -0,0 +1,183 @@
|
||||
import { metadataCacheFor, PgCache, sql } from "@umt/db";
|
||||
import { createMockTmdb, type MockTmdb } from "@umt/testkit";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { createCacheModule } from "../../src/modules/cache/index.ts";
|
||||
import { SourcesRuntime } from "../../src/modules/sources-identity/runtime.ts";
|
||||
import { metaRequestCache } from "../../src/modules/sources-identity/service/meta-subjects.ts";
|
||||
import { metaCachePolicy, regionHitStats, resetTrackedTmdbRefs } from "../../src/shared-cache.ts";
|
||||
import { bearer, createHarness, type Harness, pgAvailable } from "../auth-settings/harness.ts";
|
||||
import { DEV_KEY, json } from "../helpers.ts";
|
||||
|
||||
// 统一元数据缓存接到 TMDB / 豆瓣 / bgm:策略表、过期先用旧值后台刷新、同请求合并、失败不缓存、
|
||||
// 关注中作品 pinned 不被淘汰、按 region 统计命中(GET /cache/regions)。
|
||||
|
||||
const hasPg = await pgAvailable();
|
||||
const KEY = "0123456789abcdef0123456789abcdef";
|
||||
|
||||
const until = async (cond: () => boolean | Promise<boolean>, ms = 5000) => {
|
||||
const end = Date.now() + ms;
|
||||
while (!(await cond())) {
|
||||
if (Date.now() > end) throw new Error("等待超时");
|
||||
await new Promise((r) => setTimeout(r, 20));
|
||||
}
|
||||
};
|
||||
|
||||
describe("策略表(来源 × 资源类型)", () => {
|
||||
it("缺省:搜索 7 天(空 1 天)、详情 30 天(空 1 天)、列表 6 小时(空 1 小时)、图片 90 天", async () => {
|
||||
const { DEFAULT_STORAGE_SETTINGS } = await import("@umt/contracts");
|
||||
const t = DEFAULT_STORAGE_SETTINGS.cacheTtl;
|
||||
const day = 86_400;
|
||||
for (const s of ["tmdb", "douban", "bangumi"] as const) {
|
||||
expect(metaCachePolicy(t, s, "search")).toEqual({ ttl: 7 * day, emptyTtl: day });
|
||||
expect(metaCachePolicy(t, s, "detail")).toEqual({ ttl: 30 * day, emptyTtl: day });
|
||||
expect(metaCachePolicy(t, s, "list")).toEqual({ ttl: 6 * 3600, emptyTtl: 3600 });
|
||||
}
|
||||
expect(metaCachePolicy(t, "tmdb", "image")).toEqual({ ttl: 90 * day, emptyTtl: day });
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!hasPg)("统一元数据缓存(真实 PG)", () => {
|
||||
let h: Harness;
|
||||
let admin: ReturnType<typeof bearer>;
|
||||
let tmdb: MockTmdb;
|
||||
|
||||
beforeAll(async () => {
|
||||
h = await createHarness({ extraModules: [createCacheModule()] });
|
||||
admin = bearer(h.app, DEV_KEY);
|
||||
tmdb = await createMockTmdb({ apiKey: KEY });
|
||||
const r = await admin("PATCH", "/settings", {
|
||||
metadata: { tmdbApiKey: KEY, tmdbBaseUrl: `${tmdb.url}/3` },
|
||||
cache: { tmdbMaxSize: 16 },
|
||||
});
|
||||
expect(r.status).toBe(200);
|
||||
});
|
||||
afterAll(async () => {
|
||||
await tmdb?.close();
|
||||
await h?.close();
|
||||
});
|
||||
|
||||
const client = async () => (await new SourcesRuntime(h.ctx).tmdb())!;
|
||||
const row = async (like: string) =>
|
||||
(
|
||||
await sql<{ key: string; fresh: number; pinned: boolean }>`select key,
|
||||
extract(epoch from fresh_until - now()) / 86400 as fresh, pinned
|
||||
from kv_cache where region = 'tmdb.request' and key like ${like}`.execute(h.db)
|
||||
).rows[0];
|
||||
|
||||
it("TMDB 详情过期在宽限期内:先返回旧值,后台只刷一次;刷新后是新值", async () => {
|
||||
const c = await client();
|
||||
await c.details("movie", 872585);
|
||||
const before = tmdb.callsTo("/3/movie/872585").length;
|
||||
// 改成「已过新鲜期、还在宽限期」并在值里做个记号(热度 12345)
|
||||
await sql`update kv_cache set fresh_until = now() - interval '1 second', expires_at = now() + interval '1 day',
|
||||
value = jsonb_set(value, '{popularity}', '12345')
|
||||
where region = 'tmdb.request' and key like '/movie/872585?%'`.execute(h.db);
|
||||
const keys = (
|
||||
await sql<{
|
||||
key: string;
|
||||
}>`select key from kv_cache where region = 'tmdb.request' and key like '/movie/872585?%'`.execute(h.db)
|
||||
).rows.map((r) => r.key);
|
||||
expect(keys.length).toBeGreaterThan(0);
|
||||
// 回源慢一点,让两个调用方都在刷新完成前读到
|
||||
tmdb.fault("/3/movie/872585", { delayMs: 300, times: 10 });
|
||||
const [a, b] = await Promise.all([c.details("movie", 872585), c.details("movie", 872585)]);
|
||||
expect(a.popularity).toBe(12345);
|
||||
expect(b.popularity).toBe(12345);
|
||||
// 后台刷新:两个调用方同时拿到旧值,每个过期键只回源一次;刷新后新鲜且是新值
|
||||
await until(async () => {
|
||||
const r = await sql<{ n: number }>`select count(*)::int as n from kv_cache where region = 'tmdb.request'
|
||||
and key like '/movie/872585?%' and fresh_until > now() + interval '29 days'`.execute(h.db);
|
||||
return r.rows[0]!.n === keys.length;
|
||||
});
|
||||
expect(tmdb.callsTo("/3/movie/872585").length - before).toBe(keys.length);
|
||||
tmdb.clearFaults();
|
||||
expect((await c.details("movie", 872585)).popularity).not.toBe(12345);
|
||||
});
|
||||
|
||||
it("同请求合并:并发同一详情只回源一次;失败(5xx)不缓存", async () => {
|
||||
const c = await client();
|
||||
const n0 = tmdb.callsTo("/3/tv/95396/season/1").length;
|
||||
await Promise.all(Array.from({ length: 5 }, () => c.getJson("/tv/95396/season/1", { language: "zh-CN" })));
|
||||
expect(tmdb.callsTo("/3/tv/95396/season/1").length - n0).toBe(1);
|
||||
// 5xx 重试用尽后抛错(客户端缺省重试 3 次,退避约 3.5 秒),不写缓存
|
||||
tmdb.fault("/3/tv/100088/season/1", { status: 503, times: 10 });
|
||||
await expect(c.getJson("/tv/100088/season/1", { language: "zh-CN" })).rejects.toThrow();
|
||||
expect(await row("/tv/100088/season/1?%")).toBeUndefined();
|
||||
tmdb.clearFaults();
|
||||
});
|
||||
|
||||
it("关注中的作品:条目 pinned,条数上限淘汰时不被清掉", async () => {
|
||||
const ent = (
|
||||
await sql<{
|
||||
id: string;
|
||||
}>`insert into media_entity (kind, title) values ('series', '人生切割术') returning id`.execute(h.db)
|
||||
).rows[0]!.id;
|
||||
await sql`insert into external_identity (media_entity_id, provider, entity_type, external_id, status)
|
||||
values (${ent}, 'tmdb', 'tv', '95396', 'accepted')`.execute(h.db);
|
||||
await sql`insert into subscription (media_entity_id, media_kind, status) values (${ent}, 'series', 'active')`.execute(
|
||||
h.db,
|
||||
);
|
||||
resetTrackedTmdbRefs(h.ctx);
|
||||
await sql`delete from kv_cache where region = 'tmdb.request'`.execute(h.db);
|
||||
const c = await client();
|
||||
await c.getJson("/tv/95396", { language: "zh-CN" });
|
||||
expect((await row("/tv/95396?%"))?.pinned).toBe(true);
|
||||
// 上限 16:再写一批别的请求,关注作品的条目留着
|
||||
for (let i = 0; i < 24; i++) await c.searchRaw("movie", `不存在${i}`);
|
||||
expect(await row("/tv/95396?%")).toBeDefined();
|
||||
const n = await sql<{ n: number }>`select count(*)::int as n from kv_cache where region = 'tmdb.request'`.execute(
|
||||
h.db,
|
||||
);
|
||||
expect(n.rows[0]!.n).toBeLessThanOrEqual(16);
|
||||
});
|
||||
|
||||
it("豆瓣 / bgm 请求缓存按策略表:搜索 7 天、列表 6 小时、空结果按查无 / 列表空;失败不写", async () => {
|
||||
const cache = metaRequestCache(h.ctx);
|
||||
await cache.cached({ source: "douban", key: "GET s1", kind: "search", empty: () => false }, async () => [1]);
|
||||
await cache.cached({ source: "douban", key: "GET s0", kind: "search", empty: () => true }, async () => []);
|
||||
await cache.cached({ source: "bangumi", key: "GET l1", kind: "list", empty: () => false }, async () => ({
|
||||
data: [1],
|
||||
}));
|
||||
await expect(
|
||||
cache.cached({ source: "bangumi", key: "GET bad", kind: "list", empty: () => false }, async () => {
|
||||
throw new Error("HTTP 403");
|
||||
}),
|
||||
).rejects.toThrow("403");
|
||||
const ttl = async (region: string, key: string) =>
|
||||
Number(
|
||||
(
|
||||
await sql<{ s: number }>`select extract(epoch from fresh_until - now()) as s from kv_cache
|
||||
where region = ${region} and key = ${key}`.execute(h.db)
|
||||
).rows[0]?.s ?? -1,
|
||||
);
|
||||
expect(await ttl("douban.request", "GET s1")).toBeGreaterThan(7 * 86400 - 60);
|
||||
expect(await ttl("douban.request", "GET s0")).toBeLessThanOrEqual(86400);
|
||||
expect(await ttl("bangumi.request", "GET l1")).toBeLessThanOrEqual(6 * 3600);
|
||||
expect(await ttl("bangumi.request", "GET l1")).toBeGreaterThan(6 * 3600 - 60);
|
||||
expect(await ttl("bangumi.request", "GET bad")).toBe(-1);
|
||||
});
|
||||
|
||||
it("命中统计:各 region 计 L1 / L2 / 旧值 / 未命中 / 失败 / 后台刷新;写入 cache_stat 后接口合计并给命中率", async () => {
|
||||
const mc = metadataCacheFor(h.db);
|
||||
const pending = mc.pendingStats().get("tmdb.request");
|
||||
expect(pending?.misses).toBeGreaterThan(0);
|
||||
expect(pending?.staleHits).toBeGreaterThan(0);
|
||||
expect(pending?.refreshes).toBeGreaterThan(0);
|
||||
expect(pending?.failures).toBeGreaterThan(0);
|
||||
await mc.flushStats();
|
||||
expect(mc.pendingStats().size).toBe(0);
|
||||
// 再来一次 L2 命中(未写入 cache_stat 的也算进接口)
|
||||
await (await client()).getJson("/tv/95396", { language: "zh-CN" });
|
||||
const stats = await regionHitStats(h.ctx);
|
||||
expect(stats.get("tmdb.request")!.l2Hits).toBeGreaterThan(0);
|
||||
const list = await json(await admin("GET", "/cache/regions"));
|
||||
const tmdbRow = list.data.items.find((i: { region: string }) => i.region === "tmdb.request");
|
||||
expect(tmdbRow.hits.hitRate).toBeGreaterThan(0);
|
||||
expect(tmdbRow.hits.hitRate).toBeLessThan(1);
|
||||
expect(tmdbRow.hits).toMatchObject({ misses: expect.any(Number), l2Hits: expect.any(Number) });
|
||||
// 非元数据 region 没有命中统计
|
||||
await new PgCache(h.db).set("site.detail", "x", 1, { ttl: 60 });
|
||||
const again = await json(await admin("GET", "/cache/regions"));
|
||||
expect(again.data.items.find((i: { region: string }) => i.region === "site.detail").hits).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -40,6 +40,19 @@ const DEVIATIONS: { id: string; apply: (files: string[], scenario: string) => st
|
||||
? fs
|
||||
: fs.map((f) => (f === "poster.jpg" || f === "fanart.jpg" ? `${LOOSE.replace(/\.mkv$/, "")}-${f}` : f)),
|
||||
},
|
||||
// 4. 作品级标志 clearlogo.png(TMDB logos;MP 只在接 fanart.tv 时写 logo,我们不接 fanart.tv):
|
||||
// 写在作品海报旁边——电影影片目录 / 剧集根目录 clearlogo.png,库根的电影 <视频名>-clearlogo.png;季目录不写
|
||||
{
|
||||
id: "tmdb_clearlogo",
|
||||
apply: (fs) => [
|
||||
...fs,
|
||||
...fs.flatMap((f) => {
|
||||
const m = /^(.*?)(\/|^|-)poster\.jpg$/.exec(f);
|
||||
if (!m || /(^|\/)Season \d+$/.test(m[1]!) || /(^|\/)season(\d+|-specials)$/.test(m[1]!)) return [];
|
||||
return [`${m[1]}${m[2]}clearlogo.png`];
|
||||
}),
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const detail = (entityType: "movie" | "tv", externalId: string): TmdbRichDetail =>
|
||||
@@ -69,6 +82,7 @@ const detail = (entityType: "movie" | "tv", externalId: string): TmdbRichDetail
|
||||
originCountry: [],
|
||||
posterPath: "/poster.jpg",
|
||||
backdropPath: "/backdrop.jpg",
|
||||
logoPath: "/logo.png",
|
||||
cast: [],
|
||||
crew: [],
|
||||
externalIds: { imdb: null, tvdb: null, wikidata: null },
|
||||
@@ -124,7 +138,10 @@ describe.skipIf(skip)("刮削文件布局与 MP 差分", () => {
|
||||
"scrlayout",
|
||||
{
|
||||
ao: {
|
||||
scrapeImage: async () => ({ content: new Uint8Array([0xff, 0xd8, 0xff]), mime: "image/jpeg" }),
|
||||
scrapeImage: async (p) =>
|
||||
p.endsWith(".png")
|
||||
? { content: new Uint8Array([0x89, 0x50, 0x4e, 0x47, 1]), mime: "image/png" }
|
||||
: { content: new Uint8Array([0xff, 0xd8, 0xff]), mime: "image/jpeg" },
|
||||
scrapeTmdbDetail: async (type, id) => detail(type, id),
|
||||
scrapeTmdbSeason: async (_tv, n) => (n === 1 ? season1 : null),
|
||||
},
|
||||
|
||||
@@ -218,13 +218,13 @@ describe.skipIf(!up)("发现页接口", () => {
|
||||
expect(meta.filter((m) => m.includes("movie_showing"))).toHaveLength(3);
|
||||
expect(meta.at(-1)).toContain("start=20&count=20");
|
||||
|
||||
// 空结果:写入但用空结果 TTL(缺省 1 小时),正常 6 小时
|
||||
// 空结果:写入但用空结果 TTL(缺省 1 小时),正常 6 小时(看新鲜期;过期后还有 50% 宽限期先用旧值)
|
||||
doubanReply = () => json({ subject_collection_items: [], total: 0 });
|
||||
await req("/discover/douban?list=tv_korean");
|
||||
const rows = await cache.items(META_REQUEST_REGIONS.douban);
|
||||
const ttlOf = (part: string) => {
|
||||
const row = rows.find((x) => x.key.includes(part))!;
|
||||
return Math.round((new Date(row.expiresAt!).getTime() - Date.now()) / 60_000);
|
||||
return Math.round((new Date(row.freshUntil).getTime() - Date.now()) / 60_000);
|
||||
};
|
||||
expect(ttlOf("tv_korean")).toBeLessThanOrEqual(60);
|
||||
expect(ttlOf("tv_korean")).toBeGreaterThan(55);
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
import { existsSync, mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { createTmdbClient } from "@umt/adapters";
|
||||
import { sql } from "@umt/db";
|
||||
import { createMockTmdb, type MockTmdb, SPLIT_COUR_FIXTURE, TMDB_FIXTURES } from "@umt/testkit";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { tmdbSharedCache } from "../../src/modules/sources-identity/runtime.ts";
|
||||
import { prewarmMetadata } from "../../src/modules/sources-identity/service/metadata-prewarm.ts";
|
||||
import { enqueueWatchedRefresh } from "../../src/modules/sources-identity/service/watched.ts";
|
||||
import { tmdbCachePolicy } from "../../src/shared-cache.ts";
|
||||
import { createHarness, type Harness, pgAvailable } from "./harness.ts";
|
||||
|
||||
// 关注作品预热:进入关注时入队 prewarmMetadata(幂等键按作品 + 日期);执行后详情(含标志)、各季、选中的剧集组
|
||||
// 在 TMDB 请求缓存里且 pinned,海报 / 背景 / 标志进图片缓存;再跑一次全部命中,不发请求。
|
||||
|
||||
const skip = !(await pgAvailable());
|
||||
const PNG = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 1]);
|
||||
const JPEG = new Uint8Array([0xff, 0xd8, 0xff, 0xe0, 1]);
|
||||
|
||||
describe.skipIf(skip)("关注作品预热", () => {
|
||||
let h: Harness;
|
||||
let tmdb: MockTmdb;
|
||||
let entityId: string;
|
||||
const images: string[] = [];
|
||||
const dataDir = mkdtempSync(join(tmpdir(), "umt-prewarm-"));
|
||||
|
||||
beforeAll(async () => {
|
||||
tmdb = await createMockTmdb({
|
||||
fixtures: [
|
||||
...TMDB_FIXTURES,
|
||||
{
|
||||
...SPLIT_COUR_FIXTURE,
|
||||
rich: { ...SPLIT_COUR_FIXTURE.rich, backdrop_path: "/split-bg.jpg" },
|
||||
logos: [
|
||||
{ file_path: "/split-en.png", iso_639_1: "en" },
|
||||
{ file_path: "/split-ja.png", iso_639_1: "ja" },
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
let policy: Awaited<ReturnType<typeof tmdbCachePolicy>> | null = null;
|
||||
h = await createHarness("prewarm", {
|
||||
// 和服务端一样走统一元数据缓存(PG 请求缓存)
|
||||
tmdb: () =>
|
||||
createTmdbClient({
|
||||
apiKey: "test-tmdb-key",
|
||||
baseUrl: `${tmdb.url}/3`,
|
||||
retries: 0,
|
||||
sharedCache: tmdbSharedCache(h.ctx, policy!),
|
||||
}),
|
||||
// 图片上游(image.tmdb.org)
|
||||
fetch: async (input) => {
|
||||
const url = String(input);
|
||||
images.push(url);
|
||||
return new Response(url.endsWith(".png") ? PNG : JPEG);
|
||||
},
|
||||
});
|
||||
policy = await tmdbCachePolicy(h.ctx);
|
||||
(h.ctx.config as { DATA_DIR?: string }).DATA_DIR = dataDir;
|
||||
entityId = (
|
||||
await sql<{ id: string }>`insert into media_entity (kind, title, episode_group_id, episode_group_name)
|
||||
values ('series', '分割番', 'grp-production', '制作季') returning id`.execute(h.db)
|
||||
).rows[0]!.id;
|
||||
await sql`insert into external_identity (media_entity_id, provider, entity_type, external_id, status)
|
||||
values (${entityId}, 'tmdb', 'tv', ${String(SPLIT_COUR_FIXTURE.id)}, 'accepted')`.execute(h.db);
|
||||
});
|
||||
afterAll(async () => {
|
||||
await h?.close();
|
||||
await tmdb?.close();
|
||||
rmSync(dataDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("进入关注(建订阅)时入队预热,幂等键按作品 + 日期;同一天再进入不重复排", async () => {
|
||||
await sql`insert into subscription (media_entity_id, media_kind, status) values (${entityId}, 'series', 'active')`.execute(
|
||||
h.db,
|
||||
);
|
||||
await enqueueWatchedRefresh(h.ctx, entityId, "subscription");
|
||||
await enqueueWatchedRefresh(h.ctx, entityId, "download");
|
||||
const jobs = h.engine.enqueued.filter((e) => e.task === "prewarmMetadata");
|
||||
expect(jobs).toHaveLength(1);
|
||||
expect(jobs[0]).toMatchObject({
|
||||
input: { mediaEntityId: entityId },
|
||||
key: `prewarmMetadata:${entityId}:${new Date().toISOString().slice(0, 10)}`,
|
||||
});
|
||||
});
|
||||
|
||||
it("执行后:详情 / 各季 / 剧集组 / 标志在缓存里且 pinned;海报、背景、标志进图片缓存", async () => {
|
||||
const r = await prewarmMetadata(h.rt, entityId);
|
||||
expect(r).toMatchObject({ status: "warmed", episodeGroup: true, images: 3, errors: [] });
|
||||
expect(r.seasons).toBe(SPLIT_COUR_FIXTURE.seasons!.length);
|
||||
const rows = (
|
||||
await sql<{ key: string; pinned: boolean }>`select key, pinned from kv_cache where region = 'tmdb.request'
|
||||
order by key`.execute(h.db)
|
||||
).rows;
|
||||
const has = (re: RegExp) => rows.filter((x) => re.test(x.key));
|
||||
const id = SPLIT_COUR_FIXTURE.id;
|
||||
expect(has(new RegExp(`^/tv/${id}\\?.*images`)).length).toBeGreaterThan(0);
|
||||
for (const s of SPLIT_COUR_FIXTURE.seasons!)
|
||||
expect(has(new RegExp(`^/tv/${id}/season/${s.season_number}\\?`)).length).toBeGreaterThan(0);
|
||||
expect(has(/^\/tv\/episode_group\/grp-production\?/)).toHaveLength(1);
|
||||
// 原语言是日语:另取了日语标志
|
||||
expect(has(new RegExp(`^/tv/${id}/images\\?include_image_language=ja`))).toHaveLength(1);
|
||||
expect(rows.every((x) => x.pinned)).toBe(true);
|
||||
for (const f of ["split.jpg", "split-bg.jpg", "split-ja.png"])
|
||||
expect(existsSync(join(dataDir, "cache/images/t/p/original", f))).toBe(true);
|
||||
expect(images.every((u) => u.startsWith("https://image.tmdb.org/t/p/original/"))).toBe(true);
|
||||
});
|
||||
|
||||
it("再预热一次全部命中:不发 TMDB 请求、不再下载图片", async () => {
|
||||
const calls = tmdb.calls.length;
|
||||
const fetched = images.length;
|
||||
const r = await prewarmMetadata(h.rt, entityId);
|
||||
expect(r.status).toBe("warmed");
|
||||
expect(tmdb.calls.length).toBe(calls);
|
||||
expect(images.length).toBe(fetched);
|
||||
});
|
||||
|
||||
it("没有 TMDB 身份的作品跳过", async () => {
|
||||
const other = (
|
||||
await sql<{ id: string }>`insert into media_entity (kind, title) values ('movie', '本地') returning id`.execute(
|
||||
h.db,
|
||||
)
|
||||
).rows[0]!.id;
|
||||
expect(await prewarmMetadata(h.rt, other)).toMatchObject({ status: "skipped", reason: "no_tmdb_identity" });
|
||||
});
|
||||
});
|
||||
@@ -69,12 +69,13 @@ describe.skipIf(!up)("M-Team 快照与任务注册", () => {
|
||||
api.setFailMode(null);
|
||||
});
|
||||
|
||||
it("模块注册 7 个任务与 3 条 outbox 路由;任务处理器经 step.run 调服务", async () => {
|
||||
it("模块注册 8 个任务与 3 条 outbox 路由;任务处理器经 step.run 调服务", async () => {
|
||||
const reg = new TaskRegistry();
|
||||
h.module.registerTasks(reg, h.ctx);
|
||||
expect(reg.names()).toEqual([
|
||||
"enrichPoolEntry",
|
||||
"ingestFeed",
|
||||
"prewarmMetadata",
|
||||
"refreshMetadata",
|
||||
"refreshSiteAccount",
|
||||
"resolveObservation",
|
||||
|
||||
Reference in new issue
Block a user