feat(cache): TMDB / 豆瓣 / Bangumi 接入统一元数据缓存与策略表
- 策略表「来源 × 资源类型」(metaCachePolicy):搜索 / 识别 7 天(空 1 天)、详情 / 季 / 剧集组 30 天(空 1 天)、 榜单 / 热门 / 每日放送 6 小时(空 1 小时)、人物 / 图片 90 天;取值来自 storage.cacheTtl, 追加 tmdbListHours / tmdbListEmptyHours / imageDays / metaDetailDays / metaMissDays,metaSearchDays 缺省改 7 天 - TMDB 客户端共享缓存端口加 fetch(带请求类别与空结果判定),服务端整条读取交给 MetadataCache; 识别缓存、douban.request / bangumi.request、douban.subject / bangumi.subject、meta.subject_miss、discover.match 都走这一层 - 豆瓣 / bgm 条目只在「不存在」时记查无,网络失败 / 被拦不缓存(本进程冷却 30 分钟) - 关注中作品(isEntityWatched 口径,含选中的剧集组)的 TMDB 条目写入 pinned、命中续期 - 各进程启动 LISTEN 失效通知并每分钟把命中计数写 cache_stat;GET /cache/regions 给元数据 region 带近 7 天命中统计 Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei
This commit is contained in:
1 parent
ad472a1a19
commit
1dca95f883
16 files changed
+687
-137
No files matched your search
+40
-12
@@ -14,7 +14,16 @@ import { dataPaths } from "../../config.ts";
|
||||
import { externalFetch } from "../../egress.ts";
|
||||
import { requireScope } from "../../http/auth.ts";
|
||||
import { errorBody } from "../../http/errors.ts";
|
||||
import { CACHE_REGIONS, sharedCache, storageSettings, tmdbCachePolicy } from "../../shared-cache.ts";
|
||||
import {
|
||||
CACHE_REGIONS,
|
||||
METADATA_REGIONS,
|
||||
metadataCache,
|
||||
type RegionHitStats,
|
||||
regionHitStats,
|
||||
sharedCache,
|
||||
storageSettings,
|
||||
tmdbCachePolicy,
|
||||
} from "../../shared-cache.ts";
|
||||
import { registerScheduledJob } from "../schedules/registry.ts";
|
||||
import type { ModuleApp, ServerContext, ServerModule } from "../types.ts";
|
||||
import { ImageCache, ImageProxyError } from "./images.ts";
|
||||
@@ -72,6 +81,13 @@ function route<D extends EndpointDef>(
|
||||
|
||||
const str = (v: unknown) => (typeof v === "string" && v ? v : null);
|
||||
|
||||
/** 命中统计 → 接口形状;命中率 = (L1 + L2 + 旧值) / (L1 + L2 + 旧值 + 未命中) */
|
||||
function hitView(h: RegionHitStats | undefined) {
|
||||
const v = h ?? { l1Hits: 0, l2Hits: 0, staleHits: 0, misses: 0, failures: 0, refreshes: 0 };
|
||||
const hit = v.l1Hits + v.l2Hits + v.staleHits;
|
||||
return { ...v, hitRate: hit + v.misses > 0 ? hit / (hit + v.misses) : null };
|
||||
}
|
||||
|
||||
export function createCacheModule(opts: CacheModuleOptions = {}): ServerModule & {
|
||||
images(ctx: ServerContext): ImageCache;
|
||||
} {
|
||||
@@ -166,7 +182,7 @@ export function createCacheModule(opts: CacheModuleOptions = {}): ServerModule &
|
||||
return out;
|
||||
});
|
||||
route(app, Api.cache.tmdbClear, async () => {
|
||||
const cache = sharedCache(ctx);
|
||||
const cache = metadataCache(ctx);
|
||||
const deleted =
|
||||
(await cache.clear(CACHE_REGIONS.tmdbRecognize)) + (await cache.clear(CACHE_REGIONS.tmdbRequest));
|
||||
ctx.logger.info({ deleted }, "TMDB 缓存已清空");
|
||||
@@ -194,18 +210,30 @@ export function createCacheModule(opts: CacheModuleOptions = {}): ServerModule &
|
||||
coalesce(sum(pg_column_size(value) + pg_column_size(key)), 0)::text as bytes,
|
||||
max(updated_at) as last
|
||||
from kv_cache group by region order by region`.execute(ctx.db);
|
||||
return {
|
||||
items: r.rows.map((x) => ({
|
||||
region: x.region,
|
||||
entries: Number(x.entries),
|
||||
expired: Number(x.expired),
|
||||
bytes: Number(x.bytes),
|
||||
lastWriteAt: x.last ? new Date(x.last).toISOString() : null,
|
||||
})),
|
||||
// 统一元数据缓存的 region 带最近 7 天命中统计(口径见 contracts CacheHitStats)
|
||||
const hits = await regionHitStats(ctx).catch(() => new Map<string, RegionHitStats>());
|
||||
const withHits = (region: string) => {
|
||||
const h = hits.get(region);
|
||||
if (!h) return METADATA_REGIONS.includes(region) ? { hits: hitView(undefined) } : {};
|
||||
return { hits: hitView(h) };
|
||||
};
|
||||
const items = r.rows.map((x) => ({
|
||||
region: x.region,
|
||||
entries: Number(x.entries),
|
||||
expired: Number(x.expired),
|
||||
bytes: Number(x.bytes),
|
||||
lastWriteAt: x.last ? new Date(x.last).toISOString() : null,
|
||||
...withHits(x.region),
|
||||
}));
|
||||
// 条目已清空、但最近有读取统计的元数据 region 也列出来
|
||||
for (const region of hits.keys())
|
||||
if (!items.some((i) => i.region === region))
|
||||
items.push({ region, entries: 0, expired: 0, bytes: 0, lastWriteAt: null, ...withHits(region) });
|
||||
items.sort((a, b) => a.region.localeCompare(b.region));
|
||||
return { items };
|
||||
});
|
||||
route(app, Api.cache.regionClear, async ({ params }) => {
|
||||
const deleted = await sharedCache(ctx).clear(params.region);
|
||||
const deleted = await metadataCache(ctx).clear(params.region);
|
||||
ctx.logger.info({ region: params.region, deleted }, "缓存区已清空");
|
||||
return { deleted };
|
||||
});
|
||||
@@ -217,7 +245,7 @@ export function createCacheModule(opts: CacheModuleOptions = {}): ServerModule &
|
||||
lane: "maintenance",
|
||||
defaults: {
|
||||
name: "缓存清理",
|
||||
description: "删除过期的 TMDB 与识别缓存",
|
||||
description: "删除过期的元数据与其它缓存",
|
||||
group: "maintain",
|
||||
schedule: { type: "interval", everyMs: 10 * 60_000 },
|
||||
firstRun: "after_interval",
|
||||
|
||||
@@ -24,10 +24,10 @@ import {
|
||||
type DiscoverTmdbQuery,
|
||||
type InventoryEntry,
|
||||
} from "@umt/contracts";
|
||||
import { pgCacheFor } from "@umt/db";
|
||||
import { type IdentityCandidate, parseRelease, resolveIdentity } from "@umt/domain";
|
||||
import { ApiException } from "../../http/errors.ts";
|
||||
import { runSandbox } from "../../sandbox/client.ts";
|
||||
import { CACHE_REGIONS, metaCachePolicy, metadataCache, storageSettings } from "../../shared-cache.ts";
|
||||
import type { SourcesRuntime } from "../sources-identity/runtime.ts";
|
||||
import { subjectRecall } from "../sources-identity/service/id-escalation.ts";
|
||||
import { metaClients, toCached } from "../sources-identity/service/meta-subjects.ts";
|
||||
@@ -41,9 +41,8 @@ import { localEntities } from "./media.ts";
|
||||
// - 每条结果按 TMDB 身份标本地状态(已订阅 / 媒体库已有 / 下载中);豆瓣 / bgm 条目匹配过 TMDB 的才知道(discover.match)。
|
||||
// 本地状态每次现查,不进缓存。
|
||||
|
||||
/** 豆瓣 / bgm 条目 → TMDB 作品的确定匹配(点开时写入,列表据此标本地状态) */
|
||||
const MATCH_REGION = "discover.match";
|
||||
const MATCH_TTL_SEC = 30 * 86_400;
|
||||
/** 豆瓣 / bgm 条目 → TMDB 作品的确定匹配(点开时写入,列表据此标本地状态);TTL 按「豆瓣 / bgm × 详情」 */
|
||||
const MATCH_REGION = CACHE_REGIONS.discoverMatch;
|
||||
const MATCH_MAX = 20_000;
|
||||
|
||||
export type InventoryLookup = (ids: string[]) => Promise<Map<string, InventoryEntry>>;
|
||||
@@ -108,12 +107,12 @@ const fromTmdb = (t: TmdbListItem): DiscoverItem => ({
|
||||
|
||||
/** 豆瓣 / bgm 条目补上记住的 TMDB 匹配,再按 TMDB 身份查本地作品与库存 */
|
||||
export async function annotate(rt: SourcesRuntime, items: DiscoverItem[], inventory: InventoryLookup) {
|
||||
const cache = pgCacheFor(rt.ctx.db);
|
||||
const cache = metadataCache(rt.ctx);
|
||||
await Promise.all(
|
||||
items
|
||||
.filter((it) => !it.tmdb && it.source !== "tmdb")
|
||||
.map(async (it) => {
|
||||
const hit = await cache.get<DiscoverMatchCandidate>(MATCH_REGION, `${it.source}:${it.id}`).catch(() => null);
|
||||
const hit = await cache.get<DiscoverMatchCandidate>(MATCH_REGION, `${it.source}:${it.id}`);
|
||||
if (hit) it.tmdb = { type: hit.type, id: hit.id };
|
||||
}),
|
||||
);
|
||||
@@ -257,9 +256,9 @@ const view = (m: MetadataCandidate): DiscoverMatchCandidate => ({
|
||||
});
|
||||
|
||||
export async function matchTmdb(rt: SourcesRuntime, req: DiscoverMatchRequest): Promise<DiscoverMatchResult> {
|
||||
const cache = pgCacheFor(rt.ctx.db);
|
||||
const cache = metadataCache(rt.ctx);
|
||||
const memoKey = `${req.source}:${req.id}`;
|
||||
const memo = await cache.get<DiscoverMatchCandidate>(MATCH_REGION, memoKey).catch(() => null);
|
||||
const memo = await cache.get<DiscoverMatchCandidate>(MATCH_REGION, memoKey);
|
||||
if (memo) return { match: memo, candidates: [memo] };
|
||||
const subject = toCached({ ...req, cover: null, rating: null, date: null, episodes: null, url: "" }, rt.now());
|
||||
const base = parseRelease(req.title);
|
||||
@@ -278,6 +277,9 @@ export async function matchTmdb(rt: SourcesRuntime, req: DiscoverMatchRequest):
|
||||
const best = res.status === "auto" && res.best ? byCand.get(res.best.candidate) : undefined;
|
||||
const match = best ? view(best) : null;
|
||||
// 只记确定的匹配;不确定 / 失败不记,下次重算
|
||||
if (match) await cache.set(MATCH_REGION, memoKey, match, { ttl: MATCH_TTL_SEC, maxsize: MATCH_MAX }).catch(() => {});
|
||||
if (match) {
|
||||
const ttl = metaCachePolicy((await storageSettings(rt.ctx)).cacheTtl, req.source, "detail").ttl;
|
||||
await cache.set(MATCH_REGION, memoKey, match, { ttl, maxsize: MATCH_MAX }).catch(() => {});
|
||||
}
|
||||
return DiscoverMatchResult.parse({ match, candidates: ranked.map(view) });
|
||||
}
|
||||
@@ -11,19 +11,13 @@ import {
|
||||
type SiteDefinition,
|
||||
type SiteRateLimiter,
|
||||
type TmdbClient,
|
||||
type TmdbRequestKind,
|
||||
type TmdbSharedCache,
|
||||
} from "@umt/adapters";
|
||||
import { DEFAULT_SETTINGS, type SitesSettings } from "@umt/contracts";
|
||||
import { pgTokenBucketFor, type SiteRow, sql } from "@umt/db";
|
||||
import { egressKey, externalFetch, siteEgress, siteFetchFor } from "../../egress.ts";
|
||||
import {
|
||||
CACHE_REGIONS,
|
||||
renewTrackedTmdbDetail,
|
||||
sharedCache,
|
||||
type TmdbCachePolicy,
|
||||
tmdbCachePolicy,
|
||||
tmdbDetailRef,
|
||||
} from "../../shared-cache.ts";
|
||||
import { CACHE_REGIONS, metadataCache, type TmdbCachePolicy, tmdbCachePolicy, watchedPin } from "../../shared-cache.ts";
|
||||
import type { ServerContext } from "../types.ts";
|
||||
import { CloudflareGate, withClearance } from "./service/cloudflare.ts";
|
||||
import { failureSampleSink } from "./service/failure-sample.ts";
|
||||
@@ -165,6 +159,8 @@ export class SourcesRuntime {
|
||||
ttlMs: policy.ttl * 1000,
|
||||
searchTtlMs: policy.searchTtl * 1000,
|
||||
emptyTtlMs: policy.emptyTtl * 1000,
|
||||
listTtlMs: policy.listTtl * 1000,
|
||||
imageTtlMs: policy.imageTtl * 1000,
|
||||
},
|
||||
sharedCache: tmdbSharedCache(this.ctx, policy),
|
||||
}
|
||||
@@ -420,28 +416,34 @@ export class SourcesRuntime {
|
||||
}
|
||||
|
||||
/**
|
||||
* TMDB 请求响应缓存:PG kv_cache region=tmdb.request;PG 出错按未命中 / 不写处理,不让 TMDB 请求失败。
|
||||
* 详情命中且属于追踪中的作品时续期(CACHE-2),续期失败只记 debug。
|
||||
* TMDB 请求响应缓存:统一元数据缓存 region=tmdb.request(L1 进程内存 + L2 kv_cache,过期先用旧值后台刷新、同请求合并)。
|
||||
* TTL 按请求类别取策略表(搜索 / 详情 / 列表 / 图片);关注中作品的条目写入 pinned、命中续期。
|
||||
* PG 出错按未命中 / 不写处理,不让 TMDB 请求失败。get / set 保留给不走 fetch 的旧调用方。
|
||||
*/
|
||||
function tmdbSharedCache(ctx: ServerContext, policy: TmdbCachePolicy): TmdbSharedCache {
|
||||
const cache = sharedCache(ctx);
|
||||
const warn = (err: unknown) =>
|
||||
ctx.logger.warn({ err: { message: (err as Error).message } }, "TMDB 缓存读写失败,按未命中处理");
|
||||
export function tmdbSharedCache(ctx: ServerContext, policy: TmdbCachePolicy): TmdbSharedCache {
|
||||
const cache = metadataCache(ctx);
|
||||
const region = CACHE_REGIONS.tmdbRequest;
|
||||
const ttlOf = (kind: TmdbRequestKind) =>
|
||||
kind === "search"
|
||||
? { ttl: policy.searchTtl, emptyTtl: policy.emptyTtl }
|
||||
: kind === "list"
|
||||
? { ttl: policy.listTtl, emptyTtl: policy.listEmptyTtl }
|
||||
: kind === "image"
|
||||
? { ttl: policy.imageTtl, emptyTtl: policy.emptyTtl }
|
||||
: { ttl: policy.ttl, emptyTtl: policy.emptyTtl };
|
||||
return {
|
||||
get: async (key) => {
|
||||
const v = await cache.get(CACHE_REGIONS.tmdbRequest, key).catch((e) => {
|
||||
warn(e);
|
||||
return null;
|
||||
});
|
||||
if (v !== null && tmdbDetailRef(key)) {
|
||||
await renewTrackedTmdbDetail(ctx, key, policy.ttl).catch((e) =>
|
||||
ctx.logger.debug({ err: { message: (e as Error).message } }, "TMDB 详情缓存续期失败"),
|
||||
);
|
||||
}
|
||||
return v;
|
||||
},
|
||||
fetch: (req, load) =>
|
||||
cache.fetch(
|
||||
region,
|
||||
req.key,
|
||||
{ ...ttlOf(req.kind), isEmpty: req.isEmpty, maxsize: policy.maxsize, pin: watchedPin(ctx, req.key) },
|
||||
load,
|
||||
),
|
||||
get: (key) => cache.get(region, key),
|
||||
set: (key, value, ttlMs) =>
|
||||
cache.set(CACHE_REGIONS.tmdbRequest, key, value, { ttl: ttlMs / 1000, maxsize: policy.maxsize }).catch(warn),
|
||||
cache
|
||||
.set(region, key, value, { ttl: ttlMs / 1000, maxsize: policy.maxsize })
|
||||
.catch((e) => ctx.logger.warn({ err: { message: (e as Error).message } }, "TMDB 缓存读写失败,按未命中处理")),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { formatDetailPage } from "@umt/adapters";
|
||||
import { pgCacheFor } from "@umt/db";
|
||||
import { DEFAULT_STORAGE_SETTINGS } from "@umt/contracts";
|
||||
import { metaCachePolicy, metadataCache, storageSettings } from "../../../shared-cache.ts";
|
||||
import type { ServerContext } from "../../types.ts";
|
||||
|
||||
// ID-3 豆瓣条目缓存(kv_cache region douban.subject,key = 豆瓣 subject id):只存标题 / 别名 / 年份,供「详情页只有豆瓣 ID」时
|
||||
@@ -10,8 +11,8 @@ import type { ServerContext } from "../../types.ts";
|
||||
// - douban_api:豆瓣移动版接口(识别升级时缓存没有才查)。bgm.tv 条目同形,存 region bangumi.subject。
|
||||
|
||||
export const DOUBAN_SUBJECT_REGION = "douban.subject";
|
||||
const TTL_SEC = 30 * 86_400;
|
||||
const MAX_ENTRIES = 20_000;
|
||||
/** 条目 TTL 取策略表「豆瓣 / bgm × 详情」(storage.cacheTtl.metaDetailDays,缺省 30 天) */
|
||||
export const META_SUBJECT_MAX_ENTRIES = 20_000;
|
||||
|
||||
export interface DoubanSubject {
|
||||
id: string;
|
||||
@@ -92,16 +93,24 @@ export function doubanSubjectFromSnippet(
|
||||
};
|
||||
}
|
||||
|
||||
type Db = Pick<ServerContext, "db">;
|
||||
type Db = Pick<ServerContext, "db"> & Partial<Pick<ServerContext, "settings" | "logger">>;
|
||||
|
||||
/** 写入缓存;站点详情来源优先(已有 site_detail 条目时不用搜索摘要覆盖) */
|
||||
/** 条目 TTL(秒):有设置读设置,否则缺省 */
|
||||
export async function metaSubjectTtl(ctx: Db): Promise<number> {
|
||||
const t = ctx.settings
|
||||
? (await storageSettings({ settings: ctx.settings })).cacheTtl
|
||||
: DEFAULT_STORAGE_SETTINGS.cacheTtl;
|
||||
return metaCachePolicy(t, "douban", "detail").ttl;
|
||||
}
|
||||
|
||||
/** 写入缓存(统一元数据缓存,两级);站点详情来源优先(已有 site_detail 条目时不用搜索摘要覆盖) */
|
||||
export async function saveDoubanSubject(ctx: Db, s: DoubanSubject, region = DOUBAN_SUBJECT_REGION): Promise<void> {
|
||||
const cache = pgCacheFor(ctx.db);
|
||||
const cache = metadataCache(ctx);
|
||||
if (s.source === "web_snippet") {
|
||||
const prev = await cache.get<DoubanSubject>(region, s.id);
|
||||
if (prev?.source === "site_detail") return;
|
||||
}
|
||||
await cache.set(region, s.id, s, { ttl: TTL_SEC, maxsize: MAX_ENTRIES });
|
||||
await cache.set(region, s.id, s, { ttl: await metaSubjectTtl(ctx), maxsize: META_SUBJECT_MAX_ENTRIES });
|
||||
}
|
||||
|
||||
export async function loadDoubanSubject(
|
||||
@@ -110,7 +119,7 @@ export async function loadDoubanSubject(
|
||||
region = DOUBAN_SUBJECT_REGION,
|
||||
): Promise<DoubanSubject | null> {
|
||||
try {
|
||||
const v = await pgCacheFor(ctx.db).get<DoubanSubject>(region, id);
|
||||
const v = await metadataCache(ctx).get<DoubanSubject>(region, id);
|
||||
return v && typeof v.title === "string" && v.title ? v : null;
|
||||
} catch {
|
||||
return null;
|
||||
|
||||
@@ -4,51 +4,56 @@ import {
|
||||
createDoubanClient,
|
||||
type DoubanClient,
|
||||
type MetaRequestCache,
|
||||
MetaSourceError,
|
||||
type MetaSubject,
|
||||
RequestLimiter,
|
||||
} from "@umt/adapters";
|
||||
import { cachedCall, pgCacheFor } from "@umt/db";
|
||||
import { normalizeTitle, toSimplified } from "@umt/domain";
|
||||
import { storageSettings } from "../../../shared-cache.ts";
|
||||
import { CACHE_REGIONS, metaCachePolicy, metadataCache, storageSettings } from "../../../shared-cache.ts";
|
||||
import type { SourcesRuntime } from "../runtime.ts";
|
||||
import { DOUBAN_SUBJECT_REGION, type DoubanSubject, loadDoubanSubject, saveDoubanSubject } from "./douban-subject.ts";
|
||||
import {
|
||||
DOUBAN_SUBJECT_REGION,
|
||||
type DoubanSubject,
|
||||
loadDoubanSubject,
|
||||
META_SUBJECT_MAX_ENTRIES,
|
||||
} from "./douban-subject.ts";
|
||||
|
||||
// 豆瓣 / bgm.tv 作为辅助信息源(对照 MP:豆瓣、Bangumi 条目 → 名称 + 年份 → TMDB 投影;身份仍以 TMDB 为准)。
|
||||
// 条目先查缓存(30 天),没有才在线查一次;查不到 / 网络失败记 6 小时负缓存,不反复打外站。
|
||||
|
||||
export const BANGUMI_SUBJECT_REGION = "bangumi.subject";
|
||||
const MISS_REGION = "meta.subject_miss";
|
||||
const MISS_TTL_SEC = 6 * 3600;
|
||||
export const BANGUMI_SUBJECT_REGION = CACHE_REGIONS.bangumiSubject;
|
||||
const MISS_REGION = CACHE_REGIONS.subjectMiss;
|
||||
/** 网络失败 / 被拦后同一条目在本进程里冷却多久不再在线查(不写缓存,只是不连着打外站) */
|
||||
const FAIL_COOLDOWN_MS = 30 * 60_000;
|
||||
const failedAt = new Map<string, number>();
|
||||
|
||||
export type MetaSource = "douban" | "bangumi";
|
||||
const REGION: Record<MetaSource, string> = { douban: DOUBAN_SUBJECT_REGION, bangumi: BANGUMI_SUBJECT_REGION };
|
||||
|
||||
/** 豆瓣 / bgm 请求缓存 region(对照 MP DoubanApi.__invoke / BangumiApi 的 @cached);key 见 metaRequestKey */
|
||||
export const META_REQUEST_REGIONS = { douban: "douban.request", bangumi: "bangumi.request" } as const;
|
||||
export const META_REQUEST_REGIONS = {
|
||||
douban: CACHE_REGIONS.doubanRequest,
|
||||
bangumi: CACHE_REGIONS.bangumiRequest,
|
||||
} as const;
|
||||
const META_REQUEST_MAXSIZE = 5000;
|
||||
/** 每源限流(进程内):豆瓣风控严,间隔大一些;缓存命中不占额度 */
|
||||
const META_LIMITERS = { douban: new RequestLimiter(2, 300), bangumi: new RequestLimiter(3, 100) };
|
||||
|
||||
/**
|
||||
* 请求缓存:搜索按 storage.cacheTtl.metaSearchDays,列表(榜单 / 热门 / 每日放送 / 趋势页)按 metaListHours,
|
||||
* 空结果按 metaEmptyHours(不超过正常 TTL);失败(网络、403、风控页)在适配器里抛出,cachedCall 不写。
|
||||
* 条目详情不走这里:douban.subject / bangumi.subject 条目级缓存(30 天)已经存了,避免同一份数据存两遍。
|
||||
* 请求缓存(统一元数据缓存):按策略表「来源 × 资源类型」取 TTL——搜索 metaSearchDays / 查无 metaMissDays,
|
||||
* 列表(榜单 / 热门 / 每日放送 / 趋势页)metaListHours / 空 metaEmptyHours;过期先用旧值后台刷新。
|
||||
* 失败(网络、403、风控页)在适配器里抛出,不写。
|
||||
* 条目详情不走这里:douban.subject / bangumi.subject 条目级缓存已经存了,避免同一份数据存两遍。
|
||||
*/
|
||||
export function metaRequestCache(ctx: SourcesRuntime["ctx"]): MetaRequestCache {
|
||||
return {
|
||||
async cached(req, load) {
|
||||
const t = (await storageSettings(ctx)).cacheTtl;
|
||||
const ttl = req.kind === "search" ? t.metaSearchDays * 86_400 : t.metaListHours * 3600;
|
||||
return cachedCall(
|
||||
pgCacheFor(ctx.db),
|
||||
const p = metaCachePolicy(t, req.source, req.kind);
|
||||
return metadataCache(ctx).fetch(
|
||||
META_REQUEST_REGIONS[req.source],
|
||||
req.key,
|
||||
{
|
||||
region: META_REQUEST_REGIONS[req.source],
|
||||
ttl,
|
||||
emptyTtl: Math.min(ttl, t.metaEmptyHours * 3600),
|
||||
emptyIf: req.empty,
|
||||
maxsize: META_REQUEST_MAXSIZE,
|
||||
},
|
||||
{ ...p, isEmpty: req.empty, maxsize: META_REQUEST_MAXSIZE },
|
||||
load,
|
||||
);
|
||||
},
|
||||
@@ -77,23 +82,38 @@ export function toCached(s: MetaSubject, now: Date): DoubanSubject {
|
||||
};
|
||||
}
|
||||
|
||||
/** 条目(缓存 → 在线);拿不到为 null */
|
||||
/**
|
||||
* 条目(缓存 → 在线,统一元数据缓存:过期先用旧值后台刷新);拿不到为 null。
|
||||
* 条目不存在记查无(metaMissDays);网络失败 / 被拦不缓存,同一条目本进程冷却 30 分钟再查。
|
||||
*/
|
||||
export async function loadSubject(rt: SourcesRuntime, source: MetaSource, id: string): Promise<DoubanSubject | null> {
|
||||
const hit = await loadDoubanSubject(rt.ctx, id, REGION[source]);
|
||||
if (hit) return hit;
|
||||
const clients = metaClients(rt);
|
||||
if (!clients) return null;
|
||||
const cache = pgCacheFor(rt.ctx.db);
|
||||
if (!clients) return loadDoubanSubject(rt.ctx, id, REGION[source]);
|
||||
const cache = metadataCache(rt.ctx);
|
||||
const missKey = `${source}:${id}`;
|
||||
if (await cache.get(MISS_REGION, missKey).catch(() => null)) return null;
|
||||
const t = (await storageSettings(rt.ctx)).cacheTtl;
|
||||
const p = metaCachePolicy(t, source, "detail");
|
||||
try {
|
||||
const s = source === "douban" ? await clients.douban.subject(id) : await clients.bangumi.subject(id);
|
||||
const cached = toCached(s, rt.now());
|
||||
await saveDoubanSubject(rt.ctx, cached, REGION[source]);
|
||||
return cached;
|
||||
const v = await cache.fetch<DoubanSubject | null>(
|
||||
REGION[source],
|
||||
id,
|
||||
{ ttl: p.ttl, maxsize: META_SUBJECT_MAX_ENTRIES, skipIf: (x) => !x?.title },
|
||||
async () => {
|
||||
if ((failedAt.get(missKey) ?? 0) > Date.now() - FAIL_COOLDOWN_MS) return null;
|
||||
if (await cache.get(MISS_REGION, missKey)) return null;
|
||||
const s = source === "douban" ? await clients.douban.subject(id) : await clients.bangumi.subject(id);
|
||||
return toCached(s, rt.now());
|
||||
},
|
||||
);
|
||||
return v && typeof v.title === "string" && v.title ? v : null;
|
||||
} catch (e) {
|
||||
rt.ctx.logger.info({ source, id, err: (e as Error).message }, "辅助信息源条目查询失败");
|
||||
await cache.set(MISS_REGION, missKey, true, { ttl: MISS_TTL_SEC, maxsize: 5000 }).catch(() => {});
|
||||
if (e instanceof MetaSourceError && e.kind === "not_found")
|
||||
await cache.set(MISS_REGION, missKey, true, { ttl: p.emptyTtl, maxsize: 5000 }).catch(() => {});
|
||||
else {
|
||||
if (failedAt.size > 5000) failedAt.clear();
|
||||
failedAt.set(missKey, Date.now());
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { MetadataCandidate } from "@umt/adapters";
|
||||
import type { ParsedRelease } from "@umt/domain";
|
||||
import { CACHE_REGIONS, sharedCache, tmdbCachePolicy } from "../../../shared-cache.ts";
|
||||
import { CACHE_REGIONS, metadataCache, tmdbCachePolicy } from "../../../shared-cache.ts";
|
||||
import type { SourcesRuntime } from "../runtime.ts";
|
||||
|
||||
// TMDB 识别缓存:移植 MP app/modules/themoviedb/cache.py 的 TmdbCache(对照表见 docs/porting/moviepilot-tmdb-cache.md)。
|
||||
@@ -92,7 +92,7 @@ export async function loadRecognizeCache(
|
||||
): Promise<RecognizeCacheEntry | null> {
|
||||
const tmdb = await rt.tmdb();
|
||||
const key = recognizeKey(meta, localeOf(tmdb));
|
||||
const cache = sharedCache(rt.ctx);
|
||||
const cache = metadataCache(rt.ctx);
|
||||
const v = await cache.get<RecognizeCacheEntry>(CACHE_REGIONS.tmdbRecognize, key).catch(() => null);
|
||||
if (!v || typeof v !== "object") return null;
|
||||
if (typeConflicted(meta, v.type, v.id)) {
|
||||
@@ -120,7 +120,7 @@ export async function saveRecognizeCache(
|
||||
const tmdb = await rt.tmdb();
|
||||
const key = recognizeKey(meta, localeOf(tmdb));
|
||||
const policy = await tmdbCachePolicy(rt.ctx);
|
||||
const cache = sharedCache(rt.ctx);
|
||||
const cache = metadataCache(rt.ctx);
|
||||
if (info) {
|
||||
if (typeConflicted(meta, info.entityType, info.externalId)) {
|
||||
rt.ctx.logger.warn({ key, title: info.title, type: info.entityType }, "识别结果类型与元数据冲突,不写入识别缓存");
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { sql } from "@umt/db";
|
||||
import { type RawBuilder, sql } from "@umt/db";
|
||||
import type { ServerContext } from "../../types.ts";
|
||||
import type { Exec } from "./common.ts";
|
||||
|
||||
@@ -17,9 +17,8 @@ import type { Exec } from "./common.ts";
|
||||
const OPEN_SUBSCRIPTION = ["active", "pending"];
|
||||
const DOWNLOAD_DONE = ["imported", "removed"];
|
||||
|
||||
export async function isEntityWatched(exec: Exec, entityId: string): Promise<boolean> {
|
||||
const r = await sql<{ watched: boolean }>`
|
||||
select (
|
||||
/** 作品 e(SQL 片段:作品 id 表达式)是否关注中;isEntityWatched 与 watchedTmdbRefs 共用一个口径 */
|
||||
const watchedExpr = (entityId: RawBuilder<unknown>) => sql<boolean>`(
|
||||
exists (select 1 from subscription s where s.media_entity_id = ${entityId} and s.status = any(${OPEN_SUBSCRIPTION}))
|
||||
or exists (select 1 from media_asset a where a.media_entity_id = ${entityId} and a.status = 'active')
|
||||
or exists (select 1 from external_identity x
|
||||
@@ -27,7 +26,7 @@ export async function isEntityWatched(exec: Exec, entityId: string): Promise<boo
|
||||
or exists (select 1 from source_release r
|
||||
where r.media_entity_id = ${entityId} and r.resolution_evidence->>'lookup' in ('search', 'manual'))
|
||||
or exists (select 1 from acquisition_plan p left join acquisition_slot sl on sl.id = p.slot_id
|
||||
where (sl.media_entity_id = ${entityId} or p.evaluation->>'mediaEntityId' = ${entityId}::text)
|
||||
where (sl.media_entity_id = ${entityId} or p.evaluation->>'mediaEntityId' = (${entityId})::text)
|
||||
and (p.evaluation->>'manual')::boolean is true)
|
||||
or exists (select 1 from download_task t
|
||||
left join source_release r on r.id = t.source_release_id
|
||||
@@ -37,7 +36,28 @@ export async function isEntityWatched(exec: Exec, entityId: string): Promise<boo
|
||||
and ${entityId} in (t.media_entity_id, r.media_entity_id, sl.media_entity_id))
|
||||
or exists (select 1 from organization_record o
|
||||
where o.media_entity_id = ${entityId} and o.status in ('queued', 'running'))
|
||||
) as watched`.execute(exec);
|
||||
)`;
|
||||
|
||||
/**
|
||||
* 关注中作品的 TMDB 条目集合(统一元数据缓存的 pinned / 续期用):`movie:<id>` / `tv:<id>` 与选中的剧集组 `group:<gid>`。
|
||||
* 口径同 isEntityWatched。
|
||||
*/
|
||||
export async function watchedTmdbRefs(exec: Exec): Promise<Set<string>> {
|
||||
const r = await sql<{ ref: string; grp: string | null }>`
|
||||
select x.entity_type || ':' || x.external_id as ref, e.episode_group_id as grp
|
||||
from external_identity x join media_entity e on e.id = x.media_entity_id
|
||||
where x.provider = 'tmdb' and x.status = 'accepted' and x.entity_type in ('movie', 'tv')
|
||||
and e.status <> 'merged' and ${watchedExpr(sql`e.id`)}`.execute(exec);
|
||||
const out = new Set<string>();
|
||||
for (const x of r.rows) {
|
||||
out.add(x.ref);
|
||||
if (x.grp) out.add(`group:${x.grp}`);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function isEntityWatched(exec: Exec, entityId: string): Promise<boolean> {
|
||||
const r = await sql<{ watched: boolean }>`select ${watchedExpr(sql`${entityId}::uuid`)} as watched`.execute(exec);
|
||||
return r.rows[0]?.watched === true;
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user