feat(cache): read cache TTLs from storage settings and renew tracked TMDB details (CACHE-2)
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
58e358e4a9
commit
85b352b8eb
10 files changed
+140
-35
No files matched your search
@@ -15,7 +15,14 @@ import {
|
||||
} from "@umt/adapters";
|
||||
import { DEFAULT_SETTINGS, type SitesSettings } from "@umt/contracts";
|
||||
import { pgTokenBucketFor, type SiteRow, sql } from "@umt/db";
|
||||
import { CACHE_REGIONS, sharedCache, type TmdbCachePolicy, tmdbCachePolicy } from "../../shared-cache.ts";
|
||||
import {
|
||||
CACHE_REGIONS,
|
||||
renewTrackedTmdbDetail,
|
||||
sharedCache,
|
||||
type TmdbCachePolicy,
|
||||
tmdbCachePolicy,
|
||||
tmdbDetailRef,
|
||||
} from "../../shared-cache.ts";
|
||||
import type { ServerContext } from "../types.ts";
|
||||
|
||||
// 模块运行时依赖:站点适配器与限流器(每站一个实例,令牌在 PG 跨进程共享)、RSS 抓取器、TMDB 客户端、时钟。
|
||||
@@ -108,7 +115,11 @@ export class SourcesRuntime {
|
||||
...(baseUrl ? { baseUrl } : {}),
|
||||
...(this.overrides.fetch ? { fetch: this.overrides.fetch } : {}),
|
||||
// 请求响应进 PG 缓存(region tmdb.request),三进程共享;TTL / 条数上限 / 空结果短 TTL 取设置(对照 MP)
|
||||
cache: { ttlMs: policy.ttl * 1000, emptyTtlMs: policy.emptyTtl * 1000 },
|
||||
cache: {
|
||||
ttlMs: policy.ttl * 1000,
|
||||
searchTtlMs: policy.searchTtl * 1000,
|
||||
emptyTtlMs: policy.emptyTtl * 1000,
|
||||
},
|
||||
sharedCache: tmdbSharedCache(this.ctx, policy),
|
||||
})
|
||||
: null;
|
||||
@@ -276,17 +287,27 @@ export class SourcesRuntime {
|
||||
}
|
||||
}
|
||||
|
||||
/** TMDB 请求响应缓存:PG kv_cache region=tmdb.request;PG 出错按未命中 / 不写处理,不让 TMDB 请求失败 */
|
||||
/**
|
||||
* TMDB 请求响应缓存:PG kv_cache region=tmdb.request;PG 出错按未命中 / 不写处理,不让 TMDB 请求失败。
|
||||
* 详情命中且属于追踪中的作品时续期(CACHE-2),续期失败只记 debug。
|
||||
*/
|
||||
function tmdbSharedCache(ctx: ServerContext, policy: TmdbCachePolicy): TmdbSharedCache {
|
||||
const cache = sharedCache(ctx);
|
||||
const warn = (err: unknown) =>
|
||||
ctx.logger.warn({ err: { message: (err as Error).message } }, "TMDB 缓存读写失败，按未命中处理");
|
||||
return {
|
||||
get: (key) =>
|
||||
cache.get(CACHE_REGIONS.tmdbRequest, key).catch((e) => {
|
||||
get: async (key) => {
|
||||
const v = await cache.get(CACHE_REGIONS.tmdbRequest, key).catch((e) => {
|
||||
warn(e);
|
||||
return null;
|
||||
}),
|
||||
});
|
||||
if (v !== null && tmdbDetailRef(key)) {
|
||||
await renewTrackedTmdbDetail(ctx, key, policy.ttl).catch((e) =>
|
||||
ctx.logger.debug({ err: { message: (e as Error).message } }, "TMDB 详情缓存续期失败"),
|
||||
);
|
||||
}
|
||||
return v;
|
||||
},
|
||||
set: (key, value, ttlMs) =>
|
||||
cache.set(CACHE_REGIONS.tmdbRequest, key, value, { ttl: ttlMs / 1000, maxsize: policy.maxsize }).catch(warn),
|
||||
};
|
||||
|
||||
@@ -1,18 +1,18 @@
|
||||
import type { SiteDetailPage } from "@umt/adapters";
|
||||
import type { SiteReleasePage } from "@umt/contracts";
|
||||
import { sql } from "@umt/db";
|
||||
import { sharedCache } from "../../../shared-cache.ts";
|
||||
import { sharedCache, storageSettings } from "../../../shared-cache.ts";
|
||||
import type { SourcesRuntime } from "../runtime.ts";
|
||||
import type { Exec } from "./common.ts";
|
||||
|
||||
// 站点详情页扩展内容(简介 / MediaInfo / 截图 / 外部 ID / 文件 / 标签 / 上传者):
|
||||
// - 只在按需抓详情时得到(工作台打开详情、release.inspect 复核),放 PG kv_cache region site.detail,
|
||||
// key = source_release.id,TTL = 站点详情缓存期(至少 24 小时);不进 source_release 列,过期即丢。
|
||||
// key = source_release.id,TTL = storage.cacheTtl.siteDetailDays(CACHE-2,缺省 7 天;不短于站点详情缓存期);
|
||||
// 不进 source_release 列,过期即丢。促销 / H&R 不靠它:下载前复核走 refreshReleaseDetail(按站点详情缓存期判断是否重抓)。
|
||||
// - 外部 ID(IMDb / 豆瓣 / TMDB)另写 source_release.raw._detail(列表快照 upsert 不覆盖它),供身份解析用。
|
||||
// - 截图对外只给代理路径 /api/v1/img/site-detail/{releaseId}/{index},原始地址留在服务端。
|
||||
|
||||
export const DETAIL_PAGE_REGION = "site.detail";
|
||||
const MIN_TTL_SEC = 86_400;
|
||||
const MAX_ENTRIES = 5_000;
|
||||
|
||||
interface StoredPage extends SiteDetailPage {
|
||||
@@ -33,8 +33,9 @@ export async function saveDetailPage(
|
||||
cacheSec: number,
|
||||
): Promise<void> {
|
||||
const stored: StoredPage = { ...page, fetchedAt: rt.now().toISOString() };
|
||||
const { siteDetailDays } = (await storageSettings(rt.ctx)).cacheTtl;
|
||||
await sharedCache(rt.ctx)
|
||||
.set(DETAIL_PAGE_REGION, srId, stored, { ttl: Math.max(cacheSec, MIN_TTL_SEC), maxsize: MAX_ENTRIES })
|
||||
.set(DETAIL_PAGE_REGION, srId, stored, { ttl: Math.max(cacheSec, siteDetailDays * 86_400), maxsize: MAX_ENTRIES })
|
||||
.catch((e) => rt.ctx.logger.warn({ err: { message: (e as Error).message } }, "详情页缓存写入失败"));
|
||||
const ids: DetailIds = {
|
||||
...(page.imdbId ? { imdbId: page.imdbId } : {}),
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { DecisionOption, PoolCandidate } from "@umt/contracts";
|
||||
import { sql } from "@umt/db";
|
||||
import { storageSettings } from "../../../shared-cache.ts";
|
||||
import type { ServerContext } from "../../types.ts";
|
||||
import { jsonb } from "../rows.ts";
|
||||
import type { SourcesRuntime } from "../runtime.ts";
|
||||
@@ -10,7 +11,7 @@ import { closeDecisions, type Exec, openDecision } from "./common.ts";
|
||||
// - 同一来源 + 同一归一化标题合并为一条事项(dedupeKey identity-group:<siteId>:<normalized>),成员 = 池里
|
||||
// decision_id 指向它且仍 pending 的条目;事项的 subject 与候选选项取自代表资源(最近一次进来的那条);
|
||||
// - 成员被识别 / 指定 / 忽略时从组里摘除,组空了才关闭事项;
|
||||
// - unmatched 池条目超过 identity.poolArchiveDays 没有新证据就归档。
|
||||
// - unmatched 池条目超过 storage.retentionDays.identityPoolArchive 天没有新证据就归档。
|
||||
|
||||
const OPEN = ["open", "investigating", "awaiting_user"] as const;
|
||||
/** 事项证据里最多列多少条成员资源(完整列表在待识别池按标题检索) */
|
||||
@@ -208,14 +209,15 @@ const ARCHIVE_EVERY_MS = 3600_000;
|
||||
let lastArchiveAt = 0;
|
||||
|
||||
/**
|
||||
* 过期归档:unmatched 池条目超过 identity.poolArchiveDays 没有新证据 → archived(仍可按状态筛出、可手动指定;
|
||||
* 过期归档:unmatched 池条目超过 storage.retentionDays.identityPoolArchive 天没有新证据 → archived(仍可按状态筛出、可手动指定;
|
||||
* 同一资源再被观察到或新增相近别名时回到 pending)。每进程每小时最多执行一次。
|
||||
*/
|
||||
export async function archiveStalePool(rt: SourcesRuntime, opts: { force?: boolean } = {}): Promise<number> {
|
||||
const now = rt.now();
|
||||
if (!opts.force && now.getTime() - lastArchiveAt < ARCHIVE_EVERY_MS) return 0;
|
||||
lastArchiveAt = now.getTime();
|
||||
const { poolArchiveDays } = await rt.ctx.settings.get("identity");
|
||||
// 与存储保留任务 storage.retention.identity_pool 同一条件(那边在 system 模块里按表直接执行)
|
||||
const poolArchiveDays = (await storageSettings(rt.ctx)).retentionDays.identityPoolArchive;
|
||||
const cutoff = new Date(now.getTime() - poolArchiveDays * 86_400_000);
|
||||
const r = await rt.ctx.db
|
||||
.updateTable("identity_candidate")
|
||||
|
||||
@@ -110,7 +110,8 @@ export async function saveRecognizeCache(
|
||||
poster_path: info.posterPath ?? null,
|
||||
backdrop_path: info.backdropPath ?? null,
|
||||
};
|
||||
await cache.set(CACHE_REGIONS.tmdbRecognize, key, entry, { ttl: policy.ttl, maxsize: policy.maxsize });
|
||||
// 识别结果 = 名称 → TMDB 条目,按搜索类 TTL(storage.cacheTtl.tmdbSearchDays)
|
||||
await cache.set(CACHE_REGIONS.tmdbRecognize, key, entry, { ttl: policy.searchTtl, maxsize: policy.maxsize });
|
||||
return;
|
||||
}
|
||||
// 负识别缓存用独立短 TTL(EMPTY_RESULT_CACHE_TTL):故障期间的空结果不被固化整个有效期
|
||||
|
||||
@@ -17,10 +17,11 @@ import {
|
||||
type WebSearchTestResult,
|
||||
} from "@umt/contracts";
|
||||
import { cachedCall, pgCacheFor, pgTokenBucketFor } from "@umt/db";
|
||||
import { storageSettings } from "../../shared-cache.ts";
|
||||
import type { ServerContext } from "../types.ts";
|
||||
|
||||
/**
|
||||
* WEB-1 网页搜索服务:Brave 客户端之上的 Key 读取、每分钟上限(跨进程令牌桶)、10 分钟 PG 缓存(kv_cache region websearch)、
|
||||
* WEB-1 网页搜索服务:Brave 客户端之上的 Key 读取、每分钟上限(跨进程令牌桶)、PG 缓存(kv_cache region websearch,TTL = storage.cacheTtl.webSearchDays)、
|
||||
* 以及 web.fetch 的白名单——按调用方(Agent 会话 / 其它调用者)记录最近一次 web.search 返回的 URL,只允许读取这些 URL。
|
||||
*/
|
||||
|
||||
@@ -36,8 +37,6 @@ export interface WebSearchModuleOptions {
|
||||
export const WEBSEARCH_CACHE_REGION = "websearch";
|
||||
/** 白名单:key = 调用方,value = 最近一次搜索的 URL */
|
||||
export const WEBSEARCH_ALLOW_REGION = "websearch.allow";
|
||||
/** 搜索结果缓存 10 分钟 */
|
||||
export const SEARCH_CACHE_TTL_SEC = 600;
|
||||
const SEARCH_CACHE_MAX = 1000;
|
||||
/** 白名单保留 6 小时(会话时长上限远小于此) */
|
||||
const ALLOW_TTL_SEC = 6 * 3600;
|
||||
@@ -142,7 +141,7 @@ export class WebSearchService {
|
||||
key,
|
||||
{
|
||||
region: WEBSEARCH_CACHE_REGION,
|
||||
ttl: SEARCH_CACHE_TTL_SEC,
|
||||
ttl: (await storageSettings(this.ctx)).cacheTtl.webSearchDays * 86_400,
|
||||
maxsize: SEARCH_CACHE_MAX,
|
||||
...(o.fresh ? { fresh: true } : {}),
|
||||
},
|
||||
|
||||
Reference in new issue
Block a user