Merge branch 'cache2'
This commit is contained in:
root committed
commit
eb9cfaa63d
32 files changed
+1927
-87
No files matched your search
@@ -1,5 +1,11 @@
|
||||
import { createCipheriv, createDecipheriv, createHmac, randomBytes, randomUUID } from "node:crypto";
|
||||
import { DEFAULT_SETTINGS, GlobalSettings, type SecretPurpose, type SettingsKey } from "@umt/contracts";
|
||||
import {
|
||||
DEFAULT_SETTINGS,
|
||||
GlobalSettings,
|
||||
type SecretPurpose,
|
||||
type SettingsKey,
|
||||
withStorageDefaults,
|
||||
} from "@umt/contracts";
|
||||
import type { DB, Kysely } from "@umt/db";
|
||||
import type { Logger } from "pino";
|
||||
import type { EventBusPort } from "../../http/sse.ts";
|
||||
@@ -335,7 +341,13 @@ export function createSettingsReader(deps: {
|
||||
for (const key of Object.keys(DEFAULT_SETTINGS) as SettingsKey[]) {
|
||||
const def = DEFAULT_SETTINGS[key];
|
||||
const v = stored.get(key);
|
||||
const merged = v && typeof v === "object" && !Array.isArray(v) ? { ...def, ...(v as object) } : def;
|
||||
// storage 是两层结构:逐层补缺省,旧库里缺子字段也能整体通过校验
|
||||
const merged =
|
||||
key === "storage"
|
||||
? withStorageDefaults(v)
|
||||
: v && typeof v === "object" && !Array.isArray(v)
|
||||
? { ...def, ...(v as object) }
|
||||
: def;
|
||||
const parsed = GlobalSettings.shape[key].safeParse(merged);
|
||||
if (!parsed.success) deps.logger.warn({ key }, "settings 值不合法，回退缺省");
|
||||
out[key] = parsed.success ? parsed.data : def;
|
||||
|
||||
+51
-4
@@ -9,6 +9,8 @@ import { TMDB_IMAGE_SIZES } from "@umt/contracts";
|
||||
// - 读缓存只在 use_cache(GLOBAL_IMAGE_CACHE)开启时;拉到的合法图片总是写盘(与 MP 一致);
|
||||
// - 合法图片按内容识别 MIME,拒绝非图片与 SVG;ETag = md5(内容),Cache-Control: public, max-age=7 天,If-None-Match 相同回 304;
|
||||
// - 清理:删除 mtime 早于 GLOBAL_IMAGE_CACHE_DAYS 天的文件与空目录(MP clear_temp → SystemUtils.clear)。
|
||||
// - CACHE-2 容量 LRU(我们加的):读盘命中时把 atime 显式写成当前时间(不依赖挂载的 relatime),
|
||||
// 超过 storage.budgets.imagesGb 时按 atime 从旧到新删到预算的 90%;按天清理保留作兜底。
|
||||
// 我们只代理 TMDB 图片域(MP SECURITY_IMAGE_DOMAINS 的 TMDB 项),上游失败回 502。
|
||||
|
||||
export const TMDB_IMAGE_HOST = "image.tmdb.org";
|
||||
@@ -51,6 +53,13 @@ export function imageMimeType(b: Uint8Array): string | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
interface ImageFile {
|
||||
path: string;
|
||||
size: number;
|
||||
mtimeMs: number;
|
||||
atimeMs: number;
|
||||
}
|
||||
|
||||
export interface ImageCacheOptions {
|
||||
/** DATA_DIR/cache/images */
|
||||
dir: string;
|
||||
@@ -104,7 +113,10 @@ export class ImageCache {
|
||||
if (useCache) {
|
||||
const cached = await fs.readFile(path).catch(() => null);
|
||||
const mime = cached?.length ? imageMimeType(cached) : null;
|
||||
if (cached && mime) return { content: cached, mime };
|
||||
if (cached && mime) {
|
||||
void this.touch(path);
|
||||
return { content: cached, mime };
|
||||
}
|
||||
}
|
||||
let res: Response;
|
||||
try {
|
||||
@@ -130,8 +142,18 @@ export class ImageCache {
|
||||
return { content, mime };
|
||||
}
|
||||
|
||||
private async files(): Promise<{ path: string; size: number; mtimeMs: number }[]> {
|
||||
const out: { path: string; size: number; mtimeMs: number }[] = [];
|
||||
/** 记一次访问:atime = 现在,mtime 不变(按天清理仍按写入时间) */
|
||||
private async touch(path: string): Promise<void> {
|
||||
try {
|
||||
const st = await fs.stat(path);
|
||||
await fs.utimes(path, new Date(), st.mtime);
|
||||
} catch {
|
||||
// 文件刚被清理等:忽略
|
||||
}
|
||||
}
|
||||
|
||||
private async files(): Promise<ImageFile[]> {
|
||||
const out: ImageFile[] = [];
|
||||
const walk = async (d: string) => {
|
||||
let names: string[];
|
||||
try {
|
||||
@@ -144,7 +166,7 @@ export class ImageCache {
|
||||
const st = await fs.stat(p).catch(() => null);
|
||||
if (!st) continue;
|
||||
if (st.isDirectory()) await walk(p);
|
||||
else out.push({ path: p, size: st.size, mtimeMs: st.mtimeMs });
|
||||
else if (!n.endsWith(".tmp")) out.push({ path: p, size: st.size, mtimeMs: st.mtimeMs, atimeMs: st.atimeMs });
|
||||
}
|
||||
};
|
||||
await walk(this.opts.dir);
|
||||
@@ -156,6 +178,31 @@ export class ImageCache {
|
||||
return { bytes: f.reduce((a, x) => a + x.size, 0), files: f.length };
|
||||
}
|
||||
|
||||
/**
|
||||
* 容量 LRU:总量超过 maxBytes 时按最近访问(atime)从旧到新删除,直到不超过 maxBytes × lowWater;
|
||||
* 没超不动。返回删除数、释放字节与清理后用量。
|
||||
*/
|
||||
async enforceBudget(
|
||||
maxBytes: number,
|
||||
lowWater = 0.9,
|
||||
): Promise<{ deleted: number; freedBytes: number; usedBytes: number }> {
|
||||
const files = await this.files();
|
||||
let used = files.reduce((a, f) => a + f.size, 0);
|
||||
if (used <= maxBytes) return { deleted: 0, freedBytes: 0, usedBytes: used };
|
||||
const target = Math.floor(maxBytes * lowWater);
|
||||
files.sort((a, b) => a.atimeMs - b.atimeMs || a.mtimeMs - b.mtimeMs);
|
||||
let deleted = 0;
|
||||
let freed = 0;
|
||||
for (const f of files) {
|
||||
if (used <= target) break;
|
||||
await fs.rm(f.path, { force: true });
|
||||
used -= f.size;
|
||||
freed += f.size;
|
||||
deleted++;
|
||||
}
|
||||
return { deleted, freedBytes: freed, usedBytes: used };
|
||||
}
|
||||
|
||||
/** SystemUtils.clear(images, days):删 mtime 早于 days 天的文件,再删空目录;返回删除数 */
|
||||
async clean(days: number, now = Date.now()): Promise<number> {
|
||||
let deleted = 0;
|
||||
|
||||
+23
-2
@@ -12,7 +12,7 @@ import type { z } from "zod";
|
||||
import { dataPaths } from "../../config.ts";
|
||||
import { requireScope } from "../../http/auth.ts";
|
||||
import { errorBody } from "../../http/errors.ts";
|
||||
import { CACHE_REGIONS, sharedCache, tmdbCachePolicy } from "../../shared-cache.ts";
|
||||
import { CACHE_REGIONS, sharedCache, storageSettings, tmdbCachePolicy } from "../../shared-cache.ts";
|
||||
import { registerScheduledJob } from "../schedules/registry.ts";
|
||||
import type { ModuleApp, ServerContext, ServerModule } from "../types.ts";
|
||||
import { ImageCache, ImageProxyError } from "./images.ts";
|
||||
@@ -22,7 +22,8 @@ import { ImageCache, ImageProxyError } from "./images.ts";
|
||||
* - PG 缓存 kv_cache(UNLOGGED,移植 MP TTLCache)的过期行物理清理任务 cache.sweep(每 10 分钟);
|
||||
* - TMDB 缓存管理(对照 MP tmdb_cache_items / tmdb_cache_clear):GET /cache/tmdb、POST /cache/tmdb/clear;
|
||||
* - 图片代理 GET /img/tmdb/{size}/{file}(会话或 API Key read,移植 MP ImageHelper / fetch_image)+ DATA_DIR/cache/images 落盘,
|
||||
* 用量 / 清空 GET /cache/images、POST /cache/images/clear,按保留天数清理的任务 cache.images_clean(每天)。
|
||||
* 用量 / 清空 GET /cache/images、POST /cache/images/clear,按保留天数清理的任务 cache.images_clean(每天);
|
||||
* - CACHE-2:图片缓存容量 LRU 任务 cache.images_lru(30 分钟,预算 storage.budgets.imagesGb,超出删最久未访问)。
|
||||
*/
|
||||
export interface CacheModuleOptions {
|
||||
/** 图片上游 fetch(测试注入) */
|
||||
@@ -203,6 +204,26 @@ export function createCacheModule(opts: CacheModuleOptions = {}): ServerModule &
|
||||
return { summary: `删除 ${n} 张过期图片` };
|
||||
},
|
||||
});
|
||||
registerScheduledJob(ctx, {
|
||||
key: "cache.images_lru",
|
||||
defaults: {
|
||||
name: "图片缓存容量控制",
|
||||
description: "图片代理缓存超过容量预算（存储设置，缺省 20 GB）时，按最久未访问删除到预算的 90%",
|
||||
group: "maintain",
|
||||
schedule: { type: "interval", everyMs: 30 * 60_000 },
|
||||
firstRun: "after_interval",
|
||||
},
|
||||
run: async () => {
|
||||
const { imagesGb } = (await storageSettings(ctx)).budgets;
|
||||
const r = await images(ctx).enforceBudget(Math.round(imagesGb * 1024 ** 3));
|
||||
const gb = (b: number) => `${Math.round((b / 1024 ** 3) * 100) / 100} GB`;
|
||||
return {
|
||||
summary: r.deleted
|
||||
? `删除 ${r.deleted} 张最久未访问的图片，释放 ${gb(r.freedBytes)}，现用 ${gb(r.usedBytes)}`
|
||||
: `现用 ${gb(r.usedBytes)}，未超过预算 ${imagesGb} GB`,
|
||||
};
|
||||
},
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -15,7 +15,14 @@ import {
|
||||
} from "@umt/adapters";
|
||||
import { DEFAULT_SETTINGS, type SitesSettings } from "@umt/contracts";
|
||||
import { pgTokenBucketFor, type SiteRow, sql } from "@umt/db";
|
||||
import { CACHE_REGIONS, sharedCache, type TmdbCachePolicy, tmdbCachePolicy } from "../../shared-cache.ts";
|
||||
import {
|
||||
CACHE_REGIONS,
|
||||
renewTrackedTmdbDetail,
|
||||
sharedCache,
|
||||
type TmdbCachePolicy,
|
||||
tmdbCachePolicy,
|
||||
tmdbDetailRef,
|
||||
} from "../../shared-cache.ts";
|
||||
import type { ServerContext } from "../types.ts";
|
||||
|
||||
// 模块运行时依赖:站点适配器与限流器(每站一个实例,令牌在 PG 跨进程共享)、RSS 抓取器、TMDB 客户端、时钟。
|
||||
@@ -108,7 +115,11 @@ export class SourcesRuntime {
|
||||
...(baseUrl ? { baseUrl } : {}),
|
||||
...(this.overrides.fetch ? { fetch: this.overrides.fetch } : {}),
|
||||
// 请求响应进 PG 缓存(region tmdb.request),三进程共享;TTL / 条数上限 / 空结果短 TTL 取设置(对照 MP)
|
||||
cache: { ttlMs: policy.ttl * 1000, emptyTtlMs: policy.emptyTtl * 1000 },
|
||||
cache: {
|
||||
ttlMs: policy.ttl * 1000,
|
||||
searchTtlMs: policy.searchTtl * 1000,
|
||||
emptyTtlMs: policy.emptyTtl * 1000,
|
||||
},
|
||||
sharedCache: tmdbSharedCache(this.ctx, policy),
|
||||
})
|
||||
: null;
|
||||
@@ -276,17 +287,27 @@ export class SourcesRuntime {
|
||||
}
|
||||
}
|
||||
|
||||
/** TMDB 请求响应缓存:PG kv_cache region=tmdb.request;PG 出错按未命中 / 不写处理,不让 TMDB 请求失败 */
|
||||
/**
|
||||
* TMDB 请求响应缓存:PG kv_cache region=tmdb.request;PG 出错按未命中 / 不写处理,不让 TMDB 请求失败。
|
||||
* 详情命中且属于追踪中的作品时续期(CACHE-2),续期失败只记 debug。
|
||||
*/
|
||||
function tmdbSharedCache(ctx: ServerContext, policy: TmdbCachePolicy): TmdbSharedCache {
|
||||
const cache = sharedCache(ctx);
|
||||
const warn = (err: unknown) =>
|
||||
ctx.logger.warn({ err: { message: (err as Error).message } }, "TMDB 缓存读写失败，按未命中处理");
|
||||
return {
|
||||
get: (key) =>
|
||||
cache.get(CACHE_REGIONS.tmdbRequest, key).catch((e) => {
|
||||
get: async (key) => {
|
||||
const v = await cache.get(CACHE_REGIONS.tmdbRequest, key).catch((e) => {
|
||||
warn(e);
|
||||
return null;
|
||||
}),
|
||||
});
|
||||
if (v !== null && tmdbDetailRef(key)) {
|
||||
await renewTrackedTmdbDetail(ctx, key, policy.ttl).catch((e) =>
|
||||
ctx.logger.debug({ err: { message: (e as Error).message } }, "TMDB 详情缓存续期失败"),
|
||||
);
|
||||
}
|
||||
return v;
|
||||
},
|
||||
set: (key, value, ttlMs) =>
|
||||
cache.set(CACHE_REGIONS.tmdbRequest, key, value, { ttl: ttlMs / 1000, maxsize: policy.maxsize }).catch(warn),
|
||||
};
|
||||
|
||||
@@ -1,18 +1,18 @@
|
||||
import type { SiteDetailPage } from "@umt/adapters";
|
||||
import type { SiteReleasePage } from "@umt/contracts";
|
||||
import { sql } from "@umt/db";
|
||||
import { sharedCache } from "../../../shared-cache.ts";
|
||||
import { sharedCache, storageSettings } from "../../../shared-cache.ts";
|
||||
import type { SourcesRuntime } from "../runtime.ts";
|
||||
import type { Exec } from "./common.ts";
|
||||
|
||||
// 站点详情页扩展内容(简介 / MediaInfo / 截图 / 外部 ID / 文件 / 标签 / 上传者):
|
||||
// - 只在按需抓详情时得到(工作台打开详情、release.inspect 复核),放 PG kv_cache region site.detail,
|
||||
// key = source_release.id,TTL = 站点详情缓存期(至少 24 小时);不进 source_release 列,过期即丢。
|
||||
// key = source_release.id,TTL = storage.cacheTtl.siteDetailDays(CACHE-2,缺省 7 天;不短于站点详情缓存期);
|
||||
// 不进 source_release 列,过期即丢。促销 / H&R 不靠它:下载前复核走 refreshReleaseDetail(按站点详情缓存期判断是否重抓)。
|
||||
// - 外部 ID(IMDb / 豆瓣 / TMDB)另写 source_release.raw._detail(列表快照 upsert 不覆盖它),供身份解析用。
|
||||
// - 截图对外只给代理路径 /api/v1/img/site-detail/{releaseId}/{index},原始地址留在服务端。
|
||||
|
||||
export const DETAIL_PAGE_REGION = "site.detail";
|
||||
const MIN_TTL_SEC = 86_400;
|
||||
const MAX_ENTRIES = 5_000;
|
||||
|
||||
interface StoredPage extends SiteDetailPage {
|
||||
@@ -33,8 +33,9 @@ export async function saveDetailPage(
|
||||
cacheSec: number,
|
||||
): Promise<void> {
|
||||
const stored: StoredPage = { ...page, fetchedAt: rt.now().toISOString() };
|
||||
const { siteDetailDays } = (await storageSettings(rt.ctx)).cacheTtl;
|
||||
await sharedCache(rt.ctx)
|
||||
.set(DETAIL_PAGE_REGION, srId, stored, { ttl: Math.max(cacheSec, MIN_TTL_SEC), maxsize: MAX_ENTRIES })
|
||||
.set(DETAIL_PAGE_REGION, srId, stored, { ttl: Math.max(cacheSec, siteDetailDays * 86_400), maxsize: MAX_ENTRIES })
|
||||
.catch((e) => rt.ctx.logger.warn({ err: { message: (e as Error).message } }, "详情页缓存写入失败"));
|
||||
const ids: DetailIds = {
|
||||
...(page.imdbId ? { imdbId: page.imdbId } : {}),
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { DecisionOption, PoolCandidate } from "@umt/contracts";
|
||||
import { sql } from "@umt/db";
|
||||
import { storageSettings } from "../../../shared-cache.ts";
|
||||
import type { ServerContext } from "../../types.ts";
|
||||
import { jsonb } from "../rows.ts";
|
||||
import type { SourcesRuntime } from "../runtime.ts";
|
||||
@@ -10,7 +11,7 @@ import { closeDecisions, type Exec, openDecision } from "./common.ts";
|
||||
// - 同一来源 + 同一归一化标题合并为一条事项(dedupeKey identity-group:<siteId>:<normalized>),成员 = 池里
|
||||
// decision_id 指向它且仍 pending 的条目;事项的 subject 与候选选项取自代表资源(最近一次进来的那条);
|
||||
// - 成员被识别 / 指定 / 忽略时从组里摘除,组空了才关闭事项;
|
||||
// - unmatched 池条目超过 identity.poolArchiveDays 没有新证据就归档。
|
||||
// - unmatched 池条目超过 storage.retentionDays.identityPoolArchive 天没有新证据就归档。
|
||||
|
||||
const OPEN = ["open", "investigating", "awaiting_user"] as const;
|
||||
/** 事项证据里最多列多少条成员资源(完整列表在待识别池按标题检索) */
|
||||
@@ -208,14 +209,15 @@ const ARCHIVE_EVERY_MS = 3600_000;
|
||||
let lastArchiveAt = 0;
|
||||
|
||||
/**
|
||||
* 过期归档:unmatched 池条目超过 identity.poolArchiveDays 没有新证据 → archived(仍可按状态筛出、可手动指定;
|
||||
* 过期归档:unmatched 池条目超过 storage.retentionDays.identityPoolArchive 天没有新证据 → archived(仍可按状态筛出、可手动指定;
|
||||
* 同一资源再被观察到或新增相近别名时回到 pending)。每进程每小时最多执行一次。
|
||||
*/
|
||||
export async function archiveStalePool(rt: SourcesRuntime, opts: { force?: boolean } = {}): Promise<number> {
|
||||
const now = rt.now();
|
||||
if (!opts.force && now.getTime() - lastArchiveAt < ARCHIVE_EVERY_MS) return 0;
|
||||
lastArchiveAt = now.getTime();
|
||||
const { poolArchiveDays } = await rt.ctx.settings.get("identity");
|
||||
// 与存储保留任务 storage.retention.identity_pool 同一条件(那边在 system 模块里按表直接执行)
|
||||
const poolArchiveDays = (await storageSettings(rt.ctx)).retentionDays.identityPoolArchive;
|
||||
const cutoff = new Date(now.getTime() - poolArchiveDays * 86_400_000);
|
||||
const r = await rt.ctx.db
|
||||
.updateTable("identity_candidate")
|
||||
|
||||
@@ -110,7 +110,8 @@ export async function saveRecognizeCache(
|
||||
poster_path: info.posterPath ?? null,
|
||||
backdrop_path: info.backdropPath ?? null,
|
||||
};
|
||||
await cache.set(CACHE_REGIONS.tmdbRecognize, key, entry, { ttl: policy.ttl, maxsize: policy.maxsize });
|
||||
// 识别结果 = 名称 → TMDB 条目,按搜索类 TTL(storage.cacheTtl.tmdbSearchDays)
|
||||
await cache.set(CACHE_REGIONS.tmdbRecognize, key, entry, { ttl: policy.searchTtl, maxsize: policy.maxsize });
|
||||
return;
|
||||
}
|
||||
// 负识别缓存用独立短 TTL(EMPTY_RESULT_CACHE_TTL):故障期间的空结果不被固化整个有效期
|
||||
|
||||
@@ -28,6 +28,8 @@ export interface HealthDeps {
|
||||
checkRoot?: typeof checkRoot;
|
||||
/** 单个路径 IO 检查超时(ms),缺省 3000 */
|
||||
ioTimeoutMs?: number;
|
||||
/** 存储段(CACHE-2);不给时该段为 unknown */
|
||||
storage?: () => Promise<{ level: HealthLevel; alerts: string[] }>;
|
||||
}
|
||||
|
||||
const LEVEL_ORDER: HealthLevel[] = ["ok", "warn", "critical"];
|
||||
@@ -384,7 +386,7 @@ export async function loadSchedules(db: Kysely<DB>, now: Date): Promise<SystemHe
|
||||
|
||||
export async function buildHealth(d: HealthDeps): Promise<SystemHealth> {
|
||||
const now = d.now?.() ?? new Date();
|
||||
const [{ processes, retiredProcesses }, db, paths, engine, schedules] = await Promise.all([
|
||||
const [{ processes, retiredProcesses }, db, paths, engine, schedules, storage] = await Promise.all([
|
||||
loadProcesses(d.db, now).catch(() => ({
|
||||
processes: [] as SystemProcess[],
|
||||
retiredProcesses: [] as SystemProcess[],
|
||||
@@ -408,6 +410,10 @@ export async function buildHealth(d: HealthDeps): Promise<SystemHealth> {
|
||||
failing: [],
|
||||
failedRuns24h: 0,
|
||||
})),
|
||||
(d.storage?.() ?? Promise.resolve({ level: "unknown" as HealthLevel, alerts: [] })).catch(() => ({
|
||||
level: "unknown" as HealthLevel,
|
||||
alerts: [] as string[],
|
||||
})),
|
||||
]);
|
||||
// SSE:存活的 api/all 进程上报的连接数合计;本进程若还没写过心跳,用实时值
|
||||
const selfCounted = processes.some((p) => !p.stale && p.instanceId === d.self.instanceId && p.sseConnections != null);
|
||||
@@ -423,6 +429,7 @@ export async function buildHealth(d: HealthDeps): Promise<SystemHealth> {
|
||||
mountLevel,
|
||||
engine.level,
|
||||
schedules.level,
|
||||
storage.level,
|
||||
]),
|
||||
checkedAt: now.toISOString(),
|
||||
api: {
|
||||
@@ -439,6 +446,7 @@ export async function buildHealth(d: HealthDeps): Promise<SystemHealth> {
|
||||
mounts: paths.mounts,
|
||||
engine,
|
||||
schedules,
|
||||
storage,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -130,11 +130,13 @@ export async function markHeartbeatStopped(db: Kysely<DB>, instanceId: string, a
|
||||
/** 裁剪:超过 24 小时没心跳的进程行、超过 7 天的采样;返回删除行数 */
|
||||
export async function pruneHeartbeats(
|
||||
db: Kysely<DB>,
|
||||
o: { now?: Date; rowRetentionMs?: number; sampleDays?: number } = {},
|
||||
o: { now?: Date; rowRetentionMs?: number; sampleDays?: number; sampleHours?: number } = {},
|
||||
): Promise<{ rows: number; samples: number }> {
|
||||
const now = o.now ?? new Date();
|
||||
const rowCutoff = new Date(now.getTime() - (o.rowRetentionMs ?? HEARTBEAT_ROW_RETENTION_MS));
|
||||
const sampleCutoff = new Date(now.getTime() - (o.sampleDays ?? SAMPLE_RETENTION_DAYS) * 24 * 3600_000);
|
||||
// CACHE-2:采样保留取 storage.retentionDays.heartbeatHours(调用方传 sampleHours);sampleDays 为旧参数
|
||||
const sampleHours = o.sampleHours ?? (o.sampleDays ?? SAMPLE_RETENTION_DAYS) * 24;
|
||||
const sampleCutoff = new Date(now.getTime() - sampleHours * 3600_000);
|
||||
const rows = await db.deleteFrom("process_heartbeat").where("last_beat_at", "<", rowCutoff).executeTakeFirst();
|
||||
const samples = await db.deleteFrom("process_heartbeat_sample").where("at", "<", sampleCutoff).executeTakeFirst();
|
||||
return { rows: Number(rows.numDeletedRows), samples: Number(samples.numDeletedRows) };
|
||||
|
||||
@@ -1,9 +1,19 @@
|
||||
import { createRoute, type RouteConfig } from "@hono/zod-openapi";
|
||||
import { Api, apiResponse, DEFAULT_SETTINGS, type EndpointDef } from "@umt/contracts";
|
||||
import {
|
||||
type Actor,
|
||||
Api,
|
||||
apiResponse,
|
||||
DEFAULT_SETTINGS,
|
||||
type EndpointDef,
|
||||
type StorageCleanupTarget,
|
||||
} from "@umt/contracts";
|
||||
import { sql } from "@umt/db";
|
||||
import type { z } from "zod";
|
||||
import { dataPaths } from "../../config.ts";
|
||||
import { requireScope } from "../../http/auth.ts";
|
||||
import { ApiException } from "../../http/errors.ts";
|
||||
import { sseConnectionCount } from "../../http/sse.ts";
|
||||
import { storageSettings } from "../../shared-cache.ts";
|
||||
import { registerScheduledJob } from "../schedules/registry.ts";
|
||||
import type { ModuleApp, ServerContext, ServerModule } from "../types.ts";
|
||||
import { buildHealth, type HealthDeps, loadHistory } from "./health.ts";
|
||||
@@ -15,6 +25,8 @@ import {
|
||||
safeHostname,
|
||||
startHeartbeatWriter,
|
||||
} from "./heartbeat.ts";
|
||||
import { type BatchOptions, CLEANUP_JOB_IDS, RETENTION_JOBS } from "./retention.ts";
|
||||
import { buildStorage, type StoragePaths, storageHealth } from "./storage.ts";
|
||||
|
||||
export { HEARTBEAT_STALE_MS, HEARTBEAT_WRITE_MS, startHeartbeatWriter } from "./heartbeat.ts";
|
||||
|
||||
@@ -23,7 +35,13 @@ export { HEARTBEAT_STALE_MS, HEARTBEAT_WRITE_MS, startHeartbeatWriter } from "./
|
||||
* - GET /system/health:进程心跳 / 事件循环延迟 / SSE 连接 / DB / 磁盘 / 挂载根 / 引擎 / 定时任务,仅 admin;
|
||||
* - GET /system/health/history?hours=:心跳采样时间序列。
|
||||
* 心跳写入:worker 进程在 worker/index.ts 里起写入器;纯 api 进程(role=api)在注册路由时起(role=all 由 worker 写 kind=all)。
|
||||
* 清理:定时任务 system.heartbeat_cleanup(维护组,1 小时)删 24 小时没心跳的进程行与 7 天前的采样。
|
||||
* 清理:定时任务 system.heartbeat_cleanup(维护组,1 小时)删 24 小时没心跳的进程行与超过
|
||||
* storage.retentionDays.heartbeatHours 的采样。
|
||||
*
|
||||
* CACHE-2 存储(docs/runbook/storage.md):
|
||||
* - GET /system/storage:表体积 top 15、库体积、kv_cache 各 region、图片 / 备份 / 会话与证据对照预算、DATA_DIR 分目录、清理任务;
|
||||
* - POST /system/storage/cleanup/{target}:把对应的维护任务排入立即执行(同 schedules run_now 语义);
|
||||
* - 保留清理任务 storage.retention.* 与每周 storage.analyze(维护组,各自开关与周期在定时任务看板改)。
|
||||
*/
|
||||
export interface SystemModuleOptions {
|
||||
/** 引擎 schema(测试覆盖);缺省 openworkflow */
|
||||
@@ -34,6 +52,12 @@ export interface SystemModuleOptions {
|
||||
apiHeartbeat?: boolean;
|
||||
/** 定时任务 system.heartbeat_cleanup 的默认周期,默认 1 小时;<=0 不登记 */
|
||||
cleanupIntervalMs?: number;
|
||||
/** 存储面板路径(测试覆盖);缺省按 DATA_DIR / AGENT_SESSION_DIR */
|
||||
storagePaths?: StoragePaths;
|
||||
/** false 时不登记保留清理任务(测试);缺省 true */
|
||||
retentionJobs?: boolean;
|
||||
/** 保留清理的分批参数(测试) */
|
||||
retentionBatch?: BatchOptions;
|
||||
}
|
||||
|
||||
type Infer<T> = T extends z.ZodType ? z.infer<T> : Record<string, never>;
|
||||
@@ -41,10 +65,11 @@ type Infer<T> = T extends z.ZodType ? z.infer<T> : Record<string, never>;
|
||||
function route<D extends EndpointDef>(
|
||||
app: ModuleApp,
|
||||
def: D,
|
||||
handler: (i: { query: Infer<D["query"]> }) => Promise<unknown>,
|
||||
handler: (i: { query: Infer<D["query"]>; params: Infer<D["params"]>; actor: Actor | undefined }) => Promise<unknown>,
|
||||
) {
|
||||
const request: RouteConfig["request"] = {};
|
||||
if (def.query) request.query = def.query as never;
|
||||
if (def.params) request.params = def.params as never;
|
||||
const r = createRoute({
|
||||
method: def.method,
|
||||
path: def.path,
|
||||
@@ -58,7 +83,13 @@ function route<D extends EndpointDef>(
|
||||
});
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 路由泛型在封装里擦除,入参已由 zod-openapi 校验
|
||||
app.openapi(r, (async (c: any) => {
|
||||
const data = def.response.parse(await handler({ query: def.query ? c.req.valid("query") : {} } as never));
|
||||
const data = def.response.parse(
|
||||
await handler({
|
||||
query: def.query ? c.req.valid("query") : {},
|
||||
params: def.params ? c.req.valid("param") : {},
|
||||
actor: c.get("actor"),
|
||||
} as never),
|
||||
);
|
||||
return c.json({ ok: true, data }, 200);
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 同上
|
||||
}) as any);
|
||||
@@ -84,6 +115,11 @@ export function createSystemModule(opts: SystemModuleOptions = {}): ServerModule
|
||||
}
|
||||
const selfKind = ctx.role === "all" ? "all" : ctx.role === "api" ? "api" : null;
|
||||
const selfId = selfKind ? instanceIdOf(selfKind, safeHostname(), Deno.pid) : undefined;
|
||||
const paths = (): StoragePaths => {
|
||||
if (opts.storagePaths) return opts.storagePaths;
|
||||
const p = dataPaths(ctx.config);
|
||||
return { dataDir: opts.healthDeps?.dataDir ?? p.dataDir, sessionDir: p.sessionDir };
|
||||
};
|
||||
|
||||
route(app, Api.system.healthDetail, async () => {
|
||||
const cache = await ctx.settings.get("cache").catch(() => DEFAULT_SETTINGS.cache);
|
||||
@@ -98,25 +134,89 @@ export function createSystemModule(opts: SystemModuleOptions = {}): ServerModule
|
||||
...(selfId ? { instanceId: selfId } : {}),
|
||||
},
|
||||
thresholds: { warnPercent: cache.resumePercent, criticalPercent: cache.hardThresholdPercent },
|
||||
storage: async () =>
|
||||
await storageHealth({ db: ctx.db, paths: paths(), settings: await storageSettings(ctx) }),
|
||||
...(opts.healthDeps ?? {}),
|
||||
});
|
||||
});
|
||||
route(app, Api.system.healthHistory, async ({ query }) => await loadHistory(ctx.db, query.hours));
|
||||
route(app, Api.system.storage, async () =>
|
||||
buildStorage({
|
||||
db: ctx.db,
|
||||
paths: paths(),
|
||||
settings: await storageSettings(ctx),
|
||||
engineSchema: opts.engineSchema ?? "openworkflow",
|
||||
}),
|
||||
);
|
||||
route(app, Api.system.storageCleanup, async ({ params, actor }) => {
|
||||
const target = params.target as StorageCleanupTarget;
|
||||
const jobId = CLEANUP_JOB_IDS[target];
|
||||
const job = await ctx.db
|
||||
.selectFrom("scheduled_job")
|
||||
.select(["id", "available"])
|
||||
.where("id", "=", jobId)
|
||||
.executeTakeFirst();
|
||||
if (!job) throw new ApiException("not_found", `清理任务 ${jobId} 还没登记（调度器未同步）`);
|
||||
if (!job.available) throw new ApiException("conflict", "任务未配置，不能执行");
|
||||
// 同 schedules 的立即执行:只置 run_requested_at,已在排队或运行中则合并
|
||||
const r = await sql`update scheduled_job set run_requested_at = now()
|
||||
where id = ${jobId} and run_requested_at is null and running_since is null`.execute(ctx.db);
|
||||
const queued = Number(r.numAffectedRows ?? 0) > 0;
|
||||
if (queued) {
|
||||
await ctx.db
|
||||
.insertInto("audit_event")
|
||||
.values({
|
||||
actor_type: actor?.type === "api_key" ? "api_key" : actor ? "user" : "system",
|
||||
actor_id: actor?.id ?? null,
|
||||
action: "storage.cleanup",
|
||||
target_type: "scheduled_job",
|
||||
target_id: jobId,
|
||||
payload: JSON.stringify({ target }),
|
||||
})
|
||||
.execute()
|
||||
.catch((err) => ctx.logger.warn({ err }, "审计写入失败"));
|
||||
}
|
||||
return { target, jobId, queued };
|
||||
});
|
||||
},
|
||||
registerTasks() {},
|
||||
start(ctx) {
|
||||
if (opts.retentionJobs !== false) {
|
||||
for (const j of RETENTION_JOBS) {
|
||||
registerScheduledJob(ctx, {
|
||||
key: j.jobId,
|
||||
defaults: {
|
||||
name: j.name,
|
||||
description: j.description,
|
||||
group: "maintain",
|
||||
schedule: { type: "interval", everyMs: j.everyMs },
|
||||
firstRun: "after_interval",
|
||||
},
|
||||
run: async ({ now }) => ({
|
||||
summary: await j.run({
|
||||
db: ctx.db,
|
||||
engineSchema: opts.engineSchema ?? "openworkflow",
|
||||
settings: await storageSettings(ctx),
|
||||
now,
|
||||
...(opts.retentionBatch ? { batch: opts.retentionBatch } : {}),
|
||||
}),
|
||||
}),
|
||||
});
|
||||
}
|
||||
}
|
||||
const everyMs = opts.cleanupIntervalMs ?? 3600_000;
|
||||
if (everyMs <= 0) return;
|
||||
registerScheduledJob(ctx, {
|
||||
key: "system.heartbeat_cleanup",
|
||||
defaults: {
|
||||
name: "进程心跳清理",
|
||||
description: "删除 24 小时没有心跳的进程行和 7 天前的心跳采样",
|
||||
description: "删除 24 小时没有心跳的进程行和超过保留期（存储设置，缺省 24 小时）的心跳采样",
|
||||
group: "maintain",
|
||||
schedule: { type: "interval", everyMs },
|
||||
},
|
||||
run: async ({ now }) => {
|
||||
const r = await pruneHeartbeats(ctx.db, { now });
|
||||
const sampleHours = (await storageSettings(ctx)).retentionDays.heartbeatHours;
|
||||
const r = await pruneHeartbeats(ctx.db, { now, sampleHours });
|
||||
return { summary: `删除 ${r.rows} 个进程行、${r.samples} 条采样` };
|
||||
},
|
||||
});
|
||||
|
||||
@@ -0,0 +1,273 @@
|
||||
import type { StorageCleanupTarget, StorageSettings } from "@umt/contracts";
|
||||
import { type DB, type Kysely, type RawBuilder, sql } from "@umt/db";
|
||||
|
||||
// CACHE-2 保留策略清理(docs/runbook/storage.md):按表按时间列分批删除流水数据。
|
||||
// - 每批:单独事务,`set local statement_timeout / lock_timeout` 限时,按 ctid 取一批再删,不做整表长事务;
|
||||
// - 每次运行有总时限,到点就停,剩下的留给下次;批与批之间让出一小段时间给业务写入;
|
||||
// - 只删已结束的数据(任务 / Agent 会话 / 能力调用看状态),进行中的不动。
|
||||
// 待识别池不删:unmatched 条目到期归档(与 sources-identity archiveStalePool 同一条件)。
|
||||
|
||||
export interface BatchOptions {
|
||||
/** 每批行数,缺省 5000 */
|
||||
batchSize?: number;
|
||||
/** 每批语句超时(ms),缺省 15000 */
|
||||
statementTimeoutMs?: number;
|
||||
/** 等锁超时(ms),缺省 3000 */
|
||||
lockTimeoutMs?: number;
|
||||
/** 本次运行总时限(ms),缺省 5 分钟 */
|
||||
budgetMs?: number;
|
||||
/** 批间停顿(ms),缺省 50 */
|
||||
pauseMs?: number;
|
||||
/** 测试注入 */
|
||||
now?: () => number;
|
||||
}
|
||||
|
||||
export interface BatchResult {
|
||||
deleted: number;
|
||||
batches: number;
|
||||
/** 到总时限还没删完 */
|
||||
truncated: boolean;
|
||||
}
|
||||
|
||||
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
||||
|
||||
/**
|
||||
* 分批删除 `table` 中满足 `where` 的行:每批 `delete … where ctid = any(array(select ctid … limit n))`,
|
||||
* 单独事务 + 语句 / 锁超时。某批超时(57014)或等锁超时(55P03)时停止本次运行,不抛错。
|
||||
*/
|
||||
export async function deleteInBatches(
|
||||
db: Kysely<DB>,
|
||||
table: RawBuilder<unknown>,
|
||||
where: RawBuilder<unknown>,
|
||||
o: BatchOptions = {},
|
||||
): Promise<BatchResult> {
|
||||
const size = o.batchSize ?? 5000;
|
||||
const now = o.now ?? Date.now;
|
||||
const deadline = now() + (o.budgetMs ?? 5 * 60_000);
|
||||
let deleted = 0;
|
||||
let batches = 0;
|
||||
for (;;) {
|
||||
let n: number;
|
||||
try {
|
||||
n = await db.transaction().execute(async (trx) => {
|
||||
await sql`select set_config('statement_timeout', ${String(o.statementTimeoutMs ?? 15_000)}, true),
|
||||
set_config('lock_timeout', ${String(o.lockTimeoutMs ?? 3000)}, true)`.execute(trx);
|
||||
const r = await sql`delete from ${table} where ctid = any(array(
|
||||
select ctid from ${table} where ${where} limit ${size}))`.execute(trx);
|
||||
return Number(r.numAffectedRows ?? 0);
|
||||
});
|
||||
} catch (e) {
|
||||
const code = (e as { code?: string }).code;
|
||||
if (code === "57014" || code === "55P03") return { deleted, batches, truncated: true };
|
||||
throw e;
|
||||
}
|
||||
batches++;
|
||||
deleted += n;
|
||||
if (n < size) return { deleted, batches, truncated: false };
|
||||
if (now() >= deadline) return { deleted, batches, truncated: true };
|
||||
if ((o.pauseMs ?? 50) > 0) await sleep(o.pauseMs ?? 50);
|
||||
}
|
||||
}
|
||||
|
||||
export interface RetentionJob {
|
||||
target: StorageCleanupTarget;
|
||||
/** scheduled_job key */
|
||||
jobId: string;
|
||||
name: string;
|
||||
description: string;
|
||||
everyMs: number;
|
||||
/** 保留期说明(存储面板) */
|
||||
retention(s: StorageSettings): string | null;
|
||||
run(c: RetentionRunContext): Promise<string>;
|
||||
}
|
||||
|
||||
export interface RetentionRunContext {
|
||||
db: Kysely<DB>;
|
||||
engineSchema: string;
|
||||
settings: StorageSettings;
|
||||
now: Date;
|
||||
batch?: BatchOptions;
|
||||
}
|
||||
|
||||
const DAY_MS = 86_400_000;
|
||||
const cutoffOf = (now: Date, days: number) => new Date(now.getTime() - days * DAY_MS);
|
||||
const t = (name: string) => sql.raw(`"${name}"`);
|
||||
const summaryOf = (label: string, r: BatchResult) =>
|
||||
`${label}删除 ${r.deleted} 行（${r.batches} 批）${r.truncated ? "，未删完，下次继续" : ""}`;
|
||||
|
||||
const HOUR = 3600_000;
|
||||
|
||||
/**
|
||||
* 保留清理任务(维护组;heartbeat 目标由 system.heartbeat_cleanup 承担,images / kv_cache 由 cache 模块承担)。
|
||||
* 顺序即存储面板的展示顺序。
|
||||
*/
|
||||
export const RETENTION_JOBS: RetentionJob[] = [
|
||||
{
|
||||
target: "observation",
|
||||
jobId: "storage.retention.observation",
|
||||
name: "观察记录清理",
|
||||
description: "删除超过保留期的 Feed 观察记录（observation，按 created_at）",
|
||||
everyMs: 6 * HOUR,
|
||||
retention: (s) => `${s.retentionDays.observation} 天`,
|
||||
async run(c) {
|
||||
const cutoff = cutoffOf(c.now, c.settings.retentionDays.observation);
|
||||
const r = await deleteInBatches(c.db, t("observation"), sql`created_at < ${cutoff}`, c.batch);
|
||||
return summaryOf("观察记录", r);
|
||||
},
|
||||
},
|
||||
{
|
||||
target: "sse_event",
|
||||
jobId: "storage.retention.sse_event",
|
||||
name: "SSE 事件清理",
|
||||
description: "删除超过保留期的 SSE 补读事件（sse_event，按 at；发布方另按条数裁剪）",
|
||||
everyMs: HOUR,
|
||||
retention: (s) => `${s.retentionDays.sseEvent} 天`,
|
||||
async run(c) {
|
||||
const cutoff = cutoffOf(c.now, c.settings.retentionDays.sseEvent);
|
||||
const r = await deleteInBatches(c.db, t("sse_event"), sql`at < ${cutoff}`, c.batch);
|
||||
return summaryOf("SSE 事件", r);
|
||||
},
|
||||
},
|
||||
{
|
||||
target: "task_run",
|
||||
jobId: "storage.retention.task_run",
|
||||
name: "任务运行清理",
|
||||
description: "删除超过保留期、已结束的业务任务（task）与引擎 run（含步骤、运行元数据）",
|
||||
everyMs: 6 * HOUR,
|
||||
retention: (s) => `${s.retentionDays.taskRun} 天`,
|
||||
async run(c) {
|
||||
const cutoff = cutoffOf(c.now, c.settings.retentionDays.taskRun);
|
||||
const e = (n: string) => sql.raw(`"${c.engineSchema}"."${n}"`);
|
||||
const task = await deleteInBatches(
|
||||
c.db,
|
||||
t("task"),
|
||||
sql`created_at < ${cutoff} and status in ('succeeded', 'failed', 'cancelled', 'dead')`,
|
||||
c.batch,
|
||||
);
|
||||
// 引擎 run:step_attempts 随 run 级联删除
|
||||
const runs = await deleteInBatches(
|
||||
c.db,
|
||||
e("workflow_runs"),
|
||||
sql`created_at < ${cutoff} and status in ('completed', 'succeeded', 'failed', 'canceled')`,
|
||||
c.batch,
|
||||
);
|
||||
// 运行元数据与 run 同时写入:早于保留期的 run 已结束(上一步删掉了),元数据按时间直接删
|
||||
const meta = await deleteInBatches(c.db, t("tasks_run_meta"), sql`created_at < ${cutoff}`, c.batch);
|
||||
const facts = await deleteInBatches(c.db, t("tasks_fact"), sql`created_at < ${cutoff}`, c.batch);
|
||||
const ledger = await deleteInBatches(
|
||||
c.db,
|
||||
t("tasks_op_ledger"),
|
||||
sql`status = 'succeeded' and started_at < ${cutoff}`,
|
||||
c.batch,
|
||||
);
|
||||
const truncated = [task, runs, meta, facts, ledger].some((x) => x.truncated);
|
||||
return `业务任务 ${task.deleted}、引擎 run ${runs.deleted}、运行元数据 ${meta.deleted + facts.deleted + ledger.deleted} 行已删除${truncated ? "，未删完，下次继续" : ""}`;
|
||||
},
|
||||
},
|
||||
{
|
||||
target: "agent_event",
|
||||
jobId: "storage.retention.agent_event",
|
||||
name: "Agent 事件清理",
|
||||
description: "删除超过保留期、会话已结束的 Agent 事件流与消息（agent_run_event / agent_run_message；会话本身保留）",
|
||||
everyMs: 6 * HOUR,
|
||||
retention: (s) => `${s.retentionDays.agentEvent} 天`,
|
||||
async run(c) {
|
||||
const cutoff = cutoffOf(c.now, c.settings.retentionDays.agentEvent);
|
||||
const ended = sql`exists (select 1 from agent_run r where r.id = run_id
|
||||
and r.status in ('succeeded', 'failed', 'cancelled', 'budget_exceeded'))`;
|
||||
const ev = await deleteInBatches(c.db, t("agent_run_event"), sql`at < ${cutoff} and ${ended}`, c.batch);
|
||||
const msg = await deleteInBatches(
|
||||
c.db,
|
||||
t("agent_run_message"),
|
||||
sql`created_at < ${cutoff} and status = 'consumed' and ${ended}`,
|
||||
c.batch,
|
||||
);
|
||||
return `Agent 事件 ${ev.deleted}、消息 ${msg.deleted} 行已删除${ev.truncated || msg.truncated ? "，未删完，下次继续" : ""}`;
|
||||
},
|
||||
},
|
||||
{
|
||||
target: "audit",
|
||||
jobId: "storage.retention.audit",
|
||||
name: "审计清理",
|
||||
description: "删除超过保留期的审计事件（audit_event，能力调用审计按能力统计的保留期另算）",
|
||||
everyMs: 24 * HOUR,
|
||||
retention: (s) => `${s.retentionDays.audit} 天`,
|
||||
async run(c) {
|
||||
const cutoff = cutoffOf(c.now, c.settings.retentionDays.audit);
|
||||
const r = await deleteInBatches(
|
||||
c.db,
|
||||
t("audit_event"),
|
||||
sql`at < ${cutoff} and target_type is distinct from 'capability'`,
|
||||
c.batch,
|
||||
);
|
||||
return summaryOf("审计事件", r);
|
||||
},
|
||||
},
|
||||
{
|
||||
target: "capability_stats",
|
||||
jobId: "storage.retention.capability_stats",
|
||||
name: "能力统计清理",
|
||||
description: "删除超过保留期的能力调用审计（audit_event target_type=capability）与已结束的能力调用记录",
|
||||
everyMs: 24 * HOUR,
|
||||
retention: (s) => `${s.retentionDays.capabilityStats} 天`,
|
||||
async run(c) {
|
||||
const cutoff = cutoffOf(c.now, c.settings.retentionDays.capabilityStats);
|
||||
const a = await deleteInBatches(
|
||||
c.db,
|
||||
t("audit_event"),
|
||||
sql`at < ${cutoff} and target_type = 'capability'`,
|
||||
c.batch,
|
||||
);
|
||||
const inv = await deleteInBatches(
|
||||
c.db,
|
||||
t("capability_invocation"),
|
||||
sql`created_at < ${cutoff} and status <> 'running'`,
|
||||
c.batch,
|
||||
);
|
||||
return `能力审计 ${a.deleted}、调用记录 ${inv.deleted} 行已删除${a.truncated || inv.truncated ? "，未删完，下次继续" : ""}`;
|
||||
},
|
||||
},
|
||||
{
|
||||
target: "identity_pool",
|
||||
jobId: "storage.retention.identity_pool",
|
||||
name: "待识别池归档",
|
||||
description: "待识别池里 unmatched 条目超过保留期没有新证据就归档（不删除，仍可按状态筛出）",
|
||||
everyMs: 6 * HOUR,
|
||||
retention: (s) => `${s.retentionDays.identityPoolArchive} 天`,
|
||||
async run(c) {
|
||||
const cutoff = cutoffOf(c.now, c.settings.retentionDays.identityPoolArchive);
|
||||
let archived = 0;
|
||||
const size = c.batch?.batchSize ?? 5000;
|
||||
for (;;) {
|
||||
const r = await sql`update identity_candidate set status = 'archived', next_review_at = null
|
||||
where id in (select id from identity_candidate where status = 'pending' and reason = 'unmatched'
|
||||
and coalesce(last_attempt_at, created_at) < ${cutoff} limit ${size})`.execute(c.db);
|
||||
const n = Number(r.numAffectedRows ?? 0);
|
||||
archived += n;
|
||||
if (n < size) break;
|
||||
}
|
||||
return `归档 ${archived} 条待识别条目`;
|
||||
},
|
||||
},
|
||||
{
|
||||
target: "analyze",
|
||||
jobId: "storage.analyze",
|
||||
name: "统计信息更新",
|
||||
description: "对全库执行 ANALYZE，刷新查询规划用的统计信息（每周）",
|
||||
everyMs: 7 * 24 * HOUR,
|
||||
retention: () => null,
|
||||
async run(c) {
|
||||
const started = Date.now();
|
||||
await sql`analyze`.execute(c.db);
|
||||
return `ANALYZE 完成，用时 ${Math.round((Date.now() - started) / 100) / 10} 秒`;
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
/** 清理目标 → 定时任务 key(heartbeat / images / kv_cache 由其它任务承担) */
|
||||
export const CLEANUP_JOB_IDS: Record<StorageCleanupTarget, string> = {
|
||||
...(Object.fromEntries(RETENTION_JOBS.map((j) => [j.target, j.jobId])) as Record<string, string>),
|
||||
heartbeat: "system.heartbeat_cleanup",
|
||||
images: "cache.images_lru",
|
||||
kv_cache: "cache.sweep",
|
||||
} as Record<StorageCleanupTarget, string>;
|
||||
@@ -0,0 +1,347 @@
|
||||
import { promises as fs } from "node:fs";
|
||||
import { join, relative } from "node:path";
|
||||
import type { HealthLevel, StorageBudgetUsage, StorageSettings, SystemStorage } from "@umt/contracts";
|
||||
import { type DB, type Kysely, sql } from "@umt/db";
|
||||
import { worst } from "./health.ts";
|
||||
import { CLEANUP_JOB_IDS, RETENTION_JOBS } from "./retention.ts";
|
||||
|
||||
// CACHE-2 存储面板(GET /system/storage)与 health 的存储档位。
|
||||
// 目录遍历可能很慢(图片缓存几十万个文件),结果按目录集合缓存 5 分钟;health 只用缓存值,过期时后台刷新,不等。
|
||||
|
||||
const GB = 1024 ** 3;
|
||||
const DIR_TTL_MS = 5 * 60_000;
|
||||
|
||||
export interface StoragePaths {
|
||||
dataDir: string;
|
||||
/** Agent 会话目录(可能在 DATA_DIR 之外) */
|
||||
sessionDir: string;
|
||||
/** 图片缓存目录;缺省 DATA_DIR/cache/images */
|
||||
imagesDir?: string;
|
||||
}
|
||||
|
||||
export interface DirUsage {
|
||||
name: string;
|
||||
path: string;
|
||||
usedBytes: number;
|
||||
files: number;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
interface DirScan {
|
||||
at: Date;
|
||||
dirs: DirUsage[];
|
||||
images: DirUsage;
|
||||
sessions: DirUsage;
|
||||
evidence: DirUsage;
|
||||
backups: DirUsage & { count: number };
|
||||
}
|
||||
|
||||
async function walk(path: string): Promise<{ bytes: number; files: number; error: string | null }> {
|
||||
let bytes = 0;
|
||||
let files = 0;
|
||||
const go = async (d: string, top: boolean): Promise<string | null> => {
|
||||
let names: string[];
|
||||
try {
|
||||
names = await fs.readdir(d);
|
||||
} catch (e) {
|
||||
const code = (e as { code?: string }).code;
|
||||
// 目录不存在按 0 计
|
||||
return top && code !== "ENOENT" ? (code ?? (e as Error).message) : null;
|
||||
}
|
||||
for (const n of names) {
|
||||
const p = join(d, n);
|
||||
const st = await fs.lstat(p).catch(() => null);
|
||||
if (!st) continue;
|
||||
if (st.isDirectory()) await go(p, false);
|
||||
else {
|
||||
bytes += st.size;
|
||||
files++;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
};
|
||||
const error = await go(path, true);
|
||||
return { bytes, files, error };
|
||||
}
|
||||
|
||||
const usageOf = async (name: string, path: string): Promise<DirUsage> => {
|
||||
const w = await walk(path);
|
||||
return { name, path, usedBytes: w.bytes, files: w.files, error: w.error };
|
||||
};
|
||||
|
||||
/** DATA_DIR 一级目录(cache / tmp 再细分一层)+ 会话目录 + 备份份数 */
|
||||
export async function scanDirs(p: StoragePaths): Promise<DirScan> {
|
||||
const dirs: DirUsage[] = [];
|
||||
const top = await fs.readdir(p.dataDir, { withFileTypes: true }).catch(() => []);
|
||||
for (const e of top.sort((a, b) => a.name.localeCompare(b.name))) {
|
||||
if (!e.isDirectory()) continue;
|
||||
const path = join(p.dataDir, e.name);
|
||||
if (e.name === "cache" || e.name === "tmp") {
|
||||
const sub = await fs.readdir(path, { withFileTypes: true }).catch(() => []);
|
||||
for (const s of sub.sort((a, b) => a.name.localeCompare(b.name))) {
|
||||
if (s.isDirectory()) dirs.push(await usageOf(`${e.name}/${s.name}`, join(path, s.name)));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
dirs.push(await usageOf(e.name, path));
|
||||
}
|
||||
const find = async (path: string, name: string) => dirs.find((d) => d.path === path) ?? (await usageOf(name, path));
|
||||
const imagesDir = p.imagesDir ?? join(p.dataDir, "cache", "images");
|
||||
const backupsDir = join(p.dataDir, "backups");
|
||||
const backupNames = await fs.readdir(backupsDir, { withFileTypes: true }).catch(() => []);
|
||||
const sessionsInData = !relative(p.dataDir, p.sessionDir).startsWith("..");
|
||||
const sessions = await find(p.sessionDir, "agent-sessions");
|
||||
if (!sessionsInData) dirs.push(sessions);
|
||||
return {
|
||||
at: new Date(),
|
||||
dirs,
|
||||
images: await find(imagesDir, "cache/images"),
|
||||
sessions,
|
||||
evidence: await find(join(p.dataDir, "tmp", "evidence"), "tmp/evidence"),
|
||||
backups: {
|
||||
...(await find(backupsDir, "backups")),
|
||||
count: backupNames.filter((b) => b.isDirectory() && !b.name.startsWith(".")).length,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const scans = new Map<string, { at: number; value: Promise<DirScan>; last: DirScan | null }>();
|
||||
|
||||
/**
|
||||
* 目录用量(缓存 5 分钟)。wait=false 时不等遍历:有旧值返回旧值(过期则后台刷新),没有返回 null。
|
||||
*/
|
||||
export async function dirScan(p: StoragePaths, o: { wait: boolean; force?: boolean }): Promise<DirScan | null> {
|
||||
const key = `${p.dataDir}\u0000${p.sessionDir}\u0000${p.imagesDir ?? ""}`;
|
||||
const cur = scans.get(key);
|
||||
if (!cur || o.force || Date.now() - cur.at > DIR_TTL_MS) {
|
||||
const value = scanDirs(p);
|
||||
const entry = { at: Date.now(), value, last: cur?.last ?? null };
|
||||
scans.set(key, entry);
|
||||
value.then(
|
||||
(v) => {
|
||||
entry.last = v;
|
||||
},
|
||||
() => {
|
||||
if (scans.get(key) === entry) scans.delete(key);
|
||||
},
|
||||
);
|
||||
if (o.wait) return await value;
|
||||
return entry.last;
|
||||
}
|
||||
if (o.wait) return await cur.value;
|
||||
return cur.last;
|
||||
}
|
||||
|
||||
/** 测试用 */
|
||||
export function resetDirScans(): void {
|
||||
scans.clear();
|
||||
}
|
||||
|
||||
/** 用量对照预算:≥ budget critical,≥ warn warn */
|
||||
export function budgetLevel(used: number | null, budget: number | null, warn: number | null): HealthLevel {
|
||||
if (used === null) return "unknown";
|
||||
if (budget !== null && used >= budget) return "critical";
|
||||
if (warn !== null && used >= warn) return "warn";
|
||||
return "ok";
|
||||
}
|
||||
|
||||
const gb = (v: number) => Math.round(v * GB);
|
||||
|
||||
function usage(used: number | null, budgetGb: number, warnGb: number, error: string | null = null): StorageBudgetUsage {
|
||||
const budgetBytes = gb(budgetGb);
|
||||
const warnBytes = gb(warnGb);
|
||||
return {
|
||||
usedBytes: used,
|
||||
budgetBytes,
|
||||
warnBytes,
|
||||
level: error ? "unknown" : budgetLevel(used, budgetBytes, warnBytes),
|
||||
error,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 图片缓存有容量 LRU(到预算就按最久未访问删),接近预算是常态:超过预算 10% 才告警(说明 LRU 没跟上),不给 critical。
|
||||
*/
|
||||
function imagesUsage(used: number | null, budgetGb: number, error: string | null): StorageBudgetUsage {
|
||||
const budgetBytes = gb(budgetGb);
|
||||
const warnBytes = Math.round(budgetBytes * 1.1);
|
||||
return {
|
||||
usedBytes: used,
|
||||
budgetBytes,
|
||||
warnBytes,
|
||||
level: error ? "unknown" : used === null ? "unknown" : used >= warnBytes ? "warn" : "ok",
|
||||
error,
|
||||
};
|
||||
}
|
||||
|
||||
const fmtGb = (b: number | null) => (b === null ? "?" : `${Math.round((b / GB) * 10) / 10} GB`);
|
||||
|
||||
export interface StorageDeps {
|
||||
db: Kysely<DB>;
|
||||
paths: StoragePaths;
|
||||
settings: StorageSettings;
|
||||
}
|
||||
|
||||
async function dbSize(db: Kysely<DB>): Promise<number> {
|
||||
const r = await sql<{ n: string }>`select pg_database_size(current_database())::text as n`.execute(db);
|
||||
return Number(r.rows[0]?.n ?? 0);
|
||||
}
|
||||
|
||||
/** 汇总档位 + 告警文案(health 与存储面板共用) */
|
||||
function levelsOf(
|
||||
s: StorageSettings,
|
||||
dbBytes: number | null,
|
||||
scan: DirScan | null,
|
||||
): {
|
||||
db: StorageBudgetUsage;
|
||||
images: StorageBudgetUsage;
|
||||
backups: StorageBudgetUsage;
|
||||
sessionsEvidence: StorageBudgetUsage;
|
||||
level: HealthLevel;
|
||||
alerts: string[];
|
||||
} {
|
||||
const b = s.budgets;
|
||||
const db = usage(dbBytes, b.pgDataGb, b.pgDataWarnGb, dbBytes === null ? "db_unavailable" : null);
|
||||
const images = imagesUsage(scan ? scan.images.usedBytes : null, b.imagesGb, scan?.images.error ?? null);
|
||||
const backups = usage(
|
||||
scan ? scan.backups.usedBytes : null,
|
||||
b.backupsGb,
|
||||
b.backupsGb * 0.8,
|
||||
scan?.backups.error ?? null,
|
||||
);
|
||||
const se = scan ? scan.sessions.usedBytes + scan.evidence.usedBytes : null;
|
||||
const sessionsEvidence = usage(
|
||||
se,
|
||||
b.sessionsEvidenceGb,
|
||||
b.sessionsEvidenceGb * 0.8,
|
||||
scan?.sessions.error ?? scan?.evidence.error ?? null,
|
||||
);
|
||||
const alerts: string[] = [];
|
||||
const say = (label: string, u: StorageBudgetUsage) => {
|
||||
if (u.level === "warn" || u.level === "critical") {
|
||||
alerts.push(
|
||||
`${label} ${fmtGb(u.usedBytes)}，${u.level === "critical" ? "已超过预算" : "超过告警线"} ${fmtGb(u.level === "critical" ? u.budgetBytes : u.warnBytes)}`,
|
||||
);
|
||||
}
|
||||
};
|
||||
say("数据库", db);
|
||||
say("图片缓存", images);
|
||||
say("备份", backups);
|
||||
say("会话与证据", sessionsEvidence);
|
||||
return {
|
||||
db,
|
||||
images,
|
||||
backups,
|
||||
sessionsEvidence,
|
||||
level: worst([db.level, images.level, backups.level, sessionsEvidence.level]),
|
||||
alerts,
|
||||
};
|
||||
}
|
||||
|
||||
/** health 的存储段:库体积实时取,目录用量只用缓存(不等遍历) */
|
||||
export async function storageHealth(d: StorageDeps): Promise<{ level: HealthLevel; alerts: string[] }> {
|
||||
const [bytes, scan] = await Promise.all([
|
||||
dbSize(d.db).catch(() => null),
|
||||
dirScan(d.paths, { wait: false }).catch(() => null),
|
||||
]);
|
||||
const l = levelsOf(d.settings, bytes, scan);
|
||||
return { level: l.level, alerts: l.alerts };
|
||||
}
|
||||
|
||||
export async function buildStorage(
|
||||
d: StorageDeps & { engineSchema: string; refresh?: boolean },
|
||||
): Promise<SystemStorage> {
|
||||
const [bytes, scan, tables, kv, jobs] = await Promise.all([
|
||||
dbSize(d.db).catch(() => null),
|
||||
dirScan(d.paths, { wait: true, force: d.refresh ?? false }).catch(() => null),
|
||||
sql<{
|
||||
schema: string;
|
||||
name: string;
|
||||
total: string;
|
||||
tbl: string;
|
||||
idx: string;
|
||||
rows: number | null;
|
||||
dead: number | null;
|
||||
vac: Date | null;
|
||||
ana: Date | null;
|
||||
}>`select n.nspname as schema, c.relname as name,
|
||||
pg_total_relation_size(c.oid)::text as total, pg_relation_size(c.oid)::text as tbl,
|
||||
pg_indexes_size(c.oid)::text as idx,
|
||||
case when c.reltuples < 0 then null else c.reltuples::bigint end as rows,
|
||||
s.n_dead_tup as dead, s.last_autovacuum as vac,
|
||||
greatest(s.last_analyze, s.last_autoanalyze) as ana
|
||||
from pg_class c join pg_namespace n on n.oid = c.relnamespace
|
||||
left join pg_stat_all_tables s on s.relid = c.oid
|
||||
where c.relkind in ('r', 'm', 'p') and n.nspname not in ('pg_catalog', 'information_schema')
|
||||
and n.nspname not like 'pg_toast%'
|
||||
order by pg_total_relation_size(c.oid) desc limit 15`.execute(d.db),
|
||||
sql<{ region: string; entries: number; expired: number; bytes: string }>`
|
||||
select region, count(*)::int as entries, count(*) filter (where expires_at <= now())::int as expired,
|
||||
coalesce(sum(pg_column_size(value) + pg_column_size(key)), 0)::text as bytes
|
||||
from kv_cache group by region order by region`
|
||||
.execute(d.db)
|
||||
.catch(() => ({ rows: [] })),
|
||||
d.db
|
||||
.selectFrom("scheduled_job")
|
||||
.select(["id", "enabled", "last_run_at", "last_status", "next_run_at"])
|
||||
.where("id", "in", Object.values(CLEANUP_JOB_IDS))
|
||||
.execute()
|
||||
.catch(() => []),
|
||||
]);
|
||||
const lastSummaries = new Map<string, string | null>();
|
||||
if (jobs.length) {
|
||||
const r = await sql<{ job_id: string; summary: string | null; error: string | null }>`
|
||||
select distinct on (job_id) job_id, summary, error from scheduled_job_run
|
||||
where job_id in (${sql.join(jobs.map((j) => j.id))}) and status <> 'running'
|
||||
order by job_id, started_at desc`
|
||||
.execute(d.db)
|
||||
.catch(() => ({ rows: [] as { job_id: string; summary: string | null; error: string | null }[] }));
|
||||
for (const x of r.rows) lastSummaries.set(x.job_id, x.summary ?? x.error);
|
||||
}
|
||||
const l = levelsOf(d.settings, bytes, scan);
|
||||
const s = d.settings;
|
||||
const retentionOf: Record<string, string | null> = {
|
||||
...Object.fromEntries(RETENTION_JOBS.map((j) => [j.target, j.retention(s)])),
|
||||
heartbeat: `${s.retentionDays.heartbeatHours} 小时`,
|
||||
images: `容量 ${s.budgets.imagesGb} GB`,
|
||||
kv_cache: null,
|
||||
};
|
||||
const iso = (v: Date | string | null) => (v ? new Date(v).toISOString() : null);
|
||||
return {
|
||||
level: l.level,
|
||||
checkedAt: new Date().toISOString(),
|
||||
dirsScannedAt: scan ? scan.at.toISOString() : null,
|
||||
db: {
|
||||
...l.db,
|
||||
tables: tables.rows.map((t) => ({
|
||||
schema: t.schema,
|
||||
name: t.name,
|
||||
totalBytes: Number(t.total),
|
||||
tableBytes: Number(t.tbl),
|
||||
indexBytes: Number(t.idx),
|
||||
rowsEstimate: t.rows === null ? null : Number(t.rows),
|
||||
deadRows: t.dead === null ? null : Number(t.dead),
|
||||
lastAutovacuumAt: iso(t.vac),
|
||||
lastAnalyzeAt: iso(t.ana),
|
||||
})),
|
||||
},
|
||||
kvCache: kv.rows.map((r) => ({ region: r.region, entries: r.entries, expired: r.expired, bytes: Number(r.bytes) })),
|
||||
images: { ...l.images, files: scan ? scan.images.files : null },
|
||||
backups: { ...l.backups, count: scan ? scan.backups.count : null },
|
||||
sessionsEvidence: l.sessionsEvidence,
|
||||
dataDir: scan?.dirs ?? [],
|
||||
cleanup: (Object.entries(CLEANUP_JOB_IDS) as [keyof typeof CLEANUP_JOB_IDS, string][]).map(([target, jobId]) => {
|
||||
const j = jobs.find((x) => x.id === jobId);
|
||||
return {
|
||||
target,
|
||||
jobId,
|
||||
retention: retentionOf[target] ?? null,
|
||||
enabled: j ? j.enabled : null,
|
||||
lastRunAt: iso(j?.last_run_at ?? null),
|
||||
lastStatus: j?.last_status ?? null,
|
||||
lastSummary: lastSummaries.get(jobId) ?? null,
|
||||
nextRunAt: iso(j?.next_run_at ?? null),
|
||||
};
|
||||
}),
|
||||
};
|
||||
}
|
||||
@@ -17,10 +17,11 @@ import {
|
||||
type WebSearchTestResult,
|
||||
} from "@umt/contracts";
|
||||
import { cachedCall, pgCacheFor, pgTokenBucketFor } from "@umt/db";
|
||||
import { storageSettings } from "../../shared-cache.ts";
|
||||
import type { ServerContext } from "../types.ts";
|
||||
|
||||
/**
|
||||
* WEB-1 网页搜索服务:Brave 客户端之上的 Key 读取、每分钟上限(跨进程令牌桶)、10 分钟 PG 缓存(kv_cache region websearch)、
|
||||
* WEB-1 网页搜索服务:Brave 客户端之上的 Key 读取、每分钟上限(跨进程令牌桶)、PG 缓存(kv_cache region websearch,TTL = storage.cacheTtl.webSearchDays)、
|
||||
* 以及 web.fetch 的白名单——按调用方(Agent 会话 / 其它调用者)记录最近一次 web.search 返回的 URL,只允许读取这些 URL。
|
||||
*/
|
||||
|
||||
@@ -36,8 +37,6 @@ export interface WebSearchModuleOptions {
|
||||
export const WEBSEARCH_CACHE_REGION = "websearch";
|
||||
/** 白名单:key = 调用方,value = 最近一次搜索的 URL */
|
||||
export const WEBSEARCH_ALLOW_REGION = "websearch.allow";
|
||||
/** 搜索结果缓存 10 分钟 */
|
||||
export const SEARCH_CACHE_TTL_SEC = 600;
|
||||
const SEARCH_CACHE_MAX = 1000;
|
||||
/** 白名单保留 6 小时(会话时长上限远小于此) */
|
||||
const ALLOW_TTL_SEC = 6 * 3600;
|
||||
@@ -142,7 +141,7 @@ export class WebSearchService {
|
||||
key,
|
||||
{
|
||||
region: WEBSEARCH_CACHE_REGION,
|
||||
ttl: SEARCH_CACHE_TTL_SEC,
|
||||
ttl: (await storageSettings(this.ctx)).cacheTtl.webSearchDays * 86_400,
|
||||
maxsize: SEARCH_CACHE_MAX,
|
||||
...(o.fresh ? { fresh: true } : {}),
|
||||
},
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
import { DEFAULT_SETTINGS } from "@umt/contracts";
|
||||
import { type PgCache, pgCacheFor } from "@umt/db";
|
||||
import { DEFAULT_SETTINGS, type StorageSettings, withStorageDefaults } from "@umt/contracts";
|
||||
import { type PgCache, pgCacheFor, sql } from "@umt/db";
|
||||
import type { ServerContext } from "./modules/types.ts";
|
||||
|
||||
// 进程级共享缓存入口:PG kv_cache(移植 MP app/runtime/cache.py 的 TTLCache,存储换成 PG,三进程共享)。
|
||||
// region 与 TTL 规则见 docs/runbook/cache.md 与 docs/porting/moviepilot-tmdb-cache.md。
|
||||
// region 与 TTL 规则见 docs/runbook/cache.md 与 docs/porting/moviepilot-tmdb-cache.md;
|
||||
// CACHE-2 起各 region 的 TTL 取值来源是 settings.storage.cacheTtl(docs/runbook/storage.md),语义不变。
|
||||
|
||||
export const CACHE_REGIONS = {
|
||||
/** TMDB 请求响应(MP TMDb.request 的 @cached):key = path?query */
|
||||
@@ -12,14 +13,27 @@ export const CACHE_REGIONS = {
|
||||
tmdbRecognize: "tmdb.recognize",
|
||||
} as const;
|
||||
|
||||
const DAY = 86_400;
|
||||
|
||||
export function sharedCache(ctx: Pick<ServerContext, "db">): PgCache {
|
||||
return pgCacheFor(ctx.db);
|
||||
}
|
||||
|
||||
/** TMDB 缓存策略(MP:CONF.meta = META_CACHE_EXPIRE 小时、CONF.tmdb 条数、EMPTY_RESULT_CACHE_TTL 秒) */
|
||||
/** settings.storage(读失败用缺省) */
|
||||
export async function storageSettings(ctx: Pick<ServerContext, "settings">): Promise<StorageSettings> {
|
||||
return withStorageDefaults(await ctx.settings.get("storage").catch(() => undefined));
|
||||
}
|
||||
|
||||
/**
|
||||
* TMDB 缓存策略(MP:CONF.meta = META_CACHE_EXPIRE、CONF.tmdb 条数、EMPTY_RESULT_CACHE_TTL)。
|
||||
* CACHE-2:MP 只有一个 CONF.meta,这里按请求类别取 storage.cacheTtl:搜索 / find 与识别结果用 searchTtl,
|
||||
* 详情(/movie/{id}、/tv/{id}、季)用 ttl(= tmdbDetailDays),空结果与未识别负缓存用 emptyTtl。
|
||||
*/
|
||||
export interface TmdbCachePolicy {
|
||||
/** 秒 */
|
||||
/** 详情 TTL(秒) */
|
||||
ttl: number;
|
||||
/** 搜索 / find / 识别结果 TTL(秒) */
|
||||
searchTtl: number;
|
||||
maxsize: number;
|
||||
/** 秒 */
|
||||
emptyTtl: number;
|
||||
@@ -27,10 +41,61 @@ export interface TmdbCachePolicy {
|
||||
|
||||
export async function tmdbCachePolicy(ctx: Pick<ServerContext, "settings">): Promise<TmdbCachePolicy> {
|
||||
const c = await ctx.settings.get("cache").catch(() => DEFAULT_SETTINGS.cache);
|
||||
const d = DEFAULT_SETTINGS.cache;
|
||||
const t = (await storageSettings(ctx)).cacheTtl;
|
||||
return {
|
||||
ttl: (c.tmdbTtlHours ?? d.tmdbTtlHours) * 3600,
|
||||
maxsize: c.tmdbMaxSize ?? d.tmdbMaxSize,
|
||||
emptyTtl: c.emptyResultTtlSec ?? d.emptyResultTtlSec,
|
||||
ttl: t.tmdbDetailDays * DAY,
|
||||
searchTtl: t.tmdbSearchDays * DAY,
|
||||
maxsize: c.tmdbMaxSize ?? DEFAULT_SETTINGS.cache.tmdbMaxSize,
|
||||
emptyTtl: t.tmdbMissDays * DAY,
|
||||
};
|
||||
}
|
||||
|
||||
/** TMDB 请求键是否是搜索类(按 searchTtl 缓存) */
|
||||
export function isTmdbSearchPath(key: string): boolean {
|
||||
return /^\/(search|find)\//.test(key);
|
||||
}
|
||||
|
||||
/** 详情键 → `movie:<id>` / `tv:<id>`;不是详情返回 null */
|
||||
export function tmdbDetailRef(key: string): string | null {
|
||||
const m = /^\/(movie|tv)\/(\d+)(?:[/?]|$)/.exec(key);
|
||||
return m ? `${m[1]}:${m[2]}` : null;
|
||||
}
|
||||
|
||||
// ── 追踪中的作品:详情缓存命中时续期 ──
|
||||
// 追踪 = 作品有未结束的订阅(active / paused / pending)且有已接受的 TMDB 身份。集合按进程缓存 5 分钟。
|
||||
|
||||
const TRACKED_TTL_MS = 5 * 60_000;
|
||||
const trackedCache = new WeakMap<object, { at: number; set: Promise<Set<string>> }>();
|
||||
|
||||
export async function trackedTmdbRefs(ctx: Pick<ServerContext, "db">, now = Date.now()): Promise<Set<string>> {
|
||||
const hit = trackedCache.get(ctx.db);
|
||||
if (hit && now - hit.at < TRACKED_TTL_MS) return hit.set;
|
||||
const set = sql<{ ref: string }>`select distinct x.entity_type || ':' || x.external_id as ref
|
||||
from subscription s join external_identity x on x.media_entity_id = s.media_entity_id
|
||||
where s.status in ('active', 'paused', 'pending') and x.provider = 'tmdb' and x.status = 'accepted'
|
||||
and x.entity_type in ('movie', 'tv')`
|
||||
.execute(ctx.db)
|
||||
.then((r) => new Set(r.rows.map((x) => x.ref)));
|
||||
trackedCache.set(ctx.db, { at: now, set });
|
||||
set.catch(() => trackedCache.delete(ctx.db));
|
||||
return set;
|
||||
}
|
||||
|
||||
/** 测试用:清掉追踪集合缓存 */
|
||||
export function resetTrackedTmdbRefs(ctx: Pick<ServerContext, "db">): void {
|
||||
trackedCache.delete(ctx.db);
|
||||
}
|
||||
|
||||
/**
|
||||
* TMDB 详情缓存命中后的续期:键属于追踪中的作品时把过期时间推到 now + 详情 TTL(只延长不缩短)。
|
||||
* 返回是否续期。失败不影响命中结果。
|
||||
*/
|
||||
export async function renewTrackedTmdbDetail(
|
||||
ctx: Pick<ServerContext, "db">,
|
||||
key: string,
|
||||
detailTtlSec: number,
|
||||
): Promise<boolean> {
|
||||
const ref = tmdbDetailRef(key);
|
||||
if (!ref || !(await trackedTmdbRefs(ctx)).has(ref)) return false;
|
||||
return await sharedCache(ctx).renew(CACHE_REGIONS.tmdbRequest, key, detailTtlSec);
|
||||
}
|
||||
Vendored
+112
-16
@@ -1,14 +1,14 @@
|
||||
import { mkdtempSync, rmSync, utimesSync } from "node:fs";
|
||||
import { existsSync, mkdtempSync, rmSync, statSync, utimesSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { PgCache, sql } from "@umt/db";
|
||||
import { createMockTmdb, type MockTmdb } from "@umt/testkit";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { imageCachePath } from "../../src/modules/cache/images.ts";
|
||||
import { ImageCache, imageCachePath } from "../../src/modules/cache/images.ts";
|
||||
import { createCacheModule } from "../../src/modules/cache/index.ts";
|
||||
import { scheduleRegistry } from "../../src/modules/schedules/registry.ts";
|
||||
import { SourcesRuntime } from "../../src/modules/sources-identity/runtime.ts";
|
||||
import { sharedCache } from "../../src/shared-cache.ts";
|
||||
import { resetTrackedTmdbRefs, sharedCache } from "../../src/shared-cache.ts";
|
||||
import { bearer, createHarness, type Harness, pgAvailable } from "../auth-settings/harness.ts";
|
||||
import { DEV_KEY, json } from "../helpers.ts";
|
||||
|
||||
@@ -53,31 +53,77 @@ describe.skipIf(!hasPg)("C2-A 缓存(真实 PG)", () => {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("TMDB 请求响应进 kv_cache(tmdb.request,TTL = cache.tmdbTtlHours,条数上限 = cache.tmdbMaxSize),另一个进程直接命中", async () => {
|
||||
it("TMDB 请求响应进 kv_cache(tmdb.request,TTL 取 storage.cacheTtl:详情 / 搜索 / 未命中,条数上限 = cache.tmdbMaxSize),另一个进程直接命中", async () => {
|
||||
const r = await admin("PATCH", "/settings", {
|
||||
metadata: { tmdbApiKey: "0123456789abcdef0123456789abcdef", tmdbBaseUrl: `${tmdb.url}/3` },
|
||||
cache: { tmdbTtlHours: 6, tmdbMaxSize: 16 },
|
||||
cache: { tmdbMaxSize: 16 },
|
||||
storage: { cacheTtl: { tmdbDetailDays: 20, tmdbSearchDays: 3, tmdbMissDays: 2 } },
|
||||
});
|
||||
expect(r.status).toBe(200);
|
||||
// 部分更新只改给出的子字段,其余保持缺省
|
||||
const st = (await json(await admin("GET", "/settings"))).data.storage;
|
||||
expect(st.cacheTtl).toMatchObject({ tmdbDetailDays: 20, tmdbSearchDays: 3, tmdbMissDays: 2, siteDetailDays: 7 });
|
||||
expect(st.budgets.imagesGb).toBe(20);
|
||||
await (await new SourcesRuntime(h.ctx).tmdb())!.details("movie", 872585);
|
||||
const calls = tmdb.calls.length;
|
||||
const rows = (
|
||||
await sql<{ key: string; hours: number }>`select key, extract(epoch from expires_at - now()) / 3600 as hours
|
||||
await sql<{ key: string; days: number }>`select key, extract(epoch from expires_at - now()) / 86400 as days
|
||||
from kv_cache where region = 'tmdb.request'`.execute(h.db)
|
||||
).rows;
|
||||
const key = rows.find((x) => x.key.startsWith("/movie/872585?"))!.key;
|
||||
const detail = rows.find((x) => x.key.startsWith("/movie/872585?"))!;
|
||||
expect(rows.every((x) => !x.key.includes("0123456789abcdef"))).toBe(true);
|
||||
expect(Number(rows[0]!.hours)).toBeGreaterThan(5.9);
|
||||
expect(await new PgCache(h.db).get("tmdb.request", key)).toMatchObject({ id: 872585 });
|
||||
expect(Number(detail.days)).toBeGreaterThan(19.9);
|
||||
expect(Number(detail.days)).toBeLessThanOrEqual(20);
|
||||
expect(await new PgCache(h.db).get("tmdb.request", detail.key)).toMatchObject({ id: 872585 });
|
||||
await (await new SourcesRuntime(h.ctx).tmdb())!.details("movie", 872585);
|
||||
expect(tmdb.calls.length).toBe(calls);
|
||||
// 空搜索结果按 EMPTY_RESULT_CACHE_TTL(30 分钟)
|
||||
const days = async (like: string) =>
|
||||
Number(
|
||||
(
|
||||
await sql<{ d: number }>`select extract(epoch from expires_at - now()) / 86400 as d
|
||||
from kv_cache where region = 'tmdb.request' and key like ${like}`.execute(h.db)
|
||||
).rows[0]!.d,
|
||||
);
|
||||
// 有结果的搜索按 tmdbSearchDays
|
||||
await (await new SourcesRuntime(h.ctx).tmdb())!.search("Oppenheimer", { type: "movie" });
|
||||
const hit = await days("/search/movie?%Oppenheimer%");
|
||||
expect(hit).toBeGreaterThan(2.9);
|
||||
expect(hit).toBeLessThanOrEqual(3);
|
||||
// 空搜索结果按 EMPTY_RESULT_CACHE_TTL 语义,取 tmdbMissDays
|
||||
await (await new SourcesRuntime(h.ctx).tmdb())!.search("完全不存在的片名c2a", { type: "movie" });
|
||||
const empty = (
|
||||
await sql<{ m: number }>`select extract(epoch from expires_at - now()) / 60 as m
|
||||
from kv_cache where region = 'tmdb.request' and key like '/search/movie?%'`.execute(h.db)
|
||||
).rows[0];
|
||||
expect(Number(empty!.m)).toBeLessThanOrEqual(30);
|
||||
const empty = await days("/search/movie?%c2a%");
|
||||
expect(empty).toBeGreaterThan(1.9);
|
||||
expect(empty).toBeLessThanOrEqual(2);
|
||||
});
|
||||
|
||||
it("追踪中的作品:详情缓存命中时续期到 now + 详情 TTL;未追踪的不动", async () => {
|
||||
const ent = (
|
||||
await sql<{
|
||||
id: string;
|
||||
}>`insert into media_entity (kind, title) values ('movie', '奥本海默') returning id`.execute(h.db)
|
||||
).rows[0]!.id;
|
||||
await sql`insert into external_identity (media_entity_id, provider, entity_type, external_id, status)
|
||||
values (${ent}, 'tmdb', 'movie', '872585', 'accepted')`.execute(h.db);
|
||||
await sql`insert into subscription (media_entity_id, media_kind, status) values (${ent}, 'movie', 'active')`.execute(
|
||||
h.db,
|
||||
);
|
||||
resetTrackedTmdbRefs(h.ctx);
|
||||
const client = (await new SourcesRuntime(h.ctx).tmdb())!;
|
||||
await client.details("movie", 872585);
|
||||
await client.details("movie", 693134);
|
||||
await sql`update kv_cache set expires_at = now() + interval '1 day'
|
||||
where region = 'tmdb.request' and (key like '/movie/872585?%' or key like '/movie/693134?%')`.execute(h.db);
|
||||
await client.details("movie", 872585);
|
||||
await client.details("movie", 693134);
|
||||
const left = async (like: string) =>
|
||||
Number(
|
||||
(
|
||||
await sql<{ d: number }>`select extract(epoch from expires_at - now()) / 86400 as d
|
||||
from kv_cache where region = 'tmdb.request' and key like ${like}`.execute(h.db)
|
||||
).rows[0]!.d,
|
||||
);
|
||||
expect(await left("/movie/872585?%")).toBeGreaterThan(19.9);
|
||||
expect(await left("/movie/693134?%")).toBeLessThanOrEqual(1);
|
||||
});
|
||||
|
||||
it("GET /cache/tmdb 列出条目(键 / 过期 / 大小)与上限;POST /cache/tmdb/clear 清空", async () => {
|
||||
@@ -88,7 +134,7 @@ describe.skipIf(!hasPg)("C2-A 缓存(真实 PG)", () => {
|
||||
{ ttl: 3600 },
|
||||
);
|
||||
const list = await json(await admin("GET", "/cache/tmdb"));
|
||||
expect(list.data).toMatchObject({ maxSize: 16, ttlHours: 6, emptyResultTtlSec: 1800 });
|
||||
expect(list.data).toMatchObject({ maxSize: 16, ttlHours: 20 * 24, emptyResultTtlSec: 2 * 86400 });
|
||||
expect(list.data.counts.request).toBeGreaterThan(0);
|
||||
const rec = list.data.items.find((i: { kind: string }) => i.kind === "recognize");
|
||||
expect(rec).toMatchObject({ tmdbId: 872585, title: "奥本海默", mediaType: "movie", posterPath: "/p.jpg" });
|
||||
@@ -143,4 +189,54 @@ describe.skipIf(!hasPg)("C2-A 缓存(真实 PG)", () => {
|
||||
expect((await json(await admin("GET", "/cache/images"))).data.files).toBe(0);
|
||||
expect((await json(await admin("POST", "/cache/images/clear"))).data.deleted).toBe(0);
|
||||
});
|
||||
|
||||
it("图片容量 LRU:超过预算按最久未访问(atime)删到预算的 90%;读盘命中刷新 atime、不动 mtime", async () => {
|
||||
const lruDir = mkdtempSync(join(tmpdir(), "umt-c2-lru-"));
|
||||
try {
|
||||
const cache = new ImageCache({
|
||||
dir: lruDir,
|
||||
upstreamBase: "https://image.tmdb.org/t/p",
|
||||
fetch: async () => new Response(new Uint8Array([...JPEG, ...new Uint8Array(92)])),
|
||||
});
|
||||
// 4 张 100 字节的图,访问时间从旧到新 a < b < c < d
|
||||
for (const f of ["a", "b", "c", "d"]) await cache.serve("w92", `${f}.jpg`, false, null);
|
||||
const p = (f: string) => join(lruDir, "t/p/w92", `${f}.jpg`);
|
||||
const base = Date.now() - 10 * 86_400_000;
|
||||
["a", "b", "c", "d"].forEach((f, i) => {
|
||||
utimesSync(p(f), new Date(base + i * 3600_000), new Date(base));
|
||||
});
|
||||
// 读盘命中 a:atime 变成现在(最近使用),mtime 仍是写入时间
|
||||
await cache.serve("w92", "a.jpg", true, null);
|
||||
await new Promise((r) => setTimeout(r, 50));
|
||||
const st = statSync(p("a"));
|
||||
expect(st.atimeMs).toBeGreaterThan(Date.now() - 60_000);
|
||||
expect(Math.abs(st.mtimeMs - base)).toBeLessThan(1000);
|
||||
// 没超预算不动
|
||||
expect(await cache.enforceBudget(1000)).toMatchObject({ deleted: 0, usedBytes: 400 });
|
||||
// 预算 300 → 删到 ≤ 270:最久未访问的 b、c 被删,a(刚读过)与 d 保留
|
||||
const r = await cache.enforceBudget(300);
|
||||
expect(r).toMatchObject({ deleted: 2, freedBytes: 200, usedBytes: 200 });
|
||||
expect(existsSync(p("a"))).toBe(true);
|
||||
expect(existsSync(p("b"))).toBe(false);
|
||||
expect(existsSync(p("c"))).toBe(false);
|
||||
expect(existsSync(p("d"))).toBe(true);
|
||||
} finally {
|
||||
rmSync(lruDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("cache.images_lru 登记为维护任务,预算取 storage.budgets.imagesGb", async () => {
|
||||
await mod.start?.(h.ctx);
|
||||
const job = scheduleRegistry(h.ctx).get("cache.images_lru");
|
||||
expect(job?.defaults).toMatchObject({ group: "maintain", schedule: { everyMs: 30 * 60_000 } });
|
||||
const out = await job!.run({
|
||||
jobId: "cache.images_lru",
|
||||
runId: "r",
|
||||
trigger: "manual",
|
||||
now: new Date(),
|
||||
config: {},
|
||||
slot: "1",
|
||||
});
|
||||
expect(out?.summary).toMatch(/未超过预算 20 GB/);
|
||||
});
|
||||
});
|
||||
@@ -468,18 +468,21 @@ describe.skipIf(!up)("resolveObservation / 身份与元数据", () => {
|
||||
).rows;
|
||||
const opp = (await rows()).find((r) => r.key === "[未知][zh-CN]Oppenheimer-2023-None");
|
||||
expect(opp?.value).toMatchObject({ id: 872585, type: "movie", title: "奥本海默" });
|
||||
expect(Number(opp!.minutes)).toBeGreaterThan(23 * 60);
|
||||
// 正常条目按搜索类 TTL(storage.cacheTtl.tmdbSearchDays,缺省 7 天)
|
||||
expect(Number(opp!.minutes)).toBeGreaterThan(7 * 24 * 60 - 5);
|
||||
expect(Number(opp!.minutes)).toBeLessThanOrEqual(7 * 24 * 60);
|
||||
// 同名同年的另一条资源:命中识别缓存,不再调 TMDB 搜索
|
||||
const searches = tmdb.callsTo(/\/3\/search\//).length;
|
||||
const again = await release("Oppenheimer.2023.1080p.BluRay.x264-OTHER");
|
||||
await resolveRelease(h.rt, { sourceReleaseId: again.id });
|
||||
expect(tmdb.callsTo(/\/3\/search\//).length).toBe(searches);
|
||||
// 查无此项 → 负缓存 {id: 0},TTL = 空结果短 TTL(30 分钟)
|
||||
// 查无此项 → 负缓存 {id: 0},TTL = 未命中 TTL(storage.cacheTtl.tmdbMissDays,缺省 1 天)
|
||||
const none = await release("Zzqxv.Nonexistent.Title.2019.1080p.WEB-DL-NOGRP");
|
||||
await resolveRelease(h.rt, { sourceReleaseId: none.id });
|
||||
const neg = (await rows()).find((r) => r.key === "[未知][zh-CN]Zzqxv Nonexistent Title-2019-None");
|
||||
expect(neg?.value).toEqual({ id: 0 });
|
||||
expect(Number(neg!.minutes)).toBeLessThanOrEqual(30);
|
||||
expect(Number(neg!.minutes)).toBeLessThanOrEqual(24 * 60);
|
||||
expect(Number(neg!.minutes)).toBeGreaterThan(24 * 60 - 5);
|
||||
const s2 = tmdb.callsTo(/\/3\/search\//).length;
|
||||
await resolveRelease(h.rt, { sourceReleaseId: none.id });
|
||||
expect(tmdb.callsTo(/\/3\/search\//).length).toBe(s2);
|
||||
|
||||
@@ -0,0 +1,320 @@
|
||||
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { DEFAULT_STORAGE_SETTINGS, SystemHealth, SystemStorage } from "@umt/contracts";
|
||||
import { type RawBuilder, sql } from "@umt/db";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { scheduleRegistry } from "../../src/modules/schedules/registry.ts";
|
||||
import { createSystemModule } from "../../src/modules/system/index.ts";
|
||||
import { CLEANUP_JOB_IDS, deleteInBatches, RETENTION_JOBS } from "../../src/modules/system/retention.ts";
|
||||
import { budgetLevel, resetDirScans } from "../../src/modules/system/storage.ts";
|
||||
import { bearer, createHarness, type Harness, pgAvailable } from "../auth-settings/harness.ts";
|
||||
import { DEV_KEY, json } from "../helpers.ts";
|
||||
|
||||
// CACHE-2 存储预算与保留:保留清理分批与边界、存储面板形状、health 存储档位、手动触发清理。
|
||||
|
||||
const hasPg = await pgAvailable();
|
||||
const DAY = 86_400_000;
|
||||
|
||||
describe("预算档位", () => {
|
||||
it("≥ 预算 critical,≥ 告警线 warn,取不到 unknown", () => {
|
||||
expect(budgetLevel(null, 100, 80)).toBe("unknown");
|
||||
expect(budgetLevel(79, 100, 80)).toBe("ok");
|
||||
expect(budgetLevel(80, 100, 80)).toBe("warn");
|
||||
expect(budgetLevel(100, 100, 80)).toBe("critical");
|
||||
});
|
||||
|
||||
it("每个清理目标都对应一个定时任务", () => {
|
||||
expect(Object.keys(CLEANUP_JOB_IDS).sort()).toEqual(
|
||||
[
|
||||
"observation",
|
||||
"sse_event",
|
||||
"heartbeat",
|
||||
"task_run",
|
||||
"agent_event",
|
||||
"audit",
|
||||
"capability_stats",
|
||||
"identity_pool",
|
||||
"analyze",
|
||||
"images",
|
||||
"kv_cache",
|
||||
].sort(),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!hasPg)("CACHE-2 存储(真实 PG)", () => {
|
||||
let h: Harness;
|
||||
let admin: ReturnType<typeof bearer>;
|
||||
const dir = mkdtempSync(join(tmpdir(), "umt-cache2-"));
|
||||
|
||||
beforeAll(async () => {
|
||||
h = await createHarness({ engine: true });
|
||||
const { createApp } = await import("../../src/http/app.ts");
|
||||
const { createAuthSettingsModule } = await import("../../src/modules/auth-settings/index.ts");
|
||||
const sys = createSystemModule({
|
||||
engineSchema: h.engineSchema,
|
||||
apiHeartbeat: false,
|
||||
storagePaths: { dataDir: dir, sessionDir: join(dir, "agent-sessions") },
|
||||
retentionBatch: { batchSize: 3, pauseMs: 0 },
|
||||
healthDeps: {
|
||||
dataDir: dir,
|
||||
diskUsage: async (path) => ({
|
||||
status: "ok",
|
||||
path,
|
||||
totalBytes: 1000,
|
||||
freeBytes: 500,
|
||||
availableBytes: 500,
|
||||
usedBytes: 500,
|
||||
usedRatio: 0.5,
|
||||
}),
|
||||
},
|
||||
});
|
||||
h.app = createApp({
|
||||
config: h.ctx.config,
|
||||
logger: h.ctx.logger,
|
||||
bus: h.bus,
|
||||
db: h.db,
|
||||
role: "all",
|
||||
version: "0.0.0-test",
|
||||
modules: [createAuthSettingsModule({ setupTokenOnStart: false, engineSchema: h.engineSchema }), sys],
|
||||
ctx: h.ctx,
|
||||
}) as Harness["app"];
|
||||
admin = bearer(h.app, DEV_KEY);
|
||||
await sys.start?.(h.ctx);
|
||||
// 目录:图片 3 KB、备份 2 份、会话 1 KB、证据 1 KB
|
||||
mkdirSync(join(dir, "cache", "images", "t", "p", "w92"), { recursive: true });
|
||||
writeFileSync(join(dir, "cache", "images", "t", "p", "w92", "a.jpg"), new Uint8Array(3072));
|
||||
for (const b of ["20260901T000000Z", "20260902T000000Z", ".20260903T000000Z.partial"]) {
|
||||
mkdirSync(join(dir, "backups", b), { recursive: true });
|
||||
writeFileSync(join(dir, "backups", b, "db.dump"), new Uint8Array(1024));
|
||||
}
|
||||
mkdirSync(join(dir, "agent-sessions"), { recursive: true });
|
||||
writeFileSync(join(dir, "agent-sessions", "s.jsonl"), new Uint8Array(1024));
|
||||
mkdirSync(join(dir, "tmp", "evidence"), { recursive: true });
|
||||
writeFileSync(join(dir, "tmp", "evidence", "e.jpg"), new Uint8Array(1024));
|
||||
resetDirScans();
|
||||
}, 60_000);
|
||||
afterAll(async () => {
|
||||
await h?.close();
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
const job = (target: string) => {
|
||||
const j = RETENTION_JOBS.find((x) => x.target === target)!;
|
||||
return (now = new Date(), settings = DEFAULT_STORAGE_SETTINGS) =>
|
||||
j.run({ db: h.db, engineSchema: h.engineSchema, settings, now, batch: { batchSize: 3, pauseMs: 0 } });
|
||||
};
|
||||
const count = async (q: RawBuilder<{ n: number }>) => Number((await q.execute(h.db)).rows[0]!.n);
|
||||
|
||||
it("保留任务登记在维护组,各自一个 key", () => {
|
||||
const reg = scheduleRegistry(h.ctx);
|
||||
for (const j of RETENTION_JOBS) {
|
||||
expect(reg.get(j.jobId)?.defaults, j.jobId).toMatchObject({ group: "maintain" });
|
||||
}
|
||||
expect(reg.get("storage.analyze")?.defaults.schedule).toMatchObject({ everyMs: 7 * DAY });
|
||||
expect(reg.get("system.heartbeat_cleanup")?.defaults.group).toBe("maintain");
|
||||
});
|
||||
|
||||
it("deleteInBatches:按批删除,只删满足条件的行;总时限到了停在中途,下次接着删", async () => {
|
||||
await sql`insert into sse_event (type, at) select 'x', now() - interval '30 days' from generate_series(1, 10)`.execute(
|
||||
h.db,
|
||||
);
|
||||
await sql`insert into sse_event (type, at) values ('keep', now())`.execute(h.db);
|
||||
// 时钟每次调用前进 1 秒,预算 1.5 秒:删完 2 批就停
|
||||
let t = 0;
|
||||
const r1 = await deleteInBatches(h.db, sql.raw("sse_event"), sql`at < now() - interval '7 days'`, {
|
||||
batchSize: 3,
|
||||
pauseMs: 0,
|
||||
budgetMs: 1500,
|
||||
now: () => (t += 1000),
|
||||
});
|
||||
expect(r1).toEqual({ deleted: 6, batches: 2, truncated: true });
|
||||
const r2 = await deleteInBatches(h.db, sql.raw("sse_event"), sql`at < now() - interval '7 days'`, {
|
||||
batchSize: 3,
|
||||
pauseMs: 0,
|
||||
});
|
||||
expect(r2).toEqual({ deleted: 4, batches: 2, truncated: false });
|
||||
expect(await count(sql<{ n: number }>`select count(*)::int as n from sse_event`)).toBe(1);
|
||||
});
|
||||
|
||||
it("每批语句超时:停止本次运行,不抛错", async () => {
|
||||
await sql`insert into sse_event (type, at) values ('slow', now() - interval '30 days')`.execute(h.db);
|
||||
const r = await deleteInBatches(
|
||||
h.db,
|
||||
sql.raw("sse_event"),
|
||||
sql`at < now() - interval '7 days' and pg_sleep(0.3) is not null`,
|
||||
{ batchSize: 3, statementTimeoutMs: 50 },
|
||||
);
|
||||
expect(r).toEqual({ deleted: 0, batches: 0, truncated: true });
|
||||
await sql`delete from sse_event where type = 'slow'`.execute(h.db);
|
||||
});
|
||||
|
||||
it("观察记录:保留边界(90 天前一刻删,之后保留)", async () => {
|
||||
const feed = (
|
||||
await sql<{
|
||||
id: string;
|
||||
}>`insert into feed_source (name, url_display) values ('rss', 'https://x/rss') returning id`.execute(h.db)
|
||||
).rows[0]!.id;
|
||||
const now = new Date();
|
||||
const ins = (guid: string, at: Date) =>
|
||||
sql`insert into observation (feed_id, guid, content_digest, title, created_at)
|
||||
values (${feed}, ${guid}, ${guid}, ${guid}, ${at})`.execute(h.db);
|
||||
for (let i = 0; i < 5; i++) await ins(`old${i}`, new Date(now.getTime() - 90 * DAY - 60_000 - i));
|
||||
await ins("edge", new Date(now.getTime() - 90 * DAY + 60_000));
|
||||
await ins("new", now);
|
||||
expect(await job("observation")(now)).toMatch(/删除 5 行（2 批）/);
|
||||
const left = await sql<{ guid: string }>`select guid from observation order by guid`.execute(h.db);
|
||||
expect(left.rows.map((r) => r.guid)).toEqual(["edge", "new"]);
|
||||
});
|
||||
|
||||
it("任务运行:只删过期且已结束的业务任务与引擎 run", async () => {
|
||||
const now = new Date();
|
||||
const old = new Date(now.getTime() - 61 * DAY);
|
||||
const ins = (key: string, status: string, at: Date) =>
|
||||
sql`insert into task (kind, idempotency_key, status, created_at) values ('k', ${key}, ${status}, ${at})`.execute(
|
||||
h.db,
|
||||
);
|
||||
await ins("t1", "succeeded", old);
|
||||
await ins("t2", "failed", old);
|
||||
await ins("t3", "running", old);
|
||||
await ins("t4", "succeeded", now);
|
||||
const t = (n: string) => sql.raw(`"${h.engineSchema}".${n}`);
|
||||
const run = (id: string, status: string, at: Date) =>
|
||||
sql`insert into ${t("workflow_runs")} (namespace_id, id, workflow_name, version, status, config, attempts, created_at, updated_at)
|
||||
values ('default', ${id}, 'w', 'v1', ${status}, '{}'::jsonb, 1, ${at}, ${at})`.execute(h.db);
|
||||
await run("r-old-done", "completed", old);
|
||||
await run("r-old-running", "running", old);
|
||||
await run("r-new", "completed", now);
|
||||
expect(await job("task_run")(now)).toMatch(/业务任务 2、引擎 run 1/);
|
||||
expect(
|
||||
(await sql<{ idempotency_key: string }>`select idempotency_key from task order by 1`.execute(h.db)).rows.map(
|
||||
(r) => r.idempotency_key,
|
||||
),
|
||||
).toEqual(["t3", "t4"]);
|
||||
expect(
|
||||
(await sql<{ id: string }>`select id from ${t("workflow_runs")} order by id`.execute(h.db)).rows.map((r) => r.id),
|
||||
).toEqual(expect.arrayContaining(["r-old-running", "r-new"]));
|
||||
expect(
|
||||
await count(sql<{ n: number }>`select count(*)::int as n from ${t("workflow_runs")} where id = 'r-old-done'`),
|
||||
).toBe(0);
|
||||
});
|
||||
|
||||
it("Agent 事件:只删已结束会话里过期的事件;审计与能力统计分开保留", async () => {
|
||||
const now = new Date();
|
||||
const runs = await sql<{ id: string; status: string }>`insert into agent_run (trigger, goal, status)
|
||||
values ('user', 'g', 'succeeded'), ('user', 'g', 'running') returning id, status`.execute(h.db);
|
||||
const done = runs.rows.find((r) => r.status === "succeeded")!.id;
|
||||
const live = runs.rows.find((r) => r.status === "running")!.id;
|
||||
const old = new Date(now.getTime() - 91 * DAY);
|
||||
await sql`insert into agent_run_event (run_id, seq, type, at) values
|
||||
(${done}, 1, 'run_started', ${old}), (${done}, 2, 'run_started', ${now}), (${live}, 1, 'run_started', ${old})`.execute(
|
||||
h.db,
|
||||
);
|
||||
expect(await job("agent_event")(now)).toMatch(/Agent 事件 1、/);
|
||||
expect(await count(sql<{ n: number }>`select count(*)::int as n from agent_run_event`)).toBe(2);
|
||||
|
||||
await sql`delete from audit_event`.execute(h.db);
|
||||
const a = (days: number, targetType: string | null) =>
|
||||
sql`insert into audit_event (actor_type, action, target_type, at)
|
||||
values ('system', 'x', ${targetType}, ${new Date(now.getTime() - days * DAY)})`.execute(h.db);
|
||||
await a(181, null);
|
||||
await a(179, null);
|
||||
await a(100, "capability");
|
||||
await a(80, "capability");
|
||||
await a(181, "capability");
|
||||
expect(await job("audit")(now)).toMatch(/审计事件删除 1 行/);
|
||||
expect(await job("capability_stats")(now)).toMatch(/能力审计 2、/);
|
||||
const left = await sql<{ t: string | null; d: number }>`select target_type as t,
|
||||
round(extract(epoch from now() - at) / 86400)::int as d from audit_event order by d`.execute(h.db);
|
||||
expect(left.rows).toEqual([
|
||||
{ t: "capability", d: 80 },
|
||||
{ t: null, d: 179 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("待识别池:unmatched 超过保留期没有新证据的归档,其余不动", async () => {
|
||||
const site = (
|
||||
await sql<{ id: string }>`insert into site (key, name, adapter, base_url)
|
||||
values ('s-cache2', 'S', 'nexusphp', 'https://s.example') returning id`.execute(h.db)
|
||||
).rows[0]!.id;
|
||||
const now = new Date();
|
||||
const cand = async (rid: string, reason: string, lastAttemptDays: number) => {
|
||||
const sr = (
|
||||
await sql<{ id: string }>`insert into source_release (site_id, site_release_id, title)
|
||||
values (${site}, ${rid}, ${rid}) returning id`.execute(h.db)
|
||||
).rows[0]!.id;
|
||||
await sql`insert into identity_candidate (source_release_id, reason, evidence_version, last_attempt_at)
|
||||
values (${sr}, ${reason}, 'v1', ${new Date(now.getTime() - lastAttemptDays * DAY)})`.execute(h.db);
|
||||
};
|
||||
await cand("r1", "unmatched", 31);
|
||||
await cand("r2", "unmatched", 29);
|
||||
await cand("r3", "ambiguous", 31);
|
||||
expect(await job("identity_pool")(now)).toBe("归档 1 条待识别条目");
|
||||
const st = await sql<{ t: string; s: string }>`select sr.title as t, c.status as s from identity_candidate c
|
||||
join source_release sr on sr.id = c.source_release_id order by sr.title`.execute(h.db);
|
||||
expect(st.rows).toEqual([
|
||||
{ t: "r1", s: "archived" },
|
||||
{ t: "r2", s: "pending" },
|
||||
{ t: "r3", s: "pending" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("ANALYZE 任务", async () => {
|
||||
expect(await job("analyze")()).toMatch(/ANALYZE 完成/);
|
||||
});
|
||||
|
||||
it("GET /system/storage:表体积 top 15、库体积对照预算、kv_cache、目录用量、清理任务", async () => {
|
||||
await sql`insert into kv_cache (region, key, value, expires_at) values
|
||||
('tmdb.request', 'a', '{}'::jsonb, now() + interval '1 day'),
|
||||
('tmdb.request', 'b', '{}'::jsonb, now() - interval '1 day')`.execute(h.db);
|
||||
const res = await admin("GET", "/system/storage");
|
||||
expect(res.status).toBe(200);
|
||||
const d = SystemStorage.parse((await json(res)).data);
|
||||
expect(d.db.tables.length).toBeGreaterThan(0);
|
||||
expect(d.db.tables.length).toBeLessThanOrEqual(15);
|
||||
expect(d.db.usedBytes).toBeGreaterThan(0);
|
||||
expect(d.db.budgetBytes).toBe(25 * 1024 ** 3);
|
||||
expect(d.db.warnBytes).toBe(20 * 1024 ** 3);
|
||||
expect(d.kvCache.find((r) => r.region === "tmdb.request")).toMatchObject({ entries: 2, expired: 1 });
|
||||
expect(d.images).toMatchObject({ usedBytes: 3072, files: 1, level: "ok", budgetBytes: 20 * 1024 ** 3 });
|
||||
expect(d.backups).toMatchObject({ usedBytes: 3072, count: 2, level: "ok" });
|
||||
expect(d.sessionsEvidence).toMatchObject({ usedBytes: 2048, level: "ok" });
|
||||
expect(d.dataDir.map((x) => x.name)).toEqual(
|
||||
expect.arrayContaining(["agent-sessions", "backups", "cache/images", "tmp/evidence"]),
|
||||
);
|
||||
const obs = d.cleanup.find((c) => c.target === "observation")!;
|
||||
expect(obs).toMatchObject({ jobId: "storage.retention.observation", retention: "90 天" });
|
||||
expect(d.cleanup.find((c) => c.target === "heartbeat")?.retention).toBe("24 小时");
|
||||
expect(d.level).toBe("ok");
|
||||
});
|
||||
|
||||
it("预算超了:存储面板与 health 都报出来", async () => {
|
||||
const r = await admin("PATCH", "/settings", {
|
||||
storage: { budgets: { pgDataWarnGb: 0.01, pgDataGb: 50 } },
|
||||
});
|
||||
expect(r.status).toBe(200);
|
||||
// 把 PG 告警线压到库体积之下验证 db warn
|
||||
const d = SystemStorage.parse((await json(await admin("GET", "/system/storage"))).data);
|
||||
expect(d.db.level).toBe("warn");
|
||||
expect(d.level).toBe("warn");
|
||||
const hl = SystemHealth.parse((await json(await admin("GET", "/system/health"))).data);
|
||||
expect(hl.storage.level).toBe("warn");
|
||||
expect(hl.storage.alerts.join("")).toMatch(/数据库/);
|
||||
expect(["warn", "critical"]).toContain(hl.level);
|
||||
await admin("PATCH", "/settings", { storage: { budgets: { pgDataWarnGb: 20, pgDataGb: 25 } } });
|
||||
});
|
||||
|
||||
it("POST /system/storage/cleanup/{target}:排入对应定时任务;重复请求合并;未知目标 422", async () => {
|
||||
await sql`insert into scheduled_job (id, name, board_group, kind, enabled, default_enabled, schedule, default_schedule)
|
||||
values ('storage.retention.audit', '审计清理', 'maintain', 'task', true, true,
|
||||
'{"type":"interval","everyMs":86400000}'::jsonb, '{"type":"interval","everyMs":86400000}'::jsonb)
|
||||
on conflict (id) do nothing`.execute(h.db);
|
||||
const r1 = await json(await admin("POST", "/system/storage/cleanup/audit"));
|
||||
expect(r1.data).toEqual({ target: "audit", jobId: "storage.retention.audit", queued: true });
|
||||
const r2 = await json(await admin("POST", "/system/storage/cleanup/audit"));
|
||||
expect(r2.data.queued).toBe(false);
|
||||
expect((await admin("POST", "/system/storage/cleanup/nope")).status).toBe(422);
|
||||
expect((await admin("POST", "/system/storage/cleanup/observation")).status).toBe(404);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user