Files
UHD_Media_Tool/packages/contracts/test/db-sync.test.ts
T
rootandClaude Opus 5.5 ad472a1a19 feat(db): 统一元数据缓存层 MetadataCache
- 两级:L1 进程内存(按 region 限条数的 LRU、5 分钟)+ L2 PG kv_cache;读 L1 → L2 → 源,写两级
- 迁移 0086:kv_cache 加 fresh_until(过期先用旧值的宽限期)与 pinned(LRU 淘汰跳过);新表 cache_stat 按天 × region 汇总命中
- stale-while-revalidate:宽限期 = TTL 的 50%(上限 7 天),旧值先返回、同 key 后台只刷一次,刷新失败继续用旧值并记警告
- 同进程同 key 并发只回源一次;回源抛错不缓存;PG 读写失败按未命中 / 不写处理
- PgCache.delete / clear 发 pg_notify(umt_cache_invalidate),listenCacheInvalidation 收到后丢 L1;监听在线才启用 L1,断线停用、重连后整体清空
- PgCache.get 只认新鲜条目,getEntry 带回旧值;renew 只续新鲜条目并可标 pinned

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei
2026-10-06 19:37:06 +08:00

335 lines
14 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { readFileSync } from "node:fs";
import { describe, expect, it } from "vitest";
import type { z } from "zod";
import * as C from "../src/index.ts";
// 与 packages/db/src/types.ts(从迁移反射生成)做文本级对照:
// 1) 每张表的列 ⇔ 对应 Row schema 的字段(camelCase),只允许显式列出的差异;
// 2) 每个 CHECK 枚举 ⇔ contracts 中对应 z.enum 的取值。
// 只读文件,不引入 @umt/db 依赖。
const typesSrc = readFileSync(new URL("../../db/src/types.ts", import.meta.url), "utf8");
const camel = (s: string) => s.replace(/_([a-z0-9])/g, (_, c: string) => c.toUpperCase());
function tableColumns(): Map<string, string[]> {
const out = new Map<string, string[]>();
for (const m of typesSrc.matchAll(/export interface (\w+)Table \{\n([\s\S]*?)\n\}/g)) {
const cols = [...m[2]!.matchAll(/^ {2}(\w+):/gm)].map((x) => x[1]!);
out.set(m[1]!, cols);
}
return out;
}
function enumAliases(): Map<string, string[]> {
const out = new Map<string, string[]>();
for (const m of typesSrc.matchAll(/export type (\w+) =([^;]+);/g)) {
if (m[1] === "Timestamptz" || m[1] === "Json" || m[2]!.includes("<")) continue;
out.set(
m[1]!,
[...m[2]!.matchAll(/"([^"]*)"/g)].map((x) => x[1]!),
);
}
return out;
}
type Obj = z.ZodObject;
/** 表(PascalCase)→ Row schema 与允许的差异 */
const ROWS: Record<string, { row: Obj; dbOnly?: string[]; apiOnly?: string[] }> = {
AcquisitionPlan: { row: C.AcquisitionPlan },
AcquisitionSlot: { row: C.AcquisitionSlot, apiOnly: ["currentAsset"] },
// upstream_id 只在服务端使用;await_wake 是挂起等待子会话 / 用户选择 / 审批时的唤醒信号(C1-B);
// mode 是内部能力上限(AG-2 取消运行模式后不再出 API):都不进 AgentRun 视图
AgentRun: { row: C.AgentRun, dbOnly: ["upstreamId", "awaitWake", "mode"] },
AgentRunEvent: { row: C.AgentRunEventRow },
AgentRunMessage: { row: C.AgentRunMessage },
CapabilityInvocation: { row: C.CapabilityInvocation },
ApiKey: { row: C.ApiKey, dbOnly: ["keyHash"] },
Approval: { row: C.Approval },
Artifact: { row: C.Artifact },
AssetLocation: { row: C.AssetLocation },
AssetOperation: { row: C.AssetOperation },
AssetUnitCoverage: { row: C.AssetUnitCoverage },
AuditEvent: { row: C.AuditEvent },
AutomationGrant: { row: C.AutomationGrant },
ContentUnit: { row: C.ContentUnit },
Decision: { row: C.Decision },
DownloadTask: { row: C.DownloadTask },
DownloaderInstance: { row: C.DownloaderInstance },
EpisodeMappingRule: { row: C.EpisodeMappingRuleRow },
ExternalIdentity: { row: C.ExternalIdentity, apiOnly: ["mediaEntityTitle"] },
FeedSource: { row: C.FeedSource, apiOnly: ["possibleGap", "warnings"] },
IdentityRevision: { row: C.IdentityRevision },
LibraryRoot: { row: C.LibraryRoot },
MediaAsset: { row: C.MediaAsset },
MediaEntity: { row: C.MediaEntity },
MediaServerInstance: { row: C.MediaServerInstance },
Observation: { row: C.Observation },
OrganizationFileResult: { row: C.OrganizationFileResult },
OrganizeSourceSnapshot: { row: C.OrganizeSourceSnapshotRow },
// requeue_count / requeued_at:整理巡检的内部计数,不外露
OrganizationRecord: { row: C.OrganizationRecord, dbOnly: ["changeXid", "requeueCount", "requeuedAt"] },
OutboxEvent: { row: C.OutboxEvent },
PathBinding: { row: C.PathBinding },
PathMapping: { row: C.PathMapping },
PolicyRevision: { row: C.PolicyRevision },
PreferenceSuppression: { row: C.PreferenceSuppression },
PreferenceTag: { row: C.PreferenceTag },
ReconcileRun: { row: C.ReconcileRun },
Secret: { row: C.Secret, dbOnly: ["ciphertext", "nonce", "authTag"] },
ScrapeRecord: { row: C.ScrapeRecordRow },
SubtitleWrite: { row: C.SubtitleWriteRow },
ServerMapping: { row: C.ServerMapping },
Settings: { row: C.SettingEntry },
// deleted_at:软删除只在服务端过滤(已删站点不出 API);capabilities 由类型 ∩ 站点定义算出(IA-2)
Site: { row: C.Site, dbOnly: ["deletedAt"], apiOnly: ["capabilities"] },
// secret 引用不外露,只给布尔
SiteUserData: { row: C.SiteUserData },
SiteAccount: {
row: C.SiteAccount,
// BR-4:ua_pending 只给解析后的 userAgentSource / effectiveUserAgent;browserOnlyCookies 取自 user_info
dbOnly: ["totpSecretRef", "sessionSecretRef", "passkeySecretRef", "uaPending"],
apiOnly: ["hasTotp", "hasSession", "hasPasskey", "userAgentSource", "effectiveUserAgent", "browserOnlyCookies"],
},
SourceRelease: { row: C.SourceRelease },
Subscription: { row: C.Subscription, apiOnly: ["mediaEntityTitle"] },
Task: { row: C.Task },
TitleAlias: { row: C.TitleAlias, apiOnly: ["mediaEntityTitle"] },
// W2-B 来源/身份/元数据(0018–0020)
FieldOverride: { row: C.FieldOverride },
IdentityCandidate: { row: C.IdentityCandidate },
IdentityRedirect: { row: C.IdentityRedirect },
MetadataField: { row: C.MetadataField },
// META-1 元数据补全记录(0049)
MetadataEnrichment: { row: C.MetadataEnrichment },
ResolutionAttempt: { row: C.ResolutionAttempt },
SiteSnapshot: { row: C.SiteSnapshot },
SiteFailureSample: { row: C.SiteFailureSample },
// W4-C 浏览器实体(0023)
// BR-4:identity 是浏览器固定身份的缓存(站点请求 UA 用),不进 API
BrowserProvider: { row: C.BrowserProviderRow, dbOnly: ["identity"] },
BrowserSession: { row: C.BrowserSession },
BrowserTab: { row: C.BrowserTab },
BrowserControlLease: { row: C.BrowserControlLease },
// W5-A 文件管理回收站(0024)
TrashItem: { row: C.TrashItemRow },
// W5-C 订阅增强(0025)
SlotDeferral: { row: C.SlotDeferral },
SlotUnit: { row: C.SlotUnit },
SubscriptionHistory: { row: C.SubscriptionHistory },
// W7-C 定时任务(0028):board_group 在 API 里叫 group;default_enabled 只用于"恢复默认";config_customized(0045)只供启动同步
// agentRuns(INT-3)是看板列表按 agent_run 汇总出来的,不是列
ScheduledJob: {
row: C.ScheduledJob,
dbOnly: ["boardGroup", "defaultEnabled", "configCustomized"],
apiOnly: ["group", "agentRuns"],
},
ScheduledJobRun: { row: C.ScheduledJobRun },
// W7-A Pi 上游(0027):secret_ref 不出 API
// 0044 远程模型表:经 GET /pi/upstreams/{id}/models 单独返回,不进上游行
PiUpstream: { row: C.PiUpstreamRow, dbOnly: ["secretRef", "models", "modelsFetchedAt"] },
// W9-D 规则组(0033):legacy_rule 只供旧 /preference-tags 别名回放
RuleGroup: { row: C.RuleGroup, dbOnly: ["legacyRule"] },
RuleAtom: { row: C.RuleAtom },
};
/** 引擎自有表(@umt/tasks 内部使用,不经 API 暴露),不要求 Row schema */
const INTERNAL_TABLES = [
"TasksRunMeta",
"TasksFact",
"TasksOpLedger",
// W2-A 登录门禁与 SSE 总线的服务端内部表:经 contracts/auth.ts 的专用 DTO 暴露,不做 Row 对照
"AdminUser",
"AdminSession",
"LoginAttempt",
"SseEvent",
// W9-C 命名模板版本登记(0032):服务端内部,经 /naming/* 暴露
"NamingTemplateVersion",
// C2-A 跨进程限流桶(0041)与 PG 缓存(0042):服务端内部,缓存经 /cache/* 的专用 DTO 暴露
"RateLimitBucket",
"KvCache",
// 统一元数据缓存命中统计(0086):服务端内部,经 GET /cache/regions 的 hits 字段暴露
"CacheStat",
// 站点请求计数(0046):服务端内部,经 GET /sites/{id}/request-stats 的专用 DTO 暴露
"SiteRequestCount",
// W9-I 进程心跳(0034):服务端内部,经 /system/health 的专用 DTO 暴露
"ProcessHeartbeat",
"ProcessHeartbeatSample",
// AG-1 MCP 服务端单行配置(0039):经 /mcp/status 与 /mcp/config 的专用 DTO 暴露
"McpServerConfig",
// C1-B 恢复码 / Passkey / WebAuthn 挑战(0037):服务端内部,经 contracts/auth.ts 的专用 DTO 暴露
"AdminRecoveryCode",
"AdminPasskey",
"WebauthnChallenge",
];
const INTERNAL_ENUMS = ["TasksOpLedgerStatus", "WebauthnChallengePurpose"];
/** DB 枚举类型名 → contracts 枚举 */
const ENUMS: Record<string, z.ZodEnum> = {
AcquisitionPlanPurpose: C.PlanPurpose,
AcquisitionPlanStatus: C.PlanStatus,
AcquisitionPlanPriority: C.AcquisitionPriority,
AcquisitionSlotStatus: C.SlotStatus,
AgentRunTrigger: C.AgentTrigger,
AgentRunStatus: C.AgentRunStatus,
AgentRunEventType: C.AgentRunEventType,
AgentRunMode: C.AgentRunMode,
AgentRunMessageStatus: C.AgentRunMessageStatus,
CapabilityInvocationStatus: C.CapabilityInvocationStatus,
ApprovalStatus: C.ApprovalStatus,
ArtifactKind: C.ArtifactKind,
AssetLocationLocationKind: C.LocationKind,
AssetLocationStatus: C.LocationStatus,
AssetOperationPhase: C.OperationPhase,
AssetOperationKind: C.OperationKind,
AssetOperationStatus: C.OperationStatus,
AssetUnitCoverageCoverage: C.CoverageKind,
AssetUnitCoverageSegment: C.Segment,
AutomationGrantStatus: C.GrantStatus,
ContentUnitKind: C.ContentUnitKind,
ContentUnitContentType: C.ContentType,
ContentUnitSegment: C.Segment,
ContentUnitStatus: C.ContentUnitStatus,
DecisionKind: C.DecisionKind,
DecisionStatus: C.DecisionStatus,
DecisionSeverity: C.Severity,
DownloadTaskOrigin: C.DownloadOrigin,
DownloadTaskStatus: C.DownloadStatus,
DownloadTaskPauseReason: C.PauseReason,
DownloaderInstanceKind: C.DownloaderKind,
DownloaderInstanceHealthStatus: C.InstanceHealth,
EpisodeMappingRuleKind: C.EpisodeMappingRuleKind,
EpisodeMappingRuleSpecial: C.EpisodeSpecialKind,
EpisodeMappingRuleStatus: C.EpisodeMappingRuleStatus,
EpisodeMappingRuleEvidenceLevel: C.EvidenceLevel,
EpisodeMappingRuleCreatedByType: C.ActorType,
ExternalIdentityEntityType: C.ExternalEntityType,
ExternalIdentityStatus: C.IdentityStatus,
ExternalIdentityEvidenceLevel: C.EvidenceLevel,
ExternalIdentityVerifyStatus: C.IdentityVerifyStatus,
FeedSourcePurpose: C.FeedPurpose,
FeedSourceKind: C.FeedKind,
FeedSourceHealthStatus: C.FeedHealth,
IdentityRevisionAction: C.IdentityRevisionAction,
IdentityRevisionFromStatus: C.IdentityStatus,
IdentityRevisionToStatus: C.IdentityStatus,
IdentityRevisionActorType: C.ActorType,
LibraryRootMediaKind: C.LibraryMediaKind,
LibraryRootManagementMode: C.ManagementMode,
LibraryRootHealthStatus: C.InstanceHealth,
MediaAssetKind: C.AssetKind,
MediaAssetStatus: C.AssetStatus,
MediaEntityKind: C.MediaKind,
MediaEntityStatus: C.MediaEntityStatus,
MediaServerInstanceKind: C.MediaServerKind,
MediaServerInstanceHealthStatus: C.InstanceHealth,
ObservationStatus: C.ObservationStatus,
OrganizationFileResultRole: C.FileRole,
OrganizationFileResultMode: C.OrganizationAction,
OrganizationFileResultStatus: C.FileResultStatus,
OrganizationRecordTrigger: C.OrganizationTrigger,
OrganizationRecordActorType: C.ActorType,
OrganizationRecordSourcePathView: C.SourcePathView,
OrganizationRecordAction: C.OrganizationAction,
OrganizationRecordStatus: C.OrganizationStatus,
OutboxEventStatus: C.OutboxStatus,
PathBindingBindingKind: C.BindingKind,
PathBindingStatus: C.BindingStatus,
PathMappingViewKind: C.PathViewKind,
PathMappingVerifyStatus: C.PathMappingVerifyStatus,
PolicyRevisionKind: C.PolicyKind,
PolicyRevisionStatus: C.PolicyStatus,
PreferenceTagKind: C.PreferenceKind,
PreferenceTagOrigin: C.PreferenceOrigin,
RuleGroupKind: C.RuleGroupKind,
RuleGroupSource: C.RuleGroupSource,
PreferenceTagStatus: C.PreferenceStatus,
ReconcileRunKind: C.ReconcileKind,
ReconcileRunMode: C.ReconcileMode,
ReconcileRunStatus: C.ReconcileStatus,
SecretPurpose: C.SecretPurpose,
ProcessHeartbeatProcessKind: C.ProcessKind,
ScheduledJobBoardGroup: C.ScheduleGroup,
ScheduledJobKind: C.ScheduleKind,
ScheduledJobLastStatus: C.ScheduleRunStatus,
ScheduledJobRunTrigger: C.ScheduleTrigger,
ScheduledJobRunStatus: C.ScheduleRunStatus,
SiteFailureSampleKind: C.SiteFailureSampleKind,
ScrapeRecordStatus: C.ScrapeStatus,
ScrapeRecordTrigger: C.ScrapeTrigger,
SubtitleWriteSource: C.SubtitleSource,
ScrapeRecordMode: C.ScrapeMode,
PiUpstreamAuthKind: C.PiAuthKind,
PiUpstreamStatus: C.PiUpstreamStatus,
PiUpstreamThinkingLevel: C.PiThinkingLevel,
ServerMappingItemType: C.ServerItemType,
ServerMappingMatchStatus: C.ServerMatchStatus,
SiteAdapter: C.SiteAdapter,
SiteHealthStatus: C.SiteHealth,
SiteCircuitState: C.CircuitState,
SiteKind: C.SiteKind,
SiteAccountAuthKind: C.SiteAuthKind,
SiteAccountStatus: C.SiteAccountStatus,
SourceReleaseResolutionStatus: C.ResolutionStatus,
SourceReleaseAvailability: C.ReleaseAvailability,
SubscriptionMediaKind: C.SubscriptionMediaKind,
SubscriptionMode: C.SubscriptionMode,
SubscriptionSeasonScope: C.SeasonScope,
SubscriptionOrigin: C.SubscriptionOrigin,
SubscriptionPriority: C.AcquisitionPriority,
SubscriptionStatus: C.SubscriptionStatus,
TaskStatus: C.TaskStatus,
TitleAliasKind: C.AliasKind,
TitleAliasSource: C.AliasSource,
// W2-B(0018–0020)
FieldOverrideSource: C.MetadataSource,
FieldOverrideActorType: C.ActorType,
IdentityCandidateStatus: C.IdentityCandidateStatus,
IdentityCandidateReason: C.IdentityCandidateReason,
IdentityRedirectActorType: C.ActorType,
MetadataFieldSource: C.MetadataSource,
MetadataEnrichmentStatus: C.EnrichmentStatus,
MetadataEnrichmentTrigger: C.EnrichmentTrigger,
ResolutionAttemptOutcome: C.ResolutionOutcome,
ResolutionAttemptActorType: C.ActorType,
SiteSnapshotKind: C.SiteSnapshotKind,
SiteSnapshotStatus: C.SiteSnapshotStatus,
// W4-C(0023)
BrowserProviderKind: C.BrowserProviderKind,
BrowserProviderHealthStatus: C.InstanceHealth,
BrowserSessionController: C.BrowserController,
BrowserTabStatus: C.BrowserTabStatus,
BrowserControlLeaseHolderKind: C.BrowserLeaseHolderKind,
BrowserControlLeaseStatus: C.BrowserLeaseStatus,
// W5-C(0025)
ContentUnitEpisodeTotalSource: C.EpisodeTotalSource,
// DH-1(0053)
ContentUnitSource: C.ContentUnitSource,
};
describe("contracts ⇔ db 表结构同步", () => {
const tables = tableColumns();
it("每张表都有 Row schema", () => {
expect([...tables.keys()].filter((t) => !INTERNAL_TABLES.includes(t)).sort()).toEqual(Object.keys(ROWS).sort());
});
for (const [table, spec] of Object.entries(ROWS)) {
it(`${table} 列与 Row 字段一致`, () => {
const dbCols = (tables.get(table) ?? []).map(camel).filter((c) => !spec.dbOnly?.includes(c));
const apiCols = Object.keys(spec.row.shape).filter((c) => !spec.apiOnly?.includes(c));
expect(apiCols.sort()).toEqual(dbCols.sort());
});
}
it("每个 DB 枚举都有对应 contracts 枚举且取值一致", () => {
const aliases = enumAliases();
for (const internal of INTERNAL_ENUMS) aliases.delete(internal);
expect([...aliases.keys()].sort()).toEqual(Object.keys(ENUMS).sort());
for (const [name, values] of aliases) {
expect(ENUMS[name]!.options, name).toEqual(values);
}
});
});