feat(sites): cross-process site rate limit via PG token bucket
Migration 0041_rate_limit adds rate_limit_bucket. PgTokenBucket (@umt/db) refills and takes tokens under SELECT ... FOR UPDATE on the DB clock and polls with jitter within a wait budget; penalize() carries Retry-After. Site adapters get createSharedSiteRateLimiter (key site:<id>), shared by api / worker / worker:agent, with an in-process serial queue in front and a local fallback when the bucket is unavailable. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
57346d438c
commit
61f585a707
12 files changed
+466
-5
No files matched your search
@@ -1,5 +1,6 @@
|
||||
import {
|
||||
createRssFetcher,
|
||||
createSharedSiteRateLimiter,
|
||||
createSiteAdapter,
|
||||
createSiteRateLimiter,
|
||||
createTmdbClient,
|
||||
@@ -12,10 +13,10 @@ import {
|
||||
type TmdbClient,
|
||||
} from "@umt/adapters";
|
||||
import { DEFAULT_SETTINGS, type SitesSettings } from "@umt/contracts";
|
||||
import type { SiteRow } from "@umt/db";
|
||||
import { pgTokenBucketFor, type SiteRow } from "@umt/db";
|
||||
import type { ServerContext } from "../types.ts";
|
||||
|
||||
// 模块运行时依赖:站点适配器与限流器(每站一个实例)、RSS 抓取器、TMDB 客户端、时钟。
|
||||
// 模块运行时依赖:站点适配器与限流器(每站一个实例,令牌在 PG 跨进程共享)、RSS 抓取器、TMDB 客户端、时钟。
|
||||
// 测试通过 createSourcesIdentityModule(overrides) 注入 fetch / TMDB / 时钟。
|
||||
|
||||
export interface RuntimeOverrides {
|
||||
@@ -28,6 +29,8 @@ export interface RuntimeOverrides {
|
||||
rss?: Parameters<typeof createRssFetcher>[0];
|
||||
/** 调度 tick 间隔(毫秒),默认 30s */
|
||||
schedulerIntervalMs?: number;
|
||||
/** false = 站点限流只用进程内令牌桶(缺省走 PG 跨进程令牌桶 rate_limit_bucket) */
|
||||
sharedRateLimit?: boolean;
|
||||
}
|
||||
|
||||
interface CachedAdapter {
|
||||
@@ -158,7 +161,21 @@ export class SourcesRuntime {
|
||||
]);
|
||||
const hit = this.adapters.get(site.id);
|
||||
if (hit && hit.sig === sig) return hit.adapter;
|
||||
const limiter = createSiteRateLimiter(limiterOptions(def, site.rate_limit, defaults.defaultRateLimit));
|
||||
const lo = limiterOptions(def, site.rate_limit, defaults.defaultRateLimit);
|
||||
// 跨进程:api / worker / worker:agent 对同一站点共用 PG 令牌桶(key = site:<id>);进程内先串行排队
|
||||
const limiter =
|
||||
this.overrides.sharedRateLimit === false
|
||||
? createSiteRateLimiter(lo)
|
||||
: createSharedSiteRateLimiter({
|
||||
...lo,
|
||||
key: `site:${site.id}`,
|
||||
bucket: pgTokenBucketFor(this.ctx.db),
|
||||
onError: (err) =>
|
||||
this.ctx.logger.warn(
|
||||
{ siteId: site.id, err: { message: (err as Error).message } },
|
||||
"站点共享限流不可用,本次退回进程内限流",
|
||||
),
|
||||
});
|
||||
const cfg = (site.config ?? {}) as { apiBaseUrl?: string };
|
||||
const baseUrl =
|
||||
def.umt.kind === "mteam" ? cfg.apiBaseUrl : sameUrl(site.base_url, def.domain) ? undefined : site.base_url;
|
||||
|
||||
Reference in new issue
Block a user