feat(sites): cross-process site rate limit via PG token bucket

Migration 0041_rate_limit adds rate_limit_bucket. PgTokenBucket (@umt/db)
refills and takes tokens under SELECT ... FOR UPDATE on the DB clock and
polls with jitter within a wait budget; penalize() carries Retry-After.
Site adapters get createSharedSiteRateLimiter (key site:<id>), shared by
api / worker / worker:agent, with an in-process serial queue in front and
a local fallback when the bucket is unavailable.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

apps/server/src/modules/sources-identity/runtime.ts

@@ -1,5 +1,6 @@
 import {
   createRssFetcher,
+  createSharedSiteRateLimiter,
   createSiteAdapter,
   createSiteRateLimiter,
   createTmdbClient,
@@ -12,10 +13,10 @@ import {
   type TmdbClient,
 } from "@umt/adapters";
 import { DEFAULT_SETTINGS, type SitesSettings } from "@umt/contracts";
-import type { SiteRow } from "@umt/db";
+import { pgTokenBucketFor, type SiteRow } from "@umt/db";
 import type { ServerContext } from "../types.ts";
 
-// 模块运行时依赖:站点适配器与限流器(每站一个实例)、RSS 抓取器、TMDB 客户端、时钟。
+// 模块运行时依赖:站点适配器与限流器(每站一个实例,令牌在 PG 跨进程共享)、RSS 抓取器、TMDB 客户端、时钟。
 // 测试通过 createSourcesIdentityModule(overrides) 注入 fetch / TMDB / 时钟。
 
 export interface RuntimeOverrides {
@@ -28,6 +29,8 @@ export interface RuntimeOverrides {
   rss?: Parameters<typeof createRssFetcher>[0];
   /** 调度 tick 间隔(毫秒),默认 30s */
   schedulerIntervalMs?: number;
+  /** false = 站点限流只用进程内令牌桶(缺省走 PG 跨进程令牌桶 rate_limit_bucket) */
+  sharedRateLimit?: boolean;
 }
 
 interface CachedAdapter {
@@ -158,7 +161,21 @@ export class SourcesRuntime {
     ]);
     const hit = this.adapters.get(site.id);
     if (hit && hit.sig === sig) return hit.adapter;
-    const limiter = createSiteRateLimiter(limiterOptions(def, site.rate_limit, defaults.defaultRateLimit));
+    const lo = limiterOptions(def, site.rate_limit, defaults.defaultRateLimit);
+    // 跨进程:api / worker / worker:agent 对同一站点共用 PG 令牌桶(key = site:<id>);进程内先串行排队
+    const limiter =
+      this.overrides.sharedRateLimit === false
+        ? createSiteRateLimiter(lo)
+        : createSharedSiteRateLimiter({
+            ...lo,
+            key: `site:${site.id}`,
+            bucket: pgTokenBucketFor(this.ctx.db),
+            onError: (err) =>
+              this.ctx.logger.warn(
+                { siteId: site.id, err: { message: (err as Error).message } },
+                "站点共享限流不可用，本次退回进程内限流",
+              ),
+          });
     const cfg = (site.config ?? {}) as { apiBaseUrl?: string };
     const baseUrl =
       def.umt.kind === "mteam" ? cfg.apiBaseUrl : sameUrl(site.base_url, def.domain) ? undefined : site.base_url;

apps/server/test/sources-identity/api.test.ts

@@ -131,6 +131,19 @@ describe.skipIf(!up)("来源管理与站点工作台 API", () => {
     expect(observed.length).toBe(r.newCount);
     const site = await json(await h.req(`/sites/${siteId}`));
     expect(site.body.data.lastSnapshotAt).not.toBeNull();
+    // 限流间隔 > 0 时令牌在 PG 跨进程共享(rate_limit_bucket,key = site:<id>);0 = 不限流只走本地
+    await h.db
+      .updateTable("site")
+      .set({ rate_limit: JSON.stringify({ minIntervalMs: 5, concurrency: 5 }) })
+      .where("id", "=", siteId)
+      .execute();
+    expect((await snapshotSite(h.rt, { siteId, kind: "manual" })).status).toBe("ok");
+    const bucket = await h.db
+      .selectFrom("rate_limit_bucket")
+      .selectAll()
+      .where("key", "=", `site:${siteId}`)
+      .executeTakeFirst();
+    expect(bucket).toMatchObject({ capacity: 5, refill_per_sec: 200 });
   });
 
   it("工作台列表:快照优先,标注来源与更新时间;筛选与分页", async () => {