feat(sites): cross-process site rate limit via PG token bucket

Migration 0041_rate_limit adds rate_limit_bucket. PgTokenBucket (@umt/db)
refills and takes tokens under SELECT ... FOR UPDATE on the DB clock and
polls with jitter within a wait budget; penalize() carries Retry-After.
Site adapters get createSharedSiteRateLimiter (key site:<id>), shared by
api / worker / worker:agent, with an in-process serial queue in front and
a local fallback when the bucket is unavailable.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

packages/adapters/src/sites/index.ts

@@ -38,7 +38,14 @@ export {
   mteamSearchParams,
   parseMteamResult,
 } from "./mteam/spider.ts";
-export { createSiteRateLimiter, type RateLimiterOptions, type SiteRateLimiter } from "./rate-limit.ts";
+export {
+  createSharedSiteRateLimiter,
+  createSiteRateLimiter,
+  type RateLimiterOptions,
+  type SharedBucketPort,
+  type SharedRateLimiterOptions,
+  type SiteRateLimiter,
+} from "./rate-limit.ts";
 export {
   createSiteAdapter,
   getSiteAdapter,

packages/adapters/src/sites/rate-limit.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from "vitest";
-import { createSiteRateLimiter } from "./rate-limit.ts";
+import { createSharedSiteRateLimiter, createSiteRateLimiter, type SharedBucketPort } from "./rate-limit.ts";
 import { parseSizeMp as parseSize } from "./spider/spider.ts";
 import { parseCount, parseSiteTime } from "./util.ts";
 
@@ -34,6 +34,52 @@ describe("站点令牌桶", () => {
   });
 });
 
+describe("跨进程站点限流器", () => {
+  it("同进程串行去共享桶扣令牌,key 与速率按站点参数;Retry-After 同时顺延共享桶", async () => {
+    const calls: { key: string; capacity: number; refillPerSec: number }[] = [];
+    let inflight = 0;
+    let maxInflight = 0;
+    const penalties: number[] = [];
+    const bucket: SharedBucketPort = {
+      async acquire(key, _cost, o) {
+        inflight++;
+        maxInflight = Math.max(maxInflight, inflight);
+        await new Promise((r) => setTimeout(r, 5));
+        inflight--;
+        calls.push({ key, capacity: o.capacity, refillPerSec: o.refillPerSec });
+        return { ok: true, waitedMs: 5 };
+      },
+      async penalize(_k, ms) {
+        penalties.push(ms);
+      },
+    };
+    const l = createSharedSiteRateLimiter({ key: "site:x", bucket, minIntervalMs: 500, burst: 2 });
+    await Promise.all([l.acquire(), l.acquire(), l.acquire()]);
+    expect(maxInflight).toBe(1);
+    expect(calls).toEqual(Array(3).fill({ key: "site:x", capacity: 2, refillPerSec: 2 }));
+    expect(l.stats.granted).toBe(3);
+    l.penalize(3000);
+    expect(penalties).toEqual([3000]);
+  });
+
+  it("共享桶不可用时退回进程内令牌桶", async () => {
+    const errors: unknown[] = [];
+    const bucket: SharedBucketPort = {
+      acquire: () => Promise.reject(new Error("db down")),
+      penalize: () => Promise.reject(new Error("db down")),
+    };
+    const l = createSharedSiteRateLimiter({
+      key: "site:y",
+      bucket,
+      minIntervalMs: 10,
+      onError: (e) => errors.push(e),
+    });
+    await l.acquire();
+    expect(l.stats.granted).toBe(1);
+    expect(errors).toHaveLength(1);
+  });
+});
+
 describe("解析辅助", () => {
   it("parseSize/parseCount/parseSiteTime", () => {
     expect(parseSize("78.42 GB")).toBe(84202833838);

packages/adapters/src/sites/rate-limit.ts

@@ -93,3 +93,81 @@ export function createSiteRateLimiter(opts: RateLimiterOptions): SiteRateLimiter
     },
   };
 }
+
+/** 跨进程令牌桶端口(@umt/db 的 PgTokenBucket 结构上满足;adapters 不依赖 db 包) */
+export interface SharedBucketPort {
+  acquire(
+    key: string,
+    cost: number,
+    opts: { capacity: number; refillPerSec: number; waitMs?: number; signal?: AbortSignal },
+  ): Promise<{ ok: boolean; waitedMs: number }>;
+  penalize(key: string, ms: number, spec: { capacity: number; refillPerSec: number }): Promise<void>;
+}
+
+export interface SharedRateLimiterOptions extends RateLimiterOptions {
+  /** 桶 key,站点用 `site:<id>` */
+  key: string;
+  bucket: SharedBucketPort;
+  /** 单轮等待预算,缺省 60s;超出后重新排队(直到 signal 中止) */
+  waitMs?: number;
+  /** 共享桶出错(DB 不可达等)时回调;本次退回进程内令牌桶,不让站点请求失败 */
+  onError?: (err: unknown) => void;
+}
+
+/**
+ * 跨进程站点限流器:进程内先串行排队(同进程并发不去 PG 抢锁),再到共享桶扣令牌;
+ * 429 Retry-After 同时顺延本地与共享桶。minIntervalMs = 0(不限流)时只走本地。
+ */
+export function createSharedSiteRateLimiter(opts: SharedRateLimiterOptions): SiteRateLimiter {
+  const local = createSiteRateLimiter(opts);
+  const interval = Math.max(0, opts.minIntervalMs);
+  const spec = { capacity: Math.max(1, opts.burst ?? 1), refillPerSec: interval > 0 ? 1000 / interval : 0 };
+  const stats = { granted: 0, waitedMs: 0 };
+  let queue: Promise<void> = Promise.resolve();
+  const onError = opts.onError ?? (() => {});
+
+  const viaLocal = async (signal?: AbortSignal) => {
+    const before = local.stats.waitedMs;
+    await local.acquire(signal);
+    stats.waitedMs += local.stats.waitedMs - before;
+  };
+
+  return {
+    stats,
+    tryAcquire() {
+      if (!local.tryAcquire()) return false;
+      stats.granted++;
+      return true;
+    },
+    penalize(ms) {
+      local.penalize(ms);
+      if (interval > 0) opts.bucket.penalize(opts.key, ms, spec).catch(onError);
+    },
+    acquire(signal) {
+      const run = queue.then(async () => {
+        if (interval === 0) {
+          await viaLocal(signal);
+        } else {
+          try {
+            for (;;) {
+              const r = await opts.bucket.acquire(opts.key, 1, {
+                ...spec,
+                waitMs: opts.waitMs ?? 60_000,
+                ...(signal ? { signal } : {}),
+              });
+              stats.waitedMs += r.waitedMs;
+              if (r.ok) break;
+            }
+          } catch (e) {
+            if (signal?.aborted) throw e;
+            onError(e);
+            await viaLocal(signal);
+          }
+        }
+        stats.granted++;
+      });
+      queue = run.catch(() => {});
+      return run;
+    },
+  };
+}