feat(adapters): RSS and PT site adapters ported from MoviePilot v3 + mocks (W1-3b)

- rss: port of MP RssHelper._parse_impl/get_rss_link semantics with safe
  XML (entities off, DTD/ENTITY rejected, size/item caps), ETag/Last-Modified,
  backoff + Retry-After, circuit breaker with manual reset, passkey redaction,
  overlap-window GUID+digest dedupe
- sites/spider: TS port of MP SiteSpider (indexer schema, search URL builder,
  field DSL, filters, Jinja templates via nunjucks, login-page detection);
  audiences definition hand-written as JSON; raw category id preserved;
  shared per-site token bucket
- sites/mteam: port of MP MTorrentSpider (params, result mapping, promo
  factors, genDlToken flow) without embedding the API key in enclosures;
  business code/message checks
- testkit: mock RSS server, mock NexusPHP site, mock M-Team API
- docs: W1-3b migration mapping appended to porting ledger

Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
rootandClaude Fable 5.1 committed 2026-09-27 18:10:34 +08:00
1 parent ef2e504a94
commit a6d049db81
47 files changed
+5414

No files matched your search

+95
View File
@@ -0,0 +1,95 @@
/**
* 站点级令牌桶:同一站点的列表/搜索/详情/下载(以及将来的浏览器操作)共享一个实例。
* 容量 burst,每 minIntervalMs 补 1 个令牌;429/Retry-After 通过 penalize 整体顺延。
*/
export interface SiteRateLimiter {
acquire(signal?: AbortSignal): Promise<void>;
/** 不等待:有令牌返回 true 并消耗 */
tryAcquire(): boolean;
/** 站点要求暂停(Retry-After 等),在 ms 内不发放令牌 */
penalize(ms: number): void;
readonly stats: { granted: number; waitedMs: number };
}
export interface RateLimiterOptions {
minIntervalMs: number;
burst?: number;
now?: () => number;
sleep?: (ms: number, signal?: AbortSignal) => Promise<void>;
}
function defaultSleep(ms: number, signal?: AbortSignal): Promise<void> {
return new Promise((resolve, reject) => {
if (signal?.aborted) {
reject(signal.reason);
return;
}
const t = setTimeout(() => {
signal?.removeEventListener("abort", onAbort);
resolve();
}, ms);
const onAbort = () => {
clearTimeout(t);
reject(signal?.reason);
};
signal?.addEventListener("abort", onAbort, { once: true });
});
}
export function createSiteRateLimiter(opts: RateLimiterOptions): SiteRateLimiter {
const interval = Math.max(0, opts.minIntervalMs);
const burst = Math.max(1, opts.burst ?? 1);
const now = opts.now ?? Date.now;
const sleep = opts.sleep ?? defaultSleep;
let tokens = burst;
let last = now();
let blockedUntil = 0;
let queue: Promise<void> = Promise.resolve();
const stats = { granted: 0, waitedMs: 0 };
const refill = () => {
const t = now();
if (interval === 0) tokens = burst;
else tokens = Math.min(burst, tokens + (t - last) / interval);
last = t;
};
const waitMs = (): number => {
refill();
const t = now();
if (t < blockedUntil) return blockedUntil - t;
if (tokens >= 1) return 0;
return Math.ceil((1 - tokens) * interval);
};
return {
stats,
tryAcquire() {
if (waitMs() > 0) return false;
tokens -= 1;
stats.granted++;
return true;
},
penalize(ms) {
blockedUntil = Math.max(blockedUntil, now() + ms);
tokens = Math.min(tokens, 0);
},
acquire(signal) {
// 串行排队,保证先到先得、不会并发抢同一个令牌
const run = queue.then(async () => {
for (;;) {
signal?.throwIfAborted();
const w = waitMs();
if (w <= 0) break;
stats.waitedMs += w;
await sleep(w, signal);
}
tokens -= 1;
stats.granted++;
});
queue = run.catch(() => {});
return run;
},
};
}