feat(adapters): RSS and PT site adapters ported from MoviePilot v3 + mocks (W1-3b)
- rss: port of MP RssHelper._parse_impl/get_rss_link semantics with safe XML (entities off, DTD/ENTITY rejected, size/item caps), ETag/Last-Modified, backoff + Retry-After, circuit breaker with manual reset, passkey redaction, overlap-window GUID+digest dedupe - sites/spider: TS port of MP SiteSpider (indexer schema, search URL builder, field DSL, filters, Jinja templates via nunjucks, login-page detection); audiences definition hand-written as JSON; raw category id preserved; shared per-site token bucket - sites/mteam: port of MP MTorrentSpider (params, result mapping, promo factors, genDlToken flow) without embedding the API key in enclosures; business code/message checks - testkit: mock RSS server, mock NexusPHP site, mock M-Team API - docs: W1-3b migration mapping appended to porting ledger Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
1 parent
ef2e504a94
commit
a6d049db81
47 files changed
+5414
No files matched your search
@@ -0,0 +1,95 @@
|
||||
/**
|
||||
* 站点级令牌桶:同一站点的列表/搜索/详情/下载(以及将来的浏览器操作)共享一个实例。
|
||||
* 容量 burst,每 minIntervalMs 补 1 个令牌;429/Retry-After 通过 penalize 整体顺延。
|
||||
*/
|
||||
|
||||
export interface SiteRateLimiter {
|
||||
acquire(signal?: AbortSignal): Promise<void>;
|
||||
/** 不等待:有令牌返回 true 并消耗 */
|
||||
tryAcquire(): boolean;
|
||||
/** 站点要求暂停(Retry-After 等),在 ms 内不发放令牌 */
|
||||
penalize(ms: number): void;
|
||||
readonly stats: { granted: number; waitedMs: number };
|
||||
}
|
||||
|
||||
export interface RateLimiterOptions {
|
||||
minIntervalMs: number;
|
||||
burst?: number;
|
||||
now?: () => number;
|
||||
sleep?: (ms: number, signal?: AbortSignal) => Promise<void>;
|
||||
}
|
||||
|
||||
function defaultSleep(ms: number, signal?: AbortSignal): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
if (signal?.aborted) {
|
||||
reject(signal.reason);
|
||||
return;
|
||||
}
|
||||
const t = setTimeout(() => {
|
||||
signal?.removeEventListener("abort", onAbort);
|
||||
resolve();
|
||||
}, ms);
|
||||
const onAbort = () => {
|
||||
clearTimeout(t);
|
||||
reject(signal?.reason);
|
||||
};
|
||||
signal?.addEventListener("abort", onAbort, { once: true });
|
||||
});
|
||||
}
|
||||
|
||||
export function createSiteRateLimiter(opts: RateLimiterOptions): SiteRateLimiter {
|
||||
const interval = Math.max(0, opts.minIntervalMs);
|
||||
const burst = Math.max(1, opts.burst ?? 1);
|
||||
const now = opts.now ?? Date.now;
|
||||
const sleep = opts.sleep ?? defaultSleep;
|
||||
let tokens = burst;
|
||||
let last = now();
|
||||
let blockedUntil = 0;
|
||||
let queue: Promise<void> = Promise.resolve();
|
||||
const stats = { granted: 0, waitedMs: 0 };
|
||||
|
||||
const refill = () => {
|
||||
const t = now();
|
||||
if (interval === 0) tokens = burst;
|
||||
else tokens = Math.min(burst, tokens + (t - last) / interval);
|
||||
last = t;
|
||||
};
|
||||
|
||||
const waitMs = (): number => {
|
||||
refill();
|
||||
const t = now();
|
||||
if (t < blockedUntil) return blockedUntil - t;
|
||||
if (tokens >= 1) return 0;
|
||||
return Math.ceil((1 - tokens) * interval);
|
||||
};
|
||||
|
||||
return {
|
||||
stats,
|
||||
tryAcquire() {
|
||||
if (waitMs() > 0) return false;
|
||||
tokens -= 1;
|
||||
stats.granted++;
|
||||
return true;
|
||||
},
|
||||
penalize(ms) {
|
||||
blockedUntil = Math.max(blockedUntil, now() + ms);
|
||||
tokens = Math.min(tokens, 0);
|
||||
},
|
||||
acquire(signal) {
|
||||
// 串行排队,保证先到先得、不会并发抢同一个令牌
|
||||
const run = queue.then(async () => {
|
||||
for (;;) {
|
||||
signal?.throwIfAborted();
|
||||
const w = waitMs();
|
||||
if (w <= 0) break;
|
||||
stats.waitedMs += w;
|
||||
await sleep(w, signal);
|
||||
}
|
||||
tokens -= 1;
|
||||
stats.granted++;
|
||||
});
|
||||
queue = run.catch(() => {});
|
||||
return run;
|
||||
},
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user