feat(adapters): RSS and PT site adapters ported from MoviePilot v3 + mocks (W1-3b)
- rss: port of MP RssHelper._parse_impl/get_rss_link semantics with safe XML (entities off, DTD/ENTITY rejected, size/item caps), ETag/Last-Modified, backoff + Retry-After, circuit breaker with manual reset, passkey redaction, overlap-window GUID+digest dedupe - sites/spider: TS port of MP SiteSpider (indexer schema, search URL builder, field DSL, filters, Jinja templates via nunjucks, login-page detection); audiences definition hand-written as JSON; raw category id preserved; shared per-site token bucket - sites/mteam: port of MP MTorrentSpider (params, result mapping, promo factors, genDlToken flow) without embedding the API key in enclosures; business code/message checks - testkit: mock RSS server, mock NexusPHP site, mock M-Team API - docs: W1-3b migration mapping appended to porting ledger Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
1 parent
ef2e504a94
commit
a6d049db81
47 files changed
+5414
No files matched your search
@@ -0,0 +1,92 @@
|
||||
import { redactText } from "../rss/redact";
|
||||
import type { SiteRateLimiter } from "./rate-limit";
|
||||
import { SiteError } from "./types";
|
||||
|
||||
export interface SiteHttpContext {
|
||||
siteId: string;
|
||||
fetch: typeof globalThis.fetch;
|
||||
limiter: SiteRateLimiter;
|
||||
timeoutMs: number;
|
||||
userAgent?: string;
|
||||
now: () => Date;
|
||||
}
|
||||
|
||||
function retryAfterMs(res: Response, now: Date): number | undefined {
|
||||
const v = res.headers.get("retry-after");
|
||||
if (!v) return undefined;
|
||||
if (/^\d+$/.test(v.trim())) return Number(v.trim()) * 1000;
|
||||
const t = Date.parse(v);
|
||||
return Number.isNaN(t) ? undefined : Math.max(0, t - now.getTime());
|
||||
}
|
||||
|
||||
/**
|
||||
* 站点请求:先取站点令牌,再发请求;429/503 统一映射并顺延限流器。
|
||||
* 返回的 Response 由调用方消费。错误消息只含脱敏后的 URL。
|
||||
*/
|
||||
export async function siteFetch(
|
||||
ctx: SiteHttpContext,
|
||||
url: string,
|
||||
init: RequestInit & { signal?: AbortSignal } = {},
|
||||
): Promise<Response> {
|
||||
await ctx.limiter.acquire(init.signal ?? undefined);
|
||||
const timeout = AbortSignal.timeout(ctx.timeoutMs);
|
||||
const signal = init.signal ? AbortSignal.any([init.signal, timeout]) : timeout;
|
||||
const headers = new Headers(init.headers);
|
||||
if (ctx.userAgent && !headers.has("user-agent")) headers.set("user-agent", ctx.userAgent);
|
||||
const safeUrl = redactText(url);
|
||||
let res: Response;
|
||||
try {
|
||||
res = await ctx.fetch(url, { ...init, headers, signal });
|
||||
} catch (e) {
|
||||
if (init.signal?.aborted) throw e;
|
||||
const reason = timeout.aborted ? "超时" : (e as Error).message;
|
||||
throw new SiteError(ctx.siteId, "network", redactText(`请求 ${safeUrl} 失败:${reason}`));
|
||||
}
|
||||
if (res.status === 429 || res.status === 503) {
|
||||
const ra = retryAfterMs(res, ctx.now());
|
||||
await res.body?.cancel().catch(() => {});
|
||||
if (ra !== undefined) ctx.limiter.penalize(ra);
|
||||
throw new SiteError(
|
||||
ctx.siteId,
|
||||
res.status === 429 ? "rate_limited" : "site_maintenance",
|
||||
`请求 ${safeUrl} 返回 HTTP ${res.status}`,
|
||||
{ httpStatus: res.status, ...(ra !== undefined ? { retryAfterMs: ra } : {}) },
|
||||
);
|
||||
}
|
||||
if (res.status >= 500) {
|
||||
await res.body?.cancel().catch(() => {});
|
||||
throw new SiteError(ctx.siteId, "site_maintenance", `请求 ${safeUrl} 返回 HTTP ${res.status}`, {
|
||||
httpStatus: res.status,
|
||||
});
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
/** 读取响应体并限制大小(默认 16 MiB,足够容纳 .torrent 与列表页)。 */
|
||||
export async function readBody(ctx: SiteHttpContext, res: Response, maxBytes = 16 * 1024 * 1024): Promise<Uint8Array> {
|
||||
if (!res.body) return new Uint8Array();
|
||||
const reader = res.body.getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
let total = 0;
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
total += value.byteLength;
|
||||
if (total > maxBytes) {
|
||||
await reader.cancel().catch(() => {});
|
||||
throw new SiteError(ctx.siteId, "parse_error", `响应超过 ${maxBytes} 字节上限`);
|
||||
}
|
||||
chunks.push(value);
|
||||
}
|
||||
const out = new Uint8Array(total);
|
||||
let off = 0;
|
||||
for (const c of chunks) {
|
||||
out.set(c, off);
|
||||
off += c.byteLength;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function readText(ctx: SiteHttpContext, res: Response): Promise<string> {
|
||||
return new TextDecoder("utf-8").decode(await readBody(ctx, res));
|
||||
}
|
||||
Reference in new issue
Block a user