feat(adapters): RSS and PT site adapters ported from MoviePilot v3 + mocks (W1-3b)

- rss: port of MP RssHelper._parse_impl/get_rss_link semantics with safe
  XML (entities off, DTD/ENTITY rejected, size/item caps), ETag/Last-Modified,
  backoff + Retry-After, circuit breaker with manual reset, passkey redaction,
  overlap-window GUID+digest dedupe
- sites/spider: TS port of MP SiteSpider (indexer schema, search URL builder,
  field DSL, filters, Jinja templates via nunjucks, login-page detection);
  audiences definition hand-written as JSON; raw category id preserved;
  shared per-site token bucket
- sites/mteam: port of MP MTorrentSpider (params, result mapping, promo
  factors, genDlToken flow) without embedding the API key in enclosures;
  business code/message checks
- testkit: mock RSS server, mock NexusPHP site, mock M-Team API
- docs: W1-3b migration mapping appended to porting ledger

Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
rootandClaude Fable 5.1 committed 2026-09-27 18:10:34 +08:00
1 parent ef2e504a94
commit a6d049db81
47 files changed
+5414

No files matched your search

+92
View File
@@ -0,0 +1,92 @@
import { redactText } from "../rss/redact";
import type { SiteRateLimiter } from "./rate-limit";
import { SiteError } from "./types";
export interface SiteHttpContext {
siteId: string;
fetch: typeof globalThis.fetch;
limiter: SiteRateLimiter;
timeoutMs: number;
userAgent?: string;
now: () => Date;
}
function retryAfterMs(res: Response, now: Date): number | undefined {
const v = res.headers.get("retry-after");
if (!v) return undefined;
if (/^\d+$/.test(v.trim())) return Number(v.trim()) * 1000;
const t = Date.parse(v);
return Number.isNaN(t) ? undefined : Math.max(0, t - now.getTime());
}
/**
* 站点请求:先取站点令牌,再发请求;429/503 统一映射并顺延限流器。
* 返回的 Response 由调用方消费。错误消息只含脱敏后的 URL。
*/
export async function siteFetch(
ctx: SiteHttpContext,
url: string,
init: RequestInit & { signal?: AbortSignal } = {},
): Promise<Response> {
await ctx.limiter.acquire(init.signal ?? undefined);
const timeout = AbortSignal.timeout(ctx.timeoutMs);
const signal = init.signal ? AbortSignal.any([init.signal, timeout]) : timeout;
const headers = new Headers(init.headers);
if (ctx.userAgent && !headers.has("user-agent")) headers.set("user-agent", ctx.userAgent);
const safeUrl = redactText(url);
let res: Response;
try {
res = await ctx.fetch(url, { ...init, headers, signal });
} catch (e) {
if (init.signal?.aborted) throw e;
const reason = timeout.aborted ? "超时" : (e as Error).message;
throw new SiteError(ctx.siteId, "network", redactText(`请求 ${safeUrl} 失败:${reason}`));
}
if (res.status === 429 || res.status === 503) {
const ra = retryAfterMs(res, ctx.now());
await res.body?.cancel().catch(() => {});
if (ra !== undefined) ctx.limiter.penalize(ra);
throw new SiteError(
ctx.siteId,
res.status === 429 ? "rate_limited" : "site_maintenance",
`请求 ${safeUrl} 返回 HTTP ${res.status}`,
{ httpStatus: res.status, ...(ra !== undefined ? { retryAfterMs: ra } : {}) },
);
}
if (res.status >= 500) {
await res.body?.cancel().catch(() => {});
throw new SiteError(ctx.siteId, "site_maintenance", `请求 ${safeUrl} 返回 HTTP ${res.status}`, {
httpStatus: res.status,
});
}
return res;
}
/** 读取响应体并限制大小(默认 16 MiB,足够容纳 .torrent 与列表页)。 */
export async function readBody(ctx: SiteHttpContext, res: Response, maxBytes = 16 * 1024 * 1024): Promise<Uint8Array> {
if (!res.body) return new Uint8Array();
const reader = res.body.getReader();
const chunks: Uint8Array[] = [];
let total = 0;
for (;;) {
const { done, value } = await reader.read();
if (done) break;
total += value.byteLength;
if (total > maxBytes) {
await reader.cancel().catch(() => {});
throw new SiteError(ctx.siteId, "parse_error", `响应超过 ${maxBytes} 字节上限`);
}
chunks.push(value);
}
const out = new Uint8Array(total);
let off = 0;
for (const c of chunks) {
out.set(c, off);
off += c.byteLength;
}
return out;
}
export async function readText(ctx: SiteHttpContext, res: Response): Promise<string> {
return new TextDecoder("utf-8").decode(await readBody(ctx, res));
}