import { redactText } from "../rss/redact.ts"; import { detectChallenge } from "./challenge.ts"; import type { SiteRateLimiter } from "./rate-limit.ts"; import { SiteError } from "./types.ts"; export interface SiteHttpContext { siteId: string; fetch: typeof globalThis.fetch; limiter: SiteRateLimiter; timeoutMs: number; userAgent?: string; now: () => Date; } function retryAfterMs(res: Response, now: Date): number | undefined { const v = res.headers.get("retry-after"); if (!v) return undefined; if (/^\d+$/.test(v.trim())) return Number(v.trim()) * 1000; const t = Date.parse(v); return Number.isNaN(t) ? undefined : Math.max(0, t - now.getTime()); } /** * 站点请求:先取站点令牌,再发请求;Cloudflare 挑战(`cf-mitigated: challenge`,或 403 / 503 / 429 的挑战页)→ `challenge` * (不是凭据错误,CF-2);其余 429/503 统一映射并顺延限流器。 * 返回的 Response 由调用方消费。错误消息只含脱敏后的 URL。 */ export async function siteFetch( ctx: SiteHttpContext, url: string, init: RequestInit & { signal?: AbortSignal } = {}, ): Promise { await ctx.limiter.acquire(init.signal ?? undefined); const timeout = AbortSignal.timeout(ctx.timeoutMs); const signal = init.signal ? AbortSignal.any([init.signal, timeout]) : timeout; const headers = new Headers(init.headers); if (ctx.userAgent && !headers.has("user-agent")) headers.set("user-agent", ctx.userAgent); const safeUrl = redactText(url); let res: Response; try { res = await ctx.fetch(url, { ...init, headers, signal }); } catch (e) { if (init.signal?.aborted) throw e; const reason = timeout.aborted ? "超时" : (e as Error).message; throw new SiteError(ctx.siteId, "network", redactText(`请求 ${safeUrl} 失败:${reason}`)); } const ch = await detectChallenge(res); res = ch.res; if (ch.rule) { await res.body?.cancel().catch(() => {}); throw new SiteError(ctx.siteId, "challenge", `请求 ${safeUrl}:站点要求人机验证(Cloudflare)`, { httpStatus: res.status, detail: { reason: "challenge", rule: ch.rule, httpStatus: res.status }, }); } if (res.status === 429 || res.status === 503) { const ra = retryAfterMs(res, ctx.now()); await res.body?.cancel().catch(() => {}); if (ra !== undefined) ctx.limiter.penalize(ra); throw new SiteError( ctx.siteId, res.status === 429 ? "rate_limited" : "site_maintenance", `请求 ${safeUrl} 返回 HTTP ${res.status}`, { httpStatus: res.status, ...(ra !== undefined ? { retryAfterMs: ra } : {}) }, ); } if (res.status >= 500) { await res.body?.cancel().catch(() => {}); throw new SiteError(ctx.siteId, "site_maintenance", `请求 ${safeUrl} 返回 HTTP ${res.status}`, { httpStatus: res.status, }); } return res; } /** 读取响应体并限制大小(默认 16 MiB,足够容纳 .torrent 与列表页)。 */ export async function readBody(ctx: SiteHttpContext, res: Response, maxBytes = 16 * 1024 * 1024): Promise { if (!res.body) return new Uint8Array(); const reader = res.body.getReader(); const chunks: Uint8Array[] = []; let total = 0; for (;;) { const { done, value } = await reader.read(); if (done) break; total += value.byteLength; if (total > maxBytes) { await reader.cancel().catch(() => {}); throw new SiteError(ctx.siteId, "parse_error", `响应超过 ${maxBytes} 字节上限`); } chunks.push(value); } const out = new Uint8Array(total); let off = 0; for (const c of chunks) { out.set(c, off); off += c.byteLength; } return out; } export async function readText(ctx: SiteHttpContext, res: Response): Promise { return new TextDecoder("utf-8").decode(await readBody(ctx, res)); }