import { redactText } from "../rss/redact.ts"; import { CHALLENGE_OUTCOME_HEADER, challengeOutcomeText, detectChallenge, isPageRequest } from "./challenge.ts"; import { buildFailureSample } from "./failure-sample.ts"; import type { SiteRateLimiter } from "./rate-limit.ts"; import { SiteError } from "./types.ts"; export interface SiteHttpContext { siteId: string; fetch: (input: RequestInfo | URL, init?: RequestInit) => Promise; limiter: SiteRateLimiter; timeoutMs: number; userAgent?: string; now: () => Date; } function retryAfterMs(res: Response, now: Date): number | undefined { const v = res.headers.get("retry-after"); if (!v) return undefined; if (/^\d+$/.test(v.trim())) return Number(v.trim()) * 1000; const t = Date.parse(v); return Number.isNaN(t) ? undefined : Math.max(0, t - now.getTime()); } /** * 站点请求:先取站点令牌,再发请求;Cloudflare 挑战(`cf-mitigated: challenge`,或 403 / 503 / 429 的挑战页, * 或页面请求返回 200 的挑战页 / 人机验证门页)→ `challenge`(附脱敏页面样本) * (不是凭据错误,CF-2);其余 429/503 统一映射并顺延限流器。 * 返回的 Response 由调用方消费。错误消息只含脱敏后的 URL。 */ export async function siteFetch( ctx: SiteHttpContext, url: string, init: RequestInit & { signal?: AbortSignal } = {}, ): Promise { await ctx.limiter.acquire(init.signal ?? undefined); const timeout = AbortSignal.timeout(ctx.timeoutMs); const signal = init.signal ? AbortSignal.any([init.signal, timeout]) : timeout; const headers = new Headers(init.headers); if (ctx.userAgent && !headers.has("user-agent")) headers.set("user-agent", ctx.userAgent); const safeUrl = redactText(url); let res: Response; try { res = await ctx.fetch(url, { ...init, headers, signal }); } catch (e) { if (init.signal?.aborted) throw e; const reason = timeout.aborted ? "超时" : (e as Error).message; throw new SiteError(ctx.siteId, "network", redactText(`请求 ${safeUrl} 失败:${reason}`)); } // 页面类请求的 200 响应也看正文:返回 200 的挑战页 / 人机验证门页(只读开头,有上限) const ch = await detectChallenge(res, { page: isPageRequest(init.method, headers.get("accept")) }); res = ch.res; if (ch.rule) { await res.body?.cancel().catch(() => {}); const outcome = res.headers.get(CHALLENGE_OUTCOME_HEADER); const message = `请求 ${safeUrl}:站点要求人机验证(Cloudflare 等),${challengeOutcomeText(outcome)}`; throw new SiteError(ctx.siteId, "challenge", message, { httpStatus: res.status, detail: { reason: "challenge", rule: ch.rule, httpStatus: res.status, ...(outcome ? { outcome } : {}) }, sample: buildFailureSample({ kind: "challenge", requestKind: "other", url, httpStatus: res.status, headers: res.headers, message, rule: ch.rule, html: ch.html, now: ctx.now(), }), }); } if (res.status === 429 || res.status === 503) { const ra = retryAfterMs(res, ctx.now()); await res.body?.cancel().catch(() => {}); if (ra !== undefined) ctx.limiter.penalize(ra); throw new SiteError( ctx.siteId, res.status === 429 ? "rate_limited" : "site_maintenance", `请求 ${safeUrl} 返回 HTTP ${res.status}`, { httpStatus: res.status, ...(ra !== undefined ? { retryAfterMs: ra } : {}) }, ); } if (res.status >= 500) { await res.body?.cancel().catch(() => {}); throw new SiteError(ctx.siteId, "site_maintenance", `请求 ${safeUrl} 返回 HTTP ${res.status}`, { httpStatus: res.status, }); } return res; } /** 读取响应体并限制大小(默认 16 MiB,足够容纳 .torrent 与列表页)。 */ export async function readBody(ctx: SiteHttpContext, res: Response, maxBytes = 16 * 1024 * 1024): Promise { if (!res.body) return new Uint8Array(); const reader = res.body.getReader(); const chunks: Uint8Array[] = []; let total = 0; for (;;) { const { done, value } = await reader.read(); if (done) break; total += value.byteLength; if (total > maxBytes) { await reader.cancel().catch(() => {}); throw new SiteError(ctx.siteId, "parse_error", `响应超过 ${maxBytes} 字节上限`); } chunks.push(value); } const out = new Uint8Array(total); let off = 0; for (const c of chunks) { out.set(c, off); off += c.byteLength; } return out; } export async function readText(ctx: SiteHttpContext, res: Response): Promise { return new TextDecoder("utf-8").decode(await readBody(ctx, res)); }