120 lines
4.6 KiB
TypeScript
120 lines
4.6 KiB
TypeScript
import { redactText } from "../rss/redact.ts";
|
|
import { CHALLENGE_OUTCOME_HEADER, challengeOutcomeText, detectChallenge, isPageRequest } from "./challenge.ts";
|
|
import { buildFailureSample } from "./failure-sample.ts";
|
|
import type { SiteRateLimiter } from "./rate-limit.ts";
|
|
import { SiteError } from "./types.ts";
|
|
|
|
export interface SiteHttpContext {
|
|
siteId: string;
|
|
fetch: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
|
limiter: SiteRateLimiter;
|
|
timeoutMs: number;
|
|
userAgent?: string;
|
|
now: () => Date;
|
|
}
|
|
|
|
function retryAfterMs(res: Response, now: Date): number | undefined {
|
|
const v = res.headers.get("retry-after");
|
|
if (!v) return undefined;
|
|
if (/^\d+$/.test(v.trim())) return Number(v.trim()) * 1000;
|
|
const t = Date.parse(v);
|
|
return Number.isNaN(t) ? undefined : Math.max(0, t - now.getTime());
|
|
}
|
|
|
|
/**
|
|
* 站点请求:先取站点令牌,再发请求;Cloudflare 挑战(`cf-mitigated: challenge`,或 403 / 503 / 429 的挑战页,
|
|
* 或页面请求返回 200 的挑战页 / 人机验证门页)→ `challenge`(附脱敏页面样本)
|
|
* (不是凭据错误,CF-2);其余 429/503 统一映射并顺延限流器。
|
|
* 返回的 Response 由调用方消费。错误消息只含脱敏后的 URL。
|
|
*/
|
|
export async function siteFetch(
|
|
ctx: SiteHttpContext,
|
|
url: string,
|
|
init: RequestInit & { signal?: AbortSignal } = {},
|
|
): Promise<Response> {
|
|
await ctx.limiter.acquire(init.signal ?? undefined);
|
|
const timeout = AbortSignal.timeout(ctx.timeoutMs);
|
|
const signal = init.signal ? AbortSignal.any([init.signal, timeout]) : timeout;
|
|
const headers = new Headers(init.headers);
|
|
if (ctx.userAgent && !headers.has("user-agent")) headers.set("user-agent", ctx.userAgent);
|
|
const safeUrl = redactText(url);
|
|
let res: Response;
|
|
try {
|
|
res = await ctx.fetch(url, { ...init, headers, signal });
|
|
} catch (e) {
|
|
if (init.signal?.aborted) throw e;
|
|
const reason = timeout.aborted ? "超时" : (e as Error).message;
|
|
throw new SiteError(ctx.siteId, "network", redactText(`请求 ${safeUrl} 失败:${reason}`));
|
|
}
|
|
// 页面类请求的 200 响应也看正文:返回 200 的挑战页 / 人机验证门页(只读开头,有上限)
|
|
const ch = await detectChallenge(res, { page: isPageRequest(init.method, headers.get("accept")) });
|
|
res = ch.res;
|
|
if (ch.rule) {
|
|
await res.body?.cancel().catch(() => {});
|
|
const outcome = res.headers.get(CHALLENGE_OUTCOME_HEADER);
|
|
const message = `请求 ${safeUrl}：站点要求人机验证（Cloudflare 等），${challengeOutcomeText(outcome)}`;
|
|
throw new SiteError(ctx.siteId, "challenge", message, {
|
|
httpStatus: res.status,
|
|
detail: { reason: "challenge", rule: ch.rule, httpStatus: res.status, ...(outcome ? { outcome } : {}) },
|
|
sample: buildFailureSample({
|
|
kind: "challenge",
|
|
requestKind: "other",
|
|
url,
|
|
httpStatus: res.status,
|
|
headers: res.headers,
|
|
message,
|
|
rule: ch.rule,
|
|
html: ch.html,
|
|
now: ctx.now(),
|
|
}),
|
|
});
|
|
}
|
|
if (res.status === 429 || res.status === 503) {
|
|
const ra = retryAfterMs(res, ctx.now());
|
|
await res.body?.cancel().catch(() => {});
|
|
if (ra !== undefined) ctx.limiter.penalize(ra);
|
|
throw new SiteError(
|
|
ctx.siteId,
|
|
res.status === 429 ? "rate_limited" : "site_maintenance",
|
|
`请求 ${safeUrl} 返回 HTTP ${res.status}`,
|
|
{ httpStatus: res.status, ...(ra !== undefined ? { retryAfterMs: ra } : {}) },
|
|
);
|
|
}
|
|
if (res.status >= 500) {
|
|
await res.body?.cancel().catch(() => {});
|
|
throw new SiteError(ctx.siteId, "site_maintenance", `请求 ${safeUrl} 返回 HTTP ${res.status}`, {
|
|
httpStatus: res.status,
|
|
});
|
|
}
|
|
return res;
|
|
}
|
|
|
|
/** 读取响应体并限制大小(默认 16 MiB,足够容纳 .torrent 与列表页)。 */
|
|
export async function readBody(ctx: SiteHttpContext, res: Response, maxBytes = 16 * 1024 * 1024): Promise<Uint8Array> {
|
|
if (!res.body) return new Uint8Array();
|
|
const reader = res.body.getReader();
|
|
const chunks: Uint8Array[] = [];
|
|
let total = 0;
|
|
for (;;) {
|
|
const { done, value } = await reader.read();
|
|
if (done) break;
|
|
total += value.byteLength;
|
|
if (total > maxBytes) {
|
|
await reader.cancel().catch(() => {});
|
|
throw new SiteError(ctx.siteId, "parse_error", `响应超过 ${maxBytes} 字节上限`);
|
|
}
|
|
chunks.push(value);
|
|
}
|
|
const out = new Uint8Array(total);
|
|
let off = 0;
|
|
for (const c of chunks) {
|
|
out.set(c, off);
|
|
off += c.byteLength;
|
|
}
|
|
return out;
|
|
}
|
|
|
|
export async function readText(ctx: SiteHttpContext, res: Response): Promise<string> {
|
|
return new TextDecoder("utf-8").decode(await readBody(ctx, res));
|
|
}
|