fix(adapters): 识别 200 状态的挑战 / 人机验证页,解析失败前兜底,失败页面留样
- detectChallenge 支持页面类请求(GET + Accept text/html)的 2xx 响应:正文只先读 256 KB, 短页且没有种子表 / 用户信息块时,认 Cloudflare 挑战标题 / 脚本与 Turnstile / hCaptcha / reCAPTCHA 门页; 带登录表单的页面不算门页。非页面请求不读正文 - siteFetch 用上面的判定;挑战错误附脱敏页面样本,文案带级联结果(x-umt-challenge) - spider 适配器:认不出列表 / 详情时先看是不是人机验证页,是就报 challenge(「搜索页需要人机验证」), 不再报改版;parse_error / challenge 都经 deps.onFailureSample 交出脱敏样本(前 64 KB,去掉个人信息块、 passkey / authkey / uid / 用户名 / Cookie / 邮箱) - classifyChallengePage 也认门页(浏览器过挑战时不再把门页当已通过) Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
4fa001750b
commit
1779e1a0f2
11 files changed
+2078
-34
No files matched your search
@@ -1,5 +1,6 @@
|
||||
// CF-2 Cloudflare 挑战识别(docs/design/cloudflare-handling.md)。
|
||||
// - HTTP 端:响应头 `cf-mitigated: challenge`(任何状态码),或 403 / 503 / 429 且正文是挑战页 → 挑战,不是凭据错误;
|
||||
// 页面类请求(GET + Accept text/html)返回 200 也可能是挑战页 / 人机验证门页(Turnstile、hCaptcha、reCAPTCHA),短页且没有站点正常内容才算;
|
||||
// - 浏览器端:挑战页分三类——自动(JS 检查,等几秒自己过)、可点(「我是人类」复选框 / Turnstile)、真验证码(选图 / 输码)。
|
||||
// 只用正则,不解析 DOM:HTTP 端要在限流器之后快速判断,浏览器端拿到的是 outerHTML + innerText 样本。
|
||||
|
||||
@@ -27,21 +28,168 @@ export function challengeRuleOfHeaders(headers: Headers): string | null {
|
||||
return headers.get("cf-mitigated")?.trim().toLowerCase() === "challenge" ? "header cf-mitigated=challenge" : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 站点请求经服务端 Cloudflare 级联后仍是挑战时,级联在返回的响应上加这个头说明处理结果
|
||||
* (needs_human / failed / throttled / no_browser / no_account),适配器据此写错误文案;
|
||||
* unhandled = 网络层没认出、适配器解析时才认出的验证页(级联没跑)。
|
||||
*/
|
||||
export const CHALLENGE_OUTCOME_HEADER = "x-umt-challenge";
|
||||
|
||||
/** 人机验证错误文案的后半句:自动处理到哪一步、要不要人 */
|
||||
export function challengeOutcomeText(outcome: string | null | undefined): string {
|
||||
switch (outcome) {
|
||||
case "needs_human":
|
||||
return "自动处理没能通过，需要你在原站视图里过一次验证";
|
||||
case "failed":
|
||||
return "已自动尝试用浏览器通过，没有成功";
|
||||
case "throttled":
|
||||
return "30 分钟内已自动尝试过，之后会再试";
|
||||
case "no_browser":
|
||||
return "没有可用的浏览器，无法自动通过";
|
||||
case "no_account":
|
||||
return "站点没有已登录的账号，不自动处理";
|
||||
case "unhandled":
|
||||
return "这类页面没有自动处理，可以在原站视图里过一次验证";
|
||||
default:
|
||||
return "会自动尝试用浏览器通过";
|
||||
}
|
||||
}
|
||||
|
||||
/** 这些状态码的响应才去看正文(Cloudflare 挑战页是 403 / 503,少数 429) */
|
||||
export const CHALLENGE_STATUSES: readonly number[] = [403, 429, 503];
|
||||
|
||||
/** 判定时最多先读这么多正文;超出部分不缓冲,照常流给调用方。挑战 / 人机验证门页都远小于这个 */
|
||||
export const CHALLENGE_PEEK_BYTES = 256 * 1024;
|
||||
|
||||
/** 页面类请求(GET + Accept text/html):只有这类请求的 2xx 响应才检查正文 */
|
||||
export function isPageRequest(method: string | null | undefined, accept: string | null | undefined): boolean {
|
||||
return (method ?? "GET").toUpperCase() === "GET" && /text\/html|application\/xhtml/i.test(accept ?? "");
|
||||
}
|
||||
|
||||
/** 人机验证组件:Turnstile / hCaptcha / reCAPTCHA(只认脚本地址与组件 class,不认文案) */
|
||||
const GATE_WIDGET =
|
||||
/challenges\.cloudflare\.com\/turnstile|class="[^"]*\bcf-turnstile\b|hcaptcha\.com\/1\/api|class="[^"]*\bh-captcha\b|google\.com\/recaptcha|recaptcha\/api\.js|class="[^"]*\bg-recaptcha\b/i;
|
||||
/** 站点正常内容的痕迹:种子表、已登录的用户信息块(NexusPHP 等)。出现任何一个就不是门页 */
|
||||
const SITE_CONTENT =
|
||||
/id="torrenttable"|<table[^>]*class="[^"]*\btorrents\b|logout\.php|userdetails\.php\?id=|usercp\.php|id="info_block"/i;
|
||||
/** 登录表单(登录页本身也可能带 Turnstile,交给登录 / 失效判定,不算门页) */
|
||||
const LOGIN_FORM = /type=["']?password|takelogin\.php/i;
|
||||
/** 门页都很短;超过这个长度的页面不按门页判 */
|
||||
const GATE_MAX_CHARS = 100_000;
|
||||
/** 页面文字里的人机验证提示(只在页面已经认不出来时用) */
|
||||
const HUMAN_TEXT = /人机验证|verify you are human|确认您是真人|验证您是真人|checking your browser/i;
|
||||
|
||||
/**
|
||||
* 判定一个站点响应是不是挑战。正文只在状态码可能是挑战时读(读过的正文重新包成 Response 返回,调用方照常消费)。
|
||||
* 返回 { res, rule }:rule 为 null 表示不是挑战。
|
||||
* 人机验证门页:页面上有 Turnstile / hCaptcha / reCAPTCHA,且没有登录表单、没有站点正常内容、页面很短。
|
||||
* 命中返回规则说明,否则 null。
|
||||
*/
|
||||
export async function detectChallenge(res: Response): Promise<{ res: Response; rule: string | null }> {
|
||||
export function challengeGateRule(html: string | null | undefined): string | null {
|
||||
if (!html || html.length > GATE_MAX_CHARS) return null;
|
||||
const w = GATE_WIDGET.exec(html);
|
||||
if (!w || LOGIN_FORM.test(html) || SITE_CONTENT.test(html)) return null;
|
||||
return `gate*=${w[0].slice(0, 40)}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* 2xx 页面是不是挑战 / 人机验证页。比非 2xx 严:正文必须完整读到(短页)、没有站点正常内容,
|
||||
* 再看挑战标题 / 脚本特征或人机验证门页。complete = 正文是否已全部读到。
|
||||
*/
|
||||
export function challengeRuleOfOkPage(html: string, complete = true): string | null {
|
||||
if (!complete || !html || html.length > GATE_MAX_CHARS || SITE_CONTENT.test(html)) return null;
|
||||
return challengeRuleOfHtml(html) ?? challengeGateRule(html);
|
||||
}
|
||||
|
||||
/**
|
||||
* 页面已经解析不出来(列表 / 详情认不出结构)时的兜底:像挑战 / 人机验证就返回规则说明。
|
||||
* 比 challengeRuleOfOkPage 宽:不要求短页、也认页面文字里的「人机验证」提示,但带登录表单的页面不算。
|
||||
*/
|
||||
export function challengeRuleOfUnrecognizedPage(html: string | null | undefined): string | null {
|
||||
if (!html) return null;
|
||||
const head = html.slice(0, 300_000);
|
||||
const cf = challengeRuleOfHtml(head);
|
||||
if (cf) return cf;
|
||||
if (LOGIN_FORM.test(head)) return null;
|
||||
const w = GATE_WIDGET.exec(head);
|
||||
if (w) return `gate*=${w[0].slice(0, 40)}`;
|
||||
const t = HUMAN_TEXT.exec(head.replace(/<script[\s\S]*?<\/script>|<style[\s\S]*?<\/style>/gi, " "));
|
||||
return t ? `text*=${t[0]}` : null;
|
||||
}
|
||||
|
||||
function concatBytes(chunks: Uint8Array[], total: number): Uint8Array<ArrayBuffer> {
|
||||
const out = new Uint8Array(total);
|
||||
let off = 0;
|
||||
for (const c of chunks) {
|
||||
out.set(c, off);
|
||||
off += c.byteLength;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** 先读最多 max 字节;返回读到的部分、是否读完,以及一个内容不变的新响应(没读完的部分接着原流) */
|
||||
async function peekBody(
|
||||
res: Response,
|
||||
max: number,
|
||||
): Promise<{ head: Uint8Array<ArrayBuffer>; complete: boolean; res: Response }> {
|
||||
const reader = (res.body as ReadableStream<Uint8Array>).getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
let total = 0;
|
||||
let complete = false;
|
||||
while (total < max) {
|
||||
const r = await reader.read();
|
||||
if (r.done) {
|
||||
complete = true;
|
||||
break;
|
||||
}
|
||||
chunks.push(r.value);
|
||||
total += r.value.byteLength;
|
||||
}
|
||||
const head = concatBytes(chunks, total);
|
||||
const init = { status: res.status, statusText: res.statusText, headers: res.headers };
|
||||
if (complete) return { head, complete, res: new Response(head, init) };
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
start(c) {
|
||||
c.enqueue(head);
|
||||
},
|
||||
async pull(c) {
|
||||
const r = await reader.read();
|
||||
if (r.done) c.close();
|
||||
else c.enqueue(r.value);
|
||||
},
|
||||
cancel(reason) {
|
||||
return reader.cancel(reason);
|
||||
},
|
||||
});
|
||||
return { head, complete, res: new Response(stream, init) };
|
||||
}
|
||||
|
||||
/**
|
||||
* 判定一个站点响应是不是挑战。
|
||||
* - 响应头 `cf-mitigated: challenge` → 挑战(不读正文);
|
||||
* - 403 / 429 / 503 → 读正文看挑战页特征;
|
||||
* - opts.page(页面类请求)且 2xx 的 HTML → 读正文看挑战页 / 人机验证门页(challengeRuleOfOkPage);
|
||||
* - 其它情况不读正文,零开销。
|
||||
* 正文最多先读 CHALLENGE_PEEK_BYTES,读过的部分重新包进返回的响应,调用方照常消费。
|
||||
* 返回 { res, rule, html }:rule 为 null 表示不是挑战;html 是读到的正文开头(没读为 null,留样用)。
|
||||
*/
|
||||
export async function detectChallenge(
|
||||
res: Response,
|
||||
opts: { page?: boolean } = {},
|
||||
): Promise<{ res: Response; rule: string | null; html: string | null }> {
|
||||
const byHeader = challengeRuleOfHeaders(res.headers);
|
||||
if (byHeader) return { res, rule: byHeader };
|
||||
if (!CHALLENGE_STATUSES.includes(res.status) || !res.body) return { res, rule: null };
|
||||
const bytes = new Uint8Array(await res.arrayBuffer());
|
||||
const html = new TextDecoder("utf-8").decode(bytes.subarray(0, 300_000));
|
||||
const copy = new Response(bytes, { status: res.status, statusText: res.statusText, headers: res.headers });
|
||||
return { res: copy, rule: challengeRuleOfHtml(html) };
|
||||
if (byHeader) return { res, rule: byHeader, html: null };
|
||||
const suspect = CHALLENGE_STATUSES.includes(res.status);
|
||||
const okPage =
|
||||
!!opts.page &&
|
||||
res.status >= 200 &&
|
||||
res.status < 300 &&
|
||||
res.status !== 204 &&
|
||||
res.status !== 205 &&
|
||||
/html|^$/i.test(res.headers.get("content-type")?.trim() ?? "");
|
||||
if ((!suspect && !okPage) || !res.body) return { res, rule: null, html: null };
|
||||
const peek = await peekBody(res, CHALLENGE_PEEK_BYTES);
|
||||
const html = new TextDecoder("utf-8").decode(peek.head);
|
||||
const rule = suspect ? challengeRuleOfHtml(html) : challengeRuleOfOkPage(html, peek.complete);
|
||||
return { res: peek.res, rule, html };
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -68,6 +216,7 @@ export function classifyChallengePage(page: { title?: string | null; html?: stri
|
||||
const rule =
|
||||
(CHALLENGE_TITLE.test(title) ? `title=${title.slice(0, 40)}` : null) ??
|
||||
challengeRuleOfHtml(html) ??
|
||||
challengeGateRule(html) ??
|
||||
(/verify you are human|确认您是真人|验证您是真人|checking (?:if the site connection is secure|your browser)/i.test(
|
||||
text,
|
||||
)
|
||||
|
||||
Reference in new issue
Block a user