fix(adapters): 识别 200 状态的挑战 / 人机验证页,解析失败前兜底,失败页面留样
- detectChallenge 支持页面类请求(GET + Accept text/html)的 2xx 响应:正文只先读 256 KB, 短页且没有种子表 / 用户信息块时,认 Cloudflare 挑战标题 / 脚本与 Turnstile / hCaptcha / reCAPTCHA 门页; 带登录表单的页面不算门页。非页面请求不读正文 - siteFetch 用上面的判定;挑战错误附脱敏页面样本,文案带级联结果(x-umt-challenge) - spider 适配器:认不出列表 / 详情时先看是不是人机验证页,是就报 challenge(「搜索页需要人机验证」), 不再报改版;parse_error / challenge 都经 deps.onFailureSample 交出脱敏样本(前 64 KB,去掉个人信息块、 passkey / authkey / uid / 用户名 / Cookie / 邮箱) - classifyChallengePage 也认门页(浏览器过挑战时不再把门页当已通过) Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei
This commit is contained in:
1 parent
4fa001750b
commit
1779e1a0f2
11 files changed
+2078
-34
No files matched your search
@@ -1,5 +1,6 @@
|
||||
import { redactText } from "../rss/redact.ts";
|
||||
import { detectChallenge } from "./challenge.ts";
|
||||
import { CHALLENGE_OUTCOME_HEADER, challengeOutcomeText, detectChallenge, isPageRequest } from "./challenge.ts";
|
||||
import { buildFailureSample } from "./failure-sample.ts";
|
||||
import type { SiteRateLimiter } from "./rate-limit.ts";
|
||||
import { SiteError } from "./types.ts";
|
||||
|
||||
@@ -21,7 +22,8 @@ function retryAfterMs(res: Response, now: Date): number | undefined {
|
||||
}
|
||||
|
||||
/**
|
||||
* 站点请求:先取站点令牌,再发请求;Cloudflare 挑战(`cf-mitigated: challenge`,或 403 / 503 / 429 的挑战页)→ `challenge`
|
||||
* 站点请求:先取站点令牌,再发请求;Cloudflare 挑战(`cf-mitigated: challenge`,或 403 / 503 / 429 的挑战页,
|
||||
* 或页面请求返回 200 的挑战页 / 人机验证门页)→ `challenge`(附脱敏页面样本)
|
||||
* (不是凭据错误,CF-2);其余 429/503 统一映射并顺延限流器。
|
||||
* 返回的 Response 由调用方消费。错误消息只含脱敏后的 URL。
|
||||
*/
|
||||
@@ -44,13 +46,27 @@ export async function siteFetch(
|
||||
const reason = timeout.aborted ? "超时" : (e as Error).message;
|
||||
throw new SiteError(ctx.siteId, "network", redactText(`请求 ${safeUrl} 失败:${reason}`));
|
||||
}
|
||||
const ch = await detectChallenge(res);
|
||||
// 页面类请求的 200 响应也看正文:返回 200 的挑战页 / 人机验证门页(只读开头,有上限)
|
||||
const ch = await detectChallenge(res, { page: isPageRequest(init.method, headers.get("accept")) });
|
||||
res = ch.res;
|
||||
if (ch.rule) {
|
||||
await res.body?.cancel().catch(() => {});
|
||||
throw new SiteError(ctx.siteId, "challenge", `请求 ${safeUrl}:站点要求人机验证(Cloudflare)`, {
|
||||
const outcome = res.headers.get(CHALLENGE_OUTCOME_HEADER);
|
||||
const message = `请求 ${safeUrl}:站点要求人机验证(Cloudflare 等),${challengeOutcomeText(outcome)}`;
|
||||
throw new SiteError(ctx.siteId, "challenge", message, {
|
||||
httpStatus: res.status,
|
||||
detail: { reason: "challenge", rule: ch.rule, httpStatus: res.status },
|
||||
detail: { reason: "challenge", rule: ch.rule, httpStatus: res.status, ...(outcome ? { outcome } : {}) },
|
||||
sample: buildFailureSample({
|
||||
kind: "challenge",
|
||||
requestKind: "other",
|
||||
url,
|
||||
httpStatus: res.status,
|
||||
headers: res.headers,
|
||||
message,
|
||||
rule: ch.rule,
|
||||
html: ch.html,
|
||||
now: ctx.now(),
|
||||
}),
|
||||
});
|
||||
}
|
||||
if (res.status === 429 || res.status === 503) {
|
||||
|
||||
Reference in new issue
Block a user