fix(adapters): 识别 200 状态的挑战 / 人机验证页,解析失败前兜底,失败页面留样

- detectChallenge 支持页面类请求(GET + Accept text/html)的 2xx 响应:正文只先读 256 KB,
  短页且没有种子表 / 用户信息块时,认 Cloudflare 挑战标题 / 脚本与 Turnstile / hCaptcha / reCAPTCHA 门页;
  带登录表单的页面不算门页。非页面请求不读正文
- siteFetch 用上面的判定;挑战错误附脱敏页面样本,文案带级联结果(x-umt-challenge)
- spider 适配器:认不出列表 / 详情时先看是不是人机验证页,是就报 challenge(「搜索页需要人机验证」),
  不再报改版;parse_error / challenge 都经 deps.onFailureSample 交出脱敏样本(前 64 KB,去掉个人信息块、
  passkey / authkey / uid / 用户名 / Cookie / 邮箱)
- classifyChallengePage 也认门页(浏览器过挑战时不再把门页当已通过)

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei
This commit is contained in:
rootandClaude Opus 5.5 committed 2026-10-06 15:35:31 +08:00
1 parent 4fa001750b
commit 1779e1a0f2
11 files changed
+2078 -34

No files matched your search

+21 -5
View File
@@ -1,5 +1,6 @@
import { redactText } from "../rss/redact.ts";
import { detectChallenge } from "./challenge.ts";
import { CHALLENGE_OUTCOME_HEADER, challengeOutcomeText, detectChallenge, isPageRequest } from "./challenge.ts";
import { buildFailureSample } from "./failure-sample.ts";
import type { SiteRateLimiter } from "./rate-limit.ts";
import { SiteError } from "./types.ts";
@@ -21,7 +22,8 @@ function retryAfterMs(res: Response, now: Date): number | undefined {
}
/**
* 站点请求:先取站点令牌,再发请求;Cloudflare 挑战(`cf-mitigated: challenge`,或 403 / 503 / 429 的挑战页)→ `challenge`
* 站点请求:先取站点令牌,再发请求;Cloudflare 挑战(`cf-mitigated: challenge`,或 403 / 503 / 429 的挑战页,
* 或页面请求返回 200 的挑战页 / 人机验证门页)→ `challenge`(附脱敏页面样本)
* (不是凭据错误,CF-2);其余 429/503 统一映射并顺延限流器。
* 返回的 Response 由调用方消费。错误消息只含脱敏后的 URL。
*/
@@ -44,13 +46,27 @@ export async function siteFetch(
const reason = timeout.aborted ? "超时" : (e as Error).message;
throw new SiteError(ctx.siteId, "network", redactText(`请求 ${safeUrl} 失败:${reason}`));
}
const ch = await detectChallenge(res);
// 页面类请求的 200 响应也看正文:返回 200 的挑战页 / 人机验证门页(只读开头,有上限)
const ch = await detectChallenge(res, { page: isPageRequest(init.method, headers.get("accept")) });
res = ch.res;
if (ch.rule) {
await res.body?.cancel().catch(() => {});
throw new SiteError(ctx.siteId, "challenge", `请求 ${safeUrl}:站点要求人机验证(Cloudflare)`, {
const outcome = res.headers.get(CHALLENGE_OUTCOME_HEADER);
const message = `请求 ${safeUrl}:站点要求人机验证(Cloudflare 等),${challengeOutcomeText(outcome)}`;
throw new SiteError(ctx.siteId, "challenge", message, {
httpStatus: res.status,
detail: { reason: "challenge", rule: ch.rule, httpStatus: res.status },
detail: { reason: "challenge", rule: ch.rule, httpStatus: res.status, ...(outcome ? { outcome } : {}) },
sample: buildFailureSample({
kind: "challenge",
requestKind: "other",
url,
httpStatus: res.status,
headers: res.headers,
message,
rule: ch.rule,
html: ch.html,
now: ctx.now(),
}),
});
}
if (res.status === 429 || res.status === 503) {