diff --git a/apps/server/src/modules/sources-identity/service/account.ts b/apps/server/src/modules/sources-identity/service/account.ts index 4d295555..d4eca063 100644 --- a/apps/server/src/modules/sources-identity/service/account.ts +++ b/apps/server/src/modules/sources-identity/service/account.ts @@ -688,8 +688,9 @@ async function markFailure( ): Promise { const now = rt.now(); const lastError = { reason, message: message.slice(0, 300), at: now.toISOString() }; + // Cloudflare 挑战(CF-2)不是凭据问题:账号状态不变、不开"登录失效"事项;过挑战由 Cloudflare 处理负责,需要人时它开事项 const status = - reason === "network" || reason === "unsupported" + reason === "network" || reason === "unsupported" || reason === "challenge" ? null : reason === "auth_expired" && acc.auth_kind === "api_key" ? "invalid" diff --git a/packages/adapters/src/sites/challenge.test.ts b/packages/adapters/src/sites/challenge.test.ts new file mode 100644 index 00000000..537772e9 --- /dev/null +++ b/packages/adapters/src/sites/challenge.test.ts @@ -0,0 +1,116 @@ +import { describe, expect, it } from "vitest"; +import { challengeRuleOfHtml, classifyChallengePage, detectChallenge } from "./challenge.ts"; +import { siteFetch } from "./http.ts"; +import { createSiteRateLimiter } from "./rate-limit.ts"; +import { SiteError } from "./types.ts"; + +// CF-2:Cloudflare 挑战识别。HTTP 端挑战不是凭据错误(403 不再算 auth_invalid、503 不再算维护); +// 浏览器端按页面分:自动 / 可点(Turnstile)/ 真验证码。 + +const CF_PAGE = `Just a moment... +
`; +const CF_JS_PAGE = `Just a moment... +`; +const NORMAL = `种子 - 观众ok`; + +function ctx(fetchImpl: typeof fetch) { + return { + siteId: "s1", + fetch: fetchImpl, + limiter: createSiteRateLimiter({ minIntervalMs: 0, burst: 10 }), + timeoutMs: 5_000, + now: () => new Date(), + }; +} +const reply = (status: number, body: string, headers: Record = {}) => + (async () => new Response(body, { status, headers: { "content-type": "text/html", ...headers } })) as typeof fetch; + +describe("challengeRuleOfHtml", () => { + it("挑战页标题 / 脚本特征命中;正常页面里注入的 challenge-platform 脚本不算", () => { + expect(challengeRuleOfHtml(CF_PAGE)).toBe("title=Just a moment..."); + expect(challengeRuleOfHtml('
')).toContain( + "challenge-form", + ); + expect(challengeRuleOfHtml(NORMAL)).toBeNull(); + expect(challengeRuleOfHtml("")).toBeNull(); + }); +}); + +describe("detectChallenge", () => { + it("cf-mitigated: challenge 不读正文;403 / 503 读正文判定并重新包装响应", async () => { + const a = await detectChallenge(new Response("x", { status: 200, headers: { "cf-mitigated": "challenge" } })); + expect(a.rule).toBe("header cf-mitigated=challenge"); + const b = await detectChallenge(new Response(CF_PAGE, { status: 403 })); + expect(b.rule).toContain("Just a moment"); + expect(await b.res.text()).toBe(CF_PAGE); + const c = await detectChallenge(new Response("Forbidden", { status: 403 })); + expect(c.rule).toBeNull(); + expect(await c.res.text()).toBe("Forbidden"); + const d = await detectChallenge(new Response(NORMAL, { status: 200 })); + expect(d.rule).toBeNull(); + }); +}); + +describe("siteFetch 挑战分类", () => { + const url = "https://audiences.example/torrents.php"; + const kind = async (f: typeof fetch) => { + try { + await siteFetch(ctx(f), url); + return "ok"; + } catch (e) { + return e instanceof SiteError ? e.kind : "other"; + } + }; + + it("403 + cf-mitigated → challenge(不是 auth_invalid),不算凭据错误", async () => { + let err: unknown; + try { + await siteFetch(ctx(reply(403, CF_PAGE, { "cf-mitigated": "challenge" })), url); + } catch (e) { + err = e; + } + expect(err).toBeInstanceOf(SiteError); + const se = err as SiteError; + expect(se.kind).toBe("challenge"); + expect(se.isAuthError).toBe(false); + expect(se.detail).toMatchObject({ reason: "challenge", rule: "header cf-mitigated=challenge", httpStatus: 403 }); + }); + + it("503 挑战页 → challenge(不是维护);403 挑战页(没有响应头)也认", async () => { + expect(await kind(reply(503, CF_PAGE))).toBe("challenge"); + expect(await kind(reply(403, CF_PAGE))).toBe("challenge"); + expect(await kind(reply(429, CF_JS_PAGE))).toBe("challenge"); + }); + + it("普通 503 仍是维护,普通 403 交给适配器(响应体可读)", async () => { + expect(await kind(reply(503, "maintenance"))).toBe("site_maintenance"); + const res = await siteFetch(ctx(reply(403, "Forbidden")), url); + expect(res.status).toBe(403); + expect(await res.text()).toBe("Forbidden"); + }); +}); + +describe("classifyChallengePage", () => { + it("不是挑战页 → none", () => { + expect(classifyChallengePage({ title: "种子", html: NORMAL, text: "种子列表" }).kind).toBe("none"); + }); + it("JS 检查 → waiting;托管 / Turnstile → interactive", () => { + expect(classifyChallengePage({ html: CF_JS_PAGE, text: "Checking your browser" }).kind).toBe("waiting"); + expect(classifyChallengePage({ html: CF_PAGE, text: "" }).kind).toBe("interactive"); + expect( + classifyChallengePage({ + title: "请稍候…", + html: "", + text: "请完成以下操作，验证您是真人。", + }).kind, + ).toBe("interactive"); + }); + it("选图 / 输码 → captcha(不自动处理)", () => { + expect(classifyChallengePage({ html: `${CF_PAGE}
`, text: "Just a moment" }).kind).toBe( + "captcha", + ); + expect(classifyChallengePage({ title: "Just a moment...", html: "", text: "请点击图中所有的红绿灯" }).kind).toBe( + "captcha", + ); + }); +}); diff --git a/packages/adapters/src/sites/challenge.ts b/packages/adapters/src/sites/challenge.ts new file mode 100644 index 00000000..0c521ab7 --- /dev/null +++ b/packages/adapters/src/sites/challenge.ts @@ -0,0 +1,83 @@ +// CF-2 Cloudflare 挑战识别(docs/design/cloudflare-handling.md)。 +// - HTTP 端:响应头 `cf-mitigated: challenge`(任何状态码),或 403 / 503 / 429 且正文是挑战页 → 挑战,不是凭据错误; +// - 浏览器端:挑战页分三类——自动(JS 检查,等几秒自己过)、可点(「我是人类」复选框 / Turnstile)、真验证码(选图 / 输码)。 +// 只用正则,不解析 DOM:HTTP 端要在限流器之后快速判断,浏览器端拿到的是 outerHTML + innerText 样本。 + +/** Cloudflare / DDoS-Guard 挑战页的标题 */ +const CHALLENGE_TITLE = /^\s*(?:just a moment|attention required! \| cloudflare|请稍候|ddos-guard|one moment, please)/i; +/** 挑战页的脚本 / 元素特征。不认 `/cdn-cgi/challenge-platform/`:Cloudflare 会往正常页面也注入这段脚本 */ +const CHALLENGE_HTML = + /cf-browser-verification|cf_chl_opt|cf-challenge-running|id="challenge-form"|id="challenge-spinner"|id="cf-please-wait"|id="trk_jschal_js"/i; + +function titleOf(html: string): string { + return /]*>([^<]*)<\/title>/i.exec(html.slice(0, 50_000))?.[1]?.trim() ?? ""; +} + +/** 页面是不是挑战页;命中返回规则说明(进技术信息),否则 null */ +export function challengeRuleOfHtml(html: string | null | undefined): string | null { + if (!html) return null; + const title = titleOf(html); + if (CHALLENGE_TITLE.test(title)) return `title=${title.slice(0, 40)}`; + const m = CHALLENGE_HTML.exec(html.slice(0, 200_000)); + return m ? `html*=${m[0]}` : null; +} + +/** 响应头判定(不读正文):`cf-mitigated: challenge` */ +export function challengeRuleOfHeaders(headers: Headers): string | null { + return headers.get("cf-mitigated")?.trim().toLowerCase() === "challenge" ? "header cf-mitigated=challenge" : null; +} + +/** 这些状态码的响应才去看正文(Cloudflare 挑战页是 403 / 503,少数 429) */ +export const CHALLENGE_STATUSES: readonly number[] = [403, 429, 503]; + +/** + * 判定一个站点响应是不是挑战。正文只在状态码可能是挑战时读(读过的正文重新包成 Response 返回,调用方照常消费)。 + * 返回 { res, rule }:rule 为 null 表示不是挑战。 + */ +export async function detectChallenge(res: Response): Promise<{ res: Response; rule: string | null }> { + const byHeader = challengeRuleOfHeaders(res.headers); + if (byHeader) return { res, rule: byHeader }; + if (!CHALLENGE_STATUSES.includes(res.status) || !res.body) return { res, rule: null }; + const bytes = new Uint8Array(await res.arrayBuffer()); + const html = new TextDecoder("utf-8").decode(bytes.subarray(0, 300_000)); + const copy = new Response(bytes, { status: res.status, statusText: res.statusText, headers: res.headers }); + return { res: copy, rule: challengeRuleOfHtml(html) }; +} + +/** + * 浏览器里打开的页面属于哪种挑战: + * - none:不是挑战页(已通过); + * - waiting:自动检查(JS / 非交互),等几秒会自己过; + * - interactive:「我是人类」复选框 / Turnstile,可以点一下; + * - captcha:选图、输验证码这类真验证码,不自动处理。 + */ +export type ChallengePageKind = "none" | "waiting" | "interactive" | "captcha"; + +const CAPTCHA_MARKERS = + /hcaptcha\.com|class="h-captcha"|g-recaptcha|recaptcha\/api|imagecode\.php|select all (?:images|squares)|请(?:依次)?点击|选择所有|拖动滑块|输入验证码|enter the (?:characters|code)/i; +const INTERACTIVE_MARKERS = + /challenges\.cloudflare\.com\/(?:turnstile|cdn-cgi)|cf-turnstile|cType\s*:\s*['"](?:managed|interactive)['"]|verify you are human|确认您是真人|验证您是真人|我是人类/i; + +export function classifyChallengePage(page: { title?: string | null; html?: string | null; text?: string | null }): { + kind: ChallengePageKind; + rule: string | null; +} { + const html = page.html ?? ""; + const text = page.text ?? ""; + const title = page.title ?? titleOf(html); + const rule = + (CHALLENGE_TITLE.test(title) ? `title=${title.slice(0, 40)}` : null) ?? + challengeRuleOfHtml(html) ?? + (/verify you are human|确认您是真人|验证您是真人|checking (?:if the site connection is secure|your browser)/i.test( + text, + ) + ? "text" + : null); + if (!rule) return { kind: "none", rule: null }; + const all = `${html.slice(0, 300_000)}\n${text.slice(0, 20_000)}`; + const cap = CAPTCHA_MARKERS.exec(all); + if (cap) return { kind: "captcha", rule: `${rule}; captcha*=${cap[0]}` }; + const inter = INTERACTIVE_MARKERS.exec(all); + if (inter) return { kind: "interactive", rule: `${rule}; interactive*=${inter[0]}` }; + return { kind: "waiting", rule }; +} diff --git a/packages/adapters/src/sites/http.ts b/packages/adapters/src/sites/http.ts index 33c88156..43e8d33f 100644 --- a/packages/adapters/src/sites/http.ts +++ b/packages/adapters/src/sites/http.ts @@ -1,4 +1,5 @@ import { redactText } from "../rss/redact.ts"; +import { detectChallenge } from "./challenge.ts"; import type { SiteRateLimiter } from "./rate-limit.ts"; import { SiteError } from "./types.ts"; @@ -20,7 +21,8 @@ function retryAfterMs(res: Response, now: Date): number | undefined { } /** - * 站点请求:先取站点令牌,再发请求;429/503 统一映射并顺延限流器。 + * 站点请求:先取站点令牌,再发请求;Cloudflare 挑战(`cf-mitigated: challenge`,或 403 / 503 / 429 的挑战页)→ `challenge` + * (不是凭据错误,CF-2);其余 429/503 统一映射并顺延限流器。 * 返回的 Response 由调用方消费。错误消息只含脱敏后的 URL。 */ export async function siteFetch( @@ -42,6 +44,15 @@ export async function siteFetch( const reason = timeout.aborted ? "超时" : (e as Error).message; throw new SiteError(ctx.siteId, "network", redactText(`请求 ${safeUrl} 失败:${reason}`)); } + const ch = await detectChallenge(res); + res = ch.res; + if (ch.rule) { + await res.body?.cancel().catch(() => {}); + throw new SiteError(ctx.siteId, "challenge", `请求 ${safeUrl}:站点要求人机验证(Cloudflare)`, { + httpStatus: res.status, + detail: { reason: "challenge", rule: ch.rule, httpStatus: res.status }, + }); + } if (res.status === 429 || res.status === 503) { const ra = retryAfterMs(res, ctx.now()); await res.body?.cancel().catch(() => {}); diff --git a/packages/adapters/src/sites/index.ts b/packages/adapters/src/sites/index.ts index 4c4a0bb4..0f19c314 100644 --- a/packages/adapters/src/sites/index.ts +++ b/packages/adapters/src/sites/index.ts @@ -1,3 +1,11 @@ +export { + CHALLENGE_STATUSES, + type ChallengePageKind, + challengeRuleOfHeaders, + challengeRuleOfHtml, + classifyChallengePage, + detectChallenge, +} from "./challenge.ts"; export { audiencesDefinition, BUILTIN_SITE_DEFINITIONS,