diff --git a/apps/server/src/modules/sources-identity/service/cloudflare.ts b/apps/server/src/modules/sources-identity/service/cloudflare.ts index 3e540f8f..0590d495 100644 --- a/apps/server/src/modules/sources-identity/service/cloudflare.ts +++ b/apps/server/src/modules/sources-identity/service/cloudflare.ts @@ -1,4 +1,4 @@ -import { detectChallenge, mergeCookieHeaders, parseCookieHeader } from "@umt/adapters"; +import { CHALLENGE_OUTCOME_HEADER, detectChallenge, mergeCookieHeaders, parseCookieHeader } from "@umt/adapters"; import { type SiteCloudflareState, siteCloudflareState } from "@umt/contracts"; import { type SiteAccountRow, type SiteRow, sql } from "@umt/db"; import { egressKey, siteEgress } from "../../../egress.ts"; @@ -11,12 +11,14 @@ import { resolveUserAgent } from "./user-agent.ts"; // 站点 HTTP 请求(SourcesRuntime.siteFetch)经这里: // 0. 站点在"浏览器代理期"(config.cloudflare.browserUntil 未到)且请求是页面(GET + Accept text/html)→ 直接由浏览器代抓; // 1. 发 HTTP;账号 UA 跟随浏览器、出口与浏览器一致、复用没失败过时带上浏览器拿到的 cf_clearance(__cf_bm 不带); -// 2. 响应是挑战(cf-mitigated: challenge,或 403 / 503 / 429 的挑战页)→ 不算凭据错误,记 lastChallengeAt; +// 2. 响应是挑战(cf-mitigated: challenge,或 403 / 503 / 429 的挑战页,或页面请求返回 200 的挑战页 / 人机验证门页) +// → 不算凭据错误,记 lastChallengeAt; // 3. 过挑战(同站 30 分钟最多一次、同一时刻只跑一次;只处理有登录态账号的站点):浏览器用该账号 profile 打开同一地址 → // ① 等它自己过(≤15 秒)→ ② 复选框 / Turnstile 派视觉子流程点一下(最多 2 次)→ ③ 选图 / 输码类转人工(待处理事项); // 4. 过了:读回 cf_clearance + navigator.userAgent;满足复用条件就存下来,HTTP 重试 1 次,结果记 cfReuse = ok / fails; // 5. 仍被挑战:页面请求改由浏览器导航取 HTML 交给原解析器,站点记 6 小时浏览器代理期;其它请求(下载、RSS、POST)照常报挑战。 -// 不接任何打码服务。 +// 没处理掉时返回的挑战响应带 `x-umt-challenge: <结果>`(needs_human / failed / throttled / no_browser / no_account), +// 适配器据此写错误文案(自动处理到哪一步、要不要人)。不接任何打码服务。 /** 同站两次过挑战的最短间隔 */ export const CF_ATTEMPT_INTERVAL_MS = 30 * 60_000; @@ -132,6 +134,13 @@ export function clearanceUsable(input: { ); } +/** 挑战响应上标出级联结果(响应头不可改,重新包一层) */ +function tagOutcome(res: Response, outcome: string): Response { + const headers = new Headers(res.headers); + headers.set(CHALLENGE_OUTCOME_HEADER, outcome); + return new Response(res.body, { status: res.status, statusText: res.statusText, headers }); +} + export class CloudflareGate { private readonly cache = new Map(); /** 进程内同站同一时刻只过一次挑战:后到的请求等同一个结果 */ @@ -323,19 +332,19 @@ export class CloudflareGate { // 1. HTTP(满足条件时带 cf_clearance) // 从没遇到过挑战的站点不查账号(每条请求少两次查询) const extra = st.lastChallengeAt ? await this.clearanceCookie(siteId, req).catch(() => null) : null; - const first = await detectChallenge(await send(extra)); + const first = await detectChallenge(await send(extra), { page: pagey }); if (!first.rule) return first.res; // 2. 挑战:记下来;没有浏览器 / 没有登录态账号 → 原样返回(适配器报 challenge,不算凭据错误) await this.patch(siteId, { lastChallengeAt: now.toISOString(), lastRule: first.rule }); const port = challengeBrowserFor(this.ctx); - if (!port) return first.res; + if (!port) return tagOutcome(first.res, "no_browser"); const acc = await this.loggedInAccount(siteId); if (!acc) { await this.patch(siteId, { lastResult: "skipped", lastError: "站点没有已登录的账号,不处理挑战" }); - return first.res; + return tagOutcome(first.res, "no_account"); } const site = await this.ctx.db.selectFrom("site").selectAll().where("id", "=", siteId).executeTakeFirst(); - if (!site?.enabled || site.deleted_at) return first.res; + if (!site?.enabled || site.deleted_at) return tagOutcome(first.res, "failed"); // 3. 过挑战 const out = await this.solveOnce(site, acc, req.url); st = await this.state(siteId, true); @@ -357,7 +366,7 @@ export class CloudflareGate { const retryExtra = await this.clearanceCookie(siteId, req).catch(() => null); if (retryExtra) { await this.rt.limiterOf(siteId)?.acquire(); - const second = await detectChallenge(await send(retryExtra)); + const second = await detectChallenge(await send(retryExtra), { page: pagey }); if (!second.rule) { await this.patch(siteId, { cfReuse: "ok", lastResult: "reuse_ok", browserUntil: null }); return second.res; @@ -387,7 +396,7 @@ export class CloudflareGate { return r; } } - return first.res; + return tagOutcome(first.res, !out ? "throttled" : out.status === "needs_human" ? "needs_human" : "failed"); } /** diff --git a/apps/server/test/sources-identity/cloudflare.test.ts b/apps/server/test/sources-identity/cloudflare.test.ts index 9939ea7f..a2efc7a8 100644 --- a/apps/server/test/sources-identity/cloudflare.test.ts +++ b/apps/server/test/sources-identity/cloudflare.test.ts @@ -21,9 +21,24 @@ import { createHarness, type Harness, pgAvailable } from "./harness.ts"; // 浏览器一侧用假端口(ChallengeBrowserPort),真实浏览器流程见 browser-cloudflare.test.ts。 const up = await pgAvailable(); + +/** 轮询到有结果(留样是异步写入的) */ +async function until(f: () => Promise, ms = 5_000): Promise { + const deadline = Date.now() + ms; + for (;;) { + const r = await f(); + if (r.length || Date.now() > deadline) return r; + await new Promise((res) => setTimeout(res, 50)); + } +} const BROWSER_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:152.0) Gecko/20100101 Firefox/152.0"; const CF_HTML = `Just a moment...
`; +/** 返回 200 的 Turnstile 门页(没有 cf-mitigated 头):搜索页单独加盾、间歇触发时见到的形态 */ +const GATE_HTML = `Audiences + +

请完成人机验证后继续

+x`; // biome-ignore lint/suspicious/noExplicitAny: 响应体断言 const json = async (r: Response): Promise => { @@ -75,6 +90,8 @@ describe.skipIf(!up)("CF-2 Cloudflare 级联", () => { pass: (h: Headers) => (h.get("cookie") ?? "").includes("cf_clearance=GOOD") && h.get("user-agent") === BROWSER_UA, challenged: 0, passed: 0, + /** 非空:挑战响应改成 200 + 这段 HTML(不带 cf-mitigated 头) */ + okPage: null as string | null, }; const seen: Headers[] = []; const solves: { url: string; accountId: string }[] = []; @@ -88,6 +105,12 @@ describe.skipIf(!up)("CF-2 Cloudflare 级联", () => { seen.push(hd); if (cf.on && !cf.pass(hd)) { cf.challenged++; + if (cf.okPage) { + return new Response(cf.okPage, { + status: 200, + headers: { "content-type": "text/html; charset=utf-8", server: "cloudflare", "cf-ray": "8c00-HKG" }, + }); + } return new Response(CF_HTML, { status: 403, headers: { "cf-mitigated": "challenge", "content-type": "text/html" }, @@ -148,6 +171,7 @@ describe.skipIf(!up)("CF-2 Cloudflare 级联", () => { cf.pass = (hd) => (hd.get("cookie") ?? "").includes("cf_clearance=GOOD") && hd.get("user-agent") === BROWSER_UA; cf.challenged = 0; cf.passed = 0; + cf.okPage = null; seen.length = 0; solves.length = 0; fetches.length = 0; @@ -305,6 +329,53 @@ describe.skipIf(!up)("CF-2 Cloudflare 级联", () => { expect(await state()).toMatchObject({ lastResult: "skipped" }); }); + it("页面请求返回 200 的 Turnstile 门页 → 进入级联:过挑战后带 cf_clearance 重试成功", async () => { + cf.okPage = GATE_HTML; + const page = await list(); + expect(page.items.length).toBeGreaterThan(0); + expect(cf.challenged).toBe(1); + expect(solves).toHaveLength(1); + expect(fetches).toEqual([]); + const st = await state(); + expect(st).toMatchObject({ cfReuse: "ok", lastResult: "reuse_ok" }); + expect(st?.lastRule).toMatch(/^gate\*=/); + }); + + it("200 的 Cloudflare 挑战页且 HTTP 过不去 → 浏览器代抓,进入浏览器代理期", async () => { + cf.okPage = CF_HTML; + cf.pass = () => false; + const page = await h.rt.siteAdapter(await site()).search("Dune"); + expect(page.items.length).toBeGreaterThan(0); + expect(cf.challenged).toBe(2); + expect(fetches).toHaveLength(1); + expect(fetches[0]).toContain("search=Dune"); + expect((await state())?.browserUntil).toBeTruthy(); + }); + + it("200 门页转人工:搜索报 challenge(文案说要人在原站视图过验证),留样标搜索、已脱敏", async () => { + cf.okPage = GATE_HTML; + solveResult = { status: "needs_human", sessionId: null, decisionId: null, reason: "选图验证码" }; + await h.db.deleteFrom("site_failure_sample").where("site_id", "=", siteId).execute(); + const err = await h.rt + .siteAdapter(await site()) + .search("Dune") + .catch((e) => e); + expect(err).toBeInstanceOf(SiteError); + expect((err as SiteError).kind).toBe("challenge"); + expect((err as SiteError).message).toContain("搜索页需要人机验证"); + expect((err as SiteError).message).toContain("原站视图"); + expect((err as SiteError).message).not.toContain("改版"); + expect((await account()).status).toBe("active"); + const rows = await until(() => + h.db.selectFrom("site_failure_sample").selectAll().where("site_id", "=", siteId).execute(), + ); + expect(rows[0]).toMatchObject({ kind: "challenge", request_kind: "search", http_status: 200 }); + expect(rows[0]?.headers).toMatchObject({ server: "cloudflare", "cf-ray": "8c00-HKG" }); + expect(rows[0]?.body).toContain("cf-turnstile"); + expect(rows[0]?.body).not.toContain("0123456789abcdef0123456789abcdef"); + expect(rows[0]?.url).toContain("search=Dune"); + }); + it("健康检测遇挑战且过不去:记 challenge(degraded),账号状态不变", async () => { solveResult = { status: "failed", reason: "浏览器服务不可用" }; fetchResult = { status: "unavailable", reason: "浏览器服务不可用" };