fix(cloudflare): 页面请求返回 200 的挑战 / 人机验证门页也进入 A→B 级联

- handle 对页面请求(GET + Accept text/html)的 2xx 响应也做挑战判定(短页、无站点内容才看正文),
  过挑战 → cf_clearance 回带重试 → 浏览器代抓的级联照常触发
- 没处理掉时返回的挑战响应带 x-umt-challenge(needs_human / failed / throttled / no_browser / no_account),
  适配器据此区分「需要你在原站视图里过一次验证」与「已自动尝试，没有成功」等文案
- 测试:200 门页过挑战后重试成功、200 挑战页走浏览器代抓、转人工时搜索报 challenge 并留脱敏样本

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei

apps/server/src/modules/sources-identity/service/cloudflare.ts

@@ -1,4 +1,4 @@
-import { detectChallenge, mergeCookieHeaders, parseCookieHeader } from "@umt/adapters";
+import { CHALLENGE_OUTCOME_HEADER, detectChallenge, mergeCookieHeaders, parseCookieHeader } from "@umt/adapters";
 import { type SiteCloudflareState, siteCloudflareState } from "@umt/contracts";
 import { type SiteAccountRow, type SiteRow, sql } from "@umt/db";
 import { egressKey, siteEgress } from "../../../egress.ts";
@@ -11,12 +11,14 @@ import { resolveUserAgent } from "./user-agent.ts";
 // 站点 HTTP 请求(SourcesRuntime.siteFetch)经这里:
 // 0. 站点在"浏览器代理期"(config.cloudflare.browserUntil 未到)且请求是页面(GET + Accept text/html)→ 直接由浏览器代抓;
 // 1. 发 HTTP;账号 UA 跟随浏览器、出口与浏览器一致、复用没失败过时带上浏览器拿到的 cf_clearance(__cf_bm 不带);
-// 2. 响应是挑战(cf-mitigated: challenge,或 403 / 503 / 429 的挑战页)→ 不算凭据错误,记 lastChallengeAt;
+// 2. 响应是挑战(cf-mitigated: challenge,或 403 / 503 / 429 的挑战页,或页面请求返回 200 的挑战页 / 人机验证门页)
+//    → 不算凭据错误,记 lastChallengeAt;
 // 3. 过挑战(同站 30 分钟最多一次、同一时刻只跑一次;只处理有登录态账号的站点):浏览器用该账号 profile 打开同一地址 →
 //    ① 等它自己过(≤15 秒)→ ② 复选框 / Turnstile 派视觉子流程点一下(最多 2 次)→ ③ 选图 / 输码类转人工(待处理事项);
 // 4. 过了:读回 cf_clearance + navigator.userAgent;满足复用条件就存下来,HTTP 重试 1 次,结果记 cfReuse = ok / fails;
 // 5. 仍被挑战:页面请求改由浏览器导航取 HTML 交给原解析器,站点记 6 小时浏览器代理期;其它请求(下载、RSS、POST)照常报挑战。
-// 不接任何打码服务。
+// 没处理掉时返回的挑战响应带 `x-umt-challenge: <结果>`(needs_human / failed / throttled / no_browser / no_account),
+// 适配器据此写错误文案(自动处理到哪一步、要不要人)。不接任何打码服务。
 
 /** 同站两次过挑战的最短间隔 */
 export const CF_ATTEMPT_INTERVAL_MS = 30 * 60_000;
@@ -132,6 +134,13 @@ export function clearanceUsable(input: {
   );
 }
 
+/** 挑战响应上标出级联结果(响应头不可改,重新包一层) */
+function tagOutcome(res: Response, outcome: string): Response {
+  const headers = new Headers(res.headers);
+  headers.set(CHALLENGE_OUTCOME_HEADER, outcome);
+  return new Response(res.body, { status: res.status, statusText: res.statusText, headers });
+}
+
 export class CloudflareGate {
   private readonly cache = new Map<string, { at: number; state: SiteCloudflareState }>();
   /** 进程内同站同一时刻只过一次挑战:后到的请求等同一个结果 */
@@ -323,19 +332,19 @@ export class CloudflareGate {
     // 1. HTTP(满足条件时带 cf_clearance)
     // 从没遇到过挑战的站点不查账号(每条请求少两次查询)
     const extra = st.lastChallengeAt ? await this.clearanceCookie(siteId, req).catch(() => null) : null;
-    const first = await detectChallenge(await send(extra));
+    const first = await detectChallenge(await send(extra), { page: pagey });
     if (!first.rule) return first.res;
     // 2. 挑战:记下来;没有浏览器 / 没有登录态账号 → 原样返回(适配器报 challenge,不算凭据错误)
     await this.patch(siteId, { lastChallengeAt: now.toISOString(), lastRule: first.rule });
     const port = challengeBrowserFor(this.ctx);
-    if (!port) return first.res;
+    if (!port) return tagOutcome(first.res, "no_browser");
     const acc = await this.loggedInAccount(siteId);
     if (!acc) {
       await this.patch(siteId, { lastResult: "skipped", lastError: "站点没有已登录的账号，不处理挑战" });
-      return first.res;
+      return tagOutcome(first.res, "no_account");
     }
     const site = await this.ctx.db.selectFrom("site").selectAll().where("id", "=", siteId).executeTakeFirst();
-    if (!site?.enabled || site.deleted_at) return first.res;
+    if (!site?.enabled || site.deleted_at) return tagOutcome(first.res, "failed");
     // 3. 过挑战
     const out = await this.solveOnce(site, acc, req.url);
     st = await this.state(siteId, true);
@@ -357,7 +366,7 @@ export class CloudflareGate {
         const retryExtra = await this.clearanceCookie(siteId, req).catch(() => null);
         if (retryExtra) {
           await this.rt.limiterOf(siteId)?.acquire();
-          const second = await detectChallenge(await send(retryExtra));
+          const second = await detectChallenge(await send(retryExtra), { page: pagey });
           if (!second.rule) {
             await this.patch(siteId, { cfReuse: "ok", lastResult: "reuse_ok", browserUntil: null });
             return second.res;
@@ -387,7 +396,7 @@ export class CloudflareGate {
         return r;
       }
     }
-    return first.res;
+    return tagOutcome(first.res, !out ? "throttled" : out.status === "needs_human" ? "needs_human" : "failed");
   }
 
   /**

apps/server/test/sources-identity/cloudflare.test.ts

@@ -21,9 +21,24 @@ import { createHarness, type Harness, pgAvailable } from "./harness.ts";
 // 浏览器一侧用假端口(ChallengeBrowserPort),真实浏览器流程见 browser-cloudflare.test.ts。
 
 const up = await pgAvailable();
+
+/** 轮询到有结果(留样是异步写入的) */
+async function until<T>(f: () => Promise<T[]>, ms = 5_000): Promise<T[]> {
+  const deadline = Date.now() + ms;
+  for (;;) {
+    const r = await f();
+    if (r.length || Date.now() > deadline) return r;
+    await new Promise((res) => setTimeout(res, 50));
+  }
+}
 const BROWSER_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:152.0) Gecko/20100101 Firefox/152.0";
 const CF_HTML = `<!DOCTYPE html><html><head><title>Just a moment...</title></head><body>
 <div id="challenge-spinner"></div><script>window._cf_chl_opt={cType: 'managed'}</script></body></html>`;
+/** 返回 200 的 Turnstile 门页(没有 cf-mitigated 头):搜索页单独加盾、间歇触发时见到的形态 */
+const GATE_HTML = `<!DOCTYPE html><html><head><title>Audiences</title>
+<script src="https://challenges.cloudflare.com/turnstile/v0/api.js" async defer></script></head>
+<body><p>请完成人机验证后继续</p><div class="cf-turnstile" data-sitekey="0x4AAAAAAA"></div>
+<a href="download.php?id=1&passkey=0123456789abcdef0123456789abcdef">x</a></body></html>`;
 
 // biome-ignore lint/suspicious/noExplicitAny: 响应体断言
 const json = async (r: Response): Promise<any> => {
@@ -75,6 +90,8 @@ describe.skipIf(!up)("CF-2 Cloudflare 级联", () => {
     pass: (h: Headers) => (h.get("cookie") ?? "").includes("cf_clearance=GOOD") && h.get("user-agent") === BROWSER_UA,
     challenged: 0,
     passed: 0,
+    /** 非空:挑战响应改成 200 + 这段 HTML(不带 cf-mitigated 头) */
+    okPage: null as string | null,
   };
   const seen: Headers[] = [];
   const solves: { url: string; accountId: string }[] = [];
@@ -88,6 +105,12 @@ describe.skipIf(!up)("CF-2 Cloudflare 级联", () => {
     seen.push(hd);
     if (cf.on && !cf.pass(hd)) {
       cf.challenged++;
+      if (cf.okPage) {
+        return new Response(cf.okPage, {
+          status: 200,
+          headers: { "content-type": "text/html; charset=utf-8", server: "cloudflare", "cf-ray": "8c00-HKG" },
+        });
+      }
       return new Response(CF_HTML, {
         status: 403,
         headers: { "cf-mitigated": "challenge", "content-type": "text/html" },
@@ -148,6 +171,7 @@ describe.skipIf(!up)("CF-2 Cloudflare 级联", () => {
     cf.pass = (hd) => (hd.get("cookie") ?? "").includes("cf_clearance=GOOD") && hd.get("user-agent") === BROWSER_UA;
     cf.challenged = 0;
     cf.passed = 0;
+    cf.okPage = null;
     seen.length = 0;
     solves.length = 0;
     fetches.length = 0;
@@ -305,6 +329,53 @@ describe.skipIf(!up)("CF-2 Cloudflare 级联", () => {
     expect(await state()).toMatchObject({ lastResult: "skipped" });
   });
 
+  it("页面请求返回 200 的 Turnstile 门页 → 进入级联:过挑战后带 cf_clearance 重试成功", async () => {
+    cf.okPage = GATE_HTML;
+    const page = await list();
+    expect(page.items.length).toBeGreaterThan(0);
+    expect(cf.challenged).toBe(1);
+    expect(solves).toHaveLength(1);
+    expect(fetches).toEqual([]);
+    const st = await state();
+    expect(st).toMatchObject({ cfReuse: "ok", lastResult: "reuse_ok" });
+    expect(st?.lastRule).toMatch(/^gate\*=/);
+  });
+
+  it("200 的 Cloudflare 挑战页且 HTTP 过不去 → 浏览器代抓,进入浏览器代理期", async () => {
+    cf.okPage = CF_HTML;
+    cf.pass = () => false;
+    const page = await h.rt.siteAdapter(await site()).search("Dune");
+    expect(page.items.length).toBeGreaterThan(0);
+    expect(cf.challenged).toBe(2);
+    expect(fetches).toHaveLength(1);
+    expect(fetches[0]).toContain("search=Dune");
+    expect((await state())?.browserUntil).toBeTruthy();
+  });
+
+  it("200 门页转人工:搜索报 challenge(文案说要人在原站视图过验证),留样标搜索、已脱敏", async () => {
+    cf.okPage = GATE_HTML;
+    solveResult = { status: "needs_human", sessionId: null, decisionId: null, reason: "选图验证码" };
+    await h.db.deleteFrom("site_failure_sample").where("site_id", "=", siteId).execute();
+    const err = await h.rt
+      .siteAdapter(await site())
+      .search("Dune")
+      .catch((e) => e);
+    expect(err).toBeInstanceOf(SiteError);
+    expect((err as SiteError).kind).toBe("challenge");
+    expect((err as SiteError).message).toContain("搜索页需要人机验证");
+    expect((err as SiteError).message).toContain("原站视图");
+    expect((err as SiteError).message).not.toContain("改版");
+    expect((await account()).status).toBe("active");
+    const rows = await until(() =>
+      h.db.selectFrom("site_failure_sample").selectAll().where("site_id", "=", siteId).execute(),
+    );
+    expect(rows[0]).toMatchObject({ kind: "challenge", request_kind: "search", http_status: 200 });
+    expect(rows[0]?.headers).toMatchObject({ server: "cloudflare", "cf-ray": "8c00-HKG" });
+    expect(rows[0]?.body).toContain("cf-turnstile");
+    expect(rows[0]?.body).not.toContain("0123456789abcdef0123456789abcdef");
+    expect(rows[0]?.url).toContain("search=Dune");
+  });
+
   it("健康检测遇挑战且过不去:记 challenge(degraded),账号状态不变", async () => {
     solveResult = { status: "failed", reason: "浏览器服务不可用" };
     fetchResult = { status: "unavailable", reason: "浏览器服务不可用" };