UHD_Media_Tool/apps/server/test/sources-identity/cloudflare.test.ts

import { SiteError } from "@umt/adapters";
import { siteCloudflareState } from "@umt/contracts";
import { createMockNexusPhpSite, type MockNexusPhpSite } from "@umt/testkit";
import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest";
import {
  type BrowserFetchOutcome,
  browserFetchable,
  CF_ATTEMPT_INTERVAL_MS,
  type ChallengeSolveOutcome,
  clearanceUsable,
  registerChallengeBrowser,
  withClearance,
} from "../../src/modules/sources-identity/service/cloudflare.ts";
import { snapshotSite } from "../../src/modules/sources-identity/service/snapshot.ts";
import { registerBrowserIdentityReader } from "../../src/modules/sources-identity/service/user-agent.ts";
import { createHarness, type Harness, pgAvailable } from "./harness.ts";

// CF-2 级联状态机:HTTP 遇 Cloudflare 挑战 → 浏览器过挑战(自动 / 视觉子流程 / 转人工)→ cf_clearance 同步给 HTTP 重试 1 次 →
// 仍被挑战则页面请求改由浏览器代抓并进入 6 小时浏览器代理期;同站 30 分钟节流、同一时刻只处理一次;挑战不算凭据错误。
// 站点 = testkit mock NexusPHP(真 HTTP),前面套一层"Cloudflare":没带合格 cf_clearance 的请求回 403 + cf-mitigated 挑战页。
// 浏览器一侧用假端口(ChallengeBrowserPort),真实浏览器流程见 browser-cloudflare.test.ts。

const up = await pgAvailable();
const BROWSER_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:152.0) Gecko/20100101 Firefox/152.0";
const CF_HTML = `<!DOCTYPE html><html><head><title>Just a moment...</title></head><body>
<div id="challenge-spinner"></div><script>window._cf_chl_opt={cType: 'managed'}</script></body></html>`;

// biome-ignore lint/suspicious/noExplicitAny: 响应体断言
const json = async (r: Response): Promise<any> => {
  const body = await r.json();
  if (!r.ok) throw new Error(`${r.status} ${JSON.stringify(body)}`);
  return body.data;
};

describe("CF-2 纯函数", () => {
  it("页面请求才能由浏览器代抓", () => {
    expect(browserFetchable({ method: "GET", accept: "text/html,application/xhtml+xml" })).toBe(true);
    expect(browserFetchable({ method: "GET", accept: null })).toBe(false);
    expect(browserFetchable({ method: "POST", accept: "text/html" })).toBe(false);
    expect(browserFetchable({ method: "GET", accept: "application/rss+xml" })).toBe(false);
  });
  it("cf_clearance 同步条件:UA 跟随浏览器 + UA 相同 + 两端出口相同 + 复用没失败过", () => {
    const base = {
      uaSource: "browser",
      requestUa: "A",
      clearanceUa: "A",
      requestEgress: "direct",
      clearanceEgress: "direct",
      cfReuse: null,
    } as const;
    expect(clearanceUsable(base)).toBe(true);
    expect(clearanceUsable({ ...base, uaSource: "manual" })).toBe(false);
    expect(clearanceUsable({ ...base, uaSource: "global" })).toBe(false);
    expect(clearanceUsable({ ...base, clearanceUa: "B" })).toBe(false);
    expect(clearanceUsable({ ...base, clearanceEgress: null })).toBe(false);
    expect(clearanceUsable({ ...base, clearanceEgress: "proxy:http://relay:7891" })).toBe(false);
    expect(clearanceUsable({ ...base, cfReuse: "fails" })).toBe(false);
  });
  it("并进 cf_clearance,去掉 __cf_bm", () => {
    expect(withClearance("a=1; __cf_bm=x", "cf_clearance=Z")).toBe("a=1; cf_clearance=Z");
    expect(withClearance(null, "cf_clearance=Z")).toBe("cf_clearance=Z");
    expect(withClearance("a=1", null)).toBe("a=1");
  });
});

describe.skipIf(!up)("CF-2 Cloudflare 级联", () => {
  let h: Harness;
  let mock: MockNexusPhpSite;
  let siteId: string;
  let accountId: string;
  /** "Cloudflare" 开关与放行规则 */
  const cf = {
    on: false,
    /** 放行判定;缺省:带 cf_clearance=GOOD 且 UA 是浏览器 UA */
    pass: (h: Headers) => (h.get("cookie") ?? "").includes("cf_clearance=GOOD") && h.get("user-agent") === BROWSER_UA,
    challenged: 0,
    passed: 0,
  };
  const seen: Headers[] = [];
  const solves: { url: string; accountId: string }[] = [];
  const fetches: string[] = [];
  let solveResult: ChallengeSolveOutcome = { status: "failed", reason: "unset" };
  let solveDelay = 0;
  let fetchResult: BrowserFetchOutcome | null = null;

  const cfFetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
    const hd = new Headers(init?.headers);
    seen.push(hd);
    if (cf.on && !cf.pass(hd)) {
      cf.challenged++;
      return new Response(CF_HTML, {
        status: 403,
        headers: { "cf-mitigated": "challenge", "content-type": "text/html" },
      });
    }
    cf.passed++;
    return globalThis.fetch(input, init);
  }) as typeof fetch;

  const site = () => h.db.selectFrom("site").selectAll().where("id", "=", siteId).executeTakeFirstOrThrow();
  const account = () =>
    h.db.selectFrom("site_account").selectAll().where("id", "=", accountId).executeTakeFirstOrThrow();
  const state = async () => siteCloudflareState((await site()).config);
  const list = async () => h.rt.siteAdapter(await site()).list(1);

  beforeAll(async () => {
    mock = await createMockNexusPhpSite({});
    h = await createHarness("cf2", { fetch: cfFetch });
    registerBrowserIdentityReader(h.ctx, async () => BROWSER_UA);
    registerChallengeBrowser(h.ctx, {
      async solve(req) {
        solves.push(req);
        if (solveDelay) await new Promise((r) => setTimeout(r, solveDelay));
        return solveResult;
      },
      async fetchHtml(req) {
        fetches.push(req.url);
        if (fetchResult) return fetchResult;
        // 浏览器已登录且过了挑战:直接取 mock 页面
        const r = await globalThis.fetch(req.url, { headers: { cookie: mock.cookie } });
        return { status: "ok", html: await r.text(), url: req.url };
      },
    });
    const s = await json(
      await h.req("/sites", {
        body: {
          key: "audiences",
          name: "观众",
          adapter: "nexusphp",
          baseUrl: `${mock.baseUrl}/`,
          rateLimit: { minIntervalMs: 0, concurrency: 10 },
        },
      }),
    );
    siteId = s.id;
    const r = await json(
      await h.req(`/sites/${siteId}/credentials`, { method: "PUT", body: { kind: "cookie", cookie: mock.cookie } }),
    );
    accountId = r.siteAccountId;
  });
  afterAll(async () => {
    await h?.close();
    await mock?.close();
  });

  beforeEach(async () => {
    cf.on = true;
    cf.pass = (hd) => (hd.get("cookie") ?? "").includes("cf_clearance=GOOD") && hd.get("user-agent") === BROWSER_UA;
    cf.challenged = 0;
    cf.passed = 0;
    seen.length = 0;
    solves.length = 0;
    fetches.length = 0;
    solveDelay = 0;
    fetchResult = null;
    solveResult = { status: "passed", by: "auto", clearance: "GOOD", userAgent: BROWSER_UA, browserEgress: "direct" };
    await h.db
      .updateTable("site")
      .set({ config: JSON.stringify({}), health_status: "healthy" })
      .where("id", "=", siteId)
      .execute();
    await h.db
      .updateTable("site_account")
      .set({ user_info: JSON.stringify({}), user_agent: null, ua_pending: null, status: "active" })
      .where("id", "=", accountId)
      .execute();
    await h.rt.cloudflare.patch(siteId, {});
  });

  it("自动通过:浏览器拿到 cf_clearance + UA,HTTP 带上重试 1 次成功,记 cfReuse = ok;之后的请求直接带", async () => {
    const page = await list();
    expect(page.items.length).toBeGreaterThan(0);
    expect(solves).toEqual([{ siteId, accountId, url: expect.stringContaining("/torrents.php") }]);
    expect(cf.challenged).toBe(1);
    expect(fetches).toEqual([]);
    const st = await state();
    expect(st).toMatchObject({ lastSolvedBy: "auto", cfReuse: "ok", lastResult: "reuse_ok", browserUntil: null });
    expect(st?.lastChallengeAt).toBeTruthy();
    expect(st?.lastRule).toBe("header cf-mitigated=challenge");
    // cf_clearance 存在 secret 里,账号 user_info 只有引用与取得时的 UA / 出口
    const info = (await account()).user_info as {
      cloudflare?: { secretRef: string; userAgent: string; egress: string };
    };
    expect(info.cloudflare).toMatchObject({ userAgent: BROWSER_UA, egress: "direct" });
    expect(h.secrets.plain.get(info.cloudflare!.secretRef)).toBe("GOOD");
    expect(JSON.stringify(info)).not.toContain("GOOD");
    // __cf_bm 不带;下一条请求直接带 cf_clearance,不再过挑战
    seen.length = 0;
    await list();
    expect(solves).toHaveLength(1);
    expect(seen[0]?.get("cookie")).toContain("cf_clearance=GOOD");
    expect(seen[0]?.get("cookie")).not.toContain("__cf_bm");
    // 账号状态不变
    expect((await account()).status).toBe("active");
  });

  it("视觉子流程通过:记 lastSolvedBy = vision", async () => {
    solveResult = { status: "passed", by: "vision", clearance: "GOOD", userAgent: BROWSER_UA, browserEgress: "direct" };
    await list();
    expect(await state()).toMatchObject({ lastSolvedBy: "vision", cfReuse: "ok" });
  });

  it("复用失败:cf_clearance 给 HTTP 仍被挑战 → cfReuse = fails,本次由浏览器代抓,6 小时浏览器代理期内页面请求直接走浏览器", async () => {
    cf.pass = () => false; // 站点绑定 TLS 指纹:HTTP 怎么都过不去
    const page = await list();
    expect(page.items.length).toBeGreaterThan(0);
    expect(cf.challenged).toBe(2); // 首次 + 带 cf_clearance 重试 1 次
    expect(fetches).toHaveLength(1);
    const st = await state();
    expect(st).toMatchObject({ cfReuse: "fails", lastResult: "reuse_fails" });
    const until = Date.parse(st!.browserUntil!);
    expect(until - Date.now()).toBeGreaterThan(5.9 * 3_600_000);
    // 代理期内:不发 HTTP,不再过挑战
    await list();
    expect(cf.challenged).toBe(2);
    expect(fetches).toHaveLength(2);
    expect(solves).toHaveLength(1);
    // 下载(不是页面请求)仍走 HTTP,报 challenge,不带失败过的 cf_clearance
    const err = await h.rt
      .siteAdapter(await site())
      .downloadTarget(page.items[0]!.siteReleaseId)
      .catch((e) => e);
    expect(err).toBeInstanceOf(SiteError);
    expect((err as SiteError).kind).toBe("challenge");
    expect(seen.at(-1)?.get("cookie") ?? "").not.toContain("cf_clearance");
    // 到期再试 HTTP
    await h.rt.cloudflare.patch(siteId, { browserUntil: new Date(Date.now() - 1000).toISOString() });
    cf.pass = () => true;
    const n = fetches.length;
    await list();
    expect(fetches).toHaveLength(n);
  });

  it("真验证码转人工:本次报 challenge(不算凭据错误),站点记 pending = human;快照不置账号失效、不开登录失效事项", async () => {
    solveResult = { status: "needs_human", sessionId: null, decisionId: null, reason: "选图验证码" };
    const r = await snapshotSite(h.rt, { siteId, kind: "manual" });
    expect(r.status).toBe("failed");
    expect(r.error).toContain("challenge");
    expect(fetches).toEqual([]);
    expect(await state()).toMatchObject({ pending: "human", lastResult: "needs_human", lastError: "选图验证码" });
    expect((await account()).status).toBe("active");
    expect((await site()).health_status).toBe("degraded");
    const d = await h.db.selectFrom("decision").select("id").where("dedupe_key", "=", `site-auth:${siteId}`).execute();
    expect(d).toEqual([]);
    expect(h.engine.enqueued.filter((e) => e.task === "refreshSiteAccount")).toEqual([]);

    // 人在原站视图里过完、交还:浏览器模块回调 afterHuman → 清掉等人处理,存下 cf_clearance,马上可以再试
    await h.rt.cloudflare.afterHuman(siteId, accountId, {
      clearance: "GOOD",
      userAgent: BROWSER_UA,
      browserEgress: "direct",
    });
    expect(await state()).toMatchObject({ pending: null, lastSolvedBy: "human", lastAttemptAt: null });
    const page = await list();
    expect(page.items.length).toBeGreaterThan(0);
    expect(solves).toHaveLength(1);
  });

  it("节流与单飞:并发的挑战只过一次;30 分钟内不再过;超过 30 分钟再过", async () => {
    solveDelay = 150;
    cf.pass = () => false;
    fetchResult = { status: "challenge", reason: "still" };
    const results = await Promise.allSettled([list(), list(), list()]);
    expect(results.every((r) => r.status === "rejected")).toBe(true);
    expect(solves).toHaveLength(1);
    // 30 分钟内:只记挑战,不过
    await list().catch(() => null);
    expect(solves).toHaveLength(1);
    expect((await state())?.lastAttemptAt).toBeTruthy();
    // 超过 30 分钟
    await h.rt.cloudflare.patch(siteId, {
      lastAttemptAt: new Date(Date.now() - CF_ATTEMPT_INTERVAL_MS - 1000).toISOString(),
    });
    await list().catch(() => null);
    expect(solves).toHaveLength(2);
  });

  it("同步条件不满足:账号手动 UA → 不把 cf_clearance 交给 HTTP(不重试),页面改由浏览器代抓", async () => {
    await h.db.updateTable("site_account").set({ user_agent: BROWSER_UA }).where("id", "=", accountId).execute();
    const page = await list();
    expect(page.items.length).toBeGreaterThan(0);
    expect(cf.challenged).toBe(1);
    expect(fetches).toHaveLength(1);
    expect(seen.every((x) => !(x.get("cookie") ?? "").includes("cf_clearance"))).toBe(true);
    expect((await state())?.browserUntil).toBeTruthy();
  });

  it("同步条件不满足:浏览器出口与 HTTP 不同(旧插件按环境变量出网)→ 不复用", async () => {
    solveResult = { status: "passed", by: "auto", clearance: "GOOD", userAgent: BROWSER_UA, browserEgress: null };
    await list();
    expect(cf.challenged).toBe(1);
    expect(fetches).toHaveLength(1);
    expect((await state())?.cfReuse).toBeNull();
  });

  it("没有登录态账号的站点不处理挑战", async () => {
    await h.db.updateTable("site_account").set({ status: "expired" }).where("id", "=", accountId).execute();
    // 适配器没有 Cookie 就不发请求;这里直接用站点 fetch(RSS 这类不要登录态的请求同理)
    const res = await h.rt.siteFetch(await site())(`${mock.baseUrl}/torrents.php`, {
      headers: { accept: "text/html" },
    });
    expect(res.status).toBe(403);
    expect(solves).toEqual([]);
    expect(fetches).toEqual([]);
    expect(await state()).toMatchObject({ lastResult: "skipped" });
  });

  it("健康检测遇挑战且过不去:记 challenge(degraded),账号状态不变", async () => {
    solveResult = { status: "failed", reason: "浏览器服务不可用" };
    fetchResult = { status: "unavailable", reason: "浏览器服务不可用" };
    const r = await json(await h.req(`/sites/${siteId}/health-check`, { method: "POST", body: {} }));
    expect(r).toMatchObject({ ok: false, status: "challenge", healthStatus: "degraded", loggedIn: null });
    expect((await account()).status).toBe("active");
  });
});