rootandClaude Opus 5.5
bd475a4ce1
- handle 对页面请求(GET + Accept text/html)的 2xx 响应也做挑战判定(短页、无站点内容才看正文), 过挑战 → cf_clearance 回带重试 → 浏览器代抓的级联照常触发 - 没处理掉时返回的挑战响应带 x-umt-challenge(needs_human / failed / throttled / no_browser / no_account), 适配器据此区分「需要你在原站视图里过一次验证」与「已自动尝试，没有成功」等文案 - 测试:200 门页过挑战后重试成功、200 挑战页走浏览器代抓、转人工时搜索报 challenge 并留脱敏样本 Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei
387 lines
18 KiB
TypeScript
387 lines
18 KiB
TypeScript
import { SiteError } from "@umt/adapters";
|
|
import { siteCloudflareState } from "@umt/contracts";
|
|
import { createMockNexusPhpSite, type MockNexusPhpSite } from "@umt/testkit";
|
|
import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest";
|
|
import {
|
|
type BrowserFetchOutcome,
|
|
browserFetchable,
|
|
CF_ATTEMPT_INTERVAL_MS,
|
|
type ChallengeSolveOutcome,
|
|
clearanceUsable,
|
|
registerChallengeBrowser,
|
|
withClearance,
|
|
} from "../../src/modules/sources-identity/service/cloudflare.ts";
|
|
import { snapshotSite } from "../../src/modules/sources-identity/service/snapshot.ts";
|
|
import { registerBrowserIdentityReader } from "../../src/modules/sources-identity/service/user-agent.ts";
|
|
import { createHarness, type Harness, pgAvailable } from "./harness.ts";
|
|
|
|
// CF-2 级联状态机:HTTP 遇 Cloudflare 挑战 → 浏览器过挑战(自动 / 视觉子流程 / 转人工)→ cf_clearance 同步给 HTTP 重试 1 次 →
|
|
// 仍被挑战则页面请求改由浏览器代抓并进入 6 小时浏览器代理期;同站 30 分钟节流、同一时刻只处理一次;挑战不算凭据错误。
|
|
// 站点 = testkit mock NexusPHP(真 HTTP),前面套一层"Cloudflare":没带合格 cf_clearance 的请求回 403 + cf-mitigated 挑战页。
|
|
// 浏览器一侧用假端口(ChallengeBrowserPort),真实浏览器流程见 browser-cloudflare.test.ts。
|
|
|
|
const up = await pgAvailable();
|
|
|
|
/** 轮询到有结果(留样是异步写入的) */
|
|
async function until<T>(f: () => Promise<T[]>, ms = 5_000): Promise<T[]> {
|
|
const deadline = Date.now() + ms;
|
|
for (;;) {
|
|
const r = await f();
|
|
if (r.length || Date.now() > deadline) return r;
|
|
await new Promise((res) => setTimeout(res, 50));
|
|
}
|
|
}
|
|
const BROWSER_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:152.0) Gecko/20100101 Firefox/152.0";
|
|
const CF_HTML = `<!DOCTYPE html><html><head><title>Just a moment...</title></head><body>
|
|
<div id="challenge-spinner"></div><script>window._cf_chl_opt={cType: 'managed'}</script></body></html>`;
|
|
/** 返回 200 的 Turnstile 门页(没有 cf-mitigated 头):搜索页单独加盾、间歇触发时见到的形态 */
|
|
const GATE_HTML = `<!DOCTYPE html><html><head><title>Audiences</title>
|
|
<script src="https://challenges.cloudflare.com/turnstile/v0/api.js" async defer></script></head>
|
|
<body><p>请完成人机验证后继续</p><div class="cf-turnstile" data-sitekey="0x4AAAAAAA"></div>
|
|
<a href="download.php?id=1&passkey=0123456789abcdef0123456789abcdef">x</a></body></html>`;
|
|
|
|
// biome-ignore lint/suspicious/noExplicitAny: 响应体断言
|
|
const json = async (r: Response): Promise<any> => {
|
|
const body = await r.json();
|
|
if (!r.ok) throw new Error(`${r.status} ${JSON.stringify(body)}`);
|
|
return body.data;
|
|
};
|
|
|
|
describe("CF-2 纯函数", () => {
|
|
it("页面请求才能由浏览器代抓", () => {
|
|
expect(browserFetchable({ method: "GET", accept: "text/html,application/xhtml+xml" })).toBe(true);
|
|
expect(browserFetchable({ method: "GET", accept: null })).toBe(false);
|
|
expect(browserFetchable({ method: "POST", accept: "text/html" })).toBe(false);
|
|
expect(browserFetchable({ method: "GET", accept: "application/rss+xml" })).toBe(false);
|
|
});
|
|
it("cf_clearance 同步条件:UA 跟随浏览器 + UA 相同 + 两端出口相同 + 复用没失败过", () => {
|
|
const base = {
|
|
uaSource: "browser",
|
|
requestUa: "A",
|
|
clearanceUa: "A",
|
|
requestEgress: "direct",
|
|
clearanceEgress: "direct",
|
|
cfReuse: null,
|
|
} as const;
|
|
expect(clearanceUsable(base)).toBe(true);
|
|
expect(clearanceUsable({ ...base, uaSource: "manual" })).toBe(false);
|
|
expect(clearanceUsable({ ...base, uaSource: "global" })).toBe(false);
|
|
expect(clearanceUsable({ ...base, clearanceUa: "B" })).toBe(false);
|
|
expect(clearanceUsable({ ...base, clearanceEgress: null })).toBe(false);
|
|
expect(clearanceUsable({ ...base, clearanceEgress: "proxy:http://relay:7891" })).toBe(false);
|
|
expect(clearanceUsable({ ...base, cfReuse: "fails" })).toBe(false);
|
|
});
|
|
it("并进 cf_clearance,去掉 __cf_bm", () => {
|
|
expect(withClearance("a=1; __cf_bm=x", "cf_clearance=Z")).toBe("a=1; cf_clearance=Z");
|
|
expect(withClearance(null, "cf_clearance=Z")).toBe("cf_clearance=Z");
|
|
expect(withClearance("a=1", null)).toBe("a=1");
|
|
});
|
|
});
|
|
|
|
describe.skipIf(!up)("CF-2 Cloudflare 级联", () => {
|
|
let h: Harness;
|
|
let mock: MockNexusPhpSite;
|
|
let siteId: string;
|
|
let accountId: string;
|
|
/** "Cloudflare" 开关与放行规则 */
|
|
const cf = {
|
|
on: false,
|
|
/** 放行判定;缺省:带 cf_clearance=GOOD 且 UA 是浏览器 UA */
|
|
pass: (h: Headers) => (h.get("cookie") ?? "").includes("cf_clearance=GOOD") && h.get("user-agent") === BROWSER_UA,
|
|
challenged: 0,
|
|
passed: 0,
|
|
/** 非空:挑战响应改成 200 + 这段 HTML(不带 cf-mitigated 头) */
|
|
okPage: null as string | null,
|
|
};
|
|
const seen: Headers[] = [];
|
|
const solves: { url: string; accountId: string }[] = [];
|
|
const fetches: string[] = [];
|
|
let solveResult: ChallengeSolveOutcome = { status: "failed", reason: "unset" };
|
|
let solveDelay = 0;
|
|
let fetchResult: BrowserFetchOutcome | null = null;
|
|
|
|
const cfFetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
|
|
const hd = new Headers(init?.headers);
|
|
seen.push(hd);
|
|
if (cf.on && !cf.pass(hd)) {
|
|
cf.challenged++;
|
|
if (cf.okPage) {
|
|
return new Response(cf.okPage, {
|
|
status: 200,
|
|
headers: { "content-type": "text/html; charset=utf-8", server: "cloudflare", "cf-ray": "8c00-HKG" },
|
|
});
|
|
}
|
|
return new Response(CF_HTML, {
|
|
status: 403,
|
|
headers: { "cf-mitigated": "challenge", "content-type": "text/html" },
|
|
});
|
|
}
|
|
cf.passed++;
|
|
return globalThis.fetch(input, init);
|
|
}) as typeof fetch;
|
|
|
|
const site = () => h.db.selectFrom("site").selectAll().where("id", "=", siteId).executeTakeFirstOrThrow();
|
|
const account = () =>
|
|
h.db.selectFrom("site_account").selectAll().where("id", "=", accountId).executeTakeFirstOrThrow();
|
|
const state = async () => siteCloudflareState((await site()).config);
|
|
const list = async () => h.rt.siteAdapter(await site()).list(1);
|
|
|
|
beforeAll(async () => {
|
|
mock = await createMockNexusPhpSite({});
|
|
h = await createHarness("cf2", { fetch: cfFetch });
|
|
registerBrowserIdentityReader(h.ctx, async () => BROWSER_UA);
|
|
registerChallengeBrowser(h.ctx, {
|
|
async solve(req) {
|
|
solves.push(req);
|
|
if (solveDelay) await new Promise((r) => setTimeout(r, solveDelay));
|
|
return solveResult;
|
|
},
|
|
async fetchHtml(req) {
|
|
fetches.push(req.url);
|
|
if (fetchResult) return fetchResult;
|
|
// 浏览器已登录且过了挑战:直接取 mock 页面
|
|
const r = await globalThis.fetch(req.url, { headers: { cookie: mock.cookie } });
|
|
return { status: "ok", html: await r.text(), url: req.url };
|
|
},
|
|
});
|
|
const s = await json(
|
|
await h.req("/sites", {
|
|
body: {
|
|
key: "audiences",
|
|
name: "观众",
|
|
adapter: "nexusphp",
|
|
baseUrl: `${mock.baseUrl}/`,
|
|
rateLimit: { minIntervalMs: 0, concurrency: 10 },
|
|
},
|
|
}),
|
|
);
|
|
siteId = s.id;
|
|
const r = await json(
|
|
await h.req(`/sites/${siteId}/credentials`, { method: "PUT", body: { kind: "cookie", cookie: mock.cookie } }),
|
|
);
|
|
accountId = r.siteAccountId;
|
|
});
|
|
afterAll(async () => {
|
|
await h?.close();
|
|
await mock?.close();
|
|
});
|
|
|
|
beforeEach(async () => {
|
|
cf.on = true;
|
|
cf.pass = (hd) => (hd.get("cookie") ?? "").includes("cf_clearance=GOOD") && hd.get("user-agent") === BROWSER_UA;
|
|
cf.challenged = 0;
|
|
cf.passed = 0;
|
|
cf.okPage = null;
|
|
seen.length = 0;
|
|
solves.length = 0;
|
|
fetches.length = 0;
|
|
solveDelay = 0;
|
|
fetchResult = null;
|
|
solveResult = { status: "passed", by: "auto", clearance: "GOOD", userAgent: BROWSER_UA, browserEgress: "direct" };
|
|
await h.db
|
|
.updateTable("site")
|
|
.set({ config: JSON.stringify({}), health_status: "healthy" })
|
|
.where("id", "=", siteId)
|
|
.execute();
|
|
await h.db
|
|
.updateTable("site_account")
|
|
.set({ user_info: JSON.stringify({}), user_agent: null, ua_pending: null, status: "active" })
|
|
.where("id", "=", accountId)
|
|
.execute();
|
|
await h.rt.cloudflare.patch(siteId, {});
|
|
});
|
|
|
|
it("自动通过:浏览器拿到 cf_clearance + UA,HTTP 带上重试 1 次成功,记 cfReuse = ok;之后的请求直接带", async () => {
|
|
const page = await list();
|
|
expect(page.items.length).toBeGreaterThan(0);
|
|
expect(solves).toEqual([{ siteId, accountId, url: expect.stringContaining("/torrents.php") }]);
|
|
expect(cf.challenged).toBe(1);
|
|
expect(fetches).toEqual([]);
|
|
const st = await state();
|
|
expect(st).toMatchObject({ lastSolvedBy: "auto", cfReuse: "ok", lastResult: "reuse_ok", browserUntil: null });
|
|
expect(st?.lastChallengeAt).toBeTruthy();
|
|
expect(st?.lastRule).toBe("header cf-mitigated=challenge");
|
|
// cf_clearance 存在 secret 里,账号 user_info 只有引用与取得时的 UA / 出口
|
|
const info = (await account()).user_info as {
|
|
cloudflare?: { secretRef: string; userAgent: string; egress: string };
|
|
};
|
|
expect(info.cloudflare).toMatchObject({ userAgent: BROWSER_UA, egress: "direct" });
|
|
expect(h.secrets.plain.get(info.cloudflare!.secretRef)).toBe("GOOD");
|
|
expect(JSON.stringify(info)).not.toContain("GOOD");
|
|
// __cf_bm 不带;下一条请求直接带 cf_clearance,不再过挑战
|
|
seen.length = 0;
|
|
await list();
|
|
expect(solves).toHaveLength(1);
|
|
expect(seen[0]?.get("cookie")).toContain("cf_clearance=GOOD");
|
|
expect(seen[0]?.get("cookie")).not.toContain("__cf_bm");
|
|
// 账号状态不变
|
|
expect((await account()).status).toBe("active");
|
|
});
|
|
|
|
it("视觉子流程通过:记 lastSolvedBy = vision", async () => {
|
|
solveResult = { status: "passed", by: "vision", clearance: "GOOD", userAgent: BROWSER_UA, browserEgress: "direct" };
|
|
await list();
|
|
expect(await state()).toMatchObject({ lastSolvedBy: "vision", cfReuse: "ok" });
|
|
});
|
|
|
|
it("复用失败:cf_clearance 给 HTTP 仍被挑战 → cfReuse = fails,本次由浏览器代抓,6 小时浏览器代理期内页面请求直接走浏览器", async () => {
|
|
cf.pass = () => false; // 站点绑定 TLS 指纹:HTTP 怎么都过不去
|
|
const page = await list();
|
|
expect(page.items.length).toBeGreaterThan(0);
|
|
expect(cf.challenged).toBe(2); // 首次 + 带 cf_clearance 重试 1 次
|
|
expect(fetches).toHaveLength(1);
|
|
const st = await state();
|
|
expect(st).toMatchObject({ cfReuse: "fails", lastResult: "reuse_fails" });
|
|
const until = Date.parse(st!.browserUntil!);
|
|
expect(until - Date.now()).toBeGreaterThan(5.9 * 3_600_000);
|
|
// 代理期内:不发 HTTP,不再过挑战
|
|
await list();
|
|
expect(cf.challenged).toBe(2);
|
|
expect(fetches).toHaveLength(2);
|
|
expect(solves).toHaveLength(1);
|
|
// 下载(不是页面请求)仍走 HTTP,报 challenge,不带失败过的 cf_clearance
|
|
const err = await h.rt
|
|
.siteAdapter(await site())
|
|
.downloadTarget(page.items[0]!.siteReleaseId)
|
|
.catch((e) => e);
|
|
expect(err).toBeInstanceOf(SiteError);
|
|
expect((err as SiteError).kind).toBe("challenge");
|
|
expect(seen.at(-1)?.get("cookie") ?? "").not.toContain("cf_clearance");
|
|
// 到期再试 HTTP
|
|
await h.rt.cloudflare.patch(siteId, { browserUntil: new Date(Date.now() - 1000).toISOString() });
|
|
cf.pass = () => true;
|
|
const n = fetches.length;
|
|
await list();
|
|
expect(fetches).toHaveLength(n);
|
|
});
|
|
|
|
it("真验证码转人工:本次报 challenge(不算凭据错误),站点记 pending = human;快照不置账号失效、不开登录失效事项", async () => {
|
|
solveResult = { status: "needs_human", sessionId: null, decisionId: null, reason: "选图验证码" };
|
|
const r = await snapshotSite(h.rt, { siteId, kind: "manual" });
|
|
expect(r.status).toBe("failed");
|
|
expect(r.error).toContain("challenge");
|
|
expect(fetches).toEqual([]);
|
|
expect(await state()).toMatchObject({ pending: "human", lastResult: "needs_human", lastError: "选图验证码" });
|
|
expect((await account()).status).toBe("active");
|
|
expect((await site()).health_status).toBe("degraded");
|
|
const d = await h.db.selectFrom("decision").select("id").where("dedupe_key", "=", `site-auth:${siteId}`).execute();
|
|
expect(d).toEqual([]);
|
|
expect(h.engine.enqueued.filter((e) => e.task === "refreshSiteAccount")).toEqual([]);
|
|
|
|
// 人在原站视图里过完、交还:浏览器模块回调 afterHuman → 清掉等人处理,存下 cf_clearance,马上可以再试
|
|
await h.rt.cloudflare.afterHuman(siteId, accountId, {
|
|
clearance: "GOOD",
|
|
userAgent: BROWSER_UA,
|
|
browserEgress: "direct",
|
|
});
|
|
expect(await state()).toMatchObject({ pending: null, lastSolvedBy: "human", lastAttemptAt: null });
|
|
const page = await list();
|
|
expect(page.items.length).toBeGreaterThan(0);
|
|
expect(solves).toHaveLength(1);
|
|
});
|
|
|
|
it("节流与单飞:并发的挑战只过一次;30 分钟内不再过;超过 30 分钟再过", async () => {
|
|
solveDelay = 150;
|
|
cf.pass = () => false;
|
|
fetchResult = { status: "challenge", reason: "still" };
|
|
const results = await Promise.allSettled([list(), list(), list()]);
|
|
expect(results.every((r) => r.status === "rejected")).toBe(true);
|
|
expect(solves).toHaveLength(1);
|
|
// 30 分钟内:只记挑战,不过
|
|
await list().catch(() => null);
|
|
expect(solves).toHaveLength(1);
|
|
expect((await state())?.lastAttemptAt).toBeTruthy();
|
|
// 超过 30 分钟
|
|
await h.rt.cloudflare.patch(siteId, {
|
|
lastAttemptAt: new Date(Date.now() - CF_ATTEMPT_INTERVAL_MS - 1000).toISOString(),
|
|
});
|
|
await list().catch(() => null);
|
|
expect(solves).toHaveLength(2);
|
|
});
|
|
|
|
it("同步条件不满足:账号手动 UA → 不把 cf_clearance 交给 HTTP(不重试),页面改由浏览器代抓", async () => {
|
|
await h.db.updateTable("site_account").set({ user_agent: BROWSER_UA }).where("id", "=", accountId).execute();
|
|
const page = await list();
|
|
expect(page.items.length).toBeGreaterThan(0);
|
|
expect(cf.challenged).toBe(1);
|
|
expect(fetches).toHaveLength(1);
|
|
expect(seen.every((x) => !(x.get("cookie") ?? "").includes("cf_clearance"))).toBe(true);
|
|
expect((await state())?.browserUntil).toBeTruthy();
|
|
});
|
|
|
|
it("同步条件不满足:浏览器出口与 HTTP 不同(旧插件按环境变量出网)→ 不复用", async () => {
|
|
solveResult = { status: "passed", by: "auto", clearance: "GOOD", userAgent: BROWSER_UA, browserEgress: null };
|
|
await list();
|
|
expect(cf.challenged).toBe(1);
|
|
expect(fetches).toHaveLength(1);
|
|
expect((await state())?.cfReuse).toBeNull();
|
|
});
|
|
|
|
it("没有登录态账号的站点不处理挑战", async () => {
|
|
await h.db.updateTable("site_account").set({ status: "expired" }).where("id", "=", accountId).execute();
|
|
// 适配器没有 Cookie 就不发请求;这里直接用站点 fetch(RSS 这类不要登录态的请求同理)
|
|
const res = await h.rt.siteFetch(await site())(`${mock.baseUrl}/torrents.php`, {
|
|
headers: { accept: "text/html" },
|
|
});
|
|
expect(res.status).toBe(403);
|
|
expect(solves).toEqual([]);
|
|
expect(fetches).toEqual([]);
|
|
expect(await state()).toMatchObject({ lastResult: "skipped" });
|
|
});
|
|
|
|
it("页面请求返回 200 的 Turnstile 门页 → 进入级联:过挑战后带 cf_clearance 重试成功", async () => {
|
|
cf.okPage = GATE_HTML;
|
|
const page = await list();
|
|
expect(page.items.length).toBeGreaterThan(0);
|
|
expect(cf.challenged).toBe(1);
|
|
expect(solves).toHaveLength(1);
|
|
expect(fetches).toEqual([]);
|
|
const st = await state();
|
|
expect(st).toMatchObject({ cfReuse: "ok", lastResult: "reuse_ok" });
|
|
expect(st?.lastRule).toMatch(/^gate\*=/);
|
|
});
|
|
|
|
it("200 的 Cloudflare 挑战页且 HTTP 过不去 → 浏览器代抓,进入浏览器代理期", async () => {
|
|
cf.okPage = CF_HTML;
|
|
cf.pass = () => false;
|
|
const page = await h.rt.siteAdapter(await site()).search("Dune");
|
|
expect(page.items.length).toBeGreaterThan(0);
|
|
expect(cf.challenged).toBe(2);
|
|
expect(fetches).toHaveLength(1);
|
|
expect(fetches[0]).toContain("search=Dune");
|
|
expect((await state())?.browserUntil).toBeTruthy();
|
|
});
|
|
|
|
it("200 门页转人工:搜索报 challenge(文案说要人在原站视图过验证),留样标搜索、已脱敏", async () => {
|
|
cf.okPage = GATE_HTML;
|
|
solveResult = { status: "needs_human", sessionId: null, decisionId: null, reason: "选图验证码" };
|
|
await h.db.deleteFrom("site_failure_sample").where("site_id", "=", siteId).execute();
|
|
const err = await h.rt
|
|
.siteAdapter(await site())
|
|
.search("Dune")
|
|
.catch((e) => e);
|
|
expect(err).toBeInstanceOf(SiteError);
|
|
expect((err as SiteError).kind).toBe("challenge");
|
|
expect((err as SiteError).message).toContain("搜索页需要人机验证");
|
|
expect((err as SiteError).message).toContain("原站视图");
|
|
expect((err as SiteError).message).not.toContain("改版");
|
|
expect((await account()).status).toBe("active");
|
|
const rows = await until(() =>
|
|
h.db.selectFrom("site_failure_sample").selectAll().where("site_id", "=", siteId).execute(),
|
|
);
|
|
expect(rows[0]).toMatchObject({ kind: "challenge", request_kind: "search", http_status: 200 });
|
|
expect(rows[0]?.headers).toMatchObject({ server: "cloudflare", "cf-ray": "8c00-HKG" });
|
|
expect(rows[0]?.body).toContain("cf-turnstile");
|
|
expect(rows[0]?.body).not.toContain("0123456789abcdef0123456789abcdef");
|
|
expect(rows[0]?.url).toContain("search=Dune");
|
|
});
|
|
|
|
it("健康检测遇挑战且过不去:记 challenge(degraded),账号状态不变", async () => {
|
|
solveResult = { status: "failed", reason: "浏览器服务不可用" };
|
|
fetchResult = { status: "unavailable", reason: "浏览器服务不可用" };
|
|
const r = await json(await h.req(`/sites/${siteId}/health-check`, { method: "POST", body: {} }));
|
|
expect(r).toMatchObject({ ok: false, status: "challenge", healthStatus: "degraded", loggedIn: null });
|
|
expect((await account()).status).toBe("active");
|
|
});
|
|
});
|