From a11ef16e40015b166ba2f8595267c580368996ac Mon Sep 17 00:00:00 2001 From: root Date: Wed, 30 Sep 2026 21:26:53 +0800 Subject: [PATCH 1/5] =?UTF-8?q?fix(sites):=20Cloudflare=20=E6=8C=91?= =?UTF-8?q?=E6=88=98=E4=B8=8D=E7=AE=97=E5=87=AD=E6=8D=AE=E9=94=99=E8=AF=AF?= =?UTF-8?q?(CF-2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - http.ts:cf-mitigated: challenge 或 403/503/429 挑战页 → SiteError(challenge),503 不再算维护、403 不再落到 auth_invalid - 挑战页分类 classifyChallengePage:自动 / 可点(Turnstile)/ 真验证码 - 刷新登录 / 用户数据遇 Cloudflare 挑战只记 last_error,账号状态不变、不开登录失效事项 Co-Authored-By: Claude Opus 5.5 --- .../sources-identity/service/account.ts | 3 +- packages/adapters/src/sites/challenge.test.ts | 116 ++++++++++++++++++ packages/adapters/src/sites/challenge.ts | 83 +++++++++++++ packages/adapters/src/sites/http.ts | 13 +- packages/adapters/src/sites/index.ts | 8 ++ 5 files changed, 221 insertions(+), 2 deletions(-) create mode 100644 packages/adapters/src/sites/challenge.test.ts create mode 100644 packages/adapters/src/sites/challenge.ts diff --git a/apps/server/src/modules/sources-identity/service/account.ts b/apps/server/src/modules/sources-identity/service/account.ts index 4d295555..d4eca063 100644 --- a/apps/server/src/modules/sources-identity/service/account.ts +++ b/apps/server/src/modules/sources-identity/service/account.ts @@ -688,8 +688,9 @@ async function markFailure( ): Promise { const now = rt.now(); const lastError = { reason, message: message.slice(0, 300), at: now.toISOString() }; + // Cloudflare 挑战(CF-2)不是凭据问题:账号状态不变、不开"登录失效"事项;过挑战由 Cloudflare 处理负责,需要人时它开事项 const status = - reason === "network" || reason === "unsupported" + reason === "network" || reason === "unsupported" || reason === "challenge" ? null : reason === "auth_expired" && acc.auth_kind === "api_key" ? "invalid" diff --git a/packages/adapters/src/sites/challenge.test.ts b/packages/adapters/src/sites/challenge.test.ts new file mode 100644 index 00000000..537772e9 --- /dev/null +++ b/packages/adapters/src/sites/challenge.test.ts @@ -0,0 +1,116 @@ +import { describe, expect, it } from "vitest"; +import { challengeRuleOfHtml, classifyChallengePage, detectChallenge } from "./challenge.ts"; +import { siteFetch } from "./http.ts"; +import { createSiteRateLimiter } from "./rate-limit.ts"; +import { SiteError } from "./types.ts"; + +// CF-2:Cloudflare 挑战识别。HTTP 端挑战不是凭据错误(403 不再算 auth_invalid、503 不再算维护); +// 浏览器端按页面分:自动 / 可点(Turnstile)/ 真验证码。 + +const CF_PAGE = `Just a moment... +
`; +const CF_JS_PAGE = `Just a moment... +`; +const NORMAL = `种子 - 观众ok`; + +function ctx(fetchImpl: typeof fetch) { + return { + siteId: "s1", + fetch: fetchImpl, + limiter: createSiteRateLimiter({ minIntervalMs: 0, burst: 10 }), + timeoutMs: 5_000, + now: () => new Date(), + }; +} +const reply = (status: number, body: string, headers: Record = {}) => + (async () => new Response(body, { status, headers: { "content-type": "text/html", ...headers } })) as typeof fetch; + +describe("challengeRuleOfHtml", () => { + it("挑战页标题 / 脚本特征命中;正常页面里注入的 challenge-platform 脚本不算", () => { + expect(challengeRuleOfHtml(CF_PAGE)).toBe("title=Just a moment..."); + expect(challengeRuleOfHtml('
')).toContain( + "challenge-form", + ); + expect(challengeRuleOfHtml(NORMAL)).toBeNull(); + expect(challengeRuleOfHtml("")).toBeNull(); + }); +}); + +describe("detectChallenge", () => { + it("cf-mitigated: challenge 不读正文;403 / 503 读正文判定并重新包装响应", async () => { + const a = await detectChallenge(new Response("x", { status: 200, headers: { "cf-mitigated": "challenge" } })); + expect(a.rule).toBe("header cf-mitigated=challenge"); + const b = await detectChallenge(new Response(CF_PAGE, { status: 403 })); + expect(b.rule).toContain("Just a moment"); + expect(await b.res.text()).toBe(CF_PAGE); + const c = await detectChallenge(new Response("Forbidden", { status: 403 })); + expect(c.rule).toBeNull(); + expect(await c.res.text()).toBe("Forbidden"); + const d = await detectChallenge(new Response(NORMAL, { status: 200 })); + expect(d.rule).toBeNull(); + }); +}); + +describe("siteFetch 挑战分类", () => { + const url = "https://audiences.example/torrents.php"; + const kind = async (f: typeof fetch) => { + try { + await siteFetch(ctx(f), url); + return "ok"; + } catch (e) { + return e instanceof SiteError ? e.kind : "other"; + } + }; + + it("403 + cf-mitigated → challenge(不是 auth_invalid),不算凭据错误", async () => { + let err: unknown; + try { + await siteFetch(ctx(reply(403, CF_PAGE, { "cf-mitigated": "challenge" })), url); + } catch (e) { + err = e; + } + expect(err).toBeInstanceOf(SiteError); + const se = err as SiteError; + expect(se.kind).toBe("challenge"); + expect(se.isAuthError).toBe(false); + expect(se.detail).toMatchObject({ reason: "challenge", rule: "header cf-mitigated=challenge", httpStatus: 403 }); + }); + + it("503 挑战页 → challenge(不是维护);403 挑战页(没有响应头)也认", async () => { + expect(await kind(reply(503, CF_PAGE))).toBe("challenge"); + expect(await kind(reply(403, CF_PAGE))).toBe("challenge"); + expect(await kind(reply(429, CF_JS_PAGE))).toBe("challenge"); + }); + + it("普通 503 仍是维护,普通 403 交给适配器(响应体可读)", async () => { + expect(await kind(reply(503, "maintenance"))).toBe("site_maintenance"); + const res = await siteFetch(ctx(reply(403, "Forbidden")), url); + expect(res.status).toBe(403); + expect(await res.text()).toBe("Forbidden"); + }); +}); + +describe("classifyChallengePage", () => { + it("不是挑战页 → none", () => { + expect(classifyChallengePage({ title: "种子", html: NORMAL, text: "种子列表" }).kind).toBe("none"); + }); + it("JS 检查 → waiting;托管 / Turnstile → interactive", () => { + expect(classifyChallengePage({ html: CF_JS_PAGE, text: "Checking your browser" }).kind).toBe("waiting"); + expect(classifyChallengePage({ html: CF_PAGE, text: "" }).kind).toBe("interactive"); + expect( + classifyChallengePage({ + title: "请稍候…", + html: "", + text: "请完成以下操作，验证您是真人。", + }).kind, + ).toBe("interactive"); + }); + it("选图 / 输码 → captcha(不自动处理)", () => { + expect(classifyChallengePage({ html: `${CF_PAGE}
`, text: "Just a moment" }).kind).toBe( + "captcha", + ); + expect(classifyChallengePage({ title: "Just a moment...", html: "", text: "请点击图中所有的红绿灯" }).kind).toBe( + "captcha", + ); + }); +}); diff --git a/packages/adapters/src/sites/challenge.ts b/packages/adapters/src/sites/challenge.ts new file mode 100644 index 00000000..0c521ab7 --- /dev/null +++ b/packages/adapters/src/sites/challenge.ts @@ -0,0 +1,83 @@ +// CF-2 Cloudflare 挑战识别(docs/design/cloudflare-handling.md)。 +// - HTTP 端:响应头 `cf-mitigated: challenge`(任何状态码),或 403 / 503 / 429 且正文是挑战页 → 挑战,不是凭据错误; +// - 浏览器端:挑战页分三类——自动(JS 检查,等几秒自己过)、可点(「我是人类」复选框 / Turnstile)、真验证码(选图 / 输码)。 +// 只用正则,不解析 DOM:HTTP 端要在限流器之后快速判断,浏览器端拿到的是 outerHTML + innerText 样本。 + +/** Cloudflare / DDoS-Guard 挑战页的标题 */ +const CHALLENGE_TITLE = /^\s*(?:just a moment|attention required! \| cloudflare|请稍候|ddos-guard|one moment, please)/i; +/** 挑战页的脚本 / 元素特征。不认 `/cdn-cgi/challenge-platform/`:Cloudflare 会往正常页面也注入这段脚本 */ +const CHALLENGE_HTML = + /cf-browser-verification|cf_chl_opt|cf-challenge-running|id="challenge-form"|id="challenge-spinner"|id="cf-please-wait"|id="trk_jschal_js"/i; + +function titleOf(html: string): string { + return /]*>([^<]*)<\/title>/i.exec(html.slice(0, 50_000))?.[1]?.trim() ?? ""; +} + +/** 页面是不是挑战页;命中返回规则说明(进技术信息),否则 null */ +export function challengeRuleOfHtml(html: string | null | undefined): string | null { + if (!html) return null; + const title = titleOf(html); + if (CHALLENGE_TITLE.test(title)) return `title=${title.slice(0, 40)}`; + const m = CHALLENGE_HTML.exec(html.slice(0, 200_000)); + return m ? `html*=${m[0]}` : null; +} + +/** 响应头判定(不读正文):`cf-mitigated: challenge` */ +export function challengeRuleOfHeaders(headers: Headers): string | null { + return headers.get("cf-mitigated")?.trim().toLowerCase() === "challenge" ? "header cf-mitigated=challenge" : null; +} + +/** 这些状态码的响应才去看正文(Cloudflare 挑战页是 403 / 503,少数 429) */ +export const CHALLENGE_STATUSES: readonly number[] = [403, 429, 503]; + +/** + * 判定一个站点响应是不是挑战。正文只在状态码可能是挑战时读(读过的正文重新包成 Response 返回,调用方照常消费)。 + * 返回 { res, rule }:rule 为 null 表示不是挑战。 + */ +export async function detectChallenge(res: Response): Promise<{ res: Response; rule: string | null }> { + const byHeader = challengeRuleOfHeaders(res.headers); + if (byHeader) return { res, rule: byHeader }; + if (!CHALLENGE_STATUSES.includes(res.status) || !res.body) return { res, rule: null }; + const bytes = new Uint8Array(await res.arrayBuffer()); + const html = new TextDecoder("utf-8").decode(bytes.subarray(0, 300_000)); + const copy = new Response(bytes, { status: res.status, statusText: res.statusText, headers: res.headers }); + return { res: copy, rule: challengeRuleOfHtml(html) }; +} + +/** + * 浏览器里打开的页面属于哪种挑战: + * - none:不是挑战页(已通过); + * - waiting:自动检查(JS / 非交互),等几秒会自己过; + * - interactive:「我是人类」复选框 / Turnstile,可以点一下; + * - captcha:选图、输验证码这类真验证码,不自动处理。 + */ +export type ChallengePageKind = "none" | "waiting" | "interactive" | "captcha"; + +const CAPTCHA_MARKERS = + /hcaptcha\.com|class="h-captcha"|g-recaptcha|recaptcha\/api|imagecode\.php|select all (?:images|squares)|请(?:依次)?点击|选择所有|拖动滑块|输入验证码|enter the (?:characters|code)/i; +const INTERACTIVE_MARKERS = + /challenges\.cloudflare\.com\/(?:turnstile|cdn-cgi)|cf-turnstile|cType\s*:\s*['"](?:managed|interactive)['"]|verify you are human|确认您是真人|验证您是真人|我是人类/i; + +export function classifyChallengePage(page: { title?: string | null; html?: string | null; text?: string | null }): { + kind: ChallengePageKind; + rule: string | null; +} { + const html = page.html ?? ""; + const text = page.text ?? ""; + const title = page.title ?? titleOf(html); + const rule = + (CHALLENGE_TITLE.test(title) ? `title=${title.slice(0, 40)}` : null) ?? + challengeRuleOfHtml(html) ?? + (/verify you are human|确认您是真人|验证您是真人|checking (?:if the site connection is secure|your browser)/i.test( + text, + ) + ? "text" + : null); + if (!rule) return { kind: "none", rule: null }; + const all = `${html.slice(0, 300_000)}\n${text.slice(0, 20_000)}`; + const cap = CAPTCHA_MARKERS.exec(all); + if (cap) return { kind: "captcha", rule: `${rule}; captcha*=${cap[0]}` }; + const inter = INTERACTIVE_MARKERS.exec(all); + if (inter) return { kind: "interactive", rule: `${rule}; interactive*=${inter[0]}` }; + return { kind: "waiting", rule }; +} diff --git a/packages/adapters/src/sites/http.ts b/packages/adapters/src/sites/http.ts index 33c88156..43e8d33f 100644 --- a/packages/adapters/src/sites/http.ts +++ b/packages/adapters/src/sites/http.ts @@ -1,4 +1,5 @@ import { redactText } from "../rss/redact.ts"; +import { detectChallenge } from "./challenge.ts"; import type { SiteRateLimiter } from "./rate-limit.ts"; import { SiteError } from "./types.ts"; @@ -20,7 +21,8 @@ function retryAfterMs(res: Response, now: Date): number | undefined { } /** - * 站点请求:先取站点令牌,再发请求;429/503 统一映射并顺延限流器。 + * 站点请求:先取站点令牌,再发请求;Cloudflare 挑战(`cf-mitigated: challenge`,或 403 / 503 / 429 的挑战页)→ `challenge` + * (不是凭据错误,CF-2);其余 429/503 统一映射并顺延限流器。 * 返回的 Response 由调用方消费。错误消息只含脱敏后的 URL。 */ export async function siteFetch( @@ -42,6 +44,15 @@ export async function siteFetch( const reason = timeout.aborted ? "超时" : (e as Error).message; throw new SiteError(ctx.siteId, "network", redactText(`请求 ${safeUrl} 失败:${reason}`)); } + const ch = await detectChallenge(res); + res = ch.res; + if (ch.rule) { + await res.body?.cancel().catch(() => {}); + throw new SiteError(ctx.siteId, "challenge", `请求 ${safeUrl}:站点要求人机验证(Cloudflare)`, { + httpStatus: res.status, + detail: { reason: "challenge", rule: ch.rule, httpStatus: res.status }, + }); + } if (res.status === 429 || res.status === 503) { const ra = retryAfterMs(res, ctx.now()); await res.body?.cancel().catch(() => {}); diff --git a/packages/adapters/src/sites/index.ts b/packages/adapters/src/sites/index.ts index 4c4a0bb4..0f19c314 100644 --- a/packages/adapters/src/sites/index.ts +++ b/packages/adapters/src/sites/index.ts @@ -1,3 +1,11 @@ +export { + CHALLENGE_STATUSES, + type ChallengePageKind, + challengeRuleOfHeaders, + challengeRuleOfHtml, + classifyChallengePage, + detectChallenge, +} from "./challenge.ts"; export { audiencesDefinition, BUILTIN_SITE_DEFINITIONS, From 6b14b0da4502223bd4c5cb87b0ca6e9aa7fe26d7 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 30 Sep 2026 21:26:53 +0800 Subject: [PATCH 2/5] =?UTF-8?q?feat(browser):=20=E6=8F=92=E4=BB=B6?= =?UTF-8?q?=E7=9C=9F=E5=AE=9E=E5=9D=90=E6=A0=87=E7=82=B9=E5=87=BB=E4=B8=8E?= =?UTF-8?q?=E9=A1=B5=E9=9D=A2=E5=86=85=E5=AE=B9=E8=AF=BB=E5=8F=96(CF-2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - umt 插件 POST /umt/tabs/{id}/mouse-click:page.mouse.click,跨域 iframe 的 Turnstile 也点得到;click 坐标优先走它,旧镜像退回 elementFromPoint - 适配器 pageContent(地址 / 标题 / outerHTML / 正文 / navigator.userAgent);setEgress 返回插件出口模式 - mock camofox 增加 Cloudflare 模式(auto / interactive / captcha)与插件鼠标点击 Co-Authored-By: Claude Opus 5.5 --- apps/server/test/e2e/e14-vision.test.ts | 103 +++++++++++++++++- .../opt/camofox/plugins/umt-cookies/index.js | 38 ++++++- .../adapters/src/browser/camofox-real.test.ts | 2 + .../adapters/src/browser/camofox/client.ts | 102 ++++++++++++++++- packages/adapters/src/browser/provider.ts | 16 ++- .../adapters/src/browser/umt-plugin.test.ts | 59 +++++++++- packages/testkit/src/browser/mock-camofox.ts | 92 ++++++++++++++++ 7 files changed, 402 insertions(+), 10 deletions(-) diff --git a/apps/server/test/e2e/e14-vision.test.ts b/apps/server/test/e2e/e14-vision.test.ts index 9678fea2..05c10704 100644 --- a/apps/server/test/e2e/e14-vision.test.ts +++ b/apps/server/test/e2e/e14-vision.test.ts @@ -22,6 +22,7 @@ type Any = any; const ORIGIN = "https://audiences.example"; const ORIGIN_2FA = "https://secure.example"; +const ORIGIN_CF = "https://cf.example"; const USER = "fixture-user"; const PASSWORD = "e14-login-pw"; const ACCESS_KEY = "camofox-e14-key"; @@ -30,6 +31,7 @@ const ACCESS_KEY = "camofox-e14-key"; const AT = { username: { x: 240, y: 64 }, password: { x: 240, y: 96 } }; let sid = ""; +let cfSid = ""; const childResults = new Map(); let mainResults: FakeToolResult[] = []; @@ -37,6 +39,16 @@ const typeUser = () => ({ sessionId: sid, ...AT.username, text: USER, allowSensi const typePassword = () => ({ sessionId: sid, ...AT.password, text: PASSWORD, pressEnter: true, allowSensitive: true }); const scripts: FakeScript[] = [ + { + // CF-2 系统发起的视觉子流程(没有父会话):截图 → 点 Turnstile 复选框(第 2 行,截图坐标 (240, 64)) + name: "e14-cf", + match: "Cloudflare 人机验证页", + steps: [ + { toolCalls: [{ capability: "browser.screenshot", input: () => ({ sessionId: cfSid }) }] }, + { toolCalls: [{ capability: "browser.click_at", input: () => ({ sessionId: cfSid, x: 240, y: 64 }) }] }, + ], + finalText: () => "已点复选框", + }, { name: "e14-main", match: "E14 主会话登录", @@ -122,6 +134,7 @@ describe.skipIf(!hasPg)("E14 视觉自动化闭环", () => { let e: E2E; let mock: MockCamofox; let mock2fa: MockCamofox; + let mockCf: MockCamofox; let evidenceDir: string; let svc: BrowserService; const ids = { site: "", site2fa: "", provider: "", provider2fa: "" }; @@ -154,8 +167,16 @@ describe.skipIf(!hasPg)("E14 视觉自动化闭环", () => { site: { ...site(ORIGIN_2FA), twoFactor: true }, viewport: { width: 1600, height: 1000 }, }); + mockCf = await createMockCamofox({ + accessKey: ACCESS_KEY, + site: { ...site(ORIGIN_CF), protectedPaths: /^$/, cloudflare: { mode: "interactive", clearance: "clr-e14" } }, + viewport: { width: 1600, height: 1000 }, + }); evidenceDir = await mkdtemp(join(tmpdir(), "umt-e14-evidence-")); - e = await createE2E("e14", { fakeScripts: scripts, browser: { evidenceDir, settleMs: 0 } }); + e = await createE2E("e14", { + fakeScripts: scripts, + browser: { evidenceDir, settleMs: 0, challengePollMs: 10, challengeAutoWaitMs: 200 }, + }); svc = browserServiceFor(e.booted.ctx)!; const ctx = e.booted.ctx; const keyRef = await ctx.secrets.create({ purpose: "other", plaintext: ACCESS_KEY }); @@ -200,6 +221,7 @@ describe.skipIf(!hasPg)("E14 视觉自动化闭环", () => { await e?.close(); await mock?.close(); await mock2fa?.close(); + await mockCf?.close(); if (evidenceDir) await rm(evidenceDir, { recursive: true, force: true }); }); @@ -335,9 +357,14 @@ describe.skipIf(!hasPg)("E14 视觉自动化闭环", () => { // 坐标换算:截图 (240, 64) → 页面 (300, 80);(240, 96) → (300, 120) expect(r[1]?.json).toMatchObject({ pageX: 300, pageY: 80 }); expect(r[2]?.json).toMatchObject({ pageX: 300, pageY: 120 }); - const evals = mock.actions.filter((a) => a.action === "evaluate").map((a) => String(a.body.expression)); - expect(evals.some((x) => x.includes("elementFromPoint(300, 80)"))).toBe(true); - expect(evals.some((x) => x.includes("elementFromPoint(300, 120)"))).toBe(true); + // CF-2:坐标点击走 umt 插件的真实鼠标事件(按页面 CSS 像素) + const clicks = mock.actions.filter((a) => a.action === "mouse-click").map((a) => [a.body.x, a.body.y]); + expect(clicks).toEqual( + expect.arrayContaining([ + [300, 80], + [300, 120], + ]), + ); expect(mock.loggedIn.size).toBe(1); // 父会话事件流里有子流程 const events = await runEvents(e, runId); @@ -372,4 +399,72 @@ describe.skipIf(!hasPg)("E14 视觉自动化闭环", () => { expect(again).not.toBe(sid); await worker.close(again); }, 60_000); + + it("CF-2 站点请求遇 Turnstile:系统发起视觉子流程(没有父会话,预算取设置)点复选框,浏览器读回 cf_clearance", async () => { + // 只留 Cloudflare 这个浏览器服务(站点请求一侧不指定服务,取启用的那个) + await e.db.updateTable("browser_provider").set({ enabled: false }).execute(); + const providerCf = ( + await e.db + .insertInto("browser_provider") + .values({ + name: "camofox-e14-cf", + kind: "camofox", + base_url: mockCf.url, + secret_ref: ( + await e.db.selectFrom("browser_provider").select("secret_ref").executeTakeFirstOrThrow() + ).secret_ref, + enabled: true, + config: JSON.stringify({ navigationTimeoutMs: 2_000, maxRetries: 0 }), + }) + .returning("id") + .executeTakeFirstOrThrow() + ).id; + const siteCf = ( + await e.db + .insertInto("site") + .values({ + key: "audiences", + name: "观众", + adapter: "nexusphp", + base_url: `${ORIGIN_CF}/`, + rate_limit: JSON.stringify({ minIntervalMs: 0, concurrency: 5 }), + }) + .returning("id") + .executeTakeFirstOrThrow() + ).id; + const ref = await e.booted.ctx.secrets.create({ purpose: "site_cookie", plaintext: "c_secure_pass=e14" }); + const accountId = ( + await e.db + .insertInto("site_account") + .values({ site_id: siteCf, label: "main", auth_kind: "cookie", secret_ref: ref, status: "active" }) + .returning("id") + .executeTakeFirstOrThrow() + ).id; + cfSid = `umt-site-${siteCf}-acct-${accountId}`; // 先占位,打开后换成会话 ID + const pending = svc.solveSiteChallenge({ siteId: siteCf, accountId, url: `${ORIGIN_CF}/torrents.php` }); + // 会话打开后脚本才知道 sessionId + await waitFor( + async () => { + const live = await svc.liveSessionFor(siteCf, accountId); + if (live) cfSid = live.id; + return !!live; + }, + 10_000, + "Cloudflare 会话打开", + 10, + ); + const out = await pending; + expect(out).toMatchObject({ status: "passed", by: "vision", clearance: "clr-e14" }); + expect(mockCf.actions.some((a) => a.action === "mouse-click" && a.body.x === 300 && a.body.y === 80)).toBe(true); + const child = await e.db + .selectFrom("agent_run") + .selectAll() + .where("spawn_reason", "=", "browser.solve_challenge") + .where("created_by", "=", "system:cloudflare") + .executeTakeFirstOrThrow(); + expect(child).toMatchObject({ trigger: "event", parent_run_id: null, status: "succeeded" }); + expect((child.budget as Any).maxToolCalls).toBe(4); + expect((child.budget as Any).maxDurationSec).toBeLessThanOrEqual(120); + expect(providerCf).toBeTruthy(); + }, 60_000); }); diff --git a/docker/browser-desktop/root/opt/camofox/plugins/umt-cookies/index.js b/docker/browser-desktop/root/opt/camofox/plugins/umt-cookies/index.js index a5ef8f9d..3f235804 100644 --- a/docker/browser-desktop/root/opt/camofox/plugins/umt-cookies/index.js +++ b/docker/browser-desktop/root/opt/camofox/plugins/umt-cookies/index.js @@ -10,6 +10,9 @@ * GET /umt/identity → 固定浏览器身份的公开部分:UA、OS、版本、生成时间(BR-4) * PUT /umt/sessions/:userId/proxy { proxy: {server, username?, password?} | null } * → 该 profile 的出口:站点开代理 = 与 HTTP 同一代理;null = 直连 + * POST /umt/tabs/:tabId/mouse-click { userId, x, y } → 按视口坐标发真实鼠标事件(page.mouse.click,CF-2): + * 跨域 iframe 里的 Cloudflare Turnstile 复选框也点得到; + * 标签页不在回 422(与"插件没有这个端点"的 404 区分) * Cookie 值只在请求/响应体里,不写日志(只记个数);代理只记主机,不记账号口令。 * * BR-4 固定浏览器身份(docs/runbook/browser.md §11):`browser:launching` 钩子里,首次用 camoufox-js 生成一份 os=windows @@ -168,6 +171,16 @@ async function windowsFontconfig(installDir, log) { } } +/** 上游会话 → 标签页状态(与 server.js findTab 相同的遍历:tabGroups: Map>) */ +function findTab(session, tabId) { + if (!session?.tabGroups) return null; + for (const group of session.tabGroups.values()) { + const t = group.get(tabId); + if (t?.page) return t; + } + return null; +} + const FIELDS = ["name", "value", "domain", "path", "expires", "httpOnly", "secure", "sameSite"]; function parseUrls(raw) { @@ -364,7 +377,30 @@ export async function register(app, ctx) { app.get("/umt/instance", ctx.auth(), (_req, res) => res.json(INSTANCE)); - log("info", "umt-cookies plugin: registered cookies / instance / identity / egress endpoints", { + // ── CF-2 真实坐标点击 ── + app.post("/umt/tabs/:tabId/mouse-click", ctx.auth(), express.json({ limit: "4kb" }), async (req, res) => { + try { + const userId = ctx.normalizeUserId(String(req.body?.userId ?? "")); + const x = Number(req.body?.x); + const y = Number(req.body?.y); + if (!userId) return res.status(400).json({ error: "userId is required" }); + if (!Number.isFinite(x) || !Number.isFinite(y) || x < 0 || y < 0 || x > 20000 || y > 20000) + return res.status(400).json({ error: "x / y must be viewport CSS pixels" }); + const tab = findTab(ctx.sessions.get(userId), req.params.tabId); + if (!tab) return res.status(422).json({ error: "tab not found" }); + const session = ctx.sessions.get(userId); + session.lastAccess = Date.now(); + tab.toolCalls = (tab.toolCalls ?? 0) + 1; + await ctx.withUserLimit(userId, () => tab.page.mouse.click(Math.round(x), Math.round(y), { delay: 60 })); + log("info", "umt mouse click", { reqId: req.reqId, userId, tabId: req.params.tabId }); + res.json({ ok: true, tabId: req.params.tabId, url: tab.page.url() }); + } catch (err) { + log("error", "umt mouse click failed", { reqId: req.reqId, error: err.message }); + res.status(500).json({ error: ctx.safeError(err) }); + } + }); + + log("info", "umt-cookies plugin: registered cookies / instance / identity / egress / mouse-click endpoints", { identityMode, proxyMode, }); diff --git a/packages/adapters/src/browser/camofox-real.test.ts b/packages/adapters/src/browser/camofox-real.test.ts index 29147700..9f3601f5 100644 --- a/packages/adapters/src/browser/camofox-real.test.ts +++ b/packages/adapters/src/browser/camofox-real.test.ts @@ -27,6 +27,8 @@ function realUpstream(handler: (c: Call) => Reply | undefined) { calls.push(call); const r = handler(call) ?? + // 真实上游(不带 umt 插件)没有 /umt/* 路由 + (call.path.startsWith("/umt/") ? { status: 404, json: { error: "Not found" } } : undefined) ?? (call.path === "/tabs" && call.method === "POST" ? { json: { tabId: "t-1", url: `${ORIGIN}/login.php`, httpStatus: 200, navigationOk: true } } : { json: { ok: true } }); diff --git a/packages/adapters/src/browser/camofox/client.ts b/packages/adapters/src/browser/camofox/client.ts index dcc0e7e4..bb6ab67c 100644 --- a/packages/adapters/src/browser/camofox/client.ts +++ b/packages/adapters/src/browser/camofox/client.ts @@ -43,6 +43,7 @@ import { type NavigationResult, type OpenSessionRequest, type OpenSessionResult, + type PageContent, type PageMetrics, profileKeyFor, type ReopenSessionRequest, @@ -143,6 +144,14 @@ const PAGE_METRICS_EXPR = `/* umt-metrics */ (() => ({ w: innerWidth, h: innerHe pw: !!document.querySelector("input[type=password]"), text: String((document.body && document.body.innerText) || "").slice(0, 4000) }))()`; +/** 页面内容表达式(mock 按 umt-page 标记识别) */ +function pageContentExpr(maxChars: number): string { + return `/* umt-page */ (() => ({ url: location.href, title: document.title, + html: document.documentElement ? document.documentElement.outerHTML.slice(0, ${maxChars}) : "", + text: String((document.body && document.body.innerText) || "").slice(0, 20000), + ua: navigator.userAgent }))()`; +} + const LOGIN_PATH = /(?:^|\/)(?:login|takelogin|signin|sign_in|auth)(?:\.php)?(?:\/|$|\?)/i; export class CamofoxBrowserProvider implements BrowserProvider { @@ -159,6 +168,8 @@ export class CamofoxBrowserProvider implements BrowserProvider { private cookieApi: "plugin" | "upstream" | "none" | null = null; /** 实例标识接口:plugin = umt 插件 GET /umt/instance;health = 只能从 /health 里找;null = 未探测 */ private instanceApi: "plugin" | "health" | null = null; + /** 真实鼠标坐标点击(CF-2):plugin = umt 插件 `POST /umt/tabs/{id}/mouse-click`;none = 插件没有,退回 elementFromPoint;null = 未探测 */ + private mouseApi: "plugin" | "none" | null = null; constructor(private readonly cfg: CamofoxConfig) { this.base = cfg.baseUrl.replace(/\/+$/, ""); @@ -419,7 +430,10 @@ export class CamofoxBrowserProvider implements BrowserProvider { * 该 profile 的出口(BR-4 出口 IP 按站点统一):umt 插件 `PUT /umt/sessions/{userId}/proxy`。 * proxy = null 直连;活动上下文出口不同时插件会先关掉它(落盘登录状态)。插件不在报 not_implemented。 */ - async setEgress(profileKey: string, proxy: BrowserEgressProxy | null): Promise<{ restarted: boolean }> { + async setEgress( + profileKey: string, + proxy: BrowserEgressProxy | null, + ): Promise<{ restarted: boolean; mode?: string }> { try { const body = await this.json( await this.request("PUT", `/umt/sessions/${encodeURIComponent(profileKey)}/proxy`, { @@ -428,7 +442,8 @@ export class CamofoxBrowserProvider implements BrowserProvider { }), "/umt/sessions/{userId}/proxy", ); - return { restarted: body.restarted === true }; + const mode = str(body.mode); + return { restarted: body.restarted === true, ...(mode ? { mode } : {}) }; } catch (e) { if (e instanceof BrowserError && e.httpStatus === 404) throw new BrowserError("not_implemented", "浏览器服务不支持按站点设出口(镜像未带新版 umt-cookies 插件)"); @@ -1047,6 +1062,22 @@ export class CamofoxBrowserProvider implements BrowserProvider { await this.withProfile(s, async () => { this.assertActor(s, ref.as); const tab = this.tabOf(s, ref); + // 坐标点击先走 umt 插件的真实鼠标事件(page.mouse.click):跨域 iframe 里的控件(Cloudflare Turnstile)与画布也点得到(CF-2) + if ( + action === "click" && + typeof payload.x === "number" && + typeof payload.y === "number" && + this.mouseApi !== "none" + ) { + const done = await this.mouseClick(s, tab, payload.x, payload.y); + if (done) { + this.touch(s, done.url ?? s.rawUrl); + if (done.url) tab.url = s.session.currentUrl; + s.currentTabId = tab.upstreamTabId; + after?.(s); + return; + } + } // 上游 click/type 只认 ref/selector(1.17.0 实测不接受 {x, y}):坐标先经 evaluate 标记该点元素,再按选择器操作, // 这样仍走上游完整的鼠标事件序列与下载守卫,而不是页面内合成 click if (typeof payload.x === "number" && typeof payload.y === "number") { @@ -1068,6 +1099,73 @@ export class CamofoxBrowserProvider implements BrowserProvider { }); } + /** + * 插件的真实鼠标点击:`POST /umt/tabs/{id}/mouse-click {userId, x, y}`(视口 CSS 像素)。插件没有这个端点(404)时记下, + * 返回 null 让调用方退回 elementFromPoint。 + */ + private async mouseClick( + s: SessionState, + tab: BrowserTab, + x: number, + y: number, + ): Promise<{ url: string | undefined } | null> { + const path = `/umt/tabs/${encodeURIComponent(tab.upstreamTabId)}/mouse-click`; + try { + const res = await this.request("POST", path, { + body: { userId: s.session.profileKey, x, y }, + timeoutMs: s.budget.timeoutMs, + }); + const out = (await res.json().catch(() => ({}))) as Json; + this.mouseApi = "plugin"; + return { url: str(out.url) }; + } catch (e) { + // 404 = 插件没有这个端点(旧镜像);插件对"标签页不在"回 422,按标签页已不存在处理(BR-3 自愈) + if (e instanceof BrowserError && e.httpStatus === 404) { + this.mouseApi = "none"; + return null; + } + if (e instanceof BrowserError && e.httpStatus === 422) + throw new BrowserError("not_found", "标签页不存在或已关闭", { httpStatus: 404 }); + throw e; + } + } + + /** + * 当前页面内容(CF-2):地址(已脱敏)、标题、outerHTML、正文样本、navigator.userAgent。只读,不要求自动化控制权; + * 上游不支持 evaluate 时返回 null。HTML 只给服务端解析器用,不进模型与日志。 + */ + async pageContent(ref: TabRef, opts: { maxHtmlBytes?: number } = {}): Promise { + const s = this.state(ref.sessionId); + const max = Math.max(1024, Math.min(opts.maxHtmlBytes ?? 8 * 1024 * 1024, 16 * 1024 * 1024)); + return this.withProfile(s, async () => { + const tab = this.tabOf(s, ref); + const path = `/tabs/${encodeURIComponent(tab.upstreamTabId)}/evaluate`; + let out: Json; + try { + const res = await this.request("POST", path, { + body: { userId: s.session.profileKey, expression: pageContentExpr(max) }, + timeoutMs: s.budget.timeoutMs, + }); + out = await this.json(res, path); + } catch (e) { + if (e instanceof BrowserError && e.kind === "not_found") throw e; + return null; + } + const r = (out.result && typeof out.result === "object" ? out.result : null) as Json | null; + if (!r || typeof r.html !== "string") return null; + const rawUrl = str(r.url) ?? s.rawUrl; + this.touch(s, rawUrl ?? s.rawUrl); + tab.url = s.session.currentUrl; + return { + url: rawUrl ? redactBrowserUrl(rawUrl) : null, + title: str(r.title) ?? "", + html: r.html, + text: str(r.text) ?? "", + userAgent: str(r.ua) ?? null, + }; + }); + } + /** 视口坐标 → 唯一属性选择器(POST /tabs/{id}/evaluate + elementFromPoint);坐标处无元素报 bad_request */ private async pointSelector(s: SessionState, tab: BrowserTab, x: number, y: number): Promise { const nonce = this.newId() diff --git a/packages/adapters/src/browser/provider.ts b/packages/adapters/src/browser/provider.ts index 4734b743..f7c42448 100644 --- a/packages/adapters/src/browser/provider.ts +++ b/packages/adapters/src/browser/provider.ts @@ -68,6 +68,11 @@ export interface BrowserProvider { * 只读,不要求自动化控制权;上游不支持 evaluate 时返回 null。 */ pageMetrics(ref: TabRef): Promise; + /** + * 当前页面内容(CF-2 挑战识别与浏览器代抓):地址(已脱敏)、标题、outerHTML、正文样本、navigator.userAgent。 + * 只读,不要求自动化控制权;上游不支持时返回 null。HTML 只给服务端解析器,不进模型与日志。 + */ + pageContent?(ref: TabRef, opts?: { maxHtmlBytes?: number }): Promise; // ── BR-2 凭据统一 ── /** @@ -108,7 +113,7 @@ export interface BrowserProvider { * 该 profile 的出口:proxy = 站点开代理时与 HTTP 同一代理,null = 直连。上游没有接口时报 BrowserError("not_implemented")。 * restarted = 活动上下文出口不同、已被关闭(下次操作按新出口重建)。 */ - setEgress?(profileKey: string, proxy: BrowserEgressProxy | null): Promise<{ restarted: boolean }>; + setEgress?(profileKey: string, proxy: BrowserEgressProxy | null): Promise<{ restarted: boolean; mode?: string }>; } export interface ReopenSessionRequest { @@ -124,6 +129,15 @@ export interface ReopenSessionResult { cookies?: { status: "injected" | "unsupported" | "failed"; count: number }; } +export interface PageContent { + url: string | null; + title: string; + html: string; + /** document.body.innerText 前 2 万字 */ + text: string; + userAgent: string | null; +} + export interface PageMetrics { viewportWidth: number; viewportHeight: number; diff --git a/packages/adapters/src/browser/umt-plugin.test.ts b/packages/adapters/src/browser/umt-plugin.test.ts index d63d88af..37baaba1 100644 --- a/packages/adapters/src/browser/umt-plugin.test.ts +++ b/packages/adapters/src/browser/umt-plugin.test.ts @@ -163,12 +163,67 @@ describe("camofox 适配器:身份与出口", () => { mock = await createMockCamofox({ accessKey: "k" }); const p = provider(mock); expect(await p.identity!()).toMatchObject({ userAgent: MOCK_CAMOFOX_IDENTITY.userAgent, os: "windows" }); - expect(await p.setEgress!("site-a", { server: "http://127.0.0.1:7891" })).toEqual({ restarted: false }); - expect(await p.setEgress!("site-b", null)).toEqual({ restarted: false }); + expect(await p.setEgress!("site-a", { server: "http://127.0.0.1:7891" })).toEqual({ + restarted: false, + mode: "site", + }); + expect(await p.setEgress!("site-b", null)).toEqual({ restarted: false, mode: "site" }); expect(mock.egress.get("site-a")).toEqual({ server: "http://127.0.0.1:7891" }); expect(mock.egress.get("site-b")).toBeNull(); }); + const budget = { domainAllowlist: ["audiences.example"], timeoutMs: 5_000, maxRetries: 0, maxBytes: 1_000_000 }; + const openCf = async (m: MockCamofox) => { + const p = provider(m); + const { session } = await p.openSession({ + providerId: "11111111-1111-4111-8111-111111111111", + siteId: "s", + accountId: "a", + profileKey: "umt-site-s-acct-a", + initialUrl: "https://audiences.example/torrents.php", + budget, + }); + return { p, ref: { sessionId: session.id } }; + }; + const cfSite = (mode: "auto" | "interactive" | "captcha") => ({ + origin: "https://audiences.example", + username: "u", + password: "p", + protectedPaths: /^$/, + cloudflare: { mode, autoPolls: 2, clearance: "clr-1" }, + }); + + it("CF-2 pageContent():地址 / 标题 / HTML / UA;挑战页分类由调用方做", async () => { + mock = await createMockCamofox({ accessKey: "k", site: cfSite("auto") }); + const { p, ref } = await openCf(mock); + const c1 = await p.pageContent!(ref); + expect(c1).toMatchObject({ title: "Just a moment...", userAgent: MOCK_CAMOFOX_IDENTITY.userAgent }); + expect(c1?.html).toContain("_cf_chl_opt"); + // 第 2 次读页面时 mock 自动放行,并写 cf_clearance + const c2 = await p.pageContent!(ref); + expect(c2?.title).not.toBe("Just a moment..."); + const cookies = await p.getCookies!("umt-site-s-acct-a", ["https://audiences.example/"]); + expect(cookies.find((c) => c.name === "cf_clearance")?.value).toBe("clr-1"); + }); + + it("CF-2 坐标点击走插件真实鼠标事件(跨域 iframe 的 Turnstile 也点得到),不再 elementFromPoint", async () => { + mock = await createMockCamofox({ accessKey: "k", site: cfSite("interactive") }); + const { p, ref } = await openCf(mock); + // 第 2 行(y = 20 + 40 + 10)是复选框 + await p.click(ref, { x: 100, y: 70 }); + expect(mock.actions.some((a) => a.action === "mouse-click" && a.body.x === 100 && a.body.y === 70)).toBe(true); + expect(mock.actions.some((a) => a.action === "evaluate")).toBe(false); + expect(mock.cfPassed.has("umt-site-s-acct-a")).toBe(true); + }); + + it("CF-2 插件没有 mouse-click(旧镜像)→ 退回 elementFromPoint + 选择器点击", async () => { + mock = await createMockCamofox({ accessKey: "k", cookieApi: "none", site: cfSite("interactive") }); + const { p, ref } = await openCf(mock); + await p.click(ref, { x: 100, y: 70 }); + expect(mock.actions.some((a) => a.action === "evaluate")).toBe(true); + expect(mock.actions.some((a) => a.action === "click")).toBe(true); + }); + it("旧插件(没有接口):identity() 为 null,setEgress() 报 not_implemented", async () => { mock = await createMockCamofox({ accessKey: "k", cookieApi: "none" }); const p = provider(mock); diff --git a/packages/testkit/src/browser/mock-camofox.ts b/packages/testkit/src/browser/mock-camofox.ts index ccc99619..4c437352 100644 --- a/packages/testkit/src/browser/mock-camofox.ts +++ b/packages/testkit/src/browser/mock-camofox.ts @@ -39,6 +39,15 @@ export interface MockCamofoxSite { * 表单登录成功后把 name=fresh 写进该 profile 的 Cookie(读回时可见)。 */ sessionCookie?: { name: string; valid: string[]; fresh: string }; + /** + * Cloudflare 挑战(CF-2):没通过的 profile 打开站点任何页面都是挑战页(标题 Just a moment...)。 + * auto = 读页面内容(umt-page)autoPolls 次后自己通过;interactive = 第 1 行是「确认您是真人」复选框, + * 用插件真实鼠标点击(`POST /umt/tabs/{id}/mouse-click`)点中才通过;captcha = 选图验证码,自动化过不去。 + * 通过后该 profile 得到 cf_clearance(值 clearance)。 + */ + cloudflare?: { mode: "auto" | "interactive" | "captcha"; autoPolls?: number; clearance?: string }; + /** 页面 HTML(umt-page 读页面内容时返回);缺省按快照行拼一段简单 HTML */ + html?: (url: string) => string; } /** mock 的 Cookie 接口形态:plugin = umt-cookies 插件(读写);upstream = 只有上游导入端点(要求 CAMOFOX_API_KEY);none = 都没有 */ @@ -128,6 +137,10 @@ export type MockCamofox = MockHttp & { readonly instanceId: string; /** 各 profile 的出口(`PUT /umt/sessions/:userId/proxy`,BR-4):null = 直连 */ egress: Map; + /** 已通过 Cloudflare 挑战的 profile(CF-2) */ + cfPassed: Set; + /** 各 profile 读页面内容(umt-page)的次数 */ + contentReads: Map; }; /** mock 的固定浏览器身份(`GET /umt/identity`,BR-4) */ @@ -353,6 +366,23 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise< /** 重启前存在过的 tabId:之后访问返回 410 */ const restartedTabs = new Set(); const site = opts.site; + const cfPassed = new Set(); + const contentReads = new Map(); + const passCf = (userId: string) => { + if (!site?.cloudflare || cfPassed.has(userId)) return; + cfPassed.add(userId); + putCookie(userId, { + name: "cf_clearance", + value: site.cloudflare.clearance ?? "cf-clearance-mock", + domain: `.${new URL(site.origin).hostname}`, + path: "/", + expires: Math.floor(Date.now() / 1000) + 365 * 86400, + httpOnly: true, + secure: true, + }); + }; + const underCf = (t: MockCamofoxTab) => + !!site?.cloudflare && !cfPassed.has(t.userId) && t.url.startsWith(new URL(site.origin).origin); const putCookie = (userId: string, c: MockCamofoxCookie) => { const jar = (cookieJars.get(userId) ?? []).filter( (x) => !(x.name === c.name && x.domain === c.domain && x.path === c.path), @@ -454,6 +484,14 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise< const pageLines = (t: MockCamofoxTab): Line[] => { if (!site) return []; + if (underCf(t)) { + const mode = site.cloudflare!.mode; + const lines: Line[] = [{ ref: "e1", role: "heading", text: "Just a moment..." }]; + if (mode === "interactive") + lines.push({ ref: "e2", role: "checkbox", text: "确认您是真人", onClick: () => undefined }); + if (mode === "captcha") lines.push({ ref: "e2", role: "img", text: "请点击图中所有的红绿灯" }); + return lines; + } const u = new URL(t.url); const passkey = site.passkey ?? CAMOFOX_FIXTURE_PASSKEY; const releases = site.releases ?? [{ id: "42", title: "Oppenheimer 2023 2160p UHD BluRay" }]; @@ -749,6 +787,36 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise< }, }, }; + // 页面内容(CF-2):地址 / 标题 / HTML / 正文 / UA + if (expr.includes("umt-page")) { + const n = (contentReads.get(t.userId) ?? 0) + 1; + contentReads.set(t.userId, n); + if (underCf(t) && site?.cloudflare?.mode === "auto" && n >= (site.cloudflare.autoPolls ?? 2)) + passCf(t.userId); + const lines = pageLines(t); + const cf = underCf(t); + const mode = site?.cloudflare?.mode; + const title = cf ? "Just a moment..." : (lines[0]?.text ?? ""); + const html = cf + ? `Just a moment...
${ + mode === "captcha" ? '
' : "" + }` + : site?.html + ? site.html(t.url) + : `${title}${lines.map((l) => `

${l.text}

`).join("")}`; + return { + json: { + ok: true, + result: { + url: t.url, + title, + html, + text: lines.map((l) => l.text).join("\n"), + ua: MOCK_CAMOFOX_IDENTITY.userAgent, + }, + }, + }; + } const pt = /elementFromPoint\((\d+), (\d+)\)/.exec(expr); const nonce = /setAttribute\("data-umt-pt", "([^"]+)"\)/.exec(expr)?.[1]; if (!pt || !nonce) return { json: { ok: true, result: null } }; @@ -811,6 +879,27 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise< return { json: { downloads: list } }; }, ], + [ + // 插件的真实鼠标点击(CF-2):坐标按行找元素并点击;挑战页的复选框点中即通过 + /^\/umt\/tabs\/([^/]+)\/mouse-click$/, + (m, q, req, b) => { + if (cookieApi !== "plugin" || req.method !== "POST") return err(404, "no route"); + const t = tabFor(m[1]!, b.userId as string | undefined); + if (!t) return err(422, "tab not found"); + actions.push({ action: "mouse-click", tabId: t.tabId, body: { ...b } }); + void q; + const y = Number(b.y); + const line = pageLines(t)[Math.floor((y - MOCK_ROW_Y0) / MOCK_ROW_H)]; + if (line) { + if (underCf(t) && site?.cloudflare?.mode === "interactive" && line.role === "checkbox") passCf(t.userId); + else { + if (line.role.startsWith("textbox")) t.focus = line.ref; + line.onClick?.(); + } + } + return { json: { ok: true, tabId: t.tabId, url: t.url } }; + }, + ], [/^\/umt\/instance$/, () => (cookieApi === "plugin" ? { json: { instanceId } } : err(404, "no route"))], [ /^\/umt\/identity$/, @@ -891,6 +980,8 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise< loggedIn, cookieJars, egress, + cfPassed, + contentReads, expireLogin(userId?: string) { if (userId) loggedIn.delete(userId); else loggedIn.clear(); @@ -905,6 +996,7 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise< if (o.loseProfiles) { loggedIn.clear(); cookieJars.clear(); + cfPassed.clear(); } }, }); From 16083942c0a26b701a0d7a62cd518950b5ef09fb Mon Sep 17 00:00:00 2001 From: root Date: Wed, 30 Sep 2026 21:26:54 +0800 Subject: [PATCH 3/5] =?UTF-8?q?feat(sites):=20=E5=87=BA=E5=8F=A3=E7=BB=9F?= =?UTF-8?q?=E4=B8=80=E4=B8=8E=20Cloudflare=20=E8=BF=87=E6=8C=91=E6=88=98?= =?UTF-8?q?=E7=BA=A7=E8=81=94(CF-2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 出口:站点请求不再隐式继承 HTTP(S)_PROXY,没开代理明确直连(创建客户端时临时拿掉代理环境变量,Deno 2.9 语义见 egress.ts), 开了走站点代理;TMDB / Brave / 图片改为显式按环境变量(行为不变) - CloudflareGate:浏览器代理期直接代抓 → HTTP(可复用时带 cf_clearance)→ 挑战 → 浏览器过挑战(同站 30 分钟一次、单飞) → cf_clearance 仅在 UA 跟随浏览器且两端同出口时给 HTTP,重试 1 次记 cfReuse → 仍被挑战则浏览器代抓并进入 6 小时代理期 - 浏览器:solveSiteChallenge(自动放行 ≤15s / 系统发起视觉子流程最多 2 次 / 真验证码转人工待处理)、fetchSiteHtml(profile 锁 + 站点令牌)、 交还后读回 cf_clearance 继续;site.config.cloudflare 状态契约 Co-Authored-By: Claude Opus 5.5 --- apps/server/src/egress.ts | 136 ++++++ apps/server/src/modules/agent/challenge.ts | 38 +- .../__tests__/browser-cloudflare.test.ts | 228 +++++++++ .../src/modules/browser/__tests__/harness.ts | 23 +- apps/server/src/modules/browser/service.ts | 325 ++++++++++++- apps/server/src/modules/cache/index.ts | 4 +- .../modules/sources-identity/routes/enrich.ts | 6 +- .../sources-identity/routes/workbench.ts | 3 +- .../src/modules/sources-identity/runtime.ts | 71 +-- .../sources-identity/service/cloudflare.ts | 431 ++++++++++++++++++ apps/server/src/modules/websearch/service.ts | 6 +- .../test/sources-identity/cloudflare.test.ts | 315 +++++++++++++ .../test/sources-identity/egress.test.ts | 133 ++++++ packages/contracts/src/sources.ts | 35 ++ 14 files changed, 1711 insertions(+), 43 deletions(-) create mode 100644 apps/server/src/egress.ts create mode 100644 apps/server/src/modules/browser/__tests__/browser-cloudflare.test.ts create mode 100644 apps/server/src/modules/sources-identity/service/cloudflare.ts create mode 100644 apps/server/test/sources-identity/cloudflare.test.ts create mode 100644 apps/server/test/sources-identity/egress.test.ts diff --git a/apps/server/src/egress.ts b/apps/server/src/egress.ts new file mode 100644 index 00000000..0318a063 --- /dev/null +++ b/apps/server/src/egress.ts @@ -0,0 +1,136 @@ +// CF-2 出口统一:每条对外 HTTP 请求走哪条出口由代码显式决定,不再隐式继承容器的 HTTP(S)_PROXY。 +// +// Deno 2.9 的语义(2026-09-30 本机实测,见 docs/runbook/sites.md「出口」): +// - `fetch` 缺省客户端与 `Deno.createHttpClient({})` 都会读 HTTP_PROXY / HTTPS_PROXY / ALL_PROXY(大小写都认)与 NO_PROXY; +// 读取发生在**客户端创建时**(缺省客户端在进程第一次 fetch 时创建); +// - `createHttpClient` 没有"不走代理"的选项,`NO_PROXY=*` 也不认;显式 `proxy` 会排在环境变量代理前面, +// 但 NO_PROXY 命中的主机连显式代理也绕过; +// - 所以直连客户端的做法是:创建那一刻把这几个环境变量临时拿掉(同步调用,同一个 isolate 内不会被别的代码插进来),建完立即还原。 +// 代理客户端同样在拿掉环境变量后创建,保证只走给定的代理。 +// +// 出口表: +// - 站点请求(适配器、RSS、登录、用户数据):站点开"使用代理"且设了代理地址 → 该代理;否则直连(siteClient); +// - TMDB、Brave、图片缓存等外网请求:照旧用环境变量里的代理(行为不变),但改成显式读 env 建客户端(externalFetch)。 + +const PROXY_ENV = ["HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy"] as const; +const NO_PROXY_ENV = ["NO_PROXY", "no_proxy"] as const; + +/** 进程启动时的代理环境变量快照(外网请求显式使用;之后的临时改动不影响) */ +const ENV_SNAPSHOT: Record = (() => { + const out: Record = {}; + try { + for (const k of [...PROXY_ENV, ...NO_PROXY_ENV]) out[k] = Deno.env.get(k); + } catch { + // 没有 env 权限:当作没有代理 + } + return out; +})(); + +/** 在"没有代理环境变量"的状态下同步执行 fn(建 HttpClient 用),结束立即还原 */ +function withoutProxyEnv(fn: () => T): T { + const saved = new Map(); + try { + for (const k of [...PROXY_ENV, ...NO_PROXY_ENV]) { + const v = Deno.env.get(k); + if (v !== undefined) { + saved.set(k, v); + Deno.env.delete(k); + } + } + } catch { + // 没有 env 权限时也没有可读的代理变量,直接建 + } + try { + return fn(); + } finally { + for (const [k, v] of saved) Deno.env.set(k, v); + } +} + +let direct: Deno.HttpClient | null = null; +const proxied = new Map(); + +/** 明确直连的客户端(不读任何代理环境变量) */ +export function directHttpClient(): Deno.HttpClient { + direct ??= withoutProxyEnv(() => Deno.createHttpClient({})); + return direct; +} + +/** 只走给定代理的客户端(同一地址复用) */ +export function proxyHttpClient(url: string): Deno.HttpClient { + let c = proxied.get(url); + if (!c) { + c = withoutProxyEnv(() => Deno.createHttpClient({ proxy: { url } })); + proxied.set(url, c); + } + return c; +} + +/** 用指定客户端发请求的 fetch */ +export function fetchVia(client: Deno.HttpClient): typeof globalThis.fetch { + return ((input: RequestInfo | URL, init?: RequestInit) => + globalThis.fetch(input, { ...init, client } as RequestInit)) as typeof globalThis.fetch; +} + +/** 站点请求的出口:开代理且有代理地址 → proxy,否则 direct */ +export interface SiteEgress { + mode: "direct" | "proxy"; + proxyUrl: string | null; +} + +export function siteEgress(proxy: boolean, proxyUrl: string | null | undefined): SiteEgress { + return proxy && proxyUrl ? { mode: "proxy", proxyUrl } : { mode: "direct", proxyUrl: null }; +} + +/** 出口的比较键(不含代理口令):direct / proxy:<主机:端口> */ +export function egressKey(e: SiteEgress): string { + if (e.mode === "direct" || !e.proxyUrl) return "direct"; + try { + const u = new URL(e.proxyUrl); + return `proxy:${u.protocol}//${u.host}`; + } catch { + return "proxy:?"; + } +} + +export function siteFetchFor(e: SiteEgress): typeof globalThis.fetch { + return fetchVia(e.mode === "proxy" && e.proxyUrl ? proxyHttpClient(e.proxyUrl) : directHttpClient()); +} + +// ---------- 外网请求(TMDB / Brave / 图片):显式按环境变量 ---------- + +/** NO_PROXY 匹配(与常见实现一致:逗号分隔;`.a.com` / `a.com` 匹配自身与子域;`*` 匹配全部;IP 精确匹配) */ +export function noProxyMatches(host: string, noProxy: string | undefined): boolean { + if (!noProxy) return false; + const h = host.toLowerCase().replace(/^\[|\]$/g, ""); + for (const raw of noProxy.split(",")) { + const p = raw.trim().toLowerCase().replace(/:\d+$/, ""); + if (!p) continue; + if (p === "*") return true; + const d = p.replace(/^\*?\./, ""); + if (h === d || h.endsWith(`.${d}`)) return true; + } + return false; +} + +/** 外网请求按环境变量应走的代理(进程启动时的快照);没有 / NO_PROXY 命中为 null */ +export function envProxyFor(target: URL, env: Record = ENV_SNAPSHOT): string | null { + const get = (k: string) => env[k] ?? env[k.toLowerCase()]; + if (noProxyMatches(target.hostname, get("NO_PROXY"))) return null; + const byScheme = target.protocol === "https:" ? get("HTTPS_PROXY") : get("HTTP_PROXY"); + const p = byScheme || get("ALL_PROXY"); + return p ? p : null; +} + +/** 外网请求用的 fetch:按环境变量显式选代理 / 直连(与之前隐式继承的行为一致) */ +export const externalFetch: typeof globalThis.fetch = ((input: RequestInfo | URL, init?: RequestInit) => { + let target: URL | null = null; + try { + target = new URL(input instanceof Request ? input.url : String(input)); + } catch { + // 交给 fetch 报错 + } + const proxy = target ? envProxyFor(target) : null; + const client = proxy ? proxyHttpClient(proxy) : directHttpClient(); + return globalThis.fetch(input, { ...init, client } as RequestInit); +}) as typeof globalThis.fetch; diff --git a/apps/server/src/modules/agent/challenge.ts b/apps/server/src/modules/agent/challenge.ts index 48a23081..4143e38b 100644 --- a/apps/server/src/modules/agent/challenge.ts +++ b/apps/server/src/modules/agent/challenge.ts @@ -20,7 +20,7 @@ import { } from "@umt/contracts"; import type { z } from "zod"; import { browserServiceFor } from "../browser/index.ts"; -import { BrowserServiceError } from "../browser/service.ts"; +import { BrowserServiceError, type ChallengeVision, registerChallengeVision } from "../browser/service.ts"; import { resolveRunUpstream } from "../pi/index.ts"; import type { ServerContext } from "../types.ts"; import { parseFakeScripts } from "./fake-scripts.ts"; @@ -62,7 +62,41 @@ const FALLBACK_SCRIPT: FakeScript = { const isUuid = (s: string) => /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(s); const clip = (s: string, n: number) => (s.length > n ? `${s.slice(0, n)}…` : s); +/** + * CF-2:系统发起的视觉子流程(站点 HTTP 遇 Cloudflare 复选框 / Turnstile 时由浏览器模块调用)。没有父会话: + * 发起人记 system:cloudflare,子会话 trigger = event;预算取设置(智能体单次 token / 费用上限,时长 ≤ 120 秒)。 + * 自动化总开关由浏览器模块先查过;子流程里的坐标动作照常经注册中心授权(spawn_reason 放行)。 + */ +export function systemChallengeVision(env: () => AgentEnv, registry: CapabilityRegistry): ChallengeVision { + return async (req) => { + const { ctx } = env(); + const s = await ctx.settings.all(); + const input: Input = { + sessionId: req.sessionId, + goal: req.goal, + // 成功与否浏览器模块还会按页面再核实一次;这里给子流程一个可判定的条件 + success: { textAbsent: req.textAbsent ?? "Just a moment" }, + maxSteps: Math.min(12, Math.max(1, req.maxSteps)), + budget: { + maxDurationSec: Math.min(120, s.agent.maxDurationSec), + ...(s.agent.maxTokensPerRun ? { maxTokens: Math.max(1000, s.agent.maxTokensPerRun) } : {}), + ...(s.agent.maxCostUsdPerRun ? { maxCostUsd: s.agent.maxCostUsdPerRun } : {}), + }, + }; + const hctx = { + actorId: "system:cloudflare", + taskId: `cloudflare:${req.sessionId}`, + scopeIds: [], + signal: AbortSignal.timeout(180_000), + source: "task", + } as HandlerContext; + const out = await solveChallenge(env(), registry, input, hctx); + return { status: out.status, reason: out.reason, childRunId: out.childRunId }; + }; +} + export function registerChallengeCapability(registry: CapabilityRegistry, env: () => AgentEnv): void { + registerChallengeVision(env().ctx, systemChallengeVision(env, registry)); registry.register(capabilitySpec("browser.solve_challenge"), async (input, ctx) => { try { return await solveChallenge(env(), registry, input, ctx); @@ -249,7 +283,7 @@ async function runChild( const child = await ctx.db .insertInto("agent_run") .values({ - trigger: parent ? "subagent" : "user", + trigger: parent ? "subagent" : hctx.actorId.startsWith("system:") ? "event" : "user", goal, status: "running", scope: JSON.stringify({ capabilities: [...VISION_CAPABILITIES], allowedKinds: CHILD_KINDS }), diff --git a/apps/server/src/modules/browser/__tests__/browser-cloudflare.test.ts b/apps/server/src/modules/browser/__tests__/browser-cloudflare.test.ts new file mode 100644 index 00000000..f8ac33ef --- /dev/null +++ b/apps/server/src/modules/browser/__tests__/browser-cloudflare.test.ts @@ -0,0 +1,228 @@ +import { siteCloudflareState } from "@umt/contracts"; +import { MOCK_CAMOFOX_IDENTITY, type MockCamofoxSite } from "@umt/testkit"; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; +import { browserServiceFor } from "../index.ts"; +import { type BrowserService, registerChallengeVision } from "../service.ts"; +import { type BrowserHarness, createBrowserHarness, type Loose, ORIGIN, pgAvailable } from "./harness.ts"; + +// CF-2 浏览器一侧(对 testkit mock camofox 的 Cloudflare 模式): +// ① 自动放行(轮询页面)② 复选框 / Turnstile 派视觉子流程,用插件真实鼠标点击 ③ 选图验证码 / 点不过 → 会话转等待 + 待处理事项; +// 人在原站视图里过完交还 → 读回 cf_clearance / UA 通知站点请求继续;浏览器代抓页面受 profile 锁与站点限流约束。 + +const up = await pgAvailable(); +const GOOD = "cf2-session-value"; +const CLEARANCE = "clr-from-browser"; +const LIST_HTML = + '种子
Oppenheimer
'; + +describe.skipIf(!up)("CF-2 浏览器过挑战 / 代抓", () => { + let h: BrowserHarness; + let svc: BrowserService; + const cfCfg: NonNullable = { mode: "auto", autoPolls: 2, clearance: CLEARANCE }; + const visionCalls: string[] = []; + let visionClick = true; + const url = `${ORIGIN}/torrents.php`; + + const sessions = async () => (await h.req("/browser/sessions")).body.data.items as Loose[]; + const closeAll = async () => { + for (const s of await sessions()) await h.req(`/browser/sessions/${s.id}`, { method: "DELETE" }); + }; + const profileKey = () => `umt-site-${h.siteId}-acct-${h.accountId}`; + + beforeAll(async () => { + h = await createBrowserHarness("cf2", { + sessionCookie: { name: "c_secure_pass", valid: [GOOD], fresh: GOOD }, + cloudflare: cfCfg, + html: (u) => + u.includes("torrents.php") ? LIST_HTML : "首页", + browserOptions: { challengePollMs: 5, challengeAutoWaitMs: 200, settleMs: 0 }, + }); + await sessions(); // 第一次请求时浏览器模块才为这个上下文建服务 + svc = browserServiceFor(h.ctx)!; + const ref = await h.secrets.create({ purpose: "site_cookie", plaintext: `c_secure_uid=1; c_secure_pass=${GOOD}` }); + await h.db + .updateTable("site_account") + .set({ auth_kind: "cookie", secret_ref: ref, session_secret_ref: null, status: "active" }) + .where("id", "=", h.accountId) + .execute(); + // 视觉子流程(假):点一下复选框所在的位置(第 2 行) + registerChallengeVision(h.ctx, async (req) => { + visionCalls.push(req.sessionId); + if (visionClick) + await svc.visionAct( + req.sessionId, + { kind: "automation", holder: "system:cloudflare" }, + (p, ref) => p.click(ref, { x: 100, y: 70 }), + { wantImage: false, sensitiveAllowed: false }, + ); + return { status: visionClick ? "solved" : "needs_human", reason: "", childRunId: null }; + }); + }, 120_000); + afterAll(async () => { + await h?.close(); + }); + beforeEach(async () => { + await closeAll(); + h.mock.cfPassed.clear(); + h.mock.contentReads.clear(); + h.mock.actions.length = 0; + for (const jar of h.mock.cookieJars.values()) + jar.splice(0, jar.length, ...jar.filter((c) => c.name !== "cf_clearance")); + cfCfg.mode = "auto"; + visionCalls.length = 0; + visionClick = true; + await h.db + .updateTable("site") + .set({ config: JSON.stringify({}), enabled: true }) + .where("id", "=", h.siteId) + .execute(); + await h.db + .updateTable("settings") + .set({ value: JSON.stringify({ enabled: true }) }) + .where("key", "=", "automation") + .execute(); + }); + + it("① 自动放行:轮询到挑战页消失,读回 cf_clearance + navigator.userAgent + 出口", async () => { + const r = await svc.solveSiteChallenge({ siteId: h.siteId, accountId: h.accountId, url }); + expect(r).toEqual({ + status: "passed", + by: "auto", + clearance: CLEARANCE, + userAgent: MOCK_CAMOFOX_IDENTITY.userAgent, + browserEgress: "direct", + }); + expect(visionCalls).toEqual([]); + // 会话仍归自动化,没有待处理事项 + const [s] = await sessions(); + expect(s.controller).toBe("automation"); + }); + + it("② 复选框 / Turnstile:等不过 → 视觉子流程用插件真实鼠标点击 → 通过(by vision)", async () => { + cfCfg.mode = "interactive"; + const r = await svc.solveSiteChallenge({ siteId: h.siteId, accountId: h.accountId, url }); + expect(r).toMatchObject({ status: "passed", by: "vision", clearance: CLEARANCE }); + expect(visionCalls).toHaveLength(1); + expect(h.mock.actions.some((a) => a.action === "mouse-click")).toBe(true); + }); + + it("③ 视觉子流程点不过(最多 2 次)→ 会话转等待,待处理事项「观众需要你在原站视图里过一次验证」", async () => { + cfCfg.mode = "interactive"; + visionClick = false; + const r = await svc.solveSiteChallenge({ siteId: h.siteId, accountId: h.accountId, url }); + expect(r).toMatchObject({ status: "needs_human" }); + expect(visionCalls).toHaveLength(2); + const s = (await h.req(`/browser/sessions/${(r as Loose).sessionId}`)).body.data as Loose; + expect(s.controller).toBe("waiting"); + expect(s.waitingReason).toBe("cf_challenge"); + const d = await h.db + .selectFrom("decision") + .selectAll() + .where("id", "=", (r as Loose).decisionId) + .executeTakeFirstOrThrow(); + expect(d.title).toBe("观众需要你在原站视图里过一次验证"); + expect(d.status).toBe("awaiting_user"); + expect((d.evidence as Loose).cloudflare).toMatchObject({ siteId: h.siteId }); + // 已在等人:再来一次不打扰,直接返回同一个等待 + const again = await svc.solveSiteChallenge({ siteId: h.siteId, accountId: h.accountId, url }); + expect(again).toMatchObject({ status: "needs_human", sessionId: (r as Loose).sessionId }); + expect(visionCalls).toHaveLength(2); + }); + + it("自动化总开关关闭:不派视觉子流程,直接转人工", async () => { + cfCfg.mode = "interactive"; + await h.db + .insertInto("settings") + .values({ key: "automation", value: JSON.stringify({ enabled: false }) }) + .onConflict((oc) => oc.column("key").doUpdateSet({ value: JSON.stringify({ enabled: false }) })) + .execute(); + const r = await svc.solveSiteChallenge({ siteId: h.siteId, accountId: h.accountId, url }); + expect(r).toMatchObject({ status: "needs_human" }); + expect(visionCalls).toEqual([]); + }); + + it("③ 选图 / 输码类:不点、不等,直接转人工;人过完交还 → 读回 cf_clearance,站点请求继续", async () => { + cfCfg.mode = "captcha"; + const r = (await svc.solveSiteChallenge({ siteId: h.siteId, accountId: h.accountId, url })) as Loose; + expect(r.status).toBe("needs_human"); + expect(r.reason).toContain("选图"); + expect(visionCalls).toEqual([]); + // 站点请求一侧记下"等人处理"(级联在 sources-identity 里做,这里直接写状态) + await h.db + .updateTable("site") + .set({ config: JSON.stringify({ cloudflare: { pending: "human", sessionId: r.sessionId } }) }) + .where("id", "=", h.siteId) + .execute(); + // 人接管,在桌面里过了验证(桌面操作不经 API:直接改 mock 状态),交还 + expect( + (await h.req(`/browser/sessions/${r.sessionId}/control/acquire`, { body: { reason: "过验证" } })).status, + ).toBe(200); + h.mock.cfPassed.add(profileKey()); + h.mock.cookieJars.get(profileKey())!.push({ + name: "cf_clearance", + value: "clr-human", + domain: ".audiences.example", + path: "/", + expires: Math.floor(Date.now() / 1000) + 3600, + httpOnly: true, + secure: true, + }); + expect((await h.req(`/browser/sessions/${r.sessionId}/control/release`, { body: {} })).status).toBe(200); + const site = await h.db.selectFrom("site").select("config").where("id", "=", h.siteId).executeTakeFirstOrThrow(); + expect(siteCloudflareState(site.config)).toMatchObject({ pending: null, lastSolvedBy: "human" }); + const acc = await h.db + .selectFrom("site_account") + .select("user_info") + .where("id", "=", h.accountId) + .executeTakeFirstOrThrow(); + const info = (acc.user_info as Loose).cloudflare; + expect(info).toMatchObject({ userAgent: MOCK_CAMOFOX_IDENTITY.userAgent, egress: "direct" }); + expect(h.secrets.plain.get(info.secretRef)).toBe("clr-human"); + // 待处理事项已关闭 + const d = await h.db + .selectFrom("decision") + .select("status") + .where("id", "=", r.decisionId) + .executeTakeFirstOrThrow(); + expect(d.status).toBe("resolved"); + }); + + it("浏览器代抓:过了挑战的 profile 导航取 outerHTML;同一 profile 并发串行(只开一个会话);每次导航取站点令牌", async () => { + h.mock.cfPassed.add(profileKey()); + const [a, b] = await Promise.all([ + svc.fetchSiteHtml({ siteId: h.siteId, accountId: h.accountId, url }), + svc.fetchSiteHtml({ siteId: h.siteId, accountId: h.accountId, url }), + ]); + expect(a).toMatchObject({ status: "ok", html: LIST_HTML }); + expect(b).toMatchObject({ status: "ok", html: LIST_HTML }); + expect(await sessions()).toHaveLength(1); + const navs = h.mock.calls.filter( + (c) => c.path.endsWith("/navigate") || (c.path === "/tabs" && c.method === "POST"), + ); + expect(navs.length).toBeGreaterThanOrEqual(2); + const count = await h.db + .selectFrom("site_request_count") + .select((eb) => eb.fn.sum("count").as("n")) + .where("site_id", "=", h.siteId) + .executeTakeFirst(); + expect(Number(count?.n ?? 0)).toBeGreaterThanOrEqual(2); + }); + + it("浏览器代抓:仍是挑战页 → challenge;会话在人工接管中 → 不打扰;站点停用 → 不可用", async () => { + cfCfg.mode = "captcha"; + expect(await svc.fetchSiteHtml({ siteId: h.siteId, accountId: h.accountId, url })).toMatchObject({ + status: "challenge", + }); + const [s] = await sessions(); + await h.req(`/browser/sessions/${s.id}/control/acquire`, { body: { reason: "看看" } }); + expect(await svc.fetchSiteHtml({ siteId: h.siteId, accountId: h.accountId, url })).toMatchObject({ + status: "unavailable", + }); + await h.req(`/browser/sessions/${s.id}/control/release`, { body: {} }); + await h.db.updateTable("site").set({ enabled: false }).where("id", "=", h.siteId).execute(); + expect(await svc.fetchSiteHtml({ siteId: h.siteId, accountId: h.accountId, url })).toMatchObject({ + status: "unavailable", + reason: "站点已停用", + }); + }); +}); diff --git a/apps/server/src/modules/browser/__tests__/harness.ts b/apps/server/src/modules/browser/__tests__/harness.ts index 6d265c53..44396451 100644 --- a/apps/server/src/modules/browser/__tests__/harness.ts +++ b/apps/server/src/modules/browser/__tests__/harness.ts @@ -3,7 +3,13 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import type { CapabilityRegistry, InvokeContext } from "@umt/agent"; import { type DB, type Kysely, migrateToLatest, sql } from "@umt/db"; -import { buildFakeTorrent, createMockCamofox, type FakeTorrent, type MockCamofox } from "@umt/testkit"; +import { + buildFakeTorrent, + createMockCamofox, + type FakeTorrent, + type MockCamofox, + type MockCamofoxSite, +} from "@umt/testkit"; import { Kysely as K, PostgresDialect } from "kysely"; import pg from "pg"; import { withSchemaLock } from "../../../../test/auth-settings/harness.ts"; @@ -19,6 +25,7 @@ import { createSourcesIdentityModule } from "../../sources-identity/index.ts"; import type { ServerContext } from "../../types.ts"; import { registerBrowserCapabilities } from "../capabilities.ts"; import { createBrowserModule } from "../index.ts"; +import type { BrowserModuleOptions } from "../service.ts"; // C1-E 浏览器模块测试装配:独立 PG schema + 真实迁移 + 交互式 mock camofox(观众风格站点)+ 真实能力注册中心。 @@ -61,6 +68,11 @@ export async function createBrowserHarness( sessionCookie?: { name: string; valid: string[]; fresh: string }; cookieApi?: "plugin" | "upstream" | "none"; cookieImportKey?: string; + /** CF-2:mock 站点的 Cloudflare 挑战与页面 HTML */ + cloudflare?: MockCamofoxSite["cloudflare"]; + html?: MockCamofoxSite["html"]; + /** 浏览器模块的额外选项(挑战轮询间隔等) */ + browserOptions?: Partial; } = {}, ): Promise { const torrent = buildFakeTorrent({ name: "Oppenheimer.2023.2160p.UHD.BluRay.x265-GRP" }); @@ -70,6 +82,8 @@ export async function createBrowserHarness( ...(opts.cookieImportKey ? { cookieImportKey: opts.cookieImportKey } : {}), site: { ...(opts.sessionCookie ? { sessionCookie: opts.sessionCookie } : {}), + ...(opts.cloudflare ? { cloudflare: opts.cloudflare } : {}), + ...(opts.html ? { html: opts.html } : {}), origin: ORIGIN, username: "fixture-user", password: PASSWORD, @@ -110,7 +124,12 @@ export async function createBrowserHarness( capabilities: registry, } as ServerContext; const sources = createSourcesIdentityModule({ tmdb: () => null }); - const browser = createBrowserModule({ runtime: (c) => sources.runtime(c), stagingDir, downloadPollMs: 20 }); + const browser = createBrowserModule({ + runtime: (c) => sources.runtime(c), + stagingDir, + downloadPollMs: 20, + ...opts.browserOptions, + }); registerBrowserCapabilities(registry, () => browser.service(ctx)); const app = createApp({ config: { NODE_ENV: "development", DEV_ADMIN_KEY: DEV_KEY }, diff --git a/apps/server/src/modules/browser/service.ts b/apps/server/src/modules/browser/service.ts index bc5b6ac4..ac902b3d 100644 --- a/apps/server/src/modules/browser/service.ts +++ b/apps/server/src/modules/browser/service.ts @@ -5,6 +5,7 @@ import { type BrowserProvider, type BrowserSecretResolver, CAMOFOX_VERIFIED_RELEASES, + classifyChallengePage, cookiesFromHeader, createCamofoxBrowserProvider, type DownloadSink, @@ -12,7 +13,9 @@ import { getSiteDefinition, headerFromCookies, hostAllowed, + InMemoryProfileLock, imageDimensions, + type PageContent, type PageMetrics, parseTorrent, prepareImage, @@ -50,8 +53,14 @@ import { type SiteRow, sql, } from "@umt/db"; +import { egressKey, siteEgress } from "../../egress.ts"; import type { SourcesRuntime } from "../sources-identity/runtime.ts"; import { refreshSiteAccount } from "../sources-identity/service/account.ts"; +import { + type BrowserFetchOutcome, + type ChallengeSolveOutcome, + registerChallengeBrowser, +} from "../sources-identity/service/cloudflare.ts"; import { readAccountCookie, registerBrowserCookieReader, @@ -85,6 +94,10 @@ export interface BrowserModuleOptions { evidenceDir?: string; /** 坐标动作之后等页面稳定再截图的毫秒数;缺省 400 */ settleMs?: number; + /** CF-2 等挑战页放行时读页面的间隔(毫秒);缺省 1500 */ + challengePollMs?: number; + /** CF-2 等 Cloudflare 自己放行的最长时间(毫秒);缺省 CF_AUTO_WAIT_MS(15 秒) */ + challengeAutoWaitMs?: number; /** 保活间隔(AG-4);缺省 BROWSER_KEEPALIVE_MS 配置(120 秒),0 = 关闭 */ keepaliveMs?: number; /** 测试替换 provider 实现 */ @@ -175,8 +188,39 @@ export interface RecoveryState { reconnect?: BrowserReconnectState; /** 最近一次自动化(Agent 能力)动作成功的时间;原站视图据此区分「Agent 正在操作」与「无人操作」 */ lastAutomationAt?: string; + /** CF-2:打开 / 重连时下发给浏览器的出口键(egressKey);插件没按站点设出口(旧镜像 / env 模式)为 null */ + egress?: string | null; } +/** + * CF-2 视觉子流程钩子(agent 模块注册):系统发起、没有父会话,预算取设置;只点「我是人类」复选框 / Turnstile 一步。 + * 返回子流程结论(solved 仍要由调用方按页面核实)。 + */ +export type ChallengeVision = (req: { + sessionId: string; + goal: string; + /** 通过后页面上不再出现的文字(成功条件) */ + textAbsent: string | null; + maxSteps: number; +}) => Promise<{ status: string; reason: string; childRunId: string | null }>; + +const visions = new WeakMap(); + +export function registerChallengeVision(ctx: object, fn: ChallengeVision): void { + visions.set(ctx, fn); +} + +/** 系统(Cloudflare 处理)发起的浏览器动作 */ +const CF_ACTOR: Actor = { kind: "automation", holder: "system:cloudflare" }; +/** 等 Cloudflare 自己放行的最长时间 */ +export const CF_AUTO_WAIT_MS = 15_000; +/** 视觉子流程点过之后再等多久 */ +const CF_AFTER_CLICK_WAIT_MS = 10_000; +/** 视觉子流程最多派几次 */ +const CF_VISION_ATTEMPTS = 2; +/** 视觉子流程步数上限:截图 → 点击 → 等待 */ +const CF_VISION_STEPS = 4; + /** 同一浏览器服务的实例标识多久查一次 */ const INSTANCE_CHECK_MS = 5_000; /** 自动重连失败后,操作触发的重连至少间隔多久 */ @@ -221,6 +265,11 @@ export class BrowserService { registerBrowserCookieReader(ctx, (siteId, accountId) => this.readProfileCookie(siteId, accountId)); // 站点请求 UA 跟随浏览器固定身份(BR-4):sources-identity 经钩子取 registerBrowserIdentityReader(ctx, () => this.identityUserAgent()); + // CF-2:站点 HTTP 遇 Cloudflare 挑战时由浏览器过挑战 / 代抓页面 + registerChallengeBrowser(ctx, { + solve: (req) => this.solveSiteChallenge(req), + fetchHtml: (req) => this.fetchSiteHtml(req), + }); } // ---------- BR-4 浏览器身份与出口 ---------- @@ -283,8 +332,8 @@ export class BrowserService { * 出口按站点统一(BR-4):站点开了代理且设置了代理地址 → 该 profile 走同一代理;否则直连。 * 旧镜像(插件没有出口接口)只记日志,浏览器照旧走容器环境变量里的代理。 */ - private async applyEgress(provider: BrowserProvider, site: SiteRow, profileKey: string): Promise { - if (!provider.setEgress) return; + private async applyEgress(provider: BrowserProvider, site: SiteRow, profileKey: string): Promise { + if (!provider.setEgress) return null; const rt = this.opts.runtime?.(this.ctx); const proxyUrl = rt ? (await rt.siteDefaults()).proxyUrl @@ -297,6 +346,8 @@ export class BrowserService { { component: "browser", siteId: site.id, egress: proxy ? "proxy" : "direct" }, "浏览器会话出口已变更，按新出口重建", ); + // 插件 env 模式:浏览器按容器环境变量出网,与站点 HTTP(明确直连 / 站点代理)不一定一致 + return r.mode && r.mode !== "site" ? null : egressKey(siteEgress(site.proxy, proxyUrl)); } catch (e) { this.ctx.logger.warn( { @@ -307,6 +358,7 @@ export class BrowserService { }, "设置浏览器会话出口失败，浏览器按容器环境变量出网", ); + return null; } } @@ -851,7 +903,7 @@ export class BrowserService { for (const m of await provider.listSessions()) if (m.profileKey === profileKey && m.controller !== "closed") provider.setController(m.id, "closed"); const prep = await this.prepareCookies(site, accountId); - await this.applyEgress(provider, site, profileKey); + const egress = await this.applyEgress(provider, site, profileKey); await this.refreshIdentity(prow, { maxAgeMs: 10 * 60_000 }).catch(() => null); await this.rateLimit(site); let res: Awaited>; @@ -904,6 +956,7 @@ export class BrowserService { needsSnapshot: true, credential, instanceId, + egress, }), }) .execute(); @@ -1141,7 +1194,11 @@ export class BrowserService { } /** 转等待处理:自动化动作被拒,生成待处理事项,等人在原站视图接管 */ - async enterWaiting(row: BrowserSessionRow, reason: string): Promise { + async enterWaiting( + row: BrowserSessionRow, + reason: string, + text?: { title: string; summary: string }, + ): Promise { const provider = this.providers.get(row.provider_id)?.provider; if (provider?.sessionInfo(row.id)) provider.setController(row.id, "waiting"); const site = await this.liveSite(row.site_id).catch(() => null); @@ -1154,8 +1211,8 @@ export class BrowserService { kind: "other", status: "awaiting_user", severity: "warning", - title: `${site?.name ?? "站点"}：原站需要人工处理`, - summary: `${REASON_TEXT[reason] ?? reason}。在站点详情的原站视图接管会话，完成后交还。`, + title: text?.title ?? `${site?.name ?? "站点"}：原站需要人工处理`, + summary: text?.summary ?? `${REASON_TEXT[reason] ?? reason}。在站点详情的原站视图接管会话，完成后交还。`, subject_type: "browser_session", subject_id: row.id, dedupe_key: `browser.waiting:${row.id}`, @@ -1327,6 +1384,14 @@ export class BrowserService { await this.persist(id, provider); // 人工在桌面 / 截图模式里登录过:交还时读回登录态写进站点账号,全站(HTTP 抓取 / RSS / 快照)生效 await this.syncCookies(id, "release"); + // CF-2:等人过 Cloudflare 验证的会话交还后,读回 cf_clearance / UA,站点请求继续 + if (opts.to === "automation") + await this.afterHumanChallenge(id).catch((e) => + this.ctx.logger.warn( + { component: "browser", sessionId: id, err: String((e as Error)?.message ?? e) }, + "交还后读回 Cloudflare 验证结果失败", + ), + ); return toLease(released); } @@ -1586,7 +1651,7 @@ export class BrowserService { if (!provider.reopenSession) throw await fail("浏览器服务不支持重新连接"); const url = restoreUrl(row.current_url, budget) ?? site.base_url; const prep = await this.prepareCookies(site, row.account_id).catch(() => null); - await this.applyEgress(provider, site, row.profile_key); + const egress = await this.applyEgress(provider, site, row.profile_key); // 浏览器重启过(可能换了镜像版本):身份重新读 await this.refreshIdentity(prow).catch(() => null); let res: Awaited>>; @@ -1606,6 +1671,7 @@ export class BrowserService { needsReconnect: false, needsSnapshot: true, instanceId: instanceId ?? rs.instanceId ?? null, + egress, reconnect: { status: "ok", trigger, at, reason: null, humanNotice: row.controller === "human" }, }); this.coordSpaces.delete(id); @@ -1894,6 +1960,248 @@ export class BrowserService { return { decisionId, controller: row.controller }; } + // ---------- CF-2 Cloudflare 挑战 ---------- + + /** 过挑战 / 代抓同一 profile 串行(本进程;跨进程的同站互斥靠 site.config.cloudflare.lastAttemptAt 条件更新) */ + private readonly cfLock = new InMemoryProfileLock(240_000); + + /** 读当前页面内容(只读,不要求自动化控制权);上游不支持为 null */ + private async readPage(id: string): Promise { + return this.run(id, CF_ACTOR, (p, ref) => (p.pageContent ? p.pageContent(ref) : Promise.resolve(null)), { + write: false, + persist: false, + background: true, + }); + } + + /** 轮询页面直到不是挑战页(或变成真验证码 / 超时);返回最后一次读到的页面与分类 */ + private async waitChallenge( + id: string, + page: PageContent | null, + ms: number, + ): Promise<{ + page: PageContent | null; + kind: ReturnType["kind"]; + rule: string | null; + }> { + let cur = page ?? (await this.readPage(id)); + let c = classifyChallengePage(cur ?? {}); + const deadline = Date.now() + ms; + while (cur && c.kind !== "none" && c.kind !== "captcha" && Date.now() < deadline) { + await new Promise((r) => setTimeout(r, this.opts.challengePollMs ?? 1_500)); + cur = await this.readPage(id); + c = classifyChallengePage(cur ?? {}); + } + return { page: cur, kind: cur ? c.kind : "none", rule: c.rule }; + } + + /** + * 该账号 profile 的自动化会话:已有且在自动化控制下就用它(导航到 url,取站点令牌);没有就打开一个(打开即导航)。 + * 人工接管中 / 等人处理中返回 null(不打扰人)。 + */ + private async cfSession( + site: SiteRow, + accountId: string, + url: string, + ): Promise<{ id: string } | { busy: BrowserSessionRow }> { + const live = await this.liveSessionFor(site.id, accountId); + if (live && live.controller !== "automation") return { busy: live }; + if (!live) return { id: await this.open({ siteId: site.id, accountId, url }) }; + await this.run(live.id, CF_ACTOR, async (p, ref, x) => { + await this.rateLimit(x.site); + await p.navigate(ref, url, x.budget); + }); + return { id: live.id }; + } + + /** 会话转"等你过验证":待处理事项写明是哪个站点、去哪里处理 */ + private async cfNeedsHuman(id: string, site: SiteRow, reason: string): Promise { + const text = { + title: `${site.name}需要你在原站视图里过一次验证`, + summary: + "站点要求人机验证（Cloudflare），系统没能自动通过。在站点详情的原站视图接管会话，完成验证后交还，站点请求会继续。", + }; + let row = await this.sessionRow(id); + if (row.controller === "automation") row = await this.enterWaiting(row, "cf_challenge", text); + else if (row.controller === "waiting") { + const rs = obj(row.recovery_state) as RecoveryState; + if (!rs.waitingDecisionId) row = await this.enterWaiting(row, "cf_challenge", text); + else { + // 视觉子流程没成功时已转等待(challenge_unsolved):事项改成 Cloudflare 的说法 + await this.ctx.db + .updateTable("decision") + .set({ title: text.title, summary: text.summary }) + .where("id", "=", rs.waitingDecisionId) + .execute(); + row = await this.patchRecovery(row, { waitingReason: "cf_challenge" }); + } + } + const decisionId = (obj(row.recovery_state) as RecoveryState).waitingDecisionId ?? null; + if (decisionId) + await this.ctx.db + .updateTable("decision") + .set({ + evidence: sql`evidence || ${JSON.stringify({ cloudflare: { siteId: site.id, reason } })}::jsonb`, + }) + .where("id", "=", decisionId) + .execute(); + this.ctx.logger.info({ component: "browser", sessionId: id, siteId: site.id, reason }, "Cloudflare 验证转人工"); + return { status: "needs_human", sessionId: id, decisionId, reason }; + } + + /** 读回 profile 里的 cf_clearance 值(只取作用于站点主机的) */ + private async cfClearance(profileKey: string, siteUrl: string): Promise { + const prow = await this.providerRow().catch(() => null); + const provider = prow ? this.providerFor(prow) : null; + if (!provider?.getCookies) return null; + const list = await provider.getCookies(profileKey, [siteUrl]).catch(() => []); + const host = new URL(siteUrl).hostname; + const hit = list.find( + (c) => + c.name === "cf_clearance" && + (host === c.domain.replace(/^\./, "") || host.endsWith(`.${c.domain.replace(/^\./, "")}`)) && + !(c.expires !== undefined && c.expires > 0 && c.expires * 1000 <= this.now().getTime()), + ); + return hit?.value ?? null; + } + + /** + * 过挑战(CF-2 级联):用该账号 profile 打开同一地址 → ① 等它自己过(≤15 秒)→ ② 复选框 / Turnstile 派视觉子流程点一下 + * (最多 2 次,系统发起,预算取设置;自动化总开关关闭时跳过)→ ③ 选图 / 输码类或仍没过:会话转等待,开待处理事项。 + * 通过后读回 cf_clearance 与 navigator.userAgent,连同浏览器出口交给调用方。 + */ + async solveSiteChallenge(req: { siteId: string; accountId: string; url: string }): Promise { + const site = await this.liveSite(req.siteId); + if (!site.enabled) return { status: "failed", reason: "站点已停用" }; + const profileKey = profileKeyFor(site.id, req.accountId); + return this.cfLock.run(profileKey, async () => { + const got = await this.cfSession(site, req.accountId, req.url); + if ("busy" in got) { + const rs = obj(got.busy.recovery_state) as RecoveryState; + if (got.busy.controller === "waiting" && rs.waitingReason === "cf_challenge") + return { + status: "needs_human", + sessionId: got.busy.id, + decisionId: rs.waitingDecisionId ?? null, + reason: "已在等你过验证", + }; + return { status: "failed", reason: "浏览器会话正由人工处理，没有打扰" }; + } + const id = got.id; + const opened = await this.sessionRow(id); + if (opened.controller !== "automation") + return { status: "failed", reason: "浏览器会话打开后在等人处理（登录已失效）" }; + // ① 等它自己过 + let w = await this.waitChallenge(id, null, this.opts.challengeAutoWaitMs ?? CF_AUTO_WAIT_MS); + if (!w.page) return { status: "failed", reason: "浏览器服务不支持读取页面内容" }; + let by: "auto" | "vision" = "auto"; + // ② 复选框 / Turnstile:视觉子流程点一下 + if (w.kind === "interactive") { + const vision = visions.get(this.ctx); + const auto = await this.ctx.settings.get("automation").catch(() => null); + if (vision && auto?.enabled && !auto.readOnlyReconcile) { + const marker = /verify you are human|确认您是真人|验证您是真人|我是人类/i.exec(w.page.text)?.[0] ?? null; + for (let i = 0; i < CF_VISION_ATTEMPTS && w.kind === "interactive"; i++) { + const r = await vision({ + sessionId: id, + goal: "这是 Cloudflare 人机验证页。只点一次「确认您是真人 / Verify you are human」复选框（Turnstile），然后等待页面跳转；不要做其它操作。如果页面要求选图片、输入验证码或别的操作，立即停止并说明需要人工。", + textAbsent: marker, + maxSteps: CF_VISION_STEPS, + }).catch((e) => ({ status: "failed", reason: String((e as Error)?.message ?? e), childRunId: null })); + this.ctx.logger.info( + { component: "browser", sessionId: id, siteId: site.id, attempt: i + 1, status: r.status }, + "Cloudflare 复选框视觉子流程结束", + ); + // 子流程没成功时会话已转等待(challenge_unsolved);下一次子流程开始时自己收回自动化控制 + w = await this.waitChallenge( + id, + null, + Math.min(CF_AFTER_CLICK_WAIT_MS, this.opts.challengeAutoWaitMs ?? CF_AFTER_CLICK_WAIT_MS), + ); + if (w.kind === "none") by = "vision"; + } + } + } + // ③ 没过 + if (w.kind !== "none") + return this.cfNeedsHuman( + id, + site, + w.kind === "captcha" ? "选图 / 输入验证码类验证，不自动处理" : `没能自动通过(${w.rule ?? w.kind})`, + ); + // 通过:子流程成功时关掉它留下的待处理;读回 cf_clearance 与 UA + const row = await this.sessionRow(id); + if (row.controller === "waiting") { + await this.resolveWaiting(row, "system:cloudflare"); + const { provider } = await this.loaded(row); + provider.setController(id, "automation"); + await this.ctx.db + .updateTable("browser_session") + .set({ + controller: "automation", + recovery_state: JSON.stringify({ + ...obj(row.recovery_state), + waitingReason: null, + waitingDecisionId: null, + }), + updated_at: this.now(), + }) + .where("id", "=", id) + .execute(); + } + const rs = obj((await this.sessionRow(id)).recovery_state) as RecoveryState; + return { + status: "passed", + by, + clearance: await this.cfClearance(profileKey, site.base_url), + userAgent: w.page?.userAgent ?? (await this.identityUserAgent().catch(() => null)), + browserEgress: rs.egress ?? null, + }; + }); + } + + /** + * 浏览器代抓一页(CF-2 回退):该账号 profile 的自动化会话导航到 url(取站点令牌,受 profile 锁约束),读 outerHTML 交给原解析器。 + * 站点停用、会话正由人工处理 / 等人处理、仍是挑战页时不返回页面。 + */ + async fetchSiteHtml(req: { siteId: string; accountId: string; url: string }): Promise { + const site = await this.liveSite(req.siteId).catch(() => null); + if (!site?.enabled) return { status: "unavailable", reason: "站点已停用" }; + const profileKey = profileKeyFor(site.id, req.accountId); + try { + return await this.cfLock.run(profileKey, async () => { + const got = await this.cfSession(site, req.accountId, req.url); + if ("busy" in got) return { status: "unavailable" as const, reason: "浏览器会话正由人工处理或在等人处理" }; + const w = await this.waitChallenge(got.id, null, this.opts.challengeAutoWaitMs ?? CF_AUTO_WAIT_MS); + if (!w.page) return { status: "unavailable" as const, reason: "浏览器服务不支持读取页面内容" }; + if (w.kind !== "none") return { status: "challenge" as const, reason: w.rule ?? w.kind }; + return { status: "ok" as const, html: w.page.html, url: w.page.url }; + }); + } catch (e) { + const err = fromBrowserError(e); + return { status: "unavailable", reason: String((err as Error)?.message ?? err).slice(0, 300) }; + } + } + + /** 人过完验证交还后:站点在等人过验证、会话是那一个 → 读回 cf_clearance / UA,通知站点请求继续 */ + private async afterHumanChallenge(id: string): Promise { + const row = await this.sessionRow(id); + if (!row.account_id) return; + const rt = this.opts.runtime?.(this.ctx); + if (!rt) return; + const st = await rt.cloudflare.state(row.site_id, true); + if (st.pending !== "human") return; + const site = await this.liveSite(row.site_id); + const page = await this.readPage(id).catch(() => null); + if (page && classifyChallengePage(page).kind !== "none") return; // 还在挑战页:继续等 + const rs = obj(row.recovery_state) as RecoveryState; + await rt.cloudflare.afterHuman(row.site_id, row.account_id, { + clearance: await this.cfClearance(row.profile_key, site.base_url), + userAgent: page?.userAgent ?? (await this.identityUserAgent().catch(() => null)), + browserEgress: rs.egress ?? null, + }); + } + // ---------- 下载 ---------- /** 暂存产物核验:解析种子、按 infohash / 站点资源 ID 匹配本地资源、会话内去重 */ @@ -1972,6 +2280,7 @@ export class BrowserService { } const REASON_TEXT: Record = { + cf_challenge: "站点要求人机验证（Cloudflare），需要你过一次", captcha: "登录页需要验证码", challenge: "页面需要人工验证", two_factor: "需要二次验证或本人设备确认", @@ -2054,7 +2363,7 @@ export function toLease(r: BrowserControlLeaseRow): BrowserControlLease { } /** 视觉子流程可以接手的等待原因(验证码 / 页面挑战 / 上次没处理成功) */ -const SOLVABLE = ["captcha", "challenge", "challenge_unsolved"]; +const SOLVABLE = ["captcha", "challenge", "challenge_unsolved", "cf_challenge"]; /** capture / visionAct 的结果(能力输出的公共部分) */ export interface VisionShot { diff --git a/apps/server/src/modules/cache/index.ts b/apps/server/src/modules/cache/index.ts index 8b1a40ad..9bf7cda5 100644 --- a/apps/server/src/modules/cache/index.ts +++ b/apps/server/src/modules/cache/index.ts @@ -10,6 +10,7 @@ import { } from "@umt/contracts"; import type { z } from "zod"; import { dataPaths } from "../../config.ts"; +import { externalFetch } from "../../egress.ts"; import { requireScope } from "../../http/auth.ts"; import { errorBody } from "../../http/errors.ts"; import { CACHE_REGIONS, sharedCache, storageSettings, tmdbCachePolicy } from "../../shared-cache.ts"; @@ -73,7 +74,8 @@ export function createCacheModule(opts: CacheModuleOptions = {}): ServerModule & if (!c) { c = new ImageCache({ dir: opts.imageDir ?? join(dataPaths(ctx.config).dataDir, "cache", "images"), - ...(opts.imageFetch ? { fetch: opts.imageFetch } : {}), + // 外网请求:显式按环境变量走代理(CF-2 出口表) + fetch: opts.imageFetch ?? externalFetch, ...(opts.imageUpstreamBase ? { upstreamBase: opts.imageUpstreamBase } : {}), }); imageCaches.set(ctx, c); diff --git a/apps/server/src/modules/sources-identity/routes/enrich.ts b/apps/server/src/modules/sources-identity/routes/enrich.ts index 8f2d292c..a3e92fd2 100644 --- a/apps/server/src/modules/sources-identity/routes/enrich.ts +++ b/apps/server/src/modules/sources-identity/routes/enrich.ts @@ -9,6 +9,7 @@ import { IdParam, MetadataEnrichOutput, } from "@umt/contracts"; +import { externalFetch } from "../../../egress.ts"; import { requireScope } from "../../../http/auth.ts"; import { ApiException, errorBody } from "../../../http/errors.ts"; import type { ModuleApp } from "../../types.ts"; @@ -122,7 +123,10 @@ export function enrichRoutes(app: ModuleApp, rt: () => SourcesRuntime): void { const srcHost = hostOf(s.data.fields.posterUrl?.sourceUrl ?? ""); if (!srcHost || !domainAllowed(srcHost, identity.enrichDomains ?? [])) return c.json(errorBody("forbidden", "海报出处不在补全白名单里"), 403); - const got = await fetchPublicImage(r.overrides.fetch ?? fetch, url, { maxBytes: POSTER_MAX_BYTES, label: "海报" }); + const got = await fetchPublicImage(r.overrides.fetch ?? externalFetch, url, { + maxBytes: POSTER_MAX_BYTES, + label: "海报", + }); if (!got.ok) return c.json(errorBody("upstream_unavailable", got.message), 502); return imageResponse(got, 86400 * 7); }); diff --git a/apps/server/src/modules/sources-identity/routes/workbench.ts b/apps/server/src/modules/sources-identity/routes/workbench.ts index 623a84d3..45561fac 100644 --- a/apps/server/src/modules/sources-identity/routes/workbench.ts +++ b/apps/server/src/modules/sources-identity/routes/workbench.ts @@ -1,5 +1,6 @@ import { Id, WorkbenchReleaseDetail, WorkbenchReleasePage, WorkbenchReleaseQuery } from "@umt/contracts"; import { z } from "zod"; +import { externalFetch } from "../../../egress.ts"; import { requireScope } from "../../../http/auth.ts"; import { errorBody } from "../../../http/errors.ts"; import type { ModuleApp } from "../../types.ts"; @@ -55,7 +56,7 @@ export function workbenchRoutes(app: ModuleApp, rt: () => SourcesRuntime) { const r = rt(); const req = await detailScreenshotRequest(r, rid, index); if (!req) return c.json(errorBody("not_found", "截图不存在或详情缓存已过期"), 404); - const got = await fetchPublicImage(r.overrides.fetch ?? fetch, req.url, { + const got = await fetchPublicImage(r.overrides.fetch ?? externalFetch, req.url, { maxBytes: SHOT_MAX_BYTES, label: "截图", ...(req.referer ? { referer: req.referer } : {}), diff --git a/apps/server/src/modules/sources-identity/runtime.ts b/apps/server/src/modules/sources-identity/runtime.ts index f4fa8c74..9bd7ad7c 100644 --- a/apps/server/src/modules/sources-identity/runtime.ts +++ b/apps/server/src/modules/sources-identity/runtime.ts @@ -15,6 +15,7 @@ import { } from "@umt/adapters"; import { DEFAULT_SETTINGS, type SitesSettings } from "@umt/contracts"; import { pgTokenBucketFor, type SiteRow, sql } from "@umt/db"; +import { egressKey, externalFetch, siteEgress, siteFetchFor } from "../../egress.ts"; import { CACHE_REGIONS, renewTrackedTmdbDetail, @@ -24,6 +25,7 @@ import { tmdbDetailRef, } from "../../shared-cache.ts"; import type { ServerContext } from "../types.ts"; +import { CloudflareGate, withClearance } from "./service/cloudflare.ts"; import { readBrowserUserAgent, resolveUserAgent, type UserAgentDefaults } from "./service/user-agent.ts"; // 模块运行时依赖:站点适配器与限流器(每站一个实例,令牌在 PG 跨进程共享)、RSS 抓取器、TMDB 客户端、时钟。 @@ -58,12 +60,15 @@ export class SourcesRuntime { private siteDefaultsCache: SitesSettings = DEFAULT_SETTINGS.sites; private siteDefaultsAt = 0; private tmdbSig: string | null = null; + /** CF-2 Cloudflare 挑战处理(站点请求经 siteFetch 进来) */ + readonly cloudflare: CloudflareGate; constructor( readonly ctx: ServerContext, readonly overrides: RuntimeOverrides = {}, ) { this.now = overrides.now ?? (() => new Date()); + this.cloudflare = new CloudflareGate(this); this.rss = createRssFetcher({ ...(overrides.fetch ? { fetch: overrides.fetch } : {}), now: this.now, @@ -101,7 +106,7 @@ export class SourcesRuntime { const { cfg } = await this.tmdbConfig(); if (!cfg) return null; let n = 0; - const base = cfg.fetch ?? globalThis.fetch; + const base = cfg.fetch ?? externalFetch; const counted: typeof fetch = (input, init) => { n++; return base(input, init); @@ -144,7 +149,8 @@ export class SourcesRuntime { ...(apiKey ? { apiKey } : {}), ...(accessToken ? { accessToken } : {}), ...(baseUrl ? { baseUrl } : {}), - ...(this.overrides.fetch ? { fetch: this.overrides.fetch } : {}), + // 外网请求:显式按环境变量走代理(CF-2 出口表;测试注入优先) + fetch: this.overrides.fetch ?? externalFetch, // 请求响应进 PG 缓存(region tmdb.request),三进程共享;TTL / 条数上限 / 空结果短 TTL 取设置(对照 MP) cache: { ttlMs: policy.ttl * 1000, @@ -168,40 +174,53 @@ export class SourcesRuntime { } /** - * 站点请求用的 fetch:测试注入优先;站点开了"使用代理"且设置了 proxyUrl 时走代理(Deno.createHttpClient)。 + * 站点请求用的 fetch:测试注入优先;站点开了"使用代理"且设置了 proxyUrl 时走代理,否则**明确直连** + * (CF-2:不再隐式继承容器的 HTTP(S)_PROXY,见 egress.ts)。 * BR-4:每条请求都带 UA(请求自己设了就不动;否则 ua 参数,缺省按"跟随浏览器"),不再发 Deno 缺省的 `Deno/x.y.z`。 * ua 可以是函数:每次请求现场取(站点适配器按当前账号取,账号切换 UA 后立即生效)。 + * CF-2:给了 site.id 的请求经 Cloudflare 处理(浏览器代理期直接走浏览器、带可复用的 cf_clearance、遇挑战过挑战 / 重试 / 浏览器代抓)。 */ siteFetch( - site: Pick, + site: Pick & Partial>, defaults: SitesSettings = this.siteDefaultsCache, ua?: string | (() => Promise), ): typeof globalThis.fetch { const base = this.transportFetch(site, defaults); - return ((input: RequestInfo | URL, init?: RequestInit) => { + const egress = this.egressKeyOf(site, defaults); + const siteId = site.id; + return (async (input: RequestInfo | URL, init?: RequestInit) => { const headers = new Headers(init?.headers ?? (input instanceof Request ? input.headers : undefined)); - if (headers.has("user-agent")) return base(input, { ...init, headers }); - const pick = typeof ua === "function" ? ua() : ua !== undefined ? Promise.resolve(ua) : this.defaultUserAgent(); - return pick.then((v) => { - headers.set("user-agent", v); - return base(input, { ...init, headers }); - }); + if (!headers.has("user-agent")) { + const pick = typeof ua === "function" ? ua() : ua !== undefined ? Promise.resolve(ua) : this.defaultUserAgent(); + headers.set("user-agent", await pick); + } + if (!siteId) return base(input, { ...init, headers }); + const url = input instanceof Request ? input.url : String(input); + const method = init?.method ?? (input instanceof Request ? input.method : "GET"); + return this.cloudflare.handle( + siteId, + { url, method, accept: headers.get("accept"), userAgent: headers.get("user-agent"), egress }, + (extra) => { + const h = new Headers(headers); + const cookie = withClearance(h.get("cookie"), extra); + if (cookie) h.set("cookie", cookie); + // 过挑战可能超过单次请求的超时:重试时原信号已中止就换一个新的超时(调用方取消照样生效在级联之外) + const signal = init?.signal?.aborted ? AbortSignal.timeout(30_000) : init?.signal; + return base(input, { ...init, headers: h, ...(signal ? { signal } : {}) }); + }, + ); }) as typeof globalThis.fetch; } - /** 不加 UA 的底层 fetch(代理 / 测试注入) */ + /** 站点 HTTP 出口键(direct / proxy:<主机>),与浏览器会话的出口同一口径 */ + egressKeyOf(site: Pick, defaults: SitesSettings = this.siteDefaultsCache): string { + return egressKey(siteEgress(site.proxy, defaults.proxyUrl)); + } + + /** 不加 UA 的底层 fetch(测试注入 / 按站点出口:代理或明确直连) */ private transportFetch(site: Pick, defaults: SitesSettings): typeof globalThis.fetch { if (this.overrides.fetch) return this.overrides.fetch; - if (!site.proxy || !defaults.proxyUrl) return globalThis.fetch; - const key = defaults.proxyUrl; - let client = this.proxyClients.get(key); - if (!client) { - client = Deno.createHttpClient({ proxy: { url: key } }); - this.proxyClients.set(key, client); - } - const c = client; - return ((input: RequestInfo | URL, init?: RequestInit) => - globalThis.fetch(input, { ...init, client: c } as RequestInit)) as typeof globalThis.fetch; + return siteFetchFor(siteEgress(site.proxy, defaults.proxyUrl)); } // ---------- BR-4 站点请求 UA ---------- @@ -253,8 +272,6 @@ export class SourcesRuntime { return resolveUserAgent(acc, d).ua; } - private readonly proxyClients = new Map(); - /** * 站点适配器:每站一个实例,列表/搜索/详情/下载共享同一个限流器。 * 站点限流参数、基址、代理或站点默认变化时重建(签名比较)。 @@ -270,10 +287,12 @@ export class SourcesRuntime { } const defaults = this.siteDefaultsCache; const proxy = site.proxy ?? false; + // Cloudflare 处理状态(config.cloudflare)经常变,不参与重建判断 + const { cloudflare: _cf, ...configSig } = (site.config ?? {}) as Record; const sig = JSON.stringify([ site.base_url, site.rate_limit, - site.config, + configSig, proxy, proxy ? defaults.proxyUrl : null, defaults.defaultRateLimit, @@ -304,7 +323,7 @@ export class SourcesRuntime { limiter, now: this.now, // 每条请求都经 siteFetch:代理 + 按账号现场取 UA(BR-4) - fetch: this.siteFetch({ proxy }, defaults, () => this.siteUserAgent(site.id)), + fetch: this.siteFetch({ id: site.id, proxy }, defaults, () => this.siteUserAgent(site.id)), ...(baseUrl ? { baseUrl } : {}), }); this.adapters.set(site.id, { sig, limiter, adapter }); diff --git a/apps/server/src/modules/sources-identity/service/cloudflare.ts b/apps/server/src/modules/sources-identity/service/cloudflare.ts new file mode 100644 index 00000000..3e540f8f --- /dev/null +++ b/apps/server/src/modules/sources-identity/service/cloudflare.ts @@ -0,0 +1,431 @@ +import { detectChallenge, mergeCookieHeaders, parseCookieHeader } from "@umt/adapters"; +import { type SiteCloudflareState, siteCloudflareState } from "@umt/contracts"; +import { type SiteAccountRow, type SiteRow, sql } from "@umt/db"; +import { egressKey, siteEgress } from "../../../egress.ts"; +import type { ServerContext } from "../../types.ts"; +import type { SourcesRuntime } from "../runtime.ts"; +import { resolveUserAgent } from "./user-agent.ts"; + +// CF-2 Cloudflare 挑战处理(docs/design/cloudflare-handling.md;运维见 docs/runbook/sites.md「Cloudflare」)。 +// +// 站点 HTTP 请求(SourcesRuntime.siteFetch)经这里: +// 0. 站点在"浏览器代理期"(config.cloudflare.browserUntil 未到)且请求是页面(GET + Accept text/html)→ 直接由浏览器代抓; +// 1. 发 HTTP;账号 UA 跟随浏览器、出口与浏览器一致、复用没失败过时带上浏览器拿到的 cf_clearance(__cf_bm 不带); +// 2. 响应是挑战(cf-mitigated: challenge,或 403 / 503 / 429 的挑战页)→ 不算凭据错误,记 lastChallengeAt; +// 3. 过挑战(同站 30 分钟最多一次、同一时刻只跑一次;只处理有登录态账号的站点):浏览器用该账号 profile 打开同一地址 → +// ① 等它自己过(≤15 秒)→ ② 复选框 / Turnstile 派视觉子流程点一下(最多 2 次)→ ③ 选图 / 输码类转人工(待处理事项); +// 4. 过了:读回 cf_clearance + navigator.userAgent;满足复用条件就存下来,HTTP 重试 1 次,结果记 cfReuse = ok / fails; +// 5. 仍被挑战:页面请求改由浏览器导航取 HTML 交给原解析器,站点记 6 小时浏览器代理期;其它请求(下载、RSS、POST)照常报挑战。 +// 不接任何打码服务。 + +/** 同站两次过挑战的最短间隔 */ +export const CF_ATTEMPT_INTERVAL_MS = 30 * 60_000; +/** HTTP 过不去时走浏览器的时长 */ +export const CF_BROWSER_MODE_MS = 6 * 3_600_000; +/** 站点状态读缓存(每条站点请求都要看是否在浏览器代理期) */ +const STATE_CACHE_MS = 5_000; + +export type ChallengeSolveOutcome = + | { + status: "passed"; + by: "auto" | "vision"; + /** 浏览器 profile 里的 cf_clearance 值;没有为 null */ + clearance: string | null; + /** 浏览器 navigator.userAgent */ + userAgent: string | null; + /** 浏览器这一侧的出口键(egressKey);按容器环境变量出网 / 不知道时 null */ + browserEgress: string | null; + } + | { status: "needs_human"; sessionId: string | null; decisionId: string | null; reason: string } + | { status: "failed"; reason: string }; + +export type BrowserFetchOutcome = + | { status: "ok"; html: string; url: string | null } + | { status: "challenge"; reason: string } + | { status: "unavailable"; reason: string }; + +/** 浏览器一侧(浏览器模块注册;sources-identity 不直接依赖浏览器模块) */ +export interface ChallengeBrowserPort { + solve(req: { siteId: string; accountId: string; url: string }): Promise; + fetchHtml(req: { siteId: string; accountId: string; url: string }): Promise; +} + +const ports = new WeakMap(); + +export function registerChallengeBrowser(ctx: object, port: ChallengeBrowserPort): void { + ports.set(ctx, port); +} + +export function challengeBrowserFor(ctx: object): ChallengeBrowserPort | null { + return ports.get(ctx) ?? null; +} + +/** user_info.cloudflare:浏览器取得的 cf_clearance(值在 secret 里)与取得时的 UA / 出口 */ +interface ClearanceInfo { + secretRef: string; + userAgent: string | null; + egress: string | null; + at: string; +} + +const obj = (v: unknown): Record => + v && typeof v === "object" && !Array.isArray(v) ? (v as Record) : {}; + +function clearanceInfo(userInfo: unknown): ClearanceInfo | null { + const c = obj(obj(userInfo).cloudflare); + return typeof c.secretRef === "string" && typeof c.at === "string" + ? { + secretRef: c.secretRef, + userAgent: typeof c.userAgent === "string" ? c.userAgent : null, + egress: typeof c.egress === "string" ? c.egress : null, + at: c.at, + } + : null; +} + +export interface ChallengeRequest { + url: string; + method: string; + /** 请求的 Accept 头 */ + accept: string | null; + /** 请求实际带的 UA */ + userAgent: string | null; + /** 该请求的 HTTP 出口键 */ + egress: string; +} + +const EMPTY: SiteCloudflareState = { + browserUntil: null, + cfReuse: null, + lastChallengeAt: null, + lastSolvedBy: null, + lastSolvedAt: null, + lastAttemptAt: null, + pending: null, + sessionId: null, + lastResult: null, + lastRule: null, + lastError: null, +}; + +/** 页面请求:浏览器导航能代替的(GET + 要 HTML) */ +export function browserFetchable(req: Pick): boolean { + return req.method.toUpperCase() === "GET" && /text\/html|application\/xhtml/i.test(req.accept ?? ""); +} + +/** cf_clearance 能不能交给 HTTP:账号 UA 跟随浏览器、UA 相同、两端出口相同、复用没失败过 */ +export function clearanceUsable(input: { + uaSource: string; + requestUa: string | null; + clearanceUa: string | null; + requestEgress: string; + clearanceEgress: string | null; + cfReuse: SiteCloudflareState["cfReuse"]; +}): boolean { + return ( + input.uaSource === "browser" && + !!input.requestUa && + input.requestUa === input.clearanceUa && + !!input.clearanceEgress && + input.clearanceEgress === input.requestEgress && + input.cfReuse !== "fails" + ); +} + +export class CloudflareGate { + private readonly cache = new Map(); + /** 进程内同站同一时刻只过一次挑战:后到的请求等同一个结果 */ + private readonly inflight = new Map>(); + + constructor(private readonly rt: SourcesRuntime) {} + + private get ctx(): ServerContext { + return this.rt.ctx; + } + + private now(): Date { + return this.rt.now(); + } + + async state(siteId: string, fresh = false): Promise { + const hit = this.cache.get(siteId); + if (!fresh && hit && Date.now() - hit.at < STATE_CACHE_MS) return hit.state; + const row = await this.ctx.db.selectFrom("site").select("config").where("id", "=", siteId).executeTakeFirst(); + const state = { ...EMPTY, ...(siteCloudflareState(row?.config) ?? {}) }; + this.cache.set(siteId, { at: Date.now(), state }); + return state; + } + + /** 合并写 site.config.cloudflare(只动这一个键) */ + async patch(siteId: string, patch: Partial): Promise { + await sql`update site set config = coalesce(config, '{}'::jsonb) || jsonb_build_object('cloudflare', + coalesce(config->'cloudflare', '{}'::jsonb) || ${JSON.stringify(patch)}::jsonb) + where id = ${siteId}`.execute(this.ctx.db); + this.cache.delete(siteId); + } + + /** + * 同站 30 分钟最多一次:条件更新 lastAttemptAt(跨进程)。抢到返回 true。 + */ + private async claimAttempt(siteId: string): Promise { + const now = this.now(); + const before = new Date(now.getTime() - CF_ATTEMPT_INTERVAL_MS); + const r = await sql<{ + id: string; + }>`update site set config = coalesce(config, '{}'::jsonb) || jsonb_build_object('cloudflare', + coalesce(config->'cloudflare', '{}'::jsonb) || jsonb_build_object('lastAttemptAt', ${now.toISOString()}::text)) + where id = ${siteId} + and (config->'cloudflare'->>'lastAttemptAt' is null + or (config->'cloudflare'->>'lastAttemptAt')::timestamptz <= ${before}) + returning id`.execute(this.ctx.db); + this.cache.delete(siteId); + return r.rows.length > 0; + } + + /** + * 有登录态的账号:与站点请求用的凭证同一个(第一个启用、未失效的 Cookie / 账密账号,且确实有 Cookie)。 + * 没有返回 null(没登录的站点不处理挑战)。 + */ + async loggedInAccount(siteId: string): Promise { + const rows = await this.ctx.db + .selectFrom("site_account") + .selectAll() + .where("site_id", "=", siteId) + .where("enabled", "=", true) + .where("status", "in", ["unknown", "active"]) + .where("auth_kind", "in", ["cookie", "password"]) + .orderBy("created_at") + .execute(); + return ( + rows.find( + (r) => (r.auth_kind === "cookie" && !!r.secret_ref) || (r.auth_kind === "password" && !!r.session_secret_ref), + ) ?? null + ); + } + + /** 这条请求能带的 cf_clearance 头片段("cf_clearance=…");不满足复用条件为 null */ + async clearanceCookie(siteId: string, req: ChallengeRequest): Promise { + const acc = await this.loggedInAccount(siteId); + if (!acc) return null; + const info = clearanceInfo(acc.user_info); + if (!info) return null; + const st = await this.state(siteId); + const d = await this.rt.uaDefaults(); + const ok = clearanceUsable({ + uaSource: resolveUserAgent(acc, d).source, + requestUa: req.userAgent, + clearanceUa: info.userAgent, + requestEgress: req.egress, + clearanceEgress: info.egress, + cfReuse: st.cfReuse, + }); + if (!ok) return null; + const v = await this.ctx.secrets.resolve(info.secretRef).catch(() => null); + return v ? `cf_clearance=${v}` : null; + } + + /** 把浏览器拿到的 cf_clearance 存进账号(secret),记下取得时的 UA 与出口 */ + private async storeClearance( + acc: SiteAccountRow, + value: string, + userAgent: string | null, + egress: string | null, + ): Promise<{ conditionsChanged: boolean }> { + const info = obj(acc.user_info); + const prev = clearanceInfo(acc.user_info); + const conditionsChanged = !!prev && (prev.userAgent !== userAgent || prev.egress !== egress); + let ref = prev?.secretRef ?? null; + if (ref) { + try { + await this.ctx.secrets.replace(ref, value); + } catch { + ref = null; + } + } + if (!ref) + ref = await this.ctx.secrets.create({ + purpose: "site_cookie", + plaintext: value, + label: `site_account:${acc.site_id}:${acc.label}:cf_clearance:${crypto.randomUUID().slice(0, 8)}`, + }); + const next: ClearanceInfo = { secretRef: ref, userAgent, egress, at: this.now().toISOString() }; + await this.ctx.db + .updateTable("site_account") + .set({ user_info: JSON.stringify({ ...info, cloudflare: next }) }) + .where("id", "=", acc.id) + .execute(); + // UA 或出口变了:之前"复用无效"的结论不再适用,重新试一次 + if (conditionsChanged) await this.patch(acc.site_id, { cfReuse: null }); + return { conditionsChanged }; + } + + /** 过挑战:进程内同站只跑一次,跨进程 30 分钟一次;被节流返回 null */ + private solveOnce(site: SiteRow, acc: SiteAccountRow, url: string): Promise { + const running = this.inflight.get(site.id); + if (running) return running; + const p = (async () => { + const port = challengeBrowserFor(this.ctx); + if (!port) return null; + if (!(await this.claimAttempt(site.id))) return null; + try { + return await port.solve({ siteId: site.id, accountId: acc.id, url }); + } catch (e) { + return { status: "failed" as const, reason: String((e as Error)?.message ?? e).slice(0, 300) }; + } + })().finally(() => this.inflight.delete(site.id)); + this.inflight.set(site.id, p); + return p; + } + + /** 浏览器代抓一页;失败 / 仍是挑战返回 null */ + private async viaBrowser(siteId: string, acc: SiteAccountRow, url: string): Promise { + const port = challengeBrowserFor(this.ctx); + if (!port) return null; + let r: BrowserFetchOutcome; + try { + r = await port.fetchHtml({ siteId, accountId: acc.id, url }); + } catch (e) { + r = { status: "unavailable", reason: String((e as Error)?.message ?? e) }; + } + if (r.status !== "ok") { + this.ctx.logger.info({ component: "cloudflare", siteId, status: r.status, reason: r.reason }, "浏览器代抓未成功"); + if (r.status === "challenge") await this.patch(siteId, { lastResult: "browser_challenge", lastError: r.reason }); + return null; + } + return new Response(r.html, { + status: 200, + headers: { "content-type": "text/html; charset=utf-8", "x-umt-via": "browser" }, + }); + } + + private log(siteId: string, msg: string, extra: Record = {}): void { + this.ctx.logger.info({ component: "cloudflare", siteId, ...extra }, msg); + } + + /** + * 站点请求入口。send(cookieExtra) 发一次 HTTP(cookieExtra 非空时并进 Cookie 头);返回交给适配器的响应 + * (挑战没处理掉时原样返回挑战响应,适配器报 challenge)。 + */ + async handle( + siteId: string, + req: ChallengeRequest, + send: (cookieExtra: string | null) => Promise, + ): Promise { + const pagey = browserFetchable(req); + let st = await this.state(siteId); + const now = this.now(); + // 0. 浏览器代理期:页面请求直接走浏览器 + if (pagey && st.browserUntil && Date.parse(st.browserUntil) > now.getTime() && challengeBrowserFor(this.ctx)) { + const acc = await this.loggedInAccount(siteId); + const r = acc ? await this.viaBrowser(siteId, acc, req.url) : null; + if (r) return r; + } + // 1. HTTP(满足条件时带 cf_clearance) + // 从没遇到过挑战的站点不查账号(每条请求少两次查询) + const extra = st.lastChallengeAt ? await this.clearanceCookie(siteId, req).catch(() => null) : null; + const first = await detectChallenge(await send(extra)); + if (!first.rule) return first.res; + // 2. 挑战:记下来;没有浏览器 / 没有登录态账号 → 原样返回(适配器报 challenge,不算凭据错误) + await this.patch(siteId, { lastChallengeAt: now.toISOString(), lastRule: first.rule }); + const port = challengeBrowserFor(this.ctx); + if (!port) return first.res; + const acc = await this.loggedInAccount(siteId); + if (!acc) { + await this.patch(siteId, { lastResult: "skipped", lastError: "站点没有已登录的账号，不处理挑战" }); + return first.res; + } + const site = await this.ctx.db.selectFrom("site").selectAll().where("id", "=", siteId).executeTakeFirst(); + if (!site?.enabled || site.deleted_at) return first.res; + // 3. 过挑战 + const out = await this.solveOnce(site, acc, req.url); + st = await this.state(siteId, true); + if (!out) this.log(siteId, "站点 Cloudflare 挑战：30 分钟内处理过，本次不再过挑战"); + else if (out.status === "passed") { + const at = this.now().toISOString(); + await this.patch(siteId, { + lastSolvedBy: out.by, + lastSolvedAt: at, + lastResult: "passed", + pending: null, + sessionId: null, + lastError: null, + }); + this.log(siteId, "站点 Cloudflare 挑战已通过", { by: out.by }); + // 4. 同步 cf_clearance 并重试 HTTP 一次 + if (out.clearance) { + await this.storeClearance(acc, out.clearance, out.userAgent, out.browserEgress); + const retryExtra = await this.clearanceCookie(siteId, req).catch(() => null); + if (retryExtra) { + await this.rt.limiterOf(siteId)?.acquire(); + const second = await detectChallenge(await send(retryExtra)); + if (!second.rule) { + await this.patch(siteId, { cfReuse: "ok", lastResult: "reuse_ok", browserUntil: null }); + return second.res; + } + await second.res.body?.cancel().catch(() => {}); + await this.patch(siteId, { cfReuse: "fails", lastResult: "reuse_fails" }); + this.log(siteId, "浏览器的 cf_clearance 给 HTTP 用仍被挑战，之后不再复用"); + } + } + } else if (out.status === "needs_human") { + await this.patch(siteId, { + pending: "human", + sessionId: out.sessionId, + lastResult: "needs_human", + lastError: out.reason, + }); + } else { + await this.patch(siteId, { lastResult: "failed", lastError: out.reason }); + } + // 5. 页面请求改由浏览器代抓,并进入 6 小时浏览器代理期 + if (pagey && !(out && out.status === "needs_human")) { + const r = await this.viaBrowser(siteId, acc, req.url); + if (r) { + const until = new Date(this.now().getTime() + CF_BROWSER_MODE_MS).toISOString(); + await this.patch(siteId, { browserUntil: until, ...(out ? {} : { lastResult: "browser" }) }); + await first.res.body?.cancel().catch(() => {}); + return r; + } + } + return first.res; + } + + /** + * 人在原站视图里过了挑战、交还会话之后(浏览器模块调用):读回的 cf_clearance / UA 存起来,清掉"等人处理"。 + */ + async afterHuman( + siteId: string, + accountId: string, + got: { clearance: string | null; userAgent: string | null; browserEgress: string | null }, + ): Promise { + const st = await this.state(siteId, true); + if (st.pending !== "human") return; + const acc = await this.ctx.db.selectFrom("site_account").selectAll().where("id", "=", accountId).executeTakeFirst(); + if (acc && got.clearance) await this.storeClearance(acc, got.clearance, got.userAgent, got.browserEgress); + await this.patch(siteId, { + pending: null, + sessionId: null, + lastSolvedBy: "human", + lastSolvedAt: this.now().toISOString(), + lastResult: "passed", + lastError: null, + // 人过了:允许马上再试一次 HTTP(不等 30 分钟) + lastAttemptAt: null, + }); + this.log(siteId, "站点 Cloudflare 挑战已由人在原站视图里通过"); + } +} + +/** Cookie 头里并进 cf_clearance(同名覆盖);__cf_bm 等其它 Cloudflare Cookie 不带 */ +export function withClearance(cookieHeader: string | null, extra: string | null): string | null { + if (!extra) return cookieHeader; + const jar = parseCookieHeader(cookieHeader); + jar.delete("__cf_bm"); + const merged = mergeCookieHeaders([...jar].map(([k, v]) => `${k}=${v}`).join("; ") || null, extra); + return merged; +} + +/** 站点出口键(与浏览器会话 applyEgress 同一口径) */ +export function siteEgressKey(proxy: boolean, proxyUrl: string | null | undefined): string { + return egressKey(siteEgress(proxy, proxyUrl)); +} diff --git a/apps/server/src/modules/websearch/service.ts b/apps/server/src/modules/websearch/service.ts index d8b46333..bfcbc7ba 100644 --- a/apps/server/src/modules/websearch/service.ts +++ b/apps/server/src/modules/websearch/service.ts @@ -17,6 +17,7 @@ import { type WebSearchTestResult, } from "@umt/contracts"; import { cachedCall, pgCacheFor, pgTokenBucketFor } from "@umt/db"; +import { externalFetch } from "../../egress.ts"; import { storageSettings } from "../../shared-cache.ts"; import type { ServerContext } from "../types.ts"; @@ -94,7 +95,8 @@ export class WebSearchService { return createBraveClient({ apiKey, ...(this.opts.braveBaseUrl ? { baseUrl: this.opts.braveBaseUrl } : {}), - ...(this.opts.fetch ? { fetch: this.opts.fetch } : {}), + // 外网请求:显式按环境变量走代理(CF-2 出口表) + fetch: this.opts.fetch ?? externalFetch, }); } @@ -184,7 +186,7 @@ export class WebSearchService { allow ? "只能读取最近一次 web.search 结果里的链接" : "先用 web.search 搜索，再读取结果里的链接", ); const page = await fetchPage(want, { - ...(this.opts.fetch ? { fetch: this.opts.fetch } : {}), + fetch: this.opts.fetch ?? externalFetch, ...(this.opts.resolveHost ? { resolve: this.opts.resolveHost } : {}), ...(o.signal ? { signal: o.signal } : {}), }); diff --git a/apps/server/test/sources-identity/cloudflare.test.ts b/apps/server/test/sources-identity/cloudflare.test.ts new file mode 100644 index 00000000..9939ea7f --- /dev/null +++ b/apps/server/test/sources-identity/cloudflare.test.ts @@ -0,0 +1,315 @@ +import { SiteError } from "@umt/adapters"; +import { siteCloudflareState } from "@umt/contracts"; +import { createMockNexusPhpSite, type MockNexusPhpSite } from "@umt/testkit"; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; +import { + type BrowserFetchOutcome, + browserFetchable, + CF_ATTEMPT_INTERVAL_MS, + type ChallengeSolveOutcome, + clearanceUsable, + registerChallengeBrowser, + withClearance, +} from "../../src/modules/sources-identity/service/cloudflare.ts"; +import { snapshotSite } from "../../src/modules/sources-identity/service/snapshot.ts"; +import { registerBrowserIdentityReader } from "../../src/modules/sources-identity/service/user-agent.ts"; +import { createHarness, type Harness, pgAvailable } from "./harness.ts"; + +// CF-2 级联状态机:HTTP 遇 Cloudflare 挑战 → 浏览器过挑战(自动 / 视觉子流程 / 转人工)→ cf_clearance 同步给 HTTP 重试 1 次 → +// 仍被挑战则页面请求改由浏览器代抓并进入 6 小时浏览器代理期;同站 30 分钟节流、同一时刻只处理一次;挑战不算凭据错误。 +// 站点 = testkit mock NexusPHP(真 HTTP),前面套一层"Cloudflare":没带合格 cf_clearance 的请求回 403 + cf-mitigated 挑战页。 +// 浏览器一侧用假端口(ChallengeBrowserPort),真实浏览器流程见 browser-cloudflare.test.ts。 + +const up = await pgAvailable(); +const BROWSER_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:152.0) Gecko/20100101 Firefox/152.0"; +const CF_HTML = `Just a moment... +
`; + +// biome-ignore lint/suspicious/noExplicitAny: 响应体断言 +const json = async (r: Response): Promise => { + const body = await r.json(); + if (!r.ok) throw new Error(`${r.status} ${JSON.stringify(body)}`); + return body.data; +}; + +describe("CF-2 纯函数", () => { + it("页面请求才能由浏览器代抓", () => { + expect(browserFetchable({ method: "GET", accept: "text/html,application/xhtml+xml" })).toBe(true); + expect(browserFetchable({ method: "GET", accept: null })).toBe(false); + expect(browserFetchable({ method: "POST", accept: "text/html" })).toBe(false); + expect(browserFetchable({ method: "GET", accept: "application/rss+xml" })).toBe(false); + }); + it("cf_clearance 同步条件:UA 跟随浏览器 + UA 相同 + 两端出口相同 + 复用没失败过", () => { + const base = { + uaSource: "browser", + requestUa: "A", + clearanceUa: "A", + requestEgress: "direct", + clearanceEgress: "direct", + cfReuse: null, + } as const; + expect(clearanceUsable(base)).toBe(true); + expect(clearanceUsable({ ...base, uaSource: "manual" })).toBe(false); + expect(clearanceUsable({ ...base, uaSource: "global" })).toBe(false); + expect(clearanceUsable({ ...base, clearanceUa: "B" })).toBe(false); + expect(clearanceUsable({ ...base, clearanceEgress: null })).toBe(false); + expect(clearanceUsable({ ...base, clearanceEgress: "proxy:http://relay:7891" })).toBe(false); + expect(clearanceUsable({ ...base, cfReuse: "fails" })).toBe(false); + }); + it("并进 cf_clearance,去掉 __cf_bm", () => { + expect(withClearance("a=1; __cf_bm=x", "cf_clearance=Z")).toBe("a=1; cf_clearance=Z"); + expect(withClearance(null, "cf_clearance=Z")).toBe("cf_clearance=Z"); + expect(withClearance("a=1", null)).toBe("a=1"); + }); +}); + +describe.skipIf(!up)("CF-2 Cloudflare 级联", () => { + let h: Harness; + let mock: MockNexusPhpSite; + let siteId: string; + let accountId: string; + /** "Cloudflare" 开关与放行规则 */ + const cf = { + on: false, + /** 放行判定;缺省:带 cf_clearance=GOOD 且 UA 是浏览器 UA */ + pass: (h: Headers) => (h.get("cookie") ?? "").includes("cf_clearance=GOOD") && h.get("user-agent") === BROWSER_UA, + challenged: 0, + passed: 0, + }; + const seen: Headers[] = []; + const solves: { url: string; accountId: string }[] = []; + const fetches: string[] = []; + let solveResult: ChallengeSolveOutcome = { status: "failed", reason: "unset" }; + let solveDelay = 0; + let fetchResult: BrowserFetchOutcome | null = null; + + const cfFetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const hd = new Headers(init?.headers); + seen.push(hd); + if (cf.on && !cf.pass(hd)) { + cf.challenged++; + return new Response(CF_HTML, { + status: 403, + headers: { "cf-mitigated": "challenge", "content-type": "text/html" }, + }); + } + cf.passed++; + return globalThis.fetch(input, init); + }) as typeof fetch; + + const site = () => h.db.selectFrom("site").selectAll().where("id", "=", siteId).executeTakeFirstOrThrow(); + const account = () => + h.db.selectFrom("site_account").selectAll().where("id", "=", accountId).executeTakeFirstOrThrow(); + const state = async () => siteCloudflareState((await site()).config); + const list = async () => h.rt.siteAdapter(await site()).list(1); + + beforeAll(async () => { + mock = await createMockNexusPhpSite({}); + h = await createHarness("cf2", { fetch: cfFetch }); + registerBrowserIdentityReader(h.ctx, async () => BROWSER_UA); + registerChallengeBrowser(h.ctx, { + async solve(req) { + solves.push(req); + if (solveDelay) await new Promise((r) => setTimeout(r, solveDelay)); + return solveResult; + }, + async fetchHtml(req) { + fetches.push(req.url); + if (fetchResult) return fetchResult; + // 浏览器已登录且过了挑战:直接取 mock 页面 + const r = await globalThis.fetch(req.url, { headers: { cookie: mock.cookie } }); + return { status: "ok", html: await r.text(), url: req.url }; + }, + }); + const s = await json( + await h.req("/sites", { + body: { + key: "audiences", + name: "观众", + adapter: "nexusphp", + baseUrl: `${mock.baseUrl}/`, + rateLimit: { minIntervalMs: 0, concurrency: 10 }, + }, + }), + ); + siteId = s.id; + const r = await json( + await h.req(`/sites/${siteId}/credentials`, { method: "PUT", body: { kind: "cookie", cookie: mock.cookie } }), + ); + accountId = r.siteAccountId; + }); + afterAll(async () => { + await h?.close(); + await mock?.close(); + }); + + beforeEach(async () => { + cf.on = true; + cf.pass = (hd) => (hd.get("cookie") ?? "").includes("cf_clearance=GOOD") && hd.get("user-agent") === BROWSER_UA; + cf.challenged = 0; + cf.passed = 0; + seen.length = 0; + solves.length = 0; + fetches.length = 0; + solveDelay = 0; + fetchResult = null; + solveResult = { status: "passed", by: "auto", clearance: "GOOD", userAgent: BROWSER_UA, browserEgress: "direct" }; + await h.db + .updateTable("site") + .set({ config: JSON.stringify({}), health_status: "healthy" }) + .where("id", "=", siteId) + .execute(); + await h.db + .updateTable("site_account") + .set({ user_info: JSON.stringify({}), user_agent: null, ua_pending: null, status: "active" }) + .where("id", "=", accountId) + .execute(); + await h.rt.cloudflare.patch(siteId, {}); + }); + + it("自动通过:浏览器拿到 cf_clearance + UA,HTTP 带上重试 1 次成功,记 cfReuse = ok;之后的请求直接带", async () => { + const page = await list(); + expect(page.items.length).toBeGreaterThan(0); + expect(solves).toEqual([{ siteId, accountId, url: expect.stringContaining("/torrents.php") }]); + expect(cf.challenged).toBe(1); + expect(fetches).toEqual([]); + const st = await state(); + expect(st).toMatchObject({ lastSolvedBy: "auto", cfReuse: "ok", lastResult: "reuse_ok", browserUntil: null }); + expect(st?.lastChallengeAt).toBeTruthy(); + expect(st?.lastRule).toBe("header cf-mitigated=challenge"); + // cf_clearance 存在 secret 里,账号 user_info 只有引用与取得时的 UA / 出口 + const info = (await account()).user_info as { + cloudflare?: { secretRef: string; userAgent: string; egress: string }; + }; + expect(info.cloudflare).toMatchObject({ userAgent: BROWSER_UA, egress: "direct" }); + expect(h.secrets.plain.get(info.cloudflare!.secretRef)).toBe("GOOD"); + expect(JSON.stringify(info)).not.toContain("GOOD"); + // __cf_bm 不带;下一条请求直接带 cf_clearance,不再过挑战 + seen.length = 0; + await list(); + expect(solves).toHaveLength(1); + expect(seen[0]?.get("cookie")).toContain("cf_clearance=GOOD"); + expect(seen[0]?.get("cookie")).not.toContain("__cf_bm"); + // 账号状态不变 + expect((await account()).status).toBe("active"); + }); + + it("视觉子流程通过:记 lastSolvedBy = vision", async () => { + solveResult = { status: "passed", by: "vision", clearance: "GOOD", userAgent: BROWSER_UA, browserEgress: "direct" }; + await list(); + expect(await state()).toMatchObject({ lastSolvedBy: "vision", cfReuse: "ok" }); + }); + + it("复用失败:cf_clearance 给 HTTP 仍被挑战 → cfReuse = fails,本次由浏览器代抓,6 小时浏览器代理期内页面请求直接走浏览器", async () => { + cf.pass = () => false; // 站点绑定 TLS 指纹:HTTP 怎么都过不去 + const page = await list(); + expect(page.items.length).toBeGreaterThan(0); + expect(cf.challenged).toBe(2); // 首次 + 带 cf_clearance 重试 1 次 + expect(fetches).toHaveLength(1); + const st = await state(); + expect(st).toMatchObject({ cfReuse: "fails", lastResult: "reuse_fails" }); + const until = Date.parse(st!.browserUntil!); + expect(until - Date.now()).toBeGreaterThan(5.9 * 3_600_000); + // 代理期内:不发 HTTP,不再过挑战 + await list(); + expect(cf.challenged).toBe(2); + expect(fetches).toHaveLength(2); + expect(solves).toHaveLength(1); + // 下载(不是页面请求)仍走 HTTP,报 challenge,不带失败过的 cf_clearance + const err = await h.rt + .siteAdapter(await site()) + .downloadTarget(page.items[0]!.siteReleaseId) + .catch((e) => e); + expect(err).toBeInstanceOf(SiteError); + expect((err as SiteError).kind).toBe("challenge"); + expect(seen.at(-1)?.get("cookie") ?? "").not.toContain("cf_clearance"); + // 到期再试 HTTP + await h.rt.cloudflare.patch(siteId, { browserUntil: new Date(Date.now() - 1000).toISOString() }); + cf.pass = () => true; + const n = fetches.length; + await list(); + expect(fetches).toHaveLength(n); + }); + + it("真验证码转人工:本次报 challenge(不算凭据错误),站点记 pending = human;快照不置账号失效、不开登录失效事项", async () => { + solveResult = { status: "needs_human", sessionId: null, decisionId: null, reason: "选图验证码" }; + const r = await snapshotSite(h.rt, { siteId, kind: "manual" }); + expect(r.status).toBe("failed"); + expect(r.error).toContain("challenge"); + expect(fetches).toEqual([]); + expect(await state()).toMatchObject({ pending: "human", lastResult: "needs_human", lastError: "选图验证码" }); + expect((await account()).status).toBe("active"); + expect((await site()).health_status).toBe("degraded"); + const d = await h.db.selectFrom("decision").select("id").where("dedupe_key", "=", `site-auth:${siteId}`).execute(); + expect(d).toEqual([]); + expect(h.engine.enqueued.filter((e) => e.task === "refreshSiteAccount")).toEqual([]); + + // 人在原站视图里过完、交还:浏览器模块回调 afterHuman → 清掉等人处理,存下 cf_clearance,马上可以再试 + await h.rt.cloudflare.afterHuman(siteId, accountId, { + clearance: "GOOD", + userAgent: BROWSER_UA, + browserEgress: "direct", + }); + expect(await state()).toMatchObject({ pending: null, lastSolvedBy: "human", lastAttemptAt: null }); + const page = await list(); + expect(page.items.length).toBeGreaterThan(0); + expect(solves).toHaveLength(1); + }); + + it("节流与单飞:并发的挑战只过一次;30 分钟内不再过;超过 30 分钟再过", async () => { + solveDelay = 150; + cf.pass = () => false; + fetchResult = { status: "challenge", reason: "still" }; + const results = await Promise.allSettled([list(), list(), list()]); + expect(results.every((r) => r.status === "rejected")).toBe(true); + expect(solves).toHaveLength(1); + // 30 分钟内:只记挑战,不过 + await list().catch(() => null); + expect(solves).toHaveLength(1); + expect((await state())?.lastAttemptAt).toBeTruthy(); + // 超过 30 分钟 + await h.rt.cloudflare.patch(siteId, { + lastAttemptAt: new Date(Date.now() - CF_ATTEMPT_INTERVAL_MS - 1000).toISOString(), + }); + await list().catch(() => null); + expect(solves).toHaveLength(2); + }); + + it("同步条件不满足:账号手动 UA → 不把 cf_clearance 交给 HTTP(不重试),页面改由浏览器代抓", async () => { + await h.db.updateTable("site_account").set({ user_agent: BROWSER_UA }).where("id", "=", accountId).execute(); + const page = await list(); + expect(page.items.length).toBeGreaterThan(0); + expect(cf.challenged).toBe(1); + expect(fetches).toHaveLength(1); + expect(seen.every((x) => !(x.get("cookie") ?? "").includes("cf_clearance"))).toBe(true); + expect((await state())?.browserUntil).toBeTruthy(); + }); + + it("同步条件不满足:浏览器出口与 HTTP 不同(旧插件按环境变量出网)→ 不复用", async () => { + solveResult = { status: "passed", by: "auto", clearance: "GOOD", userAgent: BROWSER_UA, browserEgress: null }; + await list(); + expect(cf.challenged).toBe(1); + expect(fetches).toHaveLength(1); + expect((await state())?.cfReuse).toBeNull(); + }); + + it("没有登录态账号的站点不处理挑战", async () => { + await h.db.updateTable("site_account").set({ status: "expired" }).where("id", "=", accountId).execute(); + // 适配器没有 Cookie 就不发请求;这里直接用站点 fetch(RSS 这类不要登录态的请求同理) + const res = await h.rt.siteFetch(await site())(`${mock.baseUrl}/torrents.php`, { + headers: { accept: "text/html" }, + }); + expect(res.status).toBe(403); + expect(solves).toEqual([]); + expect(fetches).toEqual([]); + expect(await state()).toMatchObject({ lastResult: "skipped" }); + }); + + it("健康检测遇挑战且过不去:记 challenge(degraded),账号状态不变", async () => { + solveResult = { status: "failed", reason: "浏览器服务不可用" }; + fetchResult = { status: "unavailable", reason: "浏览器服务不可用" }; + const r = await json(await h.req(`/sites/${siteId}/health-check`, { method: "POST", body: {} })); + expect(r).toMatchObject({ ok: false, status: "challenge", healthStatus: "degraded", loggedIn: null }); + expect((await account()).status).toBe("active"); + }); +}); diff --git a/apps/server/test/sources-identity/egress.test.ts b/apps/server/test/sources-identity/egress.test.ts new file mode 100644 index 00000000..dc8222cf --- /dev/null +++ b/apps/server/test/sources-identity/egress.test.ts @@ -0,0 +1,133 @@ +import { DEFAULT_SETTINGS } from "@umt/contracts"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { + directHttpClient, + egressKey, + envProxyFor, + fetchVia, + noProxyMatches, + proxyHttpClient, + siteEgress, +} from "../../src/egress.ts"; +import { SourcesRuntime } from "../../src/modules/sources-identity/runtime.ts"; +import type { ServerContext } from "../../src/modules/types.ts"; + +// CF-2 出口统一:站点请求不再隐式继承容器的 HTTP(S)_PROXY。开了站点代理 → 该代理;否则明确直连。 +// 本地起一个"代理"(收到请求就回 via-proxy)和一个目标(回 direct),把 HTTP_PROXY 指向前者,看请求实际走哪条。 + +const PROXY_KEYS = [ + "HTTP_PROXY", + "HTTPS_PROXY", + "ALL_PROXY", + "http_proxy", + "https_proxy", + "all_proxy", + "NO_PROXY", + "no_proxy", +]; + +describe("出口选择(直连 / 代理)", () => { + let proxy: Deno.HttpServer; + let target: Deno.HttpServer; + let proxyUrl = ""; + let targetUrl = ""; + const proxied: string[] = []; + const saved = new Map(); + + beforeAll(() => { + proxy = Deno.serve({ hostname: "127.0.0.1", port: 0, onListen() {} }, (req) => { + proxied.push(req.url); + return new Response("via-proxy"); + }); + target = Deno.serve({ hostname: "127.0.0.1", port: 0, onListen() {} }, () => new Response("direct")); + proxyUrl = `http://127.0.0.1:${proxy.addr.port}`; + targetUrl = `http://127.0.0.1:${target.addr.port}/x`; + for (const k of PROXY_KEYS) { + saved.set(k, Deno.env.get(k)); + Deno.env.delete(k); + } + // 模拟容器环境:HTTP_PROXY 指向"中继",没有 NO_PROXY + Deno.env.set("HTTP_PROXY", proxyUrl); + }); + + afterAll(async () => { + for (const [k, v] of saved) { + if (v === undefined) Deno.env.delete(k); + else Deno.env.set(k, v); + } + await proxy.shutdown(); + await target.shutdown(); + }); + + const body = async (f: typeof fetch) => (await f(targetUrl)).text(); + + it("Deno 2.9 语义:createHttpClient({}) 仍会读 HTTP_PROXY(所以直连要显式做)", async () => { + const implicit = Deno.createHttpClient({}); + try { + expect(await body(fetchVia(implicit))).toBe("via-proxy"); + } finally { + implicit.close(); + } + }); + + it("directHttpClient:环境变量里有代理也直连;创建后环境变量原样还原", async () => { + proxied.length = 0; + expect(await body(fetchVia(directHttpClient()))).toBe("direct"); + expect(proxied).toEqual([]); + expect(Deno.env.get("HTTP_PROXY")).toBe(proxyUrl); + }); + + it("proxyHttpClient:只走给定代理(NO_PROXY 命中也不绕过)", async () => { + Deno.env.set("NO_PROXY", "127.0.0.1"); + try { + proxied.length = 0; + const client = proxyHttpClient(proxyUrl); + expect(await body(fetchVia(client))).toBe("via-proxy"); + expect(proxied).toHaveLength(1); + expect(Deno.env.get("NO_PROXY")).toBe("127.0.0.1"); + } finally { + Deno.env.delete("NO_PROXY"); + } + }); + + it("SourcesRuntime.siteFetch:站点没开代理 → 直连;开了且设了代理地址 → 站点代理", async () => { + const ctx = { logger: { debug() {}, info() {}, warn() {} } } as unknown as ServerContext; + const rt = new SourcesRuntime(ctx); + const defaults = { ...DEFAULT_SETTINGS.sites, proxyUrl }; + proxied.length = 0; + expect(await body(rt.siteFetch({ proxy: false }, defaults, "UA"))).toBe("direct"); + expect(proxied).toEqual([]); + expect(await body(rt.siteFetch({ proxy: true }, defaults, "UA"))).toBe("via-proxy"); + expect(proxied).toHaveLength(1); + // 开了代理但没设代理地址 → 直连 + expect(await body(rt.siteFetch({ proxy: true }, { ...defaults, proxyUrl: null }, "UA"))).toBe("direct"); + expect(rt.egressKeyOf({ proxy: false }, defaults)).toBe("direct"); + expect(rt.egressKeyOf({ proxy: true }, defaults)).toBe(`proxy:http://127.0.0.1:${proxy.addr.port}`); + }); +}); + +describe("外网请求按环境变量(行为不变,改为显式)", () => { + it("HTTPS / HTTP / ALL_PROXY 与 NO_PROXY", () => { + const env = { HTTPS_PROXY: "http://relay:7891", HTTP_PROXY: "http://relay:7890", NO_PROXY: "localhost,.lan" }; + expect(envProxyFor(new URL("https://api.themoviedb.org/3"), env)).toBe("http://relay:7891"); + expect(envProxyFor(new URL("http://image.tmdb.org/t"), env)).toBe("http://relay:7890"); + expect(envProxyFor(new URL("http://nas.lan/x"), env)).toBeNull(); + expect(envProxyFor(new URL("https://api.search.brave.com/"), { ALL_PROXY: "socks5h://relay:1080" })).toBe( + "socks5h://relay:1080", + ); + expect(envProxyFor(new URL("https://api.search.brave.com/"), {})).toBeNull(); + expect(envProxyFor(new URL("https://x.example/"), { https_proxy: "http://lower:1" })).toBe("http://lower:1"); + }); + it("NO_PROXY 匹配", () => { + expect(noProxyMatches("a.example.com", "example.com")).toBe(true); + expect(noProxyMatches("example.com", ".example.com")).toBe(true); + expect(noProxyMatches("badexample.com", "example.com")).toBe(false); + expect(noProxyMatches("x", "*")).toBe(true); + expect(noProxyMatches("127.0.0.1", "localhost, 127.0.0.1:8080")).toBe(true); + }); + it("出口键不含代理口令", () => { + expect(egressKey(siteEgress(false, "http://u:p@relay:7891"))).toBe("direct"); + expect(egressKey(siteEgress(true, "http://u:p@relay:7891"))).toBe("proxy:http://relay:7891"); + expect(egressKey(siteEgress(true, null))).toBe("direct"); + }); +}); diff --git a/packages/contracts/src/sources.ts b/packages/contracts/src/sources.ts index c40512a4..6caf9662 100644 --- a/packages/contracts/src/sources.ts +++ b/packages/contracts/src/sources.ts @@ -124,6 +124,41 @@ export const SiteLastHealth = z.object({ }); export type SiteLastHealth = z.infer; +/** + * 站点的 Cloudflare 处理状态(CF-2,存 site.config.cloudflare;docs/runbook/sites.md「Cloudflare」): + * - browserUntil:在此之前该站能走浏览器的页面请求直接由浏览器代抓(HTTP 仍被挑战时置 6 小时,到期再试 HTTP); + * - cfReuse:浏览器拿到的 cf_clearance 给 HTTP 用的结果(ok 有效 / fails 无效,之后不再复用); + * - lastSolvedBy:最近一次怎么过的挑战(auto 等它自己过 / vision 视觉子流程点了一下 / human 人在原站视图里过); + * - pending = human:遇到选图 / 输码类验证码,等人在原站视图里过一次; + * - lastAttemptAt:最近一次尝试过挑战(同站 30 分钟最多一次)。 + */ +export const SiteCloudflareState = z.object({ + browserUntil: Timestamp.nullable().default(null), + cfReuse: z.enum(["ok", "fails"]).nullable().default(null), + lastChallengeAt: Timestamp.nullable().default(null), + lastSolvedBy: z.enum(["auto", "vision", "human"]).nullable().default(null), + lastSolvedAt: Timestamp.nullable().default(null), + lastAttemptAt: Timestamp.nullable().default(null), + pending: z.enum(["human"]).nullable().default(null), + /** 等人处理的浏览器会话(原站视图接管入口) */ + sessionId: Id.nullable().default(null), + /** 最近一次处理结果(技术信息):passed / reuse_ok / reuse_fails / browser / needs_human / failed / throttled / skipped */ + lastResult: z.string().nullable().default(null), + /** 命中的挑战规则(技术信息) */ + lastRule: z.string().nullable().default(null), + /** 最近一次失败原因(技术信息) */ + lastError: z.string().nullable().default(null), +}); +export type SiteCloudflareState = z.infer; + +/** 从 site.config 读 Cloudflare 状态;没有或格式不对返回 null */ +export function siteCloudflareState(config: unknown): SiteCloudflareState | null { + const raw = (config as { cloudflare?: unknown } | null | undefined)?.cloudflare; + if (!raw || typeof raw !== "object") return null; + const r = SiteCloudflareState.safeParse(raw); + return r.success ? r.data : null; +} + export const Site = EntityBase.extend({ key: z.string().regex(/^[a-z][a-z0-9_-]*$/), name: z.string().min(1), From f16b97151bb5367a15becebaaddc4c41580fcbba Mon Sep 17 00:00:00 2001 From: root Date: Wed, 30 Sep 2026 21:26:54 +0800 Subject: [PATCH 4/5] =?UTF-8?q?feat(web):=20=E7=AB=99=E7=82=B9=E6=98=BE?= =?UTF-8?q?=E7=A4=BA=20Cloudflare=20=E7=8A=B6=E6=80=81(CF-2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- apps/web/app/components/sites/site-card.tsx | 11 ++++ .../app/components/sites/site-fields.test.ts | 50 ++++++++++++++- apps/web/app/components/sites/site-fields.ts | 64 ++++++++++++++++++- .../app/components/sites/site-overview.tsx | 33 +++++++++- 4 files changed, 155 insertions(+), 3 deletions(-) diff --git a/apps/web/app/components/sites/site-card.tsx b/apps/web/app/components/sites/site-card.tsx index efd35c8a..883eb3db 100644 --- a/apps/web/app/components/sites/site-card.tsx +++ b/apps/web/app/components/sites/site-card.tsx @@ -29,6 +29,7 @@ import type { SiteDialog, useSiteChecks } from "./site-dialogs.tsx"; import { ACCOUNT_OVERALL, bytes, + cloudflareLine, dataAccount, hasLogin, healthLine, @@ -291,6 +292,7 @@ function AccountSiteCard({ site, checks, onDialog }: Props) { const hmsg = healthMessage(site.lastHealth); const stale = site.snapshotEnabled && snapshotStale(site); const circuit = circuitLabel(site); + const cf = cloudflareLine(site.config); const lastErr = acc && acc.status !== "active" ? acc.lastError : null; return ( @@ -321,6 +323,15 @@ function AccountSiteCard({ site, checks, onDialog }: Props) { {u?.vip ? VIP : null} {circuit ? {circuit} : null} + {cf ? ( + cf.takeover ? ( + + {cf.text} + + ) : ( + {cf.text} + ) + ) : null} {lastErr ? ( diff --git a/apps/web/app/components/sites/site-fields.test.ts b/apps/web/app/components/sites/site-fields.test.ts index c51bd956..1cd408b1 100644 --- a/apps/web/app/components/sites/site-fields.test.ts +++ b/apps/web/app/components/sites/site-fields.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { healthLine, healthMessage, healthTechItems } from "./site-fields.ts"; +import { cloudflareLine, cloudflareTechItems, healthLine, healthMessage, healthTechItems } from "./site-fields.ts"; describe("健康检测文案(HC-1)", () => { it("人机验证单独成类;失败原因一句,原始判定进技术信息", () => { @@ -24,3 +24,51 @@ describe("健康检测文案(HC-1)", () => { expect(healthTechItems({ ok: true, status: "ok", latencyMs: 1, message: null })).toEqual([]); }); }); + +describe("cloudflareLine / cloudflareTechItems(CF-2)", () => { + const now = new Date("2026-09-30T12:00:00+08:00"); + const cfg = (cloudflare: Record) => ({ cloudflare }); + it("没遇到过挑战不显示", () => { + expect(cloudflareLine({}, now)).toBeNull(); + expect(cloudflareLine(cfg({ lastSolvedBy: "auto" }), now)).toBeNull(); + expect(cloudflareTechItems({})).toEqual([]); + }); + it("需要你验证 > 浏览器代理中 > 自动通过", () => { + const at = "2026-09-30T11:00:00+08:00"; + expect(cloudflareLine(cfg({ lastChallengeAt: at, pending: "human", lastSolvedBy: "auto" }), now)).toEqual({ + text: "Cloudflare：需要你验证", + tone: "warning", + takeover: true, + }); + const until = new Date("2026-09-30T17:30:00+08:00"); + const line = cloudflareLine(cfg({ lastChallengeAt: at, browserUntil: until.toISOString() }), now); + const hm = `${String(until.getHours()).padStart(2, "0")}:${String(until.getMinutes()).padStart(2, "0")}`; + expect(line?.text).toBe(`Cloudflare：浏览器代理中，至 ${hm}`); + expect(cloudflareLine(cfg({ lastChallengeAt: at, lastSolvedBy: "vision" }), now)?.text).toBe( + "Cloudflare：自动通过", + ); + // 代理期已过、没通过过、有错误 + expect( + cloudflareLine(cfg({ lastChallengeAt: at, browserUntil: at, lastError: "浏览器服务不可用" }), now)?.text, + ).toBe("Cloudflare：没能通过"); + }); + it("原始记录进技术信息", () => { + const items = cloudflareTechItems( + cfg({ + lastChallengeAt: "2026-09-30T11:00:00+08:00", + lastRule: "header cf-mitigated=challenge", + lastResult: "reuse_fails", + lastSolvedBy: "auto", + cfReuse: "fails", + }), + ); + expect(items.map((i) => i.label)).toEqual([ + "最近遇到挑战", + "挑战规则", + "处理结果", + "最近通过方式", + "cf_clearance 复用", + ]); + expect(items.find((i) => i.label === "处理结果")?.value).toContain("reuse_fails"); + }); +}); diff --git a/apps/web/app/components/sites/site-fields.ts b/apps/web/app/components/sites/site-fields.ts index 04430cac..e3ab8ec7 100644 --- a/apps/web/app/components/sites/site-fields.ts +++ b/apps/web/app/components/sites/site-fields.ts @@ -11,6 +11,7 @@ import { type SiteListItem, type SiteLoginFailureReason, type SiteRateLimit, + siteCloudflareState, } from "@umt/contracts"; import type { StatusTone } from "~/components/console/status-badge.tsx"; import { relTime } from "~/components/workbench/release-fields.ts"; @@ -180,6 +181,64 @@ export function healthTechItems(h: SiteLastHealth | null): { label: string; valu ].filter((it) => it.value); } +/** + * Cloudflare 状态一行(CF-2):没遇到过挑战不显示。 + * 需要你验证 → 浏览器代理中,至 hh:mm → 自动通过(最近一次怎么过的)。 + */ +export function cloudflareLine( + config: unknown, + now: Date = new Date(), +): { text: string; tone: StatusTone; takeover: boolean } | null { + const st = siteCloudflareState(config); + if (!st?.lastChallengeAt) return null; + if (st.pending === "human") return { text: "Cloudflare：需要你验证", tone: "warning", takeover: true }; + if (st.browserUntil && Date.parse(st.browserUntil) > now.getTime()) { + const t = new Date(st.browserUntil); + const hm = `${String(t.getHours()).padStart(2, "0")}:${String(t.getMinutes()).padStart(2, "0")}`; + return { text: `Cloudflare：浏览器代理中，至 ${hm}`, tone: "neutral", takeover: false }; + } + if (st.lastSolvedBy) return { text: "Cloudflare：自动通过", tone: "success", takeover: false }; + if (st.lastError) return { text: "Cloudflare：没能通过", tone: "warning", takeover: false }; + return null; +} + +const CF_SOLVED_BY: Record = { + auto: "等它自动放行", + vision: "视觉子流程点了复选框", + human: "你在原站视图里通过", +}; +const CF_RESULT: Record = { + passed: "已通过", + reuse_ok: "已通过，HTTP 复用 cf_clearance 有效", + reuse_fails: "已通过，HTTP 复用 cf_clearance 无效，改由浏览器代抓", + browser: "由浏览器代抓", + browser_challenge: "浏览器也被拦下", + needs_human: "等你在原站视图里验证", + failed: "没能通过", + skipped: "没有已登录的账号，未处理", +}; + +/** Cloudflare 原始记录 → 技术信息条目 */ +export function cloudflareTechItems(config: unknown): { label: string; value: string; mono?: boolean }[] { + const st = siteCloudflareState(config); + if (!st?.lastChallengeAt) return []; + const t = (v: string | null) => (v ? new Date(v).toLocaleString("zh-CN", { hour12: false }) : ""); + return [ + { label: "最近遇到挑战", value: t(st.lastChallengeAt) }, + { label: "挑战规则", value: st.lastRule ?? "", mono: true }, + { label: "处理结果", value: st.lastResult ? `${CF_RESULT[st.lastResult] ?? st.lastResult}(${st.lastResult})` : "" }, + { label: "最近通过方式", value: st.lastSolvedBy ? (CF_SOLVED_BY[st.lastSolvedBy] ?? st.lastSolvedBy) : "" }, + { label: "最近通过时间", value: t(st.lastSolvedAt) }, + { + label: "cf_clearance 复用", + value: st.cfReuse === "ok" ? "有效(ok)" : st.cfReuse === "fails" ? "无效(fails)" : "", + }, + { label: "浏览器代理至", value: t(st.browserUntil) }, + { label: "最近尝试", value: t(st.lastAttemptAt) }, + { label: "最近错误", value: st.lastError ?? "" }, + ].filter((it) => it.value); +} + /** 限流:"20 次/分";空对象 = 默认 */ export function rateLimitText(r: SiteRateLimit | null | undefined): string { if (!r || (!r.requestsPerMinute && !r.minIntervalMs && !r.concurrency)) return "默认"; @@ -202,7 +261,10 @@ export function loginFailureText( case "two_factor": return { text: "登录失败：站点要求二次验证。填写 TOTP 密钥后重试，或改用 Cookie", suggestCookie: true }; case "challenge": - return { text: "站点返回 Cloudflare 挑战页，无法自动登录。改用 Cookie", suggestCookie: true }; + return { + text: "站点要求人机验证（Cloudflare）。会用浏览器自动过，过不去时在待处理里提醒你", + suggestCookie: false, + }; case "bad_credentials": return { text: "登录失败：账号或密码错误", suggestCookie: false }; case "account_blocked": diff --git a/apps/web/app/components/sites/site-overview.tsx b/apps/web/app/components/sites/site-overview.tsx index e094c930..59d0cb84 100644 --- a/apps/web/app/components/sites/site-overview.tsx +++ b/apps/web/app/components/sites/site-overview.tsx @@ -1,9 +1,11 @@ import type { SiteListItem } from "@umt/contracts"; +import { Link } from "react-router"; import { useFeeds } from "~/api/feeds.ts"; import { accessSummary, useSiteRequestStats, useSiteUserData } from "~/api/sites.ts"; import { DetailSection } from "~/components/console/detail-sheet.tsx"; import { ErrorState, LoadingState } from "~/components/console/empty-state.tsx"; import { KvList } from "~/components/console/kv-list.tsx"; +import { consolePath } from "~/components/console/nav.ts"; import { StatTile } from "~/components/console/stat-tile.tsx"; import { StatusBadge } from "~/components/console/status-badge.tsx"; import { TechInfo } from "~/components/console/tech-info.tsx"; @@ -12,6 +14,8 @@ import { ACCOUNT_OVERALL, AUTH_KIND_LABEL, bytes, + cloudflareLine, + cloudflareTechItems, dataAccount, hasLogin, healthLine, @@ -26,6 +30,28 @@ import { } from "./site-fields.ts"; import { UserDataChart } from "./user-data-chart.tsx"; +/** Cloudflare 状态行(CF-2);需要你验证时给原站视图入口 */ +function cloudflareRow(site: SiteListItem) { + const cf = cloudflareLine(site.config); + if (!cf) return []; + const text = cf.text.replace(/^Cloudflare：/, ""); + return [ + { + label: "Cloudflare", + value: cf.takeover ? ( + + {text} + + 去原站视图 + + + ) : ( + text + ), + }, + ]; +} + const CAP_LABEL = [ ["search", "搜索"], ["list", "列表抓取"], @@ -180,11 +206,16 @@ function AccountSiteOverview({ site }: { site: SiteListItem }) { ), }, { label: "代理", value: site.proxy ? "使用" : "不使用" }, + ...cloudflareRow(site), ...(site.kind && site.kind !== "pt" ? [{ label: "类型", value: kindLabel(site.kind) }] : []), { label: "获取方式", value: capabilityText(site) }, ]} /> - + From c38ac2a85ce7e57de4133114f85311b23a363cc0 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 30 Sep 2026 21:26:54 +0800 Subject: [PATCH 5/5] =?UTF-8?q?docs:=20CF-2=20=E5=87=BA=E5=8F=A3=E8=A1=A8?= =?UTF-8?q?=E3=80=81Cloudflare=20=E7=BA=A7=E8=81=94=E8=BF=90=E7=BB=B4?= =?UTF-8?q?=E4=B8=8E=E6=8F=92=E4=BB=B6=E7=82=B9=E5=87=BB=E6=A0=B8=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 10 +++++ docs/design/cloudflare-handling.md | 5 +++ docs/plan/00-first-version-plan.md | 2 + docs/runbook/agent.md | 2 +- docs/runbook/browser.md | 38 ++++++++++++++-- docs/runbook/sites.md | 69 +++++++++++++++++++++++++++++- 6 files changed, 120 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70b940f8..2123f2ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ ## [Unreleased] +### 新增(CF-2 Cloudflare 验证处理) +- 站点请求遇 Cloudflare 挑战(`cf-mitigated: challenge`,或 403 / 503 / 429 的挑战页)不再算凭据错误:不置账号失效、不开「登录失效」事项、不排自动重登;503 挑战不再算维护;健康检测记「需要人机验证」。 +- 过挑战级联(只处理有登录态账号的站点;同站 30 分钟最多一次、同一时刻只处理一次;不接打码服务):浏览器用该账号的 profile 打开同一地址 → 等它自己过(≤ 15 秒)→ 「我是人类」复选框 / Turnstile 由系统发起视觉子流程点一下(最多 2 次,预算取设置,自动化总开关关闭时跳过)→ 选图 / 输码类转人工,待处理事项「<站点>需要你在原站视图里过一次验证」,接管过完交还即继续。 +- 过了之后读回 `cf_clearance` 与浏览器 UA:账号 UA 跟随浏览器且两端出口一致时交给 HTTP(`__cf_bm` 仍不带),重试 1 次,按站点记复用是否有效;仍被挑战时这次页面请求改由浏览器导航取 HTML 交给原解析器,之后 6 小时该站页面请求直接走浏览器,到期再试 HTTP。 +- 站点卡片 / 概览显示「Cloudflare：自动通过 / 浏览器代理中，至 hh:mm / 需要你验证」,原始记录进技术信息。 +- camofox 插件新端点 `POST /umt/tabs/{tabId}/mouse-click`:`click_at` 发真实鼠标事件,跨域 iframe 里的 Turnstile 也点得到(需要重建 camofox 镜像)。 + +### 变更(CF-2 出口统一) +- 站点请求(适配器、RSS、登录、用户数据)不再隐式继承容器的 `HTTP(S)_PROXY`:站点没开代理明确直连,开了走站点 / 全局代理,与浏览器会话同一出口。TMDB、Brave、图片缓存等外网请求照旧按环境变量走代理,改为代码里显式指定。出口表见 `docs/runbook/sites.md`「出口」。 + ## [0.3.4] - 2026-09-30 ## [0.3.3] - 2026-09-30 diff --git a/docs/design/cloudflare-handling.md b/docs/design/cloudflare-handling.md index e5aef947..21958764 100644 --- a/docs/design/cloudflare-handling.md +++ b/docs/design/cloudflare-handling.md @@ -1,5 +1,10 @@ # CF-1 Cloudflare 验证处理(调研) +> 状态:CF-2 已按本文 §3 推荐方案 A→B 级联落地,另加出口统一(站点请求明确直连 / 站点代理)。运维与字段见 +> `docs/runbook/sites.md`「出口」「Cloudflare(CF-2)」、`docs/runbook/browser.md` §13。与本文的差异:同步 cf_clearance 单独存 +> `site_account.user_info.cloudflare`(值在 secret),普通浏览器回写仍剔除;视觉子流程最多 2 次;`site.config.cloudflare` 多了 +> `lastAttemptAt / lastSolvedAt / pending / sessionId / lastResult / lastRule / lastError`。 + ## 1. 复现(2026-09-30,HTTP 5 次、浏览器 5 次,间隔 ≥15s) | 请求 | 结果 | | --- | --- | diff --git a/docs/plan/00-first-version-plan.md b/docs/plan/00-first-version-plan.md index 39c2b153..13e1beab 100644 --- a/docs/plan/00-first-version-plan.md +++ b/docs/plan/00-first-version-plan.md @@ -212,3 +212,5 @@ NFO / 图片投影、身份巡检与挂载根因聚合、HTTP 失败自动切浏 - **健康检查误判(HC-1,用户 2026-09-30 接观众站)**:"测试"登录成功但健康检测报"未登录"——健康检查抓 index.php 找 logout 链接,与用户数据解析两套判定打架;改为以用户数据解析成功为准并细化失败原因。 - **观众详情页适配核对(DET-2)**:用真实详情页(≤3 次请求)逐项核对并修,脱敏样本进差分测试。 - **Cloudflare 验证处理(CF-1,用户 2026-09-30)**:搜索入口有时随机出 CF 验证。先用浏览器复现并研究:HTTP 端遇 CF 挑战时的处理策略(浏览器过挑战后同步 cf_clearance+UA 重试 / 该请求改由浏览器执行并解析 / 站点级"走浏览器"标记与时效),对照 MP(FlareSolverr / CloakBrowser)与 BR-4 的固定身份,出方案后实现。 + **CF-2 已实现(分支 cf2,待合并)**:出口统一(站点请求明确直连 / 站点代理)、挑战不算凭据错误、浏览器过挑战级联(自动 / 视觉子流程 / 转人工)、 + cf_clearance 有条件给 HTTP 重试、浏览器代抓与 6 小时浏览器代理期、站点页展示;见 `docs/runbook/sites.md`「Cloudflare(CF-2)」。 diff --git a/docs/runbook/agent.md b/docs/runbook/agent.md index e017cae1..452e6016 100644 --- a/docs/runbook/agent.md +++ b/docs/runbook/agent.md @@ -13,7 +13,7 @@ | 查询(read) | `catalog.lookup`、`release.search`、`release.inspect`、`inventory.query`、`subscription.list`、`decision.list`、`download.list`、`download.inspect`、`downloader.list`、`task.inspect`、`site.list`、`site.user_data` | | 计划(preview / propose) | `identity.resolve`、`download.plan`、`organize.plan`、`policy.preview` | | 执行(execute) | `download.submit`、`download.recheck`、`organize.execute`、`task.retry`、`task.cancel`、`policy.apply`、`site.health_check`、`site.refresh_login` | -| 原站浏览器 | `browser.snapshot`、`browser.navigate`、`browser.interact`、`browser.download`、`browser.screenshot`、`browser.click_at`、`browser.type_at`、`browser.scroll`、`browser.wait`、`browser.solve_challenge`(见 [browser.md](browser.md)) | +| 原站浏览器 | `browser.snapshot`、`browser.navigate`、`browser.interact`、`browser.download`、`browser.screenshot`、`browser.click_at`、`browser.type_at`、`browser.scroll`、`browser.wait`、`browser.solve_challenge`(见 [browser.md](browser.md));站点请求遇 Cloudflare 复选框时系统也会发起 `solve_challenge` 视觉子流程(发起人 `system:cloudflare`、trigger event、没有父会话,预算取设置,见 browser.md §13) | | 会话 | `agent.plan`、`agent.propose_options`、`agent.spawn`、`agent.await` | | 网页搜索 | `web.search`、`web.fetch`(见 [websearch.md](websearch.md)) | | 元数据补全 | `metadata.enrich`(propose,只出建议;网页搜索开启才提供,见 [identity.md](identity.md#元数据补全meta-1)) | diff --git a/docs/runbook/browser.md b/docs/runbook/browser.md index 67d080b6..7f5ed003 100644 --- a/docs/runbook/browser.md +++ b/docs/runbook/browser.md @@ -126,8 +126,9 @@ execute 类需要持续授权(`automation_grant`)或审批。元素引用只来 不支持时截图不发送,能力改给已脱敏的文字快照并提示;`solve_challenge` 直接判定需人工。fake 上游按支持图片处理。 - **坐标系**:服务端对每次截图读页面视口与 DPR(`evaluate`),`页面 CSS 像素 = 截图像素 × 视口宽 / 截图宽`; 输出的 `coordinates` 记录截图尺寸、原始尺寸、视口、DPR(取不到时 `assumed = true`,按原图、DPR 1)。动作之后旧坐标系作废, - 必须用新截图的坐标。点击沿用 1.17.0 的做法:`elementFromPoint` 打标记再按选择器点,**画布(canvas)内的位置、跨域 iframe - 里的控件(如 Cloudflare Turnstile)点不到具体位置**,这类挑战实际只能转人工。 + 必须用新截图的坐标。点击(CF-2 起)先走 umt 插件的 `POST /umt/tabs/{id}/mouse-click`——Playwright `page.mouse.click` 发真实鼠标事件, + **跨域 iframe 里的控件(Cloudflare Turnstile)和画布也点得到**;插件没有这个端点(旧镜像、纯截图镜像)时退回 1.17.0 的做法: + `elementFromPoint` 打标记再按选择器点(跨域 iframe / 画布点不到具体位置)。 - **动作前提**:会话在自动化控制下(人工接管 / 等待处理中一律拒绝),当前页面主机在站点域名白名单内。 - **敏感截图**:登录页地址、页面上有密码框、地址带凭据 → `sensitive = true`,默认不交给模型(不做像素遮蔽),改给已脱敏的文字快照; 要交给模型需要 `automation_grant.action = browser.sensitive_screenshot`(或 `*`)的持续授权,且调用时传 `allowSensitive: true`。 @@ -368,6 +369,7 @@ camofox 1.17.0 只有 `POST /sessions/:userId/cookies`(导入),而且要另一 | `GET /umt/instance` | `{instanceId, startedAt}`:插件加载时生成,camofox 进程重启即变(BR-3 重启检测,§10) | | `GET /umt/identity` | 固定浏览器身份的公开部分(UA / OS / 版本,BR-4,§11.1) | | `PUT /umt/sessions/{userId}/proxy` `{proxy}` | 该 profile 的出口:`{server, username?, password?}` 或 `null` 直连(BR-4,§11.5) | +| `POST /umt/tabs/{tabId}/mouse-click` `{userId, x, y}` | 按视口坐标发真实鼠标事件(`page.mouse.click`,CF-2,§13);标签页不在回 422 | 适配器先走插件;插件不在(404)再试上游导入端点(`CAMOFOX_API_KEY` 与访问凭据相同时可用,只能写不能读);都不行 → 会话照常打开, `credential.status = unsupported` 并说明。profile 落盘在 `/root/.camofox/profiles//storage-state.json` @@ -550,7 +552,7 @@ Windows 的真值);TLS / HTTP2 指纹是 Firefox 152 本身(与 OS 无关)。屏 - Cookie 账号如果一直不开原站视图、也不换 Cookie,会一直待切换(界面写"待切换:下次刷新登录,或在原站视图里用过后改为跟随浏览器")。 刷新登录对 Cookie 账号只校验,不切换。 -### 11.4 Cloudflare Cookie 仅浏览器 +### 11.4 Cloudflare Cookie 仅浏览器(CF-2 起有条件给 HTTP) `cf_clearance`、`__cf_bm`(及挑战过程中的 `cf_chl_*`)与取得它的浏览器绑定(UA + IP + TLS 指纹),Deno 的 TLS 指纹与 Firefox 必然不同, HTTP 端拿去用多半无效还可能触发风控。所以: @@ -559,6 +561,9 @@ HTTP 端拿去用多半无效还可能触发风控。所以: - 比较"有没有变化"时也不算它们(CF Cookie 轮换不会触发回写);refresh-login 从浏览器读回的 Cookie 同样剔除; - 站点账号页技术信息里显示"仅浏览器使用的 Cookie"; - 用户手填的 Cookie 里带 `cf_clearance` 不处理(用户自己的选择)。 +- **CF-2 例外**:站点遇 Cloudflare 挑战、浏览器过了之后,账号 UA 来源 = 跟随浏览器、浏览器 UA 与 HTTP UA 相同、两端出口相同时, + `cf_clearance`(只有它,`__cf_bm` 仍不带)单独存进 `site_account.user_info.cloudflare`(值在 secret)并由 HTTP 带上重试 1 次, + 结果按站点记 `cfReuse`;普通回写仍剔除它(见 sites.md「Cloudflare(CF-2)」)。 ### 11.5 出口 IP 按站点统一 @@ -642,3 +647,30 @@ camofox 按 **API 调用**判断活动,有三道空闲回收(`docker-compose.yml 核验:`apps/server/src/modules/browser/__tests__/browser-keepalive.test.ts`(登记 / 引用计数 / 复查;对 mock camofox:接管期间保活、 不记自动化活动、交还后停;桌面连接期间标签页被回收 → 保活时自愈重连;智能体会话结束后停、会话关闭撤销)。真机未核验。 + +## 13. Cloudflare 挑战(CF-2) + +站点 HTTP 遇挑战时由站点请求一侧(sources-identity `CloudflareGate`)经钩子调用浏览器模块,流程与状态见 sites.md「Cloudflare(CF-2)」。 +浏览器这边: + +- `solveSiteChallenge({siteId, accountId, url})`:该账号 profile 的自动化会话(没有就按站点打开;人工接管中 / 等人处理中不打扰)导航到 + 同一地址(取站点令牌)→ 用 `POST /tabs/{id}/evaluate` 读页面(地址、标题、outerHTML、正文样本、`navigator.userAgent`,适配器 + `pageContent`)按 `classifyChallengePage` 分类:自动(JS 检查)/ 可点(Turnstile、「确认您是真人」)/ 真验证码(hCaptcha、reCAPTCHA、 + 选图、输码)→ ① 每 1.5 秒读一次,最多 15 秒 → ② 可点的派视觉子流程(`browser.solve_challenge`,系统发起:发起人 `system:cloudflare`、 + trigger event、没有父会话;步数 4、时长 ≤ 120 秒、token / 费用取 `agent.maxTokensPerRun` / `maxCostUsdPerRun`;最多 2 次; + 自动化总开关关闭或只读对账时不派)→ ③ 没过:会话转等待 `cf_challenge`,待处理事项「<站点>需要你在原站视图里过一次验证」 + (视觉子流程失败留下的 `challenge_unsolved` 事项改成这个说法)。通过后读回 profile 里的 `cf_clearance`、UA 与会话出口(`recovery_state.egress`, + 插件按站点下发出口成功才有值)。 +- `fetchSiteHtml({siteId, accountId, url})`:同一会话导航(取站点令牌)后读 outerHTML;仍是挑战页 / 会话在人工处理 / 站点停用时不返回页面。 +- 同一 profile 的过挑战与代抓在进程内串行(profile 锁);跨进程同站 30 分钟一次靠站点状态的条件更新。 +- 交还会话(给自动化)时:站点在等人过验证 → 读回 `cf_clearance` / UA 通知站点请求继续(`CloudflareGate.afterHuman`),待处理事项随交还关闭。 +- 插件新端点 `POST /umt/tabs/{tabId}/mouse-click`:`click_at`(与原站视图截图模式的坐标点击)都先走它。**需要重建 camofox 镜像**才有 + (`docker compose build camofox && docker compose up -d camofox`,只重跑插件层);没重建时退回 elementFromPoint,Turnstile 点不到, + 可点类挑战会转人工。 + +核验:mock camofox(`cloudflare` 模式:auto / interactive / captcha)见 `browser-cloudflare.test.ts`、`umt-plugin.test.ts`、E14。 +插件鼠标点击的真实容器核验见下表(本地临时容器,只开本地页面,不访问真实验证页)。 + +| 日期 | 做法 | 结果 | +| --- | --- | --- | +| 2026-09-30 | 临时容器 `umt-cf2-camofox`(镜像 `umt-browser-desktop:1.17.0`,只读挂载新插件目录),容器内 node 起两个本地页面:父页 `localhost:8001`(标题 Just a moment...)内嵌**跨域** iframe `localhost:8002`(复选框,`change` 事件 `isTrusted` 才向父页 postMessage);适配器 `openSession` → `pageContent` → `click({x, y})` → `pageContent` | 插件路径:`POST /umt/tabs/{id}/mouse-click` 命中复选框,iframe 里收到可信事件,父页标题变为 passed;`pageContent` 读到 UA `…Windows NT 10.0; Win64; x64; rv:152.0…`。旧路径(elementFromPoint)只能标到 iframe 元素本身、按元素中心点击,点不到指定坐标。未访问任何外部站点,容器已删除 | diff --git a/docs/runbook/sites.md b/docs/runbook/sites.md index 98c993ad..448c3696 100644 --- a/docs/runbook/sites.md +++ b/docs/runbook/sites.md @@ -97,7 +97,7 @@ | `defaultHealthCheckIntervalSec` | 21600 | 定时健康检测的最短间隔(与定义取较长) | | `defaultDetailCacheSec` | 86400 | 详情缓存:期内 `GET /sites/{id}/releases/{rid}?refresh=true` 与 `release.inspect` 不再请求站点(与定义取较长);0 = 不缓存 | | `userAgent` | 空(跟随浏览器) | 全局 UA 覆盖:空 = 站点请求跟随浏览器固定身份的 UA(Windows Firefox);填了则账号没手动 UA 时都用它。优先级:账号手动 > 待切换 > 全局 > 跟随浏览器,见 browser.md §11.2 | -| `proxyUrl` | null | 站点 `proxy=true` 时,适配器、登录、用户数据、RSS 请求都经此代理(`Deno.createHttpClient`),浏览器会话也走同一代理(browser.md §11.5);地址要让 app 与 camofox 容器都能连 | +| `proxyUrl` | null | 站点 `proxy=true` 时,适配器、登录、用户数据、RSS 请求都经此代理(`Deno.createHttpClient`),浏览器会话也走同一代理(browser.md §11.5);没开代理的站点**明确直连**,不读容器的 `HTTP(S)_PROXY`(见下「出口」);地址要让 app 与 camofox 容器都能连 | | `userDataMaxSeedingPages` | 10 | 用户数据做种分页上限 | 运行时每 30 s 最多重读一次设置;设置变化会让站点适配器按新签名重建。 @@ -136,9 +136,74 @@ - 注册数:首批能力 17 → 21,启动断言 `assertCapabilityCount` 跟随 `CAPABILITY_IDS.length`。 - 未接:`agent/invoke-limits.ts` 的站点并发上限还没把 `site.health_check` / `site.refresh_login` 算作外部站点调用;Agent 默认只读白名单(`agent/runner.ts`)也还没加 `site.list` / `site.user_data`。两处都在 agent 模块,留给该模块的负责人。 +## 出口:哪些请求走哪条(CF-2) + +代码 `apps/server/src/egress.ts`。之前 app 的站点请求用 `globalThis.fetch`,Deno 会读容器的 `HTTPS_PROXY`(本机栈 override 里的 mihomo 中继), +站点没开代理时 HTTP 端走代理、浏览器却直连,两端出口 IP 不同(CF-1 实测 HKG / DFW),这是 Cloudflare 随机拦截的来源之一。 + +| 请求 | 出口 | +| --- | --- | +| 站点请求:适配器(列表 / 搜索 / 详情 / 下载 / 健康检测)、RSS、账密登录、用户数据(`SourcesRuntime.siteFetch`) | 站点开「使用代理」且设了 `sites.proxyUrl` → 该代理;否则**明确直连** | +| 浏览器会话(camofox) | 同一站点同一出口:插件按 profile 设代理或直连(browser.md §11.5);`UMT_BROWSER_PROXY_MODE=env` 时按容器环境变量(与 HTTP 不一定一致) | +| TMDB、Brave 搜索 / 网页读取、TMDB 图片缓存、海报与站点截图代理 | 照旧按环境变量 `HTTPS_PROXY` / `HTTP_PROXY` / `ALL_PROXY` / `NO_PROXY`(进程启动时读一次),改为代码里显式建客户端(`externalFetch`),行为不变 | +| 模型上游(pi SDK)、下载器、媒体服务器、camofox 本身 | 不在本表范围,仍是各自的 fetch(模型上游随环境变量) | + +**Deno 2.9 的语义**(2026-09-30 本机实测,`test/sources-identity/egress.test.ts` 锁住):缺省 `fetch` 与 `Deno.createHttpClient({})` 都读 +`HTTP_PROXY` / `HTTPS_PROXY` / `ALL_PROXY`(大小写都认)与 `NO_PROXY`,读取发生在**客户端创建时**;`createHttpClient` 没有"不走代理"的选项, +`NO_PROXY=*` 不认;显式 `proxy` 排在环境变量代理前面,但 `NO_PROXY` 命中的主机连显式代理也绕过。所以直连客户端是在创建那一刻临时拿掉 +这几个环境变量(同步调用,同一 isolate 内不会插进别的代码)、建完立即还原;代理客户端同样在拿掉后创建,只走给定代理。 + +排查:站点 HTTP 与浏览器是否同出口,在 app 容器里 `curl` 不能代表 Deno(curl 也读环境变量);看站点页技术信息里的 Cloudflare 记录, +或临时对一个会回显来源 IP 的地址各发一次(不要对 PT 站做)。 + +## Cloudflare(CF-2) + +方案见 `docs/design/cloudflare-handling.md`。代码:`packages/adapters/src/sites/challenge.ts`(识别)、 +`apps/server/src/modules/sources-identity/service/cloudflare.ts`(级联,`CloudflareGate`)、`modules/browser/service.ts` +(`solveSiteChallenge` / `fetchSiteHtml`)、`modules/agent/challenge.ts`(`systemChallengeVision`)。 + +**识别**:站点响应带 `cf-mitigated: challenge`(任何状态码),或 403 / 503 / 429 且正文是挑战页(标题 Just a moment / Attention Required、 +`cf_chl_opt` 等,不认正常页面也会注入的 `/cdn-cgi/challenge-platform/`)→ `SiteError("challenge")`。**不是凭据错误**:快照 / 详情不调 +`onAuthError`、不置账号失效、不排自动重登;健康检测记 `challenge`(degraded);刷新登录 / 用户数据遇挑战只写 `last_error`,账号状态不变 +(验证码、二次验证仍是账号状态 `challenge`)。 + +**级联**(站点请求经 `siteFetch` 自动走,调用方无感): + +1. 站点在浏览器代理期(`config.cloudflare.browserUntil` 未到)且请求是页面(GET + Accept `text/html`)→ 直接由浏览器代抓,不发 HTTP; +2. 发 HTTP;满足复用条件时带上浏览器拿到的 `cf_clearance`(`__cf_bm` 不带); +3. 遇挑战 → 记 `lastChallengeAt` / `lastRule`;只处理**有登录态账号**的站点(第一个启用、未失效、确有 Cookie 的 Cookie / 账密账号); +4. 过挑战:同站 30 分钟最多一次(`lastAttemptAt` 条件更新,跨进程)、进程内同一时刻只跑一次,浏览器用该账号 profile 打开同一地址: + ① 轮询页面等它自己过(≤ 15 秒)→ ② 「我是人类」复选框 / Turnstile:派视觉子流程点一下(系统发起、没有父会话,预算取设置,最多 2 次; + 自动化总开关关闭或只读对账时跳过)→ ③ 选图 / 输码类或仍没过:会话转等待(`cf_challenge`),待处理事项「<站点>需要你在原站视图里过一次验证」, + 站点记 `pending = human`。不接任何打码服务; +5. 通过 → 读回 `cf_clearance` 与 `navigator.userAgent`;**只有账号 UA 来源 = 跟随浏览器、UA 相同、两端出口相同**(浏览器出口按站点下发成功才算) + 时才存进账号(`site_account.user_info.cloudflare = {secretRef, userAgent, egress, at}`,值在 secret 里),HTTP 重试 1 次:过了记 + `cfReuse = ok`,仍被挑战记 `cfReuse = fails`(之后跳过复用;UA 或出口变了自动清掉重试); +6. 仍被挑战 → 页面请求改由浏览器导航取 outerHTML 交给原解析器(受站点限流令牌与 profile 锁约束),站点记 6 小时浏览器代理期,到期再试 HTTP; + 下载 .torrent、RSS、POST 这类不能代抓的请求照常报挑战。 + +人在原站视图里过完验证、交还会话(交还给自动化)→ 读回 `cf_clearance` / UA,清 `pending`,记 `lastSolvedBy = human`,并允许马上再试(清 +`lastAttemptAt`)。 + +`site.config.cloudflare` 字段(契约 `SiteCloudflareState`):`browserUntil`、`cfReuse`(ok / fails)、`lastChallengeAt`、`lastSolvedBy` +(auto / vision / human)、`lastSolvedAt`、`lastAttemptAt`、`pending`(human)、`sessionId`、`lastResult`、`lastRule`、`lastError`。 +改这个键不会让站点适配器重建。 + +**界面**:站点卡片与站点概览「站点」一栏显示一行 Cloudflare 状态(没遇到过挑战不显示):「需要你验证」(卡片上可点,进原站视图)/ +「浏览器代理中，至 hh:mm」/「自动通过」/「没能通过」;原始记录进概览的技术信息。 + +**测试**:`packages/adapters/src/sites/challenge.test.ts`(分类)、`apps/server/test/sources-identity/egress.test.ts`(出口)、 +`cloudflare.test.ts`(级联:自动通过 / 视觉 / 转人工 / 复用失败回退浏览器 / 节流单飞 / 同步条件 / 健康)、 +`modules/browser/__tests__/browser-cloudflare.test.ts`(mock camofox:自动放行、插件真实鼠标点击、选图转人工、交还继续、代抓与 profile 锁)、 +`test/e2e/e14-vision.test.ts`(系统发起的视觉子流程)。真机:未对真实站点验证(CF-2 只用 mock);插件鼠标点击见 browser.md §13。 + +**要人处理时**:站点页 Cloudflare 行显示「需要你验证」→ 原站视图接管 → 过验证 → 交还。一直过不去(站点绑定 TLS 指纹、代理 IP 被拉黑): +关掉站点代理或换出口;实在不行关闭该站点的定时快照,只保留 RSS。 + ## 待核验 1. takelogin 表单字段、二次验证字段名、错误文本全部只对 `@umt/testkit` mock 站点验证过,未连接真实站点。 2. 失败文本归类是关键词匹配(`classifyLoginError`),真实站点的措辞可能落到 `unknown`(状态记为 expired,原文截 200 字写进 `last_error`)。 -3. Cloudflare 站点只识别并转待处理,不做浏览器登录。 +3. Cloudflare:识别与级联见上「Cloudflare(CF-2)」,只对 mock 验证过;真实站点的挑战页类型、cf_clearance 能否给 HTTP 复用(是否绑 TLS) + 按站点记 `cfReuse` 在线验证。账密登录遇挑战仍不做浏览器登录。 4. 馒头用户数据:`MTorrent` 解析器的做种列表首页按 GET 请求,真实 API 可能要求 POST。