diff --git a/apps/server/src/egress.ts b/apps/server/src/egress.ts new file mode 100644 index 00000000..0318a063 --- /dev/null +++ b/apps/server/src/egress.ts @@ -0,0 +1,136 @@ +// CF-2 出口统一:每条对外 HTTP 请求走哪条出口由代码显式决定,不再隐式继承容器的 HTTP(S)_PROXY。 +// +// Deno 2.9 的语义(2026-09-30 本机实测,见 docs/runbook/sites.md「出口」): +// - `fetch` 缺省客户端与 `Deno.createHttpClient({})` 都会读 HTTP_PROXY / HTTPS_PROXY / ALL_PROXY(大小写都认)与 NO_PROXY; +// 读取发生在**客户端创建时**(缺省客户端在进程第一次 fetch 时创建); +// - `createHttpClient` 没有"不走代理"的选项,`NO_PROXY=*` 也不认;显式 `proxy` 会排在环境变量代理前面, +// 但 NO_PROXY 命中的主机连显式代理也绕过; +// - 所以直连客户端的做法是:创建那一刻把这几个环境变量临时拿掉(同步调用,同一个 isolate 内不会被别的代码插进来),建完立即还原。 +// 代理客户端同样在拿掉环境变量后创建,保证只走给定的代理。 +// +// 出口表: +// - 站点请求(适配器、RSS、登录、用户数据):站点开"使用代理"且设了代理地址 → 该代理;否则直连(siteClient); +// - TMDB、Brave、图片缓存等外网请求:照旧用环境变量里的代理(行为不变),但改成显式读 env 建客户端(externalFetch)。 + +const PROXY_ENV = ["HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy"] as const; +const NO_PROXY_ENV = ["NO_PROXY", "no_proxy"] as const; + +/** 进程启动时的代理环境变量快照(外网请求显式使用;之后的临时改动不影响) */ +const ENV_SNAPSHOT: Record = (() => { + const out: Record = {}; + try { + for (const k of [...PROXY_ENV, ...NO_PROXY_ENV]) out[k] = Deno.env.get(k); + } catch { + // 没有 env 权限:当作没有代理 + } + return out; +})(); + +/** 在"没有代理环境变量"的状态下同步执行 fn(建 HttpClient 用),结束立即还原 */ +function withoutProxyEnv(fn: () => T): T { + const saved = new Map(); + try { + for (const k of [...PROXY_ENV, ...NO_PROXY_ENV]) { + const v = Deno.env.get(k); + if (v !== undefined) { + saved.set(k, v); + Deno.env.delete(k); + } + } + } catch { + // 没有 env 权限时也没有可读的代理变量,直接建 + } + try { + return fn(); + } finally { + for (const [k, v] of saved) Deno.env.set(k, v); + } +} + +let direct: Deno.HttpClient | null = null; +const proxied = new Map(); + +/** 明确直连的客户端(不读任何代理环境变量) */ +export function directHttpClient(): Deno.HttpClient { + direct ??= withoutProxyEnv(() => Deno.createHttpClient({})); + return direct; +} + +/** 只走给定代理的客户端(同一地址复用) */ +export function proxyHttpClient(url: string): Deno.HttpClient { + let c = proxied.get(url); + if (!c) { + c = withoutProxyEnv(() => Deno.createHttpClient({ proxy: { url } })); + proxied.set(url, c); + } + return c; +} + +/** 用指定客户端发请求的 fetch */ +export function fetchVia(client: Deno.HttpClient): typeof globalThis.fetch { + return ((input: RequestInfo | URL, init?: RequestInit) => + globalThis.fetch(input, { ...init, client } as RequestInit)) as typeof globalThis.fetch; +} + +/** 站点请求的出口:开代理且有代理地址 → proxy,否则 direct */ +export interface SiteEgress { + mode: "direct" | "proxy"; + proxyUrl: string | null; +} + +export function siteEgress(proxy: boolean, proxyUrl: string | null | undefined): SiteEgress { + return proxy && proxyUrl ? { mode: "proxy", proxyUrl } : { mode: "direct", proxyUrl: null }; +} + +/** 出口的比较键(不含代理口令):direct / proxy:<主机:端口> */ +export function egressKey(e: SiteEgress): string { + if (e.mode === "direct" || !e.proxyUrl) return "direct"; + try { + const u = new URL(e.proxyUrl); + return `proxy:${u.protocol}//${u.host}`; + } catch { + return "proxy:?"; + } +} + +export function siteFetchFor(e: SiteEgress): typeof globalThis.fetch { + return fetchVia(e.mode === "proxy" && e.proxyUrl ? proxyHttpClient(e.proxyUrl) : directHttpClient()); +} + +// ---------- 外网请求(TMDB / Brave / 图片):显式按环境变量 ---------- + +/** NO_PROXY 匹配(与常见实现一致:逗号分隔;`.a.com` / `a.com` 匹配自身与子域;`*` 匹配全部;IP 精确匹配) */ +export function noProxyMatches(host: string, noProxy: string | undefined): boolean { + if (!noProxy) return false; + const h = host.toLowerCase().replace(/^\[|\]$/g, ""); + for (const raw of noProxy.split(",")) { + const p = raw.trim().toLowerCase().replace(/:\d+$/, ""); + if (!p) continue; + if (p === "*") return true; + const d = p.replace(/^\*?\./, ""); + if (h === d || h.endsWith(`.${d}`)) return true; + } + return false; +} + +/** 外网请求按环境变量应走的代理(进程启动时的快照);没有 / NO_PROXY 命中为 null */ +export function envProxyFor(target: URL, env: Record = ENV_SNAPSHOT): string | null { + const get = (k: string) => env[k] ?? env[k.toLowerCase()]; + if (noProxyMatches(target.hostname, get("NO_PROXY"))) return null; + const byScheme = target.protocol === "https:" ? get("HTTPS_PROXY") : get("HTTP_PROXY"); + const p = byScheme || get("ALL_PROXY"); + return p ? p : null; +} + +/** 外网请求用的 fetch:按环境变量显式选代理 / 直连(与之前隐式继承的行为一致) */ +export const externalFetch: typeof globalThis.fetch = ((input: RequestInfo | URL, init?: RequestInit) => { + let target: URL | null = null; + try { + target = new URL(input instanceof Request ? input.url : String(input)); + } catch { + // 交给 fetch 报错 + } + const proxy = target ? envProxyFor(target) : null; + const client = proxy ? proxyHttpClient(proxy) : directHttpClient(); + return globalThis.fetch(input, { ...init, client } as RequestInit); +}) as typeof globalThis.fetch; diff --git a/apps/server/src/modules/agent/challenge.ts b/apps/server/src/modules/agent/challenge.ts index 48a23081..4143e38b 100644 --- a/apps/server/src/modules/agent/challenge.ts +++ b/apps/server/src/modules/agent/challenge.ts @@ -20,7 +20,7 @@ import { } from "@umt/contracts"; import type { z } from "zod"; import { browserServiceFor } from "../browser/index.ts"; -import { BrowserServiceError } from "../browser/service.ts"; +import { BrowserServiceError, type ChallengeVision, registerChallengeVision } from "../browser/service.ts"; import { resolveRunUpstream } from "../pi/index.ts"; import type { ServerContext } from "../types.ts"; import { parseFakeScripts } from "./fake-scripts.ts"; @@ -62,7 +62,41 @@ const FALLBACK_SCRIPT: FakeScript = { const isUuid = (s: string) => /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(s); const clip = (s: string, n: number) => (s.length > n ? `${s.slice(0, n)}…` : s); +/** + * CF-2:系统发起的视觉子流程(站点 HTTP 遇 Cloudflare 复选框 / Turnstile 时由浏览器模块调用)。没有父会话: + * 发起人记 system:cloudflare,子会话 trigger = event;预算取设置(智能体单次 token / 费用上限,时长 ≤ 120 秒)。 + * 自动化总开关由浏览器模块先查过;子流程里的坐标动作照常经注册中心授权(spawn_reason 放行)。 + */ +export function systemChallengeVision(env: () => AgentEnv, registry: CapabilityRegistry): ChallengeVision { + return async (req) => { + const { ctx } = env(); + const s = await ctx.settings.all(); + const input: Input = { + sessionId: req.sessionId, + goal: req.goal, + // 成功与否浏览器模块还会按页面再核实一次;这里给子流程一个可判定的条件 + success: { textAbsent: req.textAbsent ?? "Just a moment" }, + maxSteps: Math.min(12, Math.max(1, req.maxSteps)), + budget: { + maxDurationSec: Math.min(120, s.agent.maxDurationSec), + ...(s.agent.maxTokensPerRun ? { maxTokens: Math.max(1000, s.agent.maxTokensPerRun) } : {}), + ...(s.agent.maxCostUsdPerRun ? { maxCostUsd: s.agent.maxCostUsdPerRun } : {}), + }, + }; + const hctx = { + actorId: "system:cloudflare", + taskId: `cloudflare:${req.sessionId}`, + scopeIds: [], + signal: AbortSignal.timeout(180_000), + source: "task", + } as HandlerContext; + const out = await solveChallenge(env(), registry, input, hctx); + return { status: out.status, reason: out.reason, childRunId: out.childRunId }; + }; +} + export function registerChallengeCapability(registry: CapabilityRegistry, env: () => AgentEnv): void { + registerChallengeVision(env().ctx, systemChallengeVision(env, registry)); registry.register(capabilitySpec("browser.solve_challenge"), async (input, ctx) => { try { return await solveChallenge(env(), registry, input, ctx); @@ -249,7 +283,7 @@ async function runChild( const child = await ctx.db .insertInto("agent_run") .values({ - trigger: parent ? "subagent" : "user", + trigger: parent ? "subagent" : hctx.actorId.startsWith("system:") ? "event" : "user", goal, status: "running", scope: JSON.stringify({ capabilities: [...VISION_CAPABILITIES], allowedKinds: CHILD_KINDS }), diff --git a/apps/server/src/modules/browser/__tests__/browser-cloudflare.test.ts b/apps/server/src/modules/browser/__tests__/browser-cloudflare.test.ts new file mode 100644 index 00000000..f8ac33ef --- /dev/null +++ b/apps/server/src/modules/browser/__tests__/browser-cloudflare.test.ts @@ -0,0 +1,228 @@ +import { siteCloudflareState } from "@umt/contracts"; +import { MOCK_CAMOFOX_IDENTITY, type MockCamofoxSite } from "@umt/testkit"; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; +import { browserServiceFor } from "../index.ts"; +import { type BrowserService, registerChallengeVision } from "../service.ts"; +import { type BrowserHarness, createBrowserHarness, type Loose, ORIGIN, pgAvailable } from "./harness.ts"; + +// CF-2 浏览器一侧(对 testkit mock camofox 的 Cloudflare 模式): +// ① 自动放行(轮询页面)② 复选框 / Turnstile 派视觉子流程,用插件真实鼠标点击 ③ 选图验证码 / 点不过 → 会话转等待 + 待处理事项; +// 人在原站视图里过完交还 → 读回 cf_clearance / UA 通知站点请求继续;浏览器代抓页面受 profile 锁与站点限流约束。 + +const up = await pgAvailable(); +const GOOD = "cf2-session-value"; +const CLEARANCE = "clr-from-browser"; +const LIST_HTML = + '种子
Oppenheimer
'; + +describe.skipIf(!up)("CF-2 浏览器过挑战 / 代抓", () => { + let h: BrowserHarness; + let svc: BrowserService; + const cfCfg: NonNullable = { mode: "auto", autoPolls: 2, clearance: CLEARANCE }; + const visionCalls: string[] = []; + let visionClick = true; + const url = `${ORIGIN}/torrents.php`; + + const sessions = async () => (await h.req("/browser/sessions")).body.data.items as Loose[]; + const closeAll = async () => { + for (const s of await sessions()) await h.req(`/browser/sessions/${s.id}`, { method: "DELETE" }); + }; + const profileKey = () => `umt-site-${h.siteId}-acct-${h.accountId}`; + + beforeAll(async () => { + h = await createBrowserHarness("cf2", { + sessionCookie: { name: "c_secure_pass", valid: [GOOD], fresh: GOOD }, + cloudflare: cfCfg, + html: (u) => + u.includes("torrents.php") ? LIST_HTML : "首页", + browserOptions: { challengePollMs: 5, challengeAutoWaitMs: 200, settleMs: 0 }, + }); + await sessions(); // 第一次请求时浏览器模块才为这个上下文建服务 + svc = browserServiceFor(h.ctx)!; + const ref = await h.secrets.create({ purpose: "site_cookie", plaintext: `c_secure_uid=1; c_secure_pass=${GOOD}` }); + await h.db + .updateTable("site_account") + .set({ auth_kind: "cookie", secret_ref: ref, session_secret_ref: null, status: "active" }) + .where("id", "=", h.accountId) + .execute(); + // 视觉子流程(假):点一下复选框所在的位置(第 2 行) + registerChallengeVision(h.ctx, async (req) => { + visionCalls.push(req.sessionId); + if (visionClick) + await svc.visionAct( + req.sessionId, + { kind: "automation", holder: "system:cloudflare" }, + (p, ref) => p.click(ref, { x: 100, y: 70 }), + { wantImage: false, sensitiveAllowed: false }, + ); + return { status: visionClick ? "solved" : "needs_human", reason: "", childRunId: null }; + }); + }, 120_000); + afterAll(async () => { + await h?.close(); + }); + beforeEach(async () => { + await closeAll(); + h.mock.cfPassed.clear(); + h.mock.contentReads.clear(); + h.mock.actions.length = 0; + for (const jar of h.mock.cookieJars.values()) + jar.splice(0, jar.length, ...jar.filter((c) => c.name !== "cf_clearance")); + cfCfg.mode = "auto"; + visionCalls.length = 0; + visionClick = true; + await h.db + .updateTable("site") + .set({ config: JSON.stringify({}), enabled: true }) + .where("id", "=", h.siteId) + .execute(); + await h.db + .updateTable("settings") + .set({ value: JSON.stringify({ enabled: true }) }) + .where("key", "=", "automation") + .execute(); + }); + + it("① 自动放行:轮询到挑战页消失,读回 cf_clearance + navigator.userAgent + 出口", async () => { + const r = await svc.solveSiteChallenge({ siteId: h.siteId, accountId: h.accountId, url }); + expect(r).toEqual({ + status: "passed", + by: "auto", + clearance: CLEARANCE, + userAgent: MOCK_CAMOFOX_IDENTITY.userAgent, + browserEgress: "direct", + }); + expect(visionCalls).toEqual([]); + // 会话仍归自动化,没有待处理事项 + const [s] = await sessions(); + expect(s.controller).toBe("automation"); + }); + + it("② 复选框 / Turnstile:等不过 → 视觉子流程用插件真实鼠标点击 → 通过(by vision)", async () => { + cfCfg.mode = "interactive"; + const r = await svc.solveSiteChallenge({ siteId: h.siteId, accountId: h.accountId, url }); + expect(r).toMatchObject({ status: "passed", by: "vision", clearance: CLEARANCE }); + expect(visionCalls).toHaveLength(1); + expect(h.mock.actions.some((a) => a.action === "mouse-click")).toBe(true); + }); + + it("③ 视觉子流程点不过(最多 2 次)→ 会话转等待,待处理事项「观众需要你在原站视图里过一次验证」", async () => { + cfCfg.mode = "interactive"; + visionClick = false; + const r = await svc.solveSiteChallenge({ siteId: h.siteId, accountId: h.accountId, url }); + expect(r).toMatchObject({ status: "needs_human" }); + expect(visionCalls).toHaveLength(2); + const s = (await h.req(`/browser/sessions/${(r as Loose).sessionId}`)).body.data as Loose; + expect(s.controller).toBe("waiting"); + expect(s.waitingReason).toBe("cf_challenge"); + const d = await h.db + .selectFrom("decision") + .selectAll() + .where("id", "=", (r as Loose).decisionId) + .executeTakeFirstOrThrow(); + expect(d.title).toBe("观众需要你在原站视图里过一次验证"); + expect(d.status).toBe("awaiting_user"); + expect((d.evidence as Loose).cloudflare).toMatchObject({ siteId: h.siteId }); + // 已在等人:再来一次不打扰,直接返回同一个等待 + const again = await svc.solveSiteChallenge({ siteId: h.siteId, accountId: h.accountId, url }); + expect(again).toMatchObject({ status: "needs_human", sessionId: (r as Loose).sessionId }); + expect(visionCalls).toHaveLength(2); + }); + + it("自动化总开关关闭:不派视觉子流程,直接转人工", async () => { + cfCfg.mode = "interactive"; + await h.db + .insertInto("settings") + .values({ key: "automation", value: JSON.stringify({ enabled: false }) }) + .onConflict((oc) => oc.column("key").doUpdateSet({ value: JSON.stringify({ enabled: false }) })) + .execute(); + const r = await svc.solveSiteChallenge({ siteId: h.siteId, accountId: h.accountId, url }); + expect(r).toMatchObject({ status: "needs_human" }); + expect(visionCalls).toEqual([]); + }); + + it("③ 选图 / 输码类:不点、不等,直接转人工;人过完交还 → 读回 cf_clearance,站点请求继续", async () => { + cfCfg.mode = "captcha"; + const r = (await svc.solveSiteChallenge({ siteId: h.siteId, accountId: h.accountId, url })) as Loose; + expect(r.status).toBe("needs_human"); + expect(r.reason).toContain("选图"); + expect(visionCalls).toEqual([]); + // 站点请求一侧记下"等人处理"(级联在 sources-identity 里做,这里直接写状态) + await h.db + .updateTable("site") + .set({ config: JSON.stringify({ cloudflare: { pending: "human", sessionId: r.sessionId } }) }) + .where("id", "=", h.siteId) + .execute(); + // 人接管,在桌面里过了验证(桌面操作不经 API:直接改 mock 状态),交还 + expect( + (await h.req(`/browser/sessions/${r.sessionId}/control/acquire`, { body: { reason: "过验证" } })).status, + ).toBe(200); + h.mock.cfPassed.add(profileKey()); + h.mock.cookieJars.get(profileKey())!.push({ + name: "cf_clearance", + value: "clr-human", + domain: ".audiences.example", + path: "/", + expires: Math.floor(Date.now() / 1000) + 3600, + httpOnly: true, + secure: true, + }); + expect((await h.req(`/browser/sessions/${r.sessionId}/control/release`, { body: {} })).status).toBe(200); + const site = await h.db.selectFrom("site").select("config").where("id", "=", h.siteId).executeTakeFirstOrThrow(); + expect(siteCloudflareState(site.config)).toMatchObject({ pending: null, lastSolvedBy: "human" }); + const acc = await h.db + .selectFrom("site_account") + .select("user_info") + .where("id", "=", h.accountId) + .executeTakeFirstOrThrow(); + const info = (acc.user_info as Loose).cloudflare; + expect(info).toMatchObject({ userAgent: MOCK_CAMOFOX_IDENTITY.userAgent, egress: "direct" }); + expect(h.secrets.plain.get(info.secretRef)).toBe("clr-human"); + // 待处理事项已关闭 + const d = await h.db + .selectFrom("decision") + .select("status") + .where("id", "=", r.decisionId) + .executeTakeFirstOrThrow(); + expect(d.status).toBe("resolved"); + }); + + it("浏览器代抓:过了挑战的 profile 导航取 outerHTML;同一 profile 并发串行(只开一个会话);每次导航取站点令牌", async () => { + h.mock.cfPassed.add(profileKey()); + const [a, b] = await Promise.all([ + svc.fetchSiteHtml({ siteId: h.siteId, accountId: h.accountId, url }), + svc.fetchSiteHtml({ siteId: h.siteId, accountId: h.accountId, url }), + ]); + expect(a).toMatchObject({ status: "ok", html: LIST_HTML }); + expect(b).toMatchObject({ status: "ok", html: LIST_HTML }); + expect(await sessions()).toHaveLength(1); + const navs = h.mock.calls.filter( + (c) => c.path.endsWith("/navigate") || (c.path === "/tabs" && c.method === "POST"), + ); + expect(navs.length).toBeGreaterThanOrEqual(2); + const count = await h.db + .selectFrom("site_request_count") + .select((eb) => eb.fn.sum("count").as("n")) + .where("site_id", "=", h.siteId) + .executeTakeFirst(); + expect(Number(count?.n ?? 0)).toBeGreaterThanOrEqual(2); + }); + + it("浏览器代抓:仍是挑战页 → challenge;会话在人工接管中 → 不打扰;站点停用 → 不可用", async () => { + cfCfg.mode = "captcha"; + expect(await svc.fetchSiteHtml({ siteId: h.siteId, accountId: h.accountId, url })).toMatchObject({ + status: "challenge", + }); + const [s] = await sessions(); + await h.req(`/browser/sessions/${s.id}/control/acquire`, { body: { reason: "看看" } }); + expect(await svc.fetchSiteHtml({ siteId: h.siteId, accountId: h.accountId, url })).toMatchObject({ + status: "unavailable", + }); + await h.req(`/browser/sessions/${s.id}/control/release`, { body: {} }); + await h.db.updateTable("site").set({ enabled: false }).where("id", "=", h.siteId).execute(); + expect(await svc.fetchSiteHtml({ siteId: h.siteId, accountId: h.accountId, url })).toMatchObject({ + status: "unavailable", + reason: "站点已停用", + }); + }); +}); diff --git a/apps/server/src/modules/browser/__tests__/harness.ts b/apps/server/src/modules/browser/__tests__/harness.ts index 6d265c53..44396451 100644 --- a/apps/server/src/modules/browser/__tests__/harness.ts +++ b/apps/server/src/modules/browser/__tests__/harness.ts @@ -3,7 +3,13 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import type { CapabilityRegistry, InvokeContext } from "@umt/agent"; import { type DB, type Kysely, migrateToLatest, sql } from "@umt/db"; -import { buildFakeTorrent, createMockCamofox, type FakeTorrent, type MockCamofox } from "@umt/testkit"; +import { + buildFakeTorrent, + createMockCamofox, + type FakeTorrent, + type MockCamofox, + type MockCamofoxSite, +} from "@umt/testkit"; import { Kysely as K, PostgresDialect } from "kysely"; import pg from "pg"; import { withSchemaLock } from "../../../../test/auth-settings/harness.ts"; @@ -19,6 +25,7 @@ import { createSourcesIdentityModule } from "../../sources-identity/index.ts"; import type { ServerContext } from "../../types.ts"; import { registerBrowserCapabilities } from "../capabilities.ts"; import { createBrowserModule } from "../index.ts"; +import type { BrowserModuleOptions } from "../service.ts"; // C1-E 浏览器模块测试装配:独立 PG schema + 真实迁移 + 交互式 mock camofox(观众风格站点)+ 真实能力注册中心。 @@ -61,6 +68,11 @@ export async function createBrowserHarness( sessionCookie?: { name: string; valid: string[]; fresh: string }; cookieApi?: "plugin" | "upstream" | "none"; cookieImportKey?: string; + /** CF-2:mock 站点的 Cloudflare 挑战与页面 HTML */ + cloudflare?: MockCamofoxSite["cloudflare"]; + html?: MockCamofoxSite["html"]; + /** 浏览器模块的额外选项(挑战轮询间隔等) */ + browserOptions?: Partial; } = {}, ): Promise { const torrent = buildFakeTorrent({ name: "Oppenheimer.2023.2160p.UHD.BluRay.x265-GRP" }); @@ -70,6 +82,8 @@ export async function createBrowserHarness( ...(opts.cookieImportKey ? { cookieImportKey: opts.cookieImportKey } : {}), site: { ...(opts.sessionCookie ? { sessionCookie: opts.sessionCookie } : {}), + ...(opts.cloudflare ? { cloudflare: opts.cloudflare } : {}), + ...(opts.html ? { html: opts.html } : {}), origin: ORIGIN, username: "fixture-user", password: PASSWORD, @@ -110,7 +124,12 @@ export async function createBrowserHarness( capabilities: registry, } as ServerContext; const sources = createSourcesIdentityModule({ tmdb: () => null }); - const browser = createBrowserModule({ runtime: (c) => sources.runtime(c), stagingDir, downloadPollMs: 20 }); + const browser = createBrowserModule({ + runtime: (c) => sources.runtime(c), + stagingDir, + downloadPollMs: 20, + ...opts.browserOptions, + }); registerBrowserCapabilities(registry, () => browser.service(ctx)); const app = createApp({ config: { NODE_ENV: "development", DEV_ADMIN_KEY: DEV_KEY }, diff --git a/apps/server/src/modules/browser/service.ts b/apps/server/src/modules/browser/service.ts index bc5b6ac4..ac902b3d 100644 --- a/apps/server/src/modules/browser/service.ts +++ b/apps/server/src/modules/browser/service.ts @@ -5,6 +5,7 @@ import { type BrowserProvider, type BrowserSecretResolver, CAMOFOX_VERIFIED_RELEASES, + classifyChallengePage, cookiesFromHeader, createCamofoxBrowserProvider, type DownloadSink, @@ -12,7 +13,9 @@ import { getSiteDefinition, headerFromCookies, hostAllowed, + InMemoryProfileLock, imageDimensions, + type PageContent, type PageMetrics, parseTorrent, prepareImage, @@ -50,8 +53,14 @@ import { type SiteRow, sql, } from "@umt/db"; +import { egressKey, siteEgress } from "../../egress.ts"; import type { SourcesRuntime } from "../sources-identity/runtime.ts"; import { refreshSiteAccount } from "../sources-identity/service/account.ts"; +import { + type BrowserFetchOutcome, + type ChallengeSolveOutcome, + registerChallengeBrowser, +} from "../sources-identity/service/cloudflare.ts"; import { readAccountCookie, registerBrowserCookieReader, @@ -85,6 +94,10 @@ export interface BrowserModuleOptions { evidenceDir?: string; /** 坐标动作之后等页面稳定再截图的毫秒数;缺省 400 */ settleMs?: number; + /** CF-2 等挑战页放行时读页面的间隔(毫秒);缺省 1500 */ + challengePollMs?: number; + /** CF-2 等 Cloudflare 自己放行的最长时间(毫秒);缺省 CF_AUTO_WAIT_MS(15 秒) */ + challengeAutoWaitMs?: number; /** 保活间隔(AG-4);缺省 BROWSER_KEEPALIVE_MS 配置(120 秒),0 = 关闭 */ keepaliveMs?: number; /** 测试替换 provider 实现 */ @@ -175,8 +188,39 @@ export interface RecoveryState { reconnect?: BrowserReconnectState; /** 最近一次自动化(Agent 能力)动作成功的时间;原站视图据此区分「Agent 正在操作」与「无人操作」 */ lastAutomationAt?: string; + /** CF-2:打开 / 重连时下发给浏览器的出口键(egressKey);插件没按站点设出口(旧镜像 / env 模式)为 null */ + egress?: string | null; } +/** + * CF-2 视觉子流程钩子(agent 模块注册):系统发起、没有父会话,预算取设置;只点「我是人类」复选框 / Turnstile 一步。 + * 返回子流程结论(solved 仍要由调用方按页面核实)。 + */ +export type ChallengeVision = (req: { + sessionId: string; + goal: string; + /** 通过后页面上不再出现的文字(成功条件) */ + textAbsent: string | null; + maxSteps: number; +}) => Promise<{ status: string; reason: string; childRunId: string | null }>; + +const visions = new WeakMap(); + +export function registerChallengeVision(ctx: object, fn: ChallengeVision): void { + visions.set(ctx, fn); +} + +/** 系统(Cloudflare 处理)发起的浏览器动作 */ +const CF_ACTOR: Actor = { kind: "automation", holder: "system:cloudflare" }; +/** 等 Cloudflare 自己放行的最长时间 */ +export const CF_AUTO_WAIT_MS = 15_000; +/** 视觉子流程点过之后再等多久 */ +const CF_AFTER_CLICK_WAIT_MS = 10_000; +/** 视觉子流程最多派几次 */ +const CF_VISION_ATTEMPTS = 2; +/** 视觉子流程步数上限:截图 → 点击 → 等待 */ +const CF_VISION_STEPS = 4; + /** 同一浏览器服务的实例标识多久查一次 */ const INSTANCE_CHECK_MS = 5_000; /** 自动重连失败后,操作触发的重连至少间隔多久 */ @@ -221,6 +265,11 @@ export class BrowserService { registerBrowserCookieReader(ctx, (siteId, accountId) => this.readProfileCookie(siteId, accountId)); // 站点请求 UA 跟随浏览器固定身份(BR-4):sources-identity 经钩子取 registerBrowserIdentityReader(ctx, () => this.identityUserAgent()); + // CF-2:站点 HTTP 遇 Cloudflare 挑战时由浏览器过挑战 / 代抓页面 + registerChallengeBrowser(ctx, { + solve: (req) => this.solveSiteChallenge(req), + fetchHtml: (req) => this.fetchSiteHtml(req), + }); } // ---------- BR-4 浏览器身份与出口 ---------- @@ -283,8 +332,8 @@ export class BrowserService { * 出口按站点统一(BR-4):站点开了代理且设置了代理地址 → 该 profile 走同一代理;否则直连。 * 旧镜像(插件没有出口接口)只记日志,浏览器照旧走容器环境变量里的代理。 */ - private async applyEgress(provider: BrowserProvider, site: SiteRow, profileKey: string): Promise { - if (!provider.setEgress) return; + private async applyEgress(provider: BrowserProvider, site: SiteRow, profileKey: string): Promise { + if (!provider.setEgress) return null; const rt = this.opts.runtime?.(this.ctx); const proxyUrl = rt ? (await rt.siteDefaults()).proxyUrl @@ -297,6 +346,8 @@ export class BrowserService { { component: "browser", siteId: site.id, egress: proxy ? "proxy" : "direct" }, "浏览器会话出口已变更,按新出口重建", ); + // 插件 env 模式:浏览器按容器环境变量出网,与站点 HTTP(明确直连 / 站点代理)不一定一致 + return r.mode && r.mode !== "site" ? null : egressKey(siteEgress(site.proxy, proxyUrl)); } catch (e) { this.ctx.logger.warn( { @@ -307,6 +358,7 @@ export class BrowserService { }, "设置浏览器会话出口失败,浏览器按容器环境变量出网", ); + return null; } } @@ -851,7 +903,7 @@ export class BrowserService { for (const m of await provider.listSessions()) if (m.profileKey === profileKey && m.controller !== "closed") provider.setController(m.id, "closed"); const prep = await this.prepareCookies(site, accountId); - await this.applyEgress(provider, site, profileKey); + const egress = await this.applyEgress(provider, site, profileKey); await this.refreshIdentity(prow, { maxAgeMs: 10 * 60_000 }).catch(() => null); await this.rateLimit(site); let res: Awaited>; @@ -904,6 +956,7 @@ export class BrowserService { needsSnapshot: true, credential, instanceId, + egress, }), }) .execute(); @@ -1141,7 +1194,11 @@ export class BrowserService { } /** 转等待处理:自动化动作被拒,生成待处理事项,等人在原站视图接管 */ - async enterWaiting(row: BrowserSessionRow, reason: string): Promise { + async enterWaiting( + row: BrowserSessionRow, + reason: string, + text?: { title: string; summary: string }, + ): Promise { const provider = this.providers.get(row.provider_id)?.provider; if (provider?.sessionInfo(row.id)) provider.setController(row.id, "waiting"); const site = await this.liveSite(row.site_id).catch(() => null); @@ -1154,8 +1211,8 @@ export class BrowserService { kind: "other", status: "awaiting_user", severity: "warning", - title: `${site?.name ?? "站点"}:原站需要人工处理`, - summary: `${REASON_TEXT[reason] ?? reason}。在站点详情的原站视图接管会话,完成后交还。`, + title: text?.title ?? `${site?.name ?? "站点"}:原站需要人工处理`, + summary: text?.summary ?? `${REASON_TEXT[reason] ?? reason}。在站点详情的原站视图接管会话,完成后交还。`, subject_type: "browser_session", subject_id: row.id, dedupe_key: `browser.waiting:${row.id}`, @@ -1327,6 +1384,14 @@ export class BrowserService { await this.persist(id, provider); // 人工在桌面 / 截图模式里登录过:交还时读回登录态写进站点账号,全站(HTTP 抓取 / RSS / 快照)生效 await this.syncCookies(id, "release"); + // CF-2:等人过 Cloudflare 验证的会话交还后,读回 cf_clearance / UA,站点请求继续 + if (opts.to === "automation") + await this.afterHumanChallenge(id).catch((e) => + this.ctx.logger.warn( + { component: "browser", sessionId: id, err: String((e as Error)?.message ?? e) }, + "交还后读回 Cloudflare 验证结果失败", + ), + ); return toLease(released); } @@ -1586,7 +1651,7 @@ export class BrowserService { if (!provider.reopenSession) throw await fail("浏览器服务不支持重新连接"); const url = restoreUrl(row.current_url, budget) ?? site.base_url; const prep = await this.prepareCookies(site, row.account_id).catch(() => null); - await this.applyEgress(provider, site, row.profile_key); + const egress = await this.applyEgress(provider, site, row.profile_key); // 浏览器重启过(可能换了镜像版本):身份重新读 await this.refreshIdentity(prow).catch(() => null); let res: Awaited>>; @@ -1606,6 +1671,7 @@ export class BrowserService { needsReconnect: false, needsSnapshot: true, instanceId: instanceId ?? rs.instanceId ?? null, + egress, reconnect: { status: "ok", trigger, at, reason: null, humanNotice: row.controller === "human" }, }); this.coordSpaces.delete(id); @@ -1894,6 +1960,248 @@ export class BrowserService { return { decisionId, controller: row.controller }; } + // ---------- CF-2 Cloudflare 挑战 ---------- + + /** 过挑战 / 代抓同一 profile 串行(本进程;跨进程的同站互斥靠 site.config.cloudflare.lastAttemptAt 条件更新) */ + private readonly cfLock = new InMemoryProfileLock(240_000); + + /** 读当前页面内容(只读,不要求自动化控制权);上游不支持为 null */ + private async readPage(id: string): Promise { + return this.run(id, CF_ACTOR, (p, ref) => (p.pageContent ? p.pageContent(ref) : Promise.resolve(null)), { + write: false, + persist: false, + background: true, + }); + } + + /** 轮询页面直到不是挑战页(或变成真验证码 / 超时);返回最后一次读到的页面与分类 */ + private async waitChallenge( + id: string, + page: PageContent | null, + ms: number, + ): Promise<{ + page: PageContent | null; + kind: ReturnType["kind"]; + rule: string | null; + }> { + let cur = page ?? (await this.readPage(id)); + let c = classifyChallengePage(cur ?? {}); + const deadline = Date.now() + ms; + while (cur && c.kind !== "none" && c.kind !== "captcha" && Date.now() < deadline) { + await new Promise((r) => setTimeout(r, this.opts.challengePollMs ?? 1_500)); + cur = await this.readPage(id); + c = classifyChallengePage(cur ?? {}); + } + return { page: cur, kind: cur ? c.kind : "none", rule: c.rule }; + } + + /** + * 该账号 profile 的自动化会话:已有且在自动化控制下就用它(导航到 url,取站点令牌);没有就打开一个(打开即导航)。 + * 人工接管中 / 等人处理中返回 null(不打扰人)。 + */ + private async cfSession( + site: SiteRow, + accountId: string, + url: string, + ): Promise<{ id: string } | { busy: BrowserSessionRow }> { + const live = await this.liveSessionFor(site.id, accountId); + if (live && live.controller !== "automation") return { busy: live }; + if (!live) return { id: await this.open({ siteId: site.id, accountId, url }) }; + await this.run(live.id, CF_ACTOR, async (p, ref, x) => { + await this.rateLimit(x.site); + await p.navigate(ref, url, x.budget); + }); + return { id: live.id }; + } + + /** 会话转"等你过验证":待处理事项写明是哪个站点、去哪里处理 */ + private async cfNeedsHuman(id: string, site: SiteRow, reason: string): Promise { + const text = { + title: `${site.name}需要你在原站视图里过一次验证`, + summary: + "站点要求人机验证(Cloudflare),系统没能自动通过。在站点详情的原站视图接管会话,完成验证后交还,站点请求会继续。", + }; + let row = await this.sessionRow(id); + if (row.controller === "automation") row = await this.enterWaiting(row, "cf_challenge", text); + else if (row.controller === "waiting") { + const rs = obj(row.recovery_state) as RecoveryState; + if (!rs.waitingDecisionId) row = await this.enterWaiting(row, "cf_challenge", text); + else { + // 视觉子流程没成功时已转等待(challenge_unsolved):事项改成 Cloudflare 的说法 + await this.ctx.db + .updateTable("decision") + .set({ title: text.title, summary: text.summary }) + .where("id", "=", rs.waitingDecisionId) + .execute(); + row = await this.patchRecovery(row, { waitingReason: "cf_challenge" }); + } + } + const decisionId = (obj(row.recovery_state) as RecoveryState).waitingDecisionId ?? null; + if (decisionId) + await this.ctx.db + .updateTable("decision") + .set({ + evidence: sql`evidence || ${JSON.stringify({ cloudflare: { siteId: site.id, reason } })}::jsonb`, + }) + .where("id", "=", decisionId) + .execute(); + this.ctx.logger.info({ component: "browser", sessionId: id, siteId: site.id, reason }, "Cloudflare 验证转人工"); + return { status: "needs_human", sessionId: id, decisionId, reason }; + } + + /** 读回 profile 里的 cf_clearance 值(只取作用于站点主机的) */ + private async cfClearance(profileKey: string, siteUrl: string): Promise { + const prow = await this.providerRow().catch(() => null); + const provider = prow ? this.providerFor(prow) : null; + if (!provider?.getCookies) return null; + const list = await provider.getCookies(profileKey, [siteUrl]).catch(() => []); + const host = new URL(siteUrl).hostname; + const hit = list.find( + (c) => + c.name === "cf_clearance" && + (host === c.domain.replace(/^\./, "") || host.endsWith(`.${c.domain.replace(/^\./, "")}`)) && + !(c.expires !== undefined && c.expires > 0 && c.expires * 1000 <= this.now().getTime()), + ); + return hit?.value ?? null; + } + + /** + * 过挑战(CF-2 级联):用该账号 profile 打开同一地址 → ① 等它自己过(≤15 秒)→ ② 复选框 / Turnstile 派视觉子流程点一下 + * (最多 2 次,系统发起,预算取设置;自动化总开关关闭时跳过)→ ③ 选图 / 输码类或仍没过:会话转等待,开待处理事项。 + * 通过后读回 cf_clearance 与 navigator.userAgent,连同浏览器出口交给调用方。 + */ + async solveSiteChallenge(req: { siteId: string; accountId: string; url: string }): Promise { + const site = await this.liveSite(req.siteId); + if (!site.enabled) return { status: "failed", reason: "站点已停用" }; + const profileKey = profileKeyFor(site.id, req.accountId); + return this.cfLock.run(profileKey, async () => { + const got = await this.cfSession(site, req.accountId, req.url); + if ("busy" in got) { + const rs = obj(got.busy.recovery_state) as RecoveryState; + if (got.busy.controller === "waiting" && rs.waitingReason === "cf_challenge") + return { + status: "needs_human", + sessionId: got.busy.id, + decisionId: rs.waitingDecisionId ?? null, + reason: "已在等你过验证", + }; + return { status: "failed", reason: "浏览器会话正由人工处理,没有打扰" }; + } + const id = got.id; + const opened = await this.sessionRow(id); + if (opened.controller !== "automation") + return { status: "failed", reason: "浏览器会话打开后在等人处理(登录已失效)" }; + // ① 等它自己过 + let w = await this.waitChallenge(id, null, this.opts.challengeAutoWaitMs ?? CF_AUTO_WAIT_MS); + if (!w.page) return { status: "failed", reason: "浏览器服务不支持读取页面内容" }; + let by: "auto" | "vision" = "auto"; + // ② 复选框 / Turnstile:视觉子流程点一下 + if (w.kind === "interactive") { + const vision = visions.get(this.ctx); + const auto = await this.ctx.settings.get("automation").catch(() => null); + if (vision && auto?.enabled && !auto.readOnlyReconcile) { + const marker = /verify you are human|确认您是真人|验证您是真人|我是人类/i.exec(w.page.text)?.[0] ?? null; + for (let i = 0; i < CF_VISION_ATTEMPTS && w.kind === "interactive"; i++) { + const r = await vision({ + sessionId: id, + goal: "这是 Cloudflare 人机验证页。只点一次「确认您是真人 / Verify you are human」复选框(Turnstile),然后等待页面跳转;不要做其它操作。如果页面要求选图片、输入验证码或别的操作,立即停止并说明需要人工。", + textAbsent: marker, + maxSteps: CF_VISION_STEPS, + }).catch((e) => ({ status: "failed", reason: String((e as Error)?.message ?? e), childRunId: null })); + this.ctx.logger.info( + { component: "browser", sessionId: id, siteId: site.id, attempt: i + 1, status: r.status }, + "Cloudflare 复选框视觉子流程结束", + ); + // 子流程没成功时会话已转等待(challenge_unsolved);下一次子流程开始时自己收回自动化控制 + w = await this.waitChallenge( + id, + null, + Math.min(CF_AFTER_CLICK_WAIT_MS, this.opts.challengeAutoWaitMs ?? CF_AFTER_CLICK_WAIT_MS), + ); + if (w.kind === "none") by = "vision"; + } + } + } + // ③ 没过 + if (w.kind !== "none") + return this.cfNeedsHuman( + id, + site, + w.kind === "captcha" ? "选图 / 输入验证码类验证,不自动处理" : `没能自动通过(${w.rule ?? w.kind})`, + ); + // 通过:子流程成功时关掉它留下的待处理;读回 cf_clearance 与 UA + const row = await this.sessionRow(id); + if (row.controller === "waiting") { + await this.resolveWaiting(row, "system:cloudflare"); + const { provider } = await this.loaded(row); + provider.setController(id, "automation"); + await this.ctx.db + .updateTable("browser_session") + .set({ + controller: "automation", + recovery_state: JSON.stringify({ + ...obj(row.recovery_state), + waitingReason: null, + waitingDecisionId: null, + }), + updated_at: this.now(), + }) + .where("id", "=", id) + .execute(); + } + const rs = obj((await this.sessionRow(id)).recovery_state) as RecoveryState; + return { + status: "passed", + by, + clearance: await this.cfClearance(profileKey, site.base_url), + userAgent: w.page?.userAgent ?? (await this.identityUserAgent().catch(() => null)), + browserEgress: rs.egress ?? null, + }; + }); + } + + /** + * 浏览器代抓一页(CF-2 回退):该账号 profile 的自动化会话导航到 url(取站点令牌,受 profile 锁约束),读 outerHTML 交给原解析器。 + * 站点停用、会话正由人工处理 / 等人处理、仍是挑战页时不返回页面。 + */ + async fetchSiteHtml(req: { siteId: string; accountId: string; url: string }): Promise { + const site = await this.liveSite(req.siteId).catch(() => null); + if (!site?.enabled) return { status: "unavailable", reason: "站点已停用" }; + const profileKey = profileKeyFor(site.id, req.accountId); + try { + return await this.cfLock.run(profileKey, async () => { + const got = await this.cfSession(site, req.accountId, req.url); + if ("busy" in got) return { status: "unavailable" as const, reason: "浏览器会话正由人工处理或在等人处理" }; + const w = await this.waitChallenge(got.id, null, this.opts.challengeAutoWaitMs ?? CF_AUTO_WAIT_MS); + if (!w.page) return { status: "unavailable" as const, reason: "浏览器服务不支持读取页面内容" }; + if (w.kind !== "none") return { status: "challenge" as const, reason: w.rule ?? w.kind }; + return { status: "ok" as const, html: w.page.html, url: w.page.url }; + }); + } catch (e) { + const err = fromBrowserError(e); + return { status: "unavailable", reason: String((err as Error)?.message ?? err).slice(0, 300) }; + } + } + + /** 人过完验证交还后:站点在等人过验证、会话是那一个 → 读回 cf_clearance / UA,通知站点请求继续 */ + private async afterHumanChallenge(id: string): Promise { + const row = await this.sessionRow(id); + if (!row.account_id) return; + const rt = this.opts.runtime?.(this.ctx); + if (!rt) return; + const st = await rt.cloudflare.state(row.site_id, true); + if (st.pending !== "human") return; + const site = await this.liveSite(row.site_id); + const page = await this.readPage(id).catch(() => null); + if (page && classifyChallengePage(page).kind !== "none") return; // 还在挑战页:继续等 + const rs = obj(row.recovery_state) as RecoveryState; + await rt.cloudflare.afterHuman(row.site_id, row.account_id, { + clearance: await this.cfClearance(row.profile_key, site.base_url), + userAgent: page?.userAgent ?? (await this.identityUserAgent().catch(() => null)), + browserEgress: rs.egress ?? null, + }); + } + // ---------- 下载 ---------- /** 暂存产物核验:解析种子、按 infohash / 站点资源 ID 匹配本地资源、会话内去重 */ @@ -1972,6 +2280,7 @@ export class BrowserService { } const REASON_TEXT: Record = { + cf_challenge: "站点要求人机验证(Cloudflare),需要你过一次", captcha: "登录页需要验证码", challenge: "页面需要人工验证", two_factor: "需要二次验证或本人设备确认", @@ -2054,7 +2363,7 @@ export function toLease(r: BrowserControlLeaseRow): BrowserControlLease { } /** 视觉子流程可以接手的等待原因(验证码 / 页面挑战 / 上次没处理成功) */ -const SOLVABLE = ["captcha", "challenge", "challenge_unsolved"]; +const SOLVABLE = ["captcha", "challenge", "challenge_unsolved", "cf_challenge"]; /** capture / visionAct 的结果(能力输出的公共部分) */ export interface VisionShot { diff --git a/apps/server/src/modules/cache/index.ts b/apps/server/src/modules/cache/index.ts index 8b1a40ad..9bf7cda5 100644 --- a/apps/server/src/modules/cache/index.ts +++ b/apps/server/src/modules/cache/index.ts @@ -10,6 +10,7 @@ import { } from "@umt/contracts"; import type { z } from "zod"; import { dataPaths } from "../../config.ts"; +import { externalFetch } from "../../egress.ts"; import { requireScope } from "../../http/auth.ts"; import { errorBody } from "../../http/errors.ts"; import { CACHE_REGIONS, sharedCache, storageSettings, tmdbCachePolicy } from "../../shared-cache.ts"; @@ -73,7 +74,8 @@ export function createCacheModule(opts: CacheModuleOptions = {}): ServerModule & if (!c) { c = new ImageCache({ dir: opts.imageDir ?? join(dataPaths(ctx.config).dataDir, "cache", "images"), - ...(opts.imageFetch ? { fetch: opts.imageFetch } : {}), + // 外网请求:显式按环境变量走代理(CF-2 出口表) + fetch: opts.imageFetch ?? externalFetch, ...(opts.imageUpstreamBase ? { upstreamBase: opts.imageUpstreamBase } : {}), }); imageCaches.set(ctx, c); diff --git a/apps/server/src/modules/sources-identity/routes/enrich.ts b/apps/server/src/modules/sources-identity/routes/enrich.ts index 8f2d292c..a3e92fd2 100644 --- a/apps/server/src/modules/sources-identity/routes/enrich.ts +++ b/apps/server/src/modules/sources-identity/routes/enrich.ts @@ -9,6 +9,7 @@ import { IdParam, MetadataEnrichOutput, } from "@umt/contracts"; +import { externalFetch } from "../../../egress.ts"; import { requireScope } from "../../../http/auth.ts"; import { ApiException, errorBody } from "../../../http/errors.ts"; import type { ModuleApp } from "../../types.ts"; @@ -122,7 +123,10 @@ export function enrichRoutes(app: ModuleApp, rt: () => SourcesRuntime): void { const srcHost = hostOf(s.data.fields.posterUrl?.sourceUrl ?? ""); if (!srcHost || !domainAllowed(srcHost, identity.enrichDomains ?? [])) return c.json(errorBody("forbidden", "海报出处不在补全白名单里"), 403); - const got = await fetchPublicImage(r.overrides.fetch ?? fetch, url, { maxBytes: POSTER_MAX_BYTES, label: "海报" }); + const got = await fetchPublicImage(r.overrides.fetch ?? externalFetch, url, { + maxBytes: POSTER_MAX_BYTES, + label: "海报", + }); if (!got.ok) return c.json(errorBody("upstream_unavailable", got.message), 502); return imageResponse(got, 86400 * 7); }); diff --git a/apps/server/src/modules/sources-identity/routes/workbench.ts b/apps/server/src/modules/sources-identity/routes/workbench.ts index 623a84d3..45561fac 100644 --- a/apps/server/src/modules/sources-identity/routes/workbench.ts +++ b/apps/server/src/modules/sources-identity/routes/workbench.ts @@ -1,5 +1,6 @@ import { Id, WorkbenchReleaseDetail, WorkbenchReleasePage, WorkbenchReleaseQuery } from "@umt/contracts"; import { z } from "zod"; +import { externalFetch } from "../../../egress.ts"; import { requireScope } from "../../../http/auth.ts"; import { errorBody } from "../../../http/errors.ts"; import type { ModuleApp } from "../../types.ts"; @@ -55,7 +56,7 @@ export function workbenchRoutes(app: ModuleApp, rt: () => SourcesRuntime) { const r = rt(); const req = await detailScreenshotRequest(r, rid, index); if (!req) return c.json(errorBody("not_found", "截图不存在或详情缓存已过期"), 404); - const got = await fetchPublicImage(r.overrides.fetch ?? fetch, req.url, { + const got = await fetchPublicImage(r.overrides.fetch ?? externalFetch, req.url, { maxBytes: SHOT_MAX_BYTES, label: "截图", ...(req.referer ? { referer: req.referer } : {}), diff --git a/apps/server/src/modules/sources-identity/runtime.ts b/apps/server/src/modules/sources-identity/runtime.ts index f4fa8c74..9bd7ad7c 100644 --- a/apps/server/src/modules/sources-identity/runtime.ts +++ b/apps/server/src/modules/sources-identity/runtime.ts @@ -15,6 +15,7 @@ import { } from "@umt/adapters"; import { DEFAULT_SETTINGS, type SitesSettings } from "@umt/contracts"; import { pgTokenBucketFor, type SiteRow, sql } from "@umt/db"; +import { egressKey, externalFetch, siteEgress, siteFetchFor } from "../../egress.ts"; import { CACHE_REGIONS, renewTrackedTmdbDetail, @@ -24,6 +25,7 @@ import { tmdbDetailRef, } from "../../shared-cache.ts"; import type { ServerContext } from "../types.ts"; +import { CloudflareGate, withClearance } from "./service/cloudflare.ts"; import { readBrowserUserAgent, resolveUserAgent, type UserAgentDefaults } from "./service/user-agent.ts"; // 模块运行时依赖:站点适配器与限流器(每站一个实例,令牌在 PG 跨进程共享)、RSS 抓取器、TMDB 客户端、时钟。 @@ -58,12 +60,15 @@ export class SourcesRuntime { private siteDefaultsCache: SitesSettings = DEFAULT_SETTINGS.sites; private siteDefaultsAt = 0; private tmdbSig: string | null = null; + /** CF-2 Cloudflare 挑战处理(站点请求经 siteFetch 进来) */ + readonly cloudflare: CloudflareGate; constructor( readonly ctx: ServerContext, readonly overrides: RuntimeOverrides = {}, ) { this.now = overrides.now ?? (() => new Date()); + this.cloudflare = new CloudflareGate(this); this.rss = createRssFetcher({ ...(overrides.fetch ? { fetch: overrides.fetch } : {}), now: this.now, @@ -101,7 +106,7 @@ export class SourcesRuntime { const { cfg } = await this.tmdbConfig(); if (!cfg) return null; let n = 0; - const base = cfg.fetch ?? globalThis.fetch; + const base = cfg.fetch ?? externalFetch; const counted: typeof fetch = (input, init) => { n++; return base(input, init); @@ -144,7 +149,8 @@ export class SourcesRuntime { ...(apiKey ? { apiKey } : {}), ...(accessToken ? { accessToken } : {}), ...(baseUrl ? { baseUrl } : {}), - ...(this.overrides.fetch ? { fetch: this.overrides.fetch } : {}), + // 外网请求:显式按环境变量走代理(CF-2 出口表;测试注入优先) + fetch: this.overrides.fetch ?? externalFetch, // 请求响应进 PG 缓存(region tmdb.request),三进程共享;TTL / 条数上限 / 空结果短 TTL 取设置(对照 MP) cache: { ttlMs: policy.ttl * 1000, @@ -168,40 +174,53 @@ export class SourcesRuntime { } /** - * 站点请求用的 fetch:测试注入优先;站点开了"使用代理"且设置了 proxyUrl 时走代理(Deno.createHttpClient)。 + * 站点请求用的 fetch:测试注入优先;站点开了"使用代理"且设置了 proxyUrl 时走代理,否则**明确直连** + * (CF-2:不再隐式继承容器的 HTTP(S)_PROXY,见 egress.ts)。 * BR-4:每条请求都带 UA(请求自己设了就不动;否则 ua 参数,缺省按"跟随浏览器"),不再发 Deno 缺省的 `Deno/x.y.z`。 * ua 可以是函数:每次请求现场取(站点适配器按当前账号取,账号切换 UA 后立即生效)。 + * CF-2:给了 site.id 的请求经 Cloudflare 处理(浏览器代理期直接走浏览器、带可复用的 cf_clearance、遇挑战过挑战 / 重试 / 浏览器代抓)。 */ siteFetch( - site: Pick, + site: Pick & Partial>, defaults: SitesSettings = this.siteDefaultsCache, ua?: string | (() => Promise), ): typeof globalThis.fetch { const base = this.transportFetch(site, defaults); - return ((input: RequestInfo | URL, init?: RequestInit) => { + const egress = this.egressKeyOf(site, defaults); + const siteId = site.id; + return (async (input: RequestInfo | URL, init?: RequestInit) => { const headers = new Headers(init?.headers ?? (input instanceof Request ? input.headers : undefined)); - if (headers.has("user-agent")) return base(input, { ...init, headers }); - const pick = typeof ua === "function" ? ua() : ua !== undefined ? Promise.resolve(ua) : this.defaultUserAgent(); - return pick.then((v) => { - headers.set("user-agent", v); - return base(input, { ...init, headers }); - }); + if (!headers.has("user-agent")) { + const pick = typeof ua === "function" ? ua() : ua !== undefined ? Promise.resolve(ua) : this.defaultUserAgent(); + headers.set("user-agent", await pick); + } + if (!siteId) return base(input, { ...init, headers }); + const url = input instanceof Request ? input.url : String(input); + const method = init?.method ?? (input instanceof Request ? input.method : "GET"); + return this.cloudflare.handle( + siteId, + { url, method, accept: headers.get("accept"), userAgent: headers.get("user-agent"), egress }, + (extra) => { + const h = new Headers(headers); + const cookie = withClearance(h.get("cookie"), extra); + if (cookie) h.set("cookie", cookie); + // 过挑战可能超过单次请求的超时:重试时原信号已中止就换一个新的超时(调用方取消照样生效在级联之外) + const signal = init?.signal?.aborted ? AbortSignal.timeout(30_000) : init?.signal; + return base(input, { ...init, headers: h, ...(signal ? { signal } : {}) }); + }, + ); }) as typeof globalThis.fetch; } - /** 不加 UA 的底层 fetch(代理 / 测试注入) */ + /** 站点 HTTP 出口键(direct / proxy:<主机>),与浏览器会话的出口同一口径 */ + egressKeyOf(site: Pick, defaults: SitesSettings = this.siteDefaultsCache): string { + return egressKey(siteEgress(site.proxy, defaults.proxyUrl)); + } + + /** 不加 UA 的底层 fetch(测试注入 / 按站点出口:代理或明确直连) */ private transportFetch(site: Pick, defaults: SitesSettings): typeof globalThis.fetch { if (this.overrides.fetch) return this.overrides.fetch; - if (!site.proxy || !defaults.proxyUrl) return globalThis.fetch; - const key = defaults.proxyUrl; - let client = this.proxyClients.get(key); - if (!client) { - client = Deno.createHttpClient({ proxy: { url: key } }); - this.proxyClients.set(key, client); - } - const c = client; - return ((input: RequestInfo | URL, init?: RequestInit) => - globalThis.fetch(input, { ...init, client: c } as RequestInit)) as typeof globalThis.fetch; + return siteFetchFor(siteEgress(site.proxy, defaults.proxyUrl)); } // ---------- BR-4 站点请求 UA ---------- @@ -253,8 +272,6 @@ export class SourcesRuntime { return resolveUserAgent(acc, d).ua; } - private readonly proxyClients = new Map(); - /** * 站点适配器:每站一个实例,列表/搜索/详情/下载共享同一个限流器。 * 站点限流参数、基址、代理或站点默认变化时重建(签名比较)。 @@ -270,10 +287,12 @@ export class SourcesRuntime { } const defaults = this.siteDefaultsCache; const proxy = site.proxy ?? false; + // Cloudflare 处理状态(config.cloudflare)经常变,不参与重建判断 + const { cloudflare: _cf, ...configSig } = (site.config ?? {}) as Record; const sig = JSON.stringify([ site.base_url, site.rate_limit, - site.config, + configSig, proxy, proxy ? defaults.proxyUrl : null, defaults.defaultRateLimit, @@ -304,7 +323,7 @@ export class SourcesRuntime { limiter, now: this.now, // 每条请求都经 siteFetch:代理 + 按账号现场取 UA(BR-4) - fetch: this.siteFetch({ proxy }, defaults, () => this.siteUserAgent(site.id)), + fetch: this.siteFetch({ id: site.id, proxy }, defaults, () => this.siteUserAgent(site.id)), ...(baseUrl ? { baseUrl } : {}), }); this.adapters.set(site.id, { sig, limiter, adapter }); diff --git a/apps/server/src/modules/sources-identity/service/cloudflare.ts b/apps/server/src/modules/sources-identity/service/cloudflare.ts new file mode 100644 index 00000000..3e540f8f --- /dev/null +++ b/apps/server/src/modules/sources-identity/service/cloudflare.ts @@ -0,0 +1,431 @@ +import { detectChallenge, mergeCookieHeaders, parseCookieHeader } from "@umt/adapters"; +import { type SiteCloudflareState, siteCloudflareState } from "@umt/contracts"; +import { type SiteAccountRow, type SiteRow, sql } from "@umt/db"; +import { egressKey, siteEgress } from "../../../egress.ts"; +import type { ServerContext } from "../../types.ts"; +import type { SourcesRuntime } from "../runtime.ts"; +import { resolveUserAgent } from "./user-agent.ts"; + +// CF-2 Cloudflare 挑战处理(docs/design/cloudflare-handling.md;运维见 docs/runbook/sites.md「Cloudflare」)。 +// +// 站点 HTTP 请求(SourcesRuntime.siteFetch)经这里: +// 0. 站点在"浏览器代理期"(config.cloudflare.browserUntil 未到)且请求是页面(GET + Accept text/html)→ 直接由浏览器代抓; +// 1. 发 HTTP;账号 UA 跟随浏览器、出口与浏览器一致、复用没失败过时带上浏览器拿到的 cf_clearance(__cf_bm 不带); +// 2. 响应是挑战(cf-mitigated: challenge,或 403 / 503 / 429 的挑战页)→ 不算凭据错误,记 lastChallengeAt; +// 3. 过挑战(同站 30 分钟最多一次、同一时刻只跑一次;只处理有登录态账号的站点):浏览器用该账号 profile 打开同一地址 → +// ① 等它自己过(≤15 秒)→ ② 复选框 / Turnstile 派视觉子流程点一下(最多 2 次)→ ③ 选图 / 输码类转人工(待处理事项); +// 4. 过了:读回 cf_clearance + navigator.userAgent;满足复用条件就存下来,HTTP 重试 1 次,结果记 cfReuse = ok / fails; +// 5. 仍被挑战:页面请求改由浏览器导航取 HTML 交给原解析器,站点记 6 小时浏览器代理期;其它请求(下载、RSS、POST)照常报挑战。 +// 不接任何打码服务。 + +/** 同站两次过挑战的最短间隔 */ +export const CF_ATTEMPT_INTERVAL_MS = 30 * 60_000; +/** HTTP 过不去时走浏览器的时长 */ +export const CF_BROWSER_MODE_MS = 6 * 3_600_000; +/** 站点状态读缓存(每条站点请求都要看是否在浏览器代理期) */ +const STATE_CACHE_MS = 5_000; + +export type ChallengeSolveOutcome = + | { + status: "passed"; + by: "auto" | "vision"; + /** 浏览器 profile 里的 cf_clearance 值;没有为 null */ + clearance: string | null; + /** 浏览器 navigator.userAgent */ + userAgent: string | null; + /** 浏览器这一侧的出口键(egressKey);按容器环境变量出网 / 不知道时 null */ + browserEgress: string | null; + } + | { status: "needs_human"; sessionId: string | null; decisionId: string | null; reason: string } + | { status: "failed"; reason: string }; + +export type BrowserFetchOutcome = + | { status: "ok"; html: string; url: string | null } + | { status: "challenge"; reason: string } + | { status: "unavailable"; reason: string }; + +/** 浏览器一侧(浏览器模块注册;sources-identity 不直接依赖浏览器模块) */ +export interface ChallengeBrowserPort { + solve(req: { siteId: string; accountId: string; url: string }): Promise; + fetchHtml(req: { siteId: string; accountId: string; url: string }): Promise; +} + +const ports = new WeakMap(); + +export function registerChallengeBrowser(ctx: object, port: ChallengeBrowserPort): void { + ports.set(ctx, port); +} + +export function challengeBrowserFor(ctx: object): ChallengeBrowserPort | null { + return ports.get(ctx) ?? null; +} + +/** user_info.cloudflare:浏览器取得的 cf_clearance(值在 secret 里)与取得时的 UA / 出口 */ +interface ClearanceInfo { + secretRef: string; + userAgent: string | null; + egress: string | null; + at: string; +} + +const obj = (v: unknown): Record => + v && typeof v === "object" && !Array.isArray(v) ? (v as Record) : {}; + +function clearanceInfo(userInfo: unknown): ClearanceInfo | null { + const c = obj(obj(userInfo).cloudflare); + return typeof c.secretRef === "string" && typeof c.at === "string" + ? { + secretRef: c.secretRef, + userAgent: typeof c.userAgent === "string" ? c.userAgent : null, + egress: typeof c.egress === "string" ? c.egress : null, + at: c.at, + } + : null; +} + +export interface ChallengeRequest { + url: string; + method: string; + /** 请求的 Accept 头 */ + accept: string | null; + /** 请求实际带的 UA */ + userAgent: string | null; + /** 该请求的 HTTP 出口键 */ + egress: string; +} + +const EMPTY: SiteCloudflareState = { + browserUntil: null, + cfReuse: null, + lastChallengeAt: null, + lastSolvedBy: null, + lastSolvedAt: null, + lastAttemptAt: null, + pending: null, + sessionId: null, + lastResult: null, + lastRule: null, + lastError: null, +}; + +/** 页面请求:浏览器导航能代替的(GET + 要 HTML) */ +export function browserFetchable(req: Pick): boolean { + return req.method.toUpperCase() === "GET" && /text\/html|application\/xhtml/i.test(req.accept ?? ""); +} + +/** cf_clearance 能不能交给 HTTP:账号 UA 跟随浏览器、UA 相同、两端出口相同、复用没失败过 */ +export function clearanceUsable(input: { + uaSource: string; + requestUa: string | null; + clearanceUa: string | null; + requestEgress: string; + clearanceEgress: string | null; + cfReuse: SiteCloudflareState["cfReuse"]; +}): boolean { + return ( + input.uaSource === "browser" && + !!input.requestUa && + input.requestUa === input.clearanceUa && + !!input.clearanceEgress && + input.clearanceEgress === input.requestEgress && + input.cfReuse !== "fails" + ); +} + +export class CloudflareGate { + private readonly cache = new Map(); + /** 进程内同站同一时刻只过一次挑战:后到的请求等同一个结果 */ + private readonly inflight = new Map>(); + + constructor(private readonly rt: SourcesRuntime) {} + + private get ctx(): ServerContext { + return this.rt.ctx; + } + + private now(): Date { + return this.rt.now(); + } + + async state(siteId: string, fresh = false): Promise { + const hit = this.cache.get(siteId); + if (!fresh && hit && Date.now() - hit.at < STATE_CACHE_MS) return hit.state; + const row = await this.ctx.db.selectFrom("site").select("config").where("id", "=", siteId).executeTakeFirst(); + const state = { ...EMPTY, ...(siteCloudflareState(row?.config) ?? {}) }; + this.cache.set(siteId, { at: Date.now(), state }); + return state; + } + + /** 合并写 site.config.cloudflare(只动这一个键) */ + async patch(siteId: string, patch: Partial): Promise { + await sql`update site set config = coalesce(config, '{}'::jsonb) || jsonb_build_object('cloudflare', + coalesce(config->'cloudflare', '{}'::jsonb) || ${JSON.stringify(patch)}::jsonb) + where id = ${siteId}`.execute(this.ctx.db); + this.cache.delete(siteId); + } + + /** + * 同站 30 分钟最多一次:条件更新 lastAttemptAt(跨进程)。抢到返回 true。 + */ + private async claimAttempt(siteId: string): Promise { + const now = this.now(); + const before = new Date(now.getTime() - CF_ATTEMPT_INTERVAL_MS); + const r = await sql<{ + id: string; + }>`update site set config = coalesce(config, '{}'::jsonb) || jsonb_build_object('cloudflare', + coalesce(config->'cloudflare', '{}'::jsonb) || jsonb_build_object('lastAttemptAt', ${now.toISOString()}::text)) + where id = ${siteId} + and (config->'cloudflare'->>'lastAttemptAt' is null + or (config->'cloudflare'->>'lastAttemptAt')::timestamptz <= ${before}) + returning id`.execute(this.ctx.db); + this.cache.delete(siteId); + return r.rows.length > 0; + } + + /** + * 有登录态的账号:与站点请求用的凭证同一个(第一个启用、未失效的 Cookie / 账密账号,且确实有 Cookie)。 + * 没有返回 null(没登录的站点不处理挑战)。 + */ + async loggedInAccount(siteId: string): Promise { + const rows = await this.ctx.db + .selectFrom("site_account") + .selectAll() + .where("site_id", "=", siteId) + .where("enabled", "=", true) + .where("status", "in", ["unknown", "active"]) + .where("auth_kind", "in", ["cookie", "password"]) + .orderBy("created_at") + .execute(); + return ( + rows.find( + (r) => (r.auth_kind === "cookie" && !!r.secret_ref) || (r.auth_kind === "password" && !!r.session_secret_ref), + ) ?? null + ); + } + + /** 这条请求能带的 cf_clearance 头片段("cf_clearance=…");不满足复用条件为 null */ + async clearanceCookie(siteId: string, req: ChallengeRequest): Promise { + const acc = await this.loggedInAccount(siteId); + if (!acc) return null; + const info = clearanceInfo(acc.user_info); + if (!info) return null; + const st = await this.state(siteId); + const d = await this.rt.uaDefaults(); + const ok = clearanceUsable({ + uaSource: resolveUserAgent(acc, d).source, + requestUa: req.userAgent, + clearanceUa: info.userAgent, + requestEgress: req.egress, + clearanceEgress: info.egress, + cfReuse: st.cfReuse, + }); + if (!ok) return null; + const v = await this.ctx.secrets.resolve(info.secretRef).catch(() => null); + return v ? `cf_clearance=${v}` : null; + } + + /** 把浏览器拿到的 cf_clearance 存进账号(secret),记下取得时的 UA 与出口 */ + private async storeClearance( + acc: SiteAccountRow, + value: string, + userAgent: string | null, + egress: string | null, + ): Promise<{ conditionsChanged: boolean }> { + const info = obj(acc.user_info); + const prev = clearanceInfo(acc.user_info); + const conditionsChanged = !!prev && (prev.userAgent !== userAgent || prev.egress !== egress); + let ref = prev?.secretRef ?? null; + if (ref) { + try { + await this.ctx.secrets.replace(ref, value); + } catch { + ref = null; + } + } + if (!ref) + ref = await this.ctx.secrets.create({ + purpose: "site_cookie", + plaintext: value, + label: `site_account:${acc.site_id}:${acc.label}:cf_clearance:${crypto.randomUUID().slice(0, 8)}`, + }); + const next: ClearanceInfo = { secretRef: ref, userAgent, egress, at: this.now().toISOString() }; + await this.ctx.db + .updateTable("site_account") + .set({ user_info: JSON.stringify({ ...info, cloudflare: next }) }) + .where("id", "=", acc.id) + .execute(); + // UA 或出口变了:之前"复用无效"的结论不再适用,重新试一次 + if (conditionsChanged) await this.patch(acc.site_id, { cfReuse: null }); + return { conditionsChanged }; + } + + /** 过挑战:进程内同站只跑一次,跨进程 30 分钟一次;被节流返回 null */ + private solveOnce(site: SiteRow, acc: SiteAccountRow, url: string): Promise { + const running = this.inflight.get(site.id); + if (running) return running; + const p = (async () => { + const port = challengeBrowserFor(this.ctx); + if (!port) return null; + if (!(await this.claimAttempt(site.id))) return null; + try { + return await port.solve({ siteId: site.id, accountId: acc.id, url }); + } catch (e) { + return { status: "failed" as const, reason: String((e as Error)?.message ?? e).slice(0, 300) }; + } + })().finally(() => this.inflight.delete(site.id)); + this.inflight.set(site.id, p); + return p; + } + + /** 浏览器代抓一页;失败 / 仍是挑战返回 null */ + private async viaBrowser(siteId: string, acc: SiteAccountRow, url: string): Promise { + const port = challengeBrowserFor(this.ctx); + if (!port) return null; + let r: BrowserFetchOutcome; + try { + r = await port.fetchHtml({ siteId, accountId: acc.id, url }); + } catch (e) { + r = { status: "unavailable", reason: String((e as Error)?.message ?? e) }; + } + if (r.status !== "ok") { + this.ctx.logger.info({ component: "cloudflare", siteId, status: r.status, reason: r.reason }, "浏览器代抓未成功"); + if (r.status === "challenge") await this.patch(siteId, { lastResult: "browser_challenge", lastError: r.reason }); + return null; + } + return new Response(r.html, { + status: 200, + headers: { "content-type": "text/html; charset=utf-8", "x-umt-via": "browser" }, + }); + } + + private log(siteId: string, msg: string, extra: Record = {}): void { + this.ctx.logger.info({ component: "cloudflare", siteId, ...extra }, msg); + } + + /** + * 站点请求入口。send(cookieExtra) 发一次 HTTP(cookieExtra 非空时并进 Cookie 头);返回交给适配器的响应 + * (挑战没处理掉时原样返回挑战响应,适配器报 challenge)。 + */ + async handle( + siteId: string, + req: ChallengeRequest, + send: (cookieExtra: string | null) => Promise, + ): Promise { + const pagey = browserFetchable(req); + let st = await this.state(siteId); + const now = this.now(); + // 0. 浏览器代理期:页面请求直接走浏览器 + if (pagey && st.browserUntil && Date.parse(st.browserUntil) > now.getTime() && challengeBrowserFor(this.ctx)) { + const acc = await this.loggedInAccount(siteId); + const r = acc ? await this.viaBrowser(siteId, acc, req.url) : null; + if (r) return r; + } + // 1. HTTP(满足条件时带 cf_clearance) + // 从没遇到过挑战的站点不查账号(每条请求少两次查询) + const extra = st.lastChallengeAt ? await this.clearanceCookie(siteId, req).catch(() => null) : null; + const first = await detectChallenge(await send(extra)); + if (!first.rule) return first.res; + // 2. 挑战:记下来;没有浏览器 / 没有登录态账号 → 原样返回(适配器报 challenge,不算凭据错误) + await this.patch(siteId, { lastChallengeAt: now.toISOString(), lastRule: first.rule }); + const port = challengeBrowserFor(this.ctx); + if (!port) return first.res; + const acc = await this.loggedInAccount(siteId); + if (!acc) { + await this.patch(siteId, { lastResult: "skipped", lastError: "站点没有已登录的账号,不处理挑战" }); + return first.res; + } + const site = await this.ctx.db.selectFrom("site").selectAll().where("id", "=", siteId).executeTakeFirst(); + if (!site?.enabled || site.deleted_at) return first.res; + // 3. 过挑战 + const out = await this.solveOnce(site, acc, req.url); + st = await this.state(siteId, true); + if (!out) this.log(siteId, "站点 Cloudflare 挑战:30 分钟内处理过,本次不再过挑战"); + else if (out.status === "passed") { + const at = this.now().toISOString(); + await this.patch(siteId, { + lastSolvedBy: out.by, + lastSolvedAt: at, + lastResult: "passed", + pending: null, + sessionId: null, + lastError: null, + }); + this.log(siteId, "站点 Cloudflare 挑战已通过", { by: out.by }); + // 4. 同步 cf_clearance 并重试 HTTP 一次 + if (out.clearance) { + await this.storeClearance(acc, out.clearance, out.userAgent, out.browserEgress); + const retryExtra = await this.clearanceCookie(siteId, req).catch(() => null); + if (retryExtra) { + await this.rt.limiterOf(siteId)?.acquire(); + const second = await detectChallenge(await send(retryExtra)); + if (!second.rule) { + await this.patch(siteId, { cfReuse: "ok", lastResult: "reuse_ok", browserUntil: null }); + return second.res; + } + await second.res.body?.cancel().catch(() => {}); + await this.patch(siteId, { cfReuse: "fails", lastResult: "reuse_fails" }); + this.log(siteId, "浏览器的 cf_clearance 给 HTTP 用仍被挑战,之后不再复用"); + } + } + } else if (out.status === "needs_human") { + await this.patch(siteId, { + pending: "human", + sessionId: out.sessionId, + lastResult: "needs_human", + lastError: out.reason, + }); + } else { + await this.patch(siteId, { lastResult: "failed", lastError: out.reason }); + } + // 5. 页面请求改由浏览器代抓,并进入 6 小时浏览器代理期 + if (pagey && !(out && out.status === "needs_human")) { + const r = await this.viaBrowser(siteId, acc, req.url); + if (r) { + const until = new Date(this.now().getTime() + CF_BROWSER_MODE_MS).toISOString(); + await this.patch(siteId, { browserUntil: until, ...(out ? {} : { lastResult: "browser" }) }); + await first.res.body?.cancel().catch(() => {}); + return r; + } + } + return first.res; + } + + /** + * 人在原站视图里过了挑战、交还会话之后(浏览器模块调用):读回的 cf_clearance / UA 存起来,清掉"等人处理"。 + */ + async afterHuman( + siteId: string, + accountId: string, + got: { clearance: string | null; userAgent: string | null; browserEgress: string | null }, + ): Promise { + const st = await this.state(siteId, true); + if (st.pending !== "human") return; + const acc = await this.ctx.db.selectFrom("site_account").selectAll().where("id", "=", accountId).executeTakeFirst(); + if (acc && got.clearance) await this.storeClearance(acc, got.clearance, got.userAgent, got.browserEgress); + await this.patch(siteId, { + pending: null, + sessionId: null, + lastSolvedBy: "human", + lastSolvedAt: this.now().toISOString(), + lastResult: "passed", + lastError: null, + // 人过了:允许马上再试一次 HTTP(不等 30 分钟) + lastAttemptAt: null, + }); + this.log(siteId, "站点 Cloudflare 挑战已由人在原站视图里通过"); + } +} + +/** Cookie 头里并进 cf_clearance(同名覆盖);__cf_bm 等其它 Cloudflare Cookie 不带 */ +export function withClearance(cookieHeader: string | null, extra: string | null): string | null { + if (!extra) return cookieHeader; + const jar = parseCookieHeader(cookieHeader); + jar.delete("__cf_bm"); + const merged = mergeCookieHeaders([...jar].map(([k, v]) => `${k}=${v}`).join("; ") || null, extra); + return merged; +} + +/** 站点出口键(与浏览器会话 applyEgress 同一口径) */ +export function siteEgressKey(proxy: boolean, proxyUrl: string | null | undefined): string { + return egressKey(siteEgress(proxy, proxyUrl)); +} diff --git a/apps/server/src/modules/websearch/service.ts b/apps/server/src/modules/websearch/service.ts index d8b46333..bfcbc7ba 100644 --- a/apps/server/src/modules/websearch/service.ts +++ b/apps/server/src/modules/websearch/service.ts @@ -17,6 +17,7 @@ import { type WebSearchTestResult, } from "@umt/contracts"; import { cachedCall, pgCacheFor, pgTokenBucketFor } from "@umt/db"; +import { externalFetch } from "../../egress.ts"; import { storageSettings } from "../../shared-cache.ts"; import type { ServerContext } from "../types.ts"; @@ -94,7 +95,8 @@ export class WebSearchService { return createBraveClient({ apiKey, ...(this.opts.braveBaseUrl ? { baseUrl: this.opts.braveBaseUrl } : {}), - ...(this.opts.fetch ? { fetch: this.opts.fetch } : {}), + // 外网请求:显式按环境变量走代理(CF-2 出口表) + fetch: this.opts.fetch ?? externalFetch, }); } @@ -184,7 +186,7 @@ export class WebSearchService { allow ? "只能读取最近一次 web.search 结果里的链接" : "先用 web.search 搜索,再读取结果里的链接", ); const page = await fetchPage(want, { - ...(this.opts.fetch ? { fetch: this.opts.fetch } : {}), + fetch: this.opts.fetch ?? externalFetch, ...(this.opts.resolveHost ? { resolve: this.opts.resolveHost } : {}), ...(o.signal ? { signal: o.signal } : {}), }); diff --git a/apps/server/test/sources-identity/cloudflare.test.ts b/apps/server/test/sources-identity/cloudflare.test.ts new file mode 100644 index 00000000..9939ea7f --- /dev/null +++ b/apps/server/test/sources-identity/cloudflare.test.ts @@ -0,0 +1,315 @@ +import { SiteError } from "@umt/adapters"; +import { siteCloudflareState } from "@umt/contracts"; +import { createMockNexusPhpSite, type MockNexusPhpSite } from "@umt/testkit"; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; +import { + type BrowserFetchOutcome, + browserFetchable, + CF_ATTEMPT_INTERVAL_MS, + type ChallengeSolveOutcome, + clearanceUsable, + registerChallengeBrowser, + withClearance, +} from "../../src/modules/sources-identity/service/cloudflare.ts"; +import { snapshotSite } from "../../src/modules/sources-identity/service/snapshot.ts"; +import { registerBrowserIdentityReader } from "../../src/modules/sources-identity/service/user-agent.ts"; +import { createHarness, type Harness, pgAvailable } from "./harness.ts"; + +// CF-2 级联状态机:HTTP 遇 Cloudflare 挑战 → 浏览器过挑战(自动 / 视觉子流程 / 转人工)→ cf_clearance 同步给 HTTP 重试 1 次 → +// 仍被挑战则页面请求改由浏览器代抓并进入 6 小时浏览器代理期;同站 30 分钟节流、同一时刻只处理一次;挑战不算凭据错误。 +// 站点 = testkit mock NexusPHP(真 HTTP),前面套一层"Cloudflare":没带合格 cf_clearance 的请求回 403 + cf-mitigated 挑战页。 +// 浏览器一侧用假端口(ChallengeBrowserPort),真实浏览器流程见 browser-cloudflare.test.ts。 + +const up = await pgAvailable(); +const BROWSER_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:152.0) Gecko/20100101 Firefox/152.0"; +const CF_HTML = `Just a moment... +
`; + +// biome-ignore lint/suspicious/noExplicitAny: 响应体断言 +const json = async (r: Response): Promise => { + const body = await r.json(); + if (!r.ok) throw new Error(`${r.status} ${JSON.stringify(body)}`); + return body.data; +}; + +describe("CF-2 纯函数", () => { + it("页面请求才能由浏览器代抓", () => { + expect(browserFetchable({ method: "GET", accept: "text/html,application/xhtml+xml" })).toBe(true); + expect(browserFetchable({ method: "GET", accept: null })).toBe(false); + expect(browserFetchable({ method: "POST", accept: "text/html" })).toBe(false); + expect(browserFetchable({ method: "GET", accept: "application/rss+xml" })).toBe(false); + }); + it("cf_clearance 同步条件:UA 跟随浏览器 + UA 相同 + 两端出口相同 + 复用没失败过", () => { + const base = { + uaSource: "browser", + requestUa: "A", + clearanceUa: "A", + requestEgress: "direct", + clearanceEgress: "direct", + cfReuse: null, + } as const; + expect(clearanceUsable(base)).toBe(true); + expect(clearanceUsable({ ...base, uaSource: "manual" })).toBe(false); + expect(clearanceUsable({ ...base, uaSource: "global" })).toBe(false); + expect(clearanceUsable({ ...base, clearanceUa: "B" })).toBe(false); + expect(clearanceUsable({ ...base, clearanceEgress: null })).toBe(false); + expect(clearanceUsable({ ...base, clearanceEgress: "proxy:http://relay:7891" })).toBe(false); + expect(clearanceUsable({ ...base, cfReuse: "fails" })).toBe(false); + }); + it("并进 cf_clearance,去掉 __cf_bm", () => { + expect(withClearance("a=1; __cf_bm=x", "cf_clearance=Z")).toBe("a=1; cf_clearance=Z"); + expect(withClearance(null, "cf_clearance=Z")).toBe("cf_clearance=Z"); + expect(withClearance("a=1", null)).toBe("a=1"); + }); +}); + +describe.skipIf(!up)("CF-2 Cloudflare 级联", () => { + let h: Harness; + let mock: MockNexusPhpSite; + let siteId: string; + let accountId: string; + /** "Cloudflare" 开关与放行规则 */ + const cf = { + on: false, + /** 放行判定;缺省:带 cf_clearance=GOOD 且 UA 是浏览器 UA */ + pass: (h: Headers) => (h.get("cookie") ?? "").includes("cf_clearance=GOOD") && h.get("user-agent") === BROWSER_UA, + challenged: 0, + passed: 0, + }; + const seen: Headers[] = []; + const solves: { url: string; accountId: string }[] = []; + const fetches: string[] = []; + let solveResult: ChallengeSolveOutcome = { status: "failed", reason: "unset" }; + let solveDelay = 0; + let fetchResult: BrowserFetchOutcome | null = null; + + const cfFetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const hd = new Headers(init?.headers); + seen.push(hd); + if (cf.on && !cf.pass(hd)) { + cf.challenged++; + return new Response(CF_HTML, { + status: 403, + headers: { "cf-mitigated": "challenge", "content-type": "text/html" }, + }); + } + cf.passed++; + return globalThis.fetch(input, init); + }) as typeof fetch; + + const site = () => h.db.selectFrom("site").selectAll().where("id", "=", siteId).executeTakeFirstOrThrow(); + const account = () => + h.db.selectFrom("site_account").selectAll().where("id", "=", accountId).executeTakeFirstOrThrow(); + const state = async () => siteCloudflareState((await site()).config); + const list = async () => h.rt.siteAdapter(await site()).list(1); + + beforeAll(async () => { + mock = await createMockNexusPhpSite({}); + h = await createHarness("cf2", { fetch: cfFetch }); + registerBrowserIdentityReader(h.ctx, async () => BROWSER_UA); + registerChallengeBrowser(h.ctx, { + async solve(req) { + solves.push(req); + if (solveDelay) await new Promise((r) => setTimeout(r, solveDelay)); + return solveResult; + }, + async fetchHtml(req) { + fetches.push(req.url); + if (fetchResult) return fetchResult; + // 浏览器已登录且过了挑战:直接取 mock 页面 + const r = await globalThis.fetch(req.url, { headers: { cookie: mock.cookie } }); + return { status: "ok", html: await r.text(), url: req.url }; + }, + }); + const s = await json( + await h.req("/sites", { + body: { + key: "audiences", + name: "观众", + adapter: "nexusphp", + baseUrl: `${mock.baseUrl}/`, + rateLimit: { minIntervalMs: 0, concurrency: 10 }, + }, + }), + ); + siteId = s.id; + const r = await json( + await h.req(`/sites/${siteId}/credentials`, { method: "PUT", body: { kind: "cookie", cookie: mock.cookie } }), + ); + accountId = r.siteAccountId; + }); + afterAll(async () => { + await h?.close(); + await mock?.close(); + }); + + beforeEach(async () => { + cf.on = true; + cf.pass = (hd) => (hd.get("cookie") ?? "").includes("cf_clearance=GOOD") && hd.get("user-agent") === BROWSER_UA; + cf.challenged = 0; + cf.passed = 0; + seen.length = 0; + solves.length = 0; + fetches.length = 0; + solveDelay = 0; + fetchResult = null; + solveResult = { status: "passed", by: "auto", clearance: "GOOD", userAgent: BROWSER_UA, browserEgress: "direct" }; + await h.db + .updateTable("site") + .set({ config: JSON.stringify({}), health_status: "healthy" }) + .where("id", "=", siteId) + .execute(); + await h.db + .updateTable("site_account") + .set({ user_info: JSON.stringify({}), user_agent: null, ua_pending: null, status: "active" }) + .where("id", "=", accountId) + .execute(); + await h.rt.cloudflare.patch(siteId, {}); + }); + + it("自动通过:浏览器拿到 cf_clearance + UA,HTTP 带上重试 1 次成功,记 cfReuse = ok;之后的请求直接带", async () => { + const page = await list(); + expect(page.items.length).toBeGreaterThan(0); + expect(solves).toEqual([{ siteId, accountId, url: expect.stringContaining("/torrents.php") }]); + expect(cf.challenged).toBe(1); + expect(fetches).toEqual([]); + const st = await state(); + expect(st).toMatchObject({ lastSolvedBy: "auto", cfReuse: "ok", lastResult: "reuse_ok", browserUntil: null }); + expect(st?.lastChallengeAt).toBeTruthy(); + expect(st?.lastRule).toBe("header cf-mitigated=challenge"); + // cf_clearance 存在 secret 里,账号 user_info 只有引用与取得时的 UA / 出口 + const info = (await account()).user_info as { + cloudflare?: { secretRef: string; userAgent: string; egress: string }; + }; + expect(info.cloudflare).toMatchObject({ userAgent: BROWSER_UA, egress: "direct" }); + expect(h.secrets.plain.get(info.cloudflare!.secretRef)).toBe("GOOD"); + expect(JSON.stringify(info)).not.toContain("GOOD"); + // __cf_bm 不带;下一条请求直接带 cf_clearance,不再过挑战 + seen.length = 0; + await list(); + expect(solves).toHaveLength(1); + expect(seen[0]?.get("cookie")).toContain("cf_clearance=GOOD"); + expect(seen[0]?.get("cookie")).not.toContain("__cf_bm"); + // 账号状态不变 + expect((await account()).status).toBe("active"); + }); + + it("视觉子流程通过:记 lastSolvedBy = vision", async () => { + solveResult = { status: "passed", by: "vision", clearance: "GOOD", userAgent: BROWSER_UA, browserEgress: "direct" }; + await list(); + expect(await state()).toMatchObject({ lastSolvedBy: "vision", cfReuse: "ok" }); + }); + + it("复用失败:cf_clearance 给 HTTP 仍被挑战 → cfReuse = fails,本次由浏览器代抓,6 小时浏览器代理期内页面请求直接走浏览器", async () => { + cf.pass = () => false; // 站点绑定 TLS 指纹:HTTP 怎么都过不去 + const page = await list(); + expect(page.items.length).toBeGreaterThan(0); + expect(cf.challenged).toBe(2); // 首次 + 带 cf_clearance 重试 1 次 + expect(fetches).toHaveLength(1); + const st = await state(); + expect(st).toMatchObject({ cfReuse: "fails", lastResult: "reuse_fails" }); + const until = Date.parse(st!.browserUntil!); + expect(until - Date.now()).toBeGreaterThan(5.9 * 3_600_000); + // 代理期内:不发 HTTP,不再过挑战 + await list(); + expect(cf.challenged).toBe(2); + expect(fetches).toHaveLength(2); + expect(solves).toHaveLength(1); + // 下载(不是页面请求)仍走 HTTP,报 challenge,不带失败过的 cf_clearance + const err = await h.rt + .siteAdapter(await site()) + .downloadTarget(page.items[0]!.siteReleaseId) + .catch((e) => e); + expect(err).toBeInstanceOf(SiteError); + expect((err as SiteError).kind).toBe("challenge"); + expect(seen.at(-1)?.get("cookie") ?? "").not.toContain("cf_clearance"); + // 到期再试 HTTP + await h.rt.cloudflare.patch(siteId, { browserUntil: new Date(Date.now() - 1000).toISOString() }); + cf.pass = () => true; + const n = fetches.length; + await list(); + expect(fetches).toHaveLength(n); + }); + + it("真验证码转人工:本次报 challenge(不算凭据错误),站点记 pending = human;快照不置账号失效、不开登录失效事项", async () => { + solveResult = { status: "needs_human", sessionId: null, decisionId: null, reason: "选图验证码" }; + const r = await snapshotSite(h.rt, { siteId, kind: "manual" }); + expect(r.status).toBe("failed"); + expect(r.error).toContain("challenge"); + expect(fetches).toEqual([]); + expect(await state()).toMatchObject({ pending: "human", lastResult: "needs_human", lastError: "选图验证码" }); + expect((await account()).status).toBe("active"); + expect((await site()).health_status).toBe("degraded"); + const d = await h.db.selectFrom("decision").select("id").where("dedupe_key", "=", `site-auth:${siteId}`).execute(); + expect(d).toEqual([]); + expect(h.engine.enqueued.filter((e) => e.task === "refreshSiteAccount")).toEqual([]); + + // 人在原站视图里过完、交还:浏览器模块回调 afterHuman → 清掉等人处理,存下 cf_clearance,马上可以再试 + await h.rt.cloudflare.afterHuman(siteId, accountId, { + clearance: "GOOD", + userAgent: BROWSER_UA, + browserEgress: "direct", + }); + expect(await state()).toMatchObject({ pending: null, lastSolvedBy: "human", lastAttemptAt: null }); + const page = await list(); + expect(page.items.length).toBeGreaterThan(0); + expect(solves).toHaveLength(1); + }); + + it("节流与单飞:并发的挑战只过一次;30 分钟内不再过;超过 30 分钟再过", async () => { + solveDelay = 150; + cf.pass = () => false; + fetchResult = { status: "challenge", reason: "still" }; + const results = await Promise.allSettled([list(), list(), list()]); + expect(results.every((r) => r.status === "rejected")).toBe(true); + expect(solves).toHaveLength(1); + // 30 分钟内:只记挑战,不过 + await list().catch(() => null); + expect(solves).toHaveLength(1); + expect((await state())?.lastAttemptAt).toBeTruthy(); + // 超过 30 分钟 + await h.rt.cloudflare.patch(siteId, { + lastAttemptAt: new Date(Date.now() - CF_ATTEMPT_INTERVAL_MS - 1000).toISOString(), + }); + await list().catch(() => null); + expect(solves).toHaveLength(2); + }); + + it("同步条件不满足:账号手动 UA → 不把 cf_clearance 交给 HTTP(不重试),页面改由浏览器代抓", async () => { + await h.db.updateTable("site_account").set({ user_agent: BROWSER_UA }).where("id", "=", accountId).execute(); + const page = await list(); + expect(page.items.length).toBeGreaterThan(0); + expect(cf.challenged).toBe(1); + expect(fetches).toHaveLength(1); + expect(seen.every((x) => !(x.get("cookie") ?? "").includes("cf_clearance"))).toBe(true); + expect((await state())?.browserUntil).toBeTruthy(); + }); + + it("同步条件不满足:浏览器出口与 HTTP 不同(旧插件按环境变量出网)→ 不复用", async () => { + solveResult = { status: "passed", by: "auto", clearance: "GOOD", userAgent: BROWSER_UA, browserEgress: null }; + await list(); + expect(cf.challenged).toBe(1); + expect(fetches).toHaveLength(1); + expect((await state())?.cfReuse).toBeNull(); + }); + + it("没有登录态账号的站点不处理挑战", async () => { + await h.db.updateTable("site_account").set({ status: "expired" }).where("id", "=", accountId).execute(); + // 适配器没有 Cookie 就不发请求;这里直接用站点 fetch(RSS 这类不要登录态的请求同理) + const res = await h.rt.siteFetch(await site())(`${mock.baseUrl}/torrents.php`, { + headers: { accept: "text/html" }, + }); + expect(res.status).toBe(403); + expect(solves).toEqual([]); + expect(fetches).toEqual([]); + expect(await state()).toMatchObject({ lastResult: "skipped" }); + }); + + it("健康检测遇挑战且过不去:记 challenge(degraded),账号状态不变", async () => { + solveResult = { status: "failed", reason: "浏览器服务不可用" }; + fetchResult = { status: "unavailable", reason: "浏览器服务不可用" }; + const r = await json(await h.req(`/sites/${siteId}/health-check`, { method: "POST", body: {} })); + expect(r).toMatchObject({ ok: false, status: "challenge", healthStatus: "degraded", loggedIn: null }); + expect((await account()).status).toBe("active"); + }); +}); diff --git a/apps/server/test/sources-identity/egress.test.ts b/apps/server/test/sources-identity/egress.test.ts new file mode 100644 index 00000000..dc8222cf --- /dev/null +++ b/apps/server/test/sources-identity/egress.test.ts @@ -0,0 +1,133 @@ +import { DEFAULT_SETTINGS } from "@umt/contracts"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { + directHttpClient, + egressKey, + envProxyFor, + fetchVia, + noProxyMatches, + proxyHttpClient, + siteEgress, +} from "../../src/egress.ts"; +import { SourcesRuntime } from "../../src/modules/sources-identity/runtime.ts"; +import type { ServerContext } from "../../src/modules/types.ts"; + +// CF-2 出口统一:站点请求不再隐式继承容器的 HTTP(S)_PROXY。开了站点代理 → 该代理;否则明确直连。 +// 本地起一个"代理"(收到请求就回 via-proxy)和一个目标(回 direct),把 HTTP_PROXY 指向前者,看请求实际走哪条。 + +const PROXY_KEYS = [ + "HTTP_PROXY", + "HTTPS_PROXY", + "ALL_PROXY", + "http_proxy", + "https_proxy", + "all_proxy", + "NO_PROXY", + "no_proxy", +]; + +describe("出口选择(直连 / 代理)", () => { + let proxy: Deno.HttpServer; + let target: Deno.HttpServer; + let proxyUrl = ""; + let targetUrl = ""; + const proxied: string[] = []; + const saved = new Map(); + + beforeAll(() => { + proxy = Deno.serve({ hostname: "127.0.0.1", port: 0, onListen() {} }, (req) => { + proxied.push(req.url); + return new Response("via-proxy"); + }); + target = Deno.serve({ hostname: "127.0.0.1", port: 0, onListen() {} }, () => new Response("direct")); + proxyUrl = `http://127.0.0.1:${proxy.addr.port}`; + targetUrl = `http://127.0.0.1:${target.addr.port}/x`; + for (const k of PROXY_KEYS) { + saved.set(k, Deno.env.get(k)); + Deno.env.delete(k); + } + // 模拟容器环境:HTTP_PROXY 指向"中继",没有 NO_PROXY + Deno.env.set("HTTP_PROXY", proxyUrl); + }); + + afterAll(async () => { + for (const [k, v] of saved) { + if (v === undefined) Deno.env.delete(k); + else Deno.env.set(k, v); + } + await proxy.shutdown(); + await target.shutdown(); + }); + + const body = async (f: typeof fetch) => (await f(targetUrl)).text(); + + it("Deno 2.9 语义:createHttpClient({}) 仍会读 HTTP_PROXY(所以直连要显式做)", async () => { + const implicit = Deno.createHttpClient({}); + try { + expect(await body(fetchVia(implicit))).toBe("via-proxy"); + } finally { + implicit.close(); + } + }); + + it("directHttpClient:环境变量里有代理也直连;创建后环境变量原样还原", async () => { + proxied.length = 0; + expect(await body(fetchVia(directHttpClient()))).toBe("direct"); + expect(proxied).toEqual([]); + expect(Deno.env.get("HTTP_PROXY")).toBe(proxyUrl); + }); + + it("proxyHttpClient:只走给定代理(NO_PROXY 命中也不绕过)", async () => { + Deno.env.set("NO_PROXY", "127.0.0.1"); + try { + proxied.length = 0; + const client = proxyHttpClient(proxyUrl); + expect(await body(fetchVia(client))).toBe("via-proxy"); + expect(proxied).toHaveLength(1); + expect(Deno.env.get("NO_PROXY")).toBe("127.0.0.1"); + } finally { + Deno.env.delete("NO_PROXY"); + } + }); + + it("SourcesRuntime.siteFetch:站点没开代理 → 直连;开了且设了代理地址 → 站点代理", async () => { + const ctx = { logger: { debug() {}, info() {}, warn() {} } } as unknown as ServerContext; + const rt = new SourcesRuntime(ctx); + const defaults = { ...DEFAULT_SETTINGS.sites, proxyUrl }; + proxied.length = 0; + expect(await body(rt.siteFetch({ proxy: false }, defaults, "UA"))).toBe("direct"); + expect(proxied).toEqual([]); + expect(await body(rt.siteFetch({ proxy: true }, defaults, "UA"))).toBe("via-proxy"); + expect(proxied).toHaveLength(1); + // 开了代理但没设代理地址 → 直连 + expect(await body(rt.siteFetch({ proxy: true }, { ...defaults, proxyUrl: null }, "UA"))).toBe("direct"); + expect(rt.egressKeyOf({ proxy: false }, defaults)).toBe("direct"); + expect(rt.egressKeyOf({ proxy: true }, defaults)).toBe(`proxy:http://127.0.0.1:${proxy.addr.port}`); + }); +}); + +describe("外网请求按环境变量(行为不变,改为显式)", () => { + it("HTTPS / HTTP / ALL_PROXY 与 NO_PROXY", () => { + const env = { HTTPS_PROXY: "http://relay:7891", HTTP_PROXY: "http://relay:7890", NO_PROXY: "localhost,.lan" }; + expect(envProxyFor(new URL("https://api.themoviedb.org/3"), env)).toBe("http://relay:7891"); + expect(envProxyFor(new URL("http://image.tmdb.org/t"), env)).toBe("http://relay:7890"); + expect(envProxyFor(new URL("http://nas.lan/x"), env)).toBeNull(); + expect(envProxyFor(new URL("https://api.search.brave.com/"), { ALL_PROXY: "socks5h://relay:1080" })).toBe( + "socks5h://relay:1080", + ); + expect(envProxyFor(new URL("https://api.search.brave.com/"), {})).toBeNull(); + expect(envProxyFor(new URL("https://x.example/"), { https_proxy: "http://lower:1" })).toBe("http://lower:1"); + }); + it("NO_PROXY 匹配", () => { + expect(noProxyMatches("a.example.com", "example.com")).toBe(true); + expect(noProxyMatches("example.com", ".example.com")).toBe(true); + expect(noProxyMatches("badexample.com", "example.com")).toBe(false); + expect(noProxyMatches("x", "*")).toBe(true); + expect(noProxyMatches("127.0.0.1", "localhost, 127.0.0.1:8080")).toBe(true); + }); + it("出口键不含代理口令", () => { + expect(egressKey(siteEgress(false, "http://u:p@relay:7891"))).toBe("direct"); + expect(egressKey(siteEgress(true, "http://u:p@relay:7891"))).toBe("proxy:http://relay:7891"); + expect(egressKey(siteEgress(true, null))).toBe("direct"); + }); +}); diff --git a/packages/contracts/src/sources.ts b/packages/contracts/src/sources.ts index c40512a4..6caf9662 100644 --- a/packages/contracts/src/sources.ts +++ b/packages/contracts/src/sources.ts @@ -124,6 +124,41 @@ export const SiteLastHealth = z.object({ }); export type SiteLastHealth = z.infer; +/** + * 站点的 Cloudflare 处理状态(CF-2,存 site.config.cloudflare;docs/runbook/sites.md「Cloudflare」): + * - browserUntil:在此之前该站能走浏览器的页面请求直接由浏览器代抓(HTTP 仍被挑战时置 6 小时,到期再试 HTTP); + * - cfReuse:浏览器拿到的 cf_clearance 给 HTTP 用的结果(ok 有效 / fails 无效,之后不再复用); + * - lastSolvedBy:最近一次怎么过的挑战(auto 等它自己过 / vision 视觉子流程点了一下 / human 人在原站视图里过); + * - pending = human:遇到选图 / 输码类验证码,等人在原站视图里过一次; + * - lastAttemptAt:最近一次尝试过挑战(同站 30 分钟最多一次)。 + */ +export const SiteCloudflareState = z.object({ + browserUntil: Timestamp.nullable().default(null), + cfReuse: z.enum(["ok", "fails"]).nullable().default(null), + lastChallengeAt: Timestamp.nullable().default(null), + lastSolvedBy: z.enum(["auto", "vision", "human"]).nullable().default(null), + lastSolvedAt: Timestamp.nullable().default(null), + lastAttemptAt: Timestamp.nullable().default(null), + pending: z.enum(["human"]).nullable().default(null), + /** 等人处理的浏览器会话(原站视图接管入口) */ + sessionId: Id.nullable().default(null), + /** 最近一次处理结果(技术信息):passed / reuse_ok / reuse_fails / browser / needs_human / failed / throttled / skipped */ + lastResult: z.string().nullable().default(null), + /** 命中的挑战规则(技术信息) */ + lastRule: z.string().nullable().default(null), + /** 最近一次失败原因(技术信息) */ + lastError: z.string().nullable().default(null), +}); +export type SiteCloudflareState = z.infer; + +/** 从 site.config 读 Cloudflare 状态;没有或格式不对返回 null */ +export function siteCloudflareState(config: unknown): SiteCloudflareState | null { + const raw = (config as { cloudflare?: unknown } | null | undefined)?.cloudflare; + if (!raw || typeof raw !== "object") return null; + const r = SiteCloudflareState.safeParse(raw); + return r.success ? r.data : null; +} + export const Site = EntityBase.extend({ key: z.string().regex(/^[a-z][a-z0-9_-]*$/), name: z.string().min(1),