Merge branch 'cf2'
Co-Authored-By: Claude Fable 5.1 <[email protected]> # Conflicts: # docs/plan/00-first-version-plan.md
This commit is contained in:
root committed
commit
1ce937345e
36 files changed
+2609
-64
No files matched your search
@@ -4,6 +4,16 @@
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### 新增(CF-2 Cloudflare 验证处理)
|
||||
- 站点请求遇 Cloudflare 挑战(`cf-mitigated: challenge`,或 403 / 503 / 429 的挑战页)不再算凭据错误:不置账号失效、不开「登录失效」事项、不排自动重登;503 挑战不再算维护;健康检测记「需要人机验证」。
|
||||
- 过挑战级联(只处理有登录态账号的站点;同站 30 分钟最多一次、同一时刻只处理一次;不接打码服务):浏览器用该账号的 profile 打开同一地址 → 等它自己过(≤ 15 秒)→ 「我是人类」复选框 / Turnstile 由系统发起视觉子流程点一下(最多 2 次,预算取设置,自动化总开关关闭时跳过)→ 选图 / 输码类转人工,待处理事项「<站点>需要你在原站视图里过一次验证」,接管过完交还即继续。
|
||||
- 过了之后读回 `cf_clearance` 与浏览器 UA:账号 UA 跟随浏览器且两端出口一致时交给 HTTP(`__cf_bm` 仍不带),重试 1 次,按站点记复用是否有效;仍被挑战时这次页面请求改由浏览器导航取 HTML 交给原解析器,之后 6 小时该站页面请求直接走浏览器,到期再试 HTTP。
|
||||
- 站点卡片 / 概览显示「Cloudflare：自动通过 / 浏览器代理中，至 hh:mm / 需要你验证」,原始记录进技术信息。
|
||||
- camofox 插件新端点 `POST /umt/tabs/{tabId}/mouse-click`:`click_at` 发真实鼠标事件,跨域 iframe 里的 Turnstile 也点得到(需要重建 camofox 镜像)。
|
||||
|
||||
### 变更(CF-2 出口统一)
|
||||
- 站点请求(适配器、RSS、登录、用户数据)不再隐式继承容器的 `HTTP(S)_PROXY`:站点没开代理明确直连,开了走站点 / 全局代理,与浏览器会话同一出口。TMDB、Brave、图片缓存等外网请求照旧按环境变量走代理,改为代码里显式指定。出口表见 `docs/runbook/sites.md`「出口」。
|
||||
|
||||
## [0.3.4] - 2026-09-30
|
||||
|
||||
## [0.3.3] - 2026-09-30
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
// CF-2 出口统一:每条对外 HTTP 请求走哪条出口由代码显式决定,不再隐式继承容器的 HTTP(S)_PROXY。
|
||||
//
|
||||
// Deno 2.9 的语义(2026-09-30 本机实测,见 docs/runbook/sites.md「出口」):
|
||||
// - `fetch` 缺省客户端与 `Deno.createHttpClient({})` 都会读 HTTP_PROXY / HTTPS_PROXY / ALL_PROXY(大小写都认)与 NO_PROXY;
|
||||
// 读取发生在**客户端创建时**(缺省客户端在进程第一次 fetch 时创建);
|
||||
// - `createHttpClient` 没有"不走代理"的选项,`NO_PROXY=*` 也不认;显式 `proxy` 会排在环境变量代理前面,
|
||||
// 但 NO_PROXY 命中的主机连显式代理也绕过;
|
||||
// - 所以直连客户端的做法是:创建那一刻把这几个环境变量临时拿掉(同步调用,同一个 isolate 内不会被别的代码插进来),建完立即还原。
|
||||
// 代理客户端同样在拿掉环境变量后创建,保证只走给定的代理。
|
||||
//
|
||||
// 出口表:
|
||||
// - 站点请求(适配器、RSS、登录、用户数据):站点开"使用代理"且设了代理地址 → 该代理;否则直连(siteClient);
|
||||
// - TMDB、Brave、图片缓存等外网请求:照旧用环境变量里的代理(行为不变),但改成显式读 env 建客户端(externalFetch)。
|
||||
|
||||
const PROXY_ENV = ["HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy"] as const;
|
||||
const NO_PROXY_ENV = ["NO_PROXY", "no_proxy"] as const;
|
||||
|
||||
/** 进程启动时的代理环境变量快照(外网请求显式使用;之后的临时改动不影响) */
|
||||
const ENV_SNAPSHOT: Record<string, string | undefined> = (() => {
|
||||
const out: Record<string, string | undefined> = {};
|
||||
try {
|
||||
for (const k of [...PROXY_ENV, ...NO_PROXY_ENV]) out[k] = Deno.env.get(k);
|
||||
} catch {
|
||||
// 没有 env 权限:当作没有代理
|
||||
}
|
||||
return out;
|
||||
})();
|
||||
|
||||
/** 在"没有代理环境变量"的状态下同步执行 fn(建 HttpClient 用),结束立即还原 */
|
||||
function withoutProxyEnv<T>(fn: () => T): T {
|
||||
const saved = new Map<string, string>();
|
||||
try {
|
||||
for (const k of [...PROXY_ENV, ...NO_PROXY_ENV]) {
|
||||
const v = Deno.env.get(k);
|
||||
if (v !== undefined) {
|
||||
saved.set(k, v);
|
||||
Deno.env.delete(k);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// 没有 env 权限时也没有可读的代理变量,直接建
|
||||
}
|
||||
try {
|
||||
return fn();
|
||||
} finally {
|
||||
for (const [k, v] of saved) Deno.env.set(k, v);
|
||||
}
|
||||
}
|
||||
|
||||
let direct: Deno.HttpClient | null = null;
|
||||
const proxied = new Map<string, Deno.HttpClient>();
|
||||
|
||||
/** 明确直连的客户端(不读任何代理环境变量) */
|
||||
export function directHttpClient(): Deno.HttpClient {
|
||||
direct ??= withoutProxyEnv(() => Deno.createHttpClient({}));
|
||||
return direct;
|
||||
}
|
||||
|
||||
/** 只走给定代理的客户端(同一地址复用) */
|
||||
export function proxyHttpClient(url: string): Deno.HttpClient {
|
||||
let c = proxied.get(url);
|
||||
if (!c) {
|
||||
c = withoutProxyEnv(() => Deno.createHttpClient({ proxy: { url } }));
|
||||
proxied.set(url, c);
|
||||
}
|
||||
return c;
|
||||
}
|
||||
|
||||
/** 用指定客户端发请求的 fetch */
|
||||
export function fetchVia(client: Deno.HttpClient): typeof globalThis.fetch {
|
||||
return ((input: RequestInfo | URL, init?: RequestInit) =>
|
||||
globalThis.fetch(input, { ...init, client } as RequestInit)) as typeof globalThis.fetch;
|
||||
}
|
||||
|
||||
/** 站点请求的出口:开代理且有代理地址 → proxy,否则 direct */
|
||||
export interface SiteEgress {
|
||||
mode: "direct" | "proxy";
|
||||
proxyUrl: string | null;
|
||||
}
|
||||
|
||||
export function siteEgress(proxy: boolean, proxyUrl: string | null | undefined): SiteEgress {
|
||||
return proxy && proxyUrl ? { mode: "proxy", proxyUrl } : { mode: "direct", proxyUrl: null };
|
||||
}
|
||||
|
||||
/** 出口的比较键(不含代理口令):direct / proxy:<主机:端口> */
|
||||
export function egressKey(e: SiteEgress): string {
|
||||
if (e.mode === "direct" || !e.proxyUrl) return "direct";
|
||||
try {
|
||||
const u = new URL(e.proxyUrl);
|
||||
return `proxy:${u.protocol}//${u.host}`;
|
||||
} catch {
|
||||
return "proxy:?";
|
||||
}
|
||||
}
|
||||
|
||||
export function siteFetchFor(e: SiteEgress): typeof globalThis.fetch {
|
||||
return fetchVia(e.mode === "proxy" && e.proxyUrl ? proxyHttpClient(e.proxyUrl) : directHttpClient());
|
||||
}
|
||||
|
||||
// ---------- 外网请求(TMDB / Brave / 图片):显式按环境变量 ----------
|
||||
|
||||
/** NO_PROXY 匹配(与常见实现一致:逗号分隔;`.a.com` / `a.com` 匹配自身与子域;`*` 匹配全部;IP 精确匹配) */
|
||||
export function noProxyMatches(host: string, noProxy: string | undefined): boolean {
|
||||
if (!noProxy) return false;
|
||||
const h = host.toLowerCase().replace(/^\[|\]$/g, "");
|
||||
for (const raw of noProxy.split(",")) {
|
||||
const p = raw.trim().toLowerCase().replace(/:\d+$/, "");
|
||||
if (!p) continue;
|
||||
if (p === "*") return true;
|
||||
const d = p.replace(/^\*?\./, "");
|
||||
if (h === d || h.endsWith(`.${d}`)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** 外网请求按环境变量应走的代理(进程启动时的快照);没有 / NO_PROXY 命中为 null */
|
||||
export function envProxyFor(target: URL, env: Record<string, string | undefined> = ENV_SNAPSHOT): string | null {
|
||||
const get = (k: string) => env[k] ?? env[k.toLowerCase()];
|
||||
if (noProxyMatches(target.hostname, get("NO_PROXY"))) return null;
|
||||
const byScheme = target.protocol === "https:" ? get("HTTPS_PROXY") : get("HTTP_PROXY");
|
||||
const p = byScheme || get("ALL_PROXY");
|
||||
return p ? p : null;
|
||||
}
|
||||
|
||||
/** 外网请求用的 fetch:按环境变量显式选代理 / 直连(与之前隐式继承的行为一致) */
|
||||
export const externalFetch: typeof globalThis.fetch = ((input: RequestInfo | URL, init?: RequestInit) => {
|
||||
let target: URL | null = null;
|
||||
try {
|
||||
target = new URL(input instanceof Request ? input.url : String(input));
|
||||
} catch {
|
||||
// 交给 fetch 报错
|
||||
}
|
||||
const proxy = target ? envProxyFor(target) : null;
|
||||
const client = proxy ? proxyHttpClient(proxy) : directHttpClient();
|
||||
return globalThis.fetch(input, { ...init, client } as RequestInit);
|
||||
}) as typeof globalThis.fetch;
|
||||
@@ -20,7 +20,7 @@ import {
|
||||
} from "@umt/contracts";
|
||||
import type { z } from "zod";
|
||||
import { browserServiceFor } from "../browser/index.ts";
|
||||
import { BrowserServiceError } from "../browser/service.ts";
|
||||
import { BrowserServiceError, type ChallengeVision, registerChallengeVision } from "../browser/service.ts";
|
||||
import { resolveRunUpstream } from "../pi/index.ts";
|
||||
import type { ServerContext } from "../types.ts";
|
||||
import { parseFakeScripts } from "./fake-scripts.ts";
|
||||
@@ -62,7 +62,41 @@ const FALLBACK_SCRIPT: FakeScript = {
|
||||
const isUuid = (s: string) => /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(s);
|
||||
const clip = (s: string, n: number) => (s.length > n ? `${s.slice(0, n)}…` : s);
|
||||
|
||||
/**
|
||||
* CF-2:系统发起的视觉子流程(站点 HTTP 遇 Cloudflare 复选框 / Turnstile 时由浏览器模块调用)。没有父会话:
|
||||
* 发起人记 system:cloudflare,子会话 trigger = event;预算取设置(智能体单次 token / 费用上限,时长 ≤ 120 秒)。
|
||||
* 自动化总开关由浏览器模块先查过;子流程里的坐标动作照常经注册中心授权(spawn_reason 放行)。
|
||||
*/
|
||||
export function systemChallengeVision(env: () => AgentEnv, registry: CapabilityRegistry): ChallengeVision {
|
||||
return async (req) => {
|
||||
const { ctx } = env();
|
||||
const s = await ctx.settings.all();
|
||||
const input: Input = {
|
||||
sessionId: req.sessionId,
|
||||
goal: req.goal,
|
||||
// 成功与否浏览器模块还会按页面再核实一次;这里给子流程一个可判定的条件
|
||||
success: { textAbsent: req.textAbsent ?? "Just a moment" },
|
||||
maxSteps: Math.min(12, Math.max(1, req.maxSteps)),
|
||||
budget: {
|
||||
maxDurationSec: Math.min(120, s.agent.maxDurationSec),
|
||||
...(s.agent.maxTokensPerRun ? { maxTokens: Math.max(1000, s.agent.maxTokensPerRun) } : {}),
|
||||
...(s.agent.maxCostUsdPerRun ? { maxCostUsd: s.agent.maxCostUsdPerRun } : {}),
|
||||
},
|
||||
};
|
||||
const hctx = {
|
||||
actorId: "system:cloudflare",
|
||||
taskId: `cloudflare:${req.sessionId}`,
|
||||
scopeIds: [],
|
||||
signal: AbortSignal.timeout(180_000),
|
||||
source: "task",
|
||||
} as HandlerContext;
|
||||
const out = await solveChallenge(env(), registry, input, hctx);
|
||||
return { status: out.status, reason: out.reason, childRunId: out.childRunId };
|
||||
};
|
||||
}
|
||||
|
||||
export function registerChallengeCapability(registry: CapabilityRegistry, env: () => AgentEnv): void {
|
||||
registerChallengeVision(env().ctx, systemChallengeVision(env, registry));
|
||||
registry.register(capabilitySpec("browser.solve_challenge"), async (input, ctx) => {
|
||||
try {
|
||||
return await solveChallenge(env(), registry, input, ctx);
|
||||
@@ -249,7 +283,7 @@ async function runChild(
|
||||
const child = await ctx.db
|
||||
.insertInto("agent_run")
|
||||
.values({
|
||||
trigger: parent ? "subagent" : "user",
|
||||
trigger: parent ? "subagent" : hctx.actorId.startsWith("system:") ? "event" : "user",
|
||||
goal,
|
||||
status: "running",
|
||||
scope: JSON.stringify({ capabilities: [...VISION_CAPABILITIES], allowedKinds: CHILD_KINDS }),
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
import { siteCloudflareState } from "@umt/contracts";
|
||||
import { MOCK_CAMOFOX_IDENTITY, type MockCamofoxSite } from "@umt/testkit";
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest";
|
||||
import { browserServiceFor } from "../index.ts";
|
||||
import { type BrowserService, registerChallengeVision } from "../service.ts";
|
||||
import { type BrowserHarness, createBrowserHarness, type Loose, ORIGIN, pgAvailable } from "./harness.ts";
|
||||
|
||||
// CF-2 浏览器一侧(对 testkit mock camofox 的 Cloudflare 模式):
|
||||
// ① 自动放行(轮询页面)② 复选框 / Turnstile 派视觉子流程,用插件真实鼠标点击 ③ 选图验证码 / 点不过 → 会话转等待 + 待处理事项;
|
||||
// 人在原站视图里过完交还 → 读回 cf_clearance / UA 通知站点请求继续;浏览器代抓页面受 profile 锁与站点限流约束。
|
||||
|
||||
const up = await pgAvailable();
|
||||
const GOOD = "cf2-session-value";
|
||||
const CLEARANCE = "clr-from-browser";
|
||||
const LIST_HTML =
|
||||
'<html><head><title>种子</title></head><body><table class="torrents"><tr><td>Oppenheimer</td></tr></table></body></html>';
|
||||
|
||||
describe.skipIf(!up)("CF-2 浏览器过挑战 / 代抓", () => {
|
||||
let h: BrowserHarness;
|
||||
let svc: BrowserService;
|
||||
const cfCfg: NonNullable<MockCamofoxSite["cloudflare"]> = { mode: "auto", autoPolls: 2, clearance: CLEARANCE };
|
||||
const visionCalls: string[] = [];
|
||||
let visionClick = true;
|
||||
const url = `${ORIGIN}/torrents.php`;
|
||||
|
||||
const sessions = async () => (await h.req("/browser/sessions")).body.data.items as Loose[];
|
||||
const closeAll = async () => {
|
||||
for (const s of await sessions()) await h.req(`/browser/sessions/${s.id}`, { method: "DELETE" });
|
||||
};
|
||||
const profileKey = () => `umt-site-${h.siteId}-acct-${h.accountId}`;
|
||||
|
||||
beforeAll(async () => {
|
||||
h = await createBrowserHarness("cf2", {
|
||||
sessionCookie: { name: "c_secure_pass", valid: [GOOD], fresh: GOOD },
|
||||
cloudflare: cfCfg,
|
||||
html: (u) =>
|
||||
u.includes("torrents.php") ? LIST_HTML : "<html><head><title>首页</title></head><body></body></html>",
|
||||
browserOptions: { challengePollMs: 5, challengeAutoWaitMs: 200, settleMs: 0 },
|
||||
});
|
||||
await sessions(); // 第一次请求时浏览器模块才为这个上下文建服务
|
||||
svc = browserServiceFor(h.ctx)!;
|
||||
const ref = await h.secrets.create({ purpose: "site_cookie", plaintext: `c_secure_uid=1; c_secure_pass=${GOOD}` });
|
||||
await h.db
|
||||
.updateTable("site_account")
|
||||
.set({ auth_kind: "cookie", secret_ref: ref, session_secret_ref: null, status: "active" })
|
||||
.where("id", "=", h.accountId)
|
||||
.execute();
|
||||
// 视觉子流程(假):点一下复选框所在的位置(第 2 行)
|
||||
registerChallengeVision(h.ctx, async (req) => {
|
||||
visionCalls.push(req.sessionId);
|
||||
if (visionClick)
|
||||
await svc.visionAct(
|
||||
req.sessionId,
|
||||
{ kind: "automation", holder: "system:cloudflare" },
|
||||
(p, ref) => p.click(ref, { x: 100, y: 70 }),
|
||||
{ wantImage: false, sensitiveAllowed: false },
|
||||
);
|
||||
return { status: visionClick ? "solved" : "needs_human", reason: "", childRunId: null };
|
||||
});
|
||||
}, 120_000);
|
||||
afterAll(async () => {
|
||||
await h?.close();
|
||||
});
|
||||
beforeEach(async () => {
|
||||
await closeAll();
|
||||
h.mock.cfPassed.clear();
|
||||
h.mock.contentReads.clear();
|
||||
h.mock.actions.length = 0;
|
||||
for (const jar of h.mock.cookieJars.values())
|
||||
jar.splice(0, jar.length, ...jar.filter((c) => c.name !== "cf_clearance"));
|
||||
cfCfg.mode = "auto";
|
||||
visionCalls.length = 0;
|
||||
visionClick = true;
|
||||
await h.db
|
||||
.updateTable("site")
|
||||
.set({ config: JSON.stringify({}), enabled: true })
|
||||
.where("id", "=", h.siteId)
|
||||
.execute();
|
||||
await h.db
|
||||
.updateTable("settings")
|
||||
.set({ value: JSON.stringify({ enabled: true }) })
|
||||
.where("key", "=", "automation")
|
||||
.execute();
|
||||
});
|
||||
|
||||
it("① 自动放行:轮询到挑战页消失,读回 cf_clearance + navigator.userAgent + 出口", async () => {
|
||||
const r = await svc.solveSiteChallenge({ siteId: h.siteId, accountId: h.accountId, url });
|
||||
expect(r).toEqual({
|
||||
status: "passed",
|
||||
by: "auto",
|
||||
clearance: CLEARANCE,
|
||||
userAgent: MOCK_CAMOFOX_IDENTITY.userAgent,
|
||||
browserEgress: "direct",
|
||||
});
|
||||
expect(visionCalls).toEqual([]);
|
||||
// 会话仍归自动化,没有待处理事项
|
||||
const [s] = await sessions();
|
||||
expect(s.controller).toBe("automation");
|
||||
});
|
||||
|
||||
it("② 复选框 / Turnstile:等不过 → 视觉子流程用插件真实鼠标点击 → 通过(by vision)", async () => {
|
||||
cfCfg.mode = "interactive";
|
||||
const r = await svc.solveSiteChallenge({ siteId: h.siteId, accountId: h.accountId, url });
|
||||
expect(r).toMatchObject({ status: "passed", by: "vision", clearance: CLEARANCE });
|
||||
expect(visionCalls).toHaveLength(1);
|
||||
expect(h.mock.actions.some((a) => a.action === "mouse-click")).toBe(true);
|
||||
});
|
||||
|
||||
it("③ 视觉子流程点不过(最多 2 次)→ 会话转等待,待处理事项「观众需要你在原站视图里过一次验证」", async () => {
|
||||
cfCfg.mode = "interactive";
|
||||
visionClick = false;
|
||||
const r = await svc.solveSiteChallenge({ siteId: h.siteId, accountId: h.accountId, url });
|
||||
expect(r).toMatchObject({ status: "needs_human" });
|
||||
expect(visionCalls).toHaveLength(2);
|
||||
const s = (await h.req(`/browser/sessions/${(r as Loose).sessionId}`)).body.data as Loose;
|
||||
expect(s.controller).toBe("waiting");
|
||||
expect(s.waitingReason).toBe("cf_challenge");
|
||||
const d = await h.db
|
||||
.selectFrom("decision")
|
||||
.selectAll()
|
||||
.where("id", "=", (r as Loose).decisionId)
|
||||
.executeTakeFirstOrThrow();
|
||||
expect(d.title).toBe("观众需要你在原站视图里过一次验证");
|
||||
expect(d.status).toBe("awaiting_user");
|
||||
expect((d.evidence as Loose).cloudflare).toMatchObject({ siteId: h.siteId });
|
||||
// 已在等人:再来一次不打扰,直接返回同一个等待
|
||||
const again = await svc.solveSiteChallenge({ siteId: h.siteId, accountId: h.accountId, url });
|
||||
expect(again).toMatchObject({ status: "needs_human", sessionId: (r as Loose).sessionId });
|
||||
expect(visionCalls).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("自动化总开关关闭:不派视觉子流程,直接转人工", async () => {
|
||||
cfCfg.mode = "interactive";
|
||||
await h.db
|
||||
.insertInto("settings")
|
||||
.values({ key: "automation", value: JSON.stringify({ enabled: false }) })
|
||||
.onConflict((oc) => oc.column("key").doUpdateSet({ value: JSON.stringify({ enabled: false }) }))
|
||||
.execute();
|
||||
const r = await svc.solveSiteChallenge({ siteId: h.siteId, accountId: h.accountId, url });
|
||||
expect(r).toMatchObject({ status: "needs_human" });
|
||||
expect(visionCalls).toEqual([]);
|
||||
});
|
||||
|
||||
it("③ 选图 / 输码类:不点、不等,直接转人工;人过完交还 → 读回 cf_clearance,站点请求继续", async () => {
|
||||
cfCfg.mode = "captcha";
|
||||
const r = (await svc.solveSiteChallenge({ siteId: h.siteId, accountId: h.accountId, url })) as Loose;
|
||||
expect(r.status).toBe("needs_human");
|
||||
expect(r.reason).toContain("选图");
|
||||
expect(visionCalls).toEqual([]);
|
||||
// 站点请求一侧记下"等人处理"(级联在 sources-identity 里做,这里直接写状态)
|
||||
await h.db
|
||||
.updateTable("site")
|
||||
.set({ config: JSON.stringify({ cloudflare: { pending: "human", sessionId: r.sessionId } }) })
|
||||
.where("id", "=", h.siteId)
|
||||
.execute();
|
||||
// 人接管,在桌面里过了验证(桌面操作不经 API:直接改 mock 状态),交还
|
||||
expect(
|
||||
(await h.req(`/browser/sessions/${r.sessionId}/control/acquire`, { body: { reason: "过验证" } })).status,
|
||||
).toBe(200);
|
||||
h.mock.cfPassed.add(profileKey());
|
||||
h.mock.cookieJars.get(profileKey())!.push({
|
||||
name: "cf_clearance",
|
||||
value: "clr-human",
|
||||
domain: ".audiences.example",
|
||||
path: "/",
|
||||
expires: Math.floor(Date.now() / 1000) + 3600,
|
||||
httpOnly: true,
|
||||
secure: true,
|
||||
});
|
||||
expect((await h.req(`/browser/sessions/${r.sessionId}/control/release`, { body: {} })).status).toBe(200);
|
||||
const site = await h.db.selectFrom("site").select("config").where("id", "=", h.siteId).executeTakeFirstOrThrow();
|
||||
expect(siteCloudflareState(site.config)).toMatchObject({ pending: null, lastSolvedBy: "human" });
|
||||
const acc = await h.db
|
||||
.selectFrom("site_account")
|
||||
.select("user_info")
|
||||
.where("id", "=", h.accountId)
|
||||
.executeTakeFirstOrThrow();
|
||||
const info = (acc.user_info as Loose).cloudflare;
|
||||
expect(info).toMatchObject({ userAgent: MOCK_CAMOFOX_IDENTITY.userAgent, egress: "direct" });
|
||||
expect(h.secrets.plain.get(info.secretRef)).toBe("clr-human");
|
||||
// 待处理事项已关闭
|
||||
const d = await h.db
|
||||
.selectFrom("decision")
|
||||
.select("status")
|
||||
.where("id", "=", r.decisionId)
|
||||
.executeTakeFirstOrThrow();
|
||||
expect(d.status).toBe("resolved");
|
||||
});
|
||||
|
||||
it("浏览器代抓:过了挑战的 profile 导航取 outerHTML;同一 profile 并发串行(只开一个会话);每次导航取站点令牌", async () => {
|
||||
h.mock.cfPassed.add(profileKey());
|
||||
const [a, b] = await Promise.all([
|
||||
svc.fetchSiteHtml({ siteId: h.siteId, accountId: h.accountId, url }),
|
||||
svc.fetchSiteHtml({ siteId: h.siteId, accountId: h.accountId, url }),
|
||||
]);
|
||||
expect(a).toMatchObject({ status: "ok", html: LIST_HTML });
|
||||
expect(b).toMatchObject({ status: "ok", html: LIST_HTML });
|
||||
expect(await sessions()).toHaveLength(1);
|
||||
const navs = h.mock.calls.filter(
|
||||
(c) => c.path.endsWith("/navigate") || (c.path === "/tabs" && c.method === "POST"),
|
||||
);
|
||||
expect(navs.length).toBeGreaterThanOrEqual(2);
|
||||
const count = await h.db
|
||||
.selectFrom("site_request_count")
|
||||
.select((eb) => eb.fn.sum<number>("count").as("n"))
|
||||
.where("site_id", "=", h.siteId)
|
||||
.executeTakeFirst();
|
||||
expect(Number(count?.n ?? 0)).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
|
||||
it("浏览器代抓:仍是挑战页 → challenge;会话在人工接管中 → 不打扰;站点停用 → 不可用", async () => {
|
||||
cfCfg.mode = "captcha";
|
||||
expect(await svc.fetchSiteHtml({ siteId: h.siteId, accountId: h.accountId, url })).toMatchObject({
|
||||
status: "challenge",
|
||||
});
|
||||
const [s] = await sessions();
|
||||
await h.req(`/browser/sessions/${s.id}/control/acquire`, { body: { reason: "看看" } });
|
||||
expect(await svc.fetchSiteHtml({ siteId: h.siteId, accountId: h.accountId, url })).toMatchObject({
|
||||
status: "unavailable",
|
||||
});
|
||||
await h.req(`/browser/sessions/${s.id}/control/release`, { body: {} });
|
||||
await h.db.updateTable("site").set({ enabled: false }).where("id", "=", h.siteId).execute();
|
||||
expect(await svc.fetchSiteHtml({ siteId: h.siteId, accountId: h.accountId, url })).toMatchObject({
|
||||
status: "unavailable",
|
||||
reason: "站点已停用",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -3,7 +3,13 @@ import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { CapabilityRegistry, InvokeContext } from "@umt/agent";
|
||||
import { type DB, type Kysely, migrateToLatest, sql } from "@umt/db";
|
||||
import { buildFakeTorrent, createMockCamofox, type FakeTorrent, type MockCamofox } from "@umt/testkit";
|
||||
import {
|
||||
buildFakeTorrent,
|
||||
createMockCamofox,
|
||||
type FakeTorrent,
|
||||
type MockCamofox,
|
||||
type MockCamofoxSite,
|
||||
} from "@umt/testkit";
|
||||
import { Kysely as K, PostgresDialect } from "kysely";
|
||||
import pg from "pg";
|
||||
import { withSchemaLock } from "../../../../test/auth-settings/harness.ts";
|
||||
@@ -19,6 +25,7 @@ import { createSourcesIdentityModule } from "../../sources-identity/index.ts";
|
||||
import type { ServerContext } from "../../types.ts";
|
||||
import { registerBrowserCapabilities } from "../capabilities.ts";
|
||||
import { createBrowserModule } from "../index.ts";
|
||||
import type { BrowserModuleOptions } from "../service.ts";
|
||||
|
||||
// C1-E 浏览器模块测试装配:独立 PG schema + 真实迁移 + 交互式 mock camofox(观众风格站点)+ 真实能力注册中心。
|
||||
|
||||
@@ -61,6 +68,11 @@ export async function createBrowserHarness(
|
||||
sessionCookie?: { name: string; valid: string[]; fresh: string };
|
||||
cookieApi?: "plugin" | "upstream" | "none";
|
||||
cookieImportKey?: string;
|
||||
/** CF-2:mock 站点的 Cloudflare 挑战与页面 HTML */
|
||||
cloudflare?: MockCamofoxSite["cloudflare"];
|
||||
html?: MockCamofoxSite["html"];
|
||||
/** 浏览器模块的额外选项(挑战轮询间隔等) */
|
||||
browserOptions?: Partial<BrowserModuleOptions>;
|
||||
} = {},
|
||||
): Promise<BrowserHarness> {
|
||||
const torrent = buildFakeTorrent({ name: "Oppenheimer.2023.2160p.UHD.BluRay.x265-GRP" });
|
||||
@@ -70,6 +82,8 @@ export async function createBrowserHarness(
|
||||
...(opts.cookieImportKey ? { cookieImportKey: opts.cookieImportKey } : {}),
|
||||
site: {
|
||||
...(opts.sessionCookie ? { sessionCookie: opts.sessionCookie } : {}),
|
||||
...(opts.cloudflare ? { cloudflare: opts.cloudflare } : {}),
|
||||
...(opts.html ? { html: opts.html } : {}),
|
||||
origin: ORIGIN,
|
||||
username: "fixture-user",
|
||||
password: PASSWORD,
|
||||
@@ -110,7 +124,12 @@ export async function createBrowserHarness(
|
||||
capabilities: registry,
|
||||
} as ServerContext;
|
||||
const sources = createSourcesIdentityModule({ tmdb: () => null });
|
||||
const browser = createBrowserModule({ runtime: (c) => sources.runtime(c), stagingDir, downloadPollMs: 20 });
|
||||
const browser = createBrowserModule({
|
||||
runtime: (c) => sources.runtime(c),
|
||||
stagingDir,
|
||||
downloadPollMs: 20,
|
||||
...opts.browserOptions,
|
||||
});
|
||||
registerBrowserCapabilities(registry, () => browser.service(ctx));
|
||||
const app = createApp({
|
||||
config: { NODE_ENV: "development", DEV_ADMIN_KEY: DEV_KEY },
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
type BrowserProvider,
|
||||
type BrowserSecretResolver,
|
||||
CAMOFOX_VERIFIED_RELEASES,
|
||||
classifyChallengePage,
|
||||
cookiesFromHeader,
|
||||
createCamofoxBrowserProvider,
|
||||
type DownloadSink,
|
||||
@@ -12,7 +13,9 @@ import {
|
||||
getSiteDefinition,
|
||||
headerFromCookies,
|
||||
hostAllowed,
|
||||
InMemoryProfileLock,
|
||||
imageDimensions,
|
||||
type PageContent,
|
||||
type PageMetrics,
|
||||
parseTorrent,
|
||||
prepareImage,
|
||||
@@ -50,8 +53,14 @@ import {
|
||||
type SiteRow,
|
||||
sql,
|
||||
} from "@umt/db";
|
||||
import { egressKey, siteEgress } from "../../egress.ts";
|
||||
import type { SourcesRuntime } from "../sources-identity/runtime.ts";
|
||||
import { refreshSiteAccount } from "../sources-identity/service/account.ts";
|
||||
import {
|
||||
type BrowserFetchOutcome,
|
||||
type ChallengeSolveOutcome,
|
||||
registerChallengeBrowser,
|
||||
} from "../sources-identity/service/cloudflare.ts";
|
||||
import {
|
||||
readAccountCookie,
|
||||
registerBrowserCookieReader,
|
||||
@@ -85,6 +94,10 @@ export interface BrowserModuleOptions {
|
||||
evidenceDir?: string;
|
||||
/** 坐标动作之后等页面稳定再截图的毫秒数;缺省 400 */
|
||||
settleMs?: number;
|
||||
/** CF-2 等挑战页放行时读页面的间隔(毫秒);缺省 1500 */
|
||||
challengePollMs?: number;
|
||||
/** CF-2 等 Cloudflare 自己放行的最长时间(毫秒);缺省 CF_AUTO_WAIT_MS(15 秒) */
|
||||
challengeAutoWaitMs?: number;
|
||||
/** 保活间隔(AG-4);缺省 BROWSER_KEEPALIVE_MS 配置(120 秒),0 = 关闭 */
|
||||
keepaliveMs?: number;
|
||||
/** 测试替换 provider 实现 */
|
||||
@@ -175,8 +188,39 @@ export interface RecoveryState {
|
||||
reconnect?: BrowserReconnectState;
|
||||
/** 最近一次自动化(Agent 能力)动作成功的时间;原站视图据此区分「Agent 正在操作」与「无人操作」 */
|
||||
lastAutomationAt?: string;
|
||||
/** CF-2:打开 / 重连时下发给浏览器的出口键(egressKey);插件没按站点设出口(旧镜像 / env 模式)为 null */
|
||||
egress?: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* CF-2 视觉子流程钩子(agent 模块注册):系统发起、没有父会话,预算取设置;只点「我是人类」复选框 / Turnstile 一步。
|
||||
* 返回子流程结论(solved 仍要由调用方按页面核实)。
|
||||
*/
|
||||
export type ChallengeVision = (req: {
|
||||
sessionId: string;
|
||||
goal: string;
|
||||
/** 通过后页面上不再出现的文字(成功条件) */
|
||||
textAbsent: string | null;
|
||||
maxSteps: number;
|
||||
}) => Promise<{ status: string; reason: string; childRunId: string | null }>;
|
||||
|
||||
const visions = new WeakMap<object, ChallengeVision>();
|
||||
|
||||
export function registerChallengeVision(ctx: object, fn: ChallengeVision): void {
|
||||
visions.set(ctx, fn);
|
||||
}
|
||||
|
||||
/** 系统(Cloudflare 处理)发起的浏览器动作 */
|
||||
const CF_ACTOR: Actor = { kind: "automation", holder: "system:cloudflare" };
|
||||
/** 等 Cloudflare 自己放行的最长时间 */
|
||||
export const CF_AUTO_WAIT_MS = 15_000;
|
||||
/** 视觉子流程点过之后再等多久 */
|
||||
const CF_AFTER_CLICK_WAIT_MS = 10_000;
|
||||
/** 视觉子流程最多派几次 */
|
||||
const CF_VISION_ATTEMPTS = 2;
|
||||
/** 视觉子流程步数上限:截图 → 点击 → 等待 */
|
||||
const CF_VISION_STEPS = 4;
|
||||
|
||||
/** 同一浏览器服务的实例标识多久查一次 */
|
||||
const INSTANCE_CHECK_MS = 5_000;
|
||||
/** 自动重连失败后,操作触发的重连至少间隔多久 */
|
||||
@@ -221,6 +265,11 @@ export class BrowserService {
|
||||
registerBrowserCookieReader(ctx, (siteId, accountId) => this.readProfileCookie(siteId, accountId));
|
||||
// 站点请求 UA 跟随浏览器固定身份(BR-4):sources-identity 经钩子取
|
||||
registerBrowserIdentityReader(ctx, () => this.identityUserAgent());
|
||||
// CF-2:站点 HTTP 遇 Cloudflare 挑战时由浏览器过挑战 / 代抓页面
|
||||
registerChallengeBrowser(ctx, {
|
||||
solve: (req) => this.solveSiteChallenge(req),
|
||||
fetchHtml: (req) => this.fetchSiteHtml(req),
|
||||
});
|
||||
}
|
||||
|
||||
// ---------- BR-4 浏览器身份与出口 ----------
|
||||
@@ -283,8 +332,8 @@ export class BrowserService {
|
||||
* 出口按站点统一(BR-4):站点开了代理且设置了代理地址 → 该 profile 走同一代理;否则直连。
|
||||
* 旧镜像(插件没有出口接口)只记日志,浏览器照旧走容器环境变量里的代理。
|
||||
*/
|
||||
private async applyEgress(provider: BrowserProvider, site: SiteRow, profileKey: string): Promise<void> {
|
||||
if (!provider.setEgress) return;
|
||||
private async applyEgress(provider: BrowserProvider, site: SiteRow, profileKey: string): Promise<string | null> {
|
||||
if (!provider.setEgress) return null;
|
||||
const rt = this.opts.runtime?.(this.ctx);
|
||||
const proxyUrl = rt
|
||||
? (await rt.siteDefaults()).proxyUrl
|
||||
@@ -297,6 +346,8 @@ export class BrowserService {
|
||||
{ component: "browser", siteId: site.id, egress: proxy ? "proxy" : "direct" },
|
||||
"浏览器会话出口已变更，按新出口重建",
|
||||
);
|
||||
// 插件 env 模式:浏览器按容器环境变量出网,与站点 HTTP(明确直连 / 站点代理)不一定一致
|
||||
return r.mode && r.mode !== "site" ? null : egressKey(siteEgress(site.proxy, proxyUrl));
|
||||
} catch (e) {
|
||||
this.ctx.logger.warn(
|
||||
{
|
||||
@@ -307,6 +358,7 @@ export class BrowserService {
|
||||
},
|
||||
"设置浏览器会话出口失败，浏览器按容器环境变量出网",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -851,7 +903,7 @@ export class BrowserService {
|
||||
for (const m of await provider.listSessions())
|
||||
if (m.profileKey === profileKey && m.controller !== "closed") provider.setController(m.id, "closed");
|
||||
const prep = await this.prepareCookies(site, accountId);
|
||||
await this.applyEgress(provider, site, profileKey);
|
||||
const egress = await this.applyEgress(provider, site, profileKey);
|
||||
await this.refreshIdentity(prow, { maxAgeMs: 10 * 60_000 }).catch(() => null);
|
||||
await this.rateLimit(site);
|
||||
let res: Awaited<ReturnType<BrowserProvider["openSession"]>>;
|
||||
@@ -904,6 +956,7 @@ export class BrowserService {
|
||||
needsSnapshot: true,
|
||||
credential,
|
||||
instanceId,
|
||||
egress,
|
||||
}),
|
||||
})
|
||||
.execute();
|
||||
@@ -1141,7 +1194,11 @@ export class BrowserService {
|
||||
}
|
||||
|
||||
/** 转等待处理:自动化动作被拒,生成待处理事项,等人在原站视图接管 */
|
||||
async enterWaiting(row: BrowserSessionRow, reason: string): Promise<BrowserSessionRow> {
|
||||
async enterWaiting(
|
||||
row: BrowserSessionRow,
|
||||
reason: string,
|
||||
text?: { title: string; summary: string },
|
||||
): Promise<BrowserSessionRow> {
|
||||
const provider = this.providers.get(row.provider_id)?.provider;
|
||||
if (provider?.sessionInfo(row.id)) provider.setController(row.id, "waiting");
|
||||
const site = await this.liveSite(row.site_id).catch(() => null);
|
||||
@@ -1154,8 +1211,8 @@ export class BrowserService {
|
||||
kind: "other",
|
||||
status: "awaiting_user",
|
||||
severity: "warning",
|
||||
title: `${site?.name ?? "站点"}：原站需要人工处理`,
|
||||
summary: `${REASON_TEXT[reason] ?? reason}。在站点详情的原站视图接管会话，完成后交还。`,
|
||||
title: text?.title ?? `${site?.name ?? "站点"}：原站需要人工处理`,
|
||||
summary: text?.summary ?? `${REASON_TEXT[reason] ?? reason}。在站点详情的原站视图接管会话，完成后交还。`,
|
||||
subject_type: "browser_session",
|
||||
subject_id: row.id,
|
||||
dedupe_key: `browser.waiting:${row.id}`,
|
||||
@@ -1327,6 +1384,14 @@ export class BrowserService {
|
||||
await this.persist(id, provider);
|
||||
// 人工在桌面 / 截图模式里登录过:交还时读回登录态写进站点账号,全站(HTTP 抓取 / RSS / 快照)生效
|
||||
await this.syncCookies(id, "release");
|
||||
// CF-2:等人过 Cloudflare 验证的会话交还后,读回 cf_clearance / UA,站点请求继续
|
||||
if (opts.to === "automation")
|
||||
await this.afterHumanChallenge(id).catch((e) =>
|
||||
this.ctx.logger.warn(
|
||||
{ component: "browser", sessionId: id, err: String((e as Error)?.message ?? e) },
|
||||
"交还后读回 Cloudflare 验证结果失败",
|
||||
),
|
||||
);
|
||||
return toLease(released);
|
||||
}
|
||||
|
||||
@@ -1586,7 +1651,7 @@ export class BrowserService {
|
||||
if (!provider.reopenSession) throw await fail("浏览器服务不支持重新连接");
|
||||
const url = restoreUrl(row.current_url, budget) ?? site.base_url;
|
||||
const prep = await this.prepareCookies(site, row.account_id).catch(() => null);
|
||||
await this.applyEgress(provider, site, row.profile_key);
|
||||
const egress = await this.applyEgress(provider, site, row.profile_key);
|
||||
// 浏览器重启过(可能换了镜像版本):身份重新读
|
||||
await this.refreshIdentity(prow).catch(() => null);
|
||||
let res: Awaited<ReturnType<NonNullable<BrowserProvider["reopenSession"]>>>;
|
||||
@@ -1606,6 +1671,7 @@ export class BrowserService {
|
||||
needsReconnect: false,
|
||||
needsSnapshot: true,
|
||||
instanceId: instanceId ?? rs.instanceId ?? null,
|
||||
egress,
|
||||
reconnect: { status: "ok", trigger, at, reason: null, humanNotice: row.controller === "human" },
|
||||
});
|
||||
this.coordSpaces.delete(id);
|
||||
@@ -1894,6 +1960,248 @@ export class BrowserService {
|
||||
return { decisionId, controller: row.controller };
|
||||
}
|
||||
|
||||
// ---------- CF-2 Cloudflare 挑战 ----------
|
||||
|
||||
/** 过挑战 / 代抓同一 profile 串行(本进程;跨进程的同站互斥靠 site.config.cloudflare.lastAttemptAt 条件更新) */
|
||||
private readonly cfLock = new InMemoryProfileLock(240_000);
|
||||
|
||||
/** 读当前页面内容(只读,不要求自动化控制权);上游不支持为 null */
|
||||
private async readPage(id: string): Promise<PageContent | null> {
|
||||
return this.run(id, CF_ACTOR, (p, ref) => (p.pageContent ? p.pageContent(ref) : Promise.resolve(null)), {
|
||||
write: false,
|
||||
persist: false,
|
||||
background: true,
|
||||
});
|
||||
}
|
||||
|
||||
/** 轮询页面直到不是挑战页(或变成真验证码 / 超时);返回最后一次读到的页面与分类 */
|
||||
private async waitChallenge(
|
||||
id: string,
|
||||
page: PageContent | null,
|
||||
ms: number,
|
||||
): Promise<{
|
||||
page: PageContent | null;
|
||||
kind: ReturnType<typeof classifyChallengePage>["kind"];
|
||||
rule: string | null;
|
||||
}> {
|
||||
let cur = page ?? (await this.readPage(id));
|
||||
let c = classifyChallengePage(cur ?? {});
|
||||
const deadline = Date.now() + ms;
|
||||
while (cur && c.kind !== "none" && c.kind !== "captcha" && Date.now() < deadline) {
|
||||
await new Promise((r) => setTimeout(r, this.opts.challengePollMs ?? 1_500));
|
||||
cur = await this.readPage(id);
|
||||
c = classifyChallengePage(cur ?? {});
|
||||
}
|
||||
return { page: cur, kind: cur ? c.kind : "none", rule: c.rule };
|
||||
}
|
||||
|
||||
/**
|
||||
* 该账号 profile 的自动化会话:已有且在自动化控制下就用它(导航到 url,取站点令牌);没有就打开一个(打开即导航)。
|
||||
* 人工接管中 / 等人处理中返回 null(不打扰人)。
|
||||
*/
|
||||
private async cfSession(
|
||||
site: SiteRow,
|
||||
accountId: string,
|
||||
url: string,
|
||||
): Promise<{ id: string } | { busy: BrowserSessionRow }> {
|
||||
const live = await this.liveSessionFor(site.id, accountId);
|
||||
if (live && live.controller !== "automation") return { busy: live };
|
||||
if (!live) return { id: await this.open({ siteId: site.id, accountId, url }) };
|
||||
await this.run(live.id, CF_ACTOR, async (p, ref, x) => {
|
||||
await this.rateLimit(x.site);
|
||||
await p.navigate(ref, url, x.budget);
|
||||
});
|
||||
return { id: live.id };
|
||||
}
|
||||
|
||||
/** 会话转"等你过验证":待处理事项写明是哪个站点、去哪里处理 */
|
||||
private async cfNeedsHuman(id: string, site: SiteRow, reason: string): Promise<ChallengeSolveOutcome> {
|
||||
const text = {
|
||||
title: `${site.name}需要你在原站视图里过一次验证`,
|
||||
summary:
|
||||
"站点要求人机验证（Cloudflare），系统没能自动通过。在站点详情的原站视图接管会话，完成验证后交还，站点请求会继续。",
|
||||
};
|
||||
let row = await this.sessionRow(id);
|
||||
if (row.controller === "automation") row = await this.enterWaiting(row, "cf_challenge", text);
|
||||
else if (row.controller === "waiting") {
|
||||
const rs = obj(row.recovery_state) as RecoveryState;
|
||||
if (!rs.waitingDecisionId) row = await this.enterWaiting(row, "cf_challenge", text);
|
||||
else {
|
||||
// 视觉子流程没成功时已转等待(challenge_unsolved):事项改成 Cloudflare 的说法
|
||||
await this.ctx.db
|
||||
.updateTable("decision")
|
||||
.set({ title: text.title, summary: text.summary })
|
||||
.where("id", "=", rs.waitingDecisionId)
|
||||
.execute();
|
||||
row = await this.patchRecovery(row, { waitingReason: "cf_challenge" });
|
||||
}
|
||||
}
|
||||
const decisionId = (obj(row.recovery_state) as RecoveryState).waitingDecisionId ?? null;
|
||||
if (decisionId)
|
||||
await this.ctx.db
|
||||
.updateTable("decision")
|
||||
.set({
|
||||
evidence: sql`evidence || ${JSON.stringify({ cloudflare: { siteId: site.id, reason } })}::jsonb`,
|
||||
})
|
||||
.where("id", "=", decisionId)
|
||||
.execute();
|
||||
this.ctx.logger.info({ component: "browser", sessionId: id, siteId: site.id, reason }, "Cloudflare 验证转人工");
|
||||
return { status: "needs_human", sessionId: id, decisionId, reason };
|
||||
}
|
||||
|
||||
/** 读回 profile 里的 cf_clearance 值(只取作用于站点主机的) */
|
||||
private async cfClearance(profileKey: string, siteUrl: string): Promise<string | null> {
|
||||
const prow = await this.providerRow().catch(() => null);
|
||||
const provider = prow ? this.providerFor(prow) : null;
|
||||
if (!provider?.getCookies) return null;
|
||||
const list = await provider.getCookies(profileKey, [siteUrl]).catch(() => []);
|
||||
const host = new URL(siteUrl).hostname;
|
||||
const hit = list.find(
|
||||
(c) =>
|
||||
c.name === "cf_clearance" &&
|
||||
(host === c.domain.replace(/^\./, "") || host.endsWith(`.${c.domain.replace(/^\./, "")}`)) &&
|
||||
!(c.expires !== undefined && c.expires > 0 && c.expires * 1000 <= this.now().getTime()),
|
||||
);
|
||||
return hit?.value ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 过挑战(CF-2 级联):用该账号 profile 打开同一地址 → ① 等它自己过(≤15 秒)→ ② 复选框 / Turnstile 派视觉子流程点一下
|
||||
* (最多 2 次,系统发起,预算取设置;自动化总开关关闭时跳过)→ ③ 选图 / 输码类或仍没过:会话转等待,开待处理事项。
|
||||
* 通过后读回 cf_clearance 与 navigator.userAgent,连同浏览器出口交给调用方。
|
||||
*/
|
||||
async solveSiteChallenge(req: { siteId: string; accountId: string; url: string }): Promise<ChallengeSolveOutcome> {
|
||||
const site = await this.liveSite(req.siteId);
|
||||
if (!site.enabled) return { status: "failed", reason: "站点已停用" };
|
||||
const profileKey = profileKeyFor(site.id, req.accountId);
|
||||
return this.cfLock.run(profileKey, async () => {
|
||||
const got = await this.cfSession(site, req.accountId, req.url);
|
||||
if ("busy" in got) {
|
||||
const rs = obj(got.busy.recovery_state) as RecoveryState;
|
||||
if (got.busy.controller === "waiting" && rs.waitingReason === "cf_challenge")
|
||||
return {
|
||||
status: "needs_human",
|
||||
sessionId: got.busy.id,
|
||||
decisionId: rs.waitingDecisionId ?? null,
|
||||
reason: "已在等你过验证",
|
||||
};
|
||||
return { status: "failed", reason: "浏览器会话正由人工处理，没有打扰" };
|
||||
}
|
||||
const id = got.id;
|
||||
const opened = await this.sessionRow(id);
|
||||
if (opened.controller !== "automation")
|
||||
return { status: "failed", reason: "浏览器会话打开后在等人处理（登录已失效）" };
|
||||
// ① 等它自己过
|
||||
let w = await this.waitChallenge(id, null, this.opts.challengeAutoWaitMs ?? CF_AUTO_WAIT_MS);
|
||||
if (!w.page) return { status: "failed", reason: "浏览器服务不支持读取页面内容" };
|
||||
let by: "auto" | "vision" = "auto";
|
||||
// ② 复选框 / Turnstile:视觉子流程点一下
|
||||
if (w.kind === "interactive") {
|
||||
const vision = visions.get(this.ctx);
|
||||
const auto = await this.ctx.settings.get("automation").catch(() => null);
|
||||
if (vision && auto?.enabled && !auto.readOnlyReconcile) {
|
||||
const marker = /verify you are human|确认您是真人|验证您是真人|我是人类/i.exec(w.page.text)?.[0] ?? null;
|
||||
for (let i = 0; i < CF_VISION_ATTEMPTS && w.kind === "interactive"; i++) {
|
||||
const r = await vision({
|
||||
sessionId: id,
|
||||
goal: "这是 Cloudflare 人机验证页。只点一次「确认您是真人 / Verify you are human」复选框（Turnstile），然后等待页面跳转；不要做其它操作。如果页面要求选图片、输入验证码或别的操作，立即停止并说明需要人工。",
|
||||
textAbsent: marker,
|
||||
maxSteps: CF_VISION_STEPS,
|
||||
}).catch((e) => ({ status: "failed", reason: String((e as Error)?.message ?? e), childRunId: null }));
|
||||
this.ctx.logger.info(
|
||||
{ component: "browser", sessionId: id, siteId: site.id, attempt: i + 1, status: r.status },
|
||||
"Cloudflare 复选框视觉子流程结束",
|
||||
);
|
||||
// 子流程没成功时会话已转等待(challenge_unsolved);下一次子流程开始时自己收回自动化控制
|
||||
w = await this.waitChallenge(
|
||||
id,
|
||||
null,
|
||||
Math.min(CF_AFTER_CLICK_WAIT_MS, this.opts.challengeAutoWaitMs ?? CF_AFTER_CLICK_WAIT_MS),
|
||||
);
|
||||
if (w.kind === "none") by = "vision";
|
||||
}
|
||||
}
|
||||
}
|
||||
// ③ 没过
|
||||
if (w.kind !== "none")
|
||||
return this.cfNeedsHuman(
|
||||
id,
|
||||
site,
|
||||
w.kind === "captcha" ? "选图 / 输入验证码类验证，不自动处理" : `没能自动通过(${w.rule ?? w.kind})`,
|
||||
);
|
||||
// 通过:子流程成功时关掉它留下的待处理;读回 cf_clearance 与 UA
|
||||
const row = await this.sessionRow(id);
|
||||
if (row.controller === "waiting") {
|
||||
await this.resolveWaiting(row, "system:cloudflare");
|
||||
const { provider } = await this.loaded(row);
|
||||
provider.setController(id, "automation");
|
||||
await this.ctx.db
|
||||
.updateTable("browser_session")
|
||||
.set({
|
||||
controller: "automation",
|
||||
recovery_state: JSON.stringify({
|
||||
...obj(row.recovery_state),
|
||||
waitingReason: null,
|
||||
waitingDecisionId: null,
|
||||
}),
|
||||
updated_at: this.now(),
|
||||
})
|
||||
.where("id", "=", id)
|
||||
.execute();
|
||||
}
|
||||
const rs = obj((await this.sessionRow(id)).recovery_state) as RecoveryState;
|
||||
return {
|
||||
status: "passed",
|
||||
by,
|
||||
clearance: await this.cfClearance(profileKey, site.base_url),
|
||||
userAgent: w.page?.userAgent ?? (await this.identityUserAgent().catch(() => null)),
|
||||
browserEgress: rs.egress ?? null,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 浏览器代抓一页(CF-2 回退):该账号 profile 的自动化会话导航到 url(取站点令牌,受 profile 锁约束),读 outerHTML 交给原解析器。
|
||||
* 站点停用、会话正由人工处理 / 等人处理、仍是挑战页时不返回页面。
|
||||
*/
|
||||
async fetchSiteHtml(req: { siteId: string; accountId: string; url: string }): Promise<BrowserFetchOutcome> {
|
||||
const site = await this.liveSite(req.siteId).catch(() => null);
|
||||
if (!site?.enabled) return { status: "unavailable", reason: "站点已停用" };
|
||||
const profileKey = profileKeyFor(site.id, req.accountId);
|
||||
try {
|
||||
return await this.cfLock.run(profileKey, async () => {
|
||||
const got = await this.cfSession(site, req.accountId, req.url);
|
||||
if ("busy" in got) return { status: "unavailable" as const, reason: "浏览器会话正由人工处理或在等人处理" };
|
||||
const w = await this.waitChallenge(got.id, null, this.opts.challengeAutoWaitMs ?? CF_AUTO_WAIT_MS);
|
||||
if (!w.page) return { status: "unavailable" as const, reason: "浏览器服务不支持读取页面内容" };
|
||||
if (w.kind !== "none") return { status: "challenge" as const, reason: w.rule ?? w.kind };
|
||||
return { status: "ok" as const, html: w.page.html, url: w.page.url };
|
||||
});
|
||||
} catch (e) {
|
||||
const err = fromBrowserError(e);
|
||||
return { status: "unavailable", reason: String((err as Error)?.message ?? err).slice(0, 300) };
|
||||
}
|
||||
}
|
||||
|
||||
/** 人过完验证交还后:站点在等人过验证、会话是那一个 → 读回 cf_clearance / UA,通知站点请求继续 */
|
||||
private async afterHumanChallenge(id: string): Promise<void> {
|
||||
const row = await this.sessionRow(id);
|
||||
if (!row.account_id) return;
|
||||
const rt = this.opts.runtime?.(this.ctx);
|
||||
if (!rt) return;
|
||||
const st = await rt.cloudflare.state(row.site_id, true);
|
||||
if (st.pending !== "human") return;
|
||||
const site = await this.liveSite(row.site_id);
|
||||
const page = await this.readPage(id).catch(() => null);
|
||||
if (page && classifyChallengePage(page).kind !== "none") return; // 还在挑战页:继续等
|
||||
const rs = obj(row.recovery_state) as RecoveryState;
|
||||
await rt.cloudflare.afterHuman(row.site_id, row.account_id, {
|
||||
clearance: await this.cfClearance(row.profile_key, site.base_url),
|
||||
userAgent: page?.userAgent ?? (await this.identityUserAgent().catch(() => null)),
|
||||
browserEgress: rs.egress ?? null,
|
||||
});
|
||||
}
|
||||
|
||||
// ---------- 下载 ----------
|
||||
|
||||
/** 暂存产物核验:解析种子、按 infohash / 站点资源 ID 匹配本地资源、会话内去重 */
|
||||
@@ -1972,6 +2280,7 @@ export class BrowserService {
|
||||
}
|
||||
|
||||
const REASON_TEXT: Record<string, string> = {
|
||||
cf_challenge: "站点要求人机验证（Cloudflare），需要你过一次",
|
||||
captcha: "登录页需要验证码",
|
||||
challenge: "页面需要人工验证",
|
||||
two_factor: "需要二次验证或本人设备确认",
|
||||
@@ -2054,7 +2363,7 @@ export function toLease(r: BrowserControlLeaseRow): BrowserControlLease {
|
||||
}
|
||||
|
||||
/** 视觉子流程可以接手的等待原因(验证码 / 页面挑战 / 上次没处理成功) */
|
||||
const SOLVABLE = ["captcha", "challenge", "challenge_unsolved"];
|
||||
const SOLVABLE = ["captcha", "challenge", "challenge_unsolved", "cf_challenge"];
|
||||
|
||||
/** capture / visionAct 的结果(能力输出的公共部分) */
|
||||
export interface VisionShot {
|
||||
|
||||
+3
-1
@@ -10,6 +10,7 @@ import {
|
||||
} from "@umt/contracts";
|
||||
import type { z } from "zod";
|
||||
import { dataPaths } from "../../config.ts";
|
||||
import { externalFetch } from "../../egress.ts";
|
||||
import { requireScope } from "../../http/auth.ts";
|
||||
import { errorBody } from "../../http/errors.ts";
|
||||
import { CACHE_REGIONS, sharedCache, storageSettings, tmdbCachePolicy } from "../../shared-cache.ts";
|
||||
@@ -73,7 +74,8 @@ export function createCacheModule(opts: CacheModuleOptions = {}): ServerModule &
|
||||
if (!c) {
|
||||
c = new ImageCache({
|
||||
dir: opts.imageDir ?? join(dataPaths(ctx.config).dataDir, "cache", "images"),
|
||||
...(opts.imageFetch ? { fetch: opts.imageFetch } : {}),
|
||||
// 外网请求:显式按环境变量走代理(CF-2 出口表)
|
||||
fetch: opts.imageFetch ?? externalFetch,
|
||||
...(opts.imageUpstreamBase ? { upstreamBase: opts.imageUpstreamBase } : {}),
|
||||
});
|
||||
imageCaches.set(ctx, c);
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
IdParam,
|
||||
MetadataEnrichOutput,
|
||||
} from "@umt/contracts";
|
||||
import { externalFetch } from "../../../egress.ts";
|
||||
import { requireScope } from "../../../http/auth.ts";
|
||||
import { ApiException, errorBody } from "../../../http/errors.ts";
|
||||
import type { ModuleApp } from "../../types.ts";
|
||||
@@ -122,7 +123,10 @@ export function enrichRoutes(app: ModuleApp, rt: () => SourcesRuntime): void {
|
||||
const srcHost = hostOf(s.data.fields.posterUrl?.sourceUrl ?? "");
|
||||
if (!srcHost || !domainAllowed(srcHost, identity.enrichDomains ?? []))
|
||||
return c.json(errorBody("forbidden", "海报出处不在补全白名单里"), 403);
|
||||
const got = await fetchPublicImage(r.overrides.fetch ?? fetch, url, { maxBytes: POSTER_MAX_BYTES, label: "海报" });
|
||||
const got = await fetchPublicImage(r.overrides.fetch ?? externalFetch, url, {
|
||||
maxBytes: POSTER_MAX_BYTES,
|
||||
label: "海报",
|
||||
});
|
||||
if (!got.ok) return c.json(errorBody("upstream_unavailable", got.message), 502);
|
||||
return imageResponse(got, 86400 * 7);
|
||||
});
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { Id, WorkbenchReleaseDetail, WorkbenchReleasePage, WorkbenchReleaseQuery } from "@umt/contracts";
|
||||
import { z } from "zod";
|
||||
import { externalFetch } from "../../../egress.ts";
|
||||
import { requireScope } from "../../../http/auth.ts";
|
||||
import { errorBody } from "../../../http/errors.ts";
|
||||
import type { ModuleApp } from "../../types.ts";
|
||||
@@ -55,7 +56,7 @@ export function workbenchRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
|
||||
const r = rt();
|
||||
const req = await detailScreenshotRequest(r, rid, index);
|
||||
if (!req) return c.json(errorBody("not_found", "截图不存在或详情缓存已过期"), 404);
|
||||
const got = await fetchPublicImage(r.overrides.fetch ?? fetch, req.url, {
|
||||
const got = await fetchPublicImage(r.overrides.fetch ?? externalFetch, req.url, {
|
||||
maxBytes: SHOT_MAX_BYTES,
|
||||
label: "截图",
|
||||
...(req.referer ? { referer: req.referer } : {}),
|
||||
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
} from "@umt/adapters";
|
||||
import { DEFAULT_SETTINGS, type SitesSettings } from "@umt/contracts";
|
||||
import { pgTokenBucketFor, type SiteRow, sql } from "@umt/db";
|
||||
import { egressKey, externalFetch, siteEgress, siteFetchFor } from "../../egress.ts";
|
||||
import {
|
||||
CACHE_REGIONS,
|
||||
renewTrackedTmdbDetail,
|
||||
@@ -24,6 +25,7 @@ import {
|
||||
tmdbDetailRef,
|
||||
} from "../../shared-cache.ts";
|
||||
import type { ServerContext } from "../types.ts";
|
||||
import { CloudflareGate, withClearance } from "./service/cloudflare.ts";
|
||||
import { readBrowserUserAgent, resolveUserAgent, type UserAgentDefaults } from "./service/user-agent.ts";
|
||||
|
||||
// 模块运行时依赖:站点适配器与限流器(每站一个实例,令牌在 PG 跨进程共享)、RSS 抓取器、TMDB 客户端、时钟。
|
||||
@@ -58,12 +60,15 @@ export class SourcesRuntime {
|
||||
private siteDefaultsCache: SitesSettings = DEFAULT_SETTINGS.sites;
|
||||
private siteDefaultsAt = 0;
|
||||
private tmdbSig: string | null = null;
|
||||
/** CF-2 Cloudflare 挑战处理(站点请求经 siteFetch 进来) */
|
||||
readonly cloudflare: CloudflareGate;
|
||||
|
||||
constructor(
|
||||
readonly ctx: ServerContext,
|
||||
readonly overrides: RuntimeOverrides = {},
|
||||
) {
|
||||
this.now = overrides.now ?? (() => new Date());
|
||||
this.cloudflare = new CloudflareGate(this);
|
||||
this.rss = createRssFetcher({
|
||||
...(overrides.fetch ? { fetch: overrides.fetch } : {}),
|
||||
now: this.now,
|
||||
@@ -101,7 +106,7 @@ export class SourcesRuntime {
|
||||
const { cfg } = await this.tmdbConfig();
|
||||
if (!cfg) return null;
|
||||
let n = 0;
|
||||
const base = cfg.fetch ?? globalThis.fetch;
|
||||
const base = cfg.fetch ?? externalFetch;
|
||||
const counted: typeof fetch = (input, init) => {
|
||||
n++;
|
||||
return base(input, init);
|
||||
@@ -144,7 +149,8 @@ export class SourcesRuntime {
|
||||
...(apiKey ? { apiKey } : {}),
|
||||
...(accessToken ? { accessToken } : {}),
|
||||
...(baseUrl ? { baseUrl } : {}),
|
||||
...(this.overrides.fetch ? { fetch: this.overrides.fetch } : {}),
|
||||
// 外网请求:显式按环境变量走代理(CF-2 出口表;测试注入优先)
|
||||
fetch: this.overrides.fetch ?? externalFetch,
|
||||
// 请求响应进 PG 缓存(region tmdb.request),三进程共享;TTL / 条数上限 / 空结果短 TTL 取设置(对照 MP)
|
||||
cache: {
|
||||
ttlMs: policy.ttl * 1000,
|
||||
@@ -168,40 +174,53 @@ export class SourcesRuntime {
|
||||
}
|
||||
|
||||
/**
|
||||
* 站点请求用的 fetch:测试注入优先;站点开了"使用代理"且设置了 proxyUrl 时走代理(Deno.createHttpClient)。
|
||||
* 站点请求用的 fetch:测试注入优先;站点开了"使用代理"且设置了 proxyUrl 时走代理,否则**明确直连**
|
||||
* (CF-2:不再隐式继承容器的 HTTP(S)_PROXY,见 egress.ts)。
|
||||
* BR-4:每条请求都带 UA(请求自己设了就不动;否则 ua 参数,缺省按"跟随浏览器"),不再发 Deno 缺省的 `Deno/x.y.z`。
|
||||
* ua 可以是函数:每次请求现场取(站点适配器按当前账号取,账号切换 UA 后立即生效)。
|
||||
* CF-2:给了 site.id 的请求经 Cloudflare 处理(浏览器代理期直接走浏览器、带可复用的 cf_clearance、遇挑战过挑战 / 重试 / 浏览器代抓)。
|
||||
*/
|
||||
siteFetch(
|
||||
site: Pick<SiteRow, "proxy">,
|
||||
site: Pick<SiteRow, "proxy"> & Partial<Pick<SiteRow, "id">>,
|
||||
defaults: SitesSettings = this.siteDefaultsCache,
|
||||
ua?: string | (() => Promise<string>),
|
||||
): typeof globalThis.fetch {
|
||||
const base = this.transportFetch(site, defaults);
|
||||
return ((input: RequestInfo | URL, init?: RequestInit) => {
|
||||
const egress = this.egressKeyOf(site, defaults);
|
||||
const siteId = site.id;
|
||||
return (async (input: RequestInfo | URL, init?: RequestInit) => {
|
||||
const headers = new Headers(init?.headers ?? (input instanceof Request ? input.headers : undefined));
|
||||
if (headers.has("user-agent")) return base(input, { ...init, headers });
|
||||
const pick = typeof ua === "function" ? ua() : ua !== undefined ? Promise.resolve(ua) : this.defaultUserAgent();
|
||||
return pick.then((v) => {
|
||||
headers.set("user-agent", v);
|
||||
return base(input, { ...init, headers });
|
||||
});
|
||||
if (!headers.has("user-agent")) {
|
||||
const pick = typeof ua === "function" ? ua() : ua !== undefined ? Promise.resolve(ua) : this.defaultUserAgent();
|
||||
headers.set("user-agent", await pick);
|
||||
}
|
||||
if (!siteId) return base(input, { ...init, headers });
|
||||
const url = input instanceof Request ? input.url : String(input);
|
||||
const method = init?.method ?? (input instanceof Request ? input.method : "GET");
|
||||
return this.cloudflare.handle(
|
||||
siteId,
|
||||
{ url, method, accept: headers.get("accept"), userAgent: headers.get("user-agent"), egress },
|
||||
(extra) => {
|
||||
const h = new Headers(headers);
|
||||
const cookie = withClearance(h.get("cookie"), extra);
|
||||
if (cookie) h.set("cookie", cookie);
|
||||
// 过挑战可能超过单次请求的超时:重试时原信号已中止就换一个新的超时(调用方取消照样生效在级联之外)
|
||||
const signal = init?.signal?.aborted ? AbortSignal.timeout(30_000) : init?.signal;
|
||||
return base(input, { ...init, headers: h, ...(signal ? { signal } : {}) });
|
||||
},
|
||||
);
|
||||
}) as typeof globalThis.fetch;
|
||||
}
|
||||
|
||||
/** 不加 UA 的底层 fetch(代理 / 测试注入) */
|
||||
/** 站点 HTTP 出口键(direct / proxy:<主机>),与浏览器会话的出口同一口径 */
|
||||
egressKeyOf(site: Pick<SiteRow, "proxy">, defaults: SitesSettings = this.siteDefaultsCache): string {
|
||||
return egressKey(siteEgress(site.proxy, defaults.proxyUrl));
|
||||
}
|
||||
|
||||
/** 不加 UA 的底层 fetch(测试注入 / 按站点出口:代理或明确直连) */
|
||||
private transportFetch(site: Pick<SiteRow, "proxy">, defaults: SitesSettings): typeof globalThis.fetch {
|
||||
if (this.overrides.fetch) return this.overrides.fetch;
|
||||
if (!site.proxy || !defaults.proxyUrl) return globalThis.fetch;
|
||||
const key = defaults.proxyUrl;
|
||||
let client = this.proxyClients.get(key);
|
||||
if (!client) {
|
||||
client = Deno.createHttpClient({ proxy: { url: key } });
|
||||
this.proxyClients.set(key, client);
|
||||
}
|
||||
const c = client;
|
||||
return ((input: RequestInfo | URL, init?: RequestInit) =>
|
||||
globalThis.fetch(input, { ...init, client: c } as RequestInit)) as typeof globalThis.fetch;
|
||||
return siteFetchFor(siteEgress(site.proxy, defaults.proxyUrl));
|
||||
}
|
||||
|
||||
// ---------- BR-4 站点请求 UA ----------
|
||||
@@ -253,8 +272,6 @@ export class SourcesRuntime {
|
||||
return resolveUserAgent(acc, d).ua;
|
||||
}
|
||||
|
||||
private readonly proxyClients = new Map<string, Deno.HttpClient>();
|
||||
|
||||
/**
|
||||
* 站点适配器:每站一个实例,列表/搜索/详情/下载共享同一个限流器。
|
||||
* 站点限流参数、基址、代理或站点默认变化时重建(签名比较)。
|
||||
@@ -270,10 +287,12 @@ export class SourcesRuntime {
|
||||
}
|
||||
const defaults = this.siteDefaultsCache;
|
||||
const proxy = site.proxy ?? false;
|
||||
// Cloudflare 处理状态(config.cloudflare)经常变,不参与重建判断
|
||||
const { cloudflare: _cf, ...configSig } = (site.config ?? {}) as Record<string, unknown>;
|
||||
const sig = JSON.stringify([
|
||||
site.base_url,
|
||||
site.rate_limit,
|
||||
site.config,
|
||||
configSig,
|
||||
proxy,
|
||||
proxy ? defaults.proxyUrl : null,
|
||||
defaults.defaultRateLimit,
|
||||
@@ -304,7 +323,7 @@ export class SourcesRuntime {
|
||||
limiter,
|
||||
now: this.now,
|
||||
// 每条请求都经 siteFetch:代理 + 按账号现场取 UA(BR-4)
|
||||
fetch: this.siteFetch({ proxy }, defaults, () => this.siteUserAgent(site.id)),
|
||||
fetch: this.siteFetch({ id: site.id, proxy }, defaults, () => this.siteUserAgent(site.id)),
|
||||
...(baseUrl ? { baseUrl } : {}),
|
||||
});
|
||||
this.adapters.set(site.id, { sig, limiter, adapter });
|
||||
|
||||
@@ -688,8 +688,9 @@ async function markFailure(
|
||||
): Promise<SiteLoginResult> {
|
||||
const now = rt.now();
|
||||
const lastError = { reason, message: message.slice(0, 300), at: now.toISOString() };
|
||||
// Cloudflare 挑战(CF-2)不是凭据问题:账号状态不变、不开"登录失效"事项;过挑战由 Cloudflare 处理负责,需要人时它开事项
|
||||
const status =
|
||||
reason === "network" || reason === "unsupported"
|
||||
reason === "network" || reason === "unsupported" || reason === "challenge"
|
||||
? null
|
||||
: reason === "auth_expired" && acc.auth_kind === "api_key"
|
||||
? "invalid"
|
||||
|
||||
@@ -0,0 +1,431 @@
|
||||
import { detectChallenge, mergeCookieHeaders, parseCookieHeader } from "@umt/adapters";
|
||||
import { type SiteCloudflareState, siteCloudflareState } from "@umt/contracts";
|
||||
import { type SiteAccountRow, type SiteRow, sql } from "@umt/db";
|
||||
import { egressKey, siteEgress } from "../../../egress.ts";
|
||||
import type { ServerContext } from "../../types.ts";
|
||||
import type { SourcesRuntime } from "../runtime.ts";
|
||||
import { resolveUserAgent } from "./user-agent.ts";
|
||||
|
||||
// CF-2 Cloudflare 挑战处理(docs/design/cloudflare-handling.md;运维见 docs/runbook/sites.md「Cloudflare」)。
|
||||
//
|
||||
// 站点 HTTP 请求(SourcesRuntime.siteFetch)经这里:
|
||||
// 0. 站点在"浏览器代理期"(config.cloudflare.browserUntil 未到)且请求是页面(GET + Accept text/html)→ 直接由浏览器代抓;
|
||||
// 1. 发 HTTP;账号 UA 跟随浏览器、出口与浏览器一致、复用没失败过时带上浏览器拿到的 cf_clearance(__cf_bm 不带);
|
||||
// 2. 响应是挑战(cf-mitigated: challenge,或 403 / 503 / 429 的挑战页)→ 不算凭据错误,记 lastChallengeAt;
|
||||
// 3. 过挑战(同站 30 分钟最多一次、同一时刻只跑一次;只处理有登录态账号的站点):浏览器用该账号 profile 打开同一地址 →
|
||||
// ① 等它自己过(≤15 秒)→ ② 复选框 / Turnstile 派视觉子流程点一下(最多 2 次)→ ③ 选图 / 输码类转人工(待处理事项);
|
||||
// 4. 过了:读回 cf_clearance + navigator.userAgent;满足复用条件就存下来,HTTP 重试 1 次,结果记 cfReuse = ok / fails;
|
||||
// 5. 仍被挑战:页面请求改由浏览器导航取 HTML 交给原解析器,站点记 6 小时浏览器代理期;其它请求(下载、RSS、POST)照常报挑战。
|
||||
// 不接任何打码服务。
|
||||
|
||||
/** 同站两次过挑战的最短间隔 */
|
||||
export const CF_ATTEMPT_INTERVAL_MS = 30 * 60_000;
|
||||
/** HTTP 过不去时走浏览器的时长 */
|
||||
export const CF_BROWSER_MODE_MS = 6 * 3_600_000;
|
||||
/** 站点状态读缓存(每条站点请求都要看是否在浏览器代理期) */
|
||||
const STATE_CACHE_MS = 5_000;
|
||||
|
||||
export type ChallengeSolveOutcome =
|
||||
| {
|
||||
status: "passed";
|
||||
by: "auto" | "vision";
|
||||
/** 浏览器 profile 里的 cf_clearance 值;没有为 null */
|
||||
clearance: string | null;
|
||||
/** 浏览器 navigator.userAgent */
|
||||
userAgent: string | null;
|
||||
/** 浏览器这一侧的出口键(egressKey);按容器环境变量出网 / 不知道时 null */
|
||||
browserEgress: string | null;
|
||||
}
|
||||
| { status: "needs_human"; sessionId: string | null; decisionId: string | null; reason: string }
|
||||
| { status: "failed"; reason: string };
|
||||
|
||||
export type BrowserFetchOutcome =
|
||||
| { status: "ok"; html: string; url: string | null }
|
||||
| { status: "challenge"; reason: string }
|
||||
| { status: "unavailable"; reason: string };
|
||||
|
||||
/** 浏览器一侧(浏览器模块注册;sources-identity 不直接依赖浏览器模块) */
|
||||
export interface ChallengeBrowserPort {
|
||||
solve(req: { siteId: string; accountId: string; url: string }): Promise<ChallengeSolveOutcome>;
|
||||
fetchHtml(req: { siteId: string; accountId: string; url: string }): Promise<BrowserFetchOutcome>;
|
||||
}
|
||||
|
||||
const ports = new WeakMap<object, ChallengeBrowserPort>();
|
||||
|
||||
export function registerChallengeBrowser(ctx: object, port: ChallengeBrowserPort): void {
|
||||
ports.set(ctx, port);
|
||||
}
|
||||
|
||||
export function challengeBrowserFor(ctx: object): ChallengeBrowserPort | null {
|
||||
return ports.get(ctx) ?? null;
|
||||
}
|
||||
|
||||
/** user_info.cloudflare:浏览器取得的 cf_clearance(值在 secret 里)与取得时的 UA / 出口 */
|
||||
interface ClearanceInfo {
|
||||
secretRef: string;
|
||||
userAgent: string | null;
|
||||
egress: string | null;
|
||||
at: string;
|
||||
}
|
||||
|
||||
const obj = (v: unknown): Record<string, unknown> =>
|
||||
v && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : {};
|
||||
|
||||
function clearanceInfo(userInfo: unknown): ClearanceInfo | null {
|
||||
const c = obj(obj(userInfo).cloudflare);
|
||||
return typeof c.secretRef === "string" && typeof c.at === "string"
|
||||
? {
|
||||
secretRef: c.secretRef,
|
||||
userAgent: typeof c.userAgent === "string" ? c.userAgent : null,
|
||||
egress: typeof c.egress === "string" ? c.egress : null,
|
||||
at: c.at,
|
||||
}
|
||||
: null;
|
||||
}
|
||||
|
||||
export interface ChallengeRequest {
|
||||
url: string;
|
||||
method: string;
|
||||
/** 请求的 Accept 头 */
|
||||
accept: string | null;
|
||||
/** 请求实际带的 UA */
|
||||
userAgent: string | null;
|
||||
/** 该请求的 HTTP 出口键 */
|
||||
egress: string;
|
||||
}
|
||||
|
||||
const EMPTY: SiteCloudflareState = {
|
||||
browserUntil: null,
|
||||
cfReuse: null,
|
||||
lastChallengeAt: null,
|
||||
lastSolvedBy: null,
|
||||
lastSolvedAt: null,
|
||||
lastAttemptAt: null,
|
||||
pending: null,
|
||||
sessionId: null,
|
||||
lastResult: null,
|
||||
lastRule: null,
|
||||
lastError: null,
|
||||
};
|
||||
|
||||
/** 页面请求:浏览器导航能代替的(GET + 要 HTML) */
|
||||
export function browserFetchable(req: Pick<ChallengeRequest, "method" | "accept">): boolean {
|
||||
return req.method.toUpperCase() === "GET" && /text\/html|application\/xhtml/i.test(req.accept ?? "");
|
||||
}
|
||||
|
||||
/** cf_clearance 能不能交给 HTTP:账号 UA 跟随浏览器、UA 相同、两端出口相同、复用没失败过 */
|
||||
export function clearanceUsable(input: {
|
||||
uaSource: string;
|
||||
requestUa: string | null;
|
||||
clearanceUa: string | null;
|
||||
requestEgress: string;
|
||||
clearanceEgress: string | null;
|
||||
cfReuse: SiteCloudflareState["cfReuse"];
|
||||
}): boolean {
|
||||
return (
|
||||
input.uaSource === "browser" &&
|
||||
!!input.requestUa &&
|
||||
input.requestUa === input.clearanceUa &&
|
||||
!!input.clearanceEgress &&
|
||||
input.clearanceEgress === input.requestEgress &&
|
||||
input.cfReuse !== "fails"
|
||||
);
|
||||
}
|
||||
|
||||
export class CloudflareGate {
|
||||
private readonly cache = new Map<string, { at: number; state: SiteCloudflareState }>();
|
||||
/** 进程内同站同一时刻只过一次挑战:后到的请求等同一个结果 */
|
||||
private readonly inflight = new Map<string, Promise<ChallengeSolveOutcome | null>>();
|
||||
|
||||
constructor(private readonly rt: SourcesRuntime) {}
|
||||
|
||||
private get ctx(): ServerContext {
|
||||
return this.rt.ctx;
|
||||
}
|
||||
|
||||
private now(): Date {
|
||||
return this.rt.now();
|
||||
}
|
||||
|
||||
async state(siteId: string, fresh = false): Promise<SiteCloudflareState> {
|
||||
const hit = this.cache.get(siteId);
|
||||
if (!fresh && hit && Date.now() - hit.at < STATE_CACHE_MS) return hit.state;
|
||||
const row = await this.ctx.db.selectFrom("site").select("config").where("id", "=", siteId).executeTakeFirst();
|
||||
const state = { ...EMPTY, ...(siteCloudflareState(row?.config) ?? {}) };
|
||||
this.cache.set(siteId, { at: Date.now(), state });
|
||||
return state;
|
||||
}
|
||||
|
||||
/** 合并写 site.config.cloudflare(只动这一个键) */
|
||||
async patch(siteId: string, patch: Partial<SiteCloudflareState>): Promise<void> {
|
||||
await sql`update site set config = coalesce(config, '{}'::jsonb) || jsonb_build_object('cloudflare',
|
||||
coalesce(config->'cloudflare', '{}'::jsonb) || ${JSON.stringify(patch)}::jsonb)
|
||||
where id = ${siteId}`.execute(this.ctx.db);
|
||||
this.cache.delete(siteId);
|
||||
}
|
||||
|
||||
/**
|
||||
* 同站 30 分钟最多一次:条件更新 lastAttemptAt(跨进程)。抢到返回 true。
|
||||
*/
|
||||
private async claimAttempt(siteId: string): Promise<boolean> {
|
||||
const now = this.now();
|
||||
const before = new Date(now.getTime() - CF_ATTEMPT_INTERVAL_MS);
|
||||
const r = await sql<{
|
||||
id: string;
|
||||
}>`update site set config = coalesce(config, '{}'::jsonb) || jsonb_build_object('cloudflare',
|
||||
coalesce(config->'cloudflare', '{}'::jsonb) || jsonb_build_object('lastAttemptAt', ${now.toISOString()}::text))
|
||||
where id = ${siteId}
|
||||
and (config->'cloudflare'->>'lastAttemptAt' is null
|
||||
or (config->'cloudflare'->>'lastAttemptAt')::timestamptz <= ${before})
|
||||
returning id`.execute(this.ctx.db);
|
||||
this.cache.delete(siteId);
|
||||
return r.rows.length > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* 有登录态的账号:与站点请求用的凭证同一个(第一个启用、未失效的 Cookie / 账密账号,且确实有 Cookie)。
|
||||
* 没有返回 null(没登录的站点不处理挑战)。
|
||||
*/
|
||||
async loggedInAccount(siteId: string): Promise<SiteAccountRow | null> {
|
||||
const rows = await this.ctx.db
|
||||
.selectFrom("site_account")
|
||||
.selectAll()
|
||||
.where("site_id", "=", siteId)
|
||||
.where("enabled", "=", true)
|
||||
.where("status", "in", ["unknown", "active"])
|
||||
.where("auth_kind", "in", ["cookie", "password"])
|
||||
.orderBy("created_at")
|
||||
.execute();
|
||||
return (
|
||||
rows.find(
|
||||
(r) => (r.auth_kind === "cookie" && !!r.secret_ref) || (r.auth_kind === "password" && !!r.session_secret_ref),
|
||||
) ?? null
|
||||
);
|
||||
}
|
||||
|
||||
/** 这条请求能带的 cf_clearance 头片段("cf_clearance=…");不满足复用条件为 null */
|
||||
async clearanceCookie(siteId: string, req: ChallengeRequest): Promise<string | null> {
|
||||
const acc = await this.loggedInAccount(siteId);
|
||||
if (!acc) return null;
|
||||
const info = clearanceInfo(acc.user_info);
|
||||
if (!info) return null;
|
||||
const st = await this.state(siteId);
|
||||
const d = await this.rt.uaDefaults();
|
||||
const ok = clearanceUsable({
|
||||
uaSource: resolveUserAgent(acc, d).source,
|
||||
requestUa: req.userAgent,
|
||||
clearanceUa: info.userAgent,
|
||||
requestEgress: req.egress,
|
||||
clearanceEgress: info.egress,
|
||||
cfReuse: st.cfReuse,
|
||||
});
|
||||
if (!ok) return null;
|
||||
const v = await this.ctx.secrets.resolve(info.secretRef).catch(() => null);
|
||||
return v ? `cf_clearance=${v}` : null;
|
||||
}
|
||||
|
||||
/** 把浏览器拿到的 cf_clearance 存进账号(secret),记下取得时的 UA 与出口 */
|
||||
private async storeClearance(
|
||||
acc: SiteAccountRow,
|
||||
value: string,
|
||||
userAgent: string | null,
|
||||
egress: string | null,
|
||||
): Promise<{ conditionsChanged: boolean }> {
|
||||
const info = obj(acc.user_info);
|
||||
const prev = clearanceInfo(acc.user_info);
|
||||
const conditionsChanged = !!prev && (prev.userAgent !== userAgent || prev.egress !== egress);
|
||||
let ref = prev?.secretRef ?? null;
|
||||
if (ref) {
|
||||
try {
|
||||
await this.ctx.secrets.replace(ref, value);
|
||||
} catch {
|
||||
ref = null;
|
||||
}
|
||||
}
|
||||
if (!ref)
|
||||
ref = await this.ctx.secrets.create({
|
||||
purpose: "site_cookie",
|
||||
plaintext: value,
|
||||
label: `site_account:${acc.site_id}:${acc.label}:cf_clearance:${crypto.randomUUID().slice(0, 8)}`,
|
||||
});
|
||||
const next: ClearanceInfo = { secretRef: ref, userAgent, egress, at: this.now().toISOString() };
|
||||
await this.ctx.db
|
||||
.updateTable("site_account")
|
||||
.set({ user_info: JSON.stringify({ ...info, cloudflare: next }) })
|
||||
.where("id", "=", acc.id)
|
||||
.execute();
|
||||
// UA 或出口变了:之前"复用无效"的结论不再适用,重新试一次
|
||||
if (conditionsChanged) await this.patch(acc.site_id, { cfReuse: null });
|
||||
return { conditionsChanged };
|
||||
}
|
||||
|
||||
/** 过挑战:进程内同站只跑一次,跨进程 30 分钟一次;被节流返回 null */
|
||||
private solveOnce(site: SiteRow, acc: SiteAccountRow, url: string): Promise<ChallengeSolveOutcome | null> {
|
||||
const running = this.inflight.get(site.id);
|
||||
if (running) return running;
|
||||
const p = (async () => {
|
||||
const port = challengeBrowserFor(this.ctx);
|
||||
if (!port) return null;
|
||||
if (!(await this.claimAttempt(site.id))) return null;
|
||||
try {
|
||||
return await port.solve({ siteId: site.id, accountId: acc.id, url });
|
||||
} catch (e) {
|
||||
return { status: "failed" as const, reason: String((e as Error)?.message ?? e).slice(0, 300) };
|
||||
}
|
||||
})().finally(() => this.inflight.delete(site.id));
|
||||
this.inflight.set(site.id, p);
|
||||
return p;
|
||||
}
|
||||
|
||||
/** 浏览器代抓一页;失败 / 仍是挑战返回 null */
|
||||
private async viaBrowser(siteId: string, acc: SiteAccountRow, url: string): Promise<Response | null> {
|
||||
const port = challengeBrowserFor(this.ctx);
|
||||
if (!port) return null;
|
||||
let r: BrowserFetchOutcome;
|
||||
try {
|
||||
r = await port.fetchHtml({ siteId, accountId: acc.id, url });
|
||||
} catch (e) {
|
||||
r = { status: "unavailable", reason: String((e as Error)?.message ?? e) };
|
||||
}
|
||||
if (r.status !== "ok") {
|
||||
this.ctx.logger.info({ component: "cloudflare", siteId, status: r.status, reason: r.reason }, "浏览器代抓未成功");
|
||||
if (r.status === "challenge") await this.patch(siteId, { lastResult: "browser_challenge", lastError: r.reason });
|
||||
return null;
|
||||
}
|
||||
return new Response(r.html, {
|
||||
status: 200,
|
||||
headers: { "content-type": "text/html; charset=utf-8", "x-umt-via": "browser" },
|
||||
});
|
||||
}
|
||||
|
||||
private log(siteId: string, msg: string, extra: Record<string, unknown> = {}): void {
|
||||
this.ctx.logger.info({ component: "cloudflare", siteId, ...extra }, msg);
|
||||
}
|
||||
|
||||
/**
|
||||
* 站点请求入口。send(cookieExtra) 发一次 HTTP(cookieExtra 非空时并进 Cookie 头);返回交给适配器的响应
|
||||
* (挑战没处理掉时原样返回挑战响应,适配器报 challenge)。
|
||||
*/
|
||||
async handle(
|
||||
siteId: string,
|
||||
req: ChallengeRequest,
|
||||
send: (cookieExtra: string | null) => Promise<Response>,
|
||||
): Promise<Response> {
|
||||
const pagey = browserFetchable(req);
|
||||
let st = await this.state(siteId);
|
||||
const now = this.now();
|
||||
// 0. 浏览器代理期:页面请求直接走浏览器
|
||||
if (pagey && st.browserUntil && Date.parse(st.browserUntil) > now.getTime() && challengeBrowserFor(this.ctx)) {
|
||||
const acc = await this.loggedInAccount(siteId);
|
||||
const r = acc ? await this.viaBrowser(siteId, acc, req.url) : null;
|
||||
if (r) return r;
|
||||
}
|
||||
// 1. HTTP(满足条件时带 cf_clearance)
|
||||
// 从没遇到过挑战的站点不查账号(每条请求少两次查询)
|
||||
const extra = st.lastChallengeAt ? await this.clearanceCookie(siteId, req).catch(() => null) : null;
|
||||
const first = await detectChallenge(await send(extra));
|
||||
if (!first.rule) return first.res;
|
||||
// 2. 挑战:记下来;没有浏览器 / 没有登录态账号 → 原样返回(适配器报 challenge,不算凭据错误)
|
||||
await this.patch(siteId, { lastChallengeAt: now.toISOString(), lastRule: first.rule });
|
||||
const port = challengeBrowserFor(this.ctx);
|
||||
if (!port) return first.res;
|
||||
const acc = await this.loggedInAccount(siteId);
|
||||
if (!acc) {
|
||||
await this.patch(siteId, { lastResult: "skipped", lastError: "站点没有已登录的账号，不处理挑战" });
|
||||
return first.res;
|
||||
}
|
||||
const site = await this.ctx.db.selectFrom("site").selectAll().where("id", "=", siteId).executeTakeFirst();
|
||||
if (!site?.enabled || site.deleted_at) return first.res;
|
||||
// 3. 过挑战
|
||||
const out = await this.solveOnce(site, acc, req.url);
|
||||
st = await this.state(siteId, true);
|
||||
if (!out) this.log(siteId, "站点 Cloudflare 挑战：30 分钟内处理过，本次不再过挑战");
|
||||
else if (out.status === "passed") {
|
||||
const at = this.now().toISOString();
|
||||
await this.patch(siteId, {
|
||||
lastSolvedBy: out.by,
|
||||
lastSolvedAt: at,
|
||||
lastResult: "passed",
|
||||
pending: null,
|
||||
sessionId: null,
|
||||
lastError: null,
|
||||
});
|
||||
this.log(siteId, "站点 Cloudflare 挑战已通过", { by: out.by });
|
||||
// 4. 同步 cf_clearance 并重试 HTTP 一次
|
||||
if (out.clearance) {
|
||||
await this.storeClearance(acc, out.clearance, out.userAgent, out.browserEgress);
|
||||
const retryExtra = await this.clearanceCookie(siteId, req).catch(() => null);
|
||||
if (retryExtra) {
|
||||
await this.rt.limiterOf(siteId)?.acquire();
|
||||
const second = await detectChallenge(await send(retryExtra));
|
||||
if (!second.rule) {
|
||||
await this.patch(siteId, { cfReuse: "ok", lastResult: "reuse_ok", browserUntil: null });
|
||||
return second.res;
|
||||
}
|
||||
await second.res.body?.cancel().catch(() => {});
|
||||
await this.patch(siteId, { cfReuse: "fails", lastResult: "reuse_fails" });
|
||||
this.log(siteId, "浏览器的 cf_clearance 给 HTTP 用仍被挑战，之后不再复用");
|
||||
}
|
||||
}
|
||||
} else if (out.status === "needs_human") {
|
||||
await this.patch(siteId, {
|
||||
pending: "human",
|
||||
sessionId: out.sessionId,
|
||||
lastResult: "needs_human",
|
||||
lastError: out.reason,
|
||||
});
|
||||
} else {
|
||||
await this.patch(siteId, { lastResult: "failed", lastError: out.reason });
|
||||
}
|
||||
// 5. 页面请求改由浏览器代抓,并进入 6 小时浏览器代理期
|
||||
if (pagey && !(out && out.status === "needs_human")) {
|
||||
const r = await this.viaBrowser(siteId, acc, req.url);
|
||||
if (r) {
|
||||
const until = new Date(this.now().getTime() + CF_BROWSER_MODE_MS).toISOString();
|
||||
await this.patch(siteId, { browserUntil: until, ...(out ? {} : { lastResult: "browser" }) });
|
||||
await first.res.body?.cancel().catch(() => {});
|
||||
return r;
|
||||
}
|
||||
}
|
||||
return first.res;
|
||||
}
|
||||
|
||||
/**
|
||||
* 人在原站视图里过了挑战、交还会话之后(浏览器模块调用):读回的 cf_clearance / UA 存起来,清掉"等人处理"。
|
||||
*/
|
||||
async afterHuman(
|
||||
siteId: string,
|
||||
accountId: string,
|
||||
got: { clearance: string | null; userAgent: string | null; browserEgress: string | null },
|
||||
): Promise<void> {
|
||||
const st = await this.state(siteId, true);
|
||||
if (st.pending !== "human") return;
|
||||
const acc = await this.ctx.db.selectFrom("site_account").selectAll().where("id", "=", accountId).executeTakeFirst();
|
||||
if (acc && got.clearance) await this.storeClearance(acc, got.clearance, got.userAgent, got.browserEgress);
|
||||
await this.patch(siteId, {
|
||||
pending: null,
|
||||
sessionId: null,
|
||||
lastSolvedBy: "human",
|
||||
lastSolvedAt: this.now().toISOString(),
|
||||
lastResult: "passed",
|
||||
lastError: null,
|
||||
// 人过了:允许马上再试一次 HTTP(不等 30 分钟)
|
||||
lastAttemptAt: null,
|
||||
});
|
||||
this.log(siteId, "站点 Cloudflare 挑战已由人在原站视图里通过");
|
||||
}
|
||||
}
|
||||
|
||||
/** Cookie 头里并进 cf_clearance(同名覆盖);__cf_bm 等其它 Cloudflare Cookie 不带 */
|
||||
export function withClearance(cookieHeader: string | null, extra: string | null): string | null {
|
||||
if (!extra) return cookieHeader;
|
||||
const jar = parseCookieHeader(cookieHeader);
|
||||
jar.delete("__cf_bm");
|
||||
const merged = mergeCookieHeaders([...jar].map(([k, v]) => `${k}=${v}`).join("; ") || null, extra);
|
||||
return merged;
|
||||
}
|
||||
|
||||
/** 站点出口键(与浏览器会话 applyEgress 同一口径) */
|
||||
export function siteEgressKey(proxy: boolean, proxyUrl: string | null | undefined): string {
|
||||
return egressKey(siteEgress(proxy, proxyUrl));
|
||||
}
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
type WebSearchTestResult,
|
||||
} from "@umt/contracts";
|
||||
import { cachedCall, pgCacheFor, pgTokenBucketFor } from "@umt/db";
|
||||
import { externalFetch } from "../../egress.ts";
|
||||
import { storageSettings } from "../../shared-cache.ts";
|
||||
import type { ServerContext } from "../types.ts";
|
||||
|
||||
@@ -94,7 +95,8 @@ export class WebSearchService {
|
||||
return createBraveClient({
|
||||
apiKey,
|
||||
...(this.opts.braveBaseUrl ? { baseUrl: this.opts.braveBaseUrl } : {}),
|
||||
...(this.opts.fetch ? { fetch: this.opts.fetch } : {}),
|
||||
// 外网请求:显式按环境变量走代理(CF-2 出口表)
|
||||
fetch: this.opts.fetch ?? externalFetch,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -184,7 +186,7 @@ export class WebSearchService {
|
||||
allow ? "只能读取最近一次 web.search 结果里的链接" : "先用 web.search 搜索，再读取结果里的链接",
|
||||
);
|
||||
const page = await fetchPage(want, {
|
||||
...(this.opts.fetch ? { fetch: this.opts.fetch } : {}),
|
||||
fetch: this.opts.fetch ?? externalFetch,
|
||||
...(this.opts.resolveHost ? { resolve: this.opts.resolveHost } : {}),
|
||||
...(o.signal ? { signal: o.signal } : {}),
|
||||
});
|
||||
|
||||
@@ -22,6 +22,7 @@ type Any = any;
|
||||
|
||||
const ORIGIN = "https://audiences.example";
|
||||
const ORIGIN_2FA = "https://secure.example";
|
||||
const ORIGIN_CF = "https://cf.example";
|
||||
const USER = "fixture-user";
|
||||
const PASSWORD = "e14-login-pw";
|
||||
const ACCESS_KEY = "camofox-e14-key";
|
||||
@@ -30,6 +31,7 @@ const ACCESS_KEY = "camofox-e14-key";
|
||||
const AT = { username: { x: 240, y: 64 }, password: { x: 240, y: 96 } };
|
||||
|
||||
let sid = "";
|
||||
let cfSid = "";
|
||||
const childResults = new Map<string, FakeToolResult[]>();
|
||||
let mainResults: FakeToolResult[] = [];
|
||||
|
||||
@@ -37,6 +39,16 @@ const typeUser = () => ({ sessionId: sid, ...AT.username, text: USER, allowSensi
|
||||
const typePassword = () => ({ sessionId: sid, ...AT.password, text: PASSWORD, pressEnter: true, allowSensitive: true });
|
||||
|
||||
const scripts: FakeScript[] = [
|
||||
{
|
||||
// CF-2 系统发起的视觉子流程(没有父会话):截图 → 点 Turnstile 复选框(第 2 行,截图坐标 (240, 64))
|
||||
name: "e14-cf",
|
||||
match: "Cloudflare 人机验证页",
|
||||
steps: [
|
||||
{ toolCalls: [{ capability: "browser.screenshot", input: () => ({ sessionId: cfSid }) }] },
|
||||
{ toolCalls: [{ capability: "browser.click_at", input: () => ({ sessionId: cfSid, x: 240, y: 64 }) }] },
|
||||
],
|
||||
finalText: () => "已点复选框",
|
||||
},
|
||||
{
|
||||
name: "e14-main",
|
||||
match: "E14 主会话登录",
|
||||
@@ -122,6 +134,7 @@ describe.skipIf(!hasPg)("E14 视觉自动化闭环", () => {
|
||||
let e: E2E;
|
||||
let mock: MockCamofox;
|
||||
let mock2fa: MockCamofox;
|
||||
let mockCf: MockCamofox;
|
||||
let evidenceDir: string;
|
||||
let svc: BrowserService;
|
||||
const ids = { site: "", site2fa: "", provider: "", provider2fa: "" };
|
||||
@@ -154,8 +167,16 @@ describe.skipIf(!hasPg)("E14 视觉自动化闭环", () => {
|
||||
site: { ...site(ORIGIN_2FA), twoFactor: true },
|
||||
viewport: { width: 1600, height: 1000 },
|
||||
});
|
||||
mockCf = await createMockCamofox({
|
||||
accessKey: ACCESS_KEY,
|
||||
site: { ...site(ORIGIN_CF), protectedPaths: /^$/, cloudflare: { mode: "interactive", clearance: "clr-e14" } },
|
||||
viewport: { width: 1600, height: 1000 },
|
||||
});
|
||||
evidenceDir = await mkdtemp(join(tmpdir(), "umt-e14-evidence-"));
|
||||
e = await createE2E("e14", { fakeScripts: scripts, browser: { evidenceDir, settleMs: 0 } });
|
||||
e = await createE2E("e14", {
|
||||
fakeScripts: scripts,
|
||||
browser: { evidenceDir, settleMs: 0, challengePollMs: 10, challengeAutoWaitMs: 200 },
|
||||
});
|
||||
svc = browserServiceFor(e.booted.ctx)!;
|
||||
const ctx = e.booted.ctx;
|
||||
const keyRef = await ctx.secrets.create({ purpose: "other", plaintext: ACCESS_KEY });
|
||||
@@ -200,6 +221,7 @@ describe.skipIf(!hasPg)("E14 视觉自动化闭环", () => {
|
||||
await e?.close();
|
||||
await mock?.close();
|
||||
await mock2fa?.close();
|
||||
await mockCf?.close();
|
||||
if (evidenceDir) await rm(evidenceDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
@@ -335,9 +357,14 @@ describe.skipIf(!hasPg)("E14 视觉自动化闭环", () => {
|
||||
// 坐标换算:截图 (240, 64) → 页面 (300, 80);(240, 96) → (300, 120)
|
||||
expect(r[1]?.json).toMatchObject({ pageX: 300, pageY: 80 });
|
||||
expect(r[2]?.json).toMatchObject({ pageX: 300, pageY: 120 });
|
||||
const evals = mock.actions.filter((a) => a.action === "evaluate").map((a) => String(a.body.expression));
|
||||
expect(evals.some((x) => x.includes("elementFromPoint(300, 80)"))).toBe(true);
|
||||
expect(evals.some((x) => x.includes("elementFromPoint(300, 120)"))).toBe(true);
|
||||
// CF-2:坐标点击走 umt 插件的真实鼠标事件(按页面 CSS 像素)
|
||||
const clicks = mock.actions.filter((a) => a.action === "mouse-click").map((a) => [a.body.x, a.body.y]);
|
||||
expect(clicks).toEqual(
|
||||
expect.arrayContaining([
|
||||
[300, 80],
|
||||
[300, 120],
|
||||
]),
|
||||
);
|
||||
expect(mock.loggedIn.size).toBe(1);
|
||||
// 父会话事件流里有子流程
|
||||
const events = await runEvents(e, runId);
|
||||
@@ -372,4 +399,72 @@ describe.skipIf(!hasPg)("E14 视觉自动化闭环", () => {
|
||||
expect(again).not.toBe(sid);
|
||||
await worker.close(again);
|
||||
}, 60_000);
|
||||
|
||||
it("CF-2 站点请求遇 Turnstile:系统发起视觉子流程(没有父会话,预算取设置)点复选框,浏览器读回 cf_clearance", async () => {
|
||||
// 只留 Cloudflare 这个浏览器服务(站点请求一侧不指定服务,取启用的那个)
|
||||
await e.db.updateTable("browser_provider").set({ enabled: false }).execute();
|
||||
const providerCf = (
|
||||
await e.db
|
||||
.insertInto("browser_provider")
|
||||
.values({
|
||||
name: "camofox-e14-cf",
|
||||
kind: "camofox",
|
||||
base_url: mockCf.url,
|
||||
secret_ref: (
|
||||
await e.db.selectFrom("browser_provider").select("secret_ref").executeTakeFirstOrThrow()
|
||||
).secret_ref,
|
||||
enabled: true,
|
||||
config: JSON.stringify({ navigationTimeoutMs: 2_000, maxRetries: 0 }),
|
||||
})
|
||||
.returning("id")
|
||||
.executeTakeFirstOrThrow()
|
||||
).id;
|
||||
const siteCf = (
|
||||
await e.db
|
||||
.insertInto("site")
|
||||
.values({
|
||||
key: "audiences",
|
||||
name: "观众",
|
||||
adapter: "nexusphp",
|
||||
base_url: `${ORIGIN_CF}/`,
|
||||
rate_limit: JSON.stringify({ minIntervalMs: 0, concurrency: 5 }),
|
||||
})
|
||||
.returning("id")
|
||||
.executeTakeFirstOrThrow()
|
||||
).id;
|
||||
const ref = await e.booted.ctx.secrets.create({ purpose: "site_cookie", plaintext: "c_secure_pass=e14" });
|
||||
const accountId = (
|
||||
await e.db
|
||||
.insertInto("site_account")
|
||||
.values({ site_id: siteCf, label: "main", auth_kind: "cookie", secret_ref: ref, status: "active" })
|
||||
.returning("id")
|
||||
.executeTakeFirstOrThrow()
|
||||
).id;
|
||||
cfSid = `umt-site-${siteCf}-acct-${accountId}`; // 先占位,打开后换成会话 ID
|
||||
const pending = svc.solveSiteChallenge({ siteId: siteCf, accountId, url: `${ORIGIN_CF}/torrents.php` });
|
||||
// 会话打开后脚本才知道 sessionId
|
||||
await waitFor(
|
||||
async () => {
|
||||
const live = await svc.liveSessionFor(siteCf, accountId);
|
||||
if (live) cfSid = live.id;
|
||||
return !!live;
|
||||
},
|
||||
10_000,
|
||||
"Cloudflare 会话打开",
|
||||
10,
|
||||
);
|
||||
const out = await pending;
|
||||
expect(out).toMatchObject({ status: "passed", by: "vision", clearance: "clr-e14" });
|
||||
expect(mockCf.actions.some((a) => a.action === "mouse-click" && a.body.x === 300 && a.body.y === 80)).toBe(true);
|
||||
const child = await e.db
|
||||
.selectFrom("agent_run")
|
||||
.selectAll()
|
||||
.where("spawn_reason", "=", "browser.solve_challenge")
|
||||
.where("created_by", "=", "system:cloudflare")
|
||||
.executeTakeFirstOrThrow();
|
||||
expect(child).toMatchObject({ trigger: "event", parent_run_id: null, status: "succeeded" });
|
||||
expect((child.budget as Any).maxToolCalls).toBe(4);
|
||||
expect((child.budget as Any).maxDurationSec).toBeLessThanOrEqual(120);
|
||||
expect(providerCf).toBeTruthy();
|
||||
}, 60_000);
|
||||
});
|
||||
@@ -0,0 +1,315 @@
|
||||
import { SiteError } from "@umt/adapters";
|
||||
import { siteCloudflareState } from "@umt/contracts";
|
||||
import { createMockNexusPhpSite, type MockNexusPhpSite } from "@umt/testkit";
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
type BrowserFetchOutcome,
|
||||
browserFetchable,
|
||||
CF_ATTEMPT_INTERVAL_MS,
|
||||
type ChallengeSolveOutcome,
|
||||
clearanceUsable,
|
||||
registerChallengeBrowser,
|
||||
withClearance,
|
||||
} from "../../src/modules/sources-identity/service/cloudflare.ts";
|
||||
import { snapshotSite } from "../../src/modules/sources-identity/service/snapshot.ts";
|
||||
import { registerBrowserIdentityReader } from "../../src/modules/sources-identity/service/user-agent.ts";
|
||||
import { createHarness, type Harness, pgAvailable } from "./harness.ts";
|
||||
|
||||
// CF-2 级联状态机:HTTP 遇 Cloudflare 挑战 → 浏览器过挑战(自动 / 视觉子流程 / 转人工)→ cf_clearance 同步给 HTTP 重试 1 次 →
|
||||
// 仍被挑战则页面请求改由浏览器代抓并进入 6 小时浏览器代理期;同站 30 分钟节流、同一时刻只处理一次;挑战不算凭据错误。
|
||||
// 站点 = testkit mock NexusPHP(真 HTTP),前面套一层"Cloudflare":没带合格 cf_clearance 的请求回 403 + cf-mitigated 挑战页。
|
||||
// 浏览器一侧用假端口(ChallengeBrowserPort),真实浏览器流程见 browser-cloudflare.test.ts。
|
||||
|
||||
const up = await pgAvailable();
|
||||
const BROWSER_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:152.0) Gecko/20100101 Firefox/152.0";
|
||||
const CF_HTML = `<!DOCTYPE html><html><head><title>Just a moment...</title></head><body>
|
||||
<div id="challenge-spinner"></div><script>window._cf_chl_opt={cType: 'managed'}</script></body></html>`;
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 响应体断言
|
||||
const json = async (r: Response): Promise<any> => {
|
||||
const body = await r.json();
|
||||
if (!r.ok) throw new Error(`${r.status} ${JSON.stringify(body)}`);
|
||||
return body.data;
|
||||
};
|
||||
|
||||
describe("CF-2 纯函数", () => {
|
||||
it("页面请求才能由浏览器代抓", () => {
|
||||
expect(browserFetchable({ method: "GET", accept: "text/html,application/xhtml+xml" })).toBe(true);
|
||||
expect(browserFetchable({ method: "GET", accept: null })).toBe(false);
|
||||
expect(browserFetchable({ method: "POST", accept: "text/html" })).toBe(false);
|
||||
expect(browserFetchable({ method: "GET", accept: "application/rss+xml" })).toBe(false);
|
||||
});
|
||||
it("cf_clearance 同步条件:UA 跟随浏览器 + UA 相同 + 两端出口相同 + 复用没失败过", () => {
|
||||
const base = {
|
||||
uaSource: "browser",
|
||||
requestUa: "A",
|
||||
clearanceUa: "A",
|
||||
requestEgress: "direct",
|
||||
clearanceEgress: "direct",
|
||||
cfReuse: null,
|
||||
} as const;
|
||||
expect(clearanceUsable(base)).toBe(true);
|
||||
expect(clearanceUsable({ ...base, uaSource: "manual" })).toBe(false);
|
||||
expect(clearanceUsable({ ...base, uaSource: "global" })).toBe(false);
|
||||
expect(clearanceUsable({ ...base, clearanceUa: "B" })).toBe(false);
|
||||
expect(clearanceUsable({ ...base, clearanceEgress: null })).toBe(false);
|
||||
expect(clearanceUsable({ ...base, clearanceEgress: "proxy:http://relay:7891" })).toBe(false);
|
||||
expect(clearanceUsable({ ...base, cfReuse: "fails" })).toBe(false);
|
||||
});
|
||||
it("并进 cf_clearance,去掉 __cf_bm", () => {
|
||||
expect(withClearance("a=1; __cf_bm=x", "cf_clearance=Z")).toBe("a=1; cf_clearance=Z");
|
||||
expect(withClearance(null, "cf_clearance=Z")).toBe("cf_clearance=Z");
|
||||
expect(withClearance("a=1", null)).toBe("a=1");
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!up)("CF-2 Cloudflare 级联", () => {
|
||||
let h: Harness;
|
||||
let mock: MockNexusPhpSite;
|
||||
let siteId: string;
|
||||
let accountId: string;
|
||||
/** "Cloudflare" 开关与放行规则 */
|
||||
const cf = {
|
||||
on: false,
|
||||
/** 放行判定;缺省:带 cf_clearance=GOOD 且 UA 是浏览器 UA */
|
||||
pass: (h: Headers) => (h.get("cookie") ?? "").includes("cf_clearance=GOOD") && h.get("user-agent") === BROWSER_UA,
|
||||
challenged: 0,
|
||||
passed: 0,
|
||||
};
|
||||
const seen: Headers[] = [];
|
||||
const solves: { url: string; accountId: string }[] = [];
|
||||
const fetches: string[] = [];
|
||||
let solveResult: ChallengeSolveOutcome = { status: "failed", reason: "unset" };
|
||||
let solveDelay = 0;
|
||||
let fetchResult: BrowserFetchOutcome | null = null;
|
||||
|
||||
const cfFetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
|
||||
const hd = new Headers(init?.headers);
|
||||
seen.push(hd);
|
||||
if (cf.on && !cf.pass(hd)) {
|
||||
cf.challenged++;
|
||||
return new Response(CF_HTML, {
|
||||
status: 403,
|
||||
headers: { "cf-mitigated": "challenge", "content-type": "text/html" },
|
||||
});
|
||||
}
|
||||
cf.passed++;
|
||||
return globalThis.fetch(input, init);
|
||||
}) as typeof fetch;
|
||||
|
||||
const site = () => h.db.selectFrom("site").selectAll().where("id", "=", siteId).executeTakeFirstOrThrow();
|
||||
const account = () =>
|
||||
h.db.selectFrom("site_account").selectAll().where("id", "=", accountId).executeTakeFirstOrThrow();
|
||||
const state = async () => siteCloudflareState((await site()).config);
|
||||
const list = async () => h.rt.siteAdapter(await site()).list(1);
|
||||
|
||||
beforeAll(async () => {
|
||||
mock = await createMockNexusPhpSite({});
|
||||
h = await createHarness("cf2", { fetch: cfFetch });
|
||||
registerBrowserIdentityReader(h.ctx, async () => BROWSER_UA);
|
||||
registerChallengeBrowser(h.ctx, {
|
||||
async solve(req) {
|
||||
solves.push(req);
|
||||
if (solveDelay) await new Promise((r) => setTimeout(r, solveDelay));
|
||||
return solveResult;
|
||||
},
|
||||
async fetchHtml(req) {
|
||||
fetches.push(req.url);
|
||||
if (fetchResult) return fetchResult;
|
||||
// 浏览器已登录且过了挑战:直接取 mock 页面
|
||||
const r = await globalThis.fetch(req.url, { headers: { cookie: mock.cookie } });
|
||||
return { status: "ok", html: await r.text(), url: req.url };
|
||||
},
|
||||
});
|
||||
const s = await json(
|
||||
await h.req("/sites", {
|
||||
body: {
|
||||
key: "audiences",
|
||||
name: "观众",
|
||||
adapter: "nexusphp",
|
||||
baseUrl: `${mock.baseUrl}/`,
|
||||
rateLimit: { minIntervalMs: 0, concurrency: 10 },
|
||||
},
|
||||
}),
|
||||
);
|
||||
siteId = s.id;
|
||||
const r = await json(
|
||||
await h.req(`/sites/${siteId}/credentials`, { method: "PUT", body: { kind: "cookie", cookie: mock.cookie } }),
|
||||
);
|
||||
accountId = r.siteAccountId;
|
||||
});
|
||||
afterAll(async () => {
|
||||
await h?.close();
|
||||
await mock?.close();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
cf.on = true;
|
||||
cf.pass = (hd) => (hd.get("cookie") ?? "").includes("cf_clearance=GOOD") && hd.get("user-agent") === BROWSER_UA;
|
||||
cf.challenged = 0;
|
||||
cf.passed = 0;
|
||||
seen.length = 0;
|
||||
solves.length = 0;
|
||||
fetches.length = 0;
|
||||
solveDelay = 0;
|
||||
fetchResult = null;
|
||||
solveResult = { status: "passed", by: "auto", clearance: "GOOD", userAgent: BROWSER_UA, browserEgress: "direct" };
|
||||
await h.db
|
||||
.updateTable("site")
|
||||
.set({ config: JSON.stringify({}), health_status: "healthy" })
|
||||
.where("id", "=", siteId)
|
||||
.execute();
|
||||
await h.db
|
||||
.updateTable("site_account")
|
||||
.set({ user_info: JSON.stringify({}), user_agent: null, ua_pending: null, status: "active" })
|
||||
.where("id", "=", accountId)
|
||||
.execute();
|
||||
await h.rt.cloudflare.patch(siteId, {});
|
||||
});
|
||||
|
||||
it("自动通过:浏览器拿到 cf_clearance + UA,HTTP 带上重试 1 次成功,记 cfReuse = ok;之后的请求直接带", async () => {
|
||||
const page = await list();
|
||||
expect(page.items.length).toBeGreaterThan(0);
|
||||
expect(solves).toEqual([{ siteId, accountId, url: expect.stringContaining("/torrents.php") }]);
|
||||
expect(cf.challenged).toBe(1);
|
||||
expect(fetches).toEqual([]);
|
||||
const st = await state();
|
||||
expect(st).toMatchObject({ lastSolvedBy: "auto", cfReuse: "ok", lastResult: "reuse_ok", browserUntil: null });
|
||||
expect(st?.lastChallengeAt).toBeTruthy();
|
||||
expect(st?.lastRule).toBe("header cf-mitigated=challenge");
|
||||
// cf_clearance 存在 secret 里,账号 user_info 只有引用与取得时的 UA / 出口
|
||||
const info = (await account()).user_info as {
|
||||
cloudflare?: { secretRef: string; userAgent: string; egress: string };
|
||||
};
|
||||
expect(info.cloudflare).toMatchObject({ userAgent: BROWSER_UA, egress: "direct" });
|
||||
expect(h.secrets.plain.get(info.cloudflare!.secretRef)).toBe("GOOD");
|
||||
expect(JSON.stringify(info)).not.toContain("GOOD");
|
||||
// __cf_bm 不带;下一条请求直接带 cf_clearance,不再过挑战
|
||||
seen.length = 0;
|
||||
await list();
|
||||
expect(solves).toHaveLength(1);
|
||||
expect(seen[0]?.get("cookie")).toContain("cf_clearance=GOOD");
|
||||
expect(seen[0]?.get("cookie")).not.toContain("__cf_bm");
|
||||
// 账号状态不变
|
||||
expect((await account()).status).toBe("active");
|
||||
});
|
||||
|
||||
it("视觉子流程通过:记 lastSolvedBy = vision", async () => {
|
||||
solveResult = { status: "passed", by: "vision", clearance: "GOOD", userAgent: BROWSER_UA, browserEgress: "direct" };
|
||||
await list();
|
||||
expect(await state()).toMatchObject({ lastSolvedBy: "vision", cfReuse: "ok" });
|
||||
});
|
||||
|
||||
it("复用失败:cf_clearance 给 HTTP 仍被挑战 → cfReuse = fails,本次由浏览器代抓,6 小时浏览器代理期内页面请求直接走浏览器", async () => {
|
||||
cf.pass = () => false; // 站点绑定 TLS 指纹:HTTP 怎么都过不去
|
||||
const page = await list();
|
||||
expect(page.items.length).toBeGreaterThan(0);
|
||||
expect(cf.challenged).toBe(2); // 首次 + 带 cf_clearance 重试 1 次
|
||||
expect(fetches).toHaveLength(1);
|
||||
const st = await state();
|
||||
expect(st).toMatchObject({ cfReuse: "fails", lastResult: "reuse_fails" });
|
||||
const until = Date.parse(st!.browserUntil!);
|
||||
expect(until - Date.now()).toBeGreaterThan(5.9 * 3_600_000);
|
||||
// 代理期内:不发 HTTP,不再过挑战
|
||||
await list();
|
||||
expect(cf.challenged).toBe(2);
|
||||
expect(fetches).toHaveLength(2);
|
||||
expect(solves).toHaveLength(1);
|
||||
// 下载(不是页面请求)仍走 HTTP,报 challenge,不带失败过的 cf_clearance
|
||||
const err = await h.rt
|
||||
.siteAdapter(await site())
|
||||
.downloadTarget(page.items[0]!.siteReleaseId)
|
||||
.catch((e) => e);
|
||||
expect(err).toBeInstanceOf(SiteError);
|
||||
expect((err as SiteError).kind).toBe("challenge");
|
||||
expect(seen.at(-1)?.get("cookie") ?? "").not.toContain("cf_clearance");
|
||||
// 到期再试 HTTP
|
||||
await h.rt.cloudflare.patch(siteId, { browserUntil: new Date(Date.now() - 1000).toISOString() });
|
||||
cf.pass = () => true;
|
||||
const n = fetches.length;
|
||||
await list();
|
||||
expect(fetches).toHaveLength(n);
|
||||
});
|
||||
|
||||
it("真验证码转人工:本次报 challenge(不算凭据错误),站点记 pending = human;快照不置账号失效、不开登录失效事项", async () => {
|
||||
solveResult = { status: "needs_human", sessionId: null, decisionId: null, reason: "选图验证码" };
|
||||
const r = await snapshotSite(h.rt, { siteId, kind: "manual" });
|
||||
expect(r.status).toBe("failed");
|
||||
expect(r.error).toContain("challenge");
|
||||
expect(fetches).toEqual([]);
|
||||
expect(await state()).toMatchObject({ pending: "human", lastResult: "needs_human", lastError: "选图验证码" });
|
||||
expect((await account()).status).toBe("active");
|
||||
expect((await site()).health_status).toBe("degraded");
|
||||
const d = await h.db.selectFrom("decision").select("id").where("dedupe_key", "=", `site-auth:${siteId}`).execute();
|
||||
expect(d).toEqual([]);
|
||||
expect(h.engine.enqueued.filter((e) => e.task === "refreshSiteAccount")).toEqual([]);
|
||||
|
||||
// 人在原站视图里过完、交还:浏览器模块回调 afterHuman → 清掉等人处理,存下 cf_clearance,马上可以再试
|
||||
await h.rt.cloudflare.afterHuman(siteId, accountId, {
|
||||
clearance: "GOOD",
|
||||
userAgent: BROWSER_UA,
|
||||
browserEgress: "direct",
|
||||
});
|
||||
expect(await state()).toMatchObject({ pending: null, lastSolvedBy: "human", lastAttemptAt: null });
|
||||
const page = await list();
|
||||
expect(page.items.length).toBeGreaterThan(0);
|
||||
expect(solves).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("节流与单飞:并发的挑战只过一次;30 分钟内不再过;超过 30 分钟再过", async () => {
|
||||
solveDelay = 150;
|
||||
cf.pass = () => false;
|
||||
fetchResult = { status: "challenge", reason: "still" };
|
||||
const results = await Promise.allSettled([list(), list(), list()]);
|
||||
expect(results.every((r) => r.status === "rejected")).toBe(true);
|
||||
expect(solves).toHaveLength(1);
|
||||
// 30 分钟内:只记挑战,不过
|
||||
await list().catch(() => null);
|
||||
expect(solves).toHaveLength(1);
|
||||
expect((await state())?.lastAttemptAt).toBeTruthy();
|
||||
// 超过 30 分钟
|
||||
await h.rt.cloudflare.patch(siteId, {
|
||||
lastAttemptAt: new Date(Date.now() - CF_ATTEMPT_INTERVAL_MS - 1000).toISOString(),
|
||||
});
|
||||
await list().catch(() => null);
|
||||
expect(solves).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("同步条件不满足:账号手动 UA → 不把 cf_clearance 交给 HTTP(不重试),页面改由浏览器代抓", async () => {
|
||||
await h.db.updateTable("site_account").set({ user_agent: BROWSER_UA }).where("id", "=", accountId).execute();
|
||||
const page = await list();
|
||||
expect(page.items.length).toBeGreaterThan(0);
|
||||
expect(cf.challenged).toBe(1);
|
||||
expect(fetches).toHaveLength(1);
|
||||
expect(seen.every((x) => !(x.get("cookie") ?? "").includes("cf_clearance"))).toBe(true);
|
||||
expect((await state())?.browserUntil).toBeTruthy();
|
||||
});
|
||||
|
||||
it("同步条件不满足:浏览器出口与 HTTP 不同(旧插件按环境变量出网)→ 不复用", async () => {
|
||||
solveResult = { status: "passed", by: "auto", clearance: "GOOD", userAgent: BROWSER_UA, browserEgress: null };
|
||||
await list();
|
||||
expect(cf.challenged).toBe(1);
|
||||
expect(fetches).toHaveLength(1);
|
||||
expect((await state())?.cfReuse).toBeNull();
|
||||
});
|
||||
|
||||
it("没有登录态账号的站点不处理挑战", async () => {
|
||||
await h.db.updateTable("site_account").set({ status: "expired" }).where("id", "=", accountId).execute();
|
||||
// 适配器没有 Cookie 就不发请求;这里直接用站点 fetch(RSS 这类不要登录态的请求同理)
|
||||
const res = await h.rt.siteFetch(await site())(`${mock.baseUrl}/torrents.php`, {
|
||||
headers: { accept: "text/html" },
|
||||
});
|
||||
expect(res.status).toBe(403);
|
||||
expect(solves).toEqual([]);
|
||||
expect(fetches).toEqual([]);
|
||||
expect(await state()).toMatchObject({ lastResult: "skipped" });
|
||||
});
|
||||
|
||||
it("健康检测遇挑战且过不去:记 challenge(degraded),账号状态不变", async () => {
|
||||
solveResult = { status: "failed", reason: "浏览器服务不可用" };
|
||||
fetchResult = { status: "unavailable", reason: "浏览器服务不可用" };
|
||||
const r = await json(await h.req(`/sites/${siteId}/health-check`, { method: "POST", body: {} }));
|
||||
expect(r).toMatchObject({ ok: false, status: "challenge", healthStatus: "degraded", loggedIn: null });
|
||||
expect((await account()).status).toBe("active");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,133 @@
|
||||
import { DEFAULT_SETTINGS } from "@umt/contracts";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import {
|
||||
directHttpClient,
|
||||
egressKey,
|
||||
envProxyFor,
|
||||
fetchVia,
|
||||
noProxyMatches,
|
||||
proxyHttpClient,
|
||||
siteEgress,
|
||||
} from "../../src/egress.ts";
|
||||
import { SourcesRuntime } from "../../src/modules/sources-identity/runtime.ts";
|
||||
import type { ServerContext } from "../../src/modules/types.ts";
|
||||
|
||||
// CF-2 出口统一:站点请求不再隐式继承容器的 HTTP(S)_PROXY。开了站点代理 → 该代理;否则明确直连。
|
||||
// 本地起一个"代理"(收到请求就回 via-proxy)和一个目标(回 direct),把 HTTP_PROXY 指向前者,看请求实际走哪条。
|
||||
|
||||
const PROXY_KEYS = [
|
||||
"HTTP_PROXY",
|
||||
"HTTPS_PROXY",
|
||||
"ALL_PROXY",
|
||||
"http_proxy",
|
||||
"https_proxy",
|
||||
"all_proxy",
|
||||
"NO_PROXY",
|
||||
"no_proxy",
|
||||
];
|
||||
|
||||
describe("出口选择(直连 / 代理)", () => {
|
||||
let proxy: Deno.HttpServer<Deno.NetAddr>;
|
||||
let target: Deno.HttpServer<Deno.NetAddr>;
|
||||
let proxyUrl = "";
|
||||
let targetUrl = "";
|
||||
const proxied: string[] = [];
|
||||
const saved = new Map<string, string | undefined>();
|
||||
|
||||
beforeAll(() => {
|
||||
proxy = Deno.serve({ hostname: "127.0.0.1", port: 0, onListen() {} }, (req) => {
|
||||
proxied.push(req.url);
|
||||
return new Response("via-proxy");
|
||||
});
|
||||
target = Deno.serve({ hostname: "127.0.0.1", port: 0, onListen() {} }, () => new Response("direct"));
|
||||
proxyUrl = `http://127.0.0.1:${proxy.addr.port}`;
|
||||
targetUrl = `http://127.0.0.1:${target.addr.port}/x`;
|
||||
for (const k of PROXY_KEYS) {
|
||||
saved.set(k, Deno.env.get(k));
|
||||
Deno.env.delete(k);
|
||||
}
|
||||
// 模拟容器环境:HTTP_PROXY 指向"中继",没有 NO_PROXY
|
||||
Deno.env.set("HTTP_PROXY", proxyUrl);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
for (const [k, v] of saved) {
|
||||
if (v === undefined) Deno.env.delete(k);
|
||||
else Deno.env.set(k, v);
|
||||
}
|
||||
await proxy.shutdown();
|
||||
await target.shutdown();
|
||||
});
|
||||
|
||||
const body = async (f: typeof fetch) => (await f(targetUrl)).text();
|
||||
|
||||
it("Deno 2.9 语义:createHttpClient({}) 仍会读 HTTP_PROXY(所以直连要显式做)", async () => {
|
||||
const implicit = Deno.createHttpClient({});
|
||||
try {
|
||||
expect(await body(fetchVia(implicit))).toBe("via-proxy");
|
||||
} finally {
|
||||
implicit.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("directHttpClient:环境变量里有代理也直连;创建后环境变量原样还原", async () => {
|
||||
proxied.length = 0;
|
||||
expect(await body(fetchVia(directHttpClient()))).toBe("direct");
|
||||
expect(proxied).toEqual([]);
|
||||
expect(Deno.env.get("HTTP_PROXY")).toBe(proxyUrl);
|
||||
});
|
||||
|
||||
it("proxyHttpClient:只走给定代理(NO_PROXY 命中也不绕过)", async () => {
|
||||
Deno.env.set("NO_PROXY", "127.0.0.1");
|
||||
try {
|
||||
proxied.length = 0;
|
||||
const client = proxyHttpClient(proxyUrl);
|
||||
expect(await body(fetchVia(client))).toBe("via-proxy");
|
||||
expect(proxied).toHaveLength(1);
|
||||
expect(Deno.env.get("NO_PROXY")).toBe("127.0.0.1");
|
||||
} finally {
|
||||
Deno.env.delete("NO_PROXY");
|
||||
}
|
||||
});
|
||||
|
||||
it("SourcesRuntime.siteFetch:站点没开代理 → 直连;开了且设了代理地址 → 站点代理", async () => {
|
||||
const ctx = { logger: { debug() {}, info() {}, warn() {} } } as unknown as ServerContext;
|
||||
const rt = new SourcesRuntime(ctx);
|
||||
const defaults = { ...DEFAULT_SETTINGS.sites, proxyUrl };
|
||||
proxied.length = 0;
|
||||
expect(await body(rt.siteFetch({ proxy: false }, defaults, "UA"))).toBe("direct");
|
||||
expect(proxied).toEqual([]);
|
||||
expect(await body(rt.siteFetch({ proxy: true }, defaults, "UA"))).toBe("via-proxy");
|
||||
expect(proxied).toHaveLength(1);
|
||||
// 开了代理但没设代理地址 → 直连
|
||||
expect(await body(rt.siteFetch({ proxy: true }, { ...defaults, proxyUrl: null }, "UA"))).toBe("direct");
|
||||
expect(rt.egressKeyOf({ proxy: false }, defaults)).toBe("direct");
|
||||
expect(rt.egressKeyOf({ proxy: true }, defaults)).toBe(`proxy:http://127.0.0.1:${proxy.addr.port}`);
|
||||
});
|
||||
});
|
||||
|
||||
describe("外网请求按环境变量(行为不变,改为显式)", () => {
|
||||
it("HTTPS / HTTP / ALL_PROXY 与 NO_PROXY", () => {
|
||||
const env = { HTTPS_PROXY: "http://relay:7891", HTTP_PROXY: "http://relay:7890", NO_PROXY: "localhost,.lan" };
|
||||
expect(envProxyFor(new URL("https://api.themoviedb.org/3"), env)).toBe("http://relay:7891");
|
||||
expect(envProxyFor(new URL("http://image.tmdb.org/t"), env)).toBe("http://relay:7890");
|
||||
expect(envProxyFor(new URL("http://nas.lan/x"), env)).toBeNull();
|
||||
expect(envProxyFor(new URL("https://api.search.brave.com/"), { ALL_PROXY: "socks5h://relay:1080" })).toBe(
|
||||
"socks5h://relay:1080",
|
||||
);
|
||||
expect(envProxyFor(new URL("https://api.search.brave.com/"), {})).toBeNull();
|
||||
expect(envProxyFor(new URL("https://x.example/"), { https_proxy: "http://lower:1" })).toBe("http://lower:1");
|
||||
});
|
||||
it("NO_PROXY 匹配", () => {
|
||||
expect(noProxyMatches("a.example.com", "example.com")).toBe(true);
|
||||
expect(noProxyMatches("example.com", ".example.com")).toBe(true);
|
||||
expect(noProxyMatches("badexample.com", "example.com")).toBe(false);
|
||||
expect(noProxyMatches("x", "*")).toBe(true);
|
||||
expect(noProxyMatches("127.0.0.1", "localhost, 127.0.0.1:8080")).toBe(true);
|
||||
});
|
||||
it("出口键不含代理口令", () => {
|
||||
expect(egressKey(siteEgress(false, "http://u:p@relay:7891"))).toBe("direct");
|
||||
expect(egressKey(siteEgress(true, "http://u:p@relay:7891"))).toBe("proxy:http://relay:7891");
|
||||
expect(egressKey(siteEgress(true, null))).toBe("direct");
|
||||
});
|
||||
});
|
||||
@@ -29,6 +29,7 @@ import type { SiteDialog, useSiteChecks } from "./site-dialogs.tsx";
|
||||
import {
|
||||
ACCOUNT_OVERALL,
|
||||
bytes,
|
||||
cloudflareLine,
|
||||
dataAccount,
|
||||
hasLogin,
|
||||
healthLine,
|
||||
@@ -291,6 +292,7 @@ function AccountSiteCard({ site, checks, onDialog }: Props) {
|
||||
const hmsg = healthMessage(site.lastHealth);
|
||||
const stale = site.snapshotEnabled && snapshotStale(site);
|
||||
const circuit = circuitLabel(site);
|
||||
const cf = cloudflareLine(site.config);
|
||||
const lastErr = acc && acc.status !== "active" ? acc.lastError : null;
|
||||
|
||||
return (
|
||||
@@ -321,6 +323,15 @@ function AccountSiteCard({ site, checks, onDialog }: Props) {
|
||||
<AccountBadges site={site} />
|
||||
{u?.vip ? <StatusBadge tone="neutral">VIP</StatusBadge> : null}
|
||||
{circuit ? <StatusBadge tone="warning">{circuit}</StatusBadge> : null}
|
||||
{cf ? (
|
||||
cf.takeover ? (
|
||||
<Link to={`${consolePath("sites", site.id)}?tab=origin`} title="在原站视图里接管会话，完成验证后交还">
|
||||
<StatusBadge tone={cf.tone}>{cf.text}</StatusBadge>
|
||||
</Link>
|
||||
) : (
|
||||
<StatusBadge tone={cf.tone}>{cf.text}</StatusBadge>
|
||||
)
|
||||
) : null}
|
||||
</div>
|
||||
|
||||
{lastErr ? (
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { healthLine, healthMessage, healthTechItems } from "./site-fields.ts";
|
||||
import { cloudflareLine, cloudflareTechItems, healthLine, healthMessage, healthTechItems } from "./site-fields.ts";
|
||||
|
||||
describe("健康检测文案(HC-1)", () => {
|
||||
it("人机验证单独成类;失败原因一句,原始判定进技术信息", () => {
|
||||
@@ -24,3 +24,51 @@ describe("健康检测文案(HC-1)", () => {
|
||||
expect(healthTechItems({ ok: true, status: "ok", latencyMs: 1, message: null })).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("cloudflareLine / cloudflareTechItems(CF-2)", () => {
|
||||
const now = new Date("2026-09-30T12:00:00+08:00");
|
||||
const cfg = (cloudflare: Record<string, unknown>) => ({ cloudflare });
|
||||
it("没遇到过挑战不显示", () => {
|
||||
expect(cloudflareLine({}, now)).toBeNull();
|
||||
expect(cloudflareLine(cfg({ lastSolvedBy: "auto" }), now)).toBeNull();
|
||||
expect(cloudflareTechItems({})).toEqual([]);
|
||||
});
|
||||
it("需要你验证 > 浏览器代理中 > 自动通过", () => {
|
||||
const at = "2026-09-30T11:00:00+08:00";
|
||||
expect(cloudflareLine(cfg({ lastChallengeAt: at, pending: "human", lastSolvedBy: "auto" }), now)).toEqual({
|
||||
text: "Cloudflare：需要你验证",
|
||||
tone: "warning",
|
||||
takeover: true,
|
||||
});
|
||||
const until = new Date("2026-09-30T17:30:00+08:00");
|
||||
const line = cloudflareLine(cfg({ lastChallengeAt: at, browserUntil: until.toISOString() }), now);
|
||||
const hm = `${String(until.getHours()).padStart(2, "0")}:${String(until.getMinutes()).padStart(2, "0")}`;
|
||||
expect(line?.text).toBe(`Cloudflare：浏览器代理中，至 ${hm}`);
|
||||
expect(cloudflareLine(cfg({ lastChallengeAt: at, lastSolvedBy: "vision" }), now)?.text).toBe(
|
||||
"Cloudflare：自动通过",
|
||||
);
|
||||
// 代理期已过、没通过过、有错误
|
||||
expect(
|
||||
cloudflareLine(cfg({ lastChallengeAt: at, browserUntil: at, lastError: "浏览器服务不可用" }), now)?.text,
|
||||
).toBe("Cloudflare：没能通过");
|
||||
});
|
||||
it("原始记录进技术信息", () => {
|
||||
const items = cloudflareTechItems(
|
||||
cfg({
|
||||
lastChallengeAt: "2026-09-30T11:00:00+08:00",
|
||||
lastRule: "header cf-mitigated=challenge",
|
||||
lastResult: "reuse_fails",
|
||||
lastSolvedBy: "auto",
|
||||
cfReuse: "fails",
|
||||
}),
|
||||
);
|
||||
expect(items.map((i) => i.label)).toEqual([
|
||||
"最近遇到挑战",
|
||||
"挑战规则",
|
||||
"处理结果",
|
||||
"最近通过方式",
|
||||
"cf_clearance 复用",
|
||||
]);
|
||||
expect(items.find((i) => i.label === "处理结果")?.value).toContain("reuse_fails");
|
||||
});
|
||||
});
|
||||
@@ -11,6 +11,7 @@ import {
|
||||
type SiteListItem,
|
||||
type SiteLoginFailureReason,
|
||||
type SiteRateLimit,
|
||||
siteCloudflareState,
|
||||
} from "@umt/contracts";
|
||||
import type { StatusTone } from "~/components/console/status-badge.tsx";
|
||||
import { relTime } from "~/components/workbench/release-fields.ts";
|
||||
@@ -180,6 +181,64 @@ export function healthTechItems(h: SiteLastHealth | null): { label: string; valu
|
||||
].filter((it) => it.value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Cloudflare 状态一行(CF-2):没遇到过挑战不显示。
|
||||
* 需要你验证 → 浏览器代理中,至 hh:mm → 自动通过(最近一次怎么过的)。
|
||||
*/
|
||||
export function cloudflareLine(
|
||||
config: unknown,
|
||||
now: Date = new Date(),
|
||||
): { text: string; tone: StatusTone; takeover: boolean } | null {
|
||||
const st = siteCloudflareState(config);
|
||||
if (!st?.lastChallengeAt) return null;
|
||||
if (st.pending === "human") return { text: "Cloudflare：需要你验证", tone: "warning", takeover: true };
|
||||
if (st.browserUntil && Date.parse(st.browserUntil) > now.getTime()) {
|
||||
const t = new Date(st.browserUntil);
|
||||
const hm = `${String(t.getHours()).padStart(2, "0")}:${String(t.getMinutes()).padStart(2, "0")}`;
|
||||
return { text: `Cloudflare：浏览器代理中，至 ${hm}`, tone: "neutral", takeover: false };
|
||||
}
|
||||
if (st.lastSolvedBy) return { text: "Cloudflare：自动通过", tone: "success", takeover: false };
|
||||
if (st.lastError) return { text: "Cloudflare：没能通过", tone: "warning", takeover: false };
|
||||
return null;
|
||||
}
|
||||
|
||||
const CF_SOLVED_BY: Record<string, string> = {
|
||||
auto: "等它自动放行",
|
||||
vision: "视觉子流程点了复选框",
|
||||
human: "你在原站视图里通过",
|
||||
};
|
||||
const CF_RESULT: Record<string, string> = {
|
||||
passed: "已通过",
|
||||
reuse_ok: "已通过，HTTP 复用 cf_clearance 有效",
|
||||
reuse_fails: "已通过，HTTP 复用 cf_clearance 无效，改由浏览器代抓",
|
||||
browser: "由浏览器代抓",
|
||||
browser_challenge: "浏览器也被拦下",
|
||||
needs_human: "等你在原站视图里验证",
|
||||
failed: "没能通过",
|
||||
skipped: "没有已登录的账号，未处理",
|
||||
};
|
||||
|
||||
/** Cloudflare 原始记录 → 技术信息条目 */
|
||||
export function cloudflareTechItems(config: unknown): { label: string; value: string; mono?: boolean }[] {
|
||||
const st = siteCloudflareState(config);
|
||||
if (!st?.lastChallengeAt) return [];
|
||||
const t = (v: string | null) => (v ? new Date(v).toLocaleString("zh-CN", { hour12: false }) : "");
|
||||
return [
|
||||
{ label: "最近遇到挑战", value: t(st.lastChallengeAt) },
|
||||
{ label: "挑战规则", value: st.lastRule ?? "", mono: true },
|
||||
{ label: "处理结果", value: st.lastResult ? `${CF_RESULT[st.lastResult] ?? st.lastResult}(${st.lastResult})` : "" },
|
||||
{ label: "最近通过方式", value: st.lastSolvedBy ? (CF_SOLVED_BY[st.lastSolvedBy] ?? st.lastSolvedBy) : "" },
|
||||
{ label: "最近通过时间", value: t(st.lastSolvedAt) },
|
||||
{
|
||||
label: "cf_clearance 复用",
|
||||
value: st.cfReuse === "ok" ? "有效(ok)" : st.cfReuse === "fails" ? "无效(fails)" : "",
|
||||
},
|
||||
{ label: "浏览器代理至", value: t(st.browserUntil) },
|
||||
{ label: "最近尝试", value: t(st.lastAttemptAt) },
|
||||
{ label: "最近错误", value: st.lastError ?? "" },
|
||||
].filter((it) => it.value);
|
||||
}
|
||||
|
||||
/** 限流:"20 次/分";空对象 = 默认 */
|
||||
export function rateLimitText(r: SiteRateLimit | null | undefined): string {
|
||||
if (!r || (!r.requestsPerMinute && !r.minIntervalMs && !r.concurrency)) return "默认";
|
||||
@@ -202,7 +261,10 @@ export function loginFailureText(
|
||||
case "two_factor":
|
||||
return { text: "登录失败：站点要求二次验证。填写 TOTP 密钥后重试，或改用 Cookie", suggestCookie: true };
|
||||
case "challenge":
|
||||
return { text: "站点返回 Cloudflare 挑战页，无法自动登录。改用 Cookie", suggestCookie: true };
|
||||
return {
|
||||
text: "站点要求人机验证（Cloudflare）。会用浏览器自动过，过不去时在待处理里提醒你",
|
||||
suggestCookie: false,
|
||||
};
|
||||
case "bad_credentials":
|
||||
return { text: "登录失败：账号或密码错误", suggestCookie: false };
|
||||
case "account_blocked":
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
import type { SiteListItem } from "@umt/contracts";
|
||||
import { Link } from "react-router";
|
||||
import { useFeeds } from "~/api/feeds.ts";
|
||||
import { accessSummary, useSiteRequestStats, useSiteUserData } from "~/api/sites.ts";
|
||||
import { DetailSection } from "~/components/console/detail-sheet.tsx";
|
||||
import { ErrorState, LoadingState } from "~/components/console/empty-state.tsx";
|
||||
import { KvList } from "~/components/console/kv-list.tsx";
|
||||
import { consolePath } from "~/components/console/nav.ts";
|
||||
import { StatTile } from "~/components/console/stat-tile.tsx";
|
||||
import { StatusBadge } from "~/components/console/status-badge.tsx";
|
||||
import { TechInfo } from "~/components/console/tech-info.tsx";
|
||||
@@ -12,6 +14,8 @@ import {
|
||||
ACCOUNT_OVERALL,
|
||||
AUTH_KIND_LABEL,
|
||||
bytes,
|
||||
cloudflareLine,
|
||||
cloudflareTechItems,
|
||||
dataAccount,
|
||||
hasLogin,
|
||||
healthLine,
|
||||
@@ -26,6 +30,28 @@ import {
|
||||
} from "./site-fields.ts";
|
||||
import { UserDataChart } from "./user-data-chart.tsx";
|
||||
|
||||
/** Cloudflare 状态行(CF-2);需要你验证时给原站视图入口 */
|
||||
function cloudflareRow(site: SiteListItem) {
|
||||
const cf = cloudflareLine(site.config);
|
||||
if (!cf) return [];
|
||||
const text = cf.text.replace(/^Cloudflare：/, "");
|
||||
return [
|
||||
{
|
||||
label: "Cloudflare",
|
||||
value: cf.takeover ? (
|
||||
<span>
|
||||
<span className="text-warning">{text}</span>
|
||||
<Link className="ml-2 text-meta underline" to={`${consolePath("sites", site.id)}?tab=origin`}>
|
||||
去原站视图
|
||||
</Link>
|
||||
</span>
|
||||
) : (
|
||||
text
|
||||
),
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
const CAP_LABEL = [
|
||||
["search", "搜索"],
|
||||
["list", "列表抓取"],
|
||||
@@ -180,11 +206,16 @@ function AccountSiteOverview({ site }: { site: SiteListItem }) {
|
||||
),
|
||||
},
|
||||
{ label: "代理", value: site.proxy ? "使用" : "不使用" },
|
||||
...cloudflareRow(site),
|
||||
...(site.kind && site.kind !== "pt" ? [{ label: "类型", value: kindLabel(site.kind) }] : []),
|
||||
{ label: "获取方式", value: capabilityText(site) },
|
||||
]}
|
||||
/>
|
||||
<TechInfo className="mt-2" columns={1} items={healthTechItems(site.lastHealth)} />
|
||||
<TechInfo
|
||||
className="mt-2"
|
||||
columns={1}
|
||||
items={[...healthTechItems(site.lastHealth), ...cloudflareTechItems(site.config)]}
|
||||
/>
|
||||
</DetailSection>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -10,6 +10,9 @@
|
||||
* GET /umt/identity → 固定浏览器身份的公开部分:UA、OS、版本、生成时间(BR-4)
|
||||
* PUT /umt/sessions/:userId/proxy { proxy: {server, username?, password?} | null }
|
||||
* → 该 profile 的出口:站点开代理 = 与 HTTP 同一代理;null = 直连
|
||||
* POST /umt/tabs/:tabId/mouse-click { userId, x, y } → 按视口坐标发真实鼠标事件(page.mouse.click,CF-2):
|
||||
* 跨域 iframe 里的 Cloudflare Turnstile 复选框也点得到;
|
||||
* 标签页不在回 422(与"插件没有这个端点"的 404 区分)
|
||||
* Cookie 值只在请求/响应体里,不写日志(只记个数);代理只记主机,不记账号口令。
|
||||
*
|
||||
* BR-4 固定浏览器身份(docs/runbook/browser.md §11):`browser:launching` 钩子里,首次用 camoufox-js 生成一份 os=windows
|
||||
@@ -168,6 +171,16 @@ async function windowsFontconfig(installDir, log) {
|
||||
}
|
||||
}
|
||||
|
||||
/** 上游会话 → 标签页状态(与 server.js findTab 相同的遍历:tabGroups: Map<sessionKey, Map<tabId, TabState>>) */
|
||||
function findTab(session, tabId) {
|
||||
if (!session?.tabGroups) return null;
|
||||
for (const group of session.tabGroups.values()) {
|
||||
const t = group.get(tabId);
|
||||
if (t?.page) return t;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
const FIELDS = ["name", "value", "domain", "path", "expires", "httpOnly", "secure", "sameSite"];
|
||||
|
||||
function parseUrls(raw) {
|
||||
@@ -364,7 +377,30 @@ export async function register(app, ctx) {
|
||||
|
||||
app.get("/umt/instance", ctx.auth(), (_req, res) => res.json(INSTANCE));
|
||||
|
||||
log("info", "umt-cookies plugin: registered cookies / instance / identity / egress endpoints", {
|
||||
// ── CF-2 真实坐标点击 ──
|
||||
app.post("/umt/tabs/:tabId/mouse-click", ctx.auth(), express.json({ limit: "4kb" }), async (req, res) => {
|
||||
try {
|
||||
const userId = ctx.normalizeUserId(String(req.body?.userId ?? ""));
|
||||
const x = Number(req.body?.x);
|
||||
const y = Number(req.body?.y);
|
||||
if (!userId) return res.status(400).json({ error: "userId is required" });
|
||||
if (!Number.isFinite(x) || !Number.isFinite(y) || x < 0 || y < 0 || x > 20000 || y > 20000)
|
||||
return res.status(400).json({ error: "x / y must be viewport CSS pixels" });
|
||||
const tab = findTab(ctx.sessions.get(userId), req.params.tabId);
|
||||
if (!tab) return res.status(422).json({ error: "tab not found" });
|
||||
const session = ctx.sessions.get(userId);
|
||||
session.lastAccess = Date.now();
|
||||
tab.toolCalls = (tab.toolCalls ?? 0) + 1;
|
||||
await ctx.withUserLimit(userId, () => tab.page.mouse.click(Math.round(x), Math.round(y), { delay: 60 }));
|
||||
log("info", "umt mouse click", { reqId: req.reqId, userId, tabId: req.params.tabId });
|
||||
res.json({ ok: true, tabId: req.params.tabId, url: tab.page.url() });
|
||||
} catch (err) {
|
||||
log("error", "umt mouse click failed", { reqId: req.reqId, error: err.message });
|
||||
res.status(500).json({ error: ctx.safeError(err) });
|
||||
}
|
||||
});
|
||||
|
||||
log("info", "umt-cookies plugin: registered cookies / instance / identity / egress / mouse-click endpoints", {
|
||||
identityMode,
|
||||
proxyMode,
|
||||
});
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
# CF-1 Cloudflare 验证处理(调研)
|
||||
|
||||
> 状态:CF-2 已按本文 §3 推荐方案 A→B 级联落地,另加出口统一(站点请求明确直连 / 站点代理)。运维与字段见
|
||||
> `docs/runbook/sites.md`「出口」「Cloudflare(CF-2)」、`docs/runbook/browser.md` §13。与本文的差异:同步 cf_clearance 单独存
|
||||
> `site_account.user_info.cloudflare`(值在 secret),普通浏览器回写仍剔除;视觉子流程最多 2 次;`site.config.cloudflare` 多了
|
||||
> `lastAttemptAt / lastSolvedAt / pending / sessionId / lastResult / lastRule / lastError`。
|
||||
|
||||
## 1. 复现(2026-09-30,HTTP 5 次、浏览器 5 次,间隔 ≥15s)
|
||||
| 请求 | 结果 |
|
||||
| --- | --- |
|
||||
|
||||
@@ -213,3 +213,5 @@ NFO / 图片投影、身份巡检与挂载根因聚合、HTTP 失败自动切浏
|
||||
- **观众详情页适配核对(DET-2)**:用真实详情页(≤3 次请求)逐项核对并修,脱敏样本进差分测试。
|
||||
- **Cloudflare 验证处理(CF-1,用户 2026-09-30)**:搜索入口有时随机出 CF 验证。先用浏览器复现并研究:HTTP 端遇 CF 挑战时的处理策略(浏览器过挑战后同步 cf_clearance+UA 重试 / 该请求改由浏览器执行并解析 / 站点级"走浏览器"标记与时效),对照 MP(FlareSolverr / CloakBrowser)与 BR-4 的固定身份,出方案后实现。
|
||||
- DET-2 结果:观众详情页修了促销/到期、上传时间、做种数、hash、副标题、标签、上传者、简介清洗、截图过滤;站点没有 TMDB 与单种 H&R。**后续**:资源增加豆瓣 / IMDb 评分字段(页面有,本工具没字段);观众文件清单是点开后另加载的端点,待核对(需 1–2 次请求)。
|
||||
**CF-2 已实现(分支 cf2,待合并)**:出口统一(站点请求明确直连 / 站点代理)、挑战不算凭据错误、浏览器过挑战级联(自动 / 视觉子流程 / 转人工)、
|
||||
cf_clearance 有条件给 HTTP 重试、浏览器代抓与 6 小时浏览器代理期、站点页展示;见 `docs/runbook/sites.md`「Cloudflare(CF-2)」。
|
||||
@@ -13,7 +13,7 @@
|
||||
| 查询(read) | `catalog.lookup`、`release.search`、`release.inspect`、`inventory.query`、`subscription.list`、`decision.list`、`download.list`、`download.inspect`、`downloader.list`、`task.inspect`、`site.list`、`site.user_data` |
|
||||
| 计划(preview / propose) | `identity.resolve`、`download.plan`、`organize.plan`、`policy.preview` |
|
||||
| 执行(execute) | `download.submit`、`download.recheck`、`organize.execute`、`task.retry`、`task.cancel`、`policy.apply`、`site.health_check`、`site.refresh_login` |
|
||||
| 原站浏览器 | `browser.snapshot`、`browser.navigate`、`browser.interact`、`browser.download`、`browser.screenshot`、`browser.click_at`、`browser.type_at`、`browser.scroll`、`browser.wait`、`browser.solve_challenge`(见 [browser.md](browser.md)) |
|
||||
| 原站浏览器 | `browser.snapshot`、`browser.navigate`、`browser.interact`、`browser.download`、`browser.screenshot`、`browser.click_at`、`browser.type_at`、`browser.scroll`、`browser.wait`、`browser.solve_challenge`(见 [browser.md](browser.md));站点请求遇 Cloudflare 复选框时系统也会发起 `solve_challenge` 视觉子流程(发起人 `system:cloudflare`、trigger event、没有父会话,预算取设置,见 browser.md §13) |
|
||||
| 会话 | `agent.plan`、`agent.propose_options`、`agent.spawn`、`agent.await` |
|
||||
| 网页搜索 | `web.search`、`web.fetch`(见 [websearch.md](websearch.md)) |
|
||||
| 元数据补全 | `metadata.enrich`(propose,只出建议;网页搜索开启才提供,见 [identity.md](identity.md#元数据补全meta-1)) |
|
||||
|
||||
+35
-3
@@ -126,8 +126,9 @@ execute 类需要持续授权(`automation_grant`)或审批。元素引用只来
|
||||
不支持时截图不发送,能力改给已脱敏的文字快照并提示;`solve_challenge` 直接判定需人工。fake 上游按支持图片处理。
|
||||
- **坐标系**:服务端对每次截图读页面视口与 DPR(`evaluate`),`页面 CSS 像素 = 截图像素 × 视口宽 / 截图宽`;
|
||||
输出的 `coordinates` 记录截图尺寸、原始尺寸、视口、DPR(取不到时 `assumed = true`,按原图、DPR 1)。动作之后旧坐标系作废,
|
||||
必须用新截图的坐标。点击沿用 1.17.0 的做法:`elementFromPoint` 打标记再按选择器点,**画布(canvas)内的位置、跨域 iframe
|
||||
里的控件(如 Cloudflare Turnstile)点不到具体位置**,这类挑战实际只能转人工。
|
||||
必须用新截图的坐标。点击(CF-2 起)先走 umt 插件的 `POST /umt/tabs/{id}/mouse-click`——Playwright `page.mouse.click` 发真实鼠标事件,
|
||||
**跨域 iframe 里的控件(Cloudflare Turnstile)和画布也点得到**;插件没有这个端点(旧镜像、纯截图镜像)时退回 1.17.0 的做法:
|
||||
`elementFromPoint` 打标记再按选择器点(跨域 iframe / 画布点不到具体位置)。
|
||||
- **动作前提**:会话在自动化控制下(人工接管 / 等待处理中一律拒绝),当前页面主机在站点域名白名单内。
|
||||
- **敏感截图**:登录页地址、页面上有密码框、地址带凭据 → `sensitive = true`,默认不交给模型(不做像素遮蔽),改给已脱敏的文字快照;
|
||||
要交给模型需要 `automation_grant.action = browser.sensitive_screenshot`(或 `*`)的持续授权,且调用时传 `allowSensitive: true`。
|
||||
@@ -368,6 +369,7 @@ camofox 1.17.0 只有 `POST /sessions/:userId/cookies`(导入),而且要另一
|
||||
| `GET /umt/instance` | `{instanceId, startedAt}`:插件加载时生成,camofox 进程重启即变(BR-3 重启检测,§10) |
|
||||
| `GET /umt/identity` | 固定浏览器身份的公开部分(UA / OS / 版本,BR-4,§11.1) |
|
||||
| `PUT /umt/sessions/{userId}/proxy` `{proxy}` | 该 profile 的出口:`{server, username?, password?}` 或 `null` 直连(BR-4,§11.5) |
|
||||
| `POST /umt/tabs/{tabId}/mouse-click` `{userId, x, y}` | 按视口坐标发真实鼠标事件(`page.mouse.click`,CF-2,§13);标签页不在回 422 |
|
||||
|
||||
适配器先走插件;插件不在(404)再试上游导入端点(`CAMOFOX_API_KEY` 与访问凭据相同时可用,只能写不能读);都不行 → 会话照常打开,
|
||||
`credential.status = unsupported` 并说明。profile 落盘在 `/root/.camofox/profiles/<sha256(userId) 前 32 位>/storage-state.json`
|
||||
@@ -550,7 +552,7 @@ Windows 的真值);TLS / HTTP2 指纹是 Firefox 152 本身(与 OS 无关)。屏
|
||||
- Cookie 账号如果一直不开原站视图、也不换 Cookie,会一直待切换(界面写"待切换:下次刷新登录,或在原站视图里用过后改为跟随浏览器")。
|
||||
刷新登录对 Cookie 账号只校验,不切换。
|
||||
|
||||
### 11.4 Cloudflare Cookie 仅浏览器
|
||||
### 11.4 Cloudflare Cookie 仅浏览器(CF-2 起有条件给 HTTP)
|
||||
|
||||
`cf_clearance`、`__cf_bm`(及挑战过程中的 `cf_chl_*`)与取得它的浏览器绑定(UA + IP + TLS 指纹),Deno 的 TLS 指纹与 Firefox 必然不同,
|
||||
HTTP 端拿去用多半无效还可能触发风控。所以:
|
||||
@@ -559,6 +561,9 @@ HTTP 端拿去用多半无效还可能触发风控。所以:
|
||||
- 比较"有没有变化"时也不算它们(CF Cookie 轮换不会触发回写);refresh-login 从浏览器读回的 Cookie 同样剔除;
|
||||
- 站点账号页技术信息里显示"仅浏览器使用的 Cookie";
|
||||
- 用户手填的 Cookie 里带 `cf_clearance` 不处理(用户自己的选择)。
|
||||
- **CF-2 例外**:站点遇 Cloudflare 挑战、浏览器过了之后,账号 UA 来源 = 跟随浏览器、浏览器 UA 与 HTTP UA 相同、两端出口相同时,
|
||||
`cf_clearance`(只有它,`__cf_bm` 仍不带)单独存进 `site_account.user_info.cloudflare`(值在 secret)并由 HTTP 带上重试 1 次,
|
||||
结果按站点记 `cfReuse`;普通回写仍剔除它(见 sites.md「Cloudflare(CF-2)」)。
|
||||
|
||||
### 11.5 出口 IP 按站点统一
|
||||
|
||||
@@ -642,3 +647,30 @@ camofox 按 **API 调用**判断活动,有三道空闲回收(`docker-compose.yml
|
||||
|
||||
核验:`apps/server/src/modules/browser/__tests__/browser-keepalive.test.ts`(登记 / 引用计数 / 复查;对 mock camofox:接管期间保活、
|
||||
不记自动化活动、交还后停;桌面连接期间标签页被回收 → 保活时自愈重连;智能体会话结束后停、会话关闭撤销)。真机未核验。
|
||||
|
||||
## 13. Cloudflare 挑战(CF-2)
|
||||
|
||||
站点 HTTP 遇挑战时由站点请求一侧(sources-identity `CloudflareGate`)经钩子调用浏览器模块,流程与状态见 sites.md「Cloudflare(CF-2)」。
|
||||
浏览器这边:
|
||||
|
||||
- `solveSiteChallenge({siteId, accountId, url})`:该账号 profile 的自动化会话(没有就按站点打开;人工接管中 / 等人处理中不打扰)导航到
|
||||
同一地址(取站点令牌)→ 用 `POST /tabs/{id}/evaluate` 读页面(地址、标题、outerHTML、正文样本、`navigator.userAgent`,适配器
|
||||
`pageContent`)按 `classifyChallengePage` 分类:自动(JS 检查)/ 可点(Turnstile、「确认您是真人」)/ 真验证码(hCaptcha、reCAPTCHA、
|
||||
选图、输码)→ ① 每 1.5 秒读一次,最多 15 秒 → ② 可点的派视觉子流程(`browser.solve_challenge`,系统发起:发起人 `system:cloudflare`、
|
||||
trigger event、没有父会话;步数 4、时长 ≤ 120 秒、token / 费用取 `agent.maxTokensPerRun` / `maxCostUsdPerRun`;最多 2 次;
|
||||
自动化总开关关闭或只读对账时不派)→ ③ 没过:会话转等待 `cf_challenge`,待处理事项「<站点>需要你在原站视图里过一次验证」
|
||||
(视觉子流程失败留下的 `challenge_unsolved` 事项改成这个说法)。通过后读回 profile 里的 `cf_clearance`、UA 与会话出口(`recovery_state.egress`,
|
||||
插件按站点下发出口成功才有值)。
|
||||
- `fetchSiteHtml({siteId, accountId, url})`:同一会话导航(取站点令牌)后读 outerHTML;仍是挑战页 / 会话在人工处理 / 站点停用时不返回页面。
|
||||
- 同一 profile 的过挑战与代抓在进程内串行(profile 锁);跨进程同站 30 分钟一次靠站点状态的条件更新。
|
||||
- 交还会话(给自动化)时:站点在等人过验证 → 读回 `cf_clearance` / UA 通知站点请求继续(`CloudflareGate.afterHuman`),待处理事项随交还关闭。
|
||||
- 插件新端点 `POST /umt/tabs/{tabId}/mouse-click`:`click_at`(与原站视图截图模式的坐标点击)都先走它。**需要重建 camofox 镜像**才有
|
||||
(`docker compose build camofox && docker compose up -d camofox`,只重跑插件层);没重建时退回 elementFromPoint,Turnstile 点不到,
|
||||
可点类挑战会转人工。
|
||||
|
||||
核验:mock camofox(`cloudflare` 模式:auto / interactive / captcha)见 `browser-cloudflare.test.ts`、`umt-plugin.test.ts`、E14。
|
||||
插件鼠标点击的真实容器核验见下表(本地临时容器,只开本地页面,不访问真实验证页)。
|
||||
|
||||
| 日期 | 做法 | 结果 |
|
||||
| --- | --- | --- |
|
||||
| 2026-09-30 | 临时容器 `umt-cf2-camofox`(镜像 `umt-browser-desktop:1.17.0`,只读挂载新插件目录),容器内 node 起两个本地页面:父页 `localhost:8001`(标题 Just a moment...)内嵌**跨域** iframe `localhost:8002`(复选框,`change` 事件 `isTrusted` 才向父页 postMessage);适配器 `openSession` → `pageContent` → `click({x, y})` → `pageContent` | 插件路径:`POST /umt/tabs/{id}/mouse-click` 命中复选框,iframe 里收到可信事件,父页标题变为 passed;`pageContent` 读到 UA `…Windows NT 10.0; Win64; x64; rv:152.0…`。旧路径(elementFromPoint)只能标到 iframe 元素本身、按元素中心点击,点不到指定坐标。未访问任何外部站点,容器已删除 |
|
||||
+67
-2
@@ -97,7 +97,7 @@
|
||||
| `defaultHealthCheckIntervalSec` | 21600 | 定时健康检测的最短间隔(与定义取较长) |
|
||||
| `defaultDetailCacheSec` | 86400 | 详情缓存:期内 `GET /sites/{id}/releases/{rid}?refresh=true` 与 `release.inspect` 不再请求站点(与定义取较长);0 = 不缓存 |
|
||||
| `userAgent` | 空(跟随浏览器) | 全局 UA 覆盖:空 = 站点请求跟随浏览器固定身份的 UA(Windows Firefox);填了则账号没手动 UA 时都用它。优先级:账号手动 > 待切换 > 全局 > 跟随浏览器,见 browser.md §11.2 |
|
||||
| `proxyUrl` | null | 站点 `proxy=true` 时,适配器、登录、用户数据、RSS 请求都经此代理(`Deno.createHttpClient`),浏览器会话也走同一代理(browser.md §11.5);地址要让 app 与 camofox 容器都能连 |
|
||||
| `proxyUrl` | null | 站点 `proxy=true` 时,适配器、登录、用户数据、RSS 请求都经此代理(`Deno.createHttpClient`),浏览器会话也走同一代理(browser.md §11.5);没开代理的站点**明确直连**,不读容器的 `HTTP(S)_PROXY`(见下「出口」);地址要让 app 与 camofox 容器都能连 |
|
||||
| `userDataMaxSeedingPages` | 10 | 用户数据做种分页上限 |
|
||||
|
||||
运行时每 30 s 最多重读一次设置;设置变化会让站点适配器按新签名重建。
|
||||
@@ -137,9 +137,74 @@
|
||||
- 注册数:首批能力 17 → 21,启动断言 `assertCapabilityCount` 跟随 `CAPABILITY_IDS.length`。
|
||||
- 未接:`agent/invoke-limits.ts` 的站点并发上限还没把 `site.health_check` / `site.refresh_login` 算作外部站点调用;Agent 默认只读白名单(`agent/runner.ts`)也还没加 `site.list` / `site.user_data`。两处都在 agent 模块,留给该模块的负责人。
|
||||
|
||||
## 出口:哪些请求走哪条(CF-2)
|
||||
|
||||
代码 `apps/server/src/egress.ts`。之前 app 的站点请求用 `globalThis.fetch`,Deno 会读容器的 `HTTPS_PROXY`(本机栈 override 里的 mihomo 中继),
|
||||
站点没开代理时 HTTP 端走代理、浏览器却直连,两端出口 IP 不同(CF-1 实测 HKG / DFW),这是 Cloudflare 随机拦截的来源之一。
|
||||
|
||||
| 请求 | 出口 |
|
||||
| --- | --- |
|
||||
| 站点请求:适配器(列表 / 搜索 / 详情 / 下载 / 健康检测)、RSS、账密登录、用户数据(`SourcesRuntime.siteFetch`) | 站点开「使用代理」且设了 `sites.proxyUrl` → 该代理;否则**明确直连** |
|
||||
| 浏览器会话(camofox) | 同一站点同一出口:插件按 profile 设代理或直连(browser.md §11.5);`UMT_BROWSER_PROXY_MODE=env` 时按容器环境变量(与 HTTP 不一定一致) |
|
||||
| TMDB、Brave 搜索 / 网页读取、TMDB 图片缓存、海报与站点截图代理 | 照旧按环境变量 `HTTPS_PROXY` / `HTTP_PROXY` / `ALL_PROXY` / `NO_PROXY`(进程启动时读一次),改为代码里显式建客户端(`externalFetch`),行为不变 |
|
||||
| 模型上游(pi SDK)、下载器、媒体服务器、camofox 本身 | 不在本表范围,仍是各自的 fetch(模型上游随环境变量) |
|
||||
|
||||
**Deno 2.9 的语义**(2026-09-30 本机实测,`test/sources-identity/egress.test.ts` 锁住):缺省 `fetch` 与 `Deno.createHttpClient({})` 都读
|
||||
`HTTP_PROXY` / `HTTPS_PROXY` / `ALL_PROXY`(大小写都认)与 `NO_PROXY`,读取发生在**客户端创建时**;`createHttpClient` 没有"不走代理"的选项,
|
||||
`NO_PROXY=*` 不认;显式 `proxy` 排在环境变量代理前面,但 `NO_PROXY` 命中的主机连显式代理也绕过。所以直连客户端是在创建那一刻临时拿掉
|
||||
这几个环境变量(同步调用,同一 isolate 内不会插进别的代码)、建完立即还原;代理客户端同样在拿掉后创建,只走给定代理。
|
||||
|
||||
排查:站点 HTTP 与浏览器是否同出口,在 app 容器里 `curl` 不能代表 Deno(curl 也读环境变量);看站点页技术信息里的 Cloudflare 记录,
|
||||
或临时对一个会回显来源 IP 的地址各发一次(不要对 PT 站做)。
|
||||
|
||||
## Cloudflare(CF-2)
|
||||
|
||||
方案见 `docs/design/cloudflare-handling.md`。代码:`packages/adapters/src/sites/challenge.ts`(识别)、
|
||||
`apps/server/src/modules/sources-identity/service/cloudflare.ts`(级联,`CloudflareGate`)、`modules/browser/service.ts`
|
||||
(`solveSiteChallenge` / `fetchSiteHtml`)、`modules/agent/challenge.ts`(`systemChallengeVision`)。
|
||||
|
||||
**识别**:站点响应带 `cf-mitigated: challenge`(任何状态码),或 403 / 503 / 429 且正文是挑战页(标题 Just a moment / Attention Required、
|
||||
`cf_chl_opt` 等,不认正常页面也会注入的 `/cdn-cgi/challenge-platform/`)→ `SiteError("challenge")`。**不是凭据错误**:快照 / 详情不调
|
||||
`onAuthError`、不置账号失效、不排自动重登;健康检测记 `challenge`(degraded);刷新登录 / 用户数据遇挑战只写 `last_error`,账号状态不变
|
||||
(验证码、二次验证仍是账号状态 `challenge`)。
|
||||
|
||||
**级联**(站点请求经 `siteFetch` 自动走,调用方无感):
|
||||
|
||||
1. 站点在浏览器代理期(`config.cloudflare.browserUntil` 未到)且请求是页面(GET + Accept `text/html`)→ 直接由浏览器代抓,不发 HTTP;
|
||||
2. 发 HTTP;满足复用条件时带上浏览器拿到的 `cf_clearance`(`__cf_bm` 不带);
|
||||
3. 遇挑战 → 记 `lastChallengeAt` / `lastRule`;只处理**有登录态账号**的站点(第一个启用、未失效、确有 Cookie 的 Cookie / 账密账号);
|
||||
4. 过挑战:同站 30 分钟最多一次(`lastAttemptAt` 条件更新,跨进程)、进程内同一时刻只跑一次,浏览器用该账号 profile 打开同一地址:
|
||||
① 轮询页面等它自己过(≤ 15 秒)→ ② 「我是人类」复选框 / Turnstile:派视觉子流程点一下(系统发起、没有父会话,预算取设置,最多 2 次;
|
||||
自动化总开关关闭或只读对账时跳过)→ ③ 选图 / 输码类或仍没过:会话转等待(`cf_challenge`),待处理事项「<站点>需要你在原站视图里过一次验证」,
|
||||
站点记 `pending = human`。不接任何打码服务;
|
||||
5. 通过 → 读回 `cf_clearance` 与 `navigator.userAgent`;**只有账号 UA 来源 = 跟随浏览器、UA 相同、两端出口相同**(浏览器出口按站点下发成功才算)
|
||||
时才存进账号(`site_account.user_info.cloudflare = {secretRef, userAgent, egress, at}`,值在 secret 里),HTTP 重试 1 次:过了记
|
||||
`cfReuse = ok`,仍被挑战记 `cfReuse = fails`(之后跳过复用;UA 或出口变了自动清掉重试);
|
||||
6. 仍被挑战 → 页面请求改由浏览器导航取 outerHTML 交给原解析器(受站点限流令牌与 profile 锁约束),站点记 6 小时浏览器代理期,到期再试 HTTP;
|
||||
下载 .torrent、RSS、POST 这类不能代抓的请求照常报挑战。
|
||||
|
||||
人在原站视图里过完验证、交还会话(交还给自动化)→ 读回 `cf_clearance` / UA,清 `pending`,记 `lastSolvedBy = human`,并允许马上再试(清
|
||||
`lastAttemptAt`)。
|
||||
|
||||
`site.config.cloudflare` 字段(契约 `SiteCloudflareState`):`browserUntil`、`cfReuse`(ok / fails)、`lastChallengeAt`、`lastSolvedBy`
|
||||
(auto / vision / human)、`lastSolvedAt`、`lastAttemptAt`、`pending`(human)、`sessionId`、`lastResult`、`lastRule`、`lastError`。
|
||||
改这个键不会让站点适配器重建。
|
||||
|
||||
**界面**:站点卡片与站点概览「站点」一栏显示一行 Cloudflare 状态(没遇到过挑战不显示):「需要你验证」(卡片上可点,进原站视图)/
|
||||
「浏览器代理中，至 hh:mm」/「自动通过」/「没能通过」;原始记录进概览的技术信息。
|
||||
|
||||
**测试**:`packages/adapters/src/sites/challenge.test.ts`(分类)、`apps/server/test/sources-identity/egress.test.ts`(出口)、
|
||||
`cloudflare.test.ts`(级联:自动通过 / 视觉 / 转人工 / 复用失败回退浏览器 / 节流单飞 / 同步条件 / 健康)、
|
||||
`modules/browser/__tests__/browser-cloudflare.test.ts`(mock camofox:自动放行、插件真实鼠标点击、选图转人工、交还继续、代抓与 profile 锁)、
|
||||
`test/e2e/e14-vision.test.ts`(系统发起的视觉子流程)。真机:未对真实站点验证(CF-2 只用 mock);插件鼠标点击见 browser.md §13。
|
||||
|
||||
**要人处理时**:站点页 Cloudflare 行显示「需要你验证」→ 原站视图接管 → 过验证 → 交还。一直过不去(站点绑定 TLS 指纹、代理 IP 被拉黑):
|
||||
关掉站点代理或换出口;实在不行关闭该站点的定时快照,只保留 RSS。
|
||||
|
||||
## 待核验
|
||||
|
||||
1. takelogin 表单字段、二次验证字段名、错误文本全部只对 `@umt/testkit` mock 站点验证过,未连接真实站点。
|
||||
2. 失败文本归类是关键词匹配(`classifyLoginError`),真实站点的措辞可能落到 `unknown`(状态记为 expired,原文截 200 字写进 `last_error`)。
|
||||
3. Cloudflare 站点只识别并转待处理,不做浏览器登录。
|
||||
3. Cloudflare:识别与级联见上「Cloudflare(CF-2)」,只对 mock 验证过;真实站点的挑战页类型、cf_clearance 能否给 HTTP 复用(是否绑 TLS)
|
||||
按站点记 `cfReuse` 在线验证。账密登录遇挑战仍不做浏览器登录。
|
||||
4. 馒头用户数据:`MTorrent` 解析器的做种列表首页按 GET 请求,真实 API 可能要求 POST。
|
||||
@@ -27,6 +27,8 @@ function realUpstream(handler: (c: Call) => Reply | undefined) {
|
||||
calls.push(call);
|
||||
const r =
|
||||
handler(call) ??
|
||||
// 真实上游(不带 umt 插件)没有 /umt/* 路由
|
||||
(call.path.startsWith("/umt/") ? { status: 404, json: { error: "Not found" } } : undefined) ??
|
||||
(call.path === "/tabs" && call.method === "POST"
|
||||
? { json: { tabId: "t-1", url: `${ORIGIN}/login.php`, httpStatus: 200, navigationOk: true } }
|
||||
: { json: { ok: true } });
|
||||
|
||||
@@ -43,6 +43,7 @@ import {
|
||||
type NavigationResult,
|
||||
type OpenSessionRequest,
|
||||
type OpenSessionResult,
|
||||
type PageContent,
|
||||
type PageMetrics,
|
||||
profileKeyFor,
|
||||
type ReopenSessionRequest,
|
||||
@@ -143,6 +144,14 @@ const PAGE_METRICS_EXPR = `/* umt-metrics */ (() => ({ w: innerWidth, h: innerHe
|
||||
pw: !!document.querySelector("input[type=password]"),
|
||||
text: String((document.body && document.body.innerText) || "").slice(0, 4000) }))()`;
|
||||
|
||||
/** 页面内容表达式(mock 按 umt-page 标记识别) */
|
||||
function pageContentExpr(maxChars: number): string {
|
||||
return `/* umt-page */ (() => ({ url: location.href, title: document.title,
|
||||
html: document.documentElement ? document.documentElement.outerHTML.slice(0, ${maxChars}) : "",
|
||||
text: String((document.body && document.body.innerText) || "").slice(0, 20000),
|
||||
ua: navigator.userAgent }))()`;
|
||||
}
|
||||
|
||||
const LOGIN_PATH = /(?:^|\/)(?:login|takelogin|signin|sign_in|auth)(?:\.php)?(?:\/|$|\?)/i;
|
||||
|
||||
export class CamofoxBrowserProvider implements BrowserProvider {
|
||||
@@ -159,6 +168,8 @@ export class CamofoxBrowserProvider implements BrowserProvider {
|
||||
private cookieApi: "plugin" | "upstream" | "none" | null = null;
|
||||
/** 实例标识接口:plugin = umt 插件 GET /umt/instance;health = 只能从 /health 里找;null = 未探测 */
|
||||
private instanceApi: "plugin" | "health" | null = null;
|
||||
/** 真实鼠标坐标点击(CF-2):plugin = umt 插件 `POST /umt/tabs/{id}/mouse-click`;none = 插件没有,退回 elementFromPoint;null = 未探测 */
|
||||
private mouseApi: "plugin" | "none" | null = null;
|
||||
|
||||
constructor(private readonly cfg: CamofoxConfig) {
|
||||
this.base = cfg.baseUrl.replace(/\/+$/, "");
|
||||
@@ -419,7 +430,10 @@ export class CamofoxBrowserProvider implements BrowserProvider {
|
||||
* 该 profile 的出口(BR-4 出口 IP 按站点统一):umt 插件 `PUT /umt/sessions/{userId}/proxy`。
|
||||
* proxy = null 直连;活动上下文出口不同时插件会先关掉它(落盘登录状态)。插件不在报 not_implemented。
|
||||
*/
|
||||
async setEgress(profileKey: string, proxy: BrowserEgressProxy | null): Promise<{ restarted: boolean }> {
|
||||
async setEgress(
|
||||
profileKey: string,
|
||||
proxy: BrowserEgressProxy | null,
|
||||
): Promise<{ restarted: boolean; mode?: string }> {
|
||||
try {
|
||||
const body = await this.json(
|
||||
await this.request("PUT", `/umt/sessions/${encodeURIComponent(profileKey)}/proxy`, {
|
||||
@@ -428,7 +442,8 @@ export class CamofoxBrowserProvider implements BrowserProvider {
|
||||
}),
|
||||
"/umt/sessions/{userId}/proxy",
|
||||
);
|
||||
return { restarted: body.restarted === true };
|
||||
const mode = str(body.mode);
|
||||
return { restarted: body.restarted === true, ...(mode ? { mode } : {}) };
|
||||
} catch (e) {
|
||||
if (e instanceof BrowserError && e.httpStatus === 404)
|
||||
throw new BrowserError("not_implemented", "浏览器服务不支持按站点设出口(镜像未带新版 umt-cookies 插件)");
|
||||
@@ -1047,6 +1062,22 @@ export class CamofoxBrowserProvider implements BrowserProvider {
|
||||
await this.withProfile(s, async () => {
|
||||
this.assertActor(s, ref.as);
|
||||
const tab = this.tabOf(s, ref);
|
||||
// 坐标点击先走 umt 插件的真实鼠标事件(page.mouse.click):跨域 iframe 里的控件(Cloudflare Turnstile)与画布也点得到(CF-2)
|
||||
if (
|
||||
action === "click" &&
|
||||
typeof payload.x === "number" &&
|
||||
typeof payload.y === "number" &&
|
||||
this.mouseApi !== "none"
|
||||
) {
|
||||
const done = await this.mouseClick(s, tab, payload.x, payload.y);
|
||||
if (done) {
|
||||
this.touch(s, done.url ?? s.rawUrl);
|
||||
if (done.url) tab.url = s.session.currentUrl;
|
||||
s.currentTabId = tab.upstreamTabId;
|
||||
after?.(s);
|
||||
return;
|
||||
}
|
||||
}
|
||||
// 上游 click/type 只认 ref/selector(1.17.0 实测不接受 {x, y}):坐标先经 evaluate 标记该点元素,再按选择器操作,
|
||||
// 这样仍走上游完整的鼠标事件序列与下载守卫,而不是页面内合成 click
|
||||
if (typeof payload.x === "number" && typeof payload.y === "number") {
|
||||
@@ -1068,6 +1099,73 @@ export class CamofoxBrowserProvider implements BrowserProvider {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 插件的真实鼠标点击:`POST /umt/tabs/{id}/mouse-click {userId, x, y}`(视口 CSS 像素)。插件没有这个端点(404)时记下,
|
||||
* 返回 null 让调用方退回 elementFromPoint。
|
||||
*/
|
||||
private async mouseClick(
|
||||
s: SessionState,
|
||||
tab: BrowserTab,
|
||||
x: number,
|
||||
y: number,
|
||||
): Promise<{ url: string | undefined } | null> {
|
||||
const path = `/umt/tabs/${encodeURIComponent(tab.upstreamTabId)}/mouse-click`;
|
||||
try {
|
||||
const res = await this.request("POST", path, {
|
||||
body: { userId: s.session.profileKey, x, y },
|
||||
timeoutMs: s.budget.timeoutMs,
|
||||
});
|
||||
const out = (await res.json().catch(() => ({}))) as Json;
|
||||
this.mouseApi = "plugin";
|
||||
return { url: str(out.url) };
|
||||
} catch (e) {
|
||||
// 404 = 插件没有这个端点(旧镜像);插件对"标签页不在"回 422,按标签页已不存在处理(BR-3 自愈)
|
||||
if (e instanceof BrowserError && e.httpStatus === 404) {
|
||||
this.mouseApi = "none";
|
||||
return null;
|
||||
}
|
||||
if (e instanceof BrowserError && e.httpStatus === 422)
|
||||
throw new BrowserError("not_found", "标签页不存在或已关闭", { httpStatus: 404 });
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 当前页面内容(CF-2):地址(已脱敏)、标题、outerHTML、正文样本、navigator.userAgent。只读,不要求自动化控制权;
|
||||
* 上游不支持 evaluate 时返回 null。HTML 只给服务端解析器用,不进模型与日志。
|
||||
*/
|
||||
async pageContent(ref: TabRef, opts: { maxHtmlBytes?: number } = {}): Promise<PageContent | null> {
|
||||
const s = this.state(ref.sessionId);
|
||||
const max = Math.max(1024, Math.min(opts.maxHtmlBytes ?? 8 * 1024 * 1024, 16 * 1024 * 1024));
|
||||
return this.withProfile(s, async () => {
|
||||
const tab = this.tabOf(s, ref);
|
||||
const path = `/tabs/${encodeURIComponent(tab.upstreamTabId)}/evaluate`;
|
||||
let out: Json;
|
||||
try {
|
||||
const res = await this.request("POST", path, {
|
||||
body: { userId: s.session.profileKey, expression: pageContentExpr(max) },
|
||||
timeoutMs: s.budget.timeoutMs,
|
||||
});
|
||||
out = await this.json(res, path);
|
||||
} catch (e) {
|
||||
if (e instanceof BrowserError && e.kind === "not_found") throw e;
|
||||
return null;
|
||||
}
|
||||
const r = (out.result && typeof out.result === "object" ? out.result : null) as Json | null;
|
||||
if (!r || typeof r.html !== "string") return null;
|
||||
const rawUrl = str(r.url) ?? s.rawUrl;
|
||||
this.touch(s, rawUrl ?? s.rawUrl);
|
||||
tab.url = s.session.currentUrl;
|
||||
return {
|
||||
url: rawUrl ? redactBrowserUrl(rawUrl) : null,
|
||||
title: str(r.title) ?? "",
|
||||
html: r.html,
|
||||
text: str(r.text) ?? "",
|
||||
userAgent: str(r.ua) ?? null,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/** 视口坐标 → 唯一属性选择器(POST /tabs/{id}/evaluate + elementFromPoint);坐标处无元素报 bad_request */
|
||||
private async pointSelector(s: SessionState, tab: BrowserTab, x: number, y: number): Promise<string> {
|
||||
const nonce = this.newId()
|
||||
|
||||
@@ -68,6 +68,11 @@ export interface BrowserProvider {
|
||||
* 只读,不要求自动化控制权;上游不支持 evaluate 时返回 null。
|
||||
*/
|
||||
pageMetrics(ref: TabRef): Promise<PageMetrics | null>;
|
||||
/**
|
||||
* 当前页面内容(CF-2 挑战识别与浏览器代抓):地址(已脱敏)、标题、outerHTML、正文样本、navigator.userAgent。
|
||||
* 只读,不要求自动化控制权;上游不支持时返回 null。HTML 只给服务端解析器,不进模型与日志。
|
||||
*/
|
||||
pageContent?(ref: TabRef, opts?: { maxHtmlBytes?: number }): Promise<PageContent | null>;
|
||||
|
||||
// ── BR-2 凭据统一 ──
|
||||
/**
|
||||
@@ -108,7 +113,7 @@ export interface BrowserProvider {
|
||||
* 该 profile 的出口:proxy = 站点开代理时与 HTTP 同一代理,null = 直连。上游没有接口时报 BrowserError("not_implemented")。
|
||||
* restarted = 活动上下文出口不同、已被关闭(下次操作按新出口重建)。
|
||||
*/
|
||||
setEgress?(profileKey: string, proxy: BrowserEgressProxy | null): Promise<{ restarted: boolean }>;
|
||||
setEgress?(profileKey: string, proxy: BrowserEgressProxy | null): Promise<{ restarted: boolean; mode?: string }>;
|
||||
}
|
||||
|
||||
export interface ReopenSessionRequest {
|
||||
@@ -124,6 +129,15 @@ export interface ReopenSessionResult {
|
||||
cookies?: { status: "injected" | "unsupported" | "failed"; count: number };
|
||||
}
|
||||
|
||||
export interface PageContent {
|
||||
url: string | null;
|
||||
title: string;
|
||||
html: string;
|
||||
/** document.body.innerText 前 2 万字 */
|
||||
text: string;
|
||||
userAgent: string | null;
|
||||
}
|
||||
|
||||
export interface PageMetrics {
|
||||
viewportWidth: number;
|
||||
viewportHeight: number;
|
||||
|
||||
@@ -163,12 +163,67 @@ describe("camofox 适配器:身份与出口", () => {
|
||||
mock = await createMockCamofox({ accessKey: "k" });
|
||||
const p = provider(mock);
|
||||
expect(await p.identity!()).toMatchObject({ userAgent: MOCK_CAMOFOX_IDENTITY.userAgent, os: "windows" });
|
||||
expect(await p.setEgress!("site-a", { server: "http://127.0.0.1:7891" })).toEqual({ restarted: false });
|
||||
expect(await p.setEgress!("site-b", null)).toEqual({ restarted: false });
|
||||
expect(await p.setEgress!("site-a", { server: "http://127.0.0.1:7891" })).toEqual({
|
||||
restarted: false,
|
||||
mode: "site",
|
||||
});
|
||||
expect(await p.setEgress!("site-b", null)).toEqual({ restarted: false, mode: "site" });
|
||||
expect(mock.egress.get("site-a")).toEqual({ server: "http://127.0.0.1:7891" });
|
||||
expect(mock.egress.get("site-b")).toBeNull();
|
||||
});
|
||||
|
||||
const budget = { domainAllowlist: ["audiences.example"], timeoutMs: 5_000, maxRetries: 0, maxBytes: 1_000_000 };
|
||||
const openCf = async (m: MockCamofox) => {
|
||||
const p = provider(m);
|
||||
const { session } = await p.openSession({
|
||||
providerId: "11111111-1111-4111-8111-111111111111",
|
||||
siteId: "s",
|
||||
accountId: "a",
|
||||
profileKey: "umt-site-s-acct-a",
|
||||
initialUrl: "https://audiences.example/torrents.php",
|
||||
budget,
|
||||
});
|
||||
return { p, ref: { sessionId: session.id } };
|
||||
};
|
||||
const cfSite = (mode: "auto" | "interactive" | "captcha") => ({
|
||||
origin: "https://audiences.example",
|
||||
username: "u",
|
||||
password: "p",
|
||||
protectedPaths: /^$/,
|
||||
cloudflare: { mode, autoPolls: 2, clearance: "clr-1" },
|
||||
});
|
||||
|
||||
it("CF-2 pageContent():地址 / 标题 / HTML / UA;挑战页分类由调用方做", async () => {
|
||||
mock = await createMockCamofox({ accessKey: "k", site: cfSite("auto") });
|
||||
const { p, ref } = await openCf(mock);
|
||||
const c1 = await p.pageContent!(ref);
|
||||
expect(c1).toMatchObject({ title: "Just a moment...", userAgent: MOCK_CAMOFOX_IDENTITY.userAgent });
|
||||
expect(c1?.html).toContain("_cf_chl_opt");
|
||||
// 第 2 次读页面时 mock 自动放行,并写 cf_clearance
|
||||
const c2 = await p.pageContent!(ref);
|
||||
expect(c2?.title).not.toBe("Just a moment...");
|
||||
const cookies = await p.getCookies!("umt-site-s-acct-a", ["https://audiences.example/"]);
|
||||
expect(cookies.find((c) => c.name === "cf_clearance")?.value).toBe("clr-1");
|
||||
});
|
||||
|
||||
it("CF-2 坐标点击走插件真实鼠标事件(跨域 iframe 的 Turnstile 也点得到),不再 elementFromPoint", async () => {
|
||||
mock = await createMockCamofox({ accessKey: "k", site: cfSite("interactive") });
|
||||
const { p, ref } = await openCf(mock);
|
||||
// 第 2 行(y = 20 + 40 + 10)是复选框
|
||||
await p.click(ref, { x: 100, y: 70 });
|
||||
expect(mock.actions.some((a) => a.action === "mouse-click" && a.body.x === 100 && a.body.y === 70)).toBe(true);
|
||||
expect(mock.actions.some((a) => a.action === "evaluate")).toBe(false);
|
||||
expect(mock.cfPassed.has("umt-site-s-acct-a")).toBe(true);
|
||||
});
|
||||
|
||||
it("CF-2 插件没有 mouse-click(旧镜像)→ 退回 elementFromPoint + 选择器点击", async () => {
|
||||
mock = await createMockCamofox({ accessKey: "k", cookieApi: "none", site: cfSite("interactive") });
|
||||
const { p, ref } = await openCf(mock);
|
||||
await p.click(ref, { x: 100, y: 70 });
|
||||
expect(mock.actions.some((a) => a.action === "evaluate")).toBe(true);
|
||||
expect(mock.actions.some((a) => a.action === "click")).toBe(true);
|
||||
});
|
||||
|
||||
it("旧插件(没有接口):identity() 为 null,setEgress() 报 not_implemented", async () => {
|
||||
mock = await createMockCamofox({ accessKey: "k", cookieApi: "none" });
|
||||
const p = provider(mock);
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { challengeRuleOfHtml, classifyChallengePage, detectChallenge } from "./challenge.ts";
|
||||
import { siteFetch } from "./http.ts";
|
||||
import { createSiteRateLimiter } from "./rate-limit.ts";
|
||||
import { SiteError } from "./types.ts";
|
||||
|
||||
// CF-2:Cloudflare 挑战识别。HTTP 端挑战不是凭据错误(403 不再算 auth_invalid、503 不再算维护);
|
||||
// 浏览器端按页面分:自动 / 可点(Turnstile)/ 真验证码。
|
||||
|
||||
const CF_PAGE = `<!DOCTYPE html><html><head><title>Just a moment...</title></head><body>
|
||||
<div id="challenge-spinner"></div><script>window._cf_chl_opt={cType: 'managed'}</script></body></html>`;
|
||||
const CF_JS_PAGE = `<html><head><title>Just a moment...</title></head><body>
|
||||
<script>window._cf_chl_opt={cType: 'non-interactive'}</script></body></html>`;
|
||||
const NORMAL = `<html><head><title>种子 - 观众</title><script src="/cdn-cgi/challenge-platform/scripts/jsd/main.js"></script></head><body>ok</body></html>`;
|
||||
|
||||
function ctx(fetchImpl: typeof fetch) {
|
||||
return {
|
||||
siteId: "s1",
|
||||
fetch: fetchImpl,
|
||||
limiter: createSiteRateLimiter({ minIntervalMs: 0, burst: 10 }),
|
||||
timeoutMs: 5_000,
|
||||
now: () => new Date(),
|
||||
};
|
||||
}
|
||||
const reply = (status: number, body: string, headers: Record<string, string> = {}) =>
|
||||
(async () => new Response(body, { status, headers: { "content-type": "text/html", ...headers } })) as typeof fetch;
|
||||
|
||||
describe("challengeRuleOfHtml", () => {
|
||||
it("挑战页标题 / 脚本特征命中;正常页面里注入的 challenge-platform 脚本不算", () => {
|
||||
expect(challengeRuleOfHtml(CF_PAGE)).toBe("title=Just a moment...");
|
||||
expect(challengeRuleOfHtml('<html><body><form id="challenge-form"></form></body></html>')).toContain(
|
||||
"challenge-form",
|
||||
);
|
||||
expect(challengeRuleOfHtml(NORMAL)).toBeNull();
|
||||
expect(challengeRuleOfHtml("")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("detectChallenge", () => {
|
||||
it("cf-mitigated: challenge 不读正文;403 / 503 读正文判定并重新包装响应", async () => {
|
||||
const a = await detectChallenge(new Response("x", { status: 200, headers: { "cf-mitigated": "challenge" } }));
|
||||
expect(a.rule).toBe("header cf-mitigated=challenge");
|
||||
const b = await detectChallenge(new Response(CF_PAGE, { status: 403 }));
|
||||
expect(b.rule).toContain("Just a moment");
|
||||
expect(await b.res.text()).toBe(CF_PAGE);
|
||||
const c = await detectChallenge(new Response("Forbidden", { status: 403 }));
|
||||
expect(c.rule).toBeNull();
|
||||
expect(await c.res.text()).toBe("Forbidden");
|
||||
const d = await detectChallenge(new Response(NORMAL, { status: 200 }));
|
||||
expect(d.rule).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("siteFetch 挑战分类", () => {
|
||||
const url = "https://audiences.example/torrents.php";
|
||||
const kind = async (f: typeof fetch) => {
|
||||
try {
|
||||
await siteFetch(ctx(f), url);
|
||||
return "ok";
|
||||
} catch (e) {
|
||||
return e instanceof SiteError ? e.kind : "other";
|
||||
}
|
||||
};
|
||||
|
||||
it("403 + cf-mitigated → challenge(不是 auth_invalid),不算凭据错误", async () => {
|
||||
let err: unknown;
|
||||
try {
|
||||
await siteFetch(ctx(reply(403, CF_PAGE, { "cf-mitigated": "challenge" })), url);
|
||||
} catch (e) {
|
||||
err = e;
|
||||
}
|
||||
expect(err).toBeInstanceOf(SiteError);
|
||||
const se = err as SiteError;
|
||||
expect(se.kind).toBe("challenge");
|
||||
expect(se.isAuthError).toBe(false);
|
||||
expect(se.detail).toMatchObject({ reason: "challenge", rule: "header cf-mitigated=challenge", httpStatus: 403 });
|
||||
});
|
||||
|
||||
it("503 挑战页 → challenge(不是维护);403 挑战页(没有响应头)也认", async () => {
|
||||
expect(await kind(reply(503, CF_PAGE))).toBe("challenge");
|
||||
expect(await kind(reply(403, CF_PAGE))).toBe("challenge");
|
||||
expect(await kind(reply(429, CF_JS_PAGE))).toBe("challenge");
|
||||
});
|
||||
|
||||
it("普通 503 仍是维护,普通 403 交给适配器(响应体可读)", async () => {
|
||||
expect(await kind(reply(503, "maintenance"))).toBe("site_maintenance");
|
||||
const res = await siteFetch(ctx(reply(403, "Forbidden")), url);
|
||||
expect(res.status).toBe(403);
|
||||
expect(await res.text()).toBe("Forbidden");
|
||||
});
|
||||
});
|
||||
|
||||
describe("classifyChallengePage", () => {
|
||||
it("不是挑战页 → none", () => {
|
||||
expect(classifyChallengePage({ title: "种子", html: NORMAL, text: "种子列表" }).kind).toBe("none");
|
||||
});
|
||||
it("JS 检查 → waiting;托管 / Turnstile → interactive", () => {
|
||||
expect(classifyChallengePage({ html: CF_JS_PAGE, text: "Checking your browser" }).kind).toBe("waiting");
|
||||
expect(classifyChallengePage({ html: CF_PAGE, text: "" }).kind).toBe("interactive");
|
||||
expect(
|
||||
classifyChallengePage({
|
||||
title: "请稍候…",
|
||||
html: "<html><body></body></html>",
|
||||
text: "请完成以下操作，验证您是真人。",
|
||||
}).kind,
|
||||
).toBe("interactive");
|
||||
});
|
||||
it("选图 / 输码 → captcha(不自动处理)", () => {
|
||||
expect(classifyChallengePage({ html: `${CF_PAGE}<div class="h-captcha"></div>`, text: "Just a moment" }).kind).toBe(
|
||||
"captcha",
|
||||
);
|
||||
expect(classifyChallengePage({ title: "Just a moment...", html: "", text: "请点击图中所有的红绿灯" }).kind).toBe(
|
||||
"captcha",
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,83 @@
|
||||
// CF-2 Cloudflare 挑战识别(docs/design/cloudflare-handling.md)。
|
||||
// - HTTP 端:响应头 `cf-mitigated: challenge`(任何状态码),或 403 / 503 / 429 且正文是挑战页 → 挑战,不是凭据错误;
|
||||
// - 浏览器端:挑战页分三类——自动(JS 检查,等几秒自己过)、可点(「我是人类」复选框 / Turnstile)、真验证码(选图 / 输码)。
|
||||
// 只用正则,不解析 DOM:HTTP 端要在限流器之后快速判断,浏览器端拿到的是 outerHTML + innerText 样本。
|
||||
|
||||
/** Cloudflare / DDoS-Guard 挑战页的标题 */
|
||||
const CHALLENGE_TITLE = /^\s*(?:just a moment|attention required! \| cloudflare|请稍候|ddos-guard|one moment, please)/i;
|
||||
/** 挑战页的脚本 / 元素特征。不认 `/cdn-cgi/challenge-platform/`:Cloudflare 会往正常页面也注入这段脚本 */
|
||||
const CHALLENGE_HTML =
|
||||
/cf-browser-verification|cf_chl_opt|cf-challenge-running|id="challenge-form"|id="challenge-spinner"|id="cf-please-wait"|id="trk_jschal_js"/i;
|
||||
|
||||
function titleOf(html: string): string {
|
||||
return /<title[^>]*>([^<]*)<\/title>/i.exec(html.slice(0, 50_000))?.[1]?.trim() ?? "";
|
||||
}
|
||||
|
||||
/** 页面是不是挑战页;命中返回规则说明(进技术信息),否则 null */
|
||||
export function challengeRuleOfHtml(html: string | null | undefined): string | null {
|
||||
if (!html) return null;
|
||||
const title = titleOf(html);
|
||||
if (CHALLENGE_TITLE.test(title)) return `title=${title.slice(0, 40)}`;
|
||||
const m = CHALLENGE_HTML.exec(html.slice(0, 200_000));
|
||||
return m ? `html*=${m[0]}` : null;
|
||||
}
|
||||
|
||||
/** 响应头判定(不读正文):`cf-mitigated: challenge` */
|
||||
export function challengeRuleOfHeaders(headers: Headers): string | null {
|
||||
return headers.get("cf-mitigated")?.trim().toLowerCase() === "challenge" ? "header cf-mitigated=challenge" : null;
|
||||
}
|
||||
|
||||
/** 这些状态码的响应才去看正文(Cloudflare 挑战页是 403 / 503,少数 429) */
|
||||
export const CHALLENGE_STATUSES: readonly number[] = [403, 429, 503];
|
||||
|
||||
/**
|
||||
* 判定一个站点响应是不是挑战。正文只在状态码可能是挑战时读(读过的正文重新包成 Response 返回,调用方照常消费)。
|
||||
* 返回 { res, rule }:rule 为 null 表示不是挑战。
|
||||
*/
|
||||
export async function detectChallenge(res: Response): Promise<{ res: Response; rule: string | null }> {
|
||||
const byHeader = challengeRuleOfHeaders(res.headers);
|
||||
if (byHeader) return { res, rule: byHeader };
|
||||
if (!CHALLENGE_STATUSES.includes(res.status) || !res.body) return { res, rule: null };
|
||||
const bytes = new Uint8Array(await res.arrayBuffer());
|
||||
const html = new TextDecoder("utf-8").decode(bytes.subarray(0, 300_000));
|
||||
const copy = new Response(bytes, { status: res.status, statusText: res.statusText, headers: res.headers });
|
||||
return { res: copy, rule: challengeRuleOfHtml(html) };
|
||||
}
|
||||
|
||||
/**
|
||||
* 浏览器里打开的页面属于哪种挑战:
|
||||
* - none:不是挑战页(已通过);
|
||||
* - waiting:自动检查(JS / 非交互),等几秒会自己过;
|
||||
* - interactive:「我是人类」复选框 / Turnstile,可以点一下;
|
||||
* - captcha:选图、输验证码这类真验证码,不自动处理。
|
||||
*/
|
||||
export type ChallengePageKind = "none" | "waiting" | "interactive" | "captcha";
|
||||
|
||||
const CAPTCHA_MARKERS =
|
||||
/hcaptcha\.com|class="h-captcha"|g-recaptcha|recaptcha\/api|imagecode\.php|select all (?:images|squares)|请(?:依次)?点击|选择所有|拖动滑块|输入验证码|enter the (?:characters|code)/i;
|
||||
const INTERACTIVE_MARKERS =
|
||||
/challenges\.cloudflare\.com\/(?:turnstile|cdn-cgi)|cf-turnstile|cType\s*:\s*['"](?:managed|interactive)['"]|verify you are human|确认您是真人|验证您是真人|我是人类/i;
|
||||
|
||||
export function classifyChallengePage(page: { title?: string | null; html?: string | null; text?: string | null }): {
|
||||
kind: ChallengePageKind;
|
||||
rule: string | null;
|
||||
} {
|
||||
const html = page.html ?? "";
|
||||
const text = page.text ?? "";
|
||||
const title = page.title ?? titleOf(html);
|
||||
const rule =
|
||||
(CHALLENGE_TITLE.test(title) ? `title=${title.slice(0, 40)}` : null) ??
|
||||
challengeRuleOfHtml(html) ??
|
||||
(/verify you are human|确认您是真人|验证您是真人|checking (?:if the site connection is secure|your browser)/i.test(
|
||||
text,
|
||||
)
|
||||
? "text"
|
||||
: null);
|
||||
if (!rule) return { kind: "none", rule: null };
|
||||
const all = `${html.slice(0, 300_000)}\n${text.slice(0, 20_000)}`;
|
||||
const cap = CAPTCHA_MARKERS.exec(all);
|
||||
if (cap) return { kind: "captcha", rule: `${rule}; captcha*=${cap[0]}` };
|
||||
const inter = INTERACTIVE_MARKERS.exec(all);
|
||||
if (inter) return { kind: "interactive", rule: `${rule}; interactive*=${inter[0]}` };
|
||||
return { kind: "waiting", rule };
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
import { redactText } from "../rss/redact.ts";
|
||||
import { detectChallenge } from "./challenge.ts";
|
||||
import type { SiteRateLimiter } from "./rate-limit.ts";
|
||||
import { SiteError } from "./types.ts";
|
||||
|
||||
@@ -20,7 +21,8 @@ function retryAfterMs(res: Response, now: Date): number | undefined {
|
||||
}
|
||||
|
||||
/**
|
||||
* 站点请求:先取站点令牌,再发请求;429/503 统一映射并顺延限流器。
|
||||
* 站点请求:先取站点令牌,再发请求;Cloudflare 挑战(`cf-mitigated: challenge`,或 403 / 503 / 429 的挑战页)→ `challenge`
|
||||
* (不是凭据错误,CF-2);其余 429/503 统一映射并顺延限流器。
|
||||
* 返回的 Response 由调用方消费。错误消息只含脱敏后的 URL。
|
||||
*/
|
||||
export async function siteFetch(
|
||||
@@ -42,6 +44,15 @@ export async function siteFetch(
|
||||
const reason = timeout.aborted ? "超时" : (e as Error).message;
|
||||
throw new SiteError(ctx.siteId, "network", redactText(`请求 ${safeUrl} 失败:${reason}`));
|
||||
}
|
||||
const ch = await detectChallenge(res);
|
||||
res = ch.res;
|
||||
if (ch.rule) {
|
||||
await res.body?.cancel().catch(() => {});
|
||||
throw new SiteError(ctx.siteId, "challenge", `请求 ${safeUrl}:站点要求人机验证(Cloudflare)`, {
|
||||
httpStatus: res.status,
|
||||
detail: { reason: "challenge", rule: ch.rule, httpStatus: res.status },
|
||||
});
|
||||
}
|
||||
if (res.status === 429 || res.status === 503) {
|
||||
const ra = retryAfterMs(res, ctx.now());
|
||||
await res.body?.cancel().catch(() => {});
|
||||
|
||||
@@ -1,3 +1,11 @@
|
||||
export {
|
||||
CHALLENGE_STATUSES,
|
||||
type ChallengePageKind,
|
||||
challengeRuleOfHeaders,
|
||||
challengeRuleOfHtml,
|
||||
classifyChallengePage,
|
||||
detectChallenge,
|
||||
} from "./challenge.ts";
|
||||
export {
|
||||
audiencesDefinition,
|
||||
BUILTIN_SITE_DEFINITIONS,
|
||||
|
||||
@@ -124,6 +124,41 @@ export const SiteLastHealth = z.object({
|
||||
});
|
||||
export type SiteLastHealth = z.infer<typeof SiteLastHealth>;
|
||||
|
||||
/**
|
||||
* 站点的 Cloudflare 处理状态(CF-2,存 site.config.cloudflare;docs/runbook/sites.md「Cloudflare」):
|
||||
* - browserUntil:在此之前该站能走浏览器的页面请求直接由浏览器代抓(HTTP 仍被挑战时置 6 小时,到期再试 HTTP);
|
||||
* - cfReuse:浏览器拿到的 cf_clearance 给 HTTP 用的结果(ok 有效 / fails 无效,之后不再复用);
|
||||
* - lastSolvedBy:最近一次怎么过的挑战(auto 等它自己过 / vision 视觉子流程点了一下 / human 人在原站视图里过);
|
||||
* - pending = human:遇到选图 / 输码类验证码,等人在原站视图里过一次;
|
||||
* - lastAttemptAt:最近一次尝试过挑战(同站 30 分钟最多一次)。
|
||||
*/
|
||||
export const SiteCloudflareState = z.object({
|
||||
browserUntil: Timestamp.nullable().default(null),
|
||||
cfReuse: z.enum(["ok", "fails"]).nullable().default(null),
|
||||
lastChallengeAt: Timestamp.nullable().default(null),
|
||||
lastSolvedBy: z.enum(["auto", "vision", "human"]).nullable().default(null),
|
||||
lastSolvedAt: Timestamp.nullable().default(null),
|
||||
lastAttemptAt: Timestamp.nullable().default(null),
|
||||
pending: z.enum(["human"]).nullable().default(null),
|
||||
/** 等人处理的浏览器会话(原站视图接管入口) */
|
||||
sessionId: Id.nullable().default(null),
|
||||
/** 最近一次处理结果(技术信息):passed / reuse_ok / reuse_fails / browser / needs_human / failed / throttled / skipped */
|
||||
lastResult: z.string().nullable().default(null),
|
||||
/** 命中的挑战规则(技术信息) */
|
||||
lastRule: z.string().nullable().default(null),
|
||||
/** 最近一次失败原因(技术信息) */
|
||||
lastError: z.string().nullable().default(null),
|
||||
});
|
||||
export type SiteCloudflareState = z.infer<typeof SiteCloudflareState>;
|
||||
|
||||
/** 从 site.config 读 Cloudflare 状态;没有或格式不对返回 null */
|
||||
export function siteCloudflareState(config: unknown): SiteCloudflareState | null {
|
||||
const raw = (config as { cloudflare?: unknown } | null | undefined)?.cloudflare;
|
||||
if (!raw || typeof raw !== "object") return null;
|
||||
const r = SiteCloudflareState.safeParse(raw);
|
||||
return r.success ? r.data : null;
|
||||
}
|
||||
|
||||
export const Site = EntityBase.extend({
|
||||
key: z.string().regex(/^[a-z][a-z0-9_-]*$/),
|
||||
name: z.string().min(1),
|
||||
|
||||
@@ -39,6 +39,15 @@ export interface MockCamofoxSite {
|
||||
* 表单登录成功后把 name=fresh 写进该 profile 的 Cookie(读回时可见)。
|
||||
*/
|
||||
sessionCookie?: { name: string; valid: string[]; fresh: string };
|
||||
/**
|
||||
* Cloudflare 挑战(CF-2):没通过的 profile 打开站点任何页面都是挑战页(标题 Just a moment...)。
|
||||
* auto = 读页面内容(umt-page)autoPolls 次后自己通过;interactive = 第 1 行是「确认您是真人」复选框,
|
||||
* 用插件真实鼠标点击(`POST /umt/tabs/{id}/mouse-click`)点中才通过;captcha = 选图验证码,自动化过不去。
|
||||
* 通过后该 profile 得到 cf_clearance(值 clearance)。
|
||||
*/
|
||||
cloudflare?: { mode: "auto" | "interactive" | "captcha"; autoPolls?: number; clearance?: string };
|
||||
/** 页面 HTML(umt-page 读页面内容时返回);缺省按快照行拼一段简单 HTML */
|
||||
html?: (url: string) => string;
|
||||
}
|
||||
|
||||
/** mock 的 Cookie 接口形态:plugin = umt-cookies 插件(读写);upstream = 只有上游导入端点(要求 CAMOFOX_API_KEY);none = 都没有 */
|
||||
@@ -128,6 +137,10 @@ export type MockCamofox = MockHttp & {
|
||||
readonly instanceId: string;
|
||||
/** 各 profile 的出口(`PUT /umt/sessions/:userId/proxy`,BR-4):null = 直连 */
|
||||
egress: Map<string, { server: string; username?: string; password?: string } | null>;
|
||||
/** 已通过 Cloudflare 挑战的 profile(CF-2) */
|
||||
cfPassed: Set<string>;
|
||||
/** 各 profile 读页面内容(umt-page)的次数 */
|
||||
contentReads: Map<string, number>;
|
||||
};
|
||||
|
||||
/** mock 的固定浏览器身份(`GET /umt/identity`,BR-4) */
|
||||
@@ -353,6 +366,23 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise<
|
||||
/** 重启前存在过的 tabId:之后访问返回 410 */
|
||||
const restartedTabs = new Set<string>();
|
||||
const site = opts.site;
|
||||
const cfPassed = new Set<string>();
|
||||
const contentReads = new Map<string, number>();
|
||||
const passCf = (userId: string) => {
|
||||
if (!site?.cloudflare || cfPassed.has(userId)) return;
|
||||
cfPassed.add(userId);
|
||||
putCookie(userId, {
|
||||
name: "cf_clearance",
|
||||
value: site.cloudflare.clearance ?? "cf-clearance-mock",
|
||||
domain: `.${new URL(site.origin).hostname}`,
|
||||
path: "/",
|
||||
expires: Math.floor(Date.now() / 1000) + 365 * 86400,
|
||||
httpOnly: true,
|
||||
secure: true,
|
||||
});
|
||||
};
|
||||
const underCf = (t: MockCamofoxTab) =>
|
||||
!!site?.cloudflare && !cfPassed.has(t.userId) && t.url.startsWith(new URL(site.origin).origin);
|
||||
const putCookie = (userId: string, c: MockCamofoxCookie) => {
|
||||
const jar = (cookieJars.get(userId) ?? []).filter(
|
||||
(x) => !(x.name === c.name && x.domain === c.domain && x.path === c.path),
|
||||
@@ -454,6 +484,14 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise<
|
||||
|
||||
const pageLines = (t: MockCamofoxTab): Line[] => {
|
||||
if (!site) return [];
|
||||
if (underCf(t)) {
|
||||
const mode = site.cloudflare!.mode;
|
||||
const lines: Line[] = [{ ref: "e1", role: "heading", text: "Just a moment..." }];
|
||||
if (mode === "interactive")
|
||||
lines.push({ ref: "e2", role: "checkbox", text: "确认您是真人", onClick: () => undefined });
|
||||
if (mode === "captcha") lines.push({ ref: "e2", role: "img", text: "请点击图中所有的红绿灯" });
|
||||
return lines;
|
||||
}
|
||||
const u = new URL(t.url);
|
||||
const passkey = site.passkey ?? CAMOFOX_FIXTURE_PASSKEY;
|
||||
const releases = site.releases ?? [{ id: "42", title: "Oppenheimer 2023 2160p UHD BluRay" }];
|
||||
@@ -749,6 +787,36 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise<
|
||||
},
|
||||
},
|
||||
};
|
||||
// 页面内容(CF-2):地址 / 标题 / HTML / 正文 / UA
|
||||
if (expr.includes("umt-page")) {
|
||||
const n = (contentReads.get(t.userId) ?? 0) + 1;
|
||||
contentReads.set(t.userId, n);
|
||||
if (underCf(t) && site?.cloudflare?.mode === "auto" && n >= (site.cloudflare.autoPolls ?? 2))
|
||||
passCf(t.userId);
|
||||
const lines = pageLines(t);
|
||||
const cf = underCf(t);
|
||||
const mode = site?.cloudflare?.mode;
|
||||
const title = cf ? "Just a moment..." : (lines[0]?.text ?? "");
|
||||
const html = cf
|
||||
? `<html><head><title>Just a moment...</title></head><body><div id="challenge-spinner"></div><script>window._cf_chl_opt={cType: '${mode === "auto" ? "non-interactive" : "managed"}'}</script>${
|
||||
mode === "captcha" ? '<div class="h-captcha"></div>' : ""
|
||||
}</body></html>`
|
||||
: site?.html
|
||||
? site.html(t.url)
|
||||
: `<html><head><title>${title}</title></head><body>${lines.map((l) => `<p>${l.text}</p>`).join("")}</body></html>`;
|
||||
return {
|
||||
json: {
|
||||
ok: true,
|
||||
result: {
|
||||
url: t.url,
|
||||
title,
|
||||
html,
|
||||
text: lines.map((l) => l.text).join("\n"),
|
||||
ua: MOCK_CAMOFOX_IDENTITY.userAgent,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
const pt = /elementFromPoint\((\d+), (\d+)\)/.exec(expr);
|
||||
const nonce = /setAttribute\("data-umt-pt", "([^"]+)"\)/.exec(expr)?.[1];
|
||||
if (!pt || !nonce) return { json: { ok: true, result: null } };
|
||||
@@ -811,6 +879,27 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise<
|
||||
return { json: { downloads: list } };
|
||||
},
|
||||
],
|
||||
[
|
||||
// 插件的真实鼠标点击(CF-2):坐标按行找元素并点击;挑战页的复选框点中即通过
|
||||
/^\/umt\/tabs\/([^/]+)\/mouse-click$/,
|
||||
(m, q, req, b) => {
|
||||
if (cookieApi !== "plugin" || req.method !== "POST") return err(404, "no route");
|
||||
const t = tabFor(m[1]!, b.userId as string | undefined);
|
||||
if (!t) return err(422, "tab not found");
|
||||
actions.push({ action: "mouse-click", tabId: t.tabId, body: { ...b } });
|
||||
void q;
|
||||
const y = Number(b.y);
|
||||
const line = pageLines(t)[Math.floor((y - MOCK_ROW_Y0) / MOCK_ROW_H)];
|
||||
if (line) {
|
||||
if (underCf(t) && site?.cloudflare?.mode === "interactive" && line.role === "checkbox") passCf(t.userId);
|
||||
else {
|
||||
if (line.role.startsWith("textbox")) t.focus = line.ref;
|
||||
line.onClick?.();
|
||||
}
|
||||
}
|
||||
return { json: { ok: true, tabId: t.tabId, url: t.url } };
|
||||
},
|
||||
],
|
||||
[/^\/umt\/instance$/, () => (cookieApi === "plugin" ? { json: { instanceId } } : err(404, "no route"))],
|
||||
[
|
||||
/^\/umt\/identity$/,
|
||||
@@ -891,6 +980,8 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise<
|
||||
loggedIn,
|
||||
cookieJars,
|
||||
egress,
|
||||
cfPassed,
|
||||
contentReads,
|
||||
expireLogin(userId?: string) {
|
||||
if (userId) loggedIn.delete(userId);
|
||||
else loggedIn.clear();
|
||||
@@ -905,6 +996,7 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise<
|
||||
if (o.loseProfiles) {
|
||||
loggedIn.clear();
|
||||
cookieJars.clear();
|
||||
cfPassed.clear();
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user