feat(sites): 出口统一与 Cloudflare 过挑战级联(CF-2)
- 出口:站点请求不再隐式继承 HTTP(S)_PROXY,没开代理明确直连(创建客户端时临时拿掉代理环境变量,Deno 2.9 语义见 egress.ts), 开了走站点代理;TMDB / Brave / 图片改为显式按环境变量(行为不变) - CloudflareGate:浏览器代理期直接代抓 → HTTP(可复用时带 cf_clearance)→ 挑战 → 浏览器过挑战(同站 30 分钟一次、单飞) → cf_clearance 仅在 UA 跟随浏览器且两端同出口时给 HTTP,重试 1 次记 cfReuse → 仍被挑战则浏览器代抓并进入 6 小时代理期 - 浏览器:solveSiteChallenge(自动放行 ≤15s / 系统发起视觉子流程最多 2 次 / 真验证码转人工待处理)、fetchSiteHtml(profile 锁 + 站点令牌)、 交还后读回 cf_clearance 继续;site.config.cloudflare 状态契约 Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
6b14b0da45
commit
16083942c0
14 files changed
+1711
-43
No files matched your search
@@ -0,0 +1,136 @@
|
||||
// CF-2 出口统一:每条对外 HTTP 请求走哪条出口由代码显式决定,不再隐式继承容器的 HTTP(S)_PROXY。
|
||||
//
|
||||
// Deno 2.9 的语义(2026-09-30 本机实测,见 docs/runbook/sites.md「出口」):
|
||||
// - `fetch` 缺省客户端与 `Deno.createHttpClient({})` 都会读 HTTP_PROXY / HTTPS_PROXY / ALL_PROXY(大小写都认)与 NO_PROXY;
|
||||
// 读取发生在**客户端创建时**(缺省客户端在进程第一次 fetch 时创建);
|
||||
// - `createHttpClient` 没有"不走代理"的选项,`NO_PROXY=*` 也不认;显式 `proxy` 会排在环境变量代理前面,
|
||||
// 但 NO_PROXY 命中的主机连显式代理也绕过;
|
||||
// - 所以直连客户端的做法是:创建那一刻把这几个环境变量临时拿掉(同步调用,同一个 isolate 内不会被别的代码插进来),建完立即还原。
|
||||
// 代理客户端同样在拿掉环境变量后创建,保证只走给定的代理。
|
||||
//
|
||||
// 出口表:
|
||||
// - 站点请求(适配器、RSS、登录、用户数据):站点开"使用代理"且设了代理地址 → 该代理;否则直连(siteClient);
|
||||
// - TMDB、Brave、图片缓存等外网请求:照旧用环境变量里的代理(行为不变),但改成显式读 env 建客户端(externalFetch)。
|
||||
|
||||
const PROXY_ENV = ["HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy"] as const;
|
||||
const NO_PROXY_ENV = ["NO_PROXY", "no_proxy"] as const;
|
||||
|
||||
/** 进程启动时的代理环境变量快照(外网请求显式使用;之后的临时改动不影响) */
|
||||
const ENV_SNAPSHOT: Record<string, string | undefined> = (() => {
|
||||
const out: Record<string, string | undefined> = {};
|
||||
try {
|
||||
for (const k of [...PROXY_ENV, ...NO_PROXY_ENV]) out[k] = Deno.env.get(k);
|
||||
} catch {
|
||||
// 没有 env 权限:当作没有代理
|
||||
}
|
||||
return out;
|
||||
})();
|
||||
|
||||
/** 在"没有代理环境变量"的状态下同步执行 fn(建 HttpClient 用),结束立即还原 */
|
||||
function withoutProxyEnv<T>(fn: () => T): T {
|
||||
const saved = new Map<string, string>();
|
||||
try {
|
||||
for (const k of [...PROXY_ENV, ...NO_PROXY_ENV]) {
|
||||
const v = Deno.env.get(k);
|
||||
if (v !== undefined) {
|
||||
saved.set(k, v);
|
||||
Deno.env.delete(k);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// 没有 env 权限时也没有可读的代理变量,直接建
|
||||
}
|
||||
try {
|
||||
return fn();
|
||||
} finally {
|
||||
for (const [k, v] of saved) Deno.env.set(k, v);
|
||||
}
|
||||
}
|
||||
|
||||
let direct: Deno.HttpClient | null = null;
|
||||
const proxied = new Map<string, Deno.HttpClient>();
|
||||
|
||||
/** 明确直连的客户端(不读任何代理环境变量) */
|
||||
export function directHttpClient(): Deno.HttpClient {
|
||||
direct ??= withoutProxyEnv(() => Deno.createHttpClient({}));
|
||||
return direct;
|
||||
}
|
||||
|
||||
/** 只走给定代理的客户端(同一地址复用) */
|
||||
export function proxyHttpClient(url: string): Deno.HttpClient {
|
||||
let c = proxied.get(url);
|
||||
if (!c) {
|
||||
c = withoutProxyEnv(() => Deno.createHttpClient({ proxy: { url } }));
|
||||
proxied.set(url, c);
|
||||
}
|
||||
return c;
|
||||
}
|
||||
|
||||
/** 用指定客户端发请求的 fetch */
|
||||
export function fetchVia(client: Deno.HttpClient): typeof globalThis.fetch {
|
||||
return ((input: RequestInfo | URL, init?: RequestInit) =>
|
||||
globalThis.fetch(input, { ...init, client } as RequestInit)) as typeof globalThis.fetch;
|
||||
}
|
||||
|
||||
/** 站点请求的出口:开代理且有代理地址 → proxy,否则 direct */
|
||||
export interface SiteEgress {
|
||||
mode: "direct" | "proxy";
|
||||
proxyUrl: string | null;
|
||||
}
|
||||
|
||||
export function siteEgress(proxy: boolean, proxyUrl: string | null | undefined): SiteEgress {
|
||||
return proxy && proxyUrl ? { mode: "proxy", proxyUrl } : { mode: "direct", proxyUrl: null };
|
||||
}
|
||||
|
||||
/** 出口的比较键(不含代理口令):direct / proxy:<主机:端口> */
|
||||
export function egressKey(e: SiteEgress): string {
|
||||
if (e.mode === "direct" || !e.proxyUrl) return "direct";
|
||||
try {
|
||||
const u = new URL(e.proxyUrl);
|
||||
return `proxy:${u.protocol}//${u.host}`;
|
||||
} catch {
|
||||
return "proxy:?";
|
||||
}
|
||||
}
|
||||
|
||||
export function siteFetchFor(e: SiteEgress): typeof globalThis.fetch {
|
||||
return fetchVia(e.mode === "proxy" && e.proxyUrl ? proxyHttpClient(e.proxyUrl) : directHttpClient());
|
||||
}
|
||||
|
||||
// ---------- 外网请求(TMDB / Brave / 图片):显式按环境变量 ----------
|
||||
|
||||
/** NO_PROXY 匹配(与常见实现一致:逗号分隔;`.a.com` / `a.com` 匹配自身与子域;`*` 匹配全部;IP 精确匹配) */
|
||||
export function noProxyMatches(host: string, noProxy: string | undefined): boolean {
|
||||
if (!noProxy) return false;
|
||||
const h = host.toLowerCase().replace(/^\[|\]$/g, "");
|
||||
for (const raw of noProxy.split(",")) {
|
||||
const p = raw.trim().toLowerCase().replace(/:\d+$/, "");
|
||||
if (!p) continue;
|
||||
if (p === "*") return true;
|
||||
const d = p.replace(/^\*?\./, "");
|
||||
if (h === d || h.endsWith(`.${d}`)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** 外网请求按环境变量应走的代理(进程启动时的快照);没有 / NO_PROXY 命中为 null */
|
||||
export function envProxyFor(target: URL, env: Record<string, string | undefined> = ENV_SNAPSHOT): string | null {
|
||||
const get = (k: string) => env[k] ?? env[k.toLowerCase()];
|
||||
if (noProxyMatches(target.hostname, get("NO_PROXY"))) return null;
|
||||
const byScheme = target.protocol === "https:" ? get("HTTPS_PROXY") : get("HTTP_PROXY");
|
||||
const p = byScheme || get("ALL_PROXY");
|
||||
return p ? p : null;
|
||||
}
|
||||
|
||||
/** 外网请求用的 fetch:按环境变量显式选代理 / 直连(与之前隐式继承的行为一致) */
|
||||
export const externalFetch: typeof globalThis.fetch = ((input: RequestInfo | URL, init?: RequestInit) => {
|
||||
let target: URL | null = null;
|
||||
try {
|
||||
target = new URL(input instanceof Request ? input.url : String(input));
|
||||
} catch {
|
||||
// 交给 fetch 报错
|
||||
}
|
||||
const proxy = target ? envProxyFor(target) : null;
|
||||
const client = proxy ? proxyHttpClient(proxy) : directHttpClient();
|
||||
return globalThis.fetch(input, { ...init, client } as RequestInit);
|
||||
}) as typeof globalThis.fetch;
|
||||
@@ -20,7 +20,7 @@ import {
|
||||
} from "@umt/contracts";
|
||||
import type { z } from "zod";
|
||||
import { browserServiceFor } from "../browser/index.ts";
|
||||
import { BrowserServiceError } from "../browser/service.ts";
|
||||
import { BrowserServiceError, type ChallengeVision, registerChallengeVision } from "../browser/service.ts";
|
||||
import { resolveRunUpstream } from "../pi/index.ts";
|
||||
import type { ServerContext } from "../types.ts";
|
||||
import { parseFakeScripts } from "./fake-scripts.ts";
|
||||
@@ -62,7 +62,41 @@ const FALLBACK_SCRIPT: FakeScript = {
|
||||
const isUuid = (s: string) => /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(s);
|
||||
const clip = (s: string, n: number) => (s.length > n ? `${s.slice(0, n)}…` : s);
|
||||
|
||||
/**
|
||||
* CF-2:系统发起的视觉子流程(站点 HTTP 遇 Cloudflare 复选框 / Turnstile 时由浏览器模块调用)。没有父会话:
|
||||
* 发起人记 system:cloudflare,子会话 trigger = event;预算取设置(智能体单次 token / 费用上限,时长 ≤ 120 秒)。
|
||||
* 自动化总开关由浏览器模块先查过;子流程里的坐标动作照常经注册中心授权(spawn_reason 放行)。
|
||||
*/
|
||||
export function systemChallengeVision(env: () => AgentEnv, registry: CapabilityRegistry): ChallengeVision {
|
||||
return async (req) => {
|
||||
const { ctx } = env();
|
||||
const s = await ctx.settings.all();
|
||||
const input: Input = {
|
||||
sessionId: req.sessionId,
|
||||
goal: req.goal,
|
||||
// 成功与否浏览器模块还会按页面再核实一次;这里给子流程一个可判定的条件
|
||||
success: { textAbsent: req.textAbsent ?? "Just a moment" },
|
||||
maxSteps: Math.min(12, Math.max(1, req.maxSteps)),
|
||||
budget: {
|
||||
maxDurationSec: Math.min(120, s.agent.maxDurationSec),
|
||||
...(s.agent.maxTokensPerRun ? { maxTokens: Math.max(1000, s.agent.maxTokensPerRun) } : {}),
|
||||
...(s.agent.maxCostUsdPerRun ? { maxCostUsd: s.agent.maxCostUsdPerRun } : {}),
|
||||
},
|
||||
};
|
||||
const hctx = {
|
||||
actorId: "system:cloudflare",
|
||||
taskId: `cloudflare:${req.sessionId}`,
|
||||
scopeIds: [],
|
||||
signal: AbortSignal.timeout(180_000),
|
||||
source: "task",
|
||||
} as HandlerContext;
|
||||
const out = await solveChallenge(env(), registry, input, hctx);
|
||||
return { status: out.status, reason: out.reason, childRunId: out.childRunId };
|
||||
};
|
||||
}
|
||||
|
||||
export function registerChallengeCapability(registry: CapabilityRegistry, env: () => AgentEnv): void {
|
||||
registerChallengeVision(env().ctx, systemChallengeVision(env, registry));
|
||||
registry.register(capabilitySpec("browser.solve_challenge"), async (input, ctx) => {
|
||||
try {
|
||||
return await solveChallenge(env(), registry, input, ctx);
|
||||
@@ -249,7 +283,7 @@ async function runChild(
|
||||
const child = await ctx.db
|
||||
.insertInto("agent_run")
|
||||
.values({
|
||||
trigger: parent ? "subagent" : "user",
|
||||
trigger: parent ? "subagent" : hctx.actorId.startsWith("system:") ? "event" : "user",
|
||||
goal,
|
||||
status: "running",
|
||||
scope: JSON.stringify({ capabilities: [...VISION_CAPABILITIES], allowedKinds: CHILD_KINDS }),
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
import { siteCloudflareState } from "@umt/contracts";
|
||||
import { MOCK_CAMOFOX_IDENTITY, type MockCamofoxSite } from "@umt/testkit";
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest";
|
||||
import { browserServiceFor } from "../index.ts";
|
||||
import { type BrowserService, registerChallengeVision } from "../service.ts";
|
||||
import { type BrowserHarness, createBrowserHarness, type Loose, ORIGIN, pgAvailable } from "./harness.ts";
|
||||
|
||||
// CF-2 浏览器一侧(对 testkit mock camofox 的 Cloudflare 模式):
|
||||
// ① 自动放行(轮询页面)② 复选框 / Turnstile 派视觉子流程,用插件真实鼠标点击 ③ 选图验证码 / 点不过 → 会话转等待 + 待处理事项;
|
||||
// 人在原站视图里过完交还 → 读回 cf_clearance / UA 通知站点请求继续;浏览器代抓页面受 profile 锁与站点限流约束。
|
||||
|
||||
const up = await pgAvailable();
|
||||
const GOOD = "cf2-session-value";
|
||||
const CLEARANCE = "clr-from-browser";
|
||||
const LIST_HTML =
|
||||
'<html><head><title>种子</title></head><body><table class="torrents"><tr><td>Oppenheimer</td></tr></table></body></html>';
|
||||
|
||||
describe.skipIf(!up)("CF-2 浏览器过挑战 / 代抓", () => {
|
||||
let h: BrowserHarness;
|
||||
let svc: BrowserService;
|
||||
const cfCfg: NonNullable<MockCamofoxSite["cloudflare"]> = { mode: "auto", autoPolls: 2, clearance: CLEARANCE };
|
||||
const visionCalls: string[] = [];
|
||||
let visionClick = true;
|
||||
const url = `${ORIGIN}/torrents.php`;
|
||||
|
||||
const sessions = async () => (await h.req("/browser/sessions")).body.data.items as Loose[];
|
||||
const closeAll = async () => {
|
||||
for (const s of await sessions()) await h.req(`/browser/sessions/${s.id}`, { method: "DELETE" });
|
||||
};
|
||||
const profileKey = () => `umt-site-${h.siteId}-acct-${h.accountId}`;
|
||||
|
||||
beforeAll(async () => {
|
||||
h = await createBrowserHarness("cf2", {
|
||||
sessionCookie: { name: "c_secure_pass", valid: [GOOD], fresh: GOOD },
|
||||
cloudflare: cfCfg,
|
||||
html: (u) =>
|
||||
u.includes("torrents.php") ? LIST_HTML : "<html><head><title>首页</title></head><body></body></html>",
|
||||
browserOptions: { challengePollMs: 5, challengeAutoWaitMs: 200, settleMs: 0 },
|
||||
});
|
||||
await sessions(); // 第一次请求时浏览器模块才为这个上下文建服务
|
||||
svc = browserServiceFor(h.ctx)!;
|
||||
const ref = await h.secrets.create({ purpose: "site_cookie", plaintext: `c_secure_uid=1; c_secure_pass=${GOOD}` });
|
||||
await h.db
|
||||
.updateTable("site_account")
|
||||
.set({ auth_kind: "cookie", secret_ref: ref, session_secret_ref: null, status: "active" })
|
||||
.where("id", "=", h.accountId)
|
||||
.execute();
|
||||
// 视觉子流程(假):点一下复选框所在的位置(第 2 行)
|
||||
registerChallengeVision(h.ctx, async (req) => {
|
||||
visionCalls.push(req.sessionId);
|
||||
if (visionClick)
|
||||
await svc.visionAct(
|
||||
req.sessionId,
|
||||
{ kind: "automation", holder: "system:cloudflare" },
|
||||
(p, ref) => p.click(ref, { x: 100, y: 70 }),
|
||||
{ wantImage: false, sensitiveAllowed: false },
|
||||
);
|
||||
return { status: visionClick ? "solved" : "needs_human", reason: "", childRunId: null };
|
||||
});
|
||||
}, 120_000);
|
||||
afterAll(async () => {
|
||||
await h?.close();
|
||||
});
|
||||
beforeEach(async () => {
|
||||
await closeAll();
|
||||
h.mock.cfPassed.clear();
|
||||
h.mock.contentReads.clear();
|
||||
h.mock.actions.length = 0;
|
||||
for (const jar of h.mock.cookieJars.values())
|
||||
jar.splice(0, jar.length, ...jar.filter((c) => c.name !== "cf_clearance"));
|
||||
cfCfg.mode = "auto";
|
||||
visionCalls.length = 0;
|
||||
visionClick = true;
|
||||
await h.db
|
||||
.updateTable("site")
|
||||
.set({ config: JSON.stringify({}), enabled: true })
|
||||
.where("id", "=", h.siteId)
|
||||
.execute();
|
||||
await h.db
|
||||
.updateTable("settings")
|
||||
.set({ value: JSON.stringify({ enabled: true }) })
|
||||
.where("key", "=", "automation")
|
||||
.execute();
|
||||
});
|
||||
|
||||
it("① 自动放行:轮询到挑战页消失,读回 cf_clearance + navigator.userAgent + 出口", async () => {
|
||||
const r = await svc.solveSiteChallenge({ siteId: h.siteId, accountId: h.accountId, url });
|
||||
expect(r).toEqual({
|
||||
status: "passed",
|
||||
by: "auto",
|
||||
clearance: CLEARANCE,
|
||||
userAgent: MOCK_CAMOFOX_IDENTITY.userAgent,
|
||||
browserEgress: "direct",
|
||||
});
|
||||
expect(visionCalls).toEqual([]);
|
||||
// 会话仍归自动化,没有待处理事项
|
||||
const [s] = await sessions();
|
||||
expect(s.controller).toBe("automation");
|
||||
});
|
||||
|
||||
it("② 复选框 / Turnstile:等不过 → 视觉子流程用插件真实鼠标点击 → 通过(by vision)", async () => {
|
||||
cfCfg.mode = "interactive";
|
||||
const r = await svc.solveSiteChallenge({ siteId: h.siteId, accountId: h.accountId, url });
|
||||
expect(r).toMatchObject({ status: "passed", by: "vision", clearance: CLEARANCE });
|
||||
expect(visionCalls).toHaveLength(1);
|
||||
expect(h.mock.actions.some((a) => a.action === "mouse-click")).toBe(true);
|
||||
});
|
||||
|
||||
it("③ 视觉子流程点不过(最多 2 次)→ 会话转等待,待处理事项「观众需要你在原站视图里过一次验证」", async () => {
|
||||
cfCfg.mode = "interactive";
|
||||
visionClick = false;
|
||||
const r = await svc.solveSiteChallenge({ siteId: h.siteId, accountId: h.accountId, url });
|
||||
expect(r).toMatchObject({ status: "needs_human" });
|
||||
expect(visionCalls).toHaveLength(2);
|
||||
const s = (await h.req(`/browser/sessions/${(r as Loose).sessionId}`)).body.data as Loose;
|
||||
expect(s.controller).toBe("waiting");
|
||||
expect(s.waitingReason).toBe("cf_challenge");
|
||||
const d = await h.db
|
||||
.selectFrom("decision")
|
||||
.selectAll()
|
||||
.where("id", "=", (r as Loose).decisionId)
|
||||
.executeTakeFirstOrThrow();
|
||||
expect(d.title).toBe("观众需要你在原站视图里过一次验证");
|
||||
expect(d.status).toBe("awaiting_user");
|
||||
expect((d.evidence as Loose).cloudflare).toMatchObject({ siteId: h.siteId });
|
||||
// 已在等人:再来一次不打扰,直接返回同一个等待
|
||||
const again = await svc.solveSiteChallenge({ siteId: h.siteId, accountId: h.accountId, url });
|
||||
expect(again).toMatchObject({ status: "needs_human", sessionId: (r as Loose).sessionId });
|
||||
expect(visionCalls).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("自动化总开关关闭:不派视觉子流程,直接转人工", async () => {
|
||||
cfCfg.mode = "interactive";
|
||||
await h.db
|
||||
.insertInto("settings")
|
||||
.values({ key: "automation", value: JSON.stringify({ enabled: false }) })
|
||||
.onConflict((oc) => oc.column("key").doUpdateSet({ value: JSON.stringify({ enabled: false }) }))
|
||||
.execute();
|
||||
const r = await svc.solveSiteChallenge({ siteId: h.siteId, accountId: h.accountId, url });
|
||||
expect(r).toMatchObject({ status: "needs_human" });
|
||||
expect(visionCalls).toEqual([]);
|
||||
});
|
||||
|
||||
it("③ 选图 / 输码类:不点、不等,直接转人工;人过完交还 → 读回 cf_clearance,站点请求继续", async () => {
|
||||
cfCfg.mode = "captcha";
|
||||
const r = (await svc.solveSiteChallenge({ siteId: h.siteId, accountId: h.accountId, url })) as Loose;
|
||||
expect(r.status).toBe("needs_human");
|
||||
expect(r.reason).toContain("选图");
|
||||
expect(visionCalls).toEqual([]);
|
||||
// 站点请求一侧记下"等人处理"(级联在 sources-identity 里做,这里直接写状态)
|
||||
await h.db
|
||||
.updateTable("site")
|
||||
.set({ config: JSON.stringify({ cloudflare: { pending: "human", sessionId: r.sessionId } }) })
|
||||
.where("id", "=", h.siteId)
|
||||
.execute();
|
||||
// 人接管,在桌面里过了验证(桌面操作不经 API:直接改 mock 状态),交还
|
||||
expect(
|
||||
(await h.req(`/browser/sessions/${r.sessionId}/control/acquire`, { body: { reason: "过验证" } })).status,
|
||||
).toBe(200);
|
||||
h.mock.cfPassed.add(profileKey());
|
||||
h.mock.cookieJars.get(profileKey())!.push({
|
||||
name: "cf_clearance",
|
||||
value: "clr-human",
|
||||
domain: ".audiences.example",
|
||||
path: "/",
|
||||
expires: Math.floor(Date.now() / 1000) + 3600,
|
||||
httpOnly: true,
|
||||
secure: true,
|
||||
});
|
||||
expect((await h.req(`/browser/sessions/${r.sessionId}/control/release`, { body: {} })).status).toBe(200);
|
||||
const site = await h.db.selectFrom("site").select("config").where("id", "=", h.siteId).executeTakeFirstOrThrow();
|
||||
expect(siteCloudflareState(site.config)).toMatchObject({ pending: null, lastSolvedBy: "human" });
|
||||
const acc = await h.db
|
||||
.selectFrom("site_account")
|
||||
.select("user_info")
|
||||
.where("id", "=", h.accountId)
|
||||
.executeTakeFirstOrThrow();
|
||||
const info = (acc.user_info as Loose).cloudflare;
|
||||
expect(info).toMatchObject({ userAgent: MOCK_CAMOFOX_IDENTITY.userAgent, egress: "direct" });
|
||||
expect(h.secrets.plain.get(info.secretRef)).toBe("clr-human");
|
||||
// 待处理事项已关闭
|
||||
const d = await h.db
|
||||
.selectFrom("decision")
|
||||
.select("status")
|
||||
.where("id", "=", r.decisionId)
|
||||
.executeTakeFirstOrThrow();
|
||||
expect(d.status).toBe("resolved");
|
||||
});
|
||||
|
||||
it("浏览器代抓:过了挑战的 profile 导航取 outerHTML;同一 profile 并发串行(只开一个会话);每次导航取站点令牌", async () => {
|
||||
h.mock.cfPassed.add(profileKey());
|
||||
const [a, b] = await Promise.all([
|
||||
svc.fetchSiteHtml({ siteId: h.siteId, accountId: h.accountId, url }),
|
||||
svc.fetchSiteHtml({ siteId: h.siteId, accountId: h.accountId, url }),
|
||||
]);
|
||||
expect(a).toMatchObject({ status: "ok", html: LIST_HTML });
|
||||
expect(b).toMatchObject({ status: "ok", html: LIST_HTML });
|
||||
expect(await sessions()).toHaveLength(1);
|
||||
const navs = h.mock.calls.filter(
|
||||
(c) => c.path.endsWith("/navigate") || (c.path === "/tabs" && c.method === "POST"),
|
||||
);
|
||||
expect(navs.length).toBeGreaterThanOrEqual(2);
|
||||
const count = await h.db
|
||||
.selectFrom("site_request_count")
|
||||
.select((eb) => eb.fn.sum<number>("count").as("n"))
|
||||
.where("site_id", "=", h.siteId)
|
||||
.executeTakeFirst();
|
||||
expect(Number(count?.n ?? 0)).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
|
||||
it("浏览器代抓:仍是挑战页 → challenge;会话在人工接管中 → 不打扰;站点停用 → 不可用", async () => {
|
||||
cfCfg.mode = "captcha";
|
||||
expect(await svc.fetchSiteHtml({ siteId: h.siteId, accountId: h.accountId, url })).toMatchObject({
|
||||
status: "challenge",
|
||||
});
|
||||
const [s] = await sessions();
|
||||
await h.req(`/browser/sessions/${s.id}/control/acquire`, { body: { reason: "看看" } });
|
||||
expect(await svc.fetchSiteHtml({ siteId: h.siteId, accountId: h.accountId, url })).toMatchObject({
|
||||
status: "unavailable",
|
||||
});
|
||||
await h.req(`/browser/sessions/${s.id}/control/release`, { body: {} });
|
||||
await h.db.updateTable("site").set({ enabled: false }).where("id", "=", h.siteId).execute();
|
||||
expect(await svc.fetchSiteHtml({ siteId: h.siteId, accountId: h.accountId, url })).toMatchObject({
|
||||
status: "unavailable",
|
||||
reason: "站点已停用",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -3,7 +3,13 @@ import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { CapabilityRegistry, InvokeContext } from "@umt/agent";
|
||||
import { type DB, type Kysely, migrateToLatest, sql } from "@umt/db";
|
||||
import { buildFakeTorrent, createMockCamofox, type FakeTorrent, type MockCamofox } from "@umt/testkit";
|
||||
import {
|
||||
buildFakeTorrent,
|
||||
createMockCamofox,
|
||||
type FakeTorrent,
|
||||
type MockCamofox,
|
||||
type MockCamofoxSite,
|
||||
} from "@umt/testkit";
|
||||
import { Kysely as K, PostgresDialect } from "kysely";
|
||||
import pg from "pg";
|
||||
import { withSchemaLock } from "../../../../test/auth-settings/harness.ts";
|
||||
@@ -19,6 +25,7 @@ import { createSourcesIdentityModule } from "../../sources-identity/index.ts";
|
||||
import type { ServerContext } from "../../types.ts";
|
||||
import { registerBrowserCapabilities } from "../capabilities.ts";
|
||||
import { createBrowserModule } from "../index.ts";
|
||||
import type { BrowserModuleOptions } from "../service.ts";
|
||||
|
||||
// C1-E 浏览器模块测试装配:独立 PG schema + 真实迁移 + 交互式 mock camofox(观众风格站点)+ 真实能力注册中心。
|
||||
|
||||
@@ -61,6 +68,11 @@ export async function createBrowserHarness(
|
||||
sessionCookie?: { name: string; valid: string[]; fresh: string };
|
||||
cookieApi?: "plugin" | "upstream" | "none";
|
||||
cookieImportKey?: string;
|
||||
/** CF-2:mock 站点的 Cloudflare 挑战与页面 HTML */
|
||||
cloudflare?: MockCamofoxSite["cloudflare"];
|
||||
html?: MockCamofoxSite["html"];
|
||||
/** 浏览器模块的额外选项(挑战轮询间隔等) */
|
||||
browserOptions?: Partial<BrowserModuleOptions>;
|
||||
} = {},
|
||||
): Promise<BrowserHarness> {
|
||||
const torrent = buildFakeTorrent({ name: "Oppenheimer.2023.2160p.UHD.BluRay.x265-GRP" });
|
||||
@@ -70,6 +82,8 @@ export async function createBrowserHarness(
|
||||
...(opts.cookieImportKey ? { cookieImportKey: opts.cookieImportKey } : {}),
|
||||
site: {
|
||||
...(opts.sessionCookie ? { sessionCookie: opts.sessionCookie } : {}),
|
||||
...(opts.cloudflare ? { cloudflare: opts.cloudflare } : {}),
|
||||
...(opts.html ? { html: opts.html } : {}),
|
||||
origin: ORIGIN,
|
||||
username: "fixture-user",
|
||||
password: PASSWORD,
|
||||
@@ -110,7 +124,12 @@ export async function createBrowserHarness(
|
||||
capabilities: registry,
|
||||
} as ServerContext;
|
||||
const sources = createSourcesIdentityModule({ tmdb: () => null });
|
||||
const browser = createBrowserModule({ runtime: (c) => sources.runtime(c), stagingDir, downloadPollMs: 20 });
|
||||
const browser = createBrowserModule({
|
||||
runtime: (c) => sources.runtime(c),
|
||||
stagingDir,
|
||||
downloadPollMs: 20,
|
||||
...opts.browserOptions,
|
||||
});
|
||||
registerBrowserCapabilities(registry, () => browser.service(ctx));
|
||||
const app = createApp({
|
||||
config: { NODE_ENV: "development", DEV_ADMIN_KEY: DEV_KEY },
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
type BrowserProvider,
|
||||
type BrowserSecretResolver,
|
||||
CAMOFOX_VERIFIED_RELEASES,
|
||||
classifyChallengePage,
|
||||
cookiesFromHeader,
|
||||
createCamofoxBrowserProvider,
|
||||
type DownloadSink,
|
||||
@@ -12,7 +13,9 @@ import {
|
||||
getSiteDefinition,
|
||||
headerFromCookies,
|
||||
hostAllowed,
|
||||
InMemoryProfileLock,
|
||||
imageDimensions,
|
||||
type PageContent,
|
||||
type PageMetrics,
|
||||
parseTorrent,
|
||||
prepareImage,
|
||||
@@ -50,8 +53,14 @@ import {
|
||||
type SiteRow,
|
||||
sql,
|
||||
} from "@umt/db";
|
||||
import { egressKey, siteEgress } from "../../egress.ts";
|
||||
import type { SourcesRuntime } from "../sources-identity/runtime.ts";
|
||||
import { refreshSiteAccount } from "../sources-identity/service/account.ts";
|
||||
import {
|
||||
type BrowserFetchOutcome,
|
||||
type ChallengeSolveOutcome,
|
||||
registerChallengeBrowser,
|
||||
} from "../sources-identity/service/cloudflare.ts";
|
||||
import {
|
||||
readAccountCookie,
|
||||
registerBrowserCookieReader,
|
||||
@@ -85,6 +94,10 @@ export interface BrowserModuleOptions {
|
||||
evidenceDir?: string;
|
||||
/** 坐标动作之后等页面稳定再截图的毫秒数;缺省 400 */
|
||||
settleMs?: number;
|
||||
/** CF-2 等挑战页放行时读页面的间隔(毫秒);缺省 1500 */
|
||||
challengePollMs?: number;
|
||||
/** CF-2 等 Cloudflare 自己放行的最长时间(毫秒);缺省 CF_AUTO_WAIT_MS(15 秒) */
|
||||
challengeAutoWaitMs?: number;
|
||||
/** 保活间隔(AG-4);缺省 BROWSER_KEEPALIVE_MS 配置(120 秒),0 = 关闭 */
|
||||
keepaliveMs?: number;
|
||||
/** 测试替换 provider 实现 */
|
||||
@@ -175,8 +188,39 @@ export interface RecoveryState {
|
||||
reconnect?: BrowserReconnectState;
|
||||
/** 最近一次自动化(Agent 能力)动作成功的时间;原站视图据此区分「Agent 正在操作」与「无人操作」 */
|
||||
lastAutomationAt?: string;
|
||||
/** CF-2:打开 / 重连时下发给浏览器的出口键(egressKey);插件没按站点设出口(旧镜像 / env 模式)为 null */
|
||||
egress?: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* CF-2 视觉子流程钩子(agent 模块注册):系统发起、没有父会话,预算取设置;只点「我是人类」复选框 / Turnstile 一步。
|
||||
* 返回子流程结论(solved 仍要由调用方按页面核实)。
|
||||
*/
|
||||
export type ChallengeVision = (req: {
|
||||
sessionId: string;
|
||||
goal: string;
|
||||
/** 通过后页面上不再出现的文字(成功条件) */
|
||||
textAbsent: string | null;
|
||||
maxSteps: number;
|
||||
}) => Promise<{ status: string; reason: string; childRunId: string | null }>;
|
||||
|
||||
const visions = new WeakMap<object, ChallengeVision>();
|
||||
|
||||
export function registerChallengeVision(ctx: object, fn: ChallengeVision): void {
|
||||
visions.set(ctx, fn);
|
||||
}
|
||||
|
||||
/** 系统(Cloudflare 处理)发起的浏览器动作 */
|
||||
const CF_ACTOR: Actor = { kind: "automation", holder: "system:cloudflare" };
|
||||
/** 等 Cloudflare 自己放行的最长时间 */
|
||||
export const CF_AUTO_WAIT_MS = 15_000;
|
||||
/** 视觉子流程点过之后再等多久 */
|
||||
const CF_AFTER_CLICK_WAIT_MS = 10_000;
|
||||
/** 视觉子流程最多派几次 */
|
||||
const CF_VISION_ATTEMPTS = 2;
|
||||
/** 视觉子流程步数上限:截图 → 点击 → 等待 */
|
||||
const CF_VISION_STEPS = 4;
|
||||
|
||||
/** 同一浏览器服务的实例标识多久查一次 */
|
||||
const INSTANCE_CHECK_MS = 5_000;
|
||||
/** 自动重连失败后,操作触发的重连至少间隔多久 */
|
||||
@@ -221,6 +265,11 @@ export class BrowserService {
|
||||
registerBrowserCookieReader(ctx, (siteId, accountId) => this.readProfileCookie(siteId, accountId));
|
||||
// 站点请求 UA 跟随浏览器固定身份(BR-4):sources-identity 经钩子取
|
||||
registerBrowserIdentityReader(ctx, () => this.identityUserAgent());
|
||||
// CF-2:站点 HTTP 遇 Cloudflare 挑战时由浏览器过挑战 / 代抓页面
|
||||
registerChallengeBrowser(ctx, {
|
||||
solve: (req) => this.solveSiteChallenge(req),
|
||||
fetchHtml: (req) => this.fetchSiteHtml(req),
|
||||
});
|
||||
}
|
||||
|
||||
// ---------- BR-4 浏览器身份与出口 ----------
|
||||
@@ -283,8 +332,8 @@ export class BrowserService {
|
||||
* 出口按站点统一(BR-4):站点开了代理且设置了代理地址 → 该 profile 走同一代理;否则直连。
|
||||
* 旧镜像(插件没有出口接口)只记日志,浏览器照旧走容器环境变量里的代理。
|
||||
*/
|
||||
private async applyEgress(provider: BrowserProvider, site: SiteRow, profileKey: string): Promise<void> {
|
||||
if (!provider.setEgress) return;
|
||||
private async applyEgress(provider: BrowserProvider, site: SiteRow, profileKey: string): Promise<string | null> {
|
||||
if (!provider.setEgress) return null;
|
||||
const rt = this.opts.runtime?.(this.ctx);
|
||||
const proxyUrl = rt
|
||||
? (await rt.siteDefaults()).proxyUrl
|
||||
@@ -297,6 +346,8 @@ export class BrowserService {
|
||||
{ component: "browser", siteId: site.id, egress: proxy ? "proxy" : "direct" },
|
||||
"浏览器会话出口已变更，按新出口重建",
|
||||
);
|
||||
// 插件 env 模式:浏览器按容器环境变量出网,与站点 HTTP(明确直连 / 站点代理)不一定一致
|
||||
return r.mode && r.mode !== "site" ? null : egressKey(siteEgress(site.proxy, proxyUrl));
|
||||
} catch (e) {
|
||||
this.ctx.logger.warn(
|
||||
{
|
||||
@@ -307,6 +358,7 @@ export class BrowserService {
|
||||
},
|
||||
"设置浏览器会话出口失败，浏览器按容器环境变量出网",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -851,7 +903,7 @@ export class BrowserService {
|
||||
for (const m of await provider.listSessions())
|
||||
if (m.profileKey === profileKey && m.controller !== "closed") provider.setController(m.id, "closed");
|
||||
const prep = await this.prepareCookies(site, accountId);
|
||||
await this.applyEgress(provider, site, profileKey);
|
||||
const egress = await this.applyEgress(provider, site, profileKey);
|
||||
await this.refreshIdentity(prow, { maxAgeMs: 10 * 60_000 }).catch(() => null);
|
||||
await this.rateLimit(site);
|
||||
let res: Awaited<ReturnType<BrowserProvider["openSession"]>>;
|
||||
@@ -904,6 +956,7 @@ export class BrowserService {
|
||||
needsSnapshot: true,
|
||||
credential,
|
||||
instanceId,
|
||||
egress,
|
||||
}),
|
||||
})
|
||||
.execute();
|
||||
@@ -1141,7 +1194,11 @@ export class BrowserService {
|
||||
}
|
||||
|
||||
/** 转等待处理:自动化动作被拒,生成待处理事项,等人在原站视图接管 */
|
||||
async enterWaiting(row: BrowserSessionRow, reason: string): Promise<BrowserSessionRow> {
|
||||
async enterWaiting(
|
||||
row: BrowserSessionRow,
|
||||
reason: string,
|
||||
text?: { title: string; summary: string },
|
||||
): Promise<BrowserSessionRow> {
|
||||
const provider = this.providers.get(row.provider_id)?.provider;
|
||||
if (provider?.sessionInfo(row.id)) provider.setController(row.id, "waiting");
|
||||
const site = await this.liveSite(row.site_id).catch(() => null);
|
||||
@@ -1154,8 +1211,8 @@ export class BrowserService {
|
||||
kind: "other",
|
||||
status: "awaiting_user",
|
||||
severity: "warning",
|
||||
title: `${site?.name ?? "站点"}：原站需要人工处理`,
|
||||
summary: `${REASON_TEXT[reason] ?? reason}。在站点详情的原站视图接管会话，完成后交还。`,
|
||||
title: text?.title ?? `${site?.name ?? "站点"}：原站需要人工处理`,
|
||||
summary: text?.summary ?? `${REASON_TEXT[reason] ?? reason}。在站点详情的原站视图接管会话，完成后交还。`,
|
||||
subject_type: "browser_session",
|
||||
subject_id: row.id,
|
||||
dedupe_key: `browser.waiting:${row.id}`,
|
||||
@@ -1327,6 +1384,14 @@ export class BrowserService {
|
||||
await this.persist(id, provider);
|
||||
// 人工在桌面 / 截图模式里登录过:交还时读回登录态写进站点账号,全站(HTTP 抓取 / RSS / 快照)生效
|
||||
await this.syncCookies(id, "release");
|
||||
// CF-2:等人过 Cloudflare 验证的会话交还后,读回 cf_clearance / UA,站点请求继续
|
||||
if (opts.to === "automation")
|
||||
await this.afterHumanChallenge(id).catch((e) =>
|
||||
this.ctx.logger.warn(
|
||||
{ component: "browser", sessionId: id, err: String((e as Error)?.message ?? e) },
|
||||
"交还后读回 Cloudflare 验证结果失败",
|
||||
),
|
||||
);
|
||||
return toLease(released);
|
||||
}
|
||||
|
||||
@@ -1586,7 +1651,7 @@ export class BrowserService {
|
||||
if (!provider.reopenSession) throw await fail("浏览器服务不支持重新连接");
|
||||
const url = restoreUrl(row.current_url, budget) ?? site.base_url;
|
||||
const prep = await this.prepareCookies(site, row.account_id).catch(() => null);
|
||||
await this.applyEgress(provider, site, row.profile_key);
|
||||
const egress = await this.applyEgress(provider, site, row.profile_key);
|
||||
// 浏览器重启过(可能换了镜像版本):身份重新读
|
||||
await this.refreshIdentity(prow).catch(() => null);
|
||||
let res: Awaited<ReturnType<NonNullable<BrowserProvider["reopenSession"]>>>;
|
||||
@@ -1606,6 +1671,7 @@ export class BrowserService {
|
||||
needsReconnect: false,
|
||||
needsSnapshot: true,
|
||||
instanceId: instanceId ?? rs.instanceId ?? null,
|
||||
egress,
|
||||
reconnect: { status: "ok", trigger, at, reason: null, humanNotice: row.controller === "human" },
|
||||
});
|
||||
this.coordSpaces.delete(id);
|
||||
@@ -1894,6 +1960,248 @@ export class BrowserService {
|
||||
return { decisionId, controller: row.controller };
|
||||
}
|
||||
|
||||
// ---------- CF-2 Cloudflare 挑战 ----------
|
||||
|
||||
/** 过挑战 / 代抓同一 profile 串行(本进程;跨进程的同站互斥靠 site.config.cloudflare.lastAttemptAt 条件更新) */
|
||||
private readonly cfLock = new InMemoryProfileLock(240_000);
|
||||
|
||||
/** 读当前页面内容(只读,不要求自动化控制权);上游不支持为 null */
|
||||
private async readPage(id: string): Promise<PageContent | null> {
|
||||
return this.run(id, CF_ACTOR, (p, ref) => (p.pageContent ? p.pageContent(ref) : Promise.resolve(null)), {
|
||||
write: false,
|
||||
persist: false,
|
||||
background: true,
|
||||
});
|
||||
}
|
||||
|
||||
/** 轮询页面直到不是挑战页(或变成真验证码 / 超时);返回最后一次读到的页面与分类 */
|
||||
private async waitChallenge(
|
||||
id: string,
|
||||
page: PageContent | null,
|
||||
ms: number,
|
||||
): Promise<{
|
||||
page: PageContent | null;
|
||||
kind: ReturnType<typeof classifyChallengePage>["kind"];
|
||||
rule: string | null;
|
||||
}> {
|
||||
let cur = page ?? (await this.readPage(id));
|
||||
let c = classifyChallengePage(cur ?? {});
|
||||
const deadline = Date.now() + ms;
|
||||
while (cur && c.kind !== "none" && c.kind !== "captcha" && Date.now() < deadline) {
|
||||
await new Promise((r) => setTimeout(r, this.opts.challengePollMs ?? 1_500));
|
||||
cur = await this.readPage(id);
|
||||
c = classifyChallengePage(cur ?? {});
|
||||
}
|
||||
return { page: cur, kind: cur ? c.kind : "none", rule: c.rule };
|
||||
}
|
||||
|
||||
/**
|
||||
* 该账号 profile 的自动化会话:已有且在自动化控制下就用它(导航到 url,取站点令牌);没有就打开一个(打开即导航)。
|
||||
* 人工接管中 / 等人处理中返回 null(不打扰人)。
|
||||
*/
|
||||
private async cfSession(
|
||||
site: SiteRow,
|
||||
accountId: string,
|
||||
url: string,
|
||||
): Promise<{ id: string } | { busy: BrowserSessionRow }> {
|
||||
const live = await this.liveSessionFor(site.id, accountId);
|
||||
if (live && live.controller !== "automation") return { busy: live };
|
||||
if (!live) return { id: await this.open({ siteId: site.id, accountId, url }) };
|
||||
await this.run(live.id, CF_ACTOR, async (p, ref, x) => {
|
||||
await this.rateLimit(x.site);
|
||||
await p.navigate(ref, url, x.budget);
|
||||
});
|
||||
return { id: live.id };
|
||||
}
|
||||
|
||||
/** 会话转"等你过验证":待处理事项写明是哪个站点、去哪里处理 */
|
||||
private async cfNeedsHuman(id: string, site: SiteRow, reason: string): Promise<ChallengeSolveOutcome> {
|
||||
const text = {
|
||||
title: `${site.name}需要你在原站视图里过一次验证`,
|
||||
summary:
|
||||
"站点要求人机验证（Cloudflare），系统没能自动通过。在站点详情的原站视图接管会话，完成验证后交还，站点请求会继续。",
|
||||
};
|
||||
let row = await this.sessionRow(id);
|
||||
if (row.controller === "automation") row = await this.enterWaiting(row, "cf_challenge", text);
|
||||
else if (row.controller === "waiting") {
|
||||
const rs = obj(row.recovery_state) as RecoveryState;
|
||||
if (!rs.waitingDecisionId) row = await this.enterWaiting(row, "cf_challenge", text);
|
||||
else {
|
||||
// 视觉子流程没成功时已转等待(challenge_unsolved):事项改成 Cloudflare 的说法
|
||||
await this.ctx.db
|
||||
.updateTable("decision")
|
||||
.set({ title: text.title, summary: text.summary })
|
||||
.where("id", "=", rs.waitingDecisionId)
|
||||
.execute();
|
||||
row = await this.patchRecovery(row, { waitingReason: "cf_challenge" });
|
||||
}
|
||||
}
|
||||
const decisionId = (obj(row.recovery_state) as RecoveryState).waitingDecisionId ?? null;
|
||||
if (decisionId)
|
||||
await this.ctx.db
|
||||
.updateTable("decision")
|
||||
.set({
|
||||
evidence: sql`evidence || ${JSON.stringify({ cloudflare: { siteId: site.id, reason } })}::jsonb`,
|
||||
})
|
||||
.where("id", "=", decisionId)
|
||||
.execute();
|
||||
this.ctx.logger.info({ component: "browser", sessionId: id, siteId: site.id, reason }, "Cloudflare 验证转人工");
|
||||
return { status: "needs_human", sessionId: id, decisionId, reason };
|
||||
}
|
||||
|
||||
/** 读回 profile 里的 cf_clearance 值(只取作用于站点主机的) */
|
||||
private async cfClearance(profileKey: string, siteUrl: string): Promise<string | null> {
|
||||
const prow = await this.providerRow().catch(() => null);
|
||||
const provider = prow ? this.providerFor(prow) : null;
|
||||
if (!provider?.getCookies) return null;
|
||||
const list = await provider.getCookies(profileKey, [siteUrl]).catch(() => []);
|
||||
const host = new URL(siteUrl).hostname;
|
||||
const hit = list.find(
|
||||
(c) =>
|
||||
c.name === "cf_clearance" &&
|
||||
(host === c.domain.replace(/^\./, "") || host.endsWith(`.${c.domain.replace(/^\./, "")}`)) &&
|
||||
!(c.expires !== undefined && c.expires > 0 && c.expires * 1000 <= this.now().getTime()),
|
||||
);
|
||||
return hit?.value ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 过挑战(CF-2 级联):用该账号 profile 打开同一地址 → ① 等它自己过(≤15 秒)→ ② 复选框 / Turnstile 派视觉子流程点一下
|
||||
* (最多 2 次,系统发起,预算取设置;自动化总开关关闭时跳过)→ ③ 选图 / 输码类或仍没过:会话转等待,开待处理事项。
|
||||
* 通过后读回 cf_clearance 与 navigator.userAgent,连同浏览器出口交给调用方。
|
||||
*/
|
||||
async solveSiteChallenge(req: { siteId: string; accountId: string; url: string }): Promise<ChallengeSolveOutcome> {
|
||||
const site = await this.liveSite(req.siteId);
|
||||
if (!site.enabled) return { status: "failed", reason: "站点已停用" };
|
||||
const profileKey = profileKeyFor(site.id, req.accountId);
|
||||
return this.cfLock.run(profileKey, async () => {
|
||||
const got = await this.cfSession(site, req.accountId, req.url);
|
||||
if ("busy" in got) {
|
||||
const rs = obj(got.busy.recovery_state) as RecoveryState;
|
||||
if (got.busy.controller === "waiting" && rs.waitingReason === "cf_challenge")
|
||||
return {
|
||||
status: "needs_human",
|
||||
sessionId: got.busy.id,
|
||||
decisionId: rs.waitingDecisionId ?? null,
|
||||
reason: "已在等你过验证",
|
||||
};
|
||||
return { status: "failed", reason: "浏览器会话正由人工处理，没有打扰" };
|
||||
}
|
||||
const id = got.id;
|
||||
const opened = await this.sessionRow(id);
|
||||
if (opened.controller !== "automation")
|
||||
return { status: "failed", reason: "浏览器会话打开后在等人处理（登录已失效）" };
|
||||
// ① 等它自己过
|
||||
let w = await this.waitChallenge(id, null, this.opts.challengeAutoWaitMs ?? CF_AUTO_WAIT_MS);
|
||||
if (!w.page) return { status: "failed", reason: "浏览器服务不支持读取页面内容" };
|
||||
let by: "auto" | "vision" = "auto";
|
||||
// ② 复选框 / Turnstile:视觉子流程点一下
|
||||
if (w.kind === "interactive") {
|
||||
const vision = visions.get(this.ctx);
|
||||
const auto = await this.ctx.settings.get("automation").catch(() => null);
|
||||
if (vision && auto?.enabled && !auto.readOnlyReconcile) {
|
||||
const marker = /verify you are human|确认您是真人|验证您是真人|我是人类/i.exec(w.page.text)?.[0] ?? null;
|
||||
for (let i = 0; i < CF_VISION_ATTEMPTS && w.kind === "interactive"; i++) {
|
||||
const r = await vision({
|
||||
sessionId: id,
|
||||
goal: "这是 Cloudflare 人机验证页。只点一次「确认您是真人 / Verify you are human」复选框（Turnstile），然后等待页面跳转；不要做其它操作。如果页面要求选图片、输入验证码或别的操作，立即停止并说明需要人工。",
|
||||
textAbsent: marker,
|
||||
maxSteps: CF_VISION_STEPS,
|
||||
}).catch((e) => ({ status: "failed", reason: String((e as Error)?.message ?? e), childRunId: null }));
|
||||
this.ctx.logger.info(
|
||||
{ component: "browser", sessionId: id, siteId: site.id, attempt: i + 1, status: r.status },
|
||||
"Cloudflare 复选框视觉子流程结束",
|
||||
);
|
||||
// 子流程没成功时会话已转等待(challenge_unsolved);下一次子流程开始时自己收回自动化控制
|
||||
w = await this.waitChallenge(
|
||||
id,
|
||||
null,
|
||||
Math.min(CF_AFTER_CLICK_WAIT_MS, this.opts.challengeAutoWaitMs ?? CF_AFTER_CLICK_WAIT_MS),
|
||||
);
|
||||
if (w.kind === "none") by = "vision";
|
||||
}
|
||||
}
|
||||
}
|
||||
// ③ 没过
|
||||
if (w.kind !== "none")
|
||||
return this.cfNeedsHuman(
|
||||
id,
|
||||
site,
|
||||
w.kind === "captcha" ? "选图 / 输入验证码类验证，不自动处理" : `没能自动通过(${w.rule ?? w.kind})`,
|
||||
);
|
||||
// 通过:子流程成功时关掉它留下的待处理;读回 cf_clearance 与 UA
|
||||
const row = await this.sessionRow(id);
|
||||
if (row.controller === "waiting") {
|
||||
await this.resolveWaiting(row, "system:cloudflare");
|
||||
const { provider } = await this.loaded(row);
|
||||
provider.setController(id, "automation");
|
||||
await this.ctx.db
|
||||
.updateTable("browser_session")
|
||||
.set({
|
||||
controller: "automation",
|
||||
recovery_state: JSON.stringify({
|
||||
...obj(row.recovery_state),
|
||||
waitingReason: null,
|
||||
waitingDecisionId: null,
|
||||
}),
|
||||
updated_at: this.now(),
|
||||
})
|
||||
.where("id", "=", id)
|
||||
.execute();
|
||||
}
|
||||
const rs = obj((await this.sessionRow(id)).recovery_state) as RecoveryState;
|
||||
return {
|
||||
status: "passed",
|
||||
by,
|
||||
clearance: await this.cfClearance(profileKey, site.base_url),
|
||||
userAgent: w.page?.userAgent ?? (await this.identityUserAgent().catch(() => null)),
|
||||
browserEgress: rs.egress ?? null,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 浏览器代抓一页(CF-2 回退):该账号 profile 的自动化会话导航到 url(取站点令牌,受 profile 锁约束),读 outerHTML 交给原解析器。
|
||||
* 站点停用、会话正由人工处理 / 等人处理、仍是挑战页时不返回页面。
|
||||
*/
|
||||
async fetchSiteHtml(req: { siteId: string; accountId: string; url: string }): Promise<BrowserFetchOutcome> {
|
||||
const site = await this.liveSite(req.siteId).catch(() => null);
|
||||
if (!site?.enabled) return { status: "unavailable", reason: "站点已停用" };
|
||||
const profileKey = profileKeyFor(site.id, req.accountId);
|
||||
try {
|
||||
return await this.cfLock.run(profileKey, async () => {
|
||||
const got = await this.cfSession(site, req.accountId, req.url);
|
||||
if ("busy" in got) return { status: "unavailable" as const, reason: "浏览器会话正由人工处理或在等人处理" };
|
||||
const w = await this.waitChallenge(got.id, null, this.opts.challengeAutoWaitMs ?? CF_AUTO_WAIT_MS);
|
||||
if (!w.page) return { status: "unavailable" as const, reason: "浏览器服务不支持读取页面内容" };
|
||||
if (w.kind !== "none") return { status: "challenge" as const, reason: w.rule ?? w.kind };
|
||||
return { status: "ok" as const, html: w.page.html, url: w.page.url };
|
||||
});
|
||||
} catch (e) {
|
||||
const err = fromBrowserError(e);
|
||||
return { status: "unavailable", reason: String((err as Error)?.message ?? err).slice(0, 300) };
|
||||
}
|
||||
}
|
||||
|
||||
/** 人过完验证交还后:站点在等人过验证、会话是那一个 → 读回 cf_clearance / UA,通知站点请求继续 */
|
||||
private async afterHumanChallenge(id: string): Promise<void> {
|
||||
const row = await this.sessionRow(id);
|
||||
if (!row.account_id) return;
|
||||
const rt = this.opts.runtime?.(this.ctx);
|
||||
if (!rt) return;
|
||||
const st = await rt.cloudflare.state(row.site_id, true);
|
||||
if (st.pending !== "human") return;
|
||||
const site = await this.liveSite(row.site_id);
|
||||
const page = await this.readPage(id).catch(() => null);
|
||||
if (page && classifyChallengePage(page).kind !== "none") return; // 还在挑战页:继续等
|
||||
const rs = obj(row.recovery_state) as RecoveryState;
|
||||
await rt.cloudflare.afterHuman(row.site_id, row.account_id, {
|
||||
clearance: await this.cfClearance(row.profile_key, site.base_url),
|
||||
userAgent: page?.userAgent ?? (await this.identityUserAgent().catch(() => null)),
|
||||
browserEgress: rs.egress ?? null,
|
||||
});
|
||||
}
|
||||
|
||||
// ---------- 下载 ----------
|
||||
|
||||
/** 暂存产物核验:解析种子、按 infohash / 站点资源 ID 匹配本地资源、会话内去重 */
|
||||
@@ -1972,6 +2280,7 @@ export class BrowserService {
|
||||
}
|
||||
|
||||
const REASON_TEXT: Record<string, string> = {
|
||||
cf_challenge: "站点要求人机验证（Cloudflare），需要你过一次",
|
||||
captcha: "登录页需要验证码",
|
||||
challenge: "页面需要人工验证",
|
||||
two_factor: "需要二次验证或本人设备确认",
|
||||
@@ -2054,7 +2363,7 @@ export function toLease(r: BrowserControlLeaseRow): BrowserControlLease {
|
||||
}
|
||||
|
||||
/** 视觉子流程可以接手的等待原因(验证码 / 页面挑战 / 上次没处理成功) */
|
||||
const SOLVABLE = ["captcha", "challenge", "challenge_unsolved"];
|
||||
const SOLVABLE = ["captcha", "challenge", "challenge_unsolved", "cf_challenge"];
|
||||
|
||||
/** capture / visionAct 的结果(能力输出的公共部分) */
|
||||
export interface VisionShot {
|
||||
|
||||
+3
-1
@@ -10,6 +10,7 @@ import {
|
||||
} from "@umt/contracts";
|
||||
import type { z } from "zod";
|
||||
import { dataPaths } from "../../config.ts";
|
||||
import { externalFetch } from "../../egress.ts";
|
||||
import { requireScope } from "../../http/auth.ts";
|
||||
import { errorBody } from "../../http/errors.ts";
|
||||
import { CACHE_REGIONS, sharedCache, storageSettings, tmdbCachePolicy } from "../../shared-cache.ts";
|
||||
@@ -73,7 +74,8 @@ export function createCacheModule(opts: CacheModuleOptions = {}): ServerModule &
|
||||
if (!c) {
|
||||
c = new ImageCache({
|
||||
dir: opts.imageDir ?? join(dataPaths(ctx.config).dataDir, "cache", "images"),
|
||||
...(opts.imageFetch ? { fetch: opts.imageFetch } : {}),
|
||||
// 外网请求:显式按环境变量走代理(CF-2 出口表)
|
||||
fetch: opts.imageFetch ?? externalFetch,
|
||||
...(opts.imageUpstreamBase ? { upstreamBase: opts.imageUpstreamBase } : {}),
|
||||
});
|
||||
imageCaches.set(ctx, c);
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
IdParam,
|
||||
MetadataEnrichOutput,
|
||||
} from "@umt/contracts";
|
||||
import { externalFetch } from "../../../egress.ts";
|
||||
import { requireScope } from "../../../http/auth.ts";
|
||||
import { ApiException, errorBody } from "../../../http/errors.ts";
|
||||
import type { ModuleApp } from "../../types.ts";
|
||||
@@ -122,7 +123,10 @@ export function enrichRoutes(app: ModuleApp, rt: () => SourcesRuntime): void {
|
||||
const srcHost = hostOf(s.data.fields.posterUrl?.sourceUrl ?? "");
|
||||
if (!srcHost || !domainAllowed(srcHost, identity.enrichDomains ?? []))
|
||||
return c.json(errorBody("forbidden", "海报出处不在补全白名单里"), 403);
|
||||
const got = await fetchPublicImage(r.overrides.fetch ?? fetch, url, { maxBytes: POSTER_MAX_BYTES, label: "海报" });
|
||||
const got = await fetchPublicImage(r.overrides.fetch ?? externalFetch, url, {
|
||||
maxBytes: POSTER_MAX_BYTES,
|
||||
label: "海报",
|
||||
});
|
||||
if (!got.ok) return c.json(errorBody("upstream_unavailable", got.message), 502);
|
||||
return imageResponse(got, 86400 * 7);
|
||||
});
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { Id, WorkbenchReleaseDetail, WorkbenchReleasePage, WorkbenchReleaseQuery } from "@umt/contracts";
|
||||
import { z } from "zod";
|
||||
import { externalFetch } from "../../../egress.ts";
|
||||
import { requireScope } from "../../../http/auth.ts";
|
||||
import { errorBody } from "../../../http/errors.ts";
|
||||
import type { ModuleApp } from "../../types.ts";
|
||||
@@ -55,7 +56,7 @@ export function workbenchRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
|
||||
const r = rt();
|
||||
const req = await detailScreenshotRequest(r, rid, index);
|
||||
if (!req) return c.json(errorBody("not_found", "截图不存在或详情缓存已过期"), 404);
|
||||
const got = await fetchPublicImage(r.overrides.fetch ?? fetch, req.url, {
|
||||
const got = await fetchPublicImage(r.overrides.fetch ?? externalFetch, req.url, {
|
||||
maxBytes: SHOT_MAX_BYTES,
|
||||
label: "截图",
|
||||
...(req.referer ? { referer: req.referer } : {}),
|
||||
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
} from "@umt/adapters";
|
||||
import { DEFAULT_SETTINGS, type SitesSettings } from "@umt/contracts";
|
||||
import { pgTokenBucketFor, type SiteRow, sql } from "@umt/db";
|
||||
import { egressKey, externalFetch, siteEgress, siteFetchFor } from "../../egress.ts";
|
||||
import {
|
||||
CACHE_REGIONS,
|
||||
renewTrackedTmdbDetail,
|
||||
@@ -24,6 +25,7 @@ import {
|
||||
tmdbDetailRef,
|
||||
} from "../../shared-cache.ts";
|
||||
import type { ServerContext } from "../types.ts";
|
||||
import { CloudflareGate, withClearance } from "./service/cloudflare.ts";
|
||||
import { readBrowserUserAgent, resolveUserAgent, type UserAgentDefaults } from "./service/user-agent.ts";
|
||||
|
||||
// 模块运行时依赖:站点适配器与限流器(每站一个实例,令牌在 PG 跨进程共享)、RSS 抓取器、TMDB 客户端、时钟。
|
||||
@@ -58,12 +60,15 @@ export class SourcesRuntime {
|
||||
private siteDefaultsCache: SitesSettings = DEFAULT_SETTINGS.sites;
|
||||
private siteDefaultsAt = 0;
|
||||
private tmdbSig: string | null = null;
|
||||
/** CF-2 Cloudflare 挑战处理(站点请求经 siteFetch 进来) */
|
||||
readonly cloudflare: CloudflareGate;
|
||||
|
||||
constructor(
|
||||
readonly ctx: ServerContext,
|
||||
readonly overrides: RuntimeOverrides = {},
|
||||
) {
|
||||
this.now = overrides.now ?? (() => new Date());
|
||||
this.cloudflare = new CloudflareGate(this);
|
||||
this.rss = createRssFetcher({
|
||||
...(overrides.fetch ? { fetch: overrides.fetch } : {}),
|
||||
now: this.now,
|
||||
@@ -101,7 +106,7 @@ export class SourcesRuntime {
|
||||
const { cfg } = await this.tmdbConfig();
|
||||
if (!cfg) return null;
|
||||
let n = 0;
|
||||
const base = cfg.fetch ?? globalThis.fetch;
|
||||
const base = cfg.fetch ?? externalFetch;
|
||||
const counted: typeof fetch = (input, init) => {
|
||||
n++;
|
||||
return base(input, init);
|
||||
@@ -144,7 +149,8 @@ export class SourcesRuntime {
|
||||
...(apiKey ? { apiKey } : {}),
|
||||
...(accessToken ? { accessToken } : {}),
|
||||
...(baseUrl ? { baseUrl } : {}),
|
||||
...(this.overrides.fetch ? { fetch: this.overrides.fetch } : {}),
|
||||
// 外网请求:显式按环境变量走代理(CF-2 出口表;测试注入优先)
|
||||
fetch: this.overrides.fetch ?? externalFetch,
|
||||
// 请求响应进 PG 缓存(region tmdb.request),三进程共享;TTL / 条数上限 / 空结果短 TTL 取设置(对照 MP)
|
||||
cache: {
|
||||
ttlMs: policy.ttl * 1000,
|
||||
@@ -168,40 +174,53 @@ export class SourcesRuntime {
|
||||
}
|
||||
|
||||
/**
|
||||
* 站点请求用的 fetch:测试注入优先;站点开了"使用代理"且设置了 proxyUrl 时走代理(Deno.createHttpClient)。
|
||||
* 站点请求用的 fetch:测试注入优先;站点开了"使用代理"且设置了 proxyUrl 时走代理,否则**明确直连**
|
||||
* (CF-2:不再隐式继承容器的 HTTP(S)_PROXY,见 egress.ts)。
|
||||
* BR-4:每条请求都带 UA(请求自己设了就不动;否则 ua 参数,缺省按"跟随浏览器"),不再发 Deno 缺省的 `Deno/x.y.z`。
|
||||
* ua 可以是函数:每次请求现场取(站点适配器按当前账号取,账号切换 UA 后立即生效)。
|
||||
* CF-2:给了 site.id 的请求经 Cloudflare 处理(浏览器代理期直接走浏览器、带可复用的 cf_clearance、遇挑战过挑战 / 重试 / 浏览器代抓)。
|
||||
*/
|
||||
siteFetch(
|
||||
site: Pick<SiteRow, "proxy">,
|
||||
site: Pick<SiteRow, "proxy"> & Partial<Pick<SiteRow, "id">>,
|
||||
defaults: SitesSettings = this.siteDefaultsCache,
|
||||
ua?: string | (() => Promise<string>),
|
||||
): typeof globalThis.fetch {
|
||||
const base = this.transportFetch(site, defaults);
|
||||
return ((input: RequestInfo | URL, init?: RequestInit) => {
|
||||
const egress = this.egressKeyOf(site, defaults);
|
||||
const siteId = site.id;
|
||||
return (async (input: RequestInfo | URL, init?: RequestInit) => {
|
||||
const headers = new Headers(init?.headers ?? (input instanceof Request ? input.headers : undefined));
|
||||
if (headers.has("user-agent")) return base(input, { ...init, headers });
|
||||
const pick = typeof ua === "function" ? ua() : ua !== undefined ? Promise.resolve(ua) : this.defaultUserAgent();
|
||||
return pick.then((v) => {
|
||||
headers.set("user-agent", v);
|
||||
return base(input, { ...init, headers });
|
||||
});
|
||||
if (!headers.has("user-agent")) {
|
||||
const pick = typeof ua === "function" ? ua() : ua !== undefined ? Promise.resolve(ua) : this.defaultUserAgent();
|
||||
headers.set("user-agent", await pick);
|
||||
}
|
||||
if (!siteId) return base(input, { ...init, headers });
|
||||
const url = input instanceof Request ? input.url : String(input);
|
||||
const method = init?.method ?? (input instanceof Request ? input.method : "GET");
|
||||
return this.cloudflare.handle(
|
||||
siteId,
|
||||
{ url, method, accept: headers.get("accept"), userAgent: headers.get("user-agent"), egress },
|
||||
(extra) => {
|
||||
const h = new Headers(headers);
|
||||
const cookie = withClearance(h.get("cookie"), extra);
|
||||
if (cookie) h.set("cookie", cookie);
|
||||
// 过挑战可能超过单次请求的超时:重试时原信号已中止就换一个新的超时(调用方取消照样生效在级联之外)
|
||||
const signal = init?.signal?.aborted ? AbortSignal.timeout(30_000) : init?.signal;
|
||||
return base(input, { ...init, headers: h, ...(signal ? { signal } : {}) });
|
||||
},
|
||||
);
|
||||
}) as typeof globalThis.fetch;
|
||||
}
|
||||
|
||||
/** 不加 UA 的底层 fetch(代理 / 测试注入) */
|
||||
/** 站点 HTTP 出口键(direct / proxy:<主机>),与浏览器会话的出口同一口径 */
|
||||
egressKeyOf(site: Pick<SiteRow, "proxy">, defaults: SitesSettings = this.siteDefaultsCache): string {
|
||||
return egressKey(siteEgress(site.proxy, defaults.proxyUrl));
|
||||
}
|
||||
|
||||
/** 不加 UA 的底层 fetch(测试注入 / 按站点出口:代理或明确直连) */
|
||||
private transportFetch(site: Pick<SiteRow, "proxy">, defaults: SitesSettings): typeof globalThis.fetch {
|
||||
if (this.overrides.fetch) return this.overrides.fetch;
|
||||
if (!site.proxy || !defaults.proxyUrl) return globalThis.fetch;
|
||||
const key = defaults.proxyUrl;
|
||||
let client = this.proxyClients.get(key);
|
||||
if (!client) {
|
||||
client = Deno.createHttpClient({ proxy: { url: key } });
|
||||
this.proxyClients.set(key, client);
|
||||
}
|
||||
const c = client;
|
||||
return ((input: RequestInfo | URL, init?: RequestInit) =>
|
||||
globalThis.fetch(input, { ...init, client: c } as RequestInit)) as typeof globalThis.fetch;
|
||||
return siteFetchFor(siteEgress(site.proxy, defaults.proxyUrl));
|
||||
}
|
||||
|
||||
// ---------- BR-4 站点请求 UA ----------
|
||||
@@ -253,8 +272,6 @@ export class SourcesRuntime {
|
||||
return resolveUserAgent(acc, d).ua;
|
||||
}
|
||||
|
||||
private readonly proxyClients = new Map<string, Deno.HttpClient>();
|
||||
|
||||
/**
|
||||
* 站点适配器:每站一个实例,列表/搜索/详情/下载共享同一个限流器。
|
||||
* 站点限流参数、基址、代理或站点默认变化时重建(签名比较)。
|
||||
@@ -270,10 +287,12 @@ export class SourcesRuntime {
|
||||
}
|
||||
const defaults = this.siteDefaultsCache;
|
||||
const proxy = site.proxy ?? false;
|
||||
// Cloudflare 处理状态(config.cloudflare)经常变,不参与重建判断
|
||||
const { cloudflare: _cf, ...configSig } = (site.config ?? {}) as Record<string, unknown>;
|
||||
const sig = JSON.stringify([
|
||||
site.base_url,
|
||||
site.rate_limit,
|
||||
site.config,
|
||||
configSig,
|
||||
proxy,
|
||||
proxy ? defaults.proxyUrl : null,
|
||||
defaults.defaultRateLimit,
|
||||
@@ -304,7 +323,7 @@ export class SourcesRuntime {
|
||||
limiter,
|
||||
now: this.now,
|
||||
// 每条请求都经 siteFetch:代理 + 按账号现场取 UA(BR-4)
|
||||
fetch: this.siteFetch({ proxy }, defaults, () => this.siteUserAgent(site.id)),
|
||||
fetch: this.siteFetch({ id: site.id, proxy }, defaults, () => this.siteUserAgent(site.id)),
|
||||
...(baseUrl ? { baseUrl } : {}),
|
||||
});
|
||||
this.adapters.set(site.id, { sig, limiter, adapter });
|
||||
|
||||
@@ -0,0 +1,431 @@
|
||||
import { detectChallenge, mergeCookieHeaders, parseCookieHeader } from "@umt/adapters";
|
||||
import { type SiteCloudflareState, siteCloudflareState } from "@umt/contracts";
|
||||
import { type SiteAccountRow, type SiteRow, sql } from "@umt/db";
|
||||
import { egressKey, siteEgress } from "../../../egress.ts";
|
||||
import type { ServerContext } from "../../types.ts";
|
||||
import type { SourcesRuntime } from "../runtime.ts";
|
||||
import { resolveUserAgent } from "./user-agent.ts";
|
||||
|
||||
// CF-2 Cloudflare 挑战处理(docs/design/cloudflare-handling.md;运维见 docs/runbook/sites.md「Cloudflare」)。
|
||||
//
|
||||
// 站点 HTTP 请求(SourcesRuntime.siteFetch)经这里:
|
||||
// 0. 站点在"浏览器代理期"(config.cloudflare.browserUntil 未到)且请求是页面(GET + Accept text/html)→ 直接由浏览器代抓;
|
||||
// 1. 发 HTTP;账号 UA 跟随浏览器、出口与浏览器一致、复用没失败过时带上浏览器拿到的 cf_clearance(__cf_bm 不带);
|
||||
// 2. 响应是挑战(cf-mitigated: challenge,或 403 / 503 / 429 的挑战页)→ 不算凭据错误,记 lastChallengeAt;
|
||||
// 3. 过挑战(同站 30 分钟最多一次、同一时刻只跑一次;只处理有登录态账号的站点):浏览器用该账号 profile 打开同一地址 →
|
||||
// ① 等它自己过(≤15 秒)→ ② 复选框 / Turnstile 派视觉子流程点一下(最多 2 次)→ ③ 选图 / 输码类转人工(待处理事项);
|
||||
// 4. 过了:读回 cf_clearance + navigator.userAgent;满足复用条件就存下来,HTTP 重试 1 次,结果记 cfReuse = ok / fails;
|
||||
// 5. 仍被挑战:页面请求改由浏览器导航取 HTML 交给原解析器,站点记 6 小时浏览器代理期;其它请求(下载、RSS、POST)照常报挑战。
|
||||
// 不接任何打码服务。
|
||||
|
||||
/** 同站两次过挑战的最短间隔 */
|
||||
export const CF_ATTEMPT_INTERVAL_MS = 30 * 60_000;
|
||||
/** HTTP 过不去时走浏览器的时长 */
|
||||
export const CF_BROWSER_MODE_MS = 6 * 3_600_000;
|
||||
/** 站点状态读缓存(每条站点请求都要看是否在浏览器代理期) */
|
||||
const STATE_CACHE_MS = 5_000;
|
||||
|
||||
export type ChallengeSolveOutcome =
|
||||
| {
|
||||
status: "passed";
|
||||
by: "auto" | "vision";
|
||||
/** 浏览器 profile 里的 cf_clearance 值;没有为 null */
|
||||
clearance: string | null;
|
||||
/** 浏览器 navigator.userAgent */
|
||||
userAgent: string | null;
|
||||
/** 浏览器这一侧的出口键(egressKey);按容器环境变量出网 / 不知道时 null */
|
||||
browserEgress: string | null;
|
||||
}
|
||||
| { status: "needs_human"; sessionId: string | null; decisionId: string | null; reason: string }
|
||||
| { status: "failed"; reason: string };
|
||||
|
||||
export type BrowserFetchOutcome =
|
||||
| { status: "ok"; html: string; url: string | null }
|
||||
| { status: "challenge"; reason: string }
|
||||
| { status: "unavailable"; reason: string };
|
||||
|
||||
/** 浏览器一侧(浏览器模块注册;sources-identity 不直接依赖浏览器模块) */
|
||||
export interface ChallengeBrowserPort {
|
||||
solve(req: { siteId: string; accountId: string; url: string }): Promise<ChallengeSolveOutcome>;
|
||||
fetchHtml(req: { siteId: string; accountId: string; url: string }): Promise<BrowserFetchOutcome>;
|
||||
}
|
||||
|
||||
const ports = new WeakMap<object, ChallengeBrowserPort>();
|
||||
|
||||
export function registerChallengeBrowser(ctx: object, port: ChallengeBrowserPort): void {
|
||||
ports.set(ctx, port);
|
||||
}
|
||||
|
||||
export function challengeBrowserFor(ctx: object): ChallengeBrowserPort | null {
|
||||
return ports.get(ctx) ?? null;
|
||||
}
|
||||
|
||||
/** user_info.cloudflare:浏览器取得的 cf_clearance(值在 secret 里)与取得时的 UA / 出口 */
|
||||
interface ClearanceInfo {
|
||||
secretRef: string;
|
||||
userAgent: string | null;
|
||||
egress: string | null;
|
||||
at: string;
|
||||
}
|
||||
|
||||
const obj = (v: unknown): Record<string, unknown> =>
|
||||
v && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : {};
|
||||
|
||||
function clearanceInfo(userInfo: unknown): ClearanceInfo | null {
|
||||
const c = obj(obj(userInfo).cloudflare);
|
||||
return typeof c.secretRef === "string" && typeof c.at === "string"
|
||||
? {
|
||||
secretRef: c.secretRef,
|
||||
userAgent: typeof c.userAgent === "string" ? c.userAgent : null,
|
||||
egress: typeof c.egress === "string" ? c.egress : null,
|
||||
at: c.at,
|
||||
}
|
||||
: null;
|
||||
}
|
||||
|
||||
export interface ChallengeRequest {
|
||||
url: string;
|
||||
method: string;
|
||||
/** 请求的 Accept 头 */
|
||||
accept: string | null;
|
||||
/** 请求实际带的 UA */
|
||||
userAgent: string | null;
|
||||
/** 该请求的 HTTP 出口键 */
|
||||
egress: string;
|
||||
}
|
||||
|
||||
const EMPTY: SiteCloudflareState = {
|
||||
browserUntil: null,
|
||||
cfReuse: null,
|
||||
lastChallengeAt: null,
|
||||
lastSolvedBy: null,
|
||||
lastSolvedAt: null,
|
||||
lastAttemptAt: null,
|
||||
pending: null,
|
||||
sessionId: null,
|
||||
lastResult: null,
|
||||
lastRule: null,
|
||||
lastError: null,
|
||||
};
|
||||
|
||||
/** 页面请求:浏览器导航能代替的(GET + 要 HTML) */
|
||||
export function browserFetchable(req: Pick<ChallengeRequest, "method" | "accept">): boolean {
|
||||
return req.method.toUpperCase() === "GET" && /text\/html|application\/xhtml/i.test(req.accept ?? "");
|
||||
}
|
||||
|
||||
/** cf_clearance 能不能交给 HTTP:账号 UA 跟随浏览器、UA 相同、两端出口相同、复用没失败过 */
|
||||
export function clearanceUsable(input: {
|
||||
uaSource: string;
|
||||
requestUa: string | null;
|
||||
clearanceUa: string | null;
|
||||
requestEgress: string;
|
||||
clearanceEgress: string | null;
|
||||
cfReuse: SiteCloudflareState["cfReuse"];
|
||||
}): boolean {
|
||||
return (
|
||||
input.uaSource === "browser" &&
|
||||
!!input.requestUa &&
|
||||
input.requestUa === input.clearanceUa &&
|
||||
!!input.clearanceEgress &&
|
||||
input.clearanceEgress === input.requestEgress &&
|
||||
input.cfReuse !== "fails"
|
||||
);
|
||||
}
|
||||
|
||||
export class CloudflareGate {
|
||||
private readonly cache = new Map<string, { at: number; state: SiteCloudflareState }>();
|
||||
/** 进程内同站同一时刻只过一次挑战:后到的请求等同一个结果 */
|
||||
private readonly inflight = new Map<string, Promise<ChallengeSolveOutcome | null>>();
|
||||
|
||||
constructor(private readonly rt: SourcesRuntime) {}
|
||||
|
||||
private get ctx(): ServerContext {
|
||||
return this.rt.ctx;
|
||||
}
|
||||
|
||||
private now(): Date {
|
||||
return this.rt.now();
|
||||
}
|
||||
|
||||
async state(siteId: string, fresh = false): Promise<SiteCloudflareState> {
|
||||
const hit = this.cache.get(siteId);
|
||||
if (!fresh && hit && Date.now() - hit.at < STATE_CACHE_MS) return hit.state;
|
||||
const row = await this.ctx.db.selectFrom("site").select("config").where("id", "=", siteId).executeTakeFirst();
|
||||
const state = { ...EMPTY, ...(siteCloudflareState(row?.config) ?? {}) };
|
||||
this.cache.set(siteId, { at: Date.now(), state });
|
||||
return state;
|
||||
}
|
||||
|
||||
/** 合并写 site.config.cloudflare(只动这一个键) */
|
||||
async patch(siteId: string, patch: Partial<SiteCloudflareState>): Promise<void> {
|
||||
await sql`update site set config = coalesce(config, '{}'::jsonb) || jsonb_build_object('cloudflare',
|
||||
coalesce(config->'cloudflare', '{}'::jsonb) || ${JSON.stringify(patch)}::jsonb)
|
||||
where id = ${siteId}`.execute(this.ctx.db);
|
||||
this.cache.delete(siteId);
|
||||
}
|
||||
|
||||
/**
|
||||
* 同站 30 分钟最多一次:条件更新 lastAttemptAt(跨进程)。抢到返回 true。
|
||||
*/
|
||||
private async claimAttempt(siteId: string): Promise<boolean> {
|
||||
const now = this.now();
|
||||
const before = new Date(now.getTime() - CF_ATTEMPT_INTERVAL_MS);
|
||||
const r = await sql<{
|
||||
id: string;
|
||||
}>`update site set config = coalesce(config, '{}'::jsonb) || jsonb_build_object('cloudflare',
|
||||
coalesce(config->'cloudflare', '{}'::jsonb) || jsonb_build_object('lastAttemptAt', ${now.toISOString()}::text))
|
||||
where id = ${siteId}
|
||||
and (config->'cloudflare'->>'lastAttemptAt' is null
|
||||
or (config->'cloudflare'->>'lastAttemptAt')::timestamptz <= ${before})
|
||||
returning id`.execute(this.ctx.db);
|
||||
this.cache.delete(siteId);
|
||||
return r.rows.length > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* 有登录态的账号:与站点请求用的凭证同一个(第一个启用、未失效的 Cookie / 账密账号,且确实有 Cookie)。
|
||||
* 没有返回 null(没登录的站点不处理挑战)。
|
||||
*/
|
||||
async loggedInAccount(siteId: string): Promise<SiteAccountRow | null> {
|
||||
const rows = await this.ctx.db
|
||||
.selectFrom("site_account")
|
||||
.selectAll()
|
||||
.where("site_id", "=", siteId)
|
||||
.where("enabled", "=", true)
|
||||
.where("status", "in", ["unknown", "active"])
|
||||
.where("auth_kind", "in", ["cookie", "password"])
|
||||
.orderBy("created_at")
|
||||
.execute();
|
||||
return (
|
||||
rows.find(
|
||||
(r) => (r.auth_kind === "cookie" && !!r.secret_ref) || (r.auth_kind === "password" && !!r.session_secret_ref),
|
||||
) ?? null
|
||||
);
|
||||
}
|
||||
|
||||
/** 这条请求能带的 cf_clearance 头片段("cf_clearance=…");不满足复用条件为 null */
|
||||
async clearanceCookie(siteId: string, req: ChallengeRequest): Promise<string | null> {
|
||||
const acc = await this.loggedInAccount(siteId);
|
||||
if (!acc) return null;
|
||||
const info = clearanceInfo(acc.user_info);
|
||||
if (!info) return null;
|
||||
const st = await this.state(siteId);
|
||||
const d = await this.rt.uaDefaults();
|
||||
const ok = clearanceUsable({
|
||||
uaSource: resolveUserAgent(acc, d).source,
|
||||
requestUa: req.userAgent,
|
||||
clearanceUa: info.userAgent,
|
||||
requestEgress: req.egress,
|
||||
clearanceEgress: info.egress,
|
||||
cfReuse: st.cfReuse,
|
||||
});
|
||||
if (!ok) return null;
|
||||
const v = await this.ctx.secrets.resolve(info.secretRef).catch(() => null);
|
||||
return v ? `cf_clearance=${v}` : null;
|
||||
}
|
||||
|
||||
/** 把浏览器拿到的 cf_clearance 存进账号(secret),记下取得时的 UA 与出口 */
|
||||
private async storeClearance(
|
||||
acc: SiteAccountRow,
|
||||
value: string,
|
||||
userAgent: string | null,
|
||||
egress: string | null,
|
||||
): Promise<{ conditionsChanged: boolean }> {
|
||||
const info = obj(acc.user_info);
|
||||
const prev = clearanceInfo(acc.user_info);
|
||||
const conditionsChanged = !!prev && (prev.userAgent !== userAgent || prev.egress !== egress);
|
||||
let ref = prev?.secretRef ?? null;
|
||||
if (ref) {
|
||||
try {
|
||||
await this.ctx.secrets.replace(ref, value);
|
||||
} catch {
|
||||
ref = null;
|
||||
}
|
||||
}
|
||||
if (!ref)
|
||||
ref = await this.ctx.secrets.create({
|
||||
purpose: "site_cookie",
|
||||
plaintext: value,
|
||||
label: `site_account:${acc.site_id}:${acc.label}:cf_clearance:${crypto.randomUUID().slice(0, 8)}`,
|
||||
});
|
||||
const next: ClearanceInfo = { secretRef: ref, userAgent, egress, at: this.now().toISOString() };
|
||||
await this.ctx.db
|
||||
.updateTable("site_account")
|
||||
.set({ user_info: JSON.stringify({ ...info, cloudflare: next }) })
|
||||
.where("id", "=", acc.id)
|
||||
.execute();
|
||||
// UA 或出口变了:之前"复用无效"的结论不再适用,重新试一次
|
||||
if (conditionsChanged) await this.patch(acc.site_id, { cfReuse: null });
|
||||
return { conditionsChanged };
|
||||
}
|
||||
|
||||
/** 过挑战:进程内同站只跑一次,跨进程 30 分钟一次;被节流返回 null */
|
||||
private solveOnce(site: SiteRow, acc: SiteAccountRow, url: string): Promise<ChallengeSolveOutcome | null> {
|
||||
const running = this.inflight.get(site.id);
|
||||
if (running) return running;
|
||||
const p = (async () => {
|
||||
const port = challengeBrowserFor(this.ctx);
|
||||
if (!port) return null;
|
||||
if (!(await this.claimAttempt(site.id))) return null;
|
||||
try {
|
||||
return await port.solve({ siteId: site.id, accountId: acc.id, url });
|
||||
} catch (e) {
|
||||
return { status: "failed" as const, reason: String((e as Error)?.message ?? e).slice(0, 300) };
|
||||
}
|
||||
})().finally(() => this.inflight.delete(site.id));
|
||||
this.inflight.set(site.id, p);
|
||||
return p;
|
||||
}
|
||||
|
||||
/** 浏览器代抓一页;失败 / 仍是挑战返回 null */
|
||||
private async viaBrowser(siteId: string, acc: SiteAccountRow, url: string): Promise<Response | null> {
|
||||
const port = challengeBrowserFor(this.ctx);
|
||||
if (!port) return null;
|
||||
let r: BrowserFetchOutcome;
|
||||
try {
|
||||
r = await port.fetchHtml({ siteId, accountId: acc.id, url });
|
||||
} catch (e) {
|
||||
r = { status: "unavailable", reason: String((e as Error)?.message ?? e) };
|
||||
}
|
||||
if (r.status !== "ok") {
|
||||
this.ctx.logger.info({ component: "cloudflare", siteId, status: r.status, reason: r.reason }, "浏览器代抓未成功");
|
||||
if (r.status === "challenge") await this.patch(siteId, { lastResult: "browser_challenge", lastError: r.reason });
|
||||
return null;
|
||||
}
|
||||
return new Response(r.html, {
|
||||
status: 200,
|
||||
headers: { "content-type": "text/html; charset=utf-8", "x-umt-via": "browser" },
|
||||
});
|
||||
}
|
||||
|
||||
private log(siteId: string, msg: string, extra: Record<string, unknown> = {}): void {
|
||||
this.ctx.logger.info({ component: "cloudflare", siteId, ...extra }, msg);
|
||||
}
|
||||
|
||||
/**
|
||||
* 站点请求入口。send(cookieExtra) 发一次 HTTP(cookieExtra 非空时并进 Cookie 头);返回交给适配器的响应
|
||||
* (挑战没处理掉时原样返回挑战响应,适配器报 challenge)。
|
||||
*/
|
||||
async handle(
|
||||
siteId: string,
|
||||
req: ChallengeRequest,
|
||||
send: (cookieExtra: string | null) => Promise<Response>,
|
||||
): Promise<Response> {
|
||||
const pagey = browserFetchable(req);
|
||||
let st = await this.state(siteId);
|
||||
const now = this.now();
|
||||
// 0. 浏览器代理期:页面请求直接走浏览器
|
||||
if (pagey && st.browserUntil && Date.parse(st.browserUntil) > now.getTime() && challengeBrowserFor(this.ctx)) {
|
||||
const acc = await this.loggedInAccount(siteId);
|
||||
const r = acc ? await this.viaBrowser(siteId, acc, req.url) : null;
|
||||
if (r) return r;
|
||||
}
|
||||
// 1. HTTP(满足条件时带 cf_clearance)
|
||||
// 从没遇到过挑战的站点不查账号(每条请求少两次查询)
|
||||
const extra = st.lastChallengeAt ? await this.clearanceCookie(siteId, req).catch(() => null) : null;
|
||||
const first = await detectChallenge(await send(extra));
|
||||
if (!first.rule) return first.res;
|
||||
// 2. 挑战:记下来;没有浏览器 / 没有登录态账号 → 原样返回(适配器报 challenge,不算凭据错误)
|
||||
await this.patch(siteId, { lastChallengeAt: now.toISOString(), lastRule: first.rule });
|
||||
const port = challengeBrowserFor(this.ctx);
|
||||
if (!port) return first.res;
|
||||
const acc = await this.loggedInAccount(siteId);
|
||||
if (!acc) {
|
||||
await this.patch(siteId, { lastResult: "skipped", lastError: "站点没有已登录的账号，不处理挑战" });
|
||||
return first.res;
|
||||
}
|
||||
const site = await this.ctx.db.selectFrom("site").selectAll().where("id", "=", siteId).executeTakeFirst();
|
||||
if (!site?.enabled || site.deleted_at) return first.res;
|
||||
// 3. 过挑战
|
||||
const out = await this.solveOnce(site, acc, req.url);
|
||||
st = await this.state(siteId, true);
|
||||
if (!out) this.log(siteId, "站点 Cloudflare 挑战：30 分钟内处理过，本次不再过挑战");
|
||||
else if (out.status === "passed") {
|
||||
const at = this.now().toISOString();
|
||||
await this.patch(siteId, {
|
||||
lastSolvedBy: out.by,
|
||||
lastSolvedAt: at,
|
||||
lastResult: "passed",
|
||||
pending: null,
|
||||
sessionId: null,
|
||||
lastError: null,
|
||||
});
|
||||
this.log(siteId, "站点 Cloudflare 挑战已通过", { by: out.by });
|
||||
// 4. 同步 cf_clearance 并重试 HTTP 一次
|
||||
if (out.clearance) {
|
||||
await this.storeClearance(acc, out.clearance, out.userAgent, out.browserEgress);
|
||||
const retryExtra = await this.clearanceCookie(siteId, req).catch(() => null);
|
||||
if (retryExtra) {
|
||||
await this.rt.limiterOf(siteId)?.acquire();
|
||||
const second = await detectChallenge(await send(retryExtra));
|
||||
if (!second.rule) {
|
||||
await this.patch(siteId, { cfReuse: "ok", lastResult: "reuse_ok", browserUntil: null });
|
||||
return second.res;
|
||||
}
|
||||
await second.res.body?.cancel().catch(() => {});
|
||||
await this.patch(siteId, { cfReuse: "fails", lastResult: "reuse_fails" });
|
||||
this.log(siteId, "浏览器的 cf_clearance 给 HTTP 用仍被挑战，之后不再复用");
|
||||
}
|
||||
}
|
||||
} else if (out.status === "needs_human") {
|
||||
await this.patch(siteId, {
|
||||
pending: "human",
|
||||
sessionId: out.sessionId,
|
||||
lastResult: "needs_human",
|
||||
lastError: out.reason,
|
||||
});
|
||||
} else {
|
||||
await this.patch(siteId, { lastResult: "failed", lastError: out.reason });
|
||||
}
|
||||
// 5. 页面请求改由浏览器代抓,并进入 6 小时浏览器代理期
|
||||
if (pagey && !(out && out.status === "needs_human")) {
|
||||
const r = await this.viaBrowser(siteId, acc, req.url);
|
||||
if (r) {
|
||||
const until = new Date(this.now().getTime() + CF_BROWSER_MODE_MS).toISOString();
|
||||
await this.patch(siteId, { browserUntil: until, ...(out ? {} : { lastResult: "browser" }) });
|
||||
await first.res.body?.cancel().catch(() => {});
|
||||
return r;
|
||||
}
|
||||
}
|
||||
return first.res;
|
||||
}
|
||||
|
||||
/**
|
||||
* 人在原站视图里过了挑战、交还会话之后(浏览器模块调用):读回的 cf_clearance / UA 存起来,清掉"等人处理"。
|
||||
*/
|
||||
async afterHuman(
|
||||
siteId: string,
|
||||
accountId: string,
|
||||
got: { clearance: string | null; userAgent: string | null; browserEgress: string | null },
|
||||
): Promise<void> {
|
||||
const st = await this.state(siteId, true);
|
||||
if (st.pending !== "human") return;
|
||||
const acc = await this.ctx.db.selectFrom("site_account").selectAll().where("id", "=", accountId).executeTakeFirst();
|
||||
if (acc && got.clearance) await this.storeClearance(acc, got.clearance, got.userAgent, got.browserEgress);
|
||||
await this.patch(siteId, {
|
||||
pending: null,
|
||||
sessionId: null,
|
||||
lastSolvedBy: "human",
|
||||
lastSolvedAt: this.now().toISOString(),
|
||||
lastResult: "passed",
|
||||
lastError: null,
|
||||
// 人过了:允许马上再试一次 HTTP(不等 30 分钟)
|
||||
lastAttemptAt: null,
|
||||
});
|
||||
this.log(siteId, "站点 Cloudflare 挑战已由人在原站视图里通过");
|
||||
}
|
||||
}
|
||||
|
||||
/** Cookie 头里并进 cf_clearance(同名覆盖);__cf_bm 等其它 Cloudflare Cookie 不带 */
|
||||
export function withClearance(cookieHeader: string | null, extra: string | null): string | null {
|
||||
if (!extra) return cookieHeader;
|
||||
const jar = parseCookieHeader(cookieHeader);
|
||||
jar.delete("__cf_bm");
|
||||
const merged = mergeCookieHeaders([...jar].map(([k, v]) => `${k}=${v}`).join("; ") || null, extra);
|
||||
return merged;
|
||||
}
|
||||
|
||||
/** 站点出口键(与浏览器会话 applyEgress 同一口径) */
|
||||
export function siteEgressKey(proxy: boolean, proxyUrl: string | null | undefined): string {
|
||||
return egressKey(siteEgress(proxy, proxyUrl));
|
||||
}
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
type WebSearchTestResult,
|
||||
} from "@umt/contracts";
|
||||
import { cachedCall, pgCacheFor, pgTokenBucketFor } from "@umt/db";
|
||||
import { externalFetch } from "../../egress.ts";
|
||||
import { storageSettings } from "../../shared-cache.ts";
|
||||
import type { ServerContext } from "../types.ts";
|
||||
|
||||
@@ -94,7 +95,8 @@ export class WebSearchService {
|
||||
return createBraveClient({
|
||||
apiKey,
|
||||
...(this.opts.braveBaseUrl ? { baseUrl: this.opts.braveBaseUrl } : {}),
|
||||
...(this.opts.fetch ? { fetch: this.opts.fetch } : {}),
|
||||
// 外网请求:显式按环境变量走代理(CF-2 出口表)
|
||||
fetch: this.opts.fetch ?? externalFetch,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -184,7 +186,7 @@ export class WebSearchService {
|
||||
allow ? "只能读取最近一次 web.search 结果里的链接" : "先用 web.search 搜索，再读取结果里的链接",
|
||||
);
|
||||
const page = await fetchPage(want, {
|
||||
...(this.opts.fetch ? { fetch: this.opts.fetch } : {}),
|
||||
fetch: this.opts.fetch ?? externalFetch,
|
||||
...(this.opts.resolveHost ? { resolve: this.opts.resolveHost } : {}),
|
||||
...(o.signal ? { signal: o.signal } : {}),
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user