feat(sites): 出口统一与 Cloudflare 过挑战级联(CF-2)
- 出口:站点请求不再隐式继承 HTTP(S)_PROXY,没开代理明确直连(创建客户端时临时拿掉代理环境变量,Deno 2.9 语义见 egress.ts), 开了走站点代理;TMDB / Brave / 图片改为显式按环境变量(行为不变) - CloudflareGate:浏览器代理期直接代抓 → HTTP(可复用时带 cf_clearance)→ 挑战 → 浏览器过挑战(同站 30 分钟一次、单飞) → cf_clearance 仅在 UA 跟随浏览器且两端同出口时给 HTTP,重试 1 次记 cfReuse → 仍被挑战则浏览器代抓并进入 6 小时代理期 - 浏览器:solveSiteChallenge(自动放行 ≤15s / 系统发起视觉子流程最多 2 次 / 真验证码转人工待处理)、fetchSiteHtml(profile 锁 + 站点令牌)、 交还后读回 cf_clearance 继续;site.config.cloudflare 状态契约 Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
6b14b0da45
commit
16083942c0
14 files changed
+1711
-43
No files matched your search
@@ -0,0 +1,228 @@
|
||||
import { siteCloudflareState } from "@umt/contracts";
|
||||
import { MOCK_CAMOFOX_IDENTITY, type MockCamofoxSite } from "@umt/testkit";
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest";
|
||||
import { browserServiceFor } from "../index.ts";
|
||||
import { type BrowserService, registerChallengeVision } from "../service.ts";
|
||||
import { type BrowserHarness, createBrowserHarness, type Loose, ORIGIN, pgAvailable } from "./harness.ts";
|
||||
|
||||
// CF-2 浏览器一侧(对 testkit mock camofox 的 Cloudflare 模式):
|
||||
// ① 自动放行(轮询页面)② 复选框 / Turnstile 派视觉子流程,用插件真实鼠标点击 ③ 选图验证码 / 点不过 → 会话转等待 + 待处理事项;
|
||||
// 人在原站视图里过完交还 → 读回 cf_clearance / UA 通知站点请求继续;浏览器代抓页面受 profile 锁与站点限流约束。
|
||||
|
||||
const up = await pgAvailable();
|
||||
const GOOD = "cf2-session-value";
|
||||
const CLEARANCE = "clr-from-browser";
|
||||
const LIST_HTML =
|
||||
'<html><head><title>种子</title></head><body><table class="torrents"><tr><td>Oppenheimer</td></tr></table></body></html>';
|
||||
|
||||
describe.skipIf(!up)("CF-2 浏览器过挑战 / 代抓", () => {
|
||||
let h: BrowserHarness;
|
||||
let svc: BrowserService;
|
||||
const cfCfg: NonNullable<MockCamofoxSite["cloudflare"]> = { mode: "auto", autoPolls: 2, clearance: CLEARANCE };
|
||||
const visionCalls: string[] = [];
|
||||
let visionClick = true;
|
||||
const url = `${ORIGIN}/torrents.php`;
|
||||
|
||||
const sessions = async () => (await h.req("/browser/sessions")).body.data.items as Loose[];
|
||||
const closeAll = async () => {
|
||||
for (const s of await sessions()) await h.req(`/browser/sessions/${s.id}`, { method: "DELETE" });
|
||||
};
|
||||
const profileKey = () => `umt-site-${h.siteId}-acct-${h.accountId}`;
|
||||
|
||||
beforeAll(async () => {
|
||||
h = await createBrowserHarness("cf2", {
|
||||
sessionCookie: { name: "c_secure_pass", valid: [GOOD], fresh: GOOD },
|
||||
cloudflare: cfCfg,
|
||||
html: (u) =>
|
||||
u.includes("torrents.php") ? LIST_HTML : "<html><head><title>首页</title></head><body></body></html>",
|
||||
browserOptions: { challengePollMs: 5, challengeAutoWaitMs: 200, settleMs: 0 },
|
||||
});
|
||||
await sessions(); // 第一次请求时浏览器模块才为这个上下文建服务
|
||||
svc = browserServiceFor(h.ctx)!;
|
||||
const ref = await h.secrets.create({ purpose: "site_cookie", plaintext: `c_secure_uid=1; c_secure_pass=${GOOD}` });
|
||||
await h.db
|
||||
.updateTable("site_account")
|
||||
.set({ auth_kind: "cookie", secret_ref: ref, session_secret_ref: null, status: "active" })
|
||||
.where("id", "=", h.accountId)
|
||||
.execute();
|
||||
// 视觉子流程(假):点一下复选框所在的位置(第 2 行)
|
||||
registerChallengeVision(h.ctx, async (req) => {
|
||||
visionCalls.push(req.sessionId);
|
||||
if (visionClick)
|
||||
await svc.visionAct(
|
||||
req.sessionId,
|
||||
{ kind: "automation", holder: "system:cloudflare" },
|
||||
(p, ref) => p.click(ref, { x: 100, y: 70 }),
|
||||
{ wantImage: false, sensitiveAllowed: false },
|
||||
);
|
||||
return { status: visionClick ? "solved" : "needs_human", reason: "", childRunId: null };
|
||||
});
|
||||
}, 120_000);
|
||||
afterAll(async () => {
|
||||
await h?.close();
|
||||
});
|
||||
beforeEach(async () => {
|
||||
await closeAll();
|
||||
h.mock.cfPassed.clear();
|
||||
h.mock.contentReads.clear();
|
||||
h.mock.actions.length = 0;
|
||||
for (const jar of h.mock.cookieJars.values())
|
||||
jar.splice(0, jar.length, ...jar.filter((c) => c.name !== "cf_clearance"));
|
||||
cfCfg.mode = "auto";
|
||||
visionCalls.length = 0;
|
||||
visionClick = true;
|
||||
await h.db
|
||||
.updateTable("site")
|
||||
.set({ config: JSON.stringify({}), enabled: true })
|
||||
.where("id", "=", h.siteId)
|
||||
.execute();
|
||||
await h.db
|
||||
.updateTable("settings")
|
||||
.set({ value: JSON.stringify({ enabled: true }) })
|
||||
.where("key", "=", "automation")
|
||||
.execute();
|
||||
});
|
||||
|
||||
it("① 自动放行:轮询到挑战页消失,读回 cf_clearance + navigator.userAgent + 出口", async () => {
|
||||
const r = await svc.solveSiteChallenge({ siteId: h.siteId, accountId: h.accountId, url });
|
||||
expect(r).toEqual({
|
||||
status: "passed",
|
||||
by: "auto",
|
||||
clearance: CLEARANCE,
|
||||
userAgent: MOCK_CAMOFOX_IDENTITY.userAgent,
|
||||
browserEgress: "direct",
|
||||
});
|
||||
expect(visionCalls).toEqual([]);
|
||||
// 会话仍归自动化,没有待处理事项
|
||||
const [s] = await sessions();
|
||||
expect(s.controller).toBe("automation");
|
||||
});
|
||||
|
||||
it("② 复选框 / Turnstile:等不过 → 视觉子流程用插件真实鼠标点击 → 通过(by vision)", async () => {
|
||||
cfCfg.mode = "interactive";
|
||||
const r = await svc.solveSiteChallenge({ siteId: h.siteId, accountId: h.accountId, url });
|
||||
expect(r).toMatchObject({ status: "passed", by: "vision", clearance: CLEARANCE });
|
||||
expect(visionCalls).toHaveLength(1);
|
||||
expect(h.mock.actions.some((a) => a.action === "mouse-click")).toBe(true);
|
||||
});
|
||||
|
||||
it("③ 视觉子流程点不过(最多 2 次)→ 会话转等待,待处理事项「观众需要你在原站视图里过一次验证」", async () => {
|
||||
cfCfg.mode = "interactive";
|
||||
visionClick = false;
|
||||
const r = await svc.solveSiteChallenge({ siteId: h.siteId, accountId: h.accountId, url });
|
||||
expect(r).toMatchObject({ status: "needs_human" });
|
||||
expect(visionCalls).toHaveLength(2);
|
||||
const s = (await h.req(`/browser/sessions/${(r as Loose).sessionId}`)).body.data as Loose;
|
||||
expect(s.controller).toBe("waiting");
|
||||
expect(s.waitingReason).toBe("cf_challenge");
|
||||
const d = await h.db
|
||||
.selectFrom("decision")
|
||||
.selectAll()
|
||||
.where("id", "=", (r as Loose).decisionId)
|
||||
.executeTakeFirstOrThrow();
|
||||
expect(d.title).toBe("观众需要你在原站视图里过一次验证");
|
||||
expect(d.status).toBe("awaiting_user");
|
||||
expect((d.evidence as Loose).cloudflare).toMatchObject({ siteId: h.siteId });
|
||||
// 已在等人:再来一次不打扰,直接返回同一个等待
|
||||
const again = await svc.solveSiteChallenge({ siteId: h.siteId, accountId: h.accountId, url });
|
||||
expect(again).toMatchObject({ status: "needs_human", sessionId: (r as Loose).sessionId });
|
||||
expect(visionCalls).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("自动化总开关关闭:不派视觉子流程,直接转人工", async () => {
|
||||
cfCfg.mode = "interactive";
|
||||
await h.db
|
||||
.insertInto("settings")
|
||||
.values({ key: "automation", value: JSON.stringify({ enabled: false }) })
|
||||
.onConflict((oc) => oc.column("key").doUpdateSet({ value: JSON.stringify({ enabled: false }) }))
|
||||
.execute();
|
||||
const r = await svc.solveSiteChallenge({ siteId: h.siteId, accountId: h.accountId, url });
|
||||
expect(r).toMatchObject({ status: "needs_human" });
|
||||
expect(visionCalls).toEqual([]);
|
||||
});
|
||||
|
||||
it("③ 选图 / 输码类:不点、不等,直接转人工;人过完交还 → 读回 cf_clearance,站点请求继续", async () => {
|
||||
cfCfg.mode = "captcha";
|
||||
const r = (await svc.solveSiteChallenge({ siteId: h.siteId, accountId: h.accountId, url })) as Loose;
|
||||
expect(r.status).toBe("needs_human");
|
||||
expect(r.reason).toContain("选图");
|
||||
expect(visionCalls).toEqual([]);
|
||||
// 站点请求一侧记下"等人处理"(级联在 sources-identity 里做,这里直接写状态)
|
||||
await h.db
|
||||
.updateTable("site")
|
||||
.set({ config: JSON.stringify({ cloudflare: { pending: "human", sessionId: r.sessionId } }) })
|
||||
.where("id", "=", h.siteId)
|
||||
.execute();
|
||||
// 人接管,在桌面里过了验证(桌面操作不经 API:直接改 mock 状态),交还
|
||||
expect(
|
||||
(await h.req(`/browser/sessions/${r.sessionId}/control/acquire`, { body: { reason: "过验证" } })).status,
|
||||
).toBe(200);
|
||||
h.mock.cfPassed.add(profileKey());
|
||||
h.mock.cookieJars.get(profileKey())!.push({
|
||||
name: "cf_clearance",
|
||||
value: "clr-human",
|
||||
domain: ".audiences.example",
|
||||
path: "/",
|
||||
expires: Math.floor(Date.now() / 1000) + 3600,
|
||||
httpOnly: true,
|
||||
secure: true,
|
||||
});
|
||||
expect((await h.req(`/browser/sessions/${r.sessionId}/control/release`, { body: {} })).status).toBe(200);
|
||||
const site = await h.db.selectFrom("site").select("config").where("id", "=", h.siteId).executeTakeFirstOrThrow();
|
||||
expect(siteCloudflareState(site.config)).toMatchObject({ pending: null, lastSolvedBy: "human" });
|
||||
const acc = await h.db
|
||||
.selectFrom("site_account")
|
||||
.select("user_info")
|
||||
.where("id", "=", h.accountId)
|
||||
.executeTakeFirstOrThrow();
|
||||
const info = (acc.user_info as Loose).cloudflare;
|
||||
expect(info).toMatchObject({ userAgent: MOCK_CAMOFOX_IDENTITY.userAgent, egress: "direct" });
|
||||
expect(h.secrets.plain.get(info.secretRef)).toBe("clr-human");
|
||||
// 待处理事项已关闭
|
||||
const d = await h.db
|
||||
.selectFrom("decision")
|
||||
.select("status")
|
||||
.where("id", "=", r.decisionId)
|
||||
.executeTakeFirstOrThrow();
|
||||
expect(d.status).toBe("resolved");
|
||||
});
|
||||
|
||||
it("浏览器代抓:过了挑战的 profile 导航取 outerHTML;同一 profile 并发串行(只开一个会话);每次导航取站点令牌", async () => {
|
||||
h.mock.cfPassed.add(profileKey());
|
||||
const [a, b] = await Promise.all([
|
||||
svc.fetchSiteHtml({ siteId: h.siteId, accountId: h.accountId, url }),
|
||||
svc.fetchSiteHtml({ siteId: h.siteId, accountId: h.accountId, url }),
|
||||
]);
|
||||
expect(a).toMatchObject({ status: "ok", html: LIST_HTML });
|
||||
expect(b).toMatchObject({ status: "ok", html: LIST_HTML });
|
||||
expect(await sessions()).toHaveLength(1);
|
||||
const navs = h.mock.calls.filter(
|
||||
(c) => c.path.endsWith("/navigate") || (c.path === "/tabs" && c.method === "POST"),
|
||||
);
|
||||
expect(navs.length).toBeGreaterThanOrEqual(2);
|
||||
const count = await h.db
|
||||
.selectFrom("site_request_count")
|
||||
.select((eb) => eb.fn.sum<number>("count").as("n"))
|
||||
.where("site_id", "=", h.siteId)
|
||||
.executeTakeFirst();
|
||||
expect(Number(count?.n ?? 0)).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
|
||||
it("浏览器代抓:仍是挑战页 → challenge;会话在人工接管中 → 不打扰;站点停用 → 不可用", async () => {
|
||||
cfCfg.mode = "captcha";
|
||||
expect(await svc.fetchSiteHtml({ siteId: h.siteId, accountId: h.accountId, url })).toMatchObject({
|
||||
status: "challenge",
|
||||
});
|
||||
const [s] = await sessions();
|
||||
await h.req(`/browser/sessions/${s.id}/control/acquire`, { body: { reason: "看看" } });
|
||||
expect(await svc.fetchSiteHtml({ siteId: h.siteId, accountId: h.accountId, url })).toMatchObject({
|
||||
status: "unavailable",
|
||||
});
|
||||
await h.req(`/browser/sessions/${s.id}/control/release`, { body: {} });
|
||||
await h.db.updateTable("site").set({ enabled: false }).where("id", "=", h.siteId).execute();
|
||||
expect(await svc.fetchSiteHtml({ siteId: h.siteId, accountId: h.accountId, url })).toMatchObject({
|
||||
status: "unavailable",
|
||||
reason: "站点已停用",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -3,7 +3,13 @@ import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { CapabilityRegistry, InvokeContext } from "@umt/agent";
|
||||
import { type DB, type Kysely, migrateToLatest, sql } from "@umt/db";
|
||||
import { buildFakeTorrent, createMockCamofox, type FakeTorrent, type MockCamofox } from "@umt/testkit";
|
||||
import {
|
||||
buildFakeTorrent,
|
||||
createMockCamofox,
|
||||
type FakeTorrent,
|
||||
type MockCamofox,
|
||||
type MockCamofoxSite,
|
||||
} from "@umt/testkit";
|
||||
import { Kysely as K, PostgresDialect } from "kysely";
|
||||
import pg from "pg";
|
||||
import { withSchemaLock } from "../../../../test/auth-settings/harness.ts";
|
||||
@@ -19,6 +25,7 @@ import { createSourcesIdentityModule } from "../../sources-identity/index.ts";
|
||||
import type { ServerContext } from "../../types.ts";
|
||||
import { registerBrowserCapabilities } from "../capabilities.ts";
|
||||
import { createBrowserModule } from "../index.ts";
|
||||
import type { BrowserModuleOptions } from "../service.ts";
|
||||
|
||||
// C1-E 浏览器模块测试装配:独立 PG schema + 真实迁移 + 交互式 mock camofox(观众风格站点)+ 真实能力注册中心。
|
||||
|
||||
@@ -61,6 +68,11 @@ export async function createBrowserHarness(
|
||||
sessionCookie?: { name: string; valid: string[]; fresh: string };
|
||||
cookieApi?: "plugin" | "upstream" | "none";
|
||||
cookieImportKey?: string;
|
||||
/** CF-2:mock 站点的 Cloudflare 挑战与页面 HTML */
|
||||
cloudflare?: MockCamofoxSite["cloudflare"];
|
||||
html?: MockCamofoxSite["html"];
|
||||
/** 浏览器模块的额外选项(挑战轮询间隔等) */
|
||||
browserOptions?: Partial<BrowserModuleOptions>;
|
||||
} = {},
|
||||
): Promise<BrowserHarness> {
|
||||
const torrent = buildFakeTorrent({ name: "Oppenheimer.2023.2160p.UHD.BluRay.x265-GRP" });
|
||||
@@ -70,6 +82,8 @@ export async function createBrowserHarness(
|
||||
...(opts.cookieImportKey ? { cookieImportKey: opts.cookieImportKey } : {}),
|
||||
site: {
|
||||
...(opts.sessionCookie ? { sessionCookie: opts.sessionCookie } : {}),
|
||||
...(opts.cloudflare ? { cloudflare: opts.cloudflare } : {}),
|
||||
...(opts.html ? { html: opts.html } : {}),
|
||||
origin: ORIGIN,
|
||||
username: "fixture-user",
|
||||
password: PASSWORD,
|
||||
@@ -110,7 +124,12 @@ export async function createBrowserHarness(
|
||||
capabilities: registry,
|
||||
} as ServerContext;
|
||||
const sources = createSourcesIdentityModule({ tmdb: () => null });
|
||||
const browser = createBrowserModule({ runtime: (c) => sources.runtime(c), stagingDir, downloadPollMs: 20 });
|
||||
const browser = createBrowserModule({
|
||||
runtime: (c) => sources.runtime(c),
|
||||
stagingDir,
|
||||
downloadPollMs: 20,
|
||||
...opts.browserOptions,
|
||||
});
|
||||
registerBrowserCapabilities(registry, () => browser.service(ctx));
|
||||
const app = createApp({
|
||||
config: { NODE_ENV: "development", DEV_ADMIN_KEY: DEV_KEY },
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
type BrowserProvider,
|
||||
type BrowserSecretResolver,
|
||||
CAMOFOX_VERIFIED_RELEASES,
|
||||
classifyChallengePage,
|
||||
cookiesFromHeader,
|
||||
createCamofoxBrowserProvider,
|
||||
type DownloadSink,
|
||||
@@ -12,7 +13,9 @@ import {
|
||||
getSiteDefinition,
|
||||
headerFromCookies,
|
||||
hostAllowed,
|
||||
InMemoryProfileLock,
|
||||
imageDimensions,
|
||||
type PageContent,
|
||||
type PageMetrics,
|
||||
parseTorrent,
|
||||
prepareImage,
|
||||
@@ -50,8 +53,14 @@ import {
|
||||
type SiteRow,
|
||||
sql,
|
||||
} from "@umt/db";
|
||||
import { egressKey, siteEgress } from "../../egress.ts";
|
||||
import type { SourcesRuntime } from "../sources-identity/runtime.ts";
|
||||
import { refreshSiteAccount } from "../sources-identity/service/account.ts";
|
||||
import {
|
||||
type BrowserFetchOutcome,
|
||||
type ChallengeSolveOutcome,
|
||||
registerChallengeBrowser,
|
||||
} from "../sources-identity/service/cloudflare.ts";
|
||||
import {
|
||||
readAccountCookie,
|
||||
registerBrowserCookieReader,
|
||||
@@ -85,6 +94,10 @@ export interface BrowserModuleOptions {
|
||||
evidenceDir?: string;
|
||||
/** 坐标动作之后等页面稳定再截图的毫秒数;缺省 400 */
|
||||
settleMs?: number;
|
||||
/** CF-2 等挑战页放行时读页面的间隔(毫秒);缺省 1500 */
|
||||
challengePollMs?: number;
|
||||
/** CF-2 等 Cloudflare 自己放行的最长时间(毫秒);缺省 CF_AUTO_WAIT_MS(15 秒) */
|
||||
challengeAutoWaitMs?: number;
|
||||
/** 保活间隔(AG-4);缺省 BROWSER_KEEPALIVE_MS 配置(120 秒),0 = 关闭 */
|
||||
keepaliveMs?: number;
|
||||
/** 测试替换 provider 实现 */
|
||||
@@ -175,8 +188,39 @@ export interface RecoveryState {
|
||||
reconnect?: BrowserReconnectState;
|
||||
/** 最近一次自动化(Agent 能力)动作成功的时间;原站视图据此区分「Agent 正在操作」与「无人操作」 */
|
||||
lastAutomationAt?: string;
|
||||
/** CF-2:打开 / 重连时下发给浏览器的出口键(egressKey);插件没按站点设出口(旧镜像 / env 模式)为 null */
|
||||
egress?: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* CF-2 视觉子流程钩子(agent 模块注册):系统发起、没有父会话,预算取设置;只点「我是人类」复选框 / Turnstile 一步。
|
||||
* 返回子流程结论(solved 仍要由调用方按页面核实)。
|
||||
*/
|
||||
export type ChallengeVision = (req: {
|
||||
sessionId: string;
|
||||
goal: string;
|
||||
/** 通过后页面上不再出现的文字(成功条件) */
|
||||
textAbsent: string | null;
|
||||
maxSteps: number;
|
||||
}) => Promise<{ status: string; reason: string; childRunId: string | null }>;
|
||||
|
||||
const visions = new WeakMap<object, ChallengeVision>();
|
||||
|
||||
export function registerChallengeVision(ctx: object, fn: ChallengeVision): void {
|
||||
visions.set(ctx, fn);
|
||||
}
|
||||
|
||||
/** 系统(Cloudflare 处理)发起的浏览器动作 */
|
||||
const CF_ACTOR: Actor = { kind: "automation", holder: "system:cloudflare" };
|
||||
/** 等 Cloudflare 自己放行的最长时间 */
|
||||
export const CF_AUTO_WAIT_MS = 15_000;
|
||||
/** 视觉子流程点过之后再等多久 */
|
||||
const CF_AFTER_CLICK_WAIT_MS = 10_000;
|
||||
/** 视觉子流程最多派几次 */
|
||||
const CF_VISION_ATTEMPTS = 2;
|
||||
/** 视觉子流程步数上限:截图 → 点击 → 等待 */
|
||||
const CF_VISION_STEPS = 4;
|
||||
|
||||
/** 同一浏览器服务的实例标识多久查一次 */
|
||||
const INSTANCE_CHECK_MS = 5_000;
|
||||
/** 自动重连失败后,操作触发的重连至少间隔多久 */
|
||||
@@ -221,6 +265,11 @@ export class BrowserService {
|
||||
registerBrowserCookieReader(ctx, (siteId, accountId) => this.readProfileCookie(siteId, accountId));
|
||||
// 站点请求 UA 跟随浏览器固定身份(BR-4):sources-identity 经钩子取
|
||||
registerBrowserIdentityReader(ctx, () => this.identityUserAgent());
|
||||
// CF-2:站点 HTTP 遇 Cloudflare 挑战时由浏览器过挑战 / 代抓页面
|
||||
registerChallengeBrowser(ctx, {
|
||||
solve: (req) => this.solveSiteChallenge(req),
|
||||
fetchHtml: (req) => this.fetchSiteHtml(req),
|
||||
});
|
||||
}
|
||||
|
||||
// ---------- BR-4 浏览器身份与出口 ----------
|
||||
@@ -283,8 +332,8 @@ export class BrowserService {
|
||||
* 出口按站点统一(BR-4):站点开了代理且设置了代理地址 → 该 profile 走同一代理;否则直连。
|
||||
* 旧镜像(插件没有出口接口)只记日志,浏览器照旧走容器环境变量里的代理。
|
||||
*/
|
||||
private async applyEgress(provider: BrowserProvider, site: SiteRow, profileKey: string): Promise<void> {
|
||||
if (!provider.setEgress) return;
|
||||
private async applyEgress(provider: BrowserProvider, site: SiteRow, profileKey: string): Promise<string | null> {
|
||||
if (!provider.setEgress) return null;
|
||||
const rt = this.opts.runtime?.(this.ctx);
|
||||
const proxyUrl = rt
|
||||
? (await rt.siteDefaults()).proxyUrl
|
||||
@@ -297,6 +346,8 @@ export class BrowserService {
|
||||
{ component: "browser", siteId: site.id, egress: proxy ? "proxy" : "direct" },
|
||||
"浏览器会话出口已变更，按新出口重建",
|
||||
);
|
||||
// 插件 env 模式:浏览器按容器环境变量出网,与站点 HTTP(明确直连 / 站点代理)不一定一致
|
||||
return r.mode && r.mode !== "site" ? null : egressKey(siteEgress(site.proxy, proxyUrl));
|
||||
} catch (e) {
|
||||
this.ctx.logger.warn(
|
||||
{
|
||||
@@ -307,6 +358,7 @@ export class BrowserService {
|
||||
},
|
||||
"设置浏览器会话出口失败，浏览器按容器环境变量出网",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -851,7 +903,7 @@ export class BrowserService {
|
||||
for (const m of await provider.listSessions())
|
||||
if (m.profileKey === profileKey && m.controller !== "closed") provider.setController(m.id, "closed");
|
||||
const prep = await this.prepareCookies(site, accountId);
|
||||
await this.applyEgress(provider, site, profileKey);
|
||||
const egress = await this.applyEgress(provider, site, profileKey);
|
||||
await this.refreshIdentity(prow, { maxAgeMs: 10 * 60_000 }).catch(() => null);
|
||||
await this.rateLimit(site);
|
||||
let res: Awaited<ReturnType<BrowserProvider["openSession"]>>;
|
||||
@@ -904,6 +956,7 @@ export class BrowserService {
|
||||
needsSnapshot: true,
|
||||
credential,
|
||||
instanceId,
|
||||
egress,
|
||||
}),
|
||||
})
|
||||
.execute();
|
||||
@@ -1141,7 +1194,11 @@ export class BrowserService {
|
||||
}
|
||||
|
||||
/** 转等待处理:自动化动作被拒,生成待处理事项,等人在原站视图接管 */
|
||||
async enterWaiting(row: BrowserSessionRow, reason: string): Promise<BrowserSessionRow> {
|
||||
async enterWaiting(
|
||||
row: BrowserSessionRow,
|
||||
reason: string,
|
||||
text?: { title: string; summary: string },
|
||||
): Promise<BrowserSessionRow> {
|
||||
const provider = this.providers.get(row.provider_id)?.provider;
|
||||
if (provider?.sessionInfo(row.id)) provider.setController(row.id, "waiting");
|
||||
const site = await this.liveSite(row.site_id).catch(() => null);
|
||||
@@ -1154,8 +1211,8 @@ export class BrowserService {
|
||||
kind: "other",
|
||||
status: "awaiting_user",
|
||||
severity: "warning",
|
||||
title: `${site?.name ?? "站点"}：原站需要人工处理`,
|
||||
summary: `${REASON_TEXT[reason] ?? reason}。在站点详情的原站视图接管会话，完成后交还。`,
|
||||
title: text?.title ?? `${site?.name ?? "站点"}：原站需要人工处理`,
|
||||
summary: text?.summary ?? `${REASON_TEXT[reason] ?? reason}。在站点详情的原站视图接管会话，完成后交还。`,
|
||||
subject_type: "browser_session",
|
||||
subject_id: row.id,
|
||||
dedupe_key: `browser.waiting:${row.id}`,
|
||||
@@ -1327,6 +1384,14 @@ export class BrowserService {
|
||||
await this.persist(id, provider);
|
||||
// 人工在桌面 / 截图模式里登录过:交还时读回登录态写进站点账号,全站(HTTP 抓取 / RSS / 快照)生效
|
||||
await this.syncCookies(id, "release");
|
||||
// CF-2:等人过 Cloudflare 验证的会话交还后,读回 cf_clearance / UA,站点请求继续
|
||||
if (opts.to === "automation")
|
||||
await this.afterHumanChallenge(id).catch((e) =>
|
||||
this.ctx.logger.warn(
|
||||
{ component: "browser", sessionId: id, err: String((e as Error)?.message ?? e) },
|
||||
"交还后读回 Cloudflare 验证结果失败",
|
||||
),
|
||||
);
|
||||
return toLease(released);
|
||||
}
|
||||
|
||||
@@ -1586,7 +1651,7 @@ export class BrowserService {
|
||||
if (!provider.reopenSession) throw await fail("浏览器服务不支持重新连接");
|
||||
const url = restoreUrl(row.current_url, budget) ?? site.base_url;
|
||||
const prep = await this.prepareCookies(site, row.account_id).catch(() => null);
|
||||
await this.applyEgress(provider, site, row.profile_key);
|
||||
const egress = await this.applyEgress(provider, site, row.profile_key);
|
||||
// 浏览器重启过(可能换了镜像版本):身份重新读
|
||||
await this.refreshIdentity(prow).catch(() => null);
|
||||
let res: Awaited<ReturnType<NonNullable<BrowserProvider["reopenSession"]>>>;
|
||||
@@ -1606,6 +1671,7 @@ export class BrowserService {
|
||||
needsReconnect: false,
|
||||
needsSnapshot: true,
|
||||
instanceId: instanceId ?? rs.instanceId ?? null,
|
||||
egress,
|
||||
reconnect: { status: "ok", trigger, at, reason: null, humanNotice: row.controller === "human" },
|
||||
});
|
||||
this.coordSpaces.delete(id);
|
||||
@@ -1894,6 +1960,248 @@ export class BrowserService {
|
||||
return { decisionId, controller: row.controller };
|
||||
}
|
||||
|
||||
// ---------- CF-2 Cloudflare 挑战 ----------
|
||||
|
||||
/** 过挑战 / 代抓同一 profile 串行(本进程;跨进程的同站互斥靠 site.config.cloudflare.lastAttemptAt 条件更新) */
|
||||
private readonly cfLock = new InMemoryProfileLock(240_000);
|
||||
|
||||
/** 读当前页面内容(只读,不要求自动化控制权);上游不支持为 null */
|
||||
private async readPage(id: string): Promise<PageContent | null> {
|
||||
return this.run(id, CF_ACTOR, (p, ref) => (p.pageContent ? p.pageContent(ref) : Promise.resolve(null)), {
|
||||
write: false,
|
||||
persist: false,
|
||||
background: true,
|
||||
});
|
||||
}
|
||||
|
||||
/** 轮询页面直到不是挑战页(或变成真验证码 / 超时);返回最后一次读到的页面与分类 */
|
||||
private async waitChallenge(
|
||||
id: string,
|
||||
page: PageContent | null,
|
||||
ms: number,
|
||||
): Promise<{
|
||||
page: PageContent | null;
|
||||
kind: ReturnType<typeof classifyChallengePage>["kind"];
|
||||
rule: string | null;
|
||||
}> {
|
||||
let cur = page ?? (await this.readPage(id));
|
||||
let c = classifyChallengePage(cur ?? {});
|
||||
const deadline = Date.now() + ms;
|
||||
while (cur && c.kind !== "none" && c.kind !== "captcha" && Date.now() < deadline) {
|
||||
await new Promise((r) => setTimeout(r, this.opts.challengePollMs ?? 1_500));
|
||||
cur = await this.readPage(id);
|
||||
c = classifyChallengePage(cur ?? {});
|
||||
}
|
||||
return { page: cur, kind: cur ? c.kind : "none", rule: c.rule };
|
||||
}
|
||||
|
||||
/**
|
||||
* 该账号 profile 的自动化会话:已有且在自动化控制下就用它(导航到 url,取站点令牌);没有就打开一个(打开即导航)。
|
||||
* 人工接管中 / 等人处理中返回 null(不打扰人)。
|
||||
*/
|
||||
private async cfSession(
|
||||
site: SiteRow,
|
||||
accountId: string,
|
||||
url: string,
|
||||
): Promise<{ id: string } | { busy: BrowserSessionRow }> {
|
||||
const live = await this.liveSessionFor(site.id, accountId);
|
||||
if (live && live.controller !== "automation") return { busy: live };
|
||||
if (!live) return { id: await this.open({ siteId: site.id, accountId, url }) };
|
||||
await this.run(live.id, CF_ACTOR, async (p, ref, x) => {
|
||||
await this.rateLimit(x.site);
|
||||
await p.navigate(ref, url, x.budget);
|
||||
});
|
||||
return { id: live.id };
|
||||
}
|
||||
|
||||
/** 会话转"等你过验证":待处理事项写明是哪个站点、去哪里处理 */
|
||||
private async cfNeedsHuman(id: string, site: SiteRow, reason: string): Promise<ChallengeSolveOutcome> {
|
||||
const text = {
|
||||
title: `${site.name}需要你在原站视图里过一次验证`,
|
||||
summary:
|
||||
"站点要求人机验证（Cloudflare），系统没能自动通过。在站点详情的原站视图接管会话，完成验证后交还，站点请求会继续。",
|
||||
};
|
||||
let row = await this.sessionRow(id);
|
||||
if (row.controller === "automation") row = await this.enterWaiting(row, "cf_challenge", text);
|
||||
else if (row.controller === "waiting") {
|
||||
const rs = obj(row.recovery_state) as RecoveryState;
|
||||
if (!rs.waitingDecisionId) row = await this.enterWaiting(row, "cf_challenge", text);
|
||||
else {
|
||||
// 视觉子流程没成功时已转等待(challenge_unsolved):事项改成 Cloudflare 的说法
|
||||
await this.ctx.db
|
||||
.updateTable("decision")
|
||||
.set({ title: text.title, summary: text.summary })
|
||||
.where("id", "=", rs.waitingDecisionId)
|
||||
.execute();
|
||||
row = await this.patchRecovery(row, { waitingReason: "cf_challenge" });
|
||||
}
|
||||
}
|
||||
const decisionId = (obj(row.recovery_state) as RecoveryState).waitingDecisionId ?? null;
|
||||
if (decisionId)
|
||||
await this.ctx.db
|
||||
.updateTable("decision")
|
||||
.set({
|
||||
evidence: sql`evidence || ${JSON.stringify({ cloudflare: { siteId: site.id, reason } })}::jsonb`,
|
||||
})
|
||||
.where("id", "=", decisionId)
|
||||
.execute();
|
||||
this.ctx.logger.info({ component: "browser", sessionId: id, siteId: site.id, reason }, "Cloudflare 验证转人工");
|
||||
return { status: "needs_human", sessionId: id, decisionId, reason };
|
||||
}
|
||||
|
||||
/** 读回 profile 里的 cf_clearance 值(只取作用于站点主机的) */
|
||||
private async cfClearance(profileKey: string, siteUrl: string): Promise<string | null> {
|
||||
const prow = await this.providerRow().catch(() => null);
|
||||
const provider = prow ? this.providerFor(prow) : null;
|
||||
if (!provider?.getCookies) return null;
|
||||
const list = await provider.getCookies(profileKey, [siteUrl]).catch(() => []);
|
||||
const host = new URL(siteUrl).hostname;
|
||||
const hit = list.find(
|
||||
(c) =>
|
||||
c.name === "cf_clearance" &&
|
||||
(host === c.domain.replace(/^\./, "") || host.endsWith(`.${c.domain.replace(/^\./, "")}`)) &&
|
||||
!(c.expires !== undefined && c.expires > 0 && c.expires * 1000 <= this.now().getTime()),
|
||||
);
|
||||
return hit?.value ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 过挑战(CF-2 级联):用该账号 profile 打开同一地址 → ① 等它自己过(≤15 秒)→ ② 复选框 / Turnstile 派视觉子流程点一下
|
||||
* (最多 2 次,系统发起,预算取设置;自动化总开关关闭时跳过)→ ③ 选图 / 输码类或仍没过:会话转等待,开待处理事项。
|
||||
* 通过后读回 cf_clearance 与 navigator.userAgent,连同浏览器出口交给调用方。
|
||||
*/
|
||||
async solveSiteChallenge(req: { siteId: string; accountId: string; url: string }): Promise<ChallengeSolveOutcome> {
|
||||
const site = await this.liveSite(req.siteId);
|
||||
if (!site.enabled) return { status: "failed", reason: "站点已停用" };
|
||||
const profileKey = profileKeyFor(site.id, req.accountId);
|
||||
return this.cfLock.run(profileKey, async () => {
|
||||
const got = await this.cfSession(site, req.accountId, req.url);
|
||||
if ("busy" in got) {
|
||||
const rs = obj(got.busy.recovery_state) as RecoveryState;
|
||||
if (got.busy.controller === "waiting" && rs.waitingReason === "cf_challenge")
|
||||
return {
|
||||
status: "needs_human",
|
||||
sessionId: got.busy.id,
|
||||
decisionId: rs.waitingDecisionId ?? null,
|
||||
reason: "已在等你过验证",
|
||||
};
|
||||
return { status: "failed", reason: "浏览器会话正由人工处理，没有打扰" };
|
||||
}
|
||||
const id = got.id;
|
||||
const opened = await this.sessionRow(id);
|
||||
if (opened.controller !== "automation")
|
||||
return { status: "failed", reason: "浏览器会话打开后在等人处理（登录已失效）" };
|
||||
// ① 等它自己过
|
||||
let w = await this.waitChallenge(id, null, this.opts.challengeAutoWaitMs ?? CF_AUTO_WAIT_MS);
|
||||
if (!w.page) return { status: "failed", reason: "浏览器服务不支持读取页面内容" };
|
||||
let by: "auto" | "vision" = "auto";
|
||||
// ② 复选框 / Turnstile:视觉子流程点一下
|
||||
if (w.kind === "interactive") {
|
||||
const vision = visions.get(this.ctx);
|
||||
const auto = await this.ctx.settings.get("automation").catch(() => null);
|
||||
if (vision && auto?.enabled && !auto.readOnlyReconcile) {
|
||||
const marker = /verify you are human|确认您是真人|验证您是真人|我是人类/i.exec(w.page.text)?.[0] ?? null;
|
||||
for (let i = 0; i < CF_VISION_ATTEMPTS && w.kind === "interactive"; i++) {
|
||||
const r = await vision({
|
||||
sessionId: id,
|
||||
goal: "这是 Cloudflare 人机验证页。只点一次「确认您是真人 / Verify you are human」复选框（Turnstile），然后等待页面跳转；不要做其它操作。如果页面要求选图片、输入验证码或别的操作，立即停止并说明需要人工。",
|
||||
textAbsent: marker,
|
||||
maxSteps: CF_VISION_STEPS,
|
||||
}).catch((e) => ({ status: "failed", reason: String((e as Error)?.message ?? e), childRunId: null }));
|
||||
this.ctx.logger.info(
|
||||
{ component: "browser", sessionId: id, siteId: site.id, attempt: i + 1, status: r.status },
|
||||
"Cloudflare 复选框视觉子流程结束",
|
||||
);
|
||||
// 子流程没成功时会话已转等待(challenge_unsolved);下一次子流程开始时自己收回自动化控制
|
||||
w = await this.waitChallenge(
|
||||
id,
|
||||
null,
|
||||
Math.min(CF_AFTER_CLICK_WAIT_MS, this.opts.challengeAutoWaitMs ?? CF_AFTER_CLICK_WAIT_MS),
|
||||
);
|
||||
if (w.kind === "none") by = "vision";
|
||||
}
|
||||
}
|
||||
}
|
||||
// ③ 没过
|
||||
if (w.kind !== "none")
|
||||
return this.cfNeedsHuman(
|
||||
id,
|
||||
site,
|
||||
w.kind === "captcha" ? "选图 / 输入验证码类验证，不自动处理" : `没能自动通过(${w.rule ?? w.kind})`,
|
||||
);
|
||||
// 通过:子流程成功时关掉它留下的待处理;读回 cf_clearance 与 UA
|
||||
const row = await this.sessionRow(id);
|
||||
if (row.controller === "waiting") {
|
||||
await this.resolveWaiting(row, "system:cloudflare");
|
||||
const { provider } = await this.loaded(row);
|
||||
provider.setController(id, "automation");
|
||||
await this.ctx.db
|
||||
.updateTable("browser_session")
|
||||
.set({
|
||||
controller: "automation",
|
||||
recovery_state: JSON.stringify({
|
||||
...obj(row.recovery_state),
|
||||
waitingReason: null,
|
||||
waitingDecisionId: null,
|
||||
}),
|
||||
updated_at: this.now(),
|
||||
})
|
||||
.where("id", "=", id)
|
||||
.execute();
|
||||
}
|
||||
const rs = obj((await this.sessionRow(id)).recovery_state) as RecoveryState;
|
||||
return {
|
||||
status: "passed",
|
||||
by,
|
||||
clearance: await this.cfClearance(profileKey, site.base_url),
|
||||
userAgent: w.page?.userAgent ?? (await this.identityUserAgent().catch(() => null)),
|
||||
browserEgress: rs.egress ?? null,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 浏览器代抓一页(CF-2 回退):该账号 profile 的自动化会话导航到 url(取站点令牌,受 profile 锁约束),读 outerHTML 交给原解析器。
|
||||
* 站点停用、会话正由人工处理 / 等人处理、仍是挑战页时不返回页面。
|
||||
*/
|
||||
async fetchSiteHtml(req: { siteId: string; accountId: string; url: string }): Promise<BrowserFetchOutcome> {
|
||||
const site = await this.liveSite(req.siteId).catch(() => null);
|
||||
if (!site?.enabled) return { status: "unavailable", reason: "站点已停用" };
|
||||
const profileKey = profileKeyFor(site.id, req.accountId);
|
||||
try {
|
||||
return await this.cfLock.run(profileKey, async () => {
|
||||
const got = await this.cfSession(site, req.accountId, req.url);
|
||||
if ("busy" in got) return { status: "unavailable" as const, reason: "浏览器会话正由人工处理或在等人处理" };
|
||||
const w = await this.waitChallenge(got.id, null, this.opts.challengeAutoWaitMs ?? CF_AUTO_WAIT_MS);
|
||||
if (!w.page) return { status: "unavailable" as const, reason: "浏览器服务不支持读取页面内容" };
|
||||
if (w.kind !== "none") return { status: "challenge" as const, reason: w.rule ?? w.kind };
|
||||
return { status: "ok" as const, html: w.page.html, url: w.page.url };
|
||||
});
|
||||
} catch (e) {
|
||||
const err = fromBrowserError(e);
|
||||
return { status: "unavailable", reason: String((err as Error)?.message ?? err).slice(0, 300) };
|
||||
}
|
||||
}
|
||||
|
||||
/** 人过完验证交还后:站点在等人过验证、会话是那一个 → 读回 cf_clearance / UA,通知站点请求继续 */
|
||||
private async afterHumanChallenge(id: string): Promise<void> {
|
||||
const row = await this.sessionRow(id);
|
||||
if (!row.account_id) return;
|
||||
const rt = this.opts.runtime?.(this.ctx);
|
||||
if (!rt) return;
|
||||
const st = await rt.cloudflare.state(row.site_id, true);
|
||||
if (st.pending !== "human") return;
|
||||
const site = await this.liveSite(row.site_id);
|
||||
const page = await this.readPage(id).catch(() => null);
|
||||
if (page && classifyChallengePage(page).kind !== "none") return; // 还在挑战页:继续等
|
||||
const rs = obj(row.recovery_state) as RecoveryState;
|
||||
await rt.cloudflare.afterHuman(row.site_id, row.account_id, {
|
||||
clearance: await this.cfClearance(row.profile_key, site.base_url),
|
||||
userAgent: page?.userAgent ?? (await this.identityUserAgent().catch(() => null)),
|
||||
browserEgress: rs.egress ?? null,
|
||||
});
|
||||
}
|
||||
|
||||
// ---------- 下载 ----------
|
||||
|
||||
/** 暂存产物核验:解析种子、按 infohash / 站点资源 ID 匹配本地资源、会话内去重 */
|
||||
@@ -1972,6 +2280,7 @@ export class BrowserService {
|
||||
}
|
||||
|
||||
const REASON_TEXT: Record<string, string> = {
|
||||
cf_challenge: "站点要求人机验证（Cloudflare），需要你过一次",
|
||||
captcha: "登录页需要验证码",
|
||||
challenge: "页面需要人工验证",
|
||||
two_factor: "需要二次验证或本人设备确认",
|
||||
@@ -2054,7 +2363,7 @@ export function toLease(r: BrowserControlLeaseRow): BrowserControlLease {
|
||||
}
|
||||
|
||||
/** 视觉子流程可以接手的等待原因(验证码 / 页面挑战 / 上次没处理成功) */
|
||||
const SOLVABLE = ["captcha", "challenge", "challenge_unsolved"];
|
||||
const SOLVABLE = ["captcha", "challenge", "challenge_unsolved", "cf_challenge"];
|
||||
|
||||
/** capture / visionAct 的结果(能力输出的公共部分) */
|
||||
export interface VisionShot {
|
||||
|
||||
Reference in new issue
Block a user