feat(sites): 出口统一与 Cloudflare 过挑战级联(CF-2)

- 出口:站点请求不再隐式继承 HTTP(S)_PROXY,没开代理明确直连(创建客户端时临时拿掉代理环境变量,Deno 2.9 语义见 egress.ts),
  开了走站点代理;TMDB / Brave / 图片改为显式按环境变量(行为不变)
- CloudflareGate:浏览器代理期直接代抓 → HTTP(可复用时带 cf_clearance)→ 挑战 → 浏览器过挑战(同站 30 分钟一次、单飞)
  → cf_clearance 仅在 UA 跟随浏览器且两端同出口时给 HTTP,重试 1 次记 cfReuse → 仍被挑战则浏览器代抓并进入 6 小时代理期
- 浏览器:solveSiteChallenge(自动放行 ≤15s / 系统发起视觉子流程最多 2 次 / 真验证码转人工待处理)、fetchSiteHtml(profile 锁 + 站点令牌)、
  交还后读回 cf_clearance 继续;site.config.cloudflare 状态契约

Co-Authored-By: Claude Opus 5.5 <[email protected]>

apps/server/src/modules/browser/service.ts

@@ -5,6 +5,7 @@ import {
   type BrowserProvider,
   type BrowserSecretResolver,
   CAMOFOX_VERIFIED_RELEASES,
+  classifyChallengePage,
   cookiesFromHeader,
   createCamofoxBrowserProvider,
   type DownloadSink,
@@ -12,7 +13,9 @@ import {
   getSiteDefinition,
   headerFromCookies,
   hostAllowed,
+  InMemoryProfileLock,
   imageDimensions,
+  type PageContent,
   type PageMetrics,
   parseTorrent,
   prepareImage,
@@ -50,8 +53,14 @@ import {
   type SiteRow,
   sql,
 } from "@umt/db";
+import { egressKey, siteEgress } from "../../egress.ts";
 import type { SourcesRuntime } from "../sources-identity/runtime.ts";
 import { refreshSiteAccount } from "../sources-identity/service/account.ts";
+import {
+  type BrowserFetchOutcome,
+  type ChallengeSolveOutcome,
+  registerChallengeBrowser,
+} from "../sources-identity/service/cloudflare.ts";
 import {
   readAccountCookie,
   registerBrowserCookieReader,
@@ -85,6 +94,10 @@ export interface BrowserModuleOptions {
   evidenceDir?: string;
   /** 坐标动作之后等页面稳定再截图的毫秒数;缺省 400 */
   settleMs?: number;
+  /** CF-2 等挑战页放行时读页面的间隔(毫秒);缺省 1500 */
+  challengePollMs?: number;
+  /** CF-2 等 Cloudflare 自己放行的最长时间(毫秒);缺省 CF_AUTO_WAIT_MS(15 秒) */
+  challengeAutoWaitMs?: number;
   /** 保活间隔(AG-4);缺省 BROWSER_KEEPALIVE_MS 配置(120 秒),0 = 关闭 */
   keepaliveMs?: number;
   /** 测试替换 provider 实现 */
@@ -175,8 +188,39 @@ export interface RecoveryState {
   reconnect?: BrowserReconnectState;
   /** 最近一次自动化(Agent 能力)动作成功的时间;原站视图据此区分「Agent 正在操作」与「无人操作」 */
   lastAutomationAt?: string;
+  /** CF-2:打开 / 重连时下发给浏览器的出口键(egressKey);插件没按站点设出口(旧镜像 / env 模式)为 null */
+  egress?: string | null;
 }
 
+/**
+ * CF-2 视觉子流程钩子(agent 模块注册):系统发起、没有父会话,预算取设置;只点「我是人类」复选框 / Turnstile 一步。
+ * 返回子流程结论(solved 仍要由调用方按页面核实)。
+ */
+export type ChallengeVision = (req: {
+  sessionId: string;
+  goal: string;
+  /** 通过后页面上不再出现的文字(成功条件) */
+  textAbsent: string | null;
+  maxSteps: number;
+}) => Promise<{ status: string; reason: string; childRunId: string | null }>;
+
+const visions = new WeakMap<object, ChallengeVision>();
+
+export function registerChallengeVision(ctx: object, fn: ChallengeVision): void {
+  visions.set(ctx, fn);
+}
+
+/** 系统(Cloudflare 处理)发起的浏览器动作 */
+const CF_ACTOR: Actor = { kind: "automation", holder: "system:cloudflare" };
+/** 等 Cloudflare 自己放行的最长时间 */
+export const CF_AUTO_WAIT_MS = 15_000;
+/** 视觉子流程点过之后再等多久 */
+const CF_AFTER_CLICK_WAIT_MS = 10_000;
+/** 视觉子流程最多派几次 */
+const CF_VISION_ATTEMPTS = 2;
+/** 视觉子流程步数上限:截图 → 点击 → 等待 */
+const CF_VISION_STEPS = 4;
+
 /** 同一浏览器服务的实例标识多久查一次 */
 const INSTANCE_CHECK_MS = 5_000;
 /** 自动重连失败后,操作触发的重连至少间隔多久 */
@@ -221,6 +265,11 @@ export class BrowserService {
     registerBrowserCookieReader(ctx, (siteId, accountId) => this.readProfileCookie(siteId, accountId));
     // 站点请求 UA 跟随浏览器固定身份(BR-4):sources-identity 经钩子取
     registerBrowserIdentityReader(ctx, () => this.identityUserAgent());
+    // CF-2:站点 HTTP 遇 Cloudflare 挑战时由浏览器过挑战 / 代抓页面
+    registerChallengeBrowser(ctx, {
+      solve: (req) => this.solveSiteChallenge(req),
+      fetchHtml: (req) => this.fetchSiteHtml(req),
+    });
   }
 
   // ---------- BR-4 浏览器身份与出口 ----------
@@ -283,8 +332,8 @@ export class BrowserService {
    * 出口按站点统一(BR-4):站点开了代理且设置了代理地址 → 该 profile 走同一代理;否则直连。
    * 旧镜像(插件没有出口接口)只记日志,浏览器照旧走容器环境变量里的代理。
    */
-  private async applyEgress(provider: BrowserProvider, site: SiteRow, profileKey: string): Promise<void> {
-    if (!provider.setEgress) return;
+  private async applyEgress(provider: BrowserProvider, site: SiteRow, profileKey: string): Promise<string | null> {
+    if (!provider.setEgress) return null;
     const rt = this.opts.runtime?.(this.ctx);
     const proxyUrl = rt
       ? (await rt.siteDefaults()).proxyUrl
@@ -297,6 +346,8 @@ export class BrowserService {
           { component: "browser", siteId: site.id, egress: proxy ? "proxy" : "direct" },
           "浏览器会话出口已变更，按新出口重建",
         );
+      // 插件 env 模式:浏览器按容器环境变量出网,与站点 HTTP(明确直连 / 站点代理)不一定一致
+      return r.mode && r.mode !== "site" ? null : egressKey(siteEgress(site.proxy, proxyUrl));
     } catch (e) {
       this.ctx.logger.warn(
         {
@@ -307,6 +358,7 @@ export class BrowserService {
         },
         "设置浏览器会话出口失败，浏览器按容器环境变量出网",
       );
+      return null;
     }
   }
 
@@ -851,7 +903,7 @@ export class BrowserService {
     for (const m of await provider.listSessions())
       if (m.profileKey === profileKey && m.controller !== "closed") provider.setController(m.id, "closed");
     const prep = await this.prepareCookies(site, accountId);
-    await this.applyEgress(provider, site, profileKey);
+    const egress = await this.applyEgress(provider, site, profileKey);
     await this.refreshIdentity(prow, { maxAgeMs: 10 * 60_000 }).catch(() => null);
     await this.rateLimit(site);
     let res: Awaited<ReturnType<BrowserProvider["openSession"]>>;
@@ -904,6 +956,7 @@ export class BrowserService {
             needsSnapshot: true,
             credential,
             instanceId,
+            egress,
           }),
         })
         .execute();
@@ -1141,7 +1194,11 @@ export class BrowserService {
   }
 
   /** 转等待处理:自动化动作被拒,生成待处理事项,等人在原站视图接管 */
-  async enterWaiting(row: BrowserSessionRow, reason: string): Promise<BrowserSessionRow> {
+  async enterWaiting(
+    row: BrowserSessionRow,
+    reason: string,
+    text?: { title: string; summary: string },
+  ): Promise<BrowserSessionRow> {
     const provider = this.providers.get(row.provider_id)?.provider;
     if (provider?.sessionInfo(row.id)) provider.setController(row.id, "waiting");
     const site = await this.liveSite(row.site_id).catch(() => null);
@@ -1154,8 +1211,8 @@ export class BrowserService {
           kind: "other",
           status: "awaiting_user",
           severity: "warning",
-          title: `${site?.name ?? "站点"}：原站需要人工处理`,
-          summary: `${REASON_TEXT[reason] ?? reason}。在站点详情的原站视图接管会话，完成后交还。`,
+          title: text?.title ?? `${site?.name ?? "站点"}：原站需要人工处理`,
+          summary: text?.summary ?? `${REASON_TEXT[reason] ?? reason}。在站点详情的原站视图接管会话，完成后交还。`,
           subject_type: "browser_session",
           subject_id: row.id,
           dedupe_key: `browser.waiting:${row.id}`,
@@ -1327,6 +1384,14 @@ export class BrowserService {
     await this.persist(id, provider);
     // 人工在桌面 / 截图模式里登录过:交还时读回登录态写进站点账号,全站(HTTP 抓取 / RSS / 快照)生效
     await this.syncCookies(id, "release");
+    // CF-2:等人过 Cloudflare 验证的会话交还后,读回 cf_clearance / UA,站点请求继续
+    if (opts.to === "automation")
+      await this.afterHumanChallenge(id).catch((e) =>
+        this.ctx.logger.warn(
+          { component: "browser", sessionId: id, err: String((e as Error)?.message ?? e) },
+          "交还后读回 Cloudflare 验证结果失败",
+        ),
+      );
     return toLease(released);
   }
 
@@ -1586,7 +1651,7 @@ export class BrowserService {
     if (!provider.reopenSession) throw await fail("浏览器服务不支持重新连接");
     const url = restoreUrl(row.current_url, budget) ?? site.base_url;
     const prep = await this.prepareCookies(site, row.account_id).catch(() => null);
-    await this.applyEgress(provider, site, row.profile_key);
+    const egress = await this.applyEgress(provider, site, row.profile_key);
     // 浏览器重启过(可能换了镜像版本):身份重新读
     await this.refreshIdentity(prow).catch(() => null);
     let res: Awaited<ReturnType<NonNullable<BrowserProvider["reopenSession"]>>>;
@@ -1606,6 +1671,7 @@ export class BrowserService {
       needsReconnect: false,
       needsSnapshot: true,
       instanceId: instanceId ?? rs.instanceId ?? null,
+      egress,
       reconnect: { status: "ok", trigger, at, reason: null, humanNotice: row.controller === "human" },
     });
     this.coordSpaces.delete(id);
@@ -1894,6 +1960,248 @@ export class BrowserService {
     return { decisionId, controller: row.controller };
   }
 
+  // ---------- CF-2 Cloudflare 挑战 ----------
+
+  /** 过挑战 / 代抓同一 profile 串行(本进程;跨进程的同站互斥靠 site.config.cloudflare.lastAttemptAt 条件更新) */
+  private readonly cfLock = new InMemoryProfileLock(240_000);
+
+  /** 读当前页面内容(只读,不要求自动化控制权);上游不支持为 null */
+  private async readPage(id: string): Promise<PageContent | null> {
+    return this.run(id, CF_ACTOR, (p, ref) => (p.pageContent ? p.pageContent(ref) : Promise.resolve(null)), {
+      write: false,
+      persist: false,
+      background: true,
+    });
+  }
+
+  /** 轮询页面直到不是挑战页(或变成真验证码 / 超时);返回最后一次读到的页面与分类 */
+  private async waitChallenge(
+    id: string,
+    page: PageContent | null,
+    ms: number,
+  ): Promise<{
+    page: PageContent | null;
+    kind: ReturnType<typeof classifyChallengePage>["kind"];
+    rule: string | null;
+  }> {
+    let cur = page ?? (await this.readPage(id));
+    let c = classifyChallengePage(cur ?? {});
+    const deadline = Date.now() + ms;
+    while (cur && c.kind !== "none" && c.kind !== "captcha" && Date.now() < deadline) {
+      await new Promise((r) => setTimeout(r, this.opts.challengePollMs ?? 1_500));
+      cur = await this.readPage(id);
+      c = classifyChallengePage(cur ?? {});
+    }
+    return { page: cur, kind: cur ? c.kind : "none", rule: c.rule };
+  }
+
+  /**
+   * 该账号 profile 的自动化会话:已有且在自动化控制下就用它(导航到 url,取站点令牌);没有就打开一个(打开即导航)。
+   * 人工接管中 / 等人处理中返回 null(不打扰人)。
+   */
+  private async cfSession(
+    site: SiteRow,
+    accountId: string,
+    url: string,
+  ): Promise<{ id: string } | { busy: BrowserSessionRow }> {
+    const live = await this.liveSessionFor(site.id, accountId);
+    if (live && live.controller !== "automation") return { busy: live };
+    if (!live) return { id: await this.open({ siteId: site.id, accountId, url }) };
+    await this.run(live.id, CF_ACTOR, async (p, ref, x) => {
+      await this.rateLimit(x.site);
+      await p.navigate(ref, url, x.budget);
+    });
+    return { id: live.id };
+  }
+
+  /** 会话转"等你过验证":待处理事项写明是哪个站点、去哪里处理 */
+  private async cfNeedsHuman(id: string, site: SiteRow, reason: string): Promise<ChallengeSolveOutcome> {
+    const text = {
+      title: `${site.name}需要你在原站视图里过一次验证`,
+      summary:
+        "站点要求人机验证（Cloudflare），系统没能自动通过。在站点详情的原站视图接管会话，完成验证后交还，站点请求会继续。",
+    };
+    let row = await this.sessionRow(id);
+    if (row.controller === "automation") row = await this.enterWaiting(row, "cf_challenge", text);
+    else if (row.controller === "waiting") {
+      const rs = obj(row.recovery_state) as RecoveryState;
+      if (!rs.waitingDecisionId) row = await this.enterWaiting(row, "cf_challenge", text);
+      else {
+        // 视觉子流程没成功时已转等待(challenge_unsolved):事项改成 Cloudflare 的说法
+        await this.ctx.db
+          .updateTable("decision")
+          .set({ title: text.title, summary: text.summary })
+          .where("id", "=", rs.waitingDecisionId)
+          .execute();
+        row = await this.patchRecovery(row, { waitingReason: "cf_challenge" });
+      }
+    }
+    const decisionId = (obj(row.recovery_state) as RecoveryState).waitingDecisionId ?? null;
+    if (decisionId)
+      await this.ctx.db
+        .updateTable("decision")
+        .set({
+          evidence: sql`evidence || ${JSON.stringify({ cloudflare: { siteId: site.id, reason } })}::jsonb`,
+        })
+        .where("id", "=", decisionId)
+        .execute();
+    this.ctx.logger.info({ component: "browser", sessionId: id, siteId: site.id, reason }, "Cloudflare 验证转人工");
+    return { status: "needs_human", sessionId: id, decisionId, reason };
+  }
+
+  /** 读回 profile 里的 cf_clearance 值(只取作用于站点主机的) */
+  private async cfClearance(profileKey: string, siteUrl: string): Promise<string | null> {
+    const prow = await this.providerRow().catch(() => null);
+    const provider = prow ? this.providerFor(prow) : null;
+    if (!provider?.getCookies) return null;
+    const list = await provider.getCookies(profileKey, [siteUrl]).catch(() => []);
+    const host = new URL(siteUrl).hostname;
+    const hit = list.find(
+      (c) =>
+        c.name === "cf_clearance" &&
+        (host === c.domain.replace(/^\./, "") || host.endsWith(`.${c.domain.replace(/^\./, "")}`)) &&
+        !(c.expires !== undefined && c.expires > 0 && c.expires * 1000 <= this.now().getTime()),
+    );
+    return hit?.value ?? null;
+  }
+
+  /**
+   * 过挑战(CF-2 级联):用该账号 profile 打开同一地址 → ① 等它自己过(≤15 秒)→ ② 复选框 / Turnstile 派视觉子流程点一下
+   * (最多 2 次,系统发起,预算取设置;自动化总开关关闭时跳过)→ ③ 选图 / 输码类或仍没过:会话转等待,开待处理事项。
+   * 通过后读回 cf_clearance 与 navigator.userAgent,连同浏览器出口交给调用方。
+   */
+  async solveSiteChallenge(req: { siteId: string; accountId: string; url: string }): Promise<ChallengeSolveOutcome> {
+    const site = await this.liveSite(req.siteId);
+    if (!site.enabled) return { status: "failed", reason: "站点已停用" };
+    const profileKey = profileKeyFor(site.id, req.accountId);
+    return this.cfLock.run(profileKey, async () => {
+      const got = await this.cfSession(site, req.accountId, req.url);
+      if ("busy" in got) {
+        const rs = obj(got.busy.recovery_state) as RecoveryState;
+        if (got.busy.controller === "waiting" && rs.waitingReason === "cf_challenge")
+          return {
+            status: "needs_human",
+            sessionId: got.busy.id,
+            decisionId: rs.waitingDecisionId ?? null,
+            reason: "已在等你过验证",
+          };
+        return { status: "failed", reason: "浏览器会话正由人工处理，没有打扰" };
+      }
+      const id = got.id;
+      const opened = await this.sessionRow(id);
+      if (opened.controller !== "automation")
+        return { status: "failed", reason: "浏览器会话打开后在等人处理（登录已失效）" };
+      // ① 等它自己过
+      let w = await this.waitChallenge(id, null, this.opts.challengeAutoWaitMs ?? CF_AUTO_WAIT_MS);
+      if (!w.page) return { status: "failed", reason: "浏览器服务不支持读取页面内容" };
+      let by: "auto" | "vision" = "auto";
+      // ② 复选框 / Turnstile:视觉子流程点一下
+      if (w.kind === "interactive") {
+        const vision = visions.get(this.ctx);
+        const auto = await this.ctx.settings.get("automation").catch(() => null);
+        if (vision && auto?.enabled && !auto.readOnlyReconcile) {
+          const marker = /verify you are human|确认您是真人|验证您是真人|我是人类/i.exec(w.page.text)?.[0] ?? null;
+          for (let i = 0; i < CF_VISION_ATTEMPTS && w.kind === "interactive"; i++) {
+            const r = await vision({
+              sessionId: id,
+              goal: "这是 Cloudflare 人机验证页。只点一次「确认您是真人 / Verify you are human」复选框（Turnstile），然后等待页面跳转；不要做其它操作。如果页面要求选图片、输入验证码或别的操作，立即停止并说明需要人工。",
+              textAbsent: marker,
+              maxSteps: CF_VISION_STEPS,
+            }).catch((e) => ({ status: "failed", reason: String((e as Error)?.message ?? e), childRunId: null }));
+            this.ctx.logger.info(
+              { component: "browser", sessionId: id, siteId: site.id, attempt: i + 1, status: r.status },
+              "Cloudflare 复选框视觉子流程结束",
+            );
+            // 子流程没成功时会话已转等待(challenge_unsolved);下一次子流程开始时自己收回自动化控制
+            w = await this.waitChallenge(
+              id,
+              null,
+              Math.min(CF_AFTER_CLICK_WAIT_MS, this.opts.challengeAutoWaitMs ?? CF_AFTER_CLICK_WAIT_MS),
+            );
+            if (w.kind === "none") by = "vision";
+          }
+        }
+      }
+      // ③ 没过
+      if (w.kind !== "none")
+        return this.cfNeedsHuman(
+          id,
+          site,
+          w.kind === "captcha" ? "选图 / 输入验证码类验证，不自动处理" : `没能自动通过(${w.rule ?? w.kind})`,
+        );
+      // 通过:子流程成功时关掉它留下的待处理;读回 cf_clearance 与 UA
+      const row = await this.sessionRow(id);
+      if (row.controller === "waiting") {
+        await this.resolveWaiting(row, "system:cloudflare");
+        const { provider } = await this.loaded(row);
+        provider.setController(id, "automation");
+        await this.ctx.db
+          .updateTable("browser_session")
+          .set({
+            controller: "automation",
+            recovery_state: JSON.stringify({
+              ...obj(row.recovery_state),
+              waitingReason: null,
+              waitingDecisionId: null,
+            }),
+            updated_at: this.now(),
+          })
+          .where("id", "=", id)
+          .execute();
+      }
+      const rs = obj((await this.sessionRow(id)).recovery_state) as RecoveryState;
+      return {
+        status: "passed",
+        by,
+        clearance: await this.cfClearance(profileKey, site.base_url),
+        userAgent: w.page?.userAgent ?? (await this.identityUserAgent().catch(() => null)),
+        browserEgress: rs.egress ?? null,
+      };
+    });
+  }
+
+  /**
+   * 浏览器代抓一页(CF-2 回退):该账号 profile 的自动化会话导航到 url(取站点令牌,受 profile 锁约束),读 outerHTML 交给原解析器。
+   * 站点停用、会话正由人工处理 / 等人处理、仍是挑战页时不返回页面。
+   */
+  async fetchSiteHtml(req: { siteId: string; accountId: string; url: string }): Promise<BrowserFetchOutcome> {
+    const site = await this.liveSite(req.siteId).catch(() => null);
+    if (!site?.enabled) return { status: "unavailable", reason: "站点已停用" };
+    const profileKey = profileKeyFor(site.id, req.accountId);
+    try {
+      return await this.cfLock.run(profileKey, async () => {
+        const got = await this.cfSession(site, req.accountId, req.url);
+        if ("busy" in got) return { status: "unavailable" as const, reason: "浏览器会话正由人工处理或在等人处理" };
+        const w = await this.waitChallenge(got.id, null, this.opts.challengeAutoWaitMs ?? CF_AUTO_WAIT_MS);
+        if (!w.page) return { status: "unavailable" as const, reason: "浏览器服务不支持读取页面内容" };
+        if (w.kind !== "none") return { status: "challenge" as const, reason: w.rule ?? w.kind };
+        return { status: "ok" as const, html: w.page.html, url: w.page.url };
+      });
+    } catch (e) {
+      const err = fromBrowserError(e);
+      return { status: "unavailable", reason: String((err as Error)?.message ?? err).slice(0, 300) };
+    }
+  }
+
+  /** 人过完验证交还后:站点在等人过验证、会话是那一个 → 读回 cf_clearance / UA,通知站点请求继续 */
+  private async afterHumanChallenge(id: string): Promise<void> {
+    const row = await this.sessionRow(id);
+    if (!row.account_id) return;
+    const rt = this.opts.runtime?.(this.ctx);
+    if (!rt) return;
+    const st = await rt.cloudflare.state(row.site_id, true);
+    if (st.pending !== "human") return;
+    const site = await this.liveSite(row.site_id);
+    const page = await this.readPage(id).catch(() => null);
+    if (page && classifyChallengePage(page).kind !== "none") return; // 还在挑战页:继续等
+    const rs = obj(row.recovery_state) as RecoveryState;
+    await rt.cloudflare.afterHuman(row.site_id, row.account_id, {
+      clearance: await this.cfClearance(row.profile_key, site.base_url),
+      userAgent: page?.userAgent ?? (await this.identityUserAgent().catch(() => null)),
+      browserEgress: rs.egress ?? null,
+    });
+  }
+
   // ---------- 下载 ----------
 
   /** 暂存产物核验:解析种子、按 infohash / 站点资源 ID 匹配本地资源、会话内去重 */
@@ -1972,6 +2280,7 @@ export class BrowserService {
 }
 
 const REASON_TEXT: Record<string, string> = {
+  cf_challenge: "站点要求人机验证（Cloudflare），需要你过一次",
   captcha: "登录页需要验证码",
   challenge: "页面需要人工验证",
   two_factor: "需要二次验证或本人设备确认",
@@ -2054,7 +2363,7 @@ export function toLease(r: BrowserControlLeaseRow): BrowserControlLease {
 }
 
 /** 视觉子流程可以接手的等待原因(验证码 / 页面挑战 / 上次没处理成功) */
-const SOLVABLE = ["captcha", "challenge", "challenge_unsolved"];
+const SOLVABLE = ["captcha", "challenge", "challenge_unsolved", "cf_challenge"];
 
 /** capture / visionAct 的结果(能力输出的公共部分) */
 export interface VisionShot {