feat(sites): 出口统一与 Cloudflare 过挑战级联(CF-2)

- 出口:站点请求不再隐式继承 HTTP(S)_PROXY,没开代理明确直连(创建客户端时临时拿掉代理环境变量,Deno 2.9 语义见 egress.ts), 开了走站点代理;TMDB / Brave / 图片改为显式按环境变量(行为不变) - CloudflareGate:浏览器代理期直接代抓 → HTTP(可复用时带 cf_clearance)→ 挑战 → 浏览器过挑战(同站 30 分钟一次、单飞) → cf_clearance 仅在 UA 跟随浏览器且两端同出口时给 HTTP,重试 1 次记 cfReuse → 仍被挑战则浏览器代抓并进入 6 小时代理期 - 浏览器:solveSiteChallenge(自动放行 ≤15s / 系统发起视觉子流程最多 2 次 / 真验证码转人工待处理)、fetchSiteHtml(profile 锁 + 站点令牌)、交还后读回 cf_clearance 继续;site.config.cloudflare 状态契约 Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-09-30 21:26:54 +08:00
1 parent 6b14b0da45
commit 16083942c0
@@ -0,0 +1,133 @@
+import { DEFAULT_SETTINGS } from "@umt/contracts";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import {
+  directHttpClient,
+  egressKey,
+  envProxyFor,
+  fetchVia,
+  noProxyMatches,
+  proxyHttpClient,
+  siteEgress,
+} from "../../src/egress.ts";
+import { SourcesRuntime } from "../../src/modules/sources-identity/runtime.ts";
+import type { ServerContext } from "../../src/modules/types.ts";
+
+// CF-2 出口统一:站点请求不再隐式继承容器的 HTTP(S)_PROXY。开了站点代理 → 该代理;否则明确直连。
+// 本地起一个"代理"(收到请求就回 via-proxy)和一个目标(回 direct),把 HTTP_PROXY 指向前者,看请求实际走哪条。
+
+const PROXY_KEYS = [
+  "HTTP_PROXY",
+  "HTTPS_PROXY",
+  "ALL_PROXY",
+  "http_proxy",
+  "https_proxy",
+  "all_proxy",
+  "NO_PROXY",
+  "no_proxy",
+];
+
+describe("出口选择(直连 / 代理)", () => {
+  let proxy: Deno.HttpServer<Deno.NetAddr>;
+  let target: Deno.HttpServer<Deno.NetAddr>;
+  let proxyUrl = "";
+  let targetUrl = "";
+  const proxied: string[] = [];
+  const saved = new Map<string, string | undefined>();
+
+  beforeAll(() => {
+    proxy = Deno.serve({ hostname: "127.0.0.1", port: 0, onListen() {} }, (req) => {
+      proxied.push(req.url);
+      return new Response("via-proxy");
+    });
+    target = Deno.serve({ hostname: "127.0.0.1", port: 0, onListen() {} }, () => new Response("direct"));
+    proxyUrl = `http://127.0.0.1:${proxy.addr.port}`;
+    targetUrl = `http://127.0.0.1:${target.addr.port}/x`;
+    for (const k of PROXY_KEYS) {
+      saved.set(k, Deno.env.get(k));
+      Deno.env.delete(k);
+    }
+    // 模拟容器环境:HTTP_PROXY 指向"中继",没有 NO_PROXY
+    Deno.env.set("HTTP_PROXY", proxyUrl);
+  });
+
+  afterAll(async () => {
+    for (const [k, v] of saved) {
+      if (v === undefined) Deno.env.delete(k);
+      else Deno.env.set(k, v);
+    }
+    await proxy.shutdown();
+    await target.shutdown();
+  });
+
+  const body = async (f: typeof fetch) => (await f(targetUrl)).text();
+
+  it("Deno 2.9 语义:createHttpClient({}) 仍会读 HTTP_PROXY(所以直连要显式做)", async () => {
+    const implicit = Deno.createHttpClient({});
+    try {
+      expect(await body(fetchVia(implicit))).toBe("via-proxy");
+    } finally {
+      implicit.close();
+    }
+  });
+
+  it("directHttpClient:环境变量里有代理也直连;创建后环境变量原样还原", async () => {
+    proxied.length = 0;
+    expect(await body(fetchVia(directHttpClient()))).toBe("direct");
+    expect(proxied).toEqual([]);
+    expect(Deno.env.get("HTTP_PROXY")).toBe(proxyUrl);
+  });
+
+  it("proxyHttpClient:只走给定代理(NO_PROXY 命中也不绕过)", async () => {
+    Deno.env.set("NO_PROXY", "127.0.0.1");
+    try {
+      proxied.length = 0;
+      const client = proxyHttpClient(proxyUrl);
+      expect(await body(fetchVia(client))).toBe("via-proxy");
+      expect(proxied).toHaveLength(1);
+      expect(Deno.env.get("NO_PROXY")).toBe("127.0.0.1");
+    } finally {
+      Deno.env.delete("NO_PROXY");
+    }
+  });
+
+  it("SourcesRuntime.siteFetch:站点没开代理 → 直连;开了且设了代理地址 → 站点代理", async () => {
+    const ctx = { logger: { debug() {}, info() {}, warn() {} } } as unknown as ServerContext;
+    const rt = new SourcesRuntime(ctx);
+    const defaults = { ...DEFAULT_SETTINGS.sites, proxyUrl };
+    proxied.length = 0;
+    expect(await body(rt.siteFetch({ proxy: false }, defaults, "UA"))).toBe("direct");
+    expect(proxied).toEqual([]);
+    expect(await body(rt.siteFetch({ proxy: true }, defaults, "UA"))).toBe("via-proxy");
+    expect(proxied).toHaveLength(1);
+    // 开了代理但没设代理地址 → 直连
+    expect(await body(rt.siteFetch({ proxy: true }, { ...defaults, proxyUrl: null }, "UA"))).toBe("direct");
+    expect(rt.egressKeyOf({ proxy: false }, defaults)).toBe("direct");
+    expect(rt.egressKeyOf({ proxy: true }, defaults)).toBe(`proxy:http://127.0.0.1:${proxy.addr.port}`);
+  });
+});
+
+describe("外网请求按环境变量(行为不变,改为显式)", () => {
+  it("HTTPS / HTTP / ALL_PROXY 与 NO_PROXY", () => {
+    const env = { HTTPS_PROXY: "http://relay:7891", HTTP_PROXY: "http://relay:7890", NO_PROXY: "localhost,.lan" };
+    expect(envProxyFor(new URL("https://api.themoviedb.org/3"), env)).toBe("http://relay:7891");
+    expect(envProxyFor(new URL("http://image.tmdb.org/t"), env)).toBe("http://relay:7890");
+    expect(envProxyFor(new URL("http://nas.lan/x"), env)).toBeNull();
+    expect(envProxyFor(new URL("https://api.search.brave.com/"), { ALL_PROXY: "socks5h://relay:1080" })).toBe(
+      "socks5h://relay:1080",
+    );
+    expect(envProxyFor(new URL("https://api.search.brave.com/"), {})).toBeNull();
+    expect(envProxyFor(new URL("https://x.example/"), { https_proxy: "http://lower:1" })).toBe("http://lower:1");
+  });
+  it("NO_PROXY 匹配", () => {
+    expect(noProxyMatches("a.example.com", "example.com")).toBe(true);
+    expect(noProxyMatches("example.com", ".example.com")).toBe(true);
+    expect(noProxyMatches("badexample.com", "example.com")).toBe(false);
+    expect(noProxyMatches("x", "*")).toBe(true);
+    expect(noProxyMatches("127.0.0.1", "localhost, 127.0.0.1:8080")).toBe(true);
+  });
+  it("出口键不含代理口令", () => {
+    expect(egressKey(siteEgress(false, "http://u:p@relay:7891"))).toBe("direct");
+    expect(egressKey(siteEgress(true, "http://u:p@relay:7891"))).toBe("proxy:http://relay:7891");
+    expect(egressKey(siteEgress(true, null))).toBe("direct");
+  });
+});