feat(sites): 出口统一与 Cloudflare 过挑战级联(CF-2)
- 出口:站点请求不再隐式继承 HTTP(S)_PROXY,没开代理明确直连(创建客户端时临时拿掉代理环境变量,Deno 2.9 语义见 egress.ts), 开了走站点代理;TMDB / Brave / 图片改为显式按环境变量(行为不变) - CloudflareGate:浏览器代理期直接代抓 → HTTP(可复用时带 cf_clearance)→ 挑战 → 浏览器过挑战(同站 30 分钟一次、单飞) → cf_clearance 仅在 UA 跟随浏览器且两端同出口时给 HTTP,重试 1 次记 cfReuse → 仍被挑战则浏览器代抓并进入 6 小时代理期 - 浏览器:solveSiteChallenge(自动放行 ≤15s / 系统发起视觉子流程最多 2 次 / 真验证码转人工待处理)、fetchSiteHtml(profile 锁 + 站点令牌)、 交还后读回 cf_clearance 继续;site.config.cloudflare 状态契约 Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
6b14b0da45
commit
16083942c0
14 files changed
+1711
-43
No files matched your search
@@ -15,6 +15,7 @@ import {
|
||||
} from "@umt/adapters";
|
||||
import { DEFAULT_SETTINGS, type SitesSettings } from "@umt/contracts";
|
||||
import { pgTokenBucketFor, type SiteRow, sql } from "@umt/db";
|
||||
import { egressKey, externalFetch, siteEgress, siteFetchFor } from "../../egress.ts";
|
||||
import {
|
||||
CACHE_REGIONS,
|
||||
renewTrackedTmdbDetail,
|
||||
@@ -24,6 +25,7 @@ import {
|
||||
tmdbDetailRef,
|
||||
} from "../../shared-cache.ts";
|
||||
import type { ServerContext } from "../types.ts";
|
||||
import { CloudflareGate, withClearance } from "./service/cloudflare.ts";
|
||||
import { readBrowserUserAgent, resolveUserAgent, type UserAgentDefaults } from "./service/user-agent.ts";
|
||||
|
||||
// 模块运行时依赖:站点适配器与限流器(每站一个实例,令牌在 PG 跨进程共享)、RSS 抓取器、TMDB 客户端、时钟。
|
||||
@@ -58,12 +60,15 @@ export class SourcesRuntime {
|
||||
private siteDefaultsCache: SitesSettings = DEFAULT_SETTINGS.sites;
|
||||
private siteDefaultsAt = 0;
|
||||
private tmdbSig: string | null = null;
|
||||
/** CF-2 Cloudflare 挑战处理(站点请求经 siteFetch 进来) */
|
||||
readonly cloudflare: CloudflareGate;
|
||||
|
||||
constructor(
|
||||
readonly ctx: ServerContext,
|
||||
readonly overrides: RuntimeOverrides = {},
|
||||
) {
|
||||
this.now = overrides.now ?? (() => new Date());
|
||||
this.cloudflare = new CloudflareGate(this);
|
||||
this.rss = createRssFetcher({
|
||||
...(overrides.fetch ? { fetch: overrides.fetch } : {}),
|
||||
now: this.now,
|
||||
@@ -101,7 +106,7 @@ export class SourcesRuntime {
|
||||
const { cfg } = await this.tmdbConfig();
|
||||
if (!cfg) return null;
|
||||
let n = 0;
|
||||
const base = cfg.fetch ?? globalThis.fetch;
|
||||
const base = cfg.fetch ?? externalFetch;
|
||||
const counted: typeof fetch = (input, init) => {
|
||||
n++;
|
||||
return base(input, init);
|
||||
@@ -144,7 +149,8 @@ export class SourcesRuntime {
|
||||
...(apiKey ? { apiKey } : {}),
|
||||
...(accessToken ? { accessToken } : {}),
|
||||
...(baseUrl ? { baseUrl } : {}),
|
||||
...(this.overrides.fetch ? { fetch: this.overrides.fetch } : {}),
|
||||
// 外网请求:显式按环境变量走代理(CF-2 出口表;测试注入优先)
|
||||
fetch: this.overrides.fetch ?? externalFetch,
|
||||
// 请求响应进 PG 缓存(region tmdb.request),三进程共享;TTL / 条数上限 / 空结果短 TTL 取设置(对照 MP)
|
||||
cache: {
|
||||
ttlMs: policy.ttl * 1000,
|
||||
@@ -168,40 +174,53 @@ export class SourcesRuntime {
|
||||
}
|
||||
|
||||
/**
|
||||
* 站点请求用的 fetch:测试注入优先;站点开了"使用代理"且设置了 proxyUrl 时走代理(Deno.createHttpClient)。
|
||||
* 站点请求用的 fetch:测试注入优先;站点开了"使用代理"且设置了 proxyUrl 时走代理,否则**明确直连**
|
||||
* (CF-2:不再隐式继承容器的 HTTP(S)_PROXY,见 egress.ts)。
|
||||
* BR-4:每条请求都带 UA(请求自己设了就不动;否则 ua 参数,缺省按"跟随浏览器"),不再发 Deno 缺省的 `Deno/x.y.z`。
|
||||
* ua 可以是函数:每次请求现场取(站点适配器按当前账号取,账号切换 UA 后立即生效)。
|
||||
* CF-2:给了 site.id 的请求经 Cloudflare 处理(浏览器代理期直接走浏览器、带可复用的 cf_clearance、遇挑战过挑战 / 重试 / 浏览器代抓)。
|
||||
*/
|
||||
siteFetch(
|
||||
site: Pick<SiteRow, "proxy">,
|
||||
site: Pick<SiteRow, "proxy"> & Partial<Pick<SiteRow, "id">>,
|
||||
defaults: SitesSettings = this.siteDefaultsCache,
|
||||
ua?: string | (() => Promise<string>),
|
||||
): typeof globalThis.fetch {
|
||||
const base = this.transportFetch(site, defaults);
|
||||
return ((input: RequestInfo | URL, init?: RequestInit) => {
|
||||
const egress = this.egressKeyOf(site, defaults);
|
||||
const siteId = site.id;
|
||||
return (async (input: RequestInfo | URL, init?: RequestInit) => {
|
||||
const headers = new Headers(init?.headers ?? (input instanceof Request ? input.headers : undefined));
|
||||
if (headers.has("user-agent")) return base(input, { ...init, headers });
|
||||
const pick = typeof ua === "function" ? ua() : ua !== undefined ? Promise.resolve(ua) : this.defaultUserAgent();
|
||||
return pick.then((v) => {
|
||||
headers.set("user-agent", v);
|
||||
return base(input, { ...init, headers });
|
||||
});
|
||||
if (!headers.has("user-agent")) {
|
||||
const pick = typeof ua === "function" ? ua() : ua !== undefined ? Promise.resolve(ua) : this.defaultUserAgent();
|
||||
headers.set("user-agent", await pick);
|
||||
}
|
||||
if (!siteId) return base(input, { ...init, headers });
|
||||
const url = input instanceof Request ? input.url : String(input);
|
||||
const method = init?.method ?? (input instanceof Request ? input.method : "GET");
|
||||
return this.cloudflare.handle(
|
||||
siteId,
|
||||
{ url, method, accept: headers.get("accept"), userAgent: headers.get("user-agent"), egress },
|
||||
(extra) => {
|
||||
const h = new Headers(headers);
|
||||
const cookie = withClearance(h.get("cookie"), extra);
|
||||
if (cookie) h.set("cookie", cookie);
|
||||
// 过挑战可能超过单次请求的超时:重试时原信号已中止就换一个新的超时(调用方取消照样生效在级联之外)
|
||||
const signal = init?.signal?.aborted ? AbortSignal.timeout(30_000) : init?.signal;
|
||||
return base(input, { ...init, headers: h, ...(signal ? { signal } : {}) });
|
||||
},
|
||||
);
|
||||
}) as typeof globalThis.fetch;
|
||||
}
|
||||
|
||||
/** 不加 UA 的底层 fetch(代理 / 测试注入) */
|
||||
/** 站点 HTTP 出口键(direct / proxy:<主机>),与浏览器会话的出口同一口径 */
|
||||
egressKeyOf(site: Pick<SiteRow, "proxy">, defaults: SitesSettings = this.siteDefaultsCache): string {
|
||||
return egressKey(siteEgress(site.proxy, defaults.proxyUrl));
|
||||
}
|
||||
|
||||
/** 不加 UA 的底层 fetch(测试注入 / 按站点出口:代理或明确直连) */
|
||||
private transportFetch(site: Pick<SiteRow, "proxy">, defaults: SitesSettings): typeof globalThis.fetch {
|
||||
if (this.overrides.fetch) return this.overrides.fetch;
|
||||
if (!site.proxy || !defaults.proxyUrl) return globalThis.fetch;
|
||||
const key = defaults.proxyUrl;
|
||||
let client = this.proxyClients.get(key);
|
||||
if (!client) {
|
||||
client = Deno.createHttpClient({ proxy: { url: key } });
|
||||
this.proxyClients.set(key, client);
|
||||
}
|
||||
const c = client;
|
||||
return ((input: RequestInfo | URL, init?: RequestInit) =>
|
||||
globalThis.fetch(input, { ...init, client: c } as RequestInit)) as typeof globalThis.fetch;
|
||||
return siteFetchFor(siteEgress(site.proxy, defaults.proxyUrl));
|
||||
}
|
||||
|
||||
// ---------- BR-4 站点请求 UA ----------
|
||||
@@ -253,8 +272,6 @@ export class SourcesRuntime {
|
||||
return resolveUserAgent(acc, d).ua;
|
||||
}
|
||||
|
||||
private readonly proxyClients = new Map<string, Deno.HttpClient>();
|
||||
|
||||
/**
|
||||
* 站点适配器:每站一个实例,列表/搜索/详情/下载共享同一个限流器。
|
||||
* 站点限流参数、基址、代理或站点默认变化时重建(签名比较)。
|
||||
@@ -270,10 +287,12 @@ export class SourcesRuntime {
|
||||
}
|
||||
const defaults = this.siteDefaultsCache;
|
||||
const proxy = site.proxy ?? false;
|
||||
// Cloudflare 处理状态(config.cloudflare)经常变,不参与重建判断
|
||||
const { cloudflare: _cf, ...configSig } = (site.config ?? {}) as Record<string, unknown>;
|
||||
const sig = JSON.stringify([
|
||||
site.base_url,
|
||||
site.rate_limit,
|
||||
site.config,
|
||||
configSig,
|
||||
proxy,
|
||||
proxy ? defaults.proxyUrl : null,
|
||||
defaults.defaultRateLimit,
|
||||
@@ -304,7 +323,7 @@ export class SourcesRuntime {
|
||||
limiter,
|
||||
now: this.now,
|
||||
// 每条请求都经 siteFetch:代理 + 按账号现场取 UA(BR-4)
|
||||
fetch: this.siteFetch({ proxy }, defaults, () => this.siteUserAgent(site.id)),
|
||||
fetch: this.siteFetch({ id: site.id, proxy }, defaults, () => this.siteUserAgent(site.id)),
|
||||
...(baseUrl ? { baseUrl } : {}),
|
||||
});
|
||||
this.adapters.set(site.id, { sig, limiter, adapter });
|
||||
|
||||
Reference in new issue
Block a user