feat(sites): site requests follow browser UA, smooth UA switch, CF cookies browser-only (BR-4)
- every outbound site request (adapter list/search/detail/download/health, RSS, login, user data) carries the account UA via SourcesRuntime.siteFetch; no more Deno default UA - priority: account manual > pending > global setting > browser identity (fallback: same-shape Windows Firefox UA); settings UA default is empty - migration 0056: existing cookie/password accounts keep their old UA as ua_pending until re-login or browser cookie write-back switches cookie+UA in one transaction; browser_provider.identity caches the browser identity - cf_clearance/__cf_bm stripped on browser write-back (names kept only) - browser sessions get per-site egress (same proxy as HTTP, or direct) - site account page shows UA source; raw UA in tech info Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
ebd07c9230
commit
adc853aa64
21 files changed
+955
-72
No files matched your search
@@ -0,0 +1,79 @@
|
||||
import { MOCK_CAMOFOX_IDENTITY } from "@umt/testkit";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { sourcesIdentityModule } from "../../sources-identity/index.ts";
|
||||
import { readBrowserUserAgent } from "../../sources-identity/service/user-agent.ts";
|
||||
import { egressProxy } from "../service.ts";
|
||||
import { type BrowserHarness, createBrowserHarness, type Loose, ORIGIN, pgAvailable } from "./harness.ts";
|
||||
|
||||
// BR-4:打开会话前按站点下发出口(站点开代理 = 与 HTTP 同一代理,否则直连);浏览器固定身份写进 browser_provider.identity,
|
||||
// 站点请求 UA 经钩子跟随它。
|
||||
|
||||
const up = await pgAvailable();
|
||||
|
||||
describe("egressProxy", () => {
|
||||
it("URL → 上下文代理(口令拆出);非法为 null", () => {
|
||||
expect(egressProxy("http://u:p%[email protected]:7891")).toEqual({
|
||||
server: "http://172.17.0.1:7891",
|
||||
username: "u",
|
||||
password: "p@ss",
|
||||
});
|
||||
expect(egressProxy("socks5://127.0.0.1:1080")).toEqual({ server: "socks5://127.0.0.1:1080" });
|
||||
expect(egressProxy("not a url")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!up)("BR-4 浏览器身份与出口", () => {
|
||||
let h: BrowserHarness;
|
||||
|
||||
const open = async () =>
|
||||
(await h.req("/browser/sessions", { body: { siteId: h.siteId, accountId: null, url: `${ORIGIN}/torrents.php` } }))
|
||||
.body.data as Loose;
|
||||
const closeAll = async () => {
|
||||
for (const s of (await h.req("/browser/sessions")).body.data.items as Loose[])
|
||||
await h.req(`/browser/sessions/${s.id}`, { method: "DELETE" });
|
||||
};
|
||||
|
||||
beforeAll(async () => {
|
||||
h = await createBrowserHarness("br4id");
|
||||
}, 120_000);
|
||||
afterAll(async () => {
|
||||
await h?.close();
|
||||
});
|
||||
|
||||
it("站点不开代理:该 profile 直连;打开时读浏览器身份存库,站点请求 UA 跟随", async () => {
|
||||
const s = await open();
|
||||
expect(h.mock.egress.get(s.profileKey)).toBeNull();
|
||||
const calls = h.mock.calls.map((c) => `${c.method} ${c.path}`);
|
||||
expect(calls.indexOf(`PUT /umt/sessions/${encodeURIComponent(s.profileKey)}/proxy`)).toBeLessThan(
|
||||
calls.lastIndexOf("POST /tabs"),
|
||||
);
|
||||
const row = await h.db
|
||||
.selectFrom("browser_provider")
|
||||
.select("identity")
|
||||
.where("id", "=", h.providerId)
|
||||
.executeTakeFirstOrThrow();
|
||||
expect(row.identity).toMatchObject({ userAgent: MOCK_CAMOFOX_IDENTITY.userAgent, os: "windows" });
|
||||
expect(await readBrowserUserAgent(h.ctx)).toBe(MOCK_CAMOFOX_IDENTITY.userAgent);
|
||||
expect(await sourcesIdentityModule.runtime(h.ctx).defaultUserAgent()).toBe(MOCK_CAMOFOX_IDENTITY.userAgent);
|
||||
await closeAll();
|
||||
});
|
||||
|
||||
it("站点开代理且设了代理地址:该 profile 走同一代理(口令单独传)", async () => {
|
||||
await h.db.updateTable("site").set({ proxy: true }).where("id", "=", h.siteId).execute();
|
||||
await h.db
|
||||
.insertInto("settings")
|
||||
.values({ key: "sites", value: JSON.stringify({ proxyUrl: "http://user:[email protected]:7891" }) })
|
||||
.onConflict((oc) =>
|
||||
oc.column("key").doUpdateSet({ value: JSON.stringify({ proxyUrl: "http://user:[email protected]:7891" }) }),
|
||||
)
|
||||
.execute();
|
||||
const s = await open();
|
||||
expect(h.mock.egress.get(s.profileKey)).toEqual({
|
||||
server: "http://172.17.0.1:7891",
|
||||
username: "user",
|
||||
password: "pw",
|
||||
});
|
||||
await closeAll();
|
||||
await h.db.updateTable("site").set({ proxy: false }).where("id", "=", h.siteId).execute();
|
||||
});
|
||||
});
|
||||
@@ -29,8 +29,10 @@ import type {
|
||||
BrowserCoordinateSpace,
|
||||
BrowserCredentialState,
|
||||
BrowserDownloadVerification,
|
||||
BrowserEgressProxy,
|
||||
BrowserEvidenceRef,
|
||||
BrowserHealth,
|
||||
BrowserIdentity,
|
||||
BrowserProviderView,
|
||||
BrowserReconnectState,
|
||||
BrowserSession,
|
||||
@@ -55,6 +57,7 @@ import {
|
||||
registerBrowserCookieReader,
|
||||
writeBrowserCookie,
|
||||
} from "../sources-identity/service/credentials.ts";
|
||||
import { registerBrowserIdentityReader } from "../sources-identity/service/user-agent.ts";
|
||||
import type { ServerContext } from "../types.ts";
|
||||
|
||||
// C1-E 浏览器服务层:BrowserProvider(camofox)之上的会话/标签/控制权持久化(数据库为准)、
|
||||
@@ -199,6 +202,95 @@ export class BrowserService {
|
||||
this.now = opts.now ?? (() => new Date());
|
||||
// refresh-login 发现 Cookie 失效时从浏览器 profile 读回一次(sources-identity 经钩子调用,不直接依赖本模块)
|
||||
registerBrowserCookieReader(ctx, (siteId, accountId) => this.readProfileCookie(siteId, accountId));
|
||||
// 站点请求 UA 跟随浏览器固定身份(BR-4):sources-identity 经钩子取
|
||||
registerBrowserIdentityReader(ctx, () => this.identityUserAgent());
|
||||
}
|
||||
|
||||
// ---------- BR-4 浏览器身份与出口 ----------
|
||||
|
||||
/** 身份读取失败后多久再试(浏览器服务没起 / 旧镜像时不每条站点请求都去问) */
|
||||
private identityRetryAt = 0;
|
||||
|
||||
/**
|
||||
* 浏览器固定身份的 UA:库里 browser_provider.identity 6 小时内读过就用它;否则问浏览器服务并写回。
|
||||
* 没有启用的浏览器服务 / 插件不支持时返回库里的旧值或 null。
|
||||
*/
|
||||
async identityUserAgent(): Promise<string | null> {
|
||||
const prow = await this.providerRow().catch(() => null);
|
||||
if (!prow) return null;
|
||||
const stored = obj(prow.identity) as { userAgent?: string; fetchedAt?: string };
|
||||
const age = stored.fetchedAt ? Date.now() - Date.parse(stored.fetchedAt) : Number.POSITIVE_INFINITY;
|
||||
if (stored.userAgent && age < 6 * 3_600_000) return stored.userAgent;
|
||||
if (Date.now() < this.identityRetryAt) return stored.userAgent ?? null;
|
||||
const fresh = await this.refreshIdentity(prow).catch(() => null);
|
||||
return fresh?.userAgent ?? stored.userAgent ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 读浏览器服务的固定身份写进 browser_provider.identity(UA / OS / 版本 / 生成时间 + 读取时间)。
|
||||
* maxAgeMs:库里的值比这新就不问。失败 5 分钟内不重试。
|
||||
*/
|
||||
async refreshIdentity(prow: BrowserProviderRow, opts: { maxAgeMs?: number } = {}): Promise<BrowserIdentity | null> {
|
||||
const stored = obj(prow.identity) as Partial<BrowserIdentity> & { fetchedAt?: string };
|
||||
if (opts.maxAgeMs !== undefined && stored.userAgent && stored.fetchedAt) {
|
||||
if (Date.now() - Date.parse(stored.fetchedAt) < opts.maxAgeMs) return stored as BrowserIdentity;
|
||||
}
|
||||
const provider = this.providerFor(prow);
|
||||
if (!provider.identity) return null;
|
||||
let id: BrowserIdentity | null;
|
||||
try {
|
||||
id = await provider.identity();
|
||||
} catch (e) {
|
||||
this.identityRetryAt = Date.now() + 5 * 60_000;
|
||||
throw e;
|
||||
}
|
||||
if (!id) {
|
||||
this.identityRetryAt = Date.now() + 5 * 60_000;
|
||||
return null;
|
||||
}
|
||||
this.identityRetryAt = 0;
|
||||
await this.ctx.db
|
||||
.updateTable("browser_provider")
|
||||
.set({ identity: JSON.stringify({ ...id, fetchedAt: this.now().toISOString() }) })
|
||||
.where("id", "=", prow.id)
|
||||
.execute();
|
||||
if (stored.userAgent !== id.userAgent)
|
||||
this.ctx.logger.info(
|
||||
{ component: "browser", providerId: prow.id, os: id.os, firefoxVersion: id.firefoxVersion },
|
||||
"浏览器身份已更新，站点请求 UA 跟随",
|
||||
);
|
||||
return id;
|
||||
}
|
||||
|
||||
/**
|
||||
* 出口按站点统一(BR-4):站点开了代理且设置了代理地址 → 该 profile 走同一代理;否则直连。
|
||||
* 旧镜像(插件没有出口接口)只记日志,浏览器照旧走容器环境变量里的代理。
|
||||
*/
|
||||
private async applyEgress(provider: BrowserProvider, site: SiteRow, profileKey: string): Promise<void> {
|
||||
if (!provider.setEgress) return;
|
||||
const rt = this.opts.runtime?.(this.ctx);
|
||||
const proxyUrl = rt
|
||||
? (await rt.siteDefaults()).proxyUrl
|
||||
: ((await this.ctx.settings.get("sites").catch(() => null))?.proxyUrl ?? null);
|
||||
const proxy = site.proxy && proxyUrl ? egressProxy(proxyUrl) : null;
|
||||
try {
|
||||
const r = await provider.setEgress(profileKey, proxy);
|
||||
if (r.restarted)
|
||||
this.ctx.logger.info(
|
||||
{ component: "browser", siteId: site.id, egress: proxy ? "proxy" : "direct" },
|
||||
"浏览器会话出口已变更，按新出口重建",
|
||||
);
|
||||
} catch (e) {
|
||||
this.ctx.logger.warn(
|
||||
{
|
||||
component: "browser",
|
||||
siteId: site.id,
|
||||
egress: proxy ? "proxy" : "direct",
|
||||
err: String((e as Error)?.message ?? e),
|
||||
},
|
||||
"设置浏览器会话出口失败，浏览器按容器环境变量出网",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- 浏览器服务 ----------
|
||||
@@ -486,6 +578,8 @@ export class BrowserService {
|
||||
.where("id", "=", row.id)
|
||||
.returningAll()
|
||||
.executeTakeFirstOrThrow();
|
||||
// 探测时顺便读一次浏览器身份(站点请求 UA 跟随它,BR-4)
|
||||
if (health.status !== "down" && row.enabled) await this.refreshIdentity(row).catch(() => null);
|
||||
}
|
||||
const caps = obj(row.capabilities);
|
||||
out.push({
|
||||
@@ -740,6 +834,8 @@ export class BrowserService {
|
||||
for (const m of await provider.listSessions())
|
||||
if (m.profileKey === profileKey && m.controller !== "closed") provider.setController(m.id, "closed");
|
||||
const prep = await this.prepareCookies(site, accountId);
|
||||
await this.applyEgress(provider, site, profileKey);
|
||||
await this.refreshIdentity(prow, { maxAgeMs: 10 * 60_000 }).catch(() => null);
|
||||
await this.rateLimit(site);
|
||||
let res: Awaited<ReturnType<BrowserProvider["openSession"]>>;
|
||||
try {
|
||||
@@ -1416,6 +1512,9 @@ export class BrowserService {
|
||||
if (!provider.reopenSession) throw await fail("浏览器服务不支持重新连接");
|
||||
const url = restoreUrl(row.current_url, budget) ?? site.base_url;
|
||||
const prep = await this.prepareCookies(site, row.account_id).catch(() => null);
|
||||
await this.applyEgress(provider, site, row.profile_key);
|
||||
// 浏览器重启过(可能换了镜像版本):身份重新读
|
||||
await this.refreshIdentity(prow).catch(() => null);
|
||||
let res: Awaited<ReturnType<NonNullable<BrowserProvider["reopenSession"]>>>;
|
||||
try {
|
||||
await this.rateLimit(site);
|
||||
@@ -1944,3 +2043,18 @@ export function imageSize(data: Uint8Array): { width: number | null; height: num
|
||||
}
|
||||
|
||||
export type { ElementTarget };
|
||||
|
||||
/** 代理地址 → 浏览器上下文代理(账号口令拆出来;Playwright 不认 URL 里的口令) */
|
||||
export function egressProxy(url: string): BrowserEgressProxy | null {
|
||||
try {
|
||||
const u = new URL(url);
|
||||
if (!["http:", "https:", "socks4:", "socks5:"].includes(u.protocol) || !u.hostname) return null;
|
||||
return {
|
||||
server: `${u.protocol}//${u.host}`,
|
||||
...(u.username ? { username: decodeURIComponent(u.username) } : {}),
|
||||
...(u.password ? { password: decodeURIComponent(u.password) } : {}),
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user