From ebd07c923001dfc6aaa531a8b2ee3b37a0b884c8 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 30 Sep 2026 03:17:06 +0800 Subject: [PATCH 1/3] feat(browser): fixed Windows identity and per-site egress in umt-cookies plugin (BR-4) - browser:launching hook generates a Windows fingerprint once (camoufox-js, os=windows), persists it to /root/.camofox/umt-identity.json and reuses it on every launch; Firefox/Camoufox upgrades only bump UA/version fields - Windows fontconfig with absolute font dir (upstream path was broken) - GET /umt/identity (public UA/OS/version) and PUT /umt/sessions/:id/proxy (per-context proxy; Firefox no longer uses container HTTP_PROXY env) - adapter identity()/setEgress(), splitBrowserOnlyCookies for cf_clearance/__cf_bm Co-Authored-By: Claude Opus 5.5 --- .../camofox/plugins/umt-cookies/identity.js | 157 +++++++++++ .../opt/camofox/plugins/umt-cookies/index.js | 265 +++++++++++++++++- .../adapters/src/browser/camofox/client.ts | 52 +++- packages/adapters/src/browser/cookies.ts | 24 +- packages/adapters/src/browser/provider.ts | 11 + .../adapters/src/browser/umt-plugin.test.ts | 178 ++++++++++++ packages/contracts/src/browser.ts | 31 ++ packages/testkit/src/browser/mock-camofox.ts | 33 +++ 8 files changed, 746 insertions(+), 5 deletions(-) create mode 100644 docker/browser-desktop/root/opt/camofox/plugins/umt-cookies/identity.js create mode 100644 packages/adapters/src/browser/umt-plugin.test.ts diff --git a/docker/browser-desktop/root/opt/camofox/plugins/umt-cookies/identity.js b/docker/browser-desktop/root/opt/camofox/plugins/umt-cookies/identity.js new file mode 100644 index 00000000..b3a8e572 --- /dev/null +++ b/docker/browser-desktop/root/opt/camofox/plugins/umt-cookies/identity.js @@ -0,0 +1,157 @@ +/** + * 固定浏览器身份(BR-4)的纯函数部分:不依赖 camoufox-js,单测直接导入(packages/adapters/src/browser/umt-identity.test.ts)。 + * + * Camoufox 的指纹是一份 config 字典(navigator.* / screen.* / webGl:* / fonts / 各种 seed …),经 CAMOU_CONFIG_n 环境变量 + * 交给浏览器进程。身份文件保存"指纹部分"的 config;每次启动把上游生成的 config 里的指纹部分整份换成身份文件里的, + * 只保留运行期字段(humanize、locale、geolocation、timezone、WebRTC 地址、扩展路径)。 + */ + +export const IDENTITY_SCHEMA = 1; + +/** 运行期字段:由 camofox 启动参数决定(CAMOFOX_LOCALE、代理 geoip、humanize、扩展),不进身份文件 */ +const RUNTIME_KEYS = new Set(["humanize", "addons", "allowMainWorld", "timezone"]); +const RUNTIME_PREFIXES = ["humanize:", "locale:", "geolocation:", "webrtc:"]; + +export function isRuntimeKey(key) { + return RUNTIME_KEYS.has(key) || RUNTIME_PREFIXES.some((p) => key.startsWith(p)); +} + +/** 从启动参数的 env 里拼回 config(CAMOU_CONFIG_1..n 按序拼接) */ +export function decodeConfig(env) { + const parts = []; + for (let i = 1; env && typeof env[`CAMOU_CONFIG_${i}`] === "string"; i++) parts.push(env[`CAMOU_CONFIG_${i}`]); + if (!parts.length) return null; + try { + const v = JSON.parse(parts.join("")); + return v && typeof v === "object" && !Array.isArray(v) ? v : null; + } catch { + return null; + } +} + +/** 把 config 写回 env:先删掉旧的 CAMOU_CONFIG_n,再按 Linux 的分片长度(32767)切片 */ +export function encodeConfig(env, config) { + for (const k of Object.keys(env)) if (/^CAMOU_CONFIG_\d+$/.test(k)) delete env[k]; + const s = JSON.stringify(config); + const chunk = 32767; + for (let i = 0; i < s.length; i += chunk) env[`CAMOU_CONFIG_${Math.floor(i / chunk) + 1}`] = s.slice(i, i + chunk); + return env; +} + +/** 指纹部分(去掉运行期字段) */ +export function fingerprintPart(config) { + const out = {}; + for (const [k, v] of Object.entries(config ?? {})) if (!isRuntimeKey(k)) out[k] = v; + return out; +} + +/** 上游生成的 config + 身份 → 实际启动用的 config:运行期字段取上游,其余一律取身份 */ +export function mergeLaunchConfig(upstream, identityConfig) { + const out = {}; + for (const [k, v] of Object.entries(upstream ?? {})) if (isRuntimeKey(k)) out[k] = v; + return { ...out, ...fingerprintPart(identityConfig) }; +} + +/** Firefox 主版本号:UA 里的 rv:NNN.0 / Firefox/NNN.0 */ +export function firefoxMajorOf(ua) { + const m = /(?:rv:|Firefox\/)(\d{2,3})\.\d/.exec(String(ua ?? "")); + return m ? m[1] : null; +} + +/** 与 camoufox-js fromBrowserforge 相同的版本号替换:三位数主版本 + ".0" */ +function bumpString(s, major) { + return s.replaceAll(/(? (p ? `${p.server}|${p.username ?? ""}|${p.password ?? ""}` : "direct"); + return k(a) === k(b); +} diff --git a/docker/browser-desktop/root/opt/camofox/plugins/umt-cookies/index.js b/docker/browser-desktop/root/opt/camofox/plugins/umt-cookies/index.js index 4757aab5..a5ef8f9d 100644 --- a/docker/browser-desktop/root/opt/camofox/plugins/umt-cookies/index.js +++ b/docker/browser-desktop/root/opt/camofox/plugins/umt-cookies/index.js @@ -7,14 +7,166 @@ * GET /umt/sessions/:userId/cookies?urls=a,b → 读作用于这些地址的 Cookie(活动上下文优先,否则读落盘文件) * GET /umt/instance → { instanceId, startedAt }:插件加载时生成,进程重启即变 * (BR-3:app 据此发现容器重启,把会话标为需要重新连接) - * Cookie 值只在请求/响应体里,不写日志(只记个数)。 + * GET /umt/identity → 固定浏览器身份的公开部分:UA、OS、版本、生成时间(BR-4) + * PUT /umt/sessions/:userId/proxy { proxy: {server, username?, password?} | null } + * → 该 profile 的出口:站点开代理 = 与 HTTP 同一代理;null = 直连 + * Cookie 值只在请求/响应体里,不写日志(只记个数);代理只记主机,不记账号口令。 + * + * BR-4 固定浏览器身份(docs/runbook/browser.md §11):`browser:launching` 钩子里,首次用 camoufox-js 生成一份 os=windows + * 的指纹 config 落盘到持久卷(缺省 /root/.camofox/umt-identity.json,与 profile 同一个卷),之后每次启动把上游随机生成的 + * 指纹部分整份换成它;Camoufox / Firefox 升级时只改 UA / 版本字段。字体按 Windows 走 Camoufox 自带的 fontconfig。 + * 出口(UMT_BROWSER_PROXY_MODE,缺省 site):Firefox 不用容器的 HTTP(S)_PROXY 环境变量(network.proxy.type = 0 直连), + * 按 profile 在 `session:creating` 里给 BrowserContext 设代理(Playwright 按上下文代理,Juggler 原生支持);env = 保持旧行为。 */ -import { randomUUID } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; import express from "express"; import { getUserPersistencePaths } from "../../lib/persistence.js"; +import { + decodeConfig, + encodeConfig, + fingerprintPart, + firefoxMajorOf, + IDENTITY_SCHEMA, + mergeLaunchConfig, + normalizeProxy, + publicIdentity, + sameProxy, + upgradeIdentity, + validIdentity, +} from "./identity.js"; const INSTANCE = { instanceId: randomUUID(), startedAt: new Date().toISOString() }; +const IDENTITY_OS = "windows"; +/** 身份里固定的 Firefox 首选项(WebGL2 开关与 WebGL 取样配套) */ +const IDENTITY_PREFS = ["webgl.enable-webgl2", "webgl.force-enabled"]; + +async function camoufoxApi() { + const [main, pkg] = await Promise.all([import("camoufox-js"), import("camoufox-js/dist/pkgman.js")]); + return { launchOptions: main.launchOptions, installedVerStr: pkg.installedVerStr }; +} + +/** 用 camoufox-js 按目标 OS 生成一份完整指纹(与上游启动时的生成流程相同,只是 os 固定) */ +async function sampleIdentity(executablePath) { + const { launchOptions } = await camoufoxApi(); + const out = await launchOptions({ + os: IDENTITY_OS, + headless: false, + i_know_what_im_doing: true, + exclude_addons: ["UBO"], + env: {}, + ...(executablePath ? { executable_path: executablePath } : {}), + }); + const config = fingerprintPart(decodeConfig(out.env) ?? {}); + const prefs = {}; + for (const k of IDENTITY_PREFS) if (out.firefoxUserPrefs?.[k] !== undefined) prefs[k] = out.firefoxUserPrefs[k]; + return { config, prefs }; +} + +function createIdentityStore({ file, log }) { + let cached = null; + let loading = null; + const save = async (id) => { + await fs.mkdir(path.dirname(file), { recursive: true }); + const tmp = `${file}.${process.pid}.tmp`; + await fs.writeFile(tmp, JSON.stringify(id, null, 2), { mode: 0o600 }); + await fs.rename(tmp, file); + }; + const load = async (executablePath) => { + const { installedVerStr } = await camoufoxApi(); + let full = null; + try { + full = installedVerStr(); + } catch {} + const major = full ? String(full).split(".")[0] : null; + let id = null; + try { + const v = JSON.parse(await fs.readFile(file, "utf8")); + if (validIdentity(v)) id = v; + else { + log("warn", "umt identity file invalid; regenerating", { file }); + await fs.rename(file, `${file}.bad-${Date.now()}`).catch(() => {}); + } + } catch (err) { + if (err.code !== "ENOENT") { + log("warn", "umt identity file unreadable; regenerating", { file, error: err.message }); + await fs.rename(file, `${file}.bad-${Date.now()}`).catch(() => {}); + } + } + const now = new Date().toISOString(); + if (!id) { + const s = await sampleIdentity(executablePath); + id = { + schema: IDENTITY_SCHEMA, + os: IDENTITY_OS, + config: s.config, + prefs: s.prefs, + firefoxVersion: firefoxMajorOf(s.config["navigator.userAgent"]) ?? major, + camoufoxVersion: full, + fingerprintId: createHash("sha256").update(JSON.stringify(s.config)).digest("hex").slice(0, 12), + revision: 1, + createdAt: now, + updatedAt: now, + }; + await save(id); + log("info", "umt identity created", { + os: id.os, + fingerprintId: id.fingerprintId, + firefoxVersion: id.firefoxVersion, + }); + } else if ((major && id.firefoxVersion !== major) || (full && id.camoufoxVersion !== full)) { + const sample = await sampleIdentity(executablePath).catch(() => null); + const r = upgradeIdentity(id, { firefoxVersion: major, sample: sample?.config, camoufoxVersion: full, now }); + if (r.changed) { + id = r.identity; + await save(id); + log("info", "umt identity upgraded (UA / version fields only)", { + fingerprintId: id.fingerprintId, + revision: id.revision, + firefoxVersion: id.firefoxVersion, + }); + } + } + return id; + }; + return { + get: (executablePath) => { + if (cached) return Promise.resolve(cached); + loading ??= load(executablePath) + .then((id) => { + cached = id; + return id; + }) + .finally(() => { + loading = null; + }); + return loading; + }, + }; +} + +/** + * Windows 字体:Camoufox 自带 fontconfig/windows(只含自带字体目录 + Windows 缺省字族映射),但目录写的是相对当前目录的 + * `fonts`;浏览器进程的当前目录是 camofox 的 /opt/camofox,不是安装目录。复制一份,把字体目录改成绝对路径。 + */ +async function windowsFontconfig(installDir, log) { + const src = path.join(installDir, "fontconfig", IDENTITY_OS, "fonts.conf"); + try { + const text = await fs.readFile(src, "utf8"); + const dir = path.join(os.tmpdir(), "umt-fontconfig-windows"); + await fs.mkdir(dir, { recursive: true }); + await fs.writeFile( + path.join(dir, "fonts.conf"), + text.replace(/fonts<\/dir>/, `${path.join(installDir, "fonts")}`), + ); + return dir; + } catch (err) { + log("warn", "umt identity: windows fontconfig not found; fonts follow upstream", { error: err.message }); + return null; + } +} const FIELDS = ["name", "value", "domain", "path", "expires", "httpOnly", "secure", "sameSite"]; @@ -46,6 +198,110 @@ function matches(c, urls, nowSec) { export async function register(app, ctx) { const { log } = ctx; const profileDir = process.env.CAMOFOX_PROFILE_DIR || ctx.config.profileDir; + // 身份与代理表放在 profile 目录的上一级(同一个持久卷 /root/.camofox) + const stateDir = process.env.UMT_STATE_DIR || path.dirname(profileDir || "/root/.camofox/profiles"); + const installDir = process.env.CAMOUFOX_INSTALL_DIR || "/opt/camoufox"; + const identityMode = (process.env.UMT_BROWSER_IDENTITY || "fixed").toLowerCase(); + const proxyMode = (process.env.UMT_BROWSER_PROXY_MODE || "site").toLowerCase(); + const identity = createIdentityStore({ + file: process.env.UMT_IDENTITY_FILE || path.join(stateDir, "umt-identity.json"), + log, + }); + + // ── BR-4 固定浏览器身份 ── + ctx.events.on("browser:launching", async ({ options }) => { + if (identityMode === "fixed") { + try { + const id = await identity.get(options.executablePath); + const merged = mergeLaunchConfig(decodeConfig(options.env) ?? {}, id.config); + options.env = encodeConfig({ ...(options.env ?? {}) }, merged); + const fc = await windowsFontconfig(installDir, log); + if (fc) options.env.FONTCONFIG_PATH = fc; + options.firefoxUserPrefs = { ...(options.firefoxUserPrefs ?? {}), ...(id.prefs ?? {}) }; + log("info", "umt identity applied", { os: id.os, fingerprintId: id.fingerprintId, revision: id.revision }); + } catch (err) { + // 身份不可用时照常启动(上游随机指纹),app 读 /umt/identity 失败会退回内置 UA + log("error", "umt identity failed; launching with upstream fingerprint", { error: err.message }); + } + } + // 出口按站点:不走容器环境变量里的代理;上游自己的代理池(CAMOFOX 代理配置)优先 + if (proxyMode === "site" && !options.proxy) { + options.firefoxUserPrefs = { ...(options.firefoxUserPrefs ?? {}), "network.proxy.type": 0 }; + } + }); + + app.get("/umt/identity", ctx.auth(), async (_req, res) => { + if (identityMode !== "fixed") return res.status(404).json({ error: "fixed identity disabled" }); + try { + res.json(publicIdentity(await identity.get())); + } catch (err) { + log("error", "umt identity read failed", { error: err.message }); + res.status(500).json({ error: ctx.safeError(err) }); + } + }); + + // ── BR-4 出口按站点 ── + const proxyFile = path.join(stateDir, "umt-proxies.json"); + /** userId → 代理(null = 直连);持久化,camofox 重启后 app 重开会话前也会重新下发 */ + const proxies = new Map(); + /** userId → 活动上下文创建时实际用的代理 */ + const applied = new Map(); + try { + const saved = JSON.parse(await fs.readFile(proxyFile, "utf8")); + for (const [k, v] of Object.entries(saved ?? {})) { + const p = normalizeProxy(v); + if (p !== undefined) proxies.set(k, p); + } + } catch {} + const saveProxies = async () => { + await fs.mkdir(stateDir, { recursive: true }); + const tmp = `${proxyFile}.${process.pid}.tmp`; + await fs.writeFile(tmp, JSON.stringify(Object.fromEntries(proxies)), { mode: 0o600 }); + await fs.rename(tmp, proxyFile); + }; + const hostOf = (p) => (p ? p.server.replace(/^[a-z0-9]+:\/\//, "") : null); + + ctx.events.on("session:creating", ({ userId, contextOptions }) => { + if (proxyMode !== "site") return; + if (contextOptions.proxy) { + applied.set(userId, contextOptions.proxy); + return; + } + const p = proxies.get(userId) ?? null; + if (p) contextOptions.proxy = { ...p }; + applied.set(userId, p); + log("info", "umt session egress", { userId, egress: p ? "proxy" : "direct", proxyHost: hostOf(p) }); + }); + ctx.events.on("session:destroyed", ({ userId }) => applied.delete(userId)); + + app.put("/umt/sessions/:userId/proxy", ctx.auth(), express.json({ limit: "8kb" }), async (req, res) => { + try { + const userId = ctx.normalizeUserId(req.params.userId); + const p = normalizeProxy(req.body?.proxy === undefined ? undefined : req.body.proxy); + if (p === undefined) return res.status(400).json({ error: "proxy must be {server} or null" }); + const before = proxies.has(userId) ? proxies.get(userId) : undefined; + proxies.set(userId, p); + if (before === undefined || !sameProxy(before, p)) await saveProxies(); + // 活动上下文的出口与新设置不同:关掉它(持久化插件先落盘登录状态),下次请求按新出口重建 + let restarted = false; + const live = ctx.sessions.get(userId); + if (proxyMode === "site" && live && applied.has(userId) && !sameProxy(applied.get(userId), p)) { + await ctx.destroySession(userId, { reason: "umt_egress_changed" }); + restarted = true; + } + log("info", "umt egress set", { + reqId: req.reqId, + userId, + egress: p ? "proxy" : "direct", + proxyHost: hostOf(p), + restarted, + }); + res.json({ ok: true, userId, egress: p ? "proxy" : "direct", mode: proxyMode, restarted }); + } catch (err) { + log("error", "umt egress set failed", { reqId: req.reqId, error: err.message }); + res.status(500).json({ error: ctx.safeError(err) }); + } + }); app.post("/umt/sessions/:userId/cookies", ctx.auth(), express.json({ limit: "512kb" }), async (req, res) => { try { @@ -108,5 +364,8 @@ export async function register(app, ctx) { app.get("/umt/instance", ctx.auth(), (_req, res) => res.json(INSTANCE)); - log("info", "umt-cookies plugin: registered GET/POST /umt/sessions/:userId/cookies, GET /umt/instance"); + log("info", "umt-cookies plugin: registered cookies / instance / identity / egress endpoints", { + identityMode, + proxyMode, + }); } diff --git a/packages/adapters/src/browser/camofox/client.ts b/packages/adapters/src/browser/camofox/client.ts index 2c9f45d3..5229f147 100644 --- a/packages/adapters/src/browser/camofox/client.ts +++ b/packages/adapters/src/browser/camofox/client.ts @@ -19,7 +19,9 @@ import type { BrowserCapabilities, BrowserControlLease, BrowserCookie, + BrowserEgressProxy, BrowserHealth, + BrowserIdentity, BrowserSession, BrowserSnapshot, BrowserTab, @@ -175,7 +177,7 @@ export class CamofoxBrowserProvider implements BrowserProvider { } private async request( - method: "GET" | "POST" | "DELETE", + method: "GET" | "POST" | "PUT" | "DELETE", path: string, opts: { query?: Record; body?: Json; timeoutMs?: number; auth?: boolean } = {}, ): Promise { @@ -386,6 +388,54 @@ export class CamofoxBrowserProvider implements BrowserProvider { return typeof v === "string" || typeof v === "number" ? String(v) : null; } + /** + * 固定浏览器身份(BR-4):umt 插件 `GET /umt/identity`(UA、OS、版本、生成时间,不含指纹细节); + * 插件不在或身份关闭(404)返回 null。 + */ + async identity(): Promise { + let body: Json; + try { + body = await this.json(await this.request("GET", "/umt/identity", { timeoutMs: 10_000 }), "/umt/identity"); + } catch (e) { + if (e instanceof BrowserError && e.httpStatus === 404) return null; + throw e; + } + const userAgent = str(body.userAgent); + if (!userAgent) return null; + return { + userAgent, + os: str(body.os) ?? null, + platform: str(body.platform) ?? null, + firefoxVersion: str(body.firefoxVersion) ?? null, + camoufoxVersion: str(body.camoufoxVersion) ?? null, + fingerprintId: str(body.fingerprintId) ?? null, + revision: int(body.revision) ?? 1, + createdAt: str(body.createdAt) ?? null, + updatedAt: str(body.updatedAt) ?? null, + }; + } + + /** + * 该 profile 的出口(BR-4 出口 IP 按站点统一):umt 插件 `PUT /umt/sessions/{userId}/proxy`。 + * proxy = null 直连;活动上下文出口不同时插件会先关掉它(落盘登录状态)。插件不在报 not_implemented。 + */ + async setEgress(profileKey: string, proxy: BrowserEgressProxy | null): Promise<{ restarted: boolean }> { + try { + const body = await this.json( + await this.request("PUT", `/umt/sessions/${encodeURIComponent(profileKey)}/proxy`, { + body: { proxy } as unknown as Json, + timeoutMs: 15_000, + }), + "/umt/sessions/{userId}/proxy", + ); + return { restarted: body.restarted === true }; + } catch (e) { + if (e instanceof BrowserError && e.httpStatus === 404) + throw new BrowserError("not_implemented", "浏览器服务不支持按站点设出口(镜像未带新版 umt-cookies 插件)"); + throw e; + } + } + /** * 重建会话的上游上下文(BR-3):上游重启 / 回收后旧标签页都不在了。先(可选)注入 Cookie,再用同一 userId / sessionKey * 开新标签页并导航到 url;旧的打开中标签页标 lost,旧元素引用作废。不检查控制者:接管中的会话重连后仍归原控制者。 diff --git a/packages/adapters/src/browser/cookies.ts b/packages/adapters/src/browser/cookies.ts index c85bd030..5b7d6ed4 100644 --- a/packages/adapters/src/browser/cookies.ts +++ b/packages/adapters/src/browser/cookies.ts @@ -1,4 +1,4 @@ -import type { BrowserCookie } from "@umt/contracts"; +import { BROWSER_ONLY_COOKIES, type BrowserCookie } from "@umt/contracts"; // BR-2 凭据统一:站点账号里存的是 HTTP `Cookie:` 请求头串("a=1; b=2"),浏览器 profile 里是带域名/路径/期限的 Cookie 对象。 // 这里只做两者互转与比较;值是凭据,不打印、不进错误信息。 @@ -34,6 +34,28 @@ export function mergeCookieHeaders(...headers: (string | null | undefined)[]): s return jar.size ? formatCookieHeader(jar) : null; } +/** 仅浏览器用的 Cookie(Cloudflare 人机验证):名字精确匹配,或 cf_chl_ 前缀(挑战过程中的临时 Cookie) */ +export function isBrowserOnlyCookie(name: string): boolean { + return (BROWSER_ONLY_COOKIES as readonly string[]).includes(name) || name.startsWith("cf_chl_"); +} + +/** + * 浏览器 → HTTP 方向:剔除仅浏览器用的 Cookie(BR-4)。返回剩下的头串(没有剩余为 null)与被剔除的名字(不含值)。 + */ +export function splitBrowserOnlyCookies(header: string | null | undefined): { + header: string | null; + browserOnly: string[]; +} { + const jar = parseCookieHeader(header); + const browserOnly: string[] = []; + for (const name of [...jar.keys()]) + if (isBrowserOnlyCookie(name)) { + jar.delete(name); + browserOnly.push(name); + } + return { header: jar.size ? formatCookieHeader(jar) : null, browserOnly: browserOnly.sort() }; +} + /** 两个 Cookie 头串是否等价(忽略顺序与空白) */ export function sameCookieHeader(a: string | null | undefined, b: string | null | undefined): boolean { const x = parseCookieHeader(a); diff --git a/packages/adapters/src/browser/provider.ts b/packages/adapters/src/browser/provider.ts index 74b65921..b40f5b17 100644 --- a/packages/adapters/src/browser/provider.ts +++ b/packages/adapters/src/browser/provider.ts @@ -2,7 +2,9 @@ import type { BrowserCapabilities, BrowserControlLease, BrowserCookie, + BrowserEgressProxy, BrowserHealth, + BrowserIdentity, BrowserLeaseHolderKind, BrowserProviderKind, BrowserSession, @@ -90,6 +92,15 @@ export interface BrowserProvider { * 原有打开中的标签页一律标 lost。不检查控制者(接管中的会话也能重连,控制权不变)。 */ reopenSession?(sessionId: string, req: ReopenSessionRequest): Promise; + + // ── BR-4 浏览器 UA 与指纹统一、出口按站点 ── + /** 固定浏览器身份的公开部分(UA、OS、版本);上游没有时返回 null */ + identity?(): Promise; + /** + * 该 profile 的出口:proxy = 站点开代理时与 HTTP 同一代理,null = 直连。上游没有接口时报 BrowserError("not_implemented")。 + * restarted = 活动上下文出口不同、已被关闭(下次操作按新出口重建)。 + */ + setEgress?(profileKey: string, proxy: BrowserEgressProxy | null): Promise<{ restarted: boolean }>; } export interface ReopenSessionRequest { diff --git a/packages/adapters/src/browser/umt-plugin.test.ts b/packages/adapters/src/browser/umt-plugin.test.ts new file mode 100644 index 00000000..d63d88af --- /dev/null +++ b/packages/adapters/src/browser/umt-plugin.test.ts @@ -0,0 +1,178 @@ +import { createMockCamofox, MOCK_CAMOFOX_IDENTITY, type MockCamofox } from "@umt/testkit"; +import { afterEach, describe, expect, it } from "vitest"; +import { createCamofoxBrowserProvider, splitBrowserOnlyCookies, staticBrowserSecrets } from "./index.ts"; + +// BR-4:umt-cookies 插件的固定身份纯函数(docker/browser-desktop/.../umt-cookies/identity.js)、 +// 适配器的 identity() / setEgress()、仅浏览器 Cookie 的剔除。 + +// 插件是容器里的纯 JS(不进 tsc 模块图):按路径动态导入 +const PLUGIN = new URL( + "../../../../docker/browser-desktop/root/opt/camofox/plugins/umt-cookies/identity.js", + import.meta.url, +).href; +// biome-ignore lint/suspicious/noExplicitAny: 插件是无类型 JS +const plugin = async (): Promise => import(PLUGIN); + +const UA152 = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:152.0) Gecko/20100101 Firefox/152.0"; + +describe("umt-cookies 插件:固定身份", () => { + it("config ⇄ CAMOU_CONFIG_n 分片往返;旧分片先清掉", async () => { + const p = await plugin(); + const big = { "navigator.userAgent": UA152, pad: "x".repeat(70_000) }; + const env: Record = { CAMOU_CONFIG_1: "stale", CAMOU_CONFIG_9: "stale", PATH: "/bin" }; + p.encodeConfig(env, big); + expect(Object.keys(env).filter((k) => k.startsWith("CAMOU_CONFIG_"))).toEqual([ + "CAMOU_CONFIG_1", + "CAMOU_CONFIG_2", + "CAMOU_CONFIG_3", + ]); + expect(env.PATH).toBe("/bin"); + expect(p.decodeConfig(env)).toEqual(big); + expect(p.decodeConfig({})).toBeNull(); + }); + + it("启动 config:运行期字段取上游,指纹字段一律取身份", async () => { + const p = await plugin(); + const upstream = { + "navigator.userAgent": "Mozilla/5.0 (X11; Linux x86_64; rv:152.0) Gecko/20100101 Firefox/152.0", + "screen.width": 1920, + humanize: true, + "humanize:maxTime": 2, + "locale:language": "zh", + "geolocation:latitude": 1, + timezone: "Asia/Shanghai", + addons: ["/a"], + "webrtc:ipv4": "1.2.3.4", + }; + const identity = { "navigator.userAgent": UA152, "navigator.platform": "Win32", "screen.width": 2560 }; + expect(p.mergeLaunchConfig(upstream, identity)).toEqual({ + humanize: true, + "humanize:maxTime": 2, + "locale:language": "zh", + "geolocation:latitude": 1, + timezone: "Asia/Shanghai", + addons: ["/a"], + "webrtc:ipv4": "1.2.3.4", + "navigator.userAgent": UA152, + "navigator.platform": "Win32", + "screen.width": 2560, + }); + expect(p.fingerprintPart(upstream)).toEqual({ + "navigator.userAgent": upstream["navigator.userAgent"], + "screen.width": 1920, + }); + }); + + it("Firefox 升级:只改 navigator / headers 里的版本号,其余保留;新字段从样本补齐;版本相同不改", async () => { + const p = await plugin(); + const id = { + schema: 1, + os: "windows", + firefoxVersion: "140", + camoufoxVersion: "140.0-a", + fingerprintId: "fp1", + revision: 1, + createdAt: "2026-01-01T00:00:00Z", + updatedAt: "2026-01-01T00:00:00Z", + config: { + "navigator.userAgent": UA152.replaceAll("152.0", "140.0"), + "navigator.appVersion": "5.0 (Windows)", + "headers.User-Agent": UA152.replaceAll("152.0", "140.0"), + "screen.width": 2560, + "webGl:renderer": "ANGLE 140.0 build", + "canvas:seed": 42, + }, + }; + const sample = { "navigator.userAgent": "other", "audio:seed": 7, "screen.width": 1 }; + const r = p.upgradeIdentity(id, { firefoxVersion: "152", sample, camoufoxVersion: "152.0.4", now: "N" }); + expect(r.changed).toBe(true); + expect(r.identity.config).toEqual({ + "navigator.userAgent": UA152, + "navigator.appVersion": "5.0 (Windows)", + "headers.User-Agent": UA152, + "screen.width": 2560, + "webGl:renderer": "ANGLE 140.0 build", + "canvas:seed": 42, + "audio:seed": 7, + }); + expect(r.identity).toMatchObject({ firefoxVersion: "152", revision: 2, fingerprintId: "fp1", updatedAt: "N" }); + expect(p.publicIdentity(r.identity)).toEqual({ + userAgent: UA152, + os: "windows", + platform: null, + firefoxVersion: "152", + camoufoxVersion: "152.0.4", + fingerprintId: "fp1", + revision: 2, + schema: 1, + createdAt: "2026-01-01T00:00:00Z", + updatedAt: "N", + }); + const same = p.upgradeIdentity(r.identity, { firefoxVersion: "152", sample, camoufoxVersion: "152.0.4", now: "M" }); + expect(same.changed).toBe(false); + expect(p.validIdentity(r.identity)).toBe(true); + expect(p.validIdentity({ schema: 1, config: {} })).toBe(false); + expect(p.firefoxMajorOf(UA152)).toBe("152"); + }); + + it("出口代理规范化:口令从 URL 拆出;非法协议拒绝;null = 直连", async () => { + const p = await plugin(); + expect(p.normalizeProxy(null)).toBeNull(); + expect(p.normalizeProxy({ server: "http://u%40x:p%3Aw@172.17.0.1:7891/" })).toEqual({ + server: "http://172.17.0.1:7891", + username: "u@x", + password: "p:w", + }); + expect(p.normalizeProxy({ server: "socks5://h:1080", username: "a", password: "b" })).toEqual({ + server: "socks5://h:1080", + username: "a", + password: "b", + }); + expect(p.normalizeProxy({ server: "ftp://h" })).toBeUndefined(); + expect(p.normalizeProxy({})).toBeUndefined(); + expect(p.sameProxy(null, null)).toBe(true); + expect(p.sameProxy({ server: "http://a:1" }, null)).toBe(false); + }); +}); + +describe("仅浏览器 Cookie", () => { + it("剔除 cf_clearance / __cf_bm / cf_chl_*,只返回名字", () => { + expect(splitBrowserOnlyCookies("a=1; cf_clearance=S; __cf_bm=T; cf_chl_rc_m=1")).toEqual({ + header: "a=1", + browserOnly: ["__cf_bm", "cf_chl_rc_m", "cf_clearance"], + }); + expect(splitBrowserOnlyCookies("cf_clearance=S")).toEqual({ header: null, browserOnly: ["cf_clearance"] }); + expect(splitBrowserOnlyCookies(null)).toEqual({ header: null, browserOnly: [] }); + }); +}); + +describe("camofox 适配器:身份与出口", () => { + let mock: MockCamofox | null = null; + afterEach(async () => { + await mock?.close(); + mock = null; + }); + const provider = (m: MockCamofox) => + createCamofoxBrowserProvider({ + providerId: "11111111-1111-4111-8111-111111111111", + baseUrl: m.url, + secrets: staticBrowserSecrets({ "provider:11111111-1111-4111-8111-111111111111": "k" }), + }); + + it("identity() 读插件身份;setEgress() 下发代理 / 直连", async () => { + mock = await createMockCamofox({ accessKey: "k" }); + const p = provider(mock); + expect(await p.identity!()).toMatchObject({ userAgent: MOCK_CAMOFOX_IDENTITY.userAgent, os: "windows" }); + expect(await p.setEgress!("site-a", { server: "http://127.0.0.1:7891" })).toEqual({ restarted: false }); + expect(await p.setEgress!("site-b", null)).toEqual({ restarted: false }); + expect(mock.egress.get("site-a")).toEqual({ server: "http://127.0.0.1:7891" }); + expect(mock.egress.get("site-b")).toBeNull(); + }); + + it("旧插件(没有接口):identity() 为 null,setEgress() 报 not_implemented", async () => { + mock = await createMockCamofox({ accessKey: "k", cookieApi: "none" }); + const p = provider(mock); + expect(await p.identity!()).toBeNull(); + await expect(p.setEgress!("x", null)).rejects.toMatchObject({ kind: "not_implemented" }); + }); +}); diff --git a/packages/contracts/src/browser.ts b/packages/contracts/src/browser.ts index ff56542d..58c78390 100644 --- a/packages/contracts/src/browser.ts +++ b/packages/contracts/src/browser.ts @@ -304,6 +304,37 @@ export const BrowserCookie = z.object({ }); export type BrowserCookie = z.infer; +// ── BR-4 浏览器 UA 与指纹统一 ── + +/** 固定浏览器身份的公开部分(umt 插件 GET /umt/identity):不含指纹细节 */ +export const BrowserIdentity = z.object({ + userAgent: z.string().min(1), + os: z.string().nullable(), + platform: z.string().nullable(), + firefoxVersion: z.string().nullable(), + camoufoxVersion: z.string().nullable(), + /** 首次生成时的指纹摘要(升级只改 UA / 版本,摘要不变) */ + fingerprintId: z.string().nullable(), + /** 每次升级改写 UA / 版本字段 +1 */ + revision: z.number().int(), + createdAt: z.string().nullable(), + updatedAt: z.string().nullable(), +}); +export type BrowserIdentity = z.infer; + +/** 浏览器会话出口代理(Playwright 形状);null = 直连 */ +export interface BrowserEgressProxy { + server: string; + username?: string; + password?: string; +} + +/** + * Cloudflare 的人机验证 Cookie:与取得它的浏览器(UA、IP、TLS 指纹)绑定,HTTP 端(Deno)TLS 指纹不同,拿去用多半无效还可能 + * 触发风控。浏览器回写账号时剔除,只留在浏览器 profile 里("仅浏览器")。 + */ +export const BROWSER_ONLY_COOKIES = ["cf_clearance", "__cf_bm"] as const; + /** 站点账号登录态(Cookie)来源:手填 / 账密自动登录 / 浏览器回写 */ export const SiteCookieSource = z.enum(["manual", "login", "browser"]); export type SiteCookieSource = z.infer; diff --git a/packages/testkit/src/browser/mock-camofox.ts b/packages/testkit/src/browser/mock-camofox.ts index 6096c57f..ccc99619 100644 --- a/packages/testkit/src/browser/mock-camofox.ts +++ b/packages/testkit/src/browser/mock-camofox.ts @@ -126,6 +126,22 @@ export type MockCamofox = MockHttp & { restart(opts?: { loseProfiles?: boolean }): void; /** 当前实例标识 */ readonly instanceId: string; + /** 各 profile 的出口(`PUT /umt/sessions/:userId/proxy`,BR-4):null = 直连 */ + egress: Map; +}; + +/** mock 的固定浏览器身份(`GET /umt/identity`,BR-4) */ +export const MOCK_CAMOFOX_IDENTITY = { + userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:152.0) Gecko/20100101 Firefox/152.0", + os: "windows", + platform: "Win32", + firefoxVersion: "152", + camoufoxVersion: "152.0.4-mock", + fingerprintId: "mockfp000001", + revision: 1, + schema: 1, + createdAt: "2026-09-30T00:00:00.000Z", + updatedAt: "2026-09-30T00:00:00.000Z", }; /** 1x1 透明 PNG */ @@ -331,6 +347,7 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise< const actions: MockCamofoxAction[] = []; const loggedIn = new Set(); const cookieJars = new Map(); + const egress = new Map(); const cookieApi = opts.cookieApi ?? "plugin"; let instanceId = `inst-${crypto.randomUUID()}`; /** 重启前存在过的 tabId:之后访问返回 410 */ @@ -795,6 +812,21 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise< }, ], [/^\/umt\/instance$/, () => (cookieApi === "plugin" ? { json: { instanceId } } : err(404, "no route"))], + [ + /^\/umt\/identity$/, + () => (cookieApi === "plugin" ? { json: { ...MOCK_CAMOFOX_IDENTITY } } : err(404, "no route")), + ], + [ + /^\/umt\/sessions\/([^/]+)\/proxy$/, + (m, _q, req, b) => { + if (cookieApi !== "plugin" || req.method !== "PUT") return err(404, "no route"); + const userId = decodeURIComponent(m[1]!); + const p = b.proxy as { server?: string } | null | undefined; + if (p !== null && (!p || typeof p.server !== "string")) return err(400, "proxy must be {server} or null"); + egress.set(userId, p === null ? null : { ...(p as { server: string }) }); + return { json: { ok: true, userId, egress: p ? "proxy" : "direct", mode: "site", restarted: false } }; + }, + ], [ /^\/umt\/sessions\/([^/]+)\/cookies$/, (m, q, req, b) => { @@ -858,6 +890,7 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise< actions, loggedIn, cookieJars, + egress, expireLogin(userId?: string) { if (userId) loggedIn.delete(userId); else loggedIn.clear(); From adc853aa641fa912af2d103fc0b4fdb7b87b48f9 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 30 Sep 2026 03:17:13 +0800 Subject: [PATCH 2/3] feat(sites): site requests follow browser UA, smooth UA switch, CF cookies browser-only (BR-4) - every outbound site request (adapter list/search/detail/download/health, RSS, login, user data) carries the account UA via SourcesRuntime.siteFetch; no more Deno default UA - priority: account manual > pending > global setting > browser identity (fallback: same-shape Windows Firefox UA); settings UA default is empty - migration 0056: existing cookie/password accounts keep their old UA as ua_pending until re-login or browser cookie write-back switches cookie+UA in one transaction; browser_provider.identity caches the browser identity - cf_clearance/__cf_bm stripped on browser write-back (names kept only) - browser sessions get per-site egress (same proxy as HTTP, or direct) - site account page shows UA source; raw UA in tech info Co-Authored-By: Claude Opus 5.5 --- .../src/modules/auth-settings/provided.ts | 10 +- .../__tests__/browser-identity.test.ts | 79 +++++ apps/server/src/modules/browser/service.ts | 114 ++++++++ .../sources-identity/routes/sources.ts | 14 +- .../src/modules/sources-identity/runtime.ts | 76 ++++- .../sources-identity/service/account.ts | 54 +++- .../sources-identity/service/credentials.ts | 96 +++--- .../modules/sources-identity/service/feeds.ts | 7 +- .../sources-identity/service/user-agent.ts | 69 +++++ apps/server/src/modules/types.ts | 3 +- .../migration-browser-identity.test.ts | 100 +++++++ .../test/sources-identity/user-agent.test.ts | 273 ++++++++++++++++++ apps/web/app/components/settings/sites.tsx | 6 +- .../components/sites/credential-fields.tsx | 7 +- .../components/sites/site-accounts-tab.tsx | 23 +- packages/adapters/src/rss/fetch.ts | 9 +- packages/contracts/src/settings.ts | 19 +- packages/contracts/src/sources.ts | 11 +- packages/contracts/test/db-sync.test.ts | 8 +- .../db/migrations/0056_browser_identity.ts | 47 +++ packages/db/src/types.ts | 2 + 21 files changed, 955 insertions(+), 72 deletions(-) create mode 100644 apps/server/src/modules/browser/__tests__/browser-identity.test.ts create mode 100644 apps/server/src/modules/sources-identity/service/user-agent.ts create mode 100644 apps/server/test/sources-identity/migration-browser-identity.test.ts create mode 100644 apps/server/test/sources-identity/user-agent.test.ts create mode 100644 packages/db/migrations/0056_browser_identity.ts diff --git a/apps/server/src/modules/auth-settings/provided.ts b/apps/server/src/modules/auth-settings/provided.ts index 430be9d9..2c5d5d90 100644 --- a/apps/server/src/modules/auth-settings/provided.ts +++ b/apps/server/src/modules/auth-settings/provided.ts @@ -114,12 +114,16 @@ function open( */ export function createSecretStore(deps: { db: Kysely; masterKey: string }): SecretService { const { db, masterKey } = deps; - const replace = async (secretRef: string, plaintext: string) => { + const replace = async (secretRef: string, plaintext: string, exec: Kysely = db) => { assertRef(secretRef); - const row = await db.selectFrom("secret").select(["id", "purpose"]).where("id", "=", secretRef).executeTakeFirst(); + const row = await exec + .selectFrom("secret") + .select(["id", "purpose"]) + .where("id", "=", secretRef) + .executeTakeFirst(); if (!row) throw new SecretError("not_found", secretRef); const s = seal(masterKey, row.id, row.purpose, plaintext); - await db + await exec .updateTable("secret") .set({ ciphertext: s.ciphertext, diff --git a/apps/server/src/modules/browser/__tests__/browser-identity.test.ts b/apps/server/src/modules/browser/__tests__/browser-identity.test.ts new file mode 100644 index 00000000..0e970c17 --- /dev/null +++ b/apps/server/src/modules/browser/__tests__/browser-identity.test.ts @@ -0,0 +1,79 @@ +import { MOCK_CAMOFOX_IDENTITY } from "@umt/testkit"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { sourcesIdentityModule } from "../../sources-identity/index.ts"; +import { readBrowserUserAgent } from "../../sources-identity/service/user-agent.ts"; +import { egressProxy } from "../service.ts"; +import { type BrowserHarness, createBrowserHarness, type Loose, ORIGIN, pgAvailable } from "./harness.ts"; + +// BR-4:打开会话前按站点下发出口(站点开代理 = 与 HTTP 同一代理,否则直连);浏览器固定身份写进 browser_provider.identity, +// 站点请求 UA 经钩子跟随它。 + +const up = await pgAvailable(); + +describe("egressProxy", () => { + it("URL → 上下文代理(口令拆出);非法为 null", () => { + expect(egressProxy("http://u:p%40ss@172.17.0.1:7891")).toEqual({ + server: "http://172.17.0.1:7891", + username: "u", + password: "p@ss", + }); + expect(egressProxy("socks5://127.0.0.1:1080")).toEqual({ server: "socks5://127.0.0.1:1080" }); + expect(egressProxy("not a url")).toBeNull(); + }); +}); + +describe.skipIf(!up)("BR-4 浏览器身份与出口", () => { + let h: BrowserHarness; + + const open = async () => + (await h.req("/browser/sessions", { body: { siteId: h.siteId, accountId: null, url: `${ORIGIN}/torrents.php` } })) + .body.data as Loose; + const closeAll = async () => { + for (const s of (await h.req("/browser/sessions")).body.data.items as Loose[]) + await h.req(`/browser/sessions/${s.id}`, { method: "DELETE" }); + }; + + beforeAll(async () => { + h = await createBrowserHarness("br4id"); + }, 120_000); + afterAll(async () => { + await h?.close(); + }); + + it("站点不开代理:该 profile 直连;打开时读浏览器身份存库,站点请求 UA 跟随", async () => { + const s = await open(); + expect(h.mock.egress.get(s.profileKey)).toBeNull(); + const calls = h.mock.calls.map((c) => `${c.method} ${c.path}`); + expect(calls.indexOf(`PUT /umt/sessions/${encodeURIComponent(s.profileKey)}/proxy`)).toBeLessThan( + calls.lastIndexOf("POST /tabs"), + ); + const row = await h.db + .selectFrom("browser_provider") + .select("identity") + .where("id", "=", h.providerId) + .executeTakeFirstOrThrow(); + expect(row.identity).toMatchObject({ userAgent: MOCK_CAMOFOX_IDENTITY.userAgent, os: "windows" }); + expect(await readBrowserUserAgent(h.ctx)).toBe(MOCK_CAMOFOX_IDENTITY.userAgent); + expect(await sourcesIdentityModule.runtime(h.ctx).defaultUserAgent()).toBe(MOCK_CAMOFOX_IDENTITY.userAgent); + await closeAll(); + }); + + it("站点开代理且设了代理地址:该 profile 走同一代理(口令单独传)", async () => { + await h.db.updateTable("site").set({ proxy: true }).where("id", "=", h.siteId).execute(); + await h.db + .insertInto("settings") + .values({ key: "sites", value: JSON.stringify({ proxyUrl: "http://user:pw@172.17.0.1:7891" }) }) + .onConflict((oc) => + oc.column("key").doUpdateSet({ value: JSON.stringify({ proxyUrl: "http://user:pw@172.17.0.1:7891" }) }), + ) + .execute(); + const s = await open(); + expect(h.mock.egress.get(s.profileKey)).toEqual({ + server: "http://172.17.0.1:7891", + username: "user", + password: "pw", + }); + await closeAll(); + await h.db.updateTable("site").set({ proxy: false }).where("id", "=", h.siteId).execute(); + }); +}); diff --git a/apps/server/src/modules/browser/service.ts b/apps/server/src/modules/browser/service.ts index 524e5c61..8fdda9b3 100644 --- a/apps/server/src/modules/browser/service.ts +++ b/apps/server/src/modules/browser/service.ts @@ -29,8 +29,10 @@ import type { BrowserCoordinateSpace, BrowserCredentialState, BrowserDownloadVerification, + BrowserEgressProxy, BrowserEvidenceRef, BrowserHealth, + BrowserIdentity, BrowserProviderView, BrowserReconnectState, BrowserSession, @@ -55,6 +57,7 @@ import { registerBrowserCookieReader, writeBrowserCookie, } from "../sources-identity/service/credentials.ts"; +import { registerBrowserIdentityReader } from "../sources-identity/service/user-agent.ts"; import type { ServerContext } from "../types.ts"; // C1-E 浏览器服务层:BrowserProvider(camofox)之上的会话/标签/控制权持久化(数据库为准)、 @@ -199,6 +202,95 @@ export class BrowserService { this.now = opts.now ?? (() => new Date()); // refresh-login 发现 Cookie 失效时从浏览器 profile 读回一次(sources-identity 经钩子调用,不直接依赖本模块) registerBrowserCookieReader(ctx, (siteId, accountId) => this.readProfileCookie(siteId, accountId)); + // 站点请求 UA 跟随浏览器固定身份(BR-4):sources-identity 经钩子取 + registerBrowserIdentityReader(ctx, () => this.identityUserAgent()); + } + + // ---------- BR-4 浏览器身份与出口 ---------- + + /** 身份读取失败后多久再试(浏览器服务没起 / 旧镜像时不每条站点请求都去问) */ + private identityRetryAt = 0; + + /** + * 浏览器固定身份的 UA:库里 browser_provider.identity 6 小时内读过就用它;否则问浏览器服务并写回。 + * 没有启用的浏览器服务 / 插件不支持时返回库里的旧值或 null。 + */ + async identityUserAgent(): Promise { + const prow = await this.providerRow().catch(() => null); + if (!prow) return null; + const stored = obj(prow.identity) as { userAgent?: string; fetchedAt?: string }; + const age = stored.fetchedAt ? Date.now() - Date.parse(stored.fetchedAt) : Number.POSITIVE_INFINITY; + if (stored.userAgent && age < 6 * 3_600_000) return stored.userAgent; + if (Date.now() < this.identityRetryAt) return stored.userAgent ?? null; + const fresh = await this.refreshIdentity(prow).catch(() => null); + return fresh?.userAgent ?? stored.userAgent ?? null; + } + + /** + * 读浏览器服务的固定身份写进 browser_provider.identity(UA / OS / 版本 / 生成时间 + 读取时间)。 + * maxAgeMs:库里的值比这新就不问。失败 5 分钟内不重试。 + */ + async refreshIdentity(prow: BrowserProviderRow, opts: { maxAgeMs?: number } = {}): Promise { + const stored = obj(prow.identity) as Partial & { fetchedAt?: string }; + if (opts.maxAgeMs !== undefined && stored.userAgent && stored.fetchedAt) { + if (Date.now() - Date.parse(stored.fetchedAt) < opts.maxAgeMs) return stored as BrowserIdentity; + } + const provider = this.providerFor(prow); + if (!provider.identity) return null; + let id: BrowserIdentity | null; + try { + id = await provider.identity(); + } catch (e) { + this.identityRetryAt = Date.now() + 5 * 60_000; + throw e; + } + if (!id) { + this.identityRetryAt = Date.now() + 5 * 60_000; + return null; + } + this.identityRetryAt = 0; + await this.ctx.db + .updateTable("browser_provider") + .set({ identity: JSON.stringify({ ...id, fetchedAt: this.now().toISOString() }) }) + .where("id", "=", prow.id) + .execute(); + if (stored.userAgent !== id.userAgent) + this.ctx.logger.info( + { component: "browser", providerId: prow.id, os: id.os, firefoxVersion: id.firefoxVersion }, + "浏览器身份已更新，站点请求 UA 跟随", + ); + return id; + } + + /** + * 出口按站点统一(BR-4):站点开了代理且设置了代理地址 → 该 profile 走同一代理;否则直连。 + * 旧镜像(插件没有出口接口)只记日志,浏览器照旧走容器环境变量里的代理。 + */ + private async applyEgress(provider: BrowserProvider, site: SiteRow, profileKey: string): Promise { + if (!provider.setEgress) return; + const rt = this.opts.runtime?.(this.ctx); + const proxyUrl = rt + ? (await rt.siteDefaults()).proxyUrl + : ((await this.ctx.settings.get("sites").catch(() => null))?.proxyUrl ?? null); + const proxy = site.proxy && proxyUrl ? egressProxy(proxyUrl) : null; + try { + const r = await provider.setEgress(profileKey, proxy); + if (r.restarted) + this.ctx.logger.info( + { component: "browser", siteId: site.id, egress: proxy ? "proxy" : "direct" }, + "浏览器会话出口已变更，按新出口重建", + ); + } catch (e) { + this.ctx.logger.warn( + { + component: "browser", + siteId: site.id, + egress: proxy ? "proxy" : "direct", + err: String((e as Error)?.message ?? e), + }, + "设置浏览器会话出口失败，浏览器按容器环境变量出网", + ); + } } // ---------- 浏览器服务 ---------- @@ -486,6 +578,8 @@ export class BrowserService { .where("id", "=", row.id) .returningAll() .executeTakeFirstOrThrow(); + // 探测时顺便读一次浏览器身份(站点请求 UA 跟随它,BR-4) + if (health.status !== "down" && row.enabled) await this.refreshIdentity(row).catch(() => null); } const caps = obj(row.capabilities); out.push({ @@ -740,6 +834,8 @@ export class BrowserService { for (const m of await provider.listSessions()) if (m.profileKey === profileKey && m.controller !== "closed") provider.setController(m.id, "closed"); const prep = await this.prepareCookies(site, accountId); + await this.applyEgress(provider, site, profileKey); + await this.refreshIdentity(prow, { maxAgeMs: 10 * 60_000 }).catch(() => null); await this.rateLimit(site); let res: Awaited>; try { @@ -1416,6 +1512,9 @@ export class BrowserService { if (!provider.reopenSession) throw await fail("浏览器服务不支持重新连接"); const url = restoreUrl(row.current_url, budget) ?? site.base_url; const prep = await this.prepareCookies(site, row.account_id).catch(() => null); + await this.applyEgress(provider, site, row.profile_key); + // 浏览器重启过(可能换了镜像版本):身份重新读 + await this.refreshIdentity(prow).catch(() => null); let res: Awaited>>; try { await this.rateLimit(site); @@ -1944,3 +2043,18 @@ export function imageSize(data: Uint8Array): { width: number | null; height: num } export type { ElementTarget }; + +/** 代理地址 → 浏览器上下文代理(账号口令拆出来;Playwright 不认 URL 里的口令) */ +export function egressProxy(url: string): BrowserEgressProxy | null { + try { + const u = new URL(url); + if (!["http:", "https:", "socks4:", "socks5:"].includes(u.protocol) || !u.hostname) return null; + return { + server: `${u.protocol}//${u.host}`, + ...(u.username ? { username: decodeURIComponent(u.username) } : {}), + ...(u.password ? { password: decodeURIComponent(u.password) } : {}), + }; + } catch { + return null; + } +} diff --git a/apps/server/src/modules/sources-identity/routes/sources.ts b/apps/server/src/modules/sources-identity/routes/sources.ts index d6006d74..480c4486 100644 --- a/apps/server/src/modules/sources-identity/routes/sources.ts +++ b/apps/server/src/modules/sources-identity/routes/sources.ts @@ -128,9 +128,10 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) { ) .orderBy("created_at") .execute(); + const ua = await rt().uaDefaults(); for (const it of items) { const mine = accounts.filter((a) => a.site_id === it.id); - it.accounts = mine.map(accountToApi); + it.accounts = mine.map((a) => accountToApi(a, ua)); it.accountStatus = overallAccountStatus(mine); } } @@ -483,12 +484,13 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) { .limit(query.limit + 1) .execute(); const p = page(rows, off, query.limit); - return { items: p.items.map(accountToApi) as SiteAccount[], nextCursor: p.nextCursor }; + const ua = await rt().uaDefaults(); + return { items: p.items.map((a) => accountToApi(a, ua)) as SiteAccount[], nextCursor: p.nextCursor }; }); route(app, D.siteAccounts.get, async ({ params }) => { const row = await db().selectFrom("site_account").selectAll().where("id", "=", params.id).executeTakeFirst(); if (!row) throw new ApiException("not_found", "站点账号不存在"); - return accountToApi(row); + return accountToApi(row, await rt().uaDefaults()); }); route(app, D.siteAccounts.create, async ({ body, actor }) => { await getSite(body.siteId); @@ -510,7 +512,7 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) { .returningAll() .executeTakeFirstOrThrow(); await audit(db(), actor, "site_account.created", { type: "site_account", id: row.id }, { siteId: body.siteId }); - return accountToApi(row); + return accountToApi(row, await rt().uaDefaults()); } catch (e) { await rt() .ctx.secrets.remove(ref) @@ -539,6 +541,8 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) { patch.status = "unknown"; patch.last_error = null; patch.session_secret_ref = null; + // 新凭证 = 新登录态:不再沿用旧 UA(BR-4) + patch.ua_pending = null; } const row = await db() .transaction() @@ -585,7 +589,7 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) { ); return r; }); - return accountToApi(row); + return accountToApi(row, await rt().uaDefaults()); }); // ── Feed ── diff --git a/apps/server/src/modules/sources-identity/runtime.ts b/apps/server/src/modules/sources-identity/runtime.ts index 7c096ad7..f4fa8c74 100644 --- a/apps/server/src/modules/sources-identity/runtime.ts +++ b/apps/server/src/modules/sources-identity/runtime.ts @@ -24,6 +24,7 @@ import { tmdbDetailRef, } from "../../shared-cache.ts"; import type { ServerContext } from "../types.ts"; +import { readBrowserUserAgent, resolveUserAgent, type UserAgentDefaults } from "./service/user-agent.ts"; // 模块运行时依赖:站点适配器与限流器(每站一个实例,令牌在 PG 跨进程共享)、RSS 抓取器、TMDB 客户端、时钟。 // 测试通过 createSourcesIdentityModule(overrides) 注入 fetch / TMDB / 时钟。 @@ -168,8 +169,28 @@ export class SourcesRuntime { /** * 站点请求用的 fetch:测试注入优先;站点开了"使用代理"且设置了 proxyUrl 时走代理(Deno.createHttpClient)。 + * BR-4:每条请求都带 UA(请求自己设了就不动;否则 ua 参数,缺省按"跟随浏览器"),不再发 Deno 缺省的 `Deno/x.y.z`。 + * ua 可以是函数:每次请求现场取(站点适配器按当前账号取,账号切换 UA 后立即生效)。 */ - siteFetch(site: Pick, defaults: SitesSettings = this.siteDefaultsCache): typeof globalThis.fetch { + siteFetch( + site: Pick, + defaults: SitesSettings = this.siteDefaultsCache, + ua?: string | (() => Promise), + ): typeof globalThis.fetch { + const base = this.transportFetch(site, defaults); + return ((input: RequestInfo | URL, init?: RequestInit) => { + const headers = new Headers(init?.headers ?? (input instanceof Request ? input.headers : undefined)); + if (headers.has("user-agent")) return base(input, { ...init, headers }); + const pick = typeof ua === "function" ? ua() : ua !== undefined ? Promise.resolve(ua) : this.defaultUserAgent(); + return pick.then((v) => { + headers.set("user-agent", v); + return base(input, { ...init, headers }); + }); + }) as typeof globalThis.fetch; + } + + /** 不加 UA 的底层 fetch(代理 / 测试注入) */ + private transportFetch(site: Pick, defaults: SitesSettings): typeof globalThis.fetch { if (this.overrides.fetch) return this.overrides.fetch; if (!site.proxy || !defaults.proxyUrl) return globalThis.fetch; const key = defaults.proxyUrl; @@ -182,6 +203,56 @@ export class SourcesRuntime { return ((input: RequestInfo | URL, init?: RequestInit) => globalThis.fetch(input, { ...init, client: c } as RequestInit)) as typeof globalThis.fetch; } + + // ---------- BR-4 站点请求 UA ---------- + + private browserUa: { at: number; ua: string | null } = { at: 0, ua: null }; + + /** 浏览器固定身份的 UA(60 秒缓存);读不到为 null(调用方退回常量) */ + async browserUserAgent(): Promise { + if (Date.now() - this.browserUa.at < 60_000) return this.browserUa.ua; + let ua: string | null = this.browserUa.ua; + try { + ua = (await readBrowserUserAgent(this.ctx)) ?? ua; + } catch (err) { + this.ctx.logger.debug({ err: { message: (err as Error).message } }, "读取浏览器身份 UA 失败，沿用上次的值"); + } + this.browserUa = { at: Date.now(), ua }; + return ua; + } + + /** UA 解析用的全局部分:设置里的全局 UA + 浏览器 UA(已退回常量) */ + async uaDefaults(): Promise { + const [s, browser] = await Promise.all([this.siteDefaults(), this.browserUserAgent()]); + return { global: s.userAgent ?? "", browser: browser ?? "" }; + } + + /** 不挂账号的请求(通用 RSS 等)用的 UA:全局 UA 或跟随浏览器 */ + async defaultUserAgent(): Promise { + return resolveUserAgent(null, await this.uaDefaults()).ua; + } + + /** + * 站点请求 UA:取与站点凭证相同的账号(第一个启用、未失效、非 Passkey 的账号;没有就第一个启用的),按账号解析。 + * 不缓存:与 Cookie 同一时刻读库,账号切换 Cookie + UA 后下一条请求两者一起生效。 + */ + async siteUserAgent(siteId: string): Promise { + const d = await this.uaDefaults(); + const rows = await this.ctx.db + .selectFrom("site_account") + .select(["user_agent", "ua_pending", "status", "auth_kind"]) + .where("site_id", "=", siteId) + .where("enabled", "=", true) + .orderBy("created_at") + .execute(); + const acc = + rows.find((r) => r.auth_kind !== "passkey" && (r.status === "unknown" || r.status === "active")) ?? + rows.find((r) => r.auth_kind !== "passkey") ?? + rows[0] ?? + null; + return resolveUserAgent(acc, d).ua; + } + private readonly proxyClients = new Map(); /** @@ -232,7 +303,8 @@ export class SourcesRuntime { secrets: this.siteSecrets(site.id), limiter, now: this.now, - ...(this.overrides.fetch || proxy ? { fetch: this.siteFetch({ proxy }, defaults) } : {}), + // 每条请求都经 siteFetch:代理 + 按账号现场取 UA(BR-4) + fetch: this.siteFetch({ proxy }, defaults, () => this.siteUserAgent(site.id)), ...(baseUrl ? { baseUrl } : {}), }); this.adapters.set(site.id, { sig, limiter, adapter }); diff --git a/apps/server/src/modules/sources-identity/service/account.ts b/apps/server/src/modules/sources-identity/service/account.ts index 228a1032..bc8f6d6e 100644 --- a/apps/server/src/modules/sources-identity/service/account.ts +++ b/apps/server/src/modules/sources-identity/service/account.ts @@ -29,20 +29,25 @@ import type { SourcesRuntime } from "../runtime.ts"; import { capabilitiesOf, LOGIN_KINDS } from "../site-kinds.ts"; import { type Actor, audit, closeDecisions, type Exec, openDecision, SYSTEM } from "./common.ts"; import { cookieFromBrowser, writeBrowserCookie } from "./credentials.ts"; +import { browserOnlyCookiesOf, resolveUserAgent, type UserAgentDefaults } from "./user-agent.ts"; // W9-A 站点中心(docs/design/site-center.md):凭证写入、刷新登录(账密 takelogin / Cookie / API Key 校验)、 // 用户数据与按天历史、健康检测、删除。凭证明文只在内存里用,不进日志、审计与错误信息。 const jsonb = (v: unknown) => JSON.stringify(v ?? null); -/** site_account 行 → SiteAccount(secret 引用只给布尔) */ -export function accountToApi(row: SiteAccountRow): Record { - const { totp_secret_ref, session_secret_ref, passkey_secret_ref, ...rest } = row; +/** site_account 行 → SiteAccount(secret 引用只给布尔);ua = 全局 UA 与浏览器 UA(SourcesRuntime.uaDefaults) */ +export function accountToApi(row: SiteAccountRow, ua: UserAgentDefaults): Record { + const { totp_secret_ref, session_secret_ref, passkey_secret_ref, ua_pending: _p, ...rest } = row; + const eff = resolveUserAgent(row, ua); return { ...toApi>(rest), hasTotp: !!totp_secret_ref, hasSession: !!session_secret_ref, hasPasskey: !!passkey_secret_ref || (row.auth_kind === "passkey" && !!row.secret_ref), + userAgentSource: eff.source, + effectiveUserAgent: eff.ua, + browserOnlyCookies: browserOnlyCookiesOf(row.user_info), }; } @@ -283,6 +288,8 @@ export async function setSiteCredentials( totp_secret_ref: totpRef, login_name: body.kind === "password" ? body.username : null, user_agent: "userAgent" in body ? (body.userAgent ?? null) : null, + // 换了主凭证 = 新的登录态,不再沿用旧 UA(BR-4 待切换结束);只改 UA / 启用时保持 + ...(keepMain ? {} : { ua_pending: null }), user_info: jsonb(userInfo), ...(body.enabled !== undefined ? { enabled: body.enabled } : {}), }; @@ -388,7 +395,10 @@ export async function refreshSiteAccount( const def = getSiteDefinition(site.key); if (!def || site.deleted_at) return unchanged("unsupported", "站点没有适配器定义，无法校验登录"); const defaults = await rt.siteDefaults(); - const ua = acc.user_agent ?? defaults.userAgent; + // BR-4:current = 当前登录态配套的 UA(待切换时是旧 UA);target = 切换后的 UA(重新登录、浏览器 Cookie 用它) + const uaDefaults = await rt.uaDefaults(); + const currentUa = resolveUserAgent(acc, uaDefaults).ua; + const targetUa = resolveUserAgent(acc, uaDefaults, { ignorePending: true }).ua; let limiter: ReturnType = null; try { rt.siteAdapter(site); @@ -396,7 +406,7 @@ export async function refreshSiteAccount( } catch { // 无适配器:不限流(仅 generic_rss,前面已返回) } - const fetch = rt.siteFetch(site, defaults); + const fetch = rt.siteFetch(site, defaults, currentUa); const umt = def.umt as { kind: string; login?: Parameters[0]["login"]; loginSuccess?: string }; const profile = (def.umt as { profile?: Record }).profile; const vipLevels = (def.umt as { vipLevels?: string[] }).vipLevels; @@ -406,7 +416,10 @@ export async function refreshSiteAccount( ? (site.config as { apiBaseUrl?: string })?.apiBaseUrl || mteamApiBase(def as MteamDefinition) : null; - const fetchUserData = async (cred: { cookie?: string | null; apikey?: string | null }): Promise => { + const fetchUserData = async ( + cred: { cookie?: string | null; apikey?: string | null }, + ua: string = currentUa, + ): Promise => { const info = await refreshSiteUserData({ name: site.name, url: site.base_url, @@ -467,14 +480,15 @@ export async function refreshSiteAccount( username: acc.login_name ?? (acc.user_info as { username?: string })?.username ?? acc.label, password, totpSecret, - userAgent: ua, + userAgent: targetUa, fetch, ...(limiter ? { limiter } : {}), ...(opts.signal ? { signal: opts.signal } : {}), now: rt.now, }); + // 新登录态用 targetUa 取得:会话 Cookie 与 UA 一起切换(清掉待切换) await storeSession(rt, acc, login.cookie); - const r = await fetchUserData({ cookie: login.cookie }); + const r = await fetchUserData({ cookie: login.cookie }, targetUa); if (r.kind === "ok") return await succeed(r.info); return await succeed( null, @@ -488,7 +502,8 @@ export async function refreshSiteAccount( // BR-2:存的 Cookie 失效,但浏览器里可能已人工登录过 → 读回浏览器 profile 的 Cookie 试一次,有效才回写 const alt = await cookieFromBrowser(rt.ctx, acc, plain); if (alt) { - const r2 = await fetchUserData({ cookie: alt }); + // 浏览器里的登录态是浏览器 UA 取得的:按切换后的 UA 验证,回写时一起切换 + const r2 = await fetchUserData({ cookie: alt }, targetUa); if (r2.kind === "ok") { await writeBrowserCookie(rt.ctx, acc.id, alt, rt.now()); return await succeed(r2.info, "存的 Cookie 已失效；浏览器里的登录态有效，已回写"); @@ -515,7 +530,10 @@ export async function refreshSiteAccount( export const secretName = (acc: { site_id: string; label: string }, suffix = "") => `site_account:${acc.site_id}:${acc.label}${suffix}:${randomUUID().slice(0, 8)}`; -/** 账密登录成功:会话 Cookie 存 secret;"只换 Cookie"时凭证类型落为 cookie,密码与 TOTP 丢弃 */ +/** + * 账密登录成功:会话 Cookie 存 secret;"只换 Cookie"时凭证类型落为 cookie,密码与 TOTP 丢弃。 + * 登录用的是切换后的 UA:写 Cookie 的同一事务里清掉待切换(BR-4 Cookie + UA 原子切换)。 + */ async function storeSession(rt: SourcesRuntime, acc: SiteAccountRow, cookie: string) { const { db, secrets } = rt.ctx; const cookieOnly = (acc.user_info as { saveCookieOnly?: boolean })?.saveCookieOnly === true; @@ -531,6 +549,7 @@ async function storeSession(rt: SourcesRuntime, acc: SiteAccountRow, cookie: str session_secret_ref: null, totp_secret_ref: null, user_info: jsonb(info), + ua_pending: null, }) .where("id", "=", acc.id) .execute(); @@ -541,10 +560,16 @@ async function storeSession(rt: SourcesRuntime, acc: SiteAccountRow, cookie: str acc.secret_ref = ref; acc.session_secret_ref = null; acc.totp_secret_ref = null; + acc.ua_pending = null; return; } if (acc.session_secret_ref) { - await secrets.replace(acc.session_secret_ref, cookie); + const ref = acc.session_secret_ref; + await db.transaction().execute(async (trx) => { + await secrets.replace(ref, cookie, trx); + await trx.updateTable("site_account").set({ ua_pending: null }).where("id", "=", acc.id).execute(); + }); + acc.ua_pending = null; return; } const ref = await secrets.create({ @@ -552,8 +577,13 @@ async function storeSession(rt: SourcesRuntime, acc: SiteAccountRow, cookie: str plaintext: cookie, label: secretName(acc, ":session"), }); - await db.updateTable("site_account").set({ session_secret_ref: ref }).where("id", "=", acc.id).execute(); + await db + .updateTable("site_account") + .set({ session_secret_ref: ref, ua_pending: null }) + .where("id", "=", acc.id) + .execute(); acc.session_secret_ref = ref; + acc.ua_pending = null; } async function markSuccess( diff --git a/apps/server/src/modules/sources-identity/service/credentials.ts b/apps/server/src/modules/sources-identity/service/credentials.ts index 49391e9b..1692d1d3 100644 --- a/apps/server/src/modules/sources-identity/service/credentials.ts +++ b/apps/server/src/modules/sources-identity/service/credentials.ts @@ -1,4 +1,4 @@ -import { mergeCookieHeaders, parseCookieHeader, sameCookieHeader } from "@umt/adapters"; +import { mergeCookieHeaders, parseCookieHeader, sameCookieHeader, splitBrowserOnlyCookies } from "@umt/adapters"; import type { SiteCookieSource } from "@umt/contracts"; import type { SiteAccountRow } from "@umt/db"; import type { ServerContext } from "../../types.ts"; @@ -81,52 +81,81 @@ export async function readAccountCookie(ctx: Ctx, accountId: string): Promise { +): Promise<{ changed: boolean; count: number; uaSwitched: boolean; browserOnly: string[] }> { + const { header, browserOnly } = splitBrowserOnlyCookies(rawHeader); const count = parseCookieHeader(header).size; - if (!count) return { changed: false, count: 0 }; + const none = { changed: false, count, uaSwitched: false, browserOnly }; + if (!header || !count) return none; const acc = await ctx.db.selectFrom("site_account").selectAll().where("id", "=", accountId).executeTakeFirst(); - if (!acc || (acc.auth_kind !== "cookie" && acc.auth_kind !== "password")) return { changed: false, count }; + if (!acc || (acc.auth_kind !== "cookie" && acc.auth_kind !== "password")) return none; const cur = await readAccountCookie(ctx, accountId); - if (cur && sameCookieHeader(cur.header, header)) return { changed: false, count }; - let sessionRef = acc.session_secret_ref; - if (acc.auth_kind === "cookie") { - if (!acc.secret_ref) return { changed: false, count }; - await ctx.secrets.replace(acc.secret_ref, header); - // 旧会话 Cookie 会在合并时覆盖同名项,回写后以浏览器为准 - if (sessionRef) await ctx.secrets.replace(sessionRef, header); - } else if (sessionRef) { - await ctx.secrets.replace(sessionRef, header); - } else { - sessionRef = await ctx.secrets.create({ - purpose: "site_cookie", - plaintext: header, - label: `site_account:${acc.site_id}:${acc.label}:session:${crypto.randomUUID().slice(0, 8)}`, - }); + const info = obj(acc.user_info); + const prevOnly = Array.isArray(info.browserOnlyCookies) ? (info.browserOnlyCookies as string[]) : []; + const onlyChanged = browserOnly.length > 0 && browserOnly.join(",") !== prevOnly.join(","); + if (cur && sameCookieHeader(cur.header, header)) { + // Cookie 没变:浏览器正用这份登录态(不在登录页),待切换的 UA 也可以换了 + if (!acc.ua_pending && !onlyChanged) return none; + await ctx.db + .updateTable("site_account") + .set({ + ...(acc.ua_pending ? { ua_pending: null } : {}), + ...(onlyChanged ? { user_info: JSON.stringify({ ...info, browserOnlyCookies: browserOnly }) } : {}), + updated_at: now, + }) + .where("id", "=", acc.id) + .execute(); + if (acc.ua_pending) + ctx.logger.info({ component: "credentials", accountId }, "站点账号 UA 已切换为跟随浏览器（Cookie 未变）"); + return { ...none, uaSwitched: !!acc.ua_pending }; } + if (acc.auth_kind === "cookie" && !acc.secret_ref) return none; + // 新建会话 secret 在事务外(新行没被引用,不影响读);替换已有 secret 与账号行更新在同一事务里 + const newSession = + acc.auth_kind === "password" && !acc.session_secret_ref + ? await ctx.secrets.create({ + purpose: "site_cookie", + plaintext: header, + label: `site_account:${acc.site_id}:${acc.label}:session:${crypto.randomUUID().slice(0, 8)}`, + }) + : null; + const sessionRef = newSession ?? acc.session_secret_ref; const origin: CookieOrigin = { source: "browser", by: "browser", at: now.toISOString() }; const stale = acc.status === "expired" || acc.status === "invalid"; - await ctx.db - .updateTable("site_account") - .set({ - user_info: JSON.stringify({ ...obj(acc.user_info), cookieOrigin: origin }), - session_secret_ref: sessionRef, - ...(stale ? { status: "unknown" as const, last_error: null } : {}), - updated_at: now, - }) - .where("id", "=", acc.id) - .execute(); + await ctx.db.transaction().execute(async (trx) => { + if (acc.auth_kind === "cookie") { + await ctx.secrets.replace(acc.secret_ref!, header, trx); + // 旧会话 Cookie 会在合并时覆盖同名项,回写后以浏览器为准 + if (sessionRef) await ctx.secrets.replace(sessionRef, header, trx); + } else if (!newSession && sessionRef) { + await ctx.secrets.replace(sessionRef, header, trx); + } + await trx + .updateTable("site_account") + .set({ + user_info: JSON.stringify({ ...info, cookieOrigin: origin, browserOnlyCookies: browserOnly }), + session_secret_ref: sessionRef, + ua_pending: null, + ...(stale ? { status: "unknown" as const, last_error: null } : {}), + updated_at: now, + }) + .where("id", "=", acc.id) + .execute(); + }); ctx.logger.info( - { component: "credentials", accountId, count, updatedBy: "browser" }, + { component: "credentials", accountId, count, browserOnly: browserOnly.length, updatedBy: "browser" }, "站点账号 Cookie 已从浏览器回写", ); - return { changed: true, count }; + return { changed: true, count, uaSwitched: !!acc.ua_pending, browserOnly }; } // ---------- 浏览器读回钩子(浏览器模块注册;sources-identity 不直接依赖浏览器模块) ---------- @@ -148,7 +177,8 @@ export async function cookieFromBrowser( ): Promise { const reader = readers.get(ctx); if (!reader) return null; - const header = await reader(acc.site_id, acc.id).catch(() => null); + // 仅浏览器用的 Cloudflare Cookie 不给站点请求(BR-4) + const header = splitBrowserOnlyCookies(await reader(acc.site_id, acc.id).catch(() => null)).header; if (!header || sameCookieHeader(header, current)) return null; return header; } diff --git a/apps/server/src/modules/sources-identity/service/feeds.ts b/apps/server/src/modules/sources-identity/service/feeds.ts index 971f7a03..5101f064 100644 --- a/apps/server/src/modules/sources-identity/service/feeds.ts +++ b/apps/server/src/modules/sources-identity/service/feeds.ts @@ -159,7 +159,12 @@ export async function ingestFeed( if (!feed.url_secret_ref) return { status: "skipped", reason: "no_url", ...empty }; const url = await secrets.resolve(feed.url_secret_ref); - const result = await rt.rss.fetchFeed(url, stateFromRow(feed), { ignoreBackoff: !!opts.manual }); + // BR-4:挂站点的 Feed 与站点其它请求同一 UA、同一出口(站点开代理时走代理);通用 Feed 用全局 UA / 跟随浏览器 + const result = await rt.rss.fetchFeed(url, stateFromRow(feed), { + ignoreBackoff: !!opts.manual, + userAgent: site ? await rt.siteUserAgent(site.id) : await rt.defaultUserAgent(), + fetch: rt.siteFetch(site ?? { proxy: false }), + }); // 挂站点的 Feed:每次真正发出的 RSS 请求也记进站点请求计数(熔断 / 退避跳过的不算) if (site && result.kind !== "skipped") rt.countSiteRequest(site.id); if (site) await renormalizeFeedReleases(rt, site).catch(() => {}); diff --git a/apps/server/src/modules/sources-identity/service/user-agent.ts b/apps/server/src/modules/sources-identity/service/user-agent.ts new file mode 100644 index 00000000..008001c0 --- /dev/null +++ b/apps/server/src/modules/sources-identity/service/user-agent.ts @@ -0,0 +1,69 @@ +import { FALLBACK_SITE_USER_AGENT } from "@umt/contracts"; +import type { SiteAccountRow } from "@umt/db"; +import type { ServerContext } from "../../types.ts"; + +// BR-4 站点请求 UA:账号手动 > 待切换(沿用取得当前 Cookie 时的旧 UA)> 设置里的全局 UA > 跟随浏览器(camofox 固定身份的 UA, +// 读不到时用同型的 Windows Firefox 常量)。站点适配器、RSS、登录、用户数据、种子下载都经 SourcesRuntime.siteFetch 带上它。 + +export type UserAgentSource = "browser" | "manual" | "global" | "pending"; + +export interface UserAgentDefaults { + /** settings.sites.userAgent;空 = 跟随浏览器 */ + global: string; + /** 浏览器身份 UA(已退回常量) */ + browser: string; +} + +type UaFields = Pick; + +/** + * 生效 UA。ignorePending = 按"切换之后"算(重新登录 / 浏览器 Cookie 回写时用新 UA,成功后清掉待切换)。 + */ +export function resolveUserAgent( + acc: UaFields | null | undefined, + d: UserAgentDefaults, + opts: { ignorePending?: boolean } = {}, +): { ua: string; source: UserAgentSource } { + const manual = acc?.user_agent?.trim(); + if (manual) return { ua: manual, source: "manual" }; + const pending = acc?.ua_pending?.trim(); + if (pending && !opts.ignorePending) return { ua: pending, source: "pending" }; + const global = d.global.trim(); + if (global) return { ua: global, source: "global" }; + return { ua: d.browser || FALLBACK_SITE_USER_AGENT, source: "browser" }; +} + +/** user_info.browserOnlyCookies:最近一次浏览器回写时剔除的仅浏览器 Cookie 名 */ +export function browserOnlyCookiesOf(userInfo: unknown): string[] { + const v = (userInfo as { browserOnlyCookies?: unknown } | null)?.browserOnlyCookies; + return Array.isArray(v) ? v.filter((x): x is string => typeof x === "string") : []; +} + +// ---------- 浏览器身份读取钩子(浏览器模块注册;sources-identity 不直接依赖浏览器模块) ---------- + +/** 返回浏览器固定身份的 UA;没有浏览器服务 / 插件时 null */ +export type BrowserIdentityReader = () => Promise; + +const readers = new WeakMap(); + +export function registerBrowserIdentityReader(ctx: object, reader: BrowserIdentityReader): void { + readers.set(ctx, reader); +} + +/** + * 浏览器身份 UA:浏览器模块在本进程装配了就问它(它读库里缓存的身份,过期再问浏览器服务); + * 没装配时只读 browser_provider.identity(其它进程写入的)。 + */ +export async function readBrowserUserAgent(ctx: Pick): Promise { + const reader = readers.get(ctx); + if (reader) return reader(); + const row = await ctx.db + .selectFrom("browser_provider") + .select("identity") + .where("enabled", "=", true) + .where("kind", "<>", "none") + .orderBy("created_at") + .executeTakeFirst(); + const ua = (row?.identity as { userAgent?: unknown } | null | undefined)?.userAgent; + return typeof ua === "string" && ua ? ua : null; +} diff --git a/apps/server/src/modules/types.ts b/apps/server/src/modules/types.ts index e3f60619..2c5951ca 100644 --- a/apps/server/src/modules/types.ts +++ b/apps/server/src/modules/types.ts @@ -47,7 +47,8 @@ export interface OutboxPort extends OutboxWriter { export interface SecretStore { resolve(secretRef: string): Promise; create(input: { purpose: SecretPurpose; plaintext: string; label?: string }): Promise; - replace(secretRef: string, plaintext: string): Promise; + /** exec:在调用方事务里写(BR-4 Cookie 与 UA 原子切换);缺省用自己的连接 */ + replace(secretRef: string, plaintext: string, exec?: Kysely): Promise; remove(secretRef: string): Promise; } diff --git a/apps/server/test/sources-identity/migration-browser-identity.test.ts b/apps/server/test/sources-identity/migration-browser-identity.test.ts new file mode 100644 index 00000000..66ddbf00 --- /dev/null +++ b/apps/server/test/sources-identity/migration-browser-identity.test.ts @@ -0,0 +1,100 @@ +import { createMigrator, type DB, migrateToLatest, sql } from "@umt/db"; +import { Kysely, PostgresDialect } from "kysely"; +import pg from "pg"; +import { afterAll, describe, expect, it } from "vitest"; +import { PG_URL, pgAvailable, withSchemaLock } from "../auth-settings/harness.ts"; + +// 0056:存量 Cookie / 账密账号标"待切换"(沿用旧缺省 UA);手动 UA 与 API Key 账号不动;设置里的旧缺省 UA 清掉 = 跟随浏览器。 + +const hasPg = await pgAvailable(); +const LEGACY = + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36"; + +describe.skipIf(!hasPg)("0056 浏览器身份迁移", () => { + const schemas: string[] = []; + const dbs: Kysely[] = []; + const admin = async (q: string) => { + const c = new pg.Client({ connectionString: PG_URL }); + await c.connect(); + await c.query(q).finally(() => c.end()); + }; + afterAll(async () => { + for (const d of dbs) await d.destroy(); + await withSchemaLock(async () => { + for (const s of schemas) await admin(`drop schema if exists ${s} cascade`); + }); + }); + + const setup = async (settingsUa: string | null) => { + const schema = `br4_mig_${process.pid.toString(36)}${Math.random().toString(36).slice(2, 6)}`; + schemas.push(schema); + await admin(`create schema ${schema}`); + const db = new Kysely({ + dialect: new PostgresDialect({ + pool: new pg.Pool({ connectionString: PG_URL, max: 2, options: `-c search_path=${schema},public` }), + }), + }); + dbs.push(db); + await withSchemaLock(async () => { + const r = await createMigrator(db, { migrationTableSchema: schema }).migrateTo("0055_site_kind"); + if (r.error) throw r.error; + }); + await sql`insert into site (key, name, adapter, base_url) values ('audiences', '观众', 'nexusphp', 'https://a.example/')`.execute( + db, + ); + await sql`insert into secret (id, name, purpose, ciphertext, nonce) values + ('00000000-0000-4000-8000-000000000001', 's1', 'site_cookie', 'x', 'x'), + ('00000000-0000-4000-8000-000000000002', 's2', 'site_password', 'x', 'x'), + ('00000000-0000-4000-8000-000000000003', 's3', 'site_api_key', 'x', 'x'), + ('00000000-0000-4000-8000-000000000004', 's4', 'site_cookie', 'x', 'x')`.execute(db); + await sql`insert into site_account (site_id, label, auth_kind, secret_ref, user_agent) values + ((select id from site), 'cookie', 'cookie', '00000000-0000-4000-8000-000000000001', null), + ((select id from site), 'password', 'password', '00000000-0000-4000-8000-000000000002', null), + ((select id from site), 'apikey', 'api_key', '00000000-0000-4000-8000-000000000003', null), + ((select id from site), 'manual', 'cookie', '00000000-0000-4000-8000-000000000004', 'My-UA/1'), + ((select id from site), 'empty', 'cookie', null, null)`.execute(db); + if (settingsUa !== null) + await sql`insert into settings (key, value) values ('sites', ${JSON.stringify({ userAgent: settingsUa, proxyUrl: null })}::jsonb)`.execute( + db, + ); + await withSchemaLock(() => migrateToLatest(db, { migrationTableSchema: schema })); + const accounts = await sql<{ label: string; ua_pending: string | null }>` + select label, ua_pending from site_account order by label`.execute(db); + const settings = await sql<{ + value: Record; + }>`select value from settings where key = 'sites'`.execute(db); + return { db, accounts: Object.fromEntries(accounts.rows.map((r) => [r.label, r.ua_pending])), settings }; + }; + + it("设置仍是旧缺省:Cookie / 账密账号待切换到旧 UA,设置里的 UA 清掉(跟随浏览器)", async () => { + const r = await setup(LEGACY); + expect(r.accounts).toEqual({ apikey: null, cookie: LEGACY, empty: null, manual: null, password: LEGACY }); + expect(r.settings.rows[0]!.value).not.toHaveProperty("userAgent"); + expect(r.settings.rows[0]!.value).toHaveProperty("proxyUrl", null); + const cols = await sql<{ column_name: string }>`select column_name from information_schema.columns + where table_schema = current_schema() and table_name = 'browser_provider' and column_name = 'identity'`.execute( + r.db, + ); + expect(cols.rows).toHaveLength(1); + }); + + it("用户改过全局 UA:保留为全局 UA,存量账号待切换到它;没有设置行时按旧缺省", async () => { + const r = await setup("Custom-UA/9"); + expect(r.accounts.cookie).toBe("Custom-UA/9"); + expect(r.settings.rows[0]!.value).toHaveProperty("userAgent", "Custom-UA/9"); + const r2 = await setup(null); + expect(r2.accounts.password).toBe(LEGACY); + expect(r2.settings.rows).toHaveLength(0); + }); + + it("回滚只删列", async () => { + const r = await setup(LEGACY); + await withSchemaLock(async () => { + const m = await createMigrator(r.db, { migrationTableSchema: schemas.at(-1)! }).migrateTo("0055_site_kind"); + if (m.error) throw m.error; + }); + const cols = await sql<{ column_name: string }>`select column_name from information_schema.columns + where table_schema = current_schema() and column_name in ('ua_pending', 'identity')`.execute(r.db); + expect(cols.rows).toHaveLength(0); + }); +}); diff --git a/apps/server/test/sources-identity/user-agent.test.ts b/apps/server/test/sources-identity/user-agent.test.ts new file mode 100644 index 00000000..8357d461 --- /dev/null +++ b/apps/server/test/sources-identity/user-agent.test.ts @@ -0,0 +1,273 @@ +import { FALLBACK_SITE_USER_AGENT, LEGACY_SITE_USER_AGENT } from "@umt/contracts"; +import { createMockNexusPhpSite, createMockRssServer, type MockNexusPhpSite, type MockRssServer } from "@umt/testkit"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { + readAccountCookie, + registerBrowserCookieReader, + writeBrowserCookie, +} from "../../src/modules/sources-identity/service/credentials.ts"; +import { ingestFeed } from "../../src/modules/sources-identity/service/feeds.ts"; +import { + registerBrowserIdentityReader, + resolveUserAgent, +} from "../../src/modules/sources-identity/service/user-agent.ts"; +import { createHarness, type Harness, pgAvailable, seedFeed } from "./harness.ts"; + +// BR-4 站点请求 UA:每条出站请求都带账号 UA(不再是 Deno 缺省 UA)、优先级(手动 > 待切换 > 全局 > 跟随浏览器)、 +// 平滑迁移(待切换账号沿用旧 UA,重新登录 / 浏览器回写 Cookie 时 Cookie + UA 一起切换)、Cloudflare Cookie 仅浏览器。 + +const up = await pgAvailable(); +const BROWSER_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:152.0) Gecko/20100101 Firefox/152.0"; + +// biome-ignore lint/suspicious/noExplicitAny: 响应体断言 +const json = async (r: Response): Promise => { + const body = await r.json(); + if (!r.ok) throw new Error(`${r.status} ${JSON.stringify(body)}`); + return body.data; +}; + +type RecordedRequest = MockNexusPhpSite["requests"][number]; +const uaOf = (r: RecordedRequest) => String(r.headers["user-agent"] ?? ""); + +describe("resolveUserAgent 优先级", () => { + const d = { global: "", browser: "B" }; + it("手动 > 待切换 > 全局 > 跟随浏览器 > 常量", () => { + expect(resolveUserAgent({ user_agent: "M", ua_pending: "P" }, { global: "G", browser: "B" })).toEqual({ + ua: "M", + source: "manual", + }); + expect(resolveUserAgent({ user_agent: null, ua_pending: "P" }, { global: "G", browser: "B" })).toEqual({ + ua: "P", + source: "pending", + }); + expect( + resolveUserAgent({ user_agent: null, ua_pending: "P" }, { global: "G", browser: "B" }, { ignorePending: true }), + ).toEqual({ ua: "G", source: "global" }); + expect(resolveUserAgent({ user_agent: " ", ua_pending: null }, d)).toEqual({ ua: "B", source: "browser" }); + expect(resolveUserAgent(null, { global: "", browser: "" })).toEqual({ + ua: FALLBACK_SITE_USER_AGENT, + source: "browser", + }); + }); +}); + +describe.skipIf(!up)("BR-4 站点请求 UA 与 Cookie 切换", () => { + let h: Harness; + let mock: MockNexusPhpSite; + let rss: MockRssServer; + let siteId: string; + let accountId: string; + let browserUa: string | null = BROWSER_UA; + let browserCookie: string | null = null; + + const account = () => + h.db.selectFrom("site_account").selectAll().where("id", "=", accountId).executeTakeFirstOrThrow(); + const site = () => h.db.selectFrom("site").selectAll().where("id", "=", siteId).executeTakeFirstOrThrow(); + const apiAccount = async () => json(await h.req(`/site-accounts/${accountId}`)); + const refresh = async () => + (await json(await h.req(`/sites/${siteId}/refresh-login`, { body: { wait: true, accountId } }))).result; + const since = (n: number) => mock.requests.slice(n); + + beforeAll(async () => { + mock = await createMockNexusPhpSite({ username: "alice", password: "s3cret-pw" }); + rss = await createMockRssServer({ + passkey: "PK", + items: [ + { + guid: "br4-1", + title: "Some.Movie.2024.1080p.BluRay-GRP", + link: `${mock.baseUrl}/details.php?id=900001`, + enclosureUrl: `${mock.baseUrl}/download.php?id=900001&passkey=PK`, + size: 1024 ** 3, + pubDate: new Date("2026-09-20T08:00:00Z"), + categories: ["电影"], + }, + ], + }); + h = await createHarness("br4ua"); + registerBrowserIdentityReader(h.ctx, async () => browserUa); + registerBrowserCookieReader(h.ctx, async () => browserCookie); + const s = await json( + await h.req("/sites", { + body: { + key: "audiences", + name: "观众", + adapter: "nexusphp", + baseUrl: `${mock.baseUrl}/`, + rateLimit: { minIntervalMs: 0, concurrency: 10 }, + }, + }), + ); + siteId = s.id; + const r = await json( + await h.req(`/sites/${siteId}/credentials`, { + method: "PUT", + body: { kind: "password", username: "alice", password: "s3cret-pw" }, + }), + ); + accountId = r.siteAccountId; + }); + afterAll(async () => { + await h?.close(); + await mock?.close(); + await rss?.close(); + }); + + it("登录、用户数据、列表 / 搜索 / 详情 / 下载 / 健康检测、RSS:每条请求都带跟随浏览器的 UA", async () => { + const n0 = mock.requests.length; + expect(await refresh()).toMatchObject({ ok: true, status: "active" }); + const adapter = h.rt.siteAdapter(await site()); + const page = await adapter.list(0); + const id = page.items[0]!.siteReleaseId; + await adapter.search("Movie"); + await adapter.detail(id); + const target = await adapter.downloadTarget(id); + expect(target.kind).toBe("torrent"); + expect((await adapter.healthCheck()).ok).toBe(true); + const f = await seedFeed(h, { siteId, url: rss.feedUrl, name: "br4", accountId }); + expect((await ingestFeed(h.rt, f.id)).status).toBe("ok"); + + const reqs = [...since(n0), ...rss.requests]; + const paths = new Set(since(n0).map((r) => r.path.split("?")[0])); + for (const p of ["/takelogin.php", "/torrents.php", "/details.php", "/download.php"]) expect(paths).toContain(p); + expect(rss.requests.length).toBeGreaterThan(0); + for (const r of reqs) { + expect(uaOf(r), `${r.method} ${r.path}`).toBe(BROWSER_UA); + expect(uaOf(r)).not.toMatch(/Deno/i); + } + expect(await apiAccount()).toMatchObject({ + userAgentSource: "browser", + effectiveUserAgent: BROWSER_UA, + userAgent: null, + }); + }); + + it("浏览器身份读不到:退回同型 Windows Firefox 常量;设置里的全局 UA 优先于跟随浏览器,账号手动再优先", async () => { + browserUa = null; + // runtime 缓存浏览器 UA 60 秒:换一个 runtime 读 + (h.rt as unknown as { browserUa: { at: number } }).browserUa.at = 0; + let n = mock.requests.length; + await h.rt.siteAdapter(await site()).healthCheck(); + expect(since(n).map(uaOf)).toEqual(since(n).map(() => FALLBACK_SITE_USER_AGENT)); + + await h.db + .insertInto("settings") + .values({ key: "sites", value: JSON.stringify({ userAgent: "Global-UA/1.0" }) }) + .execute(); + n = mock.requests.length; + await h.rt.siteDefaults(); + await h.rt.siteAdapter(await site()).healthCheck(); + expect(new Set(since(n).map(uaOf))).toEqual(new Set(["Global-UA/1.0"])); + expect(await apiAccount()).toMatchObject({ userAgentSource: "global", effectiveUserAgent: "Global-UA/1.0" }); + + await h.db.updateTable("site_account").set({ user_agent: "Manual-UA/2.0" }).where("id", "=", accountId).execute(); + n = mock.requests.length; + await h.rt.siteAdapter(await site()).healthCheck(); + expect(new Set(since(n).map(uaOf))).toEqual(new Set(["Manual-UA/2.0"])); + expect(await apiAccount()).toMatchObject({ userAgentSource: "manual", userAgent: "Manual-UA/2.0" }); + + await h.db.updateTable("site_account").set({ user_agent: null }).where("id", "=", accountId).execute(); + await h.db.deleteFrom("settings").where("key", "=", "sites").execute(); + await h.rt.siteDefaults(); + browserUa = BROWSER_UA; + (h.rt as unknown as { browserUa: { at: number } }).browserUa.at = 0; + }); + + it("待切换:沿用旧 UA;重新登录用浏览器 UA,会话 Cookie 与 UA 一起切换", async () => { + await h.db + .updateTable("site_account") + .set({ ua_pending: LEGACY_SITE_USER_AGENT }) + .where("id", "=", accountId) + .execute(); + expect(await apiAccount()).toMatchObject({ + userAgentSource: "pending", + effectiveUserAgent: LEGACY_SITE_USER_AGENT, + }); + let n = mock.requests.length; + await h.rt.siteAdapter(await site()).healthCheck(); + expect(new Set(since(n).map(uaOf))).toEqual(new Set([LEGACY_SITE_USER_AGENT])); + + n = mock.requests.length; + expect(await refresh()).toMatchObject({ ok: true }); + const login = since(n).find((r) => r.path.startsWith("/takelogin.php")); + expect(login && uaOf(login)).toBe(BROWSER_UA); + expect((await account()).ua_pending).toBeNull(); + n = mock.requests.length; + await h.rt.siteAdapter(await site()).healthCheck(); + expect(new Set(since(n).map(uaOf))).toEqual(new Set([BROWSER_UA])); + expect(await apiAccount()).toMatchObject({ userAgentSource: "browser" }); + }); + + it("浏览器回写 Cookie:原子切换 UA;cf_clearance / __cf_bm 只留在浏览器,站点请求不用", async () => { + const r = await json( + await h.req(`/sites/${siteId}/credentials`, { + method: "PUT", + body: { kind: "cookie", label: "c", cookie: "c_secure_uid=1; c_secure_pass=old" }, + }), + ); + const cid = r.siteAccountId as string; + await h.db.updateTable("site_account").set({ ua_pending: LEGACY_SITE_USER_AGENT }).where("id", "=", cid).execute(); + + const w = await writeBrowserCookie( + h.ctx, + cid, + "c_secure_uid=1; c_secure_pass=from-browser; cf_clearance=CFSECRET; __cf_bm=BMSECRET", + ); + expect(w).toMatchObject({ changed: true, count: 2, uaSwitched: true, browserOnly: ["__cf_bm", "cf_clearance"] }); + const acc = await readAccountCookie(h.ctx, cid); + expect(acc!.header).toBe("c_secure_pass=from-browser; c_secure_uid=1"); + expect(acc!.header).not.toContain("CFSECRET"); + const row = await h.db.selectFrom("site_account").selectAll().where("id", "=", cid).executeTakeFirstOrThrow(); + expect(row.ua_pending).toBeNull(); + expect(row.user_info).toMatchObject({ browserOnlyCookies: ["__cf_bm", "cf_clearance"] }); + const view = await json(await h.req(`/site-accounts/${cid}`)); + expect(view).toMatchObject({ userAgentSource: "browser", browserOnlyCookies: ["__cf_bm", "cf_clearance"] }); + expect(JSON.stringify(view)).not.toContain("CFSECRET"); + + // 只多了 CF Cookie、其余不变:不算变化 + expect( + await writeBrowserCookie(h.ctx, cid, "c_secure_uid=1; c_secure_pass=from-browser; cf_clearance=OTHER; __cf_bm=X"), + ).toMatchObject({ changed: false }); + // Cookie 不变但仍待切换:只切 UA + await h.db.updateTable("site_account").set({ ua_pending: LEGACY_SITE_USER_AGENT }).where("id", "=", cid).execute(); + expect(await writeBrowserCookie(h.ctx, cid, "c_secure_uid=1; c_secure_pass=from-browser")).toMatchObject({ + changed: false, + uaSwitched: true, + }); + expect( + (await h.db.selectFrom("site_account").select("ua_pending").where("id", "=", cid).executeTakeFirstOrThrow()) + .ua_pending, + ).toBeNull(); + await h.req(`/site-accounts/${cid}`, { method: "DELETE" }); + }); + + it("refresh-login 从浏览器读回的 Cookie 也剔除 CF Cookie;换主凭证结束待切换", async () => { + await h.db + .updateTable("site_account") + .set({ ua_pending: LEGACY_SITE_USER_AGENT }) + .where("id", "=", accountId) + .execute(); + await json( + await h.req(`/sites/${siteId}/credentials`, { + method: "PUT", + body: { kind: "cookie", cookie: "c_secure_uid=1; c_secure_pass=stale" }, + }), + ); + expect((await account()).ua_pending).toBeNull(); + await h.db + .updateTable("site_account") + .set({ ua_pending: LEGACY_SITE_USER_AGENT }) + .where("id", "=", accountId) + .execute(); + browserCookie = `${mock.cookie}; cf_clearance=CFSECRET`; + const n = mock.requests.length; + expect(await refresh()).toMatchObject({ ok: true, status: "active" }); + const acc = await readAccountCookie(h.ctx, accountId); + expect(acc!.header).not.toContain("cf_clearance"); + expect((await account()).ua_pending).toBeNull(); + // 验证浏览器 Cookie 的请求用浏览器 UA,且没有带 cf_clearance + const withBrowserCookie = since(n).filter((r) => String(r.headers.cookie ?? "").includes("mock")); + for (const r of since(n)) expect(String(r.headers.cookie ?? "")).not.toContain("CFSECRET"); + expect(withBrowserCookie.every((r) => uaOf(r) === BROWSER_UA)).toBe(true); + }); +}); diff --git a/apps/web/app/components/settings/sites.tsx b/apps/web/app/components/settings/sites.tsx index 0529f0ce..761c765d 100644 --- a/apps/web/app/components/settings/sites.tsx +++ b/apps/web/app/components/settings/sites.tsx @@ -55,7 +55,8 @@ const DEFAULTS_CARD: CardSpec = { name: "sites.userAgent", label: "User-Agent", kind: "text", - hint: "账号没填 UA 时，登录和拉取用户数据用这个", + placeholder: "跟随浏览器", + hint: "留空跟随浏览器（推荐），站点请求和浏览器用同一个 UA。填了则账号没单独填 UA 时都用它，和浏览器不一致", wide: true, }, { @@ -96,7 +97,8 @@ const DEFAULTS_CARD: CardSpec = { defaultSnapshotMaxPages: num(v["sites.defaultSnapshotMaxPages"]), defaultHealthCheckIntervalSec: healthH === undefined ? undefined : Math.round(healthH * 3600), defaultDetailCacheSec: detailH === undefined ? undefined : Math.round(detailH * 3600), - userAgent: str(v["sites.userAgent"]) || undefined, + // 空串 = 跟随浏览器(要能清空,不能用 undefined 表示"不改") + userAgent: str(v["sites.userAgent"]), proxyUrl: str(v["sites.proxyUrl"]) || null, }, }; diff --git a/apps/web/app/components/sites/credential-fields.tsx b/apps/web/app/components/sites/credential-fields.tsx index b1cb5c7b..033d97ae 100644 --- a/apps/web/app/components/sites/credential-fields.tsx +++ b/apps/web/app/components/sites/credential-fields.tsx @@ -234,10 +234,15 @@ export function CredentialFields({ set("userAgent", e.target.value)} disabled={disabled} diff --git a/apps/web/app/components/sites/site-accounts-tab.tsx b/apps/web/app/components/sites/site-accounts-tab.tsx index 814ef84d..91394541 100644 --- a/apps/web/app/components/sites/site-accounts-tab.tsx +++ b/apps/web/app/components/sites/site-accounts-tab.tsx @@ -10,6 +10,7 @@ import { FormField } from "~/components/console/form-field.tsx"; import { KvList } from "~/components/console/kv-list.tsx"; import { SecretInput } from "~/components/console/secret-input.tsx"; import { StatusBadge } from "~/components/console/status-badge.tsx"; +import { TechInfo } from "~/components/console/tech-info.tsx"; import { Button } from "~/components/ui/button.tsx"; import { relTime } from "~/components/workbench/release-fields.ts"; import { errorMessage } from "~/lib/api.ts"; @@ -143,7 +144,7 @@ function AccountBlock({ label: "会话 Cookie", value: a.authKind === "password" ? (a.hasSession ? "已保存" : "还没有") : null, }, - { label: "User-Agent", value: a.userAgent ?? "站点默认", wide: true, mono: !!a.userAgent }, + { label: "User-Agent", value: UA_SOURCE_TEXT[a.userAgentSource], wide: true }, ...(a.lastError ? [ { @@ -159,12 +160,32 @@ function AccountBlock({ : []), ]} /> + ); } +/** 站点请求 UA 的来源(BR-4);原始 UA 在技术信息里 */ +const UA_SOURCE_TEXT: Record = { + browser: "跟随浏览器", + manual: "手动填写", + global: "使用设置里的全局 UA", + pending: "待切换：下次刷新登录，或在原站视图里用过后改为跟随浏览器", +}; + const ACTION_LABEL: Record = { "site_account.login_refreshed": "刷新登录", "site_account.login_refresh_requested": "请求刷新登录", diff --git a/packages/adapters/src/rss/fetch.ts b/packages/adapters/src/rss/fetch.ts index cb555cb3..658c4bc0 100644 --- a/packages/adapters/src/rss/fetch.ts +++ b/packages/adapters/src/rss/fetch.ts @@ -28,6 +28,10 @@ export interface FetchFeedOptions { signal?: AbortSignal; /** 手动刷新:忽略退避窗口(不忽略熔断) */ ignoreBackoff?: boolean; + /** 本次请求的 UA(覆盖 RssFetcherOptions.userAgent;BR-4 按站点账号取) */ + userAgent?: string; + /** 本次请求用的 fetch(覆盖 RssFetcherOptions.fetch;BR-4 站点代理 = 与站点其它请求同一出口) */ + fetch?: typeof globalThis.fetch; } export function initialFeedState(): FeedFetchState { @@ -158,7 +162,8 @@ export function createRssFetcher(options: RssFetcherOptions = {}): RssFetcher { const headers: Record = { Accept: "application/rss+xml, application/atom+xml, application/xml;q=0.9, text/xml;q=0.8, */*;q=0.1", }; - if (options.userAgent) headers["User-Agent"] = options.userAgent; + const ua = opts.userAgent ?? options.userAgent; + if (ua) headers["User-Agent"] = ua; if (state.etag) headers["If-None-Match"] = state.etag; if (state.lastModified) headers["If-Modified-Since"] = state.lastModified; @@ -166,7 +171,7 @@ export function createRssFetcher(options: RssFetcherOptions = {}): RssFetcher { const signal = opts.signal ? AbortSignal.any([opts.signal, timeout]) : timeout; let res: Response; try { - res = await doFetch(url, { headers, signal, redirect: "follow" }); + res = await (opts.fetch ?? doFetch)(url, { headers, signal, redirect: "follow" }); } catch (e) { if (opts.signal?.aborted) throw e; const err = e as Error; diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 68ea6c14..e0731fe4 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -254,8 +254,15 @@ export const MetadataSettings = z.object({ }); export type MetadataSettings = z.infer; -/** 固定桌面 Chrome UA(账号未设 UA 时使用) */ -export const DEFAULT_SITE_USER_AGENT = +/** + * 读不到浏览器身份(没有浏览器服务 / 旧镜像 / 暂时连不上)时站点请求用的 UA:与浏览器身份同型的 Windows Firefox + * (camofox 1.17.0 = Camoufox 152;升级浏览器镜像时同步改这里,BR-4)。 + */ +export const FALLBACK_SITE_USER_AGENT = + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:152.0) Gecko/20100101 Firefox/152.0"; + +/** BR-4 之前的缺省 UA(Windows Chrome 140):迁移时据此把存量账号标"待切换",设置里的旧缺省值清空改为跟随浏览器 */ +export const LEGACY_SITE_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36"; /** 站点默认(site-center §8):站点行的限流 / 快照周期为空时继承;站点和凭证在站点页管理 */ @@ -268,8 +275,10 @@ export const SitesSettings = z.object({ .int() .min(300) .max(7 * 24 * 3600), - /** 账号未设 UA 时使用 */ - userAgent: z.string().min(1).max(500), + /** + * 全局 UA 覆盖:空串 = 跟随浏览器(缺省,BR-4);填了则账号没有手动 UA 时都用它(浏览器仍是自己的 UA,两端会不一致) + */ + userAgent: z.string().trim().max(500), /** 站点开"使用代理"时生效;null = 不用代理 */ proxyUrl: z.url().nullable(), /** 用户数据做种分页上限 */ @@ -645,7 +654,7 @@ export const DEFAULT_SETTINGS: GlobalSettings = { sites: { defaultRateLimit: { requestsPerMinute: 20, concurrency: 2 }, defaultSnapshotIntervalSec: 3600, - userAgent: DEFAULT_SITE_USER_AGENT, + userAgent: "", proxyUrl: null, userDataMaxSeedingPages: 10, defaultSnapshotMaxPages: 1, diff --git a/packages/contracts/src/sources.ts b/packages/contracts/src/sources.ts index 81c5da88..c45167b8 100644 --- a/packages/contracts/src/sources.ts +++ b/packages/contracts/src/sources.ts @@ -252,8 +252,17 @@ export const SiteAccount = EntityBase.extend({ userInfo: JsonObject, /** 账密登录名 */ loginName: z.string().nullable(), - /** 空 = 全局默认 UA(settings.sites.userAgent) */ + /** 账号手动填的 UA;空 = 不覆盖(见 userAgentSource / effectiveUserAgent) */ userAgent: z.string().nullable(), + /** + * 当前生效 UA 的来源(BR-4):browser = 跟随浏览器;manual = 账号手动填写;global = 设置里的全局 UA; + * pending = 待切换(沿用取得当前 Cookie 时的旧 UA,下次重新登录或浏览器回写 Cookie 时换成浏览器 UA) + */ + userAgentSource: z.enum(["browser", "manual", "global", "pending"]), + /** 当前站点请求实际发送的 UA(技术信息) */ + effectiveUserAgent: z.string(), + /** 仅浏览器用、没有回写给站点请求的 Cookie 名(Cloudflare 验证 Cookie,不含值) */ + browserOnlyCookies: z.array(z.string()), /** 最近一次登录 / 校验成功(Cookie、API Key 校验成功也算) */ lastLoginAt: Timestamp.nullable(), lastError: SiteAccountError.nullable(), diff --git a/packages/contracts/test/db-sync.test.ts b/packages/contracts/test/db-sync.test.ts index a6ee807e..bb763cea 100644 --- a/packages/contracts/test/db-sync.test.ts +++ b/packages/contracts/test/db-sync.test.ts @@ -83,8 +83,9 @@ const ROWS: Record SiteUserData: { row: C.SiteUserData }, SiteAccount: { row: C.SiteAccount, - dbOnly: ["totpSecretRef", "sessionSecretRef", "passkeySecretRef"], - apiOnly: ["hasTotp", "hasSession", "hasPasskey"], + // BR-4:ua_pending 只给解析后的 userAgentSource / effectiveUserAgent;browserOnlyCookies 取自 user_info + dbOnly: ["totpSecretRef", "sessionSecretRef", "passkeySecretRef", "uaPending"], + apiOnly: ["hasTotp", "hasSession", "hasPasskey", "userAgentSource", "effectiveUserAgent", "browserOnlyCookies"], }, SourceRelease: { row: C.SourceRelease }, Subscription: { row: C.Subscription, apiOnly: ["mediaEntityTitle"] }, @@ -100,7 +101,8 @@ const ROWS: Record ResolutionAttempt: { row: C.ResolutionAttempt }, SiteSnapshot: { row: C.SiteSnapshot }, // W4-C 浏览器实体(0023) - BrowserProvider: { row: C.BrowserProviderRow }, + // BR-4:identity 是浏览器固定身份的缓存(站点请求 UA 用),不进 API + BrowserProvider: { row: C.BrowserProviderRow, dbOnly: ["identity"] }, BrowserSession: { row: C.BrowserSession }, BrowserTab: { row: C.BrowserTab }, BrowserControlLease: { row: C.BrowserControlLease }, diff --git a/packages/db/migrations/0056_browser_identity.ts b/packages/db/migrations/0056_browser_identity.ts new file mode 100644 index 00000000..6330e0e4 --- /dev/null +++ b/packages/db/migrations/0056_browser_identity.ts @@ -0,0 +1,47 @@ +import { type Kysely, sql } from "kysely"; +import { execDdl } from "../src/migration-helpers.ts"; + +// BR-4 浏览器 UA 与指纹统一(docs/runbook/browser.md §10): +// - site_account.ua_pending:待切换。存量 Cookie / 账密账号的登录态是用旧缺省 UA(Windows Chrome 140)取得的,不立即换 UA, +// 记下旧 UA 继续用;下次重新登录或浏览器回写 Cookie 时与 Cookie 一起切到浏览器 UA(清空本列)。 +// 账号手动填了 UA 的不动(手动优先);API Key / Passkey 账号与网页登录态无关,直接跟随浏览器。 +// - browser_provider.identity:浏览器服务固定身份的公开部分(UA / OS / 版本,umt 插件 GET /umt/identity)+ 读取时间。 +// - settings.sites.userAgent:仍是旧缺省值(Chrome 140)时删掉该键 → 新缺省空串 = 跟随浏览器;用户改过的保留为全局 UA。 +// down 只删列;settings 不回填(旧版本读到缺省即 Chrome 140)。 + +/** BR-4 之前的缺省 UA(与 contracts LEGACY_SITE_USER_AGENT 相同;迁移不依赖应用包) */ +export const LEGACY_UA = + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36"; + +// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型 +export async function up(db: Kysely): Promise { + await execDdl( + db, + `alter table site_account add column ua_pending text; + alter table browser_provider add column identity jsonb;`, + ); + const row = await db.selectFrom("settings").select("value").where("key", "=", "sites").executeTakeFirst(); + const stored = (row?.value as { userAgent?: unknown } | undefined)?.userAgent; + const custom = typeof stored === "string" && stored.trim() && stored.trim() !== LEGACY_UA ? stored.trim() : null; + // 存量账号取得登录态时用的 UA:设置里改过就是改过的值,否则是旧缺省 + const previous = custom ?? LEGACY_UA; + await db + .updateTable("site_account") + .set({ ua_pending: previous }) + .where("auth_kind", "in", ["cookie", "password"]) + .where((eb) => eb.or([eb("user_agent", "is", null), eb("user_agent", "=", "")])) + .where((eb) => eb.or([eb("secret_ref", "is not", null), eb("session_secret_ref", "is not", null)])) + .execute(); + if (typeof stored === "string" && stored.trim() === LEGACY_UA) { + await db.updateTable("settings").set({ value: sql`value - 'userAgent'` }).where("key", "=", "sites").execute(); + } +} + +// biome-ignore lint/suspicious/noExplicitAny: 同上 +export async function down(db: Kysely): Promise { + await execDdl( + db, + `alter table site_account drop column if exists ua_pending; + alter table browser_provider drop column if exists identity;`, + ); +} diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index e83dd8df..61c0b627 100644 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -569,6 +569,7 @@ export interface BrowserProviderTable { config: Generated; created_at: GeneratedTimestamptz; updated_at: GeneratedTimestamptz; + identity: Json | null; } export interface BrowserSessionTable { @@ -1409,6 +1410,7 @@ export interface SiteAccountTable { last_error: Json | null; user_data: Json | null; user_data_at: Timestamptz | null; + ua_pending: string | null; } export interface SiteRequestCountTable { From 9ea0d6bed16febfd3b9919fb276d2a69b5981196 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 30 Sep 2026 03:17:13 +0800 Subject: [PATCH 3/3] docs(browser): BR-4 runbook for browser identity, UA priority, migration, CF cookies, egress Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 11 +++ docs/plan/00-first-version-plan.md | 1 + docs/runbook/browser.md | 147 ++++++++++++++++++++++++++++- docs/runbook/sites.md | 4 +- 4 files changed, 160 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5b09160..49b1f99a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,17 @@ ## [Unreleased] +### 新增(BR-4 浏览器 UA 与指纹统一) +- 固定浏览器身份:浏览器镜像的 umt-cookies 插件首次启动时生成一套 Windows 指纹(UA、navigator、屏幕、WebGL、Windows 字体、随机种子)存进持久卷(`/root/.camofox/umt-identity.json`),之后每次启动复用,不再每次随机;Camoufox / Firefox 升级时只改 UA 与版本字段。修正上游字体配置路径,Windows 字体(含微软雅黑)按 Windows 渲染。新增 `GET /umt/identity`(访问凭据保护,只含 UA / OS / 版本 / 时间)。 +- 站点请求 UA 跟随浏览器:列表 / 搜索 / 详情 / 下载种子 / 健康检测、RSS、登录、用户数据都带账号 UA,不再发 `Deno/2.9.7`。优先级:账号手动 > 待切换 > 设置里的全局 UA > 跟随浏览器(读不到浏览器身份时用同型 Windows Firefox UA)。站点账号页显示 UA 来源(跟随浏览器 / 手动填写 / 待切换),原始 UA 在技术信息里。 +- 出口按站点统一:站点开了"使用代理"时浏览器会话也走同一代理,没开时直连;浏览器不再使用容器环境变量里的代理(`UMT_BROWSER_PROXY_MODE=env` 可退回)。插件新增 `PUT /umt/sessions/{userId}/proxy`。 +- Cloudflare 验证 Cookie(`cf_clearance`、`__cf_bm`)只留在浏览器:回写站点账号时剔除,只记名字,站点请求不用。 + +### 变更(BR-4) +- 设置 → 站点默认 → User-Agent 缺省改为空(跟随浏览器),原缺省 Windows Chrome 140 去掉;挂站点的 RSS 与站点其它请求同一出口(站点开代理时 RSS 也走代理)。 +- 迁移 0056:存量 Cookie / 账密账号标"待切换",继续用取得登录态时的旧 UA,下次重新登录或浏览器回写 Cookie 时 Cookie 与 UA 一起切换;设置里仍是旧缺省 UA 的清空。 +- 上线需重建 app 与 camofox(`docker compose build app camofox && docker compose up -d app camofox`);camofox 只有插件层变化,不重新下载浏览器包。见 `docs/runbook/browser.md` §11。 + ## [0.3.1] - 2026-09-30 ### 新增 diff --git a/docs/plan/00-first-version-plan.md b/docs/plan/00-first-version-plan.md index fb702e7d..fe1b5af0 100644 --- a/docs/plan/00-first-version-plan.md +++ b/docs/plan/00-first-version-plan.md @@ -199,6 +199,7 @@ NFO / 图片投影、身份巡检与挂载根因聚合、HTTP 失败自动切浏 - 偏离与补充:获取方式比任务单多一项 `list`(列表抓取 / 快照,调度需要区分);"计划"仍在概览组(用户只提了任务);公开 BT 站目前没有站点定义,实际只有 RSS(有定义后自动获得搜索 / 列表);`POST /feed-sources` 不带站点时自动归入同主机的自定义 RSS 源(兼容旧调用方,而不是 400)。截图 `apps/web/test/screenshots/ia2/`(不进 git)。 - **浏览器 UA 与指纹(BR-4,用户 2026-09-30:不着急,先研究明白)**:现状 HTTP 默认 Windows Chrome 140 UA、浏览器是 camofox(Firefox 内核)自生成 UA,同一站点账号 Cookie 两边互注 → UA 不一致。方向:每个站点账号一套固定 UA(跟随 camofox 的 Firefox)+ 固定指纹,浏览器与 HTTP 共用,不轮换,浏览器升级时同步。先做只读调研(camofox 能否按会话指定/持久化指纹、各站会话与 UA 绑定情况、MP 做法),出报告再定实现。 - 调研见 `docs/design/browser-ua-research.md`。用户决定(2026-09-30):对外伪装 **Windows**;全实例**一套固定指纹**(插件 `browser:launching` 首次生成落盘、之后复用);浏览器与 HTTP **按站点统一出口 IP**;站点适配器 / RSS 目前发 `Deno/2.9.7` UA 的问题**和 BR-4 一起做**(不单独先修)。cf_clearance 等 CF Cookie 回写时标"仅浏览器"。平滑迁移:下次重新登录 / 浏览器回写 Cookie 时原子切换 Cookie+UA。 + - **已实现(br4 分支,待合并)**:固定 Windows 身份(插件落盘 `/root/.camofox/umt-identity.json`、`GET /umt/identity`)、站点请求 UA 全链路跟随浏览器(优先级 手动 > 待切换 > 全局 > 浏览器)、迁移 0056 待切换、CF Cookie 仅浏览器、出口按站点(浏览器按上下文代理,不再用容器环境变量代理)。运维与限制见 `docs/runbook/browser.md` §11;上线需重建 app + camofox。 - **下载计划并入下载面板(DL-3,用户 2026-09-30 同意)**:acquisition_plan 后台保留(批准绑定、预算/空间预占、防重复、升级/换源追踪);界面去掉「下载计划」页签——等批准/等空间/等预算的计划作为「进行中」行显示(需操作的同时进待处理);已提交的计划并入对应下载任务,详情加「为什么下这个」;失败/过期进下载历史,细节进技术信息;用户可见处不再出现"下载计划",统一说"下载"。 - **文件预览不加 ffmpeg(用户 2026-09-30)**:视频缩略图 / 截图条不做,镜像不引入 ffmpeg(+141MB);`docs/design/file-preview-research.md` 对应项标为不做。 - 用户 2026-09-30:BR-4 直接做完上线;《征途》重复单元不再处理(刷新时自动合并即可)。 diff --git a/docs/runbook/browser.md b/docs/runbook/browser.md index 32510d83..cedd7e87 100644 --- a/docs/runbook/browser.md +++ b/docs/runbook/browser.md @@ -365,6 +365,8 @@ camofox 1.17.0 只有 `POST /sessions/:userId/cookies`(导入),而且要另一 | `POST /umt/sessions/{userId}/cookies` `{cookies}` | `context.addCookies`,并触发持久化插件立即落盘(与上游导入端点相同) | | `GET /umt/sessions/{userId}/cookies?urls=a,b` | 作用于这些地址的 Cookie:会话活着读 `context.cookies(urls)`,否则读 profile 落盘文件 | | `GET /umt/instance` | `{instanceId, startedAt}`:插件加载时生成,camofox 进程重启即变(BR-3 重启检测,§10) | +| `GET /umt/identity` | 固定浏览器身份的公开部分(UA / OS / 版本,BR-4,§11.1) | +| `PUT /umt/sessions/{userId}/proxy` `{proxy}` | 该 profile 的出口:`{server, username?, password?}` 或 `null` 直连(BR-4,§11.5) | 适配器先走插件;插件不在(404)再试上游导入端点(`CAMOFOX_API_KEY` 与访问凭据相同时可用,只能写不能读);都不行 → 会话照常打开, `credential.status = unsupported` 并说明。profile 落盘在 `/root/.camofox/profiles//storage-state.json` @@ -393,7 +395,8 @@ camofox 1.17.0 只有 `POST /sessions/:userId/cookies`(导入),而且要另一 | 定时 | 会话有动作时,距上次读回(或创建)满 10 分钟,后台读回一次 | | refresh-login 发现 Cookie 失效 | Cookie 账号:从浏览器 profile 读回(不要求会话开着,读落盘文件),与存的不同就拿它验证一次,有效才回写并标 active | -读回只取作用于站点主机的、未过期的 Cookie(统计 / CDN / 第三方域名一律丢弃),与存储比较(忽略顺序),有变化才写: +读回只取作用于站点主机的、未过期的 Cookie(统计 / CDN / 第三方域名一律丢弃),剔除 Cloudflare 的 `cf_clearance` / `__cf_bm`(仅浏览器,§11.4), +与存储比较(忽略顺序),有变化才写(同时结束 UA 待切换,§11.3): Cookie 账号替换 `secret_ref`,账密账号替换(或新建)会话 Cookie;`site_account.user_info.cookieOrigin = {source: "browser", by: "browser", at}`, `expired` / `invalid` 的账号回到 `unknown`。来源判定:`cookieOrigin.at` 不早于 secret 最近写入(`rotated_at`,留 5 秒)才算浏览器回写, 之后再手填或自动登录会让它变回 `manual` / `login`。 @@ -460,3 +463,145 @@ camofox 容器重启(`docker compose up -d` 重建、OOM、手动 restart)后, - 真机:未对真实 camofox 容器重启核验。桌面镜像要重建后才有 `GET /umt/instance`(`docker compose build camofox && docker compose up -d camofox`); 重建前只有被动发现(截图 / 操作时自愈),桌面模式要等一次截图或 Agent 操作才会重连。 + +## 11. 浏览器身份、站点请求 UA 与出口(BR-4) + +同一个站点账号的登录态在浏览器(camofox / Firefox)和 HTTP 端(Deno)之间互注(§9),所以两端要**看起来是同一台设备**: +同一个 UA、稳定的指纹、同一个出口 IP。用户决定(2026-09-30):对外伪装 **Windows**;全实例**一套固定指纹**; +浏览器与 HTTP **按站点统一出口**。调研见 `docs/design/browser-ua-research.md`。 + +代码:插件 `docker/browser-desktop/root/opt/camofox/plugins/umt-cookies/{index.js,identity.js}`; +`apps/server/src/modules/sources-identity/{runtime.ts,service/user-agent.ts,service/credentials.ts,service/account.ts}`; +`apps/server/src/modules/browser/service.ts`(`identityUserAgent` / `refreshIdentity` / `applyEgress`);迁移 `0056_browser_identity`。 + +### 11.1 固定浏览器身份(插件 `browser:launching`) + +- **首次启动**:插件用 camoufox-js 的 `launchOptions({ os: "windows" })` 生成一份完整指纹——BrowserForge 的 Windows Firefox + 指纹(navigator / screen / 硬件)、Camoufox 的 Windows 字族表、按 Windows 取样的 WebGL(vendor / renderer / 参数)、 + 字体间距 / 音频 / canvas 随机种子、`window.history.length`、canvas 抗锯齿偏移——去掉运行期字段(`humanize`、`locale:*`、 + `geolocation:*`、`timezone`、`webrtc:*`、`addons`)后落盘 **`/root/.camofox/umt-identity.json`**(compose 的持久卷 `umt-camofox`, + 与 profile 同一个卷;属主 abc、0600)。WebGL2 开关等配套 Firefox 首选项一起存。 +- **之后每次启动**(容器重启、浏览器空闲 5 分钟退出后再启动):把上游随机生成的 config 里的指纹部分整份换成身份文件里的, + 运行期字段仍取上游。浏览器进程日志 `umt identity applied {os, fingerprintId, revision}`。 +- **升级**:Camoufox 版本或 Firefox 主版本与文件里记的不同 → 只改 `navigator.*` / `headers.*` 里的版本号(与 camoufox-js 生成时 + 同一个替换规则),新版本多出的指纹字段从同 OS 新样本补齐,其余一律保留;`revision + 1`,`fingerprintId` 不变 + (日志 `umt identity upgraded`)。站点看到的是"同一台电脑升级了 Firefox"。 +- **重置**(换一台"新设备",所有站点都会看到新设备):停 camofox,删 `umt-identity.json`,再启动。 +- **开关与路径**:`UMT_BROWSER_IDENTITY=off` 关掉(回到上游每次随机的 Linux 指纹);`UMT_IDENTITY_FILE` / `UMT_STATE_DIR` 改路径。 + 文件损坏会改名 `.bad-<时间>` 后重新生成。 +- **`GET /umt/identity`**(访问凭据保护):`{userAgent, os, platform, firefoxVersion, camoufoxVersion, fingerprintId, revision, + schema, createdAt, updatedAt}`,不含指纹细节。app 存进 `browser_provider.identity`(加 `fetchedAt`):打开会话时(10 分钟内读过就不读)、 + 重连时、设置页探测浏览器服务时读;站点请求取它时超过 6 小时会重读,读不到 5 分钟内不再试。 + +**Windows 伪装的一致性**(camoufox-js 0.11.5 源码 + 2026-09-30 本地临时容器实测,只开本地页面): + +| 项 | 结果 | +| --- | --- | +| UA / 请求头 | `Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:152.0) Gecko/20100101 Firefox/152.0`;HTTP `User-Agent` 头同值 | +| navigator | `platform = Win32`、`oscpu = Windows NT 10.0; Win64; x64`;并发数、语言等来自同一份 Windows 指纹 | +| WebGL | 按 Windows 取样:`Google Inc. (Intel)` / `ANGLE (Intel, Intel(R) HD Graphics 400 Direct3D11 vs_5_0 ps_5_0)`(D3D11,Windows 才有) | +| 字体 | 字族白名单 = Camoufox 的 Windows 字体表;字形来自 Camoufox 自带的 Windows 字体(`/opt/camoufox/fonts/windows`:Arial、Segoe UI、微软雅黑、宋体、等线……)。实测"微软雅黑"可用且宽度与回退字体不同,Noto / DejaVu 这类 Linux 字体不可见;中文页面以微软雅黑渲染 | +| 跨重启 | 重启容器后 `fingerprintId`、屏幕尺寸、WebGL 不变;模拟 140 → 152 升级只改了 UA 版本 | + +发现并修正的上游问题:camoufox-js 0.11.5 把 `FONTCONFIG_PATH` 设成 `/opt/camoufox/fontconfig/lin`(不存在;Camoufox 152 的目录是 +`linux` / `macos` / `windows`),而 fontconfig 里的字体目录写的是相对当前目录的 `fonts`(浏览器进程当前目录是 `/opt/camofox`)。 +插件把 `fontconfig/windows/fonts.conf` 复制到临时目录、字体目录改成 `/opt/camoufox/fonts` 绝对路径,再指给 `FONTCONFIG_PATH`。 + +限制:渲染仍是 Linux 上的 Firefox——字体光栅化、canvas / 音频的像素级结果与真 Windows 不同(Camoufox 用固定种子加噪,稳定但不是 +Windows 的真值);TLS / HTTP2 指纹是 Firefox 152 本身(与 OS 无关)。屏幕尺寸来自指纹(如 2560×1440),桌面模式窗口是 1280×900, +相当于没最大化的窗口。时区跟容器 `TZ`(缺省 Asia/Shanghai);语言跟指纹(缺省 en-US),要中文设 `CAMOFOX_LOCALE`。 +指纹是进程级的:所有站点 / 账号共用这一套(用户决定;每账号一套要每账号一个浏览器进程,不做)。 + +### 11.2 站点请求 UA + +站点请求的 UA 按账号解析,**优先级从高到低**: + +| 来源(`SiteAccount.userAgentSource`) | 取值 | 界面 | +| --- | --- | --- | +| `manual` | 账号手动填的 UA(`site_account.user_agent`) | 手动填写 | +| `pending` | 待切换:取得当前登录态时用的旧 UA(`site_account.ua_pending`,§11.3) | 待切换:… | +| `global` | 设置 → 站点默认 → User-Agent(`settings.sites.userAgent`,非空时) | 使用设置里的全局 UA | +| `browser` | 浏览器固定身份的 UA(`browser_provider.identity.userAgent`);读不到时用常量 `FALLBACK_SITE_USER_AGENT`(同型 Windows Firefox 152) | 跟随浏览器 | + +- 覆盖面:站点适配器(列表 / 搜索 / 详情 / 下载 .torrent / 健康检测)、RSS、账密登录、用户数据、Cookie 失效时的浏览器读回验证—— + 全部经 `SourcesRuntime.siteFetch`:请求自己没设 UA 就补上;适配器每条请求现场按"与站点凭证同一个账号"(第一个启用、未失效、非 Passkey) + 取 UA,与 Cookie 同一时刻读库,不缓存。之前适配器与 RSS 发的是 Deno 缺省 UA `Deno/2.9.7`,已不再出现 + (测试 `apps/server/test/sources-identity/user-agent.test.ts` 逐条断言)。不挂站点的请求(通用 RSS)用全局 UA / 跟随浏览器。 +- 设置里的 User-Agent **缺省留空 = 跟随浏览器**(原缺省 Windows Chrome 140 已去掉);填了会让 HTTP 与浏览器 UA 不一致,只在确有需要时用。 +- 账号"更新凭证"里的 UA 留空 = 不覆盖;Cookie 是从别的浏览器复制的、且站点校验 UA 时才填那个浏览器的 UA。 +- 不在范围:TMDB、图片代理(不带凭据的公开请求)、下载器、浏览器服务本身。 +- 升级 camofox 镜像(Firefox 大版本变)时:浏览器身份自动升级(§11.1),app 最迟 6 小时(或下次打开会话 / 重连)跟上; + 同时把 `packages/contracts/src/settings.ts` 的 `FALLBACK_SITE_USER_AGENT` 改成新版本号。 + +### 11.3 平滑迁移(待切换) + +改 UA 一般不会让 NexusPHP 的 Cookie 失效,但前置 Cloudflare / WAF 的站点可能要求重新过挑战。所以**不立即给存量账号换 UA**: + +- 迁移 `0056_browser_identity`:Cookie / 账密账号(有凭证、没手动 UA)记 `ua_pending` = 取得当前登录态时的 UA——设置里改过全局 UA + 就是那个值,否则是旧缺省 Chrome 140;API Key / Passkey 账号与网页登录态无关,直接跟随浏览器;设置里的 UA 仍是旧缺省值时删掉 + (= 跟随浏览器),改过的保留为全局 UA。 +- 待切换期间,这个账号的所有站点请求(包括原来发 Deno UA 的适配器 / RSS)都用旧 UA,与登录态一致。 +- **切换时机**(Cookie 与 UA 同一事务写,下一条站点请求两者一起生效): + 1. 重新登录:账密账号 refresh-login 走 takelogin 时用切换后的 UA,拿到会话 Cookie 后清 `ua_pending`; + 2. 浏览器回写 Cookie(§9.2 的交还 / 关闭 / 定时 / refresh-login 读回):浏览器是用自己的 UA 取得 / 使用这份登录态的,回写时清 `ua_pending`; + Cookie 没变但浏览器不在登录页、正用着这份登录态,也清; + 3. 换主凭证(填新 Cookie / 新密码):新登录态,直接跟随浏览器。 +- Cookie 账号如果一直不开原站视图、也不换 Cookie,会一直待切换(界面写"待切换:下次刷新登录,或在原站视图里用过后改为跟随浏览器")。 + 刷新登录对 Cookie 账号只校验,不切换。 + +### 11.4 Cloudflare Cookie 仅浏览器 + +`cf_clearance`、`__cf_bm`(及挑战过程中的 `cf_chl_*`)与取得它的浏览器绑定(UA + IP + TLS 指纹),Deno 的 TLS 指纹与 Firefox 必然不同, +HTTP 端拿去用多半无效还可能触发风控。所以: + +- 浏览器回写账号时剔除这些 Cookie,只在 `site_account.user_info.browserOnlyCookies` 记名字(不含值);它们留在浏览器 profile 里照常用; +- 比较"有没有变化"时也不算它们(CF Cookie 轮换不会触发回写);refresh-login 从浏览器读回的 Cookie 同样剔除; +- 站点账号页技术信息里显示"仅浏览器使用的 Cookie"; +- 用户手填的 Cookie 里带 `cf_clearance` 不处理(用户自己的选择)。 + +### 11.5 出口 IP 按站点统一 + +实测(本地临时容器):**Firefox 会用容器的 `HTTP_PROXY` / `HTTPS_PROXY` 环境变量**(Firefox 缺省"使用系统代理",Linux 下没有 +GSettings 就读环境变量)。所以本机栈此前的状态是:`docker-compose.override.yml` 给 camofox 设了 `HTTP_PROXY=http://172.17.0.1:7891`, +浏览器**所有**站点都走代理;HTTP 端只有开了"使用代理"的站点走代理——同一账号两端出口 IP 不同。 + +现在(`UMT_BROWSER_PROXY_MODE=site`,缺省): + +- 插件在启动时设 `network.proxy.type = 0`:浏览器**不再用**容器环境变量里的代理,缺省直连; +- app 在打开会话 / 重连前调 `PUT /umt/sessions/{userId}/proxy`:站点开了"使用代理"且设置了代理地址 → 该站点 profile 的 + BrowserContext 走同一个代理(Playwright 按上下文代理,Firefox / Juggler 原生支持;URL 里的账号口令拆成 username / password); + 否则 `null` 直连。插件把这张表存在 `/root/.camofox/umt-proxies.json`(0600,代理带口令时含口令),camofox 重启后照用; +- 该 profile 已有活动上下文、出口不同 → 插件先关掉它(持久化插件先落盘登录状态),下一次操作按新出口重建; +- 实测:直连 profile 访问测试域名走不到环境变量里的代理;代理 profile 的请求到了指定代理;改出口时活动上下文被重建。 + +限制与注意: + +- **代理地址要让 app 与 camofox 两个容器都能连**(如本机的 socat 中继 `http://172.17.0.1:7891` 可以;`http://127.0.0.1:7890` 在 + camofox 容器里连不到)。 +- 站点"使用代理"开关或代理地址改了,已打开的浏览器会话在**下一次打开 / 重连**时才按新出口;要立即生效就关掉会话重开。 +- 匿名会话与 Agent 打开的站点会话同样按站点出口(浏览器会话都属于某个站点)。 +- 旧镜像 / 插件没有出口接口:app 只记日志,浏览器照旧按环境变量出网。`UMT_BROWSER_PROXY_MODE=env` 可退回旧行为。 +- SOCKS 代理的 DNS 解析位置未核验(HTTP 代理由代理端解析,与 Deno 的代理客户端一致)。 +- 本机栈重建 camofox 后,憨憨(未开代理)的浏览器会话会从"经代理"变为**直连**,与 HTTP 端一致;override 里 camofox 的 + `HTTP_PROXY` 可以留着(Firefox 不再用它),也可以删掉。 + +### 11.6 上线步骤与核验 + +需要重建 **app**(代码 + 迁移 0056)和 **camofox**(插件): + +```sh +docker compose --env-file .env.deploy build app camofox +docker compose --env-file .env.deploy up -d app camofox +docker compose exec camofox curl -s -H "Authorization: Bearer $CAMOFOX_ACCESS_KEY" http://127.0.0.1:9377/umt/identity +``` + +camofox 镜像只有 `COPY root/ /` 与其后改插件清单的一层变化;前面的 apt 层与从 `camofox-browser:1.17.0` 复制 node / camofox / +Camoufox 浏览器包的层都走构建缓存,**不会重新下载 Camoufox(约 630 MB)**。2026-09-30 在本机用临时标签实测:带 +`--build-arg APT_MIRROR=mirrors.ustc.edu.cn`(与 `.env.deploy` 相同)时 apt / 复制层全部 `CACHED`,只重跑插件两层(< 1 秒)。 +注意**构建参数要与上次一致**(用同一个 `--env-file .env.deploy`):不带 `APT_MIRROR` 构建会让 apt 层缓存失效,重新 apt 安装(浏览器包 +仍从本地 `camofox-browser:1.17.0` 复制,不从网上下)。首次启动生成身份,之后 profile 卷里的登录状态照常可用。 + +核验:`apps/server/test/sources-identity/user-agent.test.ts`(每条出站请求的 UA、优先级、待切换、CF Cookie)、 +`migration-browser-identity.test.ts`、`apps/server/src/modules/browser/__tests__/browser-identity.test.ts`(出口下发、身份存库)、 +`packages/adapters/src/browser/umt-plugin.test.ts`(插件纯函数、适配器接口);真机:本地临时容器(`--network none`,只开容器内本地 +页面)核验了 §11.1 表格与 §11.5 出口行为,未对真实站点发请求。 diff --git a/docs/runbook/sites.md b/docs/runbook/sites.md index 47b71e15..98c993ad 100644 --- a/docs/runbook/sites.md +++ b/docs/runbook/sites.md @@ -96,8 +96,8 @@ | `defaultSnapshotMaxPages` | 1 | 定时 / 手动快照最多抓几页(与定义取较小);补漏有自己的页数预算,实时补查固定 1 页 | | `defaultHealthCheckIntervalSec` | 21600 | 定时健康检测的最短间隔(与定义取较长) | | `defaultDetailCacheSec` | 86400 | 详情缓存:期内 `GET /sites/{id}/releases/{rid}?refresh=true` 与 `release.inspect` 不再请求站点(与定义取较长);0 = 不缓存 | -| `userAgent` | 固定桌面 Chrome UA | 账号未设 UA 时用于登录与用户数据 | -| `proxyUrl` | null | 站点 `proxy=true` 时,适配器、登录、用户数据请求都经此代理(`Deno.createHttpClient`) | +| `userAgent` | 空(跟随浏览器) | 全局 UA 覆盖:空 = 站点请求跟随浏览器固定身份的 UA(Windows Firefox);填了则账号没手动 UA 时都用它。优先级:账号手动 > 待切换 > 全局 > 跟随浏览器,见 browser.md §11.2 | +| `proxyUrl` | null | 站点 `proxy=true` 时,适配器、登录、用户数据、RSS 请求都经此代理(`Deno.createHttpClient`),浏览器会话也走同一代理(browser.md §11.5);地址要让 app 与 camofox 容器都能连 | | `userDataMaxSeedingPages` | 10 | 用户数据做种分页上限 | 运行时每 30 s 最多重读一次设置;设置变化会让站点适配器按新签名重建。