feat(sites): site requests follow browser UA, smooth UA switch, CF cookies browser-only (BR-4)

- every outbound site request (adapter list/search/detail/download/health,
  RSS, login, user data) carries the account UA via SourcesRuntime.siteFetch;
  no more Deno default UA
- priority: account manual > pending > global setting > browser identity
  (fallback: same-shape Windows Firefox UA); settings UA default is empty
- migration 0056: existing cookie/password accounts keep their old UA as
  ua_pending until re-login or browser cookie write-back switches cookie+UA
  in one transaction; browser_provider.identity caches the browser identity
- cf_clearance/__cf_bm stripped on browser write-back (names kept only)
- browser sessions get per-site egress (same proxy as HTTP, or direct)
- site account page shows UA source; raw UA in tech info

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed 2026-09-30 03:17:13 +08:00
1 parent ebd07c9230
commit adc853aa64
21 files changed
+949 -66

No files matched your search

@@ -114,12 +114,16 @@ function open(
*/
export function createSecretStore(deps: { db: Kysely<DB>; masterKey: string }): SecretService {
const { db, masterKey } = deps;
const replace = async (secretRef: string, plaintext: string) => {
const replace = async (secretRef: string, plaintext: string, exec: Kysely<DB> = db) => {
assertRef(secretRef);
const row = await db.selectFrom("secret").select(["id", "purpose"]).where("id", "=", secretRef).executeTakeFirst();
const row = await exec
.selectFrom("secret")
.select(["id", "purpose"])
.where("id", "=", secretRef)
.executeTakeFirst();
if (!row) throw new SecretError("not_found", secretRef);
const s = seal(masterKey, row.id, row.purpose, plaintext);
await db
await exec
.updateTable("secret")
.set({
ciphertext: s.ciphertext,
@@ -0,0 +1,79 @@
import { MOCK_CAMOFOX_IDENTITY } from "@umt/testkit";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { sourcesIdentityModule } from "../../sources-identity/index.ts";
import { readBrowserUserAgent } from "../../sources-identity/service/user-agent.ts";
import { egressProxy } from "../service.ts";
import { type BrowserHarness, createBrowserHarness, type Loose, ORIGIN, pgAvailable } from "./harness.ts";
// BR-4:打开会话前按站点下发出口(站点开代理 = 与 HTTP 同一代理,否则直连);浏览器固定身份写进 browser_provider.identity,
// 站点请求 UA 经钩子跟随它。
const up = await pgAvailable();
describe("egressProxy", () => {
it("URL → 上下文代理(口令拆出);非法为 null", () => {
expect(egressProxy("http://u:p%[email protected]:7891")).toEqual({
server: "http://172.17.0.1:7891",
username: "u",
password: "p@ss",
});
expect(egressProxy("socks5://127.0.0.1:1080")).toEqual({ server: "socks5://127.0.0.1:1080" });
expect(egressProxy("not a url")).toBeNull();
});
});
describe.skipIf(!up)("BR-4 浏览器身份与出口", () => {
let h: BrowserHarness;
const open = async () =>
(await h.req("/browser/sessions", { body: { siteId: h.siteId, accountId: null, url: `${ORIGIN}/torrents.php` } }))
.body.data as Loose;
const closeAll = async () => {
for (const s of (await h.req("/browser/sessions")).body.data.items as Loose[])
await h.req(`/browser/sessions/${s.id}`, { method: "DELETE" });
};
beforeAll(async () => {
h = await createBrowserHarness("br4id");
}, 120_000);
afterAll(async () => {
await h?.close();
});
it("站点不开代理:该 profile 直连;打开时读浏览器身份存库,站点请求 UA 跟随", async () => {
const s = await open();
expect(h.mock.egress.get(s.profileKey)).toBeNull();
const calls = h.mock.calls.map((c) => `${c.method} ${c.path}`);
expect(calls.indexOf(`PUT /umt/sessions/${encodeURIComponent(s.profileKey)}/proxy`)).toBeLessThan(
calls.lastIndexOf("POST /tabs"),
);
const row = await h.db
.selectFrom("browser_provider")
.select("identity")
.where("id", "=", h.providerId)
.executeTakeFirstOrThrow();
expect(row.identity).toMatchObject({ userAgent: MOCK_CAMOFOX_IDENTITY.userAgent, os: "windows" });
expect(await readBrowserUserAgent(h.ctx)).toBe(MOCK_CAMOFOX_IDENTITY.userAgent);
expect(await sourcesIdentityModule.runtime(h.ctx).defaultUserAgent()).toBe(MOCK_CAMOFOX_IDENTITY.userAgent);
await closeAll();
});
it("站点开代理且设了代理地址:该 profile 走同一代理(口令单独传)", async () => {
await h.db.updateTable("site").set({ proxy: true }).where("id", "=", h.siteId).execute();
await h.db
.insertInto("settings")
.values({ key: "sites", value: JSON.stringify({ proxyUrl: "http://user:[email protected]:7891" }) })
.onConflict((oc) =>
oc.column("key").doUpdateSet({ value: JSON.stringify({ proxyUrl: "http://user:[email protected]:7891" }) }),
)
.execute();
const s = await open();
expect(h.mock.egress.get(s.profileKey)).toEqual({
server: "http://172.17.0.1:7891",
username: "user",
password: "pw",
});
await closeAll();
await h.db.updateTable("site").set({ proxy: false }).where("id", "=", h.siteId).execute();
});
});
+114
View File
@@ -29,8 +29,10 @@ import type {
BrowserCoordinateSpace,
BrowserCredentialState,
BrowserDownloadVerification,
BrowserEgressProxy,
BrowserEvidenceRef,
BrowserHealth,
BrowserIdentity,
BrowserProviderView,
BrowserReconnectState,
BrowserSession,
@@ -55,6 +57,7 @@ import {
registerBrowserCookieReader,
writeBrowserCookie,
} from "../sources-identity/service/credentials.ts";
import { registerBrowserIdentityReader } from "../sources-identity/service/user-agent.ts";
import type { ServerContext } from "../types.ts";
// C1-E 浏览器服务层:BrowserProvider(camofox)之上的会话/标签/控制权持久化(数据库为准)、
@@ -199,6 +202,95 @@ export class BrowserService {
this.now = opts.now ?? (() => new Date());
// refresh-login 发现 Cookie 失效时从浏览器 profile 读回一次(sources-identity 经钩子调用,不直接依赖本模块)
registerBrowserCookieReader(ctx, (siteId, accountId) => this.readProfileCookie(siteId, accountId));
// 站点请求 UA 跟随浏览器固定身份(BR-4):sources-identity 经钩子取
registerBrowserIdentityReader(ctx, () => this.identityUserAgent());
}
// ---------- BR-4 浏览器身份与出口 ----------
/** 身份读取失败后多久再试(浏览器服务没起 / 旧镜像时不每条站点请求都去问) */
private identityRetryAt = 0;
/**
* 浏览器固定身份的 UA:库里 browser_provider.identity 6 小时内读过就用它;否则问浏览器服务并写回。
* 没有启用的浏览器服务 / 插件不支持时返回库里的旧值或 null。
*/
async identityUserAgent(): Promise<string | null> {
const prow = await this.providerRow().catch(() => null);
if (!prow) return null;
const stored = obj(prow.identity) as { userAgent?: string; fetchedAt?: string };
const age = stored.fetchedAt ? Date.now() - Date.parse(stored.fetchedAt) : Number.POSITIVE_INFINITY;
if (stored.userAgent && age < 6 * 3_600_000) return stored.userAgent;
if (Date.now() < this.identityRetryAt) return stored.userAgent ?? null;
const fresh = await this.refreshIdentity(prow).catch(() => null);
return fresh?.userAgent ?? stored.userAgent ?? null;
}
/**
* 读浏览器服务的固定身份写进 browser_provider.identity(UA / OS / 版本 / 生成时间 + 读取时间)。
* maxAgeMs:库里的值比这新就不问。失败 5 分钟内不重试。
*/
async refreshIdentity(prow: BrowserProviderRow, opts: { maxAgeMs?: number } = {}): Promise<BrowserIdentity | null> {
const stored = obj(prow.identity) as Partial<BrowserIdentity> & { fetchedAt?: string };
if (opts.maxAgeMs !== undefined && stored.userAgent && stored.fetchedAt) {
if (Date.now() - Date.parse(stored.fetchedAt) < opts.maxAgeMs) return stored as BrowserIdentity;
}
const provider = this.providerFor(prow);
if (!provider.identity) return null;
let id: BrowserIdentity | null;
try {
id = await provider.identity();
} catch (e) {
this.identityRetryAt = Date.now() + 5 * 60_000;
throw e;
}
if (!id) {
this.identityRetryAt = Date.now() + 5 * 60_000;
return null;
}
this.identityRetryAt = 0;
await this.ctx.db
.updateTable("browser_provider")
.set({ identity: JSON.stringify({ ...id, fetchedAt: this.now().toISOString() }) })
.where("id", "=", prow.id)
.execute();
if (stored.userAgent !== id.userAgent)
this.ctx.logger.info(
{ component: "browser", providerId: prow.id, os: id.os, firefoxVersion: id.firefoxVersion },
"浏览器身份已更新,站点请求 UA 跟随",
);
return id;
}
/**
* 出口按站点统一(BR-4):站点开了代理且设置了代理地址 → 该 profile 走同一代理;否则直连。
* 旧镜像(插件没有出口接口)只记日志,浏览器照旧走容器环境变量里的代理。
*/
private async applyEgress(provider: BrowserProvider, site: SiteRow, profileKey: string): Promise<void> {
if (!provider.setEgress) return;
const rt = this.opts.runtime?.(this.ctx);
const proxyUrl = rt
? (await rt.siteDefaults()).proxyUrl
: ((await this.ctx.settings.get("sites").catch(() => null))?.proxyUrl ?? null);
const proxy = site.proxy && proxyUrl ? egressProxy(proxyUrl) : null;
try {
const r = await provider.setEgress(profileKey, proxy);
if (r.restarted)
this.ctx.logger.info(
{ component: "browser", siteId: site.id, egress: proxy ? "proxy" : "direct" },
"浏览器会话出口已变更,按新出口重建",
);
} catch (e) {
this.ctx.logger.warn(
{
component: "browser",
siteId: site.id,
egress: proxy ? "proxy" : "direct",
err: String((e as Error)?.message ?? e),
},
"设置浏览器会话出口失败,浏览器按容器环境变量出网",
);
}
}
// ---------- 浏览器服务 ----------
@@ -486,6 +578,8 @@ export class BrowserService {
.where("id", "=", row.id)
.returningAll()
.executeTakeFirstOrThrow();
// 探测时顺便读一次浏览器身份(站点请求 UA 跟随它,BR-4)
if (health.status !== "down" && row.enabled) await this.refreshIdentity(row).catch(() => null);
}
const caps = obj(row.capabilities);
out.push({
@@ -740,6 +834,8 @@ export class BrowserService {
for (const m of await provider.listSessions())
if (m.profileKey === profileKey && m.controller !== "closed") provider.setController(m.id, "closed");
const prep = await this.prepareCookies(site, accountId);
await this.applyEgress(provider, site, profileKey);
await this.refreshIdentity(prow, { maxAgeMs: 10 * 60_000 }).catch(() => null);
await this.rateLimit(site);
let res: Awaited<ReturnType<BrowserProvider["openSession"]>>;
try {
@@ -1416,6 +1512,9 @@ export class BrowserService {
if (!provider.reopenSession) throw await fail("浏览器服务不支持重新连接");
const url = restoreUrl(row.current_url, budget) ?? site.base_url;
const prep = await this.prepareCookies(site, row.account_id).catch(() => null);
await this.applyEgress(provider, site, row.profile_key);
// 浏览器重启过(可能换了镜像版本):身份重新读
await this.refreshIdentity(prow).catch(() => null);
let res: Awaited<ReturnType<NonNullable<BrowserProvider["reopenSession"]>>>;
try {
await this.rateLimit(site);
@@ -1944,3 +2043,18 @@ export function imageSize(data: Uint8Array): { width: number | null; height: num
}
export type { ElementTarget };
/** 代理地址 → 浏览器上下文代理(账号口令拆出来;Playwright 不认 URL 里的口令) */
export function egressProxy(url: string): BrowserEgressProxy | null {
try {
const u = new URL(url);
if (!["http:", "https:", "socks4:", "socks5:"].includes(u.protocol) || !u.hostname) return null;
return {
server: `${u.protocol}//${u.host}`,
...(u.username ? { username: decodeURIComponent(u.username) } : {}),
...(u.password ? { password: decodeURIComponent(u.password) } : {}),
};
} catch {
return null;
}
}
@@ -128,9 +128,10 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
)
.orderBy("created_at")
.execute();
const ua = await rt().uaDefaults();
for (const it of items) {
const mine = accounts.filter((a) => a.site_id === it.id);
it.accounts = mine.map(accountToApi);
it.accounts = mine.map((a) => accountToApi(a, ua));
it.accountStatus = overallAccountStatus(mine);
}
}
@@ -483,12 +484,13 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
.limit(query.limit + 1)
.execute();
const p = page(rows, off, query.limit);
return { items: p.items.map(accountToApi) as SiteAccount[], nextCursor: p.nextCursor };
const ua = await rt().uaDefaults();
return { items: p.items.map((a) => accountToApi(a, ua)) as SiteAccount[], nextCursor: p.nextCursor };
});
route(app, D.siteAccounts.get, async ({ params }) => {
const row = await db().selectFrom("site_account").selectAll().where("id", "=", params.id).executeTakeFirst();
if (!row) throw new ApiException("not_found", "站点账号不存在");
return accountToApi(row);
return accountToApi(row, await rt().uaDefaults());
});
route(app, D.siteAccounts.create, async ({ body, actor }) => {
await getSite(body.siteId);
@@ -510,7 +512,7 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
.returningAll()
.executeTakeFirstOrThrow();
await audit(db(), actor, "site_account.created", { type: "site_account", id: row.id }, { siteId: body.siteId });
return accountToApi(row);
return accountToApi(row, await rt().uaDefaults());
} catch (e) {
await rt()
.ctx.secrets.remove(ref)
@@ -539,6 +541,8 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
patch.status = "unknown";
patch.last_error = null;
patch.session_secret_ref = null;
// 新凭证 = 新登录态:不再沿用旧 UA(BR-4)
patch.ua_pending = null;
}
const row = await db()
.transaction()
@@ -585,7 +589,7 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
);
return r;
});
return accountToApi(row);
return accountToApi(row, await rt().uaDefaults());
});
// ── Feed ──
@@ -24,6 +24,7 @@ import {
tmdbDetailRef,
} from "../../shared-cache.ts";
import type { ServerContext } from "../types.ts";
import { readBrowserUserAgent, resolveUserAgent, type UserAgentDefaults } from "./service/user-agent.ts";
// 模块运行时依赖:站点适配器与限流器(每站一个实例,令牌在 PG 跨进程共享)、RSS 抓取器、TMDB 客户端、时钟。
// 测试通过 createSourcesIdentityModule(overrides) 注入 fetch / TMDB / 时钟。
@@ -168,8 +169,28 @@ export class SourcesRuntime {
/**
* 站点请求用的 fetch:测试注入优先;站点开了"使用代理"且设置了 proxyUrl 时走代理(Deno.createHttpClient)。
* BR-4:每条请求都带 UA(请求自己设了就不动;否则 ua 参数,缺省按"跟随浏览器"),不再发 Deno 缺省的 `Deno/x.y.z`。
* ua 可以是函数:每次请求现场取(站点适配器按当前账号取,账号切换 UA 后立即生效)。
*/
siteFetch(site: Pick<SiteRow, "proxy">, defaults: SitesSettings = this.siteDefaultsCache): typeof globalThis.fetch {
siteFetch(
site: Pick<SiteRow, "proxy">,
defaults: SitesSettings = this.siteDefaultsCache,
ua?: string | (() => Promise<string>),
): typeof globalThis.fetch {
const base = this.transportFetch(site, defaults);
return ((input: RequestInfo | URL, init?: RequestInit) => {
const headers = new Headers(init?.headers ?? (input instanceof Request ? input.headers : undefined));
if (headers.has("user-agent")) return base(input, { ...init, headers });
const pick = typeof ua === "function" ? ua() : ua !== undefined ? Promise.resolve(ua) : this.defaultUserAgent();
return pick.then((v) => {
headers.set("user-agent", v);
return base(input, { ...init, headers });
});
}) as typeof globalThis.fetch;
}
/** 不加 UA 的底层 fetch(代理 / 测试注入) */
private transportFetch(site: Pick<SiteRow, "proxy">, defaults: SitesSettings): typeof globalThis.fetch {
if (this.overrides.fetch) return this.overrides.fetch;
if (!site.proxy || !defaults.proxyUrl) return globalThis.fetch;
const key = defaults.proxyUrl;
@@ -182,6 +203,56 @@ export class SourcesRuntime {
return ((input: RequestInfo | URL, init?: RequestInit) =>
globalThis.fetch(input, { ...init, client: c } as RequestInit)) as typeof globalThis.fetch;
}
// ---------- BR-4 站点请求 UA ----------
private browserUa: { at: number; ua: string | null } = { at: 0, ua: null };
/** 浏览器固定身份的 UA(60 秒缓存);读不到为 null(调用方退回常量) */
async browserUserAgent(): Promise<string | null> {
if (Date.now() - this.browserUa.at < 60_000) return this.browserUa.ua;
let ua: string | null = this.browserUa.ua;
try {
ua = (await readBrowserUserAgent(this.ctx)) ?? ua;
} catch (err) {
this.ctx.logger.debug({ err: { message: (err as Error).message } }, "读取浏览器身份 UA 失败,沿用上次的值");
}
this.browserUa = { at: Date.now(), ua };
return ua;
}
/** UA 解析用的全局部分:设置里的全局 UA + 浏览器 UA(已退回常量) */
async uaDefaults(): Promise<UserAgentDefaults> {
const [s, browser] = await Promise.all([this.siteDefaults(), this.browserUserAgent()]);
return { global: s.userAgent ?? "", browser: browser ?? "" };
}
/** 不挂账号的请求(通用 RSS 等)用的 UA:全局 UA 或跟随浏览器 */
async defaultUserAgent(): Promise<string> {
return resolveUserAgent(null, await this.uaDefaults()).ua;
}
/**
* 站点请求 UA:取与站点凭证相同的账号(第一个启用、未失效、非 Passkey 的账号;没有就第一个启用的),按账号解析。
* 不缓存:与 Cookie 同一时刻读库,账号切换 Cookie + UA 后下一条请求两者一起生效。
*/
async siteUserAgent(siteId: string): Promise<string> {
const d = await this.uaDefaults();
const rows = await this.ctx.db
.selectFrom("site_account")
.select(["user_agent", "ua_pending", "status", "auth_kind"])
.where("site_id", "=", siteId)
.where("enabled", "=", true)
.orderBy("created_at")
.execute();
const acc =
rows.find((r) => r.auth_kind !== "passkey" && (r.status === "unknown" || r.status === "active")) ??
rows.find((r) => r.auth_kind !== "passkey") ??
rows[0] ??
null;
return resolveUserAgent(acc, d).ua;
}
private readonly proxyClients = new Map<string, Deno.HttpClient>();
/**
@@ -232,7 +303,8 @@ export class SourcesRuntime {
secrets: this.siteSecrets(site.id),
limiter,
now: this.now,
...(this.overrides.fetch || proxy ? { fetch: this.siteFetch({ proxy }, defaults) } : {}),
// 每条请求都经 siteFetch:代理 + 按账号现场取 UA(BR-4)
fetch: this.siteFetch({ proxy }, defaults, () => this.siteUserAgent(site.id)),
...(baseUrl ? { baseUrl } : {}),
});
this.adapters.set(site.id, { sig, limiter, adapter });
@@ -29,20 +29,25 @@ import type { SourcesRuntime } from "../runtime.ts";
import { capabilitiesOf, LOGIN_KINDS } from "../site-kinds.ts";
import { type Actor, audit, closeDecisions, type Exec, openDecision, SYSTEM } from "./common.ts";
import { cookieFromBrowser, writeBrowserCookie } from "./credentials.ts";
import { browserOnlyCookiesOf, resolveUserAgent, type UserAgentDefaults } from "./user-agent.ts";
// W9-A 站点中心(docs/design/site-center.md):凭证写入、刷新登录(账密 takelogin / Cookie / API Key 校验)、
// 用户数据与按天历史、健康检测、删除。凭证明文只在内存里用,不进日志、审计与错误信息。
const jsonb = (v: unknown) => JSON.stringify(v ?? null);
/** site_account 行 → SiteAccount(secret 引用只给布尔) */
export function accountToApi(row: SiteAccountRow): Record<string, unknown> {
const { totp_secret_ref, session_secret_ref, passkey_secret_ref, ...rest } = row;
/** site_account 行 → SiteAccount(secret 引用只给布尔);ua = 全局 UA 与浏览器 UA(SourcesRuntime.uaDefaults) */
export function accountToApi(row: SiteAccountRow, ua: UserAgentDefaults): Record<string, unknown> {
const { totp_secret_ref, session_secret_ref, passkey_secret_ref, ua_pending: _p, ...rest } = row;
const eff = resolveUserAgent(row, ua);
return {
...toApi<Record<string, unknown>>(rest),
hasTotp: !!totp_secret_ref,
hasSession: !!session_secret_ref,
hasPasskey: !!passkey_secret_ref || (row.auth_kind === "passkey" && !!row.secret_ref),
userAgentSource: eff.source,
effectiveUserAgent: eff.ua,
browserOnlyCookies: browserOnlyCookiesOf(row.user_info),
};
}
@@ -283,6 +288,8 @@ export async function setSiteCredentials(
totp_secret_ref: totpRef,
login_name: body.kind === "password" ? body.username : null,
user_agent: "userAgent" in body ? (body.userAgent ?? null) : null,
// 换了主凭证 = 新的登录态,不再沿用旧 UA(BR-4 待切换结束);只改 UA / 启用时保持
...(keepMain ? {} : { ua_pending: null }),
user_info: jsonb(userInfo),
...(body.enabled !== undefined ? { enabled: body.enabled } : {}),
};
@@ -388,7 +395,10 @@ export async function refreshSiteAccount(
const def = getSiteDefinition(site.key);
if (!def || site.deleted_at) return unchanged("unsupported", "站点没有适配器定义,无法校验登录");
const defaults = await rt.siteDefaults();
const ua = acc.user_agent ?? defaults.userAgent;
// BR-4:current = 当前登录态配套的 UA(待切换时是旧 UA);target = 切换后的 UA(重新登录、浏览器 Cookie 用它)
const uaDefaults = await rt.uaDefaults();
const currentUa = resolveUserAgent(acc, uaDefaults).ua;
const targetUa = resolveUserAgent(acc, uaDefaults, { ignorePending: true }).ua;
let limiter: ReturnType<SourcesRuntime["limiterOf"]> = null;
try {
rt.siteAdapter(site);
@@ -396,7 +406,7 @@ export async function refreshSiteAccount(
} catch {
// 无适配器:不限流(仅 generic_rss,前面已返回)
}
const fetch = rt.siteFetch(site, defaults);
const fetch = rt.siteFetch(site, defaults, currentUa);
const umt = def.umt as { kind: string; login?: Parameters<typeof loginNexusPhp>[0]["login"]; loginSuccess?: string };
const profile = (def.umt as { profile?: Record<string, string> }).profile;
const vipLevels = (def.umt as { vipLevels?: string[] }).vipLevels;
@@ -406,7 +416,10 @@ export async function refreshSiteAccount(
? (site.config as { apiBaseUrl?: string })?.apiBaseUrl || mteamApiBase(def as MteamDefinition)
: null;
const fetchUserData = async (cred: { cookie?: string | null; apikey?: string | null }): Promise<UserDataOutcome> => {
const fetchUserData = async (
cred: { cookie?: string | null; apikey?: string | null },
ua: string = currentUa,
): Promise<UserDataOutcome> => {
const info = await refreshSiteUserData({
name: site.name,
url: site.base_url,
@@ -467,14 +480,15 @@ export async function refreshSiteAccount(
username: acc.login_name ?? (acc.user_info as { username?: string })?.username ?? acc.label,
password,
totpSecret,
userAgent: ua,
userAgent: targetUa,
fetch,
...(limiter ? { limiter } : {}),
...(opts.signal ? { signal: opts.signal } : {}),
now: rt.now,
});
// 新登录态用 targetUa 取得:会话 Cookie 与 UA 一起切换(清掉待切换)
await storeSession(rt, acc, login.cookie);
const r = await fetchUserData({ cookie: login.cookie });
const r = await fetchUserData({ cookie: login.cookie }, targetUa);
if (r.kind === "ok") return await succeed(r.info);
return await succeed(
null,
@@ -488,7 +502,8 @@ export async function refreshSiteAccount(
// BR-2:存的 Cookie 失效,但浏览器里可能已人工登录过 → 读回浏览器 profile 的 Cookie 试一次,有效才回写
const alt = await cookieFromBrowser(rt.ctx, acc, plain);
if (alt) {
const r2 = await fetchUserData({ cookie: alt });
// 浏览器里的登录态是浏览器 UA 取得的:按切换后的 UA 验证,回写时一起切换
const r2 = await fetchUserData({ cookie: alt }, targetUa);
if (r2.kind === "ok") {
await writeBrowserCookie(rt.ctx, acc.id, alt, rt.now());
return await succeed(r2.info, "存的 Cookie 已失效;浏览器里的登录态有效,已回写");
@@ -515,7 +530,10 @@ export async function refreshSiteAccount(
export const secretName = (acc: { site_id: string; label: string }, suffix = "") =>
`site_account:${acc.site_id}:${acc.label}${suffix}:${randomUUID().slice(0, 8)}`;
/** 账密登录成功:会话 Cookie 存 secret;"只换 Cookie"时凭证类型落为 cookie,密码与 TOTP 丢弃 */
/**
* 账密登录成功:会话 Cookie 存 secret;"只换 Cookie"时凭证类型落为 cookie,密码与 TOTP 丢弃。
* 登录用的是切换后的 UA:写 Cookie 的同一事务里清掉待切换(BR-4 Cookie + UA 原子切换)。
*/
async function storeSession(rt: SourcesRuntime, acc: SiteAccountRow, cookie: string) {
const { db, secrets } = rt.ctx;
const cookieOnly = (acc.user_info as { saveCookieOnly?: boolean })?.saveCookieOnly === true;
@@ -531,6 +549,7 @@ async function storeSession(rt: SourcesRuntime, acc: SiteAccountRow, cookie: str
session_secret_ref: null,
totp_secret_ref: null,
user_info: jsonb(info),
ua_pending: null,
})
.where("id", "=", acc.id)
.execute();
@@ -541,10 +560,16 @@ async function storeSession(rt: SourcesRuntime, acc: SiteAccountRow, cookie: str
acc.secret_ref = ref;
acc.session_secret_ref = null;
acc.totp_secret_ref = null;
acc.ua_pending = null;
return;
}
if (acc.session_secret_ref) {
await secrets.replace(acc.session_secret_ref, cookie);
const ref = acc.session_secret_ref;
await db.transaction().execute(async (trx) => {
await secrets.replace(ref, cookie, trx);
await trx.updateTable("site_account").set({ ua_pending: null }).where("id", "=", acc.id).execute();
});
acc.ua_pending = null;
return;
}
const ref = await secrets.create({
@@ -552,8 +577,13 @@ async function storeSession(rt: SourcesRuntime, acc: SiteAccountRow, cookie: str
plaintext: cookie,
label: secretName(acc, ":session"),
});
await db.updateTable("site_account").set({ session_secret_ref: ref }).where("id", "=", acc.id).execute();
await db
.updateTable("site_account")
.set({ session_secret_ref: ref, ua_pending: null })
.where("id", "=", acc.id)
.execute();
acc.session_secret_ref = ref;
acc.ua_pending = null;
}
async function markSuccess(
@@ -1,4 +1,4 @@
import { mergeCookieHeaders, parseCookieHeader, sameCookieHeader } from "@umt/adapters";
import { mergeCookieHeaders, parseCookieHeader, sameCookieHeader, splitBrowserOnlyCookies } from "@umt/adapters";
import type { SiteCookieSource } from "@umt/contracts";
import type { SiteAccountRow } from "@umt/db";
import type { ServerContext } from "../../types.ts";
@@ -81,52 +81,81 @@ export async function readAccountCookie(ctx: Ctx, accountId: string): Promise<Ac
/**
* 浏览器回写:与当前 Cookie 比较(忽略顺序),有变化才写。
* Cookie 账号替换 secret_ref;账密账号替换(或新建)会话 Cookie。失效状态的账号回到 unknown(下次抓取 / 健康检测再验证)。
* BR-4:
* - Cloudflare 验证 Cookie(cf_clearance、__cf_bm)只留在浏览器 profile,不写给站点请求,只在 user_info.browserOnlyCookies 记名字;
* - 浏览器用自己的 UA 取得 / 用着这份登录态:写 Cookie 的同一事务里清掉"待切换",站点请求从下一条起 Cookie + UA 一起换。
* 返回是否写入。调用方负责只传该站域名的 Cookie,并确认浏览器当前不在登录页。
*/
export async function writeBrowserCookie(
ctx: Ctx,
accountId: string,
header: string,
rawHeader: string,
now: Date = new Date(),
): Promise<{ changed: boolean; count: number }> {
): Promise<{ changed: boolean; count: number; uaSwitched: boolean; browserOnly: string[] }> {
const { header, browserOnly } = splitBrowserOnlyCookies(rawHeader);
const count = parseCookieHeader(header).size;
if (!count) return { changed: false, count: 0 };
const none = { changed: false, count, uaSwitched: false, browserOnly };
if (!header || !count) return none;
const acc = await ctx.db.selectFrom("site_account").selectAll().where("id", "=", accountId).executeTakeFirst();
if (!acc || (acc.auth_kind !== "cookie" && acc.auth_kind !== "password")) return { changed: false, count };
if (!acc || (acc.auth_kind !== "cookie" && acc.auth_kind !== "password")) return none;
const cur = await readAccountCookie(ctx, accountId);
if (cur && sameCookieHeader(cur.header, header)) return { changed: false, count };
let sessionRef = acc.session_secret_ref;
if (acc.auth_kind === "cookie") {
if (!acc.secret_ref) return { changed: false, count };
await ctx.secrets.replace(acc.secret_ref, header);
// 旧会话 Cookie 会在合并时覆盖同名项,回写后以浏览器为准
if (sessionRef) await ctx.secrets.replace(sessionRef, header);
} else if (sessionRef) {
await ctx.secrets.replace(sessionRef, header);
} else {
sessionRef = await ctx.secrets.create({
purpose: "site_cookie",
plaintext: header,
label: `site_account:${acc.site_id}:${acc.label}:session:${crypto.randomUUID().slice(0, 8)}`,
});
}
const origin: CookieOrigin = { source: "browser", by: "browser", at: now.toISOString() };
const stale = acc.status === "expired" || acc.status === "invalid";
const info = obj(acc.user_info);
const prevOnly = Array.isArray(info.browserOnlyCookies) ? (info.browserOnlyCookies as string[]) : [];
const onlyChanged = browserOnly.length > 0 && browserOnly.join(",") !== prevOnly.join(",");
if (cur && sameCookieHeader(cur.header, header)) {
// Cookie 没变:浏览器正用这份登录态(不在登录页),待切换的 UA 也可以换了
if (!acc.ua_pending && !onlyChanged) return none;
await ctx.db
.updateTable("site_account")
.set({
user_info: JSON.stringify({ ...obj(acc.user_info), cookieOrigin: origin }),
...(acc.ua_pending ? { ua_pending: null } : {}),
...(onlyChanged ? { user_info: JSON.stringify({ ...info, browserOnlyCookies: browserOnly }) } : {}),
updated_at: now,
})
.where("id", "=", acc.id)
.execute();
if (acc.ua_pending)
ctx.logger.info({ component: "credentials", accountId }, "站点账号 UA 已切换为跟随浏览器(Cookie 未变)");
return { ...none, uaSwitched: !!acc.ua_pending };
}
if (acc.auth_kind === "cookie" && !acc.secret_ref) return none;
// 新建会话 secret 在事务外(新行没被引用,不影响读);替换已有 secret 与账号行更新在同一事务里
const newSession =
acc.auth_kind === "password" && !acc.session_secret_ref
? await ctx.secrets.create({
purpose: "site_cookie",
plaintext: header,
label: `site_account:${acc.site_id}:${acc.label}:session:${crypto.randomUUID().slice(0, 8)}`,
})
: null;
const sessionRef = newSession ?? acc.session_secret_ref;
const origin: CookieOrigin = { source: "browser", by: "browser", at: now.toISOString() };
const stale = acc.status === "expired" || acc.status === "invalid";
await ctx.db.transaction().execute(async (trx) => {
if (acc.auth_kind === "cookie") {
await ctx.secrets.replace(acc.secret_ref!, header, trx);
// 旧会话 Cookie 会在合并时覆盖同名项,回写后以浏览器为准
if (sessionRef) await ctx.secrets.replace(sessionRef, header, trx);
} else if (!newSession && sessionRef) {
await ctx.secrets.replace(sessionRef, header, trx);
}
await trx
.updateTable("site_account")
.set({
user_info: JSON.stringify({ ...info, cookieOrigin: origin, browserOnlyCookies: browserOnly }),
session_secret_ref: sessionRef,
ua_pending: null,
...(stale ? { status: "unknown" as const, last_error: null } : {}),
updated_at: now,
})
.where("id", "=", acc.id)
.execute();
});
ctx.logger.info(
{ component: "credentials", accountId, count, updatedBy: "browser" },
{ component: "credentials", accountId, count, browserOnly: browserOnly.length, updatedBy: "browser" },
"站点账号 Cookie 已从浏览器回写",
);
return { changed: true, count };
return { changed: true, count, uaSwitched: !!acc.ua_pending, browserOnly };
}
// ---------- 浏览器读回钩子(浏览器模块注册;sources-identity 不直接依赖浏览器模块) ----------
@@ -148,7 +177,8 @@ export async function cookieFromBrowser(
): Promise<string | null> {
const reader = readers.get(ctx);
if (!reader) return null;
const header = await reader(acc.site_id, acc.id).catch(() => null);
// 仅浏览器用的 Cloudflare Cookie 不给站点请求(BR-4)
const header = splitBrowserOnlyCookies(await reader(acc.site_id, acc.id).catch(() => null)).header;
if (!header || sameCookieHeader(header, current)) return null;
return header;
}
@@ -159,7 +159,12 @@ export async function ingestFeed(
if (!feed.url_secret_ref) return { status: "skipped", reason: "no_url", ...empty };
const url = await secrets.resolve(feed.url_secret_ref);
const result = await rt.rss.fetchFeed(url, stateFromRow(feed), { ignoreBackoff: !!opts.manual });
// BR-4:挂站点的 Feed 与站点其它请求同一 UA、同一出口(站点开代理时走代理);通用 Feed 用全局 UA / 跟随浏览器
const result = await rt.rss.fetchFeed(url, stateFromRow(feed), {
ignoreBackoff: !!opts.manual,
userAgent: site ? await rt.siteUserAgent(site.id) : await rt.defaultUserAgent(),
fetch: rt.siteFetch(site ?? { proxy: false }),
});
// 挂站点的 Feed:每次真正发出的 RSS 请求也记进站点请求计数(熔断 / 退避跳过的不算)
if (site && result.kind !== "skipped") rt.countSiteRequest(site.id);
if (site) await renormalizeFeedReleases(rt, site).catch(() => {});
@@ -0,0 +1,69 @@
import { FALLBACK_SITE_USER_AGENT } from "@umt/contracts";
import type { SiteAccountRow } from "@umt/db";
import type { ServerContext } from "../../types.ts";
// BR-4 站点请求 UA:账号手动 > 待切换(沿用取得当前 Cookie 时的旧 UA)> 设置里的全局 UA > 跟随浏览器(camofox 固定身份的 UA,
// 读不到时用同型的 Windows Firefox 常量)。站点适配器、RSS、登录、用户数据、种子下载都经 SourcesRuntime.siteFetch 带上它。
export type UserAgentSource = "browser" | "manual" | "global" | "pending";
export interface UserAgentDefaults {
/** settings.sites.userAgent;空 = 跟随浏览器 */
global: string;
/** 浏览器身份 UA(已退回常量) */
browser: string;
}
type UaFields = Pick<SiteAccountRow, "user_agent" | "ua_pending">;
/**
* 生效 UA。ignorePending = 按"切换之后"算(重新登录 / 浏览器 Cookie 回写时用新 UA,成功后清掉待切换)。
*/
export function resolveUserAgent(
acc: UaFields | null | undefined,
d: UserAgentDefaults,
opts: { ignorePending?: boolean } = {},
): { ua: string; source: UserAgentSource } {
const manual = acc?.user_agent?.trim();
if (manual) return { ua: manual, source: "manual" };
const pending = acc?.ua_pending?.trim();
if (pending && !opts.ignorePending) return { ua: pending, source: "pending" };
const global = d.global.trim();
if (global) return { ua: global, source: "global" };
return { ua: d.browser || FALLBACK_SITE_USER_AGENT, source: "browser" };
}
/** user_info.browserOnlyCookies:最近一次浏览器回写时剔除的仅浏览器 Cookie 名 */
export function browserOnlyCookiesOf(userInfo: unknown): string[] {
const v = (userInfo as { browserOnlyCookies?: unknown } | null)?.browserOnlyCookies;
return Array.isArray(v) ? v.filter((x): x is string => typeof x === "string") : [];
}
// ---------- 浏览器身份读取钩子(浏览器模块注册;sources-identity 不直接依赖浏览器模块) ----------
/** 返回浏览器固定身份的 UA;没有浏览器服务 / 插件时 null */
export type BrowserIdentityReader = () => Promise<string | null>;
const readers = new WeakMap<object, BrowserIdentityReader>();
export function registerBrowserIdentityReader(ctx: object, reader: BrowserIdentityReader): void {
readers.set(ctx, reader);
}
/**
* 浏览器身份 UA:浏览器模块在本进程装配了就问它(它读库里缓存的身份,过期再问浏览器服务);
* 没装配时只读 browser_provider.identity(其它进程写入的)。
*/
export async function readBrowserUserAgent(ctx: Pick<ServerContext, "db">): Promise<string | null> {
const reader = readers.get(ctx);
if (reader) return reader();
const row = await ctx.db
.selectFrom("browser_provider")
.select("identity")
.where("enabled", "=", true)
.where("kind", "<>", "none")
.orderBy("created_at")
.executeTakeFirst();
const ua = (row?.identity as { userAgent?: unknown } | null | undefined)?.userAgent;
return typeof ua === "string" && ua ? ua : null;
}
+2 -1
View File
@@ -47,7 +47,8 @@ export interface OutboxPort extends OutboxWriter {
export interface SecretStore {
resolve(secretRef: string): Promise<string>;
create(input: { purpose: SecretPurpose; plaintext: string; label?: string }): Promise<string>;
replace(secretRef: string, plaintext: string): Promise<void>;
/** exec:在调用方事务里写(BR-4 Cookie 与 UA 原子切换);缺省用自己的连接 */
replace(secretRef: string, plaintext: string, exec?: Kysely<DB>): Promise<void>;
remove(secretRef: string): Promise<void>;
}
@@ -0,0 +1,100 @@
import { createMigrator, type DB, migrateToLatest, sql } from "@umt/db";
import { Kysely, PostgresDialect } from "kysely";
import pg from "pg";
import { afterAll, describe, expect, it } from "vitest";
import { PG_URL, pgAvailable, withSchemaLock } from "../auth-settings/harness.ts";
// 0056:存量 Cookie / 账密账号标"待切换"(沿用旧缺省 UA);手动 UA 与 API Key 账号不动;设置里的旧缺省 UA 清掉 = 跟随浏览器。
const hasPg = await pgAvailable();
const LEGACY =
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36";
describe.skipIf(!hasPg)("0056 浏览器身份迁移", () => {
const schemas: string[] = [];
const dbs: Kysely<DB>[] = [];
const admin = async (q: string) => {
const c = new pg.Client({ connectionString: PG_URL });
await c.connect();
await c.query(q).finally(() => c.end());
};
afterAll(async () => {
for (const d of dbs) await d.destroy();
await withSchemaLock(async () => {
for (const s of schemas) await admin(`drop schema if exists ${s} cascade`);
});
});
const setup = async (settingsUa: string | null) => {
const schema = `br4_mig_${process.pid.toString(36)}${Math.random().toString(36).slice(2, 6)}`;
schemas.push(schema);
await admin(`create schema ${schema}`);
const db = new Kysely<DB>({
dialect: new PostgresDialect({
pool: new pg.Pool({ connectionString: PG_URL, max: 2, options: `-c search_path=${schema},public` }),
}),
});
dbs.push(db);
await withSchemaLock(async () => {
const r = await createMigrator(db, { migrationTableSchema: schema }).migrateTo("0055_site_kind");
if (r.error) throw r.error;
});
await sql`insert into site (key, name, adapter, base_url) values ('audiences', '观众', 'nexusphp', 'https://a.example/')`.execute(
db,
);
await sql`insert into secret (id, name, purpose, ciphertext, nonce) values
('00000000-0000-4000-8000-000000000001', 's1', 'site_cookie', 'x', 'x'),
('00000000-0000-4000-8000-000000000002', 's2', 'site_password', 'x', 'x'),
('00000000-0000-4000-8000-000000000003', 's3', 'site_api_key', 'x', 'x'),
('00000000-0000-4000-8000-000000000004', 's4', 'site_cookie', 'x', 'x')`.execute(db);
await sql`insert into site_account (site_id, label, auth_kind, secret_ref, user_agent) values
((select id from site), 'cookie', 'cookie', '00000000-0000-4000-8000-000000000001', null),
((select id from site), 'password', 'password', '00000000-0000-4000-8000-000000000002', null),
((select id from site), 'apikey', 'api_key', '00000000-0000-4000-8000-000000000003', null),
((select id from site), 'manual', 'cookie', '00000000-0000-4000-8000-000000000004', 'My-UA/1'),
((select id from site), 'empty', 'cookie', null, null)`.execute(db);
if (settingsUa !== null)
await sql`insert into settings (key, value) values ('sites', ${JSON.stringify({ userAgent: settingsUa, proxyUrl: null })}::jsonb)`.execute(
db,
);
await withSchemaLock(() => migrateToLatest(db, { migrationTableSchema: schema }));
const accounts = await sql<{ label: string; ua_pending: string | null }>`
select label, ua_pending from site_account order by label`.execute(db);
const settings = await sql<{
value: Record<string, unknown>;
}>`select value from settings where key = 'sites'`.execute(db);
return { db, accounts: Object.fromEntries(accounts.rows.map((r) => [r.label, r.ua_pending])), settings };
};
it("设置仍是旧缺省:Cookie / 账密账号待切换到旧 UA,设置里的 UA 清掉(跟随浏览器)", async () => {
const r = await setup(LEGACY);
expect(r.accounts).toEqual({ apikey: null, cookie: LEGACY, empty: null, manual: null, password: LEGACY });
expect(r.settings.rows[0]!.value).not.toHaveProperty("userAgent");
expect(r.settings.rows[0]!.value).toHaveProperty("proxyUrl", null);
const cols = await sql<{ column_name: string }>`select column_name from information_schema.columns
where table_schema = current_schema() and table_name = 'browser_provider' and column_name = 'identity'`.execute(
r.db,
);
expect(cols.rows).toHaveLength(1);
});
it("用户改过全局 UA:保留为全局 UA,存量账号待切换到它;没有设置行时按旧缺省", async () => {
const r = await setup("Custom-UA/9");
expect(r.accounts.cookie).toBe("Custom-UA/9");
expect(r.settings.rows[0]!.value).toHaveProperty("userAgent", "Custom-UA/9");
const r2 = await setup(null);
expect(r2.accounts.password).toBe(LEGACY);
expect(r2.settings.rows).toHaveLength(0);
});
it("回滚只删列", async () => {
const r = await setup(LEGACY);
await withSchemaLock(async () => {
const m = await createMigrator(r.db, { migrationTableSchema: schemas.at(-1)! }).migrateTo("0055_site_kind");
if (m.error) throw m.error;
});
const cols = await sql<{ column_name: string }>`select column_name from information_schema.columns
where table_schema = current_schema() and column_name in ('ua_pending', 'identity')`.execute(r.db);
expect(cols.rows).toHaveLength(0);
});
});
@@ -0,0 +1,273 @@
import { FALLBACK_SITE_USER_AGENT, LEGACY_SITE_USER_AGENT } from "@umt/contracts";
import { createMockNexusPhpSite, createMockRssServer, type MockNexusPhpSite, type MockRssServer } from "@umt/testkit";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import {
readAccountCookie,
registerBrowserCookieReader,
writeBrowserCookie,
} from "../../src/modules/sources-identity/service/credentials.ts";
import { ingestFeed } from "../../src/modules/sources-identity/service/feeds.ts";
import {
registerBrowserIdentityReader,
resolveUserAgent,
} from "../../src/modules/sources-identity/service/user-agent.ts";
import { createHarness, type Harness, pgAvailable, seedFeed } from "./harness.ts";
// BR-4 站点请求 UA:每条出站请求都带账号 UA(不再是 Deno 缺省 UA)、优先级(手动 > 待切换 > 全局 > 跟随浏览器)、
// 平滑迁移(待切换账号沿用旧 UA,重新登录 / 浏览器回写 Cookie 时 Cookie + UA 一起切换)、Cloudflare Cookie 仅浏览器。
const up = await pgAvailable();
const BROWSER_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:152.0) Gecko/20100101 Firefox/152.0";
// biome-ignore lint/suspicious/noExplicitAny: 响应体断言
const json = async (r: Response): Promise<any> => {
const body = await r.json();
if (!r.ok) throw new Error(`${r.status} ${JSON.stringify(body)}`);
return body.data;
};
type RecordedRequest = MockNexusPhpSite["requests"][number];
const uaOf = (r: RecordedRequest) => String(r.headers["user-agent"] ?? "");
describe("resolveUserAgent 优先级", () => {
const d = { global: "", browser: "B" };
it("手动 > 待切换 > 全局 > 跟随浏览器 > 常量", () => {
expect(resolveUserAgent({ user_agent: "M", ua_pending: "P" }, { global: "G", browser: "B" })).toEqual({
ua: "M",
source: "manual",
});
expect(resolveUserAgent({ user_agent: null, ua_pending: "P" }, { global: "G", browser: "B" })).toEqual({
ua: "P",
source: "pending",
});
expect(
resolveUserAgent({ user_agent: null, ua_pending: "P" }, { global: "G", browser: "B" }, { ignorePending: true }),
).toEqual({ ua: "G", source: "global" });
expect(resolveUserAgent({ user_agent: " ", ua_pending: null }, d)).toEqual({ ua: "B", source: "browser" });
expect(resolveUserAgent(null, { global: "", browser: "" })).toEqual({
ua: FALLBACK_SITE_USER_AGENT,
source: "browser",
});
});
});
describe.skipIf(!up)("BR-4 站点请求 UA 与 Cookie 切换", () => {
let h: Harness;
let mock: MockNexusPhpSite;
let rss: MockRssServer;
let siteId: string;
let accountId: string;
let browserUa: string | null = BROWSER_UA;
let browserCookie: string | null = null;
const account = () =>
h.db.selectFrom("site_account").selectAll().where("id", "=", accountId).executeTakeFirstOrThrow();
const site = () => h.db.selectFrom("site").selectAll().where("id", "=", siteId).executeTakeFirstOrThrow();
const apiAccount = async () => json(await h.req(`/site-accounts/${accountId}`));
const refresh = async () =>
(await json(await h.req(`/sites/${siteId}/refresh-login`, { body: { wait: true, accountId } }))).result;
const since = (n: number) => mock.requests.slice(n);
beforeAll(async () => {
mock = await createMockNexusPhpSite({ username: "alice", password: "s3cret-pw" });
rss = await createMockRssServer({
passkey: "PK",
items: [
{
guid: "br4-1",
title: "Some.Movie.2024.1080p.BluRay-GRP",
link: `${mock.baseUrl}/details.php?id=900001`,
enclosureUrl: `${mock.baseUrl}/download.php?id=900001&passkey=PK`,
size: 1024 ** 3,
pubDate: new Date("2026-09-20T08:00:00Z"),
categories: ["电影"],
},
],
});
h = await createHarness("br4ua");
registerBrowserIdentityReader(h.ctx, async () => browserUa);
registerBrowserCookieReader(h.ctx, async () => browserCookie);
const s = await json(
await h.req("/sites", {
body: {
key: "audiences",
name: "观众",
adapter: "nexusphp",
baseUrl: `${mock.baseUrl}/`,
rateLimit: { minIntervalMs: 0, concurrency: 10 },
},
}),
);
siteId = s.id;
const r = await json(
await h.req(`/sites/${siteId}/credentials`, {
method: "PUT",
body: { kind: "password", username: "alice", password: "s3cret-pw" },
}),
);
accountId = r.siteAccountId;
});
afterAll(async () => {
await h?.close();
await mock?.close();
await rss?.close();
});
it("登录、用户数据、列表 / 搜索 / 详情 / 下载 / 健康检测、RSS:每条请求都带跟随浏览器的 UA", async () => {
const n0 = mock.requests.length;
expect(await refresh()).toMatchObject({ ok: true, status: "active" });
const adapter = h.rt.siteAdapter(await site());
const page = await adapter.list(0);
const id = page.items[0]!.siteReleaseId;
await adapter.search("Movie");
await adapter.detail(id);
const target = await adapter.downloadTarget(id);
expect(target.kind).toBe("torrent");
expect((await adapter.healthCheck()).ok).toBe(true);
const f = await seedFeed(h, { siteId, url: rss.feedUrl, name: "br4", accountId });
expect((await ingestFeed(h.rt, f.id)).status).toBe("ok");
const reqs = [...since(n0), ...rss.requests];
const paths = new Set(since(n0).map((r) => r.path.split("?")[0]));
for (const p of ["/takelogin.php", "/torrents.php", "/details.php", "/download.php"]) expect(paths).toContain(p);
expect(rss.requests.length).toBeGreaterThan(0);
for (const r of reqs) {
expect(uaOf(r), `${r.method} ${r.path}`).toBe(BROWSER_UA);
expect(uaOf(r)).not.toMatch(/Deno/i);
}
expect(await apiAccount()).toMatchObject({
userAgentSource: "browser",
effectiveUserAgent: BROWSER_UA,
userAgent: null,
});
});
it("浏览器身份读不到:退回同型 Windows Firefox 常量;设置里的全局 UA 优先于跟随浏览器,账号手动再优先", async () => {
browserUa = null;
// runtime 缓存浏览器 UA 60 秒:换一个 runtime 读
(h.rt as unknown as { browserUa: { at: number } }).browserUa.at = 0;
let n = mock.requests.length;
await h.rt.siteAdapter(await site()).healthCheck();
expect(since(n).map(uaOf)).toEqual(since(n).map(() => FALLBACK_SITE_USER_AGENT));
await h.db
.insertInto("settings")
.values({ key: "sites", value: JSON.stringify({ userAgent: "Global-UA/1.0" }) })
.execute();
n = mock.requests.length;
await h.rt.siteDefaults();
await h.rt.siteAdapter(await site()).healthCheck();
expect(new Set(since(n).map(uaOf))).toEqual(new Set(["Global-UA/1.0"]));
expect(await apiAccount()).toMatchObject({ userAgentSource: "global", effectiveUserAgent: "Global-UA/1.0" });
await h.db.updateTable("site_account").set({ user_agent: "Manual-UA/2.0" }).where("id", "=", accountId).execute();
n = mock.requests.length;
await h.rt.siteAdapter(await site()).healthCheck();
expect(new Set(since(n).map(uaOf))).toEqual(new Set(["Manual-UA/2.0"]));
expect(await apiAccount()).toMatchObject({ userAgentSource: "manual", userAgent: "Manual-UA/2.0" });
await h.db.updateTable("site_account").set({ user_agent: null }).where("id", "=", accountId).execute();
await h.db.deleteFrom("settings").where("key", "=", "sites").execute();
await h.rt.siteDefaults();
browserUa = BROWSER_UA;
(h.rt as unknown as { browserUa: { at: number } }).browserUa.at = 0;
});
it("待切换:沿用旧 UA;重新登录用浏览器 UA,会话 Cookie 与 UA 一起切换", async () => {
await h.db
.updateTable("site_account")
.set({ ua_pending: LEGACY_SITE_USER_AGENT })
.where("id", "=", accountId)
.execute();
expect(await apiAccount()).toMatchObject({
userAgentSource: "pending",
effectiveUserAgent: LEGACY_SITE_USER_AGENT,
});
let n = mock.requests.length;
await h.rt.siteAdapter(await site()).healthCheck();
expect(new Set(since(n).map(uaOf))).toEqual(new Set([LEGACY_SITE_USER_AGENT]));
n = mock.requests.length;
expect(await refresh()).toMatchObject({ ok: true });
const login = since(n).find((r) => r.path.startsWith("/takelogin.php"));
expect(login && uaOf(login)).toBe(BROWSER_UA);
expect((await account()).ua_pending).toBeNull();
n = mock.requests.length;
await h.rt.siteAdapter(await site()).healthCheck();
expect(new Set(since(n).map(uaOf))).toEqual(new Set([BROWSER_UA]));
expect(await apiAccount()).toMatchObject({ userAgentSource: "browser" });
});
it("浏览器回写 Cookie:原子切换 UA;cf_clearance / __cf_bm 只留在浏览器,站点请求不用", async () => {
const r = await json(
await h.req(`/sites/${siteId}/credentials`, {
method: "PUT",
body: { kind: "cookie", label: "c", cookie: "c_secure_uid=1; c_secure_pass=old" },
}),
);
const cid = r.siteAccountId as string;
await h.db.updateTable("site_account").set({ ua_pending: LEGACY_SITE_USER_AGENT }).where("id", "=", cid).execute();
const w = await writeBrowserCookie(
h.ctx,
cid,
"c_secure_uid=1; c_secure_pass=from-browser; cf_clearance=CFSECRET; __cf_bm=BMSECRET",
);
expect(w).toMatchObject({ changed: true, count: 2, uaSwitched: true, browserOnly: ["__cf_bm", "cf_clearance"] });
const acc = await readAccountCookie(h.ctx, cid);
expect(acc!.header).toBe("c_secure_pass=from-browser; c_secure_uid=1");
expect(acc!.header).not.toContain("CFSECRET");
const row = await h.db.selectFrom("site_account").selectAll().where("id", "=", cid).executeTakeFirstOrThrow();
expect(row.ua_pending).toBeNull();
expect(row.user_info).toMatchObject({ browserOnlyCookies: ["__cf_bm", "cf_clearance"] });
const view = await json(await h.req(`/site-accounts/${cid}`));
expect(view).toMatchObject({ userAgentSource: "browser", browserOnlyCookies: ["__cf_bm", "cf_clearance"] });
expect(JSON.stringify(view)).not.toContain("CFSECRET");
// 只多了 CF Cookie、其余不变:不算变化
expect(
await writeBrowserCookie(h.ctx, cid, "c_secure_uid=1; c_secure_pass=from-browser; cf_clearance=OTHER; __cf_bm=X"),
).toMatchObject({ changed: false });
// Cookie 不变但仍待切换:只切 UA
await h.db.updateTable("site_account").set({ ua_pending: LEGACY_SITE_USER_AGENT }).where("id", "=", cid).execute();
expect(await writeBrowserCookie(h.ctx, cid, "c_secure_uid=1; c_secure_pass=from-browser")).toMatchObject({
changed: false,
uaSwitched: true,
});
expect(
(await h.db.selectFrom("site_account").select("ua_pending").where("id", "=", cid).executeTakeFirstOrThrow())
.ua_pending,
).toBeNull();
await h.req(`/site-accounts/${cid}`, { method: "DELETE" });
});
it("refresh-login 从浏览器读回的 Cookie 也剔除 CF Cookie;换主凭证结束待切换", async () => {
await h.db
.updateTable("site_account")
.set({ ua_pending: LEGACY_SITE_USER_AGENT })
.where("id", "=", accountId)
.execute();
await json(
await h.req(`/sites/${siteId}/credentials`, {
method: "PUT",
body: { kind: "cookie", cookie: "c_secure_uid=1; c_secure_pass=stale" },
}),
);
expect((await account()).ua_pending).toBeNull();
await h.db
.updateTable("site_account")
.set({ ua_pending: LEGACY_SITE_USER_AGENT })
.where("id", "=", accountId)
.execute();
browserCookie = `${mock.cookie}; cf_clearance=CFSECRET`;
const n = mock.requests.length;
expect(await refresh()).toMatchObject({ ok: true, status: "active" });
const acc = await readAccountCookie(h.ctx, accountId);
expect(acc!.header).not.toContain("cf_clearance");
expect((await account()).ua_pending).toBeNull();
// 验证浏览器 Cookie 的请求用浏览器 UA,且没有带 cf_clearance
const withBrowserCookie = since(n).filter((r) => String(r.headers.cookie ?? "").includes("mock"));
for (const r of since(n)) expect(String(r.headers.cookie ?? "")).not.toContain("CFSECRET");
expect(withBrowserCookie.every((r) => uaOf(r) === BROWSER_UA)).toBe(true);
});
});
+4 -2
View File
@@ -55,7 +55,8 @@ const DEFAULTS_CARD: CardSpec = {
name: "sites.userAgent",
label: "User-Agent",
kind: "text",
hint: "账号没填 UA 时,登录和拉取用户数据用这个",
placeholder: "跟随浏览器",
hint: "留空跟随浏览器(推荐),站点请求和浏览器用同一个 UA。填了则账号没单独填 UA 时都用它,和浏览器不一致",
wide: true,
},
{
@@ -96,7 +97,8 @@ const DEFAULTS_CARD: CardSpec = {
defaultSnapshotMaxPages: num(v["sites.defaultSnapshotMaxPages"]),
defaultHealthCheckIntervalSec: healthH === undefined ? undefined : Math.round(healthH * 3600),
defaultDetailCacheSec: detailH === undefined ? undefined : Math.round(detailH * 3600),
userAgent: str(v["sites.userAgent"]) || undefined,
// 空串 = 跟随浏览器(要能清空,不能用 undefined 表示"不改")
userAgent: str(v["sites.userAgent"]),
proxyUrl: str(v["sites.proxyUrl"]) || null,
},
};
@@ -234,10 +234,15 @@ export function CredentialFields({
<FormField
label="User-Agent"
htmlFor={id("ua")}
hint={value.kind === "cookie" ? "Cookie 与浏览器 UA 绑定时填写;留空用站点默认" : "留空用站点默认"}
hint={
value.kind === "cookie"
? "留空跟随浏览器(推荐)。Cookie 是从别的浏览器复制的、且站点校验 UA 时,填那个浏览器的 UA"
: "留空跟随浏览器(推荐)"
}
>
<Input
id={id("ua")}
placeholder="跟随浏览器"
value={value.userAgent}
onChange={(e) => set("userAgent", e.target.value)}
disabled={disabled}
@@ -10,6 +10,7 @@ import { FormField } from "~/components/console/form-field.tsx";
import { KvList } from "~/components/console/kv-list.tsx";
import { SecretInput } from "~/components/console/secret-input.tsx";
import { StatusBadge } from "~/components/console/status-badge.tsx";
import { TechInfo } from "~/components/console/tech-info.tsx";
import { Button } from "~/components/ui/button.tsx";
import { relTime } from "~/components/workbench/release-fields.ts";
import { errorMessage } from "~/lib/api.ts";
@@ -143,7 +144,7 @@ function AccountBlock({
label: "会话 Cookie",
value: a.authKind === "password" ? (a.hasSession ? "已保存" : "还没有") : null,
},
{ label: "User-Agent", value: a.userAgent ?? "站点默认", wide: true, mono: !!a.userAgent },
{ label: "User-Agent", value: UA_SOURCE_TEXT[a.userAgentSource], wide: true },
...(a.lastError
? [
{
@@ -159,12 +160,32 @@ function AccountBlock({
: []),
]}
/>
<TechInfo
className="mt-3"
columns={1}
items={[
{ label: "当前 User-Agent", value: a.effectiveUserAgent, wide: true },
{
label: "仅浏览器使用的 Cookie",
value: a.browserOnlyCookies.length ? a.browserOnlyCookies.join(", ") : null,
wide: true,
},
]}
/>
</section>
<LoginHistory accountId={a.id} />
</div>
);
}
/** 站点请求 UA 的来源(BR-4);原始 UA 在技术信息里 */
const UA_SOURCE_TEXT: Record<SiteAccount["userAgentSource"], string> = {
browser: "跟随浏览器",
manual: "手动填写",
global: "使用设置里的全局 UA",
pending: "待切换:下次刷新登录,或在原站视图里用过后改为跟随浏览器",
};
const ACTION_LABEL: Record<string, string> = {
"site_account.login_refreshed": "刷新登录",
"site_account.login_refresh_requested": "请求刷新登录",
+7 -2
View File
@@ -28,6 +28,10 @@ export interface FetchFeedOptions {
signal?: AbortSignal;
/** 手动刷新:忽略退避窗口(不忽略熔断) */
ignoreBackoff?: boolean;
/** 本次请求的 UA(覆盖 RssFetcherOptions.userAgent;BR-4 按站点账号取) */
userAgent?: string;
/** 本次请求用的 fetch(覆盖 RssFetcherOptions.fetch;BR-4 站点代理 = 与站点其它请求同一出口) */
fetch?: typeof globalThis.fetch;
}
export function initialFeedState(): FeedFetchState {
@@ -158,7 +162,8 @@ export function createRssFetcher(options: RssFetcherOptions = {}): RssFetcher {
const headers: Record<string, string> = {
Accept: "application/rss+xml, application/atom+xml, application/xml;q=0.9, text/xml;q=0.8, */*;q=0.1",
};
if (options.userAgent) headers["User-Agent"] = options.userAgent;
const ua = opts.userAgent ?? options.userAgent;
if (ua) headers["User-Agent"] = ua;
if (state.etag) headers["If-None-Match"] = state.etag;
if (state.lastModified) headers["If-Modified-Since"] = state.lastModified;
@@ -166,7 +171,7 @@ export function createRssFetcher(options: RssFetcherOptions = {}): RssFetcher {
const signal = opts.signal ? AbortSignal.any([opts.signal, timeout]) : timeout;
let res: Response;
try {
res = await doFetch(url, { headers, signal, redirect: "follow" });
res = await (opts.fetch ?? doFetch)(url, { headers, signal, redirect: "follow" });
} catch (e) {
if (opts.signal?.aborted) throw e;
const err = e as Error;
+14 -5
View File
@@ -254,8 +254,15 @@ export const MetadataSettings = z.object({
});
export type MetadataSettings = z.infer<typeof MetadataSettings>;
/** 固定桌面 Chrome UA(账号未设 UA 时使用) */
export const DEFAULT_SITE_USER_AGENT =
/**
* 读不到浏览器身份(没有浏览器服务 / 旧镜像 / 暂时连不上)时站点请求用的 UA:与浏览器身份同型的 Windows Firefox
* (camofox 1.17.0 = Camoufox 152;升级浏览器镜像时同步改这里,BR-4)。
*/
export const FALLBACK_SITE_USER_AGENT =
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:152.0) Gecko/20100101 Firefox/152.0";
/** BR-4 之前的缺省 UA(Windows Chrome 140):迁移时据此把存量账号标"待切换",设置里的旧缺省值清空改为跟随浏览器 */
export const LEGACY_SITE_USER_AGENT =
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36";
/** 站点默认(site-center §8):站点行的限流 / 快照周期为空时继承;站点和凭证在站点页管理 */
@@ -268,8 +275,10 @@ export const SitesSettings = z.object({
.int()
.min(300)
.max(7 * 24 * 3600),
/** 账号未设 UA 时使用 */
userAgent: z.string().min(1).max(500),
/**
* 全局 UA 覆盖:空串 = 跟随浏览器(缺省,BR-4);填了则账号没有手动 UA 时都用它(浏览器仍是自己的 UA,两端会不一致)
*/
userAgent: z.string().trim().max(500),
/** 站点开"使用代理"时生效;null = 不用代理 */
proxyUrl: z.url().nullable(),
/** 用户数据做种分页上限 */
@@ -645,7 +654,7 @@ export const DEFAULT_SETTINGS: GlobalSettings = {
sites: {
defaultRateLimit: { requestsPerMinute: 20, concurrency: 2 },
defaultSnapshotIntervalSec: 3600,
userAgent: DEFAULT_SITE_USER_AGENT,
userAgent: "",
proxyUrl: null,
userDataMaxSeedingPages: 10,
defaultSnapshotMaxPages: 1,
+10 -1
View File
@@ -252,8 +252,17 @@ export const SiteAccount = EntityBase.extend({
userInfo: JsonObject,
/** 账密登录名 */
loginName: z.string().nullable(),
/** 空 = 全局默认 UA(settings.sites.userAgent) */
/** 账号手动填的 UA;空 = 不覆盖(见 userAgentSource / effectiveUserAgent) */
userAgent: z.string().nullable(),
/**
* 当前生效 UA 的来源(BR-4):browser = 跟随浏览器;manual = 账号手动填写;global = 设置里的全局 UA;
* pending = 待切换(沿用取得当前 Cookie 时的旧 UA,下次重新登录或浏览器回写 Cookie 时换成浏览器 UA)
*/
userAgentSource: z.enum(["browser", "manual", "global", "pending"]),
/** 当前站点请求实际发送的 UA(技术信息) */
effectiveUserAgent: z.string(),
/** 仅浏览器用、没有回写给站点请求的 Cookie 名(Cloudflare 验证 Cookie,不含值) */
browserOnlyCookies: z.array(z.string()),
/** 最近一次登录 / 校验成功(Cookie、API Key 校验成功也算) */
lastLoginAt: Timestamp.nullable(),
lastError: SiteAccountError.nullable(),
+5 -3
View File
@@ -83,8 +83,9 @@ const ROWS: Record<string, { row: Obj; dbOnly?: string[]; apiOnly?: string[] }>
SiteUserData: { row: C.SiteUserData },
SiteAccount: {
row: C.SiteAccount,
dbOnly: ["totpSecretRef", "sessionSecretRef", "passkeySecretRef"],
apiOnly: ["hasTotp", "hasSession", "hasPasskey"],
// BR-4:ua_pending 只给解析后的 userAgentSource / effectiveUserAgent;browserOnlyCookies 取自 user_info
dbOnly: ["totpSecretRef", "sessionSecretRef", "passkeySecretRef", "uaPending"],
apiOnly: ["hasTotp", "hasSession", "hasPasskey", "userAgentSource", "effectiveUserAgent", "browserOnlyCookies"],
},
SourceRelease: { row: C.SourceRelease },
Subscription: { row: C.Subscription, apiOnly: ["mediaEntityTitle"] },
@@ -100,7 +101,8 @@ const ROWS: Record<string, { row: Obj; dbOnly?: string[]; apiOnly?: string[] }>
ResolutionAttempt: { row: C.ResolutionAttempt },
SiteSnapshot: { row: C.SiteSnapshot },
// W4-C 浏览器实体(0023)
BrowserProvider: { row: C.BrowserProviderRow },
// BR-4:identity 是浏览器固定身份的缓存(站点请求 UA 用),不进 API
BrowserProvider: { row: C.BrowserProviderRow, dbOnly: ["identity"] },
BrowserSession: { row: C.BrowserSession },
BrowserTab: { row: C.BrowserTab },
BrowserControlLease: { row: C.BrowserControlLease },
@@ -0,0 +1,47 @@
import { type Kysely, sql } from "kysely";
import { execDdl } from "../src/migration-helpers.ts";
// BR-4 浏览器 UA 与指纹统一(docs/runbook/browser.md §10):
// - site_account.ua_pending:待切换。存量 Cookie / 账密账号的登录态是用旧缺省 UA(Windows Chrome 140)取得的,不立即换 UA,
// 记下旧 UA 继续用;下次重新登录或浏览器回写 Cookie 时与 Cookie 一起切到浏览器 UA(清空本列)。
// 账号手动填了 UA 的不动(手动优先);API Key / Passkey 账号与网页登录态无关,直接跟随浏览器。
// - browser_provider.identity:浏览器服务固定身份的公开部分(UA / OS / 版本,umt 插件 GET /umt/identity)+ 读取时间。
// - settings.sites.userAgent:仍是旧缺省值(Chrome 140)时删掉该键 → 新缺省空串 = 跟随浏览器;用户改过的保留为全局 UA。
// down 只删列;settings 不回填(旧版本读到缺省即 Chrome 140)。
/** BR-4 之前的缺省 UA(与 contracts LEGACY_SITE_USER_AGENT 相同;迁移不依赖应用包) */
export const LEGACY_UA =
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36";
// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型
export async function up(db: Kysely<any>): Promise<void> {
await execDdl(
db,
`alter table site_account add column ua_pending text;
alter table browser_provider add column identity jsonb;`,
);
const row = await db.selectFrom("settings").select("value").where("key", "=", "sites").executeTakeFirst();
const stored = (row?.value as { userAgent?: unknown } | undefined)?.userAgent;
const custom = typeof stored === "string" && stored.trim() && stored.trim() !== LEGACY_UA ? stored.trim() : null;
// 存量账号取得登录态时用的 UA:设置里改过就是改过的值,否则是旧缺省
const previous = custom ?? LEGACY_UA;
await db
.updateTable("site_account")
.set({ ua_pending: previous })
.where("auth_kind", "in", ["cookie", "password"])
.where((eb) => eb.or([eb("user_agent", "is", null), eb("user_agent", "=", "")]))
.where((eb) => eb.or([eb("secret_ref", "is not", null), eb("session_secret_ref", "is not", null)]))
.execute();
if (typeof stored === "string" && stored.trim() === LEGACY_UA) {
await db.updateTable("settings").set({ value: sql`value - 'userAgent'` }).where("key", "=", "sites").execute();
}
}
// biome-ignore lint/suspicious/noExplicitAny: 同上
export async function down(db: Kysely<any>): Promise<void> {
await execDdl(
db,
`alter table site_account drop column if exists ua_pending;
alter table browser_provider drop column if exists identity;`,
);
}
+2
View File
@@ -569,6 +569,7 @@ export interface BrowserProviderTable {
config: Generated<Json>;
created_at: GeneratedTimestamptz;
updated_at: GeneratedTimestamptz;
identity: Json | null;
}
export interface BrowserSessionTable {
@@ -1409,6 +1410,7 @@ export interface SiteAccountTable {
last_error: Json | null;
user_data: Json | null;
user_data_at: Timestamptz | null;
ua_pending: string | null;
}
export interface SiteRequestCountTable {