feat(sites): site requests follow browser UA, smooth UA switch, CF cookies browser-only (BR-4)
- every outbound site request (adapter list/search/detail/download/health, RSS, login, user data) carries the account UA via SourcesRuntime.siteFetch; no more Deno default UA - priority: account manual > pending > global setting > browser identity (fallback: same-shape Windows Firefox UA); settings UA default is empty - migration 0056: existing cookie/password accounts keep their old UA as ua_pending until re-login or browser cookie write-back switches cookie+UA in one transaction; browser_provider.identity caches the browser identity - cf_clearance/__cf_bm stripped on browser write-back (names kept only) - browser sessions get per-site egress (same proxy as HTTP, or direct) - site account page shows UA source; raw UA in tech info Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
ebd07c9230
commit
adc853aa64
21 files changed
+955
-72
No files matched your search
@@ -28,6 +28,10 @@ export interface FetchFeedOptions {
|
||||
signal?: AbortSignal;
|
||||
/** 手动刷新:忽略退避窗口(不忽略熔断) */
|
||||
ignoreBackoff?: boolean;
|
||||
/** 本次请求的 UA(覆盖 RssFetcherOptions.userAgent;BR-4 按站点账号取) */
|
||||
userAgent?: string;
|
||||
/** 本次请求用的 fetch(覆盖 RssFetcherOptions.fetch;BR-4 站点代理 = 与站点其它请求同一出口) */
|
||||
fetch?: typeof globalThis.fetch;
|
||||
}
|
||||
|
||||
export function initialFeedState(): FeedFetchState {
|
||||
@@ -158,7 +162,8 @@ export function createRssFetcher(options: RssFetcherOptions = {}): RssFetcher {
|
||||
const headers: Record<string, string> = {
|
||||
Accept: "application/rss+xml, application/atom+xml, application/xml;q=0.9, text/xml;q=0.8, */*;q=0.1",
|
||||
};
|
||||
if (options.userAgent) headers["User-Agent"] = options.userAgent;
|
||||
const ua = opts.userAgent ?? options.userAgent;
|
||||
if (ua) headers["User-Agent"] = ua;
|
||||
if (state.etag) headers["If-None-Match"] = state.etag;
|
||||
if (state.lastModified) headers["If-Modified-Since"] = state.lastModified;
|
||||
|
||||
@@ -166,7 +171,7 @@ export function createRssFetcher(options: RssFetcherOptions = {}): RssFetcher {
|
||||
const signal = opts.signal ? AbortSignal.any([opts.signal, timeout]) : timeout;
|
||||
let res: Response;
|
||||
try {
|
||||
res = await doFetch(url, { headers, signal, redirect: "follow" });
|
||||
res = await (opts.fetch ?? doFetch)(url, { headers, signal, redirect: "follow" });
|
||||
} catch (e) {
|
||||
if (opts.signal?.aborted) throw e;
|
||||
const err = e as Error;
|
||||
|
||||
@@ -254,8 +254,15 @@ export const MetadataSettings = z.object({
|
||||
});
|
||||
export type MetadataSettings = z.infer<typeof MetadataSettings>;
|
||||
|
||||
/** 固定桌面 Chrome UA(账号未设 UA 时使用) */
|
||||
export const DEFAULT_SITE_USER_AGENT =
|
||||
/**
|
||||
* 读不到浏览器身份(没有浏览器服务 / 旧镜像 / 暂时连不上)时站点请求用的 UA:与浏览器身份同型的 Windows Firefox
|
||||
* (camofox 1.17.0 = Camoufox 152;升级浏览器镜像时同步改这里,BR-4)。
|
||||
*/
|
||||
export const FALLBACK_SITE_USER_AGENT =
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:152.0) Gecko/20100101 Firefox/152.0";
|
||||
|
||||
/** BR-4 之前的缺省 UA(Windows Chrome 140):迁移时据此把存量账号标"待切换",设置里的旧缺省值清空改为跟随浏览器 */
|
||||
export const LEGACY_SITE_USER_AGENT =
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36";
|
||||
|
||||
/** 站点默认(site-center §8):站点行的限流 / 快照周期为空时继承;站点和凭证在站点页管理 */
|
||||
@@ -268,8 +275,10 @@ export const SitesSettings = z.object({
|
||||
.int()
|
||||
.min(300)
|
||||
.max(7 * 24 * 3600),
|
||||
/** 账号未设 UA 时使用 */
|
||||
userAgent: z.string().min(1).max(500),
|
||||
/**
|
||||
* 全局 UA 覆盖:空串 = 跟随浏览器(缺省,BR-4);填了则账号没有手动 UA 时都用它(浏览器仍是自己的 UA,两端会不一致)
|
||||
*/
|
||||
userAgent: z.string().trim().max(500),
|
||||
/** 站点开"使用代理"时生效;null = 不用代理 */
|
||||
proxyUrl: z.url().nullable(),
|
||||
/** 用户数据做种分页上限 */
|
||||
@@ -645,7 +654,7 @@ export const DEFAULT_SETTINGS: GlobalSettings = {
|
||||
sites: {
|
||||
defaultRateLimit: { requestsPerMinute: 20, concurrency: 2 },
|
||||
defaultSnapshotIntervalSec: 3600,
|
||||
userAgent: DEFAULT_SITE_USER_AGENT,
|
||||
userAgent: "",
|
||||
proxyUrl: null,
|
||||
userDataMaxSeedingPages: 10,
|
||||
defaultSnapshotMaxPages: 1,
|
||||
|
||||
@@ -252,8 +252,17 @@ export const SiteAccount = EntityBase.extend({
|
||||
userInfo: JsonObject,
|
||||
/** 账密登录名 */
|
||||
loginName: z.string().nullable(),
|
||||
/** 空 = 全局默认 UA(settings.sites.userAgent) */
|
||||
/** 账号手动填的 UA;空 = 不覆盖(见 userAgentSource / effectiveUserAgent) */
|
||||
userAgent: z.string().nullable(),
|
||||
/**
|
||||
* 当前生效 UA 的来源(BR-4):browser = 跟随浏览器;manual = 账号手动填写;global = 设置里的全局 UA;
|
||||
* pending = 待切换(沿用取得当前 Cookie 时的旧 UA,下次重新登录或浏览器回写 Cookie 时换成浏览器 UA)
|
||||
*/
|
||||
userAgentSource: z.enum(["browser", "manual", "global", "pending"]),
|
||||
/** 当前站点请求实际发送的 UA(技术信息) */
|
||||
effectiveUserAgent: z.string(),
|
||||
/** 仅浏览器用、没有回写给站点请求的 Cookie 名(Cloudflare 验证 Cookie,不含值) */
|
||||
browserOnlyCookies: z.array(z.string()),
|
||||
/** 最近一次登录 / 校验成功(Cookie、API Key 校验成功也算) */
|
||||
lastLoginAt: Timestamp.nullable(),
|
||||
lastError: SiteAccountError.nullable(),
|
||||
|
||||
@@ -83,8 +83,9 @@ const ROWS: Record<string, { row: Obj; dbOnly?: string[]; apiOnly?: string[] }>
|
||||
SiteUserData: { row: C.SiteUserData },
|
||||
SiteAccount: {
|
||||
row: C.SiteAccount,
|
||||
dbOnly: ["totpSecretRef", "sessionSecretRef", "passkeySecretRef"],
|
||||
apiOnly: ["hasTotp", "hasSession", "hasPasskey"],
|
||||
// BR-4:ua_pending 只给解析后的 userAgentSource / effectiveUserAgent;browserOnlyCookies 取自 user_info
|
||||
dbOnly: ["totpSecretRef", "sessionSecretRef", "passkeySecretRef", "uaPending"],
|
||||
apiOnly: ["hasTotp", "hasSession", "hasPasskey", "userAgentSource", "effectiveUserAgent", "browserOnlyCookies"],
|
||||
},
|
||||
SourceRelease: { row: C.SourceRelease },
|
||||
Subscription: { row: C.Subscription, apiOnly: ["mediaEntityTitle"] },
|
||||
@@ -100,7 +101,8 @@ const ROWS: Record<string, { row: Obj; dbOnly?: string[]; apiOnly?: string[] }>
|
||||
ResolutionAttempt: { row: C.ResolutionAttempt },
|
||||
SiteSnapshot: { row: C.SiteSnapshot },
|
||||
// W4-C 浏览器实体(0023)
|
||||
BrowserProvider: { row: C.BrowserProviderRow },
|
||||
// BR-4:identity 是浏览器固定身份的缓存(站点请求 UA 用),不进 API
|
||||
BrowserProvider: { row: C.BrowserProviderRow, dbOnly: ["identity"] },
|
||||
BrowserSession: { row: C.BrowserSession },
|
||||
BrowserTab: { row: C.BrowserTab },
|
||||
BrowserControlLease: { row: C.BrowserControlLease },
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
import { type Kysely, sql } from "kysely";
|
||||
import { execDdl } from "../src/migration-helpers.ts";
|
||||
|
||||
// BR-4 浏览器 UA 与指纹统一(docs/runbook/browser.md §10):
|
||||
// - site_account.ua_pending:待切换。存量 Cookie / 账密账号的登录态是用旧缺省 UA(Windows Chrome 140)取得的,不立即换 UA,
|
||||
// 记下旧 UA 继续用;下次重新登录或浏览器回写 Cookie 时与 Cookie 一起切到浏览器 UA(清空本列)。
|
||||
// 账号手动填了 UA 的不动(手动优先);API Key / Passkey 账号与网页登录态无关,直接跟随浏览器。
|
||||
// - browser_provider.identity:浏览器服务固定身份的公开部分(UA / OS / 版本,umt 插件 GET /umt/identity)+ 读取时间。
|
||||
// - settings.sites.userAgent:仍是旧缺省值(Chrome 140)时删掉该键 → 新缺省空串 = 跟随浏览器;用户改过的保留为全局 UA。
|
||||
// down 只删列;settings 不回填(旧版本读到缺省即 Chrome 140)。
|
||||
|
||||
/** BR-4 之前的缺省 UA(与 contracts LEGACY_SITE_USER_AGENT 相同;迁移不依赖应用包) */
|
||||
export const LEGACY_UA =
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36";
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型
|
||||
export async function up(db: Kysely<any>): Promise<void> {
|
||||
await execDdl(
|
||||
db,
|
||||
`alter table site_account add column ua_pending text;
|
||||
alter table browser_provider add column identity jsonb;`,
|
||||
);
|
||||
const row = await db.selectFrom("settings").select("value").where("key", "=", "sites").executeTakeFirst();
|
||||
const stored = (row?.value as { userAgent?: unknown } | undefined)?.userAgent;
|
||||
const custom = typeof stored === "string" && stored.trim() && stored.trim() !== LEGACY_UA ? stored.trim() : null;
|
||||
// 存量账号取得登录态时用的 UA:设置里改过就是改过的值,否则是旧缺省
|
||||
const previous = custom ?? LEGACY_UA;
|
||||
await db
|
||||
.updateTable("site_account")
|
||||
.set({ ua_pending: previous })
|
||||
.where("auth_kind", "in", ["cookie", "password"])
|
||||
.where((eb) => eb.or([eb("user_agent", "is", null), eb("user_agent", "=", "")]))
|
||||
.where((eb) => eb.or([eb("secret_ref", "is not", null), eb("session_secret_ref", "is not", null)]))
|
||||
.execute();
|
||||
if (typeof stored === "string" && stored.trim() === LEGACY_UA) {
|
||||
await db.updateTable("settings").set({ value: sql`value - 'userAgent'` }).where("key", "=", "sites").execute();
|
||||
}
|
||||
}
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 同上
|
||||
export async function down(db: Kysely<any>): Promise<void> {
|
||||
await execDdl(
|
||||
db,
|
||||
`alter table site_account drop column if exists ua_pending;
|
||||
alter table browser_provider drop column if exists identity;`,
|
||||
);
|
||||
}
|
||||
@@ -569,6 +569,7 @@ export interface BrowserProviderTable {
|
||||
config: Generated<Json>;
|
||||
created_at: GeneratedTimestamptz;
|
||||
updated_at: GeneratedTimestamptz;
|
||||
identity: Json | null;
|
||||
}
|
||||
|
||||
export interface BrowserSessionTable {
|
||||
@@ -1409,6 +1410,7 @@ export interface SiteAccountTable {
|
||||
last_error: Json | null;
|
||||
user_data: Json | null;
|
||||
user_data_at: Timestamptz | null;
|
||||
ua_pending: string | null;
|
||||
}
|
||||
|
||||
export interface SiteRequestCountTable {
|
||||
|
||||
Reference in new issue
Block a user