feat(sites): site requests follow browser UA, smooth UA switch, CF cookies browser-only (BR-4)
- every outbound site request (adapter list/search/detail/download/health, RSS, login, user data) carries the account UA via SourcesRuntime.siteFetch; no more Deno default UA - priority: account manual > pending > global setting > browser identity (fallback: same-shape Windows Firefox UA); settings UA default is empty - migration 0056: existing cookie/password accounts keep their old UA as ua_pending until re-login or browser cookie write-back switches cookie+UA in one transaction; browser_provider.identity caches the browser identity - cf_clearance/__cf_bm stripped on browser write-back (names kept only) - browser sessions get per-site egress (same proxy as HTTP, or direct) - site account page shows UA source; raw UA in tech info Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
ebd07c9230
commit
adc853aa64
21 files changed
+955
-72
No files matched your search
@@ -114,12 +114,16 @@ function open(
|
||||
*/
|
||||
export function createSecretStore(deps: { db: Kysely<DB>; masterKey: string }): SecretService {
|
||||
const { db, masterKey } = deps;
|
||||
const replace = async (secretRef: string, plaintext: string) => {
|
||||
const replace = async (secretRef: string, plaintext: string, exec: Kysely<DB> = db) => {
|
||||
assertRef(secretRef);
|
||||
const row = await db.selectFrom("secret").select(["id", "purpose"]).where("id", "=", secretRef).executeTakeFirst();
|
||||
const row = await exec
|
||||
.selectFrom("secret")
|
||||
.select(["id", "purpose"])
|
||||
.where("id", "=", secretRef)
|
||||
.executeTakeFirst();
|
||||
if (!row) throw new SecretError("not_found", secretRef);
|
||||
const s = seal(masterKey, row.id, row.purpose, plaintext);
|
||||
await db
|
||||
await exec
|
||||
.updateTable("secret")
|
||||
.set({
|
||||
ciphertext: s.ciphertext,
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
import { MOCK_CAMOFOX_IDENTITY } from "@umt/testkit";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { sourcesIdentityModule } from "../../sources-identity/index.ts";
|
||||
import { readBrowserUserAgent } from "../../sources-identity/service/user-agent.ts";
|
||||
import { egressProxy } from "../service.ts";
|
||||
import { type BrowserHarness, createBrowserHarness, type Loose, ORIGIN, pgAvailable } from "./harness.ts";
|
||||
|
||||
// BR-4:打开会话前按站点下发出口(站点开代理 = 与 HTTP 同一代理,否则直连);浏览器固定身份写进 browser_provider.identity,
|
||||
// 站点请求 UA 经钩子跟随它。
|
||||
|
||||
const up = await pgAvailable();
|
||||
|
||||
describe("egressProxy", () => {
|
||||
it("URL → 上下文代理(口令拆出);非法为 null", () => {
|
||||
expect(egressProxy("http://u:p%[email protected]:7891")).toEqual({
|
||||
server: "http://172.17.0.1:7891",
|
||||
username: "u",
|
||||
password: "p@ss",
|
||||
});
|
||||
expect(egressProxy("socks5://127.0.0.1:1080")).toEqual({ server: "socks5://127.0.0.1:1080" });
|
||||
expect(egressProxy("not a url")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!up)("BR-4 浏览器身份与出口", () => {
|
||||
let h: BrowserHarness;
|
||||
|
||||
const open = async () =>
|
||||
(await h.req("/browser/sessions", { body: { siteId: h.siteId, accountId: null, url: `${ORIGIN}/torrents.php` } }))
|
||||
.body.data as Loose;
|
||||
const closeAll = async () => {
|
||||
for (const s of (await h.req("/browser/sessions")).body.data.items as Loose[])
|
||||
await h.req(`/browser/sessions/${s.id}`, { method: "DELETE" });
|
||||
};
|
||||
|
||||
beforeAll(async () => {
|
||||
h = await createBrowserHarness("br4id");
|
||||
}, 120_000);
|
||||
afterAll(async () => {
|
||||
await h?.close();
|
||||
});
|
||||
|
||||
it("站点不开代理:该 profile 直连;打开时读浏览器身份存库,站点请求 UA 跟随", async () => {
|
||||
const s = await open();
|
||||
expect(h.mock.egress.get(s.profileKey)).toBeNull();
|
||||
const calls = h.mock.calls.map((c) => `${c.method} ${c.path}`);
|
||||
expect(calls.indexOf(`PUT /umt/sessions/${encodeURIComponent(s.profileKey)}/proxy`)).toBeLessThan(
|
||||
calls.lastIndexOf("POST /tabs"),
|
||||
);
|
||||
const row = await h.db
|
||||
.selectFrom("browser_provider")
|
||||
.select("identity")
|
||||
.where("id", "=", h.providerId)
|
||||
.executeTakeFirstOrThrow();
|
||||
expect(row.identity).toMatchObject({ userAgent: MOCK_CAMOFOX_IDENTITY.userAgent, os: "windows" });
|
||||
expect(await readBrowserUserAgent(h.ctx)).toBe(MOCK_CAMOFOX_IDENTITY.userAgent);
|
||||
expect(await sourcesIdentityModule.runtime(h.ctx).defaultUserAgent()).toBe(MOCK_CAMOFOX_IDENTITY.userAgent);
|
||||
await closeAll();
|
||||
});
|
||||
|
||||
it("站点开代理且设了代理地址:该 profile 走同一代理(口令单独传)", async () => {
|
||||
await h.db.updateTable("site").set({ proxy: true }).where("id", "=", h.siteId).execute();
|
||||
await h.db
|
||||
.insertInto("settings")
|
||||
.values({ key: "sites", value: JSON.stringify({ proxyUrl: "http://user:[email protected]:7891" }) })
|
||||
.onConflict((oc) =>
|
||||
oc.column("key").doUpdateSet({ value: JSON.stringify({ proxyUrl: "http://user:[email protected]:7891" }) }),
|
||||
)
|
||||
.execute();
|
||||
const s = await open();
|
||||
expect(h.mock.egress.get(s.profileKey)).toEqual({
|
||||
server: "http://172.17.0.1:7891",
|
||||
username: "user",
|
||||
password: "pw",
|
||||
});
|
||||
await closeAll();
|
||||
await h.db.updateTable("site").set({ proxy: false }).where("id", "=", h.siteId).execute();
|
||||
});
|
||||
});
|
||||
@@ -29,8 +29,10 @@ import type {
|
||||
BrowserCoordinateSpace,
|
||||
BrowserCredentialState,
|
||||
BrowserDownloadVerification,
|
||||
BrowserEgressProxy,
|
||||
BrowserEvidenceRef,
|
||||
BrowserHealth,
|
||||
BrowserIdentity,
|
||||
BrowserProviderView,
|
||||
BrowserReconnectState,
|
||||
BrowserSession,
|
||||
@@ -55,6 +57,7 @@ import {
|
||||
registerBrowserCookieReader,
|
||||
writeBrowserCookie,
|
||||
} from "../sources-identity/service/credentials.ts";
|
||||
import { registerBrowserIdentityReader } from "../sources-identity/service/user-agent.ts";
|
||||
import type { ServerContext } from "../types.ts";
|
||||
|
||||
// C1-E 浏览器服务层:BrowserProvider(camofox)之上的会话/标签/控制权持久化(数据库为准)、
|
||||
@@ -199,6 +202,95 @@ export class BrowserService {
|
||||
this.now = opts.now ?? (() => new Date());
|
||||
// refresh-login 发现 Cookie 失效时从浏览器 profile 读回一次(sources-identity 经钩子调用,不直接依赖本模块)
|
||||
registerBrowserCookieReader(ctx, (siteId, accountId) => this.readProfileCookie(siteId, accountId));
|
||||
// 站点请求 UA 跟随浏览器固定身份(BR-4):sources-identity 经钩子取
|
||||
registerBrowserIdentityReader(ctx, () => this.identityUserAgent());
|
||||
}
|
||||
|
||||
// ---------- BR-4 浏览器身份与出口 ----------
|
||||
|
||||
/** 身份读取失败后多久再试(浏览器服务没起 / 旧镜像时不每条站点请求都去问) */
|
||||
private identityRetryAt = 0;
|
||||
|
||||
/**
|
||||
* 浏览器固定身份的 UA:库里 browser_provider.identity 6 小时内读过就用它;否则问浏览器服务并写回。
|
||||
* 没有启用的浏览器服务 / 插件不支持时返回库里的旧值或 null。
|
||||
*/
|
||||
async identityUserAgent(): Promise<string | null> {
|
||||
const prow = await this.providerRow().catch(() => null);
|
||||
if (!prow) return null;
|
||||
const stored = obj(prow.identity) as { userAgent?: string; fetchedAt?: string };
|
||||
const age = stored.fetchedAt ? Date.now() - Date.parse(stored.fetchedAt) : Number.POSITIVE_INFINITY;
|
||||
if (stored.userAgent && age < 6 * 3_600_000) return stored.userAgent;
|
||||
if (Date.now() < this.identityRetryAt) return stored.userAgent ?? null;
|
||||
const fresh = await this.refreshIdentity(prow).catch(() => null);
|
||||
return fresh?.userAgent ?? stored.userAgent ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 读浏览器服务的固定身份写进 browser_provider.identity(UA / OS / 版本 / 生成时间 + 读取时间)。
|
||||
* maxAgeMs:库里的值比这新就不问。失败 5 分钟内不重试。
|
||||
*/
|
||||
async refreshIdentity(prow: BrowserProviderRow, opts: { maxAgeMs?: number } = {}): Promise<BrowserIdentity | null> {
|
||||
const stored = obj(prow.identity) as Partial<BrowserIdentity> & { fetchedAt?: string };
|
||||
if (opts.maxAgeMs !== undefined && stored.userAgent && stored.fetchedAt) {
|
||||
if (Date.now() - Date.parse(stored.fetchedAt) < opts.maxAgeMs) return stored as BrowserIdentity;
|
||||
}
|
||||
const provider = this.providerFor(prow);
|
||||
if (!provider.identity) return null;
|
||||
let id: BrowserIdentity | null;
|
||||
try {
|
||||
id = await provider.identity();
|
||||
} catch (e) {
|
||||
this.identityRetryAt = Date.now() + 5 * 60_000;
|
||||
throw e;
|
||||
}
|
||||
if (!id) {
|
||||
this.identityRetryAt = Date.now() + 5 * 60_000;
|
||||
return null;
|
||||
}
|
||||
this.identityRetryAt = 0;
|
||||
await this.ctx.db
|
||||
.updateTable("browser_provider")
|
||||
.set({ identity: JSON.stringify({ ...id, fetchedAt: this.now().toISOString() }) })
|
||||
.where("id", "=", prow.id)
|
||||
.execute();
|
||||
if (stored.userAgent !== id.userAgent)
|
||||
this.ctx.logger.info(
|
||||
{ component: "browser", providerId: prow.id, os: id.os, firefoxVersion: id.firefoxVersion },
|
||||
"浏览器身份已更新,站点请求 UA 跟随",
|
||||
);
|
||||
return id;
|
||||
}
|
||||
|
||||
/**
|
||||
* 出口按站点统一(BR-4):站点开了代理且设置了代理地址 → 该 profile 走同一代理;否则直连。
|
||||
* 旧镜像(插件没有出口接口)只记日志,浏览器照旧走容器环境变量里的代理。
|
||||
*/
|
||||
private async applyEgress(provider: BrowserProvider, site: SiteRow, profileKey: string): Promise<void> {
|
||||
if (!provider.setEgress) return;
|
||||
const rt = this.opts.runtime?.(this.ctx);
|
||||
const proxyUrl = rt
|
||||
? (await rt.siteDefaults()).proxyUrl
|
||||
: ((await this.ctx.settings.get("sites").catch(() => null))?.proxyUrl ?? null);
|
||||
const proxy = site.proxy && proxyUrl ? egressProxy(proxyUrl) : null;
|
||||
try {
|
||||
const r = await provider.setEgress(profileKey, proxy);
|
||||
if (r.restarted)
|
||||
this.ctx.logger.info(
|
||||
{ component: "browser", siteId: site.id, egress: proxy ? "proxy" : "direct" },
|
||||
"浏览器会话出口已变更,按新出口重建",
|
||||
);
|
||||
} catch (e) {
|
||||
this.ctx.logger.warn(
|
||||
{
|
||||
component: "browser",
|
||||
siteId: site.id,
|
||||
egress: proxy ? "proxy" : "direct",
|
||||
err: String((e as Error)?.message ?? e),
|
||||
},
|
||||
"设置浏览器会话出口失败,浏览器按容器环境变量出网",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- 浏览器服务 ----------
|
||||
@@ -486,6 +578,8 @@ export class BrowserService {
|
||||
.where("id", "=", row.id)
|
||||
.returningAll()
|
||||
.executeTakeFirstOrThrow();
|
||||
// 探测时顺便读一次浏览器身份(站点请求 UA 跟随它,BR-4)
|
||||
if (health.status !== "down" && row.enabled) await this.refreshIdentity(row).catch(() => null);
|
||||
}
|
||||
const caps = obj(row.capabilities);
|
||||
out.push({
|
||||
@@ -740,6 +834,8 @@ export class BrowserService {
|
||||
for (const m of await provider.listSessions())
|
||||
if (m.profileKey === profileKey && m.controller !== "closed") provider.setController(m.id, "closed");
|
||||
const prep = await this.prepareCookies(site, accountId);
|
||||
await this.applyEgress(provider, site, profileKey);
|
||||
await this.refreshIdentity(prow, { maxAgeMs: 10 * 60_000 }).catch(() => null);
|
||||
await this.rateLimit(site);
|
||||
let res: Awaited<ReturnType<BrowserProvider["openSession"]>>;
|
||||
try {
|
||||
@@ -1416,6 +1512,9 @@ export class BrowserService {
|
||||
if (!provider.reopenSession) throw await fail("浏览器服务不支持重新连接");
|
||||
const url = restoreUrl(row.current_url, budget) ?? site.base_url;
|
||||
const prep = await this.prepareCookies(site, row.account_id).catch(() => null);
|
||||
await this.applyEgress(provider, site, row.profile_key);
|
||||
// 浏览器重启过(可能换了镜像版本):身份重新读
|
||||
await this.refreshIdentity(prow).catch(() => null);
|
||||
let res: Awaited<ReturnType<NonNullable<BrowserProvider["reopenSession"]>>>;
|
||||
try {
|
||||
await this.rateLimit(site);
|
||||
@@ -1944,3 +2043,18 @@ export function imageSize(data: Uint8Array): { width: number | null; height: num
|
||||
}
|
||||
|
||||
export type { ElementTarget };
|
||||
|
||||
/** 代理地址 → 浏览器上下文代理(账号口令拆出来;Playwright 不认 URL 里的口令) */
|
||||
export function egressProxy(url: string): BrowserEgressProxy | null {
|
||||
try {
|
||||
const u = new URL(url);
|
||||
if (!["http:", "https:", "socks4:", "socks5:"].includes(u.protocol) || !u.hostname) return null;
|
||||
return {
|
||||
server: `${u.protocol}//${u.host}`,
|
||||
...(u.username ? { username: decodeURIComponent(u.username) } : {}),
|
||||
...(u.password ? { password: decodeURIComponent(u.password) } : {}),
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -128,9 +128,10 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
|
||||
)
|
||||
.orderBy("created_at")
|
||||
.execute();
|
||||
const ua = await rt().uaDefaults();
|
||||
for (const it of items) {
|
||||
const mine = accounts.filter((a) => a.site_id === it.id);
|
||||
it.accounts = mine.map(accountToApi);
|
||||
it.accounts = mine.map((a) => accountToApi(a, ua));
|
||||
it.accountStatus = overallAccountStatus(mine);
|
||||
}
|
||||
}
|
||||
@@ -483,12 +484,13 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
|
||||
.limit(query.limit + 1)
|
||||
.execute();
|
||||
const p = page(rows, off, query.limit);
|
||||
return { items: p.items.map(accountToApi) as SiteAccount[], nextCursor: p.nextCursor };
|
||||
const ua = await rt().uaDefaults();
|
||||
return { items: p.items.map((a) => accountToApi(a, ua)) as SiteAccount[], nextCursor: p.nextCursor };
|
||||
});
|
||||
route(app, D.siteAccounts.get, async ({ params }) => {
|
||||
const row = await db().selectFrom("site_account").selectAll().where("id", "=", params.id).executeTakeFirst();
|
||||
if (!row) throw new ApiException("not_found", "站点账号不存在");
|
||||
return accountToApi(row);
|
||||
return accountToApi(row, await rt().uaDefaults());
|
||||
});
|
||||
route(app, D.siteAccounts.create, async ({ body, actor }) => {
|
||||
await getSite(body.siteId);
|
||||
@@ -510,7 +512,7 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
|
||||
.returningAll()
|
||||
.executeTakeFirstOrThrow();
|
||||
await audit(db(), actor, "site_account.created", { type: "site_account", id: row.id }, { siteId: body.siteId });
|
||||
return accountToApi(row);
|
||||
return accountToApi(row, await rt().uaDefaults());
|
||||
} catch (e) {
|
||||
await rt()
|
||||
.ctx.secrets.remove(ref)
|
||||
@@ -539,6 +541,8 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
|
||||
patch.status = "unknown";
|
||||
patch.last_error = null;
|
||||
patch.session_secret_ref = null;
|
||||
// 新凭证 = 新登录态:不再沿用旧 UA(BR-4)
|
||||
patch.ua_pending = null;
|
||||
}
|
||||
const row = await db()
|
||||
.transaction()
|
||||
@@ -585,7 +589,7 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
|
||||
);
|
||||
return r;
|
||||
});
|
||||
return accountToApi(row);
|
||||
return accountToApi(row, await rt().uaDefaults());
|
||||
});
|
||||
|
||||
// ── Feed ──
|
||||
|
||||
@@ -24,6 +24,7 @@ import {
|
||||
tmdbDetailRef,
|
||||
} from "../../shared-cache.ts";
|
||||
import type { ServerContext } from "../types.ts";
|
||||
import { readBrowserUserAgent, resolveUserAgent, type UserAgentDefaults } from "./service/user-agent.ts";
|
||||
|
||||
// 模块运行时依赖:站点适配器与限流器(每站一个实例,令牌在 PG 跨进程共享)、RSS 抓取器、TMDB 客户端、时钟。
|
||||
// 测试通过 createSourcesIdentityModule(overrides) 注入 fetch / TMDB / 时钟。
|
||||
@@ -168,8 +169,28 @@ export class SourcesRuntime {
|
||||
|
||||
/**
|
||||
* 站点请求用的 fetch:测试注入优先;站点开了"使用代理"且设置了 proxyUrl 时走代理(Deno.createHttpClient)。
|
||||
* BR-4:每条请求都带 UA(请求自己设了就不动;否则 ua 参数,缺省按"跟随浏览器"),不再发 Deno 缺省的 `Deno/x.y.z`。
|
||||
* ua 可以是函数:每次请求现场取(站点适配器按当前账号取,账号切换 UA 后立即生效)。
|
||||
*/
|
||||
siteFetch(site: Pick<SiteRow, "proxy">, defaults: SitesSettings = this.siteDefaultsCache): typeof globalThis.fetch {
|
||||
siteFetch(
|
||||
site: Pick<SiteRow, "proxy">,
|
||||
defaults: SitesSettings = this.siteDefaultsCache,
|
||||
ua?: string | (() => Promise<string>),
|
||||
): typeof globalThis.fetch {
|
||||
const base = this.transportFetch(site, defaults);
|
||||
return ((input: RequestInfo | URL, init?: RequestInit) => {
|
||||
const headers = new Headers(init?.headers ?? (input instanceof Request ? input.headers : undefined));
|
||||
if (headers.has("user-agent")) return base(input, { ...init, headers });
|
||||
const pick = typeof ua === "function" ? ua() : ua !== undefined ? Promise.resolve(ua) : this.defaultUserAgent();
|
||||
return pick.then((v) => {
|
||||
headers.set("user-agent", v);
|
||||
return base(input, { ...init, headers });
|
||||
});
|
||||
}) as typeof globalThis.fetch;
|
||||
}
|
||||
|
||||
/** 不加 UA 的底层 fetch(代理 / 测试注入) */
|
||||
private transportFetch(site: Pick<SiteRow, "proxy">, defaults: SitesSettings): typeof globalThis.fetch {
|
||||
if (this.overrides.fetch) return this.overrides.fetch;
|
||||
if (!site.proxy || !defaults.proxyUrl) return globalThis.fetch;
|
||||
const key = defaults.proxyUrl;
|
||||
@@ -182,6 +203,56 @@ export class SourcesRuntime {
|
||||
return ((input: RequestInfo | URL, init?: RequestInit) =>
|
||||
globalThis.fetch(input, { ...init, client: c } as RequestInit)) as typeof globalThis.fetch;
|
||||
}
|
||||
|
||||
// ---------- BR-4 站点请求 UA ----------
|
||||
|
||||
private browserUa: { at: number; ua: string | null } = { at: 0, ua: null };
|
||||
|
||||
/** 浏览器固定身份的 UA(60 秒缓存);读不到为 null(调用方退回常量) */
|
||||
async browserUserAgent(): Promise<string | null> {
|
||||
if (Date.now() - this.browserUa.at < 60_000) return this.browserUa.ua;
|
||||
let ua: string | null = this.browserUa.ua;
|
||||
try {
|
||||
ua = (await readBrowserUserAgent(this.ctx)) ?? ua;
|
||||
} catch (err) {
|
||||
this.ctx.logger.debug({ err: { message: (err as Error).message } }, "读取浏览器身份 UA 失败,沿用上次的值");
|
||||
}
|
||||
this.browserUa = { at: Date.now(), ua };
|
||||
return ua;
|
||||
}
|
||||
|
||||
/** UA 解析用的全局部分:设置里的全局 UA + 浏览器 UA(已退回常量) */
|
||||
async uaDefaults(): Promise<UserAgentDefaults> {
|
||||
const [s, browser] = await Promise.all([this.siteDefaults(), this.browserUserAgent()]);
|
||||
return { global: s.userAgent ?? "", browser: browser ?? "" };
|
||||
}
|
||||
|
||||
/** 不挂账号的请求(通用 RSS 等)用的 UA:全局 UA 或跟随浏览器 */
|
||||
async defaultUserAgent(): Promise<string> {
|
||||
return resolveUserAgent(null, await this.uaDefaults()).ua;
|
||||
}
|
||||
|
||||
/**
|
||||
* 站点请求 UA:取与站点凭证相同的账号(第一个启用、未失效、非 Passkey 的账号;没有就第一个启用的),按账号解析。
|
||||
* 不缓存:与 Cookie 同一时刻读库,账号切换 Cookie + UA 后下一条请求两者一起生效。
|
||||
*/
|
||||
async siteUserAgent(siteId: string): Promise<string> {
|
||||
const d = await this.uaDefaults();
|
||||
const rows = await this.ctx.db
|
||||
.selectFrom("site_account")
|
||||
.select(["user_agent", "ua_pending", "status", "auth_kind"])
|
||||
.where("site_id", "=", siteId)
|
||||
.where("enabled", "=", true)
|
||||
.orderBy("created_at")
|
||||
.execute();
|
||||
const acc =
|
||||
rows.find((r) => r.auth_kind !== "passkey" && (r.status === "unknown" || r.status === "active")) ??
|
||||
rows.find((r) => r.auth_kind !== "passkey") ??
|
||||
rows[0] ??
|
||||
null;
|
||||
return resolveUserAgent(acc, d).ua;
|
||||
}
|
||||
|
||||
private readonly proxyClients = new Map<string, Deno.HttpClient>();
|
||||
|
||||
/**
|
||||
@@ -232,7 +303,8 @@ export class SourcesRuntime {
|
||||
secrets: this.siteSecrets(site.id),
|
||||
limiter,
|
||||
now: this.now,
|
||||
...(this.overrides.fetch || proxy ? { fetch: this.siteFetch({ proxy }, defaults) } : {}),
|
||||
// 每条请求都经 siteFetch:代理 + 按账号现场取 UA(BR-4)
|
||||
fetch: this.siteFetch({ proxy }, defaults, () => this.siteUserAgent(site.id)),
|
||||
...(baseUrl ? { baseUrl } : {}),
|
||||
});
|
||||
this.adapters.set(site.id, { sig, limiter, adapter });
|
||||
|
||||
@@ -29,20 +29,25 @@ import type { SourcesRuntime } from "../runtime.ts";
|
||||
import { capabilitiesOf, LOGIN_KINDS } from "../site-kinds.ts";
|
||||
import { type Actor, audit, closeDecisions, type Exec, openDecision, SYSTEM } from "./common.ts";
|
||||
import { cookieFromBrowser, writeBrowserCookie } from "./credentials.ts";
|
||||
import { browserOnlyCookiesOf, resolveUserAgent, type UserAgentDefaults } from "./user-agent.ts";
|
||||
|
||||
// W9-A 站点中心(docs/design/site-center.md):凭证写入、刷新登录(账密 takelogin / Cookie / API Key 校验)、
|
||||
// 用户数据与按天历史、健康检测、删除。凭证明文只在内存里用,不进日志、审计与错误信息。
|
||||
|
||||
const jsonb = (v: unknown) => JSON.stringify(v ?? null);
|
||||
|
||||
/** site_account 行 → SiteAccount(secret 引用只给布尔) */
|
||||
export function accountToApi(row: SiteAccountRow): Record<string, unknown> {
|
||||
const { totp_secret_ref, session_secret_ref, passkey_secret_ref, ...rest } = row;
|
||||
/** site_account 行 → SiteAccount(secret 引用只给布尔);ua = 全局 UA 与浏览器 UA(SourcesRuntime.uaDefaults) */
|
||||
export function accountToApi(row: SiteAccountRow, ua: UserAgentDefaults): Record<string, unknown> {
|
||||
const { totp_secret_ref, session_secret_ref, passkey_secret_ref, ua_pending: _p, ...rest } = row;
|
||||
const eff = resolveUserAgent(row, ua);
|
||||
return {
|
||||
...toApi<Record<string, unknown>>(rest),
|
||||
hasTotp: !!totp_secret_ref,
|
||||
hasSession: !!session_secret_ref,
|
||||
hasPasskey: !!passkey_secret_ref || (row.auth_kind === "passkey" && !!row.secret_ref),
|
||||
userAgentSource: eff.source,
|
||||
effectiveUserAgent: eff.ua,
|
||||
browserOnlyCookies: browserOnlyCookiesOf(row.user_info),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -283,6 +288,8 @@ export async function setSiteCredentials(
|
||||
totp_secret_ref: totpRef,
|
||||
login_name: body.kind === "password" ? body.username : null,
|
||||
user_agent: "userAgent" in body ? (body.userAgent ?? null) : null,
|
||||
// 换了主凭证 = 新的登录态,不再沿用旧 UA(BR-4 待切换结束);只改 UA / 启用时保持
|
||||
...(keepMain ? {} : { ua_pending: null }),
|
||||
user_info: jsonb(userInfo),
|
||||
...(body.enabled !== undefined ? { enabled: body.enabled } : {}),
|
||||
};
|
||||
@@ -388,7 +395,10 @@ export async function refreshSiteAccount(
|
||||
const def = getSiteDefinition(site.key);
|
||||
if (!def || site.deleted_at) return unchanged("unsupported", "站点没有适配器定义,无法校验登录");
|
||||
const defaults = await rt.siteDefaults();
|
||||
const ua = acc.user_agent ?? defaults.userAgent;
|
||||
// BR-4:current = 当前登录态配套的 UA(待切换时是旧 UA);target = 切换后的 UA(重新登录、浏览器 Cookie 用它)
|
||||
const uaDefaults = await rt.uaDefaults();
|
||||
const currentUa = resolveUserAgent(acc, uaDefaults).ua;
|
||||
const targetUa = resolveUserAgent(acc, uaDefaults, { ignorePending: true }).ua;
|
||||
let limiter: ReturnType<SourcesRuntime["limiterOf"]> = null;
|
||||
try {
|
||||
rt.siteAdapter(site);
|
||||
@@ -396,7 +406,7 @@ export async function refreshSiteAccount(
|
||||
} catch {
|
||||
// 无适配器:不限流(仅 generic_rss,前面已返回)
|
||||
}
|
||||
const fetch = rt.siteFetch(site, defaults);
|
||||
const fetch = rt.siteFetch(site, defaults, currentUa);
|
||||
const umt = def.umt as { kind: string; login?: Parameters<typeof loginNexusPhp>[0]["login"]; loginSuccess?: string };
|
||||
const profile = (def.umt as { profile?: Record<string, string> }).profile;
|
||||
const vipLevels = (def.umt as { vipLevels?: string[] }).vipLevels;
|
||||
@@ -406,7 +416,10 @@ export async function refreshSiteAccount(
|
||||
? (site.config as { apiBaseUrl?: string })?.apiBaseUrl || mteamApiBase(def as MteamDefinition)
|
||||
: null;
|
||||
|
||||
const fetchUserData = async (cred: { cookie?: string | null; apikey?: string | null }): Promise<UserDataOutcome> => {
|
||||
const fetchUserData = async (
|
||||
cred: { cookie?: string | null; apikey?: string | null },
|
||||
ua: string = currentUa,
|
||||
): Promise<UserDataOutcome> => {
|
||||
const info = await refreshSiteUserData({
|
||||
name: site.name,
|
||||
url: site.base_url,
|
||||
@@ -467,14 +480,15 @@ export async function refreshSiteAccount(
|
||||
username: acc.login_name ?? (acc.user_info as { username?: string })?.username ?? acc.label,
|
||||
password,
|
||||
totpSecret,
|
||||
userAgent: ua,
|
||||
userAgent: targetUa,
|
||||
fetch,
|
||||
...(limiter ? { limiter } : {}),
|
||||
...(opts.signal ? { signal: opts.signal } : {}),
|
||||
now: rt.now,
|
||||
});
|
||||
// 新登录态用 targetUa 取得:会话 Cookie 与 UA 一起切换(清掉待切换)
|
||||
await storeSession(rt, acc, login.cookie);
|
||||
const r = await fetchUserData({ cookie: login.cookie });
|
||||
const r = await fetchUserData({ cookie: login.cookie }, targetUa);
|
||||
if (r.kind === "ok") return await succeed(r.info);
|
||||
return await succeed(
|
||||
null,
|
||||
@@ -488,7 +502,8 @@ export async function refreshSiteAccount(
|
||||
// BR-2:存的 Cookie 失效,但浏览器里可能已人工登录过 → 读回浏览器 profile 的 Cookie 试一次,有效才回写
|
||||
const alt = await cookieFromBrowser(rt.ctx, acc, plain);
|
||||
if (alt) {
|
||||
const r2 = await fetchUserData({ cookie: alt });
|
||||
// 浏览器里的登录态是浏览器 UA 取得的:按切换后的 UA 验证,回写时一起切换
|
||||
const r2 = await fetchUserData({ cookie: alt }, targetUa);
|
||||
if (r2.kind === "ok") {
|
||||
await writeBrowserCookie(rt.ctx, acc.id, alt, rt.now());
|
||||
return await succeed(r2.info, "存的 Cookie 已失效;浏览器里的登录态有效,已回写");
|
||||
@@ -515,7 +530,10 @@ export async function refreshSiteAccount(
|
||||
export const secretName = (acc: { site_id: string; label: string }, suffix = "") =>
|
||||
`site_account:${acc.site_id}:${acc.label}${suffix}:${randomUUID().slice(0, 8)}`;
|
||||
|
||||
/** 账密登录成功:会话 Cookie 存 secret;"只换 Cookie"时凭证类型落为 cookie,密码与 TOTP 丢弃 */
|
||||
/**
|
||||
* 账密登录成功:会话 Cookie 存 secret;"只换 Cookie"时凭证类型落为 cookie,密码与 TOTP 丢弃。
|
||||
* 登录用的是切换后的 UA:写 Cookie 的同一事务里清掉待切换(BR-4 Cookie + UA 原子切换)。
|
||||
*/
|
||||
async function storeSession(rt: SourcesRuntime, acc: SiteAccountRow, cookie: string) {
|
||||
const { db, secrets } = rt.ctx;
|
||||
const cookieOnly = (acc.user_info as { saveCookieOnly?: boolean })?.saveCookieOnly === true;
|
||||
@@ -531,6 +549,7 @@ async function storeSession(rt: SourcesRuntime, acc: SiteAccountRow, cookie: str
|
||||
session_secret_ref: null,
|
||||
totp_secret_ref: null,
|
||||
user_info: jsonb(info),
|
||||
ua_pending: null,
|
||||
})
|
||||
.where("id", "=", acc.id)
|
||||
.execute();
|
||||
@@ -541,10 +560,16 @@ async function storeSession(rt: SourcesRuntime, acc: SiteAccountRow, cookie: str
|
||||
acc.secret_ref = ref;
|
||||
acc.session_secret_ref = null;
|
||||
acc.totp_secret_ref = null;
|
||||
acc.ua_pending = null;
|
||||
return;
|
||||
}
|
||||
if (acc.session_secret_ref) {
|
||||
await secrets.replace(acc.session_secret_ref, cookie);
|
||||
const ref = acc.session_secret_ref;
|
||||
await db.transaction().execute(async (trx) => {
|
||||
await secrets.replace(ref, cookie, trx);
|
||||
await trx.updateTable("site_account").set({ ua_pending: null }).where("id", "=", acc.id).execute();
|
||||
});
|
||||
acc.ua_pending = null;
|
||||
return;
|
||||
}
|
||||
const ref = await secrets.create({
|
||||
@@ -552,8 +577,13 @@ async function storeSession(rt: SourcesRuntime, acc: SiteAccountRow, cookie: str
|
||||
plaintext: cookie,
|
||||
label: secretName(acc, ":session"),
|
||||
});
|
||||
await db.updateTable("site_account").set({ session_secret_ref: ref }).where("id", "=", acc.id).execute();
|
||||
await db
|
||||
.updateTable("site_account")
|
||||
.set({ session_secret_ref: ref, ua_pending: null })
|
||||
.where("id", "=", acc.id)
|
||||
.execute();
|
||||
acc.session_secret_ref = ref;
|
||||
acc.ua_pending = null;
|
||||
}
|
||||
|
||||
async function markSuccess(
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { mergeCookieHeaders, parseCookieHeader, sameCookieHeader } from "@umt/adapters";
|
||||
import { mergeCookieHeaders, parseCookieHeader, sameCookieHeader, splitBrowserOnlyCookies } from "@umt/adapters";
|
||||
import type { SiteCookieSource } from "@umt/contracts";
|
||||
import type { SiteAccountRow } from "@umt/db";
|
||||
import type { ServerContext } from "../../types.ts";
|
||||
@@ -81,52 +81,81 @@ export async function readAccountCookie(ctx: Ctx, accountId: string): Promise<Ac
|
||||
/**
|
||||
* 浏览器回写:与当前 Cookie 比较(忽略顺序),有变化才写。
|
||||
* Cookie 账号替换 secret_ref;账密账号替换(或新建)会话 Cookie。失效状态的账号回到 unknown(下次抓取 / 健康检测再验证)。
|
||||
* BR-4:
|
||||
* - Cloudflare 验证 Cookie(cf_clearance、__cf_bm)只留在浏览器 profile,不写给站点请求,只在 user_info.browserOnlyCookies 记名字;
|
||||
* - 浏览器用自己的 UA 取得 / 用着这份登录态:写 Cookie 的同一事务里清掉"待切换",站点请求从下一条起 Cookie + UA 一起换。
|
||||
* 返回是否写入。调用方负责只传该站域名的 Cookie,并确认浏览器当前不在登录页。
|
||||
*/
|
||||
export async function writeBrowserCookie(
|
||||
ctx: Ctx,
|
||||
accountId: string,
|
||||
header: string,
|
||||
rawHeader: string,
|
||||
now: Date = new Date(),
|
||||
): Promise<{ changed: boolean; count: number }> {
|
||||
): Promise<{ changed: boolean; count: number; uaSwitched: boolean; browserOnly: string[] }> {
|
||||
const { header, browserOnly } = splitBrowserOnlyCookies(rawHeader);
|
||||
const count = parseCookieHeader(header).size;
|
||||
if (!count) return { changed: false, count: 0 };
|
||||
const none = { changed: false, count, uaSwitched: false, browserOnly };
|
||||
if (!header || !count) return none;
|
||||
const acc = await ctx.db.selectFrom("site_account").selectAll().where("id", "=", accountId).executeTakeFirst();
|
||||
if (!acc || (acc.auth_kind !== "cookie" && acc.auth_kind !== "password")) return { changed: false, count };
|
||||
if (!acc || (acc.auth_kind !== "cookie" && acc.auth_kind !== "password")) return none;
|
||||
const cur = await readAccountCookie(ctx, accountId);
|
||||
if (cur && sameCookieHeader(cur.header, header)) return { changed: false, count };
|
||||
let sessionRef = acc.session_secret_ref;
|
||||
if (acc.auth_kind === "cookie") {
|
||||
if (!acc.secret_ref) return { changed: false, count };
|
||||
await ctx.secrets.replace(acc.secret_ref, header);
|
||||
// 旧会话 Cookie 会在合并时覆盖同名项,回写后以浏览器为准
|
||||
if (sessionRef) await ctx.secrets.replace(sessionRef, header);
|
||||
} else if (sessionRef) {
|
||||
await ctx.secrets.replace(sessionRef, header);
|
||||
} else {
|
||||
sessionRef = await ctx.secrets.create({
|
||||
purpose: "site_cookie",
|
||||
plaintext: header,
|
||||
label: `site_account:${acc.site_id}:${acc.label}:session:${crypto.randomUUID().slice(0, 8)}`,
|
||||
});
|
||||
const info = obj(acc.user_info);
|
||||
const prevOnly = Array.isArray(info.browserOnlyCookies) ? (info.browserOnlyCookies as string[]) : [];
|
||||
const onlyChanged = browserOnly.length > 0 && browserOnly.join(",") !== prevOnly.join(",");
|
||||
if (cur && sameCookieHeader(cur.header, header)) {
|
||||
// Cookie 没变:浏览器正用这份登录态(不在登录页),待切换的 UA 也可以换了
|
||||
if (!acc.ua_pending && !onlyChanged) return none;
|
||||
await ctx.db
|
||||
.updateTable("site_account")
|
||||
.set({
|
||||
...(acc.ua_pending ? { ua_pending: null } : {}),
|
||||
...(onlyChanged ? { user_info: JSON.stringify({ ...info, browserOnlyCookies: browserOnly }) } : {}),
|
||||
updated_at: now,
|
||||
})
|
||||
.where("id", "=", acc.id)
|
||||
.execute();
|
||||
if (acc.ua_pending)
|
||||
ctx.logger.info({ component: "credentials", accountId }, "站点账号 UA 已切换为跟随浏览器(Cookie 未变)");
|
||||
return { ...none, uaSwitched: !!acc.ua_pending };
|
||||
}
|
||||
if (acc.auth_kind === "cookie" && !acc.secret_ref) return none;
|
||||
// 新建会话 secret 在事务外(新行没被引用,不影响读);替换已有 secret 与账号行更新在同一事务里
|
||||
const newSession =
|
||||
acc.auth_kind === "password" && !acc.session_secret_ref
|
||||
? await ctx.secrets.create({
|
||||
purpose: "site_cookie",
|
||||
plaintext: header,
|
||||
label: `site_account:${acc.site_id}:${acc.label}:session:${crypto.randomUUID().slice(0, 8)}`,
|
||||
})
|
||||
: null;
|
||||
const sessionRef = newSession ?? acc.session_secret_ref;
|
||||
const origin: CookieOrigin = { source: "browser", by: "browser", at: now.toISOString() };
|
||||
const stale = acc.status === "expired" || acc.status === "invalid";
|
||||
await ctx.db
|
||||
.updateTable("site_account")
|
||||
.set({
|
||||
user_info: JSON.stringify({ ...obj(acc.user_info), cookieOrigin: origin }),
|
||||
session_secret_ref: sessionRef,
|
||||
...(stale ? { status: "unknown" as const, last_error: null } : {}),
|
||||
updated_at: now,
|
||||
})
|
||||
.where("id", "=", acc.id)
|
||||
.execute();
|
||||
await ctx.db.transaction().execute(async (trx) => {
|
||||
if (acc.auth_kind === "cookie") {
|
||||
await ctx.secrets.replace(acc.secret_ref!, header, trx);
|
||||
// 旧会话 Cookie 会在合并时覆盖同名项,回写后以浏览器为准
|
||||
if (sessionRef) await ctx.secrets.replace(sessionRef, header, trx);
|
||||
} else if (!newSession && sessionRef) {
|
||||
await ctx.secrets.replace(sessionRef, header, trx);
|
||||
}
|
||||
await trx
|
||||
.updateTable("site_account")
|
||||
.set({
|
||||
user_info: JSON.stringify({ ...info, cookieOrigin: origin, browserOnlyCookies: browserOnly }),
|
||||
session_secret_ref: sessionRef,
|
||||
ua_pending: null,
|
||||
...(stale ? { status: "unknown" as const, last_error: null } : {}),
|
||||
updated_at: now,
|
||||
})
|
||||
.where("id", "=", acc.id)
|
||||
.execute();
|
||||
});
|
||||
ctx.logger.info(
|
||||
{ component: "credentials", accountId, count, updatedBy: "browser" },
|
||||
{ component: "credentials", accountId, count, browserOnly: browserOnly.length, updatedBy: "browser" },
|
||||
"站点账号 Cookie 已从浏览器回写",
|
||||
);
|
||||
return { changed: true, count };
|
||||
return { changed: true, count, uaSwitched: !!acc.ua_pending, browserOnly };
|
||||
}
|
||||
|
||||
// ---------- 浏览器读回钩子(浏览器模块注册;sources-identity 不直接依赖浏览器模块) ----------
|
||||
@@ -148,7 +177,8 @@ export async function cookieFromBrowser(
|
||||
): Promise<string | null> {
|
||||
const reader = readers.get(ctx);
|
||||
if (!reader) return null;
|
||||
const header = await reader(acc.site_id, acc.id).catch(() => null);
|
||||
// 仅浏览器用的 Cloudflare Cookie 不给站点请求(BR-4)
|
||||
const header = splitBrowserOnlyCookies(await reader(acc.site_id, acc.id).catch(() => null)).header;
|
||||
if (!header || sameCookieHeader(header, current)) return null;
|
||||
return header;
|
||||
}
|
||||
@@ -159,7 +159,12 @@ export async function ingestFeed(
|
||||
if (!feed.url_secret_ref) return { status: "skipped", reason: "no_url", ...empty };
|
||||
|
||||
const url = await secrets.resolve(feed.url_secret_ref);
|
||||
const result = await rt.rss.fetchFeed(url, stateFromRow(feed), { ignoreBackoff: !!opts.manual });
|
||||
// BR-4:挂站点的 Feed 与站点其它请求同一 UA、同一出口(站点开代理时走代理);通用 Feed 用全局 UA / 跟随浏览器
|
||||
const result = await rt.rss.fetchFeed(url, stateFromRow(feed), {
|
||||
ignoreBackoff: !!opts.manual,
|
||||
userAgent: site ? await rt.siteUserAgent(site.id) : await rt.defaultUserAgent(),
|
||||
fetch: rt.siteFetch(site ?? { proxy: false }),
|
||||
});
|
||||
// 挂站点的 Feed:每次真正发出的 RSS 请求也记进站点请求计数(熔断 / 退避跳过的不算)
|
||||
if (site && result.kind !== "skipped") rt.countSiteRequest(site.id);
|
||||
if (site) await renormalizeFeedReleases(rt, site).catch(() => {});
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
import { FALLBACK_SITE_USER_AGENT } from "@umt/contracts";
|
||||
import type { SiteAccountRow } from "@umt/db";
|
||||
import type { ServerContext } from "../../types.ts";
|
||||
|
||||
// BR-4 站点请求 UA:账号手动 > 待切换(沿用取得当前 Cookie 时的旧 UA)> 设置里的全局 UA > 跟随浏览器(camofox 固定身份的 UA,
|
||||
// 读不到时用同型的 Windows Firefox 常量)。站点适配器、RSS、登录、用户数据、种子下载都经 SourcesRuntime.siteFetch 带上它。
|
||||
|
||||
export type UserAgentSource = "browser" | "manual" | "global" | "pending";
|
||||
|
||||
export interface UserAgentDefaults {
|
||||
/** settings.sites.userAgent;空 = 跟随浏览器 */
|
||||
global: string;
|
||||
/** 浏览器身份 UA(已退回常量) */
|
||||
browser: string;
|
||||
}
|
||||
|
||||
type UaFields = Pick<SiteAccountRow, "user_agent" | "ua_pending">;
|
||||
|
||||
/**
|
||||
* 生效 UA。ignorePending = 按"切换之后"算(重新登录 / 浏览器 Cookie 回写时用新 UA,成功后清掉待切换)。
|
||||
*/
|
||||
export function resolveUserAgent(
|
||||
acc: UaFields | null | undefined,
|
||||
d: UserAgentDefaults,
|
||||
opts: { ignorePending?: boolean } = {},
|
||||
): { ua: string; source: UserAgentSource } {
|
||||
const manual = acc?.user_agent?.trim();
|
||||
if (manual) return { ua: manual, source: "manual" };
|
||||
const pending = acc?.ua_pending?.trim();
|
||||
if (pending && !opts.ignorePending) return { ua: pending, source: "pending" };
|
||||
const global = d.global.trim();
|
||||
if (global) return { ua: global, source: "global" };
|
||||
return { ua: d.browser || FALLBACK_SITE_USER_AGENT, source: "browser" };
|
||||
}
|
||||
|
||||
/** user_info.browserOnlyCookies:最近一次浏览器回写时剔除的仅浏览器 Cookie 名 */
|
||||
export function browserOnlyCookiesOf(userInfo: unknown): string[] {
|
||||
const v = (userInfo as { browserOnlyCookies?: unknown } | null)?.browserOnlyCookies;
|
||||
return Array.isArray(v) ? v.filter((x): x is string => typeof x === "string") : [];
|
||||
}
|
||||
|
||||
// ---------- 浏览器身份读取钩子(浏览器模块注册;sources-identity 不直接依赖浏览器模块) ----------
|
||||
|
||||
/** 返回浏览器固定身份的 UA;没有浏览器服务 / 插件时 null */
|
||||
export type BrowserIdentityReader = () => Promise<string | null>;
|
||||
|
||||
const readers = new WeakMap<object, BrowserIdentityReader>();
|
||||
|
||||
export function registerBrowserIdentityReader(ctx: object, reader: BrowserIdentityReader): void {
|
||||
readers.set(ctx, reader);
|
||||
}
|
||||
|
||||
/**
|
||||
* 浏览器身份 UA:浏览器模块在本进程装配了就问它(它读库里缓存的身份,过期再问浏览器服务);
|
||||
* 没装配时只读 browser_provider.identity(其它进程写入的)。
|
||||
*/
|
||||
export async function readBrowserUserAgent(ctx: Pick<ServerContext, "db">): Promise<string | null> {
|
||||
const reader = readers.get(ctx);
|
||||
if (reader) return reader();
|
||||
const row = await ctx.db
|
||||
.selectFrom("browser_provider")
|
||||
.select("identity")
|
||||
.where("enabled", "=", true)
|
||||
.where("kind", "<>", "none")
|
||||
.orderBy("created_at")
|
||||
.executeTakeFirst();
|
||||
const ua = (row?.identity as { userAgent?: unknown } | null | undefined)?.userAgent;
|
||||
return typeof ua === "string" && ua ? ua : null;
|
||||
}
|
||||
@@ -47,7 +47,8 @@ export interface OutboxPort extends OutboxWriter {
|
||||
export interface SecretStore {
|
||||
resolve(secretRef: string): Promise<string>;
|
||||
create(input: { purpose: SecretPurpose; plaintext: string; label?: string }): Promise<string>;
|
||||
replace(secretRef: string, plaintext: string): Promise<void>;
|
||||
/** exec:在调用方事务里写(BR-4 Cookie 与 UA 原子切换);缺省用自己的连接 */
|
||||
replace(secretRef: string, plaintext: string, exec?: Kysely<DB>): Promise<void>;
|
||||
remove(secretRef: string): Promise<void>;
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user