feat(sites): site requests follow browser UA, smooth UA switch, CF cookies browser-only (BR-4)
- every outbound site request (adapter list/search/detail/download/health, RSS, login, user data) carries the account UA via SourcesRuntime.siteFetch; no more Deno default UA - priority: account manual > pending > global setting > browser identity (fallback: same-shape Windows Firefox UA); settings UA default is empty - migration 0056: existing cookie/password accounts keep their old UA as ua_pending until re-login or browser cookie write-back switches cookie+UA in one transaction; browser_provider.identity caches the browser identity - cf_clearance/__cf_bm stripped on browser write-back (names kept only) - browser sessions get per-site egress (same proxy as HTTP, or direct) - site account page shows UA source; raw UA in tech info Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
ebd07c9230
commit
adc853aa64
21 files changed
+955
-72
No files matched your search
@@ -0,0 +1,100 @@
|
||||
import { createMigrator, type DB, migrateToLatest, sql } from "@umt/db";
|
||||
import { Kysely, PostgresDialect } from "kysely";
|
||||
import pg from "pg";
|
||||
import { afterAll, describe, expect, it } from "vitest";
|
||||
import { PG_URL, pgAvailable, withSchemaLock } from "../auth-settings/harness.ts";
|
||||
|
||||
// 0056:存量 Cookie / 账密账号标"待切换"(沿用旧缺省 UA);手动 UA 与 API Key 账号不动;设置里的旧缺省 UA 清掉 = 跟随浏览器。
|
||||
|
||||
const hasPg = await pgAvailable();
|
||||
const LEGACY =
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36";
|
||||
|
||||
describe.skipIf(!hasPg)("0056 浏览器身份迁移", () => {
|
||||
const schemas: string[] = [];
|
||||
const dbs: Kysely<DB>[] = [];
|
||||
const admin = async (q: string) => {
|
||||
const c = new pg.Client({ connectionString: PG_URL });
|
||||
await c.connect();
|
||||
await c.query(q).finally(() => c.end());
|
||||
};
|
||||
afterAll(async () => {
|
||||
for (const d of dbs) await d.destroy();
|
||||
await withSchemaLock(async () => {
|
||||
for (const s of schemas) await admin(`drop schema if exists ${s} cascade`);
|
||||
});
|
||||
});
|
||||
|
||||
const setup = async (settingsUa: string | null) => {
|
||||
const schema = `br4_mig_${process.pid.toString(36)}${Math.random().toString(36).slice(2, 6)}`;
|
||||
schemas.push(schema);
|
||||
await admin(`create schema ${schema}`);
|
||||
const db = new Kysely<DB>({
|
||||
dialect: new PostgresDialect({
|
||||
pool: new pg.Pool({ connectionString: PG_URL, max: 2, options: `-c search_path=${schema},public` }),
|
||||
}),
|
||||
});
|
||||
dbs.push(db);
|
||||
await withSchemaLock(async () => {
|
||||
const r = await createMigrator(db, { migrationTableSchema: schema }).migrateTo("0055_site_kind");
|
||||
if (r.error) throw r.error;
|
||||
});
|
||||
await sql`insert into site (key, name, adapter, base_url) values ('audiences', '观众', 'nexusphp', 'https://a.example/')`.execute(
|
||||
db,
|
||||
);
|
||||
await sql`insert into secret (id, name, purpose, ciphertext, nonce) values
|
||||
('00000000-0000-4000-8000-000000000001', 's1', 'site_cookie', 'x', 'x'),
|
||||
('00000000-0000-4000-8000-000000000002', 's2', 'site_password', 'x', 'x'),
|
||||
('00000000-0000-4000-8000-000000000003', 's3', 'site_api_key', 'x', 'x'),
|
||||
('00000000-0000-4000-8000-000000000004', 's4', 'site_cookie', 'x', 'x')`.execute(db);
|
||||
await sql`insert into site_account (site_id, label, auth_kind, secret_ref, user_agent) values
|
||||
((select id from site), 'cookie', 'cookie', '00000000-0000-4000-8000-000000000001', null),
|
||||
((select id from site), 'password', 'password', '00000000-0000-4000-8000-000000000002', null),
|
||||
((select id from site), 'apikey', 'api_key', '00000000-0000-4000-8000-000000000003', null),
|
||||
((select id from site), 'manual', 'cookie', '00000000-0000-4000-8000-000000000004', 'My-UA/1'),
|
||||
((select id from site), 'empty', 'cookie', null, null)`.execute(db);
|
||||
if (settingsUa !== null)
|
||||
await sql`insert into settings (key, value) values ('sites', ${JSON.stringify({ userAgent: settingsUa, proxyUrl: null })}::jsonb)`.execute(
|
||||
db,
|
||||
);
|
||||
await withSchemaLock(() => migrateToLatest(db, { migrationTableSchema: schema }));
|
||||
const accounts = await sql<{ label: string; ua_pending: string | null }>`
|
||||
select label, ua_pending from site_account order by label`.execute(db);
|
||||
const settings = await sql<{
|
||||
value: Record<string, unknown>;
|
||||
}>`select value from settings where key = 'sites'`.execute(db);
|
||||
return { db, accounts: Object.fromEntries(accounts.rows.map((r) => [r.label, r.ua_pending])), settings };
|
||||
};
|
||||
|
||||
it("设置仍是旧缺省:Cookie / 账密账号待切换到旧 UA,设置里的 UA 清掉(跟随浏览器)", async () => {
|
||||
const r = await setup(LEGACY);
|
||||
expect(r.accounts).toEqual({ apikey: null, cookie: LEGACY, empty: null, manual: null, password: LEGACY });
|
||||
expect(r.settings.rows[0]!.value).not.toHaveProperty("userAgent");
|
||||
expect(r.settings.rows[0]!.value).toHaveProperty("proxyUrl", null);
|
||||
const cols = await sql<{ column_name: string }>`select column_name from information_schema.columns
|
||||
where table_schema = current_schema() and table_name = 'browser_provider' and column_name = 'identity'`.execute(
|
||||
r.db,
|
||||
);
|
||||
expect(cols.rows).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("用户改过全局 UA:保留为全局 UA,存量账号待切换到它;没有设置行时按旧缺省", async () => {
|
||||
const r = await setup("Custom-UA/9");
|
||||
expect(r.accounts.cookie).toBe("Custom-UA/9");
|
||||
expect(r.settings.rows[0]!.value).toHaveProperty("userAgent", "Custom-UA/9");
|
||||
const r2 = await setup(null);
|
||||
expect(r2.accounts.password).toBe(LEGACY);
|
||||
expect(r2.settings.rows).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("回滚只删列", async () => {
|
||||
const r = await setup(LEGACY);
|
||||
await withSchemaLock(async () => {
|
||||
const m = await createMigrator(r.db, { migrationTableSchema: schemas.at(-1)! }).migrateTo("0055_site_kind");
|
||||
if (m.error) throw m.error;
|
||||
});
|
||||
const cols = await sql<{ column_name: string }>`select column_name from information_schema.columns
|
||||
where table_schema = current_schema() and column_name in ('ua_pending', 'identity')`.execute(r.db);
|
||||
expect(cols.rows).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,273 @@
|
||||
import { FALLBACK_SITE_USER_AGENT, LEGACY_SITE_USER_AGENT } from "@umt/contracts";
|
||||
import { createMockNexusPhpSite, createMockRssServer, type MockNexusPhpSite, type MockRssServer } from "@umt/testkit";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import {
|
||||
readAccountCookie,
|
||||
registerBrowserCookieReader,
|
||||
writeBrowserCookie,
|
||||
} from "../../src/modules/sources-identity/service/credentials.ts";
|
||||
import { ingestFeed } from "../../src/modules/sources-identity/service/feeds.ts";
|
||||
import {
|
||||
registerBrowserIdentityReader,
|
||||
resolveUserAgent,
|
||||
} from "../../src/modules/sources-identity/service/user-agent.ts";
|
||||
import { createHarness, type Harness, pgAvailable, seedFeed } from "./harness.ts";
|
||||
|
||||
// BR-4 站点请求 UA:每条出站请求都带账号 UA(不再是 Deno 缺省 UA)、优先级(手动 > 待切换 > 全局 > 跟随浏览器)、
|
||||
// 平滑迁移(待切换账号沿用旧 UA,重新登录 / 浏览器回写 Cookie 时 Cookie + UA 一起切换)、Cloudflare Cookie 仅浏览器。
|
||||
|
||||
const up = await pgAvailable();
|
||||
const BROWSER_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:152.0) Gecko/20100101 Firefox/152.0";
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 响应体断言
|
||||
const json = async (r: Response): Promise<any> => {
|
||||
const body = await r.json();
|
||||
if (!r.ok) throw new Error(`${r.status} ${JSON.stringify(body)}`);
|
||||
return body.data;
|
||||
};
|
||||
|
||||
type RecordedRequest = MockNexusPhpSite["requests"][number];
|
||||
const uaOf = (r: RecordedRequest) => String(r.headers["user-agent"] ?? "");
|
||||
|
||||
describe("resolveUserAgent 优先级", () => {
|
||||
const d = { global: "", browser: "B" };
|
||||
it("手动 > 待切换 > 全局 > 跟随浏览器 > 常量", () => {
|
||||
expect(resolveUserAgent({ user_agent: "M", ua_pending: "P" }, { global: "G", browser: "B" })).toEqual({
|
||||
ua: "M",
|
||||
source: "manual",
|
||||
});
|
||||
expect(resolveUserAgent({ user_agent: null, ua_pending: "P" }, { global: "G", browser: "B" })).toEqual({
|
||||
ua: "P",
|
||||
source: "pending",
|
||||
});
|
||||
expect(
|
||||
resolveUserAgent({ user_agent: null, ua_pending: "P" }, { global: "G", browser: "B" }, { ignorePending: true }),
|
||||
).toEqual({ ua: "G", source: "global" });
|
||||
expect(resolveUserAgent({ user_agent: " ", ua_pending: null }, d)).toEqual({ ua: "B", source: "browser" });
|
||||
expect(resolveUserAgent(null, { global: "", browser: "" })).toEqual({
|
||||
ua: FALLBACK_SITE_USER_AGENT,
|
||||
source: "browser",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!up)("BR-4 站点请求 UA 与 Cookie 切换", () => {
|
||||
let h: Harness;
|
||||
let mock: MockNexusPhpSite;
|
||||
let rss: MockRssServer;
|
||||
let siteId: string;
|
||||
let accountId: string;
|
||||
let browserUa: string | null = BROWSER_UA;
|
||||
let browserCookie: string | null = null;
|
||||
|
||||
const account = () =>
|
||||
h.db.selectFrom("site_account").selectAll().where("id", "=", accountId).executeTakeFirstOrThrow();
|
||||
const site = () => h.db.selectFrom("site").selectAll().where("id", "=", siteId).executeTakeFirstOrThrow();
|
||||
const apiAccount = async () => json(await h.req(`/site-accounts/${accountId}`));
|
||||
const refresh = async () =>
|
||||
(await json(await h.req(`/sites/${siteId}/refresh-login`, { body: { wait: true, accountId } }))).result;
|
||||
const since = (n: number) => mock.requests.slice(n);
|
||||
|
||||
beforeAll(async () => {
|
||||
mock = await createMockNexusPhpSite({ username: "alice", password: "s3cret-pw" });
|
||||
rss = await createMockRssServer({
|
||||
passkey: "PK",
|
||||
items: [
|
||||
{
|
||||
guid: "br4-1",
|
||||
title: "Some.Movie.2024.1080p.BluRay-GRP",
|
||||
link: `${mock.baseUrl}/details.php?id=900001`,
|
||||
enclosureUrl: `${mock.baseUrl}/download.php?id=900001&passkey=PK`,
|
||||
size: 1024 ** 3,
|
||||
pubDate: new Date("2026-09-20T08:00:00Z"),
|
||||
categories: ["电影"],
|
||||
},
|
||||
],
|
||||
});
|
||||
h = await createHarness("br4ua");
|
||||
registerBrowserIdentityReader(h.ctx, async () => browserUa);
|
||||
registerBrowserCookieReader(h.ctx, async () => browserCookie);
|
||||
const s = await json(
|
||||
await h.req("/sites", {
|
||||
body: {
|
||||
key: "audiences",
|
||||
name: "观众",
|
||||
adapter: "nexusphp",
|
||||
baseUrl: `${mock.baseUrl}/`,
|
||||
rateLimit: { minIntervalMs: 0, concurrency: 10 },
|
||||
},
|
||||
}),
|
||||
);
|
||||
siteId = s.id;
|
||||
const r = await json(
|
||||
await h.req(`/sites/${siteId}/credentials`, {
|
||||
method: "PUT",
|
||||
body: { kind: "password", username: "alice", password: "s3cret-pw" },
|
||||
}),
|
||||
);
|
||||
accountId = r.siteAccountId;
|
||||
});
|
||||
afterAll(async () => {
|
||||
await h?.close();
|
||||
await mock?.close();
|
||||
await rss?.close();
|
||||
});
|
||||
|
||||
it("登录、用户数据、列表 / 搜索 / 详情 / 下载 / 健康检测、RSS:每条请求都带跟随浏览器的 UA", async () => {
|
||||
const n0 = mock.requests.length;
|
||||
expect(await refresh()).toMatchObject({ ok: true, status: "active" });
|
||||
const adapter = h.rt.siteAdapter(await site());
|
||||
const page = await adapter.list(0);
|
||||
const id = page.items[0]!.siteReleaseId;
|
||||
await adapter.search("Movie");
|
||||
await adapter.detail(id);
|
||||
const target = await adapter.downloadTarget(id);
|
||||
expect(target.kind).toBe("torrent");
|
||||
expect((await adapter.healthCheck()).ok).toBe(true);
|
||||
const f = await seedFeed(h, { siteId, url: rss.feedUrl, name: "br4", accountId });
|
||||
expect((await ingestFeed(h.rt, f.id)).status).toBe("ok");
|
||||
|
||||
const reqs = [...since(n0), ...rss.requests];
|
||||
const paths = new Set(since(n0).map((r) => r.path.split("?")[0]));
|
||||
for (const p of ["/takelogin.php", "/torrents.php", "/details.php", "/download.php"]) expect(paths).toContain(p);
|
||||
expect(rss.requests.length).toBeGreaterThan(0);
|
||||
for (const r of reqs) {
|
||||
expect(uaOf(r), `${r.method} ${r.path}`).toBe(BROWSER_UA);
|
||||
expect(uaOf(r)).not.toMatch(/Deno/i);
|
||||
}
|
||||
expect(await apiAccount()).toMatchObject({
|
||||
userAgentSource: "browser",
|
||||
effectiveUserAgent: BROWSER_UA,
|
||||
userAgent: null,
|
||||
});
|
||||
});
|
||||
|
||||
it("浏览器身份读不到:退回同型 Windows Firefox 常量;设置里的全局 UA 优先于跟随浏览器,账号手动再优先", async () => {
|
||||
browserUa = null;
|
||||
// runtime 缓存浏览器 UA 60 秒:换一个 runtime 读
|
||||
(h.rt as unknown as { browserUa: { at: number } }).browserUa.at = 0;
|
||||
let n = mock.requests.length;
|
||||
await h.rt.siteAdapter(await site()).healthCheck();
|
||||
expect(since(n).map(uaOf)).toEqual(since(n).map(() => FALLBACK_SITE_USER_AGENT));
|
||||
|
||||
await h.db
|
||||
.insertInto("settings")
|
||||
.values({ key: "sites", value: JSON.stringify({ userAgent: "Global-UA/1.0" }) })
|
||||
.execute();
|
||||
n = mock.requests.length;
|
||||
await h.rt.siteDefaults();
|
||||
await h.rt.siteAdapter(await site()).healthCheck();
|
||||
expect(new Set(since(n).map(uaOf))).toEqual(new Set(["Global-UA/1.0"]));
|
||||
expect(await apiAccount()).toMatchObject({ userAgentSource: "global", effectiveUserAgent: "Global-UA/1.0" });
|
||||
|
||||
await h.db.updateTable("site_account").set({ user_agent: "Manual-UA/2.0" }).where("id", "=", accountId).execute();
|
||||
n = mock.requests.length;
|
||||
await h.rt.siteAdapter(await site()).healthCheck();
|
||||
expect(new Set(since(n).map(uaOf))).toEqual(new Set(["Manual-UA/2.0"]));
|
||||
expect(await apiAccount()).toMatchObject({ userAgentSource: "manual", userAgent: "Manual-UA/2.0" });
|
||||
|
||||
await h.db.updateTable("site_account").set({ user_agent: null }).where("id", "=", accountId).execute();
|
||||
await h.db.deleteFrom("settings").where("key", "=", "sites").execute();
|
||||
await h.rt.siteDefaults();
|
||||
browserUa = BROWSER_UA;
|
||||
(h.rt as unknown as { browserUa: { at: number } }).browserUa.at = 0;
|
||||
});
|
||||
|
||||
it("待切换:沿用旧 UA;重新登录用浏览器 UA,会话 Cookie 与 UA 一起切换", async () => {
|
||||
await h.db
|
||||
.updateTable("site_account")
|
||||
.set({ ua_pending: LEGACY_SITE_USER_AGENT })
|
||||
.where("id", "=", accountId)
|
||||
.execute();
|
||||
expect(await apiAccount()).toMatchObject({
|
||||
userAgentSource: "pending",
|
||||
effectiveUserAgent: LEGACY_SITE_USER_AGENT,
|
||||
});
|
||||
let n = mock.requests.length;
|
||||
await h.rt.siteAdapter(await site()).healthCheck();
|
||||
expect(new Set(since(n).map(uaOf))).toEqual(new Set([LEGACY_SITE_USER_AGENT]));
|
||||
|
||||
n = mock.requests.length;
|
||||
expect(await refresh()).toMatchObject({ ok: true });
|
||||
const login = since(n).find((r) => r.path.startsWith("/takelogin.php"));
|
||||
expect(login && uaOf(login)).toBe(BROWSER_UA);
|
||||
expect((await account()).ua_pending).toBeNull();
|
||||
n = mock.requests.length;
|
||||
await h.rt.siteAdapter(await site()).healthCheck();
|
||||
expect(new Set(since(n).map(uaOf))).toEqual(new Set([BROWSER_UA]));
|
||||
expect(await apiAccount()).toMatchObject({ userAgentSource: "browser" });
|
||||
});
|
||||
|
||||
it("浏览器回写 Cookie:原子切换 UA;cf_clearance / __cf_bm 只留在浏览器,站点请求不用", async () => {
|
||||
const r = await json(
|
||||
await h.req(`/sites/${siteId}/credentials`, {
|
||||
method: "PUT",
|
||||
body: { kind: "cookie", label: "c", cookie: "c_secure_uid=1; c_secure_pass=old" },
|
||||
}),
|
||||
);
|
||||
const cid = r.siteAccountId as string;
|
||||
await h.db.updateTable("site_account").set({ ua_pending: LEGACY_SITE_USER_AGENT }).where("id", "=", cid).execute();
|
||||
|
||||
const w = await writeBrowserCookie(
|
||||
h.ctx,
|
||||
cid,
|
||||
"c_secure_uid=1; c_secure_pass=from-browser; cf_clearance=CFSECRET; __cf_bm=BMSECRET",
|
||||
);
|
||||
expect(w).toMatchObject({ changed: true, count: 2, uaSwitched: true, browserOnly: ["__cf_bm", "cf_clearance"] });
|
||||
const acc = await readAccountCookie(h.ctx, cid);
|
||||
expect(acc!.header).toBe("c_secure_pass=from-browser; c_secure_uid=1");
|
||||
expect(acc!.header).not.toContain("CFSECRET");
|
||||
const row = await h.db.selectFrom("site_account").selectAll().where("id", "=", cid).executeTakeFirstOrThrow();
|
||||
expect(row.ua_pending).toBeNull();
|
||||
expect(row.user_info).toMatchObject({ browserOnlyCookies: ["__cf_bm", "cf_clearance"] });
|
||||
const view = await json(await h.req(`/site-accounts/${cid}`));
|
||||
expect(view).toMatchObject({ userAgentSource: "browser", browserOnlyCookies: ["__cf_bm", "cf_clearance"] });
|
||||
expect(JSON.stringify(view)).not.toContain("CFSECRET");
|
||||
|
||||
// 只多了 CF Cookie、其余不变:不算变化
|
||||
expect(
|
||||
await writeBrowserCookie(h.ctx, cid, "c_secure_uid=1; c_secure_pass=from-browser; cf_clearance=OTHER; __cf_bm=X"),
|
||||
).toMatchObject({ changed: false });
|
||||
// Cookie 不变但仍待切换:只切 UA
|
||||
await h.db.updateTable("site_account").set({ ua_pending: LEGACY_SITE_USER_AGENT }).where("id", "=", cid).execute();
|
||||
expect(await writeBrowserCookie(h.ctx, cid, "c_secure_uid=1; c_secure_pass=from-browser")).toMatchObject({
|
||||
changed: false,
|
||||
uaSwitched: true,
|
||||
});
|
||||
expect(
|
||||
(await h.db.selectFrom("site_account").select("ua_pending").where("id", "=", cid).executeTakeFirstOrThrow())
|
||||
.ua_pending,
|
||||
).toBeNull();
|
||||
await h.req(`/site-accounts/${cid}`, { method: "DELETE" });
|
||||
});
|
||||
|
||||
it("refresh-login 从浏览器读回的 Cookie 也剔除 CF Cookie;换主凭证结束待切换", async () => {
|
||||
await h.db
|
||||
.updateTable("site_account")
|
||||
.set({ ua_pending: LEGACY_SITE_USER_AGENT })
|
||||
.where("id", "=", accountId)
|
||||
.execute();
|
||||
await json(
|
||||
await h.req(`/sites/${siteId}/credentials`, {
|
||||
method: "PUT",
|
||||
body: { kind: "cookie", cookie: "c_secure_uid=1; c_secure_pass=stale" },
|
||||
}),
|
||||
);
|
||||
expect((await account()).ua_pending).toBeNull();
|
||||
await h.db
|
||||
.updateTable("site_account")
|
||||
.set({ ua_pending: LEGACY_SITE_USER_AGENT })
|
||||
.where("id", "=", accountId)
|
||||
.execute();
|
||||
browserCookie = `${mock.cookie}; cf_clearance=CFSECRET`;
|
||||
const n = mock.requests.length;
|
||||
expect(await refresh()).toMatchObject({ ok: true, status: "active" });
|
||||
const acc = await readAccountCookie(h.ctx, accountId);
|
||||
expect(acc!.header).not.toContain("cf_clearance");
|
||||
expect((await account()).ua_pending).toBeNull();
|
||||
// 验证浏览器 Cookie 的请求用浏览器 UA,且没有带 cf_clearance
|
||||
const withBrowserCookie = since(n).filter((r) => String(r.headers.cookie ?? "").includes("mock"));
|
||||
for (const r of since(n)) expect(String(r.headers.cookie ?? "")).not.toContain("CFSECRET");
|
||||
expect(withBrowserCookie.every((r) => uaOf(r) === BROWSER_UA)).toBe(true);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user