From 05a7706ddefad9d4dba2208f4c18cb810a6d37df Mon Sep 17 00:00:00 2001 From: root Date: Mon, 28 Sep 2026 15:18:40 +0800 Subject: [PATCH] feat(sources): site center backend (credentials, refresh login, user data, health check, delete) PUT /sites/{id}/credentials by kind (moved from auth-settings, legacy body accepted), POST refresh-login (async task or wait), POST health-check, GET user-data history, DELETE /sites/{id} (soft delete, releases kept), DELETE /site-accounts/{id}, GET /sites?include=accounts; refreshSiteAccount task with login-failure decisions and auto re-login on auth_expired; scheduled jobs sites.user_data (6h) and sites.health_check (1h); settings.sites defaults for rate limit, snapshot interval, UA and proxy. Co-Authored-By: Claude Opus 5.5 --- .../modules/auth-settings/instances-routes.ts | 99 +- .../src/modules/sources-identity/defs.ts | 14 +- .../src/modules/sources-identity/index.ts | 27 + .../sources-identity/routes/decisions.ts | 10 + .../sources-identity/routes/sources.ts | 205 +++- .../src/modules/sources-identity/runtime.ts | 129 ++- .../src/modules/sources-identity/scheduler.ts | 9 +- .../sources-identity/service/account.ts | 884 ++++++++++++++++++ .../sources-identity/service/snapshot.ts | 3 + .../src/modules/sources-identity/tasks.ts | 18 + .../test/auth-settings/admin-api.test.ts | 45 +- apps/server/test/sources-identity/harness.ts | 6 +- .../test/sources-identity/mteam-tasks.test.ts | 3 +- .../test/sources-identity/site-center.test.ts | 328 +++++++ .../components/workbench/release-fields.ts | 1 + 15 files changed, 1602 insertions(+), 179 deletions(-) create mode 100644 apps/server/src/modules/sources-identity/service/account.ts create mode 100644 apps/server/test/sources-identity/site-center.test.ts diff --git a/apps/server/src/modules/auth-settings/instances-routes.ts b/apps/server/src/modules/auth-settings/instances-routes.ts index 362505d7..aa26a7e4 100644 --- a/apps/server/src/modules/auth-settings/instances-routes.ts +++ b/apps/server/src/modules/auth-settings/instances-routes.ts @@ -18,10 +18,7 @@ import { PathMappingCreate, PathMappingUpdate, type SecretPurpose, - SiteCredentialsPut, - SiteCredentialsResult, } from "@umt/contracts"; -import type { DB } from "@umt/db"; import { z } from "zod"; import { ApiException } from "../../http/errors.ts"; import { redact, redactString } from "../../logger.ts"; @@ -41,7 +38,7 @@ import { } from "./endpoint.ts"; // 实例与路径:library_root / downloader_instance / media_server_instance / path_mapping 的 CRUD + 连通性测试, -// 以及站点账号凭证录入 PUT /sites/{id}/credentials。凭证明文只在请求里出现,存为 secret 后只回 secretRef。 +// 站点账号凭证录入已迁到 sources-identity(PUT /sites/{id}/credentials)。 const OkResult = z.object({ ok: z.literal(true) }); const del = (path: string, summary: string, tag: string) => ({ @@ -470,97 +467,5 @@ export function registerInstanceRoutes(app: ModuleApp, ctx: ServerContext) { await audit(ctx, actor, "path_mapping.verify", { type: "path_mapping", id: row.id }, { ok }); return toApi(updated); }); - - // ── 站点账号凭证 ── - const PURPOSE: Record = { - cookie: "site_cookie", - passkey: "site_passkey", - api_key: "site_api_key", - password: "site_password", - }; - endpoint( - app, - { - method: "put", - path: "/sites/{id}/credentials", - scope: "admin", - summary: "录入站点账号凭证(cookie/passkey/apiKey/密码 → secret_ref,不回显)", - params: IdParam, - body: SiteCredentialsPut, - response: SiteCredentialsResult, - tags: ["sites"], - }, - async ({ params, body, actor }) => { - const site = await db().selectFrom("site").select("id").where("id", "=", params.id).executeTakeFirst(); - if (!site) throw new ApiException("not_found", "站点不存在"); - const newRef = await ctx.secrets.create({ purpose: PURPOSE[body.authKind], plaintext: body.credential }); - let oldRef: string | null = null; - let created = false; - let accountId: string; - try { - accountId = await db() - .transaction() - .execute(async (trx) => { - const cur = await trx - .selectFrom("site_account") - .select(["id", "secret_ref", "user_info"]) - .where("site_id", "=", params.id) - .where("label", "=", body.label) - .forUpdate() - .executeTakeFirst(); - const userInfo = { - ...((cur?.user_info as Record | undefined) ?? {}), - ...(body.username ? { username: body.username } : {}), - }; - const common: Partial> = { - auth_kind: body.authKind, - secret_ref: newRef, - status: "unknown", - user_info: JSON.stringify(userInfo), - ...(body.enabled !== undefined ? { enabled: body.enabled } : {}), - }; - if (cur) { - oldRef = cur.secret_ref; - await trx - .updateTable("site_account") - .set(common as never) - .where("id", "=", cur.id) - .execute(); - return cur.id; - } - created = true; - const r = await trx - .insertInto("site_account") - .values({ site_id: params.id, label: body.label, ...common } as never) - .returning("id") - .executeTakeFirstOrThrow(); - return r.id; - }); - } catch (e) { - await ctx.secrets.remove(newRef).catch(() => {}); - throw e; - } - if (oldRef) await ctx.secrets.remove(oldRef).catch(() => {}); - await audit( - ctx, - actor, - "site_account.credentials_set", - { type: "site_account", id: accountId }, - { - siteId: params.id, - label: body.label, - authKind: body.authKind, - created, - }, - ); - return { - siteAccountId: accountId, - siteId: params.id, - label: body.label, - authKind: body.authKind, - secretRef: newRef, - created, - }; - }, - ); + // 站点账号凭证 PUT /sites/{id}/credentials 已迁到 sources-identity(W9-A 站点中心,按 kind 判别) } diff --git a/apps/server/src/modules/sources-identity/defs.ts b/apps/server/src/modules/sources-identity/defs.ts index f6f8da2c..b8b2816c 100644 --- a/apps/server/src/modules/sources-identity/defs.ts +++ b/apps/server/src/modules/sources-identity/defs.ts @@ -33,6 +33,8 @@ import { SiteAccountCreate, SiteAccountUpdate, SiteCreate, + SiteListItem, + SiteListQuery, SiteUpdate, SourceRelease, TitleAlias, @@ -96,7 +98,15 @@ const TD = ["decisions"]; export const D = { sites: { - list: list("/sites", "站点列表", Site, {}, TS), + list: { + method: "get" as const, + path: "/sites", + scope: "read" as const, + summary: "站点列表(include=accounts 附带账号与整体账号状态)", + tags: TS, + query: SiteListQuery, + response: cursorPage(SiteListItem), + }, get: get("/sites/{id}", "站点详情", Site, TS), create: { method: "post" as const, @@ -107,7 +117,7 @@ export const D = { body: SiteCreate, response: Site, }, - update: write("patch", "/sites/{id}", "admin", "修改站点(启停、快照周期、限流)", SiteUpdate, Site, TS), + update: write("patch", "/sites/{id}", "admin", "修改站点(启停、快照周期、限流、代理)", SiteUpdate, Site, TS), }, siteAccounts: { list: list("/site-accounts", "站点账号列表", SiteAccount, { siteId: Id.optional() }, TS), diff --git a/apps/server/src/modules/sources-identity/index.ts b/apps/server/src/modules/sources-identity/index.ts index fdf08323..312f1b39 100644 --- a/apps/server/src/modules/sources-identity/index.ts +++ b/apps/server/src/modules/sources-identity/index.ts @@ -7,6 +7,7 @@ import { sourcesRoutes } from "./routes/sources.ts"; import { workbenchRoutes } from "./routes/workbench.ts"; import { type RuntimeOverrides, SourcesRuntime } from "./runtime.ts"; import { readOnlyReconcile, tickFeeds, tickReviews, tickSnapshots } from "./scheduler.ts"; +import { tickSiteHealth, tickSiteUserData } from "./service/account.ts"; import { registerSourcesTasks, sourcesOutboxRoutes } from "./tasks.ts"; /** @@ -60,6 +61,32 @@ export function createSourcesIdentityModule(overrides: RuntimeOverrides = {}): S job("feeds.poll", "Feed 轮询", "扫描到期的 Feed 并入队抓取(每个 Feed 仍按自己的轮询间隔)", tickFeeds); job("sites.snapshot", "站点快照", "扫描到周期的站点并入队快照", tickSnapshots); job("identity.review", "待识别复查", "到期的待识别条目重新识别", tickReviews); + // W9-A 站点中心:用户数据每 6 小时、健康检测每小时(可在定时任务页关闭或改周期) + registerScheduledJob(ctx, { + key: "sites.user_data", + defaults: { + name: "站点数据刷新", + description: "登录状态正常的站点账号刷新上传 / 下载 / 分享率等用户数据;会话失效的账密账号自动重登一次", + group: "collect", + schedule: { type: "interval", everyMs: 6 * 3_600_000 }, + firstRun: "after_interval", + }, + run: async ({ slot }) => { + if (await readOnlyReconcile(rt)) return { status: "skipped", summary: "只读对账模式" }; + return { summary: `${await tickSiteUserData(rt, slot)} 个账号` }; + }, + }); + registerScheduledJob(ctx, { + key: "sites.health_check", + defaults: { + name: "站点健康检测", + description: "已启用站点逐个检测首页连通与登录态,结果显示在站点页", + group: "collect", + schedule: { type: "interval", everyMs: 3_600_000 }, + firstRun: "after_interval", + }, + run: async () => ({ summary: `${await tickSiteHealth(rt)} 个站点` }), + }); }, }; } diff --git a/apps/server/src/modules/sources-identity/routes/decisions.ts b/apps/server/src/modules/sources-identity/routes/decisions.ts index c6e23f22..ec4e3476 100644 --- a/apps/server/src/modules/sources-identity/routes/decisions.ts +++ b/apps/server/src/modules/sources-identity/routes/decisions.ts @@ -1,3 +1,4 @@ +import { randomUUID } from "node:crypto"; import { type Decision, DecisionKind, DecisionStatus, Id, type PoolCandidate } from "@umt/contracts"; import { sql } from "@umt/db"; import { z } from "zod"; @@ -163,6 +164,15 @@ export function decisionRoutes(app: ModuleApp, rt: () => SourcesRuntime) { ); } } + // 站点登录需人工验证:"重试登录" = 入队刷新登录(结果另行更新账号状态与待处理) + if (d.dedupe_key?.startsWith("site-login:") && opt?.key === "retry") { + const accountId = d.dedupe_key.slice("site-login:".length); + await rt().ctx.engine.enqueue( + "refreshSiteAccount", + { accountId, mode: "login" }, + { idempotencyKey: `refreshSiteAccount:decision:${d.id}:${Date.now()}`, traceId: randomUUID() }, + ); + } // 上面的执行路径已按 dedupeKey 关闭;其余类型在这里记录处理结果 const row = await db() .transaction() diff --git a/apps/server/src/modules/sources-identity/routes/sources.ts b/apps/server/src/modules/sources-identity/routes/sources.ts index 55785aa2..86b073a3 100644 --- a/apps/server/src/modules/sources-identity/routes/sources.ts +++ b/apps/server/src/modules/sources-identity/routes/sources.ts @@ -7,13 +7,21 @@ import { Id, IdParam, type Observation, + OkResult, ReleaseIdentityAssign, type SecretPurpose, Site, type SiteAccount, type SiteAdapter, + SiteCredentialsPut, + SiteCredentialsResult, SiteDefinitionView, + SiteHealthResult, + SiteRefreshLoginAccepted, + SiteRefreshLoginInput, SiteSnapshot, + SiteUserDataPoint, + SiteUserDataQuery, type SourceRelease, } from "@umt/contracts"; import { sql } from "@umt/db"; @@ -24,6 +32,16 @@ import { D } from "../defs.ts"; import { route } from "../http.ts"; import { decodeCursor, feedToApi, jsonb, page, toApi, toApiList } from "../rows.ts"; import type { SourcesRuntime } from "../runtime.ts"; +import { + accountToApi, + checkSiteHealth, + deleteSite, + deleteSiteAccount, + overallAccountStatus, + refreshSiteAccount, + setSiteCredentials, + userDataHistory, +} from "../service/account.ts"; import { audit, closeDecisions } from "../service/common.ts"; import { assignReleaseIdentity } from "../service/identity-admin.ts"; @@ -61,7 +79,7 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) { response: z.object({ items: z.array(SiteDefinitionView) }), }, async () => { - const rows = await db().selectFrom("site").select(["id", "key"]).execute(); + const rows = await db().selectFrom("site").select(["id", "key"]).where("deleted_at", "is", null).execute(); const byKey = new Map(rows.map((r) => [r.key, r.id])); return { items: listSiteDefinitions().map((d) => ({ @@ -86,12 +104,31 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) { const rows = await db() .selectFrom("site") .selectAll() + .where("deleted_at", "is", null) .orderBy("key") .offset(off) .limit(query.limit + 1) .execute(); const p = page(rows, off, query.limit); - return { items: toApiList(p.items), nextCursor: p.nextCursor }; + const items = toApiList>(p.items); + if (query.include === "accounts" && p.items.length) { + const accounts = await db() + .selectFrom("site_account") + .selectAll() + .where( + "site_id", + "in", + p.items.map((s) => s.id), + ) + .orderBy("created_at") + .execute(); + for (const it of items) { + const mine = accounts.filter((a) => a.site_id === it.id); + it.accounts = mine.map(accountToApi); + it.accountStatus = overallAccountStatus(mine); + } + } + return { items, nextCursor: p.nextCursor }; }); route(app, D.sites.get, async ({ params }) => toApi(await getSite(params.id))); route(app, D.sites.create, async ({ body, actor }) => { @@ -115,6 +152,7 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) { config: jsonb(body.config ?? {}), snapshot_enabled: body.snapshotEnabled ?? false, snapshot_interval_sec: body.snapshotIntervalSec ?? null, + proxy: body.proxy ?? false, }) .returningAll() .executeTakeFirstOrThrow(); @@ -135,7 +173,15 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) { if (body.config !== undefined) patch.config = jsonb(body.config); if (body.snapshotEnabled !== undefined) patch.snapshot_enabled = body.snapshotEnabled; if (body.snapshotIntervalSec !== undefined) patch.snapshot_interval_sec = body.snapshotIntervalSec; - if (body.enabled !== undefined) return toApi(await setSiteEnabled(params.id, body.enabled, actor)); + if (body.proxy !== undefined) patch.proxy = body.proxy; + if (body.enabled !== undefined) { + if (Object.keys(patch).length) { + await db().updateTable("site").set(patch).where("id", "=", params.id).execute(); + await audit(db(), actor, "site.updated", { type: "site", id: params.id }, { fields: Object.keys(patch) }); + } + return toApi(await setSiteEnabled(params.id, body.enabled, actor)); + } + if (!Object.keys(patch).length) return toApi(await getSite(params.id)); const row = await db() .updateTable("site") .set(patch) @@ -210,6 +256,132 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) { }, ); + // ── W9-A 站点中心:删除、健康检测、刷新登录、用户数据、凭证 ── + route( + app, + { + method: "delete", + path: "/sites/{id}", + scope: "admin", + summary: "删除站点:停用后删账号与凭证;已抓到的资源和下载任务保留(标站点已关闭)", + tags: T_SITES, + params: IdParam, + response: OkResult, + }, + async ({ params, actor }) => { + const s = await getSite(params.id); + await deleteSite(rt(), s, actor); + return { ok: true as const }; + }, + ); + route( + app, + { + method: "post", + path: "/sites/{id}/health-check", + scope: "admin", + summary: "检测健康:首页连通 + 登录态,同步返回并写站点健康列", + tags: T_SITES, + params: IdParam, + response: SiteHealthResult, + }, + async ({ params, actor }) => checkSiteHealth(rt(), await getSite(params.id), actor), + ); + route( + app, + { + method: "post", + path: "/sites/{id}/refresh-login", + scope: "admin", + summary: "刷新登录:账密重新登录换 Cookie;Cookie / API Key 校验;成功后拉用户数据", + tags: T_SITES, + params: IdParam, + body: SiteRefreshLoginInput, + response: SiteRefreshLoginAccepted, + }, + async ({ params, body, actor }) => { + const s = await getSite(params.id); + let q = db().selectFrom("site_account").select(["id", "enabled", "auth_kind"]).where("site_id", "=", s.id); + q = body.accountId + ? q.where("id", "=", body.accountId) + : q.where("enabled", "=", true).where("auth_kind", "!=", "passkey"); + const acc = await q.orderBy("created_at").executeTakeFirst(); + if (!acc) { + throw new ApiException( + body.accountId ? "not_found" : "precondition_failed", + body.accountId ? "站点账号不存在" : "站点还没有凭证", + ); + } + if (body.wait) { + const result = await refreshSiteAccount(rt(), acc.id, { + mode: "login", + actor, + signal: AbortSignal.timeout(30_000), + }); + return { taskId: null, result }; + } + const r = await rt().ctx.engine.enqueue( + "refreshSiteAccount", + { accountId: acc.id, mode: "login" }, + { + idempotencyKey: `refreshSiteAccount:manual:${acc.id}:${Math.floor(Date.now() / 10_000)}`, + traceId: randomUUID(), + }, + ); + await audit(db(), actor, "site_account.login_refresh_requested", { type: "site_account", id: acc.id }); + return { taskId: r.runId, result: null }; + }, + ); + route( + app, + { + method: "get", + path: "/sites/{id}/user-data", + scope: "read", + summary: "站点账号用户数据按天历史", + tags: T_SITES, + params: IdParam, + query: SiteUserDataQuery, + response: z.object({ items: z.array(SiteUserDataPoint) }), + }, + async ({ params, query }) => { + await getSite(params.id); + return { items: await userDataHistory(rt(), params.id, query) }; + }, + ); + route( + app, + { + method: "put", + path: "/sites/{id}/credentials", + scope: "admin", + summary: "设置站点账号凭证(按 kind:password / cookie / api_key / passkey;存为 secret,不回显)", + tags: T_SITES, + params: IdParam, + body: SiteCredentialsPut, + response: SiteCredentialsResult, + }, + async ({ params, body, actor }) => setSiteCredentials(rt(), await getSite(params.id), body, actor), + ); + route( + app, + { + method: "delete", + path: "/site-accounts/{id}", + scope: "admin", + summary: "删除站点账号与其凭证", + tags: T_SITES, + params: IdParam, + response: OkResult, + }, + async ({ params, actor }) => { + const acc = await db().selectFrom("site_account").selectAll().where("id", "=", params.id).executeTakeFirst(); + if (!acc) throw new ApiException("not_found", "站点账号不存在"); + await deleteSiteAccount(rt(), acc, actor); + return { ok: true as const }; + }, + ); + // ── 站点账号 ── route(app, D.siteAccounts.list, async ({ query }) => { const off = decodeCursor(query.cursor); @@ -221,12 +393,12 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) { .limit(query.limit + 1) .execute(); const p = page(rows, off, query.limit); - return { items: toApiList(p.items), nextCursor: p.nextCursor }; + return { items: p.items.map(accountToApi) as SiteAccount[], nextCursor: p.nextCursor }; }); route(app, D.siteAccounts.get, async ({ params }) => { const row = await db().selectFrom("site_account").selectAll().where("id", "=", params.id).executeTakeFirst(); if (!row) throw new ApiException("not_found", "站点账号不存在"); - return toApi(row); + return accountToApi(row); }); route(app, D.siteAccounts.create, async ({ body, actor }) => { await getSite(body.siteId); @@ -248,7 +420,7 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) { .returningAll() .executeTakeFirstOrThrow(); await audit(db(), actor, "site_account.created", { type: "site_account", id: row.id }, { siteId: body.siteId }); - return toApi(row); + return accountToApi(row); } catch (e) { await rt() .ctx.secrets.remove(ref) @@ -273,8 +445,10 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) { label: `site_account:${acc.label}`, }); } - // 换了凭证:状态回到未验证,关闭"登录失效"待处理,站点健康复位 + // 换了凭证:状态回到未验证,清会话 Cookie,关闭"登录失效"待处理,站点健康复位 patch.status = "unknown"; + patch.last_error = null; + patch.session_secret_ref = null; } const row = await db() .transaction() @@ -300,6 +474,14 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) { { by: "credential_updated" }, `${actor.type}:${actor.id}`, ); + await closeDecisions( + rt().ctx, + trx, + `site-login:${acc.id}`, + "resolved", + { by: "credential_updated" }, + `${actor.type}:${actor.id}`, + ); } await audit( trx, @@ -313,7 +495,7 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) { ); return r; }); - return toApi(row); + return accountToApi(row); }); // ── Feed ── @@ -588,7 +770,12 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) { // ── 内部辅助 ── async function getSite(id: string) { - const row = await db().selectFrom("site").selectAll().where("id", "=", id).executeTakeFirst(); + const row = await db() + .selectFrom("site") + .selectAll() + .where("id", "=", id) + .where("deleted_at", "is", null) + .executeTakeFirst(); if (!row) throw new ApiException("not_found", "站点不存在"); return row; } diff --git a/apps/server/src/modules/sources-identity/runtime.ts b/apps/server/src/modules/sources-identity/runtime.ts index 56fa323b..18e2946f 100644 --- a/apps/server/src/modules/sources-identity/runtime.ts +++ b/apps/server/src/modules/sources-identity/runtime.ts @@ -11,6 +11,7 @@ import { type SiteRateLimiter, type TmdbClient, } from "@umt/adapters"; +import { DEFAULT_SETTINGS, type SitesSettings } from "@umt/contracts"; import type { SiteRow } from "@umt/db"; import type { ServerContext } from "../types.ts"; @@ -40,6 +41,9 @@ export class SourcesRuntime { readonly rss: RssFetcher; private readonly adapters = new Map(); private tmdbClient: TmdbClient | null | undefined; + /** 站点默认(settings.sites)缓存:siteAdapter 是同步的,读设置的地方顺手刷新 */ + private siteDefaultsCache: SitesSettings = DEFAULT_SETTINGS.sites; + private siteDefaultsAt = 0; private tmdbSig: string | null = null; constructor( @@ -101,17 +105,60 @@ export class SourcesRuntime { return this.tmdbClient; } + /** 站点默认(settings.sites),读库并刷新缓存;读失败用上次值 */ + async siteDefaults(): Promise { + try { + this.siteDefaultsCache = { ...DEFAULT_SETTINGS.sites, ...(await this.ctx.settings.get("sites")) }; + } catch { + // 保留缓存 + } + return this.siteDefaultsCache; + } + + /** + * 站点请求用的 fetch:测试注入优先;站点开了"使用代理"且设置了 proxyUrl 时走代理(Deno.createHttpClient)。 + */ + siteFetch(site: Pick, defaults: SitesSettings = this.siteDefaultsCache): typeof globalThis.fetch { + if (this.overrides.fetch) return this.overrides.fetch; + if (!site.proxy || !defaults.proxyUrl) return globalThis.fetch; + const key = defaults.proxyUrl; + let client = this.proxyClients.get(key); + if (!client) { + client = Deno.createHttpClient({ proxy: { url: key } }); + this.proxyClients.set(key, client); + } + const c = client; + return ((input: RequestInfo | URL, init?: RequestInit) => + globalThis.fetch(input, { ...init, client: c } as RequestInit)) as typeof globalThis.fetch; + } + private readonly proxyClients = new Map(); + /** * 站点适配器:每站一个实例,列表/搜索/详情/下载共享同一个限流器。 - * 站点限流参数或基址变化时重建(签名比较)。 + * 站点限流参数、基址、代理或站点默认变化时重建(签名比较)。 */ - siteAdapter(site: Pick): SiteAdapter { + siteAdapter( + site: Pick & Partial>, + ): SiteAdapter { const def = getSiteDefinition(site.key); if (!def) throw new Error(`站点 ${site.key} 没有随产品发布的适配器定义`); - const sig = JSON.stringify([site.base_url, site.rate_limit, site.config]); + if (Date.now() - this.siteDefaultsAt > 30_000) { + this.siteDefaultsAt = Date.now(); + void this.siteDefaults(); + } + const defaults = this.siteDefaultsCache; + const proxy = site.proxy ?? false; + const sig = JSON.stringify([ + site.base_url, + site.rate_limit, + site.config, + proxy, + proxy ? defaults.proxyUrl : null, + defaults.defaultRateLimit, + ]); const hit = this.adapters.get(site.id); if (hit && hit.sig === sig) return hit.adapter; - const limiter = createSiteRateLimiter(limiterOptions(def, site.rate_limit)); + const limiter = createSiteRateLimiter(limiterOptions(def, site.rate_limit, defaults.defaultRateLimit)); const cfg = (site.config ?? {}) as { apiBaseUrl?: string }; const baseUrl = def.umt.kind === "mteam" ? cfg.apiBaseUrl : sameUrl(site.base_url, def.domain) ? undefined : site.base_url; @@ -119,7 +166,7 @@ export class SourcesRuntime { secrets: this.siteSecrets(site.id), limiter, now: this.now, - ...(this.overrides.fetch ? { fetch: this.overrides.fetch } : {}), + ...(this.overrides.fetch || proxy ? { fetch: this.siteFetch({ proxy }, defaults) } : {}), ...(baseUrl ? { baseUrl } : {}), }); this.adapters.set(site.id, { sig, limiter, adapter }); @@ -131,24 +178,39 @@ export class SourcesRuntime { return this.adapters.get(siteId)?.limiter ?? null; } - /** 站点凭证:取该站第一个启用且未失效的账号,按 auth_kind 对应适配器要求的凭证类型;明文不缓存 */ + /** + * 站点凭证:取该站第一个启用且未失效的账号;明文不缓存。 + * cookie:Cookie 型账号的 secret,或账密账号登录得到的会话 Cookie;apiKey:API Key 型账号; + * passkey:账号附挂的 passkey 或 passkey 型账号(RSS / 下载用,不看登录状态)。 + */ private siteSecrets(siteId: string): SecretResolver { const { db, secrets } = this.ctx; return { async resolve({ kind }) { - const want = kind === "cookie" ? ["cookie"] : kind === "apiKey" ? ["api_key"] : ["passkey"]; - const acc = await db + let q = db .selectFrom("site_account") - .select(["secret_ref"]) + .select(["auth_kind", "secret_ref", "session_secret_ref", "passkey_secret_ref"]) .where("site_id", "=", siteId) .where("enabled", "=", true) - .where("status", "in", ["unknown", "active"]) - .where("auth_kind", "in", want as never) - .where("secret_ref", "is not", null) - .orderBy("created_at") - .executeTakeFirst(); - if (!acc?.secret_ref) return null; - return secrets.resolve(acc.secret_ref); + .orderBy("created_at"); + if (kind !== "passkey") q = q.where("status", "in", ["unknown", "active"]); + const rows = await q.execute(); + for (const r of rows) { + const ref = + kind === "cookie" + ? r.auth_kind === "cookie" + ? r.secret_ref + : r.auth_kind === "password" + ? r.session_secret_ref + : null + : kind === "apiKey" + ? r.auth_kind === "api_key" + ? r.secret_ref + : null + : (r.passkey_secret_ref ?? (r.auth_kind === "passkey" ? r.secret_ref : null)); + if (ref) return secrets.resolve(ref); + } + return null; }, }; } @@ -158,11 +220,32 @@ function sameUrl(a: string, b: string): boolean { return a.replace(/\/+$/, "") === b.replace(/\/+$/, ""); } -/** 站点行 rate_limit 优先,缺省用定义里的 umt.rateLimit */ -export function limiterOptions(def: SiteDefinition, rateLimit: unknown): { minIntervalMs: number; burst: number } { - const rl = (rateLimit ?? {}) as { requestsPerMinute?: number; minIntervalMs?: number; concurrency?: number }; - const minIntervalMs = - rl.minIntervalMs ?? - (rl.requestsPerMinute ? Math.ceil(60_000 / rl.requestsPerMinute) : def.umt.rateLimit.minIntervalMs); - return { minIntervalMs, burst: rl.concurrency ?? def.umt.rateLimit.burst }; +type RateLimitLike = { requestsPerMinute?: number; minIntervalMs?: number; concurrency?: number }; + +/** + * 站点行 rate_limit 优先;为空时用全局默认(settings.sites.defaultRateLimit),但不比定义里的 umt.rateLimit 更激进 + * (定义值是按站点实测调的下限);都没有时用定义值。 + */ +export function limiterOptions( + def: SiteDefinition, + rateLimit: unknown, + globalDefault?: RateLimitLike | null, +): { minIntervalMs: number; burst: number } { + const own = (rateLimit ?? {}) as RateLimitLike; + const hasOwn = + own.minIntervalMs !== undefined || own.requestsPerMinute !== undefined || own.concurrency !== undefined; + const interval = (rl: RateLimitLike) => + rl.minIntervalMs ?? (rl.requestsPerMinute ? Math.ceil(60_000 / rl.requestsPerMinute) : undefined); + if (hasOwn) { + return { + minIntervalMs: interval(own) ?? def.umt.rateLimit.minIntervalMs, + burst: own.concurrency ?? def.umt.rateLimit.burst, + }; + } + const g = globalDefault ?? {}; + const gi = interval(g); + return { + minIntervalMs: gi === undefined ? def.umt.rateLimit.minIntervalMs : Math.max(gi, def.umt.rateLimit.minIntervalMs), + burst: g.concurrency === undefined ? def.umt.rateLimit.burst : Math.min(g.concurrency, def.umt.rateLimit.burst), + }; } diff --git a/apps/server/src/modules/sources-identity/scheduler.ts b/apps/server/src/modules/sources-identity/scheduler.ts index 03dea559..faa3167c 100644 --- a/apps/server/src/modules/sources-identity/scheduler.ts +++ b/apps/server/src/modules/sources-identity/scheduler.ts @@ -66,10 +66,13 @@ export async function tickFeeds(rt: SourcesRuntime): Promise { export async function tickSnapshots(rt: SourcesRuntime): Promise { const { ctx } = rt; const now = rt.now(); + // 站点未设周期时用全局默认(settings.sites.defaultSnapshotIntervalSec) + const fallback = (await rt.siteDefaults()).defaultSnapshotIntervalSec; const sites = await sql<{ id: string; snapshot_interval_sec: number }>` - select id, snapshot_interval_sec from site - where enabled and snapshot_enabled and snapshot_interval_sec is not null - and (last_snapshot_at is null or last_snapshot_at + make_interval(secs => snapshot_interval_sec) <= ${now})`.execute( + select id, coalesce(snapshot_interval_sec, ${fallback}::int) as snapshot_interval_sec from site + where enabled and snapshot_enabled and deleted_at is null + and (last_snapshot_at is null + or last_snapshot_at + make_interval(secs => coalesce(snapshot_interval_sec, ${fallback}::int)) <= ${now})`.execute( ctx.db, ); for (const s of sites.rows) { diff --git a/apps/server/src/modules/sources-identity/service/account.ts b/apps/server/src/modules/sources-identity/service/account.ts new file mode 100644 index 00000000..06a5c7dc --- /dev/null +++ b/apps/server/src/modules/sources-identity/service/account.ts @@ -0,0 +1,884 @@ +import { randomUUID } from "node:crypto"; +import { + getSiteDefinition, + loginNexusPhp, + refreshSiteUserData, + SiteLoginError, + type SiteLoginFailure, + type SiteUserInfo, +} from "@umt/adapters"; +import type { + DecisionOption, + SecretPurpose, + SiteAccountStatus, + SiteAccountUserData, + SiteCredentialsPut, + SiteCredentialsResult, + SiteHealth, + SiteHealthResult, + SiteLoginResult, + SiteUserDataPoint, +} from "@umt/contracts"; +import { type SiteAccountRow, type SiteRow, sql } from "@umt/db"; +import { toApi } from "../rows.ts"; +import type { SourcesRuntime } from "../runtime.ts"; +import { type Actor, audit, closeDecisions, type Exec, openDecision, SYSTEM } from "./common.ts"; + +// W9-A 站点中心(docs/design/site-center.md):凭证写入、刷新登录(账密 takelogin / Cookie / API Key 校验)、 +// 用户数据与按天历史、健康检测、删除。凭证明文只在内存里用,不进日志、审计与错误信息。 + +const jsonb = (v: unknown) => JSON.stringify(v ?? null); + +/** site_account 行 → SiteAccount(secret 引用只给布尔) */ +export function accountToApi(row: SiteAccountRow): Record { + const { totp_secret_ref, session_secret_ref, passkey_secret_ref, ...rest } = row; + return { + ...toApi>(rest), + hasTotp: !!totp_secret_ref, + hasSession: !!session_secret_ref, + hasPasskey: !!passkey_secret_ref || (row.auth_kind === "passkey" && !!row.secret_ref), + }; +} + +/** 多账号取最差;无启用账号 = unconfigured */ +const STATUS_RANK: Record = { active: 0, unknown: 1, expired: 2, invalid: 3, challenge: 4 }; +export function overallAccountStatus(rows: Pick[]) { + const enabled = rows.filter((r) => r.enabled); + if (!enabled.length) return "unconfigured" as const; + return enabled.reduce( + (worst, r) => (STATUS_RANK[r.status] > STATUS_RANK[worst] ? r.status : worst), + "active", + ); +} + +/** 本地日期(按进程时区)YYYY-MM-DD */ +function localDay(d: Date): string { + return new Date(d.getTime() - d.getTimezoneOffset() * 60_000).toISOString().slice(0, 10); +} + +export function userDataOf(info: SiteUserInfo): SiteAccountUserData { + const upload = Math.max(0, info.upload || 0); + const download = Math.max(0, info.download || 0); + const ratio = info.ratio ? info.ratio : download > 0 ? Math.round((upload / download) * 1000) / 1000 : null; + const int = (n: number) => Math.max(0, Math.round(n || 0)); + return { + schema: info.schema ?? null, + userId: info.userId ?? null, + username: info.username ?? null, + userLevel: info.userLevel ?? null, + vip: info.vip ?? null, + joinAt: info.joinAt ?? null, + upload, + download, + ratio, + bonus: info.bonus ? info.bonus : null, + seeding: int(info.seeding), + seedingSize: Math.max(0, info.seedingSize || 0), + leeching: int(info.leeching), + leechingSize: Math.max(0, info.leechingSize || 0), + messageUnread: int(info.messageUnread), + }; +} + +// ── 凭证写入 ── + +const PURPOSE: Record = { + cookie: "site_cookie", + passkey: "site_passkey", + api_key: "site_api_key", + password: "site_password", +}; + +type NormalizedCredentials = + | { + kind: "password"; + label: string; + username: string; + password: string; + totpSecret?: string; + saveCookieOnly?: boolean; + userAgent?: string; + enabled?: boolean; + } + | { kind: "cookie"; label: string; cookie: string; userAgent?: string; enabled?: boolean } + | { kind: "api_key"; label: string; apiKey: string; enabled?: boolean } + | { kind: "passkey"; label: string; passkey: string; enabled?: boolean }; + +/** 旧格式 {authKind, credential, username?} → 新格式 */ +export function normalizeCredentials(body: SiteCredentialsPut): NormalizedCredentials { + if ("kind" in body) return body as NormalizedCredentials; + const label = body.label; + const enabled = body.enabled; + const opt = enabled === undefined ? {} : { enabled }; + switch (body.authKind) { + case "password": + return { kind: "password", label, username: body.username || label, password: body.credential, ...opt }; + case "cookie": + return { kind: "cookie", label, cookie: body.credential, ...opt }; + case "api_key": + return { kind: "api_key", label, apiKey: body.credential, ...opt }; + default: + return { kind: "passkey", label, passkey: body.credential, ...opt }; + } +} + +/** + * PUT /sites/{id}/credentials:按 (site, label) 建立或更新账号。 + * 主凭证(password / cookie / api_key)换掉旧 secret、状态回 unknown、清会话 Cookie、关闭 site-auth / site-login 待处理; + * passkey 只附挂到账号(不改登录状态)。 + */ +export async function setSiteCredentials( + rt: SourcesRuntime, + site: Pick, + input: SiteCredentialsPut, + actor: Actor, +): Promise { + const { db, secrets } = rt.ctx; + const body = normalizeCredentials(input); + const label = body.label; + const created: string[] = []; + const mk = async (purpose: SecretPurpose, plaintext: string, suffix = "") => { + const ref = await secrets.create({ purpose, plaintext, label: `site_account:${label}${suffix}` }); + created.push(ref); + return ref; + }; + const obsolete: string[] = []; + let result: SiteCredentialsResult; + try { + if (body.kind === "passkey") { + const ref = await mk("site_passkey", body.passkey, ":passkey"); + result = await db.transaction().execute(async (trx) => { + const cur = await trx + .selectFrom("site_account") + .selectAll() + .where("site_id", "=", site.id) + .where("label", "=", label) + .forUpdate() + .executeTakeFirst(); + if (cur) { + if (cur.auth_kind === "passkey") { + if (cur.secret_ref) obsolete.push(cur.secret_ref); + await trx + .updateTable("site_account") + .set({ secret_ref: ref, ...(body.enabled !== undefined ? { enabled: body.enabled } : {}) }) + .where("id", "=", cur.id) + .execute(); + } else { + if (cur.passkey_secret_ref) obsolete.push(cur.passkey_secret_ref); + await trx + .updateTable("site_account") + .set({ passkey_secret_ref: ref, ...(body.enabled !== undefined ? { enabled: body.enabled } : {}) }) + .where("id", "=", cur.id) + .execute(); + } + return { + siteAccountId: cur.id, + siteId: site.id, + label, + authKind: cur.auth_kind, + secretRef: ref, + created: false, + }; + } + const r = await trx + .insertInto("site_account") + .values({ + site_id: site.id, + label, + auth_kind: "passkey", + secret_ref: ref, + ...(body.enabled !== undefined ? { enabled: body.enabled } : {}), + }) + .returning("id") + .executeTakeFirstOrThrow(); + return { siteAccountId: r.id, siteId: site.id, label, authKind: "passkey", secretRef: ref, created: true }; + }); + } else { + const plaintext = body.kind === "password" ? body.password : body.kind === "cookie" ? body.cookie : body.apiKey; + const ref = await mk(PURPOSE[body.kind]!, plaintext); + const totpRef = + body.kind === "password" && body.totpSecret ? await mk("site_password", body.totpSecret, ":totp") : null; + result = await db.transaction().execute(async (trx) => { + const cur = await trx + .selectFrom("site_account") + .selectAll() + .where("site_id", "=", site.id) + .where("label", "=", label) + .forUpdate() + .executeTakeFirst(); + const userInfo = { + ...((cur?.user_info as Record | undefined) ?? {}), + ...(body.kind === "password" ? { username: body.username, saveCookieOnly: !!body.saveCookieOnly } : {}), + }; + if (body.kind !== "password") delete userInfo.saveCookieOnly; + const values = { + auth_kind: body.kind, + secret_ref: ref, + status: "unknown" as const, + last_error: null, + session_secret_ref: null, + totp_secret_ref: totpRef, + login_name: body.kind === "password" ? body.username : null, + user_agent: "userAgent" in body ? (body.userAgent ?? null) : null, + user_info: jsonb(userInfo), + ...(body.enabled !== undefined ? { enabled: body.enabled } : {}), + }; + let id: string; + let isNew = false; + if (cur) { + for (const r of [ + cur.auth_kind === "passkey" ? null : cur.secret_ref, + cur.session_secret_ref, + cur.totp_secret_ref, + ]) + if (r) obsolete.push(r); + // 原来是 passkey 型账号:passkey 转为附挂 + const passkey = cur.auth_kind === "passkey" && cur.secret_ref ? { passkey_secret_ref: cur.secret_ref } : {}; + await trx + .updateTable("site_account") + .set({ ...values, ...passkey }) + .where("id", "=", cur.id) + .execute(); + id = cur.id; + } else { + const r = await trx + .insertInto("site_account") + .values({ site_id: site.id, label, ...values }) + .returning("id") + .executeTakeFirstOrThrow(); + id = r.id; + isNew = true; + } + await trx + .updateTable("site") + .set({ health_status: "unknown" }) + .where("id", "=", site.id) + .where("health_status", "=", "failing") + .execute(); + const by = `${actor.type}:${actor.id ?? ""}`; + await closeDecisions(rt.ctx, trx, `site-auth:${site.id}`, "resolved", { by: "credential_updated" }, by); + await closeDecisions(rt.ctx, trx, `site-login:${id}`, "resolved", { by: "credential_updated" }, by); + return { siteAccountId: id, siteId: site.id, label, authKind: body.kind, secretRef: ref, created: isNew }; + }); + } + } catch (e) { + for (const r of created) await secrets.remove(r).catch(() => {}); + throw e; + } + for (const r of obsolete) await secrets.remove(r).catch(() => {}); + await audit( + db, + actor, + "site_account.credentials_set", + { type: "site_account", id: result.siteAccountId }, + { siteId: site.id, label, authKind: body.kind, created: result.created }, + ); + return result; +} + +// ── 刷新登录 / 用户数据 ── + +type UserDataOutcome = + | { kind: "ok"; info: SiteUserInfo } + | { kind: "auth"; message: string } + | { kind: "network"; message: string } + | { kind: "challenge"; message: string } + | { kind: "no_parser" }; + +const FAIL_STATUS: Partial> = { + captcha: "challenge", + two_factor: "challenge", + challenge: "challenge", + bad_credentials: "invalid", + account_blocked: "invalid", + auth_expired: "expired", + unknown: "expired", +}; + +export interface RefreshOptions { + /** login:重新登录 / 校验(手动刷新登录);userData:定时刷新用户数据,会话失效时账密账号自动重登 */ + mode?: "login" | "userData"; + actor?: Actor; + signal?: AbortSignal; +} + +export async function refreshSiteAccount( + rt: SourcesRuntime, + accountId: string, + opts: RefreshOptions = {}, +): Promise { + const { db, secrets } = rt.ctx; + const actor = opts.actor ?? SYSTEM; + const acc = await db.selectFrom("site_account").selectAll().where("id", "=", accountId).executeTakeFirst(); + if (!acc) throw new Error("站点账号不存在"); + const site = await db.selectFrom("site").selectAll().where("id", "=", acc.site_id).executeTakeFirstOrThrow(); + const base = { siteId: site.id, accountId: acc.id }; + const unchanged = (reason: SiteLoginFailure, message: string): SiteLoginResult => ({ + ...base, + ok: false, + status: acc.status, + reason, + message, + userData: null, + decisionId: null, + }); + const def = getSiteDefinition(site.key); + if (!def || site.deleted_at) return unchanged("unsupported", "站点没有适配器定义,无法校验登录"); + const defaults = await rt.siteDefaults(); + const ua = acc.user_agent ?? defaults.userAgent; + let limiter: ReturnType = null; + try { + rt.siteAdapter(site); + limiter = rt.limiterOf(site.id); + } catch { + // 无适配器:不限流(仅 generic_rss,前面已返回) + } + const fetch = rt.siteFetch(site, defaults); + const umt = def.umt as { kind: string; login?: Parameters[0]["login"]; loginSuccess?: string }; + const profile = (def.umt as { profile?: Record }).profile; + const vipLevels = (def.umt as { vipLevels?: string[] }).vipLevels; + const apiUrl = def.umt.kind === "mteam" ? ((site.config as { apiBaseUrl?: string })?.apiBaseUrl ?? null) : null; + + const fetchUserData = async (cred: { cookie?: string | null; apikey?: string | null }): Promise => { + const info = await refreshSiteUserData({ + name: site.name, + url: site.base_url, + schema: (def as { schema?: string }).schema ?? null, + public: true, + cookie: cred.cookie ?? null, + apikey: cred.apikey ?? null, + ua, + fetch, + ...(limiter ? { limiter } : {}), + ...(apiUrl ? { apiUrl } : {}), + fetchMessages: false, + maxSeedingPages: defaults.userDataMaxSeedingPages, + vipCss: profile?.vip_css ?? null, + vipLevels: vipLevels ?? null, + timeoutMs: 20_000, + ...(opts.signal ? { signal: opts.signal } : {}), + }); + if (!info) return { kind: "no_parser" }; + if (info.userId && !info.errMsg) return { kind: "ok", info }; + const errs = ((info.raw as { http_errors?: string[] }).http_errors ?? []).filter(Boolean); + if (errs.some((e) => /Cloudflare/i.test(e))) return { kind: "challenge", message: "站点返回 Cloudflare 挑战页" }; + // 连接失败 / 超时 / 5xx 视为网络问题;4xx 与解析失败按登录态失效处理 + const net = errs.find((e) => !/HTTP 4\d\d|JSON 解析失败/.test(e)); + if (net) return { kind: "network", message: net.slice(0, 200) }; + return { kind: "auth", message: info.errMsg ?? "未检测到登录态" }; + }; + + const viaHealth = async (): Promise => { + const h = await rt.siteAdapter(site).healthCheck(opts.signal ? { signal: opts.signal } : {}); + if (h.ok) return { kind: "no_parser" }; + if (h.status.startsWith("auth_")) return { kind: "auth", message: h.message ?? "未检测到登录态" }; + return { kind: "network", message: h.message ?? h.status }; + }; + + const fail = (reason: SiteLoginFailure, message: string) => markFailure(rt, acc, site, reason, message, actor); + const succeed = (info: SiteUserInfo | null, note?: string) => markSuccess(rt, acc, site, info, actor, note); + + try { + if (acc.auth_kind === "passkey") return unchanged("unsupported", "Passkey 只用于 RSS 和下载,不参与登录"); + if (!acc.secret_ref) return unchanged("unsupported", "账号没有凭证"); + if (acc.auth_kind === "password") { + if (umt.kind !== "nexusphp" || !umt.login) { + return unchanged("unsupported", "该站点不支持账号密码登录。改用 Cookie 或 API Key"); + } + if (opts.mode === "userData" && acc.session_secret_ref) { + const r = await fetchUserData({ cookie: await secrets.resolve(acc.session_secret_ref) }); + if (r.kind === "ok") return await succeed(r.info); + if (r.kind === "network") return await fail("network", r.message); + // 会话失效:下面重登一次 + } + const password = await secrets.resolve(acc.secret_ref); + const totpSecret = acc.totp_secret_ref ? await secrets.resolve(acc.totp_secret_ref) : null; + const login = await loginNexusPhp({ + baseUrl: site.base_url, + login: umt.login, + ...(umt.loginSuccess ? { loginSuccess: umt.loginSuccess } : {}), + username: acc.login_name ?? (acc.user_info as { username?: string })?.username ?? acc.label, + password, + totpSecret, + userAgent: ua, + fetch, + ...(limiter ? { limiter } : {}), + ...(opts.signal ? { signal: opts.signal } : {}), + now: rt.now, + }); + await storeSession(rt, acc, login.cookie); + const r = await fetchUserData({ cookie: login.cookie }); + if (r.kind === "ok") return await succeed(r.info); + return await succeed( + null, + `登录成功;用户数据未取到:${r.kind === "no_parser" ? "站点没有用户数据解析器" : r.message}`, + ); + } + const plain = await secrets.resolve(acc.secret_ref); + let r = await fetchUserData(acc.auth_kind === "cookie" ? { cookie: plain } : { apikey: plain }); + if (r.kind === "no_parser") r = await viaHealth(); + if (r.kind === "ok") return await succeed(r.info); + if (r.kind === "no_parser") return await succeed(null, "站点没有用户数据解析器,只校验了登录态"); + if (r.kind === "network") return await fail("network", r.message); + if (r.kind === "challenge") return await fail("challenge", r.message); + return await fail( + "auth_expired", + acc.auth_kind === "api_key" + ? "API Key 无效。更新 API Key 后重新刷新登录" + : "Cookie 已失效。更新 Cookie 后重新刷新登录", + ); + } catch (e) { + if (e instanceof SiteLoginError) return await fail(e.reason, e.message); + throw e; + } +} + +/** 账密登录成功:会话 Cookie 存 secret;"只换 Cookie"时凭证类型落为 cookie,密码与 TOTP 丢弃 */ +async function storeSession(rt: SourcesRuntime, acc: SiteAccountRow, cookie: string) { + const { db, secrets } = rt.ctx; + const cookieOnly = (acc.user_info as { saveCookieOnly?: boolean })?.saveCookieOnly === true; + if (cookieOnly) { + const ref = await secrets.create({ purpose: "site_cookie", plaintext: cookie, label: `site_account:${acc.label}` }); + const info = { ...(acc.user_info as Record) }; + delete info.saveCookieOnly; + await db + .updateTable("site_account") + .set({ + auth_kind: "cookie", + secret_ref: ref, + session_secret_ref: null, + totp_secret_ref: null, + user_info: jsonb(info), + }) + .where("id", "=", acc.id) + .execute(); + for (const r of [acc.secret_ref, acc.session_secret_ref, acc.totp_secret_ref]) { + if (r) await secrets.remove(r).catch(() => {}); + } + acc.auth_kind = "cookie"; + acc.secret_ref = ref; + acc.session_secret_ref = null; + acc.totp_secret_ref = null; + return; + } + if (acc.session_secret_ref) { + await secrets.replace(acc.session_secret_ref, cookie); + return; + } + const ref = await secrets.create({ + purpose: "site_cookie", + plaintext: cookie, + label: `site_account:${acc.label}:session`, + }); + await db.updateTable("site_account").set({ session_secret_ref: ref }).where("id", "=", acc.id).execute(); + acc.session_secret_ref = ref; +} + +async function markSuccess( + rt: SourcesRuntime, + acc: SiteAccountRow, + site: SiteRow, + info: SiteUserInfo | null, + actor: Actor, + note?: string, +): Promise { + const now = rt.now(); + const data = info ? userDataOf(info) : null; + await rt.ctx.db.transaction().execute(async (trx) => { + await trx + .updateTable("site_account") + .set({ + status: "active", + last_login_at: now, + last_verified_at: now, + last_error: null, + ...(data ? { user_data: jsonb(data), user_data_at: now } : {}), + }) + .where("id", "=", acc.id) + .execute(); + if (info && data) await upsertUserDataDay(trx, acc.id, now, info, data); + await trx + .updateTable("site") + .set({ health_status: "unknown" }) + .where("id", "=", site.id) + .where("health_status", "=", "failing") + .execute(); + const by = `${actor.type}:${actor.id ?? ""}`; + await closeDecisions(rt.ctx, trx, `site-login:${acc.id}`, "resolved", { by: "login_refreshed" }, by); + await closeDecisions(rt.ctx, trx, `site-auth:${site.id}`, "resolved", { by: "login_refreshed" }, by); + await audit( + trx, + actor, + "site_account.login_refreshed", + { type: "site_account", id: acc.id }, + { + siteId: site.id, + ok: true, + userData: !!data, + }, + ); + }); + return { + siteId: site.id, + accountId: acc.id, + ok: true, + status: "active", + reason: null, + message: note ?? null, + userData: data, + decisionId: null, + }; +} + +async function upsertUserDataDay(trx: Exec, accountId: string, now: Date, info: SiteUserInfo, d: SiteAccountUserData) { + const { + messageUnreadContents: _m, + seedingInfo: _s, + raw: _r, + ...rest + } = info as SiteUserInfo & { messageUnreadContents?: unknown; seedingInfo?: unknown }; + await sql` + insert into site_user_data (site_account_id, day, upload, download, ratio, bonus, seeding, seeding_size, leeching, + user_level, vip, message_unread, join_at, raw, captured_at) + values (${accountId}, ${localDay(now)}, ${Math.round(d.upload)}, ${Math.round(d.download)}, ${d.ratio}, ${d.bonus}, + ${d.seeding}, ${Math.round(d.seedingSize)}, ${d.leeching}, ${d.userLevel}, ${d.vip}, ${d.messageUnread}, + ${d.joinAt}, ${jsonb(rest)}::jsonb, ${now}) + on conflict (site_account_id, day) do update set + upload = excluded.upload, download = excluded.download, ratio = excluded.ratio, bonus = excluded.bonus, + seeding = excluded.seeding, seeding_size = excluded.seeding_size, leeching = excluded.leeching, + user_level = excluded.user_level, vip = excluded.vip, message_unread = excluded.message_unread, + join_at = excluded.join_at, raw = excluded.raw, captured_at = excluded.captured_at`.execute(trx); +} + +const DECISION_TEXT: Record string; summary: string }> = { + challenge: { + title: (s) => `站点「${s}」登录需要人工验证`, + summary: "登录页要求验证码 / 二次验证,无法自动登录。改用 Cookie 登录,或填写 TOTP 密钥后重试", + }, + invalid: { + title: (s) => `站点「${s}」登录失败`, + summary: "用户名或密码错误,或账号被封禁 / 未确认。更新凭证后重新刷新登录", + }, + expired: { + title: (s) => `站点「${s}」登录失效`, + summary: "凭证已失效。更新凭证后重新刷新登录", + }, +}; + +async function markFailure( + rt: SourcesRuntime, + acc: SiteAccountRow, + site: SiteRow, + reason: SiteLoginFailure, + message: string, + actor: Actor, +): Promise { + const now = rt.now(); + const lastError = { reason, message: message.slice(0, 300), at: now.toISOString() }; + const status = + reason === "network" || reason === "unsupported" + ? null + : reason === "auth_expired" && acc.auth_kind === "api_key" + ? "invalid" + : (FAIL_STATUS[reason] ?? "expired"); + let decisionId: string | null = null; + await rt.ctx.db.transaction().execute(async (trx) => { + await trx + .updateTable("site_account") + .set({ last_error: jsonb(lastError), ...(status ? { status, last_verified_at: now } : {}) }) + .where("id", "=", acc.id) + .execute(); + if (status) { + const text = DECISION_TEXT[status]!; + const summary = + reason === "challenge" + ? "站点返回 Cloudflare 挑战页,无法自动登录。改用 Cookie 登录" + : status === "expired" || + reason === "account_blocked" || + reason === "bad_credentials" || + reason === "auth_expired" + ? message + : text.summary; + const options: DecisionOption[] = [ + { + key: "use_cookie", + label: "改用 Cookie", + input: { siteId: site.id, accountId: acc.id, credential: "cookie" }, + }, + ...(status === "challenge" ? [{ key: "retry", label: "重试登录", input: { accountId: acc.id } }] : []), + { key: "dismiss", label: "忽略" }, + ]; + const d = await openDecision(rt.ctx, trx, { + kind: "source_unreachable", + severity: "warning", + title: text.title(site.name), + summary, + subjectType: "site_account", + subjectId: acc.id, + dedupeKey: `site-login:${acc.id}`, + evidence: { siteId: site.id, accountId: acc.id, reason, authKind: acc.auth_kind }, + options, + }); + decisionId = d.id; + } + await audit( + trx, + actor, + "site_account.login_refreshed", + { type: "site_account", id: acc.id }, + { + siteId: site.id, + ok: false, + reason, + }, + ); + }); + return { + siteId: site.id, + accountId: acc.id, + ok: false, + status: status ?? acc.status, + reason, + message, + userData: null, + decisionId, + }; +} + +/** 请求遇 auth_expired:有密码的账号 10 分钟内最多自动重登一次 */ +export async function enqueueRelogin(rt: SourcesRuntime, siteId: string): Promise { + const rows = await rt.ctx.db + .selectFrom("site_account") + .select(["id"]) + .where("site_id", "=", siteId) + .where("enabled", "=", true) + .where("auth_kind", "=", "password") + .where("secret_ref", "is not", null) + .execute(); + const slot = Math.floor(rt.now().getTime() / 600_000); + for (const r of rows) { + await rt.ctx.engine.enqueue( + "refreshSiteAccount", + { accountId: r.id, mode: "login" }, + { idempotencyKey: `refreshSiteAccount:relogin:${r.id}:${slot}`, traceId: randomUUID() }, + ); + } + return rows.length; +} + +// ── 健康检测 ── + +const HEALTH_OF: Record = { + ok: "healthy", + auth_missing: "failing", + auth_expired: "failing", + auth_invalid: "failing", + rate_limited: "degraded", + unreachable: "degraded", + site_maintenance: "degraded", + parse_error: "degraded", +}; + +/** 首页连通 + 登录态(≤ 20 s,走站点限流);写 site.last_health* 与 health_status(已关闭的站点保持 disabled) */ +export async function checkSiteHealth( + rt: SourcesRuntime, + site: SiteRow, + actor: Actor = SYSTEM, +): Promise { + const now = rt.now(); + let ok = false; + let status = "unreachable"; + let latencyMs: number | null = null; + let message: string | null = null; + const def = getSiteDefinition(site.key); + if (!def) { + message = "站点没有适配器定义,无法检测"; + status = "unsupported"; + } else { + await rt.siteDefaults(); + const h = await rt.siteAdapter(site).healthCheck({ signal: AbortSignal.timeout(20_000) }); + ok = h.ok; + status = h.status; + latencyMs = h.latencyMs; + message = h.message ?? null; + } + const loggedIn = ok ? true : status.startsWith("auth_") ? false : null; + const healthStatus: SiteHealth = !site.enabled + ? "disabled" + : def + ? (HEALTH_OF[status] ?? "degraded") + : site.health_status; + const last = { ok, status, latencyMs, message }; + await rt.ctx.db.transaction().execute(async (trx) => { + await trx + .updateTable("site") + .set({ last_health: jsonb(last), last_health_at: now, health_status: healthStatus }) + .where("id", "=", site.id) + .execute(); + // 首页显示已登录:顺手记上次登录 + if (ok) { + await trx + .updateTable("site_account") + .set({ last_login_at: now }) + .where("site_id", "=", site.id) + .where("enabled", "=", true) + .where("status", "=", "active") + .execute(); + } + if (actor.type !== "system") { + await audit(trx, actor, "site.health_checked", { type: "site", id: site.id }, { ok, status }); + } + }); + return { siteId: site.id, checkedAt: now.toISOString(), loggedIn, healthStatus, ...last }; +} + +// ── 用户数据历史 ── + +export async function userDataHistory( + rt: SourcesRuntime, + siteId: string, + q: { accountId?: string | undefined; days: number }, +): Promise { + const since = localDay(new Date(rt.now().getTime() - (q.days - 1) * 86_400_000)); + let query = rt.ctx.db + .selectFrom("site_user_data as d") + .innerJoin("site_account as a", "a.id", "d.site_account_id") + .selectAll("d") + .where("a.site_id", "=", siteId) + .where("d.day", ">=", since); + if (q.accountId) query = query.where("d.site_account_id", "=", q.accountId); + const rows = await query.orderBy("d.day").orderBy("d.site_account_id").execute(); + return rows.map((r) => ({ + siteAccountId: r.site_account_id, + day: r.day, + upload: r.upload, + download: r.download, + ratio: r.ratio, + bonus: r.bonus, + seeding: r.seeding, + seedingSize: r.seeding_size, + leeching: r.leeching, + userLevel: r.user_level, + vip: r.vip, + messageUnread: r.message_unread, + capturedAt: r.captured_at.toISOString(), + })); +} + +// ── 删除 ── + +/** + * 删除站点:停用(资源标 site_disabled、Feed 停用)→ 删账号与全部凭证 → 软删除站点行(key 改名,可再次登记同一定义)。 + * source_release 外键 restrict,资源和下载任务都保留。 + */ +export async function deleteSite(rt: SourcesRuntime, site: SiteRow, actor: Actor): Promise { + const { db, secrets } = rt.ctx; + const refs: string[] = []; + await db.transaction().execute(async (trx) => { + const accounts = await trx.selectFrom("site_account").selectAll().where("site_id", "=", site.id).execute(); + for (const a of accounts) { + for (const r of [a.secret_ref, a.session_secret_ref, a.totp_secret_ref, a.passkey_secret_ref]) + if (r) refs.push(r); + } + await trx + .updateTable("source_release") + .set({ availability: "site_disabled" }) + .where("site_id", "=", site.id) + .where("availability", "=", "active") + .execute(); + await trx + .updateTable("feed_source") + .set({ enabled: false, health_status: "disabled", site_account_id: null }) + .where("site_id", "=", site.id) + .execute(); + const by = `${actor.type}:${actor.id ?? ""}`; + for (const a of accounts) { + await closeDecisions(rt.ctx, trx, `site-login:${a.id}`, "dismissed", { by: "site_deleted" }, by); + } + await closeDecisions(rt.ctx, trx, `site-auth:${site.id}`, "dismissed", { by: "site_deleted" }, by); + await trx.deleteFrom("site_account").where("site_id", "=", site.id).execute(); + await trx + .updateTable("site") + .set({ + enabled: false, + health_status: "disabled", + snapshot_enabled: false, + deleted_at: rt.now(), + key: `${site.key}-deleted-${Date.now().toString(36)}`, + }) + .where("id", "=", site.id) + .execute(); + await audit( + trx, + actor, + "site.deleted", + { type: "site", id: site.id }, + { key: site.key, accounts: accounts.length }, + ); + }); + for (const r of refs) await secrets.remove(r).catch(() => {}); +} + +export async function deleteSiteAccount(rt: SourcesRuntime, acc: SiteAccountRow, actor: Actor): Promise { + const { db, secrets } = rt.ctx; + await db.transaction().execute(async (trx) => { + await closeDecisions( + rt.ctx, + trx, + `site-login:${acc.id}`, + "dismissed", + { by: "account_deleted" }, + `${actor.type}:${actor.id ?? ""}`, + ); + await trx.deleteFrom("site_account").where("id", "=", acc.id).execute(); + await audit(trx, actor, "site_account.deleted", { type: "site_account", id: acc.id }, { siteId: acc.site_id }); + }); + for (const r of [acc.secret_ref, acc.session_secret_ref, acc.totp_secret_ref, acc.passkey_secret_ref]) { + if (r) await secrets.remove(r).catch(() => {}); + } +} + +// ── 定时任务 ── + +/** sites.userData:启用站点里登录状态正常的账号逐个入队刷新用户数据 */ +export async function tickSiteUserData(rt: SourcesRuntime, slot: string): Promise { + const rows = await rt.ctx.db + .selectFrom("site_account as a") + .innerJoin("site as s", "s.id", "a.site_id") + .select(["a.id"]) + .where("s.enabled", "=", true) + .where("s.deleted_at", "is", null) + .where("a.enabled", "=", true) + .where("a.status", "=", "active") + .where("a.auth_kind", "!=", "passkey") + .execute(); + for (const r of rows) { + await rt.ctx.engine.enqueue( + "refreshSiteAccount", + { accountId: r.id, mode: "userData" }, + { idempotencyKey: `refreshSiteAccount:userData:${r.id}:${slot}`, traceId: randomUUID() }, + ); + } + return rows.length; +} + +/** sites.healthCheck:启用站点逐个检测(同步,走站点限流) */ +export async function tickSiteHealth(rt: SourcesRuntime): Promise { + const sites = await rt.ctx.db + .selectFrom("site") + .selectAll() + .where("enabled", "=", true) + .where("deleted_at", "is", null) + .where("adapter", "!=", "generic_rss") + .execute(); + let n = 0; + for (const s of sites) { + try { + await checkSiteHealth(rt, s); + n++; + } catch (err) { + rt.ctx.logger.warn({ siteId: s.id, err: { message: (err as Error).message } }, "站点健康检测失败"); + } + } + return n; +} diff --git a/apps/server/src/modules/sources-identity/service/snapshot.ts b/apps/server/src/modules/sources-identity/service/snapshot.ts index cb7c465e..5217798b 100644 --- a/apps/server/src/modules/sources-identity/service/snapshot.ts +++ b/apps/server/src/modules/sources-identity/service/snapshot.ts @@ -5,6 +5,7 @@ import { parseRelease } from "@umt/domain"; import { siteReleaseToSourceRelease } from "@umt/mapping"; import { jsonb } from "../rows.ts"; import type { SourcesRuntime } from "../runtime.ts"; +import { enqueueRelogin } from "./account.ts"; import { type Exec, openDecision } from "./common.ts"; // 站点快照(周期列表)、补漏(Feed 疑似缺口,有页数预算)、实时补查(工作台搜索)。 @@ -92,6 +93,8 @@ async function onAuthError(rt: SourcesRuntime, trx: Exec, site: SiteRow, e: Site evidence: { siteId: site.id, kind: e.kind, httpStatus: e.httpStatus ?? null }, options: [{ key: "reset", label: "已更新凭证,恢复访问" }], }); + // 有密码的账号:自动重登一次(同账号 10 分钟内最多一次),成功后关闭本待处理并复位站点健康 + await enqueueRelogin(rt, site.id); } export async function snapshotSite(rt: SourcesRuntime, input: SnapshotInput): Promise { diff --git a/apps/server/src/modules/sources-identity/tasks.ts b/apps/server/src/modules/sources-identity/tasks.ts index 0ab662be..4589fcb8 100644 --- a/apps/server/src/modules/sources-identity/tasks.ts +++ b/apps/server/src/modules/sources-identity/tasks.ts @@ -1,6 +1,7 @@ import { defineTask, type OutboxRouteFn, type TaskRegistry } from "@umt/tasks"; import { z } from "zod"; import type { SourcesRuntime } from "./runtime.ts"; +import { refreshSiteAccount } from "./service/account.ts"; import { ingestFeed } from "./service/feeds.ts"; import { refreshMetadata } from "./service/metadata.ts"; import { resolveRelease } from "./service/resolve.ts"; @@ -76,6 +77,23 @@ export function registerSourcesTasks(registry: TaskRegistry, rt: () => SourcesRu snapshotSite(rt(), { siteId: t.input.siteId, kind: t.input.kind, maxPages: t.input.maxPages }), ), }), + defineTask({ + name: "refreshSiteAccount", + version: 1, + inputSchema: z.object({ accountId: id, mode: z.enum(["login", "userData"]).default("login") }), + // 网络失败只记 last_error 不抛错;不自动重试(站点登录频繁重试有封号风险) + retryPolicy: { maximumAttempts: 1 }, + handler: (t) => + t.step.run("refresh", async () => { + const exists = await rt() + .ctx.db.selectFrom("site_account") + .select("id") + .where("id", "=", t.input.accountId) + .executeTakeFirst(); + if (!exists) return { skipped: "not_found" }; + return refreshSiteAccount(rt(), t.input.accountId, { mode: t.input.mode }); + }), + }), ); } diff --git a/apps/server/test/auth-settings/admin-api.test.ts b/apps/server/test/auth-settings/admin-api.test.ts index 68995cde..3b87d28c 100644 --- a/apps/server/test/auth-settings/admin-api.test.ts +++ b/apps/server/test/auth-settings/admin-api.test.ts @@ -238,43 +238,6 @@ describe.skipIf(!hasPg)("W2-A 管理 API(真实 PG)", () => { expect(v.verifyDetail.code).toBe("ENOENT"); expect((await admin("DELETE", `/path-mappings/${m.id}`)).status).toBe(200); }); - - it("PUT /sites/{id}/credentials:新建/替换账号凭证,不回显,旧 secret 删除;未知站点 404", async () => { - const site = await sql<{ id: string }>`insert into site (key, name, adapter, base_url) - values ('audiences', '观众', 'nexusphp', 'https://audiences.me') returning id`.execute(h.db); - const siteId = site.rows[0]!.id; - const r1 = await json( - await admin("PUT", `/sites/${siteId}/credentials`, { - authKind: "cookie", - credential: "c_secure_uid=1; pass=abc", - }), - ); - expect(r1.data).toMatchObject({ created: true, label: "default", authKind: "cookie" }); - expect(JSON.stringify(r1)).not.toContain("pass=abc"); - expect(await h.ctx.secrets.resolve(r1.data.secretRef)).toBe("c_secure_uid=1; pass=abc"); - const r2 = await json( - await admin("PUT", `/sites/${siteId}/credentials`, { - authKind: "password", - credential: "pw-123456", - username: "me", - }), - ); - expect(r2.data).toMatchObject({ created: false, siteAccountId: r1.data.siteAccountId }); - await expect(h.ctx.secrets.resolve(r1.data.secretRef)).rejects.toThrow(/不存在/); - const acc = await sql<{ auth_kind: string; user_info: { username?: string } }>` - select auth_kind, user_info from site_account where id = ${r1.data.siteAccountId}`.execute(h.db); - expect(acc.rows[0]).toMatchObject({ auth_kind: "password", user_info: { username: "me" } }); - expect( - ( - await admin("PUT", "/sites/00000000-0000-4000-8000-000000000000/credentials", { - authKind: "cookie", - credential: "x", - }) - ).status, - ).toBe(404); - noLeak("pass=abc"); - noLeak("pw-123456"); - }); }); describe("审计查询", () => { @@ -289,11 +252,9 @@ describe.skipIf(!hasPg)("W2-A 管理 API(真实 PG)", () => { const ids = [...p1.data.items, ...p2.data.items].map((e: { id: string }) => e.id); expect(new Set(ids).size).toBe(6); expect(p1.data.items[0].at >= p2.data.items[0].at).toBe(true); - const t = await json(await admin("GET", "/audit-events?targetType=site_account")); - expect(t.data.items.map((e: { action: string }) => e.action)).toEqual([ - "site_account.credentials_set", - "site_account.credentials_set", - ]); + const t = await json(await admin("GET", "/audit-events?targetType=api_key&limit=200")); + expect(t.data.items.length).toBeGreaterThan(0); + expect(t.data.items.every((e: { action: string }) => e.action.startsWith("api_key."))).toBe(true); }); }); diff --git a/apps/server/test/sources-identity/harness.ts b/apps/server/test/sources-identity/harness.ts index 1c54ca0f..c83a35e4 100644 --- a/apps/server/test/sources-identity/harness.ts +++ b/apps/server/test/sources-identity/harness.ts @@ -11,6 +11,7 @@ import { EventBus } from "../../src/http/sse.ts"; import { createSourcesIdentityModule } from "../../src/modules/sources-identity/index.ts"; import type { RuntimeOverrides } from "../../src/modules/sources-identity/runtime.ts"; import type { SecretStore, ServerContext, SettingsReader } from "../../src/modules/types.ts"; +import { withSchemaLock } from "../auth-settings/harness.ts"; import { DEV_KEY, memoryLogger, memoryStore } from "../helpers.ts"; export const READ_KEY = "w2b-read-key"; @@ -141,7 +142,8 @@ export async function createHarness(name: string, opts: HarnessOptions = {}): Pr pool: new pg.Pool({ connectionString: PG_URL, max: 8, options: `-c search_path=${schema},public` }), }), }) as unknown as Kysely; - await migrateToLatest(db, { migrationTableSchema: schema }); + // 与其他测试进程共用 PG:迁移内省会看到别处正在删除的 schema,用同一把咨询锁排队并重试 + await withSchemaLock(() => migrateToLatest(db, { migrationTableSchema: schema })); const { logger } = memoryLogger(); const engine = fakeEngine(); @@ -206,7 +208,7 @@ export async function createHarness(name: string, opts: HarnessOptions = {}): Pr }, async close() { await db.destroy(); - await sql.raw(`drop schema if exists ${schema} cascade`).execute(admin); + await withSchemaLock(() => sql.raw(`drop schema if exists ${schema} cascade`).execute(admin)); await admin.destroy(); }, }; diff --git a/apps/server/test/sources-identity/mteam-tasks.test.ts b/apps/server/test/sources-identity/mteam-tasks.test.ts index 39583ecc..ec3c2f90 100644 --- a/apps/server/test/sources-identity/mteam-tasks.test.ts +++ b/apps/server/test/sources-identity/mteam-tasks.test.ts @@ -69,12 +69,13 @@ describe.skipIf(!up)("M-Team 快照与任务注册", () => { api.setFailMode(null); }); - it("模块注册 4 个任务与 3 条 outbox 路由;任务处理器经 step.run 调服务", async () => { + it("模块注册 6 个任务与 3 条 outbox 路由;任务处理器经 step.run 调服务", async () => { const reg = new TaskRegistry(); h.module.registerTasks(reg, h.ctx); expect(reg.names()).toEqual([ "ingestFeed", "refreshMetadata", + "refreshSiteAccount", "resolveObservation", "searchSubscription", "snapshotSite", diff --git a/apps/server/test/sources-identity/site-center.test.ts b/apps/server/test/sources-identity/site-center.test.ts new file mode 100644 index 00000000..ad1e5911 --- /dev/null +++ b/apps/server/test/sources-identity/site-center.test.ts @@ -0,0 +1,328 @@ +import { createMockMteamApi, createMockNexusPhpSite, type MockMteamApi, type MockNexusPhpSite } from "@umt/testkit"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { scheduleRegistry } from "../../src/modules/schedules/registry.ts"; +import { refreshSiteAccount, tickSiteUserData } from "../../src/modules/sources-identity/service/account.ts"; +import { createHarness, type Harness, pgAvailable } from "./harness.ts"; + +// W9-A 站点中心:凭证(账密 / Cookie / API Key)、刷新登录、用户数据与历史、健康检测、删除、定时任务登记。 + +const up = await pgAvailable(); +const TiB = 1024 ** 4; + +// biome-ignore lint/suspicious/noExplicitAny: 响应体断言 +const json = async (r: Response): Promise => { + const body = await r.json(); + if (!r.ok) throw new Error(`${r.status} ${JSON.stringify(body)}`); + return body.data; +}; + +describe.skipIf(!up)("站点中心(mock NexusPHP)", () => { + let h: Harness; + let mock: MockNexusPhpSite; + let siteId: string; + + const put = (body: unknown) => h.req(`/sites/${siteId}/credentials`, { method: "PUT", body }); + const refresh = async (accountId?: string) => + ( + await json( + await h.req(`/sites/${siteId}/refresh-login`, { body: { wait: true, ...(accountId ? { accountId } : {}) } }), + ) + ).result; + const account = (label = "default") => + h.db + .selectFrom("site_account") + .selectAll() + .where("site_id", "=", siteId) + .where("label", "=", label) + .executeTakeFirstOrThrow(); + const openDecisions = (accountId: string) => + h.db + .selectFrom("decision") + .selectAll() + .where("dedupe_key", "=", `site-login:${accountId}`) + .where("status", "=", "open") + .execute(); + + beforeAll(async () => { + mock = await createMockNexusPhpSite({ username: "alice", password: "s3cret-pw" }); + h = await createHarness("sitecenter"); + const site = await json( + await h.req("/sites", { + body: { + key: "audiences", + name: "观众", + adapter: "nexusphp", + baseUrl: `${mock.baseUrl}/`, + rateLimit: { minIntervalMs: 0, concurrency: 10 }, + }, + }), + ); + siteId = site.id; + expect(site).toMatchObject({ proxy: false, lastHealth: null, lastHealthAt: null }); + }); + afterAll(async () => { + await h?.close(); + await mock?.close(); + }); + + it("账密登录成功:会话 Cookie 存 secret、状态 active、用户数据写入当日历史;密码不回显", async () => { + const r = await json(await put({ kind: "password", username: "alice", password: "s3cret-pw" })); + expect(r).toMatchObject({ created: true, authKind: "password", label: "default" }); + expect(JSON.stringify(r)).not.toContain("s3cret-pw"); + const res = await refresh(); + expect(res).toMatchObject({ ok: true, status: "active", reason: null }); + expect(res.userData).toMatchObject({ + username: "mockuser", + userLevel: "Elite User", + upload: 12.5 * TiB, + ratio: 20.48, + seeding: 31, + vip: null, + }); + const acc = await account(); + expect(acc.status).toBe("active"); + expect(acc.last_login_at).toBeTruthy(); + expect(acc.session_secret_ref).toBeTruthy(); + expect(h.secrets.plain.get(acc.session_secret_ref!)).toBe( + mock.cookie + .split(";") + .map((s) => s.trim()) + .join("; "), + ); + expect(acc.login_name).toBe("alice"); + const hist = await json(await h.req(`/sites/${siteId}/user-data?days=30`)); + expect(hist.items).toHaveLength(1); + expect(hist.items[0]).toMatchObject({ siteAccountId: acc.id, upload: 12.5 * TiB, seeding: 31 }); + // 同日再刷新:upsert 不新增行 + await refresh(); + expect((await json(await h.req(`/sites/${siteId}/user-data`))).items).toHaveLength(1); + // 列表带账号摘要,secret 引用不外露 + const list = await json(await h.req("/sites?include=accounts")); + const item = list.items.find((s: { id: string }) => s.id === siteId); + expect(item.accountStatus).toBe("active"); + expect(item.accounts[0]).toMatchObject({ hasSession: true, hasTotp: false, loginName: "alice", status: "active" }); + expect(item.accounts[0].userData.upload).toBe(12.5 * TiB); + expect(JSON.stringify(list)).not.toMatch(/sessionSecretRef|s3cret-pw|mock-session-secret/); + // 会话 Cookie 给适配器用 + const page = await h.rt + .siteAdapter(await h.db.selectFrom("site").selectAll().where("id", "=", siteId).executeTakeFirstOrThrow()) + .list(1); + expect(page.items.length).toBeGreaterThan(0); + }); + + it("密码错:status invalid + 待处理(无重试选项);更新凭证后回到 unknown 并关闭待处理", async () => { + await json(await put({ kind: "password", label: "bad", username: "alice", password: "wrong" })); + const acc = await account("bad"); + const res = await refresh(acc.id); + expect(res).toMatchObject({ ok: false, status: "invalid", reason: "bad_credentials" }); + const ds = await openDecisions(acc.id); + expect(ds).toHaveLength(1); + expect((ds[0]!.options as { key: string }[]).map((o) => o.key)).toEqual(["use_cookie", "dismiss"]); + await json(await put({ kind: "password", label: "bad", username: "alice", password: "s3cret-pw" })); + expect((await account("bad")).status).toBe("unknown"); + expect(await openDecisions(acc.id)).toHaveLength(0); + await json(await h.req(`/site-accounts/${acc.id}`, { method: "DELETE" })); + }); + + it("验证码 / 二次验证:status challenge + 待处理提示改用 Cookie;TOTP 密钥可过二次验证", async () => { + await json(await put({ kind: "password", label: "cap", username: "alice", password: "s3cret-pw" })); + const acc = await account("cap"); + mock.setLoginGuard("captcha"); + const r1 = await refresh(acc.id); + expect(r1).toMatchObject({ ok: false, status: "challenge", reason: "captcha" }); + const [d] = await openDecisions(acc.id); + expect(d!.title).toContain("登录需要人工验证"); + expect(d!.summary).toContain("改用 Cookie"); + expect((d!.options as { key: string }[]).map((o) => o.key)).toEqual(["use_cookie", "retry", "dismiss"]); + + mock.setLoginGuard("two_factor"); + const r2 = await refresh(acc.id); + expect(r2).toMatchObject({ ok: false, status: "challenge", reason: "two_factor" }); + await json( + await put({ + kind: "password", + label: "cap", + username: "alice", + password: "s3cret-pw", + totpSecret: "JBSWY3DPEHPK3PXP", + }), + ); + const r3 = await refresh(acc.id); + expect(r3).toMatchObject({ ok: true, status: "active" }); + expect((await account("cap")).totp_secret_ref).toBeTruthy(); + mock.setLoginGuard("none"); + await json(await h.req(`/site-accounts/${acc.id}`, { method: "DELETE" })); + }); + + it("只换 Cookie:登录成功后凭证类型落为 cookie,密码丢弃", async () => { + await json( + await put({ kind: "password", label: "co", username: "alice", password: "s3cret-pw", saveCookieOnly: true }), + ); + const before = await account("co"); + const res = await refresh(before.id); + expect(res.ok).toBe(true); + const after = await account("co"); + expect(after.auth_kind).toBe("cookie"); + expect(after.session_secret_ref).toBeNull(); + expect(h.secrets.plain.has(before.secret_ref!)).toBe(false); + expect(h.secrets.plain.get(after.secret_ref!)).toContain("c_secure_pass"); + await json(await h.req(`/site-accounts/${after.id}`, { method: "DELETE" })); + }); + + it("Cookie 失效判定:status expired + 待处理「Cookie 已失效」;网络失败不改状态", async () => { + await json(await put({ kind: "cookie", label: "ck", cookie: "c_secure_uid=1; c_secure_pass=stale" })); + const acc = await account("ck"); + const res = await refresh(acc.id); + expect(res).toMatchObject({ ok: false, status: "expired", reason: "auth_expired" }); + expect(res.message).toContain("Cookie 已失效"); + const [d] = await openDecisions(acc.id); + expect(d!.title).toContain("登录失效"); + + await json(await put({ kind: "cookie", label: "ck", cookie: mock.cookie })); + expect((await refresh(acc.id)).ok).toBe(true); + mock.failNext(1, 502); + const net = await refresh(acc.id); + expect(net).toMatchObject({ ok: false, reason: "network", status: "active" }); + expect((await account("ck")).status).toBe("active"); + expect((await account("ck")).last_error).toMatchObject({ reason: "network" }); + await json(await h.req(`/site-accounts/${acc.id}`, { method: "DELETE" })); + }); + + it("refresh-login 缺省异步入队 refreshSiteAccount;定时 sites.user_data 只对 active 账号入队", async () => { + const r = await json(await h.req(`/sites/${siteId}/refresh-login`, { body: {} })); + expect(r.taskId).toBeTruthy(); + expect(r.result).toBeNull(); + expect(h.engine.enqueued.at(-1)).toMatchObject({ task: "refreshSiteAccount", input: { mode: "login" } }); + const n = await tickSiteUserData(h.rt, "slot-1"); + expect(n).toBe(1); + expect(h.engine.enqueued.at(-1)).toMatchObject({ task: "refreshSiteAccount", input: { mode: "userData" } }); + // userData 模式:会话失效时账密账号自动重登 + mock.setLoggedIn(false); + const acc = await account(); + const res = await refreshSiteAccount(h.rt, acc.id, { mode: "userData" }); + expect(res.ok).toBe(true); + expect(mock.loginAttempts).toBeGreaterThan(0); + }); + + it("健康检测:同步返回结构化结果并写站点健康列;未登录判 failing", async () => { + const r = await json(await h.req(`/sites/${siteId}/health-check`, { body: undefined, method: "POST" })); + expect(r).toMatchObject({ siteId, ok: true, status: "ok", loggedIn: true, healthStatus: "healthy" }); + const site = await json(await h.req(`/sites/${siteId}`)); + expect(site.lastHealth).toMatchObject({ ok: true, status: "ok" }); + expect(site.lastHealthAt).toBeTruthy(); + expect(site.healthStatus).toBe("healthy"); + mock.setLoggedIn(false); + const bad = await json(await h.req(`/sites/${siteId}/health-check`, { method: "POST" })); + expect(bad).toMatchObject({ ok: false, loggedIn: false, healthStatus: "failing" }); + mock.setLoggedIn(true); + }); + + it("删除站点:账号与凭证删除,资源保留并标站点已关闭;同一定义可再登记", async () => { + const rel = await h.db + .insertInto("source_release") + .values({ site_id: siteId, site_release_id: "x1", title: "Some.Movie.2024.2160p" }) + .returning("id") + .executeTakeFirstOrThrow(); + const accs = await h.db.selectFrom("site_account").selectAll().where("site_id", "=", siteId).execute(); + const refs = accs.flatMap((a) => [a.secret_ref, a.session_secret_ref]).filter(Boolean) as string[]; + expect(refs.length).toBeGreaterThan(0); + await json(await h.req(`/sites/${siteId}`, { method: "DELETE" })); + expect((await h.req(`/sites/${siteId}`)).status).toBe(404); + const release = await h.db + .selectFrom("source_release") + .selectAll() + .where("id", "=", rel.id) + .executeTakeFirstOrThrow(); + expect(release.availability).toBe("site_disabled"); + expect(await h.db.selectFrom("site_account").selectAll().where("site_id", "=", siteId).execute()).toHaveLength(0); + for (const r of refs) expect(h.secrets.plain.has(r)).toBe(false); + const list = await json(await h.req("/sites")); + expect(list.items.some((s: { id: string }) => s.id === siteId)).toBe(false); + const again = await json( + await h.req("/sites", { + body: { key: "audiences", name: "观众", adapter: "nexusphp", baseUrl: `${mock.baseUrl}/` }, + }), + ); + expect(again.id).not.toBe(siteId); + }); + + it("定时任务登记:sites.user_data 6h(一个周期后首跑)、sites.health_check 1h", async () => { + await h.module.start?.(h.ctx); + const reg = scheduleRegistry(h.ctx); + expect(reg.get("sites.user_data")?.defaults).toMatchObject({ + group: "collect", + schedule: { type: "interval", everyMs: 6 * 3_600_000 }, + firstRun: "after_interval", + }); + expect(reg.get("sites.health_check")?.defaults).toMatchObject({ + schedule: { type: "interval", everyMs: 3_600_000 }, + }); + await h.module.stop?.(); + }); +}); + +describe.skipIf(!up)("站点中心(mock M-Team API Key)", () => { + let h: Harness; + let api: MockMteamApi; + let siteId: string; + + beforeAll(async () => { + api = await createMockMteamApi(); + h = await createHarness("sitecenter_mt"); + const site = await json( + await h.req("/sites", { + body: { + key: "mteam", + name: "馒头", + adapter: "mteam_api", + baseUrl: "https://kp.m-team.cc/", + rateLimit: { minIntervalMs: 0, concurrency: 10 }, + config: { apiBaseUrl: api.baseUrl }, + }, + }), + ); + siteId = site.id; + }); + afterAll(async () => { + await h?.close(); + await api?.close(); + }); + + it("API Key 校验:有效 → active + 用户数据;无效 → invalid;账密不支持", async () => { + await json( + await h.req(`/sites/${siteId}/credentials`, { method: "PUT", body: { kind: "api_key", apiKey: api.apiKey } }), + ); + const ok = (await json(await h.req(`/sites/${siteId}/refresh-login`, { body: { wait: true } }))).result; + expect(ok).toMatchObject({ ok: true, status: "active" }); + expect(ok.userData).toMatchObject({ userId: "10001", username: "mockuser", upload: 1000 }); + + await json( + await h.req(`/sites/${siteId}/credentials`, { method: "PUT", body: { kind: "api_key", apiKey: "wrong-key" } }), + ); + const bad = (await json(await h.req(`/sites/${siteId}/refresh-login`, { body: { wait: true } }))).result; + expect(bad).toMatchObject({ ok: false, status: "invalid", reason: "auth_expired" }); + expect(bad.message).toContain("API Key 无效"); + + await json( + await h.req(`/sites/${siteId}/credentials`, { + method: "PUT", + body: { kind: "password", label: "pw", username: "u", password: "p" }, + }), + ); + const acc = await h.db.selectFrom("site_account").select("id").where("label", "=", "pw").executeTakeFirstOrThrow(); + const pw = (await json(await h.req(`/sites/${siteId}/refresh-login`, { body: { wait: true, accountId: acc.id } }))) + .result; + expect(pw).toMatchObject({ ok: false, reason: "unsupported", status: "unknown" }); + }); + + it("旧格式凭证 {authKind, credential} 仍可用", async () => { + const r = await json( + await h.req(`/sites/${siteId}/credentials`, { + method: "PUT", + body: { label: "legacy", authKind: "passkey", credential: "pk-123" }, + }), + ); + expect(r).toMatchObject({ created: true, authKind: "passkey" }); + expect(JSON.stringify(r)).not.toContain("pk-123"); + }); +}); diff --git a/apps/web/app/components/workbench/release-fields.ts b/apps/web/app/components/workbench/release-fields.ts index d0c0a6dc..2a09afa2 100644 --- a/apps/web/app/components/workbench/release-fields.ts +++ b/apps/web/app/components/workbench/release-fields.ts @@ -76,6 +76,7 @@ export const ACCOUNT_STATUS: Record