feat(sources): site center backend (credentials, refresh login, user data, health check, delete)

PUT /sites/{id}/credentials by kind (moved from auth-settings, legacy body
accepted), POST refresh-login (async task or wait), POST health-check,
GET user-data history, DELETE /sites/{id} (soft delete, releases kept),
DELETE /site-accounts/{id}, GET /sites?include=accounts; refreshSiteAccount
task with login-failure decisions and auto re-login on auth_expired;
scheduled jobs sites.user_data (6h) and sites.health_check (1h);
settings.sites defaults for rate limit, snapshot interval, UA and proxy.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed 2026-09-28 15:18:40 +08:00
1 parent d6fca25679
commit 05a7706dde
15 files changed
+1602 -179

No files matched your search

@@ -18,10 +18,7 @@ import {
PathMappingCreate, PathMappingCreate,
PathMappingUpdate, PathMappingUpdate,
type SecretPurpose, type SecretPurpose,
SiteCredentialsPut,
SiteCredentialsResult,
} from "@umt/contracts"; } from "@umt/contracts";
import type { DB } from "@umt/db";
import { z } from "zod"; import { z } from "zod";
import { ApiException } from "../../http/errors.ts"; import { ApiException } from "../../http/errors.ts";
import { redact, redactString } from "../../logger.ts"; import { redact, redactString } from "../../logger.ts";
@@ -41,7 +38,7 @@ import {
} from "./endpoint.ts"; } from "./endpoint.ts";
// 实例与路径:library_root / downloader_instance / media_server_instance / path_mapping 的 CRUD + 连通性测试, // 实例与路径:library_root / downloader_instance / media_server_instance / path_mapping 的 CRUD + 连通性测试,
// 以及站点账号凭证录入 PUT /sites/{id}/credentials。凭证明文只在请求里出现,存为 secret 后只回 secretRef。 // 站点账号凭证录入已迁到 sources-identity(PUT /sites/{id}/credentials)。
const OkResult = z.object({ ok: z.literal(true) }); const OkResult = z.object({ ok: z.literal(true) });
const del = (path: string, summary: string, tag: string) => ({ const del = (path: string, summary: string, tag: string) => ({
@@ -470,97 +467,5 @@ export function registerInstanceRoutes(app: ModuleApp, ctx: ServerContext) {
await audit(ctx, actor, "path_mapping.verify", { type: "path_mapping", id: row.id }, { ok }); await audit(ctx, actor, "path_mapping.verify", { type: "path_mapping", id: row.id }, { ok });
return toApi(updated); return toApi(updated);
}); });
// 站点账号凭证 PUT /sites/{id}/credentials 已迁到 sources-identity(W9-A 站点中心,按 kind 判别)
// ── 站点账号凭证 ──
const PURPOSE: Record<SiteCredentialsPut["authKind"], SecretPurpose> = {
cookie: "site_cookie",
passkey: "site_passkey",
api_key: "site_api_key",
password: "site_password",
};
endpoint(
app,
{
method: "put",
path: "/sites/{id}/credentials",
scope: "admin",
summary: "录入站点账号凭证(cookie/passkey/apiKey/密码 → secret_ref,不回显)",
params: IdParam,
body: SiteCredentialsPut,
response: SiteCredentialsResult,
tags: ["sites"],
},
async ({ params, body, actor }) => {
const site = await db().selectFrom("site").select("id").where("id", "=", params.id).executeTakeFirst();
if (!site) throw new ApiException("not_found", "站点不存在");
const newRef = await ctx.secrets.create({ purpose: PURPOSE[body.authKind], plaintext: body.credential });
let oldRef: string | null = null;
let created = false;
let accountId: string;
try {
accountId = await db()
.transaction()
.execute(async (trx) => {
const cur = await trx
.selectFrom("site_account")
.select(["id", "secret_ref", "user_info"])
.where("site_id", "=", params.id)
.where("label", "=", body.label)
.forUpdate()
.executeTakeFirst();
const userInfo = {
...((cur?.user_info as Record<string, unknown> | undefined) ?? {}),
...(body.username ? { username: body.username } : {}),
};
const common: Partial<Record<keyof DB["site_account"], unknown>> = {
auth_kind: body.authKind,
secret_ref: newRef,
status: "unknown",
user_info: JSON.stringify(userInfo),
...(body.enabled !== undefined ? { enabled: body.enabled } : {}),
};
if (cur) {
oldRef = cur.secret_ref;
await trx
.updateTable("site_account")
.set(common as never)
.where("id", "=", cur.id)
.execute();
return cur.id;
}
created = true;
const r = await trx
.insertInto("site_account")
.values({ site_id: params.id, label: body.label, ...common } as never)
.returning("id")
.executeTakeFirstOrThrow();
return r.id;
});
} catch (e) {
await ctx.secrets.remove(newRef).catch(() => {});
throw e;
}
if (oldRef) await ctx.secrets.remove(oldRef).catch(() => {});
await audit(
ctx,
actor,
"site_account.credentials_set",
{ type: "site_account", id: accountId },
{
siteId: params.id,
label: body.label,
authKind: body.authKind,
created,
},
);
return {
siteAccountId: accountId,
siteId: params.id,
label: body.label,
authKind: body.authKind,
secretRef: newRef,
created,
};
},
);
} }
@@ -33,6 +33,8 @@ import {
SiteAccountCreate, SiteAccountCreate,
SiteAccountUpdate, SiteAccountUpdate,
SiteCreate, SiteCreate,
SiteListItem,
SiteListQuery,
SiteUpdate, SiteUpdate,
SourceRelease, SourceRelease,
TitleAlias, TitleAlias,
@@ -96,7 +98,15 @@ const TD = ["decisions"];
export const D = { export const D = {
sites: { sites: {
list: list("/sites", "站点列表", Site, {}, TS), list: {
method: "get" as const,
path: "/sites",
scope: "read" as const,
summary: "站点列表(include=accounts 附带账号与整体账号状态)",
tags: TS,
query: SiteListQuery,
response: cursorPage(SiteListItem),
},
get: get("/sites/{id}", "站点详情", Site, TS), get: get("/sites/{id}", "站点详情", Site, TS),
create: { create: {
method: "post" as const, method: "post" as const,
@@ -107,7 +117,7 @@ export const D = {
body: SiteCreate, body: SiteCreate,
response: Site, response: Site,
}, },
update: write("patch", "/sites/{id}", "admin", "修改站点(启停、快照周期、限流)", SiteUpdate, Site, TS), update: write("patch", "/sites/{id}", "admin", "修改站点(启停、快照周期、限流、代理)", SiteUpdate, Site, TS),
}, },
siteAccounts: { siteAccounts: {
list: list("/site-accounts", "站点账号列表", SiteAccount, { siteId: Id.optional() }, TS), list: list("/site-accounts", "站点账号列表", SiteAccount, { siteId: Id.optional() }, TS),
@@ -7,6 +7,7 @@ import { sourcesRoutes } from "./routes/sources.ts";
import { workbenchRoutes } from "./routes/workbench.ts"; import { workbenchRoutes } from "./routes/workbench.ts";
import { type RuntimeOverrides, SourcesRuntime } from "./runtime.ts"; import { type RuntimeOverrides, SourcesRuntime } from "./runtime.ts";
import { readOnlyReconcile, tickFeeds, tickReviews, tickSnapshots } from "./scheduler.ts"; import { readOnlyReconcile, tickFeeds, tickReviews, tickSnapshots } from "./scheduler.ts";
import { tickSiteHealth, tickSiteUserData } from "./service/account.ts";
import { registerSourcesTasks, sourcesOutboxRoutes } from "./tasks.ts"; import { registerSourcesTasks, sourcesOutboxRoutes } from "./tasks.ts";
/** /**
@@ -60,6 +61,32 @@ export function createSourcesIdentityModule(overrides: RuntimeOverrides = {}): S
job("feeds.poll", "Feed 轮询", "扫描到期的 Feed 并入队抓取(每个 Feed 仍按自己的轮询间隔)", tickFeeds); job("feeds.poll", "Feed 轮询", "扫描到期的 Feed 并入队抓取(每个 Feed 仍按自己的轮询间隔)", tickFeeds);
job("sites.snapshot", "站点快照", "扫描到周期的站点并入队快照", tickSnapshots); job("sites.snapshot", "站点快照", "扫描到周期的站点并入队快照", tickSnapshots);
job("identity.review", "待识别复查", "到期的待识别条目重新识别", tickReviews); job("identity.review", "待识别复查", "到期的待识别条目重新识别", tickReviews);
// W9-A 站点中心:用户数据每 6 小时、健康检测每小时(可在定时任务页关闭或改周期)
registerScheduledJob(ctx, {
key: "sites.user_data",
defaults: {
name: "站点数据刷新",
description: "登录状态正常的站点账号刷新上传 / 下载 / 分享率等用户数据;会话失效的账密账号自动重登一次",
group: "collect",
schedule: { type: "interval", everyMs: 6 * 3_600_000 },
firstRun: "after_interval",
},
run: async ({ slot }) => {
if (await readOnlyReconcile(rt)) return { status: "skipped", summary: "只读对账模式" };
return { summary: `${await tickSiteUserData(rt, slot)} 个账号` };
},
});
registerScheduledJob(ctx, {
key: "sites.health_check",
defaults: {
name: "站点健康检测",
description: "已启用站点逐个检测首页连通与登录态,结果显示在站点页",
group: "collect",
schedule: { type: "interval", everyMs: 3_600_000 },
firstRun: "after_interval",
},
run: async () => ({ summary: `${await tickSiteHealth(rt)} 个站点` }),
});
}, },
}; };
} }
@@ -1,3 +1,4 @@
import { randomUUID } from "node:crypto";
import { type Decision, DecisionKind, DecisionStatus, Id, type PoolCandidate } from "@umt/contracts"; import { type Decision, DecisionKind, DecisionStatus, Id, type PoolCandidate } from "@umt/contracts";
import { sql } from "@umt/db"; import { sql } from "@umt/db";
import { z } from "zod"; import { z } from "zod";
@@ -163,6 +164,15 @@ export function decisionRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
); );
} }
} }
// 站点登录需人工验证:"重试登录" = 入队刷新登录(结果另行更新账号状态与待处理)
if (d.dedupe_key?.startsWith("site-login:") && opt?.key === "retry") {
const accountId = d.dedupe_key.slice("site-login:".length);
await rt().ctx.engine.enqueue(
"refreshSiteAccount",
{ accountId, mode: "login" },
{ idempotencyKey: `refreshSiteAccount:decision:${d.id}:${Date.now()}`, traceId: randomUUID() },
);
}
// 上面的执行路径已按 dedupeKey 关闭;其余类型在这里记录处理结果 // 上面的执行路径已按 dedupeKey 关闭;其余类型在这里记录处理结果
const row = await db() const row = await db()
.transaction() .transaction()
@@ -7,13 +7,21 @@ import {
Id, Id,
IdParam, IdParam,
type Observation, type Observation,
OkResult,
ReleaseIdentityAssign, ReleaseIdentityAssign,
type SecretPurpose, type SecretPurpose,
Site, Site,
type SiteAccount, type SiteAccount,
type SiteAdapter, type SiteAdapter,
SiteCredentialsPut,
SiteCredentialsResult,
SiteDefinitionView, SiteDefinitionView,
SiteHealthResult,
SiteRefreshLoginAccepted,
SiteRefreshLoginInput,
SiteSnapshot, SiteSnapshot,
SiteUserDataPoint,
SiteUserDataQuery,
type SourceRelease, type SourceRelease,
} from "@umt/contracts"; } from "@umt/contracts";
import { sql } from "@umt/db"; import { sql } from "@umt/db";
@@ -24,6 +32,16 @@ import { D } from "../defs.ts";
import { route } from "../http.ts"; import { route } from "../http.ts";
import { decodeCursor, feedToApi, jsonb, page, toApi, toApiList } from "../rows.ts"; import { decodeCursor, feedToApi, jsonb, page, toApi, toApiList } from "../rows.ts";
import type { SourcesRuntime } from "../runtime.ts"; import type { SourcesRuntime } from "../runtime.ts";
import {
accountToApi,
checkSiteHealth,
deleteSite,
deleteSiteAccount,
overallAccountStatus,
refreshSiteAccount,
setSiteCredentials,
userDataHistory,
} from "../service/account.ts";
import { audit, closeDecisions } from "../service/common.ts"; import { audit, closeDecisions } from "../service/common.ts";
import { assignReleaseIdentity } from "../service/identity-admin.ts"; import { assignReleaseIdentity } from "../service/identity-admin.ts";
@@ -61,7 +79,7 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
response: z.object({ items: z.array(SiteDefinitionView) }), response: z.object({ items: z.array(SiteDefinitionView) }),
}, },
async () => { async () => {
const rows = await db().selectFrom("site").select(["id", "key"]).execute(); const rows = await db().selectFrom("site").select(["id", "key"]).where("deleted_at", "is", null).execute();
const byKey = new Map(rows.map((r) => [r.key, r.id])); const byKey = new Map(rows.map((r) => [r.key, r.id]));
return { return {
items: listSiteDefinitions().map((d) => ({ items: listSiteDefinitions().map((d) => ({
@@ -86,12 +104,31 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
const rows = await db() const rows = await db()
.selectFrom("site") .selectFrom("site")
.selectAll() .selectAll()
.where("deleted_at", "is", null)
.orderBy("key") .orderBy("key")
.offset(off) .offset(off)
.limit(query.limit + 1) .limit(query.limit + 1)
.execute(); .execute();
const p = page(rows, off, query.limit); const p = page(rows, off, query.limit);
return { items: toApiList<Site>(p.items), nextCursor: p.nextCursor }; const items = toApiList<Record<string, unknown>>(p.items);
if (query.include === "accounts" && p.items.length) {
const accounts = await db()
.selectFrom("site_account")
.selectAll()
.where(
"site_id",
"in",
p.items.map((s) => s.id),
)
.orderBy("created_at")
.execute();
for (const it of items) {
const mine = accounts.filter((a) => a.site_id === it.id);
it.accounts = mine.map(accountToApi);
it.accountStatus = overallAccountStatus(mine);
}
}
return { items, nextCursor: p.nextCursor };
}); });
route(app, D.sites.get, async ({ params }) => toApi(await getSite(params.id))); route(app, D.sites.get, async ({ params }) => toApi(await getSite(params.id)));
route(app, D.sites.create, async ({ body, actor }) => { route(app, D.sites.create, async ({ body, actor }) => {
@@ -115,6 +152,7 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
config: jsonb(body.config ?? {}), config: jsonb(body.config ?? {}),
snapshot_enabled: body.snapshotEnabled ?? false, snapshot_enabled: body.snapshotEnabled ?? false,
snapshot_interval_sec: body.snapshotIntervalSec ?? null, snapshot_interval_sec: body.snapshotIntervalSec ?? null,
proxy: body.proxy ?? false,
}) })
.returningAll() .returningAll()
.executeTakeFirstOrThrow(); .executeTakeFirstOrThrow();
@@ -135,7 +173,15 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
if (body.config !== undefined) patch.config = jsonb(body.config); if (body.config !== undefined) patch.config = jsonb(body.config);
if (body.snapshotEnabled !== undefined) patch.snapshot_enabled = body.snapshotEnabled; if (body.snapshotEnabled !== undefined) patch.snapshot_enabled = body.snapshotEnabled;
if (body.snapshotIntervalSec !== undefined) patch.snapshot_interval_sec = body.snapshotIntervalSec; if (body.snapshotIntervalSec !== undefined) patch.snapshot_interval_sec = body.snapshotIntervalSec;
if (body.enabled !== undefined) return toApi(await setSiteEnabled(params.id, body.enabled, actor)); if (body.proxy !== undefined) patch.proxy = body.proxy;
if (body.enabled !== undefined) {
if (Object.keys(patch).length) {
await db().updateTable("site").set(patch).where("id", "=", params.id).execute();
await audit(db(), actor, "site.updated", { type: "site", id: params.id }, { fields: Object.keys(patch) });
}
return toApi(await setSiteEnabled(params.id, body.enabled, actor));
}
if (!Object.keys(patch).length) return toApi(await getSite(params.id));
const row = await db() const row = await db()
.updateTable("site") .updateTable("site")
.set(patch) .set(patch)
@@ -210,6 +256,132 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
}, },
); );
// ── W9-A 站点中心:删除、健康检测、刷新登录、用户数据、凭证 ──
route(
app,
{
method: "delete",
path: "/sites/{id}",
scope: "admin",
summary: "删除站点:停用后删账号与凭证;已抓到的资源和下载任务保留(标站点已关闭)",
tags: T_SITES,
params: IdParam,
response: OkResult,
},
async ({ params, actor }) => {
const s = await getSite(params.id);
await deleteSite(rt(), s, actor);
return { ok: true as const };
},
);
route(
app,
{
method: "post",
path: "/sites/{id}/health-check",
scope: "admin",
summary: "检测健康:首页连通 + 登录态,同步返回并写站点健康列",
tags: T_SITES,
params: IdParam,
response: SiteHealthResult,
},
async ({ params, actor }) => checkSiteHealth(rt(), await getSite(params.id), actor),
);
route(
app,
{
method: "post",
path: "/sites/{id}/refresh-login",
scope: "admin",
summary: "刷新登录:账密重新登录换 Cookie;Cookie / API Key 校验;成功后拉用户数据",
tags: T_SITES,
params: IdParam,
body: SiteRefreshLoginInput,
response: SiteRefreshLoginAccepted,
},
async ({ params, body, actor }) => {
const s = await getSite(params.id);
let q = db().selectFrom("site_account").select(["id", "enabled", "auth_kind"]).where("site_id", "=", s.id);
q = body.accountId
? q.where("id", "=", body.accountId)
: q.where("enabled", "=", true).where("auth_kind", "!=", "passkey");
const acc = await q.orderBy("created_at").executeTakeFirst();
if (!acc) {
throw new ApiException(
body.accountId ? "not_found" : "precondition_failed",
body.accountId ? "站点账号不存在" : "站点还没有凭证",
);
}
if (body.wait) {
const result = await refreshSiteAccount(rt(), acc.id, {
mode: "login",
actor,
signal: AbortSignal.timeout(30_000),
});
return { taskId: null, result };
}
const r = await rt().ctx.engine.enqueue(
"refreshSiteAccount",
{ accountId: acc.id, mode: "login" },
{
idempotencyKey: `refreshSiteAccount:manual:${acc.id}:${Math.floor(Date.now() / 10_000)}`,
traceId: randomUUID(),
},
);
await audit(db(), actor, "site_account.login_refresh_requested", { type: "site_account", id: acc.id });
return { taskId: r.runId, result: null };
},
);
route(
app,
{
method: "get",
path: "/sites/{id}/user-data",
scope: "read",
summary: "站点账号用户数据按天历史",
tags: T_SITES,
params: IdParam,
query: SiteUserDataQuery,
response: z.object({ items: z.array(SiteUserDataPoint) }),
},
async ({ params, query }) => {
await getSite(params.id);
return { items: await userDataHistory(rt(), params.id, query) };
},
);
route(
app,
{
method: "put",
path: "/sites/{id}/credentials",
scope: "admin",
summary: "设置站点账号凭证(按 kind:password / cookie / api_key / passkey;存为 secret,不回显)",
tags: T_SITES,
params: IdParam,
body: SiteCredentialsPut,
response: SiteCredentialsResult,
},
async ({ params, body, actor }) => setSiteCredentials(rt(), await getSite(params.id), body, actor),
);
route(
app,
{
method: "delete",
path: "/site-accounts/{id}",
scope: "admin",
summary: "删除站点账号与其凭证",
tags: T_SITES,
params: IdParam,
response: OkResult,
},
async ({ params, actor }) => {
const acc = await db().selectFrom("site_account").selectAll().where("id", "=", params.id).executeTakeFirst();
if (!acc) throw new ApiException("not_found", "站点账号不存在");
await deleteSiteAccount(rt(), acc, actor);
return { ok: true as const };
},
);
// ── 站点账号 ── // ── 站点账号 ──
route(app, D.siteAccounts.list, async ({ query }) => { route(app, D.siteAccounts.list, async ({ query }) => {
const off = decodeCursor(query.cursor); const off = decodeCursor(query.cursor);
@@ -221,12 +393,12 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
.limit(query.limit + 1) .limit(query.limit + 1)
.execute(); .execute();
const p = page(rows, off, query.limit); const p = page(rows, off, query.limit);
return { items: toApiList<SiteAccount>(p.items), nextCursor: p.nextCursor }; return { items: p.items.map(accountToApi) as SiteAccount[], nextCursor: p.nextCursor };
}); });
route(app, D.siteAccounts.get, async ({ params }) => { route(app, D.siteAccounts.get, async ({ params }) => {
const row = await db().selectFrom("site_account").selectAll().where("id", "=", params.id).executeTakeFirst(); const row = await db().selectFrom("site_account").selectAll().where("id", "=", params.id).executeTakeFirst();
if (!row) throw new ApiException("not_found", "站点账号不存在"); if (!row) throw new ApiException("not_found", "站点账号不存在");
return toApi(row); return accountToApi(row);
}); });
route(app, D.siteAccounts.create, async ({ body, actor }) => { route(app, D.siteAccounts.create, async ({ body, actor }) => {
await getSite(body.siteId); await getSite(body.siteId);
@@ -248,7 +420,7 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
.returningAll() .returningAll()
.executeTakeFirstOrThrow(); .executeTakeFirstOrThrow();
await audit(db(), actor, "site_account.created", { type: "site_account", id: row.id }, { siteId: body.siteId }); await audit(db(), actor, "site_account.created", { type: "site_account", id: row.id }, { siteId: body.siteId });
return toApi(row); return accountToApi(row);
} catch (e) { } catch (e) {
await rt() await rt()
.ctx.secrets.remove(ref) .ctx.secrets.remove(ref)
@@ -273,8 +445,10 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
label: `site_account:${acc.label}`, label: `site_account:${acc.label}`,
}); });
} }
// 换了凭证:状态回到未验证,关闭"登录失效"待处理,站点健康复位 // 换了凭证:状态回到未验证,清会话 Cookie,关闭"登录失效"待处理,站点健康复位
patch.status = "unknown"; patch.status = "unknown";
patch.last_error = null;
patch.session_secret_ref = null;
} }
const row = await db() const row = await db()
.transaction() .transaction()
@@ -300,6 +474,14 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
{ by: "credential_updated" }, { by: "credential_updated" },
`${actor.type}:${actor.id}`, `${actor.type}:${actor.id}`,
); );
await closeDecisions(
rt().ctx,
trx,
`site-login:${acc.id}`,
"resolved",
{ by: "credential_updated" },
`${actor.type}:${actor.id}`,
);
} }
await audit( await audit(
trx, trx,
@@ -313,7 +495,7 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
); );
return r; return r;
}); });
return toApi(row); return accountToApi(row);
}); });
// ── Feed ── // ── Feed ──
@@ -588,7 +770,12 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
// ── 内部辅助 ── // ── 内部辅助 ──
async function getSite(id: string) { async function getSite(id: string) {
const row = await db().selectFrom("site").selectAll().where("id", "=", id).executeTakeFirst(); const row = await db()
.selectFrom("site")
.selectAll()
.where("id", "=", id)
.where("deleted_at", "is", null)
.executeTakeFirst();
if (!row) throw new ApiException("not_found", "站点不存在"); if (!row) throw new ApiException("not_found", "站点不存在");
return row; return row;
} }
@@ -11,6 +11,7 @@ import {
type SiteRateLimiter, type SiteRateLimiter,
type TmdbClient, type TmdbClient,
} from "@umt/adapters"; } from "@umt/adapters";
import { DEFAULT_SETTINGS, type SitesSettings } from "@umt/contracts";
import type { SiteRow } from "@umt/db"; import type { SiteRow } from "@umt/db";
import type { ServerContext } from "../types.ts"; import type { ServerContext } from "../types.ts";
@@ -40,6 +41,9 @@ export class SourcesRuntime {
readonly rss: RssFetcher; readonly rss: RssFetcher;
private readonly adapters = new Map<string, CachedAdapter>(); private readonly adapters = new Map<string, CachedAdapter>();
private tmdbClient: TmdbClient | null | undefined; private tmdbClient: TmdbClient | null | undefined;
/** 站点默认(settings.sites)缓存:siteAdapter 是同步的,读设置的地方顺手刷新 */
private siteDefaultsCache: SitesSettings = DEFAULT_SETTINGS.sites;
private siteDefaultsAt = 0;
private tmdbSig: string | null = null; private tmdbSig: string | null = null;
constructor( constructor(
@@ -101,17 +105,60 @@ export class SourcesRuntime {
return this.tmdbClient; return this.tmdbClient;
} }
/** 站点默认(settings.sites),读库并刷新缓存;读失败用上次值 */
async siteDefaults(): Promise<SitesSettings> {
try {
this.siteDefaultsCache = { ...DEFAULT_SETTINGS.sites, ...(await this.ctx.settings.get("sites")) };
} catch {
// 保留缓存
}
return this.siteDefaultsCache;
}
/**
* 站点请求用的 fetch:测试注入优先;站点开了"使用代理"且设置了 proxyUrl 时走代理(Deno.createHttpClient)。
*/
siteFetch(site: Pick<SiteRow, "proxy">, defaults: SitesSettings = this.siteDefaultsCache): typeof globalThis.fetch {
if (this.overrides.fetch) return this.overrides.fetch;
if (!site.proxy || !defaults.proxyUrl) return globalThis.fetch;
const key = defaults.proxyUrl;
let client = this.proxyClients.get(key);
if (!client) {
client = Deno.createHttpClient({ proxy: { url: key } });
this.proxyClients.set(key, client);
}
const c = client;
return ((input: RequestInfo | URL, init?: RequestInit) =>
globalThis.fetch(input, { ...init, client: c } as RequestInit)) as typeof globalThis.fetch;
}
private readonly proxyClients = new Map<string, Deno.HttpClient>();
/** /**
* 站点适配器:每站一个实例,列表/搜索/详情/下载共享同一个限流器。 * 站点适配器:每站一个实例,列表/搜索/详情/下载共享同一个限流器。
* 站点限流参数或基址变化时重建(签名比较)。 * 站点限流参数、基址、代理或站点默认变化时重建(签名比较)。
*/ */
siteAdapter(site: Pick<SiteRow, "id" | "key" | "base_url" | "rate_limit" | "config">): SiteAdapter { siteAdapter(
site: Pick<SiteRow, "id" | "key" | "base_url" | "rate_limit" | "config"> & Partial<Pick<SiteRow, "proxy">>,
): SiteAdapter {
const def = getSiteDefinition(site.key); const def = getSiteDefinition(site.key);
if (!def) throw new Error(`站点 ${site.key} 没有随产品发布的适配器定义`); if (!def) throw new Error(`站点 ${site.key} 没有随产品发布的适配器定义`);
const sig = JSON.stringify([site.base_url, site.rate_limit, site.config]); if (Date.now() - this.siteDefaultsAt > 30_000) {
this.siteDefaultsAt = Date.now();
void this.siteDefaults();
}
const defaults = this.siteDefaultsCache;
const proxy = site.proxy ?? false;
const sig = JSON.stringify([
site.base_url,
site.rate_limit,
site.config,
proxy,
proxy ? defaults.proxyUrl : null,
defaults.defaultRateLimit,
]);
const hit = this.adapters.get(site.id); const hit = this.adapters.get(site.id);
if (hit && hit.sig === sig) return hit.adapter; if (hit && hit.sig === sig) return hit.adapter;
const limiter = createSiteRateLimiter(limiterOptions(def, site.rate_limit)); const limiter = createSiteRateLimiter(limiterOptions(def, site.rate_limit, defaults.defaultRateLimit));
const cfg = (site.config ?? {}) as { apiBaseUrl?: string }; const cfg = (site.config ?? {}) as { apiBaseUrl?: string };
const baseUrl = const baseUrl =
def.umt.kind === "mteam" ? cfg.apiBaseUrl : sameUrl(site.base_url, def.domain) ? undefined : site.base_url; def.umt.kind === "mteam" ? cfg.apiBaseUrl : sameUrl(site.base_url, def.domain) ? undefined : site.base_url;
@@ -119,7 +166,7 @@ export class SourcesRuntime {
secrets: this.siteSecrets(site.id), secrets: this.siteSecrets(site.id),
limiter, limiter,
now: this.now, now: this.now,
...(this.overrides.fetch ? { fetch: this.overrides.fetch } : {}), ...(this.overrides.fetch || proxy ? { fetch: this.siteFetch({ proxy }, defaults) } : {}),
...(baseUrl ? { baseUrl } : {}), ...(baseUrl ? { baseUrl } : {}),
}); });
this.adapters.set(site.id, { sig, limiter, adapter }); this.adapters.set(site.id, { sig, limiter, adapter });
@@ -131,24 +178,39 @@ export class SourcesRuntime {
return this.adapters.get(siteId)?.limiter ?? null; return this.adapters.get(siteId)?.limiter ?? null;
} }
/** 站点凭证:取该站第一个启用且未失效的账号,按 auth_kind 对应适配器要求的凭证类型;明文不缓存 */ /**
* 站点凭证:取该站第一个启用且未失效的账号;明文不缓存。
* cookie:Cookie 型账号的 secret,或账密账号登录得到的会话 Cookie;apiKey:API Key 型账号;
* passkey:账号附挂的 passkey 或 passkey 型账号(RSS / 下载用,不看登录状态)。
*/
private siteSecrets(siteId: string): SecretResolver { private siteSecrets(siteId: string): SecretResolver {
const { db, secrets } = this.ctx; const { db, secrets } = this.ctx;
return { return {
async resolve({ kind }) { async resolve({ kind }) {
const want = kind === "cookie" ? ["cookie"] : kind === "apiKey" ? ["api_key"] : ["passkey"]; let q = db
const acc = await db
.selectFrom("site_account") .selectFrom("site_account")
.select(["secret_ref"]) .select(["auth_kind", "secret_ref", "session_secret_ref", "passkey_secret_ref"])
.where("site_id", "=", siteId) .where("site_id", "=", siteId)
.where("enabled", "=", true) .where("enabled", "=", true)
.where("status", "in", ["unknown", "active"]) .orderBy("created_at");
.where("auth_kind", "in", want as never) if (kind !== "passkey") q = q.where("status", "in", ["unknown", "active"]);
.where("secret_ref", "is not", null) const rows = await q.execute();
.orderBy("created_at") for (const r of rows) {
.executeTakeFirst(); const ref =
if (!acc?.secret_ref) return null; kind === "cookie"
return secrets.resolve(acc.secret_ref); ? r.auth_kind === "cookie"
? r.secret_ref
: r.auth_kind === "password"
? r.session_secret_ref
: null
: kind === "apiKey"
? r.auth_kind === "api_key"
? r.secret_ref
: null
: (r.passkey_secret_ref ?? (r.auth_kind === "passkey" ? r.secret_ref : null));
if (ref) return secrets.resolve(ref);
}
return null;
}, },
}; };
} }
@@ -158,11 +220,32 @@ function sameUrl(a: string, b: string): boolean {
return a.replace(/\/+$/, "") === b.replace(/\/+$/, ""); return a.replace(/\/+$/, "") === b.replace(/\/+$/, "");
} }
/** 站点行 rate_limit 优先,缺省用定义里的 umt.rateLimit */ type RateLimitLike = { requestsPerMinute?: number; minIntervalMs?: number; concurrency?: number };
export function limiterOptions(def: SiteDefinition, rateLimit: unknown): { minIntervalMs: number; burst: number } {
const rl = (rateLimit ?? {}) as { requestsPerMinute?: number; minIntervalMs?: number; concurrency?: number }; /**
const minIntervalMs = * 站点行 rate_limit 优先;为空时用全局默认(settings.sites.defaultRateLimit),但不比定义里的 umt.rateLimit 更激进
rl.minIntervalMs ?? * (定义值是按站点实测调的下限);都没有时用定义值。
(rl.requestsPerMinute ? Math.ceil(60_000 / rl.requestsPerMinute) : def.umt.rateLimit.minIntervalMs); */
return { minIntervalMs, burst: rl.concurrency ?? def.umt.rateLimit.burst }; export function limiterOptions(
def: SiteDefinition,
rateLimit: unknown,
globalDefault?: RateLimitLike | null,
): { minIntervalMs: number; burst: number } {
const own = (rateLimit ?? {}) as RateLimitLike;
const hasOwn =
own.minIntervalMs !== undefined || own.requestsPerMinute !== undefined || own.concurrency !== undefined;
const interval = (rl: RateLimitLike) =>
rl.minIntervalMs ?? (rl.requestsPerMinute ? Math.ceil(60_000 / rl.requestsPerMinute) : undefined);
if (hasOwn) {
return {
minIntervalMs: interval(own) ?? def.umt.rateLimit.minIntervalMs,
burst: own.concurrency ?? def.umt.rateLimit.burst,
};
}
const g = globalDefault ?? {};
const gi = interval(g);
return {
minIntervalMs: gi === undefined ? def.umt.rateLimit.minIntervalMs : Math.max(gi, def.umt.rateLimit.minIntervalMs),
burst: g.concurrency === undefined ? def.umt.rateLimit.burst : Math.min(g.concurrency, def.umt.rateLimit.burst),
};
} }
@@ -66,10 +66,13 @@ export async function tickFeeds(rt: SourcesRuntime): Promise<number> {
export async function tickSnapshots(rt: SourcesRuntime): Promise<number> { export async function tickSnapshots(rt: SourcesRuntime): Promise<number> {
const { ctx } = rt; const { ctx } = rt;
const now = rt.now(); const now = rt.now();
// 站点未设周期时用全局默认(settings.sites.defaultSnapshotIntervalSec)
const fallback = (await rt.siteDefaults()).defaultSnapshotIntervalSec;
const sites = await sql<{ id: string; snapshot_interval_sec: number }>` const sites = await sql<{ id: string; snapshot_interval_sec: number }>`
select id, snapshot_interval_sec from site select id, coalesce(snapshot_interval_sec, ${fallback}::int) as snapshot_interval_sec from site
where enabled and snapshot_enabled and snapshot_interval_sec is not null where enabled and snapshot_enabled and deleted_at is null
and (last_snapshot_at is null or last_snapshot_at + make_interval(secs => snapshot_interval_sec) <= ${now})`.execute( and (last_snapshot_at is null
or last_snapshot_at + make_interval(secs => coalesce(snapshot_interval_sec, ${fallback}::int)) <= ${now})`.execute(
ctx.db, ctx.db,
); );
for (const s of sites.rows) { for (const s of sites.rows) {
@@ -0,0 +1,884 @@
import { randomUUID } from "node:crypto";
import {
getSiteDefinition,
loginNexusPhp,
refreshSiteUserData,
SiteLoginError,
type SiteLoginFailure,
type SiteUserInfo,
} from "@umt/adapters";
import type {
DecisionOption,
SecretPurpose,
SiteAccountStatus,
SiteAccountUserData,
SiteCredentialsPut,
SiteCredentialsResult,
SiteHealth,
SiteHealthResult,
SiteLoginResult,
SiteUserDataPoint,
} from "@umt/contracts";
import { type SiteAccountRow, type SiteRow, sql } from "@umt/db";
import { toApi } from "../rows.ts";
import type { SourcesRuntime } from "../runtime.ts";
import { type Actor, audit, closeDecisions, type Exec, openDecision, SYSTEM } from "./common.ts";
// W9-A 站点中心(docs/design/site-center.md):凭证写入、刷新登录(账密 takelogin / Cookie / API Key 校验)、
// 用户数据与按天历史、健康检测、删除。凭证明文只在内存里用,不进日志、审计与错误信息。
const jsonb = (v: unknown) => JSON.stringify(v ?? null);
/** site_account 行 → SiteAccount(secret 引用只给布尔) */
export function accountToApi(row: SiteAccountRow): Record<string, unknown> {
const { totp_secret_ref, session_secret_ref, passkey_secret_ref, ...rest } = row;
return {
...toApi<Record<string, unknown>>(rest),
hasTotp: !!totp_secret_ref,
hasSession: !!session_secret_ref,
hasPasskey: !!passkey_secret_ref || (row.auth_kind === "passkey" && !!row.secret_ref),
};
}
/** 多账号取最差;无启用账号 = unconfigured */
const STATUS_RANK: Record<SiteAccountStatus, number> = { active: 0, unknown: 1, expired: 2, invalid: 3, challenge: 4 };
export function overallAccountStatus(rows: Pick<SiteAccountRow, "enabled" | "status">[]) {
const enabled = rows.filter((r) => r.enabled);
if (!enabled.length) return "unconfigured" as const;
return enabled.reduce<SiteAccountStatus>(
(worst, r) => (STATUS_RANK[r.status] > STATUS_RANK[worst] ? r.status : worst),
"active",
);
}
/** 本地日期(按进程时区)YYYY-MM-DD */
function localDay(d: Date): string {
return new Date(d.getTime() - d.getTimezoneOffset() * 60_000).toISOString().slice(0, 10);
}
export function userDataOf(info: SiteUserInfo): SiteAccountUserData {
const upload = Math.max(0, info.upload || 0);
const download = Math.max(0, info.download || 0);
const ratio = info.ratio ? info.ratio : download > 0 ? Math.round((upload / download) * 1000) / 1000 : null;
const int = (n: number) => Math.max(0, Math.round(n || 0));
return {
schema: info.schema ?? null,
userId: info.userId ?? null,
username: info.username ?? null,
userLevel: info.userLevel ?? null,
vip: info.vip ?? null,
joinAt: info.joinAt ?? null,
upload,
download,
ratio,
bonus: info.bonus ? info.bonus : null,
seeding: int(info.seeding),
seedingSize: Math.max(0, info.seedingSize || 0),
leeching: int(info.leeching),
leechingSize: Math.max(0, info.leechingSize || 0),
messageUnread: int(info.messageUnread),
};
}
// ── 凭证写入 ──
const PURPOSE: Record<string, SecretPurpose> = {
cookie: "site_cookie",
passkey: "site_passkey",
api_key: "site_api_key",
password: "site_password",
};
type NormalizedCredentials =
| {
kind: "password";
label: string;
username: string;
password: string;
totpSecret?: string;
saveCookieOnly?: boolean;
userAgent?: string;
enabled?: boolean;
}
| { kind: "cookie"; label: string; cookie: string; userAgent?: string; enabled?: boolean }
| { kind: "api_key"; label: string; apiKey: string; enabled?: boolean }
| { kind: "passkey"; label: string; passkey: string; enabled?: boolean };
/** 旧格式 {authKind, credential, username?} → 新格式 */
export function normalizeCredentials(body: SiteCredentialsPut): NormalizedCredentials {
if ("kind" in body) return body as NormalizedCredentials;
const label = body.label;
const enabled = body.enabled;
const opt = enabled === undefined ? {} : { enabled };
switch (body.authKind) {
case "password":
return { kind: "password", label, username: body.username || label, password: body.credential, ...opt };
case "cookie":
return { kind: "cookie", label, cookie: body.credential, ...opt };
case "api_key":
return { kind: "api_key", label, apiKey: body.credential, ...opt };
default:
return { kind: "passkey", label, passkey: body.credential, ...opt };
}
}
/**
* PUT /sites/{id}/credentials:按 (site, label) 建立或更新账号。
* 主凭证(password / cookie / api_key)换掉旧 secret、状态回 unknown、清会话 Cookie、关闭 site-auth / site-login 待处理;
* passkey 只附挂到账号(不改登录状态)。
*/
export async function setSiteCredentials(
rt: SourcesRuntime,
site: Pick<SiteRow, "id" | "health_status">,
input: SiteCredentialsPut,
actor: Actor,
): Promise<SiteCredentialsResult> {
const { db, secrets } = rt.ctx;
const body = normalizeCredentials(input);
const label = body.label;
const created: string[] = [];
const mk = async (purpose: SecretPurpose, plaintext: string, suffix = "") => {
const ref = await secrets.create({ purpose, plaintext, label: `site_account:${label}${suffix}` });
created.push(ref);
return ref;
};
const obsolete: string[] = [];
let result: SiteCredentialsResult;
try {
if (body.kind === "passkey") {
const ref = await mk("site_passkey", body.passkey, ":passkey");
result = await db.transaction().execute(async (trx) => {
const cur = await trx
.selectFrom("site_account")
.selectAll()
.where("site_id", "=", site.id)
.where("label", "=", label)
.forUpdate()
.executeTakeFirst();
if (cur) {
if (cur.auth_kind === "passkey") {
if (cur.secret_ref) obsolete.push(cur.secret_ref);
await trx
.updateTable("site_account")
.set({ secret_ref: ref, ...(body.enabled !== undefined ? { enabled: body.enabled } : {}) })
.where("id", "=", cur.id)
.execute();
} else {
if (cur.passkey_secret_ref) obsolete.push(cur.passkey_secret_ref);
await trx
.updateTable("site_account")
.set({ passkey_secret_ref: ref, ...(body.enabled !== undefined ? { enabled: body.enabled } : {}) })
.where("id", "=", cur.id)
.execute();
}
return {
siteAccountId: cur.id,
siteId: site.id,
label,
authKind: cur.auth_kind,
secretRef: ref,
created: false,
};
}
const r = await trx
.insertInto("site_account")
.values({
site_id: site.id,
label,
auth_kind: "passkey",
secret_ref: ref,
...(body.enabled !== undefined ? { enabled: body.enabled } : {}),
})
.returning("id")
.executeTakeFirstOrThrow();
return { siteAccountId: r.id, siteId: site.id, label, authKind: "passkey", secretRef: ref, created: true };
});
} else {
const plaintext = body.kind === "password" ? body.password : body.kind === "cookie" ? body.cookie : body.apiKey;
const ref = await mk(PURPOSE[body.kind]!, plaintext);
const totpRef =
body.kind === "password" && body.totpSecret ? await mk("site_password", body.totpSecret, ":totp") : null;
result = await db.transaction().execute(async (trx) => {
const cur = await trx
.selectFrom("site_account")
.selectAll()
.where("site_id", "=", site.id)
.where("label", "=", label)
.forUpdate()
.executeTakeFirst();
const userInfo = {
...((cur?.user_info as Record<string, unknown> | undefined) ?? {}),
...(body.kind === "password" ? { username: body.username, saveCookieOnly: !!body.saveCookieOnly } : {}),
};
if (body.kind !== "password") delete userInfo.saveCookieOnly;
const values = {
auth_kind: body.kind,
secret_ref: ref,
status: "unknown" as const,
last_error: null,
session_secret_ref: null,
totp_secret_ref: totpRef,
login_name: body.kind === "password" ? body.username : null,
user_agent: "userAgent" in body ? (body.userAgent ?? null) : null,
user_info: jsonb(userInfo),
...(body.enabled !== undefined ? { enabled: body.enabled } : {}),
};
let id: string;
let isNew = false;
if (cur) {
for (const r of [
cur.auth_kind === "passkey" ? null : cur.secret_ref,
cur.session_secret_ref,
cur.totp_secret_ref,
])
if (r) obsolete.push(r);
// 原来是 passkey 型账号:passkey 转为附挂
const passkey = cur.auth_kind === "passkey" && cur.secret_ref ? { passkey_secret_ref: cur.secret_ref } : {};
await trx
.updateTable("site_account")
.set({ ...values, ...passkey })
.where("id", "=", cur.id)
.execute();
id = cur.id;
} else {
const r = await trx
.insertInto("site_account")
.values({ site_id: site.id, label, ...values })
.returning("id")
.executeTakeFirstOrThrow();
id = r.id;
isNew = true;
}
await trx
.updateTable("site")
.set({ health_status: "unknown" })
.where("id", "=", site.id)
.where("health_status", "=", "failing")
.execute();
const by = `${actor.type}:${actor.id ?? ""}`;
await closeDecisions(rt.ctx, trx, `site-auth:${site.id}`, "resolved", { by: "credential_updated" }, by);
await closeDecisions(rt.ctx, trx, `site-login:${id}`, "resolved", { by: "credential_updated" }, by);
return { siteAccountId: id, siteId: site.id, label, authKind: body.kind, secretRef: ref, created: isNew };
});
}
} catch (e) {
for (const r of created) await secrets.remove(r).catch(() => {});
throw e;
}
for (const r of obsolete) await secrets.remove(r).catch(() => {});
await audit(
db,
actor,
"site_account.credentials_set",
{ type: "site_account", id: result.siteAccountId },
{ siteId: site.id, label, authKind: body.kind, created: result.created },
);
return result;
}
// ── 刷新登录 / 用户数据 ──
type UserDataOutcome =
| { kind: "ok"; info: SiteUserInfo }
| { kind: "auth"; message: string }
| { kind: "network"; message: string }
| { kind: "challenge"; message: string }
| { kind: "no_parser" };
const FAIL_STATUS: Partial<Record<SiteLoginFailure, SiteAccountStatus>> = {
captcha: "challenge",
two_factor: "challenge",
challenge: "challenge",
bad_credentials: "invalid",
account_blocked: "invalid",
auth_expired: "expired",
unknown: "expired",
};
export interface RefreshOptions {
/** login:重新登录 / 校验(手动刷新登录);userData:定时刷新用户数据,会话失效时账密账号自动重登 */
mode?: "login" | "userData";
actor?: Actor;
signal?: AbortSignal;
}
export async function refreshSiteAccount(
rt: SourcesRuntime,
accountId: string,
opts: RefreshOptions = {},
): Promise<SiteLoginResult> {
const { db, secrets } = rt.ctx;
const actor = opts.actor ?? SYSTEM;
const acc = await db.selectFrom("site_account").selectAll().where("id", "=", accountId).executeTakeFirst();
if (!acc) throw new Error("站点账号不存在");
const site = await db.selectFrom("site").selectAll().where("id", "=", acc.site_id).executeTakeFirstOrThrow();
const base = { siteId: site.id, accountId: acc.id };
const unchanged = (reason: SiteLoginFailure, message: string): SiteLoginResult => ({
...base,
ok: false,
status: acc.status,
reason,
message,
userData: null,
decisionId: null,
});
const def = getSiteDefinition(site.key);
if (!def || site.deleted_at) return unchanged("unsupported", "站点没有适配器定义,无法校验登录");
const defaults = await rt.siteDefaults();
const ua = acc.user_agent ?? defaults.userAgent;
let limiter: ReturnType<SourcesRuntime["limiterOf"]> = null;
try {
rt.siteAdapter(site);
limiter = rt.limiterOf(site.id);
} catch {
// 无适配器:不限流(仅 generic_rss,前面已返回)
}
const fetch = rt.siteFetch(site, defaults);
const umt = def.umt as { kind: string; login?: Parameters<typeof loginNexusPhp>[0]["login"]; loginSuccess?: string };
const profile = (def.umt as { profile?: Record<string, string> }).profile;
const vipLevels = (def.umt as { vipLevels?: string[] }).vipLevels;
const apiUrl = def.umt.kind === "mteam" ? ((site.config as { apiBaseUrl?: string })?.apiBaseUrl ?? null) : null;
const fetchUserData = async (cred: { cookie?: string | null; apikey?: string | null }): Promise<UserDataOutcome> => {
const info = await refreshSiteUserData({
name: site.name,
url: site.base_url,
schema: (def as { schema?: string }).schema ?? null,
public: true,
cookie: cred.cookie ?? null,
apikey: cred.apikey ?? null,
ua,
fetch,
...(limiter ? { limiter } : {}),
...(apiUrl ? { apiUrl } : {}),
fetchMessages: false,
maxSeedingPages: defaults.userDataMaxSeedingPages,
vipCss: profile?.vip_css ?? null,
vipLevels: vipLevels ?? null,
timeoutMs: 20_000,
...(opts.signal ? { signal: opts.signal } : {}),
});
if (!info) return { kind: "no_parser" };
if (info.userId && !info.errMsg) return { kind: "ok", info };
const errs = ((info.raw as { http_errors?: string[] }).http_errors ?? []).filter(Boolean);
if (errs.some((e) => /Cloudflare/i.test(e))) return { kind: "challenge", message: "站点返回 Cloudflare 挑战页" };
// 连接失败 / 超时 / 5xx 视为网络问题;4xx 与解析失败按登录态失效处理
const net = errs.find((e) => !/HTTP 4\d\d|JSON 解析失败/.test(e));
if (net) return { kind: "network", message: net.slice(0, 200) };
return { kind: "auth", message: info.errMsg ?? "未检测到登录态" };
};
const viaHealth = async (): Promise<UserDataOutcome> => {
const h = await rt.siteAdapter(site).healthCheck(opts.signal ? { signal: opts.signal } : {});
if (h.ok) return { kind: "no_parser" };
if (h.status.startsWith("auth_")) return { kind: "auth", message: h.message ?? "未检测到登录态" };
return { kind: "network", message: h.message ?? h.status };
};
const fail = (reason: SiteLoginFailure, message: string) => markFailure(rt, acc, site, reason, message, actor);
const succeed = (info: SiteUserInfo | null, note?: string) => markSuccess(rt, acc, site, info, actor, note);
try {
if (acc.auth_kind === "passkey") return unchanged("unsupported", "Passkey 只用于 RSS 和下载,不参与登录");
if (!acc.secret_ref) return unchanged("unsupported", "账号没有凭证");
if (acc.auth_kind === "password") {
if (umt.kind !== "nexusphp" || !umt.login) {
return unchanged("unsupported", "该站点不支持账号密码登录。改用 Cookie 或 API Key");
}
if (opts.mode === "userData" && acc.session_secret_ref) {
const r = await fetchUserData({ cookie: await secrets.resolve(acc.session_secret_ref) });
if (r.kind === "ok") return await succeed(r.info);
if (r.kind === "network") return await fail("network", r.message);
// 会话失效:下面重登一次
}
const password = await secrets.resolve(acc.secret_ref);
const totpSecret = acc.totp_secret_ref ? await secrets.resolve(acc.totp_secret_ref) : null;
const login = await loginNexusPhp({
baseUrl: site.base_url,
login: umt.login,
...(umt.loginSuccess ? { loginSuccess: umt.loginSuccess } : {}),
username: acc.login_name ?? (acc.user_info as { username?: string })?.username ?? acc.label,
password,
totpSecret,
userAgent: ua,
fetch,
...(limiter ? { limiter } : {}),
...(opts.signal ? { signal: opts.signal } : {}),
now: rt.now,
});
await storeSession(rt, acc, login.cookie);
const r = await fetchUserData({ cookie: login.cookie });
if (r.kind === "ok") return await succeed(r.info);
return await succeed(
null,
`登录成功;用户数据未取到:${r.kind === "no_parser" ? "站点没有用户数据解析器" : r.message}`,
);
}
const plain = await secrets.resolve(acc.secret_ref);
let r = await fetchUserData(acc.auth_kind === "cookie" ? { cookie: plain } : { apikey: plain });
if (r.kind === "no_parser") r = await viaHealth();
if (r.kind === "ok") return await succeed(r.info);
if (r.kind === "no_parser") return await succeed(null, "站点没有用户数据解析器,只校验了登录态");
if (r.kind === "network") return await fail("network", r.message);
if (r.kind === "challenge") return await fail("challenge", r.message);
return await fail(
"auth_expired",
acc.auth_kind === "api_key"
? "API Key 无效。更新 API Key 后重新刷新登录"
: "Cookie 已失效。更新 Cookie 后重新刷新登录",
);
} catch (e) {
if (e instanceof SiteLoginError) return await fail(e.reason, e.message);
throw e;
}
}
/** 账密登录成功:会话 Cookie 存 secret;"只换 Cookie"时凭证类型落为 cookie,密码与 TOTP 丢弃 */
async function storeSession(rt: SourcesRuntime, acc: SiteAccountRow, cookie: string) {
const { db, secrets } = rt.ctx;
const cookieOnly = (acc.user_info as { saveCookieOnly?: boolean })?.saveCookieOnly === true;
if (cookieOnly) {
const ref = await secrets.create({ purpose: "site_cookie", plaintext: cookie, label: `site_account:${acc.label}` });
const info = { ...(acc.user_info as Record<string, unknown>) };
delete info.saveCookieOnly;
await db
.updateTable("site_account")
.set({
auth_kind: "cookie",
secret_ref: ref,
session_secret_ref: null,
totp_secret_ref: null,
user_info: jsonb(info),
})
.where("id", "=", acc.id)
.execute();
for (const r of [acc.secret_ref, acc.session_secret_ref, acc.totp_secret_ref]) {
if (r) await secrets.remove(r).catch(() => {});
}
acc.auth_kind = "cookie";
acc.secret_ref = ref;
acc.session_secret_ref = null;
acc.totp_secret_ref = null;
return;
}
if (acc.session_secret_ref) {
await secrets.replace(acc.session_secret_ref, cookie);
return;
}
const ref = await secrets.create({
purpose: "site_cookie",
plaintext: cookie,
label: `site_account:${acc.label}:session`,
});
await db.updateTable("site_account").set({ session_secret_ref: ref }).where("id", "=", acc.id).execute();
acc.session_secret_ref = ref;
}
async function markSuccess(
rt: SourcesRuntime,
acc: SiteAccountRow,
site: SiteRow,
info: SiteUserInfo | null,
actor: Actor,
note?: string,
): Promise<SiteLoginResult> {
const now = rt.now();
const data = info ? userDataOf(info) : null;
await rt.ctx.db.transaction().execute(async (trx) => {
await trx
.updateTable("site_account")
.set({
status: "active",
last_login_at: now,
last_verified_at: now,
last_error: null,
...(data ? { user_data: jsonb(data), user_data_at: now } : {}),
})
.where("id", "=", acc.id)
.execute();
if (info && data) await upsertUserDataDay(trx, acc.id, now, info, data);
await trx
.updateTable("site")
.set({ health_status: "unknown" })
.where("id", "=", site.id)
.where("health_status", "=", "failing")
.execute();
const by = `${actor.type}:${actor.id ?? ""}`;
await closeDecisions(rt.ctx, trx, `site-login:${acc.id}`, "resolved", { by: "login_refreshed" }, by);
await closeDecisions(rt.ctx, trx, `site-auth:${site.id}`, "resolved", { by: "login_refreshed" }, by);
await audit(
trx,
actor,
"site_account.login_refreshed",
{ type: "site_account", id: acc.id },
{
siteId: site.id,
ok: true,
userData: !!data,
},
);
});
return {
siteId: site.id,
accountId: acc.id,
ok: true,
status: "active",
reason: null,
message: note ?? null,
userData: data,
decisionId: null,
};
}
async function upsertUserDataDay(trx: Exec, accountId: string, now: Date, info: SiteUserInfo, d: SiteAccountUserData) {
const {
messageUnreadContents: _m,
seedingInfo: _s,
raw: _r,
...rest
} = info as SiteUserInfo & { messageUnreadContents?: unknown; seedingInfo?: unknown };
await sql`
insert into site_user_data (site_account_id, day, upload, download, ratio, bonus, seeding, seeding_size, leeching,
user_level, vip, message_unread, join_at, raw, captured_at)
values (${accountId}, ${localDay(now)}, ${Math.round(d.upload)}, ${Math.round(d.download)}, ${d.ratio}, ${d.bonus},
${d.seeding}, ${Math.round(d.seedingSize)}, ${d.leeching}, ${d.userLevel}, ${d.vip}, ${d.messageUnread},
${d.joinAt}, ${jsonb(rest)}::jsonb, ${now})
on conflict (site_account_id, day) do update set
upload = excluded.upload, download = excluded.download, ratio = excluded.ratio, bonus = excluded.bonus,
seeding = excluded.seeding, seeding_size = excluded.seeding_size, leeching = excluded.leeching,
user_level = excluded.user_level, vip = excluded.vip, message_unread = excluded.message_unread,
join_at = excluded.join_at, raw = excluded.raw, captured_at = excluded.captured_at`.execute(trx);
}
const DECISION_TEXT: Record<string, { title: (s: string) => string; summary: string }> = {
challenge: {
title: (s) => `站点「${s}」登录需要人工验证`,
summary: "登录页要求验证码 / 二次验证,无法自动登录。改用 Cookie 登录,或填写 TOTP 密钥后重试",
},
invalid: {
title: (s) => `站点「${s}」登录失败`,
summary: "用户名或密码错误,或账号被封禁 / 未确认。更新凭证后重新刷新登录",
},
expired: {
title: (s) => `站点「${s}」登录失效`,
summary: "凭证已失效。更新凭证后重新刷新登录",
},
};
async function markFailure(
rt: SourcesRuntime,
acc: SiteAccountRow,
site: SiteRow,
reason: SiteLoginFailure,
message: string,
actor: Actor,
): Promise<SiteLoginResult> {
const now = rt.now();
const lastError = { reason, message: message.slice(0, 300), at: now.toISOString() };
const status =
reason === "network" || reason === "unsupported"
? null
: reason === "auth_expired" && acc.auth_kind === "api_key"
? "invalid"
: (FAIL_STATUS[reason] ?? "expired");
let decisionId: string | null = null;
await rt.ctx.db.transaction().execute(async (trx) => {
await trx
.updateTable("site_account")
.set({ last_error: jsonb(lastError), ...(status ? { status, last_verified_at: now } : {}) })
.where("id", "=", acc.id)
.execute();
if (status) {
const text = DECISION_TEXT[status]!;
const summary =
reason === "challenge"
? "站点返回 Cloudflare 挑战页,无法自动登录。改用 Cookie 登录"
: status === "expired" ||
reason === "account_blocked" ||
reason === "bad_credentials" ||
reason === "auth_expired"
? message
: text.summary;
const options: DecisionOption[] = [
{
key: "use_cookie",
label: "改用 Cookie",
input: { siteId: site.id, accountId: acc.id, credential: "cookie" },
},
...(status === "challenge" ? [{ key: "retry", label: "重试登录", input: { accountId: acc.id } }] : []),
{ key: "dismiss", label: "忽略" },
];
const d = await openDecision(rt.ctx, trx, {
kind: "source_unreachable",
severity: "warning",
title: text.title(site.name),
summary,
subjectType: "site_account",
subjectId: acc.id,
dedupeKey: `site-login:${acc.id}`,
evidence: { siteId: site.id, accountId: acc.id, reason, authKind: acc.auth_kind },
options,
});
decisionId = d.id;
}
await audit(
trx,
actor,
"site_account.login_refreshed",
{ type: "site_account", id: acc.id },
{
siteId: site.id,
ok: false,
reason,
},
);
});
return {
siteId: site.id,
accountId: acc.id,
ok: false,
status: status ?? acc.status,
reason,
message,
userData: null,
decisionId,
};
}
/** 请求遇 auth_expired:有密码的账号 10 分钟内最多自动重登一次 */
export async function enqueueRelogin(rt: SourcesRuntime, siteId: string): Promise<number> {
const rows = await rt.ctx.db
.selectFrom("site_account")
.select(["id"])
.where("site_id", "=", siteId)
.where("enabled", "=", true)
.where("auth_kind", "=", "password")
.where("secret_ref", "is not", null)
.execute();
const slot = Math.floor(rt.now().getTime() / 600_000);
for (const r of rows) {
await rt.ctx.engine.enqueue(
"refreshSiteAccount",
{ accountId: r.id, mode: "login" },
{ idempotencyKey: `refreshSiteAccount:relogin:${r.id}:${slot}`, traceId: randomUUID() },
);
}
return rows.length;
}
// ── 健康检测 ──
const HEALTH_OF: Record<string, SiteHealth> = {
ok: "healthy",
auth_missing: "failing",
auth_expired: "failing",
auth_invalid: "failing",
rate_limited: "degraded",
unreachable: "degraded",
site_maintenance: "degraded",
parse_error: "degraded",
};
/** 首页连通 + 登录态(≤ 20 s,走站点限流);写 site.last_health* 与 health_status(已关闭的站点保持 disabled) */
export async function checkSiteHealth(
rt: SourcesRuntime,
site: SiteRow,
actor: Actor = SYSTEM,
): Promise<SiteHealthResult> {
const now = rt.now();
let ok = false;
let status = "unreachable";
let latencyMs: number | null = null;
let message: string | null = null;
const def = getSiteDefinition(site.key);
if (!def) {
message = "站点没有适配器定义,无法检测";
status = "unsupported";
} else {
await rt.siteDefaults();
const h = await rt.siteAdapter(site).healthCheck({ signal: AbortSignal.timeout(20_000) });
ok = h.ok;
status = h.status;
latencyMs = h.latencyMs;
message = h.message ?? null;
}
const loggedIn = ok ? true : status.startsWith("auth_") ? false : null;
const healthStatus: SiteHealth = !site.enabled
? "disabled"
: def
? (HEALTH_OF[status] ?? "degraded")
: site.health_status;
const last = { ok, status, latencyMs, message };
await rt.ctx.db.transaction().execute(async (trx) => {
await trx
.updateTable("site")
.set({ last_health: jsonb(last), last_health_at: now, health_status: healthStatus })
.where("id", "=", site.id)
.execute();
// 首页显示已登录:顺手记上次登录
if (ok) {
await trx
.updateTable("site_account")
.set({ last_login_at: now })
.where("site_id", "=", site.id)
.where("enabled", "=", true)
.where("status", "=", "active")
.execute();
}
if (actor.type !== "system") {
await audit(trx, actor, "site.health_checked", { type: "site", id: site.id }, { ok, status });
}
});
return { siteId: site.id, checkedAt: now.toISOString(), loggedIn, healthStatus, ...last };
}
// ── 用户数据历史 ──
export async function userDataHistory(
rt: SourcesRuntime,
siteId: string,
q: { accountId?: string | undefined; days: number },
): Promise<SiteUserDataPoint[]> {
const since = localDay(new Date(rt.now().getTime() - (q.days - 1) * 86_400_000));
let query = rt.ctx.db
.selectFrom("site_user_data as d")
.innerJoin("site_account as a", "a.id", "d.site_account_id")
.selectAll("d")
.where("a.site_id", "=", siteId)
.where("d.day", ">=", since);
if (q.accountId) query = query.where("d.site_account_id", "=", q.accountId);
const rows = await query.orderBy("d.day").orderBy("d.site_account_id").execute();
return rows.map((r) => ({
siteAccountId: r.site_account_id,
day: r.day,
upload: r.upload,
download: r.download,
ratio: r.ratio,
bonus: r.bonus,
seeding: r.seeding,
seedingSize: r.seeding_size,
leeching: r.leeching,
userLevel: r.user_level,
vip: r.vip,
messageUnread: r.message_unread,
capturedAt: r.captured_at.toISOString(),
}));
}
// ── 删除 ──
/**
* 删除站点:停用(资源标 site_disabled、Feed 停用)→ 删账号与全部凭证 → 软删除站点行(key 改名,可再次登记同一定义)。
* source_release 外键 restrict,资源和下载任务都保留。
*/
export async function deleteSite(rt: SourcesRuntime, site: SiteRow, actor: Actor): Promise<void> {
const { db, secrets } = rt.ctx;
const refs: string[] = [];
await db.transaction().execute(async (trx) => {
const accounts = await trx.selectFrom("site_account").selectAll().where("site_id", "=", site.id).execute();
for (const a of accounts) {
for (const r of [a.secret_ref, a.session_secret_ref, a.totp_secret_ref, a.passkey_secret_ref])
if (r) refs.push(r);
}
await trx
.updateTable("source_release")
.set({ availability: "site_disabled" })
.where("site_id", "=", site.id)
.where("availability", "=", "active")
.execute();
await trx
.updateTable("feed_source")
.set({ enabled: false, health_status: "disabled", site_account_id: null })
.where("site_id", "=", site.id)
.execute();
const by = `${actor.type}:${actor.id ?? ""}`;
for (const a of accounts) {
await closeDecisions(rt.ctx, trx, `site-login:${a.id}`, "dismissed", { by: "site_deleted" }, by);
}
await closeDecisions(rt.ctx, trx, `site-auth:${site.id}`, "dismissed", { by: "site_deleted" }, by);
await trx.deleteFrom("site_account").where("site_id", "=", site.id).execute();
await trx
.updateTable("site")
.set({
enabled: false,
health_status: "disabled",
snapshot_enabled: false,
deleted_at: rt.now(),
key: `${site.key}-deleted-${Date.now().toString(36)}`,
})
.where("id", "=", site.id)
.execute();
await audit(
trx,
actor,
"site.deleted",
{ type: "site", id: site.id },
{ key: site.key, accounts: accounts.length },
);
});
for (const r of refs) await secrets.remove(r).catch(() => {});
}
export async function deleteSiteAccount(rt: SourcesRuntime, acc: SiteAccountRow, actor: Actor): Promise<void> {
const { db, secrets } = rt.ctx;
await db.transaction().execute(async (trx) => {
await closeDecisions(
rt.ctx,
trx,
`site-login:${acc.id}`,
"dismissed",
{ by: "account_deleted" },
`${actor.type}:${actor.id ?? ""}`,
);
await trx.deleteFrom("site_account").where("id", "=", acc.id).execute();
await audit(trx, actor, "site_account.deleted", { type: "site_account", id: acc.id }, { siteId: acc.site_id });
});
for (const r of [acc.secret_ref, acc.session_secret_ref, acc.totp_secret_ref, acc.passkey_secret_ref]) {
if (r) await secrets.remove(r).catch(() => {});
}
}
// ── 定时任务 ──
/** sites.userData:启用站点里登录状态正常的账号逐个入队刷新用户数据 */
export async function tickSiteUserData(rt: SourcesRuntime, slot: string): Promise<number> {
const rows = await rt.ctx.db
.selectFrom("site_account as a")
.innerJoin("site as s", "s.id", "a.site_id")
.select(["a.id"])
.where("s.enabled", "=", true)
.where("s.deleted_at", "is", null)
.where("a.enabled", "=", true)
.where("a.status", "=", "active")
.where("a.auth_kind", "!=", "passkey")
.execute();
for (const r of rows) {
await rt.ctx.engine.enqueue(
"refreshSiteAccount",
{ accountId: r.id, mode: "userData" },
{ idempotencyKey: `refreshSiteAccount:userData:${r.id}:${slot}`, traceId: randomUUID() },
);
}
return rows.length;
}
/** sites.healthCheck:启用站点逐个检测(同步,走站点限流) */
export async function tickSiteHealth(rt: SourcesRuntime): Promise<number> {
const sites = await rt.ctx.db
.selectFrom("site")
.selectAll()
.where("enabled", "=", true)
.where("deleted_at", "is", null)
.where("adapter", "!=", "generic_rss")
.execute();
let n = 0;
for (const s of sites) {
try {
await checkSiteHealth(rt, s);
n++;
} catch (err) {
rt.ctx.logger.warn({ siteId: s.id, err: { message: (err as Error).message } }, "站点健康检测失败");
}
}
return n;
}
@@ -5,6 +5,7 @@ import { parseRelease } from "@umt/domain";
import { siteReleaseToSourceRelease } from "@umt/mapping"; import { siteReleaseToSourceRelease } from "@umt/mapping";
import { jsonb } from "../rows.ts"; import { jsonb } from "../rows.ts";
import type { SourcesRuntime } from "../runtime.ts"; import type { SourcesRuntime } from "../runtime.ts";
import { enqueueRelogin } from "./account.ts";
import { type Exec, openDecision } from "./common.ts"; import { type Exec, openDecision } from "./common.ts";
// 站点快照(周期列表)、补漏(Feed 疑似缺口,有页数预算)、实时补查(工作台搜索)。 // 站点快照(周期列表)、补漏(Feed 疑似缺口,有页数预算)、实时补查(工作台搜索)。
@@ -92,6 +93,8 @@ async function onAuthError(rt: SourcesRuntime, trx: Exec, site: SiteRow, e: Site
evidence: { siteId: site.id, kind: e.kind, httpStatus: e.httpStatus ?? null }, evidence: { siteId: site.id, kind: e.kind, httpStatus: e.httpStatus ?? null },
options: [{ key: "reset", label: "已更新凭证,恢复访问" }], options: [{ key: "reset", label: "已更新凭证,恢复访问" }],
}); });
// 有密码的账号:自动重登一次(同账号 10 分钟内最多一次),成功后关闭本待处理并复位站点健康
await enqueueRelogin(rt, site.id);
} }
export async function snapshotSite(rt: SourcesRuntime, input: SnapshotInput): Promise<SnapshotResult> { export async function snapshotSite(rt: SourcesRuntime, input: SnapshotInput): Promise<SnapshotResult> {
@@ -1,6 +1,7 @@
import { defineTask, type OutboxRouteFn, type TaskRegistry } from "@umt/tasks"; import { defineTask, type OutboxRouteFn, type TaskRegistry } from "@umt/tasks";
import { z } from "zod"; import { z } from "zod";
import type { SourcesRuntime } from "./runtime.ts"; import type { SourcesRuntime } from "./runtime.ts";
import { refreshSiteAccount } from "./service/account.ts";
import { ingestFeed } from "./service/feeds.ts"; import { ingestFeed } from "./service/feeds.ts";
import { refreshMetadata } from "./service/metadata.ts"; import { refreshMetadata } from "./service/metadata.ts";
import { resolveRelease } from "./service/resolve.ts"; import { resolveRelease } from "./service/resolve.ts";
@@ -76,6 +77,23 @@ export function registerSourcesTasks(registry: TaskRegistry, rt: () => SourcesRu
snapshotSite(rt(), { siteId: t.input.siteId, kind: t.input.kind, maxPages: t.input.maxPages }), snapshotSite(rt(), { siteId: t.input.siteId, kind: t.input.kind, maxPages: t.input.maxPages }),
), ),
}), }),
defineTask({
name: "refreshSiteAccount",
version: 1,
inputSchema: z.object({ accountId: id, mode: z.enum(["login", "userData"]).default("login") }),
// 网络失败只记 last_error 不抛错;不自动重试(站点登录频繁重试有封号风险)
retryPolicy: { maximumAttempts: 1 },
handler: (t) =>
t.step.run("refresh", async () => {
const exists = await rt()
.ctx.db.selectFrom("site_account")
.select("id")
.where("id", "=", t.input.accountId)
.executeTakeFirst();
if (!exists) return { skipped: "not_found" };
return refreshSiteAccount(rt(), t.input.accountId, { mode: t.input.mode });
}),
}),
); );
} }
@@ -238,43 +238,6 @@ describe.skipIf(!hasPg)("W2-A 管理 API(真实 PG)", () => {
expect(v.verifyDetail.code).toBe("ENOENT"); expect(v.verifyDetail.code).toBe("ENOENT");
expect((await admin("DELETE", `/path-mappings/${m.id}`)).status).toBe(200); expect((await admin("DELETE", `/path-mappings/${m.id}`)).status).toBe(200);
}); });
it("PUT /sites/{id}/credentials:新建/替换账号凭证,不回显,旧 secret 删除;未知站点 404", async () => {
const site = await sql<{ id: string }>`insert into site (key, name, adapter, base_url)
values ('audiences', '观众', 'nexusphp', 'https://audiences.me') returning id`.execute(h.db);
const siteId = site.rows[0]!.id;
const r1 = await json(
await admin("PUT", `/sites/${siteId}/credentials`, {
authKind: "cookie",
credential: "c_secure_uid=1; pass=abc",
}),
);
expect(r1.data).toMatchObject({ created: true, label: "default", authKind: "cookie" });
expect(JSON.stringify(r1)).not.toContain("pass=abc");
expect(await h.ctx.secrets.resolve(r1.data.secretRef)).toBe("c_secure_uid=1; pass=abc");
const r2 = await json(
await admin("PUT", `/sites/${siteId}/credentials`, {
authKind: "password",
credential: "pw-123456",
username: "me",
}),
);
expect(r2.data).toMatchObject({ created: false, siteAccountId: r1.data.siteAccountId });
await expect(h.ctx.secrets.resolve(r1.data.secretRef)).rejects.toThrow(/不存在/);
const acc = await sql<{ auth_kind: string; user_info: { username?: string } }>`
select auth_kind, user_info from site_account where id = ${r1.data.siteAccountId}`.execute(h.db);
expect(acc.rows[0]).toMatchObject({ auth_kind: "password", user_info: { username: "me" } });
expect(
(
await admin("PUT", "/sites/00000000-0000-4000-8000-000000000000/credentials", {
authKind: "cookie",
credential: "x",
})
).status,
).toBe(404);
noLeak("pass=abc");
noLeak("pw-123456");
});
}); });
describe("审计查询", () => { describe("审计查询", () => {
@@ -289,11 +252,9 @@ describe.skipIf(!hasPg)("W2-A 管理 API(真实 PG)", () => {
const ids = [...p1.data.items, ...p2.data.items].map((e: { id: string }) => e.id); const ids = [...p1.data.items, ...p2.data.items].map((e: { id: string }) => e.id);
expect(new Set(ids).size).toBe(6); expect(new Set(ids).size).toBe(6);
expect(p1.data.items[0].at >= p2.data.items[0].at).toBe(true); expect(p1.data.items[0].at >= p2.data.items[0].at).toBe(true);
const t = await json(await admin("GET", "/audit-events?targetType=site_account")); const t = await json(await admin("GET", "/audit-events?targetType=api_key&limit=200"));
expect(t.data.items.map((e: { action: string }) => e.action)).toEqual([ expect(t.data.items.length).toBeGreaterThan(0);
"site_account.credentials_set", expect(t.data.items.every((e: { action: string }) => e.action.startsWith("api_key."))).toBe(true);
"site_account.credentials_set",
]);
}); });
}); });
+4 -2
View File
@@ -11,6 +11,7 @@ import { EventBus } from "../../src/http/sse.ts";
import { createSourcesIdentityModule } from "../../src/modules/sources-identity/index.ts"; import { createSourcesIdentityModule } from "../../src/modules/sources-identity/index.ts";
import type { RuntimeOverrides } from "../../src/modules/sources-identity/runtime.ts"; import type { RuntimeOverrides } from "../../src/modules/sources-identity/runtime.ts";
import type { SecretStore, ServerContext, SettingsReader } from "../../src/modules/types.ts"; import type { SecretStore, ServerContext, SettingsReader } from "../../src/modules/types.ts";
import { withSchemaLock } from "../auth-settings/harness.ts";
import { DEV_KEY, memoryLogger, memoryStore } from "../helpers.ts"; import { DEV_KEY, memoryLogger, memoryStore } from "../helpers.ts";
export const READ_KEY = "w2b-read-key"; export const READ_KEY = "w2b-read-key";
@@ -141,7 +142,8 @@ export async function createHarness(name: string, opts: HarnessOptions = {}): Pr
pool: new pg.Pool({ connectionString: PG_URL, max: 8, options: `-c search_path=${schema},public` }), pool: new pg.Pool({ connectionString: PG_URL, max: 8, options: `-c search_path=${schema},public` }),
}), }),
}) as unknown as Kysely<DB>; }) as unknown as Kysely<DB>;
await migrateToLatest(db, { migrationTableSchema: schema }); // 与其他测试进程共用 PG:迁移内省会看到别处正在删除的 schema,用同一把咨询锁排队并重试
await withSchemaLock(() => migrateToLatest(db, { migrationTableSchema: schema }));
const { logger } = memoryLogger(); const { logger } = memoryLogger();
const engine = fakeEngine(); const engine = fakeEngine();
@@ -206,7 +208,7 @@ export async function createHarness(name: string, opts: HarnessOptions = {}): Pr
}, },
async close() { async close() {
await db.destroy(); await db.destroy();
await sql.raw(`drop schema if exists ${schema} cascade`).execute(admin); await withSchemaLock(() => sql.raw(`drop schema if exists ${schema} cascade`).execute(admin));
await admin.destroy(); await admin.destroy();
}, },
}; };
@@ -69,12 +69,13 @@ describe.skipIf(!up)("M-Team 快照与任务注册", () => {
api.setFailMode(null); api.setFailMode(null);
}); });
it("模块注册 4 个任务与 3 条 outbox 路由;任务处理器经 step.run 调服务", async () => { it("模块注册 6 个任务与 3 条 outbox 路由;任务处理器经 step.run 调服务", async () => {
const reg = new TaskRegistry(); const reg = new TaskRegistry();
h.module.registerTasks(reg, h.ctx); h.module.registerTasks(reg, h.ctx);
expect(reg.names()).toEqual([ expect(reg.names()).toEqual([
"ingestFeed", "ingestFeed",
"refreshMetadata", "refreshMetadata",
"refreshSiteAccount",
"resolveObservation", "resolveObservation",
"searchSubscription", "searchSubscription",
"snapshotSite", "snapshotSite",
@@ -0,0 +1,328 @@
import { createMockMteamApi, createMockNexusPhpSite, type MockMteamApi, type MockNexusPhpSite } from "@umt/testkit";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { scheduleRegistry } from "../../src/modules/schedules/registry.ts";
import { refreshSiteAccount, tickSiteUserData } from "../../src/modules/sources-identity/service/account.ts";
import { createHarness, type Harness, pgAvailable } from "./harness.ts";
// W9-A 站点中心:凭证(账密 / Cookie / API Key)、刷新登录、用户数据与历史、健康检测、删除、定时任务登记。
const up = await pgAvailable();
const TiB = 1024 ** 4;
// biome-ignore lint/suspicious/noExplicitAny: 响应体断言
const json = async (r: Response): Promise<any> => {
const body = await r.json();
if (!r.ok) throw new Error(`${r.status} ${JSON.stringify(body)}`);
return body.data;
};
describe.skipIf(!up)("站点中心(mock NexusPHP)", () => {
let h: Harness;
let mock: MockNexusPhpSite;
let siteId: string;
const put = (body: unknown) => h.req(`/sites/${siteId}/credentials`, { method: "PUT", body });
const refresh = async (accountId?: string) =>
(
await json(
await h.req(`/sites/${siteId}/refresh-login`, { body: { wait: true, ...(accountId ? { accountId } : {}) } }),
)
).result;
const account = (label = "default") =>
h.db
.selectFrom("site_account")
.selectAll()
.where("site_id", "=", siteId)
.where("label", "=", label)
.executeTakeFirstOrThrow();
const openDecisions = (accountId: string) =>
h.db
.selectFrom("decision")
.selectAll()
.where("dedupe_key", "=", `site-login:${accountId}`)
.where("status", "=", "open")
.execute();
beforeAll(async () => {
mock = await createMockNexusPhpSite({ username: "alice", password: "s3cret-pw" });
h = await createHarness("sitecenter");
const site = await json(
await h.req("/sites", {
body: {
key: "audiences",
name: "观众",
adapter: "nexusphp",
baseUrl: `${mock.baseUrl}/`,
rateLimit: { minIntervalMs: 0, concurrency: 10 },
},
}),
);
siteId = site.id;
expect(site).toMatchObject({ proxy: false, lastHealth: null, lastHealthAt: null });
});
afterAll(async () => {
await h?.close();
await mock?.close();
});
it("账密登录成功:会话 Cookie 存 secret、状态 active、用户数据写入当日历史;密码不回显", async () => {
const r = await json(await put({ kind: "password", username: "alice", password: "s3cret-pw" }));
expect(r).toMatchObject({ created: true, authKind: "password", label: "default" });
expect(JSON.stringify(r)).not.toContain("s3cret-pw");
const res = await refresh();
expect(res).toMatchObject({ ok: true, status: "active", reason: null });
expect(res.userData).toMatchObject({
username: "mockuser",
userLevel: "Elite User",
upload: 12.5 * TiB,
ratio: 20.48,
seeding: 31,
vip: null,
});
const acc = await account();
expect(acc.status).toBe("active");
expect(acc.last_login_at).toBeTruthy();
expect(acc.session_secret_ref).toBeTruthy();
expect(h.secrets.plain.get(acc.session_secret_ref!)).toBe(
mock.cookie
.split(";")
.map((s) => s.trim())
.join("; "),
);
expect(acc.login_name).toBe("alice");
const hist = await json(await h.req(`/sites/${siteId}/user-data?days=30`));
expect(hist.items).toHaveLength(1);
expect(hist.items[0]).toMatchObject({ siteAccountId: acc.id, upload: 12.5 * TiB, seeding: 31 });
// 同日再刷新:upsert 不新增行
await refresh();
expect((await json(await h.req(`/sites/${siteId}/user-data`))).items).toHaveLength(1);
// 列表带账号摘要,secret 引用不外露
const list = await json(await h.req("/sites?include=accounts"));
const item = list.items.find((s: { id: string }) => s.id === siteId);
expect(item.accountStatus).toBe("active");
expect(item.accounts[0]).toMatchObject({ hasSession: true, hasTotp: false, loginName: "alice", status: "active" });
expect(item.accounts[0].userData.upload).toBe(12.5 * TiB);
expect(JSON.stringify(list)).not.toMatch(/sessionSecretRef|s3cret-pw|mock-session-secret/);
// 会话 Cookie 给适配器用
const page = await h.rt
.siteAdapter(await h.db.selectFrom("site").selectAll().where("id", "=", siteId).executeTakeFirstOrThrow())
.list(1);
expect(page.items.length).toBeGreaterThan(0);
});
it("密码错:status invalid + 待处理(无重试选项);更新凭证后回到 unknown 并关闭待处理", async () => {
await json(await put({ kind: "password", label: "bad", username: "alice", password: "wrong" }));
const acc = await account("bad");
const res = await refresh(acc.id);
expect(res).toMatchObject({ ok: false, status: "invalid", reason: "bad_credentials" });
const ds = await openDecisions(acc.id);
expect(ds).toHaveLength(1);
expect((ds[0]!.options as { key: string }[]).map((o) => o.key)).toEqual(["use_cookie", "dismiss"]);
await json(await put({ kind: "password", label: "bad", username: "alice", password: "s3cret-pw" }));
expect((await account("bad")).status).toBe("unknown");
expect(await openDecisions(acc.id)).toHaveLength(0);
await json(await h.req(`/site-accounts/${acc.id}`, { method: "DELETE" }));
});
it("验证码 / 二次验证:status challenge + 待处理提示改用 Cookie;TOTP 密钥可过二次验证", async () => {
await json(await put({ kind: "password", label: "cap", username: "alice", password: "s3cret-pw" }));
const acc = await account("cap");
mock.setLoginGuard("captcha");
const r1 = await refresh(acc.id);
expect(r1).toMatchObject({ ok: false, status: "challenge", reason: "captcha" });
const [d] = await openDecisions(acc.id);
expect(d!.title).toContain("登录需要人工验证");
expect(d!.summary).toContain("改用 Cookie");
expect((d!.options as { key: string }[]).map((o) => o.key)).toEqual(["use_cookie", "retry", "dismiss"]);
mock.setLoginGuard("two_factor");
const r2 = await refresh(acc.id);
expect(r2).toMatchObject({ ok: false, status: "challenge", reason: "two_factor" });
await json(
await put({
kind: "password",
label: "cap",
username: "alice",
password: "s3cret-pw",
totpSecret: "JBSWY3DPEHPK3PXP",
}),
);
const r3 = await refresh(acc.id);
expect(r3).toMatchObject({ ok: true, status: "active" });
expect((await account("cap")).totp_secret_ref).toBeTruthy();
mock.setLoginGuard("none");
await json(await h.req(`/site-accounts/${acc.id}`, { method: "DELETE" }));
});
it("只换 Cookie:登录成功后凭证类型落为 cookie,密码丢弃", async () => {
await json(
await put({ kind: "password", label: "co", username: "alice", password: "s3cret-pw", saveCookieOnly: true }),
);
const before = await account("co");
const res = await refresh(before.id);
expect(res.ok).toBe(true);
const after = await account("co");
expect(after.auth_kind).toBe("cookie");
expect(after.session_secret_ref).toBeNull();
expect(h.secrets.plain.has(before.secret_ref!)).toBe(false);
expect(h.secrets.plain.get(after.secret_ref!)).toContain("c_secure_pass");
await json(await h.req(`/site-accounts/${after.id}`, { method: "DELETE" }));
});
it("Cookie 失效判定:status expired + 待处理「Cookie 已失效」;网络失败不改状态", async () => {
await json(await put({ kind: "cookie", label: "ck", cookie: "c_secure_uid=1; c_secure_pass=stale" }));
const acc = await account("ck");
const res = await refresh(acc.id);
expect(res).toMatchObject({ ok: false, status: "expired", reason: "auth_expired" });
expect(res.message).toContain("Cookie 已失效");
const [d] = await openDecisions(acc.id);
expect(d!.title).toContain("登录失效");
await json(await put({ kind: "cookie", label: "ck", cookie: mock.cookie }));
expect((await refresh(acc.id)).ok).toBe(true);
mock.failNext(1, 502);
const net = await refresh(acc.id);
expect(net).toMatchObject({ ok: false, reason: "network", status: "active" });
expect((await account("ck")).status).toBe("active");
expect((await account("ck")).last_error).toMatchObject({ reason: "network" });
await json(await h.req(`/site-accounts/${acc.id}`, { method: "DELETE" }));
});
it("refresh-login 缺省异步入队 refreshSiteAccount;定时 sites.user_data 只对 active 账号入队", async () => {
const r = await json(await h.req(`/sites/${siteId}/refresh-login`, { body: {} }));
expect(r.taskId).toBeTruthy();
expect(r.result).toBeNull();
expect(h.engine.enqueued.at(-1)).toMatchObject({ task: "refreshSiteAccount", input: { mode: "login" } });
const n = await tickSiteUserData(h.rt, "slot-1");
expect(n).toBe(1);
expect(h.engine.enqueued.at(-1)).toMatchObject({ task: "refreshSiteAccount", input: { mode: "userData" } });
// userData 模式:会话失效时账密账号自动重登
mock.setLoggedIn(false);
const acc = await account();
const res = await refreshSiteAccount(h.rt, acc.id, { mode: "userData" });
expect(res.ok).toBe(true);
expect(mock.loginAttempts).toBeGreaterThan(0);
});
it("健康检测:同步返回结构化结果并写站点健康列;未登录判 failing", async () => {
const r = await json(await h.req(`/sites/${siteId}/health-check`, { body: undefined, method: "POST" }));
expect(r).toMatchObject({ siteId, ok: true, status: "ok", loggedIn: true, healthStatus: "healthy" });
const site = await json(await h.req(`/sites/${siteId}`));
expect(site.lastHealth).toMatchObject({ ok: true, status: "ok" });
expect(site.lastHealthAt).toBeTruthy();
expect(site.healthStatus).toBe("healthy");
mock.setLoggedIn(false);
const bad = await json(await h.req(`/sites/${siteId}/health-check`, { method: "POST" }));
expect(bad).toMatchObject({ ok: false, loggedIn: false, healthStatus: "failing" });
mock.setLoggedIn(true);
});
it("删除站点:账号与凭证删除,资源保留并标站点已关闭;同一定义可再登记", async () => {
const rel = await h.db
.insertInto("source_release")
.values({ site_id: siteId, site_release_id: "x1", title: "Some.Movie.2024.2160p" })
.returning("id")
.executeTakeFirstOrThrow();
const accs = await h.db.selectFrom("site_account").selectAll().where("site_id", "=", siteId).execute();
const refs = accs.flatMap((a) => [a.secret_ref, a.session_secret_ref]).filter(Boolean) as string[];
expect(refs.length).toBeGreaterThan(0);
await json(await h.req(`/sites/${siteId}`, { method: "DELETE" }));
expect((await h.req(`/sites/${siteId}`)).status).toBe(404);
const release = await h.db
.selectFrom("source_release")
.selectAll()
.where("id", "=", rel.id)
.executeTakeFirstOrThrow();
expect(release.availability).toBe("site_disabled");
expect(await h.db.selectFrom("site_account").selectAll().where("site_id", "=", siteId).execute()).toHaveLength(0);
for (const r of refs) expect(h.secrets.plain.has(r)).toBe(false);
const list = await json(await h.req("/sites"));
expect(list.items.some((s: { id: string }) => s.id === siteId)).toBe(false);
const again = await json(
await h.req("/sites", {
body: { key: "audiences", name: "观众", adapter: "nexusphp", baseUrl: `${mock.baseUrl}/` },
}),
);
expect(again.id).not.toBe(siteId);
});
it("定时任务登记:sites.user_data 6h(一个周期后首跑)、sites.health_check 1h", async () => {
await h.module.start?.(h.ctx);
const reg = scheduleRegistry(h.ctx);
expect(reg.get("sites.user_data")?.defaults).toMatchObject({
group: "collect",
schedule: { type: "interval", everyMs: 6 * 3_600_000 },
firstRun: "after_interval",
});
expect(reg.get("sites.health_check")?.defaults).toMatchObject({
schedule: { type: "interval", everyMs: 3_600_000 },
});
await h.module.stop?.();
});
});
describe.skipIf(!up)("站点中心(mock M-Team API Key)", () => {
let h: Harness;
let api: MockMteamApi;
let siteId: string;
beforeAll(async () => {
api = await createMockMteamApi();
h = await createHarness("sitecenter_mt");
const site = await json(
await h.req("/sites", {
body: {
key: "mteam",
name: "馒头",
adapter: "mteam_api",
baseUrl: "https://kp.m-team.cc/",
rateLimit: { minIntervalMs: 0, concurrency: 10 },
config: { apiBaseUrl: api.baseUrl },
},
}),
);
siteId = site.id;
});
afterAll(async () => {
await h?.close();
await api?.close();
});
it("API Key 校验:有效 → active + 用户数据;无效 → invalid;账密不支持", async () => {
await json(
await h.req(`/sites/${siteId}/credentials`, { method: "PUT", body: { kind: "api_key", apiKey: api.apiKey } }),
);
const ok = (await json(await h.req(`/sites/${siteId}/refresh-login`, { body: { wait: true } }))).result;
expect(ok).toMatchObject({ ok: true, status: "active" });
expect(ok.userData).toMatchObject({ userId: "10001", username: "mockuser", upload: 1000 });
await json(
await h.req(`/sites/${siteId}/credentials`, { method: "PUT", body: { kind: "api_key", apiKey: "wrong-key" } }),
);
const bad = (await json(await h.req(`/sites/${siteId}/refresh-login`, { body: { wait: true } }))).result;
expect(bad).toMatchObject({ ok: false, status: "invalid", reason: "auth_expired" });
expect(bad.message).toContain("API Key 无效");
await json(
await h.req(`/sites/${siteId}/credentials`, {
method: "PUT",
body: { kind: "password", label: "pw", username: "u", password: "p" },
}),
);
const acc = await h.db.selectFrom("site_account").select("id").where("label", "=", "pw").executeTakeFirstOrThrow();
const pw = (await json(await h.req(`/sites/${siteId}/refresh-login`, { body: { wait: true, accountId: acc.id } })))
.result;
expect(pw).toMatchObject({ ok: false, reason: "unsupported", status: "unknown" });
});
it("旧格式凭证 {authKind, credential} 仍可用", async () => {
const r = await json(
await h.req(`/sites/${siteId}/credentials`, {
method: "PUT",
body: { label: "legacy", authKind: "passkey", credential: "pk-123" },
}),
);
expect(r).toMatchObject({ created: true, authKind: "passkey" });
expect(JSON.stringify(r)).not.toContain("pk-123");
});
});
@@ -76,6 +76,7 @@ export const ACCOUNT_STATUS: Record<SiteAccount["status"], { label: string; tone
active: { label: "有效", tone: "success" }, active: { label: "有效", tone: "success" },
invalid: { label: "凭证无效", tone: "destructive" }, invalid: { label: "凭证无效", tone: "destructive" },
expired: { label: "已过期", tone: "warning" }, expired: { label: "已过期", tone: "warning" },
challenge: { label: "需验证", tone: "warning" },
}; };
/** 站点下的账号总体状态:有效优先,其次最差的那个 */ /** 站点下的账号总体状态:有效优先,其次最差的那个 */