rootandClaude Opus 5.5
61f585a707
Migration 0041_rate_limit adds rate_limit_bucket. PgTokenBucket (@umt/db) refills and takes tokens under SELECT ... FOR UPDATE on the DB clock and polls with jitter within a wait budget; penalize() carries Retry-After. Site adapters get createSharedSiteRateLimiter (key site:<id>), shared by api / worker / worker:agent, with an in-process serial queue in front and a local fallback when the bucket is unavailable. Co-Authored-By: Claude Opus 5.5 <[email protected]>
252 lines
13 KiB
TypeScript
252 lines
13 KiB
TypeScript
import { createMockNexusPhpSite, type MockNexusPhpSite } from "@umt/testkit";
|
|
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
|
import { snapshotSite } from "../../src/modules/sources-identity/service/snapshot.ts";
|
|
import { liveSearch } from "../../src/modules/sources-identity/service/workbench.ts";
|
|
import { createHarness, EXEC_KEY, type Harness, json, pgAvailable, READ_KEY } from "./harness.ts";
|
|
|
|
const up = await pgAvailable();
|
|
|
|
describe.skipIf(!up)("来源管理与站点工作台 API", () => {
|
|
let h: Harness;
|
|
let mock: MockNexusPhpSite;
|
|
let siteId: string;
|
|
|
|
beforeAll(async () => {
|
|
mock = await createMockNexusPhpSite();
|
|
h = await createHarness("api");
|
|
});
|
|
afterAll(async () => {
|
|
await h?.close();
|
|
await mock?.close();
|
|
});
|
|
|
|
it("鉴权:无 key 401,只读 key 写站点 403,organize.execute 不能改站点配置", async () => {
|
|
expect((await h.app.request("/api/v1/sites")).status).toBe(401);
|
|
expect((await h.req("/sites", { key: READ_KEY })).status).toBe(200);
|
|
const body = { key: "audiences", name: "观众", adapter: "nexusphp", baseUrl: `${mock.baseUrl}/` };
|
|
expect((await h.req("/sites", { key: READ_KEY, body })).status).toBe(403);
|
|
expect((await h.req("/sites", { key: EXEC_KEY, body })).status).toBe(403);
|
|
});
|
|
|
|
it("站点注册表 + 登记站点;适配器类型与定义不符 400;重复 409;校验失败 422", async () => {
|
|
const defs = await json(await h.req("/site-definitions"));
|
|
expect(defs.status).toBe(200);
|
|
const aud = defs.body.data.items.find((d: { key: string }) => d.key === "audiences");
|
|
expect(aud).toMatchObject({ adapter: "nexusphp", siteId: null, shippedEnabled: true });
|
|
expect(defs.body.data.items.find((d: { key: string }) => d.key === "mteam")?.adapter).toBe("mteam_api");
|
|
|
|
const bad = await json(
|
|
await h.req("/sites", {
|
|
body: { key: "audiences", name: "观众", adapter: "mteam_api", baseUrl: `${mock.baseUrl}/` },
|
|
}),
|
|
);
|
|
expect(bad.status).toBe(400);
|
|
const unknown = await json(
|
|
await h.req("/sites", { body: { key: "nosuch", name: "x", adapter: "nexusphp", baseUrl: "https://x.example/" } }),
|
|
);
|
|
expect(unknown.status).toBe(400);
|
|
expect(
|
|
(await h.req("/sites", { body: { key: "BAD KEY", name: "x", adapter: "nexusphp", baseUrl: "nope" } })).status,
|
|
).toBe(422);
|
|
|
|
const ok = await json(
|
|
await h.req("/sites", {
|
|
body: {
|
|
key: "audiences",
|
|
name: "观众",
|
|
adapter: "nexusphp",
|
|
baseUrl: `${mock.baseUrl}/`,
|
|
rateLimit: { minIntervalMs: 0, concurrency: 4 },
|
|
snapshotEnabled: true,
|
|
snapshotIntervalSec: 3600,
|
|
},
|
|
}),
|
|
);
|
|
expect(ok.status).toBe(200);
|
|
siteId = ok.body.data.id;
|
|
expect(
|
|
(
|
|
await h.req("/sites", {
|
|
body: { key: "audiences", name: "观众", adapter: "nexusphp", baseUrl: `${mock.baseUrl}/` },
|
|
})
|
|
).status,
|
|
).toBe(409);
|
|
const defs2 = await json(await h.req("/site-definitions"));
|
|
expect(defs2.body.data.items.find((d: { key: string }) => d.key === "audiences").siteId).toBe(siteId);
|
|
});
|
|
|
|
it("站点账号:凭证只写(存 secret),响应里只有 secret_ref", async () => {
|
|
const r = await json(
|
|
await h.req("/site-accounts", { body: { siteId, label: "主号", authKind: "cookie", credential: mock.cookie } }),
|
|
);
|
|
expect(r.status).toBe(200);
|
|
expect(JSON.stringify(r.body)).not.toContain(mock.cookie);
|
|
expect(h.secrets.plain.get(r.body.data.secretRef)).toBe(mock.cookie);
|
|
expect(
|
|
(await h.req("/site-accounts", { body: { siteId, label: "主号", authKind: "cookie", credential: "x" } })).status,
|
|
).toBe(409);
|
|
});
|
|
|
|
it("Feed CRUD:URL 存 secret,对外只给脱敏 URL;启停;立即抓取入队", async () => {
|
|
const url = "https://audiences.example/torrentrss.php?rows=50&passkey=abcdef0123456789";
|
|
const c = await json(
|
|
await h.req("/feed-sources", { body: { siteId, name: "电影", purpose: "movie", url, pollIntervalSec: 600 } }),
|
|
);
|
|
expect(c.status).toBe(200);
|
|
expect(c.body.data.urlDisplay).toContain("passkey=%5BREDACTED%5D".slice(0, 8));
|
|
expect(JSON.stringify(c.body)).not.toContain("abcdef0123456789");
|
|
const id = c.body.data.id;
|
|
expect((await h.req("/feed-sources", { body: { siteId, name: "x", url: "not a url" } })).status).toBe(422);
|
|
const off = await json(await h.req(`/feed-sources/${id}/disable`, { body: {} }));
|
|
expect(off.body.data).toMatchObject({ enabled: false, healthStatus: "disabled" });
|
|
expect((await h.req(`/feed-sources/${id}/fetch`, { body: {} })).status).toBe(412);
|
|
await h.req(`/feed-sources/${id}/enable`, { body: {} });
|
|
const f = await json(await h.req(`/feed-sources/${id}/fetch`, { body: {} }));
|
|
expect(f.status).toBe(200);
|
|
expect(h.engine.enqueued.at(-1)).toMatchObject({ task: "ingestFeed", input: { feedSourceId: id, manual: true } });
|
|
const patched = await json(
|
|
await h.req(`/feed-sources/${id}`, {
|
|
method: "PATCH",
|
|
body: { url: "https://audiences.example/rss?passkey=NEWPASSKEY99" },
|
|
}),
|
|
);
|
|
expect(JSON.stringify(patched.body)).not.toContain("NEWPASSKEY99");
|
|
expect(h.secrets.plain.get(patched.body.data.urlSecretRef)).toContain("NEWPASSKEY99");
|
|
expect((await h.req("/feed-sources/00000000-0000-4000-8000-000000000000")).status).toBe(404);
|
|
});
|
|
|
|
it("快照:列表页 upsert 到 source_release,记录覆盖范围;站点级限流器每站一个实例", async () => {
|
|
const r = await snapshotSite(h.rt, { siteId, kind: "periodic" });
|
|
expect(r.status).toBe("ok");
|
|
expect(r.pages).toEqual([1]);
|
|
expect(r.newCount).toBe(r.itemCount);
|
|
const limiter = h.rt.limiterOf(siteId);
|
|
expect(limiter).not.toBeNull();
|
|
await snapshotSite(h.rt, { siteId, kind: "manual" });
|
|
expect(h.rt.limiterOf(siteId)).toBe(limiter);
|
|
expect(limiter!.stats.granted).toBeGreaterThanOrEqual(2);
|
|
const snaps = await json(await h.req(`/sites/${siteId}/snapshots`));
|
|
expect(snaps.body.data.items[0]).toMatchObject({ kind: "manual", status: "ok", pages: [1] });
|
|
const observed = await h.outbox("release.observed");
|
|
expect(observed.length).toBe(r.newCount);
|
|
const site = await json(await h.req(`/sites/${siteId}`));
|
|
expect(site.body.data.lastSnapshotAt).not.toBeNull();
|
|
// 限流间隔 > 0 时令牌在 PG 跨进程共享(rate_limit_bucket,key = site:<id>);0 = 不限流只走本地
|
|
await h.db
|
|
.updateTable("site")
|
|
.set({ rate_limit: JSON.stringify({ minIntervalMs: 5, concurrency: 5 }) })
|
|
.where("id", "=", siteId)
|
|
.execute();
|
|
expect((await snapshotSite(h.rt, { siteId, kind: "manual" })).status).toBe("ok");
|
|
const bucket = await h.db
|
|
.selectFrom("rate_limit_bucket")
|
|
.selectAll()
|
|
.where("key", "=", `site:${siteId}`)
|
|
.executeTakeFirst();
|
|
expect(bucket).toMatchObject({ capacity: 5, refill_per_sec: 200 });
|
|
});
|
|
|
|
it("工作台列表:快照优先,标注来源与更新时间;筛选与分页", async () => {
|
|
const l = await json(await h.req(`/sites/${siteId}/releases?limit=2`));
|
|
expect(l.status).toBe(200);
|
|
expect(l.body.data.items).toHaveLength(2);
|
|
expect(l.body.data.nextCursor).not.toBeNull();
|
|
expect(l.body.data.coverage).toMatchObject({ siteEnabled: true, stale: false, live: false });
|
|
expect(l.body.data.items[0]).toMatchObject({ origin: "snapshot", entity: null });
|
|
expect(l.body.data.items[0].dataUpdatedAt).toBeTruthy();
|
|
const p2 = await json(await h.req(`/sites/${siteId}/releases?limit=2&cursor=${l.body.data.nextCursor}`));
|
|
expect(p2.body.data.items[0].id).not.toBe(l.body.data.items[0].id);
|
|
const free = await json(await h.req(`/sites/${siteId}/releases?promotion=free`));
|
|
expect(free.body.data.items.every((i: { promotion: { kind: string } }) => i.promotion.kind === "free")).toBe(true);
|
|
const q = await json(await h.req(`/sites/${siteId}/releases?q=Dune&live=false`));
|
|
expect(q.body.data.items.map((i: { siteReleaseId: string }) => i.siteReleaseId)).toContain("812345");
|
|
});
|
|
|
|
it("实时补查:并发相同搜索合并为一次站点请求;API 标注实时来源", async () => {
|
|
const site = await h.db.selectFrom("site").selectAll().where("id", "=", siteId).executeTakeFirstOrThrow();
|
|
const searches = () => mock.requests.filter((r) => r.path.includes("search=")).length;
|
|
const before = searches();
|
|
const [a, b] = await Promise.all([liveSearch(h.rt, site, "Bear"), liveSearch(h.rt, site, "bear")]);
|
|
expect(searches() - before).toBe(1);
|
|
expect([a.coalesced, b.coalesced].sort()).toEqual([false, true]);
|
|
expect(a.error).toBeNull();
|
|
const r = await json(await h.req(`/sites/${siteId}/releases?q=Last&live=true`));
|
|
expect(r.body.data.coverage).toMatchObject({ live: true, liveError: null });
|
|
expect(r.body.data.items.map((i: { siteReleaseId: string }) => i.siteReleaseId)).toContain("812348");
|
|
const snaps = await h.db.selectFrom("site_snapshot").selectAll().where("kind", "=", "live_search").execute();
|
|
expect(snaps).toHaveLength(2);
|
|
});
|
|
|
|
it("详情:易变字段复核时间;refresh 失败保留既有数据", async () => {
|
|
const rel = await h.db
|
|
.selectFrom("source_release")
|
|
.selectAll()
|
|
.where("site_release_id", "=", "812345")
|
|
.executeTakeFirstOrThrow();
|
|
const d = await json(await h.req(`/sites/${siteId}/releases/${rel.id}?refresh=true`));
|
|
expect(d.status).toBe(200);
|
|
expect(d.body.data.refreshError).toBeNull();
|
|
expect(d.body.data.volatile.recheckBeforeDownload).toBe(true);
|
|
expect(d.body.data.volatile.detailFetchedAt).not.toBeNull();
|
|
mock.setLoggedIn(false);
|
|
const d2 = await json(await h.req(`/sites/${siteId}/releases/${rel.id}?refresh=true`));
|
|
expect(d2.status).toBe(200);
|
|
expect(d2.body.data.refreshError).toMatch(/auth/);
|
|
expect(d2.body.data.release.title).toBe(rel.title);
|
|
// 登录失效 → 账号标失效 + 待处理,周期快照不再访问
|
|
const acc = await h.db
|
|
.selectFrom("site_account")
|
|
.selectAll()
|
|
.where("site_id", "=", siteId)
|
|
.executeTakeFirstOrThrow();
|
|
expect(["invalid", "expired"]).toContain(acc.status);
|
|
expect(
|
|
(await h.db.selectFrom("decision").selectAll().where("dedupe_key", "=", `site-auth:${siteId}`).execute()).length,
|
|
).toBe(1);
|
|
const n = mock.requests.length;
|
|
expect((await snapshotSite(h.rt, { siteId, kind: "periodic" })).reason).toBe("auth_invalid");
|
|
expect(mock.requests.length).toBe(n);
|
|
// 换凭证:账号复位、待处理关闭
|
|
mock.setLoggedIn(true);
|
|
await h.req(`/site-accounts/${acc.id}`, { method: "PATCH", body: { credential: mock.cookie } });
|
|
const d3 = await h.db
|
|
.selectFrom("decision")
|
|
.select(["status"])
|
|
.where("dedupe_key", "=", `site-auth:${siteId}`)
|
|
.executeTakeFirstOrThrow();
|
|
expect(d3.status).toBe("resolved");
|
|
expect((await snapshotSite(h.rt, { siteId, kind: "periodic" })).status).toBe("ok");
|
|
});
|
|
|
|
it("关闭站点:停止网络访问,历史资源仍可查并标注 site_disabled", async () => {
|
|
const off = await json(await h.req(`/sites/${siteId}/disable`, { body: {} }));
|
|
expect(off.body.data).toMatchObject({ enabled: false, healthStatus: "disabled" });
|
|
const n = mock.requests.length;
|
|
const l = await json(await h.req(`/sites/${siteId}/releases?q=Nothing.Here&live=true`));
|
|
expect(l.body.data.coverage).toMatchObject({ siteEnabled: false, live: false });
|
|
expect(l.body.data.coverage.liveError).toMatch(/关闭/);
|
|
expect(mock.requests.length).toBe(n);
|
|
const all = await json(await h.req(`/sites/${siteId}/releases`));
|
|
expect(all.body.data.items.length).toBeGreaterThan(0);
|
|
expect(all.body.data.items.every((i: { availability: string }) => i.availability === "site_disabled")).toBe(true);
|
|
expect((await h.req(`/sites/${siteId}/snapshot`, { body: {} })).status).toBe(412);
|
|
await h.req(`/sites/${siteId}/enable`, { body: {} });
|
|
const back = await json(await h.req(`/sites/${siteId}/releases?limit=1`));
|
|
expect(back.body.data.items[0].availability).toBe("active");
|
|
});
|
|
|
|
it("来源资源 / 观察 / 作品列表与 OpenAPI 文档", async () => {
|
|
const l = await json(await h.req(`/source-releases?siteId=${siteId}&q=Dune`));
|
|
expect(l.body.data.items.length).toBeGreaterThan(0);
|
|
expect((await h.req("/source-releases/not-a-uuid")).status).toBe(422);
|
|
expect((await json(await h.req("/observations"))).status).toBe(200);
|
|
expect((await json(await h.req("/media-entities"))).status).toBe(200);
|
|
expect((await json(await h.req("/identity-pool"))).body.data.items).toEqual([]);
|
|
const doc = await json(await h.app.request("/api/openapi.json"));
|
|
const paths = Object.keys((doc.body as unknown as { paths: Record<string, unknown> }).paths);
|
|
expect(paths).toEqual(
|
|
expect.arrayContaining(["/api/v1/sites/{id}/releases", "/api/v1/identity-pool", "/api/v1/decisions/summary"]),
|
|
);
|
|
});
|
|
});
|