import { createMockRssServer, type MockFeedItem, type MockRssServer } from "@umt/testkit"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { schedulerTick } from "../../src/modules/sources-identity/scheduler.ts"; import { extractSiteReleaseId, ingestFeed } from "../../src/modules/sources-identity/service/feeds.ts"; import { sourcesOutboxRoutes } from "../../src/modules/sources-identity/tasks.ts"; import { createHarness, type Harness, pgAvailable, seedFeed, seedSite } from "./harness.ts"; const up = await pgAvailable(); const BASE = "https://audiences.example"; const item = (guid: string, id: string, title: string, extra: Partial = {}): MockFeedItem => ({ guid, title, link: `${BASE}/details.php?id=${id}&hit=1`, enclosureUrl: `${BASE}/download.php?id=${id}&passkey=SECRETPK123`, size: 1024 * 1024 * 1024, pubDate: new Date("2026-09-20T08:00:00Z"), categories: ["电影"], ...extra, }); describe.skipIf(!up)("ingestFeed", () => { let h: Harness; const servers: MockRssServer[] = []; let siteId: string; let accountId: string | null; beforeAll(async () => { h = await createHarness("ingest"); const s = await seedSite(h, { baseUrl: `${BASE}/`, cookie: "c=1" }); siteId = s.site.id; accountId = s.accountId; }); afterAll(async () => { for (const s of servers) await s.close(); await h?.close(); }); const feedServer = async (items: MockFeedItem[], passkey = "FEEDPK") => { const s = await createMockRssServer({ items, passkey }); servers.push(s); return s; }; it("从链接推出站点资源 ID", () => { expect(extractSiteReleaseId({ guid: "x", link: `${BASE}/details.php?id=812345&hit=1` })).toBe("812345"); expect(extractSiteReleaseId({ guid: "x", link: "https://kp.m-team.cc/detail/912233" })).toBe("912233"); expect(extractSiteReleaseId({ guid: "abc", link: null })).toBeNull(); expect(extractSiteReleaseId({ guid: "12345", link: null })).toBe("12345"); }); it("三个 Feed 看到同一资源只建一条 source_release,保留三条观察;release.observed 只发一次", async () => { const title = "Dune.Part.Two.2024.2160p.UHD.BluRay.REMUX-FraMeSToR"; const a = await feedServer([item("movie-1", "812345", title)]); const b = await feedServer([item("promo-xyz", "812345", title)]); const c = await feedServer([item("https://audiences.example/details.php?id=812345", "812345", title)]); const feeds = [ await seedFeed(h, { siteId, url: a.feedUrl, name: "电影", purpose: "movie", accountId }), await seedFeed(h, { siteId, url: b.feedUrl, name: "促销", purpose: "promotion" }), await seedFeed(h, { siteId, url: c.feedUrl, name: "综合" }), ]; for (const f of feeds) expect((await ingestFeed(h.rt, f.id)).status).toBe("ok"); const rels = await h.db.selectFrom("source_release").selectAll().where("site_release_id", "=", "812345").execute(); expect(rels).toHaveLength(1); const obs = await h.db.selectFrom("observation").selectAll().where("source_release_id", "=", rels[0]!.id).execute(); expect(new Set(obs.map((o) => o.feed_id)).size).toBe(3); expect(obs.every((o) => o.status === "linked")).toBe(true); const observed = (await h.outbox("release.observed")).filter((e) => e.payload.sourceReleaseId === rels[0]!.id); expect(observed).toHaveLength(1); expect(observed[0]!.payload.isNew).toBe(true); // 时间:来源发布时间与首次发现分开保存 expect(rels[0]!.source_published_at?.toISOString()).toBe("2026-09-20T08:00:00.000Z"); expect(rels[0]!.first_seen_at.getTime()).toBeGreaterThan(rels[0]!.source_published_at!.getTime()); // feed 状态持久化:游标与条件请求头 const f0 = await h.db .selectFrom("feed_source") .selectAll() .where("id", "=", feeds[0]!.id) .executeTakeFirstOrThrow(); expect(f0.health_status).toBe("healthy"); expect(f0.etag).toBeTruthy(); expect((f0.cursor as { window: { lastGuidSet: string[] } }).window.lastGuidSet).toContain("movie-1"); expect(f0.next_fetch_at!.getTime()).toBeGreaterThan(Date.now()); }); it("enclosure/passkey 只存脱敏值;未变化时 304 不产生新条目", async () => { const s = await feedServer([item("g-1", "900001", "Some.Movie.2020.1080p.WEB-DL-GRP")], "TOPSECRETKEY"); const f = await seedFeed(h, { siteId, url: s.feedUrl, name: "脱敏" }); await ingestFeed(h.rt, f.id); const dump = JSON.stringify(await h.db.selectFrom("observation").selectAll().where("feed_id", "=", f.id).execute()) + JSON.stringify( await h.db.selectFrom("source_release").selectAll().where("site_release_id", "=", "900001").execute(), ) + JSON.stringify(await h.db.selectFrom("feed_source").selectAll().where("id", "=", f.id).execute()); expect(dump).not.toContain("SECRETPK123"); expect(dump).not.toContain("TOPSECRETKEY"); expect(dump).toContain("[REDACTED]"); const again = await ingestFeed(h.rt, f.id); expect(again.status).toBe("not_modified"); expect(s.notModifiedCount).toBe(1); }); it("同 GUID 内容变化视为更新:观察与资源标题更新,重新触发识别", async () => { const s = await feedServer([item("upd-1", "900100", "Show.2021.1080p.WEB-DL-AAA")]); const f = await seedFeed(h, { siteId, url: s.feedUrl, name: "更新" }); await ingestFeed(h.rt, f.id); s.updateItem("upd-1", { title: "Show.2021.2160p.WEB-DL-AAA" }); const r = await ingestFeed(h.rt, f.id); expect(r).toMatchObject({ status: "ok", newItems: 0, updatedItems: 1, newReleases: 0 }); const obs = await h.db.selectFrom("observation").selectAll().where("feed_id", "=", f.id).execute(); expect(obs).toHaveLength(1); expect(obs[0]!.title).toBe("Show.2021.2160p.WEB-DL-AAA"); const rel = await h.db .selectFrom("source_release") .selectAll() .where("site_release_id", "=", "900100") .executeTakeFirstOrThrow(); expect(rel.title).toBe("Show.2021.2160p.WEB-DL-AAA"); const ev = (await h.outbox("release.observed")).filter((e) => e.payload.sourceReleaseId === rel.id); expect(ev.map((e) => e.payload.isNew)).toEqual([true, false]); }); it("窗口零重叠 → feed.gap_suspected,并路由到有预算的补漏快照", async () => { const s = await feedServer([item("gap-1", "900201", "A.2020.1080p-X"), item("gap-2", "900202", "B.2020.1080p-X")]); const f = await seedFeed(h, { siteId, url: s.feedUrl, name: "缺口" }); await ingestFeed(h.rt, f.id); s.setItems([item("gap-9", "900209", "C.2020.1080p-X")]); const r = await ingestFeed(h.rt, f.id); expect(r.possibleGap).toBe(true); const gap = (await h.outbox("feed.gap_suspected")).find((e) => e.payload.feedId === f.id); expect(gap).toBeDefined(); const route = sourcesOutboxRoutes["feed.gap_suspected"]!({ id: "1", type: "feed.gap_suspected", payload: gap!.payload, at: new Date(), }); expect(route).toMatchObject({ task: "snapshotSite", input: { siteId, kind: "backfill", maxPages: 3 } }); // 响应派生字段 possibleGap;下一次衔接上的抓取后清空 const got = (await (await h.req(`/feed-sources/${f.id}`)).json()) as { data: { possibleGap: { backfillQueued: boolean } | null }; }; expect(got.data.possibleGap).toMatchObject({ backfillQueued: true }); s.setItems([item("gap-9", "900209", "C.2020.1080p-X"), item("gap-10", "900210", "D.2020.1080p-X")]); await ingestFeed(h.rt, f.id); const after = (await (await h.req(`/feed-sources/${f.id}`)).json()) as { data: { possibleGap: unknown } }; expect(after.data.possibleGap).toBeNull(); }); it("登录失效(403):账号标失效 + 待处理,不重试", async () => { const s = await feedServer([item("auth-1", "900300", "X.2020.1080p-Y")], "RIGHTKEY"); const badUrl = s.feedUrl.replace("RIGHTKEY", "WRONGKEY"); const f = await seedFeed(h, { siteId, url: badUrl, name: "失效", accountId }); const r = await ingestFeed(h.rt, f.id); expect(r.status).toBe("auth_failed"); const acc = await h.db .selectFrom("site_account") .selectAll() .where("id", "=", accountId!) .executeTakeFirstOrThrow(); expect(acc.status).toBe("invalid"); const d = await h.db .selectFrom("decision") .selectAll() .where("dedupe_key", "=", `feed-auth:${f.id}`) .executeTakeFirstOrThrow(); expect(d.kind).toBe("source_unreachable"); expect(JSON.stringify(d)).not.toContain("WRONGKEY"); const row = await h.db.selectFrom("feed_source").selectAll().where("id", "=", f.id).executeTakeFirstOrThrow(); expect(row.health_status).toBe("failing"); expect(row.backoff_until!.getTime()).toBeGreaterThan(Date.now() + 23 * 3600_000); // 再次调度:退避期内不发请求 const before = s.requests.length; const again = await ingestFeed(h.rt, f.id); expect(again.status).toBe("skipped"); expect(s.requests.length).toBe(before); // 调度器也不会把它当作到期 await h.db .updateTable("feed_source") .set({ next_fetch_at: new Date(0) }) .where("id", "=", f.id) .execute(); await schedulerTick(h.rt); const due = (await h.outbox("feed.due")).filter((e) => e.payload.feedId === f.id); expect(due).toHaveLength(0); // 恢复账号给后续用例 await h.db.updateTable("site_account").set({ status: "active" }).where("id", "=", accountId!).execute(); }); it("调度器:到期 Feed 发 feed.due(幂等),快照到期入队 snapshotSite", async () => { const s = await feedServer([item("sch-1", "900400", "S.2020.1080p-Z")]); const f = await seedFeed(h, { siteId, url: s.feedUrl, name: "调度" }); const t1 = await schedulerTick(h.rt); expect(t1.feedsDue).toBeGreaterThanOrEqual(1); const due = (await h.outbox("feed.due")).filter((e) => e.payload.feedId === f.id); expect(due).toHaveLength(1); expect( sourcesOutboxRoutes["feed.due"]!({ id: "1", type: "feed.due", payload: due[0]!.payload, at: new Date() }), ).toEqual({ task: "ingestFeed", input: { feedSourceId: f.id, manual: false }, }); // next_fetch_at 已顺延:紧接着的 tick 不再重复 await schedulerTick(h.rt); expect((await h.outbox("feed.due")).filter((e) => e.payload.feedId === f.id)).toHaveLength(1); expect( h.engine.enqueued.some((e) => e.task === "snapshotSite" && (e.input as { siteId: string }).siteId === siteId), ).toBe(true); }); it("站点关闭后 Feed 不再抓取", async () => { const s = await feedServer([item("off-1", "900500", "O.2020.1080p-Z")]); const f = await seedFeed(h, { siteId, url: s.feedUrl, name: "关站" }); await h.db.updateTable("site").set({ enabled: false }).where("id", "=", siteId).execute(); const r = await ingestFeed(h.rt, f.id); expect(r).toMatchObject({ status: "skipped", reason: "site_disabled" }); expect(s.requests.length).toBe(0); await h.db.updateTable("site").set({ enabled: true }).where("id", "=", siteId).execute(); }); });