UHD_Media_Tool/apps/server/test/sources-identity/ingest.test.ts

import { createMockRssServer, type MockFeedItem, type MockRssServer } from "@umt/testkit";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { schedulerTick } from "../../src/modules/sources-identity/scheduler.ts";
import { extractSiteReleaseId, ingestFeed } from "../../src/modules/sources-identity/service/feeds.ts";
import { sourcesOutboxRoutes } from "../../src/modules/sources-identity/tasks.ts";
import { createHarness, type Harness, pgAvailable, seedFeed, seedSite } from "./harness.ts";

const up = await pgAvailable();
const BASE = "https://audiences.example";

const item = (guid: string, id: string, title: string, extra: Partial<MockFeedItem> = {}): MockFeedItem => ({
  guid,
  title,
  link: `${BASE}/details.php?id=${id}&hit=1`,
  enclosureUrl: `${BASE}/download.php?id=${id}&passkey=SECRETPK123`,
  size: 1024 * 1024 * 1024,
  pubDate: new Date("2026-09-20T08:00:00Z"),
  categories: ["电影"],
  ...extra,
});

describe.skipIf(!up)("ingestFeed", () => {
  let h: Harness;
  const servers: MockRssServer[] = [];
  let siteId: string;
  let accountId: string | null;

  beforeAll(async () => {
    h = await createHarness("ingest");
    const s = await seedSite(h, { baseUrl: `${BASE}/`, cookie: "c=1" });
    siteId = s.site.id;
    accountId = s.accountId;
  });
  afterAll(async () => {
    for (const s of servers) await s.close();
    await h?.close();
  });

  const feedServer = async (items: MockFeedItem[], passkey = "FEEDPK") => {
    const s = await createMockRssServer({ items, passkey });
    servers.push(s);
    return s;
  };

  it("从链接推出站点资源 ID", () => {
    expect(extractSiteReleaseId({ guid: "x", link: `${BASE}/details.php?id=812345&hit=1` })).toBe("812345");
    expect(extractSiteReleaseId({ guid: "x", link: "https://kp.m-team.cc/detail/912233" })).toBe("912233");
    expect(extractSiteReleaseId({ guid: "abc", link: null })).toBeNull();
    expect(extractSiteReleaseId({ guid: "12345", link: null })).toBe("12345");
  });

  it("三个 Feed 看到同一资源只建一条 source_release,保留三条观察;release.observed 只发一次", async () => {
    const title = "Dune.Part.Two.2024.2160p.UHD.BluRay.REMUX-FraMeSToR";
    const a = await feedServer([item("movie-1", "812345", title)]);
    const b = await feedServer([item("promo-xyz", "812345", title)]);
    const c = await feedServer([item("https://audiences.example/details.php?id=812345", "812345", title)]);
    const feeds = [
      await seedFeed(h, { siteId, url: a.feedUrl, name: "电影", purpose: "movie", accountId }),
      await seedFeed(h, { siteId, url: b.feedUrl, name: "促销", purpose: "promotion" }),
      await seedFeed(h, { siteId, url: c.feedUrl, name: "综合" }),
    ];
    for (const f of feeds) expect((await ingestFeed(h.rt, f.id)).status).toBe("ok");

    const rels = await h.db.selectFrom("source_release").selectAll().where("site_release_id", "=", "812345").execute();
    expect(rels).toHaveLength(1);
    const obs = await h.db.selectFrom("observation").selectAll().where("source_release_id", "=", rels[0]!.id).execute();
    expect(new Set(obs.map((o) => o.feed_id)).size).toBe(3);
    expect(obs.every((o) => o.status === "linked")).toBe(true);
    const observed = (await h.outbox("release.observed")).filter((e) => e.payload.sourceReleaseId === rels[0]!.id);
    expect(observed).toHaveLength(1);
    expect(observed[0]!.payload.isNew).toBe(true);
    // 时间:来源发布时间与首次发现分开保存
    expect(rels[0]!.source_published_at?.toISOString()).toBe("2026-09-20T08:00:00.000Z");
    expect(rels[0]!.first_seen_at.getTime()).toBeGreaterThan(rels[0]!.source_published_at!.getTime());
    // feed 状态持久化:游标与条件请求头
    const f0 = await h.db
      .selectFrom("feed_source")
      .selectAll()
      .where("id", "=", feeds[0]!.id)
      .executeTakeFirstOrThrow();
    expect(f0.health_status).toBe("healthy");
    expect(f0.etag).toBeTruthy();
    expect((f0.cursor as { window: { lastGuidSet: string[] } }).window.lastGuidSet).toContain("movie-1");
    expect(f0.next_fetch_at!.getTime()).toBeGreaterThan(Date.now());
  });

  it("enclosure/passkey 只存脱敏值;未变化时 304 不产生新条目", async () => {
    const s = await feedServer([item("g-1", "900001", "Some.Movie.2020.1080p.WEB-DL-GRP")], "TOPSECRETKEY");
    const f = await seedFeed(h, { siteId, url: s.feedUrl, name: "脱敏" });
    await ingestFeed(h.rt, f.id);
    const dump =
      JSON.stringify(await h.db.selectFrom("observation").selectAll().where("feed_id", "=", f.id).execute()) +
      JSON.stringify(
        await h.db.selectFrom("source_release").selectAll().where("site_release_id", "=", "900001").execute(),
      ) +
      JSON.stringify(await h.db.selectFrom("feed_source").selectAll().where("id", "=", f.id).execute());
    expect(dump).not.toContain("SECRETPK123");
    expect(dump).not.toContain("TOPSECRETKEY");
    expect(dump).toContain("[REDACTED]");
    const again = await ingestFeed(h.rt, f.id);
    expect(again.status).toBe("not_modified");
    expect(s.notModifiedCount).toBe(1);
  });

  it("同 GUID 内容变化视为更新:观察与资源标题更新,重新触发识别", async () => {
    const s = await feedServer([item("upd-1", "900100", "Show.2021.1080p.WEB-DL-AAA")]);
    const f = await seedFeed(h, { siteId, url: s.feedUrl, name: "更新" });
    await ingestFeed(h.rt, f.id);
    s.updateItem("upd-1", { title: "Show.2021.2160p.WEB-DL-AAA" });
    const r = await ingestFeed(h.rt, f.id);
    expect(r).toMatchObject({ status: "ok", newItems: 0, updatedItems: 1, newReleases: 0 });
    const obs = await h.db.selectFrom("observation").selectAll().where("feed_id", "=", f.id).execute();
    expect(obs).toHaveLength(1);
    expect(obs[0]!.title).toBe("Show.2021.2160p.WEB-DL-AAA");
    const rel = await h.db
      .selectFrom("source_release")
      .selectAll()
      .where("site_release_id", "=", "900100")
      .executeTakeFirstOrThrow();
    expect(rel.title).toBe("Show.2021.2160p.WEB-DL-AAA");
    const ev = (await h.outbox("release.observed")).filter((e) => e.payload.sourceReleaseId === rel.id);
    expect(ev.map((e) => e.payload.isNew)).toEqual([true, false]);
  });

  it("窗口零重叠 → feed.gap_suspected,并路由到有预算的补漏快照", async () => {
    const s = await feedServer([item("gap-1", "900201", "A.2020.1080p-X"), item("gap-2", "900202", "B.2020.1080p-X")]);
    const f = await seedFeed(h, { siteId, url: s.feedUrl, name: "缺口" });
    await ingestFeed(h.rt, f.id);
    s.setItems([item("gap-9", "900209", "C.2020.1080p-X")]);
    const r = await ingestFeed(h.rt, f.id);
    expect(r.possibleGap).toBe(true);
    const gap = (await h.outbox("feed.gap_suspected")).find((e) => e.payload.feedId === f.id);
    expect(gap).toBeDefined();
    const route = sourcesOutboxRoutes["feed.gap_suspected"]!({
      id: "1",
      type: "feed.gap_suspected",
      payload: gap!.payload,
      at: new Date(),
    });
    expect(route).toMatchObject({ task: "snapshotSite", input: { siteId, kind: "backfill", maxPages: 3 } });
    // 响应派生字段 possibleGap;下一次衔接上的抓取后清空
    const got = (await (await h.req(`/feed-sources/${f.id}`)).json()) as {
      data: { possibleGap: { backfillQueued: boolean } | null };
    };
    expect(got.data.possibleGap).toMatchObject({ backfillQueued: true });
    s.setItems([item("gap-9", "900209", "C.2020.1080p-X"), item("gap-10", "900210", "D.2020.1080p-X")]);
    await ingestFeed(h.rt, f.id);
    const after = (await (await h.req(`/feed-sources/${f.id}`)).json()) as { data: { possibleGap: unknown } };
    expect(after.data.possibleGap).toBeNull();
  });

  it("登录失效(403):账号标失效 + 待处理,不重试", async () => {
    const s = await feedServer([item("auth-1", "900300", "X.2020.1080p-Y")], "RIGHTKEY");
    const badUrl = s.feedUrl.replace("RIGHTKEY", "WRONGKEY");
    const f = await seedFeed(h, { siteId, url: badUrl, name: "失效", accountId });
    const r = await ingestFeed(h.rt, f.id);
    expect(r.status).toBe("auth_failed");
    const acc = await h.db
      .selectFrom("site_account")
      .selectAll()
      .where("id", "=", accountId!)
      .executeTakeFirstOrThrow();
    expect(acc.status).toBe("invalid");
    const d = await h.db
      .selectFrom("decision")
      .selectAll()
      .where("dedupe_key", "=", `feed-auth:${f.id}`)
      .executeTakeFirstOrThrow();
    expect(d.kind).toBe("source_unreachable");
    expect(JSON.stringify(d)).not.toContain("WRONGKEY");
    const row = await h.db.selectFrom("feed_source").selectAll().where("id", "=", f.id).executeTakeFirstOrThrow();
    expect(row.health_status).toBe("failing");
    expect(row.backoff_until!.getTime()).toBeGreaterThan(Date.now() + 23 * 3600_000);
    // 再次调度:退避期内不发请求
    const before = s.requests.length;
    const again = await ingestFeed(h.rt, f.id);
    expect(again.status).toBe("skipped");
    expect(s.requests.length).toBe(before);
    // 调度器也不会把它当作到期
    await h.db
      .updateTable("feed_source")
      .set({ next_fetch_at: new Date(0) })
      .where("id", "=", f.id)
      .execute();
    await schedulerTick(h.rt);
    const due = (await h.outbox("feed.due")).filter((e) => e.payload.feedId === f.id);
    expect(due).toHaveLength(0);
    // 恢复账号给后续用例
    await h.db.updateTable("site_account").set({ status: "active" }).where("id", "=", accountId!).execute();
  });

  it("调度器:到期 Feed 发 feed.due(幂等),快照到期入队 snapshotSite", async () => {
    const s = await feedServer([item("sch-1", "900400", "S.2020.1080p-Z")]);
    const f = await seedFeed(h, { siteId, url: s.feedUrl, name: "调度" });
    const t1 = await schedulerTick(h.rt);
    expect(t1.feedsDue).toBeGreaterThanOrEqual(1);
    const due = (await h.outbox("feed.due")).filter((e) => e.payload.feedId === f.id);
    expect(due).toHaveLength(1);
    expect(
      sourcesOutboxRoutes["feed.due"]!({ id: "1", type: "feed.due", payload: due[0]!.payload, at: new Date() }),
    ).toEqual({
      task: "ingestFeed",
      input: { feedSourceId: f.id, manual: false },
    });
    // next_fetch_at 已顺延:紧接着的 tick 不再重复
    await schedulerTick(h.rt);
    expect((await h.outbox("feed.due")).filter((e) => e.payload.feedId === f.id)).toHaveLength(1);
    expect(
      h.engine.enqueued.some((e) => e.task === "snapshotSite" && (e.input as { siteId: string }).siteId === siteId),
    ).toBe(true);
  });

  it("站点关闭后 Feed 不再抓取", async () => {
    const s = await feedServer([item("off-1", "900500", "O.2020.1080p-Z")]);
    const f = await seedFeed(h, { siteId, url: s.feedUrl, name: "关站" });
    await h.db.updateTable("site").set({ enabled: false }).where("id", "=", siteId).execute();
    const r = await ingestFeed(h.rt, f.id);
    expect(r).toMatchObject({ status: "skipped", reason: "site_disabled" });
    expect(s.requests.length).toBe(0);
    await h.db.updateTable("site").set({ enabled: true }).where("id", "=", siteId).execute();
  });
});