Merge branch 'ia2'

Co-Authored-By: Claude Fable 5.1 <[email protected]> # Conflicts: # CHANGELOG.md # docs/plan/00-first-version-plan.md # packages/agent/src/system-prompt.ts
2026-09-30 00:30:42 +08:00
commit 7a24d23bbf
@@ -137,13 +137,23 @@ describe.skipIf(!up)("来源管理与站点工作台 API", () => {
    expect(other.status).toBe(200);
    expect(other.body.data.name).toBe("促销");
    expect(other.body.data.warnings?.[0]).toContain("elsewhere.example");
+    // IA-2:RSS 属于站点。不带站点新建 → 归入同主机的「自定义 RSS 源」站点(没有就建一个)
    const bare = await json(await h.req("/feed-sources", { body: { url: "https://any.example/feed.xml" } }));
-    expect(bare.body.data).toMatchObject({
+    expect(bare.body.data).toMatchObject({ name: "any.example RSS", siteAccountId: null, parserVersion: "rss@1" });
+    const bareSite = await json(await h.req(`/sites/${bare.body.data.siteId}`));
+    expect(bareSite.body.data).toMatchObject({
+      key: "rss-any-example",
      name: "any.example",
-      siteId: null,
-      siteAccountId: null,
-      parserVersion: "1",
+      kind: "rss_feed",
+      adapter: "generic_rss",
+      capabilities: ["rss"],
+      searchDefault: false,
    });
+    const again = await json(await h.req("/feed-sources", { body: { url: "https://any.example/other.xml" } }));
+    expect(again.body.data.siteId).toBe(bare.body.data.siteId);
+    // 不能把 RSS 移出站点
+    const detach = await h.req(`/feed-sources/${bare.body.data.id}`, { method: "PATCH", body: { siteId: null } });
+    expect(detach.status).toBe(400);
  });

  it("快照:列表页 upsert 到 source_release,记录覆盖范围;站点级限流器每站一个实例", async () => {
@@ -0,0 +1,90 @@
+import { createMigrator, type DB, migrateToLatest, sql } from "@umt/db";
+import { Kysely, PostgresDialect } from "kysely";
+import pg from "pg";
+import { afterAll, describe, expect, it } from "vitest";
+import { PG_URL, pgAvailable, withSchemaLock } from "../auth-settings/harness.ts";
+
+// 0055:site.kind;generic_rss 站点 → rss_feed;不属于任何站点的 RSS 按主机归入「自定义 RSS 源」站点。
+
+const hasPg = await pgAvailable();
+
+describe.skipIf(!hasPg)("0055 站点类型迁移", () => {
+  const schema = `ia2_mig_${process.pid.toString(36)}${Math.random().toString(36).slice(2, 6)}`;
+  const admin = async (q: string) => {
+    const c = new pg.Client({ connectionString: PG_URL });
+    await c.connect();
+    await c.query(q).finally(() => c.end());
+  };
+  let db: Kysely<DB>;
+
+  afterAll(async () => {
+    await db?.destroy();
+    await withSchemaLock(() => admin(`drop schema if exists ${schema} cascade`));
+  });
+
+  it("存量站点定类型;孤立 RSS 按主机建站点、清游标窗口;回滚只删列", async () => {
+    await admin(`create schema ${schema}`);
+    db = new Kysely<DB>({
+      dialect: new PostgresDialect({
+        pool: new pg.Pool({ connectionString: PG_URL, max: 2, options: `-c search_path=${schema},public` }),
+      }),
+    });
+    await withSchemaLock(async () => {
+      const r = await createMigrator(db, { migrationTableSchema: schema }).migrateTo("0054_download_task_entity");
+      if (r.error) throw r.error;
+    });
+    await sql`insert into site (key, name, adapter, base_url) values
+      ('audiences', '观众', 'nexusphp', 'https://audiences.example/'),
+      ('myrss', '旧自定义', 'generic_rss', 'https://old.example/'),
+      ('rss-nyaa-si', '占用', 'generic_rss', 'https://taken.example/')`.execute(db);
+    const cursor = JSON.stringify({ window: { lastGuidSet: ["a"], lastFetchedAt: "2026-09-01T00:00:00Z" } });
+    await sql`insert into feed_source (site_id, name, url_display, etag, cursor) values
+      (null, 'A', 'https://nyaa.si/?page=rss&q=frieren', 'W/"1"', ${cursor}::jsonb),
+      (null, 'B', 'https://nyaa.si/?page=rss&q=dune', null, ${cursor}::jsonb),
+      (null, 'C', 'https://www.example.org/feed?passkey=[REDACTED]', null, '{}'::jsonb),
+      ((select id from site where key = 'audiences'), 'D', 'https://audiences.example/rss', 'W/"2"', ${cursor}::jsonb)`.execute(
+      db,
+    );
+
+    await withSchemaLock(() => migrateToLatest(db, { migrationTableSchema: schema }));
+
+    const sites = await sql<{ key: string; name: string; kind: string; base_url: string; search_default: boolean }>`
+      select key, name, kind, base_url, search_default from site order by key`.execute(db);
+    expect(sites.rows).toEqual([
+      { key: "audiences", name: "观众", kind: "pt", base_url: "https://audiences.example/", search_default: true },
+      { key: "myrss", name: "旧自定义", kind: "rss_feed", base_url: "https://old.example/", search_default: true },
+      {
+        key: "rss-example-org",
+        name: "www.example.org",
+        kind: "rss_feed",
+        base_url: "https://www.example.org",
+        search_default: false,
+      },
+      { key: "rss-nyaa-si", name: "占用", kind: "rss_feed", base_url: "https://taken.example/", search_default: true },
+      { key: "rss-nyaa-si-2", name: "nyaa.si", kind: "rss_feed", base_url: "https://nyaa.si", search_default: false },
+    ]);
+    const feeds = await sql<{ name: string; key: string; etag: string | null; cursor: Record<string, unknown> }>`
+      select f.name, s.key, f.etag, f.cursor from feed_source f join site s on s.id = f.site_id order by f.name`.execute(
+      db,
+    );
+    expect(feeds.rows.map((f) => [f.name, f.key])).toEqual([
+      ["A", "rss-nyaa-si-2"],
+      ["B", "rss-nyaa-si-2"],
+      ["C", "rss-example-org"],
+      ["D", "audiences"],
+    ]);
+    // 迁移过来的 RSS 清掉窗口与条件请求头(下次抓取重新写成资源);原本属于站点的不动
+    expect(feeds.rows[0]).toMatchObject({ etag: null, cursor: {} });
+    expect(feeds.rows[3]!.etag).toBe('W/"2"');
+    expect(feeds.rows[3]!.cursor).toHaveProperty("window");
+
+    await withSchemaLock(async () => {
+      const r = await createMigrator(db, { migrationTableSchema: schema }).migrateTo("0054_download_task_entity");
+      if (r.error) throw r.error;
+    });
+    const cols = await sql<{ n: number }>`select count(*)::int as n from information_schema.columns
+      where table_schema = ${schema} and table_name = 'site' and column_name = 'kind'`.execute(db);
+    expect(cols.rows[0]!.n).toBe(0);
+    await withSchemaLock(() => migrateToLatest(db, { migrationTableSchema: schema }));
+  });
+});
@@ -0,0 +1,178 @@
+import { SITE_KINDS, siteCapabilities } from "@umt/contracts";
+import { createMockRssServer, type MockRssServer } from "@umt/testkit";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { tickFeeds, tickSnapshots } from "../../src/modules/sources-identity/scheduler.ts";
+import { ingestFeed } from "../../src/modules/sources-identity/service/feeds.ts";
+import { snapshotSite } from "../../src/modules/sources-identity/service/snapshot.ts";
+import { capabilitiesOf, hostSlug } from "../../src/modules/sources-identity/site-kinds.ts";
+import { createHarness, type Harness, json, pgAvailable, seedFeed, seedSite } from "./harness.ts";
+
+// IA-2 站点类型:类型声明获取方式,实际可用 = 类型 ∩ 站点定义;调度 / 搜索 / 登录按 capabilities 判断。
+
+describe("站点类型与获取方式(纯函数)", () => {
+  it("实际可用 = 类型允许的 ∩ 定义实现的;RSS 不依赖定义;torznab 未实现为空", () => {
+    const def = { search: true, list: true, detail: false };
+    expect(siteCapabilities("pt", def)).toEqual(["search", "list", "rss", "login", "userinfo"]);
+    expect(siteCapabilities("pt", null)).toEqual(["rss"]);
+    expect(siteCapabilities("public_bt", def)).toEqual(["search", "list", "rss"]);
+    expect(siteCapabilities("public_bt", null)).toEqual(["rss"]);
+    expect(siteCapabilities("rss_feed", def)).toEqual(["rss"]);
+    expect(siteCapabilities("torznab", def)).toEqual([]);
+    expect(SITE_KINDS.torznab.implemented).toBe(false);
+    expect(SITE_KINDS.rss_feed.capabilities).toEqual(["rss"]);
+  });
+
+  it("generic_rss 站点不查定义;key 片段", () => {
+    expect(capabilitiesOf({ kind: "pt", key: "audiences", adapter: "generic_rss" })).toEqual(["rss"]);
+    expect(capabilitiesOf({ kind: "pt", key: "audiences", adapter: "nexusphp" })).toContain("login");
+    expect(hostSlug("www.Nyaa.si:8443")).toBe("nyaa-si-8443");
+    expect(hostSlug("")).toBe("feed");
+  });
+});
+
+const up = await pgAvailable();
+
+describe.skipIf(!up)("站点类型:API、调度与 RSS", () => {
+  let h: Harness;
+  let rss: MockRssServer;
+
+  beforeAll(async () => {
+    h = await createHarness("kinds");
+    rss = await createMockRssServer({
+      items: [
+        {
+          guid: "nyaa-1",
+          title: "[SubsPlease] Frieren - 12 (1080p) [ABCDEF12].mkv",
+          link: "https://nyaa.example/view/1812345",
+          enclosureUrl: "https://nyaa.example/download/1812345.torrent",
+          size: 1_400_000_000,
+          pubDate: new Date("2026-09-20T08:00:00Z"),
+          categories: ["Anime"],
+        },
+      ],
+    });
+  });
+  afterAll(async () => {
+    await rss?.close();
+    await h?.close();
+  });
+
+  it("添加自定义 RSS 源:一个事务建站点 + RSS,key 按主机生成,不参与默认搜索", async () => {
+    const r = await json(
+      await h.req("/sites", {
+        body: {
+          kind: "rss_feed",
+          adapter: "generic_rss",
+          name: "Nyaa",
+          baseUrl: "https://nyaa.example/",
+          rssUrl: rss.feedUrl,
+        },
+      }),
+    );
+    expect(r.status).toBe(200);
+    expect(r.body.data).toMatchObject({
+      kind: "rss_feed",
+      capabilities: ["rss"],
+      searchDefault: false,
+      snapshotEnabled: false,
+    });
+    expect(r.body.data.key).toMatch(/^rss-/);
+    const feeds = await json(await h.req(`/feed-sources?siteId=${r.body.data.id}`));
+    expect(feeds.body.data.items).toHaveLength(1);
+    expect(feeds.body.data.items[0]).toMatchObject({ name: "Nyaa RSS", siteId: r.body.data.id });
+
+    // 抓取:自定义 RSS 源的条目按站点聚合成资源(以前无站点的 Feed 只留观察)
+    const res = await ingestFeed(h.rt, feeds.body.data.items[0].id);
+    expect(res).toMatchObject({ status: "ok", newReleases: 1 });
+    const rel = await h.db.selectFrom("source_release").selectAll().where("site_id", "=", r.body.data.id).execute();
+    expect(rel).toHaveLength(1);
+    expect(rel[0]!.title).toContain("Frieren");
+  });
+
+  it("不支持的获取方式 400:自定义 RSS 源不能登录 / 填凭证 / 抓列表;torznab 不能添加;PT 站缺 key 400", async () => {
+    const list = await json(await h.req("/sites"));
+    const feedSite = list.body.data.items.find((s: { kind: string }) => s.kind === "rss_feed");
+    for (const [path, body] of [
+      [`/sites/${feedSite.id}/refresh-login`, {}],
+      [`/sites/${feedSite.id}/snapshot`, {}],
+    ] as const) {
+      const res = await json(await h.req(path, { body }));
+      expect(res.status, path).toBe(400);
+      expect(res.body.error.message).toContain("自定义 RSS 源");
+    }
+    const cred = await h.req(`/sites/${feedSite.id}/credentials`, {
+      method: "PUT",
+      body: { kind: "cookie", cookie: "c=1" },
+    });
+    expect(cred.status).toBe(400);
+    const tz = await h.req("/sites", {
+      body: { kind: "torznab", adapter: "generic_rss", name: "Jackett", baseUrl: "https://jackett.example/" },
+    });
+    expect(tz.status).toBe(400);
+    const noKey = await h.req("/sites", {
+      body: { kind: "pt", adapter: "nexusphp", name: "x", baseUrl: "https://x.example/" },
+    });
+    expect(noKey.status).toBe(400);
+    const pubWrongAdapter = await h.req("/sites", {
+      body: { kind: "public_bt", adapter: "nexusphp", name: "x", key: "audiences", baseUrl: "https://x.example/" },
+    });
+    expect(pubWrongAdapter.status).toBe(400);
+  });
+
+  it("站点定义都是 PT 站;PT 站带登录与账号数据", async () => {
+    const defs = await json(await h.req("/site-definitions"));
+    expect(defs.body.data.items.every((d: { kind: string }) => d.kind === "pt")).toBe(true);
+    const pt = await json(
+      await h.req("/sites", {
+        body: { key: "audiences", name: "观众", adapter: "nexusphp", baseUrl: "https://audiences.invalid/" },
+      }),
+    );
+    expect(pt.status).toBe(200);
+    expect(pt.body.data.kind).toBe("pt");
+    expect(pt.body.data.capabilities).toEqual(expect.arrayContaining(["search", "list", "rss", "login", "userinfo"]));
+  });
+
+  it("调度按获取方式:快照只排能抓列表的站点;Feed 只排能抓 RSS 的站点", async () => {
+    const pt = await seedSite(h, { key: "hdhome", baseUrl: "https://hdhome.invalid/" });
+    // 自定义 RSS 源即使 snapshot_enabled 被写成 true 也不排快照
+    const feedOnly = await h.db
+      .insertInto("site")
+      .values({
+        key: "rss-only",
+        name: "only",
+        kind: "rss_feed",
+        adapter: "generic_rss",
+        base_url: "https://only.invalid/",
+        snapshot_enabled: true,
+      })
+      .returningAll()
+      .executeTakeFirstOrThrow();
+    const tz = await h.db
+      .insertInto("site")
+      .values({ key: "tz", name: "tz", kind: "torznab", adapter: "generic_rss", base_url: "https://tz.invalid/" })
+      .returningAll()
+      .executeTakeFirstOrThrow();
+    const tzFeed = await seedFeed(h, { siteId: tz.id, url: "https://tz.invalid/api?t=search" });
+    const okFeed = await seedFeed(h, { siteId: feedOnly.id, url: "https://only.invalid/rss" });
+
+    h.engine.enqueued.length = 0;
+    await tickSnapshots(h.rt);
+    const snapSites = h.engine.enqueued
+      .filter((e) => e.task === "snapshotSite")
+      .map((e) => (e.input as { siteId: string }).siteId);
+    expect(snapSites).toContain(pt.site.id);
+    expect(snapSites).not.toContain(feedOnly.id);
+
+    await tickFeeds(h.rt);
+    const due = (await h.outbox("feed.due")).map((e) => e.payload.feedId);
+    expect(due).toContain(okFeed.id);
+    expect(due).not.toContain(tzFeed.id);
+    expect((await ingestFeed(h.rt, tzFeed.id)).reason).toBe("unsupported");
+
+    // 实时搜索 / 列表快照在不支持的站点上直接跳过,不请求站点
+    expect(await snapshotSite(h.rt, { siteId: feedOnly.id, kind: "live_search", query: "x" })).toMatchObject({
+      status: "skipped",
+      reason: "unsupported",
+    });
+  });
+});