From 89d2c81de66b8df6b1b9f650254a31c7ba193419 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 30 Sep 2026 00:29:56 +0800 Subject: [PATCH] feat(sources): site kinds and capabilities (IA-2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit site.kind: pt / public_bt / rss_feed / torznab. Each kind declares the acquisition modes it may support (search, list, rss, detail, login, userinfo); a site's effective capabilities = kind ∩ site definition. torznab only defines TorznabSiteConfig and cannot be added yet. Migration 0055: generic_rss sites become rss_feed, the rest pt; feeds without a site are grouped by host into new custom RSS source sites (cursor window and conditional headers cleared so the next fetch writes releases). User DB read-only dry-run: 0 orphan feeds. Co-Authored-By: Claude Opus 5.5 --- .../migration-site-kind.test.ts | 90 +++++++++++++ packages/contracts/src/capabilities-io.ts | 6 + packages/contracts/src/sources.ts | 126 ++++++++++++++++-- packages/contracts/test/db-sync.test.ts | 5 +- packages/db/migrations/0055_site_kind.ts | 100 ++++++++++++++ packages/db/src/types.ts | 2 + 6 files changed, 317 insertions(+), 12 deletions(-) create mode 100644 apps/server/test/sources-identity/migration-site-kind.test.ts create mode 100644 packages/db/migrations/0055_site_kind.ts diff --git a/apps/server/test/sources-identity/migration-site-kind.test.ts b/apps/server/test/sources-identity/migration-site-kind.test.ts new file mode 100644 index 00000000..6e7b31e6 --- /dev/null +++ b/apps/server/test/sources-identity/migration-site-kind.test.ts @@ -0,0 +1,90 @@ +import { createMigrator, type DB, migrateToLatest, sql } from "@umt/db"; +import { Kysely, PostgresDialect } from "kysely"; +import pg from "pg"; +import { afterAll, describe, expect, it } from "vitest"; +import { PG_URL, pgAvailable, withSchemaLock } from "../auth-settings/harness.ts"; + +// 0055:site.kind;generic_rss 站点 → rss_feed;不属于任何站点的 RSS 按主机归入「自定义 RSS 源」站点。 + +const hasPg = await pgAvailable(); + +describe.skipIf(!hasPg)("0055 站点类型迁移", () => { + const schema = `ia2_mig_${process.pid.toString(36)}${Math.random().toString(36).slice(2, 6)}`; + const admin = async (q: string) => { + const c = new pg.Client({ connectionString: PG_URL }); + await c.connect(); + await c.query(q).finally(() => c.end()); + }; + let db: Kysely; + + afterAll(async () => { + await db?.destroy(); + await withSchemaLock(() => admin(`drop schema if exists ${schema} cascade`)); + }); + + it("存量站点定类型;孤立 RSS 按主机建站点、清游标窗口;回滚只删列", async () => { + await admin(`create schema ${schema}`); + db = new Kysely({ + dialect: new PostgresDialect({ + pool: new pg.Pool({ connectionString: PG_URL, max: 2, options: `-c search_path=${schema},public` }), + }), + }); + await withSchemaLock(async () => { + const r = await createMigrator(db, { migrationTableSchema: schema }).migrateTo("0054_download_task_entity"); + if (r.error) throw r.error; + }); + await sql`insert into site (key, name, adapter, base_url) values + ('audiences', '观众', 'nexusphp', 'https://audiences.example/'), + ('myrss', '旧自定义', 'generic_rss', 'https://old.example/'), + ('rss-nyaa-si', '占用', 'generic_rss', 'https://taken.example/')`.execute(db); + const cursor = JSON.stringify({ window: { lastGuidSet: ["a"], lastFetchedAt: "2026-09-01T00:00:00Z" } }); + await sql`insert into feed_source (site_id, name, url_display, etag, cursor) values + (null, 'A', 'https://nyaa.si/?page=rss&q=frieren', 'W/"1"', ${cursor}::jsonb), + (null, 'B', 'https://nyaa.si/?page=rss&q=dune', null, ${cursor}::jsonb), + (null, 'C', 'https://www.example.org/feed?passkey=[REDACTED]', null, '{}'::jsonb), + ((select id from site where key = 'audiences'), 'D', 'https://audiences.example/rss', 'W/"2"', ${cursor}::jsonb)`.execute( + db, + ); + + await withSchemaLock(() => migrateToLatest(db, { migrationTableSchema: schema })); + + const sites = await sql<{ key: string; name: string; kind: string; base_url: string; search_default: boolean }>` + select key, name, kind, base_url, search_default from site order by key`.execute(db); + expect(sites.rows).toEqual([ + { key: "audiences", name: "观众", kind: "pt", base_url: "https://audiences.example/", search_default: true }, + { key: "myrss", name: "旧自定义", kind: "rss_feed", base_url: "https://old.example/", search_default: true }, + { + key: "rss-example-org", + name: "www.example.org", + kind: "rss_feed", + base_url: "https://www.example.org", + search_default: false, + }, + { key: "rss-nyaa-si", name: "占用", kind: "rss_feed", base_url: "https://taken.example/", search_default: true }, + { key: "rss-nyaa-si-2", name: "nyaa.si", kind: "rss_feed", base_url: "https://nyaa.si", search_default: false }, + ]); + const feeds = await sql<{ name: string; key: string; etag: string | null; cursor: Record }>` + select f.name, s.key, f.etag, f.cursor from feed_source f join site s on s.id = f.site_id order by f.name`.execute( + db, + ); + expect(feeds.rows.map((f) => [f.name, f.key])).toEqual([ + ["A", "rss-nyaa-si-2"], + ["B", "rss-nyaa-si-2"], + ["C", "rss-example-org"], + ["D", "audiences"], + ]); + // 迁移过来的 RSS 清掉窗口与条件请求头(下次抓取重新写成资源);原本属于站点的不动 + expect(feeds.rows[0]).toMatchObject({ etag: null, cursor: {} }); + expect(feeds.rows[3]!.etag).toBe('W/"2"'); + expect(feeds.rows[3]!.cursor).toHaveProperty("window"); + + await withSchemaLock(async () => { + const r = await createMigrator(db, { migrationTableSchema: schema }).migrateTo("0054_download_task_entity"); + if (r.error) throw r.error; + }); + const cols = await sql<{ n: number }>`select count(*)::int as n from information_schema.columns + where table_schema = ${schema} and table_name = 'site' and column_name = 'kind'`.execute(db); + expect(cols.rows[0]!.n).toBe(0); + await withSchemaLock(() => migrateToLatest(db, { migrationTableSchema: schema })); + }); +}); diff --git a/packages/contracts/src/capabilities-io.ts b/packages/contracts/src/capabilities-io.ts index a17e948a..807cffdf 100644 --- a/packages/contracts/src/capabilities-io.ts +++ b/packages/contracts/src/capabilities-io.ts @@ -25,8 +25,10 @@ import { SiteAccountUserData, SiteAdapter, SiteAuthKind, + SiteCapability, SiteHealth, SiteHealthResult, + SiteKind, SiteLastHealth, SiteLoginResult, SiteUserDataPoint, @@ -430,6 +432,10 @@ export const SiteBrief = z.object({ id: Id, key: z.string(), name: z.string(), + /** 站点类型:pt / public_bt / rss_feed / torznab */ + kind: SiteKind, + /** 实际可用的获取方式(search / list / rss / detail / login / userinfo) */ + capabilities: z.array(SiteCapability), adapter: SiteAdapter, baseUrl: z.string(), enabled: z.boolean(), diff --git a/packages/contracts/src/sources.ts b/packages/contracts/src/sources.ts index ce90b6a7..b078f4be 100644 --- a/packages/contracts/src/sources.ts +++ b/packages/contracts/src/sources.ts @@ -8,6 +8,94 @@ import { QuickFilterParam, ReleaseQuickFacets } from "./release-facets.ts"; /** api = MP 专用 API spider(Haidan/HDDolby/RousiPro/SunnyPT/TNode/TorrentLeech/Yema) */ export const SiteAdapter = z.enum(["nexusphp", "mteam_api", "api", "generic_rss"]); export type SiteAdapter = z.infer; + +/** + * 站点类型(IA-2):PT 站之外的下载来源都作为一种站点类型。类型声明它可能支持的获取方式, + * 站点实际可用的方式 = 类型允许的 ∩ 适配器(站点定义)实现的,见 siteCapabilities。 + * torznab 只定义结构(Jackett / Prowlarr 的 Torznab 接口),尚未实现:不能添加,也不参与调度。 + */ +export const SiteKind = z.enum(["pt", "public_bt", "rss_feed", "torznab"]); +export type SiteKind = z.infer; +/** 获取方式:搜索 / 列表抓取 / RSS / 详情页 / 登录 / 账号数据(上传量、分享率、等级) */ +export const SiteCapability = z.enum(["search", "list", "rss", "detail", "login", "userinfo"]); +export type SiteCapability = z.infer; + +export interface SiteKindSpec { + /** 界面名 */ + label: string; + /** 添加站点第一步的一句说明 */ + description: string; + /** 这类站点可能支持的获取方式 */ + capabilities: readonly SiteCapability[]; + /** false = 只定义了结构,还不能添加 */ + implemented: boolean; + /** 需要从站点定义库里选(PT 站);否则只填名称和地址 */ + fromDefinition: boolean; +} + +export const SITE_KINDS: Record = { + pt: { + label: "PT 站", + description: "需要账号的私有站。从站点定义里选，支持搜索、列表抓取、RSS、登录和账号数据", + capabilities: ["search", "list", "rss", "detail", "login", "userinfo"], + implemented: true, + fromDefinition: true, + }, + public_bt: { + label: "公开 BT 站", + description: "不用登录的公开站。填网址和 RSS 地址，按 RSS 获取资源", + capabilities: ["search", "list", "rss", "detail"], + implemented: true, + fromDefinition: false, + }, + rss_feed: { + label: "自定义 RSS 源", + description: "只有一个 RSS 地址的来源，不登录、不搜索", + capabilities: ["rss"], + implemented: true, + fromDefinition: false, + }, + torznab: { + label: "Torznab 索引器", + description: "Jackett、Prowlarr 等索引器的 Torznab 接口，还不支持", + capabilities: ["search", "rss"], + implemented: false, + fromDefinition: false, + }, +}; + +/** 站点定义(适配器)声明的能力;没有定义的站点传 null */ +export type SiteDefinitionCaps = { search: boolean; list: boolean; detail: boolean } | null; + +/** + * 站点实际可用的获取方式:类型允许的 ∩ 适配器实现的。RSS 走通用解析器,不依赖站点定义; + * 搜索 / 列表 / 详情要站点定义声明支持;登录与账号数据要有站点定义(没有定义就没有登录流程)。 + */ +export function siteCapabilities(kind: SiteKind, def: SiteDefinitionCaps): SiteCapability[] { + const spec = SITE_KINDS[kind]; + if (!spec.implemented) return []; + return spec.capabilities.filter((c) => { + if (c === "rss") return true; + if (!def) return false; + if (c === "search") return def.search; + if (c === "list") return def.list; + if (c === "detail") return def.detail; + return true; + }); +} + +/** + * Torznab 索引器的站点 config(只定义结构,未实现):API 地址、API Key 的 secret 引用、分类过滤。 + * 对照 MP 的 Jackett / Prowlarr 插件:按索引器登记,搜索走 `t=search`,RSS 走不带 q 的同一接口。 + */ +export const TorznabSiteConfig = z.object({ + apiUrl: z.url(), + apiKeySecretRef: z.uuid().nullable(), + indexerId: z.string().max(200).nullable(), + categories: z.array(z.number().int().nonnegative()).default([]), +}); +export type TorznabSiteConfig = z.infer; + export const SiteHealth = z.enum(["unknown", "healthy", "degraded", "failing", "disabled"]); export type SiteHealth = z.infer; export const CircuitState = z.enum(["closed", "open", "half_open"]); @@ -34,6 +122,10 @@ export type SiteLastHealth = z.infer; export const Site = EntityBase.extend({ key: z.string().regex(/^[a-z][a-z0-9_-]*$/), name: z.string().min(1), + /** 站点类型(IA-2) */ + kind: SiteKind, + /** 派生字段(非表列):实际可用的获取方式,见 siteCapabilities */ + capabilities: z.array(SiteCapability), adapter: SiteAdapter, baseUrl: z.url(), enabled: z.boolean(), @@ -58,6 +150,7 @@ export type Site = z.infer; export const SiteCreate = Site.pick({ key: true, name: true, + kind: true, adapter: true, baseUrl: true, enabled: true, @@ -67,17 +160,28 @@ export const SiteCreate = Site.pick({ snapshotIntervalSec: true, proxy: true, searchDefault: true, -}).partial({ - enabled: true, - rateLimit: true, - config: true, - snapshotEnabled: true, - snapshotIntervalSec: true, - proxy: true, - searchDefault: true, -}); +}) + .partial({ + key: true, + kind: true, + enabled: true, + rateLimit: true, + config: true, + snapshotEnabled: true, + snapshotIntervalSec: true, + proxy: true, + searchDefault: true, + }) + .extend({ + /** + * 非 PT 站(public_bt / rss_feed)添加时一并建的 RSS 地址(只写,存为 secret)。 + * key 缺省时按地址主机名生成(rss-<主机>,重名加序号)。 + */ + rssUrl: z.url().optional(), + }); export type SiteCreate = z.infer; -export const SiteUpdate = SiteCreate.omit({ key: true }).partial(); +/** 类型添加后不改(换类型 = 删了重加) */ +export const SiteUpdate = SiteCreate.omit({ key: true, kind: true, rssUrl: true }).partial(); export type SiteUpdate = z.infer; export const SiteAuthKind = z.enum(["cookie", "passkey", "api_key", "password"]); @@ -550,6 +654,8 @@ export type SiteRequestStats = z.infer; export const SiteDefinitionView = z.object({ key: z.string(), name: z.string(), + /** 定义对应的站点类型(现有定义都是 PT 站) */ + kind: SiteKind, adapter: SiteAdapter, baseUrl: z.string(), /** 定义是否随产品启用(未实测站点为 false) */ diff --git a/packages/contracts/test/db-sync.test.ts b/packages/contracts/test/db-sync.test.ts index 197081db..a6ee807e 100644 --- a/packages/contracts/test/db-sync.test.ts +++ b/packages/contracts/test/db-sync.test.ts @@ -77,8 +77,8 @@ const ROWS: Record Secret: { row: C.Secret, dbOnly: ["ciphertext", "nonce", "authTag"] }, ServerMapping: { row: C.ServerMapping }, Settings: { row: C.SettingEntry }, - // deleted_at:软删除只在服务端过滤(已删站点不出 API) - Site: { row: C.Site, dbOnly: ["deletedAt"] }, + // deleted_at:软删除只在服务端过滤(已删站点不出 API);capabilities 由类型 ∩ 站点定义算出(IA-2) + Site: { row: C.Site, dbOnly: ["deletedAt"], apiOnly: ["capabilities"] }, // secret 引用不外露,只给布尔 SiteUserData: { row: C.SiteUserData }, SiteAccount: { @@ -248,6 +248,7 @@ const ENUMS: Record = { SiteAdapter: C.SiteAdapter, SiteHealthStatus: C.SiteHealth, SiteCircuitState: C.CircuitState, + SiteKind: C.SiteKind, SiteAccountAuthKind: C.SiteAuthKind, SiteAccountStatus: C.SiteAccountStatus, SourceReleaseResolutionStatus: C.ResolutionStatus, diff --git a/packages/db/migrations/0055_site_kind.ts b/packages/db/migrations/0055_site_kind.ts new file mode 100644 index 00000000..4ceed7d5 --- /dev/null +++ b/packages/db/migrations/0055_site_kind.ts @@ -0,0 +1,100 @@ +import { type Kysely, sql } from "kysely"; +import { execDdl, oneOf } from "../src/migration-helpers.ts"; + +// IA-2 站点类型:site.kind(pt / public_bt / rss_feed / torznab),各类型声明支持的获取方式(contracts SITE_KINDS)。 +// - 存量:adapter = generic_rss 的站点(旧"自定义 RSS 站点:只抓 RSS,不登录")→ rss_feed;其余 → pt。 +// - RSS 归属站点:不属于任何站点的 feed_source 按地址主机名归入「自定义 RSS 源」站点(同主机一个站点, +// key = rss-<主机>,重名加序号);清掉这些 Feed 的游标窗口与条件请求头,下次抓取把窗口里的条目重新写成资源 +// (之前无站点时观察记录被标 ignored/no_site,没有进 source_release)。 +// down 只删列;迁移建的站点与 Feed 归属保留(对旧版本就是普通的 generic_rss 站点)。 + +export const SITE_KINDS = ["pt", "public_bt", "rss_feed", "torznab"] as const; + +/** 主机名 → 站点 key 片段:小写,非字母数字换成 -,首尾去 -;空则 feed */ +export function hostSlug(host: string): string { + const s = host + .toLowerCase() + .replace(/^www\./, "") + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 40) + .replace(/-+$/, ""); + return s || "feed"; +} + +function hostOf(urlDisplay: string): string | null { + try { + return new URL(urlDisplay).host || null; + } catch { + return null; + } +} + +// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型 +export async function up(db: Kysely): Promise { + await execDdl( + db, + `alter table site add column kind text not null default 'pt' ${oneOf("kind", SITE_KINDS)}; + update site set kind = 'rss_feed' where adapter = 'generic_rss'; + create index site_kind_idx on site (kind) where deleted_at is null;`, + ); + + const orphans = await db + .selectFrom("feed_source") + .select(["id", "url_display", "name"]) + .where("site_id", "is", null) + .orderBy("created_at") + .execute(); + if (!orphans.length) return; + + const taken = new Set((await db.selectFrom("site").select("key").execute()).map((r) => r.key as string)); + const byHost = new Map(); + for (const f of orphans) { + const host = hostOf(f.url_display as string) ?? "unknown"; + const list = byHost.get(host) ?? []; + list.push({ id: f.id as string, name: f.name as string }); + byHost.set(host, list); + } + for (const [host, feeds] of byHost) { + const base = `rss-${hostSlug(host === "unknown" ? "" : host)}`; + let key = base; + for (let n = 2; taken.has(key); n++) key = `${base}-${n}`; + taken.add(key); + const site = await db + .insertInto("site") + .values({ + key, + name: host === "unknown" ? (feeds[0]?.name ?? "自定义 RSS 源") : host, + kind: "rss_feed", + adapter: "generic_rss", + base_url: host === "unknown" ? "https://invalid.local" : `https://${host}`, + snapshot_enabled: false, + search_default: false, + }) + .returning("id") + .executeTakeFirstOrThrow(); + await db + .updateTable("feed_source") + .set({ + site_id: site.id, + etag: null, + last_modified: null, + cursor: sql`cursor - 'window'`, + }) + .where( + "id", + "in", + feeds.map((f) => f.id), + ) + .execute(); + } +} + +// biome-ignore lint/suspicious/noExplicitAny: 同上 +export async function down(db: Kysely): Promise { + await execDdl( + db, + `drop index if exists site_kind_idx; + alter table site drop column if exists kind;`, + ); +} diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index 2aeb4171..e83dd8df 100644 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -239,6 +239,7 @@ export type ServerMappingMatchStatus = "matched" | "ambiguous" | "unmatched" | " export type SiteAdapter = "nexusphp" | "mteam_api" | "api" | "generic_rss"; export type SiteHealthStatus = "unknown" | "healthy" | "degraded" | "failing" | "disabled"; export type SiteCircuitState = "closed" | "open" | "half_open"; +export type SiteKind = "pt" | "public_bt" | "rss_feed" | "torznab"; export type SiteAccountAuthKind = "cookie" | "passkey" | "api_key" | "password"; export type SiteAccountStatus = "unknown" | "active" | "invalid" | "expired" | "challenge"; export type SiteSnapshotKind = "periodic" | "backfill" | "live_search" | "manual"; @@ -1384,6 +1385,7 @@ export interface SiteTable { proxy: Generated; deleted_at: Timestamptz | null; search_default: Generated; + kind: Generated; } export interface SiteAccountTable {