feat(sources): site kinds and capabilities (IA-2)
site.kind: pt / public_bt / rss_feed / torznab. Each kind declares the acquisition modes it may support (search, list, rss, detail, login, userinfo); a site's effective capabilities = kind ∩ site definition. torznab only defines TorznabSiteConfig and cannot be added yet. Migration 0055: generic_rss sites become rss_feed, the rest pt; feeds without a site are grouped by host into new custom RSS source sites (cursor window and conditional headers cleared so the next fetch writes releases). User DB read-only dry-run: 0 orphan feeds. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
f0e1c77e35
commit
89d2c81de6
6 files changed
+317
-12
No files matched your search
@@ -0,0 +1,100 @@
|
||||
import { type Kysely, sql } from "kysely";
|
||||
import { execDdl, oneOf } from "../src/migration-helpers.ts";
|
||||
|
||||
// IA-2 站点类型:site.kind(pt / public_bt / rss_feed / torznab),各类型声明支持的获取方式(contracts SITE_KINDS)。
|
||||
// - 存量:adapter = generic_rss 的站点(旧"自定义 RSS 站点:只抓 RSS,不登录")→ rss_feed;其余 → pt。
|
||||
// - RSS 归属站点:不属于任何站点的 feed_source 按地址主机名归入「自定义 RSS 源」站点(同主机一个站点,
|
||||
// key = rss-<主机>,重名加序号);清掉这些 Feed 的游标窗口与条件请求头,下次抓取把窗口里的条目重新写成资源
|
||||
// (之前无站点时观察记录被标 ignored/no_site,没有进 source_release)。
|
||||
// down 只删列;迁移建的站点与 Feed 归属保留(对旧版本就是普通的 generic_rss 站点)。
|
||||
|
||||
export const SITE_KINDS = ["pt", "public_bt", "rss_feed", "torznab"] as const;
|
||||
|
||||
/** 主机名 → 站点 key 片段:小写,非字母数字换成 -,首尾去 -;空则 feed */
|
||||
export function hostSlug(host: string): string {
|
||||
const s = host
|
||||
.toLowerCase()
|
||||
.replace(/^www\./, "")
|
||||
.replace(/[^a-z0-9]+/g, "-")
|
||||
.replace(/^-+|-+$/g, "")
|
||||
.slice(0, 40)
|
||||
.replace(/-+$/, "");
|
||||
return s || "feed";
|
||||
}
|
||||
|
||||
function hostOf(urlDisplay: string): string | null {
|
||||
try {
|
||||
return new URL(urlDisplay).host || null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型
|
||||
export async function up(db: Kysely<any>): Promise<void> {
|
||||
await execDdl(
|
||||
db,
|
||||
`alter table site add column kind text not null default 'pt' ${oneOf("kind", SITE_KINDS)};
|
||||
update site set kind = 'rss_feed' where adapter = 'generic_rss';
|
||||
create index site_kind_idx on site (kind) where deleted_at is null;`,
|
||||
);
|
||||
|
||||
const orphans = await db
|
||||
.selectFrom("feed_source")
|
||||
.select(["id", "url_display", "name"])
|
||||
.where("site_id", "is", null)
|
||||
.orderBy("created_at")
|
||||
.execute();
|
||||
if (!orphans.length) return;
|
||||
|
||||
const taken = new Set((await db.selectFrom("site").select("key").execute()).map((r) => r.key as string));
|
||||
const byHost = new Map<string, { id: string; name: string }[]>();
|
||||
for (const f of orphans) {
|
||||
const host = hostOf(f.url_display as string) ?? "unknown";
|
||||
const list = byHost.get(host) ?? [];
|
||||
list.push({ id: f.id as string, name: f.name as string });
|
||||
byHost.set(host, list);
|
||||
}
|
||||
for (const [host, feeds] of byHost) {
|
||||
const base = `rss-${hostSlug(host === "unknown" ? "" : host)}`;
|
||||
let key = base;
|
||||
for (let n = 2; taken.has(key); n++) key = `${base}-${n}`;
|
||||
taken.add(key);
|
||||
const site = await db
|
||||
.insertInto("site")
|
||||
.values({
|
||||
key,
|
||||
name: host === "unknown" ? (feeds[0]?.name ?? "自定义 RSS 源") : host,
|
||||
kind: "rss_feed",
|
||||
adapter: "generic_rss",
|
||||
base_url: host === "unknown" ? "https://invalid.local" : `https://${host}`,
|
||||
snapshot_enabled: false,
|
||||
search_default: false,
|
||||
})
|
||||
.returning("id")
|
||||
.executeTakeFirstOrThrow();
|
||||
await db
|
||||
.updateTable("feed_source")
|
||||
.set({
|
||||
site_id: site.id,
|
||||
etag: null,
|
||||
last_modified: null,
|
||||
cursor: sql`cursor - 'window'`,
|
||||
})
|
||||
.where(
|
||||
"id",
|
||||
"in",
|
||||
feeds.map((f) => f.id),
|
||||
)
|
||||
.execute();
|
||||
}
|
||||
}
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 同上
|
||||
export async function down(db: Kysely<any>): Promise<void> {
|
||||
await execDdl(
|
||||
db,
|
||||
`drop index if exists site_kind_idx;
|
||||
alter table site drop column if exists kind;`,
|
||||
);
|
||||
}
|
||||
@@ -239,6 +239,7 @@ export type ServerMappingMatchStatus = "matched" | "ambiguous" | "unmatched" | "
|
||||
export type SiteAdapter = "nexusphp" | "mteam_api" | "api" | "generic_rss";
|
||||
export type SiteHealthStatus = "unknown" | "healthy" | "degraded" | "failing" | "disabled";
|
||||
export type SiteCircuitState = "closed" | "open" | "half_open";
|
||||
export type SiteKind = "pt" | "public_bt" | "rss_feed" | "torznab";
|
||||
export type SiteAccountAuthKind = "cookie" | "passkey" | "api_key" | "password";
|
||||
export type SiteAccountStatus = "unknown" | "active" | "invalid" | "expired" | "challenge";
|
||||
export type SiteSnapshotKind = "periodic" | "backfill" | "live_search" | "manual";
|
||||
@@ -1384,6 +1385,7 @@ export interface SiteTable {
|
||||
proxy: Generated<boolean>;
|
||||
deleted_at: Timestamptz | null;
|
||||
search_default: Generated<boolean>;
|
||||
kind: Generated<SiteKind>;
|
||||
}
|
||||
|
||||
export interface SiteAccountTable {
|
||||
|
||||
Reference in new issue
Block a user