feat(sites): 站点请求失败留样(site_failure_sample,每站最近 5 份)
- 迁移 0077 新表 site_failure_sample:类型(parse_error / challenge)、请求类型、脱敏 URL、状态码、
关键响应头、错误文案、命中规则、脱敏页面开头(≤ 64 KB);样本几十 KB,不放进每条请求都读的 site.config
- 适配器 onFailureSample 回调接到 runtime.siteAdapter,异步写入并删到最近 5 份,写失败只记日志
- GET /sites/{id}/failure-samples(不带正文)与 /failure-samples/{sampleId}(带正文)
Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei
This commit is contained in:
1 parent
1779e1a0f2
commit
0088c1c9e7
10 files changed
+355
No files matched your search
@@ -0,0 +1,35 @@
|
||||
import type { Kysely } from "kysely";
|
||||
import { dropTables, execDdl } from "../src/migration-helpers.ts";
|
||||
|
||||
// 站点请求失败留样:列表 / 搜索 / 详情报「页面结构无法识别」或「人机验证」时,存一份脱敏后的页面开头(≤ 64 KB)
|
||||
// 与 URL(脱敏)、状态码、关键响应头、请求类型。每站只留最近 5 份(写入时顺手删旧的);站点页「技术信息」里看。
|
||||
// 单独一张表:样本有几十 KB,不放进每条站点请求都要读的 site.config。
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型
|
||||
export async function up(db: Kysely<any>): Promise<void> {
|
||||
await execDdl(
|
||||
db,
|
||||
`
|
||||
create table site_failure_sample (
|
||||
id uuid primary key default gen_random_uuid(),
|
||||
site_id uuid not null references site(id) on delete cascade,
|
||||
kind text not null check (kind in ('parse_error', 'challenge')),
|
||||
request_kind text not null,
|
||||
url text not null,
|
||||
http_status integer,
|
||||
headers jsonb not null default '{}'::jsonb,
|
||||
message text not null,
|
||||
rule text,
|
||||
body text not null,
|
||||
truncated boolean not null default false,
|
||||
created_at timestamptz not null default now()
|
||||
);
|
||||
create index site_failure_sample_site_idx on site_failure_sample (site_id, created_at desc);
|
||||
`,
|
||||
);
|
||||
}
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 同上
|
||||
export async function down(db: Kysely<any>): Promise<void> {
|
||||
await execDdl(db, dropTables(["site_failure_sample"]));
|
||||
}
|
||||
@@ -255,6 +255,7 @@ export type SiteCircuitState = "closed" | "open" | "half_open";
|
||||
export type SiteKind = "pt" | "public_bt" | "rss_feed" | "torznab" | "subtitle";
|
||||
export type SiteAccountAuthKind = "cookie" | "passkey" | "api_key" | "password";
|
||||
export type SiteAccountStatus = "unknown" | "active" | "invalid" | "expired" | "challenge";
|
||||
export type SiteFailureSampleKind = "parse_error" | "challenge";
|
||||
export type SiteSnapshotKind = "periodic" | "backfill" | "live_search" | "manual";
|
||||
export type SiteSnapshotStatus = "running" | "ok" | "partial" | "failed";
|
||||
export type SourceReleaseResolutionStatus = "unresolved" | "resolved" | "ambiguous" | "unidentifiable" | "rejected";
|
||||
@@ -1491,6 +1492,21 @@ export interface SiteAccountTable {
|
||||
ua_pending: string | null;
|
||||
}
|
||||
|
||||
export interface SiteFailureSampleTable {
|
||||
id: Generated<string>;
|
||||
site_id: string;
|
||||
kind: SiteFailureSampleKind;
|
||||
request_kind: string;
|
||||
url: string;
|
||||
http_status: number | null;
|
||||
headers: Generated<Json>;
|
||||
message: string;
|
||||
rule: string | null;
|
||||
body: string;
|
||||
truncated: Generated<boolean>;
|
||||
created_at: GeneratedTimestamptz;
|
||||
}
|
||||
|
||||
export interface SiteRequestCountTable {
|
||||
site_id: string;
|
||||
hour: Timestamptz;
|
||||
@@ -1825,6 +1841,7 @@ export interface DB {
|
||||
settings: SettingsTable;
|
||||
site: SiteTable;
|
||||
site_account: SiteAccountTable;
|
||||
site_failure_sample: SiteFailureSampleTable;
|
||||
site_request_count: SiteRequestCountTable;
|
||||
site_snapshot: SiteSnapshotTable;
|
||||
site_user_data: SiteUserDataTable;
|
||||
@@ -2051,6 +2068,9 @@ export type SiteUpdate = Updateable<SiteTable>;
|
||||
export type SiteAccountRow = Selectable<SiteAccountTable>;
|
||||
export type NewSiteAccount = Insertable<SiteAccountTable>;
|
||||
export type SiteAccountUpdate = Updateable<SiteAccountTable>;
|
||||
export type SiteFailureSampleRow = Selectable<SiteFailureSampleTable>;
|
||||
export type NewSiteFailureSample = Insertable<SiteFailureSampleTable>;
|
||||
export type SiteFailureSampleUpdate = Updateable<SiteFailureSampleTable>;
|
||||
export type SiteRequestCountRow = Selectable<SiteRequestCountTable>;
|
||||
export type NewSiteRequestCount = Insertable<SiteRequestCountTable>;
|
||||
export type SiteRequestCountUpdate = Updateable<SiteRequestCountTable>;
|
||||
|
||||
@@ -130,6 +130,8 @@ const EXPECTED_TABLES = [
|
||||
"subtitle_write",
|
||||
// 集数映射规则(0066)
|
||||
"episode_mapping_rule",
|
||||
// 站点请求失败留样(0077)
|
||||
"site_failure_sample",
|
||||
].sort();
|
||||
|
||||
/** 不走 entityTable() 的表:追加型/引擎自有表,没有 uuid id + created_at/updated_at 三件套 */
|
||||
@@ -163,6 +165,8 @@ const NON_ENTITY_TABLES = [
|
||||
"kv_cache",
|
||||
// 站点请求计数:(site_id, hour) 主键
|
||||
"site_request_count",
|
||||
// 失败留样:只追加,只有 created_at
|
||||
"site_failure_sample",
|
||||
];
|
||||
|
||||
describe.skipIf(skip)("schema migrations (real PG)", () => {
|
||||
|
||||
Reference in new issue
Block a user