From 0088c1c9e793d726e3b73de09b1f4a29daa3d156 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 6 Oct 2026 15:41:00 +0800 Subject: [PATCH] =?UTF-8?q?feat(sites):=20=E7=AB=99=E7=82=B9=E8=AF=B7?= =?UTF-8?q?=E6=B1=82=E5=A4=B1=E8=B4=A5=E7=95=99=E6=A0=B7(site=5Ffailure=5F?= =?UTF-8?q?sample,=E6=AF=8F=E7=AB=99=E6=9C=80=E8=BF=91=205=20=E4=BB=BD)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 迁移 0077 新表 site_failure_sample:类型(parse_error / challenge)、请求类型、脱敏 URL、状态码、 关键响应头、错误文案、命中规则、脱敏页面开头(≤ 64 KB);样本几十 KB,不放进每条请求都读的 site.config - 适配器 onFailureSample 回调接到 runtime.siteAdapter,异步写入并删到最近 5 份,写失败只记日志 - GET /sites/{id}/failure-samples(不带正文)与 /failure-samples/{sampleId}(带正文) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei --- .../sources-identity/routes/sources.ts | 34 +++++ .../src/modules/sources-identity/runtime.ts | 3 + .../service/failure-sample.ts | 83 +++++++++++ .../sources-identity/failure-sample.test.ts | 129 ++++++++++++++++++ packages/contracts/src/api/sources.ts | 16 +++ packages/contracts/src/sources.ts | 29 ++++ packages/contracts/test/db-sync.test.ts | 2 + .../db/migrations/0077_site_failure_sample.ts | 35 +++++ packages/db/src/types.ts | 20 +++ packages/db/test/schema.test.ts | 4 + 10 files changed, 355 insertions(+) create mode 100644 apps/server/src/modules/sources-identity/service/failure-sample.ts create mode 100644 apps/server/test/sources-identity/failure-sample.test.ts create mode 100644 packages/db/migrations/0077_site_failure_sample.ts diff --git a/apps/server/src/modules/sources-identity/routes/sources.ts b/apps/server/src/modules/sources-identity/routes/sources.ts index 25dbc4cb..98ee8aad 100644 --- a/apps/server/src/modules/sources-identity/routes/sources.ts +++ b/apps/server/src/modules/sources-identity/routes/sources.ts @@ -17,6 +17,8 @@ import { SiteCredentialsPut, SiteCredentialsResult, SiteDefinitionView, + SiteFailureSample, + SiteFailureSampleSummary, SiteHealthResult, SiteRefreshLoginAccepted, SiteRefreshLoginInput, @@ -47,6 +49,7 @@ import { userDataHistory, } from "../service/account.ts"; import { audit, closeDecisions, type Exec } from "../service/common.ts"; +import { getFailureSample, listFailureSamples } from "../service/failure-sample.ts"; import { feedDefaultsForSite } from "../service/feeds.ts"; import { assignReleaseIdentity } from "../service/identity-admin.ts"; import { ensureRssSite, requireCapability, siteToApi, uniqueSiteKey } from "../site-kinds.ts"; @@ -376,6 +379,37 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) { }, ); + route( + app, + { + method: "get", + path: "/sites/{id}/failure-samples", + scope: "read", + summary: "站点请求失败留样：最近几次页面结构无法识别 / 人机验证时的脱敏页面（列表不带正文）", + tags: T_SITES, + params: IdParam, + response: z.object({ items: z.array(SiteFailureSampleSummary) }), + }, + async ({ params }) => ({ items: await listFailureSamples(rt().ctx, params.id) }), + ); + route( + app, + { + method: "get", + path: "/sites/{id}/failure-samples/{sampleId}", + scope: "read", + summary: "一份失败留样（含脱敏后的页面正文）", + tags: T_SITES, + params: z.object({ id: Id, sampleId: Id }), + response: SiteFailureSample, + }, + async ({ params }) => { + const s = await getFailureSample(rt().ctx, params.id, params.sampleId); + if (!s) throw new ApiException("not_found", "样本不存在"); + return s; + }, + ); + // ── W9-A 站点中心:删除、健康检测、刷新登录、用户数据、凭证 ── route( app, diff --git a/apps/server/src/modules/sources-identity/runtime.ts b/apps/server/src/modules/sources-identity/runtime.ts index e5bbc4a0..64854cec 100644 --- a/apps/server/src/modules/sources-identity/runtime.ts +++ b/apps/server/src/modules/sources-identity/runtime.ts @@ -26,6 +26,7 @@ import { } from "../../shared-cache.ts"; import type { ServerContext } from "../types.ts"; import { CloudflareGate, withClearance } from "./service/cloudflare.ts"; +import { failureSampleSink } from "./service/failure-sample.ts"; import { readBrowserUserAgent, resolveUserAgent, type UserAgentDefaults } from "./service/user-agent.ts"; // 模块运行时依赖:站点适配器与限流器(每站一个实例,令牌在 PG 跨进程共享)、RSS 抓取器、TMDB 客户端、时钟。 @@ -333,6 +334,8 @@ export class SourcesRuntime { // 每条请求都经 siteFetch:代理 + 按账号现场取 UA(BR-4) fetch: this.siteFetch({ id: site.id, proxy }, defaults, () => this.siteUserAgent(site.id)), ...(baseUrl ? { baseUrl } : {}), + // parse_error / challenge 的脱敏页面样本存进 site_failure_sample(每站最近 5 份) + onFailureSample: failureSampleSink(this.ctx, site.id), }); this.adapters.set(site.id, { sig, limiter, adapter }); return adapter; diff --git a/apps/server/src/modules/sources-identity/service/failure-sample.ts b/apps/server/src/modules/sources-identity/service/failure-sample.ts new file mode 100644 index 00000000..47f9b954 --- /dev/null +++ b/apps/server/src/modules/sources-identity/service/failure-sample.ts @@ -0,0 +1,83 @@ +import type { SiteFailureSample as AdapterSample } from "@umt/adapters"; +import type { SiteFailureSample, SiteFailureSampleSummary } from "@umt/contracts"; +import { sql } from "@umt/db"; +import type { ServerContext } from "../../types.ts"; +import { toApi } from "../rows.ts"; + +// 站点请求失败留样(表 site_failure_sample,迁移 0077):适配器报 parse_error / challenge 时交出脱敏页面样本 +// (SiteAdapterDeps.onFailureSample),这里存下来,每站只留最近 SAMPLES_PER_SITE 份。站点页「技术信息」里看。 +// 样本在适配器里已脱敏(个人信息块、passkey / authkey / uid / Cookie / 邮箱),这里不再处理正文。 + +/** 每站保留的样本数 */ +export const SAMPLES_PER_SITE = 5; + +export async function recordFailureSample(ctx: ServerContext, siteId: string, s: AdapterSample): Promise { + await ctx.db + .insertInto("site_failure_sample") + .values({ + site_id: siteId, + kind: s.kind, + request_kind: s.requestKind, + url: s.url, + http_status: s.httpStatus, + headers: JSON.stringify(s.headers), + message: s.message.slice(0, 500), + rule: s.rule, + body: s.body, + truncated: s.truncated, + created_at: new Date(s.capturedAt), + }) + .execute(); + await sql`delete from site_failure_sample where site_id = ${siteId} and id not in ( + select id from site_failure_sample where site_id = ${siteId} order by created_at desc, id desc limit ${SAMPLES_PER_SITE})`.execute( + ctx.db, + ); +} + +/** 留样回调:不等写库,失败只记日志(不影响站点请求本身的结果) */ +export function failureSampleSink(ctx: ServerContext, siteId: string): (s: AdapterSample) => void { + return (s) => { + void recordFailureSample(ctx, siteId, s).catch((err) => + ctx.logger.warn({ siteId, err: { message: (err as Error).message } }, "站点失败页面留样写入失败"), + ); + }; +} + +export async function listFailureSamples(ctx: ServerContext, siteId: string): Promise { + const rows = await ctx.db + .selectFrom("site_failure_sample") + .select([ + "id", + "site_id", + "kind", + "request_kind", + "url", + "http_status", + "headers", + "message", + "rule", + "truncated", + "created_at", + sql`octet_length(body)`.as("body_bytes"), + ]) + .where("site_id", "=", siteId) + .orderBy("created_at", "desc") + .orderBy("id", "desc") + .limit(SAMPLES_PER_SITE) + .execute(); + return rows.map((r) => toApi(r)); +} + +export async function getFailureSample( + ctx: ServerContext, + siteId: string, + sampleId: string, +): Promise { + const row = await ctx.db + .selectFrom("site_failure_sample") + .selectAll() + .where("site_id", "=", siteId) + .where("id", "=", sampleId) + .executeTakeFirst(); + return row ? toApi(row) : null; +} diff --git a/apps/server/test/sources-identity/failure-sample.test.ts b/apps/server/test/sources-identity/failure-sample.test.ts new file mode 100644 index 00000000..d0aae310 --- /dev/null +++ b/apps/server/test/sources-identity/failure-sample.test.ts @@ -0,0 +1,129 @@ +import type { SiteError } from "@umt/adapters"; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; +import { SAMPLES_PER_SITE } from "../../src/modules/sources-identity/service/failure-sample.ts"; +import { snapshotSite } from "../../src/modules/sources-identity/service/snapshot.ts"; +import { createHarness, type Harness, pgAvailable, READ_KEY } from "./harness.ts"; + +// 站点请求失败留样:列表 / 搜索报「页面结构无法识别」时存脱敏页面样本(site_failure_sample),每站只留最近 5 份; +// 站点页技术信息经 GET /sites/{id}/failure-samples(列表不带正文)与 /failure-samples/{sampleId}(带正文)取。 +// 站点前面是可控的 fetch:返回什么页面由测试决定(不访问真实站点)。 + +const up = await pgAvailable(); +const SECRET = "0123456789abcdef0123456789abcdef"; +/** 改版页:认不出种子表,还带着要脱敏的东西 */ +const BROKEN = `Audiences :: 种子 +
欢迎 alice_cn UID 29191
+

邮箱 alice@example.com

dl + +
Dune
`; + +// biome-ignore lint/suspicious/noExplicitAny: 响应体断言 +const json = async (r: Response): Promise => { + const body = await r.json(); + if (!r.ok) throw new Error(`${r.status} ${JSON.stringify(body)}`); + return body.data; +}; + +/** 轮询到条件成立(留样是异步写入的) */ +async function until(f: () => Promise, ok: (v: T) => boolean, ms = 5_000): Promise { + const deadline = Date.now() + ms; + for (;;) { + const v = await f(); + if (ok(v) || Date.now() > deadline) return v; + await new Promise((r) => setTimeout(r, 50)); + } +} + +describe.skipIf(!up)("站点失败留样", () => { + let h: Harness; + let siteId: string; + let page = BROKEN; + const fetchImpl = (async () => + new Response(page, { + status: 200, + headers: { "content-type": "text/html; charset=utf-8", server: "nginx" }, + })) as typeof fetch; + const site = () => h.db.selectFrom("site").selectAll().where("id", "=", siteId).executeTakeFirstOrThrow(); + const rows = () => + h.db.selectFrom("site_failure_sample").selectAll().where("site_id", "=", siteId).orderBy("created_at").execute(); + + beforeAll(async () => { + h = await createHarness("failsample", { fetch: fetchImpl }); + const s = await json( + await h.req("/sites", { + body: { + key: "audiences", + name: "观众", + adapter: "nexusphp", + baseUrl: "https://aud.test/", + rateLimit: { minIntervalMs: 0, concurrency: 10 }, + }, + }), + ); + siteId = s.id; + await json( + await h.req(`/sites/${siteId}/credentials`, { + method: "PUT", + body: { kind: "cookie", cookie: "c_secure_uid=1; c_secure_pass=abc" }, + }), + ); + }); + afterAll(async () => { + await h?.close(); + }); + beforeEach(async () => { + page = BROKEN; + await h.db.deleteFrom("site_failure_sample").where("site_id", "=", siteId).execute(); + }); + + it("列表快照报 parse_error → 留脱敏样本(URL、状态码、关键响应头、请求类型);接口可看", async () => { + const r = await snapshotSite(h.rt, { siteId, kind: "manual" }); + expect(r.status).toBe("failed"); + expect(r.error).toContain("parse_error"); + const got = await until(rows, (x) => x.length > 0); + expect(got).toHaveLength(1); + const s = got[0]!; + expect(s).toMatchObject({ + kind: "parse_error", + request_kind: "list", + http_status: 200, + headers: { server: "nginx" }, + }); + expect(s.message).toContain("页面结构无法识别"); + for (const leak of [SECRET, "alice@example.com", "alice_cn", "29191", "topsecretvalue"]) { + expect(s.body).not.toContain(leak); + expect(s.url).not.toContain(leak); + } + expect(s.body).toContain("new-grid"); + + const list = await json(await h.req(`/sites/${siteId}/failure-samples`, { key: READ_KEY })); + expect(list.items).toHaveLength(1); + expect(list.items[0]).toMatchObject({ id: s.id, kind: "parse_error", requestKind: "list", httpStatus: 200 }); + expect(list.items[0].body).toBeUndefined(); + expect(list.items[0].bodyBytes).toBeGreaterThan(0); + const one = await json(await h.req(`/sites/${siteId}/failure-samples/${s.id}`, { key: READ_KEY })); + expect(one.body).toBe(s.body); + const miss = await h.req(`/sites/${siteId}/failure-samples/00000000-0000-4000-8000-000000000000`); + expect(miss.status).toBe(404); + }); + + it(`每站只留最近 ${SAMPLES_PER_SITE} 份`, async () => { + const adapter = h.rt.siteAdapter(await site()); + for (let i = 1; i <= 7; i++) { + const e = await adapter.search(`k${i}`).catch((x) => x); + expect((e as SiteError).kind).toBe("parse_error"); + } + const kw = (x: { url: string }[]) => x.map((r) => /search=(k\d)/.exec(r.url)?.[1]).join(","); + const got = await until(rows, (x) => kw(x) === "k3,k4,k5,k6,k7"); + expect(kw(got)).toBe("k3,k4,k5,k6,k7"); + expect(got.every((r) => r.request_kind === "search")).toBe(true); + }); + + it("正常页面不留样", async () => { + page = `Audiences :: 种子
没有种子
`; + const adapter = h.rt.siteAdapter(await site()); + await adapter.search("x").catch(() => null); + await new Promise((r) => setTimeout(r, 200)); + expect(await rows()).toEqual([]); + }); +}); diff --git a/packages/contracts/src/api/sources.ts b/packages/contracts/src/api/sources.ts index ca5e4d0c..87e6c92b 100644 --- a/packages/contracts/src/api/sources.ts +++ b/packages/contracts/src/api/sources.ts @@ -86,6 +86,22 @@ export const sourcesApi = { params: IdParam, response: sources.SiteRequestStats, }), + failureSamples: ep({ + method: "get", + path: "/sites/{id}/failure-samples", + scope: "read", + summary: "站点请求失败留样：最近几次页面结构无法识别 / 人机验证时的脱敏页面（列表不带正文）", + params: IdParam, + response: z.object({ items: z.array(sources.SiteFailureSampleSummary) }), + }), + failureSample: ep({ + method: "get", + path: "/sites/{id}/failure-samples/{sampleId}", + scope: "read", + summary: "一份失败留样（含脱敏后的页面正文）", + params: z.object({ id: Id, sampleId: Id }), + response: sources.SiteFailureSample, + }), credentials: ep({ method: "put", path: "/sites/{id}/credentials", diff --git a/packages/contracts/src/sources.ts b/packages/contracts/src/sources.ts index 105670f3..2ace97d8 100644 --- a/packages/contracts/src/sources.ts +++ b/packages/contracts/src/sources.ts @@ -701,6 +701,35 @@ export const SiteSnapshot = EntityBase.extend({ }); export type SiteSnapshot = z.infer; +/** 失败留样的类型:页面结构无法识别 / 人机验证 */ +export const SiteFailureSampleKind = z.enum(["parse_error", "challenge"]); +export type SiteFailureSampleKind = z.infer; + +/** + * 站点请求失败留样(列表 / 搜索 / 详情报 parse_error / challenge 时存的脱敏页面开头,每站最近 5 份)。 + * requestKind:search / list / detail / health / other;headers 只有 server、cf-mitigated、cf-ray、content-type。 + */ +export const SiteFailureSample = z.object({ + id: Id, + siteId: Id, + kind: SiteFailureSampleKind, + requestKind: z.string(), + /** 已脱敏 */ + url: z.string(), + httpStatus: z.number().int().nullable(), + headers: z.record(z.string(), z.string()), + message: z.string(), + rule: z.string().nullable(), + /** 脱敏后的页面开头(≤ 64 KB) */ + body: z.string(), + truncated: z.boolean(), + createdAt: Timestamp, +}); +export type SiteFailureSample = z.infer; +/** 列表里不带页面正文,只给字节数 */ +export const SiteFailureSampleSummary = SiteFailureSample.omit({ body: true }).extend({ bodyBytes: NonNegInt }); +export type SiteFailureSampleSummary = z.infer; + /** 站点访问默认(站点定义 umt.defaults 与 settings.sites 合并后,取更保守的一方) */ export const SiteAccessDefaults = z.object({ snapshotIntervalSec: z.number().int(), diff --git a/packages/contracts/test/db-sync.test.ts b/packages/contracts/test/db-sync.test.ts index 261bb4ad..026c736f 100644 --- a/packages/contracts/test/db-sync.test.ts +++ b/packages/contracts/test/db-sync.test.ts @@ -105,6 +105,7 @@ const ROWS: Record MetadataEnrichment: { row: C.MetadataEnrichment }, ResolutionAttempt: { row: C.ResolutionAttempt }, SiteSnapshot: { row: C.SiteSnapshot }, + SiteFailureSample: { row: C.SiteFailureSample }, // W4-C 浏览器实体(0023) // BR-4:identity 是浏览器固定身份的缓存(站点请求 UA 用),不进 API BrowserProvider: { row: C.BrowserProviderRow, dbOnly: ["identity"] }, @@ -252,6 +253,7 @@ const ENUMS: Record = { ScheduledJobLastStatus: C.ScheduleRunStatus, ScheduledJobRunTrigger: C.ScheduleTrigger, ScheduledJobRunStatus: C.ScheduleRunStatus, + SiteFailureSampleKind: C.SiteFailureSampleKind, ScrapeRecordStatus: C.ScrapeStatus, ScrapeRecordTrigger: C.ScrapeTrigger, SubtitleWriteSource: C.SubtitleSource, diff --git a/packages/db/migrations/0077_site_failure_sample.ts b/packages/db/migrations/0077_site_failure_sample.ts new file mode 100644 index 00000000..ccf76b5c --- /dev/null +++ b/packages/db/migrations/0077_site_failure_sample.ts @@ -0,0 +1,35 @@ +import type { Kysely } from "kysely"; +import { dropTables, execDdl } from "../src/migration-helpers.ts"; + +// 站点请求失败留样:列表 / 搜索 / 详情报「页面结构无法识别」或「人机验证」时,存一份脱敏后的页面开头(≤ 64 KB) +// 与 URL(脱敏)、状态码、关键响应头、请求类型。每站只留最近 5 份(写入时顺手删旧的);站点页「技术信息」里看。 +// 单独一张表:样本有几十 KB,不放进每条站点请求都要读的 site.config。 + +// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型 +export async function up(db: Kysely): Promise { + await execDdl( + db, + ` +create table site_failure_sample ( + id uuid primary key default gen_random_uuid(), + site_id uuid not null references site(id) on delete cascade, + kind text not null check (kind in ('parse_error', 'challenge')), + request_kind text not null, + url text not null, + http_status integer, + headers jsonb not null default '{}'::jsonb, + message text not null, + rule text, + body text not null, + truncated boolean not null default false, + created_at timestamptz not null default now() +); +create index site_failure_sample_site_idx on site_failure_sample (site_id, created_at desc); +`, + ); +} + +// biome-ignore lint/suspicious/noExplicitAny: 同上 +export async function down(db: Kysely): Promise { + await execDdl(db, dropTables(["site_failure_sample"])); +} diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index 4daaf3b8..3c4af318 100644 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -255,6 +255,7 @@ export type SiteCircuitState = "closed" | "open" | "half_open"; export type SiteKind = "pt" | "public_bt" | "rss_feed" | "torznab" | "subtitle"; export type SiteAccountAuthKind = "cookie" | "passkey" | "api_key" | "password"; export type SiteAccountStatus = "unknown" | "active" | "invalid" | "expired" | "challenge"; +export type SiteFailureSampleKind = "parse_error" | "challenge"; export type SiteSnapshotKind = "periodic" | "backfill" | "live_search" | "manual"; export type SiteSnapshotStatus = "running" | "ok" | "partial" | "failed"; export type SourceReleaseResolutionStatus = "unresolved" | "resolved" | "ambiguous" | "unidentifiable" | "rejected"; @@ -1491,6 +1492,21 @@ export interface SiteAccountTable { ua_pending: string | null; } +export interface SiteFailureSampleTable { + id: Generated; + site_id: string; + kind: SiteFailureSampleKind; + request_kind: string; + url: string; + http_status: number | null; + headers: Generated; + message: string; + rule: string | null; + body: string; + truncated: Generated; + created_at: GeneratedTimestamptz; +} + export interface SiteRequestCountTable { site_id: string; hour: Timestamptz; @@ -1825,6 +1841,7 @@ export interface DB { settings: SettingsTable; site: SiteTable; site_account: SiteAccountTable; + site_failure_sample: SiteFailureSampleTable; site_request_count: SiteRequestCountTable; site_snapshot: SiteSnapshotTable; site_user_data: SiteUserDataTable; @@ -2051,6 +2068,9 @@ export type SiteUpdate = Updateable; export type SiteAccountRow = Selectable; export type NewSiteAccount = Insertable; export type SiteAccountUpdate = Updateable; +export type SiteFailureSampleRow = Selectable; +export type NewSiteFailureSample = Insertable; +export type SiteFailureSampleUpdate = Updateable; export type SiteRequestCountRow = Selectable; export type NewSiteRequestCount = Insertable; export type SiteRequestCountUpdate = Updateable; diff --git a/packages/db/test/schema.test.ts b/packages/db/test/schema.test.ts index cb37d6a4..0b21163b 100644 --- a/packages/db/test/schema.test.ts +++ b/packages/db/test/schema.test.ts @@ -130,6 +130,8 @@ const EXPECTED_TABLES = [ "subtitle_write", // 集数映射规则(0066) "episode_mapping_rule", + // 站点请求失败留样(0077) + "site_failure_sample", ].sort(); /** 不走 entityTable() 的表:追加型/引擎自有表,没有 uuid id + created_at/updated_at 三件套 */ @@ -163,6 +165,8 @@ const NON_ENTITY_TABLES = [ "kv_cache", // 站点请求计数:(site_id, hour) 主键 "site_request_count", + // 失败留样:只追加,只有 created_at + "site_failure_sample", ]; describe.skipIf(skip)("schema migrations (real PG)", () => {