diff --git a/apps/server/src/modules/sources-identity/routes/sources.ts b/apps/server/src/modules/sources-identity/routes/sources.ts index 25dbc4cb..98ee8aad 100644 --- a/apps/server/src/modules/sources-identity/routes/sources.ts +++ b/apps/server/src/modules/sources-identity/routes/sources.ts @@ -17,6 +17,8 @@ import { SiteCredentialsPut, SiteCredentialsResult, SiteDefinitionView, + SiteFailureSample, + SiteFailureSampleSummary, SiteHealthResult, SiteRefreshLoginAccepted, SiteRefreshLoginInput, @@ -47,6 +49,7 @@ import { userDataHistory, } from "../service/account.ts"; import { audit, closeDecisions, type Exec } from "../service/common.ts"; +import { getFailureSample, listFailureSamples } from "../service/failure-sample.ts"; import { feedDefaultsForSite } from "../service/feeds.ts"; import { assignReleaseIdentity } from "../service/identity-admin.ts"; import { ensureRssSite, requireCapability, siteToApi, uniqueSiteKey } from "../site-kinds.ts"; @@ -376,6 +379,37 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) { }, ); + route( + app, + { + method: "get", + path: "/sites/{id}/failure-samples", + scope: "read", + summary: "站点请求失败留样:最近几次页面结构无法识别 / 人机验证时的脱敏页面(列表不带正文)", + tags: T_SITES, + params: IdParam, + response: z.object({ items: z.array(SiteFailureSampleSummary) }), + }, + async ({ params }) => ({ items: await listFailureSamples(rt().ctx, params.id) }), + ); + route( + app, + { + method: "get", + path: "/sites/{id}/failure-samples/{sampleId}", + scope: "read", + summary: "一份失败留样(含脱敏后的页面正文)", + tags: T_SITES, + params: z.object({ id: Id, sampleId: Id }), + response: SiteFailureSample, + }, + async ({ params }) => { + const s = await getFailureSample(rt().ctx, params.id, params.sampleId); + if (!s) throw new ApiException("not_found", "样本不存在"); + return s; + }, + ); + // ── W9-A 站点中心:删除、健康检测、刷新登录、用户数据、凭证 ── route( app, diff --git a/apps/server/src/modules/sources-identity/runtime.ts b/apps/server/src/modules/sources-identity/runtime.ts index e5bbc4a0..64854cec 100644 --- a/apps/server/src/modules/sources-identity/runtime.ts +++ b/apps/server/src/modules/sources-identity/runtime.ts @@ -26,6 +26,7 @@ import { } from "../../shared-cache.ts"; import type { ServerContext } from "../types.ts"; import { CloudflareGate, withClearance } from "./service/cloudflare.ts"; +import { failureSampleSink } from "./service/failure-sample.ts"; import { readBrowserUserAgent, resolveUserAgent, type UserAgentDefaults } from "./service/user-agent.ts"; // 模块运行时依赖:站点适配器与限流器(每站一个实例,令牌在 PG 跨进程共享)、RSS 抓取器、TMDB 客户端、时钟。 @@ -333,6 +334,8 @@ export class SourcesRuntime { // 每条请求都经 siteFetch:代理 + 按账号现场取 UA(BR-4) fetch: this.siteFetch({ id: site.id, proxy }, defaults, () => this.siteUserAgent(site.id)), ...(baseUrl ? { baseUrl } : {}), + // parse_error / challenge 的脱敏页面样本存进 site_failure_sample(每站最近 5 份) + onFailureSample: failureSampleSink(this.ctx, site.id), }); this.adapters.set(site.id, { sig, limiter, adapter }); return adapter; diff --git a/apps/server/src/modules/sources-identity/service/cloudflare.ts b/apps/server/src/modules/sources-identity/service/cloudflare.ts index 3e540f8f..0590d495 100644 --- a/apps/server/src/modules/sources-identity/service/cloudflare.ts +++ b/apps/server/src/modules/sources-identity/service/cloudflare.ts @@ -1,4 +1,4 @@ -import { detectChallenge, mergeCookieHeaders, parseCookieHeader } from "@umt/adapters"; +import { CHALLENGE_OUTCOME_HEADER, detectChallenge, mergeCookieHeaders, parseCookieHeader } from "@umt/adapters"; import { type SiteCloudflareState, siteCloudflareState } from "@umt/contracts"; import { type SiteAccountRow, type SiteRow, sql } from "@umt/db"; import { egressKey, siteEgress } from "../../../egress.ts"; @@ -11,12 +11,14 @@ import { resolveUserAgent } from "./user-agent.ts"; // 站点 HTTP 请求(SourcesRuntime.siteFetch)经这里: // 0. 站点在"浏览器代理期"(config.cloudflare.browserUntil 未到)且请求是页面(GET + Accept text/html)→ 直接由浏览器代抓; // 1. 发 HTTP;账号 UA 跟随浏览器、出口与浏览器一致、复用没失败过时带上浏览器拿到的 cf_clearance(__cf_bm 不带); -// 2. 响应是挑战(cf-mitigated: challenge,或 403 / 503 / 429 的挑战页)→ 不算凭据错误,记 lastChallengeAt; +// 2. 响应是挑战(cf-mitigated: challenge,或 403 / 503 / 429 的挑战页,或页面请求返回 200 的挑战页 / 人机验证门页) +// → 不算凭据错误,记 lastChallengeAt; // 3. 过挑战(同站 30 分钟最多一次、同一时刻只跑一次;只处理有登录态账号的站点):浏览器用该账号 profile 打开同一地址 → // ① 等它自己过(≤15 秒)→ ② 复选框 / Turnstile 派视觉子流程点一下(最多 2 次)→ ③ 选图 / 输码类转人工(待处理事项); // 4. 过了:读回 cf_clearance + navigator.userAgent;满足复用条件就存下来,HTTP 重试 1 次,结果记 cfReuse = ok / fails; // 5. 仍被挑战:页面请求改由浏览器导航取 HTML 交给原解析器,站点记 6 小时浏览器代理期;其它请求(下载、RSS、POST)照常报挑战。 -// 不接任何打码服务。 +// 没处理掉时返回的挑战响应带 `x-umt-challenge: <结果>`(needs_human / failed / throttled / no_browser / no_account), +// 适配器据此写错误文案(自动处理到哪一步、要不要人)。不接任何打码服务。 /** 同站两次过挑战的最短间隔 */ export const CF_ATTEMPT_INTERVAL_MS = 30 * 60_000; @@ -132,6 +134,13 @@ export function clearanceUsable(input: { ); } +/** 挑战响应上标出级联结果(响应头不可改,重新包一层) */ +function tagOutcome(res: Response, outcome: string): Response { + const headers = new Headers(res.headers); + headers.set(CHALLENGE_OUTCOME_HEADER, outcome); + return new Response(res.body, { status: res.status, statusText: res.statusText, headers }); +} + export class CloudflareGate { private readonly cache = new Map(); /** 进程内同站同一时刻只过一次挑战:后到的请求等同一个结果 */ @@ -323,19 +332,19 @@ export class CloudflareGate { // 1. HTTP(满足条件时带 cf_clearance) // 从没遇到过挑战的站点不查账号(每条请求少两次查询) const extra = st.lastChallengeAt ? await this.clearanceCookie(siteId, req).catch(() => null) : null; - const first = await detectChallenge(await send(extra)); + const first = await detectChallenge(await send(extra), { page: pagey }); if (!first.rule) return first.res; // 2. 挑战:记下来;没有浏览器 / 没有登录态账号 → 原样返回(适配器报 challenge,不算凭据错误) await this.patch(siteId, { lastChallengeAt: now.toISOString(), lastRule: first.rule }); const port = challengeBrowserFor(this.ctx); - if (!port) return first.res; + if (!port) return tagOutcome(first.res, "no_browser"); const acc = await this.loggedInAccount(siteId); if (!acc) { await this.patch(siteId, { lastResult: "skipped", lastError: "站点没有已登录的账号,不处理挑战" }); - return first.res; + return tagOutcome(first.res, "no_account"); } const site = await this.ctx.db.selectFrom("site").selectAll().where("id", "=", siteId).executeTakeFirst(); - if (!site?.enabled || site.deleted_at) return first.res; + if (!site?.enabled || site.deleted_at) return tagOutcome(first.res, "failed"); // 3. 过挑战 const out = await this.solveOnce(site, acc, req.url); st = await this.state(siteId, true); @@ -357,7 +366,7 @@ export class CloudflareGate { const retryExtra = await this.clearanceCookie(siteId, req).catch(() => null); if (retryExtra) { await this.rt.limiterOf(siteId)?.acquire(); - const second = await detectChallenge(await send(retryExtra)); + const second = await detectChallenge(await send(retryExtra), { page: pagey }); if (!second.rule) { await this.patch(siteId, { cfReuse: "ok", lastResult: "reuse_ok", browserUntil: null }); return second.res; @@ -387,7 +396,7 @@ export class CloudflareGate { return r; } } - return first.res; + return tagOutcome(first.res, !out ? "throttled" : out.status === "needs_human" ? "needs_human" : "failed"); } /** diff --git a/apps/server/src/modules/sources-identity/service/failure-sample.ts b/apps/server/src/modules/sources-identity/service/failure-sample.ts new file mode 100644 index 00000000..47f9b954 --- /dev/null +++ b/apps/server/src/modules/sources-identity/service/failure-sample.ts @@ -0,0 +1,83 @@ +import type { SiteFailureSample as AdapterSample } from "@umt/adapters"; +import type { SiteFailureSample, SiteFailureSampleSummary } from "@umt/contracts"; +import { sql } from "@umt/db"; +import type { ServerContext } from "../../types.ts"; +import { toApi } from "../rows.ts"; + +// 站点请求失败留样(表 site_failure_sample,迁移 0077):适配器报 parse_error / challenge 时交出脱敏页面样本 +// (SiteAdapterDeps.onFailureSample),这里存下来,每站只留最近 SAMPLES_PER_SITE 份。站点页「技术信息」里看。 +// 样本在适配器里已脱敏(个人信息块、passkey / authkey / uid / Cookie / 邮箱),这里不再处理正文。 + +/** 每站保留的样本数 */ +export const SAMPLES_PER_SITE = 5; + +export async function recordFailureSample(ctx: ServerContext, siteId: string, s: AdapterSample): Promise { + await ctx.db + .insertInto("site_failure_sample") + .values({ + site_id: siteId, + kind: s.kind, + request_kind: s.requestKind, + url: s.url, + http_status: s.httpStatus, + headers: JSON.stringify(s.headers), + message: s.message.slice(0, 500), + rule: s.rule, + body: s.body, + truncated: s.truncated, + created_at: new Date(s.capturedAt), + }) + .execute(); + await sql`delete from site_failure_sample where site_id = ${siteId} and id not in ( + select id from site_failure_sample where site_id = ${siteId} order by created_at desc, id desc limit ${SAMPLES_PER_SITE})`.execute( + ctx.db, + ); +} + +/** 留样回调:不等写库,失败只记日志(不影响站点请求本身的结果) */ +export function failureSampleSink(ctx: ServerContext, siteId: string): (s: AdapterSample) => void { + return (s) => { + void recordFailureSample(ctx, siteId, s).catch((err) => + ctx.logger.warn({ siteId, err: { message: (err as Error).message } }, "站点失败页面留样写入失败"), + ); + }; +} + +export async function listFailureSamples(ctx: ServerContext, siteId: string): Promise { + const rows = await ctx.db + .selectFrom("site_failure_sample") + .select([ + "id", + "site_id", + "kind", + "request_kind", + "url", + "http_status", + "headers", + "message", + "rule", + "truncated", + "created_at", + sql`octet_length(body)`.as("body_bytes"), + ]) + .where("site_id", "=", siteId) + .orderBy("created_at", "desc") + .orderBy("id", "desc") + .limit(SAMPLES_PER_SITE) + .execute(); + return rows.map((r) => toApi(r)); +} + +export async function getFailureSample( + ctx: ServerContext, + siteId: string, + sampleId: string, +): Promise { + const row = await ctx.db + .selectFrom("site_failure_sample") + .selectAll() + .where("site_id", "=", siteId) + .where("id", "=", sampleId) + .executeTakeFirst(); + return row ? toApi(row) : null; +} diff --git a/apps/server/test/sources-identity/cloudflare.test.ts b/apps/server/test/sources-identity/cloudflare.test.ts index 9939ea7f..a2efc7a8 100644 --- a/apps/server/test/sources-identity/cloudflare.test.ts +++ b/apps/server/test/sources-identity/cloudflare.test.ts @@ -21,9 +21,24 @@ import { createHarness, type Harness, pgAvailable } from "./harness.ts"; // 浏览器一侧用假端口(ChallengeBrowserPort),真实浏览器流程见 browser-cloudflare.test.ts。 const up = await pgAvailable(); + +/** 轮询到有结果(留样是异步写入的) */ +async function until(f: () => Promise, ms = 5_000): Promise { + const deadline = Date.now() + ms; + for (;;) { + const r = await f(); + if (r.length || Date.now() > deadline) return r; + await new Promise((res) => setTimeout(res, 50)); + } +} const BROWSER_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:152.0) Gecko/20100101 Firefox/152.0"; const CF_HTML = `Just a moment...
`; +/** 返回 200 的 Turnstile 门页(没有 cf-mitigated 头):搜索页单独加盾、间歇触发时见到的形态 */ +const GATE_HTML = `Audiences + +

请完成人机验证后继续

+x`; // biome-ignore lint/suspicious/noExplicitAny: 响应体断言 const json = async (r: Response): Promise => { @@ -75,6 +90,8 @@ describe.skipIf(!up)("CF-2 Cloudflare 级联", () => { pass: (h: Headers) => (h.get("cookie") ?? "").includes("cf_clearance=GOOD") && h.get("user-agent") === BROWSER_UA, challenged: 0, passed: 0, + /** 非空:挑战响应改成 200 + 这段 HTML(不带 cf-mitigated 头) */ + okPage: null as string | null, }; const seen: Headers[] = []; const solves: { url: string; accountId: string }[] = []; @@ -88,6 +105,12 @@ describe.skipIf(!up)("CF-2 Cloudflare 级联", () => { seen.push(hd); if (cf.on && !cf.pass(hd)) { cf.challenged++; + if (cf.okPage) { + return new Response(cf.okPage, { + status: 200, + headers: { "content-type": "text/html; charset=utf-8", server: "cloudflare", "cf-ray": "8c00-HKG" }, + }); + } return new Response(CF_HTML, { status: 403, headers: { "cf-mitigated": "challenge", "content-type": "text/html" }, @@ -148,6 +171,7 @@ describe.skipIf(!up)("CF-2 Cloudflare 级联", () => { cf.pass = (hd) => (hd.get("cookie") ?? "").includes("cf_clearance=GOOD") && hd.get("user-agent") === BROWSER_UA; cf.challenged = 0; cf.passed = 0; + cf.okPage = null; seen.length = 0; solves.length = 0; fetches.length = 0; @@ -305,6 +329,53 @@ describe.skipIf(!up)("CF-2 Cloudflare 级联", () => { expect(await state()).toMatchObject({ lastResult: "skipped" }); }); + it("页面请求返回 200 的 Turnstile 门页 → 进入级联:过挑战后带 cf_clearance 重试成功", async () => { + cf.okPage = GATE_HTML; + const page = await list(); + expect(page.items.length).toBeGreaterThan(0); + expect(cf.challenged).toBe(1); + expect(solves).toHaveLength(1); + expect(fetches).toEqual([]); + const st = await state(); + expect(st).toMatchObject({ cfReuse: "ok", lastResult: "reuse_ok" }); + expect(st?.lastRule).toMatch(/^gate\*=/); + }); + + it("200 的 Cloudflare 挑战页且 HTTP 过不去 → 浏览器代抓,进入浏览器代理期", async () => { + cf.okPage = CF_HTML; + cf.pass = () => false; + const page = await h.rt.siteAdapter(await site()).search("Dune"); + expect(page.items.length).toBeGreaterThan(0); + expect(cf.challenged).toBe(2); + expect(fetches).toHaveLength(1); + expect(fetches[0]).toContain("search=Dune"); + expect((await state())?.browserUntil).toBeTruthy(); + }); + + it("200 门页转人工:搜索报 challenge(文案说要人在原站视图过验证),留样标搜索、已脱敏", async () => { + cf.okPage = GATE_HTML; + solveResult = { status: "needs_human", sessionId: null, decisionId: null, reason: "选图验证码" }; + await h.db.deleteFrom("site_failure_sample").where("site_id", "=", siteId).execute(); + const err = await h.rt + .siteAdapter(await site()) + .search("Dune") + .catch((e) => e); + expect(err).toBeInstanceOf(SiteError); + expect((err as SiteError).kind).toBe("challenge"); + expect((err as SiteError).message).toContain("搜索页需要人机验证"); + expect((err as SiteError).message).toContain("原站视图"); + expect((err as SiteError).message).not.toContain("改版"); + expect((await account()).status).toBe("active"); + const rows = await until(() => + h.db.selectFrom("site_failure_sample").selectAll().where("site_id", "=", siteId).execute(), + ); + expect(rows[0]).toMatchObject({ kind: "challenge", request_kind: "search", http_status: 200 }); + expect(rows[0]?.headers).toMatchObject({ server: "cloudflare", "cf-ray": "8c00-HKG" }); + expect(rows[0]?.body).toContain("cf-turnstile"); + expect(rows[0]?.body).not.toContain("0123456789abcdef0123456789abcdef"); + expect(rows[0]?.url).toContain("search=Dune"); + }); + it("健康检测遇挑战且过不去:记 challenge(degraded),账号状态不变", async () => { solveResult = { status: "failed", reason: "浏览器服务不可用" }; fetchResult = { status: "unavailable", reason: "浏览器服务不可用" }; diff --git a/apps/server/test/sources-identity/failure-sample.test.ts b/apps/server/test/sources-identity/failure-sample.test.ts new file mode 100644 index 00000000..d0aae310 --- /dev/null +++ b/apps/server/test/sources-identity/failure-sample.test.ts @@ -0,0 +1,129 @@ +import type { SiteError } from "@umt/adapters"; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; +import { SAMPLES_PER_SITE } from "../../src/modules/sources-identity/service/failure-sample.ts"; +import { snapshotSite } from "../../src/modules/sources-identity/service/snapshot.ts"; +import { createHarness, type Harness, pgAvailable, READ_KEY } from "./harness.ts"; + +// 站点请求失败留样:列表 / 搜索报「页面结构无法识别」时存脱敏页面样本(site_failure_sample),每站只留最近 5 份; +// 站点页技术信息经 GET /sites/{id}/failure-samples(列表不带正文)与 /failure-samples/{sampleId}(带正文)取。 +// 站点前面是可控的 fetch:返回什么页面由测试决定(不访问真实站点)。 + +const up = await pgAvailable(); +const SECRET = "0123456789abcdef0123456789abcdef"; +/** 改版页:认不出种子表,还带着要脱敏的东西 */ +const BROKEN = `Audiences :: 种子 +
欢迎 alice_cn UID 29191
+

邮箱 alice@example.com

dl + +
Dune
`; + +// biome-ignore lint/suspicious/noExplicitAny: 响应体断言 +const json = async (r: Response): Promise => { + const body = await r.json(); + if (!r.ok) throw new Error(`${r.status} ${JSON.stringify(body)}`); + return body.data; +}; + +/** 轮询到条件成立(留样是异步写入的) */ +async function until(f: () => Promise, ok: (v: T) => boolean, ms = 5_000): Promise { + const deadline = Date.now() + ms; + for (;;) { + const v = await f(); + if (ok(v) || Date.now() > deadline) return v; + await new Promise((r) => setTimeout(r, 50)); + } +} + +describe.skipIf(!up)("站点失败留样", () => { + let h: Harness; + let siteId: string; + let page = BROKEN; + const fetchImpl = (async () => + new Response(page, { + status: 200, + headers: { "content-type": "text/html; charset=utf-8", server: "nginx" }, + })) as typeof fetch; + const site = () => h.db.selectFrom("site").selectAll().where("id", "=", siteId).executeTakeFirstOrThrow(); + const rows = () => + h.db.selectFrom("site_failure_sample").selectAll().where("site_id", "=", siteId).orderBy("created_at").execute(); + + beforeAll(async () => { + h = await createHarness("failsample", { fetch: fetchImpl }); + const s = await json( + await h.req("/sites", { + body: { + key: "audiences", + name: "观众", + adapter: "nexusphp", + baseUrl: "https://aud.test/", + rateLimit: { minIntervalMs: 0, concurrency: 10 }, + }, + }), + ); + siteId = s.id; + await json( + await h.req(`/sites/${siteId}/credentials`, { + method: "PUT", + body: { kind: "cookie", cookie: "c_secure_uid=1; c_secure_pass=abc" }, + }), + ); + }); + afterAll(async () => { + await h?.close(); + }); + beforeEach(async () => { + page = BROKEN; + await h.db.deleteFrom("site_failure_sample").where("site_id", "=", siteId).execute(); + }); + + it("列表快照报 parse_error → 留脱敏样本(URL、状态码、关键响应头、请求类型);接口可看", async () => { + const r = await snapshotSite(h.rt, { siteId, kind: "manual" }); + expect(r.status).toBe("failed"); + expect(r.error).toContain("parse_error"); + const got = await until(rows, (x) => x.length > 0); + expect(got).toHaveLength(1); + const s = got[0]!; + expect(s).toMatchObject({ + kind: "parse_error", + request_kind: "list", + http_status: 200, + headers: { server: "nginx" }, + }); + expect(s.message).toContain("页面结构无法识别"); + for (const leak of [SECRET, "alice@example.com", "alice_cn", "29191", "topsecretvalue"]) { + expect(s.body).not.toContain(leak); + expect(s.url).not.toContain(leak); + } + expect(s.body).toContain("new-grid"); + + const list = await json(await h.req(`/sites/${siteId}/failure-samples`, { key: READ_KEY })); + expect(list.items).toHaveLength(1); + expect(list.items[0]).toMatchObject({ id: s.id, kind: "parse_error", requestKind: "list", httpStatus: 200 }); + expect(list.items[0].body).toBeUndefined(); + expect(list.items[0].bodyBytes).toBeGreaterThan(0); + const one = await json(await h.req(`/sites/${siteId}/failure-samples/${s.id}`, { key: READ_KEY })); + expect(one.body).toBe(s.body); + const miss = await h.req(`/sites/${siteId}/failure-samples/00000000-0000-4000-8000-000000000000`); + expect(miss.status).toBe(404); + }); + + it(`每站只留最近 ${SAMPLES_PER_SITE} 份`, async () => { + const adapter = h.rt.siteAdapter(await site()); + for (let i = 1; i <= 7; i++) { + const e = await adapter.search(`k${i}`).catch((x) => x); + expect((e as SiteError).kind).toBe("parse_error"); + } + const kw = (x: { url: string }[]) => x.map((r) => /search=(k\d)/.exec(r.url)?.[1]).join(","); + const got = await until(rows, (x) => kw(x) === "k3,k4,k5,k6,k7"); + expect(kw(got)).toBe("k3,k4,k5,k6,k7"); + expect(got.every((r) => r.request_kind === "search")).toBe(true); + }); + + it("正常页面不留样", async () => { + page = `Audiences :: 种子
没有种子
`; + const adapter = h.rt.siteAdapter(await site()); + await adapter.search("x").catch(() => null); + await new Promise((r) => setTimeout(r, 200)); + expect(await rows()).toEqual([]); + }); +}); diff --git a/apps/web/app/api/sites.ts b/apps/web/app/api/sites.ts index 7a29b6f4..9001b477 100644 --- a/apps/web/app/api/sites.ts +++ b/apps/web/app/api/sites.ts @@ -9,6 +9,8 @@ import { type SiteCredentialsBody, SiteCredentialsResult, SiteDefinitionView, + SiteFailureSample, + SiteFailureSampleSummary, SiteHealthResult, SiteListItem, SiteRefreshLoginAccepted, @@ -35,6 +37,8 @@ export const siteKeys = { accounts: (siteId?: string) => ["sites", "accounts", siteId ?? "all"] as const, snapshots: (siteId: string) => ["sites", "snapshots", siteId] as const, requestStats: (siteId: string) => ["sites", "request-stats", siteId] as const, + failureSamples: (siteId: string) => ["sites", "failure-samples", siteId] as const, + failureSample: (siteId: string, sampleId: string) => ["sites", "failure-samples", siteId, sampleId] as const, definitions: () => ["sites", "definitions"] as const, userData: (siteId: string, accountId?: string, days = 30) => ["sites", "user-data", siteId, accountId ?? "first", days] as const, @@ -105,6 +109,33 @@ export function useSiteRequestStats(siteId: string | undefined) { }); } +/** 最近的失败留样(页面结构无法识别 / 人机验证;每站最近 5 份,不带正文) */ +export function useSiteFailureSamples(siteId: string | undefined) { + return useQuery({ + queryKey: siteKeys.failureSamples(siteId ?? ""), + enabled: !!siteId, + queryFn: ({ signal }) => + apiParsed(z.object({ items: z.array(SiteFailureSampleSummary) }), "/api/v1/sites/:id/failure-samples", { + pathParams: { id: siteId ?? "" }, + signal, + }), + }); +} + +/** 一份失败留样(带脱敏后的页面正文);展开时才取 */ +export function useSiteFailureSample(siteId: string, sampleId: string, enabled: boolean) { + return useQuery({ + queryKey: siteKeys.failureSample(siteId, sampleId), + enabled, + staleTime: Number.POSITIVE_INFINITY, + queryFn: ({ signal }) => + apiParsed(SiteFailureSample, "/api/v1/sites/:id/failure-samples/:sampleId", { + pathParams: { id: siteId, sampleId }, + signal, + }), + }); +} + /** 访问默认的一句话说明:"快照每 60 分钟 1 页 · 健康检测每 6 小时 · 详情缓存 24 小时" */ export function accessSummary(s: SiteRequestStats): string { const dur = (sec: number) => (sec % 3600 === 0 ? `${sec / 3600} 小时` : `${Math.round(sec / 60)} 分钟`); diff --git a/apps/web/app/components/sites/site-failure-samples.test.ts b/apps/web/app/components/sites/site-failure-samples.test.ts new file mode 100644 index 00000000..ff397efc --- /dev/null +++ b/apps/web/app/components/sites/site-failure-samples.test.ts @@ -0,0 +1,56 @@ +import type { SiteFailureSample } from "@umt/contracts"; +import { createElement } from "react"; +import { renderToStaticMarkup } from "react-dom/server"; +import { describe, expect, it } from "vitest"; +import { SiteFailureSamples } from "./site-failure-samples.tsx"; +import { failureSampleFileName, failureSampleItems, failureSampleText, failureSampleTitle } from "./site-fields.ts"; + +// 站点页技术信息里的失败样本:一行说清是搜索 / 列表、人机验证还是页面结构变化;展开看地址、响应头、页面,可下载文本。 + +const sample: SiteFailureSample = { + id: "0f1e2d3c-4b5a-4987-8a6b-5c4d3e2f1a0b", + siteId: "1a2b3c4d-5e6f-4a7b-8c9d-0e1f2a3b4c5d", + kind: "challenge", + requestKind: "search", + url: "https://audiences.me/torrents.php?search=Dune&passkey=[REDACTED]", + httpStatus: 200, + headers: { server: "cloudflare", "cf-ray": "8c00-HKG" }, + message: "搜索:搜索页需要人机验证(Cloudflare 等),需要你在原站视图里过一次验证", + rule: "gate*=cf-turnstile", + body: '
', + truncated: false, + createdAt: "2026-10-06T07:30:05.000Z", +}; +const { body: _b, ...summary } = sample; +const row = { ...summary, bodyBytes: 44 }; + +describe("站点失败样本展示", () => { + it("标题区分请求类型与失败类型", () => { + expect(failureSampleTitle(sample)).toBe("搜索 · 人机验证 · HTTP 200"); + expect(failureSampleTitle({ kind: "parse_error", requestKind: "list", httpStatus: null })).toBe( + "列表 · 页面结构无法识别", + ); + }); + + it("技术信息条目:地址、错误、规则、关键响应头、页面大小", () => { + const items = failureSampleItems(row); + expect(items.map((i) => i.label)).toEqual(["地址", "错误", "命中规则", "server", "cf-ray", "页面"]); + expect(items[0]?.value).toContain("passkey=[REDACTED]"); + }); + + it("下载文本带请求信息头;文件名带请求类型、类型与时间", () => { + const text = failureSampleText(sample); + expect(text).toContain("# 地址:https://audiences.me/torrents.php"); + expect(text).toContain("# cf-ray: 8c00-HKG"); + expect(text.endsWith(sample.body)).toBe(true); + expect(failureSampleFileName(sample)).toBe("site-sample-search-challenge-20261006-073005.html.txt"); + }); + + it("列表渲染:每份一行(收起),没有样本不渲染", () => { + const html = renderToStaticMarkup(createElement(SiteFailureSamples, { siteId: sample.siteId, items: [row] })); + expect(html).toContain("最近的失败页面"); + expect(html).toContain("搜索 · 人机验证 · HTTP 200"); + expect(html).toContain('aria-expanded="false"'); + expect(renderToStaticMarkup(createElement(SiteFailureSamples, { siteId: sample.siteId, items: [] }))).toBe(""); + }); +}); diff --git a/apps/web/app/components/sites/site-failure-samples.tsx b/apps/web/app/components/sites/site-failure-samples.tsx new file mode 100644 index 00000000..eec8592f --- /dev/null +++ b/apps/web/app/components/sites/site-failure-samples.tsx @@ -0,0 +1,81 @@ +import type { SiteFailureSample, SiteFailureSampleSummary } from "@umt/contracts"; +import { Download } from "lucide-react"; +import { useState } from "react"; +import { useSiteFailureSample } from "~/api/sites.ts"; +import { ErrorState, LoadingState } from "~/components/console/empty-state.tsx"; +import { KvList } from "~/components/console/kv-list.tsx"; +import { Button } from "~/components/ui/button.tsx"; +import { relTime } from "~/components/workbench/release-fields.ts"; +import { failureSampleFileName, failureSampleItems, failureSampleText, failureSampleTitle } from "./site-fields.ts"; + +// 站点请求失败留样(技术信息里):最近几次「页面结构无法识别」/「人机验证」时站点返回的页面(已脱敏), +// 用来分清是站点改版还是被人机验证拦了。列表不带正文,展开时再取。 + +function download(s: SiteFailureSample) { + const url = URL.createObjectURL(new Blob([failureSampleText(s)], { type: "text/plain;charset=utf-8" })); + const a = document.createElement("a"); + a.href = url; + a.download = failureSampleFileName(s); + a.click(); + URL.revokeObjectURL(url); +} + +function SampleBody({ siteId, sampleId }: { siteId: string; sampleId: string }) { + const q = useSiteFailureSample(siteId, sampleId, true); + if (q.isPending) return ; + if (q.error) return void q.refetch()} />; + return ( +
+
+ +
+
+        {q.data.body || "(页面是空的)"}
+      
+
+ ); +} + +/** 一份样本:标题一行,展开看请求信息与页面 */ +export function FailureSampleRow({ siteId, sample }: { siteId: string; sample: SiteFailureSampleSummary }) { + const [open, setOpen] = useState(false); + return ( +
  • + + {open ? ( +
    + + +
    + ) : null} +
  • + ); +} + +/** 最近的失败样本(列表由调用方用 useSiteFailureSamples 取);没有就不渲染 */ +export function SiteFailureSamples({ siteId, items }: { siteId: string; items: SiteFailureSampleSummary[] }) { + if (!items.length) return null; + return ( +
    +

    最近的失败页面(已脱敏,留最近 5 份)

    +
      + {items.map((s) => ( + + ))} +
    +
    + ); +} diff --git a/apps/web/app/components/sites/site-fields.ts b/apps/web/app/components/sites/site-fields.ts index 1c43017f..770fcae2 100644 --- a/apps/web/app/components/sites/site-fields.ts +++ b/apps/web/app/components/sites/site-fields.ts @@ -6,6 +6,8 @@ import { type SiteAuthKind, type SiteCapability, type SiteDefinitionView, + type SiteFailureSample, + type SiteFailureSampleSummary, type SiteKind, type SiteLastHealth, type SiteListItem, @@ -240,6 +242,58 @@ export function cloudflareTechItems(config: unknown): { label: string; value: st ].filter((it) => it.value); } +const SAMPLE_KIND: Record = { parse_error: "页面结构无法识别", challenge: "人机验证" }; +const SAMPLE_REQUEST: Record = { + search: "搜索", + list: "列表", + detail: "详情", + health: "健康检测", + other: "请求", +}; + +/** 失败样本一行:"搜索 · 人机验证 · HTTP 200" */ +export function failureSampleTitle(s: Pick): string { + return [ + SAMPLE_REQUEST[s.requestKind] ?? s.requestKind, + SAMPLE_KIND[s.kind] ?? s.kind, + s.httpStatus !== null ? `HTTP ${s.httpStatus}` : null, + ] + .filter(Boolean) + .join(" · "); +} + +/** 失败样本的技术信息条目(地址、响应头、规则、文案) */ +export function failureSampleItems(s: SiteFailureSampleSummary): { label: string; value: string; mono?: boolean }[] { + return [ + { label: "地址", value: s.url, mono: true }, + { label: "错误", value: s.message }, + { label: "命中规则", value: s.rule ?? "", mono: true }, + ...Object.entries(s.headers).map(([k, v]) => ({ label: k, value: v, mono: true })), + { label: "页面", value: `${formatBytes(s.bodyBytes)}${s.truncated ? "(只留了开头)" : ""}` }, + ].filter((it) => it.value); +} + +/** 下载用的文本:头部是请求信息,后面是脱敏后的页面 */ +export function failureSampleText(s: SiteFailureSample): string { + const head = [ + `# 站点失败样本(已脱敏)`, + `# 时间:${s.createdAt}`, + `# 类型:${failureSampleTitle(s)}`, + `# 地址:${s.url}`, + ...Object.entries(s.headers).map(([k, v]) => `# ${k}: ${v}`), + ...(s.rule ? [`# 规则:${s.rule}`] : []), + `# 错误:${s.message}`, + ...(s.truncated ? ["# 页面超过 64 KB,只留了开头"] : []), + ]; + return `${head.join("\n")}\n\n${s.body}`; +} + +/** 下载文件名:site-sample-<请求类型>-<类型>-<时间>.html.txt */ +export function failureSampleFileName(s: Pick): string { + const t = s.createdAt.replace(/[-:]/g, "").replace(/\..*$/, "").replace("T", "-"); + return `site-sample-${s.requestKind}-${s.kind}-${t}.html.txt`; +} + /** 限流:"20 次/分";空对象 = 默认 */ export function rateLimitText(r: SiteRateLimit | null | undefined): string { if (!r || (!r.requestsPerMinute && !r.minIntervalMs && !r.concurrency)) return "默认"; diff --git a/apps/web/app/components/sites/site-overview.tsx b/apps/web/app/components/sites/site-overview.tsx index c25aac7a..5b1df31d 100644 --- a/apps/web/app/components/sites/site-overview.tsx +++ b/apps/web/app/components/sites/site-overview.tsx @@ -1,7 +1,7 @@ import type { SiteListItem } from "@umt/contracts"; import { Link } from "react-router"; import { useFeeds } from "~/api/feeds.ts"; -import { accessSummary, useSiteRequestStats, useSiteUserData } from "~/api/sites.ts"; +import { accessSummary, useSiteFailureSamples, useSiteRequestStats, useSiteUserData } from "~/api/sites.ts"; import { DetailSection } from "~/components/console/detail-sheet.tsx"; import { ErrorState, LoadingState } from "~/components/console/empty-state.tsx"; import { KvList } from "~/components/console/kv-list.tsx"; @@ -10,6 +10,7 @@ import { StatTile } from "~/components/console/stat-tile.tsx"; import { StatusBadge } from "~/components/console/status-badge.tsx"; import { TechInfo } from "~/components/console/tech-info.tsx"; import { relTime } from "~/components/workbench/release-fields.ts"; +import { SiteFailureSamples } from "./site-failure-samples.tsx"; import { ACCOUNT_OVERALL, AUTH_KIND_LABEL, @@ -116,6 +117,7 @@ function AccountSiteOverview({ site }: { site: SiteListItem }) { const u = dAcc?.userData ?? null; const history = useSiteUserData(site.id, dAcc?.id, 30); const stats = useSiteRequestStats(site.id); + const samples = useSiteFailureSamples(site.id).data?.items ?? []; const st = ACCOUNT_OVERALL[overallStatus(site)]; const h = healthLine(site.lastHealth, site.lastHealthAt); @@ -215,7 +217,9 @@ function AccountSiteOverview({ site }: { site: SiteListItem }) { className="mt-2" columns={1} items={[...healthTechItems(site.lastHealth), ...cloudflareTechItems(site.config)]} - /> + > + {samples.length ? : null} + diff --git a/docs/runbook/sites.md b/docs/runbook/sites.md index c23f8c40..9e396c39 100644 --- a/docs/runbook/sites.md +++ b/docs/runbook/sites.md @@ -165,7 +165,8 @@ (`solveSiteChallenge` / `fetchSiteHtml`)、`modules/agent/challenge.ts`(`systemChallengeVision`)。 **识别**:站点响应带 `cf-mitigated: challenge`(任何状态码),或 403 / 503 / 429 且正文是挑战页(标题 Just a moment / Attention Required、 -`cf_chl_opt` 等,不认正常页面也会注入的 `/cdn-cgi/challenge-platform/`)→ `SiteError("challenge")`。**不是凭据错误**:快照 / 详情不调 +`cf_chl_opt` 等,不认正常页面也会注入的 `/cdn-cgi/challenge-platform/`),或页面请求返回 200 的挑战页 / 人机验证门页(见下「搜索页人机验证」) +→ `SiteError("challenge")`。**不是凭据错误**:快照 / 详情不调 `onAuthError`、不置账号失效、不排自动重登;健康检测记 `challenge`(degraded);刷新登录 / 用户数据遇挑战只写 `last_error`,账号状态不变 (验证码、二次验证仍是账号状态 `challenge`)。 @@ -202,6 +203,51 @@ **要人处理时**:站点页 Cloudflare 行显示「需要你验证」→ 原站视图接管 → 过验证 → 交还。一直过不去(站点绑定 TLS 指纹、代理 IP 被拉黑): 关掉站点代理或换出口;实在不行关闭该站点的定时快照,只保留 RSS。 +### 搜索页人机验证(200 状态的挑战页) + +现象(2026-10-06,观众):只有搜索页单独加了盾、实际搜索时间歇触发,列表快照与健康检测都正常;应用当时报 +`parse_error: 搜索:页面结构无法识别(可能改版)`,事后重搜又好了,触发那一刻的页面没留下。原因是挑战 / 人机验证页可能以 **200** 返回, +以前只看响应头和 403 / 503 / 429,这类页面落到「页面结构无法识别」,级联完全没触发。现在分两层: + +1. **网络层**(`challenge.ts detectChallenge`,经 `cloudflare.ts handle` 与 `http.ts siteFetch`):页面请求(GET + Accept `text/html`) + 的 2xx HTML 响应也看正文,只先读 256 KB(`CHALLENGE_PEEK_BYTES`,其余照常流给解析器;非页面请求不读)。判定要同时满足: + 正文已读完且不超过 10 万字符、没有站点正常内容(`id="torrenttable"`、`table.torrents`、`logout.php`、`userdetails.php?id=`、 + `usercp.php`、`#info_block` 任一出现就不算),再命中 Cloudflare 挑战标题 / 脚本,或人机验证组件(Turnstile:`cf-turnstile`、 + `challenges.cloudflare.com/turnstile`;hCaptcha;reCAPTCHA)且没有登录表单(`type=password` / `takelogin.php`,登录页带 Turnstile + 仍按登录失效处理)。命中即进入上面的 A→B 级联(过挑战 → cf_clearance 回带重试 → 浏览器代抓)。 +2. **适配器兜底**(`spider/adapter.ts`):列表 / 搜索 / 详情认不出结构、即将报「页面结构无法识别」前,用 + `challengeRuleOfUnrecognizedPage` 再看一次(不限长度、不看站点内容;也认页面文字里的「人机验证 / verify you are human」), + 像就报 `challenge`,不报改版。这类页面级联没跑过,文案提示去原站视图过一次。 + +错误文案(搜索覆盖、快照错误里能看到): +- 人机验证:`搜索:搜索页需要人机验证(Cloudflare 等),<处理结果>`。处理结果来自级联打在挑战响应上的 `x-umt-challenge` 头: + 「自动处理没能通过,需要你在原站视图里过一次验证」(needs_human)、「已自动尝试用浏览器通过,没有成功」(failed)、 + 「30 分钟内已自动尝试过,之后会再试」(throttled)、「没有可用的浏览器,无法自动通过」、「站点没有已登录的账号,不自动处理」; + 适配器兜底认出的是「这类页面没有自动处理,可以在原站视图里过一次验证」。 +- 页面结构变化:`搜索:页面结构无法识别(可能改版)(页面样本见站点页技术信息)`。 + +### 失败样本在哪看 + +站点请求报 `parse_error`(页面结构无法识别 / 响应为空)或 `challenge` 时,适配器交出一份脱敏页面样本(`SiteAdapterDeps.onFailureSample`), +`runtime.siteAdapter` 接到后异步写进表 `site_failure_sample`(迁移 0077),**每站只留最近 5 份**,写失败只记日志。覆盖所有走 +`siteAdapter` 的调用:作品页 / 搜索页的搜索、列表快照、详情刷新、健康检测(首页遇挑战)。目前只有 NexusPHP 通用 spider 适配器留样。 + +- 内容:请求类型(search / list / detail / health)、类型(parse_error / challenge)、脱敏 URL、状态码、响应头 `server` / `cf-mitigated` / + `cf-ray` / `content-type`、错误文案、命中规则、时间、页面前 64 KB(超出标「只留了开头」)。 +- 脱敏(`packages/adapters/src/sites/failure-sample.ts redactPageHtml`):整块删掉 `#info_block`(当前用户的用户名、分享率、魔力); + URL 凭据参数(passkey / authkey / rsskey / token / key …)与 uid 参数、`userdetails.php?id=` 的 id 与链接文字、「UID 12345」、 + Cookie 名值(`c_secure_*`、`cf_clearance`、`PHPSESSID` …)、表单 / 脚本里的密钥字段、邮箱、32~64 位十六进制串(passkey 形态)。 +- 看:站点页 → 概览 →「站点」一栏的「技术信息」→「最近的失败页面」,展开看请求信息与页面,可「下载文本」; + 接口 `GET /api/v1/sites/{id}/failure-samples`(不带正文)、`GET /api/v1/sites/{id}/failure-samples/{sampleId}`(带正文)。 +- 排查:类型是人机验证、页面是 Turnstile / 挑战页 → 站点加了盾,看 Cloudflare 行与级联记录;类型是页面结构无法识别、页面是正常的 + 种子页 → 站点改版,按样本改站点定义;页面是站点自己的提示页(如「搜索太频繁」)→ 看站点访问默认与限流。 + +**测试**:`packages/adapters/src/sites/challenge.test.ts`(200 挑战页 / 门页、正常搜索页 / 列表页 / 带 Turnstile 的登录页不误判、 +读取上限)、`spider/challenge-page.test.ts`(搜索页 / 列表页兜底、登录页不受影响、parse_error 留样脱敏)、`failure-sample.test.ts`、 +`apps/server/test/sources-identity/cloudflare.test.ts`(200 门页过挑战重试、200 挑战页走浏览器代抓、转人工时留样)、 +`failure-sample.test.ts`(留样、只留 5 份、接口)、`apps/web/app/components/sites/site-failure-samples.test.ts`。 +正常搜索页 fixture `packages/adapters/test/fixtures/audiences-search.html.txt`(观众真实搜索页,脱敏、删减到 4 行)。 + ## 待核验 1. takelogin 表单字段、二次验证字段名、错误文本全部只对 `@umt/testkit` mock 站点验证过,未连接真实站点。 diff --git a/packages/adapters/src/sites/challenge.test.ts b/packages/adapters/src/sites/challenge.test.ts index 537772e9..364435f3 100644 --- a/packages/adapters/src/sites/challenge.test.ts +++ b/packages/adapters/src/sites/challenge.test.ts @@ -1,5 +1,14 @@ +import { readFileSync } from "node:fs"; import { describe, expect, it } from "vitest"; -import { challengeRuleOfHtml, classifyChallengePage, detectChallenge } from "./challenge.ts"; +import { + CHALLENGE_OUTCOME_HEADER, + CHALLENGE_PEEK_BYTES, + challengeRuleOfHtml, + challengeRuleOfUnrecognizedPage, + classifyChallengePage, + detectChallenge, + isPageRequest, +} from "./challenge.ts"; import { siteFetch } from "./http.ts"; import { createSiteRateLimiter } from "./rate-limit.ts"; import { SiteError } from "./types.ts"; @@ -114,3 +123,145 @@ describe("classifyChallengePage", () => { ); }); }); + +// ── 200 状态的挑战页 / 人机验证门页(搜索页单独加盾、间歇触发)── + +const fixture = (rel: string) => readFileSync(new URL(rel, import.meta.url), "utf8"); +/** 观众真实搜索页(脱敏、删减到 4 行,``) */ +const AUD_SEARCH = fixture("../../test/fixtures/audiences-search.html.txt"); +/** 观众真实列表页(DET-2 样本,已脱敏) */ +const AUD_LIST = fixture("../../test/differential/inputs/indexer/rl-audiences-list.html.txt"); +/** Turnstile 门页:站点自己出的「请完成人机验证」短页,没有种子表、没有用户信息块 */ +const TURNSTILE_GATE = `Audiences + +

    请完成人机验证后继续

    +
    `; +const HCAPTCHA_GATE = `Security check +
    `; +/** 登录页本身带 Turnstile:交给登录判定,不算门页 */ +const LOGIN_WITH_TURNSTILE = `Audiences :: 登录 - Powered by NexusPHP + + +
    `; +/** 正常页面里注入了 challenge-platform 脚本,侧栏还有个签到 Turnstile:有种子表 / 用户信息块 → 不是门页 */ +const NORMAL_WITH_WIDGET = `Audiences :: 种子 + + +
    +
    row
    `; + +const page200 = (body: string, headers: Record = {}) => + new Response(body, { status: 200, headers: { "content-type": "text/html; charset=utf-8", ...headers } }); + +describe("detectChallenge:页面请求的 200 响应", () => { + it("200 的 Cloudflare 挑战页(标题 / 脚本)→ 挑战", async () => { + expect((await detectChallenge(page200(CF_PAGE), { page: true })).rule).toContain("Just a moment"); + const noTitle = `x`; + expect((await detectChallenge(page200(noTitle), { page: true })).rule).toContain("cf_chl_opt"); + }); + + it("200 的 Turnstile / hCaptcha 门页 → 挑战,正文原样交还", async () => { + const a = await detectChallenge(page200(TURNSTILE_GATE), { page: true }); + expect(a.rule).toMatch(/^gate\*=/); + expect(a.html).toBe(TURNSTILE_GATE); + expect(await a.res.text()).toBe(TURNSTILE_GATE); + expect((await detectChallenge(page200(HCAPTCHA_GATE), { page: true })).rule).toContain("hcaptcha"); + }); + + it("正常搜索页 / 列表页 / 带 Turnstile 的登录页 / 注入脚本的正常页 → 不误判", async () => { + for (const body of [AUD_SEARCH, AUD_LIST, LOGIN_WITH_TURNSTILE, NORMAL_WITH_WIDGET, NORMAL]) { + const r = await detectChallenge(page200(body), { page: true }); + expect(r.rule).toBeNull(); + expect(await r.res.text()).toBe(body); + } + }); + + it("不是页面请求 / 不是 HTML:不读正文(原响应对象直接返回)", async () => { + const res = page200(CF_PAGE); + const r = await detectChallenge(res); + expect(r.rule).toBeNull(); + expect(r.res).toBe(res); + expect(res.bodyUsed).toBe(false); + const json = new Response('{"title":"Just a moment"}', { + status: 200, + headers: { "content-type": "application/json" }, + }); + const j = await detectChallenge(json, { page: true }); + expect(j.res).toBe(json); + }); + + it("正文超过读取上限:只读开头不判门页,其余照常流给调用方", async () => { + const big = `${TURNSTILE_GATE}${"".repeat(Math.ceil((CHALLENGE_PEEK_BYTES * 2) / 12))}`; + const r = await detectChallenge(page200(big), { page: true }); + expect(r.rule).toBeNull(); + expect(await r.res.text()).toBe(big); + }); + + it("isPageRequest:GET + Accept text/html", () => { + expect(isPageRequest("GET", "text/html,application/xhtml+xml")).toBe(true); + expect(isPageRequest(undefined, "text/html")).toBe(true); + expect(isPageRequest("POST", "text/html")).toBe(false); + expect(isPageRequest("GET", "application/json")).toBe(false); + expect(isPageRequest("GET", null)).toBe(false); + }); +}); + +describe("siteFetch:200 门页 → challenge 并附脱敏样本", () => { + const url = "https://audiences.example/torrents.php?search=x&passkey=0123456789abcdef0123456789abcdef"; + const html = { accept: "text/html,application/xhtml+xml" }; + + it("页面请求遇 200 门页 → challenge;样本 URL 脱敏、带关键响应头;级联结果进文案", async () => { + let err: unknown; + try { + await siteFetch( + ctx( + reply(200, TURNSTILE_GATE, { + server: "cloudflare", + "cf-ray": "abc-HKG", + [CHALLENGE_OUTCOME_HEADER]: "needs_human", + }), + ), + url, + { headers: html }, + ); + } catch (e) { + err = e; + } + const se = err as SiteError; + expect(se).toBeInstanceOf(SiteError); + expect(se.kind).toBe("challenge"); + expect(se.message).toContain("原站视图"); + expect(se.detail).toMatchObject({ reason: "challenge", outcome: "needs_human", httpStatus: 200 }); + expect(se.sample).toMatchObject({ + kind: "challenge", + httpStatus: 200, + headers: { server: "cloudflare", "cf-ray": "abc-HKG" }, + }); + expect(se.sample?.url).not.toContain("0123456789abcdef"); + expect(se.sample?.body).toContain("cf-turnstile"); + }); + + it("同一页面不是页面请求(没要 HTML)→ 照常返回", async () => { + const res = await siteFetch(ctx(reply(200, TURNSTILE_GATE)), url); + expect(await res.text()).toBe(TURNSTILE_GATE); + }); +}); + +describe("challengeRuleOfUnrecognizedPage(解析失败时的兜底)", () => { + it("站点外壳里的人机验证页(有用户信息块、很长)也认;登录表单不算", () => { + const shell = `Audiences +
    ${"x".repeat(150_000)}

    请完成人机验证

    `; + expect(challengeRuleOfUnrecognizedPage(shell)).toContain("cf-turnstile"); + expect(challengeRuleOfUnrecognizedPage("

    搜索需要先完成人机验证

    ")).toBe( + "text*=人机验证", + ); + expect(challengeRuleOfUnrecognizedPage(LOGIN_WITH_TURNSTILE)).toBeNull(); + expect(challengeRuleOfUnrecognizedPage("
    改版
    ")).toBeNull(); + }); + + it("classifyChallengePage 也认门页:Turnstile → interactive,hCaptcha → captcha", () => { + expect(classifyChallengePage({ html: TURNSTILE_GATE, text: "请完成人机验证后继续" }).kind).toBe("interactive"); + expect(classifyChallengePage({ html: HCAPTCHA_GATE, text: "" }).kind).toBe("captcha"); + expect(classifyChallengePage({ html: LOGIN_WITH_TURNSTILE, text: "登录" }).kind).toBe("none"); + }); +}); diff --git a/packages/adapters/src/sites/challenge.ts b/packages/adapters/src/sites/challenge.ts index 0c521ab7..5c213986 100644 --- a/packages/adapters/src/sites/challenge.ts +++ b/packages/adapters/src/sites/challenge.ts @@ -1,5 +1,6 @@ // CF-2 Cloudflare 挑战识别(docs/design/cloudflare-handling.md)。 // - HTTP 端:响应头 `cf-mitigated: challenge`(任何状态码),或 403 / 503 / 429 且正文是挑战页 → 挑战,不是凭据错误; +// 页面类请求(GET + Accept text/html)返回 200 也可能是挑战页 / 人机验证门页(Turnstile、hCaptcha、reCAPTCHA),短页且没有站点正常内容才算; // - 浏览器端:挑战页分三类——自动(JS 检查,等几秒自己过)、可点(「我是人类」复选框 / Turnstile)、真验证码(选图 / 输码)。 // 只用正则,不解析 DOM:HTTP 端要在限流器之后快速判断,浏览器端拿到的是 outerHTML + innerText 样本。 @@ -27,21 +28,168 @@ export function challengeRuleOfHeaders(headers: Headers): string | null { return headers.get("cf-mitigated")?.trim().toLowerCase() === "challenge" ? "header cf-mitigated=challenge" : null; } +/** + * 站点请求经服务端 Cloudflare 级联后仍是挑战时,级联在返回的响应上加这个头说明处理结果 + * (needs_human / failed / throttled / no_browser / no_account),适配器据此写错误文案; + * unhandled = 网络层没认出、适配器解析时才认出的验证页(级联没跑)。 + */ +export const CHALLENGE_OUTCOME_HEADER = "x-umt-challenge"; + +/** 人机验证错误文案的后半句:自动处理到哪一步、要不要人 */ +export function challengeOutcomeText(outcome: string | null | undefined): string { + switch (outcome) { + case "needs_human": + return "自动处理没能通过,需要你在原站视图里过一次验证"; + case "failed": + return "已自动尝试用浏览器通过,没有成功"; + case "throttled": + return "30 分钟内已自动尝试过,之后会再试"; + case "no_browser": + return "没有可用的浏览器,无法自动通过"; + case "no_account": + return "站点没有已登录的账号,不自动处理"; + case "unhandled": + return "这类页面没有自动处理,可以在原站视图里过一次验证"; + default: + return "会自动尝试用浏览器通过"; + } +} + /** 这些状态码的响应才去看正文(Cloudflare 挑战页是 403 / 503,少数 429) */ export const CHALLENGE_STATUSES: readonly number[] = [403, 429, 503]; +/** 判定时最多先读这么多正文;超出部分不缓冲,照常流给调用方。挑战 / 人机验证门页都远小于这个 */ +export const CHALLENGE_PEEK_BYTES = 256 * 1024; + +/** 页面类请求(GET + Accept text/html):只有这类请求的 2xx 响应才检查正文 */ +export function isPageRequest(method: string | null | undefined, accept: string | null | undefined): boolean { + return (method ?? "GET").toUpperCase() === "GET" && /text\/html|application\/xhtml/i.test(accept ?? ""); +} + +/** 人机验证组件:Turnstile / hCaptcha / reCAPTCHA(只认脚本地址与组件 class,不认文案) */ +const GATE_WIDGET = + /challenges\.cloudflare\.com\/turnstile|class="[^"]*\bcf-turnstile\b|hcaptcha\.com\/1\/api|class="[^"]*\bh-captcha\b|google\.com\/recaptcha|recaptcha\/api\.js|class="[^"]*\bg-recaptcha\b/i; +/** 站点正常内容的痕迹:种子表、已登录的用户信息块(NexusPHP 等)。出现任何一个就不是门页 */ +const SITE_CONTENT = + /id="torrenttable"|]*class="[^"]*\btorrents\b|logout\.php|userdetails\.php\?id=|usercp\.php|id="info_block"/i; +/** 登录表单(登录页本身也可能带 Turnstile,交给登录 / 失效判定,不算门页) */ +const LOGIN_FORM = /type=["']?password|takelogin\.php/i; +/** 门页都很短;超过这个长度的页面不按门页判 */ +const GATE_MAX_CHARS = 100_000; +/** 页面文字里的人机验证提示(只在页面已经认不出来时用) */ +const HUMAN_TEXT = /人机验证|verify you are human|确认您是真人|验证您是真人|checking your browser/i; + /** - * 判定一个站点响应是不是挑战。正文只在状态码可能是挑战时读(读过的正文重新包成 Response 返回,调用方照常消费)。 - * 返回 { res, rule }:rule 为 null 表示不是挑战。 + * 人机验证门页:页面上有 Turnstile / hCaptcha / reCAPTCHA,且没有登录表单、没有站点正常内容、页面很短。 + * 命中返回规则说明,否则 null。 */ -export async function detectChallenge(res: Response): Promise<{ res: Response; rule: string | null }> { +export function challengeGateRule(html: string | null | undefined): string | null { + if (!html || html.length > GATE_MAX_CHARS) return null; + const w = GATE_WIDGET.exec(html); + if (!w || LOGIN_FORM.test(html) || SITE_CONTENT.test(html)) return null; + return `gate*=${w[0].slice(0, 40)}`; +} + +/** + * 2xx 页面是不是挑战 / 人机验证页。比非 2xx 严:正文必须完整读到(短页)、没有站点正常内容, + * 再看挑战标题 / 脚本特征或人机验证门页。complete = 正文是否已全部读到。 + */ +export function challengeRuleOfOkPage(html: string, complete = true): string | null { + if (!complete || !html || html.length > GATE_MAX_CHARS || SITE_CONTENT.test(html)) return null; + return challengeRuleOfHtml(html) ?? challengeGateRule(html); +} + +/** + * 页面已经解析不出来(列表 / 详情认不出结构)时的兜底:像挑战 / 人机验证就返回规则说明。 + * 比 challengeRuleOfOkPage 宽:不要求短页、也认页面文字里的「人机验证」提示,但带登录表单的页面不算。 + */ +export function challengeRuleOfUnrecognizedPage(html: string | null | undefined): string | null { + if (!html) return null; + const head = html.slice(0, 300_000); + const cf = challengeRuleOfHtml(head); + if (cf) return cf; + if (LOGIN_FORM.test(head)) return null; + const w = GATE_WIDGET.exec(head); + if (w) return `gate*=${w[0].slice(0, 40)}`; + const t = HUMAN_TEXT.exec(head.replace(/|/gi, " ")); + return t ? `text*=${t[0]}` : null; +} + +function concatBytes(chunks: Uint8Array[], total: number): Uint8Array { + const out = new Uint8Array(total); + let off = 0; + for (const c of chunks) { + out.set(c, off); + off += c.byteLength; + } + return out; +} + +/** 先读最多 max 字节;返回读到的部分、是否读完,以及一个内容不变的新响应(没读完的部分接着原流) */ +async function peekBody( + res: Response, + max: number, +): Promise<{ head: Uint8Array; complete: boolean; res: Response }> { + const reader = (res.body as ReadableStream).getReader(); + const chunks: Uint8Array[] = []; + let total = 0; + let complete = false; + while (total < max) { + const r = await reader.read(); + if (r.done) { + complete = true; + break; + } + chunks.push(r.value); + total += r.value.byteLength; + } + const head = concatBytes(chunks, total); + const init = { status: res.status, statusText: res.statusText, headers: res.headers }; + if (complete) return { head, complete, res: new Response(head, init) }; + const stream = new ReadableStream({ + start(c) { + c.enqueue(head); + }, + async pull(c) { + const r = await reader.read(); + if (r.done) c.close(); + else c.enqueue(r.value); + }, + cancel(reason) { + return reader.cancel(reason); + }, + }); + return { head, complete, res: new Response(stream, init) }; +} + +/** + * 判定一个站点响应是不是挑战。 + * - 响应头 `cf-mitigated: challenge` → 挑战(不读正文); + * - 403 / 429 / 503 → 读正文看挑战页特征; + * - opts.page(页面类请求)且 2xx 的 HTML → 读正文看挑战页 / 人机验证门页(challengeRuleOfOkPage); + * - 其它情况不读正文,零开销。 + * 正文最多先读 CHALLENGE_PEEK_BYTES,读过的部分重新包进返回的响应,调用方照常消费。 + * 返回 { res, rule, html }:rule 为 null 表示不是挑战;html 是读到的正文开头(没读为 null,留样用)。 + */ +export async function detectChallenge( + res: Response, + opts: { page?: boolean } = {}, +): Promise<{ res: Response; rule: string | null; html: string | null }> { const byHeader = challengeRuleOfHeaders(res.headers); - if (byHeader) return { res, rule: byHeader }; - if (!CHALLENGE_STATUSES.includes(res.status) || !res.body) return { res, rule: null }; - const bytes = new Uint8Array(await res.arrayBuffer()); - const html = new TextDecoder("utf-8").decode(bytes.subarray(0, 300_000)); - const copy = new Response(bytes, { status: res.status, statusText: res.statusText, headers: res.headers }); - return { res: copy, rule: challengeRuleOfHtml(html) }; + if (byHeader) return { res, rule: byHeader, html: null }; + const suspect = CHALLENGE_STATUSES.includes(res.status); + const okPage = + !!opts.page && + res.status >= 200 && + res.status < 300 && + res.status !== 204 && + res.status !== 205 && + /html|^$/i.test(res.headers.get("content-type")?.trim() ?? ""); + if ((!suspect && !okPage) || !res.body) return { res, rule: null, html: null }; + const peek = await peekBody(res, CHALLENGE_PEEK_BYTES); + const html = new TextDecoder("utf-8").decode(peek.head); + const rule = suspect ? challengeRuleOfHtml(html) : challengeRuleOfOkPage(html, peek.complete); + return { res: peek.res, rule, html }; } /** @@ -68,6 +216,7 @@ export function classifyChallengePage(page: { title?: string | null; html?: stri const rule = (CHALLENGE_TITLE.test(title) ? `title=${title.slice(0, 40)}` : null) ?? challengeRuleOfHtml(html) ?? + challengeGateRule(html) ?? (/verify you are human|确认您是真人|验证您是真人|checking (?:if the site connection is secure|your browser)/i.test( text, ) diff --git a/packages/adapters/src/sites/deps.ts b/packages/adapters/src/sites/deps.ts index 57bd3ed9..b9e42f08 100644 --- a/packages/adapters/src/sites/deps.ts +++ b/packages/adapters/src/sites/deps.ts @@ -1,3 +1,4 @@ +import type { SiteFailureSample } from "./failure-sample.ts"; import type { SiteHttpContext } from "./http.ts"; import { createSiteRateLimiter, type SiteRateLimiter } from "./rate-limit.ts"; import type { SecretResolver } from "./types.ts"; @@ -12,6 +13,8 @@ export interface SiteAdapterDeps { userAgent?: string; /** 覆盖站点基址(测试指向 mock) */ baseUrl?: string; + /** 请求报 parse_error / challenge 时的页面样本(已脱敏);调用方自行保存,抛错不影响请求结果 */ + onFailureSample?: (sample: SiteFailureSample) => void; } export function buildHttpContext( diff --git a/packages/adapters/src/sites/failure-sample.test.ts b/packages/adapters/src/sites/failure-sample.test.ts new file mode 100644 index 00000000..7b3dd794 --- /dev/null +++ b/packages/adapters/src/sites/failure-sample.test.ts @@ -0,0 +1,68 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; +import { buildFailureSample, FAILURE_SAMPLE_MAX_BYTES, redactPageHtml } from "./failure-sample.ts"; + +// 失败留样的脱敏:passkey / authkey / uid / 用户名 / Cookie / 邮箱 / 个人信息块都不进样本;截到 64 KB。 + +const AUD_SEARCH = readFileSync(new URL("../../test/fixtures/audiences-search.html.txt", import.meta.url), "utf8"); + +describe("redactPageHtml", () => { + it("下载链接密钥、表单 / 脚本里的密钥、Cookie、邮箱、用户链接都抹掉,普通内容保留", () => { + const key = "0123456789abcdef0123456789abcdef"; + const html = `dl +rss + +

    联系 bob@example.org

    bobby UID 4242 +Dune.Part.Two.2024`; + const out = redactPageHtml(html); + for (const leak of [ + key, + "abc123def", + "zzzzzzzz9999", + "tok_ABCDEFGH", + "hunter2xyz", + "bob@example.org", + "4242", + "bobby", + "777", + ]) { + expect(out).not.toContain(leak); + } + expect(out).toContain("download.php?id=9"); + expect(out).toContain("Dune.Part.Two.2024"); + }); + + it("真实搜索页:个人信息块整块删掉,种子表保留", () => { + const out = redactPageHtml(AUD_SEARCH); + expect(AUD_SEARCH).toContain("fixtureuser"); + expect(out).not.toContain("fixtureuser"); + expect(out).toContain(""); + expect(out).toContain('id="torrenttable"'); + }); +}); + +describe("buildFailureSample", () => { + it("截到 64 KB、URL 脱敏、只留关键响应头", () => { + const s = buildFailureSample({ + kind: "parse_error", + requestKind: "search", + url: "https://aud.test/torrents.php?search=x&passkey=secretsecret&uid=5", + httpStatus: 200, + headers: new Headers({ + server: "cloudflare", + "cf-ray": "1-HKG", + "set-cookie": "a=b", + "cf-mitigated": "challenge", + }), + message: "搜索:页面结构无法识别", + html: AUD_SEARCH, + now: new Date("2026-10-06T00:00:00Z"), + }); + expect(new TextEncoder().encode(s.body).byteLength).toBeLessThanOrEqual(FAILURE_SAMPLE_MAX_BYTES); + expect(s.truncated).toBe(true); + expect(s.url).not.toContain("secretsecret"); + expect(s.url).not.toContain("uid=5"); + expect(s.headers).toEqual({ server: "cloudflare", "cf-ray": "1-HKG", "cf-mitigated": "challenge" }); + expect(s.capturedAt).toBe("2026-10-06T00:00:00.000Z"); + }); +}); diff --git a/packages/adapters/src/sites/failure-sample.ts b/packages/adapters/src/sites/failure-sample.ts new file mode 100644 index 00000000..d971649a --- /dev/null +++ b/packages/adapters/src/sites/failure-sample.ts @@ -0,0 +1,124 @@ +import { redactString } from "@umt/contracts"; +import { redactText, redactUrl } from "../rss/redact.ts"; + +// 站点请求失败留样:请求报「页面结构无法识别」(parse_error)或「人机验证」(challenge)时,留一份脱敏后的页面开头, +// 站点页「技术信息」里可看 / 下载,用来区分是站点改版还是被人机验证拦了(间歇触发的那一刻页面不会再出现)。 + +/** 页面样本最多保留的字节数 */ +export const FAILURE_SAMPLE_MAX_BYTES = 64 * 1024; + +/** 请求类型:搜索 / 列表 / 详情 / 健康检测 / 其它 */ +export type SiteRequestKind = "search" | "list" | "detail" | "health" | "other"; + +/** 留样只记这几个响应头 */ +const KEPT_HEADERS = ["server", "cf-mitigated", "cf-ray", "content-type"] as const; + +export interface SiteFailureSample { + kind: "parse_error" | "challenge"; + requestKind: SiteRequestKind; + /** 已脱敏 */ + url: string; + httpStatus: number | null; + headers: Record; + capturedAt: string; + /** 报出去的错误文案 */ + message: string; + /** 命中的挑战 / 门页规则 */ + rule: string | null; + /** 脱敏后的页面开头(≤ FAILURE_SAMPLE_MAX_BYTES 字节) */ + body: string; + /** 页面是否被截断 */ + truncated: boolean; +} + +/** URL 里除了通用凭据参数,还要抹掉的用户标识参数 */ +const EXTRA_KEYS = ["uid", "userid", "user_id"]; +const EMAIL_RE = /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g; +/** NexusPHP passkey / authkey 等:32 位以上十六进制串(info hash 是 40 位,也一并抹掉,样本不需要) */ +const HEX_KEY_RE = /\b[a-f0-9]{32,64}\b/gi; +/** Cookie 名=值(页面脚本里偶尔会出现) */ +const COOKIE_PAIR_RE = + /\b(c_secure_[a-z_]+|cf_clearance|__cf_bm|phpsessid|PHPSESSID|session(?:id)?|remember_[a-z0-9_]+)=([^;\s"'<>&]+)/gi; +/** 表单 / 脚本里的密钥字段:name="passkey" value="…"、passkey: "…" */ +const SECRET_FIELD_RE = + /\b(pass[_-]?key|auth[_-]?key|torrent[_-]?pass|rss[_-]?key|api[_-]?key|token|csrf[_-]?token|secret)(["']?\s*(?:[:=]|value=)\s*["']?)([^"'\s<>&;,]{6,})/gi; +/** 用户页链接里的用户 id 与链接文字(用户名) */ +const USER_LINK_RE = /(userdetails\.php\?id=)\d+/gi; +const USER_NAME_RE = /(]*userdetails\.php[^>]*>)((?:\s*<[^>]+>)*)([^<]+)/gi; + +/** 页面里的「UID 12345」 */ +const UID_TEXT_RE = /\bUID\s*[::]?\s*\d+/gi; + +/** + * 删掉 id 为 id 的整个元素(按同名标签计数找配对的结束标签);NexusPHP 的 `#info_block` 是当前用户的个人信息块 + * (用户名、分享率、魔力…),样本里不需要。找不到原样返回。 + */ +function dropElementById(html: string, id: string): string { + const open = new RegExp(`<([a-z][a-z0-9]*)\\b[^>]*\\bid=["']?${id}\\b[^>]*>`, "i").exec(html); + if (!open) return html; + const tag = (open[1] as string).toLowerCase(); + const re = new RegExp(`<(/?)${tag}\\b[^>]*>`, "gi"); + re.lastIndex = open.index + open[0].length; + let depth = 1; + for (let m = re.exec(html); m; m = re.exec(html)) { + depth += m[1] ? -1 : 1; + if (depth === 0) return `${html.slice(0, open.index)}${html.slice(m.index + m[0].length)}`; + } + return `${html.slice(0, open.index)}`; +} + +/** 页面文本脱敏:个人信息块、凭据参数、用户 id / 用户名、Cookie、邮箱、密钥样式的十六进制串 */ +export function redactPageHtml(html: string): string { + return redactString( + redactText(dropElementById(html, "info_block"), EXTRA_KEYS) + .replace(UID_TEXT_RE, "UID [REDACTED]") + .replace(USER_NAME_RE, "$1$2[USER]") + .replace(USER_LINK_RE, "$1[REDACTED]") + .replace(COOKIE_PAIR_RE, "$1=[REDACTED]") + .replace(SECRET_FIELD_RE, "$1$2[REDACTED]") + .replace(EMAIL_RE, "[EMAIL]") + .replace(HEX_KEY_RE, "[HEX]"), + ); +} + +/** 先脱敏、再按字节截到上限(截断处的半个字符丢掉) */ +function clip(text: string, maxBytes: number): { body: string; truncated: boolean } { + const bytes = new TextEncoder().encode(text); + if (bytes.byteLength <= maxBytes) return { body: text, truncated: false }; + return { body: new TextDecoder().decode(bytes.subarray(0, maxBytes)).replace(/�$/, ""), truncated: true }; +} + +export function buildFailureSample(input: { + kind: SiteFailureSample["kind"]; + requestKind: SiteRequestKind; + url: string; + httpStatus?: number | null; + headers?: Headers | null; + message: string; + rule?: string | null; + html: string | null | undefined; + now: Date; +}): SiteFailureSample { + const headers: Record = {}; + for (const k of KEPT_HEADERS) { + const v = input.headers?.get(k); + if (v) headers[k] = v.slice(0, 200); + } + // 脱敏只看前 4 倍上限(截断前先脱敏,避免截断把密钥切成两半漏掉) + const { body, truncated } = clip( + redactPageHtml((input.html ?? "").slice(0, FAILURE_SAMPLE_MAX_BYTES * 4)), + FAILURE_SAMPLE_MAX_BYTES, + ); + return { + kind: input.kind, + requestKind: input.requestKind, + url: redactUrl(input.url, EXTRA_KEYS), + httpStatus: input.httpStatus ?? null, + headers, + capturedAt: input.now.toISOString(), + message: input.message, + rule: input.rule ?? null, + body, + truncated: truncated || (input.html?.length ?? 0) > FAILURE_SAMPLE_MAX_BYTES * 4, + }; +} diff --git a/packages/adapters/src/sites/http.ts b/packages/adapters/src/sites/http.ts index 43e8d33f..bd5b031e 100644 --- a/packages/adapters/src/sites/http.ts +++ b/packages/adapters/src/sites/http.ts @@ -1,5 +1,6 @@ import { redactText } from "../rss/redact.ts"; -import { detectChallenge } from "./challenge.ts"; +import { CHALLENGE_OUTCOME_HEADER, challengeOutcomeText, detectChallenge, isPageRequest } from "./challenge.ts"; +import { buildFailureSample } from "./failure-sample.ts"; import type { SiteRateLimiter } from "./rate-limit.ts"; import { SiteError } from "./types.ts"; @@ -21,7 +22,8 @@ function retryAfterMs(res: Response, now: Date): number | undefined { } /** - * 站点请求:先取站点令牌,再发请求;Cloudflare 挑战(`cf-mitigated: challenge`,或 403 / 503 / 429 的挑战页)→ `challenge` + * 站点请求:先取站点令牌,再发请求;Cloudflare 挑战(`cf-mitigated: challenge`,或 403 / 503 / 429 的挑战页, + * 或页面请求返回 200 的挑战页 / 人机验证门页)→ `challenge`(附脱敏页面样本) * (不是凭据错误,CF-2);其余 429/503 统一映射并顺延限流器。 * 返回的 Response 由调用方消费。错误消息只含脱敏后的 URL。 */ @@ -44,13 +46,27 @@ export async function siteFetch( const reason = timeout.aborted ? "超时" : (e as Error).message; throw new SiteError(ctx.siteId, "network", redactText(`请求 ${safeUrl} 失败:${reason}`)); } - const ch = await detectChallenge(res); + // 页面类请求的 200 响应也看正文:返回 200 的挑战页 / 人机验证门页(只读开头,有上限) + const ch = await detectChallenge(res, { page: isPageRequest(init.method, headers.get("accept")) }); res = ch.res; if (ch.rule) { await res.body?.cancel().catch(() => {}); - throw new SiteError(ctx.siteId, "challenge", `请求 ${safeUrl}:站点要求人机验证(Cloudflare)`, { + const outcome = res.headers.get(CHALLENGE_OUTCOME_HEADER); + const message = `请求 ${safeUrl}:站点要求人机验证(Cloudflare 等),${challengeOutcomeText(outcome)}`; + throw new SiteError(ctx.siteId, "challenge", message, { httpStatus: res.status, - detail: { reason: "challenge", rule: ch.rule, httpStatus: res.status }, + detail: { reason: "challenge", rule: ch.rule, httpStatus: res.status, ...(outcome ? { outcome } : {}) }, + sample: buildFailureSample({ + kind: "challenge", + requestKind: "other", + url, + httpStatus: res.status, + headers: res.headers, + message, + rule: ch.rule, + html: ch.html, + now: ctx.now(), + }), }); } if (res.status === 429 || res.status === 503) { diff --git a/packages/adapters/src/sites/index.ts b/packages/adapters/src/sites/index.ts index 0f19c314..e0a9ac79 100644 --- a/packages/adapters/src/sites/index.ts +++ b/packages/adapters/src/sites/index.ts @@ -1,10 +1,17 @@ export { + CHALLENGE_OUTCOME_HEADER, + CHALLENGE_PEEK_BYTES, CHALLENGE_STATUSES, type ChallengePageKind, + challengeGateRule, + challengeOutcomeText, challengeRuleOfHeaders, challengeRuleOfHtml, + challengeRuleOfOkPage, + challengeRuleOfUnrecognizedPage, classifyChallengePage, detectChallenge, + isPageRequest, } from "./challenge.ts"; export { audiencesDefinition, @@ -34,6 +41,13 @@ export { structureDescription, } from "./detail-description.ts"; export { type FormattedDetailPage, formatDetailPage } from "./detail-format.ts"; +export { + buildFailureSample, + FAILURE_SAMPLE_MAX_BYTES, + redactPageHtml, + type SiteFailureSample, + type SiteRequestKind, +} from "./failure-sample.ts"; export { type FeedMediaType, normalizeSiteFeedItem, diff --git a/packages/adapters/src/sites/spider/adapter.ts b/packages/adapters/src/sites/spider/adapter.ts index 33af4865..d2df99ac 100644 --- a/packages/adapters/src/sites/spider/adapter.ts +++ b/packages/adapters/src/sites/spider/adapter.ts @@ -14,7 +14,9 @@ import { inspectTorrentIsolated } from "../../isolation.ts"; import { redactText } from "../../rss/redact.ts"; +import { challengeOutcomeText, challengeRuleOfUnrecognizedPage } from "../challenge.ts"; import { buildHttpContext, type SiteAdapterDeps } from "../deps.ts"; +import { buildFailureSample, type SiteRequestKind } from "../failure-sample.ts"; import { readBody, readText, siteFetch } from "../http.ts"; import { type DownloadTarget, @@ -107,6 +109,23 @@ export function judgeLoginPage( }; } +/** 抓到的页面:正文与留样用的响应信息 */ +interface FetchedPage { + html: string; + url: string; + status: number; + headers: Headers; +} + +/** 错误文案里的页面名 */ +const PAGE_LABEL: Record = { + search: "搜索页", + list: "列表页", + detail: "详情页", + health: "首页", + other: "页面", +}; + export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdapterDeps): SiteAdapter { const def: IndexerDefinition = deps.baseUrl ? { ...input, domain: deps.baseUrl } : input; const ext = def.umt; @@ -119,13 +138,73 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap return c; } - async function fetchPage(url: string, what: string, opts: SiteCallOptions): Promise { - const res = await siteFetch(http, url, { - method: "GET", - headers: { cookie: await cookie(), accept: "text/html,application/xhtml+xml", referer: base }, - redirect: "manual", - ...(opts.signal ? { signal: opts.signal } : {}), + /** 留样:交给调用方保存;回调出错不影响请求结果 */ + function report(e: SiteError): SiteError { + if (e.sample && deps.onFailureSample) { + try { + deps.onFailureSample(e.sample); + } catch { + // 留样失败不影响请求 + } + } + return e; + } + + /** 页面认不出来时的错误:像人机验证就报 challenge,否则 parse_error;都附页面样本 */ + function unrecognized(page: FetchedPage, what: string, kind: SiteRequestKind, parseMessage: string): SiteError { + const rule = challengeRuleOfUnrecognizedPage(page.html); + const message = rule + ? `${what}:${PAGE_LABEL[kind]}需要人机验证(Cloudflare 等),${challengeOutcomeText("unhandled")}` + : `${parseMessage}(页面样本见站点页技术信息)`; + const sample = buildFailureSample({ + kind: rule ? "challenge" : "parse_error", + requestKind: kind, + url: page.url, + httpStatus: page.status, + headers: page.headers, + message, + rule, + html: page.html, + now: http.now(), }); + return report( + new SiteError(def.id, rule ? "challenge" : "parse_error", message, { + httpStatus: page.status, + ...(rule ? { detail: { reason: "challenge", rule, httpStatus: page.status } } : {}), + sample, + }), + ); + } + + async function fetchPage( + url: string, + what: string, + opts: SiteCallOptions, + kind: SiteRequestKind, + ): Promise { + let res: Response; + try { + res = await siteFetch(http, url, { + method: "GET", + headers: { cookie: await cookie(), accept: "text/html,application/xhtml+xml", referer: base }, + redirect: "manual", + ...(opts.signal ? { signal: opts.signal } : {}), + }); + } catch (e) { + if (e instanceof SiteError && e.kind === "challenge") { + // 网络层判出的人机验证:换成带请求类型的文案,样本补上请求类型 + const outcome = typeof e.detail?.outcome === "string" ? e.detail.outcome : null; + const message = `${what}:${PAGE_LABEL[kind]}需要人机验证(Cloudflare 等),${challengeOutcomeText(outcome)}`; + throw report( + new SiteError(def.id, "challenge", message, { + ...(e.httpStatus !== undefined ? { httpStatus: e.httpStatus } : {}), + ...(e.detail ? { detail: e.detail } : {}), + ...(e.sample ? { sample: { ...e.sample, requestKind: kind, message } } : {}), + }), + ); + } + throw e; + } if (res.status >= 300 && res.status < 400) { const loc = res.headers.get("location") ?? ""; await res.body?.cancel().catch(() => {}); @@ -155,7 +234,7 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap httpStatus: res.status, }); } - return readText(http, res); + return { html: await readText(http, res), url, status: res.status, headers: res.headers }; } async function run( @@ -172,17 +251,19 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap page: Math.max(1, page) - 1, // 对外 1 起,MP 内部 0 起 }); if (!spider.canSearch()) throw new SiteError(def.id, "unsupported", `${what}:站点定义不支持该媒体类型`); - const html = await fetchPage(spider.getSearchUrl(), what, opts); - const parsed = spider.parse(html); + const kind: SiteRequestKind = keyword ? "search" : "list"; + const page$ = await fetchPage(spider.getSearchUrl(), what, opts, kind); + const parsed = spider.parse(page$.html); if (parsed.loginPage) throw new SiteError(def.id, "auth_expired", `${what}:返回登录或权限提示页`); - if (parsed.isError || !parsed.$) throw new SiteError(def.id, "parse_error", `${what}:响应为空`); + if (parsed.isError || !parsed.$) throw unrecognized(page$, what, kind, `${what}:响应为空`); const $ = parsed.$; if (parsed.torrents.length === 0) { const text = $("body").text(); const hasList = $(normalizeSelector(spider.def.torrents.list.selector)).length > 0 || $("table.torrents").length > 0; const empty = (ext.emptyMarkers ?? []).some((m) => text.includes(m)); - if (!hasList && !empty) throw new SiteError(def.id, "parse_error", `${what}:页面结构无法识别(可能改版)`); + // 认不出列表:先看是不是人机验证页(搜索页单独加盾、间歇触发),不是才报改版 + if (!hasList && !empty) throw unrecognized(page$, what, kind, `${what}:页面结构无法识别(可能改版)`); if (ext.loginSuccess && $(ext.loginSuccess).length === 0 && !empty) { throw new SiteError(def.id, "auth_expired", `${what}:未找到登录标志`); } @@ -237,18 +318,20 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap if (!/^\d+$/.test(siteReleaseId)) throw new SiteError(def.id, "not_found", "资源 ID 非法"); if (!ext.detail) throw new SiteError(def.id, "unsupported", "站点定义未配置详情页解析"); const url = new URL(`${ext.detail.path}?id=${siteReleaseId}&hit=1`, base).toString(); - const html = await fetchPage(url, `详情 ${siteReleaseId}`, opts); - const r = parseDetailPage(html, def, siteReleaseId, http.now().toISOString()); + const detailPage = await fetchPage(url, `详情 ${siteReleaseId}`, opts, "detail"); + const r = parseDetailPage(detailPage.html, def, siteReleaseId, http.now().toISOString()); if (r.kind === "login") throw new SiteError(def.id, "auth_expired", `详情 ${siteReleaseId}:返回登录页`); if (r.kind === "not_found") throw new SiteError(def.id, "not_found", `详情 ${siteReleaseId}:资源不存在`); - if (!r.detail) throw new SiteError(def.id, "parse_error", `详情 ${siteReleaseId}:页面结构无法识别`); + if (!r.detail) { + throw unrecognized(detailPage, `详情 ${siteReleaseId}`, "detail", `详情 ${siteReleaseId}:页面结构无法识别`); + } // 文件清单不在页面上(NexusPHP 点「查看列表」才 AJAX 取)→ 按需 1 次;失败只是没有文件清单 const flPath = ext.detail.page?.fileListPath; if (flPath && r.detail.page && !r.detail.page.files.length) { try { const flUrl = new URL(flPath.replace("{id}", siteReleaseId), base).toString(); - const frag = await fetchPage(flUrl, `文件清单 ${siteReleaseId}`, opts); - r.detail.page.files = filesFromFileListHtml(loadHtml(frag)); + const frag = await fetchPage(flUrl, `文件清单 ${siteReleaseId}`, opts, "detail"); + r.detail.page.files = filesFromFileListHtml(loadHtml(frag.html)); } catch (e) { if (e instanceof SiteError && e.isAuthError) throw e; } @@ -317,8 +400,8 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap const started = Date.now(); const checkedAt = http.now().toISOString(); try { - const html = await fetchPage(new URL("index.php", base).toString(), "健康检查", opts); - const v = judgeLoginPage(html, ext.loginSuccess); + const home = await fetchPage(new URL("index.php", base).toString(), "健康检查", opts, "health"); + const v = judgeLoginPage(home.html, ext.loginSuccess); const latencyMs = Date.now() - started; if (v.ok) return { siteId: def.id, status: "ok", ok: true, checkedAt, latencyMs, detail: v.detail }; return { diff --git a/packages/adapters/src/sites/spider/challenge-page.test.ts b/packages/adapters/src/sites/spider/challenge-page.test.ts new file mode 100644 index 00000000..039fc771 --- /dev/null +++ b/packages/adapters/src/sites/spider/challenge-page.test.ts @@ -0,0 +1,123 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; +import { + audiencesDefinition, + createSiteRateLimiter, + createSpiderSiteAdapter, + SiteError, + type SiteFailureSample, + staticSecretResolver, +} from "../../index.ts"; + +// 搜索页单独加盾(间歇触发):返回 200 的挑战 / 人机验证页不再报「页面结构无法识别」,报 challenge 并留样; +// 真正认不出的页面才报 parse_error(也留样)。 + +const AUD_SEARCH = readFileSync(new URL("../../../test/fixtures/audiences-search.html.txt", import.meta.url), "utf8"); +const CF_200 = `Just a moment... +
    `; +/** 站点外壳里的人机验证页:有用户信息块(网络层的严格判定不认),适配器解析失败时兜底认出 */ +const SHELL_GATE = `Audiences :: 搜索 +
    someone 退出
    +

    搜索前请完成人机验证

    +
    `; +const LOGIN_WITH_TURNSTILE = `Audiences :: 登录 - Powered by NexusPHP +
    +
    `; +/** 改版页:带上各种要脱敏的东西 */ +const SECRET = "0123456789abcdef0123456789abcdef"; +const BROKEN = `Audiences :: 种子 +
    欢迎 alice_cn 邮箱 alice@example.com
    +dl + + +
    ${"
    Dune
    ".repeat(5000)}
    `; + +function adapter(body: string, status = 200, headers: Record = {}) { + const samples: SiteFailureSample[] = []; + const fetchImpl = (async () => + new Response(body, { + status, + headers: { "content-type": "text/html; charset=utf-8", ...headers }, + })) as typeof fetch; + const a = createSpiderSiteAdapter(audiencesDefinition, { + baseUrl: "https://aud.test/", + secrets: staticSecretResolver({ [audiencesDefinition.id]: { cookie: "c_secure_uid=1; c_secure_pass=abc" } }), + limiter: createSiteRateLimiter({ minIntervalMs: 0, burst: 10 }), + fetch: fetchImpl, + onFailureSample: (s) => samples.push(s), + }); + return { a, samples }; +} + +async function err(p: Promise): Promise { + try { + await p; + } catch (e) { + expect(e).toBeInstanceOf(SiteError); + return e as SiteError; + } + throw new Error("expected SiteError"); +} + +describe("spider 搜索页人机验证", () => { + it("正常搜索页(真实样本)照常解析,不留样", async () => { + const { a, samples } = adapter(AUD_SEARCH); + const page = await a.search("药屋少女的呢喃"); + expect(page.items.length).toBeGreaterThan(0); + expect(samples).toEqual([]); + }); + + it("200 的 Cloudflare 挑战页 → challenge(网络层认出),文案说搜索页需要人机验证,留样标搜索", async () => { + const { a, samples } = adapter(CF_200, 200, { server: "cloudflare", "cf-ray": "8c-HKG" }); + const e = await err(a.search("Dune")); + expect(e.kind).toBe("challenge"); + expect(e.isAuthError).toBe(false); + expect(e.message).toContain("搜索页需要人机验证"); + expect(e.message).not.toContain("改版"); + expect(samples).toHaveLength(1); + expect(samples[0]).toMatchObject({ kind: "challenge", requestKind: "search", httpStatus: 200 }); + expect(samples[0]?.headers["cf-ray"]).toBe("8c-HKG"); + expect(samples[0]?.url).toContain("torrents.php"); + }); + + it("站点外壳里的人机验证页:网络层不认,解析失败前兜底 → challenge(不报改版)", async () => { + const { a, samples } = adapter(SHELL_GATE); + const e = await err(a.search("Dune")); + expect(e.kind).toBe("challenge"); + expect(e.message).toContain("搜索页需要人机验证"); + expect(e.detail?.rule).toContain("turnstile"); + expect(samples[0]).toMatchObject({ kind: "challenge", requestKind: "search" }); + expect(samples[0]?.body).not.toContain("29191"); + expect(samples[0]?.body).not.toContain("someone"); + }); + + it("列表页同样处理,文案按列表", async () => { + const { a, samples } = adapter(SHELL_GATE); + const e = await err(a.list(1)); + expect(e.kind).toBe("challenge"); + expect(e.message).toContain("列表页需要人机验证"); + expect(samples[0]?.requestKind).toBe("list"); + }); + + it("带 Turnstile 的登录页仍按登录失效处理,不留样", async () => { + const { a, samples } = adapter(LOGIN_WITH_TURNSTILE); + const e = await err(a.search("Dune")); + expect(e.kind).toBe("auth_expired"); + expect(samples).toEqual([]); + }); + + it("真正认不出的页面 → parse_error,留脱敏样本(≤ 64 KB)", async () => { + const { a, samples } = adapter(BROKEN, 200, { server: "nginx" }); + const e = await err(a.search("Dune")); + expect(e.kind).toBe("parse_error"); + expect(e.message).toContain("页面结构无法识别"); + expect(samples).toHaveLength(1); + const s = samples[0] as SiteFailureSample; + expect(s).toMatchObject({ kind: "parse_error", requestKind: "search", httpStatus: 200, truncated: true }); + expect(new TextEncoder().encode(s.body).byteLength).toBeLessThanOrEqual(64 * 1024); + for (const leak of [SECRET, "alice@example.com", "alice_cn", "29191", "topsecretvalue"]) { + expect(s.body).not.toContain(leak); + } + expect(s.body).toContain("download.php?id=1"); + }); +}); diff --git a/packages/adapters/src/sites/types.ts b/packages/adapters/src/sites/types.ts index 9cbe5fe7..383de115 100644 --- a/packages/adapters/src/sites/types.ts +++ b/packages/adapters/src/sites/types.ts @@ -1,5 +1,7 @@ /** 站点适配器自有类型(不依赖 @umt/contracts;W2 负责映射到 source_release / site_snapshot)。 */ +import type { SiteFailureSample } from "./failure-sample.ts"; + /** api = 专用 API spider(spiders/*,Haidan/HDDolby/RousiPro/SunnyPT/TNode/TorrentLeech/Yema) */ export type SiteKind = "nexusphp" | "mteam" | "api"; @@ -205,11 +207,13 @@ export class SiteError extends Error { readonly retryAfterMs?: number; /** 原始判定(健康检查的技术信息用) */ readonly detail?: SiteHealthDetail; + /** 失败页面样本(已脱敏;parse_error / challenge 时由适配器附上,交给 deps.onFailureSample 留样) */ + readonly sample?: SiteFailureSample; constructor( siteId: string, kind: SiteErrorKind, message: string, - opts: { httpStatus?: number; retryAfterMs?: number; detail?: SiteHealthDetail } = {}, + opts: { httpStatus?: number; retryAfterMs?: number; detail?: SiteHealthDetail; sample?: SiteFailureSample } = {}, ) { super(message); this.name = "SiteError"; @@ -218,6 +222,7 @@ export class SiteError extends Error { if (opts.httpStatus !== undefined) this.httpStatus = opts.httpStatus; if (opts.retryAfterMs !== undefined) this.retryAfterMs = opts.retryAfterMs; if (opts.detail !== undefined) this.detail = opts.detail; + if (opts.sample !== undefined) this.sample = opts.sample; } /** 凭据类错误:应立即置账号验证失败,不再重试 */ get isAuthError(): boolean { diff --git a/packages/adapters/test/fixtures/audiences-search.html.txt b/packages/adapters/test/fixtures/audiences-search.html.txt new file mode 100644 index 00000000..2b7eddd9 --- /dev/null +++ b/packages/adapters/test/fixtures/audiences-search.html.txt @@ -0,0 +1,1308 @@ + + + + + + Audiences :: 搜索结果 - 药屋少女的呢喃 - Powered by NexusPHP + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    +
    +
    +
    + + +
    + +
    + +
    +
    + + + + + + +
    +
    全局优惠生效:官组FREE
    优惠开始:2026-10-04 00:00:00 (GMT+08:00)
    优惠截止:2026-10-08 23:59:59 (GMT+08:00)
    +
    +
    + +
    +
    + + +
    +
    + + + + + + + + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    类型 + + + + 标题 + + + + + + + + + +
    音乐Blu-ray DIY
    +
    Hyakka Ryoran 2025 1080p BluRay AVC LPCM 2.0-DIY@Audies Free
    官方中字DIY动画禁转百花繚乱 · 幾田りら/ 《药屋少女的呢喃》第二季OP1 【原盘DIY / 歌词字幕】*奇怪的DIY增加了
    11年
    6月
    476.16
    MB
    70163-
    匿名
    剧集Blu-ray DIY
    +
    The Apothecary Diaries S01 2023 1080p BluRay AVC LPCM 2.0-DIY@Audies Free
    官方中字官字组DIY动画完结药屋少女的呢喃 / 薬屋のひとりごと / The Apothecary Diaries 【DIY / 双语简繁字幕】【正片全24集+特典全24集】
    02年
    2月
    176.73
    GB
    120257-
    匿名
    +
    +
    \ No newline at end of file diff --git a/packages/contracts/src/api/sources.ts b/packages/contracts/src/api/sources.ts index ca5e4d0c..87e6c92b 100644 --- a/packages/contracts/src/api/sources.ts +++ b/packages/contracts/src/api/sources.ts @@ -86,6 +86,22 @@ export const sourcesApi = { params: IdParam, response: sources.SiteRequestStats, }), + failureSamples: ep({ + method: "get", + path: "/sites/{id}/failure-samples", + scope: "read", + summary: "站点请求失败留样:最近几次页面结构无法识别 / 人机验证时的脱敏页面(列表不带正文)", + params: IdParam, + response: z.object({ items: z.array(sources.SiteFailureSampleSummary) }), + }), + failureSample: ep({ + method: "get", + path: "/sites/{id}/failure-samples/{sampleId}", + scope: "read", + summary: "一份失败留样(含脱敏后的页面正文)", + params: z.object({ id: Id, sampleId: Id }), + response: sources.SiteFailureSample, + }), credentials: ep({ method: "put", path: "/sites/{id}/credentials", diff --git a/packages/contracts/src/sources.ts b/packages/contracts/src/sources.ts index 105670f3..2ace97d8 100644 --- a/packages/contracts/src/sources.ts +++ b/packages/contracts/src/sources.ts @@ -701,6 +701,35 @@ export const SiteSnapshot = EntityBase.extend({ }); export type SiteSnapshot = z.infer; +/** 失败留样的类型:页面结构无法识别 / 人机验证 */ +export const SiteFailureSampleKind = z.enum(["parse_error", "challenge"]); +export type SiteFailureSampleKind = z.infer; + +/** + * 站点请求失败留样(列表 / 搜索 / 详情报 parse_error / challenge 时存的脱敏页面开头,每站最近 5 份)。 + * requestKind:search / list / detail / health / other;headers 只有 server、cf-mitigated、cf-ray、content-type。 + */ +export const SiteFailureSample = z.object({ + id: Id, + siteId: Id, + kind: SiteFailureSampleKind, + requestKind: z.string(), + /** 已脱敏 */ + url: z.string(), + httpStatus: z.number().int().nullable(), + headers: z.record(z.string(), z.string()), + message: z.string(), + rule: z.string().nullable(), + /** 脱敏后的页面开头(≤ 64 KB) */ + body: z.string(), + truncated: z.boolean(), + createdAt: Timestamp, +}); +export type SiteFailureSample = z.infer; +/** 列表里不带页面正文,只给字节数 */ +export const SiteFailureSampleSummary = SiteFailureSample.omit({ body: true }).extend({ bodyBytes: NonNegInt }); +export type SiteFailureSampleSummary = z.infer; + /** 站点访问默认(站点定义 umt.defaults 与 settings.sites 合并后,取更保守的一方) */ export const SiteAccessDefaults = z.object({ snapshotIntervalSec: z.number().int(), diff --git a/packages/contracts/test/db-sync.test.ts b/packages/contracts/test/db-sync.test.ts index 261bb4ad..026c736f 100644 --- a/packages/contracts/test/db-sync.test.ts +++ b/packages/contracts/test/db-sync.test.ts @@ -105,6 +105,7 @@ const ROWS: Record MetadataEnrichment: { row: C.MetadataEnrichment }, ResolutionAttempt: { row: C.ResolutionAttempt }, SiteSnapshot: { row: C.SiteSnapshot }, + SiteFailureSample: { row: C.SiteFailureSample }, // W4-C 浏览器实体(0023) // BR-4:identity 是浏览器固定身份的缓存(站点请求 UA 用),不进 API BrowserProvider: { row: C.BrowserProviderRow, dbOnly: ["identity"] }, @@ -252,6 +253,7 @@ const ENUMS: Record = { ScheduledJobLastStatus: C.ScheduleRunStatus, ScheduledJobRunTrigger: C.ScheduleTrigger, ScheduledJobRunStatus: C.ScheduleRunStatus, + SiteFailureSampleKind: C.SiteFailureSampleKind, ScrapeRecordStatus: C.ScrapeStatus, ScrapeRecordTrigger: C.ScrapeTrigger, SubtitleWriteSource: C.SubtitleSource, diff --git a/packages/db/migrations/0077_site_failure_sample.ts b/packages/db/migrations/0077_site_failure_sample.ts new file mode 100644 index 00000000..ccf76b5c --- /dev/null +++ b/packages/db/migrations/0077_site_failure_sample.ts @@ -0,0 +1,35 @@ +import type { Kysely } from "kysely"; +import { dropTables, execDdl } from "../src/migration-helpers.ts"; + +// 站点请求失败留样:列表 / 搜索 / 详情报「页面结构无法识别」或「人机验证」时,存一份脱敏后的页面开头(≤ 64 KB) +// 与 URL(脱敏)、状态码、关键响应头、请求类型。每站只留最近 5 份(写入时顺手删旧的);站点页「技术信息」里看。 +// 单独一张表:样本有几十 KB,不放进每条站点请求都要读的 site.config。 + +// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型 +export async function up(db: Kysely): Promise { + await execDdl( + db, + ` +create table site_failure_sample ( + id uuid primary key default gen_random_uuid(), + site_id uuid not null references site(id) on delete cascade, + kind text not null check (kind in ('parse_error', 'challenge')), + request_kind text not null, + url text not null, + http_status integer, + headers jsonb not null default '{}'::jsonb, + message text not null, + rule text, + body text not null, + truncated boolean not null default false, + created_at timestamptz not null default now() +); +create index site_failure_sample_site_idx on site_failure_sample (site_id, created_at desc); +`, + ); +} + +// biome-ignore lint/suspicious/noExplicitAny: 同上 +export async function down(db: Kysely): Promise { + await execDdl(db, dropTables(["site_failure_sample"])); +} diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index 4daaf3b8..3c4af318 100644 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -255,6 +255,7 @@ export type SiteCircuitState = "closed" | "open" | "half_open"; export type SiteKind = "pt" | "public_bt" | "rss_feed" | "torznab" | "subtitle"; export type SiteAccountAuthKind = "cookie" | "passkey" | "api_key" | "password"; export type SiteAccountStatus = "unknown" | "active" | "invalid" | "expired" | "challenge"; +export type SiteFailureSampleKind = "parse_error" | "challenge"; export type SiteSnapshotKind = "periodic" | "backfill" | "live_search" | "manual"; export type SiteSnapshotStatus = "running" | "ok" | "partial" | "failed"; export type SourceReleaseResolutionStatus = "unresolved" | "resolved" | "ambiguous" | "unidentifiable" | "rejected"; @@ -1491,6 +1492,21 @@ export interface SiteAccountTable { ua_pending: string | null; } +export interface SiteFailureSampleTable { + id: Generated; + site_id: string; + kind: SiteFailureSampleKind; + request_kind: string; + url: string; + http_status: number | null; + headers: Generated; + message: string; + rule: string | null; + body: string; + truncated: Generated; + created_at: GeneratedTimestamptz; +} + export interface SiteRequestCountTable { site_id: string; hour: Timestamptz; @@ -1825,6 +1841,7 @@ export interface DB { settings: SettingsTable; site: SiteTable; site_account: SiteAccountTable; + site_failure_sample: SiteFailureSampleTable; site_request_count: SiteRequestCountTable; site_snapshot: SiteSnapshotTable; site_user_data: SiteUserDataTable; @@ -2051,6 +2068,9 @@ export type SiteUpdate = Updateable; export type SiteAccountRow = Selectable; export type NewSiteAccount = Insertable; export type SiteAccountUpdate = Updateable; +export type SiteFailureSampleRow = Selectable; +export type NewSiteFailureSample = Insertable; +export type SiteFailureSampleUpdate = Updateable; export type SiteRequestCountRow = Selectable; export type NewSiteRequestCount = Insertable; export type SiteRequestCountUpdate = Updateable; diff --git a/packages/db/test/schema.test.ts b/packages/db/test/schema.test.ts index cb37d6a4..0b21163b 100644 --- a/packages/db/test/schema.test.ts +++ b/packages/db/test/schema.test.ts @@ -130,6 +130,8 @@ const EXPECTED_TABLES = [ "subtitle_write", // 集数映射规则(0066) "episode_mapping_rule", + // 站点请求失败留样(0077) + "site_failure_sample", ].sort(); /** 不走 entityTable() 的表:追加型/引擎自有表,没有 uuid id + created_at/updated_at 三件套 */ @@ -163,6 +165,8 @@ const NON_ENTITY_TABLES = [ "kv_cache", // 站点请求计数:(site_id, hour) 主键 "site_request_count", + // 失败留样:只追加,只有 created_at + "site_failure_sample", ]; describe.skipIf(skip)("schema migrations (real PG)", () => {