From 1779e1a0f25e600c5200399bf7faddeb9efdfb55 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 6 Oct 2026 15:35:31 +0800 Subject: [PATCH] =?UTF-8?q?fix(adapters):=20=E8=AF=86=E5=88=AB=20200=20?= =?UTF-8?q?=E7=8A=B6=E6=80=81=E7=9A=84=E6=8C=91=E6=88=98=20/=20=E4=BA=BA?= =?UTF-8?q?=E6=9C=BA=E9=AA=8C=E8=AF=81=E9=A1=B5,=E8=A7=A3=E6=9E=90?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5=E5=89=8D=E5=85=9C=E5=BA=95,=E5=A4=B1?= =?UTF-8?q?=E8=B4=A5=E9=A1=B5=E9=9D=A2=E7=95=99=E6=A0=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - detectChallenge 支持页面类请求(GET + Accept text/html)的 2xx 响应:正文只先读 256 KB, 短页且没有种子表 / 用户信息块时,认 Cloudflare 挑战标题 / 脚本与 Turnstile / hCaptcha / reCAPTCHA 门页; 带登录表单的页面不算门页。非页面请求不读正文 - siteFetch 用上面的判定;挑战错误附脱敏页面样本,文案带级联结果(x-umt-challenge) - spider 适配器:认不出列表 / 详情时先看是不是人机验证页,是就报 challenge(「搜索页需要人机验证」), 不再报改版;parse_error / challenge 都经 deps.onFailureSample 交出脱敏样本(前 64 KB,去掉个人信息块、 passkey / authkey / uid / 用户名 / Cookie / 邮箱) - classifyChallengePage 也认门页(浏览器过挑战时不再把门页当已通过) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei --- packages/adapters/src/sites/challenge.test.ts | 153 +- packages/adapters/src/sites/challenge.ts | 167 ++- packages/adapters/src/sites/deps.ts | 3 + .../adapters/src/sites/failure-sample.test.ts | 68 + packages/adapters/src/sites/failure-sample.ts | 124 ++ packages/adapters/src/sites/http.ts | 26 +- packages/adapters/src/sites/index.ts | 14 + packages/adapters/src/sites/spider/adapter.ts | 119 +- .../src/sites/spider/challenge-page.test.ts | 123 ++ packages/adapters/src/sites/types.ts | 7 +- .../test/fixtures/audiences-search.html.txt | 1308 +++++++++++++++++ 11 files changed, 2078 insertions(+), 34 deletions(-) create mode 100644 packages/adapters/src/sites/failure-sample.test.ts create mode 100644 packages/adapters/src/sites/failure-sample.ts create mode 100644 packages/adapters/src/sites/spider/challenge-page.test.ts create mode 100644 packages/adapters/test/fixtures/audiences-search.html.txt diff --git a/packages/adapters/src/sites/challenge.test.ts b/packages/adapters/src/sites/challenge.test.ts index 537772e9..364435f3 100644 --- a/packages/adapters/src/sites/challenge.test.ts +++ b/packages/adapters/src/sites/challenge.test.ts @@ -1,5 +1,14 @@ +import { readFileSync } from "node:fs"; import { describe, expect, it } from "vitest"; -import { challengeRuleOfHtml, classifyChallengePage, detectChallenge } from "./challenge.ts"; +import { + CHALLENGE_OUTCOME_HEADER, + CHALLENGE_PEEK_BYTES, + challengeRuleOfHtml, + challengeRuleOfUnrecognizedPage, + classifyChallengePage, + detectChallenge, + isPageRequest, +} from "./challenge.ts"; import { siteFetch } from "./http.ts"; import { createSiteRateLimiter } from "./rate-limit.ts"; import { SiteError } from "./types.ts"; @@ -114,3 +123,145 @@ describe("classifyChallengePage", () => { ); }); }); + +// ── 200 状态的挑战页 / 人机验证门页(搜索页单独加盾、间歇触发)── + +const fixture = (rel: string) => readFileSync(new URL(rel, import.meta.url), "utf8"); +/** 观众真实搜索页(脱敏、删减到 4 行,``) */ +const AUD_SEARCH = fixture("../../test/fixtures/audiences-search.html.txt"); +/** 观众真实列表页(DET-2 样本,已脱敏) */ +const AUD_LIST = fixture("../../test/differential/inputs/indexer/rl-audiences-list.html.txt"); +/** Turnstile 门页:站点自己出的「请完成人机验证」短页,没有种子表、没有用户信息块 */ +const TURNSTILE_GATE = `Audiences + +

请完成人机验证后继续

+
`; +const HCAPTCHA_GATE = `Security check +
`; +/** 登录页本身带 Turnstile:交给登录判定,不算门页 */ +const LOGIN_WITH_TURNSTILE = `Audiences :: 登录 - Powered by NexusPHP + + +
`; +/** 正常页面里注入了 challenge-platform 脚本,侧栏还有个签到 Turnstile:有种子表 / 用户信息块 → 不是门页 */ +const NORMAL_WITH_WIDGET = `Audiences :: 种子 + + +
+
row
`; + +const page200 = (body: string, headers: Record = {}) => + new Response(body, { status: 200, headers: { "content-type": "text/html; charset=utf-8", ...headers } }); + +describe("detectChallenge:页面请求的 200 响应", () => { + it("200 的 Cloudflare 挑战页(标题 / 脚本)→ 挑战", async () => { + expect((await detectChallenge(page200(CF_PAGE), { page: true })).rule).toContain("Just a moment"); + const noTitle = `x`; + expect((await detectChallenge(page200(noTitle), { page: true })).rule).toContain("cf_chl_opt"); + }); + + it("200 的 Turnstile / hCaptcha 门页 → 挑战,正文原样交还", async () => { + const a = await detectChallenge(page200(TURNSTILE_GATE), { page: true }); + expect(a.rule).toMatch(/^gate\*=/); + expect(a.html).toBe(TURNSTILE_GATE); + expect(await a.res.text()).toBe(TURNSTILE_GATE); + expect((await detectChallenge(page200(HCAPTCHA_GATE), { page: true })).rule).toContain("hcaptcha"); + }); + + it("正常搜索页 / 列表页 / 带 Turnstile 的登录页 / 注入脚本的正常页 → 不误判", async () => { + for (const body of [AUD_SEARCH, AUD_LIST, LOGIN_WITH_TURNSTILE, NORMAL_WITH_WIDGET, NORMAL]) { + const r = await detectChallenge(page200(body), { page: true }); + expect(r.rule).toBeNull(); + expect(await r.res.text()).toBe(body); + } + }); + + it("不是页面请求 / 不是 HTML:不读正文(原响应对象直接返回)", async () => { + const res = page200(CF_PAGE); + const r = await detectChallenge(res); + expect(r.rule).toBeNull(); + expect(r.res).toBe(res); + expect(res.bodyUsed).toBe(false); + const json = new Response('{"title":"Just a moment"}', { + status: 200, + headers: { "content-type": "application/json" }, + }); + const j = await detectChallenge(json, { page: true }); + expect(j.res).toBe(json); + }); + + it("正文超过读取上限:只读开头不判门页,其余照常流给调用方", async () => { + const big = `${TURNSTILE_GATE}${"".repeat(Math.ceil((CHALLENGE_PEEK_BYTES * 2) / 12))}`; + const r = await detectChallenge(page200(big), { page: true }); + expect(r.rule).toBeNull(); + expect(await r.res.text()).toBe(big); + }); + + it("isPageRequest:GET + Accept text/html", () => { + expect(isPageRequest("GET", "text/html,application/xhtml+xml")).toBe(true); + expect(isPageRequest(undefined, "text/html")).toBe(true); + expect(isPageRequest("POST", "text/html")).toBe(false); + expect(isPageRequest("GET", "application/json")).toBe(false); + expect(isPageRequest("GET", null)).toBe(false); + }); +}); + +describe("siteFetch:200 门页 → challenge 并附脱敏样本", () => { + const url = "https://audiences.example/torrents.php?search=x&passkey=0123456789abcdef0123456789abcdef"; + const html = { accept: "text/html,application/xhtml+xml" }; + + it("页面请求遇 200 门页 → challenge;样本 URL 脱敏、带关键响应头;级联结果进文案", async () => { + let err: unknown; + try { + await siteFetch( + ctx( + reply(200, TURNSTILE_GATE, { + server: "cloudflare", + "cf-ray": "abc-HKG", + [CHALLENGE_OUTCOME_HEADER]: "needs_human", + }), + ), + url, + { headers: html }, + ); + } catch (e) { + err = e; + } + const se = err as SiteError; + expect(se).toBeInstanceOf(SiteError); + expect(se.kind).toBe("challenge"); + expect(se.message).toContain("原站视图"); + expect(se.detail).toMatchObject({ reason: "challenge", outcome: "needs_human", httpStatus: 200 }); + expect(se.sample).toMatchObject({ + kind: "challenge", + httpStatus: 200, + headers: { server: "cloudflare", "cf-ray": "abc-HKG" }, + }); + expect(se.sample?.url).not.toContain("0123456789abcdef"); + expect(se.sample?.body).toContain("cf-turnstile"); + }); + + it("同一页面不是页面请求(没要 HTML)→ 照常返回", async () => { + const res = await siteFetch(ctx(reply(200, TURNSTILE_GATE)), url); + expect(await res.text()).toBe(TURNSTILE_GATE); + }); +}); + +describe("challengeRuleOfUnrecognizedPage(解析失败时的兜底)", () => { + it("站点外壳里的人机验证页(有用户信息块、很长)也认;登录表单不算", () => { + const shell = `Audiences +
${"x".repeat(150_000)}

请完成人机验证

`; + expect(challengeRuleOfUnrecognizedPage(shell)).toContain("cf-turnstile"); + expect(challengeRuleOfUnrecognizedPage("

搜索需要先完成人机验证

")).toBe( + "text*=人机验证", + ); + expect(challengeRuleOfUnrecognizedPage(LOGIN_WITH_TURNSTILE)).toBeNull(); + expect(challengeRuleOfUnrecognizedPage("
改版
")).toBeNull(); + }); + + it("classifyChallengePage 也认门页:Turnstile → interactive,hCaptcha → captcha", () => { + expect(classifyChallengePage({ html: TURNSTILE_GATE, text: "请完成人机验证后继续" }).kind).toBe("interactive"); + expect(classifyChallengePage({ html: HCAPTCHA_GATE, text: "" }).kind).toBe("captcha"); + expect(classifyChallengePage({ html: LOGIN_WITH_TURNSTILE, text: "登录" }).kind).toBe("none"); + }); +}); diff --git a/packages/adapters/src/sites/challenge.ts b/packages/adapters/src/sites/challenge.ts index 0c521ab7..5c213986 100644 --- a/packages/adapters/src/sites/challenge.ts +++ b/packages/adapters/src/sites/challenge.ts @@ -1,5 +1,6 @@ // CF-2 Cloudflare 挑战识别(docs/design/cloudflare-handling.md)。 // - HTTP 端:响应头 `cf-mitigated: challenge`(任何状态码),或 403 / 503 / 429 且正文是挑战页 → 挑战,不是凭据错误; +// 页面类请求(GET + Accept text/html)返回 200 也可能是挑战页 / 人机验证门页(Turnstile、hCaptcha、reCAPTCHA),短页且没有站点正常内容才算; // - 浏览器端:挑战页分三类——自动(JS 检查,等几秒自己过)、可点(「我是人类」复选框 / Turnstile)、真验证码(选图 / 输码)。 // 只用正则,不解析 DOM:HTTP 端要在限流器之后快速判断,浏览器端拿到的是 outerHTML + innerText 样本。 @@ -27,21 +28,168 @@ export function challengeRuleOfHeaders(headers: Headers): string | null { return headers.get("cf-mitigated")?.trim().toLowerCase() === "challenge" ? "header cf-mitigated=challenge" : null; } +/** + * 站点请求经服务端 Cloudflare 级联后仍是挑战时,级联在返回的响应上加这个头说明处理结果 + * (needs_human / failed / throttled / no_browser / no_account),适配器据此写错误文案; + * unhandled = 网络层没认出、适配器解析时才认出的验证页(级联没跑)。 + */ +export const CHALLENGE_OUTCOME_HEADER = "x-umt-challenge"; + +/** 人机验证错误文案的后半句:自动处理到哪一步、要不要人 */ +export function challengeOutcomeText(outcome: string | null | undefined): string { + switch (outcome) { + case "needs_human": + return "自动处理没能通过，需要你在原站视图里过一次验证"; + case "failed": + return "已自动尝试用浏览器通过，没有成功"; + case "throttled": + return "30 分钟内已自动尝试过，之后会再试"; + case "no_browser": + return "没有可用的浏览器，无法自动通过"; + case "no_account": + return "站点没有已登录的账号，不自动处理"; + case "unhandled": + return "这类页面没有自动处理，可以在原站视图里过一次验证"; + default: + return "会自动尝试用浏览器通过"; + } +} + /** 这些状态码的响应才去看正文(Cloudflare 挑战页是 403 / 503,少数 429) */ export const CHALLENGE_STATUSES: readonly number[] = [403, 429, 503]; +/** 判定时最多先读这么多正文;超出部分不缓冲,照常流给调用方。挑战 / 人机验证门页都远小于这个 */ +export const CHALLENGE_PEEK_BYTES = 256 * 1024; + +/** 页面类请求(GET + Accept text/html):只有这类请求的 2xx 响应才检查正文 */ +export function isPageRequest(method: string | null | undefined, accept: string | null | undefined): boolean { + return (method ?? "GET").toUpperCase() === "GET" && /text\/html|application\/xhtml/i.test(accept ?? ""); +} + +/** 人机验证组件:Turnstile / hCaptcha / reCAPTCHA(只认脚本地址与组件 class,不认文案) */ +const GATE_WIDGET = + /challenges\.cloudflare\.com\/turnstile|class="[^"]*\bcf-turnstile\b|hcaptcha\.com\/1\/api|class="[^"]*\bh-captcha\b|google\.com\/recaptcha|recaptcha\/api\.js|class="[^"]*\bg-recaptcha\b/i; +/** 站点正常内容的痕迹:种子表、已登录的用户信息块(NexusPHP 等)。出现任何一个就不是门页 */ +const SITE_CONTENT = + /id="torrenttable"|]*class="[^"]*\btorrents\b|logout\.php|userdetails\.php\?id=|usercp\.php|id="info_block"/i; +/** 登录表单(登录页本身也可能带 Turnstile,交给登录 / 失效判定,不算门页) */ +const LOGIN_FORM = /type=["']?password|takelogin\.php/i; +/** 门页都很短;超过这个长度的页面不按门页判 */ +const GATE_MAX_CHARS = 100_000; +/** 页面文字里的人机验证提示(只在页面已经认不出来时用) */ +const HUMAN_TEXT = /人机验证|verify you are human|确认您是真人|验证您是真人|checking your browser/i; + /** - * 判定一个站点响应是不是挑战。正文只在状态码可能是挑战时读(读过的正文重新包成 Response 返回,调用方照常消费)。 - * 返回 { res, rule }:rule 为 null 表示不是挑战。 + * 人机验证门页:页面上有 Turnstile / hCaptcha / reCAPTCHA,且没有登录表单、没有站点正常内容、页面很短。 + * 命中返回规则说明,否则 null。 */ -export async function detectChallenge(res: Response): Promise<{ res: Response; rule: string | null }> { +export function challengeGateRule(html: string | null | undefined): string | null { + if (!html || html.length > GATE_MAX_CHARS) return null; + const w = GATE_WIDGET.exec(html); + if (!w || LOGIN_FORM.test(html) || SITE_CONTENT.test(html)) return null; + return `gate*=${w[0].slice(0, 40)}`; +} + +/** + * 2xx 页面是不是挑战 / 人机验证页。比非 2xx 严:正文必须完整读到(短页)、没有站点正常内容, + * 再看挑战标题 / 脚本特征或人机验证门页。complete = 正文是否已全部读到。 + */ +export function challengeRuleOfOkPage(html: string, complete = true): string | null { + if (!complete || !html || html.length > GATE_MAX_CHARS || SITE_CONTENT.test(html)) return null; + return challengeRuleOfHtml(html) ?? challengeGateRule(html); +} + +/** + * 页面已经解析不出来(列表 / 详情认不出结构)时的兜底:像挑战 / 人机验证就返回规则说明。 + * 比 challengeRuleOfOkPage 宽:不要求短页、也认页面文字里的「人机验证」提示,但带登录表单的页面不算。 + */ +export function challengeRuleOfUnrecognizedPage(html: string | null | undefined): string | null { + if (!html) return null; + const head = html.slice(0, 300_000); + const cf = challengeRuleOfHtml(head); + if (cf) return cf; + if (LOGIN_FORM.test(head)) return null; + const w = GATE_WIDGET.exec(head); + if (w) return `gate*=${w[0].slice(0, 40)}`; + const t = HUMAN_TEXT.exec(head.replace(/|/gi, " ")); + return t ? `text*=${t[0]}` : null; +} + +function concatBytes(chunks: Uint8Array[], total: number): Uint8Array { + const out = new Uint8Array(total); + let off = 0; + for (const c of chunks) { + out.set(c, off); + off += c.byteLength; + } + return out; +} + +/** 先读最多 max 字节;返回读到的部分、是否读完,以及一个内容不变的新响应(没读完的部分接着原流) */ +async function peekBody( + res: Response, + max: number, +): Promise<{ head: Uint8Array; complete: boolean; res: Response }> { + const reader = (res.body as ReadableStream).getReader(); + const chunks: Uint8Array[] = []; + let total = 0; + let complete = false; + while (total < max) { + const r = await reader.read(); + if (r.done) { + complete = true; + break; + } + chunks.push(r.value); + total += r.value.byteLength; + } + const head = concatBytes(chunks, total); + const init = { status: res.status, statusText: res.statusText, headers: res.headers }; + if (complete) return { head, complete, res: new Response(head, init) }; + const stream = new ReadableStream({ + start(c) { + c.enqueue(head); + }, + async pull(c) { + const r = await reader.read(); + if (r.done) c.close(); + else c.enqueue(r.value); + }, + cancel(reason) { + return reader.cancel(reason); + }, + }); + return { head, complete, res: new Response(stream, init) }; +} + +/** + * 判定一个站点响应是不是挑战。 + * - 响应头 `cf-mitigated: challenge` → 挑战(不读正文); + * - 403 / 429 / 503 → 读正文看挑战页特征; + * - opts.page(页面类请求)且 2xx 的 HTML → 读正文看挑战页 / 人机验证门页(challengeRuleOfOkPage); + * - 其它情况不读正文,零开销。 + * 正文最多先读 CHALLENGE_PEEK_BYTES,读过的部分重新包进返回的响应,调用方照常消费。 + * 返回 { res, rule, html }:rule 为 null 表示不是挑战;html 是读到的正文开头(没读为 null,留样用)。 + */ +export async function detectChallenge( + res: Response, + opts: { page?: boolean } = {}, +): Promise<{ res: Response; rule: string | null; html: string | null }> { const byHeader = challengeRuleOfHeaders(res.headers); - if (byHeader) return { res, rule: byHeader }; - if (!CHALLENGE_STATUSES.includes(res.status) || !res.body) return { res, rule: null }; - const bytes = new Uint8Array(await res.arrayBuffer()); - const html = new TextDecoder("utf-8").decode(bytes.subarray(0, 300_000)); - const copy = new Response(bytes, { status: res.status, statusText: res.statusText, headers: res.headers }); - return { res: copy, rule: challengeRuleOfHtml(html) }; + if (byHeader) return { res, rule: byHeader, html: null }; + const suspect = CHALLENGE_STATUSES.includes(res.status); + const okPage = + !!opts.page && + res.status >= 200 && + res.status < 300 && + res.status !== 204 && + res.status !== 205 && + /html|^$/i.test(res.headers.get("content-type")?.trim() ?? ""); + if ((!suspect && !okPage) || !res.body) return { res, rule: null, html: null }; + const peek = await peekBody(res, CHALLENGE_PEEK_BYTES); + const html = new TextDecoder("utf-8").decode(peek.head); + const rule = suspect ? challengeRuleOfHtml(html) : challengeRuleOfOkPage(html, peek.complete); + return { res: peek.res, rule, html }; } /** @@ -68,6 +216,7 @@ export function classifyChallengePage(page: { title?: string | null; html?: stri const rule = (CHALLENGE_TITLE.test(title) ? `title=${title.slice(0, 40)}` : null) ?? challengeRuleOfHtml(html) ?? + challengeGateRule(html) ?? (/verify you are human|确认您是真人|验证您是真人|checking (?:if the site connection is secure|your browser)/i.test( text, ) diff --git a/packages/adapters/src/sites/deps.ts b/packages/adapters/src/sites/deps.ts index 57bd3ed9..b9e42f08 100644 --- a/packages/adapters/src/sites/deps.ts +++ b/packages/adapters/src/sites/deps.ts @@ -1,3 +1,4 @@ +import type { SiteFailureSample } from "./failure-sample.ts"; import type { SiteHttpContext } from "./http.ts"; import { createSiteRateLimiter, type SiteRateLimiter } from "./rate-limit.ts"; import type { SecretResolver } from "./types.ts"; @@ -12,6 +13,8 @@ export interface SiteAdapterDeps { userAgent?: string; /** 覆盖站点基址(测试指向 mock) */ baseUrl?: string; + /** 请求报 parse_error / challenge 时的页面样本(已脱敏);调用方自行保存,抛错不影响请求结果 */ + onFailureSample?: (sample: SiteFailureSample) => void; } export function buildHttpContext( diff --git a/packages/adapters/src/sites/failure-sample.test.ts b/packages/adapters/src/sites/failure-sample.test.ts new file mode 100644 index 00000000..7b3dd794 --- /dev/null +++ b/packages/adapters/src/sites/failure-sample.test.ts @@ -0,0 +1,68 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; +import { buildFailureSample, FAILURE_SAMPLE_MAX_BYTES, redactPageHtml } from "./failure-sample.ts"; + +// 失败留样的脱敏:passkey / authkey / uid / 用户名 / Cookie / 邮箱 / 个人信息块都不进样本;截到 64 KB。 + +const AUD_SEARCH = readFileSync(new URL("../../test/fixtures/audiences-search.html.txt", import.meta.url), "utf8"); + +describe("redactPageHtml", () => { + it("下载链接密钥、表单 / 脚本里的密钥、Cookie、邮箱、用户链接都抹掉,普通内容保留", () => { + const key = "0123456789abcdef0123456789abcdef"; + const html = `dl +rss + +

联系 bob@example.org

bobby UID 4242 +Dune.Part.Two.2024`; + const out = redactPageHtml(html); + for (const leak of [ + key, + "abc123def", + "zzzzzzzz9999", + "tok_ABCDEFGH", + "hunter2xyz", + "bob@example.org", + "4242", + "bobby", + "777", + ]) { + expect(out).not.toContain(leak); + } + expect(out).toContain("download.php?id=9"); + expect(out).toContain("Dune.Part.Two.2024"); + }); + + it("真实搜索页:个人信息块整块删掉,种子表保留", () => { + const out = redactPageHtml(AUD_SEARCH); + expect(AUD_SEARCH).toContain("fixtureuser"); + expect(out).not.toContain("fixtureuser"); + expect(out).toContain(""); + expect(out).toContain('id="torrenttable"'); + }); +}); + +describe("buildFailureSample", () => { + it("截到 64 KB、URL 脱敏、只留关键响应头", () => { + const s = buildFailureSample({ + kind: "parse_error", + requestKind: "search", + url: "https://aud.test/torrents.php?search=x&passkey=secretsecret&uid=5", + httpStatus: 200, + headers: new Headers({ + server: "cloudflare", + "cf-ray": "1-HKG", + "set-cookie": "a=b", + "cf-mitigated": "challenge", + }), + message: "搜索:页面结构无法识别", + html: AUD_SEARCH, + now: new Date("2026-10-06T00:00:00Z"), + }); + expect(new TextEncoder().encode(s.body).byteLength).toBeLessThanOrEqual(FAILURE_SAMPLE_MAX_BYTES); + expect(s.truncated).toBe(true); + expect(s.url).not.toContain("secretsecret"); + expect(s.url).not.toContain("uid=5"); + expect(s.headers).toEqual({ server: "cloudflare", "cf-ray": "1-HKG", "cf-mitigated": "challenge" }); + expect(s.capturedAt).toBe("2026-10-06T00:00:00.000Z"); + }); +}); diff --git a/packages/adapters/src/sites/failure-sample.ts b/packages/adapters/src/sites/failure-sample.ts new file mode 100644 index 00000000..d971649a --- /dev/null +++ b/packages/adapters/src/sites/failure-sample.ts @@ -0,0 +1,124 @@ +import { redactString } from "@umt/contracts"; +import { redactText, redactUrl } from "../rss/redact.ts"; + +// 站点请求失败留样:请求报「页面结构无法识别」(parse_error)或「人机验证」(challenge)时,留一份脱敏后的页面开头, +// 站点页「技术信息」里可看 / 下载,用来区分是站点改版还是被人机验证拦了(间歇触发的那一刻页面不会再出现)。 + +/** 页面样本最多保留的字节数 */ +export const FAILURE_SAMPLE_MAX_BYTES = 64 * 1024; + +/** 请求类型:搜索 / 列表 / 详情 / 健康检测 / 其它 */ +export type SiteRequestKind = "search" | "list" | "detail" | "health" | "other"; + +/** 留样只记这几个响应头 */ +const KEPT_HEADERS = ["server", "cf-mitigated", "cf-ray", "content-type"] as const; + +export interface SiteFailureSample { + kind: "parse_error" | "challenge"; + requestKind: SiteRequestKind; + /** 已脱敏 */ + url: string; + httpStatus: number | null; + headers: Record; + capturedAt: string; + /** 报出去的错误文案 */ + message: string; + /** 命中的挑战 / 门页规则 */ + rule: string | null; + /** 脱敏后的页面开头(≤ FAILURE_SAMPLE_MAX_BYTES 字节) */ + body: string; + /** 页面是否被截断 */ + truncated: boolean; +} + +/** URL 里除了通用凭据参数,还要抹掉的用户标识参数 */ +const EXTRA_KEYS = ["uid", "userid", "user_id"]; +const EMAIL_RE = /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g; +/** NexusPHP passkey / authkey 等:32 位以上十六进制串(info hash 是 40 位,也一并抹掉,样本不需要) */ +const HEX_KEY_RE = /\b[a-f0-9]{32,64}\b/gi; +/** Cookie 名=值(页面脚本里偶尔会出现) */ +const COOKIE_PAIR_RE = + /\b(c_secure_[a-z_]+|cf_clearance|__cf_bm|phpsessid|PHPSESSID|session(?:id)?|remember_[a-z0-9_]+)=([^;\s"'<>&]+)/gi; +/** 表单 / 脚本里的密钥字段:name="passkey" value="…"、passkey: "…" */ +const SECRET_FIELD_RE = + /\b(pass[_-]?key|auth[_-]?key|torrent[_-]?pass|rss[_-]?key|api[_-]?key|token|csrf[_-]?token|secret)(["']?\s*(?:[:=]|value=)\s*["']?)([^"'\s<>&;,]{6,})/gi; +/** 用户页链接里的用户 id 与链接文字(用户名) */ +const USER_LINK_RE = /(userdetails\.php\?id=)\d+/gi; +const USER_NAME_RE = /(]*userdetails\.php[^>]*>)((?:\s*<[^>]+>)*)([^<]+)/gi; + +/** 页面里的「UID 12345」 */ +const UID_TEXT_RE = /\bUID\s*[:：]?\s*\d+/gi; + +/** + * 删掉 id 为 id 的整个元素(按同名标签计数找配对的结束标签);NexusPHP 的 `#info_block` 是当前用户的个人信息块 + * (用户名、分享率、魔力…),样本里不需要。找不到原样返回。 + */ +function dropElementById(html: string, id: string): string { + const open = new RegExp(`<([a-z][a-z0-9]*)\\b[^>]*\\bid=["']?${id}\\b[^>]*>`, "i").exec(html); + if (!open) return html; + const tag = (open[1] as string).toLowerCase(); + const re = new RegExp(`<(/?)${tag}\\b[^>]*>`, "gi"); + re.lastIndex = open.index + open[0].length; + let depth = 1; + for (let m = re.exec(html); m; m = re.exec(html)) { + depth += m[1] ? -1 : 1; + if (depth === 0) return `${html.slice(0, open.index)}${html.slice(m.index + m[0].length)}`; + } + return `${html.slice(0, open.index)}`; +} + +/** 页面文本脱敏:个人信息块、凭据参数、用户 id / 用户名、Cookie、邮箱、密钥样式的十六进制串 */ +export function redactPageHtml(html: string): string { + return redactString( + redactText(dropElementById(html, "info_block"), EXTRA_KEYS) + .replace(UID_TEXT_RE, "UID [REDACTED]") + .replace(USER_NAME_RE, "$1$2[USER]") + .replace(USER_LINK_RE, "$1[REDACTED]") + .replace(COOKIE_PAIR_RE, "$1=[REDACTED]") + .replace(SECRET_FIELD_RE, "$1$2[REDACTED]") + .replace(EMAIL_RE, "[EMAIL]") + .replace(HEX_KEY_RE, "[HEX]"), + ); +} + +/** 先脱敏、再按字节截到上限(截断处的半个字符丢掉) */ +function clip(text: string, maxBytes: number): { body: string; truncated: boolean } { + const bytes = new TextEncoder().encode(text); + if (bytes.byteLength <= maxBytes) return { body: text, truncated: false }; + return { body: new TextDecoder().decode(bytes.subarray(0, maxBytes)).replace(/�$/, ""), truncated: true }; +} + +export function buildFailureSample(input: { + kind: SiteFailureSample["kind"]; + requestKind: SiteRequestKind; + url: string; + httpStatus?: number | null; + headers?: Headers | null; + message: string; + rule?: string | null; + html: string | null | undefined; + now: Date; +}): SiteFailureSample { + const headers: Record = {}; + for (const k of KEPT_HEADERS) { + const v = input.headers?.get(k); + if (v) headers[k] = v.slice(0, 200); + } + // 脱敏只看前 4 倍上限(截断前先脱敏,避免截断把密钥切成两半漏掉) + const { body, truncated } = clip( + redactPageHtml((input.html ?? "").slice(0, FAILURE_SAMPLE_MAX_BYTES * 4)), + FAILURE_SAMPLE_MAX_BYTES, + ); + return { + kind: input.kind, + requestKind: input.requestKind, + url: redactUrl(input.url, EXTRA_KEYS), + httpStatus: input.httpStatus ?? null, + headers, + capturedAt: input.now.toISOString(), + message: input.message, + rule: input.rule ?? null, + body, + truncated: truncated || (input.html?.length ?? 0) > FAILURE_SAMPLE_MAX_BYTES * 4, + }; +} diff --git a/packages/adapters/src/sites/http.ts b/packages/adapters/src/sites/http.ts index 43e8d33f..bd5b031e 100644 --- a/packages/adapters/src/sites/http.ts +++ b/packages/adapters/src/sites/http.ts @@ -1,5 +1,6 @@ import { redactText } from "../rss/redact.ts"; -import { detectChallenge } from "./challenge.ts"; +import { CHALLENGE_OUTCOME_HEADER, challengeOutcomeText, detectChallenge, isPageRequest } from "./challenge.ts"; +import { buildFailureSample } from "./failure-sample.ts"; import type { SiteRateLimiter } from "./rate-limit.ts"; import { SiteError } from "./types.ts"; @@ -21,7 +22,8 @@ function retryAfterMs(res: Response, now: Date): number | undefined { } /** - * 站点请求:先取站点令牌,再发请求;Cloudflare 挑战(`cf-mitigated: challenge`,或 403 / 503 / 429 的挑战页)→ `challenge` + * 站点请求:先取站点令牌,再发请求;Cloudflare 挑战(`cf-mitigated: challenge`,或 403 / 503 / 429 的挑战页, + * 或页面请求返回 200 的挑战页 / 人机验证门页)→ `challenge`(附脱敏页面样本) * (不是凭据错误,CF-2);其余 429/503 统一映射并顺延限流器。 * 返回的 Response 由调用方消费。错误消息只含脱敏后的 URL。 */ @@ -44,13 +46,27 @@ export async function siteFetch( const reason = timeout.aborted ? "超时" : (e as Error).message; throw new SiteError(ctx.siteId, "network", redactText(`请求 ${safeUrl} 失败:${reason}`)); } - const ch = await detectChallenge(res); + // 页面类请求的 200 响应也看正文:返回 200 的挑战页 / 人机验证门页(只读开头,有上限) + const ch = await detectChallenge(res, { page: isPageRequest(init.method, headers.get("accept")) }); res = ch.res; if (ch.rule) { await res.body?.cancel().catch(() => {}); - throw new SiteError(ctx.siteId, "challenge", `请求 ${safeUrl}:站点要求人机验证(Cloudflare)`, { + const outcome = res.headers.get(CHALLENGE_OUTCOME_HEADER); + const message = `请求 ${safeUrl}：站点要求人机验证（Cloudflare 等），${challengeOutcomeText(outcome)}`; + throw new SiteError(ctx.siteId, "challenge", message, { httpStatus: res.status, - detail: { reason: "challenge", rule: ch.rule, httpStatus: res.status }, + detail: { reason: "challenge", rule: ch.rule, httpStatus: res.status, ...(outcome ? { outcome } : {}) }, + sample: buildFailureSample({ + kind: "challenge", + requestKind: "other", + url, + httpStatus: res.status, + headers: res.headers, + message, + rule: ch.rule, + html: ch.html, + now: ctx.now(), + }), }); } if (res.status === 429 || res.status === 503) { diff --git a/packages/adapters/src/sites/index.ts b/packages/adapters/src/sites/index.ts index 0f19c314..e0a9ac79 100644 --- a/packages/adapters/src/sites/index.ts +++ b/packages/adapters/src/sites/index.ts @@ -1,10 +1,17 @@ export { + CHALLENGE_OUTCOME_HEADER, + CHALLENGE_PEEK_BYTES, CHALLENGE_STATUSES, type ChallengePageKind, + challengeGateRule, + challengeOutcomeText, challengeRuleOfHeaders, challengeRuleOfHtml, + challengeRuleOfOkPage, + challengeRuleOfUnrecognizedPage, classifyChallengePage, detectChallenge, + isPageRequest, } from "./challenge.ts"; export { audiencesDefinition, @@ -34,6 +41,13 @@ export { structureDescription, } from "./detail-description.ts"; export { type FormattedDetailPage, formatDetailPage } from "./detail-format.ts"; +export { + buildFailureSample, + FAILURE_SAMPLE_MAX_BYTES, + redactPageHtml, + type SiteFailureSample, + type SiteRequestKind, +} from "./failure-sample.ts"; export { type FeedMediaType, normalizeSiteFeedItem, diff --git a/packages/adapters/src/sites/spider/adapter.ts b/packages/adapters/src/sites/spider/adapter.ts index 33af4865..d2df99ac 100644 --- a/packages/adapters/src/sites/spider/adapter.ts +++ b/packages/adapters/src/sites/spider/adapter.ts @@ -14,7 +14,9 @@ import { inspectTorrentIsolated } from "../../isolation.ts"; import { redactText } from "../../rss/redact.ts"; +import { challengeOutcomeText, challengeRuleOfUnrecognizedPage } from "../challenge.ts"; import { buildHttpContext, type SiteAdapterDeps } from "../deps.ts"; +import { buildFailureSample, type SiteRequestKind } from "../failure-sample.ts"; import { readBody, readText, siteFetch } from "../http.ts"; import { type DownloadTarget, @@ -107,6 +109,23 @@ export function judgeLoginPage( }; } +/** 抓到的页面:正文与留样用的响应信息 */ +interface FetchedPage { + html: string; + url: string; + status: number; + headers: Headers; +} + +/** 错误文案里的页面名 */ +const PAGE_LABEL: Record = { + search: "搜索页", + list: "列表页", + detail: "详情页", + health: "首页", + other: "页面", +}; + export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdapterDeps): SiteAdapter { const def: IndexerDefinition = deps.baseUrl ? { ...input, domain: deps.baseUrl } : input; const ext = def.umt; @@ -119,13 +138,73 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap return c; } - async function fetchPage(url: string, what: string, opts: SiteCallOptions): Promise { - const res = await siteFetch(http, url, { - method: "GET", - headers: { cookie: await cookie(), accept: "text/html,application/xhtml+xml", referer: base }, - redirect: "manual", - ...(opts.signal ? { signal: opts.signal } : {}), + /** 留样:交给调用方保存;回调出错不影响请求结果 */ + function report(e: SiteError): SiteError { + if (e.sample && deps.onFailureSample) { + try { + deps.onFailureSample(e.sample); + } catch { + // 留样失败不影响请求 + } + } + return e; + } + + /** 页面认不出来时的错误:像人机验证就报 challenge,否则 parse_error;都附页面样本 */ + function unrecognized(page: FetchedPage, what: string, kind: SiteRequestKind, parseMessage: string): SiteError { + const rule = challengeRuleOfUnrecognizedPage(page.html); + const message = rule + ? `${what}：${PAGE_LABEL[kind]}需要人机验证（Cloudflare 等），${challengeOutcomeText("unhandled")}` + : `${parseMessage}（页面样本见站点页技术信息）`; + const sample = buildFailureSample({ + kind: rule ? "challenge" : "parse_error", + requestKind: kind, + url: page.url, + httpStatus: page.status, + headers: page.headers, + message, + rule, + html: page.html, + now: http.now(), }); + return report( + new SiteError(def.id, rule ? "challenge" : "parse_error", message, { + httpStatus: page.status, + ...(rule ? { detail: { reason: "challenge", rule, httpStatus: page.status } } : {}), + sample, + }), + ); + } + + async function fetchPage( + url: string, + what: string, + opts: SiteCallOptions, + kind: SiteRequestKind, + ): Promise { + let res: Response; + try { + res = await siteFetch(http, url, { + method: "GET", + headers: { cookie: await cookie(), accept: "text/html,application/xhtml+xml", referer: base }, + redirect: "manual", + ...(opts.signal ? { signal: opts.signal } : {}), + }); + } catch (e) { + if (e instanceof SiteError && e.kind === "challenge") { + // 网络层判出的人机验证:换成带请求类型的文案,样本补上请求类型 + const outcome = typeof e.detail?.outcome === "string" ? e.detail.outcome : null; + const message = `${what}：${PAGE_LABEL[kind]}需要人机验证（Cloudflare 等），${challengeOutcomeText(outcome)}`; + throw report( + new SiteError(def.id, "challenge", message, { + ...(e.httpStatus !== undefined ? { httpStatus: e.httpStatus } : {}), + ...(e.detail ? { detail: e.detail } : {}), + ...(e.sample ? { sample: { ...e.sample, requestKind: kind, message } } : {}), + }), + ); + } + throw e; + } if (res.status >= 300 && res.status < 400) { const loc = res.headers.get("location") ?? ""; await res.body?.cancel().catch(() => {}); @@ -155,7 +234,7 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap httpStatus: res.status, }); } - return readText(http, res); + return { html: await readText(http, res), url, status: res.status, headers: res.headers }; } async function run( @@ -172,17 +251,19 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap page: Math.max(1, page) - 1, // 对外 1 起,MP 内部 0 起 }); if (!spider.canSearch()) throw new SiteError(def.id, "unsupported", `${what}:站点定义不支持该媒体类型`); - const html = await fetchPage(spider.getSearchUrl(), what, opts); - const parsed = spider.parse(html); + const kind: SiteRequestKind = keyword ? "search" : "list"; + const page$ = await fetchPage(spider.getSearchUrl(), what, opts, kind); + const parsed = spider.parse(page$.html); if (parsed.loginPage) throw new SiteError(def.id, "auth_expired", `${what}:返回登录或权限提示页`); - if (parsed.isError || !parsed.$) throw new SiteError(def.id, "parse_error", `${what}:响应为空`); + if (parsed.isError || !parsed.$) throw unrecognized(page$, what, kind, `${what}:响应为空`); const $ = parsed.$; if (parsed.torrents.length === 0) { const text = $("body").text(); const hasList = $(normalizeSelector(spider.def.torrents.list.selector)).length > 0 || $("table.torrents").length > 0; const empty = (ext.emptyMarkers ?? []).some((m) => text.includes(m)); - if (!hasList && !empty) throw new SiteError(def.id, "parse_error", `${what}:页面结构无法识别(可能改版)`); + // 认不出列表:先看是不是人机验证页(搜索页单独加盾、间歇触发),不是才报改版 + if (!hasList && !empty) throw unrecognized(page$, what, kind, `${what}:页面结构无法识别(可能改版)`); if (ext.loginSuccess && $(ext.loginSuccess).length === 0 && !empty) { throw new SiteError(def.id, "auth_expired", `${what}:未找到登录标志`); } @@ -237,18 +318,20 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap if (!/^\d+$/.test(siteReleaseId)) throw new SiteError(def.id, "not_found", "资源 ID 非法"); if (!ext.detail) throw new SiteError(def.id, "unsupported", "站点定义未配置详情页解析"); const url = new URL(`${ext.detail.path}?id=${siteReleaseId}&hit=1`, base).toString(); - const html = await fetchPage(url, `详情 ${siteReleaseId}`, opts); - const r = parseDetailPage(html, def, siteReleaseId, http.now().toISOString()); + const detailPage = await fetchPage(url, `详情 ${siteReleaseId}`, opts, "detail"); + const r = parseDetailPage(detailPage.html, def, siteReleaseId, http.now().toISOString()); if (r.kind === "login") throw new SiteError(def.id, "auth_expired", `详情 ${siteReleaseId}:返回登录页`); if (r.kind === "not_found") throw new SiteError(def.id, "not_found", `详情 ${siteReleaseId}:资源不存在`); - if (!r.detail) throw new SiteError(def.id, "parse_error", `详情 ${siteReleaseId}:页面结构无法识别`); + if (!r.detail) { + throw unrecognized(detailPage, `详情 ${siteReleaseId}`, "detail", `详情 ${siteReleaseId}:页面结构无法识别`); + } // 文件清单不在页面上(NexusPHP 点「查看列表」才 AJAX 取)→ 按需 1 次;失败只是没有文件清单 const flPath = ext.detail.page?.fileListPath; if (flPath && r.detail.page && !r.detail.page.files.length) { try { const flUrl = new URL(flPath.replace("{id}", siteReleaseId), base).toString(); - const frag = await fetchPage(flUrl, `文件清单 ${siteReleaseId}`, opts); - r.detail.page.files = filesFromFileListHtml(loadHtml(frag)); + const frag = await fetchPage(flUrl, `文件清单 ${siteReleaseId}`, opts, "detail"); + r.detail.page.files = filesFromFileListHtml(loadHtml(frag.html)); } catch (e) { if (e instanceof SiteError && e.isAuthError) throw e; } @@ -317,8 +400,8 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap const started = Date.now(); const checkedAt = http.now().toISOString(); try { - const html = await fetchPage(new URL("index.php", base).toString(), "健康检查", opts); - const v = judgeLoginPage(html, ext.loginSuccess); + const home = await fetchPage(new URL("index.php", base).toString(), "健康检查", opts, "health"); + const v = judgeLoginPage(home.html, ext.loginSuccess); const latencyMs = Date.now() - started; if (v.ok) return { siteId: def.id, status: "ok", ok: true, checkedAt, latencyMs, detail: v.detail }; return { diff --git a/packages/adapters/src/sites/spider/challenge-page.test.ts b/packages/adapters/src/sites/spider/challenge-page.test.ts new file mode 100644 index 00000000..039fc771 --- /dev/null +++ b/packages/adapters/src/sites/spider/challenge-page.test.ts @@ -0,0 +1,123 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; +import { + audiencesDefinition, + createSiteRateLimiter, + createSpiderSiteAdapter, + SiteError, + type SiteFailureSample, + staticSecretResolver, +} from "../../index.ts"; + +// 搜索页单独加盾(间歇触发):返回 200 的挑战 / 人机验证页不再报「页面结构无法识别」,报 challenge 并留样; +// 真正认不出的页面才报 parse_error(也留样)。 + +const AUD_SEARCH = readFileSync(new URL("../../../test/fixtures/audiences-search.html.txt", import.meta.url), "utf8"); +const CF_200 = `Just a moment... +
`; +/** 站点外壳里的人机验证页:有用户信息块(网络层的严格判定不认),适配器解析失败时兜底认出 */ +const SHELL_GATE = `Audiences :: 搜索 +
someone 退出
+

搜索前请完成人机验证

+
`; +const LOGIN_WITH_TURNSTILE = `Audiences :: 登录 - Powered by NexusPHP +
+
`; +/** 改版页:带上各种要脱敏的东西 */ +const SECRET = "0123456789abcdef0123456789abcdef"; +const BROKEN = `Audiences :: 种子 +
欢迎 alice_cn 邮箱 alice@example.com
+dl + + +
${"
Dune
".repeat(5000)}
`; + +function adapter(body: string, status = 200, headers: Record = {}) { + const samples: SiteFailureSample[] = []; + const fetchImpl = (async () => + new Response(body, { + status, + headers: { "content-type": "text/html; charset=utf-8", ...headers }, + })) as typeof fetch; + const a = createSpiderSiteAdapter(audiencesDefinition, { + baseUrl: "https://aud.test/", + secrets: staticSecretResolver({ [audiencesDefinition.id]: { cookie: "c_secure_uid=1; c_secure_pass=abc" } }), + limiter: createSiteRateLimiter({ minIntervalMs: 0, burst: 10 }), + fetch: fetchImpl, + onFailureSample: (s) => samples.push(s), + }); + return { a, samples }; +} + +async function err(p: Promise): Promise { + try { + await p; + } catch (e) { + expect(e).toBeInstanceOf(SiteError); + return e as SiteError; + } + throw new Error("expected SiteError"); +} + +describe("spider 搜索页人机验证", () => { + it("正常搜索页(真实样本)照常解析,不留样", async () => { + const { a, samples } = adapter(AUD_SEARCH); + const page = await a.search("药屋少女的呢喃"); + expect(page.items.length).toBeGreaterThan(0); + expect(samples).toEqual([]); + }); + + it("200 的 Cloudflare 挑战页 → challenge(网络层认出),文案说搜索页需要人机验证,留样标搜索", async () => { + const { a, samples } = adapter(CF_200, 200, { server: "cloudflare", "cf-ray": "8c-HKG" }); + const e = await err(a.search("Dune")); + expect(e.kind).toBe("challenge"); + expect(e.isAuthError).toBe(false); + expect(e.message).toContain("搜索页需要人机验证"); + expect(e.message).not.toContain("改版"); + expect(samples).toHaveLength(1); + expect(samples[0]).toMatchObject({ kind: "challenge", requestKind: "search", httpStatus: 200 }); + expect(samples[0]?.headers["cf-ray"]).toBe("8c-HKG"); + expect(samples[0]?.url).toContain("torrents.php"); + }); + + it("站点外壳里的人机验证页:网络层不认,解析失败前兜底 → challenge(不报改版)", async () => { + const { a, samples } = adapter(SHELL_GATE); + const e = await err(a.search("Dune")); + expect(e.kind).toBe("challenge"); + expect(e.message).toContain("搜索页需要人机验证"); + expect(e.detail?.rule).toContain("turnstile"); + expect(samples[0]).toMatchObject({ kind: "challenge", requestKind: "search" }); + expect(samples[0]?.body).not.toContain("29191"); + expect(samples[0]?.body).not.toContain("someone"); + }); + + it("列表页同样处理,文案按列表", async () => { + const { a, samples } = adapter(SHELL_GATE); + const e = await err(a.list(1)); + expect(e.kind).toBe("challenge"); + expect(e.message).toContain("列表页需要人机验证"); + expect(samples[0]?.requestKind).toBe("list"); + }); + + it("带 Turnstile 的登录页仍按登录失效处理,不留样", async () => { + const { a, samples } = adapter(LOGIN_WITH_TURNSTILE); + const e = await err(a.search("Dune")); + expect(e.kind).toBe("auth_expired"); + expect(samples).toEqual([]); + }); + + it("真正认不出的页面 → parse_error,留脱敏样本(≤ 64 KB)", async () => { + const { a, samples } = adapter(BROKEN, 200, { server: "nginx" }); + const e = await err(a.search("Dune")); + expect(e.kind).toBe("parse_error"); + expect(e.message).toContain("页面结构无法识别"); + expect(samples).toHaveLength(1); + const s = samples[0] as SiteFailureSample; + expect(s).toMatchObject({ kind: "parse_error", requestKind: "search", httpStatus: 200, truncated: true }); + expect(new TextEncoder().encode(s.body).byteLength).toBeLessThanOrEqual(64 * 1024); + for (const leak of [SECRET, "alice@example.com", "alice_cn", "29191", "topsecretvalue"]) { + expect(s.body).not.toContain(leak); + } + expect(s.body).toContain("download.php?id=1"); + }); +}); diff --git a/packages/adapters/src/sites/types.ts b/packages/adapters/src/sites/types.ts index 9cbe5fe7..383de115 100644 --- a/packages/adapters/src/sites/types.ts +++ b/packages/adapters/src/sites/types.ts @@ -1,5 +1,7 @@ /** 站点适配器自有类型(不依赖 @umt/contracts;W2 负责映射到 source_release / site_snapshot)。 */ +import type { SiteFailureSample } from "./failure-sample.ts"; + /** api = 专用 API spider(spiders/*,Haidan/HDDolby/RousiPro/SunnyPT/TNode/TorrentLeech/Yema) */ export type SiteKind = "nexusphp" | "mteam" | "api"; @@ -205,11 +207,13 @@ export class SiteError extends Error { readonly retryAfterMs?: number; /** 原始判定(健康检查的技术信息用) */ readonly detail?: SiteHealthDetail; + /** 失败页面样本(已脱敏;parse_error / challenge 时由适配器附上,交给 deps.onFailureSample 留样) */ + readonly sample?: SiteFailureSample; constructor( siteId: string, kind: SiteErrorKind, message: string, - opts: { httpStatus?: number; retryAfterMs?: number; detail?: SiteHealthDetail } = {}, + opts: { httpStatus?: number; retryAfterMs?: number; detail?: SiteHealthDetail; sample?: SiteFailureSample } = {}, ) { super(message); this.name = "SiteError"; @@ -218,6 +222,7 @@ export class SiteError extends Error { if (opts.httpStatus !== undefined) this.httpStatus = opts.httpStatus; if (opts.retryAfterMs !== undefined) this.retryAfterMs = opts.retryAfterMs; if (opts.detail !== undefined) this.detail = opts.detail; + if (opts.sample !== undefined) this.sample = opts.sample; } /** 凭据类错误:应立即置账号验证失败,不再重试 */ get isAuthError(): boolean { diff --git a/packages/adapters/test/fixtures/audiences-search.html.txt b/packages/adapters/test/fixtures/audiences-search.html.txt new file mode 100644 index 00000000..2b7eddd9 --- /dev/null +++ b/packages/adapters/test/fixtures/audiences-search.html.txt @@ -0,0 +1,1308 @@ + + + + + + Audiences :: 搜索结果 - 药屋少女的呢喃 - Powered by NexusPHP + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+
+ + +
+ +
+ +
+
+ + + + + + +
+
+
+ + + + +
+ + + + + + +
+
+
+
+
+
+
+ fixtureuser avatar +
+
+
个人概览
+
+ fixtureuser +
+ 无限 + 28.503 TB + 0.00 KB + + + 1,708,365.7 + + + + 477,181.6 + + + + + ↑ 14 + / + ↓ 13 + + + + + 0/0/0 + +
+
+
+
+
+ + +
+
+ + + 签到 + + + + 收件箱 + 22 + + + + 发件箱 + 0 + +
+
+
+
+
+ 		+
+
+ +
+
全局优惠生效：官组FREE
优惠开始：2026-10-04 00:00:00 (GMT+08:00)
优惠截止：2026-10-08 23:59:59 (GMT+08:00)
+
+
+
+
+
+
+
+
+ +
+
兰香如故 +AudiesCC +我不是大师 +蜘蛛侠 +仙逆 +功夫女足 +余红旧事 +凡人修仙传 +斗破苍穹 +蜘蛛侠：崭新之日 +绿灯军团 +缎带英雄 +牧神记 +镖人 +无可替代 +
+
+
+ +
+
+
+
+
免费 +
国语 +粤语 +中字 +官字组 +
官方 +原创 +DIY +首发 +应求 +
Dolby Vision +HDR10 +HDR10+ +
禁转 +限转 +零魔 +完结 +
动画 +MV +卡拉OK +LIVE现场 +演唱会 +音乐专辑 +
+
+
+
+ + +
+
+
+
+
+ + +
+
+ + + + + + + +
+
+ 当前页筛选 + + + + +
+
+
+
+
+
标题
+
+ + +
+
+
+
简介
+
+ + +
+
+
+
+
+
评分
+
+
+ 豆瓣 +
+ + - + +
+
+
+ IMDb +
+ + - + +
+
+
+
+
+ +
+
+ 烂番茄 +
+ + - + +
+
+
+ Metacritic +
+ + - + +
+
+
+
+
+
+
发布时间
+
+
+ 最小 + + 天 + + 时 + + 分 +
+
+ 最大 + + 天 + + 时 + + 分 +
+
+
+
+
大小 (GB)
+
+ + - + +
+
+
+
做种
+
+ + - + +
+
+
+
下载
+
+ + - + +
+
+
+
完成人数
+
+ + - + +
+
+
+
进度 (%)
+
+ + - + +
+
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
类型 + + + + 标题 + + + + + + + + + +
音乐Blu-ray DIY
+
Hyakka Ryoran 2025 1080p BluRay AVC LPCM 2.0-DIY@Audies Free
官方中字DIY动画禁转百花繚乱 · 幾田りら/ 《药屋少女的呢喃》第二季OP1 【原盘DIY / 歌词字幕】*奇怪的DIY增加了
11年
6月		476.16
MB		70163-
匿名
剧集Blu-ray DIY
+
The Apothecary Diaries S01 2023 1080p BluRay AVC LPCM 2.0-DIY@Audies Free
官方中字官字组DIY动画完结药屋少女的呢喃 / 薬屋のひとりごと / The Apothecary Diaries 【DIY / 双语简繁字幕】【正片全24集+特典全24集】
02年
2月		176.73
GB		120257-
匿名
+
+
\ No newline at end of file