fix(adapters): 识别 200 状态的挑战 / 人机验证页,解析失败前兜底,失败页面留样
- detectChallenge 支持页面类请求(GET + Accept text/html)的 2xx 响应:正文只先读 256 KB, 短页且没有种子表 / 用户信息块时,认 Cloudflare 挑战标题 / 脚本与 Turnstile / hCaptcha / reCAPTCHA 门页; 带登录表单的页面不算门页。非页面请求不读正文 - siteFetch 用上面的判定;挑战错误附脱敏页面样本,文案带级联结果(x-umt-challenge) - spider 适配器:认不出列表 / 详情时先看是不是人机验证页,是就报 challenge(「搜索页需要人机验证」), 不再报改版;parse_error / challenge 都经 deps.onFailureSample 交出脱敏样本(前 64 KB,去掉个人信息块、 passkey / authkey / uid / 用户名 / Cookie / 邮箱) - classifyChallengePage 也认门页(浏览器过挑战时不再把门页当已通过) Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
4fa001750b
commit
1779e1a0f2
11 files changed
+2078
-34
No files matched your search
@@ -1,5 +1,14 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { challengeRuleOfHtml, classifyChallengePage, detectChallenge } from "./challenge.ts";
|
||||
import {
|
||||
CHALLENGE_OUTCOME_HEADER,
|
||||
CHALLENGE_PEEK_BYTES,
|
||||
challengeRuleOfHtml,
|
||||
challengeRuleOfUnrecognizedPage,
|
||||
classifyChallengePage,
|
||||
detectChallenge,
|
||||
isPageRequest,
|
||||
} from "./challenge.ts";
|
||||
import { siteFetch } from "./http.ts";
|
||||
import { createSiteRateLimiter } from "./rate-limit.ts";
|
||||
import { SiteError } from "./types.ts";
|
||||
@@ -114,3 +123,145 @@ describe("classifyChallengePage", () => {
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ── 200 状态的挑战页 / 人机验证门页(搜索页单独加盾、间歇触发)──
|
||||
|
||||
const fixture = (rel: string) => readFileSync(new URL(rel, import.meta.url), "utf8");
|
||||
/** 观众真实搜索页(脱敏、删减到 4 行,`<table class="torrents torrents-table" id="torrenttable">`) */
|
||||
const AUD_SEARCH = fixture("../../test/fixtures/audiences-search.html.txt");
|
||||
/** 观众真实列表页(DET-2 样本,已脱敏) */
|
||||
const AUD_LIST = fixture("../../test/differential/inputs/indexer/rl-audiences-list.html.txt");
|
||||
/** Turnstile 门页:站点自己出的「请完成人机验证」短页,没有种子表、没有用户信息块 */
|
||||
const TURNSTILE_GATE = `<!DOCTYPE html><html><head><title>Audiences</title>
|
||||
<script src="https://challenges.cloudflare.com/turnstile/v0/api.js" async defer></script></head>
|
||||
<body><form method="post" action="/verify"><p>请完成人机验证后继续</p>
|
||||
<div class="cf-turnstile" data-sitekey="0x4AAAAAAA"></div></form></body></html>`;
|
||||
const HCAPTCHA_GATE = `<html><head><title>Security check</title><script src="https://hcaptcha.com/1/api.js"></script></head>
|
||||
<body><div class="h-captcha" data-sitekey="x"></div></body></html>`;
|
||||
/** 登录页本身带 Turnstile:交给登录判定,不算门页 */
|
||||
const LOGIN_WITH_TURNSTILE = `<html><head><title>Audiences :: 登录 - Powered by NexusPHP</title>
|
||||
<script src="https://challenges.cloudflare.com/turnstile/v0/api.js"></script></head><body>
|
||||
<form method="post" action="takelogin.php"><input type="text" name="username"><input type="password" name="password">
|
||||
<div class="cf-turnstile" data-sitekey="0x4AAAAAAA"></div></form></body></html>`;
|
||||
/** 正常页面里注入了 challenge-platform 脚本,侧栏还有个签到 Turnstile:有种子表 / 用户信息块 → 不是门页 */
|
||||
const NORMAL_WITH_WIDGET = `<html><head><title>Audiences :: 种子</title>
|
||||
<script src="/cdn-cgi/challenge-platform/scripts/jsd/main.js"></script></head><body>
|
||||
<div id="info_block"><a href="userdetails.php?id=1">me</a> <a href="logout.php">退出</a></div>
|
||||
<div class="cf-turnstile" data-sitekey="x"></div>
|
||||
<table class="torrents" id="torrenttable"><tr><td>row</td></tr></table></body></html>`;
|
||||
|
||||
const page200 = (body: string, headers: Record<string, string> = {}) =>
|
||||
new Response(body, { status: 200, headers: { "content-type": "text/html; charset=utf-8", ...headers } });
|
||||
|
||||
describe("detectChallenge:页面请求的 200 响应", () => {
|
||||
it("200 的 Cloudflare 挑战页(标题 / 脚本)→ 挑战", async () => {
|
||||
expect((await detectChallenge(page200(CF_PAGE), { page: true })).rule).toContain("Just a moment");
|
||||
const noTitle = `<html><head><title>x</title></head><body><script>window._cf_chl_opt={}</script></body></html>`;
|
||||
expect((await detectChallenge(page200(noTitle), { page: true })).rule).toContain("cf_chl_opt");
|
||||
});
|
||||
|
||||
it("200 的 Turnstile / hCaptcha 门页 → 挑战,正文原样交还", async () => {
|
||||
const a = await detectChallenge(page200(TURNSTILE_GATE), { page: true });
|
||||
expect(a.rule).toMatch(/^gate\*=/);
|
||||
expect(a.html).toBe(TURNSTILE_GATE);
|
||||
expect(await a.res.text()).toBe(TURNSTILE_GATE);
|
||||
expect((await detectChallenge(page200(HCAPTCHA_GATE), { page: true })).rule).toContain("hcaptcha");
|
||||
});
|
||||
|
||||
it("正常搜索页 / 列表页 / 带 Turnstile 的登录页 / 注入脚本的正常页 → 不误判", async () => {
|
||||
for (const body of [AUD_SEARCH, AUD_LIST, LOGIN_WITH_TURNSTILE, NORMAL_WITH_WIDGET, NORMAL]) {
|
||||
const r = await detectChallenge(page200(body), { page: true });
|
||||
expect(r.rule).toBeNull();
|
||||
expect(await r.res.text()).toBe(body);
|
||||
}
|
||||
});
|
||||
|
||||
it("不是页面请求 / 不是 HTML:不读正文(原响应对象直接返回)", async () => {
|
||||
const res = page200(CF_PAGE);
|
||||
const r = await detectChallenge(res);
|
||||
expect(r.rule).toBeNull();
|
||||
expect(r.res).toBe(res);
|
||||
expect(res.bodyUsed).toBe(false);
|
||||
const json = new Response('{"title":"Just a moment"}', {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
const j = await detectChallenge(json, { page: true });
|
||||
expect(j.res).toBe(json);
|
||||
});
|
||||
|
||||
it("正文超过读取上限:只读开头不判门页,其余照常流给调用方", async () => {
|
||||
const big = `${TURNSTILE_GATE}${"<!-- pad -->".repeat(Math.ceil((CHALLENGE_PEEK_BYTES * 2) / 12))}`;
|
||||
const r = await detectChallenge(page200(big), { page: true });
|
||||
expect(r.rule).toBeNull();
|
||||
expect(await r.res.text()).toBe(big);
|
||||
});
|
||||
|
||||
it("isPageRequest:GET + Accept text/html", () => {
|
||||
expect(isPageRequest("GET", "text/html,application/xhtml+xml")).toBe(true);
|
||||
expect(isPageRequest(undefined, "text/html")).toBe(true);
|
||||
expect(isPageRequest("POST", "text/html")).toBe(false);
|
||||
expect(isPageRequest("GET", "application/json")).toBe(false);
|
||||
expect(isPageRequest("GET", null)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("siteFetch:200 门页 → challenge 并附脱敏样本", () => {
|
||||
const url = "https://audiences.example/torrents.php?search=x&passkey=0123456789abcdef0123456789abcdef";
|
||||
const html = { accept: "text/html,application/xhtml+xml" };
|
||||
|
||||
it("页面请求遇 200 门页 → challenge;样本 URL 脱敏、带关键响应头;级联结果进文案", async () => {
|
||||
let err: unknown;
|
||||
try {
|
||||
await siteFetch(
|
||||
ctx(
|
||||
reply(200, TURNSTILE_GATE, {
|
||||
server: "cloudflare",
|
||||
"cf-ray": "abc-HKG",
|
||||
[CHALLENGE_OUTCOME_HEADER]: "needs_human",
|
||||
}),
|
||||
),
|
||||
url,
|
||||
{ headers: html },
|
||||
);
|
||||
} catch (e) {
|
||||
err = e;
|
||||
}
|
||||
const se = err as SiteError;
|
||||
expect(se).toBeInstanceOf(SiteError);
|
||||
expect(se.kind).toBe("challenge");
|
||||
expect(se.message).toContain("原站视图");
|
||||
expect(se.detail).toMatchObject({ reason: "challenge", outcome: "needs_human", httpStatus: 200 });
|
||||
expect(se.sample).toMatchObject({
|
||||
kind: "challenge",
|
||||
httpStatus: 200,
|
||||
headers: { server: "cloudflare", "cf-ray": "abc-HKG" },
|
||||
});
|
||||
expect(se.sample?.url).not.toContain("0123456789abcdef");
|
||||
expect(se.sample?.body).toContain("cf-turnstile");
|
||||
});
|
||||
|
||||
it("同一页面不是页面请求(没要 HTML)→ 照常返回", async () => {
|
||||
const res = await siteFetch(ctx(reply(200, TURNSTILE_GATE)), url);
|
||||
expect(await res.text()).toBe(TURNSTILE_GATE);
|
||||
});
|
||||
});
|
||||
|
||||
describe("challengeRuleOfUnrecognizedPage(解析失败时的兜底)", () => {
|
||||
it("站点外壳里的人机验证页(有用户信息块、很长)也认;登录表单不算", () => {
|
||||
const shell = `<html><head><title>Audiences</title></head><body><div id="info_block"><a href="logout.php">退出</a></div>
|
||||
<div>${"x".repeat(150_000)}</div><p>请完成人机验证</p><div class="cf-turnstile"></div></body></html>`;
|
||||
expect(challengeRuleOfUnrecognizedPage(shell)).toContain("cf-turnstile");
|
||||
expect(challengeRuleOfUnrecognizedPage("<html><body><p>搜索需要先完成人机验证</p></body></html>")).toBe(
|
||||
"text*=人机验证",
|
||||
);
|
||||
expect(challengeRuleOfUnrecognizedPage(LOGIN_WITH_TURNSTILE)).toBeNull();
|
||||
expect(challengeRuleOfUnrecognizedPage("<html><body><div class='new-layout'>改版</div></body></html>")).toBeNull();
|
||||
});
|
||||
|
||||
it("classifyChallengePage 也认门页:Turnstile → interactive,hCaptcha → captcha", () => {
|
||||
expect(classifyChallengePage({ html: TURNSTILE_GATE, text: "请完成人机验证后继续" }).kind).toBe("interactive");
|
||||
expect(classifyChallengePage({ html: HCAPTCHA_GATE, text: "" }).kind).toBe("captcha");
|
||||
expect(classifyChallengePage({ html: LOGIN_WITH_TURNSTILE, text: "登录" }).kind).toBe("none");
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,6 @@
|
||||
// CF-2 Cloudflare 挑战识别(docs/design/cloudflare-handling.md)。
|
||||
// - HTTP 端:响应头 `cf-mitigated: challenge`(任何状态码),或 403 / 503 / 429 且正文是挑战页 → 挑战,不是凭据错误;
|
||||
// 页面类请求(GET + Accept text/html)返回 200 也可能是挑战页 / 人机验证门页(Turnstile、hCaptcha、reCAPTCHA),短页且没有站点正常内容才算;
|
||||
// - 浏览器端:挑战页分三类——自动(JS 检查,等几秒自己过)、可点(「我是人类」复选框 / Turnstile)、真验证码(选图 / 输码)。
|
||||
// 只用正则,不解析 DOM:HTTP 端要在限流器之后快速判断,浏览器端拿到的是 outerHTML + innerText 样本。
|
||||
|
||||
@@ -27,21 +28,168 @@ export function challengeRuleOfHeaders(headers: Headers): string | null {
|
||||
return headers.get("cf-mitigated")?.trim().toLowerCase() === "challenge" ? "header cf-mitigated=challenge" : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 站点请求经服务端 Cloudflare 级联后仍是挑战时,级联在返回的响应上加这个头说明处理结果
|
||||
* (needs_human / failed / throttled / no_browser / no_account),适配器据此写错误文案;
|
||||
* unhandled = 网络层没认出、适配器解析时才认出的验证页(级联没跑)。
|
||||
*/
|
||||
export const CHALLENGE_OUTCOME_HEADER = "x-umt-challenge";
|
||||
|
||||
/** 人机验证错误文案的后半句:自动处理到哪一步、要不要人 */
|
||||
export function challengeOutcomeText(outcome: string | null | undefined): string {
|
||||
switch (outcome) {
|
||||
case "needs_human":
|
||||
return "自动处理没能通过，需要你在原站视图里过一次验证";
|
||||
case "failed":
|
||||
return "已自动尝试用浏览器通过，没有成功";
|
||||
case "throttled":
|
||||
return "30 分钟内已自动尝试过，之后会再试";
|
||||
case "no_browser":
|
||||
return "没有可用的浏览器，无法自动通过";
|
||||
case "no_account":
|
||||
return "站点没有已登录的账号，不自动处理";
|
||||
case "unhandled":
|
||||
return "这类页面没有自动处理，可以在原站视图里过一次验证";
|
||||
default:
|
||||
return "会自动尝试用浏览器通过";
|
||||
}
|
||||
}
|
||||
|
||||
/** 这些状态码的响应才去看正文(Cloudflare 挑战页是 403 / 503,少数 429) */
|
||||
export const CHALLENGE_STATUSES: readonly number[] = [403, 429, 503];
|
||||
|
||||
/** 判定时最多先读这么多正文;超出部分不缓冲,照常流给调用方。挑战 / 人机验证门页都远小于这个 */
|
||||
export const CHALLENGE_PEEK_BYTES = 256 * 1024;
|
||||
|
||||
/** 页面类请求(GET + Accept text/html):只有这类请求的 2xx 响应才检查正文 */
|
||||
export function isPageRequest(method: string | null | undefined, accept: string | null | undefined): boolean {
|
||||
return (method ?? "GET").toUpperCase() === "GET" && /text\/html|application\/xhtml/i.test(accept ?? "");
|
||||
}
|
||||
|
||||
/** 人机验证组件:Turnstile / hCaptcha / reCAPTCHA(只认脚本地址与组件 class,不认文案) */
|
||||
const GATE_WIDGET =
|
||||
/challenges\.cloudflare\.com\/turnstile|class="[^"]*\bcf-turnstile\b|hcaptcha\.com\/1\/api|class="[^"]*\bh-captcha\b|google\.com\/recaptcha|recaptcha\/api\.js|class="[^"]*\bg-recaptcha\b/i;
|
||||
/** 站点正常内容的痕迹:种子表、已登录的用户信息块(NexusPHP 等)。出现任何一个就不是门页 */
|
||||
const SITE_CONTENT =
|
||||
/id="torrenttable"|<table[^>]*class="[^"]*\btorrents\b|logout\.php|userdetails\.php\?id=|usercp\.php|id="info_block"/i;
|
||||
/** 登录表单(登录页本身也可能带 Turnstile,交给登录 / 失效判定,不算门页) */
|
||||
const LOGIN_FORM = /type=["']?password|takelogin\.php/i;
|
||||
/** 门页都很短;超过这个长度的页面不按门页判 */
|
||||
const GATE_MAX_CHARS = 100_000;
|
||||
/** 页面文字里的人机验证提示(只在页面已经认不出来时用) */
|
||||
const HUMAN_TEXT = /人机验证|verify you are human|确认您是真人|验证您是真人|checking your browser/i;
|
||||
|
||||
/**
|
||||
* 判定一个站点响应是不是挑战。正文只在状态码可能是挑战时读(读过的正文重新包成 Response 返回,调用方照常消费)。
|
||||
* 返回 { res, rule }:rule 为 null 表示不是挑战。
|
||||
* 人机验证门页:页面上有 Turnstile / hCaptcha / reCAPTCHA,且没有登录表单、没有站点正常内容、页面很短。
|
||||
* 命中返回规则说明,否则 null。
|
||||
*/
|
||||
export async function detectChallenge(res: Response): Promise<{ res: Response; rule: string | null }> {
|
||||
export function challengeGateRule(html: string | null | undefined): string | null {
|
||||
if (!html || html.length > GATE_MAX_CHARS) return null;
|
||||
const w = GATE_WIDGET.exec(html);
|
||||
if (!w || LOGIN_FORM.test(html) || SITE_CONTENT.test(html)) return null;
|
||||
return `gate*=${w[0].slice(0, 40)}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* 2xx 页面是不是挑战 / 人机验证页。比非 2xx 严:正文必须完整读到(短页)、没有站点正常内容,
|
||||
* 再看挑战标题 / 脚本特征或人机验证门页。complete = 正文是否已全部读到。
|
||||
*/
|
||||
export function challengeRuleOfOkPage(html: string, complete = true): string | null {
|
||||
if (!complete || !html || html.length > GATE_MAX_CHARS || SITE_CONTENT.test(html)) return null;
|
||||
return challengeRuleOfHtml(html) ?? challengeGateRule(html);
|
||||
}
|
||||
|
||||
/**
|
||||
* 页面已经解析不出来(列表 / 详情认不出结构)时的兜底:像挑战 / 人机验证就返回规则说明。
|
||||
* 比 challengeRuleOfOkPage 宽:不要求短页、也认页面文字里的「人机验证」提示,但带登录表单的页面不算。
|
||||
*/
|
||||
export function challengeRuleOfUnrecognizedPage(html: string | null | undefined): string | null {
|
||||
if (!html) return null;
|
||||
const head = html.slice(0, 300_000);
|
||||
const cf = challengeRuleOfHtml(head);
|
||||
if (cf) return cf;
|
||||
if (LOGIN_FORM.test(head)) return null;
|
||||
const w = GATE_WIDGET.exec(head);
|
||||
if (w) return `gate*=${w[0].slice(0, 40)}`;
|
||||
const t = HUMAN_TEXT.exec(head.replace(/<script[\s\S]*?<\/script>|<style[\s\S]*?<\/style>/gi, " "));
|
||||
return t ? `text*=${t[0]}` : null;
|
||||
}
|
||||
|
||||
function concatBytes(chunks: Uint8Array[], total: number): Uint8Array<ArrayBuffer> {
|
||||
const out = new Uint8Array(total);
|
||||
let off = 0;
|
||||
for (const c of chunks) {
|
||||
out.set(c, off);
|
||||
off += c.byteLength;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** 先读最多 max 字节;返回读到的部分、是否读完,以及一个内容不变的新响应(没读完的部分接着原流) */
|
||||
async function peekBody(
|
||||
res: Response,
|
||||
max: number,
|
||||
): Promise<{ head: Uint8Array<ArrayBuffer>; complete: boolean; res: Response }> {
|
||||
const reader = (res.body as ReadableStream<Uint8Array>).getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
let total = 0;
|
||||
let complete = false;
|
||||
while (total < max) {
|
||||
const r = await reader.read();
|
||||
if (r.done) {
|
||||
complete = true;
|
||||
break;
|
||||
}
|
||||
chunks.push(r.value);
|
||||
total += r.value.byteLength;
|
||||
}
|
||||
const head = concatBytes(chunks, total);
|
||||
const init = { status: res.status, statusText: res.statusText, headers: res.headers };
|
||||
if (complete) return { head, complete, res: new Response(head, init) };
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
start(c) {
|
||||
c.enqueue(head);
|
||||
},
|
||||
async pull(c) {
|
||||
const r = await reader.read();
|
||||
if (r.done) c.close();
|
||||
else c.enqueue(r.value);
|
||||
},
|
||||
cancel(reason) {
|
||||
return reader.cancel(reason);
|
||||
},
|
||||
});
|
||||
return { head, complete, res: new Response(stream, init) };
|
||||
}
|
||||
|
||||
/**
|
||||
* 判定一个站点响应是不是挑战。
|
||||
* - 响应头 `cf-mitigated: challenge` → 挑战(不读正文);
|
||||
* - 403 / 429 / 503 → 读正文看挑战页特征;
|
||||
* - opts.page(页面类请求)且 2xx 的 HTML → 读正文看挑战页 / 人机验证门页(challengeRuleOfOkPage);
|
||||
* - 其它情况不读正文,零开销。
|
||||
* 正文最多先读 CHALLENGE_PEEK_BYTES,读过的部分重新包进返回的响应,调用方照常消费。
|
||||
* 返回 { res, rule, html }:rule 为 null 表示不是挑战;html 是读到的正文开头(没读为 null,留样用)。
|
||||
*/
|
||||
export async function detectChallenge(
|
||||
res: Response,
|
||||
opts: { page?: boolean } = {},
|
||||
): Promise<{ res: Response; rule: string | null; html: string | null }> {
|
||||
const byHeader = challengeRuleOfHeaders(res.headers);
|
||||
if (byHeader) return { res, rule: byHeader };
|
||||
if (!CHALLENGE_STATUSES.includes(res.status) || !res.body) return { res, rule: null };
|
||||
const bytes = new Uint8Array(await res.arrayBuffer());
|
||||
const html = new TextDecoder("utf-8").decode(bytes.subarray(0, 300_000));
|
||||
const copy = new Response(bytes, { status: res.status, statusText: res.statusText, headers: res.headers });
|
||||
return { res: copy, rule: challengeRuleOfHtml(html) };
|
||||
if (byHeader) return { res, rule: byHeader, html: null };
|
||||
const suspect = CHALLENGE_STATUSES.includes(res.status);
|
||||
const okPage =
|
||||
!!opts.page &&
|
||||
res.status >= 200 &&
|
||||
res.status < 300 &&
|
||||
res.status !== 204 &&
|
||||
res.status !== 205 &&
|
||||
/html|^$/i.test(res.headers.get("content-type")?.trim() ?? "");
|
||||
if ((!suspect && !okPage) || !res.body) return { res, rule: null, html: null };
|
||||
const peek = await peekBody(res, CHALLENGE_PEEK_BYTES);
|
||||
const html = new TextDecoder("utf-8").decode(peek.head);
|
||||
const rule = suspect ? challengeRuleOfHtml(html) : challengeRuleOfOkPage(html, peek.complete);
|
||||
return { res: peek.res, rule, html };
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -68,6 +216,7 @@ export function classifyChallengePage(page: { title?: string | null; html?: stri
|
||||
const rule =
|
||||
(CHALLENGE_TITLE.test(title) ? `title=${title.slice(0, 40)}` : null) ??
|
||||
challengeRuleOfHtml(html) ??
|
||||
challengeGateRule(html) ??
|
||||
(/verify you are human|确认您是真人|验证您是真人|checking (?:if the site connection is secure|your browser)/i.test(
|
||||
text,
|
||||
)
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { SiteFailureSample } from "./failure-sample.ts";
|
||||
import type { SiteHttpContext } from "./http.ts";
|
||||
import { createSiteRateLimiter, type SiteRateLimiter } from "./rate-limit.ts";
|
||||
import type { SecretResolver } from "./types.ts";
|
||||
@@ -12,6 +13,8 @@ export interface SiteAdapterDeps {
|
||||
userAgent?: string;
|
||||
/** 覆盖站点基址(测试指向 mock) */
|
||||
baseUrl?: string;
|
||||
/** 请求报 parse_error / challenge 时的页面样本(已脱敏);调用方自行保存,抛错不影响请求结果 */
|
||||
onFailureSample?: (sample: SiteFailureSample) => void;
|
||||
}
|
||||
|
||||
export function buildHttpContext(
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { buildFailureSample, FAILURE_SAMPLE_MAX_BYTES, redactPageHtml } from "./failure-sample.ts";
|
||||
|
||||
// 失败留样的脱敏:passkey / authkey / uid / 用户名 / Cookie / 邮箱 / 个人信息块都不进样本;截到 64 KB。
|
||||
|
||||
const AUD_SEARCH = readFileSync(new URL("../../test/fixtures/audiences-search.html.txt", import.meta.url), "utf8");
|
||||
|
||||
describe("redactPageHtml", () => {
|
||||
it("下载链接密钥、表单 / 脚本里的密钥、Cookie、邮箱、用户链接都抹掉,普通内容保留", () => {
|
||||
const key = "0123456789abcdef0123456789abcdef";
|
||||
const html = `<a href="download.php?id=9&passkey=${key}">dl</a>
|
||||
<a href="https://x.test/rss.php?uid=777&rsskey=abc123def">rss</a>
|
||||
<input name="authkey" value="zzzzzzzz9999"><script>var token = "tok_ABCDEFGH"; c_secure_pass=hunter2xyz;</script>
|
||||
<p>联系 [email protected]</p><a href="userdetails.php?id=4242"><b>bobby</b></a> UID 4242
|
||||
<td class="rowfollow">Dune.Part.Two.2024</td>`;
|
||||
const out = redactPageHtml(html);
|
||||
for (const leak of [
|
||||
key,
|
||||
"abc123def",
|
||||
"zzzzzzzz9999",
|
||||
"tok_ABCDEFGH",
|
||||
"hunter2xyz",
|
||||
"[email protected]",
|
||||
"4242",
|
||||
"bobby",
|
||||
"777",
|
||||
]) {
|
||||
expect(out).not.toContain(leak);
|
||||
}
|
||||
expect(out).toContain("download.php?id=9");
|
||||
expect(out).toContain("Dune.Part.Two.2024");
|
||||
});
|
||||
|
||||
it("真实搜索页:个人信息块整块删掉,种子表保留", () => {
|
||||
const out = redactPageHtml(AUD_SEARCH);
|
||||
expect(AUD_SEARCH).toContain("fixtureuser");
|
||||
expect(out).not.toContain("fixtureuser");
|
||||
expect(out).toContain("<!-- #info_block 已删除 -->");
|
||||
expect(out).toContain('id="torrenttable"');
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildFailureSample", () => {
|
||||
it("截到 64 KB、URL 脱敏、只留关键响应头", () => {
|
||||
const s = buildFailureSample({
|
||||
kind: "parse_error",
|
||||
requestKind: "search",
|
||||
url: "https://aud.test/torrents.php?search=x&passkey=secretsecret&uid=5",
|
||||
httpStatus: 200,
|
||||
headers: new Headers({
|
||||
server: "cloudflare",
|
||||
"cf-ray": "1-HKG",
|
||||
"set-cookie": "a=b",
|
||||
"cf-mitigated": "challenge",
|
||||
}),
|
||||
message: "搜索:页面结构无法识别",
|
||||
html: AUD_SEARCH,
|
||||
now: new Date("2026-10-06T00:00:00Z"),
|
||||
});
|
||||
expect(new TextEncoder().encode(s.body).byteLength).toBeLessThanOrEqual(FAILURE_SAMPLE_MAX_BYTES);
|
||||
expect(s.truncated).toBe(true);
|
||||
expect(s.url).not.toContain("secretsecret");
|
||||
expect(s.url).not.toContain("uid=5");
|
||||
expect(s.headers).toEqual({ server: "cloudflare", "cf-ray": "1-HKG", "cf-mitigated": "challenge" });
|
||||
expect(s.capturedAt).toBe("2026-10-06T00:00:00.000Z");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,124 @@
|
||||
import { redactString } from "@umt/contracts";
|
||||
import { redactText, redactUrl } from "../rss/redact.ts";
|
||||
|
||||
// 站点请求失败留样:请求报「页面结构无法识别」(parse_error)或「人机验证」(challenge)时,留一份脱敏后的页面开头,
|
||||
// 站点页「技术信息」里可看 / 下载,用来区分是站点改版还是被人机验证拦了(间歇触发的那一刻页面不会再出现)。
|
||||
|
||||
/** 页面样本最多保留的字节数 */
|
||||
export const FAILURE_SAMPLE_MAX_BYTES = 64 * 1024;
|
||||
|
||||
/** 请求类型:搜索 / 列表 / 详情 / 健康检测 / 其它 */
|
||||
export type SiteRequestKind = "search" | "list" | "detail" | "health" | "other";
|
||||
|
||||
/** 留样只记这几个响应头 */
|
||||
const KEPT_HEADERS = ["server", "cf-mitigated", "cf-ray", "content-type"] as const;
|
||||
|
||||
export interface SiteFailureSample {
|
||||
kind: "parse_error" | "challenge";
|
||||
requestKind: SiteRequestKind;
|
||||
/** 已脱敏 */
|
||||
url: string;
|
||||
httpStatus: number | null;
|
||||
headers: Record<string, string>;
|
||||
capturedAt: string;
|
||||
/** 报出去的错误文案 */
|
||||
message: string;
|
||||
/** 命中的挑战 / 门页规则 */
|
||||
rule: string | null;
|
||||
/** 脱敏后的页面开头(≤ FAILURE_SAMPLE_MAX_BYTES 字节) */
|
||||
body: string;
|
||||
/** 页面是否被截断 */
|
||||
truncated: boolean;
|
||||
}
|
||||
|
||||
/** URL 里除了通用凭据参数,还要抹掉的用户标识参数 */
|
||||
const EXTRA_KEYS = ["uid", "userid", "user_id"];
|
||||
const EMAIL_RE = /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g;
|
||||
/** NexusPHP passkey / authkey 等:32 位以上十六进制串(info hash 是 40 位,也一并抹掉,样本不需要) */
|
||||
const HEX_KEY_RE = /\b[a-f0-9]{32,64}\b/gi;
|
||||
/** Cookie 名=值(页面脚本里偶尔会出现) */
|
||||
const COOKIE_PAIR_RE =
|
||||
/\b(c_secure_[a-z_]+|cf_clearance|__cf_bm|phpsessid|PHPSESSID|session(?:id)?|remember_[a-z0-9_]+)=([^;\s"'<>&]+)/gi;
|
||||
/** 表单 / 脚本里的密钥字段:name="passkey" value="…"、passkey: "…" */
|
||||
const SECRET_FIELD_RE =
|
||||
/\b(pass[_-]?key|auth[_-]?key|torrent[_-]?pass|rss[_-]?key|api[_-]?key|token|csrf[_-]?token|secret)(["']?\s*(?:[:=]|value=)\s*["']?)([^"'\s<>&;,]{6,})/gi;
|
||||
/** 用户页链接里的用户 id 与链接文字(用户名) */
|
||||
const USER_LINK_RE = /(userdetails\.php\?id=)\d+/gi;
|
||||
const USER_NAME_RE = /(<a\b[^>]*userdetails\.php[^>]*>)((?:\s*<[^>]+>)*)([^<]+)/gi;
|
||||
|
||||
/** 页面里的「UID 12345」 */
|
||||
const UID_TEXT_RE = /\bUID\s*[:：]?\s*\d+/gi;
|
||||
|
||||
/**
|
||||
* 删掉 id 为 id 的整个元素(按同名标签计数找配对的结束标签);NexusPHP 的 `#info_block` 是当前用户的个人信息块
|
||||
* (用户名、分享率、魔力…),样本里不需要。找不到原样返回。
|
||||
*/
|
||||
function dropElementById(html: string, id: string): string {
|
||||
const open = new RegExp(`<([a-z][a-z0-9]*)\\b[^>]*\\bid=["']?${id}\\b[^>]*>`, "i").exec(html);
|
||||
if (!open) return html;
|
||||
const tag = (open[1] as string).toLowerCase();
|
||||
const re = new RegExp(`<(/?)${tag}\\b[^>]*>`, "gi");
|
||||
re.lastIndex = open.index + open[0].length;
|
||||
let depth = 1;
|
||||
for (let m = re.exec(html); m; m = re.exec(html)) {
|
||||
depth += m[1] ? -1 : 1;
|
||||
if (depth === 0) return `${html.slice(0, open.index)}<!-- #${id} 已删除 -->${html.slice(m.index + m[0].length)}`;
|
||||
}
|
||||
return `${html.slice(0, open.index)}<!-- #${id} 已删除 -->`;
|
||||
}
|
||||
|
||||
/** 页面文本脱敏:个人信息块、凭据参数、用户 id / 用户名、Cookie、邮箱、密钥样式的十六进制串 */
|
||||
export function redactPageHtml(html: string): string {
|
||||
return redactString(
|
||||
redactText(dropElementById(html, "info_block"), EXTRA_KEYS)
|
||||
.replace(UID_TEXT_RE, "UID [REDACTED]")
|
||||
.replace(USER_NAME_RE, "$1$2[USER]")
|
||||
.replace(USER_LINK_RE, "$1[REDACTED]")
|
||||
.replace(COOKIE_PAIR_RE, "$1=[REDACTED]")
|
||||
.replace(SECRET_FIELD_RE, "$1$2[REDACTED]")
|
||||
.replace(EMAIL_RE, "[EMAIL]")
|
||||
.replace(HEX_KEY_RE, "[HEX]"),
|
||||
);
|
||||
}
|
||||
|
||||
/** 先脱敏、再按字节截到上限(截断处的半个字符丢掉) */
|
||||
function clip(text: string, maxBytes: number): { body: string; truncated: boolean } {
|
||||
const bytes = new TextEncoder().encode(text);
|
||||
if (bytes.byteLength <= maxBytes) return { body: text, truncated: false };
|
||||
return { body: new TextDecoder().decode(bytes.subarray(0, maxBytes)).replace(/�$/, ""), truncated: true };
|
||||
}
|
||||
|
||||
export function buildFailureSample(input: {
|
||||
kind: SiteFailureSample["kind"];
|
||||
requestKind: SiteRequestKind;
|
||||
url: string;
|
||||
httpStatus?: number | null;
|
||||
headers?: Headers | null;
|
||||
message: string;
|
||||
rule?: string | null;
|
||||
html: string | null | undefined;
|
||||
now: Date;
|
||||
}): SiteFailureSample {
|
||||
const headers: Record<string, string> = {};
|
||||
for (const k of KEPT_HEADERS) {
|
||||
const v = input.headers?.get(k);
|
||||
if (v) headers[k] = v.slice(0, 200);
|
||||
}
|
||||
// 脱敏只看前 4 倍上限(截断前先脱敏,避免截断把密钥切成两半漏掉)
|
||||
const { body, truncated } = clip(
|
||||
redactPageHtml((input.html ?? "").slice(0, FAILURE_SAMPLE_MAX_BYTES * 4)),
|
||||
FAILURE_SAMPLE_MAX_BYTES,
|
||||
);
|
||||
return {
|
||||
kind: input.kind,
|
||||
requestKind: input.requestKind,
|
||||
url: redactUrl(input.url, EXTRA_KEYS),
|
||||
httpStatus: input.httpStatus ?? null,
|
||||
headers,
|
||||
capturedAt: input.now.toISOString(),
|
||||
message: input.message,
|
||||
rule: input.rule ?? null,
|
||||
body,
|
||||
truncated: truncated || (input.html?.length ?? 0) > FAILURE_SAMPLE_MAX_BYTES * 4,
|
||||
};
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
import { redactText } from "../rss/redact.ts";
|
||||
import { detectChallenge } from "./challenge.ts";
|
||||
import { CHALLENGE_OUTCOME_HEADER, challengeOutcomeText, detectChallenge, isPageRequest } from "./challenge.ts";
|
||||
import { buildFailureSample } from "./failure-sample.ts";
|
||||
import type { SiteRateLimiter } from "./rate-limit.ts";
|
||||
import { SiteError } from "./types.ts";
|
||||
|
||||
@@ -21,7 +22,8 @@ function retryAfterMs(res: Response, now: Date): number | undefined {
|
||||
}
|
||||
|
||||
/**
|
||||
* 站点请求:先取站点令牌,再发请求;Cloudflare 挑战(`cf-mitigated: challenge`,或 403 / 503 / 429 的挑战页)→ `challenge`
|
||||
* 站点请求:先取站点令牌,再发请求;Cloudflare 挑战(`cf-mitigated: challenge`,或 403 / 503 / 429 的挑战页,
|
||||
* 或页面请求返回 200 的挑战页 / 人机验证门页)→ `challenge`(附脱敏页面样本)
|
||||
* (不是凭据错误,CF-2);其余 429/503 统一映射并顺延限流器。
|
||||
* 返回的 Response 由调用方消费。错误消息只含脱敏后的 URL。
|
||||
*/
|
||||
@@ -44,13 +46,27 @@ export async function siteFetch(
|
||||
const reason = timeout.aborted ? "超时" : (e as Error).message;
|
||||
throw new SiteError(ctx.siteId, "network", redactText(`请求 ${safeUrl} 失败:${reason}`));
|
||||
}
|
||||
const ch = await detectChallenge(res);
|
||||
// 页面类请求的 200 响应也看正文:返回 200 的挑战页 / 人机验证门页(只读开头,有上限)
|
||||
const ch = await detectChallenge(res, { page: isPageRequest(init.method, headers.get("accept")) });
|
||||
res = ch.res;
|
||||
if (ch.rule) {
|
||||
await res.body?.cancel().catch(() => {});
|
||||
throw new SiteError(ctx.siteId, "challenge", `请求 ${safeUrl}:站点要求人机验证(Cloudflare)`, {
|
||||
const outcome = res.headers.get(CHALLENGE_OUTCOME_HEADER);
|
||||
const message = `请求 ${safeUrl}：站点要求人机验证（Cloudflare 等），${challengeOutcomeText(outcome)}`;
|
||||
throw new SiteError(ctx.siteId, "challenge", message, {
|
||||
httpStatus: res.status,
|
||||
detail: { reason: "challenge", rule: ch.rule, httpStatus: res.status },
|
||||
detail: { reason: "challenge", rule: ch.rule, httpStatus: res.status, ...(outcome ? { outcome } : {}) },
|
||||
sample: buildFailureSample({
|
||||
kind: "challenge",
|
||||
requestKind: "other",
|
||||
url,
|
||||
httpStatus: res.status,
|
||||
headers: res.headers,
|
||||
message,
|
||||
rule: ch.rule,
|
||||
html: ch.html,
|
||||
now: ctx.now(),
|
||||
}),
|
||||
});
|
||||
}
|
||||
if (res.status === 429 || res.status === 503) {
|
||||
|
||||
@@ -1,10 +1,17 @@
|
||||
export {
|
||||
CHALLENGE_OUTCOME_HEADER,
|
||||
CHALLENGE_PEEK_BYTES,
|
||||
CHALLENGE_STATUSES,
|
||||
type ChallengePageKind,
|
||||
challengeGateRule,
|
||||
challengeOutcomeText,
|
||||
challengeRuleOfHeaders,
|
||||
challengeRuleOfHtml,
|
||||
challengeRuleOfOkPage,
|
||||
challengeRuleOfUnrecognizedPage,
|
||||
classifyChallengePage,
|
||||
detectChallenge,
|
||||
isPageRequest,
|
||||
} from "./challenge.ts";
|
||||
export {
|
||||
audiencesDefinition,
|
||||
@@ -34,6 +41,13 @@ export {
|
||||
structureDescription,
|
||||
} from "./detail-description.ts";
|
||||
export { type FormattedDetailPage, formatDetailPage } from "./detail-format.ts";
|
||||
export {
|
||||
buildFailureSample,
|
||||
FAILURE_SAMPLE_MAX_BYTES,
|
||||
redactPageHtml,
|
||||
type SiteFailureSample,
|
||||
type SiteRequestKind,
|
||||
} from "./failure-sample.ts";
|
||||
export {
|
||||
type FeedMediaType,
|
||||
normalizeSiteFeedItem,
|
||||
|
||||
@@ -14,7 +14,9 @@
|
||||
|
||||
import { inspectTorrentIsolated } from "../../isolation.ts";
|
||||
import { redactText } from "../../rss/redact.ts";
|
||||
import { challengeOutcomeText, challengeRuleOfUnrecognizedPage } from "../challenge.ts";
|
||||
import { buildHttpContext, type SiteAdapterDeps } from "../deps.ts";
|
||||
import { buildFailureSample, type SiteRequestKind } from "../failure-sample.ts";
|
||||
import { readBody, readText, siteFetch } from "../http.ts";
|
||||
import {
|
||||
type DownloadTarget,
|
||||
@@ -107,6 +109,23 @@ export function judgeLoginPage(
|
||||
};
|
||||
}
|
||||
|
||||
/** 抓到的页面:正文与留样用的响应信息 */
|
||||
interface FetchedPage {
|
||||
html: string;
|
||||
url: string;
|
||||
status: number;
|
||||
headers: Headers;
|
||||
}
|
||||
|
||||
/** 错误文案里的页面名 */
|
||||
const PAGE_LABEL: Record<SiteRequestKind, string> = {
|
||||
search: "搜索页",
|
||||
list: "列表页",
|
||||
detail: "详情页",
|
||||
health: "首页",
|
||||
other: "页面",
|
||||
};
|
||||
|
||||
export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdapterDeps): SiteAdapter {
|
||||
const def: IndexerDefinition = deps.baseUrl ? { ...input, domain: deps.baseUrl } : input;
|
||||
const ext = def.umt;
|
||||
@@ -119,13 +138,73 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap
|
||||
return c;
|
||||
}
|
||||
|
||||
async function fetchPage(url: string, what: string, opts: SiteCallOptions): Promise<string> {
|
||||
const res = await siteFetch(http, url, {
|
||||
method: "GET",
|
||||
headers: { cookie: await cookie(), accept: "text/html,application/xhtml+xml", referer: base },
|
||||
redirect: "manual",
|
||||
...(opts.signal ? { signal: opts.signal } : {}),
|
||||
/** 留样:交给调用方保存;回调出错不影响请求结果 */
|
||||
function report(e: SiteError): SiteError {
|
||||
if (e.sample && deps.onFailureSample) {
|
||||
try {
|
||||
deps.onFailureSample(e.sample);
|
||||
} catch {
|
||||
// 留样失败不影响请求
|
||||
}
|
||||
}
|
||||
return e;
|
||||
}
|
||||
|
||||
/** 页面认不出来时的错误:像人机验证就报 challenge,否则 parse_error;都附页面样本 */
|
||||
function unrecognized(page: FetchedPage, what: string, kind: SiteRequestKind, parseMessage: string): SiteError {
|
||||
const rule = challengeRuleOfUnrecognizedPage(page.html);
|
||||
const message = rule
|
||||
? `${what}：${PAGE_LABEL[kind]}需要人机验证（Cloudflare 等），${challengeOutcomeText("unhandled")}`
|
||||
: `${parseMessage}（页面样本见站点页技术信息）`;
|
||||
const sample = buildFailureSample({
|
||||
kind: rule ? "challenge" : "parse_error",
|
||||
requestKind: kind,
|
||||
url: page.url,
|
||||
httpStatus: page.status,
|
||||
headers: page.headers,
|
||||
message,
|
||||
rule,
|
||||
html: page.html,
|
||||
now: http.now(),
|
||||
});
|
||||
return report(
|
||||
new SiteError(def.id, rule ? "challenge" : "parse_error", message, {
|
||||
httpStatus: page.status,
|
||||
...(rule ? { detail: { reason: "challenge", rule, httpStatus: page.status } } : {}),
|
||||
sample,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async function fetchPage(
|
||||
url: string,
|
||||
what: string,
|
||||
opts: SiteCallOptions,
|
||||
kind: SiteRequestKind,
|
||||
): Promise<FetchedPage> {
|
||||
let res: Response;
|
||||
try {
|
||||
res = await siteFetch(http, url, {
|
||||
method: "GET",
|
||||
headers: { cookie: await cookie(), accept: "text/html,application/xhtml+xml", referer: base },
|
||||
redirect: "manual",
|
||||
...(opts.signal ? { signal: opts.signal } : {}),
|
||||
});
|
||||
} catch (e) {
|
||||
if (e instanceof SiteError && e.kind === "challenge") {
|
||||
// 网络层判出的人机验证:换成带请求类型的文案,样本补上请求类型
|
||||
const outcome = typeof e.detail?.outcome === "string" ? e.detail.outcome : null;
|
||||
const message = `${what}：${PAGE_LABEL[kind]}需要人机验证（Cloudflare 等），${challengeOutcomeText(outcome)}`;
|
||||
throw report(
|
||||
new SiteError(def.id, "challenge", message, {
|
||||
...(e.httpStatus !== undefined ? { httpStatus: e.httpStatus } : {}),
|
||||
...(e.detail ? { detail: e.detail } : {}),
|
||||
...(e.sample ? { sample: { ...e.sample, requestKind: kind, message } } : {}),
|
||||
}),
|
||||
);
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
if (res.status >= 300 && res.status < 400) {
|
||||
const loc = res.headers.get("location") ?? "";
|
||||
await res.body?.cancel().catch(() => {});
|
||||
@@ -155,7 +234,7 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap
|
||||
httpStatus: res.status,
|
||||
});
|
||||
}
|
||||
return readText(http, res);
|
||||
return { html: await readText(http, res), url, status: res.status, headers: res.headers };
|
||||
}
|
||||
|
||||
async function run(
|
||||
@@ -172,17 +251,19 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap
|
||||
page: Math.max(1, page) - 1, // 对外 1 起,MP 内部 0 起
|
||||
});
|
||||
if (!spider.canSearch()) throw new SiteError(def.id, "unsupported", `${what}:站点定义不支持该媒体类型`);
|
||||
const html = await fetchPage(spider.getSearchUrl(), what, opts);
|
||||
const parsed = spider.parse(html);
|
||||
const kind: SiteRequestKind = keyword ? "search" : "list";
|
||||
const page$ = await fetchPage(spider.getSearchUrl(), what, opts, kind);
|
||||
const parsed = spider.parse(page$.html);
|
||||
if (parsed.loginPage) throw new SiteError(def.id, "auth_expired", `${what}:返回登录或权限提示页`);
|
||||
if (parsed.isError || !parsed.$) throw new SiteError(def.id, "parse_error", `${what}:响应为空`);
|
||||
if (parsed.isError || !parsed.$) throw unrecognized(page$, what, kind, `${what}:响应为空`);
|
||||
const $ = parsed.$;
|
||||
if (parsed.torrents.length === 0) {
|
||||
const text = $("body").text();
|
||||
const hasList =
|
||||
$(normalizeSelector(spider.def.torrents.list.selector)).length > 0 || $("table.torrents").length > 0;
|
||||
const empty = (ext.emptyMarkers ?? []).some((m) => text.includes(m));
|
||||
if (!hasList && !empty) throw new SiteError(def.id, "parse_error", `${what}:页面结构无法识别(可能改版)`);
|
||||
// 认不出列表:先看是不是人机验证页(搜索页单独加盾、间歇触发),不是才报改版
|
||||
if (!hasList && !empty) throw unrecognized(page$, what, kind, `${what}:页面结构无法识别(可能改版)`);
|
||||
if (ext.loginSuccess && $(ext.loginSuccess).length === 0 && !empty) {
|
||||
throw new SiteError(def.id, "auth_expired", `${what}:未找到登录标志`);
|
||||
}
|
||||
@@ -237,18 +318,20 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap
|
||||
if (!/^\d+$/.test(siteReleaseId)) throw new SiteError(def.id, "not_found", "资源 ID 非法");
|
||||
if (!ext.detail) throw new SiteError(def.id, "unsupported", "站点定义未配置详情页解析");
|
||||
const url = new URL(`${ext.detail.path}?id=${siteReleaseId}&hit=1`, base).toString();
|
||||
const html = await fetchPage(url, `详情 ${siteReleaseId}`, opts);
|
||||
const r = parseDetailPage(html, def, siteReleaseId, http.now().toISOString());
|
||||
const detailPage = await fetchPage(url, `详情 ${siteReleaseId}`, opts, "detail");
|
||||
const r = parseDetailPage(detailPage.html, def, siteReleaseId, http.now().toISOString());
|
||||
if (r.kind === "login") throw new SiteError(def.id, "auth_expired", `详情 ${siteReleaseId}:返回登录页`);
|
||||
if (r.kind === "not_found") throw new SiteError(def.id, "not_found", `详情 ${siteReleaseId}:资源不存在`);
|
||||
if (!r.detail) throw new SiteError(def.id, "parse_error", `详情 ${siteReleaseId}:页面结构无法识别`);
|
||||
if (!r.detail) {
|
||||
throw unrecognized(detailPage, `详情 ${siteReleaseId}`, "detail", `详情 ${siteReleaseId}:页面结构无法识别`);
|
||||
}
|
||||
// 文件清单不在页面上(NexusPHP 点「查看列表」才 AJAX 取)→ 按需 1 次;失败只是没有文件清单
|
||||
const flPath = ext.detail.page?.fileListPath;
|
||||
if (flPath && r.detail.page && !r.detail.page.files.length) {
|
||||
try {
|
||||
const flUrl = new URL(flPath.replace("{id}", siteReleaseId), base).toString();
|
||||
const frag = await fetchPage(flUrl, `文件清单 ${siteReleaseId}`, opts);
|
||||
r.detail.page.files = filesFromFileListHtml(loadHtml(frag));
|
||||
const frag = await fetchPage(flUrl, `文件清单 ${siteReleaseId}`, opts, "detail");
|
||||
r.detail.page.files = filesFromFileListHtml(loadHtml(frag.html));
|
||||
} catch (e) {
|
||||
if (e instanceof SiteError && e.isAuthError) throw e;
|
||||
}
|
||||
@@ -317,8 +400,8 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap
|
||||
const started = Date.now();
|
||||
const checkedAt = http.now().toISOString();
|
||||
try {
|
||||
const html = await fetchPage(new URL("index.php", base).toString(), "健康检查", opts);
|
||||
const v = judgeLoginPage(html, ext.loginSuccess);
|
||||
const home = await fetchPage(new URL("index.php", base).toString(), "健康检查", opts, "health");
|
||||
const v = judgeLoginPage(home.html, ext.loginSuccess);
|
||||
const latencyMs = Date.now() - started;
|
||||
if (v.ok) return { siteId: def.id, status: "ok", ok: true, checkedAt, latencyMs, detail: v.detail };
|
||||
return {
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
audiencesDefinition,
|
||||
createSiteRateLimiter,
|
||||
createSpiderSiteAdapter,
|
||||
SiteError,
|
||||
type SiteFailureSample,
|
||||
staticSecretResolver,
|
||||
} from "../../index.ts";
|
||||
|
||||
// 搜索页单独加盾(间歇触发):返回 200 的挑战 / 人机验证页不再报「页面结构无法识别」,报 challenge 并留样;
|
||||
// 真正认不出的页面才报 parse_error(也留样)。
|
||||
|
||||
const AUD_SEARCH = readFileSync(new URL("../../../test/fixtures/audiences-search.html.txt", import.meta.url), "utf8");
|
||||
const CF_200 = `<!DOCTYPE html><html><head><title>Just a moment...</title></head><body>
|
||||
<div id="challenge-spinner"></div><script>window._cf_chl_opt={cType: 'managed'}</script></body></html>`;
|
||||
/** 站点外壳里的人机验证页:有用户信息块(网络层的严格判定不认),适配器解析失败时兜底认出 */
|
||||
const SHELL_GATE = `<html><head><title>Audiences :: 搜索</title></head><body>
|
||||
<table id="info_block"><tr><td><a href="userdetails.php?id=29191">someone</a> <a href="logout.php">退出</a></td></tr></table>
|
||||
<div class="main"><p>搜索前请完成人机验证</p><div class="cf-turnstile" data-sitekey="0x4AAA"></div>
|
||||
<script src="https://challenges.cloudflare.com/turnstile/v0/api.js"></script></div></body></html>`;
|
||||
const LOGIN_WITH_TURNSTILE = `<html><head><title>Audiences :: 登录 - Powered by NexusPHP</title></head><body>
|
||||
<form method="post" action="takelogin.php"><input type="password" name="password">
|
||||
<div class="cf-turnstile" data-sitekey="x"></div></form></body></html>`;
|
||||
/** 改版页:带上各种要脱敏的东西 */
|
||||
const SECRET = "0123456789abcdef0123456789abcdef";
|
||||
const BROKEN = `<html><head><title>Audiences :: 种子</title></head><body>
|
||||
<div id="info_block">欢迎 <a href="userdetails.php?id=29191"><b>alice_cn</b></a> 邮箱 [email protected]</div>
|
||||
<a href="download.php?id=1&passkey=${SECRET}">dl</a>
|
||||
<script>var authkey = "${SECRET}"; document.cookie = "c_secure_pass=topsecretvalue";</script>
|
||||
<input type="hidden" name="passkey" value="${SECRET}">
|
||||
<div class="new-grid">${"<div class='card'>Dune</div>".repeat(5000)}</div></body></html>`;
|
||||
|
||||
function adapter(body: string, status = 200, headers: Record<string, string> = {}) {
|
||||
const samples: SiteFailureSample[] = [];
|
||||
const fetchImpl = (async () =>
|
||||
new Response(body, {
|
||||
status,
|
||||
headers: { "content-type": "text/html; charset=utf-8", ...headers },
|
||||
})) as typeof fetch;
|
||||
const a = createSpiderSiteAdapter(audiencesDefinition, {
|
||||
baseUrl: "https://aud.test/",
|
||||
secrets: staticSecretResolver({ [audiencesDefinition.id]: { cookie: "c_secure_uid=1; c_secure_pass=abc" } }),
|
||||
limiter: createSiteRateLimiter({ minIntervalMs: 0, burst: 10 }),
|
||||
fetch: fetchImpl,
|
||||
onFailureSample: (s) => samples.push(s),
|
||||
});
|
||||
return { a, samples };
|
||||
}
|
||||
|
||||
async function err(p: Promise<unknown>): Promise<SiteError> {
|
||||
try {
|
||||
await p;
|
||||
} catch (e) {
|
||||
expect(e).toBeInstanceOf(SiteError);
|
||||
return e as SiteError;
|
||||
}
|
||||
throw new Error("expected SiteError");
|
||||
}
|
||||
|
||||
describe("spider 搜索页人机验证", () => {
|
||||
it("正常搜索页(真实样本)照常解析,不留样", async () => {
|
||||
const { a, samples } = adapter(AUD_SEARCH);
|
||||
const page = await a.search("药屋少女的呢喃");
|
||||
expect(page.items.length).toBeGreaterThan(0);
|
||||
expect(samples).toEqual([]);
|
||||
});
|
||||
|
||||
it("200 的 Cloudflare 挑战页 → challenge(网络层认出),文案说搜索页需要人机验证,留样标搜索", async () => {
|
||||
const { a, samples } = adapter(CF_200, 200, { server: "cloudflare", "cf-ray": "8c-HKG" });
|
||||
const e = await err(a.search("Dune"));
|
||||
expect(e.kind).toBe("challenge");
|
||||
expect(e.isAuthError).toBe(false);
|
||||
expect(e.message).toContain("搜索页需要人机验证");
|
||||
expect(e.message).not.toContain("改版");
|
||||
expect(samples).toHaveLength(1);
|
||||
expect(samples[0]).toMatchObject({ kind: "challenge", requestKind: "search", httpStatus: 200 });
|
||||
expect(samples[0]?.headers["cf-ray"]).toBe("8c-HKG");
|
||||
expect(samples[0]?.url).toContain("torrents.php");
|
||||
});
|
||||
|
||||
it("站点外壳里的人机验证页:网络层不认,解析失败前兜底 → challenge(不报改版)", async () => {
|
||||
const { a, samples } = adapter(SHELL_GATE);
|
||||
const e = await err(a.search("Dune"));
|
||||
expect(e.kind).toBe("challenge");
|
||||
expect(e.message).toContain("搜索页需要人机验证");
|
||||
expect(e.detail?.rule).toContain("turnstile");
|
||||
expect(samples[0]).toMatchObject({ kind: "challenge", requestKind: "search" });
|
||||
expect(samples[0]?.body).not.toContain("29191");
|
||||
expect(samples[0]?.body).not.toContain("someone");
|
||||
});
|
||||
|
||||
it("列表页同样处理,文案按列表", async () => {
|
||||
const { a, samples } = adapter(SHELL_GATE);
|
||||
const e = await err(a.list(1));
|
||||
expect(e.kind).toBe("challenge");
|
||||
expect(e.message).toContain("列表页需要人机验证");
|
||||
expect(samples[0]?.requestKind).toBe("list");
|
||||
});
|
||||
|
||||
it("带 Turnstile 的登录页仍按登录失效处理,不留样", async () => {
|
||||
const { a, samples } = adapter(LOGIN_WITH_TURNSTILE);
|
||||
const e = await err(a.search("Dune"));
|
||||
expect(e.kind).toBe("auth_expired");
|
||||
expect(samples).toEqual([]);
|
||||
});
|
||||
|
||||
it("真正认不出的页面 → parse_error,留脱敏样本(≤ 64 KB)", async () => {
|
||||
const { a, samples } = adapter(BROKEN, 200, { server: "nginx" });
|
||||
const e = await err(a.search("Dune"));
|
||||
expect(e.kind).toBe("parse_error");
|
||||
expect(e.message).toContain("页面结构无法识别");
|
||||
expect(samples).toHaveLength(1);
|
||||
const s = samples[0] as SiteFailureSample;
|
||||
expect(s).toMatchObject({ kind: "parse_error", requestKind: "search", httpStatus: 200, truncated: true });
|
||||
expect(new TextEncoder().encode(s.body).byteLength).toBeLessThanOrEqual(64 * 1024);
|
||||
for (const leak of [SECRET, "[email protected]", "alice_cn", "29191", "topsecretvalue"]) {
|
||||
expect(s.body).not.toContain(leak);
|
||||
}
|
||||
expect(s.body).toContain("download.php?id=1");
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,7 @@
|
||||
/** 站点适配器自有类型(不依赖 @umt/contracts;W2 负责映射到 source_release / site_snapshot)。 */
|
||||
|
||||
import type { SiteFailureSample } from "./failure-sample.ts";
|
||||
|
||||
/** api = 专用 API spider(spiders/*,Haidan/HDDolby/RousiPro/SunnyPT/TNode/TorrentLeech/Yema) */
|
||||
export type SiteKind = "nexusphp" | "mteam" | "api";
|
||||
|
||||
@@ -205,11 +207,13 @@ export class SiteError extends Error {
|
||||
readonly retryAfterMs?: number;
|
||||
/** 原始判定(健康检查的技术信息用) */
|
||||
readonly detail?: SiteHealthDetail;
|
||||
/** 失败页面样本(已脱敏;parse_error / challenge 时由适配器附上,交给 deps.onFailureSample 留样) */
|
||||
readonly sample?: SiteFailureSample;
|
||||
constructor(
|
||||
siteId: string,
|
||||
kind: SiteErrorKind,
|
||||
message: string,
|
||||
opts: { httpStatus?: number; retryAfterMs?: number; detail?: SiteHealthDetail } = {},
|
||||
opts: { httpStatus?: number; retryAfterMs?: number; detail?: SiteHealthDetail; sample?: SiteFailureSample } = {},
|
||||
) {
|
||||
super(message);
|
||||
this.name = "SiteError";
|
||||
@@ -218,6 +222,7 @@ export class SiteError extends Error {
|
||||
if (opts.httpStatus !== undefined) this.httpStatus = opts.httpStatus;
|
||||
if (opts.retryAfterMs !== undefined) this.retryAfterMs = opts.retryAfterMs;
|
||||
if (opts.detail !== undefined) this.detail = opts.detail;
|
||||
if (opts.sample !== undefined) this.sample = opts.sample;
|
||||
}
|
||||
/** 凭据类错误:应立即置账号验证失败,不再重试 */
|
||||
get isAuthError(): boolean {
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
Reference in new issue
Block a user