fix(adapters): 识别 200 状态的挑战 / 人机验证页,解析失败前兜底,失败页面留样
- detectChallenge 支持页面类请求(GET + Accept text/html)的 2xx 响应:正文只先读 256 KB, 短页且没有种子表 / 用户信息块时,认 Cloudflare 挑战标题 / 脚本与 Turnstile / hCaptcha / reCAPTCHA 门页; 带登录表单的页面不算门页。非页面请求不读正文 - siteFetch 用上面的判定;挑战错误附脱敏页面样本,文案带级联结果(x-umt-challenge) - spider 适配器:认不出列表 / 详情时先看是不是人机验证页,是就报 challenge(「搜索页需要人机验证」), 不再报改版;parse_error / challenge 都经 deps.onFailureSample 交出脱敏样本(前 64 KB,去掉个人信息块、 passkey / authkey / uid / 用户名 / Cookie / 邮箱) - classifyChallengePage 也认门页(浏览器过挑战时不再把门页当已通过) Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
4fa001750b
commit
1779e1a0f2
11 files changed
+2078
-34
No files matched your search
@@ -14,7 +14,9 @@
|
||||
|
||||
import { inspectTorrentIsolated } from "../../isolation.ts";
|
||||
import { redactText } from "../../rss/redact.ts";
|
||||
import { challengeOutcomeText, challengeRuleOfUnrecognizedPage } from "../challenge.ts";
|
||||
import { buildHttpContext, type SiteAdapterDeps } from "../deps.ts";
|
||||
import { buildFailureSample, type SiteRequestKind } from "../failure-sample.ts";
|
||||
import { readBody, readText, siteFetch } from "../http.ts";
|
||||
import {
|
||||
type DownloadTarget,
|
||||
@@ -107,6 +109,23 @@ export function judgeLoginPage(
|
||||
};
|
||||
}
|
||||
|
||||
/** 抓到的页面:正文与留样用的响应信息 */
|
||||
interface FetchedPage {
|
||||
html: string;
|
||||
url: string;
|
||||
status: number;
|
||||
headers: Headers;
|
||||
}
|
||||
|
||||
/** 错误文案里的页面名 */
|
||||
const PAGE_LABEL: Record<SiteRequestKind, string> = {
|
||||
search: "搜索页",
|
||||
list: "列表页",
|
||||
detail: "详情页",
|
||||
health: "首页",
|
||||
other: "页面",
|
||||
};
|
||||
|
||||
export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdapterDeps): SiteAdapter {
|
||||
const def: IndexerDefinition = deps.baseUrl ? { ...input, domain: deps.baseUrl } : input;
|
||||
const ext = def.umt;
|
||||
@@ -119,13 +138,73 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap
|
||||
return c;
|
||||
}
|
||||
|
||||
async function fetchPage(url: string, what: string, opts: SiteCallOptions): Promise<string> {
|
||||
const res = await siteFetch(http, url, {
|
||||
method: "GET",
|
||||
headers: { cookie: await cookie(), accept: "text/html,application/xhtml+xml", referer: base },
|
||||
redirect: "manual",
|
||||
...(opts.signal ? { signal: opts.signal } : {}),
|
||||
/** 留样:交给调用方保存;回调出错不影响请求结果 */
|
||||
function report(e: SiteError): SiteError {
|
||||
if (e.sample && deps.onFailureSample) {
|
||||
try {
|
||||
deps.onFailureSample(e.sample);
|
||||
} catch {
|
||||
// 留样失败不影响请求
|
||||
}
|
||||
}
|
||||
return e;
|
||||
}
|
||||
|
||||
/** 页面认不出来时的错误:像人机验证就报 challenge,否则 parse_error;都附页面样本 */
|
||||
function unrecognized(page: FetchedPage, what: string, kind: SiteRequestKind, parseMessage: string): SiteError {
|
||||
const rule = challengeRuleOfUnrecognizedPage(page.html);
|
||||
const message = rule
|
||||
? `${what}：${PAGE_LABEL[kind]}需要人机验证（Cloudflare 等），${challengeOutcomeText("unhandled")}`
|
||||
: `${parseMessage}（页面样本见站点页技术信息）`;
|
||||
const sample = buildFailureSample({
|
||||
kind: rule ? "challenge" : "parse_error",
|
||||
requestKind: kind,
|
||||
url: page.url,
|
||||
httpStatus: page.status,
|
||||
headers: page.headers,
|
||||
message,
|
||||
rule,
|
||||
html: page.html,
|
||||
now: http.now(),
|
||||
});
|
||||
return report(
|
||||
new SiteError(def.id, rule ? "challenge" : "parse_error", message, {
|
||||
httpStatus: page.status,
|
||||
...(rule ? { detail: { reason: "challenge", rule, httpStatus: page.status } } : {}),
|
||||
sample,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async function fetchPage(
|
||||
url: string,
|
||||
what: string,
|
||||
opts: SiteCallOptions,
|
||||
kind: SiteRequestKind,
|
||||
): Promise<FetchedPage> {
|
||||
let res: Response;
|
||||
try {
|
||||
res = await siteFetch(http, url, {
|
||||
method: "GET",
|
||||
headers: { cookie: await cookie(), accept: "text/html,application/xhtml+xml", referer: base },
|
||||
redirect: "manual",
|
||||
...(opts.signal ? { signal: opts.signal } : {}),
|
||||
});
|
||||
} catch (e) {
|
||||
if (e instanceof SiteError && e.kind === "challenge") {
|
||||
// 网络层判出的人机验证:换成带请求类型的文案,样本补上请求类型
|
||||
const outcome = typeof e.detail?.outcome === "string" ? e.detail.outcome : null;
|
||||
const message = `${what}：${PAGE_LABEL[kind]}需要人机验证（Cloudflare 等），${challengeOutcomeText(outcome)}`;
|
||||
throw report(
|
||||
new SiteError(def.id, "challenge", message, {
|
||||
...(e.httpStatus !== undefined ? { httpStatus: e.httpStatus } : {}),
|
||||
...(e.detail ? { detail: e.detail } : {}),
|
||||
...(e.sample ? { sample: { ...e.sample, requestKind: kind, message } } : {}),
|
||||
}),
|
||||
);
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
if (res.status >= 300 && res.status < 400) {
|
||||
const loc = res.headers.get("location") ?? "";
|
||||
await res.body?.cancel().catch(() => {});
|
||||
@@ -155,7 +234,7 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap
|
||||
httpStatus: res.status,
|
||||
});
|
||||
}
|
||||
return readText(http, res);
|
||||
return { html: await readText(http, res), url, status: res.status, headers: res.headers };
|
||||
}
|
||||
|
||||
async function run(
|
||||
@@ -172,17 +251,19 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap
|
||||
page: Math.max(1, page) - 1, // 对外 1 起,MP 内部 0 起
|
||||
});
|
||||
if (!spider.canSearch()) throw new SiteError(def.id, "unsupported", `${what}:站点定义不支持该媒体类型`);
|
||||
const html = await fetchPage(spider.getSearchUrl(), what, opts);
|
||||
const parsed = spider.parse(html);
|
||||
const kind: SiteRequestKind = keyword ? "search" : "list";
|
||||
const page$ = await fetchPage(spider.getSearchUrl(), what, opts, kind);
|
||||
const parsed = spider.parse(page$.html);
|
||||
if (parsed.loginPage) throw new SiteError(def.id, "auth_expired", `${what}:返回登录或权限提示页`);
|
||||
if (parsed.isError || !parsed.$) throw new SiteError(def.id, "parse_error", `${what}:响应为空`);
|
||||
if (parsed.isError || !parsed.$) throw unrecognized(page$, what, kind, `${what}:响应为空`);
|
||||
const $ = parsed.$;
|
||||
if (parsed.torrents.length === 0) {
|
||||
const text = $("body").text();
|
||||
const hasList =
|
||||
$(normalizeSelector(spider.def.torrents.list.selector)).length > 0 || $("table.torrents").length > 0;
|
||||
const empty = (ext.emptyMarkers ?? []).some((m) => text.includes(m));
|
||||
if (!hasList && !empty) throw new SiteError(def.id, "parse_error", `${what}:页面结构无法识别(可能改版)`);
|
||||
// 认不出列表:先看是不是人机验证页(搜索页单独加盾、间歇触发),不是才报改版
|
||||
if (!hasList && !empty) throw unrecognized(page$, what, kind, `${what}:页面结构无法识别(可能改版)`);
|
||||
if (ext.loginSuccess && $(ext.loginSuccess).length === 0 && !empty) {
|
||||
throw new SiteError(def.id, "auth_expired", `${what}:未找到登录标志`);
|
||||
}
|
||||
@@ -237,18 +318,20 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap
|
||||
if (!/^\d+$/.test(siteReleaseId)) throw new SiteError(def.id, "not_found", "资源 ID 非法");
|
||||
if (!ext.detail) throw new SiteError(def.id, "unsupported", "站点定义未配置详情页解析");
|
||||
const url = new URL(`${ext.detail.path}?id=${siteReleaseId}&hit=1`, base).toString();
|
||||
const html = await fetchPage(url, `详情 ${siteReleaseId}`, opts);
|
||||
const r = parseDetailPage(html, def, siteReleaseId, http.now().toISOString());
|
||||
const detailPage = await fetchPage(url, `详情 ${siteReleaseId}`, opts, "detail");
|
||||
const r = parseDetailPage(detailPage.html, def, siteReleaseId, http.now().toISOString());
|
||||
if (r.kind === "login") throw new SiteError(def.id, "auth_expired", `详情 ${siteReleaseId}:返回登录页`);
|
||||
if (r.kind === "not_found") throw new SiteError(def.id, "not_found", `详情 ${siteReleaseId}:资源不存在`);
|
||||
if (!r.detail) throw new SiteError(def.id, "parse_error", `详情 ${siteReleaseId}:页面结构无法识别`);
|
||||
if (!r.detail) {
|
||||
throw unrecognized(detailPage, `详情 ${siteReleaseId}`, "detail", `详情 ${siteReleaseId}:页面结构无法识别`);
|
||||
}
|
||||
// 文件清单不在页面上(NexusPHP 点「查看列表」才 AJAX 取)→ 按需 1 次;失败只是没有文件清单
|
||||
const flPath = ext.detail.page?.fileListPath;
|
||||
if (flPath && r.detail.page && !r.detail.page.files.length) {
|
||||
try {
|
||||
const flUrl = new URL(flPath.replace("{id}", siteReleaseId), base).toString();
|
||||
const frag = await fetchPage(flUrl, `文件清单 ${siteReleaseId}`, opts);
|
||||
r.detail.page.files = filesFromFileListHtml(loadHtml(frag));
|
||||
const frag = await fetchPage(flUrl, `文件清单 ${siteReleaseId}`, opts, "detail");
|
||||
r.detail.page.files = filesFromFileListHtml(loadHtml(frag.html));
|
||||
} catch (e) {
|
||||
if (e instanceof SiteError && e.isAuthError) throw e;
|
||||
}
|
||||
@@ -317,8 +400,8 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap
|
||||
const started = Date.now();
|
||||
const checkedAt = http.now().toISOString();
|
||||
try {
|
||||
const html = await fetchPage(new URL("index.php", base).toString(), "健康检查", opts);
|
||||
const v = judgeLoginPage(html, ext.loginSuccess);
|
||||
const home = await fetchPage(new URL("index.php", base).toString(), "健康检查", opts, "health");
|
||||
const v = judgeLoginPage(home.html, ext.loginSuccess);
|
||||
const latencyMs = Date.now() - started;
|
||||
if (v.ok) return { siteId: def.id, status: "ok", ok: true, checkedAt, latencyMs, detail: v.detail };
|
||||
return {
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
audiencesDefinition,
|
||||
createSiteRateLimiter,
|
||||
createSpiderSiteAdapter,
|
||||
SiteError,
|
||||
type SiteFailureSample,
|
||||
staticSecretResolver,
|
||||
} from "../../index.ts";
|
||||
|
||||
// 搜索页单独加盾(间歇触发):返回 200 的挑战 / 人机验证页不再报「页面结构无法识别」,报 challenge 并留样;
|
||||
// 真正认不出的页面才报 parse_error(也留样)。
|
||||
|
||||
const AUD_SEARCH = readFileSync(new URL("../../../test/fixtures/audiences-search.html.txt", import.meta.url), "utf8");
|
||||
const CF_200 = `<!DOCTYPE html><html><head><title>Just a moment...</title></head><body>
|
||||
<div id="challenge-spinner"></div><script>window._cf_chl_opt={cType: 'managed'}</script></body></html>`;
|
||||
/** 站点外壳里的人机验证页:有用户信息块(网络层的严格判定不认),适配器解析失败时兜底认出 */
|
||||
const SHELL_GATE = `<html><head><title>Audiences :: 搜索</title></head><body>
|
||||
<table id="info_block"><tr><td><a href="userdetails.php?id=29191">someone</a> <a href="logout.php">退出</a></td></tr></table>
|
||||
<div class="main"><p>搜索前请完成人机验证</p><div class="cf-turnstile" data-sitekey="0x4AAA"></div>
|
||||
<script src="https://challenges.cloudflare.com/turnstile/v0/api.js"></script></div></body></html>`;
|
||||
const LOGIN_WITH_TURNSTILE = `<html><head><title>Audiences :: 登录 - Powered by NexusPHP</title></head><body>
|
||||
<form method="post" action="takelogin.php"><input type="password" name="password">
|
||||
<div class="cf-turnstile" data-sitekey="x"></div></form></body></html>`;
|
||||
/** 改版页:带上各种要脱敏的东西 */
|
||||
const SECRET = "0123456789abcdef0123456789abcdef";
|
||||
const BROKEN = `<html><head><title>Audiences :: 种子</title></head><body>
|
||||
<div id="info_block">欢迎 <a href="userdetails.php?id=29191"><b>alice_cn</b></a> 邮箱 [email protected]</div>
|
||||
<a href="download.php?id=1&passkey=${SECRET}">dl</a>
|
||||
<script>var authkey = "${SECRET}"; document.cookie = "c_secure_pass=topsecretvalue";</script>
|
||||
<input type="hidden" name="passkey" value="${SECRET}">
|
||||
<div class="new-grid">${"<div class='card'>Dune</div>".repeat(5000)}</div></body></html>`;
|
||||
|
||||
function adapter(body: string, status = 200, headers: Record<string, string> = {}) {
|
||||
const samples: SiteFailureSample[] = [];
|
||||
const fetchImpl = (async () =>
|
||||
new Response(body, {
|
||||
status,
|
||||
headers: { "content-type": "text/html; charset=utf-8", ...headers },
|
||||
})) as typeof fetch;
|
||||
const a = createSpiderSiteAdapter(audiencesDefinition, {
|
||||
baseUrl: "https://aud.test/",
|
||||
secrets: staticSecretResolver({ [audiencesDefinition.id]: { cookie: "c_secure_uid=1; c_secure_pass=abc" } }),
|
||||
limiter: createSiteRateLimiter({ minIntervalMs: 0, burst: 10 }),
|
||||
fetch: fetchImpl,
|
||||
onFailureSample: (s) => samples.push(s),
|
||||
});
|
||||
return { a, samples };
|
||||
}
|
||||
|
||||
async function err(p: Promise<unknown>): Promise<SiteError> {
|
||||
try {
|
||||
await p;
|
||||
} catch (e) {
|
||||
expect(e).toBeInstanceOf(SiteError);
|
||||
return e as SiteError;
|
||||
}
|
||||
throw new Error("expected SiteError");
|
||||
}
|
||||
|
||||
describe("spider 搜索页人机验证", () => {
|
||||
it("正常搜索页(真实样本)照常解析,不留样", async () => {
|
||||
const { a, samples } = adapter(AUD_SEARCH);
|
||||
const page = await a.search("药屋少女的呢喃");
|
||||
expect(page.items.length).toBeGreaterThan(0);
|
||||
expect(samples).toEqual([]);
|
||||
});
|
||||
|
||||
it("200 的 Cloudflare 挑战页 → challenge(网络层认出),文案说搜索页需要人机验证,留样标搜索", async () => {
|
||||
const { a, samples } = adapter(CF_200, 200, { server: "cloudflare", "cf-ray": "8c-HKG" });
|
||||
const e = await err(a.search("Dune"));
|
||||
expect(e.kind).toBe("challenge");
|
||||
expect(e.isAuthError).toBe(false);
|
||||
expect(e.message).toContain("搜索页需要人机验证");
|
||||
expect(e.message).not.toContain("改版");
|
||||
expect(samples).toHaveLength(1);
|
||||
expect(samples[0]).toMatchObject({ kind: "challenge", requestKind: "search", httpStatus: 200 });
|
||||
expect(samples[0]?.headers["cf-ray"]).toBe("8c-HKG");
|
||||
expect(samples[0]?.url).toContain("torrents.php");
|
||||
});
|
||||
|
||||
it("站点外壳里的人机验证页:网络层不认,解析失败前兜底 → challenge(不报改版)", async () => {
|
||||
const { a, samples } = adapter(SHELL_GATE);
|
||||
const e = await err(a.search("Dune"));
|
||||
expect(e.kind).toBe("challenge");
|
||||
expect(e.message).toContain("搜索页需要人机验证");
|
||||
expect(e.detail?.rule).toContain("turnstile");
|
||||
expect(samples[0]).toMatchObject({ kind: "challenge", requestKind: "search" });
|
||||
expect(samples[0]?.body).not.toContain("29191");
|
||||
expect(samples[0]?.body).not.toContain("someone");
|
||||
});
|
||||
|
||||
it("列表页同样处理,文案按列表", async () => {
|
||||
const { a, samples } = adapter(SHELL_GATE);
|
||||
const e = await err(a.list(1));
|
||||
expect(e.kind).toBe("challenge");
|
||||
expect(e.message).toContain("列表页需要人机验证");
|
||||
expect(samples[0]?.requestKind).toBe("list");
|
||||
});
|
||||
|
||||
it("带 Turnstile 的登录页仍按登录失效处理,不留样", async () => {
|
||||
const { a, samples } = adapter(LOGIN_WITH_TURNSTILE);
|
||||
const e = await err(a.search("Dune"));
|
||||
expect(e.kind).toBe("auth_expired");
|
||||
expect(samples).toEqual([]);
|
||||
});
|
||||
|
||||
it("真正认不出的页面 → parse_error,留脱敏样本(≤ 64 KB)", async () => {
|
||||
const { a, samples } = adapter(BROKEN, 200, { server: "nginx" });
|
||||
const e = await err(a.search("Dune"));
|
||||
expect(e.kind).toBe("parse_error");
|
||||
expect(e.message).toContain("页面结构无法识别");
|
||||
expect(samples).toHaveLength(1);
|
||||
const s = samples[0] as SiteFailureSample;
|
||||
expect(s).toMatchObject({ kind: "parse_error", requestKind: "search", httpStatus: 200, truncated: true });
|
||||
expect(new TextEncoder().encode(s.body).byteLength).toBeLessThanOrEqual(64 * 1024);
|
||||
for (const leak of [SECRET, "[email protected]", "alice_cn", "29191", "topsecretvalue"]) {
|
||||
expect(s.body).not.toContain(leak);
|
||||
}
|
||||
expect(s.body).toContain("download.php?id=1");
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user