fix(adapters): 识别 200 状态的挑战 / 人机验证页,解析失败前兜底,失败页面留样
- detectChallenge 支持页面类请求(GET + Accept text/html)的 2xx 响应:正文只先读 256 KB, 短页且没有种子表 / 用户信息块时,认 Cloudflare 挑战标题 / 脚本与 Turnstile / hCaptcha / reCAPTCHA 门页; 带登录表单的页面不算门页。非页面请求不读正文 - siteFetch 用上面的判定;挑战错误附脱敏页面样本,文案带级联结果(x-umt-challenge) - spider 适配器:认不出列表 / 详情时先看是不是人机验证页,是就报 challenge(「搜索页需要人机验证」), 不再报改版;parse_error / challenge 都经 deps.onFailureSample 交出脱敏样本(前 64 KB,去掉个人信息块、 passkey / authkey / uid / 用户名 / Cookie / 邮箱) - classifyChallengePage 也认门页(浏览器过挑战时不再把门页当已通过) Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
4fa001750b
commit
1779e1a0f2
11 files changed
+2078
-34
No files matched your search
@@ -14,7 +14,9 @@
|
||||
|
||||
import { inspectTorrentIsolated } from "../../isolation.ts";
|
||||
import { redactText } from "../../rss/redact.ts";
|
||||
import { challengeOutcomeText, challengeRuleOfUnrecognizedPage } from "../challenge.ts";
|
||||
import { buildHttpContext, type SiteAdapterDeps } from "../deps.ts";
|
||||
import { buildFailureSample, type SiteRequestKind } from "../failure-sample.ts";
|
||||
import { readBody, readText, siteFetch } from "../http.ts";
|
||||
import {
|
||||
type DownloadTarget,
|
||||
@@ -107,6 +109,23 @@ export function judgeLoginPage(
|
||||
};
|
||||
}
|
||||
|
||||
/** 抓到的页面:正文与留样用的响应信息 */
|
||||
interface FetchedPage {
|
||||
html: string;
|
||||
url: string;
|
||||
status: number;
|
||||
headers: Headers;
|
||||
}
|
||||
|
||||
/** 错误文案里的页面名 */
|
||||
const PAGE_LABEL: Record<SiteRequestKind, string> = {
|
||||
search: "搜索页",
|
||||
list: "列表页",
|
||||
detail: "详情页",
|
||||
health: "首页",
|
||||
other: "页面",
|
||||
};
|
||||
|
||||
export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdapterDeps): SiteAdapter {
|
||||
const def: IndexerDefinition = deps.baseUrl ? { ...input, domain: deps.baseUrl } : input;
|
||||
const ext = def.umt;
|
||||
@@ -119,13 +138,73 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap
|
||||
return c;
|
||||
}
|
||||
|
||||
async function fetchPage(url: string, what: string, opts: SiteCallOptions): Promise<string> {
|
||||
const res = await siteFetch(http, url, {
|
||||
method: "GET",
|
||||
headers: { cookie: await cookie(), accept: "text/html,application/xhtml+xml", referer: base },
|
||||
redirect: "manual",
|
||||
...(opts.signal ? { signal: opts.signal } : {}),
|
||||
/** 留样:交给调用方保存;回调出错不影响请求结果 */
|
||||
function report(e: SiteError): SiteError {
|
||||
if (e.sample && deps.onFailureSample) {
|
||||
try {
|
||||
deps.onFailureSample(e.sample);
|
||||
} catch {
|
||||
// 留样失败不影响请求
|
||||
}
|
||||
}
|
||||
return e;
|
||||
}
|
||||
|
||||
/** 页面认不出来时的错误:像人机验证就报 challenge,否则 parse_error;都附页面样本 */
|
||||
function unrecognized(page: FetchedPage, what: string, kind: SiteRequestKind, parseMessage: string): SiteError {
|
||||
const rule = challengeRuleOfUnrecognizedPage(page.html);
|
||||
const message = rule
|
||||
? `${what}：${PAGE_LABEL[kind]}需要人机验证（Cloudflare 等），${challengeOutcomeText("unhandled")}`
|
||||
: `${parseMessage}（页面样本见站点页技术信息）`;
|
||||
const sample = buildFailureSample({
|
||||
kind: rule ? "challenge" : "parse_error",
|
||||
requestKind: kind,
|
||||
url: page.url,
|
||||
httpStatus: page.status,
|
||||
headers: page.headers,
|
||||
message,
|
||||
rule,
|
||||
html: page.html,
|
||||
now: http.now(),
|
||||
});
|
||||
return report(
|
||||
new SiteError(def.id, rule ? "challenge" : "parse_error", message, {
|
||||
httpStatus: page.status,
|
||||
...(rule ? { detail: { reason: "challenge", rule, httpStatus: page.status } } : {}),
|
||||
sample,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async function fetchPage(
|
||||
url: string,
|
||||
what: string,
|
||||
opts: SiteCallOptions,
|
||||
kind: SiteRequestKind,
|
||||
): Promise<FetchedPage> {
|
||||
let res: Response;
|
||||
try {
|
||||
res = await siteFetch(http, url, {
|
||||
method: "GET",
|
||||
headers: { cookie: await cookie(), accept: "text/html,application/xhtml+xml", referer: base },
|
||||
redirect: "manual",
|
||||
...(opts.signal ? { signal: opts.signal } : {}),
|
||||
});
|
||||
} catch (e) {
|
||||
if (e instanceof SiteError && e.kind === "challenge") {
|
||||
// 网络层判出的人机验证:换成带请求类型的文案,样本补上请求类型
|
||||
const outcome = typeof e.detail?.outcome === "string" ? e.detail.outcome : null;
|
||||
const message = `${what}：${PAGE_LABEL[kind]}需要人机验证（Cloudflare 等），${challengeOutcomeText(outcome)}`;
|
||||
throw report(
|
||||
new SiteError(def.id, "challenge", message, {
|
||||
...(e.httpStatus !== undefined ? { httpStatus: e.httpStatus } : {}),
|
||||
...(e.detail ? { detail: e.detail } : {}),
|
||||
...(e.sample ? { sample: { ...e.sample, requestKind: kind, message } } : {}),
|
||||
}),
|
||||
);
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
if (res.status >= 300 && res.status < 400) {
|
||||
const loc = res.headers.get("location") ?? "";
|
||||
await res.body?.cancel().catch(() => {});
|
||||
@@ -155,7 +234,7 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap
|
||||
httpStatus: res.status,
|
||||
});
|
||||
}
|
||||
return readText(http, res);
|
||||
return { html: await readText(http, res), url, status: res.status, headers: res.headers };
|
||||
}
|
||||
|
||||
async function run(
|
||||
@@ -172,17 +251,19 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap
|
||||
page: Math.max(1, page) - 1, // 对外 1 起,MP 内部 0 起
|
||||
});
|
||||
if (!spider.canSearch()) throw new SiteError(def.id, "unsupported", `${what}:站点定义不支持该媒体类型`);
|
||||
const html = await fetchPage(spider.getSearchUrl(), what, opts);
|
||||
const parsed = spider.parse(html);
|
||||
const kind: SiteRequestKind = keyword ? "search" : "list";
|
||||
const page$ = await fetchPage(spider.getSearchUrl(), what, opts, kind);
|
||||
const parsed = spider.parse(page$.html);
|
||||
if (parsed.loginPage) throw new SiteError(def.id, "auth_expired", `${what}:返回登录或权限提示页`);
|
||||
if (parsed.isError || !parsed.$) throw new SiteError(def.id, "parse_error", `${what}:响应为空`);
|
||||
if (parsed.isError || !parsed.$) throw unrecognized(page$, what, kind, `${what}:响应为空`);
|
||||
const $ = parsed.$;
|
||||
if (parsed.torrents.length === 0) {
|
||||
const text = $("body").text();
|
||||
const hasList =
|
||||
$(normalizeSelector(spider.def.torrents.list.selector)).length > 0 || $("table.torrents").length > 0;
|
||||
const empty = (ext.emptyMarkers ?? []).some((m) => text.includes(m));
|
||||
if (!hasList && !empty) throw new SiteError(def.id, "parse_error", `${what}:页面结构无法识别(可能改版)`);
|
||||
// 认不出列表:先看是不是人机验证页(搜索页单独加盾、间歇触发),不是才报改版
|
||||
if (!hasList && !empty) throw unrecognized(page$, what, kind, `${what}:页面结构无法识别(可能改版)`);
|
||||
if (ext.loginSuccess && $(ext.loginSuccess).length === 0 && !empty) {
|
||||
throw new SiteError(def.id, "auth_expired", `${what}:未找到登录标志`);
|
||||
}
|
||||
@@ -237,18 +318,20 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap
|
||||
if (!/^\d+$/.test(siteReleaseId)) throw new SiteError(def.id, "not_found", "资源 ID 非法");
|
||||
if (!ext.detail) throw new SiteError(def.id, "unsupported", "站点定义未配置详情页解析");
|
||||
const url = new URL(`${ext.detail.path}?id=${siteReleaseId}&hit=1`, base).toString();
|
||||
const html = await fetchPage(url, `详情 ${siteReleaseId}`, opts);
|
||||
const r = parseDetailPage(html, def, siteReleaseId, http.now().toISOString());
|
||||
const detailPage = await fetchPage(url, `详情 ${siteReleaseId}`, opts, "detail");
|
||||
const r = parseDetailPage(detailPage.html, def, siteReleaseId, http.now().toISOString());
|
||||
if (r.kind === "login") throw new SiteError(def.id, "auth_expired", `详情 ${siteReleaseId}:返回登录页`);
|
||||
if (r.kind === "not_found") throw new SiteError(def.id, "not_found", `详情 ${siteReleaseId}:资源不存在`);
|
||||
if (!r.detail) throw new SiteError(def.id, "parse_error", `详情 ${siteReleaseId}:页面结构无法识别`);
|
||||
if (!r.detail) {
|
||||
throw unrecognized(detailPage, `详情 ${siteReleaseId}`, "detail", `详情 ${siteReleaseId}:页面结构无法识别`);
|
||||
}
|
||||
// 文件清单不在页面上(NexusPHP 点「查看列表」才 AJAX 取)→ 按需 1 次;失败只是没有文件清单
|
||||
const flPath = ext.detail.page?.fileListPath;
|
||||
if (flPath && r.detail.page && !r.detail.page.files.length) {
|
||||
try {
|
||||
const flUrl = new URL(flPath.replace("{id}", siteReleaseId), base).toString();
|
||||
const frag = await fetchPage(flUrl, `文件清单 ${siteReleaseId}`, opts);
|
||||
r.detail.page.files = filesFromFileListHtml(loadHtml(frag));
|
||||
const frag = await fetchPage(flUrl, `文件清单 ${siteReleaseId}`, opts, "detail");
|
||||
r.detail.page.files = filesFromFileListHtml(loadHtml(frag.html));
|
||||
} catch (e) {
|
||||
if (e instanceof SiteError && e.isAuthError) throw e;
|
||||
}
|
||||
@@ -317,8 +400,8 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap
|
||||
const started = Date.now();
|
||||
const checkedAt = http.now().toISOString();
|
||||
try {
|
||||
const html = await fetchPage(new URL("index.php", base).toString(), "健康检查", opts);
|
||||
const v = judgeLoginPage(html, ext.loginSuccess);
|
||||
const home = await fetchPage(new URL("index.php", base).toString(), "健康检查", opts, "health");
|
||||
const v = judgeLoginPage(home.html, ext.loginSuccess);
|
||||
const latencyMs = Date.now() - started;
|
||||
if (v.ok) return { siteId: def.id, status: "ok", ok: true, checkedAt, latencyMs, detail: v.detail };
|
||||
return {
|
||||
|
||||
Reference in new issue
Block a user