fix(adapters): 识别 200 状态的挑战 / 人机验证页,解析失败前兜底,失败页面留样
- detectChallenge 支持页面类请求(GET + Accept text/html)的 2xx 响应:正文只先读 256 KB, 短页且没有种子表 / 用户信息块时,认 Cloudflare 挑战标题 / 脚本与 Turnstile / hCaptcha / reCAPTCHA 门页; 带登录表单的页面不算门页。非页面请求不读正文 - siteFetch 用上面的判定;挑战错误附脱敏页面样本,文案带级联结果(x-umt-challenge) - spider 适配器:认不出列表 / 详情时先看是不是人机验证页,是就报 challenge(「搜索页需要人机验证」), 不再报改版;parse_error / challenge 都经 deps.onFailureSample 交出脱敏样本(前 64 KB,去掉个人信息块、 passkey / authkey / uid / 用户名 / Cookie / 邮箱) - classifyChallengePage 也认门页(浏览器过挑战时不再把门页当已通过) Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
4fa001750b
commit
1779e1a0f2
11 files changed
+2078
-34
No files matched your search
@@ -0,0 +1,124 @@
|
||||
import { redactString } from "@umt/contracts";
|
||||
import { redactText, redactUrl } from "../rss/redact.ts";
|
||||
|
||||
// 站点请求失败留样:请求报「页面结构无法识别」(parse_error)或「人机验证」(challenge)时,留一份脱敏后的页面开头,
|
||||
// 站点页「技术信息」里可看 / 下载,用来区分是站点改版还是被人机验证拦了(间歇触发的那一刻页面不会再出现)。
|
||||
|
||||
/** 页面样本最多保留的字节数 */
|
||||
export const FAILURE_SAMPLE_MAX_BYTES = 64 * 1024;
|
||||
|
||||
/** 请求类型:搜索 / 列表 / 详情 / 健康检测 / 其它 */
|
||||
export type SiteRequestKind = "search" | "list" | "detail" | "health" | "other";
|
||||
|
||||
/** 留样只记这几个响应头 */
|
||||
const KEPT_HEADERS = ["server", "cf-mitigated", "cf-ray", "content-type"] as const;
|
||||
|
||||
export interface SiteFailureSample {
|
||||
kind: "parse_error" | "challenge";
|
||||
requestKind: SiteRequestKind;
|
||||
/** 已脱敏 */
|
||||
url: string;
|
||||
httpStatus: number | null;
|
||||
headers: Record<string, string>;
|
||||
capturedAt: string;
|
||||
/** 报出去的错误文案 */
|
||||
message: string;
|
||||
/** 命中的挑战 / 门页规则 */
|
||||
rule: string | null;
|
||||
/** 脱敏后的页面开头(≤ FAILURE_SAMPLE_MAX_BYTES 字节) */
|
||||
body: string;
|
||||
/** 页面是否被截断 */
|
||||
truncated: boolean;
|
||||
}
|
||||
|
||||
/** URL 里除了通用凭据参数,还要抹掉的用户标识参数 */
|
||||
const EXTRA_KEYS = ["uid", "userid", "user_id"];
|
||||
const EMAIL_RE = /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g;
|
||||
/** NexusPHP passkey / authkey 等:32 位以上十六进制串(info hash 是 40 位,也一并抹掉,样本不需要) */
|
||||
const HEX_KEY_RE = /\b[a-f0-9]{32,64}\b/gi;
|
||||
/** Cookie 名=值(页面脚本里偶尔会出现) */
|
||||
const COOKIE_PAIR_RE =
|
||||
/\b(c_secure_[a-z_]+|cf_clearance|__cf_bm|phpsessid|PHPSESSID|session(?:id)?|remember_[a-z0-9_]+)=([^;\s"'<>&]+)/gi;
|
||||
/** 表单 / 脚本里的密钥字段:name="passkey" value="…"、passkey: "…" */
|
||||
const SECRET_FIELD_RE =
|
||||
/\b(pass[_-]?key|auth[_-]?key|torrent[_-]?pass|rss[_-]?key|api[_-]?key|token|csrf[_-]?token|secret)(["']?\s*(?:[:=]|value=)\s*["']?)([^"'\s<>&;,]{6,})/gi;
|
||||
/** 用户页链接里的用户 id 与链接文字(用户名) */
|
||||
const USER_LINK_RE = /(userdetails\.php\?id=)\d+/gi;
|
||||
const USER_NAME_RE = /(<a\b[^>]*userdetails\.php[^>]*>)((?:\s*<[^>]+>)*)([^<]+)/gi;
|
||||
|
||||
/** 页面里的「UID 12345」 */
|
||||
const UID_TEXT_RE = /\bUID\s*[:：]?\s*\d+/gi;
|
||||
|
||||
/**
|
||||
* 删掉 id 为 id 的整个元素(按同名标签计数找配对的结束标签);NexusPHP 的 `#info_block` 是当前用户的个人信息块
|
||||
* (用户名、分享率、魔力…),样本里不需要。找不到原样返回。
|
||||
*/
|
||||
function dropElementById(html: string, id: string): string {
|
||||
const open = new RegExp(`<([a-z][a-z0-9]*)\\b[^>]*\\bid=["']?${id}\\b[^>]*>`, "i").exec(html);
|
||||
if (!open) return html;
|
||||
const tag = (open[1] as string).toLowerCase();
|
||||
const re = new RegExp(`<(/?)${tag}\\b[^>]*>`, "gi");
|
||||
re.lastIndex = open.index + open[0].length;
|
||||
let depth = 1;
|
||||
for (let m = re.exec(html); m; m = re.exec(html)) {
|
||||
depth += m[1] ? -1 : 1;
|
||||
if (depth === 0) return `${html.slice(0, open.index)}<!-- #${id} 已删除 -->${html.slice(m.index + m[0].length)}`;
|
||||
}
|
||||
return `${html.slice(0, open.index)}<!-- #${id} 已删除 -->`;
|
||||
}
|
||||
|
||||
/** 页面文本脱敏:个人信息块、凭据参数、用户 id / 用户名、Cookie、邮箱、密钥样式的十六进制串 */
|
||||
export function redactPageHtml(html: string): string {
|
||||
return redactString(
|
||||
redactText(dropElementById(html, "info_block"), EXTRA_KEYS)
|
||||
.replace(UID_TEXT_RE, "UID [REDACTED]")
|
||||
.replace(USER_NAME_RE, "$1$2[USER]")
|
||||
.replace(USER_LINK_RE, "$1[REDACTED]")
|
||||
.replace(COOKIE_PAIR_RE, "$1=[REDACTED]")
|
||||
.replace(SECRET_FIELD_RE, "$1$2[REDACTED]")
|
||||
.replace(EMAIL_RE, "[EMAIL]")
|
||||
.replace(HEX_KEY_RE, "[HEX]"),
|
||||
);
|
||||
}
|
||||
|
||||
/** 先脱敏、再按字节截到上限(截断处的半个字符丢掉) */
|
||||
function clip(text: string, maxBytes: number): { body: string; truncated: boolean } {
|
||||
const bytes = new TextEncoder().encode(text);
|
||||
if (bytes.byteLength <= maxBytes) return { body: text, truncated: false };
|
||||
return { body: new TextDecoder().decode(bytes.subarray(0, maxBytes)).replace(/�$/, ""), truncated: true };
|
||||
}
|
||||
|
||||
export function buildFailureSample(input: {
|
||||
kind: SiteFailureSample["kind"];
|
||||
requestKind: SiteRequestKind;
|
||||
url: string;
|
||||
httpStatus?: number | null;
|
||||
headers?: Headers | null;
|
||||
message: string;
|
||||
rule?: string | null;
|
||||
html: string | null | undefined;
|
||||
now: Date;
|
||||
}): SiteFailureSample {
|
||||
const headers: Record<string, string> = {};
|
||||
for (const k of KEPT_HEADERS) {
|
||||
const v = input.headers?.get(k);
|
||||
if (v) headers[k] = v.slice(0, 200);
|
||||
}
|
||||
// 脱敏只看前 4 倍上限(截断前先脱敏,避免截断把密钥切成两半漏掉)
|
||||
const { body, truncated } = clip(
|
||||
redactPageHtml((input.html ?? "").slice(0, FAILURE_SAMPLE_MAX_BYTES * 4)),
|
||||
FAILURE_SAMPLE_MAX_BYTES,
|
||||
);
|
||||
return {
|
||||
kind: input.kind,
|
||||
requestKind: input.requestKind,
|
||||
url: redactUrl(input.url, EXTRA_KEYS),
|
||||
httpStatus: input.httpStatus ?? null,
|
||||
headers,
|
||||
capturedAt: input.now.toISOString(),
|
||||
message: input.message,
|
||||
rule: input.rule ?? null,
|
||||
body,
|
||||
truncated: truncated || (input.html?.length ?? 0) > FAILURE_SAMPLE_MAX_BYTES * 4,
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user