import { redactString } from "@umt/contracts"; import { redactText, redactUrl } from "../rss/redact.ts"; // 站点请求失败留样:请求报「页面结构无法识别」(parse_error)或「人机验证」(challenge)时,留一份脱敏后的页面开头, // 站点页「技术信息」里可看 / 下载,用来区分是站点改版还是被人机验证拦了(间歇触发的那一刻页面不会再出现)。 /** 页面样本最多保留的字节数 */ export const FAILURE_SAMPLE_MAX_BYTES = 64 * 1024; /** 请求类型:搜索 / 列表 / 详情 / 健康检测 / 其它 */ export type SiteRequestKind = "search" | "list" | "detail" | "health" | "other"; /** 留样只记这几个响应头 */ const KEPT_HEADERS = ["server", "cf-mitigated", "cf-ray", "content-type"] as const; export interface SiteFailureSample { kind: "parse_error" | "challenge"; requestKind: SiteRequestKind; /** 已脱敏 */ url: string; httpStatus: number | null; headers: Record; capturedAt: string; /** 报出去的错误文案 */ message: string; /** 命中的挑战 / 门页规则 */ rule: string | null; /** 脱敏后的页面开头(≤ FAILURE_SAMPLE_MAX_BYTES 字节) */ body: string; /** 页面是否被截断 */ truncated: boolean; } /** URL 里除了通用凭据参数,还要抹掉的用户标识参数 */ const EXTRA_KEYS = ["uid", "userid", "user_id"]; const EMAIL_RE = /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g; /** NexusPHP passkey / authkey 等:32 位以上十六进制串(info hash 是 40 位,也一并抹掉,样本不需要) */ const HEX_KEY_RE = /\b[a-f0-9]{32,64}\b/gi; /** Cookie 名=值(页面脚本里偶尔会出现) */ const COOKIE_PAIR_RE = /\b(c_secure_[a-z_]+|cf_clearance|__cf_bm|phpsessid|PHPSESSID|session(?:id)?|remember_[a-z0-9_]+)=([^;\s"'<>&]+)/gi; /** 表单 / 脚本里的密钥字段:name="passkey" value="…"、passkey: "…" */ const SECRET_FIELD_RE = /\b(pass[_-]?key|auth[_-]?key|torrent[_-]?pass|rss[_-]?key|api[_-]?key|token|csrf[_-]?token|secret)(["']?\s*(?:[:=]|value=)\s*["']?)([^"'\s<>&;,]{6,})/gi; /** 用户页链接里的用户 id 与链接文字(用户名) */ const USER_LINK_RE = /(userdetails\.php\?id=)\d+/gi; const USER_NAME_RE = /(]*userdetails\.php[^>]*>)((?:\s*<[^>]+>)*)([^<]+)/gi; /** 页面里的「UID 12345」 */ const UID_TEXT_RE = /\bUID\s*[::]?\s*\d+/gi; /** * 删掉 id 为 id 的整个元素(按同名标签计数找配对的结束标签);NexusPHP 的 `#info_block` 是当前用户的个人信息块 * (用户名、分享率、魔力…),样本里不需要。找不到原样返回。 */ function dropElementById(html: string, id: string): string { const open = new RegExp(`<([a-z][a-z0-9]*)\\b[^>]*\\bid=["']?${id}\\b[^>]*>`, "i").exec(html); if (!open) return html; const tag = (open[1] as string).toLowerCase(); const re = new RegExp(`<(/?)${tag}\\b[^>]*>`, "gi"); re.lastIndex = open.index + open[0].length; let depth = 1; for (let m = re.exec(html); m; m = re.exec(html)) { depth += m[1] ? -1 : 1; if (depth === 0) return `${html.slice(0, open.index)}${html.slice(m.index + m[0].length)}`; } return `${html.slice(0, open.index)}`; } /** 页面文本脱敏:个人信息块、凭据参数、用户 id / 用户名、Cookie、邮箱、密钥样式的十六进制串 */ export function redactPageHtml(html: string): string { return redactString( redactText(dropElementById(html, "info_block"), EXTRA_KEYS) .replace(UID_TEXT_RE, "UID [REDACTED]") .replace(USER_NAME_RE, "$1$2[USER]") .replace(USER_LINK_RE, "$1[REDACTED]") .replace(COOKIE_PAIR_RE, "$1=[REDACTED]") .replace(SECRET_FIELD_RE, "$1$2[REDACTED]") .replace(EMAIL_RE, "[EMAIL]") .replace(HEX_KEY_RE, "[HEX]"), ); } /** 先脱敏、再按字节截到上限(截断处的半个字符丢掉) */ function clip(text: string, maxBytes: number): { body: string; truncated: boolean } { const bytes = new TextEncoder().encode(text); if (bytes.byteLength <= maxBytes) return { body: text, truncated: false }; return { body: new TextDecoder().decode(bytes.subarray(0, maxBytes)).replace(/�$/, ""), truncated: true }; } export function buildFailureSample(input: { kind: SiteFailureSample["kind"]; requestKind: SiteRequestKind; url: string; httpStatus?: number | null; headers?: Headers | null; message: string; rule?: string | null; html: string | null | undefined; now: Date; }): SiteFailureSample { const headers: Record = {}; for (const k of KEPT_HEADERS) { const v = input.headers?.get(k); if (v) headers[k] = v.slice(0, 200); } // 脱敏只看前 4 倍上限(截断前先脱敏,避免截断把密钥切成两半漏掉) const { body, truncated } = clip( redactPageHtml((input.html ?? "").slice(0, FAILURE_SAMPLE_MAX_BYTES * 4)), FAILURE_SAMPLE_MAX_BYTES, ); return { kind: input.kind, requestKind: input.requestKind, url: redactUrl(input.url, EXTRA_KEYS), httpStatus: input.httpStatus ?? null, headers, capturedAt: input.now.toISOString(), message: input.message, rule: input.rule ?? null, body, truncated: truncated || (input.html?.length ?? 0) > FAILURE_SAMPLE_MAX_BYTES * 4, }; }