import { readFileSync } from "node:fs"; import { describe, expect, it } from "vitest"; import { audiencesDefinition, createSiteRateLimiter, createSpiderSiteAdapter, SiteError, type SiteFailureSample, staticSecretResolver, } from "../../index.ts"; // 搜索页单独加盾(间歇触发):返回 200 的挑战 / 人机验证页不再报「页面结构无法识别」,报 challenge 并留样; // 真正认不出的页面才报 parse_error(也留样)。 const AUD_SEARCH = readFileSync(new URL("../../../test/fixtures/audiences-search.html.txt", import.meta.url), "utf8"); const CF_200 = `Just a moment...
`; /** 站点外壳里的人机验证页:有用户信息块(网络层的严格判定不认),适配器解析失败时兜底认出 */ const SHELL_GATE = `Audiences :: 搜索
someone 退出

搜索前请完成人机验证

`; const LOGIN_WITH_TURNSTILE = `Audiences :: 登录 - Powered by NexusPHP
`; /** 改版页:带上各种要脱敏的东西 */ const SECRET = "0123456789abcdef0123456789abcdef"; const BROKEN = `Audiences :: 种子
欢迎 alice_cn 邮箱 alice@example.com
dl
${"
Dune
".repeat(5000)}
`; function adapter(body: string, status = 200, headers: Record = {}) { const samples: SiteFailureSample[] = []; const fetchImpl = (async () => new Response(body, { status, headers: { "content-type": "text/html; charset=utf-8", ...headers }, })) as (input: RequestInfo | URL, init?: RequestInit) => Promise; const a = createSpiderSiteAdapter(audiencesDefinition, { baseUrl: "https://aud.test/", secrets: staticSecretResolver({ [audiencesDefinition.id]: { cookie: "c_secure_uid=1; c_secure_pass=abc" } }), limiter: createSiteRateLimiter({ minIntervalMs: 0, burst: 10 }), fetch: fetchImpl, onFailureSample: (s) => samples.push(s), }); return { a, samples }; } async function err(p: Promise): Promise { try { await p; } catch (e) { expect(e).toBeInstanceOf(SiteError); return e as SiteError; } throw new Error("expected SiteError"); } describe("spider 搜索页人机验证", () => { it("正常搜索页(真实样本)照常解析,不留样", async () => { const { a, samples } = adapter(AUD_SEARCH); const page = await a.search("药屋少女的呢喃"); expect(page.items.length).toBeGreaterThan(0); expect(samples).toEqual([]); }); it("200 的 Cloudflare 挑战页 → challenge(网络层认出),文案说搜索页需要人机验证,留样标搜索", async () => { const { a, samples } = adapter(CF_200, 200, { server: "cloudflare", "cf-ray": "8c-HKG" }); const e = await err(a.search("Dune")); expect(e.kind).toBe("challenge"); expect(e.isAuthError).toBe(false); expect(e.message).toContain("搜索页需要人机验证"); expect(e.message).not.toContain("改版"); expect(samples).toHaveLength(1); expect(samples[0]).toMatchObject({ kind: "challenge", requestKind: "search", httpStatus: 200 }); expect(samples[0]?.headers["cf-ray"]).toBe("8c-HKG"); expect(samples[0]?.url).toContain("torrents.php"); }); it("站点外壳里的人机验证页:网络层不认,解析失败前兜底 → challenge(不报改版)", async () => { const { a, samples } = adapter(SHELL_GATE); const e = await err(a.search("Dune")); expect(e.kind).toBe("challenge"); expect(e.message).toContain("搜索页需要人机验证"); expect(e.detail?.rule).toContain("turnstile"); expect(samples[0]).toMatchObject({ kind: "challenge", requestKind: "search" }); expect(samples[0]?.body).not.toContain("29191"); expect(samples[0]?.body).not.toContain("someone"); }); it("列表页同样处理,文案按列表", async () => { const { a, samples } = adapter(SHELL_GATE); const e = await err(a.list(1)); expect(e.kind).toBe("challenge"); expect(e.message).toContain("列表页需要人机验证"); expect(samples[0]?.requestKind).toBe("list"); }); it("带 Turnstile 的登录页仍按登录失效处理,不留样", async () => { const { a, samples } = adapter(LOGIN_WITH_TURNSTILE); const e = await err(a.search("Dune")); expect(e.kind).toBe("auth_expired"); expect(samples).toEqual([]); }); it("真正认不出的页面 → parse_error,留脱敏样本(≤ 64 KB)", async () => { const { a, samples } = adapter(BROKEN, 200, { server: "nginx" }); const e = await err(a.search("Dune")); expect(e.kind).toBe("parse_error"); expect(e.message).toContain("页面结构无法识别"); expect(samples).toHaveLength(1); const s = samples[0] as SiteFailureSample; expect(s).toMatchObject({ kind: "parse_error", requestKind: "search", httpStatus: 200, truncated: true }); expect(new TextEncoder().encode(s.body).byteLength).toBeLessThanOrEqual(64 * 1024); for (const leak of [SECRET, "alice@example.com", "alice_cn", "29191", "topsecretvalue"]) { expect(s.body).not.toContain(leak); } expect(s.body).toContain("download.php?id=1"); }); });