merge: fix/n-site-challenge
Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei
This commit is contained in:
rootandClaude Opus 5.5 committed
commit
8306dfb544
29 files changed
+2793
-42
No files matched your search
@@ -17,6 +17,8 @@ import {
|
||||
SiteCredentialsPut,
|
||||
SiteCredentialsResult,
|
||||
SiteDefinitionView,
|
||||
SiteFailureSample,
|
||||
SiteFailureSampleSummary,
|
||||
SiteHealthResult,
|
||||
SiteRefreshLoginAccepted,
|
||||
SiteRefreshLoginInput,
|
||||
@@ -47,6 +49,7 @@ import {
|
||||
userDataHistory,
|
||||
} from "../service/account.ts";
|
||||
import { audit, closeDecisions, type Exec } from "../service/common.ts";
|
||||
import { getFailureSample, listFailureSamples } from "../service/failure-sample.ts";
|
||||
import { feedDefaultsForSite } from "../service/feeds.ts";
|
||||
import { assignReleaseIdentity } from "../service/identity-admin.ts";
|
||||
import { ensureRssSite, requireCapability, siteToApi, uniqueSiteKey } from "../site-kinds.ts";
|
||||
@@ -376,6 +379,37 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
|
||||
},
|
||||
);
|
||||
|
||||
route(
|
||||
app,
|
||||
{
|
||||
method: "get",
|
||||
path: "/sites/{id}/failure-samples",
|
||||
scope: "read",
|
||||
summary: "站点请求失败留样：最近几次页面结构无法识别 / 人机验证时的脱敏页面（列表不带正文）",
|
||||
tags: T_SITES,
|
||||
params: IdParam,
|
||||
response: z.object({ items: z.array(SiteFailureSampleSummary) }),
|
||||
},
|
||||
async ({ params }) => ({ items: await listFailureSamples(rt().ctx, params.id) }),
|
||||
);
|
||||
route(
|
||||
app,
|
||||
{
|
||||
method: "get",
|
||||
path: "/sites/{id}/failure-samples/{sampleId}",
|
||||
scope: "read",
|
||||
summary: "一份失败留样（含脱敏后的页面正文）",
|
||||
tags: T_SITES,
|
||||
params: z.object({ id: Id, sampleId: Id }),
|
||||
response: SiteFailureSample,
|
||||
},
|
||||
async ({ params }) => {
|
||||
const s = await getFailureSample(rt().ctx, params.id, params.sampleId);
|
||||
if (!s) throw new ApiException("not_found", "样本不存在");
|
||||
return s;
|
||||
},
|
||||
);
|
||||
|
||||
// ── W9-A 站点中心:删除、健康检测、刷新登录、用户数据、凭证 ──
|
||||
route(
|
||||
app,
|
||||
|
||||
@@ -26,6 +26,7 @@ import {
|
||||
} from "../../shared-cache.ts";
|
||||
import type { ServerContext } from "../types.ts";
|
||||
import { CloudflareGate, withClearance } from "./service/cloudflare.ts";
|
||||
import { failureSampleSink } from "./service/failure-sample.ts";
|
||||
import { readBrowserUserAgent, resolveUserAgent, type UserAgentDefaults } from "./service/user-agent.ts";
|
||||
|
||||
// 模块运行时依赖:站点适配器与限流器(每站一个实例,令牌在 PG 跨进程共享)、RSS 抓取器、TMDB 客户端、时钟。
|
||||
@@ -333,6 +334,8 @@ export class SourcesRuntime {
|
||||
// 每条请求都经 siteFetch:代理 + 按账号现场取 UA(BR-4)
|
||||
fetch: this.siteFetch({ id: site.id, proxy }, defaults, () => this.siteUserAgent(site.id)),
|
||||
...(baseUrl ? { baseUrl } : {}),
|
||||
// parse_error / challenge 的脱敏页面样本存进 site_failure_sample(每站最近 5 份)
|
||||
onFailureSample: failureSampleSink(this.ctx, site.id),
|
||||
});
|
||||
this.adapters.set(site.id, { sig, limiter, adapter });
|
||||
return adapter;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { detectChallenge, mergeCookieHeaders, parseCookieHeader } from "@umt/adapters";
|
||||
import { CHALLENGE_OUTCOME_HEADER, detectChallenge, mergeCookieHeaders, parseCookieHeader } from "@umt/adapters";
|
||||
import { type SiteCloudflareState, siteCloudflareState } from "@umt/contracts";
|
||||
import { type SiteAccountRow, type SiteRow, sql } from "@umt/db";
|
||||
import { egressKey, siteEgress } from "../../../egress.ts";
|
||||
@@ -11,12 +11,14 @@ import { resolveUserAgent } from "./user-agent.ts";
|
||||
// 站点 HTTP 请求(SourcesRuntime.siteFetch)经这里:
|
||||
// 0. 站点在"浏览器代理期"(config.cloudflare.browserUntil 未到)且请求是页面(GET + Accept text/html)→ 直接由浏览器代抓;
|
||||
// 1. 发 HTTP;账号 UA 跟随浏览器、出口与浏览器一致、复用没失败过时带上浏览器拿到的 cf_clearance(__cf_bm 不带);
|
||||
// 2. 响应是挑战(cf-mitigated: challenge,或 403 / 503 / 429 的挑战页)→ 不算凭据错误,记 lastChallengeAt;
|
||||
// 2. 响应是挑战(cf-mitigated: challenge,或 403 / 503 / 429 的挑战页,或页面请求返回 200 的挑战页 / 人机验证门页)
|
||||
// → 不算凭据错误,记 lastChallengeAt;
|
||||
// 3. 过挑战(同站 30 分钟最多一次、同一时刻只跑一次;只处理有登录态账号的站点):浏览器用该账号 profile 打开同一地址 →
|
||||
// ① 等它自己过(≤15 秒)→ ② 复选框 / Turnstile 派视觉子流程点一下(最多 2 次)→ ③ 选图 / 输码类转人工(待处理事项);
|
||||
// 4. 过了:读回 cf_clearance + navigator.userAgent;满足复用条件就存下来,HTTP 重试 1 次,结果记 cfReuse = ok / fails;
|
||||
// 5. 仍被挑战:页面请求改由浏览器导航取 HTML 交给原解析器,站点记 6 小时浏览器代理期;其它请求(下载、RSS、POST)照常报挑战。
|
||||
// 不接任何打码服务。
|
||||
// 没处理掉时返回的挑战响应带 `x-umt-challenge: <结果>`(needs_human / failed / throttled / no_browser / no_account),
|
||||
// 适配器据此写错误文案(自动处理到哪一步、要不要人)。不接任何打码服务。
|
||||
|
||||
/** 同站两次过挑战的最短间隔 */
|
||||
export const CF_ATTEMPT_INTERVAL_MS = 30 * 60_000;
|
||||
@@ -132,6 +134,13 @@ export function clearanceUsable(input: {
|
||||
);
|
||||
}
|
||||
|
||||
/** 挑战响应上标出级联结果(响应头不可改,重新包一层) */
|
||||
function tagOutcome(res: Response, outcome: string): Response {
|
||||
const headers = new Headers(res.headers);
|
||||
headers.set(CHALLENGE_OUTCOME_HEADER, outcome);
|
||||
return new Response(res.body, { status: res.status, statusText: res.statusText, headers });
|
||||
}
|
||||
|
||||
export class CloudflareGate {
|
||||
private readonly cache = new Map<string, { at: number; state: SiteCloudflareState }>();
|
||||
/** 进程内同站同一时刻只过一次挑战:后到的请求等同一个结果 */
|
||||
@@ -323,19 +332,19 @@ export class CloudflareGate {
|
||||
// 1. HTTP(满足条件时带 cf_clearance)
|
||||
// 从没遇到过挑战的站点不查账号(每条请求少两次查询)
|
||||
const extra = st.lastChallengeAt ? await this.clearanceCookie(siteId, req).catch(() => null) : null;
|
||||
const first = await detectChallenge(await send(extra));
|
||||
const first = await detectChallenge(await send(extra), { page: pagey });
|
||||
if (!first.rule) return first.res;
|
||||
// 2. 挑战:记下来;没有浏览器 / 没有登录态账号 → 原样返回(适配器报 challenge,不算凭据错误)
|
||||
await this.patch(siteId, { lastChallengeAt: now.toISOString(), lastRule: first.rule });
|
||||
const port = challengeBrowserFor(this.ctx);
|
||||
if (!port) return first.res;
|
||||
if (!port) return tagOutcome(first.res, "no_browser");
|
||||
const acc = await this.loggedInAccount(siteId);
|
||||
if (!acc) {
|
||||
await this.patch(siteId, { lastResult: "skipped", lastError: "站点没有已登录的账号，不处理挑战" });
|
||||
return first.res;
|
||||
return tagOutcome(first.res, "no_account");
|
||||
}
|
||||
const site = await this.ctx.db.selectFrom("site").selectAll().where("id", "=", siteId).executeTakeFirst();
|
||||
if (!site?.enabled || site.deleted_at) return first.res;
|
||||
if (!site?.enabled || site.deleted_at) return tagOutcome(first.res, "failed");
|
||||
// 3. 过挑战
|
||||
const out = await this.solveOnce(site, acc, req.url);
|
||||
st = await this.state(siteId, true);
|
||||
@@ -357,7 +366,7 @@ export class CloudflareGate {
|
||||
const retryExtra = await this.clearanceCookie(siteId, req).catch(() => null);
|
||||
if (retryExtra) {
|
||||
await this.rt.limiterOf(siteId)?.acquire();
|
||||
const second = await detectChallenge(await send(retryExtra));
|
||||
const second = await detectChallenge(await send(retryExtra), { page: pagey });
|
||||
if (!second.rule) {
|
||||
await this.patch(siteId, { cfReuse: "ok", lastResult: "reuse_ok", browserUntil: null });
|
||||
return second.res;
|
||||
@@ -387,7 +396,7 @@ export class CloudflareGate {
|
||||
return r;
|
||||
}
|
||||
}
|
||||
return first.res;
|
||||
return tagOutcome(first.res, !out ? "throttled" : out.status === "needs_human" ? "needs_human" : "failed");
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
import type { SiteFailureSample as AdapterSample } from "@umt/adapters";
|
||||
import type { SiteFailureSample, SiteFailureSampleSummary } from "@umt/contracts";
|
||||
import { sql } from "@umt/db";
|
||||
import type { ServerContext } from "../../types.ts";
|
||||
import { toApi } from "../rows.ts";
|
||||
|
||||
// 站点请求失败留样(表 site_failure_sample,迁移 0077):适配器报 parse_error / challenge 时交出脱敏页面样本
|
||||
// (SiteAdapterDeps.onFailureSample),这里存下来,每站只留最近 SAMPLES_PER_SITE 份。站点页「技术信息」里看。
|
||||
// 样本在适配器里已脱敏(个人信息块、passkey / authkey / uid / Cookie / 邮箱),这里不再处理正文。
|
||||
|
||||
/** 每站保留的样本数 */
|
||||
export const SAMPLES_PER_SITE = 5;
|
||||
|
||||
export async function recordFailureSample(ctx: ServerContext, siteId: string, s: AdapterSample): Promise<void> {
|
||||
await ctx.db
|
||||
.insertInto("site_failure_sample")
|
||||
.values({
|
||||
site_id: siteId,
|
||||
kind: s.kind,
|
||||
request_kind: s.requestKind,
|
||||
url: s.url,
|
||||
http_status: s.httpStatus,
|
||||
headers: JSON.stringify(s.headers),
|
||||
message: s.message.slice(0, 500),
|
||||
rule: s.rule,
|
||||
body: s.body,
|
||||
truncated: s.truncated,
|
||||
created_at: new Date(s.capturedAt),
|
||||
})
|
||||
.execute();
|
||||
await sql`delete from site_failure_sample where site_id = ${siteId} and id not in (
|
||||
select id from site_failure_sample where site_id = ${siteId} order by created_at desc, id desc limit ${SAMPLES_PER_SITE})`.execute(
|
||||
ctx.db,
|
||||
);
|
||||
}
|
||||
|
||||
/** 留样回调:不等写库,失败只记日志(不影响站点请求本身的结果) */
|
||||
export function failureSampleSink(ctx: ServerContext, siteId: string): (s: AdapterSample) => void {
|
||||
return (s) => {
|
||||
void recordFailureSample(ctx, siteId, s).catch((err) =>
|
||||
ctx.logger.warn({ siteId, err: { message: (err as Error).message } }, "站点失败页面留样写入失败"),
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
export async function listFailureSamples(ctx: ServerContext, siteId: string): Promise<SiteFailureSampleSummary[]> {
|
||||
const rows = await ctx.db
|
||||
.selectFrom("site_failure_sample")
|
||||
.select([
|
||||
"id",
|
||||
"site_id",
|
||||
"kind",
|
||||
"request_kind",
|
||||
"url",
|
||||
"http_status",
|
||||
"headers",
|
||||
"message",
|
||||
"rule",
|
||||
"truncated",
|
||||
"created_at",
|
||||
sql<number>`octet_length(body)`.as("body_bytes"),
|
||||
])
|
||||
.where("site_id", "=", siteId)
|
||||
.orderBy("created_at", "desc")
|
||||
.orderBy("id", "desc")
|
||||
.limit(SAMPLES_PER_SITE)
|
||||
.execute();
|
||||
return rows.map((r) => toApi<SiteFailureSampleSummary>(r));
|
||||
}
|
||||
|
||||
export async function getFailureSample(
|
||||
ctx: ServerContext,
|
||||
siteId: string,
|
||||
sampleId: string,
|
||||
): Promise<SiteFailureSample | null> {
|
||||
const row = await ctx.db
|
||||
.selectFrom("site_failure_sample")
|
||||
.selectAll()
|
||||
.where("site_id", "=", siteId)
|
||||
.where("id", "=", sampleId)
|
||||
.executeTakeFirst();
|
||||
return row ? toApi<SiteFailureSample>(row) : null;
|
||||
}
|
||||
@@ -21,9 +21,24 @@ import { createHarness, type Harness, pgAvailable } from "./harness.ts";
|
||||
// 浏览器一侧用假端口(ChallengeBrowserPort),真实浏览器流程见 browser-cloudflare.test.ts。
|
||||
|
||||
const up = await pgAvailable();
|
||||
|
||||
/** 轮询到有结果(留样是异步写入的) */
|
||||
async function until<T>(f: () => Promise<T[]>, ms = 5_000): Promise<T[]> {
|
||||
const deadline = Date.now() + ms;
|
||||
for (;;) {
|
||||
const r = await f();
|
||||
if (r.length || Date.now() > deadline) return r;
|
||||
await new Promise((res) => setTimeout(res, 50));
|
||||
}
|
||||
}
|
||||
const BROWSER_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:152.0) Gecko/20100101 Firefox/152.0";
|
||||
const CF_HTML = `<!DOCTYPE html><html><head><title>Just a moment...</title></head><body>
|
||||
<div id="challenge-spinner"></div><script>window._cf_chl_opt={cType: 'managed'}</script></body></html>`;
|
||||
/** 返回 200 的 Turnstile 门页(没有 cf-mitigated 头):搜索页单独加盾、间歇触发时见到的形态 */
|
||||
const GATE_HTML = `<!DOCTYPE html><html><head><title>Audiences</title>
|
||||
<script src="https://challenges.cloudflare.com/turnstile/v0/api.js" async defer></script></head>
|
||||
<body><p>请完成人机验证后继续</p><div class="cf-turnstile" data-sitekey="0x4AAAAAAA"></div>
|
||||
<a href="download.php?id=1&passkey=0123456789abcdef0123456789abcdef">x</a></body></html>`;
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 响应体断言
|
||||
const json = async (r: Response): Promise<any> => {
|
||||
@@ -75,6 +90,8 @@ describe.skipIf(!up)("CF-2 Cloudflare 级联", () => {
|
||||
pass: (h: Headers) => (h.get("cookie") ?? "").includes("cf_clearance=GOOD") && h.get("user-agent") === BROWSER_UA,
|
||||
challenged: 0,
|
||||
passed: 0,
|
||||
/** 非空:挑战响应改成 200 + 这段 HTML(不带 cf-mitigated 头) */
|
||||
okPage: null as string | null,
|
||||
};
|
||||
const seen: Headers[] = [];
|
||||
const solves: { url: string; accountId: string }[] = [];
|
||||
@@ -88,6 +105,12 @@ describe.skipIf(!up)("CF-2 Cloudflare 级联", () => {
|
||||
seen.push(hd);
|
||||
if (cf.on && !cf.pass(hd)) {
|
||||
cf.challenged++;
|
||||
if (cf.okPage) {
|
||||
return new Response(cf.okPage, {
|
||||
status: 200,
|
||||
headers: { "content-type": "text/html; charset=utf-8", server: "cloudflare", "cf-ray": "8c00-HKG" },
|
||||
});
|
||||
}
|
||||
return new Response(CF_HTML, {
|
||||
status: 403,
|
||||
headers: { "cf-mitigated": "challenge", "content-type": "text/html" },
|
||||
@@ -148,6 +171,7 @@ describe.skipIf(!up)("CF-2 Cloudflare 级联", () => {
|
||||
cf.pass = (hd) => (hd.get("cookie") ?? "").includes("cf_clearance=GOOD") && hd.get("user-agent") === BROWSER_UA;
|
||||
cf.challenged = 0;
|
||||
cf.passed = 0;
|
||||
cf.okPage = null;
|
||||
seen.length = 0;
|
||||
solves.length = 0;
|
||||
fetches.length = 0;
|
||||
@@ -305,6 +329,53 @@ describe.skipIf(!up)("CF-2 Cloudflare 级联", () => {
|
||||
expect(await state()).toMatchObject({ lastResult: "skipped" });
|
||||
});
|
||||
|
||||
it("页面请求返回 200 的 Turnstile 门页 → 进入级联:过挑战后带 cf_clearance 重试成功", async () => {
|
||||
cf.okPage = GATE_HTML;
|
||||
const page = await list();
|
||||
expect(page.items.length).toBeGreaterThan(0);
|
||||
expect(cf.challenged).toBe(1);
|
||||
expect(solves).toHaveLength(1);
|
||||
expect(fetches).toEqual([]);
|
||||
const st = await state();
|
||||
expect(st).toMatchObject({ cfReuse: "ok", lastResult: "reuse_ok" });
|
||||
expect(st?.lastRule).toMatch(/^gate\*=/);
|
||||
});
|
||||
|
||||
it("200 的 Cloudflare 挑战页且 HTTP 过不去 → 浏览器代抓,进入浏览器代理期", async () => {
|
||||
cf.okPage = CF_HTML;
|
||||
cf.pass = () => false;
|
||||
const page = await h.rt.siteAdapter(await site()).search("Dune");
|
||||
expect(page.items.length).toBeGreaterThan(0);
|
||||
expect(cf.challenged).toBe(2);
|
||||
expect(fetches).toHaveLength(1);
|
||||
expect(fetches[0]).toContain("search=Dune");
|
||||
expect((await state())?.browserUntil).toBeTruthy();
|
||||
});
|
||||
|
||||
it("200 门页转人工:搜索报 challenge(文案说要人在原站视图过验证),留样标搜索、已脱敏", async () => {
|
||||
cf.okPage = GATE_HTML;
|
||||
solveResult = { status: "needs_human", sessionId: null, decisionId: null, reason: "选图验证码" };
|
||||
await h.db.deleteFrom("site_failure_sample").where("site_id", "=", siteId).execute();
|
||||
const err = await h.rt
|
||||
.siteAdapter(await site())
|
||||
.search("Dune")
|
||||
.catch((e) => e);
|
||||
expect(err).toBeInstanceOf(SiteError);
|
||||
expect((err as SiteError).kind).toBe("challenge");
|
||||
expect((err as SiteError).message).toContain("搜索页需要人机验证");
|
||||
expect((err as SiteError).message).toContain("原站视图");
|
||||
expect((err as SiteError).message).not.toContain("改版");
|
||||
expect((await account()).status).toBe("active");
|
||||
const rows = await until(() =>
|
||||
h.db.selectFrom("site_failure_sample").selectAll().where("site_id", "=", siteId).execute(),
|
||||
);
|
||||
expect(rows[0]).toMatchObject({ kind: "challenge", request_kind: "search", http_status: 200 });
|
||||
expect(rows[0]?.headers).toMatchObject({ server: "cloudflare", "cf-ray": "8c00-HKG" });
|
||||
expect(rows[0]?.body).toContain("cf-turnstile");
|
||||
expect(rows[0]?.body).not.toContain("0123456789abcdef0123456789abcdef");
|
||||
expect(rows[0]?.url).toContain("search=Dune");
|
||||
});
|
||||
|
||||
it("健康检测遇挑战且过不去:记 challenge(degraded),账号状态不变", async () => {
|
||||
solveResult = { status: "failed", reason: "浏览器服务不可用" };
|
||||
fetchResult = { status: "unavailable", reason: "浏览器服务不可用" };
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
import type { SiteError } from "@umt/adapters";
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest";
|
||||
import { SAMPLES_PER_SITE } from "../../src/modules/sources-identity/service/failure-sample.ts";
|
||||
import { snapshotSite } from "../../src/modules/sources-identity/service/snapshot.ts";
|
||||
import { createHarness, type Harness, pgAvailable, READ_KEY } from "./harness.ts";
|
||||
|
||||
// 站点请求失败留样:列表 / 搜索报「页面结构无法识别」时存脱敏页面样本(site_failure_sample),每站只留最近 5 份;
|
||||
// 站点页技术信息经 GET /sites/{id}/failure-samples(列表不带正文)与 /failure-samples/{sampleId}(带正文)取。
|
||||
// 站点前面是可控的 fetch:返回什么页面由测试决定(不访问真实站点)。
|
||||
|
||||
const up = await pgAvailable();
|
||||
const SECRET = "0123456789abcdef0123456789abcdef";
|
||||
/** 改版页:认不出种子表,还带着要脱敏的东西 */
|
||||
const BROKEN = `<html><head><title>Audiences :: 种子</title></head><body>
|
||||
<table id="info_block"><tr><td>欢迎 <a href="userdetails.php?id=29191"><b>alice_cn</b></a> UID 29191</td></tr></table>
|
||||
<p>邮箱 [email protected]</p><a href="download.php?id=1&passkey=${SECRET}">dl</a>
|
||||
<script>var authkey = "${SECRET}"; document.cookie = "c_secure_pass=topsecretvalue";</script>
|
||||
<div class="new-grid"><div class="card">Dune</div></div></body></html>`;
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 响应体断言
|
||||
const json = async (r: Response): Promise<any> => {
|
||||
const body = await r.json();
|
||||
if (!r.ok) throw new Error(`${r.status} ${JSON.stringify(body)}`);
|
||||
return body.data;
|
||||
};
|
||||
|
||||
/** 轮询到条件成立(留样是异步写入的) */
|
||||
async function until<T>(f: () => Promise<T>, ok: (v: T) => boolean, ms = 5_000): Promise<T> {
|
||||
const deadline = Date.now() + ms;
|
||||
for (;;) {
|
||||
const v = await f();
|
||||
if (ok(v) || Date.now() > deadline) return v;
|
||||
await new Promise((r) => setTimeout(r, 50));
|
||||
}
|
||||
}
|
||||
|
||||
describe.skipIf(!up)("站点失败留样", () => {
|
||||
let h: Harness;
|
||||
let siteId: string;
|
||||
let page = BROKEN;
|
||||
const fetchImpl = (async () =>
|
||||
new Response(page, {
|
||||
status: 200,
|
||||
headers: { "content-type": "text/html; charset=utf-8", server: "nginx" },
|
||||
})) as typeof fetch;
|
||||
const site = () => h.db.selectFrom("site").selectAll().where("id", "=", siteId).executeTakeFirstOrThrow();
|
||||
const rows = () =>
|
||||
h.db.selectFrom("site_failure_sample").selectAll().where("site_id", "=", siteId).orderBy("created_at").execute();
|
||||
|
||||
beforeAll(async () => {
|
||||
h = await createHarness("failsample", { fetch: fetchImpl });
|
||||
const s = await json(
|
||||
await h.req("/sites", {
|
||||
body: {
|
||||
key: "audiences",
|
||||
name: "观众",
|
||||
adapter: "nexusphp",
|
||||
baseUrl: "https://aud.test/",
|
||||
rateLimit: { minIntervalMs: 0, concurrency: 10 },
|
||||
},
|
||||
}),
|
||||
);
|
||||
siteId = s.id;
|
||||
await json(
|
||||
await h.req(`/sites/${siteId}/credentials`, {
|
||||
method: "PUT",
|
||||
body: { kind: "cookie", cookie: "c_secure_uid=1; c_secure_pass=abc" },
|
||||
}),
|
||||
);
|
||||
});
|
||||
afterAll(async () => {
|
||||
await h?.close();
|
||||
});
|
||||
beforeEach(async () => {
|
||||
page = BROKEN;
|
||||
await h.db.deleteFrom("site_failure_sample").where("site_id", "=", siteId).execute();
|
||||
});
|
||||
|
||||
it("列表快照报 parse_error → 留脱敏样本(URL、状态码、关键响应头、请求类型);接口可看", async () => {
|
||||
const r = await snapshotSite(h.rt, { siteId, kind: "manual" });
|
||||
expect(r.status).toBe("failed");
|
||||
expect(r.error).toContain("parse_error");
|
||||
const got = await until(rows, (x) => x.length > 0);
|
||||
expect(got).toHaveLength(1);
|
||||
const s = got[0]!;
|
||||
expect(s).toMatchObject({
|
||||
kind: "parse_error",
|
||||
request_kind: "list",
|
||||
http_status: 200,
|
||||
headers: { server: "nginx" },
|
||||
});
|
||||
expect(s.message).toContain("页面结构无法识别");
|
||||
for (const leak of [SECRET, "[email protected]", "alice_cn", "29191", "topsecretvalue"]) {
|
||||
expect(s.body).not.toContain(leak);
|
||||
expect(s.url).not.toContain(leak);
|
||||
}
|
||||
expect(s.body).toContain("new-grid");
|
||||
|
||||
const list = await json(await h.req(`/sites/${siteId}/failure-samples`, { key: READ_KEY }));
|
||||
expect(list.items).toHaveLength(1);
|
||||
expect(list.items[0]).toMatchObject({ id: s.id, kind: "parse_error", requestKind: "list", httpStatus: 200 });
|
||||
expect(list.items[0].body).toBeUndefined();
|
||||
expect(list.items[0].bodyBytes).toBeGreaterThan(0);
|
||||
const one = await json(await h.req(`/sites/${siteId}/failure-samples/${s.id}`, { key: READ_KEY }));
|
||||
expect(one.body).toBe(s.body);
|
||||
const miss = await h.req(`/sites/${siteId}/failure-samples/00000000-0000-4000-8000-000000000000`);
|
||||
expect(miss.status).toBe(404);
|
||||
});
|
||||
|
||||
it(`每站只留最近 ${SAMPLES_PER_SITE} 份`, async () => {
|
||||
const adapter = h.rt.siteAdapter(await site());
|
||||
for (let i = 1; i <= 7; i++) {
|
||||
const e = await adapter.search(`k${i}`).catch((x) => x);
|
||||
expect((e as SiteError).kind).toBe("parse_error");
|
||||
}
|
||||
const kw = (x: { url: string }[]) => x.map((r) => /search=(k\d)/.exec(r.url)?.[1]).join(",");
|
||||
const got = await until(rows, (x) => kw(x) === "k3,k4,k5,k6,k7");
|
||||
expect(kw(got)).toBe("k3,k4,k5,k6,k7");
|
||||
expect(got.every((r) => r.request_kind === "search")).toBe(true);
|
||||
});
|
||||
|
||||
it("正常页面不留样", async () => {
|
||||
page = `<html><head><title>Audiences :: 种子</title></head><body><table class="torrents"><tr><td>没有种子</td></tr></table></body></html>`;
|
||||
const adapter = h.rt.siteAdapter(await site());
|
||||
await adapter.search("x").catch(() => null);
|
||||
await new Promise((r) => setTimeout(r, 200));
|
||||
expect(await rows()).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -9,6 +9,8 @@ import {
|
||||
type SiteCredentialsBody,
|
||||
SiteCredentialsResult,
|
||||
SiteDefinitionView,
|
||||
SiteFailureSample,
|
||||
SiteFailureSampleSummary,
|
||||
SiteHealthResult,
|
||||
SiteListItem,
|
||||
SiteRefreshLoginAccepted,
|
||||
@@ -35,6 +37,8 @@ export const siteKeys = {
|
||||
accounts: (siteId?: string) => ["sites", "accounts", siteId ?? "all"] as const,
|
||||
snapshots: (siteId: string) => ["sites", "snapshots", siteId] as const,
|
||||
requestStats: (siteId: string) => ["sites", "request-stats", siteId] as const,
|
||||
failureSamples: (siteId: string) => ["sites", "failure-samples", siteId] as const,
|
||||
failureSample: (siteId: string, sampleId: string) => ["sites", "failure-samples", siteId, sampleId] as const,
|
||||
definitions: () => ["sites", "definitions"] as const,
|
||||
userData: (siteId: string, accountId?: string, days = 30) =>
|
||||
["sites", "user-data", siteId, accountId ?? "first", days] as const,
|
||||
@@ -105,6 +109,33 @@ export function useSiteRequestStats(siteId: string | undefined) {
|
||||
});
|
||||
}
|
||||
|
||||
/** 最近的失败留样(页面结构无法识别 / 人机验证;每站最近 5 份,不带正文) */
|
||||
export function useSiteFailureSamples(siteId: string | undefined) {
|
||||
return useQuery({
|
||||
queryKey: siteKeys.failureSamples(siteId ?? ""),
|
||||
enabled: !!siteId,
|
||||
queryFn: ({ signal }) =>
|
||||
apiParsed(z.object({ items: z.array(SiteFailureSampleSummary) }), "/api/v1/sites/:id/failure-samples", {
|
||||
pathParams: { id: siteId ?? "" },
|
||||
signal,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
/** 一份失败留样(带脱敏后的页面正文);展开时才取 */
|
||||
export function useSiteFailureSample(siteId: string, sampleId: string, enabled: boolean) {
|
||||
return useQuery({
|
||||
queryKey: siteKeys.failureSample(siteId, sampleId),
|
||||
enabled,
|
||||
staleTime: Number.POSITIVE_INFINITY,
|
||||
queryFn: ({ signal }) =>
|
||||
apiParsed(SiteFailureSample, "/api/v1/sites/:id/failure-samples/:sampleId", {
|
||||
pathParams: { id: siteId, sampleId },
|
||||
signal,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
/** 访问默认的一句话说明:"快照每 60 分钟 1 页 · 健康检测每 6 小时 · 详情缓存 24 小时" */
|
||||
export function accessSummary(s: SiteRequestStats): string {
|
||||
const dur = (sec: number) => (sec % 3600 === 0 ? `${sec / 3600} 小时` : `${Math.round(sec / 60)} 分钟`);
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
import type { SiteFailureSample } from "@umt/contracts";
|
||||
import { createElement } from "react";
|
||||
import { renderToStaticMarkup } from "react-dom/server";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { SiteFailureSamples } from "./site-failure-samples.tsx";
|
||||
import { failureSampleFileName, failureSampleItems, failureSampleText, failureSampleTitle } from "./site-fields.ts";
|
||||
|
||||
// 站点页技术信息里的失败样本:一行说清是搜索 / 列表、人机验证还是页面结构变化;展开看地址、响应头、页面,可下载文本。
|
||||
|
||||
const sample: SiteFailureSample = {
|
||||
id: "0f1e2d3c-4b5a-4987-8a6b-5c4d3e2f1a0b",
|
||||
siteId: "1a2b3c4d-5e6f-4a7b-8c9d-0e1f2a3b4c5d",
|
||||
kind: "challenge",
|
||||
requestKind: "search",
|
||||
url: "https://audiences.me/torrents.php?search=Dune&passkey=[REDACTED]",
|
||||
httpStatus: 200,
|
||||
headers: { server: "cloudflare", "cf-ray": "8c00-HKG" },
|
||||
message: "搜索：搜索页需要人机验证（Cloudflare 等），需要你在原站视图里过一次验证",
|
||||
rule: "gate*=cf-turnstile",
|
||||
body: '<html><div class="cf-turnstile"></div></html>',
|
||||
truncated: false,
|
||||
createdAt: "2026-10-06T07:30:05.000Z",
|
||||
};
|
||||
const { body: _b, ...summary } = sample;
|
||||
const row = { ...summary, bodyBytes: 44 };
|
||||
|
||||
describe("站点失败样本展示", () => {
|
||||
it("标题区分请求类型与失败类型", () => {
|
||||
expect(failureSampleTitle(sample)).toBe("搜索 · 人机验证 · HTTP 200");
|
||||
expect(failureSampleTitle({ kind: "parse_error", requestKind: "list", httpStatus: null })).toBe(
|
||||
"列表 · 页面结构无法识别",
|
||||
);
|
||||
});
|
||||
|
||||
it("技术信息条目:地址、错误、规则、关键响应头、页面大小", () => {
|
||||
const items = failureSampleItems(row);
|
||||
expect(items.map((i) => i.label)).toEqual(["地址", "错误", "命中规则", "server", "cf-ray", "页面"]);
|
||||
expect(items[0]?.value).toContain("passkey=[REDACTED]");
|
||||
});
|
||||
|
||||
it("下载文本带请求信息头;文件名带请求类型、类型与时间", () => {
|
||||
const text = failureSampleText(sample);
|
||||
expect(text).toContain("# 地址：https://audiences.me/torrents.php");
|
||||
expect(text).toContain("# cf-ray: 8c00-HKG");
|
||||
expect(text.endsWith(sample.body)).toBe(true);
|
||||
expect(failureSampleFileName(sample)).toBe("site-sample-search-challenge-20261006-073005.html.txt");
|
||||
});
|
||||
|
||||
it("列表渲染:每份一行(收起),没有样本不渲染", () => {
|
||||
const html = renderToStaticMarkup(createElement(SiteFailureSamples, { siteId: sample.siteId, items: [row] }));
|
||||
expect(html).toContain("最近的失败页面");
|
||||
expect(html).toContain("搜索 · 人机验证 · HTTP 200");
|
||||
expect(html).toContain('aria-expanded="false"');
|
||||
expect(renderToStaticMarkup(createElement(SiteFailureSamples, { siteId: sample.siteId, items: [] }))).toBe("");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,81 @@
|
||||
import type { SiteFailureSample, SiteFailureSampleSummary } from "@umt/contracts";
|
||||
import { Download } from "lucide-react";
|
||||
import { useState } from "react";
|
||||
import { useSiteFailureSample } from "~/api/sites.ts";
|
||||
import { ErrorState, LoadingState } from "~/components/console/empty-state.tsx";
|
||||
import { KvList } from "~/components/console/kv-list.tsx";
|
||||
import { Button } from "~/components/ui/button.tsx";
|
||||
import { relTime } from "~/components/workbench/release-fields.ts";
|
||||
import { failureSampleFileName, failureSampleItems, failureSampleText, failureSampleTitle } from "./site-fields.ts";
|
||||
|
||||
// 站点请求失败留样(技术信息里):最近几次「页面结构无法识别」/「人机验证」时站点返回的页面(已脱敏),
|
||||
// 用来分清是站点改版还是被人机验证拦了。列表不带正文,展开时再取。
|
||||
|
||||
function download(s: SiteFailureSample) {
|
||||
const url = URL.createObjectURL(new Blob([failureSampleText(s)], { type: "text/plain;charset=utf-8" }));
|
||||
const a = document.createElement("a");
|
||||
a.href = url;
|
||||
a.download = failureSampleFileName(s);
|
||||
a.click();
|
||||
URL.revokeObjectURL(url);
|
||||
}
|
||||
|
||||
function SampleBody({ siteId, sampleId }: { siteId: string; sampleId: string }) {
|
||||
const q = useSiteFailureSample(siteId, sampleId, true);
|
||||
if (q.isPending) return <LoadingState rows={2} />;
|
||||
if (q.error) return <ErrorState plain title="读取样本失败" error={q.error} onRetry={() => void q.refetch()} />;
|
||||
return (
|
||||
<div className="grid gap-2">
|
||||
<div>
|
||||
<Button size="xs" variant="outline" onClick={() => download(q.data)}>
|
||||
<Download aria-hidden />
|
||||
下载文本
|
||||
</Button>
|
||||
</div>
|
||||
<pre className="max-h-80 overflow-auto rounded-lg bg-muted/40 p-2 font-mono text-caption break-all whitespace-pre-wrap text-foreground">
|
||||
{q.data.body || "（页面是空的）"}
|
||||
</pre>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/** 一份样本:标题一行,展开看请求信息与页面 */
|
||||
export function FailureSampleRow({ siteId, sample }: { siteId: string; sample: SiteFailureSampleSummary }) {
|
||||
const [open, setOpen] = useState(false);
|
||||
return (
|
||||
<li className="min-w-0 rounded-lg border border-border/60 px-3 py-2" data-slot="failure-sample">
|
||||
<button
|
||||
type="button"
|
||||
aria-expanded={open}
|
||||
onClick={() => setOpen(!open)}
|
||||
className="flex w-full flex-wrap items-baseline justify-between gap-x-3 text-left text-sm text-foreground"
|
||||
>
|
||||
<span>{failureSampleTitle(sample)}</span>
|
||||
<span className="text-meta text-muted-foreground">
|
||||
{relTime(sample.createdAt)} · {open ? "收起" : "查看"}
|
||||
</span>
|
||||
</button>
|
||||
{open ? (
|
||||
<div className="mt-2 grid gap-3">
|
||||
<KvList columns={1} items={failureSampleItems(sample)} />
|
||||
<SampleBody siteId={siteId} sampleId={sample.id} />
|
||||
</div>
|
||||
) : null}
|
||||
</li>
|
||||
);
|
||||
}
|
||||
|
||||
/** 最近的失败样本(列表由调用方用 useSiteFailureSamples 取);没有就不渲染 */
|
||||
export function SiteFailureSamples({ siteId, items }: { siteId: string; items: SiteFailureSampleSummary[] }) {
|
||||
if (!items.length) return null;
|
||||
return (
|
||||
<section className="grid gap-2">
|
||||
<h3 className="text-meta text-muted-foreground">最近的失败页面（已脱敏，留最近 5 份）</h3>
|
||||
<ul className="grid gap-2">
|
||||
{items.map((s) => (
|
||||
<FailureSampleRow key={s.id} siteId={siteId} sample={s} />
|
||||
))}
|
||||
</ul>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
@@ -6,6 +6,8 @@ import {
|
||||
type SiteAuthKind,
|
||||
type SiteCapability,
|
||||
type SiteDefinitionView,
|
||||
type SiteFailureSample,
|
||||
type SiteFailureSampleSummary,
|
||||
type SiteKind,
|
||||
type SiteLastHealth,
|
||||
type SiteListItem,
|
||||
@@ -240,6 +242,58 @@ export function cloudflareTechItems(config: unknown): { label: string; value: st
|
||||
].filter((it) => it.value);
|
||||
}
|
||||
|
||||
const SAMPLE_KIND: Record<string, string> = { parse_error: "页面结构无法识别", challenge: "人机验证" };
|
||||
const SAMPLE_REQUEST: Record<string, string> = {
|
||||
search: "搜索",
|
||||
list: "列表",
|
||||
detail: "详情",
|
||||
health: "健康检测",
|
||||
other: "请求",
|
||||
};
|
||||
|
||||
/** 失败样本一行:"搜索 · 人机验证 · HTTP 200" */
|
||||
export function failureSampleTitle(s: Pick<SiteFailureSampleSummary, "kind" | "requestKind" | "httpStatus">): string {
|
||||
return [
|
||||
SAMPLE_REQUEST[s.requestKind] ?? s.requestKind,
|
||||
SAMPLE_KIND[s.kind] ?? s.kind,
|
||||
s.httpStatus !== null ? `HTTP ${s.httpStatus}` : null,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(" · ");
|
||||
}
|
||||
|
||||
/** 失败样本的技术信息条目(地址、响应头、规则、文案) */
|
||||
export function failureSampleItems(s: SiteFailureSampleSummary): { label: string; value: string; mono?: boolean }[] {
|
||||
return [
|
||||
{ label: "地址", value: s.url, mono: true },
|
||||
{ label: "错误", value: s.message },
|
||||
{ label: "命中规则", value: s.rule ?? "", mono: true },
|
||||
...Object.entries(s.headers).map(([k, v]) => ({ label: k, value: v, mono: true })),
|
||||
{ label: "页面", value: `${formatBytes(s.bodyBytes)}${s.truncated ? "（只留了开头）" : ""}` },
|
||||
].filter((it) => it.value);
|
||||
}
|
||||
|
||||
/** 下载用的文本:头部是请求信息,后面是脱敏后的页面 */
|
||||
export function failureSampleText(s: SiteFailureSample): string {
|
||||
const head = [
|
||||
`# 站点失败样本（已脱敏）`,
|
||||
`# 时间：${s.createdAt}`,
|
||||
`# 类型：${failureSampleTitle(s)}`,
|
||||
`# 地址：${s.url}`,
|
||||
...Object.entries(s.headers).map(([k, v]) => `# ${k}: ${v}`),
|
||||
...(s.rule ? [`# 规则：${s.rule}`] : []),
|
||||
`# 错误：${s.message}`,
|
||||
...(s.truncated ? ["# 页面超过 64 KB，只留了开头"] : []),
|
||||
];
|
||||
return `${head.join("\n")}\n\n${s.body}`;
|
||||
}
|
||||
|
||||
/** 下载文件名:site-sample-<请求类型>-<类型>-<时间>.html.txt */
|
||||
export function failureSampleFileName(s: Pick<SiteFailureSample, "kind" | "requestKind" | "createdAt">): string {
|
||||
const t = s.createdAt.replace(/[-:]/g, "").replace(/\..*$/, "").replace("T", "-");
|
||||
return `site-sample-${s.requestKind}-${s.kind}-${t}.html.txt`;
|
||||
}
|
||||
|
||||
/** 限流:"20 次/分";空对象 = 默认 */
|
||||
export function rateLimitText(r: SiteRateLimit | null | undefined): string {
|
||||
if (!r || (!r.requestsPerMinute && !r.minIntervalMs && !r.concurrency)) return "默认";
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import type { SiteListItem } from "@umt/contracts";
|
||||
import { Link } from "react-router";
|
||||
import { useFeeds } from "~/api/feeds.ts";
|
||||
import { accessSummary, useSiteRequestStats, useSiteUserData } from "~/api/sites.ts";
|
||||
import { accessSummary, useSiteFailureSamples, useSiteRequestStats, useSiteUserData } from "~/api/sites.ts";
|
||||
import { DetailSection } from "~/components/console/detail-sheet.tsx";
|
||||
import { ErrorState, LoadingState } from "~/components/console/empty-state.tsx";
|
||||
import { KvList } from "~/components/console/kv-list.tsx";
|
||||
@@ -10,6 +10,7 @@ import { StatTile } from "~/components/console/stat-tile.tsx";
|
||||
import { StatusBadge } from "~/components/console/status-badge.tsx";
|
||||
import { TechInfo } from "~/components/console/tech-info.tsx";
|
||||
import { relTime } from "~/components/workbench/release-fields.ts";
|
||||
import { SiteFailureSamples } from "./site-failure-samples.tsx";
|
||||
import {
|
||||
ACCOUNT_OVERALL,
|
||||
AUTH_KIND_LABEL,
|
||||
@@ -116,6 +117,7 @@ function AccountSiteOverview({ site }: { site: SiteListItem }) {
|
||||
const u = dAcc?.userData ?? null;
|
||||
const history = useSiteUserData(site.id, dAcc?.id, 30);
|
||||
const stats = useSiteRequestStats(site.id);
|
||||
const samples = useSiteFailureSamples(site.id).data?.items ?? [];
|
||||
const st = ACCOUNT_OVERALL[overallStatus(site)];
|
||||
const h = healthLine(site.lastHealth, site.lastHealthAt);
|
||||
|
||||
@@ -215,7 +217,9 @@ function AccountSiteOverview({ site }: { site: SiteListItem }) {
|
||||
className="mt-2"
|
||||
columns={1}
|
||||
items={[...healthTechItems(site.lastHealth), ...cloudflareTechItems(site.config)]}
|
||||
/>
|
||||
>
|
||||
{samples.length ? <SiteFailureSamples siteId={site.id} items={samples} /> : null}
|
||||
</TechInfo>
|
||||
</DetailSection>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
+47
-1
@@ -165,7 +165,8 @@
|
||||
(`solveSiteChallenge` / `fetchSiteHtml`)、`modules/agent/challenge.ts`(`systemChallengeVision`)。
|
||||
|
||||
**识别**:站点响应带 `cf-mitigated: challenge`(任何状态码),或 403 / 503 / 429 且正文是挑战页(标题 Just a moment / Attention Required、
|
||||
`cf_chl_opt` 等,不认正常页面也会注入的 `/cdn-cgi/challenge-platform/`)→ `SiteError("challenge")`。**不是凭据错误**:快照 / 详情不调
|
||||
`cf_chl_opt` 等,不认正常页面也会注入的 `/cdn-cgi/challenge-platform/`),或页面请求返回 200 的挑战页 / 人机验证门页(见下「搜索页人机验证」)
|
||||
→ `SiteError("challenge")`。**不是凭据错误**:快照 / 详情不调
|
||||
`onAuthError`、不置账号失效、不排自动重登;健康检测记 `challenge`(degraded);刷新登录 / 用户数据遇挑战只写 `last_error`,账号状态不变
|
||||
(验证码、二次验证仍是账号状态 `challenge`)。
|
||||
|
||||
@@ -202,6 +203,51 @@
|
||||
**要人处理时**:站点页 Cloudflare 行显示「需要你验证」→ 原站视图接管 → 过验证 → 交还。一直过不去(站点绑定 TLS 指纹、代理 IP 被拉黑):
|
||||
关掉站点代理或换出口;实在不行关闭该站点的定时快照,只保留 RSS。
|
||||
|
||||
### 搜索页人机验证(200 状态的挑战页)
|
||||
|
||||
现象(2026-10-06,观众):只有搜索页单独加了盾、实际搜索时间歇触发,列表快照与健康检测都正常;应用当时报
|
||||
`parse_error: 搜索:页面结构无法识别(可能改版)`,事后重搜又好了,触发那一刻的页面没留下。原因是挑战 / 人机验证页可能以 **200** 返回,
|
||||
以前只看响应头和 403 / 503 / 429,这类页面落到「页面结构无法识别」,级联完全没触发。现在分两层:
|
||||
|
||||
1. **网络层**(`challenge.ts detectChallenge`,经 `cloudflare.ts handle` 与 `http.ts siteFetch`):页面请求(GET + Accept `text/html`)
|
||||
的 2xx HTML 响应也看正文,只先读 256 KB(`CHALLENGE_PEEK_BYTES`,其余照常流给解析器;非页面请求不读)。判定要同时满足:
|
||||
正文已读完且不超过 10 万字符、没有站点正常内容(`id="torrenttable"`、`table.torrents`、`logout.php`、`userdetails.php?id=`、
|
||||
`usercp.php`、`#info_block` 任一出现就不算),再命中 Cloudflare 挑战标题 / 脚本,或人机验证组件(Turnstile:`cf-turnstile`、
|
||||
`challenges.cloudflare.com/turnstile`;hCaptcha;reCAPTCHA)且没有登录表单(`type=password` / `takelogin.php`,登录页带 Turnstile
|
||||
仍按登录失效处理)。命中即进入上面的 A→B 级联(过挑战 → cf_clearance 回带重试 → 浏览器代抓)。
|
||||
2. **适配器兜底**(`spider/adapter.ts`):列表 / 搜索 / 详情认不出结构、即将报「页面结构无法识别」前,用
|
||||
`challengeRuleOfUnrecognizedPage` 再看一次(不限长度、不看站点内容;也认页面文字里的「人机验证 / verify you are human」),
|
||||
像就报 `challenge`,不报改版。这类页面级联没跑过,文案提示去原站视图过一次。
|
||||
|
||||
错误文案(搜索覆盖、快照错误里能看到):
|
||||
- 人机验证:`搜索：搜索页需要人机验证（Cloudflare 等），<处理结果>`。处理结果来自级联打在挑战响应上的 `x-umt-challenge` 头:
|
||||
「自动处理没能通过，需要你在原站视图里过一次验证」(needs_human)、「已自动尝试用浏览器通过，没有成功」(failed)、
|
||||
「30 分钟内已自动尝试过，之后会再试」(throttled)、「没有可用的浏览器，无法自动通过」、「站点没有已登录的账号，不自动处理」;
|
||||
适配器兜底认出的是「这类页面没有自动处理，可以在原站视图里过一次验证」。
|
||||
- 页面结构变化:`搜索:页面结构无法识别(可能改版)（页面样本见站点页技术信息）`。
|
||||
|
||||
### 失败样本在哪看
|
||||
|
||||
站点请求报 `parse_error`(页面结构无法识别 / 响应为空)或 `challenge` 时,适配器交出一份脱敏页面样本(`SiteAdapterDeps.onFailureSample`),
|
||||
`runtime.siteAdapter` 接到后异步写进表 `site_failure_sample`(迁移 0077),**每站只留最近 5 份**,写失败只记日志。覆盖所有走
|
||||
`siteAdapter` 的调用:作品页 / 搜索页的搜索、列表快照、详情刷新、健康检测(首页遇挑战)。目前只有 NexusPHP 通用 spider 适配器留样。
|
||||
|
||||
- 内容:请求类型(search / list / detail / health)、类型(parse_error / challenge)、脱敏 URL、状态码、响应头 `server` / `cf-mitigated` /
|
||||
`cf-ray` / `content-type`、错误文案、命中规则、时间、页面前 64 KB(超出标「只留了开头」)。
|
||||
- 脱敏(`packages/adapters/src/sites/failure-sample.ts redactPageHtml`):整块删掉 `#info_block`(当前用户的用户名、分享率、魔力);
|
||||
URL 凭据参数(passkey / authkey / rsskey / token / key …)与 uid 参数、`userdetails.php?id=` 的 id 与链接文字、「UID 12345」、
|
||||
Cookie 名值(`c_secure_*`、`cf_clearance`、`PHPSESSID` …)、表单 / 脚本里的密钥字段、邮箱、32～64 位十六进制串(passkey 形态)。
|
||||
- 看:站点页 → 概览 →「站点」一栏的「技术信息」→「最近的失败页面」,展开看请求信息与页面,可「下载文本」;
|
||||
接口 `GET /api/v1/sites/{id}/failure-samples`(不带正文)、`GET /api/v1/sites/{id}/failure-samples/{sampleId}`(带正文)。
|
||||
- 排查:类型是人机验证、页面是 Turnstile / 挑战页 → 站点加了盾,看 Cloudflare 行与级联记录;类型是页面结构无法识别、页面是正常的
|
||||
种子页 → 站点改版,按样本改站点定义;页面是站点自己的提示页(如「搜索太频繁」)→ 看站点访问默认与限流。
|
||||
|
||||
**测试**:`packages/adapters/src/sites/challenge.test.ts`(200 挑战页 / 门页、正常搜索页 / 列表页 / 带 Turnstile 的登录页不误判、
|
||||
读取上限)、`spider/challenge-page.test.ts`(搜索页 / 列表页兜底、登录页不受影响、parse_error 留样脱敏)、`failure-sample.test.ts`、
|
||||
`apps/server/test/sources-identity/cloudflare.test.ts`(200 门页过挑战重试、200 挑战页走浏览器代抓、转人工时留样)、
|
||||
`failure-sample.test.ts`(留样、只留 5 份、接口)、`apps/web/app/components/sites/site-failure-samples.test.ts`。
|
||||
正常搜索页 fixture `packages/adapters/test/fixtures/audiences-search.html.txt`(观众真实搜索页,脱敏、删减到 4 行)。
|
||||
|
||||
## 待核验
|
||||
|
||||
1. takelogin 表单字段、二次验证字段名、错误文本全部只对 `@umt/testkit` mock 站点验证过,未连接真实站点。
|
||||
|
||||
@@ -1,5 +1,14 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { challengeRuleOfHtml, classifyChallengePage, detectChallenge } from "./challenge.ts";
|
||||
import {
|
||||
CHALLENGE_OUTCOME_HEADER,
|
||||
CHALLENGE_PEEK_BYTES,
|
||||
challengeRuleOfHtml,
|
||||
challengeRuleOfUnrecognizedPage,
|
||||
classifyChallengePage,
|
||||
detectChallenge,
|
||||
isPageRequest,
|
||||
} from "./challenge.ts";
|
||||
import { siteFetch } from "./http.ts";
|
||||
import { createSiteRateLimiter } from "./rate-limit.ts";
|
||||
import { SiteError } from "./types.ts";
|
||||
@@ -114,3 +123,145 @@ describe("classifyChallengePage", () => {
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ── 200 状态的挑战页 / 人机验证门页(搜索页单独加盾、间歇触发)──
|
||||
|
||||
const fixture = (rel: string) => readFileSync(new URL(rel, import.meta.url), "utf8");
|
||||
/** 观众真实搜索页(脱敏、删减到 4 行,`<table class="torrents torrents-table" id="torrenttable">`) */
|
||||
const AUD_SEARCH = fixture("../../test/fixtures/audiences-search.html.txt");
|
||||
/** 观众真实列表页(DET-2 样本,已脱敏) */
|
||||
const AUD_LIST = fixture("../../test/differential/inputs/indexer/rl-audiences-list.html.txt");
|
||||
/** Turnstile 门页:站点自己出的「请完成人机验证」短页,没有种子表、没有用户信息块 */
|
||||
const TURNSTILE_GATE = `<!DOCTYPE html><html><head><title>Audiences</title>
|
||||
<script src="https://challenges.cloudflare.com/turnstile/v0/api.js" async defer></script></head>
|
||||
<body><form method="post" action="/verify"><p>请完成人机验证后继续</p>
|
||||
<div class="cf-turnstile" data-sitekey="0x4AAAAAAA"></div></form></body></html>`;
|
||||
const HCAPTCHA_GATE = `<html><head><title>Security check</title><script src="https://hcaptcha.com/1/api.js"></script></head>
|
||||
<body><div class="h-captcha" data-sitekey="x"></div></body></html>`;
|
||||
/** 登录页本身带 Turnstile:交给登录判定,不算门页 */
|
||||
const LOGIN_WITH_TURNSTILE = `<html><head><title>Audiences :: 登录 - Powered by NexusPHP</title>
|
||||
<script src="https://challenges.cloudflare.com/turnstile/v0/api.js"></script></head><body>
|
||||
<form method="post" action="takelogin.php"><input type="text" name="username"><input type="password" name="password">
|
||||
<div class="cf-turnstile" data-sitekey="0x4AAAAAAA"></div></form></body></html>`;
|
||||
/** 正常页面里注入了 challenge-platform 脚本,侧栏还有个签到 Turnstile:有种子表 / 用户信息块 → 不是门页 */
|
||||
const NORMAL_WITH_WIDGET = `<html><head><title>Audiences :: 种子</title>
|
||||
<script src="/cdn-cgi/challenge-platform/scripts/jsd/main.js"></script></head><body>
|
||||
<div id="info_block"><a href="userdetails.php?id=1">me</a> <a href="logout.php">退出</a></div>
|
||||
<div class="cf-turnstile" data-sitekey="x"></div>
|
||||
<table class="torrents" id="torrenttable"><tr><td>row</td></tr></table></body></html>`;
|
||||
|
||||
const page200 = (body: string, headers: Record<string, string> = {}) =>
|
||||
new Response(body, { status: 200, headers: { "content-type": "text/html; charset=utf-8", ...headers } });
|
||||
|
||||
describe("detectChallenge:页面请求的 200 响应", () => {
|
||||
it("200 的 Cloudflare 挑战页(标题 / 脚本)→ 挑战", async () => {
|
||||
expect((await detectChallenge(page200(CF_PAGE), { page: true })).rule).toContain("Just a moment");
|
||||
const noTitle = `<html><head><title>x</title></head><body><script>window._cf_chl_opt={}</script></body></html>`;
|
||||
expect((await detectChallenge(page200(noTitle), { page: true })).rule).toContain("cf_chl_opt");
|
||||
});
|
||||
|
||||
it("200 的 Turnstile / hCaptcha 门页 → 挑战,正文原样交还", async () => {
|
||||
const a = await detectChallenge(page200(TURNSTILE_GATE), { page: true });
|
||||
expect(a.rule).toMatch(/^gate\*=/);
|
||||
expect(a.html).toBe(TURNSTILE_GATE);
|
||||
expect(await a.res.text()).toBe(TURNSTILE_GATE);
|
||||
expect((await detectChallenge(page200(HCAPTCHA_GATE), { page: true })).rule).toContain("hcaptcha");
|
||||
});
|
||||
|
||||
it("正常搜索页 / 列表页 / 带 Turnstile 的登录页 / 注入脚本的正常页 → 不误判", async () => {
|
||||
for (const body of [AUD_SEARCH, AUD_LIST, LOGIN_WITH_TURNSTILE, NORMAL_WITH_WIDGET, NORMAL]) {
|
||||
const r = await detectChallenge(page200(body), { page: true });
|
||||
expect(r.rule).toBeNull();
|
||||
expect(await r.res.text()).toBe(body);
|
||||
}
|
||||
});
|
||||
|
||||
it("不是页面请求 / 不是 HTML:不读正文(原响应对象直接返回)", async () => {
|
||||
const res = page200(CF_PAGE);
|
||||
const r = await detectChallenge(res);
|
||||
expect(r.rule).toBeNull();
|
||||
expect(r.res).toBe(res);
|
||||
expect(res.bodyUsed).toBe(false);
|
||||
const json = new Response('{"title":"Just a moment"}', {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
const j = await detectChallenge(json, { page: true });
|
||||
expect(j.res).toBe(json);
|
||||
});
|
||||
|
||||
it("正文超过读取上限:只读开头不判门页,其余照常流给调用方", async () => {
|
||||
const big = `${TURNSTILE_GATE}${"<!-- pad -->".repeat(Math.ceil((CHALLENGE_PEEK_BYTES * 2) / 12))}`;
|
||||
const r = await detectChallenge(page200(big), { page: true });
|
||||
expect(r.rule).toBeNull();
|
||||
expect(await r.res.text()).toBe(big);
|
||||
});
|
||||
|
||||
it("isPageRequest:GET + Accept text/html", () => {
|
||||
expect(isPageRequest("GET", "text/html,application/xhtml+xml")).toBe(true);
|
||||
expect(isPageRequest(undefined, "text/html")).toBe(true);
|
||||
expect(isPageRequest("POST", "text/html")).toBe(false);
|
||||
expect(isPageRequest("GET", "application/json")).toBe(false);
|
||||
expect(isPageRequest("GET", null)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("siteFetch:200 门页 → challenge 并附脱敏样本", () => {
|
||||
const url = "https://audiences.example/torrents.php?search=x&passkey=0123456789abcdef0123456789abcdef";
|
||||
const html = { accept: "text/html,application/xhtml+xml" };
|
||||
|
||||
it("页面请求遇 200 门页 → challenge;样本 URL 脱敏、带关键响应头;级联结果进文案", async () => {
|
||||
let err: unknown;
|
||||
try {
|
||||
await siteFetch(
|
||||
ctx(
|
||||
reply(200, TURNSTILE_GATE, {
|
||||
server: "cloudflare",
|
||||
"cf-ray": "abc-HKG",
|
||||
[CHALLENGE_OUTCOME_HEADER]: "needs_human",
|
||||
}),
|
||||
),
|
||||
url,
|
||||
{ headers: html },
|
||||
);
|
||||
} catch (e) {
|
||||
err = e;
|
||||
}
|
||||
const se = err as SiteError;
|
||||
expect(se).toBeInstanceOf(SiteError);
|
||||
expect(se.kind).toBe("challenge");
|
||||
expect(se.message).toContain("原站视图");
|
||||
expect(se.detail).toMatchObject({ reason: "challenge", outcome: "needs_human", httpStatus: 200 });
|
||||
expect(se.sample).toMatchObject({
|
||||
kind: "challenge",
|
||||
httpStatus: 200,
|
||||
headers: { server: "cloudflare", "cf-ray": "abc-HKG" },
|
||||
});
|
||||
expect(se.sample?.url).not.toContain("0123456789abcdef");
|
||||
expect(se.sample?.body).toContain("cf-turnstile");
|
||||
});
|
||||
|
||||
it("同一页面不是页面请求(没要 HTML)→ 照常返回", async () => {
|
||||
const res = await siteFetch(ctx(reply(200, TURNSTILE_GATE)), url);
|
||||
expect(await res.text()).toBe(TURNSTILE_GATE);
|
||||
});
|
||||
});
|
||||
|
||||
describe("challengeRuleOfUnrecognizedPage(解析失败时的兜底)", () => {
|
||||
it("站点外壳里的人机验证页(有用户信息块、很长)也认;登录表单不算", () => {
|
||||
const shell = `<html><head><title>Audiences</title></head><body><div id="info_block"><a href="logout.php">退出</a></div>
|
||||
<div>${"x".repeat(150_000)}</div><p>请完成人机验证</p><div class="cf-turnstile"></div></body></html>`;
|
||||
expect(challengeRuleOfUnrecognizedPage(shell)).toContain("cf-turnstile");
|
||||
expect(challengeRuleOfUnrecognizedPage("<html><body><p>搜索需要先完成人机验证</p></body></html>")).toBe(
|
||||
"text*=人机验证",
|
||||
);
|
||||
expect(challengeRuleOfUnrecognizedPage(LOGIN_WITH_TURNSTILE)).toBeNull();
|
||||
expect(challengeRuleOfUnrecognizedPage("<html><body><div class='new-layout'>改版</div></body></html>")).toBeNull();
|
||||
});
|
||||
|
||||
it("classifyChallengePage 也认门页:Turnstile → interactive,hCaptcha → captcha", () => {
|
||||
expect(classifyChallengePage({ html: TURNSTILE_GATE, text: "请完成人机验证后继续" }).kind).toBe("interactive");
|
||||
expect(classifyChallengePage({ html: HCAPTCHA_GATE, text: "" }).kind).toBe("captcha");
|
||||
expect(classifyChallengePage({ html: LOGIN_WITH_TURNSTILE, text: "登录" }).kind).toBe("none");
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,6 @@
|
||||
// CF-2 Cloudflare 挑战识别(docs/design/cloudflare-handling.md)。
|
||||
// - HTTP 端:响应头 `cf-mitigated: challenge`(任何状态码),或 403 / 503 / 429 且正文是挑战页 → 挑战,不是凭据错误;
|
||||
// 页面类请求(GET + Accept text/html)返回 200 也可能是挑战页 / 人机验证门页(Turnstile、hCaptcha、reCAPTCHA),短页且没有站点正常内容才算;
|
||||
// - 浏览器端:挑战页分三类——自动(JS 检查,等几秒自己过)、可点(「我是人类」复选框 / Turnstile)、真验证码(选图 / 输码)。
|
||||
// 只用正则,不解析 DOM:HTTP 端要在限流器之后快速判断,浏览器端拿到的是 outerHTML + innerText 样本。
|
||||
|
||||
@@ -27,21 +28,168 @@ export function challengeRuleOfHeaders(headers: Headers): string | null {
|
||||
return headers.get("cf-mitigated")?.trim().toLowerCase() === "challenge" ? "header cf-mitigated=challenge" : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 站点请求经服务端 Cloudflare 级联后仍是挑战时,级联在返回的响应上加这个头说明处理结果
|
||||
* (needs_human / failed / throttled / no_browser / no_account),适配器据此写错误文案;
|
||||
* unhandled = 网络层没认出、适配器解析时才认出的验证页(级联没跑)。
|
||||
*/
|
||||
export const CHALLENGE_OUTCOME_HEADER = "x-umt-challenge";
|
||||
|
||||
/** 人机验证错误文案的后半句:自动处理到哪一步、要不要人 */
|
||||
export function challengeOutcomeText(outcome: string | null | undefined): string {
|
||||
switch (outcome) {
|
||||
case "needs_human":
|
||||
return "自动处理没能通过，需要你在原站视图里过一次验证";
|
||||
case "failed":
|
||||
return "已自动尝试用浏览器通过，没有成功";
|
||||
case "throttled":
|
||||
return "30 分钟内已自动尝试过，之后会再试";
|
||||
case "no_browser":
|
||||
return "没有可用的浏览器，无法自动通过";
|
||||
case "no_account":
|
||||
return "站点没有已登录的账号，不自动处理";
|
||||
case "unhandled":
|
||||
return "这类页面没有自动处理，可以在原站视图里过一次验证";
|
||||
default:
|
||||
return "会自动尝试用浏览器通过";
|
||||
}
|
||||
}
|
||||
|
||||
/** 这些状态码的响应才去看正文(Cloudflare 挑战页是 403 / 503,少数 429) */
|
||||
export const CHALLENGE_STATUSES: readonly number[] = [403, 429, 503];
|
||||
|
||||
/** 判定时最多先读这么多正文;超出部分不缓冲,照常流给调用方。挑战 / 人机验证门页都远小于这个 */
|
||||
export const CHALLENGE_PEEK_BYTES = 256 * 1024;
|
||||
|
||||
/** 页面类请求(GET + Accept text/html):只有这类请求的 2xx 响应才检查正文 */
|
||||
export function isPageRequest(method: string | null | undefined, accept: string | null | undefined): boolean {
|
||||
return (method ?? "GET").toUpperCase() === "GET" && /text\/html|application\/xhtml/i.test(accept ?? "");
|
||||
}
|
||||
|
||||
/** 人机验证组件:Turnstile / hCaptcha / reCAPTCHA(只认脚本地址与组件 class,不认文案) */
|
||||
const GATE_WIDGET =
|
||||
/challenges\.cloudflare\.com\/turnstile|class="[^"]*\bcf-turnstile\b|hcaptcha\.com\/1\/api|class="[^"]*\bh-captcha\b|google\.com\/recaptcha|recaptcha\/api\.js|class="[^"]*\bg-recaptcha\b/i;
|
||||
/** 站点正常内容的痕迹:种子表、已登录的用户信息块(NexusPHP 等)。出现任何一个就不是门页 */
|
||||
const SITE_CONTENT =
|
||||
/id="torrenttable"|<table[^>]*class="[^"]*\btorrents\b|logout\.php|userdetails\.php\?id=|usercp\.php|id="info_block"/i;
|
||||
/** 登录表单(登录页本身也可能带 Turnstile,交给登录 / 失效判定,不算门页) */
|
||||
const LOGIN_FORM = /type=["']?password|takelogin\.php/i;
|
||||
/** 门页都很短;超过这个长度的页面不按门页判 */
|
||||
const GATE_MAX_CHARS = 100_000;
|
||||
/** 页面文字里的人机验证提示(只在页面已经认不出来时用) */
|
||||
const HUMAN_TEXT = /人机验证|verify you are human|确认您是真人|验证您是真人|checking your browser/i;
|
||||
|
||||
/**
|
||||
* 判定一个站点响应是不是挑战。正文只在状态码可能是挑战时读(读过的正文重新包成 Response 返回,调用方照常消费)。
|
||||
* 返回 { res, rule }:rule 为 null 表示不是挑战。
|
||||
* 人机验证门页:页面上有 Turnstile / hCaptcha / reCAPTCHA,且没有登录表单、没有站点正常内容、页面很短。
|
||||
* 命中返回规则说明,否则 null。
|
||||
*/
|
||||
export async function detectChallenge(res: Response): Promise<{ res: Response; rule: string | null }> {
|
||||
export function challengeGateRule(html: string | null | undefined): string | null {
|
||||
if (!html || html.length > GATE_MAX_CHARS) return null;
|
||||
const w = GATE_WIDGET.exec(html);
|
||||
if (!w || LOGIN_FORM.test(html) || SITE_CONTENT.test(html)) return null;
|
||||
return `gate*=${w[0].slice(0, 40)}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* 2xx 页面是不是挑战 / 人机验证页。比非 2xx 严:正文必须完整读到(短页)、没有站点正常内容,
|
||||
* 再看挑战标题 / 脚本特征或人机验证门页。complete = 正文是否已全部读到。
|
||||
*/
|
||||
export function challengeRuleOfOkPage(html: string, complete = true): string | null {
|
||||
if (!complete || !html || html.length > GATE_MAX_CHARS || SITE_CONTENT.test(html)) return null;
|
||||
return challengeRuleOfHtml(html) ?? challengeGateRule(html);
|
||||
}
|
||||
|
||||
/**
|
||||
* 页面已经解析不出来(列表 / 详情认不出结构)时的兜底:像挑战 / 人机验证就返回规则说明。
|
||||
* 比 challengeRuleOfOkPage 宽:不要求短页、也认页面文字里的「人机验证」提示,但带登录表单的页面不算。
|
||||
*/
|
||||
export function challengeRuleOfUnrecognizedPage(html: string | null | undefined): string | null {
|
||||
if (!html) return null;
|
||||
const head = html.slice(0, 300_000);
|
||||
const cf = challengeRuleOfHtml(head);
|
||||
if (cf) return cf;
|
||||
if (LOGIN_FORM.test(head)) return null;
|
||||
const w = GATE_WIDGET.exec(head);
|
||||
if (w) return `gate*=${w[0].slice(0, 40)}`;
|
||||
const t = HUMAN_TEXT.exec(head.replace(/<script[\s\S]*?<\/script>|<style[\s\S]*?<\/style>/gi, " "));
|
||||
return t ? `text*=${t[0]}` : null;
|
||||
}
|
||||
|
||||
function concatBytes(chunks: Uint8Array[], total: number): Uint8Array<ArrayBuffer> {
|
||||
const out = new Uint8Array(total);
|
||||
let off = 0;
|
||||
for (const c of chunks) {
|
||||
out.set(c, off);
|
||||
off += c.byteLength;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** 先读最多 max 字节;返回读到的部分、是否读完,以及一个内容不变的新响应(没读完的部分接着原流) */
|
||||
async function peekBody(
|
||||
res: Response,
|
||||
max: number,
|
||||
): Promise<{ head: Uint8Array<ArrayBuffer>; complete: boolean; res: Response }> {
|
||||
const reader = (res.body as ReadableStream<Uint8Array>).getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
let total = 0;
|
||||
let complete = false;
|
||||
while (total < max) {
|
||||
const r = await reader.read();
|
||||
if (r.done) {
|
||||
complete = true;
|
||||
break;
|
||||
}
|
||||
chunks.push(r.value);
|
||||
total += r.value.byteLength;
|
||||
}
|
||||
const head = concatBytes(chunks, total);
|
||||
const init = { status: res.status, statusText: res.statusText, headers: res.headers };
|
||||
if (complete) return { head, complete, res: new Response(head, init) };
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
start(c) {
|
||||
c.enqueue(head);
|
||||
},
|
||||
async pull(c) {
|
||||
const r = await reader.read();
|
||||
if (r.done) c.close();
|
||||
else c.enqueue(r.value);
|
||||
},
|
||||
cancel(reason) {
|
||||
return reader.cancel(reason);
|
||||
},
|
||||
});
|
||||
return { head, complete, res: new Response(stream, init) };
|
||||
}
|
||||
|
||||
/**
|
||||
* 判定一个站点响应是不是挑战。
|
||||
* - 响应头 `cf-mitigated: challenge` → 挑战(不读正文);
|
||||
* - 403 / 429 / 503 → 读正文看挑战页特征;
|
||||
* - opts.page(页面类请求)且 2xx 的 HTML → 读正文看挑战页 / 人机验证门页(challengeRuleOfOkPage);
|
||||
* - 其它情况不读正文,零开销。
|
||||
* 正文最多先读 CHALLENGE_PEEK_BYTES,读过的部分重新包进返回的响应,调用方照常消费。
|
||||
* 返回 { res, rule, html }:rule 为 null 表示不是挑战;html 是读到的正文开头(没读为 null,留样用)。
|
||||
*/
|
||||
export async function detectChallenge(
|
||||
res: Response,
|
||||
opts: { page?: boolean } = {},
|
||||
): Promise<{ res: Response; rule: string | null; html: string | null }> {
|
||||
const byHeader = challengeRuleOfHeaders(res.headers);
|
||||
if (byHeader) return { res, rule: byHeader };
|
||||
if (!CHALLENGE_STATUSES.includes(res.status) || !res.body) return { res, rule: null };
|
||||
const bytes = new Uint8Array(await res.arrayBuffer());
|
||||
const html = new TextDecoder("utf-8").decode(bytes.subarray(0, 300_000));
|
||||
const copy = new Response(bytes, { status: res.status, statusText: res.statusText, headers: res.headers });
|
||||
return { res: copy, rule: challengeRuleOfHtml(html) };
|
||||
if (byHeader) return { res, rule: byHeader, html: null };
|
||||
const suspect = CHALLENGE_STATUSES.includes(res.status);
|
||||
const okPage =
|
||||
!!opts.page &&
|
||||
res.status >= 200 &&
|
||||
res.status < 300 &&
|
||||
res.status !== 204 &&
|
||||
res.status !== 205 &&
|
||||
/html|^$/i.test(res.headers.get("content-type")?.trim() ?? "");
|
||||
if ((!suspect && !okPage) || !res.body) return { res, rule: null, html: null };
|
||||
const peek = await peekBody(res, CHALLENGE_PEEK_BYTES);
|
||||
const html = new TextDecoder("utf-8").decode(peek.head);
|
||||
const rule = suspect ? challengeRuleOfHtml(html) : challengeRuleOfOkPage(html, peek.complete);
|
||||
return { res: peek.res, rule, html };
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -68,6 +216,7 @@ export function classifyChallengePage(page: { title?: string | null; html?: stri
|
||||
const rule =
|
||||
(CHALLENGE_TITLE.test(title) ? `title=${title.slice(0, 40)}` : null) ??
|
||||
challengeRuleOfHtml(html) ??
|
||||
challengeGateRule(html) ??
|
||||
(/verify you are human|确认您是真人|验证您是真人|checking (?:if the site connection is secure|your browser)/i.test(
|
||||
text,
|
||||
)
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { SiteFailureSample } from "./failure-sample.ts";
|
||||
import type { SiteHttpContext } from "./http.ts";
|
||||
import { createSiteRateLimiter, type SiteRateLimiter } from "./rate-limit.ts";
|
||||
import type { SecretResolver } from "./types.ts";
|
||||
@@ -12,6 +13,8 @@ export interface SiteAdapterDeps {
|
||||
userAgent?: string;
|
||||
/** 覆盖站点基址(测试指向 mock) */
|
||||
baseUrl?: string;
|
||||
/** 请求报 parse_error / challenge 时的页面样本(已脱敏);调用方自行保存,抛错不影响请求结果 */
|
||||
onFailureSample?: (sample: SiteFailureSample) => void;
|
||||
}
|
||||
|
||||
export function buildHttpContext(
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { buildFailureSample, FAILURE_SAMPLE_MAX_BYTES, redactPageHtml } from "./failure-sample.ts";
|
||||
|
||||
// 失败留样的脱敏:passkey / authkey / uid / 用户名 / Cookie / 邮箱 / 个人信息块都不进样本;截到 64 KB。
|
||||
|
||||
const AUD_SEARCH = readFileSync(new URL("../../test/fixtures/audiences-search.html.txt", import.meta.url), "utf8");
|
||||
|
||||
describe("redactPageHtml", () => {
|
||||
it("下载链接密钥、表单 / 脚本里的密钥、Cookie、邮箱、用户链接都抹掉,普通内容保留", () => {
|
||||
const key = "0123456789abcdef0123456789abcdef";
|
||||
const html = `<a href="download.php?id=9&passkey=${key}">dl</a>
|
||||
<a href="https://x.test/rss.php?uid=777&rsskey=abc123def">rss</a>
|
||||
<input name="authkey" value="zzzzzzzz9999"><script>var token = "tok_ABCDEFGH"; c_secure_pass=hunter2xyz;</script>
|
||||
<p>联系 [email protected]</p><a href="userdetails.php?id=4242"><b>bobby</b></a> UID 4242
|
||||
<td class="rowfollow">Dune.Part.Two.2024</td>`;
|
||||
const out = redactPageHtml(html);
|
||||
for (const leak of [
|
||||
key,
|
||||
"abc123def",
|
||||
"zzzzzzzz9999",
|
||||
"tok_ABCDEFGH",
|
||||
"hunter2xyz",
|
||||
"[email protected]",
|
||||
"4242",
|
||||
"bobby",
|
||||
"777",
|
||||
]) {
|
||||
expect(out).not.toContain(leak);
|
||||
}
|
||||
expect(out).toContain("download.php?id=9");
|
||||
expect(out).toContain("Dune.Part.Two.2024");
|
||||
});
|
||||
|
||||
it("真实搜索页:个人信息块整块删掉,种子表保留", () => {
|
||||
const out = redactPageHtml(AUD_SEARCH);
|
||||
expect(AUD_SEARCH).toContain("fixtureuser");
|
||||
expect(out).not.toContain("fixtureuser");
|
||||
expect(out).toContain("<!-- #info_block 已删除 -->");
|
||||
expect(out).toContain('id="torrenttable"');
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildFailureSample", () => {
|
||||
it("截到 64 KB、URL 脱敏、只留关键响应头", () => {
|
||||
const s = buildFailureSample({
|
||||
kind: "parse_error",
|
||||
requestKind: "search",
|
||||
url: "https://aud.test/torrents.php?search=x&passkey=secretsecret&uid=5",
|
||||
httpStatus: 200,
|
||||
headers: new Headers({
|
||||
server: "cloudflare",
|
||||
"cf-ray": "1-HKG",
|
||||
"set-cookie": "a=b",
|
||||
"cf-mitigated": "challenge",
|
||||
}),
|
||||
message: "搜索:页面结构无法识别",
|
||||
html: AUD_SEARCH,
|
||||
now: new Date("2026-10-06T00:00:00Z"),
|
||||
});
|
||||
expect(new TextEncoder().encode(s.body).byteLength).toBeLessThanOrEqual(FAILURE_SAMPLE_MAX_BYTES);
|
||||
expect(s.truncated).toBe(true);
|
||||
expect(s.url).not.toContain("secretsecret");
|
||||
expect(s.url).not.toContain("uid=5");
|
||||
expect(s.headers).toEqual({ server: "cloudflare", "cf-ray": "1-HKG", "cf-mitigated": "challenge" });
|
||||
expect(s.capturedAt).toBe("2026-10-06T00:00:00.000Z");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,124 @@
|
||||
import { redactString } from "@umt/contracts";
|
||||
import { redactText, redactUrl } from "../rss/redact.ts";
|
||||
|
||||
// 站点请求失败留样:请求报「页面结构无法识别」(parse_error)或「人机验证」(challenge)时,留一份脱敏后的页面开头,
|
||||
// 站点页「技术信息」里可看 / 下载,用来区分是站点改版还是被人机验证拦了(间歇触发的那一刻页面不会再出现)。
|
||||
|
||||
/** 页面样本最多保留的字节数 */
|
||||
export const FAILURE_SAMPLE_MAX_BYTES = 64 * 1024;
|
||||
|
||||
/** 请求类型:搜索 / 列表 / 详情 / 健康检测 / 其它 */
|
||||
export type SiteRequestKind = "search" | "list" | "detail" | "health" | "other";
|
||||
|
||||
/** 留样只记这几个响应头 */
|
||||
const KEPT_HEADERS = ["server", "cf-mitigated", "cf-ray", "content-type"] as const;
|
||||
|
||||
export interface SiteFailureSample {
|
||||
kind: "parse_error" | "challenge";
|
||||
requestKind: SiteRequestKind;
|
||||
/** 已脱敏 */
|
||||
url: string;
|
||||
httpStatus: number | null;
|
||||
headers: Record<string, string>;
|
||||
capturedAt: string;
|
||||
/** 报出去的错误文案 */
|
||||
message: string;
|
||||
/** 命中的挑战 / 门页规则 */
|
||||
rule: string | null;
|
||||
/** 脱敏后的页面开头(≤ FAILURE_SAMPLE_MAX_BYTES 字节) */
|
||||
body: string;
|
||||
/** 页面是否被截断 */
|
||||
truncated: boolean;
|
||||
}
|
||||
|
||||
/** URL 里除了通用凭据参数,还要抹掉的用户标识参数 */
|
||||
const EXTRA_KEYS = ["uid", "userid", "user_id"];
|
||||
const EMAIL_RE = /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g;
|
||||
/** NexusPHP passkey / authkey 等:32 位以上十六进制串(info hash 是 40 位,也一并抹掉,样本不需要) */
|
||||
const HEX_KEY_RE = /\b[a-f0-9]{32,64}\b/gi;
|
||||
/** Cookie 名=值(页面脚本里偶尔会出现) */
|
||||
const COOKIE_PAIR_RE =
|
||||
/\b(c_secure_[a-z_]+|cf_clearance|__cf_bm|phpsessid|PHPSESSID|session(?:id)?|remember_[a-z0-9_]+)=([^;\s"'<>&]+)/gi;
|
||||
/** 表单 / 脚本里的密钥字段:name="passkey" value="…"、passkey: "…" */
|
||||
const SECRET_FIELD_RE =
|
||||
/\b(pass[_-]?key|auth[_-]?key|torrent[_-]?pass|rss[_-]?key|api[_-]?key|token|csrf[_-]?token|secret)(["']?\s*(?:[:=]|value=)\s*["']?)([^"'\s<>&;,]{6,})/gi;
|
||||
/** 用户页链接里的用户 id 与链接文字(用户名) */
|
||||
const USER_LINK_RE = /(userdetails\.php\?id=)\d+/gi;
|
||||
const USER_NAME_RE = /(<a\b[^>]*userdetails\.php[^>]*>)((?:\s*<[^>]+>)*)([^<]+)/gi;
|
||||
|
||||
/** 页面里的「UID 12345」 */
|
||||
const UID_TEXT_RE = /\bUID\s*[:：]?\s*\d+/gi;
|
||||
|
||||
/**
|
||||
* 删掉 id 为 id 的整个元素(按同名标签计数找配对的结束标签);NexusPHP 的 `#info_block` 是当前用户的个人信息块
|
||||
* (用户名、分享率、魔力…),样本里不需要。找不到原样返回。
|
||||
*/
|
||||
function dropElementById(html: string, id: string): string {
|
||||
const open = new RegExp(`<([a-z][a-z0-9]*)\\b[^>]*\\bid=["']?${id}\\b[^>]*>`, "i").exec(html);
|
||||
if (!open) return html;
|
||||
const tag = (open[1] as string).toLowerCase();
|
||||
const re = new RegExp(`<(/?)${tag}\\b[^>]*>`, "gi");
|
||||
re.lastIndex = open.index + open[0].length;
|
||||
let depth = 1;
|
||||
for (let m = re.exec(html); m; m = re.exec(html)) {
|
||||
depth += m[1] ? -1 : 1;
|
||||
if (depth === 0) return `${html.slice(0, open.index)}<!-- #${id} 已删除 -->${html.slice(m.index + m[0].length)}`;
|
||||
}
|
||||
return `${html.slice(0, open.index)}<!-- #${id} 已删除 -->`;
|
||||
}
|
||||
|
||||
/** 页面文本脱敏:个人信息块、凭据参数、用户 id / 用户名、Cookie、邮箱、密钥样式的十六进制串 */
|
||||
export function redactPageHtml(html: string): string {
|
||||
return redactString(
|
||||
redactText(dropElementById(html, "info_block"), EXTRA_KEYS)
|
||||
.replace(UID_TEXT_RE, "UID [REDACTED]")
|
||||
.replace(USER_NAME_RE, "$1$2[USER]")
|
||||
.replace(USER_LINK_RE, "$1[REDACTED]")
|
||||
.replace(COOKIE_PAIR_RE, "$1=[REDACTED]")
|
||||
.replace(SECRET_FIELD_RE, "$1$2[REDACTED]")
|
||||
.replace(EMAIL_RE, "[EMAIL]")
|
||||
.replace(HEX_KEY_RE, "[HEX]"),
|
||||
);
|
||||
}
|
||||
|
||||
/** 先脱敏、再按字节截到上限(截断处的半个字符丢掉) */
|
||||
function clip(text: string, maxBytes: number): { body: string; truncated: boolean } {
|
||||
const bytes = new TextEncoder().encode(text);
|
||||
if (bytes.byteLength <= maxBytes) return { body: text, truncated: false };
|
||||
return { body: new TextDecoder().decode(bytes.subarray(0, maxBytes)).replace(/�$/, ""), truncated: true };
|
||||
}
|
||||
|
||||
export function buildFailureSample(input: {
|
||||
kind: SiteFailureSample["kind"];
|
||||
requestKind: SiteRequestKind;
|
||||
url: string;
|
||||
httpStatus?: number | null;
|
||||
headers?: Headers | null;
|
||||
message: string;
|
||||
rule?: string | null;
|
||||
html: string | null | undefined;
|
||||
now: Date;
|
||||
}): SiteFailureSample {
|
||||
const headers: Record<string, string> = {};
|
||||
for (const k of KEPT_HEADERS) {
|
||||
const v = input.headers?.get(k);
|
||||
if (v) headers[k] = v.slice(0, 200);
|
||||
}
|
||||
// 脱敏只看前 4 倍上限(截断前先脱敏,避免截断把密钥切成两半漏掉)
|
||||
const { body, truncated } = clip(
|
||||
redactPageHtml((input.html ?? "").slice(0, FAILURE_SAMPLE_MAX_BYTES * 4)),
|
||||
FAILURE_SAMPLE_MAX_BYTES,
|
||||
);
|
||||
return {
|
||||
kind: input.kind,
|
||||
requestKind: input.requestKind,
|
||||
url: redactUrl(input.url, EXTRA_KEYS),
|
||||
httpStatus: input.httpStatus ?? null,
|
||||
headers,
|
||||
capturedAt: input.now.toISOString(),
|
||||
message: input.message,
|
||||
rule: input.rule ?? null,
|
||||
body,
|
||||
truncated: truncated || (input.html?.length ?? 0) > FAILURE_SAMPLE_MAX_BYTES * 4,
|
||||
};
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
import { redactText } from "../rss/redact.ts";
|
||||
import { detectChallenge } from "./challenge.ts";
|
||||
import { CHALLENGE_OUTCOME_HEADER, challengeOutcomeText, detectChallenge, isPageRequest } from "./challenge.ts";
|
||||
import { buildFailureSample } from "./failure-sample.ts";
|
||||
import type { SiteRateLimiter } from "./rate-limit.ts";
|
||||
import { SiteError } from "./types.ts";
|
||||
|
||||
@@ -21,7 +22,8 @@ function retryAfterMs(res: Response, now: Date): number | undefined {
|
||||
}
|
||||
|
||||
/**
|
||||
* 站点请求:先取站点令牌,再发请求;Cloudflare 挑战(`cf-mitigated: challenge`,或 403 / 503 / 429 的挑战页)→ `challenge`
|
||||
* 站点请求:先取站点令牌,再发请求;Cloudflare 挑战(`cf-mitigated: challenge`,或 403 / 503 / 429 的挑战页,
|
||||
* 或页面请求返回 200 的挑战页 / 人机验证门页)→ `challenge`(附脱敏页面样本)
|
||||
* (不是凭据错误,CF-2);其余 429/503 统一映射并顺延限流器。
|
||||
* 返回的 Response 由调用方消费。错误消息只含脱敏后的 URL。
|
||||
*/
|
||||
@@ -44,13 +46,27 @@ export async function siteFetch(
|
||||
const reason = timeout.aborted ? "超时" : (e as Error).message;
|
||||
throw new SiteError(ctx.siteId, "network", redactText(`请求 ${safeUrl} 失败:${reason}`));
|
||||
}
|
||||
const ch = await detectChallenge(res);
|
||||
// 页面类请求的 200 响应也看正文:返回 200 的挑战页 / 人机验证门页(只读开头,有上限)
|
||||
const ch = await detectChallenge(res, { page: isPageRequest(init.method, headers.get("accept")) });
|
||||
res = ch.res;
|
||||
if (ch.rule) {
|
||||
await res.body?.cancel().catch(() => {});
|
||||
throw new SiteError(ctx.siteId, "challenge", `请求 ${safeUrl}:站点要求人机验证(Cloudflare)`, {
|
||||
const outcome = res.headers.get(CHALLENGE_OUTCOME_HEADER);
|
||||
const message = `请求 ${safeUrl}：站点要求人机验证（Cloudflare 等），${challengeOutcomeText(outcome)}`;
|
||||
throw new SiteError(ctx.siteId, "challenge", message, {
|
||||
httpStatus: res.status,
|
||||
detail: { reason: "challenge", rule: ch.rule, httpStatus: res.status },
|
||||
detail: { reason: "challenge", rule: ch.rule, httpStatus: res.status, ...(outcome ? { outcome } : {}) },
|
||||
sample: buildFailureSample({
|
||||
kind: "challenge",
|
||||
requestKind: "other",
|
||||
url,
|
||||
httpStatus: res.status,
|
||||
headers: res.headers,
|
||||
message,
|
||||
rule: ch.rule,
|
||||
html: ch.html,
|
||||
now: ctx.now(),
|
||||
}),
|
||||
});
|
||||
}
|
||||
if (res.status === 429 || res.status === 503) {
|
||||
|
||||
@@ -1,10 +1,17 @@
|
||||
export {
|
||||
CHALLENGE_OUTCOME_HEADER,
|
||||
CHALLENGE_PEEK_BYTES,
|
||||
CHALLENGE_STATUSES,
|
||||
type ChallengePageKind,
|
||||
challengeGateRule,
|
||||
challengeOutcomeText,
|
||||
challengeRuleOfHeaders,
|
||||
challengeRuleOfHtml,
|
||||
challengeRuleOfOkPage,
|
||||
challengeRuleOfUnrecognizedPage,
|
||||
classifyChallengePage,
|
||||
detectChallenge,
|
||||
isPageRequest,
|
||||
} from "./challenge.ts";
|
||||
export {
|
||||
audiencesDefinition,
|
||||
@@ -34,6 +41,13 @@ export {
|
||||
structureDescription,
|
||||
} from "./detail-description.ts";
|
||||
export { type FormattedDetailPage, formatDetailPage } from "./detail-format.ts";
|
||||
export {
|
||||
buildFailureSample,
|
||||
FAILURE_SAMPLE_MAX_BYTES,
|
||||
redactPageHtml,
|
||||
type SiteFailureSample,
|
||||
type SiteRequestKind,
|
||||
} from "./failure-sample.ts";
|
||||
export {
|
||||
type FeedMediaType,
|
||||
normalizeSiteFeedItem,
|
||||
|
||||
@@ -14,7 +14,9 @@
|
||||
|
||||
import { inspectTorrentIsolated } from "../../isolation.ts";
|
||||
import { redactText } from "../../rss/redact.ts";
|
||||
import { challengeOutcomeText, challengeRuleOfUnrecognizedPage } from "../challenge.ts";
|
||||
import { buildHttpContext, type SiteAdapterDeps } from "../deps.ts";
|
||||
import { buildFailureSample, type SiteRequestKind } from "../failure-sample.ts";
|
||||
import { readBody, readText, siteFetch } from "../http.ts";
|
||||
import {
|
||||
type DownloadTarget,
|
||||
@@ -107,6 +109,23 @@ export function judgeLoginPage(
|
||||
};
|
||||
}
|
||||
|
||||
/** 抓到的页面:正文与留样用的响应信息 */
|
||||
interface FetchedPage {
|
||||
html: string;
|
||||
url: string;
|
||||
status: number;
|
||||
headers: Headers;
|
||||
}
|
||||
|
||||
/** 错误文案里的页面名 */
|
||||
const PAGE_LABEL: Record<SiteRequestKind, string> = {
|
||||
search: "搜索页",
|
||||
list: "列表页",
|
||||
detail: "详情页",
|
||||
health: "首页",
|
||||
other: "页面",
|
||||
};
|
||||
|
||||
export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdapterDeps): SiteAdapter {
|
||||
const def: IndexerDefinition = deps.baseUrl ? { ...input, domain: deps.baseUrl } : input;
|
||||
const ext = def.umt;
|
||||
@@ -119,13 +138,73 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap
|
||||
return c;
|
||||
}
|
||||
|
||||
async function fetchPage(url: string, what: string, opts: SiteCallOptions): Promise<string> {
|
||||
const res = await siteFetch(http, url, {
|
||||
/** 留样:交给调用方保存;回调出错不影响请求结果 */
|
||||
function report(e: SiteError): SiteError {
|
||||
if (e.sample && deps.onFailureSample) {
|
||||
try {
|
||||
deps.onFailureSample(e.sample);
|
||||
} catch {
|
||||
// 留样失败不影响请求
|
||||
}
|
||||
}
|
||||
return e;
|
||||
}
|
||||
|
||||
/** 页面认不出来时的错误:像人机验证就报 challenge,否则 parse_error;都附页面样本 */
|
||||
function unrecognized(page: FetchedPage, what: string, kind: SiteRequestKind, parseMessage: string): SiteError {
|
||||
const rule = challengeRuleOfUnrecognizedPage(page.html);
|
||||
const message = rule
|
||||
? `${what}：${PAGE_LABEL[kind]}需要人机验证（Cloudflare 等），${challengeOutcomeText("unhandled")}`
|
||||
: `${parseMessage}（页面样本见站点页技术信息）`;
|
||||
const sample = buildFailureSample({
|
||||
kind: rule ? "challenge" : "parse_error",
|
||||
requestKind: kind,
|
||||
url: page.url,
|
||||
httpStatus: page.status,
|
||||
headers: page.headers,
|
||||
message,
|
||||
rule,
|
||||
html: page.html,
|
||||
now: http.now(),
|
||||
});
|
||||
return report(
|
||||
new SiteError(def.id, rule ? "challenge" : "parse_error", message, {
|
||||
httpStatus: page.status,
|
||||
...(rule ? { detail: { reason: "challenge", rule, httpStatus: page.status } } : {}),
|
||||
sample,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async function fetchPage(
|
||||
url: string,
|
||||
what: string,
|
||||
opts: SiteCallOptions,
|
||||
kind: SiteRequestKind,
|
||||
): Promise<FetchedPage> {
|
||||
let res: Response;
|
||||
try {
|
||||
res = await siteFetch(http, url, {
|
||||
method: "GET",
|
||||
headers: { cookie: await cookie(), accept: "text/html,application/xhtml+xml", referer: base },
|
||||
redirect: "manual",
|
||||
...(opts.signal ? { signal: opts.signal } : {}),
|
||||
});
|
||||
} catch (e) {
|
||||
if (e instanceof SiteError && e.kind === "challenge") {
|
||||
// 网络层判出的人机验证:换成带请求类型的文案,样本补上请求类型
|
||||
const outcome = typeof e.detail?.outcome === "string" ? e.detail.outcome : null;
|
||||
const message = `${what}：${PAGE_LABEL[kind]}需要人机验证（Cloudflare 等），${challengeOutcomeText(outcome)}`;
|
||||
throw report(
|
||||
new SiteError(def.id, "challenge", message, {
|
||||
...(e.httpStatus !== undefined ? { httpStatus: e.httpStatus } : {}),
|
||||
...(e.detail ? { detail: e.detail } : {}),
|
||||
...(e.sample ? { sample: { ...e.sample, requestKind: kind, message } } : {}),
|
||||
}),
|
||||
);
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
if (res.status >= 300 && res.status < 400) {
|
||||
const loc = res.headers.get("location") ?? "";
|
||||
await res.body?.cancel().catch(() => {});
|
||||
@@ -155,7 +234,7 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap
|
||||
httpStatus: res.status,
|
||||
});
|
||||
}
|
||||
return readText(http, res);
|
||||
return { html: await readText(http, res), url, status: res.status, headers: res.headers };
|
||||
}
|
||||
|
||||
async function run(
|
||||
@@ -172,17 +251,19 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap
|
||||
page: Math.max(1, page) - 1, // 对外 1 起,MP 内部 0 起
|
||||
});
|
||||
if (!spider.canSearch()) throw new SiteError(def.id, "unsupported", `${what}:站点定义不支持该媒体类型`);
|
||||
const html = await fetchPage(spider.getSearchUrl(), what, opts);
|
||||
const parsed = spider.parse(html);
|
||||
const kind: SiteRequestKind = keyword ? "search" : "list";
|
||||
const page$ = await fetchPage(spider.getSearchUrl(), what, opts, kind);
|
||||
const parsed = spider.parse(page$.html);
|
||||
if (parsed.loginPage) throw new SiteError(def.id, "auth_expired", `${what}:返回登录或权限提示页`);
|
||||
if (parsed.isError || !parsed.$) throw new SiteError(def.id, "parse_error", `${what}:响应为空`);
|
||||
if (parsed.isError || !parsed.$) throw unrecognized(page$, what, kind, `${what}:响应为空`);
|
||||
const $ = parsed.$;
|
||||
if (parsed.torrents.length === 0) {
|
||||
const text = $("body").text();
|
||||
const hasList =
|
||||
$(normalizeSelector(spider.def.torrents.list.selector)).length > 0 || $("table.torrents").length > 0;
|
||||
const empty = (ext.emptyMarkers ?? []).some((m) => text.includes(m));
|
||||
if (!hasList && !empty) throw new SiteError(def.id, "parse_error", `${what}:页面结构无法识别(可能改版)`);
|
||||
// 认不出列表:先看是不是人机验证页(搜索页单独加盾、间歇触发),不是才报改版
|
||||
if (!hasList && !empty) throw unrecognized(page$, what, kind, `${what}:页面结构无法识别(可能改版)`);
|
||||
if (ext.loginSuccess && $(ext.loginSuccess).length === 0 && !empty) {
|
||||
throw new SiteError(def.id, "auth_expired", `${what}:未找到登录标志`);
|
||||
}
|
||||
@@ -237,18 +318,20 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap
|
||||
if (!/^\d+$/.test(siteReleaseId)) throw new SiteError(def.id, "not_found", "资源 ID 非法");
|
||||
if (!ext.detail) throw new SiteError(def.id, "unsupported", "站点定义未配置详情页解析");
|
||||
const url = new URL(`${ext.detail.path}?id=${siteReleaseId}&hit=1`, base).toString();
|
||||
const html = await fetchPage(url, `详情 ${siteReleaseId}`, opts);
|
||||
const r = parseDetailPage(html, def, siteReleaseId, http.now().toISOString());
|
||||
const detailPage = await fetchPage(url, `详情 ${siteReleaseId}`, opts, "detail");
|
||||
const r = parseDetailPage(detailPage.html, def, siteReleaseId, http.now().toISOString());
|
||||
if (r.kind === "login") throw new SiteError(def.id, "auth_expired", `详情 ${siteReleaseId}:返回登录页`);
|
||||
if (r.kind === "not_found") throw new SiteError(def.id, "not_found", `详情 ${siteReleaseId}:资源不存在`);
|
||||
if (!r.detail) throw new SiteError(def.id, "parse_error", `详情 ${siteReleaseId}:页面结构无法识别`);
|
||||
if (!r.detail) {
|
||||
throw unrecognized(detailPage, `详情 ${siteReleaseId}`, "detail", `详情 ${siteReleaseId}:页面结构无法识别`);
|
||||
}
|
||||
// 文件清单不在页面上(NexusPHP 点「查看列表」才 AJAX 取)→ 按需 1 次;失败只是没有文件清单
|
||||
const flPath = ext.detail.page?.fileListPath;
|
||||
if (flPath && r.detail.page && !r.detail.page.files.length) {
|
||||
try {
|
||||
const flUrl = new URL(flPath.replace("{id}", siteReleaseId), base).toString();
|
||||
const frag = await fetchPage(flUrl, `文件清单 ${siteReleaseId}`, opts);
|
||||
r.detail.page.files = filesFromFileListHtml(loadHtml(frag));
|
||||
const frag = await fetchPage(flUrl, `文件清单 ${siteReleaseId}`, opts, "detail");
|
||||
r.detail.page.files = filesFromFileListHtml(loadHtml(frag.html));
|
||||
} catch (e) {
|
||||
if (e instanceof SiteError && e.isAuthError) throw e;
|
||||
}
|
||||
@@ -317,8 +400,8 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap
|
||||
const started = Date.now();
|
||||
const checkedAt = http.now().toISOString();
|
||||
try {
|
||||
const html = await fetchPage(new URL("index.php", base).toString(), "健康检查", opts);
|
||||
const v = judgeLoginPage(html, ext.loginSuccess);
|
||||
const home = await fetchPage(new URL("index.php", base).toString(), "健康检查", opts, "health");
|
||||
const v = judgeLoginPage(home.html, ext.loginSuccess);
|
||||
const latencyMs = Date.now() - started;
|
||||
if (v.ok) return { siteId: def.id, status: "ok", ok: true, checkedAt, latencyMs, detail: v.detail };
|
||||
return {
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
audiencesDefinition,
|
||||
createSiteRateLimiter,
|
||||
createSpiderSiteAdapter,
|
||||
SiteError,
|
||||
type SiteFailureSample,
|
||||
staticSecretResolver,
|
||||
} from "../../index.ts";
|
||||
|
||||
// 搜索页单独加盾(间歇触发):返回 200 的挑战 / 人机验证页不再报「页面结构无法识别」,报 challenge 并留样;
|
||||
// 真正认不出的页面才报 parse_error(也留样)。
|
||||
|
||||
const AUD_SEARCH = readFileSync(new URL("../../../test/fixtures/audiences-search.html.txt", import.meta.url), "utf8");
|
||||
const CF_200 = `<!DOCTYPE html><html><head><title>Just a moment...</title></head><body>
|
||||
<div id="challenge-spinner"></div><script>window._cf_chl_opt={cType: 'managed'}</script></body></html>`;
|
||||
/** 站点外壳里的人机验证页:有用户信息块(网络层的严格判定不认),适配器解析失败时兜底认出 */
|
||||
const SHELL_GATE = `<html><head><title>Audiences :: 搜索</title></head><body>
|
||||
<table id="info_block"><tr><td><a href="userdetails.php?id=29191">someone</a> <a href="logout.php">退出</a></td></tr></table>
|
||||
<div class="main"><p>搜索前请完成人机验证</p><div class="cf-turnstile" data-sitekey="0x4AAA"></div>
|
||||
<script src="https://challenges.cloudflare.com/turnstile/v0/api.js"></script></div></body></html>`;
|
||||
const LOGIN_WITH_TURNSTILE = `<html><head><title>Audiences :: 登录 - Powered by NexusPHP</title></head><body>
|
||||
<form method="post" action="takelogin.php"><input type="password" name="password">
|
||||
<div class="cf-turnstile" data-sitekey="x"></div></form></body></html>`;
|
||||
/** 改版页:带上各种要脱敏的东西 */
|
||||
const SECRET = "0123456789abcdef0123456789abcdef";
|
||||
const BROKEN = `<html><head><title>Audiences :: 种子</title></head><body>
|
||||
<div id="info_block">欢迎 <a href="userdetails.php?id=29191"><b>alice_cn</b></a> 邮箱 [email protected]</div>
|
||||
<a href="download.php?id=1&passkey=${SECRET}">dl</a>
|
||||
<script>var authkey = "${SECRET}"; document.cookie = "c_secure_pass=topsecretvalue";</script>
|
||||
<input type="hidden" name="passkey" value="${SECRET}">
|
||||
<div class="new-grid">${"<div class='card'>Dune</div>".repeat(5000)}</div></body></html>`;
|
||||
|
||||
function adapter(body: string, status = 200, headers: Record<string, string> = {}) {
|
||||
const samples: SiteFailureSample[] = [];
|
||||
const fetchImpl = (async () =>
|
||||
new Response(body, {
|
||||
status,
|
||||
headers: { "content-type": "text/html; charset=utf-8", ...headers },
|
||||
})) as typeof fetch;
|
||||
const a = createSpiderSiteAdapter(audiencesDefinition, {
|
||||
baseUrl: "https://aud.test/",
|
||||
secrets: staticSecretResolver({ [audiencesDefinition.id]: { cookie: "c_secure_uid=1; c_secure_pass=abc" } }),
|
||||
limiter: createSiteRateLimiter({ minIntervalMs: 0, burst: 10 }),
|
||||
fetch: fetchImpl,
|
||||
onFailureSample: (s) => samples.push(s),
|
||||
});
|
||||
return { a, samples };
|
||||
}
|
||||
|
||||
async function err(p: Promise<unknown>): Promise<SiteError> {
|
||||
try {
|
||||
await p;
|
||||
} catch (e) {
|
||||
expect(e).toBeInstanceOf(SiteError);
|
||||
return e as SiteError;
|
||||
}
|
||||
throw new Error("expected SiteError");
|
||||
}
|
||||
|
||||
describe("spider 搜索页人机验证", () => {
|
||||
it("正常搜索页(真实样本)照常解析,不留样", async () => {
|
||||
const { a, samples } = adapter(AUD_SEARCH);
|
||||
const page = await a.search("药屋少女的呢喃");
|
||||
expect(page.items.length).toBeGreaterThan(0);
|
||||
expect(samples).toEqual([]);
|
||||
});
|
||||
|
||||
it("200 的 Cloudflare 挑战页 → challenge(网络层认出),文案说搜索页需要人机验证,留样标搜索", async () => {
|
||||
const { a, samples } = adapter(CF_200, 200, { server: "cloudflare", "cf-ray": "8c-HKG" });
|
||||
const e = await err(a.search("Dune"));
|
||||
expect(e.kind).toBe("challenge");
|
||||
expect(e.isAuthError).toBe(false);
|
||||
expect(e.message).toContain("搜索页需要人机验证");
|
||||
expect(e.message).not.toContain("改版");
|
||||
expect(samples).toHaveLength(1);
|
||||
expect(samples[0]).toMatchObject({ kind: "challenge", requestKind: "search", httpStatus: 200 });
|
||||
expect(samples[0]?.headers["cf-ray"]).toBe("8c-HKG");
|
||||
expect(samples[0]?.url).toContain("torrents.php");
|
||||
});
|
||||
|
||||
it("站点外壳里的人机验证页:网络层不认,解析失败前兜底 → challenge(不报改版)", async () => {
|
||||
const { a, samples } = adapter(SHELL_GATE);
|
||||
const e = await err(a.search("Dune"));
|
||||
expect(e.kind).toBe("challenge");
|
||||
expect(e.message).toContain("搜索页需要人机验证");
|
||||
expect(e.detail?.rule).toContain("turnstile");
|
||||
expect(samples[0]).toMatchObject({ kind: "challenge", requestKind: "search" });
|
||||
expect(samples[0]?.body).not.toContain("29191");
|
||||
expect(samples[0]?.body).not.toContain("someone");
|
||||
});
|
||||
|
||||
it("列表页同样处理,文案按列表", async () => {
|
||||
const { a, samples } = adapter(SHELL_GATE);
|
||||
const e = await err(a.list(1));
|
||||
expect(e.kind).toBe("challenge");
|
||||
expect(e.message).toContain("列表页需要人机验证");
|
||||
expect(samples[0]?.requestKind).toBe("list");
|
||||
});
|
||||
|
||||
it("带 Turnstile 的登录页仍按登录失效处理,不留样", async () => {
|
||||
const { a, samples } = adapter(LOGIN_WITH_TURNSTILE);
|
||||
const e = await err(a.search("Dune"));
|
||||
expect(e.kind).toBe("auth_expired");
|
||||
expect(samples).toEqual([]);
|
||||
});
|
||||
|
||||
it("真正认不出的页面 → parse_error,留脱敏样本(≤ 64 KB)", async () => {
|
||||
const { a, samples } = adapter(BROKEN, 200, { server: "nginx" });
|
||||
const e = await err(a.search("Dune"));
|
||||
expect(e.kind).toBe("parse_error");
|
||||
expect(e.message).toContain("页面结构无法识别");
|
||||
expect(samples).toHaveLength(1);
|
||||
const s = samples[0] as SiteFailureSample;
|
||||
expect(s).toMatchObject({ kind: "parse_error", requestKind: "search", httpStatus: 200, truncated: true });
|
||||
expect(new TextEncoder().encode(s.body).byteLength).toBeLessThanOrEqual(64 * 1024);
|
||||
for (const leak of [SECRET, "[email protected]", "alice_cn", "29191", "topsecretvalue"]) {
|
||||
expect(s.body).not.toContain(leak);
|
||||
}
|
||||
expect(s.body).toContain("download.php?id=1");
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,7 @@
|
||||
/** 站点适配器自有类型(不依赖 @umt/contracts;W2 负责映射到 source_release / site_snapshot)。 */
|
||||
|
||||
import type { SiteFailureSample } from "./failure-sample.ts";
|
||||
|
||||
/** api = 专用 API spider(spiders/*,Haidan/HDDolby/RousiPro/SunnyPT/TNode/TorrentLeech/Yema) */
|
||||
export type SiteKind = "nexusphp" | "mteam" | "api";
|
||||
|
||||
@@ -205,11 +207,13 @@ export class SiteError extends Error {
|
||||
readonly retryAfterMs?: number;
|
||||
/** 原始判定(健康检查的技术信息用) */
|
||||
readonly detail?: SiteHealthDetail;
|
||||
/** 失败页面样本(已脱敏;parse_error / challenge 时由适配器附上,交给 deps.onFailureSample 留样) */
|
||||
readonly sample?: SiteFailureSample;
|
||||
constructor(
|
||||
siteId: string,
|
||||
kind: SiteErrorKind,
|
||||
message: string,
|
||||
opts: { httpStatus?: number; retryAfterMs?: number; detail?: SiteHealthDetail } = {},
|
||||
opts: { httpStatus?: number; retryAfterMs?: number; detail?: SiteHealthDetail; sample?: SiteFailureSample } = {},
|
||||
) {
|
||||
super(message);
|
||||
this.name = "SiteError";
|
||||
@@ -218,6 +222,7 @@ export class SiteError extends Error {
|
||||
if (opts.httpStatus !== undefined) this.httpStatus = opts.httpStatus;
|
||||
if (opts.retryAfterMs !== undefined) this.retryAfterMs = opts.retryAfterMs;
|
||||
if (opts.detail !== undefined) this.detail = opts.detail;
|
||||
if (opts.sample !== undefined) this.sample = opts.sample;
|
||||
}
|
||||
/** 凭据类错误:应立即置账号验证失败,不再重试 */
|
||||
get isAuthError(): boolean {
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -86,6 +86,22 @@ export const sourcesApi = {
|
||||
params: IdParam,
|
||||
response: sources.SiteRequestStats,
|
||||
}),
|
||||
failureSamples: ep({
|
||||
method: "get",
|
||||
path: "/sites/{id}/failure-samples",
|
||||
scope: "read",
|
||||
summary: "站点请求失败留样：最近几次页面结构无法识别 / 人机验证时的脱敏页面（列表不带正文）",
|
||||
params: IdParam,
|
||||
response: z.object({ items: z.array(sources.SiteFailureSampleSummary) }),
|
||||
}),
|
||||
failureSample: ep({
|
||||
method: "get",
|
||||
path: "/sites/{id}/failure-samples/{sampleId}",
|
||||
scope: "read",
|
||||
summary: "一份失败留样（含脱敏后的页面正文）",
|
||||
params: z.object({ id: Id, sampleId: Id }),
|
||||
response: sources.SiteFailureSample,
|
||||
}),
|
||||
credentials: ep({
|
||||
method: "put",
|
||||
path: "/sites/{id}/credentials",
|
||||
|
||||
@@ -701,6 +701,35 @@ export const SiteSnapshot = EntityBase.extend({
|
||||
});
|
||||
export type SiteSnapshot = z.infer<typeof SiteSnapshot>;
|
||||
|
||||
/** 失败留样的类型:页面结构无法识别 / 人机验证 */
|
||||
export const SiteFailureSampleKind = z.enum(["parse_error", "challenge"]);
|
||||
export type SiteFailureSampleKind = z.infer<typeof SiteFailureSampleKind>;
|
||||
|
||||
/**
|
||||
* 站点请求失败留样(列表 / 搜索 / 详情报 parse_error / challenge 时存的脱敏页面开头,每站最近 5 份)。
|
||||
* requestKind:search / list / detail / health / other;headers 只有 server、cf-mitigated、cf-ray、content-type。
|
||||
*/
|
||||
export const SiteFailureSample = z.object({
|
||||
id: Id,
|
||||
siteId: Id,
|
||||
kind: SiteFailureSampleKind,
|
||||
requestKind: z.string(),
|
||||
/** 已脱敏 */
|
||||
url: z.string(),
|
||||
httpStatus: z.number().int().nullable(),
|
||||
headers: z.record(z.string(), z.string()),
|
||||
message: z.string(),
|
||||
rule: z.string().nullable(),
|
||||
/** 脱敏后的页面开头(≤ 64 KB) */
|
||||
body: z.string(),
|
||||
truncated: z.boolean(),
|
||||
createdAt: Timestamp,
|
||||
});
|
||||
export type SiteFailureSample = z.infer<typeof SiteFailureSample>;
|
||||
/** 列表里不带页面正文,只给字节数 */
|
||||
export const SiteFailureSampleSummary = SiteFailureSample.omit({ body: true }).extend({ bodyBytes: NonNegInt });
|
||||
export type SiteFailureSampleSummary = z.infer<typeof SiteFailureSampleSummary>;
|
||||
|
||||
/** 站点访问默认(站点定义 umt.defaults 与 settings.sites 合并后,取更保守的一方) */
|
||||
export const SiteAccessDefaults = z.object({
|
||||
snapshotIntervalSec: z.number().int(),
|
||||
|
||||
@@ -105,6 +105,7 @@ const ROWS: Record<string, { row: Obj; dbOnly?: string[]; apiOnly?: string[] }>
|
||||
MetadataEnrichment: { row: C.MetadataEnrichment },
|
||||
ResolutionAttempt: { row: C.ResolutionAttempt },
|
||||
SiteSnapshot: { row: C.SiteSnapshot },
|
||||
SiteFailureSample: { row: C.SiteFailureSample },
|
||||
// W4-C 浏览器实体(0023)
|
||||
// BR-4:identity 是浏览器固定身份的缓存(站点请求 UA 用),不进 API
|
||||
BrowserProvider: { row: C.BrowserProviderRow, dbOnly: ["identity"] },
|
||||
@@ -252,6 +253,7 @@ const ENUMS: Record<string, z.ZodEnum> = {
|
||||
ScheduledJobLastStatus: C.ScheduleRunStatus,
|
||||
ScheduledJobRunTrigger: C.ScheduleTrigger,
|
||||
ScheduledJobRunStatus: C.ScheduleRunStatus,
|
||||
SiteFailureSampleKind: C.SiteFailureSampleKind,
|
||||
ScrapeRecordStatus: C.ScrapeStatus,
|
||||
ScrapeRecordTrigger: C.ScrapeTrigger,
|
||||
SubtitleWriteSource: C.SubtitleSource,
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
import type { Kysely } from "kysely";
|
||||
import { dropTables, execDdl } from "../src/migration-helpers.ts";
|
||||
|
||||
// 站点请求失败留样:列表 / 搜索 / 详情报「页面结构无法识别」或「人机验证」时,存一份脱敏后的页面开头(≤ 64 KB)
|
||||
// 与 URL(脱敏)、状态码、关键响应头、请求类型。每站只留最近 5 份(写入时顺手删旧的);站点页「技术信息」里看。
|
||||
// 单独一张表:样本有几十 KB,不放进每条站点请求都要读的 site.config。
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型
|
||||
export async function up(db: Kysely<any>): Promise<void> {
|
||||
await execDdl(
|
||||
db,
|
||||
`
|
||||
create table site_failure_sample (
|
||||
id uuid primary key default gen_random_uuid(),
|
||||
site_id uuid not null references site(id) on delete cascade,
|
||||
kind text not null check (kind in ('parse_error', 'challenge')),
|
||||
request_kind text not null,
|
||||
url text not null,
|
||||
http_status integer,
|
||||
headers jsonb not null default '{}'::jsonb,
|
||||
message text not null,
|
||||
rule text,
|
||||
body text not null,
|
||||
truncated boolean not null default false,
|
||||
created_at timestamptz not null default now()
|
||||
);
|
||||
create index site_failure_sample_site_idx on site_failure_sample (site_id, created_at desc);
|
||||
`,
|
||||
);
|
||||
}
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 同上
|
||||
export async function down(db: Kysely<any>): Promise<void> {
|
||||
await execDdl(db, dropTables(["site_failure_sample"]));
|
||||
}
|
||||
@@ -255,6 +255,7 @@ export type SiteCircuitState = "closed" | "open" | "half_open";
|
||||
export type SiteKind = "pt" | "public_bt" | "rss_feed" | "torznab" | "subtitle";
|
||||
export type SiteAccountAuthKind = "cookie" | "passkey" | "api_key" | "password";
|
||||
export type SiteAccountStatus = "unknown" | "active" | "invalid" | "expired" | "challenge";
|
||||
export type SiteFailureSampleKind = "parse_error" | "challenge";
|
||||
export type SiteSnapshotKind = "periodic" | "backfill" | "live_search" | "manual";
|
||||
export type SiteSnapshotStatus = "running" | "ok" | "partial" | "failed";
|
||||
export type SourceReleaseResolutionStatus = "unresolved" | "resolved" | "ambiguous" | "unidentifiable" | "rejected";
|
||||
@@ -1491,6 +1492,21 @@ export interface SiteAccountTable {
|
||||
ua_pending: string | null;
|
||||
}
|
||||
|
||||
export interface SiteFailureSampleTable {
|
||||
id: Generated<string>;
|
||||
site_id: string;
|
||||
kind: SiteFailureSampleKind;
|
||||
request_kind: string;
|
||||
url: string;
|
||||
http_status: number | null;
|
||||
headers: Generated<Json>;
|
||||
message: string;
|
||||
rule: string | null;
|
||||
body: string;
|
||||
truncated: Generated<boolean>;
|
||||
created_at: GeneratedTimestamptz;
|
||||
}
|
||||
|
||||
export interface SiteRequestCountTable {
|
||||
site_id: string;
|
||||
hour: Timestamptz;
|
||||
@@ -1825,6 +1841,7 @@ export interface DB {
|
||||
settings: SettingsTable;
|
||||
site: SiteTable;
|
||||
site_account: SiteAccountTable;
|
||||
site_failure_sample: SiteFailureSampleTable;
|
||||
site_request_count: SiteRequestCountTable;
|
||||
site_snapshot: SiteSnapshotTable;
|
||||
site_user_data: SiteUserDataTable;
|
||||
@@ -2051,6 +2068,9 @@ export type SiteUpdate = Updateable<SiteTable>;
|
||||
export type SiteAccountRow = Selectable<SiteAccountTable>;
|
||||
export type NewSiteAccount = Insertable<SiteAccountTable>;
|
||||
export type SiteAccountUpdate = Updateable<SiteAccountTable>;
|
||||
export type SiteFailureSampleRow = Selectable<SiteFailureSampleTable>;
|
||||
export type NewSiteFailureSample = Insertable<SiteFailureSampleTable>;
|
||||
export type SiteFailureSampleUpdate = Updateable<SiteFailureSampleTable>;
|
||||
export type SiteRequestCountRow = Selectable<SiteRequestCountTable>;
|
||||
export type NewSiteRequestCount = Insertable<SiteRequestCountTable>;
|
||||
export type SiteRequestCountUpdate = Updateable<SiteRequestCountTable>;
|
||||
|
||||
@@ -130,6 +130,8 @@ const EXPECTED_TABLES = [
|
||||
"subtitle_write",
|
||||
// 集数映射规则(0066)
|
||||
"episode_mapping_rule",
|
||||
// 站点请求失败留样(0077)
|
||||
"site_failure_sample",
|
||||
].sort();
|
||||
|
||||
/** 不走 entityTable() 的表:追加型/引擎自有表,没有 uuid id + created_at/updated_at 三件套 */
|
||||
@@ -163,6 +165,8 @@ const NON_ENTITY_TABLES = [
|
||||
"kv_cache",
|
||||
// 站点请求计数:(site_id, hour) 主键
|
||||
"site_request_count",
|
||||
// 失败留样:只追加,只有 created_at
|
||||
"site_failure_sample",
|
||||
];
|
||||
|
||||
describe.skipIf(skip)("schema migrations (real PG)", () => {
|
||||
|
||||
Reference in new issue
Block a user