From b44ebc2ba1f7f5784c1b13359b5957e70830c37b Mon Sep 17 00:00:00 2001 From: root Date: Thu, 5 Nov 2026 23:59:09 +0800 Subject: [PATCH] fix(sites): preserve remote throttling reasons and HTTP diagnostics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 行为对照：MoviePilot v3.2.4，6ca7b294/f4a68f36；复用现有站点共享桶，明确频控按远端秒数冷却，错误标题脱敏；修复有界响应读取对单个大数据块的限制。 --- packages/adapters/src/sites/challenge.ts | 10 +- packages/adapters/src/sites/http.test.ts | 136 +++++++++++++++++++++++ packages/adapters/src/sites/http.ts | 100 +++++++++++++++-- 3 files changed, 233 insertions(+), 13 deletions(-) create mode 100644 packages/adapters/src/sites/http.test.ts diff --git a/packages/adapters/src/sites/challenge.ts b/packages/adapters/src/sites/challenge.ts index a9027833..b1af3f13 100644 --- a/packages/adapters/src/sites/challenge.ts +++ b/packages/adapters/src/sites/challenge.ts @@ -126,7 +126,7 @@ function concatBytes(chunks: Uint8Array[], total: number): Uint8Array; complete: boolean; res: Response }> { @@ -134,14 +134,17 @@ async function peekBody( const chunks: Uint8Array[] = []; let total = 0; let complete = false; + let remainder: Uint8Array | undefined; while (total < max) { const r = await reader.read(); if (r.done) { complete = true; break; } - chunks.push(r.value); - total += r.value.byteLength; + const length = Math.min(r.value.byteLength, max - total); + chunks.push(r.value.subarray(0, length)); + total += length; + if (length < r.value.byteLength) remainder = r.value.subarray(length); } const head = concatBytes(chunks, total); const init = { status: res.status, statusText: res.statusText, headers: res.headers }; @@ -149,6 +152,7 @@ async function peekBody( const stream = new ReadableStream({ start(c) { c.enqueue(head); + if (remainder) c.enqueue(remainder); }, async pull(c) { const r = await reader.read(); diff --git a/packages/adapters/src/sites/http.test.ts b/packages/adapters/src/sites/http.test.ts new file mode 100644 index 00000000..1064f53a --- /dev/null +++ b/packages/adapters/src/sites/http.test.ts @@ -0,0 +1,136 @@ +import { describe, expect, it, vi } from "vitest"; +import { peekBody } from "./challenge.ts"; +import { type SiteHttpContext, siteFetch } from "./http.ts"; +import { createSharedSiteRateLimiter, createSiteRateLimiter } from "./rate-limit.ts"; +import { SiteError } from "./types.ts"; + +const url = "https://tracker.example/download?id=1&passkey=private-key"; +const frequency = "Request frequency limit, please try in 23 seconds."; + +function context(response: Response) { + const limiter = createSiteRateLimiter({ minIntervalMs: 0 }); + const penalize = vi.spyOn(limiter, "penalize"); + const fetch = vi.fn(async () => response); + const ctx: SiteHttpContext = { siteId: "tracker", fetch, limiter, timeoutMs: 1000, now: () => new Date(0) }; + return { ctx, penalize, fetch }; +} + +describe("站点远端频控", () => { + it.each([200, 403, 429])("HTTP %i 的HTML频控保留原因并按明确秒数顺延", async (status) => { + const { ctx, penalize, fetch } = context( + new Response(`

Blocked

${frequency}

`, { + status, + headers: { "content-type": "text/html" }, + }), + ); + const error = await siteFetch(ctx, url).catch((e: unknown) => e); + expect(error).toBeInstanceOf(SiteError); + expect(error).toMatchObject({ kind: "rate_limited", httpStatus: status, retryAfterMs: 23_000, isAuthError: false }); + expect((error as Error).message).toContain(frequency); + expect((error as Error).message).not.toContain("private-key"); + expect(penalize).toHaveBeenCalledExactlyOnceWith(23_000); + expect(fetch).toHaveBeenCalledTimes(1); + }); + + it.each([ + ["相同種子當天最多下載12次", 86_400_000], + ["Too many requests, retry after 90 minutes", 5_400_000], + [frequency, 23_000], + ])("JSON限流提示保留站点原因：%s", async (message, waitMs) => { + const { ctx, penalize } = context(Response.json({ code: "1", msg: message })); + const error = await siteFetch(ctx, url).catch((e: unknown) => e); + expect(error).toMatchObject({ kind: "rate_limited", retryAfterMs: waitMs }); + expect((error as Error).message).toContain(message); + expect(penalize).toHaveBeenCalledExactlyOnceWith(waitMs); + }); + + it("Retry-After比正文要求更长时不缩短冷却", async () => { + const { ctx, penalize } = context( + new Response(`

${frequency}

`, { status: 429, headers: { "retry-after": "60" } }), + ); + await expect(siteFetch(ctx, url)).rejects.toMatchObject({ retryAfterMs: 60_000 }); + expect(penalize).toHaveBeenCalledExactlyOnceWith(60_000); + }); + + it("频控顺延已有站点共享桶，API与下载主机仍用原站点ID", async () => { + const bucket = { + acquire: vi.fn(async () => ({ ok: true, waitedMs: 0 })), + penalize: vi.fn(async () => {}), + }; + const { ctx } = context(Response.json({ message: frequency })); + ctx.limiter = createSharedSiteRateLimiter({ key: "site:original-id", bucket, minIntervalMs: 1000 }); + await expect(siteFetch(ctx, "https://temporary-download.example/file")).rejects.toMatchObject({ + kind: "rate_limited", + }); + expect(bucket.penalize).toHaveBeenCalledExactlyOnceWith("site:original-id", 23_000, { + capacity: 1, + refillPerSec: 1, + }); + }); + + it("挑战页优先于频控文字，不将Cloudflare错误当作凭据错误", async () => { + const { ctx, penalize } = context( + new Response(`Just a moment...

${frequency}

`, { status: 403 }), + ); + await expect(siteFetch(ctx, url)).rejects.toMatchObject({ kind: "challenge", isAuthError: false }); + expect(penalize).not.toHaveBeenCalled(); + }); + + it.each([ + '{"message":"too many requests"', + '{"message":{"error":"too many requests"}}', + '["too many requests"]', + '{"message":"success"}', + "
站点限流规则说明
", + ])("不把非错误或无效结构猜成限流：%s", async (body) => { + const { ctx, penalize } = context(new Response(body)); + expect(await (await siteFetch(ctx, url)).text()).toBe(body); + expect(penalize).not.toHaveBeenCalled(); + }); + + it("有效种子的文件名即使包含频控句子也原样返回", async () => { + const bytes = new TextEncoder().encode(`d4:infod4:name${frequency.length}:${frequency}ee`); + const { ctx, penalize } = context(new Response(bytes, { headers: { "content-type": "application/x-bittorrent" } })); + expect(new Uint8Array(await (await siteFetch(ctx, url)).arrayBuffer())).toEqual(bytes); + expect(penalize).not.toHaveBeenCalled(); + }); +}); + +describe("站点HTTP失败诊断", () => { + it.each([429, 500, 503])("HTTP %i附状态和规范化页面标题，正文敏感信息不进入错误", async (status) => { + const { ctx } = context( + new Response( + `\n Service & offline passkey=private-key session=session-secret token: "token-secret" user@example.com\ncookie=body-private`, + { status, headers: { "content-type": "text/html" } }, + ), + ); + const error = await siteFetch(ctx, url).catch((e: unknown) => e); + expect(error).toMatchObject({ httpStatus: status, detail: { reason: "http", httpStatus: status } }); + expect((error as Error).message).toContain(`HTTP ${status}`); + expect((error as Error).message).toContain("页面标题：Service & offline"); + expect((error as Error).message).not.toMatch( + /private-key|session-secret|token-secret|user@example.com|body-private/, + ); + expect((error as SiteError).detail?.title).not.toMatch(/\n| {2}/); + expect((error as SiteError).detail?.title?.length).toBeLessThanOrEqual(160); + }); + + it("未识别的403保持原响应给适配器判定", async () => { + const body = "Forbidden

No permission

"; + const { ctx } = context(new Response(body, { status: 403 })); + const res = await siteFetch(ctx, url); + expect(res.status).toBe(403); + expect(await res.text()).toBe(body); + }); + + it("单次stream块超过读取预算时只检查前缀，返回正文保持完整", async () => { + const body = `${"x".repeat(70_000)}

${frequency}

`; + const { ctx, penalize } = context(new Response(body, { headers: { "content-type": "text/html" } })); + expect(await (await siteFetch(ctx, url)).text()).toBe(body); + expect(penalize).not.toHaveBeenCalled(); + const bytes = new TextEncoder().encode(body); + const peek = await peekBody(new Response(bytes), 123); + expect(peek.head.byteLength).toBe(123); + expect(new Uint8Array(await peek.res.arrayBuffer())).toEqual(bytes); + }); +}); diff --git a/packages/adapters/src/sites/http.ts b/packages/adapters/src/sites/http.ts index d9272a15..e8c79682 100644 --- a/packages/adapters/src/sites/http.ts +++ b/packages/adapters/src/sites/http.ts @@ -1,6 +1,13 @@ import { redactText } from "../rss/redact.ts"; -import { CHALLENGE_OUTCOME_HEADER, challengeOutcomeText, detectChallenge, isPageRequest } from "./challenge.ts"; -import { buildFailureSample } from "./failure-sample.ts"; +import { + CHALLENGE_OUTCOME_HEADER, + challengeOutcomeText, + detectChallenge, + isPageRequest, + peekBody, +} from "./challenge.ts"; +import { stripMarkup } from "./detail-description.ts"; +import { buildFailureSample, redactPageHtml } from "./failure-sample.ts"; import type { SiteRateLimiter } from "./rate-limit.ts"; import { SiteError } from "./types.ts"; @@ -21,10 +28,61 @@ function retryAfterMs(res: Response, now: Date): number | undefined { return Number.isNaN(t) ? undefined : Math.max(0, t - now.getTime()); } +/** 只解释明确的错误提示，不从种子字节或正常页面中的普通“限流”字样猜测失败。 */ +function rateLimitOf(body: string, status: number): { message: string; waitMs: number } | null { + let message: string | null = null; + try { + const data: unknown = JSON.parse(body); + if (data && typeof data === "object" && !Array.isArray(data)) { + const fields = data as { message?: unknown; msg?: unknown }; + if (typeof fields.message === "string") message = fields.message; + else if (typeof fields.msg === "string") message = fields.msg; + } + } catch { + // 非 JSON 只认明确的频控句子，避免把正常列表或二进制文件里的词当成错误。 + if (!/^\s*[[{]/.test(body) && (/^\s* Number.MAX_SAFE_INTEGER / 1000) return null; + return { + message: redactPageHtml(stripMarkup(message)).replace(/\s+/g, " ").trim().slice(0, 240), + waitMs: seconds * 1000, + }; +} + +function pageTitle(body: string | null): string | undefined { + const title = body && /]*>([\s\S]*?)<\/title>/i.exec(body.slice(0, 4096))?.[1]; + const safe = title && redactPageHtml(stripMarkup(title)).replace(/\s+/g, " ").trim().slice(0, 160); + return safe || undefined; +} + /** * 站点请求:先取站点令牌,再发请求;Cloudflare 挑战(`cf-mitigated: challenge`,或 403 / 503 / 429 的挑战页, * 或页面请求返回 200 的挑战页 / 人机验证门页)→ `challenge`(附脱敏页面样本) - * (不是凭据错误,CF-2);其余 429/503 统一映射并顺延限流器。 + * (不是凭据错误,CF-2);明确的远端频控按提示冷却,其余 429/503 按 Retry-After 顺延限流器。 * 返回的 Response 由调用方消费。错误消息只含脱敏后的 URL。 */ export async function siteFetch( @@ -69,21 +127,43 @@ export async function siteFetch( }), }); } + // 与挑战识别共用有界读取，重建响应保留调用方尚未消费的正文；种子等已知二进制不读。 + let body = ch.html; + const contentType = res.headers.get("content-type") ?? ""; + if (body === null && res.body && (!contentType || /text\/|json|xml/i.test(contentType))) { + const peek = await peekBody(res, 64 * 1024); + res = peek.res; + body = new TextDecoder().decode(peek.head); + } + const limited = body === null ? null : rateLimitOf(body, res.status); + if (limited) { + await res.body?.cancel().catch(() => {}); + const waitMs = Math.max(limited.waitMs, retryAfterMs(res, ctx.now()) ?? 0); + ctx.limiter.penalize(waitMs); + throw new SiteError(ctx.siteId, "rate_limited", `请求 ${safeUrl} 返回 HTTP ${res.status}：${limited.message}`, { + httpStatus: res.status, + retryAfterMs: waitMs, + detail: { reason: "rate_limited", httpStatus: res.status }, + }); + } + const title = pageTitle(body); + const httpMessage = `请求 ${safeUrl} 返回 HTTP ${res.status}${title ? `，页面标题：${title}` : ""}`; + const detail = { reason: "http", httpStatus: res.status, ...(title ? { title } : {}) }; if (res.status === 429 || res.status === 503) { const ra = retryAfterMs(res, ctx.now()); await res.body?.cancel().catch(() => {}); if (ra !== undefined) ctx.limiter.penalize(ra); - throw new SiteError( - ctx.siteId, - res.status === 429 ? "rate_limited" : "site_maintenance", - `请求 ${safeUrl} 返回 HTTP ${res.status}`, - { httpStatus: res.status, ...(ra !== undefined ? { retryAfterMs: ra } : {}) }, - ); + throw new SiteError(ctx.siteId, res.status === 429 ? "rate_limited" : "site_maintenance", httpMessage, { + httpStatus: res.status, + detail, + ...(ra !== undefined ? { retryAfterMs: ra } : {}), + }); } if (res.status >= 500) { await res.body?.cancel().catch(() => {}); - throw new SiteError(ctx.siteId, "site_maintenance", `请求 ${safeUrl} 返回 HTTP ${res.status}`, { + throw new SiteError(ctx.siteId, "site_maintenance", httpMessage, { httpStatus: res.status, + detail, }); } return res;