feat(sites): 站点请求失败留样(site_failure_sample,每站最近 5 份)

- 迁移 0077 新表 site_failure_sample:类型(parse_error / challenge)、请求类型、脱敏 URL、状态码、
  关键响应头、错误文案、命中规则、脱敏页面开头(≤ 64 KB);样本几十 KB,不放进每条请求都读的 site.config
- 适配器 onFailureSample 回调接到 runtime.siteAdapter,异步写入并删到最近 5 份,写失败只记日志
- GET /sites/{id}/failure-samples(不带正文)与 /failure-samples/{sampleId}(带正文)

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei

apps/server/src/modules/sources-identity/routes/sources.ts

@@ -17,6 +17,8 @@ import {
   SiteCredentialsPut,
   SiteCredentialsResult,
   SiteDefinitionView,
+  SiteFailureSample,
+  SiteFailureSampleSummary,
   SiteHealthResult,
   SiteRefreshLoginAccepted,
   SiteRefreshLoginInput,
@@ -47,6 +49,7 @@ import {
   userDataHistory,
 } from "../service/account.ts";
 import { audit, closeDecisions, type Exec } from "../service/common.ts";
+import { getFailureSample, listFailureSamples } from "../service/failure-sample.ts";
 import { feedDefaultsForSite } from "../service/feeds.ts";
 import { assignReleaseIdentity } from "../service/identity-admin.ts";
 import { ensureRssSite, requireCapability, siteToApi, uniqueSiteKey } from "../site-kinds.ts";
@@ -376,6 +379,37 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
     },
   );
 
+  route(
+    app,
+    {
+      method: "get",
+      path: "/sites/{id}/failure-samples",
+      scope: "read",
+      summary: "站点请求失败留样：最近几次页面结构无法识别 / 人机验证时的脱敏页面（列表不带正文）",
+      tags: T_SITES,
+      params: IdParam,
+      response: z.object({ items: z.array(SiteFailureSampleSummary) }),
+    },
+    async ({ params }) => ({ items: await listFailureSamples(rt().ctx, params.id) }),
+  );
+  route(
+    app,
+    {
+      method: "get",
+      path: "/sites/{id}/failure-samples/{sampleId}",
+      scope: "read",
+      summary: "一份失败留样（含脱敏后的页面正文）",
+      tags: T_SITES,
+      params: z.object({ id: Id, sampleId: Id }),
+      response: SiteFailureSample,
+    },
+    async ({ params }) => {
+      const s = await getFailureSample(rt().ctx, params.id, params.sampleId);
+      if (!s) throw new ApiException("not_found", "样本不存在");
+      return s;
+    },
+  );
+
   // ── W9-A 站点中心:删除、健康检测、刷新登录、用户数据、凭证 ──
   route(
     app,

apps/server/src/modules/sources-identity/runtime.ts

@@ -26,6 +26,7 @@ import {
 } from "../../shared-cache.ts";
 import type { ServerContext } from "../types.ts";
 import { CloudflareGate, withClearance } from "./service/cloudflare.ts";
+import { failureSampleSink } from "./service/failure-sample.ts";
 import { readBrowserUserAgent, resolveUserAgent, type UserAgentDefaults } from "./service/user-agent.ts";
 
 // 模块运行时依赖:站点适配器与限流器(每站一个实例,令牌在 PG 跨进程共享)、RSS 抓取器、TMDB 客户端、时钟。
@@ -333,6 +334,8 @@ export class SourcesRuntime {
       // 每条请求都经 siteFetch:代理 + 按账号现场取 UA(BR-4)
       fetch: this.siteFetch({ id: site.id, proxy }, defaults, () => this.siteUserAgent(site.id)),
       ...(baseUrl ? { baseUrl } : {}),
+      // parse_error / challenge 的脱敏页面样本存进 site_failure_sample(每站最近 5 份)
+      onFailureSample: failureSampleSink(this.ctx, site.id),
     });
     this.adapters.set(site.id, { sig, limiter, adapter });
     return adapter;

apps/server/src/modules/sources-identity/service/failure-sample.ts

@@ -0,0 +1,83 @@
+import type { SiteFailureSample as AdapterSample } from "@umt/adapters";
+import type { SiteFailureSample, SiteFailureSampleSummary } from "@umt/contracts";
+import { sql } from "@umt/db";
+import type { ServerContext } from "../../types.ts";
+import { toApi } from "../rows.ts";
+
+// 站点请求失败留样(表 site_failure_sample,迁移 0077):适配器报 parse_error / challenge 时交出脱敏页面样本
+// (SiteAdapterDeps.onFailureSample),这里存下来,每站只留最近 SAMPLES_PER_SITE 份。站点页「技术信息」里看。
+// 样本在适配器里已脱敏(个人信息块、passkey / authkey / uid / Cookie / 邮箱),这里不再处理正文。
+
+/** 每站保留的样本数 */
+export const SAMPLES_PER_SITE = 5;
+
+export async function recordFailureSample(ctx: ServerContext, siteId: string, s: AdapterSample): Promise<void> {
+  await ctx.db
+    .insertInto("site_failure_sample")
+    .values({
+      site_id: siteId,
+      kind: s.kind,
+      request_kind: s.requestKind,
+      url: s.url,
+      http_status: s.httpStatus,
+      headers: JSON.stringify(s.headers),
+      message: s.message.slice(0, 500),
+      rule: s.rule,
+      body: s.body,
+      truncated: s.truncated,
+      created_at: new Date(s.capturedAt),
+    })
+    .execute();
+  await sql`delete from site_failure_sample where site_id = ${siteId} and id not in (
+      select id from site_failure_sample where site_id = ${siteId} order by created_at desc, id desc limit ${SAMPLES_PER_SITE})`.execute(
+    ctx.db,
+  );
+}
+
+/** 留样回调:不等写库,失败只记日志(不影响站点请求本身的结果) */
+export function failureSampleSink(ctx: ServerContext, siteId: string): (s: AdapterSample) => void {
+  return (s) => {
+    void recordFailureSample(ctx, siteId, s).catch((err) =>
+      ctx.logger.warn({ siteId, err: { message: (err as Error).message } }, "站点失败页面留样写入失败"),
+    );
+  };
+}
+
+export async function listFailureSamples(ctx: ServerContext, siteId: string): Promise<SiteFailureSampleSummary[]> {
+  const rows = await ctx.db
+    .selectFrom("site_failure_sample")
+    .select([
+      "id",
+      "site_id",
+      "kind",
+      "request_kind",
+      "url",
+      "http_status",
+      "headers",
+      "message",
+      "rule",
+      "truncated",
+      "created_at",
+      sql<number>`octet_length(body)`.as("body_bytes"),
+    ])
+    .where("site_id", "=", siteId)
+    .orderBy("created_at", "desc")
+    .orderBy("id", "desc")
+    .limit(SAMPLES_PER_SITE)
+    .execute();
+  return rows.map((r) => toApi<SiteFailureSampleSummary>(r));
+}
+
+export async function getFailureSample(
+  ctx: ServerContext,
+  siteId: string,
+  sampleId: string,
+): Promise<SiteFailureSample | null> {
+  const row = await ctx.db
+    .selectFrom("site_failure_sample")
+    .selectAll()
+    .where("site_id", "=", siteId)
+    .where("id", "=", sampleId)
+    .executeTakeFirst();
+  return row ? toApi<SiteFailureSample>(row) : null;
+}

apps/server/test/sources-identity/failure-sample.test.ts

@@ -0,0 +1,129 @@
+import type { SiteError } from "@umt/adapters";
+import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest";
+import { SAMPLES_PER_SITE } from "../../src/modules/sources-identity/service/failure-sample.ts";
+import { snapshotSite } from "../../src/modules/sources-identity/service/snapshot.ts";
+import { createHarness, type Harness, pgAvailable, READ_KEY } from "./harness.ts";
+
+// 站点请求失败留样:列表 / 搜索报「页面结构无法识别」时存脱敏页面样本(site_failure_sample),每站只留最近 5 份;
+// 站点页技术信息经 GET /sites/{id}/failure-samples(列表不带正文)与 /failure-samples/{sampleId}(带正文)取。
+// 站点前面是可控的 fetch:返回什么页面由测试决定(不访问真实站点)。
+
+const up = await pgAvailable();
+const SECRET = "0123456789abcdef0123456789abcdef";
+/** 改版页:认不出种子表,还带着要脱敏的东西 */
+const BROKEN = `<html><head><title>Audiences :: 种子</title></head><body>
+<table id="info_block"><tr><td>欢迎 <a href="userdetails.php?id=29191"><b>alice_cn</b></a> UID 29191</td></tr></table>
+<p>邮箱 [email protected]</p><a href="download.php?id=1&passkey=${SECRET}">dl</a>
+<script>var authkey = "${SECRET}"; document.cookie = "c_secure_pass=topsecretvalue";</script>
+<div class="new-grid"><div class="card">Dune</div></div></body></html>`;
+
+// biome-ignore lint/suspicious/noExplicitAny: 响应体断言
+const json = async (r: Response): Promise<any> => {
+  const body = await r.json();
+  if (!r.ok) throw new Error(`${r.status} ${JSON.stringify(body)}`);
+  return body.data;
+};
+
+/** 轮询到条件成立(留样是异步写入的) */
+async function until<T>(f: () => Promise<T>, ok: (v: T) => boolean, ms = 5_000): Promise<T> {
+  const deadline = Date.now() + ms;
+  for (;;) {
+    const v = await f();
+    if (ok(v) || Date.now() > deadline) return v;
+    await new Promise((r) => setTimeout(r, 50));
+  }
+}
+
+describe.skipIf(!up)("站点失败留样", () => {
+  let h: Harness;
+  let siteId: string;
+  let page = BROKEN;
+  const fetchImpl = (async () =>
+    new Response(page, {
+      status: 200,
+      headers: { "content-type": "text/html; charset=utf-8", server: "nginx" },
+    })) as typeof fetch;
+  const site = () => h.db.selectFrom("site").selectAll().where("id", "=", siteId).executeTakeFirstOrThrow();
+  const rows = () =>
+    h.db.selectFrom("site_failure_sample").selectAll().where("site_id", "=", siteId).orderBy("created_at").execute();
+
+  beforeAll(async () => {
+    h = await createHarness("failsample", { fetch: fetchImpl });
+    const s = await json(
+      await h.req("/sites", {
+        body: {
+          key: "audiences",
+          name: "观众",
+          adapter: "nexusphp",
+          baseUrl: "https://aud.test/",
+          rateLimit: { minIntervalMs: 0, concurrency: 10 },
+        },
+      }),
+    );
+    siteId = s.id;
+    await json(
+      await h.req(`/sites/${siteId}/credentials`, {
+        method: "PUT",
+        body: { kind: "cookie", cookie: "c_secure_uid=1; c_secure_pass=abc" },
+      }),
+    );
+  });
+  afterAll(async () => {
+    await h?.close();
+  });
+  beforeEach(async () => {
+    page = BROKEN;
+    await h.db.deleteFrom("site_failure_sample").where("site_id", "=", siteId).execute();
+  });
+
+  it("列表快照报 parse_error → 留脱敏样本(URL、状态码、关键响应头、请求类型);接口可看", async () => {
+    const r = await snapshotSite(h.rt, { siteId, kind: "manual" });
+    expect(r.status).toBe("failed");
+    expect(r.error).toContain("parse_error");
+    const got = await until(rows, (x) => x.length > 0);
+    expect(got).toHaveLength(1);
+    const s = got[0]!;
+    expect(s).toMatchObject({
+      kind: "parse_error",
+      request_kind: "list",
+      http_status: 200,
+      headers: { server: "nginx" },
+    });
+    expect(s.message).toContain("页面结构无法识别");
+    for (const leak of [SECRET, "[email protected]", "alice_cn", "29191", "topsecretvalue"]) {
+      expect(s.body).not.toContain(leak);
+      expect(s.url).not.toContain(leak);
+    }
+    expect(s.body).toContain("new-grid");
+
+    const list = await json(await h.req(`/sites/${siteId}/failure-samples`, { key: READ_KEY }));
+    expect(list.items).toHaveLength(1);
+    expect(list.items[0]).toMatchObject({ id: s.id, kind: "parse_error", requestKind: "list", httpStatus: 200 });
+    expect(list.items[0].body).toBeUndefined();
+    expect(list.items[0].bodyBytes).toBeGreaterThan(0);
+    const one = await json(await h.req(`/sites/${siteId}/failure-samples/${s.id}`, { key: READ_KEY }));
+    expect(one.body).toBe(s.body);
+    const miss = await h.req(`/sites/${siteId}/failure-samples/00000000-0000-4000-8000-000000000000`);
+    expect(miss.status).toBe(404);
+  });
+
+  it(`每站只留最近 ${SAMPLES_PER_SITE} 份`, async () => {
+    const adapter = h.rt.siteAdapter(await site());
+    for (let i = 1; i <= 7; i++) {
+      const e = await adapter.search(`k${i}`).catch((x) => x);
+      expect((e as SiteError).kind).toBe("parse_error");
+    }
+    const kw = (x: { url: string }[]) => x.map((r) => /search=(k\d)/.exec(r.url)?.[1]).join(",");
+    const got = await until(rows, (x) => kw(x) === "k3,k4,k5,k6,k7");
+    expect(kw(got)).toBe("k3,k4,k5,k6,k7");
+    expect(got.every((r) => r.request_kind === "search")).toBe(true);
+  });
+
+  it("正常页面不留样", async () => {
+    page = `<html><head><title>Audiences :: 种子</title></head><body><table class="torrents"><tr><td>没有种子</td></tr></table></body></html>`;
+    const adapter = h.rt.siteAdapter(await site());
+    await adapter.search("x").catch(() => null);
+    await new Promise((r) => setTimeout(r, 200));
+    expect(await rows()).toEqual([]);
+  });
+});