UHD_Media_Tool/packages/adapters/src/websearch/page.ts

import { redactUrl } from "@umt/contracts";
import { WebSearchError } from "./errors.ts";
import { assertPublicUrl, defaultResolver, type HostResolver } from "./net-guard.ts";
import { htmlToText, sanitizeForModel } from "./text.ts";

/**
 * 读取一个公开网页的纯文本(web.fetch 的网络部分)。白名单(只能读最近一次搜索结果)由服务端检查;
 * 这里负责:只允许 http(s)、每一跳(含重定向)都不能落到内网、超时、体积上限、只收 text/html | text/plain、HTML 转纯文本。
 */

export interface FetchPageOptions {
  /** 缺省 10s(含重定向与读正文) */
  timeoutMs?: number;
  /** 最多读多少字节,缺省 512 KB;超出部分丢弃并标 truncated */
  maxBytes?: number;
  /** 交给模型的正文最多多少字符,缺省 20000 */
  maxChars?: number;
  /** 最多跟随几次重定向,缺省 3 */
  maxRedirects?: number;
  userAgent?: string;
  fetch?: typeof fetch;
  resolve?: HostResolver;
  signal?: AbortSignal;
}

export interface FetchedPage {
  url: string;
  finalUrl: string;
  status: number;
  contentType: string;
  title: string | null;
  text: string;
  truncated: boolean;
  bytes: number;
  /** HTML 的 og:image / twitter:image(绝对 http(s) 地址,已脱敏);没有为 null */
  image: string | null;
}

export const PAGE_MAX_BYTES = 512 * 1024;
export const PAGE_MAX_CHARS = 20_000;
const DEFAULT_UA = "Mozilla/5.0 (compatible; umt-websearch/1.0)";

async function readCapped(res: Response, max: number): Promise<{ bytes: Uint8Array; truncated: boolean }> {
  if (!res.body) return { bytes: new Uint8Array(), truncated: false };
  const reader = res.body.getReader();
  const chunks: Uint8Array[] = [];
  let size = 0;
  let truncated = false;
  while (true) {
    const { done, value } = await reader.read();
    if (done) break;
    if (size + value.byteLength > max) {
      chunks.push(value.subarray(0, max - size));
      size = max;
      truncated = true;
      await reader.cancel().catch(() => {});
      break;
    }
    chunks.push(value);
    size += value.byteLength;
  }
  const out = new Uint8Array(size);
  let off = 0;
  for (const c of chunks) {
    out.set(c, off);
    off += c.byteLength;
  }
  return { bytes: out, truncated };
}

function charsetOf(contentType: string): string {
  const m = contentType.match(/charset\s*=\s*"?([\w-]+)"?/i);
  return m?.[1]?.toLowerCase() ?? "utf-8";
}

function decode(bytes: Uint8Array, contentType: string): string {
  let cs = charsetOf(contentType);
  // HTML 没在响应头声明字符集时看 <meta charset>
  if (!/charset/i.test(contentType)) {
    const head = new TextDecoder("latin1").decode(bytes.subarray(0, 2048));
    const m = head.match(/<meta[^>]+charset\s*=\s*["']?([\w-]+)/i);
    if (m?.[1]) cs = m[1].toLowerCase();
  }
  try {
    return new TextDecoder(cs, { fatal: false }).decode(bytes);
  } catch {
    return new TextDecoder("utf-8", { fatal: false }).decode(bytes);
  }
}

export async function fetchPage(raw: string, opts: FetchPageOptions = {}): Promise<FetchedPage> {
  const doFetch = opts.fetch ?? fetch;
  const resolve = opts.resolve ?? defaultResolver;
  const timeoutMs = opts.timeoutMs ?? 10_000;
  const maxBytes = opts.maxBytes ?? PAGE_MAX_BYTES;
  const maxRedirects = opts.maxRedirects ?? 3;
  const timeout = AbortSignal.timeout(timeoutMs);
  const signal = opts.signal ? AbortSignal.any([opts.signal, timeout]) : timeout;

  let url = await assertPublicUrl(raw, resolve);
  let res: Response | null = null;
  try {
    for (let hop = 0; ; hop++) {
      res = await doFetch(url.href, {
        redirect: "manual",
        signal,
        headers: {
          "user-agent": opts.userAgent ?? DEFAULT_UA,
          accept: "text/html,text/plain;q=0.9,*/*;q=0.1",
        },
      });
      if (res.status >= 300 && res.status < 400 && res.headers.get("location")) {
        await res.body?.cancel().catch(() => {});
        if (hop >= maxRedirects) throw new WebSearchError("blocked", `重定向超过 ${maxRedirects} 次`);
        // 每一跳都重新检查,防止公网地址 302 到内网
        url = await assertPublicUrl(new URL(res.headers.get("location")!, url).href, resolve);
        continue;
      }
      break;
    }
  } catch (err) {
    if (err instanceof WebSearchError) throw err;
    if (timeout.aborted) throw new WebSearchError("timeout", `网页 ${timeoutMs / 1000} 秒内没有响应`);
    if (opts.signal?.aborted) throw err;
    throw new WebSearchError("network", `读取网页失败：${redactUrl(String((err as Error).message ?? err))}`);
  }
  const r = res!;
  if (r.status >= 400) {
    await r.body?.cancel().catch(() => {});
    throw new WebSearchError(r.status >= 500 ? "upstream" : "bad_request", `网页返回 HTTP ${r.status}`, {
      status: r.status,
    });
  }
  const contentType = (r.headers.get("content-type") ?? "").toLowerCase();
  const mime = contentType.split(";")[0]!.trim();
  const isHtml = mime === "text/html" || mime === "application/xhtml+xml";
  if (!isHtml && mime !== "text/plain") {
    await r.body?.cancel().catch(() => {});
    throw new WebSearchError("unsupported", `只读取 text/html 与 text/plain，这个网页是 ${mime || "未知类型"}`);
  }
  let body: { bytes: Uint8Array; truncated: boolean };
  try {
    body = await readCapped(r, maxBytes);
  } catch (err) {
    if (timeout.aborted) throw new WebSearchError("timeout", `网页 ${timeoutMs / 1000} 秒内没有读完`);
    throw new WebSearchError("network", `读取网页失败：${redactUrl(String((err as Error).message ?? err))}`);
  }
  const decoded = decode(body.bytes, contentType);
  const page = isHtml ? htmlToText(decoded) : { title: null, text: decoded.replace(/\r\n?/g, "\n").trim() };
  const clean = sanitizeForModel(page.text, opts.maxChars ?? PAGE_MAX_CHARS);
  return {
    url: redactUrl(raw),
    finalUrl: redactUrl(url.href),
    status: r.status,
    contentType: mime,
    title: page.title ? sanitizeForModel(page.title, 300).text : null,
    text: clean.text,
    truncated: body.truncated || clean.truncated,
    bytes: body.bytes.byteLength,
    image: isHtml ? ogImage(decoded, url) : null,
  };
}

/** <meta property="og:image" content="…">(属性顺序不限),退而取 twitter:image;相对地址按页面地址补全 */
export function ogImage(html: string, base: URL): string | null {
  const head = html.slice(0, 200_000);
  for (const key of ["og:image", "og:image:url", "twitter:image"]) {
    for (const m of head.matchAll(/<meta\b[^>]*>/gi)) {
      const tag = m[0];
      const name = tag.match(/\b(?:property|name)\s*=\s*["']([^"']+)["']/i)?.[1]?.toLowerCase();
      if (name !== key) continue;
      const content = tag.match(/\bcontent\s*=\s*["']([^"']+)["']/i)?.[1]?.trim();
      if (!content) continue;
      try {
        const u = new URL(content.replace(/&amp;/g, "&"), base);
        if (u.protocol !== "http:" && u.protocol !== "https:") continue;
        return redactUrl(u.href).slice(0, 2048);
      } catch {}
    }
  }
  return null;
}