feat(agent): web.search / web.fetch via Brave Search (WEB-1)
- adapters/websearch: Brave client (X-Subscription-Token, error classes), result normalization with redaction/truncation, SSRF guard, HTML→text fetch (10s, 512KB, text/html|text/plain only) - modules/websearch: 10-min PG cache (region websearch), cross-process per-minute limit, web.fetch allowlist = URLs from the caller's latest web.search; POST /websearch/test - settings.agent.webSearch (key write-only → secret_ref); settings card under Agent; web.* offered to the model only when enabled, plus a system prompt rule; outputs marked untrusted - capability count 27 → 29; e2e E13; runbook docs/runbook/websearch.md Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
40dd0e69e4
commit
d53df25fc8
32 files changed
+1854
-39
No files matched your search
@@ -7,3 +7,4 @@ export * from "./qbittorrent/index.ts";
|
||||
export * from "./rss/index.ts";
|
||||
export * from "./sites/index.ts";
|
||||
export * from "./tmdb/index.ts";
|
||||
export * from "./websearch/index.ts";
|
||||
@@ -0,0 +1,141 @@
|
||||
import { redactUrl, type WebSearchResult } from "@umt/contracts";
|
||||
import { WebSearchError } from "./errors.ts";
|
||||
import { sanitizeForModel, stripInlineHtml } from "./text.ts";
|
||||
|
||||
/**
|
||||
* Brave Web Search API 客户端(GET /res/v1/web/search,X-Subscription-Token 鉴权)。
|
||||
* 只做请求、错误分类与结果归一化;限流、缓存、Key 存取在服务端(modules/websearch)。
|
||||
* 文档:https://api-dashboard.search.brave.com/app/documentation/web-search/query
|
||||
*/
|
||||
|
||||
export const BRAVE_BASE_URL = "https://api.search.brave.com";
|
||||
|
||||
export interface BraveClientOptions {
|
||||
apiKey: string;
|
||||
/** 测试指向 mock;缺省官方地址 */
|
||||
baseUrl?: string;
|
||||
/** 缺省 10s */
|
||||
timeoutMs?: number;
|
||||
fetch?: typeof fetch;
|
||||
}
|
||||
|
||||
export interface BraveSearchParams {
|
||||
q: string;
|
||||
/** 1–20 */
|
||||
count: number;
|
||||
/** 两位国家码或 ALL */
|
||||
country?: string | null;
|
||||
search_lang?: string | null;
|
||||
safesearch?: "off" | "moderate" | "strict";
|
||||
freshness?: "pd" | "pw" | "pm" | "py";
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export interface BraveSearchResponse {
|
||||
results: WebSearchResult[];
|
||||
durationMs: number;
|
||||
}
|
||||
|
||||
/** 单条结果的上限(字符) */
|
||||
export const TITLE_MAX = 200;
|
||||
export const SNIPPET_MAX = 500;
|
||||
export const URL_MAX = 2048;
|
||||
|
||||
type Raw = Record<string, unknown>;
|
||||
const str = (v: unknown) => (typeof v === "string" ? v : "");
|
||||
|
||||
/** Brave web.results → 归一化结果:去 HTML、脱敏、截断;没有 http(s) 链接的丢弃 */
|
||||
export function normalizeBraveResults(body: unknown): WebSearchResult[] {
|
||||
const web = (body as Raw | null)?.web as Raw | undefined;
|
||||
const list = Array.isArray(web?.results) ? (web.results as Raw[]) : [];
|
||||
const out: WebSearchResult[] = [];
|
||||
for (const r of list) {
|
||||
const url = str(r.url);
|
||||
let host: string;
|
||||
try {
|
||||
const u = new URL(url);
|
||||
if (u.protocol !== "http:" && u.protocol !== "https:") continue;
|
||||
host = u.hostname;
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (url.length > URL_MAX) continue;
|
||||
const extra = Array.isArray(r.extra_snippets) ? (r.extra_snippets as unknown[]).map(str).filter(Boolean) : [];
|
||||
const snippet = [str(r.description), ...extra.slice(0, 1)].map(stripInlineHtml).filter(Boolean).join(" ");
|
||||
const age = str(r.age) || str(r.page_age) || null;
|
||||
out.push({
|
||||
title: sanitizeForModel(stripInlineHtml(str(r.title)) || host, TITLE_MAX).text,
|
||||
url: redactUrl(url),
|
||||
snippet: sanitizeForModel(snippet, SNIPPET_MAX).text,
|
||||
age: age ? age.slice(0, 64) : null,
|
||||
source: str((r.meta_url as Raw | undefined)?.hostname) || host,
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function retryAfterMs(res: Response): number | undefined {
|
||||
// Brave:X-RateLimit-Reset 为逗号分隔的秒数(第一项是每秒窗口);也兼容标准 Retry-After
|
||||
const ra = res.headers.get("retry-after") ?? res.headers.get("x-ratelimit-reset")?.split(",")[0];
|
||||
const n = ra ? Number(ra.trim()) : Number.NaN;
|
||||
return Number.isFinite(n) && n >= 0 ? Math.ceil(n * 1000) : undefined;
|
||||
}
|
||||
|
||||
export function createBraveClient(opts: BraveClientOptions) {
|
||||
const base = (opts.baseUrl ?? BRAVE_BASE_URL).replace(/\/+$/, "");
|
||||
const doFetch = opts.fetch ?? fetch;
|
||||
const timeoutMs = opts.timeoutMs ?? 10_000;
|
||||
const scrub = (s: string) => redactUrl(s, [opts.apiKey]);
|
||||
|
||||
async function search(p: BraveSearchParams): Promise<BraveSearchResponse> {
|
||||
if (!opts.apiKey) throw new WebSearchError("auth", "未配置搜索服务 API Key");
|
||||
const q = new URLSearchParams({ q: p.q, count: String(Math.min(20, Math.max(1, p.count))) });
|
||||
if (p.country) q.set("country", p.country);
|
||||
if (p.search_lang) q.set("search_lang", p.search_lang);
|
||||
if (p.safesearch) q.set("safesearch", p.safesearch);
|
||||
if (p.freshness) q.set("freshness", p.freshness);
|
||||
q.set("text_decorations", "false");
|
||||
const timeout = AbortSignal.timeout(timeoutMs);
|
||||
const signal = p.signal ? AbortSignal.any([p.signal, timeout]) : timeout;
|
||||
const started = Date.now();
|
||||
let res: Response;
|
||||
try {
|
||||
res = await doFetch(`${base}/res/v1/web/search?${q}`, {
|
||||
headers: { accept: "application/json", "x-subscription-token": opts.apiKey },
|
||||
signal,
|
||||
});
|
||||
} catch (err) {
|
||||
if (timeout.aborted) throw new WebSearchError("timeout", `搜索服务 ${timeoutMs / 1000} 秒内没有响应`);
|
||||
if (p.signal?.aborted) throw err;
|
||||
throw new WebSearchError("network", `连不上搜索服务：${scrub(String((err as Error).message ?? err))}`);
|
||||
}
|
||||
if (!res.ok) {
|
||||
const text = scrub((await res.text().catch(() => "")).slice(0, 300));
|
||||
const s = res.status;
|
||||
if (s === 401 || s === 403) throw new WebSearchError("auth", `搜索服务拒绝了 API Key(HTTP ${s})`, { status: s });
|
||||
if (s === 402) throw new WebSearchError("quota", "搜索服务额度已用尽或订阅不含网页搜索", { status: s });
|
||||
if (s === 429) {
|
||||
const ra = retryAfterMs(res);
|
||||
throw new WebSearchError("rate", "搜索服务限流，稍后再试", {
|
||||
status: s,
|
||||
...(ra !== undefined ? { retryAfterMs: ra } : {}),
|
||||
});
|
||||
}
|
||||
if (s === 400 || s === 422)
|
||||
throw new WebSearchError("bad_request", `搜索参数不被接受(HTTP ${s})：${text}`, { status: s });
|
||||
throw new WebSearchError("upstream", `搜索服务出错(HTTP ${s})`, { status: s });
|
||||
}
|
||||
let body: unknown;
|
||||
try {
|
||||
body = await res.json();
|
||||
} catch {
|
||||
throw new WebSearchError("protocol", "搜索服务返回的不是 JSON");
|
||||
}
|
||||
if (!body || typeof body !== "object") throw new WebSearchError("protocol", "搜索服务返回结构不对");
|
||||
return { results: normalizeBraveResults(body).slice(0, p.count), durationMs: Date.now() - started };
|
||||
}
|
||||
|
||||
return { search };
|
||||
}
|
||||
|
||||
export type BraveClient = ReturnType<typeof createBraveClient>;
|
||||
@@ -0,0 +1,33 @@
|
||||
/** 网页搜索 / 网页读取的错误分类(调用方按 kind 映射成能力错误;message 不含 Key) */
|
||||
export type WebSearchErrorKind =
|
||||
/** Key 缺失、无效或被吊销(401/403) */
|
||||
| "auth"
|
||||
/** 上游限流(429)或本地每分钟上限 */
|
||||
| "rate"
|
||||
/** 额度用尽 / 订阅不含该接口(402) */
|
||||
| "quota"
|
||||
/** 参数不被接受(400/422) */
|
||||
| "bad_request"
|
||||
/** 上游 5xx */
|
||||
| "upstream"
|
||||
| "network"
|
||||
| "timeout"
|
||||
/** 响应不是预期结构 */
|
||||
| "protocol"
|
||||
/** web.fetch:地址不允许(内网、非 http(s)、不在搜索结果里) */
|
||||
| "blocked"
|
||||
/** web.fetch:内容类型不是 text/html | text/plain */
|
||||
| "unsupported";
|
||||
|
||||
export class WebSearchError extends Error {
|
||||
override readonly name = "WebSearchError";
|
||||
constructor(
|
||||
readonly kind: WebSearchErrorKind,
|
||||
message: string,
|
||||
readonly detail: { status?: number; retryAfterMs?: number } = {},
|
||||
) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
export const isWebSearchError = (e: unknown): e is WebSearchError => e instanceof WebSearchError;
|
||||
@@ -0,0 +1,5 @@
|
||||
export * from "./brave.ts";
|
||||
export * from "./errors.ts";
|
||||
export * from "./net-guard.ts";
|
||||
export * from "./page.ts";
|
||||
export { htmlToText, stripInlineHtml } from "./text.ts";
|
||||
@@ -0,0 +1,146 @@
|
||||
import { lookup } from "node:dns/promises";
|
||||
import { isIP } from "node:net";
|
||||
import { WebSearchError } from "./errors.ts";
|
||||
|
||||
// web.fetch 的地址守卫:只允许 http(s)、默认端口以外也可,但主机不能是内网 / 回环 / 链路本地 / 保留地址。
|
||||
// 主机名先看后缀(localhost、.local、.internal、单标签名),再解析 DNS,任一解析结果落在内网段即拒绝。
|
||||
// 仍有 DNS 重绑定的窗口(检查与连接之间重新解析),见 docs/runbook/websearch.md「已知限制」。
|
||||
|
||||
/** 解析主机名 → IP 列表(测试注入) */
|
||||
export type HostResolver = (host: string) => Promise<string[]>;
|
||||
|
||||
export const defaultResolver: HostResolver = async (host) => {
|
||||
const r = await lookup(host, { all: true, verbatim: true });
|
||||
return r.map((x) => x.address);
|
||||
};
|
||||
|
||||
const BLOCKED_SUFFIX = [".localhost", ".local", ".internal", ".lan", ".home.arpa", ".intranet", ".corp"];
|
||||
|
||||
function v4ToInt(ip: string): number | null {
|
||||
const parts = ip.split(".");
|
||||
if (parts.length !== 4) return null;
|
||||
let n = 0;
|
||||
for (const p of parts) {
|
||||
if (!/^\d{1,3}$/.test(p)) return null;
|
||||
const v = Number(p);
|
||||
if (v > 255) return null;
|
||||
n = n * 256 + v;
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
const V4_BLOCKS: [string, number][] = [
|
||||
["0.0.0.0", 8],
|
||||
["10.0.0.0", 8],
|
||||
["100.64.0.0", 10],
|
||||
["127.0.0.0", 8],
|
||||
["169.254.0.0", 16],
|
||||
["172.16.0.0", 12],
|
||||
["192.0.0.0", 24],
|
||||
["192.0.2.0", 24],
|
||||
["192.168.0.0", 16],
|
||||
["198.18.0.0", 15],
|
||||
["198.51.100.0", 24],
|
||||
["203.0.113.0", 24],
|
||||
["224.0.0.0", 4],
|
||||
["240.0.0.0", 4],
|
||||
];
|
||||
|
||||
function v4Blocked(ip: string): boolean {
|
||||
const n = v4ToInt(ip);
|
||||
if (n === null) return true;
|
||||
return V4_BLOCKS.some(([base, bits]) => {
|
||||
const b = v4ToInt(base)!;
|
||||
const size = 2 ** (32 - bits);
|
||||
return n >= b && n < b + size;
|
||||
});
|
||||
}
|
||||
|
||||
/** 把 IPv6 展开成 8 组 16 位整数;非法返回 null */
|
||||
function v6Groups(ip: string): number[] | null {
|
||||
let s = ip.toLowerCase();
|
||||
const pct = s.indexOf("%");
|
||||
if (pct >= 0) s = s.slice(0, pct);
|
||||
// 末尾是点分 IPv4(::ffff:1.2.3.4)
|
||||
const m = s.match(/^(.*:)(\d+\.\d+\.\d+\.\d+)$/);
|
||||
if (m) {
|
||||
const n = v4ToInt(m[2]!);
|
||||
if (n === null) return null;
|
||||
s = `${m[1]}${(n >>> 16).toString(16)}:${(n & 0xffff).toString(16)}`;
|
||||
}
|
||||
const halves = s.split("::");
|
||||
if (halves.length > 2) return null;
|
||||
const head = halves[0] ? halves[0].split(":") : [];
|
||||
const tail = halves.length === 2 && halves[1] ? halves[1].split(":") : [];
|
||||
const fill = halves.length === 2 ? 8 - head.length - tail.length : 0;
|
||||
if (fill < 0) return null;
|
||||
const groups = [...head, ...Array(fill).fill("0"), ...tail];
|
||||
if (groups.length !== 8) return null;
|
||||
const out = groups.map((g) => (/^[0-9a-f]{1,4}$/.test(g) ? Number.parseInt(g, 16) : Number.NaN));
|
||||
return out.some((x) => Number.isNaN(x)) ? null : out;
|
||||
}
|
||||
|
||||
function v6Blocked(ip: string): boolean {
|
||||
const g = v6Groups(ip);
|
||||
if (!g) return true;
|
||||
const allZero = (from: number, to: number) => g.slice(from, to).every((x) => x === 0);
|
||||
if (allZero(0, 8)) return true; // ::
|
||||
if (allZero(0, 7) && g[7] === 1) return true; // ::1
|
||||
// IPv4 映射 / 兼容 / NAT64:按内嵌的 IPv4 判断
|
||||
const embedded = `${g[6]! >> 8}.${g[6]! & 255}.${g[7]! >> 8}.${g[7]! & 255}`;
|
||||
if (allZero(0, 5) && (g[5] === 0xffff || g[5] === 0)) return v4Blocked(embedded);
|
||||
if (g[0] === 0x64 && g[1] === 0xff9b && allZero(2, 6)) return v4Blocked(embedded);
|
||||
const first = g[0]!;
|
||||
if ((first & 0xfe00) === 0xfc00) return true; // fc00::/7 唯一本地
|
||||
if ((first & 0xffc0) === 0xfe80) return true; // fe80::/10 链路本地
|
||||
if ((first & 0xffc0) === 0xfec0) return true; // fec0::/10 站点本地(已弃用)
|
||||
if ((first & 0xff00) === 0xff00) return true; // 组播
|
||||
if (first === 0x2001 && g[1] === 0x0db8) return true; // 文档
|
||||
return false;
|
||||
}
|
||||
|
||||
/** IP 是否落在内网 / 回环 / 链路本地 / 保留段 */
|
||||
export function isBlockedIp(ip: string): boolean {
|
||||
const v = isIP(ip);
|
||||
if (v === 4) return v4Blocked(ip);
|
||||
if (v === 6) return v6Blocked(ip);
|
||||
return true;
|
||||
}
|
||||
|
||||
/** 主机名本身(不解析)是否就是内网名 */
|
||||
export function isBlockedHostname(host: string): boolean {
|
||||
const h = host.toLowerCase().replace(/\.$/, "");
|
||||
if (!h) return true;
|
||||
if (h === "localhost" || BLOCKED_SUFFIX.some((s) => h.endsWith(s))) return true;
|
||||
const bare = h.startsWith("[") && h.endsWith("]") ? h.slice(1, -1) : h;
|
||||
if (isIP(bare)) return isBlockedIp(bare);
|
||||
// 单标签名(intranet、nas…)只会解析到本地网络
|
||||
if (!h.includes(".")) return true;
|
||||
// 十进制 / 十六进制写法的 IPv4(http://2130706433/、0x7f.1)
|
||||
if (/^(0x[0-9a-f]+|\d+)(\.(0x[0-9a-f]+|\d+))*$/i.test(h)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/** 校验 URL:只允许 http(s);主机名与所有解析结果都不能是内网地址。返回规范化后的 URL */
|
||||
export async function assertPublicUrl(raw: string, resolve: HostResolver = defaultResolver): Promise<URL> {
|
||||
let u: URL;
|
||||
try {
|
||||
u = new URL(raw);
|
||||
} catch {
|
||||
throw new WebSearchError("blocked", "不是合法的网址");
|
||||
}
|
||||
if (u.protocol !== "http:" && u.protocol !== "https:") throw new WebSearchError("blocked", "只能读取 http(s) 网址");
|
||||
if (u.username || u.password) throw new WebSearchError("blocked", "网址里不能带账号密码");
|
||||
if (isBlockedHostname(u.hostname)) throw new WebSearchError("blocked", "不能读取内网或本机地址");
|
||||
const bare = u.hostname.startsWith("[") ? u.hostname.slice(1, -1) : u.hostname;
|
||||
if (isIP(bare)) return u;
|
||||
let ips: string[];
|
||||
try {
|
||||
ips = await resolve(u.hostname);
|
||||
} catch {
|
||||
throw new WebSearchError("network", `解析不到主机 ${u.hostname}`);
|
||||
}
|
||||
if (ips.length === 0) throw new WebSearchError("network", `解析不到主机 ${u.hostname}`);
|
||||
if (ips.some(isBlockedIp)) throw new WebSearchError("blocked", "不能读取内网或本机地址");
|
||||
return u;
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
import { redactUrl } from "@umt/contracts";
|
||||
import { WebSearchError } from "./errors.ts";
|
||||
import { assertPublicUrl, defaultResolver, type HostResolver } from "./net-guard.ts";
|
||||
import { htmlToText, sanitizeForModel } from "./text.ts";
|
||||
|
||||
/**
|
||||
* 读取一个公开网页的纯文本(web.fetch 的网络部分)。白名单(只能读最近一次搜索结果)由服务端检查;
|
||||
* 这里负责:只允许 http(s)、每一跳(含重定向)都不能落到内网、超时、体积上限、只收 text/html | text/plain、HTML 转纯文本。
|
||||
*/
|
||||
|
||||
export interface FetchPageOptions {
|
||||
/** 缺省 10s(含重定向与读正文) */
|
||||
timeoutMs?: number;
|
||||
/** 最多读多少字节,缺省 512 KB;超出部分丢弃并标 truncated */
|
||||
maxBytes?: number;
|
||||
/** 交给模型的正文最多多少字符,缺省 20000 */
|
||||
maxChars?: number;
|
||||
/** 最多跟随几次重定向,缺省 3 */
|
||||
maxRedirects?: number;
|
||||
userAgent?: string;
|
||||
fetch?: typeof fetch;
|
||||
resolve?: HostResolver;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export interface FetchedPage {
|
||||
url: string;
|
||||
finalUrl: string;
|
||||
status: number;
|
||||
contentType: string;
|
||||
title: string | null;
|
||||
text: string;
|
||||
truncated: boolean;
|
||||
bytes: number;
|
||||
}
|
||||
|
||||
export const PAGE_MAX_BYTES = 512 * 1024;
|
||||
export const PAGE_MAX_CHARS = 20_000;
|
||||
const DEFAULT_UA = "Mozilla/5.0 (compatible; umt-websearch/1.0)";
|
||||
|
||||
async function readCapped(res: Response, max: number): Promise<{ bytes: Uint8Array; truncated: boolean }> {
|
||||
if (!res.body) return { bytes: new Uint8Array(), truncated: false };
|
||||
const reader = res.body.getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
let size = 0;
|
||||
let truncated = false;
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
if (size + value.byteLength > max) {
|
||||
chunks.push(value.subarray(0, max - size));
|
||||
size = max;
|
||||
truncated = true;
|
||||
await reader.cancel().catch(() => {});
|
||||
break;
|
||||
}
|
||||
chunks.push(value);
|
||||
size += value.byteLength;
|
||||
}
|
||||
const out = new Uint8Array(size);
|
||||
let off = 0;
|
||||
for (const c of chunks) {
|
||||
out.set(c, off);
|
||||
off += c.byteLength;
|
||||
}
|
||||
return { bytes: out, truncated };
|
||||
}
|
||||
|
||||
function charsetOf(contentType: string): string {
|
||||
const m = contentType.match(/charset\s*=\s*"?([\w-]+)"?/i);
|
||||
return m?.[1]?.toLowerCase() ?? "utf-8";
|
||||
}
|
||||
|
||||
function decode(bytes: Uint8Array, contentType: string): string {
|
||||
let cs = charsetOf(contentType);
|
||||
// HTML 没在响应头声明字符集时看 <meta charset>
|
||||
if (!/charset/i.test(contentType)) {
|
||||
const head = new TextDecoder("latin1").decode(bytes.subarray(0, 2048));
|
||||
const m = head.match(/<meta[^>]+charset\s*=\s*["']?([\w-]+)/i);
|
||||
if (m?.[1]) cs = m[1].toLowerCase();
|
||||
}
|
||||
try {
|
||||
return new TextDecoder(cs, { fatal: false }).decode(bytes);
|
||||
} catch {
|
||||
return new TextDecoder("utf-8", { fatal: false }).decode(bytes);
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchPage(raw: string, opts: FetchPageOptions = {}): Promise<FetchedPage> {
|
||||
const doFetch = opts.fetch ?? fetch;
|
||||
const resolve = opts.resolve ?? defaultResolver;
|
||||
const timeoutMs = opts.timeoutMs ?? 10_000;
|
||||
const maxBytes = opts.maxBytes ?? PAGE_MAX_BYTES;
|
||||
const maxRedirects = opts.maxRedirects ?? 3;
|
||||
const timeout = AbortSignal.timeout(timeoutMs);
|
||||
const signal = opts.signal ? AbortSignal.any([opts.signal, timeout]) : timeout;
|
||||
|
||||
let url = await assertPublicUrl(raw, resolve);
|
||||
let res: Response | null = null;
|
||||
try {
|
||||
for (let hop = 0; ; hop++) {
|
||||
res = await doFetch(url.href, {
|
||||
redirect: "manual",
|
||||
signal,
|
||||
headers: {
|
||||
"user-agent": opts.userAgent ?? DEFAULT_UA,
|
||||
accept: "text/html,text/plain;q=0.9,*/*;q=0.1",
|
||||
},
|
||||
});
|
||||
if (res.status >= 300 && res.status < 400 && res.headers.get("location")) {
|
||||
await res.body?.cancel().catch(() => {});
|
||||
if (hop >= maxRedirects) throw new WebSearchError("blocked", `重定向超过 ${maxRedirects} 次`);
|
||||
// 每一跳都重新检查,防止公网地址 302 到内网
|
||||
url = await assertPublicUrl(new URL(res.headers.get("location")!, url).href, resolve);
|
||||
continue;
|
||||
}
|
||||
break;
|
||||
}
|
||||
} catch (err) {
|
||||
if (err instanceof WebSearchError) throw err;
|
||||
if (timeout.aborted) throw new WebSearchError("timeout", `网页 ${timeoutMs / 1000} 秒内没有响应`);
|
||||
if (opts.signal?.aborted) throw err;
|
||||
throw new WebSearchError("network", `读取网页失败：${redactUrl(String((err as Error).message ?? err))}`);
|
||||
}
|
||||
const r = res!;
|
||||
if (r.status >= 400) {
|
||||
await r.body?.cancel().catch(() => {});
|
||||
throw new WebSearchError(r.status >= 500 ? "upstream" : "bad_request", `网页返回 HTTP ${r.status}`, {
|
||||
status: r.status,
|
||||
});
|
||||
}
|
||||
const contentType = (r.headers.get("content-type") ?? "").toLowerCase();
|
||||
const mime = contentType.split(";")[0]!.trim();
|
||||
const isHtml = mime === "text/html" || mime === "application/xhtml+xml";
|
||||
if (!isHtml && mime !== "text/plain") {
|
||||
await r.body?.cancel().catch(() => {});
|
||||
throw new WebSearchError("unsupported", `只读取 text/html 与 text/plain，这个网页是 ${mime || "未知类型"}`);
|
||||
}
|
||||
let body: { bytes: Uint8Array; truncated: boolean };
|
||||
try {
|
||||
body = await readCapped(r, maxBytes);
|
||||
} catch (err) {
|
||||
if (timeout.aborted) throw new WebSearchError("timeout", `网页 ${timeoutMs / 1000} 秒内没有读完`);
|
||||
throw new WebSearchError("network", `读取网页失败：${redactUrl(String((err as Error).message ?? err))}`);
|
||||
}
|
||||
const decoded = decode(body.bytes, contentType);
|
||||
const page = isHtml ? htmlToText(decoded) : { title: null, text: decoded.replace(/\r\n?/g, "\n").trim() };
|
||||
const clean = sanitizeForModel(page.text, opts.maxChars ?? PAGE_MAX_CHARS);
|
||||
return {
|
||||
url: redactUrl(raw),
|
||||
finalUrl: redactUrl(url.href),
|
||||
status: r.status,
|
||||
contentType: mime,
|
||||
title: page.title ? sanitizeForModel(page.title, 300).text : null,
|
||||
text: clean.text,
|
||||
truncated: body.truncated || clean.truncated,
|
||||
bytes: body.bytes.byteLength,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
import { redactString } from "@umt/contracts";
|
||||
import { load } from "cheerio";
|
||||
|
||||
// 网页文本工具:HTML → 纯文本(去脚本、样式、表单等非正文元素)、实体解码、截断与脱敏。
|
||||
|
||||
const NAMED: Record<string, string> = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " " };
|
||||
|
||||
/** 去掉摘要里的内联标签(Brave 用 <strong> 标关键字)并解码实体 */
|
||||
export function stripInlineHtml(s: string): string {
|
||||
return s
|
||||
.replace(/<[^>]*>/g, "")
|
||||
.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (m, e: string) => {
|
||||
if (e[0] === "#") {
|
||||
const code = e[1] === "x" || e[1] === "X" ? Number.parseInt(e.slice(2), 16) : Number(e.slice(1));
|
||||
return Number.isFinite(code) && code > 0 && code < 0x110000 ? String.fromCodePoint(code) : m;
|
||||
}
|
||||
return NAMED[e.toLowerCase()] ?? m;
|
||||
})
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
/** 按字符截断,末尾加 … */
|
||||
export function truncate(s: string, max: number): { text: string; truncated: boolean } {
|
||||
if (s.length <= max) return { text: s, truncated: false };
|
||||
return { text: `${s.slice(0, Math.max(0, max - 1))}…`, truncated: true };
|
||||
}
|
||||
|
||||
/** 交给模型前的清理:脱敏(URL 参数里的 passkey / token 等)+ 截断 */
|
||||
export function sanitizeForModel(s: string, max: number): { text: string; truncated: boolean } {
|
||||
return truncate(redactString(s), max);
|
||||
}
|
||||
|
||||
const DROP = "script,style,noscript,svg,canvas,iframe,object,embed,form,input,select,button,link,meta";
|
||||
const BLOCK =
|
||||
"p,div,section,article,main,header,footer,aside,nav,li,ul,ol,table,tr,h1,h2,h3,h4,h5,h6,pre,blockquote,dd,dt,br,hr";
|
||||
|
||||
/** HTML → { title, text }:去脚本样式等,块级元素换行,合并空白 */
|
||||
export function htmlToText(html: string): { title: string | null; text: string } {
|
||||
// <template> 的内容不在 DOM 文本里,但 Wiki.js 等把正文放在 <template slot="contents">:先改成普通元素
|
||||
const $ = load(html.replace(/<template\b/gi, "<div").replace(/<\/template>/gi, "</div>"));
|
||||
const title = $("title").first().text().replace(/\s+/g, " ").trim() || null;
|
||||
$(DROP).remove();
|
||||
$("head").remove();
|
||||
$(BLOCK).each((_i, el) => {
|
||||
$(el).prepend("\n").append("\n");
|
||||
});
|
||||
const body = $("body").length ? $("body").text() : $.root().text();
|
||||
const text = body
|
||||
.split("\n")
|
||||
.map((l) => l.replace(/[ \t\f\v ]+/g, " ").trim())
|
||||
.filter((l) => l.length > 0)
|
||||
.join("\n");
|
||||
return { title, text };
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
assertPublicUrl,
|
||||
createBraveClient,
|
||||
fetchPage,
|
||||
htmlToText,
|
||||
isBlockedHostname,
|
||||
isBlockedIp,
|
||||
normalizeBraveResults,
|
||||
WebSearchError,
|
||||
} from "./index.ts";
|
||||
|
||||
const KEY = "BSA-test-key-1234567890";
|
||||
const publicDns = async () => ["93.184.216.34"];
|
||||
|
||||
function jsonRes(body: unknown, status = 200, headers: Record<string, string> = {}) {
|
||||
return new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json", ...headers } });
|
||||
}
|
||||
|
||||
const BRAVE_BODY = {
|
||||
web: {
|
||||
results: [
|
||||
{
|
||||
title: "MoviePilot <strong>v3</strong> 订阅规则",
|
||||
url: "https://example.org/mp?passkey=abcdef0123456789abcdef0123456789",
|
||||
description: "订阅规则 & <strong>过滤</strong>",
|
||||
age: "2 days ago",
|
||||
meta_url: { hostname: "example.org" },
|
||||
},
|
||||
{ title: "ftp", url: "ftp://example.org/x", description: "丢弃" },
|
||||
{ title: "", url: "https://news.example.com/a", description: "x".repeat(2000) },
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
describe("Brave 客户端", () => {
|
||||
it("带 X-Subscription-Token 与查询参数;结果归一化、去标签、脱敏、截断", async () => {
|
||||
let seen: { url: string; headers: Headers } | undefined;
|
||||
const c = createBraveClient({
|
||||
apiKey: KEY,
|
||||
fetch: (async (url: string, init: RequestInit) => {
|
||||
seen = { url, headers: new Headers(init.headers) };
|
||||
return jsonRes(BRAVE_BODY);
|
||||
}) as typeof fetch,
|
||||
});
|
||||
const r = await c.search({
|
||||
q: "MoviePilot v3",
|
||||
count: 5,
|
||||
country: "CN",
|
||||
search_lang: "zh-hans",
|
||||
safesearch: "moderate",
|
||||
freshness: "pw",
|
||||
});
|
||||
const u = new URL(seen!.url);
|
||||
expect(u.pathname).toBe("/res/v1/web/search");
|
||||
expect(Object.fromEntries(u.searchParams)).toMatchObject({
|
||||
q: "MoviePilot v3",
|
||||
count: "5",
|
||||
country: "CN",
|
||||
search_lang: "zh-hans",
|
||||
safesearch: "moderate",
|
||||
freshness: "pw",
|
||||
});
|
||||
expect(seen!.headers.get("x-subscription-token")).toBe(KEY);
|
||||
expect(r.results).toHaveLength(2);
|
||||
expect(r.results[0]).toMatchObject({
|
||||
title: "MoviePilot v3 订阅规则",
|
||||
snippet: "订阅规则 & 过滤",
|
||||
age: "2 days ago",
|
||||
source: "example.org",
|
||||
});
|
||||
expect(r.results[0]!.url).not.toContain("abcdef0123456789");
|
||||
expect(r.results[1]!.title).toBe("news.example.com");
|
||||
expect(r.results[1]!.snippet.length).toBeLessThanOrEqual(500);
|
||||
expect(r.results[1]!.age).toBeNull();
|
||||
});
|
||||
|
||||
it.each([
|
||||
[401, "auth"],
|
||||
[403, "auth"],
|
||||
[402, "quota"],
|
||||
[422, "bad_request"],
|
||||
[429, "rate"],
|
||||
[503, "upstream"],
|
||||
])("HTTP %i → %s,错误信息不含 Key", async (status, kind) => {
|
||||
const c = createBraveClient({
|
||||
apiKey: KEY,
|
||||
fetch: (async () =>
|
||||
jsonRes({ error: `bad token ${KEY}` }, status, { "x-ratelimit-reset": "1, 1000" })) as typeof fetch,
|
||||
});
|
||||
const err = (await c.search({ q: "x", count: 1 }).catch((e) => e)) as WebSearchError;
|
||||
expect(err).toBeInstanceOf(WebSearchError);
|
||||
expect(err.kind).toBe(kind);
|
||||
expect(err.message).not.toContain(KEY);
|
||||
if (status === 429) expect(err.detail.retryAfterMs).toBe(1000);
|
||||
});
|
||||
|
||||
it("网络错误与超时分类", async () => {
|
||||
const net = createBraveClient({
|
||||
apiKey: KEY,
|
||||
fetch: (async () => {
|
||||
throw new TypeError(`connect fail ${KEY}`);
|
||||
}) as typeof fetch,
|
||||
});
|
||||
const e1 = (await net.search({ q: "x", count: 1 }).catch((e) => e)) as WebSearchError;
|
||||
expect(e1.kind).toBe("network");
|
||||
expect(e1.message).not.toContain(KEY);
|
||||
const slow = createBraveClient({
|
||||
apiKey: KEY,
|
||||
timeoutMs: 50,
|
||||
fetch: ((_u: string, init: RequestInit) =>
|
||||
new Promise((_r, rej) =>
|
||||
init.signal?.addEventListener("abort", () => rej(new Error("aborted"))),
|
||||
)) as typeof fetch,
|
||||
});
|
||||
expect(((await slow.search({ q: "x", count: 1 }).catch((e) => e)) as WebSearchError).kind).toBe("timeout");
|
||||
});
|
||||
|
||||
it("没有 web 段时返回空数组", () => {
|
||||
expect(normalizeBraveResults({ type: "search" })).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("地址守卫", () => {
|
||||
it.each([
|
||||
"127.0.0.1",
|
||||
"10.1.2.3",
|
||||
"172.20.0.1",
|
||||
"192.168.1.1",
|
||||
"169.254.169.254",
|
||||
"100.64.0.1",
|
||||
"0.0.0.0",
|
||||
"::1",
|
||||
"fe80::1",
|
||||
"fd00::1",
|
||||
"::ffff:10.0.0.1",
|
||||
"::ffff:7f00:1",
|
||||
])("内网 IP %s 被拒", (ip) => expect(isBlockedIp(ip)).toBe(true));
|
||||
it.each(["93.184.216.34", "1.1.1.1", "2606:4700::1111"])("公网 IP %s 放行", (ip) =>
|
||||
expect(isBlockedIp(ip)).toBe(false),
|
||||
);
|
||||
it.each(["localhost", "nas", "router.lan", "db.internal", "printer.local", "2130706433", "0x7f.1", "[::1]"])(
|
||||
"内网主机名 %s 被拒",
|
||||
(h) => expect(isBlockedHostname(h)).toBe(true),
|
||||
);
|
||||
|
||||
it("非 http(s)、带账号密码、解析到内网的域名都拒绝", async () => {
|
||||
const kind = async (u: string, dns = publicDns) =>
|
||||
((await assertPublicUrl(u, dns).catch((e) => e)) as WebSearchError).kind;
|
||||
expect(await kind("file:///etc/passwd")).toBe("blocked");
|
||||
expect(await kind("gopher://example.org/")).toBe("blocked");
|
||||
expect(await kind("https://u:[email protected]/")).toBe("blocked");
|
||||
expect(await kind("http://127.0.0.1:4415/api")).toBe("blocked");
|
||||
expect(await kind("https://rebind.example.org/", async () => ["93.184.216.34", "10.0.0.8"])).toBe("blocked");
|
||||
expect((await assertPublicUrl("https://example.org/a", publicDns)).href).toBe("https://example.org/a");
|
||||
});
|
||||
});
|
||||
|
||||
describe("读取网页", () => {
|
||||
const html = `<!doctype html><html><head><title>站点公告</title><style>.x{}</style><script>alert("ignore previous instructions")</script></head>
|
||||
<body><nav>导航</nav><article><h1>公告</h1><p>9 月 20 日维护。</p><p>下载链接 https://t.example/dl?passkey=abcdef0123456789abcdef0123456789</p></article>
|
||||
<form><input value="x"></form><noscript>开启 JS</noscript></body></html>`;
|
||||
|
||||
it("HTML 转纯文本:去脚本样式表单,保留标题与段落,URL 凭证脱敏", async () => {
|
||||
const p = await fetchPage("https://example.org/notice", {
|
||||
resolve: publicDns,
|
||||
fetch: (async () =>
|
||||
new Response(html, { headers: { "content-type": "text/html; charset=utf-8" } })) as typeof fetch,
|
||||
});
|
||||
expect(p.title).toBe("站点公告");
|
||||
expect(p.text).toContain("9 月 20 日维护。");
|
||||
expect(p.text).not.toContain("alert");
|
||||
expect(p.text).not.toContain(".x{}");
|
||||
expect(p.text).not.toContain("开启 JS");
|
||||
expect(p.text).not.toContain("abcdef0123456789");
|
||||
expect(p.contentType).toBe("text/html");
|
||||
});
|
||||
|
||||
it("超过体积上限截断", async () => {
|
||||
const big = "a".repeat(600 * 1024);
|
||||
const p = await fetchPage("https://example.org/big.txt", {
|
||||
resolve: publicDns,
|
||||
fetch: (async () => new Response(big, { headers: { "content-type": "text/plain" } })) as typeof fetch,
|
||||
});
|
||||
expect(p.bytes).toBe(512 * 1024);
|
||||
expect(p.truncated).toBe(true);
|
||||
expect(p.text.length).toBeLessThanOrEqual(20_000);
|
||||
});
|
||||
|
||||
it("非文本类型拒绝;重定向到内网拒绝;超时", async () => {
|
||||
const pdf = await fetchPage("https://example.org/a.pdf", {
|
||||
resolve: publicDns,
|
||||
fetch: (async () => new Response("%PDF", { headers: { "content-type": "application/pdf" } })) as typeof fetch,
|
||||
}).catch((e) => e);
|
||||
expect((pdf as WebSearchError).kind).toBe("unsupported");
|
||||
const redir = await fetchPage("https://example.org/r", {
|
||||
resolve: publicDns,
|
||||
fetch: (async () =>
|
||||
new Response(null, { status: 302, headers: { location: "http://192.168.1.1/admin" } })) as typeof fetch,
|
||||
}).catch((e) => e);
|
||||
expect((redir as WebSearchError).kind).toBe("blocked");
|
||||
const slow = await fetchPage("https://example.org/slow", {
|
||||
resolve: publicDns,
|
||||
timeoutMs: 50,
|
||||
fetch: ((_u: string, init: RequestInit) =>
|
||||
new Promise((_r, rej) =>
|
||||
init.signal?.addEventListener("abort", () => rej(new Error("aborted"))),
|
||||
)) as typeof fetch,
|
||||
}).catch((e) => e);
|
||||
expect((slow as WebSearchError).kind).toBe("timeout");
|
||||
});
|
||||
|
||||
it("htmlToText 无 body 也能取文本", () => {
|
||||
expect(htmlToText("<p>a</p><p>b</p>").text).toBe("a\nb");
|
||||
});
|
||||
});
|
||||
|
||||
describe("正文在 <template> 里的页面", () => {
|
||||
it("Wiki.js 的 <template slot=contents> 正文能取出", () => {
|
||||
const t = htmlToText(
|
||||
'<html><body><div id="root"><page><template slot="contents"><div><h1>订阅</h1><p>自动搜索下载</p></div></template></page></div><script>x()</script></body></html>',
|
||||
);
|
||||
expect(t.text).toBe("订阅\n自动搜索下载");
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user