feat(agent): web.search / web.fetch via Brave Search (WEB-1)
- adapters/websearch: Brave client (X-Subscription-Token, error classes), result normalization with redaction/truncation, SSRF guard, HTML→text fetch (10s, 512KB, text/html|text/plain only) - modules/websearch: 10-min PG cache (region websearch), cross-process per-minute limit, web.fetch allowlist = URLs from the caller's latest web.search; POST /websearch/test - settings.agent.webSearch (key write-only → secret_ref); settings card under Agent; web.* offered to the model only when enabled, plus a system prompt rule; outputs marked untrusted - capability count 27 → 29; e2e E13; runbook docs/runbook/websearch.md Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
40dd0e69e4
commit
d53df25fc8
32 files changed
+1854
-39
No files matched your search
@@ -0,0 +1,159 @@
|
||||
import { redactUrl } from "@umt/contracts";
|
||||
import { WebSearchError } from "./errors.ts";
|
||||
import { assertPublicUrl, defaultResolver, type HostResolver } from "./net-guard.ts";
|
||||
import { htmlToText, sanitizeForModel } from "./text.ts";
|
||||
|
||||
/**
|
||||
* 读取一个公开网页的纯文本(web.fetch 的网络部分)。白名单(只能读最近一次搜索结果)由服务端检查;
|
||||
* 这里负责:只允许 http(s)、每一跳(含重定向)都不能落到内网、超时、体积上限、只收 text/html | text/plain、HTML 转纯文本。
|
||||
*/
|
||||
|
||||
export interface FetchPageOptions {
|
||||
/** 缺省 10s(含重定向与读正文) */
|
||||
timeoutMs?: number;
|
||||
/** 最多读多少字节,缺省 512 KB;超出部分丢弃并标 truncated */
|
||||
maxBytes?: number;
|
||||
/** 交给模型的正文最多多少字符,缺省 20000 */
|
||||
maxChars?: number;
|
||||
/** 最多跟随几次重定向,缺省 3 */
|
||||
maxRedirects?: number;
|
||||
userAgent?: string;
|
||||
fetch?: typeof fetch;
|
||||
resolve?: HostResolver;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export interface FetchedPage {
|
||||
url: string;
|
||||
finalUrl: string;
|
||||
status: number;
|
||||
contentType: string;
|
||||
title: string | null;
|
||||
text: string;
|
||||
truncated: boolean;
|
||||
bytes: number;
|
||||
}
|
||||
|
||||
export const PAGE_MAX_BYTES = 512 * 1024;
|
||||
export const PAGE_MAX_CHARS = 20_000;
|
||||
const DEFAULT_UA = "Mozilla/5.0 (compatible; umt-websearch/1.0)";
|
||||
|
||||
async function readCapped(res: Response, max: number): Promise<{ bytes: Uint8Array; truncated: boolean }> {
|
||||
if (!res.body) return { bytes: new Uint8Array(), truncated: false };
|
||||
const reader = res.body.getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
let size = 0;
|
||||
let truncated = false;
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
if (size + value.byteLength > max) {
|
||||
chunks.push(value.subarray(0, max - size));
|
||||
size = max;
|
||||
truncated = true;
|
||||
await reader.cancel().catch(() => {});
|
||||
break;
|
||||
}
|
||||
chunks.push(value);
|
||||
size += value.byteLength;
|
||||
}
|
||||
const out = new Uint8Array(size);
|
||||
let off = 0;
|
||||
for (const c of chunks) {
|
||||
out.set(c, off);
|
||||
off += c.byteLength;
|
||||
}
|
||||
return { bytes: out, truncated };
|
||||
}
|
||||
|
||||
function charsetOf(contentType: string): string {
|
||||
const m = contentType.match(/charset\s*=\s*"?([\w-]+)"?/i);
|
||||
return m?.[1]?.toLowerCase() ?? "utf-8";
|
||||
}
|
||||
|
||||
function decode(bytes: Uint8Array, contentType: string): string {
|
||||
let cs = charsetOf(contentType);
|
||||
// HTML 没在响应头声明字符集时看 <meta charset>
|
||||
if (!/charset/i.test(contentType)) {
|
||||
const head = new TextDecoder("latin1").decode(bytes.subarray(0, 2048));
|
||||
const m = head.match(/<meta[^>]+charset\s*=\s*["']?([\w-]+)/i);
|
||||
if (m?.[1]) cs = m[1].toLowerCase();
|
||||
}
|
||||
try {
|
||||
return new TextDecoder(cs, { fatal: false }).decode(bytes);
|
||||
} catch {
|
||||
return new TextDecoder("utf-8", { fatal: false }).decode(bytes);
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchPage(raw: string, opts: FetchPageOptions = {}): Promise<FetchedPage> {
|
||||
const doFetch = opts.fetch ?? fetch;
|
||||
const resolve = opts.resolve ?? defaultResolver;
|
||||
const timeoutMs = opts.timeoutMs ?? 10_000;
|
||||
const maxBytes = opts.maxBytes ?? PAGE_MAX_BYTES;
|
||||
const maxRedirects = opts.maxRedirects ?? 3;
|
||||
const timeout = AbortSignal.timeout(timeoutMs);
|
||||
const signal = opts.signal ? AbortSignal.any([opts.signal, timeout]) : timeout;
|
||||
|
||||
let url = await assertPublicUrl(raw, resolve);
|
||||
let res: Response | null = null;
|
||||
try {
|
||||
for (let hop = 0; ; hop++) {
|
||||
res = await doFetch(url.href, {
|
||||
redirect: "manual",
|
||||
signal,
|
||||
headers: {
|
||||
"user-agent": opts.userAgent ?? DEFAULT_UA,
|
||||
accept: "text/html,text/plain;q=0.9,*/*;q=0.1",
|
||||
},
|
||||
});
|
||||
if (res.status >= 300 && res.status < 400 && res.headers.get("location")) {
|
||||
await res.body?.cancel().catch(() => {});
|
||||
if (hop >= maxRedirects) throw new WebSearchError("blocked", `重定向超过 ${maxRedirects} 次`);
|
||||
// 每一跳都重新检查,防止公网地址 302 到内网
|
||||
url = await assertPublicUrl(new URL(res.headers.get("location")!, url).href, resolve);
|
||||
continue;
|
||||
}
|
||||
break;
|
||||
}
|
||||
} catch (err) {
|
||||
if (err instanceof WebSearchError) throw err;
|
||||
if (timeout.aborted) throw new WebSearchError("timeout", `网页 ${timeoutMs / 1000} 秒内没有响应`);
|
||||
if (opts.signal?.aborted) throw err;
|
||||
throw new WebSearchError("network", `读取网页失败：${redactUrl(String((err as Error).message ?? err))}`);
|
||||
}
|
||||
const r = res!;
|
||||
if (r.status >= 400) {
|
||||
await r.body?.cancel().catch(() => {});
|
||||
throw new WebSearchError(r.status >= 500 ? "upstream" : "bad_request", `网页返回 HTTP ${r.status}`, {
|
||||
status: r.status,
|
||||
});
|
||||
}
|
||||
const contentType = (r.headers.get("content-type") ?? "").toLowerCase();
|
||||
const mime = contentType.split(";")[0]!.trim();
|
||||
const isHtml = mime === "text/html" || mime === "application/xhtml+xml";
|
||||
if (!isHtml && mime !== "text/plain") {
|
||||
await r.body?.cancel().catch(() => {});
|
||||
throw new WebSearchError("unsupported", `只读取 text/html 与 text/plain，这个网页是 ${mime || "未知类型"}`);
|
||||
}
|
||||
let body: { bytes: Uint8Array; truncated: boolean };
|
||||
try {
|
||||
body = await readCapped(r, maxBytes);
|
||||
} catch (err) {
|
||||
if (timeout.aborted) throw new WebSearchError("timeout", `网页 ${timeoutMs / 1000} 秒内没有读完`);
|
||||
throw new WebSearchError("network", `读取网页失败：${redactUrl(String((err as Error).message ?? err))}`);
|
||||
}
|
||||
const decoded = decode(body.bytes, contentType);
|
||||
const page = isHtml ? htmlToText(decoded) : { title: null, text: decoded.replace(/\r\n?/g, "\n").trim() };
|
||||
const clean = sanitizeForModel(page.text, opts.maxChars ?? PAGE_MAX_CHARS);
|
||||
return {
|
||||
url: redactUrl(raw),
|
||||
finalUrl: redactUrl(url.href),
|
||||
status: r.status,
|
||||
contentType: mime,
|
||||
title: page.title ? sanitizeForModel(page.title, 300).text : null,
|
||||
text: clean.text,
|
||||
truncated: body.truncated || clean.truncated,
|
||||
bytes: body.bytes.byteLength,
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user