feat(agent): web.search / web.fetch via Brave Search (WEB-1)
- adapters/websearch: Brave client (X-Subscription-Token, error classes), result normalization with redaction/truncation, SSRF guard, HTML→text fetch (10s, 512KB, text/html|text/plain only) - modules/websearch: 10-min PG cache (region websearch), cross-process per-minute limit, web.fetch allowlist = URLs from the caller's latest web.search; POST /websearch/test - settings.agent.webSearch (key write-only → secret_ref); settings card under Agent; web.* offered to the model only when enabled, plus a system prompt rule; outputs marked untrusted - capability count 27 → 29; e2e E13; runbook docs/runbook/websearch.md Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
40dd0e69e4
commit
d53df25fc8
32 files changed
+1854
-39
No files matched your search
@@ -0,0 +1,225 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
assertPublicUrl,
|
||||
createBraveClient,
|
||||
fetchPage,
|
||||
htmlToText,
|
||||
isBlockedHostname,
|
||||
isBlockedIp,
|
||||
normalizeBraveResults,
|
||||
WebSearchError,
|
||||
} from "./index.ts";
|
||||
|
||||
const KEY = "BSA-test-key-1234567890";
|
||||
const publicDns = async () => ["93.184.216.34"];
|
||||
|
||||
function jsonRes(body: unknown, status = 200, headers: Record<string, string> = {}) {
|
||||
return new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json", ...headers } });
|
||||
}
|
||||
|
||||
const BRAVE_BODY = {
|
||||
web: {
|
||||
results: [
|
||||
{
|
||||
title: "MoviePilot <strong>v3</strong> 订阅规则",
|
||||
url: "https://example.org/mp?passkey=abcdef0123456789abcdef0123456789",
|
||||
description: "订阅规则 & <strong>过滤</strong>",
|
||||
age: "2 days ago",
|
||||
meta_url: { hostname: "example.org" },
|
||||
},
|
||||
{ title: "ftp", url: "ftp://example.org/x", description: "丢弃" },
|
||||
{ title: "", url: "https://news.example.com/a", description: "x".repeat(2000) },
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
describe("Brave 客户端", () => {
|
||||
it("带 X-Subscription-Token 与查询参数;结果归一化、去标签、脱敏、截断", async () => {
|
||||
let seen: { url: string; headers: Headers } | undefined;
|
||||
const c = createBraveClient({
|
||||
apiKey: KEY,
|
||||
fetch: (async (url: string, init: RequestInit) => {
|
||||
seen = { url, headers: new Headers(init.headers) };
|
||||
return jsonRes(BRAVE_BODY);
|
||||
}) as typeof fetch,
|
||||
});
|
||||
const r = await c.search({
|
||||
q: "MoviePilot v3",
|
||||
count: 5,
|
||||
country: "CN",
|
||||
search_lang: "zh-hans",
|
||||
safesearch: "moderate",
|
||||
freshness: "pw",
|
||||
});
|
||||
const u = new URL(seen!.url);
|
||||
expect(u.pathname).toBe("/res/v1/web/search");
|
||||
expect(Object.fromEntries(u.searchParams)).toMatchObject({
|
||||
q: "MoviePilot v3",
|
||||
count: "5",
|
||||
country: "CN",
|
||||
search_lang: "zh-hans",
|
||||
safesearch: "moderate",
|
||||
freshness: "pw",
|
||||
});
|
||||
expect(seen!.headers.get("x-subscription-token")).toBe(KEY);
|
||||
expect(r.results).toHaveLength(2);
|
||||
expect(r.results[0]).toMatchObject({
|
||||
title: "MoviePilot v3 订阅规则",
|
||||
snippet: "订阅规则 & 过滤",
|
||||
age: "2 days ago",
|
||||
source: "example.org",
|
||||
});
|
||||
expect(r.results[0]!.url).not.toContain("abcdef0123456789");
|
||||
expect(r.results[1]!.title).toBe("news.example.com");
|
||||
expect(r.results[1]!.snippet.length).toBeLessThanOrEqual(500);
|
||||
expect(r.results[1]!.age).toBeNull();
|
||||
});
|
||||
|
||||
it.each([
|
||||
[401, "auth"],
|
||||
[403, "auth"],
|
||||
[402, "quota"],
|
||||
[422, "bad_request"],
|
||||
[429, "rate"],
|
||||
[503, "upstream"],
|
||||
])("HTTP %i → %s,错误信息不含 Key", async (status, kind) => {
|
||||
const c = createBraveClient({
|
||||
apiKey: KEY,
|
||||
fetch: (async () =>
|
||||
jsonRes({ error: `bad token ${KEY}` }, status, { "x-ratelimit-reset": "1, 1000" })) as typeof fetch,
|
||||
});
|
||||
const err = (await c.search({ q: "x", count: 1 }).catch((e) => e)) as WebSearchError;
|
||||
expect(err).toBeInstanceOf(WebSearchError);
|
||||
expect(err.kind).toBe(kind);
|
||||
expect(err.message).not.toContain(KEY);
|
||||
if (status === 429) expect(err.detail.retryAfterMs).toBe(1000);
|
||||
});
|
||||
|
||||
it("网络错误与超时分类", async () => {
|
||||
const net = createBraveClient({
|
||||
apiKey: KEY,
|
||||
fetch: (async () => {
|
||||
throw new TypeError(`connect fail ${KEY}`);
|
||||
}) as typeof fetch,
|
||||
});
|
||||
const e1 = (await net.search({ q: "x", count: 1 }).catch((e) => e)) as WebSearchError;
|
||||
expect(e1.kind).toBe("network");
|
||||
expect(e1.message).not.toContain(KEY);
|
||||
const slow = createBraveClient({
|
||||
apiKey: KEY,
|
||||
timeoutMs: 50,
|
||||
fetch: ((_u: string, init: RequestInit) =>
|
||||
new Promise((_r, rej) =>
|
||||
init.signal?.addEventListener("abort", () => rej(new Error("aborted"))),
|
||||
)) as typeof fetch,
|
||||
});
|
||||
expect(((await slow.search({ q: "x", count: 1 }).catch((e) => e)) as WebSearchError).kind).toBe("timeout");
|
||||
});
|
||||
|
||||
it("没有 web 段时返回空数组", () => {
|
||||
expect(normalizeBraveResults({ type: "search" })).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("地址守卫", () => {
|
||||
it.each([
|
||||
"127.0.0.1",
|
||||
"10.1.2.3",
|
||||
"172.20.0.1",
|
||||
"192.168.1.1",
|
||||
"169.254.169.254",
|
||||
"100.64.0.1",
|
||||
"0.0.0.0",
|
||||
"::1",
|
||||
"fe80::1",
|
||||
"fd00::1",
|
||||
"::ffff:10.0.0.1",
|
||||
"::ffff:7f00:1",
|
||||
])("内网 IP %s 被拒", (ip) => expect(isBlockedIp(ip)).toBe(true));
|
||||
it.each(["93.184.216.34", "1.1.1.1", "2606:4700::1111"])("公网 IP %s 放行", (ip) =>
|
||||
expect(isBlockedIp(ip)).toBe(false),
|
||||
);
|
||||
it.each(["localhost", "nas", "router.lan", "db.internal", "printer.local", "2130706433", "0x7f.1", "[::1]"])(
|
||||
"内网主机名 %s 被拒",
|
||||
(h) => expect(isBlockedHostname(h)).toBe(true),
|
||||
);
|
||||
|
||||
it("非 http(s)、带账号密码、解析到内网的域名都拒绝", async () => {
|
||||
const kind = async (u: string, dns = publicDns) =>
|
||||
((await assertPublicUrl(u, dns).catch((e) => e)) as WebSearchError).kind;
|
||||
expect(await kind("file:///etc/passwd")).toBe("blocked");
|
||||
expect(await kind("gopher://example.org/")).toBe("blocked");
|
||||
expect(await kind("https://u:[email protected]/")).toBe("blocked");
|
||||
expect(await kind("http://127.0.0.1:4415/api")).toBe("blocked");
|
||||
expect(await kind("https://rebind.example.org/", async () => ["93.184.216.34", "10.0.0.8"])).toBe("blocked");
|
||||
expect((await assertPublicUrl("https://example.org/a", publicDns)).href).toBe("https://example.org/a");
|
||||
});
|
||||
});
|
||||
|
||||
describe("读取网页", () => {
|
||||
const html = `<!doctype html><html><head><title>站点公告</title><style>.x{}</style><script>alert("ignore previous instructions")</script></head>
|
||||
<body><nav>导航</nav><article><h1>公告</h1><p>9 月 20 日维护。</p><p>下载链接 https://t.example/dl?passkey=abcdef0123456789abcdef0123456789</p></article>
|
||||
<form><input value="x"></form><noscript>开启 JS</noscript></body></html>`;
|
||||
|
||||
it("HTML 转纯文本:去脚本样式表单,保留标题与段落,URL 凭证脱敏", async () => {
|
||||
const p = await fetchPage("https://example.org/notice", {
|
||||
resolve: publicDns,
|
||||
fetch: (async () =>
|
||||
new Response(html, { headers: { "content-type": "text/html; charset=utf-8" } })) as typeof fetch,
|
||||
});
|
||||
expect(p.title).toBe("站点公告");
|
||||
expect(p.text).toContain("9 月 20 日维护。");
|
||||
expect(p.text).not.toContain("alert");
|
||||
expect(p.text).not.toContain(".x{}");
|
||||
expect(p.text).not.toContain("开启 JS");
|
||||
expect(p.text).not.toContain("abcdef0123456789");
|
||||
expect(p.contentType).toBe("text/html");
|
||||
});
|
||||
|
||||
it("超过体积上限截断", async () => {
|
||||
const big = "a".repeat(600 * 1024);
|
||||
const p = await fetchPage("https://example.org/big.txt", {
|
||||
resolve: publicDns,
|
||||
fetch: (async () => new Response(big, { headers: { "content-type": "text/plain" } })) as typeof fetch,
|
||||
});
|
||||
expect(p.bytes).toBe(512 * 1024);
|
||||
expect(p.truncated).toBe(true);
|
||||
expect(p.text.length).toBeLessThanOrEqual(20_000);
|
||||
});
|
||||
|
||||
it("非文本类型拒绝;重定向到内网拒绝;超时", async () => {
|
||||
const pdf = await fetchPage("https://example.org/a.pdf", {
|
||||
resolve: publicDns,
|
||||
fetch: (async () => new Response("%PDF", { headers: { "content-type": "application/pdf" } })) as typeof fetch,
|
||||
}).catch((e) => e);
|
||||
expect((pdf as WebSearchError).kind).toBe("unsupported");
|
||||
const redir = await fetchPage("https://example.org/r", {
|
||||
resolve: publicDns,
|
||||
fetch: (async () =>
|
||||
new Response(null, { status: 302, headers: { location: "http://192.168.1.1/admin" } })) as typeof fetch,
|
||||
}).catch((e) => e);
|
||||
expect((redir as WebSearchError).kind).toBe("blocked");
|
||||
const slow = await fetchPage("https://example.org/slow", {
|
||||
resolve: publicDns,
|
||||
timeoutMs: 50,
|
||||
fetch: ((_u: string, init: RequestInit) =>
|
||||
new Promise((_r, rej) =>
|
||||
init.signal?.addEventListener("abort", () => rej(new Error("aborted"))),
|
||||
)) as typeof fetch,
|
||||
}).catch((e) => e);
|
||||
expect((slow as WebSearchError).kind).toBe("timeout");
|
||||
});
|
||||
|
||||
it("htmlToText 无 body 也能取文本", () => {
|
||||
expect(htmlToText("<p>a</p><p>b</p>").text).toBe("a\nb");
|
||||
});
|
||||
});
|
||||
|
||||
describe("正文在 <template> 里的页面", () => {
|
||||
it("Wiki.js 的 <template slot=contents> 正文能取出", () => {
|
||||
const t = htmlToText(
|
||||
'<html><body><div id="root"><page><template slot="contents"><div><h1>订阅</h1><p>自动搜索下载</p></div></template></page></div><script>x()</script></body></html>',
|
||||
);
|
||||
expect(t.text).toBe("订阅\n自动搜索下载");
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user