- adapters/websearch: Brave client (X-Subscription-Token, error classes), result normalization with redaction/truncation, SSRF guard, HTML→text fetch (10s, 512KB, text/html|text/plain only) - modules/websearch: 10-min PG cache (region websearch), cross-process per-minute limit, web.fetch allowlist = URLs from the caller's latest web.search; POST /websearch/test - settings.agent.webSearch (key write-only → secret_ref); settings card under Agent; web.* offered to the model only when enabled, plus a system prompt rule; outputs marked untrusted - capability count 27 → 29; e2e E13; runbook docs/runbook/websearch.md Co-Authored-By: Claude Opus 5.5 <[email protected]>
147 lines
5.7 KiB
TypeScript
147 lines
5.7 KiB
TypeScript
import { lookup } from "node:dns/promises";
|
|
import { isIP } from "node:net";
|
|
import { WebSearchError } from "./errors.ts";
|
|
|
|
// web.fetch 的地址守卫:只允许 http(s)、默认端口以外也可,但主机不能是内网 / 回环 / 链路本地 / 保留地址。
|
|
// 主机名先看后缀(localhost、.local、.internal、单标签名),再解析 DNS,任一解析结果落在内网段即拒绝。
|
|
// 仍有 DNS 重绑定的窗口(检查与连接之间重新解析),见 docs/runbook/websearch.md「已知限制」。
|
|
|
|
/** 解析主机名 → IP 列表(测试注入) */
|
|
export type HostResolver = (host: string) => Promise<string[]>;
|
|
|
|
export const defaultResolver: HostResolver = async (host) => {
|
|
const r = await lookup(host, { all: true, verbatim: true });
|
|
return r.map((x) => x.address);
|
|
};
|
|
|
|
const BLOCKED_SUFFIX = [".localhost", ".local", ".internal", ".lan", ".home.arpa", ".intranet", ".corp"];
|
|
|
|
function v4ToInt(ip: string): number | null {
|
|
const parts = ip.split(".");
|
|
if (parts.length !== 4) return null;
|
|
let n = 0;
|
|
for (const p of parts) {
|
|
if (!/^\d{1,3}$/.test(p)) return null;
|
|
const v = Number(p);
|
|
if (v > 255) return null;
|
|
n = n * 256 + v;
|
|
}
|
|
return n;
|
|
}
|
|
|
|
const V4_BLOCKS: [string, number][] = [
|
|
["0.0.0.0", 8],
|
|
["10.0.0.0", 8],
|
|
["100.64.0.0", 10],
|
|
["127.0.0.0", 8],
|
|
["169.254.0.0", 16],
|
|
["172.16.0.0", 12],
|
|
["192.0.0.0", 24],
|
|
["192.0.2.0", 24],
|
|
["192.168.0.0", 16],
|
|
["198.18.0.0", 15],
|
|
["198.51.100.0", 24],
|
|
["203.0.113.0", 24],
|
|
["224.0.0.0", 4],
|
|
["240.0.0.0", 4],
|
|
];
|
|
|
|
function v4Blocked(ip: string): boolean {
|
|
const n = v4ToInt(ip);
|
|
if (n === null) return true;
|
|
return V4_BLOCKS.some(([base, bits]) => {
|
|
const b = v4ToInt(base)!;
|
|
const size = 2 ** (32 - bits);
|
|
return n >= b && n < b + size;
|
|
});
|
|
}
|
|
|
|
/** 把 IPv6 展开成 8 组 16 位整数;非法返回 null */
|
|
function v6Groups(ip: string): number[] | null {
|
|
let s = ip.toLowerCase();
|
|
const pct = s.indexOf("%");
|
|
if (pct >= 0) s = s.slice(0, pct);
|
|
// 末尾是点分 IPv4(::ffff:1.2.3.4)
|
|
const m = s.match(/^(.*:)(\d+\.\d+\.\d+\.\d+)$/);
|
|
if (m) {
|
|
const n = v4ToInt(m[2]!);
|
|
if (n === null) return null;
|
|
s = `${m[1]}${(n >>> 16).toString(16)}:${(n & 0xffff).toString(16)}`;
|
|
}
|
|
const halves = s.split("::");
|
|
if (halves.length > 2) return null;
|
|
const head = halves[0] ? halves[0].split(":") : [];
|
|
const tail = halves.length === 2 && halves[1] ? halves[1].split(":") : [];
|
|
const fill = halves.length === 2 ? 8 - head.length - tail.length : 0;
|
|
if (fill < 0) return null;
|
|
const groups = [...head, ...Array(fill).fill("0"), ...tail];
|
|
if (groups.length !== 8) return null;
|
|
const out = groups.map((g) => (/^[0-9a-f]{1,4}$/.test(g) ? Number.parseInt(g, 16) : Number.NaN));
|
|
return out.some((x) => Number.isNaN(x)) ? null : out;
|
|
}
|
|
|
|
function v6Blocked(ip: string): boolean {
|
|
const g = v6Groups(ip);
|
|
if (!g) return true;
|
|
const allZero = (from: number, to: number) => g.slice(from, to).every((x) => x === 0);
|
|
if (allZero(0, 8)) return true; // ::
|
|
if (allZero(0, 7) && g[7] === 1) return true; // ::1
|
|
// IPv4 映射 / 兼容 / NAT64:按内嵌的 IPv4 判断
|
|
const embedded = `${g[6]! >> 8}.${g[6]! & 255}.${g[7]! >> 8}.${g[7]! & 255}`;
|
|
if (allZero(0, 5) && (g[5] === 0xffff || g[5] === 0)) return v4Blocked(embedded);
|
|
if (g[0] === 0x64 && g[1] === 0xff9b && allZero(2, 6)) return v4Blocked(embedded);
|
|
const first = g[0]!;
|
|
if ((first & 0xfe00) === 0xfc00) return true; // fc00::/7 唯一本地
|
|
if ((first & 0xffc0) === 0xfe80) return true; // fe80::/10 链路本地
|
|
if ((first & 0xffc0) === 0xfec0) return true; // fec0::/10 站点本地(已弃用)
|
|
if ((first & 0xff00) === 0xff00) return true; // 组播
|
|
if (first === 0x2001 && g[1] === 0x0db8) return true; // 文档
|
|
return false;
|
|
}
|
|
|
|
/** IP 是否落在内网 / 回环 / 链路本地 / 保留段 */
|
|
export function isBlockedIp(ip: string): boolean {
|
|
const v = isIP(ip);
|
|
if (v === 4) return v4Blocked(ip);
|
|
if (v === 6) return v6Blocked(ip);
|
|
return true;
|
|
}
|
|
|
|
/** 主机名本身(不解析)是否就是内网名 */
|
|
export function isBlockedHostname(host: string): boolean {
|
|
const h = host.toLowerCase().replace(/\.$/, "");
|
|
if (!h) return true;
|
|
if (h === "localhost" || BLOCKED_SUFFIX.some((s) => h.endsWith(s))) return true;
|
|
const bare = h.startsWith("[") && h.endsWith("]") ? h.slice(1, -1) : h;
|
|
if (isIP(bare)) return isBlockedIp(bare);
|
|
// 单标签名(intranet、nas…)只会解析到本地网络
|
|
if (!h.includes(".")) return true;
|
|
// 十进制 / 十六进制写法的 IPv4(http://2130706433/、0x7f.1)
|
|
if (/^(0x[0-9a-f]+|\d+)(\.(0x[0-9a-f]+|\d+))*$/i.test(h)) return true;
|
|
return false;
|
|
}
|
|
|
|
/** 校验 URL:只允许 http(s);主机名与所有解析结果都不能是内网地址。返回规范化后的 URL */
|
|
export async function assertPublicUrl(raw: string, resolve: HostResolver = defaultResolver): Promise<URL> {
|
|
let u: URL;
|
|
try {
|
|
u = new URL(raw);
|
|
} catch {
|
|
throw new WebSearchError("blocked", "不是合法的网址");
|
|
}
|
|
if (u.protocol !== "http:" && u.protocol !== "https:") throw new WebSearchError("blocked", "只能读取 http(s) 网址");
|
|
if (u.username || u.password) throw new WebSearchError("blocked", "网址里不能带账号密码");
|
|
if (isBlockedHostname(u.hostname)) throw new WebSearchError("blocked", "不能读取内网或本机地址");
|
|
const bare = u.hostname.startsWith("[") ? u.hostname.slice(1, -1) : u.hostname;
|
|
if (isIP(bare)) return u;
|
|
let ips: string[];
|
|
try {
|
|
ips = await resolve(u.hostname);
|
|
} catch {
|
|
throw new WebSearchError("network", `解析不到主机 ${u.hostname}`);
|
|
}
|
|
if (ips.length === 0) throw new WebSearchError("network", `解析不到主机 ${u.hostname}`);
|
|
if (ips.some(isBlockedIp)) throw new WebSearchError("blocked", "不能读取内网或本机地址");
|
|
return u;
|
|
}
|