rootandClaude Opus 5.5
d53df25fc8
- adapters/websearch: Brave client (X-Subscription-Token, error classes), result normalization with redaction/truncation, SSRF guard, HTML→text fetch (10s, 512KB, text/html|text/plain only) - modules/websearch: 10-min PG cache (region websearch), cross-process per-minute limit, web.fetch allowlist = URLs from the caller's latest web.search; POST /websearch/test - settings.agent.webSearch (key write-only → secret_ref); settings card under Agent; web.* offered to the model only when enabled, plus a system prompt rule; outputs marked untrusted - capability count 27 → 29; e2e E13; runbook docs/runbook/websearch.md Co-Authored-By: Claude Opus 5.5 <[email protected]>
56 lines
2.4 KiB
TypeScript
56 lines
2.4 KiB
TypeScript
import { redactString } from "@umt/contracts";
|
||
import { load } from "cheerio";
|
||
|
||
// 网页文本工具:HTML → 纯文本(去脚本、样式、表单等非正文元素)、实体解码、截断与脱敏。
|
||
|
||
const NAMED: Record<string, string> = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " " };
|
||
|
||
/** 去掉摘要里的内联标签(Brave 用 <strong> 标关键字)并解码实体 */
|
||
export function stripInlineHtml(s: string): string {
|
||
return s
|
||
.replace(/<[^>]*>/g, "")
|
||
.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (m, e: string) => {
|
||
if (e[0] === "#") {
|
||
const code = e[1] === "x" || e[1] === "X" ? Number.parseInt(e.slice(2), 16) : Number(e.slice(1));
|
||
return Number.isFinite(code) && code > 0 && code < 0x110000 ? String.fromCodePoint(code) : m;
|
||
}
|
||
return NAMED[e.toLowerCase()] ?? m;
|
||
})
|
||
.replace(/\s+/g, " ")
|
||
.trim();
|
||
}
|
||
|
||
/** 按字符截断,末尾加 … */
|
||
export function truncate(s: string, max: number): { text: string; truncated: boolean } {
|
||
if (s.length <= max) return { text: s, truncated: false };
|
||
return { text: `${s.slice(0, Math.max(0, max - 1))}…`, truncated: true };
|
||
}
|
||
|
||
/** 交给模型前的清理:脱敏(URL 参数里的 passkey / token 等)+ 截断 */
|
||
export function sanitizeForModel(s: string, max: number): { text: string; truncated: boolean } {
|
||
return truncate(redactString(s), max);
|
||
}
|
||
|
||
const DROP = "script,style,noscript,svg,canvas,iframe,object,embed,form,input,select,button,link,meta";
|
||
const BLOCK =
|
||
"p,div,section,article,main,header,footer,aside,nav,li,ul,ol,table,tr,h1,h2,h3,h4,h5,h6,pre,blockquote,dd,dt,br,hr";
|
||
|
||
/** HTML → { title, text }:去脚本样式等,块级元素换行,合并空白 */
|
||
export function htmlToText(html: string): { title: string | null; text: string } {
|
||
// <template> 的内容不在 DOM 文本里,但 Wiki.js 等把正文放在 <template slot="contents">:先改成普通元素
|
||
const $ = load(html.replace(/<template\b/gi, "<div").replace(/<\/template>/gi, "</div>"));
|
||
const title = $("title").first().text().replace(/\s+/g, " ").trim() || null;
|
||
$(DROP).remove();
|
||
$("head").remove();
|
||
$(BLOCK).each((_i, el) => {
|
||
$(el).prepend("\n").append("\n");
|
||
});
|
||
const body = $("body").length ? $("body").text() : $.root().text();
|
||
const text = body
|
||
.split("\n")
|
||
.map((l) => l.replace(/[ \t\f\v ]+/g, " ").trim())
|
||
.filter((l) => l.length > 0)
|
||
.join("\n");
|
||
return { title, text };
|
||
}
|