feat(agent): web.search / web.fetch via Brave Search (WEB-1)
- adapters/websearch: Brave client (X-Subscription-Token, error classes), result normalization with redaction/truncation, SSRF guard, HTML→text fetch (10s, 512KB, text/html|text/plain only) - modules/websearch: 10-min PG cache (region websearch), cross-process per-minute limit, web.fetch allowlist = URLs from the caller's latest web.search; POST /websearch/test - settings.agent.webSearch (key write-only → secret_ref); settings card under Agent; web.* offered to the model only when enabled, plus a system prompt rule; outputs marked untrusted - capability count 27 → 29; e2e E13; runbook docs/runbook/websearch.md Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
40dd0e69e4
commit
d53df25fc8
32 files changed
+1854
-39
No files matched your search
@@ -0,0 +1,44 @@
|
||||
import { WebSearchError } from "@umt/adapters";
|
||||
import { CapabilityError, type CapabilityErrorCode, type CapabilityRegistry } from "@umt/agent";
|
||||
import { capabilitySpec } from "@umt/contracts";
|
||||
import { allowScopeKey, type WebSearchService, WebSearchServiceError } from "./service.ts";
|
||||
|
||||
// web.search / web.fetch(read,low):错误按类别映射成能力错误;输出本身带 trust = untrusted 与不可信提示,
|
||||
// 进入模型上下文前 PiAgentRuntime 还会整体过一遍 redactForModel。
|
||||
|
||||
const CODE: Record<string, CapabilityErrorCode> = {
|
||||
disabled: "precondition",
|
||||
not_allowed: "precondition",
|
||||
blocked: "precondition",
|
||||
unsupported: "precondition",
|
||||
bad_request: "precondition",
|
||||
auth: "precondition",
|
||||
quota: "budget",
|
||||
rate: "budget",
|
||||
upstream: "precondition",
|
||||
network: "precondition",
|
||||
timeout: "precondition",
|
||||
protocol: "precondition",
|
||||
};
|
||||
|
||||
async function guard<T>(fn: () => Promise<T>): Promise<T> {
|
||||
try {
|
||||
return await fn();
|
||||
} catch (e) {
|
||||
if (e instanceof WebSearchError || e instanceof WebSearchServiceError)
|
||||
throw new CapabilityError(CODE[e.kind] ?? "precondition", e.message, { kind: e.kind });
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
export function registerWebSearchCapabilities(registry: CapabilityRegistry, svc: () => WebSearchService): void {
|
||||
registry.register(capabilitySpec("web.search"), (input, c) =>
|
||||
guard(async () => {
|
||||
const { durationMs: _d, ...out } = await svc().search(input, { scopeKey: allowScopeKey(c), signal: c.signal });
|
||||
return out;
|
||||
}),
|
||||
);
|
||||
registry.register(capabilitySpec("web.fetch"), (input, c) =>
|
||||
guard(() => svc().fetch(input.url, { scopeKey: allowScopeKey(c), signal: c.signal })),
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import type { ServerContext, ServerModule } from "../types.ts";
|
||||
import { registerWebSearchCapabilities } from "./capabilities.ts";
|
||||
import { registerWebSearchRoutes } from "./routes.ts";
|
||||
import { type WebSearchModuleOptions, WebSearchService } from "./service.ts";
|
||||
|
||||
export type { WebSearchModuleOptions } from "./service.ts";
|
||||
|
||||
/**
|
||||
* WEB-1 网页搜索(docs/runbook/websearch.md):Brave Web Search 客户端(@umt/adapters websearch)之上的
|
||||
* web.search / web.fetch 能力与 POST /websearch/test。不是任意网络代理:web.fetch 只读最近一次搜索结果里的链接,
|
||||
* 拒绝内网地址与非 http(s);结果与正文标为不可信数据。设置在 settings.agent.webSearch,Key 存 secret。
|
||||
*/
|
||||
export function createWebSearchModule(opts: WebSearchModuleOptions = {}): ServerModule & {
|
||||
service(ctx: ServerContext): WebSearchService;
|
||||
} {
|
||||
const services = new WeakMap<ServerContext, WebSearchService>();
|
||||
const service = (ctx: ServerContext) => {
|
||||
let s = services.get(ctx);
|
||||
if (!s) {
|
||||
s = new WebSearchService(ctx, opts);
|
||||
services.set(ctx, s);
|
||||
}
|
||||
return s;
|
||||
};
|
||||
return {
|
||||
name: "websearch",
|
||||
service,
|
||||
registerRoutes(app, ctx) {
|
||||
registerWebSearchRoutes(app, () => service(ctx));
|
||||
},
|
||||
registerTasks() {},
|
||||
registerCapabilities(registry, ctx) {
|
||||
registerWebSearchCapabilities(registry, () => service(ctx));
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export const webSearchModule = createWebSearchModule();
|
||||
@@ -0,0 +1,29 @@
|
||||
import { createRoute } from "@hono/zod-openapi";
|
||||
import { Api, apiResponse } from "@umt/contracts";
|
||||
import { requireScope } from "../../http/auth.ts";
|
||||
import type { ModuleApp } from "../types.ts";
|
||||
import type { WebSearchService } from "./service.ts";
|
||||
|
||||
// POST /websearch/test:设置页「测试」按钮,用已保存的 Key 查一次;失败不抛 4xx,结果里给 ok=false 与原因
|
||||
|
||||
export function registerWebSearchRoutes(app: ModuleApp, svc: () => WebSearchService) {
|
||||
const def = Api.websearch.test;
|
||||
const r = createRoute({
|
||||
method: def.method,
|
||||
path: def.path,
|
||||
summary: def.summary,
|
||||
tags: ["websearch"],
|
||||
middleware: [requireScope(def.scope)] as never,
|
||||
request: { body: { content: { "application/json": { schema: def.body as never } }, required: true } },
|
||||
responses: {
|
||||
200: { description: def.summary, content: { "application/json": { schema: apiResponse(def.response) } } },
|
||||
},
|
||||
});
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 路由泛型在封装里擦除,入参已由 zod-openapi 校验
|
||||
app.openapi(r, (async (c: any) => {
|
||||
const body = c.req.valid("json") as { query?: string };
|
||||
const data = def.response.parse(await svc().test(body.query));
|
||||
return c.json({ ok: true, data }, 200);
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 同上
|
||||
}) as any);
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
createBraveClient,
|
||||
fetchPage,
|
||||
type HostResolver,
|
||||
WebSearchError,
|
||||
type WebSearchErrorKind,
|
||||
} from "@umt/adapters";
|
||||
import {
|
||||
DEFAULT_WEB_SEARCH,
|
||||
UNTRUSTED_WEB_NOTICE,
|
||||
type WebFetchOutput,
|
||||
type WebSearchFreshness,
|
||||
type WebSearchOutput,
|
||||
type WebSearchResult,
|
||||
type WebSearchSettings,
|
||||
type WebSearchTestResult,
|
||||
} from "@umt/contracts";
|
||||
import { cachedCall, pgCacheFor, pgTokenBucketFor } from "@umt/db";
|
||||
import type { ServerContext } from "../types.ts";
|
||||
|
||||
/**
|
||||
* WEB-1 网页搜索服务:Brave 客户端之上的 Key 读取、每分钟上限(跨进程令牌桶)、10 分钟 PG 缓存(kv_cache region websearch)、
|
||||
* 以及 web.fetch 的白名单——按调用方(Agent 会话 / 其它调用者)记录最近一次 web.search 返回的 URL,只允许读取这些 URL。
|
||||
*/
|
||||
|
||||
export interface WebSearchModuleOptions {
|
||||
/** 搜索与网页读取用的 fetch(测试注入 mock) */
|
||||
fetch?: typeof fetch;
|
||||
/** Brave 基址(测试指向 mock) */
|
||||
braveBaseUrl?: string;
|
||||
/** DNS 解析(测试注入);缺省系统解析 */
|
||||
resolveHost?: HostResolver;
|
||||
}
|
||||
|
||||
export const WEBSEARCH_CACHE_REGION = "websearch";
|
||||
/** 白名单:key = 调用方,value = 最近一次搜索的 URL */
|
||||
export const WEBSEARCH_ALLOW_REGION = "websearch.allow";
|
||||
/** 搜索结果缓存 10 分钟 */
|
||||
export const SEARCH_CACHE_TTL_SEC = 600;
|
||||
const SEARCH_CACHE_MAX = 1000;
|
||||
/** 白名单保留 6 小时(会话时长上限远小于此) */
|
||||
const ALLOW_TTL_SEC = 6 * 3600;
|
||||
/** 取不到令牌时最多等多久 */
|
||||
const RATE_WAIT_MS = 3000;
|
||||
export const TEST_QUERY = "MoviePilot";
|
||||
|
||||
export class WebSearchServiceError extends Error {
|
||||
constructor(
|
||||
readonly kind: WebSearchErrorKind | "disabled" | "not_allowed",
|
||||
message: string,
|
||||
) {
|
||||
super(message);
|
||||
this.name = "WebSearchServiceError";
|
||||
}
|
||||
}
|
||||
|
||||
/** URL 比较用的规范形式(去掉 #片段) */
|
||||
export function canonicalUrl(raw: string): string | null {
|
||||
try {
|
||||
const u = new URL(raw);
|
||||
u.hash = "";
|
||||
return u.href;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** web.fetch 白名单的归属:Agent 会话按 run,其它调用方按 actor */
|
||||
export const allowScopeKey = (c: { agentRunId?: string | undefined; actorId: string }) =>
|
||||
c.agentRunId ? `run:${c.agentRunId}` : `actor:${c.actorId}`;
|
||||
|
||||
export class WebSearchService {
|
||||
constructor(
|
||||
private readonly ctx: ServerContext,
|
||||
private readonly opts: WebSearchModuleOptions = {},
|
||||
) {}
|
||||
|
||||
async settings(): Promise<WebSearchSettings> {
|
||||
const a = await this.ctx.settings.get("agent").catch(() => null);
|
||||
return { ...DEFAULT_WEB_SEARCH, ...(a?.webSearch ?? {}) };
|
||||
}
|
||||
|
||||
/** 设置里开启且存了 Key 才向模型提供 web.* */
|
||||
async enabled(): Promise<boolean> {
|
||||
const s = await this.settings();
|
||||
return s.provider !== "none" && !!s.apiKeySecretRef;
|
||||
}
|
||||
|
||||
private async client(s: WebSearchSettings) {
|
||||
if (s.provider === "none")
|
||||
throw new WebSearchServiceError("disabled", "网页搜索未开启，在设置 → Agent → 网页搜索里开启");
|
||||
if (!s.apiKeySecretRef) throw new WebSearchServiceError("auth", "未保存搜索服务 API Key");
|
||||
const apiKey = await this.ctx.secrets.resolve(s.apiKeySecretRef);
|
||||
return createBraveClient({
|
||||
apiKey,
|
||||
...(this.opts.braveBaseUrl ? { baseUrl: this.opts.braveBaseUrl } : {}),
|
||||
...(this.opts.fetch ? { fetch: this.opts.fetch } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
private async takeToken(s: WebSearchSettings, signal?: AbortSignal) {
|
||||
const rpm = s.requestsPerMinute;
|
||||
const r = await pgTokenBucketFor(this.ctx.db).acquire(`websearch:${s.provider}`, 1, {
|
||||
capacity: rpm,
|
||||
refillPerSec: rpm / 60,
|
||||
waitMs: RATE_WAIT_MS,
|
||||
...(signal ? { signal } : {}),
|
||||
});
|
||||
if (!r.ok) throw new WebSearchServiceError("rate", `网页搜索已达每分钟上限（${rpm} 次），稍后再试`);
|
||||
}
|
||||
|
||||
/**
|
||||
* 搜索:命中缓存直接返回(不计上限);否则扣令牌后查询并缓存 10 分钟。
|
||||
* scopeKey 给了就把结果 URL 记为该调用方的 web.fetch 白名单(替换上一次的)。
|
||||
*/
|
||||
async search(
|
||||
input: { query: string; count?: number | undefined; freshness?: WebSearchFreshness | undefined },
|
||||
o: { scopeKey?: string; fresh?: boolean; signal?: AbortSignal } = {},
|
||||
): Promise<WebSearchOutput & { durationMs: number }> {
|
||||
const s = await this.settings();
|
||||
const client = await this.client(s);
|
||||
const count = Math.min(input.count ?? s.maxResults, s.maxResults);
|
||||
const params = {
|
||||
q: input.query.trim(),
|
||||
count,
|
||||
country: s.country === "ALL" ? null : s.country,
|
||||
search_lang: s.searchLang,
|
||||
safesearch: s.safesearch,
|
||||
...(input.freshness ? { freshness: input.freshness } : {}),
|
||||
};
|
||||
const key = createHash("sha256")
|
||||
.update(
|
||||
JSON.stringify([s.provider, params.q, count, s.country, s.searchLang, s.safesearch, input.freshness ?? null]),
|
||||
)
|
||||
.digest("hex")
|
||||
.slice(0, 40);
|
||||
let hit = true;
|
||||
const started = Date.now();
|
||||
const results = await cachedCall<WebSearchResult[]>(
|
||||
pgCacheFor(this.ctx.db),
|
||||
key,
|
||||
{
|
||||
region: WEBSEARCH_CACHE_REGION,
|
||||
ttl: SEARCH_CACHE_TTL_SEC,
|
||||
maxsize: SEARCH_CACHE_MAX,
|
||||
...(o.fresh ? { fresh: true } : {}),
|
||||
},
|
||||
async () => {
|
||||
hit = false;
|
||||
await this.takeToken(s, o.signal);
|
||||
const r = await client.search({ ...params, ...(o.signal ? { signal: o.signal } : {}) });
|
||||
return r.results;
|
||||
},
|
||||
);
|
||||
if (o.scopeKey) {
|
||||
const urls = results.map((r) => canonicalUrl(r.url)).filter((u): u is string => !!u);
|
||||
await pgCacheFor(this.ctx.db).set(
|
||||
WEBSEARCH_ALLOW_REGION,
|
||||
o.scopeKey,
|
||||
{ urls, query: params.q },
|
||||
{ ttl: ALLOW_TTL_SEC, maxsize: 5000 },
|
||||
);
|
||||
}
|
||||
return {
|
||||
trust: "untrusted",
|
||||
notice: UNTRUSTED_WEB_NOTICE,
|
||||
query: params.q,
|
||||
results,
|
||||
cached: hit,
|
||||
durationMs: Date.now() - started,
|
||||
};
|
||||
}
|
||||
|
||||
/** 读取网页:必须是该调用方最近一次搜索返回过的 URL;网络层再拒绝内网与非 http(s) */
|
||||
async fetch(url: string, o: { scopeKey: string; signal?: AbortSignal }): Promise<WebFetchOutput> {
|
||||
const s = await this.settings();
|
||||
if (s.provider === "none")
|
||||
throw new WebSearchServiceError("disabled", "网页搜索未开启，在设置 → Agent → 网页搜索里开启");
|
||||
const want = canonicalUrl(url.trim());
|
||||
const allow = await pgCacheFor(this.ctx.db).get<{ urls: string[] }>(WEBSEARCH_ALLOW_REGION, o.scopeKey);
|
||||
if (!want || !allow?.urls.includes(want))
|
||||
throw new WebSearchServiceError(
|
||||
"not_allowed",
|
||||
allow ? "只能读取最近一次 web.search 结果里的链接" : "先用 web.search 搜索，再读取结果里的链接",
|
||||
);
|
||||
const page = await fetchPage(want, {
|
||||
...(this.opts.fetch ? { fetch: this.opts.fetch } : {}),
|
||||
...(this.opts.resolveHost ? { resolve: this.opts.resolveHost } : {}),
|
||||
...(o.signal ? { signal: o.signal } : {}),
|
||||
});
|
||||
return { trust: "untrusted", notice: UNTRUSTED_WEB_NOTICE, ...page };
|
||||
}
|
||||
|
||||
/** 设置页「测试」:用已保存的 Key 实际查一次(不走缓存) */
|
||||
async test(query?: string): Promise<WebSearchTestResult> {
|
||||
const s = await this.settings();
|
||||
const q = query?.trim() || TEST_QUERY;
|
||||
const started = Date.now();
|
||||
try {
|
||||
const r = await this.search({ query: q, count: Math.min(3, s.maxResults) }, { fresh: true });
|
||||
return {
|
||||
ok: true,
|
||||
provider: s.provider,
|
||||
query: q,
|
||||
count: r.results.length,
|
||||
durationMs: Date.now() - started,
|
||||
results: r.results.map(({ title, url, source }) => ({ title, url, source })),
|
||||
error: null,
|
||||
};
|
||||
} catch (err) {
|
||||
const e = err as WebSearchError | WebSearchServiceError;
|
||||
if (!(e instanceof WebSearchError) && !(e instanceof WebSearchServiceError)) throw err;
|
||||
return {
|
||||
ok: false,
|
||||
provider: s.provider,
|
||||
query: q,
|
||||
count: 0,
|
||||
durationMs: Date.now() - started,
|
||||
results: [],
|
||||
error: { code: e.kind, message: e.message },
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user