feat(pi-web-search): 统一 web_search/web_fetch 并由主模型路由搜索厂商

7 个按厂商命名的搜索工具收敛为 2 个统一工具，schema 与提示词不再暴露 provider 概念：web_search 由当前主模型按内置规则在 Tavily/Exa/Keenable 间路由（会话内 LRU 缓存，失败按可用性降级），web_fetch 固定走 Exa 并静默降级。三家均改为纯 fetch 直连——Exa 直接对 mcp.exa.ai 发 JSON-RPC 并缓存会话，彻底移除 pi-mcp-adapter、@tavily/pi-extension 与 @keenable/pi-search 依赖。 Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
2026-10-26 10:45:22 +00:00 · 2026-10-19 20:13:37 -07:00
parent 6fe6cb29ae
commit ee1e6999d7
32 changed files with 1690 additions and 1188 deletions
@@ -0,0 +1,282 @@
+/**
+ * pi-web-search: one unified web_search + web_fetch surface.
+ *
+ * The main model never sees provider names — web_search routes internally via
+ * the session's current model (see router.ts), and web_fetch always reads
+ * pages through Exa with graceful degradation. Provider clients are plain
+ * fetch calls (clients/*) — no MCP adapter, no upstream extension wrappers.
+ */
+
+import type { AgentToolResult, ExtensionAPI } from "@earendil-works/pi-coding-agent";
+
+import { loadSearchConfigEnv } from "../extensions/search-config.ts";
+import { exaFetchPages, exaSearch } from "./clients/exa.ts";
+import { keenableFetch, keenableSearch } from "./clients/keenable.ts";
+import { tavilyAvailable, tavilyExtract, tavilySearch } from "./clients/tavily.ts";
+import {
+	availableProviders,
+	routeSearch,
+	type SearchProvider,
+	type RouterContext,
+} from "./router.ts";
+
+const DEFAULT_MAX_RESULTS = 8;
+const MAX_FETCH_URLS = 10;
+const DEFAULT_FETCH_MAX_CHARACTERS = 3000;
+
+export interface WebSearchParams {
+	query: string;
+	max_results?: number;
+	site?: string;
+	published_after?: string;
+	published_before?: string;
+	depth?: "basic" | "advanced";
+}
+
+export interface WebFetchParams {
+	urls: string | string[];
+	max_characters?: number;
+}
+
+export type ProviderRunner = (params: WebSearchParams, signal?: AbortSignal) => Promise<string>;
+export type FetchRunner = (urls: string[], maxCharacters: number, signal?: AbortSignal) => Promise<string>;
+
+function formatSearchPayload(payload: unknown): string {
+	return JSON.stringify(payload, null, 2);
+}
+
+export function toTavilyParams(params: WebSearchParams) {
+	return {
+		query: params.query,
+		max_results: params.max_results ?? DEFAULT_MAX_RESULTS,
+		search_depth: (params.depth ?? "basic") as "basic" | "advanced",
+		include_domains: params.site ? [params.site] : undefined,
+		start_date: params.published_after,
+		end_date: params.published_before,
+	};
+}
+
+export function toExaParams(params: WebSearchParams) {
+	return {
+		query: params.query,
+		numResults: params.max_results ?? DEFAULT_MAX_RESULTS,
+		includeDomains: params.site ? [params.site] : undefined,
+		startPublishedDate: params.published_after,
+		endPublishedDate: params.published_before,
+	};
+}
+
+export function toKeenableParams(params: WebSearchParams) {
+	return {
+		query: params.query,
+		site: params.site,
+		published_after: params.published_after,
+		published_before: params.published_before,
+	};
+}
+
+/** Default per-provider parameter mapping from the unified schema. */
+export function createProviderRunners(
+	overrides: Partial<Record<SearchProvider, ProviderRunner>> = {},
+): Record<SearchProvider, ProviderRunner> {
+	return {
+		tavily:
+			overrides.tavily ??
+			(async (params, signal) => formatSearchPayload(await tavilySearch(toTavilyParams(params), signal))),
+		exa: overrides.exa ?? ((params, signal) => exaSearch(toExaParams(params), signal)),
+		keenable:
+			overrides.keenable ??
+			(async (params, signal) => formatSearchPayload(await keenableSearch(toKeenableParams(params), signal))),
+	};
+}
+
+function errorText(error: unknown): string {
+	return error instanceof Error ? error.message : String(error);
+}
+
+function toResult(text: string, details: Record<string, unknown>): AgentToolResult {
+	return { content: [{ type: "text", text }], details };
+}
+
+/**
+ * Route once via the main model, then try the picked provider first and fall
+ * back across the remaining available providers on transport errors.
+ */
+export async function dispatchSearch(
+	params: WebSearchParams,
+	ctx: RouterContext,
+	routeCache: Map<string, SearchProvider>,
+	signal?: AbortSignal,
+	runners: Record<SearchProvider, ProviderRunner> = createProviderRunners(),
+): Promise<AgentToolResult> {
+	const decision = await routeSearch(params, ctx, routeCache, signal);
+	const order = [decision.provider, ...availableProviders().filter((p) => p !== decision.provider)];
+	let lastError: unknown;
+	for (const provider of order) {
+		try {
+			const text = await runners[provider](params, signal);
+			return toResult(text, {
+				provider,
+				routedBy: provider === decision.provider ? decision.routedBy : "provider-fallback",
+			});
+		} catch (error) {
+			lastError = error;
+		}
+	}
+	return toResult(`web_search failed on every available provider: ${errorText(lastError)}`, { error: true });
+}
+
+export function normalizeUrls(urls: string | string[]): string[] {
+	const list = (Array.isArray(urls) ? urls : [urls]).map((url) => url.trim()).filter(Boolean);
+	return [...new Set(list)];
+}
+
+/** web_fetch always prefers Exa; Keenable and Tavily are silent degradation paths. */
+export function createFetchRunners(
+	overrides: Partial<Record<SearchProvider, FetchRunner>> = {},
+): Record<SearchProvider, FetchRunner> {
+	return {
+		exa: overrides.exa ?? ((urls, maxCharacters, signal) => exaFetchPages(urls, maxCharacters, signal)),
+		keenable:
+			overrides.keenable ??
+			(async (urls, _maxCharacters, signal) => {
+				const pages = await Promise.all(urls.map((url) => keenableFetch(url, signal)));
+				return pages
+					.map((page) => `# ${page.title ?? page.url ?? "page"}\n${page.content ?? "(no content)"}`)
+					.join("\n\n---\n\n");
+			}),
+		tavily:
+			overrides.tavily ??
+			(async (urls, _maxCharacters, signal) => {
+				const { results, failed } = await tavilyExtract(urls, signal);
+				const body = results
+					.map((page) => `# ${page.url ?? "page"}\n${page.content ?? "(no content)"}`)
+					.join("\n\n---\n\n");
+				return failed.length ? `${body}\n\nFailed URLs: ${failed.join(", ")}` : body;
+			}),
+	};
+}
+
+export async function dispatchFetch(
+	params: WebFetchParams,
+	signal?: AbortSignal,
+	fetchers: Record<SearchProvider, FetchRunner> = createFetchRunners(),
+): Promise<AgentToolResult> {
+	const urls = normalizeUrls(params.urls);
+	if (!urls.length) {
+		return toResult("web_fetch needs at least one http(s) URL.", { error: true });
+	}
+	if (urls.length > MAX_FETCH_URLS) {
+		return toResult(`web_fetch accepts at most ${MAX_FETCH_URLS} URLs per call; received ${urls.length}.`, { error: true });
+	}
+	const maxCharacters = params.max_characters ?? DEFAULT_FETCH_MAX_CHARACTERS;
+	const order: SearchProvider[] = ["exa", "keenable"];
+	if (tavilyAvailable()) order.push("tavily");
+
+	let lastError: unknown;
+	for (const provider of order) {
+		try {
+			const text = await fetchers[provider](urls, maxCharacters, signal);
+			return toResult(text, { provider });
+		} catch (error) {
+			lastError = error;
+		}
+	}
+	return toResult(`web_fetch failed on every available provider: ${errorText(lastError)}`, { error: true });
+}
+
+export default function webSearchExtension(pi: ExtensionAPI): void {
+	// Keys may already be loaded by extensions/search-config.ts; this is a
+	// harmless idempotent safety net for standalone loading.
+	loadSearchConfigEnv();
+	const routeCache = new Map<string, SearchProvider>();
+
+	pi.registerTool({
+		name: "web_search",
+		label: "Web Search",
+		description:
+			"Search the web for current or citable information and return a compact list of candidate " +
+			"pages (title, url, summary). Provider selection is automatic. Follow up with web_fetch to " +
+			"read the most relevant pages.",
+		promptSnippet: "Search the web for current or citable information.",
+		promptGuidelines: [
+			"Use web_search when the task needs current, real-world, or citable web information.",
+			"Keep queries focused; split multi-topic requests into separate web_search calls.",
+			"Use site and published_* filters only when they materially narrow the results.",
+			"After web_search, read the most relevant pages with web_fetch before relying on a claim.",
+		],
+		parameters: {
+			type: "object",
+			properties: {
+				query: {
+					type: "string",
+					description: "The search query — a natural-language description of the ideal page, not just keywords.",
+				},
+				max_results: {
+					type: "number",
+					description: `Maximum number of results to return (default ${DEFAULT_MAX_RESULTS}).`,
+				},
+				site: {
+					type: "string",
+					description: "Restrict results to a single domain, e.g. 'arxiv.org'.",
+				},
+				published_after: {
+					type: "string",
+					description: "Only pages published on/after this date (YYYY-MM-DD).",
+				},
+				published_before: {
+					type: "string",
+					description: "Only pages published on/before this date (YYYY-MM-DD).",
+				},
+				depth: {
+					type: "string",
+					enum: ["basic", "advanced"],
+					description: "'basic' (default) balanced search; 'advanced' for higher precision at higher cost.",
+				},
+			},
+			required: ["query"],
+			additionalProperties: false,
+		},
+		async execute(_toolCallId, params, signal, _onUpdate, ctx) {
+			return dispatchSearch(params as WebSearchParams, ctx as unknown as RouterContext, routeCache, signal);
+		},
+	});
+
+	pi.registerTool({
+		name: "web_fetch",
+		label: "Web Fetch",
+		description:
+			"Fetch the full content of one or more http(s) URLs as clean markdown. Use after web_search " +
+			"to read selected pages, or directly when the user provides a page to read.",
+		promptSnippet: "Read the full content of known URLs.",
+		promptGuidelines: [
+			"Use web_fetch to read the full content of a URL found via web_search or provided by the user.",
+			"Batch multiple URLs into a single web_fetch call when possible.",
+		],
+		parameters: {
+			type: "object",
+			properties: {
+				urls: {
+					anyOf: [
+						{ type: "string", description: "A single http(s) URL to read." },
+						{
+							type: "array",
+							items: { type: "string" },
+							maxItems: MAX_FETCH_URLS,
+							description: `A list of http(s) URLs to read (max ${MAX_FETCH_URLS}).`,
+						},
+					],
+				},
+				max_characters: {
+					type: "number",
+					description: `Maximum characters to extract per page (default ${DEFAULT_FETCH_MAX_CHARACTERS}).`,
+				},
+			},
+			required: ["urls"],
+			additionalProperties: false,
+		},
+		async execute(_toolCallId, params, signal) {
+			return dispatchFetch(params as WebFetchParams, signal);
+		},
+	});
+}