feat(pi-web-search): 统一 web_search/web_fetch 并由主模型路由搜索厂商

7 个按厂商命名的搜索工具收敛为 2 个统一工具，schema 与提示词不再暴露 provider 概念：web_search 由当前主模型按内置规则在 Tavily/Exa/Keenable 间路由（会话内 LRU 缓存，失败按可用性降级），web_fetch 固定走 Exa 并静默降级。三家均改为纯 fetch 直连——Exa 直接对 mcp.exa.ai 发 JSON-RPC 并缓存会话，彻底移除 pi-mcp-adapter、@tavily/pi-extension 与 @keenable/pi-search 依赖。 Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
2026-10-26 10:45:22 +00:00 · 2026-10-19 20:13:37 -07:00
parent 6fe6cb29ae
commit ee1e6999d7
32 changed files with 1690 additions and 1188 deletions
@@ -0,0 +1,222 @@
+/**
+ * Keenable REST client, adapted from @keenable/pi-search (keenable-client.ts).
+ * Keyless by default (`/v1/*\/public`), authenticated when KEENABLE_API_KEY is
+ * present (`/v1/*`), HTTPS-only base URL, `X-Keenable-Title` attribution.
+ */
+
+import { isIP } from "node:net";
+
+export class KeenableError extends Error {}
+
+const CLIENT_VERSION = "0.1.0";
+const DEFAULT_BASE_URL = "https://api.keenable.ai";
+const BASE_URL_ENV = "KEENABLE_API_URL";
+const API_KEY_ENV = "KEENABLE_API_KEY";
+const ATTRIBUTION_TITLE = "Pi";
+const USER_AGENT = `pi-web-search/${CLIENT_VERSION}`;
+const DEFAULT_TIMEOUT_MS = 30_000;
+
+export type FetchLike = (input: string | URL, init?: RequestInit) => Promise<Response>;
+
+export interface KeenableSearchParams {
+	query: string;
+	mode?: "pro";
+	site?: string;
+	published_after?: string;
+	published_before?: string;
+}
+
+export interface KeenableSearchResult {
+	title?: string;
+	url?: string;
+	description?: string;
+	published_at?: string;
+	acquired_at?: string;
+	[key: string]: unknown;
+}
+
+export interface KeenableFetchResult {
+	url?: string;
+	title?: string;
+	content?: string;
+	description?: string;
+	author?: string;
+	published_at?: string;
+	[key: string]: unknown;
+}
+
+/** Resolve the API base URL from `KEENABLE_API_URL`, enforcing HTTPS (http only for loopback). */
+function resolveBaseUrl(): string {
+	const base = (process.env[BASE_URL_ENV] || DEFAULT_BASE_URL).replace(/\/+$/u, "");
+	let parsed: URL;
+	try {
+		parsed = new URL(base);
+	} catch {
+		throw new KeenableError(`${BASE_URL_ENV} must be a valid URL with a host, got ${base}`);
+	}
+	if (parsed.hostname) {
+		if (parsed.protocol === "https:") return base;
+		if (parsed.protocol === "http:" && ["localhost", "127.0.0.1", "::1"].includes(parsed.hostname)) {
+			return base;
+		}
+	}
+	throw new KeenableError(`${BASE_URL_ENV} must be an https:// URL with a host, got ${base}`);
+}
+
+/** Non-blank API key from the env, or undefined to use the keyless public tier. */
+function resolveApiKey(): string | undefined {
+	const key = (process.env[API_KEY_ENV] ?? "").trim();
+	return key || undefined;
+}
+
+function buildHeaders(apiKey: string | undefined): Record<string, string> {
+	const headers: Record<string, string> = {
+		Accept: "application/json",
+		"User-Agent": USER_AGENT,
+		"X-Keenable-Title": ATTRIBUTION_TITLE,
+	};
+	if (apiKey) headers["X-API-Key"] = apiKey;
+	return headers;
+}
+
+function withTimeout(signal: AbortSignal | undefined, timeoutMs: number): AbortSignal {
+	const timeout = AbortSignal.timeout(timeoutMs);
+	return signal ? AbortSignal.any([signal, timeout]) : timeout;
+}
+
+function isPrivateIp(host: string): boolean {
+	const v = isIP(host);
+	if (v === 4) {
+		const p = host.split(".").map(Number);
+		return (
+			p[0] === 10 ||
+			p[0] === 127 ||
+			p[0] === 0 ||
+			(p[0] === 169 && p[1] === 254) ||
+			(p[0] === 172 && p[1] >= 16 && p[1] <= 31) ||
+			(p[0] === 192 && p[1] === 168) ||
+			(p[0] === 100 && p[1] >= 64 && p[1] <= 127) ||
+			(p[0] === 198 && (p[1] === 18 || p[1] === 19)) ||
+			p[0] >= 224
+		);
+	}
+	if (v === 6) {
+		const h = host.toLowerCase();
+		return h === "::1" || h === "::" || h.startsWith("fe80") || h.startsWith("fc") || h.startsWith("fd");
+	}
+	return false;
+}
+
+/** Refuse private/internal fetch targets before sending (SSRF). */
+function assertPublicFetchTarget(rawUrl: string): void {
+	let parsed: URL;
+	try {
+		parsed = new URL(rawUrl);
+	} catch {
+		throw new KeenableError(`Not a valid URL: ${rawUrl}`);
+	}
+	if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+		throw new KeenableError(`Refusing to fetch a non-http(s) URL: ${rawUrl}`);
+	}
+	const host = (parsed.hostname || "").toLowerCase();
+	if (!host) throw new KeenableError(`Refusing to fetch a URL with no host: ${rawUrl}`);
+	if (host === "localhost" || host === "metadata.google.internal" || isPrivateIp(host)) {
+		throw new KeenableError(`Refusing to fetch a private/internal host: ${host}`);
+	}
+}
+
+async function raiseForStatus(response: Response, keyed: boolean): Promise<void> {
+	if (response.ok) return;
+	if (!keyed && response.status === 429) {
+		throw new KeenableError(
+			"Keenable keyless requests hit their rate limit. Set KEENABLE_API_KEY to raise the limits.",
+		);
+	}
+	let detail = "";
+	try {
+		const body = (await response.json()) as Record<string, unknown>;
+		detail = String(body.message ?? body.error ?? body.detail ?? "");
+	} catch {
+		detail = (await response.text().catch(() => "")).trim();
+	}
+	const label =
+		{
+			401: "Keenable authentication failed (401)",
+			402: "Keenable: insufficient credits (402)",
+			429: "Keenable rate limit exceeded (429)",
+		}[response.status] ?? `Keenable API error (${response.status})`;
+	throw new KeenableError(detail ? `${label}: ${detail}` : label);
+}
+
+async function readJsonObject(response: Response): Promise<Record<string, unknown>> {
+	let data: unknown;
+	try {
+		data = await response.json();
+	} catch {
+		throw new KeenableError("Keenable API returned a non-JSON response.");
+	}
+	if (!data || typeof data !== "object" || Array.isArray(data)) {
+		throw new KeenableError("Unexpected (non-object) response from the Keenable API.");
+	}
+	return data as Record<string, unknown>;
+}
+
+/** POST /v1/search (keyed) or /v1/search/public (keyless). */
+export async function keenableSearch(
+	params: KeenableSearchParams,
+	signal: AbortSignal | undefined,
+	fetchImpl: FetchLike = fetch,
+): Promise<KeenableSearchResult[]> {
+	const apiKey = resolveApiKey();
+	const path = apiKey ? "/v1/search" : "/v1/search/public";
+	const payload: Record<string, unknown> = { query: params.query, mode: params.mode ?? "pro" };
+	for (const field of ["site", "published_after", "published_before"] as const) {
+		const value = params[field];
+		if (value) payload[field] = value;
+	}
+
+	let response: Response;
+	try {
+		response = await fetchImpl(`${resolveBaseUrl()}${path}`, {
+			method: "POST",
+			headers: { ...buildHeaders(apiKey), "Content-Type": "application/json" },
+			body: JSON.stringify(payload),
+			signal: withTimeout(signal, DEFAULT_TIMEOUT_MS),
+		});
+	} catch (err) {
+		throw new KeenableError(`Could not reach the Keenable API: ${(err as Error).message}`);
+	}
+	await raiseForStatus(response, Boolean(apiKey));
+	const data = await readJsonObject(response);
+	const results = data.results;
+	if (!Array.isArray(results)) {
+		throw new KeenableError("Unexpected response from the Keenable search API (no results array).");
+	}
+	return results as KeenableSearchResult[];
+}
+
+/** GET /v1/fetch?url= (keyed) or /v1/fetch/public?url= (keyless). */
+export async function keenableFetch(
+	url: string,
+	signal: AbortSignal | undefined,
+	fetchImpl: FetchLike = fetch,
+): Promise<KeenableFetchResult> {
+	assertPublicFetchTarget(url);
+	const apiKey = resolveApiKey();
+	const path = apiKey ? "/v1/fetch" : "/v1/fetch/public";
+	const endpoint = new URL(`${resolveBaseUrl()}${path}`);
+	endpoint.searchParams.set("url", url);
+
+	let response: Response;
+	try {
+		response = await fetchImpl(endpoint, {
+			method: "GET",
+			headers: buildHeaders(apiKey),
+			signal: withTimeout(signal, DEFAULT_TIMEOUT_MS),
+		});
+	} catch (err) {
+		throw new KeenableError(`Could not reach the Keenable API: ${(err as Error).message}`);
+	}
+	await raiseForStatus(response, Boolean(apiKey));
+	return (await readJsonObject(response)) as KeenableFetchResult;
+}