From ee1e6999d7b0ce62661b3c7709e55100b9bc631d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BA=91=E6=9C=8D=E5=8A=A1=E9=83=A8-=E5=8F=B6=E6=9E=97?= =?UTF-8?q?=E7=AB=8B?= <1361666059@qq.com> Date: Mon, 19 Oct 2026 20:13:37 -0700 Subject: [PATCH] =?UTF-8?q?feat(pi-web-search):=20=E7=BB=9F=E4=B8=80=20web?= =?UTF-8?q?=5Fsearch/web=5Ffetch=20=E5=B9=B6=E7=94=B1=E4=B8=BB=E6=A8=A1?= =?UTF-8?q?=E5=9E=8B=E8=B7=AF=E7=94=B1=E6=90=9C=E7=B4=A2=E5=8E=82=E5=95=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 7 个按厂商命名的搜索工具收敛为 2 个统一工具，schema 与提示词不再暴露 provider 概念：web_search 由当前主模型按内置规则在 Tavily/Exa/Keenable 间路由（会话内 LRU 缓存，失败按可用性降级），web_fetch 固定走 Exa 并 静默降级。三家均改为纯 fetch 直连——Exa 直接对 mcp.exa.ai 发 JSON-RPC 并缓存会话，彻底移除 pi-mcp-adapter、@tavily/pi-extension 与 @keenable/pi-search 依赖。 Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- AGENTS.md | 12 +- README.md | 12 +- config/pi-permission-policy.jsonc | 5 - extensions/exa-tool-prefix.ts | 57 -- extensions/mcp-config.ts | 26 - extensions/mcp.ts | 12 - extensions/tavily-override.ts | 21 - extensions/tavily-tool-prefix.ts | 25 - extensions/tool-routing.ts | 22 +- package-lock.json | 833 +-------------------- package.json | 12 +- pi-tool-search/README.md | 8 +- pi-tool-search/extensions/bundle-groups.ts | 32 +- pi-tool-search/test/bundle-groups.test.ts | 8 +- pi-tool-search/test/extension.test.ts | 26 +- pi-web-search/LICENSE | 21 + pi-web-search/README.md | 35 + pi-web-search/clients/exa.ts | 243 ++++++ pi-web-search/clients/keenable.ts | 222 ++++++ pi-web-search/clients/tavily.ts | 125 ++++ pi-web-search/index.ts | 282 +++++++ pi-web-search/package.json | 42 ++ pi-web-search/router.ts | 163 ++++ pi-web-search/test/dispatch.test.ts | 178 +++++ pi-web-search/test/exa-client.test.ts | 140 ++++ pi-web-search/test/router.test.ts | 109 +++ tests/exa-override.test.ts | 45 -- tests/extension-load.test.ts | 5 + tests/mcp.test.ts | 63 -- tests/tavily-override.test.ts | 22 - tests/tool-routing.test.ts | 25 +- tests/web-search.test.ts | 47 ++ 32 files changed, 1690 insertions(+), 1188 deletions(-) delete mode 100644 extensions/exa-tool-prefix.ts delete mode 100644 extensions/mcp-config.ts delete mode 100644 extensions/mcp.ts delete mode 100644 extensions/tavily-override.ts delete mode 100644 extensions/tavily-tool-prefix.ts create mode 100644 pi-web-search/LICENSE create mode 100644 pi-web-search/README.md create mode 100644 pi-web-search/clients/exa.ts create mode 100644 pi-web-search/clients/keenable.ts create mode 100644 pi-web-search/clients/tavily.ts create mode 100644 pi-web-search/index.ts create mode 100644 pi-web-search/package.json create mode 100644 pi-web-search/router.ts create mode 100644 pi-web-search/test/dispatch.test.ts create mode 100644 pi-web-search/test/exa-client.test.ts create mode 100644 pi-web-search/test/router.test.ts delete mode 100644 tests/exa-override.test.ts delete mode 100644 tests/mcp.test.ts delete mode 100644 tests/tavily-override.test.ts create mode 100644 tests/web-search.test.ts diff --git a/AGENTS.md b/AGENTS.md index f5e79f9..ece66a2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,10 +21,8 @@ - 初始导入快照：`3f9d35cd078d18a141eb5a6d8f4fc5010d756280`（`v0.18.0`）。 - 该目录从明确 tag 导入并由本仓库直接维护，不使用 npm 预编译产物，不是 submodule，也不保留上游 `.pi` 状态；源码、测试、wrapper 与权限集成实现继续保留，但当前不属于根组合包运行时依赖，`extensions/subagents.ts` 也不在默认扩展加载列表中。 - `extensions/fff-override.ts`：强制使用 FFF 官方 `override` 模式，统一接管 `find`、`grep`、`multi_grep` 和 FFF 的 `@` 补全；显式 CLI flag 仍遵循 FFF 官方优先级。 -- `extensions/tavily-override.ts`：加载 `@tavily/pi-extension`，把上游工具重命名为 `tavily_web_search` / `tavily_web_fetch`，并同步改写 label 与工具提示中的内部名称。 -- `@keenable/pi-search`：加载 `keenable_search` / `keenable_fetch` 及其 skill，默认 keyless，`KEENABLE_API_KEY` 仅用于提高速率限制。 +- `pi-web-search/`：完全自维护的统一在线搜索扩展。对模型只暴露 `web_search` / `web_fetch`，schema 与提示词不含厂商名；`web_search` 由当前主模型（`ctx.model` + `ctx.modelRegistry.complete`）按内置路由规则在 Tavily / Exa / Keenable 间选择并经统一参数翻译调用，`details` 记录实际 provider 与路由方式，决策按会话内 LRU 缓存，路由失败或被选厂商报错按可用性顺序降级；`web_fetch` 固定走 Exa `web_fetch_exa`，失败时静默降级 Keenable → Tavily。`clients/` 为纯 `fetch` 直连：Tavily 走 `api.tavily.com`（必须有 key），Keenable 复刻上游 client 保留 keyless `/public` 端点与 SSRF 防护，Exa 直接对 `mcp.exa.ai/mcp` 发 JSON-RPC（缓存 `mcp-session-id`，404 自动重建会话，key 只走 `x-api-key` 头），不使用 `pi-mcp-adapter` 或任何上游扩展包。 - `search_config.sh` 与 `extensions/search-config.ts`：以交互式或参数方式把三家搜索 key 写入用户级 `search.env`（权限 `600`），并在搜索扩展初始化前加载；显式进程环境变量优先。 -- `extensions/mcp.ts`：通过单个共享 `pi-mcp-adapter` 实例连接 Exa 托管 MCP，避免重复注册 Pi 的全局 MCP flag 与命令；Exa key 只通过 `x-api-key` 请求头发送，原始工具映射为统一的来源前缀形式。CodeGraph 不走 MCP。 - `extensions/codegraph.ts`：注册原生 `codegraph_explore` 工具，以 `execFile` argv 形式调用机器级 `codegraph` CLI（每次调用为短命进程，无常驻 server/daemon）；未初始化项目先经 `ctx.ui.confirm` 确认再 `codegraph init` 并自动重试，拒绝或失败则把结果打回让模型改用 `find`/`grep`/`read`/LSP；结果按 `codegraph.db` mtime 做有界缓存，索引一变即失效。 - `extensions/plugin-init.ts`：已移除；原 `/plugin_init` 的 CodeGraph 初始化职责由 `extensions/codegraph.ts` 在首次 `codegraph_explore` 调用内按需确认完成。 - `context-mode`：加载上游 Pi adapter 与 skills，提供 `ctx_*` 工具、隔离式大输出处理和会话连续性。 @@ -66,7 +64,7 @@ - 上游参考一：，快照 `59c95110aacd8131db71e21296765dd80b72fae8`（`v2.9.1`）。 - 上游来源二：，导入快照 `c4d6ca8fb484ba51684296ba8b4715f8c7b3ac91`（`0.1.4`）。 - `pi-compaction` 采用全新的 `pi-compaction` version 1 Session 记录和 `settings.json.piCompaction` 配置，不读取或迁移旧 `context-prune-*`、`contextPrune` 或 `openai-codex-native-compaction` 状态；切换后应使用新 Session。两份上游 MIT 许可证及来源差异保留在包内。 -- 根包还固定安装 `@tavily/pi-extension@0.1.2`、`@keenable/pi-search@0.1.2`、`pi-hashline-edit@0.8.3`、`typescript-language-server@5.3.0` 和 `typescript@6.0.3`；`@gotgenes/pi-permission-system`、`@smoose/pi-notify`、`pi-ask-user`、`pi-chrome`、`pi-compaction`、`pi-hermes-memory`、`pi-minimal-footer`、`pi-ssh`、`pi-tool-search` 与 `pi-worktree` 分别指向仓库内对应目录的本地 `file:` 依赖；`pi-subagents/` 仅保留源码，不作为根运行时依赖。 +- 根包还固定安装 `pi-hashline-edit@0.8.3`、`typescript-language-server@5.3.0` 和 `typescript@6.0.3`；`@gotgenes/pi-permission-system`、`@smoose/pi-notify`、`pi-ask-user`、`pi-chrome`、`pi-compaction`、`pi-hermes-memory`、`pi-minimal-footer`、`pi-ssh`、`pi-tool-search`、`pi-web-search` 与 `pi-worktree` 分别指向仓库内对应目录的本地 `file:` 依赖；`pi-subagents/` 仅保留源码，不作为根运行时依赖。 - `install.sh`：先安装根组合包，再通过 `scripts/rebuild-bundle-config.mjs` 整体重建 `/my-pi/` 受管配置目录，然后交互检查 Kitty/Solarized Dark、Oh My Zsh/Powerlevel10k/Zsh 插件、CodeGraph、Kotlin LSP、Java 21+ 和 JDT LS；缺失项目只在用户明确选择 `Y` 后安装或配置。Powerlevel10k 默认配置来自仓库内置的 `config/p10k.zsh`（当前 Rainbow/ASCII 单行紧凑主题）。 - `update.sh`：先通过 `pi update` 升级根组合包，再只升级当前已安装的终端环境和机器级依赖；未安装项直接跳过。不得在 `pi update` 前预清理安装缓存；`pi update` 失败时 fail-fast 停止——Pi 1.3.0 会在下一次 `pi update` 时依据其 `.pi-update-incomplete` 标记自行重跑 `git clean -fdx` 与依赖安装完成自愈，本脚本不做人工恢复。升级成功后调用可定位的更新后组合包 Node helper `scripts/rebuild-bundle-config.mjs` 整体重建 `/my-pi/`：先按 `config/bundle-manifest.json` 校验全部来源与目标，在 agent 目录内构建临时树，再删除旧 `my-pi/`（符号链接只删链接不跟随）并原子替换，不备份、不迁移、不合并不属于组合包的内容。无法从 `PI_PACKAGE_SOURCE` 定位更新后 helper 时必须报告失败，不能回退到可能陈旧的脚本目录 helper。其他组件升级前先查询并比较本地与远端版本，只有版本不同时才下载或替换；Powerlevel10k 配置优先从 `pi update` 后的已安装组合包读取，并与 `.zshrc` 受管块一起按内容比较后增量同步。 - `uninstall.sh`：先移除根组合包，其他共享终端环境和机器工具不卸载。不得删除 `.pi/agent/pi-hermes-memory/`、`projects-memory/`、`.hippo/` 或任何用户记忆数据。 @@ -81,7 +79,7 @@ - Context Mode 负责避免批量读取、命令研究和网页原始内容直接撑大上下文；FFF 仍负责精确字面搜索，RTK 仍处理未走 Context Mode 的普通输出。 - `chrome_snapshot`、`chrome_find` 以及带 `includeSnapshot=true` 的交互工具必须把完整结构化快照写入当前项目 `.pi/chrome-context/` 下的 Session 临时文件，工具结果只返回相对路径与有界摘要；模型必须立即用 `ctx_execute_file` 提取下一步所需信息，不得用 `read` / `cat` 把原始 JSON 重新塞入上下文。每个 Session/目标只保留最新快照，页面交互后旧快照视为失效，Session shutdown、撤销或授权过期时清理。 - Hashline 默认以 2 字符行哈希覆盖内置 `read` / `edit`，`grep` 和 `replaceText` 均关闭；它保证锚定编辑而不负责压缩，大文件上下文节省必须来自先定位/隔离分析、再做最小范围 `offset/limit` 读取。 -- 工具路由规则不得替换 Pi 默认系统提示词：代码结构、调用关系和待修改 symbol 优先 CodeGraph，定义/引用/类型/诊断优先 LSP；工作区内大文件探索、分析和总结优先 `ctx_execute_file`，日志、构建与不可预测命令输出优先 Context Mode；字面搜索先用 FFF `find` 收敛文件位置，再在收敛路径内用 `grep`/`multi_grep` 获取行号；真正修改前才用小范围 Hashline `read` 获取新鲜锚点并用锚定 `edit`。在线搜索按场景只先选一个服务：广泛发现、新闻和候选来源用 Tavily，官方/技术/论文和高确定性来源用 Exa，中文、站点限定和日期筛选用 Keenable；只有必要时才跨服务复核。遇到大量结果、截断或上限时继续缩小范围，不得靠提高 limit 或倾倒全部结果解决。 +- 工具路由规则不得替换 Pi 默认系统提示词：代码结构、调用关系和待修改 symbol 优先 CodeGraph，定义/引用/类型/诊断优先 LSP；工作区内大文件探索、分析和总结优先 `ctx_execute_file`，日志、构建与不可预测命令输出优先 Context Mode；字面搜索先用 FFF `find` 收敛文件位置，再在收敛路径内用 `grep`/`multi_grep` 获取行号；真正修改前才用小范围 Hashline `read` 获取新鲜锚点并用锚定 `edit`。在线搜索由 `pi-web-search` 内部路由，模型不感知厂商：路由规则为广泛发现、新闻和候选来源偏 Tavily，官方/技术/论文和高确定性来源偏 Exa，中文、站点限定和日期筛选偏 Keenable；路由决策由当前主模型做出，确定性逻辑只处理可用性与失败兜底，不得改写成启发式断路。遇到大量结果、截断或上限时继续缩小范围，不得靠提高 limit 或倾倒全部结果解决。 - `ask_user_question` 只用于模型继续任务所必需的用户决定、偏好、确认或澄清；不得重复询问直接用户消息中已有的信息。工具支持交互式 TUI 与具备 Pi extension UI bridge 的 RPC 客户端、顺序执行和最多 8 题；RPC 必须使用标准 `select` / `input` 对话框逐题收集，JSON/print 等无 UI 模式保持拒绝。取消或 abort 必须返回 `cancelled: true`，不得伪造答案，且 `/qna` 不属于该扩展。其成功 `details.answers` 必须保留 `question` / `answer` 字段，以匹配 AutoReview 已有的可信结构化用户交互边界。 - Tool Search 默认常驻核心 `read` / `write` / `edit` / `bash` / `grep` / `find`、`codegraph_explore`、`lsp_diagnostics`、`ctx_execute` / `ctx_execute_file` / `ctx_batch_execute`、六个记忆工具与 `context_tree_query`，以及 `tool_search`；Windows 上内置 `powershell` 必须与 `bash` 同属固定本地 Shell 核心工具，非 Windows 平台必须从目录中过滤。配置从 `/my-pi/tool-search/config.json` 读取（由 rebuild 脚本部署 `config/tool-search.json`），文件缺失或无效时回退 `BUNDLE_TOOL_SEARCH_DEFAULTS`；扩展自身不播种、不迁移，也不再读写 `settings.json` 的 `toolSearch` 段。其余组合包工具必须优先使用 `pi-tool-search/extensions/bundle-groups.ts` 中提交的权威工作流分组；精确已有分组必须直接激活，不得等待无关未知工具的模型增强。默认最多 5 个动态组、模型组最多 8 个工具、动态工具总数最多 28；`groups` 可原子激活最多 5 个精确组，完整请求超限时不得改变当前状态；加载或组内调用更新 LRU，超限先移除最久未用的非请求组。自然语言查询必须优先精确工具名和已提交 workflow alias，弱匹配或近似同分只能返回候选，不得任意激活；`/tool-search-status` 必须显示当前容量、LRU 顺序与最近淘汰原因。 - `pi-chrome` 在未授权时不注册 Chrome 工具；显式授权后出现的 21 个 `chrome_*` 工具必须由 Tool Search 分入 `chrome-navigation`、`chrome-interaction`、`chrome-debugging`，不得落入 unknown-tools 模型增强路径。授权撤销或过期时必须停用全部 21 个工具，包括 `chrome_find` / `chrome_inspect`；Tool Search 重新激活 schema 不能替代每次执行时的授权校验。 @@ -91,8 +89,8 @@ - `.pi-debug/` 是 `/dump-system-prompt` 生成的本地诊断目录，不提交到仓库，也不作为组合包运行时配置源。 - CodeGraph 是机器级依赖：`install.sh`/`update.sh` 经官方安装器管理 `codegraph` CLI，组合包不内置其运行时。安装、升级和普通启动流程不得执行 `codegraph init`；只有 `codegraph_explore` 在未初始化项目上被调用、且用户在 `ctx.ui.confirm` 中明确确认后，扩展才可以 `codegraph init` 当前目录。init 失败必须如实报告并让模型回退内置工具；不得改动 CodeGraph 更新或遥测设置。 - `codegraph_explore` 为原生 Pi 工具而非 MCP 工具：每次调用通过 `execFile`/`pi.exec` 以 argv 形式运行 `codegraph explore`（带超时、不经 shell），结果按 `.codegraph/codegraph.db` mtime 缓存；CLI 缺失、init 被拒绝或执行失败都不得抛出或阻止其他工具，必须以文本结果打回并引导模型改用 `find`/`grep`/`read`/LSP。 -- 所有 MCP 服务必须合并到 `extensions/mcp.ts` 创建的唯一 `pi-mcp-adapter` 实例；不得新增独立 adapter 扩展入口，否则会重复注册 `--mcp-config`、`/mcp` 等全局接口。根测试必须扫描全部本地扩展入口并强制这一不变量。 -- Exa MCP 使用托管 Streamable HTTP 端点和 `eager` lifecycle；`EXA_API_KEY` 通过 `x-api-key` 请求头发送，不得放入 URL、仓库文件或日志。所有在线搜索工具统一采用“来源名 + 原始语义工具名”的命名形式，例如 `tavily_web_search`、`exa_web_search`、`keenable_search`。Exa 高级搜索必须显式约束结果数量和文本长度；Tavily 输出默认控制 `max_results` 且非必要不请求 raw content；Keenable 优先利用中文、站点和日期筛选能力。 +- 组合包不加载任何 MCP adapter；Exa 由 `pi-web-search/clients/exa.ts` 以纯 `fetch` JSON-RPC 直连托管端点 `https://mcp.exa.ai/mcp`：`initialize` 一次并缓存 `mcp-session-id`，随后 `tools/call`，HTTP 404 视为会话失效自动重建并重试一次。`EXA_API_KEY` 通过 `x-api-key` 请求头发送，不得放入 URL、仓库文件或日志。 +- 在线搜索对模型只暴露 `web_search` / `web_fetch` 两个统一工具名，不得重新引入按厂商命名的工具；`web_search` 必须经 `router.ts` 的主模型分类选择 provider，`web_fetch` 固定优先 Exa。Exa 搜索统一走 `web_search_advanced_exa` 并显式约束 `numResults` 与 `textMaxCharacters`；Tavily 默认 `search_depth: basic` 且必须设置 `TAVILY_API_KEY` 才参与路由；Keenable keyless 可用，`KEENABLE_API_KEY` 仅提高速率限制。 - 搜索 key 默认保存在 `${XDG_CONFIG_HOME:-$HOME/.config}/my-pi/search.env`，文件必须为 `600` 且不得提交；`search_config.sh` 只输出配置状态，禁止回显 key。 - `extensions/context-view.ts` 是只读的本地 effective-context observer，只读取 `pi-compaction/src/service.ts` 发布的 projection metrics，不注册生产 `context` writer；`pi-minimal-footer` 优先显示同一 effective usage，snapshot 尚未建立时才回退宿主原始 usage。 - `pi-compaction` 负责已经进入 Session 的历史投影，与 Context Mode 的输入隔离职责互补。首次 Session hydration 允许一次完整扫描；后续普通 `context` hot path 必须复用不可变 snapshot 或只处理 delta，不得调用 summarizer、写 Session 或重新构建完整 branch。 diff --git a/README.md b/README.md index ae1185c..8d2b26b 100644 --- a/README.md +++ b/README.md @@ -5,9 +5,7 @@ - `@ff-labs/pi-fff@0.10.5`：使用官方 `override` 模式接管 `find`、`grep` 和 `multi_grep`。 - 本仓库维护的 `pi-rtk-optimizer`：只处理 Bash/read 等输出压缩，默认不改写命令，也不处理搜索工具结果。 - `context-mode@1.0.169`：通过 Pi adapter 提供 `ctx_*` 工具、上下文隔离和会话连续性。 -- `@tavily/pi-extension@0.1.2`：提供 Tavily 广泛搜索与页面提取；通过本地 override 暴露为 `tavily_web_search` / `tavily_web_fetch`。 -- `@keenable/pi-search@0.1.2`：提供紧凑的 `keenable_search` / `keenable_fetch`，并加载其搜索 skill；API key 可选。 -- `pi-mcp-adapter@2.26.0`：通过单个共享 adapter 实例接入 Exa 托管 MCP，避免重复注册 Pi 的全局 MCP flag 与命令。 +- 本仓库自维护的 `pi-web-search`：对模型只暴露 `web_search` / `web_fetch` 两个工具；`web_search` 由当前主模型在 Tavily / Exa / Keenable 间路由，`web_fetch` 固定走 Exa，三家均以纯 `fetch` 直连（Exa 为 `mcp.exa.ai` 的 JSON-RPC，不经 MCP adapter）。 - `extensions/codegraph.ts`：注册原生 `codegraph_explore` 工具，每次调用以短命 `codegraph explore` CLI 进程完成，无常驻 server/daemon；未初始化项目经确认后才 `codegraph init`，结果按索引库 mtime 缓存。 - `extensions/context-view.ts`：通过 `/context` 只读显示 `pi-compaction` 的 effective context，不建立第二套上下文口径。 - `pi-lsp@0.1.7`：为 TypeScript/JavaScript、Kotlin 和 Java 提供 LSP 诊断、跳转与符号工具；TypeScript 后端由组合包内置。 @@ -262,13 +260,9 @@ FFF 仍负责精确字面搜索；RTK 继续压缩未走 Context Mode 的 Bash `MY_PI_SEARCH_CONFIG` 或 `--config` 可覆盖路径。组合包在搜索扩展初始化前加载该文件， 但调用 Pi 时显式传入的同名环境变量优先。配置文件不属于仓库，不会随安装包提交。 -Tavily 通过 `extensions/tavily-override.ts` 加载官方扩展。override 只改工具身份，不改官方执行逻辑：上游 `web_search` / `web_fetch` 被注册为 `tavily_web_search` / `tavily_web_fetch`，label 和提示词中的内部工具引用也同步带上 Tavily 标识。使用前需设置 `TAVILY_API_KEY`。Tavily 适合广泛发现，但来源可能混杂且输出较长，应从较小的 `max_results` 开始，非必要不返回 raw content。 +三个服务由本仓库自维护的 `pi-web-search/` 统一封装：对模型只暴露 `web_search` / `web_fetch` 两个工具，schema 与提示词中不出现任何厂商名；每次 `web_search` 先由当前主模型（`ctx.model` + `ctx.modelRegistry.complete`，零额外配置）按内置路由规则在候选厂商中选一个，再按统一参数翻译后调用，`details` 记录实际 provider 与路由方式；选择结果按会话内 LRU 缓存，路由失败或被选厂商报错时按可用性顺序自动降级。`web_fetch` 固定调用 Exa 的 `web_fetch_exa` 提取正文（默认每页 3000 字符上限），失败时静默降级到 Keenable、再到 Tavily。 -Keenable 直接加载 npm 包的扩展和 skill，默认无需 key；设置 `KEENABLE_API_KEY` 可提高速率限制。它适合中文内容、`site` 限定和发布日期筛选。搜索结果保持紧凑，只对选中的 URL 调用 `keenable_fetch`。 - -`extensions/mcp.ts` 通过单个 `pi-mcp-adapter` 实例连接 Exa，避免多个 adapter 重复注册 `--mcp-config`、`/mcp` 和 `/mcp-auth`。Exa 连接 `https://mcp.exa.ai/mcp`，并把 MCP 原始工具映射为统一的来源前缀形式：`exa_web_search`、`exa_web_fetch` 和 `exa_web_search_advanced`。配置 `EXA_API_KEY` 后通过 `x-api-key` 请求头认证，不把 key 放进 URL。Exa 适合官方文档、技术资料、版本信息和论文；高级搜索必须显式限制结果数量与返回文本长度，避免响应过大。 - -根测试会扫描所有本地扩展入口，强制只有 `extensions/mcp.ts` 可以创建或导入 `pi-mcp-adapter`。新增 MCP 服务必须合并进这个共享实例，不能再增加独立 adapter 入口；隔离 Pi 加载仍作为发布前的运行时验证。 +`pi-web-search/clients/` 全部是纯 `fetch` 实现：Tavily 直连 `api.tavily.com/search|extract`（必须有 `TAVILY_API_KEY`，无 key 时不参与路由）；Keenable 复刻上游 client，默认走 keyless `/public` 端点，`KEENABLE_API_KEY` 只提高速率限制并保留 SSRF host 防护；Exa 直接对 `https://mcp.exa.ai/mcp` 发 JSON-RPC——initialize 一次并缓存 `mcp-session-id`、随后 `tools/call`，会话失效（HTTP 404）自动重建，key 只通过 `x-api-key` 请求头发送，不再依赖 `pi-mcp-adapter`。Exa 搜索统一走 `web_search_advanced_exa` 并显式限制结果数量与 `textMaxCharacters`，避免响应过大。 ### CodeGraph 是原生工具 diff --git a/config/pi-permission-policy.jsonc b/config/pi-permission-policy.jsonc index bde4c9a..20c64c3 100644 --- a/config/pi-permission-policy.jsonc +++ b/config/pi-permission-policy.jsonc @@ -136,11 +136,6 @@ "wget *": "ask", "printenv*": "ask" }, - "mcp": { - "*": "ask", - "mcp_status": "allow", - "mcp_list": "allow" - }, "skill": { "*": "allow" } diff --git a/extensions/exa-tool-prefix.ts b/extensions/exa-tool-prefix.ts deleted file mode 100644 index c8dae6c..0000000 --- a/extensions/exa-tool-prefix.ts +++ /dev/null @@ -1,57 +0,0 @@ -const EXA_TOOL_ALIASES: Record = { - web_search_exa: "exa_web_search", - web_fetch_exa: "exa_web_fetch", - web_search_advanced_exa: "exa_web_search_advanced", -}; - -export type ExaToolDefinition = { - name: string; - label: string; - [key: string]: unknown; -}; - -export function mapExaToolName(name: string): string { - return EXA_TOOL_ALIASES[name] ?? name; -} - -export function prefixExaTool(definition: T): T { - const name = mapExaToolName(definition.name); - if (name === definition.name) return definition; - - const displayName = name - .slice("exa_".length) - .split("_") - .map((part) => part.charAt(0).toUpperCase() + part.slice(1)) - .join(" "); - return { - ...definition, - name, - label: `Exa ${displayName}`, - } as T; -} - -type ToolApi = { - registerTool: (tool: unknown) => unknown; - setActiveTools: (names: string[]) => unknown; - unregisterTool?: (name: string) => boolean; -}; - -export function createExaToolApiOverride(pi: T): T { - const api = pi as T & ToolApi; - return new Proxy(pi, { - get(target, property) { - if (property === "registerTool") { - return (definition: ExaToolDefinition) => api.registerTool(prefixExaTool(definition)); - } - if (property === "setActiveTools") { - return (names: string[]) => api.setActiveTools([...new Set(names.map(mapExaToolName))]); - } - if (property === "unregisterTool") { - return api.unregisterTool ? (name: string) => api.unregisterTool?.(mapExaToolName(name)) : undefined; - } - - const value = Reflect.get(target, property, target); - return typeof value === "function" ? value.bind(target) : value; - }, - }) as T; -} diff --git a/extensions/mcp-config.ts b/extensions/mcp-config.ts deleted file mode 100644 index 38f080f..0000000 --- a/extensions/mcp-config.ts +++ /dev/null @@ -1,26 +0,0 @@ - -export const EXA_TOOLS = ["web_search_exa", "web_fetch_exa", "web_search_advanced_exa"]; -const EXA_MCP_URL = `https://mcp.exa.ai/mcp?tools=${EXA_TOOLS.join(",")}`; - -export function createMcpConfig(env: Record) { - const hasExaApiKey = Boolean(env.EXA_API_KEY?.trim()); - return { - settings: { - disableProxyTool: true, - scriptMode: false, - }, - mcpServers: { - exa: { - url: EXA_MCP_URL, - ...(hasExaApiKey ? { headers: { "x-api-key": "$env:EXA_API_KEY" } } : {}), - protocolVersion: "auto" as const, - lifecycle: "eager" as const, - directTools: EXA_TOOLS, - includeTools: EXA_TOOLS, - toolPrefix: "none" as const, - approveTools: false, - exposeResources: false, - }, - }, - }; -} diff --git a/extensions/mcp.ts b/extensions/mcp.ts deleted file mode 100644 index a688421..0000000 --- a/extensions/mcp.ts +++ /dev/null @@ -1,12 +0,0 @@ -import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; -import { createMcpAdapter } from "pi-mcp-adapter"; -import { createExaToolApiOverride } from "./exa-tool-prefix.ts"; -import { createMcpConfig } from "./mcp-config.ts"; -import { loadSearchConfigEnv } from "./search-config.ts"; - -/** Connect Pi to Exa's hosted MCP through the shared adapter instance. CodeGraph is a native tool (see codegraph.ts), not an MCP server. */ -export default function mcpExtension(pi: ExtensionAPI): void { - const searchEnvironment = loadSearchConfigEnv(); - const mcpAdapter = createMcpAdapter({ config: createMcpConfig(searchEnvironment) }); - mcpAdapter(createExaToolApiOverride(pi)); -} diff --git a/extensions/tavily-override.ts b/extensions/tavily-override.ts deleted file mode 100644 index a9011b8..0000000 --- a/extensions/tavily-override.ts +++ /dev/null @@ -1,21 +0,0 @@ -import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; -import tavilyExtension from "../node_modules/@tavily/pi-extension/index.ts"; -import { prefixTavilyTool, type TavilyToolDefinition } from "./tavily-tool-prefix.ts"; - - -/** Load Tavily while making the provider explicit in every registered tool name. */ -export default function tavilyOverrideExtension(pi: ExtensionAPI): void { - const prefixedPi = new Proxy(pi, { - get(target, property) { - if (property === "registerTool") { - return (definition: TavilyToolDefinition) => - (target.registerTool as (tool: unknown) => void)(prefixTavilyTool(definition)); - } - - const value = Reflect.get(target, property, target); - return typeof value === "function" ? value.bind(target) : value; - }, - }); - - tavilyExtension(prefixedPi); -} diff --git a/extensions/tavily-tool-prefix.ts b/extensions/tavily-tool-prefix.ts deleted file mode 100644 index e3dfea7..0000000 --- a/extensions/tavily-tool-prefix.ts +++ /dev/null @@ -1,25 +0,0 @@ -const TAVILY_TOOL_PREFIX = "tavily_"; - -export type TavilyToolDefinition = { - name: string; - label: string; - description: string; - promptSnippet?: string; - promptGuidelines?: string[]; - [key: string]: unknown; -}; - -function rewriteToolReferences(text: string): string { - return text.replace(/\bweb_search\b/g, "tavily_web_search").replace(/\bweb_fetch\b/g, "tavily_web_fetch"); -} - -export function prefixTavilyTool(definition: T): T { - return { - ...definition, - name: `${TAVILY_TOOL_PREFIX}${definition.name}`, - label: definition.label.startsWith("Tavily ") ? definition.label : `Tavily ${definition.label}`, - description: rewriteToolReferences(definition.description), - promptSnippet: definition.promptSnippet ? rewriteToolReferences(definition.promptSnippet) : undefined, - promptGuidelines: definition.promptGuidelines?.map(rewriteToolReferences), - } as T; -} diff --git a/extensions/tool-routing.ts b/extensions/tool-routing.ts index 33645d0..2cf6bb6 100644 --- a/extensions/tool-routing.ts +++ b/extensions/tool-routing.ts @@ -62,27 +62,9 @@ export function buildToolRoutingSection(selectedTools: SelectedTools): string { ); } - const tavilyTools = ["tavily_web_search", "tavily_web_fetch"]; - const exaTools = ["exa_web_search", "exa_web_fetch", "exa_web_search_advanced"]; - const keenableTools = ["keenable_search", "keenable_fetch"]; - if (hasAnyTool(selectedTools, [...tavilyTools, ...exaTools, ...keenableTools])) { + if (hasAnyTool(selectedTools, ["web_search", "web_fetch"])) { rules.push( - "- Choose one online search provider based on the task instead of querying Tavily, Exa, and Keenable by default; cross-check with another provider only when source quality or uncertainty warrants it.", - ); - } - if (hasAnyTool(selectedTools, tavilyTools)) { - rules.push( - "- Use Tavily tools for broad discovery when keywords are uncertain, for news/recent/trending topics, or when many candidate sources are useful. Tavily results can mix official pages, blogs, forums, and secondary sources, so keep max_results modest, avoid raw content unless needed, and fetch only selected pages when depth matters.", - ); - } - if (hasAnyTool(selectedTools, exaTools)) { - rules.push( - "- Use Exa tools when source precision matters: official or API documentation, release/version information, academic papers, research material, and high-quality fact confirmation. Use exa_web_fetch to continue reading selected pages; for exa_web_search_advanced, explicitly bound the result count and returned text length to prevent oversized responses.", - ); - } - if (hasAnyTool(selectedTools, keenableTools)) { - rules.push( - "- Use Keenable tools for Chinese-language pages, policies and government notices, site-restricted search, and publication-date filtering. Prefer its compact search results and fetch only selected pages to keep returned content controlled; KEENABLE_API_KEY is optional but raises rate limits.", + "- Use web_search for current, real-world, or citable web information; provider selection is handled internally. Keep queries focused and use site/published_* filters only when they materially narrow results. Then read selected pages with web_fetch instead of repeating similar searches.", ); } diff --git a/package-lock.json b/package-lock.json index 44fd42a..05925fd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,10 +10,8 @@ "dependencies": { "@ff-labs/pi-fff": "0.10.5", "@gotgenes/pi-permission-system": "file:./pi-permission-system", - "@keenable/pi-search": "0.1.2", "@ogulcancelik/pi-minimal-footer": "file:./pi-minimal-footer", "@smoose/pi-notify": "file:./pi-notify", - "@tavily/pi-extension": "0.1.2", "context-mode": "1.0.169", "jiti": "2.7.0", "pi-ask-user": "file:./pi-ask-user", @@ -22,9 +20,9 @@ "pi-hashline-edit": "0.8.3", "pi-hermes-memory": "file:./pi-hermes-memory", "pi-lsp": "0.1.7", - "pi-mcp-adapter": "2.26.0", "pi-ssh": "file:./pi-ssh", "pi-tool-search": "file:./pi-tool-search", + "pi-web-search": "file:./pi-web-search", "pi-worktree": "file:./pi-worktree", "ssh2": "1.17.0", "typescript": "6.0.3", @@ -3340,82 +3338,12 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, - "node_modules/@keenable/pi-search": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/@keenable/pi-search/-/pi-search-0.1.2.tgz", - "integrity": "sha512-3akVIB+0vgaSdsqwFZ46nTHOnjEywgKEkho/l6ixVkJS07T8A8W8gLsg26L526DlaZo+EMW+44EbZASTfKZU6A==", - "license": "MIT", - "peerDependencies": { - "@earendil-works/pi-ai": "*", - "@earendil-works/pi-coding-agent": "*", - "typebox": "*" - } - }, "node_modules/@mixmark-io/domino": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/@mixmark-io/domino/-/domino-2.2.0.tgz", "integrity": "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==", "license": "BSD-2-Clause" }, - "node_modules/@modelcontextprotocol/client": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@modelcontextprotocol/client/-/client-2.0.0.tgz", - "integrity": "sha512-8f1OghQ2rjzIOfqgUCP+8GiUWqRs89njoWLNqAe8kWmDePv3s1fZXseej+QXemssEuuOvLLmLO/kqM3IQHtISw==", - "license": "MIT", - "dependencies": { - "@modelcontextprotocol/core": "2.0.0", - "cross-spawn": "^7.0.5", - "eventsource": "^3.0.2", - "eventsource-parser": "^3.0.0", - "jose": "^6.1.3", - "pkce-challenge": "^5.0.0", - "zod": "^4.2.0" - }, - "engines": { - "node": ">=20" - } - }, - "node_modules/@modelcontextprotocol/core": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@modelcontextprotocol/core/-/core-2.0.0.tgz", - "integrity": "sha512-pJCEwGG7Lfr/+PQp9ZTwKXNeO5wzbfKL7H3MYpCorM4oFBoQrdjnBgEoqG+RjhsvS1FKrDbKux+M1HhlnGWqcA==", - "license": "MIT", - "dependencies": { - "zod": "^4.2.0" - }, - "engines": { - "node": ">=20" - } - }, - "node_modules/@modelcontextprotocol/ext-apps": { - "version": "1.7.5", - "resolved": "https://registry.npmjs.org/@modelcontextprotocol/ext-apps/-/ext-apps-1.7.5.tgz", - "integrity": "sha512-TjPH2S2y5UEGKhmI6+XGFuqfqOV4ppe1x6DA3txnUaEWkgtA4G5vo14jGKFZmegdkZ1H4QMLyujLvoU1BEdnAg==", - "license": "MIT", - "workspaces": [ - "examples/*" - ], - "dependencies": { - "@standard-schema/spec": "^1.1.0" - }, - "engines": { - "node": ">=20" - }, - "peerDependencies": { - "@modelcontextprotocol/sdk": "^1.29.0", - "react": "^17.0.0 || ^18.0.0 || ^19.0.0", - "react-dom": "^17.0.0 || ^18.0.0 || ^19.0.0", - "zod": "^3.25.0 || ^4.0.0" - }, - "peerDependenciesMeta": { - "react": { - "optional": true - }, - "react-dom": { - "optional": true - } - } - }, "node_modules/@modelcontextprotocol/sdk": { "version": "1.30.0", "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.30.0.tgz", @@ -3456,225 +3384,6 @@ } } }, - "node_modules/@napi-rs/keyring": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@napi-rs/keyring/-/keyring-1.3.0.tgz", - "integrity": "sha512-WrOw/bcXm0f9qHkumlT1QlArXSTWqaY9sunsDpOk+yCCorCKMxvWT/a3xko4EYHVdeZoh00yI2TydXn6eyICDA==", - "license": "MIT", - "engines": { - "node": ">= 10" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/Brooooooklyn" - }, - "optionalDependencies": { - "@napi-rs/keyring-darwin-arm64": "1.3.0", - "@napi-rs/keyring-darwin-x64": "1.3.0", - "@napi-rs/keyring-freebsd-x64": "1.3.0", - "@napi-rs/keyring-linux-arm-gnueabihf": "1.3.0", - "@napi-rs/keyring-linux-arm64-gnu": "1.3.0", - "@napi-rs/keyring-linux-arm64-musl": "1.3.0", - "@napi-rs/keyring-linux-riscv64-gnu": "1.3.0", - "@napi-rs/keyring-linux-x64-gnu": "1.3.0", - "@napi-rs/keyring-linux-x64-musl": "1.3.0", - "@napi-rs/keyring-win32-arm64-msvc": "1.3.0", - "@napi-rs/keyring-win32-ia32-msvc": "1.3.0", - "@napi-rs/keyring-win32-x64-msvc": "1.3.0" - } - }, - "node_modules/@napi-rs/keyring-darwin-arm64": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@napi-rs/keyring-darwin-arm64/-/keyring-darwin-arm64-1.3.0.tgz", - "integrity": "sha512-pl76hJvdYUBn6I24bXiOBMA9nbDapo3I5B+f3OorjDU4dUMSypXeKbOVehJe8fhgTiH24flMyTS3aAIy43xegQ==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/keyring-darwin-x64": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@napi-rs/keyring-darwin-x64/-/keyring-darwin-x64-1.3.0.tgz", - "integrity": "sha512-YcJtEV5LA3cvA4z3BurgxH5IhTsW1JfIvcAAcqcecwk06Si9F9NqkxbZVIfDwQ8oRHgaBmT3zZJnLAotCrVahw==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/keyring-freebsd-x64": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@napi-rs/keyring-freebsd-x64/-/keyring-freebsd-x64-1.3.0.tgz", - "integrity": "sha512-vlLf31TGhfRAaxLDBhg8b89ss0HHD/lyNmL5F3UjSaz5CUXElsJmKYq9fqA/B+cZKUEUcLHHGhF0I/CqcFdaVw==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/keyring-linux-arm-gnueabihf": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@napi-rs/keyring-linux-arm-gnueabihf/-/keyring-linux-arm-gnueabihf-1.3.0.tgz", - "integrity": "sha512-KiWdMMu/Inz/bHHIAGrnF7r54FZDYXuHO6UFF/rhIrshUsxbMG1Rl9lEymNtqqsVo927G0VYcb02FzWQ3iBQRQ==", - "cpu": [ - "arm" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/keyring-linux-arm64-gnu": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@napi-rs/keyring-linux-arm64-gnu/-/keyring-linux-arm64-gnu-1.3.0.tgz", - "integrity": "sha512-eyKGpY40lm9Jvs1aD294XRH4y7+TlJM0YVAryZeXA6TX0mb4gMkxVXwSQv7MCwgah7raeUd0dKUb4BPAYIgcMg==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/keyring-linux-arm64-musl": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@napi-rs/keyring-linux-arm64-musl/-/keyring-linux-arm64-musl-1.3.0.tgz", - "integrity": "sha512-iIK6JWHXAJqDrEyLY3TmswwloVyt2vj+04TZnew+uSJ9gnDO8EwRbp3/iw3LpWaXiDO7VomGO6y8I0Id8uBZSw==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/keyring-linux-riscv64-gnu": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@napi-rs/keyring-linux-riscv64-gnu/-/keyring-linux-riscv64-gnu-1.3.0.tgz", - "integrity": "sha512-/PGqrwn6EwgtK6vccASSXJRfOSP4vN1F4ASsIQ+7MdrK6hNvAJ1FZPrIuD5gGGdxezo3F++To2Wq7DbuGIeuNQ==", - "cpu": [ - "riscv64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/keyring-linux-x64-gnu": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@napi-rs/keyring-linux-x64-gnu/-/keyring-linux-x64-gnu-1.3.0.tgz", - "integrity": "sha512-2PDK1WKWTu9lBGq9VvNEkSlQD3O7YwVpmnyN2M3cy4v7NJ/8gDMd9GXv3G+FVXN13uhp4gnnPBS+ScefmEeD2A==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/keyring-linux-x64-musl": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@napi-rs/keyring-linux-x64-musl/-/keyring-linux-x64-musl-1.3.0.tgz", - "integrity": "sha512-oJ2HkX8YUo46QBkn0pG+HuIKQNqr523q6vBobCn+P95s4C4K6/kLBqHY/1bg5J4ap31DzsznhnFKcfBNBsjCnw==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/keyring-win32-arm64-msvc": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@napi-rs/keyring-win32-arm64-msvc/-/keyring-win32-arm64-msvc-1.3.0.tgz", - "integrity": "sha512-tOd3c/uAaeoE4ycVlmAdSvygz0Zt3zdca6Y7gokBeIbaRDWpjDIUOpU3MvML59XAaqyuKGsVVu0F/DZb1lHPmw==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/keyring-win32-ia32-msvc": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@napi-rs/keyring-win32-ia32-msvc/-/keyring-win32-ia32-msvc-1.3.0.tgz", - "integrity": "sha512-sPSqeAFZMGqP1R++M2JTza7GQJJ/TpCo6JU6Vcd4jnebvOaEDs9b7eipakU1PJdSvhpC2yXMCNRk9gXfrhuwHQ==", - "cpu": [ - "ia32" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/keyring-win32-x64-msvc": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@napi-rs/keyring-win32-x64-msvc/-/keyring-win32-x64-msvc-1.3.0.tgz", - "integrity": "sha512-4DnCWXwDc0HRKwyRlG5y0VhKZW2tNRQfKKfyj6IX/KWfDNyq9hn4n+GL1auyDcOO/v8PwnhmYo2+rOOqCkvvOg==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } - }, "node_modules/@ogulcancelik/pi-minimal-footer": { "resolved": "pi-minimal-footer", "link": true @@ -3699,18 +3408,6 @@ "url": "https://github.com/sponsors/Boshen" } }, - "node_modules/@pkgr/core": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/@pkgr/core/-/core-0.1.2.tgz", - "integrity": "sha512-fdDH1LSGfZdTH2sxdpVMw31BanV28K/Gry0cVFxaNP77neJSkd82mM8ErPNYs9e+0O7SdHBLTDzDgwUuy18RnQ==", - "license": "MIT", - "engines": { - "node": "^12.20.0 || ^14.18.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/unts" - } - }, "node_modules/@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -4565,31 +4262,9 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "dev": true, "license": "MIT" }, - "node_modules/@tavily/core": { - "version": "0.7.7", - "resolved": "https://registry.npmjs.org/@tavily/core/-/core-0.7.7.tgz", - "integrity": "sha512-6Sc+u4nPEdzLB2D7Ubut/sLI2wR3Wc81P1VCJNTsdFtvgLOpVGGGEYAG9VUCLOzVCeEUT5SzHRIMSnsqJ4Pdow==", - "license": "MIT", - "dependencies": { - "axios": "^1.7.7", - "https-proxy-agent": "^7.0.6", - "js-tiktoken": "^1.0.14" - } - }, - "node_modules/@tavily/pi-extension": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/@tavily/pi-extension/-/pi-extension-0.1.2.tgz", - "integrity": "sha512-12RFWh1ni7rvr+jIHOu1uKpidWeBIVgAMeOvWayUxejab5CQIRchz+83o0f1Fw0wIOH55cziCynO1nOSX2s95Q==", - "dependencies": { - "@tavily/core": "^0.7.3" - }, - "peerDependencies": { - "@earendil-works/pi-ai": "*", - "@earendil-works/pi-coding-agent": "*" - } - }, "node_modules/@tokenizer/inflate": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/@tokenizer/inflate/-/inflate-0.4.1.tgz", @@ -4972,6 +4647,7 @@ "version": "7.1.4", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "dev": true, "license": "MIT", "engines": { "node": ">= 14" @@ -5041,49 +4717,6 @@ "node": ">=12" } }, - "node_modules/asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "license": "MIT" - }, - "node_modules/axios": { - "version": "1.19.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.19.0.tgz", - "integrity": "sha512-ht/iuYZXEjFxLH/Hkezgd7m6JKlHHXEUSneaDz8uZe1Gj5QZtCnpyDsckvAiEnT89OEbCLmnte4R4sn7P0EKFw==", - "license": "MIT", - "dependencies": { - "follow-redirects": "^1.16.0", - "form-data": "^4.0.6", - "https-proxy-agent": "^5.0.1", - "proxy-from-env": "^2.1.0" - } - }, - "node_modules/axios/node_modules/agent-base": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", - "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", - "license": "MIT", - "dependencies": { - "debug": "4" - }, - "engines": { - "node": ">= 6.0.0" - } - }, - "node_modules/axios/node_modules/https-proxy-agent": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", - "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", - "license": "MIT", - "dependencies": { - "agent-base": "6", - "debug": "4" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -5241,21 +4874,6 @@ "node": ">=10.0.0" } }, - "node_modules/bundle-name": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/bundle-name/-/bundle-name-4.1.0.tgz", - "integrity": "sha512-tjwM5exMg6BGRI+kNmTntNsvdZS1X8BFYS6tnJ2hdH0kVxM6/eVZ2xy+FqStSWvYmtfFMDLIxurorHwDKfDz5Q==", - "license": "MIT", - "dependencies": { - "run-applescript": "^7.0.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -5310,18 +4928,6 @@ "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", "license": "ISC" }, - "node_modules/combined-stream": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", - "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "license": "MIT", - "dependencies": { - "delayed-stream": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, "node_modules/content-disposition": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", @@ -5503,55 +5109,6 @@ "node": ">=4.0.0" } }, - "node_modules/default-browser": { - "version": "5.5.1", - "resolved": "https://registry.npmjs.org/default-browser/-/default-browser-5.5.1.tgz", - "integrity": "sha512-m1pAzaJgZ/gssEqlOhJkPJp8Xly7QyW6xcrkUa2KKcDeDSEMP7X8xipU3snUcfisTQx0w1AGae+9UtJSfVnXGw==", - "license": "MIT", - "dependencies": { - "bundle-name": "^4.1.0", - "default-browser-id": "^5.0.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/default-browser-id": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/default-browser-id/-/default-browser-id-5.0.1.tgz", - "integrity": "sha512-x1VCxdX4t+8wVfd1so/9w+vQ4vx7lKd2Qp5tDRutErwmR85OgmfX7RlLRMWafRMY7hbEiXIbudNrjOAPa/hL8Q==", - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/define-lazy-prop": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-3.0.0.tgz", - "integrity": "sha512-N+MeXYoqr3pOgn8xfyRPREN7gHakLYjhsHhWGT3fWAiL4IkAt0iDw14QiiEm2bE30c5XX5q0FtAA3CK5f9/BUg==", - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "license": "MIT", - "engines": { - "node": ">=0.4.0" - } - }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -5664,21 +5221,6 @@ "node": ">= 0.4" } }, - "node_modules/es-set-tostringtag": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", - "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.6", - "has-tostringtag": "^1.0.2", - "hasown": "^2.0.2" - }, - "engines": { - "node": ">= 0.4" - } - }, "node_modules/esbuild": { "version": "0.28.2", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", @@ -6007,63 +5549,6 @@ "url": "https://opencollective.com/express" } }, - "node_modules/follow-redirects": { - "version": "1.16.0", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz", - "integrity": "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==", - "funding": [ - { - "type": "individual", - "url": "https://github.com/sponsors/RubenVerborgh" - } - ], - "license": "MIT", - "engines": { - "node": ">=4.0" - }, - "peerDependenciesMeta": { - "debug": { - "optional": true - } - } - }, - "node_modules/form-data": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.6.tgz", - "integrity": "sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==", - "license": "MIT", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.4", - "mime-types": "^2.1.35" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/form-data/node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/form-data/node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "license": "MIT", - "dependencies": { - "mime-db": "1.52.0" - }, - "engines": { - "node": ">= 0.6" - } - }, "node_modules/formdata-polyfill": { "version": "4.0.10", "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", @@ -6263,21 +5748,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/has-tostringtag": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", - "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", - "license": "MIT", - "dependencies": { - "has-symbols": "^1.0.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/hasown": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", @@ -6337,6 +5807,7 @@ "version": "7.0.6", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "dev": true, "license": "MIT", "dependencies": { "agent-base": "^7.1.2", @@ -6412,60 +5883,12 @@ "node": ">= 0.10" } }, - "node_modules/is-docker": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-3.0.0.tgz", - "integrity": "sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ==", - "license": "MIT", - "bin": { - "is-docker": "cli.js" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/is-inside-container": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-inside-container/-/is-inside-container-1.0.0.tgz", - "integrity": "sha512-KIYLCCJghfHZxqjYBE7rEy0OBuTd5xCHS7tHVgvCLkx7StIoaxwNW3hCALgEUjFfeRk+MG/Qxmp/vtETEF3tRA==", - "license": "MIT", - "dependencies": { - "is-docker": "^3.0.0" - }, - "bin": { - "is-inside-container": "cli.js" - }, - "engines": { - "node": ">=14.16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/is-promise": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", "license": "MIT" }, - "node_modules/is-wsl": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-3.1.1.tgz", - "integrity": "sha512-e6rvdUCiQCAuumZslxRJWR/Doq4VpPR82kqclvcS0efgt430SlGIk05vdCN58+VrzgtIcfNODjozVielycD4Sw==", - "license": "MIT", - "dependencies": { - "is-inside-container": "^1.0.0" - }, - "engines": { - "node": ">=16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -6490,15 +5913,6 @@ "url": "https://github.com/sponsors/panva" } }, - "node_modules/js-tiktoken": { - "version": "1.0.21", - "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.21.tgz", - "integrity": "sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g==", - "license": "MIT", - "dependencies": { - "base64-js": "^1.5.1" - } - }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -7130,24 +6544,6 @@ "wrappy": "1" } }, - "node_modules/open": { - "version": "10.2.0", - "resolved": "https://registry.npmjs.org/open/-/open-10.2.0.tgz", - "integrity": "sha512-YgBpdJHPyQ2UE5x+hlSXcnejzAvD0b22U2OuAP+8OnlJT+PjWPxtgmGqKKc+RgTM63U9gN0YzrYc71R2WT/hTA==", - "license": "MIT", - "dependencies": { - "default-browser": "^5.2.1", - "define-lazy-prop": "^3.0.0", - "is-inside-container": "^1.0.0", - "wsl-utils": "^0.1.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/openai": { "version": "6.40.0", "resolved": "https://registry.npmjs.org/openai/-/openai-6.40.0.tgz", @@ -7274,49 +6670,6 @@ "typebox": "*" } }, - "node_modules/pi-mcp-adapter": { - "version": "2.26.0", - "resolved": "https://registry.npmjs.org/pi-mcp-adapter/-/pi-mcp-adapter-2.26.0.tgz", - "integrity": "sha512-S1L1cTvYdDUIkNGvg/dR1v3ljfxeq6nAA8ffnB+xB5nZH4DSU8Ks9KdmKgmEH4TAEsSxGgv194i6WYORHF0Clg==", - "license": "MIT", - "dependencies": { - "@modelcontextprotocol/client": "2.0.0", - "@modelcontextprotocol/core": "2.0.0", - "@modelcontextprotocol/ext-apps": "^1.2.2", - "@napi-rs/keyring": "^1.3.0", - "ajv": "^8.17.1", - "ajv-formats": "^3.0.1", - "cross-spawn": "^7.0.6", - "open": "^10.2.0", - "recheck": "^4.5.0", - "smol-toml": "^1.6.1", - "strip-json-comments": "^5.0.3", - "zod": "^3.25.0 || ^4.0.0" - }, - "bin": { - "pi-mcp-adapter": "cli.js" - }, - "engines": { - "node": ">=20" - }, - "peerDependencies": { - "@earendil-works/pi-ai": "^0.84.1", - "@earendil-works/pi-tui": "*", - "typebox": "*", - "zod": "^3.25.0 || ^4.0.0" - }, - "peerDependenciesMeta": { - "@earendil-works/pi-ai": { - "optional": true - }, - "@earendil-works/pi-tui": { - "optional": true - }, - "typebox": { - "optional": true - } - } - }, "node_modules/pi-ssh": { "resolved": "pi-ssh", "link": true @@ -7325,6 +6678,10 @@ "resolved": "pi-tool-search", "link": true }, + "node_modules/pi-web-search": { + "resolved": "pi-web-search", + "link": true + }, "node_modules/pi-worktree": { "resolved": "pi-worktree", "link": true @@ -7450,15 +6807,6 @@ "node": ">= 0.10" } }, - "node_modules/proxy-from-env": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-2.1.0.tgz", - "integrity": "sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==", - "license": "MIT", - "engines": { - "node": ">=10" - } - }, "node_modules/pump": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", @@ -7551,84 +6899,6 @@ "node": ">= 6" } }, - "node_modules/recheck": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/recheck/-/recheck-4.5.0.tgz", - "integrity": "sha512-kPnbOV6Zfx9a25AZ++28fI1q78L/UVRQmmuazwVRPfiiqpMs+WbOU69Shx820XgfKWfak0JH75PUvZMFtRGSsw==", - "license": "MIT", - "dependencies": { - "synckit": "0.9.2" - }, - "engines": { - "node": ">=20" - }, - "optionalDependencies": { - "recheck-jar": "4.5.0", - "recheck-linux-x64": "4.5.0", - "recheck-macos-arm64": "4.5.0", - "recheck-macos-x64": "4.5.0", - "recheck-windows-x64": "4.5.0" - } - }, - "node_modules/recheck-jar": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/recheck-jar/-/recheck-jar-4.5.0.tgz", - "integrity": "sha512-Ad7oCQmY8cQLzd3QVNXjzZ+S6MbImGhR4AaW2yiGzteOfMV45522rt6nSzFyt8p3mCEaMcm/4MoZrMSxUcCbrA==", - "license": "MIT", - "optional": true - }, - "node_modules/recheck-linux-x64": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/recheck-linux-x64/-/recheck-linux-x64-4.5.0.tgz", - "integrity": "sha512-52kXsR/v+IbGIKYYFZfSZcgse/Ci9IA2HnuzrtvRRcfODkcUGe4n72ESQ8nOPwrdHFg9i4j9/YyPh1HWWgpJ6A==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/recheck-macos-arm64": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/recheck-macos-arm64/-/recheck-macos-arm64-4.5.0.tgz", - "integrity": "sha512-qIyK3dRuLkORQvv0b59fZZRXweSmjjWaoA4K8Kgifz0anMBH4pqsDV6plBlgjcRmW9yC12wErIRzifREaKnk2w==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ] - }, - "node_modules/recheck-macos-x64": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/recheck-macos-x64/-/recheck-macos-x64-4.5.0.tgz", - "integrity": "sha512-1wp/eiLxcjC/Ex4wurlrS/LGzt8IiF4TiK5sEjldu4HVAKdNCnnmsS9a5vFpfcikDz4ZuZlLlTi1VbQTxHlwZg==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ] - }, - "node_modules/recheck-windows-x64": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/recheck-windows-x64/-/recheck-windows-x64-4.5.0.tgz", - "integrity": "sha512-ekBKwAp0oKkMULn5zgmHEYLwSJfkfb95AbTtbDkQazNkqYw9PRD/mVyFUR6Ff2IeRyZI0gxy+N2AKBISWydhug==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ] - }, "node_modules/require-from-string": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", @@ -7768,18 +7038,6 @@ "node": ">= 18" } }, - "node_modules/run-applescript": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/run-applescript/-/run-applescript-7.1.0.tgz", - "integrity": "sha512-DPe5pVFaAsinSaV6QjQ6gdiedWDcRCbUuiQfQa2wmWV7+xC9bGulGI8+TdRmoFkAPaBXk8CrAbnlY2ISniJ47Q==", - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -8020,18 +7278,6 @@ "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==", "license": "MIT" }, - "node_modules/smol-toml": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.8.0.tgz", - "integrity": "sha512-kCZr2V3ch9i00x8zXRhjUNVcjG9ijES5dDudkXvUVCT5QlJNQWElSJdZqyPemffHoLNUYwOcou0Fy+ojN0uHSQ==", - "license": "BSD-3-Clause", - "engines": { - "node": ">= 18" - }, - "funding": { - "url": "https://github.com/sponsors/cyyynthia" - } - }, "node_modules/source-map-js": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", @@ -8106,18 +7352,6 @@ "url": "https://github.com/chalk/strip-ansi?sponsor=1" } }, - "node_modules/strip-json-comments": { - "version": "5.0.3", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-5.0.3.tgz", - "integrity": "sha512-1tB5mhVo7U+ETBKNf92xT4hrQa3pm0MZ0PQvuDnWgAAGHDsfp4lPSpiS6psrSiet87wyGPh9ft6wmhOMQ0hDiw==", - "license": "MIT", - "engines": { - "node": ">=14.16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/strtok3": { "version": "10.3.5", "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-10.3.5.tgz", @@ -8134,22 +7368,6 @@ "url": "https://github.com/sponsors/Borewit" } }, - "node_modules/synckit": { - "version": "0.9.2", - "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.9.2.tgz", - "integrity": "sha512-vrozgXDQwYO72vHjUb/HnFbQx1exDjoKzqx23aXEg2a9VIg2TSFZ8FmeZpTjUCFMYw7mpX4BE2SFu8wI7asYsw==", - "license": "MIT", - "dependencies": { - "@pkgr/core": "^0.1.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": "^14.18.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/unts" - } - }, "node_modules/tar-fs": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.5.tgz", @@ -8279,6 +7497,7 @@ "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true, "license": "0BSD" }, "node_modules/tsx": { @@ -8721,21 +7940,6 @@ } } }, - "node_modules/wsl-utils": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/wsl-utils/-/wsl-utils-0.1.0.tgz", - "integrity": "sha512-h3Fbisa2nKGPxCpm89Hk33lBLsnaGBvctQopaBSOW/uIs6FTe1ATyAnKFJrzVs9vpGdsTe73WF3V4lIsk4Gacw==", - "license": "MIT", - "dependencies": { - "is-wsl": "^3.1.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/xxhashjs": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/xxhashjs/-/xxhashjs-0.2.2.tgz", @@ -8947,6 +8151,25 @@ } } }, + "pi-web-search": { + "version": "0.1.0", + "license": "MIT", + "engines": { + "node": ">=22.19.0" + }, + "peerDependencies": { + "@earendil-works/pi-coding-agent": "*", + "typebox": "*" + }, + "peerDependenciesMeta": { + "@earendil-works/pi-coding-agent": { + "optional": true + }, + "typebox": { + "optional": true + } + } + }, "pi-worktree": { "version": "0.1.0", "license": "MIT", diff --git a/package.json b/package.json index b87ed13..15fa1ad 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "pi-tool-search/CHANGELOG.md", "pi-tool-search/LICENSE", "pi-tool-search/UPSTREAM.md", + "pi-web-search", "pi-worktree", "pi-permission-system/LICENSES", "pi-permission-system/docs/auto-review", @@ -50,9 +51,7 @@ "extensions": [ "./extensions/search-config.ts", "./extensions/fff-override.ts", - "./extensions/tavily-override.ts", - "./node_modules/@keenable/pi-search/src/index.ts", - "./extensions/mcp.ts", + "./pi-web-search/index.ts", "./extensions/codegraph.ts", "./pi-rtk-optimizer/index.ts", "./extensions/hashline.ts", @@ -73,17 +72,14 @@ "./pi-tool-search/extensions/index.ts" ], "skills": [ - "./node_modules/context-mode/skills", - "./node_modules/@keenable/pi-search/skills" + "./node_modules/context-mode/skills" ] }, "dependencies": { "@ff-labs/pi-fff": "0.10.5", "@gotgenes/pi-permission-system": "file:./pi-permission-system", - "@keenable/pi-search": "0.1.2", "@ogulcancelik/pi-minimal-footer": "file:./pi-minimal-footer", "@smoose/pi-notify": "file:./pi-notify", - "@tavily/pi-extension": "0.1.2", "context-mode": "1.0.169", "jiti": "2.7.0", "pi-ask-user": "file:./pi-ask-user", @@ -92,9 +88,9 @@ "pi-hashline-edit": "0.8.3", "pi-hermes-memory": "file:./pi-hermes-memory", "pi-lsp": "0.1.7", - "pi-mcp-adapter": "2.26.0", "pi-ssh": "file:./pi-ssh", "pi-tool-search": "file:./pi-tool-search", + "pi-web-search": "file:./pi-web-search", "pi-worktree": "file:./pi-worktree", "ssh2": "1.17.0", "typescript": "6.0.3", diff --git a/pi-tool-search/README.md b/pi-tool-search/README.md index 990eb63..d6b0866 100644 --- a/pi-tool-search/README.md +++ b/pi-tool-search/README.md @@ -32,9 +32,7 @@ Run `/tool-search-status` to inspect the current catalog source, pinned count, d | `ssh-remote-search` | `ssh_find`, `ssh_grep` | | `ssh-remote-shell` | `ssh_cd`, `ssh_bash` | | `code-intelligence` | CodeGraph and all LSP tools | -| `web-tavily` | Tavily search/fetch | -| `web-exa` | Exa search/advanced/fetch | -| `web-keenable` | Keenable search/fetch | +| `web-search` | `web_search`, `web_fetch` (pi-web-search unified tools) | | `context-execution` | `ctx_execute`, `ctx_execute_file`, `ctx_batch_execute` | | `context-knowledge` | Context indexing, searching, and web-doc ingestion | | `context-observability` | Context stats, doctor, and Insight | @@ -55,7 +53,7 @@ Pinned tools are removed from these dynamic groups at runtime. The defaults pin Call `tool_search` with an exact catalog group id when possible: ```json -{ "group": "web-tavily" } +{ "group": "web-search" } ``` A workflow can atomically activate up to five exact groups in one call. The complete request must fit the configured group and tool limits; otherwise no active state is changed: @@ -114,7 +112,7 @@ Example override: ```json { "groupOverrides": { - "web-tavily": ["tavily_web_search", "tavily_web_fetch"], + "web-search": ["web_search", "web_fetch"], "lsp": ["lsp_definition", "lsp_references", "lsp_hover", "lsp_symbols", "lsp_diagnostics"] } } diff --git a/pi-tool-search/extensions/bundle-groups.ts b/pi-tool-search/extensions/bundle-groups.ts index 6175789..6a65591 100644 --- a/pi-tool-search/extensions/bundle-groups.ts +++ b/pi-tool-search/extensions/bundle-groups.ts @@ -77,31 +77,13 @@ export const BUNDLE_GROUP_DEFINITIONS: BundleGroupDefinition[] = [ tools: ["codegraph_explore", "lsp_definition", "lsp_references", "lsp_hover", "lsp_symbols", "lsp_diagnostics"], }, { - id: "web-tavily", - title: "Tavily web discovery", - summary: "Discover broad, current, or news-oriented web sources with Tavily and fetch selected pages.", - useWhen: ["Keywords are uncertain, many candidate sources help, or current/news coverage is needed"], - avoidWhen: ["Official technical sources or Chinese site/date constraints are more important"], - keywords: ["tavily", "web", "news", "discovery", "网页", "新闻", "广泛搜索"], - tools: ["tavily_web_search", "tavily_web_fetch"], - }, - { - id: "web-exa", - title: "Exa precision research", - summary: "Find precise official, technical, academic, company, or API sources with Exa and read selected pages.", - useWhen: ["Source precision, official documentation, papers, releases, or advanced filters matter"], - avoidWhen: ["Broad news discovery or Chinese policy search is the primary need"], - keywords: ["exa", "official docs", "paper", "api", "research", "官方文档", "论文", "精确搜索"], - tools: ["exa_web_search", "exa_web_search_advanced", "exa_web_fetch"], - }, - { - id: "web-keenable", - title: "Keenable focused search", - summary: "Search Chinese-language, site-restricted, policy, government, or date-filtered sources and fetch selected pages.", - useWhen: ["Chinese pages, policies, government notices, domain restrictions, or publication dates matter"], - avoidWhen: ["Broad international discovery or academic semantic search is a better fit"], - keywords: ["keenable", "Chinese", "policy", "site", "date", "中文", "政策", "站点", "日期"], - tools: ["keenable_search", "keenable_fetch"], + id: "web-search", + title: "Web search and fetch", + summary: "Search the web for current or citable information and read selected pages as clean markdown.", + useWhen: ["Current events, external facts, documentation, papers, or page content are needed"], + avoidWhen: ["The answer is already in the workspace or local code tools suffice"], + keywords: ["web", "search", "fetch", "news", "docs", "网页", "搜索", "新闻", "文档"], + tools: ["web_search", "web_fetch"], }, { id: "context-execution", diff --git a/pi-tool-search/test/bundle-groups.test.ts b/pi-tool-search/test/bundle-groups.test.ts index 7c0e91e..86604f4 100644 --- a/pi-tool-search/test/bundle-groups.test.ts +++ b/pi-tool-search/test/bundle-groups.test.ts @@ -26,7 +26,7 @@ test("bundle seed names are unique and cover the declared my-pi tools", () => { assert.equal(result.coveredNames.size, names.length); assert.equal(result.catalog.tools.length, names.length); assert.equal(result.catalog.tools.find((card) => card.name === "ctx_purge")?.primaryGroup, "context-administration"); - assert.equal(result.catalog.tools.find((card) => card.name === "tavily_web_fetch")?.primaryGroup, "web-tavily"); + assert.equal(result.catalog.tools.find((card) => card.name === "web_fetch")?.primaryGroup, "web-search"); assert.equal(result.catalog.tools.find((card) => card.name === "memory_search")?.primaryGroup, "memory-recall"); assert.equal(result.catalog.tools.find((card) => card.name === "session_search")?.primaryGroup, "memory-recall"); assert.equal(result.catalog.tools.find((card) => card.name === "memory_add")?.primaryGroup, "memory-management"); @@ -51,11 +51,11 @@ test("bundle seed names are unique and cover the declared my-pi tools", () => { }); test("user overrides take priority and unknown tools retain deterministic fallback groups", () => { - const tools = [tool("tavily_web_search"), tool("custom_analyze")]; - const configured = { maxToolsPerGroup: 8, groupOverrides: { preferred: ["tavily_web_search"] } }; + const tools = [tool("web_search"), tool("custom_analyze")]; + const configured = { maxToolsPerGroup: 8, groupOverrides: { preferred: ["web_search"] } }; const hash = buildManifestHash(tools, configured); const result = createBundleCatalog(tools, hash, configured); assert.deepEqual(result.unknownTools.map((item) => item.name), ["custom_analyze"]); - assert.equal(result.catalog.tools.find((card) => card.name === "tavily_web_search")?.primaryGroup, "preferred"); + assert.equal(result.catalog.tools.find((card) => card.name === "web_search")?.primaryGroup, "preferred"); assert.equal(result.catalog.tools.find((card) => card.name === "custom_analyze")?.primaryGroup, "custom"); }); diff --git a/pi-tool-search/test/extension.test.ts b/pi-tool-search/test/extension.test.ts index 56db30f..9068fd2 100644 --- a/pi-tool-search/test/extension.test.ts +++ b/pi-tool-search/test/extension.test.ts @@ -174,8 +174,8 @@ test("builds and caches a validated catalog with the current model on first sear const prompts: string[] = []; const sourceTools = [ sourceTool("read", "Read a file"), - sourceTool("web_search", "Search the public web with filters and return sources"), - sourceTool("web_fetch", "Fetch the complete content of a selected web page"), + sourceTool("research_search", "Search the public web with filters and return sources"), + sourceTool("research_fetch", "Fetch the complete content of a selected web page"), ]; const api = { getAllTools: () => sourceTools, @@ -197,12 +197,12 @@ test("builds and caches a validated catalog with the current model on first sear summary: "Search and fetch current web sources.", useWhen: ["Current external facts are needed"], avoidWhen: ["The answer is entirely local"], - tools: ["web_search", "web_fetch"], + tools: ["research_search", "research_fetch"], }, ], tools: [ - { name: "web_search", summary: "Find web sources.", useWhen: ["Discover sources"], avoidWhen: [], keywords: ["search", "搜索"], primaryGroup: "web-research" }, - { name: "web_fetch", summary: "Read a selected source.", useWhen: ["A URL is known"], avoidWhen: [], keywords: ["fetch", "抓取"], primaryGroup: "web-research" }, + { name: "research_search", summary: "Find web sources.", useWhen: ["Discover sources"], avoidWhen: [], keywords: ["search", "搜索"], primaryGroup: "web-research" }, + { name: "research_fetch", summary: "Read a selected source.", useWhen: ["A URL is known"], avoidWhen: [], keywords: ["fetch", "抓取"], primaryGroup: "web-research" }, ], }; const usage = { input: 1, output: 1, cacheRead: 0, cacheWrite: 0, totalTokens: 2, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }; @@ -250,8 +250,8 @@ test("uses the precomputed bundle groups without calling a model", async () => { let modelCalls = 0; const sourceTools = [ sourceTool("read", "Read a file"), - sourceTool("tavily_web_search", "Search broadly for current web sources"), - sourceTool("tavily_web_fetch", "Fetch a selected Tavily source"), + sourceTool("web_search", "Search broadly for current web sources"), + sourceTool("web_fetch", "Fetch a selected web source"), sourceTool("ctx_execute", "Run commands in a context sandbox"), sourceTool("ctx_execute_file", "Analyze a file in a context sandbox"), sourceTool("ctx_batch_execute", "Run a bounded command batch"), @@ -292,12 +292,12 @@ test("uses the precomputed bundle groups without calling a model", async () => { const result = await registered.get("tool_search")?.execute( "call-2", - { query: "tavily_web_search" }, + { query: "web_search" }, undefined, undefined, ctx, ); - assert.equal(result?.details.loadedGroup, "web-tavily"); + assert.equal(result?.details.loadedGroup, "web-search"); assert.equal(result?.details.catalogSource, "bundle"); assert.equal(modelCalls, 0); await assert.rejects(readFile(join(agentDir, "my-pi", "tool-search", "catalog-v1.json"), "utf8"), /ENOENT/); @@ -337,8 +337,8 @@ test("loads an exact known group without waiting for unknown-tool model enrichme let modelCalls = 0; const sourceTools = [ sourceTool("read", "Read a file"), - sourceTool("tavily_web_search", "Search broadly for current web sources"), - sourceTool("tavily_web_fetch", "Fetch a selected Tavily source"), + sourceTool("web_search", "Search broadly for current web sources"), + sourceTool("web_fetch", "Fetch a selected web source"), sourceTool("unrecognized_extra", "An unrecognized third-party capability"), ]; const api = { @@ -366,12 +366,12 @@ test("loads an exact known group without waiting for unknown-tool model enrichme const result = await registered.get("tool_search")?.execute( "call-1", - { group: "web-tavily" }, + { group: "web-search" }, undefined, undefined, ctx, ); - assert.equal(result?.details.loadedGroup, "web-tavily"); + assert.equal(result?.details.loadedGroup, "web-search"); assert.equal(result?.details.catalogSource, "hybrid"); assert.equal(modelCalls, 0); } finally { diff --git a/pi-web-search/LICENSE b/pi-web-search/LICENSE new file mode 100644 index 0000000..57a1a7e --- /dev/null +++ b/pi-web-search/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 sia + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/pi-web-search/README.md b/pi-web-search/README.md new file mode 100644 index 0000000..5ad2c9b --- /dev/null +++ b/pi-web-search/README.md @@ -0,0 +1,35 @@ +# pi-web-search + +Unified web search and fetch for Pi. The main model only ever sees two tools — +`web_search` and `web_fetch` — provider selection happens inside the extension. + +- `web_search(query, max_results?, site?, published_after?, published_before?, depth?)` + routes each query to Tavily, Exa, or Keenable using the session's current main + model (`ctx.model` + `ctx.modelRegistry.complete`), guided by per-provider + rules. Results include `details.provider` / `details.routedBy` for debugging. +- `web_fetch(urls, max_characters?)` always reads pages via Exa + (`web_fetch_exa`), degrading silently to Keenable then Tavily. + +## Providers (`clients/`) + +Plain `fetch` calls — no MCP adapter, no upstream extension wrappers: + +- `clients/tavily.ts` — `POST api.tavily.com/search|extract`, Bearer key. + Requires `TAVILY_API_KEY`. +- `clients/keenable.ts` — `api.keenable.ai`, keyless `/public` endpoints by + default, `KEENABLE_API_KEY` raises rate limits. Adapted from + `@keenable/pi-search`. +- `clients/exa.ts` — JSON-RPC over `mcp.exa.ai/mcp` (Streamable HTTP): + `initialize` once, cache `mcp-session-id`, `tools/call` per request, re-init + on session expiry. `EXA_API_KEY` optional (`x-api-key` header). + +Keys are loaded from `~/.config/my-pi/search.env` by `extensions/search-config.ts` +(written by `search_config.sh`); explicit environment variables win. + +## Routing + +`router.ts` asks the current model to answer with exactly one provider name, +given a prompt containing the per-provider strengths and the query's requested +filters. Decisions are cached per session (LRU, 32 entries). Failure paths are +deterministic: an unroutable answer or a dead provider falls back across the +remaining available providers — never a heuristic choice over the model. diff --git a/pi-web-search/clients/exa.ts b/pi-web-search/clients/exa.ts new file mode 100644 index 0000000..e2feb8b --- /dev/null +++ b/pi-web-search/clients/exa.ts @@ -0,0 +1,243 @@ +/** + * Exa client: direct JSON-RPC over the hosted endpoint https://mcp.exa.ai/mcp. + * This is a plain HTTPS POST client (no MCP adapter, no SDK): initialize once, + * cache the `mcp-session-id` response header, then POST `tools/call` per call. + * Keyless works; EXA_API_KEY is sent as the `x-api-key` header when present. + */ + +export class ExaError extends Error {} + +const EXA_MCP_URL = "https://mcp.exa.ai/mcp?tools=web_search_exa,web_search_advanced_exa,web_fetch_exa"; +const PROTOCOL_VERSION = "2025-03-26"; +const SESSION_TIMEOUT_MS = 15_000; +const CALL_TIMEOUT_MS = 60_000; + +/** Cap page text per result so advanced search responses stay bounded. */ +export const EXA_SEARCH_TEXT_MAX_CHARACTERS = 2000; +export const EXA_FETCH_MAX_CHARACTERS = 3000; + +export type FetchLike = (input: string | URL, init?: RequestInit) => Promise; + +export interface ExaSearchParams { + query: string; + numResults?: number; + includeDomains?: string[]; + startPublishedDate?: string; + endPublishedDate?: string; + textMaxCharacters?: number; +} + +let cachedSessionId: string | undefined; +let pendingSession: Promise | undefined; +let nextRequestId = 1; + +/** Clear the cached MCP session (used on expiry and by tests). */ +export function resetExaSession(): void { + cachedSessionId = undefined; +} + +function baseHeaders(): Record { + const headers: Record = { + "content-type": "application/json", + accept: "application/json, text/event-stream", + }; + const key = process.env.EXA_API_KEY?.trim(); + if (key) headers["x-api-key"] = key; + return headers; +} + +function withTimeout(signal: AbortSignal | undefined, timeoutMs: number): AbortSignal { + const timeout = AbortSignal.timeout(timeoutMs); + return signal ? AbortSignal.any([signal, timeout]) : timeout; +} + +/** + * Extract the JSON-RPC message for `id` from a raw JSON body or an SSE stream + * (one or more `data: {...}` lines). + */ +export function parseRpcPayload(bodyText: string, id: number): Record { + const candidates: string[] = []; + const trimmed = bodyText.trim(); + if (trimmed.startsWith("{")) candidates.push(trimmed); + for (const line of trimmed.split(/\r?\n/u)) { + if (line.startsWith("data:")) candidates.push(line.slice(5).trim()); + } + for (const candidate of candidates) { + try { + const parsed = JSON.parse(candidate) as Record; + if (parsed && typeof parsed === "object" && !Array.isArray(parsed) && parsed.id === id) { + return parsed; + } + } catch { + // Not JSON — try the next candidate line. + } + } + throw new ExaError("Exa endpoint returned an unreadable response."); +} + +async function post( + body: Record, + sessionId: string | undefined, + timeoutMs: number, + signal: AbortSignal | undefined, + fetchImpl: FetchLike, +): Promise<{ status: number; sessionId?: string; text: string }> { + const headers = baseHeaders(); + if (sessionId) headers["mcp-session-id"] = sessionId; + let response: Response; + try { + response = await fetchImpl(EXA_MCP_URL, { + method: "POST", + headers, + body: JSON.stringify(body), + signal: withTimeout(signal, timeoutMs), + }); + } catch (error) { + throw new ExaError(`Could not reach the Exa endpoint: ${(error as Error).message}`); + } + return { + status: response.status, + sessionId: response.headers.get("mcp-session-id") ?? undefined, + text: await response.text(), + }; +} + +async function initialize(signal: AbortSignal | undefined, fetchImpl: FetchLike): Promise { + const id = nextRequestId++; + const res = await post( + { + jsonrpc: "2.0", + id, + method: "initialize", + params: { + protocolVersion: PROTOCOL_VERSION, + capabilities: {}, + clientInfo: { name: "my-pi-web-search", version: "0.1.0" }, + }, + }, + undefined, + SESSION_TIMEOUT_MS, + signal, + fetchImpl, + ); + if (res.status !== 200) throw new ExaError(`Exa initialize failed (HTTP ${res.status}).`); + const sessionId = res.sessionId; + if (!sessionId) throw new ExaError("Exa endpoint did not return an MCP session id."); + const payload = parseRpcPayload(res.text, id); + if (payload.error) throw new ExaError(`Exa initialize error: ${(payload.error as { message?: string }).message ?? "unknown"}`); + + const notice = await post( + { jsonrpc: "2.0", method: "notifications/initialized" }, + sessionId, + SESSION_TIMEOUT_MS, + signal, + fetchImpl, + ); + if (notice.status >= 400) { + throw new ExaError(`Exa initialized notification failed (HTTP ${notice.status}).`); + } + return sessionId; +} + +function sessionId(signal: AbortSignal | undefined, fetchImpl: FetchLike): Promise { + if (cachedSessionId) return Promise.resolve(cachedSessionId); + pendingSession ??= initialize(signal, fetchImpl) + .then((id) => { + cachedSessionId = id; + return id; + }) + .finally(() => { + pendingSession = undefined; + }); + return pendingSession; +} + +function contentText(result: Record): string { + const content = Array.isArray(result.content) ? (result.content as Record[]) : []; + return content + .filter((part) => part?.type === "text" && typeof part.text === "string") + .map((part) => part.text as string) + .join("\n") + .trim(); +} + +async function callTool( + toolName: string, + args: Record, + signal: AbortSignal | undefined, + fetchImpl: FetchLike, + allowRetry: boolean, +): Promise> { + const sid = await sessionId(signal, fetchImpl); + const id = nextRequestId++; + const res = await post( + { jsonrpc: "2.0", id, method: "tools/call", params: { name: toolName, arguments: args } }, + sid, + CALL_TIMEOUT_MS, + signal, + fetchImpl, + ); + if (res.status === 404 && allowRetry) { + // Session expired server-side: drop it, re-initialize, retry once. + cachedSessionId = undefined; + return callTool(toolName, args, signal, fetchImpl, false); + } + if (res.status !== 200) { + throw new ExaError(`Exa ${toolName} failed (HTTP ${res.status}).`); + } + const payload = parseRpcPayload(res.text, id); + if (payload.error) { + const message = (payload.error as { message?: string }).message ?? "unknown"; + if (allowRetry && /session/iu.test(message)) { + cachedSessionId = undefined; + return callTool(toolName, args, signal, fetchImpl, false); + } + throw new ExaError(`Exa ${toolName} error: ${message}`); + } + const result = (payload.result ?? {}) as Record; + if (result.isError) { + throw new ExaError(`Exa ${toolName} failed: ${contentText(result) || "unknown error"}`); + } + return result; +} + +/** tools/call web_search_advanced_exa — precise search with bounded text per result. */ +export async function exaSearch( + params: ExaSearchParams, + signal?: AbortSignal, + fetchImpl: FetchLike = fetch, +): Promise { + const result = await callTool( + "web_search_advanced_exa", + { + query: params.query, + type: "auto", + numResults: params.numResults ?? 8, + textMaxCharacters: params.textMaxCharacters ?? EXA_SEARCH_TEXT_MAX_CHARACTERS, + ...(params.includeDomains?.length ? { includeDomains: params.includeDomains } : {}), + ...(params.startPublishedDate ? { startPublishedDate: params.startPublishedDate } : {}), + ...(params.endPublishedDate ? { endPublishedDate: params.endPublishedDate } : {}), + }, + signal, + fetchImpl, + true, + ); + return contentText(result) || "Exa returned no content."; +} + +/** tools/call web_fetch_exa — full page content as clean markdown for a batch of URLs. */ +export async function exaFetchPages( + urls: string[], + maxCharacters?: number, + signal?: AbortSignal, + fetchImpl: FetchLike = fetch, +): Promise { + const result = await callTool( + "web_fetch_exa", + { urls, maxCharacters: maxCharacters ?? EXA_FETCH_MAX_CHARACTERS }, + signal, + fetchImpl, + true, + ); + return contentText(result) || "Exa returned no content."; +} diff --git a/pi-web-search/clients/keenable.ts b/pi-web-search/clients/keenable.ts new file mode 100644 index 0000000..70b6324 --- /dev/null +++ b/pi-web-search/clients/keenable.ts @@ -0,0 +1,222 @@ +/** + * Keenable REST client, adapted from @keenable/pi-search (keenable-client.ts). + * Keyless by default (`/v1/*\/public`), authenticated when KEENABLE_API_KEY is + * present (`/v1/*`), HTTPS-only base URL, `X-Keenable-Title` attribution. + */ + +import { isIP } from "node:net"; + +export class KeenableError extends Error {} + +const CLIENT_VERSION = "0.1.0"; +const DEFAULT_BASE_URL = "https://api.keenable.ai"; +const BASE_URL_ENV = "KEENABLE_API_URL"; +const API_KEY_ENV = "KEENABLE_API_KEY"; +const ATTRIBUTION_TITLE = "Pi"; +const USER_AGENT = `pi-web-search/${CLIENT_VERSION}`; +const DEFAULT_TIMEOUT_MS = 30_000; + +export type FetchLike = (input: string | URL, init?: RequestInit) => Promise; + +export interface KeenableSearchParams { + query: string; + mode?: "pro"; + site?: string; + published_after?: string; + published_before?: string; +} + +export interface KeenableSearchResult { + title?: string; + url?: string; + description?: string; + published_at?: string; + acquired_at?: string; + [key: string]: unknown; +} + +export interface KeenableFetchResult { + url?: string; + title?: string; + content?: string; + description?: string; + author?: string; + published_at?: string; + [key: string]: unknown; +} + +/** Resolve the API base URL from `KEENABLE_API_URL`, enforcing HTTPS (http only for loopback). */ +function resolveBaseUrl(): string { + const base = (process.env[BASE_URL_ENV] || DEFAULT_BASE_URL).replace(/\/+$/u, ""); + let parsed: URL; + try { + parsed = new URL(base); + } catch { + throw new KeenableError(`${BASE_URL_ENV} must be a valid URL with a host, got ${base}`); + } + if (parsed.hostname) { + if (parsed.protocol === "https:") return base; + if (parsed.protocol === "http:" && ["localhost", "127.0.0.1", "::1"].includes(parsed.hostname)) { + return base; + } + } + throw new KeenableError(`${BASE_URL_ENV} must be an https:// URL with a host, got ${base}`); +} + +/** Non-blank API key from the env, or undefined to use the keyless public tier. */ +function resolveApiKey(): string | undefined { + const key = (process.env[API_KEY_ENV] ?? "").trim(); + return key || undefined; +} + +function buildHeaders(apiKey: string | undefined): Record { + const headers: Record = { + Accept: "application/json", + "User-Agent": USER_AGENT, + "X-Keenable-Title": ATTRIBUTION_TITLE, + }; + if (apiKey) headers["X-API-Key"] = apiKey; + return headers; +} + +function withTimeout(signal: AbortSignal | undefined, timeoutMs: number): AbortSignal { + const timeout = AbortSignal.timeout(timeoutMs); + return signal ? AbortSignal.any([signal, timeout]) : timeout; +} + +function isPrivateIp(host: string): boolean { + const v = isIP(host); + if (v === 4) { + const p = host.split(".").map(Number); + return ( + p[0] === 10 || + p[0] === 127 || + p[0] === 0 || + (p[0] === 169 && p[1] === 254) || + (p[0] === 172 && p[1] >= 16 && p[1] <= 31) || + (p[0] === 192 && p[1] === 168) || + (p[0] === 100 && p[1] >= 64 && p[1] <= 127) || + (p[0] === 198 && (p[1] === 18 || p[1] === 19)) || + p[0] >= 224 + ); + } + if (v === 6) { + const h = host.toLowerCase(); + return h === "::1" || h === "::" || h.startsWith("fe80") || h.startsWith("fc") || h.startsWith("fd"); + } + return false; +} + +/** Refuse private/internal fetch targets before sending (SSRF). */ +function assertPublicFetchTarget(rawUrl: string): void { + let parsed: URL; + try { + parsed = new URL(rawUrl); + } catch { + throw new KeenableError(`Not a valid URL: ${rawUrl}`); + } + if (parsed.protocol !== "http:" && parsed.protocol !== "https:") { + throw new KeenableError(`Refusing to fetch a non-http(s) URL: ${rawUrl}`); + } + const host = (parsed.hostname || "").toLowerCase(); + if (!host) throw new KeenableError(`Refusing to fetch a URL with no host: ${rawUrl}`); + if (host === "localhost" || host === "metadata.google.internal" || isPrivateIp(host)) { + throw new KeenableError(`Refusing to fetch a private/internal host: ${host}`); + } +} + +async function raiseForStatus(response: Response, keyed: boolean): Promise { + if (response.ok) return; + if (!keyed && response.status === 429) { + throw new KeenableError( + "Keenable keyless requests hit their rate limit. Set KEENABLE_API_KEY to raise the limits.", + ); + } + let detail = ""; + try { + const body = (await response.json()) as Record; + detail = String(body.message ?? body.error ?? body.detail ?? ""); + } catch { + detail = (await response.text().catch(() => "")).trim(); + } + const label = + { + 401: "Keenable authentication failed (401)", + 402: "Keenable: insufficient credits (402)", + 429: "Keenable rate limit exceeded (429)", + }[response.status] ?? `Keenable API error (${response.status})`; + throw new KeenableError(detail ? `${label}: ${detail}` : label); +} + +async function readJsonObject(response: Response): Promise> { + let data: unknown; + try { + data = await response.json(); + } catch { + throw new KeenableError("Keenable API returned a non-JSON response."); + } + if (!data || typeof data !== "object" || Array.isArray(data)) { + throw new KeenableError("Unexpected (non-object) response from the Keenable API."); + } + return data as Record; +} + +/** POST /v1/search (keyed) or /v1/search/public (keyless). */ +export async function keenableSearch( + params: KeenableSearchParams, + signal: AbortSignal | undefined, + fetchImpl: FetchLike = fetch, +): Promise { + const apiKey = resolveApiKey(); + const path = apiKey ? "/v1/search" : "/v1/search/public"; + const payload: Record = { query: params.query, mode: params.mode ?? "pro" }; + for (const field of ["site", "published_after", "published_before"] as const) { + const value = params[field]; + if (value) payload[field] = value; + } + + let response: Response; + try { + response = await fetchImpl(`${resolveBaseUrl()}${path}`, { + method: "POST", + headers: { ...buildHeaders(apiKey), "Content-Type": "application/json" }, + body: JSON.stringify(payload), + signal: withTimeout(signal, DEFAULT_TIMEOUT_MS), + }); + } catch (err) { + throw new KeenableError(`Could not reach the Keenable API: ${(err as Error).message}`); + } + await raiseForStatus(response, Boolean(apiKey)); + const data = await readJsonObject(response); + const results = data.results; + if (!Array.isArray(results)) { + throw new KeenableError("Unexpected response from the Keenable search API (no results array)."); + } + return results as KeenableSearchResult[]; +} + +/** GET /v1/fetch?url= (keyed) or /v1/fetch/public?url= (keyless). */ +export async function keenableFetch( + url: string, + signal: AbortSignal | undefined, + fetchImpl: FetchLike = fetch, +): Promise { + assertPublicFetchTarget(url); + const apiKey = resolveApiKey(); + const path = apiKey ? "/v1/fetch" : "/v1/fetch/public"; + const endpoint = new URL(`${resolveBaseUrl()}${path}`); + endpoint.searchParams.set("url", url); + + let response: Response; + try { + response = await fetchImpl(endpoint, { + method: "GET", + headers: buildHeaders(apiKey), + signal: withTimeout(signal, DEFAULT_TIMEOUT_MS), + }); + } catch (err) { + throw new KeenableError(`Could not reach the Keenable API: ${(err as Error).message}`); + } + await raiseForStatus(response, Boolean(apiKey)); + return (await readJsonObject(response)) as KeenableFetchResult; +} diff --git a/pi-web-search/clients/tavily.ts b/pi-web-search/clients/tavily.ts new file mode 100644 index 0000000..fc1a2fb --- /dev/null +++ b/pi-web-search/clients/tavily.ts @@ -0,0 +1,125 @@ +/** + * Tavily REST client: POST https://api.tavily.com/search|extract with a Bearer + * key. Requires TAVILY_API_KEY — there is no keyless tier. + */ + +export class TavilyError extends Error {} + +const TAVILY_BASE_URL = "https://api.tavily.com"; +const SEARCH_TIMEOUT_MS = 45_000; +const EXTRACT_TIMEOUT_MS = 60_000; +const CLIENT_SOURCE = "my-pi-web-search"; + +export type FetchLike = (input: string | URL, init?: RequestInit) => Promise; + +export interface TavilySearchParams { + query: string; + max_results?: number; + search_depth?: "basic" | "advanced"; + topic?: "general" | "news" | "finance"; + include_domains?: string[]; + start_date?: string; + end_date?: string; +} + +export interface TavilySearchResult { + title?: string; + url?: string; + content?: string; + published_date?: string; + [key: string]: unknown; +} + +export function tavilyAvailable(env: Record = process.env): boolean { + return Boolean(env.TAVILY_API_KEY?.trim()); +} + +function withTimeout(signal: AbortSignal | undefined, timeoutMs: number): AbortSignal { + const timeout = AbortSignal.timeout(timeoutMs); + return signal ? AbortSignal.any([signal, timeout]) : timeout; +} + +async function post( + endpoint: string, + body: Record, + timeoutMs: number, + signal: AbortSignal | undefined, + fetchImpl: FetchLike, +): Promise> { + const apiKey = process.env.TAVILY_API_KEY?.trim(); + if (!apiKey) { + throw new TavilyError("Tavily is not configured: TAVILY_API_KEY is missing."); + } + let response: Response; + try { + response = await fetchImpl(`${TAVILY_BASE_URL}/${endpoint}`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${apiKey}`, + "X-Client-Source": CLIENT_SOURCE, + }, + body: JSON.stringify(body), + signal: withTimeout(signal, timeoutMs), + }); + } catch (error) { + throw new TavilyError(`Could not reach the Tavily API: ${(error as Error).message}`); + } + if (!response.ok) { + let detail = ""; + try { + const data = (await response.json()) as Record; + detail = String(data.error ?? data.message ?? ""); + } catch { + detail = ""; + } + throw new TavilyError(`Tavily API error (${response.status})${detail ? `: ${detail}` : ""}`); + } + return (await response.json()) as Record; +} + +/** POST /search — broad discovery; returns the answer plus a compact result list. */ +export async function tavilySearch( + params: TavilySearchParams, + signal?: AbortSignal, + fetchImpl: FetchLike = fetch, +): Promise<{ answer?: string; results: TavilySearchResult[] }> { + const payload: Record = { + query: params.query, + max_results: params.max_results, + search_depth: params.search_depth, + topic: params.topic, + include_domains: params.include_domains, + start_date: params.start_date, + end_date: params.end_date, + }; + for (const key of Object.keys(payload)) { + if (payload[key] === undefined) delete payload[key]; + } + const data = await post("search", payload, SEARCH_TIMEOUT_MS, signal, fetchImpl); + const rawResults = Array.isArray(data.results) ? (data.results as Record[]) : []; + return { + answer: typeof data.answer === "string" ? data.answer : undefined, + results: rawResults.map((result) => ({ + title: result.title, + url: result.url, + content: result.content, + published_date: result.publishedDate ?? result.published_date, + })), + }; +} + +/** POST /extract — full page content as markdown for a batch of URLs. */ +export async function tavilyExtract( + urls: string[], + signal?: AbortSignal, + fetchImpl: FetchLike = fetch, +): Promise<{ results: Array<{ url?: string; content?: string }>; failed: string[] }> { + const data = await post("extract", { urls, format: "markdown" }, EXTRACT_TIMEOUT_MS, signal, fetchImpl); + const rawResults = Array.isArray(data.results) ? (data.results as Record[]) : []; + const rawFailed = Array.isArray(data.failed_results) ? (data.failed_results as Record[]) : []; + return { + results: rawResults.map((result) => ({ url: result.url as string | undefined, content: result.raw_content as string | undefined })), + failed: rawFailed.map((result) => String(result.url ?? "unknown")), + }; +} diff --git a/pi-web-search/index.ts b/pi-web-search/index.ts new file mode 100644 index 0000000..4fbac61 --- /dev/null +++ b/pi-web-search/index.ts @@ -0,0 +1,282 @@ +/** + * pi-web-search: one unified web_search + web_fetch surface. + * + * The main model never sees provider names — web_search routes internally via + * the session's current model (see router.ts), and web_fetch always reads + * pages through Exa with graceful degradation. Provider clients are plain + * fetch calls (clients/*) — no MCP adapter, no upstream extension wrappers. + */ + +import type { AgentToolResult, ExtensionAPI } from "@earendil-works/pi-coding-agent"; + +import { loadSearchConfigEnv } from "../extensions/search-config.ts"; +import { exaFetchPages, exaSearch } from "./clients/exa.ts"; +import { keenableFetch, keenableSearch } from "./clients/keenable.ts"; +import { tavilyAvailable, tavilyExtract, tavilySearch } from "./clients/tavily.ts"; +import { + availableProviders, + routeSearch, + type SearchProvider, + type RouterContext, +} from "./router.ts"; + +const DEFAULT_MAX_RESULTS = 8; +const MAX_FETCH_URLS = 10; +const DEFAULT_FETCH_MAX_CHARACTERS = 3000; + +export interface WebSearchParams { + query: string; + max_results?: number; + site?: string; + published_after?: string; + published_before?: string; + depth?: "basic" | "advanced"; +} + +export interface WebFetchParams { + urls: string | string[]; + max_characters?: number; +} + +export type ProviderRunner = (params: WebSearchParams, signal?: AbortSignal) => Promise; +export type FetchRunner = (urls: string[], maxCharacters: number, signal?: AbortSignal) => Promise; + +function formatSearchPayload(payload: unknown): string { + return JSON.stringify(payload, null, 2); +} + +export function toTavilyParams(params: WebSearchParams) { + return { + query: params.query, + max_results: params.max_results ?? DEFAULT_MAX_RESULTS, + search_depth: (params.depth ?? "basic") as "basic" | "advanced", + include_domains: params.site ? [params.site] : undefined, + start_date: params.published_after, + end_date: params.published_before, + }; +} + +export function toExaParams(params: WebSearchParams) { + return { + query: params.query, + numResults: params.max_results ?? DEFAULT_MAX_RESULTS, + includeDomains: params.site ? [params.site] : undefined, + startPublishedDate: params.published_after, + endPublishedDate: params.published_before, + }; +} + +export function toKeenableParams(params: WebSearchParams) { + return { + query: params.query, + site: params.site, + published_after: params.published_after, + published_before: params.published_before, + }; +} + +/** Default per-provider parameter mapping from the unified schema. */ +export function createProviderRunners( + overrides: Partial> = {}, +): Record { + return { + tavily: + overrides.tavily ?? + (async (params, signal) => formatSearchPayload(await tavilySearch(toTavilyParams(params), signal))), + exa: overrides.exa ?? ((params, signal) => exaSearch(toExaParams(params), signal)), + keenable: + overrides.keenable ?? + (async (params, signal) => formatSearchPayload(await keenableSearch(toKeenableParams(params), signal))), + }; +} + +function errorText(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function toResult(text: string, details: Record): AgentToolResult { + return { content: [{ type: "text", text }], details }; +} + +/** + * Route once via the main model, then try the picked provider first and fall + * back across the remaining available providers on transport errors. + */ +export async function dispatchSearch( + params: WebSearchParams, + ctx: RouterContext, + routeCache: Map, + signal?: AbortSignal, + runners: Record = createProviderRunners(), +): Promise { + const decision = await routeSearch(params, ctx, routeCache, signal); + const order = [decision.provider, ...availableProviders().filter((p) => p !== decision.provider)]; + let lastError: unknown; + for (const provider of order) { + try { + const text = await runners[provider](params, signal); + return toResult(text, { + provider, + routedBy: provider === decision.provider ? decision.routedBy : "provider-fallback", + }); + } catch (error) { + lastError = error; + } + } + return toResult(`web_search failed on every available provider: ${errorText(lastError)}`, { error: true }); +} + +export function normalizeUrls(urls: string | string[]): string[] { + const list = (Array.isArray(urls) ? urls : [urls]).map((url) => url.trim()).filter(Boolean); + return [...new Set(list)]; +} + +/** web_fetch always prefers Exa; Keenable and Tavily are silent degradation paths. */ +export function createFetchRunners( + overrides: Partial> = {}, +): Record { + return { + exa: overrides.exa ?? ((urls, maxCharacters, signal) => exaFetchPages(urls, maxCharacters, signal)), + keenable: + overrides.keenable ?? + (async (urls, _maxCharacters, signal) => { + const pages = await Promise.all(urls.map((url) => keenableFetch(url, signal))); + return pages + .map((page) => `# ${page.title ?? page.url ?? "page"}\n${page.content ?? "(no content)"}`) + .join("\n\n---\n\n"); + }), + tavily: + overrides.tavily ?? + (async (urls, _maxCharacters, signal) => { + const { results, failed } = await tavilyExtract(urls, signal); + const body = results + .map((page) => `# ${page.url ?? "page"}\n${page.content ?? "(no content)"}`) + .join("\n\n---\n\n"); + return failed.length ? `${body}\n\nFailed URLs: ${failed.join(", ")}` : body; + }), + }; +} + +export async function dispatchFetch( + params: WebFetchParams, + signal?: AbortSignal, + fetchers: Record = createFetchRunners(), +): Promise { + const urls = normalizeUrls(params.urls); + if (!urls.length) { + return toResult("web_fetch needs at least one http(s) URL.", { error: true }); + } + if (urls.length > MAX_FETCH_URLS) { + return toResult(`web_fetch accepts at most ${MAX_FETCH_URLS} URLs per call; received ${urls.length}.`, { error: true }); + } + const maxCharacters = params.max_characters ?? DEFAULT_FETCH_MAX_CHARACTERS; + const order: SearchProvider[] = ["exa", "keenable"]; + if (tavilyAvailable()) order.push("tavily"); + + let lastError: unknown; + for (const provider of order) { + try { + const text = await fetchers[provider](urls, maxCharacters, signal); + return toResult(text, { provider }); + } catch (error) { + lastError = error; + } + } + return toResult(`web_fetch failed on every available provider: ${errorText(lastError)}`, { error: true }); +} + +export default function webSearchExtension(pi: ExtensionAPI): void { + // Keys may already be loaded by extensions/search-config.ts; this is a + // harmless idempotent safety net for standalone loading. + loadSearchConfigEnv(); + const routeCache = new Map(); + + pi.registerTool({ + name: "web_search", + label: "Web Search", + description: + "Search the web for current or citable information and return a compact list of candidate " + + "pages (title, url, summary). Provider selection is automatic. Follow up with web_fetch to " + + "read the most relevant pages.", + promptSnippet: "Search the web for current or citable information.", + promptGuidelines: [ + "Use web_search when the task needs current, real-world, or citable web information.", + "Keep queries focused; split multi-topic requests into separate web_search calls.", + "Use site and published_* filters only when they materially narrow the results.", + "After web_search, read the most relevant pages with web_fetch before relying on a claim.", + ], + parameters: { + type: "object", + properties: { + query: { + type: "string", + description: "The search query — a natural-language description of the ideal page, not just keywords.", + }, + max_results: { + type: "number", + description: `Maximum number of results to return (default ${DEFAULT_MAX_RESULTS}).`, + }, + site: { + type: "string", + description: "Restrict results to a single domain, e.g. 'arxiv.org'.", + }, + published_after: { + type: "string", + description: "Only pages published on/after this date (YYYY-MM-DD).", + }, + published_before: { + type: "string", + description: "Only pages published on/before this date (YYYY-MM-DD).", + }, + depth: { + type: "string", + enum: ["basic", "advanced"], + description: "'basic' (default) balanced search; 'advanced' for higher precision at higher cost.", + }, + }, + required: ["query"], + additionalProperties: false, + }, + async execute(_toolCallId, params, signal, _onUpdate, ctx) { + return dispatchSearch(params as WebSearchParams, ctx as unknown as RouterContext, routeCache, signal); + }, + }); + + pi.registerTool({ + name: "web_fetch", + label: "Web Fetch", + description: + "Fetch the full content of one or more http(s) URLs as clean markdown. Use after web_search " + + "to read selected pages, or directly when the user provides a page to read.", + promptSnippet: "Read the full content of known URLs.", + promptGuidelines: [ + "Use web_fetch to read the full content of a URL found via web_search or provided by the user.", + "Batch multiple URLs into a single web_fetch call when possible.", + ], + parameters: { + type: "object", + properties: { + urls: { + anyOf: [ + { type: "string", description: "A single http(s) URL to read." }, + { + type: "array", + items: { type: "string" }, + maxItems: MAX_FETCH_URLS, + description: `A list of http(s) URLs to read (max ${MAX_FETCH_URLS}).`, + }, + ], + }, + max_characters: { + type: "number", + description: `Maximum characters to extract per page (default ${DEFAULT_FETCH_MAX_CHARACTERS}).`, + }, + }, + required: ["urls"], + additionalProperties: false, + }, + async execute(_toolCallId, params, signal) { + return dispatchFetch(params as WebFetchParams, signal); + }, + }); +} diff --git a/pi-web-search/package.json b/pi-web-search/package.json new file mode 100644 index 0000000..cc92048 --- /dev/null +++ b/pi-web-search/package.json @@ -0,0 +1,42 @@ +{ + "name": "pi-web-search", + "version": "0.1.0", + "description": "Unified web search and fetch tools with main-model provider routing (Tavily, Exa, Keenable).", + "private": true, + "type": "module", + "license": "MIT", + "main": "./index.ts", + "exports": { + ".": "./index.ts" + }, + "files": [ + "index.ts", + "router.ts", + "clients", + "README.md", + "LICENSE" + ], + "pi": { + "extensions": [ + "./index.ts" + ] + }, + "scripts": { + "test": "node --test test/*.test.ts" + }, + "peerDependencies": { + "@earendil-works/pi-coding-agent": "*", + "typebox": "*" + }, + "peerDependenciesMeta": { + "@earendil-works/pi-coding-agent": { + "optional": true + }, + "typebox": { + "optional": true + } + }, + "engines": { + "node": ">=22.19.0" + } +} diff --git a/pi-web-search/router.ts b/pi-web-search/router.ts new file mode 100644 index 0000000..1517ef3 --- /dev/null +++ b/pi-web-search/router.ts @@ -0,0 +1,163 @@ +/** + * Provider router for web_search. The decision is delegated to the session's + * current main model (ctx.model via ctx.modelRegistry.complete) so no extra + * model configuration is needed; deterministic rules only handle availability + * and failure fallbacks, never the routing itself. + */ + +export type SearchProvider = "tavily" | "exa" | "keenable"; + +export interface SearchRequest { + query: string; + site?: string; + published_after?: string; + published_before?: string; + depth?: string; +} + +export type RoutedBy = "model" | "cache" | "fallback"; + +export interface RouteDecision { + provider: SearchProvider; + routedBy: RoutedBy; +} + +const ROUTER_TIMEOUT_MS = 15_000; +export const ROUTE_CACHE_LIMIT = 32; + +/** Order used only when the model cannot decide or no candidates remain. */ +export const PROVIDER_FALLBACK_ORDER: SearchProvider[] = ["tavily", "exa", "keenable"]; + +const PROVIDER_RULES: Record = { + tavily: + "Broad discovery. Best when keywords are uncertain, for news and recent or trending topics, " + + "and when many candidate sources are useful. Results can mix official pages, blogs, forums, " + + "and secondary articles; returned content is naturally long.", + exa: + "Precision sources. Best for official or API documentation, release notes and version " + + "information, academic papers and research material, and fact confirmation that needs " + + "high-quality sources.", + keenable: + "Focused and compact. Best for Chinese-language pages (policies, government notices), " + + "site-restricted search, publication-date filtering, and tight control over returned " + + "content length.", +}; + +/** Tavily requires a key; Exa and Keenable both work keyless. */ +export function availableProviders(env: Record = process.env): SearchProvider[] { + const providers: SearchProvider[] = []; + if (env.TAVILY_API_KEY?.trim()) providers.push("tavily"); + providers.push("exa", "keenable"); + return providers; +} + +export function buildRouterPrompt(request: SearchRequest, candidates: SearchProvider[]): string { + const rules = candidates.map((provider) => `- ${provider}: ${PROVIDER_RULES[provider]}`).join("\n"); + const lines = [ + "Pick the single best web search provider for this query.", + `Answer with exactly one word: ${candidates.join(", ")}.`, + "", + rules, + "", + `Query: ${request.query}`, + ]; + if (request.site) lines.push(`The caller asked to restrict results to the site: ${request.site}`); + if (request.published_after || request.published_before) { + lines.push( + `The caller asked for a publication-date filter: ${request.published_after ?? "…"} to ${request.published_before ?? "…"}`, + ); + } + if (request.depth === "advanced") lines.push("The caller asked for a deep, high-precision search."); + lines.push("Provider:"); + return lines.join("\n"); +} + +/** First candidate-matching provider token in the model's reply, else undefined. */ +export function parseProviderChoice(text: string, candidates: SearchProvider[]): SearchProvider | undefined { + const matches = text.toLowerCase().match(/\b(tavily|exa|keenable)\b/gu) ?? []; + for (const match of matches) { + const pick = match as SearchProvider; + if (candidates.includes(pick)) return pick; + } + return undefined; +} + +export function routeCacheKey(request: SearchRequest): string { + return [ + request.query.trim().toLowerCase(), + request.site ?? "", + request.published_after ?? "", + request.published_before ?? "", + request.depth ?? "", + ].join(" "); +} + +interface ModelMessage { + stopReason?: string; + errorMessage?: string; + content?: Array<{ type?: string; text?: string }>; +} + +/** Minimal structural view of the extension context needed for routing. */ +export interface RouterContext { + model?: unknown; + modelRegistry?: { + complete?: (model: unknown, request: unknown, options: unknown) => Promise; + }; +} + +export async function routeSearch( + request: SearchRequest, + ctx: RouterContext, + cache: Map, + signal?: AbortSignal, + env: Record = process.env, +): Promise { + const candidates = availableProviders(env); + const fallback = PROVIDER_FALLBACK_ORDER.find((provider) => candidates.includes(provider)) ?? candidates[0]; + if (!fallback || candidates.length <= 1) { + return { provider: fallback ?? "keenable", routedBy: "fallback" }; + } + + const key = routeCacheKey(request); + const cached = cache.get(key); + if (cached !== undefined && candidates.includes(cached)) { + return { provider: cached, routedBy: "cache" }; + } + + const complete = ctx.modelRegistry?.complete; + if (!ctx.model || typeof complete !== "function") { + return { provider: fallback, routedBy: "fallback" }; + } + + try { + const timeout = AbortSignal.timeout(ROUTER_TIMEOUT_MS); + const message = await complete.call(ctx.modelRegistry, ctx.model, { + messages: [ + { + role: "user", + content: [{ type: "text", text: buildRouterPrompt(request, candidates) }], + timestamp: Date.now(), + }, + ], + }, { + signal: signal ? AbortSignal.any([signal, timeout]) : timeout, + }); + if (message.stopReason === "error" || message.stopReason === "aborted") { + throw new Error(message.errorMessage ?? `routing stopped: ${message.stopReason}`); + } + const text = (message.content ?? []) + .flatMap((part) => (part.type === "text" && typeof part.text === "string" ? [part.text] : [])) + .join("\n"); + const pick = parseProviderChoice(text, candidates); + if (!pick) return { provider: fallback, routedBy: "fallback" }; + if (!cache.has(key) && cache.size >= ROUTE_CACHE_LIMIT) { + const oldest = cache.keys().next().value; + if (oldest !== undefined) cache.delete(oldest); + } + cache.set(key, pick); + return { provider: pick, routedBy: "model" }; + } catch { + return { provider: fallback, routedBy: "fallback" }; + } +} diff --git a/pi-web-search/test/dispatch.test.ts b/pi-web-search/test/dispatch.test.ts new file mode 100644 index 0000000..cfe7469 --- /dev/null +++ b/pi-web-search/test/dispatch.test.ts @@ -0,0 +1,178 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; + +import webSearchExtension, { + createFetchRunners, + createProviderRunners, + dispatchFetch, + dispatchSearch, + normalizeUrls, + toExaParams, + toKeenableParams, + toTavilyParams, + type WebSearchParams, +} from "../index.ts"; + +const PARAMS: WebSearchParams = { query: "release notes" }; + +function ctxAnswering(answer: string, calls: { count: number } = { count: 0 }) { + return { + model: { provider: "test", id: "main" }, + modelRegistry: { + complete: async () => { + calls.count += 1; + return { content: [{ type: "text", text: answer }] }; + }, + }, + }; +} + +function withEnv(key: string, value: string | undefined, fn: () => Promise | T): Promise | T { + const previous = process.env[key]; + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + const restore = () => { + if (previous === undefined) delete process.env[key]; + else process.env[key] = previous; + }; + try { + const out = fn(); + if (out instanceof Promise) return out.finally(restore); + restore(); + return out; + } catch (error) { + restore(); + throw error; + } +} + +test("extension registers only web_search and web_fetch, with no provider in the schema", () => { + const tools = new Map }; description?: string }>(); + const pi = { + registerTool(definition: { name: string }) { + tools.set(definition.name, definition as never); + }, + on() {}, + } as unknown as ExtensionAPI; + + webSearchExtension(pi); + assert.deepEqual([...tools.keys()].sort(), ["web_fetch", "web_search"]); + const schemaText = JSON.stringify([...tools.values()].map((t) => t.parameters)); + assert.doesNotMatch(schemaText, /tavily|exa|keenable|"provider"/iu); +}); + +test("dispatchSearch runs the model-picked provider first and records details", async () => { + await withEnv("TAVILY_API_KEY", "k", async () => { + const ran: string[] = []; + const runners = createProviderRunners({ + exa: async () => { + ran.push("exa"); + return "exa results"; + }, + }); + const result = await dispatchSearch(PARAMS, ctxAnswering("exa"), new Map(), undefined, runners); + assert.deepEqual(ran, ["exa"]); + assert.equal(result.content[0]!.type === "text" && result.content[0]!.text, "exa results"); + assert.deepEqual(result.details, { provider: "exa", routedBy: "model" }); + }); +}); + +test("dispatchSearch falls back across remaining providers when the pick fails", async () => { + await withEnv("TAVILY_API_KEY", "k", async () => { + const ran: string[] = []; + const runners = createProviderRunners({ + tavily: async () => { + ran.push("tavily"); + throw new Error("tavily down"); + }, + exa: async () => { + ran.push("exa"); + return "exa results"; + }, + }); + const result = await dispatchSearch(PARAMS, ctxAnswering("tavily"), new Map(), undefined, runners); + assert.deepEqual(ran, ["tavily", "exa"]); + assert.equal((result.details as Record).provider, "exa"); + assert.equal((result.details as Record).routedBy, "provider-fallback"); + }); +}); + +test("dispatchSearch reports an error result when every provider fails", async () => { + await withEnv("TAVILY_API_KEY", "k", async () => { + const runners = createProviderRunners({ + tavily: async () => { throw new Error("t"); }, + exa: async () => { throw new Error("e"); }, + keenable: async () => { throw new Error("k"); }, + }); + const result = await dispatchSearch(PARAMS, ctxAnswering("exa"), new Map(), undefined, runners); + assert.equal((result.details as Record).error, true); + assert.match(result.content[0]!.type === "text" ? result.content[0]!.text : "", /failed on every available provider/); + }); +}); + +test("normalizeUrls accepts a string or array, trims, and deduplicates", () => { + assert.deepEqual(normalizeUrls(" https://a.example "), ["https://a.example"]); + assert.deepEqual(normalizeUrls(["https://a", " https://a ", "https://b"]), ["https://a", "https://b"]); +}); + +test("dispatchFetch prefers exa and degrades to keenable without a model call", async () => { + await withEnv("TAVILY_API_KEY", undefined, async () => { + const fetchers = createFetchRunners({ + exa: async (urls) => { + throw new Error(`exa down for ${urls.length}`); + }, + keenable: async (urls) => `keenable pages: ${urls.join(",")}`, + }); + const result = await dispatchFetch({ urls: ["https://a.example", "https://b.example"] }, undefined, fetchers); + assert.equal((result.details as Record).provider, "keenable"); + assert.match(result.content[0]!.type === "text" ? result.content[0]!.text : "", /keenable pages: https:\/\/a\.example,https:\/\/b\.example/); + }); +}); + +test("dispatchFetch validates input and caps the URL count", async () => { + const empty = await dispatchFetch({ urls: [] }, undefined); + assert.equal((empty.details as Record).error, true); + + const tooMany = await dispatchFetch({ urls: Array.from({ length: 11 }, (_, i) => `https://e${i}.example`) }, undefined); + assert.equal((tooMany.details as Record).error, true); + assert.match(tooMany.content[0]!.type === "text" ? tooMany.content[0]!.text : "", /at most 10/); +}); + +test("unified params map onto each provider's native arguments", () => { + const params: WebSearchParams = { + query: "q", + max_results: 4, + site: "a.dev", + published_after: "2025-01-01", + published_before: "2025-06-01", + depth: "advanced", + }; + assert.deepEqual(toTavilyParams(params), { + query: "q", + max_results: 4, + search_depth: "advanced", + include_domains: ["a.dev"], + start_date: "2025-01-01", + end_date: "2025-06-01", + }); + assert.deepEqual(toExaParams(params), { + query: "q", + numResults: 4, + includeDomains: ["a.dev"], + startPublishedDate: "2025-01-01", + endPublishedDate: "2025-06-01", + }); + assert.deepEqual(toKeenableParams(params), { + query: "q", + site: "a.dev", + published_after: "2025-01-01", + published_before: "2025-06-01", + }); + + const defaults = toTavilyParams({ query: "q" }); + assert.equal(defaults.max_results, 8); + assert.equal(defaults.search_depth, "basic"); + assert.equal(defaults.include_domains, undefined); +}); diff --git a/pi-web-search/test/exa-client.test.ts b/pi-web-search/test/exa-client.test.ts new file mode 100644 index 0000000..0e109b8 --- /dev/null +++ b/pi-web-search/test/exa-client.test.ts @@ -0,0 +1,140 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { ExaError, exaFetchPages, exaSearch, parseRpcPayload, resetExaSession } from "../clients/exa.ts"; + +function sse(payload: Record): string { + return `event: message\ndata: ${JSON.stringify(payload)}\n\n`; +} + +interface RecordedCall { + method?: string; + params?: Record; + headers: Record; +} + +/** + * A fetch stub that speaks the Exa endpoint protocol: initialize answers with + * the scripted session id and the request's own id, notifications get a bare + * 202, and tools/call delegates to the supplied handler (default 404). + */ +function exaStub(options: { + sessionIds?: string[]; + onToolCall?: (args: Record, id: number, session: string | undefined) => Response; +} = {}) { + const calls: RecordedCall[] = []; + let sessionIndex = 0; + const fetchImpl = async (_url: string | URL, init?: RequestInit): Promise => { + const headers = (init?.headers ?? {}) as Record; + const body = JSON.parse(String(init?.body)) as { id?: number; method?: string; params?: Record }; + calls.push({ method: body.method, params: body.params, headers }); + if (body.method === "initialize") { + const sid = options.sessionIds?.[Math.min(sessionIndex++, (options.sessionIds?.length ?? 1) - 1)] ?? `sid-${sessionIndex}`; + return new Response( + sse({ jsonrpc: "2.0", id: body.id, result: { protocolVersion: "2025-03-26", capabilities: {}, serverInfo: { name: "exa" } } }), + { status: 200, headers: { "mcp-session-id": sid, "content-type": "text/event-stream" } }, + ); + } + if (body.method === "notifications/initialized") return new Response("", { status: 202 }); + if (body.method === "tools/call") { + return options.onToolCall?.(body.params ?? {}, body.id ?? 0, headers["mcp-session-id"]) ?? new Response("Not Found", { status: 404 }); + } + return new Response("unexpected", { status: 500 }); + }; + return { fetchImpl, calls }; +} + +function okToolResult(id: number, text: string): Response { + return new Response( + sse({ jsonrpc: "2.0", id, result: { content: [{ type: "text", text }] } }), + { status: 200, headers: { "content-type": "text/event-stream" } }, + ); +} + +test("parseRpcPayload reads JSON bodies and SSE data lines, matching the request id", () => { + const direct = JSON.stringify({ jsonrpc: "2.0", id: 7, result: { ok: true } }); + assert.deepEqual(parseRpcPayload(direct, 7).result, { ok: true }); + + const body = + `event: message\ndata: {"jsonrpc":"2.0","id":1,"result":{"a":1}}\n\n` + + `event: message\ndata: {"jsonrpc":"2.0","id":2,"result":{"b":2}}\n\n`; + assert.deepEqual(parseRpcPayload(body, 2).result, { b: 2 }); + + assert.throws(() => parseRpcPayload("not json at all", 1), ExaError); + assert.throws(() => parseRpcPayload(sse({ jsonrpc: "2.0", id: 99, result: {} }), 1), ExaError); +}); + +test("exaSearch initializes a session, notifies, then posts tools/call with the session id", async () => { + resetExaSession(); + const { fetchImpl, calls } = exaStub({ + sessionIds: ["sid-A"], + onToolCall: (_args, id) => okToolResult(id, "search results"), + }); + + const text = await exaSearch({ query: "docs", numResults: 3, includeDomains: ["python.org"] }, undefined, fetchImpl); + assert.equal(text, "search results"); + assert.deepEqual(calls.map((c) => c.method), ["initialize", "notifications/initialized", "tools/call"]); + + const call = calls[2]!; + assert.equal(call.headers["mcp-session-id"], "sid-A"); + assert.equal((call.params as { name: string }).name, "web_search_advanced_exa"); + const args = (call.params as { arguments: Record }).arguments; + assert.equal(args.numResults, 3); + assert.equal(args.type, "auto"); + assert.deepEqual(args.includeDomains, ["python.org"]); + assert.ok((args.textMaxCharacters as number) > 0); +}); + +test("exaSearch reuses the cached session across calls", async () => { + resetExaSession(); + const { fetchImpl, calls } = exaStub({ + sessionIds: ["sid-1"], + onToolCall: (args, id) => okToolResult(id, `result for ${(args as { name: string }).name}`), + }); + + await exaSearch({ query: "a" }, undefined, fetchImpl); + const text = await exaFetchPages(["https://example.com"], 500, undefined, fetchImpl); + assert.match(text, /web_fetch_exa/); + assert.equal(calls.length, 4); + assert.equal((calls[3]!.params as { name: string }).name, "web_fetch_exa"); + assert.equal(calls[3]!.headers["mcp-session-id"], "sid-1"); + assert.equal((calls[3]!.params as { arguments: { maxCharacters: number } }).arguments.maxCharacters, 500); +}); + +test("a 404 from tools/call drops the session, re-initializes, and retries once", async () => { + resetExaSession(); + let toolCalls = 0; + const { fetchImpl, calls } = exaStub({ + sessionIds: ["stale-sid", "fresh-sid"], + onToolCall: (_args, id) => { + toolCalls += 1; + return toolCalls === 1 ? new Response("Not Found", { status: 404 }) : okToolResult(id, "after re-init"); + }, + }); + + const text = await exaSearch({ query: "retry me" }, undefined, fetchImpl); + assert.equal(text, "after re-init"); + assert.deepEqual(calls.map((c) => c.method), [ + "initialize", + "notifications/initialized", + "tools/call", + "initialize", + "notifications/initialized", + "tools/call", + ]); + assert.equal(calls[5]!.headers["mcp-session-id"], "fresh-sid"); +}); + +test("isError tool results and HTTP failures raise ExaError", async () => { + resetExaSession(); + const isError = exaStub({ + onToolCall: (_args, id) => + new Response(sse({ jsonrpc: "2.0", id, result: { isError: true, content: [{ type: "text", text: "bad args" }] } }), { status: 200 }), + }); + await assert.rejects(exaFetchPages(["https://example.com"], undefined, undefined, isError.fetchImpl), /bad args/); + + resetExaSession(); + const httpFail = exaStub({ onToolCall: () => new Response("boom", { status: 503 }) }); + // Session init succeeds; tools/call hits a non-404 HTTP error. + await assert.rejects(exaSearch({ query: "x" }, undefined, httpFail.fetchImpl), /HTTP 503/); +}); diff --git a/pi-web-search/test/router.test.ts b/pi-web-search/test/router.test.ts new file mode 100644 index 0000000..344031b --- /dev/null +++ b/pi-web-search/test/router.test.ts @@ -0,0 +1,109 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + availableProviders, + buildRouterPrompt, + parseProviderChoice, + routeSearch, + ROUTE_CACHE_LIMIT, + type RouterContext, + type SearchProvider, +} from "../router.ts"; + +const ALL: SearchProvider[] = ["tavily", "exa", "keenable"]; +const ENV_ALL_KEYS = { TAVILY_API_KEY: "test-key" }; + +function ctxAnswering(answer: string, calls: { count: number } = { count: 0 }): RouterContext { + return { + model: { provider: "test", id: "main" }, + modelRegistry: { + complete: async () => { + calls.count += 1; + return { content: [{ type: "text", text: answer }] }; + }, + }, + }; +} + +test("availability: tavily needs a key, exa and keenable are keyless", () => { + assert.deepEqual(availableProviders({}), ["exa", "keenable"]); + assert.deepEqual(availableProviders(ENV_ALL_KEYS), ["tavily", "exa", "keenable"]); +}); + +test("router prompt lists only candidate rules plus the query and caller hints", () => { + const prompt = buildRouterPrompt( + { query: "python 3.14 release notes", site: "python.org", published_after: "2025-10-01" }, + ALL, + ); + assert.match(prompt, /- tavily: /); + assert.match(prompt, /- exa: /); + assert.match(prompt, /- keenable: /); + assert.match(prompt, /Query: python 3\.14 release notes/); + assert.match(prompt, /site filter.*python\.org|site: python\.org/); + assert.match(prompt, /Provider:$/); + + const subset = buildRouterPrompt({ query: "hi" }, ["exa", "keenable"]); + assert.doesNotMatch(subset, /- tavily/); + assert.match(subset, /exactly one word: exa, keenable/); +}); + +test("parseProviderChoice takes the first available candidate token", () => { + assert.equal(parseProviderChoice("tavily", ALL), "tavily"); + assert.equal(parseProviderChoice("I would pick Exa here.", ALL), "exa"); + assert.equal(parseProviderChoice("keenable!", ALL), "keenable"); + assert.equal(parseProviderChoice("google maybe", ALL), undefined); + // Model picked an unavailable provider but mentioned a valid one later. + assert.equal(parseProviderChoice("tavily or keenable", ["exa", "keenable"]), "keenable"); + assert.equal(parseProviderChoice("tavily", ["exa", "keenable"]), undefined); +}); + +test("routeSearch honors the model pick and caches identical requests", async () => { + const calls = { count: 0 }; + const ctx = ctxAnswering("exa", calls); + const cache = new Map(); + const request = { query: "official MCP spec" }; + + const first = await routeSearch(request, ctx, cache, undefined, ENV_ALL_KEYS); + assert.deepEqual(first, { provider: "exa", routedBy: "model" }); + + const second = await routeSearch(request, ctx, cache, undefined, ENV_ALL_KEYS); + assert.deepEqual(second, { provider: "exa", routedBy: "cache" }); + assert.equal(calls.count, 1); +}); + +test("routeSearch falls back to the first available provider on unroutable or failed model calls", async () => { + const cache = new Map(); + const unparseable = await routeSearch({ query: "x" }, ctxAnswering("no idea"), cache, undefined, ENV_ALL_KEYS); + assert.deepEqual(unparseable, { provider: "tavily", routedBy: "fallback" }); + + const throwing: RouterContext = { + model: {}, + modelRegistry: { complete: async () => { throw new Error("boom"); } }, + }; + const failed = await routeSearch({ query: "x" }, throwing, cache, undefined, ENV_ALL_KEYS); + assert.deepEqual(failed, { provider: "tavily", routedBy: "fallback" }); +}); + +test("routeSearch without a tavily key excludes it from candidates", async () => { + const cache = new Map(); + const decision = await routeSearch({ query: "x" }, ctxAnswering("tavily"), cache, undefined, {}); + // "tavily" is not a candidate → unroutable → first available fallback. + assert.deepEqual(decision, { provider: "exa", routedBy: "fallback" }); +}); + +test("routeSearch works without a model", async () => { + const cache = new Map(); + const decision = await routeSearch({ query: "x" }, {}, cache, undefined, ENV_ALL_KEYS); + assert.deepEqual(decision, { provider: "tavily", routedBy: "fallback" }); +}); + +test("route cache evicts the oldest entry at the limit", async () => { + const cache = new Map(); + for (let i = 0; i < ROUTE_CACHE_LIMIT; i++) cache.set(`dummy-${i}`, "exa"); + const decision = await routeSearch({ query: "fresh query" }, ctxAnswering("keenable"), cache, undefined, ENV_ALL_KEYS); + assert.equal(decision.provider, "keenable"); + assert.equal(cache.size, ROUTE_CACHE_LIMIT); + assert.equal(cache.has("dummy-0"), false); + assert.equal(cache.has("dummy-1"), true); +}); diff --git a/tests/exa-override.test.ts b/tests/exa-override.test.ts deleted file mode 100644 index 515efd0..0000000 --- a/tests/exa-override.test.ts +++ /dev/null @@ -1,45 +0,0 @@ -import assert from "node:assert/strict"; -import test from "node:test"; - -import { createExaToolApiOverride, mapExaToolName, prefixExaTool } from "../extensions/exa-tool-prefix.ts"; - -test("Exa tools use a provider prefix followed by the original semantic name", () => { - assert.equal(mapExaToolName("web_search_exa"), "exa_web_search"); - assert.equal(mapExaToolName("web_fetch_exa"), "exa_web_fetch"); - assert.equal(mapExaToolName("web_search_advanced_exa"), "exa_web_search_advanced"); - assert.equal(mapExaToolName("unrelated_tool"), "unrelated_tool"); - - const tool = prefixExaTool({ name: "web_search_exa", label: "MCP: web_search_exa" }); - assert.equal(tool.name, "exa_web_search"); - assert.equal(tool.label, "Exa Web Search"); -}); - -test("Exa API override keeps registered and active tool names aligned", () => { - const registered: Array<{ name: string; label: string }> = []; - let active: string[] = []; - const unregistered: string[] = []; - const api = createExaToolApiOverride({ - registerTool(tool: unknown) { - registered.push(tool as { name: string; label: string }); - }, - setActiveTools(names: string[]) { - active = names; - }, - unregisterTool(name: string) { - unregistered.push(name); - return true; - }, - }); - - api.registerTool({ name: "web_fetch_exa", label: "MCP: web_fetch_exa" }); - api.registerTool({ name: "codegraph_explore", label: "MCP: codegraph_explore" }); - api.setActiveTools(["read", "web_fetch_exa", "web_fetch_exa", "codegraph_explore"]); - api.unregisterTool("web_fetch_exa"); - api.unregisterTool("codegraph_explore"); - - assert.equal(registered[0]?.name, "exa_web_fetch"); - assert.equal(registered[0]?.label, "Exa Web Fetch"); - assert.equal(registered[1]?.name, "codegraph_explore"); - assert.deepEqual(active, ["read", "exa_web_fetch", "codegraph_explore"]); - assert.deepEqual(unregistered, ["exa_web_fetch", "codegraph_explore"]); -}); diff --git a/tests/extension-load.test.ts b/tests/extension-load.test.ts index de5197e..76cb3ee 100644 --- a/tests/extension-load.test.ts +++ b/tests/extension-load.test.ts @@ -35,6 +35,11 @@ test("all package extensions load together without global registration conflicts packageJson.pi.extensions.includes("./extensions/codegraph.ts"), "the bundle must expose the native codegraph_explore tool", ); + assert.ok( + packageJson.pi.extensions.includes("./pi-web-search/index.ts"), + "the bundle must load the unified pi-web-search tools", + ); + assert.equal(packageJson.dependencies["pi-web-search"], "file:./pi-web-search"); assert.equal( packageJson.pi.extensions.includes("./extensions/plugin-init.ts"), false, diff --git a/tests/mcp.test.ts b/tests/mcp.test.ts deleted file mode 100644 index fda20e6..0000000 --- a/tests/mcp.test.ts +++ /dev/null @@ -1,63 +0,0 @@ -import assert from "node:assert/strict"; -import { readFile } from "node:fs/promises"; -import test from "node:test"; - -import { createMcpConfig, EXA_TOOLS } from "../extensions/mcp-config.ts"; - -test("the shared MCP adapter config only connects Exa", () => { - assert.deepEqual(createMcpConfig({}), { - settings: { - disableProxyTool: true, - scriptMode: false, - }, - mcpServers: { - exa: { - url: `https://mcp.exa.ai/mcp?tools=${EXA_TOOLS.join(",")}`, - protocolVersion: "auto", - lifecycle: "eager", - directTools: EXA_TOOLS, - includeTools: EXA_TOOLS, - toolPrefix: "none", - approveTools: false, - exposeResources: false, - }, - }, - }); -}); - -test("Exa API key is resolved from the environment into a header and never placed in the URL", () => { - const config = createMcpConfig({ EXA_API_KEY: "exa-test-key" }); - assert.deepEqual(config.mcpServers.exa.headers, { "x-api-key": "$env:EXA_API_KEY" }); - assert.equal(config.mcpServers.exa.url?.includes("exa-test-key"), false); - assert.equal(JSON.stringify(config).includes("exa-test-key"), false); -}); - -test("the package has exactly one local MCP adapter owner", async () => { - const packageJson = JSON.parse(await readFile(new URL("../package.json", import.meta.url), "utf8")) as { - pi: { extensions: string[] }; - }; - const localEntries = packageJson.pi.extensions.filter((entry) => !entry.startsWith("./node_modules/")); - const adapterOwners: string[] = []; - for (const entry of localEntries) { - const source = await readFile(new URL(`../${entry.slice(2)}`, import.meta.url), "utf8"); - if (source.includes("createMcpAdapter") || source.includes('from "pi-mcp-adapter"')) { - adapterOwners.push(entry); - } - } - assert.deepEqual(adapterOwners, ["./extensions/mcp.ts"]); -}); - -test("search credentials load before all search providers", async () => { - const packageJson = JSON.parse(await readFile(new URL("../package.json", import.meta.url), "utf8")) as { - pi: { extensions: string[] }; - }; - const entries = packageJson.pi.extensions; - const loaderIndex = entries.indexOf("./extensions/search-config.ts"); - for (const provider of [ - "./extensions/tavily-override.ts", - "./node_modules/@keenable/pi-search/src/index.ts", - "./extensions/mcp.ts", - ]) { - assert.ok(loaderIndex >= 0 && loaderIndex < entries.indexOf(provider), `${provider} must load after search config`); - } -}); diff --git a/tests/tavily-override.test.ts b/tests/tavily-override.test.ts deleted file mode 100644 index ee795d6..0000000 --- a/tests/tavily-override.test.ts +++ /dev/null @@ -1,22 +0,0 @@ -import assert from "node:assert/strict"; -import test from "node:test"; - -import { prefixTavilyTool } from "../extensions/tavily-tool-prefix.ts"; - -test("Tavily override prefixes tool identity and internal tool references", () => { - const tool = prefixTavilyTool({ - name: "web_search", - label: "Web Search", - description: "Use web_search, then web_fetch with Tavily.", - promptSnippet: "Call web_search.", - promptGuidelines: ["Use web_fetch after web_search."], - parameters: {} as never, - execute: async () => ({ content: [{ type: "text" as const, text: "ok" }] }), - }); - - assert.equal(tool.name, "tavily_web_search"); - assert.equal(tool.label, "Tavily Web Search"); - assert.equal(tool.description, "Use tavily_web_search, then tavily_web_fetch with Tavily."); - assert.equal(tool.promptSnippet, "Call tavily_web_search."); - assert.deepEqual(tool.promptGuidelines, ["Use tavily_web_fetch after tavily_web_search."]); -}); diff --git a/tests/tool-routing.test.ts b/tests/tool-routing.test.ts index b34a839..5d3608e 100644 --- a/tests/tool-routing.test.ts +++ b/tests/tool-routing.test.ts @@ -73,21 +73,15 @@ test("LSP routing covers navigation and post-edit diagnostics", () => { assert.match(section, /post-edit diagnostics/); }); -test("online search routing selects providers by task", () => { - const tavily = buildToolRoutingSection(["tavily_web_search", "tavily_web_fetch"]); - assert.match(tavily, /broad discovery when keywords are uncertain/); - assert.match(tavily, /news\/recent\/trending topics/); - assert.doesNotMatch(tavily, /official or API documentation/); +test("online search routing uses the unified web tools without naming providers", () => { + const section = buildToolRoutingSection(["web_search", "web_fetch"]); + assert.match(section, /Use web_search for current, real-world, or citable web information/); + assert.match(section, /provider selection is handled internally/); + assert.match(section, /read selected pages with web_fetch/); + assert.doesNotMatch(section, /tavily|exa|keenable/iu); - const exa = buildToolRoutingSection(["exa_web_search", "exa_web_fetch", "exa_web_search_advanced"]); - assert.match(exa, /official or API documentation/); - assert.match(exa, /explicitly bound the result count and returned text length/); - assert.doesNotMatch(exa, /Chinese-language pages/); - - const keenable = buildToolRoutingSection(["keenable_search", "keenable_fetch"]); - assert.match(keenable, /Chinese-language pages, policies and government notices/); - assert.match(keenable, /site-restricted search/); - assert.match(keenable, /publication-date filtering/); + const searchOnly = buildToolRoutingSection(["web_search"]); + assert.match(searchOnly, /Use web_search/); }); test("routing includes only guidance for active optional tools", () => { @@ -98,8 +92,7 @@ test("routing includes only guidance for active optional tools", () => { assert.doesNotMatch(section, /Use Context Mode/); assert.doesNotMatch(section, /ctx_execute_file is confined/); assert.doesNotMatch(section, /fresh LINE#HASH anchors/); - assert.doesNotMatch(section, /Choose one online search provider/); - assert.doesNotMatch(section, /broad discovery when keywords are uncertain/); + assert.doesNotMatch(section, /Use web_search for current/); assert.match(section, /Use read only when exact source is needed/); }); diff --git a/tests/web-search.test.ts b/tests/web-search.test.ts new file mode 100644 index 0000000..34bb6b0 --- /dev/null +++ b/tests/web-search.test.ts @@ -0,0 +1,47 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +const packageJson = JSON.parse(await readFile(new URL("../package.json", import.meta.url), "utf8")) as { + dependencies: Record; + files: string[]; + pi: { extensions: string[]; skills: string[] }; +}; + +test("pi-web-search is the single search surface, loading after search-config", () => { + const entries = packageJson.pi.extensions; + assert.ok(entries.includes("./pi-web-search/index.ts"), "the bundle must load pi-web-search"); + assert.ok( + entries.indexOf("./extensions/search-config.ts") < entries.indexOf("./pi-web-search/index.ts"), + "search credentials must load before pi-web-search", + ); + for (const retired of [ + "./extensions/mcp.ts", + "./extensions/tavily-override.ts", + "./node_modules/@keenable/pi-search/src/index.ts", + ]) { + assert.equal(entries.includes(retired), false, `${retired} must no longer be loaded`); + } +}); + +test("pi-web-search ships as a maintained file: dependency without upstream search packages", () => { + assert.equal(packageJson.dependencies["pi-web-search"], "file:./pi-web-search"); + assert.ok(packageJson.files.includes("pi-web-search")); + assert.equal(packageJson.dependencies["@tavily/pi-extension"], undefined); + assert.equal(packageJson.dependencies["@keenable/pi-search"], undefined); + assert.equal(packageJson.dependencies["pi-mcp-adapter"], undefined, "Exa is a direct client now — no MCP adapter"); +}); + +test("no extension in the bundle loads the MCP adapter or upstream search extensions", async () => { + for (const entry of packageJson.pi.extensions.filter((e) => !e.startsWith("./node_modules/"))) { + const source = await readFile(new URL(`../${entry.slice(2)}`, import.meta.url), "utf8"); + assert.equal(source.includes("pi-mcp-adapter"), false, `${entry} must not import pi-mcp-adapter`); + assert.equal(source.includes("@tavily/pi-extension"), false, `${entry} must not import @tavily/pi-extension`); + } +}); + +test("the keenable skill path is gone with the upstream package", () => { + for (const skillPath of packageJson.pi.skills) { + assert.equal(skillPath.includes("@keenable"), false); + } +});