mirror of
https://bitbucket.org/siakitem/my-pi.git
synced 2026-08-28 08:35:57 +00:00
feat: vendor grouped tool search
This commit is contained in:
云服务部-叶林立
@@ -0,0 +1,4 @@
|
||||
node_modules/
|
||||
dist/
|
||||
*.js
|
||||
*.d.ts
|
||||
@@ -0,0 +1,69 @@
|
||||
# Changelog
|
||||
|
||||
## Unreleased — my-pi maintained source
|
||||
|
||||
### Changed
|
||||
- Imported upstream `v0.3.6` at `ddfb23646fd3957b791214de278e23aa393c9b13` for direct local maintenance.
|
||||
- Migrated runtime imports to `@earendil-works/pi-coding-agent` and current `typebox`.
|
||||
- Replaced the hidden steer/retry compatibility loop with current Pi dynamic-tool propagation.
|
||||
- Replaced the truncated name manifest and add-only lifetime with validated model-generated Tool Cards, workflow groups, and bounded LRU activation.
|
||||
- Added deterministic no-model fallback grouping, private manifest-hash caching, exact group overrides, lazy `/tool-search-rebuild`, and nested model usage accounting.
|
||||
- Added limits for active groups, generated group size, and total dynamic tools, plus current-host catalog/config/LRU tests.
|
||||
- Added checked-in authoritative groups for every tool exposed by the standard my-pi bundle, eliminating first-use model generation and user caching unless unrecognized third-party tools are present.
|
||||
|
||||
## [0.3.6] - 2026-04-24
|
||||
|
||||
### Bug Fixes
|
||||
- Clear footer status when `toolSearch.showToolSearchFooterStatus` is `false`, and re-read setting each refresh so settings changes take effect without stale status.
|
||||
- Add explicit `showToolSearchFooterStatus` config name with backward compatibility for older status keys.
|
||||
|
||||
## [0.3.5] - 2026-04-23
|
||||
|
||||
### Other
|
||||
- Add `pi install npm:pi-tool-search` command to README
|
||||
|
||||
## [0.3.4] - 2026-04-23
|
||||
|
||||
### Other
|
||||
- Clarify core defaults and token-saving purpose
|
||||
|
||||
## [0.3.3] - 2026-04-23
|
||||
|
||||
### Bug Fixes
|
||||
- Refresh active tools on every `turn_start`, not only fresh user prompts, so unlocked tools stay available during agent-loop continuations
|
||||
- Queue hidden steer hint after successful `tool_search` so agent can continue/retry without waiting for another user message
|
||||
- Stop showing visible retry guidance in `tool_search` results and narrow hidden retry hint so successful same-turn tool calls are not repeated
|
||||
|
||||
### Other
|
||||
- Document same-response activation caveat and recovery behavior in `README.md`
|
||||
|
||||
## [0.3.2] - 2026-04-23
|
||||
|
||||
### Bug Fixes
|
||||
- Split `tool_search` description into "Already active" and "Hidden" sections so LLM skips redundant enable calls
|
||||
- Add `grep` and `find` to default core tools (always enabled alongside `read`, `write`, `edit`, `bash`)
|
||||
|
||||
## [0.3.1] - 2026-04-23
|
||||
|
||||
### Other
|
||||
- Add repository field to package.json
|
||||
|
||||
## 0.3.0
|
||||
|
||||
- Renamed from `pi-lazy-tools` to `pi-tool-search`
|
||||
- Config key changed: `lazyTools` → `toolSearch` in `settings.json`
|
||||
- `showStatus` config option: show/hide `N / total tools` footer status (default: on)
|
||||
- Provider-agnostic: removed payload-level filtering, relies solely on `setActiveTools`
|
||||
- `readUserConfig()` consolidates all settings reads into one call
|
||||
|
||||
## 0.2.0
|
||||
|
||||
- User config: add `"toolSearch": { "alwaysEnabled": ["lsp", "grep"] }` to `settings.json` to pre-unlock tools beyond the defaults
|
||||
- Reads config at each `session_start` — no reinstall needed after changes
|
||||
|
||||
## 0.1.0
|
||||
|
||||
- Initial release
|
||||
- Manifest-aware `tool_search` gate
|
||||
- `names: string[]` batch enabling
|
||||
- Per-turn manifest refresh via `before_agent_start`
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 Tuan Son
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,123 @@
|
||||
# pi-tool-search
|
||||
|
||||
Bounded, model-assisted tool-group loading for Pi. This source is maintained directly in the `my-pi` bundle from upstream `v0.3.6`; see [UPSTREAM.md](./UPSTREAM.md).
|
||||
|
||||
## Why
|
||||
|
||||
A name plus a truncated first sentence can misrepresent a tool, while an add-only active set eventually restores every full schema in a long session. This maintained version instead builds validated retrieval metadata from complete tool definitions, presents compact group cards, activates whole workflow groups, and caps the dynamic active set with LRU eviction.
|
||||
|
||||
Generated metadata never replaces executable schemas. Exact names, parameter types, required fields, and source definitions remain authoritative in Pi. The repository provides curated metadata for its own tools; a model may generate only summaries, usage guidance, keywords, and assignments for additional unrecognized tools.
|
||||
|
||||
## Catalog lifecycle
|
||||
|
||||
1. `session_start` keeps `tool_search`, Pi core tools, and configured `alwaysEnabled` names active.
|
||||
2. Hidden tool definitions are hashed with the grouping constraints.
|
||||
3. Every tool exposed by the standard `my-pi` bundle is matched against the checked-in catalog in `extensions/bundle-groups.ts`. If all tools are recognized, no model call and no user cache are needed.
|
||||
4. User `groupOverrides` take priority over checked-in assignments. Optional or unavailable bundle tools are simply filtered out of their predefined groups.
|
||||
5. Additional user or third-party tools receive deterministic prefix/source groups immediately. A matching private model-enriched cache at `<agent-dir>/tool-search/catalog-v1.json` is reused when present.
|
||||
6. Only when unrecognized tools exist does the first `tool_search` call ask the current authenticated model to enrich the complete catalog. Validation requires all checked-in bundle tools to remain in their predefined groups before a `0600` cache is accepted.
|
||||
7. Missing authentication, invalid JSON, timeout, cancellation, changed bundle assignments, or cache errors leave the checked-in plus deterministic hybrid catalog usable.
|
||||
|
||||
A standard bundle installation therefore sends no tool definitions to a model during catalog setup. Complete definitions are sent to the selected provider only when extra unrecognized tools require enrichment; that nested call's usage is attached to the `tool_search` result.
|
||||
|
||||
Run `/tool-search-rebuild` to remove model enrichment. Standard bundle tools immediately return to the checked-in catalog; extra tools can be enriched lazily on the next search.
|
||||
|
||||
## Precomputed my-pi groups
|
||||
|
||||
| Group | Bundle tools |
|
||||
|---|---|
|
||||
| `filesystem-navigation` | `ls`, `multi_grep` |
|
||||
| `code-intelligence` | CodeGraph and all LSP tools |
|
||||
| `web-tavily` | Tavily search/fetch |
|
||||
| `web-exa` | Exa search/advanced/fetch |
|
||||
| `web-keenable` | Keenable search/fetch |
|
||||
| `context-execution` | `ctx_execute`, `ctx_execute_file`, `ctx_batch_execute` |
|
||||
| `context-knowledge` | Context indexing, searching, and web-doc ingestion |
|
||||
| `context-observability` | Context stats, doctor, and Insight |
|
||||
| `context-administration` | Context upgrade and destructive purge |
|
||||
| `memory-recall` | Memory/session search and condensed-output recovery |
|
||||
| `memory-management` | Durable memory add/replace/remove |
|
||||
| `skill-management` | Pi-native skill management |
|
||||
| `mcp-management` | Shared MCP adapter management |
|
||||
|
||||
Pinned tools are removed from these dynamic groups at runtime. For example, the default-pinned `codegraph_explore` and `lsp_diagnostics` do not consume the `code-intelligence` group budget.
|
||||
|
||||
## Loading and eviction
|
||||
|
||||
Call `tool_search` with an exact catalog group id when possible:
|
||||
|
||||
```json
|
||||
{ "group": "web-tavily" }
|
||||
```
|
||||
|
||||
A natural-language `query` can rank checked-in, generated, and deterministic group metadata when the group is unclear. Loading a group exposes the original full schemas for all tools in that group on Pi's next model request.
|
||||
|
||||
Pinned tools do not consume the dynamic group budget. Dynamic groups update their recency when loaded or when any member starts executing. Loading beyond `maxActiveGroups` or `maxDynamicTools` evicts least-recently-used, non-pinned groups before activating the new group.
|
||||
|
||||
The first loads are purely additive and retain Pi's native deferred-loading propagation. An eviction necessarily removes and adds schemas in one transition, so Pi uses its safe fallback rather than treating that transition as purely additive.
|
||||
|
||||
## Configuration
|
||||
|
||||
The bundle adds these fields only when missing and never replaces explicit user choices:
|
||||
|
||||
```json
|
||||
{
|
||||
"toolSearch": {
|
||||
"alwaysEnabled": ["codegraph_explore", "lsp_diagnostics"],
|
||||
"showToolSearchFooterStatus": false,
|
||||
"maxActiveGroups": 3,
|
||||
"maxToolsPerGroup": 8,
|
||||
"maxDynamicTools": 20,
|
||||
"groupOverrides": {}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
| Key | Bundle default | Description |
|
||||
|---|---|---|
|
||||
| `alwaysEnabled` | `codegraph_explore`, `lsp_diagnostics` | Exact names pinned alongside Pi core tools |
|
||||
| `showToolSearchFooterStatus` | `false` | Show active tool/group counts |
|
||||
| `maxActiveGroups` | `3` | Maximum simultaneously active dynamic groups |
|
||||
| `maxToolsPerGroup` | `8` | Maximum size accepted from model-generated groups |
|
||||
| `maxDynamicTools` | `20` | Hard cap across active dynamic group tools |
|
||||
| `groupOverrides` | `{}` | Exact tool lists that override checked-in, generated, or fallback assignments |
|
||||
|
||||
Example override:
|
||||
|
||||
```json
|
||||
{
|
||||
"toolSearch": {
|
||||
"groupOverrides": {
|
||||
"web-tavily": ["tavily_web_search", "tavily_web_fetch"],
|
||||
"lsp": ["lsp_definition", "lsp_references", "lsp_hover", "lsp_symbols", "lsp_diagnostics"]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
An explicit override larger than `maxDynamicTools` remains visible but cannot be activated until the limit or override is corrected. Invalid `settings.json` or an invalid `toolSearch` value is never overwritten.
|
||||
|
||||
## Current host integration
|
||||
|
||||
This version directly targets `@earendil-works/pi-coding-agent`, current `typebox`, `ModelRegistry.complete()`, and `ExtensionAPI.setActiveTools()`. It does not load or alias `@mariozechner/pi-coding-agent`, install another Pi runtime, rewrite provider payloads, proxy tool execution, or enqueue hidden `sendMessage` loops.
|
||||
|
||||
## Repository layout
|
||||
|
||||
- `extensions/index.ts` — Pi lifecycle, model call, activation, and group LRU
|
||||
- `extensions/catalog.ts` — hashing, strict catalog validation, fallback grouping, cache, and ranking
|
||||
- `extensions/bundle-groups.ts` — checked-in authoritative groups for tools exposed by `my-pi`
|
||||
- `extensions/config.ts` — safe bundle-default deployment and config parsing
|
||||
- `test/` — catalog, config, model-generation, and active-set tests
|
||||
- `docs/dynamic-tool-loading.md` — current Pi propagation and replacement behavior
|
||||
- `UPSTREAM.md` — imported snapshot and local divergence
|
||||
|
||||
## Development
|
||||
|
||||
```bash
|
||||
npm install
|
||||
npm run typecheck
|
||||
npm run test
|
||||
npm run build
|
||||
```
|
||||
|
||||
The root bundle loads `./pi-tool-search/extensions/index.ts` directly and references this package through `file:./pi-tool-search`.
|
||||
@@ -0,0 +1,18 @@
|
||||
# Upstream provenance
|
||||
|
||||
- Upstream: <https://github.com/tuansondinh/pi-tool-search>
|
||||
- Imported tag: `v0.3.6`
|
||||
- Imported commit: `ddfb23646fd3957b791214de278e23aa393c9b13`
|
||||
- Snapshot date: 2026-08-19
|
||||
- Upstream package version: `0.3.6`
|
||||
- License declared by upstream package metadata: MIT
|
||||
|
||||
The upstream repository did not contain a `LICENSE` file at this snapshot. This directory adds the standard MIT text using the author identity declared in upstream `package.json`.
|
||||
|
||||
## Local maintenance
|
||||
|
||||
This directory is maintained directly by the `my-pi` repository. It is not a submodule and must not contain a nested `.git`, `.pi` state, `node_modules`, or generated build output.
|
||||
|
||||
The initial local adaptation replaces the retired `@mariozechner/pi-coding-agent` and `@sinclair/typebox` imports with the current `@earendil-works/pi-coding-agent` and `typebox` host APIs, and removes the hidden `sendMessage` steer workaround.
|
||||
|
||||
The maintained implementation now diverges further from the upstream add-only exact-name manifest: it ships authoritative groups for the tools exposed by `my-pi`, optionally builds strictly validated model metadata only for unrecognized third-party tools, activates original schemas by group, and bounds long-session growth with pinned tools plus group-level LRU eviction. Standard bundle users need neither first-use model generation nor a user cache. Pure additions use current Pi's native incremental propagation; replacements intentionally use the host's safe fallback.
|
||||
@@ -0,0 +1,17 @@
|
||||
# Current dynamic tool-group loading
|
||||
|
||||
`pi-tool-search` keeps full low-frequency schemas hidden behind a compact, validated group catalog.
|
||||
|
||||
1. `session_start` activates `tool_search`, the six Pi core tools, and configured `alwaysEnabled` names. These pinned tools do not consume the dynamic-group budget.
|
||||
2. Hidden definitions are hashed with grouping constraints and matched against `extensions/bundle-groups.ts`. Available standard tools receive checked-in groups; unavailable optional tools are omitted.
|
||||
3. If every hidden tool is recognized, the bundle catalog is used directly. No model call or per-user cache is required.
|
||||
4. Additional tools receive immediate deterministic prefix/source groups. A matching model-enriched cache is reused only if it preserves all checked-in bundle assignments.
|
||||
5. Only a hybrid catalog with unrecognized tools can call `ModelRegistry.complete()`. The prompt marks bundle assignments as fixed; validation rejects any response that moves them.
|
||||
6. Code validation still requires every exact tool name exactly once and rejects unknown names, duplicates, oversized generated groups, omissions, and tool-card/group mismatches. Generated text never changes executable schemas.
|
||||
7. Loading a group calls `setActiveTools()` with its original full tool definitions. Pi supplies newly added schemas on the next model request.
|
||||
8. Group load and member execution update an in-session LRU counter. Loading past `maxActiveGroups` or `maxDynamicTools` removes the least-recently-used dynamic groups.
|
||||
9. `turn_start` notices tool or grouping-configuration changes by hash, resets stale dynamic groups, and reconstructs the checked-in/hybrid catalog.
|
||||
|
||||
A purely additive first load receives Pi's native dynamic-tool result propagation. A replacement that evicts one group while adding another is intentionally non-additive; current Pi detects the removal and uses its safe next-request fallback instead of attaching an invalid additive-only hint.
|
||||
|
||||
A model cannot call a schema it had not received in an already-generated assistant response. `tool_search` therefore returns normally, and the next model request sees the activated group's complete schemas. No provider payload rewriting, proxy dispatch, hidden steer message, old package alias, or duplicate Pi runtime is used.
|
||||
@@ -0,0 +1,134 @@
|
||||
# Same-turn tool activation race — findings
|
||||
|
||||
> Historical upstream document from snapshot `v0.3.6`. The locally maintained implementation no longer uses the old hidden steer/retry mitigation described below. Current behavior is documented in [`dynamic-tool-loading.md`](./dynamic-tool-loading.md).
|
||||
|
||||
## Bug
|
||||
|
||||
When the LLM emits parallel tool calls in a single assistant message — for
|
||||
example `tool_search(["subagent"])` together with `subagent(...)` — the second
|
||||
call fails with `Tool subagent not found`.
|
||||
|
||||
Live repro (observed in this project, 2026-04-23):
|
||||
|
||||
1. Turn A: `tool_search(["subagent"])` → `Enabled: subagent`; `subagent(...)`
|
||||
in the same reply → ❌ `Tool not found`.
|
||||
2. Turn B (agent continuation, no user input): `subagent(...)` → ❌ still not
|
||||
found.
|
||||
3. Turn C (after fresh user message): `subagent(...)` → ✅ works.
|
||||
|
||||
## Root cause
|
||||
|
||||
Two compounding issues in pi-coding-agent core.
|
||||
|
||||
### 1. Tool schema is frozen per provider request
|
||||
|
||||
`setActiveTools(["tool_search", ...unlocked])` updates `agent.state.tools`, but
|
||||
the outgoing LLM request payload (`tools: [...]`) is built **before**
|
||||
`tool_search` executes. The model that produced the parallel call never saw
|
||||
`subagent` in its tool schema — it guessed based on `tool_search`'s
|
||||
description.
|
||||
|
||||
### 2. `before_agent_start` only fires on user-initiated turns
|
||||
|
||||
The extension's hook that re-applies `setActiveTools` runs once per fresh user
|
||||
turn. Agentic-loop continuations (model → tool → model → tool…) do **not**
|
||||
re-trigger it. So even after `tool_search` mutates state in Turn A, the next
|
||||
LLM call in the same loop still uses the stale schema from Turn A's start.
|
||||
|
||||
## What we checked
|
||||
|
||||
- `pi-coding-agent/dist/core/agent-session.js:550` —
|
||||
`setActiveToolsByName` assigns `this.agent.state.tools = tools`. Pure state
|
||||
mutation, no schema re-send.
|
||||
- `pi-coding-agent/dist/core/agent-session.js:172` — `beforeToolCall` hook
|
||||
fires only for resolved tools; unknown names never reach it.
|
||||
- `pi-coding-agent/dist/core/sdk.js:195` — `onPayload(payload, model)`
|
||||
receives the **already-built, provider-specific** request. Tools are baked
|
||||
in.
|
||||
- `ExtensionAPI.getAllTools(): ToolInfo[]` — `ToolInfo = Pick<ToolDefinition,
|
||||
"name"|"description"|"parameters"> & { sourceInfo }`. **No `execute`.**
|
||||
- `ExtensionAPI.on("tool_call", ...)` result type is `{ block?, reason? }` —
|
||||
cannot replace the tool result.
|
||||
|
||||
## Extension-only fixes considered
|
||||
|
||||
| Option | Fixes parallel calls | Fixes mid-loop | Fragile | Viable |
|
||||
|---|---|---|---|---|
|
||||
| Prompt mitigation (tell model to call `tool_search` alone) | ❌ | ❌ | — | ✅ shipped |
|
||||
| `before_provider_request` payload rewrite | ❌ (too late) | ✅ | ✅ yes | ⚠️ |
|
||||
| Proxy `invoke(tool, args)` tool | ✅ | ✅ | — | ❌ dead |
|
||||
|
||||
### Why the proxy `invoke` pattern is dead
|
||||
|
||||
`pi.getAllTools()` returns metadata only. The callable `execute()` lives on
|
||||
`ToolDefinition` / `RegisteredTool`, but no public API returns those. An
|
||||
extension cannot dispatch another extension's tool programmatically.
|
||||
|
||||
### Why `before_provider_request` rewrite is fragile
|
||||
|
||||
The hook receives the raw provider payload. To inject unlocked tool schemas
|
||||
mid-loop you would need per-provider surgery:
|
||||
|
||||
- OpenAI: `payload.tools = [{ type: "function", function: { name, description, parameters } }]`
|
||||
- Anthropic: `payload.tools = [{ name, description, input_schema }]`
|
||||
- Gemini: `payload.tools = [{ functionDeclarations: [...] }]`
|
||||
- plus Azure / Copilot / Codex adapters, plus TypeBox → JSON-Schema
|
||||
conversion with provider-specific quirks (Gemini's restricted subset, etc.).
|
||||
|
||||
Breaks every time a provider is added or its payload shape changes. Duplicates
|
||||
logic pi-coding-agent already has internally.
|
||||
|
||||
It also does **not** fix the parallel-call case — by the time
|
||||
`before_provider_request` fires, the model has already emitted both calls.
|
||||
|
||||
## Current mitigation (shipped)
|
||||
|
||||
`extensions/index.ts` updates:
|
||||
|
||||
- `tool_search` description explicitly says: "STOP after calling
|
||||
tool_search. Do NOT call newly-enabled tools in same response. Tool
|
||||
schema is frozen for current response."
|
||||
- `promptSnippet` reinforces rule.
|
||||
- Active tools now refresh on every `turn_start`, not only fresh user prompts,
|
||||
so unlocked tools stay available during agent-loop continuations.
|
||||
- Successful `tool_search` queues hidden steer hint telling model to continue
|
||||
original task in next turn and retry any same-response failure immediately.
|
||||
|
||||
This does not make same-response parallel call succeed, but it removes need for
|
||||
fresh user message in common recovery path.
|
||||
|
||||
## Real fix (upstream)
|
||||
|
||||
The structural fix belongs in `pi-coding-agent`:
|
||||
|
||||
1. **Re-resolve active tools per batch iteration.** Don't snapshot at batch
|
||||
start; look up each tool name when it's about to be dispatched.
|
||||
2. **Sequential dispatch when a mutating tool is in the batch.** Or always
|
||||
sequential — parallel `Promise.all` dispatch races `setActiveTools`.
|
||||
3. **Re-send schema after mid-loop `setActiveTools`.** Or mark the active-tool
|
||||
set as dirty and rebuild payload on next provider request.
|
||||
4. **Graceful unknown-tool result.** When a dispatched tool name is not in the
|
||||
active set, return a structured tool_result (`"tool X not active this
|
||||
turn — retry next turn"`) instead of falling through to MCP or throwing.
|
||||
|
||||
With (1)+(2) alone, the parallel-call case works. With (3), mid-loop
|
||||
continuations also work. (4) makes the failure mode recoverable by the model
|
||||
when it does speculate.
|
||||
|
||||
## Recommended action
|
||||
|
||||
- Keep prompt mitigation + per-turn refresh + hidden retry hint in
|
||||
`extensions/index.ts`.
|
||||
- Document remaining limitation in README (same-response parallel call with
|
||||
`tool_search` can still fail until core is fixed).
|
||||
- File upstream issue against `pi-coding-agent` referencing this doc.
|
||||
|
||||
## File map
|
||||
|
||||
| Path | Role |
|
||||
|---|---|
|
||||
| `extensions/index.ts` | This extension; prompt mitigation lives here |
|
||||
| `pi-coding-agent/dist/core/agent-session.js:550` | `setActiveToolsByName` — state-only mutation |
|
||||
| `pi-coding-agent/dist/core/agent-session.js:172` | `beforeToolCall` hook — only fires for resolved tools |
|
||||
| `pi-coding-agent/dist/core/sdk.js:195` | `onPayload` — already-built payload; provider-specific |
|
||||
| `pi-coding-agent/dist/core/extensions/types.d.ts` | `ToolInfo` (metadata only), `ToolCallEventResult` (`{block, reason}`) |
|
||||
@@ -0,0 +1,234 @@
|
||||
import type { ToolInfo } from "@earendil-works/pi-coding-agent";
|
||||
|
||||
import {
|
||||
CATALOG_VERSION,
|
||||
applyCatalogOverrides,
|
||||
createFallbackCatalog,
|
||||
type CatalogConstraints,
|
||||
type GroupCard,
|
||||
type ToolCard,
|
||||
type ToolCatalog,
|
||||
} from "./catalog.ts";
|
||||
|
||||
interface BundleGroupDefinition {
|
||||
id: string;
|
||||
title: string;
|
||||
summary: string;
|
||||
useWhen: string[];
|
||||
avoidWhen: string[];
|
||||
keywords: string[];
|
||||
tools: string[];
|
||||
}
|
||||
|
||||
/** Curated groups for every non-core tool exposed by the my-pi bundle. */
|
||||
export const BUNDLE_GROUP_DEFINITIONS: BundleGroupDefinition[] = [
|
||||
{
|
||||
id: "filesystem-navigation",
|
||||
title: "Filesystem navigation",
|
||||
summary: "Inspect directory layouts and run bounded multi-pattern literal searches.",
|
||||
useWhen: ["You need directory entries or several literal searches after narrowing a path"],
|
||||
avoidWhen: ["Code relationships or types are the real question"],
|
||||
keywords: ["files", "directory", "list", "multi grep", "目录", "文件", "多模式搜索"],
|
||||
tools: ["ls", "multi_grep"],
|
||||
},
|
||||
{
|
||||
id: "code-intelligence",
|
||||
title: "Code intelligence",
|
||||
summary: "Explore code relationships and query language-server definitions, references, types, symbols, and diagnostics.",
|
||||
useWhen: ["Understanding architecture, call paths, symbol identity, types, references, or compiler diagnostics"],
|
||||
avoidWhen: ["Only an exact literal text match is required"],
|
||||
keywords: ["codegraph", "lsp", "definition", "references", "diagnostics", "代码结构", "定义", "引用", "诊断"],
|
||||
tools: ["codegraph_explore", "lsp_definition", "lsp_references", "lsp_hover", "lsp_symbols", "lsp_diagnostics"],
|
||||
},
|
||||
{
|
||||
id: "web-tavily",
|
||||
title: "Tavily web discovery",
|
||||
summary: "Discover broad, current, or news-oriented web sources with Tavily and fetch selected pages.",
|
||||
useWhen: ["Keywords are uncertain, many candidate sources help, or current/news coverage is needed"],
|
||||
avoidWhen: ["Official technical sources or Chinese site/date constraints are more important"],
|
||||
keywords: ["tavily", "web", "news", "discovery", "网页", "新闻", "广泛搜索"],
|
||||
tools: ["tavily_web_search", "tavily_web_fetch"],
|
||||
},
|
||||
{
|
||||
id: "web-exa",
|
||||
title: "Exa precision research",
|
||||
summary: "Find precise official, technical, academic, company, or API sources with Exa and read selected pages.",
|
||||
useWhen: ["Source precision, official documentation, papers, releases, or advanced filters matter"],
|
||||
avoidWhen: ["Broad news discovery or Chinese policy search is the primary need"],
|
||||
keywords: ["exa", "official docs", "paper", "api", "research", "官方文档", "论文", "精确搜索"],
|
||||
tools: ["exa_web_search", "exa_web_search_advanced", "exa_web_fetch"],
|
||||
},
|
||||
{
|
||||
id: "web-keenable",
|
||||
title: "Keenable focused search",
|
||||
summary: "Search Chinese-language, site-restricted, policy, government, or date-filtered sources and fetch selected pages.",
|
||||
useWhen: ["Chinese pages, policies, government notices, domain restrictions, or publication dates matter"],
|
||||
avoidWhen: ["Broad international discovery or academic semantic search is a better fit"],
|
||||
keywords: ["keenable", "Chinese", "policy", "site", "date", "中文", "政策", "站点", "日期"],
|
||||
tools: ["keenable_search", "keenable_fetch"],
|
||||
},
|
||||
{
|
||||
id: "context-execution",
|
||||
title: "Context-isolated execution",
|
||||
summary: "Run commands or analyze large files in a sandbox while returning only compact derived output.",
|
||||
useWhen: ["Logs, tests, builds, generated data, commands, or workspace files may produce large output"],
|
||||
avoidWhen: ["Exact source lines are needed for an anchored edit"],
|
||||
keywords: ["execute", "large file", "logs", "tests", "build", "大文件", "日志", "测试", "构建"],
|
||||
tools: ["ctx_execute", "ctx_execute_file", "ctx_batch_execute"],
|
||||
},
|
||||
{
|
||||
id: "context-knowledge",
|
||||
title: "Context knowledge base",
|
||||
summary: "Index local or web documentation and retrieve focused passages from the persistent context-mode knowledge base.",
|
||||
useWhen: ["Documentation or session knowledge should be stored and queried without rereading raw content"],
|
||||
avoidWhen: ["A one-shot small source can be read directly"],
|
||||
keywords: ["index", "search", "knowledge base", "fetch docs", "索引", "知识库", "文档检索"],
|
||||
tools: ["ctx_index", "ctx_search", "ctx_fetch_and_index"],
|
||||
},
|
||||
{
|
||||
id: "context-observability",
|
||||
title: "Context observability",
|
||||
summary: "Inspect context-mode health, savings statistics, and the hosted Insight dashboard.",
|
||||
useWhen: ["Diagnosing context-mode or reviewing context savings and analytics"],
|
||||
avoidWhen: ["The task is normal code or data processing"],
|
||||
keywords: ["stats", "doctor", "insight", "health", "统计", "诊断", "上下文节省"],
|
||||
tools: ["ctx_stats", "ctx_doctor", "ctx_insight"],
|
||||
},
|
||||
{
|
||||
id: "context-administration",
|
||||
title: "Context administration",
|
||||
summary: "Upgrade context-mode or destructively purge indexed context data.",
|
||||
useWhen: ["The user explicitly asks to upgrade context-mode or purge a named scope"],
|
||||
avoidWhen: ["Routine searching, indexing, or performance inspection"],
|
||||
keywords: ["upgrade", "purge", "delete", "升级", "清除", "删除知识库"],
|
||||
tools: ["ctx_upgrade", "ctx_purge"],
|
||||
},
|
||||
{
|
||||
id: "memory-recall",
|
||||
title: "Memory and session recall",
|
||||
summary: "Search durable memories, prior sessions, or recover pruned tool-call output.",
|
||||
useWhen: ["Past decisions, preferences, failures, conversations, or condensed outputs may answer the question"],
|
||||
avoidWhen: ["The required evidence is already in the current visible context"],
|
||||
keywords: ["memory", "session", "history", "recover", "记忆", "历史会话", "恢复输出"],
|
||||
tools: ["memory_search", "session_search", "context_tree_query"],
|
||||
},
|
||||
{
|
||||
id: "memory-management",
|
||||
title: "Memory management",
|
||||
summary: "Add, replace, or remove durable user, project, global, or failure memories.",
|
||||
useWhen: ["A stable preference, correction, environment fact, convention, or durable lesson should change"],
|
||||
avoidWhen: ["The information is temporary task progress"],
|
||||
keywords: ["remember", "add memory", "replace memory", "forget", "记住", "更新记忆", "删除记忆"],
|
||||
tools: ["memory_add", "memory_replace", "memory_remove"],
|
||||
},
|
||||
{
|
||||
id: "skill-management",
|
||||
title: "Procedural skill management",
|
||||
summary: "Create, inspect, patch, update, or remove reusable Pi-native procedural skills.",
|
||||
useWhen: ["A reusable workflow or non-obvious procedure should persist across sessions"],
|
||||
avoidWhen: ["Saving one-off task state or a generic summary"],
|
||||
keywords: ["skill", "procedure", "workflow", "技能", "流程", "工作流"],
|
||||
tools: ["skill_manage"],
|
||||
},
|
||||
{
|
||||
id: "mcp-management",
|
||||
title: "MCP management",
|
||||
summary: "Inspect and manage MCP server connections exposed by Pi's shared adapter.",
|
||||
useWhen: ["The user explicitly asks about MCP server status or operations"],
|
||||
avoidWhen: ["A mapped first-class MCP tool already handles the task"],
|
||||
keywords: ["mcp", "server", "connection", "MCP 服务", "连接"],
|
||||
tools: ["mcp"],
|
||||
},
|
||||
];
|
||||
|
||||
export interface BundleCatalogResult {
|
||||
catalog: ToolCatalog;
|
||||
coveredNames: Set<string>;
|
||||
unknownTools: ToolInfo[];
|
||||
}
|
||||
|
||||
function compactDescription(tool: ToolInfo): string {
|
||||
return tool.description.replace(/\s+/gu, " ").trim().slice(0, 240) || `Use the ${tool.name} tool.`;
|
||||
}
|
||||
|
||||
export function createBundleCatalog(
|
||||
tools: ToolInfo[],
|
||||
manifestHash: string,
|
||||
constraints: CatalogConstraints,
|
||||
): BundleCatalogResult {
|
||||
const available = new Map(tools.map((tool) => [tool.name, tool]));
|
||||
const coveredNames = new Set<string>();
|
||||
const groups: GroupCard[] = [];
|
||||
const cards: ToolCard[] = [];
|
||||
|
||||
for (const definition of BUNDLE_GROUP_DEFINITIONS) {
|
||||
const members = definition.tools.flatMap((name) => {
|
||||
const tool = available.get(name);
|
||||
return tool ? [tool] : [];
|
||||
});
|
||||
if (members.length === 0) continue;
|
||||
for (const tool of members) {
|
||||
coveredNames.add(tool.name);
|
||||
cards.push({
|
||||
name: tool.name,
|
||||
summary: compactDescription(tool),
|
||||
useWhen: [...definition.useWhen],
|
||||
avoidWhen: [...definition.avoidWhen],
|
||||
keywords: [...new Set([...definition.keywords, ...tool.name.split(/[_-]/u)])],
|
||||
primaryGroup: definition.id,
|
||||
});
|
||||
}
|
||||
groups.push({
|
||||
id: definition.id,
|
||||
title: definition.title,
|
||||
summary: definition.summary,
|
||||
useWhen: [...definition.useWhen],
|
||||
avoidWhen: [...definition.avoidWhen],
|
||||
tools: members.map((tool) => tool.name),
|
||||
});
|
||||
}
|
||||
|
||||
const unknownTools = tools.filter((tool) => !coveredNames.has(tool.name));
|
||||
const fallback = createFallbackCatalog(unknownTools, manifestHash, {
|
||||
maxToolsPerGroup: constraints.maxToolsPerGroup,
|
||||
groupOverrides: {},
|
||||
});
|
||||
const reservedIds = new Set(groups.map((group) => group.id));
|
||||
const renamedGroups = new Map<string, string>();
|
||||
for (const group of fallback.groups) {
|
||||
let id = group.id;
|
||||
while (reservedIds.has(id)) id = `custom-${id}`;
|
||||
reservedIds.add(id);
|
||||
renamedGroups.set(group.id, id);
|
||||
groups.push({ ...group, id, title: id.replace(/-/gu, " ") });
|
||||
}
|
||||
for (const card of fallback.tools) {
|
||||
cards.push({ ...card, primaryGroup: renamedGroups.get(card.primaryGroup) ?? card.primaryGroup });
|
||||
}
|
||||
|
||||
const overridden = applyCatalogOverrides(groups, cards, constraints.groupOverrides);
|
||||
return {
|
||||
catalog: {
|
||||
version: CATALOG_VERSION,
|
||||
manifestHash,
|
||||
generatedAt: new Date().toISOString(),
|
||||
generatedBy: "my-pi bundle seed",
|
||||
groups: overridden.groups,
|
||||
tools: overridden.tools,
|
||||
},
|
||||
coveredNames,
|
||||
unknownTools,
|
||||
};
|
||||
}
|
||||
|
||||
export function catalogPreservesBundleAssignments(
|
||||
candidate: ToolCatalog,
|
||||
bundleCatalog: ToolCatalog,
|
||||
coveredNames: Set<string>,
|
||||
): boolean {
|
||||
const expected = new Map(
|
||||
bundleCatalog.tools.filter((tool) => coveredNames.has(tool.name)).map((tool) => [tool.name, tool.primaryGroup]),
|
||||
);
|
||||
const actual = new Map(candidate.tools.map((tool) => [tool.name, tool.primaryGroup]));
|
||||
return [...expected].every(([name, group]) => actual.get(name) === group);
|
||||
}
|
||||
@@ -0,0 +1,366 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { chmodSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
|
||||
import type { ToolInfo } from "@earendil-works/pi-coding-agent";
|
||||
|
||||
export const CATALOG_VERSION = 1;
|
||||
|
||||
export interface ToolCard {
|
||||
name: string;
|
||||
summary: string;
|
||||
useWhen: string[];
|
||||
avoidWhen: string[];
|
||||
keywords: string[];
|
||||
primaryGroup: string;
|
||||
}
|
||||
|
||||
export interface GroupCard {
|
||||
id: string;
|
||||
title: string;
|
||||
summary: string;
|
||||
useWhen: string[];
|
||||
avoidWhen: string[];
|
||||
tools: string[];
|
||||
}
|
||||
|
||||
export interface ToolCatalog {
|
||||
version: number;
|
||||
manifestHash: string;
|
||||
generatedAt: string;
|
||||
generatedBy?: string;
|
||||
groups: GroupCard[];
|
||||
tools: ToolCard[];
|
||||
}
|
||||
|
||||
export interface CatalogConstraints {
|
||||
maxToolsPerGroup: number;
|
||||
groupOverrides: Record<string, string[]>;
|
||||
}
|
||||
|
||||
interface StoredCatalog {
|
||||
version: number;
|
||||
catalog: ToolCatalog;
|
||||
}
|
||||
|
||||
function compactText(value: unknown, maxLength: number): string {
|
||||
if (typeof value !== "string") return "";
|
||||
return value.replace(/\s+/gu, " ").trim().slice(0, maxLength);
|
||||
}
|
||||
|
||||
function compactStrings(value: unknown, maxItems: number, maxLength: number): string[] {
|
||||
if (!Array.isArray(value)) return [];
|
||||
return value
|
||||
.filter((item): item is string => typeof item === "string")
|
||||
.map((item) => compactText(item, maxLength))
|
||||
.filter(Boolean)
|
||||
.slice(0, maxItems);
|
||||
}
|
||||
|
||||
function normalizeGroupId(value: string): string {
|
||||
return value
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/gu, "-")
|
||||
.replace(/^-+|-+$/gu, "")
|
||||
.slice(0, 64);
|
||||
}
|
||||
|
||||
function stableValue(value: unknown, seen = new WeakSet<object>()): unknown {
|
||||
if (value === null || typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
||||
return value;
|
||||
}
|
||||
if (Array.isArray(value)) return value.map((item) => stableValue(item, seen));
|
||||
if (typeof value !== "object") return undefined;
|
||||
if (seen.has(value)) return "[Circular]";
|
||||
seen.add(value);
|
||||
const output: Record<string, unknown> = {};
|
||||
for (const key of Object.keys(value as Record<string, unknown>).sort()) {
|
||||
const child = stableValue((value as Record<string, unknown>)[key], seen);
|
||||
if (child !== undefined) output[key] = child;
|
||||
}
|
||||
seen.delete(value);
|
||||
return output;
|
||||
}
|
||||
|
||||
export function buildManifestHash(tools: ToolInfo[], constraints: CatalogConstraints): string {
|
||||
const manifest = tools
|
||||
.map((tool) => ({
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
parameters: stableValue(tool.parameters),
|
||||
promptGuidelines: tool.promptGuidelines,
|
||||
sourceInfo: stableValue(tool.sourceInfo),
|
||||
}))
|
||||
.sort((left, right) => left.name.localeCompare(right.name));
|
||||
const payload = stableValue({ manifest, constraints });
|
||||
return createHash("sha256").update(JSON.stringify(payload)).digest("hex");
|
||||
}
|
||||
|
||||
export function buildCatalogModelInput(tools: ToolInfo[]): string {
|
||||
const input = tools
|
||||
.map((tool) => ({
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
parameters: stableValue(tool.parameters),
|
||||
promptGuidelines: tool.promptGuidelines,
|
||||
sourceInfo: stableValue(tool.sourceInfo),
|
||||
}))
|
||||
.sort((left, right) => left.name.localeCompare(right.name));
|
||||
return JSON.stringify(input);
|
||||
}
|
||||
|
||||
function fallbackGroupKey(tool: ToolInfo): string {
|
||||
const prefix = tool.name.includes("_") ? tool.name.split("_", 1)[0] : "";
|
||||
if (prefix && prefix.length > 1) return normalizeGroupId(prefix);
|
||||
|
||||
const source = stableValue(tool.sourceInfo);
|
||||
if (source && typeof source === "object") {
|
||||
const record = source as Record<string, unknown>;
|
||||
for (const key of ["package", "name", "path", "source"]) {
|
||||
const candidate = record[key];
|
||||
if (typeof candidate !== "string") continue;
|
||||
const parts = candidate.split(/[\\/]/u);
|
||||
const normalized = normalizeGroupId(parts.at(-1)?.replace(/\.[^.]+$/u, "") ?? candidate);
|
||||
if (normalized) return normalized;
|
||||
}
|
||||
}
|
||||
return "other-tools";
|
||||
}
|
||||
|
||||
export function applyCatalogOverrides(
|
||||
groups: GroupCard[],
|
||||
toolCards: ToolCard[],
|
||||
overrides: Record<string, string[]>,
|
||||
): { groups: GroupCard[]; tools: ToolCard[] } {
|
||||
const knownNames = new Set(toolCards.map((tool) => tool.name));
|
||||
const overridden = new Map<string, string>();
|
||||
for (const [rawId, names] of Object.entries(overrides)) {
|
||||
const id = normalizeGroupId(rawId);
|
||||
if (!id) continue;
|
||||
for (const name of names) {
|
||||
if (knownNames.has(name)) overridden.set(name, id);
|
||||
}
|
||||
}
|
||||
if (overridden.size === 0) return { groups, tools: toolCards };
|
||||
|
||||
const byId = new Map<string, GroupCard>();
|
||||
for (const group of groups) {
|
||||
const remaining = group.tools.filter((name) => !overridden.has(name));
|
||||
if (remaining.length > 0) byId.set(group.id, { ...group, tools: remaining });
|
||||
}
|
||||
for (const [name, id] of overridden) {
|
||||
const existing = byId.get(id);
|
||||
if (existing) existing.tools.push(name);
|
||||
else {
|
||||
byId.set(id, {
|
||||
id,
|
||||
title: id.replace(/-/gu, " "),
|
||||
summary: `Configured tool group for ${id.replace(/-/gu, " ")}.`,
|
||||
useWhen: [],
|
||||
avoidWhen: [],
|
||||
tools: [name],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const updatedTools = toolCards.map((tool) => ({
|
||||
...tool,
|
||||
primaryGroup: overridden.get(tool.name) ?? tool.primaryGroup,
|
||||
}));
|
||||
return { groups: [...byId.values()], tools: updatedTools };
|
||||
}
|
||||
|
||||
export function createFallbackCatalog(
|
||||
tools: ToolInfo[],
|
||||
manifestHash: string,
|
||||
constraints: CatalogConstraints,
|
||||
): ToolCatalog {
|
||||
const cards = tools.map<ToolCard>((tool) => ({
|
||||
name: tool.name,
|
||||
summary: compactText(tool.description, 240) || `Use the ${tool.name} tool.`,
|
||||
useWhen: [],
|
||||
avoidWhen: [],
|
||||
keywords: tool.name.split(/[_-]/u).filter(Boolean),
|
||||
primaryGroup: fallbackGroupKey(tool),
|
||||
}));
|
||||
|
||||
const buckets = new Map<string, ToolCard[]>();
|
||||
for (const card of cards) {
|
||||
const bucket = buckets.get(card.primaryGroup) ?? [];
|
||||
bucket.push(card);
|
||||
buckets.set(card.primaryGroup, bucket);
|
||||
}
|
||||
|
||||
const groups: GroupCard[] = [];
|
||||
for (const [baseId, bucket] of [...buckets].sort(([left], [right]) => left.localeCompare(right))) {
|
||||
for (let index = 0; index < bucket.length; index += constraints.maxToolsPerGroup) {
|
||||
const chunk = bucket.slice(index, index + constraints.maxToolsPerGroup);
|
||||
const suffix = index === 0 ? "" : `-${Math.floor(index / constraints.maxToolsPerGroup) + 1}`;
|
||||
const id = `${baseId}${suffix}`;
|
||||
for (const card of chunk) card.primaryGroup = id;
|
||||
groups.push({
|
||||
id,
|
||||
title: id.replace(/-/gu, " "),
|
||||
summary: compactText(chunk.map((tool) => tool.summary).join("; "), 280),
|
||||
useWhen: [],
|
||||
avoidWhen: [],
|
||||
tools: chunk.map((tool) => tool.name),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const overridden = applyCatalogOverrides(groups, cards, constraints.groupOverrides);
|
||||
return {
|
||||
version: CATALOG_VERSION,
|
||||
manifestHash,
|
||||
generatedAt: new Date().toISOString(),
|
||||
groups: overridden.groups,
|
||||
tools: overridden.tools,
|
||||
};
|
||||
}
|
||||
|
||||
function parseJsonObject(text: string): unknown {
|
||||
const unfenced = text.replace(/^\s*```(?:json)?\s*/iu, "").replace(/\s*```\s*$/u, "");
|
||||
const start = unfenced.indexOf("{");
|
||||
const end = unfenced.lastIndexOf("}");
|
||||
if (start < 0 || end <= start) throw new Error("catalog response did not contain a JSON object");
|
||||
return JSON.parse(unfenced.slice(start, end + 1));
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
export function parseGeneratedCatalog(
|
||||
text: string,
|
||||
tools: ToolInfo[],
|
||||
manifestHash: string,
|
||||
constraints: CatalogConstraints,
|
||||
generatedBy: string,
|
||||
): ToolCatalog {
|
||||
const parsed = parseJsonObject(text);
|
||||
if (!isRecord(parsed) || !Array.isArray(parsed.groups) || !Array.isArray(parsed.tools)) {
|
||||
throw new Error("catalog response must contain groups and tools arrays");
|
||||
}
|
||||
|
||||
const expectedNames = new Set(tools.map((tool) => tool.name));
|
||||
const cards: ToolCard[] = [];
|
||||
const cardNames = new Set<string>();
|
||||
for (const value of parsed.tools) {
|
||||
if (!isRecord(value) || typeof value.name !== "string" || typeof value.summary !== "string" || typeof value.primaryGroup !== "string") {
|
||||
throw new Error("every generated tool card needs name, summary, and primaryGroup");
|
||||
}
|
||||
if (!expectedNames.has(value.name) || cardNames.has(value.name)) throw new Error(`invalid or duplicate tool card: ${value.name}`);
|
||||
const primaryGroup = normalizeGroupId(value.primaryGroup);
|
||||
if (!primaryGroup) throw new Error(`invalid primary group for ${value.name}`);
|
||||
cardNames.add(value.name);
|
||||
cards.push({
|
||||
name: value.name,
|
||||
summary: compactText(value.summary, 240),
|
||||
useWhen: compactStrings(value.useWhen, 5, 160),
|
||||
avoidWhen: compactStrings(value.avoidWhen, 5, 160),
|
||||
keywords: compactStrings(value.keywords, 16, 60),
|
||||
primaryGroup,
|
||||
});
|
||||
}
|
||||
if (cardNames.size !== expectedNames.size) throw new Error("generated tool cards omitted one or more tools");
|
||||
|
||||
const groups: GroupCard[] = [];
|
||||
const groupIds = new Set<string>();
|
||||
const assignedNames = new Set<string>();
|
||||
for (const value of parsed.groups) {
|
||||
if (!isRecord(value) || typeof value.id !== "string" || typeof value.title !== "string" || typeof value.summary !== "string" || !Array.isArray(value.tools)) {
|
||||
throw new Error("every generated group needs id, title, summary, and tools");
|
||||
}
|
||||
const id = normalizeGroupId(value.id);
|
||||
if (!id || groupIds.has(id)) throw new Error(`invalid or duplicate group: ${value.id}`);
|
||||
const names = value.tools.filter((name): name is string => typeof name === "string");
|
||||
if (names.length === 0 || names.length > constraints.maxToolsPerGroup) throw new Error(`group ${id} has an invalid size`);
|
||||
for (const name of names) {
|
||||
if (!expectedNames.has(name) || assignedNames.has(name)) throw new Error(`invalid or duplicate group assignment: ${name}`);
|
||||
assignedNames.add(name);
|
||||
}
|
||||
groupIds.add(id);
|
||||
groups.push({
|
||||
id,
|
||||
title: compactText(value.title, 80),
|
||||
summary: compactText(value.summary, 280),
|
||||
useWhen: compactStrings(value.useWhen, 5, 160),
|
||||
avoidWhen: compactStrings(value.avoidWhen, 5, 160),
|
||||
tools: names,
|
||||
});
|
||||
}
|
||||
if (assignedNames.size !== expectedNames.size) throw new Error("generated groups omitted one or more tools");
|
||||
for (const card of cards) {
|
||||
const group = groups.find((candidate) => candidate.id === card.primaryGroup);
|
||||
if (!group?.tools.includes(card.name)) throw new Error(`tool card/group mismatch for ${card.name}`);
|
||||
}
|
||||
|
||||
const overridden = applyCatalogOverrides(groups, cards, constraints.groupOverrides);
|
||||
return {
|
||||
version: CATALOG_VERSION,
|
||||
manifestHash,
|
||||
generatedAt: new Date().toISOString(),
|
||||
generatedBy,
|
||||
groups: overridden.groups,
|
||||
tools: overridden.tools,
|
||||
};
|
||||
}
|
||||
|
||||
export function readCachedCatalog(
|
||||
cachePath: string,
|
||||
tools: ToolInfo[],
|
||||
manifestHash: string,
|
||||
constraints: CatalogConstraints,
|
||||
): ToolCatalog | undefined {
|
||||
try {
|
||||
const stored: unknown = JSON.parse(readFileSync(cachePath, "utf8"));
|
||||
if (!isRecord(stored) || stored.version !== CATALOG_VERSION || !isRecord(stored.catalog)) return undefined;
|
||||
const catalog = stored.catalog as unknown as ToolCatalog;
|
||||
if (catalog.manifestHash !== manifestHash) return undefined;
|
||||
return parseGeneratedCatalog(JSON.stringify(catalog), tools, manifestHash, constraints, catalog.generatedBy ?? "cached model");
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
export function writeCachedCatalog(cachePath: string, catalog: ToolCatalog): void {
|
||||
mkdirSync(dirname(cachePath), { recursive: true, mode: 0o700 });
|
||||
const temporaryPath = `${cachePath}.${process.pid}.tmp`;
|
||||
const stored: StoredCatalog = { version: CATALOG_VERSION, catalog };
|
||||
writeFileSync(temporaryPath, `${JSON.stringify(stored, null, 2)}\n`, { encoding: "utf8", mode: 0o600 });
|
||||
chmodSync(temporaryPath, 0o600);
|
||||
renameSync(temporaryPath, cachePath);
|
||||
}
|
||||
|
||||
function queryTerms(value: string): Set<string> {
|
||||
const normalized = value.toLowerCase();
|
||||
const terms = new Set(normalized.match(/[\p{L}\p{N}]+/gu) ?? []);
|
||||
const cjk = [...normalized].filter((character) => /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]/u.test(character));
|
||||
for (const character of cjk) terms.add(character);
|
||||
for (let index = 0; index + 1 < cjk.length; index += 1) terms.add(`${cjk[index]}${cjk[index + 1]}`);
|
||||
return terms;
|
||||
}
|
||||
|
||||
export function rankGroups(catalog: ToolCatalog, query: string): Array<{ group: GroupCard; score: number }> {
|
||||
const querySet = queryTerms(query);
|
||||
return catalog.groups
|
||||
.map((group) => {
|
||||
const cards = catalog.tools.filter((tool) => tool.primaryGroup === group.id);
|
||||
const searchable = [
|
||||
group.id,
|
||||
group.title,
|
||||
group.summary,
|
||||
...group.useWhen,
|
||||
...group.avoidWhen,
|
||||
...cards.flatMap((tool) => [tool.name, tool.summary, ...tool.useWhen, ...tool.keywords]),
|
||||
].join(" ");
|
||||
const terms = queryTerms(searchable);
|
||||
let score = 0;
|
||||
for (const term of querySet) if (terms.has(term)) score += term.length > 1 ? 3 : 1;
|
||||
if (group.id === query.trim().toLowerCase()) score += 100;
|
||||
return { group, score };
|
||||
})
|
||||
.sort((left, right) => right.score - left.score || left.group.id.localeCompare(right.group.id));
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
|
||||
export type ToolSearchDefaultResult = "updated" | "unchanged" | "skipped-invalid";
|
||||
|
||||
export const BUNDLE_TOOL_SEARCH_DEFAULTS = {
|
||||
alwaysEnabled: ["codegraph_explore", "lsp_diagnostics"],
|
||||
showToolSearchFooterStatus: false,
|
||||
maxActiveGroups: 3,
|
||||
maxToolsPerGroup: 8,
|
||||
maxDynamicTools: 20,
|
||||
groupOverrides: {},
|
||||
} as const;
|
||||
|
||||
export interface ToolSearchConfig {
|
||||
alwaysEnabled: string[];
|
||||
showToolSearchFooterStatus: boolean;
|
||||
maxActiveGroups: number;
|
||||
maxToolsPerGroup: number;
|
||||
maxDynamicTools: number;
|
||||
groupOverrides: Record<string, string[]>;
|
||||
}
|
||||
|
||||
function isObject(value: unknown): value is Record<string, unknown> {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function positiveInteger(value: unknown, fallback: number): number {
|
||||
return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : fallback;
|
||||
}
|
||||
|
||||
function stringList(value: unknown): string[] {
|
||||
if (!Array.isArray(value)) return [];
|
||||
return [...new Set(value.filter((item): item is string => typeof item === "string" && item.length > 0))];
|
||||
}
|
||||
|
||||
function readGroupOverrides(value: unknown): Record<string, string[]> {
|
||||
if (!isObject(value)) return {};
|
||||
const overrides: Record<string, string[]> = {};
|
||||
for (const [groupId, names] of Object.entries(value)) {
|
||||
const tools = stringList(names);
|
||||
if (groupId.trim() && tools.length > 0) overrides[groupId] = tools;
|
||||
}
|
||||
return overrides;
|
||||
}
|
||||
|
||||
export function bundleFallbackConfig(): ToolSearchConfig {
|
||||
return {
|
||||
alwaysEnabled: [...BUNDLE_TOOL_SEARCH_DEFAULTS.alwaysEnabled],
|
||||
showToolSearchFooterStatus: BUNDLE_TOOL_SEARCH_DEFAULTS.showToolSearchFooterStatus,
|
||||
maxActiveGroups: BUNDLE_TOOL_SEARCH_DEFAULTS.maxActiveGroups,
|
||||
maxToolsPerGroup: BUNDLE_TOOL_SEARCH_DEFAULTS.maxToolsPerGroup,
|
||||
maxDynamicTools: BUNDLE_TOOL_SEARCH_DEFAULTS.maxDynamicTools,
|
||||
groupOverrides: {},
|
||||
};
|
||||
}
|
||||
|
||||
export function readToolSearchConfig(agentDir: string): ToolSearchConfig {
|
||||
const fallback = bundleFallbackConfig();
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(readFileSync(join(agentDir, "settings.json"), "utf8"));
|
||||
if (!isObject(parsed) || !isObject(parsed.toolSearch)) return fallback;
|
||||
const config = parsed.toolSearch;
|
||||
return {
|
||||
alwaysEnabled: Array.isArray(config.alwaysEnabled) ? stringList(config.alwaysEnabled) : fallback.alwaysEnabled,
|
||||
showToolSearchFooterStatus:
|
||||
typeof config.showToolSearchFooterStatus === "boolean"
|
||||
? config.showToolSearchFooterStatus
|
||||
: fallback.showToolSearchFooterStatus,
|
||||
maxActiveGroups: positiveInteger(config.maxActiveGroups, fallback.maxActiveGroups),
|
||||
maxToolsPerGroup: positiveInteger(config.maxToolsPerGroup, fallback.maxToolsPerGroup),
|
||||
maxDynamicTools: positiveInteger(config.maxDynamicTools, fallback.maxDynamicTools),
|
||||
groupOverrides: readGroupOverrides(config.groupOverrides),
|
||||
};
|
||||
} catch {
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
/** Add bundle defaults without replacing explicit user choices. */
|
||||
export function ensureToolSearchDefaults(agentDir: string): ToolSearchDefaultResult {
|
||||
const targetPath = join(agentDir, "settings.json");
|
||||
let settings: Record<string, unknown> = {};
|
||||
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(readFileSync(targetPath, "utf8"));
|
||||
if (!isObject(parsed)) return "skipped-invalid";
|
||||
settings = parsed;
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== "ENOENT") return "skipped-invalid";
|
||||
}
|
||||
|
||||
const existing = settings.toolSearch;
|
||||
if (existing !== undefined && !isObject(existing)) return "skipped-invalid";
|
||||
|
||||
const toolSearch = { ...(existing ?? {}) };
|
||||
let changed = false;
|
||||
for (const [key, value] of Object.entries(BUNDLE_TOOL_SEARCH_DEFAULTS)) {
|
||||
if (Object.hasOwn(toolSearch, key)) continue;
|
||||
toolSearch[key] = Array.isArray(value) ? [...value] : isObject(value) ? { ...value } : value;
|
||||
changed = true;
|
||||
}
|
||||
if (!changed) return "unchanged";
|
||||
|
||||
settings.toolSearch = toolSearch;
|
||||
mkdirSync(dirname(targetPath), { recursive: true });
|
||||
const temporaryPath = `${targetPath}.my-pi.tmp`;
|
||||
writeFileSync(temporaryPath, `${JSON.stringify(settings, null, 2)}\n`, "utf8");
|
||||
renameSync(temporaryPath, targetPath);
|
||||
return "updated";
|
||||
}
|
||||
@@ -0,0 +1,423 @@
|
||||
/**
|
||||
* pi-tool-search — model-built tool groups with bounded dynamic schema loading.
|
||||
*
|
||||
* Imported from https://github.com/tuansondinh/pi-tool-search and maintained
|
||||
* locally from snapshot ddfb23646fd3957b791214de278e23aa393c9b13 (v0.3.6).
|
||||
*/
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { unlinkSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { getAgentDir, type ExtensionAPI, type ExtensionContext, type ToolInfo } from "@earendil-works/pi-coding-agent";
|
||||
import { Type } from "typebox";
|
||||
|
||||
import {
|
||||
buildCatalogModelInput,
|
||||
buildManifestHash,
|
||||
createFallbackCatalog,
|
||||
parseGeneratedCatalog,
|
||||
rankGroups,
|
||||
readCachedCatalog,
|
||||
writeCachedCatalog,
|
||||
type CatalogConstraints,
|
||||
type GroupCard,
|
||||
type ToolCatalog,
|
||||
} from "./catalog.ts";
|
||||
import {
|
||||
catalogPreservesBundleAssignments,
|
||||
createBundleCatalog,
|
||||
type BundleCatalogResult,
|
||||
} from "./bundle-groups.ts";
|
||||
import { ensureToolSearchDefaults, readToolSearchConfig } from "./config.ts";
|
||||
|
||||
const TOOL_SEARCH_NAME = "tool_search";
|
||||
const CORE_TOOLS = ["read", "write", "edit", "bash", "grep", "find"];
|
||||
|
||||
type CatalogSource = "bundle" | "cache" | "fallback" | "hybrid" | "model";
|
||||
type ModelUsage = Awaited<ReturnType<ExtensionContext["modelRegistry"]["complete"]>>["usage"];
|
||||
|
||||
function unique(values: Iterable<string>): string[] {
|
||||
return [...new Set(values)];
|
||||
}
|
||||
|
||||
function catalogPrompt(toolsJson: string, maxToolsPerGroup: number, fixedGroupsJson: string): string {
|
||||
return [
|
||||
"Build a compact retrieval catalog for the provided Pi tools.",
|
||||
"The catalog is metadata only: never rename tools or invent parameters.",
|
||||
"Group tools that are commonly needed together in one workflow, but keep providers or security-sensitive administration separate when that improves routing.",
|
||||
`Every tool must appear exactly once and every group must contain 1-${maxToolsPerGroup} tools.`,
|
||||
"Tools listed in <fixed-groups> must keep exactly those primaryGroup ids; these are curated bundle assignments.",
|
||||
"Summaries must preserve capability, important boundaries, and when not to use a tool.",
|
||||
"Keywords should include common English terms and useful Chinese equivalents when applicable.",
|
||||
"Return JSON only with this exact shape:",
|
||||
'{"groups":[{"id":"kebab-case","title":"...","summary":"...","useWhen":["..."],"avoidWhen":["..."],"tools":["exact_tool_name"]}],"tools":[{"name":"exact_tool_name","summary":"...","useWhen":["..."],"avoidWhen":["..."],"keywords":["..."],"primaryGroup":"kebab-case"}]}',
|
||||
"Do not include markdown fences or explanatory text.",
|
||||
"",
|
||||
"<fixed-groups>",
|
||||
fixedGroupsJson,
|
||||
"</fixed-groups>",
|
||||
"",
|
||||
"<tools>",
|
||||
toolsJson,
|
||||
"</tools>",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function responseText(content: Array<{ type: string; text?: string }>): string {
|
||||
return content
|
||||
.filter((item): item is { type: string; text: string } => item.type === "text" && typeof item.text === "string")
|
||||
.map((item) => item.text)
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
function groupDescription(catalog: ToolCatalog, source: CatalogSource): string {
|
||||
const sourceLabel =
|
||||
source === "bundle"
|
||||
? "precomputed my-pi bundle catalog; no model generation required"
|
||||
: source === "hybrid"
|
||||
? "my-pi bundle catalog plus deterministic groups for unrecognized tools; the first search may enrich the unknown tools"
|
||||
: source === "fallback"
|
||||
? "deterministic fallback"
|
||||
: `${source} catalog`;
|
||||
const groups = catalog.groups
|
||||
.map((group) => {
|
||||
const when = group.useWhen.slice(0, 2).join("; ");
|
||||
const detail = [group.summary, when].filter(Boolean).join(" Use when: ").slice(0, 360);
|
||||
return ` ${group.id}: ${detail}`;
|
||||
})
|
||||
.join("\n");
|
||||
return [
|
||||
"Activate a complete tool group for the current task. Prefer an exact group id from this catalog; use query only when no id clearly matches.",
|
||||
`Catalog source: ${sourceLabel}.`,
|
||||
"Available groups:",
|
||||
groups || " (no hidden tool groups)",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export default function toolSearchExtension(pi: ExtensionAPI): void {
|
||||
const agentDir = getAgentDir();
|
||||
const cachePath = join(agentDir, "tool-search", "catalog-v1.json");
|
||||
const defaultResult = ensureToolSearchDefaults(agentDir);
|
||||
if (defaultResult === "skipped-invalid") {
|
||||
console.warn("my-pi: skipped pi-tool-search defaults because settings.json or toolSearch is invalid");
|
||||
}
|
||||
|
||||
let config = readToolSearchConfig(agentDir);
|
||||
let tools: ToolInfo[] = [];
|
||||
let catalog: ToolCatalog = createFallbackCatalog([], "", {
|
||||
maxToolsPerGroup: Math.min(config.maxToolsPerGroup, config.maxDynamicTools),
|
||||
groupOverrides: {},
|
||||
});
|
||||
let catalogSource: CatalogSource = "fallback";
|
||||
let manifestHash = "";
|
||||
let policySignature = "";
|
||||
let attemptedGenerationHash: string | undefined;
|
||||
let clock = 0;
|
||||
const activeGroups = new Map<string, number>();
|
||||
const pinnedTools = new Set<string>();
|
||||
let bundleState: BundleCatalogResult = { catalog, coveredNames: new Set(), unknownTools: [] };
|
||||
|
||||
function constraints(): CatalogConstraints {
|
||||
return {
|
||||
maxToolsPerGroup: Math.min(config.maxToolsPerGroup, config.maxDynamicTools),
|
||||
groupOverrides: config.groupOverrides,
|
||||
};
|
||||
}
|
||||
|
||||
function installBaseCatalog(useCache: boolean): void {
|
||||
bundleState = createBundleCatalog(tools, manifestHash, constraints());
|
||||
if (bundleState.unknownTools.length === 0) {
|
||||
catalog = bundleState.catalog;
|
||||
catalogSource = "bundle";
|
||||
return;
|
||||
}
|
||||
const cached = useCache ? readCachedCatalog(cachePath, tools, manifestHash, constraints()) : undefined;
|
||||
if (cached && catalogPreservesBundleAssignments(cached, bundleState.catalog, bundleState.coveredNames)) {
|
||||
catalog = cached;
|
||||
catalogSource = "cache";
|
||||
return;
|
||||
}
|
||||
catalog = bundleState.catalog;
|
||||
catalogSource = "hybrid";
|
||||
}
|
||||
|
||||
function fixedBundleGroupsJson(): string {
|
||||
return JSON.stringify(
|
||||
bundleState.catalog.groups
|
||||
.map((group) => ({
|
||||
id: group.id,
|
||||
tools: group.tools.filter((name) => bundleState.coveredNames.has(name)),
|
||||
}))
|
||||
.filter((group) => group.tools.length > 0),
|
||||
);
|
||||
}
|
||||
|
||||
function groupById(id: string): GroupCard | undefined {
|
||||
return catalog.groups.find((group) => group.id === id);
|
||||
}
|
||||
|
||||
function dynamicToolCount(): number {
|
||||
return unique([...activeGroups.keys()].flatMap((id) => groupById(id)?.tools ?? [])).length;
|
||||
}
|
||||
|
||||
function leastRecentlyUsedGroup(): string | undefined {
|
||||
return [...activeGroups].sort((left, right) => left[1] - right[1] || left[0].localeCompare(right[0]))[0]?.[0];
|
||||
}
|
||||
|
||||
function applyActiveTools(): void {
|
||||
const groupTools = [...activeGroups.keys()].flatMap((id) => groupById(id)?.tools ?? []);
|
||||
pi.setActiveTools(unique([TOOL_SEARCH_NAME, ...pinnedTools, ...groupTools]));
|
||||
}
|
||||
|
||||
function updateStatus(ctx: Pick<ExtensionContext, "ui">): void {
|
||||
const activeToolCount = unique([TOOL_SEARCH_NAME, ...pinnedTools, ...[...activeGroups.keys()].flatMap((id) => groupById(id)?.tools ?? [])]).length;
|
||||
ctx.ui.setStatus(
|
||||
"tool-search",
|
||||
config.showToolSearchFooterStatus
|
||||
? `${activeToolCount} / ${tools.length + 1} tools · ${activeGroups.size} / ${config.maxActiveGroups} groups`
|
||||
: undefined,
|
||||
);
|
||||
}
|
||||
|
||||
function registerToolSearch(): void {
|
||||
pi.registerTool({
|
||||
name: TOOL_SEARCH_NAME,
|
||||
label: "Tool Search",
|
||||
description: groupDescription(catalog, catalogSource),
|
||||
promptSnippet: `Activate relevant tool groups on demand; at most ${config.maxActiveGroups} dynamic groups remain active`,
|
||||
parameters: Type.Object({
|
||||
group: Type.Optional(Type.String({ description: "Exact group id from the tool_search catalog" })),
|
||||
query: Type.Optional(Type.String({ description: "Natural-language task used to rank groups when an exact id is unclear" })),
|
||||
}),
|
||||
async execute(_toolCallId, params, signal, _onUpdate, ctx) {
|
||||
const requestedId = params.group?.trim().toLowerCase();
|
||||
const preGenerationGroup = requestedId ? groupById(requestedId) : undefined;
|
||||
const generation = await ensureModelCatalog(ctx, signal);
|
||||
const lines: string[] = [];
|
||||
if (generation.notice) lines.push(generation.notice);
|
||||
|
||||
let selected = requestedId ? groupById(requestedId) : undefined;
|
||||
if (!selected && preGenerationGroup) {
|
||||
const previousNames = new Set(preGenerationGroup.tools);
|
||||
selected = [...catalog.groups]
|
||||
.map((group) => ({ group, overlap: group.tools.filter((name) => previousNames.has(name)).length }))
|
||||
.sort((left, right) => right.overlap - left.overlap || left.group.id.localeCompare(right.group.id))
|
||||
.find((candidate) => candidate.overlap > 0)?.group;
|
||||
if (selected) lines.push(`Mapped initial fallback group ${preGenerationGroup.id} to generated group ${selected.id}.`);
|
||||
}
|
||||
const ranked = params.query ? rankGroups(catalog, params.query).slice(0, 3) : [];
|
||||
if (!selected && params.query && (ranked[0]?.score ?? 0) > 0) selected = ranked[0]?.group;
|
||||
|
||||
if (!selected) {
|
||||
const candidates = ranked.length > 0 ? ranked : catalog.groups.slice(0, 5).map((group) => ({ group, score: 0 }));
|
||||
lines.push(
|
||||
params.group ? `Unknown group: ${params.group}` : "No group was activated. Provide an exact group id from the catalog.",
|
||||
`Candidates: ${candidates.map(({ group }) => group.id).join(", ") || "none"}`,
|
||||
);
|
||||
return {
|
||||
content: [{ type: "text", text: lines.join("\n") }],
|
||||
details: { loadedGroup: undefined, evictedGroups: [], activeGroups: [...activeGroups.keys()], candidates: candidates.map(({ group, score }) => ({ id: group.id, score })), catalogSource },
|
||||
...(generation.usage ? { usage: generation.usage } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
const groupTools = unique(selected.tools);
|
||||
if (groupTools.length > config.maxDynamicTools) {
|
||||
lines.push(`Group ${selected.id} has ${groupTools.length} tools, exceeding maxDynamicTools=${config.maxDynamicTools}.`);
|
||||
return {
|
||||
content: [{ type: "text", text: lines.join("\n") }],
|
||||
details: { loadedGroup: undefined, evictedGroups: [], activeGroups: [...activeGroups.keys()], candidates: [], catalogSource },
|
||||
...(generation.usage ? { usage: generation.usage } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
const evictedGroups: string[] = [];
|
||||
if (!activeGroups.has(selected.id)) {
|
||||
while (
|
||||
activeGroups.size >= config.maxActiveGroups ||
|
||||
(activeGroups.size > 0 && dynamicToolCount() + groupTools.length > config.maxDynamicTools)
|
||||
) {
|
||||
const evicted = leastRecentlyUsedGroup();
|
||||
if (!evicted) break;
|
||||
activeGroups.delete(evicted);
|
||||
evictedGroups.push(evicted);
|
||||
}
|
||||
}
|
||||
activeGroups.set(selected.id, ++clock);
|
||||
applyActiveTools();
|
||||
registerToolSearch();
|
||||
updateStatus(ctx);
|
||||
|
||||
lines.push(`Loaded group: ${selected.id} (${groupTools.join(", ")})`);
|
||||
if (evictedGroups.length > 0) lines.push(`Evicted least-recently-used: ${evictedGroups.join(", ")}`);
|
||||
lines.push(`Active groups: ${[...activeGroups.keys()].join(", ")}`);
|
||||
return {
|
||||
content: [{ type: "text", text: lines.join("\n") }],
|
||||
details: {
|
||||
loadedGroup: selected.id,
|
||||
evictedGroups,
|
||||
activeGroups: [...activeGroups.keys()],
|
||||
candidates: ranked.map(({ group, score }) => ({ id: group.id, score })),
|
||||
catalogSource,
|
||||
},
|
||||
...(generation.usage ? { usage: generation.usage } : {}),
|
||||
};
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function ensureModelCatalog(
|
||||
ctx: ExtensionContext,
|
||||
signal: AbortSignal | undefined,
|
||||
): Promise<{ notice?: string; usage?: ModelUsage }> {
|
||||
if (catalogSource !== "hybrid" || attemptedGenerationHash === manifestHash || tools.length === 0) return {};
|
||||
attemptedGenerationHash = manifestHash;
|
||||
if (!ctx.model || !ctx.modelRegistry.hasConfiguredAuth(ctx.model)) {
|
||||
return { notice: "Using the precomputed bundle catalog plus deterministic groups for unrecognized tools because the current model is unavailable or unauthenticated." };
|
||||
}
|
||||
let generationUsage: ModelUsage | undefined;
|
||||
|
||||
try {
|
||||
const response = await ctx.modelRegistry.complete(
|
||||
ctx.model,
|
||||
{
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: catalogPrompt(
|
||||
buildCatalogModelInput(tools),
|
||||
constraints().maxToolsPerGroup,
|
||||
fixedBundleGroupsJson(),
|
||||
),
|
||||
},
|
||||
],
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
signal,
|
||||
reasoningEffort: "low",
|
||||
cacheRetention: "none",
|
||||
sessionId: randomUUID(),
|
||||
maxTokens: 12_000,
|
||||
},
|
||||
);
|
||||
generationUsage = response.usage;
|
||||
catalog = parseGeneratedCatalog(
|
||||
responseText(response.content),
|
||||
tools,
|
||||
manifestHash,
|
||||
constraints(),
|
||||
`${ctx.model.provider}/${ctx.model.id}`,
|
||||
);
|
||||
if (!catalogPreservesBundleAssignments(catalog, bundleState.catalog, bundleState.coveredNames)) {
|
||||
throw new Error("generated catalog changed one or more curated my-pi bundle assignments");
|
||||
}
|
||||
catalogSource = "model";
|
||||
try {
|
||||
writeCachedCatalog(cachePath, catalog);
|
||||
} catch (error) {
|
||||
console.warn(`my-pi: could not cache pi-tool-search catalog: ${error instanceof Error ? error.message : String(error)}`);
|
||||
}
|
||||
registerToolSearch();
|
||||
return {
|
||||
notice: `Generated and cached ${catalog.groups.length} groups while preserving the precomputed my-pi assignments.`,
|
||||
usage: generationUsage,
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
notice: `Model enrichment failed; using the precomputed bundle catalog plus deterministic unknown-tool groups (${error instanceof Error ? error.message : String(error)}).`,
|
||||
...(generationUsage ? { usage: generationUsage } : {}),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function refreshState(ctx: Pick<ExtensionContext, "ui">, forceReset: boolean): void {
|
||||
const nextConfig = readToolSearchConfig(agentDir);
|
||||
const allTools = pi.getAllTools().filter((tool) => tool.name !== TOOL_SEARCH_NAME);
|
||||
const availableNames = new Set(allTools.map((tool) => tool.name));
|
||||
const nextPinned = new Set([...CORE_TOOLS, ...nextConfig.alwaysEnabled].filter((name) => availableNames.has(name)));
|
||||
const hiddenTools = allTools.filter((tool) => !nextPinned.has(tool.name));
|
||||
const nextConstraints = {
|
||||
maxToolsPerGroup: Math.min(nextConfig.maxToolsPerGroup, nextConfig.maxDynamicTools),
|
||||
groupOverrides: nextConfig.groupOverrides,
|
||||
};
|
||||
const nextHash = buildManifestHash(hiddenTools, nextConstraints);
|
||||
const nextPolicySignature = JSON.stringify({
|
||||
alwaysEnabled: [...nextPinned].sort(),
|
||||
maxActiveGroups: nextConfig.maxActiveGroups,
|
||||
maxDynamicTools: nextConfig.maxDynamicTools,
|
||||
showToolSearchFooterStatus: nextConfig.showToolSearchFooterStatus,
|
||||
});
|
||||
const catalogChanged = forceReset || nextHash !== manifestHash;
|
||||
const policyChanged = nextPolicySignature !== policySignature;
|
||||
|
||||
config = nextConfig;
|
||||
tools = hiddenTools;
|
||||
pinnedTools.clear();
|
||||
for (const name of nextPinned) pinnedTools.add(name);
|
||||
manifestHash = nextHash;
|
||||
policySignature = nextPolicySignature;
|
||||
|
||||
if (catalogChanged) {
|
||||
activeGroups.clear();
|
||||
clock = 0;
|
||||
attemptedGenerationHash = undefined;
|
||||
installBaseCatalog(true);
|
||||
}
|
||||
|
||||
let capacityChanged = false;
|
||||
while (activeGroups.size > config.maxActiveGroups || dynamicToolCount() > config.maxDynamicTools) {
|
||||
const evicted = leastRecentlyUsedGroup();
|
||||
if (!evicted) break;
|
||||
activeGroups.delete(evicted);
|
||||
capacityChanged = true;
|
||||
}
|
||||
if (catalogChanged || policyChanged || capacityChanged) {
|
||||
registerToolSearch();
|
||||
applyActiveTools();
|
||||
}
|
||||
updateStatus(ctx);
|
||||
}
|
||||
|
||||
pi.registerCommand("tool-search-rebuild", {
|
||||
description: "Invalidate the generated tool-group catalog; rebuild lazily on the next tool_search call",
|
||||
handler: async (_args, ctx) => {
|
||||
try {
|
||||
unlinkSync(cachePath);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
|
||||
}
|
||||
activeGroups.clear();
|
||||
attemptedGenerationHash = undefined;
|
||||
installBaseCatalog(false);
|
||||
registerToolSearch();
|
||||
applyActiveTools();
|
||||
updateStatus(ctx);
|
||||
if (ctx.hasUI) {
|
||||
const message =
|
||||
bundleState.unknownTools.length === 0
|
||||
? "Restored the precomputed my-pi tool-group catalog; no model rebuild is needed"
|
||||
: "Restored my-pi groups; unrecognized tools may be enriched on the next tool_search call";
|
||||
ctx.ui.notify(message, "info");
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
pi.on("session_start", (_event, ctx) => {
|
||||
refreshState(ctx, true);
|
||||
});
|
||||
|
||||
pi.on("turn_start", (_event, ctx) => {
|
||||
refreshState(ctx, false);
|
||||
});
|
||||
|
||||
pi.on("tool_execution_start", (event) => {
|
||||
const group = catalog.tools.find((tool) => tool.name === event.toolName)?.primaryGroup;
|
||||
if (group && activeGroups.has(group)) activeGroups.set(group, ++clock);
|
||||
});
|
||||
}
|
||||
Generated
+68
@@ -0,0 +1,68 @@
|
||||
{
|
||||
"name": "pi-tool-search",
|
||||
"version": "0.3.6",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "pi-tool-search",
|
||||
"version": "0.3.6",
|
||||
"license": "MIT",
|
||||
"devDependencies": {
|
||||
"@types/node": "24.10.13",
|
||||
"typebox": "1.3.7",
|
||||
"typescript": "6.0.3"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@earendil-works/pi-coding-agent": "*",
|
||||
"typebox": "*"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"@earendil-works/pi-coding-agent": {
|
||||
"optional": true
|
||||
},
|
||||
"typebox": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/@types/node": {
|
||||
"version": "24.10.13",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.13.tgz",
|
||||
"integrity": "sha512-oH72nZRfDv9lADUBSo104Aq7gPHpQZc4BTx38r9xf9pg5LfP6EzSyH2n7qFmmxRQXh7YlUXODcYsg6PuTDSxGg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"undici-types": "~7.16.0"
|
||||
}
|
||||
},
|
||||
"node_modules/typebox": {
|
||||
"version": "1.3.7",
|
||||
"resolved": "https://registry.npmjs.org/typebox/-/typebox-1.3.7.tgz",
|
||||
"integrity": "sha512-meKuifc33Pccx0O6PdIzYMq3Og8zvP4TIi/a+Bw3AEMZMxOD0+RHGQvpglEe6Zdy3wZ8nqn/j95h8LUZLk/6Hg==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/typescript": {
|
||||
"version": "6.0.3",
|
||||
"resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz",
|
||||
"integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"bin": {
|
||||
"tsc": "bin/tsc",
|
||||
"tsserver": "bin/tsserver"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14.17"
|
||||
}
|
||||
},
|
||||
"node_modules/undici-types": {
|
||||
"version": "7.16.0",
|
||||
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz",
|
||||
"integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
{
|
||||
"name": "pi-tool-search",
|
||||
"version": "0.3.6",
|
||||
"description": "Model-built tool groups with bounded LRU schema loading for Pi",
|
||||
"author": "Tuan Son <tuansondinh96@gmail.com>",
|
||||
"license": "MIT",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./extensions/index.ts",
|
||||
"exports": {
|
||||
".": "./extensions/index.ts"
|
||||
},
|
||||
"files": [
|
||||
"extensions",
|
||||
"docs",
|
||||
"README.md",
|
||||
"CHANGELOG.md",
|
||||
"LICENSE",
|
||||
"UPSTREAM.md"
|
||||
],
|
||||
"pi": {
|
||||
"extensions": [
|
||||
"./extensions/index.ts"
|
||||
]
|
||||
},
|
||||
"scripts": {
|
||||
"build": "rm -rf dist && tsc -p tsconfig.json --noCheck --outDir dist",
|
||||
"typecheck": "tsc -p tsconfig.json --noEmit",
|
||||
"test": "node --test test/*.test.ts",
|
||||
"check": "npm run typecheck && npm run test && npm run build"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@earendil-works/pi-coding-agent": "*",
|
||||
"typebox": "*"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"@earendil-works/pi-coding-agent": {
|
||||
"optional": true
|
||||
},
|
||||
"typebox": {
|
||||
"optional": true
|
||||
}
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "24.10.13",
|
||||
"typebox": "1.3.7",
|
||||
"typescript": "6.0.3"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://github.com/tuansondinh/pi-tool-search.git"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
|
||||
import type { ToolInfo } from "@earendil-works/pi-coding-agent";
|
||||
import { BUNDLE_GROUP_DEFINITIONS, createBundleCatalog } from "../extensions/bundle-groups.ts";
|
||||
import { buildManifestHash } from "../extensions/catalog.ts";
|
||||
|
||||
function tool(name: string): ToolInfo {
|
||||
return {
|
||||
name,
|
||||
description: `Full description for ${name}`,
|
||||
parameters: { type: "object", properties: {} } as ToolInfo["parameters"],
|
||||
sourceInfo: { source: "extension", scope: "user", path: `/test/${name}.ts` } as unknown as ToolInfo["sourceInfo"],
|
||||
};
|
||||
}
|
||||
|
||||
const constraints = { maxToolsPerGroup: 8, groupOverrides: {} };
|
||||
|
||||
test("bundle seed names are unique and cover the declared my-pi tools", () => {
|
||||
const names = BUNDLE_GROUP_DEFINITIONS.flatMap((group) => group.tools);
|
||||
assert.equal(new Set(names).size, names.length);
|
||||
const tools = names.map(tool);
|
||||
const hash = buildManifestHash(tools, constraints);
|
||||
const result = createBundleCatalog(tools, hash, constraints);
|
||||
assert.deepEqual(result.unknownTools, []);
|
||||
assert.equal(result.coveredNames.size, names.length);
|
||||
assert.equal(result.catalog.tools.length, names.length);
|
||||
assert.equal(result.catalog.tools.find((card) => card.name === "ctx_purge")?.primaryGroup, "context-administration");
|
||||
assert.equal(result.catalog.tools.find((card) => card.name === "tavily_web_fetch")?.primaryGroup, "web-tavily");
|
||||
});
|
||||
|
||||
test("user overrides take priority and unknown tools retain deterministic fallback groups", () => {
|
||||
const tools = [tool("tavily_web_search"), tool("custom_analyze")];
|
||||
const configured = { maxToolsPerGroup: 8, groupOverrides: { preferred: ["tavily_web_search"] } };
|
||||
const hash = buildManifestHash(tools, configured);
|
||||
const result = createBundleCatalog(tools, hash, configured);
|
||||
assert.deepEqual(result.unknownTools.map((item) => item.name), ["custom_analyze"]);
|
||||
assert.equal(result.catalog.tools.find((card) => card.name === "tavily_web_search")?.primaryGroup, "preferred");
|
||||
assert.equal(result.catalog.tools.find((card) => card.name === "custom_analyze")?.primaryGroup, "custom");
|
||||
});
|
||||
@@ -0,0 +1,82 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
|
||||
import type { ToolInfo } from "@earendil-works/pi-coding-agent";
|
||||
import {
|
||||
buildManifestHash,
|
||||
createFallbackCatalog,
|
||||
parseGeneratedCatalog,
|
||||
rankGroups,
|
||||
readCachedCatalog,
|
||||
writeCachedCatalog,
|
||||
} from "../extensions/catalog.ts";
|
||||
|
||||
function tool(name: string, description: string): ToolInfo {
|
||||
return {
|
||||
name,
|
||||
description,
|
||||
parameters: { type: "object", properties: { query: { type: "string" } } } as ToolInfo["parameters"],
|
||||
sourceInfo: { source: "extension", scope: "user", path: `/test/${name}.ts` } as unknown as ToolInfo["sourceInfo"],
|
||||
};
|
||||
}
|
||||
|
||||
const constraints = { maxToolsPerGroup: 2, groupOverrides: {} };
|
||||
|
||||
test("fallback grouping chunks large prefixes and applies explicit overrides", () => {
|
||||
const tools = [tool("ctx_one", "First"), tool("ctx_two", "Second"), tool("ctx_three", "Third")];
|
||||
const configured = { maxToolsPerGroup: 2, groupOverrides: { preferred: ["ctx_three"] } };
|
||||
const hash = buildManifestHash(tools, configured);
|
||||
const catalog = createFallbackCatalog(tools, hash, configured);
|
||||
assert.deepEqual(catalog.groups.find((group) => group.id === "preferred")?.tools, ["ctx_three"]);
|
||||
assert.ok(catalog.groups.every((group) => group.tools.length <= 2));
|
||||
assert.equal(catalog.tools.find((card) => card.name === "ctx_three")?.primaryGroup, "preferred");
|
||||
});
|
||||
|
||||
test("validates exact generated assignments and ranks generated metadata", () => {
|
||||
const tools = [tool("web_search", "Find sources"), tool("web_fetch", "Read a source")];
|
||||
const hash = buildManifestHash(tools, constraints);
|
||||
const generated = {
|
||||
groups: [
|
||||
{
|
||||
id: "web-research",
|
||||
title: "Web research",
|
||||
summary: "Find and read current sources.",
|
||||
useWhen: ["需要网页搜索"],
|
||||
avoidWhen: [],
|
||||
tools: ["web_search", "web_fetch"],
|
||||
},
|
||||
],
|
||||
tools: [
|
||||
{ name: "web_search", summary: "Find sources", useWhen: [], avoidWhen: [], keywords: ["搜索"], primaryGroup: "web-research" },
|
||||
{ name: "web_fetch", summary: "Read sources", useWhen: [], avoidWhen: [], keywords: ["抓取"], primaryGroup: "web-research" },
|
||||
],
|
||||
};
|
||||
const catalog = parseGeneratedCatalog(JSON.stringify(generated), tools, hash, constraints, "test/model");
|
||||
assert.equal(rankGroups(catalog, "帮我搜索网页")[0]?.group.id, "web-research");
|
||||
assert.throws(
|
||||
() => parseGeneratedCatalog(JSON.stringify({ ...generated, tools: generated.tools.slice(0, 1) }), tools, hash, constraints, "test/model"),
|
||||
/omitted/,
|
||||
);
|
||||
});
|
||||
|
||||
test("writes a private cache and rejects a stale manifest hash", async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), "tool-search-catalog-"));
|
||||
try {
|
||||
const tools = [tool("web_search", "Find sources")];
|
||||
const hash = buildManifestHash(tools, constraints);
|
||||
const generated = {
|
||||
groups: [{ id: "web", title: "Web", summary: "Find sources", useWhen: [], avoidWhen: [], tools: ["web_search"] }],
|
||||
tools: [{ name: "web_search", summary: "Find sources", useWhen: [], avoidWhen: [], keywords: [], primaryGroup: "web" }],
|
||||
};
|
||||
const catalog = parseGeneratedCatalog(JSON.stringify(generated), tools, hash, constraints, "test/model");
|
||||
const path = join(directory, "nested", "catalog.json");
|
||||
writeCachedCatalog(path, catalog);
|
||||
assert.equal(readCachedCatalog(path, tools, hash, constraints)?.groups[0]?.id, "web");
|
||||
assert.equal(readCachedCatalog(path, tools, "stale", constraints), undefined);
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,105 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
|
||||
import { BUNDLE_TOOL_SEARCH_DEFAULTS, ensureToolSearchDefaults, readToolSearchConfig } from "../extensions/config.ts";
|
||||
|
||||
async function withAgentDir(run: (agentDir: string) => Promise<void>): Promise<void> {
|
||||
const agentDir = await mkdtemp(join(tmpdir(), "my-pi-tool-search-"));
|
||||
try {
|
||||
await run(agentDir);
|
||||
} finally {
|
||||
await rm(agentDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
test("writes bundle defaults when toolSearch is absent", async () => {
|
||||
await withAgentDir(async (agentDir) => {
|
||||
assert.equal(ensureToolSearchDefaults(agentDir), "updated");
|
||||
assert.deepEqual(JSON.parse(await readFile(join(agentDir, "settings.json"), "utf8")), {
|
||||
toolSearch: {
|
||||
alwaysEnabled: [...BUNDLE_TOOL_SEARCH_DEFAULTS.alwaysEnabled],
|
||||
showToolSearchFooterStatus: false,
|
||||
maxActiveGroups: 3,
|
||||
maxToolsPerGroup: 8,
|
||||
maxDynamicTools: 20,
|
||||
groupOverrides: {},
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test("fills missing defaults and preserves explicit settings", async () => {
|
||||
await withAgentDir(async (agentDir) => {
|
||||
const path = join(agentDir, "settings.json");
|
||||
await writeFile(path, JSON.stringify({ theme: "dark", toolSearch: { alwaysEnabled: ["multi_grep"], maxActiveGroups: 2 } }), "utf8");
|
||||
assert.equal(ensureToolSearchDefaults(agentDir), "updated");
|
||||
assert.deepEqual(JSON.parse(await readFile(path, "utf8")), {
|
||||
theme: "dark",
|
||||
toolSearch: {
|
||||
alwaysEnabled: ["multi_grep"],
|
||||
maxActiveGroups: 2,
|
||||
showToolSearchFooterStatus: false,
|
||||
maxToolsPerGroup: 8,
|
||||
maxDynamicTools: 20,
|
||||
groupOverrides: {},
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test("normalizes invalid runtime values without overwriting the file", async () => {
|
||||
await withAgentDir(async (agentDir) => {
|
||||
const path = join(agentDir, "settings.json");
|
||||
const original = JSON.stringify({
|
||||
toolSearch: {
|
||||
alwaysEnabled: ["one", "one", 3],
|
||||
showToolSearchFooterStatus: "no",
|
||||
maxActiveGroups: 0,
|
||||
maxToolsPerGroup: 4,
|
||||
maxDynamicTools: -1,
|
||||
groupOverrides: { web: ["search", 2], empty: [] },
|
||||
},
|
||||
});
|
||||
await writeFile(path, original, "utf8");
|
||||
assert.deepEqual(readToolSearchConfig(agentDir), {
|
||||
alwaysEnabled: ["one"],
|
||||
showToolSearchFooterStatus: false,
|
||||
maxActiveGroups: 3,
|
||||
maxToolsPerGroup: 4,
|
||||
maxDynamicTools: 20,
|
||||
groupOverrides: { web: ["search"] },
|
||||
});
|
||||
assert.equal(await readFile(path, "utf8"), original);
|
||||
});
|
||||
});
|
||||
|
||||
test("preserves explicit complete configuration", async () => {
|
||||
await withAgentDir(async (agentDir) => {
|
||||
const path = join(agentDir, "settings.json");
|
||||
const original = `${JSON.stringify({
|
||||
toolSearch: {
|
||||
alwaysEnabled: [],
|
||||
showToolSearchFooterStatus: true,
|
||||
maxActiveGroups: 1,
|
||||
maxToolsPerGroup: 2,
|
||||
maxDynamicTools: 2,
|
||||
groupOverrides: { custom: ["one"] },
|
||||
},
|
||||
}, null, 2)}\n`;
|
||||
await writeFile(path, original, "utf8");
|
||||
assert.equal(ensureToolSearchDefaults(agentDir), "unchanged");
|
||||
assert.equal(await readFile(path, "utf8"), original);
|
||||
});
|
||||
});
|
||||
|
||||
test("does not overwrite malformed settings", async () => {
|
||||
await withAgentDir(async (agentDir) => {
|
||||
const path = join(agentDir, "settings.json");
|
||||
await writeFile(path, "{not-json", "utf8");
|
||||
assert.equal(ensureToolSearchDefaults(agentDir), "skipped-invalid");
|
||||
assert.equal(await readFile(path, "utf8"), "{not-json");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,251 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
|
||||
import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
|
||||
import toolSearchExtension from "../extensions/index.ts";
|
||||
|
||||
interface ToolSearchResult {
|
||||
details: {
|
||||
loadedGroup?: string;
|
||||
evictedGroups: string[];
|
||||
activeGroups: string[];
|
||||
catalogSource: string;
|
||||
};
|
||||
usage?: unknown;
|
||||
}
|
||||
|
||||
interface RegisteredTool {
|
||||
execute(
|
||||
id: string,
|
||||
params: { group?: string; query?: string },
|
||||
signal: AbortSignal | undefined,
|
||||
onUpdate: undefined,
|
||||
ctx: ExtensionContext,
|
||||
): Promise<ToolSearchResult>;
|
||||
}
|
||||
|
||||
function sourceTool(name: string, description: string) {
|
||||
return {
|
||||
name,
|
||||
description,
|
||||
parameters: { type: "object", properties: {} },
|
||||
sourceInfo: { type: "extension", path: `/test/${name}.ts` },
|
||||
};
|
||||
}
|
||||
|
||||
function testContext(overrides: Partial<ExtensionContext> = {}): ExtensionContext {
|
||||
return {
|
||||
ui: { setStatus: () => {} },
|
||||
model: undefined,
|
||||
modelRegistry: { hasConfiguredAuth: () => false },
|
||||
...overrides,
|
||||
} as unknown as ExtensionContext;
|
||||
}
|
||||
|
||||
test("loads whole groups and evicts the least-recently-used group", async () => {
|
||||
const agentDir = await mkdtemp(join(tmpdir(), "my-pi-tool-search-extension-"));
|
||||
const previousAgentDir = process.env.PI_CODING_AGENT_DIR;
|
||||
process.env.PI_CODING_AGENT_DIR = agentDir;
|
||||
|
||||
try {
|
||||
await writeFile(
|
||||
join(agentDir, "settings.json"),
|
||||
JSON.stringify({
|
||||
toolSearch: {
|
||||
alwaysEnabled: ["codegraph_explore", "lsp_diagnostics"],
|
||||
showToolSearchFooterStatus: false,
|
||||
maxActiveGroups: 2,
|
||||
maxToolsPerGroup: 4,
|
||||
maxDynamicTools: 8,
|
||||
groupOverrides: {},
|
||||
},
|
||||
}),
|
||||
);
|
||||
const handlers = new Map<string, (...args: any[]) => unknown>();
|
||||
const registered = new Map<string, RegisteredTool>();
|
||||
const activeCalls: string[][] = [];
|
||||
const sourceTools = [
|
||||
sourceTool("read", "Read a file"),
|
||||
sourceTool("bash", "Run a command"),
|
||||
sourceTool("codegraph_explore", "Explore code relationships"),
|
||||
sourceTool("lsp_diagnostics", "Read diagnostics"),
|
||||
sourceTool("alpha_one", "First alpha capability with all important details"),
|
||||
sourceTool("alpha_two", "Second alpha capability"),
|
||||
sourceTool("beta_one", "Beta capability"),
|
||||
sourceTool("gamma_one", "Gamma capability"),
|
||||
];
|
||||
|
||||
const api = {
|
||||
getAllTools: () => sourceTools,
|
||||
getActiveTools: () => activeCalls.at(-1) ?? sourceTools.map((tool) => tool.name),
|
||||
registerTool: (definition: unknown) => {
|
||||
const tool = definition as RegisteredTool & { name: string };
|
||||
registered.set(tool.name, tool);
|
||||
},
|
||||
registerCommand: () => {},
|
||||
setActiveTools: (names: string[]) => activeCalls.push([...names]),
|
||||
on: (event: string, handler: (...args: any[]) => unknown) => handlers.set(event, handler),
|
||||
} as unknown as ExtensionAPI;
|
||||
|
||||
toolSearchExtension(api);
|
||||
const ctx = testContext();
|
||||
handlers.get("session_start")?.({}, ctx);
|
||||
|
||||
assert.ok(activeCalls.at(-1)?.includes("tool_search"));
|
||||
assert.ok(activeCalls.at(-1)?.includes("codegraph_explore"));
|
||||
assert.ok(!activeCalls.at(-1)?.includes("alpha_one"));
|
||||
|
||||
const tool = registered.get("tool_search");
|
||||
const alpha = await tool?.execute("call-1", { group: "alpha" }, undefined, undefined, ctx);
|
||||
assert.equal(alpha?.details.loadedGroup, "alpha");
|
||||
assert.ok(activeCalls.at(-1)?.includes("alpha_one"));
|
||||
assert.ok(activeCalls.at(-1)?.includes("alpha_two"));
|
||||
assert.ok(activeCalls.at(-2)?.every((name) => activeCalls.at(-1)?.includes(name)));
|
||||
|
||||
await tool?.execute("call-2", { group: "beta" }, undefined, undefined, ctx);
|
||||
handlers.get("tool_execution_start")?.({ toolName: "alpha_one" }, ctx);
|
||||
const gamma = await tool?.execute("call-3", { group: "gamma" }, undefined, undefined, ctx);
|
||||
assert.deepEqual(gamma?.details.evictedGroups, ["beta"]);
|
||||
assert.deepEqual(gamma?.details.activeGroups, ["alpha", "gamma"]);
|
||||
assert.ok(activeCalls.at(-1)?.includes("alpha_one"));
|
||||
assert.ok(activeCalls.at(-1)?.includes("gamma_one"));
|
||||
assert.ok(!activeCalls.at(-1)?.includes("beta_one"));
|
||||
} finally {
|
||||
if (previousAgentDir === undefined) delete process.env.PI_CODING_AGENT_DIR;
|
||||
else process.env.PI_CODING_AGENT_DIR = previousAgentDir;
|
||||
await rm(agentDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("builds and caches a validated catalog with the current model on first search", async () => {
|
||||
const agentDir = await mkdtemp(join(tmpdir(), "my-pi-tool-search-model-"));
|
||||
const previousAgentDir = process.env.PI_CODING_AGENT_DIR;
|
||||
process.env.PI_CODING_AGENT_DIR = agentDir;
|
||||
|
||||
try {
|
||||
const handlers = new Map<string, (...args: any[]) => unknown>();
|
||||
const registered = new Map<string, RegisteredTool>();
|
||||
const prompts: string[] = [];
|
||||
const sourceTools = [
|
||||
sourceTool("read", "Read a file"),
|
||||
sourceTool("web_search", "Search the public web with filters and return sources"),
|
||||
sourceTool("web_fetch", "Fetch the complete content of a selected web page"),
|
||||
];
|
||||
const api = {
|
||||
getAllTools: () => sourceTools,
|
||||
registerTool: (definition: unknown) => {
|
||||
const tool = definition as RegisteredTool & { name: string };
|
||||
registered.set(tool.name, tool);
|
||||
},
|
||||
registerCommand: () => {},
|
||||
setActiveTools: () => {},
|
||||
on: (event: string, handler: (...args: any[]) => unknown) => handlers.set(event, handler),
|
||||
} as unknown as ExtensionAPI;
|
||||
toolSearchExtension(api);
|
||||
|
||||
const generated = {
|
||||
groups: [
|
||||
{
|
||||
id: "web-research",
|
||||
title: "Web research",
|
||||
summary: "Search and fetch current web sources.",
|
||||
useWhen: ["Current external facts are needed"],
|
||||
avoidWhen: ["The answer is entirely local"],
|
||||
tools: ["web_search", "web_fetch"],
|
||||
},
|
||||
],
|
||||
tools: [
|
||||
{ name: "web_search", summary: "Find web sources.", useWhen: ["Discover sources"], avoidWhen: [], keywords: ["search", "搜索"], primaryGroup: "web-research" },
|
||||
{ name: "web_fetch", summary: "Read a selected source.", useWhen: ["A URL is known"], avoidWhen: [], keywords: ["fetch", "抓取"], primaryGroup: "web-research" },
|
||||
],
|
||||
};
|
||||
const usage = { input: 1, output: 1, cacheRead: 0, cacheWrite: 0, totalTokens: 2, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } };
|
||||
const ctx = testContext({
|
||||
model: { provider: "test", id: "catalog-model" } as ExtensionContext["model"],
|
||||
modelRegistry: {
|
||||
hasConfiguredAuth: () => true,
|
||||
complete: async (_model: unknown, context: { messages: Array<{ content: Array<{ text: string }> }> }) => {
|
||||
prompts.push(context.messages[0]?.content[0]?.text ?? "");
|
||||
return { content: [{ type: "text", text: JSON.stringify(generated) }], usage };
|
||||
},
|
||||
} as unknown as ExtensionContext["modelRegistry"],
|
||||
});
|
||||
handlers.get("session_start")?.({}, ctx);
|
||||
|
||||
const result = await registered.get("tool_search")?.execute(
|
||||
"call-1",
|
||||
{ group: "web-research" },
|
||||
undefined,
|
||||
undefined,
|
||||
ctx,
|
||||
);
|
||||
assert.equal(result?.details.loadedGroup, "web-research");
|
||||
assert.equal(result?.details.catalogSource, "model");
|
||||
assert.deepEqual(result?.usage, usage);
|
||||
assert.match(prompts[0] ?? "", /Search the public web with filters and return sources/);
|
||||
const cache = JSON.parse(await readFile(join(agentDir, "tool-search", "catalog-v1.json"), "utf8"));
|
||||
assert.equal(cache.catalog.groups[0].id, "web-research");
|
||||
} finally {
|
||||
if (previousAgentDir === undefined) delete process.env.PI_CODING_AGENT_DIR;
|
||||
else process.env.PI_CODING_AGENT_DIR = previousAgentDir;
|
||||
await rm(agentDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("uses the precomputed bundle groups without calling a model", async () => {
|
||||
const agentDir = await mkdtemp(join(tmpdir(), "my-pi-tool-search-bundle-"));
|
||||
const previousAgentDir = process.env.PI_CODING_AGENT_DIR;
|
||||
process.env.PI_CODING_AGENT_DIR = agentDir;
|
||||
|
||||
try {
|
||||
const handlers = new Map<string, (...args: any[]) => unknown>();
|
||||
const registered = new Map<string, RegisteredTool>();
|
||||
let modelCalls = 0;
|
||||
const sourceTools = [
|
||||
sourceTool("read", "Read a file"),
|
||||
sourceTool("tavily_web_search", "Search broadly for current web sources"),
|
||||
sourceTool("tavily_web_fetch", "Fetch a selected Tavily source"),
|
||||
];
|
||||
const api = {
|
||||
getAllTools: () => sourceTools,
|
||||
registerTool: (definition: unknown) => {
|
||||
const tool = definition as RegisteredTool & { name: string };
|
||||
registered.set(tool.name, tool);
|
||||
},
|
||||
registerCommand: () => {},
|
||||
setActiveTools: () => {},
|
||||
on: (event: string, handler: (...args: any[]) => unknown) => handlers.set(event, handler),
|
||||
} as unknown as ExtensionAPI;
|
||||
toolSearchExtension(api);
|
||||
const ctx = testContext({
|
||||
model: { provider: "test", id: "catalog-model" } as ExtensionContext["model"],
|
||||
modelRegistry: {
|
||||
hasConfiguredAuth: () => true,
|
||||
complete: async () => {
|
||||
modelCalls += 1;
|
||||
throw new Error("the bundle catalog should not invoke a model");
|
||||
},
|
||||
} as unknown as ExtensionContext["modelRegistry"],
|
||||
});
|
||||
handlers.get("session_start")?.({}, ctx);
|
||||
|
||||
const result = await registered.get("tool_search")?.execute(
|
||||
"call-1",
|
||||
{ group: "web-tavily" },
|
||||
undefined,
|
||||
undefined,
|
||||
ctx,
|
||||
);
|
||||
assert.equal(result?.details.loadedGroup, "web-tavily");
|
||||
assert.equal(result?.details.catalogSource, "bundle");
|
||||
assert.equal(modelCalls, 0);
|
||||
await assert.rejects(readFile(join(agentDir, "tool-search", "catalog-v1.json"), "utf8"), /ENOENT/);
|
||||
} finally {
|
||||
if (previousAgentDir === undefined) delete process.env.PI_CODING_AGENT_DIR;
|
||||
else process.env.PI_CODING_AGENT_DIR = previousAgentDir;
|
||||
await rm(agentDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2023",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"strict": true,
|
||||
"skipLibCheck": true,
|
||||
"allowImportingTsExtensions": true,
|
||||
"rewriteRelativeImportExtensions": true,
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["extensions/**/*.ts", "test/**/*.ts"]
|
||||
}
|
||||
Reference in New Issue
Block a user