From 2a6bf8c9291f2f0375ce50d4cfbc274457acb42b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BA=91=E6=9C=8D=E5=8A=A1=E9=83=A8-=E5=8F=B6=E6=9E=97?= =?UTF-8?q?=E7=AB=8B?= <1361666059@qq.com> Date: Wed, 21 Oct 2026 20:25:44 -0700 Subject: [PATCH] =?UTF-8?q?feat(pi-overflow):=20=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E7=BB=9F=E4=B8=80=E7=9A=84=E6=96=B0=E5=B7=A5=E5=85=B7=E7=BB=93?= =?UTF-8?q?=E6=9E=9C=E5=B1=95=E7=A4=BA=E6=88=AA=E6=96=AD=E6=89=A9=E5=B1=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 此前截断分散在多层且互相冲突：Pi bash 保留尾部 50KB，RTK 兜底截断又只留 开头 12000 字符（砍掉报错与完整输出路径），pi-chrome 截开头 30K，web_fetch 各 provider 不一致。新扩展统一负责"新结果给模型看多少"： - capture.ts 最先加载暂存原文，index.ts 最后加载（system-prompt 之前） - 超过 24K 字符时原文落盘 /my-pi/pi-overflow/（700/600、原子写）， 优先取 bash fullOutputPath 完整文件；模型看到 头 + 中段信号行 + 尾 的有界 视图，行号基于落盘原文、结尾永远可见 - overflow_query：按 id 正则搜索、行区间、jsonPath 只读查询，id 校验防穿越 - 收编：RTK truncate 默认关闭、删除 pi-chrome truncateText、web_fetch 默认 每页 20000（上限 100000）、web_search max_results 限定 1–20 - pi-compaction 遮蔽带 details.overflow.id 的结果时在占位符提示 overflow_query - 登记 tools.jsonc / routing / 权限 allow / tool-search 常驻与分组 Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- AGENTS.md | 3 +- README.md | 9 + config/pi-overflow.json | 13 + config/pi-permission-policy.jsonc | 3 + config/prompt/routing.jsonc | 4 + config/prompt/tools.jsonc | 1 + config/rtk-optimizer.json | 2 +- config/tool-search.json | 3 +- package-lock.json | 28 ++ package.json | 11 + pi-chrome/UPSTREAM.md | 1 + .../extensions/chrome-profile-bridge/index.ts | 16 +- pi-compaction/AGENTS.md | 2 +- pi-compaction/README.md | 1 + pi-compaction/UPSTREAM.md | 1 + pi-compaction/src/masking.ts | 24 +- pi-compaction/test/masking.test.ts | 21 ++ pi-overflow/AGENTS.md | 18 + pi-overflow/LICENSE | 21 ++ pi-overflow/README.md | 66 ++++ pi-overflow/UPSTREAM.md | 8 + pi-overflow/capture.ts | 30 ++ pi-overflow/index.ts | 167 ++++++++++ pi-overflow/package.json | 49 +++ pi-overflow/settings.ts | 100 ++++++ pi-overflow/src/query.ts | 188 +++++++++++ pi-overflow/src/stash.ts | 35 ++ pi-overflow/src/store.ts | 267 +++++++++++++++ pi-overflow/src/view.ts | 246 ++++++++++++++ pi-overflow/test/handler.test.ts | 314 ++++++++++++++++++ pi-overflow/test/query.test.ts | 112 +++++++ pi-overflow/test/view.test.ts | 86 +++++ pi-rtk-optimizer/AGENTS.md | 2 +- pi-rtk-optimizer/README.md | 4 +- pi-rtk-optimizer/UPSTREAM.md | 1 + pi-tool-search/CHANGELOG.md | 1 + pi-tool-search/README.md | 5 +- pi-tool-search/docs/dynamic-tool-loading.md | 2 +- pi-tool-search/extensions/bundle-groups.ts | 6 +- pi-tool-search/extensions/config.ts | 1 + pi-tool-search/test/config.test.ts | 1 + pi-web-search/README.md | 5 +- pi-web-search/index.ts | 23 +- tests/extension-load.test.ts | 15 + 44 files changed, 1882 insertions(+), 34 deletions(-) create mode 100644 config/pi-overflow.json create mode 100644 pi-overflow/AGENTS.md create mode 100644 pi-overflow/LICENSE create mode 100644 pi-overflow/README.md create mode 100644 pi-overflow/UPSTREAM.md create mode 100644 pi-overflow/capture.ts create mode 100644 pi-overflow/index.ts create mode 100644 pi-overflow/package.json create mode 100644 pi-overflow/settings.ts create mode 100644 pi-overflow/src/query.ts create mode 100644 pi-overflow/src/stash.ts create mode 100644 pi-overflow/src/store.ts create mode 100644 pi-overflow/src/view.ts create mode 100644 pi-overflow/test/handler.test.ts create mode 100644 pi-overflow/test/query.test.ts create mode 100644 pi-overflow/test/view.test.ts diff --git a/AGENTS.md b/AGENTS.md index f0de8c8..d5fc777 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,6 +19,7 @@ | `pi-minimal-footer/` | 紧凑 footer：上下文仪表、订阅用量条、扩展状态 | | `pi-notify/` | Kitty OSC 99 完成与权限确认通知，精确聚焦来源窗口 | | `pi-compaction/` | 压力触发的观察遮蔽（Pi context_edit）、阈值整段摘要、effective token、native compaction | +| `pi-overflow/` | 新工具结果统一截断：超阈值原文落盘 + 有界视图 + `overflow_query` | | `extensions/fff-override.ts` | FFF `override` 模式接管 `find`/`grep`/`multi_grep` 与 `@` 补全 | | `extensions/codegraph.ts` | 原生 `codegraph_explore`，自有 stdio JSON-RPC 直连 `codegraph serve`（`codegraph-client.ts`） | | `extensions/context-mode.ts` | 经 `prompt-guard.ts` 加载 context-mode adapter 与 skills（`context-mode-filter.ts` 过滤冲突锚点） | @@ -35,7 +36,7 @@ ## 全局硬规则 - 系统提示词单一所有权：`extensions/system-prompt.ts` 排在 `pi.extensions` 最后，是唯一允许在 `before_agent_start` 返回 `systemPrompt` 的扩展；其他扩展只能经 `systemPromptOptions.sections.` 贡献分区，第三方扩展必须经 `extensions/` wrapper + `prompt-guard.ts` 加载，新增工具必须在 `config/prompt/tools.jsonc` 登记。宿主传入 `customPrompt`（子 Agent / SYSTEM.md）时不覆盖其开场白；子 Agent 有独立提示词画像 `config/prompt/subagent/`（preamble + 内置 Agent），判定规则与路由 scope/`subagentExclude` 细则见 `extensions/AGENTS.md`。 -- 加载顺序：`permission-system` → `subagents` → `pi-ssh` → `pi-chrome` → `pi-tool-search` → `system-prompt`（最后）；`pi-hermes-memory` 必须排在 `pi-compaction` 之前（`session_before_compact` 取消语义），`pi-compaction` 在 `pi-minimal-footer` 与 `context-view` 之前。 +- 加载顺序：`pi-overflow/capture.ts` 必须排在最前（先于 RTK 等结果改写扩展暂存原文），`permission-system` → `subagents` → `pi-ssh` → `pi-chrome` → `pi-tool-search` → `pi-overflow/index.ts` → `system-prompt`（最后）；`pi-hermes-memory` 必须排在 `pi-compaction` 之前（`session_before_compact` 取消语义），`pi-compaction` 在 `pi-minimal-footer` 与 `context-view` 之前。 - 依赖：外部依赖在根 `package.json` 用精确版本，自维护扩展一律 `file:` 本地依赖并更新锁文件；Pi 核心包只作 optional host peer，根 runtime deps 不含 Pi runtime，任何目录不得再装一套 Pi runtime；各子包用于测试的 `@earendil-works/*` devDependencies 一律写 `"latest"`，经 `scripts/update-pi-devdeps.sh` 统一刷新；原生构建依赖只按锁定版本进根 `allowScripts`（当前仅 `better-sqlite3@12.11.1`；pi-hermes-memory 的 `better-sqlite3@13` 用包内 N-API 预编译，不新增放行）。不要用仓库级 `.pi/settings.json` 代替组合包依赖。根 `files` 对子包按运行时子路径白名单列出（源码、package.json、README/LICENSE/CHANGELOG/UPSTREAM），不得整目录列入，以免打入嵌套 node_modules、dist 和测试；由 `tests/package-files.test.ts` 守护。 - 配置：`config/` 下全部文件是权威源、必须提交；不得读写 `settings.json` 自定义段、agent 目录独立配置文件或以环境变量作持久默认值；`config/` 只放机器无关值，路径类默认值在代码中运行时计算。`/my-pi/` 只放可丢弃运行时状态（日志、缓存），连同旧 `extensions/pi-permission-system/` 残留、构建产物与依赖目录都不提交；用户数据（记忆、会话、`auth.json` 等）不属于组合包配置，也不得放入该目录。 - 组合包不加载任何 MCP adapter/SDK：Exa 与 CodeGraph 均用自有直连客户端。 diff --git a/README.md b/README.md index 58967c5..47bc72f 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ - 本仓库维护的 `pi-minimal-footer`：基于 `@ogulcancelik/pi-minimal-footer@0.1.10`，用紧凑的上下文仪表和订阅用量条替换默认 footer。 - 本仓库维护的 `pi-notify`：基于 `@smoose/pi-notify@0.1.1`，使用 Kitty OSC 99 在 Pi 完全 settled 或权限系统即将请求人工确认时通知，并支持点击精确聚焦来源 Kitty 窗口。 - 本仓库维护的 `pi-compaction`：压力触发时以 Pi `context_edit` 批量遮蔽旧工具结果、空闲时按阈值触发整段摘要、effective token 计量、`context_tree_query`、portable checkpoint 与 Codex native compaction。 +- 本仓库自维护的 `pi-overflow`：统一的新工具结果展示截断——超阈值原文落盘、向模型展示「头 + 信号行 + 尾」的有界视图，并提供 `overflow_query` 按 id 搜索、行区间或 JSON 路径取回原文。 - 本仓库维护的 `pi-ask-user`：提供模型主动调用的 `ask_user_question`；TUI 中保留单题、多题、选择、文本、自定义回答与提交前复核，Paseo 等兼容 RPC 客户端通过标准 Pi extension UI 逐题交互。 - 本仓库自维护的 `pi-worktree`：纯斜杠命令（`/wt` / `/worktree`）git worktree 管理，`/wt new` 一键拆分、`/wt merge` 一键合并回来源分支（默认 ff-only，可选 `--squash`），通过 `SessionManager.forkFrom` + `switchSession` 切换会话 cwd；登记状态在 `/pi-worktree/state.json`，不注册模型工具、不 push。 - 本仓库维护的 `pi-tool-search`：从完整工具定义生成并缓存经过校验的工作流分组，默认以最多 5 个动态组、28 个动态工具的 LRU 策略加载原始完整 schema，并支持原子批量激活。 @@ -386,3 +387,11 @@ compaction 和排队 follow-up 完成前不会提前通知；同时监听 `pi-pe 原生远程压缩同时支持 `openai-codex` + `openai-codex-responses` 和 `sub-api` + `openai-responses`，使用触发时当前会话模型。SubAPI 保留现有模型配置，从 `baseUrl` 调用公共 `/responses` 并使用 `compaction_trigger` 流式协议；优先保留非空显式 `Authorization`，否则使用 API key，也支持仅凭非空 `cf-aig-authorization` 的代理认证。缺失或空白凭据在请求前拒绝。SubAPI 不要求客户端解析 ChatGPT account ID，也不调用独立 `/responses/compact`；Codex 的 JWT 要求保持不变。网关部署须支持 Sub2API 原生压缩 v2；请求失败沿用 Pi 回退，切换模型或 Provider 时使用 portable summary。 本实现不读取或迁移旧 `contextPrune`、`context-prune-*` 或旧 Codex native checkpoint；切换后应创建新 Session。它与 Context Mode 职责互补：Context Mode 防止大型原始内容进入会话，Pi Compaction 管理已经进入会话的长期历史。 + +### Pi Overflow + +组合包自维护的 `pi-overflow/` 统一负责「每个新工具结果给模型看多少」。它有两个入口：`capture.ts` 排在 `pi.extensions` 最前，只把原始结果文本暂存到进程内 Map（容量 64），绝不修改结果；`index.ts` 排在 `system-prompt.ts` 之前、所有其他扩展之后，此时 RTK 等 handler 已经完成变换——它用最终可见文本判断是否超阈值，落盘的原文取 bash 的 `details.fullOutputPath` 文件、暂存原文或当前文本三者中的最完整版本，视图与其中的行号则一律基于落盘原文生成，保证与 `overflow_query` 读取的内容一致。 + +当最终文本仍超过 `thresholdChars`（默认 24000）时，模型看到的是确定性的「头部 40 行 + 中段匹配 error/warn 等信号行（至多 40 条）+ 尾部 120 行」视图，并写入 `details.overflow.id`；当上游已把结果压短但原文超阈值时，只追加一行 `[pi-overflow id=…]` 恢复标记。原文以 `sha1(sessionId+toolCallId)` 前 12 位 hex 为 id 存入 `/my-pi/pi-overflow/`（目录 `700`、文件 `600`、原子写入），session 启动时按 `retentionDays`/`maxStoreBytes` 清理。模型可用 `overflow_query` 按 id 正则搜索（带上下文合并）、按行区间或 `jsonPath` 读取；`read`/`edit`/`write`/`overflow_query`/`ask_user_question` 与嵌套调用永远豁免。配置为组合包内 `config/pi-overflow.json`，详见 [`pi-overflow/README.md`](pi-overflow/README.md)。 + +RTK 的兜底截断与 pi-chrome 的 30K 字符截断已收编进本扩展：`config/rtk-optimizer.json` 中 `outputCompaction.truncate.enabled` 默认关闭，chrome_* 工具返回完整文本；pi-web-search 的 `web_fetch` 默认 `max_characters` 提高到 20000（上限 100000），`web_search` 的 `max_results` 约束在 1–20。 diff --git a/config/pi-overflow.json b/config/pi-overflow.json new file mode 100644 index 0000000..99c5edd --- /dev/null +++ b/config/pi-overflow.json @@ -0,0 +1,13 @@ +{ + "enabled": true, + "thresholdChars": 24000, + "headLines": 40, + "tailLines": 120, + "maxSignalLines": 40, + "signalPattern": "\\b(error|errors|fail|failed|failure|warn|warning|exception|traceback|panic|fatal)\\b", + "maxLineChars": 500, + "exemptTools": ["read", "edit", "write", "overflow_query", "ask_user_question"], + "retentionDays": 7, + "maxStoreBytes": 1073741824, + "maxEntryBytes": 67108864 +} diff --git a/config/pi-permission-policy.jsonc b/config/pi-permission-policy.jsonc index 89a33cb..a6d40be 100644 --- a/config/pi-permission-policy.jsonc +++ b/config/pi-permission-policy.jsonc @@ -86,6 +86,9 @@ // index and never touches the workspace. It is pinned in Tool Search's // alwaysEnabled set, so it must not pay an AutoReview round trip per turn. "context_tree_query": "allow", + // pi-overflow's inspector reads only the extension's own stored output + // directory (id-validated); pinned in Tool Search's alwaysEnabled set. + "overflow_query": "allow", // Model-chosen URLs can exfiltrate data — same reason curl/wget ask. "web_fetch": "ask", diff --git a/config/prompt/routing.jsonc b/config/prompt/routing.jsonc index 2443663..cbbe536 100644 --- a/config/prompt/routing.jsonc +++ b/config/prompt/routing.jsonc @@ -65,6 +65,10 @@ "whenAll": ["chrome_snapshot", "ctx_execute_file"], "text": "After chrome_snapshot or a Chrome action with includeSnapshot=true, immediately analyze the returned workspace-relative capture path with ctx_execute_file; never use read or cat for the raw JSON. Print only the uids, text, or state needed for the next action, and take a fresh snapshot after any interaction because the previous capture may be stale." }, + { + "whenAny": ["overflow_query"], + "text": "When a tool result starts with or ends with a [pi-overflow id=…] marker, the full original output is stored; inspect it with overflow_query (search, line ranges, or jsonPath) instead of re-running the command or dumping it again." + }, { "whenAny": ["read"], "text": "Use read only when exact source is needed or the file is small. Before editing, request the smallest useful offset/limit range; do not read a complete large file merely to copy, compare, hash, count, explore, or summarize it." diff --git a/config/prompt/tools.jsonc b/config/prompt/tools.jsonc index 7b0e970..f6e133d 100644 --- a/config/prompt/tools.jsonc +++ b/config/prompt/tools.jsonc @@ -62,6 +62,7 @@ "memory_remove": { "inherit": true }, "skill_manage": { "inherit": true }, "context_tree_query": { "inherit": true }, + "overflow_query": { "inherit": true }, "ask_user_question": { "inherit": true }, diff --git a/config/rtk-optimizer.json b/config/rtk-optimizer.json index f671e39..1b7450b 100644 --- a/config/rtk-optimizer.json +++ b/config/rtk-optimizer.json @@ -11,7 +11,7 @@ "enabled": false }, "truncate": { - "enabled": true, + "enabled": false, "maxChars": 12000 }, "sourceCodeFilteringEnabled": false, diff --git a/config/tool-search.json b/config/tool-search.json index 357a049..b2b7601 100644 --- a/config/tool-search.json +++ b/config/tool-search.json @@ -11,7 +11,8 @@ "memory_replace", "memory_remove", "skill_manage", - "context_tree_query" + "context_tree_query", + "overflow_query" ], "showToolSearchFooterStatus": false, "maxActiveGroups": 5, diff --git a/package-lock.json b/package-lock.json index a2841c2..6aa14e0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,6 +21,7 @@ "pi-hashline-edit": "0.8.3", "pi-hermes-memory": "file:./pi-hermes-memory", "pi-lsp": "0.1.7", + "pi-overflow": "file:./pi-overflow", "pi-ssh": "file:./pi-ssh", "pi-tool-search": "file:./pi-tool-search", "pi-web-search": "file:./pi-web-search", @@ -5571,6 +5572,10 @@ "typebox": "*" } }, + "node_modules/pi-overflow": { + "resolved": "pi-overflow", + "link": true + }, "node_modules/pi-ssh": { "resolved": "pi-ssh", "link": true @@ -7093,6 +7098,29 @@ "@earendil-works/pi-coding-agent": "*" } }, + "pi-overflow": { + "version": "0.1.0", + "license": "MIT", + "devDependencies": { + "@earendil-works/pi-coding-agent": "latest", + "typebox": "latest" + }, + "engines": { + "node": ">=22.19.0" + }, + "peerDependencies": { + "@earendil-works/pi-coding-agent": "*", + "typebox": "*" + }, + "peerDependenciesMeta": { + "@earendil-works/pi-coding-agent": { + "optional": true + }, + "typebox": { + "optional": true + } + } + }, "pi-permission-system": { "name": "@gotgenes/pi-permission-system", "version": "26.2.1", diff --git a/package.json b/package.json index efdfc8d..15ebbae 100644 --- a/package.json +++ b/package.json @@ -59,6 +59,14 @@ "pi-subagents/SECURITY.md", "pi-subagents/LICENSE", "pi-subagents/UPSTREAM.md", + "pi-overflow/index.ts", + "pi-overflow/capture.ts", + "pi-overflow/settings.ts", + "pi-overflow/src", + "pi-overflow/package.json", + "pi-overflow/README.md", + "pi-overflow/LICENSE", + "pi-overflow/UPSTREAM.md", "pi-notify/src", "pi-notify/package.json", "pi-notify/README.md", @@ -111,6 +119,7 @@ ], "pi": { "extensions": [ + "./pi-overflow/capture.ts", "./extensions/fff-override.ts", "./pi-web-search/index.ts", "./extensions/codegraph.ts", @@ -130,6 +139,7 @@ "./pi-worktree/index.ts", "./pi-chrome/extensions/chrome-profile-bridge/index.ts", "./pi-tool-search/extensions/index.ts", + "./pi-overflow/index.ts", "./extensions/system-prompt.ts" ], "skills": [ @@ -149,6 +159,7 @@ "pi-compaction": "file:./pi-compaction", "pi-hashline-edit": "0.8.3", "pi-hermes-memory": "file:./pi-hermes-memory", + "pi-overflow": "file:./pi-overflow", "pi-lsp": "0.1.7", "pi-ssh": "file:./pi-ssh", "pi-tool-search": "file:./pi-tool-search", diff --git a/pi-chrome/UPSTREAM.md b/pi-chrome/UPSTREAM.md index e4aa63b..e223775 100644 --- a/pi-chrome/UPSTREAM.md +++ b/pi-chrome/UPSTREAM.md @@ -21,5 +21,6 @@ - click/type/fill/key 上的 `includeSnapshot=true` 会保存动作后的快照并返回其捕获路径，而不是把快照嵌入工具结果。 - 授权后的 `before_agent_start` 不再返回 `systemPrompt`：Chrome 使用提示写入 `event.systemPromptOptions.sections.chrome_profile_bridge` 结构化分区，由组合包单一提示词出口 `extensions/system-prompt.ts` 统一组装，顺序由 `config/prompt/sections.jsonc` 控制；未授权时不贡献任何提示词内容。 - 移除 `PI_CHROME_BRIDGE_HOST`/`PI_CHROME_BRIDGE_PORT` 环境变量，host/port 固定为 `127.0.0.1:17318`（Companion 扩展 `service_worker.js` 同样硬编码该地址，单侧可配只会断连且非回环 host 有安全风险）。 +- 移除 `MAX_TEXT_CHARS`/`truncateText` 的 30K 字符文本截断：chrome_* 工具返回完整文本，超长结果的展示截断由 pi-overflow 统一负责；`MAX_ELEMENTS` 等条数上限保留不变。 同步上游时，请以导入 commit 为基准比较并选择性移植改动。不得整体覆盖这些本地生命周期、隐私和 Context Mode 集成。 diff --git a/pi-chrome/extensions/chrome-profile-bridge/index.ts b/pi-chrome/extensions/chrome-profile-bridge/index.ts index 8f8914d..ec215f6 100644 --- a/pi-chrome/extensions/chrome-profile-bridge/index.ts +++ b/pi-chrome/extensions/chrome-profile-bridge/index.ts @@ -67,14 +67,8 @@ const PI_CHROME_AUTH_KEY = "__piChromeProfileBridgeAuth__"; const DEFAULT_HOST = "127.0.0.1"; const DEFAULT_PORT = 17318; const DEFAULT_TIMEOUT_MS = 30_000; -const MAX_TEXT_CHARS = 30_000; const MAX_ELEMENTS = 80; -function truncateText(text: string, maxChars = MAX_TEXT_CHARS): string { - if (text.length <= maxChars) return text; - return `${text.slice(0, maxChars)}\n\n[truncated ${text.length - maxChars} characters]`; -} - function safeJson(value: unknown): string { return JSON.stringify(value, null, 2); } @@ -142,7 +136,7 @@ function formatChromeInspect(inspect: any): string { lines.push("\n## Ancestors"); for (const a of inspect.ancestors.slice(0, 6)) lines.push(`- ${a.uid} ${a.role || a.tag} ${compactLine(a.label || a.selector, 120)}`); } - return truncateText(lines.join("\n")); + return lines.join("\n"); } function extensionRoot(): string { @@ -1512,7 +1506,7 @@ Usage rules: : typeof value === "string" ? value : safeJson(value) ?? "undefined"; - return { content: [{ type: "text", text: truncateText(text) }], details: { value: value as Json } }; + return { content: [{ type: "text", text }], details: { value: value as Json } }; }, }); @@ -1691,7 +1685,7 @@ Usage rules: }), async execute(_id, params, signal): Promise { const result = await authorizedBridgeSend("page.console.list", withBackground(params), DEFAULT_TIMEOUT_MS, signal); - return { content: [{ type: "text", text: truncateText(safeJson(result)) }], details: { result: result as Json } }; + return { content: [{ type: "text", text: safeJson(result) }], details: { result: result as Json } }; }, }); @@ -1713,7 +1707,7 @@ Usage rules: }), async execute(_id, params, signal): Promise { const result = await authorizedBridgeSend("page.network.list", withBackground(params), DEFAULT_TIMEOUT_MS, signal); - return { content: [{ type: "text", text: truncateText(safeJson(result)) }], details: { result: result as Json } }; + return { content: [{ type: "text", text: safeJson(result) }], details: { result: result as Json } }; }, }); @@ -1733,7 +1727,7 @@ Usage rules: }), async execute(_id, params, signal): Promise { const result = await authorizedBridgeSend("page.network.get", withBackground(params), DEFAULT_TIMEOUT_MS, signal); - return { content: [{ type: "text", text: truncateText(safeJson(result)) }], details: { result: result as Json } }; + return { content: [{ type: "text", text: safeJson(result) }], details: { result: result as Json } }; }, }); diff --git a/pi-compaction/AGENTS.md b/pi-compaction/AGENTS.md index 666beee..8faf557 100644 --- a/pi-compaction/AGENTS.md +++ b/pi-compaction/AGENTS.md @@ -7,7 +7,7 @@ - `pi-compaction` 负责已经进入 Session 的历史投影，与 Context Mode 的输入隔离职责互补。首次 Session hydration 允许一次完整扫描；后续普通 `context` hot path 必须复用不可变 snapshot 或只处理 delta，不得调用 summarizer、写 Session 或重新构建完整 branch。 - L2 遮蔽只用 Pi 原生 `context_edit`（`{ type: "context_edit", targetId, replacement }`）处理保护窗口外的旧 toolResult：成批提交、受 trigger/target 回差与 `minClearTokens` 约束，不逐条即时替换；不为遮蔽写 `segment-summary` 等 custom entry，原始条目、UI 与导出保持不变。 - 不得改写最近 `keepRecentTurns` 轮；不得遮蔽 user/assistant 消息、`excludeTools` 中的工具、`SKILL.md` 的 `read` 结果、已遮蔽或已省略的条目、以及小于 `minEntryTokens` 的结果。 -- 遮蔽占位符必须确定性、单行、以 `MASK_PREFIX`（`"[pi-compaction] Earlier "`）开头，并在文本中指向 `context_tree_query` 恢复；`context_tree_query` 与 native checkpoint 的 portable summary 都必须有界。 +- 遮蔽占位符必须确定性、单行、以 `MASK_PREFIX`（`"[pi-compaction] Earlier "`）开头，并在文本中指向 `context_tree_query` 恢复；`context_tree_query` 与 native checkpoint 的 portable summary 都必须有界。被遮蔽的 toolResult 若带 `details.overflow.id`（pi-overflow 已存原文），占位符末尾追加 `overflow_query` 提示——只读该公开字段，不得 import pi-overflow 内部实现。 - L3 整段摘要只能从空闲的 `agent_settled` 通过 `ctx.compact` 触发（`ctx.compact` 会中止当前运行，非空闲时调用会破坏运行）；同一 leaf 不重复触发。阈值以绝对 token 计并受窗口比例约束（`effectiveThreshold = min(tokens, floor(ratio × contextWindow))`）。 - `context` 钩子不得改写消息——非 checkpoint 路径只更新 snapshot 并返回 undefined；唯一例外是 native checkpoint 路径：checkpoint 之后的条目在投影时必须应用各 target 最新的 `context_edit`（null 省略、字符串规范成 text block），因为 `sessionEntryToContextMessages` 不处理 edits。 - `session_before_compact` 只负责生成 portable/native 摘要，不得按自身阈值取消 Pi 的压缩（Pi 的 `contextTokens > window - reserveTokens` 是最终安全网）。 diff --git a/pi-compaction/README.md b/pi-compaction/README.md index 4999f14..c0e7abb 100644 --- a/pi-compaction/README.md +++ b/pi-compaction/README.md @@ -79,6 +79,7 @@ Compaction entries still carry `pi-compaction` version 1 details; native checkpo ## Interfaces - `context_tree_query`: recover the original output of a tool result that pi-compaction cleared from context, by tool call ID (1–20 ids per call, bounded output). Use it when a `[pi-compaction] Earlier` placeholder lacks evidence needed for the current task. +- pi-overflow linkage: when the masked tool result carries `details.overflow.id` (pi-overflow stored the oversized original), the placeholder ends with `Full original stored: overflow_query {"id":"…"}`. pi-compaction only reads that public field and never imports pi-overflow internals. - `/compaction-status`: show raw/effective tokens, reclaimed tokens, confidence, segments, masked results, and generation. - Service `my-pi.pi-compaction.service.v1`: `getSnapshot`, `getMetrics`, and `query(sessionId, ids)` — `query` returns a placeholder record only for tool call IDs that are currently masked, so consumers such as the permission system's AutoReview see the same evidence the model does. diff --git a/pi-compaction/UPSTREAM.md b/pi-compaction/UPSTREAM.md index 2323aae..724802e 100644 --- a/pi-compaction/UPSTREAM.md +++ b/pi-compaction/UPSTREAM.md @@ -33,3 +33,4 @@ ## 本地改动记录 - 上下文管理重构为 L2/L3：L2 在上下文压力下经 `turn_end` 提交 Pi 原生 `context_edit`，把保护窗口外的旧 toolResult 批量遮蔽为指向 `context_tree_query` 的确定性占位符（原始条目保留、不写 custom entry）；L3 只在空闲 `agent_settled` 且达到绝对 token 阈值（受窗口比例约束）时调用一次 `ctx.compact`。删除逐条确定性存根、后台语义 worker、`softRatio`/`hardRatio`/`targetRatio` 配置与 `session_before_compact` 的自行阈值取消；Pi 原生阈值压缩保留为最终安全网。 +- L2 遮蔽占位符联动 pi-overflow：被遮蔽的 toolResult 若携带 `details.overflow.id`，占位符末尾追加 `overflow_query {"id":"…"}` 恢复提示。 diff --git a/pi-compaction/src/masking.ts b/pi-compaction/src/masking.ts index fd7b30e..e66defe 100644 --- a/pi-compaction/src/masking.ts +++ b/pi-compaction/src/masking.ts @@ -52,14 +52,25 @@ function json(value: unknown): string { } } -function maskPlaceholder(toolName: string, args: unknown, toolCallId: string, originalTokens: number): string { +function maskPlaceholder( + toolName: string, + args: unknown, + toolCallId: string, + originalTokens: number, + overflowId?: string, +): string { let argsText = json(args); if (argsText.length > MAX_PLACEHOLDER_ARGS_CHARS) { argsText = `${argsText.slice(0, MAX_PLACEHOLDER_ARGS_CHARS)}…`; } + // `details.overflow.id` is pi-overflow's public contract — read the field + // only, never import its implementation. + const overflowHint = typeof overflowId === "string" && overflowId.length > 0 + ? ` Full original stored: overflow_query {"id":"${overflowId}"}.` + : ""; return `${MASK_PREFIX}${toolName} result cleared to save context (~${originalTokens} tokens). ` + `Call args: ${argsText}. Re-run the call if still needed, or recover the original with ` + - `context_tree_query {"toolCallIds":["${toolCallId}"]}.`; + `context_tree_query {"toolCallIds":["${toolCallId}"]}.${overflowHint}`; } function isSkillRead(toolName: string, args: unknown): boolean { @@ -136,7 +147,14 @@ export function planMasking(params: { if (contentText(message.content).startsWith(MASK_PREFIX)) continue; const originalTokens = params.estimate(message); if (originalTokens < masking.minEntryTokens) continue; - const placeholder = maskPlaceholder(toolName, args, message.toolCallId, originalTokens); + const overflowId = (message.details as { overflow?: { id?: unknown } } | undefined)?.overflow?.id; + const placeholder = maskPlaceholder( + toolName, + args, + message.toolCallId, + originalTokens, + typeof overflowId === "string" ? overflowId : undefined, + ); const placeholderTokens = params.estimate({ ...message, content: [{ type: "text", text: placeholder }] }); clearedTokens += originalTokens - placeholderTokens; masked.push({ diff --git a/pi-compaction/test/masking.test.ts b/pi-compaction/test/masking.test.ts index 69731c5..fb9b6f0 100644 --- a/pi-compaction/test/masking.test.ts +++ b/pi-compaction/test/masking.test.ts @@ -137,6 +137,27 @@ test("placeholder is deterministic, names the tool and call id, and truncates ar assert.ok(placeholder.includes("…")); }); +test("a masked result carrying details.overflow.id gets an overflow_query hint in the placeholder", () => { + const withOverflow = { + ...toolResultMessage("r1", "c1", "a".repeat(4_000)), + details: { overflow: { id: "0123456789ab", totalLines: 5, totalBytes: 4000, truncated: true } }, + }; + const withoutOverflow = toolResultMessage("r2", "c2", "b".repeat(4_000)); + const entries = [ + entry("e1", assistantMessage("a1", [{ id: "c1", name: "bash" }, { id: "c2", name: "bash" }])), + entry("e2", withOverflow), + entry("e3", withoutOverflow), + ...protect(), + ]; + const masking = { ...MASKING, minClearTokens: 1 }; + const plan = planMasking({ contextEntries: entries, currentTokens: 3_500, contextWindow: 0, masking, estimate }); + assert.equal(plan.masked.length, 2); + const first = plan.masked.find((m) => m.toolCallId === "c1")!.placeholder; + const second = plan.masked.find((m) => m.toolCallId === "c2")!.placeholder; + assert.ok(first.includes(` Full original stored: overflow_query {"id":"0123456789ab"}.`)); + assert.ok(!second.includes("overflow_query")); +}); + test("the small-window ratio caps both trigger and target", () => { const masking: MaskingConfig = { ...MASKING, diff --git a/pi-overflow/AGENTS.md b/pi-overflow/AGENTS.md new file mode 100644 index 0000000..459d1de --- /dev/null +++ b/pi-overflow/AGENTS.md @@ -0,0 +1,18 @@ +# my-pi 维护约定（pi-overflow） + +- 完全自维护（无上游）的新结果展示截断扩展，负责"每个新 tool_result 给模型看多少"：`capture.ts` 最先加载只暂存原文，`index.ts` 最后加载（在 `system-prompt.ts` 之前）生成有界视图并注册 `overflow_query`。 +- 根 `pi.extensions` 顺序是硬约束：`./pi-overflow/capture.ts` 必须第一，`./pi-overflow/index.ts` 必须在所有其他扩展之后、`./extensions/system-prompt.ts` 之前；跨扩展暂存走 `globalThis[Symbol.for("my-pi.pi-overflow.stash.v1")]`，容量 64 条、淘汰最旧。 +- `tool_result` handler 链中 capture 不得修改结果；index 改写 content 时必须原样回传 `structuredContent`、保留 image 部分、合并而非丢弃已有 `details`（如 RTK 压缩元数据、bash `fullOutputPath`）。 +- `read`/`edit`/`write`/`overflow_query`/`ask_user_question` 与嵌套调用（`parentToolCallId`）永远豁免；read 靠自身 offset/limit 分页，不得破坏 hashline 锚点。 +- 视图与行号一律基于刚落盘的存储原文（不是 current 文本）：文件来源用流式逐行扫描收集 head/环形 tail/signals，文本来源切同构片段；视图必须确定性、不超 `thresholdChars`、结尾永远可见；落盘原文优先取 `details.fullOutputPath` 文件（超 `maxEntryBytes` 只留头尾各半），其次暂存原文，再次当前文本。 +- 超预算收缩优先级为 tail > signals > head：head 先缩到 min(10, headLines)，signals 逐步减半至 10 再到 0，再按约 35/65 缩 head/tail，按字符截断兜底。 +- signal 扫描与 `overflow_query` 的 search 正则只匹配每行前 10,000 字符；query 输出每行按 `maxLineChars` 截断。 +- 存储位于 `/my-pi/pi-overflow/`（目录 `700`、文件 `600`、原子写入），id 为 `sha1(sessionId + "\0" + toolCallId)` 前 12 位 hex；`overflow_query` 的 id 必须匹配 `^[0-9a-f]{12}$` 防路径穿越；`session_start` 按 `retentionDays`/`maxStoreBytes` 清理并删除超过 1 小时的 `*.tmp-*` 残留，失败不得抛出。 +- 持久配置只读组合包 `config/pi-overflow.json`（`settings.ts` 逐字段校验、非法回退默认、signalPattern 编译失败回退默认且不敏感大小写）；不得读写 `settings.json` 或以环境变量作持久默认。 +- `overflow_query` 只读本扩展自己的存储目录；非法 id/正则/JSON/路径一律返回清楚错误文本，不得抛出。 +- 与 pi-compaction 的唯一联动是被遮蔽 toolResult 的 `details.overflow.id` 公开字段——pi-compaction 只读它、不得 import 本包任何实现。 +- 测试必须注入临时存储目录，不得写真实 agent 目录。 + +## 验证 + +- 在本目录内至少运行 `npm test` 与 `npm run check`；涉及加载入口、`pi.extensions` 顺序或 `files` 白名单时还要运行根 `node --test tests/extension-load.test.ts` 与 `tests/package-files.test.ts`。 diff --git a/pi-overflow/LICENSE b/pi-overflow/LICENSE new file mode 100644 index 0000000..57a1a7e --- /dev/null +++ b/pi-overflow/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 sia + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/pi-overflow/README.md b/pi-overflow/README.md new file mode 100644 index 0000000..55804aa --- /dev/null +++ b/pi-overflow/README.md @@ -0,0 +1,66 @@ +# pi-overflow + +Bounded display for oversized tool results. When a tool result exceeds +`thresholdChars`, pi-overflow persists the full original to disk and shows the +model a deterministic view — head lines, signal lines (error/fail/warn/…), and +tail lines — plus an `overflow_query` tool to search or read the stored +original by id. + +## Two-stage load order + +- `capture.ts` — **first** in `pi.extensions`. A `tool_result` handler that + never modifies anything; it stashes the untouched original text in + `globalThis[Symbol.for("my-pi.pi-overflow.stash.v1")]` (capacity 64) so the + bound stage can still reach the pre-RTK original. +- `index.ts` — **last** content extension, right before `system-prompt.ts`. + Its `tool_result` handler sees the final text after every other handler + (RTK semantic compaction included) has run, then: + - resolves the *original*: `details.fullOutputPath` file (streamed, capped + at `maxEntryBytes` head+tail) → stashed raw text → current text; + - if the current text still exceeds `thresholdChars`, replaces it with the + bounded view **built from the stored file** — line numbers and totals in + the view are exactly what `overflow_query` will read, even when RTK or + bash already cut the in-context text (file sources are scanned + line-by-line in a stream, never held in memory); + - if the current text fits but the original did not (an earlier extension + already cut it), keeps the text and appends a one-line + `[pi-overflow id=…]` recovery marker; + - preserves image parts and `structuredContent`, and merges + `details.overflow = { id, totalLines, totalBytes, truncated }` into the + existing details. +- Exempt tools (`read`, `edit`, `write`, `overflow_query`, + `ask_user_question`) and nested calls (`parentToolCallId`) are never + touched. + +## Stored originals + +`/my-pi/pi-overflow/.txt` + `.json` metadata, where +`id = sha1(sessionId + "\0" + toolCallId)[0:12]`. Directories are `0700`, +files `0600`, writes are atomic. `session_start` cleans entries older than +`retentionDays` and evicts oldest-first past `maxStoreBytes`. + +## `overflow_query` + +- `id` (required, `^[0-9a-f]{12}$` — path traversal safe) +- `search` — case-insensitive regex; hits come back with `context` lines and + adjacent hits merge; capped by `maxResults` +- `lines: { start, end }` — 1-based inclusive line range +- `jsonPath` — dot/bracket path into stored JSON (`a.b[3].c`) +- No selector → metadata + first-screen view. Output is always bounded by + `thresholdChars` and reports whether it was truncated. + +## Configuration + +`config/pi-overflow.json` inside the installed bundle. Every field is +validated independently and falls back to its default; an uncompilable +`signalPattern` falls back to the default pattern (case-insensitive). + +## Development + +```sh +npm test # node --test test/*.test.ts +npm run check # strip-types syntax check + tests +``` + +Tests inject a temporary store root — they never write to a real agent +directory. diff --git a/pi-overflow/UPSTREAM.md b/pi-overflow/UPSTREAM.md new file mode 100644 index 0000000..16de9c2 --- /dev/null +++ b/pi-overflow/UPSTREAM.md @@ -0,0 +1,8 @@ +# 上游来源 + +- 本项目完全自维护，无上游；未从任何外部仓库或 npm 包导入源码。 +- 实现仅依赖 Pi 扩展宿主 API（`@earendil-works/pi-coding-agent`）与 `typebox`，两者均为组合包 peer 依赖。 + +## 本地改动记录 + +- 初始实现：`capture.ts`（首个加载，暂存原始 tool_result 文本）、`index.ts`（最后加载，生成有界视图 + 注册 `overflow_query`）、`src/`（视图、存储、查询、跨扩展暂存区）。 diff --git a/pi-overflow/capture.ts b/pi-overflow/capture.ts new file mode 100644 index 0000000..2902e4d --- /dev/null +++ b/pi-overflow/capture.ts @@ -0,0 +1,30 @@ +import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; +import { readOverflowSettings } from "./settings.ts"; +import { getStash, stashResult } from "./src/stash.ts"; + +/** + * Pre-bound capture: loaded FIRST in pi.extensions, before RTK or any other + * result-rewriting extension. It only stashes the untouched original text — + * it never modifies the result. The bound stage (`index.ts`, loaded last) + * consumes the stash after all other handlers have run. + */ +export default function overflowCapture(pi: ExtensionAPI): void { + const settings = readOverflowSettings(); + pi.on("tool_result", (event) => { + if (!settings.enabled) return undefined; + if (settings.exemptTools.includes(event.toolName)) return undefined; + if (event.parentToolCallId) return undefined; + const rawText = event.content + .filter((part): part is { type: "text"; text: string } => part.type === "text") + .map((part) => part.text) + .join("\n"); + const fullOutputPath = + typeof event.details?.fullOutputPath === "string" ? event.details.fullOutputPath : undefined; + if (rawText.length <= settings.thresholdChars && !fullOutputPath) return undefined; + stashResult(getStash(), event.toolCallId, { + rawText, + ...(fullOutputPath ? { fullOutputPath } : {}), + }); + return undefined; + }); +} diff --git a/pi-overflow/index.ts b/pi-overflow/index.ts new file mode 100644 index 0000000..c8fa739 --- /dev/null +++ b/pi-overflow/index.ts @@ -0,0 +1,167 @@ +import { getAgentDir, type ExtensionAPI } from "@earendil-works/pi-coding-agent"; +import { Type } from "typebox"; +import { stat } from "node:fs/promises"; +import { join } from "node:path"; +import { readOverflowSettings, type OverflowSettings } from "./settings.ts"; +import { getStash } from "./src/stash.ts"; +import { buildView, collectViewFromFile, collectViewFromText } from "./src/view.ts"; +import { cleanStore, entryId, storeEntry } from "./src/store.ts"; +import { runOverflowQuery } from "./src/query.ts"; + +function formatBytes(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1_048_576) return `${(bytes / 1024).toFixed(1)} KB`; + return `${(bytes / 1_048_576).toFixed(1)} MB`; +} + +export interface OverflowExtensionOptions { + settings?: OverflowSettings; + storeRoot?: string; +} + +/** + * Bound stage: loaded LAST among content extensions (before system-prompt). + * By the time this handler runs, every other tool_result handler — RTK + * included — has already transformed the result, so the bounded view is + * built from the final visible text while the stored original comes from + * capture.ts's stash or bash's fullOutputPath file. + */ +export default function overflowExtension(pi: ExtensionAPI, options: OverflowExtensionOptions = {}): void { + const settings = options.settings ?? readOverflowSettings(); + const storeRoot = () => options.storeRoot ?? join(getAgentDir(), "my-pi", "pi-overflow"); + + pi.on("session_start", () => { + // Retention/size cleanup never throws (cleanStore swallows failures) + // and the returned promise lets tests await it. + return cleanStore(storeRoot(), settings.retentionDays, settings.maxStoreBytes); + }); + + pi.on("tool_result", async (event, ctx) => { + const stash = getStash(); + const staged = stash.get(event.toolCallId); + try { + if ( + !settings.enabled || + settings.exemptTools.includes(event.toolName) || + event.parentToolCallId + ) { + return undefined; + } + const current = event.content + .filter((part): part is { type: "text"; text: string } => part.type === "text") + .map((part) => part.text) + .join("\n"); + const fullOutputPath = + typeof event.details?.fullOutputPath === "string" + ? event.details.fullOutputPath + : staged?.fullOutputPath; + + // Resolve the original: bash's spilled file wins (it is complete even + // when the in-context text was already cut), then the capture stash, + // then the current text itself. + let source: { kind: "text"; text: string } | { kind: "file"; path: string }; + let originalSize: number; + if (fullOutputPath) { + try { + const info = await stat(fullOutputPath); + if (!info.isFile()) throw new Error("not a file"); + source = { kind: "file", path: fullOutputPath }; + originalSize = info.size; + } catch { + const fallback = staged?.rawText ?? current; + source = { kind: "text", text: fallback }; + originalSize = fallback.length; + } + } else { + const fallback = staged?.rawText ?? current; + source = { kind: "text", text: fallback }; + originalSize = fallback.length; + } + + if (current.length <= settings.thresholdChars && originalSize <= settings.thresholdChars) { + return undefined; + } + + const sessionId = ctx.sessionManager?.getSessionId?.() ?? "session"; + const id = entryId(sessionId, event.toolCallId); + const stored = await storeEntry( + storeRoot(), + id, + source, + { tool: event.toolName, args: event.input }, + settings.maxEntryBytes, + ); + const overflow = { + id, + totalLines: stored.totalLines, + totalBytes: stored.totalBytes, + truncated: current.length > settings.thresholdChars, + }; + + const images = event.content.filter((part) => part.type !== "text"); + let text: string; + if (current.length > settings.thresholdChars) { + // The view always describes the stored file — its line numbers and + // totals must match what overflow_query reads, which is not true + // of `current` when RTK rewrote or bash pre-truncated the result. + const viewInput = + stored.storedText !== undefined + ? collectViewFromText(stored.storedText, settings) + : await collectViewFromFile(join(storeRoot(), `${id}.txt`), settings); + text = buildView(viewInput, settings, { id, tool: event.toolName }); + } else { + text = `${current}\n[pi-overflow id=${id}: full original output stored (${stored.totalLines} lines, ${formatBytes(stored.totalBytes)}); use overflow_query to inspect it]`; + } + + return { + content: [{ type: "text", text }, ...images], + structuredContent: event.structuredContent, + details: { ...event.details, overflow }, + }; + } finally { + stash.delete(event.toolCallId); + } + }); + + pi.registerTool({ + name: "overflow_query", + label: "Overflow Query", + description: + "Inspect the full original of a tool output that pi-overflow stored because it was too long to show " + + "(search with regex, read line ranges, or extract a JSON path).", + promptSnippet: "Inspect a stored oversized tool output by id: regex search, line ranges, or JSON path.", + promptGuidelines: [ + "When a tool result carries a [pi-overflow id=…] marker, the model is seeing a bounded view; the complete original is stored on disk.", + "Call overflow_query with the marker's id and a regex, line range, or jsonPath instead of re-running the tool or asking for the full dump.", + ], + parameters: Type.Object({ + id: Type.String({ description: "12-hex storage id from a [pi-overflow id=…] marker." }), + search: Type.Optional( + Type.String({ description: "Case-insensitive regular expression matched per line." }), + ), + context: Type.Optional( + Type.Number({ minimum: 0, maximum: 10, description: "Context lines around each search hit (default 2)." }), + ), + lines: Type.Optional( + Type.Object({ + start: Type.Number({ minimum: 1, description: "First line to return (1-based, inclusive)." }), + end: Type.Number({ minimum: 1, description: "Last line to return (inclusive)." }), + }), + ), + jsonPath: Type.Optional( + Type.String({ description: "Dot/bracket path into the stored JSON, e.g. snapshot.nodes[3].name." }), + ), + maxResults: Type.Optional( + Type.Number({ minimum: 1, maximum: 200, description: "Maximum search hits to report (default 50)." }), + ), + }), + async execute(_toolCallId, params) { + const result = await runOverflowQuery(storeRoot(), params, settings); + return { + content: [{ type: "text", text: result.text }], + details: { error: result.error === true, truncated: result.truncated }, + isError: result.error === true, + }; + }, + }); +} diff --git a/pi-overflow/package.json b/pi-overflow/package.json new file mode 100644 index 0000000..fb28612 --- /dev/null +++ b/pi-overflow/package.json @@ -0,0 +1,49 @@ +{ + "name": "pi-overflow", + "version": "0.1.0", + "description": "Bounded views for oversized tool results: store the full original, show head/signals/tail, recover with overflow_query.", + "private": true, + "type": "module", + "license": "MIT", + "main": "./index.ts", + "exports": { + ".": "./index.ts" + }, + "files": [ + "index.ts", + "capture.ts", + "settings.ts", + "src", + "README.md", + "LICENSE" + ], + "pi": { + "extensions": [ + "./capture.ts", + "./index.ts" + ] + }, + "scripts": { + "test": "node --test test/*.test.ts", + "check": "node --experimental-strip-types --check index.ts && node --test test/*.test.ts" + }, + "peerDependencies": { + "@earendil-works/pi-coding-agent": "*", + "typebox": "*" + }, + "peerDependenciesMeta": { + "@earendil-works/pi-coding-agent": { + "optional": true + }, + "typebox": { + "optional": true + } + }, + "devDependencies": { + "@earendil-works/pi-coding-agent": "latest", + "typebox": "latest" + }, + "engines": { + "node": ">=22.19.0" + } +} diff --git a/pi-overflow/settings.ts b/pi-overflow/settings.ts new file mode 100644 index 0000000..2018ba2 --- /dev/null +++ b/pi-overflow/settings.ts @@ -0,0 +1,100 @@ +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; + +export interface OverflowSettings { + enabled: boolean; + thresholdChars: number; + headLines: number; + tailLines: number; + maxSignalLines: number; + signalPattern: string; + signalRegex: RegExp; + maxLineChars: number; + exemptTools: string[]; + retentionDays: number; + maxStoreBytes: number; + maxEntryBytes: number; +} + +export const DEFAULT_SIGNAL_PATTERN = + "\\b(error|errors|fail|failed|failure|warn|warning|exception|traceback|panic|fatal)\\b"; + +export const OVERFLOW_DEFAULTS = { + enabled: true, + thresholdChars: 24_000, + headLines: 40, + tailLines: 120, + maxSignalLines: 40, + signalPattern: DEFAULT_SIGNAL_PATTERN, + maxLineChars: 500, + exemptTools: ["read", "edit", "write", "overflow_query", "ask_user_question"], + retentionDays: 7, + maxStoreBytes: 1_073_741_824, + maxEntryBytes: 67_108_864, +} as const; + +// Bundle-level `config/` is the single source of truth: the extension reads +// `config/pi-overflow.json` from inside the installed bundle (resolved +// relative to this package-root source file), never `settings.json`. +export const OVERFLOW_CONFIG_PATH = fileURLToPath( + new URL("../config/pi-overflow.json", import.meta.url), +); + +function isObject(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function bool(value: unknown, fallback: boolean): boolean { + return typeof value === "boolean" ? value : fallback; +} + +function intInRange(value: unknown, fallback: number, min: number, max: number): number { + return typeof value === "number" && Number.isInteger(value) && value >= min && value <= max + ? value + : fallback; +} + +function compileSignalPattern(value: unknown): { pattern: string; regex: RegExp } { + const pattern = typeof value === "string" && value.length > 0 ? value : DEFAULT_SIGNAL_PATTERN; + try { + return { pattern, regex: new RegExp(pattern, "i") }; + } catch { + return { pattern: DEFAULT_SIGNAL_PATTERN, regex: new RegExp(DEFAULT_SIGNAL_PATTERN, "i") }; + } +} + +function stringList(value: unknown, fallback: readonly string[]): string[] { + if (!Array.isArray(value)) return [...fallback]; + const list = value.filter((item): item is string => typeof item === "string" && item.length > 0); + return [...new Set(list)]; +} + +/** + * Read `config/pi-overflow.json`. Every field is validated independently and + * falls back to its bundle default when missing or invalid; an unreadable or + * non-object file yields all defaults. Tests inject an explicit path. + */ +export function readOverflowSettings(configPath: string = OVERFLOW_CONFIG_PATH): OverflowSettings { + let parsed: unknown = {}; + try { + parsed = JSON.parse(readFileSync(configPath, "utf8")); + } catch { + parsed = {}; + } + const config = isObject(parsed) ? parsed : {}; + const signal = compileSignalPattern(config.signalPattern); + return { + enabled: bool(config.enabled, OVERFLOW_DEFAULTS.enabled), + thresholdChars: intInRange(config.thresholdChars, OVERFLOW_DEFAULTS.thresholdChars, 1, 10_000_000), + headLines: intInRange(config.headLines, OVERFLOW_DEFAULTS.headLines, 0, 10_000), + tailLines: intInRange(config.tailLines, OVERFLOW_DEFAULTS.tailLines, 0, 10_000), + maxSignalLines: intInRange(config.maxSignalLines, OVERFLOW_DEFAULTS.maxSignalLines, 0, 1_000), + signalPattern: signal.pattern, + signalRegex: signal.regex, + maxLineChars: intInRange(config.maxLineChars, OVERFLOW_DEFAULTS.maxLineChars, 10, 100_000), + exemptTools: stringList(config.exemptTools, OVERFLOW_DEFAULTS.exemptTools), + retentionDays: intInRange(config.retentionDays, OVERFLOW_DEFAULTS.retentionDays, 0, 3_650), + maxStoreBytes: intInRange(config.maxStoreBytes, OVERFLOW_DEFAULTS.maxStoreBytes, 1_024, 1_099_511_627_776), + maxEntryBytes: intInRange(config.maxEntryBytes, OVERFLOW_DEFAULTS.maxEntryBytes, 1_024, 1_099_511_627_776), + }; +} diff --git a/pi-overflow/src/query.ts b/pi-overflow/src/query.ts new file mode 100644 index 0000000..caae239 --- /dev/null +++ b/pi-overflow/src/query.ts @@ -0,0 +1,188 @@ +import { ID_PATTERN, readStoredEntry, type EntryMeta } from "./store.ts"; +import type { OverflowSettings } from "../settings.ts"; + +export interface QueryParams { + id: string; + search?: string; + context?: number; + lines?: { start: number; end: number }; + jsonPath?: string; + maxResults?: number; +} + +export interface QueryResult { + text: string; + truncated: boolean; + error?: boolean; +} + +function fail(text: string): QueryResult { + return { text, truncated: false, error: true }; +} + +function boundOutput(text: string, limit: number): QueryResult { + if (text.length <= limit) return { text, truncated: false }; + const marker = `\n[output truncated at ${limit} chars; narrow the query (smaller line range, tighter regex, or jsonPath)]`; + const keep = Math.max(0, limit - marker.length); + return { text: text.slice(0, keep) + marker, truncated: true }; +} + +function metaLine(id: string, meta: EntryMeta | null): string { + if (!meta) return `[pi-overflow id=${id}]`; + return `[pi-overflow id=${id} tool=${meta.tool} ${meta.totalLines} lines / ${meta.totalBytes} bytes]`; +} + +/** Regexes only inspect this prefix of each stored line — a model-supplied + * pattern must not backtrack catastrophically across a megabyte-long line. */ +const MATCH_SCAN_CHARS = 10_000; + +function clipLine(line: string, maxLineChars: number): string { + return line.length > maxLineChars ? `${line.slice(0, maxLineChars)}…` : line; +} + +function evalJsonPath(root: unknown, path: string): { ok: true; value: unknown } | { ok: false; error: string } { + const tokens: Array = []; + const pattern = /(?:^|\.)([A-Za-z_$][A-Za-z0-9_$]*)|\[\s*(\d+)\s*\]|\[\s*["']([^"']+)["']\s*\]/g; + let index = 0; + let match: RegExpExecArray | null; + while ((match = pattern.exec(path)) !== null) { + if (match.index !== index) return { ok: false, error: `cannot parse jsonPath near "${path.slice(index)}"` }; + index = pattern.lastIndex; + if (match[1] !== undefined) tokens.push(match[1]); + else if (match[2] !== undefined) tokens.push(Number(match[2])); + else tokens.push(match[3] as string); + } + if (index !== path.length || tokens.length === 0) { + return { ok: false, error: `cannot parse jsonPath "${path}"` }; + } + let value: unknown = root; + for (const token of tokens) { + if (value === null || value === undefined) { + return { ok: false, error: `jsonPath stops at null/undefined before "${String(token)}"` }; + } + if (typeof token === "number") { + if (!Array.isArray(value) || token >= value.length) { + return { ok: false, error: `index [${token}] out of range` }; + } + value = value[token]; + } else { + if (typeof value !== "object" || Array.isArray(value)) { + return { ok: false, error: `"${token}" is not a property of this value` }; + } + const record = value as Record; + if (!(token in record)) return { ok: false, error: `property "${token}" not found` }; + value = record[token]; + } + } + return { ok: true, value }; +} + +function mergeRanges(hits: number[], context: number): Array<{ start: number; end: number }> { + const ranges: Array<{ start: number; end: number }> = []; + for (const hit of hits) { + const start = Math.max(1, hit - context); + const end = hit + context; + const last = ranges[ranges.length - 1]; + if (last && start <= last.end + 1) { + last.end = Math.max(last.end, end); + } else { + ranges.push({ start, end }); + } + } + return ranges; +} + +/** + * Execute an overflow_query against the store. Never throws on bad input — + * invalid ids, regexes, JSON, and paths all come back as clear error text. + */ +export async function runOverflowQuery( + root: string, + params: QueryParams, + settings: OverflowSettings, +): Promise { + if (typeof params.id !== "string" || !ID_PATTERN.test(params.id)) { + return fail(`invalid overflow id "${String(params.id)}" — expected 12 lowercase hex chars from a [pi-overflow id=…] marker`); + } + let stored: { text: string; meta: EntryMeta | null }; + try { + stored = await readStoredEntry(root, params.id); + } catch { + return fail(`no stored output for id ${params.id} (it may have been cleaned up or never stored)`); + } + const lines = stored.text.split("\n"); + const head = metaLine(params.id, stored.meta); + const context = Math.min(Math.max(Math.floor(params.context ?? 2), 0), 10); + const maxResults = Math.min(Math.max(Math.floor(params.maxResults ?? 50), 1), 200); + + if (typeof params.search === "string") { + let regex: RegExp; + try { + regex = new RegExp(params.search, "i"); + } catch (error) { + return fail(`invalid search regex: ${error instanceof Error ? error.message : String(error)}`); + } + const hits: number[] = []; + for (let index = 0; index < lines.length && hits.length < maxResults; index++) { + regex.lastIndex = 0; + if (regex.test(lines[index].slice(0, MATCH_SCAN_CHARS))) hits.push(index + 1); + } + if (hits.length === 0) { + return boundOutput(`${head}\nno lines match /${params.search}/i`, settings.thresholdChars); + } + const parts = [head, `${hits.length}${hits.length >= maxResults ? "+" : ""} matching lines:`]; + for (const range of mergeRanges(hits, context)) { + const end = Math.min(range.end, lines.length); + parts.push(`--- lines ${range.start}-${end} ---`); + for (let line = range.start; line <= end; line++) { + parts.push(`L${line}: ${clipLine(lines[line - 1], settings.maxLineChars)}`); + } + } + return boundOutput(parts.join("\n"), settings.thresholdChars); + } + + if (params.lines && typeof params.lines === "object") { + const start = Math.floor(params.lines.start); + const end = Math.floor(params.lines.end); + if (!Number.isFinite(start) || !Number.isFinite(end) || start < 1 || end < start) { + return fail(`invalid line range {start: ${params.lines.start}, end: ${params.lines.end}} — need 1 <= start <= end`); + } + const clippedEnd = Math.min(end, lines.length); + if (start > lines.length) { + return fail(`start ${start} is beyond the stored ${lines.length} lines`); + } + const parts = [head, `--- lines ${start}-${clippedEnd} ---`]; + for (let line = start; line <= clippedEnd; line++) { + parts.push(`L${line}: ${clipLine(lines[line - 1], settings.maxLineChars)}`); + } + return boundOutput(parts.join("\n"), settings.thresholdChars); + } + + if (typeof params.jsonPath === "string") { + let parsed: unknown; + try { + parsed = JSON.parse(stored.text); + } catch (error) { + return fail(`stored output is not valid JSON: ${error instanceof Error ? error.message : String(error)}`); + } + const result = evalJsonPath(parsed, params.jsonPath); + if (!result.ok) return fail(`jsonPath error: ${result.error}`); + let rendered: string; + try { + rendered = JSON.stringify(result.value, null, 2) ?? "undefined"; + } catch { + rendered = String(result.value); + } + return boundOutput(`${head}\njsonPath ${params.jsonPath}:\n${rendered}`, settings.thresholdChars); + } + + // No selector: metadata plus a bounded head/tail preview. + const headCount = Math.min(settings.headLines, lines.length); + const tailCount = Math.min(settings.tailLines, Math.max(0, lines.length - headCount)); + const parts = [head, `--- lines 1-${headCount} ---`, ...lines.slice(0, headCount)]; + if (tailCount > 0) { + parts.push(`… ${lines.length - headCount - tailCount} lines omitted …`); + parts.push(`--- lines ${lines.length - tailCount + 1}-${lines.length} ---`, ...lines.slice(lines.length - tailCount)); + } + return boundOutput(parts.join("\n"), settings.thresholdChars); +} diff --git a/pi-overflow/src/stash.ts b/pi-overflow/src/stash.ts new file mode 100644 index 0000000..967fc5e --- /dev/null +++ b/pi-overflow/src/stash.ts @@ -0,0 +1,35 @@ +/** + * Cross-extension staging for pre-bound originals. `capture.ts` (loaded + * first) stashes the raw tool_result text; `index.ts` (loaded last) consumes + * it after every other handler — RTK in particular — has run. Sharing goes + * through `globalThis[Symbol.for(...)]`, the same mechanism pi-compaction + * uses for its metrics service. + */ + +export interface StashedResult { + rawText: string; + fullOutputPath?: string; +} + +const STASH_KEY = Symbol.for("my-pi.pi-overflow.stash.v1"); +const STASH_LIMIT = 64; + +export function getStash(): Map { + const scope = globalThis as Record; + let stash = scope[STASH_KEY] as Map | undefined; + if (!(stash instanceof Map)) { + stash = new Map(); + scope[STASH_KEY] = stash; + } + return stash; +} + +export function stashResult(stash: Map, toolCallId: string, entry: StashedResult): void { + stash.delete(toolCallId); + stash.set(toolCallId, entry); + while (stash.size > STASH_LIMIT) { + const oldest = stash.keys().next().value; + if (oldest === undefined) break; + stash.delete(oldest); + } +} diff --git a/pi-overflow/src/store.ts b/pi-overflow/src/store.ts new file mode 100644 index 0000000..c9b0be4 --- /dev/null +++ b/pi-overflow/src/store.ts @@ -0,0 +1,267 @@ +import { createHash } from "node:crypto"; +import { createReadStream, createWriteStream, type WriteStream } from "node:fs"; +import { chmod, mkdir, open, readFile, readdir, rename, stat, unlink, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { pipeline } from "node:stream/promises"; + +export interface EntryMeta { + tool: string; + argsSummary: string; + totalLines: number; + totalBytes: number; + createdAt: string; + omittedMiddle?: boolean; +} + +export interface StoredEntry { + id: string; + totalLines: number; + totalBytes: number; + /** Present when the source was text: the exact content written to disk. */ + storedText?: string; +} + +export const ID_PATTERN = /^[0-9a-f]{12}$/; + +const TMP_MAX_AGE_MS = 3_600_000; + +export function entryId(sessionId: string, toolCallId: string): string { + return createHash("sha1").update(`${sessionId}\0${toolCallId}`).digest("hex").slice(0, 12); +} + +async function ensureDir(dir: string): Promise { + await mkdir(dir, { recursive: true, mode: 0o700 }); + try { + await chmod(dir, 0o700); + } catch { + // Best-effort permission tightening only. + } +} + +function tmpPath(path: string): string { + return `${path}.tmp-${process.pid}-${Math.random().toString(36).slice(2)}`; +} + +async function writeFileAtomic(path: string, content: string | Buffer): Promise { + const tmp = tmpPath(path); + await writeFile(tmp, content, { mode: 0o600 }); + await rename(tmp, path); +} + +function countNewlines(text: string): number { + let count = 0; + for (let index = 0; index < text.length; index++) { + if (text.charCodeAt(index) === 10) count++; + } + return count; +} + +export function lineCount(text: string): number { + if (text.length === 0) return 0; + return countNewlines(text) + (text.endsWith("\n") ? 0 : 1); +} + +function truncateSummary(value: string, maxChars = 300): string { + return value.length > maxChars ? `${value.slice(0, maxChars)}…` : value; +} + +export function argsSummary(input: unknown): string { + try { + return truncateSummary(JSON.stringify(input) ?? "undefined"); + } catch { + return "[unserializable]"; + } +} + +/** + * Copy `sourcePath` into `/.txt` without holding it all in memory. + * Files larger than `maxEntryBytes` keep the first and last halves with an + * omission marker between them. + */ +async function copyFileBounded( + sourcePath: string, + destPath: string, + maxEntryBytes: number, +): Promise<{ bytes: number; omittedMiddle: boolean }> { + const info = await stat(sourcePath); + if (!info.isFile()) throw new Error("full output path is not a file"); + const size = info.size; + if (size <= maxEntryBytes) { + const tmp = tmpPath(destPath); + const out: WriteStream = createWriteStream(tmp, { mode: 0o600 }); + await pipeline(createReadStream(sourcePath), out); + await rename(tmp, destPath); + return { bytes: size, omittedMiddle: false }; + } + const half = Math.floor(maxEntryBytes / 2); + const omitted = size - half * 2; + const source = await open(sourcePath, "r"); + try { + const head = Buffer.alloc(half); + const tail = Buffer.alloc(half); + await source.read(head, 0, half, 0); + await source.read(tail, 0, half, size - half); + const marker = Buffer.from(`\n… ${omitted} bytes omitted (exceeded ${maxEntryBytes}-byte store limit) …\n`); + await writeFileAtomic(destPath, Buffer.concat([head, marker, tail])); + } finally { + await source.close(); + } + return { bytes: size, omittedMiddle: true }; +} + +async function countFileLines(path: string): Promise { + let count = 0; + let lastByte = -1; + let seen = false; + for await (const chunk of createReadStream(path)) { + seen = true; + const buffer = chunk as Buffer; + for (let index = 0; index < buffer.length; index++) { + if (buffer[index] === 10) count++; + } + if (buffer.length > 0) lastByte = buffer[buffer.length - 1]; + } + if (!seen) return 0; + return count + (lastByte === 10 ? 0 : 1); +} + +/** + * Persist an original tool output under `/.txt` plus `.json` + * metadata. Directories are 0700, files 0600, writes are atomic. The caller + * chooses `root` (defaults come from getAgentDir() in index.ts; tests inject + * a temp directory). + */ +export async function storeEntry( + root: string, + id: string, + source: { kind: "text"; text: string } | { kind: "file"; path: string }, + meta: { tool: string; args: unknown }, + maxEntryBytes: number, +): Promise { + await ensureDir(root); + const textPath = join(root, `${id}.txt`); + const metaPath = join(root, `${id}.json`); + let totalLines = 0; + let totalBytes = 0; + let omittedMiddle = false; + let storedText: string | undefined; + if (source.kind === "text") { + let text = source.text; + const bytes = Buffer.byteLength(text, "utf8"); + totalBytes = bytes; + if (bytes > maxEntryBytes) { + const halfChars = Math.floor(maxEntryBytes / 2); + const omitted = bytes - halfChars * 2; + text = `${text.slice(0, halfChars)}\n… ${omitted} bytes omitted (exceeded ${maxEntryBytes}-byte store limit) …\n${text.slice(text.length - halfChars)}`; + omittedMiddle = true; + } + totalLines = lineCount(text); + storedText = text; + await writeFileAtomic(textPath, text); + } else { + const copied = await copyFileBounded(source.path, textPath, maxEntryBytes); + totalBytes = copied.bytes; + omittedMiddle = copied.omittedMiddle; + totalLines = await countFileLines(textPath).catch(() => 0); + } + const entry: EntryMeta = { + tool: meta.tool, + argsSummary: argsSummary(meta.args), + totalLines, + totalBytes, + createdAt: new Date().toISOString(), + ...(omittedMiddle ? { omittedMiddle: true } : {}), + }; + await writeFileAtomic(metaPath, `${JSON.stringify(entry, null, 2)}\n`); + return { id, totalLines, totalBytes, ...(storedText !== undefined ? { storedText } : {}) }; +} + +export async function readStoredEntry(root: string, id: string): Promise<{ text: string; meta: EntryMeta | null }> { + if (!ID_PATTERN.test(id)) throw new Error(`invalid overflow id: ${id}`); + const textPath = join(root, `${id}.txt`); + const metaPath = join(root, `${id}.json`); + const text = await readFile(textPath, "utf8"); + let meta: EntryMeta | null = null; + try { + const parsed = JSON.parse(await readFile(metaPath, "utf8")); + if (parsed && typeof parsed === "object") meta = parsed as EntryMeta; + } catch { + meta = null; + } + return { text, meta }; +} + +/** + * Delete tmp files left by interrupted writes (older than one hour), entries + * older than `retentionDays`, then evict oldest-first until the store is + * under `maxStoreBytes`. All failures are swallowed — cleanup must never + * break a session. + */ +export async function cleanStore(root: string, retentionDays: number, maxStoreBytes: number): Promise { + try { + const now = Date.now(); + const cutoff = now - retentionDays * 86_400_000; + const names = await readdir(root); + // Stale tmp files from interrupted writes never become entries; remove + // them once they are clearly abandoned. + for (const name of names) { + if (!name.includes(".tmp-")) continue; + try { + const info = await stat(join(root, name)); + if (info.isFile() && info.mtimeMs < now - TMP_MAX_AGE_MS) { + await unlink(join(root, name)); + } + } catch { + // Already gone or unreadable — nothing to do. + } + } + const ids = new Set( + names + .filter((name) => !name.includes(".tmp-")) + .map((name) => name.replace(/\.(txt|json)$/, "")), + ); + const entries: Array<{ id: string; bytes: number; mtime: number }> = []; + for (const id of ids) { + let bytes = 0; + let mtime = 0; + let seen = false; + for (const suffix of ["txt", "json"] as const) { + const file = join(root, `${id}.${suffix}`); + try { + const info = await stat(file); + bytes += info.size; + mtime = Math.max(mtime, info.mtimeMs); + seen = true; + } catch { + // Missing half of a pair still counts toward cleanup. + } + } + if (seen) entries.push({ id, bytes, mtime }); + } + const remove = async (id: string) => { + for (const suffix of ["txt", "json"] as const) { + await unlink(join(root, `${id}.${suffix}`)).catch(() => {}); + } + }; + let total = 0; + const survivors: typeof entries = []; + for (const entry of entries) { + if (entry.mtime < cutoff) { + await remove(entry.id); + } else { + total += entry.bytes; + survivors.push(entry); + } + } + if (total > maxStoreBytes) { + survivors.sort((a, b) => a.mtime - b.mtime); + for (const entry of survivors) { + if (total <= maxStoreBytes) break; + await remove(entry.id); + total -= entry.bytes; + } + } + } catch { + // Cleanup failures never propagate. + } +} diff --git a/pi-overflow/src/view.ts b/pi-overflow/src/view.ts new file mode 100644 index 0000000..b5c1dee --- /dev/null +++ b/pi-overflow/src/view.ts @@ -0,0 +1,246 @@ +import { createReadStream } from "node:fs"; +import { open, stat } from "node:fs/promises"; +import type { OverflowSettings } from "../settings.ts"; + +export interface ViewMeta { + id: string; + tool: string; +} + +export interface SignalHit { + line: number; + text: string; +} + +/** + * Pre-collected view segments of the STORED original (never the current — + * possibly RTK- or bash-truncated — text). Line numbers here are the stored + * file's line numbers, which is exactly what overflow_query reads. + */ +export interface ViewInput { + head: string[]; + signals: SignalHit[]; + tail: string[]; + totalLines: number; + totalBytes: number; + /** First ~thresholdChars chars of the stored content, for char fallback. */ + rawHead: string; + /** Last ~thresholdChars chars of the stored content, for char fallback. */ + rawTail: string; +} + +/** Regexes only inspect this prefix of each line — catastrophic backtracking + * on a giant single-line blob is not allowed to stall the handler. */ +export const MATCH_SCAN_CHARS = 10_000; + +function clip(line: string, maxLineChars: number): string { + return line.length > maxLineChars ? `${line.slice(0, maxLineChars)}…` : line; +} + +/** Slice a text original into the same view segments a file scan produces. */ +export function collectViewFromText(text: string, settings: OverflowSettings): ViewInput { + const lines = text.split("\n"); + const total = lines.length; + const headCount = Math.min(settings.headLines, total); + const tailCount = Math.min(settings.tailLines, Math.max(0, total - headCount)); + const head = lines.slice(0, headCount).map((line) => clip(line, settings.maxLineChars)); + const tail = tailCount > 0 ? lines.slice(total - tailCount).map((line) => clip(line, settings.maxLineChars)) : []; + const signals: SignalHit[] = []; + const regex = new RegExp(settings.signalRegex.source, "i"); + for ( + let index = headCount; + index < total - tailCount && signals.length < settings.maxSignalLines; + index++ + ) { + regex.lastIndex = 0; + if (regex.test(lines[index].slice(0, MATCH_SCAN_CHARS))) { + signals.push({ line: index + 1, text: clip(lines[index], settings.maxLineChars) }); + } + } + return { + head, + signals, + tail, + totalLines: total, + totalBytes: Buffer.byteLength(text, "utf8"), + rawHead: text.slice(0, settings.thresholdChars), + rawTail: text.slice(Math.max(0, text.length - settings.thresholdChars)), + }; +} + +/** + * Stream the stored file line by line — never materialize it. Collects the + * first headLines lines, keeps the last tailLines lines in a ring buffer, + * and records at most maxSignalLines signal hits (line number + clipped + * text); hits landing inside the final tail range are dropped afterwards. + */ +export async function collectViewFromFile(path: string, settings: OverflowSettings): Promise { + const head: string[] = []; + const tail: string[] = []; + const signals: SignalHit[] = []; + let totalLines = 0; + let remainder = ""; + const regex = new RegExp(settings.signalRegex.source, "i"); + + const handleLine = (line: string) => { + totalLines++; + if (head.length < settings.headLines) { + head.push(clip(line, settings.maxLineChars)); + } else if (signals.length < settings.maxSignalLines) { + regex.lastIndex = 0; + if (regex.test(line.slice(0, MATCH_SCAN_CHARS))) { + signals.push({ line: totalLines, text: clip(line, settings.maxLineChars) }); + } + } + tail.push(clip(line, settings.maxLineChars)); + if (tail.length > settings.tailLines) tail.shift(); + }; + + for await (const chunk of createReadStream(path, { encoding: "utf8" })) { + remainder += chunk; + let index = remainder.indexOf("\n"); + while (index >= 0) { + handleLine(remainder.slice(0, index)); + remainder = remainder.slice(index + 1); + index = remainder.indexOf("\n"); + } + } + if (remainder.length > 0) handleLine(remainder); + + const tailStart = totalLines - tail.length + 1; + const middleSignals = signals.filter((hit) => hit.line < tailStart); + + const info = await stat(path); + const take = Math.min(settings.thresholdChars, info.size); + let rawHead = ""; + let rawTail = ""; + if (take > 0) { + const handle = await open(path, "r"); + try { + const headBuffer = Buffer.alloc(take); + const tailBuffer = Buffer.alloc(take); + await handle.read(headBuffer, 0, take, 0); + await handle.read(tailBuffer, 0, take, info.size - take); + rawHead = headBuffer.toString("utf8"); + rawTail = tailBuffer.toString("utf8"); + } finally { + await handle.close(); + } + } + + return { + head, + signals: middleSignals, + tail, + totalLines, + totalBytes: info.size, + rawHead, + rawTail, + }; +} + +function formatBytes(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1_048_576) return `${(bytes / 1024).toFixed(1)} KB`; + if (bytes < 1_073_741_824) return `${(bytes / 1_048_576).toFixed(1)} MB`; + return `${(bytes / 1_073_741_824).toFixed(2)} GB`; +} + +function signalDescription(pattern: string): string { + const compact = pattern.replace(/\\b/g, "").replace(/[()]/g, ""); + return compact.length > 60 ? `${compact.slice(0, 60)}…` : compact; +} + +/** + * Deterministic bounded view of the stored original: head lines, signal + * lines from the middle section, and tail lines — all with the stored + * file's line numbers. Output never exceeds settings.thresholdChars and the + * tail of the stored content is always visible. + * + * Shrink order under budget pressure is tail > signals > head: head first + * collapses to min(10, headLines), signals halve stepwise to 10 then 0, then + * head/tail shrink ~35/65 with tail preferred, and char truncation of the + * stored head/tail is the final fallback. + */ +export function buildView(input: ViewInput, settings: OverflowSettings, meta: ViewMeta): string { + const budget = settings.thresholdChars; + const header = + `[pi-overflow id=${meta.id} tool=${meta.tool} ${input.totalLines} lines / ${formatBytes(input.totalBytes)}` + + ` — full output stored; use overflow_query to search or read ranges]`; + + const render = (headLines: string[], signals: SignalHit[], tailLines: string[]): string => { + const parts: string[] = [header]; + if (headLines.length > 0) { + parts.push(`--- lines 1-${headLines.length} ---`, ...headLines); + } + if (signals.length > 0) { + parts.push( + `--- signals (lines matching ${signalDescription(settings.signalPattern)}, up to ${settings.maxSignalLines}) ---`, + ...signals.map((hit) => `L${hit.line}: ${hit.text}`), + ); + } + if (tailLines.length > 0) { + const first = input.totalLines - tailLines.length + 1; + parts.push(`--- lines ${first}-${input.totalLines} ---`, ...tailLines); + } + return parts.join("\n"); + }; + + const charFallback = (): string => { + const markerReserve = 64; + const bodyBudget = Math.max(64, budget - header.length - markerReserve); + const headChars = Math.floor(bodyBudget * 0.35); + const tailChars = bodyBudget - headChars; + const headText = input.rawHead.slice(0, headChars); + const tailText = input.rawTail.slice(Math.max(0, input.rawTail.length - tailChars)); + const omitted = Math.max(0, input.totalBytes - headText.length - tailText.length); + return `${header}\n${headText}\n… ${omitted} chars omitted …\n${tailText}`; + }; + + let headList = input.head; + let signalList = input.signals; + let tailList = input.tail; + + let result = render(headList, signalList, tailList); + if (result.length <= budget) return result; + + // 1. Collapse head to its minimum useful window. + headList = headList.slice(0, Math.min(10, settings.headLines)); + result = render(headList, signalList, tailList); + if (result.length <= budget) return result; + + // 2. Halve signals stepwise until 10 remain, then drop them entirely. + while (signalList.length > 10) { + signalList = signalList.slice(0, Math.ceil(signalList.length / 2)); + result = render(headList, signalList, tailList); + if (result.length <= budget) return result; + } + if (signalList.length > 0) { + signalList = []; + result = render(headList, signalList, tailList); + if (result.length <= budget) return result; + } + + // When head+tail already span every stored line there is no line-level + // slack left: go straight to char truncation so the ending survives. + if (input.head.length + input.tail.length >= input.totalLines) return charFallback(); + + // 3. Shrink head/tail ~35/65; tail keeps the larger share and always keeps + // its last line (the stored ending). + while (headList.length + tailList.length > 0) { + result = render(headList, signalList, tailList); + if (result.length <= budget) return result; + const headChars = headList.reduce((sum, line) => sum + line.length + 1, 0); + const tailChars = tailList.reduce((sum, line) => sum + line.length + 1, 0); + if (tailList.length > 1 && (tailChars >= headChars * (65 / 35) || headList.length === 0)) { + tailList.shift(); + } else if (headList.length > 0) { + headList.pop(); + } else { + tailList.shift(); + } + } + + // 4. Last resort: keep the stored text's own head 35% / tail 65%. + return charFallback(); +} diff --git a/pi-overflow/test/handler.test.ts b/pi-overflow/test/handler.test.ts new file mode 100644 index 0000000..b613a9b --- /dev/null +++ b/pi-overflow/test/handler.test.ts @@ -0,0 +1,314 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtemp, readFile, readdir, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import overflowCapture from "../capture.ts"; +import overflowExtension from "../index.ts"; +import { getStash } from "../src/stash.ts"; +import { readOverflowSettings, OVERFLOW_DEFAULTS, type OverflowSettings } from "../settings.ts"; + +function harness() { + const handlers = new Map(); + const tools = new Map(); + const pi: any = { + on(name: string, handler: Function) { + const list = handlers.get(name) ?? []; + list.push(handler); + handlers.set(name, list); + }, + registerTool(tool: any) { tools.set(tool.name, tool); }, + }; + return { pi, handlers, tools }; +} + +function ctx() { + return { sessionManager: { getSessionId: () => "session-test" } } as any; +} + +function settings(overrides: Partial = {}): OverflowSettings { + return { ...readOverflowSettings("/nonexistent/pi-overflow.json"), ...overrides }; +} + +function textEvent(toolCallId: string, text: string, extra: Record = {}) { + return { + toolCallId, + toolName: "bash", + input: { command: "run" }, + content: [{ type: "text", text }], + details: {}, + ...extra, + }; +} + +async function runHandlers(handlers: Map, event: any, context: any) { + let current = event; + for (const handler of handlers.get("tool_result") ?? []) { + const result = await handler(current, context); + if (result) current = { ...current, ...result }; + } + return current; +} + +async function makeStore() { + return mkdtemp(join(tmpdir(), "pi-overflow-test-")); +} + +test("capture stashes the original before another handler shortens it; bound stage stores original but views current", async () => { + const root = await makeStore(); + getStash().clear(); + const capture = harness(); + overflowCapture(capture.pi); + const bound = harness(); + overflowExtension(bound.pi, { storeRoot: root, settings: settings() }); + + const big = Array.from({ length: 3000 }, (_, i) => `orig ${i}`).join("\n"); + const event = textEvent("call-1", big); + const afterCapture = await runHandlers(capture.handlers, event, ctx()); + assert.equal(afterCapture, event, "capture must not modify the result"); + assert.ok(getStash().has("call-1"), "capture must stash the original"); + + // Simulate RTK: replace content with a shortened version that is still + // over threshold. + const rtkShortened = Array.from({ length: 900 }, (_, i) => `rtk ${i}`).join("\n") + "\n" + "z".repeat(30_000); + const mid = { ...afterCapture, content: [{ type: "text", text: rtkShortened }] }; + const final = await runHandlers(bound.handlers, mid, ctx()); + + assert.ok(!getStash().has("call-1"), "stash entry must be consumed"); + const text = final.content[0].text; + assert.match(text, /\[pi-overflow id=[0-9a-f]{12} tool=bash/); + assert.ok(text.length <= settings().thresholdChars); + + const id = final.details.overflow.id; + const stored = await readFile(join(root, `${id}.txt`), "utf8"); + assert.ok(stored.includes("orig 0"), "store must keep the ORIGINAL, not the RTK-shortened text"); + assert.ok(stored.includes("orig 2999")); + assert.equal(final.details.overflow.truncated, true); +}); + +test("details.fullOutputPath file is used as the original", async () => { + const root = await makeStore(); + getStash().clear(); + const bound = harness(); + overflowExtension(bound.pi, { storeRoot: root, settings: settings() }); + + const spill = join(root, "spill.txt"); + await writeFile(spill, Array.from({ length: 5000 }, (_, i) => `spill ${i}`).join("\n")); + const event = textEvent("call-2", "x".repeat(30_000), { details: { fullOutputPath: spill } }); + const final = await runHandlers(bound.handlers, event, ctx()); + const stored = await readFile(join(root, `${final.details.overflow.id}.txt`), "utf8"); + assert.ok(stored.includes("spill 0")); + assert.ok(stored.includes("spill 4999")); +}); + +test("view line numbers track the STORED file, not the current text", async () => { + const root = await makeStore(); + getStash().clear(); + const bound = harness(); + overflowExtension(bound.pi, { storeRoot: root, settings: settings() }); + + // bash spill: 5000 lines on disk, but the in-context result only carries + // the last ~100 lines (still over threshold via padding). + const spill = join(root, "spill-5000.txt"); + await writeFile( + spill, + Array.from({ length: 5000 }, (_, i) => `spill line ${i + 1}`).join("\n"), + ); + const current = Array.from({ length: 100 }, (_, i) => `spill line ${4901 + i}`).join("\n") + "\n" + "pad".repeat(10_000); + const event = textEvent("call-lines", current, { details: { fullOutputPath: spill } }); + const final = await runHandlers(bound.handlers, event, ctx()); + + const text = final.content[0].text; + const id = final.details.overflow.id; + assert.match(text, /--- lines 4881-5000 ---/, "tail range must come from the stored 5000-line file"); + assert.match(text, /spill line 5000/); + assert.match(text, /\[pi-overflow id=[0-9a-f]{12} tool=bash 5000 lines/); + + // overflow_query with the view's tail range returns the same lines. + const tool = bound.tools.get("overflow_query"); + const result = await tool.execute("q1", { id, lines: { start: 4881, end: 4883 } }); + assert.match(result.content[0].text, /L4881: spill line 4881/); + assert.match(result.content[0].text, /L4883: spill line 4883/); +}); + +test("RTK scenario: view line numbers come from the stored original text", async () => { + const root = await makeStore(); + getStash().clear(); + const capture = harness(); + overflowCapture(capture.pi); + const bound = harness(); + overflowExtension(bound.pi, { storeRoot: root, settings: settings() }); + + const original = Array.from({ length: 3000 }, (_, i) => `orig ${i + 1}`).join("\n"); + await runHandlers(capture.handlers, textEvent("call-rtk", original), ctx()); + // RTK rewrites the visible text but it stays over threshold. + const mid = { + ...textEvent("call-rtk", ""), + content: [{ type: "text", text: "z".repeat(30_000) }], + }; + const final = await runHandlers(bound.handlers, mid, ctx()); + const text = final.content[0].text; + assert.match(text, /--- lines 2881-3000 ---/, "view tail must be the stored original's tail"); + assert.ok(text.includes("orig 3000")); +}); + +test("file sources over maxEntryBytes keep real head and tail halves", async () => { + const root = await makeStore(); + const { storeEntry } = await import("../src/store.ts"); + const src = join(root, "big.bin"); + const content = "HEAD-".repeat(200) + "MID-".repeat(2000) + "TAIL-".repeat(200); + await writeFile(src, content); + const maxEntryBytes = 2000; + await storeEntry(root, "0123456789ab", { kind: "file", path: src }, { tool: "bash", args: {} }, maxEntryBytes); + const stored = await readFile(join(root, "0123456789ab.txt"), "utf8"); + assert.ok(stored.startsWith("HEAD-")); + assert.ok(stored.endsWith("TAIL-")); + assert.match(stored, /bytes omitted/); + // Head and tail halves must equal the source's real bytes. + const half = Math.floor(maxEntryBytes / 2); + assert.equal(stored.slice(0, half), content.slice(0, half)); + assert.equal(stored.slice(-half), content.slice(-half)); + assert.ok(!stored.includes("\0".repeat(100)), "must not store zeroed content"); +}); + +test("interrupted-write tmp files older than an hour are cleaned", async () => { + const root = await makeStore(); + const bound = harness(); + overflowExtension(bound.pi, { storeRoot: root, settings: settings() }); + + const staleTmp = join(root, "deadbeef.txt.tmp-999-zzz"); + const freshTmp = join(root, "livebeef.txt.tmp-111-aaa"); + await writeFile(staleTmp, "partial"); + await writeFile(freshTmp, "partial"); + const old = new Date(Date.now() - 2 * 3_600_000); + const { utimes } = await import("node:fs/promises"); + await utimes(staleTmp, old, old); + + for (const handler of bound.handlers.get("session_start") ?? []) await handler({}, ctx()); + const { existsSync } = await import("node:fs"); + assert.ok(!existsSync(staleTmp), "stale tmp must be removed"); + assert.ok(existsSync(freshTmp), "fresh tmp must survive"); +}); + +test("current under threshold but original over → one-line marker appended", async () => { + const root = await makeStore(); + getStash().clear(); + const capture = harness(); + overflowCapture(capture.pi); + const bound = harness(); + overflowExtension(bound.pi, { storeRoot: root, settings: settings() }); + + const big = "a".repeat(30_000); + const event = textEvent("call-3", big); + await runHandlers(capture.handlers, event, ctx()); + const mid = { ...event, content: [{ type: "text", text: "short summary" }] }; + const final = await runHandlers(bound.handlers, mid, ctx()); + + const text = final.content[0].text; + assert.ok(text.startsWith("short summary")); + assert.match(text, /\[pi-overflow id=[0-9a-f]{12}: full original output stored \(\d+ lines, .*; use overflow_query/); + assert.equal(final.details.overflow.truncated, false); + assert.ok(final.details.overflow.id); +}); + +test("exempt tools and nested calls are ignored", async () => { + const root = await makeStore(); + getStash().clear(); + const bound = harness(); + overflowExtension(bound.pi, { storeRoot: root, settings: settings() }); + + const big = "x".repeat(30_000); + const exempt = await runHandlers(bound.handlers, textEvent("c-ex", big, { toolName: "read" }), ctx()); + assert.equal(exempt.content[0].text, big, "read results must pass through untouched"); + + const nested = await runHandlers(bound.handlers, textEvent("c-nest", big, { parentToolCallId: "parent" }), ctx()); + assert.equal(nested.content[0].text, big, "nested calls must pass through untouched"); +}); + +test("image parts and structuredContent survive", async () => { + const root = await makeStore(); + getStash().clear(); + const bound = harness(); + overflowExtension(bound.pi, { storeRoot: root, settings: settings() }); + + const event = textEvent("call-4", "x".repeat(30_000)); + event.content.push({ type: "image", data: "b64" } as any); + (event as any).structuredContent = { rows: 3 }; + const final = await runHandlers(bound.handlers, event, ctx()); + assert.deepEqual(final.structuredContent, { rows: 3 }); + assert.equal(final.content.at(-1).type, "image"); + assert.equal(final.content.at(-1).data, "b64"); +}); + +test("existing details fields survive the merge", async () => { + const root = await makeStore(); + getStash().clear(); + const bound = harness(); + overflowExtension(bound.pi, { storeRoot: root, settings: settings() }); + + const event = textEvent("call-5", "x".repeat(30_000), { details: { rtk: { saved: 100 }, fullOutputPath: undefined } }); + const final = await runHandlers(bound.handlers, event, ctx()); + assert.deepEqual(final.details.rtk, { saved: 100 }); + assert.ok(final.details.overflow.id); +}); + +test("files are 0600 and directories 0700", async () => { + const root = await makeStore(); + getStash().clear(); + const bound = harness(); + overflowExtension(bound.pi, { storeRoot: root, settings: settings() }); + + const final = await runHandlers(bound.handlers, textEvent("call-6", "x".repeat(30_000)), ctx()); + const id = final.details.overflow.id; + const dirMode = (await stat(root)).mode & 0o777; + const fileMode = (await stat(join(root, `${id}.txt`))).mode & 0o777; + const metaMode = (await stat(join(root, `${id}.json`))).mode & 0o777; + assert.equal(dirMode, 0o700); + assert.equal(fileMode, 0o600); + assert.equal(metaMode, 0o600); +}); + +test("retention and total-size cleanup remove old entries", async () => { + const root = await makeStore(); + const bound = harness(); + overflowExtension(bound.pi, { + storeRoot: root, + settings: settings({ retentionDays: 1, maxStoreBytes: 100 }), + }); + + // An old entry (mtime in the past) must be deleted by age. + const oldId = "aaaaaaaaaaaa"; + await writeFile(join(root, `${oldId}.txt`), "old"); + await writeFile(join(root, `${oldId}.json`), "{}"); + const past = new Date(Date.now() - 10 * 86_400_000); + const { utimes } = await import("node:fs/promises"); + await utimes(join(root, `${oldId}.txt`), past, past); + await utimes(join(root, `${oldId}.json`), past, past); + + for (const handler of bound.handlers.get("session_start") ?? []) await handler({}, ctx()); + const names = await readdir(root); + assert.ok(!names.some((n) => n.startsWith(oldId)), "old entry must be deleted"); + + // Fresh entries exceeding maxStoreBytes get evicted oldest-first. + for (const [i, id] of ["bbbbbbbbbbbb", "cccccccccccc", "dddddddddddd"].entries()) { + await writeFile(join(root, `${id}.txt`), "y".repeat(80)); + const t = new Date(Date.now() - (3 - i) * 60_000); + await utimes(join(root, `${id}.txt`), t, t); + } + for (const handler of bound.handlers.get("session_start") ?? []) await handler({}, ctx()); + const after = (await readdir(root)).filter((n) => n.endsWith(".txt")); + assert.ok(after.length <= 2, `expected eviction under maxStoreBytes, got ${after.join(",")}`); + assert.ok(!after.includes("bbbbbbbbbbbb.txt"), "oldest entry must be evicted first"); +}); + +test("disabled setting clears the stash and returns undefined", async () => { + const root = await makeStore(); + getStash().clear(); + const bound = harness(); + overflowExtension(bound.pi, { storeRoot: root, settings: settings({ enabled: false }) }); + getStash().set("c-off", { rawText: "x".repeat(30_000) }); + const event = textEvent("c-off", "x".repeat(30_000)); + const result = await runHandlers(bound.handlers, event, ctx()); + assert.equal(result, event); + assert.ok(!getStash().has("c-off")); +}); diff --git a/pi-overflow/test/query.test.ts b/pi-overflow/test/query.test.ts new file mode 100644 index 0000000..dc696a5 --- /dev/null +++ b/pi-overflow/test/query.test.ts @@ -0,0 +1,112 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtemp, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { runOverflowQuery } from "../src/query.ts"; +import { storeEntry } from "../src/store.ts"; +import { readOverflowSettings, type OverflowSettings } from "../settings.ts"; + +function settings(overrides: Partial = {}): OverflowSettings { + return { ...readOverflowSettings("/nonexistent/pi-overflow.json"), ...overrides }; +} + +async function store(text: string, id = "0123456789ab") { + const root = await mkdtemp(join(tmpdir(), "pi-overflow-query-")); + await storeEntry(root, id, { kind: "text", text }, { tool: "bash", args: { command: "run" } }, 1_000_000); + return root; +} + +test("search returns hits with merged context", async () => { + const lines = Array.from({ length: 100 }, (_, i) => (i === 9 || i === 11 || i === 50 ? `line ${i + 1} FAIL` : `line ${i + 1}`)); + const root = await store(lines.join("\n")); + const result = await runOverflowQuery(root, { id: "0123456789ab", search: "FAIL", context: 1 }, settings()); + assert.ok(!result.error); + assert.match(result.text, /L10: line 10 FAIL/); + assert.match(result.text, /L12: line 12 FAIL/); + assert.match(result.text, /L51: line 51 FAIL/); + // hits at lines 10 and 12 with context 1 merge into one range 9-13 + assert.match(result.text, /--- lines 9-13 ---/); + assert.match(result.text, /--- lines 50-52 ---/); +}); + +test("line-range retrieval returns numbered lines", async () => { + const lines = Array.from({ length: 50 }, (_, i) => `row ${i + 1}`); + const root = await store(lines.join("\n")); + const result = await runOverflowQuery(root, { id: "0123456789ab", lines: { start: 10, end: 15 } }, settings()); + assert.match(result.text, /--- lines 10-15 ---/); + assert.match(result.text, /L10: row 10/); + assert.match(result.text, /L15: row 15/); + assert.ok(!result.text.includes("row 16")); +}); + +test("jsonPath extracts nested values", async () => { + const doc = JSON.stringify({ snapshot: { nodes: [{ name: "a" }, { name: "b" }, { name: "c" }] } }); + const root = await store(doc); + const result = await runOverflowQuery(root, { id: "0123456789ab", jsonPath: "snapshot.nodes[2].name" }, settings()); + assert.match(result.text, /"c"/); +}); + +test("invalid id, regex, JSON and path return clear errors without throwing", async () => { + const root = await store('{"a": {"b": 1}}'); + assert.ok((await runOverflowQuery(root, { id: "../etc/passwd" }, settings())).error); + assert.ok((await runOverflowQuery(root, { id: "zzzzzzzzzzzz" }, settings())).error); + assert.ok((await runOverflowQuery(root, { id: "0123456789ab", search: "([" }, settings())).error); + const goodPath = await runOverflowQuery(root, { id: "0123456789ab", jsonPath: "a.b" }, settings()); + assert.ok(!goodPath.error, "valid path on valid JSON must succeed"); + const badPath = await runOverflowQuery(root, { id: "0123456789ab", jsonPath: "a.b.c!!" }, settings()); + assert.ok(badPath.error); + const missing = await runOverflowQuery(root, { id: "ffffffffffff" }, settings()); + assert.ok(missing.error); +}); + +test("non-JSON content + jsonPath reports the parse failure", async () => { + const root = await store("not json at all"); + const result = await runOverflowQuery(root, { id: "0123456789ab", jsonPath: "a" }, settings()); + assert.ok(result.error); + assert.match(result.text, /not valid JSON/); +}); + +test("output is bounded by thresholdChars and marked truncated", async () => { + const lines = Array.from({ length: 5000 }, (_, i) => `line ${i + 1} pad ${"p".repeat(40)}`); + const root = await store(lines.join("\n")); + const result = await runOverflowQuery( + root, + { id: "0123456789ab", lines: { start: 1, end: 5000 } }, + settings({ thresholdChars: 4000 }), + ); + assert.ok(result.text.length <= 4000); + assert.equal(result.truncated, true); + assert.match(result.text, /truncated/); +}); + +test("lines and search output clip each line to maxLineChars", async () => { + const long = `row ${"q".repeat(2_000)}`; + const root = await store(`first\n${long}\n${long} ERROR\nlast`); + const s = settings({ maxLineChars: 60 }); + const lines = await runOverflowQuery(root, { id: "0123456789ab", lines: { start: 2, end: 3 } }, s); + assert.ok(lines.text.includes("q".repeat(56) + "…")); + assert.ok(!lines.text.includes("q".repeat(61))); + const search = await runOverflowQuery(root, { id: "0123456789ab", search: "ERROR", context: 0 }, s); + assert.ok(search.text.includes("q".repeat(56) + "…")); + assert.ok(!search.text.includes("q".repeat(61))); +}); + +test("search only scans the first 10000 chars of each line", async () => { + const earlyHit = `hit ${"a".repeat(100)} FAIL`; + const lateHit = `${"a".repeat(10_500)} FAIL`; + const root = await store(`${earlyHit}\n${lateHit}\nplain`); + const result = await runOverflowQuery(root, { id: "0123456789ab", search: "FAIL", context: 0 }, settings()); + assert.match(result.text, /L1: hit /); + assert.ok(!result.text.includes("L2:"), "match beyond char 10000 must not be reported"); +}); + +test("default query returns metadata and a first-screen view", async () => { + const lines = Array.from({ length: 300 }, (_, i) => `row ${i + 1}`); + const root = await store(lines.join("\n")); + const result = await runOverflowQuery(root, { id: "0123456789ab" }, settings()); + assert.ok(!result.error); + assert.match(result.text, /\[pi-overflow id=0123456789ab tool=bash \d+ lines \/ \d+ bytes\]/); + assert.match(result.text, /--- lines 1-40 ---/); + assert.ok(result.text.includes("row 300"), "tail of the stored file must appear"); +}); diff --git a/pi-overflow/test/view.test.ts b/pi-overflow/test/view.test.ts new file mode 100644 index 0000000..0df7f01 --- /dev/null +++ b/pi-overflow/test/view.test.ts @@ -0,0 +1,86 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { buildView, collectViewFromText } from "../src/view.ts"; +import { readOverflowSettings, type OverflowSettings } from "../settings.ts"; + +function settings(overrides: Partial = {}): OverflowSettings { + return { + ...readOverflowSettings("/nonexistent/pi-overflow.json"), + ...overrides, + }; +} + +const meta = { id: "abc123def456", tool: "bash" }; + +function view(text: string, s = settings()) { + return buildView(collectViewFromText(text, s), s, meta); +} + +test("below threshold remains unchanged", () => { + const out = view("line one\nline two\nline three"); + assert.ok(out.includes("line one")); + assert.ok(out.includes("line three")); +}); + +test("above threshold includes header, line ranges, signal lines and tail", () => { + const lines: string[] = []; + for (let i = 1; i <= 1000; i++) lines.push(`row ${i} ${i === 500 ? "ERROR boom" : "ok"}`); + const out = view(lines.join("\n")); + assert.match(out, /\[pi-overflow id=abc123def456 tool=bash 1000 lines/); + assert.match(out, /--- lines 1-40 ---/); + assert.match(out, /L500: row 500 ERROR boom/); + assert.match(out, /--- lines 881-1000 ---/); + assert.ok(out.includes("row 1000")); +}); + +test("output never exceeds thresholdChars", () => { + const lines = Array.from({ length: 5000 }, (_, i) => `line ${i} ${"x".repeat(60)}`); + const s = settings({ thresholdChars: 2000 }); + const out = view(lines.join("\n"), s); + assert.ok(out.length <= 2000, `view ${out.length} exceeds budget`); + assert.ok(out.includes("line 4999"), "tail must survive"); +}); + +test("few long lines still expose the ending", () => { + const lines = ["start", ...Array.from({ length: 5 }, (_, i) => `middle${i} ${"y".repeat(4000)}`), "THE_END_MARKER"]; + const s = settings({ thresholdChars: 3000, headLines: 40, tailLines: 120 }); + const out = view(lines.join("\n"), s); + assert.ok(out.includes("THE_END_MARKER")); + assert.ok(out.length <= 3000 + 100, `view ${out.length} far exceeds budget`); +}); + +test("same input produces identical output", () => { + const lines = Array.from({ length: 800 }, (_, i) => `entry ${i} ${i % 77 === 0 ? "warning" : "fine"}`); + const text = lines.join("\n"); + assert.equal(view(text), view(text)); +}); + +test("a single line over maxLineChars is truncated", () => { + const long = "z".repeat(2000); + const out = view(`a\n${long}\nb`, settings({ maxLineChars: 50 })); + assert.ok(out.includes("z".repeat(50) + "…")); + assert.ok(!out.includes("z".repeat(51))); +}); + +test("signals outrank head and survive when head+tail exceed the budget", () => { + // ~150-char lines: head 40 + tail 120 = 160 lines * 150 = ~24K plus + // overhead, over budget. Shrink order must keep some signals while head + // collapses to 10 lines before the 35/65 phase kicks in. + const lines = Array.from({ length: 400 }, (_, i) => { + const marker = i === 200 ? "ERROR detected here" : "ok"; + return `line ${i + 1} ${marker} ${"x".repeat(140)}`; + }); + const out = view(lines.join("\n")); + assert.match(out, /--- signals/); + assert.match(out, /L201: line 201 ERROR detected here/); + // head collapsed below the configured 40 lines + assert.match(out, /--- lines 1-(?:[0-9]|10) ---/); + assert.ok(out.includes("line 400")); +}); + +test("signal hits inside the tail region are not duplicated", () => { + const lines = Array.from({ length: 100 }, (_, i) => `line ${i + 1} ${"y".repeat(40)}`); + lines[99] = "line 100 tail-side ERROR"; + const out = view(lines.join("\n")); + assert.ok(!out.includes("L100:"), "tail-region hit must not appear in signals"); +}); diff --git a/pi-rtk-optimizer/AGENTS.md b/pi-rtk-optimizer/AGENTS.md index c1c352c..6085e7e 100644 --- a/pi-rtk-optimizer/AGENTS.md +++ b/pi-rtk-optimizer/AGENTS.md @@ -2,7 +2,7 @@ - 扩展入口是 `pi-rtk-optimizer/index.ts`，主要实现位于 `pi-rtk-optimizer/src/`。 - 组合包运行环境为 Node.js 22.19 或更高版本；本子目录开发验证还需要 npm、Bun 和项目声明的开发依赖。 -- RTK 默认只压缩非搜索输出，包括本地 `bash` 与远端 `ssh_bash` 的 ANSI 清理、测试聚合、构建过滤、Git 压缩、Lint 聚合和兜底截断；`ssh_bash` 只复用输出处理，远端命令不得进入 RTK rewrite。 +- RTK 默认只压缩非搜索输出，包括本地 `bash` 与远端 `ssh_bash` 的 ANSI 清理、测试聚合、构建过滤、Git 压缩、Lint 聚合；兜底截断默认关闭，新结果的展示截断由 pi-overflow 统一负责；`ssh_bash` 只复用输出处理，远端命令不得进入 RTK rewrite。 - RTK 命令改写默认关闭（`commandRewritingEnabled` 默认 `false`）；默认安装不依赖系统 `rtk` CLI，只有用户主动开启命令改写时才需要 `rtk rewrite`，其命令支持策略由已安装的 `rtk rewrite` 决定，不在扩展内重复维护规则。 - FFF override 独占搜索工具与搜索输出；RTK 不得处理本地或远端搜索工具的调用/结果（含 `grep` 工具结果），也不得通过默认命令改写接管 `rg`、`grep`、`find`、`fd`。 - `readCompaction.enabled`、`sourceCodeFilteringEnabled` 和 `smartTruncate.enabled` 当前均默认 `false`，源码读取保持原样；未经用户明确决定，不因节省上下文而改变这些默认值。若以后开启 read 压缩，优先考虑 `readCompaction + smartTruncate`，源码过滤仍独立评估；必须保留精确 `offset/limit` 读取、短文件和行锚点的完整性。 diff --git a/pi-rtk-optimizer/README.md b/pi-rtk-optimizer/README.md index d2a7ef1..64d5133 100644 --- a/pi-rtk-optimizer/README.md +++ b/pi-rtk-optimizer/README.md @@ -154,7 +154,7 @@ When `read` output uses Pi hashline/anchor prefixes, the compactor treats each a |--------|------|---------|-------|-------------| | `outputCompaction.smartTruncate.enabled` | boolean | `false` | — | Enable smart line-based truncation for read output when read compaction is enabled | | `outputCompaction.smartTruncate.maxLines` | number | `220` | 40–4000 | Maximum lines after smart truncation | -| `outputCompaction.truncate.enabled` | boolean | `true` | — | Enable hard character truncation | +| `outputCompaction.truncate.enabled` | boolean | `false` | — | Enable hard character truncation (off in this bundle — pi-overflow owns display truncation) | | `outputCompaction.truncate.maxChars` | number | `12000` | 1000–200000 | Maximum characters in final output | ### Source Code Filtering Levels @@ -195,7 +195,7 @@ When `read` output uses Pi hashline/anchor prefixes, the compactor treats each a "maxLines": 220 }, "truncate": { - "enabled": true, + "enabled": false, "maxChars": 12000 } } diff --git a/pi-rtk-optimizer/UPSTREAM.md b/pi-rtk-optimizer/UPSTREAM.md index 92a497d..0f0066f 100644 --- a/pi-rtk-optimizer/UPSTREAM.md +++ b/pi-rtk-optimizer/UPSTREAM.md @@ -13,3 +13,4 @@ - 默认只负责非搜索工具输出压缩；RTK 命令改写收敛为可选能力（`commandRewritingEnabled` 默认 `false`）。 - `readCompaction`、`sourceCodeFiltering`、`smartTruncate` 默认关闭，源码读取保持原样。 +- `outputCompaction.truncate` 兜底截断默认关闭（组合包 `config/rtk-optimizer.json`），新结果的展示截断由 pi-overflow 统一负责；RTK 只保留语义压缩。 diff --git a/pi-tool-search/CHANGELOG.md b/pi-tool-search/CHANGELOG.md index d86276d..bfc1a9a 100644 --- a/pi-tool-search/CHANGELOG.md +++ b/pi-tool-search/CHANGELOG.md @@ -12,6 +12,7 @@ - Added checked-in authoritative groups for every tool exposed by the standard my-pi bundle, eliminating first-use model generation and user caching unless unrecognized third-party tools are present. - Replaced the Hermes and third-party Hippo assignments with the five tools exposed by the official Hippo Pi extension, grouped into recall and management workflows. - Replaced the retired Hippo `hippo_*` assignments with the vendored Hermes memory extension's six tools, grouped into `memory-recall` (`memory_search`, `session_search`, `context_tree_query`) and `memory-management` (`memory_add`, `memory_replace`, `memory_remove`, `skill_manage`). +- Added `overflow_query` to `memory-recall` and pinned it in `alwaysEnabled` — inspecting a pi-overflow-stored original must not depend on a `tool_search` activation. - Added the enabled bundle's `Agent`, `get_subagent_result`, and `steer_subagent` tools to a checked-in `subagents` orchestration group. - Added authoritative `ssh-connection`, `ssh-remote-files`, `ssh-remote-search`, and `ssh-remote-shell` groups for the maintained `pi-ssh` tools, keeping reviewed connection, bounded discovery, structured file operations, and reviewed shell execution separate; the connection workflow now requires sequential `ssh_connect` to succeed before dependent tools. - Added sequential `ssh_cd` to the authoritative remote-shell workflow and direct the model to wait for a successful workspace change before dependent remote calls, avoiding both the unknown-tools fallback and cwd races. diff --git a/pi-tool-search/README.md b/pi-tool-search/README.md index 73c9581..f4874a5 100644 --- a/pi-tool-search/README.md +++ b/pi-tool-search/README.md @@ -87,7 +87,8 @@ Pi reads this configuration directly from the installed bundle's `config/tool-se "memory_replace", "memory_remove", "skill_manage", - "context_tree_query" + "context_tree_query", + "overflow_query" ], "showToolSearchFooterStatus": false, "maxActiveGroups": 5, @@ -99,7 +100,7 @@ Pi reads this configuration directly from the installed bundle's `config/tool-se | Key | Bundle default | Description | |---|---|---| -| `alwaysEnabled` | CodeGraph explore, LSP diagnostics, Context execution trio, all six Hermes memory tools, `context_tree_query` | Exact names pinned alongside Pi core tools | +| `alwaysEnabled` | CodeGraph explore, LSP diagnostics, Context execution trio, all six Hermes memory tools, `context_tree_query`, `overflow_query` | Exact names pinned alongside Pi core tools | | `showToolSearchFooterStatus` | `false` | Show active tool/group counts | | `maxActiveGroups` | `5` | Maximum simultaneously active dynamic groups | | `maxToolsPerGroup` | `8` | Maximum size accepted from model-generated groups | diff --git a/pi-tool-search/docs/dynamic-tool-loading.md b/pi-tool-search/docs/dynamic-tool-loading.md index 8f9b37d..c329198 100644 --- a/pi-tool-search/docs/dynamic-tool-loading.md +++ b/pi-tool-search/docs/dynamic-tool-loading.md @@ -13,7 +13,7 @@ 9. Natural-language lookup gives exact tool names and workflow aliases more weight than descriptive overlap. Weak or closely tied matches return candidates without activation. 10. `turn_start` notices tool or grouping-configuration changes by hash, resets stale dynamic groups, and reconstructs the checked-in/hybrid catalog. -The standard bundle assigns the Hermes memory extension's six tools to recall and management groups: `memory_search` and `session_search` join `context_tree_query` in `memory-recall`, while `memory_add`, `memory_replace`, `memory_remove`, and `skill_manage` form `memory-management`. All six are pinned by default, because the injected `` tells the model to consult and write memory during normal work, and a schema the model cannot see would silently disable that loop. Third-party `tff-memory_*` names and the retired Hippo `hippo_*` names are not seeded. It also assigns `Agent`, `get_subagent_result`, and `steer_subagent` to the `subagents` orchestration group. +The standard bundle assigns the Hermes memory extension's six tools to recall and management groups: `memory_search`, `session_search`, and `overflow_query` join `context_tree_query` in `memory-recall`, while `memory_add`, `memory_replace`, `memory_remove`, and `skill_manage` form `memory-management`. All six are pinned by default, because the injected `` tells the model to consult and write memory during normal work, and a schema the model cannot see would silently disable that loop. Third-party `tff-memory_*` names and the retired Hippo `hippo_*` names are not seeded. It also assigns `Agent`, `get_subagent_result`, and `steer_subagent` to the `subagents` orchestration group. The maintained `pi-ssh` tools are also split by workflow: sequential reviewed agent-controlled connection lives in `ssh-connection`, structured `ssh_read`/`ssh_write`/`ssh_edit` file operations live in `ssh-remote-files`, bounded `ssh_find`/`ssh_grep` discovery lives in `ssh-remote-search`, and sequential reviewed workspace changes plus reviewed command execution live in `ssh-remote-shell`. The workflows tell the model to call `ssh_connect` or `ssh_cd` separately and wait for success before issuing remote calls that depend on the new connection or cwd. The maintained `ask_user_question` tool has its own `user-interaction` group so the model can load the complete interactive schema only when a missing decision or preference must come from the user. The pinned `pi-chrome` bridge registers its 21 `chrome_*` tools only after explicit per-session authorization. Once registered, catalog refresh assigns them to `chrome-navigation`, `chrome-interaction`, and `chrome-debugging`; the 28-tool default allows all three groups to coexist. Revocation remains enforced inside every upstream tool execution even if a schema is later reactivated by Tool Search. diff --git a/pi-tool-search/extensions/bundle-groups.ts b/pi-tool-search/extensions/bundle-groups.ts index f1f12a2..1d8d1ea 100644 --- a/pi-tool-search/extensions/bundle-groups.ts +++ b/pi-tool-search/extensions/bundle-groups.ts @@ -124,11 +124,11 @@ export const BUNDLE_GROUP_DEFINITIONS: BundleGroupDefinition[] = [ { id: "memory-recall", title: "Persistent memory recall", - summary: "Search durable user, global, project, and failure memories, search indexed past conversations, or recover pruned tool-call output.", + summary: "Search durable user, global, project, and failure memories, search indexed past conversations, or recover pruned or overflow-stored tool-call output.", useWhen: ["The task may depend on durable context from previous sessions: preferences, project conventions, prior decisions, known failures, corrections, or condensed outputs"], avoidWhen: ["The required evidence is already in the current visible context, or the question is a generic one-off that durable memory would not help"], - keywords: ["memory", "recall", "search", "session", "history", "recover", "记忆", "召回", "检索", "历史会话", "恢复输出"], - tools: ["memory_search", "session_search", "context_tree_query"], + keywords: ["memory", "recall", "search", "session", "history", "recover", "overflow", "记忆", "召回", "检索", "历史会话", "恢复输出"], + tools: ["memory_search", "session_search", "context_tree_query", "overflow_query"], }, { id: "memory-management", diff --git a/pi-tool-search/extensions/config.ts b/pi-tool-search/extensions/config.ts index 7c9ff40..f0de663 100644 --- a/pi-tool-search/extensions/config.ts +++ b/pi-tool-search/extensions/config.ts @@ -35,6 +35,7 @@ export const BUNDLE_TOOL_SEARCH_DEFAULTS = { "memory_remove", "skill_manage", "context_tree_query", + "overflow_query", ], showToolSearchFooterStatus: false, maxActiveGroups: 5, diff --git a/pi-tool-search/test/config.test.ts b/pi-tool-search/test/config.test.ts index 4fb7e08..c370e6b 100644 --- a/pi-tool-search/test/config.test.ts +++ b/pi-tool-search/test/config.test.ts @@ -43,6 +43,7 @@ test("bundle defaults keep the committed tool list", () => { "memory_remove", "skill_manage", "context_tree_query", + "overflow_query", ]); assert.deepEqual(bundleFallbackConfig(), { alwaysEnabled: [...BUNDLE_TOOL_SEARCH_DEFAULTS.alwaysEnabled], diff --git a/pi-web-search/README.md b/pi-web-search/README.md index c0ca71e..40cde5c 100644 --- a/pi-web-search/README.md +++ b/pi-web-search/README.md @@ -6,9 +6,12 @@ Unified web search and fetch for Pi. The main model only ever sees two tools — - `web_search(query, max_results?, site?, published_after?, published_before?, depth?)` routes each query to Tavily, Exa, or Keenable using the session's current main model (`ctx.model` + `ctx.modelRegistry.complete`), guided by per-provider - rules. Results include `details.provider` / `details.routedBy` for debugging. + rules. `max_results` defaults to 8 and is clamped to 1–20. Results include + `details.provider` / `details.routedBy` for debugging. - `web_fetch(urls, max_characters?)` always reads pages via Exa (`web_fetch_exa`), degrading silently to Keenable then Tavily. + `max_characters` defaults to 20,000 and is clamped to 1–100,000 per page; + oversized fetch output is further bounded by pi-overflow downstream. ## Providers (`clients/`) diff --git a/pi-web-search/index.ts b/pi-web-search/index.ts index f33de04..a7b419e 100644 --- a/pi-web-search/index.ts +++ b/pi-web-search/index.ts @@ -21,8 +21,15 @@ import { } from "./router.ts"; const DEFAULT_MAX_RESULTS = 8; +const MAX_RESULTS_LIMIT = 20; const MAX_FETCH_URLS = 10; -const DEFAULT_FETCH_MAX_CHARACTERS = 3000; +const DEFAULT_FETCH_MAX_CHARACTERS = 20000; +const FETCH_MAX_CHARACTERS_LIMIT = 100000; + +function clampInt(value: number | undefined, fallback: number, min: number, max: number): number { + if (typeof value !== "number" || !Number.isFinite(value)) return fallback; + return Math.min(Math.max(Math.floor(value), min), max); +} export interface WebSearchParams { query: string; @@ -48,7 +55,7 @@ function formatSearchPayload(payload: unknown): string { export function toTavilyParams(params: WebSearchParams) { return { query: params.query, - max_results: params.max_results ?? DEFAULT_MAX_RESULTS, + max_results: clampInt(params.max_results, DEFAULT_MAX_RESULTS, 1, MAX_RESULTS_LIMIT), search_depth: (params.depth ?? "basic") as "basic" | "advanced", include_domains: params.site ? [params.site] : undefined, start_date: params.published_after, @@ -59,7 +66,7 @@ export function toTavilyParams(params: WebSearchParams) { export function toExaParams(params: WebSearchParams) { return { query: params.query, - numResults: params.max_results ?? DEFAULT_MAX_RESULTS, + numResults: clampInt(params.max_results, DEFAULT_MAX_RESULTS, 1, MAX_RESULTS_LIMIT), includeDomains: params.site ? [params.site] : undefined, startPublishedDate: params.published_after, endPublishedDate: params.published_before, @@ -171,7 +178,7 @@ export async function dispatchFetch( if (urls.length > MAX_FETCH_URLS) { return toResult(`web_fetch accepts at most ${MAX_FETCH_URLS} URLs per call; received ${urls.length}.`, { error: true }); } - const maxCharacters = params.max_characters ?? DEFAULT_FETCH_MAX_CHARACTERS; + const maxCharacters = clampInt(params.max_characters, DEFAULT_FETCH_MAX_CHARACTERS, 1, FETCH_MAX_CHARACTERS_LIMIT); const order: SearchProvider[] = ["exa", "keenable"]; if (tavilyAvailable(resolve)) order.push("tavily"); @@ -213,7 +220,9 @@ export default function webSearchExtension(pi: ExtensionAPI): void { }, max_results: { type: "number", - description: `Maximum number of results to return (default ${DEFAULT_MAX_RESULTS}).`, + minimum: 1, + maximum: MAX_RESULTS_LIMIT, + description: `Maximum number of results to return (default ${DEFAULT_MAX_RESULTS}, max ${MAX_RESULTS_LIMIT}).`, }, site: { type: "string", @@ -268,7 +277,9 @@ export default function webSearchExtension(pi: ExtensionAPI): void { }, max_characters: { type: "number", - description: `Maximum characters to extract per page (default ${DEFAULT_FETCH_MAX_CHARACTERS}).`, + minimum: 1, + maximum: FETCH_MAX_CHARACTERS_LIMIT, + description: `Maximum characters to extract per page (default ${DEFAULT_FETCH_MAX_CHARACTERS}, max ${FETCH_MAX_CHARACTERS_LIMIT}).`, }, }, required: ["urls"], diff --git a/tests/extension-load.test.ts b/tests/extension-load.test.ts index 64f7538..1d66435 100644 --- a/tests/extension-load.test.ts +++ b/tests/extension-load.test.ts @@ -198,6 +198,21 @@ test("all package extensions load together without global registration conflicts "./extensions/system-prompt.ts", "the single system-prompt owner must be the last loaded extension", ); + // pi-overflow has two entry points with strict ordering: capture.ts must + // run first so it sees results before RTK rewrites them, and index.ts must + // run last (before system-prompt) so it bounds the final visible text. + assert.equal( + packageJson.pi.extensions.at(0), + "./pi-overflow/capture.ts", + "pi-overflow capture must be the first loaded extension", + ); + assert.equal(packageJson.dependencies["pi-overflow"], "file:./pi-overflow"); + assert.ok(filesIncludePackageDir(packageJson.files, "pi-overflow"), "the packed bundle must include pi-overflow source"); + assert.equal( + packageJson.pi.extensions.at(-2), + "./pi-overflow/index.ts", + "pi-overflow bound stage must load after all other content extensions and before system-prompt", + ); assert.ok( packageJson.pi.extensions.includes("./extensions/context-mode.ts"), "Context Mode must load through the bundle's prompt-guarded wrapper",