mirror of
https://bitbucket.org/siakitem/my-pi.git
synced 2026-10-26 10:45:22 +00:00
feat(pi-overflow): 新增统一的新工具结果展示截断扩展
此前截断分散在多层且互相冲突：Pi bash 保留尾部 50KB，RTK 兜底截断又只留 开头 12000 字符（砍掉报错与完整输出路径），pi-chrome 截开头 30K，web_fetch 各 provider 不一致。新扩展统一负责"新结果给模型看多少"： - capture.ts 最先加载暂存原文，index.ts 最后加载（system-prompt 之前） - 超过 24K 字符时原文落盘 <agentDir>/my-pi/pi-overflow/（700/600、原子写）， 优先取 bash fullOutputPath 完整文件；模型看到 头 + 中段信号行 + 尾 的有界 视图，行号基于落盘原文、结尾永远可见 - overflow_query：按 id 正则搜索、行区间、jsonPath 只读查询，id 校验防穿越 - 收编：RTK truncate 默认关闭、删除 pi-chrome truncateText、web_fetch 默认 每页 20000（上限 100000）、web_search max_results 限定 1–20 - pi-compaction 遮蔽带 details.overflow.id 的结果时在占位符提示 overflow_query - 登记 tools.jsonc / routing / 权限 allow / tool-search 常驻与分组 Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
云服务部-叶林立
co-authored by
Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
parent
a6d8416c76
commit
2a6bf8c929
@@ -0,0 +1,314 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtemp, readFile, readdir, stat, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import overflowCapture from "../capture.ts";
|
||||
import overflowExtension from "../index.ts";
|
||||
import { getStash } from "../src/stash.ts";
|
||||
import { readOverflowSettings, OVERFLOW_DEFAULTS, type OverflowSettings } from "../settings.ts";
|
||||
|
||||
function harness() {
|
||||
const handlers = new Map<string, Function[]>();
|
||||
const tools = new Map<string, any>();
|
||||
const pi: any = {
|
||||
on(name: string, handler: Function) {
|
||||
const list = handlers.get(name) ?? [];
|
||||
list.push(handler);
|
||||
handlers.set(name, list);
|
||||
},
|
||||
registerTool(tool: any) { tools.set(tool.name, tool); },
|
||||
};
|
||||
return { pi, handlers, tools };
|
||||
}
|
||||
|
||||
function ctx() {
|
||||
return { sessionManager: { getSessionId: () => "session-test" } } as any;
|
||||
}
|
||||
|
||||
function settings(overrides: Partial<OverflowSettings> = {}): OverflowSettings {
|
||||
return { ...readOverflowSettings("/nonexistent/pi-overflow.json"), ...overrides };
|
||||
}
|
||||
|
||||
function textEvent(toolCallId: string, text: string, extra: Record<string, unknown> = {}) {
|
||||
return {
|
||||
toolCallId,
|
||||
toolName: "bash",
|
||||
input: { command: "run" },
|
||||
content: [{ type: "text", text }],
|
||||
details: {},
|
||||
...extra,
|
||||
};
|
||||
}
|
||||
|
||||
async function runHandlers(handlers: Map<string, Function[]>, event: any, context: any) {
|
||||
let current = event;
|
||||
for (const handler of handlers.get("tool_result") ?? []) {
|
||||
const result = await handler(current, context);
|
||||
if (result) current = { ...current, ...result };
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
async function makeStore() {
|
||||
return mkdtemp(join(tmpdir(), "pi-overflow-test-"));
|
||||
}
|
||||
|
||||
test("capture stashes the original before another handler shortens it; bound stage stores original but views current", async () => {
|
||||
const root = await makeStore();
|
||||
getStash().clear();
|
||||
const capture = harness();
|
||||
overflowCapture(capture.pi);
|
||||
const bound = harness();
|
||||
overflowExtension(bound.pi, { storeRoot: root, settings: settings() });
|
||||
|
||||
const big = Array.from({ length: 3000 }, (_, i) => `orig ${i}`).join("\n");
|
||||
const event = textEvent("call-1", big);
|
||||
const afterCapture = await runHandlers(capture.handlers, event, ctx());
|
||||
assert.equal(afterCapture, event, "capture must not modify the result");
|
||||
assert.ok(getStash().has("call-1"), "capture must stash the original");
|
||||
|
||||
// Simulate RTK: replace content with a shortened version that is still
|
||||
// over threshold.
|
||||
const rtkShortened = Array.from({ length: 900 }, (_, i) => `rtk ${i}`).join("\n") + "\n" + "z".repeat(30_000);
|
||||
const mid = { ...afterCapture, content: [{ type: "text", text: rtkShortened }] };
|
||||
const final = await runHandlers(bound.handlers, mid, ctx());
|
||||
|
||||
assert.ok(!getStash().has("call-1"), "stash entry must be consumed");
|
||||
const text = final.content[0].text;
|
||||
assert.match(text, /\[pi-overflow id=[0-9a-f]{12} tool=bash/);
|
||||
assert.ok(text.length <= settings().thresholdChars);
|
||||
|
||||
const id = final.details.overflow.id;
|
||||
const stored = await readFile(join(root, `${id}.txt`), "utf8");
|
||||
assert.ok(stored.includes("orig 0"), "store must keep the ORIGINAL, not the RTK-shortened text");
|
||||
assert.ok(stored.includes("orig 2999"));
|
||||
assert.equal(final.details.overflow.truncated, true);
|
||||
});
|
||||
|
||||
test("details.fullOutputPath file is used as the original", async () => {
|
||||
const root = await makeStore();
|
||||
getStash().clear();
|
||||
const bound = harness();
|
||||
overflowExtension(bound.pi, { storeRoot: root, settings: settings() });
|
||||
|
||||
const spill = join(root, "spill.txt");
|
||||
await writeFile(spill, Array.from({ length: 5000 }, (_, i) => `spill ${i}`).join("\n"));
|
||||
const event = textEvent("call-2", "x".repeat(30_000), { details: { fullOutputPath: spill } });
|
||||
const final = await runHandlers(bound.handlers, event, ctx());
|
||||
const stored = await readFile(join(root, `${final.details.overflow.id}.txt`), "utf8");
|
||||
assert.ok(stored.includes("spill 0"));
|
||||
assert.ok(stored.includes("spill 4999"));
|
||||
});
|
||||
|
||||
test("view line numbers track the STORED file, not the current text", async () => {
|
||||
const root = await makeStore();
|
||||
getStash().clear();
|
||||
const bound = harness();
|
||||
overflowExtension(bound.pi, { storeRoot: root, settings: settings() });
|
||||
|
||||
// bash spill: 5000 lines on disk, but the in-context result only carries
|
||||
// the last ~100 lines (still over threshold via padding).
|
||||
const spill = join(root, "spill-5000.txt");
|
||||
await writeFile(
|
||||
spill,
|
||||
Array.from({ length: 5000 }, (_, i) => `spill line ${i + 1}`).join("\n"),
|
||||
);
|
||||
const current = Array.from({ length: 100 }, (_, i) => `spill line ${4901 + i}`).join("\n") + "\n" + "pad".repeat(10_000);
|
||||
const event = textEvent("call-lines", current, { details: { fullOutputPath: spill } });
|
||||
const final = await runHandlers(bound.handlers, event, ctx());
|
||||
|
||||
const text = final.content[0].text;
|
||||
const id = final.details.overflow.id;
|
||||
assert.match(text, /--- lines 4881-5000 ---/, "tail range must come from the stored 5000-line file");
|
||||
assert.match(text, /spill line 5000/);
|
||||
assert.match(text, /\[pi-overflow id=[0-9a-f]{12} tool=bash 5000 lines/);
|
||||
|
||||
// overflow_query with the view's tail range returns the same lines.
|
||||
const tool = bound.tools.get("overflow_query");
|
||||
const result = await tool.execute("q1", { id, lines: { start: 4881, end: 4883 } });
|
||||
assert.match(result.content[0].text, /L4881: spill line 4881/);
|
||||
assert.match(result.content[0].text, /L4883: spill line 4883/);
|
||||
});
|
||||
|
||||
test("RTK scenario: view line numbers come from the stored original text", async () => {
|
||||
const root = await makeStore();
|
||||
getStash().clear();
|
||||
const capture = harness();
|
||||
overflowCapture(capture.pi);
|
||||
const bound = harness();
|
||||
overflowExtension(bound.pi, { storeRoot: root, settings: settings() });
|
||||
|
||||
const original = Array.from({ length: 3000 }, (_, i) => `orig ${i + 1}`).join("\n");
|
||||
await runHandlers(capture.handlers, textEvent("call-rtk", original), ctx());
|
||||
// RTK rewrites the visible text but it stays over threshold.
|
||||
const mid = {
|
||||
...textEvent("call-rtk", ""),
|
||||
content: [{ type: "text", text: "z".repeat(30_000) }],
|
||||
};
|
||||
const final = await runHandlers(bound.handlers, mid, ctx());
|
||||
const text = final.content[0].text;
|
||||
assert.match(text, /--- lines 2881-3000 ---/, "view tail must be the stored original's tail");
|
||||
assert.ok(text.includes("orig 3000"));
|
||||
});
|
||||
|
||||
test("file sources over maxEntryBytes keep real head and tail halves", async () => {
|
||||
const root = await makeStore();
|
||||
const { storeEntry } = await import("../src/store.ts");
|
||||
const src = join(root, "big.bin");
|
||||
const content = "HEAD-".repeat(200) + "MID-".repeat(2000) + "TAIL-".repeat(200);
|
||||
await writeFile(src, content);
|
||||
const maxEntryBytes = 2000;
|
||||
await storeEntry(root, "0123456789ab", { kind: "file", path: src }, { tool: "bash", args: {} }, maxEntryBytes);
|
||||
const stored = await readFile(join(root, "0123456789ab.txt"), "utf8");
|
||||
assert.ok(stored.startsWith("HEAD-"));
|
||||
assert.ok(stored.endsWith("TAIL-"));
|
||||
assert.match(stored, /bytes omitted/);
|
||||
// Head and tail halves must equal the source's real bytes.
|
||||
const half = Math.floor(maxEntryBytes / 2);
|
||||
assert.equal(stored.slice(0, half), content.slice(0, half));
|
||||
assert.equal(stored.slice(-half), content.slice(-half));
|
||||
assert.ok(!stored.includes("\0".repeat(100)), "must not store zeroed content");
|
||||
});
|
||||
|
||||
test("interrupted-write tmp files older than an hour are cleaned", async () => {
|
||||
const root = await makeStore();
|
||||
const bound = harness();
|
||||
overflowExtension(bound.pi, { storeRoot: root, settings: settings() });
|
||||
|
||||
const staleTmp = join(root, "deadbeef.txt.tmp-999-zzz");
|
||||
const freshTmp = join(root, "livebeef.txt.tmp-111-aaa");
|
||||
await writeFile(staleTmp, "partial");
|
||||
await writeFile(freshTmp, "partial");
|
||||
const old = new Date(Date.now() - 2 * 3_600_000);
|
||||
const { utimes } = await import("node:fs/promises");
|
||||
await utimes(staleTmp, old, old);
|
||||
|
||||
for (const handler of bound.handlers.get("session_start") ?? []) await handler({}, ctx());
|
||||
const { existsSync } = await import("node:fs");
|
||||
assert.ok(!existsSync(staleTmp), "stale tmp must be removed");
|
||||
assert.ok(existsSync(freshTmp), "fresh tmp must survive");
|
||||
});
|
||||
|
||||
test("current under threshold but original over → one-line marker appended", async () => {
|
||||
const root = await makeStore();
|
||||
getStash().clear();
|
||||
const capture = harness();
|
||||
overflowCapture(capture.pi);
|
||||
const bound = harness();
|
||||
overflowExtension(bound.pi, { storeRoot: root, settings: settings() });
|
||||
|
||||
const big = "a".repeat(30_000);
|
||||
const event = textEvent("call-3", big);
|
||||
await runHandlers(capture.handlers, event, ctx());
|
||||
const mid = { ...event, content: [{ type: "text", text: "short summary" }] };
|
||||
const final = await runHandlers(bound.handlers, mid, ctx());
|
||||
|
||||
const text = final.content[0].text;
|
||||
assert.ok(text.startsWith("short summary"));
|
||||
assert.match(text, /\[pi-overflow id=[0-9a-f]{12}: full original output stored \(\d+ lines, .*; use overflow_query/);
|
||||
assert.equal(final.details.overflow.truncated, false);
|
||||
assert.ok(final.details.overflow.id);
|
||||
});
|
||||
|
||||
test("exempt tools and nested calls are ignored", async () => {
|
||||
const root = await makeStore();
|
||||
getStash().clear();
|
||||
const bound = harness();
|
||||
overflowExtension(bound.pi, { storeRoot: root, settings: settings() });
|
||||
|
||||
const big = "x".repeat(30_000);
|
||||
const exempt = await runHandlers(bound.handlers, textEvent("c-ex", big, { toolName: "read" }), ctx());
|
||||
assert.equal(exempt.content[0].text, big, "read results must pass through untouched");
|
||||
|
||||
const nested = await runHandlers(bound.handlers, textEvent("c-nest", big, { parentToolCallId: "parent" }), ctx());
|
||||
assert.equal(nested.content[0].text, big, "nested calls must pass through untouched");
|
||||
});
|
||||
|
||||
test("image parts and structuredContent survive", async () => {
|
||||
const root = await makeStore();
|
||||
getStash().clear();
|
||||
const bound = harness();
|
||||
overflowExtension(bound.pi, { storeRoot: root, settings: settings() });
|
||||
|
||||
const event = textEvent("call-4", "x".repeat(30_000));
|
||||
event.content.push({ type: "image", data: "b64" } as any);
|
||||
(event as any).structuredContent = { rows: 3 };
|
||||
const final = await runHandlers(bound.handlers, event, ctx());
|
||||
assert.deepEqual(final.structuredContent, { rows: 3 });
|
||||
assert.equal(final.content.at(-1).type, "image");
|
||||
assert.equal(final.content.at(-1).data, "b64");
|
||||
});
|
||||
|
||||
test("existing details fields survive the merge", async () => {
|
||||
const root = await makeStore();
|
||||
getStash().clear();
|
||||
const bound = harness();
|
||||
overflowExtension(bound.pi, { storeRoot: root, settings: settings() });
|
||||
|
||||
const event = textEvent("call-5", "x".repeat(30_000), { details: { rtk: { saved: 100 }, fullOutputPath: undefined } });
|
||||
const final = await runHandlers(bound.handlers, event, ctx());
|
||||
assert.deepEqual(final.details.rtk, { saved: 100 });
|
||||
assert.ok(final.details.overflow.id);
|
||||
});
|
||||
|
||||
test("files are 0600 and directories 0700", async () => {
|
||||
const root = await makeStore();
|
||||
getStash().clear();
|
||||
const bound = harness();
|
||||
overflowExtension(bound.pi, { storeRoot: root, settings: settings() });
|
||||
|
||||
const final = await runHandlers(bound.handlers, textEvent("call-6", "x".repeat(30_000)), ctx());
|
||||
const id = final.details.overflow.id;
|
||||
const dirMode = (await stat(root)).mode & 0o777;
|
||||
const fileMode = (await stat(join(root, `${id}.txt`))).mode & 0o777;
|
||||
const metaMode = (await stat(join(root, `${id}.json`))).mode & 0o777;
|
||||
assert.equal(dirMode, 0o700);
|
||||
assert.equal(fileMode, 0o600);
|
||||
assert.equal(metaMode, 0o600);
|
||||
});
|
||||
|
||||
test("retention and total-size cleanup remove old entries", async () => {
|
||||
const root = await makeStore();
|
||||
const bound = harness();
|
||||
overflowExtension(bound.pi, {
|
||||
storeRoot: root,
|
||||
settings: settings({ retentionDays: 1, maxStoreBytes: 100 }),
|
||||
});
|
||||
|
||||
// An old entry (mtime in the past) must be deleted by age.
|
||||
const oldId = "aaaaaaaaaaaa";
|
||||
await writeFile(join(root, `${oldId}.txt`), "old");
|
||||
await writeFile(join(root, `${oldId}.json`), "{}");
|
||||
const past = new Date(Date.now() - 10 * 86_400_000);
|
||||
const { utimes } = await import("node:fs/promises");
|
||||
await utimes(join(root, `${oldId}.txt`), past, past);
|
||||
await utimes(join(root, `${oldId}.json`), past, past);
|
||||
|
||||
for (const handler of bound.handlers.get("session_start") ?? []) await handler({}, ctx());
|
||||
const names = await readdir(root);
|
||||
assert.ok(!names.some((n) => n.startsWith(oldId)), "old entry must be deleted");
|
||||
|
||||
// Fresh entries exceeding maxStoreBytes get evicted oldest-first.
|
||||
for (const [i, id] of ["bbbbbbbbbbbb", "cccccccccccc", "dddddddddddd"].entries()) {
|
||||
await writeFile(join(root, `${id}.txt`), "y".repeat(80));
|
||||
const t = new Date(Date.now() - (3 - i) * 60_000);
|
||||
await utimes(join(root, `${id}.txt`), t, t);
|
||||
}
|
||||
for (const handler of bound.handlers.get("session_start") ?? []) await handler({}, ctx());
|
||||
const after = (await readdir(root)).filter((n) => n.endsWith(".txt"));
|
||||
assert.ok(after.length <= 2, `expected eviction under maxStoreBytes, got ${after.join(",")}`);
|
||||
assert.ok(!after.includes("bbbbbbbbbbbb.txt"), "oldest entry must be evicted first");
|
||||
});
|
||||
|
||||
test("disabled setting clears the stash and returns undefined", async () => {
|
||||
const root = await makeStore();
|
||||
getStash().clear();
|
||||
const bound = harness();
|
||||
overflowExtension(bound.pi, { storeRoot: root, settings: settings({ enabled: false }) });
|
||||
getStash().set("c-off", { rawText: "x".repeat(30_000) });
|
||||
const event = textEvent("c-off", "x".repeat(30_000));
|
||||
const result = await runHandlers(bound.handlers, event, ctx());
|
||||
assert.equal(result, event);
|
||||
assert.ok(!getStash().has("c-off"));
|
||||
});
|
||||
Reference in New Issue
Block a user