feat: enable permission-aware subagents

2026-08-28 08:35:57 +00:00 · 2026-08-26 10:49:26 +08:00
parent d3bf562189
commit 7571ba6dd9
185 changed files with 48365 additions and 26 deletions
@@ -0,0 +1,81 @@
+/**
+ * isolated-provider.e2e.test.ts — reachability guard for PR #152 (issue #151).
+ *
+ * PR #152 fixes isolated subagents dropping extension-registered custom providers
+ * on Pi >= 0.80.8 (the `modelRegistry` → `modelRuntime` migration). agent-runner
+ * forwards the parent's runtime, read off the ModelRegistry facade as
+ * `ctx.modelRegistry.runtime` via `as unknown as { runtime }`.
+ *
+ * That forwarding is already guarded by the unit test in test/agent-runner.test.ts
+ * ("passes the parent model runtime …") — but against a MOCK whose `.runtime` is
+ * hand-set. The mock cannot catch the one thing that would silently break the fix:
+ * `.runtime` is a `private readonly` field on the real ModelRegistry, absent from
+ * the public type AND the package exports. If a future Pi renames it, makes it a
+ * true #private, or moves the module, the cast quietly yields `undefined`, the fix
+ * omits `modelRuntime`, and the bug returns with no failing test.
+ *
+ * This test closes exactly that gap and nothing else: it asserts the real facade
+ * exposes a runtime-reachable `.runtime` that IS the runtime it wraps. It is not a
+ * guard for the forwarding itself (that's the unit test's job) — a fuller e2e that
+ * drives real `runAgent` end-to-end is tracked as a follow-up.
+ *
+ * VERSION GATE: `.runtime` only exists in the post-migration facade world (Pi >=
+ * 0.80.8, where `ModelRuntime` is first exported). The repo's dev dependency is
+ * pinned post-migration, so this runs by default — but it stays gated because CI
+ * also runs the suite against the peer-range floor (see .github/workflows/ci.yml),
+ * where Pi predates the migration. There we DYNAMICALLY import Pi and skip
+ * cleanly; a static `import { ModelRuntime }` would be a link-time error.
+ */
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, describe, expect, it } from "vitest";
+
+// Dynamic, so this file LOADS on pre-migration Pi (a static `import { ModelRuntime }`
+// would be a link-time error there — 0.80.6 doesn't export it).
+const pi = (await import("@earendil-works/pi-coding-agent")) as Record<string, unknown>;
+const ModelRuntime = pi.ModelRuntime as
+  | { create(opts?: Record<string, unknown>): Promise<ModelRuntimeLike> }
+  | undefined;
+
+// The migration is exactly "ModelRuntime now exists". Absent ⇒ pre-0.80.8 ⇒ there
+// is no `.runtime` facade to guard.
+const MIGRATED = typeof ModelRuntime?.create === "function";
+const RT = ModelRuntime as { create(opts?: Record<string, unknown>): Promise<ModelRuntimeLike> };
+
+// The one method the reach scenario needs; `.runtime` itself is private (reached below).
+interface ModelRuntimeLike {
+  registerProvider(id: string, config: Record<string, unknown>): void;
+}
+
+const tmpDirs: string[] = [];
+afterAll(() => {
+  for (const d of tmpDirs.splice(0)) rmSync(d, { recursive: true, force: true });
+});
+
+describe.skipIf(!MIGRATED)("PR #152 reach: real ModelRegistry exposes .runtime (Pi >= 0.80.8)", () => {
+  it("ctx.modelRegistry.runtime is reachable and IS the runtime it wraps", async () => {
+    // A real, configured runtime — as an extension leaves it after registerProvider.
+    const dir = mkdtempSync(join(tmpdir(), "iso-prov-"));
+    tmpDirs.push(dir);
+    const runtime = await RT.create({
+      authPath: join(dir, "auth.json"),
+      modelsPath: join(dir, "models.json"),
+      allowModelNetwork: false,
+    });
+
+    // `.runtime` is private and not in the package exports — reach the compiled
+    // class by file path, exactly the field the patch's cast depends on. If Pi
+    // moves/renames/#privates it, THIS line fails loudly instead of the fix
+    // silently no-op'ing back to the #151 bug.
+    const indexUrl = import.meta.resolve("@earendil-works/pi-coding-agent");
+    const mrUrl = indexUrl.replace(/index\.js$/, "core/model-registry.js");
+    const { ModelRegistry } = (await import(mrUrl)) as {
+      ModelRegistry: new (rt: ModelRuntimeLike) => { runtime?: unknown };
+    };
+
+    const facade = new ModelRegistry(runtime);
+    // This is the exact expression agent-runner reads (`ctx.modelRegistry.runtime`).
+    expect((facade as { runtime?: unknown }).runtime).toBe(runtime);
+  });
+});
@@ -0,0 +1,97 @@
+/**
+ * mention-clone-tool-reachability.e2e.test.ts — reachability guard for the one
+ * tool the mention clone is built around.
+ *
+ * `runMentionClone` hands a session ONE tool and expects the model to call it.
+ * Whether that tool ever reaches the model is decided entirely inside Pi, by
+ * `createAgentSession`'s allowlist plumbing — and the unit tests cannot see it:
+ * their `createAgentSession` is a mock that hands `customTools[0]` straight to
+ * the model turn, so a session option that silently strips the tool passes
+ * every one of them.
+ *
+ * That is not hypothetical. The clone shipped with `noTools: "all"` on the
+ * reading its doc comment invites ("start with no tools enabled" — no
+ * built-ins, keep mine). Pi turns that flag into an EMPTY allowlist, and an
+ * empty array is truthy, so `AgentSession` builds an empty `Set` and
+ * `isAllowedTool` rejects every name — custom tools are filtered by the same
+ * predicate as built-ins. Every mention was prompted with no tools, answered in
+ * prose, and fell back to a direct start with a warning. The unit suite stayed
+ * green throughout.
+ *
+ * So this asserts against a REAL session, on the two things a mock cannot
+ * establish:
+ *   1. the clone's `Agent` tool is actually active on it, and
+ *   2. nothing else is — the invisible turn cannot read, write or run anything.
+ *
+ * No network/LLM: a faux provider satisfies session construction, and the
+ * assertion is on the constructed tool set rather than on a model turn.
+ */
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+// Real pi-mono session construction; a cold first run under full-suite CPU
+// contention can exceed vitest's 5s default.
+vi.setConfig({ testTimeout: 30_000 });
+
+// Hoisted so the (lifted) mock factory can reach it. Everything except the
+// capture is the real module — the point is to construct a REAL session.
+const { sessions } = vi.hoisted(() => ({ sessions: [] as any[] }));
+
+vi.mock("@earendil-works/pi-coding-agent", async () => {
+  const actual = await vi.importActual<any>("@earendil-works/pi-coding-agent");
+  return {
+    ...actual,
+    createAgentSession: async (opts: any) => {
+      const created = await actual.createAgentSession(opts);
+      sessions.push(created.session);
+      return created;
+    },
+  };
+});
+
+import { runMentionClone } from "../../src/mention-clone.js";
+import { fauxModelBackend } from "../helpers/faux-model-backend.js";
+import { registerFauxProvider } from "../helpers/pi-ai.js";
+
+describe("mention clone tool reachability against real pi-mono", () => {
+  let cwd: string;
+  let faux: ReturnType<typeof registerFauxProvider>;
+
+  beforeEach(() => {
+    sessions.length = 0;
+    cwd = mkdtempSync(join(tmpdir(), "subagents-mention-clone-"));
+    faux = registerFauxProvider({ provider: "faux", models: [{ id: "faux-1", contextWindow: 200_000 }] });
+  });
+  afterEach(() => {
+    faux.unregister();
+    rmSync(cwd, { recursive: true, force: true });
+  });
+
+  it("the clone's Agent tool is live on the real session, and it is the only one", async () => {
+    const model = faux.getModel();
+    const backend = fauxModelBackend(model);
+    const ctx: any = {
+      cwd,
+      model,
+      getSystemPrompt: () => "PARENT",
+      // mention-clone reads the runtime off the registry facade, the same shim
+      // agent-runner carries for Pi >= 0.80.8.
+      modelRegistry: { ...backend.modelRegistry, runtime: backend.modelRuntime },
+      sessionManager: { getEntries: () => [], getLeafId: () => undefined },
+    };
+
+    // Never called: the assertion is on what the session exposes, not on the
+    // faux model deciding to use it.
+    const agentTool = { name: "Agent", execute: vi.fn() } as any;
+
+    // Never rejects by contract; a faux turn that cannot complete is fine,
+    // because the tool set is fixed at construction.
+    await runMentionClone({ ctx, type: "Explore", message: "go", agentTool });
+
+    expect(sessions).toHaveLength(1);
+    // The bug this file exists for: with an empty allowlist this is `[]`.
+    expect(sessions[0].getActiveToolNames()).toEqual(["Agent"]);
+  });
+});
@@ -0,0 +1,145 @@
+/**
+ * tool-veto-reachability.e2e.test.ts — reachability guard for the `ext:` turn-1
+ * tool veto (issue #125).
+ *
+ * `installExtensionToolScope` enforces `ext:` narrowing two ways. Re-narrowing the
+ * ACTIVE set on `turn_end` is built entirely on public API (`getAllTools`,
+ * `getActiveToolNames`, `setActiveToolsByName`) and is covered by the unit tests.
+ * The second half is not: turn 1 cannot be narrowed at all — `before_agent_start`
+ * fires INSIDE `prompt()` and may widen the tool set, but `createContextSnapshot()`
+ * freezes that turn's tools immediately after, leaving no window — so out-of-scope
+ * calls are vetoed at call time by wrapping `session.agent.beforeToolCall`.
+ *
+ * That wrap is the one place this extension reaches past the documented surface:
+ *   - `ExtensionBindings` has no tool_call hook, so there is no SDK-level way to
+ *     inject a veto into a session we construct. Pi exposes the veto to EXTENSIONS
+ *     as `pi.on("tool_call") -> { block, reason }`, but we are the SDK caller here,
+ *     not an extension bound to the child session.
+ *   - So we wrap the property Pi itself installs in the AgentSession constructor
+ *     (`_installAgentToolHooks`), chaining to the prior hook so Pi's own `tool_call`
+ *     dispatch still runs.
+ *
+ * The unit tests assert our wrapper's behavior against a MOCK session whose `agent`
+ * is a hand-written `{ beforeToolCall: undefined }`. That mock cannot catch the one
+ * thing that would silently break the veto: if a future Pi renames `beforeToolCall`,
+ * stops installing it, makes `agent` non-enumerable/private, or moves the veto
+ * elsewhere, our assignment lands on a property nothing reads. Every test still
+ * passes, and out-of-scope tools become callable on turn 1 with no failing test.
+ *
+ * This guard closes exactly that gap and nothing else. It asserts against a REAL
+ * session that:
+ *   1. Pi installs its own `beforeToolCall` (so there IS a prior hook to chain), and
+ *   2. after `runAgent`, ours is installed and vetoes an out-of-scope tool in the
+ *      `{ block, reason }` shape Pi honors.
+ *
+ * No network/LLM: a faux Model satisfies `createAgentSession`, and the veto is
+ * invoked directly rather than through a model turn.
+ */
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { runAgent } from "../../src/agent-runner.js";
+import { registerAgents } from "../../src/agent-types.js";
+import type { AgentConfig } from "../../src/types.js";
+import { registerFauxProvider } from "../helpers/pi-ai.js";
+
+// Real pi-mono (loader + dynamic extension import + session construction).
+vi.setConfig({ testTimeout: 30_000 });
+
+/** Registers `alpha_read` / `alpha_write`; reused so no new fixture is needed. */
+const ALPHA = resolve(fileURLToPath(new URL("../fixtures/ext-alpha.mjs", import.meta.url)));
+/** Registers `beta_tool` — loaded but NOT selected by the `ext:` selector below. */
+const BETA = resolve(fileURLToPath(new URL("../fixtures/ext-beta.mjs", import.meta.url)));
+
+function makePi() {
+  return { exec: async () => ({ code: 1, stdout: "", stderr: "" }) } as any;
+}
+
+describe("tool veto reachability against real pi-mono", () => {
+  let cwd: string;
+  let faux: ReturnType<typeof registerFauxProvider>;
+
+  beforeEach(() => {
+    cwd = mkdtempSync(join(tmpdir(), "subagents-veto-"));
+    faux = registerFauxProvider({
+      provider: "faux",
+      models: [{ id: "faux-1", contextWindow: 200_000 }],
+    });
+  });
+  afterEach(() => {
+    faux.unregister();
+    rmSync(cwd, { recursive: true, force: true });
+  });
+
+  it("pi installs a chainable beforeToolCall, and runAgent's veto blocks out-of-scope tools", async () => {
+    registerAgents(
+      new Map([
+        [
+          "veto",
+          {
+            name: "veto",
+            description: "veto guard",
+            builtinToolNames: ["read"],
+            // Select alpha only — beta loads (its handlers run) but is muted.
+            extensions: [ALPHA, BETA],
+            extSelectors: ["ext:ext-alpha.mjs"],
+            skills: false,
+            systemPrompt: "You are veto.",
+            promptMode: "replace",
+            inheritContext: false,
+            runInBackground: false,
+            isolated: false,
+          } as AgentConfig,
+        ],
+      ]),
+    );
+
+    const model = faux.getModel();
+    const modelRegistry: any = {
+      find: () => model,
+      getAll: () => [model],
+      getAvailable: () => [model],
+      hasConfiguredAuth: () => true,
+      isUsingOAuth: () => false,
+      getApiKeyAndHeaders: async () => ({ apiKey: "faux", headers: {} }),
+      registerProvider: () => {},
+      unregisterProvider: () => {},
+    };
+    const ctx: any = { cwd, getSystemPrompt: () => "PARENT", model, modelRegistry };
+
+    let priorIsFunction: boolean | undefined;
+    let session: any;
+    try {
+      await runAgent(ctx, "veto", "go", {
+        pi: makePi(),
+        model,
+        onSessionCreated: (s: any) => {
+          session = s;
+          // By onSessionCreated our wrapper is already installed, so this being a
+          // function proves the property is reachable and writable. Pi installing
+          // its own in the constructor is what gives us something to chain to —
+          // asserted below via the in-scope path returning undefined rather than
+          // throwing on a missing prior hook.
+          priorIsFunction = typeof s.agent?.beforeToolCall === "function";
+        },
+      });
+    } catch {
+      // A faux-model turn may not complete; the veto is fixed at construction.
+    }
+
+    expect(priorIsFunction).toBe(true);
+
+    // Out of scope: beta loaded but the ext: flip did not select it.
+    await expect(
+      session.agent.beforeToolCall({ toolCall: { name: "beta_tool" }, args: {} }),
+    ).resolves.toMatchObject({ block: true, reason: expect.any(String) });
+
+    // In scope: must NOT be blocked. Reaching Pi's own prior hook without throwing
+    // also proves the chain is intact (a clobbered/absent prior would surface here).
+    await expect(
+      session.agent.beforeToolCall({ toolCall: { name: "alpha_read" }, args: {} }),
+    ).resolves.toSatisfy((r: any) => !r?.block);
+  });
+});
@@ -0,0 +1,148 @@
+/**
+ * usage-reaches-session-stats.e2e.test.ts — the premise #193 rests on, checked
+ * against the REAL pi runtime.
+ *
+ * Every unit test for usage reporting asserts that our tool results carry a
+ * `usage` field. None of them can establish the thing that makes carrying it
+ * worth doing: that pi picks it up. That happens entirely inside pi —
+ * `createToolResultMessage` copies `AgentToolResult.usage` onto the persisted
+ * message, and `getSessionStats()` folds `toolResult.usage` into the tokens and
+ * cost the footer, the statusline and `/cost` read. Mock pi, and a release that
+ * stopped doing either would leave the whole feature reporting into a void with
+ * a green suite.
+ *
+ * So this drives a real `AgentSession` and reads its real `getSessionStats()`,
+ * with the exact object `PendingUsagePool.drain()` produces — including the
+ * `cacheRead` our own display total drops but this report must carry, and the
+ * cost breakdown whose `total` pi reads with no guard at all.
+ *
+ * No network/LLM and no model turn: the message is appended through pi's own
+ * `sessionManager.appendMessage`, because what is under test is the accounting,
+ * not the streaming that would normally produce the message.
+ *
+ * This test is also what set the peer floor. Pi began folding `toolResult.usage`
+ * into `getSessionStats()` in 0.81.0, when the computation moved to walking
+ * session entries through `addUsageToTotals`; every 0.80.x sums assistant
+ * messages alone and drops the field. Running unconditionally is the point —
+ * against a Pi that does not aggregate, this fails rather than skipping, which
+ * is how the range stays honest. `peerDependencies` moved to `>=0.81.0` for
+ * exactly this reason, so the CI floor job runs it too.
+ */
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { createAgentSession, SessionManager } from "@earendil-works/pi-coding-agent";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { PendingUsagePool } from "../../src/usage.js";
+import { fauxModelBackend } from "../helpers/faux-model-backend.js";
+import { registerFauxProvider } from "../helpers/pi-ai.js";
+
+// Real pi session construction; a cold first run under full-suite CPU
+// contention can exceed vitest's 5s default.
+vi.setConfig({ testTimeout: 30_000 });
+
+describe("subagent usage reaches the parent session's stats (real pi)", () => {
+  let cwd: string;
+  let faux: ReturnType<typeof registerFauxProvider>;
+
+  beforeEach(() => {
+    cwd = mkdtempSync(join(tmpdir(), "subagents-usage-e2e-"));
+    faux = registerFauxProvider({ provider: "faux", models: [{ id: "faux-1", contextWindow: 200_000 }] });
+  });
+  afterEach(() => {
+    faux.unregister();
+    rmSync(cwd, { recursive: true, force: true });
+  });
+
+  /** A real session, in memory, on a faux model. */
+  async function realSession() {
+    const model = faux.getModel();
+    const backend = fauxModelBackend(model);
+    const { session } = await createAgentSession({
+      cwd,
+      sessionManager: SessionManager.inMemory(cwd),
+      model: model as any,
+      modelRegistry: backend.modelRegistry,
+      modelRuntime: backend.modelRuntime,
+      tools: [],
+    } as any);
+    return session;
+  }
+
+  /** The tool result our `Agent` tool returns, as pi would persist it. */
+  function toolResultCarrying(usage: unknown) {
+    return {
+      role: "toolResult" as const,
+      toolCallId: "tc-1",
+      toolName: "Agent",
+      content: [{ type: "text" as const, text: "Agent completed." }],
+      isError: false,
+      timestamp: 1,
+      usage,
+    };
+  }
+
+  it("pi adds our reported tokens and cost to getSessionStats()", async () => {
+    const session = await realSession();
+    try {
+      const before = session.getSessionStats();
+
+      const pool = new PendingUsagePool();
+      pool.add({ input: 1000, output: 400, cacheWrite: 100, cacheRead: 9000, cost: 0.0123 });
+      pool.add({ input: 2000, output: 600, cacheWrite: 200, cacheRead: 18_000, cost: 0.0077 });
+      const usage = pool.drain();
+
+      session.sessionManager.appendMessage(toolResultCarrying(usage) as any);
+      const after = session.getSessionStats();
+
+      // Exactly what we reported, on every component pi tracks — cacheRead
+      // included, which is the one pi counts for its own messages and our own
+      // display total leaves out.
+      expect(after.tokens.input - before.tokens.input).toBe(3000);
+      expect(after.tokens.output - before.tokens.output).toBe(1000);
+      expect(after.tokens.cacheWrite - before.tokens.cacheWrite).toBe(300);
+      expect(after.tokens.cacheRead - before.tokens.cacheRead).toBe(27_000);
+
+      // The cost: the whole point of the feature for anyone watching a
+      // statusline. `addUsageToTotals` reads `usage.cost.total` with no guard,
+      // so an incomplete object would have thrown before reaching here.
+      expect(after.cost - before.cost).toBeCloseTo(0.02, 10);
+    } finally {
+      session.dispose?.();
+    }
+  });
+
+  it("leaves the context-window percentage alone", async () => {
+    // pi derives context usage from assistant messages only. If that ever
+    // changed, a delegating session would look like it was filling its context
+    // with work that happened somewhere else entirely — and users would compact
+    // for no reason.
+    const session = await realSession();
+    try {
+      const before = session.getSessionStats().contextUsage?.percent ?? null;
+
+      const pool = new PendingUsagePool();
+      pool.add({ input: 150_000, output: 400, cacheWrite: 100, cost: 1.5 });
+      session.sessionManager.appendMessage(toolResultCarrying(pool.drain()) as any);
+
+      expect(session.getSessionStats().contextUsage?.percent ?? null).toBe(before);
+    } finally {
+      session.dispose?.();
+    }
+  });
+
+  it("counts nothing for a tool result that carries no usage", async () => {
+    // The `reportUsage: false` shape, and every other tool in the session.
+    const session = await realSession();
+    try {
+      const before = session.getSessionStats();
+      session.sessionManager.appendMessage(toolResultCarrying(undefined) as any);
+      const after = session.getSessionStats();
+
+      expect(after.tokens.input).toBe(before.tokens.input);
+      expect(after.cost).toBe(before.cost);
+    } finally {
+      session.dispose?.();
+    }
+  });
+});