my-pi/pi-permission-system/test/auto-review-runtime.test.ts

import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";

import type { ExtensionAPI, ModelRegistry, SessionManager } from "@earendil-works/pi-coding-agent";
import {
  DEFAULT_AUTO_REVIEW_ANALYSIS_CONFIG,
  DEFAULT_AUTO_REVIEW_CONFIG,
} from "#src/auto-review/config";
import { AutoReviewLedger } from "#src/auto-review/ledger";
import { installBuiltinAutoReview } from "#src/auto-review/runtime";
import { getAnalysisStateDir } from "#src/config-paths";
import type { ConfigReader } from "#src/config-store";
import type { PermissionsService, PromptPermissionDetails } from "#src/service";
import type { ReviewLogger } from "#src/session-logger";

type AnalysisRunner = NonNullable<
  Parameters<typeof installBuiltinAutoReview>[0]["runAnalysis"]
>;

const roots: string[] = [];

afterEach(() => {
  for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
});

function seedPending(agentDir: string) {
  const analysis = { ...DEFAULT_AUTO_REVIEW_ANALYSIS_CONFIG, threshold: 1 };
  const ledger = new AutoReviewLedger(getAnalysisStateDir(agentDir));
  ledger.recordDecision({
    details: {
      requestId: "request-1",
      source: "tool",
      agentName: "main",
      surface: "bash",
      toolName: "bash",
      command: "npm install example",
    } as unknown as PromptPermissionDetails,
    assessment: {
      outcome: "allow",
      riskLevel: "low",
      userAuthorization: "high",
      rationale: "directly authorized",
    },
    policyRevision: "policy-a",
    config: DEFAULT_AUTO_REVIEW_CONFIG,
    analysis,
  });
  return { analysis, ledger };
}

function harness(agentDir: string, analysis: typeof DEFAULT_AUTO_REVIEW_ANALYSIS_CONFIG, runAnalysis: Parameters<typeof installBuiltinAutoReview>[0]["runAnalysis"]) {
  const sendMessage = vi.fn();
  const registerAuthorizer = vi.fn(() => () => {});
  const review = vi.fn();
  const controller = installBuiltinAutoReview({
    pi: { sendMessage } as unknown as ExtensionAPI,
    service: { registerAuthorizer } as unknown as PermissionsService,
    config: {
      current: () => ({
        debugLog: false,
        permissionReviewLog: true,
        yoloMode: false,
        doublePressToConfirm: true,
        autoReview: DEFAULT_AUTO_REVIEW_CONFIG,
        analysis,
      }),
    } as ConfigReader,
    logger: { review } as unknown as ReviewLogger,
    agentDir,
    runAnalysis,
  });
  controller.handleSessionStart({
    modelRegistry: {} as ModelRegistry,
    sessionManager: { getBranch: () => [] } as unknown as Pick<SessionManager, "getBranch">,
  });
  return { controller, sendMessage, registerAuthorizer, review };
}

describe("built-in AutoReview lifecycle", () => {
  it("analyzes a persisted pending checkpoint at agent_settled and follows up in the same session", async () => {
    const agentDir = mkdtempSync(join(tmpdir(), "auto-review-runtime-"));
    roots.push(agentDir);
    const { analysis } = seedPending(agentDir);
    const runAnalysis = vi.fn<AnalysisRunner>(async () => ({
      summary: "No unconditional policy changes recommended.",
      allowCandidates: [],
      denyCandidates: [],
      keepReview: [],
      hardGuardCandidates: [],
    }));
    const { controller, sendMessage, registerAuthorizer } = harness(agentDir, analysis, runAnalysis);

    await controller.handleAgentSettled();

    expect(registerAuthorizer).toHaveBeenCalledWith("auto-review", expect.any(Function));
    expect(runAnalysis).toHaveBeenCalledTimes(1);
    expect(runAnalysis.mock.calls[0]?.[0].batch).toMatchObject({ fromSequence: 1, toSequence: 1 });
    expect(sendMessage).toHaveBeenCalledWith(
      expect.objectContaining({ customType: "permission-policy-analysis-ready", display: true }),
      { deliverAs: "followUp", triggerTurn: true },
    );
    expect(new AutoReviewLedger(getAnalysisStateDir(agentDir)).hasPendingAnalysis()).toBe(false);

    await controller.handleAgentSettled();
    expect(runAnalysis).toHaveBeenCalledTimes(1);
  });

  it("keeps the checkpoint pending when background analysis fails", async () => {
    const agentDir = mkdtempSync(join(tmpdir(), "auto-review-runtime-"));
    roots.push(agentDir);
    const { analysis } = seedPending(agentDir);
    const runAnalysis = vi.fn<AnalysisRunner>(async () => {
      throw new Error("analysis unavailable");
    });
    const { controller, sendMessage, review } = harness(agentDir, analysis, runAnalysis);

    await controller.handleAgentSettled();

    expect(sendMessage).not.toHaveBeenCalled();
    expect(review).toHaveBeenCalledWith("auto_review.analysis_failed", { error: "analysis unavailable" });
    expect(new AutoReviewLedger(getAnalysisStateDir(agentDir)).hasPendingAnalysis()).toBe(true);
  });

  it("discards a stale result after session replacement without advancing the checkpoint", async () => {
    const agentDir = mkdtempSync(join(tmpdir(), "auto-review-runtime-"));
    roots.push(agentDir);
    const { analysis } = seedPending(agentDir);
    let resolveAnalysis!: (proposal: Awaited<ReturnType<AnalysisRunner>>) => void;
    const runAnalysis = vi.fn<AnalysisRunner>(() => new Promise((resolve) => {
      resolveAnalysis = resolve;
    }));
    const { controller, sendMessage, review } = harness(agentDir, analysis, runAnalysis);

    const pending = controller.handleAgentSettled();
    expect(runAnalysis).toHaveBeenCalledTimes(1);
    controller.handleSessionShutdown();
    controller.handleSessionStart({
      modelRegistry: {} as ModelRegistry,
      sessionManager: { getBranch: () => [] } as unknown as Pick<SessionManager, "getBranch">,
    });
    resolveAnalysis({
      summary: "stale proposal",
      allowCandidates: [],
      denyCandidates: [],
      keepReview: [],
      hardGuardCandidates: [],
    });
    await pending;

    expect(sendMessage).not.toHaveBeenCalled();
    expect(review).not.toHaveBeenCalledWith("auto_review.analysis_completed", expect.anything());
    expect(new AutoReviewLedger(getAnalysisStateDir(agentDir)).hasPendingAnalysis()).toBe(true);
  });

  it("does not advance the checkpoint when same-session delivery fails", async () => {
    const agentDir = mkdtempSync(join(tmpdir(), "auto-review-runtime-"));
    roots.push(agentDir);
    const { analysis } = seedPending(agentDir);
    const runAnalysis = vi.fn<AnalysisRunner>(async () => ({
      summary: "proposal",
      allowCandidates: [],
      denyCandidates: [],
      keepReview: [],
      hardGuardCandidates: [],
    }));
    const { controller, sendMessage, review } = harness(agentDir, analysis, runAnalysis);
    sendMessage.mockImplementation(() => {
      throw new Error("session delivery unavailable");
    });

    await controller.handleAgentSettled();

    expect(review).toHaveBeenCalledWith("auto_review.analysis_failed", {
      error: "session delivery unavailable",
    });
    expect(new AutoReviewLedger(getAnalysisStateDir(agentDir)).hasPendingAnalysis()).toBe(true);
  });
});