import type { ReviewModelRegistry } from '../src/model.js' import type { Api, AssistantMessage, AssistantMessageEventStream, Model, Provider, SimpleStreamOptions, } from '@earendil-works/pi-ai' import type { SessionEntry } from '@earendil-works/pi-coding-agent' import type { AuthorizerLog, PermissionQuery, PromptPermissionDetails } from '@gotgenes/pi-permission-system' import { afterEach, describe, expect, it, vi } from 'vitest' import { DenialCircuitBreaker } from '../src/circuit-breaker.js' import { autoReviewConfigSchema } from '../src/config.js' import { createPermissionReviewer } from '../src/reviewer.js' function createModel(): Model { return { id: 'review-model', name: 'Review Model', api: 'openai-responses', provider: 'custom-review', baseUrl: 'https://review.example/v1', reasoning: true, input: ['text'], cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, }, contextWindow: 128_000, maxTokens: 16_000, } } function assistantMessage(text: string, stopReason: AssistantMessage['stopReason'] = 'stop'): AssistantMessage { return { role: 'assistant', content: [{ type: 'text', text }], api: 'openai-responses', provider: 'custom-review', model: 'review-model', usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0, }, }, stopReason, timestamp: 0, } } function streamFrom(result: () => Promise): AssistantMessageEventStream { return { result } as AssistantMessageEventStream } function userEntry(): SessionEntry { return { type: 'message', id: 'user-1', parentId: null, timestamp: '2026-07-23T00:00:00.000Z', message: { role: 'user', content: 'Please run the requested operation.', timestamp: 0, }, } } function userInteractionEntries(): SessionEntry[] { return [ { type: 'message', id: 'interaction-call', parentId: 'user-1', timestamp: '2026-07-23T00:00:30.000Z', message: { role: 'assistant', content: [ { type: 'toolCall', id: 'question-1', name: 'ask_user_question', arguments: { questions: [] }, }, ], api: 'openai-responses', provider: 'test', model: 'test', usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, }, stopReason: 'toolUse', timestamp: 0, }, }, { type: 'message', id: 'interaction-1', parentId: 'interaction-call', timestamp: '2026-07-23T00:01:00.000Z', message: { role: 'toolResult', toolCallId: 'question-1', toolName: 'ask_user_question', content: [{ type: 'text', text: 'untrusted presentation text' }], details: { cancelled: false, answers: [{ question: 'Choose a mode?', answer: 'Safe mode' }], }, isError: false, timestamp: 0, }, }, ] } function details(overrides: Partial = {}): PromptPermissionDetails { const surface = overrides.surface ?? 'bash' const value = overrides.value ?? overrides.command ?? overrides.path ?? 'pnpm publish' return { requestId: 'request-1', source: 'tool_call', agentName: null, toolName: 'bash', command: 'pnpm publish', surface, payload: { kind: surface === 'bash' ? 'bash' : surface === 'path' ? 'path' : 'tool', request: { requester: { agentName: null, forwarded: false, sessionId: null }, surface, toolName: overrides.toolName ?? 'bash', invokedToolName: null, value, matchedPattern: '*', commandContext: null, executedUnit: null, }, evidence: [], annotations: [], }, ...overrides, } } interface TestLog extends AuthorizerLog { review: ReturnType> debug: ReturnType> } function createLog(): TestLog { return { review: vi.fn(), debug: vi.fn(), } } interface HarnessOptions { responses?: Array auth?: Awaited> timeoutMs?: number resultFactory?: (options: SimpleStreamOptions) => Promise providerLookup?: 'native' | 'missing' | 'throwing' sessionEntries?: SessionEntry[] } function createHarness(options: HarnessOptions = {}) { const model = createModel() const responses = [...(options.responses ?? [assistantMessage('{"outcome":"allow"}')])] const streamSimple = vi.fn((_model: Model, _context: unknown, streamOptions: SimpleStreamOptions = {}) => streamFrom(async () => { if (options.resultFactory !== undefined) { return options.resultFactory(streamOptions) } const next = responses.shift() if (next instanceof Error) { throw next } if (next === undefined) { throw new Error('no fake response') } return next }), ) const provider = { id: 'custom-review', name: 'Custom Review', auth: {}, getModels: () => [model], stream: streamSimple, streamSimple, } as unknown as Provider const getApiKeyAndHeaders = vi.fn(async () => Promise.resolve( options.auth ?? { ok: true as const, apiKey: 'secret-key', headers: { 'x-review': 'enabled' }, env: { REVIEW_REGION: 'test' }, }, ), ) const registryBase = { find: vi.fn(() => model), getAll: vi.fn(() => [model]), getApiKeyAndHeaders, } const providerLookup = vi.fn(() => provider) let registry: ReviewModelRegistry switch (options.providerLookup ?? 'native') { case 'native': registry = { ...registryBase, getProvider: providerLookup } break case 'missing': registry = { ...registryBase, getProvider: vi.fn(() => undefined) } break case 'throwing': registry = { ...registryBase, getProvider: () => { throw new Error('provider lookup failed') }, } break } const circuitBreaker = new DenialCircuitBreaker() const getBranch = vi.fn(() => options.sessionEntries ?? [userEntry()]) const authorize = createPermissionReviewer( { config: autoReviewConfigSchema.parse({ provider: 'custom-review', model: 'review-model', timeoutMs: options.timeoutMs ?? 90_000, }), registry, sessionManager: { getBranch }, circuitBreaker, }, { now: () => 0, retryDelaysMs: [0, 0], sleep: async () => Promise.resolve(), }, ) return { authorize, circuitBreaker, getApiKeyAndHeaders, getBranch, registry, streamSimple, } } const query = {} as PermissionQuery describe('permission reviewer', () => { afterEach(() => { vi.useRealTimers() }) it('passes Pi-managed auth to a tool-free provider call and allows', async () => { const harness = createHarness() const log = createLog() await expect(harness.authorize(details(), query, log)).resolves.toEqual({ kind: 'allow', }) expect(harness.getApiKeyAndHeaders.mock.calls).toHaveLength(1) const [, context, options] = harness.streamSimple.mock.calls[0] ?? [] expect(context).toMatchObject({ messages: [{ role: 'user' }], }) expect(context).not.toHaveProperty('tools') expect((context as { systemPrompt?: string }).systemPrompt).toContain( 'source field is "user" or "user_interaction"', ) expect(harness.getBranch).toHaveBeenCalledOnce() expect(options).toMatchObject({ apiKey: 'secret-key', headers: { 'x-review': 'enabled' }, env: { REVIEW_REGION: 'test' }, maxRetries: 0, maxTokens: 1_000, reasoning: 'low', }) expect(log.review.mock.calls[0]?.[1]).toMatchObject({ policyRevision: 'openai-codex/c4f42d161ae44a8d696ee9fb595709661979d187+pi1', contextSource: 'active-branch', transcriptEntriesRetained: 1, transcriptEntriesOmitted: 0, transcriptEntriesTruncated: 0, directUserEntriesRetained: 1, directUserEntriesOmitted: 0, directUserEntriesTruncated: 0, userInteractionEntriesRetained: 0, userInteractionEntriesOmitted: 0, userInteractionEntriesTruncated: 0, latestTrustedEntryRetained: true, }) }) it('sends canonical structured user interactions to the provider', async () => { const harness = createHarness({ sessionEntries: [userEntry(), ...userInteractionEntries()], }) await expect(harness.authorize(details(), query, createLog())).resolves.toEqual({ kind: 'allow' }) const [, context] = harness.streamSimple.mock.calls[0] ?? [] const userPrompt = ( context as { messages: Array<{ content: string }> } ).messages[0]?.content expect(userPrompt).toContain('"source":"user_interaction"') expect(userPrompt).toContain('[{\\"question\\":\\"Choose a mode?\\",\\"answer\\":\\"Safe mode\\"}]') expect(userPrompt).not.toContain('untrusted presentation text') }) it('returns a teaching denial without persisting the rationale', async () => { const harness = createHarness({ responses: [ assistantMessage( '{"risk_level":"high","user_authorization":"unknown","outcome":"deny","rationale":"Publishing was not authorized."}', ), ], }) const log = createLog() const result = await harness.authorize(details({ surface: 'path', path: '.env' }), query, log) expect(result.kind).toBe('deny') if (result.kind === 'deny') { expect(result.reason).toContain('Publishing was not authorized.') } expect(log.review.mock.calls[0]?.[0]).toBe('auto_review.decision') expect(log.review.mock.calls[0]?.[1]).toMatchObject({ outcome: 'deny', riskLevel: 'high', userAuthorization: 'unknown', }) expect(log.review.mock.calls[0]?.[1]).not.toHaveProperty('rationale') expect(log.review.mock.calls[0]?.[1]).not.toHaveProperty('surface') }) it('retries transient provider failures within the same review', async () => { const harness = createHarness({ responses: [ new Error('temporary failure'), assistantMessage('', 'error'), assistantMessage('{"outcome":"allow"}'), ], }) await expect(harness.authorize(details(), query, createLog())).resolves.toEqual({ kind: 'allow' }) expect(harness.streamSimple.mock.calls).toHaveLength(3) }) it('defers malformed output and missing auth to the human authorizer', async () => { const malformed = createHarness({ responses: [assistantMessage('not json')], }) const malformedLog = createLog() await expect(malformed.authorize(details(), query, malformedLog)).resolves.toEqual({ kind: 'defer' }) expect(malformedLog.review.mock.calls[0]?.[0]).toBe('auto_review.decision') expect(malformedLog.review.mock.calls[0]?.[1]).toMatchObject({ errorCategory: 'invalid-response', }) const missingAuth = createHarness({ auth: { ok: false, error: 'not configured' }, }) await expect(missingAuth.authorize(details(), query, createLog())).resolves.toEqual({ kind: 'defer' }) expect(missingAuth.streamSimple.mock.calls).toHaveLength(0) }) it('contains unsupported and throwing provider lookup failures', async () => { const missing = createHarness({ providerLookup: 'missing' }) const missingLog = createLog() await expect(missing.authorize(details(), query, missingLog)).resolves.toEqual({ kind: 'defer' }) expect(missingLog.review.mock.calls[0]?.[1]).toMatchObject({ errorCategory: 'provider-unresolved', }) const throwing = createHarness({ providerLookup: 'throwing' }) const throwingLog = createLog() await expect(throwing.authorize(details(), query, throwingLog)).resolves.toEqual({ kind: 'defer' }) expect(throwingLog.review.mock.calls[0]?.[1]).toMatchObject({ errorCategory: 'internal-error', }) }) it('defers when review logging throws', async () => { const harness = createHarness() const log = createLog() log.review.mockImplementation(() => { throw new Error('log unavailable') }) await expect(harness.authorize(details(), query, log)).resolves.toEqual({ kind: 'defer' }) }) it('opens the per-turn circuit after three consecutive denials', async () => { const denial = assistantMessage('{"outcome":"deny","rationale":"Not authorized."}') const harness = createHarness({ responses: [denial, denial, denial], }) for (let index = 0; index < 3; index += 1) { await expect( harness.authorize(details({ requestId: `request-${index}` }), query, createLog()), ).resolves.toMatchObject({ kind: 'deny' }) } const circuitLog = createLog() const circuitResult = await harness.authorize(details({ requestId: 'request-4' }), query, circuitLog) expect(circuitResult.kind).toBe('deny') if (circuitResult.kind === 'deny') { expect(circuitResult.reason).toContain('explicit approval') } expect(harness.streamSimple.mock.calls).toHaveLength(3) expect(circuitLog.review.mock.calls[0]?.[0]).toBe('auto_review.circuit_open') }) it('opens the per-turn circuit after ten non-consecutive denials in the recent window', async () => { const denial = assistantMessage('{"outcome":"deny","rationale":"Not authorized."}') const allow = assistantMessage('{"outcome":"allow"}') const responses = Array.from({ length: 10 }, () => [denial, allow]).flat() const harness = createHarness({ responses }) for (let index = 0; index < 19; index += 1) { await harness.authorize(details({ requestId: `request-${index}` }), query, createLog()) } await expect( harness.authorize(details({ requestId: 'request-circuit' }), query, createLog()), ).resolves.toMatchObject({ kind: 'deny' }) expect(harness.streamSimple.mock.calls).toHaveLength(19) harness.circuitBreaker.resetTurn() await expect(harness.authorize(details({ requestId: 'request-new-turn' }), query, createLog())).resolves.toEqual({ kind: 'allow', }) expect(harness.streamSimple.mock.calls).toHaveLength(20) }) it('aborts at the total timeout and defers', async () => { vi.useFakeTimers() const harness = createHarness({ timeoutMs: 5, resultFactory: async streamOptions => new Promise((_resolve, reject) => { streamOptions.signal?.addEventListener('abort', () => reject(new Error('aborted')), { once: true }) }), }) const log = createLog() const result = harness.authorize(details(), query, log) await vi.advanceTimersByTimeAsync(10) await expect(result).resolves.toEqual({ kind: 'defer' }) expect(log.review.mock.calls[0]?.[0]).toBe('auto_review.decision') expect(log.review.mock.calls[0]?.[1]).toMatchObject({ errorCategory: 'timeout', }) }) })