feat: vendor permission auto-review source

2026-08-28 08:35:57 +00:00 · 2026-08-19 07:19:50 +08:00
parent 04ad87b54d
commit cf4fef10b8
42 changed files with 10311 additions and 103 deletions
@@ -0,0 +1,487 @@
+import type { ReviewModelRegistry } from '../src/model.js'
+import type {
+  Api,
+  AssistantMessage,
+  AssistantMessageEventStream,
+  Model,
+  Provider,
+  SimpleStreamOptions,
+} from '@earendil-works/pi-ai'
+import type { SessionEntry } from '@earendil-works/pi-coding-agent'
+import type { AuthorizerLog, PermissionQuery, PromptPermissionDetails } from '@gotgenes/pi-permission-system'
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { DenialCircuitBreaker } from '../src/circuit-breaker.js'
+import { autoReviewConfigSchema } from '../src/config.js'
+import { createPermissionReviewer } from '../src/reviewer.js'
+
+function createModel(): Model<Api> {
+  return {
+    id: 'review-model',
+    name: 'Review Model',
+    api: 'openai-responses',
+    provider: 'custom-review',
+    baseUrl: 'https://review.example/v1',
+    reasoning: true,
+    input: ['text'],
+    cost: {
+      input: 0,
+      output: 0,
+      cacheRead: 0,
+      cacheWrite: 0,
+    },
+    contextWindow: 128_000,
+    maxTokens: 16_000,
+  }
+}
+
+function assistantMessage(text: string, stopReason: AssistantMessage['stopReason'] = 'stop'): AssistantMessage {
+  return {
+    role: 'assistant',
+    content: [{ type: 'text', text }],
+    api: 'openai-responses',
+    provider: 'custom-review',
+    model: 'review-model',
+    usage: {
+      input: 0,
+      output: 0,
+      cacheRead: 0,
+      cacheWrite: 0,
+      totalTokens: 0,
+      cost: {
+        input: 0,
+        output: 0,
+        cacheRead: 0,
+        cacheWrite: 0,
+        total: 0,
+      },
+    },
+    stopReason,
+    timestamp: 0,
+  }
+}
+
+function streamFrom(result: () => Promise<AssistantMessage>): AssistantMessageEventStream {
+  return { result } as AssistantMessageEventStream
+}
+
+function userEntry(): SessionEntry {
+  return {
+    type: 'message',
+    id: 'user-1',
+    parentId: null,
+    timestamp: '2026-07-23T00:00:00.000Z',
+    message: {
+      role: 'user',
+      content: 'Please run the requested operation.',
+      timestamp: 0,
+    },
+  }
+}
+
+function userInteractionEntries(): SessionEntry[] {
+  return [
+    {
+      type: 'message',
+      id: 'interaction-call',
+      parentId: 'user-1',
+      timestamp: '2026-07-23T00:00:30.000Z',
+      message: {
+        role: 'assistant',
+        content: [
+          {
+            type: 'toolCall',
+            id: 'question-1',
+            name: 'ask_user_question',
+            arguments: { questions: [] },
+          },
+        ],
+        api: 'openai-responses',
+        provider: 'test',
+        model: 'test',
+        usage: {
+          input: 0,
+          output: 0,
+          cacheRead: 0,
+          cacheWrite: 0,
+          totalTokens: 0,
+          cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
+        },
+        stopReason: 'toolUse',
+        timestamp: 0,
+      },
+    },
+    {
+      type: 'message',
+      id: 'interaction-1',
+      parentId: 'interaction-call',
+      timestamp: '2026-07-23T00:01:00.000Z',
+      message: {
+        role: 'toolResult',
+        toolCallId: 'question-1',
+        toolName: 'ask_user_question',
+        content: [{ type: 'text', text: 'untrusted presentation text' }],
+        details: {
+          cancelled: false,
+          answers: [{ question: 'Choose a mode?', answer: 'Safe mode' }],
+        },
+        isError: false,
+        timestamp: 0,
+      },
+    },
+  ]
+}
+
+function details(overrides: Partial<PromptPermissionDetails> = {}): PromptPermissionDetails {
+  const surface = overrides.surface ?? 'bash'
+  const value = overrides.value ?? overrides.command ?? overrides.path ?? 'pnpm publish'
+  return {
+    requestId: 'request-1',
+    source: 'tool_call',
+    agentName: null,
+    toolName: 'bash',
+    command: 'pnpm publish',
+    surface,
+    payload: {
+      kind: surface === 'bash' ? 'bash' : surface === 'path' ? 'path' : 'tool',
+      request: {
+        requester: { agentName: null, forwarded: false, sessionId: null },
+        surface,
+        toolName: overrides.toolName ?? 'bash',
+        invokedToolName: null,
+        value,
+        matchedPattern: '*',
+        commandContext: null,
+        executedUnit: null,
+      },
+      evidence: [],
+      annotations: [],
+    },
+    ...overrides,
+  }
+}
+
+interface TestLog extends AuthorizerLog {
+  review: ReturnType<typeof vi.fn<AuthorizerLog['review']>>
+  debug: ReturnType<typeof vi.fn<AuthorizerLog['debug']>>
+}
+
+function createLog(): TestLog {
+  return {
+    review: vi.fn<AuthorizerLog['review']>(),
+    debug: vi.fn<AuthorizerLog['debug']>(),
+  }
+}
+
+interface HarnessOptions {
+  responses?: Array<AssistantMessage | Error>
+  auth?: Awaited<ReturnType<ReviewModelRegistry['getApiKeyAndHeaders']>>
+  timeoutMs?: number
+  resultFactory?: (options: SimpleStreamOptions) => Promise<AssistantMessage>
+  providerLookup?: 'native' | 'missing' | 'throwing'
+  sessionEntries?: SessionEntry[]
+}
+
+function createHarness(options: HarnessOptions = {}) {
+  const model = createModel()
+  const responses = [...(options.responses ?? [assistantMessage('{"outcome":"allow"}')])]
+  const streamSimple = vi.fn((_model: Model<Api>, _context: unknown, streamOptions: SimpleStreamOptions = {}) =>
+    streamFrom(async () => {
+      if (options.resultFactory !== undefined) {
+        return options.resultFactory(streamOptions)
+      }
+      const next = responses.shift()
+      if (next instanceof Error) {
+        throw next
+      }
+      if (next === undefined) {
+        throw new Error('no fake response')
+      }
+      return next
+    }),
+  )
+  const provider = {
+    id: 'custom-review',
+    name: 'Custom Review',
+    auth: {},
+    getModels: () => [model],
+    stream: streamSimple,
+    streamSimple,
+  } as unknown as Provider
+  const getApiKeyAndHeaders = vi.fn(async () =>
+    Promise.resolve(
+      options.auth ?? {
+        ok: true as const,
+        apiKey: 'secret-key',
+        headers: { 'x-review': 'enabled' },
+        env: { REVIEW_REGION: 'test' },
+      },
+    ),
+  )
+  const registryBase = {
+    find: vi.fn(() => model),
+    getAll: vi.fn(() => [model]),
+    getApiKeyAndHeaders,
+  }
+  const providerLookup = vi.fn(() => provider)
+  let registry: ReviewModelRegistry
+  switch (options.providerLookup ?? 'native') {
+    case 'native':
+      registry = { ...registryBase, getProvider: providerLookup }
+      break
+    case 'missing':
+      registry = { ...registryBase, getProvider: vi.fn(() => undefined) }
+      break
+    case 'throwing':
+      registry = {
+        ...registryBase,
+        getProvider: () => {
+          throw new Error('provider lookup failed')
+        },
+      }
+      break
+  }
+  const circuitBreaker = new DenialCircuitBreaker()
+  const getBranch = vi.fn(() => options.sessionEntries ?? [userEntry()])
+  const authorize = createPermissionReviewer(
+    {
+      config: autoReviewConfigSchema.parse({
+        provider: 'custom-review',
+        model: 'review-model',
+        timeoutMs: options.timeoutMs ?? 90_000,
+      }),
+      registry,
+      sessionManager: { getBranch },
+      circuitBreaker,
+    },
+    {
+      now: () => 0,
+      retryDelaysMs: [0, 0],
+      sleep: async () => Promise.resolve(),
+    },
+  )
+
+  return {
+    authorize,
+    circuitBreaker,
+    getApiKeyAndHeaders,
+    getBranch,
+    registry,
+    streamSimple,
+  }
+}
+
+const query = {} as PermissionQuery
+
+describe('permission reviewer', () => {
+  afterEach(() => {
+    vi.useRealTimers()
+  })
+
+  it('passes Pi-managed auth to a tool-free provider call and allows', async () => {
+    const harness = createHarness()
+    const log = createLog()
+
+    await expect(harness.authorize(details(), query, log)).resolves.toEqual({
+      kind: 'allow',
+    })
+
+    expect(harness.getApiKeyAndHeaders.mock.calls).toHaveLength(1)
+    const [, context, options] = harness.streamSimple.mock.calls[0] ?? []
+    expect(context).toMatchObject({
+      messages: [{ role: 'user' }],
+    })
+    expect(context).not.toHaveProperty('tools')
+    expect((context as { systemPrompt?: string }).systemPrompt).toContain(
+      'source field is "user" or "user_interaction"',
+    )
+    expect(harness.getBranch).toHaveBeenCalledOnce()
+    expect(options).toMatchObject({
+      apiKey: 'secret-key',
+      headers: { 'x-review': 'enabled' },
+      env: { REVIEW_REGION: 'test' },
+      maxRetries: 0,
+      maxTokens: 1_000,
+      reasoning: 'low',
+    })
+    expect(log.review.mock.calls[0]?.[1]).toMatchObject({
+      policyRevision: 'openai-codex/c4f42d161ae44a8d696ee9fb595709661979d187+pi1',
+      contextSource: 'active-branch',
+      transcriptEntriesRetained: 1,
+      transcriptEntriesOmitted: 0,
+      transcriptEntriesTruncated: 0,
+      directUserEntriesRetained: 1,
+      directUserEntriesOmitted: 0,
+      directUserEntriesTruncated: 0,
+      userInteractionEntriesRetained: 0,
+      userInteractionEntriesOmitted: 0,
+      userInteractionEntriesTruncated: 0,
+      latestTrustedEntryRetained: true,
+    })
+  })
+
+  it('sends canonical structured user interactions to the provider', async () => {
+    const harness = createHarness({
+      sessionEntries: [userEntry(), ...userInteractionEntries()],
+    })
+
+    await expect(harness.authorize(details(), query, createLog())).resolves.toEqual({ kind: 'allow' })
+
+    const [, context] = harness.streamSimple.mock.calls[0] ?? []
+    const userPrompt = (
+      context as {
+        messages: Array<{ content: string }>
+      }
+    ).messages[0]?.content
+    expect(userPrompt).toContain('"source":"user_interaction"')
+    expect(userPrompt).toContain('[{\\"question\\":\\"Choose a mode?\\",\\"answer\\":\\"Safe mode\\"}]')
+    expect(userPrompt).not.toContain('untrusted presentation text')
+  })
+
+  it('returns a teaching denial without persisting the rationale', async () => {
+    const harness = createHarness({
+      responses: [
+        assistantMessage(
+          '{"risk_level":"high","user_authorization":"unknown","outcome":"deny","rationale":"Publishing was not authorized."}',
+        ),
+      ],
+    })
+    const log = createLog()
+
+    const result = await harness.authorize(details({ surface: 'path', path: '.env' }), query, log)
+
+    expect(result.kind).toBe('deny')
+    if (result.kind === 'deny') {
+      expect(result.reason).toContain('Publishing was not authorized.')
+    }
+    expect(log.review.mock.calls[0]?.[0]).toBe('auto_review.decision')
+    expect(log.review.mock.calls[0]?.[1]).toMatchObject({
+      outcome: 'deny',
+      riskLevel: 'high',
+      userAuthorization: 'unknown',
+    })
+    expect(log.review.mock.calls[0]?.[1]).not.toHaveProperty('rationale')
+    expect(log.review.mock.calls[0]?.[1]).not.toHaveProperty('surface')
+  })
+
+  it('retries transient provider failures within the same review', async () => {
+    const harness = createHarness({
+      responses: [
+        new Error('temporary failure'),
+        assistantMessage('', 'error'),
+        assistantMessage('{"outcome":"allow"}'),
+      ],
+    })
+
+    await expect(harness.authorize(details(), query, createLog())).resolves.toEqual({ kind: 'allow' })
+    expect(harness.streamSimple.mock.calls).toHaveLength(3)
+  })
+
+  it('defers malformed output and missing auth to the human authorizer', async () => {
+    const malformed = createHarness({
+      responses: [assistantMessage('not json')],
+    })
+    const malformedLog = createLog()
+    await expect(malformed.authorize(details(), query, malformedLog)).resolves.toEqual({ kind: 'defer' })
+    expect(malformedLog.review.mock.calls[0]?.[0]).toBe('auto_review.decision')
+    expect(malformedLog.review.mock.calls[0]?.[1]).toMatchObject({
+      errorCategory: 'invalid-response',
+    })
+
+    const missingAuth = createHarness({
+      auth: { ok: false, error: 'not configured' },
+    })
+    await expect(missingAuth.authorize(details(), query, createLog())).resolves.toEqual({ kind: 'defer' })
+    expect(missingAuth.streamSimple.mock.calls).toHaveLength(0)
+  })
+
+  it('contains unsupported and throwing provider lookup failures', async () => {
+    const missing = createHarness({ providerLookup: 'missing' })
+    const missingLog = createLog()
+    await expect(missing.authorize(details(), query, missingLog)).resolves.toEqual({ kind: 'defer' })
+    expect(missingLog.review.mock.calls[0]?.[1]).toMatchObject({
+      errorCategory: 'provider-unresolved',
+    })
+
+    const throwing = createHarness({ providerLookup: 'throwing' })
+    const throwingLog = createLog()
+    await expect(throwing.authorize(details(), query, throwingLog)).resolves.toEqual({ kind: 'defer' })
+    expect(throwingLog.review.mock.calls[0]?.[1]).toMatchObject({
+      errorCategory: 'internal-error',
+    })
+  })
+
+  it('defers when review logging throws', async () => {
+    const harness = createHarness()
+    const log = createLog()
+    log.review.mockImplementation(() => {
+      throw new Error('log unavailable')
+    })
+
+    await expect(harness.authorize(details(), query, log)).resolves.toEqual({ kind: 'defer' })
+  })
+
+  it('opens the per-turn circuit after three consecutive denials', async () => {
+    const denial = assistantMessage('{"outcome":"deny","rationale":"Not authorized."}')
+    const harness = createHarness({
+      responses: [denial, denial, denial],
+    })
+
+    for (let index = 0; index < 3; index += 1) {
+      await expect(
+        harness.authorize(details({ requestId: `request-${index}` }), query, createLog()),
+      ).resolves.toMatchObject({ kind: 'deny' })
+    }
+
+    const circuitLog = createLog()
+    const circuitResult = await harness.authorize(details({ requestId: 'request-4' }), query, circuitLog)
+    expect(circuitResult.kind).toBe('deny')
+    if (circuitResult.kind === 'deny') {
+      expect(circuitResult.reason).toContain('explicit approval')
+    }
+    expect(harness.streamSimple.mock.calls).toHaveLength(3)
+    expect(circuitLog.review.mock.calls[0]?.[0]).toBe('auto_review.circuit_open')
+  })
+
+  it('opens the per-turn circuit after ten non-consecutive denials in the recent window', async () => {
+    const denial = assistantMessage('{"outcome":"deny","rationale":"Not authorized."}')
+    const allow = assistantMessage('{"outcome":"allow"}')
+    const responses = Array.from({ length: 10 }, () => [denial, allow]).flat()
+    const harness = createHarness({ responses })
+
+    for (let index = 0; index < 19; index += 1) {
+      await harness.authorize(details({ requestId: `request-${index}` }), query, createLog())
+    }
+
+    await expect(
+      harness.authorize(details({ requestId: 'request-circuit' }), query, createLog()),
+    ).resolves.toMatchObject({ kind: 'deny' })
+    expect(harness.streamSimple.mock.calls).toHaveLength(19)
+
+    harness.circuitBreaker.resetTurn()
+    await expect(harness.authorize(details({ requestId: 'request-new-turn' }), query, createLog())).resolves.toEqual({
+      kind: 'allow',
+    })
+    expect(harness.streamSimple.mock.calls).toHaveLength(20)
+  })
+
+  it('aborts at the total timeout and defers', async () => {
+    vi.useFakeTimers()
+    const harness = createHarness({
+      timeoutMs: 5,
+      resultFactory: async streamOptions =>
+        new Promise((_resolve, reject) => {
+          streamOptions.signal?.addEventListener('abort', () => reject(new Error('aborted')), { once: true })
+        }),
+    })
+    const log = createLog()
+
+    const result = harness.authorize(details(), query, log)
+    await vi.advanceTimersByTimeAsync(10)
+
+    await expect(result).resolves.toEqual({ kind: 'defer' })
+    expect(log.review.mock.calls[0]?.[0]).toBe('auto_review.decision')
+    expect(log.review.mock.calls[0]?.[1]).toMatchObject({
+      errorCategory: 'timeout',
+    })
+  })
+})