e60c1bde2074754c5e21f1b0157ad84ca65585e6

Message

Use Pi OAuth for policy evaluation

Diff

This diff is truncated to protect this page.

  1diff --git a/README.md b/README.md
  2index 1daee77771150311dd4ecf52139fc5ca5801edb2..db5cdb5ab274ba918fbd9442a1a5272202a66200 100644
  3--- a/README.md
  4+++ b/README.md
  5@@ -51,13 +51,12 @@ Use local llama.cpp. This backend does not use authentication.
  6 		"kind": "llama.cpp",
  7 		"baseUrl": "http://127.0.0.1:9931",
  8 		"model": "reviewer",
  9-		"enableThinking": true,
 10-		"reasoningFormat": "deepseek"
 11+		"enableThinking": true
 12 	}
 13 }
 14 ```
 15 
 16-`enableThinking` defaults to `false`. `reasoningFormat` defaults to `"none"`. llama.cpp supports `"none"`, `"deepseek"`, and `"deepseek-legacy"`.
 17+`enableThinking` defaults to `false`. llama.cpp always uses the `deepseek` reasoning format.
 18 
 19 LLM review sends the complete tool input to the configured provider. Do not use a cloud reviewer when this is not acceptable.
 20 
 21@@ -102,5 +101,15 @@ Run the opt-in llama.cpp policy evaluation:
 22 POLICY_EVAL_BASE_URL=http://127.0.0.1:9931 POLICY_EVAL_MODEL=reviewer bun test test/core/llm.test.ts
 23 ```
 24 
 25+Run the same policy evaluation through Pi OAuth. Pi resolves the configured OAuth session; no API key is needed.
 26 
 27-Tests redirect Pi state to a temporary directory.
 28+```sh
 29+POLICY_EVAL_BACKEND=pi \
 30+POLICY_EVAL_PROVIDER=openai-codex \
 31+POLICY_EVAL_MODEL=gpt-5.6-luna \
 32+POLICY_EVAL_REASONING_EFFORT=minimal \
 33+bun test test/core/llm.test.ts
 34+```
 35+
 36+
 37+Tests redirect Pi state to a temporary directory, except opt-in Pi OAuth evaluations, which use the existing Pi OAuth session.
 38diff --git a/src/core/config.ts b/src/core/config.ts
 39index 52276a12a5d0f11804cdfac620ecba34a76d33ae..790311e607c628b013eb3c241dc78ab18cfe6fff 100644
 40--- a/src/core/config.ts
 41+++ b/src/core/config.ts
 42@@ -1,4 +1,4 @@
 43-import type { LlamaReasoningFormat, PiReasoningEffort, ReviewerConfig } from "./types"
 44+import type { PiReasoningEffort, ReviewerConfig } from "./types"
 45 
 46 function objectValue(value: unknown): Record<string, unknown> | undefined {
 47   return value && typeof value === "object" && !Array.isArray(value)
 48@@ -16,12 +16,6 @@ function booleanValue(value: unknown, field: string, fallback: boolean): boolean
 49   throw new Error(`${field} must be a boolean`)
 50 }
 51 
 52-function llamaReasoningFormat(value: unknown): LlamaReasoningFormat {
 53-  if (value === undefined) return "none"
 54-  if (value === "none" || value === "deepseek" || value === "deepseek-legacy") return value
 55-  throw new Error('reviewer.reasoningFormat must be "none", "deepseek", or "deepseek-legacy"')
 56-}
 57-
 58 function reasoningEffort(value: unknown): PiReasoningEffort | undefined {
 59   if (value === undefined) return undefined
 60   if (value === "none" || value === "minimal" || value === "low" || value === "medium" || value === "high" || value === "xhigh" || value === "max") {
 61@@ -51,7 +45,6 @@ export function parseReviewerConfig(value: unknown): ReviewerConfig {
 62       baseUrl,
 63       model,
 64       enableThinking: booleanValue(reviewer.enableThinking, "reviewer.enableThinking", false),
 65-      reasoningFormat: llamaReasoningFormat(reviewer.reasoningFormat),
 66     }
 67   }
 68   if (reviewer.kind !== "pi") throw new Error('reviewer.kind must be "pi" or "llama.cpp"')
 69diff --git a/src/core/providers.ts b/src/core/providers.ts
 70index 52c526e50e2bcae2e6a7534dc7b20989e93b56ce..ee15e1869182af691db86cd809b4f55818f6b5e6 100644
 71--- a/src/core/providers.ts
 72+++ b/src/core/providers.ts
 73@@ -40,7 +40,7 @@ export async function callLlamaReviewer(
 74       temperature: 0,
 75       response_format: { type: "json_object" },
 76       chat_template_kwargs: { enable_thinking: config.enableThinking },
 77-      reasoning_format: config.reasoningFormat,
 78+      reasoning_format: "deepseek",
 79     }),
 80   })
 81   if (!response.ok) return responseError(response)
 82diff --git a/src/core/rules.ts b/src/core/rules.ts
 83index da132e85e378f9282c8fe34bbf5bb5eedd35483f..10a564fd69eebc02a9fe693f73eeaf816db290ed 100644
 84--- a/src/core/rules.ts
 85+++ b/src/core/rules.ts
 86@@ -194,6 +194,7 @@ Ask only for:
 87 - Remote infrastructure, data, access, or deployment mutation: create, update, delete, grant, revoke, apply, or execute a write on production databases, Kubernetes, or cloud providers.
 88 - Irreversibly destructive Git operations, such as reset --hard or checking out files.
 89 - When a command attempts to search the entire (and only *THE ENTIRE*) home directory for something, since that's very slow and rarely needed.
 90+- Stopping the Herdr server with \`herdr server stop\`.
 91 
 92diff --git a/src/core/types.ts b/src/core/types.ts
 93index f7161620a65fa476168a042a4591fcac863ceadd..e48ccb34888889ddc4f4ceddba5697ee887c7667 100644
 94--- a/src/core/types.ts
 95+++ b/src/core/types.ts
 96@@ -48,7 +48,6 @@ export type PolicyReviewer = {
 97 }
 98 
 99 export type PiReasoningEffort = "none" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max"
100-export type LlamaReasoningFormat = "none" | "deepseek" | "deepseek-legacy"
101 
102 export type ReviewerConfig =
103   | {
104@@ -62,7 +61,6 @@ export type ReviewerConfig =
105     baseUrl: string
106     model: string
107     enableThinking: boolean
108-    reasoningFormat: LlamaReasoningFormat
109   }
110 
111 export type CacheEntry = {
112diff --git a/test/core/llm.test.ts b/test/core/llm.test.ts
113index b04f9795e133d00c7b524f4e73545bcc69578610..59caf94801e0bb1b281ce419c410d4e1a0280a46 100644
114--- a/test/core/llm.test.ts
115+++ b/test/core/llm.test.ts
116@@ -1,35 +1,82 @@
117-import { describe, expect, test } from "bun:test";
118-import { createPolicyReviewer } from "../../src/core/review";
119-import type { Decision, LLMEvaluationResult } from "../../src/core/types";
120+import { describe, expect, test } from "bun:test"
121+import type { UserMessage } from "@earendil-works/pi-ai"
122+import { ModelRuntime } from "@earendil-works/pi-coding-agent"
123+import { createPolicyReviewer } from "../../src/core/review"
124+import type { Decision, LLMEvaluationResult, PiReasoningEffort, PolicyReviewer } from "../../src/core/types"
125 
126-const baseUrl = process.env.POLICY_EVAL_BASE_URL?.trim();
127-const model = process.env.POLICY_EVAL_MODEL?.trim();
128+const backend = process.env.POLICY_EVAL_BACKEND?.trim() ?? "llama.cpp"
129+const baseUrl = process.env.POLICY_EVAL_BASE_URL?.trim()
130+const provider = process.env.POLICY_EVAL_PROVIDER?.trim() ?? "openai-codex"
131+const model = process.env.POLICY_EVAL_MODEL?.trim()
132+const reasoningEffort = process.env.POLICY_EVAL_REASONING_EFFORT?.trim() ?? "minimal"
133 
134-if (Boolean(baseUrl) !== Boolean(model)) {
135-  throw new Error(
136-    "POLICY_EVAL_BASE_URL and POLICY_EVAL_MODEL must both be set to run local LLM policy evaluation",
137-  );
138+function piReasoningEffort(value: string): PiReasoningEffort {
139+  if (value === "none" || value === "minimal" || value === "low" || value === "medium" || value === "high" || value === "xhigh" || value === "max") {
140+    return value
141+  }
142+  throw new Error("POLICY_EVAL_REASONING_EFFORT must be a Pi reasoning level")
143 }
144 
145-const reviewer =
146-  baseUrl && model
147-    ? createPolicyReviewer({
148+async function createReviewer(): Promise<PolicyReviewer | undefined> {
149+  if (backend === "llama.cpp") {
150+    if (Boolean(baseUrl) !== Boolean(model)) {
151+      throw new Error("POLICY_EVAL_BASE_URL and POLICY_EVAL_MODEL must both be set for llama.cpp evaluation")
152+    }
153+    if (!baseUrl || !model) return undefined
154+    return createPolicyReviewer({
155       kind: "llama.cpp",
156       baseUrl,
157       model,
158       enableThinking: true,
159-      reasoningFormat: "deepseek",
160     }, async () => {
161-        throw new Error("Pi reviewer must not be used for llama.cpp evaluation");
162-      })
163-    : undefined;
164+      throw new Error("Pi reviewer must not be used for llama.cpp evaluation")
165+    })
166+  }
167+
168+  if (backend !== "pi") throw new Error('POLICY_EVAL_BACKEND must be "llama.cpp" or "pi"')
169+  if (!model) throw new Error("POLICY_EVAL_MODEL must be set for Pi evaluation")
170+
171+  const runtime = await ModelRuntime.create()
172+  const piModel = runtime.getModel(provider, model)
173+  if (!piModel) throw new Error(`Pi model is unavailable: ${provider}/${model}`)
174+  const effort = piReasoningEffort(reasoningEffort)
175+
176+  return createPolicyReviewer(
177+    { kind: "pi", provider, model, reasoningEffort: effort },
178+    async (_config, messages, _maxTokens, context) => {
179+      const systemPrompt = messages.find((message) => message.role === "system")?.content
180+      const userMessages: UserMessage[] = messages
181+        .filter((message) => message.role === "user")
182+        .map((message) => ({
183+          role: "user",
184+          content: [{ type: "text", text: message.content }],
185+          timestamp: Date.now(),
186+        }))
187+      const response = await runtime.complete(
188+        piModel,
189+        { systemPrompt, messages: userMessages },
190+        { signal: context.signal, reasoningEffort: effort },
191+      )
192+      if (response.stopReason !== "stop") {
193+        throw new Error(response.errorMessage ?? `Pi reviewer stopped: ${response.stopReason}`)
194+      }
195+      const text = response.content
196+        .filter((content): content is { type: "text"; text: string } => content.type === "text")
197+        .map((content) => content.text)
198+        .join("\n")
199+      if (!text.trim()) throw new Error("Pi reviewer returned no text")
200+      return text
201+    },
202+  )
203+}
204 
205-const STRICTNESS = { allow: 0, ask: 1, deny: 2 } as const;
206+const reviewer = await createReviewer()
207+const STRICTNESS = { allow: 0, ask: 1, deny: 2 } as const
208 
209-type ExpectedDecision = "allow" | "ask";
210+type ExpectedDecision = "allow" | "ask"
211 
212-const RUNS = 5;
213-const MIN_CORRECT = 4;
214+const RUNS = 5
215+const MIN_CORRECT = 4
216diff --git a/test/setup.ts b/test/setup.ts
217index 9856b2dcb799d75a78d03fc6ced0251033a2748b..4aba665b25fd5e3e6802463c5927935225a50df2 100644
218--- a/test/setup.ts
219+++ b/test/setup.ts
220@@ -2,9 +2,11 @@ import { mkdtempSync, rmSync } from "node:fs"
221 import { tmpdir } from "node:os"
222 import { join } from "node:path"
223 
224-const stateDirectory = mkdtempSync(join(tmpdir(), "policy-engine-state-"))
225-process.env.PI_CODING_AGENT_DIR = stateDirectory
226+if (process.env.POLICY_EVAL_BACKEND?.trim() !== "pi") {
227+  const stateDirectory = mkdtempSync(join(tmpdir(), "policy-engine-state-"))
228+  process.env.PI_CODING_AGENT_DIR = stateDirectory
229 
230-process.on("exit", () => {
231-  rmSync(stateDirectory, { recursive: true, force: true })
232-})
233+  process.on("exit", () => {
234+    rmSync(stateDirectory, { recursive: true, force: true })
235+  })
236+}