8433e33f85ad8e9a9ecbdeb6c02b23a5b9e9da28

Author
TheEdgeOfRage <git@theedgeofrage.com>
Committer
TheEdgeOfRage <git@theedgeofrage.com>
Date

Message

Log unparseable LLM responses

Diff

 1diff --git a/src/core/pipeline.ts b/src/core/pipeline.ts
 2index 45b639cd7e9277726e1e1973757a136f9539d977..8d54065e6a4cc82142bed48298ffe0c2cb055467 100644
 3--- a/src/core/pipeline.ts
 4+++ b/src/core/pipeline.ts
 5@@ -47,6 +47,7 @@ export function createPolicyPipeline(options: PolicyPipelineOptions) {
 6       source: result.source,
 7       timingMs: performance.now() - startedAt,
 8       sessionId: context.sessionId,
 9+      rawResponse: result.error ? result.rawResponse : undefined,
10     }
11     try {
12       await options.audit.record(entry)
13diff --git a/src/core/types.ts b/src/core/types.ts
14index 70389b7e343ad9770f8a3528006be8e2b4d7772f..203243a03e7c8b0195d5dd6ad7b89cf3225b76dd 100644
15--- a/src/core/types.ts
16+++ b/src/core/types.ts
17@@ -90,6 +90,7 @@ export type AuditEntry = {
18   source: DecisionSource
19   timingMs: number
20   sessionId?: string
21+  rawResponse?: string
22 }
23 
24 export type DecisionAudit = {
25diff --git a/src/pi/index.ts b/src/pi/index.ts
26index 80186211b297c79877ebfd42d3de8f9975a25626..7b60cb0492ac072d1f9de708ff70c891b377af77 100644
27--- a/src/pi/index.ts
28+++ b/src/pi/index.ts
29@@ -197,7 +197,10 @@ export default function policyEngine(pi: ExtensionAPI) {
30     let approved: boolean
31     try {
32       const title = "Policy approval required"
33-      const message = `${event.toolName}: ${inputSummary(event.toolName, input)}\n\n${result.decision.reason}`
34+      const rawResponse = result.error === "Failed to parse response JSON" && result.rawResponse
35+        ? `\n\nLLM response:\n${result.rawResponse}`
36+        : ""
37+      const message = `${event.toolName}: ${inputSummary(event.toolName, input)}\n\n${result.decision.reason}${rawResponse}`
38       approved = ctx.mode === "tui"
39         ? await ctx.ui.custom(
40           (tui, theme, keybindings, done) => new PolicyApprovalDialog(tui, theme, keybindings, title, message, done),
41diff --git a/test/pi/extension.test.ts b/test/pi/extension.test.ts
42index 4db1204c76fa4381b885d4adae4e45e010cf13c7..d885484fc12636cef1a1939228ce88b7af37d992 100644
43--- a/test/pi/extension.test.ts
44+++ b/test/pi/extension.test.ts
45@@ -54,7 +54,7 @@ function loadExtension() {
46   return { toolCall, sessionStart, allowBash, skipPermissions, entries, events }
47 }
48 
49-function context(hasUI = false, confirm = async () => false) {
50+function context(hasUI = false, confirm: (title: string, message: string) => Promise<boolean> = async () => false) {
51   return {
52     hasUI,
53     model: undefined,
54@@ -106,6 +106,25 @@ describe("Pi policy extension", () => {
55     ])
56   })
57 
58+  test("shows an unparseable LLM response in the approval prompt", async () => {
59+    globalThis.fetch = (async (_input, _init) => {
60+      return Response.json({ choices: [{ message: { content: "not JSON" } }] })
61+    }) as typeof fetch
62+    let message = ""
63+
64+    const extension = loadExtension()
65+    await extension.toolCall(
66+      { toolName: "unknown_tool", input: {} },
67+      context(true, async (_title: string, value: string) => {
68+        message = value
69+        return false
70+      }),
71+    )
72+
73+    expect(message).toContain("Unparseable LLM response")
74+    expect(message).toContain("LLM response:\nnot JSON")
75+  })
76+
77   test("allows base64 from a pipe", async () => {
78     let reviews = 0
79     globalThis.fetch = (async (_input, _init) => {