`8adbdc605f3a3462f11e895849b8ea79593562bf`

Author: TheEdgeOfRage <git@theedgeofrage.com>
Committer: TheEdgeOfRage <git@theedgeofrage.com>
Date: 2026-09-08 09:00 UTC

Message

llm: extend judge contract with romaji field

Diff

This diff is truncated to protect this page.

  1diff --git a/PLAN.md b/PLAN.md
  2index 79c332864f320c3c62cbeb4fbd26d8936fa11383..119a21b91732db7ba2482ce7b809cf246ecc8316 100644
  3--- a/PLAN.md
  4+++ b/PLAN.md
  5@@ -76,7 +76,7 @@ This command is reference information for the operator only. The game does not i
  6 
  7diff --git a/internal/adapters/llm_test.go b/internal/adapters/llm_test.go
  8index e6331d5ec4702336f09e3b4fa6875e3bd996f1b6..79cc9dc53ac4a4a463c194970856b0c165530b51 100644
  9--- a/internal/adapters/llm_test.go
 10+++ b/internal/adapters/llm_test.go
 11@@ -118,7 +118,7 @@ func TestNPCModelBoundsHistory(t *testing.T) {
 12 
 13 func TestJudgeModelReply(t *testing.T) {
 14 	client, cap := newFakeLLM(t, func(w http.ResponseWriter) {
 15-		sseReply(w, "SCORE|80\nFEEDBACK|good")
 16+		sseReply(w, "SCORE|80\nROMAJI|konnichiwa\nFEEDBACK|good")
 17 	})
 18 	j := &JudgeModel{Client: client}
 19 
 20@@ -126,7 +126,7 @@ func TestJudgeModelReply(t *testing.T) {
 21 	if err != nil {
 22 		t.Fatalf("Judge: %v", err)
 23 	}
 24-	if got.Score != 80 || got.Feedback != "good" {
 25+	if got.Score != 80 || got.Romaji != "konnichiwa" || got.Feedback != "good" {
 26 		t.Errorf("unexpected judge result: %+v", got)
 27 	}
 28 	var req capturedChat
 29@@ -140,7 +140,7 @@ func TestJudgeModelReply(t *testing.T) {
 30 
 31 func TestJudgeModelMalformedSurfacesContractError(t *testing.T) {
 32 	client, _ := newFakeLLM(t, func(w http.ResponseWriter) {
 33-		sseReply(w, "SCORE|not-a-number\nFEEDBACK|x")
 34+		sseReply(w, "SCORE|not-a-number\nROMAJI|x\nFEEDBACK|x")
 35 	})
 36 	j := &JudgeModel{Client: client}
 37 
 38diff --git a/internal/llm/contract.go b/internal/llm/contract.go
 39index 2602a1deff731826c0ca7a28b9b7a65d721bdb48..ffb93543c4f8445aea93c09907b8c96f43fe7c52 100644
 40--- a/internal/llm/contract.go
 41+++ b/internal/llm/contract.go
 42@@ -45,7 +45,7 @@ func kindString(k ContractErrorKind) string {
 43 
 44 var (
 45 	npcFieldNames   = []string{"ROMAJI", "KANA", "ENGLISH"}
 46-	judgeFieldNames = []string{"SCORE", "FEEDBACK"}
 47+	judgeFieldNames = []string{"SCORE", "ROMAJI", "FEEDBACK"}
 48 )
 49 
 50 // parseFields splits a raw model reply into FIELD|value lines. Surrounding
 51@@ -121,33 +121,26 @@ func ParseNPCReply(raw string) (NPCReply, error) {
 52 
 53 type JudgeResult struct {
 54 	Score    int
 55+	Romaji   string
 56 	Feedback string
 57 }
 58 
 59-// ParseJudge parses the two-field judge contract. The score must be an integer
 60+// ParseJudge parses the three-field judge contract. The score must be an integer
 61 // in 0-100; anything else is a recoverable *ContractError.
 62 func ParseJudge(raw string) (JudgeResult, error) {
 63 	vals, err := parseFields(raw, judgeFieldNames)
 64 	if err != nil {
 65 		return JudgeResult{}, err
 66 	}
 67-	scoreStr, ok := vals["SCORE"]
 68-	if !ok {
 69-		return JudgeResult{}, &ContractError{Kind: MissingField, Field: "SCORE"}
 70+	if err := requireFields(vals, judgeFieldNames); err != nil {
 71+		return JudgeResult{}, err
 72 	}
 73-	score, perr := strconv.Atoi(scoreStr)
 74+	score, perr := strconv.Atoi(vals["SCORE"])
 75 	if perr != nil {
 76 		return JudgeResult{}, &ContractError{Kind: InvalidFormat, Field: "SCORE"}
 77 	}
 78 	if score < 0 || score > 100 {
 79 		return JudgeResult{}, &ContractError{Kind: OutOfRange, Field: "SCORE"}
 80 	}
 81-	feedback, ok := vals["FEEDBACK"]
 82-	if !ok {
 83-		return JudgeResult{}, &ContractError{Kind: MissingField, Field: "FEEDBACK"}
 84-	}
 85-	if feedback == "" {
 86-		return JudgeResult{}, &ContractError{Kind: EmptyValue, Field: "FEEDBACK"}
 87-	}
 88-	return JudgeResult{Score: score, Feedback: feedback}, nil
 89+	return JudgeResult{Score: score, Romaji: vals["ROMAJI"], Feedback: vals["FEEDBACK"]}, nil
 90 }
 91diff --git a/internal/llm/contract_test.go b/internal/llm/contract_test.go
 92index 90c7b0c393e61c70bc9ef0c6600fa1e660786a55..375443921ec34ae59f26466573df39257635bd90 100644
 93--- a/internal/llm/contract_test.go
 94+++ b/internal/llm/contract_test.go
 95@@ -74,46 +74,75 @@ func ParseNPCReplyErr(raw string) error {
 96 }
 97 
 98 func TestParseJudgeHappy(t *testing.T) {
 99-	raw := "SCORE|82\nFEEDBACK|Fushin na hodo, zensei deshita."
100+	raw := "SCORE|82\nROMAJI|konnichiwa desu\nFEEDBACK|Fushin na hodo, zensei deshita."
101 	got, err := ParseJudge(raw)
102 	if err != nil {
103 		t.Fatalf("unexpected error: %v", err)
104 	}
105-	want := JudgeResult{Score: 82, Feedback: "Fushin na hodo, zensei deshita."}
106+	want := JudgeResult{Score: 82, Romaji: "konnichiwa desu", Feedback: "Fushin na hodo, zensei deshita."}
107 	if got != want {
108 		t.Errorf("got %+v, want %+v", got, want)
109 	}
110 }
111 
112+func TestParseJudgeWhitespaceTolerance(t *testing.T) {
113+	raw := "  SCORE| 82 \n ROMAJI| konnichiwa \n\tFEEDBACK| nice  \n"
114+	got, err := ParseJudge(raw)
115+	if err != nil {
116+		t.Fatalf("unexpected error: %v", err)
117+	}
118+	if got.Score != 82 || got.Romaji != "konnichiwa" || got.Feedback != "nice" {
119+		t.Errorf("got %+v", got)
120+	}
121+}
122+
123+func TestParseJudgeMissingRomaji(t *testing.T) {
124+	if kindOf(t, ParseJudgeErr("SCORE|82\nFEEDBACK|x")) != MissingField {
125+		t.Fatalf("want MissingField for missing ROMAJI")
126+	}
127+}
128+
129+func TestParseJudgeEmptyRomaji(t *testing.T) {
130+	if kindOf(t, ParseJudgeErr("SCORE|82\nROMAJI|\nFEEDBACK|x")) != EmptyValue {
131+		t.Fatalf("want EmptyValue for empty ROMAJI")
132+	}
133+}
134+
135+func TestParseJudgeDuplicateField(t *testing.T) {
136+	if kindOf(t, ParseJudgeErr("SCORE|82\nROMAJI|a\nROMAJI|b\nFEEDBACK|x")) != DuplicateField {
137+		t.Fatalf("want DuplicateField for repeated ROMAJI")
138+	}
139+}
140+
141 func TestParseJudgeOutOfRangeHigh(t *testing.T) {
142-	if kindOf(t, ParseJudgeErr("SCORE|101\nFEEDBACK|x")) != OutOfRange {
143+	if kindOf(t, ParseJudgeErr("SCORE|101\nROMAJI|x\nFEEDBACK|x")) != OutOfRange {
144 		t.Fatalf("want OutOfRange for 101")
145 	}
146 }
147 
148 func TestParseJudgeOutOfRangeNegative(t *testing.T) {
149-	if kindOf(t, ParseJudgeErr("SCORE|-1\nFEEDBACK|x")) != OutOfRange {
150+	if kindOf(t, ParseJudgeErr("SCORE|-1\nROMAJI|x\nFEEDBACK|x")) != OutOfRange {
151 		t.Fatalf("want OutOfRange for -1")
152 	}
153 }
154 
155 func TestParseJudgeNonIntegerScore(t *testing.T) {
156-	if kindOf(t, ParseJudgeErr("SCORE|abc\nFEEDBACK|x")) != InvalidFormat {
157+	if kindOf(t, ParseJudgeErr("SCORE|abc\nROMAJI|x\nFEEDBACK|x")) != InvalidFormat {
158 		t.Fatalf("want InvalidFormat for non-integer score")
159 	}
160-	if kindOf(t, ParseJudgeErr("SCORE|5.5\nFEEDBACK|x")) != InvalidFormat {
161+	if kindOf(t, ParseJudgeErr("SCORE|5.5\nROMAJI|x\nFEEDBACK|x")) != InvalidFormat {
162 		t.Fatalf("want InvalidFormat for float score")
163 	}
164 }
165 
166 func TestParseJudgeBoundaryScores(t *testing.T) {
167 	for _, s := range []string{"0", "100"} {
168-		got, err := ParseJudge("SCORE|" + s + "\nFEEDBACK|x")
169+		got, err := ParseJudge("SCORE|" + s + "\nROMAJI|romaji\nFEEDBACK|x")
170 		if err != nil {
171 			t.Fatalf("score %s: unexpected error %v", s, err)
172 		}
173-		if got.Feedback != "x" {
174-			t.Errorf("score %s: feedback = %q", s, got.Feedback)
175+		if got.Romaji != "romaji" || got.Feedback != "x" {
176+			t.Errorf("score %s: got %+v", s, got)
177 		}
178 	}
179 }
180diff --git a/internal/llm/prompt.go b/internal/llm/prompt.go
181index 240d09a5699805ed657b7f20397451f8b0c0c463..5cad800b04e88d965e5f19d6d14a0f475d7e20bd 100644
182--- a/internal/llm/prompt.go
183+++ b/internal/llm/prompt.go
184@@ -20,13 +20,17 @@ ROMAJI|<romaji of the sentence you say>
185 KANA|<the same sentence written in kana>
186 ENGLISH|<a natural English translation of that sentence>`
187 
188-// judgeSystemTemplate is persona-free and fixes the two-field judge contract.
189+// judgeSystemTemplate is persona-free and fixes the three-field judge contract.
190 // It scores fluency, naturalness, and fit to the situation, accepting any
191-// natural phrasing. It never names a game, quest, or scenario.
192+// natural phrasing, and transcribes the given line into Hepburn romaji. It never
193+// names a game, quest, or scenario.
194diff --git a/internal/llm/prompt_test.go b/internal/llm/prompt_test.go
195index 63737b79defd23cf2dcecd2463feb3ceed868b8b..29935d6e4f291af32773386958bd3a1a583b4efe 100644
196--- a/internal/llm/prompt_test.go
197+++ b/internal/llm/prompt_test.go
198@@ -62,3 +62,16 @@ func TestBuildJudgeMessagesStructure(t *testing.T) {
199 		t.Errorf("judge prompt should be persona-free: %q", msgs[0].Content)
200 	}
201 }
202+
203+func TestBuildJudgeMessagesInstructsRomajiTranscription(t *testing.T) {
204+	msgs := BuildJudgeMessages("at a station ticket window", "raw japanese line")
205+	sys := msgs[0].Content
206+	for _, field := range []string{"SCORE|", "ROMAJI|", "FEEDBACK|"} {
207+		if !strings.Contains(sys, field) {
208+			t.Errorf("judge system should fix the %s field: %q", field, sys)
209+		}
210+	}
211+	if !strings.Contains(strings.ToLower(sys), "hepburn romaji") {
212+		t.Errorf("judge system should instruct Hepburn romaji transcription: %q", sys)
213+	}
214+}