llmgw/internal/bench/eval/score_test.go

package eval

import (
	"strings"
	"testing"
)

func caseOf(t *testing.T, yamlCase string) *Case {
	t.Helper()
	s, err := Parse("t.yaml", []byte("cases:\n  - prompt: q\n"+indent(yamlCase)))
	if err != nil {
		t.Fatal(err)
	}
	return &s.Cases[0]
}

func indent(s string) string {
	var b strings.Builder
	for _, l := range strings.Split(strings.TrimSpace(s), "\n") {
		b.WriteString("    " + l + "\n")
	}
	return b.String()
}

func TestEachRuleScoresTheAnswersItShouldAndRefusesTheRest(t *testing.T) {
	for _, tc := range []struct {
		rule   string
		answer string
		want   float64
	}{
		{`exact: "42"`, "42", 1},
		{`exact: "42"`, "  42\n", 1},
		{`exact: "42"`, "42.", 0},
		{"exact: Paris\nignore_case: true", "PARIS", 1},
		{`contains: Paris`, "The capital is Paris.", 1},
		{`contains: [Paris, France]`, "Paris is in France", 1},
		{`contains: [Paris, France]`, "Paris", 0},
		{"contains: paris\nignore_case: true", "PARIS", 1},
		{`regex: '^\s*\d{4}-\d{2}-\d{2}\s*$'`, "2026-10-04", 1},
		{`regex: '^\d+$'`, "about 12", 0},
		{"regex: '^yes'\nignore_case: true", "Yes, it is", 1},
		{`number: 42`, "17 + 25 = 42", 1},
		{`number: 42`, "The answer is 42.", 1},
		{`number: 42`, "It is 41", 0},
		{`number: 1234567`, "1,234,567", 1},
		{"number: 3.14\ntolerance: 0.01", "pi is about 3.1416", 1},
		{"number: 3.14\ntolerance: 0.001", "pi is about 3.1416", 0},
		{`number: -5`, "3 - 8 = -5", 1},
		// A date's last part is a number, not a negative one.
		{`number: 4`, "from 2026-10-04", 1},
		{`number: 1`, "no number here", 0},
	} {
		c := caseOf(t, tc.rule)
		v := scoreRule(c, cleanAnswer(tc.answer))
		if v.score != tc.want {
			t.Errorf("%s against %q scored %v (%s), wanted %v", strings.ReplaceAll(tc.rule, "\n", "; "), tc.answer, v.score, v.detail, tc.want)
		}
	}
}

func TestAnAnswerIsReadAsJSONAndHeldToItsSchema(t *testing.T) {
	c := caseOf(t, `
json_schema:
  type: object
  required: [name, age, tags]
  additionalProperties: false
  properties:
    name: {type: string, minLength: 1}
    age: {type: integer, minimum: 0, maximum: 150}
    tags: {type: array, items: {type: string}, maxItems: 3}
    kind: {enum: [cat, dog]}`)
	for _, tc := range []struct {
		answer string
		pass   bool
		why    string
	}{
		{`{"name": "Ada", "age": 36, "tags": ["math"]}`, true, ""},
		{"Here it is:\n```json\n{\"name\": \"Ada\", \"age\": 36, \"tags\": []}\n```", true, ""},
		{`Sure! {"name": "Ada", "age": 36, "tags": []} Hope that helps.`, true, ""},
		{`{"name": "Ada", "age": 36.5, "tags": []}`, false, "age is a number, not integer"},
		{`{"name": "Ada", "tags": []}`, false, "has no age"},
		{`{"name": "Ada", "age": 36, "tags": [], "extra": 1}`, false, "extra is not allowed"},
		{`{"name": "Ada", "age": 200, "tags": []}`, false, "above 150"},
		{`{"name": "Ada", "age": 3, "tags": [1]}`, false, "tags[0] is an integer, not string"},
		{`{"name": "Ada", "age": 3, "tags": [], "kind": "fish"}`, false, "not one of"},
		{`{"name": "", "age": 3, "tags": []}`, false, "fewer than 1"},
		{`not json at all`, false, "not JSON"},
	} {
		v := scoreRule(c, tc.answer)
		if (v.score == 1) != tc.pass || (tc.why != "" && !strings.Contains(v.detail, tc.why)) {
			t.Errorf("%q: %v %q, wanted pass=%v saying %q", tc.answer, v.score, v.detail, tc.pass, tc.why)
		}
	}
}

func TestASchemaUsingWhatIsNotCheckedIsRefusedWhenTheSuiteIsRead(t *testing.T) {
	_, err := Parse("t.yaml", []byte("cases:\n  - prompt: q\n    json_schema: {oneOf: [{type: string}]}\n"))
	if err == nil || !strings.Contains(err.Error(), "oneOf is not supported") {
		t.Fatalf("a schema with oneOf was accepted: %v", err)
	}
}

func TestAJudgesReplyIsReadAsAScore(t *testing.T) {
	for _, tc := range []struct {
		reply string
		want  float64
		bad   bool
	}{
		{`{"score": 0.8, "reason": "mostly right"}`, 0.8, false},
		{"<think>hmm</think>\n```json\n{\"score\": 1, \"reason\": \"ok\"}\n```", 1, false},
		{`{"score": "0.25"}`, 0.25, false},
		{`{"pass": false, "reason": "wrong city"}`, 0, false},
		{`{"score": 7}`, 1, false},
		{`0.6`, 0.6, false},
		{`PASS`, 1, false},
		{`Fail: it named the wrong city.`, 0, false},
		{`I cannot say.`, 0, true},
	} {
		v, err := readJudgement(tc.reply)
		if (err != nil) != tc.bad || v.score != tc.want {
			t.Errorf("%q: %v %v, wanted %v (unreadable: %v)", tc.reply, v.score, err, tc.want, tc.bad)
		}
	}
}

func TestAReasoningModelsThinkingIsNotPartOfItsAnswer(t *testing.T) {
	if got := cleanAnswer("<think>\n17+25... carry the one\n</think>\n\n42"); got != "42" {
		t.Errorf("%q", got)
	}
	if got := cleanAnswer("<think>still thinking when it was cut off"); got != "" {
		t.Errorf("%q", got)
	}
}