llmgw/internal/cli/ctl_eval_test.go

package cli

import (
	"bytes"
	"context"
	"encoding/json"
	"io"
	"io/fs"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"testing"

	llmgw "gitea.mynodes.xyz/ilker/llmgw"
	"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)

func TestTheSuitesTheBinaryCarriesCanBeRun(t *testing.T) {
	sub, err := fs.Sub(llmgw.EvalSuites, "deploy/eval")
	if err != nil {
		t.Fatal(err)
	}
	got := eval.NewService(eval.Deps{Builtin: sub}).Suites()
	var names []string
	for _, s := range got {
		if s.Error != "" || s.Cases < 5 || s.Judge {
			t.Errorf("%s: %+v", s.Name, s)
		}
		names = append(names, s.Name)
	}
	if strings.Join(names, " ") != "arithmetic json-format" {
		t.Errorf("built in: %v", names)
	}
}

func TestCtlEvalSendsAFileAndFailsBelowTheScore(t *testing.T) {
	var mu sync.Mutex
	var sent []string
	finished := eval.Live{EvalRun: store.EvalRun{ID: "eval-1", Model: "qwen", Suite: "mine", Score: 0.5,
		Threshold: 0.8, Cases: 2, Below: 1, Revision: "abcdef1234567", Repo: "o/r", Results: []store.EvalCase{
			{ID: "add", Kind: "number", Weight: 1, Score: 1, Passed: true, Answer: "42", Detail: "read 42"},
			{ID: "mul", Kind: "number", Weight: 1, Score: 0, Answer: "43", Detail: "read 43, wanted 42"}}},
		State: eval.Done, Done: 2}
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		b, _ := io.ReadAll(r.Body)
		mu.Lock()
		sent = append(sent, r.Method+" "+r.URL.Path+" "+string(b))
		mu.Unlock()
		switch r.URL.Path {
		case "/api/v1/models/qwen/eval":
			w.WriteHeader(http.StatusAccepted)
			_ = json.NewEncoder(w).Encode(eval.Live{EvalRun: store.EvalRun{ID: "eval-1", Model: "qwen", Cases: 2},
				State: eval.Running})
		case "/api/v1/eval/runs/eval-1":
			_ = json.NewEncoder(w).Encode(finished)
		case "/api/v1/models/qwen/update":
			w.WriteHeader(http.StatusAccepted)
			_ = json.NewEncoder(w).Encode(modelserve.Job{ID: "update-1", Model: "qwen", State: modelserve.Done})
		default:
			http.NotFound(w, r)
		}
	}))
	t.Cleanup(srv.Close)
	file := filepath.Join(t.TempDir(), "mine.yaml")
	if err := os.WriteFile(file, []byte("cases:\n  - prompt: 40 + 2?\n    number: 42\n"), 0o600); err != nil {
		t.Fatal(err)
	}
	ctl := func(args ...string) (string, error) {
		cmd := newCtlCmd()
		var out bytes.Buffer
		cmd.SetOut(&out)
		cmd.SetErr(&out)
		cmd.SetArgs(append([]string{"--server", srv.URL}, args...))
		err := cmd.ExecuteContext(context.Background())
		return out.String(), err
	}

	out, err := ctl("eval", "qwen", "--suite", file, "--concurrency", "2")
	if err == nil || !strings.Contains(err.Error(), "qwen scored 0.50 on mine, below 0.80") {
		t.Fatalf("a run below its score did not fail: %v\n%s", err, out)
	}
	for _, want := range []string{"read 43, wanted 42", "1 of 2 cases passed", "weights o/r at abcdef1"} {
		if !strings.Contains(out, want) {
			t.Errorf("the output lacks %q:\n%s", want, out)
		}
	}
	mu.Lock()
	var req eval.Request
	_ = json.Unmarshal([]byte(strings.SplitN(sent[0], " ", 3)[2]), &req)
	mu.Unlock()
	if req.File != "mine.yaml" || !strings.Contains(req.Content, "40 + 2?") || req.Concurrency != 2 {
		t.Errorf("sent %+v", req)
	}

	// A broken file is said here, before anything is sent.
	if err := os.WriteFile(file, []byte("cases: []\n"), 0o600); err != nil {
		t.Fatal(err)
	}
	if _, err := ctl("eval", "qwen", "--suite", file); err == nil || !strings.Contains(err.Error(), "no cases") {
		t.Errorf("%v", err)
	}

	// An update held to a suite by name carries it.
	if _, err := ctl("model", "update", "qwen", "--eval", "arithmetic", "--min-score", "0.9"); err != nil {
		t.Fatal(err)
	}
	mu.Lock()
	last := sent[len(sent)-1]
	mu.Unlock()
	var ch modelserve.Change
	if err := json.Unmarshal([]byte(strings.SplitN(last, " ", 3)[2]), &ch); err != nil || ch.Eval == nil ||
		ch.Eval.Suite != "arithmetic" || ch.Eval.MinScore != 0.9 {
		t.Errorf("sent %s", last)
	}
	if _, err := ctl("model", "update", "qwen", "--min-score", "0.9"); err == nil {
		t.Error("--min-score without --eval was sent")
	}
}