llmgw/internal/cli/tui_quality_test.go

package cli

import (
	"encoding/json"
	"io"
	"net/http"
	"net/http/httptest"
	"strings"
	"sync"
	"testing"

	tea "github.com/charmbracelet/bubbletea"

	"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)

func TestAModelIsAskedASuiteFromItsMenuAndItsScoreShown(t *testing.T) {
	var mu sync.Mutex
	var asked eval.Request
	polls := 0
	done := eval.Live{EvalRun: store.EvalRun{ID: "eval-1", Model: "qwen", Suite: "arithmetic", Score: 0.75,
		Threshold: 0.7, Passed: true, Cases: 2, Below: 1, Results: []store.EvalCase{
			{ID: "add", Kind: "number", Score: 1, Passed: true, Detail: "read 42"},
			{ID: "mul", Kind: "number", Score: 0, Detail: "read 43, wanted 42"}}}, State: eval.Done, Done: 2}
	admin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		mu.Lock()
		defer mu.Unlock()
		switch {
		case r.URL.Path == "/api/v1/eval/suites":
			_ = json.NewEncoder(w).Encode([]eval.SuiteInfo{
				{Name: "arithmetic", From: "built in", Cases: 2, Threshold: 0.7},
				{Name: "broken", From: "/etc/llmgw/eval/broken.yaml", Error: "no cases"},
				{Name: "json-format", From: "built in", Cases: 7, Threshold: 0.7}})
		case r.Method == http.MethodPost && r.URL.Path == "/api/v1/models/qwen/eval":
			b, _ := io.ReadAll(r.Body)
			_ = json.Unmarshal(b, &asked)
			w.WriteHeader(http.StatusAccepted)
			_ = json.NewEncoder(w).Encode(eval.Live{EvalRun: store.EvalRun{ID: "eval-1", Model: "qwen",
				Suite: "arithmetic", Cases: 2}, State: eval.Running})
		case r.URL.Path == "/api/v1/eval/runs/eval-1":
			polls++
			if polls == 1 {
				_ = json.NewEncoder(w).Encode(eval.Live{EvalRun: store.EvalRun{ID: "eval-1", Model: "qwen",
					Suite: "arithmetic", Cases: 2}, State: eval.Running, Done: 1})
				return
			}
			_ = json.NewEncoder(w).Encode(done)
		case r.URL.Path == "/api/v1/eval/runs":
			older := done
			older.ID, older.Score, older.Passed = "eval-0", 0.5, false
			_ = json.NewEncoder(w).Encode([]eval.Live{done, older})
		default:
			http.NotFound(w, r)
		}
	}))
	t.Cleanup(admin.Close)
	m := newConsoleModel(admin.URL, "/nonexistent/server.yaml")
	m.loaded, m.menu, m.inPanel = true, cModels, true
	md := adminapi.ModelView{Name: "qwen"}

	// run plays a command and what it leads to, the way the program would.
	run := func(tm tea.Model, cmd tea.Cmd) tea.Model {
		for cmd != nil {
			msg := cmd()
			tm, cmd = tm.Update(msg)
			if _, ok := msg.(actionDoneMsg); ok {
				break
			}
		}
		return tm
	}
	var tm tea.Model = m
	items := m.evalItems(md)
	if len(items) != 3 || !strings.Contains(items[0].label, "quality suite") {
		t.Fatalf("%+v", items)
	}
	tm = run(tm, items[0].act(&m))
	f := tm.(consoleModel).formD
	if f == nil || f.fields[0].options == nil || len(f.fields[0].options) != 2 {
		t.Fatalf("no form offering the suites that can run: %+v", f)
	}
	tm = press(tm, "tab")
	f.fields[1].text.SetValue("2")
	tm = press(tm, "tab", "tab")
	tm, cmd := tm.Update(tea.KeyMsg{Type: tea.KeyEnter})
	if busy := tm.(consoleModel).busy; busy != "qwen: arithmetic" {
		t.Fatalf("busy %q", busy)
	}
	// Started, then followed: one poll still running, then its score.
	tm, next := tm.Update(cmd())
	if note := tm.(consoleModel).busyNote; !strings.Contains(note, "0 of 2 answered") {
		t.Errorf("progress %q", note)
	}
	tm, next = tm.Update(next())
	if note := tm.(consoleModel).busyNote; !strings.Contains(note, "1 of 2 answered") {
		t.Errorf("progress %q", note)
	}
	tm, _ = tm.Update(next())
	view := tm.View()
	mu.Lock()
	sent := asked
	mu.Unlock()
	if sent.Suite != "arithmetic" || sent.Concurrency != 2 {
		t.Errorf("asked %+v", sent)
	}
	for _, want := range []string{"qwen scored 0.75 on arithmetic", "1 of 2 cases passed", "read 43, wanted 42", "Its last runs"} {
		if !strings.Contains(view, want) {
			t.Errorf("the result does not say %q:\n%s", want, view)
		}
	}
	if c := tm.(consoleModel); c.busy != "" || c.evalWatch != "" {
		t.Errorf("still busy: %q %q", c.busy, c.evalWatch)
	}
}