package cli import ( "encoding/json" "io" "net/http" "net/http/httptest" "strings" "sync" "testing" tea "github.com/charmbracelet/bubbletea" "gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) func TestAModelIsAskedASuiteFromItsMenuAndItsScoreShown(t *testing.T) { var mu sync.Mutex var asked eval.Request polls := 0 done := eval.Live{EvalRun: store.EvalRun{ID: "eval-1", Model: "qwen", Suite: "arithmetic", Score: 0.75, Threshold: 0.7, Passed: true, Cases: 2, Below: 1, Results: []store.EvalCase{ {ID: "add", Kind: "number", Score: 1, Passed: true, Detail: "read 42"}, {ID: "mul", Kind: "number", Score: 0, Detail: "read 43, wanted 42"}}}, State: eval.Done, Done: 2} admin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { mu.Lock() defer mu.Unlock() switch { case r.URL.Path == "/api/v1/eval/suites": _ = json.NewEncoder(w).Encode([]eval.SuiteInfo{ {Name: "arithmetic", From: "built in", Cases: 2, Threshold: 0.7}, {Name: "broken", From: "/etc/llmgw/eval/broken.yaml", Error: "no cases"}, {Name: "json-format", From: "built in", Cases: 7, Threshold: 0.7}}) case r.Method == http.MethodPost && r.URL.Path == "/api/v1/models/qwen/eval": b, _ := io.ReadAll(r.Body) _ = json.Unmarshal(b, &asked) w.WriteHeader(http.StatusAccepted) _ = json.NewEncoder(w).Encode(eval.Live{EvalRun: store.EvalRun{ID: "eval-1", Model: "qwen", Suite: "arithmetic", Cases: 2}, State: eval.Running}) case r.URL.Path == "/api/v1/eval/runs/eval-1": polls++ if polls == 1 { _ = json.NewEncoder(w).Encode(eval.Live{EvalRun: store.EvalRun{ID: "eval-1", Model: "qwen", Suite: "arithmetic", Cases: 2}, State: eval.Running, Done: 1}) return } _ = json.NewEncoder(w).Encode(done) case r.URL.Path == "/api/v1/eval/runs": older := done older.ID, older.Score, older.Passed = "eval-0", 0.5, false _ = json.NewEncoder(w).Encode([]eval.Live{done, older}) default: http.NotFound(w, r) } })) t.Cleanup(admin.Close) m := newConsoleModel(admin.URL, "/nonexistent/server.yaml") m.loaded, m.menu, m.inPanel = true, cModels, true md := adminapi.ModelView{Name: "qwen"} // run plays a command and what it leads to, the way the program would. run := func(tm tea.Model, cmd tea.Cmd) tea.Model { for cmd != nil { msg := cmd() tm, cmd = tm.Update(msg) if _, ok := msg.(actionDoneMsg); ok { break } } return tm } var tm tea.Model = m items := m.evalItems(md) if len(items) != 3 || !strings.Contains(items[0].label, "quality suite") { t.Fatalf("%+v", items) } tm = run(tm, items[0].act(&m)) f := tm.(consoleModel).formD if f == nil || f.fields[0].options == nil || len(f.fields[0].options) != 2 { t.Fatalf("no form offering the suites that can run: %+v", f) } tm = press(tm, "tab") f.fields[1].text.SetValue("2") tm = press(tm, "tab", "tab") tm, cmd := tm.Update(tea.KeyMsg{Type: tea.KeyEnter}) if busy := tm.(consoleModel).busy; busy != "qwen: arithmetic" { t.Fatalf("busy %q", busy) } // Started, then followed: one poll still running, then its score. tm, next := tm.Update(cmd()) if note := tm.(consoleModel).busyNote; !strings.Contains(note, "0 of 2 answered") { t.Errorf("progress %q", note) } tm, next = tm.Update(next()) if note := tm.(consoleModel).busyNote; !strings.Contains(note, "1 of 2 answered") { t.Errorf("progress %q", note) } tm, _ = tm.Update(next()) view := tm.View() mu.Lock() sent := asked mu.Unlock() if sent.Suite != "arithmetic" || sent.Concurrency != 2 { t.Errorf("asked %+v", sent) } for _, want := range []string{"qwen scored 0.75 on arithmetic", "1 of 2 cases passed", "read 43, wanted 42", "Its last runs"} { if !strings.Contains(view, want) { t.Errorf("the result does not say %q:\n%s", want, view) } } if c := tm.(consoleModel); c.busy != "" || c.evalWatch != "" { t.Errorf("still busy: %q %q", c.busy, c.evalWatch) } }