llmgw/internal/cli/tui_quality.go

package cli

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"slices"
	"strconv"
	"strings"
	"time"

	tea "github.com/charmbracelet/bubbletea"

	"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
)

// Quality suites from the model menu: ask a model one of the suites the
// control plane has (its eval directory's, and those built in) and watch
// its score come in; list its past runs; hold its updates to one — the same
// as `llmgw ctl eval` and `llmgw ctl model gate`.

// evalSuitesMsg is the suites the control plane can run, fetched to open
// the form that runs one (or, with gate, the one that sets a model's).
type evalSuitesMsg struct {
	model  adminapi.ModelView
	suites []eval.SuiteInfo
	gate   bool
	err    error
}

// evalPollMsg is a run still going, to be asked about again.
type evalPollMsg struct {
	title string
	live  eval.Live
	err   error
}

// gateNote says, in an update's or a promotion's question, that the new
// weights are asked the model's suite first.
func gateNote(md adminapi.ModelView) string {
	g := md.Eval
	if g == nil {
		return ""
	}
	at := "its threshold"
	if g.MinScore > 0 {
		at = fmt.Sprintf("%.2f", g.MinScore)
	}
	return fmt.Sprintf("\n\nFirst the new weights are asked the quality suite %s, and must score %s;\n"+
		"below it nothing moves.", g.Suite, at)
}

// evalItems are the model menu's quality entries.
func (m consoleModel) evalItems(md adminapi.ModelView) []menuItem {
	gate := "Hold its updates to a quality suite…"
	if g := md.Eval; g != nil {
		gate = "Its updates' quality suite (" + g.Suite + ")…"
	}
	return []menuItem{
		{"Ask it a quality suite…", func(m *consoleModel) tea.Cmd { return m.fetchSuites(md, false) }},
		{"Its quality runs", func(m *consoleModel) tea.Cmd {
			return m.call("Quality runs of "+md.Name, http.MethodGet,
				"/api/v1/eval/runs?limit=15&model="+url.QueryEscape(md.Name), nil, func(raw json.RawMessage) string {
					var runs []eval.Live
					if err := json.Unmarshal(raw, &runs); err != nil {
						return string(raw)
					}
					return evalRunsText(runs)
				})
		}},
		{gate, func(m *consoleModel) tea.Cmd { return m.fetchSuites(md, true) }},
	}
}

func (m *consoleModel) fetchSuites(md adminapi.ModelView, gate bool) tea.Cmd {
	m.busy = "Quality suites"
	c := m.client
	return func() tea.Msg {
		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
		defer cancel()
		var list []eval.SuiteInfo
		err := c.get(ctx, "/api/v1/eval/suites", &list)
		return evalSuitesMsg{model: md, suites: list, gate: gate, err: err}
	}
}

func (m consoleModel) onEvalSuites(msg evalSuitesMsg) (tea.Model, tea.Cmd) {
	m.busy = ""
	md := msg.model
	var names []string
	for _, s := range msg.suites {
		if s.Error == "" {
			names = append(names, s.Name)
		}
	}
	if msg.err == nil && len(names) == 0 {
		msg.err = errors.New("the control plane has no suite it can run: put one in its control.eval_dir " +
			"(/etc/llmgw/eval); docs/install.md, \"Quality suites\", says how one is written")
	}
	if msg.err != nil {
		m.dialog, m.dialogTitle = "error", "Quality suites"
		m.dialogText = sBad.Render("✗ ") + msg.err.Error() + "\n\n" + sDim.Render("any key")
		return m, nil
	}
	about := func(name string) string {
		for _, s := range msg.suites {
			if s.Name == name {
				return fmt.Sprintf("%s: %d cases, passed at %.2f, %s", s.Name, s.Cases, s.Threshold, s.From)
			}
		}
		return ""
	}
	path := "/api/v1/models/" + url.PathEscape(md.Name)
	if msg.gate {
		current, minScore := "none", ""
		if g := md.Eval; g != nil {
			current = g.Suite
			if g.MinScore > 0 {
				minScore = strconv.FormatFloat(g.MinScore, 'f', -1, 64)
			}
			if !slices.Contains(names, current) {
				names = append(names, current)
			}
		}
		m.openForm(&formDialog{title: "Quality gate of " + md.Name,
			note: "An update or a promotion asks the new weights this suite first, and stops below its score;\n" +
				"an update tries them as a canary given none of " + md.Name + "'s requests, and takes it out again.",
			fields: []formField{
				choiceF("suite", append([]string{"none"}, names...), current),
				textF("min score", minScore, "0 to 1; empty is the suite's own threshold"),
			},
			submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
				if v[0] == "none" {
					return m.call("Quality gate of "+md.Name, http.MethodDelete, path+"/eval-gate", nil,
						func(json.RawMessage) string { return md.Name + "'s updates are no longer held to a suite." }), nil
				}
				score, err := minScoreOf(v[1])
				if err != nil {
					return nil, err
				}
				return m.call("Quality gate of "+md.Name, http.MethodPut, path+"/eval-gate",
					adminapi.EvalGateRequest{Suite: v[0], MinScore: score}, func(json.RawMessage) string {
						return fmt.Sprintf("%s's new weights must pass %s before they replace the old.", md.Name, v[0])
					}), nil
			}})
		return m, nil
	}
	m.openForm(&formDialog{title: "Ask " + md.Name + " a quality suite",
		note: "Asked on the control plane, through the router as its own requests: past quotas and guardrails.\n" +
			about(names[0]),
		fields: []formField{
			choiceF("suite", names, names[0]),
			textF("at once", "4", "cases asked at the same time"),
			textF("min score", "", "0 to 1; empty is the suite's own threshold"),
		},
		submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
			n, err := strconv.Atoi(v[1])
			if err != nil || n < 1 {
				return nil, fmt.Errorf("at once is a whole number, 1 or more")
			}
			score, err := minScoreOf(v[2])
			if err != nil {
				return nil, err
			}
			return m.startEval(md.Name, eval.Request{Suite: v[0], Concurrency: n, MinScore: score}), nil
		}})
	return m, nil
}

func minScoreOf(s string) (float64, error) {
	if s == "" {
		return 0, nil
	}
	f, err := strconv.ParseFloat(s, 64)
	if err != nil || f < 0 || f > 1 {
		return 0, fmt.Errorf("the min score is a number from 0 to 1")
	}
	return f, nil
}

// startEval starts a run and follows it.
func (m *consoleModel) startEval(model string, req eval.Request) tea.Cmd {
	title := model + ": " + req.Suite
	m.busy, m.busyNote = title, "starting…"
	c := m.client
	return func() tea.Msg {
		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
		defer cancel()
		var l eval.Live
		err := c.do(ctx, http.MethodPost, "/api/v1/models/"+url.PathEscape(model)+"/eval", req, &l)
		if err != nil {
			return actionDoneMsg{title: title, err: err}
		}
		return evalPollMsg{title: title, live: l}
	}
}

func (m consoleModel) onEvalPoll(msg evalPollMsg) (tea.Model, tea.Cmd) {
	if m.evalWatch != "" && m.evalWatch != msg.live.ID {
		return m, nil
	}
	if m.evalWatch == "" && m.busy != msg.title {
		// Esc: no longer followed here; it carries on on the control plane.
		return m, nil
	}
	m.evalWatch = msg.live.ID
	m.busy = msg.title
	m.busyNote = fmt.Sprintf("%d of %d answered · Esc stops watching (it carries on)", msg.live.Done, msg.live.Cases)
	c, id, title := m.client, msg.live.ID, msg.title
	return m, func() tea.Msg {
		time.Sleep(time.Second)
		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
		defer cancel()
		var l eval.Live
		if err := c.get(ctx, "/api/v1/eval/runs/"+url.PathEscape(id), &l); err != nil {
			return actionDoneMsg{title: title, err: err}
		}
		if l.State == eval.Running {
			return evalPollMsg{title: title, live: l}
		}
		var runs []eval.Live
		_ = c.get(ctx, "/api/v1/eval/runs?limit=6&model="+url.QueryEscape(l.Model), &runs)
		text := evalRunText(l)
		if len(runs) > 1 {
			text += "\n\nIts last runs:\n" + evalRunsText(runs)
		}
		if l.State == eval.Failed || !l.Passed {
			return actionDoneMsg{title: title, err: errors.New(text)}
		}
		return actionDoneMsg{title: title, text: text}
	}
}

// evalRunText is a finished run: its score, then each case.
func evalRunText(l eval.Live) string {
	if l.State == eval.Failed {
		return "The run could not be scored: " + l.Error
	}
	var b strings.Builder
	b.WriteString(evalSummary(l) + "\n")
	if l.Error != "" {
		b.WriteString(l.Error + "\n")
	}
	b.WriteString("\n")
	shown := min(len(l.Results), 14)
	for _, r := range l.Results[:shown] {
		mark := sOK.Render("✓")
		if !r.Passed {
			mark = sBad.Render("✗")
		}
		why := r.Detail
		if r.Error != "" {
			why = r.Error
		}
		fmt.Fprintf(&b, "%s %-18s %4.2f  %s\n", mark, truncate(r.ID, 18), r.Score, truncate(why, 60))
	}
	if len(l.Results) > shown {
		fmt.Fprintf(&b, "  and %d more: llmgw ctl eval --run %s\n", len(l.Results)-shown, l.ID)
	}
	return strings.TrimRight(b.String(), "\n")
}

// evalRunsText is a model's runs, one a line.
func evalRunsText(runs []eval.Live) string {
	if len(runs) == 0 {
		return "No runs kept yet: \"Ask it a quality suite\" on this menu makes one."
	}
	var b strings.Builder
	for _, r := range runs {
		result := sOK.Render("passed")
		switch {
		case r.State == eval.Running:
			result = fmt.Sprintf("running, %d of %d", r.Done, r.Cases)
		case r.State == eval.Failed:
			result = sBad.Render("could not run")
		case !r.Passed:
			result = sBad.Render("failed")
		}
		who := r.Model
		if r.Target != "" {
			who = r.Target
		}
		fmt.Fprintf(&b, "%-9s %-14s %-16s %4.2f of %4.2f  %s\n", ageShort(r.At), truncate(r.Suite, 14),
			truncate(who, 16), r.Score, r.Threshold, result)
	}
	return strings.TrimRight(b.String(), "\n")
}