llmgw/internal/cli/ctl_eval.go

package cli

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/fs"
	"log/slog"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/spf13/cobra"

	llmgw "gitea.mynodes.xyz/ilker/llmgw"
	"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
	"gitea.mynodes.xyz/ilker/llmgw/internal/config"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
	"gitea.mynodes.xyz/ilker/llmgw/internal/router"
)

// newEvalService is the control plane's runner of quality suites: through
// the router as its own requests, with the suites of control.eval_dir and
// those the binary carries, kept in the store when there is one.
func newEvalService(cfg config.Server, rt *router.Router, admin *control.Controller, log *slog.Logger) *eval.Service {
	builtin, err := fs.Sub(llmgw.EvalSuites, "deploy/eval")
	if err != nil {
		builtin = nil
	}
	d := eval.Deps{Call: rt.Replay, Dir: cfg.Control.EvalDir, Builtin: builtin, Log: log,
		Origin: func(name string) (string, string) {
			if m, ok := admin.Catalog().Resolve(name); ok && m.Origin != nil {
				return m.Origin.Repo, m.Origin.Revision
			}
			return "", ""
		},
		CanaryOf: func(name string) string { return admin.Catalog().CanaryOf(name) },
	}
	// Only a store that is there: a nil one in the interface would look
	// like one.
	if st := rt.Store(); st != nil {
		d.Store = st
	}
	return eval.NewService(d)
}

// suiteRequest is a suite named on the command line: a file here, sent
// whole, or else the name of one on the control plane.
func suiteRequest(arg string) (eval.Request, error) {
	if st, err := os.Stat(arg); err == nil && !st.IsDir() {
		b, err := os.ReadFile(arg)
		if err != nil {
			return eval.Request{}, err
		}
		// Read here first, so a mistake in it is said before anything is sent.
		if _, err := eval.Parse(filepath.Base(arg), b); err != nil {
			return eval.Request{}, err
		}
		return eval.Request{Content: string(b), File: filepath.Base(arg)}, nil
	}
	if strings.ContainsAny(arg, `/\`) {
		return eval.Request{}, fmt.Errorf("%s: no such file here", arg)
	}
	return eval.Request{Suite: arg}, nil
}

func newEvalCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
	var (
		suite, judge, run       string
		history, suites, detach bool
		concurrency, limit      int
		minScore                float64
	)
	c := &cobra.Command{
		Use:   "eval MODEL --suite FILE|NAME",
		Short: "Ask a model a quality suite and score its answers",
		Long: "A suite is a set of cases — a prompt or a conversation, and how its answer\n" +
			"is scored: exact, contains, regex, a number within a tolerance, JSON that\n" +
			"satisfies a schema, or a rubric another model on the cluster grades — with\n" +
			"a weight each and a score, 0 to 1, the suite is passed at (docs/install.md,\n" +
			"\"Quality suites\").\n\n" +
			"It runs on the control plane, which asks the model through the router as\n" +
			"its own requests, as a replay does: past admission, quotas, accounting and\n" +
			"the guardrails, so the score is the weights' and not a tenant's limits'.\n" +
			"--suite is a file here, sent whole, or the name of a suite in the control\n" +
			"plane's control.eval_dir (/etc/llmgw/eval) or built in: arithmetic,\n" +
			"json-format. Every run is kept, with the revision of the weights that\n" +
			"answered; --history lists them.\n\n" +
			"It exits non-zero when the model scores below the suite's threshold, or\n" +
			"--min-score.",
		Example: "  llmgw ctl eval qwen-7b --suite arithmetic\n" +
			"  llmgw ctl eval qwen-7b --suite ./support.yaml --judge llama-70b --concurrency 8\n" +
			"  llmgw ctl eval --history qwen-7b\n" +
			"  llmgw ctl eval --run eval-20261004-101500-1a2b3c4d\n" +
			"  llmgw ctl eval --suites",
		Args: cobra.MaximumNArgs(1),
		RunE: func(cmd *cobra.Command, args []string) error {
			out := cmd.OutOrStdout()
			switch {
			case suites:
				var list []eval.SuiteInfo
				if err := client().get(cmd.Context(), "/api/v1/eval/suites", &list); err != nil {
					return err
				}
				if *asJSON {
					return emitJSON(list)
				}
				w := tabTo(out)
				fmt.Fprintln(w, "SUITE\tCASES\tPASS AT\tJUDGE\tFROM\tABOUT")
				for _, s := range list {
					if s.Error != "" {
						fmt.Fprintf(w, "%s\t-\t-\t-\t%s\tcannot be run: %s\n", s.Name, s.From, truncate(s.Error, 70))
						continue
					}
					fmt.Fprintf(w, "%s\t%d\t%.2f\t%s\t%s\t%s\n", s.Name, s.Cases, s.Threshold, yesNo(s.Judge), s.From,
						truncate(s.Description, 60))
				}
				return w.Flush()
			case run != "":
				var l eval.Live
				if err := client().get(cmd.Context(), "/api/v1/eval/runs/"+url.PathEscape(run), &l); err != nil {
					return err
				}
				if l.State == eval.Running && !*asJSON {
					return followEval(cmd, client, l, false)
				}
				return showEval(cmd, l, *asJSON)
			case history:
				q := url.Values{}
				if len(args) == 1 {
					q.Set("model", args[0])
				}
				if limit > 0 {
					q.Set("limit", strconv.Itoa(limit))
				}
				var runs []eval.Live
				if err := client().get(cmd.Context(), "/api/v1/eval/runs?"+q.Encode(), &runs); err != nil {
					return err
				}
				if *asJSON {
					return emitJSON(runs)
				}
				return printEvalHistory(out, runs)
			}
			if len(args) != 1 {
				return errors.New("name the model to ask: llmgw ctl eval MODEL --suite FILE|NAME")
			}
			if suite == "" {
				return errors.New("--suite is required: a file, or a suite's name on the control plane (`llmgw ctl eval --suites`)")
			}
			req, err := suiteRequest(suite)
			if err != nil {
				return err
			}
			req.Concurrency, req.Judge, req.MinScore = concurrency, judge, minScore
			body, _ := json.Marshal(req)
			var l eval.Live
			if err := client().post(cmd.Context(), "/api/v1/models/"+url.PathEscape(args[0])+"/eval", body, &l); err != nil {
				return err
			}
			if detach {
				fmt.Fprintf(out, "started %s on the control plane\nfollow it: llmgw ctl eval --run %s\n", l.ID, l.ID)
				return nil
			}
			return followEval(cmd, client, l, *asJSON)
		},
	}
	f := c.Flags()
	f.StringVar(&suite, "suite", "", "the suite: a file here, or a suite's name on the control plane")
	f.IntVar(&concurrency, "concurrency", 4, "cases asked at once")
	f.StringVar(&judge, "judge", "", "the model that grades rubric cases, instead of the suite's judge")
	f.Float64Var(&minScore, "min-score", 0, "the score to pass at, 0 to 1, instead of the suite's threshold")
	f.BoolVar(&detach, "detach", false, "start it and return: it runs on the control plane")
	f.BoolVar(&history, "history", false, "list the runs kept, of MODEL or of every model")
	f.IntVar(&limit, "limit", 20, "with --history, how many runs")
	f.StringVar(&run, "run", "", "show one run, by its id, with each case")
	f.BoolVar(&suites, "suites", false, "list the suites the control plane can run by name")
	return c
}

func yesNo(b bool) string {
	if b {
		return "yes"
	}
	return "-"
}

// followEval waits for a run, saying how far it has got, then shows it.
func followEval(cmd *cobra.Command, client func() *ctlClient, l eval.Live, asJSON bool) error {
	out := cmd.OutOrStdout()
	if !asJSON {
		fmt.Fprintf(out, "run %s: %s asked %s, %d cases\n", l.ID, evalTarget(l), l.Suite, l.Cases)
	}
	shown := -1
	for l.State == eval.Running {
		if !asJSON && l.Done != shown {
			fmt.Fprintf(out, "  %d of %d answered\n", l.Done, l.Cases)
			shown = l.Done
		}
		select {
		case <-cmd.Context().Done():
			return cmd.Context().Err()
		case <-time.After(time.Second):
		}
		if err := client().get(cmd.Context(), "/api/v1/eval/runs/"+url.PathEscape(l.ID), &l); err != nil {
			return err
		}
	}
	if !asJSON {
		fmt.Fprintln(out)
	}
	return showEval(cmd, l, asJSON)
}

func evalTarget(l eval.Live) string {
	if l.Target != "" {
		return l.Target
	}
	return l.Model
}

// showEval prints a finished run's cases and score, and fails below it.
func showEval(cmd *cobra.Command, l eval.Live, asJSON bool) error {
	out := cmd.OutOrStdout()
	if asJSON {
		if err := emitJSON(l); err != nil {
			return err
		}
	} else {
		if l.State == eval.Failed {
			return fmt.Errorf("the run could not be scored: %s", l.Error)
		}
		w := tabTo(out)
		fmt.Fprintln(w, "CASE\tKIND\tWEIGHT\tSCORE\tTIME\tANSWER\tWHY")
		for _, r := range l.Results {
			score := fmt.Sprintf("%.2f", r.Score)
			if !r.Passed {
				score += " ✗"
			}
			why := r.Detail
			if r.Error != "" {
				why = "error: " + r.Error
			}
			fmt.Fprintf(w, "%s\t%s\t%g\t%s\t%s\t%s\t%s\n", r.ID, r.Kind, r.Weight, score,
				(time.Duration(r.TookMs) * time.Millisecond).Round(10*time.Millisecond), dash(truncate(oneLine(r.Answer), 30)),
				truncate(why, 70))
		}
		if err := w.Flush(); err != nil {
			return err
		}
		fmt.Fprintln(out)
		fmt.Fprintln(out, evalSummary(l))
		if l.Error != "" {
			fmt.Fprintln(out, l.Error)
		}
	}
	if !l.Passed {
		return fmt.Errorf("%s scored %.2f on %s, below %.2f", evalTarget(l), l.Score, l.Suite, l.Threshold)
	}
	return nil
}

func oneLine(s string) string { return strings.Join(strings.Fields(s), " ") }

// evalSummary is a run in a sentence.
func evalSummary(l eval.Live) string {
	verdict := "passed"
	if !l.Passed {
		verdict = "failed"
	}
	s := fmt.Sprintf("%s scored %.2f on %s: %d of %d cases passed, %.2f asked: %s", evalTarget(l), l.Score, l.Suite,
		l.Cases-l.Below, l.Cases, l.Threshold, verdict)
	if l.Revision != "" {
		s += fmt.Sprintf(" (weights %s at %s)", dash(l.Repo), catalog.ShortRevision(l.Revision))
	}
	return s + "."
}

func printEvalHistory(out io.Writer, runs []eval.Live) error {
	if len(runs) == 0 {
		fmt.Fprintln(out, "No runs kept: `llmgw ctl eval MODEL --suite NAME` makes one.")
		return nil
	}
	w := tabTo(out)
	fmt.Fprintln(w, "WHEN\tRUN\tMODEL\tWEIGHTS\tSUITE\tSCORE\tPASS AT\tRESULT\tFOR")
	for _, r := range runs {
		model := r.Model
		if r.Target != "" {
			model += " (" + r.Target + ")"
		}
		result := "passed"
		switch {
		case r.State == eval.Running:
			result = fmt.Sprintf("running, %d of %d", r.Done, r.Cases)
		case r.State == eval.Failed:
			result = "could not run"
		case !r.Passed:
			result = "failed"
		}
		score := fmt.Sprintf("%.2f", r.Score)
		if r.State == eval.Running {
			score = "-"
		}
		forWhat := r.Job
		if forWhat == "" {
			forWhat = r.By
		}
		fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%.2f\t%s\t%s\n", ago(r.At, time.Now()), r.ID, model,
			dash(catalog.ShortRevision(r.Revision)), r.Suite, score, r.Threshold, result, dash(forWhat))
	}
	return w.Flush()
}

// gateFlags are how an update or a promotion is held to a quality suite.
type gateFlags struct {
	suite, judge string
	minScore     float64
	skip         bool
}

func (g *gateFlags) add(c *cobra.Command) {
	f := c.Flags()
	f.StringVar(&g.suite, "eval", "", "a quality suite the new weights must pass first: a file here, or a suite's name on the control plane")
	f.Float64Var(&g.minScore, "min-score", 0, "with --eval, the score to pass at, 0 to 1; the suite's threshold otherwise")
	f.StringVar(&g.judge, "judge", "", "with --eval, the model that grades rubric cases")
	f.BoolVar(&g.skip, "skip-eval", false, "move without the model's own quality suite (`llmgw ctl model gate`)")
}

// body is the change, as the API takes it.
func (g gateFlags) body() ([]byte, error) {
	var ch modelserve.Change
	if g.suite != "" {
		if g.skip {
			return nil, errors.New("--eval and --skip-eval together: give one")
		}
		r, err := suiteRequest(g.suite)
		if err != nil {
			return nil, err
		}
		r.MinScore, r.Judge = g.minScore, g.judge
		ch.Eval = &r
	} else if g.minScore != 0 || g.judge != "" {
		return nil, errors.New("--min-score and --judge go with --eval")
	}
	ch.SkipEval = g.skip
	return json.Marshal(ch)
}

func newModelGateCmd(client func() *ctlClient) *cobra.Command {
	var (
		g   adminapi.EvalGateRequest
		off bool
	)
	c := &cobra.Command{
		Use:   "gate MODEL --eval SUITE [--min-score N]",
		Short: "Hold a model's updates and promotions to a quality suite",
		Long: "From then on `llmgw ctl model update` and `model promote` (and the\n" +
			"console's) ask the new weights the suite before they replace the old,\n" +
			"and stop below its score: an update runs them first as a canary given\n" +
			"none of the model's requests, and takes it out again when it fails. The\n" +
			"suite is one on the control plane, by name: in control.eval_dir, or\n" +
			"built in. --skip-eval on an update moves without it, once; --off takes it\n" +
			"away.",
		Example: "  llmgw ctl model gate qwen-7b --eval arithmetic --min-score 0.8\n" +
			"  llmgw ctl model gate qwen-7b --off",
		Args: cobra.ExactArgs(1),
		RunE: func(cmd *cobra.Command, args []string) error {
			path := "/api/v1/models/" + url.PathEscape(args[0]) + "/eval-gate"
			var out struct {
				Model string            `json:"model"`
				Eval  *catalog.EvalGate `json:"eval"`
			}
			if off {
				if err := client().delete(cmd.Context(), path, &out); err != nil {
					return err
				}
				fmt.Fprintf(cmd.OutOrStdout(), "%s's updates are no longer held to a quality suite\n", args[0])
				return nil
			}
			if g.Suite == "" {
				return errors.New("--eval names the suite, or --off takes the gate away")
			}
			body, _ := json.Marshal(g)
			if err := client().put(cmd.Context(), path, body, &out); err != nil {
				return err
			}
			at := "the suite's threshold"
			if g.MinScore > 0 {
				at = strconv.FormatFloat(g.MinScore, 'f', -1, 64)
			}
			fmt.Fprintf(cmd.OutOrStdout(), "%s's new weights must score at least %s on %s before they replace the old\n",
				args[0], at, g.Suite)
			return nil
		},
	}
	c.Flags().StringVar(&g.Suite, "eval", "", "the suite: its name on the control plane")
	c.Flags().Float64Var(&g.MinScore, "min-score", 0, "the score to pass at, 0 to 1; the suite's threshold otherwise")
	c.Flags().StringVar(&g.Judge, "judge", "", "the model that grades the suite's rubric cases")
	c.Flags().BoolVar(&off, "off", false, "take the gate away")
	return c
}