Quality suites, and new weights held to one: ctl eval, model update/promote --eval, model gate

A suite is a set of cases in YAML or JSON lines, each scored by exact,
contains, regex, a number within a tolerance, a JSON Schema subset, or a
judge model with a rubric; weighed into one score against a threshold.
internal/bench/eval runs one on the control plane through Router.Replay, as
the gateway's own requests, and keeps every run and its cases in the store
(migration 0028: eval_runs, eval_cases).

- llmgw ctl eval MODEL --suite FILE|NAME, --history, --run, --suites; exits
  non-zero below the score. Suites by name from control.eval_dir, or the
  two built in from deploy/eval (arithmetic, json-format).
- model update and promote take --eval SUITE --min-score N, or a model's
  standing gate (ctl model gate, eval: in the catalog): a promote asks the
  canary first and leaves it below the score; an update tries the new
  weights as a canary given no share, then promotes or removes it.
- The console's model menu: ask a suite with its progress, its runs, its
  gate.
- The audit middleware no longer hands a handler a body cut off at 64 KiB.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 00:04:43 +02:00
1 parent 30f7f3474a
commit f0477f140d
49 files changed
+4563 -46

No files matched your search

+1
View File
@@ -147,6 +147,7 @@ func newCtlCmd() *cobra.Command {
// cert works on files rather than the admin API, so --server does not
// apply to it; it lives here because this is where operators look.
newBenchCmd(client, &asJSON),
newEvalCmd(client, &asJSON),
newStatusCmd(client, &asJSON),
newModelCmd(client, &asJSON),
newScaleCmd(client),
+408
View File
@@ -0,0 +1,408 @@
package cli
import (
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"log/slog"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/spf13/cobra"
llmgw "gitea.mynodes.xyz/ilker/llmgw"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
"gitea.mynodes.xyz/ilker/llmgw/internal/router"
)
// newEvalService is the control plane's runner of quality suites: through
// the router as its own requests, with the suites of control.eval_dir and
// those the binary carries, kept in the store when there is one.
func newEvalService(cfg config.Server, rt *router.Router, admin *control.Controller, log *slog.Logger) *eval.Service {
builtin, err := fs.Sub(llmgw.EvalSuites, "deploy/eval")
if err != nil {
builtin = nil
}
d := eval.Deps{Call: rt.Replay, Dir: cfg.Control.EvalDir, Builtin: builtin, Log: log,
Origin: func(name string) (string, string) {
if m, ok := admin.Catalog().Resolve(name); ok && m.Origin != nil {
return m.Origin.Repo, m.Origin.Revision
}
return "", ""
},
CanaryOf: func(name string) string { return admin.Catalog().CanaryOf(name) },
}
// Only a store that is there: a nil one in the interface would look
// like one.
if st := rt.Store(); st != nil {
d.Store = st
}
return eval.NewService(d)
}
// suiteRequest is a suite named on the command line: a file here, sent
// whole, or else the name of one on the control plane.
func suiteRequest(arg string) (eval.Request, error) {
if st, err := os.Stat(arg); err == nil && !st.IsDir() {
b, err := os.ReadFile(arg)
if err != nil {
return eval.Request{}, err
}
// Read here first, so a mistake in it is said before anything is sent.
if _, err := eval.Parse(filepath.Base(arg), b); err != nil {
return eval.Request{}, err
}
return eval.Request{Content: string(b), File: filepath.Base(arg)}, nil
}
if strings.ContainsAny(arg, `/\`) {
return eval.Request{}, fmt.Errorf("%s: no such file here", arg)
}
return eval.Request{Suite: arg}, nil
}
func newEvalCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
var (
suite, judge, run string
history, suites, detach bool
concurrency, limit int
minScore float64
)
c := &cobra.Command{
Use: "eval MODEL --suite FILE|NAME",
Short: "Ask a model a quality suite and score its answers",
Long: "A suite is a set of cases — a prompt or a conversation, and how its answer\n" +
"is scored: exact, contains, regex, a number within a tolerance, JSON that\n" +
"satisfies a schema, or a rubric another model on the cluster grades — with\n" +
"a weight each and a score, 0 to 1, the suite is passed at (docs/install.md,\n" +
"\"Quality suites\").\n\n" +
"It runs on the control plane, which asks the model through the router as\n" +
"its own requests, as a replay does: past admission, quotas, accounting and\n" +
"the guardrails, so the score is the weights' and not a tenant's limits'.\n" +
"--suite is a file here, sent whole, or the name of a suite in the control\n" +
"plane's control.eval_dir (/etc/llmgw/eval) or built in: arithmetic,\n" +
"json-format. Every run is kept, with the revision of the weights that\n" +
"answered; --history lists them.\n\n" +
"It exits non-zero when the model scores below the suite's threshold, or\n" +
"--min-score.",
Example: " llmgw ctl eval qwen-7b --suite arithmetic\n" +
" llmgw ctl eval qwen-7b --suite ./support.yaml --judge llama-70b --concurrency 8\n" +
" llmgw ctl eval --history qwen-7b\n" +
" llmgw ctl eval --run eval-20261004-101500-1a2b3c4d\n" +
" llmgw ctl eval --suites",
Args: cobra.MaximumNArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
out := cmd.OutOrStdout()
switch {
case suites:
var list []eval.SuiteInfo
if err := client().get(cmd.Context(), "/api/v1/eval/suites", &list); err != nil {
return err
}
if *asJSON {
return emitJSON(list)
}
w := tabTo(out)
fmt.Fprintln(w, "SUITE\tCASES\tPASS AT\tJUDGE\tFROM\tABOUT")
for _, s := range list {
if s.Error != "" {
fmt.Fprintf(w, "%s\t-\t-\t-\t%s\tcannot be run: %s\n", s.Name, s.From, truncate(s.Error, 70))
continue
}
fmt.Fprintf(w, "%s\t%d\t%.2f\t%s\t%s\t%s\n", s.Name, s.Cases, s.Threshold, yesNo(s.Judge), s.From,
truncate(s.Description, 60))
}
return w.Flush()
case run != "":
var l eval.Live
if err := client().get(cmd.Context(), "/api/v1/eval/runs/"+url.PathEscape(run), &l); err != nil {
return err
}
if l.State == eval.Running && !*asJSON {
return followEval(cmd, client, l, false)
}
return showEval(cmd, l, *asJSON)
case history:
q := url.Values{}
if len(args) == 1 {
q.Set("model", args[0])
}
if limit > 0 {
q.Set("limit", strconv.Itoa(limit))
}
var runs []eval.Live
if err := client().get(cmd.Context(), "/api/v1/eval/runs?"+q.Encode(), &runs); err != nil {
return err
}
if *asJSON {
return emitJSON(runs)
}
return printEvalHistory(out, runs)
}
if len(args) != 1 {
return errors.New("name the model to ask: llmgw ctl eval MODEL --suite FILE|NAME")
}
if suite == "" {
return errors.New("--suite is required: a file, or a suite's name on the control plane (`llmgw ctl eval --suites`)")
}
req, err := suiteRequest(suite)
if err != nil {
return err
}
req.Concurrency, req.Judge, req.MinScore = concurrency, judge, minScore
body, _ := json.Marshal(req)
var l eval.Live
if err := client().post(cmd.Context(), "/api/v1/models/"+url.PathEscape(args[0])+"/eval", body, &l); err != nil {
return err
}
if detach {
fmt.Fprintf(out, "started %s on the control plane\nfollow it: llmgw ctl eval --run %s\n", l.ID, l.ID)
return nil
}
return followEval(cmd, client, l, *asJSON)
},
}
f := c.Flags()
f.StringVar(&suite, "suite", "", "the suite: a file here, or a suite's name on the control plane")
f.IntVar(&concurrency, "concurrency", 4, "cases asked at once")
f.StringVar(&judge, "judge", "", "the model that grades rubric cases, instead of the suite's judge")
f.Float64Var(&minScore, "min-score", 0, "the score to pass at, 0 to 1, instead of the suite's threshold")
f.BoolVar(&detach, "detach", false, "start it and return: it runs on the control plane")
f.BoolVar(&history, "history", false, "list the runs kept, of MODEL or of every model")
f.IntVar(&limit, "limit", 20, "with --history, how many runs")
f.StringVar(&run, "run", "", "show one run, by its id, with each case")
f.BoolVar(&suites, "suites", false, "list the suites the control plane can run by name")
return c
}
func yesNo(b bool) string {
if b {
return "yes"
}
return "-"
}
// followEval waits for a run, saying how far it has got, then shows it.
func followEval(cmd *cobra.Command, client func() *ctlClient, l eval.Live, asJSON bool) error {
out := cmd.OutOrStdout()
if !asJSON {
fmt.Fprintf(out, "run %s: %s asked %s, %d cases\n", l.ID, evalTarget(l), l.Suite, l.Cases)
}
shown := -1
for l.State == eval.Running {
if !asJSON && l.Done != shown {
fmt.Fprintf(out, " %d of %d answered\n", l.Done, l.Cases)
shown = l.Done
}
select {
case <-cmd.Context().Done():
return cmd.Context().Err()
case <-time.After(time.Second):
}
if err := client().get(cmd.Context(), "/api/v1/eval/runs/"+url.PathEscape(l.ID), &l); err != nil {
return err
}
}
if !asJSON {
fmt.Fprintln(out)
}
return showEval(cmd, l, asJSON)
}
func evalTarget(l eval.Live) string {
if l.Target != "" {
return l.Target
}
return l.Model
}
// showEval prints a finished run's cases and score, and fails below it.
func showEval(cmd *cobra.Command, l eval.Live, asJSON bool) error {
out := cmd.OutOrStdout()
if asJSON {
if err := emitJSON(l); err != nil {
return err
}
} else {
if l.State == eval.Failed {
return fmt.Errorf("the run could not be scored: %s", l.Error)
}
w := tabTo(out)
fmt.Fprintln(w, "CASE\tKIND\tWEIGHT\tSCORE\tTIME\tANSWER\tWHY")
for _, r := range l.Results {
score := fmt.Sprintf("%.2f", r.Score)
if !r.Passed {
score += " ✗"
}
why := r.Detail
if r.Error != "" {
why = "error: " + r.Error
}
fmt.Fprintf(w, "%s\t%s\t%g\t%s\t%s\t%s\t%s\n", r.ID, r.Kind, r.Weight, score,
(time.Duration(r.TookMs) * time.Millisecond).Round(10*time.Millisecond), dash(truncate(oneLine(r.Answer), 30)),
truncate(why, 70))
}
if err := w.Flush(); err != nil {
return err
}
fmt.Fprintln(out)
fmt.Fprintln(out, evalSummary(l))
if l.Error != "" {
fmt.Fprintln(out, l.Error)
}
}
if !l.Passed {
return fmt.Errorf("%s scored %.2f on %s, below %.2f", evalTarget(l), l.Score, l.Suite, l.Threshold)
}
return nil
}
func oneLine(s string) string { return strings.Join(strings.Fields(s), " ") }
// evalSummary is a run in a sentence.
func evalSummary(l eval.Live) string {
verdict := "passed"
if !l.Passed {
verdict = "failed"
}
s := fmt.Sprintf("%s scored %.2f on %s: %d of %d cases passed, %.2f asked: %s", evalTarget(l), l.Score, l.Suite,
l.Cases-l.Below, l.Cases, l.Threshold, verdict)
if l.Revision != "" {
s += fmt.Sprintf(" (weights %s at %s)", dash(l.Repo), catalog.ShortRevision(l.Revision))
}
return s + "."
}
func printEvalHistory(out io.Writer, runs []eval.Live) error {
if len(runs) == 0 {
fmt.Fprintln(out, "No runs kept: `llmgw ctl eval MODEL --suite NAME` makes one.")
return nil
}
w := tabTo(out)
fmt.Fprintln(w, "WHEN\tRUN\tMODEL\tWEIGHTS\tSUITE\tSCORE\tPASS AT\tRESULT\tFOR")
for _, r := range runs {
model := r.Model
if r.Target != "" {
model += " (" + r.Target + ")"
}
result := "passed"
switch {
case r.State == eval.Running:
result = fmt.Sprintf("running, %d of %d", r.Done, r.Cases)
case r.State == eval.Failed:
result = "could not run"
case !r.Passed:
result = "failed"
}
score := fmt.Sprintf("%.2f", r.Score)
if r.State == eval.Running {
score = "-"
}
forWhat := r.Job
if forWhat == "" {
forWhat = r.By
}
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%.2f\t%s\t%s\n", ago(r.At, time.Now()), r.ID, model,
dash(catalog.ShortRevision(r.Revision)), r.Suite, score, r.Threshold, result, dash(forWhat))
}
return w.Flush()
}
// gateFlags are how an update or a promotion is held to a quality suite.
type gateFlags struct {
suite, judge string
minScore float64
skip bool
}
func (g *gateFlags) add(c *cobra.Command) {
f := c.Flags()
f.StringVar(&g.suite, "eval", "", "a quality suite the new weights must pass first: a file here, or a suite's name on the control plane")
f.Float64Var(&g.minScore, "min-score", 0, "with --eval, the score to pass at, 0 to 1; the suite's threshold otherwise")
f.StringVar(&g.judge, "judge", "", "with --eval, the model that grades rubric cases")
f.BoolVar(&g.skip, "skip-eval", false, "move without the model's own quality suite (`llmgw ctl model gate`)")
}
// body is the change, as the API takes it.
func (g gateFlags) body() ([]byte, error) {
var ch modelserve.Change
if g.suite != "" {
if g.skip {
return nil, errors.New("--eval and --skip-eval together: give one")
}
r, err := suiteRequest(g.suite)
if err != nil {
return nil, err
}
r.MinScore, r.Judge = g.minScore, g.judge
ch.Eval = &r
} else if g.minScore != 0 || g.judge != "" {
return nil, errors.New("--min-score and --judge go with --eval")
}
ch.SkipEval = g.skip
return json.Marshal(ch)
}
func newModelGateCmd(client func() *ctlClient) *cobra.Command {
var (
g adminapi.EvalGateRequest
off bool
)
c := &cobra.Command{
Use: "gate MODEL --eval SUITE [--min-score N]",
Short: "Hold a model's updates and promotions to a quality suite",
Long: "From then on `llmgw ctl model update` and `model promote` (and the\n" +
"console's) ask the new weights the suite before they replace the old,\n" +
"and stop below its score: an update runs them first as a canary given\n" +
"none of the model's requests, and takes it out again when it fails. The\n" +
"suite is one on the control plane, by name: in control.eval_dir, or\n" +
"built in. --skip-eval on an update moves without it, once; --off takes it\n" +
"away.",
Example: " llmgw ctl model gate qwen-7b --eval arithmetic --min-score 0.8\n" +
" llmgw ctl model gate qwen-7b --off",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
path := "/api/v1/models/" + url.PathEscape(args[0]) + "/eval-gate"
var out struct {
Model string `json:"model"`
Eval *catalog.EvalGate `json:"eval"`
}
if off {
if err := client().delete(cmd.Context(), path, &out); err != nil {
return err
}
fmt.Fprintf(cmd.OutOrStdout(), "%s's updates are no longer held to a quality suite\n", args[0])
return nil
}
if g.Suite == "" {
return errors.New("--eval names the suite, or --off takes the gate away")
}
body, _ := json.Marshal(g)
if err := client().put(cmd.Context(), path, body, &out); err != nil {
return err
}
at := "the suite's threshold"
if g.MinScore > 0 {
at = strconv.FormatFloat(g.MinScore, 'f', -1, 64)
}
fmt.Fprintf(cmd.OutOrStdout(), "%s's new weights must score at least %s on %s before they replace the old\n",
args[0], at, g.Suite)
return nil
},
}
c.Flags().StringVar(&g.Suite, "eval", "", "the suite: its name on the control plane")
c.Flags().Float64Var(&g.MinScore, "min-score", 0, "the score to pass at, 0 to 1; the suite's threshold otherwise")
c.Flags().StringVar(&g.Judge, "judge", "", "the model that grades the suite's rubric cases")
c.Flags().BoolVar(&off, "off", false, "take the gate away")
return c
}
+123
View File
@@ -0,0 +1,123 @@
package cli
import (
"bytes"
"context"
"encoding/json"
"io"
"io/fs"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync"
"testing"
llmgw "gitea.mynodes.xyz/ilker/llmgw"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
func TestTheSuitesTheBinaryCarriesCanBeRun(t *testing.T) {
sub, err := fs.Sub(llmgw.EvalSuites, "deploy/eval")
if err != nil {
t.Fatal(err)
}
got := eval.NewService(eval.Deps{Builtin: sub}).Suites()
var names []string
for _, s := range got {
if s.Error != "" || s.Cases < 5 || s.Judge {
t.Errorf("%s: %+v", s.Name, s)
}
names = append(names, s.Name)
}
if strings.Join(names, " ") != "arithmetic json-format" {
t.Errorf("built in: %v", names)
}
}
func TestCtlEvalSendsAFileAndFailsBelowTheScore(t *testing.T) {
var mu sync.Mutex
var sent []string
finished := eval.Live{EvalRun: store.EvalRun{ID: "eval-1", Model: "qwen", Suite: "mine", Score: 0.5,
Threshold: 0.8, Cases: 2, Below: 1, Revision: "abcdef1234567", Repo: "o/r", Results: []store.EvalCase{
{ID: "add", Kind: "number", Weight: 1, Score: 1, Passed: true, Answer: "42", Detail: "read 42"},
{ID: "mul", Kind: "number", Weight: 1, Score: 0, Answer: "43", Detail: "read 43, wanted 42"}}},
State: eval.Done, Done: 2}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
b, _ := io.ReadAll(r.Body)
mu.Lock()
sent = append(sent, r.Method+" "+r.URL.Path+" "+string(b))
mu.Unlock()
switch r.URL.Path {
case "/api/v1/models/qwen/eval":
w.WriteHeader(http.StatusAccepted)
_ = json.NewEncoder(w).Encode(eval.Live{EvalRun: store.EvalRun{ID: "eval-1", Model: "qwen", Cases: 2},
State: eval.Running})
case "/api/v1/eval/runs/eval-1":
_ = json.NewEncoder(w).Encode(finished)
case "/api/v1/models/qwen/update":
w.WriteHeader(http.StatusAccepted)
_ = json.NewEncoder(w).Encode(modelserve.Job{ID: "update-1", Model: "qwen", State: modelserve.Done})
default:
http.NotFound(w, r)
}
}))
t.Cleanup(srv.Close)
file := filepath.Join(t.TempDir(), "mine.yaml")
if err := os.WriteFile(file, []byte("cases:\n - prompt: 40 + 2?\n number: 42\n"), 0o600); err != nil {
t.Fatal(err)
}
ctl := func(args ...string) (string, error) {
cmd := newCtlCmd()
var out bytes.Buffer
cmd.SetOut(&out)
cmd.SetErr(&out)
cmd.SetArgs(append([]string{"--server", srv.URL}, args...))
err := cmd.ExecuteContext(context.Background())
return out.String(), err
}
out, err := ctl("eval", "qwen", "--suite", file, "--concurrency", "2")
if err == nil || !strings.Contains(err.Error(), "qwen scored 0.50 on mine, below 0.80") {
t.Fatalf("a run below its score did not fail: %v\n%s", err, out)
}
for _, want := range []string{"read 43, wanted 42", "1 of 2 cases passed", "weights o/r at abcdef1"} {
if !strings.Contains(out, want) {
t.Errorf("the output lacks %q:\n%s", want, out)
}
}
mu.Lock()
var req eval.Request
_ = json.Unmarshal([]byte(strings.SplitN(sent[0], " ", 3)[2]), &req)
mu.Unlock()
if req.File != "mine.yaml" || !strings.Contains(req.Content, "40 + 2?") || req.Concurrency != 2 {
t.Errorf("sent %+v", req)
}
// A broken file is said here, before anything is sent.
if err := os.WriteFile(file, []byte("cases: []\n"), 0o600); err != nil {
t.Fatal(err)
}
if _, err := ctl("eval", "qwen", "--suite", file); err == nil || !strings.Contains(err.Error(), "no cases") {
t.Errorf("%v", err)
}
// An update held to a suite by name carries it.
if _, err := ctl("model", "update", "qwen", "--eval", "arithmetic", "--min-score", "0.9"); err != nil {
t.Fatal(err)
}
mu.Lock()
last := sent[len(sent)-1]
mu.Unlock()
var ch modelserve.Change
if err := json.Unmarshal([]byte(strings.SplitN(last, " ", 3)[2]), &ch); err != nil || ch.Eval == nil ||
ch.Eval.Suite != "arithmetic" || ch.Eval.MinScore != 0.9 {
t.Errorf("sent %s", last)
}
if _, err := ctl("model", "update", "qwen", "--min-score", "0.9"); err == nil {
t.Error("--min-score without --eval was sent")
}
}
+2 -1
View File
@@ -37,7 +37,8 @@ func newModelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
newTargetCmd(client, asJSON), newAdaptersCmd(client),
newModelServeCmd(client), newModelUpdatesCmd(client, asJSON), newModelUpdateCmd(client),
newModelRollbackCmd(client), newModelOriginCmd(client),
newModelCanaryCmd(client), newModelPromoteCmd(client), newModelCanariesCmd(client, asJSON))
newModelCanaryCmd(client), newModelPromoteCmd(client), newModelCanariesCmd(client, asJSON),
newModelGateCmd(client))
return c
}
+16 -3
View File
@@ -83,17 +83,29 @@ func newModelCanaryCmd(client func() *ctlClient) *cobra.Command {
}
func newModelPromoteCmd(client func() *ctlClient) *cobra.Command {
var detach bool
var (
detach bool
gate gateFlags
)
c := &cobra.Command{
Use: "promote MODEL",
Short: "Move a model to its canary's weights, and take the canary out",
Long: "The update the canary was a trial of: the catalog names the canary's\n" +
"files, the model's replicas are replaced one at a time, and the canary\n" +
"is removed. `llmgw ctl model rollback MODEL` goes back.",
"is removed. `llmgw ctl model rollback MODEL` goes back.\n\n" +
"Held to a quality suite (--eval, or the model's own: `llmgw ctl model\n" +
"gate`), the canary is asked it first, and below its score nothing\n" +
"moves: the canary stays, with its share, until it is taken out.",
Example: " llmgw ctl model promote qwen-7b\n" +
" llmgw ctl model promote qwen-7b --eval ./support.yaml --min-score 0.9 --judge llama-70b",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
body, err := gate.body()
if err != nil {
return err
}
var job modelserve.Job
if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/promote", []byte("{}"), &job); err != nil {
if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/promote", body, &job); err != nil {
return err
}
if detach {
@@ -103,6 +115,7 @@ func newModelPromoteCmd(client func() *ctlClient) *cobra.Command {
},
}
c.Flags().BoolVar(&detach, "detach", false, "start it and return: it runs on the control plane; `llmgw ctl jobs` follows it")
gate.add(c)
return c
}
+17 -2
View File
@@ -82,7 +82,10 @@ func updateCell(u modelserve.UpdateInfo) string {
}
func newModelUpdateCmd(client func() *ctlClient) *cobra.Command {
var detach bool
var (
detach bool
gate gateFlags
)
c := &cobra.Command{
Use: "update MODEL",
Short: "Move a model to the weights its repository publishes now",
@@ -90,12 +93,23 @@ func newModelUpdateCmd(client func() *ctlClient) *cobra.Command {
"checked against the registry's checksums and the nodes' llama.cpp; then\n" +
"the catalog names them and the replicas are replaced one at a time, each\n" +
"new one up before an old one stops, and one is asked to say hello.\n\n" +
"Held to a quality suite (--eval, or the model's own: `llmgw ctl model\n" +
"gate`), the new weights first run beside the old as MODEL-canary, given\n" +
"none of its requests, and are asked the suite; below its score they are\n" +
"taken out again and the model keeps its weights. That needs room for one\n" +
"more replica while it runs.\n\n" +
"The old files stay on the mirror: `llmgw ctl model rollback MODEL` goes\n" +
"back to them. A model from server.yaml is changed there instead.",
Example: " llmgw ctl model update qwen-7b\n" +
" llmgw ctl model update qwen-7b --eval arithmetic --min-score 0.8",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
body, err := gate.body()
if err != nil {
return err
}
var job modelserve.Job
if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/update", []byte("{}"), &job); err != nil {
if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/update", body, &job); err != nil {
return err
}
if detach {
@@ -105,6 +119,7 @@ func newModelUpdateCmd(client func() *ctlClient) *cobra.Command {
},
}
c.Flags().BoolVar(&detach, "detach", false, "start it and return: it runs on the control plane; `llmgw ctl jobs` follows it")
gate.add(c)
return c
}
+7
View File
@@ -32,6 +32,7 @@ import (
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/audit"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
@@ -286,6 +287,7 @@ func newServerCmd() *cobra.Command {
var (
adminAPI *adminapi.API
catalogSvc *modelsvc.Service
evals *eval.Service
)
if admin != nil {
api := adminapi.New(admin)
@@ -303,6 +305,9 @@ func newServerCmd() *cobra.Command {
api.SetReleases(cfg.Releases)
api.SetNotifier(notifier)
api.SetReplayer(rt.Replay)
// Quality suites, asked of a model the way a replay is.
evals = newEvalService(cfg, rt, admin, log)
api.SetEval(evals)
if st := rt.Store(); st != nil && gate != nil {
api.SetGuard(&adminapi.GuardAdmin{Library: cfg.Guard, Store: st, Invalidate: gate.Invalidate})
}
@@ -418,6 +423,8 @@ func newServerCmd() *cobra.Command {
Pool: adminAPI.PoolNodes, Notify: notifier,
// Canaries: how the router saw each side serve.
CanaryStats: rt.CanaryStats, ResetCanary: rt.ResetCanary,
// Updates and promotions held to a quality suite.
Eval: evals,
})
go serve.WatchCanaries(ctx, 30*time.Second)
adminAPI.SetServe(serve)
+4 -3
View File
@@ -423,6 +423,7 @@ func (m consoleModel) modelMenu(md adminapi.ModelView) *menuDialog {
}},
}
items = append(items, m.updateItems(md)...)
items = append(items, m.evalItems(md)...)
if md.Parked {
items = append(items, menuItem{"Try again (it is parked)", func(m *consoleModel) tea.Cmd {
return m.call("Retry "+md.Name, http.MethodPost, path+"/retry", map[string]string{}, nil)
@@ -706,7 +707,7 @@ func (m consoleModel) canaryItems(md adminapi.ModelView) []menuItem {
{"Promote the canary…", func(m *consoleModel) tea.Cmd {
m.confirmD = &confirmDialog{title: "Promote " + cn.Model,
text: md.Name + " moves to the canary's weights: its replicas are replaced one at a time,\n" +
"and the canary is removed. Going back is \"Go back to\" on this menu.",
"and the canary is removed. Going back is \"Go back to\" on this menu." + gateNote(md),
yes: func(m *consoleModel) tea.Cmd {
return m.call("Promote "+cn.Model, http.MethodPost, path+"/promote", map[string]string{}, startedSays)
}}
@@ -780,8 +781,8 @@ func (m consoleModel) updateItems(md adminapi.ModelView) []menuItem {
m.confirmD = &confirmDialog{title: "Update " + md.Name,
text: fmt.Sprintf("%s has new weights at %s: %s.\n\n"+
"They are fetched into the mirror beside the ones in use and checked, then the\n"+
"replicas are replaced one at a time. The old files stay, to go back to.",
u.Repo, catalog.ShortRevision(u.Latest), strings.Join(u.Changed, ", ")),
"replicas are replaced one at a time. The old files stay, to go back to.%s",
u.Repo, catalog.ShortRevision(u.Latest), strings.Join(u.Changed, ", "), gateNote(md)),
yes: func(m *consoleModel) tea.Cmd {
return m.call("Update "+md.Name, http.MethodPost, "/api/v1/models/"+url.PathEscape(md.Name)+"/update",
map[string]string{}, startedSays)
+18 -2
View File
@@ -154,6 +154,11 @@ type consoleModel struct {
formD *formDialog
confirmD *confirmDialog
busy string
// busyNote says how far the request under way has got, when it can;
// evalWatch is the quality run being followed, which Esc stops
// following while it carries on.
busyNote string
evalWatch string
quotas []adminapi.QuotaView
quotasErr error
@@ -337,8 +342,12 @@ func (m consoleModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
m.people = peopleData(msg)
case consoleQuotasMsg:
m.quotas, m.quotasErr = msg.rows, msg.err
case evalSuitesMsg:
return m.onEvalSuites(msg)
case evalPollMsg:
return m.onEvalPoll(msg)
case actionDoneMsg:
m.busy = ""
m.busy, m.busyNote, m.evalWatch = "", "", ""
m.dialog, m.dialogTitle = "error", msg.title
if msg.err != nil {
m.dialogText = sBad.Render("✗ ") + msg.err.Error()
@@ -367,6 +376,9 @@ func (m consoleModel) storeReplicated() bool { return len(m.data.sum.Store) > 0
func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
if m.busy != "" {
if m.evalWatch != "" && k.Type == tea.KeyEsc {
m.busy, m.busyNote, m.evalWatch = "", "", ""
}
return m, nil
}
if m.dialog != "" {
@@ -973,7 +985,11 @@ func (m consoleModel) View() string {
} else if t, d := m.docs.dialog(); d != "" && m.menu == cManual {
f.DialogTitle, f.Dialog = t, d
} else if m.busy != "" {
f.DialogTitle, f.Dialog = m.busy, m.spin.View()+" asking the control plane…"
note := "asking the control plane…"
if m.busyNote != "" {
note = m.busyNote
}
f.DialogTitle, f.Dialog = m.busy, m.spin.View()+" "+note
} else if t, d := m.actionDialog(); d != "" {
f.DialogTitle, f.Dialog = t, d
}
+291
View File
@@ -0,0 +1,291 @@
package cli
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/url"
"slices"
"strconv"
"strings"
"time"
tea "github.com/charmbracelet/bubbletea"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
)
// Quality suites from the model menu: ask a model one of the suites the
// control plane has (its eval directory's, and those built in) and watch
// its score come in; list its past runs; hold its updates to one — the same
// as `llmgw ctl eval` and `llmgw ctl model gate`.
// evalSuitesMsg is the suites the control plane can run, fetched to open
// the form that runs one (or, with gate, the one that sets a model's).
type evalSuitesMsg struct {
model adminapi.ModelView
suites []eval.SuiteInfo
gate bool
err error
}
// evalPollMsg is a run still going, to be asked about again.
type evalPollMsg struct {
title string
live eval.Live
err error
}
// gateNote says, in an update's or a promotion's question, that the new
// weights are asked the model's suite first.
func gateNote(md adminapi.ModelView) string {
g := md.Eval
if g == nil {
return ""
}
at := "its threshold"
if g.MinScore > 0 {
at = fmt.Sprintf("%.2f", g.MinScore)
}
return fmt.Sprintf("\n\nFirst the new weights are asked the quality suite %s, and must score %s;\n"+
"below it nothing moves.", g.Suite, at)
}
// evalItems are the model menu's quality entries.
func (m consoleModel) evalItems(md adminapi.ModelView) []menuItem {
gate := "Hold its updates to a quality suite…"
if g := md.Eval; g != nil {
gate = "Its updates' quality suite (" + g.Suite + ")…"
}
return []menuItem{
{"Ask it a quality suite…", func(m *consoleModel) tea.Cmd { return m.fetchSuites(md, false) }},
{"Its quality runs", func(m *consoleModel) tea.Cmd {
return m.call("Quality runs of "+md.Name, http.MethodGet,
"/api/v1/eval/runs?limit=15&model="+url.QueryEscape(md.Name), nil, func(raw json.RawMessage) string {
var runs []eval.Live
if err := json.Unmarshal(raw, &runs); err != nil {
return string(raw)
}
return evalRunsText(runs)
})
}},
{gate, func(m *consoleModel) tea.Cmd { return m.fetchSuites(md, true) }},
}
}
func (m *consoleModel) fetchSuites(md adminapi.ModelView, gate bool) tea.Cmd {
m.busy = "Quality suites"
c := m.client
return func() tea.Msg {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
var list []eval.SuiteInfo
err := c.get(ctx, "/api/v1/eval/suites", &list)
return evalSuitesMsg{model: md, suites: list, gate: gate, err: err}
}
}
func (m consoleModel) onEvalSuites(msg evalSuitesMsg) (tea.Model, tea.Cmd) {
m.busy = ""
md := msg.model
var names []string
for _, s := range msg.suites {
if s.Error == "" {
names = append(names, s.Name)
}
}
if msg.err == nil && len(names) == 0 {
msg.err = errors.New("the control plane has no suite it can run: put one in its control.eval_dir " +
"(/etc/llmgw/eval); docs/install.md, \"Quality suites\", says how one is written")
}
if msg.err != nil {
m.dialog, m.dialogTitle = "error", "Quality suites"
m.dialogText = sBad.Render("✗ ") + msg.err.Error() + "\n\n" + sDim.Render("any key")
return m, nil
}
about := func(name string) string {
for _, s := range msg.suites {
if s.Name == name {
return fmt.Sprintf("%s: %d cases, passed at %.2f, %s", s.Name, s.Cases, s.Threshold, s.From)
}
}
return ""
}
path := "/api/v1/models/" + url.PathEscape(md.Name)
if msg.gate {
current, minScore := "none", ""
if g := md.Eval; g != nil {
current = g.Suite
if g.MinScore > 0 {
minScore = strconv.FormatFloat(g.MinScore, 'f', -1, 64)
}
if !slices.Contains(names, current) {
names = append(names, current)
}
}
m.openForm(&formDialog{title: "Quality gate of " + md.Name,
note: "An update or a promotion asks the new weights this suite first, and stops below its score;\n" +
"an update tries them as a canary given none of " + md.Name + "'s requests, and takes it out again.",
fields: []formField{
choiceF("suite", append([]string{"none"}, names...), current),
textF("min score", minScore, "0 to 1; empty is the suite's own threshold"),
},
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
if v[0] == "none" {
return m.call("Quality gate of "+md.Name, http.MethodDelete, path+"/eval-gate", nil,
func(json.RawMessage) string { return md.Name + "'s updates are no longer held to a suite." }), nil
}
score, err := minScoreOf(v[1])
if err != nil {
return nil, err
}
return m.call("Quality gate of "+md.Name, http.MethodPut, path+"/eval-gate",
adminapi.EvalGateRequest{Suite: v[0], MinScore: score}, func(json.RawMessage) string {
return fmt.Sprintf("%s's new weights must pass %s before they replace the old.", md.Name, v[0])
}), nil
}})
return m, nil
}
m.openForm(&formDialog{title: "Ask " + md.Name + " a quality suite",
note: "Asked on the control plane, through the router as its own requests: past quotas and guardrails.\n" +
about(names[0]),
fields: []formField{
choiceF("suite", names, names[0]),
textF("at once", "4", "cases asked at the same time"),
textF("min score", "", "0 to 1; empty is the suite's own threshold"),
},
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
n, err := strconv.Atoi(v[1])
if err != nil || n < 1 {
return nil, fmt.Errorf("at once is a whole number, 1 or more")
}
score, err := minScoreOf(v[2])
if err != nil {
return nil, err
}
return m.startEval(md.Name, eval.Request{Suite: v[0], Concurrency: n, MinScore: score}), nil
}})
return m, nil
}
func minScoreOf(s string) (float64, error) {
if s == "" {
return 0, nil
}
f, err := strconv.ParseFloat(s, 64)
if err != nil || f < 0 || f > 1 {
return 0, fmt.Errorf("the min score is a number from 0 to 1")
}
return f, nil
}
// startEval starts a run and follows it.
func (m *consoleModel) startEval(model string, req eval.Request) tea.Cmd {
title := model + ": " + req.Suite
m.busy, m.busyNote = title, "starting…"
c := m.client
return func() tea.Msg {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
var l eval.Live
err := c.do(ctx, http.MethodPost, "/api/v1/models/"+url.PathEscape(model)+"/eval", req, &l)
if err != nil {
return actionDoneMsg{title: title, err: err}
}
return evalPollMsg{title: title, live: l}
}
}
func (m consoleModel) onEvalPoll(msg evalPollMsg) (tea.Model, tea.Cmd) {
if m.evalWatch != "" && m.evalWatch != msg.live.ID {
return m, nil
}
if m.evalWatch == "" && m.busy != msg.title {
// Esc: no longer followed here; it carries on on the control plane.
return m, nil
}
m.evalWatch = msg.live.ID
m.busy = msg.title
m.busyNote = fmt.Sprintf("%d of %d answered · Esc stops watching (it carries on)", msg.live.Done, msg.live.Cases)
c, id, title := m.client, msg.live.ID, msg.title
return m, func() tea.Msg {
time.Sleep(time.Second)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
var l eval.Live
if err := c.get(ctx, "/api/v1/eval/runs/"+url.PathEscape(id), &l); err != nil {
return actionDoneMsg{title: title, err: err}
}
if l.State == eval.Running {
return evalPollMsg{title: title, live: l}
}
var runs []eval.Live
_ = c.get(ctx, "/api/v1/eval/runs?limit=6&model="+url.QueryEscape(l.Model), &runs)
text := evalRunText(l)
if len(runs) > 1 {
text += "\n\nIts last runs:\n" + evalRunsText(runs)
}
if l.State == eval.Failed || !l.Passed {
return actionDoneMsg{title: title, err: errors.New(text)}
}
return actionDoneMsg{title: title, text: text}
}
}
// evalRunText is a finished run: its score, then each case.
func evalRunText(l eval.Live) string {
if l.State == eval.Failed {
return "The run could not be scored: " + l.Error
}
var b strings.Builder
b.WriteString(evalSummary(l) + "\n")
if l.Error != "" {
b.WriteString(l.Error + "\n")
}
b.WriteString("\n")
shown := min(len(l.Results), 14)
for _, r := range l.Results[:shown] {
mark := sOK.Render("✓")
if !r.Passed {
mark = sBad.Render("✗")
}
why := r.Detail
if r.Error != "" {
why = r.Error
}
fmt.Fprintf(&b, "%s %-18s %4.2f %s\n", mark, truncate(r.ID, 18), r.Score, truncate(why, 60))
}
if len(l.Results) > shown {
fmt.Fprintf(&b, " and %d more: llmgw ctl eval --run %s\n", len(l.Results)-shown, l.ID)
}
return strings.TrimRight(b.String(), "\n")
}
// evalRunsText is a model's runs, one a line.
func evalRunsText(runs []eval.Live) string {
if len(runs) == 0 {
return "No runs kept yet: \"Ask it a quality suite\" on this menu makes one."
}
var b strings.Builder
for _, r := range runs {
result := sOK.Render("passed")
switch {
case r.State == eval.Running:
result = fmt.Sprintf("running, %d of %d", r.Done, r.Cases)
case r.State == eval.Failed:
result = sBad.Render("could not run")
case !r.Passed:
result = sBad.Render("failed")
}
who := r.Model
if r.Target != "" {
who = r.Target
}
fmt.Fprintf(&b, "%-9s %-14s %-16s %4.2f of %4.2f %s\n", ageShort(r.At), truncate(r.Suite, 14),
truncate(who, 16), r.Score, r.Threshold, result)
}
return strings.TrimRight(b.String(), "\n")
}
+116
View File
@@ -0,0 +1,116 @@
package cli
import (
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
tea "github.com/charmbracelet/bubbletea"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
func TestAModelIsAskedASuiteFromItsMenuAndItsScoreShown(t *testing.T) {
var mu sync.Mutex
var asked eval.Request
polls := 0
done := eval.Live{EvalRun: store.EvalRun{ID: "eval-1", Model: "qwen", Suite: "arithmetic", Score: 0.75,
Threshold: 0.7, Passed: true, Cases: 2, Below: 1, Results: []store.EvalCase{
{ID: "add", Kind: "number", Score: 1, Passed: true, Detail: "read 42"},
{ID: "mul", Kind: "number", Score: 0, Detail: "read 43, wanted 42"}}}, State: eval.Done, Done: 2}
admin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
mu.Lock()
defer mu.Unlock()
switch {
case r.URL.Path == "/api/v1/eval/suites":
_ = json.NewEncoder(w).Encode([]eval.SuiteInfo{
{Name: "arithmetic", From: "built in", Cases: 2, Threshold: 0.7},
{Name: "broken", From: "/etc/llmgw/eval/broken.yaml", Error: "no cases"},
{Name: "json-format", From: "built in", Cases: 7, Threshold: 0.7}})
case r.Method == http.MethodPost && r.URL.Path == "/api/v1/models/qwen/eval":
b, _ := io.ReadAll(r.Body)
_ = json.Unmarshal(b, &asked)
w.WriteHeader(http.StatusAccepted)
_ = json.NewEncoder(w).Encode(eval.Live{EvalRun: store.EvalRun{ID: "eval-1", Model: "qwen",
Suite: "arithmetic", Cases: 2}, State: eval.Running})
case r.URL.Path == "/api/v1/eval/runs/eval-1":
polls++
if polls == 1 {
_ = json.NewEncoder(w).Encode(eval.Live{EvalRun: store.EvalRun{ID: "eval-1", Model: "qwen",
Suite: "arithmetic", Cases: 2}, State: eval.Running, Done: 1})
return
}
_ = json.NewEncoder(w).Encode(done)
case r.URL.Path == "/api/v1/eval/runs":
older := done
older.ID, older.Score, older.Passed = "eval-0", 0.5, false
_ = json.NewEncoder(w).Encode([]eval.Live{done, older})
default:
http.NotFound(w, r)
}
}))
t.Cleanup(admin.Close)
m := newConsoleModel(admin.URL, "/nonexistent/server.yaml")
m.loaded, m.menu, m.inPanel = true, cModels, true
md := adminapi.ModelView{Name: "qwen"}
// run plays a command and what it leads to, the way the program would.
run := func(tm tea.Model, cmd tea.Cmd) tea.Model {
for cmd != nil {
msg := cmd()
tm, cmd = tm.Update(msg)
if _, ok := msg.(actionDoneMsg); ok {
break
}
}
return tm
}
var tm tea.Model = m
items := m.evalItems(md)
if len(items) != 3 || !strings.Contains(items[0].label, "quality suite") {
t.Fatalf("%+v", items)
}
tm = run(tm, items[0].act(&m))
f := tm.(consoleModel).formD
if f == nil || f.fields[0].options == nil || len(f.fields[0].options) != 2 {
t.Fatalf("no form offering the suites that can run: %+v", f)
}
tm = press(tm, "tab")
f.fields[1].text.SetValue("2")
tm = press(tm, "tab", "tab")
tm, cmd := tm.Update(tea.KeyMsg{Type: tea.KeyEnter})
if busy := tm.(consoleModel).busy; busy != "qwen: arithmetic" {
t.Fatalf("busy %q", busy)
}
// Started, then followed: one poll still running, then its score.
tm, next := tm.Update(cmd())
if note := tm.(consoleModel).busyNote; !strings.Contains(note, "0 of 2 answered") {
t.Errorf("progress %q", note)
}
tm, next = tm.Update(next())
if note := tm.(consoleModel).busyNote; !strings.Contains(note, "1 of 2 answered") {
t.Errorf("progress %q", note)
}
tm, _ = tm.Update(next())
view := tm.View()
mu.Lock()
sent := asked
mu.Unlock()
if sent.Suite != "arithmetic" || sent.Concurrency != 2 {
t.Errorf("asked %+v", sent)
}
for _, want := range []string{"qwen scored 0.75 on arithmetic", "1 of 2 cases passed", "read 43, wanted 42", "Its last runs"} {
if !strings.Contains(view, want) {
t.Errorf("the result does not say %q:\n%s", want, view)
}
}
if c := tm.(consoleModel); c.busy != "" || c.evalWatch != "" {
t.Errorf("still busy: %q %q", c.busy, c.evalWatch)
}
}