Quality suites, and new weights held to one: ctl eval, model update/promote --eval, model gate

A suite is a set of cases in YAML or JSON lines, each scored by exact,
contains, regex, a number within a tolerance, a JSON Schema subset, or a
judge model with a rubric; weighed into one score against a threshold.
internal/bench/eval runs one on the control plane through Router.Replay, as
the gateway's own requests, and keeps every run and its cases in the store
(migration 0028: eval_runs, eval_cases).

- llmgw ctl eval MODEL --suite FILE|NAME, --history, --run, --suites; exits
  non-zero below the score. Suites by name from control.eval_dir, or the
  two built in from deploy/eval (arithmetic, json-format).
- model update and promote take --eval SUITE --min-score N, or a model's
  standing gate (ctl model gate, eval: in the catalog): a promote asks the
  canary first and leaves it below the score; an update tries the new
  weights as a canary given no share, then promotes or removes it.
- The console's model menu: ask a suite with its progress, its runs, its
  gate.
- The audit middleware no longer hands a handler a body cut off at 64 KiB.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 00:04:43 +02:00
1 parent 30f7f3474a
commit f0477f140d
49 files changed
+4563 -46

No files matched your search

+12 -2
View File
@@ -28,6 +28,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
@@ -43,6 +44,8 @@ import (
type API struct {
// replay sends a kept request again; nil without a router here.
replay Replayer
// evals runs quality suites against models; nil without a router here.
evals *eval.Service
// notifier is the webhooks `ctl notify test` reaches; nil without any.
notifier *notify.Notifier
// releases is where release binaries are fetched from, for upgrades.
@@ -157,6 +160,7 @@ func (a *API) Register(mux *http.ServeMux) {
mux.HandleFunc("POST /api/v1/models/{name}/update", a.updateModel)
mux.HandleFunc("POST /api/v1/models/{name}/rollback", a.rollbackModel)
a.registerCanary(mux)
a.registerEval(mux)
mux.HandleFunc("POST /api/v1/models/{name}/origin", a.setOrigin)
mux.HandleFunc("GET /api/v1/plan", a.planModel)
mux.HandleFunc("POST /api/v1/metrics/range", a.metricsBatch)
@@ -339,6 +343,8 @@ type ModelView struct {
// set on the candidate, naming the model it is a canary of.
Canary *catalog.Canary `json:"canary,omitempty"`
CanaryOf string `json:"canary_of,omitempty"`
// Eval is the quality suite its new weights must pass, when it has one.
Eval *catalog.EvalGate `json:"eval,omitempty"`
// Description is what the model is for, when somebody said.
Description string `json:"description,omitempty"`
Aliases []string `json:"aliases,omitempty"`
@@ -660,7 +666,7 @@ func (a *API) Models() []ModelView {
Parked: h.Parked, Reason: h.Reason, Unplaceable: h.Unplaceable,
Class: m.EffectiveClass(), Owner: m.Owner, Preempted: h.Preempted, HeldFor: h.HeldFor,
External: externalLabel(m), Adapters: m.Adapters,
Canary: m.Canary, CanaryOf: a.ctl.Catalog().CanaryOf(m.Name),
Canary: m.Canary, CanaryOf: a.ctl.Catalog().CanaryOf(m.Name), Eval: m.Eval,
}
for _, p := range m.Profiles {
v.Profiles = append(v.Profiles, p.Name)
@@ -1559,7 +1565,11 @@ func (a *API) updateModel(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusNotImplemented, "updates need a mirror and a store, and this control plane lacks one")
return
}
job, err := a.serve.StartUpdate(r.PathValue("name"), actorOf(r))
ch, ok := readChange(w, r)
if !ok {
return
}
job, err := a.serve.StartUpdate(r.PathValue("name"), actorOf(r), ch)
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
+5 -1
View File
@@ -74,7 +74,11 @@ func (a *API) registerCanary(mux *http.ServeMux) {
if !need(w) {
return
}
job, err := a.serve.StartPromote(r.PathValue("name"), actorOf(r))
ch, ok := readChange(w, r)
if !ok {
return
}
job, err := a.serve.StartPromote(r.PathValue("name"), actorOf(r), ch)
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
+179
View File
@@ -0,0 +1,179 @@
package adminapi
import (
"context"
"encoding/json"
"errors"
"io"
"net/http"
"strconv"
"strings"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// Quality suites: run against a model on the control plane, through the
// router as the gateway's own requests (like a replay), followed while they
// run, and kept; and a model's standing suite that its updates must pass.
// SetEval gives the API the service that runs quality suites.
func (a *API) SetEval(s *eval.Service) { a.evals = s }
// maxSuiteBytes bounds a suite sent whole: a few hundred cases with their
// prompts, which is more than a gate should take to run.
const maxSuiteBytes = 4 << 20
// EvalGateRequest sets a model's standing quality suite; an empty suite
// takes it away.
type EvalGateRequest struct {
Suite string `json:"suite"`
MinScore float64 `json:"min_score,omitempty"`
Judge string `json:"judge,omitempty"`
}
type evalGater interface {
SetEval(ctx context.Context, name string, g *catalog.EvalGate, by string) (catalog.Model, error)
}
// readChange reads how an update or a promotion is held to quality; an
// empty body is no change to how.
func readChange(w http.ResponseWriter, r *http.Request) (modelserve.Change, bool) {
var ch modelserve.Change
b, err := io.ReadAll(io.LimitReader(r.Body, maxSuiteBytes+1))
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return ch, false
}
if len(b) > maxSuiteBytes {
writeError(w, http.StatusRequestEntityTooLarge, "the suite is over 4 MiB")
return ch, false
}
if len(strings.TrimSpace(string(b))) == 0 {
return ch, true
}
if err := json.Unmarshal(b, &ch); err != nil {
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
return ch, false
}
return ch, true
}
func (a *API) registerEval(mux *http.ServeMux) {
need := func(w http.ResponseWriter) bool {
if a.evals == nil {
writeError(w, http.StatusNotImplemented, "this control plane does not route requests, so it cannot ask a model a suite")
return false
}
return true
}
mux.HandleFunc("GET /api/v1/eval/suites", func(w http.ResponseWriter, r *http.Request) {
if !need(w) {
return
}
out := a.evals.Suites()
if out == nil {
out = []eval.SuiteInfo{}
}
writeJSON(w, http.StatusOK, out)
})
// Started and answered at once: a suite against a model on CPUs takes
// minutes, longer than a client waits on one request.
mux.HandleFunc("POST /api/v1/models/{name}/eval", func(w http.ResponseWriter, r *http.Request) {
if !need(w) {
return
}
var req eval.Request
if err := json.NewDecoder(io.LimitReader(r.Body, maxSuiteBytes)).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
return
}
name := r.PathValue("name")
if a.ctl != nil {
// By its own name, so an alias's runs are filed with the rest.
m, ok := a.ctl.Catalog().Resolve(name)
if !ok {
writeError(w, http.StatusNotFound, "no model called "+name+": `llmgw ctl models` lists them")
return
}
name = m.Name
}
live, err := a.evals.Start(name, req, actorOf(r))
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusAccepted, live)
})
mux.HandleFunc("GET /api/v1/eval/runs", func(w http.ResponseWriter, r *http.Request) {
if !need(w) {
return
}
limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
out, err := a.evals.History(r.Context(), r.URL.Query().Get("model"), limit)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if out == nil {
out = []eval.Live{}
}
writeJSON(w, http.StatusOK, out)
})
mux.HandleFunc("GET /api/v1/eval/runs/{id}", func(w http.ResponseWriter, r *http.Request) {
if !need(w) {
return
}
live, err := a.evals.Get(r.Context(), r.PathValue("id"))
if errors.Is(err, store.ErrNotFound) {
writeError(w, http.StatusNotFound, "no eval run "+r.PathValue("id"))
return
}
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, live)
})
setGate := func(w http.ResponseWriter, r *http.Request, g *catalog.EvalGate) {
gater, ok := a.models.(evalGater)
if !ok {
writeError(w, http.StatusNotImplemented, "this control plane has no store to keep the setting in")
return
}
name := r.PathValue("name")
if g != nil && a.evals != nil {
if err := a.evals.Check(name, eval.Request{Suite: g.Suite, Judge: g.Judge, MinScore: g.MinScore}); err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
}
m, err := gater.SetEval(r.Context(), name, g, actorOf(r))
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]any{"model": m.Name, "eval": m.Eval})
}
mux.HandleFunc("PUT /api/v1/models/{name}/eval-gate", func(w http.ResponseWriter, r *http.Request) {
var req EvalGateRequest
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
return
}
if strings.TrimSpace(req.Suite) == "" {
writeError(w, http.StatusBadRequest, "name a suite: one in the control plane's eval directory, or built in")
return
}
if req.MinScore < 0 || req.MinScore > 1 {
writeError(w, http.StatusBadRequest, "the minimum score is 0 to 1")
return
}
setGate(w, r, &catalog.EvalGate{Suite: strings.TrimSpace(req.Suite), MinScore: req.MinScore, Judge: req.Judge})
})
mux.HandleFunc("DELETE /api/v1/models/{name}/eval-gate", func(w http.ResponseWriter, r *http.Request) {
setGate(w, r, nil)
})
}
+84
View File
@@ -0,0 +1,84 @@
package adminapi
import (
"context"
"encoding/json"
"log/slog"
"net/http"
"net/http/httptest"
"strings"
"testing"
"testing/fstest"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
)
// gatingModels is the catalog's service, keeping the gate it is given.
type gatingModels struct {
*fakeModels
gate *catalog.EvalGate
set bool
}
func (g *gatingModels) SetEval(_ context.Context, name string, gate *catalog.EvalGate, _ string) (catalog.Model, error) {
g.gate, g.set = gate, true
return catalog.Model{Name: name, Eval: gate}, nil
}
func TestASuiteIsStartedByAModelsAliasAndAGateNeedsASuiteThatIsThere(t *testing.T) {
a, _ := setup(t, []catalog.Model{{Name: "qwen", Aliases: []string{"q"}, Source: "/m.gguf",
Scaling: catalog.Scaling{Min: 1, Max: 1}, Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}}})
answer := func(_ context.Context, _, _ string, _ []byte) (int, []byte, time.Duration, error) {
return 200, []byte(`{"choices":[{"message":{"content":"42"}}]}`), 0, nil
}
a.SetEval(eval.NewService(eval.Deps{Call: answer, Log: slog.New(slog.DiscardHandler),
Builtin: fstest.MapFS{"arith.yaml": {Data: []byte("cases:\n - prompt: 40 + 2?\n number: 42\n")}}}))
models := &gatingModels{fakeModels: &fakeModels{}}
a.SetModels(models)
mux := http.NewServeMux()
a.Register(mux)
do := func(method, path, body string) *httptest.ResponseRecorder {
req := httptest.NewRequest(method, path, strings.NewReader(body))
rec := httptest.NewRecorder()
mux.ServeHTTP(rec, req)
return rec
}
rec := do(http.MethodPost, "/api/v1/models/q/eval", `{"suite": "arith"}`)
var live eval.Live
if rec.Code != http.StatusAccepted || json.Unmarshal(rec.Body.Bytes(), &live) != nil || live.Model != "qwen" {
t.Fatalf("%d %s", rec.Code, rec.Body.String())
}
if rec := do(http.MethodPost, "/api/v1/models/nobody/eval", `{"suite": "arith"}`); rec.Code != http.StatusNotFound {
t.Errorf("a model that is not there: %d %s", rec.Code, rec.Body.String())
}
if rec := do(http.MethodPost, "/api/v1/models/qwen/eval", `{"suite": "nope"}`); rec.Code != http.StatusBadRequest ||
!strings.Contains(rec.Body.String(), "--suites") {
t.Errorf("a suite that is not there: %d %s", rec.Code, rec.Body.String())
}
if rec := do(http.MethodGet, "/api/v1/eval/suites", ""); !strings.Contains(rec.Body.String(), `"name":"arith"`) {
t.Errorf("suites: %s", rec.Body.String())
}
if rec := do(http.MethodPut, "/api/v1/models/qwen/eval-gate", `{"suite": "nope"}`); rec.Code != http.StatusBadRequest || models.set {
t.Errorf("a gate on a suite that is not there was set: %d %s", rec.Code, rec.Body.String())
}
if rec := do(http.MethodPut, "/api/v1/models/qwen/eval-gate", `{"suite": "arith", "min_score": 0.9}`); rec.Code != http.StatusOK ||
models.gate == nil || models.gate.Suite != "arith" || models.gate.MinScore != 0.9 {
t.Errorf("%d %s %+v", rec.Code, rec.Body.String(), models.gate)
}
if rec := do(http.MethodDelete, "/api/v1/models/qwen/eval-gate", ""); rec.Code != http.StatusOK || models.gate != nil {
t.Errorf("%d %s", rec.Code, rec.Body.String())
}
deadline := time.Now().Add(5 * time.Second)
for live.State == eval.Running && time.Now().Before(deadline) {
time.Sleep(10 * time.Millisecond)
_ = json.Unmarshal(do(http.MethodGet, "/api/v1/eval/runs/"+live.ID, "").Body.Bytes(), &live)
}
if live.State != eval.Done || live.Score != 1 || len(live.Results) != 1 {
t.Fatalf("%+v", live)
}
}