Quality suites, and new weights held to one: ctl eval, model update/promote --eval, model gate
A suite is a set of cases in YAML or JSON lines, each scored by exact, contains, regex, a number within a tolerance, a JSON Schema subset, or a judge model with a rubric; weighed into one score against a threshold. internal/bench/eval runs one on the control plane through Router.Replay, as the gateway's own requests, and keeps every run and its cases in the store (migration 0028: eval_runs, eval_cases). - llmgw ctl eval MODEL --suite FILE|NAME, --history, --run, --suites; exits non-zero below the score. Suites by name from control.eval_dir, or the two built in from deploy/eval (arithmetic, json-format). - model update and promote take --eval SUITE --min-score N, or a model's standing gate (ctl model gate, eval: in the catalog): a promote asks the canary first and leaves it below the score; an update tries the new weights as a canary given no share, then promotes or removes it. - The console's model menu: ask a suite with its progress, its runs, its gate. - The audit middleware no longer hands a handler a body cut off at 64 KiB. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
30f7f3474a
commit
f0477f140d
49 files changed
+4563
-46
No files matched your search
@@ -0,0 +1,179 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// Quality suites: run against a model on the control plane, through the
|
||||
// router as the gateway's own requests (like a replay), followed while they
|
||||
// run, and kept; and a model's standing suite that its updates must pass.
|
||||
|
||||
// SetEval gives the API the service that runs quality suites.
|
||||
func (a *API) SetEval(s *eval.Service) { a.evals = s }
|
||||
|
||||
// maxSuiteBytes bounds a suite sent whole: a few hundred cases with their
|
||||
// prompts, which is more than a gate should take to run.
|
||||
const maxSuiteBytes = 4 << 20
|
||||
|
||||
// EvalGateRequest sets a model's standing quality suite; an empty suite
|
||||
// takes it away.
|
||||
type EvalGateRequest struct {
|
||||
Suite string `json:"suite"`
|
||||
MinScore float64 `json:"min_score,omitempty"`
|
||||
Judge string `json:"judge,omitempty"`
|
||||
}
|
||||
|
||||
type evalGater interface {
|
||||
SetEval(ctx context.Context, name string, g *catalog.EvalGate, by string) (catalog.Model, error)
|
||||
}
|
||||
|
||||
// readChange reads how an update or a promotion is held to quality; an
|
||||
// empty body is no change to how.
|
||||
func readChange(w http.ResponseWriter, r *http.Request) (modelserve.Change, bool) {
|
||||
var ch modelserve.Change
|
||||
b, err := io.ReadAll(io.LimitReader(r.Body, maxSuiteBytes+1))
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return ch, false
|
||||
}
|
||||
if len(b) > maxSuiteBytes {
|
||||
writeError(w, http.StatusRequestEntityTooLarge, "the suite is over 4 MiB")
|
||||
return ch, false
|
||||
}
|
||||
if len(strings.TrimSpace(string(b))) == 0 {
|
||||
return ch, true
|
||||
}
|
||||
if err := json.Unmarshal(b, &ch); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
|
||||
return ch, false
|
||||
}
|
||||
return ch, true
|
||||
}
|
||||
|
||||
func (a *API) registerEval(mux *http.ServeMux) {
|
||||
need := func(w http.ResponseWriter) bool {
|
||||
if a.evals == nil {
|
||||
writeError(w, http.StatusNotImplemented, "this control plane does not route requests, so it cannot ask a model a suite")
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
mux.HandleFunc("GET /api/v1/eval/suites", func(w http.ResponseWriter, r *http.Request) {
|
||||
if !need(w) {
|
||||
return
|
||||
}
|
||||
out := a.evals.Suites()
|
||||
if out == nil {
|
||||
out = []eval.SuiteInfo{}
|
||||
}
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
})
|
||||
// Started and answered at once: a suite against a model on CPUs takes
|
||||
// minutes, longer than a client waits on one request.
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/eval", func(w http.ResponseWriter, r *http.Request) {
|
||||
if !need(w) {
|
||||
return
|
||||
}
|
||||
var req eval.Request
|
||||
if err := json.NewDecoder(io.LimitReader(r.Body, maxSuiteBytes)).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
|
||||
return
|
||||
}
|
||||
name := r.PathValue("name")
|
||||
if a.ctl != nil {
|
||||
// By its own name, so an alias's runs are filed with the rest.
|
||||
m, ok := a.ctl.Catalog().Resolve(name)
|
||||
if !ok {
|
||||
writeError(w, http.StatusNotFound, "no model called "+name+": `llmgw ctl models` lists them")
|
||||
return
|
||||
}
|
||||
name = m.Name
|
||||
}
|
||||
live, err := a.evals.Start(name, req, actorOf(r))
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, live)
|
||||
})
|
||||
mux.HandleFunc("GET /api/v1/eval/runs", func(w http.ResponseWriter, r *http.Request) {
|
||||
if !need(w) {
|
||||
return
|
||||
}
|
||||
limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
|
||||
out, err := a.evals.History(r.Context(), r.URL.Query().Get("model"), limit)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
if out == nil {
|
||||
out = []eval.Live{}
|
||||
}
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
})
|
||||
mux.HandleFunc("GET /api/v1/eval/runs/{id}", func(w http.ResponseWriter, r *http.Request) {
|
||||
if !need(w) {
|
||||
return
|
||||
}
|
||||
live, err := a.evals.Get(r.Context(), r.PathValue("id"))
|
||||
if errors.Is(err, store.ErrNotFound) {
|
||||
writeError(w, http.StatusNotFound, "no eval run "+r.PathValue("id"))
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, live)
|
||||
})
|
||||
setGate := func(w http.ResponseWriter, r *http.Request, g *catalog.EvalGate) {
|
||||
gater, ok := a.models.(evalGater)
|
||||
if !ok {
|
||||
writeError(w, http.StatusNotImplemented, "this control plane has no store to keep the setting in")
|
||||
return
|
||||
}
|
||||
name := r.PathValue("name")
|
||||
if g != nil && a.evals != nil {
|
||||
if err := a.evals.Check(name, eval.Request{Suite: g.Suite, Judge: g.Judge, MinScore: g.MinScore}); err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
m, err := gater.SetEval(r.Context(), name, g, actorOf(r))
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"model": m.Name, "eval": m.Eval})
|
||||
}
|
||||
mux.HandleFunc("PUT /api/v1/models/{name}/eval-gate", func(w http.ResponseWriter, r *http.Request) {
|
||||
var req EvalGateRequest
|
||||
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
|
||||
return
|
||||
}
|
||||
if strings.TrimSpace(req.Suite) == "" {
|
||||
writeError(w, http.StatusBadRequest, "name a suite: one in the control plane's eval directory, or built in")
|
||||
return
|
||||
}
|
||||
if req.MinScore < 0 || req.MinScore > 1 {
|
||||
writeError(w, http.StatusBadRequest, "the minimum score is 0 to 1")
|
||||
return
|
||||
}
|
||||
setGate(w, r, &catalog.EvalGate{Suite: strings.TrimSpace(req.Suite), MinScore: req.MinScore, Judge: req.Judge})
|
||||
})
|
||||
mux.HandleFunc("DELETE /api/v1/models/{name}/eval-gate", func(w http.ResponseWriter, r *http.Request) {
|
||||
setGate(w, r, nil)
|
||||
})
|
||||
}
|
||||
Reference in new issue
Block a user