Merge C21 quality suites and gating

Conflicts: docs.go (manual, OpenAPI and suites all kept), modelserve Job
(trace and eval both kept). The eval routes are added to the OpenAPI route
table, update and promote documented with their optional body, and the
document regenerated.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 00:08:33 +02:00
commit d41104a452
51 files changed
+5204 -49

No files matched your search

+12 -2
View File
@@ -28,6 +28,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
@@ -43,6 +44,8 @@ import (
type API struct {
// replay sends a kept request again; nil without a router here.
replay Replayer
// evals runs quality suites against models; nil without a router here.
evals *eval.Service
// notifier is the webhooks `ctl notify test` reaches; nil without any.
notifier *notify.Notifier
// releases is where release binaries are fetched from, for upgrades.
@@ -168,6 +171,7 @@ func (a *API) Register(mux Mux) {
mux.HandleFunc("POST /api/v1/models/{name}/update", a.updateModel)
mux.HandleFunc("POST /api/v1/models/{name}/rollback", a.rollbackModel)
a.registerCanary(mux)
a.registerEval(mux)
mux.HandleFunc("POST /api/v1/models/{name}/origin", a.setOrigin)
mux.HandleFunc("GET /api/v1/plan", a.planModel)
mux.HandleFunc("POST /api/v1/metrics/range", a.metricsBatch)
@@ -379,6 +383,8 @@ type ModelView struct {
// set on the candidate, naming the model it is a canary of.
Canary *catalog.Canary `json:"canary,omitempty"`
CanaryOf string `json:"canary_of,omitempty"`
// Eval is the quality suite its new weights must pass, when it has one.
Eval *catalog.EvalGate `json:"eval,omitempty"`
// Description is what the model is for, when somebody said.
Description string `json:"description,omitempty"`
Aliases []string `json:"aliases,omitempty"`
@@ -715,7 +721,7 @@ func (a *API) Models() []ModelView {
Parked: h.Parked, Reason: h.Reason, Unplaceable: h.Unplaceable,
Class: m.EffectiveClass(), Owner: m.Owner, Preempted: h.Preempted, HeldFor: h.HeldFor,
External: externalLabel(m), Adapters: m.Adapters,
Canary: m.Canary, CanaryOf: a.ctl.Catalog().CanaryOf(m.Name),
Canary: m.Canary, CanaryOf: a.ctl.Catalog().CanaryOf(m.Name), Eval: m.Eval,
}
for _, p := range m.Profiles {
v.Profiles = append(v.Profiles, p.Name)
@@ -1632,7 +1638,11 @@ func (a *API) updateModel(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusNotImplemented, "updates need a mirror and a store, and this control plane lacks one")
return
}
job, err := a.serve.StartUpdate(r.PathValue("name"), actorOf(r))
ch, ok := readChange(w, r)
if !ok {
return
}
job, err := a.serve.StartUpdate(r.PathValue("name"), actorOf(r), ch)
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
+5 -1
View File
@@ -78,7 +78,11 @@ func (a *API) registerCanary(mux Mux) {
if !need(w) {
return
}
job, err := a.serve.StartPromote(r.PathValue("name"), actorOf(r))
ch, ok := readChange(w, r)
if !ok {
return
}
job, err := a.serve.StartPromote(r.PathValue("name"), actorOf(r), ch)
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
+186
View File
@@ -0,0 +1,186 @@
package adminapi
import (
"context"
"encoding/json"
"errors"
"io"
"net/http"
"strconv"
"strings"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// Quality suites: run against a model on the control plane, through the
// router as the gateway's own requests (like a replay), followed while they
// run, and kept; and a model's standing suite that its updates must pass.
// SetEval gives the API the service that runs quality suites.
func (a *API) SetEval(s *eval.Service) { a.evals = s }
// maxSuiteBytes bounds a suite sent whole: a few hundred cases with their
// prompts, which is more than a gate should take to run.
const maxSuiteBytes = 4 << 20
// EvalGateRequest sets a model's standing quality suite; an empty suite
// takes it away.
type EvalGateRequest struct {
Suite string `json:"suite"`
MinScore float64 `json:"min_score,omitempty"`
Judge string `json:"judge,omitempty"`
}
type evalGater interface {
SetEval(ctx context.Context, name string, g *catalog.EvalGate, by string) (catalog.Model, error)
}
// readChange reads how an update or a promotion is held to quality; an
// empty body is no change to how.
func readChange(w http.ResponseWriter, r *http.Request) (modelserve.Change, bool) {
var ch modelserve.Change
b, err := io.ReadAll(io.LimitReader(r.Body, maxSuiteBytes+1))
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return ch, false
}
if len(b) > maxSuiteBytes {
writeError(w, http.StatusRequestEntityTooLarge, "the suite is over 4 MiB")
return ch, false
}
if len(strings.TrimSpace(string(b))) == 0 {
return ch, true
}
if err := json.Unmarshal(b, &ch); err != nil {
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
return ch, false
}
return ch, true
}
func (a *API) registerEval(mux Mux) {
need := func(w http.ResponseWriter) bool {
if a.evals == nil {
writeError(w, http.StatusNotImplemented, "this control plane does not route requests, so it cannot ask a model a suite")
return false
}
return true
}
mux.HandleFunc("GET /api/v1/eval/suites", func(w http.ResponseWriter, r *http.Request) {
if !need(w) {
return
}
out := a.evals.Suites()
if out == nil {
out = []eval.SuiteInfo{}
}
writeJSON(w, http.StatusOK, out)
})
// Started and answered at once: a suite against a model on CPUs takes
// minutes, longer than a client waits on one request.
mux.HandleFunc("POST /api/v1/models/{name}/eval", func(w http.ResponseWriter, r *http.Request) {
if !need(w) {
return
}
var req eval.Request
if err := json.NewDecoder(io.LimitReader(r.Body, maxSuiteBytes)).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
return
}
name := r.PathValue("name")
if a.ctl != nil {
// By its own name, so an alias's runs are filed with the rest.
m, ok := a.ctl.Catalog().Resolve(name)
if !ok {
writeError(w, http.StatusNotFound, "no model called "+name+": `llmgw ctl models` lists them")
return
}
name = m.Name
}
live, err := a.evals.Start(name, req, actorOf(r))
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusAccepted, live)
})
mux.HandleFunc("GET /api/v1/eval/runs", func(w http.ResponseWriter, r *http.Request) {
if !need(w) {
return
}
limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
out, err := a.evals.History(r.Context(), r.URL.Query().Get("model"), limit)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if out == nil {
out = []eval.Live{}
}
writeJSON(w, http.StatusOK, out)
})
mux.HandleFunc("GET /api/v1/eval/runs/{id}", func(w http.ResponseWriter, r *http.Request) {
if !need(w) {
return
}
live, err := a.evals.Get(r.Context(), r.PathValue("id"))
if errors.Is(err, store.ErrNotFound) {
writeError(w, http.StatusNotFound, "no eval run "+r.PathValue("id"))
return
}
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, live)
})
setGate := func(w http.ResponseWriter, r *http.Request, g *catalog.EvalGate) {
gater, ok := a.models.(evalGater)
if !ok {
writeError(w, http.StatusNotImplemented, "this control plane has no store to keep the setting in")
return
}
name := r.PathValue("name")
if g != nil && a.evals != nil {
if err := a.evals.Check(name, eval.Request{Suite: g.Suite, Judge: g.Judge, MinScore: g.MinScore}); err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
}
m, err := gater.SetEval(r.Context(), name, g, actorOf(r))
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusOK, EvalGateView{Model: m.Name, Eval: m.Eval})
}
mux.HandleFunc("PUT /api/v1/models/{name}/eval-gate", func(w http.ResponseWriter, r *http.Request) {
var req EvalGateRequest
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
return
}
if strings.TrimSpace(req.Suite) == "" {
writeError(w, http.StatusBadRequest, "name a suite: one in the control plane's eval directory, or built in")
return
}
if req.MinScore < 0 || req.MinScore > 1 {
writeError(w, http.StatusBadRequest, "the minimum score is 0 to 1")
return
}
setGate(w, r, &catalog.EvalGate{Suite: strings.TrimSpace(req.Suite), MinScore: req.MinScore, Judge: req.Judge})
})
mux.HandleFunc("DELETE /api/v1/models/{name}/eval-gate", func(w http.ResponseWriter, r *http.Request) {
setGate(w, r, nil)
})
}
// EvalGateView is a model's standing quality gate, as it is now; Eval is
// absent when it has none.
type EvalGateView struct {
Model string `json:"model"`
Eval *catalog.EvalGate `json:"eval,omitempty"`
}
+84
View File
@@ -0,0 +1,84 @@
package adminapi
import (
"context"
"encoding/json"
"log/slog"
"net/http"
"net/http/httptest"
"strings"
"testing"
"testing/fstest"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
)
// gatingModels is the catalog's service, keeping the gate it is given.
type gatingModels struct {
*fakeModels
gate *catalog.EvalGate
set bool
}
func (g *gatingModels) SetEval(_ context.Context, name string, gate *catalog.EvalGate, _ string) (catalog.Model, error) {
g.gate, g.set = gate, true
return catalog.Model{Name: name, Eval: gate}, nil
}
func TestASuiteIsStartedByAModelsAliasAndAGateNeedsASuiteThatIsThere(t *testing.T) {
a, _ := setup(t, []catalog.Model{{Name: "qwen", Aliases: []string{"q"}, Source: "/m.gguf",
Scaling: catalog.Scaling{Min: 1, Max: 1}, Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}}})
answer := func(_ context.Context, _, _ string, _ []byte) (int, []byte, time.Duration, error) {
return 200, []byte(`{"choices":[{"message":{"content":"42"}}]}`), 0, nil
}
a.SetEval(eval.NewService(eval.Deps{Call: answer, Log: slog.New(slog.DiscardHandler),
Builtin: fstest.MapFS{"arith.yaml": {Data: []byte("cases:\n - prompt: 40 + 2?\n number: 42\n")}}}))
models := &gatingModels{fakeModels: &fakeModels{}}
a.SetModels(models)
mux := http.NewServeMux()
a.Register(mux)
do := func(method, path, body string) *httptest.ResponseRecorder {
req := httptest.NewRequest(method, path, strings.NewReader(body))
rec := httptest.NewRecorder()
mux.ServeHTTP(rec, req)
return rec
}
rec := do(http.MethodPost, "/api/v1/models/q/eval", `{"suite": "arith"}`)
var live eval.Live
if rec.Code != http.StatusAccepted || json.Unmarshal(rec.Body.Bytes(), &live) != nil || live.Model != "qwen" {
t.Fatalf("%d %s", rec.Code, rec.Body.String())
}
if rec := do(http.MethodPost, "/api/v1/models/nobody/eval", `{"suite": "arith"}`); rec.Code != http.StatusNotFound {
t.Errorf("a model that is not there: %d %s", rec.Code, rec.Body.String())
}
if rec := do(http.MethodPost, "/api/v1/models/qwen/eval", `{"suite": "nope"}`); rec.Code != http.StatusBadRequest ||
!strings.Contains(rec.Body.String(), "--suites") {
t.Errorf("a suite that is not there: %d %s", rec.Code, rec.Body.String())
}
if rec := do(http.MethodGet, "/api/v1/eval/suites", ""); !strings.Contains(rec.Body.String(), `"name":"arith"`) {
t.Errorf("suites: %s", rec.Body.String())
}
if rec := do(http.MethodPut, "/api/v1/models/qwen/eval-gate", `{"suite": "nope"}`); rec.Code != http.StatusBadRequest || models.set {
t.Errorf("a gate on a suite that is not there was set: %d %s", rec.Code, rec.Body.String())
}
if rec := do(http.MethodPut, "/api/v1/models/qwen/eval-gate", `{"suite": "arith", "min_score": 0.9}`); rec.Code != http.StatusOK ||
models.gate == nil || models.gate.Suite != "arith" || models.gate.MinScore != 0.9 {
t.Errorf("%d %s %+v", rec.Code, rec.Body.String(), models.gate)
}
if rec := do(http.MethodDelete, "/api/v1/models/qwen/eval-gate", ""); rec.Code != http.StatusOK || models.gate != nil {
t.Errorf("%d %s", rec.Code, rec.Body.String())
}
deadline := time.Now().Add(5 * time.Second)
for live.State == eval.Running && time.Now().Before(deadline) {
time.Sleep(10 * time.Millisecond)
_ = json.Unmarshal(do(http.MethodGet, "/api/v1/eval/runs/"+live.ID, "").Body.Bytes(), &live)
}
if live.State != eval.Done || live.Score != 1 || len(live.Results) != 1 {
t.Fatalf("%+v", live)
}
}
+25 -2
View File
@@ -5,6 +5,7 @@ import (
llmgw "gitea.mynodes.xyz/ilker/llmgw"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
@@ -58,6 +59,7 @@ var PathParams = map[string]string{
"models/{name}": "The model's name in the catalog.",
"nodes/{id}": "The node's ID, as `llmgw ctl nodes` lists it.",
"jobs/{id}": "The job's ID, as starting it answered.",
"runs/{id}": "The quality run's ID, as starting it answered.",
"requests/{id}": "The request's ID: the x-request-id a client was answered with.",
"fabrics/{id}": "The physical fabric's ID, as GET /api/v1/fabrics lists it.",
"quotas/{scope}": "The scope: org:NAME, team:ORG/TEAM, project:ORG/TEAM/PROJECT, or their IDs after the colon.",
@@ -171,7 +173,8 @@ var Routes = []Route{
{Method: "POST", Path: "/api/v1/updates/check", Summary: "Ask the registry now, about one model or every one",
Request: CheckRequest{}, OptionalBody: true, Response: []modelserve.UpdateInfo{}},
{Method: "POST", Path: "/api/v1/models/{name}/update", Summary: "Move a model to its repository's newer weights",
Response: modelserve.Job{}, Status: http.StatusAccepted},
Description: "With a quality suite, or the model's standing gate, the new weights are asked it first as a hidden canary.",
Request: modelserve.Change{}, OptionalBody: true, Response: modelserve.Job{}, Status: http.StatusAccepted},
{Method: "POST", Path: "/api/v1/models/{name}/rollback", Summary: "Go back to the weights the last update replaced",
Response: Result{}},
{Method: "POST", Path: "/api/v1/models/{name}/origin", Summary: "Record the repository a model's weights came from",
@@ -185,7 +188,27 @@ var Routes = []Route{
{Method: "DELETE", Path: "/api/v1/models/{name}/canary", Summary: "Take a canary out: its share first, then the candidate",
Response: Result{}},
{Method: "POST", Path: "/api/v1/models/{name}/promote", Summary: "Make a canary's weights the model's",
Response: modelserve.Job{}, Status: http.StatusAccepted},
Description: "With a quality suite, or the model's standing gate, the canary is asked it first and kept below the score.",
Request: modelserve.Change{}, OptionalBody: true, Response: modelserve.Job{}, Status: http.StatusAccepted},
// Quality suites.
{Method: "GET", Path: "/api/v1/eval/suites", Summary: "The quality suites the control plane can run by name",
Response: []eval.SuiteInfo{}},
{Method: "POST", Path: "/api/v1/models/{name}/eval", Summary: "Ask a model a quality suite",
Description: "Started at once and answered with the run, to be followed at /api/v1/eval/runs/{id}.",
Request: eval.Request{}, Response: eval.Live{}, Status: http.StatusAccepted},
{Method: "GET", Path: "/api/v1/eval/runs", Summary: "Quality runs, the newest first",
Query: []Param{
{Name: "model", Description: "Only this model's runs."},
{Name: "limit", Type: "integer", Description: "How many; a sensible number when absent."},
},
Response: []eval.Live{}},
{Method: "GET", Path: "/api/v1/eval/runs/{id}", Summary: "One quality run, with each case",
Response: eval.Live{}},
{Method: "PUT", Path: "/api/v1/models/{name}/eval-gate", Summary: "Hold a model's updates and promotions to a quality suite",
Request: EvalGateRequest{}, Response: EvalGateView{}},
{Method: "DELETE", Path: "/api/v1/models/{name}/eval-gate", Summary: "Stop holding a model's updates to a suite",
Response: EvalGateView{}},
// Tenancy: quotas and guardrails.
{Method: "GET", Path: "/api/v1/quotas", Summary: "Every capacity quota, with what its scope holds now",