A suite is a set of cases in YAML or JSON lines, each scored by exact, contains, regex, a number within a tolerance, a JSON Schema subset, or a judge model with a rubric; weighed into one score against a threshold. internal/bench/eval runs one on the control plane through Router.Replay, as the gateway's own requests, and keeps every run and its cases in the store (migration 0028: eval_runs, eval_cases). - llmgw ctl eval MODEL --suite FILE|NAME, --history, --run, --suites; exits non-zero below the score. Suites by name from control.eval_dir, or the two built in from deploy/eval (arithmetic, json-format). - model update and promote take --eval SUITE --min-score N, or a model's standing gate (ctl model gate, eval: in the catalog): a promote asks the canary first and leaves it below the score; an update tries the new weights as a canary given no share, then promotes or removes it. - The console's model menu: ask a suite with its progress, its runs, its gate. - The audit middleware no longer hands a handler a body cut off at 64 KiB. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
89 lines
2.5 KiB
Go
89 lines
2.5 KiB
Go
package adminapi
|
|
|
|
import (
|
|
"encoding/json"
|
|
"io"
|
|
"net/http"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
)
|
|
|
|
func (a *API) registerCanary(mux *http.ServeMux) {
|
|
need := func(w http.ResponseWriter) bool {
|
|
if a.serve == nil {
|
|
writeError(w, http.StatusNotImplemented, "a canary needs a mirror and a store, and this control plane lacks one")
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
mux.HandleFunc("GET /api/v1/canaries", func(w http.ResponseWriter, r *http.Request) {
|
|
if !need(w) {
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, a.serve.Canaries())
|
|
})
|
|
// Start one: the newer weights fetched and run beside, then given
|
|
// their share.
|
|
mux.HandleFunc("POST /api/v1/models/{name}/canary", func(w http.ResponseWriter, r *http.Request) {
|
|
if !need(w) {
|
|
return
|
|
}
|
|
var cn catalog.Canary
|
|
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&cn); err != nil {
|
|
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
|
|
return
|
|
}
|
|
job, err := a.serve.StartCanary(r.PathValue("name"), cn, actorOf(r))
|
|
if err != nil {
|
|
writeError(w, http.StatusBadRequest, err.Error())
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusAccepted, job)
|
|
})
|
|
// Change its share.
|
|
mux.HandleFunc("PUT /api/v1/models/{name}/canary", func(w http.ResponseWriter, r *http.Request) {
|
|
if !need(w) {
|
|
return
|
|
}
|
|
var req struct {
|
|
Percent int `json:"percent"`
|
|
}
|
|
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req); err != nil {
|
|
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
|
|
return
|
|
}
|
|
msg, err := a.serve.SetCanaryPercent(r.Context(), r.PathValue("name"), req.Percent, actorOf(r))
|
|
if err != nil {
|
|
writeError(w, http.StatusBadRequest, err.Error())
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, map[string]string{"result": msg})
|
|
})
|
|
mux.HandleFunc("DELETE /api/v1/models/{name}/canary", func(w http.ResponseWriter, r *http.Request) {
|
|
if !need(w) {
|
|
return
|
|
}
|
|
msg, err := a.serve.AbortCanary(r.Context(), r.PathValue("name"), actorOf(r))
|
|
if err != nil {
|
|
writeError(w, http.StatusBadRequest, err.Error())
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, map[string]string{"result": msg})
|
|
})
|
|
mux.HandleFunc("POST /api/v1/models/{name}/promote", func(w http.ResponseWriter, r *http.Request) {
|
|
if !need(w) {
|
|
return
|
|
}
|
|
ch, ok := readChange(w, r)
|
|
if !ok {
|
|
return
|
|
}
|
|
job, err := a.serve.StartPromote(r.PathValue("name"), actorOf(r), ch)
|
|
if err != nil {
|
|
writeError(w, http.StatusBadRequest, err.Error())
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusAccepted, job)
|
|
})
|
|
}
|