Quality suites, and new weights held to one: ctl eval, model update/promote --eval, model gate
A suite is a set of cases in YAML or JSON lines, each scored by exact, contains, regex, a number within a tolerance, a JSON Schema subset, or a judge model with a rubric; weighed into one score against a threshold. internal/bench/eval runs one on the control plane through Router.Replay, as the gateway's own requests, and keeps every run and its cases in the store (migration 0028: eval_runs, eval_cases). - llmgw ctl eval MODEL --suite FILE|NAME, --history, --run, --suites; exits non-zero below the score. Suites by name from control.eval_dir, or the two built in from deploy/eval (arithmetic, json-format). - model update and promote take --eval SUITE --min-score N, or a model's standing gate (ctl model gate, eval: in the catalog): a promote asks the canary first and leaves it below the score; an update tries the new weights as a canary given no share, then promotes or removes it. - The console's model menu: ask a suite with its progress, its runs, its gate. - The audit middleware no longer hands a handler a body cut off at 64 KiB. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
30f7f3474a
commit
f0477f140d
49 files changed
+4563
-46
No files matched your search
@@ -28,6 +28,7 @@ import (
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/bench"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
|
||||
@@ -43,6 +44,8 @@ import (
|
||||
type API struct {
|
||||
// replay sends a kept request again; nil without a router here.
|
||||
replay Replayer
|
||||
// evals runs quality suites against models; nil without a router here.
|
||||
evals *eval.Service
|
||||
// notifier is the webhooks `ctl notify test` reaches; nil without any.
|
||||
notifier *notify.Notifier
|
||||
// releases is where release binaries are fetched from, for upgrades.
|
||||
@@ -157,6 +160,7 @@ func (a *API) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/update", a.updateModel)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/rollback", a.rollbackModel)
|
||||
a.registerCanary(mux)
|
||||
a.registerEval(mux)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/origin", a.setOrigin)
|
||||
mux.HandleFunc("GET /api/v1/plan", a.planModel)
|
||||
mux.HandleFunc("POST /api/v1/metrics/range", a.metricsBatch)
|
||||
@@ -339,6 +343,8 @@ type ModelView struct {
|
||||
// set on the candidate, naming the model it is a canary of.
|
||||
Canary *catalog.Canary `json:"canary,omitempty"`
|
||||
CanaryOf string `json:"canary_of,omitempty"`
|
||||
// Eval is the quality suite its new weights must pass, when it has one.
|
||||
Eval *catalog.EvalGate `json:"eval,omitempty"`
|
||||
// Description is what the model is for, when somebody said.
|
||||
Description string `json:"description,omitempty"`
|
||||
Aliases []string `json:"aliases,omitempty"`
|
||||
@@ -660,7 +666,7 @@ func (a *API) Models() []ModelView {
|
||||
Parked: h.Parked, Reason: h.Reason, Unplaceable: h.Unplaceable,
|
||||
Class: m.EffectiveClass(), Owner: m.Owner, Preempted: h.Preempted, HeldFor: h.HeldFor,
|
||||
External: externalLabel(m), Adapters: m.Adapters,
|
||||
Canary: m.Canary, CanaryOf: a.ctl.Catalog().CanaryOf(m.Name),
|
||||
Canary: m.Canary, CanaryOf: a.ctl.Catalog().CanaryOf(m.Name), Eval: m.Eval,
|
||||
}
|
||||
for _, p := range m.Profiles {
|
||||
v.Profiles = append(v.Profiles, p.Name)
|
||||
@@ -1559,7 +1565,11 @@ func (a *API) updateModel(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, http.StatusNotImplemented, "updates need a mirror and a store, and this control plane lacks one")
|
||||
return
|
||||
}
|
||||
job, err := a.serve.StartUpdate(r.PathValue("name"), actorOf(r))
|
||||
ch, ok := readChange(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
job, err := a.serve.StartUpdate(r.PathValue("name"), actorOf(r), ch)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
|
||||
Reference in new issue
Block a user