Ilker ManapandClaude Opus 5.5
f0477f140d
A suite is a set of cases in YAML or JSON lines, each scored by exact, contains, regex, a number within a tolerance, a JSON Schema subset, or a judge model with a rubric; weighed into one score against a threshold. internal/bench/eval runs one on the control plane through Router.Replay, as the gateway's own requests, and keeps every run and its cases in the store (migration 0028: eval_runs, eval_cases). - llmgw ctl eval MODEL --suite FILE|NAME, --history, --run, --suites; exits non-zero below the score. Suites by name from control.eval_dir, or the two built in from deploy/eval (arithmetic, json-format). - model update and promote take --eval SUITE --min-score N, or a model's standing gate (ctl model gate, eval: in the catalog): a promote asks the canary first and leaves it below the score; an update tries the new weights as a canary given no share, then promotes or removes it. - The console's model menu: ask a suite with its progress, its runs, its gate. - The audit middleware no longer hands a handler a body cut off at 64 KiB. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
85 lines
3.4 KiB
Go
85 lines
3.4 KiB
Go
package adminapi
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"log/slog"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
"testing/fstest"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
)
|
|
|
|
// gatingModels is the catalog's service, keeping the gate it is given.
|
|
type gatingModels struct {
|
|
*fakeModels
|
|
gate *catalog.EvalGate
|
|
set bool
|
|
}
|
|
|
|
func (g *gatingModels) SetEval(_ context.Context, name string, gate *catalog.EvalGate, _ string) (catalog.Model, error) {
|
|
g.gate, g.set = gate, true
|
|
return catalog.Model{Name: name, Eval: gate}, nil
|
|
}
|
|
|
|
func TestASuiteIsStartedByAModelsAliasAndAGateNeedsASuiteThatIsThere(t *testing.T) {
|
|
a, _ := setup(t, []catalog.Model{{Name: "qwen", Aliases: []string{"q"}, Source: "/m.gguf",
|
|
Scaling: catalog.Scaling{Min: 1, Max: 1}, Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}}})
|
|
answer := func(_ context.Context, _, _ string, _ []byte) (int, []byte, time.Duration, error) {
|
|
return 200, []byte(`{"choices":[{"message":{"content":"42"}}]}`), 0, nil
|
|
}
|
|
a.SetEval(eval.NewService(eval.Deps{Call: answer, Log: slog.New(slog.DiscardHandler),
|
|
Builtin: fstest.MapFS{"arith.yaml": {Data: []byte("cases:\n - prompt: 40 + 2?\n number: 42\n")}}}))
|
|
models := &gatingModels{fakeModels: &fakeModels{}}
|
|
a.SetModels(models)
|
|
mux := http.NewServeMux()
|
|
a.Register(mux)
|
|
do := func(method, path, body string) *httptest.ResponseRecorder {
|
|
req := httptest.NewRequest(method, path, strings.NewReader(body))
|
|
rec := httptest.NewRecorder()
|
|
mux.ServeHTTP(rec, req)
|
|
return rec
|
|
}
|
|
|
|
rec := do(http.MethodPost, "/api/v1/models/q/eval", `{"suite": "arith"}`)
|
|
var live eval.Live
|
|
if rec.Code != http.StatusAccepted || json.Unmarshal(rec.Body.Bytes(), &live) != nil || live.Model != "qwen" {
|
|
t.Fatalf("%d %s", rec.Code, rec.Body.String())
|
|
}
|
|
if rec := do(http.MethodPost, "/api/v1/models/nobody/eval", `{"suite": "arith"}`); rec.Code != http.StatusNotFound {
|
|
t.Errorf("a model that is not there: %d %s", rec.Code, rec.Body.String())
|
|
}
|
|
if rec := do(http.MethodPost, "/api/v1/models/qwen/eval", `{"suite": "nope"}`); rec.Code != http.StatusBadRequest ||
|
|
!strings.Contains(rec.Body.String(), "--suites") {
|
|
t.Errorf("a suite that is not there: %d %s", rec.Code, rec.Body.String())
|
|
}
|
|
if rec := do(http.MethodGet, "/api/v1/eval/suites", ""); !strings.Contains(rec.Body.String(), `"name":"arith"`) {
|
|
t.Errorf("suites: %s", rec.Body.String())
|
|
}
|
|
|
|
if rec := do(http.MethodPut, "/api/v1/models/qwen/eval-gate", `{"suite": "nope"}`); rec.Code != http.StatusBadRequest || models.set {
|
|
t.Errorf("a gate on a suite that is not there was set: %d %s", rec.Code, rec.Body.String())
|
|
}
|
|
if rec := do(http.MethodPut, "/api/v1/models/qwen/eval-gate", `{"suite": "arith", "min_score": 0.9}`); rec.Code != http.StatusOK ||
|
|
models.gate == nil || models.gate.Suite != "arith" || models.gate.MinScore != 0.9 {
|
|
t.Errorf("%d %s %+v", rec.Code, rec.Body.String(), models.gate)
|
|
}
|
|
if rec := do(http.MethodDelete, "/api/v1/models/qwen/eval-gate", ""); rec.Code != http.StatusOK || models.gate != nil {
|
|
t.Errorf("%d %s", rec.Code, rec.Body.String())
|
|
}
|
|
|
|
deadline := time.Now().Add(5 * time.Second)
|
|
for live.State == eval.Running && time.Now().Before(deadline) {
|
|
time.Sleep(10 * time.Millisecond)
|
|
_ = json.Unmarshal(do(http.MethodGet, "/api/v1/eval/runs/"+live.ID, "").Body.Bytes(), &live)
|
|
}
|
|
if live.State != eval.Done || live.Score != 1 || len(live.Results) != 1 {
|
|
t.Fatalf("%+v", live)
|
|
}
|
|
}
|