package adminapi import ( "context" "encoding/json" "log/slog" "net/http" "net/http/httptest" "strings" "testing" "testing/fstest" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" ) // gatingModels is the catalog's service, keeping the gate it is given. type gatingModels struct { *fakeModels gate *catalog.EvalGate set bool } func (g *gatingModels) SetEval(_ context.Context, name string, gate *catalog.EvalGate, _ string) (catalog.Model, error) { g.gate, g.set = gate, true return catalog.Model{Name: name, Eval: gate}, nil } func TestASuiteIsStartedByAModelsAliasAndAGateNeedsASuiteThatIsThere(t *testing.T) { a, _ := setup(t, []catalog.Model{{Name: "qwen", Aliases: []string{"q"}, Source: "/m.gguf", Scaling: catalog.Scaling{Min: 1, Max: 1}, Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}}}) answer := func(_ context.Context, _, _ string, _ []byte) (int, []byte, time.Duration, error) { return 200, []byte(`{"choices":[{"message":{"content":"42"}}]}`), 0, nil } a.SetEval(eval.NewService(eval.Deps{Call: answer, Log: slog.New(slog.DiscardHandler), Builtin: fstest.MapFS{"arith.yaml": {Data: []byte("cases:\n - prompt: 40 + 2?\n number: 42\n")}}})) models := &gatingModels{fakeModels: &fakeModels{}} a.SetModels(models) mux := http.NewServeMux() a.Register(mux) do := func(method, path, body string) *httptest.ResponseRecorder { req := httptest.NewRequest(method, path, strings.NewReader(body)) rec := httptest.NewRecorder() mux.ServeHTTP(rec, req) return rec } rec := do(http.MethodPost, "/api/v1/models/q/eval", `{"suite": "arith"}`) var live eval.Live if rec.Code != http.StatusAccepted || json.Unmarshal(rec.Body.Bytes(), &live) != nil || live.Model != "qwen" { t.Fatalf("%d %s", rec.Code, rec.Body.String()) } if rec := do(http.MethodPost, "/api/v1/models/nobody/eval", `{"suite": "arith"}`); rec.Code != http.StatusNotFound { t.Errorf("a model that is not there: %d %s", rec.Code, rec.Body.String()) } if rec := do(http.MethodPost, "/api/v1/models/qwen/eval", `{"suite": "nope"}`); rec.Code != http.StatusBadRequest || !strings.Contains(rec.Body.String(), "--suites") { t.Errorf("a suite that is not there: %d %s", rec.Code, rec.Body.String()) } if rec := do(http.MethodGet, "/api/v1/eval/suites", ""); !strings.Contains(rec.Body.String(), `"name":"arith"`) { t.Errorf("suites: %s", rec.Body.String()) } if rec := do(http.MethodPut, "/api/v1/models/qwen/eval-gate", `{"suite": "nope"}`); rec.Code != http.StatusBadRequest || models.set { t.Errorf("a gate on a suite that is not there was set: %d %s", rec.Code, rec.Body.String()) } if rec := do(http.MethodPut, "/api/v1/models/qwen/eval-gate", `{"suite": "arith", "min_score": 0.9}`); rec.Code != http.StatusOK || models.gate == nil || models.gate.Suite != "arith" || models.gate.MinScore != 0.9 { t.Errorf("%d %s %+v", rec.Code, rec.Body.String(), models.gate) } if rec := do(http.MethodDelete, "/api/v1/models/qwen/eval-gate", ""); rec.Code != http.StatusOK || models.gate != nil { t.Errorf("%d %s", rec.Code, rec.Body.String()) } deadline := time.Now().Add(5 * time.Second) for live.State == eval.Running && time.Now().Before(deadline) { time.Sleep(10 * time.Millisecond) _ = json.Unmarshal(do(http.MethodGet, "/api/v1/eval/runs/"+live.ID, "").Body.Bytes(), &live) } if live.State != eval.Done || live.Score != 1 || len(live.Results) != 1 { t.Fatalf("%+v", live) } }