llmgw/internal/control/adminapi/eval.go

package adminapi

import (
	"context"
	"encoding/json"
	"errors"
	"io"
	"net/http"
	"strconv"
	"strings"

	"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)

// Quality suites: run against a model on the control plane, through the
// router as the gateway's own requests (like a replay), followed while they
// run, and kept; and a model's standing suite that its updates must pass.

// SetEval gives the API the service that runs quality suites.
func (a *API) SetEval(s *eval.Service) { a.evals = s }

// maxSuiteBytes bounds a suite sent whole: a few hundred cases with their
// prompts, which is more than a gate should take to run.
const maxSuiteBytes = 4 << 20

// EvalGateRequest sets a model's standing quality suite; an empty suite
// takes it away.
type EvalGateRequest struct {
	Suite    string  `json:"suite"`
	MinScore float64 `json:"min_score,omitempty"`
	Judge    string  `json:"judge,omitempty"`
}

type evalGater interface {
	SetEval(ctx context.Context, name string, g *catalog.EvalGate, by string) (catalog.Model, error)
}

// readChange reads how an update or a promotion is held to quality; an
// empty body is no change to how.
func readChange(w http.ResponseWriter, r *http.Request) (modelserve.Change, bool) {
	var ch modelserve.Change
	b, err := io.ReadAll(io.LimitReader(r.Body, maxSuiteBytes+1))
	if err != nil {
		writeError(w, http.StatusBadRequest, err.Error())
		return ch, false
	}
	if len(b) > maxSuiteBytes {
		writeError(w, http.StatusRequestEntityTooLarge, "the suite is over 4 MiB")
		return ch, false
	}
	if len(strings.TrimSpace(string(b))) == 0 {
		return ch, true
	}
	if err := json.Unmarshal(b, &ch); err != nil {
		writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
		return ch, false
	}
	return ch, true
}

func (a *API) registerEval(mux *http.ServeMux) {
	need := func(w http.ResponseWriter) bool {
		if a.evals == nil {
			writeError(w, http.StatusNotImplemented, "this control plane does not route requests, so it cannot ask a model a suite")
			return false
		}
		return true
	}
	mux.HandleFunc("GET /api/v1/eval/suites", func(w http.ResponseWriter, r *http.Request) {
		if !need(w) {
			return
		}
		out := a.evals.Suites()
		if out == nil {
			out = []eval.SuiteInfo{}
		}
		writeJSON(w, http.StatusOK, out)
	})
	// Started and answered at once: a suite against a model on CPUs takes
	// minutes, longer than a client waits on one request.
	mux.HandleFunc("POST /api/v1/models/{name}/eval", func(w http.ResponseWriter, r *http.Request) {
		if !need(w) {
			return
		}
		var req eval.Request
		if err := json.NewDecoder(io.LimitReader(r.Body, maxSuiteBytes)).Decode(&req); err != nil {
			writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
			return
		}
		name := r.PathValue("name")
		if a.ctl != nil {
			// By its own name, so an alias's runs are filed with the rest.
			m, ok := a.ctl.Catalog().Resolve(name)
			if !ok {
				writeError(w, http.StatusNotFound, "no model called "+name+": `llmgw ctl models` lists them")
				return
			}
			name = m.Name
		}
		live, err := a.evals.Start(name, req, actorOf(r))
		if err != nil {
			writeError(w, http.StatusBadRequest, err.Error())
			return
		}
		writeJSON(w, http.StatusAccepted, live)
	})
	mux.HandleFunc("GET /api/v1/eval/runs", func(w http.ResponseWriter, r *http.Request) {
		if !need(w) {
			return
		}
		limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
		out, err := a.evals.History(r.Context(), r.URL.Query().Get("model"), limit)
		if err != nil {
			writeError(w, http.StatusInternalServerError, err.Error())
			return
		}
		if out == nil {
			out = []eval.Live{}
		}
		writeJSON(w, http.StatusOK, out)
	})
	mux.HandleFunc("GET /api/v1/eval/runs/{id}", func(w http.ResponseWriter, r *http.Request) {
		if !need(w) {
			return
		}
		live, err := a.evals.Get(r.Context(), r.PathValue("id"))
		if errors.Is(err, store.ErrNotFound) {
			writeError(w, http.StatusNotFound, "no eval run "+r.PathValue("id"))
			return
		}
		if err != nil {
			writeError(w, http.StatusInternalServerError, err.Error())
			return
		}
		writeJSON(w, http.StatusOK, live)
	})
	setGate := func(w http.ResponseWriter, r *http.Request, g *catalog.EvalGate) {
		gater, ok := a.models.(evalGater)
		if !ok {
			writeError(w, http.StatusNotImplemented, "this control plane has no store to keep the setting in")
			return
		}
		name := r.PathValue("name")
		if g != nil && a.evals != nil {
			if err := a.evals.Check(name, eval.Request{Suite: g.Suite, Judge: g.Judge, MinScore: g.MinScore}); err != nil {
				writeError(w, http.StatusBadRequest, err.Error())
				return
			}
		}
		m, err := gater.SetEval(r.Context(), name, g, actorOf(r))
		if err != nil {
			writeError(w, http.StatusBadRequest, err.Error())
			return
		}
		writeJSON(w, http.StatusOK, map[string]any{"model": m.Name, "eval": m.Eval})
	}
	mux.HandleFunc("PUT /api/v1/models/{name}/eval-gate", func(w http.ResponseWriter, r *http.Request) {
		var req EvalGateRequest
		if err := json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req); err != nil {
			writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
			return
		}
		if strings.TrimSpace(req.Suite) == "" {
			writeError(w, http.StatusBadRequest, "name a suite: one in the control plane's eval directory, or built in")
			return
		}
		if req.MinScore < 0 || req.MinScore > 1 {
			writeError(w, http.StatusBadRequest, "the minimum score is 0 to 1")
			return
		}
		setGate(w, r, &catalog.EvalGate{Suite: strings.TrimSpace(req.Suite), MinScore: req.MinScore, Judge: req.Judge})
	})
	mux.HandleFunc("DELETE /api/v1/models/{name}/eval-gate", func(w http.ResponseWriter, r *http.Request) {
		setGate(w, r, nil)
	})
}