Keep what a benchmark found out, instead of leaving it in a terminal

`ctl bench` has always been able to work out what one replica can serve — it
ramps concurrency until the SLO breaks and reports the last level that held.
It printed that and nothing kept it, so the one number saying what a replica is
worth lived in whoever ran it's scrollback. Everything downstream then had to
guess: admission applies a single global default for healthy queue depth, and
sizing a model against real demand had nothing to size against.

`ctl bench --save` records it; `ctl capacity` reads it back.

Keyed by model and profile. Capacity belongs to the hardware and the engine
arguments as much as to the weights, so the same model on CPU and on an H100
are two numbers, and one key would overwrite the other on every run.

The profile is looked up from the replica rather than asked for, because the
control plane already knows which profile answers on which endpoint and a
measurement filed under the wrong one is worse than none. A target that is not
a replica — the gateway, usually — is refused with that said: benchmarking the
whole path is worth doing and says nothing about the hardware.

The conclusion is stored, not the ramp. Every level and its latencies are worth
reading once while judging whether the run was sound; keeping them would grow
without bound as a model is re-benchmarked.

Re-measuring replaces. A second run is a better answer to the same question,
not another data point, and keeping both would leave a reader choosing between
numbers with no basis for the choice.

The SLO and the date are stored with it. Capacity means nothing without the
target it was measured against, and hardware and engine builds change — a
measurement is an observation with a date on it, not a fact about the model.

A failed save is reported and not fatal: the run has already cost the GPU time.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-27 11:51:58 +02:00
1 parent 707d774c76
commit 3aa250428a
12 files changed
+743 -4

No files matched your search

+100 -2
View File
@@ -12,18 +12,22 @@ import (
"fmt"
"io"
"net/http"
"slices"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench"
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// API serves the management endpoints.
type API struct {
models Models
ctl *control.Controller
models Models
capacity Capacity
ctl *control.Controller
}
// New builds the API over a controller.
@@ -38,6 +42,8 @@ func (a *API) Register(mux *http.ServeMux) {
mux.HandleFunc("POST /api/v1/models", a.addModel)
mux.HandleFunc("DELETE /api/v1/models/{name}", a.removeModel)
mux.HandleFunc("POST /api/v1/models/{name}/scale", a.scaleModel)
mux.HandleFunc("PUT /api/v1/models/{name}/capacity", a.putCapacity)
mux.HandleFunc("GET /api/v1/capacity", a.listCapacity)
mux.HandleFunc("GET /api/v1/nodes", a.listNodes)
mux.HandleFunc("GET /api/v1/models", a.listModels)
mux.HandleFunc("GET /api/v1/replicas", a.listReplicas)
@@ -308,6 +314,98 @@ type Models interface {
Scale(ctx context.Context, name string, s catalog.Scaling, by string) (catalog.Model, error)
}
// Capacity is where measured capacity is kept. The store implements it.
type Capacity interface {
PutCapacity(ctx context.Context, c store.StoredCapacity) error
Capacities(ctx context.Context) ([]store.StoredCapacity, error)
}
// SetCapacity supplies somewhere to keep benchmark results.
func (a *API) SetCapacity(c Capacity) { a.capacity = c }
// putCapacity records what a replica of a model was measured to serve.
//
// A PUT rather than a POST: re-running a benchmark replaces what was known
// about that model on that profile. It is a better answer to the same question,
// not another data point.
func (a *API) putCapacity(w http.ResponseWriter, r *http.Request) {
if a.capacity == nil {
writeError(w, http.StatusNotImplemented,
"this control plane has no store, so a measurement recorded here would not survive a restart")
return
}
var c bench.Capacity
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<20)).Decode(&c); err != nil {
writeError(w, http.StatusBadRequest, "the measurement is not valid JSON: "+err.Error())
return
}
c.Model = r.PathValue("name")
if c.Profile == "" {
writeError(w, http.StatusBadRequest,
"a measurement needs the profile it was taken on: the same model on CPU and on a GPU are different numbers")
return
}
// The model has to exist now, which catches a typo in the name — a
// measurement filed under a misspelling is worse than none, because it
// looks like the model was never benchmarked. It is not deleted when the
// model is, though: a benchmark costs real GPU time and outliving the
// catalog entry is the point.
m, ok := a.ctl.Catalog().Resolve(c.Model)
if !ok {
writeError(w, http.StatusNotFound, "no model named "+c.Model+" is in the catalog")
return
}
if !slices.ContainsFunc(m.Profiles, func(p catalog.Profile) bool { return p.Name == c.Profile }) {
writeError(w, http.StatusBadRequest,
"model "+c.Model+" has no profile named "+c.Profile)
return
}
body, err := json.Marshal(c)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if err := a.capacity.PutCapacity(r.Context(), store.StoredCapacity{
Model: c.Model, Profile: c.Profile, Measurement: body,
MeasuredAt: c.MeasuredAt, MeasuredBy: r.Header.Get("X-Llmgw-Actor"),
}); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, c)
}
// CapacityView is one stored measurement, with who recorded it.
type CapacityView struct {
bench.Capacity
MeasuredBy string `json:"measured_by,omitempty"`
}
func (a *API) listCapacity(w http.ResponseWriter, r *http.Request) {
if a.capacity == nil {
writeJSON(w, http.StatusOK, []CapacityView{})
return
}
rows, err := a.capacity.Capacities(r.Context())
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
out := make([]CapacityView, 0, len(rows))
for _, row := range rows {
var c bench.Capacity
if err := json.Unmarshal(row.Measurement, &c); err != nil {
// Skipped rather than failing the listing: one row written by an
// older shape must not hide every other measurement.
continue
}
out = append(out, CapacityView{Capacity: c, MeasuredBy: row.MeasuredBy})
}
writeJSON(w, http.StatusOK, out)
}
// SetModels supplies the catalog-changing service.
func (a *API) SetModels(m Models) { a.models = m }