Keep what a benchmark found out, instead of leaving it in a terminal

`ctl bench` has always been able to work out what one replica can serve — it
ramps concurrency until the SLO breaks and reports the last level that held.
It printed that and nothing kept it, so the one number saying what a replica is
worth lived in whoever ran it's scrollback. Everything downstream then had to
guess: admission applies a single global default for healthy queue depth, and
sizing a model against real demand had nothing to size against.

`ctl bench --save` records it; `ctl capacity` reads it back.

Keyed by model and profile. Capacity belongs to the hardware and the engine
arguments as much as to the weights, so the same model on CPU and on an H100
are two numbers, and one key would overwrite the other on every run.

The profile is looked up from the replica rather than asked for, because the
control plane already knows which profile answers on which endpoint and a
measurement filed under the wrong one is worse than none. A target that is not
a replica — the gateway, usually — is refused with that said: benchmarking the
whole path is worth doing and says nothing about the hardware.

The conclusion is stored, not the ramp. Every level and its latencies are worth
reading once while judging whether the run was sound; keeping them would grow
without bound as a model is re-benchmarked.

Re-measuring replaces. A second run is a better answer to the same question,
not another data point, and keeping both would leave a reader choosing between
numbers with no basis for the choice.

The SLO and the date are stored with it. Capacity means nothing without the
target it was measured against, and hardware and engine builds change — a
measurement is an observation with a date on it, not a fact about the model.

A failed save is reported and not fatal: the run has already cost the GPU time.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-27 11:51:58 +02:00
1 parent 707d774c76
commit 3aa250428a
12 files changed
+743 -4

No files matched your search

+115
View File
@@ -16,6 +16,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// nullSink discards published endpoints; these tests are about the API.
@@ -418,3 +419,117 @@ func TestANegativeReplicaCountIsRefused(t *testing.T) {
t.Error("nothing should have reached the service")
}
}
type memCapacity struct{ rows []store.StoredCapacity }
func (m *memCapacity) PutCapacity(_ context.Context, c store.StoredCapacity) error {
m.rows = append(m.rows, c)
return nil
}
func (m *memCapacity) Capacities(context.Context) ([]store.StoredCapacity, error) {
return m.rows, nil
}
func capacityAPI(t *testing.T) (*API, *memCapacity, *http.ServeMux) {
t.Helper()
a, _ := setup(t, testModels())
cap := &memCapacity{}
a.SetCapacity(cap)
mux := http.NewServeMux()
a.Register(mux)
return a, cap, mux
}
func TestAMeasurementIsFiledAgainstItsModelAndProfile(t *testing.T) {
_, cap, mux := capacityAPI(t)
body := `{"profile":"tp8","max_concurrency_at_slo":8,"suggested_target_waiting":3}`
req := httptest.NewRequest(http.MethodPut, "/api/v1/models/llama/capacity", strings.NewReader(body))
req.Header.Set("X-Llmgw-Actor", "ilker")
rec := httptest.NewRecorder()
mux.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected 200, got %d: %s", rec.Code, rec.Body)
}
if len(cap.rows) != 1 {
t.Fatalf("expected one row, got %d", len(cap.rows))
}
// The model comes from the path, not the body: a measurement filed under a
// name the caller made up would be worse than none.
if cap.rows[0].Model != "llama" || cap.rows[0].Profile != "tp8" {
t.Errorf("wrong key: %s/%s", cap.rows[0].Model, cap.rows[0].Profile)
}
if cap.rows[0].MeasuredBy != "ilker" {
t.Errorf("the measurement should say who took it; got %q", cap.rows[0].MeasuredBy)
}
}
func TestAMeasurementForAModelThatIsNotThereIsRefused(t *testing.T) {
_, cap, mux := capacityAPI(t)
req := httptest.NewRequest(http.MethodPut, "/api/v1/models/typo/capacity",
strings.NewReader(`{"profile":"tp8"}`))
rec := httptest.NewRecorder()
mux.ServeHTTP(rec, req)
// A typo in the name files the measurement somewhere nobody will look, and
// the model then looks as though it was never benchmarked.
if rec.Code != http.StatusNotFound {
t.Errorf("expected 404 for an unknown model, got %d", rec.Code)
}
if len(cap.rows) != 0 {
t.Error("nothing should have been stored")
}
}
func TestAMeasurementNeedsTheProfileItWasTakenOn(t *testing.T) {
_, cap, mux := capacityAPI(t)
req := httptest.NewRequest(http.MethodPut, "/api/v1/models/llama/capacity",
strings.NewReader(`{"max_concurrency_at_slo":8}`))
rec := httptest.NewRecorder()
mux.ServeHTTP(rec, req)
// Capacity belongs to the hardware as much as the weights. Without a
// profile there is nothing to compare a later run against.
if rec.Code != http.StatusBadRequest {
t.Errorf("expected 400 without a profile, got %d", rec.Code)
}
if len(cap.rows) != 0 {
t.Error("nothing should have been stored")
}
}
func TestAProfileTheModelDoesNotHaveIsRefused(t *testing.T) {
_, _, mux := capacityAPI(t)
req := httptest.NewRequest(http.MethodPut, "/api/v1/models/llama/capacity",
strings.NewReader(`{"profile":"not-a-profile"}`))
rec := httptest.NewRecorder()
mux.ServeHTTP(rec, req)
if rec.Code != http.StatusBadRequest {
t.Errorf("expected 400 for a profile the model does not have, got %d", rec.Code)
}
}
func TestCapacityWithoutAStoreIsAnEmptyListRatherThanAnError(t *testing.T) {
a, _ := setup(t, testModels())
mux := http.NewServeMux()
a.Register(mux)
rec := httptest.NewRecorder()
mux.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/v1/capacity", nil))
// Nothing measured and no store look the same to a reader, and both mean
// "there is nothing here" — which is not a server fault.
if rec.Code != http.StatusOK {
t.Errorf("expected 200, got %d", rec.Code)
}
var rows []CapacityView
if err := json.Unmarshal(rec.Body.Bytes(), &rows); err != nil || len(rows) != 0 {
t.Errorf("expected an empty list, got %s", rec.Body)
}
}