`ctl bench` has always been able to work out what one replica can serve — it ramps concurrency until the SLO breaks and reports the last level that held. It printed that and nothing kept it, so the one number saying what a replica is worth lived in whoever ran it's scrollback. Everything downstream then had to guess: admission applies a single global default for healthy queue depth, and sizing a model against real demand had nothing to size against. `ctl bench --save` records it; `ctl capacity` reads it back. Keyed by model and profile. Capacity belongs to the hardware and the engine arguments as much as to the weights, so the same model on CPU and on an H100 are two numbers, and one key would overwrite the other on every run. The profile is looked up from the replica rather than asked for, because the control plane already knows which profile answers on which endpoint and a measurement filed under the wrong one is worse than none. A target that is not a replica — the gateway, usually — is refused with that said: benchmarking the whole path is worth doing and says nothing about the hardware. The conclusion is stored, not the ramp. Every level and its latencies are worth reading once while judging whether the run was sound; keeping them would grow without bound as a model is re-benchmarked. Re-measuring replaces. A second run is a better answer to the same question, not another data point, and keeping both would leave a reader choosing between numbers with no basis for the choice. The SLO and the date are stored with it. Capacity means nothing without the target it was measured against, and hardware and engine builds change — a measurement is an observation with a date on it, not a fact about the model. A failed save is reported and not fatal: the run has already cost the GPU time. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
138 lines
4.2 KiB
Go
138 lines
4.2 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func capacityStore(t *testing.T) *SQLite {
|
|
t.Helper()
|
|
s, err := OpenSQLite(context.Background(), filepath.Join(t.TempDir(), "cap.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = s.Close() })
|
|
return s
|
|
}
|
|
|
|
func TestAMeasurementSurvivesTheTerminalItWasPrintedIn(t *testing.T) {
|
|
s := capacityStore(t)
|
|
at := time.Now().UTC().Truncate(time.Second)
|
|
|
|
if err := s.PutCapacity(t.Context(), StoredCapacity{
|
|
Model: "qwen", Profile: "cpu",
|
|
Measurement: []byte(`{"max_concurrency_at_slo":8}`),
|
|
MeasuredAt: at, MeasuredBy: "ilker",
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
got, err := s.Capacities(t.Context())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(got) != 1 {
|
|
t.Fatalf("expected the one measurement, got %d", len(got))
|
|
}
|
|
if got[0].Model != "qwen" || got[0].Profile != "cpu" {
|
|
t.Errorf("wrong row: %+v", got[0])
|
|
}
|
|
if string(got[0].Measurement) != `{"max_concurrency_at_slo":8}` {
|
|
t.Errorf("the measurement came back changed: %s", got[0].Measurement)
|
|
}
|
|
if !got[0].MeasuredAt.UTC().Equal(at) {
|
|
t.Errorf("when it was taken is part of the answer; got %v want %v", got[0].MeasuredAt, at)
|
|
}
|
|
}
|
|
|
|
func TestTheSameModelOnTwoProfilesIsTwoMeasurements(t *testing.T) {
|
|
s := capacityStore(t)
|
|
for _, p := range []string{"cpu", "gpu"} {
|
|
if err := s.PutCapacity(t.Context(), StoredCapacity{
|
|
Model: "qwen", Profile: p, Measurement: []byte(`{}`),
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// Capacity belongs to the hardware and the engine arguments as much as to
|
|
// the weights. Keyed by model alone, benchmarking the GPU nodes would
|
|
// overwrite what is known about the CPU ones.
|
|
got, _ := s.Capacities(t.Context())
|
|
if len(got) != 2 {
|
|
t.Fatalf("expected one measurement per profile, got %d", len(got))
|
|
}
|
|
}
|
|
|
|
func TestRemeasuringReplacesRatherThanAccumulates(t *testing.T) {
|
|
s := capacityStore(t)
|
|
older := time.Now().Add(-24 * time.Hour)
|
|
if err := s.PutCapacity(t.Context(), StoredCapacity{
|
|
Model: "qwen", Profile: "cpu",
|
|
Measurement: []byte(`{"max_concurrency_at_slo":4}`), MeasuredAt: older,
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.PutCapacity(t.Context(), StoredCapacity{
|
|
Model: "qwen", Profile: "cpu",
|
|
Measurement: []byte(`{"max_concurrency_at_slo":9}`), MeasuredAt: time.Now(),
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// A second run is a better answer to the same question, not another data
|
|
// point: keeping both would leave a reader choosing between them with no
|
|
// basis for the choice.
|
|
got, _ := s.Capacities(t.Context())
|
|
if len(got) != 1 {
|
|
t.Fatalf("expected the measurement to be replaced, got %d rows", len(got))
|
|
}
|
|
if string(got[0].Measurement) != `{"max_concurrency_at_slo":9}` {
|
|
t.Errorf("the older measurement won: %s", got[0].Measurement)
|
|
}
|
|
}
|
|
|
|
func TestMeasurementsComeBackNewestFirst(t *testing.T) {
|
|
s := capacityStore(t)
|
|
now := time.Now()
|
|
_ = s.PutCapacity(t.Context(), StoredCapacity{
|
|
Model: "old", Profile: "cpu", Measurement: []byte(`{}`),
|
|
MeasuredAt: now.Add(-48 * time.Hour),
|
|
})
|
|
_ = s.PutCapacity(t.Context(), StoredCapacity{
|
|
Model: "new", Profile: "cpu", Measurement: []byte(`{}`), MeasuredAt: now,
|
|
})
|
|
|
|
got, _ := s.Capacities(t.Context())
|
|
// The old one is the one most likely to describe hardware that has since
|
|
// changed, so it is the one that should need scrolling to.
|
|
if len(got) != 2 || got[0].Model != "new" {
|
|
t.Errorf("expected the newest first, got %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestRubbishIsRefusedBeforeItIsStored(t *testing.T) {
|
|
s := capacityStore(t)
|
|
if err := s.PutCapacity(t.Context(), StoredCapacity{
|
|
Model: "qwen", Profile: "cpu", Measurement: []byte(`not json`),
|
|
}); err == nil {
|
|
t.Error("a measurement that is not JSON should be refused at the write")
|
|
}
|
|
if err := s.PutCapacity(t.Context(), StoredCapacity{
|
|
Profile: "cpu", Measurement: []byte(`{}`),
|
|
}); err == nil {
|
|
t.Error("a measurement with no model should be refused")
|
|
}
|
|
}
|
|
|
|
func TestForgettingAMeasurementThatIsNotThereIsFine(t *testing.T) {
|
|
s := capacityStore(t)
|
|
// The caller wanted it gone, and it is. Erroring would make cleaning up
|
|
// after a removed model a thing that needs a check first.
|
|
if err := s.DeleteCapacity(t.Context(), "ghost", "cpu"); err != nil {
|
|
t.Errorf("deleting an absent measurement should not error: %v", err)
|
|
}
|
|
}
|