From 3aa250428a125e455a1b4f59f7412f019241f42d Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sun, 27 Sep 2026 11:51:58 +0200 Subject: [PATCH] Keep what a benchmark found out, instead of leaving it in a terminal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `ctl bench` has always been able to work out what one replica can serve — it ramps concurrency until the SLO breaks and reports the last level that held. It printed that and nothing kept it, so the one number saying what a replica is worth lived in whoever ran it's scrollback. Everything downstream then had to guess: admission applies a single global default for healthy queue depth, and sizing a model against real demand had nothing to size against. `ctl bench --save` records it; `ctl capacity` reads it back. Keyed by model and profile. Capacity belongs to the hardware and the engine arguments as much as to the weights, so the same model on CPU and on an H100 are two numbers, and one key would overwrite the other on every run. The profile is looked up from the replica rather than asked for, because the control plane already knows which profile answers on which endpoint and a measurement filed under the wrong one is worse than none. A target that is not a replica — the gateway, usually — is refused with that said: benchmarking the whole path is worth doing and says nothing about the hardware. The conclusion is stored, not the ramp. Every level and its latencies are worth reading once while judging whether the run was sound; keeping them would grow without bound as a model is re-benchmarked. Re-measuring replaces. A second run is a better answer to the same question, not another data point, and keeping both would leave a reader choosing between numbers with no basis for the choice. The SLO and the date are stored with it. Capacity means nothing without the target it was measured against, and hardware and engine builds change — a measurement is an observation with a date on it, not a fact about the model. A failed save is reported and not fatal: the run has already cost the GPU time. Co-Authored-By: Claude Opus 5 (1M context) --- internal/bench/capacity.go | 77 ++++++++++ internal/cli/bench.go | 68 ++++++++- internal/cli/capacity.go | 77 ++++++++++ internal/cli/ctl.go | 9 +- internal/cli/server.go | 4 + internal/control/adminapi/api.go | 102 ++++++++++++- internal/control/adminapi/api_test.go | 115 +++++++++++++++ internal/store/capacity.go | 88 +++++++++++ internal/store/capacity_test.go | 137 ++++++++++++++++++ internal/store/copy.go | 4 + .../migrations/postgres/0009_capacity.sql | 34 +++++ .../store/migrations/sqlite/0009_capacity.sql | 32 ++++ 12 files changed, 743 insertions(+), 4 deletions(-) create mode 100644 internal/bench/capacity.go create mode 100644 internal/cli/capacity.go create mode 100644 internal/store/capacity.go create mode 100644 internal/store/capacity_test.go create mode 100644 internal/store/migrations/postgres/0009_capacity.sql create mode 100644 internal/store/migrations/sqlite/0009_capacity.sql diff --git a/internal/bench/capacity.go b/internal/bench/capacity.go new file mode 100644 index 0000000..18a8e06 --- /dev/null +++ b/internal/bench/capacity.go @@ -0,0 +1,77 @@ +package bench + +import "time" + +// Capacity is what one replica was measured to serve, kept so the measurement +// outlives the terminal it was printed in. +// +// The conclusion of a run, not the run. The ramp itself — every concurrency +// level, its latencies, its errors — is worth reading once, while deciding +// whether the run was sound; it is not worth carrying around afterwards, and +// storing it would grow without bound as a model is re-benchmarked. What a +// later reader needs is the number and enough provenance to judge whether it is +// still true. +type Capacity struct { + Model string `json:"model"` + Profile string `json:"profile"` + + // MaxConcurrencyAtSLO is the capacity: the highest concurrency that still + // met the target. Zero means even one request did not, which is a real + // result and not a missing one — it says this hardware cannot serve this + // model at this SLO at all. + MaxConcurrencyAtSLO int `json:"max_concurrency_at_slo"` + // ThroughputTokPerSec and the latencies are taken at that level, not at the + // peak. Peak throughput arrives long after latency has become unusable, so + // reporting it would overstate what the hardware can serve. + ThroughputTokPerSec float64 `json:"throughput_tok_per_sec"` + TTFTp99 time.Duration `json:"ttft_p99"` + TPOTp99 time.Duration `json:"tpot_p99"` + + // SuggestedTargetWaiting and SuggestedKVCeiling are the queue depth and + // cache pressure seen while the SLO was still being met, which is what + // admission should treat as healthy for this model. Admission currently + // applies one global default to every model; these are the per-model + // numbers it should be using instead. + SuggestedTargetWaiting int `json:"suggested_target_waiting"` + SuggestedKVCeiling float64 `json:"suggested_kv_ceiling"` + + // SLO is recorded because capacity is meaningless without it. The same + // replica has a different capacity against a 200ms first-token target than + // against a 2s one, and a number stored without the target it was measured + // against cannot be compared with anything. + SLO SLO `json:"slo"` + + // Target is the endpoint the load actually went to, kept so a profile + // measured against the gateway is never mistaken for one measured against + // a replica. Only the second says anything about the hardware. + Target string `json:"target"` + MeasuredAt time.Time `json:"measured_at"` + // Stale is not a field. Whether a measurement still holds depends on + // whether the hardware and the engine build have changed since, which this + // package cannot know; MeasuredAt is given so the reader can decide. +} + +// Capacity summarizes a completed run for storage. +func (r Result) Capacity(profile string) Capacity { + return Capacity{ + Model: r.Model, + Profile: profile, + MaxConcurrencyAtSLO: r.MaxConcurrencyAtSLO, + ThroughputTokPerSec: r.ThroughputTokPerSec, + TTFTp99: r.TTFTp99, + TPOTp99: r.TPOTp99, + SuggestedTargetWaiting: r.SuggestedTargetWaiting, + SuggestedKVCeiling: r.SuggestedKVCeiling, + SLO: r.SLO, + Target: r.Target, + MeasuredAt: r.RanAt, + } +} + +// Usable reports whether this measurement says anything. +// +// A run that never met the SLO, or one stopped before it finished a level, +// concludes at zero. That is worth recording — it is the answer to "can this +// run here at all" — but a caller sizing a model against it would compute a +// replica count of infinity, so it has to be asked about rather than used. +func (c Capacity) Usable() bool { return c.MaxConcurrencyAtSLO > 0 } diff --git a/internal/cli/bench.go b/internal/cli/bench.go index 3d4d2a0..55e3b29 100644 --- a/internal/cli/bench.go +++ b/internal/cli/bench.go @@ -3,14 +3,16 @@ package cli import ( "encoding/json" "fmt" + "strings" "time" "github.com/spf13/cobra" "gitea.mynodes.xyz/ilker/llmgw/internal/bench" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" ) -func newBenchCmd(asJSON *bool) *cobra.Command { +func newBenchCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { cfg := bench.DefaultConfig() var ( steps []int @@ -18,6 +20,7 @@ func newBenchCmd(asJSON *bool) *cobra.Command { metrics string engine string key string + save bool ) cmd := &cobra.Command{ @@ -78,6 +81,15 @@ func newBenchCmd(asJSON *bool) *cobra.Command { if err != nil { return err } + if save { + if err := saveCapacity(cmd, client, res, target); err != nil { + // Reported, not fatal. The run has already cost the GPU + // time; losing its conclusion to a failed write would mean + // running it again, and the numbers are on screen either + // way. + cmd.PrintErrf("the measurement was not saved: %v\n", err) + } + } if *asJSON { return json.NewEncoder(cmd.OutOrStdout()).Encode(res) } @@ -99,9 +111,63 @@ func newBenchCmd(asJSON *bool) *cobra.Command { "send every request the same prompt, measuring the cached case instead of the hardware") f.DurationVar(&cfg.SLO.TTFTp99, "ttft-p99", cfg.SLO.TTFTp99, "time-to-first-token target") f.DurationVar(&cfg.SLO.TPOTp99, "tpot-p99", cfg.SLO.TPOTp99, "per-output-token target") + f.BoolVar(&save, "save", false, + "record the capacity against the model and profile, so the number outlives this terminal") return cmd } +// saveCapacity records a run's conclusion against the profile it was measured +// on. +// +// The profile is looked up from the replica rather than asked for. Capacity +// belongs to the hardware and the engine arguments as much as to the weights, +// so a measurement filed under the wrong profile is worse than none — and the +// control plane already knows which profile is answering on which endpoint, so +// nobody needs to be trusted to type it. +func saveCapacity(cmd *cobra.Command, client func() *ctlClient, res bench.Result, target string) error { + var reps []adminapi.ReplicaView + if err := client().get(cmd.Context(), "/api/v1/replicas", &reps); err != nil { + return err + } + profile := "" + for _, rep := range reps { + if rep.Model == res.Model && sameEndpoint(rep.Endpoint, target) { + profile = rep.Profile + break + } + } + if profile == "" { + // Almost always because --target was the gateway. Benchmarking the + // whole path is a useful thing to do and says nothing about what one + // replica can serve, so there is nothing here worth filing as capacity. + return fmt.Errorf("%s is not a replica of %s, so this run measures the path rather than the hardware; "+ + "point --target at a replica endpoint from `llmgw ctl replicas`", target, res.Model) + } + + body, err := json.Marshal(res.Capacity(profile)) + if err != nil { + return err + } + var saved bench.Capacity + if err := client().put(cmd.Context(), + "/api/v1/models/"+res.Model+"/capacity", body, &saved); err != nil { + return err + } + cmd.PrintErrf("saved: %s on profile %s, %d concurrent at the SLO\n", + saved.Model, saved.Profile, saved.MaxConcurrencyAtSLO) + return nil +} + +// sameEndpoint compares two base URLs, ignoring a trailing slash. +// +// Written out because the two come from different places: one is what the +// control plane recorded, the other is what somebody typed or pasted, and a +// trailing slash is the difference that would silently make every run look like +// it was aimed at the gateway. +func sameEndpoint(a, b string) bool { + return strings.TrimSuffix(a, "/") == strings.TrimSuffix(b, "/") +} + func printConclusion(cmd *cobra.Command, res bench.Result, cfg bench.Config) { out := cmd.OutOrStdout() fmt.Fprintln(out) diff --git a/internal/cli/capacity.go b/internal/cli/capacity.go new file mode 100644 index 0000000..976ebfc --- /dev/null +++ b/internal/cli/capacity.go @@ -0,0 +1,77 @@ +package cli + +import ( + "encoding/json" + "fmt" + "text/tabwriter" + "time" + + "github.com/spf13/cobra" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" +) + +// newCapacityCmd shows what benchmarks have found out. +// +// `ctl bench` has always been able to measure this; until it could be saved, +// the number lived in whoever ran it's scrollback. This is the other half: +// somewhere to read it back, so sizing a model is looking something up rather +// than re-running an hour of load. +func newCapacityCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { + return &cobra.Command{ + Use: "capacity", + Short: "What a replica of each model was measured to serve", + Long: "Recorded by `llmgw ctl bench --save`, per model and profile — the same\n" + + "model on CPU and on a GPU are different numbers.\n\n" + + "Age matters. Hardware is replaced and engine builds change, so a\n" + + "measurement is an observation with a date on it rather than a fact about\n" + + "the model. The column is there to be read.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + var rows []adminapi.CapacityView + if err := client().get(cmd.Context(), "/api/v1/capacity", &rows); err != nil { + return err + } + if *asJSON { + return json.NewEncoder(cmd.OutOrStdout()).Encode(rows) + } + if len(rows) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), + "Nothing has been measured. Run `llmgw ctl bench --target --save`.") + return nil + } + + w := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 2, ' ', 0) + fmt.Fprintln(w, "MODEL\tPROFILE\tCONCURRENT\tTOK/S\tTTFT p99\tTPOT p99\tWAIT\tKV\tMEASURED\tBY") + for _, c := range rows { + concurrent := fmt.Sprint(c.MaxConcurrencyAtSLO) + if !c.Usable() { + // Zero is a real result — this hardware cannot serve this + // model at this SLO — and printing it as a number invites + // somebody to divide by it. + concurrent = "none at SLO" + } + fmt.Fprintf(w, "%s\t%s\t%s\t%.0f\t%s\t%s\t%d\t%.0f%%\t%s\t%s\n", + c.Model, c.Profile, concurrent, c.ThroughputTokPerSec, + ms(c.TTFTp99), ms(c.TPOTp99), + c.SuggestedTargetWaiting, c.SuggestedKVCeiling*100, + age(c.MeasuredAt), or(c.MeasuredBy, "—")) + } + return w.Flush() + }, + } +} + +func age(t time.Time) string { + if t.IsZero() { + return "unknown" + } + return time.Since(t).Round(time.Minute).String() + " ago" +} + +func or(v, fallback string) string { + if v == "" { + return fallback + } + return v +} diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go index 4100c59..9a14e3f 100644 --- a/internal/cli/ctl.go +++ b/internal/cli/ctl.go @@ -43,6 +43,12 @@ func (c *ctlClient) post(ctx context.Context, path string, body []byte, out any) return c.do(ctx, http.MethodPost, path, json.RawMessage(body), out) } +// put replaces a resource. Used where a second call is a better answer to the +// same question rather than another one: re-measuring a model's capacity. +func (c *ctlClient) put(ctx context.Context, path string, body []byte, out any) error { + return c.do(ctx, http.MethodPut, path, json.RawMessage(body), out) +} + func (c *ctlClient) delete(ctx context.Context, path string, out any) error { return c.do(ctx, http.MethodDelete, path, nil, out) } @@ -121,13 +127,14 @@ func newCtlCmd() *cobra.Command { "Move work off a node, one replica at a time"), // cert works on files rather than the admin API, so --server does not // apply to it; it lives here because this is where operators look. - newBenchCmd(&asJSON), + newBenchCmd(client, &asJSON), newStatusCmd(client, &asJSON), newModelCmd(client), newScaleCmd(client), newEngineCheckCmd(), newToolCheckCmd(), newWeightsCmd(client, &asJSON), + newCapacityCmd(client, &asJSON), newCertCmd(), ) return cmd diff --git a/internal/cli/server.go b/internal/cli/server.go index 722974f..3f4c36c 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -158,6 +158,10 @@ func newServerCmd() *cobra.Command { admin.ReplaceCatalog(ctx, built) } api.SetModels(svc) + // Benchmark results live here too. A measurement costs real + // GPU time; without somewhere to put it the number lives in + // whoever ran it's terminal and is gone by morning. + api.SetCapacity(st) } api.Register(adminMux) diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 68389de..740b16b 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -12,18 +12,22 @@ import ( "fmt" "io" "net/http" + "slices" "time" + "gitea.mynodes.xyz/ilker/llmgw/internal/bench" "gitea.mynodes.xyz/ilker/llmgw/internal/control" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) // API serves the management endpoints. type API struct { - models Models - ctl *control.Controller + models Models + capacity Capacity + ctl *control.Controller } // New builds the API over a controller. @@ -38,6 +42,8 @@ func (a *API) Register(mux *http.ServeMux) { mux.HandleFunc("POST /api/v1/models", a.addModel) mux.HandleFunc("DELETE /api/v1/models/{name}", a.removeModel) mux.HandleFunc("POST /api/v1/models/{name}/scale", a.scaleModel) + mux.HandleFunc("PUT /api/v1/models/{name}/capacity", a.putCapacity) + mux.HandleFunc("GET /api/v1/capacity", a.listCapacity) mux.HandleFunc("GET /api/v1/nodes", a.listNodes) mux.HandleFunc("GET /api/v1/models", a.listModels) mux.HandleFunc("GET /api/v1/replicas", a.listReplicas) @@ -308,6 +314,98 @@ type Models interface { Scale(ctx context.Context, name string, s catalog.Scaling, by string) (catalog.Model, error) } +// Capacity is where measured capacity is kept. The store implements it. +type Capacity interface { + PutCapacity(ctx context.Context, c store.StoredCapacity) error + Capacities(ctx context.Context) ([]store.StoredCapacity, error) +} + +// SetCapacity supplies somewhere to keep benchmark results. +func (a *API) SetCapacity(c Capacity) { a.capacity = c } + +// putCapacity records what a replica of a model was measured to serve. +// +// A PUT rather than a POST: re-running a benchmark replaces what was known +// about that model on that profile. It is a better answer to the same question, +// not another data point. +func (a *API) putCapacity(w http.ResponseWriter, r *http.Request) { + if a.capacity == nil { + writeError(w, http.StatusNotImplemented, + "this control plane has no store, so a measurement recorded here would not survive a restart") + return + } + var c bench.Capacity + if err := json.NewDecoder(io.LimitReader(r.Body, 1<<20)).Decode(&c); err != nil { + writeError(w, http.StatusBadRequest, "the measurement is not valid JSON: "+err.Error()) + return + } + c.Model = r.PathValue("name") + if c.Profile == "" { + writeError(w, http.StatusBadRequest, + "a measurement needs the profile it was taken on: the same model on CPU and on a GPU are different numbers") + return + } + + // The model has to exist now, which catches a typo in the name — a + // measurement filed under a misspelling is worse than none, because it + // looks like the model was never benchmarked. It is not deleted when the + // model is, though: a benchmark costs real GPU time and outliving the + // catalog entry is the point. + m, ok := a.ctl.Catalog().Resolve(c.Model) + if !ok { + writeError(w, http.StatusNotFound, "no model named "+c.Model+" is in the catalog") + return + } + if !slices.ContainsFunc(m.Profiles, func(p catalog.Profile) bool { return p.Name == c.Profile }) { + writeError(w, http.StatusBadRequest, + "model "+c.Model+" has no profile named "+c.Profile) + return + } + + body, err := json.Marshal(c) + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + if err := a.capacity.PutCapacity(r.Context(), store.StoredCapacity{ + Model: c.Model, Profile: c.Profile, Measurement: body, + MeasuredAt: c.MeasuredAt, MeasuredBy: r.Header.Get("X-Llmgw-Actor"), + }); err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + writeJSON(w, http.StatusOK, c) +} + +// CapacityView is one stored measurement, with who recorded it. +type CapacityView struct { + bench.Capacity + MeasuredBy string `json:"measured_by,omitempty"` +} + +func (a *API) listCapacity(w http.ResponseWriter, r *http.Request) { + if a.capacity == nil { + writeJSON(w, http.StatusOK, []CapacityView{}) + return + } + rows, err := a.capacity.Capacities(r.Context()) + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + out := make([]CapacityView, 0, len(rows)) + for _, row := range rows { + var c bench.Capacity + if err := json.Unmarshal(row.Measurement, &c); err != nil { + // Skipped rather than failing the listing: one row written by an + // older shape must not hide every other measurement. + continue + } + out = append(out, CapacityView{Capacity: c, MeasuredBy: row.MeasuredBy}) + } + writeJSON(w, http.StatusOK, out) +} + // SetModels supplies the catalog-changing service. func (a *API) SetModels(m Models) { a.models = m } diff --git a/internal/control/adminapi/api_test.go b/internal/control/adminapi/api_test.go index 4ccbfcc..daf7c48 100644 --- a/internal/control/adminapi/api_test.go +++ b/internal/control/adminapi/api_test.go @@ -16,6 +16,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" "gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) // nullSink discards published endpoints; these tests are about the API. @@ -418,3 +419,117 @@ func TestANegativeReplicaCountIsRefused(t *testing.T) { t.Error("nothing should have reached the service") } } + +type memCapacity struct{ rows []store.StoredCapacity } + +func (m *memCapacity) PutCapacity(_ context.Context, c store.StoredCapacity) error { + m.rows = append(m.rows, c) + return nil +} + +func (m *memCapacity) Capacities(context.Context) ([]store.StoredCapacity, error) { + return m.rows, nil +} + +func capacityAPI(t *testing.T) (*API, *memCapacity, *http.ServeMux) { + t.Helper() + a, _ := setup(t, testModels()) + cap := &memCapacity{} + a.SetCapacity(cap) + mux := http.NewServeMux() + a.Register(mux) + return a, cap, mux +} + +func TestAMeasurementIsFiledAgainstItsModelAndProfile(t *testing.T) { + _, cap, mux := capacityAPI(t) + + body := `{"profile":"tp8","max_concurrency_at_slo":8,"suggested_target_waiting":3}` + req := httptest.NewRequest(http.MethodPut, "/api/v1/models/llama/capacity", strings.NewReader(body)) + req.Header.Set("X-Llmgw-Actor", "ilker") + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", rec.Code, rec.Body) + } + if len(cap.rows) != 1 { + t.Fatalf("expected one row, got %d", len(cap.rows)) + } + // The model comes from the path, not the body: a measurement filed under a + // name the caller made up would be worse than none. + if cap.rows[0].Model != "llama" || cap.rows[0].Profile != "tp8" { + t.Errorf("wrong key: %s/%s", cap.rows[0].Model, cap.rows[0].Profile) + } + if cap.rows[0].MeasuredBy != "ilker" { + t.Errorf("the measurement should say who took it; got %q", cap.rows[0].MeasuredBy) + } +} + +func TestAMeasurementForAModelThatIsNotThereIsRefused(t *testing.T) { + _, cap, mux := capacityAPI(t) + + req := httptest.NewRequest(http.MethodPut, "/api/v1/models/typo/capacity", + strings.NewReader(`{"profile":"tp8"}`)) + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, req) + + // A typo in the name files the measurement somewhere nobody will look, and + // the model then looks as though it was never benchmarked. + if rec.Code != http.StatusNotFound { + t.Errorf("expected 404 for an unknown model, got %d", rec.Code) + } + if len(cap.rows) != 0 { + t.Error("nothing should have been stored") + } +} + +func TestAMeasurementNeedsTheProfileItWasTakenOn(t *testing.T) { + _, cap, mux := capacityAPI(t) + + req := httptest.NewRequest(http.MethodPut, "/api/v1/models/llama/capacity", + strings.NewReader(`{"max_concurrency_at_slo":8}`)) + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, req) + + // Capacity belongs to the hardware as much as the weights. Without a + // profile there is nothing to compare a later run against. + if rec.Code != http.StatusBadRequest { + t.Errorf("expected 400 without a profile, got %d", rec.Code) + } + if len(cap.rows) != 0 { + t.Error("nothing should have been stored") + } +} + +func TestAProfileTheModelDoesNotHaveIsRefused(t *testing.T) { + _, _, mux := capacityAPI(t) + + req := httptest.NewRequest(http.MethodPut, "/api/v1/models/llama/capacity", + strings.NewReader(`{"profile":"not-a-profile"}`)) + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusBadRequest { + t.Errorf("expected 400 for a profile the model does not have, got %d", rec.Code) + } +} + +func TestCapacityWithoutAStoreIsAnEmptyListRatherThanAnError(t *testing.T) { + a, _ := setup(t, testModels()) + mux := http.NewServeMux() + a.Register(mux) + + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/v1/capacity", nil)) + + // Nothing measured and no store look the same to a reader, and both mean + // "there is nothing here" — which is not a server fault. + if rec.Code != http.StatusOK { + t.Errorf("expected 200, got %d", rec.Code) + } + var rows []CapacityView + if err := json.Unmarshal(rec.Body.Bytes(), &rows); err != nil || len(rows) != 0 { + t.Errorf("expected an empty list, got %s", rec.Body) + } +} diff --git a/internal/store/capacity.go b/internal/store/capacity.go new file mode 100644 index 0000000..1cd57bc --- /dev/null +++ b/internal/store/capacity.go @@ -0,0 +1,88 @@ +package store + +import ( + "context" + "encoding/json" + "fmt" + "time" +) + +// StoredCapacity is a benchmark's conclusion, kept for whoever has to size the +// model later. +type StoredCapacity struct { + Model string + Profile string + // Measurement is bench.Capacity as JSON. Stored whole for the reason a + // model definition is: a measurement is a nested shape that is still + // growing, and as columns every field added would be a migration. + Measurement []byte + MeasuredAt time.Time + MeasuredBy string +} + +// PutCapacity records what a replica of a model was measured to serve. +// +// Replacing rather than accumulating. A second measurement of the same model on +// the same profile is a better answer to the same question, not another data +// point: hardware and engine builds change, and keeping every run would leave a +// reader choosing between numbers with no basis for the choice. The run itself +// is printed when it happens, for anyone who wants the ramp. +func (s *SQLite) PutCapacity(ctx context.Context, c StoredCapacity) error { + if c.Model == "" || c.Profile == "" { + return fmt.Errorf("a capacity measurement needs a model and a profile") + } + if !json.Valid(c.Measurement) { + return fmt.Errorf("capacity for %s/%s: the measurement is not valid JSON", c.Model, c.Profile) + } + if c.MeasuredAt.IsZero() { + c.MeasuredAt = time.Now() + } + _, err := s.db.ExecContext(ctx, + `INSERT INTO capacity (model, profile, measurement, measured_at, measured_by) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(model, profile) DO UPDATE SET + measurement = excluded.measurement, + measured_at = excluded.measured_at, + measured_by = excluded.measured_by`, + c.Model, c.Profile, string(c.Measurement), c.MeasuredAt, c.MeasuredBy) + return err +} + +// Capacities lists every measurement, newest first. +// +// Newest first because the question a reader usually has is "what do we know +// lately", and an old measurement is the one most likely to be describing +// hardware that has since changed. +func (s *SQLite) Capacities(ctx context.Context) ([]StoredCapacity, error) { + rows, err := s.db.QueryContext(ctx, + `SELECT model, profile, measurement, measured_at, measured_by + FROM capacity ORDER BY measured_at DESC, model, profile`) + if err != nil { + return nil, err + } + defer rows.Close() + + var out []StoredCapacity + for rows.Next() { + var c StoredCapacity + var m string + if err := rows.Scan(&c.Model, &c.Profile, &m, &c.MeasuredAt, &c.MeasuredBy); err != nil { + return nil, err + } + c.Measurement = []byte(m) + out = append(out, c) + } + return out, rows.Err() +} + +// DeleteCapacity forgets a measurement. +// +// Wanted when a model is removed, and wanted on its own: a profile whose +// hardware has been replaced is worse than no measurement, because it looks +// like knowledge. Deleting one that is not there is not an error — the caller +// wanted it gone, and it is. +func (s *SQLite) DeleteCapacity(ctx context.Context, model, profile string) error { + _, err := s.db.ExecContext(ctx, + `DELETE FROM capacity WHERE model = ? AND profile = ?`, model, profile) + return err +} diff --git a/internal/store/capacity_test.go b/internal/store/capacity_test.go new file mode 100644 index 0000000..0016b4c --- /dev/null +++ b/internal/store/capacity_test.go @@ -0,0 +1,137 @@ +package store + +import ( + "context" + "path/filepath" + "testing" + "time" +) + +func capacityStore(t *testing.T) *SQLite { + t.Helper() + s, err := OpenSQLite(context.Background(), filepath.Join(t.TempDir(), "cap.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = s.Close() }) + return s +} + +func TestAMeasurementSurvivesTheTerminalItWasPrintedIn(t *testing.T) { + s := capacityStore(t) + at := time.Now().UTC().Truncate(time.Second) + + if err := s.PutCapacity(t.Context(), StoredCapacity{ + Model: "qwen", Profile: "cpu", + Measurement: []byte(`{"max_concurrency_at_slo":8}`), + MeasuredAt: at, MeasuredBy: "ilker", + }); err != nil { + t.Fatal(err) + } + + got, err := s.Capacities(t.Context()) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 { + t.Fatalf("expected the one measurement, got %d", len(got)) + } + if got[0].Model != "qwen" || got[0].Profile != "cpu" { + t.Errorf("wrong row: %+v", got[0]) + } + if string(got[0].Measurement) != `{"max_concurrency_at_slo":8}` { + t.Errorf("the measurement came back changed: %s", got[0].Measurement) + } + if !got[0].MeasuredAt.UTC().Equal(at) { + t.Errorf("when it was taken is part of the answer; got %v want %v", got[0].MeasuredAt, at) + } +} + +func TestTheSameModelOnTwoProfilesIsTwoMeasurements(t *testing.T) { + s := capacityStore(t) + for _, p := range []string{"cpu", "gpu"} { + if err := s.PutCapacity(t.Context(), StoredCapacity{ + Model: "qwen", Profile: p, Measurement: []byte(`{}`), + }); err != nil { + t.Fatal(err) + } + } + + // Capacity belongs to the hardware and the engine arguments as much as to + // the weights. Keyed by model alone, benchmarking the GPU nodes would + // overwrite what is known about the CPU ones. + got, _ := s.Capacities(t.Context()) + if len(got) != 2 { + t.Fatalf("expected one measurement per profile, got %d", len(got)) + } +} + +func TestRemeasuringReplacesRatherThanAccumulates(t *testing.T) { + s := capacityStore(t) + older := time.Now().Add(-24 * time.Hour) + if err := s.PutCapacity(t.Context(), StoredCapacity{ + Model: "qwen", Profile: "cpu", + Measurement: []byte(`{"max_concurrency_at_slo":4}`), MeasuredAt: older, + }); err != nil { + t.Fatal(err) + } + if err := s.PutCapacity(t.Context(), StoredCapacity{ + Model: "qwen", Profile: "cpu", + Measurement: []byte(`{"max_concurrency_at_slo":9}`), MeasuredAt: time.Now(), + }); err != nil { + t.Fatal(err) + } + + // A second run is a better answer to the same question, not another data + // point: keeping both would leave a reader choosing between them with no + // basis for the choice. + got, _ := s.Capacities(t.Context()) + if len(got) != 1 { + t.Fatalf("expected the measurement to be replaced, got %d rows", len(got)) + } + if string(got[0].Measurement) != `{"max_concurrency_at_slo":9}` { + t.Errorf("the older measurement won: %s", got[0].Measurement) + } +} + +func TestMeasurementsComeBackNewestFirst(t *testing.T) { + s := capacityStore(t) + now := time.Now() + _ = s.PutCapacity(t.Context(), StoredCapacity{ + Model: "old", Profile: "cpu", Measurement: []byte(`{}`), + MeasuredAt: now.Add(-48 * time.Hour), + }) + _ = s.PutCapacity(t.Context(), StoredCapacity{ + Model: "new", Profile: "cpu", Measurement: []byte(`{}`), MeasuredAt: now, + }) + + got, _ := s.Capacities(t.Context()) + // The old one is the one most likely to describe hardware that has since + // changed, so it is the one that should need scrolling to. + if len(got) != 2 || got[0].Model != "new" { + t.Errorf("expected the newest first, got %+v", got) + } +} + +func TestRubbishIsRefusedBeforeItIsStored(t *testing.T) { + s := capacityStore(t) + if err := s.PutCapacity(t.Context(), StoredCapacity{ + Model: "qwen", Profile: "cpu", Measurement: []byte(`not json`), + }); err == nil { + t.Error("a measurement that is not JSON should be refused at the write") + } + if err := s.PutCapacity(t.Context(), StoredCapacity{ + Profile: "cpu", Measurement: []byte(`{}`), + }); err == nil { + t.Error("a measurement with no model should be refused") + } +} + +func TestForgettingAMeasurementThatIsNotThereIsFine(t *testing.T) { + s := capacityStore(t) + // The caller wanted it gone, and it is. Erroring would make cleaning up + // after a removed model a thing that needs a check first. + if err := s.DeleteCapacity(t.Context(), "ghost", "cpu"); err != nil { + t.Errorf("deleting an absent measurement should not error: %v", err) + } +} diff --git a/internal/store/copy.go b/internal/store/copy.go index 7149beb..d6d1c01 100644 --- a/internal/store/copy.go +++ b/internal/store/copy.go @@ -29,6 +29,10 @@ var copyOrder = []string{ "limits", "usage", "models", + // No foreign key to models, deliberately: a measurement outlives the + // catalog entry it was taken for, and losing it when somebody removes a + // model would mean re-running a benchmark that costs real GPU time. + "capacity", "threads", "thread_messages", "memory_notes", diff --git a/internal/store/migrations/postgres/0009_capacity.sql b/internal/store/migrations/postgres/0009_capacity.sql new file mode 100644 index 0000000..102f75b --- /dev/null +++ b/internal/store/migrations/postgres/0009_capacity.sql @@ -0,0 +1,34 @@ +-- Ported from the SQLite migration of the same number. +-- +-- What one replica of a model can actually serve, measured rather than guessed. +-- +-- `llmgw ctl bench` has always been able to work this out — it ramps +-- concurrency until the SLO breaks and reports the last level that held. Until +-- now it printed that to a terminal and nothing kept it, so the one number that +-- says what a replica is worth lived in somebody's scrollback. Every decision +-- downstream then had to guess: admission uses a single global default for +-- healthy queue depth, and anything that wanted to size a model against real +-- demand had nothing to size against. +-- +-- Keyed by model and profile, not by model alone. Capacity is a property of the +-- hardware and the engine arguments as much as of the weights: the same model +-- on a CPU profile and on an H100 profile are two different numbers, and +-- storing one would overwrite the other every time somebody benchmarked the +-- other kind of node. +-- +-- The measurement is JSON for the same reason a model definition is: it is a +-- nested shape that is still growing, and as columns every field added to it +-- would be a migration. +-- +-- Nothing here is authoritative about what a model may do — it is an +-- observation, with the time and the target it was taken from, so a reader can +-- decide whether it is still true. Hardware changes and engine builds change; +-- a capacity profile from six months ago is a hypothesis, not a fact. +CREATE TABLE capacity ( + model TEXT NOT NULL, + profile TEXT NOT NULL, + measurement TEXT NOT NULL, + measured_at TIMESTAMP NOT NULL, + measured_by TEXT NOT NULL DEFAULT '', + PRIMARY KEY (model, profile) +); diff --git a/internal/store/migrations/sqlite/0009_capacity.sql b/internal/store/migrations/sqlite/0009_capacity.sql new file mode 100644 index 0000000..907950b --- /dev/null +++ b/internal/store/migrations/sqlite/0009_capacity.sql @@ -0,0 +1,32 @@ +-- What one replica of a model can actually serve, measured rather than guessed. +-- +-- `llmgw ctl bench` has always been able to work this out — it ramps +-- concurrency until the SLO breaks and reports the last level that held. Until +-- now it printed that to a terminal and nothing kept it, so the one number that +-- says what a replica is worth lived in somebody's scrollback. Every decision +-- downstream then had to guess: admission uses a single global default for +-- healthy queue depth, and anything that wanted to size a model against real +-- demand had nothing to size against. +-- +-- Keyed by model and profile, not by model alone. Capacity is a property of the +-- hardware and the engine arguments as much as of the weights: the same model +-- on a CPU profile and on an H100 profile are two different numbers, and +-- storing one would overwrite the other every time somebody benchmarked the +-- other kind of node. +-- +-- The measurement is JSON for the same reason a model definition is: it is a +-- nested shape that is still growing, and as columns every field added to it +-- would be a migration. +-- +-- Nothing here is authoritative about what a model may do — it is an +-- observation, with the time and the target it was taken from, so a reader can +-- decide whether it is still true. Hardware changes and engine builds change; +-- a capacity profile from six months ago is a hypothesis, not a fact. +CREATE TABLE capacity ( + model TEXT NOT NULL, + profile TEXT NOT NULL, + measurement TEXT NOT NULL, + measured_at TIMESTAMP NOT NULL, + measured_by TEXT NOT NULL DEFAULT '', + PRIMARY KEY (model, profile) +);