diff --git a/internal/bench/capacity.go b/internal/bench/capacity.go new file mode 100644 index 0000000..18a8e06 --- /dev/null +++ b/internal/bench/capacity.go @@ -0,0 +1,77 @@ +package bench + +import "time" + +// Capacity is what one replica was measured to serve, kept so the measurement +// outlives the terminal it was printed in. +// +// The conclusion of a run, not the run. The ramp itself — every concurrency +// level, its latencies, its errors — is worth reading once, while deciding +// whether the run was sound; it is not worth carrying around afterwards, and +// storing it would grow without bound as a model is re-benchmarked. What a +// later reader needs is the number and enough provenance to judge whether it is +// still true. +type Capacity struct { + Model string `json:"model"` + Profile string `json:"profile"` + + // MaxConcurrencyAtSLO is the capacity: the highest concurrency that still + // met the target. Zero means even one request did not, which is a real + // result and not a missing one — it says this hardware cannot serve this + // model at this SLO at all. + MaxConcurrencyAtSLO int `json:"max_concurrency_at_slo"` + // ThroughputTokPerSec and the latencies are taken at that level, not at the + // peak. Peak throughput arrives long after latency has become unusable, so + // reporting it would overstate what the hardware can serve. + ThroughputTokPerSec float64 `json:"throughput_tok_per_sec"` + TTFTp99 time.Duration `json:"ttft_p99"` + TPOTp99 time.Duration `json:"tpot_p99"` + + // SuggestedTargetWaiting and SuggestedKVCeiling are the queue depth and + // cache pressure seen while the SLO was still being met, which is what + // admission should treat as healthy for this model. Admission currently + // applies one global default to every model; these are the per-model + // numbers it should be using instead. + SuggestedTargetWaiting int `json:"suggested_target_waiting"` + SuggestedKVCeiling float64 `json:"suggested_kv_ceiling"` + + // SLO is recorded because capacity is meaningless without it. The same + // replica has a different capacity against a 200ms first-token target than + // against a 2s one, and a number stored without the target it was measured + // against cannot be compared with anything. + SLO SLO `json:"slo"` + + // Target is the endpoint the load actually went to, kept so a profile + // measured against the gateway is never mistaken for one measured against + // a replica. Only the second says anything about the hardware. + Target string `json:"target"` + MeasuredAt time.Time `json:"measured_at"` + // Stale is not a field. Whether a measurement still holds depends on + // whether the hardware and the engine build have changed since, which this + // package cannot know; MeasuredAt is given so the reader can decide. +} + +// Capacity summarizes a completed run for storage. +func (r Result) Capacity(profile string) Capacity { + return Capacity{ + Model: r.Model, + Profile: profile, + MaxConcurrencyAtSLO: r.MaxConcurrencyAtSLO, + ThroughputTokPerSec: r.ThroughputTokPerSec, + TTFTp99: r.TTFTp99, + TPOTp99: r.TPOTp99, + SuggestedTargetWaiting: r.SuggestedTargetWaiting, + SuggestedKVCeiling: r.SuggestedKVCeiling, + SLO: r.SLO, + Target: r.Target, + MeasuredAt: r.RanAt, + } +} + +// Usable reports whether this measurement says anything. +// +// A run that never met the SLO, or one stopped before it finished a level, +// concludes at zero. That is worth recording — it is the answer to "can this +// run here at all" — but a caller sizing a model against it would compute a +// replica count of infinity, so it has to be asked about rather than used. +func (c Capacity) Usable() bool { return c.MaxConcurrencyAtSLO > 0 } diff --git a/internal/cli/bench.go b/internal/cli/bench.go index 3d4d2a0..55e3b29 100644 --- a/internal/cli/bench.go +++ b/internal/cli/bench.go @@ -3,14 +3,16 @@ package cli import ( "encoding/json" "fmt" + "strings" "time" "github.com/spf13/cobra" "gitea.mynodes.xyz/ilker/llmgw/internal/bench" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" ) -func newBenchCmd(asJSON *bool) *cobra.Command { +func newBenchCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { cfg := bench.DefaultConfig() var ( steps []int @@ -18,6 +20,7 @@ func newBenchCmd(asJSON *bool) *cobra.Command { metrics string engine string key string + save bool ) cmd := &cobra.Command{ @@ -78,6 +81,15 @@ func newBenchCmd(asJSON *bool) *cobra.Command { if err != nil { return err } + if save { + if err := saveCapacity(cmd, client, res, target); err != nil { + // Reported, not fatal. The run has already cost the GPU + // time; losing its conclusion to a failed write would mean + // running it again, and the numbers are on screen either + // way. + cmd.PrintErrf("the measurement was not saved: %v\n", err) + } + } if *asJSON { return json.NewEncoder(cmd.OutOrStdout()).Encode(res) } @@ -99,9 +111,63 @@ func newBenchCmd(asJSON *bool) *cobra.Command { "send every request the same prompt, measuring the cached case instead of the hardware") f.DurationVar(&cfg.SLO.TTFTp99, "ttft-p99", cfg.SLO.TTFTp99, "time-to-first-token target") f.DurationVar(&cfg.SLO.TPOTp99, "tpot-p99", cfg.SLO.TPOTp99, "per-output-token target") + f.BoolVar(&save, "save", false, + "record the capacity against the model and profile, so the number outlives this terminal") return cmd } +// saveCapacity records a run's conclusion against the profile it was measured +// on. +// +// The profile is looked up from the replica rather than asked for. Capacity +// belongs to the hardware and the engine arguments as much as to the weights, +// so a measurement filed under the wrong profile is worse than none — and the +// control plane already knows which profile is answering on which endpoint, so +// nobody needs to be trusted to type it. +func saveCapacity(cmd *cobra.Command, client func() *ctlClient, res bench.Result, target string) error { + var reps []adminapi.ReplicaView + if err := client().get(cmd.Context(), "/api/v1/replicas", &reps); err != nil { + return err + } + profile := "" + for _, rep := range reps { + if rep.Model == res.Model && sameEndpoint(rep.Endpoint, target) { + profile = rep.Profile + break + } + } + if profile == "" { + // Almost always because --target was the gateway. Benchmarking the + // whole path is a useful thing to do and says nothing about what one + // replica can serve, so there is nothing here worth filing as capacity. + return fmt.Errorf("%s is not a replica of %s, so this run measures the path rather than the hardware; "+ + "point --target at a replica endpoint from `llmgw ctl replicas`", target, res.Model) + } + + body, err := json.Marshal(res.Capacity(profile)) + if err != nil { + return err + } + var saved bench.Capacity + if err := client().put(cmd.Context(), + "/api/v1/models/"+res.Model+"/capacity", body, &saved); err != nil { + return err + } + cmd.PrintErrf("saved: %s on profile %s, %d concurrent at the SLO\n", + saved.Model, saved.Profile, saved.MaxConcurrencyAtSLO) + return nil +} + +// sameEndpoint compares two base URLs, ignoring a trailing slash. +// +// Written out because the two come from different places: one is what the +// control plane recorded, the other is what somebody typed or pasted, and a +// trailing slash is the difference that would silently make every run look like +// it was aimed at the gateway. +func sameEndpoint(a, b string) bool { + return strings.TrimSuffix(a, "/") == strings.TrimSuffix(b, "/") +} + func printConclusion(cmd *cobra.Command, res bench.Result, cfg bench.Config) { out := cmd.OutOrStdout() fmt.Fprintln(out) diff --git a/internal/cli/capacity.go b/internal/cli/capacity.go new file mode 100644 index 0000000..976ebfc --- /dev/null +++ b/internal/cli/capacity.go @@ -0,0 +1,77 @@ +package cli + +import ( + "encoding/json" + "fmt" + "text/tabwriter" + "time" + + "github.com/spf13/cobra" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" +) + +// newCapacityCmd shows what benchmarks have found out. +// +// `ctl bench` has always been able to measure this; until it could be saved, +// the number lived in whoever ran it's scrollback. This is the other half: +// somewhere to read it back, so sizing a model is looking something up rather +// than re-running an hour of load. +func newCapacityCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { + return &cobra.Command{ + Use: "capacity", + Short: "What a replica of each model was measured to serve", + Long: "Recorded by `llmgw ctl bench --save`, per model and profile — the same\n" + + "model on CPU and on a GPU are different numbers.\n\n" + + "Age matters. Hardware is replaced and engine builds change, so a\n" + + "measurement is an observation with a date on it rather than a fact about\n" + + "the model. The column is there to be read.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + var rows []adminapi.CapacityView + if err := client().get(cmd.Context(), "/api/v1/capacity", &rows); err != nil { + return err + } + if *asJSON { + return json.NewEncoder(cmd.OutOrStdout()).Encode(rows) + } + if len(rows) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), + "Nothing has been measured. Run `llmgw ctl bench --target --save`.") + return nil + } + + w := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 2, ' ', 0) + fmt.Fprintln(w, "MODEL\tPROFILE\tCONCURRENT\tTOK/S\tTTFT p99\tTPOT p99\tWAIT\tKV\tMEASURED\tBY") + for _, c := range rows { + concurrent := fmt.Sprint(c.MaxConcurrencyAtSLO) + if !c.Usable() { + // Zero is a real result — this hardware cannot serve this + // model at this SLO — and printing it as a number invites + // somebody to divide by it. + concurrent = "none at SLO" + } + fmt.Fprintf(w, "%s\t%s\t%s\t%.0f\t%s\t%s\t%d\t%.0f%%\t%s\t%s\n", + c.Model, c.Profile, concurrent, c.ThroughputTokPerSec, + ms(c.TTFTp99), ms(c.TPOTp99), + c.SuggestedTargetWaiting, c.SuggestedKVCeiling*100, + age(c.MeasuredAt), or(c.MeasuredBy, "—")) + } + return w.Flush() + }, + } +} + +func age(t time.Time) string { + if t.IsZero() { + return "unknown" + } + return time.Since(t).Round(time.Minute).String() + " ago" +} + +func or(v, fallback string) string { + if v == "" { + return fallback + } + return v +} diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go index 4100c59..9a14e3f 100644 --- a/internal/cli/ctl.go +++ b/internal/cli/ctl.go @@ -43,6 +43,12 @@ func (c *ctlClient) post(ctx context.Context, path string, body []byte, out any) return c.do(ctx, http.MethodPost, path, json.RawMessage(body), out) } +// put replaces a resource. Used where a second call is a better answer to the +// same question rather than another one: re-measuring a model's capacity. +func (c *ctlClient) put(ctx context.Context, path string, body []byte, out any) error { + return c.do(ctx, http.MethodPut, path, json.RawMessage(body), out) +} + func (c *ctlClient) delete(ctx context.Context, path string, out any) error { return c.do(ctx, http.MethodDelete, path, nil, out) } @@ -121,13 +127,14 @@ func newCtlCmd() *cobra.Command { "Move work off a node, one replica at a time"), // cert works on files rather than the admin API, so --server does not // apply to it; it lives here because this is where operators look. - newBenchCmd(&asJSON), + newBenchCmd(client, &asJSON), newStatusCmd(client, &asJSON), newModelCmd(client), newScaleCmd(client), newEngineCheckCmd(), newToolCheckCmd(), newWeightsCmd(client, &asJSON), + newCapacityCmd(client, &asJSON), newCertCmd(), ) return cmd diff --git a/internal/cli/server.go b/internal/cli/server.go index 722974f..3f4c36c 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -158,6 +158,10 @@ func newServerCmd() *cobra.Command { admin.ReplaceCatalog(ctx, built) } api.SetModels(svc) + // Benchmark results live here too. A measurement costs real + // GPU time; without somewhere to put it the number lives in + // whoever ran it's terminal and is gone by morning. + api.SetCapacity(st) } api.Register(adminMux) diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 68389de..740b16b 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -12,18 +12,22 @@ import ( "fmt" "io" "net/http" + "slices" "time" + "gitea.mynodes.xyz/ilker/llmgw/internal/bench" "gitea.mynodes.xyz/ilker/llmgw/internal/control" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) // API serves the management endpoints. type API struct { - models Models - ctl *control.Controller + models Models + capacity Capacity + ctl *control.Controller } // New builds the API over a controller. @@ -38,6 +42,8 @@ func (a *API) Register(mux *http.ServeMux) { mux.HandleFunc("POST /api/v1/models", a.addModel) mux.HandleFunc("DELETE /api/v1/models/{name}", a.removeModel) mux.HandleFunc("POST /api/v1/models/{name}/scale", a.scaleModel) + mux.HandleFunc("PUT /api/v1/models/{name}/capacity", a.putCapacity) + mux.HandleFunc("GET /api/v1/capacity", a.listCapacity) mux.HandleFunc("GET /api/v1/nodes", a.listNodes) mux.HandleFunc("GET /api/v1/models", a.listModels) mux.HandleFunc("GET /api/v1/replicas", a.listReplicas) @@ -308,6 +314,98 @@ type Models interface { Scale(ctx context.Context, name string, s catalog.Scaling, by string) (catalog.Model, error) } +// Capacity is where measured capacity is kept. The store implements it. +type Capacity interface { + PutCapacity(ctx context.Context, c store.StoredCapacity) error + Capacities(ctx context.Context) ([]store.StoredCapacity, error) +} + +// SetCapacity supplies somewhere to keep benchmark results. +func (a *API) SetCapacity(c Capacity) { a.capacity = c } + +// putCapacity records what a replica of a model was measured to serve. +// +// A PUT rather than a POST: re-running a benchmark replaces what was known +// about that model on that profile. It is a better answer to the same question, +// not another data point. +func (a *API) putCapacity(w http.ResponseWriter, r *http.Request) { + if a.capacity == nil { + writeError(w, http.StatusNotImplemented, + "this control plane has no store, so a measurement recorded here would not survive a restart") + return + } + var c bench.Capacity + if err := json.NewDecoder(io.LimitReader(r.Body, 1<<20)).Decode(&c); err != nil { + writeError(w, http.StatusBadRequest, "the measurement is not valid JSON: "+err.Error()) + return + } + c.Model = r.PathValue("name") + if c.Profile == "" { + writeError(w, http.StatusBadRequest, + "a measurement needs the profile it was taken on: the same model on CPU and on a GPU are different numbers") + return + } + + // The model has to exist now, which catches a typo in the name — a + // measurement filed under a misspelling is worse than none, because it + // looks like the model was never benchmarked. It is not deleted when the + // model is, though: a benchmark costs real GPU time and outliving the + // catalog entry is the point. + m, ok := a.ctl.Catalog().Resolve(c.Model) + if !ok { + writeError(w, http.StatusNotFound, "no model named "+c.Model+" is in the catalog") + return + } + if !slices.ContainsFunc(m.Profiles, func(p catalog.Profile) bool { return p.Name == c.Profile }) { + writeError(w, http.StatusBadRequest, + "model "+c.Model+" has no profile named "+c.Profile) + return + } + + body, err := json.Marshal(c) + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + if err := a.capacity.PutCapacity(r.Context(), store.StoredCapacity{ + Model: c.Model, Profile: c.Profile, Measurement: body, + MeasuredAt: c.MeasuredAt, MeasuredBy: r.Header.Get("X-Llmgw-Actor"), + }); err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + writeJSON(w, http.StatusOK, c) +} + +// CapacityView is one stored measurement, with who recorded it. +type CapacityView struct { + bench.Capacity + MeasuredBy string `json:"measured_by,omitempty"` +} + +func (a *API) listCapacity(w http.ResponseWriter, r *http.Request) { + if a.capacity == nil { + writeJSON(w, http.StatusOK, []CapacityView{}) + return + } + rows, err := a.capacity.Capacities(r.Context()) + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + out := make([]CapacityView, 0, len(rows)) + for _, row := range rows { + var c bench.Capacity + if err := json.Unmarshal(row.Measurement, &c); err != nil { + // Skipped rather than failing the listing: one row written by an + // older shape must not hide every other measurement. + continue + } + out = append(out, CapacityView{Capacity: c, MeasuredBy: row.MeasuredBy}) + } + writeJSON(w, http.StatusOK, out) +} + // SetModels supplies the catalog-changing service. func (a *API) SetModels(m Models) { a.models = m } diff --git a/internal/control/adminapi/api_test.go b/internal/control/adminapi/api_test.go index 4ccbfcc..daf7c48 100644 --- a/internal/control/adminapi/api_test.go +++ b/internal/control/adminapi/api_test.go @@ -16,6 +16,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" "gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) // nullSink discards published endpoints; these tests are about the API. @@ -418,3 +419,117 @@ func TestANegativeReplicaCountIsRefused(t *testing.T) { t.Error("nothing should have reached the service") } } + +type memCapacity struct{ rows []store.StoredCapacity } + +func (m *memCapacity) PutCapacity(_ context.Context, c store.StoredCapacity) error { + m.rows = append(m.rows, c) + return nil +} + +func (m *memCapacity) Capacities(context.Context) ([]store.StoredCapacity, error) { + return m.rows, nil +} + +func capacityAPI(t *testing.T) (*API, *memCapacity, *http.ServeMux) { + t.Helper() + a, _ := setup(t, testModels()) + cap := &memCapacity{} + a.SetCapacity(cap) + mux := http.NewServeMux() + a.Register(mux) + return a, cap, mux +} + +func TestAMeasurementIsFiledAgainstItsModelAndProfile(t *testing.T) { + _, cap, mux := capacityAPI(t) + + body := `{"profile":"tp8","max_concurrency_at_slo":8,"suggested_target_waiting":3}` + req := httptest.NewRequest(http.MethodPut, "/api/v1/models/llama/capacity", strings.NewReader(body)) + req.Header.Set("X-Llmgw-Actor", "ilker") + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", rec.Code, rec.Body) + } + if len(cap.rows) != 1 { + t.Fatalf("expected one row, got %d", len(cap.rows)) + } + // The model comes from the path, not the body: a measurement filed under a + // name the caller made up would be worse than none. + if cap.rows[0].Model != "llama" || cap.rows[0].Profile != "tp8" { + t.Errorf("wrong key: %s/%s", cap.rows[0].Model, cap.rows[0].Profile) + } + if cap.rows[0].MeasuredBy != "ilker" { + t.Errorf("the measurement should say who took it; got %q", cap.rows[0].MeasuredBy) + } +} + +func TestAMeasurementForAModelThatIsNotThereIsRefused(t *testing.T) { + _, cap, mux := capacityAPI(t) + + req := httptest.NewRequest(http.MethodPut, "/api/v1/models/typo/capacity", + strings.NewReader(`{"profile":"tp8"}`)) + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, req) + + // A typo in the name files the measurement somewhere nobody will look, and + // the model then looks as though it was never benchmarked. + if rec.Code != http.StatusNotFound { + t.Errorf("expected 404 for an unknown model, got %d", rec.Code) + } + if len(cap.rows) != 0 { + t.Error("nothing should have been stored") + } +} + +func TestAMeasurementNeedsTheProfileItWasTakenOn(t *testing.T) { + _, cap, mux := capacityAPI(t) + + req := httptest.NewRequest(http.MethodPut, "/api/v1/models/llama/capacity", + strings.NewReader(`{"max_concurrency_at_slo":8}`)) + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, req) + + // Capacity belongs to the hardware as much as the weights. Without a + // profile there is nothing to compare a later run against. + if rec.Code != http.StatusBadRequest { + t.Errorf("expected 400 without a profile, got %d", rec.Code) + } + if len(cap.rows) != 0 { + t.Error("nothing should have been stored") + } +} + +func TestAProfileTheModelDoesNotHaveIsRefused(t *testing.T) { + _, _, mux := capacityAPI(t) + + req := httptest.NewRequest(http.MethodPut, "/api/v1/models/llama/capacity", + strings.NewReader(`{"profile":"not-a-profile"}`)) + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusBadRequest { + t.Errorf("expected 400 for a profile the model does not have, got %d", rec.Code) + } +} + +func TestCapacityWithoutAStoreIsAnEmptyListRatherThanAnError(t *testing.T) { + a, _ := setup(t, testModels()) + mux := http.NewServeMux() + a.Register(mux) + + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/v1/capacity", nil)) + + // Nothing measured and no store look the same to a reader, and both mean + // "there is nothing here" — which is not a server fault. + if rec.Code != http.StatusOK { + t.Errorf("expected 200, got %d", rec.Code) + } + var rows []CapacityView + if err := json.Unmarshal(rec.Body.Bytes(), &rows); err != nil || len(rows) != 0 { + t.Errorf("expected an empty list, got %s", rec.Body) + } +} diff --git a/internal/store/capacity.go b/internal/store/capacity.go new file mode 100644 index 0000000..1cd57bc --- /dev/null +++ b/internal/store/capacity.go @@ -0,0 +1,88 @@ +package store + +import ( + "context" + "encoding/json" + "fmt" + "time" +) + +// StoredCapacity is a benchmark's conclusion, kept for whoever has to size the +// model later. +type StoredCapacity struct { + Model string + Profile string + // Measurement is bench.Capacity as JSON. Stored whole for the reason a + // model definition is: a measurement is a nested shape that is still + // growing, and as columns every field added would be a migration. + Measurement []byte + MeasuredAt time.Time + MeasuredBy string +} + +// PutCapacity records what a replica of a model was measured to serve. +// +// Replacing rather than accumulating. A second measurement of the same model on +// the same profile is a better answer to the same question, not another data +// point: hardware and engine builds change, and keeping every run would leave a +// reader choosing between numbers with no basis for the choice. The run itself +// is printed when it happens, for anyone who wants the ramp. +func (s *SQLite) PutCapacity(ctx context.Context, c StoredCapacity) error { + if c.Model == "" || c.Profile == "" { + return fmt.Errorf("a capacity measurement needs a model and a profile") + } + if !json.Valid(c.Measurement) { + return fmt.Errorf("capacity for %s/%s: the measurement is not valid JSON", c.Model, c.Profile) + } + if c.MeasuredAt.IsZero() { + c.MeasuredAt = time.Now() + } + _, err := s.db.ExecContext(ctx, + `INSERT INTO capacity (model, profile, measurement, measured_at, measured_by) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(model, profile) DO UPDATE SET + measurement = excluded.measurement, + measured_at = excluded.measured_at, + measured_by = excluded.measured_by`, + c.Model, c.Profile, string(c.Measurement), c.MeasuredAt, c.MeasuredBy) + return err +} + +// Capacities lists every measurement, newest first. +// +// Newest first because the question a reader usually has is "what do we know +// lately", and an old measurement is the one most likely to be describing +// hardware that has since changed. +func (s *SQLite) Capacities(ctx context.Context) ([]StoredCapacity, error) { + rows, err := s.db.QueryContext(ctx, + `SELECT model, profile, measurement, measured_at, measured_by + FROM capacity ORDER BY measured_at DESC, model, profile`) + if err != nil { + return nil, err + } + defer rows.Close() + + var out []StoredCapacity + for rows.Next() { + var c StoredCapacity + var m string + if err := rows.Scan(&c.Model, &c.Profile, &m, &c.MeasuredAt, &c.MeasuredBy); err != nil { + return nil, err + } + c.Measurement = []byte(m) + out = append(out, c) + } + return out, rows.Err() +} + +// DeleteCapacity forgets a measurement. +// +// Wanted when a model is removed, and wanted on its own: a profile whose +// hardware has been replaced is worse than no measurement, because it looks +// like knowledge. Deleting one that is not there is not an error — the caller +// wanted it gone, and it is. +func (s *SQLite) DeleteCapacity(ctx context.Context, model, profile string) error { + _, err := s.db.ExecContext(ctx, + `DELETE FROM capacity WHERE model = ? AND profile = ?`, model, profile) + return err +} diff --git a/internal/store/capacity_test.go b/internal/store/capacity_test.go new file mode 100644 index 0000000..0016b4c --- /dev/null +++ b/internal/store/capacity_test.go @@ -0,0 +1,137 @@ +package store + +import ( + "context" + "path/filepath" + "testing" + "time" +) + +func capacityStore(t *testing.T) *SQLite { + t.Helper() + s, err := OpenSQLite(context.Background(), filepath.Join(t.TempDir(), "cap.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = s.Close() }) + return s +} + +func TestAMeasurementSurvivesTheTerminalItWasPrintedIn(t *testing.T) { + s := capacityStore(t) + at := time.Now().UTC().Truncate(time.Second) + + if err := s.PutCapacity(t.Context(), StoredCapacity{ + Model: "qwen", Profile: "cpu", + Measurement: []byte(`{"max_concurrency_at_slo":8}`), + MeasuredAt: at, MeasuredBy: "ilker", + }); err != nil { + t.Fatal(err) + } + + got, err := s.Capacities(t.Context()) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 { + t.Fatalf("expected the one measurement, got %d", len(got)) + } + if got[0].Model != "qwen" || got[0].Profile != "cpu" { + t.Errorf("wrong row: %+v", got[0]) + } + if string(got[0].Measurement) != `{"max_concurrency_at_slo":8}` { + t.Errorf("the measurement came back changed: %s", got[0].Measurement) + } + if !got[0].MeasuredAt.UTC().Equal(at) { + t.Errorf("when it was taken is part of the answer; got %v want %v", got[0].MeasuredAt, at) + } +} + +func TestTheSameModelOnTwoProfilesIsTwoMeasurements(t *testing.T) { + s := capacityStore(t) + for _, p := range []string{"cpu", "gpu"} { + if err := s.PutCapacity(t.Context(), StoredCapacity{ + Model: "qwen", Profile: p, Measurement: []byte(`{}`), + }); err != nil { + t.Fatal(err) + } + } + + // Capacity belongs to the hardware and the engine arguments as much as to + // the weights. Keyed by model alone, benchmarking the GPU nodes would + // overwrite what is known about the CPU ones. + got, _ := s.Capacities(t.Context()) + if len(got) != 2 { + t.Fatalf("expected one measurement per profile, got %d", len(got)) + } +} + +func TestRemeasuringReplacesRatherThanAccumulates(t *testing.T) { + s := capacityStore(t) + older := time.Now().Add(-24 * time.Hour) + if err := s.PutCapacity(t.Context(), StoredCapacity{ + Model: "qwen", Profile: "cpu", + Measurement: []byte(`{"max_concurrency_at_slo":4}`), MeasuredAt: older, + }); err != nil { + t.Fatal(err) + } + if err := s.PutCapacity(t.Context(), StoredCapacity{ + Model: "qwen", Profile: "cpu", + Measurement: []byte(`{"max_concurrency_at_slo":9}`), MeasuredAt: time.Now(), + }); err != nil { + t.Fatal(err) + } + + // A second run is a better answer to the same question, not another data + // point: keeping both would leave a reader choosing between them with no + // basis for the choice. + got, _ := s.Capacities(t.Context()) + if len(got) != 1 { + t.Fatalf("expected the measurement to be replaced, got %d rows", len(got)) + } + if string(got[0].Measurement) != `{"max_concurrency_at_slo":9}` { + t.Errorf("the older measurement won: %s", got[0].Measurement) + } +} + +func TestMeasurementsComeBackNewestFirst(t *testing.T) { + s := capacityStore(t) + now := time.Now() + _ = s.PutCapacity(t.Context(), StoredCapacity{ + Model: "old", Profile: "cpu", Measurement: []byte(`{}`), + MeasuredAt: now.Add(-48 * time.Hour), + }) + _ = s.PutCapacity(t.Context(), StoredCapacity{ + Model: "new", Profile: "cpu", Measurement: []byte(`{}`), MeasuredAt: now, + }) + + got, _ := s.Capacities(t.Context()) + // The old one is the one most likely to describe hardware that has since + // changed, so it is the one that should need scrolling to. + if len(got) != 2 || got[0].Model != "new" { + t.Errorf("expected the newest first, got %+v", got) + } +} + +func TestRubbishIsRefusedBeforeItIsStored(t *testing.T) { + s := capacityStore(t) + if err := s.PutCapacity(t.Context(), StoredCapacity{ + Model: "qwen", Profile: "cpu", Measurement: []byte(`not json`), + }); err == nil { + t.Error("a measurement that is not JSON should be refused at the write") + } + if err := s.PutCapacity(t.Context(), StoredCapacity{ + Profile: "cpu", Measurement: []byte(`{}`), + }); err == nil { + t.Error("a measurement with no model should be refused") + } +} + +func TestForgettingAMeasurementThatIsNotThereIsFine(t *testing.T) { + s := capacityStore(t) + // The caller wanted it gone, and it is. Erroring would make cleaning up + // after a removed model a thing that needs a check first. + if err := s.DeleteCapacity(t.Context(), "ghost", "cpu"); err != nil { + t.Errorf("deleting an absent measurement should not error: %v", err) + } +} diff --git a/internal/store/copy.go b/internal/store/copy.go index 7149beb..d6d1c01 100644 --- a/internal/store/copy.go +++ b/internal/store/copy.go @@ -29,6 +29,10 @@ var copyOrder = []string{ "limits", "usage", "models", + // No foreign key to models, deliberately: a measurement outlives the + // catalog entry it was taken for, and losing it when somebody removes a + // model would mean re-running a benchmark that costs real GPU time. + "capacity", "threads", "thread_messages", "memory_notes", diff --git a/internal/store/migrations/postgres/0009_capacity.sql b/internal/store/migrations/postgres/0009_capacity.sql new file mode 100644 index 0000000..102f75b --- /dev/null +++ b/internal/store/migrations/postgres/0009_capacity.sql @@ -0,0 +1,34 @@ +-- Ported from the SQLite migration of the same number. +-- +-- What one replica of a model can actually serve, measured rather than guessed. +-- +-- `llmgw ctl bench` has always been able to work this out — it ramps +-- concurrency until the SLO breaks and reports the last level that held. Until +-- now it printed that to a terminal and nothing kept it, so the one number that +-- says what a replica is worth lived in somebody's scrollback. Every decision +-- downstream then had to guess: admission uses a single global default for +-- healthy queue depth, and anything that wanted to size a model against real +-- demand had nothing to size against. +-- +-- Keyed by model and profile, not by model alone. Capacity is a property of the +-- hardware and the engine arguments as much as of the weights: the same model +-- on a CPU profile and on an H100 profile are two different numbers, and +-- storing one would overwrite the other every time somebody benchmarked the +-- other kind of node. +-- +-- The measurement is JSON for the same reason a model definition is: it is a +-- nested shape that is still growing, and as columns every field added to it +-- would be a migration. +-- +-- Nothing here is authoritative about what a model may do — it is an +-- observation, with the time and the target it was taken from, so a reader can +-- decide whether it is still true. Hardware changes and engine builds change; +-- a capacity profile from six months ago is a hypothesis, not a fact. +CREATE TABLE capacity ( + model TEXT NOT NULL, + profile TEXT NOT NULL, + measurement TEXT NOT NULL, + measured_at TIMESTAMP NOT NULL, + measured_by TEXT NOT NULL DEFAULT '', + PRIMARY KEY (model, profile) +); diff --git a/internal/store/migrations/sqlite/0009_capacity.sql b/internal/store/migrations/sqlite/0009_capacity.sql new file mode 100644 index 0000000..907950b --- /dev/null +++ b/internal/store/migrations/sqlite/0009_capacity.sql @@ -0,0 +1,32 @@ +-- What one replica of a model can actually serve, measured rather than guessed. +-- +-- `llmgw ctl bench` has always been able to work this out — it ramps +-- concurrency until the SLO breaks and reports the last level that held. Until +-- now it printed that to a terminal and nothing kept it, so the one number that +-- says what a replica is worth lived in somebody's scrollback. Every decision +-- downstream then had to guess: admission uses a single global default for +-- healthy queue depth, and anything that wanted to size a model against real +-- demand had nothing to size against. +-- +-- Keyed by model and profile, not by model alone. Capacity is a property of the +-- hardware and the engine arguments as much as of the weights: the same model +-- on a CPU profile and on an H100 profile are two different numbers, and +-- storing one would overwrite the other every time somebody benchmarked the +-- other kind of node. +-- +-- The measurement is JSON for the same reason a model definition is: it is a +-- nested shape that is still growing, and as columns every field added to it +-- would be a migration. +-- +-- Nothing here is authoritative about what a model may do — it is an +-- observation, with the time and the target it was taken from, so a reader can +-- decide whether it is still true. Hardware changes and engine builds change; +-- a capacity profile from six months ago is a hypothesis, not a fact. +CREATE TABLE capacity ( + model TEXT NOT NULL, + profile TEXT NOT NULL, + measurement TEXT NOT NULL, + measured_at TIMESTAMP NOT NULL, + measured_by TEXT NOT NULL DEFAULT '', + PRIMARY KEY (model, profile) +);