Keep what a benchmark found out, instead of leaving it in a terminal
`ctl bench` has always been able to work out what one replica can serve — it ramps concurrency until the SLO breaks and reports the last level that held. It printed that and nothing kept it, so the one number saying what a replica is worth lived in whoever ran it's scrollback. Everything downstream then had to guess: admission applies a single global default for healthy queue depth, and sizing a model against real demand had nothing to size against. `ctl bench --save` records it; `ctl capacity` reads it back. Keyed by model and profile. Capacity belongs to the hardware and the engine arguments as much as to the weights, so the same model on CPU and on an H100 are two numbers, and one key would overwrite the other on every run. The profile is looked up from the replica rather than asked for, because the control plane already knows which profile answers on which endpoint and a measurement filed under the wrong one is worse than none. A target that is not a replica — the gateway, usually — is refused with that said: benchmarking the whole path is worth doing and says nothing about the hardware. The conclusion is stored, not the ramp. Every level and its latencies are worth reading once while judging whether the run was sound; keeping them would grow without bound as a model is re-benchmarked. Re-measuring replaces. A second run is a better answer to the same question, not another data point, and keeping both would leave a reader choosing between numbers with no basis for the choice. The SLO and the date are stored with it. Capacity means nothing without the target it was measured against, and hardware and engine builds change — a measurement is an observation with a date on it, not a fact about the model. A failed save is reported and not fatal: the run has already cost the GPU time. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
707d774c76
commit
3aa250428a
12 files changed
+743
-4
No files matched your search
+67
-1
@@ -3,14 +3,16 @@ package cli
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/bench"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
)
|
||||
|
||||
func newBenchCmd(asJSON *bool) *cobra.Command {
|
||||
func newBenchCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
cfg := bench.DefaultConfig()
|
||||
var (
|
||||
steps []int
|
||||
@@ -18,6 +20,7 @@ func newBenchCmd(asJSON *bool) *cobra.Command {
|
||||
metrics string
|
||||
engine string
|
||||
key string
|
||||
save bool
|
||||
)
|
||||
|
||||
cmd := &cobra.Command{
|
||||
@@ -78,6 +81,15 @@ func newBenchCmd(asJSON *bool) *cobra.Command {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if save {
|
||||
if err := saveCapacity(cmd, client, res, target); err != nil {
|
||||
// Reported, not fatal. The run has already cost the GPU
|
||||
// time; losing its conclusion to a failed write would mean
|
||||
// running it again, and the numbers are on screen either
|
||||
// way.
|
||||
cmd.PrintErrf("the measurement was not saved: %v\n", err)
|
||||
}
|
||||
}
|
||||
if *asJSON {
|
||||
return json.NewEncoder(cmd.OutOrStdout()).Encode(res)
|
||||
}
|
||||
@@ -99,9 +111,63 @@ func newBenchCmd(asJSON *bool) *cobra.Command {
|
||||
"send every request the same prompt, measuring the cached case instead of the hardware")
|
||||
f.DurationVar(&cfg.SLO.TTFTp99, "ttft-p99", cfg.SLO.TTFTp99, "time-to-first-token target")
|
||||
f.DurationVar(&cfg.SLO.TPOTp99, "tpot-p99", cfg.SLO.TPOTp99, "per-output-token target")
|
||||
f.BoolVar(&save, "save", false,
|
||||
"record the capacity against the model and profile, so the number outlives this terminal")
|
||||
return cmd
|
||||
}
|
||||
|
||||
// saveCapacity records a run's conclusion against the profile it was measured
|
||||
// on.
|
||||
//
|
||||
// The profile is looked up from the replica rather than asked for. Capacity
|
||||
// belongs to the hardware and the engine arguments as much as to the weights,
|
||||
// so a measurement filed under the wrong profile is worse than none — and the
|
||||
// control plane already knows which profile is answering on which endpoint, so
|
||||
// nobody needs to be trusted to type it.
|
||||
func saveCapacity(cmd *cobra.Command, client func() *ctlClient, res bench.Result, target string) error {
|
||||
var reps []adminapi.ReplicaView
|
||||
if err := client().get(cmd.Context(), "/api/v1/replicas", &reps); err != nil {
|
||||
return err
|
||||
}
|
||||
profile := ""
|
||||
for _, rep := range reps {
|
||||
if rep.Model == res.Model && sameEndpoint(rep.Endpoint, target) {
|
||||
profile = rep.Profile
|
||||
break
|
||||
}
|
||||
}
|
||||
if profile == "" {
|
||||
// Almost always because --target was the gateway. Benchmarking the
|
||||
// whole path is a useful thing to do and says nothing about what one
|
||||
// replica can serve, so there is nothing here worth filing as capacity.
|
||||
return fmt.Errorf("%s is not a replica of %s, so this run measures the path rather than the hardware; "+
|
||||
"point --target at a replica endpoint from `llmgw ctl replicas`", target, res.Model)
|
||||
}
|
||||
|
||||
body, err := json.Marshal(res.Capacity(profile))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var saved bench.Capacity
|
||||
if err := client().put(cmd.Context(),
|
||||
"/api/v1/models/"+res.Model+"/capacity", body, &saved); err != nil {
|
||||
return err
|
||||
}
|
||||
cmd.PrintErrf("saved: %s on profile %s, %d concurrent at the SLO\n",
|
||||
saved.Model, saved.Profile, saved.MaxConcurrencyAtSLO)
|
||||
return nil
|
||||
}
|
||||
|
||||
// sameEndpoint compares two base URLs, ignoring a trailing slash.
|
||||
//
|
||||
// Written out because the two come from different places: one is what the
|
||||
// control plane recorded, the other is what somebody typed or pasted, and a
|
||||
// trailing slash is the difference that would silently make every run look like
|
||||
// it was aimed at the gateway.
|
||||
func sameEndpoint(a, b string) bool {
|
||||
return strings.TrimSuffix(a, "/") == strings.TrimSuffix(b, "/")
|
||||
}
|
||||
|
||||
func printConclusion(cmd *cobra.Command, res bench.Result, cfg bench.Config) {
|
||||
out := cmd.OutOrStdout()
|
||||
fmt.Fprintln(out)
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"text/tabwriter"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
)
|
||||
|
||||
// newCapacityCmd shows what benchmarks have found out.
|
||||
//
|
||||
// `ctl bench` has always been able to measure this; until it could be saved,
|
||||
// the number lived in whoever ran it's scrollback. This is the other half:
|
||||
// somewhere to read it back, so sizing a model is looking something up rather
|
||||
// than re-running an hour of load.
|
||||
func newCapacityCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "capacity",
|
||||
Short: "What a replica of each model was measured to serve",
|
||||
Long: "Recorded by `llmgw ctl bench --save`, per model and profile — the same\n" +
|
||||
"model on CPU and on a GPU are different numbers.\n\n" +
|
||||
"Age matters. Hardware is replaced and engine builds change, so a\n" +
|
||||
"measurement is an observation with a date on it rather than a fact about\n" +
|
||||
"the model. The column is there to be read.",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
var rows []adminapi.CapacityView
|
||||
if err := client().get(cmd.Context(), "/api/v1/capacity", &rows); err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
return json.NewEncoder(cmd.OutOrStdout()).Encode(rows)
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
fmt.Fprintln(cmd.OutOrStdout(),
|
||||
"Nothing has been measured. Run `llmgw ctl bench <model> --target <replica> --save`.")
|
||||
return nil
|
||||
}
|
||||
|
||||
w := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 2, ' ', 0)
|
||||
fmt.Fprintln(w, "MODEL\tPROFILE\tCONCURRENT\tTOK/S\tTTFT p99\tTPOT p99\tWAIT\tKV\tMEASURED\tBY")
|
||||
for _, c := range rows {
|
||||
concurrent := fmt.Sprint(c.MaxConcurrencyAtSLO)
|
||||
if !c.Usable() {
|
||||
// Zero is a real result — this hardware cannot serve this
|
||||
// model at this SLO — and printing it as a number invites
|
||||
// somebody to divide by it.
|
||||
concurrent = "none at SLO"
|
||||
}
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%.0f\t%s\t%s\t%d\t%.0f%%\t%s\t%s\n",
|
||||
c.Model, c.Profile, concurrent, c.ThroughputTokPerSec,
|
||||
ms(c.TTFTp99), ms(c.TPOTp99),
|
||||
c.SuggestedTargetWaiting, c.SuggestedKVCeiling*100,
|
||||
age(c.MeasuredAt), or(c.MeasuredBy, "—"))
|
||||
}
|
||||
return w.Flush()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func age(t time.Time) string {
|
||||
if t.IsZero() {
|
||||
return "unknown"
|
||||
}
|
||||
return time.Since(t).Round(time.Minute).String() + " ago"
|
||||
}
|
||||
|
||||
func or(v, fallback string) string {
|
||||
if v == "" {
|
||||
return fallback
|
||||
}
|
||||
return v
|
||||
}
|
||||
+8
-1
@@ -43,6 +43,12 @@ func (c *ctlClient) post(ctx context.Context, path string, body []byte, out any)
|
||||
return c.do(ctx, http.MethodPost, path, json.RawMessage(body), out)
|
||||
}
|
||||
|
||||
// put replaces a resource. Used where a second call is a better answer to the
|
||||
// same question rather than another one: re-measuring a model's capacity.
|
||||
func (c *ctlClient) put(ctx context.Context, path string, body []byte, out any) error {
|
||||
return c.do(ctx, http.MethodPut, path, json.RawMessage(body), out)
|
||||
}
|
||||
|
||||
func (c *ctlClient) delete(ctx context.Context, path string, out any) error {
|
||||
return c.do(ctx, http.MethodDelete, path, nil, out)
|
||||
}
|
||||
@@ -121,13 +127,14 @@ func newCtlCmd() *cobra.Command {
|
||||
"Move work off a node, one replica at a time"),
|
||||
// cert works on files rather than the admin API, so --server does not
|
||||
// apply to it; it lives here because this is where operators look.
|
||||
newBenchCmd(&asJSON),
|
||||
newBenchCmd(client, &asJSON),
|
||||
newStatusCmd(client, &asJSON),
|
||||
newModelCmd(client),
|
||||
newScaleCmd(client),
|
||||
newEngineCheckCmd(),
|
||||
newToolCheckCmd(),
|
||||
newWeightsCmd(client, &asJSON),
|
||||
newCapacityCmd(client, &asJSON),
|
||||
newCertCmd(),
|
||||
)
|
||||
return cmd
|
||||
|
||||
@@ -158,6 +158,10 @@ func newServerCmd() *cobra.Command {
|
||||
admin.ReplaceCatalog(ctx, built)
|
||||
}
|
||||
api.SetModels(svc)
|
||||
// Benchmark results live here too. A measurement costs real
|
||||
// GPU time; without somewhere to put it the number lives in
|
||||
// whoever ran it's terminal and is gone by morning.
|
||||
api.SetCapacity(st)
|
||||
}
|
||||
api.Register(adminMux)
|
||||
|
||||
|
||||
Reference in new issue
Block a user