Ilker ManapandClaude Opus 5
3aa250428a
`ctl bench` has always been able to work out what one replica can serve — it ramps concurrency until the SLO breaks and reports the last level that held. It printed that and nothing kept it, so the one number saying what a replica is worth lived in whoever ran it's scrollback. Everything downstream then had to guess: admission applies a single global default for healthy queue depth, and sizing a model against real demand had nothing to size against. `ctl bench --save` records it; `ctl capacity` reads it back. Keyed by model and profile. Capacity belongs to the hardware and the engine arguments as much as to the weights, so the same model on CPU and on an H100 are two numbers, and one key would overwrite the other on every run. The profile is looked up from the replica rather than asked for, because the control plane already knows which profile answers on which endpoint and a measurement filed under the wrong one is worse than none. A target that is not a replica — the gateway, usually — is refused with that said: benchmarking the whole path is worth doing and says nothing about the hardware. The conclusion is stored, not the ramp. Every level and its latencies are worth reading once while judging whether the run was sound; keeping them would grow without bound as a model is re-benchmarked. Re-measuring replaces. A second run is a better answer to the same question, not another data point, and keeping both would leave a reader choosing between numbers with no basis for the choice. The SLO and the date are stored with it. Capacity means nothing without the target it was measured against, and hardware and engine builds change — a measurement is an observation with a date on it, not a fact about the model. A failed save is reported and not fatal: the run has already cost the GPU time. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
78 lines
2.4 KiB
Go
78 lines
2.4 KiB
Go
package cli
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"text/tabwriter"
|
|
"time"
|
|
|
|
"github.com/spf13/cobra"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
|
)
|
|
|
|
// newCapacityCmd shows what benchmarks have found out.
|
|
//
|
|
// `ctl bench` has always been able to measure this; until it could be saved,
|
|
// the number lived in whoever ran it's scrollback. This is the other half:
|
|
// somewhere to read it back, so sizing a model is looking something up rather
|
|
// than re-running an hour of load.
|
|
func newCapacityCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
|
return &cobra.Command{
|
|
Use: "capacity",
|
|
Short: "What a replica of each model was measured to serve",
|
|
Long: "Recorded by `llmgw ctl bench --save`, per model and profile — the same\n" +
|
|
"model on CPU and on a GPU are different numbers.\n\n" +
|
|
"Age matters. Hardware is replaced and engine builds change, so a\n" +
|
|
"measurement is an observation with a date on it rather than a fact about\n" +
|
|
"the model. The column is there to be read.",
|
|
Args: cobra.NoArgs,
|
|
RunE: func(cmd *cobra.Command, _ []string) error {
|
|
var rows []adminapi.CapacityView
|
|
if err := client().get(cmd.Context(), "/api/v1/capacity", &rows); err != nil {
|
|
return err
|
|
}
|
|
if *asJSON {
|
|
return json.NewEncoder(cmd.OutOrStdout()).Encode(rows)
|
|
}
|
|
if len(rows) == 0 {
|
|
fmt.Fprintln(cmd.OutOrStdout(),
|
|
"Nothing has been measured. Run `llmgw ctl bench <model> --target <replica> --save`.")
|
|
return nil
|
|
}
|
|
|
|
w := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 2, ' ', 0)
|
|
fmt.Fprintln(w, "MODEL\tPROFILE\tCONCURRENT\tTOK/S\tTTFT p99\tTPOT p99\tWAIT\tKV\tMEASURED\tBY")
|
|
for _, c := range rows {
|
|
concurrent := fmt.Sprint(c.MaxConcurrencyAtSLO)
|
|
if !c.Usable() {
|
|
// Zero is a real result — this hardware cannot serve this
|
|
// model at this SLO — and printing it as a number invites
|
|
// somebody to divide by it.
|
|
concurrent = "none at SLO"
|
|
}
|
|
fmt.Fprintf(w, "%s\t%s\t%s\t%.0f\t%s\t%s\t%d\t%.0f%%\t%s\t%s\n",
|
|
c.Model, c.Profile, concurrent, c.ThroughputTokPerSec,
|
|
ms(c.TTFTp99), ms(c.TPOTp99),
|
|
c.SuggestedTargetWaiting, c.SuggestedKVCeiling*100,
|
|
age(c.MeasuredAt), or(c.MeasuredBy, "—"))
|
|
}
|
|
return w.Flush()
|
|
},
|
|
}
|
|
}
|
|
|
|
func age(t time.Time) string {
|
|
if t.IsZero() {
|
|
return "unknown"
|
|
}
|
|
return time.Since(t).Round(time.Minute).String() + " ago"
|
|
}
|
|
|
|
func or(v, fallback string) string {
|
|
if v == "" {
|
|
return fallback
|
|
}
|
|
return v
|
|
}
|