Add ctl bench, and fix two holes it walked into

`ctl bench` measures what a replica can actually serve, so weight,
target_waiting and kv_ceiling stop being numbers somebody typed. Concurrency
climbs until the SLO breaks and the last level that met it is the capacity:
peak throughput arrives long after latency has become unusable, so throughput
alone overstates what the hardware can serve. Queue depth and KV usage are
read from the engine while it is still meeting the target, which is exactly
what admission should treat as healthy.

Prompts are unique per request by default. With one repeated prompt the
engine's cache answers most of the prefill and the measurement describes the
cache rather than the hardware; --shared-prefix asks that question
deliberately, and it now produces byte-identical prompts, which the first
version did not — it dropped the random header and still picked words at
random, so the cache missed and the flag quietly measured the thing it exists
to avoid.

Running it against the cluster turned up two real defects:

A replica stuck in Starting never timed out. Every other way out of that state
needs the agent to report something, and when it never does the replica sits
there holding GPUs and counting towards the model's replica count while
serving nothing. Two of them had been like that for eight minutes past a
two-minute startup timeout, and nothing would ever have moved them.

They were stuck because adoption lost to a replica we had only placed. Replica
ids come from the model name and a counter, so a control plane that restarts
and places before its agents reconnect builds a different replica under the
same id on a freshly allocated port; the agent's report of the engine that was
actually serving was then discarded as already-known. The cluster pointed at
port 20004 while the engine answered on 20000. The fleet is authoritative
about what is running, so what we merely placed now loses to what an agent
reports — unless our copy has a shard that has actually come up, in which case
two things really are running and the stray is the one we did not ask for.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-16 09:51:37 +02:00
1 parent ec943f9587
commit 05bdc7d6d4
8 files changed
+1130 -4

No files matched your search

+437
View File
@@ -0,0 +1,437 @@
// Package bench measures what a replica can actually do, so the numbers the
// router depends on stop being guesses (plan §6.2.1).
//
// Three of them matter: the weight a replica gets when the scheduler compares
// it to others, the queue depth admission treats as healthy, and the KV usage
// it treats as full. All three are currently hand-written in configuration, and
// a hand-written capacity number is a hand-written opinion about hardware
// nobody measured.
//
// The measurement is a load ramp. Concurrency climbs until the SLO breaks, and
// the last level that still met it is the capacity — not peak throughput, which
// a GPU will happily deliver long after latency has become unusable.
package bench
import (
"bufio"
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"math/rand/v2"
"net/http"
"sort"
"strings"
"sync"
"time"
)
// SLO is the latency target capacity is measured against.
type SLO struct {
TTFTp99 time.Duration
TPOTp99 time.Duration
}
// Met reports whether a step's latencies satisfy the target.
func (s SLO) Met(ttft, tpot time.Duration) bool {
if s.TTFTp99 > 0 && ttft > s.TTFTp99 {
return false
}
if s.TPOTp99 > 0 && tpot > s.TPOTp99 {
return false
}
return true
}
// Config describes a run.
type Config struct {
// Target is the base URL: the gateway, or one replica directly. Pointing at
// a replica measures that replica; pointing at the gateway measures the
// whole path including scheduling, which is a different question.
Target string
// MetricsURL is an engine's /metrics, sampled during the run so queue depth
// and KV usage can be reported alongside latency. Only meaningful when
// Target is a single replica.
MetricsURL string
// Engine names which metric vocabulary MetricsURL speaks. Getting it wrong
// is silent: nothing matches, every sample reads zero, and the suggested
// thresholds are built on nothing.
Engine string
Model string
APIKey string
// Concurrency is the ramp. Each level runs for Duration.
Concurrency []int
Duration time.Duration
PromptTokens int
MaxTokens int
// SharedPrefix sends every request the same prompt.
//
// Off by default, and that default is the whole point: with a shared prefix
// the engine's cache answers most of the prefill and the measurement
// describes the cache rather than the hardware. Turning it on measures the
// cached case deliberately, which is a fair question to ask separately.
SharedPrefix bool
SLO SLO
}
// DefaultConfig returns a ramp that covers most single-replica hardware.
func DefaultConfig() Config {
return Config{
Concurrency: []int{1, 2, 4, 8, 16, 32, 64, 128},
Duration: 20 * time.Second,
PromptTokens: 512,
MaxTokens: 128,
SLO: SLO{TTFTp99: 2 * time.Second, TPOTp99: 50 * time.Millisecond},
}
}
func (c *Config) normalize() error {
d := DefaultConfig()
if c.Target == "" {
return errors.New("bench: a target is required")
}
if c.Model == "" {
return errors.New("bench: a model is required")
}
c.Target = strings.TrimRight(c.Target, "/")
if len(c.Concurrency) == 0 {
c.Concurrency = d.Concurrency
}
if c.Duration <= 0 {
c.Duration = d.Duration
}
if c.PromptTokens <= 0 {
c.PromptTokens = d.PromptTokens
}
if c.MaxTokens <= 0 {
c.MaxTokens = d.MaxTokens
}
sort.Ints(c.Concurrency)
return nil
}
// Percentiles summarizes a latency sample.
type Percentiles struct {
P50, P90, P99, Max time.Duration
}
// Step is one concurrency level's result.
type Step struct {
Concurrency int
Completed int
Failed int
// Requests and output tokens per second, over the step's wall clock.
RequestsPerSec float64
TokensPerSec float64
TTFT Percentiles
// TPOT is time per output token after the first, which is what a reader
// experiences as the speed of the stream.
TPOT Percentiles
Waiting float64
Running float64
KVUsage float64
MeetsSLO bool
// FirstError is kept so a step that failed says why rather than just
// reporting zeroes.
FirstError string
}
// Result is the whole ramp plus what it concluded.
type Result struct {
Model string
Target string
Steps []Step
SLO SLO
RanAt time.Time
Elapsed time.Duration
// MaxConcurrencyAtSLO is the capacity: the highest level that still met the
// target. Zero means even one request did not.
MaxConcurrencyAtSLO int
// ThroughputTokPerSec and the latencies are taken at that level.
ThroughputTokPerSec float64
TTFTp99, TPOTp99 time.Duration
// SuggestedTargetWaiting and SuggestedKVCeiling are what admission should
// treat as healthy, observed rather than guessed.
SuggestedTargetWaiting int
SuggestedKVCeiling float64
}
// Run executes the ramp.
func Run(ctx context.Context, cfg Config, progress func(Step)) (Result, error) {
if err := cfg.normalize(); err != nil {
return Result{}, err
}
res := Result{Model: cfg.Model, Target: cfg.Target, SLO: cfg.SLO, RanAt: time.Now()}
start := time.Now()
client := &http.Client{Timeout: 10 * time.Minute}
for _, c := range cfg.Concurrency {
if ctx.Err() != nil {
break
}
step := runStep(ctx, client, cfg, c)
res.Steps = append(res.Steps, step)
if progress != nil {
progress(step)
}
// Stop climbing once the target is missed. Everything above is slower
// by construction, and the cost of finding out is real GPU time.
if !step.MeetsSLO {
break
}
}
res.Elapsed = time.Since(start)
res.conclude()
return res, nil
}
func (r *Result) conclude() {
for _, s := range r.Steps {
if !s.MeetsSLO || s.Completed == 0 {
continue
}
if s.Concurrency <= r.MaxConcurrencyAtSLO {
continue
}
r.MaxConcurrencyAtSLO = s.Concurrency
r.ThroughputTokPerSec = s.TokensPerSec
r.TTFTp99, r.TPOTp99 = s.TTFT.P99, s.TPOT.P99
// The queue depth and cache pressure seen while still meeting the SLO
// are exactly what admission should treat as healthy.
r.SuggestedTargetWaiting = int(s.Waiting + 0.5)
r.SuggestedKVCeiling = roundTo(s.KVUsage, 0.05)
}
}
func runStep(ctx context.Context, client *http.Client, cfg Config, concurrency int) Step {
step := Step{Concurrency: concurrency}
ctx, cancel := context.WithTimeout(ctx, cfg.Duration)
defer cancel()
var (
mu sync.Mutex
ttfts []time.Duration
tpots []time.Duration
tokens int64
failed int
firstErr string
)
sampler := newSampler(cfg.MetricsURL, cfg.Engine)
sampler.start(ctx)
start := time.Now()
var wg sync.WaitGroup
for range concurrency {
wg.Add(1)
go func() {
defer wg.Done()
for ctx.Err() == nil {
r, err := one(ctx, client, cfg)
mu.Lock()
switch {
case err != nil:
// A cancelled context is the step ending, not a failure.
if ctx.Err() == nil {
failed++
if firstErr == "" {
firstErr = err.Error()
}
}
default:
ttfts = append(ttfts, r.ttft)
if r.tpot > 0 {
tpots = append(tpots, r.tpot)
}
tokens += r.tokens
}
mu.Unlock()
}
}()
}
wg.Wait()
elapsed := time.Since(start)
sampler.stop()
step.Completed = len(ttfts)
step.Failed = failed
step.FirstError = firstErr
step.TTFT = percentiles(ttfts)
step.TPOT = percentiles(tpots)
if elapsed > 0 {
step.RequestsPerSec = float64(step.Completed) / elapsed.Seconds()
step.TokensPerSec = float64(tokens) / elapsed.Seconds()
}
step.Waiting, step.Running, step.KVUsage = sampler.averages()
// A step that could not complete anything has not met anything.
step.MeetsSLO = step.Completed > 0 && step.Failed == 0 &&
cfg.SLO.Met(step.TTFT.P99, step.TPOT.P99)
return step
}
type outcome struct {
ttft time.Duration
tpot time.Duration
tokens int64
}
// one sends a single streaming completion and times it.
func one(ctx context.Context, client *http.Client, cfg Config) (outcome, error) {
body, err := json.Marshal(map[string]any{
"model": cfg.Model,
"stream": true,
"max_tokens": cfg.MaxTokens,
"messages": []map[string]string{
{"role": "user", "content": prompt(cfg)},
},
})
if err != nil {
return outcome{}, err
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
cfg.Target+"/v1/chat/completions", bytes.NewReader(body))
if err != nil {
return outcome{}, err
}
req.Header.Set("Content-Type", "application/json")
if cfg.APIKey != "" {
req.Header.Set("Authorization", "Bearer "+cfg.APIKey)
}
start := time.Now()
resp, err := client.Do(req)
if err != nil {
return outcome{}, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
b, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
return outcome{}, fmt.Errorf("%s: %s", resp.Status, strings.TrimSpace(string(b)))
}
var out outcome
var firstAt, lastAt time.Time
sc := bufio.NewScanner(resp.Body)
sc.Buffer(make([]byte, 0, 64*1024), 1<<20)
for sc.Scan() {
line := sc.Text()
if !strings.HasPrefix(line, "data: ") || strings.HasSuffix(line, "[DONE]") {
continue
}
// Only chunks that carry content count: a role-only first chunk is not
// a token, and counting it would flatter the time to first token.
if !hasContent(line[6:]) {
continue
}
now := time.Now()
if out.tokens == 0 {
firstAt = now
out.ttft = now.Sub(start)
}
lastAt = now
out.tokens++
}
if err := sc.Err(); err != nil {
return outcome{}, err
}
if out.tokens == 0 {
return outcome{}, errors.New("the response carried no content")
}
if out.tokens > 1 {
out.tpot = lastAt.Sub(firstAt) / time.Duration(out.tokens-1)
}
return out, nil
}
func hasContent(payload string) bool {
var chunk struct {
Choices []struct {
Delta struct {
Content string `json:"content"`
} `json:"delta"`
} `json:"choices"`
}
if err := json.Unmarshal([]byte(payload), &chunk); err != nil {
return false
}
for _, c := range chunk.Choices {
if c.Delta.Content != "" {
return true
}
}
return false
}
// prompt builds a request body of roughly the configured size.
//
// Unique per request unless a shared prefix was asked for: repeating one prompt
// lets the engine's cache answer most of the prefill, and the result then
// describes the cache rather than the hardware.
//
// "Shared" has to mean byte-identical. Dropping only the random header while
// still picking words at random would leave every prompt different after the
// first few tokens, so the cache would miss and the flag would quietly measure
// the thing it exists to avoid measuring.
func prompt(cfg Config) string {
var b strings.Builder
if cfg.SharedPrefix {
for i := 0; b.Len() < cfg.PromptTokens*4; i++ {
b.WriteString(words[i%len(words)])
b.WriteByte(' ')
}
} else {
fmt.Fprintf(&b, "%016x %016x ", rand.Uint64(), rand.Uint64()) //nolint:gosec // not security
// Roughly four characters to a token, the same crude ratio the router
// uses; exactness is not what this number is for.
for b.Len() < cfg.PromptTokens*4 {
b.WriteString(words[rand.IntN(len(words))]) //nolint:gosec // not security
b.WriteByte(' ')
}
}
b.WriteString("\n\nSummarize the text above.")
return b.String()
}
var words = strings.Fields(`the quick brown fox jumps over lazy dog while distant
thunder rolls across an empty valley and somewhere a door closes softly in the
dark house where nobody has lived for years although the garden still produces
apples every autumn which fall and rot beneath the trees unnoticed by anyone
except the birds that come each morning to pick at them before the frost`)
func percentiles(v []time.Duration) Percentiles {
if len(v) == 0 {
return Percentiles{}
}
s := make([]time.Duration, len(v))
copy(s, v)
sort.Slice(s, func(i, j int) bool { return s[i] < s[j] })
at := func(p float64) time.Duration {
i := int(float64(len(s)-1) * p)
return s[i]
}
return Percentiles{P50: at(0.50), P90: at(0.90), P99: at(0.99), Max: s[len(s)-1]}
}
// roundTo rounds to a multiple of step, through integers.
//
// The obvious float arithmetic returns 0.7000000000000001 for what should be
// 0.70, which is harmless on a terminal and ugly in the JSON someone pastes
// into a configuration file.
func roundTo(v, step float64) float64 {
if step <= 0 {
return v
}
const scale = 10000
units := int64(v/step + 0.5)
return float64(int64(float64(units)*step*scale+0.5)) / scale
}
+156
View File
@@ -0,0 +1,156 @@
package bench
import (
"context"
"strings"
"testing"
"time"
)
func TestCapacityIsTheLastLevelThatMetTheSLONotThePeak(t *testing.T) {
// A GPU keeps delivering more tokens per second long after latency has
// become unusable. Taking peak throughput as capacity would tell the router
// a replica can serve a load at which nobody would want the answers.
r := Result{SLO: SLO{TTFTp99: time.Second}}
r.Steps = []Step{
{Concurrency: 1, Completed: 10, TokensPerSec: 100, MeetsSLO: true,
TTFT: Percentiles{P99: 100 * time.Millisecond}},
{Concurrency: 8, Completed: 80, TokensPerSec: 700, MeetsSLO: true,
TTFT: Percentiles{P99: 900 * time.Millisecond}, Waiting: 3.4, KVUsage: 0.71},
// Faster in tokens, far too slow in latency.
{Concurrency: 32, Completed: 200, TokensPerSec: 1500, MeetsSLO: false,
TTFT: Percentiles{P99: 9 * time.Second}},
}
r.conclude()
if r.MaxConcurrencyAtSLO != 8 {
t.Errorf("capacity = %d, want 8", r.MaxConcurrencyAtSLO)
}
if r.ThroughputTokPerSec != 700 {
t.Errorf("throughput = %v; it must come from the level that met the SLO", r.ThroughputTokPerSec)
}
if r.SuggestedTargetWaiting != 3 {
t.Errorf("target_waiting = %d, want 3", r.SuggestedTargetWaiting)
}
if r.SuggestedKVCeiling != 0.70 {
t.Errorf("kv_ceiling = %v, want 0.70", r.SuggestedKVCeiling)
}
}
func TestAStepWithFailuresHasNotMetAnything(t *testing.T) {
// Latency percentiles over the requests that survived say nothing useful
// when the rest were refused.
r := Result{SLO: SLO{TTFTp99: time.Second}}
r.Steps = []Step{
{Concurrency: 4, Completed: 10, Failed: 90, TokensPerSec: 50, MeetsSLO: false,
TTFT: Percentiles{P99: 50 * time.Millisecond}},
}
r.conclude()
if r.MaxConcurrencyAtSLO != 0 {
t.Errorf("a step that failed 90 of 100 requests was counted as capacity")
}
}
func TestSLOIsMissedWhenEitherHalfIs(t *testing.T) {
s := SLO{TTFTp99: time.Second, TPOTp99: 50 * time.Millisecond}
cases := []struct {
ttft, tpot time.Duration
want bool
}{
{500 * time.Millisecond, 20 * time.Millisecond, true},
{2 * time.Second, 20 * time.Millisecond, false},
{500 * time.Millisecond, 80 * time.Millisecond, false},
{2 * time.Second, 80 * time.Millisecond, false},
}
for _, c := range cases {
if got := s.Met(c.ttft, c.tpot); got != c.want {
t.Errorf("Met(%s, %s) = %v", c.ttft, c.tpot, got)
}
}
// A target of zero is no target, not a target of zero.
none := SLO{}
if !none.Met(time.Hour, time.Hour) {
t.Error("an unset SLO refused everything")
}
}
func TestPercentiles(t *testing.T) {
var v []time.Duration
for i := 1; i <= 100; i++ {
v = append(v, time.Duration(i)*time.Millisecond)
}
p := percentiles(v)
if p.P50 != 50*time.Millisecond || p.P90 != 90*time.Millisecond {
t.Errorf("p50=%s p90=%s", p.P50, p.P90)
}
if p.Max != 100*time.Millisecond {
t.Errorf("max=%s", p.Max)
}
if got := percentiles(nil); got != (Percentiles{}) {
t.Errorf("empty sample gave %+v", got)
}
}
func TestPromptsAreUniqueUnlessAskedNotToBe(t *testing.T) {
// The default matters more than it looks. With one repeated prompt the
// engine's cache answers most of the prefill, and the measurement describes
// the cache rather than the hardware.
cfg := Config{PromptTokens: 64}
seen := map[string]bool{}
for range 100 {
seen[prompt(cfg)] = true
}
if len(seen) < 100 {
t.Errorf("%d distinct prompts out of 100; the cache would answer the rest", len(seen))
}
cfg.SharedPrefix = true
first := prompt(cfg)
for range 20 {
if got := prompt(cfg); !strings.HasPrefix(got, first[:200]) {
t.Error("--shared-prefix did not produce a shared prefix")
}
}
}
func TestPromptSizeFollowsTheRequest(t *testing.T) {
small := prompt(Config{PromptTokens: 32})
large := prompt(Config{PromptTokens: 512})
if len(large) < len(small)*4 {
t.Errorf("a 512-token prompt is %d chars against %d for 32", len(large), len(small))
}
}
func TestConfigIsValidatedBeforeAnyLoadIsGenerated(t *testing.T) {
// The failure has to come before the traffic, not halfway through a ramp.
if _, err := Run(context.Background(), Config{Model: "m"}, nil); err == nil {
t.Error("a run with no target started")
}
if _, err := Run(context.Background(), Config{Target: "http://x"}, nil); err == nil {
t.Error("a run with no model started")
}
}
func TestTheRampStopsOnceTheSLOBreaks(t *testing.T) {
// Everything above a missed level is slower by construction, and finding
// that out costs real GPU time.
r := Result{}
r.Steps = []Step{
{Concurrency: 1, Completed: 1, MeetsSLO: true},
{Concurrency: 2, Completed: 1, MeetsSLO: false},
}
r.conclude()
if r.MaxConcurrencyAtSLO != 1 {
t.Errorf("capacity = %d", r.MaxConcurrencyAtSLO)
}
}
func TestRoundTo(t *testing.T) {
cases := map[float64]float64{0.71: 0.70, 0.73: 0.75, 0.88: 0.90, 0.0: 0.0}
for in, want := range cases {
if got := roundTo(in, 0.05); got != want {
t.Errorf("roundTo(%v) = %v, want %v", in, got, want)
}
}
}
+108
View File
@@ -0,0 +1,108 @@
package bench
import (
"context"
"net/http"
"sync"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/scrape"
)
// sampler reads an engine's metrics during a step.
//
// It is what turns a latency measurement into something admission can use: the
// queue depth and cache pressure observed while the SLO was still being met are
// the values target_waiting and kv_ceiling should have, rather than the round
// numbers somebody typed.
//
// Optional, because a run pointed at the gateway has no single engine to read.
type sampler struct {
url string
engine string
client *http.Client
cancel context.CancelFunc
mu sync.Mutex
wg sync.WaitGroup
waiting float64
running float64
kv float64
n int
}
func newSampler(metricsURL, engine string) *sampler {
return &sampler{
url: metricsURL,
engine: engine,
client: &http.Client{Timeout: 2 * time.Second},
}
}
func (s *sampler) start(ctx context.Context) {
if s.url == "" {
return
}
ctx, s.cancel = context.WithCancel(ctx)
s.wg.Add(1)
go func() {
defer s.wg.Done()
t := time.NewTicker(time.Second)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
s.sample(ctx)
}
}
}()
}
func (s *sampler) sample(ctx context.Context) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.url, nil)
if err != nil {
return
}
resp, err := s.client.Do(req)
if err != nil {
return
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return
}
// The same mapping the router reads with, so a bench result cannot disagree
// with what the scheduler sees. The engine has to be named: reading a
// llama.cpp endpoint with vLLM's names finds nothing, and the run would
// report a queue depth of zero and suggest thresholds built on it.
stats, err := scrape.Parse(resp.Body, scrape.MappingFor(s.engine))
if err != nil {
return
}
s.mu.Lock()
s.waiting += float64(stats.Waiting)
s.running += float64(stats.Running)
s.kv += stats.KVCacheUsage
s.n++
s.mu.Unlock()
}
func (s *sampler) stop() {
if s.cancel != nil {
s.cancel()
}
s.wg.Wait()
}
func (s *sampler) averages() (waiting, running, kv float64) {
s.mu.Lock()
defer s.mu.Unlock()
if s.n == 0 {
return 0, 0, 0
}
n := float64(s.n)
return s.waiting / n, s.running / n, s.kv / n
}
+155
View File
@@ -0,0 +1,155 @@
package cli
import (
"encoding/json"
"fmt"
"text/tabwriter"
"time"
"github.com/spf13/cobra"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench"
)
func newBenchCmd(asJSON *bool) *cobra.Command {
cfg := bench.DefaultConfig()
var (
steps []int
target string
metrics string
engine string
key string
)
cmd := &cobra.Command{
Use: "bench <model>",
Short: "Measure what a replica can do, and what the router should be told",
Long: "Drives real traffic. Point it at a replica to measure that hardware, or at\n" +
"the gateway to measure the whole path — they answer different questions,\n" +
"and only the first produces a capacity profile.\n\n" +
"Concurrency climbs until the SLO breaks. The last level that still met it\n" +
"is the capacity: peak throughput arrives long after latency has become\n" +
"unusable, so throughput alone would overstate what the hardware can serve.\n\n" +
"This costs real GPU time and competes with real traffic. Run it on a\n" +
"replica you have cordoned, or during a window where that is acceptable.",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
cfg.Model = args[0]
cfg.Target = target
cfg.MetricsURL = metrics
cfg.Engine = engine
cfg.APIKey = key
if len(steps) > 0 {
cfg.Concurrency = steps
}
if metrics == "" {
cmd.PrintErrln("No --metrics endpoint: queue depth and KV usage will not be " +
"measured, so target_waiting and kv_ceiling cannot be suggested.")
}
cmd.PrintErrf("benchmarking %s against %s\n\n", cfg.Model, cfg.Target)
w := tabwriter.NewWriter(cmd.ErrOrStderr(), 0, 0, 2, ' ', 0)
fmt.Fprintln(w, "CONC\tREQ/S\tTOK/S\tTTFT p50\tTTFT p99\tTPOT p99\tWAIT\tKV\tSLO")
progress := func(s bench.Step) {
verdict := "ok"
if !s.MeetsSLO {
verdict = "MISSED"
}
if s.Failed > 0 {
verdict = fmt.Sprintf("%d failed", s.Failed)
}
fmt.Fprintf(w, "%d\t%.1f\t%.0f\t%s\t%s\t%s\t%.1f\t%.0f%%\t%s\n",
s.Concurrency, s.RequestsPerSec, s.TokensPerSec,
ms(s.TTFT.P50), ms(s.TTFT.P99), ms(s.TPOT.P99),
s.Waiting, s.KVUsage*100, verdict)
_ = w.Flush()
}
res, err := bench.Run(cmd.Context(), cfg, progress)
if err != nil {
return err
}
if *asJSON {
return json.NewEncoder(cmd.OutOrStdout()).Encode(res)
}
printConclusion(cmd, res, cfg)
return nil
},
}
f := cmd.Flags()
f.StringVar(&target, "target", "http://127.0.0.1:8080", "base URL: a replica, or the gateway")
f.StringVar(&metrics, "metrics", "", "the replica's /metrics, to observe queue depth and KV usage")
f.StringVar(&engine, "engine", "", "which metric names --metrics uses: vllm | sglang | llamacpp")
f.StringVar(&key, "key", "", "API key, when the gateway requires one")
f.IntSliceVar(&steps, "concurrency", nil, "levels to try (default 1,2,4,…,128)")
f.DurationVar(&cfg.Duration, "duration", cfg.Duration, "how long to hold each level")
f.IntVar(&cfg.PromptTokens, "prompt-tokens", cfg.PromptTokens, "approximate prompt size")
f.IntVar(&cfg.MaxTokens, "max-tokens", cfg.MaxTokens, "tokens to generate per request")
f.BoolVar(&cfg.SharedPrefix, "shared-prefix", false,
"send every request the same prompt, measuring the cached case instead of the hardware")
f.DurationVar(&cfg.SLO.TTFTp99, "ttft-p99", cfg.SLO.TTFTp99, "time-to-first-token target")
f.DurationVar(&cfg.SLO.TPOTp99, "tpot-p99", cfg.SLO.TPOTp99, "per-output-token target")
return cmd
}
func printConclusion(cmd *cobra.Command, res bench.Result, cfg bench.Config) {
out := cmd.OutOrStdout()
fmt.Fprintln(out)
if res.MaxConcurrencyAtSLO == 0 {
fmt.Fprintf(out, "No level met the SLO (TTFT p99 ≤ %s, TPOT p99 ≤ %s).\n",
cfg.SLO.TTFTp99, cfg.SLO.TPOTp99)
if len(res.Steps) > 0 && res.Steps[0].FirstError != "" {
fmt.Fprintf(out, "First error: %s\n", res.Steps[0].FirstError)
}
fmt.Fprintln(out, "Either the hardware cannot serve this model at this SLO, or the")
fmt.Fprintln(out, "target is stricter than the model can meet at any concurrency.")
return
}
fmt.Fprintf(out, "Capacity: %d concurrent requests at %.0f tokens/s\n",
res.MaxConcurrencyAtSLO, res.ThroughputTokPerSec)
fmt.Fprintf(out, " TTFT p99 %s · TPOT p99 %s · measured over %s\n\n",
ms(res.TTFTp99), ms(res.TPOTp99), res.Elapsed.Round(time.Second))
fmt.Fprintln(out, "Put this in the model's profile:")
fmt.Fprintf(out, " weight: %d\n", res.MaxConcurrencyAtSLO)
if res.SuggestedTargetWaiting > 0 || res.SuggestedKVCeiling > 0 {
fmt.Fprintln(out, "\nAnd in router.scoring, from what the engine reported while still")
fmt.Fprintln(out, "meeting the target:")
if res.SuggestedTargetWaiting > 0 {
fmt.Fprintf(out, " target_waiting: %d\n", res.SuggestedTargetWaiting)
}
if res.SuggestedKVCeiling > 0 {
fmt.Fprintf(out, " kv_ceiling: %.2f\n", res.SuggestedKVCeiling)
}
}
if last := res.Steps[len(res.Steps)-1]; !last.MeetsSLO {
fmt.Fprintf(out, "\n%d concurrent was the first level to miss it (TTFT p99 %s).\n",
last.Concurrency, ms(last.TTFT.P99))
} else {
fmt.Fprintf(out, "\nEvery level tried met the SLO, so %d is a floor rather than a\n",
res.MaxConcurrencyAtSLO)
fmt.Fprintln(out, "ceiling — raise --concurrency to find where it actually breaks.")
}
if cfg.SharedPrefix {
fmt.Fprintln(out, "\nRun with --shared-prefix: this measures the engine's prefix cache,")
fmt.Fprintln(out, "not its hardware. Do not use these numbers as capacity.")
}
}
func ms(d time.Duration) string {
switch {
case d == 0:
return "-"
case d < time.Millisecond:
return d.Round(10 * time.Microsecond).String()
case d < time.Second:
return d.Round(time.Millisecond).String()
default:
return d.Round(10 * time.Millisecond).String()
}
}
+1
View File
@@ -108,6 +108,7 @@ func newCtlCmd() *cobra.Command {
"Move work off a node, one replica at a time"),
// cert works on files rather than the admin API, so --server does not
// apply to it; it lives here because this is where operators look.
newBenchCmd(&asJSON),
newCertCmd(),
)
return cmd
+78 -4
View File
@@ -203,6 +203,7 @@ func (r *Reconciler) Notify() {
// instead of racing a ticker.
func (r *Reconciler) Reconcile(ctx context.Context) {
r.expireNodes()
r.expireStuckReplicas()
r.drainCordonedNodes(ctx)
r.reapFailed(ctx)
@@ -211,6 +212,57 @@ func (r *Reconciler) Reconcile(ctx context.Context) {
}
}
// expireStuckReplicas fails replicas that never finished coming up.
//
// Every other path out of Starting depends on the agent reporting something. A
// replica whose agent never reports — the start was a duplicate shard id and
// became a no-op, the node went quiet in a way expiry did not catch, the engine
// wedged before its first health check — sits in Starting for as long as the
// process lives, holding its GPUs and counting towards the model's replica
// count while serving nothing.
//
// Found on a real cluster: two replicas sat in Starting for eight minutes past
// a two-minute startup timeout, and nothing would ever have moved them.
func (r *Reconciler) expireStuckReplicas() {
now := r.now()
var stuck []*Replica
r.mu.Lock()
for _, rep := range r.replicas {
if rep.State != Starting && rep.State != Placing {
continue
}
if now.Sub(rep.CreatedAt) < r.startupTimeout(rep) {
continue
}
rep.State = Failed
rep.Err = fmt.Sprintf("did not come up within %s", r.startupTimeout(rep))
stuck = append(stuck, rep)
}
r.mu.Unlock()
for _, rep := range stuck {
r.log.Warn("replica did not come up in time; failing it so it can be replaced",
"replica", rep.ID, "model", rep.Model, "state", "starting",
"timeout", r.startupTimeout(rep))
}
}
// startupTimeout is the profile's, or the configured default when the profile
// does not say — a replica with no deadline at all is the thing this exists to
// prevent.
func (r *Reconciler) startupTimeout(rep *Replica) time.Duration {
if m, ok := r.cat.Resolve(rep.Model); ok {
if p, ok := m.ProfileByName(rep.Profile); ok && p.StartupTimeout > 0 {
return p.StartupTimeout
}
}
// No profile deadline: a generous default, because the thing being prevented
// is a replica with no deadline at all rather than one with a loose one.
// Loading a large model across nodes genuinely takes tens of minutes.
return 30 * time.Minute
}
// expireNodes moves silent nodes through Unreachable to Lost, and fails the
// replicas of nodes that are gone.
//
@@ -624,10 +676,32 @@ func (r *Reconciler) Adopt(shards []AdoptedShard) (unclaimed []AdoptedShard) {
defer r.mu.Unlock()
for replicaID, group := range byReplica {
if _, known := r.replicas[replicaID]; known {
// Already tracked — an agent reconnecting rather than a control
// plane restart. Its status reports keep the state current.
continue
if existing, known := r.replicas[replicaID]; known {
// Usually this is an agent reconnecting rather than a control plane
// restart, and its status reports keep the state current.
//
// Not always. Replica ids are derived from the model name and a
// counter, so a control plane that restarts and places before the
// agents reconnect produces a *different* replica under the same
// id, on a freshly allocated port. Skipping the adoption then
// leaves the cluster pointing at a port nothing listens on, while
// the engine that is actually serving is treated as a stray.
//
// The fleet is authoritative about what is running (plan §6.5), so
// what we merely placed loses to what an agent reports — unless our
// copy has a shard that has actually come up, in which case two
// things really are running and the one we did not ask for is the
// stray.
if existing.hasLiveShard() {
unclaimed = append(unclaimed, group...)
continue
}
r.log.Warn("a running replica reported by the fleet replaces one we placed",
"replica", replicaID, "placed_endpoint", existing.Endpoint)
for _, s := range existing.Shards {
r.inv.Release(s.ID)
}
delete(r.replicas, replicaID)
}
first := group[0]
+15
View File
@@ -117,6 +117,21 @@ func (r Replica) OnNode(id string) bool {
// whole thing useless, because the others are waiting on collectives that will
// never complete. That is why there is no path here that keeps a replica
// partially alive (plan §5.4).
// hasLiveShard reports whether any shard has actually got somewhere.
//
// Starting and Pending are not evidence of anything: they are what a replica
// looks like both while an engine loads a model and when nothing was ever
// started at all. Loading means the port is open, which only a real process
// does.
func (r *Replica) hasLiveShard() bool {
for _, s := range r.Shards {
if s.State == supervisor.Ready || s.State == supervisor.Loading {
return true
}
}
return false
}
func (r *Replica) deriveState() {
if r.State == Draining || r.State == Stopped {
return
+180
View File
@@ -0,0 +1,180 @@
package reconciler
import (
"context"
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
)
// TestAReplicaThatNeverComesUpIsEventuallyFailed pins down the hole found on a
// real cluster: two replicas sat in Starting for eight minutes past a two-minute
// startup timeout, holding GPUs and counting towards the model's replica count
// while serving nothing.
//
// Every other way out of Starting needs the agent to report something. When it
// never does, this is the only thing that moves.
func TestAReplicaThatNeverComesUpIsEventuallyFailed(t *testing.T) {
m := testModel(1, catalog.Profile{
Name: "tp4", GPUClass: "h100-80g", TP: 4, StartupTimeout: 2 * time.Minute,
})
h := newHarness(t, m, gpuNode("1", 8))
ctx := context.Background()
h.r.Reconcile(ctx)
reps := h.r.Replicas()
if len(reps) != 1 || reps[0].State != Starting {
t.Fatalf("expected one starting replica, got %+v", reps)
}
// The agent says nothing at all, which is exactly the case.
h.advance(90 * time.Second)
h.r.Reconcile(ctx)
if got := h.r.Replicas()[0].State; got != Starting {
t.Fatalf("failed early, at %s: %s", 90*time.Second, got)
}
h.advance(60 * time.Second)
h.r.Reconcile(ctx)
// Failing it is not the outcome that matters; being unstuck is. The same
// reconcile reaps the failure and places a replacement, which is the whole
// point of failing it.
reps = h.r.Replicas()
for _, rep := range reps {
if rep.ID == "llama-1" && rep.State == Starting {
t.Fatal("the replica is still stuck in starting past its deadline")
}
}
// The replacement waits out the model's backoff, which is the point of the
// backoff: a model that cannot start should not be restarted in a loop.
h.advance(10 * time.Minute)
h.r.Reconcile(ctx)
reps = h.r.Replicas()
var replaced bool
for _, rep := range reps {
if rep.ID != "llama-1" {
replaced = true
}
}
if !replaced {
t.Errorf("no replacement was ever placed; replicas are %+v", reps)
}
// And the shards of the stuck one were told to stop, or its GPUs stay
// allocated to a replica nobody is waiting for any more.
h.disp.mu.Lock()
stopped := len(h.disp.stopped)
h.disp.mu.Unlock()
if stopped == 0 {
t.Error("the stuck replica's shards were never stopped")
}
}
func TestTheStuckReplicaSaysWhyItFailed(t *testing.T) {
// Reaping removes the evidence, so the reason has to be recorded when the
// deadline passes rather than found afterwards.
m := testModel(1, catalog.Profile{
Name: "tp4", GPUClass: "h100-80g", TP: 4, StartupTimeout: time.Minute,
})
h := newHarness(t, m, gpuNode("1", 8))
h.r.Reconcile(context.Background())
h.advance(2 * time.Minute)
h.r.expireStuckReplicas()
reps := h.r.Replicas()
if len(reps) != 1 || reps[0].State != Failed {
t.Fatalf("replicas: %+v", reps)
}
if !strings.Contains(reps[0].Err, "did not come up") {
t.Errorf("the error does not say what happened: %q", reps[0].Err)
}
}
func TestAReplicaThatIsLoadingIsNotFailedEarly(t *testing.T) {
// Loading a large model legitimately takes a long time. The deadline is
// what the profile says, and nothing shorter.
m := testModel(1, catalog.Profile{
Name: "tp4", GPUClass: "h100-80g", TP: 4, StartupTimeout: time.Hour,
})
h := newHarness(t, m, gpuNode("1", 8))
ctx := context.Background()
h.r.Reconcile(ctx)
h.advance(30 * time.Minute)
h.r.Reconcile(ctx)
if got := h.r.Replicas()[0].State; got == Failed {
t.Error("a replica still inside its startup timeout was failed")
}
}
// TestTheFleetWinsOverAReplicaWeOnlyPlaced is the other half of the same
// incident. Replica ids come from the model name and a counter, so a control
// plane that restarts and places before its agents reconnect builds a different
// replica under the same id, on a freshly allocated port. Dropping the agent's
// report then leaves the cluster pointing at a port nothing listens on while the
// engine that is actually serving is treated as a stray.
func TestTheFleetWinsOverAReplicaWeOnlyPlaced(t *testing.T) {
h := newHarness(t, testModel(1), gpuNode("1", 8))
ctx := context.Background()
h.r.Reconcile(ctx)
placed := h.r.Replicas()[0]
if placed.State != Starting {
t.Fatalf("state = %s", placed.State)
}
// An agent reports the same replica id, running on the port it was actually
// started with.
unclaimed := h.r.Adopt([]AdoptedShard{{
NodeID: "1", ReplicaID: placed.ID, ShardID: placed.ID + "-r0",
Rank: 0, GPUIndices: []int{0, 1, 2, 3}, HTTPPort: 20000,
State: supervisor.Ready, Model: "llama", Profile: "tp4",
Endpoint: "http://node1:20000",
}})
if len(unclaimed) != 0 {
t.Fatalf("the running engine was disowned: %+v", unclaimed)
}
reps := h.r.Replicas()
if len(reps) != 1 {
t.Fatalf("replicas: %+v", reps)
}
if reps[0].Endpoint != "http://node1:20000" {
t.Errorf("endpoint = %q; the cluster is still pointing at what we placed",
reps[0].Endpoint)
}
if reps[0].State != Ready {
t.Errorf("state = %s, want ready", reps[0].State)
}
}
func TestAnAdoptedShardDoesNotDisplaceOneThatIsActuallyRunning(t *testing.T) {
// The other direction. If our copy has a shard that has come up, two things
// really are running and the one we did not ask for is the stray — the
// agent is told to stop it rather than the live one being replaced.
h := newHarness(t, testModel(1), gpuNode("1", 8))
ctx := context.Background()
h.r.Reconcile(ctx)
placed := h.r.Replicas()[0]
h.r.OnShardStatus(placed.ID, placed.Shards[0].ID, supervisor.Ready, "")
unclaimed := h.r.Adopt([]AdoptedShard{{
NodeID: "1", ReplicaID: placed.ID, ShardID: "some-other-shard",
Rank: 0, HTTPPort: 29999, State: supervisor.Ready,
Model: "llama", Profile: "tp4", Endpoint: "http://node1:29999",
}})
if len(unclaimed) != 1 {
t.Fatalf("the stray was not disowned: %+v", unclaimed)
}
if got := h.r.Replicas()[0].Endpoint; got == "http://node1:29999" {
t.Error("a stray replaced the replica that was actually serving")
}
}