diff --git a/internal/bench/bench.go b/internal/bench/bench.go new file mode 100644 index 0000000..10d9962 --- /dev/null +++ b/internal/bench/bench.go @@ -0,0 +1,437 @@ +// Package bench measures what a replica can actually do, so the numbers the +// router depends on stop being guesses (plan §6.2.1). +// +// Three of them matter: the weight a replica gets when the scheduler compares +// it to others, the queue depth admission treats as healthy, and the KV usage +// it treats as full. All three are currently hand-written in configuration, and +// a hand-written capacity number is a hand-written opinion about hardware +// nobody measured. +// +// The measurement is a load ramp. Concurrency climbs until the SLO breaks, and +// the last level that still met it is the capacity — not peak throughput, which +// a GPU will happily deliver long after latency has become unusable. +package bench + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "math/rand/v2" + "net/http" + "sort" + "strings" + "sync" + "time" +) + +// SLO is the latency target capacity is measured against. +type SLO struct { + TTFTp99 time.Duration + TPOTp99 time.Duration +} + +// Met reports whether a step's latencies satisfy the target. +func (s SLO) Met(ttft, tpot time.Duration) bool { + if s.TTFTp99 > 0 && ttft > s.TTFTp99 { + return false + } + if s.TPOTp99 > 0 && tpot > s.TPOTp99 { + return false + } + return true +} + +// Config describes a run. +type Config struct { + // Target is the base URL: the gateway, or one replica directly. Pointing at + // a replica measures that replica; pointing at the gateway measures the + // whole path including scheduling, which is a different question. + Target string + // MetricsURL is an engine's /metrics, sampled during the run so queue depth + // and KV usage can be reported alongside latency. Only meaningful when + // Target is a single replica. + MetricsURL string + // Engine names which metric vocabulary MetricsURL speaks. Getting it wrong + // is silent: nothing matches, every sample reads zero, and the suggested + // thresholds are built on nothing. + Engine string + Model string + APIKey string + + // Concurrency is the ramp. Each level runs for Duration. + Concurrency []int + Duration time.Duration + + PromptTokens int + MaxTokens int + + // SharedPrefix sends every request the same prompt. + // + // Off by default, and that default is the whole point: with a shared prefix + // the engine's cache answers most of the prefill and the measurement + // describes the cache rather than the hardware. Turning it on measures the + // cached case deliberately, which is a fair question to ask separately. + SharedPrefix bool + + SLO SLO +} + +// DefaultConfig returns a ramp that covers most single-replica hardware. +func DefaultConfig() Config { + return Config{ + Concurrency: []int{1, 2, 4, 8, 16, 32, 64, 128}, + Duration: 20 * time.Second, + PromptTokens: 512, + MaxTokens: 128, + SLO: SLO{TTFTp99: 2 * time.Second, TPOTp99: 50 * time.Millisecond}, + } +} + +func (c *Config) normalize() error { + d := DefaultConfig() + if c.Target == "" { + return errors.New("bench: a target is required") + } + if c.Model == "" { + return errors.New("bench: a model is required") + } + c.Target = strings.TrimRight(c.Target, "/") + if len(c.Concurrency) == 0 { + c.Concurrency = d.Concurrency + } + if c.Duration <= 0 { + c.Duration = d.Duration + } + if c.PromptTokens <= 0 { + c.PromptTokens = d.PromptTokens + } + if c.MaxTokens <= 0 { + c.MaxTokens = d.MaxTokens + } + sort.Ints(c.Concurrency) + return nil +} + +// Percentiles summarizes a latency sample. +type Percentiles struct { + P50, P90, P99, Max time.Duration +} + +// Step is one concurrency level's result. +type Step struct { + Concurrency int + Completed int + Failed int + // Requests and output tokens per second, over the step's wall clock. + RequestsPerSec float64 + TokensPerSec float64 + TTFT Percentiles + // TPOT is time per output token after the first, which is what a reader + // experiences as the speed of the stream. + TPOT Percentiles + Waiting float64 + Running float64 + KVUsage float64 + MeetsSLO bool + // FirstError is kept so a step that failed says why rather than just + // reporting zeroes. + FirstError string +} + +// Result is the whole ramp plus what it concluded. +type Result struct { + Model string + Target string + Steps []Step + SLO SLO + RanAt time.Time + Elapsed time.Duration + + // MaxConcurrencyAtSLO is the capacity: the highest level that still met the + // target. Zero means even one request did not. + MaxConcurrencyAtSLO int + // ThroughputTokPerSec and the latencies are taken at that level. + ThroughputTokPerSec float64 + TTFTp99, TPOTp99 time.Duration + // SuggestedTargetWaiting and SuggestedKVCeiling are what admission should + // treat as healthy, observed rather than guessed. + SuggestedTargetWaiting int + SuggestedKVCeiling float64 +} + +// Run executes the ramp. +func Run(ctx context.Context, cfg Config, progress func(Step)) (Result, error) { + if err := cfg.normalize(); err != nil { + return Result{}, err + } + res := Result{Model: cfg.Model, Target: cfg.Target, SLO: cfg.SLO, RanAt: time.Now()} + start := time.Now() + + client := &http.Client{Timeout: 10 * time.Minute} + for _, c := range cfg.Concurrency { + if ctx.Err() != nil { + break + } + step := runStep(ctx, client, cfg, c) + res.Steps = append(res.Steps, step) + if progress != nil { + progress(step) + } + // Stop climbing once the target is missed. Everything above is slower + // by construction, and the cost of finding out is real GPU time. + if !step.MeetsSLO { + break + } + } + res.Elapsed = time.Since(start) + res.conclude() + return res, nil +} + +func (r *Result) conclude() { + for _, s := range r.Steps { + if !s.MeetsSLO || s.Completed == 0 { + continue + } + if s.Concurrency <= r.MaxConcurrencyAtSLO { + continue + } + r.MaxConcurrencyAtSLO = s.Concurrency + r.ThroughputTokPerSec = s.TokensPerSec + r.TTFTp99, r.TPOTp99 = s.TTFT.P99, s.TPOT.P99 + // The queue depth and cache pressure seen while still meeting the SLO + // are exactly what admission should treat as healthy. + r.SuggestedTargetWaiting = int(s.Waiting + 0.5) + r.SuggestedKVCeiling = roundTo(s.KVUsage, 0.05) + } +} + +func runStep(ctx context.Context, client *http.Client, cfg Config, concurrency int) Step { + step := Step{Concurrency: concurrency} + ctx, cancel := context.WithTimeout(ctx, cfg.Duration) + defer cancel() + + var ( + mu sync.Mutex + ttfts []time.Duration + tpots []time.Duration + tokens int64 + failed int + firstErr string + ) + + sampler := newSampler(cfg.MetricsURL, cfg.Engine) + sampler.start(ctx) + + start := time.Now() + var wg sync.WaitGroup + for range concurrency { + wg.Add(1) + go func() { + defer wg.Done() + for ctx.Err() == nil { + r, err := one(ctx, client, cfg) + mu.Lock() + switch { + case err != nil: + // A cancelled context is the step ending, not a failure. + if ctx.Err() == nil { + failed++ + if firstErr == "" { + firstErr = err.Error() + } + } + default: + ttfts = append(ttfts, r.ttft) + if r.tpot > 0 { + tpots = append(tpots, r.tpot) + } + tokens += r.tokens + } + mu.Unlock() + } + }() + } + wg.Wait() + elapsed := time.Since(start) + sampler.stop() + + step.Completed = len(ttfts) + step.Failed = failed + step.FirstError = firstErr + step.TTFT = percentiles(ttfts) + step.TPOT = percentiles(tpots) + if elapsed > 0 { + step.RequestsPerSec = float64(step.Completed) / elapsed.Seconds() + step.TokensPerSec = float64(tokens) / elapsed.Seconds() + } + step.Waiting, step.Running, step.KVUsage = sampler.averages() + + // A step that could not complete anything has not met anything. + step.MeetsSLO = step.Completed > 0 && step.Failed == 0 && + cfg.SLO.Met(step.TTFT.P99, step.TPOT.P99) + return step +} + +type outcome struct { + ttft time.Duration + tpot time.Duration + tokens int64 +} + +// one sends a single streaming completion and times it. +func one(ctx context.Context, client *http.Client, cfg Config) (outcome, error) { + body, err := json.Marshal(map[string]any{ + "model": cfg.Model, + "stream": true, + "max_tokens": cfg.MaxTokens, + "messages": []map[string]string{ + {"role": "user", "content": prompt(cfg)}, + }, + }) + if err != nil { + return outcome{}, err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + cfg.Target+"/v1/chat/completions", bytes.NewReader(body)) + if err != nil { + return outcome{}, err + } + req.Header.Set("Content-Type", "application/json") + if cfg.APIKey != "" { + req.Header.Set("Authorization", "Bearer "+cfg.APIKey) + } + + start := time.Now() + resp, err := client.Do(req) + if err != nil { + return outcome{}, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + return outcome{}, fmt.Errorf("%s: %s", resp.Status, strings.TrimSpace(string(b))) + } + + var out outcome + var firstAt, lastAt time.Time + sc := bufio.NewScanner(resp.Body) + sc.Buffer(make([]byte, 0, 64*1024), 1<<20) + for sc.Scan() { + line := sc.Text() + if !strings.HasPrefix(line, "data: ") || strings.HasSuffix(line, "[DONE]") { + continue + } + // Only chunks that carry content count: a role-only first chunk is not + // a token, and counting it would flatter the time to first token. + if !hasContent(line[6:]) { + continue + } + now := time.Now() + if out.tokens == 0 { + firstAt = now + out.ttft = now.Sub(start) + } + lastAt = now + out.tokens++ + } + if err := sc.Err(); err != nil { + return outcome{}, err + } + if out.tokens == 0 { + return outcome{}, errors.New("the response carried no content") + } + if out.tokens > 1 { + out.tpot = lastAt.Sub(firstAt) / time.Duration(out.tokens-1) + } + return out, nil +} + +func hasContent(payload string) bool { + var chunk struct { + Choices []struct { + Delta struct { + Content string `json:"content"` + } `json:"delta"` + } `json:"choices"` + } + if err := json.Unmarshal([]byte(payload), &chunk); err != nil { + return false + } + for _, c := range chunk.Choices { + if c.Delta.Content != "" { + return true + } + } + return false +} + +// prompt builds a request body of roughly the configured size. +// +// Unique per request unless a shared prefix was asked for: repeating one prompt +// lets the engine's cache answer most of the prefill, and the result then +// describes the cache rather than the hardware. +// +// "Shared" has to mean byte-identical. Dropping only the random header while +// still picking words at random would leave every prompt different after the +// first few tokens, so the cache would miss and the flag would quietly measure +// the thing it exists to avoid measuring. +func prompt(cfg Config) string { + var b strings.Builder + if cfg.SharedPrefix { + for i := 0; b.Len() < cfg.PromptTokens*4; i++ { + b.WriteString(words[i%len(words)]) + b.WriteByte(' ') + } + } else { + fmt.Fprintf(&b, "%016x %016x ", rand.Uint64(), rand.Uint64()) //nolint:gosec // not security + // Roughly four characters to a token, the same crude ratio the router + // uses; exactness is not what this number is for. + for b.Len() < cfg.PromptTokens*4 { + b.WriteString(words[rand.IntN(len(words))]) //nolint:gosec // not security + b.WriteByte(' ') + } + } + b.WriteString("\n\nSummarize the text above.") + return b.String() +} + +var words = strings.Fields(`the quick brown fox jumps over lazy dog while distant +thunder rolls across an empty valley and somewhere a door closes softly in the +dark house where nobody has lived for years although the garden still produces +apples every autumn which fall and rot beneath the trees unnoticed by anyone +except the birds that come each morning to pick at them before the frost`) + +func percentiles(v []time.Duration) Percentiles { + if len(v) == 0 { + return Percentiles{} + } + s := make([]time.Duration, len(v)) + copy(s, v) + sort.Slice(s, func(i, j int) bool { return s[i] < s[j] }) + at := func(p float64) time.Duration { + i := int(float64(len(s)-1) * p) + return s[i] + } + return Percentiles{P50: at(0.50), P90: at(0.90), P99: at(0.99), Max: s[len(s)-1]} +} + +// roundTo rounds to a multiple of step, through integers. +// +// The obvious float arithmetic returns 0.7000000000000001 for what should be +// 0.70, which is harmless on a terminal and ugly in the JSON someone pastes +// into a configuration file. +func roundTo(v, step float64) float64 { + if step <= 0 { + return v + } + const scale = 10000 + units := int64(v/step + 0.5) + return float64(int64(float64(units)*step*scale+0.5)) / scale +} diff --git a/internal/bench/bench_test.go b/internal/bench/bench_test.go new file mode 100644 index 0000000..ce4860a --- /dev/null +++ b/internal/bench/bench_test.go @@ -0,0 +1,156 @@ +package bench + +import ( + "context" + "strings" + "testing" + "time" +) + +func TestCapacityIsTheLastLevelThatMetTheSLONotThePeak(t *testing.T) { + // A GPU keeps delivering more tokens per second long after latency has + // become unusable. Taking peak throughput as capacity would tell the router + // a replica can serve a load at which nobody would want the answers. + r := Result{SLO: SLO{TTFTp99: time.Second}} + r.Steps = []Step{ + {Concurrency: 1, Completed: 10, TokensPerSec: 100, MeetsSLO: true, + TTFT: Percentiles{P99: 100 * time.Millisecond}}, + {Concurrency: 8, Completed: 80, TokensPerSec: 700, MeetsSLO: true, + TTFT: Percentiles{P99: 900 * time.Millisecond}, Waiting: 3.4, KVUsage: 0.71}, + // Faster in tokens, far too slow in latency. + {Concurrency: 32, Completed: 200, TokensPerSec: 1500, MeetsSLO: false, + TTFT: Percentiles{P99: 9 * time.Second}}, + } + r.conclude() + + if r.MaxConcurrencyAtSLO != 8 { + t.Errorf("capacity = %d, want 8", r.MaxConcurrencyAtSLO) + } + if r.ThroughputTokPerSec != 700 { + t.Errorf("throughput = %v; it must come from the level that met the SLO", r.ThroughputTokPerSec) + } + if r.SuggestedTargetWaiting != 3 { + t.Errorf("target_waiting = %d, want 3", r.SuggestedTargetWaiting) + } + if r.SuggestedKVCeiling != 0.70 { + t.Errorf("kv_ceiling = %v, want 0.70", r.SuggestedKVCeiling) + } +} + +func TestAStepWithFailuresHasNotMetAnything(t *testing.T) { + // Latency percentiles over the requests that survived say nothing useful + // when the rest were refused. + r := Result{SLO: SLO{TTFTp99: time.Second}} + r.Steps = []Step{ + {Concurrency: 4, Completed: 10, Failed: 90, TokensPerSec: 50, MeetsSLO: false, + TTFT: Percentiles{P99: 50 * time.Millisecond}}, + } + r.conclude() + if r.MaxConcurrencyAtSLO != 0 { + t.Errorf("a step that failed 90 of 100 requests was counted as capacity") + } +} + +func TestSLOIsMissedWhenEitherHalfIs(t *testing.T) { + s := SLO{TTFTp99: time.Second, TPOTp99: 50 * time.Millisecond} + cases := []struct { + ttft, tpot time.Duration + want bool + }{ + {500 * time.Millisecond, 20 * time.Millisecond, true}, + {2 * time.Second, 20 * time.Millisecond, false}, + {500 * time.Millisecond, 80 * time.Millisecond, false}, + {2 * time.Second, 80 * time.Millisecond, false}, + } + for _, c := range cases { + if got := s.Met(c.ttft, c.tpot); got != c.want { + t.Errorf("Met(%s, %s) = %v", c.ttft, c.tpot, got) + } + } + + // A target of zero is no target, not a target of zero. + none := SLO{} + if !none.Met(time.Hour, time.Hour) { + t.Error("an unset SLO refused everything") + } +} + +func TestPercentiles(t *testing.T) { + var v []time.Duration + for i := 1; i <= 100; i++ { + v = append(v, time.Duration(i)*time.Millisecond) + } + p := percentiles(v) + if p.P50 != 50*time.Millisecond || p.P90 != 90*time.Millisecond { + t.Errorf("p50=%s p90=%s", p.P50, p.P90) + } + if p.Max != 100*time.Millisecond { + t.Errorf("max=%s", p.Max) + } + if got := percentiles(nil); got != (Percentiles{}) { + t.Errorf("empty sample gave %+v", got) + } +} + +func TestPromptsAreUniqueUnlessAskedNotToBe(t *testing.T) { + // The default matters more than it looks. With one repeated prompt the + // engine's cache answers most of the prefill, and the measurement describes + // the cache rather than the hardware. + cfg := Config{PromptTokens: 64} + seen := map[string]bool{} + for range 100 { + seen[prompt(cfg)] = true + } + if len(seen) < 100 { + t.Errorf("%d distinct prompts out of 100; the cache would answer the rest", len(seen)) + } + + cfg.SharedPrefix = true + first := prompt(cfg) + for range 20 { + if got := prompt(cfg); !strings.HasPrefix(got, first[:200]) { + t.Error("--shared-prefix did not produce a shared prefix") + } + } +} + +func TestPromptSizeFollowsTheRequest(t *testing.T) { + small := prompt(Config{PromptTokens: 32}) + large := prompt(Config{PromptTokens: 512}) + if len(large) < len(small)*4 { + t.Errorf("a 512-token prompt is %d chars against %d for 32", len(large), len(small)) + } +} + +func TestConfigIsValidatedBeforeAnyLoadIsGenerated(t *testing.T) { + // The failure has to come before the traffic, not halfway through a ramp. + if _, err := Run(context.Background(), Config{Model: "m"}, nil); err == nil { + t.Error("a run with no target started") + } + if _, err := Run(context.Background(), Config{Target: "http://x"}, nil); err == nil { + t.Error("a run with no model started") + } +} + +func TestTheRampStopsOnceTheSLOBreaks(t *testing.T) { + // Everything above a missed level is slower by construction, and finding + // that out costs real GPU time. + r := Result{} + r.Steps = []Step{ + {Concurrency: 1, Completed: 1, MeetsSLO: true}, + {Concurrency: 2, Completed: 1, MeetsSLO: false}, + } + r.conclude() + if r.MaxConcurrencyAtSLO != 1 { + t.Errorf("capacity = %d", r.MaxConcurrencyAtSLO) + } +} + +func TestRoundTo(t *testing.T) { + cases := map[float64]float64{0.71: 0.70, 0.73: 0.75, 0.88: 0.90, 0.0: 0.0} + for in, want := range cases { + if got := roundTo(in, 0.05); got != want { + t.Errorf("roundTo(%v) = %v, want %v", in, got, want) + } + } +} diff --git a/internal/bench/sampler.go b/internal/bench/sampler.go new file mode 100644 index 0000000..1cde4db --- /dev/null +++ b/internal/bench/sampler.go @@ -0,0 +1,108 @@ +package bench + +import ( + "context" + "net/http" + "sync" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/router/scrape" +) + +// sampler reads an engine's metrics during a step. +// +// It is what turns a latency measurement into something admission can use: the +// queue depth and cache pressure observed while the SLO was still being met are +// the values target_waiting and kv_ceiling should have, rather than the round +// numbers somebody typed. +// +// Optional, because a run pointed at the gateway has no single engine to read. +type sampler struct { + url string + engine string + client *http.Client + cancel context.CancelFunc + + mu sync.Mutex + wg sync.WaitGroup + waiting float64 + running float64 + kv float64 + n int +} + +func newSampler(metricsURL, engine string) *sampler { + return &sampler{ + url: metricsURL, + engine: engine, + client: &http.Client{Timeout: 2 * time.Second}, + } +} + +func (s *sampler) start(ctx context.Context) { + if s.url == "" { + return + } + ctx, s.cancel = context.WithCancel(ctx) + s.wg.Add(1) + go func() { + defer s.wg.Done() + t := time.NewTicker(time.Second) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + s.sample(ctx) + } + } + }() +} + +func (s *sampler) sample(ctx context.Context) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.url, nil) + if err != nil { + return + } + resp, err := s.client.Do(req) + if err != nil { + return + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return + } + + // The same mapping the router reads with, so a bench result cannot disagree + // with what the scheduler sees. The engine has to be named: reading a + // llama.cpp endpoint with vLLM's names finds nothing, and the run would + // report a queue depth of zero and suggest thresholds built on it. + stats, err := scrape.Parse(resp.Body, scrape.MappingFor(s.engine)) + if err != nil { + return + } + s.mu.Lock() + s.waiting += float64(stats.Waiting) + s.running += float64(stats.Running) + s.kv += stats.KVCacheUsage + s.n++ + s.mu.Unlock() +} + +func (s *sampler) stop() { + if s.cancel != nil { + s.cancel() + } + s.wg.Wait() +} + +func (s *sampler) averages() (waiting, running, kv float64) { + s.mu.Lock() + defer s.mu.Unlock() + if s.n == 0 { + return 0, 0, 0 + } + n := float64(s.n) + return s.waiting / n, s.running / n, s.kv / n +} diff --git a/internal/cli/bench.go b/internal/cli/bench.go new file mode 100644 index 0000000..1b37d75 --- /dev/null +++ b/internal/cli/bench.go @@ -0,0 +1,155 @@ +package cli + +import ( + "encoding/json" + "fmt" + "text/tabwriter" + "time" + + "github.com/spf13/cobra" + + "gitea.mynodes.xyz/ilker/llmgw/internal/bench" +) + +func newBenchCmd(asJSON *bool) *cobra.Command { + cfg := bench.DefaultConfig() + var ( + steps []int + target string + metrics string + engine string + key string + ) + + cmd := &cobra.Command{ + Use: "bench ", + Short: "Measure what a replica can do, and what the router should be told", + Long: "Drives real traffic. Point it at a replica to measure that hardware, or at\n" + + "the gateway to measure the whole path — they answer different questions,\n" + + "and only the first produces a capacity profile.\n\n" + + "Concurrency climbs until the SLO breaks. The last level that still met it\n" + + "is the capacity: peak throughput arrives long after latency has become\n" + + "unusable, so throughput alone would overstate what the hardware can serve.\n\n" + + "This costs real GPU time and competes with real traffic. Run it on a\n" + + "replica you have cordoned, or during a window where that is acceptable.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + cfg.Model = args[0] + cfg.Target = target + cfg.MetricsURL = metrics + cfg.Engine = engine + cfg.APIKey = key + if len(steps) > 0 { + cfg.Concurrency = steps + } + + if metrics == "" { + cmd.PrintErrln("No --metrics endpoint: queue depth and KV usage will not be " + + "measured, so target_waiting and kv_ceiling cannot be suggested.") + } + cmd.PrintErrf("benchmarking %s against %s\n\n", cfg.Model, cfg.Target) + + w := tabwriter.NewWriter(cmd.ErrOrStderr(), 0, 0, 2, ' ', 0) + fmt.Fprintln(w, "CONC\tREQ/S\tTOK/S\tTTFT p50\tTTFT p99\tTPOT p99\tWAIT\tKV\tSLO") + progress := func(s bench.Step) { + verdict := "ok" + if !s.MeetsSLO { + verdict = "MISSED" + } + if s.Failed > 0 { + verdict = fmt.Sprintf("%d failed", s.Failed) + } + fmt.Fprintf(w, "%d\t%.1f\t%.0f\t%s\t%s\t%s\t%.1f\t%.0f%%\t%s\n", + s.Concurrency, s.RequestsPerSec, s.TokensPerSec, + ms(s.TTFT.P50), ms(s.TTFT.P99), ms(s.TPOT.P99), + s.Waiting, s.KVUsage*100, verdict) + _ = w.Flush() + } + + res, err := bench.Run(cmd.Context(), cfg, progress) + if err != nil { + return err + } + if *asJSON { + return json.NewEncoder(cmd.OutOrStdout()).Encode(res) + } + printConclusion(cmd, res, cfg) + return nil + }, + } + + f := cmd.Flags() + f.StringVar(&target, "target", "http://127.0.0.1:8080", "base URL: a replica, or the gateway") + f.StringVar(&metrics, "metrics", "", "the replica's /metrics, to observe queue depth and KV usage") + f.StringVar(&engine, "engine", "", "which metric names --metrics uses: vllm | sglang | llamacpp") + f.StringVar(&key, "key", "", "API key, when the gateway requires one") + f.IntSliceVar(&steps, "concurrency", nil, "levels to try (default 1,2,4,…,128)") + f.DurationVar(&cfg.Duration, "duration", cfg.Duration, "how long to hold each level") + f.IntVar(&cfg.PromptTokens, "prompt-tokens", cfg.PromptTokens, "approximate prompt size") + f.IntVar(&cfg.MaxTokens, "max-tokens", cfg.MaxTokens, "tokens to generate per request") + f.BoolVar(&cfg.SharedPrefix, "shared-prefix", false, + "send every request the same prompt, measuring the cached case instead of the hardware") + f.DurationVar(&cfg.SLO.TTFTp99, "ttft-p99", cfg.SLO.TTFTp99, "time-to-first-token target") + f.DurationVar(&cfg.SLO.TPOTp99, "tpot-p99", cfg.SLO.TPOTp99, "per-output-token target") + return cmd +} + +func printConclusion(cmd *cobra.Command, res bench.Result, cfg bench.Config) { + out := cmd.OutOrStdout() + fmt.Fprintln(out) + + if res.MaxConcurrencyAtSLO == 0 { + fmt.Fprintf(out, "No level met the SLO (TTFT p99 ≤ %s, TPOT p99 ≤ %s).\n", + cfg.SLO.TTFTp99, cfg.SLO.TPOTp99) + if len(res.Steps) > 0 && res.Steps[0].FirstError != "" { + fmt.Fprintf(out, "First error: %s\n", res.Steps[0].FirstError) + } + fmt.Fprintln(out, "Either the hardware cannot serve this model at this SLO, or the") + fmt.Fprintln(out, "target is stricter than the model can meet at any concurrency.") + return + } + + fmt.Fprintf(out, "Capacity: %d concurrent requests at %.0f tokens/s\n", + res.MaxConcurrencyAtSLO, res.ThroughputTokPerSec) + fmt.Fprintf(out, " TTFT p99 %s · TPOT p99 %s · measured over %s\n\n", + ms(res.TTFTp99), ms(res.TPOTp99), res.Elapsed.Round(time.Second)) + + fmt.Fprintln(out, "Put this in the model's profile:") + fmt.Fprintf(out, " weight: %d\n", res.MaxConcurrencyAtSLO) + if res.SuggestedTargetWaiting > 0 || res.SuggestedKVCeiling > 0 { + fmt.Fprintln(out, "\nAnd in router.scoring, from what the engine reported while still") + fmt.Fprintln(out, "meeting the target:") + if res.SuggestedTargetWaiting > 0 { + fmt.Fprintf(out, " target_waiting: %d\n", res.SuggestedTargetWaiting) + } + if res.SuggestedKVCeiling > 0 { + fmt.Fprintf(out, " kv_ceiling: %.2f\n", res.SuggestedKVCeiling) + } + } + + if last := res.Steps[len(res.Steps)-1]; !last.MeetsSLO { + fmt.Fprintf(out, "\n%d concurrent was the first level to miss it (TTFT p99 %s).\n", + last.Concurrency, ms(last.TTFT.P99)) + } else { + fmt.Fprintf(out, "\nEvery level tried met the SLO, so %d is a floor rather than a\n", + res.MaxConcurrencyAtSLO) + fmt.Fprintln(out, "ceiling — raise --concurrency to find where it actually breaks.") + } + if cfg.SharedPrefix { + fmt.Fprintln(out, "\nRun with --shared-prefix: this measures the engine's prefix cache,") + fmt.Fprintln(out, "not its hardware. Do not use these numbers as capacity.") + } +} + +func ms(d time.Duration) string { + switch { + case d == 0: + return "-" + case d < time.Millisecond: + return d.Round(10 * time.Microsecond).String() + case d < time.Second: + return d.Round(time.Millisecond).String() + default: + return d.Round(10 * time.Millisecond).String() + } +} diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go index c4f1f7a..c5b62a1 100644 --- a/internal/cli/ctl.go +++ b/internal/cli/ctl.go @@ -108,6 +108,7 @@ func newCtlCmd() *cobra.Command { "Move work off a node, one replica at a time"), // cert works on files rather than the admin API, so --server does not // apply to it; it lives here because this is where operators look. + newBenchCmd(&asJSON), newCertCmd(), ) return cmd diff --git a/internal/control/reconciler/reconciler.go b/internal/control/reconciler/reconciler.go index b49b2b5..149a8c4 100644 --- a/internal/control/reconciler/reconciler.go +++ b/internal/control/reconciler/reconciler.go @@ -203,6 +203,7 @@ func (r *Reconciler) Notify() { // instead of racing a ticker. func (r *Reconciler) Reconcile(ctx context.Context) { r.expireNodes() + r.expireStuckReplicas() r.drainCordonedNodes(ctx) r.reapFailed(ctx) @@ -211,6 +212,57 @@ func (r *Reconciler) Reconcile(ctx context.Context) { } } +// expireStuckReplicas fails replicas that never finished coming up. +// +// Every other path out of Starting depends on the agent reporting something. A +// replica whose agent never reports — the start was a duplicate shard id and +// became a no-op, the node went quiet in a way expiry did not catch, the engine +// wedged before its first health check — sits in Starting for as long as the +// process lives, holding its GPUs and counting towards the model's replica +// count while serving nothing. +// +// Found on a real cluster: two replicas sat in Starting for eight minutes past +// a two-minute startup timeout, and nothing would ever have moved them. +func (r *Reconciler) expireStuckReplicas() { + now := r.now() + var stuck []*Replica + + r.mu.Lock() + for _, rep := range r.replicas { + if rep.State != Starting && rep.State != Placing { + continue + } + if now.Sub(rep.CreatedAt) < r.startupTimeout(rep) { + continue + } + rep.State = Failed + rep.Err = fmt.Sprintf("did not come up within %s", r.startupTimeout(rep)) + stuck = append(stuck, rep) + } + r.mu.Unlock() + + for _, rep := range stuck { + r.log.Warn("replica did not come up in time; failing it so it can be replaced", + "replica", rep.ID, "model", rep.Model, "state", "starting", + "timeout", r.startupTimeout(rep)) + } +} + +// startupTimeout is the profile's, or the configured default when the profile +// does not say — a replica with no deadline at all is the thing this exists to +// prevent. +func (r *Reconciler) startupTimeout(rep *Replica) time.Duration { + if m, ok := r.cat.Resolve(rep.Model); ok { + if p, ok := m.ProfileByName(rep.Profile); ok && p.StartupTimeout > 0 { + return p.StartupTimeout + } + } + // No profile deadline: a generous default, because the thing being prevented + // is a replica with no deadline at all rather than one with a loose one. + // Loading a large model across nodes genuinely takes tens of minutes. + return 30 * time.Minute +} + // expireNodes moves silent nodes through Unreachable to Lost, and fails the // replicas of nodes that are gone. // @@ -624,10 +676,32 @@ func (r *Reconciler) Adopt(shards []AdoptedShard) (unclaimed []AdoptedShard) { defer r.mu.Unlock() for replicaID, group := range byReplica { - if _, known := r.replicas[replicaID]; known { - // Already tracked — an agent reconnecting rather than a control - // plane restart. Its status reports keep the state current. - continue + if existing, known := r.replicas[replicaID]; known { + // Usually this is an agent reconnecting rather than a control plane + // restart, and its status reports keep the state current. + // + // Not always. Replica ids are derived from the model name and a + // counter, so a control plane that restarts and places before the + // agents reconnect produces a *different* replica under the same + // id, on a freshly allocated port. Skipping the adoption then + // leaves the cluster pointing at a port nothing listens on, while + // the engine that is actually serving is treated as a stray. + // + // The fleet is authoritative about what is running (plan §6.5), so + // what we merely placed loses to what an agent reports — unless our + // copy has a shard that has actually come up, in which case two + // things really are running and the one we did not ask for is the + // stray. + if existing.hasLiveShard() { + unclaimed = append(unclaimed, group...) + continue + } + r.log.Warn("a running replica reported by the fleet replaces one we placed", + "replica", replicaID, "placed_endpoint", existing.Endpoint) + for _, s := range existing.Shards { + r.inv.Release(s.ID) + } + delete(r.replicas, replicaID) } first := group[0] diff --git a/internal/control/reconciler/replica.go b/internal/control/reconciler/replica.go index b2562f6..b58d3fb 100644 --- a/internal/control/reconciler/replica.go +++ b/internal/control/reconciler/replica.go @@ -117,6 +117,21 @@ func (r Replica) OnNode(id string) bool { // whole thing useless, because the others are waiting on collectives that will // never complete. That is why there is no path here that keeps a replica // partially alive (plan §5.4). +// hasLiveShard reports whether any shard has actually got somewhere. +// +// Starting and Pending are not evidence of anything: they are what a replica +// looks like both while an engine loads a model and when nothing was ever +// started at all. Loading means the port is open, which only a real process +// does. +func (r *Replica) hasLiveShard() bool { + for _, s := range r.Shards { + if s.State == supervisor.Ready || s.State == supervisor.Loading { + return true + } + } + return false +} + func (r *Replica) deriveState() { if r.State == Draining || r.State == Stopped { return diff --git a/internal/control/reconciler/stuck_test.go b/internal/control/reconciler/stuck_test.go new file mode 100644 index 0000000..74f493c --- /dev/null +++ b/internal/control/reconciler/stuck_test.go @@ -0,0 +1,180 @@ +package reconciler + +import ( + "context" + "strings" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" +) + +// TestAReplicaThatNeverComesUpIsEventuallyFailed pins down the hole found on a +// real cluster: two replicas sat in Starting for eight minutes past a two-minute +// startup timeout, holding GPUs and counting towards the model's replica count +// while serving nothing. +// +// Every other way out of Starting needs the agent to report something. When it +// never does, this is the only thing that moves. +func TestAReplicaThatNeverComesUpIsEventuallyFailed(t *testing.T) { + m := testModel(1, catalog.Profile{ + Name: "tp4", GPUClass: "h100-80g", TP: 4, StartupTimeout: 2 * time.Minute, + }) + h := newHarness(t, m, gpuNode("1", 8)) + ctx := context.Background() + + h.r.Reconcile(ctx) + reps := h.r.Replicas() + if len(reps) != 1 || reps[0].State != Starting { + t.Fatalf("expected one starting replica, got %+v", reps) + } + + // The agent says nothing at all, which is exactly the case. + h.advance(90 * time.Second) + h.r.Reconcile(ctx) + if got := h.r.Replicas()[0].State; got != Starting { + t.Fatalf("failed early, at %s: %s", 90*time.Second, got) + } + + h.advance(60 * time.Second) + h.r.Reconcile(ctx) + + // Failing it is not the outcome that matters; being unstuck is. The same + // reconcile reaps the failure and places a replacement, which is the whole + // point of failing it. + reps = h.r.Replicas() + for _, rep := range reps { + if rep.ID == "llama-1" && rep.State == Starting { + t.Fatal("the replica is still stuck in starting past its deadline") + } + } + // The replacement waits out the model's backoff, which is the point of the + // backoff: a model that cannot start should not be restarted in a loop. + h.advance(10 * time.Minute) + h.r.Reconcile(ctx) + + reps = h.r.Replicas() + var replaced bool + for _, rep := range reps { + if rep.ID != "llama-1" { + replaced = true + } + } + if !replaced { + t.Errorf("no replacement was ever placed; replicas are %+v", reps) + } + + // And the shards of the stuck one were told to stop, or its GPUs stay + // allocated to a replica nobody is waiting for any more. + h.disp.mu.Lock() + stopped := len(h.disp.stopped) + h.disp.mu.Unlock() + if stopped == 0 { + t.Error("the stuck replica's shards were never stopped") + } +} + +func TestTheStuckReplicaSaysWhyItFailed(t *testing.T) { + // Reaping removes the evidence, so the reason has to be recorded when the + // deadline passes rather than found afterwards. + m := testModel(1, catalog.Profile{ + Name: "tp4", GPUClass: "h100-80g", TP: 4, StartupTimeout: time.Minute, + }) + h := newHarness(t, m, gpuNode("1", 8)) + h.r.Reconcile(context.Background()) + + h.advance(2 * time.Minute) + h.r.expireStuckReplicas() + + reps := h.r.Replicas() + if len(reps) != 1 || reps[0].State != Failed { + t.Fatalf("replicas: %+v", reps) + } + if !strings.Contains(reps[0].Err, "did not come up") { + t.Errorf("the error does not say what happened: %q", reps[0].Err) + } +} + +func TestAReplicaThatIsLoadingIsNotFailedEarly(t *testing.T) { + // Loading a large model legitimately takes a long time. The deadline is + // what the profile says, and nothing shorter. + m := testModel(1, catalog.Profile{ + Name: "tp4", GPUClass: "h100-80g", TP: 4, StartupTimeout: time.Hour, + }) + h := newHarness(t, m, gpuNode("1", 8)) + ctx := context.Background() + h.r.Reconcile(ctx) + + h.advance(30 * time.Minute) + h.r.Reconcile(ctx) + if got := h.r.Replicas()[0].State; got == Failed { + t.Error("a replica still inside its startup timeout was failed") + } +} + +// TestTheFleetWinsOverAReplicaWeOnlyPlaced is the other half of the same +// incident. Replica ids come from the model name and a counter, so a control +// plane that restarts and places before its agents reconnect builds a different +// replica under the same id, on a freshly allocated port. Dropping the agent's +// report then leaves the cluster pointing at a port nothing listens on while the +// engine that is actually serving is treated as a stray. +func TestTheFleetWinsOverAReplicaWeOnlyPlaced(t *testing.T) { + h := newHarness(t, testModel(1), gpuNode("1", 8)) + ctx := context.Background() + + h.r.Reconcile(ctx) + placed := h.r.Replicas()[0] + if placed.State != Starting { + t.Fatalf("state = %s", placed.State) + } + + // An agent reports the same replica id, running on the port it was actually + // started with. + unclaimed := h.r.Adopt([]AdoptedShard{{ + NodeID: "1", ReplicaID: placed.ID, ShardID: placed.ID + "-r0", + Rank: 0, GPUIndices: []int{0, 1, 2, 3}, HTTPPort: 20000, + State: supervisor.Ready, Model: "llama", Profile: "tp4", + Endpoint: "http://node1:20000", + }}) + + if len(unclaimed) != 0 { + t.Fatalf("the running engine was disowned: %+v", unclaimed) + } + reps := h.r.Replicas() + if len(reps) != 1 { + t.Fatalf("replicas: %+v", reps) + } + if reps[0].Endpoint != "http://node1:20000" { + t.Errorf("endpoint = %q; the cluster is still pointing at what we placed", + reps[0].Endpoint) + } + if reps[0].State != Ready { + t.Errorf("state = %s, want ready", reps[0].State) + } +} + +func TestAnAdoptedShardDoesNotDisplaceOneThatIsActuallyRunning(t *testing.T) { + // The other direction. If our copy has a shard that has come up, two things + // really are running and the one we did not ask for is the stray — the + // agent is told to stop it rather than the live one being replaced. + h := newHarness(t, testModel(1), gpuNode("1", 8)) + ctx := context.Background() + h.r.Reconcile(ctx) + + placed := h.r.Replicas()[0] + h.r.OnShardStatus(placed.ID, placed.Shards[0].ID, supervisor.Ready, "") + + unclaimed := h.r.Adopt([]AdoptedShard{{ + NodeID: "1", ReplicaID: placed.ID, ShardID: "some-other-shard", + Rank: 0, HTTPPort: 29999, State: supervisor.Ready, + Model: "llama", Profile: "tp4", Endpoint: "http://node1:29999", + }}) + + if len(unclaimed) != 1 { + t.Fatalf("the stray was not disowned: %+v", unclaimed) + } + if got := h.r.Replicas()[0].Endpoint; got == "http://node1:29999" { + t.Error("a stray replaced the replica that was actually serving") + } +}