From 688b205604c56ac955fdd55e5d4edd7a34530338 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sat, 17 Oct 2026 00:23:26 +0200 Subject: [PATCH] What llmgw adds to a request, measured: ctl bench overhead MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An engine that answers at once and the gateway in front of it, in one process on loopback, given the same load straight and through the gateway: bare, with authentication and accounting, and with the guardrails' pattern checks. It reports p50 and p99, what the gateway added, and a rough CPU per thousand requests. On an Apple M5 at 100 to 2,000 requests a second (10 s a step, 72,000 requests, none failed), the gateway added 0.05–0.42 ms at p50 and at most 0.49 ms at p99. That is now in the README, with how it was measured and why the CPU figure is rough. CI runs a one-second version. Co-Authored-By: Claude Opus 5.5 (1M context) --- BACKLOG.md | 14 +- CHANGELOG.md | 8 + README.md | 27 ++ internal/cli/bench.go | 1 + internal/cli/bench_overhead.go | 421 ++++++++++++++++++++++++++++ internal/cli/bench_overhead_test.go | 33 +++ 6 files changed, 502 insertions(+), 2 deletions(-) create mode 100644 internal/cli/bench_overhead.go create mode 100644 internal/cli/bench_overhead_test.go diff --git a/BACKLOG.md b/BACKLOG.md index 58145243..fc1454c0 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -23,7 +23,8 @@ What is left, most needed first. The detail is in the item each names. another restart was giving back. 2. ~~**J5**~~ — done 2026-10-16: an upgrade refits an earlier install's judge by itself. -3. **I3** — what llmgw adds to a request, measured, in the README. (S) +3. ~~**I3**~~ — done 2026-10-16: `ctl bench overhead`, figures in the + README. 4. **I4** — several keys for one outside provider. (S) 5. **J2** — a judge the leader adds by itself is sized as init's is. (S) 6. **J3** — HA setup puts the judges on nodes of their own. (S) @@ -1522,7 +1523,16 @@ in every client with its own credentials and nobody seeing what was called. through it, a key without the grant does not see the server's tools, and every call is in `ctl audit`. -### I3. What llmgw adds to a request, measured (S) +### I3. What llmgw adds to a request, measured (S) — done 2026-10-16 +`llmgw ctl bench overhead` (`internal/cli/bench_overhead.go`): an instant +engine and the gateway in one process, four modes (direct, bare, auth, auth +and guard patterns) at rising rates. On an Apple M5 the gateway added +0.05–0.42 ms at p50 and at most 0.49 ms at p99, in the README. +`TestTheGatewaysOverheadIsMeasured` runs a second of it in CI with no bound +asserted. Left: the CPU figure is rough with everything in one process; a +gateway in a process of its own would measure it properly. The original +item follows. + Bifrost's headline is "11 µs at 5k RPS" (its own figure, and its own pages disagree with it). llmgw has none, so any question about its overhead is answered with a guess. diff --git a/CHANGELOG.md b/CHANGELOG.md index f8fa556a..c2ff303a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,14 @@ APIs change between minors without a deprecation period. ## Unreleased +### Added + +- **`llmgw ctl bench overhead`**: what the gateway adds to a request, measured against an engine that answers at + once. Gateway and engine run in one process on loopback, so no cluster is needed. The load is sent straight to the + engine and through the gateway: bare, with authentication and accounting, and with the guardrails' pattern checks. + It reports p50 and p99 latency, what the gateway added, and a rough CPU per thousand requests. On an Apple M5 the + gateway added 0.05–0.42 ms at p50 and at most 0.49 ms at p99 (README). + ### Fixed - **An upgrade refits the judge an earlier `server init` installed.** Upgrading to v0.14.7 left such a judge as it was: diff --git a/README.md b/README.md index ebc703a3..c9bfa2a3 100644 --- a/README.md +++ b/README.md @@ -257,6 +257,33 @@ Add `--json` to any command for machine-readable output. says nothing about which of six constraints bit, so it prints every candidate considered and the reason it was rejected. +## What llmgw adds to a request + +Measured with `llmgw ctl bench overhead`. It starts an engine that answers +at once and a gateway in front of it, in one process on loopback. It sends +the same load straight to the engine and through the gateway, and reports +what the gateway added. Run on an Apple M5 (10 CPUs, load about 2) on +2026-10-16, 10 s a step, 72,000 requests with none failed: + +| Through the gateway | added at p50 | added at p99 | +|---|---|---| +| bare | 0.06–0.24 ms | 0.10–0.33 ms | +| with authentication and usage accounting | 0.05–0.32 ms | 0.09–0.37 ms | +| with the guardrails' pattern checks as well | 0.10–0.42 ms | 0.15–0.49 ms | + +The range is across 100 to 2,000 requests a second. The overhead was +*lowest* at 2,000, most likely because a busier processor keeps its cores +awake. The guardrails' judge model is not in these figures: a judgment +takes seconds on CPUs (see [install.md](docs/install.md)) and is a model's +time, not the gateway's. + +The command also prints the gateway's CPU per thousand requests. Gateway, +engine and load share one process there, so that figure is rough (0.04 to +0.65 s per thousand in the run above); the latency is the one to quote. +Figures from a VM on a busy host move with the host: run it where you will +deploy, and keep the load it prints beside the result. CI runs a one-second +version on every change (`TestTheGatewaysOverheadIsMeasured`). + ## Running the router ```sh diff --git a/internal/cli/bench.go b/internal/cli/bench.go index 60c737df..6da5dfa0 100644 --- a/internal/cli/bench.go +++ b/internal/cli/bench.go @@ -150,6 +150,7 @@ func newBenchCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { f.DurationVar(&cfg.SLO.TPOTp99, "tpot-p99", cfg.SLO.TPOTp99, "per-output-token target") f.BoolVar(&save, "save", false, "record the capacity against the model and profile, so the number outlives this terminal") + cmd.AddCommand(newBenchOverheadCmd(asJSON)) return cmd } diff --git a/internal/cli/bench_overhead.go b/internal/cli/bench_overhead.go new file mode 100644 index 00000000..5ed6d1bb --- /dev/null +++ b/internal/cli/bench_overhead.go @@ -0,0 +1,421 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "slices" + "sort" + "strings" + "sync" + "syscall" + "text/tabwriter" + "time" + + "github.com/spf13/cobra" + + "gitea.mynodes.xyz/ilker/llmgw/internal/config" + "gitea.mynodes.xyz/ilker/llmgw/internal/guard" + "gitea.mynodes.xyz/ilker/llmgw/internal/router" + storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// What llmgw adds to a request, measured: the same load straight at an +// engine that answers at once, and through the gateway in front of it — +// bare, with authentication and usage accounting, and with the guardrails' +// pattern checks besides. The engine's own time is the same on both paths, +// so what differs is the gateway's. +// +// Everything runs in this process, on loopback: no cluster, no GPU, and the +// numbers are this machine's. On a machine shared with other work (a VM on +// an oversubscribed host) they move with it; compare modes run together, +// not figures from different runs. + +// overheadMode is one way a request reaches the engine. +type overheadMode struct { + name string + gateway bool + auth, guard bool +} + +var overheadModes = []overheadMode{ + {name: "direct"}, + {name: "gateway", gateway: true}, + {name: "gateway+auth", gateway: true, auth: true}, + {name: "gateway+auth+guard", gateway: true, auth: true, guard: true}, +} + +// overheadStep is one mode at one request rate. +type overheadStep struct { + Mode string `json:"mode"` + Rate int `json:"rate"` + Sent int `json:"sent"` + Failed int `json:"failed"` + P50 time.Duration `json:"p50"` + P99 time.Duration `json:"p99"` + AddedP50 time.Duration `json:"added_p50"` + AddedP99 time.Duration `json:"added_p99"` + // CPUPer1k is the process's CPU time per thousand requests beyond what + // the direct mode spent at the same rate: the gateway's. + CPUPer1k time.Duration `json:"cpu_per_1k"` + cpu time.Duration +} + +func newBenchOverheadCmd(asJSON *bool) *cobra.Command { + var ( + rates []int + each time.Duration + ) + cmd := &cobra.Command{ + Use: "overhead", + Short: "Measure what the gateway adds to a request, against an engine that answers at once", + Long: "Runs an engine that answers at once and a gateway in front of it, in this\n" + + "process on loopback, and sends the same load at each rate straight to the\n" + + "engine and through the gateway: bare, with authentication and usage\n" + + "accounting, and with the guardrails' pattern checks too. Reports each\n" + + "request's latency at p50 and p99, what the gateway added to it, and the\n" + + "gateway's CPU time per thousand requests.\n\n" + + "Needs no cluster. The figures are this machine's: on one shared with other\n" + + "work they move with it, so compare the modes of one run, and note the\n" + + "machine's load beside any figure you keep.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + steps, err := runOverhead(cmd.Context(), rates, each) + if err != nil { + return err + } + if *asJSON { + return json.NewEncoder(cmd.OutOrStdout()).Encode(steps) + } + printOverhead(cmd.OutOrStdout(), steps) + return nil + }, + } + f := cmd.Flags() + f.IntSliceVar(&rates, "rates", []int{100, 500, 1000, 2000}, "requests a second to send, one step each") + f.DurationVar(&each, "each", 3*time.Second, "how long each step sends for") + return cmd +} + +func printOverhead(w io.Writer, steps []overheadStep) { + tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0) + fmt.Fprintln(tw, "MODE\tRATE/S\tSENT\tFAILED\tP50\tP99\t+P50\t+P99\tCPU/1K REQ") + for _, s := range steps { + added := func(d time.Duration) string { + if s.Mode == "direct" { + return "-" + } + return msOf(d) + } + cpu := "-" + if s.Mode != "direct" { + cpu = msOf(s.CPUPer1k) + } + fmt.Fprintf(tw, "%s\t%d\t%d\t%d\t%s\t%s\t%s\t%s\t%s\n", s.Mode, s.Rate, s.Sent, s.Failed, + msOf(s.P50), msOf(s.P99), added(s.AddedP50), added(s.AddedP99), cpu) + } + _ = tw.Flush() + fmt.Fprintf(w, "\n%d CPUs, %s. Loopback, in one process: the figures are this machine's.\n", + runtime.NumCPU(), loadAverage()) +} + +func msOf(d time.Duration) string { return fmt.Sprintf("%.2fms", float64(d)/float64(time.Millisecond)) } + +// loadAverage is the machine's, where it can be read, to keep beside the +// figures. +func loadAverage() string { + if b, err := os.ReadFile("/proc/loadavg"); err == nil { + if f := strings.Fields(string(b)); len(f) >= 3 { + return "load " + strings.Join(f[:3], " ") + } + } + // macOS: "{ 1.23 1.10 0.98 }". + if b, err := exec.Command("sysctl", "-n", "vm.loadavg").Output(); err == nil { + if f := strings.Fields(strings.Trim(strings.TrimSpace(string(b)), "{}")); len(f) >= 3 { + return "load " + strings.Join(f[:3], " ") + } + } + return "load not read" +} + +// runOverhead measures every mode at every rate. +func runOverhead(ctx context.Context, rates []int, each time.Duration) ([]overheadStep, error) { + engine, err := instantEngine() + if err != nil { + return nil, err + } + defer engine.Close() + dir, err := os.MkdirTemp("", "llmgw-overhead-") + if err != nil { + return nil, err + } + defer os.RemoveAll(dir) + + type target struct { + url, key string + stop func() + } + targets := map[string]target{} + defer func() { + for _, t := range targets { + if t.stop != nil { + t.stop() + } + } + }() + for _, m := range overheadModes { + if !m.gateway { + targets[m.name] = target{url: engine.URL} + continue + } + url, key, stop, err := overheadGateway(ctx, engine.URL, filepath.Join(dir, m.name+".db"), m) + if err != nil { + return nil, fmt.Errorf("%s: %w", m.name, err) + } + targets[m.name] = target{url: url, key: key, stop: stop} + } + + client := &http.Client{Transport: &http.Transport{MaxIdleConns: 1024, MaxIdleConnsPerHost: 1024}, Timeout: 30 * time.Second} + var out []overheadStep + for _, rate := range rates { + var direct overheadStep + for _, m := range overheadModes { + t := targets[m.name] + s := loadStep(ctx, client, t.url, t.key, rate, each) + s.Mode = m.name + if m.name == "direct" { + direct = s + } else { + s.AddedP50, s.AddedP99 = max(0, s.P50-direct.P50), max(0, s.P99-direct.P99) + if s.Sent > 0 { + s.CPUPer1k = max(0, s.cpu-direct.cpu) * 1000 / time.Duration(s.Sent) + } + } + out = append(out, s) + } + } + return out, nil +} + +// loadStep sends rate requests a second for d, each in its own goroutine, +// and measures each one's latency and the process's CPU meanwhile. +func loadStep(ctx context.Context, client *http.Client, url, key string, rate int, d time.Duration) overheadStep { + s := overheadStep{Rate: rate} + body := []byte(`{"model":"bench","max_tokens":8,"messages":[{"role":"user","content":"What is the capital of France?"}]}`) + var ( + mu sync.Mutex + lat []time.Duration + wg sync.WaitGroup + total = int(float64(rate) * d.Seconds()) + ) + interval := time.Second / time.Duration(max(rate, 1)) + cpu0 := processCPU() + start := time.Now() + for i := range total { + if ctx.Err() != nil { + break + } + if wait := time.Until(start.Add(time.Duration(i) * interval)); wait > 0 { + time.Sleep(wait) + } + wg.Add(1) + go func() { + defer wg.Done() + req, _ := http.NewRequestWithContext(ctx, http.MethodPost, url+"/v1/chat/completions", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + if key != "" { + req.Header.Set("Authorization", "Bearer "+key) + } + t := time.Now() + resp, err := client.Do(req) + ok := err == nil && resp.StatusCode == http.StatusOK + if resp != nil { + _, _ = io.Copy(io.Discard, resp.Body) + resp.Body.Close() + } + el := time.Since(t) + mu.Lock() + s.Sent++ + if ok { + lat = append(lat, el) + } else { + s.Failed++ + } + mu.Unlock() + }() + } + wg.Wait() + s.cpu = processCPU() - cpu0 + s.P50, s.P99 = percentile(lat, 0.50), percentile(lat, 0.99) + return s +} + +func percentile(ds []time.Duration, p float64) time.Duration { + if len(ds) == 0 { + return 0 + } + ds = slices.Clone(ds) + sort.Slice(ds, func(i, j int) bool { return ds[i] < ds[j] }) + i := int(p * float64(len(ds)-1)) + return ds[i] +} + +// processCPU is the CPU time this process has used, user and system. +func processCPU() time.Duration { + var ru syscall.Rusage + if err := syscall.Getrusage(syscall.RUSAGE_SELF, &ru); err != nil { + return 0 + } + return time.Duration(ru.Utime.Nano() + ru.Stime.Nano()) +} + +// instantEngine is an OpenAI-compatible engine that answers at once, with +// the metrics the router scrapes. +func instantEngine() (*httpServer, error) { + mux := http.NewServeMux() + mux.HandleFunc("GET /metrics", func(w http.ResponseWriter, _ *http.Request) { + fmt.Fprint(w, "# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running 0\n"+ + "# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting 0\n"+ + "# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc 0.1\n") + }) + mux.HandleFunc("GET /health", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }) + mux.HandleFunc("POST /v1/chat/completions", func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"id":"b","object":"chat.completion","model":"bench","choices":[{"index":0,"message":{"role":"assistant","content":"Paris."},"finish_reason":"stop"}],"usage":{"prompt_tokens":14,"completion_tokens":2,"total_tokens":16}}`) + }) + return serveLoopback(mux) +} + +// httpServer is a handler served on a loopback port. +type httpServer struct { + URL string + srv *http.Server +} + +func (s *httpServer) Close() { _ = s.srv.Close() } + +func serveLoopback(h http.Handler) (*httpServer, error) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return nil, err + } + srv := &http.Server{Handler: h, ReadHeaderTimeout: 10 * time.Second} + go func() { _ = srv.Serve(ln) }() + return &httpServer{URL: "http://" + ln.Addr().String(), srv: srv}, nil +} + +// overheadGateway starts a gateway in front of the engine as the mode asks, +// and returns its address and, with auth, a key. +func overheadGateway(ctx context.Context, engineURL, storePath string, m overheadMode) (string, string, func(), error) { + cfg := config.Default() + cfg.Listen = "127.0.0.1:0" + cfg.Router.ScrapeInterval = 200 * time.Millisecond + cfg.Endpoints = []config.Endpoint{{ID: "bench", URL: engineURL, Models: []string{"bench"}}} + key := "" + if m.auth { + var err error + if key, err = overheadKey(ctx, storePath); err != nil { + return "", "", nil, err + } + cfg.Auth.Enabled, cfg.Auth.Store = true, storePath + // Limits would measure themselves refusing: off. + cfg.Auth.DefaultRPM, cfg.Auth.DefaultTPM = 0, 0 + } + rt, err := router.New(cfg, slog.New(slog.DiscardHandler)) + if err != nil { + return "", "", nil, err + } + if m.guard { + g, err := baselineGuard() + if err != nil { + return "", "", nil, err + } + rt.SetGuard(g) + } + rctx, cancel := context.WithCancel(ctx) + rt.Start(rctx) + srv, err := serveLoopback(rt.Handler()) + if err != nil { + cancel() + return "", "", nil, err + } + stop := func() { srv.Close(); cancel(); rt.Stop() } + // Ready once a request goes through: the endpoint is scraped first. + deadline := time.Now().Add(10 * time.Second) + for { + body := `{"model":"bench","messages":[{"role":"user","content":"hi"}]}` + req, _ := http.NewRequestWithContext(ctx, http.MethodPost, srv.URL+"/v1/chat/completions", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + if key != "" { + req.Header.Set("Authorization", "Bearer "+key) + } + resp, err := http.DefaultClient.Do(req) + if err == nil { + _, _ = io.Copy(io.Discard, resp.Body) + resp.Body.Close() + if resp.StatusCode == http.StatusOK { + return srv.URL, key, stop, nil + } + } + if time.Now().After(deadline) { + stop() + status := "no answer" + if err == nil { + status = resp.Status + } + return "", "", nil, fmt.Errorf("the gateway did not serve within 10s (%s)", status) + } + time.Sleep(100 * time.Millisecond) + } +} + +// overheadKey makes a store with one project and a key in it. +func overheadKey(ctx context.Context, path string) (string, error) { + st, err := storepkg.Open(ctx, path) + if err != nil { + return "", err + } + defer st.Close() + org, err := st.CreateOrg(ctx, "bench") + if err != nil { + return "", err + } + team, err := st.CreateTeam(ctx, org, "bench") + if err != nil { + return "", err + } + project, err := st.CreateProject(ctx, team, "bench") + if err != nil { + return "", err + } + secret, k, err := storepkg.NewKey("key-bench", "bench", project) + if err != nil { + return "", err + } + k.CreatedAt = time.Now() + return secret, st.CreateKey(ctx, k) +} + +// baselineGuard is the policy init installs, without its judge: the pattern +// checks every request passes. +func baselineGuard() (guard.Guard, error) { + m, err := config.Parse([]byte(guardSection(false))) + if err != nil { + return nil, err + } + cfg, err := config.FromMap(m) + if err != nil { + return nil, err + } + return guard.NewGate(cfg.Guard, nil, slog.New(slog.DiscardHandler)) +} diff --git a/internal/cli/bench_overhead_test.go b/internal/cli/bench_overhead_test.go new file mode 100644 index 00000000..6eec237a --- /dev/null +++ b/internal/cli/bench_overhead_test.go @@ -0,0 +1,33 @@ +package cli + +import ( + "bytes" + "strings" + "testing" + "time" +) + +// The short version, so CI sees the gateway's overhead on every change: every +// mode answers every request, and what the gateway adds is measured, not +// guessed. No bound is asserted on it, as CI's machines vary; the table is +// in the test's log. +func TestTheGatewaysOverheadIsMeasured(t *testing.T) { + steps, err := runOverhead(t.Context(), []int{50}, time.Second) + if err != nil { + t.Fatal(err) + } + if len(steps) != len(overheadModes) { + t.Fatalf("%d steps for %d modes", len(steps), len(overheadModes)) + } + for _, s := range steps { + if s.Sent == 0 || s.Failed > 0 || s.P50 <= 0 || s.P99 < s.P50 { + t.Errorf("%s: sent %d failed %d p50 %s p99 %s", s.Mode, s.Sent, s.Failed, s.P50, s.P99) + } + } + var b bytes.Buffer + printOverhead(&b, steps) + if !strings.Contains(b.String(), "gateway+auth+guard") { + t.Errorf("table:\n%s", b.String()) + } + t.Logf("\n%s", b.String()) +}