diff --git a/BACKLOG.md b/BACKLOG.md index 58145243..fc1454c0 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -23,7 +23,8 @@ What is left, most needed first. The detail is in the item each names. another restart was giving back. 2. ~~**J5**~~ — done 2026-10-16: an upgrade refits an earlier install's judge by itself. -3. **I3** — what llmgw adds to a request, measured, in the README. (S) +3. ~~**I3**~~ — done 2026-10-16: `ctl bench overhead`, figures in the + README. 4. **I4** — several keys for one outside provider. (S) 5. **J2** — a judge the leader adds by itself is sized as init's is. (S) 6. **J3** — HA setup puts the judges on nodes of their own. (S) @@ -1522,7 +1523,16 @@ in every client with its own credentials and nobody seeing what was called. through it, a key without the grant does not see the server's tools, and every call is in `ctl audit`. -### I3. What llmgw adds to a request, measured (S) +### I3. What llmgw adds to a request, measured (S) — done 2026-10-16 +`llmgw ctl bench overhead` (`internal/cli/bench_overhead.go`): an instant +engine and the gateway in one process, four modes (direct, bare, auth, auth +and guard patterns) at rising rates. On an Apple M5 the gateway added +0.05–0.42 ms at p50 and at most 0.49 ms at p99, in the README. +`TestTheGatewaysOverheadIsMeasured` runs a second of it in CI with no bound +asserted. Left: the CPU figure is rough with everything in one process; a +gateway in a process of its own would measure it properly. The original +item follows. + Bifrost's headline is "11 µs at 5k RPS" (its own figure, and its own pages disagree with it). llmgw has none, so any question about its overhead is answered with a guess. diff --git a/CHANGELOG.md b/CHANGELOG.md index f8fa556a..c2ff303a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,14 @@ APIs change between minors without a deprecation period. ## Unreleased +### Added + +- **`llmgw ctl bench overhead`**: what the gateway adds to a request, measured against an engine that answers at + once. Gateway and engine run in one process on loopback, so no cluster is needed. The load is sent straight to the + engine and through the gateway: bare, with authentication and accounting, and with the guardrails' pattern checks. + It reports p50 and p99 latency, what the gateway added, and a rough CPU per thousand requests. On an Apple M5 the + gateway added 0.05–0.42 ms at p50 and at most 0.49 ms at p99 (README). + ### Fixed - **An upgrade refits the judge an earlier `server init` installed.** Upgrading to v0.14.7 left such a judge as it was: diff --git a/README.md b/README.md index ebc703a3..c9bfa2a3 100644 --- a/README.md +++ b/README.md @@ -257,6 +257,33 @@ Add `--json` to any command for machine-readable output. says nothing about which of six constraints bit, so it prints every candidate considered and the reason it was rejected. +## What llmgw adds to a request + +Measured with `llmgw ctl bench overhead`. It starts an engine that answers +at once and a gateway in front of it, in one process on loopback. It sends +the same load straight to the engine and through the gateway, and reports +what the gateway added. Run on an Apple M5 (10 CPUs, load about 2) on +2026-10-16, 10 s a step, 72,000 requests with none failed: + +| Through the gateway | added at p50 | added at p99 | +|---|---|---| +| bare | 0.06–0.24 ms | 0.10–0.33 ms | +| with authentication and usage accounting | 0.05–0.32 ms | 0.09–0.37 ms | +| with the guardrails' pattern checks as well | 0.10–0.42 ms | 0.15–0.49 ms | + +The range is across 100 to 2,000 requests a second. The overhead was +*lowest* at 2,000, most likely because a busier processor keeps its cores +awake. The guardrails' judge model is not in these figures: a judgment +takes seconds on CPUs (see [install.md](docs/install.md)) and is a model's +time, not the gateway's. + +The command also prints the gateway's CPU per thousand requests. Gateway, +engine and load share one process there, so that figure is rough (0.04 to +0.65 s per thousand in the run above); the latency is the one to quote. +Figures from a VM on a busy host move with the host: run it where you will +deploy, and keep the load it prints beside the result. CI runs a one-second +version on every change (`TestTheGatewaysOverheadIsMeasured`). + ## Running the router ```sh diff --git a/internal/cli/bench.go b/internal/cli/bench.go index 60c737df..6da5dfa0 100644 --- a/internal/cli/bench.go +++ b/internal/cli/bench.go @@ -150,6 +150,7 @@ func newBenchCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { f.DurationVar(&cfg.SLO.TPOTp99, "tpot-p99", cfg.SLO.TPOTp99, "per-output-token target") f.BoolVar(&save, "save", false, "record the capacity against the model and profile, so the number outlives this terminal") + cmd.AddCommand(newBenchOverheadCmd(asJSON)) return cmd } diff --git a/internal/cli/bench_overhead.go b/internal/cli/bench_overhead.go new file mode 100644 index 00000000..5ed6d1bb --- /dev/null +++ b/internal/cli/bench_overhead.go @@ -0,0 +1,421 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "slices" + "sort" + "strings" + "sync" + "syscall" + "text/tabwriter" + "time" + + "github.com/spf13/cobra" + + "gitea.mynodes.xyz/ilker/llmgw/internal/config" + "gitea.mynodes.xyz/ilker/llmgw/internal/guard" + "gitea.mynodes.xyz/ilker/llmgw/internal/router" + storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// What llmgw adds to a request, measured: the same load straight at an +// engine that answers at once, and through the gateway in front of it — +// bare, with authentication and usage accounting, and with the guardrails' +// pattern checks besides. The engine's own time is the same on both paths, +// so what differs is the gateway's. +// +// Everything runs in this process, on loopback: no cluster, no GPU, and the +// numbers are this machine's. On a machine shared with other work (a VM on +// an oversubscribed host) they move with it; compare modes run together, +// not figures from different runs. + +// overheadMode is one way a request reaches the engine. +type overheadMode struct { + name string + gateway bool + auth, guard bool +} + +var overheadModes = []overheadMode{ + {name: "direct"}, + {name: "gateway", gateway: true}, + {name: "gateway+auth", gateway: true, auth: true}, + {name: "gateway+auth+guard", gateway: true, auth: true, guard: true}, +} + +// overheadStep is one mode at one request rate. +type overheadStep struct { + Mode string `json:"mode"` + Rate int `json:"rate"` + Sent int `json:"sent"` + Failed int `json:"failed"` + P50 time.Duration `json:"p50"` + P99 time.Duration `json:"p99"` + AddedP50 time.Duration `json:"added_p50"` + AddedP99 time.Duration `json:"added_p99"` + // CPUPer1k is the process's CPU time per thousand requests beyond what + // the direct mode spent at the same rate: the gateway's. + CPUPer1k time.Duration `json:"cpu_per_1k"` + cpu time.Duration +} + +func newBenchOverheadCmd(asJSON *bool) *cobra.Command { + var ( + rates []int + each time.Duration + ) + cmd := &cobra.Command{ + Use: "overhead", + Short: "Measure what the gateway adds to a request, against an engine that answers at once", + Long: "Runs an engine that answers at once and a gateway in front of it, in this\n" + + "process on loopback, and sends the same load at each rate straight to the\n" + + "engine and through the gateway: bare, with authentication and usage\n" + + "accounting, and with the guardrails' pattern checks too. Reports each\n" + + "request's latency at p50 and p99, what the gateway added to it, and the\n" + + "gateway's CPU time per thousand requests.\n\n" + + "Needs no cluster. The figures are this machine's: on one shared with other\n" + + "work they move with it, so compare the modes of one run, and note the\n" + + "machine's load beside any figure you keep.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + steps, err := runOverhead(cmd.Context(), rates, each) + if err != nil { + return err + } + if *asJSON { + return json.NewEncoder(cmd.OutOrStdout()).Encode(steps) + } + printOverhead(cmd.OutOrStdout(), steps) + return nil + }, + } + f := cmd.Flags() + f.IntSliceVar(&rates, "rates", []int{100, 500, 1000, 2000}, "requests a second to send, one step each") + f.DurationVar(&each, "each", 3*time.Second, "how long each step sends for") + return cmd +} + +func printOverhead(w io.Writer, steps []overheadStep) { + tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0) + fmt.Fprintln(tw, "MODE\tRATE/S\tSENT\tFAILED\tP50\tP99\t+P50\t+P99\tCPU/1K REQ") + for _, s := range steps { + added := func(d time.Duration) string { + if s.Mode == "direct" { + return "-" + } + return msOf(d) + } + cpu := "-" + if s.Mode != "direct" { + cpu = msOf(s.CPUPer1k) + } + fmt.Fprintf(tw, "%s\t%d\t%d\t%d\t%s\t%s\t%s\t%s\t%s\n", s.Mode, s.Rate, s.Sent, s.Failed, + msOf(s.P50), msOf(s.P99), added(s.AddedP50), added(s.AddedP99), cpu) + } + _ = tw.Flush() + fmt.Fprintf(w, "\n%d CPUs, %s. Loopback, in one process: the figures are this machine's.\n", + runtime.NumCPU(), loadAverage()) +} + +func msOf(d time.Duration) string { return fmt.Sprintf("%.2fms", float64(d)/float64(time.Millisecond)) } + +// loadAverage is the machine's, where it can be read, to keep beside the +// figures. +func loadAverage() string { + if b, err := os.ReadFile("/proc/loadavg"); err == nil { + if f := strings.Fields(string(b)); len(f) >= 3 { + return "load " + strings.Join(f[:3], " ") + } + } + // macOS: "{ 1.23 1.10 0.98 }". + if b, err := exec.Command("sysctl", "-n", "vm.loadavg").Output(); err == nil { + if f := strings.Fields(strings.Trim(strings.TrimSpace(string(b)), "{}")); len(f) >= 3 { + return "load " + strings.Join(f[:3], " ") + } + } + return "load not read" +} + +// runOverhead measures every mode at every rate. +func runOverhead(ctx context.Context, rates []int, each time.Duration) ([]overheadStep, error) { + engine, err := instantEngine() + if err != nil { + return nil, err + } + defer engine.Close() + dir, err := os.MkdirTemp("", "llmgw-overhead-") + if err != nil { + return nil, err + } + defer os.RemoveAll(dir) + + type target struct { + url, key string + stop func() + } + targets := map[string]target{} + defer func() { + for _, t := range targets { + if t.stop != nil { + t.stop() + } + } + }() + for _, m := range overheadModes { + if !m.gateway { + targets[m.name] = target{url: engine.URL} + continue + } + url, key, stop, err := overheadGateway(ctx, engine.URL, filepath.Join(dir, m.name+".db"), m) + if err != nil { + return nil, fmt.Errorf("%s: %w", m.name, err) + } + targets[m.name] = target{url: url, key: key, stop: stop} + } + + client := &http.Client{Transport: &http.Transport{MaxIdleConns: 1024, MaxIdleConnsPerHost: 1024}, Timeout: 30 * time.Second} + var out []overheadStep + for _, rate := range rates { + var direct overheadStep + for _, m := range overheadModes { + t := targets[m.name] + s := loadStep(ctx, client, t.url, t.key, rate, each) + s.Mode = m.name + if m.name == "direct" { + direct = s + } else { + s.AddedP50, s.AddedP99 = max(0, s.P50-direct.P50), max(0, s.P99-direct.P99) + if s.Sent > 0 { + s.CPUPer1k = max(0, s.cpu-direct.cpu) * 1000 / time.Duration(s.Sent) + } + } + out = append(out, s) + } + } + return out, nil +} + +// loadStep sends rate requests a second for d, each in its own goroutine, +// and measures each one's latency and the process's CPU meanwhile. +func loadStep(ctx context.Context, client *http.Client, url, key string, rate int, d time.Duration) overheadStep { + s := overheadStep{Rate: rate} + body := []byte(`{"model":"bench","max_tokens":8,"messages":[{"role":"user","content":"What is the capital of France?"}]}`) + var ( + mu sync.Mutex + lat []time.Duration + wg sync.WaitGroup + total = int(float64(rate) * d.Seconds()) + ) + interval := time.Second / time.Duration(max(rate, 1)) + cpu0 := processCPU() + start := time.Now() + for i := range total { + if ctx.Err() != nil { + break + } + if wait := time.Until(start.Add(time.Duration(i) * interval)); wait > 0 { + time.Sleep(wait) + } + wg.Add(1) + go func() { + defer wg.Done() + req, _ := http.NewRequestWithContext(ctx, http.MethodPost, url+"/v1/chat/completions", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + if key != "" { + req.Header.Set("Authorization", "Bearer "+key) + } + t := time.Now() + resp, err := client.Do(req) + ok := err == nil && resp.StatusCode == http.StatusOK + if resp != nil { + _, _ = io.Copy(io.Discard, resp.Body) + resp.Body.Close() + } + el := time.Since(t) + mu.Lock() + s.Sent++ + if ok { + lat = append(lat, el) + } else { + s.Failed++ + } + mu.Unlock() + }() + } + wg.Wait() + s.cpu = processCPU() - cpu0 + s.P50, s.P99 = percentile(lat, 0.50), percentile(lat, 0.99) + return s +} + +func percentile(ds []time.Duration, p float64) time.Duration { + if len(ds) == 0 { + return 0 + } + ds = slices.Clone(ds) + sort.Slice(ds, func(i, j int) bool { return ds[i] < ds[j] }) + i := int(p * float64(len(ds)-1)) + return ds[i] +} + +// processCPU is the CPU time this process has used, user and system. +func processCPU() time.Duration { + var ru syscall.Rusage + if err := syscall.Getrusage(syscall.RUSAGE_SELF, &ru); err != nil { + return 0 + } + return time.Duration(ru.Utime.Nano() + ru.Stime.Nano()) +} + +// instantEngine is an OpenAI-compatible engine that answers at once, with +// the metrics the router scrapes. +func instantEngine() (*httpServer, error) { + mux := http.NewServeMux() + mux.HandleFunc("GET /metrics", func(w http.ResponseWriter, _ *http.Request) { + fmt.Fprint(w, "# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running 0\n"+ + "# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting 0\n"+ + "# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc 0.1\n") + }) + mux.HandleFunc("GET /health", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }) + mux.HandleFunc("POST /v1/chat/completions", func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"id":"b","object":"chat.completion","model":"bench","choices":[{"index":0,"message":{"role":"assistant","content":"Paris."},"finish_reason":"stop"}],"usage":{"prompt_tokens":14,"completion_tokens":2,"total_tokens":16}}`) + }) + return serveLoopback(mux) +} + +// httpServer is a handler served on a loopback port. +type httpServer struct { + URL string + srv *http.Server +} + +func (s *httpServer) Close() { _ = s.srv.Close() } + +func serveLoopback(h http.Handler) (*httpServer, error) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return nil, err + } + srv := &http.Server{Handler: h, ReadHeaderTimeout: 10 * time.Second} + go func() { _ = srv.Serve(ln) }() + return &httpServer{URL: "http://" + ln.Addr().String(), srv: srv}, nil +} + +// overheadGateway starts a gateway in front of the engine as the mode asks, +// and returns its address and, with auth, a key. +func overheadGateway(ctx context.Context, engineURL, storePath string, m overheadMode) (string, string, func(), error) { + cfg := config.Default() + cfg.Listen = "127.0.0.1:0" + cfg.Router.ScrapeInterval = 200 * time.Millisecond + cfg.Endpoints = []config.Endpoint{{ID: "bench", URL: engineURL, Models: []string{"bench"}}} + key := "" + if m.auth { + var err error + if key, err = overheadKey(ctx, storePath); err != nil { + return "", "", nil, err + } + cfg.Auth.Enabled, cfg.Auth.Store = true, storePath + // Limits would measure themselves refusing: off. + cfg.Auth.DefaultRPM, cfg.Auth.DefaultTPM = 0, 0 + } + rt, err := router.New(cfg, slog.New(slog.DiscardHandler)) + if err != nil { + return "", "", nil, err + } + if m.guard { + g, err := baselineGuard() + if err != nil { + return "", "", nil, err + } + rt.SetGuard(g) + } + rctx, cancel := context.WithCancel(ctx) + rt.Start(rctx) + srv, err := serveLoopback(rt.Handler()) + if err != nil { + cancel() + return "", "", nil, err + } + stop := func() { srv.Close(); cancel(); rt.Stop() } + // Ready once a request goes through: the endpoint is scraped first. + deadline := time.Now().Add(10 * time.Second) + for { + body := `{"model":"bench","messages":[{"role":"user","content":"hi"}]}` + req, _ := http.NewRequestWithContext(ctx, http.MethodPost, srv.URL+"/v1/chat/completions", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + if key != "" { + req.Header.Set("Authorization", "Bearer "+key) + } + resp, err := http.DefaultClient.Do(req) + if err == nil { + _, _ = io.Copy(io.Discard, resp.Body) + resp.Body.Close() + if resp.StatusCode == http.StatusOK { + return srv.URL, key, stop, nil + } + } + if time.Now().After(deadline) { + stop() + status := "no answer" + if err == nil { + status = resp.Status + } + return "", "", nil, fmt.Errorf("the gateway did not serve within 10s (%s)", status) + } + time.Sleep(100 * time.Millisecond) + } +} + +// overheadKey makes a store with one project and a key in it. +func overheadKey(ctx context.Context, path string) (string, error) { + st, err := storepkg.Open(ctx, path) + if err != nil { + return "", err + } + defer st.Close() + org, err := st.CreateOrg(ctx, "bench") + if err != nil { + return "", err + } + team, err := st.CreateTeam(ctx, org, "bench") + if err != nil { + return "", err + } + project, err := st.CreateProject(ctx, team, "bench") + if err != nil { + return "", err + } + secret, k, err := storepkg.NewKey("key-bench", "bench", project) + if err != nil { + return "", err + } + k.CreatedAt = time.Now() + return secret, st.CreateKey(ctx, k) +} + +// baselineGuard is the policy init installs, without its judge: the pattern +// checks every request passes. +func baselineGuard() (guard.Guard, error) { + m, err := config.Parse([]byte(guardSection(false))) + if err != nil { + return nil, err + } + cfg, err := config.FromMap(m) + if err != nil { + return nil, err + } + return guard.NewGate(cfg.Guard, nil, slog.New(slog.DiscardHandler)) +} diff --git a/internal/cli/bench_overhead_test.go b/internal/cli/bench_overhead_test.go new file mode 100644 index 00000000..6eec237a --- /dev/null +++ b/internal/cli/bench_overhead_test.go @@ -0,0 +1,33 @@ +package cli + +import ( + "bytes" + "strings" + "testing" + "time" +) + +// The short version, so CI sees the gateway's overhead on every change: every +// mode answers every request, and what the gateway adds is measured, not +// guessed. No bound is asserted on it, as CI's machines vary; the table is +// in the test's log. +func TestTheGatewaysOverheadIsMeasured(t *testing.T) { + steps, err := runOverhead(t.Context(), []int{50}, time.Second) + if err != nil { + t.Fatal(err) + } + if len(steps) != len(overheadModes) { + t.Fatalf("%d steps for %d modes", len(steps), len(overheadModes)) + } + for _, s := range steps { + if s.Sent == 0 || s.Failed > 0 || s.P50 <= 0 || s.P99 < s.P50 { + t.Errorf("%s: sent %d failed %d p50 %s p99 %s", s.Mode, s.Sent, s.Failed, s.P50, s.P99) + } + } + var b bytes.Buffer + printOverhead(&b, steps) + if !strings.Contains(b.String(), "gateway+auth+guard") { + t.Errorf("table:\n%s", b.String()) + } + t.Logf("\n%s", b.String()) +}