From 8cc8d7139b04d68141e53de5dfec4a2ebdb82fbe Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Mon, 5 Oct 2026 00:20:25 +0200 Subject: [PATCH] SGLang read in its own metric names, started serving them, and asked for its own help The router read SGLang replicas with vLLM's names, and SGLang was started without --enable-metrics, so it served no /metrics: every scrape failed as if unreachable and the replica was routed to blind. It now gets --enable-metrics and a mapping of its own (sglang:num_running_reqs, num_queue_reqs, token_usage, the token counters and num_requests_total, as 0.5.21 publishes them). engine-check asks python -m sglang.launch_server --help instead of Python's own help, and SGLang is given --tp-size rather than the --tp argparse only takes as an abbreviation. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/install.md | 5 ++ internal/agent/engines/engines_test.go | 20 +++++ internal/agent/engines/sglang.go | 18 +++- internal/cli/enginecheck_test.go | 60 ++++++++++++- internal/router/scrape/parse.go | 35 ++++++++ internal/router/scrape/sglang_test.go | 116 +++++++++++++++++++++++++ 6 files changed, 251 insertions(+), 3 deletions(-) create mode 100644 internal/router/scrape/sglang_test.go diff --git a/docs/install.md b/docs/install.md index 114315a7..68be7040 100644 --- a/docs/install.md +++ b/docs/install.md @@ -597,6 +597,11 @@ engines: sglang: "0.5.20" # SGLang only when named ``` +SGLang is started with `--enable-metrics`, without which it serves no +`/metrics`; the router reads `sglang:num_running_reqs`, `sglang:num_queue_reqs` +and `sglang:token_usage` there. Check an SGLang against the flags llmgw builds +with `llmgw ctl engine-check sglang --bin /bin/python`. + A changed version reaches every node when it next connects. To use an engine you installed yourself on a node, name its path in that node's `agent.yaml` (`engines.llamacpp.bin`, `engines.vllm.bin`, `engines.sglang.python`, diff --git a/internal/agent/engines/engines_test.go b/internal/agent/engines/engines_test.go index c795bcf7..7d0fb9dc 100644 --- a/internal/agent/engines/engines_test.go +++ b/internal/agent/engines/engines_test.go @@ -366,6 +366,26 @@ func TestEnginesListenOnTheAddressTheyAreGiven(t *testing.T) { } } +func TestSGLangIsStartedServingItsMetrics(t *testing.T) { + // Without --enable-metrics SGLang has no /metrics, and every scrape of + // the replica fails as though it were unreachable. + if cmd := buildOne(t, &SGLang{}, singleNodeSpec()); !slices.Contains(cmd.Args, "--enable-metrics") { + t.Errorf("not started with its metrics: %v", cmd.Args) + } +} + +func TestSGLangIsAskedForItsHelpThroughItsModule(t *testing.T) { + g := &SGLang{cfg: SGLangConfig{Python: "/opt/sglang/bin/python"}} + g.UseInstalled("/opt/llmgw/engines/sglang/0.5.21-cu13/bin/python") + path, args := g.HelpCommand(Command{}) + if path != "/opt/llmgw/engines/sglang/0.5.21-cu13/bin/python" { + t.Errorf("asked %s, not the interpreter it was installed into", path) + } + if !slices.Equal(args, []string{"-m", "sglang.launch_server", "--help"}) { + t.Errorf("asked %v", args) + } +} + func TestSGLangSingleNodeOmitsMultiNodeFlags(t *testing.T) { cmd := buildOne(t, &SGLang{}, singleNodeSpec()) for _, flag := range []string{"--nnodes", "--node-rank", "--dist-init-addr"} { diff --git a/internal/agent/engines/sglang.go b/internal/agent/engines/sglang.go index ccaa5722..fa24a8c6 100644 --- a/internal/agent/engines/sglang.go +++ b/internal/agent/engines/sglang.go @@ -40,7 +40,15 @@ func (*SGLang) Name() string { return "sglang" } // HealthPath is SGLang's readiness endpoint. func (*SGLang) HealthPath() string { return "/health" } -// MetricsPath is SGLang's Prometheus endpoint. +// HelpCommand is how to ask SGLang for its help: through its module, in the +// environment it was installed into, since the command itself is llmgw's +// launcher and the interpreter's own --help describes Python. +func (g *SGLang) HelpCommand(Command) (string, []string) { + return g.python(), []string{"-m", "sglang.launch_server", "--help"} +} + +// MetricsPath is SGLang's Prometheus endpoint, served only with +// --enable-metrics. func (*SGLang) MetricsPath() string { return "/metrics" } // Validate checks the profile against the GPUs it would run on. @@ -83,7 +91,13 @@ func (g *SGLang) serverCommand(s ShardSpec) (Command, error) { "--model-path", p.ModelPath, "--host", s.Host(), "--port", strconv.Itoa(s.HTTPPort), - "--tp", strconv.Itoa(p.TP), + // The flag's whole name: SGLang's documentation writes --tp, which + // argparse accepts only as an abbreviation of --tp-size and only + // while no other flag begins with it. + "--tp-size", strconv.Itoa(p.TP), + // Without it SGLang serves no /metrics at all, and the router has + // nothing to schedule this replica by. + "--enable-metrics", } if p.ServedModelName != "" { args = append(args, "--served-model-name", p.ServedModelName) diff --git a/internal/cli/enginecheck_test.go b/internal/cli/enginecheck_test.go index 1b4d85f4..ac40b0a6 100644 --- a/internal/cli/enginecheck_test.go +++ b/internal/cli/enginecheck_test.go @@ -79,6 +79,13 @@ func TestATruncatedHelpIsSuspected(t *testing.T) { // trtllm-serve's help through its module, it prints the help given; asked // anything else, it fails, as running the launcher here would. func fakeInterpreter(t *testing.T, help string) string { + t.Helper() + return fakePython(t, "-m tensorrt_llm.commands.serve serve --help", help) +} + +// fakePython is an interpreter that prints help when asked exactly ask, and +// fails on anything else — the interpreter's own --help included. +func fakePython(t *testing.T, ask, help string) string { t.Helper() dir := t.TempDir() if err := os.WriteFile(filepath.Join(dir, "help.txt"), []byte(help), 0o644); err != nil { @@ -86,7 +93,7 @@ func fakeInterpreter(t *testing.T, help string) string { } p := filepath.Join(dir, "python") body := "#!/bin/sh\n" + - `if [ "$*" = "-m tensorrt_llm.commands.serve serve --help" ]; then cat "` + filepath.Join(dir, "help.txt") + `"; exit 0; fi` + "\n" + + `if [ "$*" = "` + ask + `" ]; then cat "` + filepath.Join(dir, "help.txt") + `"; exit 0; fi` + "\n" + "echo \"unexpected: $*\" >&2; exit 2\n" if err := os.WriteFile(p, []byte(body), 0o755); err != nil { //nolint:gosec // an executable stand-in t.Fatal(err) @@ -137,6 +144,57 @@ func TestEngineCheckAsksTrtllmServeThroughItsModule(t *testing.T) { } } +// sglangHelp is the part of python -m sglang.launch_server --help (0.5.21) +// that holds the flags llmgw builds, in argparse's layout. +const sglangHelp = `usage: sglang serve [-h] --model-path MODEL_PATH [--host HOST] [--port PORT] + +options: + -h, --help show this help message and exit + --model-path MODEL_PATH, --model MODEL_PATH + Path to model weights. + --host HOST The host of the HTTP server. + --port PORT The port of the HTTP server. + --served-model-name SERVED_MODEL_NAME + Override the model name returned by the v1/models endpoint. + --enable-metrics Enable log prometheus metrics. + --dist-init-addr DIST_INIT_ADDR, --nccl-init-addr DIST_INIT_ADDR + The host address for initializing distributed backend. + --nnodes NNODES The number of nodes. + --node-rank NODE_RANK + The node rank. + --tp-size TP_SIZE, --tensor-parallel-size TP_SIZE + The tensor parallelism size. +` + +func TestEngineCheckAsksSGLangThroughItsModule(t *testing.T) { + // The command is the interpreter running llmgw's launcher; asked for + // --help it would describe Python, and every SGLang flag would look + // missing — or, against a real python3, be checked against nothing. + py := fakePython(t, "-m sglang.launch_server --help", sglangHelp) + out, err := runEngineCheck(t, "sglang", "--bin", py) + if err != nil { + t.Fatalf("%v\n%s", err, out) + } + if !strings.Contains(out, "every flag llmgw builds is accepted") { + t.Errorf("unexpected report:\n%s", out) + } + if strings.Contains(out, "unexpected:") || strings.Contains(out, "skipped") { + t.Errorf("the help was not asked of SGLang:\n%s", out) + } + // Two nodes by default, so the rendezvous flags were among those checked. + if !strings.Contains(out, "sglang: 9 flags checked") { + t.Errorf("not every flag was checked:\n%s", out) + } +} + +func TestEngineCheckCatchesAnSGLangWithoutTheFlagForItsMetrics(t *testing.T) { + old := strings.ReplaceAll(sglangHelp, " --enable-metrics Enable log prometheus metrics.\n", "") + out, err := runEngineCheck(t, "sglang", "--bin", fakePython(t, "-m sglang.launch_server --help", old)) + if err == nil || !strings.Contains(out, "sglang: --enable-metrics") { + t.Errorf("an SGLang without --enable-metrics passed: %v\n%s", err, out) + } +} + func TestEngineCheckCatchesATensorRTLLMTooOldForTheServedName(t *testing.T) { // TensorRT-LLM 1.2 has no --served_model_name, and a replica of it would // never be seen serving the name the agent waits for. diff --git a/internal/router/scrape/parse.go b/internal/router/scrape/parse.go index ec35a65c..6b4d6161 100644 --- a/internal/router/scrape/parse.go +++ b/internal/router/scrape/parse.go @@ -48,6 +48,8 @@ func MappingFor(engine string) Mapping { return LlamaCppMapping() case "trtllm": return TRTLLMMapping() + case "sglang": + return SGLangMapping() default: return VLLMMapping() } @@ -106,6 +108,39 @@ func TRTLLMMapping() Mapping { } } +// SGLangMapping returns SGLang's metric names, as sglang.launch_server +// publishes them under /metrics (checked against 0.5.21's collector). +// +// They are there only when the server runs with --enable-metrics, which +// llmgw's launch command passes; without it /metrics is not served at all. +// SGLang exempts /metrics from its API key, so the key the poller sends is +// ignored rather than needed. +// +// "Queue" is SGLang's word for vLLM's waiting. The gauges are set by the +// scheduler as it reports its batches, and at least every thirty seconds +// while it is idle, so a server that has only just started may serve none of +// them yet and is reported incomplete until it does. +// +// token_usage is the fraction of the KV pool in use, as vLLM's +// kv_cache_usage_perc is (for a model with sliding-window layers, the fuller +// of its two pools). cache_hit_rate is a different number — how much of the +// prompts the prefix cache answered — and is not what saturation means. +// +// num_requests_total counts the requests that finished, the nearest thing it +// has to vLLM's success counter; like the token counters it is split by +// is_streaming and summed. There is no cache configuration to read block +// sizes from. +func SGLangMapping() Mapping { + return Mapping{ + Running: "sglang:num_running_reqs", + Waiting: "sglang:num_queue_reqs", + KVCacheUsage: "sglang:token_usage", + PromptTokens: "sglang:prompt_tokens_total", + GenerationTokens: "sglang:generation_tokens_total", + SuccessTotal: "sglang:num_requests_total", + } +} + // VLLMMapping returns vLLM's current metric names. func VLLMMapping() Mapping { return Mapping{ diff --git a/internal/router/scrape/sglang_test.go b/internal/router/scrape/sglang_test.go new file mode 100644 index 00000000..0c9ebe11 --- /dev/null +++ b/internal/router/scrape/sglang_test.go @@ -0,0 +1,116 @@ +package scrape + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint" +) + +// sglangMetrics is what sglang.launch_server --enable-metrics serves on +// /metrics (0.5.21), trimmed to the families that matter, one it must not be +// mistaken for, and a histogram the parser has to step over. The scheduler's +// gauges carry its ranks; the tokenizer's counters are split by whether the +// request streamed. +const sglangMetrics = `# HELP sglang:num_running_reqs The number of running requests. +# TYPE sglang:num_running_reqs gauge +sglang:num_running_reqs{engine_type="unified",model_name="qwen3-8b",moe_ep_rank="0",pp_rank="0",tp_rank="0"} 6.0 +# HELP sglang:num_queue_reqs The number of requests in the waiting queue. +# TYPE sglang:num_queue_reqs gauge +sglang:num_queue_reqs{engine_type="unified",model_name="qwen3-8b",moe_ep_rank="0",pp_rank="0",tp_rank="0"} 3.0 +# HELP sglang:cache_hit_rate The prefix cache hit rate. +# TYPE sglang:cache_hit_rate gauge +sglang:cache_hit_rate{engine_type="unified",model_name="qwen3-8b",moe_ep_rank="0",pp_rank="0",tp_rank="0"} 0.9 +# HELP sglang:token_usage The token usage. +# TYPE sglang:token_usage gauge +sglang:token_usage{engine_type="unified",model_name="qwen3-8b",moe_ep_rank="0",pp_rank="0",tp_rank="0"} 0.42 +# HELP sglang:prompt_tokens_total Number of prefill tokens processed. +# TYPE sglang:prompt_tokens_total counter +sglang:prompt_tokens_total{engine_type="unified",is_streaming="true",model_name="qwen3-8b"} 12000.0 +sglang:prompt_tokens_total{engine_type="unified",is_streaming="false",model_name="qwen3-8b"} 345.0 +# HELP sglang:generation_tokens_total Number of generation tokens processed. +# TYPE sglang:generation_tokens_total counter +sglang:generation_tokens_total{engine_type="unified",is_streaming="true",model_name="qwen3-8b"} 8000.0 +sglang:generation_tokens_total{engine_type="unified",is_streaming="false",model_name="qwen3-8b"} 64.0 +# HELP sglang:num_requests_total Number of requests processed. +# TYPE sglang:num_requests_total counter +sglang:num_requests_total{engine_type="unified",is_streaming="true",model_name="qwen3-8b"} 30.0 +sglang:num_requests_total{engine_type="unified",is_streaming="false",model_name="qwen3-8b"} 4.0 +# HELP sglang:e2e_request_latency_seconds Histogram of End-to-end request latency in seconds +# TYPE sglang:e2e_request_latency_seconds histogram +sglang:e2e_request_latency_seconds_bucket{engine_type="unified",is_streaming="true",le="0.5",model_name="qwen3-8b"} 12.0 +sglang:e2e_request_latency_seconds_bucket{engine_type="unified",is_streaming="true",le="+Inf",model_name="qwen3-8b"} 30.0 +sglang:e2e_request_latency_seconds_count{engine_type="unified",is_streaming="true",model_name="qwen3-8b"} 30.0 +sglang:e2e_request_latency_seconds_sum{engine_type="unified",is_streaming="true",model_name="qwen3-8b"} 41.2 +` + +func TestSGLangMetricsSatisfyTheContract(t *testing.T) { + s, err := Parse(strings.NewReader(sglangMetrics), MappingFor("sglang")) + if err != nil { + t.Fatal(err) + } + if s.Running != 6 || s.Waiting != 3 { + t.Errorf("running=%d waiting=%d", s.Running, s.Waiting) + } + // The pool's fill, not the prefix cache's hit rate beside it. + if s.KVCacheUsage != 0.42 { + t.Errorf("kv=%v", s.KVCacheUsage) + } + // Split by is_streaming, summed. + if s.PromptTokens != 12345 || s.GenerationTokens != 8064 { + t.Errorf("tokens=%v/%v", s.PromptTokens, s.GenerationTokens) + } + if s.SuccessTotal != 34 { + t.Errorf("success=%v", s.SuccessTotal) + } +} + +func TestSGLangIsNotReadWithVLLMsNames(t *testing.T) { + // Read with vLLM's vocabulary, an SGLang replica serves none of the + // contract and the router schedules to it blind. + var missing *MissingError + if _, err := Parse(strings.NewReader(sglangMetrics), VLLMMapping()); !errors.As(err, &missing) { + t.Fatalf("vLLM's names found something in SGLang's metrics: %v", err) + } + if MappingFor("sglang").Running == VLLMMapping().Running { + t.Error("sglang falls back to vLLM's names") + } +} + +func TestAnSGLangEndpointIsMeasuredByThePoller(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/metrics" { + http.NotFound(w, r) + return + } + _, _ = w.Write([]byte(sglangMetrics)) + })) + defer srv.Close() + + reg := endpoint.NewRegistry(endpoint.Config{}) + reg.Replace([]endpoint.Info{{ID: "s1", URL: srv.URL, Engine: "sglang", APIKey: "k1", Models: []string{"m"}}}) + p := NewPoller(reg, Config{Interval: 20 * time.Millisecond, Timeout: 2 * time.Second}, nil) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + p.Sync(ctx) + defer p.Stop() + + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + for _, snap := range reg.Snapshots("m") { + if snap.ID == "s1" && snap.Stats.Running == 6 { + if snap.Stats.Waiting != 3 || snap.Stats.KVCacheUsage != 0.42 { + t.Errorf("stats read wrong: %+v", snap.Stats) + } + return + } + } + time.Sleep(10 * time.Millisecond) + } + t.Fatal("the endpoint was never measured") +}