SGLang read in its own metric names, started serving them, and asked for its own help

The router read SGLang replicas with vLLM's names, and SGLang was started
without --enable-metrics, so it served no /metrics: every scrape failed as
if unreachable and the replica was routed to blind. It now gets
--enable-metrics and a mapping of its own (sglang:num_running_reqs,
num_queue_reqs, token_usage, the token counters and num_requests_total, as
0.5.21 publishes them). engine-check asks python -m sglang.launch_server
--help instead of Python's own help, and SGLang is given --tp-size rather
than the --tp argparse only takes as an abbreviation.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

docs/install.md

@@ -597,6 +597,11 @@ engines:
   sglang: "0.5.20"    # SGLang only when named
 ```
 
+SGLang is started with `--enable-metrics`, without which it serves no
+`/metrics`; the router reads `sglang:num_running_reqs`, `sglang:num_queue_reqs`
+and `sglang:token_usage` there. Check an SGLang against the flags llmgw builds
+with `llmgw ctl engine-check sglang --bin <its environment>/bin/python`.
+
 A changed version reaches every node when it next connects. To use an engine
 you installed yourself on a node, name its path in that node's `agent.yaml`
 (`engines.llamacpp.bin`, `engines.vllm.bin`, `engines.sglang.python`,

internal/agent/engines/engines_test.go

@@ -366,6 +366,26 @@ func TestEnginesListenOnTheAddressTheyAreGiven(t *testing.T) {
 	}
 }
 
+func TestSGLangIsStartedServingItsMetrics(t *testing.T) {
+	// Without --enable-metrics SGLang has no /metrics, and every scrape of
+	// the replica fails as though it were unreachable.
+	if cmd := buildOne(t, &SGLang{}, singleNodeSpec()); !slices.Contains(cmd.Args, "--enable-metrics") {
+		t.Errorf("not started with its metrics: %v", cmd.Args)
+	}
+}
+
+func TestSGLangIsAskedForItsHelpThroughItsModule(t *testing.T) {
+	g := &SGLang{cfg: SGLangConfig{Python: "/opt/sglang/bin/python"}}
+	g.UseInstalled("/opt/llmgw/engines/sglang/0.5.21-cu13/bin/python")
+	path, args := g.HelpCommand(Command{})
+	if path != "/opt/llmgw/engines/sglang/0.5.21-cu13/bin/python" {
+		t.Errorf("asked %s, not the interpreter it was installed into", path)
+	}
+	if !slices.Equal(args, []string{"-m", "sglang.launch_server", "--help"}) {
+		t.Errorf("asked %v", args)
+	}
+}
+
 func TestSGLangSingleNodeOmitsMultiNodeFlags(t *testing.T) {
 	cmd := buildOne(t, &SGLang{}, singleNodeSpec())
 	for _, flag := range []string{"--nnodes", "--node-rank", "--dist-init-addr"} {

internal/agent/engines/sglang.go

@@ -40,7 +40,15 @@ func (*SGLang) Name() string { return "sglang" }
 // HealthPath is SGLang's readiness endpoint.
 func (*SGLang) HealthPath() string { return "/health" }
 
-// MetricsPath is SGLang's Prometheus endpoint.
+// HelpCommand is how to ask SGLang for its help: through its module, in the
+// environment it was installed into, since the command itself is llmgw's
+// launcher and the interpreter's own --help describes Python.
+func (g *SGLang) HelpCommand(Command) (string, []string) {
+	return g.python(), []string{"-m", "sglang.launch_server", "--help"}
+}
+
+// MetricsPath is SGLang's Prometheus endpoint, served only with
+// --enable-metrics.
 func (*SGLang) MetricsPath() string { return "/metrics" }
 
 // Validate checks the profile against the GPUs it would run on.
@@ -83,7 +91,13 @@ func (g *SGLang) serverCommand(s ShardSpec) (Command, error) {
 		"--model-path", p.ModelPath,
 		"--host", s.Host(),
 		"--port", strconv.Itoa(s.HTTPPort),
-		"--tp", strconv.Itoa(p.TP),
+		// The flag's whole name: SGLang's documentation writes --tp, which
+		// argparse accepts only as an abbreviation of --tp-size and only
+		// while no other flag begins with it.
+		"--tp-size", strconv.Itoa(p.TP),
+		// Without it SGLang serves no /metrics at all, and the router has
+		// nothing to schedule this replica by.
+		"--enable-metrics",
 	}
 	if p.ServedModelName != "" {
 		args = append(args, "--served-model-name", p.ServedModelName)

internal/cli/enginecheck_test.go

@@ -79,6 +79,13 @@ func TestATruncatedHelpIsSuspected(t *testing.T) {
 // trtllm-serve's help through its module, it prints the help given; asked
 // anything else, it fails, as running the launcher here would.
 func fakeInterpreter(t *testing.T, help string) string {
+	t.Helper()
+	return fakePython(t, "-m tensorrt_llm.commands.serve serve --help", help)
+}
+
+// fakePython is an interpreter that prints help when asked exactly ask, and
+// fails on anything else — the interpreter's own --help included.
+func fakePython(t *testing.T, ask, help string) string {
 	t.Helper()
 	dir := t.TempDir()
 	if err := os.WriteFile(filepath.Join(dir, "help.txt"), []byte(help), 0o644); err != nil {
@@ -86,7 +93,7 @@ func fakeInterpreter(t *testing.T, help string) string {
 	}
 	p := filepath.Join(dir, "python")
 	body := "#!/bin/sh\n" +
-		`if [ "$*" = "-m tensorrt_llm.commands.serve serve --help" ]; then cat "` + filepath.Join(dir, "help.txt") + `"; exit 0; fi` + "\n" +
+		`if [ "$*" = "` + ask + `" ]; then cat "` + filepath.Join(dir, "help.txt") + `"; exit 0; fi` + "\n" +
 		"echo \"unexpected: $*\" >&2; exit 2\n"
 	if err := os.WriteFile(p, []byte(body), 0o755); err != nil { //nolint:gosec // an executable stand-in
 		t.Fatal(err)
@@ -137,6 +144,57 @@ func TestEngineCheckAsksTrtllmServeThroughItsModule(t *testing.T) {
 	}
 }
 
+// sglangHelp is the part of python -m sglang.launch_server --help (0.5.21)
+// that holds the flags llmgw builds, in argparse's layout.
+const sglangHelp = `usage: sglang serve [-h] --model-path MODEL_PATH [--host HOST] [--port PORT]
+
+options:
+  -h, --help            show this help message and exit
+  --model-path MODEL_PATH, --model MODEL_PATH
+                        Path to model weights.
+  --host HOST           The host of the HTTP server.
+  --port PORT           The port of the HTTP server.
+  --served-model-name SERVED_MODEL_NAME
+                        Override the model name returned by the v1/models endpoint.
+  --enable-metrics      Enable log prometheus metrics.
+  --dist-init-addr DIST_INIT_ADDR, --nccl-init-addr DIST_INIT_ADDR
+                        The host address for initializing distributed backend.
+  --nnodes NNODES       The number of nodes.
+  --node-rank NODE_RANK
+                        The node rank.
+  --tp-size TP_SIZE, --tensor-parallel-size TP_SIZE
+                        The tensor parallelism size.
+`
+
+func TestEngineCheckAsksSGLangThroughItsModule(t *testing.T) {
+	// The command is the interpreter running llmgw's launcher; asked for
+	// --help it would describe Python, and every SGLang flag would look
+	// missing — or, against a real python3, be checked against nothing.
+	py := fakePython(t, "-m sglang.launch_server --help", sglangHelp)
+	out, err := runEngineCheck(t, "sglang", "--bin", py)
+	if err != nil {
+		t.Fatalf("%v\n%s", err, out)
+	}
+	if !strings.Contains(out, "every flag llmgw builds is accepted") {
+		t.Errorf("unexpected report:\n%s", out)
+	}
+	if strings.Contains(out, "unexpected:") || strings.Contains(out, "skipped") {
+		t.Errorf("the help was not asked of SGLang:\n%s", out)
+	}
+	// Two nodes by default, so the rendezvous flags were among those checked.
+	if !strings.Contains(out, "sglang: 9 flags checked") {
+		t.Errorf("not every flag was checked:\n%s", out)
+	}
+}
+
+func TestEngineCheckCatchesAnSGLangWithoutTheFlagForItsMetrics(t *testing.T) {
+	old := strings.ReplaceAll(sglangHelp, "  --enable-metrics      Enable log prometheus metrics.\n", "")
+	out, err := runEngineCheck(t, "sglang", "--bin", fakePython(t, "-m sglang.launch_server --help", old))
+	if err == nil || !strings.Contains(out, "sglang: --enable-metrics") {
+		t.Errorf("an SGLang without --enable-metrics passed: %v\n%s", err, out)
+	}
+}
+
 func TestEngineCheckCatchesATensorRTLLMTooOldForTheServedName(t *testing.T) {
 	// TensorRT-LLM 1.2 has no --served_model_name, and a replica of it would
 	// never be seen serving the name the agent waits for.

internal/router/scrape/parse.go

@@ -48,6 +48,8 @@ func MappingFor(engine string) Mapping {
 		return LlamaCppMapping()
 	case "trtllm":
 		return TRTLLMMapping()
+	case "sglang":
+		return SGLangMapping()
 	default:
 		return VLLMMapping()
 	}
@@ -106,6 +108,39 @@ func TRTLLMMapping() Mapping {
 	}
 }
 
+// SGLangMapping returns SGLang's metric names, as sglang.launch_server
+// publishes them under /metrics (checked against 0.5.21's collector).
+//
+// They are there only when the server runs with --enable-metrics, which
+// llmgw's launch command passes; without it /metrics is not served at all.
+// SGLang exempts /metrics from its API key, so the key the poller sends is
+// ignored rather than needed.
+//
+// "Queue" is SGLang's word for vLLM's waiting. The gauges are set by the
+// scheduler as it reports its batches, and at least every thirty seconds
+// while it is idle, so a server that has only just started may serve none of
+// them yet and is reported incomplete until it does.
+//
+// token_usage is the fraction of the KV pool in use, as vLLM's
+// kv_cache_usage_perc is (for a model with sliding-window layers, the fuller
+// of its two pools). cache_hit_rate is a different number — how much of the
+// prompts the prefix cache answered — and is not what saturation means.
+//
+// num_requests_total counts the requests that finished, the nearest thing it
+// has to vLLM's success counter; like the token counters it is split by
+// is_streaming and summed. There is no cache configuration to read block
+// sizes from.
+func SGLangMapping() Mapping {
+	return Mapping{
+		Running:          "sglang:num_running_reqs",
+		Waiting:          "sglang:num_queue_reqs",
+		KVCacheUsage:     "sglang:token_usage",
+		PromptTokens:     "sglang:prompt_tokens_total",
+		GenerationTokens: "sglang:generation_tokens_total",
+		SuccessTotal:     "sglang:num_requests_total",
+	}
+}
+
 // VLLMMapping returns vLLM's current metric names.
 func VLLMMapping() Mapping {
 	return Mapping{

internal/router/scrape/sglang_test.go

@@ -0,0 +1,116 @@
+package scrape
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
+)
+
+// sglangMetrics is what sglang.launch_server --enable-metrics serves on
+// /metrics (0.5.21), trimmed to the families that matter, one it must not be
+// mistaken for, and a histogram the parser has to step over. The scheduler's
+// gauges carry its ranks; the tokenizer's counters are split by whether the
+// request streamed.
+const sglangMetrics = `# HELP sglang:num_running_reqs The number of running requests.
+# TYPE sglang:num_running_reqs gauge
+sglang:num_running_reqs{engine_type="unified",model_name="qwen3-8b",moe_ep_rank="0",pp_rank="0",tp_rank="0"} 6.0
+# HELP sglang:num_queue_reqs The number of requests in the waiting queue.
+# TYPE sglang:num_queue_reqs gauge
+sglang:num_queue_reqs{engine_type="unified",model_name="qwen3-8b",moe_ep_rank="0",pp_rank="0",tp_rank="0"} 3.0
+# HELP sglang:cache_hit_rate The prefix cache hit rate.
+# TYPE sglang:cache_hit_rate gauge
+sglang:cache_hit_rate{engine_type="unified",model_name="qwen3-8b",moe_ep_rank="0",pp_rank="0",tp_rank="0"} 0.9
+# HELP sglang:token_usage The token usage.
+# TYPE sglang:token_usage gauge
+sglang:token_usage{engine_type="unified",model_name="qwen3-8b",moe_ep_rank="0",pp_rank="0",tp_rank="0"} 0.42
+# HELP sglang:prompt_tokens_total Number of prefill tokens processed.
+# TYPE sglang:prompt_tokens_total counter
+sglang:prompt_tokens_total{engine_type="unified",is_streaming="true",model_name="qwen3-8b"} 12000.0
+sglang:prompt_tokens_total{engine_type="unified",is_streaming="false",model_name="qwen3-8b"} 345.0
+# HELP sglang:generation_tokens_total Number of generation tokens processed.
+# TYPE sglang:generation_tokens_total counter
+sglang:generation_tokens_total{engine_type="unified",is_streaming="true",model_name="qwen3-8b"} 8000.0
+sglang:generation_tokens_total{engine_type="unified",is_streaming="false",model_name="qwen3-8b"} 64.0
+# HELP sglang:num_requests_total Number of requests processed.
+# TYPE sglang:num_requests_total counter
+sglang:num_requests_total{engine_type="unified",is_streaming="true",model_name="qwen3-8b"} 30.0
+sglang:num_requests_total{engine_type="unified",is_streaming="false",model_name="qwen3-8b"} 4.0
+# HELP sglang:e2e_request_latency_seconds Histogram of End-to-end request latency in seconds
+# TYPE sglang:e2e_request_latency_seconds histogram
+sglang:e2e_request_latency_seconds_bucket{engine_type="unified",is_streaming="true",le="0.5",model_name="qwen3-8b"} 12.0
+sglang:e2e_request_latency_seconds_bucket{engine_type="unified",is_streaming="true",le="+Inf",model_name="qwen3-8b"} 30.0
+sglang:e2e_request_latency_seconds_count{engine_type="unified",is_streaming="true",model_name="qwen3-8b"} 30.0
+sglang:e2e_request_latency_seconds_sum{engine_type="unified",is_streaming="true",model_name="qwen3-8b"} 41.2
+`
+
+func TestSGLangMetricsSatisfyTheContract(t *testing.T) {
+	s, err := Parse(strings.NewReader(sglangMetrics), MappingFor("sglang"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if s.Running != 6 || s.Waiting != 3 {
+		t.Errorf("running=%d waiting=%d", s.Running, s.Waiting)
+	}
+	// The pool's fill, not the prefix cache's hit rate beside it.
+	if s.KVCacheUsage != 0.42 {
+		t.Errorf("kv=%v", s.KVCacheUsage)
+	}
+	// Split by is_streaming, summed.
+	if s.PromptTokens != 12345 || s.GenerationTokens != 8064 {
+		t.Errorf("tokens=%v/%v", s.PromptTokens, s.GenerationTokens)
+	}
+	if s.SuccessTotal != 34 {
+		t.Errorf("success=%v", s.SuccessTotal)
+	}
+}
+
+func TestSGLangIsNotReadWithVLLMsNames(t *testing.T) {
+	// Read with vLLM's vocabulary, an SGLang replica serves none of the
+	// contract and the router schedules to it blind.
+	var missing *MissingError
+	if _, err := Parse(strings.NewReader(sglangMetrics), VLLMMapping()); !errors.As(err, &missing) {
+		t.Fatalf("vLLM's names found something in SGLang's metrics: %v", err)
+	}
+	if MappingFor("sglang").Running == VLLMMapping().Running {
+		t.Error("sglang falls back to vLLM's names")
+	}
+}
+
+func TestAnSGLangEndpointIsMeasuredByThePoller(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/metrics" {
+			http.NotFound(w, r)
+			return
+		}
+		_, _ = w.Write([]byte(sglangMetrics))
+	}))
+	defer srv.Close()
+
+	reg := endpoint.NewRegistry(endpoint.Config{})
+	reg.Replace([]endpoint.Info{{ID: "s1", URL: srv.URL, Engine: "sglang", APIKey: "k1", Models: []string{"m"}}})
+	p := NewPoller(reg, Config{Interval: 20 * time.Millisecond, Timeout: 2 * time.Second}, nil)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	p.Sync(ctx)
+	defer p.Stop()
+
+	deadline := time.Now().Add(2 * time.Second)
+	for time.Now().Before(deadline) {
+		for _, snap := range reg.Snapshots("m") {
+			if snap.ID == "s1" && snap.Stats.Running == 6 {
+				if snap.Stats.Waiting != 3 || snap.Stats.KVCacheUsage != 0.42 {
+					t.Errorf("stats read wrong: %+v", snap.Stats)
+				}
+				return
+			}
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	t.Fatal("the endpoint was never measured")
+}