SGLang read in its own metric names, started serving them, and asked for its own help

The router read SGLang replicas with vLLM's names, and SGLang was started
without --enable-metrics, so it served no /metrics: every scrape failed as
if unreachable and the replica was routed to blind. It now gets
--enable-metrics and a mapping of its own (sglang:num_running_reqs,
num_queue_reqs, token_usage, the token counters and num_requests_total, as
0.5.21 publishes them). engine-check asks python -m sglang.launch_server
--help instead of Python's own help, and SGLang is given --tp-size rather
than the --tp argparse only takes as an abbreviation.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 00:20:25 +02:00
1 parent d41104a452
commit 8cc8d7139b
6 files changed
+251 -3

No files matched your search

+35
View File
@@ -48,6 +48,8 @@ func MappingFor(engine string) Mapping {
return LlamaCppMapping()
case "trtllm":
return TRTLLMMapping()
case "sglang":
return SGLangMapping()
default:
return VLLMMapping()
}
@@ -106,6 +108,39 @@ func TRTLLMMapping() Mapping {
}
}
// SGLangMapping returns SGLang's metric names, as sglang.launch_server
// publishes them under /metrics (checked against 0.5.21's collector).
//
// They are there only when the server runs with --enable-metrics, which
// llmgw's launch command passes; without it /metrics is not served at all.
// SGLang exempts /metrics from its API key, so the key the poller sends is
// ignored rather than needed.
//
// "Queue" is SGLang's word for vLLM's waiting. The gauges are set by the
// scheduler as it reports its batches, and at least every thirty seconds
// while it is idle, so a server that has only just started may serve none of
// them yet and is reported incomplete until it does.
//
// token_usage is the fraction of the KV pool in use, as vLLM's
// kv_cache_usage_perc is (for a model with sliding-window layers, the fuller
// of its two pools). cache_hit_rate is a different number — how much of the
// prompts the prefix cache answered — and is not what saturation means.
//
// num_requests_total counts the requests that finished, the nearest thing it
// has to vLLM's success counter; like the token counters it is split by
// is_streaming and summed. There is no cache configuration to read block
// sizes from.
func SGLangMapping() Mapping {
return Mapping{
Running: "sglang:num_running_reqs",
Waiting: "sglang:num_queue_reqs",
KVCacheUsage: "sglang:token_usage",
PromptTokens: "sglang:prompt_tokens_total",
GenerationTokens: "sglang:generation_tokens_total",
SuccessTotal: "sglang:num_requests_total",
}
}
// VLLMMapping returns vLLM's current metric names.
func VLLMMapping() Mapping {
return Mapping{
+116
View File
@@ -0,0 +1,116 @@
package scrape
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
)
// sglangMetrics is what sglang.launch_server --enable-metrics serves on
// /metrics (0.5.21), trimmed to the families that matter, one it must not be
// mistaken for, and a histogram the parser has to step over. The scheduler's
// gauges carry its ranks; the tokenizer's counters are split by whether the
// request streamed.
const sglangMetrics = `# HELP sglang:num_running_reqs The number of running requests.
# TYPE sglang:num_running_reqs gauge
sglang:num_running_reqs{engine_type="unified",model_name="qwen3-8b",moe_ep_rank="0",pp_rank="0",tp_rank="0"} 6.0
# HELP sglang:num_queue_reqs The number of requests in the waiting queue.
# TYPE sglang:num_queue_reqs gauge
sglang:num_queue_reqs{engine_type="unified",model_name="qwen3-8b",moe_ep_rank="0",pp_rank="0",tp_rank="0"} 3.0
# HELP sglang:cache_hit_rate The prefix cache hit rate.
# TYPE sglang:cache_hit_rate gauge
sglang:cache_hit_rate{engine_type="unified",model_name="qwen3-8b",moe_ep_rank="0",pp_rank="0",tp_rank="0"} 0.9
# HELP sglang:token_usage The token usage.
# TYPE sglang:token_usage gauge
sglang:token_usage{engine_type="unified",model_name="qwen3-8b",moe_ep_rank="0",pp_rank="0",tp_rank="0"} 0.42
# HELP sglang:prompt_tokens_total Number of prefill tokens processed.
# TYPE sglang:prompt_tokens_total counter
sglang:prompt_tokens_total{engine_type="unified",is_streaming="true",model_name="qwen3-8b"} 12000.0
sglang:prompt_tokens_total{engine_type="unified",is_streaming="false",model_name="qwen3-8b"} 345.0
# HELP sglang:generation_tokens_total Number of generation tokens processed.
# TYPE sglang:generation_tokens_total counter
sglang:generation_tokens_total{engine_type="unified",is_streaming="true",model_name="qwen3-8b"} 8000.0
sglang:generation_tokens_total{engine_type="unified",is_streaming="false",model_name="qwen3-8b"} 64.0
# HELP sglang:num_requests_total Number of requests processed.
# TYPE sglang:num_requests_total counter
sglang:num_requests_total{engine_type="unified",is_streaming="true",model_name="qwen3-8b"} 30.0
sglang:num_requests_total{engine_type="unified",is_streaming="false",model_name="qwen3-8b"} 4.0
# HELP sglang:e2e_request_latency_seconds Histogram of End-to-end request latency in seconds
# TYPE sglang:e2e_request_latency_seconds histogram
sglang:e2e_request_latency_seconds_bucket{engine_type="unified",is_streaming="true",le="0.5",model_name="qwen3-8b"} 12.0
sglang:e2e_request_latency_seconds_bucket{engine_type="unified",is_streaming="true",le="+Inf",model_name="qwen3-8b"} 30.0
sglang:e2e_request_latency_seconds_count{engine_type="unified",is_streaming="true",model_name="qwen3-8b"} 30.0
sglang:e2e_request_latency_seconds_sum{engine_type="unified",is_streaming="true",model_name="qwen3-8b"} 41.2
`
func TestSGLangMetricsSatisfyTheContract(t *testing.T) {
s, err := Parse(strings.NewReader(sglangMetrics), MappingFor("sglang"))
if err != nil {
t.Fatal(err)
}
if s.Running != 6 || s.Waiting != 3 {
t.Errorf("running=%d waiting=%d", s.Running, s.Waiting)
}
// The pool's fill, not the prefix cache's hit rate beside it.
if s.KVCacheUsage != 0.42 {
t.Errorf("kv=%v", s.KVCacheUsage)
}
// Split by is_streaming, summed.
if s.PromptTokens != 12345 || s.GenerationTokens != 8064 {
t.Errorf("tokens=%v/%v", s.PromptTokens, s.GenerationTokens)
}
if s.SuccessTotal != 34 {
t.Errorf("success=%v", s.SuccessTotal)
}
}
func TestSGLangIsNotReadWithVLLMsNames(t *testing.T) {
// Read with vLLM's vocabulary, an SGLang replica serves none of the
// contract and the router schedules to it blind.
var missing *MissingError
if _, err := Parse(strings.NewReader(sglangMetrics), VLLMMapping()); !errors.As(err, &missing) {
t.Fatalf("vLLM's names found something in SGLang's metrics: %v", err)
}
if MappingFor("sglang").Running == VLLMMapping().Running {
t.Error("sglang falls back to vLLM's names")
}
}
func TestAnSGLangEndpointIsMeasuredByThePoller(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/metrics" {
http.NotFound(w, r)
return
}
_, _ = w.Write([]byte(sglangMetrics))
}))
defer srv.Close()
reg := endpoint.NewRegistry(endpoint.Config{})
reg.Replace([]endpoint.Info{{ID: "s1", URL: srv.URL, Engine: "sglang", APIKey: "k1", Models: []string{"m"}}})
p := NewPoller(reg, Config{Interval: 20 * time.Millisecond, Timeout: 2 * time.Second}, nil)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
p.Sync(ctx)
defer p.Stop()
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
for _, snap := range reg.Snapshots("m") {
if snap.ID == "s1" && snap.Stats.Running == 6 {
if snap.Stats.Waiting != 3 || snap.Stats.KVCacheUsage != 0.42 {
t.Errorf("stats read wrong: %+v", snap.Stats)
}
return
}
}
time.Sleep(10 * time.Millisecond)
}
t.Fatal("the endpoint was never measured")
}