Ilker ManapandClaude Opus 5.5
8cc8d7139b
The router read SGLang replicas with vLLM's names, and SGLang was started without --enable-metrics, so it served no /metrics: every scrape failed as if unreachable and the replica was routed to blind. It now gets --enable-metrics and a mapping of its own (sglang:num_running_reqs, num_queue_reqs, token_usage, the token counters and num_requests_total, as 0.5.21 publishes them). engine-check asks python -m sglang.launch_server --help instead of Python's own help, and SGLang is given --tp-size rather than the --tp argparse only takes as an abbreviation. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
117 lines
4.9 KiB
Go
117 lines
4.9 KiB
Go
package scrape
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
|
)
|
|
|
|
// sglangMetrics is what sglang.launch_server --enable-metrics serves on
|
|
// /metrics (0.5.21), trimmed to the families that matter, one it must not be
|
|
// mistaken for, and a histogram the parser has to step over. The scheduler's
|
|
// gauges carry its ranks; the tokenizer's counters are split by whether the
|
|
// request streamed.
|
|
const sglangMetrics = `# HELP sglang:num_running_reqs The number of running requests.
|
|
# TYPE sglang:num_running_reqs gauge
|
|
sglang:num_running_reqs{engine_type="unified",model_name="qwen3-8b",moe_ep_rank="0",pp_rank="0",tp_rank="0"} 6.0
|
|
# HELP sglang:num_queue_reqs The number of requests in the waiting queue.
|
|
# TYPE sglang:num_queue_reqs gauge
|
|
sglang:num_queue_reqs{engine_type="unified",model_name="qwen3-8b",moe_ep_rank="0",pp_rank="0",tp_rank="0"} 3.0
|
|
# HELP sglang:cache_hit_rate The prefix cache hit rate.
|
|
# TYPE sglang:cache_hit_rate gauge
|
|
sglang:cache_hit_rate{engine_type="unified",model_name="qwen3-8b",moe_ep_rank="0",pp_rank="0",tp_rank="0"} 0.9
|
|
# HELP sglang:token_usage The token usage.
|
|
# TYPE sglang:token_usage gauge
|
|
sglang:token_usage{engine_type="unified",model_name="qwen3-8b",moe_ep_rank="0",pp_rank="0",tp_rank="0"} 0.42
|
|
# HELP sglang:prompt_tokens_total Number of prefill tokens processed.
|
|
# TYPE sglang:prompt_tokens_total counter
|
|
sglang:prompt_tokens_total{engine_type="unified",is_streaming="true",model_name="qwen3-8b"} 12000.0
|
|
sglang:prompt_tokens_total{engine_type="unified",is_streaming="false",model_name="qwen3-8b"} 345.0
|
|
# HELP sglang:generation_tokens_total Number of generation tokens processed.
|
|
# TYPE sglang:generation_tokens_total counter
|
|
sglang:generation_tokens_total{engine_type="unified",is_streaming="true",model_name="qwen3-8b"} 8000.0
|
|
sglang:generation_tokens_total{engine_type="unified",is_streaming="false",model_name="qwen3-8b"} 64.0
|
|
# HELP sglang:num_requests_total Number of requests processed.
|
|
# TYPE sglang:num_requests_total counter
|
|
sglang:num_requests_total{engine_type="unified",is_streaming="true",model_name="qwen3-8b"} 30.0
|
|
sglang:num_requests_total{engine_type="unified",is_streaming="false",model_name="qwen3-8b"} 4.0
|
|
# HELP sglang:e2e_request_latency_seconds Histogram of End-to-end request latency in seconds
|
|
# TYPE sglang:e2e_request_latency_seconds histogram
|
|
sglang:e2e_request_latency_seconds_bucket{engine_type="unified",is_streaming="true",le="0.5",model_name="qwen3-8b"} 12.0
|
|
sglang:e2e_request_latency_seconds_bucket{engine_type="unified",is_streaming="true",le="+Inf",model_name="qwen3-8b"} 30.0
|
|
sglang:e2e_request_latency_seconds_count{engine_type="unified",is_streaming="true",model_name="qwen3-8b"} 30.0
|
|
sglang:e2e_request_latency_seconds_sum{engine_type="unified",is_streaming="true",model_name="qwen3-8b"} 41.2
|
|
`
|
|
|
|
func TestSGLangMetricsSatisfyTheContract(t *testing.T) {
|
|
s, err := Parse(strings.NewReader(sglangMetrics), MappingFor("sglang"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if s.Running != 6 || s.Waiting != 3 {
|
|
t.Errorf("running=%d waiting=%d", s.Running, s.Waiting)
|
|
}
|
|
// The pool's fill, not the prefix cache's hit rate beside it.
|
|
if s.KVCacheUsage != 0.42 {
|
|
t.Errorf("kv=%v", s.KVCacheUsage)
|
|
}
|
|
// Split by is_streaming, summed.
|
|
if s.PromptTokens != 12345 || s.GenerationTokens != 8064 {
|
|
t.Errorf("tokens=%v/%v", s.PromptTokens, s.GenerationTokens)
|
|
}
|
|
if s.SuccessTotal != 34 {
|
|
t.Errorf("success=%v", s.SuccessTotal)
|
|
}
|
|
}
|
|
|
|
func TestSGLangIsNotReadWithVLLMsNames(t *testing.T) {
|
|
// Read with vLLM's vocabulary, an SGLang replica serves none of the
|
|
// contract and the router schedules to it blind.
|
|
var missing *MissingError
|
|
if _, err := Parse(strings.NewReader(sglangMetrics), VLLMMapping()); !errors.As(err, &missing) {
|
|
t.Fatalf("vLLM's names found something in SGLang's metrics: %v", err)
|
|
}
|
|
if MappingFor("sglang").Running == VLLMMapping().Running {
|
|
t.Error("sglang falls back to vLLM's names")
|
|
}
|
|
}
|
|
|
|
func TestAnSGLangEndpointIsMeasuredByThePoller(t *testing.T) {
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path != "/metrics" {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
_, _ = w.Write([]byte(sglangMetrics))
|
|
}))
|
|
defer srv.Close()
|
|
|
|
reg := endpoint.NewRegistry(endpoint.Config{})
|
|
reg.Replace([]endpoint.Info{{ID: "s1", URL: srv.URL, Engine: "sglang", APIKey: "k1", Models: []string{"m"}}})
|
|
p := NewPoller(reg, Config{Interval: 20 * time.Millisecond, Timeout: 2 * time.Second}, nil)
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
p.Sync(ctx)
|
|
defer p.Stop()
|
|
|
|
deadline := time.Now().Add(2 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
for _, snap := range reg.Snapshots("m") {
|
|
if snap.ID == "s1" && snap.Stats.Running == 6 {
|
|
if snap.Stats.Waiting != 3 || snap.Stats.KVCacheUsage != 0.42 {
|
|
t.Errorf("stats read wrong: %+v", snap.Stats)
|
|
}
|
|
return
|
|
}
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
t.Fatal("the endpoint was never measured")
|
|
}
|