package scrape import ( "context" "errors" "net/http" "net/http/httptest" "strings" "testing" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint" ) // sglangMetrics is what sglang.launch_server --enable-metrics serves on // /metrics (0.5.21), trimmed to the families that matter, one it must not be // mistaken for, and a histogram the parser has to step over. The scheduler's // gauges carry its ranks; the tokenizer's counters are split by whether the // request streamed. const sglangMetrics = `# HELP sglang:num_running_reqs The number of running requests. # TYPE sglang:num_running_reqs gauge sglang:num_running_reqs{engine_type="unified",model_name="qwen3-8b",moe_ep_rank="0",pp_rank="0",tp_rank="0"} 6.0 # HELP sglang:num_queue_reqs The number of requests in the waiting queue. # TYPE sglang:num_queue_reqs gauge sglang:num_queue_reqs{engine_type="unified",model_name="qwen3-8b",moe_ep_rank="0",pp_rank="0",tp_rank="0"} 3.0 # HELP sglang:cache_hit_rate The prefix cache hit rate. # TYPE sglang:cache_hit_rate gauge sglang:cache_hit_rate{engine_type="unified",model_name="qwen3-8b",moe_ep_rank="0",pp_rank="0",tp_rank="0"} 0.9 # HELP sglang:token_usage The token usage. # TYPE sglang:token_usage gauge sglang:token_usage{engine_type="unified",model_name="qwen3-8b",moe_ep_rank="0",pp_rank="0",tp_rank="0"} 0.42 # HELP sglang:prompt_tokens_total Number of prefill tokens processed. # TYPE sglang:prompt_tokens_total counter sglang:prompt_tokens_total{engine_type="unified",is_streaming="true",model_name="qwen3-8b"} 12000.0 sglang:prompt_tokens_total{engine_type="unified",is_streaming="false",model_name="qwen3-8b"} 345.0 # HELP sglang:generation_tokens_total Number of generation tokens processed. # TYPE sglang:generation_tokens_total counter sglang:generation_tokens_total{engine_type="unified",is_streaming="true",model_name="qwen3-8b"} 8000.0 sglang:generation_tokens_total{engine_type="unified",is_streaming="false",model_name="qwen3-8b"} 64.0 # HELP sglang:num_requests_total Number of requests processed. # TYPE sglang:num_requests_total counter sglang:num_requests_total{engine_type="unified",is_streaming="true",model_name="qwen3-8b"} 30.0 sglang:num_requests_total{engine_type="unified",is_streaming="false",model_name="qwen3-8b"} 4.0 # HELP sglang:e2e_request_latency_seconds Histogram of End-to-end request latency in seconds # TYPE sglang:e2e_request_latency_seconds histogram sglang:e2e_request_latency_seconds_bucket{engine_type="unified",is_streaming="true",le="0.5",model_name="qwen3-8b"} 12.0 sglang:e2e_request_latency_seconds_bucket{engine_type="unified",is_streaming="true",le="+Inf",model_name="qwen3-8b"} 30.0 sglang:e2e_request_latency_seconds_count{engine_type="unified",is_streaming="true",model_name="qwen3-8b"} 30.0 sglang:e2e_request_latency_seconds_sum{engine_type="unified",is_streaming="true",model_name="qwen3-8b"} 41.2 ` func TestSGLangMetricsSatisfyTheContract(t *testing.T) { s, err := Parse(strings.NewReader(sglangMetrics), MappingFor("sglang")) if err != nil { t.Fatal(err) } if s.Running != 6 || s.Waiting != 3 { t.Errorf("running=%d waiting=%d", s.Running, s.Waiting) } // The pool's fill, not the prefix cache's hit rate beside it. if s.KVCacheUsage != 0.42 { t.Errorf("kv=%v", s.KVCacheUsage) } // Split by is_streaming, summed. if s.PromptTokens != 12345 || s.GenerationTokens != 8064 { t.Errorf("tokens=%v/%v", s.PromptTokens, s.GenerationTokens) } if s.SuccessTotal != 34 { t.Errorf("success=%v", s.SuccessTotal) } } func TestSGLangIsNotReadWithVLLMsNames(t *testing.T) { // Read with vLLM's vocabulary, an SGLang replica serves none of the // contract and the router schedules to it blind. var missing *MissingError if _, err := Parse(strings.NewReader(sglangMetrics), VLLMMapping()); !errors.As(err, &missing) { t.Fatalf("vLLM's names found something in SGLang's metrics: %v", err) } if MappingFor("sglang").Running == VLLMMapping().Running { t.Error("sglang falls back to vLLM's names") } } func TestAnSGLangEndpointIsMeasuredByThePoller(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/metrics" { http.NotFound(w, r) return } _, _ = w.Write([]byte(sglangMetrics)) })) defer srv.Close() reg := endpoint.NewRegistry(endpoint.Config{}) reg.Replace([]endpoint.Info{{ID: "s1", URL: srv.URL, Engine: "sglang", APIKey: "k1", Models: []string{"m"}}}) p := NewPoller(reg, Config{Interval: 20 * time.Millisecond, Timeout: 2 * time.Second}, nil) ctx, cancel := context.WithCancel(context.Background()) defer cancel() p.Sync(ctx) defer p.Stop() deadline := time.Now().Add(2 * time.Second) for time.Now().Before(deadline) { for _, snap := range reg.Snapshots("m") { if snap.ID == "s1" && snap.Stats.Running == 6 { if snap.Stats.Waiting != 3 || snap.Stats.KVCacheUsage != 0.42 { t.Errorf("stats read wrong: %+v", snap.Stats) } return } } time.Sleep(10 * time.Millisecond) } t.Fatal("the endpoint was never measured") }