Merge engine fixes: SGLang metrics and flags, bench reading guarded metrics
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
commit
cc4de36674
12 files changed
+655
-28
No files matched your search
@@ -48,6 +48,8 @@ func MappingFor(engine string) Mapping {
|
||||
return LlamaCppMapping()
|
||||
case "trtllm":
|
||||
return TRTLLMMapping()
|
||||
case "sglang":
|
||||
return SGLangMapping()
|
||||
default:
|
||||
return VLLMMapping()
|
||||
}
|
||||
@@ -106,6 +108,39 @@ func TRTLLMMapping() Mapping {
|
||||
}
|
||||
}
|
||||
|
||||
// SGLangMapping returns SGLang's metric names, as sglang.launch_server
|
||||
// publishes them under /metrics (checked against 0.5.21's collector).
|
||||
//
|
||||
// They are there only when the server runs with --enable-metrics, which
|
||||
// llmgw's launch command passes; without it /metrics is not served at all.
|
||||
// SGLang exempts /metrics from its API key, so the key the poller sends is
|
||||
// ignored rather than needed.
|
||||
//
|
||||
// "Queue" is SGLang's word for vLLM's waiting. The gauges are set by the
|
||||
// scheduler as it reports its batches, and at least every thirty seconds
|
||||
// while it is idle, so a server that has only just started may serve none of
|
||||
// them yet and is reported incomplete until it does.
|
||||
//
|
||||
// token_usage is the fraction of the KV pool in use, as vLLM's
|
||||
// kv_cache_usage_perc is (for a model with sliding-window layers, the fuller
|
||||
// of its two pools). cache_hit_rate is a different number — how much of the
|
||||
// prompts the prefix cache answered — and is not what saturation means.
|
||||
//
|
||||
// num_requests_total counts the requests that finished, the nearest thing it
|
||||
// has to vLLM's success counter; like the token counters it is split by
|
||||
// is_streaming and summed. There is no cache configuration to read block
|
||||
// sizes from.
|
||||
func SGLangMapping() Mapping {
|
||||
return Mapping{
|
||||
Running: "sglang:num_running_reqs",
|
||||
Waiting: "sglang:num_queue_reqs",
|
||||
KVCacheUsage: "sglang:token_usage",
|
||||
PromptTokens: "sglang:prompt_tokens_total",
|
||||
GenerationTokens: "sglang:generation_tokens_total",
|
||||
SuccessTotal: "sglang:num_requests_total",
|
||||
}
|
||||
}
|
||||
|
||||
// VLLMMapping returns vLLM's current metric names.
|
||||
func VLLMMapping() Mapping {
|
||||
return Mapping{
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
package scrape
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
||||
)
|
||||
|
||||
// sglangMetrics is what sglang.launch_server --enable-metrics serves on
|
||||
// /metrics (0.5.21), trimmed to the families that matter, one it must not be
|
||||
// mistaken for, and a histogram the parser has to step over. The scheduler's
|
||||
// gauges carry its ranks; the tokenizer's counters are split by whether the
|
||||
// request streamed.
|
||||
const sglangMetrics = `# HELP sglang:num_running_reqs The number of running requests.
|
||||
# TYPE sglang:num_running_reqs gauge
|
||||
sglang:num_running_reqs{engine_type="unified",model_name="qwen3-8b",moe_ep_rank="0",pp_rank="0",tp_rank="0"} 6.0
|
||||
# HELP sglang:num_queue_reqs The number of requests in the waiting queue.
|
||||
# TYPE sglang:num_queue_reqs gauge
|
||||
sglang:num_queue_reqs{engine_type="unified",model_name="qwen3-8b",moe_ep_rank="0",pp_rank="0",tp_rank="0"} 3.0
|
||||
# HELP sglang:cache_hit_rate The prefix cache hit rate.
|
||||
# TYPE sglang:cache_hit_rate gauge
|
||||
sglang:cache_hit_rate{engine_type="unified",model_name="qwen3-8b",moe_ep_rank="0",pp_rank="0",tp_rank="0"} 0.9
|
||||
# HELP sglang:token_usage The token usage.
|
||||
# TYPE sglang:token_usage gauge
|
||||
sglang:token_usage{engine_type="unified",model_name="qwen3-8b",moe_ep_rank="0",pp_rank="0",tp_rank="0"} 0.42
|
||||
# HELP sglang:prompt_tokens_total Number of prefill tokens processed.
|
||||
# TYPE sglang:prompt_tokens_total counter
|
||||
sglang:prompt_tokens_total{engine_type="unified",is_streaming="true",model_name="qwen3-8b"} 12000.0
|
||||
sglang:prompt_tokens_total{engine_type="unified",is_streaming="false",model_name="qwen3-8b"} 345.0
|
||||
# HELP sglang:generation_tokens_total Number of generation tokens processed.
|
||||
# TYPE sglang:generation_tokens_total counter
|
||||
sglang:generation_tokens_total{engine_type="unified",is_streaming="true",model_name="qwen3-8b"} 8000.0
|
||||
sglang:generation_tokens_total{engine_type="unified",is_streaming="false",model_name="qwen3-8b"} 64.0
|
||||
# HELP sglang:num_requests_total Number of requests processed.
|
||||
# TYPE sglang:num_requests_total counter
|
||||
sglang:num_requests_total{engine_type="unified",is_streaming="true",model_name="qwen3-8b"} 30.0
|
||||
sglang:num_requests_total{engine_type="unified",is_streaming="false",model_name="qwen3-8b"} 4.0
|
||||
# HELP sglang:e2e_request_latency_seconds Histogram of End-to-end request latency in seconds
|
||||
# TYPE sglang:e2e_request_latency_seconds histogram
|
||||
sglang:e2e_request_latency_seconds_bucket{engine_type="unified",is_streaming="true",le="0.5",model_name="qwen3-8b"} 12.0
|
||||
sglang:e2e_request_latency_seconds_bucket{engine_type="unified",is_streaming="true",le="+Inf",model_name="qwen3-8b"} 30.0
|
||||
sglang:e2e_request_latency_seconds_count{engine_type="unified",is_streaming="true",model_name="qwen3-8b"} 30.0
|
||||
sglang:e2e_request_latency_seconds_sum{engine_type="unified",is_streaming="true",model_name="qwen3-8b"} 41.2
|
||||
`
|
||||
|
||||
func TestSGLangMetricsSatisfyTheContract(t *testing.T) {
|
||||
s, err := Parse(strings.NewReader(sglangMetrics), MappingFor("sglang"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if s.Running != 6 || s.Waiting != 3 {
|
||||
t.Errorf("running=%d waiting=%d", s.Running, s.Waiting)
|
||||
}
|
||||
// The pool's fill, not the prefix cache's hit rate beside it.
|
||||
if s.KVCacheUsage != 0.42 {
|
||||
t.Errorf("kv=%v", s.KVCacheUsage)
|
||||
}
|
||||
// Split by is_streaming, summed.
|
||||
if s.PromptTokens != 12345 || s.GenerationTokens != 8064 {
|
||||
t.Errorf("tokens=%v/%v", s.PromptTokens, s.GenerationTokens)
|
||||
}
|
||||
if s.SuccessTotal != 34 {
|
||||
t.Errorf("success=%v", s.SuccessTotal)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSGLangIsNotReadWithVLLMsNames(t *testing.T) {
|
||||
// Read with vLLM's vocabulary, an SGLang replica serves none of the
|
||||
// contract and the router schedules to it blind.
|
||||
var missing *MissingError
|
||||
if _, err := Parse(strings.NewReader(sglangMetrics), VLLMMapping()); !errors.As(err, &missing) {
|
||||
t.Fatalf("vLLM's names found something in SGLang's metrics: %v", err)
|
||||
}
|
||||
if MappingFor("sglang").Running == VLLMMapping().Running {
|
||||
t.Error("sglang falls back to vLLM's names")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnSGLangEndpointIsMeasuredByThePoller(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/metrics" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte(sglangMetrics))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
reg := endpoint.NewRegistry(endpoint.Config{})
|
||||
reg.Replace([]endpoint.Info{{ID: "s1", URL: srv.URL, Engine: "sglang", APIKey: "k1", Models: []string{"m"}}})
|
||||
p := NewPoller(reg, Config{Interval: 20 * time.Millisecond, Timeout: 2 * time.Second}, nil)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
p.Sync(ctx)
|
||||
defer p.Stop()
|
||||
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
for _, snap := range reg.Snapshots("m") {
|
||||
if snap.ID == "s1" && snap.Stats.Running == 6 {
|
||||
if snap.Stats.Waiting != 3 || snap.Stats.KVCacheUsage != 0.42 {
|
||||
t.Errorf("stats read wrong: %+v", snap.Stats)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
t.Fatal("the endpoint was never measured")
|
||||
}
|
||||
Reference in new issue
Block a user