SGLang read in its own metric names, started serving them, and asked for its own help
The router read SGLang replicas with vLLM's names, and SGLang was started without --enable-metrics, so it served no /metrics: every scrape failed as if unreachable and the replica was routed to blind. It now gets --enable-metrics and a mapping of its own (sglang:num_running_reqs, num_queue_reqs, token_usage, the token counters and num_requests_total, as 0.5.21 publishes them). engine-check asks python -m sglang.launch_server --help instead of Python's own help, and SGLang is given --tp-size rather than the --tp argparse only takes as an abbreviation. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d41104a452
commit
8cc8d7139b
6 files changed
+251
-3
No files matched your search
@@ -597,6 +597,11 @@ engines:
|
|||||||
sglang: "0.5.20" # SGLang only when named
|
sglang: "0.5.20" # SGLang only when named
|
||||||
```
|
```
|
||||||
|
|
||||||
|
SGLang is started with `--enable-metrics`, without which it serves no
|
||||||
|
`/metrics`; the router reads `sglang:num_running_reqs`, `sglang:num_queue_reqs`
|
||||||
|
and `sglang:token_usage` there. Check an SGLang against the flags llmgw builds
|
||||||
|
with `llmgw ctl engine-check sglang --bin <its environment>/bin/python`.
|
||||||
|
|
||||||
A changed version reaches every node when it next connects. To use an engine
|
A changed version reaches every node when it next connects. To use an engine
|
||||||
you installed yourself on a node, name its path in that node's `agent.yaml`
|
you installed yourself on a node, name its path in that node's `agent.yaml`
|
||||||
(`engines.llamacpp.bin`, `engines.vllm.bin`, `engines.sglang.python`,
|
(`engines.llamacpp.bin`, `engines.vllm.bin`, `engines.sglang.python`,
|
||||||
|
|||||||
@@ -366,6 +366,26 @@ func TestEnginesListenOnTheAddressTheyAreGiven(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSGLangIsStartedServingItsMetrics(t *testing.T) {
|
||||||
|
// Without --enable-metrics SGLang has no /metrics, and every scrape of
|
||||||
|
// the replica fails as though it were unreachable.
|
||||||
|
if cmd := buildOne(t, &SGLang{}, singleNodeSpec()); !slices.Contains(cmd.Args, "--enable-metrics") {
|
||||||
|
t.Errorf("not started with its metrics: %v", cmd.Args)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSGLangIsAskedForItsHelpThroughItsModule(t *testing.T) {
|
||||||
|
g := &SGLang{cfg: SGLangConfig{Python: "/opt/sglang/bin/python"}}
|
||||||
|
g.UseInstalled("/opt/llmgw/engines/sglang/0.5.21-cu13/bin/python")
|
||||||
|
path, args := g.HelpCommand(Command{})
|
||||||
|
if path != "/opt/llmgw/engines/sglang/0.5.21-cu13/bin/python" {
|
||||||
|
t.Errorf("asked %s, not the interpreter it was installed into", path)
|
||||||
|
}
|
||||||
|
if !slices.Equal(args, []string{"-m", "sglang.launch_server", "--help"}) {
|
||||||
|
t.Errorf("asked %v", args)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestSGLangSingleNodeOmitsMultiNodeFlags(t *testing.T) {
|
func TestSGLangSingleNodeOmitsMultiNodeFlags(t *testing.T) {
|
||||||
cmd := buildOne(t, &SGLang{}, singleNodeSpec())
|
cmd := buildOne(t, &SGLang{}, singleNodeSpec())
|
||||||
for _, flag := range []string{"--nnodes", "--node-rank", "--dist-init-addr"} {
|
for _, flag := range []string{"--nnodes", "--node-rank", "--dist-init-addr"} {
|
||||||
|
|||||||
@@ -40,7 +40,15 @@ func (*SGLang) Name() string { return "sglang" }
|
|||||||
// HealthPath is SGLang's readiness endpoint.
|
// HealthPath is SGLang's readiness endpoint.
|
||||||
func (*SGLang) HealthPath() string { return "/health" }
|
func (*SGLang) HealthPath() string { return "/health" }
|
||||||
|
|
||||||
// MetricsPath is SGLang's Prometheus endpoint.
|
// HelpCommand is how to ask SGLang for its help: through its module, in the
|
||||||
|
// environment it was installed into, since the command itself is llmgw's
|
||||||
|
// launcher and the interpreter's own --help describes Python.
|
||||||
|
func (g *SGLang) HelpCommand(Command) (string, []string) {
|
||||||
|
return g.python(), []string{"-m", "sglang.launch_server", "--help"}
|
||||||
|
}
|
||||||
|
|
||||||
|
// MetricsPath is SGLang's Prometheus endpoint, served only with
|
||||||
|
// --enable-metrics.
|
||||||
func (*SGLang) MetricsPath() string { return "/metrics" }
|
func (*SGLang) MetricsPath() string { return "/metrics" }
|
||||||
|
|
||||||
// Validate checks the profile against the GPUs it would run on.
|
// Validate checks the profile against the GPUs it would run on.
|
||||||
@@ -83,7 +91,13 @@ func (g *SGLang) serverCommand(s ShardSpec) (Command, error) {
|
|||||||
"--model-path", p.ModelPath,
|
"--model-path", p.ModelPath,
|
||||||
"--host", s.Host(),
|
"--host", s.Host(),
|
||||||
"--port", strconv.Itoa(s.HTTPPort),
|
"--port", strconv.Itoa(s.HTTPPort),
|
||||||
"--tp", strconv.Itoa(p.TP),
|
// The flag's whole name: SGLang's documentation writes --tp, which
|
||||||
|
// argparse accepts only as an abbreviation of --tp-size and only
|
||||||
|
// while no other flag begins with it.
|
||||||
|
"--tp-size", strconv.Itoa(p.TP),
|
||||||
|
// Without it SGLang serves no /metrics at all, and the router has
|
||||||
|
// nothing to schedule this replica by.
|
||||||
|
"--enable-metrics",
|
||||||
}
|
}
|
||||||
if p.ServedModelName != "" {
|
if p.ServedModelName != "" {
|
||||||
args = append(args, "--served-model-name", p.ServedModelName)
|
args = append(args, "--served-model-name", p.ServedModelName)
|
||||||
|
|||||||
@@ -79,6 +79,13 @@ func TestATruncatedHelpIsSuspected(t *testing.T) {
|
|||||||
// trtllm-serve's help through its module, it prints the help given; asked
|
// trtllm-serve's help through its module, it prints the help given; asked
|
||||||
// anything else, it fails, as running the launcher here would.
|
// anything else, it fails, as running the launcher here would.
|
||||||
func fakeInterpreter(t *testing.T, help string) string {
|
func fakeInterpreter(t *testing.T, help string) string {
|
||||||
|
t.Helper()
|
||||||
|
return fakePython(t, "-m tensorrt_llm.commands.serve serve --help", help)
|
||||||
|
}
|
||||||
|
|
||||||
|
// fakePython is an interpreter that prints help when asked exactly ask, and
|
||||||
|
// fails on anything else — the interpreter's own --help included.
|
||||||
|
func fakePython(t *testing.T, ask, help string) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
if err := os.WriteFile(filepath.Join(dir, "help.txt"), []byte(help), 0o644); err != nil {
|
if err := os.WriteFile(filepath.Join(dir, "help.txt"), []byte(help), 0o644); err != nil {
|
||||||
@@ -86,7 +93,7 @@ func fakeInterpreter(t *testing.T, help string) string {
|
|||||||
}
|
}
|
||||||
p := filepath.Join(dir, "python")
|
p := filepath.Join(dir, "python")
|
||||||
body := "#!/bin/sh\n" +
|
body := "#!/bin/sh\n" +
|
||||||
`if [ "$*" = "-m tensorrt_llm.commands.serve serve --help" ]; then cat "` + filepath.Join(dir, "help.txt") + `"; exit 0; fi` + "\n" +
|
`if [ "$*" = "` + ask + `" ]; then cat "` + filepath.Join(dir, "help.txt") + `"; exit 0; fi` + "\n" +
|
||||||
"echo \"unexpected: $*\" >&2; exit 2\n"
|
"echo \"unexpected: $*\" >&2; exit 2\n"
|
||||||
if err := os.WriteFile(p, []byte(body), 0o755); err != nil { //nolint:gosec // an executable stand-in
|
if err := os.WriteFile(p, []byte(body), 0o755); err != nil { //nolint:gosec // an executable stand-in
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -137,6 +144,57 @@ func TestEngineCheckAsksTrtllmServeThroughItsModule(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// sglangHelp is the part of python -m sglang.launch_server --help (0.5.21)
|
||||||
|
// that holds the flags llmgw builds, in argparse's layout.
|
||||||
|
const sglangHelp = `usage: sglang serve [-h] --model-path MODEL_PATH [--host HOST] [--port PORT]
|
||||||
|
|
||||||
|
options:
|
||||||
|
-h, --help show this help message and exit
|
||||||
|
--model-path MODEL_PATH, --model MODEL_PATH
|
||||||
|
Path to model weights.
|
||||||
|
--host HOST The host of the HTTP server.
|
||||||
|
--port PORT The port of the HTTP server.
|
||||||
|
--served-model-name SERVED_MODEL_NAME
|
||||||
|
Override the model name returned by the v1/models endpoint.
|
||||||
|
--enable-metrics Enable log prometheus metrics.
|
||||||
|
--dist-init-addr DIST_INIT_ADDR, --nccl-init-addr DIST_INIT_ADDR
|
||||||
|
The host address for initializing distributed backend.
|
||||||
|
--nnodes NNODES The number of nodes.
|
||||||
|
--node-rank NODE_RANK
|
||||||
|
The node rank.
|
||||||
|
--tp-size TP_SIZE, --tensor-parallel-size TP_SIZE
|
||||||
|
The tensor parallelism size.
|
||||||
|
`
|
||||||
|
|
||||||
|
func TestEngineCheckAsksSGLangThroughItsModule(t *testing.T) {
|
||||||
|
// The command is the interpreter running llmgw's launcher; asked for
|
||||||
|
// --help it would describe Python, and every SGLang flag would look
|
||||||
|
// missing — or, against a real python3, be checked against nothing.
|
||||||
|
py := fakePython(t, "-m sglang.launch_server --help", sglangHelp)
|
||||||
|
out, err := runEngineCheck(t, "sglang", "--bin", py)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%v\n%s", err, out)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, "every flag llmgw builds is accepted") {
|
||||||
|
t.Errorf("unexpected report:\n%s", out)
|
||||||
|
}
|
||||||
|
if strings.Contains(out, "unexpected:") || strings.Contains(out, "skipped") {
|
||||||
|
t.Errorf("the help was not asked of SGLang:\n%s", out)
|
||||||
|
}
|
||||||
|
// Two nodes by default, so the rendezvous flags were among those checked.
|
||||||
|
if !strings.Contains(out, "sglang: 9 flags checked") {
|
||||||
|
t.Errorf("not every flag was checked:\n%s", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEngineCheckCatchesAnSGLangWithoutTheFlagForItsMetrics(t *testing.T) {
|
||||||
|
old := strings.ReplaceAll(sglangHelp, " --enable-metrics Enable log prometheus metrics.\n", "")
|
||||||
|
out, err := runEngineCheck(t, "sglang", "--bin", fakePython(t, "-m sglang.launch_server --help", old))
|
||||||
|
if err == nil || !strings.Contains(out, "sglang: --enable-metrics") {
|
||||||
|
t.Errorf("an SGLang without --enable-metrics passed: %v\n%s", err, out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestEngineCheckCatchesATensorRTLLMTooOldForTheServedName(t *testing.T) {
|
func TestEngineCheckCatchesATensorRTLLMTooOldForTheServedName(t *testing.T) {
|
||||||
// TensorRT-LLM 1.2 has no --served_model_name, and a replica of it would
|
// TensorRT-LLM 1.2 has no --served_model_name, and a replica of it would
|
||||||
// never be seen serving the name the agent waits for.
|
// never be seen serving the name the agent waits for.
|
||||||
|
|||||||
@@ -48,6 +48,8 @@ func MappingFor(engine string) Mapping {
|
|||||||
return LlamaCppMapping()
|
return LlamaCppMapping()
|
||||||
case "trtllm":
|
case "trtllm":
|
||||||
return TRTLLMMapping()
|
return TRTLLMMapping()
|
||||||
|
case "sglang":
|
||||||
|
return SGLangMapping()
|
||||||
default:
|
default:
|
||||||
return VLLMMapping()
|
return VLLMMapping()
|
||||||
}
|
}
|
||||||
@@ -106,6 +108,39 @@ func TRTLLMMapping() Mapping {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SGLangMapping returns SGLang's metric names, as sglang.launch_server
|
||||||
|
// publishes them under /metrics (checked against 0.5.21's collector).
|
||||||
|
//
|
||||||
|
// They are there only when the server runs with --enable-metrics, which
|
||||||
|
// llmgw's launch command passes; without it /metrics is not served at all.
|
||||||
|
// SGLang exempts /metrics from its API key, so the key the poller sends is
|
||||||
|
// ignored rather than needed.
|
||||||
|
//
|
||||||
|
// "Queue" is SGLang's word for vLLM's waiting. The gauges are set by the
|
||||||
|
// scheduler as it reports its batches, and at least every thirty seconds
|
||||||
|
// while it is idle, so a server that has only just started may serve none of
|
||||||
|
// them yet and is reported incomplete until it does.
|
||||||
|
//
|
||||||
|
// token_usage is the fraction of the KV pool in use, as vLLM's
|
||||||
|
// kv_cache_usage_perc is (for a model with sliding-window layers, the fuller
|
||||||
|
// of its two pools). cache_hit_rate is a different number — how much of the
|
||||||
|
// prompts the prefix cache answered — and is not what saturation means.
|
||||||
|
//
|
||||||
|
// num_requests_total counts the requests that finished, the nearest thing it
|
||||||
|
// has to vLLM's success counter; like the token counters it is split by
|
||||||
|
// is_streaming and summed. There is no cache configuration to read block
|
||||||
|
// sizes from.
|
||||||
|
func SGLangMapping() Mapping {
|
||||||
|
return Mapping{
|
||||||
|
Running: "sglang:num_running_reqs",
|
||||||
|
Waiting: "sglang:num_queue_reqs",
|
||||||
|
KVCacheUsage: "sglang:token_usage",
|
||||||
|
PromptTokens: "sglang:prompt_tokens_total",
|
||||||
|
GenerationTokens: "sglang:generation_tokens_total",
|
||||||
|
SuccessTotal: "sglang:num_requests_total",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// VLLMMapping returns vLLM's current metric names.
|
// VLLMMapping returns vLLM's current metric names.
|
||||||
func VLLMMapping() Mapping {
|
func VLLMMapping() Mapping {
|
||||||
return Mapping{
|
return Mapping{
|
||||||
|
|||||||
@@ -0,0 +1,116 @@
|
|||||||
|
package scrape
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
||||||
|
)
|
||||||
|
|
||||||
|
// sglangMetrics is what sglang.launch_server --enable-metrics serves on
|
||||||
|
// /metrics (0.5.21), trimmed to the families that matter, one it must not be
|
||||||
|
// mistaken for, and a histogram the parser has to step over. The scheduler's
|
||||||
|
// gauges carry its ranks; the tokenizer's counters are split by whether the
|
||||||
|
// request streamed.
|
||||||
|
const sglangMetrics = `# HELP sglang:num_running_reqs The number of running requests.
|
||||||
|
# TYPE sglang:num_running_reqs gauge
|
||||||
|
sglang:num_running_reqs{engine_type="unified",model_name="qwen3-8b",moe_ep_rank="0",pp_rank="0",tp_rank="0"} 6.0
|
||||||
|
# HELP sglang:num_queue_reqs The number of requests in the waiting queue.
|
||||||
|
# TYPE sglang:num_queue_reqs gauge
|
||||||
|
sglang:num_queue_reqs{engine_type="unified",model_name="qwen3-8b",moe_ep_rank="0",pp_rank="0",tp_rank="0"} 3.0
|
||||||
|
# HELP sglang:cache_hit_rate The prefix cache hit rate.
|
||||||
|
# TYPE sglang:cache_hit_rate gauge
|
||||||
|
sglang:cache_hit_rate{engine_type="unified",model_name="qwen3-8b",moe_ep_rank="0",pp_rank="0",tp_rank="0"} 0.9
|
||||||
|
# HELP sglang:token_usage The token usage.
|
||||||
|
# TYPE sglang:token_usage gauge
|
||||||
|
sglang:token_usage{engine_type="unified",model_name="qwen3-8b",moe_ep_rank="0",pp_rank="0",tp_rank="0"} 0.42
|
||||||
|
# HELP sglang:prompt_tokens_total Number of prefill tokens processed.
|
||||||
|
# TYPE sglang:prompt_tokens_total counter
|
||||||
|
sglang:prompt_tokens_total{engine_type="unified",is_streaming="true",model_name="qwen3-8b"} 12000.0
|
||||||
|
sglang:prompt_tokens_total{engine_type="unified",is_streaming="false",model_name="qwen3-8b"} 345.0
|
||||||
|
# HELP sglang:generation_tokens_total Number of generation tokens processed.
|
||||||
|
# TYPE sglang:generation_tokens_total counter
|
||||||
|
sglang:generation_tokens_total{engine_type="unified",is_streaming="true",model_name="qwen3-8b"} 8000.0
|
||||||
|
sglang:generation_tokens_total{engine_type="unified",is_streaming="false",model_name="qwen3-8b"} 64.0
|
||||||
|
# HELP sglang:num_requests_total Number of requests processed.
|
||||||
|
# TYPE sglang:num_requests_total counter
|
||||||
|
sglang:num_requests_total{engine_type="unified",is_streaming="true",model_name="qwen3-8b"} 30.0
|
||||||
|
sglang:num_requests_total{engine_type="unified",is_streaming="false",model_name="qwen3-8b"} 4.0
|
||||||
|
# HELP sglang:e2e_request_latency_seconds Histogram of End-to-end request latency in seconds
|
||||||
|
# TYPE sglang:e2e_request_latency_seconds histogram
|
||||||
|
sglang:e2e_request_latency_seconds_bucket{engine_type="unified",is_streaming="true",le="0.5",model_name="qwen3-8b"} 12.0
|
||||||
|
sglang:e2e_request_latency_seconds_bucket{engine_type="unified",is_streaming="true",le="+Inf",model_name="qwen3-8b"} 30.0
|
||||||
|
sglang:e2e_request_latency_seconds_count{engine_type="unified",is_streaming="true",model_name="qwen3-8b"} 30.0
|
||||||
|
sglang:e2e_request_latency_seconds_sum{engine_type="unified",is_streaming="true",model_name="qwen3-8b"} 41.2
|
||||||
|
`
|
||||||
|
|
||||||
|
func TestSGLangMetricsSatisfyTheContract(t *testing.T) {
|
||||||
|
s, err := Parse(strings.NewReader(sglangMetrics), MappingFor("sglang"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if s.Running != 6 || s.Waiting != 3 {
|
||||||
|
t.Errorf("running=%d waiting=%d", s.Running, s.Waiting)
|
||||||
|
}
|
||||||
|
// The pool's fill, not the prefix cache's hit rate beside it.
|
||||||
|
if s.KVCacheUsage != 0.42 {
|
||||||
|
t.Errorf("kv=%v", s.KVCacheUsage)
|
||||||
|
}
|
||||||
|
// Split by is_streaming, summed.
|
||||||
|
if s.PromptTokens != 12345 || s.GenerationTokens != 8064 {
|
||||||
|
t.Errorf("tokens=%v/%v", s.PromptTokens, s.GenerationTokens)
|
||||||
|
}
|
||||||
|
if s.SuccessTotal != 34 {
|
||||||
|
t.Errorf("success=%v", s.SuccessTotal)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSGLangIsNotReadWithVLLMsNames(t *testing.T) {
|
||||||
|
// Read with vLLM's vocabulary, an SGLang replica serves none of the
|
||||||
|
// contract and the router schedules to it blind.
|
||||||
|
var missing *MissingError
|
||||||
|
if _, err := Parse(strings.NewReader(sglangMetrics), VLLMMapping()); !errors.As(err, &missing) {
|
||||||
|
t.Fatalf("vLLM's names found something in SGLang's metrics: %v", err)
|
||||||
|
}
|
||||||
|
if MappingFor("sglang").Running == VLLMMapping().Running {
|
||||||
|
t.Error("sglang falls back to vLLM's names")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnSGLangEndpointIsMeasuredByThePoller(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path != "/metrics" {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(sglangMetrics))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
reg := endpoint.NewRegistry(endpoint.Config{})
|
||||||
|
reg.Replace([]endpoint.Info{{ID: "s1", URL: srv.URL, Engine: "sglang", APIKey: "k1", Models: []string{"m"}}})
|
||||||
|
p := NewPoller(reg, Config{Interval: 20 * time.Millisecond, Timeout: 2 * time.Second}, nil)
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
p.Sync(ctx)
|
||||||
|
defer p.Stop()
|
||||||
|
|
||||||
|
deadline := time.Now().Add(2 * time.Second)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
for _, snap := range reg.Snapshots("m") {
|
||||||
|
if snap.ID == "s1" && snap.Stats.Running == 6 {
|
||||||
|
if snap.Stats.Waiting != 3 || snap.Stats.KVCacheUsage != 0.42 {
|
||||||
|
t.Errorf("stats read wrong: %+v", snap.Stats)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
time.Sleep(10 * time.Millisecond)
|
||||||
|
}
|
||||||
|
t.Fatal("the endpoint was never measured")
|
||||||
|
}
|
||||||
Reference in new issue
Block a user