SGLang read in its own metric names, started serving them, and asked for its own help

The router read SGLang replicas with vLLM's names, and SGLang was started
without --enable-metrics, so it served no /metrics: every scrape failed as
if unreachable and the replica was routed to blind. It now gets
--enable-metrics and a mapping of its own (sglang:num_running_reqs,
num_queue_reqs, token_usage, the token counters and num_requests_total, as
0.5.21 publishes them). engine-check asks python -m sglang.launch_server
--help instead of Python's own help, and SGLang is given --tp-size rather
than the --tp argparse only takes as an abbreviation.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 00:20:25 +02:00
1 parent d41104a452
commit 8cc8d7139b
6 files changed
+251 -3

No files matched your search

+20
View File
@@ -366,6 +366,26 @@ func TestEnginesListenOnTheAddressTheyAreGiven(t *testing.T) {
}
}
func TestSGLangIsStartedServingItsMetrics(t *testing.T) {
// Without --enable-metrics SGLang has no /metrics, and every scrape of
// the replica fails as though it were unreachable.
if cmd := buildOne(t, &SGLang{}, singleNodeSpec()); !slices.Contains(cmd.Args, "--enable-metrics") {
t.Errorf("not started with its metrics: %v", cmd.Args)
}
}
func TestSGLangIsAskedForItsHelpThroughItsModule(t *testing.T) {
g := &SGLang{cfg: SGLangConfig{Python: "/opt/sglang/bin/python"}}
g.UseInstalled("/opt/llmgw/engines/sglang/0.5.21-cu13/bin/python")
path, args := g.HelpCommand(Command{})
if path != "/opt/llmgw/engines/sglang/0.5.21-cu13/bin/python" {
t.Errorf("asked %s, not the interpreter it was installed into", path)
}
if !slices.Equal(args, []string{"-m", "sglang.launch_server", "--help"}) {
t.Errorf("asked %v", args)
}
}
func TestSGLangSingleNodeOmitsMultiNodeFlags(t *testing.T) {
cmd := buildOne(t, &SGLang{}, singleNodeSpec())
for _, flag := range []string{"--nnodes", "--node-rank", "--dist-init-addr"} {
+16 -2
View File
@@ -40,7 +40,15 @@ func (*SGLang) Name() string { return "sglang" }
// HealthPath is SGLang's readiness endpoint.
func (*SGLang) HealthPath() string { return "/health" }
// MetricsPath is SGLang's Prometheus endpoint.
// HelpCommand is how to ask SGLang for its help: through its module, in the
// environment it was installed into, since the command itself is llmgw's
// launcher and the interpreter's own --help describes Python.
func (g *SGLang) HelpCommand(Command) (string, []string) {
return g.python(), []string{"-m", "sglang.launch_server", "--help"}
}
// MetricsPath is SGLang's Prometheus endpoint, served only with
// --enable-metrics.
func (*SGLang) MetricsPath() string { return "/metrics" }
// Validate checks the profile against the GPUs it would run on.
@@ -83,7 +91,13 @@ func (g *SGLang) serverCommand(s ShardSpec) (Command, error) {
"--model-path", p.ModelPath,
"--host", s.Host(),
"--port", strconv.Itoa(s.HTTPPort),
"--tp", strconv.Itoa(p.TP),
// The flag's whole name: SGLang's documentation writes --tp, which
// argparse accepts only as an abbreviation of --tp-size and only
// while no other flag begins with it.
"--tp-size", strconv.Itoa(p.TP),
// Without it SGLang serves no /metrics at all, and the router has
// nothing to schedule this replica by.
"--enable-metrics",
}
if p.ServedModelName != "" {
args = append(args, "--served-model-name", p.ServedModelName)