SGLang read in its own metric names, started serving them, and asked for its own help
The router read SGLang replicas with vLLM's names, and SGLang was started without --enable-metrics, so it served no /metrics: every scrape failed as if unreachable and the replica was routed to blind. It now gets --enable-metrics and a mapping of its own (sglang:num_running_reqs, num_queue_reqs, token_usage, the token counters and num_requests_total, as 0.5.21 publishes them). engine-check asks python -m sglang.launch_server --help instead of Python's own help, and SGLang is given --tp-size rather than the --tp argparse only takes as an abbreviation. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d41104a452
commit
8cc8d7139b
6 files changed
+251
-3
No files matched your search
@@ -366,6 +366,26 @@ func TestEnginesListenOnTheAddressTheyAreGiven(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSGLangIsStartedServingItsMetrics(t *testing.T) {
|
||||
// Without --enable-metrics SGLang has no /metrics, and every scrape of
|
||||
// the replica fails as though it were unreachable.
|
||||
if cmd := buildOne(t, &SGLang{}, singleNodeSpec()); !slices.Contains(cmd.Args, "--enable-metrics") {
|
||||
t.Errorf("not started with its metrics: %v", cmd.Args)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSGLangIsAskedForItsHelpThroughItsModule(t *testing.T) {
|
||||
g := &SGLang{cfg: SGLangConfig{Python: "/opt/sglang/bin/python"}}
|
||||
g.UseInstalled("/opt/llmgw/engines/sglang/0.5.21-cu13/bin/python")
|
||||
path, args := g.HelpCommand(Command{})
|
||||
if path != "/opt/llmgw/engines/sglang/0.5.21-cu13/bin/python" {
|
||||
t.Errorf("asked %s, not the interpreter it was installed into", path)
|
||||
}
|
||||
if !slices.Equal(args, []string{"-m", "sglang.launch_server", "--help"}) {
|
||||
t.Errorf("asked %v", args)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSGLangSingleNodeOmitsMultiNodeFlags(t *testing.T) {
|
||||
cmd := buildOne(t, &SGLang{}, singleNodeSpec())
|
||||
for _, flag := range []string{"--nnodes", "--node-rank", "--dist-init-addr"} {
|
||||
|
||||
@@ -40,7 +40,15 @@ func (*SGLang) Name() string { return "sglang" }
|
||||
// HealthPath is SGLang's readiness endpoint.
|
||||
func (*SGLang) HealthPath() string { return "/health" }
|
||||
|
||||
// MetricsPath is SGLang's Prometheus endpoint.
|
||||
// HelpCommand is how to ask SGLang for its help: through its module, in the
|
||||
// environment it was installed into, since the command itself is llmgw's
|
||||
// launcher and the interpreter's own --help describes Python.
|
||||
func (g *SGLang) HelpCommand(Command) (string, []string) {
|
||||
return g.python(), []string{"-m", "sglang.launch_server", "--help"}
|
||||
}
|
||||
|
||||
// MetricsPath is SGLang's Prometheus endpoint, served only with
|
||||
// --enable-metrics.
|
||||
func (*SGLang) MetricsPath() string { return "/metrics" }
|
||||
|
||||
// Validate checks the profile against the GPUs it would run on.
|
||||
@@ -83,7 +91,13 @@ func (g *SGLang) serverCommand(s ShardSpec) (Command, error) {
|
||||
"--model-path", p.ModelPath,
|
||||
"--host", s.Host(),
|
||||
"--port", strconv.Itoa(s.HTTPPort),
|
||||
"--tp", strconv.Itoa(p.TP),
|
||||
// The flag's whole name: SGLang's documentation writes --tp, which
|
||||
// argparse accepts only as an abbreviation of --tp-size and only
|
||||
// while no other flag begins with it.
|
||||
"--tp-size", strconv.Itoa(p.TP),
|
||||
// Without it SGLang serves no /metrics at all, and the router has
|
||||
// nothing to schedule this replica by.
|
||||
"--enable-metrics",
|
||||
}
|
||||
if p.ServedModelName != "" {
|
||||
args = append(args, "--served-model-name", p.ServedModelName)
|
||||
|
||||
Reference in new issue
Block a user