SGLang read in its own metric names, started serving them, and asked for its own help
The router read SGLang replicas with vLLM's names, and SGLang was started without --enable-metrics, so it served no /metrics: every scrape failed as if unreachable and the replica was routed to blind. It now gets --enable-metrics and a mapping of its own (sglang:num_running_reqs, num_queue_reqs, token_usage, the token counters and num_requests_total, as 0.5.21 publishes them). engine-check asks python -m sglang.launch_server --help instead of Python's own help, and SGLang is given --tp-size rather than the --tp argparse only takes as an abbreviation. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d41104a452
commit
8cc8d7139b
6 files changed
+251
-3
No files matched your search
@@ -79,6 +79,13 @@ func TestATruncatedHelpIsSuspected(t *testing.T) {
|
||||
// trtllm-serve's help through its module, it prints the help given; asked
|
||||
// anything else, it fails, as running the launcher here would.
|
||||
func fakeInterpreter(t *testing.T, help string) string {
|
||||
t.Helper()
|
||||
return fakePython(t, "-m tensorrt_llm.commands.serve serve --help", help)
|
||||
}
|
||||
|
||||
// fakePython is an interpreter that prints help when asked exactly ask, and
|
||||
// fails on anything else — the interpreter's own --help included.
|
||||
func fakePython(t *testing.T, ask, help string) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "help.txt"), []byte(help), 0o644); err != nil {
|
||||
@@ -86,7 +93,7 @@ func fakeInterpreter(t *testing.T, help string) string {
|
||||
}
|
||||
p := filepath.Join(dir, "python")
|
||||
body := "#!/bin/sh\n" +
|
||||
`if [ "$*" = "-m tensorrt_llm.commands.serve serve --help" ]; then cat "` + filepath.Join(dir, "help.txt") + `"; exit 0; fi` + "\n" +
|
||||
`if [ "$*" = "` + ask + `" ]; then cat "` + filepath.Join(dir, "help.txt") + `"; exit 0; fi` + "\n" +
|
||||
"echo \"unexpected: $*\" >&2; exit 2\n"
|
||||
if err := os.WriteFile(p, []byte(body), 0o755); err != nil { //nolint:gosec // an executable stand-in
|
||||
t.Fatal(err)
|
||||
@@ -137,6 +144,57 @@ func TestEngineCheckAsksTrtllmServeThroughItsModule(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// sglangHelp is the part of python -m sglang.launch_server --help (0.5.21)
|
||||
// that holds the flags llmgw builds, in argparse's layout.
|
||||
const sglangHelp = `usage: sglang serve [-h] --model-path MODEL_PATH [--host HOST] [--port PORT]
|
||||
|
||||
options:
|
||||
-h, --help show this help message and exit
|
||||
--model-path MODEL_PATH, --model MODEL_PATH
|
||||
Path to model weights.
|
||||
--host HOST The host of the HTTP server.
|
||||
--port PORT The port of the HTTP server.
|
||||
--served-model-name SERVED_MODEL_NAME
|
||||
Override the model name returned by the v1/models endpoint.
|
||||
--enable-metrics Enable log prometheus metrics.
|
||||
--dist-init-addr DIST_INIT_ADDR, --nccl-init-addr DIST_INIT_ADDR
|
||||
The host address for initializing distributed backend.
|
||||
--nnodes NNODES The number of nodes.
|
||||
--node-rank NODE_RANK
|
||||
The node rank.
|
||||
--tp-size TP_SIZE, --tensor-parallel-size TP_SIZE
|
||||
The tensor parallelism size.
|
||||
`
|
||||
|
||||
func TestEngineCheckAsksSGLangThroughItsModule(t *testing.T) {
|
||||
// The command is the interpreter running llmgw's launcher; asked for
|
||||
// --help it would describe Python, and every SGLang flag would look
|
||||
// missing — or, against a real python3, be checked against nothing.
|
||||
py := fakePython(t, "-m sglang.launch_server --help", sglangHelp)
|
||||
out, err := runEngineCheck(t, "sglang", "--bin", py)
|
||||
if err != nil {
|
||||
t.Fatalf("%v\n%s", err, out)
|
||||
}
|
||||
if !strings.Contains(out, "every flag llmgw builds is accepted") {
|
||||
t.Errorf("unexpected report:\n%s", out)
|
||||
}
|
||||
if strings.Contains(out, "unexpected:") || strings.Contains(out, "skipped") {
|
||||
t.Errorf("the help was not asked of SGLang:\n%s", out)
|
||||
}
|
||||
// Two nodes by default, so the rendezvous flags were among those checked.
|
||||
if !strings.Contains(out, "sglang: 9 flags checked") {
|
||||
t.Errorf("not every flag was checked:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEngineCheckCatchesAnSGLangWithoutTheFlagForItsMetrics(t *testing.T) {
|
||||
old := strings.ReplaceAll(sglangHelp, " --enable-metrics Enable log prometheus metrics.\n", "")
|
||||
out, err := runEngineCheck(t, "sglang", "--bin", fakePython(t, "-m sglang.launch_server --help", old))
|
||||
if err == nil || !strings.Contains(out, "sglang: --enable-metrics") {
|
||||
t.Errorf("an SGLang without --enable-metrics passed: %v\n%s", err, out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEngineCheckCatchesATensorRTLLMTooOldForTheServedName(t *testing.T) {
|
||||
// TensorRT-LLM 1.2 has no --served_model_name, and a replica of it would
|
||||
// never be seen serving the name the agent waits for.
|
||||
|
||||
Reference in new issue
Block a user