Merge engine fixes: SGLang metrics and flags, bench reading guarded metrics
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
commit
cc4de36674
12 files changed
+655
-28
No files matched your search
+64
-14
@@ -2,7 +2,9 @@ package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -20,6 +22,7 @@ func newBenchCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
metrics string
|
||||
engine string
|
||||
key string
|
||||
mkey string
|
||||
save bool
|
||||
)
|
||||
|
||||
@@ -41,10 +44,25 @@ func newBenchCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
cfg.MetricsURL = metrics
|
||||
cfg.Engine = engine
|
||||
cfg.APIKey = key
|
||||
// A replica's engine demands its own key. Asked for rather than
|
||||
// typed: the admin API knows which replica answers at the target.
|
||||
if cfg.APIKey == "" {
|
||||
cfg.APIKey = replicaKey(cmd, client, target)
|
||||
cfg.MetricsKey = mkey
|
||||
// A replica's engine demands its own key, for its API and — on
|
||||
// llama.cpp and TensorRT-LLM — for its metrics. Asked for rather
|
||||
// than typed: the admin API knows which replica answers where, and
|
||||
// this runs wherever ctl does, which can reach it.
|
||||
needKeys := cfg.APIKey == "" || (metrics != "" && cfg.MetricsKey == "")
|
||||
if needKeys {
|
||||
reps, err := listReplicas(cmd, client)
|
||||
if cfg.APIKey == "" {
|
||||
cfg.APIKey = replicaKey(reps, target)
|
||||
}
|
||||
if metrics != "" && cfg.MetricsKey == "" {
|
||||
cfg.MetricsKey = replicaKey(reps, metrics)
|
||||
if err != nil {
|
||||
cmd.PrintErrf("The admin API could not be asked for the engine key of the replica at "+
|
||||
"--metrics (%v); an engine that guards its metrics will refuse them. "+
|
||||
"Pass --metrics-key to give it.\n", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(steps) > 0 {
|
||||
cfg.Concurrency = steps
|
||||
@@ -72,17 +90,29 @@ func newBenchCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
if s.Failed > 0 {
|
||||
verdict = fmt.Sprintf("%d failed", s.Failed)
|
||||
}
|
||||
// Unread is not zero: a dash, and the reason under the row.
|
||||
wait, kv := fmt.Sprintf("%.1f", s.Waiting), fmt.Sprintf("%.0f%%", s.KVUsage*100)
|
||||
if s.MetricsUnread != "" {
|
||||
wait, kv = "-", "-"
|
||||
}
|
||||
fmt.Fprintf(cmd.ErrOrStderr(), row,
|
||||
fmt.Sprint(s.Concurrency),
|
||||
fmt.Sprintf("%.1f", s.RequestsPerSec),
|
||||
fmt.Sprintf("%.0f", s.TokensPerSec),
|
||||
ms(s.TTFT.P50), ms(s.TTFT.P99), ms(s.TPOT.P99),
|
||||
fmt.Sprintf("%.1f", s.Waiting),
|
||||
fmt.Sprintf("%.0f%%", s.KVUsage*100),
|
||||
verdict)
|
||||
wait, kv, verdict)
|
||||
if s.MetricsUnread != "" {
|
||||
fmt.Fprintf(cmd.ErrOrStderr(), " metrics not read: %s\n", s.MetricsUnread)
|
||||
}
|
||||
}
|
||||
|
||||
res, err := bench.Run(cmd.Context(), cfg, progress)
|
||||
var refused *bench.MetricsRefusedError
|
||||
if errors.As(err, &refused) {
|
||||
return fmt.Errorf("%w. The engine's key is the admin API's, given for the replica "+
|
||||
"whose endpoint `llmgw ctl replicas` lists at --metrics' address; "+
|
||||
"or pass it with --metrics-key", err)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -108,6 +138,8 @@ func newBenchCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
f.StringVar(&metrics, "metrics", "", "the replica's /metrics, to observe queue depth and KV usage")
|
||||
f.StringVar(&engine, "engine", "", "which metric names --metrics uses: vllm | sglang | llamacpp | trtllm (whose are at /prometheus/metrics)")
|
||||
f.StringVar(&key, "key", "", "API key, when the gateway requires one")
|
||||
f.StringVar(&mkey, "metrics-key", "",
|
||||
"the engine key --metrics is read with (default: the replica's, from the admin API)")
|
||||
f.IntSliceVar(&steps, "concurrency", nil, "levels to try (default 1,2,4,…,128)")
|
||||
f.DurationVar(&cfg.Duration, "duration", cfg.Duration, "how long to hold each level")
|
||||
f.IntVar(&cfg.PromptTokens, "prompt-tokens", cfg.PromptTokens, "approximate prompt size")
|
||||
@@ -233,17 +265,35 @@ func ms(d time.Duration) string {
|
||||
}
|
||||
}
|
||||
|
||||
// replicaKey is the engine key of the replica at target, or "" when target is
|
||||
// not a replica (the gateway, say) or the admin API cannot be asked.
|
||||
func replicaKey(cmd *cobra.Command, client func() *ctlClient, target string) string {
|
||||
func listReplicas(cmd *cobra.Command, client func() *ctlClient) ([]adminapi.ReplicaView, error) {
|
||||
var reps []adminapi.ReplicaView
|
||||
if err := client().get(cmd.Context(), "/api/v1/replicas", &reps); err != nil {
|
||||
return ""
|
||||
}
|
||||
err := client().get(cmd.Context(), "/api/v1/replicas", &reps)
|
||||
return reps, err
|
||||
}
|
||||
|
||||
// replicaKey is the engine key of the replica serving at addr, or "" when
|
||||
// addr is not a replica's (the gateway's, say).
|
||||
//
|
||||
// Matched by scheme, host and port rather than by the whole URL, so the same
|
||||
// lookup serves the target, which is a replica's base URL, and --metrics,
|
||||
// which is a path under it: /metrics, or /prometheus/metrics on TensorRT-LLM.
|
||||
func replicaKey(reps []adminapi.ReplicaView, addr string) string {
|
||||
for _, rep := range reps {
|
||||
if sameEndpoint(rep.Endpoint, target) {
|
||||
if sameOrigin(rep.Endpoint, addr) {
|
||||
return rep.EngineKey
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func sameOrigin(a, b string) bool {
|
||||
ua, err := url.Parse(a)
|
||||
if err != nil || ua.Host == "" {
|
||||
return false
|
||||
}
|
||||
ub, err := url.Parse(b)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return ua.Scheme == ub.Scheme && ua.Host == ub.Host
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
)
|
||||
|
||||
// benchCluster is a gateway taking the client's key, a llama.cpp replica
|
||||
// whose /metrics takes only its engine key, and an admin API that lists the
|
||||
// replica with that key, as the control plane does.
|
||||
type benchCluster struct {
|
||||
gateway, engine, admin *httptest.Server
|
||||
metricsAuth atomic.Value
|
||||
}
|
||||
|
||||
func newBenchCluster(t *testing.T) *benchCluster {
|
||||
t.Helper()
|
||||
c := &benchCluster{}
|
||||
c.metricsAuth.Store("")
|
||||
c.gateway = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Authorization") != "Bearer client" {
|
||||
http.Error(w, "no", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
fmt.Fprint(w, "data: {\"choices\":[{\"delta\":{\"content\":\"a\"}}]}\n\n")
|
||||
fmt.Fprint(w, "data: {\"choices\":[{\"delta\":{\"content\":\"b\"}}]}\n\n")
|
||||
fmt.Fprint(w, "data: [DONE]\n\n")
|
||||
}))
|
||||
c.engine = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
c.metricsAuth.Store(r.Header.Get("Authorization"))
|
||||
if r.URL.Path != "/metrics" || r.Header.Get("Authorization") != "Bearer engine-key" {
|
||||
http.Error(w, `{"error":"Invalid API Key"}`, http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
fmt.Fprint(w, "llamacpp:requests_processing 1\nllamacpp:requests_deferred 3\n")
|
||||
}))
|
||||
c.admin = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_ = json.NewEncoder(w).Encode([]adminapi.ReplicaView{
|
||||
{ID: "r1", Model: "m", Profile: "cpu", Endpoint: c.engine.URL, EngineKey: "engine-key"},
|
||||
})
|
||||
}))
|
||||
t.Cleanup(func() { c.gateway.Close(); c.engine.Close(); c.admin.Close() })
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *benchCluster) bench(t *testing.T, args ...string) (string, error) {
|
||||
t.Helper()
|
||||
t.Setenv("LLMGW_ADMIN_TOKEN", "t")
|
||||
asJSON := false
|
||||
cmd := newBenchCmd(func() *ctlClient { return newCtlClient(c.admin.URL) }, &asJSON)
|
||||
var out bytes.Buffer
|
||||
cmd.SetOut(&out)
|
||||
cmd.SetErr(&out)
|
||||
cmd.SetArgs(append([]string{"m", "--concurrency", "1", "--duration", "1500ms", "--ttft-p99", "1m", "--tpot-p99", "1m"}, args...))
|
||||
err := cmd.ExecuteContext(context.Background())
|
||||
return out.String(), err
|
||||
}
|
||||
|
||||
func TestBenchReadsAGuardedEnginesMetricsWithTheKeyTheAdminAPIGives(t *testing.T) {
|
||||
// Through the gateway with a client's key, and the replica's metrics on
|
||||
// the side: the client's key is not the engine's, and the engine's is
|
||||
// found by the replica at --metrics' address, not at the target's.
|
||||
c := newBenchCluster(t)
|
||||
out, err := c.bench(t, "--target", c.gateway.URL, "--key", "client",
|
||||
"--metrics", c.engine.URL+"/metrics", "--engine", "llamacpp")
|
||||
if err != nil {
|
||||
t.Fatalf("%v\n%s", err, out)
|
||||
}
|
||||
if got := c.metricsAuth.Load(); got != "Bearer engine-key" {
|
||||
t.Errorf("metrics read with %q", got)
|
||||
}
|
||||
if !strings.Contains(out, "target_waiting: 3") {
|
||||
t.Errorf("the queue depth read was not suggested:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBenchSaysWhyAnEnginesMetricsCannotBeRead(t *testing.T) {
|
||||
// A key the engine refuses fails the run before any load, saying where
|
||||
// the right one comes from, rather than reporting an idle engine.
|
||||
c := newBenchCluster(t)
|
||||
out, err := c.bench(t, "--target", c.gateway.URL, "--key", "client",
|
||||
"--metrics", c.engine.URL+"/metrics", "--engine", "llamacpp", "--metrics-key", "wrong")
|
||||
if err == nil || !strings.Contains(err.Error(), "refused the key") || !strings.Contains(err.Error(), "--metrics-key") {
|
||||
t.Fatalf("expected the refusal explained, got %v\n%s", err, out)
|
||||
}
|
||||
}
|
||||
@@ -79,6 +79,13 @@ func TestATruncatedHelpIsSuspected(t *testing.T) {
|
||||
// trtllm-serve's help through its module, it prints the help given; asked
|
||||
// anything else, it fails, as running the launcher here would.
|
||||
func fakeInterpreter(t *testing.T, help string) string {
|
||||
t.Helper()
|
||||
return fakePython(t, "-m tensorrt_llm.commands.serve serve --help", help)
|
||||
}
|
||||
|
||||
// fakePython is an interpreter that prints help when asked exactly ask, and
|
||||
// fails on anything else — the interpreter's own --help included.
|
||||
func fakePython(t *testing.T, ask, help string) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "help.txt"), []byte(help), 0o644); err != nil {
|
||||
@@ -86,7 +93,7 @@ func fakeInterpreter(t *testing.T, help string) string {
|
||||
}
|
||||
p := filepath.Join(dir, "python")
|
||||
body := "#!/bin/sh\n" +
|
||||
`if [ "$*" = "-m tensorrt_llm.commands.serve serve --help" ]; then cat "` + filepath.Join(dir, "help.txt") + `"; exit 0; fi` + "\n" +
|
||||
`if [ "$*" = "` + ask + `" ]; then cat "` + filepath.Join(dir, "help.txt") + `"; exit 0; fi` + "\n" +
|
||||
"echo \"unexpected: $*\" >&2; exit 2\n"
|
||||
if err := os.WriteFile(p, []byte(body), 0o755); err != nil { //nolint:gosec // an executable stand-in
|
||||
t.Fatal(err)
|
||||
@@ -137,6 +144,57 @@ func TestEngineCheckAsksTrtllmServeThroughItsModule(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// sglangHelp is the part of python -m sglang.launch_server --help (0.5.21)
|
||||
// that holds the flags llmgw builds, in argparse's layout.
|
||||
const sglangHelp = `usage: sglang serve [-h] --model-path MODEL_PATH [--host HOST] [--port PORT]
|
||||
|
||||
options:
|
||||
-h, --help show this help message and exit
|
||||
--model-path MODEL_PATH, --model MODEL_PATH
|
||||
Path to model weights.
|
||||
--host HOST The host of the HTTP server.
|
||||
--port PORT The port of the HTTP server.
|
||||
--served-model-name SERVED_MODEL_NAME
|
||||
Override the model name returned by the v1/models endpoint.
|
||||
--enable-metrics Enable log prometheus metrics.
|
||||
--dist-init-addr DIST_INIT_ADDR, --nccl-init-addr DIST_INIT_ADDR
|
||||
The host address for initializing distributed backend.
|
||||
--nnodes NNODES The number of nodes.
|
||||
--node-rank NODE_RANK
|
||||
The node rank.
|
||||
--tp-size TP_SIZE, --tensor-parallel-size TP_SIZE
|
||||
The tensor parallelism size.
|
||||
`
|
||||
|
||||
func TestEngineCheckAsksSGLangThroughItsModule(t *testing.T) {
|
||||
// The command is the interpreter running llmgw's launcher; asked for
|
||||
// --help it would describe Python, and every SGLang flag would look
|
||||
// missing — or, against a real python3, be checked against nothing.
|
||||
py := fakePython(t, "-m sglang.launch_server --help", sglangHelp)
|
||||
out, err := runEngineCheck(t, "sglang", "--bin", py)
|
||||
if err != nil {
|
||||
t.Fatalf("%v\n%s", err, out)
|
||||
}
|
||||
if !strings.Contains(out, "every flag llmgw builds is accepted") {
|
||||
t.Errorf("unexpected report:\n%s", out)
|
||||
}
|
||||
if strings.Contains(out, "unexpected:") || strings.Contains(out, "skipped") {
|
||||
t.Errorf("the help was not asked of SGLang:\n%s", out)
|
||||
}
|
||||
// Two nodes by default, so the rendezvous flags were among those checked.
|
||||
if !strings.Contains(out, "sglang: 9 flags checked") {
|
||||
t.Errorf("not every flag was checked:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEngineCheckCatchesAnSGLangWithoutTheFlagForItsMetrics(t *testing.T) {
|
||||
old := strings.ReplaceAll(sglangHelp, " --enable-metrics Enable log prometheus metrics.\n", "")
|
||||
out, err := runEngineCheck(t, "sglang", "--bin", fakePython(t, "-m sglang.launch_server --help", old))
|
||||
if err == nil || !strings.Contains(out, "sglang: --enable-metrics") {
|
||||
t.Errorf("an SGLang without --enable-metrics passed: %v\n%s", err, out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEngineCheckCatchesATensorRTLLMTooOldForTheServedName(t *testing.T) {
|
||||
// TensorRT-LLM 1.2 has no --served_model_name, and a replica of it would
|
||||
// never be seen serving the name the agent waits for.
|
||||
|
||||
Reference in new issue
Block a user