Engines listen on every address their node has, with no key. Checked on the live cluster: from a neighbouring node, a request straight to an engine's port was answered — past the gateway's authentication, quotas and accounting, by anyone who could reach the node. Each replica's engine is now started with a key and refuses requests without it. The key is derived from the replica id with an HMAC over a secret kept in the store, rather than stored per replica: the router has to send the right key to replicas this process never started — adopted after a restart, or placed by the other control plane — and a function of a shared secret gives that without a table to keep in step. A key taken from one node opens that replica only. The secret is made on first use and copied with the rest of the store when moving to PostgreSQL; a new one would lock the router out of every running engine at the moment of the move. The key reaches llama.cpp and vLLM through their environment (LLAMA_API_KEY, VLLM_API_KEY) rather than their command line, which every user on the node can read; SGLang only takes it as an argument. The agent's readiness check, the router's proxy and its metrics scrape send it, the client's own Authorization still never reaches the engine, and `ctl bench` fetches the key of the replica it is pointed at from the admin API. Engines started before this keep running without a key until their replica is next started. With no store there is nowhere to keep the secret, and engines start without one, with a warning. Also formats two files that went in unformatted. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
107 lines
3.1 KiB
Go
107 lines
3.1 KiB
Go
package engines
|
|
|
|
import (
|
|
"strconv"
|
|
"sync/atomic"
|
|
)
|
|
|
|
// SGLangConfig locates the SGLang installation.
|
|
type SGLangConfig struct {
|
|
// Python is the interpreter of the SGLang virtualenv. SGLang is launched as
|
|
// a module rather than a console script, which is the documented entry
|
|
// point and avoids depending on the script being on PATH.
|
|
Python string `koanf:"python"`
|
|
}
|
|
|
|
// SGLang builds SGLang launch commands.
|
|
type SGLang struct {
|
|
cfg SGLangConfig
|
|
installed atomic.Pointer[string]
|
|
}
|
|
|
|
// UseInstalled points the engine at the interpreter of the environment the
|
|
// agent installed and checked, ahead of the configuration.
|
|
func (g *SGLang) UseInstalled(path string) { g.installed.Store(&path) }
|
|
|
|
// python is the interpreter to launch SGLang with.
|
|
func (g *SGLang) python() string {
|
|
if p := g.installed.Load(); p != nil {
|
|
return *p
|
|
}
|
|
if g.cfg.Python != "" {
|
|
return g.cfg.Python
|
|
}
|
|
return "python3"
|
|
}
|
|
|
|
// Name identifies the engine in the catalog.
|
|
func (*SGLang) Name() string { return "sglang" }
|
|
|
|
// HealthPath is SGLang's readiness endpoint.
|
|
func (*SGLang) HealthPath() string { return "/health" }
|
|
|
|
// MetricsPath is SGLang's Prometheus endpoint.
|
|
func (*SGLang) MetricsPath() string { return "/metrics" }
|
|
|
|
// Validate checks the profile against the GPUs it would run on.
|
|
func (*SGLang) Validate(p Profile, gpus []GPUInfo) error {
|
|
if err := validateCommon(p, gpus); err != nil {
|
|
return err
|
|
}
|
|
if q := quantizationOf(p.Args); q == "fp8" {
|
|
if err := requireComputeCapability(p, gpus, 8.9, "fp8 quantization"); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ServesHTTP reports whether this rank answers the API.
|
|
//
|
|
// Every SGLang rank runs the same launcher, but only node-rank 0 binds the HTTP
|
|
// port; the others exist to contribute GPUs to the tensor-parallel group.
|
|
func (*SGLang) ServesHTTP(s ShardSpec) bool { return s.Leader() }
|
|
|
|
// Commands returns the processes for one shard.
|
|
//
|
|
// SGLang supports multi-node natively, so there is only ever one process: every
|
|
// rank runs the same launcher with its own rank number and they rendezvous at
|
|
// the leader's dist-init address.
|
|
func (g *SGLang) Commands(s ShardSpec) ([]Command, error) {
|
|
c, err := g.serverCommand(s)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return []Command{c}, nil
|
|
}
|
|
|
|
func (g *SGLang) serverCommand(s ShardSpec) (Command, error) {
|
|
python := g.python()
|
|
p := s.Profile
|
|
|
|
args := []string{"-m", "sglang.launch_server",
|
|
"--model-path", p.ModelPath,
|
|
"--host", "0.0.0.0",
|
|
"--port", strconv.Itoa(s.HTTPPort),
|
|
"--tp", strconv.Itoa(p.TP),
|
|
}
|
|
if p.ServedModelName != "" {
|
|
args = append(args, "--served-model-name", p.ServedModelName)
|
|
}
|
|
if s.NNodes > 1 {
|
|
args = append(args,
|
|
"--nnodes", strconv.Itoa(s.NNodes),
|
|
"--node-rank", strconv.Itoa(s.Rank),
|
|
"--dist-init-addr", s.LeaderAddr+":"+strconv.Itoa(s.LeaderPort),
|
|
)
|
|
}
|
|
args = append(args, p.Args...)
|
|
// SGLang reads its key only from the command line, so on a node with
|
|
// other users it is visible to them in the process list.
|
|
if s.APIKey != "" && s.Leader() {
|
|
args = append(args, "--api-key", s.APIKey)
|
|
}
|
|
|
|
return Command{Name: "sglang", Path: python, Args: args, Env: materializeEnv(baseEnv(s))}, nil
|
|
}
|