vLLM was installed by hand and named in each agent's configuration. The agent now installs it: a Python 3.12 environment built by uv, which is itself a GitHub release checked against its checksum, from PyPI resolved as it stood a day after that vLLM was published and installed with --require-hashes. The resolved list is kept in the environment. The environment is built relocatable under a scratch name and renamed into place once it runs, like an archive. The default is 0.30.0, whose flags were checked against the ones llmgw builds. It is built for CUDA 13, so it needs driver 580 and a Turing GPU or newer, and 25 GB free while it installs; a node lacking one says which, and a node with no GPU skips it without being shown as a fault. gcc comes with it because Triton compiles its launcher at the first model load. SGLang installs the same way when the control plane names a version. llmgw doctor reports whether vLLM can be installed on the node. Checked by installing 0.30.0 this way on a test node: 7.7 GB, torch 2.13 for CUDA 13.0, importing cleanly; the node has no GPU, so vLLM was not started. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
102 lines
2.9 KiB
Go
102 lines
2.9 KiB
Go
package engines
|
|
|
|
import (
|
|
"strconv"
|
|
"sync/atomic"
|
|
)
|
|
|
|
// SGLangConfig locates the SGLang installation.
|
|
type SGLangConfig struct {
|
|
// Python is the interpreter of the SGLang virtualenv. SGLang is launched as
|
|
// a module rather than a console script, which is the documented entry
|
|
// point and avoids depending on the script being on PATH.
|
|
Python string `koanf:"python"`
|
|
}
|
|
|
|
// SGLang builds SGLang launch commands.
|
|
type SGLang struct {
|
|
cfg SGLangConfig
|
|
installed atomic.Pointer[string]
|
|
}
|
|
|
|
// UseInstalled points the engine at the interpreter of the environment the
|
|
// agent installed and checked, ahead of the configuration.
|
|
func (g *SGLang) UseInstalled(path string) { g.installed.Store(&path) }
|
|
|
|
// python is the interpreter to launch SGLang with.
|
|
func (g *SGLang) python() string {
|
|
if p := g.installed.Load(); p != nil {
|
|
return *p
|
|
}
|
|
if g.cfg.Python != "" {
|
|
return g.cfg.Python
|
|
}
|
|
return "python3"
|
|
}
|
|
|
|
// Name identifies the engine in the catalog.
|
|
func (*SGLang) Name() string { return "sglang" }
|
|
|
|
// HealthPath is SGLang's readiness endpoint.
|
|
func (*SGLang) HealthPath() string { return "/health" }
|
|
|
|
// MetricsPath is SGLang's Prometheus endpoint.
|
|
func (*SGLang) MetricsPath() string { return "/metrics" }
|
|
|
|
// Validate checks the profile against the GPUs it would run on.
|
|
func (*SGLang) Validate(p Profile, gpus []GPUInfo) error {
|
|
if err := validateCommon(p, gpus); err != nil {
|
|
return err
|
|
}
|
|
if q := quantizationOf(p.Args); q == "fp8" {
|
|
if err := requireComputeCapability(p, gpus, 8.9, "fp8 quantization"); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ServesHTTP reports whether this rank answers the API.
|
|
//
|
|
// Every SGLang rank runs the same launcher, but only node-rank 0 binds the HTTP
|
|
// port; the others exist to contribute GPUs to the tensor-parallel group.
|
|
func (*SGLang) ServesHTTP(s ShardSpec) bool { return s.Leader() }
|
|
|
|
// Commands returns the processes for one shard.
|
|
//
|
|
// SGLang supports multi-node natively, so there is only ever one process: every
|
|
// rank runs the same launcher with its own rank number and they rendezvous at
|
|
// the leader's dist-init address.
|
|
func (g *SGLang) Commands(s ShardSpec) ([]Command, error) {
|
|
c, err := g.serverCommand(s)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return []Command{c}, nil
|
|
}
|
|
|
|
func (g *SGLang) serverCommand(s ShardSpec) (Command, error) {
|
|
python := g.python()
|
|
p := s.Profile
|
|
|
|
args := []string{"-m", "sglang.launch_server",
|
|
"--model-path", p.ModelPath,
|
|
"--host", "0.0.0.0",
|
|
"--port", strconv.Itoa(s.HTTPPort),
|
|
"--tp", strconv.Itoa(p.TP),
|
|
}
|
|
if p.ServedModelName != "" {
|
|
args = append(args, "--served-model-name", p.ServedModelName)
|
|
}
|
|
if s.NNodes > 1 {
|
|
args = append(args,
|
|
"--nnodes", strconv.Itoa(s.NNodes),
|
|
"--node-rank", strconv.Itoa(s.Rank),
|
|
"--dist-init-addr", s.LeaderAddr+":"+strconv.Itoa(s.LeaderPort),
|
|
)
|
|
}
|
|
args = append(args, p.Args...)
|
|
|
|
return Command{Name: "sglang", Path: python, Args: args, Env: materializeEnv(baseEnv(s))}, nil
|
|
}
|