package engines import "strconv" // SGLangConfig locates the SGLang installation. type SGLangConfig struct { // Python is the interpreter of the SGLang virtualenv. SGLang is launched as // a module rather than a console script, which is the documented entry // point and avoids depending on the script being on PATH. Python string `koanf:"python"` } // SGLang builds SGLang launch commands. type SGLang struct{ cfg SGLangConfig } // Name identifies the engine in the catalog. func (*SGLang) Name() string { return "sglang" } // HealthPath is SGLang's readiness endpoint. func (*SGLang) HealthPath() string { return "/health" } // MetricsPath is SGLang's Prometheus endpoint. func (*SGLang) MetricsPath() string { return "/metrics" } // Validate checks the profile against the GPUs it would run on. func (*SGLang) Validate(p Profile, gpus []GPUInfo) error { if err := validateCommon(p, gpus); err != nil { return err } if q := quantizationOf(p.Args); q == "fp8" { if err := requireComputeCapability(p, gpus, 8.9, "fp8 quantization"); err != nil { return err } } return nil } // ServesHTTP reports whether this rank answers the API. // // Every SGLang rank runs the same launcher, but only node-rank 0 binds the HTTP // port; the others exist to contribute GPUs to the tensor-parallel group. func (*SGLang) ServesHTTP(s ShardSpec) bool { return s.Leader() } // Commands returns the processes for one shard. // // SGLang supports multi-node natively, so there is only ever one process: every // rank runs the same launcher with its own rank number and they rendezvous at // the leader's dist-init address. func (g *SGLang) Commands(s ShardSpec) ([]Command, error) { c, err := g.serverCommand(s) if err != nil { return nil, err } return []Command{c}, nil } func (g *SGLang) serverCommand(s ShardSpec) (Command, error) { python := g.cfg.Python if python == "" { python = "python3" } p := s.Profile args := []string{"-m", "sglang.launch_server", "--model-path", p.ModelPath, "--host", "0.0.0.0", "--port", strconv.Itoa(s.HTTPPort), "--tp", strconv.Itoa(p.TP), } if p.ServedModelName != "" { args = append(args, "--served-model-name", p.ServedModelName) } if s.NNodes > 1 { args = append(args, "--nnodes", strconv.Itoa(s.NNodes), "--node-rank", strconv.Itoa(s.Rank), "--dist-init-addr", s.LeaderAddr+":"+strconv.Itoa(s.LeaderPort), ) } args = append(args, p.Args...) return Command{Name: "sglang", Path: python, Args: args, Env: materializeEnv(baseEnv(s))}, nil }