Have engines listen on the management address, and take SGLang's key off its command line
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
9b61bbc983
commit
3b05ec0538
9 files changed
+148
-15
No files matched your search
@@ -152,6 +152,13 @@ APIs change between minors without a deprecation period.
|
||||
before it — onboarding the node again issues one that works. A mirror
|
||||
serving its management API with `--tls-ca` now requires `--api-client`
|
||||
(the control plane's name), since the same authority signs every node.
|
||||
- **Engines listen on the management address only**, the one the control
|
||||
plane reaches them on, instead of every address the node has — a public
|
||||
one included. (When that address is a name, or not one of the node's,
|
||||
they still listen on all.) **SGLang's key is off its command line**,
|
||||
where every user on the node could read it: it reaches the engine in its
|
||||
environment, and a short launcher hands it to SGLang inside the process.
|
||||
Both take effect as engines restart under an upgraded agent.
|
||||
|
||||
### Everything that can be changed on a running cluster, from the console
|
||||
|
||||
|
||||
+41
-2
@@ -14,10 +14,12 @@ import (
|
||||
"math/rand/v2"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -310,7 +312,7 @@ func (a *Agent) settleAdopted(sh *supervisor.Shard, o supervisor.Orphan) {
|
||||
// The same two paths a freshly started shard takes: a rank that serves HTTP
|
||||
// is health-checked, a worker is judged by staying alive.
|
||||
if eng.ServesHTTP(o.Spec) {
|
||||
base := fmt.Sprintf("http://127.0.0.1:%d", o.Spec.HTTPPort)
|
||||
base := a.engineBase(o.Spec)
|
||||
err = a.sup.WaitReady(ctx, sh, base, o.Spec.Profile.ServedModelName, a.cfg.AdoptReadyTimeout)
|
||||
} else {
|
||||
err = a.sup.WaitAlive(ctx, sh)
|
||||
@@ -885,6 +887,42 @@ func (a *Agent) watchFetch(source string, fn func(weights.Progress)) (stop func(
|
||||
}
|
||||
}
|
||||
|
||||
// bindAddr is where engines listen: the management address the control
|
||||
// plane reaches them on, not every address the node has — a public one
|
||||
// included. Empty, on every address, when that is a name rather than an
|
||||
// address this machine holds.
|
||||
func (a *Agent) bindAddr() string {
|
||||
ip, err := netip.ParseAddr(a.cfg.MgmtAddr)
|
||||
if err != nil || ip.IsUnspecified() || !localAddr(ip) {
|
||||
return ""
|
||||
}
|
||||
return ip.String()
|
||||
}
|
||||
|
||||
// engineBase is where the agent asks an engine whether it is ready: where it
|
||||
// listens, loopback when that is every address.
|
||||
func (a *Agent) engineBase(s engines.ShardSpec) string {
|
||||
host := s.BindAddr
|
||||
if host == "" {
|
||||
host = "127.0.0.1"
|
||||
}
|
||||
return "http://" + net.JoinHostPort(host, strconv.Itoa(s.HTTPPort))
|
||||
}
|
||||
|
||||
// localAddr reports whether an address is one of this machine's.
|
||||
func localAddr(ip netip.Addr) bool {
|
||||
addrs, err := net.InterfaceAddrs()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
for _, a := range addrs {
|
||||
if p, err := netip.ParsePrefix(a.String()); err == nil && p.Addr().Unmap() == ip.Unmap() {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<- *llmgwv1.AgentMessage) {
|
||||
shardID, replicaID := m.GetShardId(), m.GetReplicaId()
|
||||
|
||||
@@ -907,6 +945,7 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
|
||||
|
||||
spec, timeout := agentsvc.ShardSpecFromStart(m)
|
||||
a.localizeSpec(&spec)
|
||||
spec.BindAddr = a.bindAddr()
|
||||
|
||||
eng, err := a.reg.Get(spec.Profile.Engine)
|
||||
if err != nil {
|
||||
@@ -993,7 +1032,7 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
|
||||
// staying alive, and the replica's real readiness is the leader answering,
|
||||
// which only happens once every worker has joined (plan §11, phase 4).
|
||||
if eng.ServesHTTP(spec) {
|
||||
base := fmt.Sprintf("http://127.0.0.1:%d", spec.HTTPPort)
|
||||
base := a.engineBase(spec)
|
||||
err = a.sup.WaitReady(ctx, sh, base, spec.Profile.ServedModelName, timeout)
|
||||
} else {
|
||||
err = a.sup.WaitAlive(ctx, sh)
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||
)
|
||||
|
||||
func TestEnginesListenOnTheManagementAddressWhenItIsThisMachines(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"127.0.0.1": "127.0.0.1", // one of this machine's
|
||||
"203.0.113.9": "", // not: every address rather than none
|
||||
"gpu07.lan": "", // a name
|
||||
"0.0.0.0": "",
|
||||
"": "",
|
||||
}
|
||||
for mgmt, want := range cases {
|
||||
a := &Agent{cfg: Config{MgmtAddr: mgmt}}
|
||||
if got := a.bindAddr(); got != want {
|
||||
t.Errorf("management address %q: listens on %q, expected %q", mgmt, got, want)
|
||||
}
|
||||
}
|
||||
a := &Agent{}
|
||||
if got := a.engineBase(engines.ShardSpec{HTTPPort: 8000}); got != "http://127.0.0.1:8000" {
|
||||
t.Errorf("on every address, asked at %s", got)
|
||||
}
|
||||
if got := a.engineBase(engines.ShardSpec{HTTPPort: 8000, BindAddr: "10.0.0.147"}); got != "http://10.0.0.147:8000" {
|
||||
t.Errorf("on one address, asked at %s", got)
|
||||
}
|
||||
}
|
||||
@@ -73,9 +73,13 @@ type ShardSpec struct {
|
||||
|
||||
HTTPPort int
|
||||
Profile Profile
|
||||
// BindAddr is the address the engine listens on: the node's management
|
||||
// address, which is what the router reaches it on, set by the agent.
|
||||
// Empty listens on every address.
|
||||
BindAddr string `json:"bind_addr,omitempty"`
|
||||
|
||||
// APIKey is what the engine demands of every request. Engines listen on
|
||||
// all of a node's addresses; without a key anybody who can reach the port
|
||||
// the node's management address; without a key anybody who can reach it
|
||||
// talks to the model directly, past the gateway's authentication, quotas
|
||||
// and accounting. Given to the engine in its environment rather than on
|
||||
// its command line, which every user on the node can read.
|
||||
@@ -98,6 +102,14 @@ type ShardSpec struct {
|
||||
ProfileFP string
|
||||
}
|
||||
|
||||
// Host is where the engine listens.
|
||||
func (s ShardSpec) Host() string {
|
||||
if s.BindAddr == "" {
|
||||
return "0.0.0.0"
|
||||
}
|
||||
return s.BindAddr
|
||||
}
|
||||
|
||||
// Leader reports whether this shard serves HTTP for the replica.
|
||||
func (s ShardSpec) Leader() bool { return s.Rank == 0 }
|
||||
|
||||
|
||||
@@ -323,8 +323,8 @@ func TestSGLangMultiNodeArgs(t *testing.T) {
|
||||
if cmd.Path != "/opt/sglang/bin/python" {
|
||||
t.Errorf("Path=%q", cmd.Path)
|
||||
}
|
||||
if cmd.Args[0] != "-m" || cmd.Args[1] != "sglang.launch_server" {
|
||||
t.Errorf("not launched as a module: %v", cmd.Args[:2])
|
||||
if cmd.Args[0] != "-c" || !strings.Contains(cmd.Args[1], `run_module("sglang.launch_server"`) {
|
||||
t.Errorf("not launched as the module: %v", cmd.Args[:2])
|
||||
}
|
||||
if argValue(cmd.Args, "--nnodes") != "2" || argValue(cmd.Args, "--node-rank") != "1" {
|
||||
t.Errorf("multi-node flags wrong: %v", cmd.Args)
|
||||
@@ -334,6 +334,38 @@ func TestSGLangMultiNodeArgs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSGLangsKeyIsNotOnItsCommandLine(t *testing.T) {
|
||||
s := singleNodeSpec()
|
||||
s.APIKey = "k1"
|
||||
cmd := buildOne(t, &SGLang{}, s)
|
||||
if slices.Contains(cmd.Args, "k1") || slices.Contains(cmd.Args, "--api-key") {
|
||||
t.Errorf("the key is in the arguments: %v", cmd.Args)
|
||||
}
|
||||
if !slices.Contains(cmd.Env, sglangKeyEnv+"=k1") {
|
||||
t.Errorf("the key is not in the environment")
|
||||
}
|
||||
// A worker serves no HTTP, and is not given it.
|
||||
w := multiNodeSpec(1)
|
||||
w.APIKey = "k1"
|
||||
if cmd := buildOne(t, &SGLang{}, w); slices.Contains(cmd.Env, sglangKeyEnv+"=k1") {
|
||||
t.Error("a worker was given the key")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnginesListenOnTheAddressTheyAreGiven(t *testing.T) {
|
||||
s := singleNodeSpec()
|
||||
for _, e := range []Engine{&VLLM{}, &SGLang{}} {
|
||||
s.BindAddr = ""
|
||||
if got := argValue(buildOne(t, e, s).Args, "--host"); got != "0.0.0.0" {
|
||||
t.Errorf("%s: without an address listens on %q", e.Name(), got)
|
||||
}
|
||||
s.BindAddr = "10.0.0.147"
|
||||
if got := argValue(buildOne(t, e, s).Args, "--host"); got != "10.0.0.147" {
|
||||
t.Errorf("%s: listens on %q, expected the management address", e.Name(), got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSGLangSingleNodeOmitsMultiNodeFlags(t *testing.T) {
|
||||
cmd := buildOne(t, &SGLang{}, singleNodeSpec())
|
||||
for _, flag := range []string{"--nnodes", "--node-rank", "--dist-init-addr"} {
|
||||
|
||||
@@ -99,7 +99,7 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) {
|
||||
}
|
||||
args := []string{
|
||||
"--model", model,
|
||||
"--host", "0.0.0.0",
|
||||
"--host", s.Host(),
|
||||
"--port", strconv.Itoa(s.HTTPPort),
|
||||
// Without this there is no /metrics, and an endpoint the router cannot
|
||||
// measure is one it has to route to blind.
|
||||
|
||||
@@ -79,9 +79,9 @@ func (g *SGLang) serverCommand(s ShardSpec) (Command, error) {
|
||||
python := g.python()
|
||||
p := s.Profile
|
||||
|
||||
args := []string{"-m", "sglang.launch_server",
|
||||
args := []string{"-c", sglangLauncher,
|
||||
"--model-path", p.ModelPath,
|
||||
"--host", "0.0.0.0",
|
||||
"--host", s.Host(),
|
||||
"--port", strconv.Itoa(s.HTTPPort),
|
||||
"--tp", strconv.Itoa(p.TP),
|
||||
}
|
||||
@@ -96,11 +96,23 @@ func (g *SGLang) serverCommand(s ShardSpec) (Command, error) {
|
||||
)
|
||||
}
|
||||
args = append(args, p.Args...)
|
||||
// SGLang reads its key only from the command line, so on a node with
|
||||
// other users it is visible to them in the process list.
|
||||
env := baseEnv(s)
|
||||
if s.APIKey != "" && s.Leader() {
|
||||
args = append(args, "--api-key", s.APIKey)
|
||||
env[sglangKeyEnv] = s.APIKey
|
||||
}
|
||||
|
||||
return Command{Name: "sglang", Path: python, Args: args, Env: materializeEnv(baseEnv(s))}, nil
|
||||
return Command{Name: "sglang", Path: python, Args: args, Env: materializeEnv(env)}, nil
|
||||
}
|
||||
|
||||
// SGLang reads its key only from the command line, where every user on the
|
||||
// node can read it in the process list. So the key goes in the environment,
|
||||
// which only the engine's own user and root can read, and this launcher —
|
||||
// sglang.launch_server under another name — adds it to the arguments inside
|
||||
// the process: sys.argv is Python's copy, not what /proc shows.
|
||||
const (
|
||||
sglangKeyEnv = "LLMGW_ENGINE_KEY"
|
||||
sglangLauncher = `import os, runpy, sys
|
||||
key = os.environ.pop("` + sglangKeyEnv + `", "")
|
||||
sys.argv = ["sglang.launch_server"] + sys.argv[1:] + (["--api-key", key] if key else [])
|
||||
runpy.run_module("sglang.launch_server", run_name="__main__", alter_sys=True)
|
||||
`
|
||||
)
|
||||
@@ -2,6 +2,7 @@ package engines
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"strconv"
|
||||
)
|
||||
@@ -63,7 +64,7 @@ func (s *Sim) Commands(spec ShardSpec) ([]Command, error) {
|
||||
p := spec.Profile
|
||||
|
||||
args := []string{"sim-engine",
|
||||
"--listen", "0.0.0.0:" + strconv.Itoa(spec.HTTPPort),
|
||||
"--listen", net.JoinHostPort(spec.Host(), strconv.Itoa(spec.HTTPPort)),
|
||||
"--model", p.ServedModelName,
|
||||
}
|
||||
if spec.NNodes > 1 {
|
||||
|
||||
@@ -106,7 +106,7 @@ func (v *VLLM) serverCommand(s ShardSpec) (Command, error) {
|
||||
p := s.Profile
|
||||
|
||||
args := []string{"serve", p.ModelPath,
|
||||
"--host", "0.0.0.0",
|
||||
"--host", s.Host(),
|
||||
"--port", strconv.Itoa(s.HTTPPort),
|
||||
"--tensor-parallel-size", strconv.Itoa(p.TP),
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user