diff --git a/CHANGELOG.md b/CHANGELOG.md index 92e2225..19fcba2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -152,6 +152,13 @@ APIs change between minors without a deprecation period. before it — onboarding the node again issues one that works. A mirror serving its management API with `--tls-ca` now requires `--api-client` (the control plane's name), since the same authority signs every node. +- **Engines listen on the management address only**, the one the control + plane reaches them on, instead of every address the node has — a public + one included. (When that address is a name, or not one of the node's, + they still listen on all.) **SGLang's key is off its command line**, + where every user on the node could read it: it reaches the engine in its + environment, and a short launcher hands it to SGLang inside the process. + Both take effect as engines restart under an upgraded agent. ### Everything that can be changed on a running cluster, from the console diff --git a/internal/agent/agent.go b/internal/agent/agent.go index e26bde1..832bd2f 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -14,10 +14,12 @@ import ( "math/rand/v2" "net" "net/http" + "net/netip" "os" "path/filepath" "runtime" "slices" + "strconv" "sync" "time" @@ -310,7 +312,7 @@ func (a *Agent) settleAdopted(sh *supervisor.Shard, o supervisor.Orphan) { // The same two paths a freshly started shard takes: a rank that serves HTTP // is health-checked, a worker is judged by staying alive. if eng.ServesHTTP(o.Spec) { - base := fmt.Sprintf("http://127.0.0.1:%d", o.Spec.HTTPPort) + base := a.engineBase(o.Spec) err = a.sup.WaitReady(ctx, sh, base, o.Spec.Profile.ServedModelName, a.cfg.AdoptReadyTimeout) } else { err = a.sup.WaitAlive(ctx, sh) @@ -885,6 +887,42 @@ func (a *Agent) watchFetch(source string, fn func(weights.Progress)) (stop func( } } +// bindAddr is where engines listen: the management address the control +// plane reaches them on, not every address the node has — a public one +// included. Empty, on every address, when that is a name rather than an +// address this machine holds. +func (a *Agent) bindAddr() string { + ip, err := netip.ParseAddr(a.cfg.MgmtAddr) + if err != nil || ip.IsUnspecified() || !localAddr(ip) { + return "" + } + return ip.String() +} + +// engineBase is where the agent asks an engine whether it is ready: where it +// listens, loopback when that is every address. +func (a *Agent) engineBase(s engines.ShardSpec) string { + host := s.BindAddr + if host == "" { + host = "127.0.0.1" + } + return "http://" + net.JoinHostPort(host, strconv.Itoa(s.HTTPPort)) +} + +// localAddr reports whether an address is one of this machine's. +func localAddr(ip netip.Addr) bool { + addrs, err := net.InterfaceAddrs() + if err != nil { + return false + } + for _, a := range addrs { + if p, err := netip.ParsePrefix(a.String()); err == nil && p.Addr().Unmap() == ip.Unmap() { + return true + } + } + return false +} + func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<- *llmgwv1.AgentMessage) { shardID, replicaID := m.GetShardId(), m.GetReplicaId() @@ -907,6 +945,7 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan< spec, timeout := agentsvc.ShardSpecFromStart(m) a.localizeSpec(&spec) + spec.BindAddr = a.bindAddr() eng, err := a.reg.Get(spec.Profile.Engine) if err != nil { @@ -993,7 +1032,7 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan< // staying alive, and the replica's real readiness is the leader answering, // which only happens once every worker has joined (plan §11, phase 4). if eng.ServesHTTP(spec) { - base := fmt.Sprintf("http://127.0.0.1:%d", spec.HTTPPort) + base := a.engineBase(spec) err = a.sup.WaitReady(ctx, sh, base, spec.Profile.ServedModelName, timeout) } else { err = a.sup.WaitAlive(ctx, sh) diff --git a/internal/agent/bindaddr_test.go b/internal/agent/bindaddr_test.go new file mode 100644 index 0000000..e7ec316 --- /dev/null +++ b/internal/agent/bindaddr_test.go @@ -0,0 +1,30 @@ +package agent + +import ( + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines" +) + +func TestEnginesListenOnTheManagementAddressWhenItIsThisMachines(t *testing.T) { + cases := map[string]string{ + "127.0.0.1": "127.0.0.1", // one of this machine's + "203.0.113.9": "", // not: every address rather than none + "gpu07.lan": "", // a name + "0.0.0.0": "", + "": "", + } + for mgmt, want := range cases { + a := &Agent{cfg: Config{MgmtAddr: mgmt}} + if got := a.bindAddr(); got != want { + t.Errorf("management address %q: listens on %q, expected %q", mgmt, got, want) + } + } + a := &Agent{} + if got := a.engineBase(engines.ShardSpec{HTTPPort: 8000}); got != "http://127.0.0.1:8000" { + t.Errorf("on every address, asked at %s", got) + } + if got := a.engineBase(engines.ShardSpec{HTTPPort: 8000, BindAddr: "10.0.0.147"}); got != "http://10.0.0.147:8000" { + t.Errorf("on one address, asked at %s", got) + } +} diff --git a/internal/agent/engines/engine.go b/internal/agent/engines/engine.go index edc8829..484ffed 100644 --- a/internal/agent/engines/engine.go +++ b/internal/agent/engines/engine.go @@ -73,9 +73,13 @@ type ShardSpec struct { HTTPPort int Profile Profile + // BindAddr is the address the engine listens on: the node's management + // address, which is what the router reaches it on, set by the agent. + // Empty listens on every address. + BindAddr string `json:"bind_addr,omitempty"` // APIKey is what the engine demands of every request. Engines listen on - // all of a node's addresses; without a key anybody who can reach the port + // the node's management address; without a key anybody who can reach it // talks to the model directly, past the gateway's authentication, quotas // and accounting. Given to the engine in its environment rather than on // its command line, which every user on the node can read. @@ -98,6 +102,14 @@ type ShardSpec struct { ProfileFP string } +// Host is where the engine listens. +func (s ShardSpec) Host() string { + if s.BindAddr == "" { + return "0.0.0.0" + } + return s.BindAddr +} + // Leader reports whether this shard serves HTTP for the replica. func (s ShardSpec) Leader() bool { return s.Rank == 0 } diff --git a/internal/agent/engines/engines_test.go b/internal/agent/engines/engines_test.go index b59a24f..049f0a2 100644 --- a/internal/agent/engines/engines_test.go +++ b/internal/agent/engines/engines_test.go @@ -323,8 +323,8 @@ func TestSGLangMultiNodeArgs(t *testing.T) { if cmd.Path != "/opt/sglang/bin/python" { t.Errorf("Path=%q", cmd.Path) } - if cmd.Args[0] != "-m" || cmd.Args[1] != "sglang.launch_server" { - t.Errorf("not launched as a module: %v", cmd.Args[:2]) + if cmd.Args[0] != "-c" || !strings.Contains(cmd.Args[1], `run_module("sglang.launch_server"`) { + t.Errorf("not launched as the module: %v", cmd.Args[:2]) } if argValue(cmd.Args, "--nnodes") != "2" || argValue(cmd.Args, "--node-rank") != "1" { t.Errorf("multi-node flags wrong: %v", cmd.Args) @@ -334,6 +334,38 @@ func TestSGLangMultiNodeArgs(t *testing.T) { } } +func TestSGLangsKeyIsNotOnItsCommandLine(t *testing.T) { + s := singleNodeSpec() + s.APIKey = "k1" + cmd := buildOne(t, &SGLang{}, s) + if slices.Contains(cmd.Args, "k1") || slices.Contains(cmd.Args, "--api-key") { + t.Errorf("the key is in the arguments: %v", cmd.Args) + } + if !slices.Contains(cmd.Env, sglangKeyEnv+"=k1") { + t.Errorf("the key is not in the environment") + } + // A worker serves no HTTP, and is not given it. + w := multiNodeSpec(1) + w.APIKey = "k1" + if cmd := buildOne(t, &SGLang{}, w); slices.Contains(cmd.Env, sglangKeyEnv+"=k1") { + t.Error("a worker was given the key") + } +} + +func TestEnginesListenOnTheAddressTheyAreGiven(t *testing.T) { + s := singleNodeSpec() + for _, e := range []Engine{&VLLM{}, &SGLang{}} { + s.BindAddr = "" + if got := argValue(buildOne(t, e, s).Args, "--host"); got != "0.0.0.0" { + t.Errorf("%s: without an address listens on %q", e.Name(), got) + } + s.BindAddr = "10.0.0.147" + if got := argValue(buildOne(t, e, s).Args, "--host"); got != "10.0.0.147" { + t.Errorf("%s: listens on %q, expected the management address", e.Name(), got) + } + } +} + func TestSGLangSingleNodeOmitsMultiNodeFlags(t *testing.T) { cmd := buildOne(t, &SGLang{}, singleNodeSpec()) for _, flag := range []string{"--nnodes", "--node-rank", "--dist-init-addr"} { diff --git a/internal/agent/engines/llamacpp.go b/internal/agent/engines/llamacpp.go index 33cadf4..02a27a6 100644 --- a/internal/agent/engines/llamacpp.go +++ b/internal/agent/engines/llamacpp.go @@ -99,7 +99,7 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) { } args := []string{ "--model", model, - "--host", "0.0.0.0", + "--host", s.Host(), "--port", strconv.Itoa(s.HTTPPort), // Without this there is no /metrics, and an endpoint the router cannot // measure is one it has to route to blind. diff --git a/internal/agent/engines/sglang.go b/internal/agent/engines/sglang.go index 501c33d..ccaa572 100644 --- a/internal/agent/engines/sglang.go +++ b/internal/agent/engines/sglang.go @@ -79,9 +79,9 @@ func (g *SGLang) serverCommand(s ShardSpec) (Command, error) { python := g.python() p := s.Profile - args := []string{"-m", "sglang.launch_server", + args := []string{"-c", sglangLauncher, "--model-path", p.ModelPath, - "--host", "0.0.0.0", + "--host", s.Host(), "--port", strconv.Itoa(s.HTTPPort), "--tp", strconv.Itoa(p.TP), } @@ -96,11 +96,23 @@ func (g *SGLang) serverCommand(s ShardSpec) (Command, error) { ) } args = append(args, p.Args...) - // SGLang reads its key only from the command line, so on a node with - // other users it is visible to them in the process list. + env := baseEnv(s) if s.APIKey != "" && s.Leader() { - args = append(args, "--api-key", s.APIKey) + env[sglangKeyEnv] = s.APIKey } - - return Command{Name: "sglang", Path: python, Args: args, Env: materializeEnv(baseEnv(s))}, nil + return Command{Name: "sglang", Path: python, Args: args, Env: materializeEnv(env)}, nil } + +// SGLang reads its key only from the command line, where every user on the +// node can read it in the process list. So the key goes in the environment, +// which only the engine's own user and root can read, and this launcher — +// sglang.launch_server under another name — adds it to the arguments inside +// the process: sys.argv is Python's copy, not what /proc shows. +const ( + sglangKeyEnv = "LLMGW_ENGINE_KEY" + sglangLauncher = `import os, runpy, sys +key = os.environ.pop("` + sglangKeyEnv + `", "") +sys.argv = ["sglang.launch_server"] + sys.argv[1:] + (["--api-key", key] if key else []) +runpy.run_module("sglang.launch_server", run_name="__main__", alter_sys=True) +` +) diff --git a/internal/agent/engines/sim.go b/internal/agent/engines/sim.go index d8d608b..9590627 100644 --- a/internal/agent/engines/sim.go +++ b/internal/agent/engines/sim.go @@ -2,6 +2,7 @@ package engines import ( "fmt" + "net" "os" "strconv" ) @@ -63,7 +64,7 @@ func (s *Sim) Commands(spec ShardSpec) ([]Command, error) { p := spec.Profile args := []string{"sim-engine", - "--listen", "0.0.0.0:" + strconv.Itoa(spec.HTTPPort), + "--listen", net.JoinHostPort(spec.Host(), strconv.Itoa(spec.HTTPPort)), "--model", p.ServedModelName, } if spec.NNodes > 1 { diff --git a/internal/agent/engines/vllm.go b/internal/agent/engines/vllm.go index 0d4b532..1fff60b 100644 --- a/internal/agent/engines/vllm.go +++ b/internal/agent/engines/vllm.go @@ -106,7 +106,7 @@ func (v *VLLM) serverCommand(s ShardSpec) (Command, error) { p := s.Profile args := []string{"serve", p.ModelPath, - "--host", "0.0.0.0", + "--host", s.Host(), "--port", strconv.Itoa(s.HTTPPort), "--tensor-parallel-size", strconv.Itoa(p.TP), }