Have engines listen on the management address, and take SGLang's key off its command line

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 10:36:42 +02:00
1 parent 9b61bbc983
commit 3b05ec0538
9 files changed
+148 -15

No files matched your search

+7
View File
@@ -152,6 +152,13 @@ APIs change between minors without a deprecation period.
before it — onboarding the node again issues one that works. A mirror
serving its management API with `--tls-ca` now requires `--api-client`
(the control plane's name), since the same authority signs every node.
- **Engines listen on the management address only**, the one the control
plane reaches them on, instead of every address the node has — a public
one included. (When that address is a name, or not one of the node's,
they still listen on all.) **SGLang's key is off its command line**,
where every user on the node could read it: it reaches the engine in its
environment, and a short launcher hands it to SGLang inside the process.
Both take effect as engines restart under an upgraded agent.
### Everything that can be changed on a running cluster, from the console
+41 -2
View File
@@ -14,10 +14,12 @@ import (
"math/rand/v2"
"net"
"net/http"
"net/netip"
"os"
"path/filepath"
"runtime"
"slices"
"strconv"
"sync"
"time"
@@ -310,7 +312,7 @@ func (a *Agent) settleAdopted(sh *supervisor.Shard, o supervisor.Orphan) {
// The same two paths a freshly started shard takes: a rank that serves HTTP
// is health-checked, a worker is judged by staying alive.
if eng.ServesHTTP(o.Spec) {
base := fmt.Sprintf("http://127.0.0.1:%d", o.Spec.HTTPPort)
base := a.engineBase(o.Spec)
err = a.sup.WaitReady(ctx, sh, base, o.Spec.Profile.ServedModelName, a.cfg.AdoptReadyTimeout)
} else {
err = a.sup.WaitAlive(ctx, sh)
@@ -885,6 +887,42 @@ func (a *Agent) watchFetch(source string, fn func(weights.Progress)) (stop func(
}
}
// bindAddr is where engines listen: the management address the control
// plane reaches them on, not every address the node has — a public one
// included. Empty, on every address, when that is a name rather than an
// address this machine holds.
func (a *Agent) bindAddr() string {
ip, err := netip.ParseAddr(a.cfg.MgmtAddr)
if err != nil || ip.IsUnspecified() || !localAddr(ip) {
return ""
}
return ip.String()
}
// engineBase is where the agent asks an engine whether it is ready: where it
// listens, loopback when that is every address.
func (a *Agent) engineBase(s engines.ShardSpec) string {
host := s.BindAddr
if host == "" {
host = "127.0.0.1"
}
return "http://" + net.JoinHostPort(host, strconv.Itoa(s.HTTPPort))
}
// localAddr reports whether an address is one of this machine's.
func localAddr(ip netip.Addr) bool {
addrs, err := net.InterfaceAddrs()
if err != nil {
return false
}
for _, a := range addrs {
if p, err := netip.ParsePrefix(a.String()); err == nil && p.Addr().Unmap() == ip.Unmap() {
return true
}
}
return false
}
func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<- *llmgwv1.AgentMessage) {
shardID, replicaID := m.GetShardId(), m.GetReplicaId()
@@ -907,6 +945,7 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
spec, timeout := agentsvc.ShardSpecFromStart(m)
a.localizeSpec(&spec)
spec.BindAddr = a.bindAddr()
eng, err := a.reg.Get(spec.Profile.Engine)
if err != nil {
@@ -993,7 +1032,7 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
// staying alive, and the replica's real readiness is the leader answering,
// which only happens once every worker has joined (plan §11, phase 4).
if eng.ServesHTTP(spec) {
base := fmt.Sprintf("http://127.0.0.1:%d", spec.HTTPPort)
base := a.engineBase(spec)
err = a.sup.WaitReady(ctx, sh, base, spec.Profile.ServedModelName, timeout)
} else {
err = a.sup.WaitAlive(ctx, sh)
+30
View File
@@ -0,0 +1,30 @@
package agent
import (
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
)
func TestEnginesListenOnTheManagementAddressWhenItIsThisMachines(t *testing.T) {
cases := map[string]string{
"127.0.0.1": "127.0.0.1", // one of this machine's
"203.0.113.9": "", // not: every address rather than none
"gpu07.lan": "", // a name
"0.0.0.0": "",
"": "",
}
for mgmt, want := range cases {
a := &Agent{cfg: Config{MgmtAddr: mgmt}}
if got := a.bindAddr(); got != want {
t.Errorf("management address %q: listens on %q, expected %q", mgmt, got, want)
}
}
a := &Agent{}
if got := a.engineBase(engines.ShardSpec{HTTPPort: 8000}); got != "http://127.0.0.1:8000" {
t.Errorf("on every address, asked at %s", got)
}
if got := a.engineBase(engines.ShardSpec{HTTPPort: 8000, BindAddr: "10.0.0.147"}); got != "http://10.0.0.147:8000" {
t.Errorf("on one address, asked at %s", got)
}
}
+13 -1
View File
@@ -73,9 +73,13 @@ type ShardSpec struct {
HTTPPort int
Profile Profile
// BindAddr is the address the engine listens on: the node's management
// address, which is what the router reaches it on, set by the agent.
// Empty listens on every address.
BindAddr string `json:"bind_addr,omitempty"`
// APIKey is what the engine demands of every request. Engines listen on
// all of a node's addresses; without a key anybody who can reach the port
// the node's management address; without a key anybody who can reach it
// talks to the model directly, past the gateway's authentication, quotas
// and accounting. Given to the engine in its environment rather than on
// its command line, which every user on the node can read.
@@ -98,6 +102,14 @@ type ShardSpec struct {
ProfileFP string
}
// Host is where the engine listens.
func (s ShardSpec) Host() string {
if s.BindAddr == "" {
return "0.0.0.0"
}
return s.BindAddr
}
// Leader reports whether this shard serves HTTP for the replica.
func (s ShardSpec) Leader() bool { return s.Rank == 0 }
+34 -2
View File
@@ -323,8 +323,8 @@ func TestSGLangMultiNodeArgs(t *testing.T) {
if cmd.Path != "/opt/sglang/bin/python" {
t.Errorf("Path=%q", cmd.Path)
}
if cmd.Args[0] != "-m" || cmd.Args[1] != "sglang.launch_server" {
t.Errorf("not launched as a module: %v", cmd.Args[:2])
if cmd.Args[0] != "-c" || !strings.Contains(cmd.Args[1], `run_module("sglang.launch_server"`) {
t.Errorf("not launched as the module: %v", cmd.Args[:2])
}
if argValue(cmd.Args, "--nnodes") != "2" || argValue(cmd.Args, "--node-rank") != "1" {
t.Errorf("multi-node flags wrong: %v", cmd.Args)
@@ -334,6 +334,38 @@ func TestSGLangMultiNodeArgs(t *testing.T) {
}
}
func TestSGLangsKeyIsNotOnItsCommandLine(t *testing.T) {
s := singleNodeSpec()
s.APIKey = "k1"
cmd := buildOne(t, &SGLang{}, s)
if slices.Contains(cmd.Args, "k1") || slices.Contains(cmd.Args, "--api-key") {
t.Errorf("the key is in the arguments: %v", cmd.Args)
}
if !slices.Contains(cmd.Env, sglangKeyEnv+"=k1") {
t.Errorf("the key is not in the environment")
}
// A worker serves no HTTP, and is not given it.
w := multiNodeSpec(1)
w.APIKey = "k1"
if cmd := buildOne(t, &SGLang{}, w); slices.Contains(cmd.Env, sglangKeyEnv+"=k1") {
t.Error("a worker was given the key")
}
}
func TestEnginesListenOnTheAddressTheyAreGiven(t *testing.T) {
s := singleNodeSpec()
for _, e := range []Engine{&VLLM{}, &SGLang{}} {
s.BindAddr = ""
if got := argValue(buildOne(t, e, s).Args, "--host"); got != "0.0.0.0" {
t.Errorf("%s: without an address listens on %q", e.Name(), got)
}
s.BindAddr = "10.0.0.147"
if got := argValue(buildOne(t, e, s).Args, "--host"); got != "10.0.0.147" {
t.Errorf("%s: listens on %q, expected the management address", e.Name(), got)
}
}
}
func TestSGLangSingleNodeOmitsMultiNodeFlags(t *testing.T) {
cmd := buildOne(t, &SGLang{}, singleNodeSpec())
for _, flag := range []string{"--nnodes", "--node-rank", "--dist-init-addr"} {
+1 -1
View File
@@ -99,7 +99,7 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) {
}
args := []string{
"--model", model,
"--host", "0.0.0.0",
"--host", s.Host(),
"--port", strconv.Itoa(s.HTTPPort),
// Without this there is no /metrics, and an endpoint the router cannot
// measure is one it has to route to blind.
+19 -7
View File
@@ -79,9 +79,9 @@ func (g *SGLang) serverCommand(s ShardSpec) (Command, error) {
python := g.python()
p := s.Profile
args := []string{"-m", "sglang.launch_server",
args := []string{"-c", sglangLauncher,
"--model-path", p.ModelPath,
"--host", "0.0.0.0",
"--host", s.Host(),
"--port", strconv.Itoa(s.HTTPPort),
"--tp", strconv.Itoa(p.TP),
}
@@ -96,11 +96,23 @@ func (g *SGLang) serverCommand(s ShardSpec) (Command, error) {
)
}
args = append(args, p.Args...)
// SGLang reads its key only from the command line, so on a node with
// other users it is visible to them in the process list.
env := baseEnv(s)
if s.APIKey != "" && s.Leader() {
args = append(args, "--api-key", s.APIKey)
env[sglangKeyEnv] = s.APIKey
}
return Command{Name: "sglang", Path: python, Args: args, Env: materializeEnv(baseEnv(s))}, nil
return Command{Name: "sglang", Path: python, Args: args, Env: materializeEnv(env)}, nil
}
// SGLang reads its key only from the command line, where every user on the
// node can read it in the process list. So the key goes in the environment,
// which only the engine's own user and root can read, and this launcher —
// sglang.launch_server under another name — adds it to the arguments inside
// the process: sys.argv is Python's copy, not what /proc shows.
const (
sglangKeyEnv = "LLMGW_ENGINE_KEY"
sglangLauncher = `import os, runpy, sys
key = os.environ.pop("` + sglangKeyEnv + `", "")
sys.argv = ["sglang.launch_server"] + sys.argv[1:] + (["--api-key", key] if key else [])
runpy.run_module("sglang.launch_server", run_name="__main__", alter_sys=True)
`
)
+2 -1
View File
@@ -2,6 +2,7 @@ package engines
import (
"fmt"
"net"
"os"
"strconv"
)
@@ -63,7 +64,7 @@ func (s *Sim) Commands(spec ShardSpec) ([]Command, error) {
p := spec.Profile
args := []string{"sim-engine",
"--listen", "0.0.0.0:" + strconv.Itoa(spec.HTTPPort),
"--listen", net.JoinHostPort(spec.Host(), strconv.Itoa(spec.HTTPPort)),
"--model", p.ServedModelName,
}
if spec.NNodes > 1 {
+1 -1
View File
@@ -106,7 +106,7 @@ func (v *VLLM) serverCommand(s ShardSpec) (Command, error) {
p := s.Profile
args := []string{"serve", p.ModelPath,
"--host", "0.0.0.0",
"--host", s.Host(),
"--port", strconv.Itoa(s.HTTPPort),
"--tensor-parallel-size", strconv.Itoa(p.TP),
}