From 4d9284ce9fc236cbff099c32e980a2eacbbdd74e Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Tue, 29 Sep 2026 01:44:59 +0200 Subject: [PATCH] Find each node's management interface, and lift the agent's memlock limit The agent's defaults said eth0 before normalize looked for the interface holding the management address, so it never looked: a node on ens3 or bond0 had NCCL and Gloo bootstrap over an interface it does not have. It is now found, with eth0 only as the last resort. The agent unit sets LimitMEMLOCK=infinity. RDMA pins what it registers, and under systemd's 8 MB default an engine without the capability that lets root ignore the limit fails its first collective over InfiniBand. Co-Authored-By: Claude Opus 5.5 (1M context) --- deploy/bootstrap/onboard-node.sh | 4 ++++ deploy/systemd/llmgw-agent.service | 4 ++++ internal/agent/agent.go | 7 +++++-- internal/agent/agent_test.go | 15 +++++++++++++++ internal/cli/serverinit_cluster.go | 4 ++++ 5 files changed, 32 insertions(+), 2 deletions(-) diff --git a/deploy/bootstrap/onboard-node.sh b/deploy/bootstrap/onboard-node.sh index 5fbc27d..00c0264 100755 --- a/deploy/bootstrap/onboard-node.sh +++ b/deploy/bootstrap/onboard-node.sh @@ -226,6 +226,10 @@ ExecStart=/usr/local/bin/llmgw agent --config /etc/llmgw/agent.yaml KillMode=process Restart=on-failure RestartSec=2s +# RDMA pins the buffers it registers, and NCCL registers a great deal: with +# systemd's default 8 MB lock limit, an engine that drops the capability that +# lets root ignore it fails its first collective over InfiniBand. +LimitMEMLOCK=infinity StateDirectory=llmgw-agent [Install] diff --git a/deploy/systemd/llmgw-agent.service b/deploy/systemd/llmgw-agent.service index fd52957..75cadfa 100644 --- a/deploy/systemd/llmgw-agent.service +++ b/deploy/systemd/llmgw-agent.service @@ -11,6 +11,10 @@ ExecStart=/usr/local/bin/llmgw agent --config /etc/llmgw/agent.yaml KillMode=process Restart=on-failure RestartSec=2s +# RDMA pins the buffers it registers, and NCCL registers a great deal: with +# systemd's default 8 MB lock limit, an engine that drops the capability that +# lets root ignore it fails its first collective over InfiniBand. +LimitMEMLOCK=infinity StateDirectory=llmgw-agent [Install] diff --git a/internal/agent/agent.go b/internal/agent/agent.go index a0f4bdb..4755291 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -118,8 +118,11 @@ type Config struct { // DefaultConfig returns sensible defaults. func DefaultConfig() Config { + // No MgmtInterface: normalize finds the one holding the management + // address, and falls back to eth0 only when it cannot. Defaulted here, it + // was never found — every node said eth0 unless told otherwise, and a node + // on ens3 or bond0 had NCCL bootstrap over an interface it does not have. return Config{ - MgmtInterface: "eth0", HeartbeatInterval: 5 * time.Second, ReconnectBase: time.Second, ReconnectMax: 30 * time.Second, @@ -159,7 +162,7 @@ func (c *Config) normalize() error { c.MgmtInterface = interfaceOf(c.MgmtAddr) } if c.MgmtInterface == "" { - c.MgmtInterface = d.MgmtInterface + c.MgmtInterface = "eth0" } // The directory downloads land in is always one that is looked in. When // they were two settings, a node could fill its disk with fetched weights diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index 9ca6a39..e3bcdd4 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -813,3 +813,18 @@ func TestAShardUsesThisNodesInterfaceAndRoCEIndex(t *testing.T) { t.Error("an InfiniBand shard was given a GID index") } } + +func TestTheManagementInterfaceIsFoundNotAssumed(t *testing.T) { + // It was defaulted to eth0 before it could be looked for, so it never + // was: a node on ens3 or bond0 had NCCL bootstrap over nothing. + cfg := DefaultConfig() + cfg.Server = "http://127.0.0.1:7443" + cfg.NodeID = "n1" + cfg.MgmtAddr = "127.0.0.1" + if err := cfg.normalize(); err != nil { + t.Fatal(err) + } + if cfg.MgmtInterface == "" || cfg.MgmtInterface == "eth0" { + t.Errorf("interface = %q, want the loopback interface holding 127.0.0.1", cfg.MgmtInterface) + } +} diff --git a/internal/cli/serverinit_cluster.go b/internal/cli/serverinit_cluster.go index d7046e1..39fa441 100644 --- a/internal/cli/serverinit_cluster.go +++ b/internal/cli/serverinit_cluster.go @@ -62,6 +62,10 @@ ExecStart=/usr/local/bin/llmgw agent --config /etc/llmgw/agent.yaml KillMode=process Restart=on-failure RestartSec=2s +# RDMA pins the buffers it registers, and NCCL registers a great deal: with +# systemd's default 8 MB lock limit, an engine that drops the capability that +# lets root ignore it fails its first collective over InfiniBand. +LimitMEMLOCK=infinity StateDirectory=llmgw-agent [Install]