diff --git a/deploy/bootstrap/onboard-node.sh b/deploy/bootstrap/onboard-node.sh index 5fbc27d..00c0264 100755 --- a/deploy/bootstrap/onboard-node.sh +++ b/deploy/bootstrap/onboard-node.sh @@ -226,6 +226,10 @@ ExecStart=/usr/local/bin/llmgw agent --config /etc/llmgw/agent.yaml KillMode=process Restart=on-failure RestartSec=2s +# RDMA pins the buffers it registers, and NCCL registers a great deal: with +# systemd's default 8 MB lock limit, an engine that drops the capability that +# lets root ignore it fails its first collective over InfiniBand. +LimitMEMLOCK=infinity StateDirectory=llmgw-agent [Install] diff --git a/deploy/systemd/llmgw-agent.service b/deploy/systemd/llmgw-agent.service index fd52957..75cadfa 100644 --- a/deploy/systemd/llmgw-agent.service +++ b/deploy/systemd/llmgw-agent.service @@ -11,6 +11,10 @@ ExecStart=/usr/local/bin/llmgw agent --config /etc/llmgw/agent.yaml KillMode=process Restart=on-failure RestartSec=2s +# RDMA pins the buffers it registers, and NCCL registers a great deal: with +# systemd's default 8 MB lock limit, an engine that drops the capability that +# lets root ignore it fails its first collective over InfiniBand. +LimitMEMLOCK=infinity StateDirectory=llmgw-agent [Install] diff --git a/internal/agent/agent.go b/internal/agent/agent.go index a0f4bdb..4755291 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -118,8 +118,11 @@ type Config struct { // DefaultConfig returns sensible defaults. func DefaultConfig() Config { + // No MgmtInterface: normalize finds the one holding the management + // address, and falls back to eth0 only when it cannot. Defaulted here, it + // was never found — every node said eth0 unless told otherwise, and a node + // on ens3 or bond0 had NCCL bootstrap over an interface it does not have. return Config{ - MgmtInterface: "eth0", HeartbeatInterval: 5 * time.Second, ReconnectBase: time.Second, ReconnectMax: 30 * time.Second, @@ -159,7 +162,7 @@ func (c *Config) normalize() error { c.MgmtInterface = interfaceOf(c.MgmtAddr) } if c.MgmtInterface == "" { - c.MgmtInterface = d.MgmtInterface + c.MgmtInterface = "eth0" } // The directory downloads land in is always one that is looked in. When // they were two settings, a node could fill its disk with fetched weights diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index 9ca6a39..e3bcdd4 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -813,3 +813,18 @@ func TestAShardUsesThisNodesInterfaceAndRoCEIndex(t *testing.T) { t.Error("an InfiniBand shard was given a GID index") } } + +func TestTheManagementInterfaceIsFoundNotAssumed(t *testing.T) { + // It was defaulted to eth0 before it could be looked for, so it never + // was: a node on ens3 or bond0 had NCCL bootstrap over nothing. + cfg := DefaultConfig() + cfg.Server = "http://127.0.0.1:7443" + cfg.NodeID = "n1" + cfg.MgmtAddr = "127.0.0.1" + if err := cfg.normalize(); err != nil { + t.Fatal(err) + } + if cfg.MgmtInterface == "" || cfg.MgmtInterface == "eth0" { + t.Errorf("interface = %q, want the loopback interface holding 127.0.0.1", cfg.MgmtInterface) + } +} diff --git a/internal/cli/serverinit_cluster.go b/internal/cli/serverinit_cluster.go index d7046e1..39fa441 100644 --- a/internal/cli/serverinit_cluster.go +++ b/internal/cli/serverinit_cluster.go @@ -62,6 +62,10 @@ ExecStart=/usr/local/bin/llmgw agent --config /etc/llmgw/agent.yaml KillMode=process Restart=on-failure RestartSec=2s +# RDMA pins the buffers it registers, and NCCL registers a great deal: with +# systemd's default 8 MB lock limit, an engine that drops the capability that +# lets root ignore it fails its first collective over InfiniBand. +LimitMEMLOCK=infinity StateDirectory=llmgw-agent [Install]