Slurm's ports for the cluster only, and srun in a range of its own
CI / test (push) Successful in 9m40s
CI / test (push) Successful in 9m40s
slurm.conf sets SrunPortRange=60001-63000, so a site's firewall can let srun's step connections in. A node given a part in Slurm adds 6817-6818 and that range to llmgw's nftables table, open to the cluster's networks and closed to everyone else, as the engines' ports are. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
914985bf9f
commit
3387f97f6a
7 files changed
+90
-4
No files matched your search
@@ -98,7 +98,10 @@ type Agent struct {
|
||||
// (nil: this machine's shell).
|
||||
fwMu sync.Mutex
|
||||
fwRunner firewall.Runner
|
||||
inv discovery.Inventory
|
||||
// fwLast is the last firewall message, applied again when the node
|
||||
// is given a part in Slurm. Under fwMu.
|
||||
fwLast *llmgwv1.Firewall
|
||||
inv discovery.Inventory
|
||||
// shardReplica remembers which replica a shard belongs to, so a status
|
||||
// report can name both. The control plane keys on the replica.
|
||||
shardReplica map[string]string
|
||||
@@ -961,7 +964,7 @@ func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out cha
|
||||
case *llmgwv1.ControlMessage_TrainStart:
|
||||
go a.trainJob(ctx, m.TrainStart, out)
|
||||
case *llmgwv1.ControlMessage_Slurm:
|
||||
a.slurm.Apply(m.Slurm)
|
||||
a.slurmMessage(m.Slurm)
|
||||
case *llmgwv1.ControlMessage_TrainCancel:
|
||||
a.trainer.Cancel(m.TrainCancel.GetJobId())
|
||||
case *llmgwv1.ControlMessage_RestartAgent:
|
||||
|
||||
@@ -10,8 +10,16 @@ import (
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/firewall"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
|
||||
)
|
||||
|
||||
// slurmPorts are Slurm's on a node that runs any of it: slurmctld's and
|
||||
// slurmd's, and srun's range for the steps it starts.
|
||||
var slurmPorts = []firewall.Port{
|
||||
{From: slurm.CtldPort, To: slurm.SlurmdPort, What: "slurm: slurmctld and slurmd"},
|
||||
{From: slurm.SrunPortsFrom, To: slurm.SrunPortsTo, What: "slurm: srun, for the steps it starts"},
|
||||
}
|
||||
|
||||
// applyFirewall keeps llmgw's firewall on this node as the cluster's
|
||||
// settings say: the engines' ports and the node exporter's for the networks
|
||||
// the node reaches its control planes on, and those named, or no table at
|
||||
@@ -19,6 +27,7 @@ import (
|
||||
func (a *Agent) applyFirewall(m *llmgwv1.Firewall) {
|
||||
a.fwMu.Lock()
|
||||
defer a.fwMu.Unlock()
|
||||
a.fwLast = m
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
r := a.fwRunner
|
||||
@@ -55,6 +64,11 @@ func (a *Agent) applyFirewall(m *llmgwv1.Firewall) {
|
||||
if from > 0 && to >= from {
|
||||
ports = append(ports, firewall.Engines(from, to))
|
||||
}
|
||||
// Once there, they stay: the ruleset keeps every port it held, so a
|
||||
// node that leaves Slurm keeps them closed to everyone but the cluster.
|
||||
if a.slurm != nil && a.slurm.HasPart() {
|
||||
ports = append(ports, slurmPorts...)
|
||||
}
|
||||
changed, err := firewall.Install(ctx, r, firewall.Spec{Ports: ports, Allow: nets})
|
||||
switch {
|
||||
case err != nil:
|
||||
@@ -64,6 +78,22 @@ func (a *Agent) applyFirewall(m *llmgwv1.Firewall) {
|
||||
}
|
||||
}
|
||||
|
||||
// slurmMessage is the leader's word on this node's part in Slurm. A node
|
||||
// that gets one has the firewall applied again, with Slurm's ports.
|
||||
func (a *Agent) slurmMessage(m *llmgwv1.Slurm) {
|
||||
had := a.slurm.HasPart()
|
||||
a.slurm.Apply(m)
|
||||
if had || !a.slurm.HasPart() {
|
||||
return
|
||||
}
|
||||
a.fwMu.Lock()
|
||||
last := a.fwLast
|
||||
a.fwMu.Unlock()
|
||||
if last != nil && last.GetEnabled() {
|
||||
go a.applyFirewall(last)
|
||||
}
|
||||
}
|
||||
|
||||
// controlPlaneIPs are the addresses of the control planes an agent dials.
|
||||
func controlPlaneIPs(ctx context.Context, servers []string) []string {
|
||||
var out []string
|
||||
|
||||
@@ -6,8 +6,10 @@ import (
|
||||
"log/slog"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/slurmnode"
|
||||
)
|
||||
|
||||
// fakeShell answers the firewall package's scripts and records what was
|
||||
@@ -55,3 +57,28 @@ func TestANodeKeepsOrDropsItsFirewallAsTheClusterSays(t *testing.T) {
|
||||
t.Error("turned off, the table stayed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestANodeGivenAPartInSlurmLetsSlurmInFromTheClusterOnly(t *testing.T) {
|
||||
sh := &fakeShell{}
|
||||
a := &Agent{log: slog.New(slog.DiscardHandler), fwRunner: sh, serverList: []string{"https://10.0.0.140:7443"},
|
||||
slurm: slurmnode.New(nil, nil, "", slog.New(slog.DiscardHandler))}
|
||||
a.applyFirewall(&llmgwv1.Firewall{Enabled: true, EnginePortsFrom: 20000, EnginePortsTo: 20999})
|
||||
if strings.Contains(sh.installed, "6817") {
|
||||
t.Fatalf("Slurm's ports before the node has a part in it:\n%s", sh.installed)
|
||||
}
|
||||
|
||||
a.slurmMessage(&llmgwv1.Slurm{Name: "node05", Node: true, ConfServer: "10.0.0.148:6817"})
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
a.fwMu.Lock()
|
||||
got := sh.installed
|
||||
a.fwMu.Unlock()
|
||||
if strings.Contains(got, "6817-6818") && strings.Contains(got, "60001-63000") {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("given a part in Slurm, the rules still lack its ports:\n%s", got)
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
@@ -101,6 +101,13 @@ func (k *Keeper) Apply(m *llmgwv1.Slurm) {
|
||||
}
|
||||
}
|
||||
|
||||
// HasPart is whether the last message gives this machine a part in Slurm.
|
||||
func (k *Keeper) HasPart() bool {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
return k.want.GetController() || k.want.GetNode()
|
||||
}
|
||||
|
||||
// Resend has the status sent again on the next pass, which it starts now.
|
||||
func (k *Keeper) Resend() {
|
||||
k.mu.Lock()
|
||||
|
||||
Reference in new issue
Block a user