Slurm's jobs in containers: pyxis and enroot, installed by the agents
CI / test (push) Failing after 6m53s
CI / test (push) Failing after 6m53s
slurm.containers (ctl slurm containers on) has every Slurm machine's agent install enroot from NVIDIA's release and build pyxis against its own Slurm, both pinned and checked against their digests, and on GPU machines nvidia-container-cli from NVIDIA's repository. enroot keeps each user's containers under /run/enroot and /var/lib/enroot, needing no login session, with the shared path in every container. The leader has Slurm load pyxis only once every machine reports it: plugstack.conf's "required" fails every job on a machine without the plugin. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c3a75dcf3f
commit
57f0fd2ed0
18 files changed
+737
-9
No files matched your search
@@ -83,6 +83,9 @@ type Reported struct {
|
||||
SlurmdbdRunning bool `json:"slurmdbd_running"`
|
||||
// Shared is the shared path here: "mounted", "exported", "missing".
|
||||
Shared string `json:"shared,omitempty"`
|
||||
// Containers is what the machine has for jobs in containers, empty
|
||||
// while not both of pyxis and enroot.
|
||||
Containers string `json:"containers,omitempty"`
|
||||
// Users is how many of llmgw's users the machine has.
|
||||
Users int `json:"users"`
|
||||
State string `json:"state,omitempty"`
|
||||
@@ -109,6 +112,10 @@ type Status struct {
|
||||
Shared config.SharedStorage `json:"shared"`
|
||||
Users string `json:"users,omitempty"`
|
||||
UserCount int `json:"user_count,omitempty"`
|
||||
// Containers is whether jobs may run in containers: "on" once every
|
||||
// machine has pyxis and enroot, "installing" until then, empty when
|
||||
// slurm.containers is off.
|
||||
Containers string `json:"containers,omitempty"`
|
||||
}
|
||||
|
||||
// User is a Linux user made from an llmgw account.
|
||||
@@ -263,7 +270,7 @@ func (s *Service) Kick() {
|
||||
// OnSlurmStatus records what a node's agent says.
|
||||
func (s *Service) OnSlurmStatus(node string, st *llmgwv1.SlurmStatus) {
|
||||
r := &Reported{Version: st.GetVersion(), Hardware: st.GetHardware(), SlurmdRunning: st.GetSlurmdRunning(),
|
||||
Shared: st.GetShared(), Users: int(st.GetUsers()),
|
||||
Shared: st.GetShared(), Users: int(st.GetUsers()), Containers: st.GetContainers(),
|
||||
SlurmctldRunning: st.GetSlurmctldRunning(), SlurmdbdRunning: st.GetSlurmdbdRunning(), State: st.GetState(), Reason: st.GetReason(),
|
||||
Jobs: int(st.GetJobs()), Error: st.GetError(), FilesSHA256: st.GetFilesSha256(), At: time.Now()}
|
||||
s.mu.Lock()
|
||||
@@ -480,7 +487,26 @@ func (s *Service) files(set config.SlurmSettings) (map[string]string, []string)
|
||||
if c.ControllerAddr == "" {
|
||||
return nil, waiting
|
||||
}
|
||||
return c.Files(), waiting
|
||||
f := c.Files()
|
||||
if s.containersReady(set) {
|
||||
f["plugstack.conf"] = slurm.PlugstackConf
|
||||
}
|
||||
return f, waiting
|
||||
}
|
||||
|
||||
// containersReady is whether jobs may ask for containers: every machine of
|
||||
// the Slurm cluster says it has pyxis and enroot. Slurm fails a job on a
|
||||
// machine without the plugin it was told to load. Under s.mu.
|
||||
func (s *Service) containersReady(set config.SlurmSettings) bool {
|
||||
if !set.Containers {
|
||||
return false
|
||||
}
|
||||
for _, n := range s.machines(set) {
|
||||
if st := s.status[n]; st == nil || st.Containers == "" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// cluster is what slurm.conf says: each node as it last reported, else as
|
||||
@@ -538,6 +564,7 @@ func (s *Service) message(set config.SlurmSettings, node string, lent bool, key
|
||||
return m
|
||||
}
|
||||
m.Name, m.MungeKey, m.Lent = node, key, lent
|
||||
m.Containers = set.Containers
|
||||
m.Node = isNode
|
||||
if addr := s.controllerAddr(set); addr != "" {
|
||||
m.ConfServer = net.JoinHostPort(addr, strconv.Itoa(slurm.CtldPort))
|
||||
@@ -652,6 +679,12 @@ func (s *Service) Status() Status {
|
||||
if !set.Enabled {
|
||||
return out
|
||||
}
|
||||
if set.Containers {
|
||||
out.Containers = "installing"
|
||||
if s.containersReady(set) {
|
||||
out.Containers = "on"
|
||||
}
|
||||
}
|
||||
_, out.Waiting = s.files(set)
|
||||
conn := s.agents.ConnectedNodes()
|
||||
for _, node := range set.Nodes {
|
||||
|
||||
@@ -287,3 +287,39 @@ func TestLendingSaysWhenAReplicaWithNowhereToGoStops(t *testing.T) {
|
||||
t.Errorf("%q", d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPyxisIsLoadedOnlyOnceEveryMachineHasIt(t *testing.T) {
|
||||
// "required" in plugstack.conf fails every job on a machine without the
|
||||
// plugin; sent early, it would stop Slurm's jobs everywhere it is missing.
|
||||
r := newRig()
|
||||
r.set.Containers = true
|
||||
r.report("cuda01", &llmgwv1.SlurmStatus{Hardware: hw})
|
||||
r.tick()
|
||||
if !r.agents.last("cuda01").GetContainers() || !r.agents.last("node06").GetContainers() {
|
||||
t.Fatal("the machines were not told to set containers up")
|
||||
}
|
||||
if _, ok := r.agents.last("node06").GetFiles()["plugstack.conf"]; ok {
|
||||
t.Fatal("pyxis loaded before any machine has it")
|
||||
}
|
||||
if s := r.svc.Status().Containers; s != "installing" {
|
||||
t.Errorf("status %q", s)
|
||||
}
|
||||
r.report("cuda01", &llmgwv1.SlurmStatus{Hardware: hw, Containers: "pyxis 0.24.0, enroot 4.2.1"})
|
||||
r.tick()
|
||||
if _, ok := r.agents.last("node06").GetFiles()["plugstack.conf"]; ok {
|
||||
t.Fatal("pyxis loaded with the controller still without it")
|
||||
}
|
||||
r.report("node06", &llmgwv1.SlurmStatus{Containers: "pyxis 0.24.0, enroot 4.2.1"})
|
||||
r.tick()
|
||||
if p := r.agents.last("node06").GetFiles()["plugstack.conf"]; !strings.Contains(p, "required /usr/local/lib/slurm/spank_pyxis.so") {
|
||||
t.Fatalf("plugstack.conf %q", p)
|
||||
}
|
||||
if s := r.svc.Status().Containers; s != "on" {
|
||||
t.Errorf("status %q", s)
|
||||
}
|
||||
r.set.Containers = false
|
||||
r.tick()
|
||||
if _, ok := r.agents.last("node06").GetFiles()["plugstack.conf"]; ok || r.svc.Status().Containers != "" {
|
||||
t.Fatal("pyxis still loaded with containers off")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user