Slurm's jobs in containers: pyxis and enroot, installed by the agents
CI / test (push) Failing after 6m53s
CI / test (push) Failing after 6m53s
slurm.containers (ctl slurm containers on) has every Slurm machine's agent install enroot from NVIDIA's release and build pyxis against its own Slurm, both pinned and checked against their digests, and on GPU machines nvidia-container-cli from NVIDIA's repository. enroot keeps each user's containers under /run/enroot and /var/lib/enroot, needing no login session, with the shared path in every container. The leader has Slurm load pyxis only once every machine reports it: plugstack.conf's "required" fails every job on a machine without the plugin. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c3a75dcf3f
commit
57f0fd2ed0
18 files changed
+737
-9
No files matched your search
@@ -32,7 +32,44 @@ func newSlurmCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
cmd.AddCommand(newSlurmStatusCmd(client, asJSON), newSlurmSetupCmd(client), newSlurmOffCmd(client),
|
||||
newSlurmListCmd(client, "lend", "Let Slurm run jobs on nodes; llmgw's replicas move off first", true),
|
||||
newSlurmListCmd(client, "reclaim", "Take nodes back from Slurm; llmgw has them when their jobs end", false),
|
||||
newSlurmSharedCmd(client), newSlurmUsersCmd(client, asJSON))
|
||||
newSlurmSharedCmd(client), newSlurmUsersCmd(client, asJSON), newSlurmContainersCmd(client))
|
||||
return cmd
|
||||
}
|
||||
|
||||
func newSlurmContainersCmd(client func() *ctlClient) *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "containers",
|
||||
Short: "Jobs in containers: srun --container-image (pyxis and enroot)",
|
||||
Long: "On, the agents install enroot and pyxis on every machine of the Slurm cluster (from NVIDIA's\n" +
|
||||
"releases, at versions llmgw pins and checks) and, on GPU machines, nvidia-container-cli; once\n" +
|
||||
"every machine has them, Slurm loads pyxis and jobs may run in a container:\n\n" +
|
||||
" srun --gres=gpu:1 --container-image=nvcr.io#nvidia/pytorch:25.01-py3 python train.py\n\n" +
|
||||
"The shared path is in every container where it is on the host. Off, Slurm stops loading pyxis;\n" +
|
||||
"what was installed stays.",
|
||||
}
|
||||
var note string
|
||||
toggle := func(use, short string, on bool) *cobra.Command {
|
||||
c := &cobra.Command{
|
||||
Use: use,
|
||||
Short: short,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
return changeSettings(cmd, client, note, func(doc *settings.Doc) error {
|
||||
if !on {
|
||||
if doc.Node([]string{"slurm", "containers"}) != nil {
|
||||
doc.Unset([]string{"slurm", "containers"})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return doc.SetYAML([]string{"slurm", "containers"}, "true")
|
||||
})
|
||||
},
|
||||
}
|
||||
c.Flags().StringVar(¬e, "note", "", "why, kept with the settings' version")
|
||||
return c
|
||||
}
|
||||
cmd.AddCommand(toggle("on", "Install pyxis and enroot everywhere, and let jobs run in containers", true),
|
||||
toggle("off", "Slurm stops loading pyxis; what was installed stays", false))
|
||||
return cmd
|
||||
}
|
||||
|
||||
@@ -211,6 +248,9 @@ func newSlurmStatusCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
if st.Users != "" {
|
||||
fmt.Fprintf(w, "users: %d, made from llmgw's accounts\n", st.UserCount)
|
||||
}
|
||||
if line := containersLine(st); line != "" {
|
||||
fmt.Fprintf(w, "containers: %s\n", line)
|
||||
}
|
||||
tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0)
|
||||
fmt.Fprintln(tw, "NODE\tPHASE\tSLURM\tJOBS\tSLURMD\tSHARED\tNOTE")
|
||||
for _, n := range st.Nodes {
|
||||
@@ -322,6 +362,30 @@ func newSlurmOffCmd(client func() *ctlClient) *cobra.Command {
|
||||
return c
|
||||
}
|
||||
|
||||
// containersLine says whether jobs may run in containers, and which
|
||||
// machines are not ready yet; empty when containers are off.
|
||||
func containersLine(st slurmsvc.Status) string {
|
||||
switch st.Containers {
|
||||
case "on":
|
||||
return "on (srun --container-image)"
|
||||
case "":
|
||||
return ""
|
||||
}
|
||||
var waiting []string
|
||||
for _, n := range st.Nodes {
|
||||
if n.Status == nil || n.Status.Containers == "" {
|
||||
waiting = append(waiting, n.Node)
|
||||
}
|
||||
}
|
||||
if st.Ctld == nil || st.Ctld.Containers == "" {
|
||||
if !slices.Contains(waiting, st.Controller) {
|
||||
waiting = append(waiting, st.Controller)
|
||||
}
|
||||
}
|
||||
return "installing pyxis and enroot; jobs take them once every machine has them (waiting: " +
|
||||
strings.Join(waiting, ", ") + ")"
|
||||
}
|
||||
|
||||
// sharedLine says where the shared files are and come from.
|
||||
func sharedLine(sh config.SharedStorage) string {
|
||||
switch {
|
||||
|
||||
@@ -54,6 +54,9 @@ func (m consoleModel) slurmPanel(pw int) (string, string, string) {
|
||||
} else {
|
||||
head += "\n " + sDim.Render("users: the site's (llmgw ctl slurm users on makes them from llmgw's accounts)")
|
||||
}
|
||||
if line := containersLine(st); line != "" {
|
||||
head += "\n containers: " + line
|
||||
}
|
||||
head += "\n " + sDim.Render("A node is llmgw's until lent; lent, its replicas move off first.")
|
||||
if len(st.Waiting) > 0 {
|
||||
head += "\n " + sWarn.Render(fmt.Sprintf("not in slurm.conf until they report their hardware: %v", st.Waiting))
|
||||
|
||||
Reference in new issue
Block a user