Slurm's jobs in containers: pyxis and enroot, installed by the agents
CI / test (push) Failing after 6m53s

slurm.containers (ctl slurm containers on) has every Slurm machine's agent
install enroot from NVIDIA's release and build pyxis against its own
Slurm, both pinned and checked against their digests, and on GPU machines
nvidia-container-cli from NVIDIA's repository. enroot keeps each user's
containers under /run/enroot and /var/lib/enroot, needing no login
session, with the shared path in every container. The leader has Slurm
load pyxis only once every machine reports it: plugstack.conf's
"required" fails every job on a machine without the plugin.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-14 21:31:20 +01:00
1 parent c3a75dcf3f
commit 57f0fd2ed0
18 files changed
+737 -9

No files matched your search

+198
View File
@@ -0,0 +1,198 @@
package slurmnode
import (
"context"
"fmt"
"path/filepath"
"runtime"
"strings"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
)
// cacheDir holds what is fetched for Slurm: enroot's packages, pyxis's
// source and its build.
const cacheDir = "/var/cache/llmgw-agent/slurm"
// pyxisMarker is the pyxis version built here, against this Slurm.
func (k *Keeper) pyxisMarker() string { return k.marker + "-pyxis" }
// containers has enroot and pyxis here, set up, when want says so; what
// is there is left when it no longer does (Slurm stops loading pyxis once
// the controller's plugstack.conf goes). Answers the status line, empty
// while either is missing.
func (k *Keeper) containers(ctx context.Context, want *llmgwv1.Slurm) (string, error) {
if !want.GetContainers() {
return "", nil
}
family := k.family()
if family == "" {
return "", fmt.Errorf("neither dpkg nor rpm here: enroot and pyxis come as deb or rpm")
}
if err := k.enroot(ctx, family); err != nil {
return "", fmt.Errorf("enroot: %w", err)
}
if err := k.enrootConf(ctx, want.GetShared().GetPath()); err != nil {
return "", fmt.Errorf("enroot's settings: %w", err)
}
if _, err := k.host.LookPath("nvidia-smi"); err == nil {
if err := k.nvidiaContainer(ctx, family); err != nil {
return "", fmt.Errorf("the GPUs in containers (nvidia-container-cli): %w", err)
}
}
if err := k.pyxis(ctx, family); err != nil {
return "", fmt.Errorf("pyxis: %w", err)
}
return slurm.ContainersStatus(), nil
}
func (k *Keeper) family() string {
if _, err := k.host.LookPath("dpkg"); err == nil {
return "deb"
}
if _, err := k.host.LookPath("rpm"); err == nil {
return "rpm"
}
return ""
}
// enroot installs the pinned release when another, or none, is here.
func (k *Keeper) enroot(ctx context.Context, family string) error {
if k.installed["enroot"] {
return nil
}
if out, err := k.host.Run(ctx, nil, "enroot", "version"); err == nil && strings.TrimSpace(string(out)) == slurm.EnrootVersion {
k.installed["enroot"] = true
return nil
}
pkgs, err := slurm.EnrootPackages(family, runtime.GOARCH)
if err != nil {
return err
}
if k.inst == nil {
return fmt.Errorf("enroot missing, and this agent installs nothing")
}
var files []string
for _, p := range pkgs {
f := filepath.Join(cacheDir, p.Name)
if err := k.host.Download(ctx, p.URL(), f, p.SHA256); err != nil {
return err
}
files = append(files, f)
}
k.log.Info("installing enroot, for Slurm's jobs in containers", "version", slurm.EnrootVersion)
if err := k.inst.InstallPackages(ctx, map[string][]string{family: files}); err != nil {
return err
}
out, err := k.host.Run(ctx, nil, "enroot", "version")
if v := strings.TrimSpace(string(out)); err != nil || v != slurm.EnrootVersion {
return fmt.Errorf("enroot says %q after installing %s: %v", v, slurm.EnrootVersion, err)
}
k.installed["enroot"] = true
return nil
}
// enrootConf writes enroot's settings and makes its directories.
func (k *Keeper) enrootConf(ctx context.Context, shared string) error {
if _, err := k.write(slurm.EnrootConfPath, []byte(slurm.EnrootConf), 0o644, ""); err != nil {
return err
}
changed, err := k.write(slurm.EnrootTmpfiles, []byte(slurm.EnrootTmpfilesConf), 0o644, "")
if err != nil {
return err
}
if changed {
if out, err := k.host.Run(ctx, nil, "systemd-tmpfiles", "--create", slurm.EnrootTmpfiles); err != nil {
return fmt.Errorf("systemd-tmpfiles: %v: %s", err, lastLine(out))
}
}
for _, d := range []string{slurm.EnrootLib + "/cache", slurm.EnrootLib + "/data"} {
if _, err := k.host.Run(ctx, nil, "test", "-d", d); err == nil {
continue
}
// Each user makes their own beneath, as in /tmp.
if out, err := k.host.Run(ctx, nil, "install", "-d", "-m", "1777", d); err != nil {
return fmt.Errorf("making %s: %v: %s", d, err, lastLine(out))
}
}
if shared == "" {
return k.host.Remove(slurm.EnrootMounts)
}
_, err = k.write(slurm.EnrootMounts, []byte(slurm.EnrootSharedMount(shared)), 0o644, "")
return err
}
// nvidiaContainer installs nvidia-container-cli, which enroot's hook uses
// to give a container the job's GPUs, from NVIDIA's repository.
func (k *Keeper) nvidiaContainer(ctx context.Context, family string) error {
if k.installed["nvidia-container-cli"] {
return nil
}
if _, err := k.host.LookPath("nvidia-container-cli"); err == nil {
k.installed["nvidia-container-cli"] = true
return nil
}
if k.inst == nil {
return fmt.Errorf("nvidia-container-cli missing, and this agent installs nothing")
}
switch family {
case "deb":
if err := k.host.Download(ctx, slurm.NvidiaContainerKeyURL, slurm.NvidiaContainerKeyPath,
slurm.NvidiaContainerKeySHA256); err != nil {
return err
}
if _, err := k.write(slurm.NvidiaContainerAptList, []byte(slurm.NvidiaContainerApt), 0o644, ""); err != nil {
return err
}
case "rpm":
if _, err := k.write(slurm.NvidiaContainerYumRepo, []byte(slurm.NvidiaContainerYum), 0o644, ""); err != nil {
return err
}
}
k.log.Info("installing nvidia-container-cli, so containers get the job's GPUs")
if err := k.inst.InstallPackages(ctx, slurm.NvidiaContainerPackages); err != nil {
return err
}
if _, err := k.host.LookPath("nvidia-container-cli"); err != nil {
return fmt.Errorf("still missing after installing libnvidia-container-tools")
}
k.installed["nvidia-container-cli"] = true
return nil
}
// pyxis builds the pinned pyxis against this machine's Slurm and installs
// it, when it has not been, or was for another version.
func (k *Keeper) pyxis(ctx context.Context, family string) error {
if b, err := k.host.ReadFile(k.pyxisMarker()); err == nil && strings.TrimSpace(string(b)) == slurm.PyxisVersion {
if _, err := k.host.Run(ctx, nil, "test", "-f", slurm.PyxisPlugin); err == nil {
return nil
}
}
if k.inst == nil {
return fmt.Errorf("pyxis missing, and this agent installs nothing")
}
if err := k.inst.InstallPackages(ctx, map[string][]string{family: slurm.PyxisBuildPackages[family]}); err != nil {
return fmt.Errorf("installing what builds it: %w", err)
}
tarball := filepath.Join(cacheDir, "pyxis-"+slurm.PyxisVersion+".tar.gz")
if err := k.host.Download(ctx, slurm.PyxisSource, tarball, slurm.PyxisSHA256); err != nil {
return err
}
src := filepath.Join(cacheDir, "pyxis-"+slurm.PyxisVersion)
k.log.Info("building pyxis against this machine's Slurm", "version", slurm.PyxisVersion)
for _, c := range [][]string{
{"rm", "-rf", src},
{"tar", "-xzf", tarball, "-C", cacheDir},
{"make", "-C", src, "install"},
} {
if out, err := k.host.Run(ctx, nil, c[0], c[1:]...); err != nil {
return fmt.Errorf("%s: %v: %s", strings.Join(c, " "), err, lastLine(out))
}
}
if _, err := k.host.Run(ctx, nil, "test", "-f", slurm.PyxisPlugin); err != nil {
return fmt.Errorf("built, yet %s is not there", slurm.PyxisPlugin)
}
_, err := k.write(k.pyxisMarker(), []byte(slurm.PyxisVersion+"\n"), 0o644, "")
return err
}
@@ -0,0 +1,92 @@
package slurmnode
import (
"context"
"runtime"
"slices"
"strings"
"testing"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
)
func TestContainersInstalledFromPinnedReleases(t *testing.T) {
if _, err := slurm.EnrootPackages("deb", runtime.GOARCH); err != nil {
t.Skip(err)
}
h := newFake()
h.bins["dpkg"], h.bins["nvidia-smi"] = true, true
inst := &fakeInst{host: h}
k := New(h, inst, "/var/lib/llmgw-agent/slurm", nil)
// enroot is not here until installed, and then says its version.
h.fail["enroot version"] = true
h.fail["test -d "+slurm.EnrootLib+"/cache"] = true
h.fail["test -f "+slurm.PyxisPlugin] = true
m := nodeMsg(true)
m.Containers = true
m.Shared = &llmgwv1.SlurmShared{Path: "/shared", Nfs: "10.0.0.11:/shared"}
st := k.Reconcile(context.Background(), m)
if st.GetContainers() != "" || !strings.Contains(st.GetError(), "containers: enroot") {
t.Fatalf("ready with enroot saying nothing: %v", st)
}
pkgs, _ := slurm.EnrootPackages("deb", runtime.GOARCH)
for _, p := range pkgs {
if !slices.Contains(h.ran, "download "+p.URL()+" "+cacheDir+"/"+p.Name+" "+p.SHA256) {
t.Errorf("%s not fetched with its digest:\n%s", p.Name, strings.Join(h.ran, "\n"))
}
}
if len(inst.got["deb"]) != 2 || !strings.HasPrefix(inst.got["deb"][0], cacheDir) {
t.Errorf("installed %v", inst.got)
}
delete(h.fail, "enroot version")
h.outputs["enroot version"] = slurm.EnrootVersion + "\n"
h.bins["nvidia-container-cli"] = false
st = k.Reconcile(context.Background(), m)
if !strings.Contains(st.GetError(), "nvidia-container-cli") {
t.Errorf("a GPU machine without nvidia-container-cli is ready: %v", st)
}
if string(h.files[slurm.NvidiaContainerAptList]) != slurm.NvidiaContainerApt ||
!slices.Contains(h.ran, "download "+slurm.NvidiaContainerKeyURL+" "+slurm.NvidiaContainerKeyPath+" "+slurm.NvidiaContainerKeySHA256) {
t.Errorf("NVIDIA's repository not added with its key checked")
}
h.bins["nvidia-container-cli"] = true
st = k.Reconcile(context.Background(), m)
// Built, but the plugin is still not there.
if st.GetContainers() != "" || !strings.Contains(st.GetError(), "pyxis") {
t.Fatalf("status %v", st)
}
for _, want := range []string{"tar -xzf " + cacheDir + "/pyxis-" + slurm.PyxisVersion + ".tar.gz -C " + cacheDir,
"make -C " + cacheDir + "/pyxis-" + slurm.PyxisVersion + " install", "install -d -m 1777 " + slurm.EnrootLib + "/cache"} {
if !slices.Contains(h.ran, want) {
t.Errorf("did not run %q", want)
}
}
delete(h.fail, "test -f "+slurm.PyxisPlugin)
st = k.Reconcile(context.Background(), m)
if st.GetContainers() != slurm.ContainersStatus() || st.GetError() != "" {
t.Fatalf("status %v", st)
}
if string(h.files[slurm.EnrootConfPath]) != slurm.EnrootConf ||
!strings.Contains(string(h.files[slurm.EnrootMounts]), "/shared /shared none") {
t.Errorf("enroot's settings not written")
}
// Done: nothing fetched or built again.
h.ran = nil
k.Reconcile(context.Background(), m)
for _, c := range h.ran {
if strings.HasPrefix(c, "download") || strings.HasPrefix(c, "make") {
t.Errorf("ran %q again", c)
}
}
// Off: nothing said, nothing taken away.
m.Containers = false
if st := k.Reconcile(context.Background(), m); st.GetContainers() != "" {
t.Errorf("status %v", st)
}
}
+61
View File
@@ -13,8 +13,10 @@ import (
"encoding/hex"
"errors"
"fmt"
"io"
"io/fs"
"log/slog"
"net/http"
"os"
"os/exec"
"os/user"
@@ -43,6 +45,9 @@ type Host interface {
// MkdirAll makes path, owned by owner.
MkdirAll(path string, mode fs.FileMode, owner string) error
Remove(path string) error
// Download fetches url to path, kept only if its SHA-256 is sha; a file
// already there with it is not fetched again.
Download(ctx context.Context, url, path, sha string) error
}
// Installer installs packages by family, "deb" and "rpm".
@@ -226,6 +231,11 @@ func (k *Keeper) Reconcile(ctx context.Context, want *llmgwv1.Slurm) *llmgwv1.Sl
if usErr != nil {
k.log.Warn("keeping the Slurm users on this node", "err", usErr)
}
// Jobs in containers; Slurm runs without them, and is kept anyway.
var ctErr error
if st.Containers, ctErr = k.containers(ctx, want); ctErr != nil {
k.log.Warn("setting up jobs in containers on this node", "err", ctErr)
}
defer func() {
switch {
case st.Error != "":
@@ -233,6 +243,8 @@ func (k *Keeper) Reconcile(ctx context.Context, want *llmgwv1.Slurm) *llmgwv1.Sl
st.Error = "shared files: " + shErr.Error()
case usErr != nil:
st.Error = "users: " + usErr.Error()
case ctErr != nil:
st.Error = "containers: " + ctErr.Error()
}
}()
if want.GetController() {
@@ -642,6 +654,55 @@ func (Local) WriteFile(path string, b []byte, mode fs.FileMode, owner string) er
return os.Rename(tmp, path)
}
// Download fetches url to path through a temporary file beside it, kept
// only when its SHA-256 is sha.
func (Local) Download(ctx context.Context, url, path, sha string) error {
if f, err := os.Open(path); err == nil { //nolint:gosec // our paths
h := sha256.New()
_, cerr := io.Copy(h, f)
_ = f.Close()
if cerr == nil && hex.EncodeToString(h.Sum(nil)) == sha {
return nil
}
}
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { //nolint:gosec // system directories
return err
}
ctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
defer cancel()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return fmt.Errorf("fetching %s: %w", url, err)
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("fetching %s: %s", url, resp.Status)
}
tmp := path + ".llmgw-new"
f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) //nolint:gosec // our paths
if err != nil {
return err
}
h := sha256.New()
_, err = io.Copy(io.MultiWriter(f, h), io.LimitReader(resp.Body, 1<<30))
if cerr := f.Close(); err == nil {
err = cerr
}
if err != nil {
_ = os.Remove(tmp)
return fmt.Errorf("fetching %s: %w", url, err)
}
if got := hex.EncodeToString(h.Sum(nil)); got != sha {
_ = os.Remove(tmp)
return fmt.Errorf("%s has SHA-256 %s, not the %s llmgw was built with: not used", url, got, sha)
}
return os.Rename(tmp, path)
}
// MkdirAll makes a directory owned by owner.
func (Local) MkdirAll(path string, mode fs.FileMode, owner string) error {
if err := os.MkdirAll(path, mode); err != nil {
@@ -102,6 +102,16 @@ func (f *fakeHost) MkdirAll(string, fs.FileMode, string) error { return nil }
func (f *fakeHost) Remove(path string) error { delete(f.files, path); return nil }
func (f *fakeHost) Download(_ context.Context, url, path, sha string) error {
c := "download " + url + " " + path + " " + sha
f.ran = append(f.ran, c)
if f.fail[c] || f.fail["download "+url] {
return errors.New("fetch failed")
}
f.files[path] = []byte("fetched " + url)
return nil
}
func (f *fakeHost) count(prefix string) int {
n := 0
for _, c := range f.ran {
+65 -1
View File
@@ -32,7 +32,44 @@ func newSlurmCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
cmd.AddCommand(newSlurmStatusCmd(client, asJSON), newSlurmSetupCmd(client), newSlurmOffCmd(client),
newSlurmListCmd(client, "lend", "Let Slurm run jobs on nodes; llmgw's replicas move off first", true),
newSlurmListCmd(client, "reclaim", "Take nodes back from Slurm; llmgw has them when their jobs end", false),
newSlurmSharedCmd(client), newSlurmUsersCmd(client, asJSON))
newSlurmSharedCmd(client), newSlurmUsersCmd(client, asJSON), newSlurmContainersCmd(client))
return cmd
}
func newSlurmContainersCmd(client func() *ctlClient) *cobra.Command {
cmd := &cobra.Command{
Use: "containers",
Short: "Jobs in containers: srun --container-image (pyxis and enroot)",
Long: "On, the agents install enroot and pyxis on every machine of the Slurm cluster (from NVIDIA's\n" +
"releases, at versions llmgw pins and checks) and, on GPU machines, nvidia-container-cli; once\n" +
"every machine has them, Slurm loads pyxis and jobs may run in a container:\n\n" +
" srun --gres=gpu:1 --container-image=nvcr.io#nvidia/pytorch:25.01-py3 python train.py\n\n" +
"The shared path is in every container where it is on the host. Off, Slurm stops loading pyxis;\n" +
"what was installed stays.",
}
var note string
toggle := func(use, short string, on bool) *cobra.Command {
c := &cobra.Command{
Use: use,
Short: short,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
return changeSettings(cmd, client, note, func(doc *settings.Doc) error {
if !on {
if doc.Node([]string{"slurm", "containers"}) != nil {
doc.Unset([]string{"slurm", "containers"})
}
return nil
}
return doc.SetYAML([]string{"slurm", "containers"}, "true")
})
},
}
c.Flags().StringVar(&note, "note", "", "why, kept with the settings' version")
return c
}
cmd.AddCommand(toggle("on", "Install pyxis and enroot everywhere, and let jobs run in containers", true),
toggle("off", "Slurm stops loading pyxis; what was installed stays", false))
return cmd
}
@@ -211,6 +248,9 @@ func newSlurmStatusCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
if st.Users != "" {
fmt.Fprintf(w, "users: %d, made from llmgw's accounts\n", st.UserCount)
}
if line := containersLine(st); line != "" {
fmt.Fprintf(w, "containers: %s\n", line)
}
tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0)
fmt.Fprintln(tw, "NODE\tPHASE\tSLURM\tJOBS\tSLURMD\tSHARED\tNOTE")
for _, n := range st.Nodes {
@@ -322,6 +362,30 @@ func newSlurmOffCmd(client func() *ctlClient) *cobra.Command {
return c
}
// containersLine says whether jobs may run in containers, and which
// machines are not ready yet; empty when containers are off.
func containersLine(st slurmsvc.Status) string {
switch st.Containers {
case "on":
return "on (srun --container-image)"
case "":
return ""
}
var waiting []string
for _, n := range st.Nodes {
if n.Status == nil || n.Status.Containers == "" {
waiting = append(waiting, n.Node)
}
}
if st.Ctld == nil || st.Ctld.Containers == "" {
if !slices.Contains(waiting, st.Controller) {
waiting = append(waiting, st.Controller)
}
}
return "installing pyxis and enroot; jobs take them once every machine has them (waiting: " +
strings.Join(waiting, ", ") + ")"
}
// sharedLine says where the shared files are and come from.
func sharedLine(sh config.SharedStorage) string {
switch {
+3
View File
@@ -54,6 +54,9 @@ func (m consoleModel) slurmPanel(pw int) (string, string, string) {
} else {
head += "\n " + sDim.Render("users: the site's (llmgw ctl slurm users on makes them from llmgw's accounts)")
}
if line := containersLine(st); line != "" {
head += "\n containers: " + line
}
head += "\n " + sDim.Render("A node is llmgw's until lent; lent, its replicas move off first.")
if len(st.Waiting) > 0 {
head += "\n " + sWarn.Render(fmt.Sprintf("not in slurm.conf until they report their hardware: %v", st.Waiting))
+6
View File
@@ -54,6 +54,12 @@ type SlurmSettings struct {
// and to a node only while a job of theirs runs there (pam_slurm_adopt);
// empty leaves users to the site (a directory through sssd, or by hand).
Users string `koanf:"users"`
// Containers lets jobs run in containers: `srun --container-image
// nvcr.io#nvidia/pytorch:25.01-py3` (pyxis, with enroot under it),
// the GPUs and the shared path inside. The agents install both on
// every Slurm machine; jobs take it once every machine has them.
Containers bool `koanf:"containers"`
}
// SharedStorage is where the Slurm cluster's shared files come from: the
+35 -2
View File
@@ -83,6 +83,9 @@ type Reported struct {
SlurmdbdRunning bool `json:"slurmdbd_running"`
// Shared is the shared path here: "mounted", "exported", "missing".
Shared string `json:"shared,omitempty"`
// Containers is what the machine has for jobs in containers, empty
// while not both of pyxis and enroot.
Containers string `json:"containers,omitempty"`
// Users is how many of llmgw's users the machine has.
Users int `json:"users"`
State string `json:"state,omitempty"`
@@ -109,6 +112,10 @@ type Status struct {
Shared config.SharedStorage `json:"shared"`
Users string `json:"users,omitempty"`
UserCount int `json:"user_count,omitempty"`
// Containers is whether jobs may run in containers: "on" once every
// machine has pyxis and enroot, "installing" until then, empty when
// slurm.containers is off.
Containers string `json:"containers,omitempty"`
}
// User is a Linux user made from an llmgw account.
@@ -263,7 +270,7 @@ func (s *Service) Kick() {
// OnSlurmStatus records what a node's agent says.
func (s *Service) OnSlurmStatus(node string, st *llmgwv1.SlurmStatus) {
r := &Reported{Version: st.GetVersion(), Hardware: st.GetHardware(), SlurmdRunning: st.GetSlurmdRunning(),
Shared: st.GetShared(), Users: int(st.GetUsers()),
Shared: st.GetShared(), Users: int(st.GetUsers()), Containers: st.GetContainers(),
SlurmctldRunning: st.GetSlurmctldRunning(), SlurmdbdRunning: st.GetSlurmdbdRunning(), State: st.GetState(), Reason: st.GetReason(),
Jobs: int(st.GetJobs()), Error: st.GetError(), FilesSHA256: st.GetFilesSha256(), At: time.Now()}
s.mu.Lock()
@@ -480,7 +487,26 @@ func (s *Service) files(set config.SlurmSettings) (map[string]string, []string)
if c.ControllerAddr == "" {
return nil, waiting
}
return c.Files(), waiting
f := c.Files()
if s.containersReady(set) {
f["plugstack.conf"] = slurm.PlugstackConf
}
return f, waiting
}
// containersReady is whether jobs may ask for containers: every machine of
// the Slurm cluster says it has pyxis and enroot. Slurm fails a job on a
// machine without the plugin it was told to load. Under s.mu.
func (s *Service) containersReady(set config.SlurmSettings) bool {
if !set.Containers {
return false
}
for _, n := range s.machines(set) {
if st := s.status[n]; st == nil || st.Containers == "" {
return false
}
}
return true
}
// cluster is what slurm.conf says: each node as it last reported, else as
@@ -538,6 +564,7 @@ func (s *Service) message(set config.SlurmSettings, node string, lent bool, key
return m
}
m.Name, m.MungeKey, m.Lent = node, key, lent
m.Containers = set.Containers
m.Node = isNode
if addr := s.controllerAddr(set); addr != "" {
m.ConfServer = net.JoinHostPort(addr, strconv.Itoa(slurm.CtldPort))
@@ -652,6 +679,12 @@ func (s *Service) Status() Status {
if !set.Enabled {
return out
}
if set.Containers {
out.Containers = "installing"
if s.containersReady(set) {
out.Containers = "on"
}
}
_, out.Waiting = s.files(set)
conn := s.agents.ConnectedNodes()
for _, node := range set.Nodes {
+36
View File
@@ -287,3 +287,39 @@ func TestLendingSaysWhenAReplicaWithNowhereToGoStops(t *testing.T) {
t.Errorf("%q", d)
}
}
func TestPyxisIsLoadedOnlyOnceEveryMachineHasIt(t *testing.T) {
// "required" in plugstack.conf fails every job on a machine without the
// plugin; sent early, it would stop Slurm's jobs everywhere it is missing.
r := newRig()
r.set.Containers = true
r.report("cuda01", &llmgwv1.SlurmStatus{Hardware: hw})
r.tick()
if !r.agents.last("cuda01").GetContainers() || !r.agents.last("node06").GetContainers() {
t.Fatal("the machines were not told to set containers up")
}
if _, ok := r.agents.last("node06").GetFiles()["plugstack.conf"]; ok {
t.Fatal("pyxis loaded before any machine has it")
}
if s := r.svc.Status().Containers; s != "installing" {
t.Errorf("status %q", s)
}
r.report("cuda01", &llmgwv1.SlurmStatus{Hardware: hw, Containers: "pyxis 0.24.0, enroot 4.2.1"})
r.tick()
if _, ok := r.agents.last("node06").GetFiles()["plugstack.conf"]; ok {
t.Fatal("pyxis loaded with the controller still without it")
}
r.report("node06", &llmgwv1.SlurmStatus{Containers: "pyxis 0.24.0, enroot 4.2.1"})
r.tick()
if p := r.agents.last("node06").GetFiles()["plugstack.conf"]; !strings.Contains(p, "required /usr/local/lib/slurm/spank_pyxis.so") {
t.Fatalf("plugstack.conf %q", p)
}
if s := r.svc.Status().Containers; s != "on" {
t.Errorf("status %q", s)
}
r.set.Containers = false
r.tick()
if _, ok := r.agents.last("node06").GetFiles()["plugstack.conf"]; ok || r.svc.Status().Containers != "" {
t.Fatal("pyxis still loaded with containers off")
}
}
+1
View File
@@ -528,6 +528,7 @@ var Reference = []Block{
{Key: "partition", Type: "string", Doc: "Partition is the one partition every node is in. Empty is \"gpu\".", Nested: ""},
{Key: "shared", Type: "SharedStorage", Doc: "Shared is the file system every machine of the Slurm cluster sees at\nthe same path: home directories, datasets, the jobs' output and\ncheckpoints. Empty path: none, and a batch job's output stays on the\nnode it ran on.", Nested: "config.SharedStorage"},
{Key: "users", Type: "string", Doc: "Users is who keeps the Linux users that run jobs, the same on every\nmachine of the Slurm cluster: \"llmgw\" makes one for each active llmgw\naccount (its login from the e-mail address, a UID of llmgw's from\n200000 up, its SSH keys from the portal), let in to the controller,\nand to a node only while a job of theirs runs there (pam_slurm_adopt);\nempty leaves users to the site (a directory through sssd, or by hand).", Nested: ""},
{Key: "containers", Type: "bool", Doc: "Containers lets jobs run in containers: `srun --container-image\nnvcr.io#nvidia/pytorch:25.01-py3` (pyxis, with enroot under it),\nthe GPUs and the shared path inside. The agents install both on\nevery Slurm machine; jobs take it once every machine has them.", Nested: ""},
}},
{Package: "engines", Name: "StrataConfig", Title: "StrataConfig", Doc: "", Fields: []Field{
{Key: "dir", Type: "string", Doc: "Dir is Strata's source directory, where setup.py and serve/ are.", Nested: ""},
+139
View File
@@ -0,0 +1,139 @@
package slurm
import (
"fmt"
"strings"
)
// Jobs in containers: pyxis, Slurm's plugin that gives srun and sbatch
// --container-image, and enroot under it, which turns an image into a
// directory a job's user runs in without a daemon or root. Both are
// NVIDIA's and in no distribution's archive, so each is fetched from its
// release at a pinned version and checked against the digest here.
const (
EnrootVersion = "4.2.1"
PyxisVersion = "0.24.0"
// PyxisSource is pyxis's source, built on each machine against its own
// Slurm: a plugin built for another Slurm is refused by it.
PyxisSource = "https://github.com/NVIDIA/pyxis/archive/refs/tags/v" + PyxisVersion + ".tar.gz"
PyxisSHA256 = "9c4cdb79a67301d8ea05951aa4d2c205f40cf6145e338214b0191add8b36d6c4"
// PyxisPlugin is where it is installed (pyxis's own make install).
PyxisPlugin = "/usr/local/lib/slurm/spank_pyxis.so"
// PlugstackConf has Slurm load pyxis. Written on the controller, from
// which configless nodes fetch it, only once every machine has the
// plugin: "required" fails every job on a machine without it, and
// "optional" would run a job asking for a container on the host.
PlugstackConf = "# Written by llmgw (BACKLOG P): jobs in containers, srun --container-image.\n" +
"required " + PyxisPlugin + "\n"
// EnrootConfPath is enroot's settings.
EnrootConfPath = "/etc/enroot/enroot.conf"
// EnrootMounts has every container see the shared path.
EnrootMounts = "/etc/enroot/mounts.d/50-llmgw-shared.fstab"
// EnrootTmpfiles makes enroot's runtime directory at each boot.
EnrootTmpfiles = "/etc/tmpfiles.d/llmgw-enroot.conf"
// EnrootRun is where containers are unpacked while they run; EnrootLib
// holds each user's image cache and named containers.
EnrootRun = "/run/enroot"
EnrootLib = "/var/lib/enroot"
)
// EnrootPackage is one of enroot's release files.
type EnrootPackage struct {
Name, SHA256 string
}
// URL is where the file is.
func (p EnrootPackage) URL() string {
return "https://github.com/NVIDIA/enroot/releases/download/v" + EnrootVersion + "/" +
strings.ReplaceAll(p.Name, "+", "%2B")
}
// EnrootPackages are the files for a family and architecture (Go's
// names): enroot, and enroot+caps, which lets its helpers unpack images
// for a user without root.
func EnrootPackages(family, arch string) ([]EnrootPackage, error) {
ps, ok := enrootPackages[family+"/"+arch]
if !ok {
return nil, fmt.Errorf("enroot %s has no package for %s on %s", EnrootVersion, family, arch)
}
return ps, nil
}
var enrootPackages = map[string][]EnrootPackage{
"deb/amd64": {
{"enroot_4.2.1-1_amd64.deb", "5955c51e88df0c5ed538d87abb5264609d4bad55fa8570745ed5d4ec66c8ab91"},
{"enroot+caps_4.2.1-1_amd64.deb", "ca22652dcbdd3ebb6e2c68df781ef4301647c79dd7a41ed62899c2fd0f692750"},
},
"deb/arm64": {
{"enroot_4.2.1-1_arm64.deb", "267018b335815a6b32a562915de1f3fad579f7c9d6748e98c62e7369b40be421"},
{"enroot+caps_4.2.1-1_arm64.deb", "12b811d0c95e879e42e4ac762fa85869a5f3cc9dbb06ab483c01e12e911284a1"},
},
"rpm/amd64": {
{"enroot-4.2.1-1.el8.x86_64.rpm", "6749195e37e3a98071ec349884900b12de7677332599da0fd5d8afb5bed9233c"},
{"enroot+caps-4.2.1-1.el8.x86_64.rpm", "56321a03a1f6c4e4273bf664917eb9a174714499d5506c498db4b64275a2b9c0"},
},
"rpm/arm64": {
{"enroot-4.2.1-1.el8.aarch64.rpm", "6b6bc2526f0f0daa7d3cebe6dd33c0468b209e81eebd92def5777520042ec6fe"},
{"enroot+caps-4.2.1-1.el8.aarch64.rpm", "38cf9ae9602a9a8c3e94637bc1b34e736e8176bda656ba71d371f5e1d1e1eed9"},
},
}
// PyxisBuildPackages build pyxis: a compiler and Slurm's headers.
var PyxisBuildPackages = map[string][]string{
"deb": {"gcc", "make", "libslurm-dev"},
"rpm": {"gcc", "make", "slurm-devel"},
}
// EnrootConf is enroot's settings: each user's containers, cache and
// data apart, in directories that need no login session (a job has none,
// and llmgw's users have no pam_systemd on a node).
const EnrootConf = "# Written by llmgw (BACKLOG P): enroot, for Slurm's jobs in containers.\n" +
"ENROOT_RUNTIME_PATH " + EnrootRun + "/user-$(id -u)\n" +
"ENROOT_CACHE_PATH " + EnrootLib + "/cache/user-$(id -u)\n" +
"ENROOT_DATA_PATH " + EnrootLib + "/data/user-$(id -u)\n" +
"ENROOT_TEMP_PATH ${TMPDIR:-/tmp}\n" +
// A container's root is the user's, as on the host: no fake root.
"ENROOT_REMAP_ROOT no\n" +
"ENROOT_ROOTFS_WRITABLE yes\n"
// EnrootTmpfilesConf makes EnrootRun, open to every user as /tmp is.
const EnrootTmpfilesConf = "# Written by llmgw (BACKLOG P): enroot's runtime directory.\n" +
"d " + EnrootRun + " 1777 root root -\n"
// EnrootSharedMount has a container see path where the host does.
func EnrootSharedMount(path string) string {
return "# Written by llmgw (BACKLOG P): the Slurm cluster's shared files, in every container.\n" +
path + " " + path + " none x-create=dir,rbind,rw,nosuid\n"
}
// The NVIDIA container library, for enroot's hook that gives a container
// the job's GPUs: from NVIDIA's own repository, its key checked.
const (
NvidiaContainerKeyURL = "https://nvidia.github.io/libnvidia-container/gpgkey"
NvidiaContainerKeySHA256 = "c880576d6cf75a48e5027a871bac70fd0421ab07d2b55f30877b21f1c87959c9"
NvidiaContainerKeyPath = "/etc/apt/keyrings/llmgw-nvidia-container.asc"
NvidiaContainerAptList = "/etc/apt/sources.list.d/llmgw-nvidia-container.list"
NvidiaContainerYumRepo = "/etc/yum.repos.d/llmgw-nvidia-container.repo"
)
// NvidiaContainerApt is the apt source.
const NvidiaContainerApt = "# Written by llmgw (BACKLOG P): nvidia-container-cli, for GPUs in Slurm's containers.\n" +
"deb [signed-by=" + NvidiaContainerKeyPath + "] https://nvidia.github.io/libnvidia-container/stable/deb/$(ARCH) /\n"
// NvidiaContainerYum is the yum/dnf repository.
const NvidiaContainerYum = "# Written by llmgw (BACKLOG P): nvidia-container-cli, for GPUs in Slurm's containers.\n" +
"[llmgw-nvidia-container]\nname=NVIDIA container library\n" +
"baseurl=https://nvidia.github.io/libnvidia-container/stable/rpm/$basearch\n" +
"repo_gpgcheck=1\ngpgcheck=0\nenabled=1\ngpgkey=" + NvidiaContainerKeyURL + "\n"
// NvidiaContainerPackages has nvidia-container-cli.
var NvidiaContainerPackages = map[string][]string{"deb": {"libnvidia-container-tools"}, "rpm": {"libnvidia-container-tools"}}
// ContainersStatus is what a machine says it has.
func ContainersStatus() string {
return "pyxis " + PyxisVersion + ", enroot " + EnrootVersion
}