Slurm's jobs in containers: pyxis and enroot, installed by the agents
CI / test (push) Failing after 6m53s

slurm.containers (ctl slurm containers on) has every Slurm machine's agent
install enroot from NVIDIA's release and build pyxis against its own
Slurm, both pinned and checked against their digests, and on GPU machines
nvidia-container-cli from NVIDIA's repository. enroot keeps each user's
containers under /run/enroot and /var/lib/enroot, needing no login
session, with the shared path in every container. The leader has Slurm
load pyxis only once every machine reports it: plugstack.conf's
"required" fails every job on a machine without the plugin.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-14 21:31:20 +01:00
1 parent c3a75dcf3f
commit 57f0fd2ed0
18 files changed
+737 -9

No files matched your search

+198
View File
@@ -0,0 +1,198 @@
package slurmnode
import (
"context"
"fmt"
"path/filepath"
"runtime"
"strings"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
)
// cacheDir holds what is fetched for Slurm: enroot's packages, pyxis's
// source and its build.
const cacheDir = "/var/cache/llmgw-agent/slurm"
// pyxisMarker is the pyxis version built here, against this Slurm.
func (k *Keeper) pyxisMarker() string { return k.marker + "-pyxis" }
// containers has enroot and pyxis here, set up, when want says so; what
// is there is left when it no longer does (Slurm stops loading pyxis once
// the controller's plugstack.conf goes). Answers the status line, empty
// while either is missing.
func (k *Keeper) containers(ctx context.Context, want *llmgwv1.Slurm) (string, error) {
if !want.GetContainers() {
return "", nil
}
family := k.family()
if family == "" {
return "", fmt.Errorf("neither dpkg nor rpm here: enroot and pyxis come as deb or rpm")
}
if err := k.enroot(ctx, family); err != nil {
return "", fmt.Errorf("enroot: %w", err)
}
if err := k.enrootConf(ctx, want.GetShared().GetPath()); err != nil {
return "", fmt.Errorf("enroot's settings: %w", err)
}
if _, err := k.host.LookPath("nvidia-smi"); err == nil {
if err := k.nvidiaContainer(ctx, family); err != nil {
return "", fmt.Errorf("the GPUs in containers (nvidia-container-cli): %w", err)
}
}
if err := k.pyxis(ctx, family); err != nil {
return "", fmt.Errorf("pyxis: %w", err)
}
return slurm.ContainersStatus(), nil
}
func (k *Keeper) family() string {
if _, err := k.host.LookPath("dpkg"); err == nil {
return "deb"
}
if _, err := k.host.LookPath("rpm"); err == nil {
return "rpm"
}
return ""
}
// enroot installs the pinned release when another, or none, is here.
func (k *Keeper) enroot(ctx context.Context, family string) error {
if k.installed["enroot"] {
return nil
}
if out, err := k.host.Run(ctx, nil, "enroot", "version"); err == nil && strings.TrimSpace(string(out)) == slurm.EnrootVersion {
k.installed["enroot"] = true
return nil
}
pkgs, err := slurm.EnrootPackages(family, runtime.GOARCH)
if err != nil {
return err
}
if k.inst == nil {
return fmt.Errorf("enroot missing, and this agent installs nothing")
}
var files []string
for _, p := range pkgs {
f := filepath.Join(cacheDir, p.Name)
if err := k.host.Download(ctx, p.URL(), f, p.SHA256); err != nil {
return err
}
files = append(files, f)
}
k.log.Info("installing enroot, for Slurm's jobs in containers", "version", slurm.EnrootVersion)
if err := k.inst.InstallPackages(ctx, map[string][]string{family: files}); err != nil {
return err
}
out, err := k.host.Run(ctx, nil, "enroot", "version")
if v := strings.TrimSpace(string(out)); err != nil || v != slurm.EnrootVersion {
return fmt.Errorf("enroot says %q after installing %s: %v", v, slurm.EnrootVersion, err)
}
k.installed["enroot"] = true
return nil
}
// enrootConf writes enroot's settings and makes its directories.
func (k *Keeper) enrootConf(ctx context.Context, shared string) error {
if _, err := k.write(slurm.EnrootConfPath, []byte(slurm.EnrootConf), 0o644, ""); err != nil {
return err
}
changed, err := k.write(slurm.EnrootTmpfiles, []byte(slurm.EnrootTmpfilesConf), 0o644, "")
if err != nil {
return err
}
if changed {
if out, err := k.host.Run(ctx, nil, "systemd-tmpfiles", "--create", slurm.EnrootTmpfiles); err != nil {
return fmt.Errorf("systemd-tmpfiles: %v: %s", err, lastLine(out))
}
}
for _, d := range []string{slurm.EnrootLib + "/cache", slurm.EnrootLib + "/data"} {
if _, err := k.host.Run(ctx, nil, "test", "-d", d); err == nil {
continue
}
// Each user makes their own beneath, as in /tmp.
if out, err := k.host.Run(ctx, nil, "install", "-d", "-m", "1777", d); err != nil {
return fmt.Errorf("making %s: %v: %s", d, err, lastLine(out))
}
}
if shared == "" {
return k.host.Remove(slurm.EnrootMounts)
}
_, err = k.write(slurm.EnrootMounts, []byte(slurm.EnrootSharedMount(shared)), 0o644, "")
return err
}
// nvidiaContainer installs nvidia-container-cli, which enroot's hook uses
// to give a container the job's GPUs, from NVIDIA's repository.
func (k *Keeper) nvidiaContainer(ctx context.Context, family string) error {
if k.installed["nvidia-container-cli"] {
return nil
}
if _, err := k.host.LookPath("nvidia-container-cli"); err == nil {
k.installed["nvidia-container-cli"] = true
return nil
}
if k.inst == nil {
return fmt.Errorf("nvidia-container-cli missing, and this agent installs nothing")
}
switch family {
case "deb":
if err := k.host.Download(ctx, slurm.NvidiaContainerKeyURL, slurm.NvidiaContainerKeyPath,
slurm.NvidiaContainerKeySHA256); err != nil {
return err
}
if _, err := k.write(slurm.NvidiaContainerAptList, []byte(slurm.NvidiaContainerApt), 0o644, ""); err != nil {
return err
}
case "rpm":
if _, err := k.write(slurm.NvidiaContainerYumRepo, []byte(slurm.NvidiaContainerYum), 0o644, ""); err != nil {
return err
}
}
k.log.Info("installing nvidia-container-cli, so containers get the job's GPUs")
if err := k.inst.InstallPackages(ctx, slurm.NvidiaContainerPackages); err != nil {
return err
}
if _, err := k.host.LookPath("nvidia-container-cli"); err != nil {
return fmt.Errorf("still missing after installing libnvidia-container-tools")
}
k.installed["nvidia-container-cli"] = true
return nil
}
// pyxis builds the pinned pyxis against this machine's Slurm and installs
// it, when it has not been, or was for another version.
func (k *Keeper) pyxis(ctx context.Context, family string) error {
if b, err := k.host.ReadFile(k.pyxisMarker()); err == nil && strings.TrimSpace(string(b)) == slurm.PyxisVersion {
if _, err := k.host.Run(ctx, nil, "test", "-f", slurm.PyxisPlugin); err == nil {
return nil
}
}
if k.inst == nil {
return fmt.Errorf("pyxis missing, and this agent installs nothing")
}
if err := k.inst.InstallPackages(ctx, map[string][]string{family: slurm.PyxisBuildPackages[family]}); err != nil {
return fmt.Errorf("installing what builds it: %w", err)
}
tarball := filepath.Join(cacheDir, "pyxis-"+slurm.PyxisVersion+".tar.gz")
if err := k.host.Download(ctx, slurm.PyxisSource, tarball, slurm.PyxisSHA256); err != nil {
return err
}
src := filepath.Join(cacheDir, "pyxis-"+slurm.PyxisVersion)
k.log.Info("building pyxis against this machine's Slurm", "version", slurm.PyxisVersion)
for _, c := range [][]string{
{"rm", "-rf", src},
{"tar", "-xzf", tarball, "-C", cacheDir},
{"make", "-C", src, "install"},
} {
if out, err := k.host.Run(ctx, nil, c[0], c[1:]...); err != nil {
return fmt.Errorf("%s: %v: %s", strings.Join(c, " "), err, lastLine(out))
}
}
if _, err := k.host.Run(ctx, nil, "test", "-f", slurm.PyxisPlugin); err != nil {
return fmt.Errorf("built, yet %s is not there", slurm.PyxisPlugin)
}
_, err := k.write(k.pyxisMarker(), []byte(slurm.PyxisVersion+"\n"), 0o644, "")
return err
}
@@ -0,0 +1,92 @@
package slurmnode
import (
"context"
"runtime"
"slices"
"strings"
"testing"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
)
func TestContainersInstalledFromPinnedReleases(t *testing.T) {
if _, err := slurm.EnrootPackages("deb", runtime.GOARCH); err != nil {
t.Skip(err)
}
h := newFake()
h.bins["dpkg"], h.bins["nvidia-smi"] = true, true
inst := &fakeInst{host: h}
k := New(h, inst, "/var/lib/llmgw-agent/slurm", nil)
// enroot is not here until installed, and then says its version.
h.fail["enroot version"] = true
h.fail["test -d "+slurm.EnrootLib+"/cache"] = true
h.fail["test -f "+slurm.PyxisPlugin] = true
m := nodeMsg(true)
m.Containers = true
m.Shared = &llmgwv1.SlurmShared{Path: "/shared", Nfs: "10.0.0.11:/shared"}
st := k.Reconcile(context.Background(), m)
if st.GetContainers() != "" || !strings.Contains(st.GetError(), "containers: enroot") {
t.Fatalf("ready with enroot saying nothing: %v", st)
}
pkgs, _ := slurm.EnrootPackages("deb", runtime.GOARCH)
for _, p := range pkgs {
if !slices.Contains(h.ran, "download "+p.URL()+" "+cacheDir+"/"+p.Name+" "+p.SHA256) {
t.Errorf("%s not fetched with its digest:\n%s", p.Name, strings.Join(h.ran, "\n"))
}
}
if len(inst.got["deb"]) != 2 || !strings.HasPrefix(inst.got["deb"][0], cacheDir) {
t.Errorf("installed %v", inst.got)
}
delete(h.fail, "enroot version")
h.outputs["enroot version"] = slurm.EnrootVersion + "\n"
h.bins["nvidia-container-cli"] = false
st = k.Reconcile(context.Background(), m)
if !strings.Contains(st.GetError(), "nvidia-container-cli") {
t.Errorf("a GPU machine without nvidia-container-cli is ready: %v", st)
}
if string(h.files[slurm.NvidiaContainerAptList]) != slurm.NvidiaContainerApt ||
!slices.Contains(h.ran, "download "+slurm.NvidiaContainerKeyURL+" "+slurm.NvidiaContainerKeyPath+" "+slurm.NvidiaContainerKeySHA256) {
t.Errorf("NVIDIA's repository not added with its key checked")
}
h.bins["nvidia-container-cli"] = true
st = k.Reconcile(context.Background(), m)
// Built, but the plugin is still not there.
if st.GetContainers() != "" || !strings.Contains(st.GetError(), "pyxis") {
t.Fatalf("status %v", st)
}
for _, want := range []string{"tar -xzf " + cacheDir + "/pyxis-" + slurm.PyxisVersion + ".tar.gz -C " + cacheDir,
"make -C " + cacheDir + "/pyxis-" + slurm.PyxisVersion + " install", "install -d -m 1777 " + slurm.EnrootLib + "/cache"} {
if !slices.Contains(h.ran, want) {
t.Errorf("did not run %q", want)
}
}
delete(h.fail, "test -f "+slurm.PyxisPlugin)
st = k.Reconcile(context.Background(), m)
if st.GetContainers() != slurm.ContainersStatus() || st.GetError() != "" {
t.Fatalf("status %v", st)
}
if string(h.files[slurm.EnrootConfPath]) != slurm.EnrootConf ||
!strings.Contains(string(h.files[slurm.EnrootMounts]), "/shared /shared none") {
t.Errorf("enroot's settings not written")
}
// Done: nothing fetched or built again.
h.ran = nil
k.Reconcile(context.Background(), m)
for _, c := range h.ran {
if strings.HasPrefix(c, "download") || strings.HasPrefix(c, "make") {
t.Errorf("ran %q again", c)
}
}
// Off: nothing said, nothing taken away.
m.Containers = false
if st := k.Reconcile(context.Background(), m); st.GetContainers() != "" {
t.Errorf("status %v", st)
}
}
+61
View File
@@ -13,8 +13,10 @@ import (
"encoding/hex"
"errors"
"fmt"
"io"
"io/fs"
"log/slog"
"net/http"
"os"
"os/exec"
"os/user"
@@ -43,6 +45,9 @@ type Host interface {
// MkdirAll makes path, owned by owner.
MkdirAll(path string, mode fs.FileMode, owner string) error
Remove(path string) error
// Download fetches url to path, kept only if its SHA-256 is sha; a file
// already there with it is not fetched again.
Download(ctx context.Context, url, path, sha string) error
}
// Installer installs packages by family, "deb" and "rpm".
@@ -226,6 +231,11 @@ func (k *Keeper) Reconcile(ctx context.Context, want *llmgwv1.Slurm) *llmgwv1.Sl
if usErr != nil {
k.log.Warn("keeping the Slurm users on this node", "err", usErr)
}
// Jobs in containers; Slurm runs without them, and is kept anyway.
var ctErr error
if st.Containers, ctErr = k.containers(ctx, want); ctErr != nil {
k.log.Warn("setting up jobs in containers on this node", "err", ctErr)
}
defer func() {
switch {
case st.Error != "":
@@ -233,6 +243,8 @@ func (k *Keeper) Reconcile(ctx context.Context, want *llmgwv1.Slurm) *llmgwv1.Sl
st.Error = "shared files: " + shErr.Error()
case usErr != nil:
st.Error = "users: " + usErr.Error()
case ctErr != nil:
st.Error = "containers: " + ctErr.Error()
}
}()
if want.GetController() {
@@ -642,6 +654,55 @@ func (Local) WriteFile(path string, b []byte, mode fs.FileMode, owner string) er
return os.Rename(tmp, path)
}
// Download fetches url to path through a temporary file beside it, kept
// only when its SHA-256 is sha.
func (Local) Download(ctx context.Context, url, path, sha string) error {
if f, err := os.Open(path); err == nil { //nolint:gosec // our paths
h := sha256.New()
_, cerr := io.Copy(h, f)
_ = f.Close()
if cerr == nil && hex.EncodeToString(h.Sum(nil)) == sha {
return nil
}
}
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { //nolint:gosec // system directories
return err
}
ctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
defer cancel()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return fmt.Errorf("fetching %s: %w", url, err)
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("fetching %s: %s", url, resp.Status)
}
tmp := path + ".llmgw-new"
f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) //nolint:gosec // our paths
if err != nil {
return err
}
h := sha256.New()
_, err = io.Copy(io.MultiWriter(f, h), io.LimitReader(resp.Body, 1<<30))
if cerr := f.Close(); err == nil {
err = cerr
}
if err != nil {
_ = os.Remove(tmp)
return fmt.Errorf("fetching %s: %w", url, err)
}
if got := hex.EncodeToString(h.Sum(nil)); got != sha {
_ = os.Remove(tmp)
return fmt.Errorf("%s has SHA-256 %s, not the %s llmgw was built with: not used", url, got, sha)
}
return os.Rename(tmp, path)
}
// MkdirAll makes a directory owned by owner.
func (Local) MkdirAll(path string, mode fs.FileMode, owner string) error {
if err := os.MkdirAll(path, mode); err != nil {
@@ -102,6 +102,16 @@ func (f *fakeHost) MkdirAll(string, fs.FileMode, string) error { return nil }
func (f *fakeHost) Remove(path string) error { delete(f.files, path); return nil }
func (f *fakeHost) Download(_ context.Context, url, path, sha string) error {
c := "download " + url + " " + path + " " + sha
f.ran = append(f.ran, c)
if f.fail[c] || f.fail["download "+url] {
return errors.New("fetch failed")
}
f.files[path] = []byte("fetched " + url)
return nil
}
func (f *fakeHost) count(prefix string) int {
n := 0
for _, c := range f.ran {