Slurm's jobs in containers: pyxis and enroot, installed by the agents

slurm.containers (ctl slurm containers on) has every Slurm machine's agent
install enroot from NVIDIA's release and build pyxis against its own
Slurm, both pinned and checked against their digests, and on GPU machines
nvidia-container-cli from NVIDIA's repository. enroot keeps each user's
containers under /run/enroot and /var/lib/enroot, needing no login
session, with the shared path in every container. The leader has Slurm
load pyxis only once every machine reports it: plugstack.conf's
"required" fails every job on a machine without the plugin.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

BACKLOG.md

@@ -2088,6 +2088,18 @@ lent, partition`); the leader tells each named node's agent (root) what to be, t
   `gres/gpu=1` in AllocTRES. Found: lending a node whose GPU model has nowhere else to go waits out
   the drain grace (5 min) before the replica stops, and `ctl slurm status` says only "1 replica(s)
   still to move off"; it should say it waits for a replacement that cannot be placed, and until when.
+  Done (c3a75dc): it says the time.
+- Jobs in containers, done 2026-11-14: `slurm.containers` (`ctl slurm containers on`). Each agent
+  installs enroot 4.2.1 (NVIDIA's .deb/.rpm, sha256 pinned) and builds pyxis 0.24.0 (source tarball,
+  sha256 pinned) against its own Slurm (gcc, make, libslurm-dev); on a machine with nvidia-smi,
+  nvidia-container-cli from NVIDIA's repository, its key's digest pinned. enroot.conf keeps each
+  user's runtime, cache and data under /run/enroot and /var/lib/enroot (no login session needed), the
+  shared path bind-mounted in every container. The leader writes plugstack.conf ("required") only
+  once every machine reports both, since a machine without the plugin would fail every job. Tried on
+  the VMs: on in ~30 s; ayse's srun --container-image=ubuntu:24.04 on n1 ran as uid 200001 with
+  /shared inside and nproc 1; a two-node sbatch ran a rank on each. Seen once: a node's pull timed out
+  reaching auth.docker.io and the job failed (network, not pyxis). Not tried: GPUs in a container
+  (cuda01's agent is an older build; needs the peer protocol), Red Hat.
 - P6, Slurm's jobs in llmgw's accounting (written 2026-11-12, not started). The tenancy tree to Slurm:
   each project an account (`sacctmgr add account`, parent its team and organization), each member a user
   of it, kept by the leader from the store; `sbatch --account=PROJECT`. The records back: the leader reads

CHANGELOG.md

@@ -65,6 +65,12 @@ APIs change between minors without a deprecation period.
   there, their session put in the job (pam_slurm_adopt), so nobody uses a
   node's cards outside Slurm, nor on a node llmgw serves from. Root and the
   machines' own users are not held to it.
+- **Slurm's jobs in containers.** `llmgw ctl slurm containers on`
+  (`slurm.containers`) installs enroot and pyxis on every Slurm machine, at
+  pinned versions checked against their digests, and nvidia-container-cli on
+  GPU machines; Slurm loads pyxis once every machine has it, and then `srun
+  --container-image=...` runs a job in that image, as its user, with the
+  shared path inside.
 
 ### Changed
 

api/gen/llmgw/v1/agent.pb.go

@@ -5314,7 +5314,10 @@ type Slurm struct {
 	Users       []*SlurmUser `protobuf:"bytes,10,rep,name=users,proto3" json:"users,omitempty"`
 	// make_homes has this machine make the users' home directories: the one
 	// that exports the shared path, else the controller.
-	MakeHomes     bool `protobuf:"varint,11,opt,name=make_homes,json=makeHomes,proto3" json:"make_homes,omitempty"`
+	MakeHomes bool `protobuf:"varint,11,opt,name=make_homes,json=makeHomes,proto3" json:"make_homes,omitempty"`
+	// containers has the machine install pyxis and enroot, so jobs may run
+	// in containers (srun --container-image), and set them up.
+	Containers    bool `protobuf:"varint,12,opt,name=containers,proto3" json:"containers,omitempty"`
 	unknownFields protoimpl.UnknownFields
 	sizeCache     protoimpl.SizeCache
 }
@@ -5426,6 +5429,13 @@ func (x *Slurm) GetMakeHomes() bool {
 	return false
 }
 
+func (x *Slurm) GetContainers() bool {
+	if x != nil {
+		return x.Containers
+	}
+	return false
+}
+
 // SlurmShared is where the Slurm cluster's shared files come from.
 type SlurmShared struct {
 	state protoimpl.MessageState `protogen:"open.v1"`
@@ -5610,7 +5620,10 @@ type SlurmStatus struct {
 	// "exported", "missing", or empty when there is none.
 	Shared string `protobuf:"bytes,11,opt,name=shared,proto3" json:"shared,omitempty"`
 	// users is how many of llmgw's users this machine has.
-	Users         int32 `protobuf:"varint,12,opt,name=users,proto3" json:"users,omitempty"`
+	Users int32 `protobuf:"varint,12,opt,name=users,proto3" json:"users,omitempty"`
+	// containers is what this machine has for jobs in containers, as
+	// "pyxis 0.24.0, enroot 4.2.1"; empty while it has not got both.
+	Containers    string `protobuf:"bytes,13,opt,name=containers,proto3" json:"containers,omitempty"`
 	unknownFields protoimpl.UnknownFields
 	sizeCache     protoimpl.SizeCache
 }
@@ -5729,6 +5742,13 @@ func (x *SlurmStatus) GetUsers() int32 {
 	return 0
 }
 
+func (x *SlurmStatus) GetContainers() string {
+	if x != nil {
+		return x.Containers
+	}
+	return ""
+}
+
 var File_llmgw_v1_agent_proto protoreflect.FileDescriptor
 
 const file_llmgw_v1_agent_proto_rawDesc = "" +
@@ -6234,7 +6254,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
 	"\x04kind\x18\x02 \x01(\tR\x04kind\x12\x10\n" +
 	"\x03seq\x18\x03 \x01(\x05R\x03seq\x12\x12\n" +
 	"\x04data\x18\x04 \x01(\fR\x04data\x12\x12\n" +
-	"\x04last\x18\x05 \x01(\bR\x04last\"\xa9\x03\n" +
+	"\x04last\x18\x05 \x01(\bR\x04last\"\xc9\x03\n" +
 	"\x05Slurm\x12\x1e\n" +
 	"\n" +
 	"controller\x18\x01 \x01(\bR\n" +
@@ -6251,7 +6271,10 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
 	"\x05users\x18\n" +
 	" \x03(\v2\x13.llmgw.v1.SlurmUserR\x05users\x12\x1d\n" +
 	"\n" +
-	"make_homes\x18\v \x01(\bR\tmakeHomes\x1a8\n" +
+	"make_homes\x18\v \x01(\bR\tmakeHomes\x12\x1e\n" +
+	"\n" +
+	"containers\x18\f \x01(\bR\n" +
+	"containers\x1a8\n" +
 	"\n" +
 	"FilesEntry\x12\x10\n" +
 	"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
@@ -6266,7 +6289,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
 	"\x03uid\x18\x02 \x01(\rR\x03uid\x12\x12\n" +
 	"\x04name\x18\x03 \x01(\tR\x04name\x12\x12\n" +
 	"\x04home\x18\x04 \x01(\tR\x04home\x12\x19\n" +
-	"\bssh_keys\x18\x05 \x03(\tR\asshKeys\"\xeb\x02\n" +
+	"\bssh_keys\x18\x05 \x03(\tR\asshKeys\"\x8b\x03\n" +
 	"\vSlurmStatus\x12\x18\n" +
 	"\aversion\x18\x01 \x01(\tR\aversion\x12\x1a\n" +
 	"\bhardware\x18\x02 \x01(\tR\bhardware\x12%\n" +
@@ -6280,7 +6303,10 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
 	"\x10slurmdbd_running\x18\n" +
 	" \x01(\bR\x0fslurmdbdRunning\x12\x16\n" +
 	"\x06shared\x18\v \x01(\tR\x06shared\x12\x14\n" +
-	"\x05users\x18\f \x01(\x05R\x05users2O\n" +
+	"\x05users\x18\f \x01(\x05R\x05users\x12\x1e\n" +
+	"\n" +
+	"containers\x18\r \x01(\tR\n" +
+	"containers2O\n" +
 	"\fAgentService\x12?\n" +
 	"\aConnect\x12\x16.llmgw.v1.AgentMessage\x1a\x18.llmgw.v1.ControlMessage(\x010\x01B\x93\x01\n" +
 	"\fcom.llmgw.v1B\n" +

api/proto/llmgw/v1/agent.proto

@@ -1020,6 +1020,9 @@ message Slurm {
   // make_homes has this machine make the users' home directories: the one
   // that exports the shared path, else the controller.
   bool make_homes = 11;
+  // containers has the machine install pyxis and enroot, so jobs may run
+  // in containers (srun --container-image), and set them up.
+  bool containers = 12;
 }
 
 // SlurmShared is where the Slurm cluster's shared files come from.
@@ -1077,4 +1080,7 @@ message SlurmStatus {
   string shared = 11;
   // users is how many of llmgw's users this machine has.
   int32 users = 12;
+  // containers is what this machine has for jobs in containers, as
+  // "pyxis 0.24.0, enroot 4.2.1"; empty while it has not got both.
+  string containers = 13;
 }

docs/admin-api.openapi.json

@@ -12372,6 +12372,10 @@
             "type": "string",
             "description": "Shared is the shared path here: \"mounted\", \"exported\", \"missing\"."
           },
+          "containers": {
+            "type": "string",
+            "description": "Containers is what the machine has for jobs in containers, empty while not both of pyxis and enroot."
+          },
           "users": {
             "type": "integer",
             "description": "Users is how many of llmgw's users the machine has."
@@ -12438,6 +12442,10 @@
           },
           "user_count": {
             "type": "integer"
+          },
+          "containers": {
+            "type": "string",
+            "description": "Containers is whether jobs may run in containers: \"on\" once every machine has pyxis and enroot, \"installing\" until then, empty when slurm.containers is off."
           }
         }
       },

docs/install.md

@@ -217,6 +217,23 @@ agents install); root and the machines' own users log in as before. A site whose
 directory (sssd against Active Directory or LDAP) passes `--slurm-users
 site`, or later `llmgw ctl slurm users off`.
 
+Jobs may run in containers: `llmgw ctl slurm containers on` has the agents
+install enroot and pyxis on every Slurm machine (NVIDIA's releases, at
+versions llmgw pins and checks; pyxis is built against each machine's own
+Slurm) and, on GPU machines, nvidia-container-cli from NVIDIA's repository.
+Once every machine has them Slurm loads pyxis, and `ctl slurm status` says
+`containers: on`:
+
+```sh
+srun --gres=gpu:1 --container-image=nvcr.io#nvidia/pytorch:25.01-py3 python train.py
+```
+
+The shared path is in every container where it is on the host, and the job
+runs as its user, held to its cores and cards. Each node pulls the image
+from its registry for each job; for a large image used often, import it
+once (`enroot import -o /shared/images/pytorch.sqsh docker://nvcr.io#...`)
+and give that file to `--container-image`.
+
 **Secure by default.** What init writes is the hardened configuration, not a
 demonstration one:
 

docs/reference.md

@@ -414,6 +414,7 @@ rather than repeated.
 | `partition` | `string` | [SlurmSettings](#slurmsettings) |
 | `shared` | `SharedStorage` | [SlurmSettings](#slurmsettings) |
 | `users` | `string` | [SlurmSettings](#slurmsettings) |
+| `containers` | `bool` | [SlurmSettings](#slurmsettings) |
 | `dir` | `string` | [StrataConfig](#strataconfig) |
 | `python` | `string` | [StrataConfig](#strataconfig) |
 | `engine` | `string` | [StrataConfig](#strataconfig) |
@@ -3599,6 +3600,15 @@ account (its login from the e-mail address, a UID of llmgw's from
 and to a node only while a job of theirs runs there (pam_slurm_adopt);
 empty leaves users to the site (a directory through sssd, or by hand).
 
+### `containers`
+
+`bool`
+
+Containers lets jobs run in containers: `srun --container-image
+nvcr.io#nvidia/pytorch:25.01-py3` (pyxis, with enroot under it),
+the GPUs and the shared path inside. The agents install both on
+every Slurm machine; jobs take it once every machine has them.
+
 ## StrataConfig
 
 ### `dir`

internal/agent/slurmnode/containers.go

@@ -0,0 +1,198 @@
+package slurmnode
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+	"runtime"
+	"strings"
+
+	llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
+)
+
+// cacheDir holds what is fetched for Slurm: enroot's packages, pyxis's
+// source and its build.
+const cacheDir = "/var/cache/llmgw-agent/slurm"
+
+// pyxisMarker is the pyxis version built here, against this Slurm.
+func (k *Keeper) pyxisMarker() string { return k.marker + "-pyxis" }
+
+// containers has enroot and pyxis here, set up, when want says so; what
+// is there is left when it no longer does (Slurm stops loading pyxis once
+// the controller's plugstack.conf goes). Answers the status line, empty
+// while either is missing.
+func (k *Keeper) containers(ctx context.Context, want *llmgwv1.Slurm) (string, error) {
+	if !want.GetContainers() {
+		return "", nil
+	}
+	family := k.family()
+	if family == "" {
+		return "", fmt.Errorf("neither dpkg nor rpm here: enroot and pyxis come as deb or rpm")
+	}
+	if err := k.enroot(ctx, family); err != nil {
+		return "", fmt.Errorf("enroot: %w", err)
+	}
+	if err := k.enrootConf(ctx, want.GetShared().GetPath()); err != nil {
+		return "", fmt.Errorf("enroot's settings: %w", err)
+	}
+	if _, err := k.host.LookPath("nvidia-smi"); err == nil {
+		if err := k.nvidiaContainer(ctx, family); err != nil {
+			return "", fmt.Errorf("the GPUs in containers (nvidia-container-cli): %w", err)
+		}
+	}
+	if err := k.pyxis(ctx, family); err != nil {
+		return "", fmt.Errorf("pyxis: %w", err)
+	}
+	return slurm.ContainersStatus(), nil
+}
+
+func (k *Keeper) family() string {
+	if _, err := k.host.LookPath("dpkg"); err == nil {
+		return "deb"
+	}
+	if _, err := k.host.LookPath("rpm"); err == nil {
+		return "rpm"
+	}
+	return ""
+}
+
+// enroot installs the pinned release when another, or none, is here.
+func (k *Keeper) enroot(ctx context.Context, family string) error {
+	if k.installed["enroot"] {
+		return nil
+	}
+	if out, err := k.host.Run(ctx, nil, "enroot", "version"); err == nil && strings.TrimSpace(string(out)) == slurm.EnrootVersion {
+		k.installed["enroot"] = true
+		return nil
+	}
+	pkgs, err := slurm.EnrootPackages(family, runtime.GOARCH)
+	if err != nil {
+		return err
+	}
+	if k.inst == nil {
+		return fmt.Errorf("enroot missing, and this agent installs nothing")
+	}
+	var files []string
+	for _, p := range pkgs {
+		f := filepath.Join(cacheDir, p.Name)
+		if err := k.host.Download(ctx, p.URL(), f, p.SHA256); err != nil {
+			return err
+		}
+		files = append(files, f)
+	}
+	k.log.Info("installing enroot, for Slurm's jobs in containers", "version", slurm.EnrootVersion)
+	if err := k.inst.InstallPackages(ctx, map[string][]string{family: files}); err != nil {
+		return err
+	}
+	out, err := k.host.Run(ctx, nil, "enroot", "version")
+	if v := strings.TrimSpace(string(out)); err != nil || v != slurm.EnrootVersion {
+		return fmt.Errorf("enroot says %q after installing %s: %v", v, slurm.EnrootVersion, err)
+	}
+	k.installed["enroot"] = true
+	return nil
+}
+
+// enrootConf writes enroot's settings and makes its directories.
+func (k *Keeper) enrootConf(ctx context.Context, shared string) error {
+	if _, err := k.write(slurm.EnrootConfPath, []byte(slurm.EnrootConf), 0o644, ""); err != nil {
+		return err
+	}
+	changed, err := k.write(slurm.EnrootTmpfiles, []byte(slurm.EnrootTmpfilesConf), 0o644, "")
+	if err != nil {
+		return err
+	}
+	if changed {
+		if out, err := k.host.Run(ctx, nil, "systemd-tmpfiles", "--create", slurm.EnrootTmpfiles); err != nil {
+			return fmt.Errorf("systemd-tmpfiles: %v: %s", err, lastLine(out))
+		}
+	}
+	for _, d := range []string{slurm.EnrootLib + "/cache", slurm.EnrootLib + "/data"} {
+		if _, err := k.host.Run(ctx, nil, "test", "-d", d); err == nil {
+			continue
+		}
+		// Each user makes their own beneath, as in /tmp.
+		if out, err := k.host.Run(ctx, nil, "install", "-d", "-m", "1777", d); err != nil {
+			return fmt.Errorf("making %s: %v: %s", d, err, lastLine(out))
+		}
+	}
+	if shared == "" {
+		return k.host.Remove(slurm.EnrootMounts)
+	}
+	_, err = k.write(slurm.EnrootMounts, []byte(slurm.EnrootSharedMount(shared)), 0o644, "")
+	return err
+}
+
+// nvidiaContainer installs nvidia-container-cli, which enroot's hook uses
+// to give a container the job's GPUs, from NVIDIA's repository.
+func (k *Keeper) nvidiaContainer(ctx context.Context, family string) error {
+	if k.installed["nvidia-container-cli"] {
+		return nil
+	}
+	if _, err := k.host.LookPath("nvidia-container-cli"); err == nil {
+		k.installed["nvidia-container-cli"] = true
+		return nil
+	}
+	if k.inst == nil {
+		return fmt.Errorf("nvidia-container-cli missing, and this agent installs nothing")
+	}
+	switch family {
+	case "deb":
+		if err := k.host.Download(ctx, slurm.NvidiaContainerKeyURL, slurm.NvidiaContainerKeyPath,
+			slurm.NvidiaContainerKeySHA256); err != nil {
+			return err
+		}
+		if _, err := k.write(slurm.NvidiaContainerAptList, []byte(slurm.NvidiaContainerApt), 0o644, ""); err != nil {
+			return err
+		}
+	case "rpm":
+		if _, err := k.write(slurm.NvidiaContainerYumRepo, []byte(slurm.NvidiaContainerYum), 0o644, ""); err != nil {
+			return err
+		}
+	}
+	k.log.Info("installing nvidia-container-cli, so containers get the job's GPUs")
+	if err := k.inst.InstallPackages(ctx, slurm.NvidiaContainerPackages); err != nil {
+		return err
+	}
+	if _, err := k.host.LookPath("nvidia-container-cli"); err != nil {
+		return fmt.Errorf("still missing after installing libnvidia-container-tools")
+	}
+	k.installed["nvidia-container-cli"] = true
+	return nil
+}
+
+// pyxis builds the pinned pyxis against this machine's Slurm and installs
+// it, when it has not been, or was for another version.
+func (k *Keeper) pyxis(ctx context.Context, family string) error {
+	if b, err := k.host.ReadFile(k.pyxisMarker()); err == nil && strings.TrimSpace(string(b)) == slurm.PyxisVersion {
+		if _, err := k.host.Run(ctx, nil, "test", "-f", slurm.PyxisPlugin); err == nil {
+			return nil
+		}
+	}
+	if k.inst == nil {
+		return fmt.Errorf("pyxis missing, and this agent installs nothing")
+	}
+	if err := k.inst.InstallPackages(ctx, map[string][]string{family: slurm.PyxisBuildPackages[family]}); err != nil {
+		return fmt.Errorf("installing what builds it: %w", err)
+	}
+	tarball := filepath.Join(cacheDir, "pyxis-"+slurm.PyxisVersion+".tar.gz")
+	if err := k.host.Download(ctx, slurm.PyxisSource, tarball, slurm.PyxisSHA256); err != nil {
+		return err
+	}
+	src := filepath.Join(cacheDir, "pyxis-"+slurm.PyxisVersion)
+	k.log.Info("building pyxis against this machine's Slurm", "version", slurm.PyxisVersion)
+	for _, c := range [][]string{
+		{"rm", "-rf", src},
+		{"tar", "-xzf", tarball, "-C", cacheDir},
+		{"make", "-C", src, "install"},
+	} {
+		if out, err := k.host.Run(ctx, nil, c[0], c[1:]...); err != nil {
+			return fmt.Errorf("%s: %v: %s", strings.Join(c, " "), err, lastLine(out))
+		}
+	}
+	if _, err := k.host.Run(ctx, nil, "test", "-f", slurm.PyxisPlugin); err != nil {
+		return fmt.Errorf("built, yet %s is not there", slurm.PyxisPlugin)
+	}
+	_, err := k.write(k.pyxisMarker(), []byte(slurm.PyxisVersion+"\n"), 0o644, "")
+	return err
+}

internal/agent/slurmnode/containers_test.go

@@ -0,0 +1,92 @@
+package slurmnode
+
+import (
+	"context"
+	"runtime"
+	"slices"
+	"strings"
+	"testing"
+
+	llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
+)
+
+func TestContainersInstalledFromPinnedReleases(t *testing.T) {
+	if _, err := slurm.EnrootPackages("deb", runtime.GOARCH); err != nil {
+		t.Skip(err)
+	}
+	h := newFake()
+	h.bins["dpkg"], h.bins["nvidia-smi"] = true, true
+	inst := &fakeInst{host: h}
+	k := New(h, inst, "/var/lib/llmgw-agent/slurm", nil)
+	// enroot is not here until installed, and then says its version.
+	h.fail["enroot version"] = true
+	h.fail["test -d "+slurm.EnrootLib+"/cache"] = true
+	h.fail["test -f "+slurm.PyxisPlugin] = true
+	m := nodeMsg(true)
+	m.Containers = true
+	m.Shared = &llmgwv1.SlurmShared{Path: "/shared", Nfs: "10.0.0.11:/shared"}
+
+	st := k.Reconcile(context.Background(), m)
+	if st.GetContainers() != "" || !strings.Contains(st.GetError(), "containers: enroot") {
+		t.Fatalf("ready with enroot saying nothing: %v", st)
+	}
+	pkgs, _ := slurm.EnrootPackages("deb", runtime.GOARCH)
+	for _, p := range pkgs {
+		if !slices.Contains(h.ran, "download "+p.URL()+" "+cacheDir+"/"+p.Name+" "+p.SHA256) {
+			t.Errorf("%s not fetched with its digest:\n%s", p.Name, strings.Join(h.ran, "\n"))
+		}
+	}
+	if len(inst.got["deb"]) != 2 || !strings.HasPrefix(inst.got["deb"][0], cacheDir) {
+		t.Errorf("installed %v", inst.got)
+	}
+
+	delete(h.fail, "enroot version")
+	h.outputs["enroot version"] = slurm.EnrootVersion + "\n"
+	h.bins["nvidia-container-cli"] = false
+	st = k.Reconcile(context.Background(), m)
+	if !strings.Contains(st.GetError(), "nvidia-container-cli") {
+		t.Errorf("a GPU machine without nvidia-container-cli is ready: %v", st)
+	}
+	if string(h.files[slurm.NvidiaContainerAptList]) != slurm.NvidiaContainerApt ||
+		!slices.Contains(h.ran, "download "+slurm.NvidiaContainerKeyURL+" "+slurm.NvidiaContainerKeyPath+" "+slurm.NvidiaContainerKeySHA256) {
+		t.Errorf("NVIDIA's repository not added with its key checked")
+	}
+
+	h.bins["nvidia-container-cli"] = true
+	st = k.Reconcile(context.Background(), m)
+	// Built, but the plugin is still not there.
+	if st.GetContainers() != "" || !strings.Contains(st.GetError(), "pyxis") {
+		t.Fatalf("status %v", st)
+	}
+	for _, want := range []string{"tar -xzf " + cacheDir + "/pyxis-" + slurm.PyxisVersion + ".tar.gz -C " + cacheDir,
+		"make -C " + cacheDir + "/pyxis-" + slurm.PyxisVersion + " install", "install -d -m 1777 " + slurm.EnrootLib + "/cache"} {
+		if !slices.Contains(h.ran, want) {
+			t.Errorf("did not run %q", want)
+		}
+	}
+	delete(h.fail, "test -f "+slurm.PyxisPlugin)
+	st = k.Reconcile(context.Background(), m)
+	if st.GetContainers() != slurm.ContainersStatus() || st.GetError() != "" {
+		t.Fatalf("status %v", st)
+	}
+	if string(h.files[slurm.EnrootConfPath]) != slurm.EnrootConf ||
+		!strings.Contains(string(h.files[slurm.EnrootMounts]), "/shared /shared none") {
+		t.Errorf("enroot's settings not written")
+	}
+
+	// Done: nothing fetched or built again.
+	h.ran = nil
+	k.Reconcile(context.Background(), m)
+	for _, c := range h.ran {
+		if strings.HasPrefix(c, "download") || strings.HasPrefix(c, "make") {
+			t.Errorf("ran %q again", c)
+		}
+	}
+
+	// Off: nothing said, nothing taken away.
+	m.Containers = false
+	if st := k.Reconcile(context.Background(), m); st.GetContainers() != "" {
+		t.Errorf("status %v", st)
+	}
+}

internal/agent/slurmnode/slurmnode.go

@@ -13,8 +13,10 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
+	"io"
 	"io/fs"
 	"log/slog"
+	"net/http"
 	"os"
 	"os/exec"
 	"os/user"
@@ -43,6 +45,9 @@ type Host interface {
 	// MkdirAll makes path, owned by owner.
 	MkdirAll(path string, mode fs.FileMode, owner string) error
 	Remove(path string) error
+	// Download fetches url to path, kept only if its SHA-256 is sha; a file
+	// already there with it is not fetched again.
+	Download(ctx context.Context, url, path, sha string) error
 }
 
 // Installer installs packages by family, "deb" and "rpm".
@@ -226,6 +231,11 @@ func (k *Keeper) Reconcile(ctx context.Context, want *llmgwv1.Slurm) *llmgwv1.Sl
 	if usErr != nil {
 		k.log.Warn("keeping the Slurm users on this node", "err", usErr)
 	}
+	// Jobs in containers; Slurm runs without them, and is kept anyway.
+	var ctErr error
+	if st.Containers, ctErr = k.containers(ctx, want); ctErr != nil {
+		k.log.Warn("setting up jobs in containers on this node", "err", ctErr)
+	}
 	defer func() {
 		switch {
 		case st.Error != "":
@@ -233,6 +243,8 @@ func (k *Keeper) Reconcile(ctx context.Context, want *llmgwv1.Slurm) *llmgwv1.Sl
 			st.Error = "shared files: " + shErr.Error()
 		case usErr != nil:
 			st.Error = "users: " + usErr.Error()
+		case ctErr != nil:
+			st.Error = "containers: " + ctErr.Error()
 		}
 	}()
 	if want.GetController() {
@@ -642,6 +654,55 @@ func (Local) WriteFile(path string, b []byte, mode fs.FileMode, owner string) er
 	return os.Rename(tmp, path)
 }
 
+// Download fetches url to path through a temporary file beside it, kept
+// only when its SHA-256 is sha.
+func (Local) Download(ctx context.Context, url, path, sha string) error {
+	if f, err := os.Open(path); err == nil { //nolint:gosec // our paths
+		h := sha256.New()
+		_, cerr := io.Copy(h, f)
+		_ = f.Close()
+		if cerr == nil && hex.EncodeToString(h.Sum(nil)) == sha {
+			return nil
+		}
+	}
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { //nolint:gosec // system directories
+		return err
+	}
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
+	defer cancel()
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return err
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("fetching %s: %w", url, err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("fetching %s: %s", url, resp.Status)
+	}
+	tmp := path + ".llmgw-new"
+	f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) //nolint:gosec // our paths
+	if err != nil {
+		return err
+	}
+	h := sha256.New()
+	_, err = io.Copy(io.MultiWriter(f, h), io.LimitReader(resp.Body, 1<<30))
+	if cerr := f.Close(); err == nil {
+		err = cerr
+	}
+	if err != nil {
+		_ = os.Remove(tmp)
+		return fmt.Errorf("fetching %s: %w", url, err)
+	}
+	if got := hex.EncodeToString(h.Sum(nil)); got != sha {
+		_ = os.Remove(tmp)
+		return fmt.Errorf("%s has SHA-256 %s, not the %s llmgw was built with: not used", url, got, sha)
+	}
+	return os.Rename(tmp, path)
+}
+
 // MkdirAll makes a directory owned by owner.
 func (Local) MkdirAll(path string, mode fs.FileMode, owner string) error {
 	if err := os.MkdirAll(path, mode); err != nil {

internal/agent/slurmnode/slurmnode_test.go

@@ -102,6 +102,16 @@ func (f *fakeHost) MkdirAll(string, fs.FileMode, string) error { return nil }
 
 func (f *fakeHost) Remove(path string) error { delete(f.files, path); return nil }
 
+func (f *fakeHost) Download(_ context.Context, url, path, sha string) error {
+	c := "download " + url + " " + path + " " + sha
+	f.ran = append(f.ran, c)
+	if f.fail[c] || f.fail["download "+url] {
+		return errors.New("fetch failed")
+	}
+	f.files[path] = []byte("fetched " + url)
+	return nil
+}
+
 func (f *fakeHost) count(prefix string) int {
 	n := 0
 	for _, c := range f.ran {

internal/cli/ctl_slurm.go

@@ -32,7 +32,44 @@ func newSlurmCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
 	cmd.AddCommand(newSlurmStatusCmd(client, asJSON), newSlurmSetupCmd(client), newSlurmOffCmd(client),
 		newSlurmListCmd(client, "lend", "Let Slurm run jobs on nodes; llmgw's replicas move off first", true),
 		newSlurmListCmd(client, "reclaim", "Take nodes back from Slurm; llmgw has them when their jobs end", false),
-		newSlurmSharedCmd(client), newSlurmUsersCmd(client, asJSON))
+		newSlurmSharedCmd(client), newSlurmUsersCmd(client, asJSON), newSlurmContainersCmd(client))
+	return cmd
+}
+
+func newSlurmContainersCmd(client func() *ctlClient) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "containers",
+		Short: "Jobs in containers: srun --container-image (pyxis and enroot)",
+		Long: "On, the agents install enroot and pyxis on every machine of the Slurm cluster (from NVIDIA's\n" +
+			"releases, at versions llmgw pins and checks) and, on GPU machines, nvidia-container-cli; once\n" +
+			"every machine has them, Slurm loads pyxis and jobs may run in a container:\n\n" +
+			"  srun --gres=gpu:1 --container-image=nvcr.io#nvidia/pytorch:25.01-py3 python train.py\n\n" +
+			"The shared path is in every container where it is on the host. Off, Slurm stops loading pyxis;\n" +
+			"what was installed stays.",
+	}
+	var note string
+	toggle := func(use, short string, on bool) *cobra.Command {
+		c := &cobra.Command{
+			Use:   use,
+			Short: short,
+			Args:  cobra.NoArgs,
+			RunE: func(cmd *cobra.Command, _ []string) error {
+				return changeSettings(cmd, client, note, func(doc *settings.Doc) error {
+					if !on {
+						if doc.Node([]string{"slurm", "containers"}) != nil {
+							doc.Unset([]string{"slurm", "containers"})
+						}
+						return nil
+					}
+					return doc.SetYAML([]string{"slurm", "containers"}, "true")
+				})
+			},
+		}
+		c.Flags().StringVar(&note, "note", "", "why, kept with the settings' version")
+		return c
+	}
+	cmd.AddCommand(toggle("on", "Install pyxis and enroot everywhere, and let jobs run in containers", true),
+		toggle("off", "Slurm stops loading pyxis; what was installed stays", false))
 	return cmd
 }
 
@@ -211,6 +248,9 @@ func newSlurmStatusCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
 			if st.Users != "" {
 				fmt.Fprintf(w, "users: %d, made from llmgw's accounts\n", st.UserCount)
 			}
+			if line := containersLine(st); line != "" {
+				fmt.Fprintf(w, "containers: %s\n", line)
+			}
 			tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0)
 			fmt.Fprintln(tw, "NODE\tPHASE\tSLURM\tJOBS\tSLURMD\tSHARED\tNOTE")
 			for _, n := range st.Nodes {
@@ -322,6 +362,30 @@ func newSlurmOffCmd(client func() *ctlClient) *cobra.Command {
 	return c
 }
 
+// containersLine says whether jobs may run in containers, and which
+// machines are not ready yet; empty when containers are off.
+func containersLine(st slurmsvc.Status) string {
+	switch st.Containers {
+	case "on":
+		return "on (srun --container-image)"
+	case "":
+		return ""
+	}
+	var waiting []string
+	for _, n := range st.Nodes {
+		if n.Status == nil || n.Status.Containers == "" {
+			waiting = append(waiting, n.Node)
+		}
+	}
+	if st.Ctld == nil || st.Ctld.Containers == "" {
+		if !slices.Contains(waiting, st.Controller) {
+			waiting = append(waiting, st.Controller)
+		}
+	}
+	return "installing pyxis and enroot; jobs take them once every machine has them (waiting: " +
+		strings.Join(waiting, ", ") + ")"
+}
+
 // sharedLine says where the shared files are and come from.
 func sharedLine(sh config.SharedStorage) string {
 	switch {

internal/cli/tui_slurm.go

@@ -54,6 +54,9 @@ func (m consoleModel) slurmPanel(pw int) (string, string, string) {
 	} else {
 		head += "\n  " + sDim.Render("users: the site's (llmgw ctl slurm users on makes them from llmgw's accounts)")
 	}
+	if line := containersLine(st); line != "" {
+		head += "\n  containers: " + line
+	}
 	head += "\n  " + sDim.Render("A node is llmgw's until lent; lent, its replicas move off first.")
 	if len(st.Waiting) > 0 {
 		head += "\n  " + sWarn.Render(fmt.Sprintf("not in slurm.conf until they report their hardware: %v", st.Waiting))

internal/config/slurm.go

@@ -54,6 +54,12 @@ type SlurmSettings struct {
 	// and to a node only while a job of theirs runs there (pam_slurm_adopt);
 	// empty leaves users to the site (a directory through sssd, or by hand).
 	Users string `koanf:"users"`
+
+	// Containers lets jobs run in containers: `srun --container-image
+	// nvcr.io#nvidia/pytorch:25.01-py3` (pyxis, with enroot under it),
+	// the GPUs and the shared path inside. The agents install both on
+	// every Slurm machine; jobs take it once every machine has them.
+	Containers bool `koanf:"containers"`
 }
 
 // SharedStorage is where the Slurm cluster's shared files come from: the

internal/control/slurmsvc/service.go

@@ -83,6 +83,9 @@ type Reported struct {
 	SlurmdbdRunning  bool   `json:"slurmdbd_running"`
 	// Shared is the shared path here: "mounted", "exported", "missing".
 	Shared string `json:"shared,omitempty"`
+	// Containers is what the machine has for jobs in containers, empty
+	// while not both of pyxis and enroot.
+	Containers string `json:"containers,omitempty"`
 	// Users is how many of llmgw's users the machine has.
 	Users       int       `json:"users"`
 	State       string    `json:"state,omitempty"`
@@ -109,6 +112,10 @@ type Status struct {
 	Shared    config.SharedStorage `json:"shared"`
 	Users     string               `json:"users,omitempty"`
 	UserCount int                  `json:"user_count,omitempty"`
+	// Containers is whether jobs may run in containers: "on" once every
+	// machine has pyxis and enroot, "installing" until then, empty when
+	// slurm.containers is off.
+	Containers string `json:"containers,omitempty"`
 }
 
 // User is a Linux user made from an llmgw account.
@@ -263,7 +270,7 @@ func (s *Service) Kick() {
 // OnSlurmStatus records what a node's agent says.
 func (s *Service) OnSlurmStatus(node string, st *llmgwv1.SlurmStatus) {
 	r := &Reported{Version: st.GetVersion(), Hardware: st.GetHardware(), SlurmdRunning: st.GetSlurmdRunning(),
-		Shared: st.GetShared(), Users: int(st.GetUsers()),
+		Shared: st.GetShared(), Users: int(st.GetUsers()), Containers: st.GetContainers(),
 		SlurmctldRunning: st.GetSlurmctldRunning(), SlurmdbdRunning: st.GetSlurmdbdRunning(), State: st.GetState(), Reason: st.GetReason(),
 		Jobs: int(st.GetJobs()), Error: st.GetError(), FilesSHA256: st.GetFilesSha256(), At: time.Now()}
 	s.mu.Lock()
@@ -480,7 +487,26 @@ func (s *Service) files(set config.SlurmSettings) (map[string]string, []string)
 	if c.ControllerAddr == "" {
 		return nil, waiting
 	}
-	return c.Files(), waiting
+	f := c.Files()
+	if s.containersReady(set) {
+		f["plugstack.conf"] = slurm.PlugstackConf
+	}
+	return f, waiting
+}
+
+// containersReady is whether jobs may ask for containers: every machine of
+// the Slurm cluster says it has pyxis and enroot. Slurm fails a job on a
+// machine without the plugin it was told to load. Under s.mu.
+func (s *Service) containersReady(set config.SlurmSettings) bool {
+	if !set.Containers {
+		return false
+	}
+	for _, n := range s.machines(set) {
+		if st := s.status[n]; st == nil || st.Containers == "" {
+			return false
+		}
+	}
+	return true
 }
 
 // cluster is what slurm.conf says: each node as it last reported, else as
@@ -538,6 +564,7 @@ func (s *Service) message(set config.SlurmSettings, node string, lent bool, key
 		return m
 	}
 	m.Name, m.MungeKey, m.Lent = node, key, lent
+	m.Containers = set.Containers
 	m.Node = isNode
 	if addr := s.controllerAddr(set); addr != "" {
 		m.ConfServer = net.JoinHostPort(addr, strconv.Itoa(slurm.CtldPort))
@@ -652,6 +679,12 @@ func (s *Service) Status() Status {
 	if !set.Enabled {
 		return out
 	}
+	if set.Containers {
+		out.Containers = "installing"
+		if s.containersReady(set) {
+			out.Containers = "on"
+		}
+	}
 	_, out.Waiting = s.files(set)
 	conn := s.agents.ConnectedNodes()
 	for _, node := range set.Nodes {

internal/control/slurmsvc/service_test.go

@@ -287,3 +287,39 @@ func TestLendingSaysWhenAReplicaWithNowhereToGoStops(t *testing.T) {
 		t.Errorf("%q", d)
 	}
 }
+
+func TestPyxisIsLoadedOnlyOnceEveryMachineHasIt(t *testing.T) {
+	// "required" in plugstack.conf fails every job on a machine without the
+	// plugin; sent early, it would stop Slurm's jobs everywhere it is missing.
+	r := newRig()
+	r.set.Containers = true
+	r.report("cuda01", &llmgwv1.SlurmStatus{Hardware: hw})
+	r.tick()
+	if !r.agents.last("cuda01").GetContainers() || !r.agents.last("node06").GetContainers() {
+		t.Fatal("the machines were not told to set containers up")
+	}
+	if _, ok := r.agents.last("node06").GetFiles()["plugstack.conf"]; ok {
+		t.Fatal("pyxis loaded before any machine has it")
+	}
+	if s := r.svc.Status().Containers; s != "installing" {
+		t.Errorf("status %q", s)
+	}
+	r.report("cuda01", &llmgwv1.SlurmStatus{Hardware: hw, Containers: "pyxis 0.24.0, enroot 4.2.1"})
+	r.tick()
+	if _, ok := r.agents.last("node06").GetFiles()["plugstack.conf"]; ok {
+		t.Fatal("pyxis loaded with the controller still without it")
+	}
+	r.report("node06", &llmgwv1.SlurmStatus{Containers: "pyxis 0.24.0, enroot 4.2.1"})
+	r.tick()
+	if p := r.agents.last("node06").GetFiles()["plugstack.conf"]; !strings.Contains(p, "required /usr/local/lib/slurm/spank_pyxis.so") {
+		t.Fatalf("plugstack.conf %q", p)
+	}
+	if s := r.svc.Status().Containers; s != "on" {
+		t.Errorf("status %q", s)
+	}
+	r.set.Containers = false
+	r.tick()
+	if _, ok := r.agents.last("node06").GetFiles()["plugstack.conf"]; ok || r.svc.Status().Containers != "" {
+		t.Fatal("pyxis still loaded with containers off")
+	}
+}

internal/docgen/reference_gen.go

@@ -528,6 +528,7 @@ var Reference = []Block{
 		{Key: "partition", Type: "string", Doc: "Partition is the one partition every node is in. Empty is \"gpu\".", Nested: ""},
 		{Key: "shared", Type: "SharedStorage", Doc: "Shared is the file system every machine of the Slurm cluster sees at\nthe same path: home directories, datasets, the jobs' output and\ncheckpoints. Empty path: none, and a batch job's output stays on the\nnode it ran on.", Nested: "config.SharedStorage"},
 		{Key: "users", Type: "string", Doc: "Users is who keeps the Linux users that run jobs, the same on every\nmachine of the Slurm cluster: \"llmgw\" makes one for each active llmgw\naccount (its login from the e-mail address, a UID of llmgw's from\n200000 up, its SSH keys from the portal), let in to the controller,\nand to a node only while a job of theirs runs there (pam_slurm_adopt);\nempty leaves users to the site (a directory through sssd, or by hand).", Nested: ""},
+		{Key: "containers", Type: "bool", Doc: "Containers lets jobs run in containers: `srun --container-image\nnvcr.io#nvidia/pytorch:25.01-py3` (pyxis, with enroot under it),\nthe GPUs and the shared path inside. The agents install both on\nevery Slurm machine; jobs take it once every machine has them.", Nested: ""},
 	}},
 	{Package: "engines", Name: "StrataConfig", Title: "StrataConfig", Doc: "", Fields: []Field{
 		{Key: "dir", Type: "string", Doc: "Dir is Strata's source directory, where setup.py and serve/ are.", Nested: ""},

internal/slurm/containers.go

@@ -0,0 +1,139 @@
+package slurm
+
+import (
+	"fmt"
+	"strings"
+)
+
+// Jobs in containers: pyxis, Slurm's plugin that gives srun and sbatch
+// --container-image, and enroot under it, which turns an image into a
+// directory a job's user runs in without a daemon or root. Both are
+// NVIDIA's and in no distribution's archive, so each is fetched from its
+// release at a pinned version and checked against the digest here.
+const (
+	EnrootVersion = "4.2.1"
+	PyxisVersion  = "0.24.0"
+
+	// PyxisSource is pyxis's source, built on each machine against its own
+	// Slurm: a plugin built for another Slurm is refused by it.
+	PyxisSource = "https://github.com/NVIDIA/pyxis/archive/refs/tags/v" + PyxisVersion + ".tar.gz"
+	PyxisSHA256 = "9c4cdb79a67301d8ea05951aa4d2c205f40cf6145e338214b0191add8b36d6c4"
+
+	// PyxisPlugin is where it is installed (pyxis's own make install).
+	PyxisPlugin = "/usr/local/lib/slurm/spank_pyxis.so"
+
+	// PlugstackConf has Slurm load pyxis. Written on the controller, from
+	// which configless nodes fetch it, only once every machine has the
+	// plugin: "required" fails every job on a machine without it, and
+	// "optional" would run a job asking for a container on the host.
+	PlugstackConf = "# Written by llmgw (BACKLOG P): jobs in containers, srun --container-image.\n" +
+		"required " + PyxisPlugin + "\n"
+
+	// EnrootConfPath is enroot's settings.
+	EnrootConfPath = "/etc/enroot/enroot.conf"
+	// EnrootMounts has every container see the shared path.
+	EnrootMounts = "/etc/enroot/mounts.d/50-llmgw-shared.fstab"
+	// EnrootTmpfiles makes enroot's runtime directory at each boot.
+	EnrootTmpfiles = "/etc/tmpfiles.d/llmgw-enroot.conf"
+	// EnrootRun is where containers are unpacked while they run; EnrootLib
+	// holds each user's image cache and named containers.
+	EnrootRun = "/run/enroot"
+	EnrootLib = "/var/lib/enroot"
+)
+
+// EnrootPackage is one of enroot's release files.
+type EnrootPackage struct {
+	Name, SHA256 string
+}
+
+// URL is where the file is.
+func (p EnrootPackage) URL() string {
+	return "https://github.com/NVIDIA/enroot/releases/download/v" + EnrootVersion + "/" +
+		strings.ReplaceAll(p.Name, "+", "%2B")
+}
+
+// EnrootPackages are the files for a family and architecture (Go's
+// names): enroot, and enroot+caps, which lets its helpers unpack images
+// for a user without root.
+func EnrootPackages(family, arch string) ([]EnrootPackage, error) {
+	ps, ok := enrootPackages[family+"/"+arch]
+	if !ok {
+		return nil, fmt.Errorf("enroot %s has no package for %s on %s", EnrootVersion, family, arch)
+	}
+	return ps, nil
+}
+
+var enrootPackages = map[string][]EnrootPackage{
+	"deb/amd64": {
+		{"enroot_4.2.1-1_amd64.deb", "5955c51e88df0c5ed538d87abb5264609d4bad55fa8570745ed5d4ec66c8ab91"},
+		{"enroot+caps_4.2.1-1_amd64.deb", "ca22652dcbdd3ebb6e2c68df781ef4301647c79dd7a41ed62899c2fd0f692750"},
+	},
+	"deb/arm64": {
+		{"enroot_4.2.1-1_arm64.deb", "267018b335815a6b32a562915de1f3fad579f7c9d6748e98c62e7369b40be421"},
+		{"enroot+caps_4.2.1-1_arm64.deb", "12b811d0c95e879e42e4ac762fa85869a5f3cc9dbb06ab483c01e12e911284a1"},
+	},
+	"rpm/amd64": {
+		{"enroot-4.2.1-1.el8.x86_64.rpm", "6749195e37e3a98071ec349884900b12de7677332599da0fd5d8afb5bed9233c"},
+		{"enroot+caps-4.2.1-1.el8.x86_64.rpm", "56321a03a1f6c4e4273bf664917eb9a174714499d5506c498db4b64275a2b9c0"},
+	},
+	"rpm/arm64": {
+		{"enroot-4.2.1-1.el8.aarch64.rpm", "6b6bc2526f0f0daa7d3cebe6dd33c0468b209e81eebd92def5777520042ec6fe"},
+		{"enroot+caps-4.2.1-1.el8.aarch64.rpm", "38cf9ae9602a9a8c3e94637bc1b34e736e8176bda656ba71d371f5e1d1e1eed9"},
+	},
+}
+
+// PyxisBuildPackages build pyxis: a compiler and Slurm's headers.
+var PyxisBuildPackages = map[string][]string{
+	"deb": {"gcc", "make", "libslurm-dev"},
+	"rpm": {"gcc", "make", "slurm-devel"},
+}
+
+// EnrootConf is enroot's settings: each user's containers, cache and
+// data apart, in directories that need no login session (a job has none,
+// and llmgw's users have no pam_systemd on a node).
+const EnrootConf = "# Written by llmgw (BACKLOG P): enroot, for Slurm's jobs in containers.\n" +
+	"ENROOT_RUNTIME_PATH " + EnrootRun + "/user-$(id -u)\n" +
+	"ENROOT_CACHE_PATH " + EnrootLib + "/cache/user-$(id -u)\n" +
+	"ENROOT_DATA_PATH " + EnrootLib + "/data/user-$(id -u)\n" +
+	"ENROOT_TEMP_PATH ${TMPDIR:-/tmp}\n" +
+	// A container's root is the user's, as on the host: no fake root.
+	"ENROOT_REMAP_ROOT no\n" +
+	"ENROOT_ROOTFS_WRITABLE yes\n"
+
+// EnrootTmpfilesConf makes EnrootRun, open to every user as /tmp is.
+const EnrootTmpfilesConf = "# Written by llmgw (BACKLOG P): enroot's runtime directory.\n" +
+	"d " + EnrootRun + " 1777 root root -\n"
+
+// EnrootSharedMount has a container see path where the host does.
+func EnrootSharedMount(path string) string {
+	return "# Written by llmgw (BACKLOG P): the Slurm cluster's shared files, in every container.\n" +
+		path + " " + path + " none x-create=dir,rbind,rw,nosuid\n"
+}
+
+// The NVIDIA container library, for enroot's hook that gives a container
+// the job's GPUs: from NVIDIA's own repository, its key checked.
+const (
+	NvidiaContainerKeyURL    = "https://nvidia.github.io/libnvidia-container/gpgkey"
+	NvidiaContainerKeySHA256 = "c880576d6cf75a48e5027a871bac70fd0421ab07d2b55f30877b21f1c87959c9"
+	NvidiaContainerKeyPath   = "/etc/apt/keyrings/llmgw-nvidia-container.asc"
+	NvidiaContainerAptList   = "/etc/apt/sources.list.d/llmgw-nvidia-container.list"
+	NvidiaContainerYumRepo   = "/etc/yum.repos.d/llmgw-nvidia-container.repo"
+)
+
+// NvidiaContainerApt is the apt source.
+const NvidiaContainerApt = "# Written by llmgw (BACKLOG P): nvidia-container-cli, for GPUs in Slurm's containers.\n" +
+	"deb [signed-by=" + NvidiaContainerKeyPath + "] https://nvidia.github.io/libnvidia-container/stable/deb/$(ARCH) /\n"
+
+// NvidiaContainerYum is the yum/dnf repository.
+const NvidiaContainerYum = "# Written by llmgw (BACKLOG P): nvidia-container-cli, for GPUs in Slurm's containers.\n" +
+	"[llmgw-nvidia-container]\nname=NVIDIA container library\n" +
+	"baseurl=https://nvidia.github.io/libnvidia-container/stable/rpm/$basearch\n" +
+	"repo_gpgcheck=1\ngpgcheck=0\nenabled=1\ngpgkey=" + NvidiaContainerKeyURL + "\n"
+
+// NvidiaContainerPackages has nvidia-container-cli.
+var NvidiaContainerPackages = map[string][]string{"deb": {"libnvidia-container-tools"}, "rpm": {"libnvidia-container-tools"}}
+
+// ContainersStatus is what a machine says it has.
+func ContainersStatus() string {
+	return "pyxis " + PyxisVersion + ", enroot " + EnrootVersion
+}