diff --git a/BACKLOG.md b/BACKLOG.md index 33af0bd..dc9784f 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -2088,6 +2088,18 @@ lent, partition`); the leader tells each named node's agent (root) what to be, t `gres/gpu=1` in AllocTRES. Found: lending a node whose GPU model has nowhere else to go waits out the drain grace (5 min) before the replica stops, and `ctl slurm status` says only "1 replica(s) still to move off"; it should say it waits for a replacement that cannot be placed, and until when. + Done (c3a75dc): it says the time. +- Jobs in containers, done 2026-11-14: `slurm.containers` (`ctl slurm containers on`). Each agent + installs enroot 4.2.1 (NVIDIA's .deb/.rpm, sha256 pinned) and builds pyxis 0.24.0 (source tarball, + sha256 pinned) against its own Slurm (gcc, make, libslurm-dev); on a machine with nvidia-smi, + nvidia-container-cli from NVIDIA's repository, its key's digest pinned. enroot.conf keeps each + user's runtime, cache and data under /run/enroot and /var/lib/enroot (no login session needed), the + shared path bind-mounted in every container. The leader writes plugstack.conf ("required") only + once every machine reports both, since a machine without the plugin would fail every job. Tried on + the VMs: on in ~30 s; ayse's srun --container-image=ubuntu:24.04 on n1 ran as uid 200001 with + /shared inside and nproc 1; a two-node sbatch ran a rank on each. Seen once: a node's pull timed out + reaching auth.docker.io and the job failed (network, not pyxis). Not tried: GPUs in a container + (cuda01's agent is an older build; needs the peer protocol), Red Hat. - P6, Slurm's jobs in llmgw's accounting (written 2026-11-12, not started). The tenancy tree to Slurm: each project an account (`sacctmgr add account`, parent its team and organization), each member a user of it, kept by the leader from the store; `sbatch --account=PROJECT`. The records back: the leader reads diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e4547b..8b73139 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,6 +65,12 @@ APIs change between minors without a deprecation period. there, their session put in the job (pam_slurm_adopt), so nobody uses a node's cards outside Slurm, nor on a node llmgw serves from. Root and the machines' own users are not held to it. +- **Slurm's jobs in containers.** `llmgw ctl slurm containers on` + (`slurm.containers`) installs enroot and pyxis on every Slurm machine, at + pinned versions checked against their digests, and nvidia-container-cli on + GPU machines; Slurm loads pyxis once every machine has it, and then `srun + --container-image=...` runs a job in that image, as its user, with the + shared path inside. ### Changed diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index 524ffc6..6e4945a 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -5314,7 +5314,10 @@ type Slurm struct { Users []*SlurmUser `protobuf:"bytes,10,rep,name=users,proto3" json:"users,omitempty"` // make_homes has this machine make the users' home directories: the one // that exports the shared path, else the controller. - MakeHomes bool `protobuf:"varint,11,opt,name=make_homes,json=makeHomes,proto3" json:"make_homes,omitempty"` + MakeHomes bool `protobuf:"varint,11,opt,name=make_homes,json=makeHomes,proto3" json:"make_homes,omitempty"` + // containers has the machine install pyxis and enroot, so jobs may run + // in containers (srun --container-image), and set them up. + Containers bool `protobuf:"varint,12,opt,name=containers,proto3" json:"containers,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -5426,6 +5429,13 @@ func (x *Slurm) GetMakeHomes() bool { return false } +func (x *Slurm) GetContainers() bool { + if x != nil { + return x.Containers + } + return false +} + // SlurmShared is where the Slurm cluster's shared files come from. type SlurmShared struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -5610,7 +5620,10 @@ type SlurmStatus struct { // "exported", "missing", or empty when there is none. Shared string `protobuf:"bytes,11,opt,name=shared,proto3" json:"shared,omitempty"` // users is how many of llmgw's users this machine has. - Users int32 `protobuf:"varint,12,opt,name=users,proto3" json:"users,omitempty"` + Users int32 `protobuf:"varint,12,opt,name=users,proto3" json:"users,omitempty"` + // containers is what this machine has for jobs in containers, as + // "pyxis 0.24.0, enroot 4.2.1"; empty while it has not got both. + Containers string `protobuf:"bytes,13,opt,name=containers,proto3" json:"containers,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -5729,6 +5742,13 @@ func (x *SlurmStatus) GetUsers() int32 { return 0 } +func (x *SlurmStatus) GetContainers() string { + if x != nil { + return x.Containers + } + return "" +} + var File_llmgw_v1_agent_proto protoreflect.FileDescriptor const file_llmgw_v1_agent_proto_rawDesc = "" + @@ -6234,7 +6254,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x04kind\x18\x02 \x01(\tR\x04kind\x12\x10\n" + "\x03seq\x18\x03 \x01(\x05R\x03seq\x12\x12\n" + "\x04data\x18\x04 \x01(\fR\x04data\x12\x12\n" + - "\x04last\x18\x05 \x01(\bR\x04last\"\xa9\x03\n" + + "\x04last\x18\x05 \x01(\bR\x04last\"\xc9\x03\n" + "\x05Slurm\x12\x1e\n" + "\n" + "controller\x18\x01 \x01(\bR\n" + @@ -6251,7 +6271,10 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x05users\x18\n" + " \x03(\v2\x13.llmgw.v1.SlurmUserR\x05users\x12\x1d\n" + "\n" + - "make_homes\x18\v \x01(\bR\tmakeHomes\x1a8\n" + + "make_homes\x18\v \x01(\bR\tmakeHomes\x12\x1e\n" + + "\n" + + "containers\x18\f \x01(\bR\n" + + "containers\x1a8\n" + "\n" + "FilesEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + @@ -6266,7 +6289,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x03uid\x18\x02 \x01(\rR\x03uid\x12\x12\n" + "\x04name\x18\x03 \x01(\tR\x04name\x12\x12\n" + "\x04home\x18\x04 \x01(\tR\x04home\x12\x19\n" + - "\bssh_keys\x18\x05 \x03(\tR\asshKeys\"\xeb\x02\n" + + "\bssh_keys\x18\x05 \x03(\tR\asshKeys\"\x8b\x03\n" + "\vSlurmStatus\x12\x18\n" + "\aversion\x18\x01 \x01(\tR\aversion\x12\x1a\n" + "\bhardware\x18\x02 \x01(\tR\bhardware\x12%\n" + @@ -6280,7 +6303,10 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x10slurmdbd_running\x18\n" + " \x01(\bR\x0fslurmdbdRunning\x12\x16\n" + "\x06shared\x18\v \x01(\tR\x06shared\x12\x14\n" + - "\x05users\x18\f \x01(\x05R\x05users2O\n" + + "\x05users\x18\f \x01(\x05R\x05users\x12\x1e\n" + + "\n" + + "containers\x18\r \x01(\tR\n" + + "containers2O\n" + "\fAgentService\x12?\n" + "\aConnect\x12\x16.llmgw.v1.AgentMessage\x1a\x18.llmgw.v1.ControlMessage(\x010\x01B\x93\x01\n" + "\fcom.llmgw.v1B\n" + diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index 29bbc08..04964b5 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -1020,6 +1020,9 @@ message Slurm { // make_homes has this machine make the users' home directories: the one // that exports the shared path, else the controller. bool make_homes = 11; + // containers has the machine install pyxis and enroot, so jobs may run + // in containers (srun --container-image), and set them up. + bool containers = 12; } // SlurmShared is where the Slurm cluster's shared files come from. @@ -1077,4 +1080,7 @@ message SlurmStatus { string shared = 11; // users is how many of llmgw's users this machine has. int32 users = 12; + // containers is what this machine has for jobs in containers, as + // "pyxis 0.24.0, enroot 4.2.1"; empty while it has not got both. + string containers = 13; } diff --git a/docs/admin-api.openapi.json b/docs/admin-api.openapi.json index a6822d8..0226ea5 100644 --- a/docs/admin-api.openapi.json +++ b/docs/admin-api.openapi.json @@ -12372,6 +12372,10 @@ "type": "string", "description": "Shared is the shared path here: \"mounted\", \"exported\", \"missing\"." }, + "containers": { + "type": "string", + "description": "Containers is what the machine has for jobs in containers, empty while not both of pyxis and enroot." + }, "users": { "type": "integer", "description": "Users is how many of llmgw's users the machine has." @@ -12438,6 +12442,10 @@ }, "user_count": { "type": "integer" + }, + "containers": { + "type": "string", + "description": "Containers is whether jobs may run in containers: \"on\" once every machine has pyxis and enroot, \"installing\" until then, empty when slurm.containers is off." } } }, diff --git a/docs/install.md b/docs/install.md index 079effa..0a0f636 100644 --- a/docs/install.md +++ b/docs/install.md @@ -217,6 +217,23 @@ agents install); root and the machines' own users log in as before. A site whose directory (sssd against Active Directory or LDAP) passes `--slurm-users site`, or later `llmgw ctl slurm users off`. +Jobs may run in containers: `llmgw ctl slurm containers on` has the agents +install enroot and pyxis on every Slurm machine (NVIDIA's releases, at +versions llmgw pins and checks; pyxis is built against each machine's own +Slurm) and, on GPU machines, nvidia-container-cli from NVIDIA's repository. +Once every machine has them Slurm loads pyxis, and `ctl slurm status` says +`containers: on`: + +```sh +srun --gres=gpu:1 --container-image=nvcr.io#nvidia/pytorch:25.01-py3 python train.py +``` + +The shared path is in every container where it is on the host, and the job +runs as its user, held to its cores and cards. Each node pulls the image +from its registry for each job; for a large image used often, import it +once (`enroot import -o /shared/images/pytorch.sqsh docker://nvcr.io#...`) +and give that file to `--container-image`. + **Secure by default.** What init writes is the hardened configuration, not a demonstration one: diff --git a/docs/reference.md b/docs/reference.md index 89b3453..2b8a320 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -414,6 +414,7 @@ rather than repeated. | `partition` | `string` | [SlurmSettings](#slurmsettings) | | `shared` | `SharedStorage` | [SlurmSettings](#slurmsettings) | | `users` | `string` | [SlurmSettings](#slurmsettings) | +| `containers` | `bool` | [SlurmSettings](#slurmsettings) | | `dir` | `string` | [StrataConfig](#strataconfig) | | `python` | `string` | [StrataConfig](#strataconfig) | | `engine` | `string` | [StrataConfig](#strataconfig) | @@ -3599,6 +3600,15 @@ account (its login from the e-mail address, a UID of llmgw's from and to a node only while a job of theirs runs there (pam_slurm_adopt); empty leaves users to the site (a directory through sssd, or by hand). +### `containers` + +`bool` + +Containers lets jobs run in containers: `srun --container-image +nvcr.io#nvidia/pytorch:25.01-py3` (pyxis, with enroot under it), +the GPUs and the shared path inside. The agents install both on +every Slurm machine; jobs take it once every machine has them. + ## StrataConfig ### `dir` diff --git a/internal/agent/slurmnode/containers.go b/internal/agent/slurmnode/containers.go new file mode 100644 index 0000000..3fa5883 --- /dev/null +++ b/internal/agent/slurmnode/containers.go @@ -0,0 +1,198 @@ +package slurmnode + +import ( + "context" + "fmt" + "path/filepath" + "runtime" + "strings" + + llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1" + "gitea.mynodes.xyz/ilker/llmgw/internal/slurm" +) + +// cacheDir holds what is fetched for Slurm: enroot's packages, pyxis's +// source and its build. +const cacheDir = "/var/cache/llmgw-agent/slurm" + +// pyxisMarker is the pyxis version built here, against this Slurm. +func (k *Keeper) pyxisMarker() string { return k.marker + "-pyxis" } + +// containers has enroot and pyxis here, set up, when want says so; what +// is there is left when it no longer does (Slurm stops loading pyxis once +// the controller's plugstack.conf goes). Answers the status line, empty +// while either is missing. +func (k *Keeper) containers(ctx context.Context, want *llmgwv1.Slurm) (string, error) { + if !want.GetContainers() { + return "", nil + } + family := k.family() + if family == "" { + return "", fmt.Errorf("neither dpkg nor rpm here: enroot and pyxis come as deb or rpm") + } + if err := k.enroot(ctx, family); err != nil { + return "", fmt.Errorf("enroot: %w", err) + } + if err := k.enrootConf(ctx, want.GetShared().GetPath()); err != nil { + return "", fmt.Errorf("enroot's settings: %w", err) + } + if _, err := k.host.LookPath("nvidia-smi"); err == nil { + if err := k.nvidiaContainer(ctx, family); err != nil { + return "", fmt.Errorf("the GPUs in containers (nvidia-container-cli): %w", err) + } + } + if err := k.pyxis(ctx, family); err != nil { + return "", fmt.Errorf("pyxis: %w", err) + } + return slurm.ContainersStatus(), nil +} + +func (k *Keeper) family() string { + if _, err := k.host.LookPath("dpkg"); err == nil { + return "deb" + } + if _, err := k.host.LookPath("rpm"); err == nil { + return "rpm" + } + return "" +} + +// enroot installs the pinned release when another, or none, is here. +func (k *Keeper) enroot(ctx context.Context, family string) error { + if k.installed["enroot"] { + return nil + } + if out, err := k.host.Run(ctx, nil, "enroot", "version"); err == nil && strings.TrimSpace(string(out)) == slurm.EnrootVersion { + k.installed["enroot"] = true + return nil + } + pkgs, err := slurm.EnrootPackages(family, runtime.GOARCH) + if err != nil { + return err + } + if k.inst == nil { + return fmt.Errorf("enroot missing, and this agent installs nothing") + } + var files []string + for _, p := range pkgs { + f := filepath.Join(cacheDir, p.Name) + if err := k.host.Download(ctx, p.URL(), f, p.SHA256); err != nil { + return err + } + files = append(files, f) + } + k.log.Info("installing enroot, for Slurm's jobs in containers", "version", slurm.EnrootVersion) + if err := k.inst.InstallPackages(ctx, map[string][]string{family: files}); err != nil { + return err + } + out, err := k.host.Run(ctx, nil, "enroot", "version") + if v := strings.TrimSpace(string(out)); err != nil || v != slurm.EnrootVersion { + return fmt.Errorf("enroot says %q after installing %s: %v", v, slurm.EnrootVersion, err) + } + k.installed["enroot"] = true + return nil +} + +// enrootConf writes enroot's settings and makes its directories. +func (k *Keeper) enrootConf(ctx context.Context, shared string) error { + if _, err := k.write(slurm.EnrootConfPath, []byte(slurm.EnrootConf), 0o644, ""); err != nil { + return err + } + changed, err := k.write(slurm.EnrootTmpfiles, []byte(slurm.EnrootTmpfilesConf), 0o644, "") + if err != nil { + return err + } + if changed { + if out, err := k.host.Run(ctx, nil, "systemd-tmpfiles", "--create", slurm.EnrootTmpfiles); err != nil { + return fmt.Errorf("systemd-tmpfiles: %v: %s", err, lastLine(out)) + } + } + for _, d := range []string{slurm.EnrootLib + "/cache", slurm.EnrootLib + "/data"} { + if _, err := k.host.Run(ctx, nil, "test", "-d", d); err == nil { + continue + } + // Each user makes their own beneath, as in /tmp. + if out, err := k.host.Run(ctx, nil, "install", "-d", "-m", "1777", d); err != nil { + return fmt.Errorf("making %s: %v: %s", d, err, lastLine(out)) + } + } + if shared == "" { + return k.host.Remove(slurm.EnrootMounts) + } + _, err = k.write(slurm.EnrootMounts, []byte(slurm.EnrootSharedMount(shared)), 0o644, "") + return err +} + +// nvidiaContainer installs nvidia-container-cli, which enroot's hook uses +// to give a container the job's GPUs, from NVIDIA's repository. +func (k *Keeper) nvidiaContainer(ctx context.Context, family string) error { + if k.installed["nvidia-container-cli"] { + return nil + } + if _, err := k.host.LookPath("nvidia-container-cli"); err == nil { + k.installed["nvidia-container-cli"] = true + return nil + } + if k.inst == nil { + return fmt.Errorf("nvidia-container-cli missing, and this agent installs nothing") + } + switch family { + case "deb": + if err := k.host.Download(ctx, slurm.NvidiaContainerKeyURL, slurm.NvidiaContainerKeyPath, + slurm.NvidiaContainerKeySHA256); err != nil { + return err + } + if _, err := k.write(slurm.NvidiaContainerAptList, []byte(slurm.NvidiaContainerApt), 0o644, ""); err != nil { + return err + } + case "rpm": + if _, err := k.write(slurm.NvidiaContainerYumRepo, []byte(slurm.NvidiaContainerYum), 0o644, ""); err != nil { + return err + } + } + k.log.Info("installing nvidia-container-cli, so containers get the job's GPUs") + if err := k.inst.InstallPackages(ctx, slurm.NvidiaContainerPackages); err != nil { + return err + } + if _, err := k.host.LookPath("nvidia-container-cli"); err != nil { + return fmt.Errorf("still missing after installing libnvidia-container-tools") + } + k.installed["nvidia-container-cli"] = true + return nil +} + +// pyxis builds the pinned pyxis against this machine's Slurm and installs +// it, when it has not been, or was for another version. +func (k *Keeper) pyxis(ctx context.Context, family string) error { + if b, err := k.host.ReadFile(k.pyxisMarker()); err == nil && strings.TrimSpace(string(b)) == slurm.PyxisVersion { + if _, err := k.host.Run(ctx, nil, "test", "-f", slurm.PyxisPlugin); err == nil { + return nil + } + } + if k.inst == nil { + return fmt.Errorf("pyxis missing, and this agent installs nothing") + } + if err := k.inst.InstallPackages(ctx, map[string][]string{family: slurm.PyxisBuildPackages[family]}); err != nil { + return fmt.Errorf("installing what builds it: %w", err) + } + tarball := filepath.Join(cacheDir, "pyxis-"+slurm.PyxisVersion+".tar.gz") + if err := k.host.Download(ctx, slurm.PyxisSource, tarball, slurm.PyxisSHA256); err != nil { + return err + } + src := filepath.Join(cacheDir, "pyxis-"+slurm.PyxisVersion) + k.log.Info("building pyxis against this machine's Slurm", "version", slurm.PyxisVersion) + for _, c := range [][]string{ + {"rm", "-rf", src}, + {"tar", "-xzf", tarball, "-C", cacheDir}, + {"make", "-C", src, "install"}, + } { + if out, err := k.host.Run(ctx, nil, c[0], c[1:]...); err != nil { + return fmt.Errorf("%s: %v: %s", strings.Join(c, " "), err, lastLine(out)) + } + } + if _, err := k.host.Run(ctx, nil, "test", "-f", slurm.PyxisPlugin); err != nil { + return fmt.Errorf("built, yet %s is not there", slurm.PyxisPlugin) + } + _, err := k.write(k.pyxisMarker(), []byte(slurm.PyxisVersion+"\n"), 0o644, "") + return err +} diff --git a/internal/agent/slurmnode/containers_test.go b/internal/agent/slurmnode/containers_test.go new file mode 100644 index 0000000..b0ae342 --- /dev/null +++ b/internal/agent/slurmnode/containers_test.go @@ -0,0 +1,92 @@ +package slurmnode + +import ( + "context" + "runtime" + "slices" + "strings" + "testing" + + llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1" + "gitea.mynodes.xyz/ilker/llmgw/internal/slurm" +) + +func TestContainersInstalledFromPinnedReleases(t *testing.T) { + if _, err := slurm.EnrootPackages("deb", runtime.GOARCH); err != nil { + t.Skip(err) + } + h := newFake() + h.bins["dpkg"], h.bins["nvidia-smi"] = true, true + inst := &fakeInst{host: h} + k := New(h, inst, "/var/lib/llmgw-agent/slurm", nil) + // enroot is not here until installed, and then says its version. + h.fail["enroot version"] = true + h.fail["test -d "+slurm.EnrootLib+"/cache"] = true + h.fail["test -f "+slurm.PyxisPlugin] = true + m := nodeMsg(true) + m.Containers = true + m.Shared = &llmgwv1.SlurmShared{Path: "/shared", Nfs: "10.0.0.11:/shared"} + + st := k.Reconcile(context.Background(), m) + if st.GetContainers() != "" || !strings.Contains(st.GetError(), "containers: enroot") { + t.Fatalf("ready with enroot saying nothing: %v", st) + } + pkgs, _ := slurm.EnrootPackages("deb", runtime.GOARCH) + for _, p := range pkgs { + if !slices.Contains(h.ran, "download "+p.URL()+" "+cacheDir+"/"+p.Name+" "+p.SHA256) { + t.Errorf("%s not fetched with its digest:\n%s", p.Name, strings.Join(h.ran, "\n")) + } + } + if len(inst.got["deb"]) != 2 || !strings.HasPrefix(inst.got["deb"][0], cacheDir) { + t.Errorf("installed %v", inst.got) + } + + delete(h.fail, "enroot version") + h.outputs["enroot version"] = slurm.EnrootVersion + "\n" + h.bins["nvidia-container-cli"] = false + st = k.Reconcile(context.Background(), m) + if !strings.Contains(st.GetError(), "nvidia-container-cli") { + t.Errorf("a GPU machine without nvidia-container-cli is ready: %v", st) + } + if string(h.files[slurm.NvidiaContainerAptList]) != slurm.NvidiaContainerApt || + !slices.Contains(h.ran, "download "+slurm.NvidiaContainerKeyURL+" "+slurm.NvidiaContainerKeyPath+" "+slurm.NvidiaContainerKeySHA256) { + t.Errorf("NVIDIA's repository not added with its key checked") + } + + h.bins["nvidia-container-cli"] = true + st = k.Reconcile(context.Background(), m) + // Built, but the plugin is still not there. + if st.GetContainers() != "" || !strings.Contains(st.GetError(), "pyxis") { + t.Fatalf("status %v", st) + } + for _, want := range []string{"tar -xzf " + cacheDir + "/pyxis-" + slurm.PyxisVersion + ".tar.gz -C " + cacheDir, + "make -C " + cacheDir + "/pyxis-" + slurm.PyxisVersion + " install", "install -d -m 1777 " + slurm.EnrootLib + "/cache"} { + if !slices.Contains(h.ran, want) { + t.Errorf("did not run %q", want) + } + } + delete(h.fail, "test -f "+slurm.PyxisPlugin) + st = k.Reconcile(context.Background(), m) + if st.GetContainers() != slurm.ContainersStatus() || st.GetError() != "" { + t.Fatalf("status %v", st) + } + if string(h.files[slurm.EnrootConfPath]) != slurm.EnrootConf || + !strings.Contains(string(h.files[slurm.EnrootMounts]), "/shared /shared none") { + t.Errorf("enroot's settings not written") + } + + // Done: nothing fetched or built again. + h.ran = nil + k.Reconcile(context.Background(), m) + for _, c := range h.ran { + if strings.HasPrefix(c, "download") || strings.HasPrefix(c, "make") { + t.Errorf("ran %q again", c) + } + } + + // Off: nothing said, nothing taken away. + m.Containers = false + if st := k.Reconcile(context.Background(), m); st.GetContainers() != "" { + t.Errorf("status %v", st) + } +} diff --git a/internal/agent/slurmnode/slurmnode.go b/internal/agent/slurmnode/slurmnode.go index 6c5176b..a00b533 100644 --- a/internal/agent/slurmnode/slurmnode.go +++ b/internal/agent/slurmnode/slurmnode.go @@ -13,8 +13,10 @@ import ( "encoding/hex" "errors" "fmt" + "io" "io/fs" "log/slog" + "net/http" "os" "os/exec" "os/user" @@ -43,6 +45,9 @@ type Host interface { // MkdirAll makes path, owned by owner. MkdirAll(path string, mode fs.FileMode, owner string) error Remove(path string) error + // Download fetches url to path, kept only if its SHA-256 is sha; a file + // already there with it is not fetched again. + Download(ctx context.Context, url, path, sha string) error } // Installer installs packages by family, "deb" and "rpm". @@ -226,6 +231,11 @@ func (k *Keeper) Reconcile(ctx context.Context, want *llmgwv1.Slurm) *llmgwv1.Sl if usErr != nil { k.log.Warn("keeping the Slurm users on this node", "err", usErr) } + // Jobs in containers; Slurm runs without them, and is kept anyway. + var ctErr error + if st.Containers, ctErr = k.containers(ctx, want); ctErr != nil { + k.log.Warn("setting up jobs in containers on this node", "err", ctErr) + } defer func() { switch { case st.Error != "": @@ -233,6 +243,8 @@ func (k *Keeper) Reconcile(ctx context.Context, want *llmgwv1.Slurm) *llmgwv1.Sl st.Error = "shared files: " + shErr.Error() case usErr != nil: st.Error = "users: " + usErr.Error() + case ctErr != nil: + st.Error = "containers: " + ctErr.Error() } }() if want.GetController() { @@ -642,6 +654,55 @@ func (Local) WriteFile(path string, b []byte, mode fs.FileMode, owner string) er return os.Rename(tmp, path) } +// Download fetches url to path through a temporary file beside it, kept +// only when its SHA-256 is sha. +func (Local) Download(ctx context.Context, url, path, sha string) error { + if f, err := os.Open(path); err == nil { //nolint:gosec // our paths + h := sha256.New() + _, cerr := io.Copy(h, f) + _ = f.Close() + if cerr == nil && hex.EncodeToString(h.Sum(nil)) == sha { + return nil + } + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { //nolint:gosec // system directories + return err + } + ctx, cancel := context.WithTimeout(ctx, 10*time.Minute) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("fetching %s: %w", url, err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("fetching %s: %s", url, resp.Status) + } + tmp := path + ".llmgw-new" + f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) //nolint:gosec // our paths + if err != nil { + return err + } + h := sha256.New() + _, err = io.Copy(io.MultiWriter(f, h), io.LimitReader(resp.Body, 1<<30)) + if cerr := f.Close(); err == nil { + err = cerr + } + if err != nil { + _ = os.Remove(tmp) + return fmt.Errorf("fetching %s: %w", url, err) + } + if got := hex.EncodeToString(h.Sum(nil)); got != sha { + _ = os.Remove(tmp) + return fmt.Errorf("%s has SHA-256 %s, not the %s llmgw was built with: not used", url, got, sha) + } + return os.Rename(tmp, path) +} + // MkdirAll makes a directory owned by owner. func (Local) MkdirAll(path string, mode fs.FileMode, owner string) error { if err := os.MkdirAll(path, mode); err != nil { diff --git a/internal/agent/slurmnode/slurmnode_test.go b/internal/agent/slurmnode/slurmnode_test.go index c4b26d8..328974e 100644 --- a/internal/agent/slurmnode/slurmnode_test.go +++ b/internal/agent/slurmnode/slurmnode_test.go @@ -102,6 +102,16 @@ func (f *fakeHost) MkdirAll(string, fs.FileMode, string) error { return nil } func (f *fakeHost) Remove(path string) error { delete(f.files, path); return nil } +func (f *fakeHost) Download(_ context.Context, url, path, sha string) error { + c := "download " + url + " " + path + " " + sha + f.ran = append(f.ran, c) + if f.fail[c] || f.fail["download "+url] { + return errors.New("fetch failed") + } + f.files[path] = []byte("fetched " + url) + return nil +} + func (f *fakeHost) count(prefix string) int { n := 0 for _, c := range f.ran { diff --git a/internal/cli/ctl_slurm.go b/internal/cli/ctl_slurm.go index e03fdcc..e4aa9a7 100644 --- a/internal/cli/ctl_slurm.go +++ b/internal/cli/ctl_slurm.go @@ -32,7 +32,44 @@ func newSlurmCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { cmd.AddCommand(newSlurmStatusCmd(client, asJSON), newSlurmSetupCmd(client), newSlurmOffCmd(client), newSlurmListCmd(client, "lend", "Let Slurm run jobs on nodes; llmgw's replicas move off first", true), newSlurmListCmd(client, "reclaim", "Take nodes back from Slurm; llmgw has them when their jobs end", false), - newSlurmSharedCmd(client), newSlurmUsersCmd(client, asJSON)) + newSlurmSharedCmd(client), newSlurmUsersCmd(client, asJSON), newSlurmContainersCmd(client)) + return cmd +} + +func newSlurmContainersCmd(client func() *ctlClient) *cobra.Command { + cmd := &cobra.Command{ + Use: "containers", + Short: "Jobs in containers: srun --container-image (pyxis and enroot)", + Long: "On, the agents install enroot and pyxis on every machine of the Slurm cluster (from NVIDIA's\n" + + "releases, at versions llmgw pins and checks) and, on GPU machines, nvidia-container-cli; once\n" + + "every machine has them, Slurm loads pyxis and jobs may run in a container:\n\n" + + " srun --gres=gpu:1 --container-image=nvcr.io#nvidia/pytorch:25.01-py3 python train.py\n\n" + + "The shared path is in every container where it is on the host. Off, Slurm stops loading pyxis;\n" + + "what was installed stays.", + } + var note string + toggle := func(use, short string, on bool) *cobra.Command { + c := &cobra.Command{ + Use: use, + Short: short, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return changeSettings(cmd, client, note, func(doc *settings.Doc) error { + if !on { + if doc.Node([]string{"slurm", "containers"}) != nil { + doc.Unset([]string{"slurm", "containers"}) + } + return nil + } + return doc.SetYAML([]string{"slurm", "containers"}, "true") + }) + }, + } + c.Flags().StringVar(¬e, "note", "", "why, kept with the settings' version") + return c + } + cmd.AddCommand(toggle("on", "Install pyxis and enroot everywhere, and let jobs run in containers", true), + toggle("off", "Slurm stops loading pyxis; what was installed stays", false)) return cmd } @@ -211,6 +248,9 @@ func newSlurmStatusCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { if st.Users != "" { fmt.Fprintf(w, "users: %d, made from llmgw's accounts\n", st.UserCount) } + if line := containersLine(st); line != "" { + fmt.Fprintf(w, "containers: %s\n", line) + } tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0) fmt.Fprintln(tw, "NODE\tPHASE\tSLURM\tJOBS\tSLURMD\tSHARED\tNOTE") for _, n := range st.Nodes { @@ -322,6 +362,30 @@ func newSlurmOffCmd(client func() *ctlClient) *cobra.Command { return c } +// containersLine says whether jobs may run in containers, and which +// machines are not ready yet; empty when containers are off. +func containersLine(st slurmsvc.Status) string { + switch st.Containers { + case "on": + return "on (srun --container-image)" + case "": + return "" + } + var waiting []string + for _, n := range st.Nodes { + if n.Status == nil || n.Status.Containers == "" { + waiting = append(waiting, n.Node) + } + } + if st.Ctld == nil || st.Ctld.Containers == "" { + if !slices.Contains(waiting, st.Controller) { + waiting = append(waiting, st.Controller) + } + } + return "installing pyxis and enroot; jobs take them once every machine has them (waiting: " + + strings.Join(waiting, ", ") + ")" +} + // sharedLine says where the shared files are and come from. func sharedLine(sh config.SharedStorage) string { switch { diff --git a/internal/cli/tui_slurm.go b/internal/cli/tui_slurm.go index ba297b6..444454f 100644 --- a/internal/cli/tui_slurm.go +++ b/internal/cli/tui_slurm.go @@ -54,6 +54,9 @@ func (m consoleModel) slurmPanel(pw int) (string, string, string) { } else { head += "\n " + sDim.Render("users: the site's (llmgw ctl slurm users on makes them from llmgw's accounts)") } + if line := containersLine(st); line != "" { + head += "\n containers: " + line + } head += "\n " + sDim.Render("A node is llmgw's until lent; lent, its replicas move off first.") if len(st.Waiting) > 0 { head += "\n " + sWarn.Render(fmt.Sprintf("not in slurm.conf until they report their hardware: %v", st.Waiting)) diff --git a/internal/config/slurm.go b/internal/config/slurm.go index 99c3110..8328ec6 100644 --- a/internal/config/slurm.go +++ b/internal/config/slurm.go @@ -54,6 +54,12 @@ type SlurmSettings struct { // and to a node only while a job of theirs runs there (pam_slurm_adopt); // empty leaves users to the site (a directory through sssd, or by hand). Users string `koanf:"users"` + + // Containers lets jobs run in containers: `srun --container-image + // nvcr.io#nvidia/pytorch:25.01-py3` (pyxis, with enroot under it), + // the GPUs and the shared path inside. The agents install both on + // every Slurm machine; jobs take it once every machine has them. + Containers bool `koanf:"containers"` } // SharedStorage is where the Slurm cluster's shared files come from: the diff --git a/internal/control/slurmsvc/service.go b/internal/control/slurmsvc/service.go index 3a10c89..a7b3b6b 100644 --- a/internal/control/slurmsvc/service.go +++ b/internal/control/slurmsvc/service.go @@ -83,6 +83,9 @@ type Reported struct { SlurmdbdRunning bool `json:"slurmdbd_running"` // Shared is the shared path here: "mounted", "exported", "missing". Shared string `json:"shared,omitempty"` + // Containers is what the machine has for jobs in containers, empty + // while not both of pyxis and enroot. + Containers string `json:"containers,omitempty"` // Users is how many of llmgw's users the machine has. Users int `json:"users"` State string `json:"state,omitempty"` @@ -109,6 +112,10 @@ type Status struct { Shared config.SharedStorage `json:"shared"` Users string `json:"users,omitempty"` UserCount int `json:"user_count,omitempty"` + // Containers is whether jobs may run in containers: "on" once every + // machine has pyxis and enroot, "installing" until then, empty when + // slurm.containers is off. + Containers string `json:"containers,omitempty"` } // User is a Linux user made from an llmgw account. @@ -263,7 +270,7 @@ func (s *Service) Kick() { // OnSlurmStatus records what a node's agent says. func (s *Service) OnSlurmStatus(node string, st *llmgwv1.SlurmStatus) { r := &Reported{Version: st.GetVersion(), Hardware: st.GetHardware(), SlurmdRunning: st.GetSlurmdRunning(), - Shared: st.GetShared(), Users: int(st.GetUsers()), + Shared: st.GetShared(), Users: int(st.GetUsers()), Containers: st.GetContainers(), SlurmctldRunning: st.GetSlurmctldRunning(), SlurmdbdRunning: st.GetSlurmdbdRunning(), State: st.GetState(), Reason: st.GetReason(), Jobs: int(st.GetJobs()), Error: st.GetError(), FilesSHA256: st.GetFilesSha256(), At: time.Now()} s.mu.Lock() @@ -480,7 +487,26 @@ func (s *Service) files(set config.SlurmSettings) (map[string]string, []string) if c.ControllerAddr == "" { return nil, waiting } - return c.Files(), waiting + f := c.Files() + if s.containersReady(set) { + f["plugstack.conf"] = slurm.PlugstackConf + } + return f, waiting +} + +// containersReady is whether jobs may ask for containers: every machine of +// the Slurm cluster says it has pyxis and enroot. Slurm fails a job on a +// machine without the plugin it was told to load. Under s.mu. +func (s *Service) containersReady(set config.SlurmSettings) bool { + if !set.Containers { + return false + } + for _, n := range s.machines(set) { + if st := s.status[n]; st == nil || st.Containers == "" { + return false + } + } + return true } // cluster is what slurm.conf says: each node as it last reported, else as @@ -538,6 +564,7 @@ func (s *Service) message(set config.SlurmSettings, node string, lent bool, key return m } m.Name, m.MungeKey, m.Lent = node, key, lent + m.Containers = set.Containers m.Node = isNode if addr := s.controllerAddr(set); addr != "" { m.ConfServer = net.JoinHostPort(addr, strconv.Itoa(slurm.CtldPort)) @@ -652,6 +679,12 @@ func (s *Service) Status() Status { if !set.Enabled { return out } + if set.Containers { + out.Containers = "installing" + if s.containersReady(set) { + out.Containers = "on" + } + } _, out.Waiting = s.files(set) conn := s.agents.ConnectedNodes() for _, node := range set.Nodes { diff --git a/internal/control/slurmsvc/service_test.go b/internal/control/slurmsvc/service_test.go index d25b5e9..9bc4434 100644 --- a/internal/control/slurmsvc/service_test.go +++ b/internal/control/slurmsvc/service_test.go @@ -287,3 +287,39 @@ func TestLendingSaysWhenAReplicaWithNowhereToGoStops(t *testing.T) { t.Errorf("%q", d) } } + +func TestPyxisIsLoadedOnlyOnceEveryMachineHasIt(t *testing.T) { + // "required" in plugstack.conf fails every job on a machine without the + // plugin; sent early, it would stop Slurm's jobs everywhere it is missing. + r := newRig() + r.set.Containers = true + r.report("cuda01", &llmgwv1.SlurmStatus{Hardware: hw}) + r.tick() + if !r.agents.last("cuda01").GetContainers() || !r.agents.last("node06").GetContainers() { + t.Fatal("the machines were not told to set containers up") + } + if _, ok := r.agents.last("node06").GetFiles()["plugstack.conf"]; ok { + t.Fatal("pyxis loaded before any machine has it") + } + if s := r.svc.Status().Containers; s != "installing" { + t.Errorf("status %q", s) + } + r.report("cuda01", &llmgwv1.SlurmStatus{Hardware: hw, Containers: "pyxis 0.24.0, enroot 4.2.1"}) + r.tick() + if _, ok := r.agents.last("node06").GetFiles()["plugstack.conf"]; ok { + t.Fatal("pyxis loaded with the controller still without it") + } + r.report("node06", &llmgwv1.SlurmStatus{Containers: "pyxis 0.24.0, enroot 4.2.1"}) + r.tick() + if p := r.agents.last("node06").GetFiles()["plugstack.conf"]; !strings.Contains(p, "required /usr/local/lib/slurm/spank_pyxis.so") { + t.Fatalf("plugstack.conf %q", p) + } + if s := r.svc.Status().Containers; s != "on" { + t.Errorf("status %q", s) + } + r.set.Containers = false + r.tick() + if _, ok := r.agents.last("node06").GetFiles()["plugstack.conf"]; ok || r.svc.Status().Containers != "" { + t.Fatal("pyxis still loaded with containers off") + } +} diff --git a/internal/docgen/reference_gen.go b/internal/docgen/reference_gen.go index 09c7da2..ec5ff9a 100644 --- a/internal/docgen/reference_gen.go +++ b/internal/docgen/reference_gen.go @@ -528,6 +528,7 @@ var Reference = []Block{ {Key: "partition", Type: "string", Doc: "Partition is the one partition every node is in. Empty is \"gpu\".", Nested: ""}, {Key: "shared", Type: "SharedStorage", Doc: "Shared is the file system every machine of the Slurm cluster sees at\nthe same path: home directories, datasets, the jobs' output and\ncheckpoints. Empty path: none, and a batch job's output stays on the\nnode it ran on.", Nested: "config.SharedStorage"}, {Key: "users", Type: "string", Doc: "Users is who keeps the Linux users that run jobs, the same on every\nmachine of the Slurm cluster: \"llmgw\" makes one for each active llmgw\naccount (its login from the e-mail address, a UID of llmgw's from\n200000 up, its SSH keys from the portal), let in to the controller,\nand to a node only while a job of theirs runs there (pam_slurm_adopt);\nempty leaves users to the site (a directory through sssd, or by hand).", Nested: ""}, + {Key: "containers", Type: "bool", Doc: "Containers lets jobs run in containers: `srun --container-image\nnvcr.io#nvidia/pytorch:25.01-py3` (pyxis, with enroot under it),\nthe GPUs and the shared path inside. The agents install both on\nevery Slurm machine; jobs take it once every machine has them.", Nested: ""}, }}, {Package: "engines", Name: "StrataConfig", Title: "StrataConfig", Doc: "", Fields: []Field{ {Key: "dir", Type: "string", Doc: "Dir is Strata's source directory, where setup.py and serve/ are.", Nested: ""}, diff --git a/internal/slurm/containers.go b/internal/slurm/containers.go new file mode 100644 index 0000000..484fa10 --- /dev/null +++ b/internal/slurm/containers.go @@ -0,0 +1,139 @@ +package slurm + +import ( + "fmt" + "strings" +) + +// Jobs in containers: pyxis, Slurm's plugin that gives srun and sbatch +// --container-image, and enroot under it, which turns an image into a +// directory a job's user runs in without a daemon or root. Both are +// NVIDIA's and in no distribution's archive, so each is fetched from its +// release at a pinned version and checked against the digest here. +const ( + EnrootVersion = "4.2.1" + PyxisVersion = "0.24.0" + + // PyxisSource is pyxis's source, built on each machine against its own + // Slurm: a plugin built for another Slurm is refused by it. + PyxisSource = "https://github.com/NVIDIA/pyxis/archive/refs/tags/v" + PyxisVersion + ".tar.gz" + PyxisSHA256 = "9c4cdb79a67301d8ea05951aa4d2c205f40cf6145e338214b0191add8b36d6c4" + + // PyxisPlugin is where it is installed (pyxis's own make install). + PyxisPlugin = "/usr/local/lib/slurm/spank_pyxis.so" + + // PlugstackConf has Slurm load pyxis. Written on the controller, from + // which configless nodes fetch it, only once every machine has the + // plugin: "required" fails every job on a machine without it, and + // "optional" would run a job asking for a container on the host. + PlugstackConf = "# Written by llmgw (BACKLOG P): jobs in containers, srun --container-image.\n" + + "required " + PyxisPlugin + "\n" + + // EnrootConfPath is enroot's settings. + EnrootConfPath = "/etc/enroot/enroot.conf" + // EnrootMounts has every container see the shared path. + EnrootMounts = "/etc/enroot/mounts.d/50-llmgw-shared.fstab" + // EnrootTmpfiles makes enroot's runtime directory at each boot. + EnrootTmpfiles = "/etc/tmpfiles.d/llmgw-enroot.conf" + // EnrootRun is where containers are unpacked while they run; EnrootLib + // holds each user's image cache and named containers. + EnrootRun = "/run/enroot" + EnrootLib = "/var/lib/enroot" +) + +// EnrootPackage is one of enroot's release files. +type EnrootPackage struct { + Name, SHA256 string +} + +// URL is where the file is. +func (p EnrootPackage) URL() string { + return "https://github.com/NVIDIA/enroot/releases/download/v" + EnrootVersion + "/" + + strings.ReplaceAll(p.Name, "+", "%2B") +} + +// EnrootPackages are the files for a family and architecture (Go's +// names): enroot, and enroot+caps, which lets its helpers unpack images +// for a user without root. +func EnrootPackages(family, arch string) ([]EnrootPackage, error) { + ps, ok := enrootPackages[family+"/"+arch] + if !ok { + return nil, fmt.Errorf("enroot %s has no package for %s on %s", EnrootVersion, family, arch) + } + return ps, nil +} + +var enrootPackages = map[string][]EnrootPackage{ + "deb/amd64": { + {"enroot_4.2.1-1_amd64.deb", "5955c51e88df0c5ed538d87abb5264609d4bad55fa8570745ed5d4ec66c8ab91"}, + {"enroot+caps_4.2.1-1_amd64.deb", "ca22652dcbdd3ebb6e2c68df781ef4301647c79dd7a41ed62899c2fd0f692750"}, + }, + "deb/arm64": { + {"enroot_4.2.1-1_arm64.deb", "267018b335815a6b32a562915de1f3fad579f7c9d6748e98c62e7369b40be421"}, + {"enroot+caps_4.2.1-1_arm64.deb", "12b811d0c95e879e42e4ac762fa85869a5f3cc9dbb06ab483c01e12e911284a1"}, + }, + "rpm/amd64": { + {"enroot-4.2.1-1.el8.x86_64.rpm", "6749195e37e3a98071ec349884900b12de7677332599da0fd5d8afb5bed9233c"}, + {"enroot+caps-4.2.1-1.el8.x86_64.rpm", "56321a03a1f6c4e4273bf664917eb9a174714499d5506c498db4b64275a2b9c0"}, + }, + "rpm/arm64": { + {"enroot-4.2.1-1.el8.aarch64.rpm", "6b6bc2526f0f0daa7d3cebe6dd33c0468b209e81eebd92def5777520042ec6fe"}, + {"enroot+caps-4.2.1-1.el8.aarch64.rpm", "38cf9ae9602a9a8c3e94637bc1b34e736e8176bda656ba71d371f5e1d1e1eed9"}, + }, +} + +// PyxisBuildPackages build pyxis: a compiler and Slurm's headers. +var PyxisBuildPackages = map[string][]string{ + "deb": {"gcc", "make", "libslurm-dev"}, + "rpm": {"gcc", "make", "slurm-devel"}, +} + +// EnrootConf is enroot's settings: each user's containers, cache and +// data apart, in directories that need no login session (a job has none, +// and llmgw's users have no pam_systemd on a node). +const EnrootConf = "# Written by llmgw (BACKLOG P): enroot, for Slurm's jobs in containers.\n" + + "ENROOT_RUNTIME_PATH " + EnrootRun + "/user-$(id -u)\n" + + "ENROOT_CACHE_PATH " + EnrootLib + "/cache/user-$(id -u)\n" + + "ENROOT_DATA_PATH " + EnrootLib + "/data/user-$(id -u)\n" + + "ENROOT_TEMP_PATH ${TMPDIR:-/tmp}\n" + + // A container's root is the user's, as on the host: no fake root. + "ENROOT_REMAP_ROOT no\n" + + "ENROOT_ROOTFS_WRITABLE yes\n" + +// EnrootTmpfilesConf makes EnrootRun, open to every user as /tmp is. +const EnrootTmpfilesConf = "# Written by llmgw (BACKLOG P): enroot's runtime directory.\n" + + "d " + EnrootRun + " 1777 root root -\n" + +// EnrootSharedMount has a container see path where the host does. +func EnrootSharedMount(path string) string { + return "# Written by llmgw (BACKLOG P): the Slurm cluster's shared files, in every container.\n" + + path + " " + path + " none x-create=dir,rbind,rw,nosuid\n" +} + +// The NVIDIA container library, for enroot's hook that gives a container +// the job's GPUs: from NVIDIA's own repository, its key checked. +const ( + NvidiaContainerKeyURL = "https://nvidia.github.io/libnvidia-container/gpgkey" + NvidiaContainerKeySHA256 = "c880576d6cf75a48e5027a871bac70fd0421ab07d2b55f30877b21f1c87959c9" + NvidiaContainerKeyPath = "/etc/apt/keyrings/llmgw-nvidia-container.asc" + NvidiaContainerAptList = "/etc/apt/sources.list.d/llmgw-nvidia-container.list" + NvidiaContainerYumRepo = "/etc/yum.repos.d/llmgw-nvidia-container.repo" +) + +// NvidiaContainerApt is the apt source. +const NvidiaContainerApt = "# Written by llmgw (BACKLOG P): nvidia-container-cli, for GPUs in Slurm's containers.\n" + + "deb [signed-by=" + NvidiaContainerKeyPath + "] https://nvidia.github.io/libnvidia-container/stable/deb/$(ARCH) /\n" + +// NvidiaContainerYum is the yum/dnf repository. +const NvidiaContainerYum = "# Written by llmgw (BACKLOG P): nvidia-container-cli, for GPUs in Slurm's containers.\n" + + "[llmgw-nvidia-container]\nname=NVIDIA container library\n" + + "baseurl=https://nvidia.github.io/libnvidia-container/stable/rpm/$basearch\n" + + "repo_gpgcheck=1\ngpgcheck=0\nenabled=1\ngpgkey=" + NvidiaContainerKeyURL + "\n" + +// NvidiaContainerPackages has nvidia-container-cli. +var NvidiaContainerPackages = map[string][]string{"deb": {"libnvidia-container-tools"}, "rpm": {"libnvidia-container-tools"}} + +// ContainersStatus is what a machine says it has. +func ContainersStatus() string { + return "pyxis " + PyxisVersion + ", enroot " + EnrootVersion +}