From 57f0fd2ed0a7ddd106410e533c8ab9f6357c3280 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sat, 14 Nov 2026 21:31:20 +0100 Subject: [PATCH] Slurm's jobs in containers: pyxis and enroot, installed by the agents slurm.containers (ctl slurm containers on) has every Slurm machine's agent install enroot from NVIDIA's release and build pyxis against its own Slurm, both pinned and checked against their digests, and on GPU machines nvidia-container-cli from NVIDIA's repository. enroot keeps each user's containers under /run/enroot and /var/lib/enroot, needing no login session, with the shared path in every container. The leader has Slurm load pyxis only once every machine reports it: plugstack.conf's "required" fails every job on a machine without the plugin. Co-Authored-By: Claude Opus 5.5 --- BACKLOG.md | 12 ++ CHANGELOG.md | 6 + api/gen/llmgw/v1/agent.pb.go | 38 +++- api/proto/llmgw/v1/agent.proto | 6 + docs/admin-api.openapi.json | 8 + docs/install.md | 17 ++ docs/reference.md | 10 + internal/agent/slurmnode/containers.go | 198 ++++++++++++++++++++ internal/agent/slurmnode/containers_test.go | 92 +++++++++ internal/agent/slurmnode/slurmnode.go | 61 ++++++ internal/agent/slurmnode/slurmnode_test.go | 10 + internal/cli/ctl_slurm.go | 66 ++++++- internal/cli/tui_slurm.go | 3 + internal/config/slurm.go | 6 + internal/control/slurmsvc/service.go | 37 +++- internal/control/slurmsvc/service_test.go | 36 ++++ internal/docgen/reference_gen.go | 1 + internal/slurm/containers.go | 139 ++++++++++++++ 18 files changed, 737 insertions(+), 9 deletions(-) create mode 100644 internal/agent/slurmnode/containers.go create mode 100644 internal/agent/slurmnode/containers_test.go create mode 100644 internal/slurm/containers.go diff --git a/BACKLOG.md b/BACKLOG.md index 33af0bd..dc9784f 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -2088,6 +2088,18 @@ lent, partition`); the leader tells each named node's agent (root) what to be, t `gres/gpu=1` in AllocTRES. Found: lending a node whose GPU model has nowhere else to go waits out the drain grace (5 min) before the replica stops, and `ctl slurm status` says only "1 replica(s) still to move off"; it should say it waits for a replacement that cannot be placed, and until when. + Done (c3a75dc): it says the time. +- Jobs in containers, done 2026-11-14: `slurm.containers` (`ctl slurm containers on`). Each agent + installs enroot 4.2.1 (NVIDIA's .deb/.rpm, sha256 pinned) and builds pyxis 0.24.0 (source tarball, + sha256 pinned) against its own Slurm (gcc, make, libslurm-dev); on a machine with nvidia-smi, + nvidia-container-cli from NVIDIA's repository, its key's digest pinned. enroot.conf keeps each + user's runtime, cache and data under /run/enroot and /var/lib/enroot (no login session needed), the + shared path bind-mounted in every container. The leader writes plugstack.conf ("required") only + once every machine reports both, since a machine without the plugin would fail every job. Tried on + the VMs: on in ~30 s; ayse's srun --container-image=ubuntu:24.04 on n1 ran as uid 200001 with + /shared inside and nproc 1; a two-node sbatch ran a rank on each. Seen once: a node's pull timed out + reaching auth.docker.io and the job failed (network, not pyxis). Not tried: GPUs in a container + (cuda01's agent is an older build; needs the peer protocol), Red Hat. - P6, Slurm's jobs in llmgw's accounting (written 2026-11-12, not started). The tenancy tree to Slurm: each project an account (`sacctmgr add account`, parent its team and organization), each member a user of it, kept by the leader from the store; `sbatch --account=PROJECT`. The records back: the leader reads diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e4547b..8b73139 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,6 +65,12 @@ APIs change between minors without a deprecation period. there, their session put in the job (pam_slurm_adopt), so nobody uses a node's cards outside Slurm, nor on a node llmgw serves from. Root and the machines' own users are not held to it. +- **Slurm's jobs in containers.** `llmgw ctl slurm containers on` + (`slurm.containers`) installs enroot and pyxis on every Slurm machine, at + pinned versions checked against their digests, and nvidia-container-cli on + GPU machines; Slurm loads pyxis once every machine has it, and then `srun + --container-image=...` runs a job in that image, as its user, with the + shared path inside. ### Changed diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index 524ffc6..6e4945a 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -5314,7 +5314,10 @@ type Slurm struct { Users []*SlurmUser `protobuf:"bytes,10,rep,name=users,proto3" json:"users,omitempty"` // make_homes has this machine make the users' home directories: the one // that exports the shared path, else the controller. - MakeHomes bool `protobuf:"varint,11,opt,name=make_homes,json=makeHomes,proto3" json:"make_homes,omitempty"` + MakeHomes bool `protobuf:"varint,11,opt,name=make_homes,json=makeHomes,proto3" json:"make_homes,omitempty"` + // containers has the machine install pyxis and enroot, so jobs may run + // in containers (srun --container-image), and set them up. + Containers bool `protobuf:"varint,12,opt,name=containers,proto3" json:"containers,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -5426,6 +5429,13 @@ func (x *Slurm) GetMakeHomes() bool { return false } +func (x *Slurm) GetContainers() bool { + if x != nil { + return x.Containers + } + return false +} + // SlurmShared is where the Slurm cluster's shared files come from. type SlurmShared struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -5610,7 +5620,10 @@ type SlurmStatus struct { // "exported", "missing", or empty when there is none. Shared string `protobuf:"bytes,11,opt,name=shared,proto3" json:"shared,omitempty"` // users is how many of llmgw's users this machine has. - Users int32 `protobuf:"varint,12,opt,name=users,proto3" json:"users,omitempty"` + Users int32 `protobuf:"varint,12,opt,name=users,proto3" json:"users,omitempty"` + // containers is what this machine has for jobs in containers, as + // "pyxis 0.24.0, enroot 4.2.1"; empty while it has not got both. + Containers string `protobuf:"bytes,13,opt,name=containers,proto3" json:"containers,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -5729,6 +5742,13 @@ func (x *SlurmStatus) GetUsers() int32 { return 0 } +func (x *SlurmStatus) GetContainers() string { + if x != nil { + return x.Containers + } + return "" +} + var File_llmgw_v1_agent_proto protoreflect.FileDescriptor const file_llmgw_v1_agent_proto_rawDesc = "" + @@ -6234,7 +6254,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x04kind\x18\x02 \x01(\tR\x04kind\x12\x10\n" + "\x03seq\x18\x03 \x01(\x05R\x03seq\x12\x12\n" + "\x04data\x18\x04 \x01(\fR\x04data\x12\x12\n" + - "\x04last\x18\x05 \x01(\bR\x04last\"\xa9\x03\n" + + "\x04last\x18\x05 \x01(\bR\x04last\"\xc9\x03\n" + "\x05Slurm\x12\x1e\n" + "\n" + "controller\x18\x01 \x01(\bR\n" + @@ -6251,7 +6271,10 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x05users\x18\n" + " \x03(\v2\x13.llmgw.v1.SlurmUserR\x05users\x12\x1d\n" + "\n" + - "make_homes\x18\v \x01(\bR\tmakeHomes\x1a8\n" + + "make_homes\x18\v \x01(\bR\tmakeHomes\x12\x1e\n" + + "\n" + + "containers\x18\f \x01(\bR\n" + + "containers\x1a8\n" + "\n" + "FilesEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + @@ -6266,7 +6289,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x03uid\x18\x02 \x01(\rR\x03uid\x12\x12\n" + "\x04name\x18\x03 \x01(\tR\x04name\x12\x12\n" + "\x04home\x18\x04 \x01(\tR\x04home\x12\x19\n" + - "\bssh_keys\x18\x05 \x03(\tR\asshKeys\"\xeb\x02\n" + + "\bssh_keys\x18\x05 \x03(\tR\asshKeys\"\x8b\x03\n" + "\vSlurmStatus\x12\x18\n" + "\aversion\x18\x01 \x01(\tR\aversion\x12\x1a\n" + "\bhardware\x18\x02 \x01(\tR\bhardware\x12%\n" + @@ -6280,7 +6303,10 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x10slurmdbd_running\x18\n" + " \x01(\bR\x0fslurmdbdRunning\x12\x16\n" + "\x06shared\x18\v \x01(\tR\x06shared\x12\x14\n" + - "\x05users\x18\f \x01(\x05R\x05users2O\n" + + "\x05users\x18\f \x01(\x05R\x05users\x12\x1e\n" + + "\n" + + "containers\x18\r \x01(\tR\n" + + "containers2O\n" + "\fAgentService\x12?\n" + "\aConnect\x12\x16.llmgw.v1.AgentMessage\x1a\x18.llmgw.v1.ControlMessage(\x010\x01B\x93\x01\n" + "\fcom.llmgw.v1B\n" + diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index 29bbc08..04964b5 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -1020,6 +1020,9 @@ message Slurm { // make_homes has this machine make the users' home directories: the one // that exports the shared path, else the controller. bool make_homes = 11; + // containers has the machine install pyxis and enroot, so jobs may run + // in containers (srun --container-image), and set them up. + bool containers = 12; } // SlurmShared is where the Slurm cluster's shared files come from. @@ -1077,4 +1080,7 @@ message SlurmStatus { string shared = 11; // users is how many of llmgw's users this machine has. int32 users = 12; + // containers is what this machine has for jobs in containers, as + // "pyxis 0.24.0, enroot 4.2.1"; empty while it has not got both. + string containers = 13; } diff --git a/docs/admin-api.openapi.json b/docs/admin-api.openapi.json index a6822d8..0226ea5 100644 --- a/docs/admin-api.openapi.json +++ b/docs/admin-api.openapi.json @@ -12372,6 +12372,10 @@ "type": "string", "description": "Shared is the shared path here: \"mounted\", \"exported\", \"missing\"." }, + "containers": { + "type": "string", + "description": "Containers is what the machine has for jobs in containers, empty while not both of pyxis and enroot." + }, "users": { "type": "integer", "description": "Users is how many of llmgw's users the machine has." @@ -12438,6 +12442,10 @@ }, "user_count": { "type": "integer" + }, + "containers": { + "type": "string", + "description": "Containers is whether jobs may run in containers: \"on\" once every machine has pyxis and enroot, \"installing\" until then, empty when slurm.containers is off." } } }, diff --git a/docs/install.md b/docs/install.md index 079effa..0a0f636 100644 --- a/docs/install.md +++ b/docs/install.md @@ -217,6 +217,23 @@ agents install); root and the machines' own users log in as before. A site whose directory (sssd against Active Directory or LDAP) passes `--slurm-users site`, or later `llmgw ctl slurm users off`. +Jobs may run in containers: `llmgw ctl slurm containers on` has the agents +install enroot and pyxis on every Slurm machine (NVIDIA's releases, at +versions llmgw pins and checks; pyxis is built against each machine's own +Slurm) and, on GPU machines, nvidia-container-cli from NVIDIA's repository. +Once every machine has them Slurm loads pyxis, and `ctl slurm status` says +`containers: on`: + +```sh +srun --gres=gpu:1 --container-image=nvcr.io#nvidia/pytorch:25.01-py3 python train.py +``` + +The shared path is in every container where it is on the host, and the job +runs as its user, held to its cores and cards. Each node pulls the image +from its registry for each job; for a large image used often, import it +once (`enroot import -o /shared/images/pytorch.sqsh docker://nvcr.io#...`) +and give that file to `--container-image`. + **Secure by default.** What init writes is the hardened configuration, not a demonstration one: diff --git a/docs/reference.md b/docs/reference.md index 89b3453..2b8a320 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -414,6 +414,7 @@ rather than repeated. | `partition` | `string` | [SlurmSettings](#slurmsettings) | | `shared` | `SharedStorage` | [SlurmSettings](#slurmsettings) | | `users` | `string` | [SlurmSettings](#slurmsettings) | +| `containers` | `bool` | [SlurmSettings](#slurmsettings) | | `dir` | `string` | [StrataConfig](#strataconfig) | | `python` | `string` | [StrataConfig](#strataconfig) | | `engine` | `string` | [StrataConfig](#strataconfig) | @@ -3599,6 +3600,15 @@ account (its login from the e-mail address, a UID of llmgw's from and to a node only while a job of theirs runs there (pam_slurm_adopt); empty leaves users to the site (a directory through sssd, or by hand). +### `containers` + +`bool` + +Containers lets jobs run in containers: `srun --container-image +nvcr.io#nvidia/pytorch:25.01-py3` (pyxis, with enroot under it), +the GPUs and the shared path inside. The agents install both on +every Slurm machine; jobs take it once every machine has them. + ## StrataConfig ### `dir` diff --git a/internal/agent/slurmnode/containers.go b/internal/agent/slurmnode/containers.go new file mode 100644 index 0000000..3fa5883 --- /dev/null +++ b/internal/agent/slurmnode/containers.go @@ -0,0 +1,198 @@ +package slurmnode + +import ( + "context" + "fmt" + "path/filepath" + "runtime" + "strings" + + llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1" + "gitea.mynodes.xyz/ilker/llmgw/internal/slurm" +) + +// cacheDir holds what is fetched for Slurm: enroot's packages, pyxis's +// source and its build. +const cacheDir = "/var/cache/llmgw-agent/slurm" + +// pyxisMarker is the pyxis version built here, against this Slurm. +func (k *Keeper) pyxisMarker() string { return k.marker + "-pyxis" } + +// containers has enroot and pyxis here, set up, when want says so; what +// is there is left when it no longer does (Slurm stops loading pyxis once +// the controller's plugstack.conf goes). Answers the status line, empty +// while either is missing. +func (k *Keeper) containers(ctx context.Context, want *llmgwv1.Slurm) (string, error) { + if !want.GetContainers() { + return "", nil + } + family := k.family() + if family == "" { + return "", fmt.Errorf("neither dpkg nor rpm here: enroot and pyxis come as deb or rpm") + } + if err := k.enroot(ctx, family); err != nil { + return "", fmt.Errorf("enroot: %w", err) + } + if err := k.enrootConf(ctx, want.GetShared().GetPath()); err != nil { + return "", fmt.Errorf("enroot's settings: %w", err) + } + if _, err := k.host.LookPath("nvidia-smi"); err == nil { + if err := k.nvidiaContainer(ctx, family); err != nil { + return "", fmt.Errorf("the GPUs in containers (nvidia-container-cli): %w", err) + } + } + if err := k.pyxis(ctx, family); err != nil { + return "", fmt.Errorf("pyxis: %w", err) + } + return slurm.ContainersStatus(), nil +} + +func (k *Keeper) family() string { + if _, err := k.host.LookPath("dpkg"); err == nil { + return "deb" + } + if _, err := k.host.LookPath("rpm"); err == nil { + return "rpm" + } + return "" +} + +// enroot installs the pinned release when another, or none, is here. +func (k *Keeper) enroot(ctx context.Context, family string) error { + if k.installed["enroot"] { + return nil + } + if out, err := k.host.Run(ctx, nil, "enroot", "version"); err == nil && strings.TrimSpace(string(out)) == slurm.EnrootVersion { + k.installed["enroot"] = true + return nil + } + pkgs, err := slurm.EnrootPackages(family, runtime.GOARCH) + if err != nil { + return err + } + if k.inst == nil { + return fmt.Errorf("enroot missing, and this agent installs nothing") + } + var files []string + for _, p := range pkgs { + f := filepath.Join(cacheDir, p.Name) + if err := k.host.Download(ctx, p.URL(), f, p.SHA256); err != nil { + return err + } + files = append(files, f) + } + k.log.Info("installing enroot, for Slurm's jobs in containers", "version", slurm.EnrootVersion) + if err := k.inst.InstallPackages(ctx, map[string][]string{family: files}); err != nil { + return err + } + out, err := k.host.Run(ctx, nil, "enroot", "version") + if v := strings.TrimSpace(string(out)); err != nil || v != slurm.EnrootVersion { + return fmt.Errorf("enroot says %q after installing %s: %v", v, slurm.EnrootVersion, err) + } + k.installed["enroot"] = true + return nil +} + +// enrootConf writes enroot's settings and makes its directories. +func (k *Keeper) enrootConf(ctx context.Context, shared string) error { + if _, err := k.write(slurm.EnrootConfPath, []byte(slurm.EnrootConf), 0o644, ""); err != nil { + return err + } + changed, err := k.write(slurm.EnrootTmpfiles, []byte(slurm.EnrootTmpfilesConf), 0o644, "") + if err != nil { + return err + } + if changed { + if out, err := k.host.Run(ctx, nil, "systemd-tmpfiles", "--create", slurm.EnrootTmpfiles); err != nil { + return fmt.Errorf("systemd-tmpfiles: %v: %s", err, lastLine(out)) + } + } + for _, d := range []string{slurm.EnrootLib + "/cache", slurm.EnrootLib + "/data"} { + if _, err := k.host.Run(ctx, nil, "test", "-d", d); err == nil { + continue + } + // Each user makes their own beneath, as in /tmp. + if out, err := k.host.Run(ctx, nil, "install", "-d", "-m", "1777", d); err != nil { + return fmt.Errorf("making %s: %v: %s", d, err, lastLine(out)) + } + } + if shared == "" { + return k.host.Remove(slurm.EnrootMounts) + } + _, err = k.write(slurm.EnrootMounts, []byte(slurm.EnrootSharedMount(shared)), 0o644, "") + return err +} + +// nvidiaContainer installs nvidia-container-cli, which enroot's hook uses +// to give a container the job's GPUs, from NVIDIA's repository. +func (k *Keeper) nvidiaContainer(ctx context.Context, family string) error { + if k.installed["nvidia-container-cli"] { + return nil + } + if _, err := k.host.LookPath("nvidia-container-cli"); err == nil { + k.installed["nvidia-container-cli"] = true + return nil + } + if k.inst == nil { + return fmt.Errorf("nvidia-container-cli missing, and this agent installs nothing") + } + switch family { + case "deb": + if err := k.host.Download(ctx, slurm.NvidiaContainerKeyURL, slurm.NvidiaContainerKeyPath, + slurm.NvidiaContainerKeySHA256); err != nil { + return err + } + if _, err := k.write(slurm.NvidiaContainerAptList, []byte(slurm.NvidiaContainerApt), 0o644, ""); err != nil { + return err + } + case "rpm": + if _, err := k.write(slurm.NvidiaContainerYumRepo, []byte(slurm.NvidiaContainerYum), 0o644, ""); err != nil { + return err + } + } + k.log.Info("installing nvidia-container-cli, so containers get the job's GPUs") + if err := k.inst.InstallPackages(ctx, slurm.NvidiaContainerPackages); err != nil { + return err + } + if _, err := k.host.LookPath("nvidia-container-cli"); err != nil { + return fmt.Errorf("still missing after installing libnvidia-container-tools") + } + k.installed["nvidia-container-cli"] = true + return nil +} + +// pyxis builds the pinned pyxis against this machine's Slurm and installs +// it, when it has not been, or was for another version. +func (k *Keeper) pyxis(ctx context.Context, family string) error { + if b, err := k.host.ReadFile(k.pyxisMarker()); err == nil && strings.TrimSpace(string(b)) == slurm.PyxisVersion { + if _, err := k.host.Run(ctx, nil, "test", "-f", slurm.PyxisPlugin); err == nil { + return nil + } + } + if k.inst == nil { + return fmt.Errorf("pyxis missing, and this agent installs nothing") + } + if err := k.inst.InstallPackages(ctx, map[string][]string{family: slurm.PyxisBuildPackages[family]}); err != nil { + return fmt.Errorf("installing what builds it: %w", err) + } + tarball := filepath.Join(cacheDir, "pyxis-"+slurm.PyxisVersion+".tar.gz") + if err := k.host.Download(ctx, slurm.PyxisSource, tarball, slurm.PyxisSHA256); err != nil { + return err + } + src := filepath.Join(cacheDir, "pyxis-"+slurm.PyxisVersion) + k.log.Info("building pyxis against this machine's Slurm", "version", slurm.PyxisVersion) + for _, c := range [][]string{ + {"rm", "-rf", src}, + {"tar", "-xzf", tarball, "-C", cacheDir}, + {"make", "-C", src, "install"}, + } { + if out, err := k.host.Run(ctx, nil, c[0], c[1:]...); err != nil { + return fmt.Errorf("%s: %v: %s", strings.Join(c, " "), err, lastLine(out)) + } + } + if _, err := k.host.Run(ctx, nil, "test", "-f", slurm.PyxisPlugin); err != nil { + return fmt.Errorf("built, yet %s is not there", slurm.PyxisPlugin) + } + _, err := k.write(k.pyxisMarker(), []byte(slurm.PyxisVersion+"\n"), 0o644, "") + return err +} diff --git a/internal/agent/slurmnode/containers_test.go b/internal/agent/slurmnode/containers_test.go new file mode 100644 index 0000000..b0ae342 --- /dev/null +++ b/internal/agent/slurmnode/containers_test.go @@ -0,0 +1,92 @@ +package slurmnode + +import ( + "context" + "runtime" + "slices" + "strings" + "testing" + + llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1" + "gitea.mynodes.xyz/ilker/llmgw/internal/slurm" +) + +func TestContainersInstalledFromPinnedReleases(t *testing.T) { + if _, err := slurm.EnrootPackages("deb", runtime.GOARCH); err != nil { + t.Skip(err) + } + h := newFake() + h.bins["dpkg"], h.bins["nvidia-smi"] = true, true + inst := &fakeInst{host: h} + k := New(h, inst, "/var/lib/llmgw-agent/slurm", nil) + // enroot is not here until installed, and then says its version. + h.fail["enroot version"] = true + h.fail["test -d "+slurm.EnrootLib+"/cache"] = true + h.fail["test -f "+slurm.PyxisPlugin] = true + m := nodeMsg(true) + m.Containers = true + m.Shared = &llmgwv1.SlurmShared{Path: "/shared", Nfs: "10.0.0.11:/shared"} + + st := k.Reconcile(context.Background(), m) + if st.GetContainers() != "" || !strings.Contains(st.GetError(), "containers: enroot") { + t.Fatalf("ready with enroot saying nothing: %v", st) + } + pkgs, _ := slurm.EnrootPackages("deb", runtime.GOARCH) + for _, p := range pkgs { + if !slices.Contains(h.ran, "download "+p.URL()+" "+cacheDir+"/"+p.Name+" "+p.SHA256) { + t.Errorf("%s not fetched with its digest:\n%s", p.Name, strings.Join(h.ran, "\n")) + } + } + if len(inst.got["deb"]) != 2 || !strings.HasPrefix(inst.got["deb"][0], cacheDir) { + t.Errorf("installed %v", inst.got) + } + + delete(h.fail, "enroot version") + h.outputs["enroot version"] = slurm.EnrootVersion + "\n" + h.bins["nvidia-container-cli"] = false + st = k.Reconcile(context.Background(), m) + if !strings.Contains(st.GetError(), "nvidia-container-cli") { + t.Errorf("a GPU machine without nvidia-container-cli is ready: %v", st) + } + if string(h.files[slurm.NvidiaContainerAptList]) != slurm.NvidiaContainerApt || + !slices.Contains(h.ran, "download "+slurm.NvidiaContainerKeyURL+" "+slurm.NvidiaContainerKeyPath+" "+slurm.NvidiaContainerKeySHA256) { + t.Errorf("NVIDIA's repository not added with its key checked") + } + + h.bins["nvidia-container-cli"] = true + st = k.Reconcile(context.Background(), m) + // Built, but the plugin is still not there. + if st.GetContainers() != "" || !strings.Contains(st.GetError(), "pyxis") { + t.Fatalf("status %v", st) + } + for _, want := range []string{"tar -xzf " + cacheDir + "/pyxis-" + slurm.PyxisVersion + ".tar.gz -C " + cacheDir, + "make -C " + cacheDir + "/pyxis-" + slurm.PyxisVersion + " install", "install -d -m 1777 " + slurm.EnrootLib + "/cache"} { + if !slices.Contains(h.ran, want) { + t.Errorf("did not run %q", want) + } + } + delete(h.fail, "test -f "+slurm.PyxisPlugin) + st = k.Reconcile(context.Background(), m) + if st.GetContainers() != slurm.ContainersStatus() || st.GetError() != "" { + t.Fatalf("status %v", st) + } + if string(h.files[slurm.EnrootConfPath]) != slurm.EnrootConf || + !strings.Contains(string(h.files[slurm.EnrootMounts]), "/shared /shared none") { + t.Errorf("enroot's settings not written") + } + + // Done: nothing fetched or built again. + h.ran = nil + k.Reconcile(context.Background(), m) + for _, c := range h.ran { + if strings.HasPrefix(c, "download") || strings.HasPrefix(c, "make") { + t.Errorf("ran %q again", c) + } + } + + // Off: nothing said, nothing taken away. + m.Containers = false + if st := k.Reconcile(context.Background(), m); st.GetContainers() != "" { + t.Errorf("status %v", st) + } +} diff --git a/internal/agent/slurmnode/slurmnode.go b/internal/agent/slurmnode/slurmnode.go index 6c5176b..a00b533 100644 --- a/internal/agent/slurmnode/slurmnode.go +++ b/internal/agent/slurmnode/slurmnode.go @@ -13,8 +13,10 @@ import ( "encoding/hex" "errors" "fmt" + "io" "io/fs" "log/slog" + "net/http" "os" "os/exec" "os/user" @@ -43,6 +45,9 @@ type Host interface { // MkdirAll makes path, owned by owner. MkdirAll(path string, mode fs.FileMode, owner string) error Remove(path string) error + // Download fetches url to path, kept only if its SHA-256 is sha; a file + // already there with it is not fetched again. + Download(ctx context.Context, url, path, sha string) error } // Installer installs packages by family, "deb" and "rpm". @@ -226,6 +231,11 @@ func (k *Keeper) Reconcile(ctx context.Context, want *llmgwv1.Slurm) *llmgwv1.Sl if usErr != nil { k.log.Warn("keeping the Slurm users on this node", "err", usErr) } + // Jobs in containers; Slurm runs without them, and is kept anyway. + var ctErr error + if st.Containers, ctErr = k.containers(ctx, want); ctErr != nil { + k.log.Warn("setting up jobs in containers on this node", "err", ctErr) + } defer func() { switch { case st.Error != "": @@ -233,6 +243,8 @@ func (k *Keeper) Reconcile(ctx context.Context, want *llmgwv1.Slurm) *llmgwv1.Sl st.Error = "shared files: " + shErr.Error() case usErr != nil: st.Error = "users: " + usErr.Error() + case ctErr != nil: + st.Error = "containers: " + ctErr.Error() } }() if want.GetController() { @@ -642,6 +654,55 @@ func (Local) WriteFile(path string, b []byte, mode fs.FileMode, owner string) er return os.Rename(tmp, path) } +// Download fetches url to path through a temporary file beside it, kept +// only when its SHA-256 is sha. +func (Local) Download(ctx context.Context, url, path, sha string) error { + if f, err := os.Open(path); err == nil { //nolint:gosec // our paths + h := sha256.New() + _, cerr := io.Copy(h, f) + _ = f.Close() + if cerr == nil && hex.EncodeToString(h.Sum(nil)) == sha { + return nil + } + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { //nolint:gosec // system directories + return err + } + ctx, cancel := context.WithTimeout(ctx, 10*time.Minute) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("fetching %s: %w", url, err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("fetching %s: %s", url, resp.Status) + } + tmp := path + ".llmgw-new" + f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) //nolint:gosec // our paths + if err != nil { + return err + } + h := sha256.New() + _, err = io.Copy(io.MultiWriter(f, h), io.LimitReader(resp.Body, 1<<30)) + if cerr := f.Close(); err == nil { + err = cerr + } + if err != nil { + _ = os.Remove(tmp) + return fmt.Errorf("fetching %s: %w", url, err) + } + if got := hex.EncodeToString(h.Sum(nil)); got != sha { + _ = os.Remove(tmp) + return fmt.Errorf("%s has SHA-256 %s, not the %s llmgw was built with: not used", url, got, sha) + } + return os.Rename(tmp, path) +} + // MkdirAll makes a directory owned by owner. func (Local) MkdirAll(path string, mode fs.FileMode, owner string) error { if err := os.MkdirAll(path, mode); err != nil { diff --git a/internal/agent/slurmnode/slurmnode_test.go b/internal/agent/slurmnode/slurmnode_test.go index c4b26d8..328974e 100644 --- a/internal/agent/slurmnode/slurmnode_test.go +++ b/internal/agent/slurmnode/slurmnode_test.go @@ -102,6 +102,16 @@ func (f *fakeHost) MkdirAll(string, fs.FileMode, string) error { return nil } func (f *fakeHost) Remove(path string) error { delete(f.files, path); return nil } +func (f *fakeHost) Download(_ context.Context, url, path, sha string) error { + c := "download " + url + " " + path + " " + sha + f.ran = append(f.ran, c) + if f.fail[c] || f.fail["download "+url] { + return errors.New("fetch failed") + } + f.files[path] = []byte("fetched " + url) + return nil +} + func (f *fakeHost) count(prefix string) int { n := 0 for _, c := range f.ran { diff --git a/internal/cli/ctl_slurm.go b/internal/cli/ctl_slurm.go index e03fdcc..e4aa9a7 100644 --- a/internal/cli/ctl_slurm.go +++ b/internal/cli/ctl_slurm.go @@ -32,7 +32,44 @@ func newSlurmCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { cmd.AddCommand(newSlurmStatusCmd(client, asJSON), newSlurmSetupCmd(client), newSlurmOffCmd(client), newSlurmListCmd(client, "lend", "Let Slurm run jobs on nodes; llmgw's replicas move off first", true), newSlurmListCmd(client, "reclaim", "Take nodes back from Slurm; llmgw has them when their jobs end", false), - newSlurmSharedCmd(client), newSlurmUsersCmd(client, asJSON)) + newSlurmSharedCmd(client), newSlurmUsersCmd(client, asJSON), newSlurmContainersCmd(client)) + return cmd +} + +func newSlurmContainersCmd(client func() *ctlClient) *cobra.Command { + cmd := &cobra.Command{ + Use: "containers", + Short: "Jobs in containers: srun --container-image (pyxis and enroot)", + Long: "On, the agents install enroot and pyxis on every machine of the Slurm cluster (from NVIDIA's\n" + + "releases, at versions llmgw pins and checks) and, on GPU machines, nvidia-container-cli; once\n" + + "every machine has them, Slurm loads pyxis and jobs may run in a container:\n\n" + + " srun --gres=gpu:1 --container-image=nvcr.io#nvidia/pytorch:25.01-py3 python train.py\n\n" + + "The shared path is in every container where it is on the host. Off, Slurm stops loading pyxis;\n" + + "what was installed stays.", + } + var note string + toggle := func(use, short string, on bool) *cobra.Command { + c := &cobra.Command{ + Use: use, + Short: short, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return changeSettings(cmd, client, note, func(doc *settings.Doc) error { + if !on { + if doc.Node([]string{"slurm", "containers"}) != nil { + doc.Unset([]string{"slurm", "containers"}) + } + return nil + } + return doc.SetYAML([]string{"slurm", "containers"}, "true") + }) + }, + } + c.Flags().StringVar(¬e, "note", "", "why, kept with the settings' version") + return c + } + cmd.AddCommand(toggle("on", "Install pyxis and enroot everywhere, and let jobs run in containers", true), + toggle("off", "Slurm stops loading pyxis; what was installed stays", false)) return cmd } @@ -211,6 +248,9 @@ func newSlurmStatusCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { if st.Users != "" { fmt.Fprintf(w, "users: %d, made from llmgw's accounts\n", st.UserCount) } + if line := containersLine(st); line != "" { + fmt.Fprintf(w, "containers: %s\n", line) + } tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0) fmt.Fprintln(tw, "NODE\tPHASE\tSLURM\tJOBS\tSLURMD\tSHARED\tNOTE") for _, n := range st.Nodes { @@ -322,6 +362,30 @@ func newSlurmOffCmd(client func() *ctlClient) *cobra.Command { return c } +// containersLine says whether jobs may run in containers, and which +// machines are not ready yet; empty when containers are off. +func containersLine(st slurmsvc.Status) string { + switch st.Containers { + case "on": + return "on (srun --container-image)" + case "": + return "" + } + var waiting []string + for _, n := range st.Nodes { + if n.Status == nil || n.Status.Containers == "" { + waiting = append(waiting, n.Node) + } + } + if st.Ctld == nil || st.Ctld.Containers == "" { + if !slices.Contains(waiting, st.Controller) { + waiting = append(waiting, st.Controller) + } + } + return "installing pyxis and enroot; jobs take them once every machine has them (waiting: " + + strings.Join(waiting, ", ") + ")" +} + // sharedLine says where the shared files are and come from. func sharedLine(sh config.SharedStorage) string { switch { diff --git a/internal/cli/tui_slurm.go b/internal/cli/tui_slurm.go index ba297b6..444454f 100644 --- a/internal/cli/tui_slurm.go +++ b/internal/cli/tui_slurm.go @@ -54,6 +54,9 @@ func (m consoleModel) slurmPanel(pw int) (string, string, string) { } else { head += "\n " + sDim.Render("users: the site's (llmgw ctl slurm users on makes them from llmgw's accounts)") } + if line := containersLine(st); line != "" { + head += "\n containers: " + line + } head += "\n " + sDim.Render("A node is llmgw's until lent; lent, its replicas move off first.") if len(st.Waiting) > 0 { head += "\n " + sWarn.Render(fmt.Sprintf("not in slurm.conf until they report their hardware: %v", st.Waiting)) diff --git a/internal/config/slurm.go b/internal/config/slurm.go index 99c3110..8328ec6 100644 --- a/internal/config/slurm.go +++ b/internal/config/slurm.go @@ -54,6 +54,12 @@ type SlurmSettings struct { // and to a node only while a job of theirs runs there (pam_slurm_adopt); // empty leaves users to the site (a directory through sssd, or by hand). Users string `koanf:"users"` + + // Containers lets jobs run in containers: `srun --container-image + // nvcr.io#nvidia/pytorch:25.01-py3` (pyxis, with enroot under it), + // the GPUs and the shared path inside. The agents install both on + // every Slurm machine; jobs take it once every machine has them. + Containers bool `koanf:"containers"` } // SharedStorage is where the Slurm cluster's shared files come from: the diff --git a/internal/control/slurmsvc/service.go b/internal/control/slurmsvc/service.go index 3a10c89..a7b3b6b 100644 --- a/internal/control/slurmsvc/service.go +++ b/internal/control/slurmsvc/service.go @@ -83,6 +83,9 @@ type Reported struct { SlurmdbdRunning bool `json:"slurmdbd_running"` // Shared is the shared path here: "mounted", "exported", "missing". Shared string `json:"shared,omitempty"` + // Containers is what the machine has for jobs in containers, empty + // while not both of pyxis and enroot. + Containers string `json:"containers,omitempty"` // Users is how many of llmgw's users the machine has. Users int `json:"users"` State string `json:"state,omitempty"` @@ -109,6 +112,10 @@ type Status struct { Shared config.SharedStorage `json:"shared"` Users string `json:"users,omitempty"` UserCount int `json:"user_count,omitempty"` + // Containers is whether jobs may run in containers: "on" once every + // machine has pyxis and enroot, "installing" until then, empty when + // slurm.containers is off. + Containers string `json:"containers,omitempty"` } // User is a Linux user made from an llmgw account. @@ -263,7 +270,7 @@ func (s *Service) Kick() { // OnSlurmStatus records what a node's agent says. func (s *Service) OnSlurmStatus(node string, st *llmgwv1.SlurmStatus) { r := &Reported{Version: st.GetVersion(), Hardware: st.GetHardware(), SlurmdRunning: st.GetSlurmdRunning(), - Shared: st.GetShared(), Users: int(st.GetUsers()), + Shared: st.GetShared(), Users: int(st.GetUsers()), Containers: st.GetContainers(), SlurmctldRunning: st.GetSlurmctldRunning(), SlurmdbdRunning: st.GetSlurmdbdRunning(), State: st.GetState(), Reason: st.GetReason(), Jobs: int(st.GetJobs()), Error: st.GetError(), FilesSHA256: st.GetFilesSha256(), At: time.Now()} s.mu.Lock() @@ -480,7 +487,26 @@ func (s *Service) files(set config.SlurmSettings) (map[string]string, []string) if c.ControllerAddr == "" { return nil, waiting } - return c.Files(), waiting + f := c.Files() + if s.containersReady(set) { + f["plugstack.conf"] = slurm.PlugstackConf + } + return f, waiting +} + +// containersReady is whether jobs may ask for containers: every machine of +// the Slurm cluster says it has pyxis and enroot. Slurm fails a job on a +// machine without the plugin it was told to load. Under s.mu. +func (s *Service) containersReady(set config.SlurmSettings) bool { + if !set.Containers { + return false + } + for _, n := range s.machines(set) { + if st := s.status[n]; st == nil || st.Containers == "" { + return false + } + } + return true } // cluster is what slurm.conf says: each node as it last reported, else as @@ -538,6 +564,7 @@ func (s *Service) message(set config.SlurmSettings, node string, lent bool, key return m } m.Name, m.MungeKey, m.Lent = node, key, lent + m.Containers = set.Containers m.Node = isNode if addr := s.controllerAddr(set); addr != "" { m.ConfServer = net.JoinHostPort(addr, strconv.Itoa(slurm.CtldPort)) @@ -652,6 +679,12 @@ func (s *Service) Status() Status { if !set.Enabled { return out } + if set.Containers { + out.Containers = "installing" + if s.containersReady(set) { + out.Containers = "on" + } + } _, out.Waiting = s.files(set) conn := s.agents.ConnectedNodes() for _, node := range set.Nodes { diff --git a/internal/control/slurmsvc/service_test.go b/internal/control/slurmsvc/service_test.go index d25b5e9..9bc4434 100644 --- a/internal/control/slurmsvc/service_test.go +++ b/internal/control/slurmsvc/service_test.go @@ -287,3 +287,39 @@ func TestLendingSaysWhenAReplicaWithNowhereToGoStops(t *testing.T) { t.Errorf("%q", d) } } + +func TestPyxisIsLoadedOnlyOnceEveryMachineHasIt(t *testing.T) { + // "required" in plugstack.conf fails every job on a machine without the + // plugin; sent early, it would stop Slurm's jobs everywhere it is missing. + r := newRig() + r.set.Containers = true + r.report("cuda01", &llmgwv1.SlurmStatus{Hardware: hw}) + r.tick() + if !r.agents.last("cuda01").GetContainers() || !r.agents.last("node06").GetContainers() { + t.Fatal("the machines were not told to set containers up") + } + if _, ok := r.agents.last("node06").GetFiles()["plugstack.conf"]; ok { + t.Fatal("pyxis loaded before any machine has it") + } + if s := r.svc.Status().Containers; s != "installing" { + t.Errorf("status %q", s) + } + r.report("cuda01", &llmgwv1.SlurmStatus{Hardware: hw, Containers: "pyxis 0.24.0, enroot 4.2.1"}) + r.tick() + if _, ok := r.agents.last("node06").GetFiles()["plugstack.conf"]; ok { + t.Fatal("pyxis loaded with the controller still without it") + } + r.report("node06", &llmgwv1.SlurmStatus{Containers: "pyxis 0.24.0, enroot 4.2.1"}) + r.tick() + if p := r.agents.last("node06").GetFiles()["plugstack.conf"]; !strings.Contains(p, "required /usr/local/lib/slurm/spank_pyxis.so") { + t.Fatalf("plugstack.conf %q", p) + } + if s := r.svc.Status().Containers; s != "on" { + t.Errorf("status %q", s) + } + r.set.Containers = false + r.tick() + if _, ok := r.agents.last("node06").GetFiles()["plugstack.conf"]; ok || r.svc.Status().Containers != "" { + t.Fatal("pyxis still loaded with containers off") + } +} diff --git a/internal/docgen/reference_gen.go b/internal/docgen/reference_gen.go index 09c7da2..ec5ff9a 100644 --- a/internal/docgen/reference_gen.go +++ b/internal/docgen/reference_gen.go @@ -528,6 +528,7 @@ var Reference = []Block{ {Key: "partition", Type: "string", Doc: "Partition is the one partition every node is in. Empty is \"gpu\".", Nested: ""}, {Key: "shared", Type: "SharedStorage", Doc: "Shared is the file system every machine of the Slurm cluster sees at\nthe same path: home directories, datasets, the jobs' output and\ncheckpoints. Empty path: none, and a batch job's output stays on the\nnode it ran on.", Nested: "config.SharedStorage"}, {Key: "users", Type: "string", Doc: "Users is who keeps the Linux users that run jobs, the same on every\nmachine of the Slurm cluster: \"llmgw\" makes one for each active llmgw\naccount (its login from the e-mail address, a UID of llmgw's from\n200000 up, its SSH keys from the portal), let in to the controller,\nand to a node only while a job of theirs runs there (pam_slurm_adopt);\nempty leaves users to the site (a directory through sssd, or by hand).", Nested: ""}, + {Key: "containers", Type: "bool", Doc: "Containers lets jobs run in containers: `srun --container-image\nnvcr.io#nvidia/pytorch:25.01-py3` (pyxis, with enroot under it),\nthe GPUs and the shared path inside. The agents install both on\nevery Slurm machine; jobs take it once every machine has them.", Nested: ""}, }}, {Package: "engines", Name: "StrataConfig", Title: "StrataConfig", Doc: "", Fields: []Field{ {Key: "dir", Type: "string", Doc: "Dir is Strata's source directory, where setup.py and serve/ are.", Nested: ""}, diff --git a/internal/slurm/containers.go b/internal/slurm/containers.go new file mode 100644 index 0000000..484fa10 --- /dev/null +++ b/internal/slurm/containers.go @@ -0,0 +1,139 @@ +package slurm + +import ( + "fmt" + "strings" +) + +// Jobs in containers: pyxis, Slurm's plugin that gives srun and sbatch +// --container-image, and enroot under it, which turns an image into a +// directory a job's user runs in without a daemon or root. Both are +// NVIDIA's and in no distribution's archive, so each is fetched from its +// release at a pinned version and checked against the digest here. +const ( + EnrootVersion = "4.2.1" + PyxisVersion = "0.24.0" + + // PyxisSource is pyxis's source, built on each machine against its own + // Slurm: a plugin built for another Slurm is refused by it. + PyxisSource = "https://github.com/NVIDIA/pyxis/archive/refs/tags/v" + PyxisVersion + ".tar.gz" + PyxisSHA256 = "9c4cdb79a67301d8ea05951aa4d2c205f40cf6145e338214b0191add8b36d6c4" + + // PyxisPlugin is where it is installed (pyxis's own make install). + PyxisPlugin = "/usr/local/lib/slurm/spank_pyxis.so" + + // PlugstackConf has Slurm load pyxis. Written on the controller, from + // which configless nodes fetch it, only once every machine has the + // plugin: "required" fails every job on a machine without it, and + // "optional" would run a job asking for a container on the host. + PlugstackConf = "# Written by llmgw (BACKLOG P): jobs in containers, srun --container-image.\n" + + "required " + PyxisPlugin + "\n" + + // EnrootConfPath is enroot's settings. + EnrootConfPath = "/etc/enroot/enroot.conf" + // EnrootMounts has every container see the shared path. + EnrootMounts = "/etc/enroot/mounts.d/50-llmgw-shared.fstab" + // EnrootTmpfiles makes enroot's runtime directory at each boot. + EnrootTmpfiles = "/etc/tmpfiles.d/llmgw-enroot.conf" + // EnrootRun is where containers are unpacked while they run; EnrootLib + // holds each user's image cache and named containers. + EnrootRun = "/run/enroot" + EnrootLib = "/var/lib/enroot" +) + +// EnrootPackage is one of enroot's release files. +type EnrootPackage struct { + Name, SHA256 string +} + +// URL is where the file is. +func (p EnrootPackage) URL() string { + return "https://github.com/NVIDIA/enroot/releases/download/v" + EnrootVersion + "/" + + strings.ReplaceAll(p.Name, "+", "%2B") +} + +// EnrootPackages are the files for a family and architecture (Go's +// names): enroot, and enroot+caps, which lets its helpers unpack images +// for a user without root. +func EnrootPackages(family, arch string) ([]EnrootPackage, error) { + ps, ok := enrootPackages[family+"/"+arch] + if !ok { + return nil, fmt.Errorf("enroot %s has no package for %s on %s", EnrootVersion, family, arch) + } + return ps, nil +} + +var enrootPackages = map[string][]EnrootPackage{ + "deb/amd64": { + {"enroot_4.2.1-1_amd64.deb", "5955c51e88df0c5ed538d87abb5264609d4bad55fa8570745ed5d4ec66c8ab91"}, + {"enroot+caps_4.2.1-1_amd64.deb", "ca22652dcbdd3ebb6e2c68df781ef4301647c79dd7a41ed62899c2fd0f692750"}, + }, + "deb/arm64": { + {"enroot_4.2.1-1_arm64.deb", "267018b335815a6b32a562915de1f3fad579f7c9d6748e98c62e7369b40be421"}, + {"enroot+caps_4.2.1-1_arm64.deb", "12b811d0c95e879e42e4ac762fa85869a5f3cc9dbb06ab483c01e12e911284a1"}, + }, + "rpm/amd64": { + {"enroot-4.2.1-1.el8.x86_64.rpm", "6749195e37e3a98071ec349884900b12de7677332599da0fd5d8afb5bed9233c"}, + {"enroot+caps-4.2.1-1.el8.x86_64.rpm", "56321a03a1f6c4e4273bf664917eb9a174714499d5506c498db4b64275a2b9c0"}, + }, + "rpm/arm64": { + {"enroot-4.2.1-1.el8.aarch64.rpm", "6b6bc2526f0f0daa7d3cebe6dd33c0468b209e81eebd92def5777520042ec6fe"}, + {"enroot+caps-4.2.1-1.el8.aarch64.rpm", "38cf9ae9602a9a8c3e94637bc1b34e736e8176bda656ba71d371f5e1d1e1eed9"}, + }, +} + +// PyxisBuildPackages build pyxis: a compiler and Slurm's headers. +var PyxisBuildPackages = map[string][]string{ + "deb": {"gcc", "make", "libslurm-dev"}, + "rpm": {"gcc", "make", "slurm-devel"}, +} + +// EnrootConf is enroot's settings: each user's containers, cache and +// data apart, in directories that need no login session (a job has none, +// and llmgw's users have no pam_systemd on a node). +const EnrootConf = "# Written by llmgw (BACKLOG P): enroot, for Slurm's jobs in containers.\n" + + "ENROOT_RUNTIME_PATH " + EnrootRun + "/user-$(id -u)\n" + + "ENROOT_CACHE_PATH " + EnrootLib + "/cache/user-$(id -u)\n" + + "ENROOT_DATA_PATH " + EnrootLib + "/data/user-$(id -u)\n" + + "ENROOT_TEMP_PATH ${TMPDIR:-/tmp}\n" + + // A container's root is the user's, as on the host: no fake root. + "ENROOT_REMAP_ROOT no\n" + + "ENROOT_ROOTFS_WRITABLE yes\n" + +// EnrootTmpfilesConf makes EnrootRun, open to every user as /tmp is. +const EnrootTmpfilesConf = "# Written by llmgw (BACKLOG P): enroot's runtime directory.\n" + + "d " + EnrootRun + " 1777 root root -\n" + +// EnrootSharedMount has a container see path where the host does. +func EnrootSharedMount(path string) string { + return "# Written by llmgw (BACKLOG P): the Slurm cluster's shared files, in every container.\n" + + path + " " + path + " none x-create=dir,rbind,rw,nosuid\n" +} + +// The NVIDIA container library, for enroot's hook that gives a container +// the job's GPUs: from NVIDIA's own repository, its key checked. +const ( + NvidiaContainerKeyURL = "https://nvidia.github.io/libnvidia-container/gpgkey" + NvidiaContainerKeySHA256 = "c880576d6cf75a48e5027a871bac70fd0421ab07d2b55f30877b21f1c87959c9" + NvidiaContainerKeyPath = "/etc/apt/keyrings/llmgw-nvidia-container.asc" + NvidiaContainerAptList = "/etc/apt/sources.list.d/llmgw-nvidia-container.list" + NvidiaContainerYumRepo = "/etc/yum.repos.d/llmgw-nvidia-container.repo" +) + +// NvidiaContainerApt is the apt source. +const NvidiaContainerApt = "# Written by llmgw (BACKLOG P): nvidia-container-cli, for GPUs in Slurm's containers.\n" + + "deb [signed-by=" + NvidiaContainerKeyPath + "] https://nvidia.github.io/libnvidia-container/stable/deb/$(ARCH) /\n" + +// NvidiaContainerYum is the yum/dnf repository. +const NvidiaContainerYum = "# Written by llmgw (BACKLOG P): nvidia-container-cli, for GPUs in Slurm's containers.\n" + + "[llmgw-nvidia-container]\nname=NVIDIA container library\n" + + "baseurl=https://nvidia.github.io/libnvidia-container/stable/rpm/$basearch\n" + + "repo_gpgcheck=1\ngpgcheck=0\nenabled=1\ngpgkey=" + NvidiaContainerKeyURL + "\n" + +// NvidiaContainerPackages has nvidia-container-cli. +var NvidiaContainerPackages = map[string][]string{"deb": {"libnvidia-container-tools"}, "rpm": {"libnvidia-container-tools"}} + +// ContainersStatus is what a machine says it has. +func ContainersStatus() string { + return "pyxis " + PyxisVersion + ", enroot " + EnrootVersion +}