Ilker ManapandClaude Opus 5.5
57f0fd2ed0
CI / test (push) Failing after 6m53s
slurm.containers (ctl slurm containers on) has every Slurm machine's agent install enroot from NVIDIA's release and build pyxis against its own Slurm, both pinned and checked against their digests, and on GPU machines nvidia-container-cli from NVIDIA's repository. enroot keeps each user's containers under /run/enroot and /var/lib/enroot, needing no login session, with the shared path in every container. The leader has Slurm load pyxis only once every machine reports it: plugstack.conf's "required" fails every job on a machine without the plugin. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
140 lines
6.2 KiB
Go
140 lines
6.2 KiB
Go
package slurm
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
)
|
|
|
|
// Jobs in containers: pyxis, Slurm's plugin that gives srun and sbatch
|
|
// --container-image, and enroot under it, which turns an image into a
|
|
// directory a job's user runs in without a daemon or root. Both are
|
|
// NVIDIA's and in no distribution's archive, so each is fetched from its
|
|
// release at a pinned version and checked against the digest here.
|
|
const (
|
|
EnrootVersion = "4.2.1"
|
|
PyxisVersion = "0.24.0"
|
|
|
|
// PyxisSource is pyxis's source, built on each machine against its own
|
|
// Slurm: a plugin built for another Slurm is refused by it.
|
|
PyxisSource = "https://github.com/NVIDIA/pyxis/archive/refs/tags/v" + PyxisVersion + ".tar.gz"
|
|
PyxisSHA256 = "9c4cdb79a67301d8ea05951aa4d2c205f40cf6145e338214b0191add8b36d6c4"
|
|
|
|
// PyxisPlugin is where it is installed (pyxis's own make install).
|
|
PyxisPlugin = "/usr/local/lib/slurm/spank_pyxis.so"
|
|
|
|
// PlugstackConf has Slurm load pyxis. Written on the controller, from
|
|
// which configless nodes fetch it, only once every machine has the
|
|
// plugin: "required" fails every job on a machine without it, and
|
|
// "optional" would run a job asking for a container on the host.
|
|
PlugstackConf = "# Written by llmgw (BACKLOG P): jobs in containers, srun --container-image.\n" +
|
|
"required " + PyxisPlugin + "\n"
|
|
|
|
// EnrootConfPath is enroot's settings.
|
|
EnrootConfPath = "/etc/enroot/enroot.conf"
|
|
// EnrootMounts has every container see the shared path.
|
|
EnrootMounts = "/etc/enroot/mounts.d/50-llmgw-shared.fstab"
|
|
// EnrootTmpfiles makes enroot's runtime directory at each boot.
|
|
EnrootTmpfiles = "/etc/tmpfiles.d/llmgw-enroot.conf"
|
|
// EnrootRun is where containers are unpacked while they run; EnrootLib
|
|
// holds each user's image cache and named containers.
|
|
EnrootRun = "/run/enroot"
|
|
EnrootLib = "/var/lib/enroot"
|
|
)
|
|
|
|
// EnrootPackage is one of enroot's release files.
|
|
type EnrootPackage struct {
|
|
Name, SHA256 string
|
|
}
|
|
|
|
// URL is where the file is.
|
|
func (p EnrootPackage) URL() string {
|
|
return "https://github.com/NVIDIA/enroot/releases/download/v" + EnrootVersion + "/" +
|
|
strings.ReplaceAll(p.Name, "+", "%2B")
|
|
}
|
|
|
|
// EnrootPackages are the files for a family and architecture (Go's
|
|
// names): enroot, and enroot+caps, which lets its helpers unpack images
|
|
// for a user without root.
|
|
func EnrootPackages(family, arch string) ([]EnrootPackage, error) {
|
|
ps, ok := enrootPackages[family+"/"+arch]
|
|
if !ok {
|
|
return nil, fmt.Errorf("enroot %s has no package for %s on %s", EnrootVersion, family, arch)
|
|
}
|
|
return ps, nil
|
|
}
|
|
|
|
var enrootPackages = map[string][]EnrootPackage{
|
|
"deb/amd64": {
|
|
{"enroot_4.2.1-1_amd64.deb", "5955c51e88df0c5ed538d87abb5264609d4bad55fa8570745ed5d4ec66c8ab91"},
|
|
{"enroot+caps_4.2.1-1_amd64.deb", "ca22652dcbdd3ebb6e2c68df781ef4301647c79dd7a41ed62899c2fd0f692750"},
|
|
},
|
|
"deb/arm64": {
|
|
{"enroot_4.2.1-1_arm64.deb", "267018b335815a6b32a562915de1f3fad579f7c9d6748e98c62e7369b40be421"},
|
|
{"enroot+caps_4.2.1-1_arm64.deb", "12b811d0c95e879e42e4ac762fa85869a5f3cc9dbb06ab483c01e12e911284a1"},
|
|
},
|
|
"rpm/amd64": {
|
|
{"enroot-4.2.1-1.el8.x86_64.rpm", "6749195e37e3a98071ec349884900b12de7677332599da0fd5d8afb5bed9233c"},
|
|
{"enroot+caps-4.2.1-1.el8.x86_64.rpm", "56321a03a1f6c4e4273bf664917eb9a174714499d5506c498db4b64275a2b9c0"},
|
|
},
|
|
"rpm/arm64": {
|
|
{"enroot-4.2.1-1.el8.aarch64.rpm", "6b6bc2526f0f0daa7d3cebe6dd33c0468b209e81eebd92def5777520042ec6fe"},
|
|
{"enroot+caps-4.2.1-1.el8.aarch64.rpm", "38cf9ae9602a9a8c3e94637bc1b34e736e8176bda656ba71d371f5e1d1e1eed9"},
|
|
},
|
|
}
|
|
|
|
// PyxisBuildPackages build pyxis: a compiler and Slurm's headers.
|
|
var PyxisBuildPackages = map[string][]string{
|
|
"deb": {"gcc", "make", "libslurm-dev"},
|
|
"rpm": {"gcc", "make", "slurm-devel"},
|
|
}
|
|
|
|
// EnrootConf is enroot's settings: each user's containers, cache and
|
|
// data apart, in directories that need no login session (a job has none,
|
|
// and llmgw's users have no pam_systemd on a node).
|
|
const EnrootConf = "# Written by llmgw (BACKLOG P): enroot, for Slurm's jobs in containers.\n" +
|
|
"ENROOT_RUNTIME_PATH " + EnrootRun + "/user-$(id -u)\n" +
|
|
"ENROOT_CACHE_PATH " + EnrootLib + "/cache/user-$(id -u)\n" +
|
|
"ENROOT_DATA_PATH " + EnrootLib + "/data/user-$(id -u)\n" +
|
|
"ENROOT_TEMP_PATH ${TMPDIR:-/tmp}\n" +
|
|
// A container's root is the user's, as on the host: no fake root.
|
|
"ENROOT_REMAP_ROOT no\n" +
|
|
"ENROOT_ROOTFS_WRITABLE yes\n"
|
|
|
|
// EnrootTmpfilesConf makes EnrootRun, open to every user as /tmp is.
|
|
const EnrootTmpfilesConf = "# Written by llmgw (BACKLOG P): enroot's runtime directory.\n" +
|
|
"d " + EnrootRun + " 1777 root root -\n"
|
|
|
|
// EnrootSharedMount has a container see path where the host does.
|
|
func EnrootSharedMount(path string) string {
|
|
return "# Written by llmgw (BACKLOG P): the Slurm cluster's shared files, in every container.\n" +
|
|
path + " " + path + " none x-create=dir,rbind,rw,nosuid\n"
|
|
}
|
|
|
|
// The NVIDIA container library, for enroot's hook that gives a container
|
|
// the job's GPUs: from NVIDIA's own repository, its key checked.
|
|
const (
|
|
NvidiaContainerKeyURL = "https://nvidia.github.io/libnvidia-container/gpgkey"
|
|
NvidiaContainerKeySHA256 = "c880576d6cf75a48e5027a871bac70fd0421ab07d2b55f30877b21f1c87959c9"
|
|
NvidiaContainerKeyPath = "/etc/apt/keyrings/llmgw-nvidia-container.asc"
|
|
NvidiaContainerAptList = "/etc/apt/sources.list.d/llmgw-nvidia-container.list"
|
|
NvidiaContainerYumRepo = "/etc/yum.repos.d/llmgw-nvidia-container.repo"
|
|
)
|
|
|
|
// NvidiaContainerApt is the apt source.
|
|
const NvidiaContainerApt = "# Written by llmgw (BACKLOG P): nvidia-container-cli, for GPUs in Slurm's containers.\n" +
|
|
"deb [signed-by=" + NvidiaContainerKeyPath + "] https://nvidia.github.io/libnvidia-container/stable/deb/$(ARCH) /\n"
|
|
|
|
// NvidiaContainerYum is the yum/dnf repository.
|
|
const NvidiaContainerYum = "# Written by llmgw (BACKLOG P): nvidia-container-cli, for GPUs in Slurm's containers.\n" +
|
|
"[llmgw-nvidia-container]\nname=NVIDIA container library\n" +
|
|
"baseurl=https://nvidia.github.io/libnvidia-container/stable/rpm/$basearch\n" +
|
|
"repo_gpgcheck=1\ngpgcheck=0\nenabled=1\ngpgkey=" + NvidiaContainerKeyURL + "\n"
|
|
|
|
// NvidiaContainerPackages has nvidia-container-cli.
|
|
var NvidiaContainerPackages = map[string][]string{"deb": {"libnvidia-container-tools"}, "rpm": {"libnvidia-container-tools"}}
|
|
|
|
// ContainersStatus is what a machine says it has.
|
|
func ContainersStatus() string {
|
|
return "pyxis " + PyxisVersion + ", enroot " + EnrootVersion
|
|
}
|