package slurm import ( "fmt" "strings" ) // Jobs in containers: pyxis, Slurm's plugin that gives srun and sbatch // --container-image, and enroot under it, which turns an image into a // directory a job's user runs in without a daemon or root. Both are // NVIDIA's and in no distribution's archive, so each is fetched from its // release at a pinned version and checked against the digest here. const ( EnrootVersion = "4.2.1" PyxisVersion = "0.24.0" // PyxisSource is pyxis's source, built on each machine against its own // Slurm: a plugin built for another Slurm is refused by it. PyxisSource = "https://github.com/NVIDIA/pyxis/archive/refs/tags/v" + PyxisVersion + ".tar.gz" PyxisSHA256 = "9c4cdb79a67301d8ea05951aa4d2c205f40cf6145e338214b0191add8b36d6c4" // PyxisPlugin is where it is installed (pyxis's own make install). PyxisPlugin = "/usr/local/lib/slurm/spank_pyxis.so" // PlugstackConf has Slurm load pyxis. Written on the controller, from // which configless nodes fetch it, only once every machine has the // plugin: "required" fails every job on a machine without it, and // "optional" would run a job asking for a container on the host. PlugstackConf = "# Written by llmgw (BACKLOG P): jobs in containers, srun --container-image.\n" + "required " + PyxisPlugin + "\n" // EnrootConfPath is enroot's settings. EnrootConfPath = "/etc/enroot/enroot.conf" // EnrootMounts has every container see the shared path. EnrootMounts = "/etc/enroot/mounts.d/50-llmgw-shared.fstab" // EnrootTmpfiles makes enroot's runtime directory at each boot. EnrootTmpfiles = "/etc/tmpfiles.d/llmgw-enroot.conf" // EnrootRun is where containers are unpacked while they run; EnrootLib // holds each user's image cache and named containers. EnrootRun = "/run/enroot" EnrootLib = "/var/lib/enroot" ) // EnrootPackage is one of enroot's release files. type EnrootPackage struct { Name, SHA256 string } // URL is where the file is. func (p EnrootPackage) URL() string { return "https://github.com/NVIDIA/enroot/releases/download/v" + EnrootVersion + "/" + strings.ReplaceAll(p.Name, "+", "%2B") } // EnrootPackages are the files for a family and architecture (Go's // names): enroot, and enroot+caps, which lets its helpers unpack images // for a user without root. func EnrootPackages(family, arch string) ([]EnrootPackage, error) { ps, ok := enrootPackages[family+"/"+arch] if !ok { return nil, fmt.Errorf("enroot %s has no package for %s on %s", EnrootVersion, family, arch) } return ps, nil } var enrootPackages = map[string][]EnrootPackage{ "deb/amd64": { {"enroot_4.2.1-1_amd64.deb", "5955c51e88df0c5ed538d87abb5264609d4bad55fa8570745ed5d4ec66c8ab91"}, {"enroot+caps_4.2.1-1_amd64.deb", "ca22652dcbdd3ebb6e2c68df781ef4301647c79dd7a41ed62899c2fd0f692750"}, }, "deb/arm64": { {"enroot_4.2.1-1_arm64.deb", "267018b335815a6b32a562915de1f3fad579f7c9d6748e98c62e7369b40be421"}, {"enroot+caps_4.2.1-1_arm64.deb", "12b811d0c95e879e42e4ac762fa85869a5f3cc9dbb06ab483c01e12e911284a1"}, }, "rpm/amd64": { {"enroot-4.2.1-1.el8.x86_64.rpm", "6749195e37e3a98071ec349884900b12de7677332599da0fd5d8afb5bed9233c"}, {"enroot+caps-4.2.1-1.el8.x86_64.rpm", "56321a03a1f6c4e4273bf664917eb9a174714499d5506c498db4b64275a2b9c0"}, }, "rpm/arm64": { {"enroot-4.2.1-1.el8.aarch64.rpm", "6b6bc2526f0f0daa7d3cebe6dd33c0468b209e81eebd92def5777520042ec6fe"}, {"enroot+caps-4.2.1-1.el8.aarch64.rpm", "38cf9ae9602a9a8c3e94637bc1b34e736e8176bda656ba71d371f5e1d1e1eed9"}, }, } // PyxisBuildPackages build pyxis: a compiler and Slurm's headers. var PyxisBuildPackages = map[string][]string{ "deb": {"gcc", "make", "libslurm-dev"}, "rpm": {"gcc", "make", "slurm-devel"}, } // EnrootConf is enroot's settings: each user's containers, cache and // data apart, in directories that need no login session (a job has none, // and llmgw's users have no pam_systemd on a node). const EnrootConf = "# Written by llmgw (BACKLOG P): enroot, for Slurm's jobs in containers.\n" + "ENROOT_RUNTIME_PATH " + EnrootRun + "/user-$(id -u)\n" + "ENROOT_CACHE_PATH " + EnrootLib + "/cache/user-$(id -u)\n" + "ENROOT_DATA_PATH " + EnrootLib + "/data/user-$(id -u)\n" + "ENROOT_TEMP_PATH ${TMPDIR:-/tmp}\n" + // A container's root is the user's, as on the host: no fake root. "ENROOT_REMAP_ROOT no\n" + "ENROOT_ROOTFS_WRITABLE yes\n" // EnrootTmpfilesConf makes EnrootRun, open to every user as /tmp is. const EnrootTmpfilesConf = "# Written by llmgw (BACKLOG P): enroot's runtime directory.\n" + "d " + EnrootRun + " 1777 root root -\n" // EnrootSharedMount has a container see path where the host does. func EnrootSharedMount(path string) string { return "# Written by llmgw (BACKLOG P): the Slurm cluster's shared files, in every container.\n" + path + " " + path + " none x-create=dir,rbind,rw,nosuid\n" } // The NVIDIA container library, for enroot's hook that gives a container // the job's GPUs: from NVIDIA's own repository, its key checked. const ( NvidiaContainerKeyURL = "https://nvidia.github.io/libnvidia-container/gpgkey" NvidiaContainerKeySHA256 = "c880576d6cf75a48e5027a871bac70fd0421ab07d2b55f30877b21f1c87959c9" NvidiaContainerKeyPath = "/etc/apt/keyrings/llmgw-nvidia-container.asc" NvidiaContainerAptList = "/etc/apt/sources.list.d/llmgw-nvidia-container.list" NvidiaContainerYumRepo = "/etc/yum.repos.d/llmgw-nvidia-container.repo" ) // NvidiaContainerApt is the apt source. const NvidiaContainerApt = "# Written by llmgw (BACKLOG P): nvidia-container-cli, for GPUs in Slurm's containers.\n" + "deb [signed-by=" + NvidiaContainerKeyPath + "] https://nvidia.github.io/libnvidia-container/stable/deb/$(ARCH) /\n" // NvidiaContainerYum is the yum/dnf repository. const NvidiaContainerYum = "# Written by llmgw (BACKLOG P): nvidia-container-cli, for GPUs in Slurm's containers.\n" + "[llmgw-nvidia-container]\nname=NVIDIA container library\n" + "baseurl=https://nvidia.github.io/libnvidia-container/stable/rpm/$basearch\n" + "repo_gpgcheck=1\ngpgcheck=0\nenabled=1\ngpgkey=" + NvidiaContainerKeyURL + "\n" // NvidiaContainerPackages has nvidia-container-cli. var NvidiaContainerPackages = map[string][]string{"deb": {"libnvidia-container-tools"}, "rpm": {"libnvidia-container-tools"}} // ContainersStatus is what a machine says it has. func ContainersStatus() string { return "pyxis " + PyxisVersion + ", enroot " + EnrootVersion }