Ilker ManapandClaude Opus 5.5
57f0fd2ed0
CI / test (push) Failing after 6m53s
slurm.containers (ctl slurm containers on) has every Slurm machine's agent install enroot from NVIDIA's release and build pyxis against its own Slurm, both pinned and checked against their digests, and on GPU machines nvidia-container-cli from NVIDIA's repository. enroot keeps each user's containers under /run/enroot and /var/lib/enroot, needing no login session, with the shared path in every container. The leader has Slurm load pyxis only once every machine reports it: plugstack.conf's "required" fails every job on a machine without the plugin. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
93 lines
3.3 KiB
Go
93 lines
3.3 KiB
Go
package slurmnode
|
|
|
|
import (
|
|
"context"
|
|
"runtime"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
|
|
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
|
|
)
|
|
|
|
func TestContainersInstalledFromPinnedReleases(t *testing.T) {
|
|
if _, err := slurm.EnrootPackages("deb", runtime.GOARCH); err != nil {
|
|
t.Skip(err)
|
|
}
|
|
h := newFake()
|
|
h.bins["dpkg"], h.bins["nvidia-smi"] = true, true
|
|
inst := &fakeInst{host: h}
|
|
k := New(h, inst, "/var/lib/llmgw-agent/slurm", nil)
|
|
// enroot is not here until installed, and then says its version.
|
|
h.fail["enroot version"] = true
|
|
h.fail["test -d "+slurm.EnrootLib+"/cache"] = true
|
|
h.fail["test -f "+slurm.PyxisPlugin] = true
|
|
m := nodeMsg(true)
|
|
m.Containers = true
|
|
m.Shared = &llmgwv1.SlurmShared{Path: "/shared", Nfs: "10.0.0.11:/shared"}
|
|
|
|
st := k.Reconcile(context.Background(), m)
|
|
if st.GetContainers() != "" || !strings.Contains(st.GetError(), "containers: enroot") {
|
|
t.Fatalf("ready with enroot saying nothing: %v", st)
|
|
}
|
|
pkgs, _ := slurm.EnrootPackages("deb", runtime.GOARCH)
|
|
for _, p := range pkgs {
|
|
if !slices.Contains(h.ran, "download "+p.URL()+" "+cacheDir+"/"+p.Name+" "+p.SHA256) {
|
|
t.Errorf("%s not fetched with its digest:\n%s", p.Name, strings.Join(h.ran, "\n"))
|
|
}
|
|
}
|
|
if len(inst.got["deb"]) != 2 || !strings.HasPrefix(inst.got["deb"][0], cacheDir) {
|
|
t.Errorf("installed %v", inst.got)
|
|
}
|
|
|
|
delete(h.fail, "enroot version")
|
|
h.outputs["enroot version"] = slurm.EnrootVersion + "\n"
|
|
h.bins["nvidia-container-cli"] = false
|
|
st = k.Reconcile(context.Background(), m)
|
|
if !strings.Contains(st.GetError(), "nvidia-container-cli") {
|
|
t.Errorf("a GPU machine without nvidia-container-cli is ready: %v", st)
|
|
}
|
|
if string(h.files[slurm.NvidiaContainerAptList]) != slurm.NvidiaContainerApt ||
|
|
!slices.Contains(h.ran, "download "+slurm.NvidiaContainerKeyURL+" "+slurm.NvidiaContainerKeyPath+" "+slurm.NvidiaContainerKeySHA256) {
|
|
t.Errorf("NVIDIA's repository not added with its key checked")
|
|
}
|
|
|
|
h.bins["nvidia-container-cli"] = true
|
|
st = k.Reconcile(context.Background(), m)
|
|
// Built, but the plugin is still not there.
|
|
if st.GetContainers() != "" || !strings.Contains(st.GetError(), "pyxis") {
|
|
t.Fatalf("status %v", st)
|
|
}
|
|
for _, want := range []string{"tar -xzf " + cacheDir + "/pyxis-" + slurm.PyxisVersion + ".tar.gz -C " + cacheDir,
|
|
"make -C " + cacheDir + "/pyxis-" + slurm.PyxisVersion + " install", "install -d -m 1777 " + slurm.EnrootLib + "/cache"} {
|
|
if !slices.Contains(h.ran, want) {
|
|
t.Errorf("did not run %q", want)
|
|
}
|
|
}
|
|
delete(h.fail, "test -f "+slurm.PyxisPlugin)
|
|
st = k.Reconcile(context.Background(), m)
|
|
if st.GetContainers() != slurm.ContainersStatus() || st.GetError() != "" {
|
|
t.Fatalf("status %v", st)
|
|
}
|
|
if string(h.files[slurm.EnrootConfPath]) != slurm.EnrootConf ||
|
|
!strings.Contains(string(h.files[slurm.EnrootMounts]), "/shared /shared none") {
|
|
t.Errorf("enroot's settings not written")
|
|
}
|
|
|
|
// Done: nothing fetched or built again.
|
|
h.ran = nil
|
|
k.Reconcile(context.Background(), m)
|
|
for _, c := range h.ran {
|
|
if strings.HasPrefix(c, "download") || strings.HasPrefix(c, "make") {
|
|
t.Errorf("ran %q again", c)
|
|
}
|
|
}
|
|
|
|
// Off: nothing said, nothing taken away.
|
|
m.Containers = false
|
|
if st := k.Reconcile(context.Background(), m); st.GetContainers() != "" {
|
|
t.Errorf("status %v", st)
|
|
}
|
|
}
|