Ilker ManapandClaude Opus 5.5
57f0fd2ed0
CI / test (push) Failing after 6m53s
slurm.containers (ctl slurm containers on) has every Slurm machine's agent install enroot from NVIDIA's release and build pyxis against its own Slurm, both pinned and checked against their digests, and on GPU machines nvidia-container-cli from NVIDIA's repository. enroot keeps each user's containers under /run/enroot and /var/lib/enroot, needing no login session, with the shared path in every container. The leader has Slurm load pyxis only once every machine reports it: plugstack.conf's "required" fails every job on a machine without the plugin. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
368 lines
12 KiB
Go
368 lines
12 KiB
Go
package slurmnode
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"io/fs"
|
|
"path/filepath"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
|
|
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
|
|
)
|
|
|
|
type fakeHost struct {
|
|
files map[string][]byte
|
|
owners map[string]string
|
|
bins map[string]bool
|
|
active map[string]bool
|
|
ran []string
|
|
outputs map[string]string
|
|
fail map[string]bool
|
|
state string
|
|
reason string
|
|
}
|
|
|
|
func newFake() *fakeHost {
|
|
return &fakeHost{files: map[string][]byte{}, owners: map[string]string{}, bins: map[string]bool{},
|
|
active: map[string]bool{}, outputs: map[string]string{
|
|
"slurmd -V": "slurm-wlm 23.11.4\n",
|
|
"slurmctld -V": "slurm-wlm 23.11.4\n",
|
|
"scontrol ping": "Slurmctld(primary) at node06 is UP\n",
|
|
"slurmd -C": "NodeName=cuda01 CPUs=16 RealMemory=64000\nUpTime=1-00:00:00\n",
|
|
"hostname -s": "node06\n",
|
|
}, fail: map[string]bool{}, state: "IDLE+DRAIN", reason: slurm.ReasonNew}
|
|
}
|
|
|
|
func (f *fakeHost) Run(_ context.Context, env []string, name string, args ...string) ([]byte, error) {
|
|
c := strings.Join(append([]string{name}, args...), " ")
|
|
f.ran = append(f.ran, c)
|
|
if f.fail[c] {
|
|
return []byte("it failed"), errors.New("exit status 1")
|
|
}
|
|
switch {
|
|
case name == "systemctl" && len(args) >= 1:
|
|
u := args[len(args)-1]
|
|
switch args[0] {
|
|
case "is-active":
|
|
if f.active[u] {
|
|
return []byte("active\n"), nil
|
|
}
|
|
return []byte("inactive\n"), errors.New("exit status 3")
|
|
case "restart", "start":
|
|
f.active[u] = true
|
|
case "enable":
|
|
if slices.Contains(args, "--now") {
|
|
f.active[u] = true
|
|
}
|
|
case "disable":
|
|
f.active[u] = false
|
|
}
|
|
return []byte(f.outputs[c]), nil
|
|
case strings.HasPrefix(c, "scontrol show node"):
|
|
return []byte("NodeName=cuda01 Arch=x86_64 State=" + f.state + " Partitions=gpu Reason=" + f.reason + " [root@now]\n"), nil
|
|
case strings.HasPrefix(c, "scontrol update"):
|
|
if slices.Contains(args, "State=RESUME") {
|
|
f.state, f.reason = "IDLE", ""
|
|
} else {
|
|
f.state, f.reason = "IDLE+DRAIN", ReasonServing
|
|
}
|
|
return nil, nil
|
|
case strings.HasPrefix(c, "pgrep"):
|
|
out := "1800358 /usr/sbin/slurmstepd infinity\n" + f.outputs["jobs"]
|
|
return []byte(out), nil
|
|
}
|
|
return []byte(f.outputs[c]), nil
|
|
}
|
|
|
|
func (f *fakeHost) LookPath(name string) (string, error) {
|
|
if f.bins[name] {
|
|
return "/usr/sbin/" + name, nil
|
|
}
|
|
return "", errors.New("not found")
|
|
}
|
|
|
|
func (f *fakeHost) ReadFile(path string) ([]byte, error) {
|
|
b, ok := f.files[path]
|
|
if !ok {
|
|
return nil, fs.ErrNotExist
|
|
}
|
|
return b, nil
|
|
}
|
|
|
|
func (f *fakeHost) WriteFile(path string, b []byte, _ fs.FileMode, owner string) error {
|
|
f.files[path] = append([]byte(nil), b...)
|
|
f.owners[path] = owner
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeHost) MkdirAll(string, fs.FileMode, string) error { return nil }
|
|
|
|
func (f *fakeHost) Remove(path string) error { delete(f.files, path); return nil }
|
|
|
|
func (f *fakeHost) Download(_ context.Context, url, path, sha string) error {
|
|
c := "download " + url + " " + path + " " + sha
|
|
f.ran = append(f.ran, c)
|
|
if f.fail[c] || f.fail["download "+url] {
|
|
return errors.New("fetch failed")
|
|
}
|
|
f.files[path] = []byte("fetched " + url)
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeHost) count(prefix string) int {
|
|
n := 0
|
|
for _, c := range f.ran {
|
|
if strings.HasPrefix(c, prefix) {
|
|
n++
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
|
|
type fakeInst struct {
|
|
host *fakeHost
|
|
got map[string][]string
|
|
}
|
|
|
|
func (i *fakeInst) InstallPackages(_ context.Context, by map[string][]string) error {
|
|
i.got = by
|
|
for _, b := range []string{"munged", "scontrol", "slurmd", "slurmctld", "slurmdbd", "mariadbd", "mariadb", "exportfs", "mount.nfs"} {
|
|
i.host.bins[b] = true
|
|
}
|
|
return nil
|
|
}
|
|
|
|
var key = []byte(strings.Repeat("k", 1024))
|
|
|
|
func nodeMsg(lent bool) *llmgwv1.Slurm {
|
|
return &llmgwv1.Slurm{Node: true, Name: "cuda01", ConfServer: "10.0.0.146:6817", MungeKey: key, Lent: lent}
|
|
}
|
|
|
|
func TestNodeInstallsAndStaysDrained(t *testing.T) {
|
|
h := newFake()
|
|
inst := &fakeInst{host: h}
|
|
k := New(h, inst, "/var/lib/llmgw-agent/slurm", nil)
|
|
st := k.Reconcile(context.Background(), nodeMsg(false))
|
|
if st.GetError() != "" {
|
|
t.Fatal(st.GetError())
|
|
}
|
|
if !slices.Equal(inst.got["deb"], []string{"munge", "slurm-client", "slurmd"}) ||
|
|
!slices.Equal(inst.got["rpm"], []string{"munge", "slurm", "slurm-slurmd"}) {
|
|
t.Fatalf("installed %v", inst.got)
|
|
}
|
|
if string(h.files[slurm.MungeKeyPath]) != string(key) || h.owners[slurm.MungeKeyPath] != "munge" {
|
|
t.Fatal("the munge key was not written for munge")
|
|
}
|
|
unit := string(h.files[DropIn])
|
|
if !strings.Contains(unit, "ExecStart=/usr/sbin/slurmd --systemd --conf-server 10.0.0.146:6817 -N cuda01\n") ||
|
|
!strings.Contains(unit, "ConditionPathExists=\n") {
|
|
t.Fatalf("drop-in:\n%s", unit)
|
|
}
|
|
if !st.GetSlurmdRunning() || st.GetVersion() != "23.11.4" || !strings.HasPrefix(st.GetHardware(), "NodeName=cuda01 CPUs=16") {
|
|
t.Fatalf("status %v", st)
|
|
}
|
|
// Not lent: left drained, with llmgw's first reason.
|
|
if h.count("scontrol update") != 0 || st.GetState() != "IDLE+DRAIN" {
|
|
t.Fatalf("a node not lent was changed: %v, %v", h.ran, st)
|
|
}
|
|
if _, ok := h.files["/var/lib/llmgw-agent/slurm"]; !ok {
|
|
t.Fatal("no marker")
|
|
}
|
|
|
|
// Again, nothing changed: no install, no restart.
|
|
h.ran = nil
|
|
inst.got = nil
|
|
k.Reconcile(context.Background(), nodeMsg(false))
|
|
if inst.got != nil || h.count("systemctl restart") != 0 || h.count("systemctl enable") != 0 {
|
|
t.Fatalf("a second pass did work: %v", h.ran)
|
|
}
|
|
}
|
|
|
|
func TestLendAndReclaim(t *testing.T) {
|
|
h := newFake()
|
|
k := New(h, &fakeInst{host: h}, "/m", nil)
|
|
st := k.Reconcile(context.Background(), nodeMsg(true))
|
|
if h.count("scontrol update NodeName=cuda01 State=RESUME") != 1 || st.GetState() != "IDLE" {
|
|
t.Fatalf("a lent node was not resumed: %v %v", h.ran, st)
|
|
}
|
|
h.outputs["jobs"] = "2001 slurmstepd: [101.batch]\n2002 slurmstepd: [102.0]\n"
|
|
st = k.Reconcile(context.Background(), nodeMsg(false))
|
|
if h.count("scontrol update NodeName=cuda01 State=DRAIN Reason="+ReasonServing) != 1 {
|
|
t.Fatalf("a node taken back was not drained: %v", h.ran)
|
|
}
|
|
if st.GetJobs() != 2 || !strings.Contains(st.GetState(), "DRAIN") {
|
|
t.Fatalf("status %v", st)
|
|
}
|
|
// Drained already: not drained again.
|
|
h.ran = nil
|
|
k.Reconcile(context.Background(), nodeMsg(false))
|
|
if h.count("scontrol update") != 0 {
|
|
t.Fatalf("drained twice: %v", h.ran)
|
|
}
|
|
}
|
|
|
|
func TestAdminsDrainIsLeft(t *testing.T) {
|
|
h := newFake()
|
|
h.reason = "bad DIMM, ticket 42"
|
|
k := New(h, &fakeInst{host: h}, "/m", nil)
|
|
k.Reconcile(context.Background(), nodeMsg(true))
|
|
if h.count("scontrol update") != 0 {
|
|
t.Fatalf("llmgw resumed a node a Slurm administrator drained: %v", h.ran)
|
|
}
|
|
}
|
|
|
|
func TestClientEnvOnANodeWithoutFiles(t *testing.T) {
|
|
h := newFake()
|
|
k := New(h, &fakeInst{host: h}, "/m", nil)
|
|
if env := k.clientEnv(); len(env) != 1 || !strings.HasSuffix(env[0], "conf-cache/slurm.conf") {
|
|
t.Fatalf("env %v", env)
|
|
}
|
|
h.files[filepath.Join(slurm.ConfDir, "slurm.conf")] = []byte("x")
|
|
if env := k.clientEnv(); env != nil {
|
|
t.Fatalf("env %v on the controller", env)
|
|
}
|
|
}
|
|
|
|
func TestController(t *testing.T) {
|
|
h := newFake()
|
|
inst := &fakeInst{host: h}
|
|
k := New(h, inst, "/m", nil)
|
|
files := map[string]string{"slurm.conf": "ClusterName=llmgw\n", "gres.conf": "#\n", "cgroup.conf": "#\n"}
|
|
m := &llmgwv1.Slurm{Controller: true, MungeKey: key, Files: files}
|
|
st := k.Reconcile(context.Background(), m)
|
|
if st.GetError() != "" {
|
|
t.Fatal(st.GetError())
|
|
}
|
|
if !slices.Contains(inst.got["deb"], "slurmctld") || slices.Contains(inst.got["deb"], "slurmd") {
|
|
t.Fatalf("installed %v", inst.got)
|
|
}
|
|
if string(h.files["/etc/slurm/slurm.conf"]) != "ClusterName=llmgw\n" {
|
|
t.Fatal("slurm.conf not written")
|
|
}
|
|
if h.count("systemctl restart slurmctld") != 1 || h.count("scontrol reconfigure") != 1 {
|
|
t.Fatalf("slurmctld not started on its files: %v", h.ran)
|
|
}
|
|
if !st.GetSlurmctldRunning() || len(st.GetFilesSha256()) != 64 {
|
|
t.Fatalf("status %v", st)
|
|
}
|
|
// Accounting: MariaDB on this machine alone, the database, slurmdbd on
|
|
// it, the cluster added, all before slurmctld.
|
|
if !slices.Contains(inst.got["deb"], "slurmdbd") || !slices.Contains(inst.got["deb"], "mariadb-server") {
|
|
t.Fatalf("installed %v", inst.got)
|
|
}
|
|
if !strings.Contains(string(h.files["/etc/mysql/mariadb.conf.d/90-llmgw-slurm.cnf"]), "bind-address=127.0.0.1") {
|
|
t.Fatal("MariaDB, installed by llmgw, not kept to this machine")
|
|
}
|
|
dbd := string(h.files[slurm.DbdConfPath])
|
|
if !strings.Contains(dbd, "DbdHost=node06\n") || !strings.Contains(dbd, "StoragePass="+slurm.DbdPassword(key)+"\n") ||
|
|
h.owners[slurm.DbdConfPath] != "slurm" {
|
|
t.Fatalf("slurmdbd.conf for %q:\n%s", h.owners[slurm.DbdConfPath], dbd)
|
|
}
|
|
if _, ok := h.files[acctSQLPath]; ok {
|
|
t.Fatal("the database's statements, password and all, were left on disk")
|
|
}
|
|
if strings.Contains(strings.Join(h.ran, "\n"), slurm.DbdPassword(key)) {
|
|
t.Fatal("the password was on a command line")
|
|
}
|
|
order := strings.Join(h.ran, "\n")
|
|
if !strings.Contains(order, "sacctmgr -i add cluster llmgw") ||
|
|
strings.Index(order, "slurmdbd") > strings.Index(order, "systemctl restart slurmctld") {
|
|
t.Fatalf("accounting not before slurmctld: %v", h.ran)
|
|
}
|
|
if !st.GetSlurmdbdRunning() {
|
|
t.Fatalf("status %v", st)
|
|
}
|
|
sum := st.GetFilesSha256()
|
|
|
|
h.ran = nil
|
|
k.Reconcile(context.Background(), m)
|
|
if h.count("systemctl restart") != 0 {
|
|
t.Fatalf("restarted on the same files: %v", h.ran)
|
|
}
|
|
|
|
files["gres.conf"] = "NodeName=cuda01 Name=gpu File=/dev/nvidia0\n"
|
|
h.ran = nil
|
|
st = k.Reconcile(context.Background(), m)
|
|
if h.count("systemctl restart slurmctld") != 1 || st.GetFilesSha256() == sum {
|
|
t.Fatalf("a changed file did not restart slurmctld: %v", h.ran)
|
|
}
|
|
|
|
if st := k.Reconcile(context.Background(), &llmgwv1.Slurm{Controller: true, MungeKey: key,
|
|
Files: map[string]string{"slurm.conf": "x", "../evil": "y"}}); !strings.Contains(st.GetError(), "named") {
|
|
t.Fatalf("a path in a file name was taken: %v", st)
|
|
}
|
|
}
|
|
|
|
func TestTooOld(t *testing.T) {
|
|
h := newFake()
|
|
h.outputs["slurmd -V"] = "slurm-wlm 21.08.5\n"
|
|
k := New(h, &fakeInst{host: h}, "/m", nil)
|
|
st := k.Reconcile(context.Background(), nodeMsg(false))
|
|
if !strings.Contains(st.GetError(), "older than 22.05") {
|
|
t.Fatalf("status %v", st)
|
|
}
|
|
if _, ok := h.files[slurm.MungeKeyPath]; ok {
|
|
t.Fatal("went on with a Slurm too old")
|
|
}
|
|
}
|
|
|
|
func TestStop(t *testing.T) {
|
|
h := newFake()
|
|
k := New(h, &fakeInst{host: h}, "/m", nil)
|
|
k.Reconcile(context.Background(), nodeMsg(false))
|
|
h.ran = nil
|
|
if st := k.Reconcile(context.Background(), &llmgwv1.Slurm{}); st != nil {
|
|
t.Fatalf("status %v for no part", st)
|
|
}
|
|
if !slices.Contains(h.ran, "systemctl disable --now slurmd") {
|
|
t.Fatalf("slurmd not stopped: %v", h.ran)
|
|
}
|
|
if _, ok := h.files[DropIn]; ok {
|
|
t.Fatal("drop-in left")
|
|
}
|
|
// Never started here: nothing touched.
|
|
h2 := newFake()
|
|
k2 := New(h2, &fakeInst{host: h2}, "/m", nil)
|
|
k2.Reconcile(context.Background(), &llmgwv1.Slurm{})
|
|
if len(h2.ran) != 0 {
|
|
t.Fatalf("a machine llmgw never ran Slurm on was touched: %v", h2.ran)
|
|
}
|
|
}
|
|
|
|
func TestBadNames(t *testing.T) {
|
|
h := newFake()
|
|
k := New(h, &fakeInst{host: h}, "/m", nil)
|
|
m := nodeMsg(false)
|
|
m.Name = "cuda01 -c /etc/evil"
|
|
if st := k.Reconcile(context.Background(), m); !strings.Contains(st.GetError(), "spaces") {
|
|
t.Fatalf("status %v", st)
|
|
}
|
|
if st := k.Reconcile(context.Background(), &llmgwv1.Slurm{Node: true, Name: "a", ConfServer: "b"}); !strings.Contains(st.GetError(), "no key") {
|
|
t.Fatalf("status %v", st)
|
|
}
|
|
}
|
|
|
|
func TestAMariaDBThatWasThereKeepsItsSettings(t *testing.T) {
|
|
h := newFake()
|
|
for _, b := range []string{"munged", "scontrol", "slurmctld", "slurmdbd", "mariadbd", "mariadb", "exportfs", "mount.nfs"} {
|
|
h.bins[b] = true
|
|
}
|
|
k := New(h, &fakeInst{host: h}, "/m", nil)
|
|
st := k.Reconcile(context.Background(), &llmgwv1.Slurm{Controller: true, MungeKey: key,
|
|
Files: map[string]string{"slurm.conf": "ClusterName=llmgw\n"}})
|
|
if st.GetError() != "" {
|
|
t.Fatal(st.GetError())
|
|
}
|
|
for p := range h.files {
|
|
if strings.HasSuffix(p, ".cnf") {
|
|
t.Fatalf("a MariaDB llmgw did not install had its settings changed: %s", p)
|
|
}
|
|
}
|
|
if h.count("mariadb -e source") != 1 || !st.GetSlurmdbdRunning() {
|
|
t.Fatalf("the database was not made in the MariaDB that was there: %v", h.ran)
|
|
}
|
|
}
|