package slurmnode import ( "context" "errors" "io/fs" "path/filepath" "slices" "strings" "testing" llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1" "gitea.mynodes.xyz/ilker/llmgw/internal/slurm" ) type fakeHost struct { files map[string][]byte owners map[string]string bins map[string]bool active map[string]bool ran []string outputs map[string]string fail map[string]bool state string reason string } func newFake() *fakeHost { return &fakeHost{files: map[string][]byte{}, owners: map[string]string{}, bins: map[string]bool{}, active: map[string]bool{}, outputs: map[string]string{ "slurmd -V": "slurm-wlm 23.11.4\n", "slurmctld -V": "slurm-wlm 23.11.4\n", "scontrol ping": "Slurmctld(primary) at node06 is UP\n", "slurmd -C": "NodeName=cuda01 CPUs=16 RealMemory=64000\nUpTime=1-00:00:00\n", }, fail: map[string]bool{}, state: "IDLE+DRAIN", reason: slurm.ReasonNew} } func (f *fakeHost) Run(_ context.Context, env []string, name string, args ...string) ([]byte, error) { c := strings.Join(append([]string{name}, args...), " ") f.ran = append(f.ran, c) if f.fail[c] { return []byte("it failed"), errors.New("exit status 1") } switch { case name == "systemctl" && len(args) >= 1: u := args[len(args)-1] switch args[0] { case "is-active": if f.active[u] { return []byte("active\n"), nil } return []byte("inactive\n"), errors.New("exit status 3") case "restart", "start": f.active[u] = true case "enable": if slices.Contains(args, "--now") { f.active[u] = true } case "disable": f.active[u] = false } return nil, nil case strings.HasPrefix(c, "scontrol show node"): return []byte("NodeName=cuda01 Arch=x86_64 State=" + f.state + " Partitions=gpu Reason=" + f.reason + " [root@now]\n"), nil case strings.HasPrefix(c, "scontrol update"): if slices.Contains(args, "State=RESUME") { f.state, f.reason = "IDLE", "" } else { f.state, f.reason = "IDLE+DRAIN", ReasonServing } return nil, nil case strings.HasPrefix(c, "pgrep"): out := "1800358 /usr/sbin/slurmstepd infinity\n" + f.outputs["jobs"] return []byte(out), nil } return []byte(f.outputs[c]), nil } func (f *fakeHost) LookPath(name string) (string, error) { if f.bins[name] { return "/usr/sbin/" + name, nil } return "", errors.New("not found") } func (f *fakeHost) ReadFile(path string) ([]byte, error) { b, ok := f.files[path] if !ok { return nil, fs.ErrNotExist } return b, nil } func (f *fakeHost) WriteFile(path string, b []byte, _ fs.FileMode, owner string) error { f.files[path] = append([]byte(nil), b...) f.owners[path] = owner return nil } func (f *fakeHost) MkdirAll(string, fs.FileMode, string) error { return nil } func (f *fakeHost) Remove(path string) error { delete(f.files, path); return nil } func (f *fakeHost) count(prefix string) int { n := 0 for _, c := range f.ran { if strings.HasPrefix(c, prefix) { n++ } } return n } type fakeInst struct { host *fakeHost got map[string][]string } func (i *fakeInst) InstallPackages(_ context.Context, by map[string][]string) error { i.got = by for _, b := range []string{"munged", "scontrol", "slurmd", "slurmctld"} { i.host.bins[b] = true } return nil } var key = []byte(strings.Repeat("k", 1024)) func nodeMsg(lent bool) *llmgwv1.Slurm { return &llmgwv1.Slurm{Node: true, Name: "cuda01", ConfServer: "10.0.0.146:6817", MungeKey: key, Lent: lent} } func TestNodeInstallsAndStaysDrained(t *testing.T) { h := newFake() inst := &fakeInst{host: h} k := New(h, inst, "/var/lib/llmgw-agent/slurm", nil) st := k.Reconcile(context.Background(), nodeMsg(false)) if st.GetError() != "" { t.Fatal(st.GetError()) } if !slices.Equal(inst.got["deb"], []string{"munge", "slurm-client", "slurmd"}) || !slices.Equal(inst.got["rpm"], []string{"munge", "slurm", "slurm-slurmd"}) { t.Fatalf("installed %v", inst.got) } if string(h.files[slurm.MungeKeyPath]) != string(key) || h.owners[slurm.MungeKeyPath] != "munge" { t.Fatal("the munge key was not written for munge") } unit := string(h.files[DropIn]) if !strings.Contains(unit, "ExecStart=/usr/sbin/slurmd --systemd --conf-server 10.0.0.146:6817 -N cuda01\n") || !strings.Contains(unit, "ConditionPathExists=\n") { t.Fatalf("drop-in:\n%s", unit) } if !st.GetSlurmdRunning() || st.GetVersion() != "23.11.4" || !strings.HasPrefix(st.GetHardware(), "NodeName=cuda01 CPUs=16") { t.Fatalf("status %v", st) } // Not lent: left drained, with llmgw's first reason. if h.count("scontrol update") != 0 || st.GetState() != "IDLE+DRAIN" { t.Fatalf("a node not lent was changed: %v, %v", h.ran, st) } if _, ok := h.files["/var/lib/llmgw-agent/slurm"]; !ok { t.Fatal("no marker") } // Again, nothing changed: no install, no restart. h.ran = nil inst.got = nil k.Reconcile(context.Background(), nodeMsg(false)) if inst.got != nil || h.count("systemctl restart") != 0 || h.count("systemctl enable") != 0 { t.Fatalf("a second pass did work: %v", h.ran) } } func TestLendAndReclaim(t *testing.T) { h := newFake() k := New(h, &fakeInst{host: h}, "/m", nil) st := k.Reconcile(context.Background(), nodeMsg(true)) if h.count("scontrol update NodeName=cuda01 State=RESUME") != 1 || st.GetState() != "IDLE" { t.Fatalf("a lent node was not resumed: %v %v", h.ran, st) } h.outputs["jobs"] = "2001 slurmstepd: [101.batch]\n2002 slurmstepd: [102.0]\n" st = k.Reconcile(context.Background(), nodeMsg(false)) if h.count("scontrol update NodeName=cuda01 State=DRAIN Reason="+ReasonServing) != 1 { t.Fatalf("a node taken back was not drained: %v", h.ran) } if st.GetJobs() != 2 || !strings.Contains(st.GetState(), "DRAIN") { t.Fatalf("status %v", st) } // Drained already: not drained again. h.ran = nil k.Reconcile(context.Background(), nodeMsg(false)) if h.count("scontrol update") != 0 { t.Fatalf("drained twice: %v", h.ran) } } func TestAdminsDrainIsLeft(t *testing.T) { h := newFake() h.reason = "bad DIMM, ticket 42" k := New(h, &fakeInst{host: h}, "/m", nil) k.Reconcile(context.Background(), nodeMsg(true)) if h.count("scontrol update") != 0 { t.Fatalf("llmgw resumed a node a Slurm administrator drained: %v", h.ran) } } func TestClientEnvOnANodeWithoutFiles(t *testing.T) { h := newFake() k := New(h, &fakeInst{host: h}, "/m", nil) if env := k.clientEnv(); len(env) != 1 || !strings.HasSuffix(env[0], "conf-cache/slurm.conf") { t.Fatalf("env %v", env) } h.files[filepath.Join(slurm.ConfDir, "slurm.conf")] = []byte("x") if env := k.clientEnv(); env != nil { t.Fatalf("env %v on the controller", env) } } func TestController(t *testing.T) { h := newFake() inst := &fakeInst{host: h} k := New(h, inst, "/m", nil) files := map[string]string{"slurm.conf": "ClusterName=llmgw\n", "gres.conf": "#\n", "cgroup.conf": "#\n"} m := &llmgwv1.Slurm{Controller: true, MungeKey: key, Files: files} st := k.Reconcile(context.Background(), m) if st.GetError() != "" { t.Fatal(st.GetError()) } if !slices.Contains(inst.got["deb"], "slurmctld") || slices.Contains(inst.got["deb"], "slurmd") { t.Fatalf("installed %v", inst.got) } if string(h.files["/etc/slurm/slurm.conf"]) != "ClusterName=llmgw\n" { t.Fatal("slurm.conf not written") } if h.count("systemctl restart slurmctld") != 1 || h.count("scontrol reconfigure") != 1 { t.Fatalf("slurmctld not started on its files: %v", h.ran) } if !st.GetSlurmctldRunning() || len(st.GetFilesSha256()) != 64 { t.Fatalf("status %v", st) } sum := st.GetFilesSha256() h.ran = nil k.Reconcile(context.Background(), m) if h.count("systemctl restart") != 0 { t.Fatalf("restarted on the same files: %v", h.ran) } files["gres.conf"] = "NodeName=cuda01 Name=gpu File=/dev/nvidia0\n" h.ran = nil st = k.Reconcile(context.Background(), m) if h.count("systemctl restart slurmctld") != 1 || st.GetFilesSha256() == sum { t.Fatalf("a changed file did not restart slurmctld: %v", h.ran) } if st := k.Reconcile(context.Background(), &llmgwv1.Slurm{Controller: true, MungeKey: key, Files: map[string]string{"slurm.conf": "x", "../evil": "y"}}); !strings.Contains(st.GetError(), "named") { t.Fatalf("a path in a file name was taken: %v", st) } } func TestTooOld(t *testing.T) { h := newFake() h.outputs["slurmd -V"] = "slurm-wlm 21.08.5\n" k := New(h, &fakeInst{host: h}, "/m", nil) st := k.Reconcile(context.Background(), nodeMsg(false)) if !strings.Contains(st.GetError(), "older than 22.05") { t.Fatalf("status %v", st) } if _, ok := h.files[slurm.MungeKeyPath]; ok { t.Fatal("went on with a Slurm too old") } } func TestStop(t *testing.T) { h := newFake() k := New(h, &fakeInst{host: h}, "/m", nil) k.Reconcile(context.Background(), nodeMsg(false)) h.ran = nil if st := k.Reconcile(context.Background(), &llmgwv1.Slurm{}); st != nil { t.Fatalf("status %v for no part", st) } if h.count("systemctl disable --now slurmd") != 1 { t.Fatalf("slurmd not stopped: %v", h.ran) } if _, ok := h.files[DropIn]; ok { t.Fatal("drop-in left") } // Never started here: nothing touched. h2 := newFake() k2 := New(h2, &fakeInst{host: h2}, "/m", nil) k2.Reconcile(context.Background(), &llmgwv1.Slurm{}) if len(h2.ran) != 0 { t.Fatalf("a machine llmgw never ran Slurm on was touched: %v", h2.ran) } } func TestBadNames(t *testing.T) { h := newFake() k := New(h, &fakeInst{host: h}, "/m", nil) m := nodeMsg(false) m.Name = "cuda01 -c /etc/evil" if st := k.Reconcile(context.Background(), m); !strings.Contains(st.GetError(), "spaces") { t.Fatalf("status %v", st) } if st := k.Reconcile(context.Background(), &llmgwv1.Slurm{Node: true, Name: "a", ConfServer: "b"}); !strings.Contains(st.GetError(), "no key") { t.Fatalf("status %v", st) } }