The InvalidAccount seen on the PoC was not the reconfigure: 23.11.4 without accounting storage has its main scheduler mark a batch job so, and backfill starts it on its next pass (job 4 waited 15 s). bf_interval=10 keeps that short; the extra restart is taken out again. MailProg=/bin/true quiets the warning on every start. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
309 lines
9.5 KiB
Go
309 lines
9.5 KiB
Go
package slurmnode
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"io/fs"
|
|
"path/filepath"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
|
|
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
|
|
)
|
|
|
|
type fakeHost struct {
|
|
files map[string][]byte
|
|
owners map[string]string
|
|
bins map[string]bool
|
|
active map[string]bool
|
|
ran []string
|
|
outputs map[string]string
|
|
fail map[string]bool
|
|
state string
|
|
reason string
|
|
}
|
|
|
|
func newFake() *fakeHost {
|
|
return &fakeHost{files: map[string][]byte{}, owners: map[string]string{}, bins: map[string]bool{},
|
|
active: map[string]bool{}, outputs: map[string]string{
|
|
"slurmd -V": "slurm-wlm 23.11.4\n",
|
|
"slurmctld -V": "slurm-wlm 23.11.4\n",
|
|
"scontrol ping": "Slurmctld(primary) at node06 is UP\n",
|
|
"slurmd -C": "NodeName=cuda01 CPUs=16 RealMemory=64000\nUpTime=1-00:00:00\n",
|
|
}, fail: map[string]bool{}, state: "IDLE+DRAIN", reason: slurm.ReasonNew}
|
|
}
|
|
|
|
func (f *fakeHost) Run(_ context.Context, env []string, name string, args ...string) ([]byte, error) {
|
|
c := strings.Join(append([]string{name}, args...), " ")
|
|
f.ran = append(f.ran, c)
|
|
if f.fail[c] {
|
|
return []byte("it failed"), errors.New("exit status 1")
|
|
}
|
|
switch {
|
|
case name == "systemctl" && len(args) >= 1:
|
|
u := args[len(args)-1]
|
|
switch args[0] {
|
|
case "is-active":
|
|
if f.active[u] {
|
|
return []byte("active\n"), nil
|
|
}
|
|
return []byte("inactive\n"), errors.New("exit status 3")
|
|
case "restart", "start":
|
|
f.active[u] = true
|
|
case "enable":
|
|
if slices.Contains(args, "--now") {
|
|
f.active[u] = true
|
|
}
|
|
case "disable":
|
|
f.active[u] = false
|
|
}
|
|
return nil, nil
|
|
case strings.HasPrefix(c, "scontrol show node"):
|
|
return []byte("NodeName=cuda01 Arch=x86_64 State=" + f.state + " Partitions=gpu Reason=" + f.reason + " [root@now]\n"), nil
|
|
case strings.HasPrefix(c, "scontrol update"):
|
|
if slices.Contains(args, "State=RESUME") {
|
|
f.state, f.reason = "IDLE", ""
|
|
} else {
|
|
f.state, f.reason = "IDLE+DRAIN", ReasonServing
|
|
}
|
|
return nil, nil
|
|
case strings.HasPrefix(c, "pgrep"):
|
|
out := "1800358 /usr/sbin/slurmstepd infinity\n" + f.outputs["jobs"]
|
|
return []byte(out), nil
|
|
}
|
|
return []byte(f.outputs[c]), nil
|
|
}
|
|
|
|
func (f *fakeHost) LookPath(name string) (string, error) {
|
|
if f.bins[name] {
|
|
return "/usr/sbin/" + name, nil
|
|
}
|
|
return "", errors.New("not found")
|
|
}
|
|
|
|
func (f *fakeHost) ReadFile(path string) ([]byte, error) {
|
|
b, ok := f.files[path]
|
|
if !ok {
|
|
return nil, fs.ErrNotExist
|
|
}
|
|
return b, nil
|
|
}
|
|
|
|
func (f *fakeHost) WriteFile(path string, b []byte, _ fs.FileMode, owner string) error {
|
|
f.files[path] = append([]byte(nil), b...)
|
|
f.owners[path] = owner
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeHost) MkdirAll(string, fs.FileMode, string) error { return nil }
|
|
|
|
func (f *fakeHost) Remove(path string) error { delete(f.files, path); return nil }
|
|
|
|
func (f *fakeHost) count(prefix string) int {
|
|
n := 0
|
|
for _, c := range f.ran {
|
|
if strings.HasPrefix(c, prefix) {
|
|
n++
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
|
|
type fakeInst struct {
|
|
host *fakeHost
|
|
got map[string][]string
|
|
}
|
|
|
|
func (i *fakeInst) InstallPackages(_ context.Context, by map[string][]string) error {
|
|
i.got = by
|
|
for _, b := range []string{"munged", "scontrol", "slurmd", "slurmctld"} {
|
|
i.host.bins[b] = true
|
|
}
|
|
return nil
|
|
}
|
|
|
|
var key = []byte(strings.Repeat("k", 1024))
|
|
|
|
func nodeMsg(lent bool) *llmgwv1.Slurm {
|
|
return &llmgwv1.Slurm{Node: true, Name: "cuda01", ConfServer: "10.0.0.146:6817", MungeKey: key, Lent: lent}
|
|
}
|
|
|
|
func TestNodeInstallsAndStaysDrained(t *testing.T) {
|
|
h := newFake()
|
|
inst := &fakeInst{host: h}
|
|
k := New(h, inst, "/var/lib/llmgw-agent/slurm", nil)
|
|
st := k.Reconcile(context.Background(), nodeMsg(false))
|
|
if st.GetError() != "" {
|
|
t.Fatal(st.GetError())
|
|
}
|
|
if !slices.Equal(inst.got["deb"], []string{"munge", "slurm-client", "slurmd"}) ||
|
|
!slices.Equal(inst.got["rpm"], []string{"munge", "slurm", "slurm-slurmd"}) {
|
|
t.Fatalf("installed %v", inst.got)
|
|
}
|
|
if string(h.files[slurm.MungeKeyPath]) != string(key) || h.owners[slurm.MungeKeyPath] != "munge" {
|
|
t.Fatal("the munge key was not written for munge")
|
|
}
|
|
unit := string(h.files[DropIn])
|
|
if !strings.Contains(unit, "ExecStart=/usr/sbin/slurmd --systemd --conf-server 10.0.0.146:6817 -N cuda01\n") ||
|
|
!strings.Contains(unit, "ConditionPathExists=\n") {
|
|
t.Fatalf("drop-in:\n%s", unit)
|
|
}
|
|
if !st.GetSlurmdRunning() || st.GetVersion() != "23.11.4" || !strings.HasPrefix(st.GetHardware(), "NodeName=cuda01 CPUs=16") {
|
|
t.Fatalf("status %v", st)
|
|
}
|
|
// Not lent: left drained, with llmgw's first reason.
|
|
if h.count("scontrol update") != 0 || st.GetState() != "IDLE+DRAIN" {
|
|
t.Fatalf("a node not lent was changed: %v, %v", h.ran, st)
|
|
}
|
|
if _, ok := h.files["/var/lib/llmgw-agent/slurm"]; !ok {
|
|
t.Fatal("no marker")
|
|
}
|
|
|
|
// Again, nothing changed: no install, no restart.
|
|
h.ran = nil
|
|
inst.got = nil
|
|
k.Reconcile(context.Background(), nodeMsg(false))
|
|
if inst.got != nil || h.count("systemctl restart") != 0 || h.count("systemctl enable") != 0 {
|
|
t.Fatalf("a second pass did work: %v", h.ran)
|
|
}
|
|
}
|
|
|
|
func TestLendAndReclaim(t *testing.T) {
|
|
h := newFake()
|
|
k := New(h, &fakeInst{host: h}, "/m", nil)
|
|
st := k.Reconcile(context.Background(), nodeMsg(true))
|
|
if h.count("scontrol update NodeName=cuda01 State=RESUME") != 1 || st.GetState() != "IDLE" {
|
|
t.Fatalf("a lent node was not resumed: %v %v", h.ran, st)
|
|
}
|
|
h.outputs["jobs"] = "2001 slurmstepd: [101.batch]\n2002 slurmstepd: [102.0]\n"
|
|
st = k.Reconcile(context.Background(), nodeMsg(false))
|
|
if h.count("scontrol update NodeName=cuda01 State=DRAIN Reason="+ReasonServing) != 1 {
|
|
t.Fatalf("a node taken back was not drained: %v", h.ran)
|
|
}
|
|
if st.GetJobs() != 2 || !strings.Contains(st.GetState(), "DRAIN") {
|
|
t.Fatalf("status %v", st)
|
|
}
|
|
// Drained already: not drained again.
|
|
h.ran = nil
|
|
k.Reconcile(context.Background(), nodeMsg(false))
|
|
if h.count("scontrol update") != 0 {
|
|
t.Fatalf("drained twice: %v", h.ran)
|
|
}
|
|
}
|
|
|
|
func TestAdminsDrainIsLeft(t *testing.T) {
|
|
h := newFake()
|
|
h.reason = "bad DIMM, ticket 42"
|
|
k := New(h, &fakeInst{host: h}, "/m", nil)
|
|
k.Reconcile(context.Background(), nodeMsg(true))
|
|
if h.count("scontrol update") != 0 {
|
|
t.Fatalf("llmgw resumed a node a Slurm administrator drained: %v", h.ran)
|
|
}
|
|
}
|
|
|
|
func TestClientEnvOnANodeWithoutFiles(t *testing.T) {
|
|
h := newFake()
|
|
k := New(h, &fakeInst{host: h}, "/m", nil)
|
|
if env := k.clientEnv(); len(env) != 1 || !strings.HasSuffix(env[0], "conf-cache/slurm.conf") {
|
|
t.Fatalf("env %v", env)
|
|
}
|
|
h.files[filepath.Join(slurm.ConfDir, "slurm.conf")] = []byte("x")
|
|
if env := k.clientEnv(); env != nil {
|
|
t.Fatalf("env %v on the controller", env)
|
|
}
|
|
}
|
|
|
|
func TestController(t *testing.T) {
|
|
h := newFake()
|
|
inst := &fakeInst{host: h}
|
|
k := New(h, inst, "/m", nil)
|
|
files := map[string]string{"slurm.conf": "ClusterName=llmgw\n", "gres.conf": "#\n", "cgroup.conf": "#\n"}
|
|
m := &llmgwv1.Slurm{Controller: true, MungeKey: key, Files: files}
|
|
st := k.Reconcile(context.Background(), m)
|
|
if st.GetError() != "" {
|
|
t.Fatal(st.GetError())
|
|
}
|
|
if !slices.Contains(inst.got["deb"], "slurmctld") || slices.Contains(inst.got["deb"], "slurmd") {
|
|
t.Fatalf("installed %v", inst.got)
|
|
}
|
|
if string(h.files["/etc/slurm/slurm.conf"]) != "ClusterName=llmgw\n" {
|
|
t.Fatal("slurm.conf not written")
|
|
}
|
|
if h.count("systemctl restart slurmctld") != 1 || h.count("scontrol reconfigure") != 1 {
|
|
t.Fatalf("slurmctld not started on its files: %v", h.ran)
|
|
}
|
|
if !st.GetSlurmctldRunning() || len(st.GetFilesSha256()) != 64 {
|
|
t.Fatalf("status %v", st)
|
|
}
|
|
sum := st.GetFilesSha256()
|
|
|
|
h.ran = nil
|
|
k.Reconcile(context.Background(), m)
|
|
if h.count("systemctl restart") != 0 {
|
|
t.Fatalf("restarted on the same files: %v", h.ran)
|
|
}
|
|
|
|
files["gres.conf"] = "NodeName=cuda01 Name=gpu File=/dev/nvidia0\n"
|
|
h.ran = nil
|
|
st = k.Reconcile(context.Background(), m)
|
|
if h.count("systemctl restart slurmctld") != 1 || st.GetFilesSha256() == sum {
|
|
t.Fatalf("a changed file did not restart slurmctld: %v", h.ran)
|
|
}
|
|
|
|
if st := k.Reconcile(context.Background(), &llmgwv1.Slurm{Controller: true, MungeKey: key,
|
|
Files: map[string]string{"slurm.conf": "x", "../evil": "y"}}); !strings.Contains(st.GetError(), "named") {
|
|
t.Fatalf("a path in a file name was taken: %v", st)
|
|
}
|
|
}
|
|
|
|
func TestTooOld(t *testing.T) {
|
|
h := newFake()
|
|
h.outputs["slurmd -V"] = "slurm-wlm 21.08.5\n"
|
|
k := New(h, &fakeInst{host: h}, "/m", nil)
|
|
st := k.Reconcile(context.Background(), nodeMsg(false))
|
|
if !strings.Contains(st.GetError(), "older than 22.05") {
|
|
t.Fatalf("status %v", st)
|
|
}
|
|
if _, ok := h.files[slurm.MungeKeyPath]; ok {
|
|
t.Fatal("went on with a Slurm too old")
|
|
}
|
|
}
|
|
|
|
func TestStop(t *testing.T) {
|
|
h := newFake()
|
|
k := New(h, &fakeInst{host: h}, "/m", nil)
|
|
k.Reconcile(context.Background(), nodeMsg(false))
|
|
h.ran = nil
|
|
if st := k.Reconcile(context.Background(), &llmgwv1.Slurm{}); st != nil {
|
|
t.Fatalf("status %v for no part", st)
|
|
}
|
|
if h.count("systemctl disable --now slurmd") != 1 {
|
|
t.Fatalf("slurmd not stopped: %v", h.ran)
|
|
}
|
|
if _, ok := h.files[DropIn]; ok {
|
|
t.Fatal("drop-in left")
|
|
}
|
|
// Never started here: nothing touched.
|
|
h2 := newFake()
|
|
k2 := New(h2, &fakeInst{host: h2}, "/m", nil)
|
|
k2.Reconcile(context.Background(), &llmgwv1.Slurm{})
|
|
if len(h2.ran) != 0 {
|
|
t.Fatalf("a machine llmgw never ran Slurm on was touched: %v", h2.ran)
|
|
}
|
|
}
|
|
|
|
func TestBadNames(t *testing.T) {
|
|
h := newFake()
|
|
k := New(h, &fakeInst{host: h}, "/m", nil)
|
|
m := nodeMsg(false)
|
|
m.Name = "cuda01 -c /etc/evil"
|
|
if st := k.Reconcile(context.Background(), m); !strings.Contains(st.GetError(), "spaces") {
|
|
t.Fatalf("status %v", st)
|
|
}
|
|
if st := k.Reconcile(context.Background(), &llmgwv1.Slurm{Node: true, Name: "a", ConfServer: "b"}); !strings.Contains(st.GetError(), "no key") {
|
|
t.Fatalf("status %v", st)
|
|
}
|
|
}
|