CI / test (push) Successful in 10m57s
Since 23.11 Slurm runs badly without accounting storage: the main scheduler passes every job by as InvalidAccount. slurmdbd takes MySQL or MariaDB only, so the controller's agent installs MariaDB beside it, answering that machine alone, makes the database and its user (the password made from the munge key, never on a command line), runs slurmdbd and adds the cluster, all before slurmctld. A MariaDB that was on the machine before keeps its settings. slurm.conf accounts the jobs there, with what each used from its cgroup and the cards, and the priority is multifactor again. Port 6819 joins Slurm's in the firewall. ctl slurm status and the console say whether slurmdbd runs; the install screen, server init --slurm and the guide say what is put where. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
358 lines
12 KiB
Go
358 lines
12 KiB
Go
package slurmnode
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"io/fs"
|
|
"path/filepath"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
|
|
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
|
|
)
|
|
|
|
type fakeHost struct {
|
|
files map[string][]byte
|
|
owners map[string]string
|
|
bins map[string]bool
|
|
active map[string]bool
|
|
ran []string
|
|
outputs map[string]string
|
|
fail map[string]bool
|
|
state string
|
|
reason string
|
|
}
|
|
|
|
func newFake() *fakeHost {
|
|
return &fakeHost{files: map[string][]byte{}, owners: map[string]string{}, bins: map[string]bool{},
|
|
active: map[string]bool{}, outputs: map[string]string{
|
|
"slurmd -V": "slurm-wlm 23.11.4\n",
|
|
"slurmctld -V": "slurm-wlm 23.11.4\n",
|
|
"scontrol ping": "Slurmctld(primary) at node06 is UP\n",
|
|
"slurmd -C": "NodeName=cuda01 CPUs=16 RealMemory=64000\nUpTime=1-00:00:00\n",
|
|
"hostname -s": "node06\n",
|
|
}, fail: map[string]bool{}, state: "IDLE+DRAIN", reason: slurm.ReasonNew}
|
|
}
|
|
|
|
func (f *fakeHost) Run(_ context.Context, env []string, name string, args ...string) ([]byte, error) {
|
|
c := strings.Join(append([]string{name}, args...), " ")
|
|
f.ran = append(f.ran, c)
|
|
if f.fail[c] {
|
|
return []byte("it failed"), errors.New("exit status 1")
|
|
}
|
|
switch {
|
|
case name == "systemctl" && len(args) >= 1:
|
|
u := args[len(args)-1]
|
|
switch args[0] {
|
|
case "is-active":
|
|
if f.active[u] {
|
|
return []byte("active\n"), nil
|
|
}
|
|
return []byte("inactive\n"), errors.New("exit status 3")
|
|
case "restart", "start":
|
|
f.active[u] = true
|
|
case "enable":
|
|
if slices.Contains(args, "--now") {
|
|
f.active[u] = true
|
|
}
|
|
case "disable":
|
|
f.active[u] = false
|
|
}
|
|
return nil, nil
|
|
case strings.HasPrefix(c, "scontrol show node"):
|
|
return []byte("NodeName=cuda01 Arch=x86_64 State=" + f.state + " Partitions=gpu Reason=" + f.reason + " [root@now]\n"), nil
|
|
case strings.HasPrefix(c, "scontrol update"):
|
|
if slices.Contains(args, "State=RESUME") {
|
|
f.state, f.reason = "IDLE", ""
|
|
} else {
|
|
f.state, f.reason = "IDLE+DRAIN", ReasonServing
|
|
}
|
|
return nil, nil
|
|
case strings.HasPrefix(c, "pgrep"):
|
|
out := "1800358 /usr/sbin/slurmstepd infinity\n" + f.outputs["jobs"]
|
|
return []byte(out), nil
|
|
}
|
|
return []byte(f.outputs[c]), nil
|
|
}
|
|
|
|
func (f *fakeHost) LookPath(name string) (string, error) {
|
|
if f.bins[name] {
|
|
return "/usr/sbin/" + name, nil
|
|
}
|
|
return "", errors.New("not found")
|
|
}
|
|
|
|
func (f *fakeHost) ReadFile(path string) ([]byte, error) {
|
|
b, ok := f.files[path]
|
|
if !ok {
|
|
return nil, fs.ErrNotExist
|
|
}
|
|
return b, nil
|
|
}
|
|
|
|
func (f *fakeHost) WriteFile(path string, b []byte, _ fs.FileMode, owner string) error {
|
|
f.files[path] = append([]byte(nil), b...)
|
|
f.owners[path] = owner
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeHost) MkdirAll(string, fs.FileMode, string) error { return nil }
|
|
|
|
func (f *fakeHost) Remove(path string) error { delete(f.files, path); return nil }
|
|
|
|
func (f *fakeHost) count(prefix string) int {
|
|
n := 0
|
|
for _, c := range f.ran {
|
|
if strings.HasPrefix(c, prefix) {
|
|
n++
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
|
|
type fakeInst struct {
|
|
host *fakeHost
|
|
got map[string][]string
|
|
}
|
|
|
|
func (i *fakeInst) InstallPackages(_ context.Context, by map[string][]string) error {
|
|
i.got = by
|
|
for _, b := range []string{"munged", "scontrol", "slurmd", "slurmctld", "slurmdbd", "mariadbd", "mariadb"} {
|
|
i.host.bins[b] = true
|
|
}
|
|
return nil
|
|
}
|
|
|
|
var key = []byte(strings.Repeat("k", 1024))
|
|
|
|
func nodeMsg(lent bool) *llmgwv1.Slurm {
|
|
return &llmgwv1.Slurm{Node: true, Name: "cuda01", ConfServer: "10.0.0.146:6817", MungeKey: key, Lent: lent}
|
|
}
|
|
|
|
func TestNodeInstallsAndStaysDrained(t *testing.T) {
|
|
h := newFake()
|
|
inst := &fakeInst{host: h}
|
|
k := New(h, inst, "/var/lib/llmgw-agent/slurm", nil)
|
|
st := k.Reconcile(context.Background(), nodeMsg(false))
|
|
if st.GetError() != "" {
|
|
t.Fatal(st.GetError())
|
|
}
|
|
if !slices.Equal(inst.got["deb"], []string{"munge", "slurm-client", "slurmd"}) ||
|
|
!slices.Equal(inst.got["rpm"], []string{"munge", "slurm", "slurm-slurmd"}) {
|
|
t.Fatalf("installed %v", inst.got)
|
|
}
|
|
if string(h.files[slurm.MungeKeyPath]) != string(key) || h.owners[slurm.MungeKeyPath] != "munge" {
|
|
t.Fatal("the munge key was not written for munge")
|
|
}
|
|
unit := string(h.files[DropIn])
|
|
if !strings.Contains(unit, "ExecStart=/usr/sbin/slurmd --systemd --conf-server 10.0.0.146:6817 -N cuda01\n") ||
|
|
!strings.Contains(unit, "ConditionPathExists=\n") {
|
|
t.Fatalf("drop-in:\n%s", unit)
|
|
}
|
|
if !st.GetSlurmdRunning() || st.GetVersion() != "23.11.4" || !strings.HasPrefix(st.GetHardware(), "NodeName=cuda01 CPUs=16") {
|
|
t.Fatalf("status %v", st)
|
|
}
|
|
// Not lent: left drained, with llmgw's first reason.
|
|
if h.count("scontrol update") != 0 || st.GetState() != "IDLE+DRAIN" {
|
|
t.Fatalf("a node not lent was changed: %v, %v", h.ran, st)
|
|
}
|
|
if _, ok := h.files["/var/lib/llmgw-agent/slurm"]; !ok {
|
|
t.Fatal("no marker")
|
|
}
|
|
|
|
// Again, nothing changed: no install, no restart.
|
|
h.ran = nil
|
|
inst.got = nil
|
|
k.Reconcile(context.Background(), nodeMsg(false))
|
|
if inst.got != nil || h.count("systemctl restart") != 0 || h.count("systemctl enable") != 0 {
|
|
t.Fatalf("a second pass did work: %v", h.ran)
|
|
}
|
|
}
|
|
|
|
func TestLendAndReclaim(t *testing.T) {
|
|
h := newFake()
|
|
k := New(h, &fakeInst{host: h}, "/m", nil)
|
|
st := k.Reconcile(context.Background(), nodeMsg(true))
|
|
if h.count("scontrol update NodeName=cuda01 State=RESUME") != 1 || st.GetState() != "IDLE" {
|
|
t.Fatalf("a lent node was not resumed: %v %v", h.ran, st)
|
|
}
|
|
h.outputs["jobs"] = "2001 slurmstepd: [101.batch]\n2002 slurmstepd: [102.0]\n"
|
|
st = k.Reconcile(context.Background(), nodeMsg(false))
|
|
if h.count("scontrol update NodeName=cuda01 State=DRAIN Reason="+ReasonServing) != 1 {
|
|
t.Fatalf("a node taken back was not drained: %v", h.ran)
|
|
}
|
|
if st.GetJobs() != 2 || !strings.Contains(st.GetState(), "DRAIN") {
|
|
t.Fatalf("status %v", st)
|
|
}
|
|
// Drained already: not drained again.
|
|
h.ran = nil
|
|
k.Reconcile(context.Background(), nodeMsg(false))
|
|
if h.count("scontrol update") != 0 {
|
|
t.Fatalf("drained twice: %v", h.ran)
|
|
}
|
|
}
|
|
|
|
func TestAdminsDrainIsLeft(t *testing.T) {
|
|
h := newFake()
|
|
h.reason = "bad DIMM, ticket 42"
|
|
k := New(h, &fakeInst{host: h}, "/m", nil)
|
|
k.Reconcile(context.Background(), nodeMsg(true))
|
|
if h.count("scontrol update") != 0 {
|
|
t.Fatalf("llmgw resumed a node a Slurm administrator drained: %v", h.ran)
|
|
}
|
|
}
|
|
|
|
func TestClientEnvOnANodeWithoutFiles(t *testing.T) {
|
|
h := newFake()
|
|
k := New(h, &fakeInst{host: h}, "/m", nil)
|
|
if env := k.clientEnv(); len(env) != 1 || !strings.HasSuffix(env[0], "conf-cache/slurm.conf") {
|
|
t.Fatalf("env %v", env)
|
|
}
|
|
h.files[filepath.Join(slurm.ConfDir, "slurm.conf")] = []byte("x")
|
|
if env := k.clientEnv(); env != nil {
|
|
t.Fatalf("env %v on the controller", env)
|
|
}
|
|
}
|
|
|
|
func TestController(t *testing.T) {
|
|
h := newFake()
|
|
inst := &fakeInst{host: h}
|
|
k := New(h, inst, "/m", nil)
|
|
files := map[string]string{"slurm.conf": "ClusterName=llmgw\n", "gres.conf": "#\n", "cgroup.conf": "#\n"}
|
|
m := &llmgwv1.Slurm{Controller: true, MungeKey: key, Files: files}
|
|
st := k.Reconcile(context.Background(), m)
|
|
if st.GetError() != "" {
|
|
t.Fatal(st.GetError())
|
|
}
|
|
if !slices.Contains(inst.got["deb"], "slurmctld") || slices.Contains(inst.got["deb"], "slurmd") {
|
|
t.Fatalf("installed %v", inst.got)
|
|
}
|
|
if string(h.files["/etc/slurm/slurm.conf"]) != "ClusterName=llmgw\n" {
|
|
t.Fatal("slurm.conf not written")
|
|
}
|
|
if h.count("systemctl restart slurmctld") != 1 || h.count("scontrol reconfigure") != 1 {
|
|
t.Fatalf("slurmctld not started on its files: %v", h.ran)
|
|
}
|
|
if !st.GetSlurmctldRunning() || len(st.GetFilesSha256()) != 64 {
|
|
t.Fatalf("status %v", st)
|
|
}
|
|
// Accounting: MariaDB on this machine alone, the database, slurmdbd on
|
|
// it, the cluster added, all before slurmctld.
|
|
if !slices.Contains(inst.got["deb"], "slurmdbd") || !slices.Contains(inst.got["deb"], "mariadb-server") {
|
|
t.Fatalf("installed %v", inst.got)
|
|
}
|
|
if !strings.Contains(string(h.files["/etc/mysql/mariadb.conf.d/90-llmgw-slurm.cnf"]), "bind-address=127.0.0.1") {
|
|
t.Fatal("MariaDB, installed by llmgw, not kept to this machine")
|
|
}
|
|
dbd := string(h.files[slurm.DbdConfPath])
|
|
if !strings.Contains(dbd, "DbdHost=node06\n") || !strings.Contains(dbd, "StoragePass="+slurm.DbdPassword(key)+"\n") ||
|
|
h.owners[slurm.DbdConfPath] != "slurm" {
|
|
t.Fatalf("slurmdbd.conf for %q:\n%s", h.owners[slurm.DbdConfPath], dbd)
|
|
}
|
|
if _, ok := h.files[acctSQLPath]; ok {
|
|
t.Fatal("the database's statements, password and all, were left on disk")
|
|
}
|
|
if strings.Contains(strings.Join(h.ran, "\n"), slurm.DbdPassword(key)) {
|
|
t.Fatal("the password was on a command line")
|
|
}
|
|
order := strings.Join(h.ran, "\n")
|
|
if !strings.Contains(order, "sacctmgr -i add cluster llmgw") ||
|
|
strings.Index(order, "slurmdbd") > strings.Index(order, "systemctl restart slurmctld") {
|
|
t.Fatalf("accounting not before slurmctld: %v", h.ran)
|
|
}
|
|
if !st.GetSlurmdbdRunning() {
|
|
t.Fatalf("status %v", st)
|
|
}
|
|
sum := st.GetFilesSha256()
|
|
|
|
h.ran = nil
|
|
k.Reconcile(context.Background(), m)
|
|
if h.count("systemctl restart") != 0 {
|
|
t.Fatalf("restarted on the same files: %v", h.ran)
|
|
}
|
|
|
|
files["gres.conf"] = "NodeName=cuda01 Name=gpu File=/dev/nvidia0\n"
|
|
h.ran = nil
|
|
st = k.Reconcile(context.Background(), m)
|
|
if h.count("systemctl restart slurmctld") != 1 || st.GetFilesSha256() == sum {
|
|
t.Fatalf("a changed file did not restart slurmctld: %v", h.ran)
|
|
}
|
|
|
|
if st := k.Reconcile(context.Background(), &llmgwv1.Slurm{Controller: true, MungeKey: key,
|
|
Files: map[string]string{"slurm.conf": "x", "../evil": "y"}}); !strings.Contains(st.GetError(), "named") {
|
|
t.Fatalf("a path in a file name was taken: %v", st)
|
|
}
|
|
}
|
|
|
|
func TestTooOld(t *testing.T) {
|
|
h := newFake()
|
|
h.outputs["slurmd -V"] = "slurm-wlm 21.08.5\n"
|
|
k := New(h, &fakeInst{host: h}, "/m", nil)
|
|
st := k.Reconcile(context.Background(), nodeMsg(false))
|
|
if !strings.Contains(st.GetError(), "older than 22.05") {
|
|
t.Fatalf("status %v", st)
|
|
}
|
|
if _, ok := h.files[slurm.MungeKeyPath]; ok {
|
|
t.Fatal("went on with a Slurm too old")
|
|
}
|
|
}
|
|
|
|
func TestStop(t *testing.T) {
|
|
h := newFake()
|
|
k := New(h, &fakeInst{host: h}, "/m", nil)
|
|
k.Reconcile(context.Background(), nodeMsg(false))
|
|
h.ran = nil
|
|
if st := k.Reconcile(context.Background(), &llmgwv1.Slurm{}); st != nil {
|
|
t.Fatalf("status %v for no part", st)
|
|
}
|
|
if !slices.Contains(h.ran, "systemctl disable --now slurmd") {
|
|
t.Fatalf("slurmd not stopped: %v", h.ran)
|
|
}
|
|
if _, ok := h.files[DropIn]; ok {
|
|
t.Fatal("drop-in left")
|
|
}
|
|
// Never started here: nothing touched.
|
|
h2 := newFake()
|
|
k2 := New(h2, &fakeInst{host: h2}, "/m", nil)
|
|
k2.Reconcile(context.Background(), &llmgwv1.Slurm{})
|
|
if len(h2.ran) != 0 {
|
|
t.Fatalf("a machine llmgw never ran Slurm on was touched: %v", h2.ran)
|
|
}
|
|
}
|
|
|
|
func TestBadNames(t *testing.T) {
|
|
h := newFake()
|
|
k := New(h, &fakeInst{host: h}, "/m", nil)
|
|
m := nodeMsg(false)
|
|
m.Name = "cuda01 -c /etc/evil"
|
|
if st := k.Reconcile(context.Background(), m); !strings.Contains(st.GetError(), "spaces") {
|
|
t.Fatalf("status %v", st)
|
|
}
|
|
if st := k.Reconcile(context.Background(), &llmgwv1.Slurm{Node: true, Name: "a", ConfServer: "b"}); !strings.Contains(st.GetError(), "no key") {
|
|
t.Fatalf("status %v", st)
|
|
}
|
|
}
|
|
|
|
func TestAMariaDBThatWasThereKeepsItsSettings(t *testing.T) {
|
|
h := newFake()
|
|
for _, b := range []string{"munged", "scontrol", "slurmctld", "slurmdbd", "mariadbd", "mariadb"} {
|
|
h.bins[b] = true
|
|
}
|
|
k := New(h, &fakeInst{host: h}, "/m", nil)
|
|
st := k.Reconcile(context.Background(), &llmgwv1.Slurm{Controller: true, MungeKey: key,
|
|
Files: map[string]string{"slurm.conf": "ClusterName=llmgw\n"}})
|
|
if st.GetError() != "" {
|
|
t.Fatal(st.GetError())
|
|
}
|
|
for p := range h.files {
|
|
if strings.HasSuffix(p, ".cnf") {
|
|
t.Fatalf("a MariaDB llmgw did not install had its settings changed: %s", p)
|
|
}
|
|
}
|
|
if h.count("mariadb -e source") != 1 || !st.GetSlurmdbdRunning() {
|
|
t.Fatalf("the database was not made in the MariaDB that was there: %v", h.ran)
|
|
}
|
|
}
|