Files
llmgw/internal/agent/slurmnode/slurmnode_test.go
T
Ilker ManapandClaude Opus 5.5 748f777a56
CI / test (push) Successful in 10m57s
Slurm's accounting: slurmdbd and a MariaDB on the controller
Since 23.11 Slurm runs badly without accounting storage: the main
scheduler passes every job by as InvalidAccount. slurmdbd takes MySQL or
MariaDB only, so the controller's agent installs MariaDB beside it,
answering that machine alone, makes the database and its user (the
password made from the munge key, never on a command line), runs
slurmdbd and adds the cluster, all before slurmctld. A MariaDB that was
on the machine before keeps its settings. slurm.conf accounts the jobs
there, with what each used from its cgroup and the cards, and the
priority is multifactor again. Port 6819 joins Slurm's in the firewall.
ctl slurm status and the console say whether slurmdbd runs; the install
screen, server init --slurm and the guide say what is put where.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-12 07:59:08 +01:00

358 lines
12 KiB
Go

package slurmnode
import (
"context"
"errors"
"io/fs"
"path/filepath"
"slices"
"strings"
"testing"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
)
type fakeHost struct {
files map[string][]byte
owners map[string]string
bins map[string]bool
active map[string]bool
ran []string
outputs map[string]string
fail map[string]bool
state string
reason string
}
func newFake() *fakeHost {
return &fakeHost{files: map[string][]byte{}, owners: map[string]string{}, bins: map[string]bool{},
active: map[string]bool{}, outputs: map[string]string{
"slurmd -V": "slurm-wlm 23.11.4\n",
"slurmctld -V": "slurm-wlm 23.11.4\n",
"scontrol ping": "Slurmctld(primary) at node06 is UP\n",
"slurmd -C": "NodeName=cuda01 CPUs=16 RealMemory=64000\nUpTime=1-00:00:00\n",
"hostname -s": "node06\n",
}, fail: map[string]bool{}, state: "IDLE+DRAIN", reason: slurm.ReasonNew}
}
func (f *fakeHost) Run(_ context.Context, env []string, name string, args ...string) ([]byte, error) {
c := strings.Join(append([]string{name}, args...), " ")
f.ran = append(f.ran, c)
if f.fail[c] {
return []byte("it failed"), errors.New("exit status 1")
}
switch {
case name == "systemctl" && len(args) >= 1:
u := args[len(args)-1]
switch args[0] {
case "is-active":
if f.active[u] {
return []byte("active\n"), nil
}
return []byte("inactive\n"), errors.New("exit status 3")
case "restart", "start":
f.active[u] = true
case "enable":
if slices.Contains(args, "--now") {
f.active[u] = true
}
case "disable":
f.active[u] = false
}
return nil, nil
case strings.HasPrefix(c, "scontrol show node"):
return []byte("NodeName=cuda01 Arch=x86_64 State=" + f.state + " Partitions=gpu Reason=" + f.reason + " [root@now]\n"), nil
case strings.HasPrefix(c, "scontrol update"):
if slices.Contains(args, "State=RESUME") {
f.state, f.reason = "IDLE", ""
} else {
f.state, f.reason = "IDLE+DRAIN", ReasonServing
}
return nil, nil
case strings.HasPrefix(c, "pgrep"):
out := "1800358 /usr/sbin/slurmstepd infinity\n" + f.outputs["jobs"]
return []byte(out), nil
}
return []byte(f.outputs[c]), nil
}
func (f *fakeHost) LookPath(name string) (string, error) {
if f.bins[name] {
return "/usr/sbin/" + name, nil
}
return "", errors.New("not found")
}
func (f *fakeHost) ReadFile(path string) ([]byte, error) {
b, ok := f.files[path]
if !ok {
return nil, fs.ErrNotExist
}
return b, nil
}
func (f *fakeHost) WriteFile(path string, b []byte, _ fs.FileMode, owner string) error {
f.files[path] = append([]byte(nil), b...)
f.owners[path] = owner
return nil
}
func (f *fakeHost) MkdirAll(string, fs.FileMode, string) error { return nil }
func (f *fakeHost) Remove(path string) error { delete(f.files, path); return nil }
func (f *fakeHost) count(prefix string) int {
n := 0
for _, c := range f.ran {
if strings.HasPrefix(c, prefix) {
n++
}
}
return n
}
type fakeInst struct {
host *fakeHost
got map[string][]string
}
func (i *fakeInst) InstallPackages(_ context.Context, by map[string][]string) error {
i.got = by
for _, b := range []string{"munged", "scontrol", "slurmd", "slurmctld", "slurmdbd", "mariadbd", "mariadb"} {
i.host.bins[b] = true
}
return nil
}
var key = []byte(strings.Repeat("k", 1024))
func nodeMsg(lent bool) *llmgwv1.Slurm {
return &llmgwv1.Slurm{Node: true, Name: "cuda01", ConfServer: "10.0.0.146:6817", MungeKey: key, Lent: lent}
}
func TestNodeInstallsAndStaysDrained(t *testing.T) {
h := newFake()
inst := &fakeInst{host: h}
k := New(h, inst, "/var/lib/llmgw-agent/slurm", nil)
st := k.Reconcile(context.Background(), nodeMsg(false))
if st.GetError() != "" {
t.Fatal(st.GetError())
}
if !slices.Equal(inst.got["deb"], []string{"munge", "slurm-client", "slurmd"}) ||
!slices.Equal(inst.got["rpm"], []string{"munge", "slurm", "slurm-slurmd"}) {
t.Fatalf("installed %v", inst.got)
}
if string(h.files[slurm.MungeKeyPath]) != string(key) || h.owners[slurm.MungeKeyPath] != "munge" {
t.Fatal("the munge key was not written for munge")
}
unit := string(h.files[DropIn])
if !strings.Contains(unit, "ExecStart=/usr/sbin/slurmd --systemd --conf-server 10.0.0.146:6817 -N cuda01\n") ||
!strings.Contains(unit, "ConditionPathExists=\n") {
t.Fatalf("drop-in:\n%s", unit)
}
if !st.GetSlurmdRunning() || st.GetVersion() != "23.11.4" || !strings.HasPrefix(st.GetHardware(), "NodeName=cuda01 CPUs=16") {
t.Fatalf("status %v", st)
}
// Not lent: left drained, with llmgw's first reason.
if h.count("scontrol update") != 0 || st.GetState() != "IDLE+DRAIN" {
t.Fatalf("a node not lent was changed: %v, %v", h.ran, st)
}
if _, ok := h.files["/var/lib/llmgw-agent/slurm"]; !ok {
t.Fatal("no marker")
}
// Again, nothing changed: no install, no restart.
h.ran = nil
inst.got = nil
k.Reconcile(context.Background(), nodeMsg(false))
if inst.got != nil || h.count("systemctl restart") != 0 || h.count("systemctl enable") != 0 {
t.Fatalf("a second pass did work: %v", h.ran)
}
}
func TestLendAndReclaim(t *testing.T) {
h := newFake()
k := New(h, &fakeInst{host: h}, "/m", nil)
st := k.Reconcile(context.Background(), nodeMsg(true))
if h.count("scontrol update NodeName=cuda01 State=RESUME") != 1 || st.GetState() != "IDLE" {
t.Fatalf("a lent node was not resumed: %v %v", h.ran, st)
}
h.outputs["jobs"] = "2001 slurmstepd: [101.batch]\n2002 slurmstepd: [102.0]\n"
st = k.Reconcile(context.Background(), nodeMsg(false))
if h.count("scontrol update NodeName=cuda01 State=DRAIN Reason="+ReasonServing) != 1 {
t.Fatalf("a node taken back was not drained: %v", h.ran)
}
if st.GetJobs() != 2 || !strings.Contains(st.GetState(), "DRAIN") {
t.Fatalf("status %v", st)
}
// Drained already: not drained again.
h.ran = nil
k.Reconcile(context.Background(), nodeMsg(false))
if h.count("scontrol update") != 0 {
t.Fatalf("drained twice: %v", h.ran)
}
}
func TestAdminsDrainIsLeft(t *testing.T) {
h := newFake()
h.reason = "bad DIMM, ticket 42"
k := New(h, &fakeInst{host: h}, "/m", nil)
k.Reconcile(context.Background(), nodeMsg(true))
if h.count("scontrol update") != 0 {
t.Fatalf("llmgw resumed a node a Slurm administrator drained: %v", h.ran)
}
}
func TestClientEnvOnANodeWithoutFiles(t *testing.T) {
h := newFake()
k := New(h, &fakeInst{host: h}, "/m", nil)
if env := k.clientEnv(); len(env) != 1 || !strings.HasSuffix(env[0], "conf-cache/slurm.conf") {
t.Fatalf("env %v", env)
}
h.files[filepath.Join(slurm.ConfDir, "slurm.conf")] = []byte("x")
if env := k.clientEnv(); env != nil {
t.Fatalf("env %v on the controller", env)
}
}
func TestController(t *testing.T) {
h := newFake()
inst := &fakeInst{host: h}
k := New(h, inst, "/m", nil)
files := map[string]string{"slurm.conf": "ClusterName=llmgw\n", "gres.conf": "#\n", "cgroup.conf": "#\n"}
m := &llmgwv1.Slurm{Controller: true, MungeKey: key, Files: files}
st := k.Reconcile(context.Background(), m)
if st.GetError() != "" {
t.Fatal(st.GetError())
}
if !slices.Contains(inst.got["deb"], "slurmctld") || slices.Contains(inst.got["deb"], "slurmd") {
t.Fatalf("installed %v", inst.got)
}
if string(h.files["/etc/slurm/slurm.conf"]) != "ClusterName=llmgw\n" {
t.Fatal("slurm.conf not written")
}
if h.count("systemctl restart slurmctld") != 1 || h.count("scontrol reconfigure") != 1 {
t.Fatalf("slurmctld not started on its files: %v", h.ran)
}
if !st.GetSlurmctldRunning() || len(st.GetFilesSha256()) != 64 {
t.Fatalf("status %v", st)
}
// Accounting: MariaDB on this machine alone, the database, slurmdbd on
// it, the cluster added, all before slurmctld.
if !slices.Contains(inst.got["deb"], "slurmdbd") || !slices.Contains(inst.got["deb"], "mariadb-server") {
t.Fatalf("installed %v", inst.got)
}
if !strings.Contains(string(h.files["/etc/mysql/mariadb.conf.d/90-llmgw-slurm.cnf"]), "bind-address=127.0.0.1") {
t.Fatal("MariaDB, installed by llmgw, not kept to this machine")
}
dbd := string(h.files[slurm.DbdConfPath])
if !strings.Contains(dbd, "DbdHost=node06\n") || !strings.Contains(dbd, "StoragePass="+slurm.DbdPassword(key)+"\n") ||
h.owners[slurm.DbdConfPath] != "slurm" {
t.Fatalf("slurmdbd.conf for %q:\n%s", h.owners[slurm.DbdConfPath], dbd)
}
if _, ok := h.files[acctSQLPath]; ok {
t.Fatal("the database's statements, password and all, were left on disk")
}
if strings.Contains(strings.Join(h.ran, "\n"), slurm.DbdPassword(key)) {
t.Fatal("the password was on a command line")
}
order := strings.Join(h.ran, "\n")
if !strings.Contains(order, "sacctmgr -i add cluster llmgw") ||
strings.Index(order, "slurmdbd") > strings.Index(order, "systemctl restart slurmctld") {
t.Fatalf("accounting not before slurmctld: %v", h.ran)
}
if !st.GetSlurmdbdRunning() {
t.Fatalf("status %v", st)
}
sum := st.GetFilesSha256()
h.ran = nil
k.Reconcile(context.Background(), m)
if h.count("systemctl restart") != 0 {
t.Fatalf("restarted on the same files: %v", h.ran)
}
files["gres.conf"] = "NodeName=cuda01 Name=gpu File=/dev/nvidia0\n"
h.ran = nil
st = k.Reconcile(context.Background(), m)
if h.count("systemctl restart slurmctld") != 1 || st.GetFilesSha256() == sum {
t.Fatalf("a changed file did not restart slurmctld: %v", h.ran)
}
if st := k.Reconcile(context.Background(), &llmgwv1.Slurm{Controller: true, MungeKey: key,
Files: map[string]string{"slurm.conf": "x", "../evil": "y"}}); !strings.Contains(st.GetError(), "named") {
t.Fatalf("a path in a file name was taken: %v", st)
}
}
func TestTooOld(t *testing.T) {
h := newFake()
h.outputs["slurmd -V"] = "slurm-wlm 21.08.5\n"
k := New(h, &fakeInst{host: h}, "/m", nil)
st := k.Reconcile(context.Background(), nodeMsg(false))
if !strings.Contains(st.GetError(), "older than 22.05") {
t.Fatalf("status %v", st)
}
if _, ok := h.files[slurm.MungeKeyPath]; ok {
t.Fatal("went on with a Slurm too old")
}
}
func TestStop(t *testing.T) {
h := newFake()
k := New(h, &fakeInst{host: h}, "/m", nil)
k.Reconcile(context.Background(), nodeMsg(false))
h.ran = nil
if st := k.Reconcile(context.Background(), &llmgwv1.Slurm{}); st != nil {
t.Fatalf("status %v for no part", st)
}
if !slices.Contains(h.ran, "systemctl disable --now slurmd") {
t.Fatalf("slurmd not stopped: %v", h.ran)
}
if _, ok := h.files[DropIn]; ok {
t.Fatal("drop-in left")
}
// Never started here: nothing touched.
h2 := newFake()
k2 := New(h2, &fakeInst{host: h2}, "/m", nil)
k2.Reconcile(context.Background(), &llmgwv1.Slurm{})
if len(h2.ran) != 0 {
t.Fatalf("a machine llmgw never ran Slurm on was touched: %v", h2.ran)
}
}
func TestBadNames(t *testing.T) {
h := newFake()
k := New(h, &fakeInst{host: h}, "/m", nil)
m := nodeMsg(false)
m.Name = "cuda01 -c /etc/evil"
if st := k.Reconcile(context.Background(), m); !strings.Contains(st.GetError(), "spaces") {
t.Fatalf("status %v", st)
}
if st := k.Reconcile(context.Background(), &llmgwv1.Slurm{Node: true, Name: "a", ConfServer: "b"}); !strings.Contains(st.GetError(), "no key") {
t.Fatalf("status %v", st)
}
}
func TestAMariaDBThatWasThereKeepsItsSettings(t *testing.T) {
h := newFake()
for _, b := range []string{"munged", "scontrol", "slurmctld", "slurmdbd", "mariadbd", "mariadb"} {
h.bins[b] = true
}
k := New(h, &fakeInst{host: h}, "/m", nil)
st := k.Reconcile(context.Background(), &llmgwv1.Slurm{Controller: true, MungeKey: key,
Files: map[string]string{"slurm.conf": "ClusterName=llmgw\n"}})
if st.GetError() != "" {
t.Fatal(st.GetError())
}
for p := range h.files {
if strings.HasSuffix(p, ".cnf") {
t.Fatalf("a MariaDB llmgw did not install had its settings changed: %s", p)
}
}
if h.count("mariadb -e source") != 1 || !st.GetSlurmdbdRunning() {
t.Fatalf("the database was not made in the MariaDB that was there: %v", h.ran)
}
}