Files
llmgw/internal/agent/slurmnode/slurmnode_test.go
T
Ilker ManapandClaude Opus 5.5 d8ce573bdb
CI / test (push) Failing after 7m1s
Slurm's shared files and users: one path everywhere, accounts as Linux users
slurm.shared mounts the site's NFS export, has an llmgw node export the path
over NFS to the Slurm machines alone (2049 in the firewall), or looks for a
file system the site mounts. slurm.users "llmgw" makes each active account a
Linux user on every Slurm machine: login from its e-mail, a UID from 200000
kept in slurm_users and never reused, its home on the shared path, its SSH
keys from the portal's Profile (or ctl slurm users keys) read by sshd from
/etc/llmgw/ssh-keys. server init --slurm chooses /shared from the first
machine and llmgw's users; --slurm-shared and --slurm-users change that.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-12 21:34:34 +01:00

358 lines
12 KiB
Go

package slurmnode
import (
"context"
"errors"
"io/fs"
"path/filepath"
"slices"
"strings"
"testing"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
)
type fakeHost struct {
files map[string][]byte
owners map[string]string
bins map[string]bool
active map[string]bool
ran []string
outputs map[string]string
fail map[string]bool
state string
reason string
}
func newFake() *fakeHost {
return &fakeHost{files: map[string][]byte{}, owners: map[string]string{}, bins: map[string]bool{},
active: map[string]bool{}, outputs: map[string]string{
"slurmd -V": "slurm-wlm 23.11.4\n",
"slurmctld -V": "slurm-wlm 23.11.4\n",
"scontrol ping": "Slurmctld(primary) at node06 is UP\n",
"slurmd -C": "NodeName=cuda01 CPUs=16 RealMemory=64000\nUpTime=1-00:00:00\n",
"hostname -s": "node06\n",
}, fail: map[string]bool{}, state: "IDLE+DRAIN", reason: slurm.ReasonNew}
}
func (f *fakeHost) Run(_ context.Context, env []string, name string, args ...string) ([]byte, error) {
c := strings.Join(append([]string{name}, args...), " ")
f.ran = append(f.ran, c)
if f.fail[c] {
return []byte("it failed"), errors.New("exit status 1")
}
switch {
case name == "systemctl" && len(args) >= 1:
u := args[len(args)-1]
switch args[0] {
case "is-active":
if f.active[u] {
return []byte("active\n"), nil
}
return []byte("inactive\n"), errors.New("exit status 3")
case "restart", "start":
f.active[u] = true
case "enable":
if slices.Contains(args, "--now") {
f.active[u] = true
}
case "disable":
f.active[u] = false
}
return []byte(f.outputs[c]), nil
case strings.HasPrefix(c, "scontrol show node"):
return []byte("NodeName=cuda01 Arch=x86_64 State=" + f.state + " Partitions=gpu Reason=" + f.reason + " [root@now]\n"), nil
case strings.HasPrefix(c, "scontrol update"):
if slices.Contains(args, "State=RESUME") {
f.state, f.reason = "IDLE", ""
} else {
f.state, f.reason = "IDLE+DRAIN", ReasonServing
}
return nil, nil
case strings.HasPrefix(c, "pgrep"):
out := "1800358 /usr/sbin/slurmstepd infinity\n" + f.outputs["jobs"]
return []byte(out), nil
}
return []byte(f.outputs[c]), nil
}
func (f *fakeHost) LookPath(name string) (string, error) {
if f.bins[name] {
return "/usr/sbin/" + name, nil
}
return "", errors.New("not found")
}
func (f *fakeHost) ReadFile(path string) ([]byte, error) {
b, ok := f.files[path]
if !ok {
return nil, fs.ErrNotExist
}
return b, nil
}
func (f *fakeHost) WriteFile(path string, b []byte, _ fs.FileMode, owner string) error {
f.files[path] = append([]byte(nil), b...)
f.owners[path] = owner
return nil
}
func (f *fakeHost) MkdirAll(string, fs.FileMode, string) error { return nil }
func (f *fakeHost) Remove(path string) error { delete(f.files, path); return nil }
func (f *fakeHost) count(prefix string) int {
n := 0
for _, c := range f.ran {
if strings.HasPrefix(c, prefix) {
n++
}
}
return n
}
type fakeInst struct {
host *fakeHost
got map[string][]string
}
func (i *fakeInst) InstallPackages(_ context.Context, by map[string][]string) error {
i.got = by
for _, b := range []string{"munged", "scontrol", "slurmd", "slurmctld", "slurmdbd", "mariadbd", "mariadb", "exportfs", "mount.nfs"} {
i.host.bins[b] = true
}
return nil
}
var key = []byte(strings.Repeat("k", 1024))
func nodeMsg(lent bool) *llmgwv1.Slurm {
return &llmgwv1.Slurm{Node: true, Name: "cuda01", ConfServer: "10.0.0.146:6817", MungeKey: key, Lent: lent}
}
func TestNodeInstallsAndStaysDrained(t *testing.T) {
h := newFake()
inst := &fakeInst{host: h}
k := New(h, inst, "/var/lib/llmgw-agent/slurm", nil)
st := k.Reconcile(context.Background(), nodeMsg(false))
if st.GetError() != "" {
t.Fatal(st.GetError())
}
if !slices.Equal(inst.got["deb"], []string{"munge", "slurm-client", "slurmd"}) ||
!slices.Equal(inst.got["rpm"], []string{"munge", "slurm", "slurm-slurmd"}) {
t.Fatalf("installed %v", inst.got)
}
if string(h.files[slurm.MungeKeyPath]) != string(key) || h.owners[slurm.MungeKeyPath] != "munge" {
t.Fatal("the munge key was not written for munge")
}
unit := string(h.files[DropIn])
if !strings.Contains(unit, "ExecStart=/usr/sbin/slurmd --systemd --conf-server 10.0.0.146:6817 -N cuda01\n") ||
!strings.Contains(unit, "ConditionPathExists=\n") {
t.Fatalf("drop-in:\n%s", unit)
}
if !st.GetSlurmdRunning() || st.GetVersion() != "23.11.4" || !strings.HasPrefix(st.GetHardware(), "NodeName=cuda01 CPUs=16") {
t.Fatalf("status %v", st)
}
// Not lent: left drained, with llmgw's first reason.
if h.count("scontrol update") != 0 || st.GetState() != "IDLE+DRAIN" {
t.Fatalf("a node not lent was changed: %v, %v", h.ran, st)
}
if _, ok := h.files["/var/lib/llmgw-agent/slurm"]; !ok {
t.Fatal("no marker")
}
// Again, nothing changed: no install, no restart.
h.ran = nil
inst.got = nil
k.Reconcile(context.Background(), nodeMsg(false))
if inst.got != nil || h.count("systemctl restart") != 0 || h.count("systemctl enable") != 0 {
t.Fatalf("a second pass did work: %v", h.ran)
}
}
func TestLendAndReclaim(t *testing.T) {
h := newFake()
k := New(h, &fakeInst{host: h}, "/m", nil)
st := k.Reconcile(context.Background(), nodeMsg(true))
if h.count("scontrol update NodeName=cuda01 State=RESUME") != 1 || st.GetState() != "IDLE" {
t.Fatalf("a lent node was not resumed: %v %v", h.ran, st)
}
h.outputs["jobs"] = "2001 slurmstepd: [101.batch]\n2002 slurmstepd: [102.0]\n"
st = k.Reconcile(context.Background(), nodeMsg(false))
if h.count("scontrol update NodeName=cuda01 State=DRAIN Reason="+ReasonServing) != 1 {
t.Fatalf("a node taken back was not drained: %v", h.ran)
}
if st.GetJobs() != 2 || !strings.Contains(st.GetState(), "DRAIN") {
t.Fatalf("status %v", st)
}
// Drained already: not drained again.
h.ran = nil
k.Reconcile(context.Background(), nodeMsg(false))
if h.count("scontrol update") != 0 {
t.Fatalf("drained twice: %v", h.ran)
}
}
func TestAdminsDrainIsLeft(t *testing.T) {
h := newFake()
h.reason = "bad DIMM, ticket 42"
k := New(h, &fakeInst{host: h}, "/m", nil)
k.Reconcile(context.Background(), nodeMsg(true))
if h.count("scontrol update") != 0 {
t.Fatalf("llmgw resumed a node a Slurm administrator drained: %v", h.ran)
}
}
func TestClientEnvOnANodeWithoutFiles(t *testing.T) {
h := newFake()
k := New(h, &fakeInst{host: h}, "/m", nil)
if env := k.clientEnv(); len(env) != 1 || !strings.HasSuffix(env[0], "conf-cache/slurm.conf") {
t.Fatalf("env %v", env)
}
h.files[filepath.Join(slurm.ConfDir, "slurm.conf")] = []byte("x")
if env := k.clientEnv(); env != nil {
t.Fatalf("env %v on the controller", env)
}
}
func TestController(t *testing.T) {
h := newFake()
inst := &fakeInst{host: h}
k := New(h, inst, "/m", nil)
files := map[string]string{"slurm.conf": "ClusterName=llmgw\n", "gres.conf": "#\n", "cgroup.conf": "#\n"}
m := &llmgwv1.Slurm{Controller: true, MungeKey: key, Files: files}
st := k.Reconcile(context.Background(), m)
if st.GetError() != "" {
t.Fatal(st.GetError())
}
if !slices.Contains(inst.got["deb"], "slurmctld") || slices.Contains(inst.got["deb"], "slurmd") {
t.Fatalf("installed %v", inst.got)
}
if string(h.files["/etc/slurm/slurm.conf"]) != "ClusterName=llmgw\n" {
t.Fatal("slurm.conf not written")
}
if h.count("systemctl restart slurmctld") != 1 || h.count("scontrol reconfigure") != 1 {
t.Fatalf("slurmctld not started on its files: %v", h.ran)
}
if !st.GetSlurmctldRunning() || len(st.GetFilesSha256()) != 64 {
t.Fatalf("status %v", st)
}
// Accounting: MariaDB on this machine alone, the database, slurmdbd on
// it, the cluster added, all before slurmctld.
if !slices.Contains(inst.got["deb"], "slurmdbd") || !slices.Contains(inst.got["deb"], "mariadb-server") {
t.Fatalf("installed %v", inst.got)
}
if !strings.Contains(string(h.files["/etc/mysql/mariadb.conf.d/90-llmgw-slurm.cnf"]), "bind-address=127.0.0.1") {
t.Fatal("MariaDB, installed by llmgw, not kept to this machine")
}
dbd := string(h.files[slurm.DbdConfPath])
if !strings.Contains(dbd, "DbdHost=node06\n") || !strings.Contains(dbd, "StoragePass="+slurm.DbdPassword(key)+"\n") ||
h.owners[slurm.DbdConfPath] != "slurm" {
t.Fatalf("slurmdbd.conf for %q:\n%s", h.owners[slurm.DbdConfPath], dbd)
}
if _, ok := h.files[acctSQLPath]; ok {
t.Fatal("the database's statements, password and all, were left on disk")
}
if strings.Contains(strings.Join(h.ran, "\n"), slurm.DbdPassword(key)) {
t.Fatal("the password was on a command line")
}
order := strings.Join(h.ran, "\n")
if !strings.Contains(order, "sacctmgr -i add cluster llmgw") ||
strings.Index(order, "slurmdbd") > strings.Index(order, "systemctl restart slurmctld") {
t.Fatalf("accounting not before slurmctld: %v", h.ran)
}
if !st.GetSlurmdbdRunning() {
t.Fatalf("status %v", st)
}
sum := st.GetFilesSha256()
h.ran = nil
k.Reconcile(context.Background(), m)
if h.count("systemctl restart") != 0 {
t.Fatalf("restarted on the same files: %v", h.ran)
}
files["gres.conf"] = "NodeName=cuda01 Name=gpu File=/dev/nvidia0\n"
h.ran = nil
st = k.Reconcile(context.Background(), m)
if h.count("systemctl restart slurmctld") != 1 || st.GetFilesSha256() == sum {
t.Fatalf("a changed file did not restart slurmctld: %v", h.ran)
}
if st := k.Reconcile(context.Background(), &llmgwv1.Slurm{Controller: true, MungeKey: key,
Files: map[string]string{"slurm.conf": "x", "../evil": "y"}}); !strings.Contains(st.GetError(), "named") {
t.Fatalf("a path in a file name was taken: %v", st)
}
}
func TestTooOld(t *testing.T) {
h := newFake()
h.outputs["slurmd -V"] = "slurm-wlm 21.08.5\n"
k := New(h, &fakeInst{host: h}, "/m", nil)
st := k.Reconcile(context.Background(), nodeMsg(false))
if !strings.Contains(st.GetError(), "older than 22.05") {
t.Fatalf("status %v", st)
}
if _, ok := h.files[slurm.MungeKeyPath]; ok {
t.Fatal("went on with a Slurm too old")
}
}
func TestStop(t *testing.T) {
h := newFake()
k := New(h, &fakeInst{host: h}, "/m", nil)
k.Reconcile(context.Background(), nodeMsg(false))
h.ran = nil
if st := k.Reconcile(context.Background(), &llmgwv1.Slurm{}); st != nil {
t.Fatalf("status %v for no part", st)
}
if !slices.Contains(h.ran, "systemctl disable --now slurmd") {
t.Fatalf("slurmd not stopped: %v", h.ran)
}
if _, ok := h.files[DropIn]; ok {
t.Fatal("drop-in left")
}
// Never started here: nothing touched.
h2 := newFake()
k2 := New(h2, &fakeInst{host: h2}, "/m", nil)
k2.Reconcile(context.Background(), &llmgwv1.Slurm{})
if len(h2.ran) != 0 {
t.Fatalf("a machine llmgw never ran Slurm on was touched: %v", h2.ran)
}
}
func TestBadNames(t *testing.T) {
h := newFake()
k := New(h, &fakeInst{host: h}, "/m", nil)
m := nodeMsg(false)
m.Name = "cuda01 -c /etc/evil"
if st := k.Reconcile(context.Background(), m); !strings.Contains(st.GetError(), "spaces") {
t.Fatalf("status %v", st)
}
if st := k.Reconcile(context.Background(), &llmgwv1.Slurm{Node: true, Name: "a", ConfServer: "b"}); !strings.Contains(st.GetError(), "no key") {
t.Fatalf("status %v", st)
}
}
func TestAMariaDBThatWasThereKeepsItsSettings(t *testing.T) {
h := newFake()
for _, b := range []string{"munged", "scontrol", "slurmctld", "slurmdbd", "mariadbd", "mariadb", "exportfs", "mount.nfs"} {
h.bins[b] = true
}
k := New(h, &fakeInst{host: h}, "/m", nil)
st := k.Reconcile(context.Background(), &llmgwv1.Slurm{Controller: true, MungeKey: key,
Files: map[string]string{"slurm.conf": "ClusterName=llmgw\n"}})
if st.GetError() != "" {
t.Fatal(st.GetError())
}
for p := range h.files {
if strings.HasSuffix(p, ".cnf") {
t.Fatalf("a MariaDB llmgw did not install had its settings changed: %s", p)
}
}
if h.count("mariadb -e source") != 1 || !st.GetSlurmdbdRunning() {
t.Fatalf("the database was not made in the MariaDB that was there: %v", h.ran)
}
}