CI / test (push) Failing after 7m1s
slurm.shared mounts the site's NFS export, has an llmgw node export the path over NFS to the Slurm machines alone (2049 in the firewall), or looks for a file system the site mounts. slurm.users "llmgw" makes each active account a Linux user on every Slurm machine: login from its e-mail, a UID from 200000 kept in slurm_users and never reused, its home on the shared path, its SSH keys from the portal's Profile (or ctl slurm users keys) read by sshd from /etc/llmgw/ssh-keys. server init --slurm chooses /shared from the first machine and llmgw's users; --slurm-shared and --slurm-users change that. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
358 lines
12 KiB
Go
358 lines
12 KiB
Go
package slurmnode
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"io/fs"
|
|
"path/filepath"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
|
|
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
|
|
)
|
|
|
|
type fakeHost struct {
|
|
files map[string][]byte
|
|
owners map[string]string
|
|
bins map[string]bool
|
|
active map[string]bool
|
|
ran []string
|
|
outputs map[string]string
|
|
fail map[string]bool
|
|
state string
|
|
reason string
|
|
}
|
|
|
|
func newFake() *fakeHost {
|
|
return &fakeHost{files: map[string][]byte{}, owners: map[string]string{}, bins: map[string]bool{},
|
|
active: map[string]bool{}, outputs: map[string]string{
|
|
"slurmd -V": "slurm-wlm 23.11.4\n",
|
|
"slurmctld -V": "slurm-wlm 23.11.4\n",
|
|
"scontrol ping": "Slurmctld(primary) at node06 is UP\n",
|
|
"slurmd -C": "NodeName=cuda01 CPUs=16 RealMemory=64000\nUpTime=1-00:00:00\n",
|
|
"hostname -s": "node06\n",
|
|
}, fail: map[string]bool{}, state: "IDLE+DRAIN", reason: slurm.ReasonNew}
|
|
}
|
|
|
|
func (f *fakeHost) Run(_ context.Context, env []string, name string, args ...string) ([]byte, error) {
|
|
c := strings.Join(append([]string{name}, args...), " ")
|
|
f.ran = append(f.ran, c)
|
|
if f.fail[c] {
|
|
return []byte("it failed"), errors.New("exit status 1")
|
|
}
|
|
switch {
|
|
case name == "systemctl" && len(args) >= 1:
|
|
u := args[len(args)-1]
|
|
switch args[0] {
|
|
case "is-active":
|
|
if f.active[u] {
|
|
return []byte("active\n"), nil
|
|
}
|
|
return []byte("inactive\n"), errors.New("exit status 3")
|
|
case "restart", "start":
|
|
f.active[u] = true
|
|
case "enable":
|
|
if slices.Contains(args, "--now") {
|
|
f.active[u] = true
|
|
}
|
|
case "disable":
|
|
f.active[u] = false
|
|
}
|
|
return []byte(f.outputs[c]), nil
|
|
case strings.HasPrefix(c, "scontrol show node"):
|
|
return []byte("NodeName=cuda01 Arch=x86_64 State=" + f.state + " Partitions=gpu Reason=" + f.reason + " [root@now]\n"), nil
|
|
case strings.HasPrefix(c, "scontrol update"):
|
|
if slices.Contains(args, "State=RESUME") {
|
|
f.state, f.reason = "IDLE", ""
|
|
} else {
|
|
f.state, f.reason = "IDLE+DRAIN", ReasonServing
|
|
}
|
|
return nil, nil
|
|
case strings.HasPrefix(c, "pgrep"):
|
|
out := "1800358 /usr/sbin/slurmstepd infinity\n" + f.outputs["jobs"]
|
|
return []byte(out), nil
|
|
}
|
|
return []byte(f.outputs[c]), nil
|
|
}
|
|
|
|
func (f *fakeHost) LookPath(name string) (string, error) {
|
|
if f.bins[name] {
|
|
return "/usr/sbin/" + name, nil
|
|
}
|
|
return "", errors.New("not found")
|
|
}
|
|
|
|
func (f *fakeHost) ReadFile(path string) ([]byte, error) {
|
|
b, ok := f.files[path]
|
|
if !ok {
|
|
return nil, fs.ErrNotExist
|
|
}
|
|
return b, nil
|
|
}
|
|
|
|
func (f *fakeHost) WriteFile(path string, b []byte, _ fs.FileMode, owner string) error {
|
|
f.files[path] = append([]byte(nil), b...)
|
|
f.owners[path] = owner
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeHost) MkdirAll(string, fs.FileMode, string) error { return nil }
|
|
|
|
func (f *fakeHost) Remove(path string) error { delete(f.files, path); return nil }
|
|
|
|
func (f *fakeHost) count(prefix string) int {
|
|
n := 0
|
|
for _, c := range f.ran {
|
|
if strings.HasPrefix(c, prefix) {
|
|
n++
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
|
|
type fakeInst struct {
|
|
host *fakeHost
|
|
got map[string][]string
|
|
}
|
|
|
|
func (i *fakeInst) InstallPackages(_ context.Context, by map[string][]string) error {
|
|
i.got = by
|
|
for _, b := range []string{"munged", "scontrol", "slurmd", "slurmctld", "slurmdbd", "mariadbd", "mariadb", "exportfs", "mount.nfs"} {
|
|
i.host.bins[b] = true
|
|
}
|
|
return nil
|
|
}
|
|
|
|
var key = []byte(strings.Repeat("k", 1024))
|
|
|
|
func nodeMsg(lent bool) *llmgwv1.Slurm {
|
|
return &llmgwv1.Slurm{Node: true, Name: "cuda01", ConfServer: "10.0.0.146:6817", MungeKey: key, Lent: lent}
|
|
}
|
|
|
|
func TestNodeInstallsAndStaysDrained(t *testing.T) {
|
|
h := newFake()
|
|
inst := &fakeInst{host: h}
|
|
k := New(h, inst, "/var/lib/llmgw-agent/slurm", nil)
|
|
st := k.Reconcile(context.Background(), nodeMsg(false))
|
|
if st.GetError() != "" {
|
|
t.Fatal(st.GetError())
|
|
}
|
|
if !slices.Equal(inst.got["deb"], []string{"munge", "slurm-client", "slurmd"}) ||
|
|
!slices.Equal(inst.got["rpm"], []string{"munge", "slurm", "slurm-slurmd"}) {
|
|
t.Fatalf("installed %v", inst.got)
|
|
}
|
|
if string(h.files[slurm.MungeKeyPath]) != string(key) || h.owners[slurm.MungeKeyPath] != "munge" {
|
|
t.Fatal("the munge key was not written for munge")
|
|
}
|
|
unit := string(h.files[DropIn])
|
|
if !strings.Contains(unit, "ExecStart=/usr/sbin/slurmd --systemd --conf-server 10.0.0.146:6817 -N cuda01\n") ||
|
|
!strings.Contains(unit, "ConditionPathExists=\n") {
|
|
t.Fatalf("drop-in:\n%s", unit)
|
|
}
|
|
if !st.GetSlurmdRunning() || st.GetVersion() != "23.11.4" || !strings.HasPrefix(st.GetHardware(), "NodeName=cuda01 CPUs=16") {
|
|
t.Fatalf("status %v", st)
|
|
}
|
|
// Not lent: left drained, with llmgw's first reason.
|
|
if h.count("scontrol update") != 0 || st.GetState() != "IDLE+DRAIN" {
|
|
t.Fatalf("a node not lent was changed: %v, %v", h.ran, st)
|
|
}
|
|
if _, ok := h.files["/var/lib/llmgw-agent/slurm"]; !ok {
|
|
t.Fatal("no marker")
|
|
}
|
|
|
|
// Again, nothing changed: no install, no restart.
|
|
h.ran = nil
|
|
inst.got = nil
|
|
k.Reconcile(context.Background(), nodeMsg(false))
|
|
if inst.got != nil || h.count("systemctl restart") != 0 || h.count("systemctl enable") != 0 {
|
|
t.Fatalf("a second pass did work: %v", h.ran)
|
|
}
|
|
}
|
|
|
|
func TestLendAndReclaim(t *testing.T) {
|
|
h := newFake()
|
|
k := New(h, &fakeInst{host: h}, "/m", nil)
|
|
st := k.Reconcile(context.Background(), nodeMsg(true))
|
|
if h.count("scontrol update NodeName=cuda01 State=RESUME") != 1 || st.GetState() != "IDLE" {
|
|
t.Fatalf("a lent node was not resumed: %v %v", h.ran, st)
|
|
}
|
|
h.outputs["jobs"] = "2001 slurmstepd: [101.batch]\n2002 slurmstepd: [102.0]\n"
|
|
st = k.Reconcile(context.Background(), nodeMsg(false))
|
|
if h.count("scontrol update NodeName=cuda01 State=DRAIN Reason="+ReasonServing) != 1 {
|
|
t.Fatalf("a node taken back was not drained: %v", h.ran)
|
|
}
|
|
if st.GetJobs() != 2 || !strings.Contains(st.GetState(), "DRAIN") {
|
|
t.Fatalf("status %v", st)
|
|
}
|
|
// Drained already: not drained again.
|
|
h.ran = nil
|
|
k.Reconcile(context.Background(), nodeMsg(false))
|
|
if h.count("scontrol update") != 0 {
|
|
t.Fatalf("drained twice: %v", h.ran)
|
|
}
|
|
}
|
|
|
|
func TestAdminsDrainIsLeft(t *testing.T) {
|
|
h := newFake()
|
|
h.reason = "bad DIMM, ticket 42"
|
|
k := New(h, &fakeInst{host: h}, "/m", nil)
|
|
k.Reconcile(context.Background(), nodeMsg(true))
|
|
if h.count("scontrol update") != 0 {
|
|
t.Fatalf("llmgw resumed a node a Slurm administrator drained: %v", h.ran)
|
|
}
|
|
}
|
|
|
|
func TestClientEnvOnANodeWithoutFiles(t *testing.T) {
|
|
h := newFake()
|
|
k := New(h, &fakeInst{host: h}, "/m", nil)
|
|
if env := k.clientEnv(); len(env) != 1 || !strings.HasSuffix(env[0], "conf-cache/slurm.conf") {
|
|
t.Fatalf("env %v", env)
|
|
}
|
|
h.files[filepath.Join(slurm.ConfDir, "slurm.conf")] = []byte("x")
|
|
if env := k.clientEnv(); env != nil {
|
|
t.Fatalf("env %v on the controller", env)
|
|
}
|
|
}
|
|
|
|
func TestController(t *testing.T) {
|
|
h := newFake()
|
|
inst := &fakeInst{host: h}
|
|
k := New(h, inst, "/m", nil)
|
|
files := map[string]string{"slurm.conf": "ClusterName=llmgw\n", "gres.conf": "#\n", "cgroup.conf": "#\n"}
|
|
m := &llmgwv1.Slurm{Controller: true, MungeKey: key, Files: files}
|
|
st := k.Reconcile(context.Background(), m)
|
|
if st.GetError() != "" {
|
|
t.Fatal(st.GetError())
|
|
}
|
|
if !slices.Contains(inst.got["deb"], "slurmctld") || slices.Contains(inst.got["deb"], "slurmd") {
|
|
t.Fatalf("installed %v", inst.got)
|
|
}
|
|
if string(h.files["/etc/slurm/slurm.conf"]) != "ClusterName=llmgw\n" {
|
|
t.Fatal("slurm.conf not written")
|
|
}
|
|
if h.count("systemctl restart slurmctld") != 1 || h.count("scontrol reconfigure") != 1 {
|
|
t.Fatalf("slurmctld not started on its files: %v", h.ran)
|
|
}
|
|
if !st.GetSlurmctldRunning() || len(st.GetFilesSha256()) != 64 {
|
|
t.Fatalf("status %v", st)
|
|
}
|
|
// Accounting: MariaDB on this machine alone, the database, slurmdbd on
|
|
// it, the cluster added, all before slurmctld.
|
|
if !slices.Contains(inst.got["deb"], "slurmdbd") || !slices.Contains(inst.got["deb"], "mariadb-server") {
|
|
t.Fatalf("installed %v", inst.got)
|
|
}
|
|
if !strings.Contains(string(h.files["/etc/mysql/mariadb.conf.d/90-llmgw-slurm.cnf"]), "bind-address=127.0.0.1") {
|
|
t.Fatal("MariaDB, installed by llmgw, not kept to this machine")
|
|
}
|
|
dbd := string(h.files[slurm.DbdConfPath])
|
|
if !strings.Contains(dbd, "DbdHost=node06\n") || !strings.Contains(dbd, "StoragePass="+slurm.DbdPassword(key)+"\n") ||
|
|
h.owners[slurm.DbdConfPath] != "slurm" {
|
|
t.Fatalf("slurmdbd.conf for %q:\n%s", h.owners[slurm.DbdConfPath], dbd)
|
|
}
|
|
if _, ok := h.files[acctSQLPath]; ok {
|
|
t.Fatal("the database's statements, password and all, were left on disk")
|
|
}
|
|
if strings.Contains(strings.Join(h.ran, "\n"), slurm.DbdPassword(key)) {
|
|
t.Fatal("the password was on a command line")
|
|
}
|
|
order := strings.Join(h.ran, "\n")
|
|
if !strings.Contains(order, "sacctmgr -i add cluster llmgw") ||
|
|
strings.Index(order, "slurmdbd") > strings.Index(order, "systemctl restart slurmctld") {
|
|
t.Fatalf("accounting not before slurmctld: %v", h.ran)
|
|
}
|
|
if !st.GetSlurmdbdRunning() {
|
|
t.Fatalf("status %v", st)
|
|
}
|
|
sum := st.GetFilesSha256()
|
|
|
|
h.ran = nil
|
|
k.Reconcile(context.Background(), m)
|
|
if h.count("systemctl restart") != 0 {
|
|
t.Fatalf("restarted on the same files: %v", h.ran)
|
|
}
|
|
|
|
files["gres.conf"] = "NodeName=cuda01 Name=gpu File=/dev/nvidia0\n"
|
|
h.ran = nil
|
|
st = k.Reconcile(context.Background(), m)
|
|
if h.count("systemctl restart slurmctld") != 1 || st.GetFilesSha256() == sum {
|
|
t.Fatalf("a changed file did not restart slurmctld: %v", h.ran)
|
|
}
|
|
|
|
if st := k.Reconcile(context.Background(), &llmgwv1.Slurm{Controller: true, MungeKey: key,
|
|
Files: map[string]string{"slurm.conf": "x", "../evil": "y"}}); !strings.Contains(st.GetError(), "named") {
|
|
t.Fatalf("a path in a file name was taken: %v", st)
|
|
}
|
|
}
|
|
|
|
func TestTooOld(t *testing.T) {
|
|
h := newFake()
|
|
h.outputs["slurmd -V"] = "slurm-wlm 21.08.5\n"
|
|
k := New(h, &fakeInst{host: h}, "/m", nil)
|
|
st := k.Reconcile(context.Background(), nodeMsg(false))
|
|
if !strings.Contains(st.GetError(), "older than 22.05") {
|
|
t.Fatalf("status %v", st)
|
|
}
|
|
if _, ok := h.files[slurm.MungeKeyPath]; ok {
|
|
t.Fatal("went on with a Slurm too old")
|
|
}
|
|
}
|
|
|
|
func TestStop(t *testing.T) {
|
|
h := newFake()
|
|
k := New(h, &fakeInst{host: h}, "/m", nil)
|
|
k.Reconcile(context.Background(), nodeMsg(false))
|
|
h.ran = nil
|
|
if st := k.Reconcile(context.Background(), &llmgwv1.Slurm{}); st != nil {
|
|
t.Fatalf("status %v for no part", st)
|
|
}
|
|
if !slices.Contains(h.ran, "systemctl disable --now slurmd") {
|
|
t.Fatalf("slurmd not stopped: %v", h.ran)
|
|
}
|
|
if _, ok := h.files[DropIn]; ok {
|
|
t.Fatal("drop-in left")
|
|
}
|
|
// Never started here: nothing touched.
|
|
h2 := newFake()
|
|
k2 := New(h2, &fakeInst{host: h2}, "/m", nil)
|
|
k2.Reconcile(context.Background(), &llmgwv1.Slurm{})
|
|
if len(h2.ran) != 0 {
|
|
t.Fatalf("a machine llmgw never ran Slurm on was touched: %v", h2.ran)
|
|
}
|
|
}
|
|
|
|
func TestBadNames(t *testing.T) {
|
|
h := newFake()
|
|
k := New(h, &fakeInst{host: h}, "/m", nil)
|
|
m := nodeMsg(false)
|
|
m.Name = "cuda01 -c /etc/evil"
|
|
if st := k.Reconcile(context.Background(), m); !strings.Contains(st.GetError(), "spaces") {
|
|
t.Fatalf("status %v", st)
|
|
}
|
|
if st := k.Reconcile(context.Background(), &llmgwv1.Slurm{Node: true, Name: "a", ConfServer: "b"}); !strings.Contains(st.GetError(), "no key") {
|
|
t.Fatalf("status %v", st)
|
|
}
|
|
}
|
|
|
|
func TestAMariaDBThatWasThereKeepsItsSettings(t *testing.T) {
|
|
h := newFake()
|
|
for _, b := range []string{"munged", "scontrol", "slurmctld", "slurmdbd", "mariadbd", "mariadb", "exportfs", "mount.nfs"} {
|
|
h.bins[b] = true
|
|
}
|
|
k := New(h, &fakeInst{host: h}, "/m", nil)
|
|
st := k.Reconcile(context.Background(), &llmgwv1.Slurm{Controller: true, MungeKey: key,
|
|
Files: map[string]string{"slurm.conf": "ClusterName=llmgw\n"}})
|
|
if st.GetError() != "" {
|
|
t.Fatal(st.GetError())
|
|
}
|
|
for p := range h.files {
|
|
if strings.HasSuffix(p, ".cnf") {
|
|
t.Fatalf("a MariaDB llmgw did not install had its settings changed: %s", p)
|
|
}
|
|
}
|
|
if h.count("mariadb -e source") != 1 || !st.GetSlurmdbdRunning() {
|
|
t.Fatalf("the database was not made in the MariaDB that was there: %v", h.ran)
|
|
}
|
|
}
|