From 748f777a5627fc2130ea8ee9682ea0cac2db6a46 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Thu, 12 Nov 2026 07:54:13 +0100 Subject: [PATCH] Slurm's accounting: slurmdbd and a MariaDB on the controller Since 23.11 Slurm runs badly without accounting storage: the main scheduler passes every job by as InvalidAccount. slurmdbd takes MySQL or MariaDB only, so the controller's agent installs MariaDB beside it, answering that machine alone, makes the database and its user (the password made from the munge key, never on a command line), runs slurmdbd and adds the cluster, all before slurmctld. A MariaDB that was on the machine before keeps its settings. slurm.conf accounts the jobs there, with what each used from its cgroup and the cards, and the priority is multifactor again. Port 6819 joins Slurm's in the firewall. ctl slurm status and the console say whether slurmdbd runs; the install screen, server init --slurm and the guide say what is put where. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 7 +- api/gen/llmgw/v1/agent.pb.go | 22 ++- api/proto/llmgw/v1/agent.proto | 3 + docs/admin-api.openapi.json | 3 + docs/install.md | 13 +- internal/agent/firewall.go | 2 +- internal/agent/firewall_test.go | 2 +- internal/agent/slurmnode/dbd.go | 170 +++++++++++++++++++++ internal/agent/slurmnode/slurmnode.go | 38 ++++- internal/agent/slurmnode/slurmnode_test.go | 53 ++++++- internal/cli/ctl_slurm.go | 7 +- internal/cli/serverinit.go | 2 +- internal/cli/serverinit_choices.go | 3 +- internal/cli/tui_choices.go | 4 +- internal/cli/tui_slurm.go | 4 +- internal/cli/tui_slurm_test.go | 4 +- internal/control/slurmsvc/service.go | 3 +- internal/slurm/dbd.go | 81 ++++++++++ internal/slurm/slurm.go | 15 +- internal/slurm/slurm_test.go | 26 +++- 20 files changed, 425 insertions(+), 37 deletions(-) create mode 100644 internal/agent/slurmnode/dbd.go create mode 100644 internal/slurm/dbd.go diff --git a/CHANGELOG.md b/CHANGELOG.md index b102f7c..6a5321f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,8 +48,11 @@ APIs change between minors without a deprecation period. and gres.conf written from what each node reports. Every node stays llmgw's, drained in Slurm, until `ctl slurm lend NODE`: its replicas move off, then Slurm runs jobs there. `ctl slurm reclaim NODE` takes it back once its jobs - end. `ctl slurm status` shows each node. Slurm's ports (6817–6818, srun's - 60001–63000) answer the cluster's networks only, as llmgw's own do. + end. `ctl slurm status` and the console's **Slurm** show each node. The + controller also keeps the jobs' records: slurmdbd, in a MariaDB on that + machine alone (Slurm takes MySQL or MariaDB only). Slurm's ports + (6817–6819, srun's 60001–63000) answer the cluster's networks only, as + llmgw's own do. ### Changed diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index a89ce7f..9913d5f 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -5410,9 +5410,12 @@ type SlurmStatus struct { Error string `protobuf:"bytes,8,opt,name=error,proto3" json:"error,omitempty"` // files_sha256 is of the controller's files as written, so the leader // knows they arrived. - FilesSha256 string `protobuf:"bytes,9,opt,name=files_sha256,json=filesSha256,proto3" json:"files_sha256,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + FilesSha256 string `protobuf:"bytes,9,opt,name=files_sha256,json=filesSha256,proto3" json:"files_sha256,omitempty"` + // slurmdbd_running is the accounting daemon on the controller, which keeps + // the jobs' records in a MariaDB beside it. + SlurmdbdRunning bool `protobuf:"varint,10,opt,name=slurmdbd_running,json=slurmdbdRunning,proto3" json:"slurmdbd_running,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SlurmStatus) Reset() { @@ -5508,6 +5511,13 @@ func (x *SlurmStatus) GetFilesSha256() string { return "" } +func (x *SlurmStatus) GetSlurmdbdRunning() bool { + if x != nil { + return x.SlurmdbdRunning + } + return false +} + var File_llmgw_v1_agent_proto protoreflect.FileDescriptor const file_llmgw_v1_agent_proto_rawDesc = "" + @@ -6028,7 +6038,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\n" + "FilesEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x92\x02\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xbd\x02\n" + "\vSlurmStatus\x12\x18\n" + "\aversion\x18\x01 \x01(\tR\aversion\x12\x1a\n" + "\bhardware\x18\x02 \x01(\tR\bhardware\x12%\n" + @@ -6038,7 +6048,9 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x06reason\x18\x06 \x01(\tR\x06reason\x12\x12\n" + "\x04jobs\x18\a \x01(\x05R\x04jobs\x12\x14\n" + "\x05error\x18\b \x01(\tR\x05error\x12!\n" + - "\ffiles_sha256\x18\t \x01(\tR\vfilesSha2562O\n" + + "\ffiles_sha256\x18\t \x01(\tR\vfilesSha256\x12)\n" + + "\x10slurmdbd_running\x18\n" + + " \x01(\bR\x0fslurmdbdRunning2O\n" + "\fAgentService\x12?\n" + "\aConnect\x12\x16.llmgw.v1.AgentMessage\x1a\x18.llmgw.v1.ControlMessage(\x010\x01B\x93\x01\n" + "\fcom.llmgw.v1B\n" + diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index 42a5881..45318a2 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -1034,4 +1034,7 @@ message SlurmStatus { // files_sha256 is of the controller's files as written, so the leader // knows they arrived. string files_sha256 = 9; + // slurmdbd_running is the accounting daemon on the controller, which keeps + // the jobs' records in a MariaDB beside it. + bool slurmdbd_running = 10; } diff --git a/docs/admin-api.openapi.json b/docs/admin-api.openapi.json index 670e184..a8ba48e 100644 --- a/docs/admin-api.openapi.json +++ b/docs/admin-api.openapi.json @@ -12246,6 +12246,9 @@ "slurmctld_running": { "type": "boolean" }, + "slurmdbd_running": { + "type": "boolean" + }, "state": { "type": "string" }, diff --git a/docs/install.md b/docs/install.md index 69a38de..c56c4dd 100644 --- a/docs/install.md +++ b/docs/install.md @@ -177,14 +177,21 @@ init again to serve on it. choices screen, runs a Slurm cluster beside llmgw: slurmctld on this machine's node, slurmd on every node, those that join later too (`slurm.all_nodes`). The agents install munge and Slurm (22.05 or newer) -from the machines' own package sources. No node is Slurm's at first: each is +from the machines' own package sources, and on this machine slurmdbd, which +keeps the jobs' records (`sacct`), in a MariaDB installed beside it that +answers this machine alone. Slurm keeps its records in MySQL or MariaDB +only, not in the cluster's PostgreSQL; the database's password is made from +the cluster's munge key, so nothing new is stored. A MariaDB the machine +already had keeps its settings and only gets the database `slurm_acct_db` +and its user. Back it up with the machine (`mariadb-dump slurm_acct_db`); +records older than a year are purged. No node is Slurm's at first: each is drained in Slurm and serves models until `llmgw ctl slurm lend NODE` gives it to Slurm — its models move off first — and `llmgw ctl slurm reclaim NODE` takes it back once its jobs end. Jobs are submitted on this machine with `sbatch` and `srun`; `llmgw ctl slurm status` shows where each node is. It needs this machine to be a node, and its hostname to be a name Slurm takes. On a cluster installed without it, `llmgw ctl slurm setup --controller NODE ---all-nodes` does the same later. Slurm's ports (6817–6818, and srun's 60001–63000) answer the cluster's +--all-nodes` does the same later. Slurm's ports (6817–6819, and srun's 60001–63000) answer the cluster's networks only, as llmgw's own do; a firewall of the site's own between the nodes must let them through. @@ -3160,7 +3167,7 @@ loaded at boot by `llmgw-firewall.service`): | Machine | Ports, for the cluster only | | --- | --- | | control plane | 7443 (agents), metrics when not on loopback, 8081 (its mirror), engines and 9100 when it is a node, 2379/2380 when it holds the store's third etcd vote | -| node | engine ports (`control.port_range`, 20000–20999), 9100; with a part in Slurm also 6817–6818 (slurmctld, slurmd) and 60001–63000 (srun's `SrunPortRange`) | +| node | engine ports (`control.port_range`, 20000–20999), 9100; with a part in Slurm also 6817–6819 (slurmctld, slurmd, slurmdbd) and 60001–63000 (srun's `SrunPortRange`) | | store member | 5432, 8008, 2379, 2380, 9100 | | mirror | 8081, 8082, 9100 | diff --git a/internal/agent/firewall.go b/internal/agent/firewall.go index fb2e726..bc9193b 100644 --- a/internal/agent/firewall.go +++ b/internal/agent/firewall.go @@ -16,7 +16,7 @@ import ( // slurmPorts are Slurm's on a node that runs any of it: slurmctld's and // slurmd's, and srun's range for the steps it starts. var slurmPorts = []firewall.Port{ - {From: slurm.CtldPort, To: slurm.SlurmdPort, What: "slurm: slurmctld and slurmd"}, + {From: slurm.CtldPort, To: slurm.DbdPort, What: "slurm: slurmctld, slurmd and slurmdbd"}, {From: slurm.SrunPortsFrom, To: slurm.SrunPortsTo, What: "slurm: srun, for the steps it starts"}, } diff --git a/internal/agent/firewall_test.go b/internal/agent/firewall_test.go index 471015d..9a0df94 100644 --- a/internal/agent/firewall_test.go +++ b/internal/agent/firewall_test.go @@ -73,7 +73,7 @@ func TestANodeGivenAPartInSlurmLetsSlurmInFromTheClusterOnly(t *testing.T) { a.fwMu.Lock() got := sh.installed a.fwMu.Unlock() - if strings.Contains(got, "6817-6818") && strings.Contains(got, "60001-63000") { + if strings.Contains(got, "6817-6819") && strings.Contains(got, "60001-63000") { break } if time.Now().After(deadline) { diff --git a/internal/agent/slurmnode/dbd.go b/internal/agent/slurmnode/dbd.go new file mode 100644 index 0000000..945d31a --- /dev/null +++ b/internal/agent/slurmnode/dbd.go @@ -0,0 +1,170 @@ +package slurmnode + +import ( + "bufio" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "strings" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/slurm" +) + +// mariaDBConfDirs are where each family's MariaDB reads its settings: Debian +// and Ubuntu's, then Red Hat's. +var mariaDBConfDirs = []string{"/etc/mysql/mariadb.conf.d", "/etc/my.cnf.d"} + +const mariaDBConfName = "90-llmgw-slurm.cnf" + +// acctSQLPath is where the database's statements are put for the client to +// read, never on its command line: they hold the password. +const acctSQLPath = "/run/llmgw-slurm-acct.sql" + +// accounting keeps slurmdbd on the controller: MariaDB on this machine +// alone, the database and its user, slurmdbd on them, and the cluster in it. +// Before slurmctld, which reports to it from its first start. +func (k *Keeper) accounting(ctx context.Context, key []byte, files map[string]string) error { + password := slurm.DbdPassword(key) + changed, err := k.mariaDBSettings(ctx) + if err != nil { + return err + } + switch { + case changed: + k.log.Info("MariaDB, for Slurm's accounting, starts on this machine alone") + if err := k.systemctl(ctx, "enable", "--now", "mariadb", "&&restart"); err != nil { + return err + } + case !k.active(ctx, "mariadb"): + if err := k.systemctl(ctx, "enable", "--now", "mariadb"); err != nil { + return err + } + } + client, err := k.mariaDBClient() + if err != nil { + return err + } + if err := k.waitFor(ctx, "MariaDB", client, "-e", "SELECT 1"); err != nil { + return err + } + // The database and its user, once each time the agent starts or the + // password is new: cheap, and a database restored from elsewhere gets + // its user back. + sum := sha256.Sum256([]byte(password)) + if mark := hex.EncodeToString(sum[:]); k.acctDone != mark { + if err := k.host.WriteFile(acctSQLPath, []byte(slurm.AcctSQL(password)), 0o600, ""); err != nil { + return err + } + out, err := k.host.Run(ctx, nil, client, "-e", "source "+acctSQLPath) + _ = k.host.Remove(acctSQLPath) + if err != nil { + return fmt.Errorf("making Slurm's database: %v: %s", err, lastLine(out)) + } + k.acctDone = mark + } + + out, err := k.host.Run(ctx, nil, "hostname", "-s") + host := strings.TrimSpace(string(out)) + if err != nil || host == "" { + return fmt.Errorf("this machine's short hostname: %v: %s", err, lastLine(out)) + } + changed, err = k.write(slurm.DbdConfPath, []byte(slurm.DbdConf(host, password)), 0o600, "slurm") + if err != nil { + return err + } + switch { + case changed: + k.log.Info("slurmdbd starts again on new settings") + if err := k.systemctl(ctx, "enable", "--now", "slurmdbd", "&&restart"); err != nil { + return err + } + case !k.active(ctx, "slurmdbd"): + if err := k.systemctl(ctx, "enable", "--now", "slurmdbd"); err != nil { + return err + } + } + if err := k.waitFor(ctx, "slurmdbd", "sacctmgr", "-n", "-P", "show", "cluster", "format=cluster"); err != nil { + return err + } + name := clusterName(files["slurm.conf"]) + if name == "" || k.clusterAdded == name { + return nil + } + out, err = k.host.Run(ctx, nil, "sacctmgr", "-n", "-P", "show", "cluster", "format=cluster") + if err != nil { + return fmt.Errorf("sacctmgr show cluster: %v: %s", err, lastLine(out)) + } + for _, l := range strings.Split(string(out), "\n") { + if strings.TrimSpace(l) == name { + k.clusterAdded = name + return nil + } + } + k.log.Info("the cluster is added to Slurm's accounting", "cluster", name) + if out, err := k.host.Run(ctx, nil, "sacctmgr", "-i", "add", "cluster", name); err != nil { + return fmt.Errorf("sacctmgr add cluster %s: %v: %s", name, err, lastLine(out)) + } + k.clusterAdded = name + return nil +} + +// mariaDBSettings keeps MariaDB to this machine, when llmgw installed it; +// one that was here before llmgw keeps its own settings, and only gets the +// database and its user. +func (k *Keeper) mariaDBSettings(ctx context.Context) (bool, error) { + if _, err := k.host.ReadFile(k.mariaDBMarker()); err != nil { + return false, nil + } + for _, d := range mariaDBConfDirs { + if _, err := k.host.Run(ctx, nil, "test", "-d", d); err == nil { + return k.write(d+"/"+mariaDBConfName, []byte(slurm.MariaDBConf), 0o644, "") + } + } + return false, fmt.Errorf("MariaDB has none of %s to put its settings in", strings.Join(mariaDBConfDirs, ", ")) +} + +// mariaDBMarker is there when llmgw installed MariaDB on this machine. +func (k *Keeper) mariaDBMarker() string { return k.marker + "-mariadb" } + +// mariaDBClient is MariaDB's command-line client, under either of its +// names. +func (k *Keeper) mariaDBClient() (string, error) { + for _, c := range []string{"mariadb", "mysql"} { + if _, err := k.host.LookPath(c); err == nil { + return c, nil + } + } + return "", errors.New("MariaDB's client (mariadb or mysql) is not installed") +} + +// waitFor runs a command until it succeeds, half a minute at most. +func (k *Keeper) waitFor(ctx context.Context, what, name string, args ...string) error { + var last []byte + for range 30 { + out, err := k.host.Run(ctx, nil, name, args...) + if err == nil { + return nil + } + last = out + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(pingWait): + } + } + return fmt.Errorf("%s did not answer after starting: %s", what, lastLine(last)) +} + +// clusterName is ClusterName in slurm.conf. +func clusterName(conf string) string { + sc := bufio.NewScanner(strings.NewReader(conf)) + for sc.Scan() { + if v, ok := strings.CutPrefix(strings.TrimSpace(sc.Text()), "ClusterName="); ok { + return strings.TrimSpace(v) + } + } + return "" +} diff --git a/internal/agent/slurmnode/slurmnode.go b/internal/agent/slurmnode/slurmnode.go index 49e1837..8f15ae6 100644 --- a/internal/agent/slurmnode/slurmnode.go +++ b/internal/agent/slurmnode/slurmnode.go @@ -77,6 +77,10 @@ type Keeper struct { // minute's check does not look again. installed map[string]bool files string + // acctDone is the digest of the password the database's user was last + // given; clusterAdded the cluster last found in the accounting. + acctDone string + clusterAdded string } // New is a Keeper; marker is a file it keeps while it runs Slurm here, so @@ -207,6 +211,19 @@ func (k *Keeper) Reconcile(ctx context.Context, want *llmgwv1.Slurm) *llmgwv1.Sl return fail("munge", err) } if want.GetController() { + // Accounting first: slurmctld reports to it from its first start. + // Without it slurmctld still runs jobs, a pass of backfill late + // each; so a failure here is said, and slurmctld is kept anyway. + var acctErr error + if acctErr = k.accounting(ctx, want.GetMungeKey(), want.GetFiles()); acctErr != nil { + k.log.Warn("keeping Slurm's accounting on this node", "err", acctErr) + } + st.SlurmdbdRunning = k.active(ctx, "slurmdbd") + defer func() { + if acctErr != nil && st.Error == "" { + st.Error = "slurmdbd: " + acctErr.Error() + } + }() sum, err := k.controller(ctx, want.GetFiles()) st.FilesSha256 = sum if err != nil { @@ -229,6 +246,8 @@ func (k *Keeper) ensurePackages(ctx context.Context, want *llmgwv1.Slurm) error } if want.GetController() { need["slurmctld"] = [2]string{"slurmctld", "slurm-slurmctld"} + need["slurmdbd"] = [2]string{"slurmdbd", "slurm-slurmdbd"} + need["mariadbd"] = [2]string{"mariadb-server", "mariadb-server"} } if want.GetNode() { need["slurmd"] = [2]string{"slurmd", "slurm-slurmd"} @@ -257,6 +276,14 @@ func (k *Keeper) ensurePackages(ctx context.Context, want *llmgwv1.Slurm) error if err := k.inst.InstallPackages(ctx, pkgs); err != nil { return err } + if slices.Contains(pkgs["deb"], "mariadb-server") { + // Installed by llmgw: its settings are llmgw's to keep. One that was + // here before is another's, and is left as it is. + if err := k.host.WriteFile(k.mariaDBMarker(), []byte("llmgw installed MariaDB here, for Slurm's accounting\n"), + 0o644, ""); err != nil { + return err + } + } for bin := range need { if _, err := k.host.LookPath(bin); err != nil { return fmt.Errorf("%s is still missing after installing %s", bin, strings.Join(pkgs["deb"], ", ")) @@ -483,14 +510,16 @@ func (k *Keeper) stop(ctx context.Context) { if _, err := k.host.ReadFile(k.marker); err != nil { return } - k.log.Info("this node has no part in Slurm any more: slurmd and slurmctld stopped; the packages stay") - for _, u := range []string{"slurmd", "slurmctld"} { + k.log.Info("this node has no part in Slurm any more: slurmd, slurmctld and slurmdbd stopped; the packages and MariaDB stay") + // MariaDB stays: it may have been the machine's before, and holds the + // jobs' records. + for _, u := range []string{"slurmd", "slurmctld", "slurmdbd"} { _ = k.systemctl(ctx, "disable", "--now", u) } _ = k.host.Remove(DropIn) _ = k.systemctl(ctx, "daemon-reload") _ = k.host.Remove(k.marker) - k.files = "" + k.files, k.acctDone, k.clusterAdded = "", "", "" } // write writes a file when it differs, saying whether it did. @@ -551,7 +580,8 @@ func (Local) LookPath(name string) (string, error) { if p, err := exec.LookPath(name); err == nil { return p, nil } - for _, d := range []string{"/usr/sbin", "/sbin", "/usr/local/sbin"} { + // /usr/libexec: Red Hat's mariadbd. + for _, d := range []string{"/usr/sbin", "/sbin", "/usr/local/sbin", "/usr/libexec"} { p := filepath.Join(d, name) if fi, err := os.Stat(p); err == nil && !fi.IsDir() { return p, nil diff --git a/internal/agent/slurmnode/slurmnode_test.go b/internal/agent/slurmnode/slurmnode_test.go index c1dc80c..2f70c0d 100644 --- a/internal/agent/slurmnode/slurmnode_test.go +++ b/internal/agent/slurmnode/slurmnode_test.go @@ -32,6 +32,7 @@ func newFake() *fakeHost { "slurmctld -V": "slurm-wlm 23.11.4\n", "scontrol ping": "Slurmctld(primary) at node06 is UP\n", "slurmd -C": "NodeName=cuda01 CPUs=16 RealMemory=64000\nUpTime=1-00:00:00\n", + "hostname -s": "node06\n", }, fail: map[string]bool{}, state: "IDLE+DRAIN", reason: slurm.ReasonNew} } @@ -118,7 +119,7 @@ type fakeInst struct { func (i *fakeInst) InstallPackages(_ context.Context, by map[string][]string) error { i.got = by - for _, b := range []string{"munged", "scontrol", "slurmd", "slurmctld"} { + for _, b := range []string{"munged", "scontrol", "slurmd", "slurmctld", "slurmdbd", "mariadbd", "mariadb"} { i.host.bins[b] = true } return nil @@ -237,6 +238,33 @@ func TestController(t *testing.T) { if !st.GetSlurmctldRunning() || len(st.GetFilesSha256()) != 64 { t.Fatalf("status %v", st) } + // Accounting: MariaDB on this machine alone, the database, slurmdbd on + // it, the cluster added, all before slurmctld. + if !slices.Contains(inst.got["deb"], "slurmdbd") || !slices.Contains(inst.got["deb"], "mariadb-server") { + t.Fatalf("installed %v", inst.got) + } + if !strings.Contains(string(h.files["/etc/mysql/mariadb.conf.d/90-llmgw-slurm.cnf"]), "bind-address=127.0.0.1") { + t.Fatal("MariaDB, installed by llmgw, not kept to this machine") + } + dbd := string(h.files[slurm.DbdConfPath]) + if !strings.Contains(dbd, "DbdHost=node06\n") || !strings.Contains(dbd, "StoragePass="+slurm.DbdPassword(key)+"\n") || + h.owners[slurm.DbdConfPath] != "slurm" { + t.Fatalf("slurmdbd.conf for %q:\n%s", h.owners[slurm.DbdConfPath], dbd) + } + if _, ok := h.files[acctSQLPath]; ok { + t.Fatal("the database's statements, password and all, were left on disk") + } + if strings.Contains(strings.Join(h.ran, "\n"), slurm.DbdPassword(key)) { + t.Fatal("the password was on a command line") + } + order := strings.Join(h.ran, "\n") + if !strings.Contains(order, "sacctmgr -i add cluster llmgw") || + strings.Index(order, "slurmdbd") > strings.Index(order, "systemctl restart slurmctld") { + t.Fatalf("accounting not before slurmctld: %v", h.ran) + } + if !st.GetSlurmdbdRunning() { + t.Fatalf("status %v", st) + } sum := st.GetFilesSha256() h.ran = nil @@ -279,7 +307,7 @@ func TestStop(t *testing.T) { if st := k.Reconcile(context.Background(), &llmgwv1.Slurm{}); st != nil { t.Fatalf("status %v for no part", st) } - if h.count("systemctl disable --now slurmd") != 1 { + if !slices.Contains(h.ran, "systemctl disable --now slurmd") { t.Fatalf("slurmd not stopped: %v", h.ran) } if _, ok := h.files[DropIn]; ok { @@ -306,3 +334,24 @@ func TestBadNames(t *testing.T) { t.Fatalf("status %v", st) } } + +func TestAMariaDBThatWasThereKeepsItsSettings(t *testing.T) { + h := newFake() + for _, b := range []string{"munged", "scontrol", "slurmctld", "slurmdbd", "mariadbd", "mariadb"} { + h.bins[b] = true + } + k := New(h, &fakeInst{host: h}, "/m", nil) + st := k.Reconcile(context.Background(), &llmgwv1.Slurm{Controller: true, MungeKey: key, + Files: map[string]string{"slurm.conf": "ClusterName=llmgw\n"}}) + if st.GetError() != "" { + t.Fatal(st.GetError()) + } + for p := range h.files { + if strings.HasSuffix(p, ".cnf") { + t.Fatalf("a MariaDB llmgw did not install had its settings changed: %s", p) + } + } + if h.count("mariadb -e source") != 1 || !st.GetSlurmdbdRunning() { + t.Fatalf("the database was not made in the MariaDB that was there: %v", h.ran) + } +} diff --git a/internal/cli/ctl_slurm.go b/internal/cli/ctl_slurm.go index cc7e604..1ba5abe 100644 --- a/internal/cli/ctl_slurm.go +++ b/internal/cli/ctl_slurm.go @@ -55,8 +55,10 @@ func newSlurmStatusCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { fmt.Fprintln(w, ": not heard from yet") case c.Error != "": fmt.Fprintf(w, ": %s\n", c.Error) + case c.SlurmctldRunning && c.SlurmdbdRunning: + fmt.Fprintf(w, ": slurmctld %s and slurmdbd running\n", c.Version) case c.SlurmctldRunning: - fmt.Fprintf(w, ": slurmctld %s running\n", c.Version) + fmt.Fprintf(w, ": slurmctld %s running, slurmdbd not (the jobs' records)\n", c.Version) default: fmt.Fprintln(w, ": slurmctld not running") } @@ -96,7 +98,8 @@ func newSlurmSetupCmd(client func() *ctlClient) *cobra.Command { Use: "setup --controller NODE --nodes NODE,... | --all-nodes", Short: "Run Slurm on llmgw's nodes: slurmctld on one, slurmd on those named", Long: "Turns slurm.enabled on and names the nodes, or with --all-nodes every node. Their agents install munge and Slurm (22.05 or\n" + - "newer) from the machines' own package sources. Every node starts drained in Slurm and stays\n" + + "newer) from the machines' own package sources; the controller also slurmdbd, the jobs' records, in a\n" + + "MariaDB on that machine alone (one that was there keeps its settings). Every node starts drained in Slurm and stays\n" + "llmgw's until `llmgw ctl slurm lend` gives it to Slurm.", Example: " llmgw ctl slurm setup --controller node06 --nodes cuda01,cuda02\n" + " llmgw ctl slurm setup --controller node06 --all-nodes", diff --git a/internal/cli/serverinit.go b/internal/cli/serverinit.go index bb1da4d..22d2f90 100644 --- a/internal/cli/serverinit.go +++ b/internal/cli/serverinit.go @@ -262,7 +262,7 @@ func newServerInitCmd() *cobra.Command { f.BoolVar(&in.media, "media", false, "pictures and 3D: the nodes install ComfyUI, and a picture model and a 3D model are added once a card fits one") f.BoolVar(&in.slurm, "slurm", false, - "Slurm for training and batch jobs: slurmctld on this machine, slurmd on every node, each llmgw's until 'llmgw ctl slurm lend'") + "Slurm for training and batch jobs: slurmctld and slurmdbd (with a MariaDB) on this machine, slurmd on every node, each llmgw's until 'llmgw ctl slurm lend'") f.DurationVar(&in.evictAfterIdle, "evict-after-idle", 0, "how long a model loaded on demand goes unasked-for before another may have its card (0: 2m)") f.DurationVar(&in.minLifetime, "min-lifetime", 0, diff --git a/internal/cli/serverinit_choices.go b/internal/cli/serverinit_choices.go index bc0ef9f..c28fc9f 100644 --- a/internal/cli/serverinit_choices.go +++ b/internal/cli/serverinit_choices.go @@ -350,7 +350,8 @@ func (in *serverInit) tuneConfig(body string) string { // slurmSection is server.yaml's slurm block as an install chooses it: // slurmctld on this machine's node, slurmd on every node. func slurmSection(controller string) string { - return "# Slurm beside llmgw, chosen at install: slurmctld on this machine, slurmd on\n" + + return "# Slurm beside llmgw, chosen at install: slurmctld, and slurmdbd with a MariaDB\n" + + "# for the jobs' records, on this machine; slurmd on\n" + "# every node, those that join later too. Each node is llmgw's, drained in\n" + "# Slurm, until 'llmgw ctl slurm lend NODE' gives it to Slurm; 'llmgw ctl slurm\n" + "# reclaim NODE' takes it back once its jobs end.\n" + diff --git a/internal/cli/tui_choices.go b/internal/cli/tui_choices.go index ead679d..0a49056 100644 --- a/internal/cli/tui_choices.go +++ b/internal/cli/tui_choices.go @@ -287,7 +287,9 @@ func newInstallForm(in *serverInit, host string, found []string, ha bool) *insta title: "Slurm", on: in.slurm, about: "A Slurm cluster beside llmgw, for training and other batch work: slurmctld on this machine, " + "slurmd on every node, those that join later too. The agents install munge and Slurm from the " + - "machines' own package sources.\n\n" + + "machines' own package sources, and on this machine slurmdbd, the jobs' records, with a MariaDB " + + "that answers this machine alone (Slurm keeps its records in MySQL or MariaDB only, not in the " + + "cluster's PostgreSQL). Its port 6819, with Slurm's others, answers the cluster's networks only.\n\n" + "Each node stays llmgw's, drained in Slurm, until 'llmgw ctl slurm lend NODE' gives it to Slurm: " + "its models move off first. 'llmgw ctl slurm reclaim NODE' takes it back once its jobs end. Jobs " + "are submitted on this machine with sbatch and srun.", diff --git a/internal/cli/tui_slurm.go b/internal/cli/tui_slurm.go index 11f42ae..6cb845d 100644 --- a/internal/cli/tui_slurm.go +++ b/internal/cli/tui_slurm.go @@ -35,8 +35,10 @@ func (m consoleModel) slurmPanel(pw int) (string, string, string) { case c == nil: case c.Error != "": ctl = sBad.Render(c.Error) + case c.SlurmctldRunning && c.SlurmdbdRunning: + ctl = sOK.Render("slurmctld " + c.Version + " and slurmdbd running") case c.SlurmctldRunning: - ctl = sOK.Render("slurmctld " + c.Version + " running") + ctl = sOK.Render("slurmctld "+c.Version+" running") + ", " + sWarn.Render("slurmdbd not") default: ctl = sBad.Render("slurmctld not running") } diff --git a/internal/cli/tui_slurm_test.go b/internal/cli/tui_slurm_test.go index b6d03c2..87393ce 100644 --- a/internal/cli/tui_slurm_test.go +++ b/internal/cli/tui_slurm_test.go @@ -37,7 +37,7 @@ func TestTheConsoleLendsANodeToSlurmFromItsRow(t *testing.T) { m := newConsoleModel(srv.URL, "/nonexistent/server.yaml") m.loaded, m.menu, m.inPanel = true, cSlurm, true m.data.slurm = slurmsvc.Status{Enabled: true, Cluster: "llmgw", Controller: "node06", Partition: "gpu", - Ctld: &slurmsvc.Reported{SlurmctldRunning: true, Version: "23.11.4"}, + Ctld: &slurmsvc.Reported{SlurmctldRunning: true, SlurmdbdRunning: true, Version: "23.11.4"}, Nodes: []slurmsvc.NodeStatus{ {Node: "node05", Phase: slurmsvc.Lent, Connected: true, Status: &slurmsvc.Reported{State: "MIXED", Jobs: 2, SlurmdRunning: true}}, @@ -46,7 +46,7 @@ func TestTheConsoleLendsANodeToSlurmFromItsRow(t *testing.T) { }} var tm tea.Model = m view := tm.View() - for _, want := range []string{"slurmctld 23.11.4 running", "node05", "Slurm's", "MIXED", "cuda01", "llmgw's"} { + for _, want := range []string{"slurmctld 23.11.4 and slurmdbd running", "node05", "Slurm's", "MIXED", "cuda01", "llmgw's"} { if !strings.Contains(view, want) { t.Errorf("the Slurm view has no %q:\n%s", want, view) } diff --git a/internal/control/slurmsvc/service.go b/internal/control/slurmsvc/service.go index 839014e..53cabd9 100644 --- a/internal/control/slurmsvc/service.go +++ b/internal/control/slurmsvc/service.go @@ -75,6 +75,7 @@ type Reported struct { Hardware string `json:"hardware,omitempty"` SlurmdRunning bool `json:"slurmd_running"` SlurmctldRunning bool `json:"slurmctld_running"` + SlurmdbdRunning bool `json:"slurmdbd_running"` State string `json:"state,omitempty"` Reason string `json:"reason,omitempty"` Jobs int `json:"jobs"` @@ -197,7 +198,7 @@ func (s *Service) Kick() { // OnSlurmStatus records what a node's agent says. func (s *Service) OnSlurmStatus(node string, st *llmgwv1.SlurmStatus) { r := &Reported{Version: st.GetVersion(), Hardware: st.GetHardware(), SlurmdRunning: st.GetSlurmdRunning(), - SlurmctldRunning: st.GetSlurmctldRunning(), State: st.GetState(), Reason: st.GetReason(), + SlurmctldRunning: st.GetSlurmctldRunning(), SlurmdbdRunning: st.GetSlurmdbdRunning(), State: st.GetState(), Reason: st.GetReason(), Jobs: int(st.GetJobs()), Error: st.GetError(), FilesSHA256: st.GetFilesSha256(), At: time.Now()} s.mu.Lock() prev := s.status[node] diff --git a/internal/slurm/dbd.go b/internal/slurm/dbd.go new file mode 100644 index 0000000..a4b5835 --- /dev/null +++ b/internal/slurm/dbd.go @@ -0,0 +1,81 @@ +package slurm + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" +) + +// Accounting: slurmdbd on the controller, keeping the jobs' records in a +// MariaDB beside it that answers this machine alone. slurmdbd takes MySQL or +// MariaDB only, so it is not the cluster's PostgreSQL. Since 23.11 Slurm +// cannot run without it well: its main scheduler passes every job by as +// InvalidAccount, and backfill starts it a pass later. +const ( + // DbdPort is slurmdbd's, which slurmctld and sacct reach. + DbdPort = 6819 + // DbdConfPath is slurmdbd's file, readable by the slurm user alone: it + // holds the database's password. + DbdConfPath = ConfDir + "/slurmdbd.conf" + // AcctDB and AcctUser are the database and its user. + AcctDB = "slurm_acct_db" + AcctUser = "slurm" +) + +// DbdPassword is the database user's password, made from the munge key: +// every controller the cluster has gets the same one, and nothing new is +// kept anywhere. +func DbdPassword(mungeKey []byte) string { + h := sha256.New() + h.Write([]byte("llmgw slurmdbd password\x00")) + h.Write(mungeKey) + return hex.EncodeToString(h.Sum(nil))[:40] +} + +// DbdConf is slurmdbd.conf for the controller host, named as `hostname -s` +// says, which slurmdbd checks it runs on. +func DbdConf(host, password string) string { + return header + fmt.Sprintf(`AuthType=auth/munge +DbdHost=%s +DbdPort=%d +SlurmUser=slurm +LogFile=%s/slurmdbd.log +PidFile=/run/slurmdbd.pid +StorageType=accounting_storage/mysql +StorageHost=localhost +StorageUser=%s +StoragePass=%s +StorageLoc=%s +# Kept a year; older records are purged, a month at a time. +PurgeEventAfter=12months +PurgeJobAfter=12months +PurgeResvAfter=12months +PurgeStepAfter=12months +PurgeSuspendAfter=12months +PurgeTXNAfter=12months +PurgeUsageAfter=24months +`, host, DbdPort, LogDir, AcctUser, password, AcctDB) +} + +// MariaDBConf is the database server's settings: this machine alone, and +// what Slurm's documentation asks of InnoDB. +const MariaDBConf = `# Written by llmgw (BACKLOG P): the database of Slurm's accounting +# (slurmdbd), on this machine alone. +[mysqld] +bind-address=127.0.0.1 +skip-networking=0 +innodb_buffer_pool_size=256M +innodb_lock_wait_timeout=900 +` + +// AcctSQL makes the database and its user, and gives the user the password +// again: run on every start of the controller's part, it changes nothing +// that is already so. +func AcctSQL(password string) string { + return fmt.Sprintf(`CREATE DATABASE IF NOT EXISTS %[1]s; +CREATE USER IF NOT EXISTS '%[2]s'@'localhost' IDENTIFIED BY '%[3]s'; +ALTER USER '%[2]s'@'localhost' IDENTIFIED BY '%[3]s'; +GRANT ALL ON %[1]s.* TO '%[2]s'@'localhost'; +FLUSH PRIVILEGES; +`, AcctDB, AcctUser, password) +} diff --git a/internal/slurm/slurm.go b/internal/slurm/slurm.go index 411d00f..07c1e52 100644 --- a/internal/slurm/slurm.go +++ b/internal/slurm/slurm.go @@ -196,6 +196,9 @@ func (c Cluster) SlurmConf() string { b.WriteString(header) fmt.Fprintf(&b, "ClusterName=%s\n", c.Name) fmt.Fprintf(&b, "SlurmctldHost=%s(%s)\n", c.Controller, c.ControllerAddr) + // The jobs' records, in slurmdbd beside slurmctld (DbdConf). + fmt.Fprintf(&b, "AccountingStorageType=accounting_storage/slurmdbd\nAccountingStorageHost=%s\nAccountingStoragePort=%d\n", + c.ControllerAddr, DbdPort) b.WriteString(`SlurmctldParameters=enable_configless SlurmctldPort=6817 SlurmdPort=6818 @@ -212,17 +215,13 @@ SlurmctldPidFile=/run/slurmctld.pid SlurmdPidFile=/run/slurmd.pid ProctrackType=proctrack/cgroup TaskPlugin=task/cgroup,task/affinity -# Nothing gathered for accounting until there is somewhere to keep it -# (slurmdbd, later); gathered and not kept is an error in every log. -JobAcctGatherType=jobacct_gather/none +# What each job used, from its cgroup, for the records; the cards too. +JobAcctGatherType=jobacct_gather/cgroup +AccountingStorageTRES=gres/gpu SelectType=select/cons_tres SelectTypeParameters=CR_Core_Memory SchedulerType=sched/backfill -# First come, first served: multifactor's fair share needs accounting -# storage (slurmdbd), which there is none of yet. Without it, 23.11's main -# scheduler passes every job by as InvalidAccount, and backfill starts it -# on its next pass (bf_interval above). -PriorityType=priority/basic +PriorityType=priority/multifactor GresTypes=gpu MpiDefault=none # No mail sent; without it every start says MailProg is invalid. diff --git a/internal/slurm/slurm_test.go b/internal/slurm/slurm_test.go index 22caf93..f6698e3 100644 --- a/internal/slurm/slurm_test.go +++ b/internal/slurm/slurm_test.go @@ -165,7 +165,29 @@ func TestALentNodeIsNotWrittenDrained(t *testing.T) { if line == "" || strings.Contains(line, "DRAIN") { t.Fatalf("a lent node, written %q: every start of slurmctld would drain it again", line) } - if !strings.Contains(conf, "PriorityType=priority/basic\n") { - t.Error("multifactor without accounting: the main scheduler passes every job by as InvalidAccount") +} + +func TestTheJobsAreAccountedInSlurmdbd(t *testing.T) { + conf := testCluster().SlurmConf() + for _, want := range []string{ + "AccountingStorageType=accounting_storage/slurmdbd\n", + "AccountingStorageHost=10.0.0.146\n", + "AccountingStoragePort=6819\n", + "JobAcctGatherType=jobacct_gather/cgroup\n", + "AccountingStorageTRES=gres/gpu\n", + } { + if !strings.Contains(conf, want) { + t.Errorf("slurm.conf has no %q", want) + } + } + dbd := DbdConf("node06", DbdPassword([]byte("key"))) + for _, want := range []string{"DbdHost=node06\n", "DbdPort=6819\n", "StorageType=accounting_storage/mysql\n", + "StorageHost=localhost\n", "StorageLoc=slurm_acct_db\n", "AuthType=auth/munge\n"} { + if !strings.Contains(dbd, want) { + t.Errorf("slurmdbd.conf has no %q:\n%s", want, dbd) + } + } + if DbdPassword([]byte("key")) == DbdPassword([]byte("other")) || len(DbdPassword([]byte("key"))) != 40 { + t.Error("the password does not follow the key") } }