Slurm's accounting: slurmdbd and a MariaDB on the controller

Since 23.11 Slurm runs badly without accounting storage: the main
scheduler passes every job by as InvalidAccount. slurmdbd takes MySQL or
MariaDB only, so the controller's agent installs MariaDB beside it,
answering that machine alone, makes the database and its user (the
password made from the munge key, never on a command line), runs
slurmdbd and adds the cluster, all before slurmctld. A MariaDB that was
on the machine before keeps its settings. slurm.conf accounts the jobs
there, with what each used from its cgroup and the cards, and the
priority is multifactor again. Port 6819 joins Slurm's in the firewall.
ctl slurm status and the console say whether slurmdbd runs; the install
screen, server init --slurm and the guide say what is put where.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

CHANGELOG.md

@@ -48,8 +48,11 @@ APIs change between minors without a deprecation period.
   and gres.conf written from what each node reports. Every node stays llmgw's,
   drained in Slurm, until `ctl slurm lend NODE`: its replicas move off, then
   Slurm runs jobs there. `ctl slurm reclaim NODE` takes it back once its jobs
-  end. `ctl slurm status` shows each node. Slurm's ports (6817–6818, srun's
-  60001–63000) answer the cluster's networks only, as llmgw's own do.
+  end. `ctl slurm status` and the console's **Slurm** show each node. The
+  controller also keeps the jobs' records: slurmdbd, in a MariaDB on that
+  machine alone (Slurm takes MySQL or MariaDB only). Slurm's ports
+  (6817–6819, srun's 60001–63000) answer the cluster's networks only, as
+  llmgw's own do.
 
 ### Changed
 

api/gen/llmgw/v1/agent.pb.go

@@ -5410,9 +5410,12 @@ type SlurmStatus struct {
 	Error string `protobuf:"bytes,8,opt,name=error,proto3" json:"error,omitempty"`
 	// files_sha256 is of the controller's files as written, so the leader
 	// knows they arrived.
-	FilesSha256   string `protobuf:"bytes,9,opt,name=files_sha256,json=filesSha256,proto3" json:"files_sha256,omitempty"`
-	unknownFields protoimpl.UnknownFields
-	sizeCache     protoimpl.SizeCache
+	FilesSha256 string `protobuf:"bytes,9,opt,name=files_sha256,json=filesSha256,proto3" json:"files_sha256,omitempty"`
+	// slurmdbd_running is the accounting daemon on the controller, which keeps
+	// the jobs' records in a MariaDB beside it.
+	SlurmdbdRunning bool `protobuf:"varint,10,opt,name=slurmdbd_running,json=slurmdbdRunning,proto3" json:"slurmdbd_running,omitempty"`
+	unknownFields   protoimpl.UnknownFields
+	sizeCache       protoimpl.SizeCache
 }
 
 func (x *SlurmStatus) Reset() {
@@ -5508,6 +5511,13 @@ func (x *SlurmStatus) GetFilesSha256() string {
 	return ""
 }
 
+func (x *SlurmStatus) GetSlurmdbdRunning() bool {
+	if x != nil {
+		return x.SlurmdbdRunning
+	}
+	return false
+}
+
 var File_llmgw_v1_agent_proto protoreflect.FileDescriptor
 
 const file_llmgw_v1_agent_proto_rawDesc = "" +
@@ -6028,7 +6038,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
 	"\n" +
 	"FilesEntry\x12\x10\n" +
 	"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
-	"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x92\x02\n" +
+	"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xbd\x02\n" +
 	"\vSlurmStatus\x12\x18\n" +
 	"\aversion\x18\x01 \x01(\tR\aversion\x12\x1a\n" +
 	"\bhardware\x18\x02 \x01(\tR\bhardware\x12%\n" +
@@ -6038,7 +6048,9 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
 	"\x06reason\x18\x06 \x01(\tR\x06reason\x12\x12\n" +
 	"\x04jobs\x18\a \x01(\x05R\x04jobs\x12\x14\n" +
 	"\x05error\x18\b \x01(\tR\x05error\x12!\n" +
-	"\ffiles_sha256\x18\t \x01(\tR\vfilesSha2562O\n" +
+	"\ffiles_sha256\x18\t \x01(\tR\vfilesSha256\x12)\n" +
+	"\x10slurmdbd_running\x18\n" +
+	" \x01(\bR\x0fslurmdbdRunning2O\n" +
 	"\fAgentService\x12?\n" +
 	"\aConnect\x12\x16.llmgw.v1.AgentMessage\x1a\x18.llmgw.v1.ControlMessage(\x010\x01B\x93\x01\n" +
 	"\fcom.llmgw.v1B\n" +

api/proto/llmgw/v1/agent.proto

@@ -1034,4 +1034,7 @@ message SlurmStatus {
   // files_sha256 is of the controller's files as written, so the leader
   // knows they arrived.
   string files_sha256 = 9;
+  // slurmdbd_running is the accounting daemon on the controller, which keeps
+  // the jobs' records in a MariaDB beside it.
+  bool slurmdbd_running = 10;
 }

docs/admin-api.openapi.json

@@ -12246,6 +12246,9 @@
           "slurmctld_running": {
             "type": "boolean"
           },
+          "slurmdbd_running": {
+            "type": "boolean"
+          },
           "state": {
             "type": "string"
           },

docs/install.md

@@ -177,14 +177,21 @@ init again to serve on it.
 choices screen, runs a Slurm cluster beside llmgw: slurmctld on this
 machine's node, slurmd on every node, those that join later too
 (`slurm.all_nodes`). The agents install munge and Slurm (22.05 or newer)
-from the machines' own package sources. No node is Slurm's at first: each is
+from the machines' own package sources, and on this machine slurmdbd, which
+keeps the jobs' records (`sacct`), in a MariaDB installed beside it that
+answers this machine alone. Slurm keeps its records in MySQL or MariaDB
+only, not in the cluster's PostgreSQL; the database's password is made from
+the cluster's munge key, so nothing new is stored. A MariaDB the machine
+already had keeps its settings and only gets the database `slurm_acct_db`
+and its user. Back it up with the machine (`mariadb-dump slurm_acct_db`);
+records older than a year are purged. No node is Slurm's at first: each is
 drained in Slurm and serves models until `llmgw ctl slurm lend NODE` gives it
 to Slurm — its models move off first — and `llmgw ctl slurm reclaim NODE`
 takes it back once its jobs end. Jobs are submitted on this machine with
 `sbatch` and `srun`; `llmgw ctl slurm status` shows where each node is. It
 needs this machine to be a node, and its hostname to be a name Slurm takes.
 On a cluster installed without it, `llmgw ctl slurm setup --controller NODE
---all-nodes` does the same later. Slurm's ports (6817–6818, and srun's 60001–63000) answer the cluster's
+--all-nodes` does the same later. Slurm's ports (6817–6819, and srun's 60001–63000) answer the cluster's
 networks only, as llmgw's own do; a firewall of the site's own between the
 nodes must let them through.
 
@@ -3160,7 +3167,7 @@ loaded at boot by `llmgw-firewall.service`):
 | Machine | Ports, for the cluster only |
 | --- | --- |
 | control plane | 7443 (agents), metrics when not on loopback, 8081 (its mirror), engines and 9100 when it is a node, 2379/2380 when it holds the store's third etcd vote |
-| node | engine ports (`control.port_range`, 20000–20999), 9100; with a part in Slurm also 6817–6818 (slurmctld, slurmd) and 60001–63000 (srun's `SrunPortRange`) |
+| node | engine ports (`control.port_range`, 20000–20999), 9100; with a part in Slurm also 6817–6819 (slurmctld, slurmd, slurmdbd) and 60001–63000 (srun's `SrunPortRange`) |
 | store member | 5432, 8008, 2379, 2380, 9100 |
 | mirror | 8081, 8082, 9100 |
 

internal/agent/firewall.go

@@ -16,7 +16,7 @@ import (
 // slurmPorts are Slurm's on a node that runs any of it: slurmctld's and
 // slurmd's, and srun's range for the steps it starts.
 var slurmPorts = []firewall.Port{
-	{From: slurm.CtldPort, To: slurm.SlurmdPort, What: "slurm: slurmctld and slurmd"},
+	{From: slurm.CtldPort, To: slurm.DbdPort, What: "slurm: slurmctld, slurmd and slurmdbd"},
 	{From: slurm.SrunPortsFrom, To: slurm.SrunPortsTo, What: "slurm: srun, for the steps it starts"},
 }
 

internal/agent/firewall_test.go

@@ -73,7 +73,7 @@ func TestANodeGivenAPartInSlurmLetsSlurmInFromTheClusterOnly(t *testing.T) {
 		a.fwMu.Lock()
 		got := sh.installed
 		a.fwMu.Unlock()
-		if strings.Contains(got, "6817-6818") && strings.Contains(got, "60001-63000") {
+		if strings.Contains(got, "6817-6819") && strings.Contains(got, "60001-63000") {
 			break
 		}
 		if time.Now().After(deadline) {

internal/agent/slurmnode/dbd.go

@@ -0,0 +1,170 @@
+package slurmnode
+
+import (
+	"bufio"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
+)
+
+// mariaDBConfDirs are where each family's MariaDB reads its settings: Debian
+// and Ubuntu's, then Red Hat's.
+var mariaDBConfDirs = []string{"/etc/mysql/mariadb.conf.d", "/etc/my.cnf.d"}
+
+const mariaDBConfName = "90-llmgw-slurm.cnf"
+
+// acctSQLPath is where the database's statements are put for the client to
+// read, never on its command line: they hold the password.
+const acctSQLPath = "/run/llmgw-slurm-acct.sql"
+
+// accounting keeps slurmdbd on the controller: MariaDB on this machine
+// alone, the database and its user, slurmdbd on them, and the cluster in it.
+// Before slurmctld, which reports to it from its first start.
+func (k *Keeper) accounting(ctx context.Context, key []byte, files map[string]string) error {
+	password := slurm.DbdPassword(key)
+	changed, err := k.mariaDBSettings(ctx)
+	if err != nil {
+		return err
+	}
+	switch {
+	case changed:
+		k.log.Info("MariaDB, for Slurm's accounting, starts on this machine alone")
+		if err := k.systemctl(ctx, "enable", "--now", "mariadb", "&&restart"); err != nil {
+			return err
+		}
+	case !k.active(ctx, "mariadb"):
+		if err := k.systemctl(ctx, "enable", "--now", "mariadb"); err != nil {
+			return err
+		}
+	}
+	client, err := k.mariaDBClient()
+	if err != nil {
+		return err
+	}
+	if err := k.waitFor(ctx, "MariaDB", client, "-e", "SELECT 1"); err != nil {
+		return err
+	}
+	// The database and its user, once each time the agent starts or the
+	// password is new: cheap, and a database restored from elsewhere gets
+	// its user back.
+	sum := sha256.Sum256([]byte(password))
+	if mark := hex.EncodeToString(sum[:]); k.acctDone != mark {
+		if err := k.host.WriteFile(acctSQLPath, []byte(slurm.AcctSQL(password)), 0o600, ""); err != nil {
+			return err
+		}
+		out, err := k.host.Run(ctx, nil, client, "-e", "source "+acctSQLPath)
+		_ = k.host.Remove(acctSQLPath)
+		if err != nil {
+			return fmt.Errorf("making Slurm's database: %v: %s", err, lastLine(out))
+		}
+		k.acctDone = mark
+	}
+
+	out, err := k.host.Run(ctx, nil, "hostname", "-s")
+	host := strings.TrimSpace(string(out))
+	if err != nil || host == "" {
+		return fmt.Errorf("this machine's short hostname: %v: %s", err, lastLine(out))
+	}
+	changed, err = k.write(slurm.DbdConfPath, []byte(slurm.DbdConf(host, password)), 0o600, "slurm")
+	if err != nil {
+		return err
+	}
+	switch {
+	case changed:
+		k.log.Info("slurmdbd starts again on new settings")
+		if err := k.systemctl(ctx, "enable", "--now", "slurmdbd", "&&restart"); err != nil {
+			return err
+		}
+	case !k.active(ctx, "slurmdbd"):
+		if err := k.systemctl(ctx, "enable", "--now", "slurmdbd"); err != nil {
+			return err
+		}
+	}
+	if err := k.waitFor(ctx, "slurmdbd", "sacctmgr", "-n", "-P", "show", "cluster", "format=cluster"); err != nil {
+		return err
+	}
+	name := clusterName(files["slurm.conf"])
+	if name == "" || k.clusterAdded == name {
+		return nil
+	}
+	out, err = k.host.Run(ctx, nil, "sacctmgr", "-n", "-P", "show", "cluster", "format=cluster")
+	if err != nil {
+		return fmt.Errorf("sacctmgr show cluster: %v: %s", err, lastLine(out))
+	}
+	for _, l := range strings.Split(string(out), "\n") {
+		if strings.TrimSpace(l) == name {
+			k.clusterAdded = name
+			return nil
+		}
+	}
+	k.log.Info("the cluster is added to Slurm's accounting", "cluster", name)
+	if out, err := k.host.Run(ctx, nil, "sacctmgr", "-i", "add", "cluster", name); err != nil {
+		return fmt.Errorf("sacctmgr add cluster %s: %v: %s", name, err, lastLine(out))
+	}
+	k.clusterAdded = name
+	return nil
+}
+
+// mariaDBSettings keeps MariaDB to this machine, when llmgw installed it;
+// one that was here before llmgw keeps its own settings, and only gets the
+// database and its user.
+func (k *Keeper) mariaDBSettings(ctx context.Context) (bool, error) {
+	if _, err := k.host.ReadFile(k.mariaDBMarker()); err != nil {
+		return false, nil
+	}
+	for _, d := range mariaDBConfDirs {
+		if _, err := k.host.Run(ctx, nil, "test", "-d", d); err == nil {
+			return k.write(d+"/"+mariaDBConfName, []byte(slurm.MariaDBConf), 0o644, "")
+		}
+	}
+	return false, fmt.Errorf("MariaDB has none of %s to put its settings in", strings.Join(mariaDBConfDirs, ", "))
+}
+
+// mariaDBMarker is there when llmgw installed MariaDB on this machine.
+func (k *Keeper) mariaDBMarker() string { return k.marker + "-mariadb" }
+
+// mariaDBClient is MariaDB's command-line client, under either of its
+// names.
+func (k *Keeper) mariaDBClient() (string, error) {
+	for _, c := range []string{"mariadb", "mysql"} {
+		if _, err := k.host.LookPath(c); err == nil {
+			return c, nil
+		}
+	}
+	return "", errors.New("MariaDB's client (mariadb or mysql) is not installed")
+}
+
+// waitFor runs a command until it succeeds, half a minute at most.
+func (k *Keeper) waitFor(ctx context.Context, what, name string, args ...string) error {
+	var last []byte
+	for range 30 {
+		out, err := k.host.Run(ctx, nil, name, args...)
+		if err == nil {
+			return nil
+		}
+		last = out
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(pingWait):
+		}
+	}
+	return fmt.Errorf("%s did not answer after starting: %s", what, lastLine(last))
+}
+
+// clusterName is ClusterName in slurm.conf.
+func clusterName(conf string) string {
+	sc := bufio.NewScanner(strings.NewReader(conf))
+	for sc.Scan() {
+		if v, ok := strings.CutPrefix(strings.TrimSpace(sc.Text()), "ClusterName="); ok {
+			return strings.TrimSpace(v)
+		}
+	}
+	return ""
+}

internal/agent/slurmnode/slurmnode.go

@@ -77,6 +77,10 @@ type Keeper struct {
 	// minute's check does not look again.
 	installed map[string]bool
 	files     string
+	// acctDone is the digest of the password the database's user was last
+	// given; clusterAdded the cluster last found in the accounting.
+	acctDone     string
+	clusterAdded string
 }
 
 // New is a Keeper; marker is a file it keeps while it runs Slurm here, so
@@ -207,6 +211,19 @@ func (k *Keeper) Reconcile(ctx context.Context, want *llmgwv1.Slurm) *llmgwv1.Sl
 		return fail("munge", err)
 	}
 	if want.GetController() {
+		// Accounting first: slurmctld reports to it from its first start.
+		// Without it slurmctld still runs jobs, a pass of backfill late
+		// each; so a failure here is said, and slurmctld is kept anyway.
+		var acctErr error
+		if acctErr = k.accounting(ctx, want.GetMungeKey(), want.GetFiles()); acctErr != nil {
+			k.log.Warn("keeping Slurm's accounting on this node", "err", acctErr)
+		}
+		st.SlurmdbdRunning = k.active(ctx, "slurmdbd")
+		defer func() {
+			if acctErr != nil && st.Error == "" {
+				st.Error = "slurmdbd: " + acctErr.Error()
+			}
+		}()
 		sum, err := k.controller(ctx, want.GetFiles())
 		st.FilesSha256 = sum
 		if err != nil {
@@ -229,6 +246,8 @@ func (k *Keeper) ensurePackages(ctx context.Context, want *llmgwv1.Slurm) error
 	}
 	if want.GetController() {
 		need["slurmctld"] = [2]string{"slurmctld", "slurm-slurmctld"}
+		need["slurmdbd"] = [2]string{"slurmdbd", "slurm-slurmdbd"}
+		need["mariadbd"] = [2]string{"mariadb-server", "mariadb-server"}
 	}
 	if want.GetNode() {
 		need["slurmd"] = [2]string{"slurmd", "slurm-slurmd"}
@@ -257,6 +276,14 @@ func (k *Keeper) ensurePackages(ctx context.Context, want *llmgwv1.Slurm) error
 	if err := k.inst.InstallPackages(ctx, pkgs); err != nil {
 		return err
 	}
+	if slices.Contains(pkgs["deb"], "mariadb-server") {
+		// Installed by llmgw: its settings are llmgw's to keep. One that was
+		// here before is another's, and is left as it is.
+		if err := k.host.WriteFile(k.mariaDBMarker(), []byte("llmgw installed MariaDB here, for Slurm's accounting\n"),
+			0o644, ""); err != nil {
+			return err
+		}
+	}
 	for bin := range need {
 		if _, err := k.host.LookPath(bin); err != nil {
 			return fmt.Errorf("%s is still missing after installing %s", bin, strings.Join(pkgs["deb"], ", "))
@@ -483,14 +510,16 @@ func (k *Keeper) stop(ctx context.Context) {
 	if _, err := k.host.ReadFile(k.marker); err != nil {
 		return
 	}
-	k.log.Info("this node has no part in Slurm any more: slurmd and slurmctld stopped; the packages stay")
-	for _, u := range []string{"slurmd", "slurmctld"} {
+	k.log.Info("this node has no part in Slurm any more: slurmd, slurmctld and slurmdbd stopped; the packages and MariaDB stay")
+	// MariaDB stays: it may have been the machine's before, and holds the
+	// jobs' records.
+	for _, u := range []string{"slurmd", "slurmctld", "slurmdbd"} {
 		_ = k.systemctl(ctx, "disable", "--now", u)
 	}
 	_ = k.host.Remove(DropIn)
 	_ = k.systemctl(ctx, "daemon-reload")
 	_ = k.host.Remove(k.marker)
-	k.files = ""
+	k.files, k.acctDone, k.clusterAdded = "", "", ""
 }
 
 // write writes a file when it differs, saying whether it did.
@@ -551,7 +580,8 @@ func (Local) LookPath(name string) (string, error) {
 	if p, err := exec.LookPath(name); err == nil {
 		return p, nil
 	}
-	for _, d := range []string{"/usr/sbin", "/sbin", "/usr/local/sbin"} {
+	// /usr/libexec: Red Hat's mariadbd.
+	for _, d := range []string{"/usr/sbin", "/sbin", "/usr/local/sbin", "/usr/libexec"} {
 		p := filepath.Join(d, name)
 		if fi, err := os.Stat(p); err == nil && !fi.IsDir() {
 			return p, nil

internal/agent/slurmnode/slurmnode_test.go

@@ -32,6 +32,7 @@ func newFake() *fakeHost {
 			"slurmctld -V":  "slurm-wlm 23.11.4\n",
 			"scontrol ping": "Slurmctld(primary) at node06 is UP\n",
 			"slurmd -C":     "NodeName=cuda01 CPUs=16 RealMemory=64000\nUpTime=1-00:00:00\n",
+			"hostname -s":   "node06\n",
 		}, fail: map[string]bool{}, state: "IDLE+DRAIN", reason: slurm.ReasonNew}
 }
 
@@ -118,7 +119,7 @@ type fakeInst struct {
 
 func (i *fakeInst) InstallPackages(_ context.Context, by map[string][]string) error {
 	i.got = by
-	for _, b := range []string{"munged", "scontrol", "slurmd", "slurmctld"} {
+	for _, b := range []string{"munged", "scontrol", "slurmd", "slurmctld", "slurmdbd", "mariadbd", "mariadb"} {
 		i.host.bins[b] = true
 	}
 	return nil
@@ -237,6 +238,33 @@ func TestController(t *testing.T) {
 	if !st.GetSlurmctldRunning() || len(st.GetFilesSha256()) != 64 {
 		t.Fatalf("status %v", st)
 	}
+	// Accounting: MariaDB on this machine alone, the database, slurmdbd on
+	// it, the cluster added, all before slurmctld.
+	if !slices.Contains(inst.got["deb"], "slurmdbd") || !slices.Contains(inst.got["deb"], "mariadb-server") {
+		t.Fatalf("installed %v", inst.got)
+	}
+	if !strings.Contains(string(h.files["/etc/mysql/mariadb.conf.d/90-llmgw-slurm.cnf"]), "bind-address=127.0.0.1") {
+		t.Fatal("MariaDB, installed by llmgw, not kept to this machine")
+	}
+	dbd := string(h.files[slurm.DbdConfPath])
+	if !strings.Contains(dbd, "DbdHost=node06\n") || !strings.Contains(dbd, "StoragePass="+slurm.DbdPassword(key)+"\n") ||
+		h.owners[slurm.DbdConfPath] != "slurm" {
+		t.Fatalf("slurmdbd.conf for %q:\n%s", h.owners[slurm.DbdConfPath], dbd)
+	}
+	if _, ok := h.files[acctSQLPath]; ok {
+		t.Fatal("the database's statements, password and all, were left on disk")
+	}
+	if strings.Contains(strings.Join(h.ran, "\n"), slurm.DbdPassword(key)) {
+		t.Fatal("the password was on a command line")
+	}
+	order := strings.Join(h.ran, "\n")
+	if !strings.Contains(order, "sacctmgr -i add cluster llmgw") ||
+		strings.Index(order, "slurmdbd") > strings.Index(order, "systemctl restart slurmctld") {
+		t.Fatalf("accounting not before slurmctld: %v", h.ran)
+	}
+	if !st.GetSlurmdbdRunning() {
+		t.Fatalf("status %v", st)
+	}
 	sum := st.GetFilesSha256()
 
 	h.ran = nil
@@ -279,7 +307,7 @@ func TestStop(t *testing.T) {
 	if st := k.Reconcile(context.Background(), &llmgwv1.Slurm{}); st != nil {
 		t.Fatalf("status %v for no part", st)
 	}
-	if h.count("systemctl disable --now slurmd") != 1 {
+	if !slices.Contains(h.ran, "systemctl disable --now slurmd") {
 		t.Fatalf("slurmd not stopped: %v", h.ran)
 	}
 	if _, ok := h.files[DropIn]; ok {
@@ -306,3 +334,24 @@ func TestBadNames(t *testing.T) {
 		t.Fatalf("status %v", st)
 	}
 }
+
+func TestAMariaDBThatWasThereKeepsItsSettings(t *testing.T) {
+	h := newFake()
+	for _, b := range []string{"munged", "scontrol", "slurmctld", "slurmdbd", "mariadbd", "mariadb"} {
+		h.bins[b] = true
+	}
+	k := New(h, &fakeInst{host: h}, "/m", nil)
+	st := k.Reconcile(context.Background(), &llmgwv1.Slurm{Controller: true, MungeKey: key,
+		Files: map[string]string{"slurm.conf": "ClusterName=llmgw\n"}})
+	if st.GetError() != "" {
+		t.Fatal(st.GetError())
+	}
+	for p := range h.files {
+		if strings.HasSuffix(p, ".cnf") {
+			t.Fatalf("a MariaDB llmgw did not install had its settings changed: %s", p)
+		}
+	}
+	if h.count("mariadb -e source") != 1 || !st.GetSlurmdbdRunning() {
+		t.Fatalf("the database was not made in the MariaDB that was there: %v", h.ran)
+	}
+}

internal/cli/ctl_slurm.go

@@ -55,8 +55,10 @@ func newSlurmStatusCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
 				fmt.Fprintln(w, ": not heard from yet")
 			case c.Error != "":
 				fmt.Fprintf(w, ": %s\n", c.Error)
+			case c.SlurmctldRunning && c.SlurmdbdRunning:
+				fmt.Fprintf(w, ": slurmctld %s and slurmdbd running\n", c.Version)
 			case c.SlurmctldRunning:
-				fmt.Fprintf(w, ": slurmctld %s running\n", c.Version)
+				fmt.Fprintf(w, ": slurmctld %s running, slurmdbd not (the jobs' records)\n", c.Version)
 			default:
 				fmt.Fprintln(w, ": slurmctld not running")
 			}
@@ -96,7 +98,8 @@ func newSlurmSetupCmd(client func() *ctlClient) *cobra.Command {
 		Use:   "setup --controller NODE --nodes NODE,... | --all-nodes",
 		Short: "Run Slurm on llmgw's nodes: slurmctld on one, slurmd on those named",
 		Long: "Turns slurm.enabled on and names the nodes, or with --all-nodes every node. Their agents install munge and Slurm (22.05 or\n" +
-			"newer) from the machines' own package sources. Every node starts drained in Slurm and stays\n" +
+			"newer) from the machines' own package sources; the controller also slurmdbd, the jobs' records, in a\n" +
+			"MariaDB on that machine alone (one that was there keeps its settings). Every node starts drained in Slurm and stays\n" +
 			"llmgw's until `llmgw ctl slurm lend` gives it to Slurm.",
 		Example: "  llmgw ctl slurm setup --controller node06 --nodes cuda01,cuda02\n" +
 			"  llmgw ctl slurm setup --controller node06 --all-nodes",

internal/cli/serverinit.go

@@ -262,7 +262,7 @@ func newServerInitCmd() *cobra.Command {
 	f.BoolVar(&in.media, "media", false,
 		"pictures and 3D: the nodes install ComfyUI, and a picture model and a 3D model are added once a card fits one")
 	f.BoolVar(&in.slurm, "slurm", false,
-		"Slurm for training and batch jobs: slurmctld on this machine, slurmd on every node, each llmgw's until 'llmgw ctl slurm lend'")
+		"Slurm for training and batch jobs: slurmctld and slurmdbd (with a MariaDB) on this machine, slurmd on every node, each llmgw's until 'llmgw ctl slurm lend'")
 	f.DurationVar(&in.evictAfterIdle, "evict-after-idle", 0,
 		"how long a model loaded on demand goes unasked-for before another may have its card (0: 2m)")
 	f.DurationVar(&in.minLifetime, "min-lifetime", 0,

internal/cli/serverinit_choices.go

@@ -350,7 +350,8 @@ func (in *serverInit) tuneConfig(body string) string {
 // slurmSection is server.yaml's slurm block as an install chooses it:
 // slurmctld on this machine's node, slurmd on every node.
 func slurmSection(controller string) string {
-	return "# Slurm beside llmgw, chosen at install: slurmctld on this machine, slurmd on\n" +
+	return "# Slurm beside llmgw, chosen at install: slurmctld, and slurmdbd with a MariaDB\n" +
+		"# for the jobs' records, on this machine; slurmd on\n" +
 		"# every node, those that join later too. Each node is llmgw's, drained in\n" +
 		"# Slurm, until 'llmgw ctl slurm lend NODE' gives it to Slurm; 'llmgw ctl slurm\n" +
 		"# reclaim NODE' takes it back once its jobs end.\n" +

internal/cli/tui_choices.go

@@ -287,7 +287,9 @@ func newInstallForm(in *serverInit, host string, found []string, ha bool) *insta
 		title: "Slurm", on: in.slurm,
 		about: "A Slurm cluster beside llmgw, for training and other batch work: slurmctld on this machine, " +
 			"slurmd on every node, those that join later too. The agents install munge and Slurm from the " +
-			"machines' own package sources.\n\n" +
+			"machines' own package sources, and on this machine slurmdbd, the jobs' records, with a MariaDB " +
+			"that answers this machine alone (Slurm keeps its records in MySQL or MariaDB only, not in the " +
+			"cluster's PostgreSQL). Its port 6819, with Slurm's others, answers the cluster's networks only.\n\n" +
 			"Each node stays llmgw's, drained in Slurm, until 'llmgw ctl slurm lend NODE' gives it to Slurm: " +
 			"its models move off first. 'llmgw ctl slurm reclaim NODE' takes it back once its jobs end. Jobs " +
 			"are submitted on this machine with sbatch and srun.",

internal/cli/tui_slurm.go

@@ -35,8 +35,10 @@ func (m consoleModel) slurmPanel(pw int) (string, string, string) {
 	case c == nil:
 	case c.Error != "":
 		ctl = sBad.Render(c.Error)
+	case c.SlurmctldRunning && c.SlurmdbdRunning:
+		ctl = sOK.Render("slurmctld " + c.Version + " and slurmdbd running")
 	case c.SlurmctldRunning:
-		ctl = sOK.Render("slurmctld " + c.Version + " running")
+		ctl = sOK.Render("slurmctld "+c.Version+" running") + ", " + sWarn.Render("slurmdbd not")
 	default:
 		ctl = sBad.Render("slurmctld not running")
 	}

internal/cli/tui_slurm_test.go

@@ -37,7 +37,7 @@ func TestTheConsoleLendsANodeToSlurmFromItsRow(t *testing.T) {
 	m := newConsoleModel(srv.URL, "/nonexistent/server.yaml")
 	m.loaded, m.menu, m.inPanel = true, cSlurm, true
 	m.data.slurm = slurmsvc.Status{Enabled: true, Cluster: "llmgw", Controller: "node06", Partition: "gpu",
-		Ctld: &slurmsvc.Reported{SlurmctldRunning: true, Version: "23.11.4"},
+		Ctld: &slurmsvc.Reported{SlurmctldRunning: true, SlurmdbdRunning: true, Version: "23.11.4"},
 		Nodes: []slurmsvc.NodeStatus{
 			{Node: "node05", Phase: slurmsvc.Lent, Connected: true,
 				Status: &slurmsvc.Reported{State: "MIXED", Jobs: 2, SlurmdRunning: true}},
@@ -46,7 +46,7 @@ func TestTheConsoleLendsANodeToSlurmFromItsRow(t *testing.T) {
 		}}
 	var tm tea.Model = m
 	view := tm.View()
-	for _, want := range []string{"slurmctld 23.11.4 running", "node05", "Slurm's", "MIXED", "cuda01", "llmgw's"} {
+	for _, want := range []string{"slurmctld 23.11.4 and slurmdbd running", "node05", "Slurm's", "MIXED", "cuda01", "llmgw's"} {
 		if !strings.Contains(view, want) {
 			t.Errorf("the Slurm view has no %q:\n%s", want, view)
 		}

internal/control/slurmsvc/service.go

@@ -75,6 +75,7 @@ type Reported struct {
 	Hardware         string    `json:"hardware,omitempty"`
 	SlurmdRunning    bool      `json:"slurmd_running"`
 	SlurmctldRunning bool      `json:"slurmctld_running"`
+	SlurmdbdRunning  bool      `json:"slurmdbd_running"`
 	State            string    `json:"state,omitempty"`
 	Reason           string    `json:"reason,omitempty"`
 	Jobs             int       `json:"jobs"`
@@ -197,7 +198,7 @@ func (s *Service) Kick() {
 // OnSlurmStatus records what a node's agent says.
 func (s *Service) OnSlurmStatus(node string, st *llmgwv1.SlurmStatus) {
 	r := &Reported{Version: st.GetVersion(), Hardware: st.GetHardware(), SlurmdRunning: st.GetSlurmdRunning(),
-		SlurmctldRunning: st.GetSlurmctldRunning(), State: st.GetState(), Reason: st.GetReason(),
+		SlurmctldRunning: st.GetSlurmctldRunning(), SlurmdbdRunning: st.GetSlurmdbdRunning(), State: st.GetState(), Reason: st.GetReason(),
 		Jobs: int(st.GetJobs()), Error: st.GetError(), FilesSHA256: st.GetFilesSha256(), At: time.Now()}
 	s.mu.Lock()
 	prev := s.status[node]

internal/slurm/dbd.go

@@ -0,0 +1,81 @@
+package slurm
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+)
+
+// Accounting: slurmdbd on the controller, keeping the jobs' records in a
+// MariaDB beside it that answers this machine alone. slurmdbd takes MySQL or
+// MariaDB only, so it is not the cluster's PostgreSQL. Since 23.11 Slurm
+// cannot run without it well: its main scheduler passes every job by as
+// InvalidAccount, and backfill starts it a pass later.
+const (
+	// DbdPort is slurmdbd's, which slurmctld and sacct reach.
+	DbdPort = 6819
+	// DbdConfPath is slurmdbd's file, readable by the slurm user alone: it
+	// holds the database's password.
+	DbdConfPath = ConfDir + "/slurmdbd.conf"
+	// AcctDB and AcctUser are the database and its user.
+	AcctDB   = "slurm_acct_db"
+	AcctUser = "slurm"
+)
+
+// DbdPassword is the database user's password, made from the munge key:
+// every controller the cluster has gets the same one, and nothing new is
+// kept anywhere.
+func DbdPassword(mungeKey []byte) string {
+	h := sha256.New()
+	h.Write([]byte("llmgw slurmdbd password\x00"))
+	h.Write(mungeKey)
+	return hex.EncodeToString(h.Sum(nil))[:40]
+}
+
+// DbdConf is slurmdbd.conf for the controller host, named as `hostname -s`
+// says, which slurmdbd checks it runs on.
+func DbdConf(host, password string) string {
+	return header + fmt.Sprintf(`AuthType=auth/munge
+DbdHost=%s
+DbdPort=%d
+SlurmUser=slurm
+LogFile=%s/slurmdbd.log
+PidFile=/run/slurmdbd.pid
+StorageType=accounting_storage/mysql
+StorageHost=localhost
+StorageUser=%s
+StoragePass=%s
+StorageLoc=%s
+# Kept a year; older records are purged, a month at a time.
+PurgeEventAfter=12months
+PurgeJobAfter=12months
+PurgeResvAfter=12months
+PurgeStepAfter=12months
+PurgeSuspendAfter=12months
+PurgeTXNAfter=12months
+PurgeUsageAfter=24months
+`, host, DbdPort, LogDir, AcctUser, password, AcctDB)
+}
+
+// MariaDBConf is the database server's settings: this machine alone, and
+// what Slurm's documentation asks of InnoDB.
+const MariaDBConf = `# Written by llmgw (BACKLOG P): the database of Slurm's accounting
+# (slurmdbd), on this machine alone.
+[mysqld]
+bind-address=127.0.0.1
+skip-networking=0
+innodb_buffer_pool_size=256M
+innodb_lock_wait_timeout=900
+`
+
+// AcctSQL makes the database and its user, and gives the user the password
+// again: run on every start of the controller's part, it changes nothing
+// that is already so.
+func AcctSQL(password string) string {
+	return fmt.Sprintf(`CREATE DATABASE IF NOT EXISTS %[1]s;
+CREATE USER IF NOT EXISTS '%[2]s'@'localhost' IDENTIFIED BY '%[3]s';
+ALTER USER '%[2]s'@'localhost' IDENTIFIED BY '%[3]s';
+GRANT ALL ON %[1]s.* TO '%[2]s'@'localhost';
+FLUSH PRIVILEGES;
+`, AcctDB, AcctUser, password)
+}

internal/slurm/slurm.go

@@ -196,6 +196,9 @@ func (c Cluster) SlurmConf() string {
 	b.WriteString(header)
 	fmt.Fprintf(&b, "ClusterName=%s\n", c.Name)
 	fmt.Fprintf(&b, "SlurmctldHost=%s(%s)\n", c.Controller, c.ControllerAddr)
+	// The jobs' records, in slurmdbd beside slurmctld (DbdConf).
+	fmt.Fprintf(&b, "AccountingStorageType=accounting_storage/slurmdbd\nAccountingStorageHost=%s\nAccountingStoragePort=%d\n",
+		c.ControllerAddr, DbdPort)
 	b.WriteString(`SlurmctldParameters=enable_configless
 SlurmctldPort=6817
 SlurmdPort=6818
@@ -212,17 +215,13 @@ SlurmctldPidFile=/run/slurmctld.pid
 SlurmdPidFile=/run/slurmd.pid
 ProctrackType=proctrack/cgroup
 TaskPlugin=task/cgroup,task/affinity
-# Nothing gathered for accounting until there is somewhere to keep it
-# (slurmdbd, later); gathered and not kept is an error in every log.
-JobAcctGatherType=jobacct_gather/none
+# What each job used, from its cgroup, for the records; the cards too.
+JobAcctGatherType=jobacct_gather/cgroup
+AccountingStorageTRES=gres/gpu
 SelectType=select/cons_tres
 SelectTypeParameters=CR_Core_Memory
 SchedulerType=sched/backfill
-# First come, first served: multifactor's fair share needs accounting
-# storage (slurmdbd), which there is none of yet. Without it, 23.11's main
-# scheduler passes every job by as InvalidAccount, and backfill starts it
-# on its next pass (bf_interval above).
-PriorityType=priority/basic
+PriorityType=priority/multifactor
 GresTypes=gpu
 MpiDefault=none
 # No mail sent; without it every start says MailProg is invalid.

internal/slurm/slurm_test.go

@@ -165,7 +165,29 @@ func TestALentNodeIsNotWrittenDrained(t *testing.T) {
 	if line == "" || strings.Contains(line, "DRAIN") {
 		t.Fatalf("a lent node, written %q: every start of slurmctld would drain it again", line)
 	}
-	if !strings.Contains(conf, "PriorityType=priority/basic\n") {
-		t.Error("multifactor without accounting: the main scheduler passes every job by as InvalidAccount")
+}
+
+func TestTheJobsAreAccountedInSlurmdbd(t *testing.T) {
+	conf := testCluster().SlurmConf()
+	for _, want := range []string{
+		"AccountingStorageType=accounting_storage/slurmdbd\n",
+		"AccountingStorageHost=10.0.0.146\n",
+		"AccountingStoragePort=6819\n",
+		"JobAcctGatherType=jobacct_gather/cgroup\n",
+		"AccountingStorageTRES=gres/gpu\n",
+	} {
+		if !strings.Contains(conf, want) {
+			t.Errorf("slurm.conf has no %q", want)
+		}
+	}
+	dbd := DbdConf("node06", DbdPassword([]byte("key")))
+	for _, want := range []string{"DbdHost=node06\n", "DbdPort=6819\n", "StorageType=accounting_storage/mysql\n",
+		"StorageHost=localhost\n", "StorageLoc=slurm_acct_db\n", "AuthType=auth/munge\n"} {
+		if !strings.Contains(dbd, want) {
+			t.Errorf("slurmdbd.conf has no %q:\n%s", want, dbd)
+		}
+	}
+	if DbdPassword([]byte("key")) == DbdPassword([]byte("other")) || len(DbdPassword([]byte("key"))) != 40 {
+		t.Error("the password does not follow the key")
 	}
 }