diff --git a/CHANGELOG.md b/CHANGELOG.md index b102f7c..6a5321f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,8 +48,11 @@ APIs change between minors without a deprecation period. and gres.conf written from what each node reports. Every node stays llmgw's, drained in Slurm, until `ctl slurm lend NODE`: its replicas move off, then Slurm runs jobs there. `ctl slurm reclaim NODE` takes it back once its jobs - end. `ctl slurm status` shows each node. Slurm's ports (6817–6818, srun's - 60001–63000) answer the cluster's networks only, as llmgw's own do. + end. `ctl slurm status` and the console's **Slurm** show each node. The + controller also keeps the jobs' records: slurmdbd, in a MariaDB on that + machine alone (Slurm takes MySQL or MariaDB only). Slurm's ports + (6817–6819, srun's 60001–63000) answer the cluster's networks only, as + llmgw's own do. ### Changed diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index a89ce7f..9913d5f 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -5410,9 +5410,12 @@ type SlurmStatus struct { Error string `protobuf:"bytes,8,opt,name=error,proto3" json:"error,omitempty"` // files_sha256 is of the controller's files as written, so the leader // knows they arrived. - FilesSha256 string `protobuf:"bytes,9,opt,name=files_sha256,json=filesSha256,proto3" json:"files_sha256,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + FilesSha256 string `protobuf:"bytes,9,opt,name=files_sha256,json=filesSha256,proto3" json:"files_sha256,omitempty"` + // slurmdbd_running is the accounting daemon on the controller, which keeps + // the jobs' records in a MariaDB beside it. + SlurmdbdRunning bool `protobuf:"varint,10,opt,name=slurmdbd_running,json=slurmdbdRunning,proto3" json:"slurmdbd_running,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SlurmStatus) Reset() { @@ -5508,6 +5511,13 @@ func (x *SlurmStatus) GetFilesSha256() string { return "" } +func (x *SlurmStatus) GetSlurmdbdRunning() bool { + if x != nil { + return x.SlurmdbdRunning + } + return false +} + var File_llmgw_v1_agent_proto protoreflect.FileDescriptor const file_llmgw_v1_agent_proto_rawDesc = "" + @@ -6028,7 +6038,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\n" + "FilesEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x92\x02\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xbd\x02\n" + "\vSlurmStatus\x12\x18\n" + "\aversion\x18\x01 \x01(\tR\aversion\x12\x1a\n" + "\bhardware\x18\x02 \x01(\tR\bhardware\x12%\n" + @@ -6038,7 +6048,9 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x06reason\x18\x06 \x01(\tR\x06reason\x12\x12\n" + "\x04jobs\x18\a \x01(\x05R\x04jobs\x12\x14\n" + "\x05error\x18\b \x01(\tR\x05error\x12!\n" + - "\ffiles_sha256\x18\t \x01(\tR\vfilesSha2562O\n" + + "\ffiles_sha256\x18\t \x01(\tR\vfilesSha256\x12)\n" + + "\x10slurmdbd_running\x18\n" + + " \x01(\bR\x0fslurmdbdRunning2O\n" + "\fAgentService\x12?\n" + "\aConnect\x12\x16.llmgw.v1.AgentMessage\x1a\x18.llmgw.v1.ControlMessage(\x010\x01B\x93\x01\n" + "\fcom.llmgw.v1B\n" + diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index 42a5881..45318a2 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -1034,4 +1034,7 @@ message SlurmStatus { // files_sha256 is of the controller's files as written, so the leader // knows they arrived. string files_sha256 = 9; + // slurmdbd_running is the accounting daemon on the controller, which keeps + // the jobs' records in a MariaDB beside it. + bool slurmdbd_running = 10; } diff --git a/docs/admin-api.openapi.json b/docs/admin-api.openapi.json index 670e184..a8ba48e 100644 --- a/docs/admin-api.openapi.json +++ b/docs/admin-api.openapi.json @@ -12246,6 +12246,9 @@ "slurmctld_running": { "type": "boolean" }, + "slurmdbd_running": { + "type": "boolean" + }, "state": { "type": "string" }, diff --git a/docs/install.md b/docs/install.md index 69a38de..c56c4dd 100644 --- a/docs/install.md +++ b/docs/install.md @@ -177,14 +177,21 @@ init again to serve on it. choices screen, runs a Slurm cluster beside llmgw: slurmctld on this machine's node, slurmd on every node, those that join later too (`slurm.all_nodes`). The agents install munge and Slurm (22.05 or newer) -from the machines' own package sources. No node is Slurm's at first: each is +from the machines' own package sources, and on this machine slurmdbd, which +keeps the jobs' records (`sacct`), in a MariaDB installed beside it that +answers this machine alone. Slurm keeps its records in MySQL or MariaDB +only, not in the cluster's PostgreSQL; the database's password is made from +the cluster's munge key, so nothing new is stored. A MariaDB the machine +already had keeps its settings and only gets the database `slurm_acct_db` +and its user. Back it up with the machine (`mariadb-dump slurm_acct_db`); +records older than a year are purged. No node is Slurm's at first: each is drained in Slurm and serves models until `llmgw ctl slurm lend NODE` gives it to Slurm — its models move off first — and `llmgw ctl slurm reclaim NODE` takes it back once its jobs end. Jobs are submitted on this machine with `sbatch` and `srun`; `llmgw ctl slurm status` shows where each node is. It needs this machine to be a node, and its hostname to be a name Slurm takes. On a cluster installed without it, `llmgw ctl slurm setup --controller NODE ---all-nodes` does the same later. Slurm's ports (6817–6818, and srun's 60001–63000) answer the cluster's +--all-nodes` does the same later. Slurm's ports (6817–6819, and srun's 60001–63000) answer the cluster's networks only, as llmgw's own do; a firewall of the site's own between the nodes must let them through. @@ -3160,7 +3167,7 @@ loaded at boot by `llmgw-firewall.service`): | Machine | Ports, for the cluster only | | --- | --- | | control plane | 7443 (agents), metrics when not on loopback, 8081 (its mirror), engines and 9100 when it is a node, 2379/2380 when it holds the store's third etcd vote | -| node | engine ports (`control.port_range`, 20000–20999), 9100; with a part in Slurm also 6817–6818 (slurmctld, slurmd) and 60001–63000 (srun's `SrunPortRange`) | +| node | engine ports (`control.port_range`, 20000–20999), 9100; with a part in Slurm also 6817–6819 (slurmctld, slurmd, slurmdbd) and 60001–63000 (srun's `SrunPortRange`) | | store member | 5432, 8008, 2379, 2380, 9100 | | mirror | 8081, 8082, 9100 | diff --git a/internal/agent/firewall.go b/internal/agent/firewall.go index fb2e726..bc9193b 100644 --- a/internal/agent/firewall.go +++ b/internal/agent/firewall.go @@ -16,7 +16,7 @@ import ( // slurmPorts are Slurm's on a node that runs any of it: slurmctld's and // slurmd's, and srun's range for the steps it starts. var slurmPorts = []firewall.Port{ - {From: slurm.CtldPort, To: slurm.SlurmdPort, What: "slurm: slurmctld and slurmd"}, + {From: slurm.CtldPort, To: slurm.DbdPort, What: "slurm: slurmctld, slurmd and slurmdbd"}, {From: slurm.SrunPortsFrom, To: slurm.SrunPortsTo, What: "slurm: srun, for the steps it starts"}, } diff --git a/internal/agent/firewall_test.go b/internal/agent/firewall_test.go index 471015d..9a0df94 100644 --- a/internal/agent/firewall_test.go +++ b/internal/agent/firewall_test.go @@ -73,7 +73,7 @@ func TestANodeGivenAPartInSlurmLetsSlurmInFromTheClusterOnly(t *testing.T) { a.fwMu.Lock() got := sh.installed a.fwMu.Unlock() - if strings.Contains(got, "6817-6818") && strings.Contains(got, "60001-63000") { + if strings.Contains(got, "6817-6819") && strings.Contains(got, "60001-63000") { break } if time.Now().After(deadline) { diff --git a/internal/agent/slurmnode/dbd.go b/internal/agent/slurmnode/dbd.go new file mode 100644 index 0000000..945d31a --- /dev/null +++ b/internal/agent/slurmnode/dbd.go @@ -0,0 +1,170 @@ +package slurmnode + +import ( + "bufio" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "strings" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/slurm" +) + +// mariaDBConfDirs are where each family's MariaDB reads its settings: Debian +// and Ubuntu's, then Red Hat's. +var mariaDBConfDirs = []string{"/etc/mysql/mariadb.conf.d", "/etc/my.cnf.d"} + +const mariaDBConfName = "90-llmgw-slurm.cnf" + +// acctSQLPath is where the database's statements are put for the client to +// read, never on its command line: they hold the password. +const acctSQLPath = "/run/llmgw-slurm-acct.sql" + +// accounting keeps slurmdbd on the controller: MariaDB on this machine +// alone, the database and its user, slurmdbd on them, and the cluster in it. +// Before slurmctld, which reports to it from its first start. +func (k *Keeper) accounting(ctx context.Context, key []byte, files map[string]string) error { + password := slurm.DbdPassword(key) + changed, err := k.mariaDBSettings(ctx) + if err != nil { + return err + } + switch { + case changed: + k.log.Info("MariaDB, for Slurm's accounting, starts on this machine alone") + if err := k.systemctl(ctx, "enable", "--now", "mariadb", "&&restart"); err != nil { + return err + } + case !k.active(ctx, "mariadb"): + if err := k.systemctl(ctx, "enable", "--now", "mariadb"); err != nil { + return err + } + } + client, err := k.mariaDBClient() + if err != nil { + return err + } + if err := k.waitFor(ctx, "MariaDB", client, "-e", "SELECT 1"); err != nil { + return err + } + // The database and its user, once each time the agent starts or the + // password is new: cheap, and a database restored from elsewhere gets + // its user back. + sum := sha256.Sum256([]byte(password)) + if mark := hex.EncodeToString(sum[:]); k.acctDone != mark { + if err := k.host.WriteFile(acctSQLPath, []byte(slurm.AcctSQL(password)), 0o600, ""); err != nil { + return err + } + out, err := k.host.Run(ctx, nil, client, "-e", "source "+acctSQLPath) + _ = k.host.Remove(acctSQLPath) + if err != nil { + return fmt.Errorf("making Slurm's database: %v: %s", err, lastLine(out)) + } + k.acctDone = mark + } + + out, err := k.host.Run(ctx, nil, "hostname", "-s") + host := strings.TrimSpace(string(out)) + if err != nil || host == "" { + return fmt.Errorf("this machine's short hostname: %v: %s", err, lastLine(out)) + } + changed, err = k.write(slurm.DbdConfPath, []byte(slurm.DbdConf(host, password)), 0o600, "slurm") + if err != nil { + return err + } + switch { + case changed: + k.log.Info("slurmdbd starts again on new settings") + if err := k.systemctl(ctx, "enable", "--now", "slurmdbd", "&&restart"); err != nil { + return err + } + case !k.active(ctx, "slurmdbd"): + if err := k.systemctl(ctx, "enable", "--now", "slurmdbd"); err != nil { + return err + } + } + if err := k.waitFor(ctx, "slurmdbd", "sacctmgr", "-n", "-P", "show", "cluster", "format=cluster"); err != nil { + return err + } + name := clusterName(files["slurm.conf"]) + if name == "" || k.clusterAdded == name { + return nil + } + out, err = k.host.Run(ctx, nil, "sacctmgr", "-n", "-P", "show", "cluster", "format=cluster") + if err != nil { + return fmt.Errorf("sacctmgr show cluster: %v: %s", err, lastLine(out)) + } + for _, l := range strings.Split(string(out), "\n") { + if strings.TrimSpace(l) == name { + k.clusterAdded = name + return nil + } + } + k.log.Info("the cluster is added to Slurm's accounting", "cluster", name) + if out, err := k.host.Run(ctx, nil, "sacctmgr", "-i", "add", "cluster", name); err != nil { + return fmt.Errorf("sacctmgr add cluster %s: %v: %s", name, err, lastLine(out)) + } + k.clusterAdded = name + return nil +} + +// mariaDBSettings keeps MariaDB to this machine, when llmgw installed it; +// one that was here before llmgw keeps its own settings, and only gets the +// database and its user. +func (k *Keeper) mariaDBSettings(ctx context.Context) (bool, error) { + if _, err := k.host.ReadFile(k.mariaDBMarker()); err != nil { + return false, nil + } + for _, d := range mariaDBConfDirs { + if _, err := k.host.Run(ctx, nil, "test", "-d", d); err == nil { + return k.write(d+"/"+mariaDBConfName, []byte(slurm.MariaDBConf), 0o644, "") + } + } + return false, fmt.Errorf("MariaDB has none of %s to put its settings in", strings.Join(mariaDBConfDirs, ", ")) +} + +// mariaDBMarker is there when llmgw installed MariaDB on this machine. +func (k *Keeper) mariaDBMarker() string { return k.marker + "-mariadb" } + +// mariaDBClient is MariaDB's command-line client, under either of its +// names. +func (k *Keeper) mariaDBClient() (string, error) { + for _, c := range []string{"mariadb", "mysql"} { + if _, err := k.host.LookPath(c); err == nil { + return c, nil + } + } + return "", errors.New("MariaDB's client (mariadb or mysql) is not installed") +} + +// waitFor runs a command until it succeeds, half a minute at most. +func (k *Keeper) waitFor(ctx context.Context, what, name string, args ...string) error { + var last []byte + for range 30 { + out, err := k.host.Run(ctx, nil, name, args...) + if err == nil { + return nil + } + last = out + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(pingWait): + } + } + return fmt.Errorf("%s did not answer after starting: %s", what, lastLine(last)) +} + +// clusterName is ClusterName in slurm.conf. +func clusterName(conf string) string { + sc := bufio.NewScanner(strings.NewReader(conf)) + for sc.Scan() { + if v, ok := strings.CutPrefix(strings.TrimSpace(sc.Text()), "ClusterName="); ok { + return strings.TrimSpace(v) + } + } + return "" +} diff --git a/internal/agent/slurmnode/slurmnode.go b/internal/agent/slurmnode/slurmnode.go index 49e1837..8f15ae6 100644 --- a/internal/agent/slurmnode/slurmnode.go +++ b/internal/agent/slurmnode/slurmnode.go @@ -77,6 +77,10 @@ type Keeper struct { // minute's check does not look again. installed map[string]bool files string + // acctDone is the digest of the password the database's user was last + // given; clusterAdded the cluster last found in the accounting. + acctDone string + clusterAdded string } // New is a Keeper; marker is a file it keeps while it runs Slurm here, so @@ -207,6 +211,19 @@ func (k *Keeper) Reconcile(ctx context.Context, want *llmgwv1.Slurm) *llmgwv1.Sl return fail("munge", err) } if want.GetController() { + // Accounting first: slurmctld reports to it from its first start. + // Without it slurmctld still runs jobs, a pass of backfill late + // each; so a failure here is said, and slurmctld is kept anyway. + var acctErr error + if acctErr = k.accounting(ctx, want.GetMungeKey(), want.GetFiles()); acctErr != nil { + k.log.Warn("keeping Slurm's accounting on this node", "err", acctErr) + } + st.SlurmdbdRunning = k.active(ctx, "slurmdbd") + defer func() { + if acctErr != nil && st.Error == "" { + st.Error = "slurmdbd: " + acctErr.Error() + } + }() sum, err := k.controller(ctx, want.GetFiles()) st.FilesSha256 = sum if err != nil { @@ -229,6 +246,8 @@ func (k *Keeper) ensurePackages(ctx context.Context, want *llmgwv1.Slurm) error } if want.GetController() { need["slurmctld"] = [2]string{"slurmctld", "slurm-slurmctld"} + need["slurmdbd"] = [2]string{"slurmdbd", "slurm-slurmdbd"} + need["mariadbd"] = [2]string{"mariadb-server", "mariadb-server"} } if want.GetNode() { need["slurmd"] = [2]string{"slurmd", "slurm-slurmd"} @@ -257,6 +276,14 @@ func (k *Keeper) ensurePackages(ctx context.Context, want *llmgwv1.Slurm) error if err := k.inst.InstallPackages(ctx, pkgs); err != nil { return err } + if slices.Contains(pkgs["deb"], "mariadb-server") { + // Installed by llmgw: its settings are llmgw's to keep. One that was + // here before is another's, and is left as it is. + if err := k.host.WriteFile(k.mariaDBMarker(), []byte("llmgw installed MariaDB here, for Slurm's accounting\n"), + 0o644, ""); err != nil { + return err + } + } for bin := range need { if _, err := k.host.LookPath(bin); err != nil { return fmt.Errorf("%s is still missing after installing %s", bin, strings.Join(pkgs["deb"], ", ")) @@ -483,14 +510,16 @@ func (k *Keeper) stop(ctx context.Context) { if _, err := k.host.ReadFile(k.marker); err != nil { return } - k.log.Info("this node has no part in Slurm any more: slurmd and slurmctld stopped; the packages stay") - for _, u := range []string{"slurmd", "slurmctld"} { + k.log.Info("this node has no part in Slurm any more: slurmd, slurmctld and slurmdbd stopped; the packages and MariaDB stay") + // MariaDB stays: it may have been the machine's before, and holds the + // jobs' records. + for _, u := range []string{"slurmd", "slurmctld", "slurmdbd"} { _ = k.systemctl(ctx, "disable", "--now", u) } _ = k.host.Remove(DropIn) _ = k.systemctl(ctx, "daemon-reload") _ = k.host.Remove(k.marker) - k.files = "" + k.files, k.acctDone, k.clusterAdded = "", "", "" } // write writes a file when it differs, saying whether it did. @@ -551,7 +580,8 @@ func (Local) LookPath(name string) (string, error) { if p, err := exec.LookPath(name); err == nil { return p, nil } - for _, d := range []string{"/usr/sbin", "/sbin", "/usr/local/sbin"} { + // /usr/libexec: Red Hat's mariadbd. + for _, d := range []string{"/usr/sbin", "/sbin", "/usr/local/sbin", "/usr/libexec"} { p := filepath.Join(d, name) if fi, err := os.Stat(p); err == nil && !fi.IsDir() { return p, nil diff --git a/internal/agent/slurmnode/slurmnode_test.go b/internal/agent/slurmnode/slurmnode_test.go index c1dc80c..2f70c0d 100644 --- a/internal/agent/slurmnode/slurmnode_test.go +++ b/internal/agent/slurmnode/slurmnode_test.go @@ -32,6 +32,7 @@ func newFake() *fakeHost { "slurmctld -V": "slurm-wlm 23.11.4\n", "scontrol ping": "Slurmctld(primary) at node06 is UP\n", "slurmd -C": "NodeName=cuda01 CPUs=16 RealMemory=64000\nUpTime=1-00:00:00\n", + "hostname -s": "node06\n", }, fail: map[string]bool{}, state: "IDLE+DRAIN", reason: slurm.ReasonNew} } @@ -118,7 +119,7 @@ type fakeInst struct { func (i *fakeInst) InstallPackages(_ context.Context, by map[string][]string) error { i.got = by - for _, b := range []string{"munged", "scontrol", "slurmd", "slurmctld"} { + for _, b := range []string{"munged", "scontrol", "slurmd", "slurmctld", "slurmdbd", "mariadbd", "mariadb"} { i.host.bins[b] = true } return nil @@ -237,6 +238,33 @@ func TestController(t *testing.T) { if !st.GetSlurmctldRunning() || len(st.GetFilesSha256()) != 64 { t.Fatalf("status %v", st) } + // Accounting: MariaDB on this machine alone, the database, slurmdbd on + // it, the cluster added, all before slurmctld. + if !slices.Contains(inst.got["deb"], "slurmdbd") || !slices.Contains(inst.got["deb"], "mariadb-server") { + t.Fatalf("installed %v", inst.got) + } + if !strings.Contains(string(h.files["/etc/mysql/mariadb.conf.d/90-llmgw-slurm.cnf"]), "bind-address=127.0.0.1") { + t.Fatal("MariaDB, installed by llmgw, not kept to this machine") + } + dbd := string(h.files[slurm.DbdConfPath]) + if !strings.Contains(dbd, "DbdHost=node06\n") || !strings.Contains(dbd, "StoragePass="+slurm.DbdPassword(key)+"\n") || + h.owners[slurm.DbdConfPath] != "slurm" { + t.Fatalf("slurmdbd.conf for %q:\n%s", h.owners[slurm.DbdConfPath], dbd) + } + if _, ok := h.files[acctSQLPath]; ok { + t.Fatal("the database's statements, password and all, were left on disk") + } + if strings.Contains(strings.Join(h.ran, "\n"), slurm.DbdPassword(key)) { + t.Fatal("the password was on a command line") + } + order := strings.Join(h.ran, "\n") + if !strings.Contains(order, "sacctmgr -i add cluster llmgw") || + strings.Index(order, "slurmdbd") > strings.Index(order, "systemctl restart slurmctld") { + t.Fatalf("accounting not before slurmctld: %v", h.ran) + } + if !st.GetSlurmdbdRunning() { + t.Fatalf("status %v", st) + } sum := st.GetFilesSha256() h.ran = nil @@ -279,7 +307,7 @@ func TestStop(t *testing.T) { if st := k.Reconcile(context.Background(), &llmgwv1.Slurm{}); st != nil { t.Fatalf("status %v for no part", st) } - if h.count("systemctl disable --now slurmd") != 1 { + if !slices.Contains(h.ran, "systemctl disable --now slurmd") { t.Fatalf("slurmd not stopped: %v", h.ran) } if _, ok := h.files[DropIn]; ok { @@ -306,3 +334,24 @@ func TestBadNames(t *testing.T) { t.Fatalf("status %v", st) } } + +func TestAMariaDBThatWasThereKeepsItsSettings(t *testing.T) { + h := newFake() + for _, b := range []string{"munged", "scontrol", "slurmctld", "slurmdbd", "mariadbd", "mariadb"} { + h.bins[b] = true + } + k := New(h, &fakeInst{host: h}, "/m", nil) + st := k.Reconcile(context.Background(), &llmgwv1.Slurm{Controller: true, MungeKey: key, + Files: map[string]string{"slurm.conf": "ClusterName=llmgw\n"}}) + if st.GetError() != "" { + t.Fatal(st.GetError()) + } + for p := range h.files { + if strings.HasSuffix(p, ".cnf") { + t.Fatalf("a MariaDB llmgw did not install had its settings changed: %s", p) + } + } + if h.count("mariadb -e source") != 1 || !st.GetSlurmdbdRunning() { + t.Fatalf("the database was not made in the MariaDB that was there: %v", h.ran) + } +} diff --git a/internal/cli/ctl_slurm.go b/internal/cli/ctl_slurm.go index cc7e604..1ba5abe 100644 --- a/internal/cli/ctl_slurm.go +++ b/internal/cli/ctl_slurm.go @@ -55,8 +55,10 @@ func newSlurmStatusCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { fmt.Fprintln(w, ": not heard from yet") case c.Error != "": fmt.Fprintf(w, ": %s\n", c.Error) + case c.SlurmctldRunning && c.SlurmdbdRunning: + fmt.Fprintf(w, ": slurmctld %s and slurmdbd running\n", c.Version) case c.SlurmctldRunning: - fmt.Fprintf(w, ": slurmctld %s running\n", c.Version) + fmt.Fprintf(w, ": slurmctld %s running, slurmdbd not (the jobs' records)\n", c.Version) default: fmt.Fprintln(w, ": slurmctld not running") } @@ -96,7 +98,8 @@ func newSlurmSetupCmd(client func() *ctlClient) *cobra.Command { Use: "setup --controller NODE --nodes NODE,... | --all-nodes", Short: "Run Slurm on llmgw's nodes: slurmctld on one, slurmd on those named", Long: "Turns slurm.enabled on and names the nodes, or with --all-nodes every node. Their agents install munge and Slurm (22.05 or\n" + - "newer) from the machines' own package sources. Every node starts drained in Slurm and stays\n" + + "newer) from the machines' own package sources; the controller also slurmdbd, the jobs' records, in a\n" + + "MariaDB on that machine alone (one that was there keeps its settings). Every node starts drained in Slurm and stays\n" + "llmgw's until `llmgw ctl slurm lend` gives it to Slurm.", Example: " llmgw ctl slurm setup --controller node06 --nodes cuda01,cuda02\n" + " llmgw ctl slurm setup --controller node06 --all-nodes", diff --git a/internal/cli/serverinit.go b/internal/cli/serverinit.go index bb1da4d..22d2f90 100644 --- a/internal/cli/serverinit.go +++ b/internal/cli/serverinit.go @@ -262,7 +262,7 @@ func newServerInitCmd() *cobra.Command { f.BoolVar(&in.media, "media", false, "pictures and 3D: the nodes install ComfyUI, and a picture model and a 3D model are added once a card fits one") f.BoolVar(&in.slurm, "slurm", false, - "Slurm for training and batch jobs: slurmctld on this machine, slurmd on every node, each llmgw's until 'llmgw ctl slurm lend'") + "Slurm for training and batch jobs: slurmctld and slurmdbd (with a MariaDB) on this machine, slurmd on every node, each llmgw's until 'llmgw ctl slurm lend'") f.DurationVar(&in.evictAfterIdle, "evict-after-idle", 0, "how long a model loaded on demand goes unasked-for before another may have its card (0: 2m)") f.DurationVar(&in.minLifetime, "min-lifetime", 0, diff --git a/internal/cli/serverinit_choices.go b/internal/cli/serverinit_choices.go index bc0ef9f..c28fc9f 100644 --- a/internal/cli/serverinit_choices.go +++ b/internal/cli/serverinit_choices.go @@ -350,7 +350,8 @@ func (in *serverInit) tuneConfig(body string) string { // slurmSection is server.yaml's slurm block as an install chooses it: // slurmctld on this machine's node, slurmd on every node. func slurmSection(controller string) string { - return "# Slurm beside llmgw, chosen at install: slurmctld on this machine, slurmd on\n" + + return "# Slurm beside llmgw, chosen at install: slurmctld, and slurmdbd with a MariaDB\n" + + "# for the jobs' records, on this machine; slurmd on\n" + "# every node, those that join later too. Each node is llmgw's, drained in\n" + "# Slurm, until 'llmgw ctl slurm lend NODE' gives it to Slurm; 'llmgw ctl slurm\n" + "# reclaim NODE' takes it back once its jobs end.\n" + diff --git a/internal/cli/tui_choices.go b/internal/cli/tui_choices.go index ead679d..0a49056 100644 --- a/internal/cli/tui_choices.go +++ b/internal/cli/tui_choices.go @@ -287,7 +287,9 @@ func newInstallForm(in *serverInit, host string, found []string, ha bool) *insta title: "Slurm", on: in.slurm, about: "A Slurm cluster beside llmgw, for training and other batch work: slurmctld on this machine, " + "slurmd on every node, those that join later too. The agents install munge and Slurm from the " + - "machines' own package sources.\n\n" + + "machines' own package sources, and on this machine slurmdbd, the jobs' records, with a MariaDB " + + "that answers this machine alone (Slurm keeps its records in MySQL or MariaDB only, not in the " + + "cluster's PostgreSQL). Its port 6819, with Slurm's others, answers the cluster's networks only.\n\n" + "Each node stays llmgw's, drained in Slurm, until 'llmgw ctl slurm lend NODE' gives it to Slurm: " + "its models move off first. 'llmgw ctl slurm reclaim NODE' takes it back once its jobs end. Jobs " + "are submitted on this machine with sbatch and srun.", diff --git a/internal/cli/tui_slurm.go b/internal/cli/tui_slurm.go index 11f42ae..6cb845d 100644 --- a/internal/cli/tui_slurm.go +++ b/internal/cli/tui_slurm.go @@ -35,8 +35,10 @@ func (m consoleModel) slurmPanel(pw int) (string, string, string) { case c == nil: case c.Error != "": ctl = sBad.Render(c.Error) + case c.SlurmctldRunning && c.SlurmdbdRunning: + ctl = sOK.Render("slurmctld " + c.Version + " and slurmdbd running") case c.SlurmctldRunning: - ctl = sOK.Render("slurmctld " + c.Version + " running") + ctl = sOK.Render("slurmctld "+c.Version+" running") + ", " + sWarn.Render("slurmdbd not") default: ctl = sBad.Render("slurmctld not running") } diff --git a/internal/cli/tui_slurm_test.go b/internal/cli/tui_slurm_test.go index b6d03c2..87393ce 100644 --- a/internal/cli/tui_slurm_test.go +++ b/internal/cli/tui_slurm_test.go @@ -37,7 +37,7 @@ func TestTheConsoleLendsANodeToSlurmFromItsRow(t *testing.T) { m := newConsoleModel(srv.URL, "/nonexistent/server.yaml") m.loaded, m.menu, m.inPanel = true, cSlurm, true m.data.slurm = slurmsvc.Status{Enabled: true, Cluster: "llmgw", Controller: "node06", Partition: "gpu", - Ctld: &slurmsvc.Reported{SlurmctldRunning: true, Version: "23.11.4"}, + Ctld: &slurmsvc.Reported{SlurmctldRunning: true, SlurmdbdRunning: true, Version: "23.11.4"}, Nodes: []slurmsvc.NodeStatus{ {Node: "node05", Phase: slurmsvc.Lent, Connected: true, Status: &slurmsvc.Reported{State: "MIXED", Jobs: 2, SlurmdRunning: true}}, @@ -46,7 +46,7 @@ func TestTheConsoleLendsANodeToSlurmFromItsRow(t *testing.T) { }} var tm tea.Model = m view := tm.View() - for _, want := range []string{"slurmctld 23.11.4 running", "node05", "Slurm's", "MIXED", "cuda01", "llmgw's"} { + for _, want := range []string{"slurmctld 23.11.4 and slurmdbd running", "node05", "Slurm's", "MIXED", "cuda01", "llmgw's"} { if !strings.Contains(view, want) { t.Errorf("the Slurm view has no %q:\n%s", want, view) } diff --git a/internal/control/slurmsvc/service.go b/internal/control/slurmsvc/service.go index 839014e..53cabd9 100644 --- a/internal/control/slurmsvc/service.go +++ b/internal/control/slurmsvc/service.go @@ -75,6 +75,7 @@ type Reported struct { Hardware string `json:"hardware,omitempty"` SlurmdRunning bool `json:"slurmd_running"` SlurmctldRunning bool `json:"slurmctld_running"` + SlurmdbdRunning bool `json:"slurmdbd_running"` State string `json:"state,omitempty"` Reason string `json:"reason,omitempty"` Jobs int `json:"jobs"` @@ -197,7 +198,7 @@ func (s *Service) Kick() { // OnSlurmStatus records what a node's agent says. func (s *Service) OnSlurmStatus(node string, st *llmgwv1.SlurmStatus) { r := &Reported{Version: st.GetVersion(), Hardware: st.GetHardware(), SlurmdRunning: st.GetSlurmdRunning(), - SlurmctldRunning: st.GetSlurmctldRunning(), State: st.GetState(), Reason: st.GetReason(), + SlurmctldRunning: st.GetSlurmctldRunning(), SlurmdbdRunning: st.GetSlurmdbdRunning(), State: st.GetState(), Reason: st.GetReason(), Jobs: int(st.GetJobs()), Error: st.GetError(), FilesSHA256: st.GetFilesSha256(), At: time.Now()} s.mu.Lock() prev := s.status[node] diff --git a/internal/slurm/dbd.go b/internal/slurm/dbd.go new file mode 100644 index 0000000..a4b5835 --- /dev/null +++ b/internal/slurm/dbd.go @@ -0,0 +1,81 @@ +package slurm + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" +) + +// Accounting: slurmdbd on the controller, keeping the jobs' records in a +// MariaDB beside it that answers this machine alone. slurmdbd takes MySQL or +// MariaDB only, so it is not the cluster's PostgreSQL. Since 23.11 Slurm +// cannot run without it well: its main scheduler passes every job by as +// InvalidAccount, and backfill starts it a pass later. +const ( + // DbdPort is slurmdbd's, which slurmctld and sacct reach. + DbdPort = 6819 + // DbdConfPath is slurmdbd's file, readable by the slurm user alone: it + // holds the database's password. + DbdConfPath = ConfDir + "/slurmdbd.conf" + // AcctDB and AcctUser are the database and its user. + AcctDB = "slurm_acct_db" + AcctUser = "slurm" +) + +// DbdPassword is the database user's password, made from the munge key: +// every controller the cluster has gets the same one, and nothing new is +// kept anywhere. +func DbdPassword(mungeKey []byte) string { + h := sha256.New() + h.Write([]byte("llmgw slurmdbd password\x00")) + h.Write(mungeKey) + return hex.EncodeToString(h.Sum(nil))[:40] +} + +// DbdConf is slurmdbd.conf for the controller host, named as `hostname -s` +// says, which slurmdbd checks it runs on. +func DbdConf(host, password string) string { + return header + fmt.Sprintf(`AuthType=auth/munge +DbdHost=%s +DbdPort=%d +SlurmUser=slurm +LogFile=%s/slurmdbd.log +PidFile=/run/slurmdbd.pid +StorageType=accounting_storage/mysql +StorageHost=localhost +StorageUser=%s +StoragePass=%s +StorageLoc=%s +# Kept a year; older records are purged, a month at a time. +PurgeEventAfter=12months +PurgeJobAfter=12months +PurgeResvAfter=12months +PurgeStepAfter=12months +PurgeSuspendAfter=12months +PurgeTXNAfter=12months +PurgeUsageAfter=24months +`, host, DbdPort, LogDir, AcctUser, password, AcctDB) +} + +// MariaDBConf is the database server's settings: this machine alone, and +// what Slurm's documentation asks of InnoDB. +const MariaDBConf = `# Written by llmgw (BACKLOG P): the database of Slurm's accounting +# (slurmdbd), on this machine alone. +[mysqld] +bind-address=127.0.0.1 +skip-networking=0 +innodb_buffer_pool_size=256M +innodb_lock_wait_timeout=900 +` + +// AcctSQL makes the database and its user, and gives the user the password +// again: run on every start of the controller's part, it changes nothing +// that is already so. +func AcctSQL(password string) string { + return fmt.Sprintf(`CREATE DATABASE IF NOT EXISTS %[1]s; +CREATE USER IF NOT EXISTS '%[2]s'@'localhost' IDENTIFIED BY '%[3]s'; +ALTER USER '%[2]s'@'localhost' IDENTIFIED BY '%[3]s'; +GRANT ALL ON %[1]s.* TO '%[2]s'@'localhost'; +FLUSH PRIVILEGES; +`, AcctDB, AcctUser, password) +} diff --git a/internal/slurm/slurm.go b/internal/slurm/slurm.go index 411d00f..07c1e52 100644 --- a/internal/slurm/slurm.go +++ b/internal/slurm/slurm.go @@ -196,6 +196,9 @@ func (c Cluster) SlurmConf() string { b.WriteString(header) fmt.Fprintf(&b, "ClusterName=%s\n", c.Name) fmt.Fprintf(&b, "SlurmctldHost=%s(%s)\n", c.Controller, c.ControllerAddr) + // The jobs' records, in slurmdbd beside slurmctld (DbdConf). + fmt.Fprintf(&b, "AccountingStorageType=accounting_storage/slurmdbd\nAccountingStorageHost=%s\nAccountingStoragePort=%d\n", + c.ControllerAddr, DbdPort) b.WriteString(`SlurmctldParameters=enable_configless SlurmctldPort=6817 SlurmdPort=6818 @@ -212,17 +215,13 @@ SlurmctldPidFile=/run/slurmctld.pid SlurmdPidFile=/run/slurmd.pid ProctrackType=proctrack/cgroup TaskPlugin=task/cgroup,task/affinity -# Nothing gathered for accounting until there is somewhere to keep it -# (slurmdbd, later); gathered and not kept is an error in every log. -JobAcctGatherType=jobacct_gather/none +# What each job used, from its cgroup, for the records; the cards too. +JobAcctGatherType=jobacct_gather/cgroup +AccountingStorageTRES=gres/gpu SelectType=select/cons_tres SelectTypeParameters=CR_Core_Memory SchedulerType=sched/backfill -# First come, first served: multifactor's fair share needs accounting -# storage (slurmdbd), which there is none of yet. Without it, 23.11's main -# scheduler passes every job by as InvalidAccount, and backfill starts it -# on its next pass (bf_interval above). -PriorityType=priority/basic +PriorityType=priority/multifactor GresTypes=gpu MpiDefault=none # No mail sent; without it every start says MailProg is invalid. diff --git a/internal/slurm/slurm_test.go b/internal/slurm/slurm_test.go index 22caf93..f6698e3 100644 --- a/internal/slurm/slurm_test.go +++ b/internal/slurm/slurm_test.go @@ -165,7 +165,29 @@ func TestALentNodeIsNotWrittenDrained(t *testing.T) { if line == "" || strings.Contains(line, "DRAIN") { t.Fatalf("a lent node, written %q: every start of slurmctld would drain it again", line) } - if !strings.Contains(conf, "PriorityType=priority/basic\n") { - t.Error("multifactor without accounting: the main scheduler passes every job by as InvalidAccount") +} + +func TestTheJobsAreAccountedInSlurmdbd(t *testing.T) { + conf := testCluster().SlurmConf() + for _, want := range []string{ + "AccountingStorageType=accounting_storage/slurmdbd\n", + "AccountingStorageHost=10.0.0.146\n", + "AccountingStoragePort=6819\n", + "JobAcctGatherType=jobacct_gather/cgroup\n", + "AccountingStorageTRES=gres/gpu\n", + } { + if !strings.Contains(conf, want) { + t.Errorf("slurm.conf has no %q", want) + } + } + dbd := DbdConf("node06", DbdPassword([]byte("key"))) + for _, want := range []string{"DbdHost=node06\n", "DbdPort=6819\n", "StorageType=accounting_storage/mysql\n", + "StorageHost=localhost\n", "StorageLoc=slurm_acct_db\n", "AuthType=auth/munge\n"} { + if !strings.Contains(dbd, want) { + t.Errorf("slurmdbd.conf has no %q:\n%s", want, dbd) + } + } + if DbdPassword([]byte("key")) == DbdPassword([]byte("other")) || len(DbdPassword([]byte("key"))) != 40 { + t.Error("the password does not follow the key") } }