Slurm's accounting: slurmdbd and a MariaDB on the controller
CI / test (push) Successful in 10m57s

Since 23.11 Slurm runs badly without accounting storage: the main
scheduler passes every job by as InvalidAccount. slurmdbd takes MySQL or
MariaDB only, so the controller's agent installs MariaDB beside it,
answering that machine alone, makes the database and its user (the
password made from the munge key, never on a command line), runs
slurmdbd and adds the cluster, all before slurmctld. A MariaDB that was
on the machine before keeps its settings. slurm.conf accounts the jobs
there, with what each used from its cgroup and the cards, and the
priority is multifactor again. Port 6819 joins Slurm's in the firewall.
ctl slurm status and the console say whether slurmdbd runs; the install
screen, server init --slurm and the guide say what is put where.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-12 07:59:08 +01:00
1 parent 02774769a7
commit 748f777a56
20 files changed
+425 -37

No files matched your search

+5 -2
View File
@@ -55,8 +55,10 @@ func newSlurmStatusCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
fmt.Fprintln(w, ": not heard from yet")
case c.Error != "":
fmt.Fprintf(w, ": %s\n", c.Error)
case c.SlurmctldRunning && c.SlurmdbdRunning:
fmt.Fprintf(w, ": slurmctld %s and slurmdbd running\n", c.Version)
case c.SlurmctldRunning:
fmt.Fprintf(w, ": slurmctld %s running\n", c.Version)
fmt.Fprintf(w, ": slurmctld %s running, slurmdbd not (the jobs' records)\n", c.Version)
default:
fmt.Fprintln(w, ": slurmctld not running")
}
@@ -96,7 +98,8 @@ func newSlurmSetupCmd(client func() *ctlClient) *cobra.Command {
Use: "setup --controller NODE --nodes NODE,... | --all-nodes",
Short: "Run Slurm on llmgw's nodes: slurmctld on one, slurmd on those named",
Long: "Turns slurm.enabled on and names the nodes, or with --all-nodes every node. Their agents install munge and Slurm (22.05 or\n" +
"newer) from the machines' own package sources. Every node starts drained in Slurm and stays\n" +
"newer) from the machines' own package sources; the controller also slurmdbd, the jobs' records, in a\n" +
"MariaDB on that machine alone (one that was there keeps its settings). Every node starts drained in Slurm and stays\n" +
"llmgw's until `llmgw ctl slurm lend` gives it to Slurm.",
Example: " llmgw ctl slurm setup --controller node06 --nodes cuda01,cuda02\n" +
" llmgw ctl slurm setup --controller node06 --all-nodes",
+1 -1
View File
@@ -262,7 +262,7 @@ func newServerInitCmd() *cobra.Command {
f.BoolVar(&in.media, "media", false,
"pictures and 3D: the nodes install ComfyUI, and a picture model and a 3D model are added once a card fits one")
f.BoolVar(&in.slurm, "slurm", false,
"Slurm for training and batch jobs: slurmctld on this machine, slurmd on every node, each llmgw's until 'llmgw ctl slurm lend'")
"Slurm for training and batch jobs: slurmctld and slurmdbd (with a MariaDB) on this machine, slurmd on every node, each llmgw's until 'llmgw ctl slurm lend'")
f.DurationVar(&in.evictAfterIdle, "evict-after-idle", 0,
"how long a model loaded on demand goes unasked-for before another may have its card (0: 2m)")
f.DurationVar(&in.minLifetime, "min-lifetime", 0,
+2 -1
View File
@@ -350,7 +350,8 @@ func (in *serverInit) tuneConfig(body string) string {
// slurmSection is server.yaml's slurm block as an install chooses it:
// slurmctld on this machine's node, slurmd on every node.
func slurmSection(controller string) string {
return "# Slurm beside llmgw, chosen at install: slurmctld on this machine, slurmd on\n" +
return "# Slurm beside llmgw, chosen at install: slurmctld, and slurmdbd with a MariaDB\n" +
"# for the jobs' records, on this machine; slurmd on\n" +
"# every node, those that join later too. Each node is llmgw's, drained in\n" +
"# Slurm, until 'llmgw ctl slurm lend NODE' gives it to Slurm; 'llmgw ctl slurm\n" +
"# reclaim NODE' takes it back once its jobs end.\n" +
+3 -1
View File
@@ -287,7 +287,9 @@ func newInstallForm(in *serverInit, host string, found []string, ha bool) *insta
title: "Slurm", on: in.slurm,
about: "A Slurm cluster beside llmgw, for training and other batch work: slurmctld on this machine, " +
"slurmd on every node, those that join later too. The agents install munge and Slurm from the " +
"machines' own package sources.\n\n" +
"machines' own package sources, and on this machine slurmdbd, the jobs' records, with a MariaDB " +
"that answers this machine alone (Slurm keeps its records in MySQL or MariaDB only, not in the " +
"cluster's PostgreSQL). Its port 6819, with Slurm's others, answers the cluster's networks only.\n\n" +
"Each node stays llmgw's, drained in Slurm, until 'llmgw ctl slurm lend NODE' gives it to Slurm: " +
"its models move off first. 'llmgw ctl slurm reclaim NODE' takes it back once its jobs end. Jobs " +
"are submitted on this machine with sbatch and srun.",
+3 -1
View File
@@ -35,8 +35,10 @@ func (m consoleModel) slurmPanel(pw int) (string, string, string) {
case c == nil:
case c.Error != "":
ctl = sBad.Render(c.Error)
case c.SlurmctldRunning && c.SlurmdbdRunning:
ctl = sOK.Render("slurmctld " + c.Version + " and slurmdbd running")
case c.SlurmctldRunning:
ctl = sOK.Render("slurmctld " + c.Version + " running")
ctl = sOK.Render("slurmctld "+c.Version+" running") + ", " + sWarn.Render("slurmdbd not")
default:
ctl = sBad.Render("slurmctld not running")
}
+2 -2
View File
@@ -37,7 +37,7 @@ func TestTheConsoleLendsANodeToSlurmFromItsRow(t *testing.T) {
m := newConsoleModel(srv.URL, "/nonexistent/server.yaml")
m.loaded, m.menu, m.inPanel = true, cSlurm, true
m.data.slurm = slurmsvc.Status{Enabled: true, Cluster: "llmgw", Controller: "node06", Partition: "gpu",
Ctld: &slurmsvc.Reported{SlurmctldRunning: true, Version: "23.11.4"},
Ctld: &slurmsvc.Reported{SlurmctldRunning: true, SlurmdbdRunning: true, Version: "23.11.4"},
Nodes: []slurmsvc.NodeStatus{
{Node: "node05", Phase: slurmsvc.Lent, Connected: true,
Status: &slurmsvc.Reported{State: "MIXED", Jobs: 2, SlurmdRunning: true}},
@@ -46,7 +46,7 @@ func TestTheConsoleLendsANodeToSlurmFromItsRow(t *testing.T) {
}}
var tm tea.Model = m
view := tm.View()
for _, want := range []string{"slurmctld 23.11.4 running", "node05", "Slurm's", "MIXED", "cuda01", "llmgw's"} {
for _, want := range []string{"slurmctld 23.11.4 and slurmdbd running", "node05", "Slurm's", "MIXED", "cuda01", "llmgw's"} {
if !strings.Contains(view, want) {
t.Errorf("the Slurm view has no %q:\n%s", want, view)
}