From 087552136fa65f3895a41ffde0a485902c6e845f Mon Sep 17 00:00:00 2001
From: Ilker Manap <ilker@Ilkers-MacBook-Pro.local>
Date: Wed, 11 Nov 2026 13:16:31 +0100
Subject: [PATCH] Slurm: slurmctld started again after a reconfigure; no
 accounting gathered

Found on the PoC: Slurm 23.11.4 without accounting storage left batch jobs
pending InvalidAccount after scontrol reconfigure until slurmctld was
restarted. And jobacct_gather with nowhere to keep it logs an error per job.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 internal/agent/slurmnode/slurmnode.go      | 12 ++++++++++++
 internal/agent/slurmnode/slurmnode_test.go |  4 ++--
 internal/slurm/slurm.go                    |  4 +++-
 3 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/internal/agent/slurmnode/slurmnode.go b/internal/agent/slurmnode/slurmnode.go
index 3bb1ceb..cdaa800 100644
--- a/internal/agent/slurmnode/slurmnode.go
+++ b/internal/agent/slurmnode/slurmnode.go
@@ -326,6 +326,18 @@ func (k *Keeper) controller(ctx context.Context, files map[string]string) (strin
 		if out, err := k.host.Run(ctx, nil, "scontrol", "reconfigure"); err != nil {
 			return sum, fmt.Errorf("scontrol reconfigure: %v: %s", err, lastLine(out))
 		}
+		// And starts again after it: found on the PoC, 23.11.4 without
+		// accounting storage left every batch job pending InvalidAccount
+		// after a reconfigure, until slurmctld was restarted.
+		if err := k.waitPing(ctx); err != nil {
+			return sum, err
+		}
+		if err := k.systemctl(ctx, "restart", "slurmctld"); err != nil {
+			return sum, err
+		}
+		if err := k.waitPing(ctx); err != nil {
+			return sum, err
+		}
 		k.files = sum
 		return sum, nil
 	}
diff --git a/internal/agent/slurmnode/slurmnode_test.go b/internal/agent/slurmnode/slurmnode_test.go
index c1dc80c..63b34db 100644
--- a/internal/agent/slurmnode/slurmnode_test.go
+++ b/internal/agent/slurmnode/slurmnode_test.go
@@ -231,7 +231,7 @@ func TestController(t *testing.T) {
 	if string(h.files["/etc/slurm/slurm.conf"]) != "ClusterName=llmgw\n" {
 		t.Fatal("slurm.conf not written")
 	}
-	if h.count("systemctl restart slurmctld") != 1 || h.count("scontrol reconfigure") != 1 {
+	if h.count("systemctl restart slurmctld") != 2 || h.count("scontrol reconfigure") != 1 {
 		t.Fatalf("slurmctld not started on its files: %v", h.ran)
 	}
 	if !st.GetSlurmctldRunning() || len(st.GetFilesSha256()) != 64 {
@@ -248,7 +248,7 @@ func TestController(t *testing.T) {
 	files["gres.conf"] = "NodeName=cuda01 Name=gpu File=/dev/nvidia0\n"
 	h.ran = nil
 	st = k.Reconcile(context.Background(), m)
-	if h.count("systemctl restart slurmctld") != 1 || st.GetFilesSha256() == sum {
+	if h.count("systemctl restart slurmctld") != 2 || st.GetFilesSha256() == sum {
 		t.Fatalf("a changed file did not restart slurmctld: %v", h.ran)
 	}
 
diff --git a/internal/slurm/slurm.go b/internal/slurm/slurm.go
index 9e86b3d..c709862 100644
--- a/internal/slurm/slurm.go
+++ b/internal/slurm/slurm.go
@@ -201,7 +201,9 @@ SlurmctldPidFile=/run/slurmctld.pid
 SlurmdPidFile=/run/slurmd.pid
 ProctrackType=proctrack/cgroup
 TaskPlugin=task/cgroup,task/affinity
-JobAcctGatherType=jobacct_gather/cgroup
+# Nothing gathered for accounting until there is somewhere to keep it
+# (slurmdbd, later); gathered and not kept is an error in every log.
+JobAcctGatherType=jobacct_gather/none
 SelectType=select/cons_tres
 SelectTypeParameters=CR_Core_Memory
 SchedulerType=sched/backfill