Slurm: slurmctld started again after a reconfigure; no accounting gathered

Found on the PoC: Slurm 23.11.4 without accounting storage left batch jobs
pending InvalidAccount after scontrol reconfigure until slurmctld was
restarted. And jobacct_gather with nowhere to keep it logs an error per job.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/agent/slurmnode/slurmnode.go

@@ -326,6 +326,18 @@ func (k *Keeper) controller(ctx context.Context, files map[string]string) (strin
 		if out, err := k.host.Run(ctx, nil, "scontrol", "reconfigure"); err != nil {
 			return sum, fmt.Errorf("scontrol reconfigure: %v: %s", err, lastLine(out))
 		}
+		// And starts again after it: found on the PoC, 23.11.4 without
+		// accounting storage left every batch job pending InvalidAccount
+		// after a reconfigure, until slurmctld was restarted.
+		if err := k.waitPing(ctx); err != nil {
+			return sum, err
+		}
+		if err := k.systemctl(ctx, "restart", "slurmctld"); err != nil {
+			return sum, err
+		}
+		if err := k.waitPing(ctx); err != nil {
+			return sum, err
+		}
 		k.files = sum
 		return sum, nil
 	}

internal/agent/slurmnode/slurmnode_test.go

@@ -231,7 +231,7 @@ func TestController(t *testing.T) {
 	if string(h.files["/etc/slurm/slurm.conf"]) != "ClusterName=llmgw\n" {
 		t.Fatal("slurm.conf not written")
 	}
-	if h.count("systemctl restart slurmctld") != 1 || h.count("scontrol reconfigure") != 1 {
+	if h.count("systemctl restart slurmctld") != 2 || h.count("scontrol reconfigure") != 1 {
 		t.Fatalf("slurmctld not started on its files: %v", h.ran)
 	}
 	if !st.GetSlurmctldRunning() || len(st.GetFilesSha256()) != 64 {
@@ -248,7 +248,7 @@ func TestController(t *testing.T) {
 	files["gres.conf"] = "NodeName=cuda01 Name=gpu File=/dev/nvidia0\n"
 	h.ran = nil
 	st = k.Reconcile(context.Background(), m)
-	if h.count("systemctl restart slurmctld") != 1 || st.GetFilesSha256() == sum {
+	if h.count("systemctl restart slurmctld") != 2 || st.GetFilesSha256() == sum {
 		t.Fatalf("a changed file did not restart slurmctld: %v", h.ran)
 	}
 

internal/slurm/slurm.go

@@ -201,7 +201,9 @@ SlurmctldPidFile=/run/slurmctld.pid
 SlurmdPidFile=/run/slurmd.pid
 ProctrackType=proctrack/cgroup
 TaskPlugin=task/cgroup,task/affinity
-JobAcctGatherType=jobacct_gather/cgroup
+# Nothing gathered for accounting until there is somewhere to keep it
+# (slurmdbd, later); gathered and not kept is an error in every log.
+JobAcctGatherType=jobacct_gather/none
 SelectType=select/cons_tres
 SelectTypeParameters=CR_Core_Memory
 SchedulerType=sched/backfill