Slurm: no second restart; backfill every 10 s, MailProg set

The InvalidAccount seen on the PoC was not the reconfigure: 23.11.4 without
accounting storage has its main scheduler mark a batch job so, and backfill
starts it on its next pass (job 4 waited 15 s). bf_interval=10 keeps that
short; the extra restart is taken out again. MailProg=/bin/true quiets the
warning on every start.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/agent/slurmnode/slurmnode.go

@@ -326,18 +326,6 @@ func (k *Keeper) controller(ctx context.Context, files map[string]string) (strin
 		if out, err := k.host.Run(ctx, nil, "scontrol", "reconfigure"); err != nil {
 			return sum, fmt.Errorf("scontrol reconfigure: %v: %s", err, lastLine(out))
 		}
-		// And starts again after it: found on the PoC, 23.11.4 without
-		// accounting storage left every batch job pending InvalidAccount
-		// after a reconfigure, until slurmctld was restarted.
-		if err := k.waitPing(ctx); err != nil {
-			return sum, err
-		}
-		if err := k.systemctl(ctx, "restart", "slurmctld"); err != nil {
-			return sum, err
-		}
-		if err := k.waitPing(ctx); err != nil {
-			return sum, err
-		}
 		k.files = sum
 		return sum, nil
 	}

internal/agent/slurmnode/slurmnode_test.go

@@ -231,7 +231,7 @@ func TestController(t *testing.T) {
 	if string(h.files["/etc/slurm/slurm.conf"]) != "ClusterName=llmgw\n" {
 		t.Fatal("slurm.conf not written")
 	}
-	if h.count("systemctl restart slurmctld") != 2 || h.count("scontrol reconfigure") != 1 {
+	if h.count("systemctl restart slurmctld") != 1 || h.count("scontrol reconfigure") != 1 {
 		t.Fatalf("slurmctld not started on its files: %v", h.ran)
 	}
 	if !st.GetSlurmctldRunning() || len(st.GetFilesSha256()) != 64 {
@@ -248,7 +248,7 @@ func TestController(t *testing.T) {
 	files["gres.conf"] = "NodeName=cuda01 Name=gpu File=/dev/nvidia0\n"
 	h.ran = nil
 	st = k.Reconcile(context.Background(), m)
-	if h.count("systemctl restart slurmctld") != 2 || st.GetFilesSha256() == sum {
+	if h.count("systemctl restart slurmctld") != 1 || st.GetFilesSha256() == sum {
 		t.Fatalf("a changed file did not restart slurmctld: %v", h.ran)
 	}
 

internal/slurm/slurm.go

@@ -209,6 +209,11 @@ SelectTypeParameters=CR_Core_Memory
 SchedulerType=sched/backfill
 GresTypes=gpu
 MpiDefault=none
+# No mail sent; without it every start says MailProg is invalid.
+MailProg=/bin/true
+# Without accounting storage, 23.11's main scheduler marks a batch job
+# InvalidAccount; backfill starts it on its next pass, so it is kept short.
+SchedulerParameters=bf_interval=10
 # A node that went down and comes back is taken again; one llmgw drained
 # stays drained until llmgw resumes it.
 ReturnToService=2