diff --git a/internal/agent/slurmnode/slurmnode.go b/internal/agent/slurmnode/slurmnode.go index cdaa800..3bb1ceb 100644 --- a/internal/agent/slurmnode/slurmnode.go +++ b/internal/agent/slurmnode/slurmnode.go @@ -326,18 +326,6 @@ func (k *Keeper) controller(ctx context.Context, files map[string]string) (strin if out, err := k.host.Run(ctx, nil, "scontrol", "reconfigure"); err != nil { return sum, fmt.Errorf("scontrol reconfigure: %v: %s", err, lastLine(out)) } - // And starts again after it: found on the PoC, 23.11.4 without - // accounting storage left every batch job pending InvalidAccount - // after a reconfigure, until slurmctld was restarted. - if err := k.waitPing(ctx); err != nil { - return sum, err - } - if err := k.systemctl(ctx, "restart", "slurmctld"); err != nil { - return sum, err - } - if err := k.waitPing(ctx); err != nil { - return sum, err - } k.files = sum return sum, nil } diff --git a/internal/agent/slurmnode/slurmnode_test.go b/internal/agent/slurmnode/slurmnode_test.go index 63b34db..c1dc80c 100644 --- a/internal/agent/slurmnode/slurmnode_test.go +++ b/internal/agent/slurmnode/slurmnode_test.go @@ -231,7 +231,7 @@ func TestController(t *testing.T) { if string(h.files["/etc/slurm/slurm.conf"]) != "ClusterName=llmgw\n" { t.Fatal("slurm.conf not written") } - if h.count("systemctl restart slurmctld") != 2 || h.count("scontrol reconfigure") != 1 { + if h.count("systemctl restart slurmctld") != 1 || h.count("scontrol reconfigure") != 1 { t.Fatalf("slurmctld not started on its files: %v", h.ran) } if !st.GetSlurmctldRunning() || len(st.GetFilesSha256()) != 64 { @@ -248,7 +248,7 @@ func TestController(t *testing.T) { files["gres.conf"] = "NodeName=cuda01 Name=gpu File=/dev/nvidia0\n" h.ran = nil st = k.Reconcile(context.Background(), m) - if h.count("systemctl restart slurmctld") != 2 || st.GetFilesSha256() == sum { + if h.count("systemctl restart slurmctld") != 1 || st.GetFilesSha256() == sum { t.Fatalf("a changed file did not restart slurmctld: %v", h.ran) } diff --git a/internal/slurm/slurm.go b/internal/slurm/slurm.go index c709862..3c4d6cc 100644 --- a/internal/slurm/slurm.go +++ b/internal/slurm/slurm.go @@ -209,6 +209,11 @@ SelectTypeParameters=CR_Core_Memory SchedulerType=sched/backfill GresTypes=gpu MpiDefault=none +# No mail sent; without it every start says MailProg is invalid. +MailProg=/bin/true +# Without accounting storage, 23.11's main scheduler marks a batch job +# InvalidAccount; backfill starts it on its next pass, so it is kept short. +SchedulerParameters=bf_interval=10 # A node that went down and comes back is taken again; one llmgw drained # stays drained until llmgw resumes it. ReturnToService=2