Slurm: a new leader with nothing kept waits for the nodes a while

On a cluster upgraded to the kept nodes, the first leader finds none and
wrote slurm.conf without the nodes it had not heard from yet, as before.
A node neither reported nor kept now holds the controller's files back
for slurmsvc.HoldFor (2 minutes) from the leader's first pass.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/control/slurmsvc/failover_test.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"strings"
 	"testing"
+	"time"
 
 	llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/config"
@@ -115,3 +116,18 @@ func TestWithTheStoreUnreadTheControllerIsSentNothingThatLeavesANodeOut(t *testi
 		t.Fatal("with every node reported, the controller still got no files")
 	}
 }
+
+func TestWithNothingKeptANewLeaderWaitsForTheNodesAWhile(t *testing.T) {
+	r := newRig()
+	r.svc.SetMemory(StoreMemory(&fakeKV{m: map[string]string{}}))
+	r.tick()
+	if m := r.agents.last("node06"); m != nil {
+		t.Fatalf("with nothing kept, the controller was sent %v before cuda01 reported", m)
+	}
+	defer func(d time.Duration) { HoldFor = d }(HoldFor)
+	HoldFor = 0
+	r.tick()
+	if m := r.agents.last("node06"); !m.GetController() || strings.Contains(m.GetFiles()["slurm.conf"], "NodeName=cuda01") {
+		t.Fatalf("after the wait, the controller was told %v", m)
+	}
+}

internal/control/slurmsvc/service.go

@@ -123,8 +123,18 @@ type Service struct {
 	mem    Memory
 	known  Kept
 	loaded bool
+	// started is the first pass, from which a node neither heard from nor
+	// kept is waited for, for HoldFor.
+	started time.Time
 }
 
+// HoldFor is how long a new leader waits for a node it neither has heard
+// from nor finds kept before it writes the controller's files without it:
+// on a cluster that kept nothing yet (one upgraded to this), the nodes
+// report within seconds of a new leader, and the controller keeps the
+// node and its jobs meanwhile.
+var HoldFor = 2 * time.Minute
+
 // New is a Service; key gives the munge key, made once from a cluster
 // secret.
 func New(settings func() config.SlurmSettings, inv Inventory, agents Agents,
@@ -216,6 +226,9 @@ func (s *Service) Tick(ctx context.Context) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	s.recall(ctx)
+	if s.started.IsZero() {
+		s.started = time.Now()
+	}
 	phases := s.handOver(set)
 	var key []byte
 	if set.Enabled {
@@ -232,7 +245,7 @@ func (s *Service) Tick(ctx context.Context) {
 	files, waiting := s.files(set)
 	// Not yet knowing what the last leader knew, a slurm.conf without some
 	// node would take it, and its jobs, from the controller: kept back.
-	hold := s.mem != nil && !s.loaded && len(waiting) > 0
+	hold := s.mem != nil && len(waiting) > 0 && (!s.loaded || time.Since(s.started) < HoldFor)
 	for _, node := range s.agents.ConnectedNodes() {
 		if hold && node == set.Controller {
 			continue