Slurm: jobs start on submission, and a lent node stays open across restarts

Without accounting storage, 23.11's default priority/multifactor has the
main scheduler pass every job by as InvalidAccount; backfill started it a
pass later. slurm.conf sets priority/basic.

Every node was written State=DRAIN, so each start of slurmctld on new
files drained a lent node again until its agent resumed it. A lent node
is written without it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

BACKLOG.md

@@ -2039,7 +2039,11 @@ lent, partition`); the leader tells each named node's agent (root) what to be, t
   InvalidAccount and backfill starts them, one a pass (bf_interval=10: 6-10 s each). So many small jobs
   want slurmdbd. — Wrong cause, found 2026-11-11: a job asking no memory got the whole node's (ReqTRES
   mem=all), so a node ran one job at a time and each next one waited for backfill's pass. Fixed: the
-  partition's DefMemPerCPU is the leanest node's share; slurmdbd is not needed for this. Not tried yet: a GPU node (gres).
+  partition's DefMemPerCPU is the leanest node's share; slurmdbd is not needed for this. And InvalidAccount is
+  real: 23.11 defaults to priority/multifactor, whose main scheduler passes every job by without
+  accounting storage; PriorityType=priority/basic starts a job on submission. Also found: slurm.conf
+  wrote every node State=DRAIN, so each slurmctld start (on any change of the files) drained a lent node
+  until its agent resumed it, up to a minute; a lent node is now written without it. Not tried yet: a GPU node (gres).
 - Failover while lent, tried 2026-11-11 (v0.16.3-23): the first try killed the job (NODE_FAIL). A new
   leader had heard from no node yet and sent the controller a slurm.conf without them; slurmctld,
   restarted on it, dropped node05 and its job. Fixed (141a1f4, 0774e90): slurm.conf's nodes and the

internal/control/slurmsvc/service.go

@@ -417,13 +417,16 @@ func (s *Service) cluster(set config.SlurmSettings) (slurm.Cluster, []string) {
 	c.ControllerAddr = s.controllerAddr(set)
 	var waiting []string
 	for _, node := range set.Nodes {
-		if sn, ok := s.reported(node); ok {
-			c.Nodes = append(c.Nodes, sn)
-		} else if sn, ok := s.known.Nodes[node]; ok {
-			c.Nodes = append(c.Nodes, sn)
-		} else {
-			waiting = append(waiting, node)
+		sn, ok := s.reported(node)
+		if !ok {
+			sn, ok = s.known.Nodes[node]
 		}
+		if !ok {
+			waiting = append(waiting, node)
+			continue
+		}
+		sn.Lent = set.IsLent(node)
+		c.Nodes = append(c.Nodes, sn)
 	}
 	return c, waiting
 }

internal/slurm/slurm.go

@@ -120,6 +120,9 @@ type Node struct {
 	Addr string
 	Hardware
 	GPUs []GPU
+	// Lent is whether llmgw has lent it to Slurm: it is not written
+	// drained, or every start of slurmctld on these files drains it again.
+	Lent bool `json:"-"`
 }
 
 // Cluster is everything slurm.conf says.
@@ -215,6 +218,10 @@ JobAcctGatherType=jobacct_gather/none
 SelectType=select/cons_tres
 SelectTypeParameters=CR_Core_Memory
 SchedulerType=sched/backfill
+# Without accounting storage, 23.11's default (multifactor) has the main
+# scheduler pass every job by as InvalidAccount, for backfill to start a
+# pass later; basic is first come, first served, started on submission.
+PriorityType=priority/basic
 GresTypes=gpu
 MpiDefault=none
 # No mail sent; without it every start says MailProg is invalid.
@@ -255,7 +262,11 @@ SlurmdTimeout=300
 		}
 		// Drained until llmgw lends it: a node new to the controller would
 		// otherwise take jobs in the minute before its agent drains it.
-		fmt.Fprintf(&b, " State=DRAIN Reason=%q\n", ReasonNew)
+		if n.Lent {
+			b.WriteString("\n")
+		} else {
+			fmt.Fprintf(&b, " State=DRAIN Reason=%q\n", ReasonNew)
+		}
 	}
 	if len(names) > 0 {
 		fmt.Fprintf(&b, "PartitionName=%s Nodes=%s Default=YES MaxTime=INFINITE State=UP DefMemPerCPU=%d\n",

internal/slurm/slurm_test.go

@@ -151,3 +151,21 @@ func TestParseVersion(t *testing.T) {
 		t.Error("no version was read as one")
 	}
 }
+
+func TestALentNodeIsNotWrittenDrained(t *testing.T) {
+	c := testCluster()
+	c.Nodes[0].Lent = true
+	conf := c.SlurmConf()
+	line := ""
+	for _, l := range strings.Split(conf, "\n") {
+		if strings.HasPrefix(l, "NodeName="+c.Nodes[0].Name+" ") {
+			line = l
+		}
+	}
+	if line == "" || strings.Contains(line, "DRAIN") {
+		t.Fatalf("a lent node, written %q: every start of slurmctld would drain it again", line)
+	}
+	if !strings.Contains(conf, "PriorityType=priority/basic\n") {
+		t.Error("multifactor without accounting: the main scheduler passes every job by as InvalidAccount")
+	}
+}