From bab0766ee1778152691e46847f5cc2f4950155ba Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Wed, 11 Nov 2026 23:18:22 +0100 Subject: [PATCH] Slurm: jobs start on submission, and a lent node stays open across restarts Without accounting storage, 23.11's default priority/multifactor has the main scheduler pass every job by as InvalidAccount; backfill started it a pass later. slurm.conf sets priority/basic. Every node was written State=DRAIN, so each start of slurmctld on new files drained a lent node again until its agent resumed it. A lent node is written without it. Co-Authored-By: Claude Opus 5.5 --- BACKLOG.md | 6 +++++- internal/control/slurmsvc/service.go | 15 +++++++++------ internal/slurm/slurm.go | 13 ++++++++++++- internal/slurm/slurm_test.go | 18 ++++++++++++++++++ 4 files changed, 44 insertions(+), 8 deletions(-) diff --git a/BACKLOG.md b/BACKLOG.md index f8d5efe..b5dd08d 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -2039,7 +2039,11 @@ lent, partition`); the leader tells each named node's agent (root) what to be, t InvalidAccount and backfill starts them, one a pass (bf_interval=10: 6-10 s each). So many small jobs want slurmdbd. — Wrong cause, found 2026-11-11: a job asking no memory got the whole node's (ReqTRES mem=all), so a node ran one job at a time and each next one waited for backfill's pass. Fixed: the - partition's DefMemPerCPU is the leanest node's share; slurmdbd is not needed for this. Not tried yet: a GPU node (gres). + partition's DefMemPerCPU is the leanest node's share; slurmdbd is not needed for this. And InvalidAccount is + real: 23.11 defaults to priority/multifactor, whose main scheduler passes every job by without + accounting storage; PriorityType=priority/basic starts a job on submission. Also found: slurm.conf + wrote every node State=DRAIN, so each slurmctld start (on any change of the files) drained a lent node + until its agent resumed it, up to a minute; a lent node is now written without it. Not tried yet: a GPU node (gres). - Failover while lent, tried 2026-11-11 (v0.16.3-23): the first try killed the job (NODE_FAIL). A new leader had heard from no node yet and sent the controller a slurm.conf without them; slurmctld, restarted on it, dropped node05 and its job. Fixed (141a1f4, 0774e90): slurm.conf's nodes and the diff --git a/internal/control/slurmsvc/service.go b/internal/control/slurmsvc/service.go index a3f067c..839014e 100644 --- a/internal/control/slurmsvc/service.go +++ b/internal/control/slurmsvc/service.go @@ -417,13 +417,16 @@ func (s *Service) cluster(set config.SlurmSettings) (slurm.Cluster, []string) { c.ControllerAddr = s.controllerAddr(set) var waiting []string for _, node := range set.Nodes { - if sn, ok := s.reported(node); ok { - c.Nodes = append(c.Nodes, sn) - } else if sn, ok := s.known.Nodes[node]; ok { - c.Nodes = append(c.Nodes, sn) - } else { - waiting = append(waiting, node) + sn, ok := s.reported(node) + if !ok { + sn, ok = s.known.Nodes[node] } + if !ok { + waiting = append(waiting, node) + continue + } + sn.Lent = set.IsLent(node) + c.Nodes = append(c.Nodes, sn) } return c, waiting } diff --git a/internal/slurm/slurm.go b/internal/slurm/slurm.go index 741e159..b13f578 100644 --- a/internal/slurm/slurm.go +++ b/internal/slurm/slurm.go @@ -120,6 +120,9 @@ type Node struct { Addr string Hardware GPUs []GPU + // Lent is whether llmgw has lent it to Slurm: it is not written + // drained, or every start of slurmctld on these files drains it again. + Lent bool `json:"-"` } // Cluster is everything slurm.conf says. @@ -215,6 +218,10 @@ JobAcctGatherType=jobacct_gather/none SelectType=select/cons_tres SelectTypeParameters=CR_Core_Memory SchedulerType=sched/backfill +# Without accounting storage, 23.11's default (multifactor) has the main +# scheduler pass every job by as InvalidAccount, for backfill to start a +# pass later; basic is first come, first served, started on submission. +PriorityType=priority/basic GresTypes=gpu MpiDefault=none # No mail sent; without it every start says MailProg is invalid. @@ -255,7 +262,11 @@ SlurmdTimeout=300 } // Drained until llmgw lends it: a node new to the controller would // otherwise take jobs in the minute before its agent drains it. - fmt.Fprintf(&b, " State=DRAIN Reason=%q\n", ReasonNew) + if n.Lent { + b.WriteString("\n") + } else { + fmt.Fprintf(&b, " State=DRAIN Reason=%q\n", ReasonNew) + } } if len(names) > 0 { fmt.Fprintf(&b, "PartitionName=%s Nodes=%s Default=YES MaxTime=INFINITE State=UP DefMemPerCPU=%d\n", diff --git a/internal/slurm/slurm_test.go b/internal/slurm/slurm_test.go index 033e525..22caf93 100644 --- a/internal/slurm/slurm_test.go +++ b/internal/slurm/slurm_test.go @@ -151,3 +151,21 @@ func TestParseVersion(t *testing.T) { t.Error("no version was read as one") } } + +func TestALentNodeIsNotWrittenDrained(t *testing.T) { + c := testCluster() + c.Nodes[0].Lent = true + conf := c.SlurmConf() + line := "" + for _, l := range strings.Split(conf, "\n") { + if strings.HasPrefix(l, "NodeName="+c.Nodes[0].Name+" ") { + line = l + } + } + if line == "" || strings.Contains(line, "DRAIN") { + t.Fatalf("a lent node, written %q: every start of slurmctld would drain it again", line) + } + if !strings.Contains(conf, "PriorityType=priority/basic\n") { + t.Error("multifactor without accounting: the main scheduler passes every job by as InvalidAccount") + } +}