From 5db920bf2c7edfbfb565cdef74af6367a0a94c63 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Wed, 11 Nov 2026 23:11:24 +0100 Subject: [PATCH] Slurm: a job that asks for no memory gets a share, not the whole node Tried on the PoC: five one-second jobs on an 8-CPU node started 10 s apart. Each asked for no memory, so each was given the node's all, ran alone, and the next waited for backfill's pass. Not accounting, as thought. The partition's DefMemPerCPU is now the leanest node's memory per CPU. Co-Authored-By: Claude Opus 5.5 --- BACKLOG.md | 6 ++++-- internal/slurm/slurm.go | 16 ++++++++++++---- internal/slurm/slurm_test.go | 2 ++ 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/BACKLOG.md b/BACKLOG.md index 2c78d3a..f8d5efe 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -2027,7 +2027,7 @@ lent, partition`); the leader tells each named node's agent (root) what to be, t - Phases: P1 the `slurm` package (conf, key, parsers) and settings; P2 the agent side (proto, install, munge, slurmd/slurmctld, status); P3 the leader side (conf from reports, lend/reclaim); P4 `ctl slurm` (status, lend, reclaim) and a TUI screen; P5 `ctl train import` of a Slurm job's output into the mirror. - Later: a backup slurmctld on a second node, accounting (slurmdbd on the Patroni PG), users from LDAP/AD. + Later: a backup slurmctld on a second node, accounting (slurmdbd; it takes MySQL/MariaDB only, not the Patroni PG), users from LDAP/AD. - Tried on the PoC 2026-11-11 (dev build v0.16.3-17-g849760c-slurm on 140, 150, node05, node06; Ubuntu 24.04, Slurm 23.11.4 from the archive): `ctl slurm setup --controller llmgw-node06 --nodes llmgw-node05` installed munge, slurmctld and slurmd, node05 entered drained with its slurmd -C hardware (8 CPUs, @@ -2037,7 +2037,9 @@ lent, partition`); the leader tells each named node's agent (root) what to be, t from the daemon); the packaged unit is Type=notify (slurmd needs --systemd); slurmd's own "slurmstepd infinity" was counted as a job; without accounting storage 23.11's main scheduler marks batch jobs InvalidAccount and backfill starts them, one a pass (bf_interval=10: 6-10 s each). So many small jobs - want slurmdbd. Not tried yet: a GPU node (gres). + want slurmdbd. — Wrong cause, found 2026-11-11: a job asking no memory got the whole node's (ReqTRES + mem=all), so a node ran one job at a time and each next one waited for backfill's pass. Fixed: the + partition's DefMemPerCPU is the leanest node's share; slurmdbd is not needed for this. Not tried yet: a GPU node (gres). - Failover while lent, tried 2026-11-11 (v0.16.3-23): the first try killed the job (NODE_FAIL). A new leader had heard from no node yet and sent the controller a slurm.conf without them; slurmctld, restarted on it, dropped node05 and its job. Fixed (141a1f4, 0774e90): slurm.conf's nodes and the diff --git a/internal/slurm/slurm.go b/internal/slurm/slurm.go index 594a0d7..741e159 100644 --- a/internal/slurm/slurm.go +++ b/internal/slurm/slurm.go @@ -219,8 +219,7 @@ GresTypes=gpu MpiDefault=none # No mail sent; without it every start says MailProg is invalid. MailProg=/bin/true -# Without accounting storage, 23.11's main scheduler marks a batch job -# InvalidAccount; backfill starts it on its next pass, so it is kept short. +# Backfill looks again every 10 s, not every 30. SchedulerParameters=bf_interval=10 # A node that went down and comes back is taken again; one llmgw drained # stays drained until llmgw resumes it. @@ -230,6 +229,15 @@ SlurmdTimeout=300 `) nodes := c.ready() names := make([]string, 0, len(nodes)) + // A job that asks for no memory gets this much per CPU: the share of + // the leanest node. Without it it gets the whole node's, and the jobs + // on a node run one at a time. + var perCPU int64 + for _, n := range nodes { + if share := int64(float64(n.RealMemory)*memoryKept) / int64(n.CPUs); perCPU == 0 || share < perCPU { + perCPU = share + } + } for _, n := range nodes { names = append(names, n.Name) fmt.Fprintf(&b, "NodeName=%s NodeAddr=%s CPUs=%d", n.Name, n.Addr, n.CPUs) @@ -250,8 +258,8 @@ SlurmdTimeout=300 fmt.Fprintf(&b, " State=DRAIN Reason=%q\n", ReasonNew) } if len(names) > 0 { - fmt.Fprintf(&b, "PartitionName=%s Nodes=%s Default=YES MaxTime=INFINITE State=UP\n", - c.Partition, strings.Join(names, ",")) + fmt.Fprintf(&b, "PartitionName=%s Nodes=%s Default=YES MaxTime=INFINITE State=UP DefMemPerCPU=%d\n", + c.Partition, strings.Join(names, ","), perCPU) } return b.String() } diff --git a/internal/slurm/slurm_test.go b/internal/slurm/slurm_test.go index 4511b91..033e525 100644 --- a/internal/slurm/slurm_test.go +++ b/internal/slurm/slurm_test.go @@ -82,6 +82,8 @@ func TestSlurmConf(t *testing.T) { "NodeName=cuda01 NodeAddr=10.0.0.147 CPUs=16 RealMemory=62080 Gres=gpu:rtx3090:1 State=DRAIN Reason=\"llmgw: not lent yet\"\n", "NodeName=site2 NodeAddr=10.1.0.5 CPUs=32 Boards=1 SocketsPerBoard=2 CoresPerSocket=8 ThreadsPerCore=2 RealMemory=97000 Gres=gpu:t4:1,gpu:v100:2 State=DRAIN Reason=\"llmgw: not lent yet\"\n", "PartitionName=gpu Nodes=cuda01,site2 Default=YES", + // site2's share, the leaner: 97000 MiB for 32 CPUs. + " DefMemPerCPU=3031\n", } { if !strings.Contains(conf, want) { t.Errorf("slurm.conf has no %q:\n%s", want, conf)