diff --git a/BACKLOG.md b/BACKLOG.md index 2c78d3a..f8d5efe 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -2027,7 +2027,7 @@ lent, partition`); the leader tells each named node's agent (root) what to be, t - Phases: P1 the `slurm` package (conf, key, parsers) and settings; P2 the agent side (proto, install, munge, slurmd/slurmctld, status); P3 the leader side (conf from reports, lend/reclaim); P4 `ctl slurm` (status, lend, reclaim) and a TUI screen; P5 `ctl train import` of a Slurm job's output into the mirror. - Later: a backup slurmctld on a second node, accounting (slurmdbd on the Patroni PG), users from LDAP/AD. + Later: a backup slurmctld on a second node, accounting (slurmdbd; it takes MySQL/MariaDB only, not the Patroni PG), users from LDAP/AD. - Tried on the PoC 2026-11-11 (dev build v0.16.3-17-g849760c-slurm on 140, 150, node05, node06; Ubuntu 24.04, Slurm 23.11.4 from the archive): `ctl slurm setup --controller llmgw-node06 --nodes llmgw-node05` installed munge, slurmctld and slurmd, node05 entered drained with its slurmd -C hardware (8 CPUs, @@ -2037,7 +2037,9 @@ lent, partition`); the leader tells each named node's agent (root) what to be, t from the daemon); the packaged unit is Type=notify (slurmd needs --systemd); slurmd's own "slurmstepd infinity" was counted as a job; without accounting storage 23.11's main scheduler marks batch jobs InvalidAccount and backfill starts them, one a pass (bf_interval=10: 6-10 s each). So many small jobs - want slurmdbd. Not tried yet: a GPU node (gres). + want slurmdbd. — Wrong cause, found 2026-11-11: a job asking no memory got the whole node's (ReqTRES + mem=all), so a node ran one job at a time and each next one waited for backfill's pass. Fixed: the + partition's DefMemPerCPU is the leanest node's share; slurmdbd is not needed for this. Not tried yet: a GPU node (gres). - Failover while lent, tried 2026-11-11 (v0.16.3-23): the first try killed the job (NODE_FAIL). A new leader had heard from no node yet and sent the controller a slurm.conf without them; slurmctld, restarted on it, dropped node05 and its job. Fixed (141a1f4, 0774e90): slurm.conf's nodes and the diff --git a/internal/slurm/slurm.go b/internal/slurm/slurm.go index 594a0d7..741e159 100644 --- a/internal/slurm/slurm.go +++ b/internal/slurm/slurm.go @@ -219,8 +219,7 @@ GresTypes=gpu MpiDefault=none # No mail sent; without it every start says MailProg is invalid. MailProg=/bin/true -# Without accounting storage, 23.11's main scheduler marks a batch job -# InvalidAccount; backfill starts it on its next pass, so it is kept short. +# Backfill looks again every 10 s, not every 30. SchedulerParameters=bf_interval=10 # A node that went down and comes back is taken again; one llmgw drained # stays drained until llmgw resumes it. @@ -230,6 +229,15 @@ SlurmdTimeout=300 `) nodes := c.ready() names := make([]string, 0, len(nodes)) + // A job that asks for no memory gets this much per CPU: the share of + // the leanest node. Without it it gets the whole node's, and the jobs + // on a node run one at a time. + var perCPU int64 + for _, n := range nodes { + if share := int64(float64(n.RealMemory)*memoryKept) / int64(n.CPUs); perCPU == 0 || share < perCPU { + perCPU = share + } + } for _, n := range nodes { names = append(names, n.Name) fmt.Fprintf(&b, "NodeName=%s NodeAddr=%s CPUs=%d", n.Name, n.Addr, n.CPUs) @@ -250,8 +258,8 @@ SlurmdTimeout=300 fmt.Fprintf(&b, " State=DRAIN Reason=%q\n", ReasonNew) } if len(names) > 0 { - fmt.Fprintf(&b, "PartitionName=%s Nodes=%s Default=YES MaxTime=INFINITE State=UP\n", - c.Partition, strings.Join(names, ",")) + fmt.Fprintf(&b, "PartitionName=%s Nodes=%s Default=YES MaxTime=INFINITE State=UP DefMemPerCPU=%d\n", + c.Partition, strings.Join(names, ","), perCPU) } return b.String() } diff --git a/internal/slurm/slurm_test.go b/internal/slurm/slurm_test.go index 4511b91..033e525 100644 --- a/internal/slurm/slurm_test.go +++ b/internal/slurm/slurm_test.go @@ -82,6 +82,8 @@ func TestSlurmConf(t *testing.T) { "NodeName=cuda01 NodeAddr=10.0.0.147 CPUs=16 RealMemory=62080 Gres=gpu:rtx3090:1 State=DRAIN Reason=\"llmgw: not lent yet\"\n", "NodeName=site2 NodeAddr=10.1.0.5 CPUs=32 Boards=1 SocketsPerBoard=2 CoresPerSocket=8 ThreadsPerCore=2 RealMemory=97000 Gres=gpu:t4:1,gpu:v100:2 State=DRAIN Reason=\"llmgw: not lent yet\"\n", "PartitionName=gpu Nodes=cuda01,site2 Default=YES", + // site2's share, the leaner: 97000 MiB for 32 CPUs. + " DefMemPerCPU=3031\n", } { if !strings.Contains(conf, want) { t.Errorf("slurm.conf has no %q:\n%s", want, conf)