Slurm: a job that asks for no memory gets a share, not the whole node
CI / test (push) Successful in 10m34s

Tried on the PoC: five one-second jobs on an 8-CPU node started 10 s
apart. Each asked for no memory, so each was given the node's all, ran
alone, and the next waited for backfill's pass. Not accounting, as
thought. The partition's DefMemPerCPU is now the leanest node's memory
per CPU.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-11 23:11:24 +01:00
1 parent 356f7e0ced
commit 5db920bf2c
3 files changed
+18 -6

No files matched your search

+12 -4
View File
@@ -219,8 +219,7 @@ GresTypes=gpu
MpiDefault=none
# No mail sent; without it every start says MailProg is invalid.
MailProg=/bin/true
# Without accounting storage, 23.11's main scheduler marks a batch job
# InvalidAccount; backfill starts it on its next pass, so it is kept short.
# Backfill looks again every 10 s, not every 30.
SchedulerParameters=bf_interval=10
# A node that went down and comes back is taken again; one llmgw drained
# stays drained until llmgw resumes it.
@@ -230,6 +229,15 @@ SlurmdTimeout=300
`)
nodes := c.ready()
names := make([]string, 0, len(nodes))
// A job that asks for no memory gets this much per CPU: the share of
// the leanest node. Without it it gets the whole node's, and the jobs
// on a node run one at a time.
var perCPU int64
for _, n := range nodes {
if share := int64(float64(n.RealMemory)*memoryKept) / int64(n.CPUs); perCPU == 0 || share < perCPU {
perCPU = share
}
}
for _, n := range nodes {
names = append(names, n.Name)
fmt.Fprintf(&b, "NodeName=%s NodeAddr=%s CPUs=%d", n.Name, n.Addr, n.CPUs)
@@ -250,8 +258,8 @@ SlurmdTimeout=300
fmt.Fprintf(&b, " State=DRAIN Reason=%q\n", ReasonNew)
}
if len(names) > 0 {
fmt.Fprintf(&b, "PartitionName=%s Nodes=%s Default=YES MaxTime=INFINITE State=UP\n",
c.Partition, strings.Join(names, ","))
fmt.Fprintf(&b, "PartitionName=%s Nodes=%s Default=YES MaxTime=INFINITE State=UP DefMemPerCPU=%d\n",
c.Partition, strings.Join(names, ","), perCPU)
}
return b.String()
}
+2
View File
@@ -82,6 +82,8 @@ func TestSlurmConf(t *testing.T) {
"NodeName=cuda01 NodeAddr=10.0.0.147 CPUs=16 RealMemory=62080 Gres=gpu:rtx3090:1 State=DRAIN Reason=\"llmgw: not lent yet\"\n",
"NodeName=site2 NodeAddr=10.1.0.5 CPUs=32 Boards=1 SocketsPerBoard=2 CoresPerSocket=8 ThreadsPerCore=2 RealMemory=97000 Gres=gpu:t4:1,gpu:v100:2 State=DRAIN Reason=\"llmgw: not lent yet\"\n",
"PartitionName=gpu Nodes=cuda01,site2 Default=YES",
// site2's share, the leaner: 97000 MiB for 32 CPUs.
" DefMemPerCPU=3031\n",
} {
if !strings.Contains(conf, want) {
t.Errorf("slurm.conf has no %q:\n%s", want, conf)