Slurm: a job that asks for no memory gets a share, not the whole node
CI / test (push) Successful in 10m34s
CI / test (push) Successful in 10m34s
Tried on the PoC: five one-second jobs on an 8-CPU node started 10 s apart. Each asked for no memory, so each was given the node's all, ran alone, and the next waited for backfill's pass. Not accounting, as thought. The partition's DefMemPerCPU is now the leanest node's memory per CPU. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
356f7e0ced
commit
5db920bf2c
3 files changed
+18
-6
No files matched your search
+12
-4
@@ -219,8 +219,7 @@ GresTypes=gpu
|
||||
MpiDefault=none
|
||||
# No mail sent; without it every start says MailProg is invalid.
|
||||
MailProg=/bin/true
|
||||
# Without accounting storage, 23.11's main scheduler marks a batch job
|
||||
# InvalidAccount; backfill starts it on its next pass, so it is kept short.
|
||||
# Backfill looks again every 10 s, not every 30.
|
||||
SchedulerParameters=bf_interval=10
|
||||
# A node that went down and comes back is taken again; one llmgw drained
|
||||
# stays drained until llmgw resumes it.
|
||||
@@ -230,6 +229,15 @@ SlurmdTimeout=300
|
||||
`)
|
||||
nodes := c.ready()
|
||||
names := make([]string, 0, len(nodes))
|
||||
// A job that asks for no memory gets this much per CPU: the share of
|
||||
// the leanest node. Without it it gets the whole node's, and the jobs
|
||||
// on a node run one at a time.
|
||||
var perCPU int64
|
||||
for _, n := range nodes {
|
||||
if share := int64(float64(n.RealMemory)*memoryKept) / int64(n.CPUs); perCPU == 0 || share < perCPU {
|
||||
perCPU = share
|
||||
}
|
||||
}
|
||||
for _, n := range nodes {
|
||||
names = append(names, n.Name)
|
||||
fmt.Fprintf(&b, "NodeName=%s NodeAddr=%s CPUs=%d", n.Name, n.Addr, n.CPUs)
|
||||
@@ -250,8 +258,8 @@ SlurmdTimeout=300
|
||||
fmt.Fprintf(&b, " State=DRAIN Reason=%q\n", ReasonNew)
|
||||
}
|
||||
if len(names) > 0 {
|
||||
fmt.Fprintf(&b, "PartitionName=%s Nodes=%s Default=YES MaxTime=INFINITE State=UP\n",
|
||||
c.Partition, strings.Join(names, ","))
|
||||
fmt.Fprintf(&b, "PartitionName=%s Nodes=%s Default=YES MaxTime=INFINITE State=UP DefMemPerCPU=%d\n",
|
||||
c.Partition, strings.Join(names, ","), perCPU)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
@@ -82,6 +82,8 @@ func TestSlurmConf(t *testing.T) {
|
||||
"NodeName=cuda01 NodeAddr=10.0.0.147 CPUs=16 RealMemory=62080 Gres=gpu:rtx3090:1 State=DRAIN Reason=\"llmgw: not lent yet\"\n",
|
||||
"NodeName=site2 NodeAddr=10.1.0.5 CPUs=32 Boards=1 SocketsPerBoard=2 CoresPerSocket=8 ThreadsPerCore=2 RealMemory=97000 Gres=gpu:t4:1,gpu:v100:2 State=DRAIN Reason=\"llmgw: not lent yet\"\n",
|
||||
"PartitionName=gpu Nodes=cuda01,site2 Default=YES",
|
||||
// site2's share, the leaner: 97000 MiB for 32 CPUs.
|
||||
" DefMemPerCPU=3031\n",
|
||||
} {
|
||||
if !strings.Contains(conf, want) {
|
||||
t.Errorf("slurm.conf has no %q:\n%s", want, conf)
|
||||
|
||||
Reference in new issue
Block a user