CI / test (push) Successful in 10m34s
Tried on the PoC: five one-second jobs on an 8-CPU node started 10 s apart. Each asked for no memory, so each was given the node's all, ran alone, and the next waited for backfill's pass. Not accounting, as thought. The partition's DefMemPerCPU is now the leanest node's memory per CPU. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
353 lines
11 KiB
Go
353 lines
11 KiB
Go
// Package slurm is what llmgw knows of Slurm (BACKLOG P): the files a
|
|
// controller runs on, written from what llmgw knows of its nodes, the munge
|
|
// key every machine shares, and reading what slurmd and scontrol say.
|
|
//
|
|
// Nothing here runs anything. The agent installs and starts the daemons; the
|
|
// leader decides what each node is and writes the files with this package.
|
|
package slurm
|
|
|
|
import (
|
|
"crypto/hkdf"
|
|
"crypto/sha256"
|
|
"fmt"
|
|
"regexp"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
// Paths are where the files are on every distribution llmgw installs on:
|
|
// Debian and Ubuntu since Slurm 20.11, and EPEL.
|
|
const (
|
|
ConfDir = "/etc/slurm"
|
|
MungeKeyPath = "/etc/munge/munge.key"
|
|
StateDir = "/var/spool/slurmctld"
|
|
SpoolDir = "/var/spool/slurmd"
|
|
LogDir = "/var/log/slurm"
|
|
// CtldPort is slurmctld's, which configless slurmd fetch their files
|
|
// from.
|
|
CtldPort = 6817
|
|
// SlurmdPort is slurmd's, which the controller and srun reach.
|
|
SlurmdPort = 6818
|
|
// SrunPortsFrom and SrunPortsTo are where srun listens for the steps it
|
|
// starts: a range of its own, so a firewall can let it in (Slurm's own
|
|
// example range). Without one srun takes any ephemeral port.
|
|
SrunPortsFrom = 60001
|
|
SrunPortsTo = 63000
|
|
)
|
|
|
|
// ReasonNew is why a node new to slurm.conf starts drained. Every reason
|
|
// llmgw drains with starts with "llmgw", and only those are resumed by it.
|
|
const ReasonNew = "llmgw: not lent yet"
|
|
|
|
// MinVersion is the oldest Slurm llmgw runs: the first to put jobs in
|
|
// cgroup v2, which every distribution llmgw supports now mounts alone.
|
|
var MinVersion = [2]int{22, 5}
|
|
|
|
// MungeKeyLen is the length munge's own tool writes.
|
|
const MungeKeyLen = 1024
|
|
|
|
// MungeKey is the key every machine of the Slurm cluster shares, derived
|
|
// from a cluster secret, so a control plane that takes over writes the same
|
|
// one and nobody copies it by hand.
|
|
func MungeKey(secret string) ([]byte, error) {
|
|
if len(secret) < 32 {
|
|
return nil, fmt.Errorf("the secret a munge key is made from is too short")
|
|
}
|
|
return hkdf.Key(sha256.New, []byte(secret), nil, "llmgw slurm munge key v1", MungeKeyLen)
|
|
}
|
|
|
|
// Hardware is a node's line from `slurmd -C`: how slurmd counts it, which is
|
|
// what slurm.conf must say or the controller refuses the node.
|
|
type Hardware struct {
|
|
CPUs int
|
|
Boards int
|
|
SocketsPerBoard int
|
|
CoresPerSocket int
|
|
ThreadsPerCore int
|
|
// RealMemory is in MiB.
|
|
RealMemory int64
|
|
}
|
|
|
|
// ParseHardware reads `slurmd -C`.
|
|
func ParseHardware(out string) (Hardware, error) {
|
|
var h Hardware
|
|
for _, line := range strings.Split(out, "\n") {
|
|
if !strings.HasPrefix(strings.TrimSpace(line), "NodeName=") {
|
|
continue
|
|
}
|
|
for _, f := range strings.Fields(line) {
|
|
k, v, _ := strings.Cut(f, "=")
|
|
n, err := strconv.ParseInt(v, 10, 64)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
switch k {
|
|
case "CPUs":
|
|
h.CPUs = int(n)
|
|
case "Boards":
|
|
h.Boards = int(n)
|
|
case "SocketsPerBoard":
|
|
h.SocketsPerBoard = int(n)
|
|
case "CoresPerSocket":
|
|
h.CoresPerSocket = int(n)
|
|
case "ThreadsPerCore":
|
|
h.ThreadsPerCore = int(n)
|
|
case "RealMemory":
|
|
h.RealMemory = n
|
|
}
|
|
}
|
|
break
|
|
}
|
|
if h.CPUs <= 0 || h.RealMemory <= 0 {
|
|
return Hardware{}, fmt.Errorf("slurmd -C said no CPUs or memory: %q", strings.TrimSpace(out))
|
|
}
|
|
return h, nil
|
|
}
|
|
|
|
// GPU is one of a node's cards.
|
|
type GPU struct {
|
|
// Minor is N of /dev/nvidiaN.
|
|
Minor int
|
|
// Name is the driver's, as nvidia-smi gives it.
|
|
Name string
|
|
}
|
|
|
|
// Node is one machine slurmd runs on.
|
|
type Node struct {
|
|
Name string
|
|
// Addr is where slurmctld reaches it: its llmgw management address.
|
|
Addr string
|
|
Hardware
|
|
GPUs []GPU
|
|
}
|
|
|
|
// Cluster is everything slurm.conf says.
|
|
type Cluster struct {
|
|
Name string
|
|
Controller string
|
|
ControllerAddr string
|
|
Partition string
|
|
// Nodes without hardware (not reported yet) are left out of the files
|
|
// until they are.
|
|
Nodes []Node
|
|
}
|
|
|
|
// gpuModel is a word that names a card's model: A100, H100, V100, T4, L40S.
|
|
var gpuModel = regexp.MustCompile(`^[a-z]{1,2}\d{1,4}[a-z]?$`)
|
|
|
|
// GPUType is a card's name as a Slurm GRES type: "NVIDIA A100-SXM4-80GB" is
|
|
// a100, "NVIDIA GeForce RTX 3090" rtx3090, "Tesla T4" t4. A name it cannot
|
|
// read is "gpu".
|
|
func GPUType(name string) string {
|
|
words := strings.Fields(strings.ToLower(strings.NewReplacer("-", " ", "_", " ").Replace(name)))
|
|
for i, w := range words {
|
|
if (w == "rtx" || w == "gtx") && i+1 < len(words) {
|
|
return w + words[i+1]
|
|
}
|
|
if gpuModel.MatchString(w) {
|
|
return w
|
|
}
|
|
}
|
|
return "gpu"
|
|
}
|
|
|
|
// gpuGroups are a node's cards by type, the types sorted.
|
|
func gpuGroups(gs []GPU) ([]string, map[string][]int) {
|
|
by := map[string][]int{}
|
|
for _, g := range gs {
|
|
t := GPUType(g.Name)
|
|
by[t] = append(by[t], g.Minor)
|
|
}
|
|
types := make([]string, 0, len(by))
|
|
for t, ms := range by {
|
|
slices.Sort(ms)
|
|
types = append(types, t)
|
|
}
|
|
slices.Sort(types)
|
|
return types, by
|
|
}
|
|
|
|
// memoryKept is what is left out of a node's RealMemory: slurmd -C reads
|
|
// what the kernel says now, which is a little less after a reboot, and a
|
|
// node whose memory is below what slurm.conf says is refused.
|
|
const memoryKept = 0.97
|
|
|
|
func (c Cluster) ready() []Node {
|
|
var out []Node
|
|
for _, n := range c.Nodes {
|
|
if n.CPUs > 0 && n.RealMemory > 0 {
|
|
out = append(out, n)
|
|
}
|
|
}
|
|
slices.SortFunc(out, func(a, b Node) int { return strings.Compare(a.Name, b.Name) })
|
|
return out
|
|
}
|
|
|
|
const header = "# Written by llmgw (BACKLOG P), on the node that runs slurmctld; the nodes read it\n" +
|
|
"# from there. Change it through the cluster's settings (slurm.*), not here.\n"
|
|
|
|
// SlurmConf is slurm.conf.
|
|
func (c Cluster) SlurmConf() string {
|
|
var b strings.Builder
|
|
b.WriteString(header)
|
|
fmt.Fprintf(&b, "ClusterName=%s\n", c.Name)
|
|
fmt.Fprintf(&b, "SlurmctldHost=%s(%s)\n", c.Controller, c.ControllerAddr)
|
|
b.WriteString(`SlurmctldParameters=enable_configless
|
|
SlurmctldPort=6817
|
|
SlurmdPort=6818
|
|
SrunPortRange=60001-63000
|
|
AuthType=auth/munge
|
|
CredType=cred/munge
|
|
SlurmUser=slurm
|
|
SlurmdUser=root
|
|
StateSaveLocation=` + StateDir + `
|
|
SlurmdSpoolDir=` + SpoolDir + `
|
|
SlurmctldLogFile=` + LogDir + `/slurmctld.log
|
|
SlurmdLogFile=` + LogDir + `/slurmd.log
|
|
SlurmctldPidFile=/run/slurmctld.pid
|
|
SlurmdPidFile=/run/slurmd.pid
|
|
ProctrackType=proctrack/cgroup
|
|
TaskPlugin=task/cgroup,task/affinity
|
|
# Nothing gathered for accounting until there is somewhere to keep it
|
|
# (slurmdbd, later); gathered and not kept is an error in every log.
|
|
JobAcctGatherType=jobacct_gather/none
|
|
SelectType=select/cons_tres
|
|
SelectTypeParameters=CR_Core_Memory
|
|
SchedulerType=sched/backfill
|
|
GresTypes=gpu
|
|
MpiDefault=none
|
|
# No mail sent; without it every start says MailProg is invalid.
|
|
MailProg=/bin/true
|
|
# Backfill looks again every 10 s, not every 30.
|
|
SchedulerParameters=bf_interval=10
|
|
# A node that went down and comes back is taken again; one llmgw drained
|
|
# stays drained until llmgw resumes it.
|
|
ReturnToService=2
|
|
SlurmctldTimeout=120
|
|
SlurmdTimeout=300
|
|
`)
|
|
nodes := c.ready()
|
|
names := make([]string, 0, len(nodes))
|
|
// A job that asks for no memory gets this much per CPU: the share of
|
|
// the leanest node. Without it it gets the whole node's, and the jobs
|
|
// on a node run one at a time.
|
|
var perCPU int64
|
|
for _, n := range nodes {
|
|
if share := int64(float64(n.RealMemory)*memoryKept) / int64(n.CPUs); perCPU == 0 || share < perCPU {
|
|
perCPU = share
|
|
}
|
|
}
|
|
for _, n := range nodes {
|
|
names = append(names, n.Name)
|
|
fmt.Fprintf(&b, "NodeName=%s NodeAddr=%s CPUs=%d", n.Name, n.Addr, n.CPUs)
|
|
if n.Boards > 0 && n.SocketsPerBoard > 0 && n.CoresPerSocket > 0 && n.ThreadsPerCore > 0 {
|
|
fmt.Fprintf(&b, " Boards=%d SocketsPerBoard=%d CoresPerSocket=%d ThreadsPerCore=%d",
|
|
n.Boards, n.SocketsPerBoard, n.CoresPerSocket, n.ThreadsPerCore)
|
|
}
|
|
fmt.Fprintf(&b, " RealMemory=%d", int64(float64(n.RealMemory)*memoryKept))
|
|
if types, by := gpuGroups(n.GPUs); len(types) > 0 {
|
|
gres := make([]string, 0, len(types))
|
|
for _, t := range types {
|
|
gres = append(gres, fmt.Sprintf("gpu:%s:%d", t, len(by[t])))
|
|
}
|
|
fmt.Fprintf(&b, " Gres=%s", strings.Join(gres, ","))
|
|
}
|
|
// Drained until llmgw lends it: a node new to the controller would
|
|
// otherwise take jobs in the minute before its agent drains it.
|
|
fmt.Fprintf(&b, " State=DRAIN Reason=%q\n", ReasonNew)
|
|
}
|
|
if len(names) > 0 {
|
|
fmt.Fprintf(&b, "PartitionName=%s Nodes=%s Default=YES MaxTime=INFINITE State=UP DefMemPerCPU=%d\n",
|
|
c.Partition, strings.Join(names, ","), perCPU)
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
// GresConf is gres.conf: each node's cards by device file, so a job is given
|
|
// the cards it asked for and kept off the others.
|
|
func (c Cluster) GresConf() string {
|
|
var b strings.Builder
|
|
b.WriteString(header)
|
|
for _, n := range c.ready() {
|
|
types, by := gpuGroups(n.GPUs)
|
|
for _, t := range types {
|
|
files := make([]string, 0, len(by[t]))
|
|
for _, m := range by[t] {
|
|
files = append(files, fmt.Sprintf("/dev/nvidia%d", m))
|
|
}
|
|
fmt.Fprintf(&b, "NodeName=%s Name=gpu Type=%s File=%s\n", n.Name, t, strings.Join(files, ","))
|
|
}
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
// CgroupConf is cgroup.conf: a job kept to its cores, memory and cards.
|
|
func (c Cluster) CgroupConf() string {
|
|
return header + `CgroupPlugin=autodetect
|
|
ConstrainCores=yes
|
|
ConstrainDevices=yes
|
|
ConstrainRAMSpace=yes
|
|
`
|
|
}
|
|
|
|
// Files are the controller's files, by name in ConfDir.
|
|
func (c Cluster) Files() map[string]string {
|
|
return map[string]string{"slurm.conf": c.SlurmConf(), "gres.conf": c.GresConf(), "cgroup.conf": c.CgroupConf()}
|
|
}
|
|
|
|
// NodeState is what `scontrol show node -o NAME` says of a node.
|
|
type NodeState struct {
|
|
// State is Slurm's, with its flags: IDLE, ALLOCATED, MIXED+DRAIN,
|
|
// DOWN+NOT_RESPONDING.
|
|
State string
|
|
Reason string
|
|
}
|
|
|
|
// Drained is whether the controller gives the node nothing new.
|
|
func (s NodeState) Drained() bool {
|
|
return strings.Contains(s.State, "DRAIN") || strings.Contains(s.State, "DOWN")
|
|
}
|
|
|
|
// ParseNodeState reads `scontrol show node -o NAME`, one line of key=value.
|
|
func ParseNodeState(out string) (NodeState, error) {
|
|
line := strings.TrimSpace(out)
|
|
if !strings.HasPrefix(line, "NodeName=") {
|
|
return NodeState{}, fmt.Errorf("scontrol said %q", line)
|
|
}
|
|
var s NodeState
|
|
if i := strings.Index(line, " State="); i >= 0 {
|
|
s.State, _, _ = strings.Cut(line[i+len(" State="):], " ")
|
|
}
|
|
// Reason is last but for its [user@time] and may have spaces.
|
|
if i := strings.Index(line, " Reason="); i >= 0 {
|
|
r := line[i+len(" Reason="):]
|
|
if j := strings.LastIndex(r, " ["); j >= 0 {
|
|
r = r[:j]
|
|
}
|
|
s.Reason = strings.TrimSpace(r)
|
|
}
|
|
return s, nil
|
|
}
|
|
|
|
// ParseVersion reads "slurm 23.11.4" or "slurm-wlm 21.08.5" as major, minor.
|
|
func ParseVersion(out string) ([2]int, error) {
|
|
for _, f := range strings.Fields(out) {
|
|
parts := strings.Split(f, ".")
|
|
if len(parts) < 2 {
|
|
continue
|
|
}
|
|
major, err1 := strconv.Atoi(parts[0])
|
|
minor, err2 := strconv.Atoi(parts[1])
|
|
if err1 == nil && err2 == nil {
|
|
return [2]int{major, minor}, nil
|
|
}
|
|
}
|
|
return [2]int{}, fmt.Errorf("no version in %q", strings.TrimSpace(out))
|
|
}
|
|
|
|
// TooOld is whether v is older than MinVersion.
|
|
func TooOld(v [2]int) bool {
|
|
return v[0] < MinVersion[0] || (v[0] == MinVersion[0] && v[1] < MinVersion[1])
|
|
}
|