CI / test (push) Successful in 9m40s
slurm.conf sets SrunPortRange=60001-63000, so a site's firewall can let srun's step connections in. A node given a part in Slurm adds 6817-6818 and that range to llmgw's nftables table, open to the cluster's networks and closed to everyone else, as the engines' ports are. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
345 lines
11 KiB
Go
345 lines
11 KiB
Go
// Package slurm is what llmgw knows of Slurm (BACKLOG P): the files a
|
|
// controller runs on, written from what llmgw knows of its nodes, the munge
|
|
// key every machine shares, and reading what slurmd and scontrol say.
|
|
//
|
|
// Nothing here runs anything. The agent installs and starts the daemons; the
|
|
// leader decides what each node is and writes the files with this package.
|
|
package slurm
|
|
|
|
import (
|
|
"crypto/hkdf"
|
|
"crypto/sha256"
|
|
"fmt"
|
|
"regexp"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
// Paths are where the files are on every distribution llmgw installs on:
|
|
// Debian and Ubuntu since Slurm 20.11, and EPEL.
|
|
const (
|
|
ConfDir = "/etc/slurm"
|
|
MungeKeyPath = "/etc/munge/munge.key"
|
|
StateDir = "/var/spool/slurmctld"
|
|
SpoolDir = "/var/spool/slurmd"
|
|
LogDir = "/var/log/slurm"
|
|
// CtldPort is slurmctld's, which configless slurmd fetch their files
|
|
// from.
|
|
CtldPort = 6817
|
|
// SlurmdPort is slurmd's, which the controller and srun reach.
|
|
SlurmdPort = 6818
|
|
// SrunPortsFrom and SrunPortsTo are where srun listens for the steps it
|
|
// starts: a range of its own, so a firewall can let it in (Slurm's own
|
|
// example range). Without one srun takes any ephemeral port.
|
|
SrunPortsFrom = 60001
|
|
SrunPortsTo = 63000
|
|
)
|
|
|
|
// ReasonNew is why a node new to slurm.conf starts drained. Every reason
|
|
// llmgw drains with starts with "llmgw", and only those are resumed by it.
|
|
const ReasonNew = "llmgw: not lent yet"
|
|
|
|
// MinVersion is the oldest Slurm llmgw runs: the first to put jobs in
|
|
// cgroup v2, which every distribution llmgw supports now mounts alone.
|
|
var MinVersion = [2]int{22, 5}
|
|
|
|
// MungeKeyLen is the length munge's own tool writes.
|
|
const MungeKeyLen = 1024
|
|
|
|
// MungeKey is the key every machine of the Slurm cluster shares, derived
|
|
// from a cluster secret, so a control plane that takes over writes the same
|
|
// one and nobody copies it by hand.
|
|
func MungeKey(secret string) ([]byte, error) {
|
|
if len(secret) < 32 {
|
|
return nil, fmt.Errorf("the secret a munge key is made from is too short")
|
|
}
|
|
return hkdf.Key(sha256.New, []byte(secret), nil, "llmgw slurm munge key v1", MungeKeyLen)
|
|
}
|
|
|
|
// Hardware is a node's line from `slurmd -C`: how slurmd counts it, which is
|
|
// what slurm.conf must say or the controller refuses the node.
|
|
type Hardware struct {
|
|
CPUs int
|
|
Boards int
|
|
SocketsPerBoard int
|
|
CoresPerSocket int
|
|
ThreadsPerCore int
|
|
// RealMemory is in MiB.
|
|
RealMemory int64
|
|
}
|
|
|
|
// ParseHardware reads `slurmd -C`.
|
|
func ParseHardware(out string) (Hardware, error) {
|
|
var h Hardware
|
|
for _, line := range strings.Split(out, "\n") {
|
|
if !strings.HasPrefix(strings.TrimSpace(line), "NodeName=") {
|
|
continue
|
|
}
|
|
for _, f := range strings.Fields(line) {
|
|
k, v, _ := strings.Cut(f, "=")
|
|
n, err := strconv.ParseInt(v, 10, 64)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
switch k {
|
|
case "CPUs":
|
|
h.CPUs = int(n)
|
|
case "Boards":
|
|
h.Boards = int(n)
|
|
case "SocketsPerBoard":
|
|
h.SocketsPerBoard = int(n)
|
|
case "CoresPerSocket":
|
|
h.CoresPerSocket = int(n)
|
|
case "ThreadsPerCore":
|
|
h.ThreadsPerCore = int(n)
|
|
case "RealMemory":
|
|
h.RealMemory = n
|
|
}
|
|
}
|
|
break
|
|
}
|
|
if h.CPUs <= 0 || h.RealMemory <= 0 {
|
|
return Hardware{}, fmt.Errorf("slurmd -C said no CPUs or memory: %q", strings.TrimSpace(out))
|
|
}
|
|
return h, nil
|
|
}
|
|
|
|
// GPU is one of a node's cards.
|
|
type GPU struct {
|
|
// Minor is N of /dev/nvidiaN.
|
|
Minor int
|
|
// Name is the driver's, as nvidia-smi gives it.
|
|
Name string
|
|
}
|
|
|
|
// Node is one machine slurmd runs on.
|
|
type Node struct {
|
|
Name string
|
|
// Addr is where slurmctld reaches it: its llmgw management address.
|
|
Addr string
|
|
Hardware
|
|
GPUs []GPU
|
|
}
|
|
|
|
// Cluster is everything slurm.conf says.
|
|
type Cluster struct {
|
|
Name string
|
|
Controller string
|
|
ControllerAddr string
|
|
Partition string
|
|
// Nodes without hardware (not reported yet) are left out of the files
|
|
// until they are.
|
|
Nodes []Node
|
|
}
|
|
|
|
// gpuModel is a word that names a card's model: A100, H100, V100, T4, L40S.
|
|
var gpuModel = regexp.MustCompile(`^[a-z]{1,2}\d{1,4}[a-z]?$`)
|
|
|
|
// GPUType is a card's name as a Slurm GRES type: "NVIDIA A100-SXM4-80GB" is
|
|
// a100, "NVIDIA GeForce RTX 3090" rtx3090, "Tesla T4" t4. A name it cannot
|
|
// read is "gpu".
|
|
func GPUType(name string) string {
|
|
words := strings.Fields(strings.ToLower(strings.NewReplacer("-", " ", "_", " ").Replace(name)))
|
|
for i, w := range words {
|
|
if (w == "rtx" || w == "gtx") && i+1 < len(words) {
|
|
return w + words[i+1]
|
|
}
|
|
if gpuModel.MatchString(w) {
|
|
return w
|
|
}
|
|
}
|
|
return "gpu"
|
|
}
|
|
|
|
// gpuGroups are a node's cards by type, the types sorted.
|
|
func gpuGroups(gs []GPU) ([]string, map[string][]int) {
|
|
by := map[string][]int{}
|
|
for _, g := range gs {
|
|
t := GPUType(g.Name)
|
|
by[t] = append(by[t], g.Minor)
|
|
}
|
|
types := make([]string, 0, len(by))
|
|
for t, ms := range by {
|
|
slices.Sort(ms)
|
|
types = append(types, t)
|
|
}
|
|
slices.Sort(types)
|
|
return types, by
|
|
}
|
|
|
|
// memoryKept is what is left out of a node's RealMemory: slurmd -C reads
|
|
// what the kernel says now, which is a little less after a reboot, and a
|
|
// node whose memory is below what slurm.conf says is refused.
|
|
const memoryKept = 0.97
|
|
|
|
func (c Cluster) ready() []Node {
|
|
var out []Node
|
|
for _, n := range c.Nodes {
|
|
if n.CPUs > 0 && n.RealMemory > 0 {
|
|
out = append(out, n)
|
|
}
|
|
}
|
|
slices.SortFunc(out, func(a, b Node) int { return strings.Compare(a.Name, b.Name) })
|
|
return out
|
|
}
|
|
|
|
const header = "# Written by llmgw (BACKLOG P), on the node that runs slurmctld; the nodes read it\n" +
|
|
"# from there. Change it through the cluster's settings (slurm.*), not here.\n"
|
|
|
|
// SlurmConf is slurm.conf.
|
|
func (c Cluster) SlurmConf() string {
|
|
var b strings.Builder
|
|
b.WriteString(header)
|
|
fmt.Fprintf(&b, "ClusterName=%s\n", c.Name)
|
|
fmt.Fprintf(&b, "SlurmctldHost=%s(%s)\n", c.Controller, c.ControllerAddr)
|
|
b.WriteString(`SlurmctldParameters=enable_configless
|
|
SlurmctldPort=6817
|
|
SlurmdPort=6818
|
|
SrunPortRange=60001-63000
|
|
AuthType=auth/munge
|
|
CredType=cred/munge
|
|
SlurmUser=slurm
|
|
SlurmdUser=root
|
|
StateSaveLocation=` + StateDir + `
|
|
SlurmdSpoolDir=` + SpoolDir + `
|
|
SlurmctldLogFile=` + LogDir + `/slurmctld.log
|
|
SlurmdLogFile=` + LogDir + `/slurmd.log
|
|
SlurmctldPidFile=/run/slurmctld.pid
|
|
SlurmdPidFile=/run/slurmd.pid
|
|
ProctrackType=proctrack/cgroup
|
|
TaskPlugin=task/cgroup,task/affinity
|
|
# Nothing gathered for accounting until there is somewhere to keep it
|
|
# (slurmdbd, later); gathered and not kept is an error in every log.
|
|
JobAcctGatherType=jobacct_gather/none
|
|
SelectType=select/cons_tres
|
|
SelectTypeParameters=CR_Core_Memory
|
|
SchedulerType=sched/backfill
|
|
GresTypes=gpu
|
|
MpiDefault=none
|
|
# No mail sent; without it every start says MailProg is invalid.
|
|
MailProg=/bin/true
|
|
# Without accounting storage, 23.11's main scheduler marks a batch job
|
|
# InvalidAccount; backfill starts it on its next pass, so it is kept short.
|
|
SchedulerParameters=bf_interval=10
|
|
# A node that went down and comes back is taken again; one llmgw drained
|
|
# stays drained until llmgw resumes it.
|
|
ReturnToService=2
|
|
SlurmctldTimeout=120
|
|
SlurmdTimeout=300
|
|
`)
|
|
nodes := c.ready()
|
|
names := make([]string, 0, len(nodes))
|
|
for _, n := range nodes {
|
|
names = append(names, n.Name)
|
|
fmt.Fprintf(&b, "NodeName=%s NodeAddr=%s CPUs=%d", n.Name, n.Addr, n.CPUs)
|
|
if n.Boards > 0 && n.SocketsPerBoard > 0 && n.CoresPerSocket > 0 && n.ThreadsPerCore > 0 {
|
|
fmt.Fprintf(&b, " Boards=%d SocketsPerBoard=%d CoresPerSocket=%d ThreadsPerCore=%d",
|
|
n.Boards, n.SocketsPerBoard, n.CoresPerSocket, n.ThreadsPerCore)
|
|
}
|
|
fmt.Fprintf(&b, " RealMemory=%d", int64(float64(n.RealMemory)*memoryKept))
|
|
if types, by := gpuGroups(n.GPUs); len(types) > 0 {
|
|
gres := make([]string, 0, len(types))
|
|
for _, t := range types {
|
|
gres = append(gres, fmt.Sprintf("gpu:%s:%d", t, len(by[t])))
|
|
}
|
|
fmt.Fprintf(&b, " Gres=%s", strings.Join(gres, ","))
|
|
}
|
|
// Drained until llmgw lends it: a node new to the controller would
|
|
// otherwise take jobs in the minute before its agent drains it.
|
|
fmt.Fprintf(&b, " State=DRAIN Reason=%q\n", ReasonNew)
|
|
}
|
|
if len(names) > 0 {
|
|
fmt.Fprintf(&b, "PartitionName=%s Nodes=%s Default=YES MaxTime=INFINITE State=UP\n",
|
|
c.Partition, strings.Join(names, ","))
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
// GresConf is gres.conf: each node's cards by device file, so a job is given
|
|
// the cards it asked for and kept off the others.
|
|
func (c Cluster) GresConf() string {
|
|
var b strings.Builder
|
|
b.WriteString(header)
|
|
for _, n := range c.ready() {
|
|
types, by := gpuGroups(n.GPUs)
|
|
for _, t := range types {
|
|
files := make([]string, 0, len(by[t]))
|
|
for _, m := range by[t] {
|
|
files = append(files, fmt.Sprintf("/dev/nvidia%d", m))
|
|
}
|
|
fmt.Fprintf(&b, "NodeName=%s Name=gpu Type=%s File=%s\n", n.Name, t, strings.Join(files, ","))
|
|
}
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
// CgroupConf is cgroup.conf: a job kept to its cores, memory and cards.
|
|
func (c Cluster) CgroupConf() string {
|
|
return header + `CgroupPlugin=autodetect
|
|
ConstrainCores=yes
|
|
ConstrainDevices=yes
|
|
ConstrainRAMSpace=yes
|
|
`
|
|
}
|
|
|
|
// Files are the controller's files, by name in ConfDir.
|
|
func (c Cluster) Files() map[string]string {
|
|
return map[string]string{"slurm.conf": c.SlurmConf(), "gres.conf": c.GresConf(), "cgroup.conf": c.CgroupConf()}
|
|
}
|
|
|
|
// NodeState is what `scontrol show node -o NAME` says of a node.
|
|
type NodeState struct {
|
|
// State is Slurm's, with its flags: IDLE, ALLOCATED, MIXED+DRAIN,
|
|
// DOWN+NOT_RESPONDING.
|
|
State string
|
|
Reason string
|
|
}
|
|
|
|
// Drained is whether the controller gives the node nothing new.
|
|
func (s NodeState) Drained() bool {
|
|
return strings.Contains(s.State, "DRAIN") || strings.Contains(s.State, "DOWN")
|
|
}
|
|
|
|
// ParseNodeState reads `scontrol show node -o NAME`, one line of key=value.
|
|
func ParseNodeState(out string) (NodeState, error) {
|
|
line := strings.TrimSpace(out)
|
|
if !strings.HasPrefix(line, "NodeName=") {
|
|
return NodeState{}, fmt.Errorf("scontrol said %q", line)
|
|
}
|
|
var s NodeState
|
|
if i := strings.Index(line, " State="); i >= 0 {
|
|
s.State, _, _ = strings.Cut(line[i+len(" State="):], " ")
|
|
}
|
|
// Reason is last but for its [user@time] and may have spaces.
|
|
if i := strings.Index(line, " Reason="); i >= 0 {
|
|
r := line[i+len(" Reason="):]
|
|
if j := strings.LastIndex(r, " ["); j >= 0 {
|
|
r = r[:j]
|
|
}
|
|
s.Reason = strings.TrimSpace(r)
|
|
}
|
|
return s, nil
|
|
}
|
|
|
|
// ParseVersion reads "slurm 23.11.4" or "slurm-wlm 21.08.5" as major, minor.
|
|
func ParseVersion(out string) ([2]int, error) {
|
|
for _, f := range strings.Fields(out) {
|
|
parts := strings.Split(f, ".")
|
|
if len(parts) < 2 {
|
|
continue
|
|
}
|
|
major, err1 := strconv.Atoi(parts[0])
|
|
minor, err2 := strconv.Atoi(parts[1])
|
|
if err1 == nil && err2 == nil {
|
|
return [2]int{major, minor}, nil
|
|
}
|
|
}
|
|
return [2]int{}, fmt.Errorf("no version in %q", strings.TrimSpace(out))
|
|
}
|
|
|
|
// TooOld is whether v is older than MinVersion.
|
|
func TooOld(v [2]int) bool {
|
|
return v[0] < MinVersion[0] || (v[0] == MinVersion[0] && v[1] < MinVersion[1])
|
|
}
|