CI / test (push) Successful in 10m57s
Since 23.11 Slurm runs badly without accounting storage: the main scheduler passes every job by as InvalidAccount. slurmdbd takes MySQL or MariaDB only, so the controller's agent installs MariaDB beside it, answering that machine alone, makes the database and its user (the password made from the munge key, never on a command line), runs slurmdbd and adds the cluster, all before slurmctld. A MariaDB that was on the machine before keeps its settings. slurm.conf accounts the jobs there, with what each used from its cgroup and the cards, and the priority is multifactor again. Port 6819 joins Slurm's in the firewall. ctl slurm status and the console say whether slurmdbd runs; the install screen, server init --slurm and the guide say what is put where. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
364 lines
11 KiB
Go
364 lines
11 KiB
Go
// Package slurm is what llmgw knows of Slurm (BACKLOG P): the files a
|
|
// controller runs on, written from what llmgw knows of its nodes, the munge
|
|
// key every machine shares, and reading what slurmd and scontrol say.
|
|
//
|
|
// Nothing here runs anything. The agent installs and starts the daemons; the
|
|
// leader decides what each node is and writes the files with this package.
|
|
package slurm
|
|
|
|
import (
|
|
"crypto/hkdf"
|
|
"crypto/sha256"
|
|
"fmt"
|
|
"regexp"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
// Paths are where the files are on every distribution llmgw installs on:
|
|
// Debian and Ubuntu since Slurm 20.11, and EPEL.
|
|
const (
|
|
ConfDir = "/etc/slurm"
|
|
MungeKeyPath = "/etc/munge/munge.key"
|
|
StateDir = "/var/spool/slurmctld"
|
|
SpoolDir = "/var/spool/slurmd"
|
|
LogDir = "/var/log/slurm"
|
|
// CtldPort is slurmctld's, which configless slurmd fetch their files
|
|
// from.
|
|
CtldPort = 6817
|
|
// SlurmdPort is slurmd's, which the controller and srun reach.
|
|
SlurmdPort = 6818
|
|
// SrunPortsFrom and SrunPortsTo are where srun listens for the steps it
|
|
// starts: a range of its own, so a firewall can let it in (Slurm's own
|
|
// example range). Without one srun takes any ephemeral port.
|
|
SrunPortsFrom = 60001
|
|
SrunPortsTo = 63000
|
|
)
|
|
|
|
// ReasonNew is why a node new to slurm.conf starts drained. Every reason
|
|
// llmgw drains with starts with "llmgw", and only those are resumed by it.
|
|
const ReasonNew = "llmgw: not lent yet"
|
|
|
|
// MinVersion is the oldest Slurm llmgw runs: the first to put jobs in
|
|
// cgroup v2, which every distribution llmgw supports now mounts alone.
|
|
var MinVersion = [2]int{22, 5}
|
|
|
|
// MungeKeyLen is the length munge's own tool writes.
|
|
const MungeKeyLen = 1024
|
|
|
|
// MungeKey is the key every machine of the Slurm cluster shares, derived
|
|
// from a cluster secret, so a control plane that takes over writes the same
|
|
// one and nobody copies it by hand.
|
|
func MungeKey(secret string) ([]byte, error) {
|
|
if len(secret) < 32 {
|
|
return nil, fmt.Errorf("the secret a munge key is made from is too short")
|
|
}
|
|
return hkdf.Key(sha256.New, []byte(secret), nil, "llmgw slurm munge key v1", MungeKeyLen)
|
|
}
|
|
|
|
// Hardware is a node's line from `slurmd -C`: how slurmd counts it, which is
|
|
// what slurm.conf must say or the controller refuses the node.
|
|
type Hardware struct {
|
|
CPUs int
|
|
Boards int
|
|
SocketsPerBoard int
|
|
CoresPerSocket int
|
|
ThreadsPerCore int
|
|
// RealMemory is in MiB.
|
|
RealMemory int64
|
|
}
|
|
|
|
// ParseHardware reads `slurmd -C`.
|
|
func ParseHardware(out string) (Hardware, error) {
|
|
var h Hardware
|
|
for _, line := range strings.Split(out, "\n") {
|
|
if !strings.HasPrefix(strings.TrimSpace(line), "NodeName=") {
|
|
continue
|
|
}
|
|
for _, f := range strings.Fields(line) {
|
|
k, v, _ := strings.Cut(f, "=")
|
|
n, err := strconv.ParseInt(v, 10, 64)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
switch k {
|
|
case "CPUs":
|
|
h.CPUs = int(n)
|
|
case "Boards":
|
|
h.Boards = int(n)
|
|
case "SocketsPerBoard":
|
|
h.SocketsPerBoard = int(n)
|
|
case "CoresPerSocket":
|
|
h.CoresPerSocket = int(n)
|
|
case "ThreadsPerCore":
|
|
h.ThreadsPerCore = int(n)
|
|
case "RealMemory":
|
|
h.RealMemory = n
|
|
}
|
|
}
|
|
break
|
|
}
|
|
if h.CPUs <= 0 || h.RealMemory <= 0 {
|
|
return Hardware{}, fmt.Errorf("slurmd -C said no CPUs or memory: %q", strings.TrimSpace(out))
|
|
}
|
|
return h, nil
|
|
}
|
|
|
|
// GPU is one of a node's cards.
|
|
type GPU struct {
|
|
// Minor is N of /dev/nvidiaN.
|
|
Minor int
|
|
// Name is the driver's, as nvidia-smi gives it.
|
|
Name string
|
|
}
|
|
|
|
// Node is one machine slurmd runs on.
|
|
type Node struct {
|
|
Name string
|
|
// Addr is where slurmctld reaches it: its llmgw management address.
|
|
Addr string
|
|
Hardware
|
|
GPUs []GPU
|
|
// Lent is whether llmgw has lent it to Slurm: it is not written
|
|
// drained, or every start of slurmctld on these files drains it again.
|
|
Lent bool `json:"-"`
|
|
}
|
|
|
|
// Cluster is everything slurm.conf says.
|
|
type Cluster struct {
|
|
Name string
|
|
Controller string
|
|
ControllerAddr string
|
|
Partition string
|
|
// Nodes without hardware (not reported yet) are left out of the files
|
|
// until they are.
|
|
Nodes []Node
|
|
}
|
|
|
|
// gpuModel is a word that names a card's model: A100, H100, V100, T4, L40S.
|
|
var gpuModel = regexp.MustCompile(`^[a-z]{1,2}\d{1,4}[a-z]?$`)
|
|
|
|
// GPUType is a card's name as a Slurm GRES type: "NVIDIA A100-SXM4-80GB" is
|
|
// a100, "NVIDIA GeForce RTX 3090" rtx3090, "Tesla T4" t4. A name it cannot
|
|
// read is "gpu".
|
|
func GPUType(name string) string {
|
|
words := strings.Fields(strings.ToLower(strings.NewReplacer("-", " ", "_", " ").Replace(name)))
|
|
for i, w := range words {
|
|
if (w == "rtx" || w == "gtx") && i+1 < len(words) {
|
|
return w + words[i+1]
|
|
}
|
|
if gpuModel.MatchString(w) {
|
|
return w
|
|
}
|
|
}
|
|
return "gpu"
|
|
}
|
|
|
|
// gpuGroups are a node's cards by type, the types sorted.
|
|
func gpuGroups(gs []GPU) ([]string, map[string][]int) {
|
|
by := map[string][]int{}
|
|
for _, g := range gs {
|
|
t := GPUType(g.Name)
|
|
by[t] = append(by[t], g.Minor)
|
|
}
|
|
types := make([]string, 0, len(by))
|
|
for t, ms := range by {
|
|
slices.Sort(ms)
|
|
types = append(types, t)
|
|
}
|
|
slices.Sort(types)
|
|
return types, by
|
|
}
|
|
|
|
// memoryKept is what is left out of a node's RealMemory: slurmd -C reads
|
|
// what the kernel says now, which is a little less after a reboot, and a
|
|
// node whose memory is below what slurm.conf says is refused.
|
|
const memoryKept = 0.97
|
|
|
|
func (c Cluster) ready() []Node {
|
|
var out []Node
|
|
for _, n := range c.Nodes {
|
|
if n.CPUs > 0 && n.RealMemory > 0 {
|
|
out = append(out, n)
|
|
}
|
|
}
|
|
slices.SortFunc(out, func(a, b Node) int { return strings.Compare(a.Name, b.Name) })
|
|
return out
|
|
}
|
|
|
|
const header = "# Written by llmgw (BACKLOG P), on the node that runs slurmctld; the nodes read it\n" +
|
|
"# from there. Change it through the cluster's settings (slurm.*), not here.\n"
|
|
|
|
// SlurmConf is slurm.conf.
|
|
func (c Cluster) SlurmConf() string {
|
|
var b strings.Builder
|
|
b.WriteString(header)
|
|
fmt.Fprintf(&b, "ClusterName=%s\n", c.Name)
|
|
fmt.Fprintf(&b, "SlurmctldHost=%s(%s)\n", c.Controller, c.ControllerAddr)
|
|
// The jobs' records, in slurmdbd beside slurmctld (DbdConf).
|
|
fmt.Fprintf(&b, "AccountingStorageType=accounting_storage/slurmdbd\nAccountingStorageHost=%s\nAccountingStoragePort=%d\n",
|
|
c.ControllerAddr, DbdPort)
|
|
b.WriteString(`SlurmctldParameters=enable_configless
|
|
SlurmctldPort=6817
|
|
SlurmdPort=6818
|
|
SrunPortRange=60001-63000
|
|
AuthType=auth/munge
|
|
CredType=cred/munge
|
|
SlurmUser=slurm
|
|
SlurmdUser=root
|
|
StateSaveLocation=` + StateDir + `
|
|
SlurmdSpoolDir=` + SpoolDir + `
|
|
SlurmctldLogFile=` + LogDir + `/slurmctld.log
|
|
SlurmdLogFile=` + LogDir + `/slurmd.log
|
|
SlurmctldPidFile=/run/slurmctld.pid
|
|
SlurmdPidFile=/run/slurmd.pid
|
|
ProctrackType=proctrack/cgroup
|
|
TaskPlugin=task/cgroup,task/affinity
|
|
# What each job used, from its cgroup, for the records; the cards too.
|
|
JobAcctGatherType=jobacct_gather/cgroup
|
|
AccountingStorageTRES=gres/gpu
|
|
SelectType=select/cons_tres
|
|
SelectTypeParameters=CR_Core_Memory
|
|
SchedulerType=sched/backfill
|
|
PriorityType=priority/multifactor
|
|
GresTypes=gpu
|
|
MpiDefault=none
|
|
# No mail sent; without it every start says MailProg is invalid.
|
|
MailProg=/bin/true
|
|
# Backfill looks again every 10 s, not every 30.
|
|
SchedulerParameters=bf_interval=10
|
|
# A node that went down and comes back is taken again; one llmgw drained
|
|
# stays drained until llmgw resumes it.
|
|
ReturnToService=2
|
|
SlurmctldTimeout=120
|
|
SlurmdTimeout=300
|
|
`)
|
|
nodes := c.ready()
|
|
names := make([]string, 0, len(nodes))
|
|
// A job that asks for no memory gets this much per CPU: the share of
|
|
// the leanest node. Without it it gets the whole node's, and the jobs
|
|
// on a node run one at a time.
|
|
var perCPU int64
|
|
for _, n := range nodes {
|
|
if share := int64(float64(n.RealMemory)*memoryKept) / int64(n.CPUs); perCPU == 0 || share < perCPU {
|
|
perCPU = share
|
|
}
|
|
}
|
|
for _, n := range nodes {
|
|
names = append(names, n.Name)
|
|
fmt.Fprintf(&b, "NodeName=%s NodeAddr=%s CPUs=%d", n.Name, n.Addr, n.CPUs)
|
|
if n.Boards > 0 && n.SocketsPerBoard > 0 && n.CoresPerSocket > 0 && n.ThreadsPerCore > 0 {
|
|
fmt.Fprintf(&b, " Boards=%d SocketsPerBoard=%d CoresPerSocket=%d ThreadsPerCore=%d",
|
|
n.Boards, n.SocketsPerBoard, n.CoresPerSocket, n.ThreadsPerCore)
|
|
}
|
|
fmt.Fprintf(&b, " RealMemory=%d", int64(float64(n.RealMemory)*memoryKept))
|
|
if types, by := gpuGroups(n.GPUs); len(types) > 0 {
|
|
gres := make([]string, 0, len(types))
|
|
for _, t := range types {
|
|
gres = append(gres, fmt.Sprintf("gpu:%s:%d", t, len(by[t])))
|
|
}
|
|
fmt.Fprintf(&b, " Gres=%s", strings.Join(gres, ","))
|
|
}
|
|
// Drained until llmgw lends it: a node new to the controller would
|
|
// otherwise take jobs in the minute before its agent drains it.
|
|
if n.Lent {
|
|
b.WriteString("\n")
|
|
} else {
|
|
fmt.Fprintf(&b, " State=DRAIN Reason=%q\n", ReasonNew)
|
|
}
|
|
}
|
|
if len(names) > 0 {
|
|
fmt.Fprintf(&b, "PartitionName=%s Nodes=%s Default=YES MaxTime=INFINITE State=UP DefMemPerCPU=%d\n",
|
|
c.Partition, strings.Join(names, ","), perCPU)
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
// GresConf is gres.conf: each node's cards by device file, so a job is given
|
|
// the cards it asked for and kept off the others.
|
|
func (c Cluster) GresConf() string {
|
|
var b strings.Builder
|
|
b.WriteString(header)
|
|
for _, n := range c.ready() {
|
|
types, by := gpuGroups(n.GPUs)
|
|
for _, t := range types {
|
|
files := make([]string, 0, len(by[t]))
|
|
for _, m := range by[t] {
|
|
files = append(files, fmt.Sprintf("/dev/nvidia%d", m))
|
|
}
|
|
fmt.Fprintf(&b, "NodeName=%s Name=gpu Type=%s File=%s\n", n.Name, t, strings.Join(files, ","))
|
|
}
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
// CgroupConf is cgroup.conf: a job kept to its cores, memory and cards.
|
|
func (c Cluster) CgroupConf() string {
|
|
return header + `CgroupPlugin=autodetect
|
|
ConstrainCores=yes
|
|
ConstrainDevices=yes
|
|
ConstrainRAMSpace=yes
|
|
`
|
|
}
|
|
|
|
// Files are the controller's files, by name in ConfDir.
|
|
func (c Cluster) Files() map[string]string {
|
|
return map[string]string{"slurm.conf": c.SlurmConf(), "gres.conf": c.GresConf(), "cgroup.conf": c.CgroupConf()}
|
|
}
|
|
|
|
// NodeState is what `scontrol show node -o NAME` says of a node.
|
|
type NodeState struct {
|
|
// State is Slurm's, with its flags: IDLE, ALLOCATED, MIXED+DRAIN,
|
|
// DOWN+NOT_RESPONDING.
|
|
State string
|
|
Reason string
|
|
}
|
|
|
|
// Drained is whether the controller gives the node nothing new.
|
|
func (s NodeState) Drained() bool {
|
|
return strings.Contains(s.State, "DRAIN") || strings.Contains(s.State, "DOWN")
|
|
}
|
|
|
|
// ParseNodeState reads `scontrol show node -o NAME`, one line of key=value.
|
|
func ParseNodeState(out string) (NodeState, error) {
|
|
line := strings.TrimSpace(out)
|
|
if !strings.HasPrefix(line, "NodeName=") {
|
|
return NodeState{}, fmt.Errorf("scontrol said %q", line)
|
|
}
|
|
var s NodeState
|
|
if i := strings.Index(line, " State="); i >= 0 {
|
|
s.State, _, _ = strings.Cut(line[i+len(" State="):], " ")
|
|
}
|
|
// Reason is last but for its [user@time] and may have spaces.
|
|
if i := strings.Index(line, " Reason="); i >= 0 {
|
|
r := line[i+len(" Reason="):]
|
|
if j := strings.LastIndex(r, " ["); j >= 0 {
|
|
r = r[:j]
|
|
}
|
|
s.Reason = strings.TrimSpace(r)
|
|
}
|
|
return s, nil
|
|
}
|
|
|
|
// ParseVersion reads "slurm 23.11.4" or "slurm-wlm 21.08.5" as major, minor.
|
|
func ParseVersion(out string) ([2]int, error) {
|
|
for _, f := range strings.Fields(out) {
|
|
parts := strings.Split(f, ".")
|
|
if len(parts) < 2 {
|
|
continue
|
|
}
|
|
major, err1 := strconv.Atoi(parts[0])
|
|
minor, err2 := strconv.Atoi(parts[1])
|
|
if err1 == nil && err2 == nil {
|
|
return [2]int{major, minor}, nil
|
|
}
|
|
}
|
|
return [2]int{}, fmt.Errorf("no version in %q", strings.TrimSpace(out))
|
|
}
|
|
|
|
// TooOld is whether v is older than MinVersion.
|
|
func TooOld(v [2]int) bool {
|
|
return v[0] < MinVersion[0] || (v[0] == MinVersion[0] && v[1] < MinVersion[1])
|
|
}
|