A Slurm message says whether the node runs slurmctld on the leader's files, slurmd configless, both or neither, and whether it is lent. The agent installs munge and Slurm from the machine's sources (22.05 or newer, for cgroup v2), writes the munge key, starts slurmd through a drop-in with --conf-server, resumes a lent node that llmgw drained and drains one that is not, never touching a drain a Slurm administrator made, and reports its version, slurmd -C, state and jobs in SlurmStatus. Told it has no part, it stops what it started. New nodes enter slurm.conf drained. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
330 lines
9.9 KiB
Go
330 lines
9.9 KiB
Go
// Package slurm is what llmgw knows of Slurm (BACKLOG P): the files a
|
|
// controller runs on, written from what llmgw knows of its nodes, the munge
|
|
// key every machine shares, and reading what slurmd and scontrol say.
|
|
//
|
|
// Nothing here runs anything. The agent installs and starts the daemons; the
|
|
// leader decides what each node is and writes the files with this package.
|
|
package slurm
|
|
|
|
import (
|
|
"crypto/hkdf"
|
|
"crypto/sha256"
|
|
"fmt"
|
|
"regexp"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
// Paths are where the files are on every distribution llmgw installs on:
|
|
// Debian and Ubuntu since Slurm 20.11, and EPEL.
|
|
const (
|
|
ConfDir = "/etc/slurm"
|
|
MungeKeyPath = "/etc/munge/munge.key"
|
|
StateDir = "/var/spool/slurmctld"
|
|
SpoolDir = "/var/spool/slurmd"
|
|
LogDir = "/var/log/slurm"
|
|
// CtldPort is slurmctld's, which configless slurmd fetch their files
|
|
// from.
|
|
CtldPort = 6817
|
|
)
|
|
|
|
// ReasonNew is why a node new to slurm.conf starts drained. Every reason
|
|
// llmgw drains with starts with "llmgw", and only those are resumed by it.
|
|
const ReasonNew = "llmgw: not lent yet"
|
|
|
|
// MinVersion is the oldest Slurm llmgw runs: the first to put jobs in
|
|
// cgroup v2, which every distribution llmgw supports now mounts alone.
|
|
var MinVersion = [2]int{22, 5}
|
|
|
|
// MungeKeyLen is the length munge's own tool writes.
|
|
const MungeKeyLen = 1024
|
|
|
|
// MungeKey is the key every machine of the Slurm cluster shares, derived
|
|
// from a cluster secret, so a control plane that takes over writes the same
|
|
// one and nobody copies it by hand.
|
|
func MungeKey(secret string) ([]byte, error) {
|
|
if len(secret) < 32 {
|
|
return nil, fmt.Errorf("the secret a munge key is made from is too short")
|
|
}
|
|
return hkdf.Key(sha256.New, []byte(secret), nil, "llmgw slurm munge key v1", MungeKeyLen)
|
|
}
|
|
|
|
// Hardware is a node's line from `slurmd -C`: how slurmd counts it, which is
|
|
// what slurm.conf must say or the controller refuses the node.
|
|
type Hardware struct {
|
|
CPUs int
|
|
Boards int
|
|
SocketsPerBoard int
|
|
CoresPerSocket int
|
|
ThreadsPerCore int
|
|
// RealMemory is in MiB.
|
|
RealMemory int64
|
|
}
|
|
|
|
// ParseHardware reads `slurmd -C`.
|
|
func ParseHardware(out string) (Hardware, error) {
|
|
var h Hardware
|
|
for _, line := range strings.Split(out, "\n") {
|
|
if !strings.HasPrefix(strings.TrimSpace(line), "NodeName=") {
|
|
continue
|
|
}
|
|
for _, f := range strings.Fields(line) {
|
|
k, v, _ := strings.Cut(f, "=")
|
|
n, err := strconv.ParseInt(v, 10, 64)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
switch k {
|
|
case "CPUs":
|
|
h.CPUs = int(n)
|
|
case "Boards":
|
|
h.Boards = int(n)
|
|
case "SocketsPerBoard":
|
|
h.SocketsPerBoard = int(n)
|
|
case "CoresPerSocket":
|
|
h.CoresPerSocket = int(n)
|
|
case "ThreadsPerCore":
|
|
h.ThreadsPerCore = int(n)
|
|
case "RealMemory":
|
|
h.RealMemory = n
|
|
}
|
|
}
|
|
break
|
|
}
|
|
if h.CPUs <= 0 || h.RealMemory <= 0 {
|
|
return Hardware{}, fmt.Errorf("slurmd -C said no CPUs or memory: %q", strings.TrimSpace(out))
|
|
}
|
|
return h, nil
|
|
}
|
|
|
|
// GPU is one of a node's cards.
|
|
type GPU struct {
|
|
// Minor is N of /dev/nvidiaN.
|
|
Minor int
|
|
// Name is the driver's, as nvidia-smi gives it.
|
|
Name string
|
|
}
|
|
|
|
// Node is one machine slurmd runs on.
|
|
type Node struct {
|
|
Name string
|
|
// Addr is where slurmctld reaches it: its llmgw management address.
|
|
Addr string
|
|
Hardware
|
|
GPUs []GPU
|
|
}
|
|
|
|
// Cluster is everything slurm.conf says.
|
|
type Cluster struct {
|
|
Name string
|
|
Controller string
|
|
ControllerAddr string
|
|
Partition string
|
|
// Nodes without hardware (not reported yet) are left out of the files
|
|
// until they are.
|
|
Nodes []Node
|
|
}
|
|
|
|
// gpuModel is a word that names a card's model: A100, H100, V100, T4, L40S.
|
|
var gpuModel = regexp.MustCompile(`^[a-z]{1,2}\d{1,4}[a-z]?$`)
|
|
|
|
// GPUType is a card's name as a Slurm GRES type: "NVIDIA A100-SXM4-80GB" is
|
|
// a100, "NVIDIA GeForce RTX 3090" rtx3090, "Tesla T4" t4. A name it cannot
|
|
// read is "gpu".
|
|
func GPUType(name string) string {
|
|
words := strings.Fields(strings.ToLower(strings.NewReplacer("-", " ", "_", " ").Replace(name)))
|
|
for i, w := range words {
|
|
if (w == "rtx" || w == "gtx") && i+1 < len(words) {
|
|
return w + words[i+1]
|
|
}
|
|
if gpuModel.MatchString(w) {
|
|
return w
|
|
}
|
|
}
|
|
return "gpu"
|
|
}
|
|
|
|
// gpuGroups are a node's cards by type, the types sorted.
|
|
func gpuGroups(gs []GPU) ([]string, map[string][]int) {
|
|
by := map[string][]int{}
|
|
for _, g := range gs {
|
|
t := GPUType(g.Name)
|
|
by[t] = append(by[t], g.Minor)
|
|
}
|
|
types := make([]string, 0, len(by))
|
|
for t, ms := range by {
|
|
slices.Sort(ms)
|
|
types = append(types, t)
|
|
}
|
|
slices.Sort(types)
|
|
return types, by
|
|
}
|
|
|
|
// memoryKept is what is left out of a node's RealMemory: slurmd -C reads
|
|
// what the kernel says now, which is a little less after a reboot, and a
|
|
// node whose memory is below what slurm.conf says is refused.
|
|
const memoryKept = 0.97
|
|
|
|
func (c Cluster) ready() []Node {
|
|
var out []Node
|
|
for _, n := range c.Nodes {
|
|
if n.CPUs > 0 && n.RealMemory > 0 {
|
|
out = append(out, n)
|
|
}
|
|
}
|
|
slices.SortFunc(out, func(a, b Node) int { return strings.Compare(a.Name, b.Name) })
|
|
return out
|
|
}
|
|
|
|
const header = "# Written by llmgw (BACKLOG P), on the node that runs slurmctld; the nodes read it\n" +
|
|
"# from there. Change it through the cluster's settings (slurm.*), not here.\n"
|
|
|
|
// SlurmConf is slurm.conf.
|
|
func (c Cluster) SlurmConf() string {
|
|
var b strings.Builder
|
|
b.WriteString(header)
|
|
fmt.Fprintf(&b, "ClusterName=%s\n", c.Name)
|
|
fmt.Fprintf(&b, "SlurmctldHost=%s(%s)\n", c.Controller, c.ControllerAddr)
|
|
b.WriteString(`SlurmctldParameters=enable_configless
|
|
SlurmctldPort=6817
|
|
SlurmdPort=6818
|
|
AuthType=auth/munge
|
|
CredType=cred/munge
|
|
SlurmUser=slurm
|
|
SlurmdUser=root
|
|
StateSaveLocation=` + StateDir + `
|
|
SlurmdSpoolDir=` + SpoolDir + `
|
|
SlurmctldLogFile=` + LogDir + `/slurmctld.log
|
|
SlurmdLogFile=` + LogDir + `/slurmd.log
|
|
SlurmctldPidFile=/run/slurmctld.pid
|
|
SlurmdPidFile=/run/slurmd.pid
|
|
ProctrackType=proctrack/cgroup
|
|
TaskPlugin=task/cgroup,task/affinity
|
|
JobAcctGatherType=jobacct_gather/cgroup
|
|
SelectType=select/cons_tres
|
|
SelectTypeParameters=CR_Core_Memory
|
|
SchedulerType=sched/backfill
|
|
GresTypes=gpu
|
|
MpiDefault=none
|
|
# A node that went down and comes back is taken again; one llmgw drained
|
|
# stays drained until llmgw resumes it.
|
|
ReturnToService=2
|
|
SlurmctldTimeout=120
|
|
SlurmdTimeout=300
|
|
`)
|
|
nodes := c.ready()
|
|
names := make([]string, 0, len(nodes))
|
|
for _, n := range nodes {
|
|
names = append(names, n.Name)
|
|
fmt.Fprintf(&b, "NodeName=%s NodeAddr=%s CPUs=%d", n.Name, n.Addr, n.CPUs)
|
|
if n.Boards > 0 && n.SocketsPerBoard > 0 && n.CoresPerSocket > 0 && n.ThreadsPerCore > 0 {
|
|
fmt.Fprintf(&b, " Boards=%d SocketsPerBoard=%d CoresPerSocket=%d ThreadsPerCore=%d",
|
|
n.Boards, n.SocketsPerBoard, n.CoresPerSocket, n.ThreadsPerCore)
|
|
}
|
|
fmt.Fprintf(&b, " RealMemory=%d", int64(float64(n.RealMemory)*memoryKept))
|
|
if types, by := gpuGroups(n.GPUs); len(types) > 0 {
|
|
gres := make([]string, 0, len(types))
|
|
for _, t := range types {
|
|
gres = append(gres, fmt.Sprintf("gpu:%s:%d", t, len(by[t])))
|
|
}
|
|
fmt.Fprintf(&b, " Gres=%s", strings.Join(gres, ","))
|
|
}
|
|
// Drained until llmgw lends it: a node new to the controller would
|
|
// otherwise take jobs in the minute before its agent drains it.
|
|
fmt.Fprintf(&b, " State=DRAIN Reason=%q\n", ReasonNew)
|
|
}
|
|
if len(names) > 0 {
|
|
fmt.Fprintf(&b, "PartitionName=%s Nodes=%s Default=YES MaxTime=INFINITE State=UP\n",
|
|
c.Partition, strings.Join(names, ","))
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
// GresConf is gres.conf: each node's cards by device file, so a job is given
|
|
// the cards it asked for and kept off the others.
|
|
func (c Cluster) GresConf() string {
|
|
var b strings.Builder
|
|
b.WriteString(header)
|
|
for _, n := range c.ready() {
|
|
types, by := gpuGroups(n.GPUs)
|
|
for _, t := range types {
|
|
files := make([]string, 0, len(by[t]))
|
|
for _, m := range by[t] {
|
|
files = append(files, fmt.Sprintf("/dev/nvidia%d", m))
|
|
}
|
|
fmt.Fprintf(&b, "NodeName=%s Name=gpu Type=%s File=%s\n", n.Name, t, strings.Join(files, ","))
|
|
}
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
// CgroupConf is cgroup.conf: a job kept to its cores, memory and cards.
|
|
func (c Cluster) CgroupConf() string {
|
|
return header + `CgroupPlugin=autodetect
|
|
ConstrainCores=yes
|
|
ConstrainDevices=yes
|
|
ConstrainRAMSpace=yes
|
|
`
|
|
}
|
|
|
|
// Files are the controller's files, by name in ConfDir.
|
|
func (c Cluster) Files() map[string]string {
|
|
return map[string]string{"slurm.conf": c.SlurmConf(), "gres.conf": c.GresConf(), "cgroup.conf": c.CgroupConf()}
|
|
}
|
|
|
|
// NodeState is what `scontrol show node -o NAME` says of a node.
|
|
type NodeState struct {
|
|
// State is Slurm's, with its flags: IDLE, ALLOCATED, MIXED+DRAIN,
|
|
// DOWN+NOT_RESPONDING.
|
|
State string
|
|
Reason string
|
|
}
|
|
|
|
// Drained is whether the controller gives the node nothing new.
|
|
func (s NodeState) Drained() bool {
|
|
return strings.Contains(s.State, "DRAIN") || strings.Contains(s.State, "DOWN")
|
|
}
|
|
|
|
// ParseNodeState reads `scontrol show node -o NAME`, one line of key=value.
|
|
func ParseNodeState(out string) (NodeState, error) {
|
|
line := strings.TrimSpace(out)
|
|
if !strings.HasPrefix(line, "NodeName=") {
|
|
return NodeState{}, fmt.Errorf("scontrol said %q", line)
|
|
}
|
|
var s NodeState
|
|
if i := strings.Index(line, " State="); i >= 0 {
|
|
s.State, _, _ = strings.Cut(line[i+len(" State="):], " ")
|
|
}
|
|
// Reason is last but for its [user@time] and may have spaces.
|
|
if i := strings.Index(line, " Reason="); i >= 0 {
|
|
r := line[i+len(" Reason="):]
|
|
if j := strings.LastIndex(r, " ["); j >= 0 {
|
|
r = r[:j]
|
|
}
|
|
s.Reason = strings.TrimSpace(r)
|
|
}
|
|
return s, nil
|
|
}
|
|
|
|
// ParseVersion reads "slurm 23.11.4" or "slurm-wlm 21.08.5" as major, minor.
|
|
func ParseVersion(out string) ([2]int, error) {
|
|
for _, f := range strings.Fields(out) {
|
|
parts := strings.Split(f, ".")
|
|
if len(parts) < 2 {
|
|
continue
|
|
}
|
|
major, err1 := strconv.Atoi(parts[0])
|
|
minor, err2 := strconv.Atoi(parts[1])
|
|
if err1 == nil && err2 == nil {
|
|
return [2]int{major, minor}, nil
|
|
}
|
|
}
|
|
return [2]int{}, fmt.Errorf("no version in %q", strings.TrimSpace(out))
|
|
}
|
|
|
|
// TooOld is whether v is older than MinVersion.
|
|
func TooOld(v [2]int) bool {
|
|
return v[0] < MinVersion[0] || (v[0] == MinVersion[0] && v[1] < MinVersion[1])
|
|
}
|