Files
llmgw/internal/slurm/slurm.go
T
Ilker ManapandClaude Opus 5.5 849760c67d Slurm: no second restart; backfill every 10 s, MailProg set
The InvalidAccount seen on the PoC was not the reconfigure: 23.11.4 without
accounting storage has its main scheduler mark a batch job so, and backfill
starts it on its next pass (job 4 waited 15 s). bf_interval=10 keeps that
short; the extra restart is taken out again. MailProg=/bin/true quiets the
warning on every start.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-11 13:18:35 +01:00

337 lines
10 KiB
Go

// Package slurm is what llmgw knows of Slurm (BACKLOG P): the files a
// controller runs on, written from what llmgw knows of its nodes, the munge
// key every machine shares, and reading what slurmd and scontrol say.
//
// Nothing here runs anything. The agent installs and starts the daemons; the
// leader decides what each node is and writes the files with this package.
package slurm
import (
"crypto/hkdf"
"crypto/sha256"
"fmt"
"regexp"
"slices"
"strconv"
"strings"
)
// Paths are where the files are on every distribution llmgw installs on:
// Debian and Ubuntu since Slurm 20.11, and EPEL.
const (
ConfDir = "/etc/slurm"
MungeKeyPath = "/etc/munge/munge.key"
StateDir = "/var/spool/slurmctld"
SpoolDir = "/var/spool/slurmd"
LogDir = "/var/log/slurm"
// CtldPort is slurmctld's, which configless slurmd fetch their files
// from.
CtldPort = 6817
)
// ReasonNew is why a node new to slurm.conf starts drained. Every reason
// llmgw drains with starts with "llmgw", and only those are resumed by it.
const ReasonNew = "llmgw: not lent yet"
// MinVersion is the oldest Slurm llmgw runs: the first to put jobs in
// cgroup v2, which every distribution llmgw supports now mounts alone.
var MinVersion = [2]int{22, 5}
// MungeKeyLen is the length munge's own tool writes.
const MungeKeyLen = 1024
// MungeKey is the key every machine of the Slurm cluster shares, derived
// from a cluster secret, so a control plane that takes over writes the same
// one and nobody copies it by hand.
func MungeKey(secret string) ([]byte, error) {
if len(secret) < 32 {
return nil, fmt.Errorf("the secret a munge key is made from is too short")
}
return hkdf.Key(sha256.New, []byte(secret), nil, "llmgw slurm munge key v1", MungeKeyLen)
}
// Hardware is a node's line from `slurmd -C`: how slurmd counts it, which is
// what slurm.conf must say or the controller refuses the node.
type Hardware struct {
CPUs int
Boards int
SocketsPerBoard int
CoresPerSocket int
ThreadsPerCore int
// RealMemory is in MiB.
RealMemory int64
}
// ParseHardware reads `slurmd -C`.
func ParseHardware(out string) (Hardware, error) {
var h Hardware
for _, line := range strings.Split(out, "\n") {
if !strings.HasPrefix(strings.TrimSpace(line), "NodeName=") {
continue
}
for _, f := range strings.Fields(line) {
k, v, _ := strings.Cut(f, "=")
n, err := strconv.ParseInt(v, 10, 64)
if err != nil {
continue
}
switch k {
case "CPUs":
h.CPUs = int(n)
case "Boards":
h.Boards = int(n)
case "SocketsPerBoard":
h.SocketsPerBoard = int(n)
case "CoresPerSocket":
h.CoresPerSocket = int(n)
case "ThreadsPerCore":
h.ThreadsPerCore = int(n)
case "RealMemory":
h.RealMemory = n
}
}
break
}
if h.CPUs <= 0 || h.RealMemory <= 0 {
return Hardware{}, fmt.Errorf("slurmd -C said no CPUs or memory: %q", strings.TrimSpace(out))
}
return h, nil
}
// GPU is one of a node's cards.
type GPU struct {
// Minor is N of /dev/nvidiaN.
Minor int
// Name is the driver's, as nvidia-smi gives it.
Name string
}
// Node is one machine slurmd runs on.
type Node struct {
Name string
// Addr is where slurmctld reaches it: its llmgw management address.
Addr string
Hardware
GPUs []GPU
}
// Cluster is everything slurm.conf says.
type Cluster struct {
Name string
Controller string
ControllerAddr string
Partition string
// Nodes without hardware (not reported yet) are left out of the files
// until they are.
Nodes []Node
}
// gpuModel is a word that names a card's model: A100, H100, V100, T4, L40S.
var gpuModel = regexp.MustCompile(`^[a-z]{1,2}\d{1,4}[a-z]?$`)
// GPUType is a card's name as a Slurm GRES type: "NVIDIA A100-SXM4-80GB" is
// a100, "NVIDIA GeForce RTX 3090" rtx3090, "Tesla T4" t4. A name it cannot
// read is "gpu".
func GPUType(name string) string {
words := strings.Fields(strings.ToLower(strings.NewReplacer("-", " ", "_", " ").Replace(name)))
for i, w := range words {
if (w == "rtx" || w == "gtx") && i+1 < len(words) {
return w + words[i+1]
}
if gpuModel.MatchString(w) {
return w
}
}
return "gpu"
}
// gpuGroups are a node's cards by type, the types sorted.
func gpuGroups(gs []GPU) ([]string, map[string][]int) {
by := map[string][]int{}
for _, g := range gs {
t := GPUType(g.Name)
by[t] = append(by[t], g.Minor)
}
types := make([]string, 0, len(by))
for t, ms := range by {
slices.Sort(ms)
types = append(types, t)
}
slices.Sort(types)
return types, by
}
// memoryKept is what is left out of a node's RealMemory: slurmd -C reads
// what the kernel says now, which is a little less after a reboot, and a
// node whose memory is below what slurm.conf says is refused.
const memoryKept = 0.97
func (c Cluster) ready() []Node {
var out []Node
for _, n := range c.Nodes {
if n.CPUs > 0 && n.RealMemory > 0 {
out = append(out, n)
}
}
slices.SortFunc(out, func(a, b Node) int { return strings.Compare(a.Name, b.Name) })
return out
}
const header = "# Written by llmgw (BACKLOG P), on the node that runs slurmctld; the nodes read it\n" +
"# from there. Change it through the cluster's settings (slurm.*), not here.\n"
// SlurmConf is slurm.conf.
func (c Cluster) SlurmConf() string {
var b strings.Builder
b.WriteString(header)
fmt.Fprintf(&b, "ClusterName=%s\n", c.Name)
fmt.Fprintf(&b, "SlurmctldHost=%s(%s)\n", c.Controller, c.ControllerAddr)
b.WriteString(`SlurmctldParameters=enable_configless
SlurmctldPort=6817
SlurmdPort=6818
AuthType=auth/munge
CredType=cred/munge
SlurmUser=slurm
SlurmdUser=root
StateSaveLocation=` + StateDir + `
SlurmdSpoolDir=` + SpoolDir + `
SlurmctldLogFile=` + LogDir + `/slurmctld.log
SlurmdLogFile=` + LogDir + `/slurmd.log
SlurmctldPidFile=/run/slurmctld.pid
SlurmdPidFile=/run/slurmd.pid
ProctrackType=proctrack/cgroup
TaskPlugin=task/cgroup,task/affinity
# Nothing gathered for accounting until there is somewhere to keep it
# (slurmdbd, later); gathered and not kept is an error in every log.
JobAcctGatherType=jobacct_gather/none
SelectType=select/cons_tres
SelectTypeParameters=CR_Core_Memory
SchedulerType=sched/backfill
GresTypes=gpu
MpiDefault=none
# No mail sent; without it every start says MailProg is invalid.
MailProg=/bin/true
# Without accounting storage, 23.11's main scheduler marks a batch job
# InvalidAccount; backfill starts it on its next pass, so it is kept short.
SchedulerParameters=bf_interval=10
# A node that went down and comes back is taken again; one llmgw drained
# stays drained until llmgw resumes it.
ReturnToService=2
SlurmctldTimeout=120
SlurmdTimeout=300
`)
nodes := c.ready()
names := make([]string, 0, len(nodes))
for _, n := range nodes {
names = append(names, n.Name)
fmt.Fprintf(&b, "NodeName=%s NodeAddr=%s CPUs=%d", n.Name, n.Addr, n.CPUs)
if n.Boards > 0 && n.SocketsPerBoard > 0 && n.CoresPerSocket > 0 && n.ThreadsPerCore > 0 {
fmt.Fprintf(&b, " Boards=%d SocketsPerBoard=%d CoresPerSocket=%d ThreadsPerCore=%d",
n.Boards, n.SocketsPerBoard, n.CoresPerSocket, n.ThreadsPerCore)
}
fmt.Fprintf(&b, " RealMemory=%d", int64(float64(n.RealMemory)*memoryKept))
if types, by := gpuGroups(n.GPUs); len(types) > 0 {
gres := make([]string, 0, len(types))
for _, t := range types {
gres = append(gres, fmt.Sprintf("gpu:%s:%d", t, len(by[t])))
}
fmt.Fprintf(&b, " Gres=%s", strings.Join(gres, ","))
}
// Drained until llmgw lends it: a node new to the controller would
// otherwise take jobs in the minute before its agent drains it.
fmt.Fprintf(&b, " State=DRAIN Reason=%q\n", ReasonNew)
}
if len(names) > 0 {
fmt.Fprintf(&b, "PartitionName=%s Nodes=%s Default=YES MaxTime=INFINITE State=UP\n",
c.Partition, strings.Join(names, ","))
}
return b.String()
}
// GresConf is gres.conf: each node's cards by device file, so a job is given
// the cards it asked for and kept off the others.
func (c Cluster) GresConf() string {
var b strings.Builder
b.WriteString(header)
for _, n := range c.ready() {
types, by := gpuGroups(n.GPUs)
for _, t := range types {
files := make([]string, 0, len(by[t]))
for _, m := range by[t] {
files = append(files, fmt.Sprintf("/dev/nvidia%d", m))
}
fmt.Fprintf(&b, "NodeName=%s Name=gpu Type=%s File=%s\n", n.Name, t, strings.Join(files, ","))
}
}
return b.String()
}
// CgroupConf is cgroup.conf: a job kept to its cores, memory and cards.
func (c Cluster) CgroupConf() string {
return header + `CgroupPlugin=autodetect
ConstrainCores=yes
ConstrainDevices=yes
ConstrainRAMSpace=yes
`
}
// Files are the controller's files, by name in ConfDir.
func (c Cluster) Files() map[string]string {
return map[string]string{"slurm.conf": c.SlurmConf(), "gres.conf": c.GresConf(), "cgroup.conf": c.CgroupConf()}
}
// NodeState is what `scontrol show node -o NAME` says of a node.
type NodeState struct {
// State is Slurm's, with its flags: IDLE, ALLOCATED, MIXED+DRAIN,
// DOWN+NOT_RESPONDING.
State string
Reason string
}
// Drained is whether the controller gives the node nothing new.
func (s NodeState) Drained() bool {
return strings.Contains(s.State, "DRAIN") || strings.Contains(s.State, "DOWN")
}
// ParseNodeState reads `scontrol show node -o NAME`, one line of key=value.
func ParseNodeState(out string) (NodeState, error) {
line := strings.TrimSpace(out)
if !strings.HasPrefix(line, "NodeName=") {
return NodeState{}, fmt.Errorf("scontrol said %q", line)
}
var s NodeState
if i := strings.Index(line, " State="); i >= 0 {
s.State, _, _ = strings.Cut(line[i+len(" State="):], " ")
}
// Reason is last but for its [user@time] and may have spaces.
if i := strings.Index(line, " Reason="); i >= 0 {
r := line[i+len(" Reason="):]
if j := strings.LastIndex(r, " ["); j >= 0 {
r = r[:j]
}
s.Reason = strings.TrimSpace(r)
}
return s, nil
}
// ParseVersion reads "slurm 23.11.4" or "slurm-wlm 21.08.5" as major, minor.
func ParseVersion(out string) ([2]int, error) {
for _, f := range strings.Fields(out) {
parts := strings.Split(f, ".")
if len(parts) < 2 {
continue
}
major, err1 := strconv.Atoi(parts[0])
minor, err2 := strconv.Atoi(parts[1])
if err1 == nil && err2 == nil {
return [2]int{major, minor}, nil
}
}
return [2]int{}, fmt.Errorf("no version in %q", strings.TrimSpace(out))
}
// TooOld is whether v is older than MinVersion.
func TooOld(v [2]int) bool {
return v[0] < MinVersion[0] || (v[0] == MinVersion[0] && v[1] < MinVersion[1])
}