Files
llmgw/internal/slurm/slurm.go
T
Ilker ManapandClaude Opus 5.5 748f777a56
CI / test (push) Successful in 10m57s
Slurm's accounting: slurmdbd and a MariaDB on the controller
Since 23.11 Slurm runs badly without accounting storage: the main
scheduler passes every job by as InvalidAccount. slurmdbd takes MySQL or
MariaDB only, so the controller's agent installs MariaDB beside it,
answering that machine alone, makes the database and its user (the
password made from the munge key, never on a command line), runs
slurmdbd and adds the cluster, all before slurmctld. A MariaDB that was
on the machine before keeps its settings. slurm.conf accounts the jobs
there, with what each used from its cgroup and the cards, and the
priority is multifactor again. Port 6819 joins Slurm's in the firewall.
ctl slurm status and the console say whether slurmdbd runs; the install
screen, server init --slurm and the guide say what is put where.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-12 07:59:08 +01:00

364 lines
11 KiB
Go

// Package slurm is what llmgw knows of Slurm (BACKLOG P): the files a
// controller runs on, written from what llmgw knows of its nodes, the munge
// key every machine shares, and reading what slurmd and scontrol say.
//
// Nothing here runs anything. The agent installs and starts the daemons; the
// leader decides what each node is and writes the files with this package.
package slurm
import (
"crypto/hkdf"
"crypto/sha256"
"fmt"
"regexp"
"slices"
"strconv"
"strings"
)
// Paths are where the files are on every distribution llmgw installs on:
// Debian and Ubuntu since Slurm 20.11, and EPEL.
const (
ConfDir = "/etc/slurm"
MungeKeyPath = "/etc/munge/munge.key"
StateDir = "/var/spool/slurmctld"
SpoolDir = "/var/spool/slurmd"
LogDir = "/var/log/slurm"
// CtldPort is slurmctld's, which configless slurmd fetch their files
// from.
CtldPort = 6817
// SlurmdPort is slurmd's, which the controller and srun reach.
SlurmdPort = 6818
// SrunPortsFrom and SrunPortsTo are where srun listens for the steps it
// starts: a range of its own, so a firewall can let it in (Slurm's own
// example range). Without one srun takes any ephemeral port.
SrunPortsFrom = 60001
SrunPortsTo = 63000
)
// ReasonNew is why a node new to slurm.conf starts drained. Every reason
// llmgw drains with starts with "llmgw", and only those are resumed by it.
const ReasonNew = "llmgw: not lent yet"
// MinVersion is the oldest Slurm llmgw runs: the first to put jobs in
// cgroup v2, which every distribution llmgw supports now mounts alone.
var MinVersion = [2]int{22, 5}
// MungeKeyLen is the length munge's own tool writes.
const MungeKeyLen = 1024
// MungeKey is the key every machine of the Slurm cluster shares, derived
// from a cluster secret, so a control plane that takes over writes the same
// one and nobody copies it by hand.
func MungeKey(secret string) ([]byte, error) {
if len(secret) < 32 {
return nil, fmt.Errorf("the secret a munge key is made from is too short")
}
return hkdf.Key(sha256.New, []byte(secret), nil, "llmgw slurm munge key v1", MungeKeyLen)
}
// Hardware is a node's line from `slurmd -C`: how slurmd counts it, which is
// what slurm.conf must say or the controller refuses the node.
type Hardware struct {
CPUs int
Boards int
SocketsPerBoard int
CoresPerSocket int
ThreadsPerCore int
// RealMemory is in MiB.
RealMemory int64
}
// ParseHardware reads `slurmd -C`.
func ParseHardware(out string) (Hardware, error) {
var h Hardware
for _, line := range strings.Split(out, "\n") {
if !strings.HasPrefix(strings.TrimSpace(line), "NodeName=") {
continue
}
for _, f := range strings.Fields(line) {
k, v, _ := strings.Cut(f, "=")
n, err := strconv.ParseInt(v, 10, 64)
if err != nil {
continue
}
switch k {
case "CPUs":
h.CPUs = int(n)
case "Boards":
h.Boards = int(n)
case "SocketsPerBoard":
h.SocketsPerBoard = int(n)
case "CoresPerSocket":
h.CoresPerSocket = int(n)
case "ThreadsPerCore":
h.ThreadsPerCore = int(n)
case "RealMemory":
h.RealMemory = n
}
}
break
}
if h.CPUs <= 0 || h.RealMemory <= 0 {
return Hardware{}, fmt.Errorf("slurmd -C said no CPUs or memory: %q", strings.TrimSpace(out))
}
return h, nil
}
// GPU is one of a node's cards.
type GPU struct {
// Minor is N of /dev/nvidiaN.
Minor int
// Name is the driver's, as nvidia-smi gives it.
Name string
}
// Node is one machine slurmd runs on.
type Node struct {
Name string
// Addr is where slurmctld reaches it: its llmgw management address.
Addr string
Hardware
GPUs []GPU
// Lent is whether llmgw has lent it to Slurm: it is not written
// drained, or every start of slurmctld on these files drains it again.
Lent bool `json:"-"`
}
// Cluster is everything slurm.conf says.
type Cluster struct {
Name string
Controller string
ControllerAddr string
Partition string
// Nodes without hardware (not reported yet) are left out of the files
// until they are.
Nodes []Node
}
// gpuModel is a word that names a card's model: A100, H100, V100, T4, L40S.
var gpuModel = regexp.MustCompile(`^[a-z]{1,2}\d{1,4}[a-z]?$`)
// GPUType is a card's name as a Slurm GRES type: "NVIDIA A100-SXM4-80GB" is
// a100, "NVIDIA GeForce RTX 3090" rtx3090, "Tesla T4" t4. A name it cannot
// read is "gpu".
func GPUType(name string) string {
words := strings.Fields(strings.ToLower(strings.NewReplacer("-", " ", "_", " ").Replace(name)))
for i, w := range words {
if (w == "rtx" || w == "gtx") && i+1 < len(words) {
return w + words[i+1]
}
if gpuModel.MatchString(w) {
return w
}
}
return "gpu"
}
// gpuGroups are a node's cards by type, the types sorted.
func gpuGroups(gs []GPU) ([]string, map[string][]int) {
by := map[string][]int{}
for _, g := range gs {
t := GPUType(g.Name)
by[t] = append(by[t], g.Minor)
}
types := make([]string, 0, len(by))
for t, ms := range by {
slices.Sort(ms)
types = append(types, t)
}
slices.Sort(types)
return types, by
}
// memoryKept is what is left out of a node's RealMemory: slurmd -C reads
// what the kernel says now, which is a little less after a reboot, and a
// node whose memory is below what slurm.conf says is refused.
const memoryKept = 0.97
func (c Cluster) ready() []Node {
var out []Node
for _, n := range c.Nodes {
if n.CPUs > 0 && n.RealMemory > 0 {
out = append(out, n)
}
}
slices.SortFunc(out, func(a, b Node) int { return strings.Compare(a.Name, b.Name) })
return out
}
const header = "# Written by llmgw (BACKLOG P), on the node that runs slurmctld; the nodes read it\n" +
"# from there. Change it through the cluster's settings (slurm.*), not here.\n"
// SlurmConf is slurm.conf.
func (c Cluster) SlurmConf() string {
var b strings.Builder
b.WriteString(header)
fmt.Fprintf(&b, "ClusterName=%s\n", c.Name)
fmt.Fprintf(&b, "SlurmctldHost=%s(%s)\n", c.Controller, c.ControllerAddr)
// The jobs' records, in slurmdbd beside slurmctld (DbdConf).
fmt.Fprintf(&b, "AccountingStorageType=accounting_storage/slurmdbd\nAccountingStorageHost=%s\nAccountingStoragePort=%d\n",
c.ControllerAddr, DbdPort)
b.WriteString(`SlurmctldParameters=enable_configless
SlurmctldPort=6817
SlurmdPort=6818
SrunPortRange=60001-63000
AuthType=auth/munge
CredType=cred/munge
SlurmUser=slurm
SlurmdUser=root
StateSaveLocation=` + StateDir + `
SlurmdSpoolDir=` + SpoolDir + `
SlurmctldLogFile=` + LogDir + `/slurmctld.log
SlurmdLogFile=` + LogDir + `/slurmd.log
SlurmctldPidFile=/run/slurmctld.pid
SlurmdPidFile=/run/slurmd.pid
ProctrackType=proctrack/cgroup
TaskPlugin=task/cgroup,task/affinity
# What each job used, from its cgroup, for the records; the cards too.
JobAcctGatherType=jobacct_gather/cgroup
AccountingStorageTRES=gres/gpu
SelectType=select/cons_tres
SelectTypeParameters=CR_Core_Memory
SchedulerType=sched/backfill
PriorityType=priority/multifactor
GresTypes=gpu
MpiDefault=none
# No mail sent; without it every start says MailProg is invalid.
MailProg=/bin/true
# Backfill looks again every 10 s, not every 30.
SchedulerParameters=bf_interval=10
# A node that went down and comes back is taken again; one llmgw drained
# stays drained until llmgw resumes it.
ReturnToService=2
SlurmctldTimeout=120
SlurmdTimeout=300
`)
nodes := c.ready()
names := make([]string, 0, len(nodes))
// A job that asks for no memory gets this much per CPU: the share of
// the leanest node. Without it it gets the whole node's, and the jobs
// on a node run one at a time.
var perCPU int64
for _, n := range nodes {
if share := int64(float64(n.RealMemory)*memoryKept) / int64(n.CPUs); perCPU == 0 || share < perCPU {
perCPU = share
}
}
for _, n := range nodes {
names = append(names, n.Name)
fmt.Fprintf(&b, "NodeName=%s NodeAddr=%s CPUs=%d", n.Name, n.Addr, n.CPUs)
if n.Boards > 0 && n.SocketsPerBoard > 0 && n.CoresPerSocket > 0 && n.ThreadsPerCore > 0 {
fmt.Fprintf(&b, " Boards=%d SocketsPerBoard=%d CoresPerSocket=%d ThreadsPerCore=%d",
n.Boards, n.SocketsPerBoard, n.CoresPerSocket, n.ThreadsPerCore)
}
fmt.Fprintf(&b, " RealMemory=%d", int64(float64(n.RealMemory)*memoryKept))
if types, by := gpuGroups(n.GPUs); len(types) > 0 {
gres := make([]string, 0, len(types))
for _, t := range types {
gres = append(gres, fmt.Sprintf("gpu:%s:%d", t, len(by[t])))
}
fmt.Fprintf(&b, " Gres=%s", strings.Join(gres, ","))
}
// Drained until llmgw lends it: a node new to the controller would
// otherwise take jobs in the minute before its agent drains it.
if n.Lent {
b.WriteString("\n")
} else {
fmt.Fprintf(&b, " State=DRAIN Reason=%q\n", ReasonNew)
}
}
if len(names) > 0 {
fmt.Fprintf(&b, "PartitionName=%s Nodes=%s Default=YES MaxTime=INFINITE State=UP DefMemPerCPU=%d\n",
c.Partition, strings.Join(names, ","), perCPU)
}
return b.String()
}
// GresConf is gres.conf: each node's cards by device file, so a job is given
// the cards it asked for and kept off the others.
func (c Cluster) GresConf() string {
var b strings.Builder
b.WriteString(header)
for _, n := range c.ready() {
types, by := gpuGroups(n.GPUs)
for _, t := range types {
files := make([]string, 0, len(by[t]))
for _, m := range by[t] {
files = append(files, fmt.Sprintf("/dev/nvidia%d", m))
}
fmt.Fprintf(&b, "NodeName=%s Name=gpu Type=%s File=%s\n", n.Name, t, strings.Join(files, ","))
}
}
return b.String()
}
// CgroupConf is cgroup.conf: a job kept to its cores, memory and cards.
func (c Cluster) CgroupConf() string {
return header + `CgroupPlugin=autodetect
ConstrainCores=yes
ConstrainDevices=yes
ConstrainRAMSpace=yes
`
}
// Files are the controller's files, by name in ConfDir.
func (c Cluster) Files() map[string]string {
return map[string]string{"slurm.conf": c.SlurmConf(), "gres.conf": c.GresConf(), "cgroup.conf": c.CgroupConf()}
}
// NodeState is what `scontrol show node -o NAME` says of a node.
type NodeState struct {
// State is Slurm's, with its flags: IDLE, ALLOCATED, MIXED+DRAIN,
// DOWN+NOT_RESPONDING.
State string
Reason string
}
// Drained is whether the controller gives the node nothing new.
func (s NodeState) Drained() bool {
return strings.Contains(s.State, "DRAIN") || strings.Contains(s.State, "DOWN")
}
// ParseNodeState reads `scontrol show node -o NAME`, one line of key=value.
func ParseNodeState(out string) (NodeState, error) {
line := strings.TrimSpace(out)
if !strings.HasPrefix(line, "NodeName=") {
return NodeState{}, fmt.Errorf("scontrol said %q", line)
}
var s NodeState
if i := strings.Index(line, " State="); i >= 0 {
s.State, _, _ = strings.Cut(line[i+len(" State="):], " ")
}
// Reason is last but for its [user@time] and may have spaces.
if i := strings.Index(line, " Reason="); i >= 0 {
r := line[i+len(" Reason="):]
if j := strings.LastIndex(r, " ["); j >= 0 {
r = r[:j]
}
s.Reason = strings.TrimSpace(r)
}
return s, nil
}
// ParseVersion reads "slurm 23.11.4" or "slurm-wlm 21.08.5" as major, minor.
func ParseVersion(out string) ([2]int, error) {
for _, f := range strings.Fields(out) {
parts := strings.Split(f, ".")
if len(parts) < 2 {
continue
}
major, err1 := strconv.Atoi(parts[0])
minor, err2 := strconv.Atoi(parts[1])
if err1 == nil && err2 == nil {
return [2]int{major, minor}, nil
}
}
return [2]int{}, fmt.Errorf("no version in %q", strings.TrimSpace(out))
}
// TooOld is whether v is older than MinVersion.
func TooOld(v [2]int) bool {
return v[0] < MinVersion[0] || (v[0] == MinVersion[0] && v[1] < MinVersion[1])
}