Files
llmgw/internal/slurm/slurm.go
T
Ilker ManapandClaude Opus 5.5 35de47444c Slurm beside llmgw, P2: the agent keeps its node's part
A Slurm message says whether the node runs slurmctld on the leader's files,
slurmd configless, both or neither, and whether it is lent. The agent
installs munge and Slurm from the machine's sources (22.05 or newer, for
cgroup v2), writes the munge key, starts slurmd through a drop-in with
--conf-server, resumes a lent node that llmgw drained and drains one that is
not, never touching a drain a Slurm administrator made, and reports its
version, slurmd -C, state and jobs in SlurmStatus. Told it has no part, it
stops what it started. New nodes enter slurm.conf drained.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-11 12:25:41 +01:00

330 lines
9.9 KiB
Go

// Package slurm is what llmgw knows of Slurm (BACKLOG P): the files a
// controller runs on, written from what llmgw knows of its nodes, the munge
// key every machine shares, and reading what slurmd and scontrol say.
//
// Nothing here runs anything. The agent installs and starts the daemons; the
// leader decides what each node is and writes the files with this package.
package slurm
import (
"crypto/hkdf"
"crypto/sha256"
"fmt"
"regexp"
"slices"
"strconv"
"strings"
)
// Paths are where the files are on every distribution llmgw installs on:
// Debian and Ubuntu since Slurm 20.11, and EPEL.
const (
ConfDir = "/etc/slurm"
MungeKeyPath = "/etc/munge/munge.key"
StateDir = "/var/spool/slurmctld"
SpoolDir = "/var/spool/slurmd"
LogDir = "/var/log/slurm"
// CtldPort is slurmctld's, which configless slurmd fetch their files
// from.
CtldPort = 6817
)
// ReasonNew is why a node new to slurm.conf starts drained. Every reason
// llmgw drains with starts with "llmgw", and only those are resumed by it.
const ReasonNew = "llmgw: not lent yet"
// MinVersion is the oldest Slurm llmgw runs: the first to put jobs in
// cgroup v2, which every distribution llmgw supports now mounts alone.
var MinVersion = [2]int{22, 5}
// MungeKeyLen is the length munge's own tool writes.
const MungeKeyLen = 1024
// MungeKey is the key every machine of the Slurm cluster shares, derived
// from a cluster secret, so a control plane that takes over writes the same
// one and nobody copies it by hand.
func MungeKey(secret string) ([]byte, error) {
if len(secret) < 32 {
return nil, fmt.Errorf("the secret a munge key is made from is too short")
}
return hkdf.Key(sha256.New, []byte(secret), nil, "llmgw slurm munge key v1", MungeKeyLen)
}
// Hardware is a node's line from `slurmd -C`: how slurmd counts it, which is
// what slurm.conf must say or the controller refuses the node.
type Hardware struct {
CPUs int
Boards int
SocketsPerBoard int
CoresPerSocket int
ThreadsPerCore int
// RealMemory is in MiB.
RealMemory int64
}
// ParseHardware reads `slurmd -C`.
func ParseHardware(out string) (Hardware, error) {
var h Hardware
for _, line := range strings.Split(out, "\n") {
if !strings.HasPrefix(strings.TrimSpace(line), "NodeName=") {
continue
}
for _, f := range strings.Fields(line) {
k, v, _ := strings.Cut(f, "=")
n, err := strconv.ParseInt(v, 10, 64)
if err != nil {
continue
}
switch k {
case "CPUs":
h.CPUs = int(n)
case "Boards":
h.Boards = int(n)
case "SocketsPerBoard":
h.SocketsPerBoard = int(n)
case "CoresPerSocket":
h.CoresPerSocket = int(n)
case "ThreadsPerCore":
h.ThreadsPerCore = int(n)
case "RealMemory":
h.RealMemory = n
}
}
break
}
if h.CPUs <= 0 || h.RealMemory <= 0 {
return Hardware{}, fmt.Errorf("slurmd -C said no CPUs or memory: %q", strings.TrimSpace(out))
}
return h, nil
}
// GPU is one of a node's cards.
type GPU struct {
// Minor is N of /dev/nvidiaN.
Minor int
// Name is the driver's, as nvidia-smi gives it.
Name string
}
// Node is one machine slurmd runs on.
type Node struct {
Name string
// Addr is where slurmctld reaches it: its llmgw management address.
Addr string
Hardware
GPUs []GPU
}
// Cluster is everything slurm.conf says.
type Cluster struct {
Name string
Controller string
ControllerAddr string
Partition string
// Nodes without hardware (not reported yet) are left out of the files
// until they are.
Nodes []Node
}
// gpuModel is a word that names a card's model: A100, H100, V100, T4, L40S.
var gpuModel = regexp.MustCompile(`^[a-z]{1,2}\d{1,4}[a-z]?$`)
// GPUType is a card's name as a Slurm GRES type: "NVIDIA A100-SXM4-80GB" is
// a100, "NVIDIA GeForce RTX 3090" rtx3090, "Tesla T4" t4. A name it cannot
// read is "gpu".
func GPUType(name string) string {
words := strings.Fields(strings.ToLower(strings.NewReplacer("-", " ", "_", " ").Replace(name)))
for i, w := range words {
if (w == "rtx" || w == "gtx") && i+1 < len(words) {
return w + words[i+1]
}
if gpuModel.MatchString(w) {
return w
}
}
return "gpu"
}
// gpuGroups are a node's cards by type, the types sorted.
func gpuGroups(gs []GPU) ([]string, map[string][]int) {
by := map[string][]int{}
for _, g := range gs {
t := GPUType(g.Name)
by[t] = append(by[t], g.Minor)
}
types := make([]string, 0, len(by))
for t, ms := range by {
slices.Sort(ms)
types = append(types, t)
}
slices.Sort(types)
return types, by
}
// memoryKept is what is left out of a node's RealMemory: slurmd -C reads
// what the kernel says now, which is a little less after a reboot, and a
// node whose memory is below what slurm.conf says is refused.
const memoryKept = 0.97
func (c Cluster) ready() []Node {
var out []Node
for _, n := range c.Nodes {
if n.CPUs > 0 && n.RealMemory > 0 {
out = append(out, n)
}
}
slices.SortFunc(out, func(a, b Node) int { return strings.Compare(a.Name, b.Name) })
return out
}
const header = "# Written by llmgw (BACKLOG P), on the node that runs slurmctld; the nodes read it\n" +
"# from there. Change it through the cluster's settings (slurm.*), not here.\n"
// SlurmConf is slurm.conf.
func (c Cluster) SlurmConf() string {
var b strings.Builder
b.WriteString(header)
fmt.Fprintf(&b, "ClusterName=%s\n", c.Name)
fmt.Fprintf(&b, "SlurmctldHost=%s(%s)\n", c.Controller, c.ControllerAddr)
b.WriteString(`SlurmctldParameters=enable_configless
SlurmctldPort=6817
SlurmdPort=6818
AuthType=auth/munge
CredType=cred/munge
SlurmUser=slurm
SlurmdUser=root
StateSaveLocation=` + StateDir + `
SlurmdSpoolDir=` + SpoolDir + `
SlurmctldLogFile=` + LogDir + `/slurmctld.log
SlurmdLogFile=` + LogDir + `/slurmd.log
SlurmctldPidFile=/run/slurmctld.pid
SlurmdPidFile=/run/slurmd.pid
ProctrackType=proctrack/cgroup
TaskPlugin=task/cgroup,task/affinity
JobAcctGatherType=jobacct_gather/cgroup
SelectType=select/cons_tres
SelectTypeParameters=CR_Core_Memory
SchedulerType=sched/backfill
GresTypes=gpu
MpiDefault=none
# A node that went down and comes back is taken again; one llmgw drained
# stays drained until llmgw resumes it.
ReturnToService=2
SlurmctldTimeout=120
SlurmdTimeout=300
`)
nodes := c.ready()
names := make([]string, 0, len(nodes))
for _, n := range nodes {
names = append(names, n.Name)
fmt.Fprintf(&b, "NodeName=%s NodeAddr=%s CPUs=%d", n.Name, n.Addr, n.CPUs)
if n.Boards > 0 && n.SocketsPerBoard > 0 && n.CoresPerSocket > 0 && n.ThreadsPerCore > 0 {
fmt.Fprintf(&b, " Boards=%d SocketsPerBoard=%d CoresPerSocket=%d ThreadsPerCore=%d",
n.Boards, n.SocketsPerBoard, n.CoresPerSocket, n.ThreadsPerCore)
}
fmt.Fprintf(&b, " RealMemory=%d", int64(float64(n.RealMemory)*memoryKept))
if types, by := gpuGroups(n.GPUs); len(types) > 0 {
gres := make([]string, 0, len(types))
for _, t := range types {
gres = append(gres, fmt.Sprintf("gpu:%s:%d", t, len(by[t])))
}
fmt.Fprintf(&b, " Gres=%s", strings.Join(gres, ","))
}
// Drained until llmgw lends it: a node new to the controller would
// otherwise take jobs in the minute before its agent drains it.
fmt.Fprintf(&b, " State=DRAIN Reason=%q\n", ReasonNew)
}
if len(names) > 0 {
fmt.Fprintf(&b, "PartitionName=%s Nodes=%s Default=YES MaxTime=INFINITE State=UP\n",
c.Partition, strings.Join(names, ","))
}
return b.String()
}
// GresConf is gres.conf: each node's cards by device file, so a job is given
// the cards it asked for and kept off the others.
func (c Cluster) GresConf() string {
var b strings.Builder
b.WriteString(header)
for _, n := range c.ready() {
types, by := gpuGroups(n.GPUs)
for _, t := range types {
files := make([]string, 0, len(by[t]))
for _, m := range by[t] {
files = append(files, fmt.Sprintf("/dev/nvidia%d", m))
}
fmt.Fprintf(&b, "NodeName=%s Name=gpu Type=%s File=%s\n", n.Name, t, strings.Join(files, ","))
}
}
return b.String()
}
// CgroupConf is cgroup.conf: a job kept to its cores, memory and cards.
func (c Cluster) CgroupConf() string {
return header + `CgroupPlugin=autodetect
ConstrainCores=yes
ConstrainDevices=yes
ConstrainRAMSpace=yes
`
}
// Files are the controller's files, by name in ConfDir.
func (c Cluster) Files() map[string]string {
return map[string]string{"slurm.conf": c.SlurmConf(), "gres.conf": c.GresConf(), "cgroup.conf": c.CgroupConf()}
}
// NodeState is what `scontrol show node -o NAME` says of a node.
type NodeState struct {
// State is Slurm's, with its flags: IDLE, ALLOCATED, MIXED+DRAIN,
// DOWN+NOT_RESPONDING.
State string
Reason string
}
// Drained is whether the controller gives the node nothing new.
func (s NodeState) Drained() bool {
return strings.Contains(s.State, "DRAIN") || strings.Contains(s.State, "DOWN")
}
// ParseNodeState reads `scontrol show node -o NAME`, one line of key=value.
func ParseNodeState(out string) (NodeState, error) {
line := strings.TrimSpace(out)
if !strings.HasPrefix(line, "NodeName=") {
return NodeState{}, fmt.Errorf("scontrol said %q", line)
}
var s NodeState
if i := strings.Index(line, " State="); i >= 0 {
s.State, _, _ = strings.Cut(line[i+len(" State="):], " ")
}
// Reason is last but for its [user@time] and may have spaces.
if i := strings.Index(line, " Reason="); i >= 0 {
r := line[i+len(" Reason="):]
if j := strings.LastIndex(r, " ["); j >= 0 {
r = r[:j]
}
s.Reason = strings.TrimSpace(r)
}
return s, nil
}
// ParseVersion reads "slurm 23.11.4" or "slurm-wlm 21.08.5" as major, minor.
func ParseVersion(out string) ([2]int, error) {
for _, f := range strings.Fields(out) {
parts := strings.Split(f, ".")
if len(parts) < 2 {
continue
}
major, err1 := strconv.Atoi(parts[0])
minor, err2 := strconv.Atoi(parts[1])
if err1 == nil && err2 == nil {
return [2]int{major, minor}, nil
}
}
return [2]int{}, fmt.Errorf("no version in %q", strings.TrimSpace(out))
}
// TooOld is whether v is older than MinVersion.
func TooOld(v [2]int) bool {
return v[0] < MinVersion[0] || (v[0] == MinVersion[0] && v[1] < MinVersion[1])
}