// Package slurm is what llmgw knows of Slurm (BACKLOG P): the files a // controller runs on, written from what llmgw knows of its nodes, the munge // key every machine shares, and reading what slurmd and scontrol say. // // Nothing here runs anything. The agent installs and starts the daemons; the // leader decides what each node is and writes the files with this package. package slurm import ( "crypto/hkdf" "crypto/sha256" "fmt" "regexp" "slices" "strconv" "strings" ) // Paths are where the files are on every distribution llmgw installs on: // Debian and Ubuntu since Slurm 20.11, and EPEL. const ( ConfDir = "/etc/slurm" MungeKeyPath = "/etc/munge/munge.key" StateDir = "/var/spool/slurmctld" SpoolDir = "/var/spool/slurmd" LogDir = "/var/log/slurm" // CtldPort is slurmctld's, which configless slurmd fetch their files // from. CtldPort = 6817 ) // ReasonNew is why a node new to slurm.conf starts drained. Every reason // llmgw drains with starts with "llmgw", and only those are resumed by it. const ReasonNew = "llmgw: not lent yet" // MinVersion is the oldest Slurm llmgw runs: the first to put jobs in // cgroup v2, which every distribution llmgw supports now mounts alone. var MinVersion = [2]int{22, 5} // MungeKeyLen is the length munge's own tool writes. const MungeKeyLen = 1024 // MungeKey is the key every machine of the Slurm cluster shares, derived // from a cluster secret, so a control plane that takes over writes the same // one and nobody copies it by hand. func MungeKey(secret string) ([]byte, error) { if len(secret) < 32 { return nil, fmt.Errorf("the secret a munge key is made from is too short") } return hkdf.Key(sha256.New, []byte(secret), nil, "llmgw slurm munge key v1", MungeKeyLen) } // Hardware is a node's line from `slurmd -C`: how slurmd counts it, which is // what slurm.conf must say or the controller refuses the node. type Hardware struct { CPUs int Boards int SocketsPerBoard int CoresPerSocket int ThreadsPerCore int // RealMemory is in MiB. RealMemory int64 } // ParseHardware reads `slurmd -C`. func ParseHardware(out string) (Hardware, error) { var h Hardware for _, line := range strings.Split(out, "\n") { if !strings.HasPrefix(strings.TrimSpace(line), "NodeName=") { continue } for _, f := range strings.Fields(line) { k, v, _ := strings.Cut(f, "=") n, err := strconv.ParseInt(v, 10, 64) if err != nil { continue } switch k { case "CPUs": h.CPUs = int(n) case "Boards": h.Boards = int(n) case "SocketsPerBoard": h.SocketsPerBoard = int(n) case "CoresPerSocket": h.CoresPerSocket = int(n) case "ThreadsPerCore": h.ThreadsPerCore = int(n) case "RealMemory": h.RealMemory = n } } break } if h.CPUs <= 0 || h.RealMemory <= 0 { return Hardware{}, fmt.Errorf("slurmd -C said no CPUs or memory: %q", strings.TrimSpace(out)) } return h, nil } // GPU is one of a node's cards. type GPU struct { // Minor is N of /dev/nvidiaN. Minor int // Name is the driver's, as nvidia-smi gives it. Name string } // Node is one machine slurmd runs on. type Node struct { Name string // Addr is where slurmctld reaches it: its llmgw management address. Addr string Hardware GPUs []GPU } // Cluster is everything slurm.conf says. type Cluster struct { Name string Controller string ControllerAddr string Partition string // Nodes without hardware (not reported yet) are left out of the files // until they are. Nodes []Node } // gpuModel is a word that names a card's model: A100, H100, V100, T4, L40S. var gpuModel = regexp.MustCompile(`^[a-z]{1,2}\d{1,4}[a-z]?$`) // GPUType is a card's name as a Slurm GRES type: "NVIDIA A100-SXM4-80GB" is // a100, "NVIDIA GeForce RTX 3090" rtx3090, "Tesla T4" t4. A name it cannot // read is "gpu". func GPUType(name string) string { words := strings.Fields(strings.ToLower(strings.NewReplacer("-", " ", "_", " ").Replace(name))) for i, w := range words { if (w == "rtx" || w == "gtx") && i+1 < len(words) { return w + words[i+1] } if gpuModel.MatchString(w) { return w } } return "gpu" } // gpuGroups are a node's cards by type, the types sorted. func gpuGroups(gs []GPU) ([]string, map[string][]int) { by := map[string][]int{} for _, g := range gs { t := GPUType(g.Name) by[t] = append(by[t], g.Minor) } types := make([]string, 0, len(by)) for t, ms := range by { slices.Sort(ms) types = append(types, t) } slices.Sort(types) return types, by } // memoryKept is what is left out of a node's RealMemory: slurmd -C reads // what the kernel says now, which is a little less after a reboot, and a // node whose memory is below what slurm.conf says is refused. const memoryKept = 0.97 func (c Cluster) ready() []Node { var out []Node for _, n := range c.Nodes { if n.CPUs > 0 && n.RealMemory > 0 { out = append(out, n) } } slices.SortFunc(out, func(a, b Node) int { return strings.Compare(a.Name, b.Name) }) return out } const header = "# Written by llmgw (BACKLOG P), on the node that runs slurmctld; the nodes read it\n" + "# from there. Change it through the cluster's settings (slurm.*), not here.\n" // SlurmConf is slurm.conf. func (c Cluster) SlurmConf() string { var b strings.Builder b.WriteString(header) fmt.Fprintf(&b, "ClusterName=%s\n", c.Name) fmt.Fprintf(&b, "SlurmctldHost=%s(%s)\n", c.Controller, c.ControllerAddr) b.WriteString(`SlurmctldParameters=enable_configless SlurmctldPort=6817 SlurmdPort=6818 AuthType=auth/munge CredType=cred/munge SlurmUser=slurm SlurmdUser=root StateSaveLocation=` + StateDir + ` SlurmdSpoolDir=` + SpoolDir + ` SlurmctldLogFile=` + LogDir + `/slurmctld.log SlurmdLogFile=` + LogDir + `/slurmd.log SlurmctldPidFile=/run/slurmctld.pid SlurmdPidFile=/run/slurmd.pid ProctrackType=proctrack/cgroup TaskPlugin=task/cgroup,task/affinity # Nothing gathered for accounting until there is somewhere to keep it # (slurmdbd, later); gathered and not kept is an error in every log. JobAcctGatherType=jobacct_gather/none SelectType=select/cons_tres SelectTypeParameters=CR_Core_Memory SchedulerType=sched/backfill GresTypes=gpu MpiDefault=none # No mail sent; without it every start says MailProg is invalid. MailProg=/bin/true # Without accounting storage, 23.11's main scheduler marks a batch job # InvalidAccount; backfill starts it on its next pass, so it is kept short. SchedulerParameters=bf_interval=10 # A node that went down and comes back is taken again; one llmgw drained # stays drained until llmgw resumes it. ReturnToService=2 SlurmctldTimeout=120 SlurmdTimeout=300 `) nodes := c.ready() names := make([]string, 0, len(nodes)) for _, n := range nodes { names = append(names, n.Name) fmt.Fprintf(&b, "NodeName=%s NodeAddr=%s CPUs=%d", n.Name, n.Addr, n.CPUs) if n.Boards > 0 && n.SocketsPerBoard > 0 && n.CoresPerSocket > 0 && n.ThreadsPerCore > 0 { fmt.Fprintf(&b, " Boards=%d SocketsPerBoard=%d CoresPerSocket=%d ThreadsPerCore=%d", n.Boards, n.SocketsPerBoard, n.CoresPerSocket, n.ThreadsPerCore) } fmt.Fprintf(&b, " RealMemory=%d", int64(float64(n.RealMemory)*memoryKept)) if types, by := gpuGroups(n.GPUs); len(types) > 0 { gres := make([]string, 0, len(types)) for _, t := range types { gres = append(gres, fmt.Sprintf("gpu:%s:%d", t, len(by[t]))) } fmt.Fprintf(&b, " Gres=%s", strings.Join(gres, ",")) } // Drained until llmgw lends it: a node new to the controller would // otherwise take jobs in the minute before its agent drains it. fmt.Fprintf(&b, " State=DRAIN Reason=%q\n", ReasonNew) } if len(names) > 0 { fmt.Fprintf(&b, "PartitionName=%s Nodes=%s Default=YES MaxTime=INFINITE State=UP\n", c.Partition, strings.Join(names, ",")) } return b.String() } // GresConf is gres.conf: each node's cards by device file, so a job is given // the cards it asked for and kept off the others. func (c Cluster) GresConf() string { var b strings.Builder b.WriteString(header) for _, n := range c.ready() { types, by := gpuGroups(n.GPUs) for _, t := range types { files := make([]string, 0, len(by[t])) for _, m := range by[t] { files = append(files, fmt.Sprintf("/dev/nvidia%d", m)) } fmt.Fprintf(&b, "NodeName=%s Name=gpu Type=%s File=%s\n", n.Name, t, strings.Join(files, ",")) } } return b.String() } // CgroupConf is cgroup.conf: a job kept to its cores, memory and cards. func (c Cluster) CgroupConf() string { return header + `CgroupPlugin=autodetect ConstrainCores=yes ConstrainDevices=yes ConstrainRAMSpace=yes ` } // Files are the controller's files, by name in ConfDir. func (c Cluster) Files() map[string]string { return map[string]string{"slurm.conf": c.SlurmConf(), "gres.conf": c.GresConf(), "cgroup.conf": c.CgroupConf()} } // NodeState is what `scontrol show node -o NAME` says of a node. type NodeState struct { // State is Slurm's, with its flags: IDLE, ALLOCATED, MIXED+DRAIN, // DOWN+NOT_RESPONDING. State string Reason string } // Drained is whether the controller gives the node nothing new. func (s NodeState) Drained() bool { return strings.Contains(s.State, "DRAIN") || strings.Contains(s.State, "DOWN") } // ParseNodeState reads `scontrol show node -o NAME`, one line of key=value. func ParseNodeState(out string) (NodeState, error) { line := strings.TrimSpace(out) if !strings.HasPrefix(line, "NodeName=") { return NodeState{}, fmt.Errorf("scontrol said %q", line) } var s NodeState if i := strings.Index(line, " State="); i >= 0 { s.State, _, _ = strings.Cut(line[i+len(" State="):], " ") } // Reason is last but for its [user@time] and may have spaces. if i := strings.Index(line, " Reason="); i >= 0 { r := line[i+len(" Reason="):] if j := strings.LastIndex(r, " ["); j >= 0 { r = r[:j] } s.Reason = strings.TrimSpace(r) } return s, nil } // ParseVersion reads "slurm 23.11.4" or "slurm-wlm 21.08.5" as major, minor. func ParseVersion(out string) ([2]int, error) { for _, f := range strings.Fields(out) { parts := strings.Split(f, ".") if len(parts) < 2 { continue } major, err1 := strconv.Atoi(parts[0]) minor, err2 := strconv.Atoi(parts[1]) if err1 == nil && err2 == nil { return [2]int{major, minor}, nil } } return [2]int{}, fmt.Errorf("no version in %q", strings.TrimSpace(out)) } // TooOld is whether v is older than MinVersion. func TooOld(v [2]int) bool { return v[0] < MinVersion[0] || (v[0] == MinVersion[0] && v[1] < MinVersion[1]) }