From d439d507d746da83e465bebf284a082bbc041653 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Wed, 11 Nov 2026 12:20:31 +0100 Subject: [PATCH] Slurm beside llmgw, P1: its settings and the files a controller runs on slurm.{enabled,cluster_name,controller,nodes,lent,partition} in the cluster's settings, checked; internal/slurm writes slurm.conf (configless), gres.conf and cgroup.conf from the nodes' slurmd -C lines and GPUs, derives the munge key from a cluster secret, and reads scontrol and version output. Co-Authored-By: Claude Opus 5.5 --- BACKLOG.md | 16 ++ docs/reference.md | 60 ++++++ internal/config/config.go | 7 + internal/config/slurm.go | 95 +++++++++ internal/config/slurm_test.go | 36 ++++ internal/docgen/reference_gen.go | 9 + internal/docgen/render.go | 1 + internal/slurm/slurm.go | 323 +++++++++++++++++++++++++++++++ internal/slurm/slurm_test.go | 151 +++++++++++++++ 9 files changed, 698 insertions(+) create mode 100644 internal/config/slurm.go create mode 100644 internal/config/slurm_test.go create mode 100644 internal/slurm/slurm.go create mode 100644 internal/slurm/slurm_test.go diff --git a/BACKLOG.md b/BACKLOG.md index 1058091..16c467c 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -2013,6 +2013,22 @@ Recommendation: keep F4 in llmgw for small single-card adapters tied to KB grant build only A's first step (N GPUs on one node, placed by the control plane, with resume) if customers ask for mid-size jobs without wanting Slurm. +**B taken (2026-11-11).** Design: the cluster's settings say it (`slurm: enabled, cluster_name, controller, nodes, +lent, partition`); the leader tells each named node's agent (root) what to be, the agents install and run it. +- The controller is an llmgw node (its agent runs slurmctld); `nodes` run slurmd configless + (`--conf-server`), so slurm.conf, gres.conf and cgroup.conf live only on the controller. +- The munge key is derived from the cluster secret (HKDF), so nothing new is stored or copied by hand. +- Each node reports `slurmd -C`; the leader writes slurm.conf from those lines, its mgmt address and its GPUs + (`Gres=gpu::N`, gres.conf with the device files), and sends it to the controller again when one changes. +- A node in `nodes` belongs to llmgw until it is in `lent`: drained in Slurm (reason "llmgw: serving models"). + Lent: llmgw drains it as `ctl drain` does (kept in force each tick, as facility holds are, so a new leader + keeps it) and, once no replica is left, its agent resumes it in Slurm. Taken out of `lent`: the agent drains + it in Slurm, reports the jobs left, and when none are the leader makes it ready for llmgw again. +- Phases: P1 the `slurm` package (conf, key, parsers) and settings; P2 the agent side (proto, install, munge, + slurmd/slurmctld, status); P3 the leader side (conf from reports, lend/reclaim); P4 `ctl slurm` + (status, lend, reclaim) and a TUI screen; P5 `ctl train import` of a Slurm job's output into the mirror. + Later: a backup slurmctld on a second node, accounting (slurmdbd on the Patroni PG), users from LDAP/AD. + ## D. Facts worth knowing before starting - Admin API: `127.0.0.1:9090`, behind the token in diff --git a/docs/reference.md b/docs/reference.md index 1888d4a..d582b95 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -376,6 +376,7 @@ rather than repeated. | `mail` | `mail.Config` | [Server](#server) | | `serving` | `ServingSettings` | [Server](#server) | | `firewall` | `FirewallSettings` | [Server](#server) | +| `slurm` | `SlurmSettings` | [Server](#server) | | `presets` | `PresetSettings` | [Server](#server) | | `on_demand` | `OnDemandSettings` | [Server](#server) | | `agents` | `AgentSettings` | [Server](#server) | @@ -401,6 +402,12 @@ rather than repeated. | `timeout` | `time.Duration` | [Service](#service) | | `stopped` | `bool` | [ServingSettings](#servingsettings) | | `bin` | `string` | [SimConfig](#simconfig) | +| `enabled` | `bool` | [SlurmSettings](#slurmsettings) | +| `cluster_name` | `string` | [SlurmSettings](#slurmsettings) | +| `controller` | `string` | [SlurmSettings](#slurmsettings) | +| `nodes` | `[]string` | [SlurmSettings](#slurmsettings) | +| `lent` | `[]string` | [SlurmSettings](#slurmsettings) | +| `partition` | `string` | [SlurmSettings](#slurmsettings) | | `dir` | `string` | [StrataConfig](#strataconfig) | | `python` | `string` | [StrataConfig](#strataconfig) | | `engine` | `string` | [StrataConfig](#strataconfig) | @@ -3279,6 +3286,15 @@ Firewall turns llmgw's firewall on every machine on or off. Takes a [FirewallSettings](#firewallsettings) block. +### `slurm` + +`SlurmSettings` + +Slurm is a Slurm cluster run beside llmgw on some of its nodes, and +which of them it may use now. + +Takes a [SlurmSettings](#slurmsettings) block. + ### `presets` `PresetSettings` @@ -3480,6 +3496,50 @@ entries ask. In force on every control plane within seconds. Bin is the llmgw binary itself; the engine is one of its subcommands. Empty means whatever started this agent, which is right almost always. +## SlurmSettings + +### `enabled` + +`bool` + +Enabled runs Slurm on the nodes below. False stops llmgw keeping it; +what was installed is left installed, and every node is llmgw's. + +### `cluster_name` + +`string` + +ClusterName is Slurm's ClusterName. Empty is "llmgw". + +### `controller` + +`string` + +Controller is the llmgw node whose agent runs slurmctld. It needs no +GPU, and need not be one of Nodes. + +### `nodes` + +`[]string` + +Nodes are the llmgw nodes that run slurmd. Each stays llmgw's, drained +in Slurm, until it is also in Lent. + +### `lent` + +`[]string` + +Lent are the nodes of Nodes Slurm may run jobs on now. A node put here +is drained of llmgw's replicas first, as `ctl drain` does, and resumed +in Slurm once none is left; a node taken out is drained in Slurm and +given back to llmgw when its last job ends. + +### `partition` + +`string` + +Partition is the one partition every node is in. Empty is "gpu". + ## StrataConfig ### `dir` diff --git a/internal/config/config.go b/internal/config/config.go index f56c46f..b23d258 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -158,6 +158,10 @@ type Server struct { // Firewall turns llmgw's firewall on every machine on or off. Firewall FirewallSettings `koanf:"firewall"` + // Slurm is a Slurm cluster run beside llmgw on some of its nodes, and + // which of them it may use now. + Slurm SlurmSettings `koanf:"slurm"` + // Presets is how the models that come with llmgw reach the catalog. Presets PresetSettings `koanf:"presets"` @@ -960,6 +964,9 @@ func (s Server) Validate() error { if _, err := s.Facility.Policy(); err != nil { return err } + if err := s.Slurm.Check(); err != nil { + return err + } if s.Control.Timezone != "" { if _, err := time.LoadLocation(s.Control.Timezone); err != nil { return fmt.Errorf("control.timezone %q: %w", s.Control.Timezone, err) diff --git a/internal/config/slurm.go b/internal/config/slurm.go new file mode 100644 index 0000000..938ba56 --- /dev/null +++ b/internal/config/slurm.go @@ -0,0 +1,95 @@ +package config + +import ( + "fmt" + "regexp" + "slices" +) + +// SlurmSettings is a Slurm cluster run beside llmgw on some of its nodes +// (BACKLOG P), for training and other batch work llmgw does not schedule. +// The leader tells each named node's agent what to run; the agents install +// it from the machine's package sources and keep it going. +type SlurmSettings struct { + // Enabled runs Slurm on the nodes below. False stops llmgw keeping it; + // what was installed is left installed, and every node is llmgw's. + Enabled bool `koanf:"enabled"` + + // ClusterName is Slurm's ClusterName. Empty is "llmgw". + ClusterName string `koanf:"cluster_name"` + + // Controller is the llmgw node whose agent runs slurmctld. It needs no + // GPU, and need not be one of Nodes. + Controller string `koanf:"controller"` + + // Nodes are the llmgw nodes that run slurmd. Each stays llmgw's, drained + // in Slurm, until it is also in Lent. + Nodes []string `koanf:"nodes"` + + // Lent are the nodes of Nodes Slurm may run jobs on now. A node put here + // is drained of llmgw's replicas first, as `ctl drain` does, and resumed + // in Slurm once none is left; a node taken out is drained in Slurm and + // given back to llmgw when its last job ends. + Lent []string `koanf:"lent"` + + // Partition is the one partition every node is in. Empty is "gpu". + Partition string `koanf:"partition"` +} + +// slurmName is what Slurm takes as a node, cluster or partition name. +var slurmName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]{0,62}$`) + +// Cluster is ClusterName or its default. +func (s SlurmSettings) Cluster() string { + if s.ClusterName == "" { + return "llmgw" + } + return s.ClusterName +} + +// PartitionName is Partition or its default. +func (s SlurmSettings) PartitionName() string { + if s.Partition == "" { + return "gpu" + } + return s.Partition +} + +// IsLent is whether Slurm may use the node now. +func (s SlurmSettings) IsLent(node string) bool { + return s.Enabled && slices.Contains(s.Nodes, node) && slices.Contains(s.Lent, node) +} + +// Check refuses settings Slurm could not run on. +func (s SlurmSettings) Check() error { + if !s.Enabled { + return nil + } + if s.Controller == "" { + return fmt.Errorf("slurm.controller is empty: name the node that runs slurmctld") + } + if !slurmName.MatchString(s.Controller) { + return fmt.Errorf("slurm.controller %q is not a name Slurm takes", s.Controller) + } + for _, n := range []struct{ key, v string }{{"slurm.cluster_name", s.ClusterName}, {"slurm.partition", s.Partition}} { + if n.v != "" && !slurmName.MatchString(n.v) { + return fmt.Errorf("%s %q is not a name Slurm takes: letters, digits, _ . -", n.key, n.v) + } + } + seen := map[string]bool{} + for i, n := range s.Nodes { + if !slurmName.MatchString(n) { + return fmt.Errorf("slurm.nodes[%d] %q is not a name Slurm takes", i, n) + } + if seen[n] { + return fmt.Errorf("slurm.nodes names %s twice", n) + } + seen[n] = true + } + for i, n := range s.Lent { + if !seen[n] { + return fmt.Errorf("slurm.lent[%d] %q is not in slurm.nodes: only a node that runs slurmd can be lent", i, n) + } + } + return nil +} diff --git a/internal/config/slurm_test.go b/internal/config/slurm_test.go new file mode 100644 index 0000000..c4fdf9f --- /dev/null +++ b/internal/config/slurm_test.go @@ -0,0 +1,36 @@ +package config + +import ( + "strings" + "testing" +) + +func TestSlurmCheck(t *testing.T) { + ok := SlurmSettings{Enabled: true, Controller: "node06", Nodes: []string{"cuda01", "site2"}, Lent: []string{"cuda01"}} + if err := ok.Check(); err != nil { + t.Fatal(err) + } + if !ok.IsLent("cuda01") || ok.IsLent("site2") || ok.IsLent("node06") { + t.Error("IsLent is wrong") + } + if ok.Cluster() != "llmgw" || ok.PartitionName() != "gpu" { + t.Error("defaults are wrong") + } + if (SlurmSettings{Lent: []string{"x"}}).Check() != nil { + t.Error("settings that are off were checked") + } + for want, s := range map[string]SlurmSettings{ + "controller is empty": {Enabled: true}, + "not a name": {Enabled: true, Controller: "bad name"}, + "twice": {Enabled: true, Controller: "c", Nodes: []string{"a", "a"}}, + "not in slurm.nodes": {Enabled: true, Controller: "c", Nodes: []string{"a"}, Lent: []string{"b"}}, + "slurm.partition": {Enabled: true, Controller: "c", Partition: "a b"}, + } { + if err := s.Check(); err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("%+v: got %v, want %q", s, err, want) + } + } + if (SlurmSettings{Enabled: false, Lent: []string{"a"}}).IsLent("a") { + t.Error("a node is lent with Slurm off") + } +} diff --git a/internal/docgen/reference_gen.go b/internal/docgen/reference_gen.go index c7a28b9..0e8abe5 100644 --- a/internal/docgen/reference_gen.go +++ b/internal/docgen/reference_gen.go @@ -480,6 +480,7 @@ var Reference = []Block{ {Key: "mail", Type: "mail.Config", Doc: "Mail is where the gateway's few emails go out from: a link to set a\nforgotten password. Unset, none is sent and people ask an operator.", Nested: "mail.Config"}, {Key: "serving", Type: "ServingSettings", Doc: "Serving stops and starts the whole cluster's service (llmgw stop, llmgw\nstart).", Nested: "config.ServingSettings"}, {Key: "firewall", Type: "FirewallSettings", Doc: "Firewall turns llmgw's firewall on every machine on or off.", Nested: "config.FirewallSettings"}, + {Key: "slurm", Type: "SlurmSettings", Doc: "Slurm is a Slurm cluster run beside llmgw on some of its nodes, and\nwhich of them it may use now.", Nested: "config.SlurmSettings"}, {Key: "presets", Type: "PresetSettings", Doc: "Presets is how the models that come with llmgw reach the catalog.", Nested: "config.PresetSettings"}, {Key: "on_demand", Type: "OnDemandSettings", Doc: "OnDemand is how models that load when asked for take turns on the\ncards.", Nested: "config.OnDemandSettings"}, {Key: "agents", Type: "AgentSettings", Doc: "Agents are the nodes' agent settings: agent.yaml's keys but those an\nagent needs to start, for every node (default) and for one\n(nodes.). Sent to the agents as they change; an agent whose\nsettings changed starts again on them, its engines serving\nthroughout.", Nested: "config.AgentSettings"}, @@ -512,6 +513,14 @@ var Reference = []Block{ {Package: "engines", Name: "SimConfig", Title: "SimConfig", Doc: "", Fields: []Field{ {Key: "bin", Type: "string", Doc: "Bin is the llmgw binary itself; the engine is one of its subcommands.\nEmpty means whatever started this agent, which is right almost always.", Nested: ""}, }}, + {Package: "config", Name: "SlurmSettings", Title: "SlurmSettings", Doc: "", Fields: []Field{ + {Key: "enabled", Type: "bool", Doc: "Enabled runs Slurm on the nodes below. False stops llmgw keeping it;\nwhat was installed is left installed, and every node is llmgw's.", Nested: ""}, + {Key: "cluster_name", Type: "string", Doc: "ClusterName is Slurm's ClusterName. Empty is \"llmgw\".", Nested: ""}, + {Key: "controller", Type: "string", Doc: "Controller is the llmgw node whose agent runs slurmctld. It needs no\nGPU, and need not be one of Nodes.", Nested: ""}, + {Key: "nodes", Type: "[]string", Doc: "Nodes are the llmgw nodes that run slurmd. Each stays llmgw's, drained\nin Slurm, until it is also in Lent.", Nested: ""}, + {Key: "lent", Type: "[]string", Doc: "Lent are the nodes of Nodes Slurm may run jobs on now. A node put here\nis drained of llmgw's replicas first, as `ctl drain` does, and resumed\nin Slurm once none is left; a node taken out is drained in Slurm and\ngiven back to llmgw when its last job ends.", Nested: ""}, + {Key: "partition", Type: "string", Doc: "Partition is the one partition every node is in. Empty is \"gpu\".", Nested: ""}, + }}, {Package: "engines", Name: "StrataConfig", Title: "StrataConfig", Doc: "", Fields: []Field{ {Key: "dir", Type: "string", Doc: "Dir is Strata's source directory, where setup.py and serve/ are.", Nested: ""}, {Key: "python", Type: "string", Doc: "Python is the interpreter of its environment; Dir's .venv when unset.", Nested: ""}, diff --git a/internal/docgen/render.go b/internal/docgen/render.go index 18afb32..e7583c6 100644 --- a/internal/docgen/render.go +++ b/internal/docgen/render.go @@ -41,6 +41,7 @@ var Sources = []string{ "internal/config/serving.go", "internal/config/presets.go", "internal/config/ondemand.go", + "internal/config/slurm.go", "internal/config/agents.go", "internal/mail/mail.go", "internal/notify/notify.go", diff --git a/internal/slurm/slurm.go b/internal/slurm/slurm.go new file mode 100644 index 0000000..0e80cb2 --- /dev/null +++ b/internal/slurm/slurm.go @@ -0,0 +1,323 @@ +// Package slurm is what llmgw knows of Slurm (BACKLOG P): the files a +// controller runs on, written from what llmgw knows of its nodes, the munge +// key every machine shares, and reading what slurmd and scontrol say. +// +// Nothing here runs anything. The agent installs and starts the daemons; the +// leader decides what each node is and writes the files with this package. +package slurm + +import ( + "crypto/hkdf" + "crypto/sha256" + "fmt" + "regexp" + "slices" + "strconv" + "strings" +) + +// Paths are where the files are on every distribution llmgw installs on: +// Debian and Ubuntu since Slurm 20.11, and EPEL. +const ( + ConfDir = "/etc/slurm" + MungeKeyPath = "/etc/munge/munge.key" + StateDir = "/var/spool/slurmctld" + SpoolDir = "/var/spool/slurmd" + LogDir = "/var/log/slurm" + // CtldPort is slurmctld's, which configless slurmd fetch their files + // from. + CtldPort = 6817 +) + +// MinVersion is the oldest Slurm llmgw runs: the first to put jobs in +// cgroup v2, which every distribution llmgw supports now mounts alone. +var MinVersion = [2]int{22, 5} + +// MungeKeyLen is the length munge's own tool writes. +const MungeKeyLen = 1024 + +// MungeKey is the key every machine of the Slurm cluster shares, derived +// from a cluster secret, so a control plane that takes over writes the same +// one and nobody copies it by hand. +func MungeKey(secret string) ([]byte, error) { + if len(secret) < 32 { + return nil, fmt.Errorf("the secret a munge key is made from is too short") + } + return hkdf.Key(sha256.New, []byte(secret), nil, "llmgw slurm munge key v1", MungeKeyLen) +} + +// Hardware is a node's line from `slurmd -C`: how slurmd counts it, which is +// what slurm.conf must say or the controller refuses the node. +type Hardware struct { + CPUs int + Boards int + SocketsPerBoard int + CoresPerSocket int + ThreadsPerCore int + // RealMemory is in MiB. + RealMemory int64 +} + +// ParseHardware reads `slurmd -C`. +func ParseHardware(out string) (Hardware, error) { + var h Hardware + for _, line := range strings.Split(out, "\n") { + if !strings.HasPrefix(strings.TrimSpace(line), "NodeName=") { + continue + } + for _, f := range strings.Fields(line) { + k, v, _ := strings.Cut(f, "=") + n, err := strconv.ParseInt(v, 10, 64) + if err != nil { + continue + } + switch k { + case "CPUs": + h.CPUs = int(n) + case "Boards": + h.Boards = int(n) + case "SocketsPerBoard": + h.SocketsPerBoard = int(n) + case "CoresPerSocket": + h.CoresPerSocket = int(n) + case "ThreadsPerCore": + h.ThreadsPerCore = int(n) + case "RealMemory": + h.RealMemory = n + } + } + break + } + if h.CPUs <= 0 || h.RealMemory <= 0 { + return Hardware{}, fmt.Errorf("slurmd -C said no CPUs or memory: %q", strings.TrimSpace(out)) + } + return h, nil +} + +// GPU is one of a node's cards. +type GPU struct { + // Minor is N of /dev/nvidiaN. + Minor int + // Name is the driver's, as nvidia-smi gives it. + Name string +} + +// Node is one machine slurmd runs on. +type Node struct { + Name string + // Addr is where slurmctld reaches it: its llmgw management address. + Addr string + Hardware + GPUs []GPU +} + +// Cluster is everything slurm.conf says. +type Cluster struct { + Name string + Controller string + ControllerAddr string + Partition string + // Nodes without hardware (not reported yet) are left out of the files + // until they are. + Nodes []Node +} + +// gpuModel is a word that names a card's model: A100, H100, V100, T4, L40S. +var gpuModel = regexp.MustCompile(`^[a-z]{1,2}\d{1,4}[a-z]?$`) + +// GPUType is a card's name as a Slurm GRES type: "NVIDIA A100-SXM4-80GB" is +// a100, "NVIDIA GeForce RTX 3090" rtx3090, "Tesla T4" t4. A name it cannot +// read is "gpu". +func GPUType(name string) string { + words := strings.Fields(strings.ToLower(strings.NewReplacer("-", " ", "_", " ").Replace(name))) + for i, w := range words { + if (w == "rtx" || w == "gtx") && i+1 < len(words) { + return w + words[i+1] + } + if gpuModel.MatchString(w) { + return w + } + } + return "gpu" +} + +// gpuGroups are a node's cards by type, the types sorted. +func gpuGroups(gs []GPU) ([]string, map[string][]int) { + by := map[string][]int{} + for _, g := range gs { + t := GPUType(g.Name) + by[t] = append(by[t], g.Minor) + } + types := make([]string, 0, len(by)) + for t, ms := range by { + slices.Sort(ms) + types = append(types, t) + } + slices.Sort(types) + return types, by +} + +// memoryKept is what is left out of a node's RealMemory: slurmd -C reads +// what the kernel says now, which is a little less after a reboot, and a +// node whose memory is below what slurm.conf says is refused. +const memoryKept = 0.97 + +func (c Cluster) ready() []Node { + var out []Node + for _, n := range c.Nodes { + if n.CPUs > 0 && n.RealMemory > 0 { + out = append(out, n) + } + } + slices.SortFunc(out, func(a, b Node) int { return strings.Compare(a.Name, b.Name) }) + return out +} + +const header = "# Written by llmgw (BACKLOG P), on the node that runs slurmctld; the nodes read it\n" + + "# from there. Change it through the cluster's settings (slurm.*), not here.\n" + +// SlurmConf is slurm.conf. +func (c Cluster) SlurmConf() string { + var b strings.Builder + b.WriteString(header) + fmt.Fprintf(&b, "ClusterName=%s\n", c.Name) + fmt.Fprintf(&b, "SlurmctldHost=%s(%s)\n", c.Controller, c.ControllerAddr) + b.WriteString(`SlurmctldParameters=enable_configless +SlurmctldPort=6817 +SlurmdPort=6818 +AuthType=auth/munge +CredType=cred/munge +SlurmUser=slurm +SlurmdUser=root +StateSaveLocation=` + StateDir + ` +SlurmdSpoolDir=` + SpoolDir + ` +SlurmctldLogFile=` + LogDir + `/slurmctld.log +SlurmdLogFile=` + LogDir + `/slurmd.log +SlurmctldPidFile=/run/slurmctld.pid +SlurmdPidFile=/run/slurmd.pid +ProctrackType=proctrack/cgroup +TaskPlugin=task/cgroup,task/affinity +JobAcctGatherType=jobacct_gather/cgroup +SelectType=select/cons_tres +SelectTypeParameters=CR_Core_Memory +SchedulerType=sched/backfill +GresTypes=gpu +MpiDefault=none +# A node that went down and comes back is taken again; one llmgw drained +# stays drained until llmgw resumes it. +ReturnToService=2 +SlurmctldTimeout=120 +SlurmdTimeout=300 +`) + nodes := c.ready() + names := make([]string, 0, len(nodes)) + for _, n := range nodes { + names = append(names, n.Name) + fmt.Fprintf(&b, "NodeName=%s NodeAddr=%s CPUs=%d", n.Name, n.Addr, n.CPUs) + if n.Boards > 0 && n.SocketsPerBoard > 0 && n.CoresPerSocket > 0 && n.ThreadsPerCore > 0 { + fmt.Fprintf(&b, " Boards=%d SocketsPerBoard=%d CoresPerSocket=%d ThreadsPerCore=%d", + n.Boards, n.SocketsPerBoard, n.CoresPerSocket, n.ThreadsPerCore) + } + fmt.Fprintf(&b, " RealMemory=%d", int64(float64(n.RealMemory)*memoryKept)) + if types, by := gpuGroups(n.GPUs); len(types) > 0 { + gres := make([]string, 0, len(types)) + for _, t := range types { + gres = append(gres, fmt.Sprintf("gpu:%s:%d", t, len(by[t]))) + } + fmt.Fprintf(&b, " Gres=%s", strings.Join(gres, ",")) + } + b.WriteString(" State=UNKNOWN\n") + } + if len(names) > 0 { + fmt.Fprintf(&b, "PartitionName=%s Nodes=%s Default=YES MaxTime=INFINITE State=UP\n", + c.Partition, strings.Join(names, ",")) + } + return b.String() +} + +// GresConf is gres.conf: each node's cards by device file, so a job is given +// the cards it asked for and kept off the others. +func (c Cluster) GresConf() string { + var b strings.Builder + b.WriteString(header) + for _, n := range c.ready() { + types, by := gpuGroups(n.GPUs) + for _, t := range types { + files := make([]string, 0, len(by[t])) + for _, m := range by[t] { + files = append(files, fmt.Sprintf("/dev/nvidia%d", m)) + } + fmt.Fprintf(&b, "NodeName=%s Name=gpu Type=%s File=%s\n", n.Name, t, strings.Join(files, ",")) + } + } + return b.String() +} + +// CgroupConf is cgroup.conf: a job kept to its cores, memory and cards. +func (c Cluster) CgroupConf() string { + return header + `CgroupPlugin=autodetect +ConstrainCores=yes +ConstrainDevices=yes +ConstrainRAMSpace=yes +` +} + +// Files are the controller's files, by name in ConfDir. +func (c Cluster) Files() map[string]string { + return map[string]string{"slurm.conf": c.SlurmConf(), "gres.conf": c.GresConf(), "cgroup.conf": c.CgroupConf()} +} + +// NodeState is what `scontrol show node -o NAME` says of a node. +type NodeState struct { + // State is Slurm's, with its flags: IDLE, ALLOCATED, MIXED+DRAIN, + // DOWN+NOT_RESPONDING. + State string + Reason string +} + +// Drained is whether the controller gives the node nothing new. +func (s NodeState) Drained() bool { + return strings.Contains(s.State, "DRAIN") || strings.Contains(s.State, "DOWN") +} + +// ParseNodeState reads `scontrol show node -o NAME`, one line of key=value. +func ParseNodeState(out string) (NodeState, error) { + line := strings.TrimSpace(out) + if !strings.HasPrefix(line, "NodeName=") { + return NodeState{}, fmt.Errorf("scontrol said %q", line) + } + var s NodeState + if i := strings.Index(line, " State="); i >= 0 { + s.State, _, _ = strings.Cut(line[i+len(" State="):], " ") + } + // Reason is last but for its [user@time] and may have spaces. + if i := strings.Index(line, " Reason="); i >= 0 { + r := line[i+len(" Reason="):] + if j := strings.LastIndex(r, " ["); j >= 0 { + r = r[:j] + } + s.Reason = strings.TrimSpace(r) + } + return s, nil +} + +// ParseVersion reads "slurm 23.11.4" or "slurm-wlm 21.08.5" as major, minor. +func ParseVersion(out string) ([2]int, error) { + for _, f := range strings.Fields(out) { + parts := strings.Split(f, ".") + if len(parts) < 2 { + continue + } + major, err1 := strconv.Atoi(parts[0]) + minor, err2 := strconv.Atoi(parts[1]) + if err1 == nil && err2 == nil { + return [2]int{major, minor}, nil + } + } + return [2]int{}, fmt.Errorf("no version in %q", strings.TrimSpace(out)) +} + +// TooOld is whether v is older than MinVersion. +func TooOld(v [2]int) bool { + return v[0] < MinVersion[0] || (v[0] == MinVersion[0] && v[1] < MinVersion[1]) +} diff --git a/internal/slurm/slurm_test.go b/internal/slurm/slurm_test.go new file mode 100644 index 0000000..0138a11 --- /dev/null +++ b/internal/slurm/slurm_test.go @@ -0,0 +1,151 @@ +package slurm + +import ( + "bytes" + "strings" + "testing" +) + +func TestMungeKey(t *testing.T) { + s := strings.Repeat("ab", 32) + a, err := MungeKey(s) + if err != nil { + t.Fatal(err) + } + b, _ := MungeKey(s) + if len(a) != MungeKeyLen || !bytes.Equal(a, b) { + t.Fatalf("a key is %d bytes and the same each time: got %d, equal %v", MungeKeyLen, len(a), bytes.Equal(a, b)) + } + c, _ := MungeKey(strings.Repeat("cd", 32)) + if bytes.Equal(a, c) { + t.Fatal("two secrets made one key") + } + if _, err := MungeKey("short"); err == nil { + t.Fatal("a short secret was taken") + } +} + +func TestParseHardware(t *testing.T) { + out := "NodeName=cuda01 CPUs=16 Boards=1 SocketsPerBoard=2 CoresPerSocket=4 ThreadsPerCore=2 RealMemory=64302\n" + + "UpTime=3-01:02:03\n" + h, err := ParseHardware(out) + if err != nil { + t.Fatal(err) + } + want := Hardware{CPUs: 16, Boards: 1, SocketsPerBoard: 2, CoresPerSocket: 4, ThreadsPerCore: 2, RealMemory: 64302} + if h != want { + t.Fatalf("got %+v", h) + } + if _, err := ParseHardware("slurmd: error: something\n"); err == nil { + t.Fatal("an error was read as hardware") + } +} + +func TestGPUType(t *testing.T) { + for name, want := range map[string]string{ + "NVIDIA A100-SXM4-80GB": "a100", + "NVIDIA H100 80GB HBM3": "h100", + "Tesla V100-SXM2-32GB": "v100", + "Tesla T4": "t4", + "NVIDIA GeForce RTX 3090": "rtx3090", + "NVIDIA RTX A6000": "rtxa6000", + "NVIDIA L40S": "l40s", + "Quadro RTX 8000": "rtx8000", + "NVIDIA GeForce GTX 1080": "gtx1080", + "": "gpu", + "Some Accelerator Unknown": "gpu", + } { + if got := GPUType(name); got != want { + t.Errorf("GPUType(%q) = %q, want %q", name, got, want) + } + } +} + +func testCluster() Cluster { + return Cluster{Name: "llmgw", Controller: "node06", ControllerAddr: "10.0.0.146", Partition: "gpu", + Nodes: []Node{ + {Name: "site2", Addr: "10.1.0.5", + Hardware: Hardware{CPUs: 32, Boards: 1, SocketsPerBoard: 2, CoresPerSocket: 8, ThreadsPerCore: 2, RealMemory: 100000}, + GPUs: []GPU{{Minor: 2, Name: "Tesla T4"}, {Minor: 0, Name: "Tesla V100-SXM2-32GB"}, {Minor: 1, Name: "Tesla V100-SXM2-32GB"}}}, + {Name: "cuda01", Addr: "10.0.0.147", Hardware: Hardware{CPUs: 16, RealMemory: 64000}, + GPUs: []GPU{{Minor: 0, Name: "NVIDIA GeForce RTX 3090"}}}, + {Name: "waiting", Addr: "10.0.0.9"}, + }} +} + +func TestSlurmConf(t *testing.T) { + conf := testCluster().SlurmConf() + for _, want := range []string{ + "ClusterName=llmgw\n", + "SlurmctldHost=node06(10.0.0.146)\n", + "SlurmctldParameters=enable_configless\n", + "NodeName=cuda01 NodeAddr=10.0.0.147 CPUs=16 RealMemory=62080 Gres=gpu:rtx3090:1 State=UNKNOWN\n", + "NodeName=site2 NodeAddr=10.1.0.5 CPUs=32 Boards=1 SocketsPerBoard=2 CoresPerSocket=8 ThreadsPerCore=2 RealMemory=97000 Gres=gpu:t4:1,gpu:v100:2 State=UNKNOWN\n", + "PartitionName=gpu Nodes=cuda01,site2 Default=YES", + } { + if !strings.Contains(conf, want) { + t.Errorf("slurm.conf has no %q:\n%s", want, conf) + } + } + if strings.Contains(conf, "waiting") { + t.Error("a node that has not reported its hardware is in slurm.conf") + } + if strings.Index(conf, "NodeName=cuda01") > strings.Index(conf, "NodeName=site2") { + t.Error("nodes are not in order") + } +} + +func TestSlurmConfNoNodes(t *testing.T) { + c := testCluster() + c.Nodes = nil + if strings.Contains(c.SlurmConf(), "PartitionName") { + t.Error("a partition with no nodes would stop slurmctld starting") + } +} + +func TestGresConf(t *testing.T) { + g := testCluster().GresConf() + for _, want := range []string{ + "NodeName=cuda01 Name=gpu Type=rtx3090 File=/dev/nvidia0\n", + "NodeName=site2 Name=gpu Type=t4 File=/dev/nvidia2\n", + "NodeName=site2 Name=gpu Type=v100 File=/dev/nvidia0,/dev/nvidia1\n", + } { + if !strings.Contains(g, want) { + t.Errorf("gres.conf has no %q:\n%s", want, g) + } + } +} + +func TestParseNodeState(t *testing.T) { + out := "NodeName=cuda01 Arch=x86_64 CoresPerSocket=8 CPUAlloc=0 State=IDLE+DRAIN ThreadsPerCore=2 " + + "Partitions=gpu Reason=llmgw: serving models [root@2026-11-11T10:00:00]\n" + s, err := ParseNodeState(out) + if err != nil { + t.Fatal(err) + } + if s.State != "IDLE+DRAIN" || s.Reason != "llmgw: serving models" || !s.Drained() { + t.Fatalf("got %+v", s) + } + s, _ = ParseNodeState("NodeName=cuda01 State=MIXED Partitions=gpu\n") + if s.Drained() || s.Reason != "" { + t.Fatalf("got %+v", s) + } + if _, err := ParseNodeState("Node cuda02 not found\n"); err == nil { + t.Fatal("an unknown node was read") + } +} + +func TestParseVersion(t *testing.T) { + for out, want := range map[string][2]int{"slurm 23.11.4\n": {23, 11}, "slurm-wlm 21.08.5": {21, 8}, "slurm 22.05.9": {22, 5}} { + v, err := ParseVersion(out) + if err != nil || v != want { + t.Errorf("ParseVersion(%q) = %v, %v", out, v, err) + } + } + if !TooOld([2]int{21, 8}) || TooOld([2]int{22, 5}) || TooOld([2]int{23, 2}) { + t.Error("TooOld is wrong") + } + if _, err := ParseVersion("command not found"); err == nil { + t.Error("no version was read as one") + } +}