Files
llmgw/internal/config/slurm.go
T
Ilker ManapandClaude Opus 5.5 d439d507d7 Slurm beside llmgw, P1: its settings and the files a controller runs on
slurm.{enabled,cluster_name,controller,nodes,lent,partition} in the cluster's
settings, checked; internal/slurm writes slurm.conf (configless), gres.conf
and cgroup.conf from the nodes' slurmd -C lines and GPUs, derives the munge
key from a cluster secret, and reads scontrol and version output.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-11 12:20:31 +01:00

96 lines
3.0 KiB
Go

package config
import (
"fmt"
"regexp"
"slices"
)
// SlurmSettings is a Slurm cluster run beside llmgw on some of its nodes
// (BACKLOG P), for training and other batch work llmgw does not schedule.
// The leader tells each named node's agent what to run; the agents install
// it from the machine's package sources and keep it going.
type SlurmSettings struct {
// Enabled runs Slurm on the nodes below. False stops llmgw keeping it;
// what was installed is left installed, and every node is llmgw's.
Enabled bool `koanf:"enabled"`
// ClusterName is Slurm's ClusterName. Empty is "llmgw".
ClusterName string `koanf:"cluster_name"`
// Controller is the llmgw node whose agent runs slurmctld. It needs no
// GPU, and need not be one of Nodes.
Controller string `koanf:"controller"`
// Nodes are the llmgw nodes that run slurmd. Each stays llmgw's, drained
// in Slurm, until it is also in Lent.
Nodes []string `koanf:"nodes"`
// Lent are the nodes of Nodes Slurm may run jobs on now. A node put here
// is drained of llmgw's replicas first, as `ctl drain` does, and resumed
// in Slurm once none is left; a node taken out is drained in Slurm and
// given back to llmgw when its last job ends.
Lent []string `koanf:"lent"`
// Partition is the one partition every node is in. Empty is "gpu".
Partition string `koanf:"partition"`
}
// slurmName is what Slurm takes as a node, cluster or partition name.
var slurmName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]{0,62}$`)
// Cluster is ClusterName or its default.
func (s SlurmSettings) Cluster() string {
if s.ClusterName == "" {
return "llmgw"
}
return s.ClusterName
}
// PartitionName is Partition or its default.
func (s SlurmSettings) PartitionName() string {
if s.Partition == "" {
return "gpu"
}
return s.Partition
}
// IsLent is whether Slurm may use the node now.
func (s SlurmSettings) IsLent(node string) bool {
return s.Enabled && slices.Contains(s.Nodes, node) && slices.Contains(s.Lent, node)
}
// Check refuses settings Slurm could not run on.
func (s SlurmSettings) Check() error {
if !s.Enabled {
return nil
}
if s.Controller == "" {
return fmt.Errorf("slurm.controller is empty: name the node that runs slurmctld")
}
if !slurmName.MatchString(s.Controller) {
return fmt.Errorf("slurm.controller %q is not a name Slurm takes", s.Controller)
}
for _, n := range []struct{ key, v string }{{"slurm.cluster_name", s.ClusterName}, {"slurm.partition", s.Partition}} {
if n.v != "" && !slurmName.MatchString(n.v) {
return fmt.Errorf("%s %q is not a name Slurm takes: letters, digits, _ . -", n.key, n.v)
}
}
seen := map[string]bool{}
for i, n := range s.Nodes {
if !slurmName.MatchString(n) {
return fmt.Errorf("slurm.nodes[%d] %q is not a name Slurm takes", i, n)
}
if seen[n] {
return fmt.Errorf("slurm.nodes names %s twice", n)
}
seen[n] = true
}
for i, n := range s.Lent {
if !seen[n] {
return fmt.Errorf("slurm.lent[%d] %q is not in slurm.nodes: only a node that runs slurmd can be lent", i, n)
}
}
return nil
}