Slurm beside llmgw, P1: its settings and the files a controller runs on
slurm.{enabled,cluster_name,controller,nodes,lent,partition} in the cluster's
settings, checked; internal/slurm writes slurm.conf (configless), gres.conf
and cgroup.conf from the nodes' slurmd -C lines and GPUs, derives the munge
key from a cluster secret, and reads scontrol and version output.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
89767c8be1
commit
d439d507d7
9 files changed
+698
No files matched your search
@@ -158,6 +158,10 @@ type Server struct {
|
||||
// Firewall turns llmgw's firewall on every machine on or off.
|
||||
Firewall FirewallSettings `koanf:"firewall"`
|
||||
|
||||
// Slurm is a Slurm cluster run beside llmgw on some of its nodes, and
|
||||
// which of them it may use now.
|
||||
Slurm SlurmSettings `koanf:"slurm"`
|
||||
|
||||
// Presets is how the models that come with llmgw reach the catalog.
|
||||
Presets PresetSettings `koanf:"presets"`
|
||||
|
||||
@@ -960,6 +964,9 @@ func (s Server) Validate() error {
|
||||
if _, err := s.Facility.Policy(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.Slurm.Check(); err != nil {
|
||||
return err
|
||||
}
|
||||
if s.Control.Timezone != "" {
|
||||
if _, err := time.LoadLocation(s.Control.Timezone); err != nil {
|
||||
return fmt.Errorf("control.timezone %q: %w", s.Control.Timezone, err)
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"slices"
|
||||
)
|
||||
|
||||
// SlurmSettings is a Slurm cluster run beside llmgw on some of its nodes
|
||||
// (BACKLOG P), for training and other batch work llmgw does not schedule.
|
||||
// The leader tells each named node's agent what to run; the agents install
|
||||
// it from the machine's package sources and keep it going.
|
||||
type SlurmSettings struct {
|
||||
// Enabled runs Slurm on the nodes below. False stops llmgw keeping it;
|
||||
// what was installed is left installed, and every node is llmgw's.
|
||||
Enabled bool `koanf:"enabled"`
|
||||
|
||||
// ClusterName is Slurm's ClusterName. Empty is "llmgw".
|
||||
ClusterName string `koanf:"cluster_name"`
|
||||
|
||||
// Controller is the llmgw node whose agent runs slurmctld. It needs no
|
||||
// GPU, and need not be one of Nodes.
|
||||
Controller string `koanf:"controller"`
|
||||
|
||||
// Nodes are the llmgw nodes that run slurmd. Each stays llmgw's, drained
|
||||
// in Slurm, until it is also in Lent.
|
||||
Nodes []string `koanf:"nodes"`
|
||||
|
||||
// Lent are the nodes of Nodes Slurm may run jobs on now. A node put here
|
||||
// is drained of llmgw's replicas first, as `ctl drain` does, and resumed
|
||||
// in Slurm once none is left; a node taken out is drained in Slurm and
|
||||
// given back to llmgw when its last job ends.
|
||||
Lent []string `koanf:"lent"`
|
||||
|
||||
// Partition is the one partition every node is in. Empty is "gpu".
|
||||
Partition string `koanf:"partition"`
|
||||
}
|
||||
|
||||
// slurmName is what Slurm takes as a node, cluster or partition name.
|
||||
var slurmName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]{0,62}$`)
|
||||
|
||||
// Cluster is ClusterName or its default.
|
||||
func (s SlurmSettings) Cluster() string {
|
||||
if s.ClusterName == "" {
|
||||
return "llmgw"
|
||||
}
|
||||
return s.ClusterName
|
||||
}
|
||||
|
||||
// PartitionName is Partition or its default.
|
||||
func (s SlurmSettings) PartitionName() string {
|
||||
if s.Partition == "" {
|
||||
return "gpu"
|
||||
}
|
||||
return s.Partition
|
||||
}
|
||||
|
||||
// IsLent is whether Slurm may use the node now.
|
||||
func (s SlurmSettings) IsLent(node string) bool {
|
||||
return s.Enabled && slices.Contains(s.Nodes, node) && slices.Contains(s.Lent, node)
|
||||
}
|
||||
|
||||
// Check refuses settings Slurm could not run on.
|
||||
func (s SlurmSettings) Check() error {
|
||||
if !s.Enabled {
|
||||
return nil
|
||||
}
|
||||
if s.Controller == "" {
|
||||
return fmt.Errorf("slurm.controller is empty: name the node that runs slurmctld")
|
||||
}
|
||||
if !slurmName.MatchString(s.Controller) {
|
||||
return fmt.Errorf("slurm.controller %q is not a name Slurm takes", s.Controller)
|
||||
}
|
||||
for _, n := range []struct{ key, v string }{{"slurm.cluster_name", s.ClusterName}, {"slurm.partition", s.Partition}} {
|
||||
if n.v != "" && !slurmName.MatchString(n.v) {
|
||||
return fmt.Errorf("%s %q is not a name Slurm takes: letters, digits, _ . -", n.key, n.v)
|
||||
}
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for i, n := range s.Nodes {
|
||||
if !slurmName.MatchString(n) {
|
||||
return fmt.Errorf("slurm.nodes[%d] %q is not a name Slurm takes", i, n)
|
||||
}
|
||||
if seen[n] {
|
||||
return fmt.Errorf("slurm.nodes names %s twice", n)
|
||||
}
|
||||
seen[n] = true
|
||||
}
|
||||
for i, n := range s.Lent {
|
||||
if !seen[n] {
|
||||
return fmt.Errorf("slurm.lent[%d] %q is not in slurm.nodes: only a node that runs slurmd can be lent", i, n)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSlurmCheck(t *testing.T) {
|
||||
ok := SlurmSettings{Enabled: true, Controller: "node06", Nodes: []string{"cuda01", "site2"}, Lent: []string{"cuda01"}}
|
||||
if err := ok.Check(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !ok.IsLent("cuda01") || ok.IsLent("site2") || ok.IsLent("node06") {
|
||||
t.Error("IsLent is wrong")
|
||||
}
|
||||
if ok.Cluster() != "llmgw" || ok.PartitionName() != "gpu" {
|
||||
t.Error("defaults are wrong")
|
||||
}
|
||||
if (SlurmSettings{Lent: []string{"x"}}).Check() != nil {
|
||||
t.Error("settings that are off were checked")
|
||||
}
|
||||
for want, s := range map[string]SlurmSettings{
|
||||
"controller is empty": {Enabled: true},
|
||||
"not a name": {Enabled: true, Controller: "bad name"},
|
||||
"twice": {Enabled: true, Controller: "c", Nodes: []string{"a", "a"}},
|
||||
"not in slurm.nodes": {Enabled: true, Controller: "c", Nodes: []string{"a"}, Lent: []string{"b"}},
|
||||
"slurm.partition": {Enabled: true, Controller: "c", Partition: "a b"},
|
||||
} {
|
||||
if err := s.Check(); err == nil || !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("%+v: got %v, want %q", s, err, want)
|
||||
}
|
||||
}
|
||||
if (SlurmSettings{Enabled: false, Lent: []string{"a"}}).IsLent("a") {
|
||||
t.Error("a node is lent with Slurm off")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user