CI / test (push) Successful in 9m30s
The choices screen has a Slurm part, and --slurm writes slurmctld on this machine's node and slurmd on every node, those that join later too, into server.yaml. slurm.all_nodes (ctl slurm setup --all-nodes) takes the nodes from the leader's inventory. No node is lent at install. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
116 lines
3.6 KiB
Go
116 lines
3.6 KiB
Go
package config
|
|
|
|
import (
|
|
"fmt"
|
|
"regexp"
|
|
"slices"
|
|
)
|
|
|
|
// SlurmSettings is a Slurm cluster run beside llmgw on some of its nodes
|
|
// (BACKLOG P), for training and other batch work llmgw does not schedule.
|
|
// The leader tells each named node's agent what to run; the agents install
|
|
// it from the machine's package sources and keep it going.
|
|
type SlurmSettings struct {
|
|
// Enabled runs Slurm on the nodes below. False stops llmgw keeping it;
|
|
// what was installed is left installed, and every node is llmgw's.
|
|
Enabled bool `koanf:"enabled"`
|
|
|
|
// ClusterName is Slurm's ClusterName. Empty is "llmgw".
|
|
ClusterName string `koanf:"cluster_name"`
|
|
|
|
// Controller is the llmgw node whose agent runs slurmctld. It needs no
|
|
// GPU, and need not be one of Nodes.
|
|
Controller string `koanf:"controller"`
|
|
|
|
// Nodes are the llmgw nodes that run slurmd. Each stays llmgw's, drained
|
|
// in Slurm, until it is also in Lent.
|
|
Nodes []string `koanf:"nodes"`
|
|
|
|
// AllNodes runs slurmd on every llmgw node, those that join later too,
|
|
// besides any named in Nodes. Each is llmgw's until it is lent, as any.
|
|
AllNodes bool `koanf:"all_nodes"`
|
|
|
|
// Lent are the nodes of Nodes Slurm may run jobs on now. A node put here
|
|
// is drained of llmgw's replicas first, as `ctl drain` does, and resumed
|
|
// in Slurm once none is left; a node taken out is drained in Slurm and
|
|
// given back to llmgw when its last job ends.
|
|
Lent []string `koanf:"lent"`
|
|
|
|
// Partition is the one partition every node is in. Empty is "gpu".
|
|
Partition string `koanf:"partition"`
|
|
}
|
|
|
|
// slurmName is what Slurm takes as a node, cluster or partition name.
|
|
var slurmName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]{0,62}$`)
|
|
|
|
// Cluster is ClusterName or its default.
|
|
func (s SlurmSettings) Cluster() string {
|
|
if s.ClusterName == "" {
|
|
return "llmgw"
|
|
}
|
|
return s.ClusterName
|
|
}
|
|
|
|
// PartitionName is Partition or its default.
|
|
func (s SlurmSettings) PartitionName() string {
|
|
if s.Partition == "" {
|
|
return "gpu"
|
|
}
|
|
return s.Partition
|
|
}
|
|
|
|
// IsLent is whether Slurm may use the node now.
|
|
func (s SlurmSettings) IsLent(node string) bool {
|
|
return s.Enabled && (s.AllNodes || slices.Contains(s.Nodes, node)) && slices.Contains(s.Lent, node)
|
|
}
|
|
|
|
// WithNodes is the settings with every node of ids in Nodes when AllNodes:
|
|
// the nodes that run slurmd, as the leader's inventory has them.
|
|
func (s SlurmSettings) WithNodes(ids []string) SlurmSettings {
|
|
if !s.AllNodes {
|
|
return s
|
|
}
|
|
nodes := slices.Clone(s.Nodes)
|
|
for _, id := range ids {
|
|
if !slices.Contains(nodes, id) {
|
|
nodes = append(nodes, id)
|
|
}
|
|
}
|
|
s.Nodes = nodes
|
|
return s
|
|
}
|
|
|
|
// Check refuses settings Slurm could not run on.
|
|
func (s SlurmSettings) Check() error {
|
|
if !s.Enabled {
|
|
return nil
|
|
}
|
|
if s.Controller == "" {
|
|
return fmt.Errorf("slurm.controller is empty: name the node that runs slurmctld")
|
|
}
|
|
if !slurmName.MatchString(s.Controller) {
|
|
return fmt.Errorf("slurm.controller %q is not a name Slurm takes", s.Controller)
|
|
}
|
|
for _, n := range []struct{ key, v string }{{"slurm.cluster_name", s.ClusterName}, {"slurm.partition", s.Partition}} {
|
|
if n.v != "" && !slurmName.MatchString(n.v) {
|
|
return fmt.Errorf("%s %q is not a name Slurm takes: letters, digits, _ . -", n.key, n.v)
|
|
}
|
|
}
|
|
seen := map[string]bool{}
|
|
for i, n := range s.Nodes {
|
|
if !slurmName.MatchString(n) {
|
|
return fmt.Errorf("slurm.nodes[%d] %q is not a name Slurm takes", i, n)
|
|
}
|
|
if seen[n] {
|
|
return fmt.Errorf("slurm.nodes names %s twice", n)
|
|
}
|
|
seen[n] = true
|
|
}
|
|
for i, n := range s.Lent {
|
|
if !seen[n] && !s.AllNodes {
|
|
return fmt.Errorf("slurm.lent[%d] %q is not in slurm.nodes: only a node that runs slurmd can be lent", i, n)
|
|
}
|
|
}
|
|
return nil
|
|
}
|