package config import ( "fmt" "regexp" "slices" ) // SlurmSettings is a Slurm cluster run beside llmgw on some of its nodes // (BACKLOG P), for training and other batch work llmgw does not schedule. // The leader tells each named node's agent what to run; the agents install // it from the machine's package sources and keep it going. type SlurmSettings struct { // Enabled runs Slurm on the nodes below. False stops llmgw keeping it; // what was installed is left installed, and every node is llmgw's. Enabled bool `koanf:"enabled"` // ClusterName is Slurm's ClusterName. Empty is "llmgw". ClusterName string `koanf:"cluster_name"` // Controller is the llmgw node whose agent runs slurmctld. It needs no // GPU, and need not be one of Nodes. Controller string `koanf:"controller"` // Nodes are the llmgw nodes that run slurmd. Each stays llmgw's, drained // in Slurm, until it is also in Lent. Nodes []string `koanf:"nodes"` // AllNodes runs slurmd on every llmgw node, those that join later too, // besides any named in Nodes. Each is llmgw's until it is lent, as any. AllNodes bool `koanf:"all_nodes"` // Lent are the nodes of Nodes Slurm may run jobs on now. A node put here // is drained of llmgw's replicas first, as `ctl drain` does, and resumed // in Slurm once none is left; a node taken out is drained in Slurm and // given back to llmgw when its last job ends. Lent []string `koanf:"lent"` // Partition is the one partition every node is in. Empty is "gpu". Partition string `koanf:"partition"` } // slurmName is what Slurm takes as a node, cluster or partition name. var slurmName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]{0,62}$`) // Cluster is ClusterName or its default. func (s SlurmSettings) Cluster() string { if s.ClusterName == "" { return "llmgw" } return s.ClusterName } // PartitionName is Partition or its default. func (s SlurmSettings) PartitionName() string { if s.Partition == "" { return "gpu" } return s.Partition } // IsLent is whether Slurm may use the node now. func (s SlurmSettings) IsLent(node string) bool { return s.Enabled && (s.AllNodes || slices.Contains(s.Nodes, node)) && slices.Contains(s.Lent, node) } // WithNodes is the settings with every node of ids in Nodes when AllNodes: // the nodes that run slurmd, as the leader's inventory has them. func (s SlurmSettings) WithNodes(ids []string) SlurmSettings { if !s.AllNodes { return s } nodes := slices.Clone(s.Nodes) for _, id := range ids { if !slices.Contains(nodes, id) { nodes = append(nodes, id) } } s.Nodes = nodes return s } // Check refuses settings Slurm could not run on. func (s SlurmSettings) Check() error { if !s.Enabled { return nil } if s.Controller == "" { return fmt.Errorf("slurm.controller is empty: name the node that runs slurmctld") } if !slurmName.MatchString(s.Controller) { return fmt.Errorf("slurm.controller %q is not a name Slurm takes", s.Controller) } for _, n := range []struct{ key, v string }{{"slurm.cluster_name", s.ClusterName}, {"slurm.partition", s.Partition}} { if n.v != "" && !slurmName.MatchString(n.v) { return fmt.Errorf("%s %q is not a name Slurm takes: letters, digits, _ . -", n.key, n.v) } } seen := map[string]bool{} for i, n := range s.Nodes { if !slurmName.MatchString(n) { return fmt.Errorf("slurm.nodes[%d] %q is not a name Slurm takes", i, n) } if seen[n] { return fmt.Errorf("slurm.nodes names %s twice", n) } seen[n] = true } for i, n := range s.Lent { if !seen[n] && !s.AllNodes { return fmt.Errorf("slurm.lent[%d] %q is not in slurm.nodes: only a node that runs slurmd can be lent", i, n) } } return nil }