Slurm in the first install: server init --slurm, and slurm.all_nodes
CI / test (push) Successful in 9m30s

The choices screen has a Slurm part, and --slurm writes slurmctld on this
machine's node and slurmd on every node, those that join later too, into
server.yaml. slurm.all_nodes (ctl slurm setup --all-nodes) takes the nodes
from the leader's inventory. No node is lent at install.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-11 14:50:35 +01:00
1 parent b15944aa1f
commit 914985bf9f
13 files changed
+208 -20

No files matched your search

+25 -9
View File
@@ -91,17 +91,19 @@ func newSlurmStatusCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
func newSlurmSetupCmd(client func() *ctlClient) *cobra.Command {
var controller, cluster, partition, note string
var nodes []string
var all bool
c := &cobra.Command{
Use: "setup --controller NODE --nodes NODE,...",
Use: "setup --controller NODE --nodes NODE,... | --all-nodes",
Short: "Run Slurm on llmgw's nodes: slurmctld on one, slurmd on those named",
Long: "Turns slurm.enabled on and names the nodes. Their agents install munge and Slurm (22.05 or\n" +
Long: "Turns slurm.enabled on and names the nodes, or with --all-nodes every node. Their agents install munge and Slurm (22.05 or\n" +
"newer) from the machines' own package sources. Every node starts drained in Slurm and stays\n" +
"llmgw's until `llmgw ctl slurm lend` gives it to Slurm.",
Example: " llmgw ctl slurm setup --controller node06 --nodes cuda01,cuda02",
Args: cobra.NoArgs,
Example: " llmgw ctl slurm setup --controller node06 --nodes cuda01,cuda02\n" +
" llmgw ctl slurm setup --controller node06 --all-nodes",
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
if controller == "" || len(nodes) == 0 {
return fmt.Errorf("give --controller and --nodes")
if controller == "" || (len(nodes) == 0 && !all) {
return fmt.Errorf("give --controller, and --nodes or --all-nodes")
}
return changeSettings(cmd, client, note, func(doc *settings.Doc) error {
if err := doc.SetYAML([]string{"slurm", "enabled"}, "true"); err != nil {
@@ -110,12 +112,15 @@ func newSlurmSetupCmd(client func() *ctlClient) *cobra.Command {
if err := doc.SetYAML([]string{"slurm", "controller"}, yamlString(controller)); err != nil {
return err
}
if err := doc.SetYAML([]string{"slurm", "nodes"}, yamlList(nodes)); err != nil {
if err := setList(doc, []string{"slurm", "nodes"}, nodes); err != nil {
return err
}
if err := doc.SetYAML([]string{"slurm", "all_nodes"}, fmt.Sprint(all)); err != nil {
return err
}
// Lent nodes no longer named are not lent.
lent := slices.DeleteFunc(docList(doc, []string{"slurm", "lent"}),
func(n string) bool { return !slices.Contains(nodes, n) })
func(n string) bool { return !all && !slices.Contains(nodes, n) })
if err := setList(doc, []string{"slurm", "lent"}, lent); err != nil {
return err
}
@@ -132,6 +137,7 @@ func newSlurmSetupCmd(client func() *ctlClient) *cobra.Command {
}
c.Flags().StringVar(&controller, "controller", "", "the node that runs slurmctld")
c.Flags().StringSliceVar(&nodes, "nodes", nil, "the nodes that run slurmd")
c.Flags().BoolVar(&all, "all-nodes", false, "every node runs slurmd, those that join later too")
c.Flags().StringVar(&cluster, "cluster-name", "", "Slurm's ClusterName (llmgw)")
c.Flags().StringVar(&partition, "partition", "", "the partition every node is in (gpu)")
c.Flags().StringVar(&note, "note", "", "why, kept with the settings' version")
@@ -173,10 +179,11 @@ func newSlurmListCmd(client func() *ctlClient, use, short string, add bool) *cob
RunE: func(cmd *cobra.Command, args []string) error {
return changeSettings(cmd, client, note, func(doc *settings.Doc) error {
nodes := docList(doc, []string{"slurm", "nodes"})
all := docBool(doc, []string{"slurm", "all_nodes"})
lent := docList(doc, []string{"slurm", "lent"})
for _, n := range args {
switch {
case add && !slices.Contains(nodes, n):
case add && !all && !slices.Contains(nodes, n):
return fmt.Errorf("%s does not run slurmd: add it with `llmgw ctl slurm setup --nodes`", n)
case add && !slices.Contains(lent, n):
lent = append(lent, n)
@@ -205,6 +212,15 @@ func docList(doc *settings.Doc, path []string) []string {
return out
}
// docBool reads a boolean from the settings, false when unset.
func docBool(doc *settings.Doc, path []string) bool {
var b bool
if n := doc.Node(path); n != nil {
_ = n.Decode(&b)
}
return b
}
func setList(doc *settings.Doc, path []string, list []string) error {
if len(list) == 0 {
if doc.Node(path) != nil {