Slurm in the first install: server init --slurm, and slurm.all_nodes
CI / test (push) Successful in 9m30s
CI / test (push) Successful in 9m30s
The choices screen has a Slurm part, and --slurm writes slurmctld on this machine's node and slurmd on every node, those that join later too, into server.yaml. slurm.all_nodes (ctl slurm setup --all-nodes) takes the nodes from the leader's inventory. No node is lent at install. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b15944aa1f
commit
914985bf9f
13 files changed
+208
-20
No files matched your search
@@ -41,6 +41,7 @@ type Inventory interface {
|
||||
Get(id string) (inventory.Node, bool)
|
||||
SetState(id string, st inventory.NodeState) bool
|
||||
Allocations(nodeID string) []inventory.Allocation
|
||||
Nodes() []inventory.Node
|
||||
}
|
||||
|
||||
// Phase is where a node of slurm.nodes is between llmgw and Slurm.
|
||||
@@ -128,6 +129,20 @@ func New(settings func() config.SlurmSettings, inv Inventory, agents Agents,
|
||||
kick: make(chan struct{}, 1)}
|
||||
}
|
||||
|
||||
// current is the settings, slurm.all_nodes resolved to the nodes the
|
||||
// inventory has.
|
||||
func (s *Service) current() config.SlurmSettings {
|
||||
set := s.settings()
|
||||
if !set.AllNodes {
|
||||
return set
|
||||
}
|
||||
var ids []string
|
||||
for _, n := range s.inv.Nodes() {
|
||||
ids = append(ids, n.ID)
|
||||
}
|
||||
return set.WithNodes(ids)
|
||||
}
|
||||
|
||||
// Every is how often the service looks again without being asked.
|
||||
var Every = 10 * time.Second
|
||||
|
||||
@@ -182,7 +197,7 @@ func (s *Service) OnRegister(node string) {
|
||||
// Tick does one pass: the nodes handed over as far as they can be, and
|
||||
// each told what it is.
|
||||
func (s *Service) Tick(ctx context.Context) {
|
||||
set := s.settings()
|
||||
set := s.current()
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
phases := s.handOver(set)
|
||||
@@ -361,7 +376,7 @@ func (s *Service) message(set config.SlurmSettings, node string, lent bool, key
|
||||
|
||||
// Status is the cluster as the service sees it now.
|
||||
func (s *Service) Status() Status {
|
||||
set := s.settings()
|
||||
set := s.current()
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
out := Status{Enabled: set.Enabled, Cluster: set.Cluster(), Controller: set.Controller,
|
||||
|
||||
@@ -2,6 +2,7 @@ package slurmsvc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -36,6 +37,15 @@ func (f *fakeInv) Allocations(id string) []inventory.Allocation {
|
||||
return make([]inventory.Allocation, f.allocs[id])
|
||||
}
|
||||
|
||||
func (f *fakeInv) Nodes() []inventory.Node {
|
||||
var out []inventory.Node
|
||||
for _, n := range f.nodes {
|
||||
out = append(out, *n)
|
||||
}
|
||||
slices.SortFunc(out, func(a, b inventory.Node) int { return strings.Compare(a.ID, b.ID) })
|
||||
return out
|
||||
}
|
||||
|
||||
type fakeAgents struct {
|
||||
conn []string
|
||||
got map[string][]*llmgwv1.Slurm
|
||||
@@ -241,3 +251,24 @@ func TestNoControllerYet(t *testing.T) {
|
||||
t.Fatalf("a node was started with no controller to fetch its files from: %v", m)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllNodes(t *testing.T) {
|
||||
r := newRig()
|
||||
r.set.Nodes, r.set.AllNodes = nil, true
|
||||
r.inv.nodes["cuda02"] = &inventory.Node{ID: "cuda02", MgmtAddr: "10.0.0.150"}
|
||||
r.agents.conn = append(r.agents.conn, "cuda02")
|
||||
r.tick()
|
||||
for _, n := range []string{"cuda01", "cuda02", "node06"} {
|
||||
if !r.agents.last(n).GetNode() {
|
||||
t.Fatalf("%s, a node that joined, does not run slurmd", n)
|
||||
}
|
||||
}
|
||||
if got := len(r.svc.Status().Nodes); got != 3 {
|
||||
t.Fatalf("%d nodes in the status", got)
|
||||
}
|
||||
r.set.Lent = []string{"cuda02"}
|
||||
r.tick()
|
||||
if p := r.svc.Status().Nodes[1]; p.Node != "cuda02" || p.Phase != Lent {
|
||||
t.Fatalf("cuda02 not lent: %+v", p)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user