Slurm: a new leader with nothing kept waits for the nodes a while
CI / test (push) Failing after 7m57s
CI / test (push) Failing after 7m57s
On a cluster upgraded to the kept nodes, the first leader finds none and wrote slurm.conf without the nodes it had not heard from yet, as before. A node neither reported nor kept now holds the controller's files back for slurmsvc.HoldFor (2 minutes) from the leader's first pass. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
141a1f46c5
commit
0774e901c0
2 files changed
+30
-1
No files matched your search
@@ -5,6 +5,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||||
@@ -115,3 +116,18 @@ func TestWithTheStoreUnreadTheControllerIsSentNothingThatLeavesANodeOut(t *testi
|
|||||||
t.Fatal("with every node reported, the controller still got no files")
|
t.Fatal("with every node reported, the controller still got no files")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestWithNothingKeptANewLeaderWaitsForTheNodesAWhile(t *testing.T) {
|
||||||
|
r := newRig()
|
||||||
|
r.svc.SetMemory(StoreMemory(&fakeKV{m: map[string]string{}}))
|
||||||
|
r.tick()
|
||||||
|
if m := r.agents.last("node06"); m != nil {
|
||||||
|
t.Fatalf("with nothing kept, the controller was sent %v before cuda01 reported", m)
|
||||||
|
}
|
||||||
|
defer func(d time.Duration) { HoldFor = d }(HoldFor)
|
||||||
|
HoldFor = 0
|
||||||
|
r.tick()
|
||||||
|
if m := r.agents.last("node06"); !m.GetController() || strings.Contains(m.GetFiles()["slurm.conf"], "NodeName=cuda01") {
|
||||||
|
t.Fatalf("after the wait, the controller was told %v", m)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -123,8 +123,18 @@ type Service struct {
|
|||||||
mem Memory
|
mem Memory
|
||||||
known Kept
|
known Kept
|
||||||
loaded bool
|
loaded bool
|
||||||
|
// started is the first pass, from which a node neither heard from nor
|
||||||
|
// kept is waited for, for HoldFor.
|
||||||
|
started time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// HoldFor is how long a new leader waits for a node it neither has heard
|
||||||
|
// from nor finds kept before it writes the controller's files without it:
|
||||||
|
// on a cluster that kept nothing yet (one upgraded to this), the nodes
|
||||||
|
// report within seconds of a new leader, and the controller keeps the
|
||||||
|
// node and its jobs meanwhile.
|
||||||
|
var HoldFor = 2 * time.Minute
|
||||||
|
|
||||||
// New is a Service; key gives the munge key, made once from a cluster
|
// New is a Service; key gives the munge key, made once from a cluster
|
||||||
// secret.
|
// secret.
|
||||||
func New(settings func() config.SlurmSettings, inv Inventory, agents Agents,
|
func New(settings func() config.SlurmSettings, inv Inventory, agents Agents,
|
||||||
@@ -216,6 +226,9 @@ func (s *Service) Tick(ctx context.Context) {
|
|||||||
s.mu.Lock()
|
s.mu.Lock()
|
||||||
defer s.mu.Unlock()
|
defer s.mu.Unlock()
|
||||||
s.recall(ctx)
|
s.recall(ctx)
|
||||||
|
if s.started.IsZero() {
|
||||||
|
s.started = time.Now()
|
||||||
|
}
|
||||||
phases := s.handOver(set)
|
phases := s.handOver(set)
|
||||||
var key []byte
|
var key []byte
|
||||||
if set.Enabled {
|
if set.Enabled {
|
||||||
@@ -232,7 +245,7 @@ func (s *Service) Tick(ctx context.Context) {
|
|||||||
files, waiting := s.files(set)
|
files, waiting := s.files(set)
|
||||||
// Not yet knowing what the last leader knew, a slurm.conf without some
|
// Not yet knowing what the last leader knew, a slurm.conf without some
|
||||||
// node would take it, and its jobs, from the controller: kept back.
|
// node would take it, and its jobs, from the controller: kept back.
|
||||||
hold := s.mem != nil && !s.loaded && len(waiting) > 0
|
hold := s.mem != nil && len(waiting) > 0 && (!s.loaded || time.Since(s.started) < HoldFor)
|
||||||
for _, node := range s.agents.ConnectedNodes() {
|
for _, node := range s.agents.ConnectedNodes() {
|
||||||
if hold && node == set.Controller {
|
if hold && node == set.Controller {
|
||||||
continue
|
continue
|
||||||
|
|||||||
Reference in new issue
Block a user