Slurm: a new leader with nothing kept waits for the nodes a while
CI / test (push) Failing after 7m57s
CI / test (push) Failing after 7m57s
On a cluster upgraded to the kept nodes, the first leader finds none and wrote slurm.conf without the nodes it had not heard from yet, as before. A node neither reported nor kept now holds the controller's files back for slurmsvc.HoldFor (2 minutes) from the leader's first pass. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
141a1f46c5
commit
0774e901c0
2 files changed
+30
-1
No files matched your search
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
@@ -115,3 +116,18 @@ func TestWithTheStoreUnreadTheControllerIsSentNothingThatLeavesANodeOut(t *testi
|
||||
t.Fatal("with every node reported, the controller still got no files")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithNothingKeptANewLeaderWaitsForTheNodesAWhile(t *testing.T) {
|
||||
r := newRig()
|
||||
r.svc.SetMemory(StoreMemory(&fakeKV{m: map[string]string{}}))
|
||||
r.tick()
|
||||
if m := r.agents.last("node06"); m != nil {
|
||||
t.Fatalf("with nothing kept, the controller was sent %v before cuda01 reported", m)
|
||||
}
|
||||
defer func(d time.Duration) { HoldFor = d }(HoldFor)
|
||||
HoldFor = 0
|
||||
r.tick()
|
||||
if m := r.agents.last("node06"); !m.GetController() || strings.Contains(m.GetFiles()["slurm.conf"], "NodeName=cuda01") {
|
||||
t.Fatalf("after the wait, the controller was told %v", m)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user