Slurm: a new leader with nothing kept waits for the nodes a while
CI / test (push) Failing after 7m57s
CI / test (push) Failing after 7m57s
On a cluster upgraded to the kept nodes, the first leader finds none and wrote slurm.conf without the nodes it had not heard from yet, as before. A node neither reported nor kept now holds the controller's files back for slurmsvc.HoldFor (2 minutes) from the leader's first pass. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
141a1f46c5
commit
0774e901c0
2 files changed
+30
-1
No files matched your search
@@ -123,8 +123,18 @@ type Service struct {
|
||||
mem Memory
|
||||
known Kept
|
||||
loaded bool
|
||||
// started is the first pass, from which a node neither heard from nor
|
||||
// kept is waited for, for HoldFor.
|
||||
started time.Time
|
||||
}
|
||||
|
||||
// HoldFor is how long a new leader waits for a node it neither has heard
|
||||
// from nor finds kept before it writes the controller's files without it:
|
||||
// on a cluster that kept nothing yet (one upgraded to this), the nodes
|
||||
// report within seconds of a new leader, and the controller keeps the
|
||||
// node and its jobs meanwhile.
|
||||
var HoldFor = 2 * time.Minute
|
||||
|
||||
// New is a Service; key gives the munge key, made once from a cluster
|
||||
// secret.
|
||||
func New(settings func() config.SlurmSettings, inv Inventory, agents Agents,
|
||||
@@ -216,6 +226,9 @@ func (s *Service) Tick(ctx context.Context) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.recall(ctx)
|
||||
if s.started.IsZero() {
|
||||
s.started = time.Now()
|
||||
}
|
||||
phases := s.handOver(set)
|
||||
var key []byte
|
||||
if set.Enabled {
|
||||
@@ -232,7 +245,7 @@ func (s *Service) Tick(ctx context.Context) {
|
||||
files, waiting := s.files(set)
|
||||
// Not yet knowing what the last leader knew, a slurm.conf without some
|
||||
// node would take it, and its jobs, from the controller: kept back.
|
||||
hold := s.mem != nil && !s.loaded && len(waiting) > 0
|
||||
hold := s.mem != nil && len(waiting) > 0 && (!s.loaded || time.Since(s.started) < HoldFor)
|
||||
for _, node := range s.agents.ConnectedNodes() {
|
||||
if hold && node == set.Controller {
|
||||
continue
|
||||
|
||||
Reference in new issue
Block a user