A stopping control plane gives up the lease before it lets its agents go
With the agent server closed rather than drained, the process could exit before Run's release had reached the store, and the standby waited out the lease. The lease is now given up first, synchronously, and nothing takes it again after. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
c2b4985abe
commit
964b659849
3 files changed
+82
-4
No files matched your search
@@ -635,6 +635,12 @@ func newServerCmd() *cobra.Command {
|
||||
log.Info("shutting down")
|
||||
sctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
// The lease first: a standby polls for it, and every agent
|
||||
// about to be let go will be refused by a standby that has
|
||||
// not taken it yet.
|
||||
if admin != nil && admin.Leadership() != nil {
|
||||
admin.Leadership().StepDown()
|
||||
}
|
||||
if agentSrv != nil {
|
||||
// Closed, not shut down gracefully: an agent's stream
|
||||
// never goes idle, so Shutdown would wait out its whole
|
||||
|
||||
@@ -52,6 +52,8 @@ type Leadership struct {
|
||||
renew time.Duration
|
||||
|
||||
leader atomic.Bool
|
||||
// stopped is set by StepDown: nothing takes the lease again after it.
|
||||
stopped atomic.Bool
|
||||
}
|
||||
|
||||
// NewLeadership prepares leadership over a store.
|
||||
@@ -118,8 +120,16 @@ func (l *Leadership) Run(ctx context.Context) {
|
||||
// process: a line per attempt would bury everything else, and the thing an
|
||||
// operator needs to see is the moment leadership moved.
|
||||
func (l *Leadership) attempt(ctx context.Context) {
|
||||
if l.stopped.Load() {
|
||||
return
|
||||
}
|
||||
lease, err := l.store.AcquireLeadership(ctx, controlRole, l.owner, l.ttl)
|
||||
switch {
|
||||
case err == nil && l.stopped.Load():
|
||||
// Stepped down while this was on its way to the store: give back
|
||||
// what it just took.
|
||||
l.leader.Store(true)
|
||||
l.step(ctx)
|
||||
case err == nil:
|
||||
if l.leader.CompareAndSwap(false, true) {
|
||||
l.log.Info("this control plane is now placing work", "owner", l.owner)
|
||||
@@ -146,10 +156,24 @@ func (l *Leadership) attempt(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// StepDown gives the lease up now, for a process on its way out: called
|
||||
// before the agents are let go, so the standby they move to is already the
|
||||
// leader, and before the process exits, which would otherwise race the
|
||||
// release in Run and leave the standby to wait out the lease. Run's own
|
||||
// release after it is then nothing.
|
||||
func (l *Leadership) StepDown() {
|
||||
if l.store == nil {
|
||||
return
|
||||
}
|
||||
l.stopped.Store(true)
|
||||
l.step(context.Background())
|
||||
}
|
||||
|
||||
// step gives the lease up on the way out, so a standby takes over in seconds
|
||||
// rather than waiting out the clock.
|
||||
func (l *Leadership) step(ctx context.Context) {
|
||||
if !l.leader.Load() {
|
||||
// Once: StepDown and Run's own way out both come here.
|
||||
if !l.leader.CompareAndSwap(true, false) {
|
||||
return
|
||||
}
|
||||
// A fresh context: the one that just ended is the reason this is running.
|
||||
@@ -159,9 +183,6 @@ func (l *Leadership) step(ctx context.Context) {
|
||||
l.log.Warn("could not release leadership; a standby will wait it out", "err", err)
|
||||
return
|
||||
}
|
||||
// Not leading from here on, so nothing in this process goes on acting as
|
||||
// if it were while it shuts down.
|
||||
l.leader.Store(false)
|
||||
l.log.Info("released leadership", "owner", l.owner)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
package control
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
func TestAStoppingLeaderHandsOverAtOnceAndNeverTakesItBack(t *testing.T) {
|
||||
st, err := store.Open(t.Context(), filepath.Join(t.TempDir(), "s.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer st.Close()
|
||||
log := slog.New(slog.DiscardHandler)
|
||||
a := NewLeadership(st, time.Minute, log)
|
||||
b := NewLeadership(st, time.Minute, log)
|
||||
|
||||
a.attempt(t.Context())
|
||||
b.attempt(t.Context())
|
||||
if !a.IsLeader() || b.IsLeader() {
|
||||
t.Fatalf("a %v, b %v", a.IsLeader(), b.IsLeader())
|
||||
}
|
||||
|
||||
// Stepping down gives the lease back, rather than leaving b to wait out
|
||||
// the minute it was taken for.
|
||||
a.StepDown()
|
||||
if a.IsLeader() {
|
||||
t.Error("still leading after stepping down")
|
||||
}
|
||||
// A renewal already on its way when the process began to stop must not
|
||||
// take it back.
|
||||
a.attempt(t.Context())
|
||||
if a.IsLeader() {
|
||||
t.Error("took the lease again after stepping down")
|
||||
}
|
||||
b.attempt(t.Context())
|
||||
if !b.IsLeader() {
|
||||
t.Error("the standby could not take the lease that was given up")
|
||||
}
|
||||
lease, err := st.Leadership(t.Context(), ControlRole)
|
||||
if err != nil || lease.Owner != b.Owner() {
|
||||
t.Errorf("the lease is %q (%v), not b's", lease.Owner, err)
|
||||
}
|
||||
if got := HolderName(b.Owner()); got == "" || got == b.Owner() {
|
||||
t.Errorf("holder name %q from %q", got, b.Owner())
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user