From 964b659849c9c5abdc169b318db7623ec02cc161 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Mon, 5 Oct 2026 02:31:13 +0200 Subject: [PATCH] A stopping control plane gives up the lease before it lets its agents go With the agent server closed rather than drained, the process could exit before Run's release had reached the store, and the standby waited out the lease. The lease is now given up first, synchronously, and nothing takes it again after. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/cli/server.go | 6 ++++ internal/control/leadership.go | 29 +++++++++++++--- internal/control/leadership_test.go | 51 +++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 4 deletions(-) create mode 100644 internal/control/leadership_test.go diff --git a/internal/cli/server.go b/internal/cli/server.go index 173810c..b1fd48d 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -635,6 +635,12 @@ func newServerCmd() *cobra.Command { log.Info("shutting down") sctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() + // The lease first: a standby polls for it, and every agent + // about to be let go will be refused by a standby that has + // not taken it yet. + if admin != nil && admin.Leadership() != nil { + admin.Leadership().StepDown() + } if agentSrv != nil { // Closed, not shut down gracefully: an agent's stream // never goes idle, so Shutdown would wait out its whole diff --git a/internal/control/leadership.go b/internal/control/leadership.go index 493a6a6..512d168 100644 --- a/internal/control/leadership.go +++ b/internal/control/leadership.go @@ -52,6 +52,8 @@ type Leadership struct { renew time.Duration leader atomic.Bool + // stopped is set by StepDown: nothing takes the lease again after it. + stopped atomic.Bool } // NewLeadership prepares leadership over a store. @@ -118,8 +120,16 @@ func (l *Leadership) Run(ctx context.Context) { // process: a line per attempt would bury everything else, and the thing an // operator needs to see is the moment leadership moved. func (l *Leadership) attempt(ctx context.Context) { + if l.stopped.Load() { + return + } lease, err := l.store.AcquireLeadership(ctx, controlRole, l.owner, l.ttl) switch { + case err == nil && l.stopped.Load(): + // Stepped down while this was on its way to the store: give back + // what it just took. + l.leader.Store(true) + l.step(ctx) case err == nil: if l.leader.CompareAndSwap(false, true) { l.log.Info("this control plane is now placing work", "owner", l.owner) @@ -146,10 +156,24 @@ func (l *Leadership) attempt(ctx context.Context) { } } +// StepDown gives the lease up now, for a process on its way out: called +// before the agents are let go, so the standby they move to is already the +// leader, and before the process exits, which would otherwise race the +// release in Run and leave the standby to wait out the lease. Run's own +// release after it is then nothing. +func (l *Leadership) StepDown() { + if l.store == nil { + return + } + l.stopped.Store(true) + l.step(context.Background()) +} + // step gives the lease up on the way out, so a standby takes over in seconds // rather than waiting out the clock. func (l *Leadership) step(ctx context.Context) { - if !l.leader.Load() { + // Once: StepDown and Run's own way out both come here. + if !l.leader.CompareAndSwap(true, false) { return } // A fresh context: the one that just ended is the reason this is running. @@ -159,9 +183,6 @@ func (l *Leadership) step(ctx context.Context) { l.log.Warn("could not release leadership; a standby will wait it out", "err", err) return } - // Not leading from here on, so nothing in this process goes on acting as - // if it were while it shuts down. - l.leader.Store(false) l.log.Info("released leadership", "owner", l.owner) } diff --git a/internal/control/leadership_test.go b/internal/control/leadership_test.go new file mode 100644 index 0000000..6367f98 --- /dev/null +++ b/internal/control/leadership_test.go @@ -0,0 +1,51 @@ +package control + +import ( + "log/slog" + "path/filepath" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +func TestAStoppingLeaderHandsOverAtOnceAndNeverTakesItBack(t *testing.T) { + st, err := store.Open(t.Context(), filepath.Join(t.TempDir(), "s.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + log := slog.New(slog.DiscardHandler) + a := NewLeadership(st, time.Minute, log) + b := NewLeadership(st, time.Minute, log) + + a.attempt(t.Context()) + b.attempt(t.Context()) + if !a.IsLeader() || b.IsLeader() { + t.Fatalf("a %v, b %v", a.IsLeader(), b.IsLeader()) + } + + // Stepping down gives the lease back, rather than leaving b to wait out + // the minute it was taken for. + a.StepDown() + if a.IsLeader() { + t.Error("still leading after stepping down") + } + // A renewal already on its way when the process began to stop must not + // take it back. + a.attempt(t.Context()) + if a.IsLeader() { + t.Error("took the lease again after stepping down") + } + b.attempt(t.Context()) + if !b.IsLeader() { + t.Error("the standby could not take the lease that was given up") + } + lease, err := st.Leadership(t.Context(), ControlRole) + if err != nil || lease.Owner != b.Owner() { + t.Errorf("the lease is %q (%v), not b's", lease.Owner, err) + } + if got := HolderName(b.Owner()); got == "" || got == b.Owner() { + t.Errorf("holder name %q from %q", got, b.Owner()) + } +}