diff --git a/internal/cli/server.go b/internal/cli/server.go index 173810cd..b1fd48d1 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -635,6 +635,12 @@ func newServerCmd() *cobra.Command { log.Info("shutting down") sctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() + // The lease first: a standby polls for it, and every agent + // about to be let go will be refused by a standby that has + // not taken it yet. + if admin != nil && admin.Leadership() != nil { + admin.Leadership().StepDown() + } if agentSrv != nil { // Closed, not shut down gracefully: an agent's stream // never goes idle, so Shutdown would wait out its whole diff --git a/internal/control/leadership.go b/internal/control/leadership.go index 493a6a63..512d1680 100644 --- a/internal/control/leadership.go +++ b/internal/control/leadership.go @@ -52,6 +52,8 @@ type Leadership struct { renew time.Duration leader atomic.Bool + // stopped is set by StepDown: nothing takes the lease again after it. + stopped atomic.Bool } // NewLeadership prepares leadership over a store. @@ -118,8 +120,16 @@ func (l *Leadership) Run(ctx context.Context) { // process: a line per attempt would bury everything else, and the thing an // operator needs to see is the moment leadership moved. func (l *Leadership) attempt(ctx context.Context) { + if l.stopped.Load() { + return + } lease, err := l.store.AcquireLeadership(ctx, controlRole, l.owner, l.ttl) switch { + case err == nil && l.stopped.Load(): + // Stepped down while this was on its way to the store: give back + // what it just took. + l.leader.Store(true) + l.step(ctx) case err == nil: if l.leader.CompareAndSwap(false, true) { l.log.Info("this control plane is now placing work", "owner", l.owner) @@ -146,10 +156,24 @@ func (l *Leadership) attempt(ctx context.Context) { } } +// StepDown gives the lease up now, for a process on its way out: called +// before the agents are let go, so the standby they move to is already the +// leader, and before the process exits, which would otherwise race the +// release in Run and leave the standby to wait out the lease. Run's own +// release after it is then nothing. +func (l *Leadership) StepDown() { + if l.store == nil { + return + } + l.stopped.Store(true) + l.step(context.Background()) +} + // step gives the lease up on the way out, so a standby takes over in seconds // rather than waiting out the clock. func (l *Leadership) step(ctx context.Context) { - if !l.leader.Load() { + // Once: StepDown and Run's own way out both come here. + if !l.leader.CompareAndSwap(true, false) { return } // A fresh context: the one that just ended is the reason this is running. @@ -159,9 +183,6 @@ func (l *Leadership) step(ctx context.Context) { l.log.Warn("could not release leadership; a standby will wait it out", "err", err) return } - // Not leading from here on, so nothing in this process goes on acting as - // if it were while it shuts down. - l.leader.Store(false) l.log.Info("released leadership", "owner", l.owner) } diff --git a/internal/control/leadership_test.go b/internal/control/leadership_test.go new file mode 100644 index 00000000..6367f989 --- /dev/null +++ b/internal/control/leadership_test.go @@ -0,0 +1,51 @@ +package control + +import ( + "log/slog" + "path/filepath" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +func TestAStoppingLeaderHandsOverAtOnceAndNeverTakesItBack(t *testing.T) { + st, err := store.Open(t.Context(), filepath.Join(t.TempDir(), "s.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + log := slog.New(slog.DiscardHandler) + a := NewLeadership(st, time.Minute, log) + b := NewLeadership(st, time.Minute, log) + + a.attempt(t.Context()) + b.attempt(t.Context()) + if !a.IsLeader() || b.IsLeader() { + t.Fatalf("a %v, b %v", a.IsLeader(), b.IsLeader()) + } + + // Stepping down gives the lease back, rather than leaving b to wait out + // the minute it was taken for. + a.StepDown() + if a.IsLeader() { + t.Error("still leading after stepping down") + } + // A renewal already on its way when the process began to stop must not + // take it back. + a.attempt(t.Context()) + if a.IsLeader() { + t.Error("took the lease again after stepping down") + } + b.attempt(t.Context()) + if !b.IsLeader() { + t.Error("the standby could not take the lease that was given up") + } + lease, err := st.Leadership(t.Context(), ControlRole) + if err != nil || lease.Owner != b.Owner() { + t.Errorf("the lease is %q (%v), not b's", lease.Owner, err) + } + if got := HolderName(b.Owner()); got == "" || got == b.Owner() { + t.Errorf("holder name %q from %q", got, b.Owner()) + } +}