package control import ( "context" "errors" "log/slog" "path/filepath" "testing" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) func TestAStoppingLeaderHandsOverAtOnceAndNeverTakesItBack(t *testing.T) { st, err := store.Open(t.Context(), filepath.Join(t.TempDir(), "s.db")) if err != nil { t.Fatal(err) } defer st.Close() log := slog.New(slog.DiscardHandler) a := NewLeadership(st, time.Minute, log) b := NewLeadership(st, time.Minute, log) a.attempt(t.Context()) b.attempt(t.Context()) if !a.IsLeader() || b.IsLeader() { t.Fatalf("a %v, b %v", a.IsLeader(), b.IsLeader()) } // Stepping down gives the lease back, rather than leaving b to wait out // the minute it was taken for. a.StepDown() if a.IsLeader() { t.Error("still leading after stepping down") } // A renewal already on its way when the process began to stop must not // take it back. a.attempt(t.Context()) if a.IsLeader() { t.Error("took the lease again after stepping down") } b.attempt(t.Context()) if !b.IsLeader() { t.Error("the standby could not take the lease that was given up") } lease, err := st.Leadership(t.Context(), ControlRole) if err != nil || lease.Owner != b.Owner() { t.Errorf("the lease is %q (%v), not b's", lease.Owner, err) } if got := HolderName(b.Owner()); got == "" || got == b.Owner() { t.Errorf("holder name %q from %q", got, b.Owner()) } } func TestAStandbyIsToldWhenItBecomesTheLeader(t *testing.T) { st, err := store.Open(t.Context(), filepath.Join(t.TempDir(), "s.db")) if err != nil { t.Fatal(err) } defer st.Close() log := slog.New(slog.DiscardHandler) a := NewLeadership(st, time.Minute, log) b := NewLeadership(st, time.Minute, log) led := make(chan struct{}, 4) b.OnLead(func() { led <- struct{}{} }) ctx := t.Context() a.attempt(ctx) b.attempt(ctx) if !a.IsLeader() || b.IsLeader() { t.Fatal("a should lead, b stand by") } a.StepDown() b.attempt(ctx) select { case <-led: case <-time.After(time.Second): t.Fatal("b became the leader and was not told") } } // cuttable is a store that can be out of reach, as one whose primary was lost. type cuttable struct { *store.DB down bool } func (c *cuttable) AcquireLeadership(ctx context.Context, role, owner string, ttl time.Duration) (store.Lease, error) { if c.down { return store.Lease{}, errors.New("dial error: timeout") } return c.DB.AcquireLeadership(ctx, role, owner, ttl) } func (c *cuttable) Leadership(ctx context.Context, role string) (store.Lease, error) { if c.down { return store.Lease{}, errors.New("dial error: timeout") } return c.DB.Leadership(ctx, role) } // The store out of reach for longer than the lease lives, as while Patroni // promotes a standby: when it answers again the lease has lapsed for all, // and the leader keeps it rather than a standby taking it for nothing. A // leader that is truly gone is still replaced, a round later. func TestAStoreThatReturnsLeavesTheLeaderLeading(t *testing.T) { st, err := store.Open(t.Context(), filepath.Join(t.TempDir(), "s.db")) if err != nil { t.Fatal(err) } defer st.Close() c := &cuttable{DB: st} log := slog.New(slog.DiscardHandler) ctx := t.Context() const ttl = 150 * time.Millisecond a, b := NewLeadership(c, ttl, log), NewLeadership(c, ttl, log) a.attempt(ctx) b.attempt(ctx) if !a.IsLeader() || b.IsLeader() { t.Fatal("a should lead, b stand by") } c.down = true a.attempt(ctx) b.attempt(ctx) time.Sleep(ttl + 20*time.Millisecond) // the lease lapses while nobody reaches the store c.down = false b.attempt(ctx) // the standby is first to ask if b.IsLeader() { t.Fatal("a standby took the lease the moment the store came back") } a.attempt(ctx) if !a.IsLeader() { t.Fatal("the leader did not keep the lease") } time.Sleep(a.renew) b.attempt(ctx) if b.IsLeader() { t.Error("the standby took the lease from a leader renewing it") } // Now the leader is gone: the standby takes over after its round. c.down = true b.attempt(ctx) time.Sleep(ttl + 20*time.Millisecond) c.down = false b.attempt(ctx) if b.IsLeader() { t.Fatal("took it at once") } time.Sleep(b.renew) b.attempt(ctx) if !b.IsLeader() { t.Error("a leader that was gone was not replaced") } }