package store import ( "context" "errors" "sync" "testing" "time" ) // leadershipSuite is the same assertions against whichever store it is given, // because the whole point of the lease is that it behaves identically on both // — and the one that matters for HA is the one it was never tested on. func leadershipSuite(t *testing.T, open func(*testing.T) *DB) { ctx := context.Background() t.Run("the first to ask gets it", func(t *testing.T) { s := open(t) lease, err := s.AcquireLeadership(ctx, "control", "a", time.Minute) if err != nil { t.Fatal(err) } if lease.Owner != "a" { t.Errorf("owner = %q", lease.Owner) } }) t.Run("a second is refused and told who holds it", func(t *testing.T) { s := open(t) if _, err := s.AcquireLeadership(ctx, "control", "a", time.Minute); err != nil { t.Fatal(err) } held, err := s.AcquireLeadership(ctx, "control", "b", time.Minute) if !errors.Is(err, ErrNotLeader) { t.Fatalf("a second control plane got %v", err) } if held.Owner != "a" { t.Errorf("refused without naming the holder: %+v", held) } }) t.Run("the holder renews without losing when it started", func(t *testing.T) { s := open(t) first, err := s.AcquireLeadership(ctx, "control", "a", time.Minute) if err != nil { t.Fatal(err) } time.Sleep(5 * time.Millisecond) again, err := s.AcquireLeadership(ctx, "control", "a", time.Minute) if err != nil { t.Fatalf("the holder could not renew: %v", err) } if !again.ExpiresAt.After(first.ExpiresAt) { t.Error("renewing did not move the expiry") } if !again.AcquiredAt.Equal(first.AcquiredAt) { t.Error("renewing reset when this leader took over") } }) t.Run("a lapsed lease is taken", func(t *testing.T) { // The reason a lease and not a lock: a leader that was powered off // cannot release anything, and nobody should have to be paged to make // a standby take over. s := open(t) if _, err := s.AcquireLeadership(ctx, "control", "a", 20*time.Millisecond); err != nil { t.Fatal(err) } time.Sleep(40 * time.Millisecond) lease, err := s.AcquireLeadership(ctx, "control", "b", time.Minute) if err != nil { t.Fatalf("a lapsed lease was not takeable: %v", err) } if lease.Owner != "b" { t.Errorf("owner after takeover = %q", lease.Owner) } }) t.Run("releasing hands over at once", func(t *testing.T) { s := open(t) if _, err := s.AcquireLeadership(ctx, "control", "a", time.Hour); err != nil { t.Fatal(err) } if err := s.ReleaseLeadership(ctx, "control", "a"); err != nil { t.Fatal(err) } if _, err := s.AcquireLeadership(ctx, "control", "b", time.Minute); err != nil { t.Errorf("a released lease was not free: %v", err) } }) t.Run("a former leader cannot release the current one", func(t *testing.T) { // A process that lost the lease and is shutting down slowly must not // take it from whoever has it now — which is exactly what an // unconditional delete would do. s := open(t) if _, err := s.AcquireLeadership(ctx, "control", "a", 20*time.Millisecond); err != nil { t.Fatal(err) } time.Sleep(40 * time.Millisecond) if _, err := s.AcquireLeadership(ctx, "control", "b", time.Hour); err != nil { t.Fatal(err) } if err := s.ReleaseLeadership(ctx, "control", "a"); err != nil { t.Fatal(err) } held, err := s.Leadership(ctx, "control") if err != nil { t.Fatal(err) } if held.Owner != "b" { t.Errorf("the old leader released the new one's lease: %+v", held) } }) t.Run("only one of many wins", func(t *testing.T) { // The case the whole table exists for. Two control planes starting // together is the ordinary way this gets tested in production. s := open(t) const racers = 8 var wg sync.WaitGroup won := make([]bool, racers) for i := range racers { wg.Add(1) go func() { defer wg.Done() _, err := s.AcquireLeadership(ctx, "control", string(rune('a'+i)), time.Minute) won[i] = err == nil }() } wg.Wait() n := 0 for _, w := range won { if w { n++ } } if n != 1 { t.Errorf("%d of %d processes believed they were leader", n, racers) } }) } func TestLeadershipOnSQLite(t *testing.T) { leadershipSuite(t, func(t *testing.T) *DB { return openDB(t) }) } func TestLeadershipOnPostgres(t *testing.T) { leadershipSuite(t, openPG) }