package store import ( "context" "errors" "time" ) // ErrNotLeader means somebody else holds the lease and it has not lapsed. var ErrNotLeader = errors.New("store: another control plane holds the lease") // Lease is who is in charge and until when. type Lease struct { Owner string ExpiresAt time.Time AcquiredAt time.Time } // AcquireLeadership takes or renews the lease for a role. // // Renewing and taking are the same statement on purpose: a leader that renews // takes the same path as one starting fresh, so there is no way for the two to // disagree about what a held lease looks like. It succeeds when the row is // absent, when this process already owns it, or when whoever owned it let it // lapse; otherwise it returns ErrNotLeader. // // The clock is the caller's rather than the database's, which is a compromise // worth naming. A database clock would be one clock for everyone; taking it // means a round trip that says now() inside every comparison, in SQL that is // spelled differently in each dialect. Two control planes on machines whose // clocks disagree by more than the lease will fight, so keep them on NTP — // which any cluster running InfiniBand already has to be. func (s *DB) AcquireLeadership(ctx context.Context, role, owner string, ttl time.Duration) (Lease, error) { now := time.Now() expires := now.Add(ttl) // One statement, so two processes racing cannot both believe they won: // the WHERE decides, and the loser changes no rows. res, err := s.db.ExecContext(ctx, `INSERT INTO leadership (role, owner, expires_at, acquired_at) VALUES (?, ?, ?, ?) ON CONFLICT (role) DO UPDATE SET owner = excluded.owner, expires_at = excluded.expires_at, acquired_at = CASE WHEN leadership.owner = excluded.owner THEN leadership.acquired_at ELSE excluded.acquired_at END WHERE leadership.owner = excluded.owner OR leadership.expires_at < ?`, role, owner, expires.UnixMilli(), now.UnixMilli(), now.UnixMilli()) if err != nil { return Lease{}, err } n, err := res.RowsAffected() if err != nil { return Lease{}, err } if n == 0 { // Lost, and the caller is told who to, because "not leader" without a // name is the start of a long afternoon. The lease is returned beside // the error rather than instead of it. held, lerr := s.Leadership(ctx, role) if lerr != nil { return Lease{}, lerr } return held, ErrNotLeader } return s.Leadership(ctx, role) } // Leadership reports who holds a role's lease. func (s *DB) Leadership(ctx context.Context, role string) (Lease, error) { var owner string var expires, acquired int64 err := s.db.QueryRowContext(ctx, `SELECT owner, expires_at, acquired_at FROM leadership WHERE role = ?`, role). Scan(&owner, &expires, &acquired) if err != nil { return Lease{}, err } return Lease{ Owner: owner, ExpiresAt: fromMillis(expires), AcquiredAt: fromMillis(acquired), }, nil } // ReleaseLeadership gives the lease up, so a standby takes over in seconds // rather than waiting out the clock. // // Only the holder can release it: a process that has already lost the lease // must not be able to take it from whoever has it now, which is exactly what // an unconditional delete during a slow shutdown would do. func (s *DB) ReleaseLeadership(ctx context.Context, role, owner string) error { _, err := s.db.ExecContext(ctx, `DELETE FROM leadership WHERE role = ? AND owner = ?`, role, owner) return err }