Files
llmgw/internal/control/leadership.go
T
Ilker ManapandClaude Opus 5.5 b09922b29b
CI / test (push) Failing after 30m3s
Leadership stays put while the store's primary is replaced
Measured again after the store fixes. Cutting the store primary's power no
longer hung requests, but leadership moved anyway: the lease had lapsed for
everyone while Patroni promoted the standby, and a standby asked first. The
new leader then spent about 25 s without its nodes, and so without its
judges.

A standby that saw the store out of reach now leaves the lease alone for one
renew round after the store returns. The default lease is 10 s, and a node
detects a control plane that died silently in 8 s (5 s idle, 3 s ping)
rather than 15.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-15 23:17:58 +02:00

299 lines
9.6 KiB
Go

package control
import (
"context"
"crypto/rand"
"database/sql"
"encoding/hex"
"errors"
"log/slog"
"os"
"slices"
"strings"
"sync"
"sync/atomic"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// LeaseStore is the part of the store leadership needs.
//
// An interface so the controller does not take a dependency on a concrete
// database for one feature, and so a deployment with no store at all — a
// router-only run, or a single control plane on a static configuration — keeps
// working with nothing to configure.
type LeaseStore interface {
AcquireLeadership(ctx context.Context, role, owner string, ttl time.Duration) (store.Lease, error)
ReleaseLeadership(ctx context.Context, role, owner string) error
}
// controlRole is the one thing there is to be leader of: placing work.
const controlRole = "control-plane"
// Leadership decides whether this process is the one that places work.
//
// Two control planes against one store both reconcile: both see a model
// wanting two replicas and none running, and both place. The result is twice
// the replicas, each holding GPUs the other believes are free, and nothing
// crashes to say so.
//
// A standby is not idle — it serves its own routing and keeps its connection
// to the store — it simply does not place, evict or drain. That makes running
// a second one safe rather than clever, which is the property worth having
// first: the common failure is not a planned failover, it is somebody starting
// a second control plane by accident.
type Leadership struct {
store LeaseStore
log *slog.Logger
// owner identifies this process. Random rather than the hostname, so a
// restart on the same machine cannot be fooled by its own stale lease.
owner string
ttl time.Duration
// renew is how often the holder pushes the expiry out. Well under ttl, so
// one slow query does not cost the leadership.
renew time.Duration
leader atomic.Bool
// stopped is set by StepDown: nothing takes the lease again after it.
stopped atomic.Bool
hookMu sync.Mutex
onLead []func()
// cut is a store this process could not reach, and back when it was
// reached again: Run's alone, so unlocked.
cut bool
back time.Time
}
// LeaseReader reads who holds a lease without competing for it: a store
// that has it lets a standby hold back after the store returns.
type LeaseReader interface {
Leadership(ctx context.Context, role string) (store.Lease, error)
}
// OnLead is called each time this process becomes the leader, after
// standing by: what it read at start may have been changed since by the
// leader before it, and is read again.
func (l *Leadership) OnLead(f func()) {
l.hookMu.Lock()
l.onLead = append(l.onLead, f)
l.hookMu.Unlock()
}
func (l *Leadership) becameLeader() {
l.hookMu.Lock()
hooks := slices.Clone(l.onLead)
l.hookMu.Unlock()
for _, f := range hooks {
go f()
}
}
// NewLeadership prepares leadership over a store.
//
// A nil store means there is nobody to coordinate with, and this process is
// the leader by definition — which is every single-control-plane deployment,
// and must stay the case without anyone opting in.
func NewLeadership(st LeaseStore, ttl time.Duration, log *slog.Logger) *Leadership {
if log == nil {
log = slog.Default()
}
if ttl <= 0 {
// Long enough to survive a slow database or a brief network stall,
// short enough that a standby takes over before anyone has finished
// reading the alert. A store out of reach for longer — its primary
// being replaced — does not move it: see holdBack.
ttl = 10 * time.Second
}
l := &Leadership{
store: st,
log: log,
owner: newOwnerID(),
ttl: ttl,
renew: ttl / 3,
}
if st == nil {
l.leader.Store(true)
}
return l
}
// IsLeader reports whether this process should be placing work.
func (l *Leadership) IsLeader() bool { return l.leader.Load() }
// Owner is this process's id, for logs and for ctl.
func (l *Leadership) Owner() string { return l.owner }
// Run keeps the lease for as long as it can hold it, and keeps trying when it
// cannot.
//
// A standby polls at the same interval the leader renews at, so a lapsed lease
// is picked up within about a third of its life rather than at the end of it.
func (l *Leadership) Run(ctx context.Context) {
if l.store == nil {
return
}
t := time.NewTicker(l.renew)
defer t.Stop()
l.attempt(ctx)
for {
select {
case <-ctx.Done():
l.step(ctx)
return
case <-t.C:
l.attempt(ctx)
}
}
}
// attempt takes or renews the lease, logging only transitions.
//
// Only transitions, because this runs every few seconds for the life of the
// process: a line per attempt would bury everything else, and the thing an
// operator needs to see is the moment leadership moved.
func (l *Leadership) attempt(ctx context.Context) {
if l.stopped.Load() {
return
}
if l.holdBack(ctx) {
return
}
lease, err := l.store.AcquireLeadership(ctx, controlRole, l.owner, l.ttl)
switch {
case err == nil && l.stopped.Load():
// Stepped down while this was on its way to the store: give back
// what it just took.
l.leader.Store(true)
l.step(ctx)
case err == nil:
if l.leader.CompareAndSwap(false, true) {
l.log.Info("this control plane is now placing work", "owner", l.owner)
l.becameLeader()
}
case errors.Is(err, store.ErrNotLeader):
if l.leader.CompareAndSwap(true, false) {
// Losing a lease that was held is worth a warning: it means this
// process was too slow to renew, and whatever made it slow is
// still true.
l.log.Warn("lost leadership; standing by",
"owner", l.owner, "now", lease.Owner)
} else if l.leader.Load() == false && lease.Owner != "" {
l.log.Debug("standing by", "leader", lease.Owner,
"until", lease.ExpiresAt.Format(time.RFC3339))
}
default:
// A store that cannot be reached is not evidence that somebody else
// is leading. Keeping the claim is the safer of two bad options: two
// leaders is worse than one, but no leader at all means nothing is
// placed and nothing is evicted, and a brief database blip should not
// stop a cluster from healing itself.
l.log.Warn("could not reach the store to renew leadership; keeping it for now",
"err", err)
l.cut, l.back = true, time.Time{}
}
}
// holdBack is a standby leaving a lapsed lease alone for one round after the
// store returns from being out of reach.
//
// A store's primary that is lost takes longer to replace than the lease
// lives, so the lease has lapsed by the time the store answers again — for
// the leader as for the standbys, which all failed to reach it alike. The
// first to ask then took it, and it was as often a standby as the leader:
// leadership moved for nothing, and the new leader spent half a minute
// learning from the agents where the judges were, refusing requests
// meanwhile. The leader renews within a round of the store's return, so a
// standby that waits that round takes the lease only from a leader that is
// gone.
func (l *Leadership) holdBack(ctx context.Context) bool {
if l.leader.Load() || !l.cut {
return false
}
r, ok := l.store.(LeaseReader)
if !ok {
l.cut = false
return false
}
if _, err := r.Leadership(ctx, controlRole); err != nil && !errors.Is(err, sql.ErrNoRows) {
l.back = time.Time{}
return true
}
if l.back.IsZero() {
l.back = time.Now()
l.log.Info("the store answers again; leaving the lease to its holder for a round", "owner", l.owner)
}
// A round less a tenth, for the ticker's own jitter.
if time.Since(l.back) < l.renew-l.renew/10 {
return true
}
l.cut, l.back = false, time.Time{}
return false
}
// StepDown gives the lease up now, for a process on its way out: called
// before the agents are let go, so the standby they move to is already the
// leader, and before the process exits, which would otherwise race the
// release in Run and leave the standby to wait out the lease. Run's own
// release after it is then nothing.
func (l *Leadership) StepDown() {
if l.store == nil {
return
}
l.stopped.Store(true)
l.step(context.Background())
}
// step gives the lease up on the way out, so a standby takes over in seconds
// rather than waiting out the clock.
func (l *Leadership) step(ctx context.Context) {
// Once: StepDown and Run's own way out both come here.
if !l.leader.CompareAndSwap(true, false) {
return
}
// A fresh context: the one that just ended is the reason this is running.
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 3*time.Second)
defer cancel()
if err := l.store.ReleaseLeadership(ctx, controlRole, l.owner); err != nil {
l.log.Warn("could not release leadership; a standby will wait it out", "err", err)
return
}
l.log.Info("released leadership", "owner", l.owner)
}
// newOwnerID is random, not the hostname.
//
// A hostname would let a restarted process match its own stale lease and
// believe it had been leading all along — including through whatever gap made
// it restart, during which another process may have taken over and started
// placing.
func newOwnerID() string {
var b [8]byte
if _, err := rand.Read(b[:]); err != nil {
// Only reached if the system has no entropy, where a pid and the time
// is still better than a fixed string.
return "ctl-" + hex.EncodeToString([]byte(time.Now().Format(time.RFC3339Nano)))[:16]
}
host, _ := os.Hostname()
if host == "" {
host = "ctl"
}
return host + "-" + hex.EncodeToString(b[:])
}
// ControlRole is the lease's name in the store, for readers that ask who
// holds it without competing.
const ControlRole = controlRole
// HolderName is the machine an owner id names: the hostname it was made
// from, without the random part.
func HolderName(owner string) string {
if i := strings.LastIndexByte(owner, '-'); i > 0 {
return owner[:i]
}
return owner
}