Files
llmgw/internal/control/leadership.go
T
Ilker ManapandClaude Opus 5.5 964b659849 A stopping control plane gives up the lease before it lets its agents go
With the agent server closed rather than drained, the process could exit
before Run's release had reached the store, and the standby waited out the
lease. The lease is now given up first, synchronously, and nothing takes it
again after.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-05 02:31:13 +02:00

221 lines
7.2 KiB
Go

package control
import (
"context"
"crypto/rand"
"encoding/hex"
"errors"
"log/slog"
"os"
"strings"
"sync/atomic"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// LeaseStore is the part of the store leadership needs.
//
// An interface so the controller does not take a dependency on a concrete
// database for one feature, and so a deployment with no store at all — a
// router-only run, or a single control plane on a static configuration — keeps
// working with nothing to configure.
type LeaseStore interface {
AcquireLeadership(ctx context.Context, role, owner string, ttl time.Duration) (store.Lease, error)
ReleaseLeadership(ctx context.Context, role, owner string) error
}
// controlRole is the one thing there is to be leader of: placing work.
const controlRole = "control-plane"
// Leadership decides whether this process is the one that places work.
//
// Two control planes against one store both reconcile: both see a model
// wanting two replicas and none running, and both place. The result is twice
// the replicas, each holding GPUs the other believes are free, and nothing
// crashes to say so.
//
// A standby is not idle — it serves its own routing and keeps its connection
// to the store — it simply does not place, evict or drain. That makes running
// a second one safe rather than clever, which is the property worth having
// first: the common failure is not a planned failover, it is somebody starting
// a second control plane by accident.
type Leadership struct {
store LeaseStore
log *slog.Logger
// owner identifies this process. Random rather than the hostname, so a
// restart on the same machine cannot be fooled by its own stale lease.
owner string
ttl time.Duration
// renew is how often the holder pushes the expiry out. Well under ttl, so
// one slow query does not cost the leadership.
renew time.Duration
leader atomic.Bool
// stopped is set by StepDown: nothing takes the lease again after it.
stopped atomic.Bool
}
// NewLeadership prepares leadership over a store.
//
// A nil store means there is nobody to coordinate with, and this process is
// the leader by definition — which is every single-control-plane deployment,
// and must stay the case without anyone opting in.
func NewLeadership(st LeaseStore, ttl time.Duration, log *slog.Logger) *Leadership {
if log == nil {
log = slog.Default()
}
if ttl <= 0 {
// Long enough to survive a slow database or a brief network stall,
// short enough that a standby takes over before anyone has finished
// reading the alert.
ttl = 15 * time.Second
}
l := &Leadership{
store: st,
log: log,
owner: newOwnerID(),
ttl: ttl,
renew: ttl / 3,
}
if st == nil {
l.leader.Store(true)
}
return l
}
// IsLeader reports whether this process should be placing work.
func (l *Leadership) IsLeader() bool { return l.leader.Load() }
// Owner is this process's id, for logs and for ctl.
func (l *Leadership) Owner() string { return l.owner }
// Run keeps the lease for as long as it can hold it, and keeps trying when it
// cannot.
//
// A standby polls at the same interval the leader renews at, so a lapsed lease
// is picked up within about a third of its life rather than at the end of it.
func (l *Leadership) Run(ctx context.Context) {
if l.store == nil {
return
}
t := time.NewTicker(l.renew)
defer t.Stop()
l.attempt(ctx)
for {
select {
case <-ctx.Done():
l.step(ctx)
return
case <-t.C:
l.attempt(ctx)
}
}
}
// attempt takes or renews the lease, logging only transitions.
//
// Only transitions, because this runs every few seconds for the life of the
// process: a line per attempt would bury everything else, and the thing an
// operator needs to see is the moment leadership moved.
func (l *Leadership) attempt(ctx context.Context) {
if l.stopped.Load() {
return
}
lease, err := l.store.AcquireLeadership(ctx, controlRole, l.owner, l.ttl)
switch {
case err == nil && l.stopped.Load():
// Stepped down while this was on its way to the store: give back
// what it just took.
l.leader.Store(true)
l.step(ctx)
case err == nil:
if l.leader.CompareAndSwap(false, true) {
l.log.Info("this control plane is now placing work", "owner", l.owner)
}
case errors.Is(err, store.ErrNotLeader):
if l.leader.CompareAndSwap(true, false) {
// Losing a lease that was held is worth a warning: it means this
// process was too slow to renew, and whatever made it slow is
// still true.
l.log.Warn("lost leadership; standing by",
"owner", l.owner, "now", lease.Owner)
} else if l.leader.Load() == false && lease.Owner != "" {
l.log.Debug("standing by", "leader", lease.Owner,
"until", lease.ExpiresAt.Format(time.RFC3339))
}
default:
// A store that cannot be reached is not evidence that somebody else
// is leading. Keeping the claim is the safer of two bad options: two
// leaders is worse than one, but no leader at all means nothing is
// placed and nothing is evicted, and a brief database blip should not
// stop a cluster from healing itself.
l.log.Warn("could not reach the store to renew leadership; keeping it for now",
"err", err)
}
}
// StepDown gives the lease up now, for a process on its way out: called
// before the agents are let go, so the standby they move to is already the
// leader, and before the process exits, which would otherwise race the
// release in Run and leave the standby to wait out the lease. Run's own
// release after it is then nothing.
func (l *Leadership) StepDown() {
if l.store == nil {
return
}
l.stopped.Store(true)
l.step(context.Background())
}
// step gives the lease up on the way out, so a standby takes over in seconds
// rather than waiting out the clock.
func (l *Leadership) step(ctx context.Context) {
// Once: StepDown and Run's own way out both come here.
if !l.leader.CompareAndSwap(true, false) {
return
}
// A fresh context: the one that just ended is the reason this is running.
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 3*time.Second)
defer cancel()
if err := l.store.ReleaseLeadership(ctx, controlRole, l.owner); err != nil {
l.log.Warn("could not release leadership; a standby will wait it out", "err", err)
return
}
l.log.Info("released leadership", "owner", l.owner)
}
// newOwnerID is random, not the hostname.
//
// A hostname would let a restarted process match its own stale lease and
// believe it had been leading all along — including through whatever gap made
// it restart, during which another process may have taken over and started
// placing.
func newOwnerID() string {
var b [8]byte
if _, err := rand.Read(b[:]); err != nil {
// Only reached if the system has no entropy, where a pid and the time
// is still better than a fixed string.
return "ctl-" + hex.EncodeToString([]byte(time.Now().Format(time.RFC3339Nano)))[:16]
}
host, _ := os.Hostname()
if host == "" {
host = "ctl"
}
return host + "-" + hex.EncodeToString(b[:])
}
// ControlRole is the lease's name in the store, for readers that ask who
// holds it without competing.
const ControlRole = controlRole
// HolderName is the machine an owner id names: the hostname it was made
// from, without the random part.
func HolderName(owner string) string {
if i := strings.LastIndexByte(owner, '-'); i > 0 {
return owner[:i]
}
return owner
}