A control plane being stopped held every agent's stream for the thirty seconds of a graceful shutdown, so the standby led within four seconds and had no nodes for thirty. The agent server is now closed, not drained. A control plane that loses its lease while it lives on lets its agents go, rather than keeping nodes the new leader cannot reach. ctl status and the console's Overview say which control plane leads, which stand by, and whether the floating address is on the leader. The console adds a control plane (Do → Add a control plane), asking for the machine and the floating address. server add says when the mirror would refuse the new control plane. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
186 lines
6.2 KiB
Go
186 lines
6.2 KiB
Go
package control
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/hex"
|
|
"errors"
|
|
"log/slog"
|
|
"os"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
// LeaseStore is the part of the store leadership needs.
|
|
//
|
|
// An interface so the controller does not take a dependency on a concrete
|
|
// database for one feature, and so a deployment with no store at all — a
|
|
// router-only run, or a single control plane on a static configuration — keeps
|
|
// working with nothing to configure.
|
|
type LeaseStore interface {
|
|
AcquireLeadership(ctx context.Context, role, owner string, ttl time.Duration) (store.Lease, error)
|
|
ReleaseLeadership(ctx context.Context, role, owner string) error
|
|
}
|
|
|
|
// controlRole is the one thing there is to be leader of: placing work.
|
|
const controlRole = "control-plane"
|
|
|
|
// Leadership decides whether this process is the one that places work.
|
|
//
|
|
// Two control planes against one store both reconcile: both see a model
|
|
// wanting two replicas and none running, and both place. The result is twice
|
|
// the replicas, each holding GPUs the other believes are free, and nothing
|
|
// crashes to say so.
|
|
//
|
|
// A standby is not idle — it serves its own routing and keeps its connection
|
|
// to the store — it simply does not place, evict or drain. That makes running
|
|
// a second one safe rather than clever, which is the property worth having
|
|
// first: the common failure is not a planned failover, it is somebody starting
|
|
// a second control plane by accident.
|
|
type Leadership struct {
|
|
store LeaseStore
|
|
log *slog.Logger
|
|
// owner identifies this process. Random rather than the hostname, so a
|
|
// restart on the same machine cannot be fooled by its own stale lease.
|
|
owner string
|
|
ttl time.Duration
|
|
// renew is how often the holder pushes the expiry out. Well under ttl, so
|
|
// one slow query does not cost the leadership.
|
|
renew time.Duration
|
|
|
|
leader atomic.Bool
|
|
}
|
|
|
|
// NewLeadership prepares leadership over a store.
|
|
//
|
|
// A nil store means there is nobody to coordinate with, and this process is
|
|
// the leader by definition — which is every single-control-plane deployment,
|
|
// and must stay the case without anyone opting in.
|
|
func NewLeadership(st LeaseStore, ttl time.Duration, log *slog.Logger) *Leadership {
|
|
if log == nil {
|
|
log = slog.Default()
|
|
}
|
|
if ttl <= 0 {
|
|
// Long enough to survive a slow database or a brief network stall,
|
|
// short enough that a standby takes over before anyone has finished
|
|
// reading the alert.
|
|
ttl = 15 * time.Second
|
|
}
|
|
l := &Leadership{
|
|
store: st,
|
|
log: log,
|
|
owner: newOwnerID(),
|
|
ttl: ttl,
|
|
renew: ttl / 3,
|
|
}
|
|
if st == nil {
|
|
l.leader.Store(true)
|
|
}
|
|
return l
|
|
}
|
|
|
|
// IsLeader reports whether this process should be placing work.
|
|
func (l *Leadership) IsLeader() bool { return l.leader.Load() }
|
|
|
|
// Owner is this process's id, for logs and for ctl.
|
|
func (l *Leadership) Owner() string { return l.owner }
|
|
|
|
// Run keeps the lease for as long as it can hold it, and keeps trying when it
|
|
// cannot.
|
|
//
|
|
// A standby polls at the same interval the leader renews at, so a lapsed lease
|
|
// is picked up within about a third of its life rather than at the end of it.
|
|
func (l *Leadership) Run(ctx context.Context) {
|
|
if l.store == nil {
|
|
return
|
|
}
|
|
t := time.NewTicker(l.renew)
|
|
defer t.Stop()
|
|
|
|
l.attempt(ctx)
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
l.step(ctx)
|
|
return
|
|
case <-t.C:
|
|
l.attempt(ctx)
|
|
}
|
|
}
|
|
}
|
|
|
|
// attempt takes or renews the lease, logging only transitions.
|
|
//
|
|
// Only transitions, because this runs every few seconds for the life of the
|
|
// process: a line per attempt would bury everything else, and the thing an
|
|
// operator needs to see is the moment leadership moved.
|
|
func (l *Leadership) attempt(ctx context.Context) {
|
|
lease, err := l.store.AcquireLeadership(ctx, controlRole, l.owner, l.ttl)
|
|
switch {
|
|
case err == nil:
|
|
if l.leader.CompareAndSwap(false, true) {
|
|
l.log.Info("this control plane is now placing work", "owner", l.owner)
|
|
}
|
|
case errors.Is(err, store.ErrNotLeader):
|
|
if l.leader.CompareAndSwap(true, false) {
|
|
// Losing a lease that was held is worth a warning: it means this
|
|
// process was too slow to renew, and whatever made it slow is
|
|
// still true.
|
|
l.log.Warn("lost leadership; standing by",
|
|
"owner", l.owner, "now", lease.Owner)
|
|
} else if l.leader.Load() == false && lease.Owner != "" {
|
|
l.log.Debug("standing by", "leader", lease.Owner,
|
|
"until", lease.ExpiresAt.Format(time.RFC3339))
|
|
}
|
|
default:
|
|
// A store that cannot be reached is not evidence that somebody else
|
|
// is leading. Keeping the claim is the safer of two bad options: two
|
|
// leaders is worse than one, but no leader at all means nothing is
|
|
// placed and nothing is evicted, and a brief database blip should not
|
|
// stop a cluster from healing itself.
|
|
l.log.Warn("could not reach the store to renew leadership; keeping it for now",
|
|
"err", err)
|
|
}
|
|
}
|
|
|
|
// step gives the lease up on the way out, so a standby takes over in seconds
|
|
// rather than waiting out the clock.
|
|
func (l *Leadership) step(ctx context.Context) {
|
|
if !l.leader.Load() {
|
|
return
|
|
}
|
|
// A fresh context: the one that just ended is the reason this is running.
|
|
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 3*time.Second)
|
|
defer cancel()
|
|
if err := l.store.ReleaseLeadership(ctx, controlRole, l.owner); err != nil {
|
|
l.log.Warn("could not release leadership; a standby will wait it out", "err", err)
|
|
return
|
|
}
|
|
// Not leading from here on, so nothing in this process goes on acting as
|
|
// if it were while it shuts down.
|
|
l.leader.Store(false)
|
|
l.log.Info("released leadership", "owner", l.owner)
|
|
}
|
|
|
|
// newOwnerID is random, not the hostname.
|
|
//
|
|
// A hostname would let a restarted process match its own stale lease and
|
|
// believe it had been leading all along — including through whatever gap made
|
|
// it restart, during which another process may have taken over and started
|
|
// placing.
|
|
func newOwnerID() string {
|
|
var b [8]byte
|
|
if _, err := rand.Read(b[:]); err != nil {
|
|
// Only reached if the system has no entropy, where a pid and the time
|
|
// is still better than a fixed string.
|
|
return "ctl-" + hex.EncodeToString([]byte(time.Now().Format(time.RFC3339Nano)))[:16]
|
|
}
|
|
host, _ := os.Hostname()
|
|
if host == "" {
|
|
host = "ctl"
|
|
}
|
|
return host + "-" + hex.EncodeToString(b[:])
|
|
}
|