CI / test (push) Successful in 1m35s
The first thing HA needs that a shared store does not give. Two control planes against one store both reconcile: both see a model wanting two replicas and none running, and both place. The model ends up with twice what it asked for, each replica holding GPUs the other's inventory believes are free — and nothing crashes to say so. A lease in a table rather than a lock held by a connection. PostgreSQL has advisory locks and SQLite has nothing like them, and this store keeps one set of queries for both; a row with an expiry works everywhere, and it survives a leader being powered off rather than asked to stop, which is the case that matters. Taking and renewing are the same statement, so the two cannot disagree about what a held lease looks like, and it is one statement, so two processes racing cannot both believe they won. The owner id is random rather than the hostname. A hostname would let a restarted process match its own stale lease and believe it had been leading all along — including through the gap that made it restart, during which somebody else may have taken over and started placing. A store that cannot be reached is not evidence that somebody else is leading, so an unreachable store keeps the claim. Two leaders is worse than one, but no leader means nothing is placed and nothing is evicted, and a brief database blip should not stop a cluster healing itself. Unset means leader. Every deployment that existed before this, every single-control-plane install and every test acts unconditionally, with nothing to configure. The same seven assertions run against both backends, which is how the test harness was caught inheriting a lease between subtests — it passed on SQLite, where each subtest gets a fresh file, and failed only on PostgreSQL. What this does not yet do is move agents. A standby holding an agent's stream is one the leader cannot command, so failover is degraded rather than clean until agents know more than one address. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
183 lines
6.0 KiB
Go
183 lines
6.0 KiB
Go
package control
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/hex"
|
|
"errors"
|
|
"log/slog"
|
|
"os"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
// LeaseStore is the part of the store leadership needs.
|
|
//
|
|
// An interface so the controller does not take a dependency on a concrete
|
|
// database for one feature, and so a deployment with no store at all — a
|
|
// router-only run, or a single control plane on a static configuration — keeps
|
|
// working with nothing to configure.
|
|
type LeaseStore interface {
|
|
AcquireLeadership(ctx context.Context, role, owner string, ttl time.Duration) (store.Lease, error)
|
|
ReleaseLeadership(ctx context.Context, role, owner string) error
|
|
}
|
|
|
|
// controlRole is the one thing there is to be leader of: placing work.
|
|
const controlRole = "control-plane"
|
|
|
|
// Leadership decides whether this process is the one that places work.
|
|
//
|
|
// Two control planes against one store both reconcile: both see a model
|
|
// wanting two replicas and none running, and both place. The result is twice
|
|
// the replicas, each holding GPUs the other believes are free, and nothing
|
|
// crashes to say so.
|
|
//
|
|
// A standby is not idle — it serves its own routing and keeps its connection
|
|
// to the store — it simply does not place, evict or drain. That makes running
|
|
// a second one safe rather than clever, which is the property worth having
|
|
// first: the common failure is not a planned failover, it is somebody starting
|
|
// a second control plane by accident.
|
|
type Leadership struct {
|
|
store LeaseStore
|
|
log *slog.Logger
|
|
// owner identifies this process. Random rather than the hostname, so a
|
|
// restart on the same machine cannot be fooled by its own stale lease.
|
|
owner string
|
|
ttl time.Duration
|
|
// renew is how often the holder pushes the expiry out. Well under ttl, so
|
|
// one slow query does not cost the leadership.
|
|
renew time.Duration
|
|
|
|
leader atomic.Bool
|
|
}
|
|
|
|
// NewLeadership prepares leadership over a store.
|
|
//
|
|
// A nil store means there is nobody to coordinate with, and this process is
|
|
// the leader by definition — which is every single-control-plane deployment,
|
|
// and must stay the case without anyone opting in.
|
|
func NewLeadership(st LeaseStore, ttl time.Duration, log *slog.Logger) *Leadership {
|
|
if log == nil {
|
|
log = slog.Default()
|
|
}
|
|
if ttl <= 0 {
|
|
// Long enough to survive a slow database or a brief network stall,
|
|
// short enough that a standby takes over before anyone has finished
|
|
// reading the alert.
|
|
ttl = 15 * time.Second
|
|
}
|
|
l := &Leadership{
|
|
store: st,
|
|
log: log,
|
|
owner: newOwnerID(),
|
|
ttl: ttl,
|
|
renew: ttl / 3,
|
|
}
|
|
if st == nil {
|
|
l.leader.Store(true)
|
|
}
|
|
return l
|
|
}
|
|
|
|
// IsLeader reports whether this process should be placing work.
|
|
func (l *Leadership) IsLeader() bool { return l.leader.Load() }
|
|
|
|
// Owner is this process's id, for logs and for ctl.
|
|
func (l *Leadership) Owner() string { return l.owner }
|
|
|
|
// Run keeps the lease for as long as it can hold it, and keeps trying when it
|
|
// cannot.
|
|
//
|
|
// A standby polls at the same interval the leader renews at, so a lapsed lease
|
|
// is picked up within about a third of its life rather than at the end of it.
|
|
func (l *Leadership) Run(ctx context.Context) {
|
|
if l.store == nil {
|
|
return
|
|
}
|
|
t := time.NewTicker(l.renew)
|
|
defer t.Stop()
|
|
|
|
l.attempt(ctx)
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
l.step(ctx)
|
|
return
|
|
case <-t.C:
|
|
l.attempt(ctx)
|
|
}
|
|
}
|
|
}
|
|
|
|
// attempt takes or renews the lease, logging only transitions.
|
|
//
|
|
// Only transitions, because this runs every few seconds for the life of the
|
|
// process: a line per attempt would bury everything else, and the thing an
|
|
// operator needs to see is the moment leadership moved.
|
|
func (l *Leadership) attempt(ctx context.Context) {
|
|
lease, err := l.store.AcquireLeadership(ctx, controlRole, l.owner, l.ttl)
|
|
switch {
|
|
case err == nil:
|
|
if l.leader.CompareAndSwap(false, true) {
|
|
l.log.Info("this control plane is now placing work", "owner", l.owner)
|
|
}
|
|
case errors.Is(err, store.ErrNotLeader):
|
|
if l.leader.CompareAndSwap(true, false) {
|
|
// Losing a lease that was held is worth a warning: it means this
|
|
// process was too slow to renew, and whatever made it slow is
|
|
// still true.
|
|
l.log.Warn("lost leadership; standing by",
|
|
"owner", l.owner, "now", lease.Owner)
|
|
} else if l.leader.Load() == false && lease.Owner != "" {
|
|
l.log.Debug("standing by", "leader", lease.Owner,
|
|
"until", lease.ExpiresAt.Format(time.RFC3339))
|
|
}
|
|
default:
|
|
// A store that cannot be reached is not evidence that somebody else
|
|
// is leading. Keeping the claim is the safer of two bad options: two
|
|
// leaders is worse than one, but no leader at all means nothing is
|
|
// placed and nothing is evicted, and a brief database blip should not
|
|
// stop a cluster from healing itself.
|
|
l.log.Warn("could not reach the store to renew leadership; keeping it for now",
|
|
"err", err)
|
|
}
|
|
}
|
|
|
|
// step gives the lease up on the way out, so a standby takes over in seconds
|
|
// rather than waiting out the clock.
|
|
func (l *Leadership) step(ctx context.Context) {
|
|
if !l.leader.Load() {
|
|
return
|
|
}
|
|
// A fresh context: the one that just ended is the reason this is running.
|
|
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 3*time.Second)
|
|
defer cancel()
|
|
if err := l.store.ReleaseLeadership(ctx, controlRole, l.owner); err != nil {
|
|
l.log.Warn("could not release leadership; a standby will wait it out", "err", err)
|
|
return
|
|
}
|
|
l.log.Info("released leadership", "owner", l.owner)
|
|
}
|
|
|
|
// newOwnerID is random, not the hostname.
|
|
//
|
|
// A hostname would let a restarted process match its own stale lease and
|
|
// believe it had been leading all along — including through whatever gap made
|
|
// it restart, during which another process may have taken over and started
|
|
// placing.
|
|
func newOwnerID() string {
|
|
var b [8]byte
|
|
if _, err := rand.Read(b[:]); err != nil {
|
|
// Only reached if the system has no entropy, where a pid and the time
|
|
// is still better than a fixed string.
|
|
return "ctl-" + hex.EncodeToString([]byte(time.Now().Format(time.RFC3339Nano)))[:16]
|
|
}
|
|
host, _ := os.Hostname()
|
|
if host == "" {
|
|
host = "ctl"
|
|
}
|
|
return host + "-" + hex.EncodeToString(b[:])
|
|
}
|