diff --git a/internal/cli/server.go b/internal/cli/server.go index 904342b..e58efb3 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -28,6 +28,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelsvc" "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" "gitea.mynodes.xyz/ilker/llmgw/internal/router" + storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store" "gitea.mynodes.xyz/ilker/llmgw/internal/version" "gitea.mynodes.xyz/ilker/llmgw/internal/web" ) @@ -335,6 +336,20 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live if u, ok := sink.(control.UsageSource); ok { ctl.SetUsageSource(u) } + // With a shared store, two control planes can see each other, so this one + // competes rather than assuming it is alone. Without one there is nobody + // to compete with and it places unconditionally, exactly as before. + // The same shape as the usage assertion above, and for the same reason: + // the sink is the router when the router is serving, and nothing when a + // control plane runs on its own. + if h, ok := sink.(interface{ Store() *storepkg.SQLite }); ok { + if st := h.Store(); st != nil { + lead := control.NewLeadership(st, cfg.Control.LeaseTTL, log) + ctl.SetLeadership(lead) + log.Info("competing to be the control plane that places work", + "owner", lead.Owner()) + } + } svc := agentsvc.New(ctl.Inventory(), ctl.Reconciler(), requireIdentity, log) fwd.set(svc) diff --git a/internal/config/config.go b/internal/config/config.go index 6071484..05858c8 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -177,6 +177,15 @@ type Control struct { PortRange []int `koanf:"port_range"` // MgmtInterface is the ethernet interface NCCL bootstraps over. MgmtInterface string `koanf:"mgmt_interface"` + + // LeaseTTL is how long this control plane's claim to be the one placing + // work stands without being renewed. + // + // Only meaningful with a shared store, where a second control plane can + // see the claim. Long enough to survive a slow query or a brief network + // stall, short enough that a standby takes over before anyone has finished + // reading the alert. Zero takes the default. + LeaseTTL time.Duration `koanf:"lease_ttl"` } // Router holds routing behaviour. diff --git a/internal/control/control.go b/internal/control/control.go index 77ef90b..00b08be 100644 --- a/internal/control/control.go +++ b/internal/control/control.go @@ -75,6 +75,7 @@ type Controller struct { inv *inventory.Store cat *catalog.Live rec *reconciler.Reconciler + lead *Leadership disp reconciler.Dispatcher sink EndpointSink usage UsageSource @@ -172,6 +173,22 @@ func (c *Controller) Reconciler() *reconciler.Reconciler { return c.rec } // Catalog exposes the model catalog. func (c *Controller) Catalog() *catalog.Catalog { return c.cat.Load() } +// SetLeadership makes this control plane compete for the right to place work, +// instead of assuming it is the only one. +// +// Wired after construction because deciding leadership needs a store, and the +// store is opened by the router. Left unset, this process places +// unconditionally — which is what a single control plane wants, and what every +// deployment that existed before this had. +func (c *Controller) SetLeadership(l *Leadership) { + c.lead = l + c.rec.SetLeadership(l) +} + +// IsLeader reports whether this process is placing work. True when leadership +// was never configured, because then there is nobody to compete with. +func (c *Controller) IsLeader() bool { return c.lead == nil || c.lead.IsLeader() } + // WeightRemover can ask a node to delete cached model files. // // Separate from Dispatcher because it is not part of scheduling and not every @@ -189,6 +206,9 @@ func (c *Controller) WeightRemover() WeightRemover { // Start runs the reconciler and begins publishing endpoints. func (c *Controller) Start(ctx context.Context) { + if c.lead != nil { + go c.lead.Run(ctx) + } c.rec.Start(ctx) go func() { diff --git a/internal/control/leadership.go b/internal/control/leadership.go new file mode 100644 index 0000000..178b32d --- /dev/null +++ b/internal/control/leadership.go @@ -0,0 +1,182 @@ +package control + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "log/slog" + "os" + "sync/atomic" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// LeaseStore is the part of the store leadership needs. +// +// An interface so the controller does not take a dependency on a concrete +// database for one feature, and so a deployment with no store at all — a +// router-only run, or a single control plane on a static configuration — keeps +// working with nothing to configure. +type LeaseStore interface { + AcquireLeadership(ctx context.Context, role, owner string, ttl time.Duration) (store.Lease, error) + ReleaseLeadership(ctx context.Context, role, owner string) error +} + +// controlRole is the one thing there is to be leader of: placing work. +const controlRole = "control-plane" + +// Leadership decides whether this process is the one that places work. +// +// Two control planes against one store both reconcile: both see a model +// wanting two replicas and none running, and both place. The result is twice +// the replicas, each holding GPUs the other believes are free, and nothing +// crashes to say so. +// +// A standby is not idle — it serves its own routing and keeps its connection +// to the store — it simply does not place, evict or drain. That makes running +// a second one safe rather than clever, which is the property worth having +// first: the common failure is not a planned failover, it is somebody starting +// a second control plane by accident. +type Leadership struct { + store LeaseStore + log *slog.Logger + // owner identifies this process. Random rather than the hostname, so a + // restart on the same machine cannot be fooled by its own stale lease. + owner string + ttl time.Duration + // renew is how often the holder pushes the expiry out. Well under ttl, so + // one slow query does not cost the leadership. + renew time.Duration + + leader atomic.Bool +} + +// NewLeadership prepares leadership over a store. +// +// A nil store means there is nobody to coordinate with, and this process is +// the leader by definition — which is every single-control-plane deployment, +// and must stay the case without anyone opting in. +func NewLeadership(st LeaseStore, ttl time.Duration, log *slog.Logger) *Leadership { + if log == nil { + log = slog.Default() + } + if ttl <= 0 { + // Long enough to survive a slow database or a brief network stall, + // short enough that a standby takes over before anyone has finished + // reading the alert. + ttl = 15 * time.Second + } + l := &Leadership{ + store: st, + log: log, + owner: newOwnerID(), + ttl: ttl, + renew: ttl / 3, + } + if st == nil { + l.leader.Store(true) + } + return l +} + +// IsLeader reports whether this process should be placing work. +func (l *Leadership) IsLeader() bool { return l.leader.Load() } + +// Owner is this process's id, for logs and for ctl. +func (l *Leadership) Owner() string { return l.owner } + +// Run keeps the lease for as long as it can hold it, and keeps trying when it +// cannot. +// +// A standby polls at the same interval the leader renews at, so a lapsed lease +// is picked up within about a third of its life rather than at the end of it. +func (l *Leadership) Run(ctx context.Context) { + if l.store == nil { + return + } + t := time.NewTicker(l.renew) + defer t.Stop() + + l.attempt(ctx) + for { + select { + case <-ctx.Done(): + l.step(ctx) + return + case <-t.C: + l.attempt(ctx) + } + } +} + +// attempt takes or renews the lease, logging only transitions. +// +// Only transitions, because this runs every few seconds for the life of the +// process: a line per attempt would bury everything else, and the thing an +// operator needs to see is the moment leadership moved. +func (l *Leadership) attempt(ctx context.Context) { + lease, err := l.store.AcquireLeadership(ctx, controlRole, l.owner, l.ttl) + switch { + case err == nil: + if l.leader.CompareAndSwap(false, true) { + l.log.Info("this control plane is now placing work", "owner", l.owner) + } + case errors.Is(err, store.ErrNotLeader): + if l.leader.CompareAndSwap(true, false) { + // Losing a lease that was held is worth a warning: it means this + // process was too slow to renew, and whatever made it slow is + // still true. + l.log.Warn("lost leadership; standing by", + "owner", l.owner, "now", lease.Owner) + } else if l.leader.Load() == false && lease.Owner != "" { + l.log.Debug("standing by", "leader", lease.Owner, + "until", lease.ExpiresAt.Format(time.RFC3339)) + } + default: + // A store that cannot be reached is not evidence that somebody else + // is leading. Keeping the claim is the safer of two bad options: two + // leaders is worse than one, but no leader at all means nothing is + // placed and nothing is evicted, and a brief database blip should not + // stop a cluster from healing itself. + l.log.Warn("could not reach the store to renew leadership; keeping it for now", + "err", err) + } +} + +// step gives the lease up on the way out, so a standby takes over in seconds +// rather than waiting out the clock. +func (l *Leadership) step(ctx context.Context) { + if !l.leader.Load() { + return + } + // A fresh context: the one that just ended is the reason this is running. + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 3*time.Second) + defer cancel() + if err := l.store.ReleaseLeadership(ctx, controlRole, l.owner); err != nil { + l.log.Warn("could not release leadership; a standby will wait it out", "err", err) + return + } + l.log.Info("released leadership", "owner", l.owner) +} + +// newOwnerID is random, not the hostname. +// +// A hostname would let a restarted process match its own stale lease and +// believe it had been leading all along — including through whatever gap made +// it restart, during which another process may have taken over and started +// placing. +func newOwnerID() string { + var b [8]byte + if _, err := rand.Read(b[:]); err != nil { + // Only reached if the system has no entropy, where a pid and the time + // is still better than a fixed string. + return "ctl-" + hex.EncodeToString([]byte(time.Now().Format(time.RFC3339Nano)))[:16] + } + host, _ := os.Hostname() + if host == "" { + host = "ctl" + } + return host + "-" + hex.EncodeToString(b[:]) +} diff --git a/internal/control/reconciler/leadership_test.go b/internal/control/reconciler/leadership_test.go new file mode 100644 index 0000000..89c0f7e --- /dev/null +++ b/internal/control/reconciler/leadership_test.go @@ -0,0 +1,48 @@ +package reconciler + +import ( + "testing" +) + +type fixedLeader bool + +func (f fixedLeader) IsLeader() bool { return bool(f) } + +func TestAStandbyPlacesNothing(t *testing.T) { + // The failure this prevents: two control planes against one store both see + // a model wanting replicas and none running, and both place. The model + // ends up with twice what it asked for, each replica holding GPUs the + // other's inventory believes are free, and nothing crashes to say so. + h := newHarness(t, testModel(1), gpuNode("n1", 8)) + h.r.SetLeadership(fixedLeader(false)) + + h.r.Reconcile(t.Context()) + + if got := len(h.r.Replicas()); got != 0 { + t.Errorf("a standby placed %d replicas", got) + } +} + +func TestALeaderPlacesAsBefore(t *testing.T) { + h := newHarness(t, testModel(1), gpuNode("n1", 8)) + h.r.SetLeadership(fixedLeader(true)) + + h.r.Reconcile(t.Context()) + + if got := len(h.r.Replicas()); got == 0 { + t.Error("a leader placed nothing") + } +} + +func TestWithoutLeadershipConfiguredNothingChanges(t *testing.T) { + // Every single-control-plane deployment, and every test written before + // leadership existed. Unset must mean "act", not "wait for permission + // from a thing that was never wired up". + h := newHarness(t, testModel(1), gpuNode("n1", 8)) + + h.r.Reconcile(t.Context()) + + if got := len(h.r.Replicas()); got == 0 { + t.Error("a reconciler with no leadership configured placed nothing") + } +} diff --git a/internal/control/reconciler/reconciler.go b/internal/control/reconciler/reconciler.go index 71c41dd..d870a5f 100644 --- a/internal/control/reconciler/reconciler.go +++ b/internal/control/reconciler/reconciler.go @@ -166,6 +166,9 @@ func (c *Config) normalize() { // Reconciler compares the catalog against what is running and closes the gap. type Reconciler struct { + // leader gates every action; nil means act unconditionally. + leader Leader + cfg Config cat *catalog.Live inv *inventory.Store @@ -235,6 +238,17 @@ func (r *Reconciler) SetClock(now func() time.Time) { r.startedAt = now() } +// Leader reports whether this process should be acting on the fleet. +type Leader interface{ IsLeader() bool } + +// SetLeadership gates every action on holding leadership. +// +// A setter rather than a constructor argument because the thing that decides +// leadership needs a store, and the store is wired after the controller. Nil +// leaves the reconciler acting unconditionally, which is what a single control +// plane wants and what every test expects. +func (r *Reconciler) SetLeadership(l Leader) { r.leader = l } + // Start runs the loop until Close. func (r *Reconciler) Start(ctx context.Context) { go func() { @@ -278,6 +292,16 @@ func (r *Reconciler) Notify() { // Reconcile performs one pass. It is exported so tests can drive it directly // instead of racing a ticker. func (r *Reconciler) Reconcile(ctx context.Context) { + // A standby does nothing at all here. Everything below issues commands to + // agents — placing, evicting, draining, reaping — and two control planes + // doing that against one fleet is how a model ends up with twice the + // replicas it asked for, each holding GPUs the other believes are free. + // + // Unset means leader, so a single control plane and every test behave + // exactly as they did before leadership existed. + if r.leader != nil && !r.leader.IsLeader() { + return + } r.expireNodes() r.expireStuckReplicas() r.drainCordonedNodes(ctx) diff --git a/internal/store/copy.go b/internal/store/copy.go index 6110a31..7149beb 100644 --- a/internal/store/copy.go +++ b/internal/store/copy.go @@ -34,6 +34,17 @@ var copyOrder = []string{ "memory_notes", } +// notCopied are tables deliberately left behind. +// +// Leadership is a claim about a process that is running right now. Copying it +// would hand the new store a leader that does not exist, and the standby +// waiting on that lease would wait out its full expiry before noticing — at +// the exact moment somebody is watching a migration and wondering why nothing +// is being placed. +var notCopied = map[string]string{ + "leadership": "a lease on a running process, meaningless in a new store", +} + // CopyResult is how many rows each table received. type CopyResult struct { Table string diff --git a/internal/store/copy_test.go b/internal/store/copy_test.go index fe701b0..050a92a 100644 --- a/internal/store/copy_test.go +++ b/internal/store/copy_test.go @@ -41,9 +41,17 @@ func TestEveryTableIsInTheCopyOrder(t *testing.T) { t.Fatal("found no tables in the migrations, so this test proves nothing") } for _, name := range declared { - if !slices.Contains(copyOrder, name) { - t.Errorf("%q is created by a migration but nothing copies it", name) + if slices.Contains(copyOrder, name) { + continue } + // Left behind on purpose, with the reason written down beside it. + if why, ok := notCopied[name]; ok { + if why == "" { + t.Errorf("%q is excluded from the copy with no reason given", name) + } + continue + } + t.Errorf("%q is created by a migration but nothing copies it", name) } } diff --git a/internal/store/leadership.go b/internal/store/leadership.go new file mode 100644 index 0000000..1b999c9 --- /dev/null +++ b/internal/store/leadership.go @@ -0,0 +1,96 @@ +package store + +import ( + "context" + "errors" + "time" +) + +// ErrNotLeader means somebody else holds the lease and it has not lapsed. +var ErrNotLeader = errors.New("store: another control plane holds the lease") + +// Lease is who is in charge and until when. +type Lease struct { + Owner string + ExpiresAt time.Time + AcquiredAt time.Time +} + +// AcquireLeadership takes or renews the lease for a role. +// +// Renewing and taking are the same statement on purpose: a leader that renews +// takes the same path as one starting fresh, so there is no way for the two to +// disagree about what a held lease looks like. It succeeds when the row is +// absent, when this process already owns it, or when whoever owned it let it +// lapse; otherwise it returns ErrNotLeader. +// +// The clock is the caller's rather than the database's, which is a compromise +// worth naming. A database clock would be one clock for everyone; taking it +// means a round trip that says now() inside every comparison, in SQL that is +// spelled differently in each dialect. Two control planes on machines whose +// clocks disagree by more than the lease will fight, so keep them on NTP — +// which any cluster running InfiniBand already has to be. +func (s *SQLite) AcquireLeadership(ctx context.Context, role, owner string, ttl time.Duration) (Lease, error) { + now := time.Now() + expires := now.Add(ttl) + + // One statement, so two processes racing cannot both believe they won: + // the WHERE decides, and the loser changes no rows. + res, err := s.db.ExecContext(ctx, + `INSERT INTO leadership (role, owner, expires_at, acquired_at) VALUES (?, ?, ?, ?) + ON CONFLICT (role) DO UPDATE SET + owner = excluded.owner, + expires_at = excluded.expires_at, + acquired_at = CASE WHEN leadership.owner = excluded.owner + THEN leadership.acquired_at + ELSE excluded.acquired_at END + WHERE leadership.owner = excluded.owner OR leadership.expires_at < ?`, + role, owner, expires.UnixMilli(), now.UnixMilli(), now.UnixMilli()) + if err != nil { + return Lease{}, err + } + n, err := res.RowsAffected() + if err != nil { + return Lease{}, err + } + if n == 0 { + // Lost, and the caller is told who to, because "not leader" without a + // name is the start of a long afternoon. The lease is returned beside + // the error rather than instead of it. + held, lerr := s.Leadership(ctx, role) + if lerr != nil { + return Lease{}, lerr + } + return held, ErrNotLeader + } + return s.Leadership(ctx, role) +} + +// Leadership reports who holds a role's lease. +func (s *SQLite) Leadership(ctx context.Context, role string) (Lease, error) { + var owner string + var expires, acquired int64 + err := s.db.QueryRowContext(ctx, + `SELECT owner, expires_at, acquired_at FROM leadership WHERE role = ?`, role). + Scan(&owner, &expires, &acquired) + if err != nil { + return Lease{}, err + } + return Lease{ + Owner: owner, + ExpiresAt: fromMillis(expires), + AcquiredAt: fromMillis(acquired), + }, nil +} + +// ReleaseLeadership gives the lease up, so a standby takes over in seconds +// rather than waiting out the clock. +// +// Only the holder can release it: a process that has already lost the lease +// must not be able to take it from whoever has it now, which is exactly what +// an unconditional delete during a slow shutdown would do. +func (s *SQLite) ReleaseLeadership(ctx context.Context, role, owner string) error { + _, err := s.db.ExecContext(ctx, + `DELETE FROM leadership WHERE role = ? AND owner = ?`, role, owner) + return err +} diff --git a/internal/store/leadership_test.go b/internal/store/leadership_test.go new file mode 100644 index 0000000..d12746d --- /dev/null +++ b/internal/store/leadership_test.go @@ -0,0 +1,151 @@ +package store + +import ( + "context" + "errors" + "sync" + "testing" + "time" +) + +// leadershipSuite is the same assertions against whichever store it is given, +// because the whole point of the lease is that it behaves identically on both +// — and the one that matters for HA is the one it was never tested on. +func leadershipSuite(t *testing.T, open func(*testing.T) *SQLite) { + ctx := context.Background() + + t.Run("the first to ask gets it", func(t *testing.T) { + s := open(t) + lease, err := s.AcquireLeadership(ctx, "control", "a", time.Minute) + if err != nil { + t.Fatal(err) + } + if lease.Owner != "a" { + t.Errorf("owner = %q", lease.Owner) + } + }) + + t.Run("a second is refused and told who holds it", func(t *testing.T) { + s := open(t) + if _, err := s.AcquireLeadership(ctx, "control", "a", time.Minute); err != nil { + t.Fatal(err) + } + held, err := s.AcquireLeadership(ctx, "control", "b", time.Minute) + if !errors.Is(err, ErrNotLeader) { + t.Fatalf("a second control plane got %v", err) + } + if held.Owner != "a" { + t.Errorf("refused without naming the holder: %+v", held) + } + }) + + t.Run("the holder renews without losing when it started", func(t *testing.T) { + s := open(t) + first, err := s.AcquireLeadership(ctx, "control", "a", time.Minute) + if err != nil { + t.Fatal(err) + } + time.Sleep(5 * time.Millisecond) + again, err := s.AcquireLeadership(ctx, "control", "a", time.Minute) + if err != nil { + t.Fatalf("the holder could not renew: %v", err) + } + if !again.ExpiresAt.After(first.ExpiresAt) { + t.Error("renewing did not move the expiry") + } + if !again.AcquiredAt.Equal(first.AcquiredAt) { + t.Error("renewing reset when this leader took over") + } + }) + + t.Run("a lapsed lease is taken", func(t *testing.T) { + // The reason a lease and not a lock: a leader that was powered off + // cannot release anything, and nobody should have to be paged to make + // a standby take over. + s := open(t) + if _, err := s.AcquireLeadership(ctx, "control", "a", 20*time.Millisecond); err != nil { + t.Fatal(err) + } + time.Sleep(40 * time.Millisecond) + lease, err := s.AcquireLeadership(ctx, "control", "b", time.Minute) + if err != nil { + t.Fatalf("a lapsed lease was not takeable: %v", err) + } + if lease.Owner != "b" { + t.Errorf("owner after takeover = %q", lease.Owner) + } + }) + + t.Run("releasing hands over at once", func(t *testing.T) { + s := open(t) + if _, err := s.AcquireLeadership(ctx, "control", "a", time.Hour); err != nil { + t.Fatal(err) + } + if err := s.ReleaseLeadership(ctx, "control", "a"); err != nil { + t.Fatal(err) + } + if _, err := s.AcquireLeadership(ctx, "control", "b", time.Minute); err != nil { + t.Errorf("a released lease was not free: %v", err) + } + }) + + t.Run("a former leader cannot release the current one", func(t *testing.T) { + // A process that lost the lease and is shutting down slowly must not + // take it from whoever has it now — which is exactly what an + // unconditional delete would do. + s := open(t) + if _, err := s.AcquireLeadership(ctx, "control", "a", 20*time.Millisecond); err != nil { + t.Fatal(err) + } + time.Sleep(40 * time.Millisecond) + if _, err := s.AcquireLeadership(ctx, "control", "b", time.Hour); err != nil { + t.Fatal(err) + } + if err := s.ReleaseLeadership(ctx, "control", "a"); err != nil { + t.Fatal(err) + } + held, err := s.Leadership(ctx, "control") + if err != nil { + t.Fatal(err) + } + if held.Owner != "b" { + t.Errorf("the old leader released the new one's lease: %+v", held) + } + }) + + t.Run("only one of many wins", func(t *testing.T) { + // The case the whole table exists for. Two control planes starting + // together is the ordinary way this gets tested in production. + s := open(t) + const racers = 8 + var wg sync.WaitGroup + won := make([]bool, racers) + for i := range racers { + wg.Add(1) + go func() { + defer wg.Done() + _, err := s.AcquireLeadership(ctx, "control", string(rune('a'+i)), time.Minute) + won[i] = err == nil + }() + } + wg.Wait() + + n := 0 + for _, w := range won { + if w { + n++ + } + } + if n != 1 { + t.Errorf("%d of %d processes believed they were leader", n, racers) + } + }) +} + +func TestLeadershipOnSQLite(t *testing.T) { + leadershipSuite(t, func(t *testing.T) *SQLite { return openDB(t) }) +} + +func TestLeadershipOnPostgres(t *testing.T) { + leadershipSuite(t, openPG) +} diff --git a/internal/store/migrations/postgres/0008_leadership.sql b/internal/store/migrations/postgres/0008_leadership.sql new file mode 100644 index 0000000..38f0637 --- /dev/null +++ b/internal/store/migrations/postgres/0008_leadership.sql @@ -0,0 +1,29 @@ +-- Ported from the SQLite migration of the same number. BIGINT because +-- PostgreSQL's INTEGER is 32 bits and these are milliseconds since the epoch. + +-- A lease on being the control plane that places work. +-- +-- Two control planes against one store both reconcile: both see a model +-- wanting two replicas and none running, and both place. The result is twice +-- the replicas, each holding GPUs the other's inventory believes are free — +-- the same failure a control plane restart used to produce before adoption, +-- and harder to see because nothing crashed. +-- +-- A lease rather than a lock held by the connection. PostgreSQL has advisory +-- locks and SQLite has nothing like them, and this store keeps one set of +-- queries for both; a row with an expiry works everywhere and survives the +-- leader being killed rather than asked to stop. +-- +-- One row, enforced by the primary key: there is one thing to be leader of. +CREATE TABLE leadership ( + role TEXT PRIMARY KEY, + -- Who holds it. An opaque id per process, so a process cannot be fooled by + -- its own stale lease after a restart on the same host. + owner TEXT NOT NULL, + -- When the lease lapses. The holder renews well before this; anyone else + -- may take it once the clock has passed, which is what makes a leader that + -- was powered off recoverable without anyone being paged. + expires_at BIGINT NOT NULL, + -- For an operator asking who has been in charge and since when. + acquired_at BIGINT NOT NULL +); diff --git a/internal/store/migrations/sqlite/0008_leadership.sql b/internal/store/migrations/sqlite/0008_leadership.sql new file mode 100644 index 0000000..0d9b920 --- /dev/null +++ b/internal/store/migrations/sqlite/0008_leadership.sql @@ -0,0 +1,26 @@ +-- A lease on being the control plane that places work. +-- +-- Two control planes against one store both reconcile: both see a model +-- wanting two replicas and none running, and both place. The result is twice +-- the replicas, each holding GPUs the other's inventory believes are free — +-- the same failure a control plane restart used to produce before adoption, +-- and harder to see because nothing crashed. +-- +-- A lease rather than a lock held by the connection. PostgreSQL has advisory +-- locks and SQLite has nothing like them, and this store keeps one set of +-- queries for both; a row with an expiry works everywhere and survives the +-- leader being killed rather than asked to stop. +-- +-- One row, enforced by the primary key: there is one thing to be leader of. +CREATE TABLE leadership ( + role TEXT PRIMARY KEY, + -- Who holds it. An opaque id per process, so a process cannot be fooled by + -- its own stale lease after a restart on the same host. + owner TEXT NOT NULL, + -- When the lease lapses. The holder renews well before this; anyone else + -- may take it once the clock has passed, which is what makes a leader that + -- was powered off recoverable without anyone being paged. + expires_at INTEGER NOT NULL, + -- For an operator asking who has been in charge and since when. + acquired_at INTEGER NOT NULL +); diff --git a/internal/store/postgres_test.go b/internal/store/postgres_test.go index 1b72d15..6748211 100644 --- a/internal/store/postgres_test.go +++ b/internal/store/postgres_test.go @@ -33,9 +33,13 @@ func openPG(t *testing.T) *SQLite { // CASCADE because the tables reference each other, and TRUNCATE rather // than DELETE so identity columns restart too. _, err = s.db.ExecContext(context.Background(), + // leadership included, and it is not in copyOrder: a subtest that + // inherits the previous one's lease is refused for reasons that have + // nothing to do with what it is testing. It passed on SQLite, where + // every subtest gets a fresh file, and failed only here. `TRUNCATE thread_messages, threads, memory_notes, usage, api_keys, key_claims, limits, models, sessions, users, projects, teams, - organizations RESTART IDENTITY CASCADE`) + organizations, leadership RESTART IDENTITY CASCADE`) if err != nil { t.Fatalf("clearing the test database: %v", err) }