Let the catalog change while the cluster is running
CI / test (push) Successful in 1m29s

Adding a model was an operator procedure with downtime: edit a file on the
control plane, restart it, wait out the settling window, watch replicas
churn while adoption catches up. I did that eight times in a day, and one
of the restarts cost two minutes of outage while stalled replicas timed
out. None of it was necessary. The reconciler already compares desired
against actual on every pass, so a new catalog is simply a different
desired.

A Catalog stays immutable — New is the only way to build one and it refuses
an invalid one — and what changes is which immutable catalog is current.
Readers take the pointer and finish their operation against it, so a swap
mid-request is invisible rather than a race. The router is told rather than
asked, the same shape as the endpoint list and for the same reason: a push
happens once per change, a pull would happen once per request.

The router still seeds from configuration as well as being pushed to. A
router with a static endpoint list and no control plane is a supported mode
and has nothing to push it a catalog; without seeding it would resolve no
model at all.

Which turned up a bug that was latent while catalogs were fixed.
reconcileModel only ever visits models that are in the catalog, so a
replica whose model had been removed was visited by nothing: it kept its
GPUs, kept counting, and kept running an engine for a name no client could
ask for. The endpoint publisher skipped such a replica with the comment
"the reconciler will tear it down", which was not true. It is now, and the
comment says what actually happens instead of what was assumed.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-17 01:08:21 +02:00
1 parent 9d8cd5e284
commit 1c57a4e378
10 files changed
+261 -34

No files matched your search

+40 -5
View File
@@ -137,7 +137,7 @@ func (c *Config) normalize() {
// Reconciler compares the catalog against what is running and closes the gap.
type Reconciler struct {
cfg Config
cat *catalog.Catalog
cat *catalog.Live
inv *inventory.Store
sched *placement.Scheduler
disp Dispatcher
@@ -171,7 +171,7 @@ type Reconciler struct {
}
// New builds a reconciler.
func New(cfg Config, cat *catalog.Catalog, inv *inventory.Store, disp Dispatcher, log *slog.Logger) *Reconciler {
func New(cfg Config, cat *catalog.Live, inv *inventory.Store, disp Dispatcher, log *slog.Logger) *Reconciler {
cfg.normalize()
if log == nil {
log = slog.Default()
@@ -252,9 +252,44 @@ func (r *Reconciler) Reconcile(ctx context.Context) {
return
}
for _, m := range r.cat.Models() {
for _, m := range r.cat.Load().Models() {
r.reconcileModel(ctx, m)
}
r.stopReplicasOfRemovedModels(ctx)
}
// stopReplicasOfRemovedModels tears down what the catalog no longer asks for.
//
// reconcileModel only ever visits models that are in the catalog, so a replica
// whose model has been removed was visited by nothing: it kept its GPUs, kept
// counting towards the inventory, and kept running an engine for a name no
// client could ask for. The endpoint publisher already skipped it with the
// comment "the reconciler will tear it down", which was simply not true.
//
// It mattered little while a catalog could only change by restarting the
// process. It matters now that models come and go while the cluster runs, which
// is the whole point of making the catalog live.
//
// Ordered after the model loop so a rebuild elsewhere has already claimed what
// it needs: freeing these GPUs first would let a replacement land on hardware
// that is about to be freed anyway, and churn two nodes instead of one.
func (r *Reconciler) stopReplicasOfRemovedModels(ctx context.Context) {
cat := r.cat.Load()
r.mu.Lock()
var gone []*Replica
for _, rep := range r.replicas {
if _, ok := cat.Resolve(rep.Model); !ok {
gone = append(gone, rep)
}
}
r.mu.Unlock()
for _, rep := range gone {
r.log.Info("stopping a replica whose model left the catalog",
"replica", rep.ID, "model", rep.Model)
r.stopReplica(ctx, rep.ID)
}
}
// fleetHasSettled reports whether placement may run yet.
@@ -380,7 +415,7 @@ func (r *Reconciler) expireStuckReplicas() {
// does not say — a replica with no deadline at all is the thing this exists to
// prevent.
func (r *Reconciler) startupTimeout(rep *Replica) time.Duration {
if m, ok := r.cat.Resolve(rep.Model); ok {
if m, ok := r.cat.Load().Resolve(rep.Model); ok {
if p, ok := m.ProfileByName(rep.Profile); ok && p.StartupTimeout > 0 {
return p.StartupTimeout
}
@@ -890,7 +925,7 @@ func (r *Reconciler) Adopt(nodeID string, shards []AdoptedShard) (unclaimed []Ad
}
first := group[0]
m, ok := r.cat.Resolve(first.Model)
m, ok := r.cat.Load().Resolve(first.Model)
if !ok {
unclaimed = append(unclaimed, group...)
r.reservePorts(group)
+72 -3
View File
@@ -134,7 +134,7 @@ func newHarness(t *testing.T, m catalog.Model, nodes ...inventory.Node) *harness
cfg.UnreachableAfter = 15 * time.Second
cfg.LostAfter = 60 * time.Second
r := New(cfg, cat, inv, disp, slog.New(slog.DiscardHandler))
r := New(cfg, catalog.NewLive(cat), inv, disp, slog.New(slog.DiscardHandler))
r.SetClock(c.now)
// Nodes were just registered, so their LastSeen is wall-clock now, which is
// far from the fake clock's epoch. Re-stamp them onto the fake timeline.
@@ -485,7 +485,7 @@ func TestScaleDownDrainsFirst(t *testing.T) {
if err != nil {
t.Fatal(err)
}
h.r.cat = cat
h.r.cat.Store(cat)
h.disp.reset()
h.r.Reconcile(t.Context())
@@ -1012,7 +1012,7 @@ func settlingHarness(t *testing.T, m catalog.Model) (*Reconciler, *inventory.Sto
inv := inventory.NewStore()
disp := &fakeDispatcher{}
c := newClock()
r := New(DefaultConfig(), cat, inv, disp, slog.New(slog.DiscardHandler))
r := New(DefaultConfig(), catalog.NewLive(cat), inv, disp, slog.New(slog.DiscardHandler))
r.SetClock(c.now)
return r, inv, disp, c
}
@@ -1229,3 +1229,72 @@ func TestAnEmptyUsageReportLeavesEverythingAlone(t *testing.T) {
t.Errorf("idle=%s known=%v; an empty report erased what was known", idle, known)
}
}
func TestAModelAddedWhileRunningIsPlaced(t *testing.T) {
// The reason the catalog stopped being fixed at startup. Adding a model was
// an operator procedure with downtime: edit a file on the control plane,
// restart it, wait out the settling window, watch adoption churn. None of
// that was necessary — the loop already compares desired against actual
// every pass, so a new catalog is simply a different desired.
h := newHarness(t, testModel(1), gpuNode("1", 8), gpuNode("2", 8))
h.r.Reconcile(t.Context())
if got := len(h.r.Replicas()); got != 1 {
t.Fatalf("expected the original model's replica, got %d", got)
}
second := testModel(1)
second.Name = "llama-two"
next, err := catalog.New([]catalog.Model{testModel(1), second})
if err != nil {
t.Fatal(err)
}
h.r.cat.Store(next)
h.r.Reconcile(t.Context())
byModel := map[string]int{}
for _, rep := range h.r.Replicas() {
byModel[rep.Model]++
}
if byModel["llama-two"] == 0 {
t.Errorf("a model added while running was never placed: %v", byModel)
}
if byModel["llama"] == 0 {
t.Errorf("the model that was already running was disturbed: %v", byModel)
}
}
func TestAModelRemovedWhileRunningIsStopped(t *testing.T) {
// The other direction, and the one with consequences: a replica whose model
// has left the catalog holds GPUs for something nothing can ask for.
first := testModel(1)
second := testModel(1)
second.Name = "llama-two"
both, err := catalog.New([]catalog.Model{first, second})
if err != nil {
t.Fatal(err)
}
h := newHarness(t, first, gpuNode("1", 8), gpuNode("2", 8))
h.r.cat.Store(both)
h.r.Reconcile(t.Context())
h.r.Reconcile(t.Context())
if len(h.r.Replicas()) < 2 {
t.Fatalf("expected both models placed, got %d", len(h.r.Replicas()))
}
only, err := catalog.New([]catalog.Model{first})
if err != nil {
t.Fatal(err)
}
h.r.cat.Store(only)
// Several passes: stopping is deliberate rather than immediate, and a
// replica is not torn down in the same breath it is noticed.
for range 4 {
h.r.Reconcile(t.Context())
}
for _, rep := range h.r.Replicas() {
if rep.Model == "llama-two" && rep.State.Counts() {
t.Errorf("a replica of a removed model is still counted: %+v", rep)
}
}
}