Adding a model was an operator procedure with downtime: edit a file on the control plane, restart it, wait out the settling window, watch replicas churn while adoption catches up. I did that eight times in a day, and one of the restarts cost two minutes of outage while stalled replicas timed out. None of it was necessary. The reconciler already compares desired against actual on every pass, so a new catalog is simply a different desired. A Catalog stays immutable — New is the only way to build one and it refuses an invalid one — and what changes is which immutable catalog is current. Readers take the pointer and finish their operation against it, so a swap mid-request is invisible rather than a race. The router is told rather than asked, the same shape as the endpoint list and for the same reason: a push happens once per change, a pull would happen once per request. The router still seeds from configuration as well as being pushed to. A router with a static endpoint list and no control plane is a supported mode and has nothing to push it a catalog; without seeding it would resolve no model at all. Which turned up a bug that was latent while catalogs were fixed. reconcileModel only ever visits models that are in the catalog, so a replica whose model had been removed was visited by nothing: it kept its GPUs, kept counting, and kept running an engine for a name no client could ask for. The endpoint publisher skipped such a replica with the comment "the reconciler will tear it down", which was not true. It is now, and the comment says what actually happens instead of what was assumed. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
9d8cd5e284
commit
1c57a4e378
10 files changed
+261
-34
No files matched your search
@@ -137,7 +137,7 @@ func (c *Config) normalize() {
|
||||
// Reconciler compares the catalog against what is running and closes the gap.
|
||||
type Reconciler struct {
|
||||
cfg Config
|
||||
cat *catalog.Catalog
|
||||
cat *catalog.Live
|
||||
inv *inventory.Store
|
||||
sched *placement.Scheduler
|
||||
disp Dispatcher
|
||||
@@ -171,7 +171,7 @@ type Reconciler struct {
|
||||
}
|
||||
|
||||
// New builds a reconciler.
|
||||
func New(cfg Config, cat *catalog.Catalog, inv *inventory.Store, disp Dispatcher, log *slog.Logger) *Reconciler {
|
||||
func New(cfg Config, cat *catalog.Live, inv *inventory.Store, disp Dispatcher, log *slog.Logger) *Reconciler {
|
||||
cfg.normalize()
|
||||
if log == nil {
|
||||
log = slog.Default()
|
||||
@@ -252,9 +252,44 @@ func (r *Reconciler) Reconcile(ctx context.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
for _, m := range r.cat.Models() {
|
||||
for _, m := range r.cat.Load().Models() {
|
||||
r.reconcileModel(ctx, m)
|
||||
}
|
||||
r.stopReplicasOfRemovedModels(ctx)
|
||||
}
|
||||
|
||||
// stopReplicasOfRemovedModels tears down what the catalog no longer asks for.
|
||||
//
|
||||
// reconcileModel only ever visits models that are in the catalog, so a replica
|
||||
// whose model has been removed was visited by nothing: it kept its GPUs, kept
|
||||
// counting towards the inventory, and kept running an engine for a name no
|
||||
// client could ask for. The endpoint publisher already skipped it with the
|
||||
// comment "the reconciler will tear it down", which was simply not true.
|
||||
//
|
||||
// It mattered little while a catalog could only change by restarting the
|
||||
// process. It matters now that models come and go while the cluster runs, which
|
||||
// is the whole point of making the catalog live.
|
||||
//
|
||||
// Ordered after the model loop so a rebuild elsewhere has already claimed what
|
||||
// it needs: freeing these GPUs first would let a replacement land on hardware
|
||||
// that is about to be freed anyway, and churn two nodes instead of one.
|
||||
func (r *Reconciler) stopReplicasOfRemovedModels(ctx context.Context) {
|
||||
cat := r.cat.Load()
|
||||
|
||||
r.mu.Lock()
|
||||
var gone []*Replica
|
||||
for _, rep := range r.replicas {
|
||||
if _, ok := cat.Resolve(rep.Model); !ok {
|
||||
gone = append(gone, rep)
|
||||
}
|
||||
}
|
||||
r.mu.Unlock()
|
||||
|
||||
for _, rep := range gone {
|
||||
r.log.Info("stopping a replica whose model left the catalog",
|
||||
"replica", rep.ID, "model", rep.Model)
|
||||
r.stopReplica(ctx, rep.ID)
|
||||
}
|
||||
}
|
||||
|
||||
// fleetHasSettled reports whether placement may run yet.
|
||||
@@ -380,7 +415,7 @@ func (r *Reconciler) expireStuckReplicas() {
|
||||
// does not say — a replica with no deadline at all is the thing this exists to
|
||||
// prevent.
|
||||
func (r *Reconciler) startupTimeout(rep *Replica) time.Duration {
|
||||
if m, ok := r.cat.Resolve(rep.Model); ok {
|
||||
if m, ok := r.cat.Load().Resolve(rep.Model); ok {
|
||||
if p, ok := m.ProfileByName(rep.Profile); ok && p.StartupTimeout > 0 {
|
||||
return p.StartupTimeout
|
||||
}
|
||||
@@ -890,7 +925,7 @@ func (r *Reconciler) Adopt(nodeID string, shards []AdoptedShard) (unclaimed []Ad
|
||||
}
|
||||
|
||||
first := group[0]
|
||||
m, ok := r.cat.Resolve(first.Model)
|
||||
m, ok := r.cat.Load().Resolve(first.Model)
|
||||
if !ok {
|
||||
unclaimed = append(unclaimed, group...)
|
||||
r.reservePorts(group)
|
||||
|
||||
@@ -134,7 +134,7 @@ func newHarness(t *testing.T, m catalog.Model, nodes ...inventory.Node) *harness
|
||||
cfg.UnreachableAfter = 15 * time.Second
|
||||
cfg.LostAfter = 60 * time.Second
|
||||
|
||||
r := New(cfg, cat, inv, disp, slog.New(slog.DiscardHandler))
|
||||
r := New(cfg, catalog.NewLive(cat), inv, disp, slog.New(slog.DiscardHandler))
|
||||
r.SetClock(c.now)
|
||||
// Nodes were just registered, so their LastSeen is wall-clock now, which is
|
||||
// far from the fake clock's epoch. Re-stamp them onto the fake timeline.
|
||||
@@ -485,7 +485,7 @@ func TestScaleDownDrainsFirst(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h.r.cat = cat
|
||||
h.r.cat.Store(cat)
|
||||
|
||||
h.disp.reset()
|
||||
h.r.Reconcile(t.Context())
|
||||
@@ -1012,7 +1012,7 @@ func settlingHarness(t *testing.T, m catalog.Model) (*Reconciler, *inventory.Sto
|
||||
inv := inventory.NewStore()
|
||||
disp := &fakeDispatcher{}
|
||||
c := newClock()
|
||||
r := New(DefaultConfig(), cat, inv, disp, slog.New(slog.DiscardHandler))
|
||||
r := New(DefaultConfig(), catalog.NewLive(cat), inv, disp, slog.New(slog.DiscardHandler))
|
||||
r.SetClock(c.now)
|
||||
return r, inv, disp, c
|
||||
}
|
||||
@@ -1229,3 +1229,72 @@ func TestAnEmptyUsageReportLeavesEverythingAlone(t *testing.T) {
|
||||
t.Errorf("idle=%s known=%v; an empty report erased what was known", idle, known)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModelAddedWhileRunningIsPlaced(t *testing.T) {
|
||||
// The reason the catalog stopped being fixed at startup. Adding a model was
|
||||
// an operator procedure with downtime: edit a file on the control plane,
|
||||
// restart it, wait out the settling window, watch adoption churn. None of
|
||||
// that was necessary — the loop already compares desired against actual
|
||||
// every pass, so a new catalog is simply a different desired.
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 8), gpuNode("2", 8))
|
||||
h.r.Reconcile(t.Context())
|
||||
if got := len(h.r.Replicas()); got != 1 {
|
||||
t.Fatalf("expected the original model's replica, got %d", got)
|
||||
}
|
||||
|
||||
second := testModel(1)
|
||||
second.Name = "llama-two"
|
||||
next, err := catalog.New([]catalog.Model{testModel(1), second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h.r.cat.Store(next)
|
||||
|
||||
h.r.Reconcile(t.Context())
|
||||
byModel := map[string]int{}
|
||||
for _, rep := range h.r.Replicas() {
|
||||
byModel[rep.Model]++
|
||||
}
|
||||
if byModel["llama-two"] == 0 {
|
||||
t.Errorf("a model added while running was never placed: %v", byModel)
|
||||
}
|
||||
if byModel["llama"] == 0 {
|
||||
t.Errorf("the model that was already running was disturbed: %v", byModel)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModelRemovedWhileRunningIsStopped(t *testing.T) {
|
||||
// The other direction, and the one with consequences: a replica whose model
|
||||
// has left the catalog holds GPUs for something nothing can ask for.
|
||||
first := testModel(1)
|
||||
second := testModel(1)
|
||||
second.Name = "llama-two"
|
||||
both, err := catalog.New([]catalog.Model{first, second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h := newHarness(t, first, gpuNode("1", 8), gpuNode("2", 8))
|
||||
h.r.cat.Store(both)
|
||||
h.r.Reconcile(t.Context())
|
||||
h.r.Reconcile(t.Context())
|
||||
if len(h.r.Replicas()) < 2 {
|
||||
t.Fatalf("expected both models placed, got %d", len(h.r.Replicas()))
|
||||
}
|
||||
|
||||
only, err := catalog.New([]catalog.Model{first})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h.r.cat.Store(only)
|
||||
|
||||
// Several passes: stopping is deliberate rather than immediate, and a
|
||||
// replica is not torn down in the same breath it is noticed.
|
||||
for range 4 {
|
||||
h.r.Reconcile(t.Context())
|
||||
}
|
||||
for _, rep := range h.r.Replicas() {
|
||||
if rep.Model == "llama-two" && rep.State.Counts() {
|
||||
t.Errorf("a replica of a removed model is still counted: %+v", rep)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user