diff --git a/internal/control/reconciler/evict.go b/internal/control/reconciler/evict.go index 7807545..212607a 100644 --- a/internal/control/reconciler/evict.go +++ b/internal/control/reconciler/evict.go @@ -64,9 +64,26 @@ func (r *Reconciler) makeRoomFor(ctx context.Context, m catalog.Model) bool { attribute.Int("llmgw.candidates", len(candidates))) defer span.End() r.stopReplica(ctx, best.ID, "evicted, idle, to make room for "+m.Name) + r.forgetWantIfGone(best.Model, best.ID) return true } +// forgetWantIfGone takes an evicted model off the wanted list when that was +// its last replica. Still wanted, it was placed again on the next pass — +// ahead of the model it made room for whenever it came first in the catalog — +// and the card went back to the model nobody was asking for. Asked again, it +// is wanted again. +func (r *Reconciler) forgetWantIfGone(model, stopped string) { + r.mu.Lock() + defer r.mu.Unlock() + for _, rep := range r.replicas { + if rep.Model == model && rep.ID != stopped && rep.State.Counts() { + return + } + } + delete(r.wanted, model) +} + // evictable lists replicas that could be stopped for a model that needs room, // longest idle first. func (r *Reconciler) evictable(wantedBy string) []evictionCandidate { diff --git a/internal/control/reconciler/reconciler_test.go b/internal/control/reconciler/reconciler_test.go index f504837..5a0b04a 100644 --- a/internal/control/reconciler/reconciler_test.go +++ b/internal/control/reconciler/reconciler_test.go @@ -1708,6 +1708,43 @@ func TestAFreshlyStartedReplicaIsNotEvicted(t *testing.T) { } } +func TestAnEvictedModelDoesNotTakeTheCardBack(t *testing.T) { + // flux evicted for triposr was placed again two seconds later: still + // wanted, and first in the catalog, it won the card it had just given up. + h := newHarness(t, testModel(1), gpuNode("1", 4)) + h.r.cat.Store(twoOnDemandModels(t)) + h.r.Want("llama") + h.r.Reconcile(t.Context()) + h.markReady() + first := h.r.Replicas() + if len(first) != 1 { + t.Fatalf("setup: %d replicas", len(first)) + } + h.r.RecordUsage(map[string]time.Time{first[0].ID: h.clock.now()}) + h.advance(DefaultConfig().MinLifetime + time.Minute) + + h.r.Want("second") + for range 4 { + h.r.Reconcile(t.Context()) + h.markReady() + h.advance(10 * time.Second) + } + var models []string + for _, rep := range h.r.Replicas() { + if rep.State.Counts() { + models = append(models, rep.Model) + } + } + if !slices.Equal(models, []string{"second"}) { + t.Errorf("running after the eviction: %v, want [second]", models) + } + + // Asked for again, it is wanted again. + if !h.r.Want("llama") || h.r.baseFor(mustResolve(t, h, "llama")) != 1 { + t.Error("an evicted model asked for again is not wanted") + } +} + func TestAModelWaitingForAnotherOnDemandOneToStepAsideIsNotUnservable(t *testing.T) { // A 3D model asked for straight after a picture, on a cluster with one // card: the picture model holds it, too freshly started to evict. That