package reconciler import ( "context" "fmt" "sort" "time" "go.opentelemetry.io/otel/attribute" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" ) // evictionCandidate is a replica that could be stopped to make room. type evictionCandidate struct { ID string Model string Idle time.Duration GPUs int // Nodes is how many machines the replica spans. More than one is left // alone — see makeRoomFor. Nodes int } // makeRoomFor stops an idle replica so a wanted model can be placed. // // Without this, on-demand loading works right up to the moment the cluster is // full and then stops meaning anything: the model somebody just asked for // cannot be placed, and the cards are held by a model nobody has asked for in // an hour. That is the exact situation the feature exists to resolve, and it is // the only situation in which it matters. // // Deliberately conservative, on three counts. // // Only idle replicas of on-demand models. A model at min 1 is a promise that it // stays up, and quietly breaking that to serve a different request would make // every guarantee in the catalog conditional on load. // // Only single-node replicas. Evicting a multi-node replica frees several // machines at once and takes tens of minutes to rebuild, which is a decision // worth making deliberately rather than as a side effect of one request. // // One at a time. Freeing everything that qualifies would empty a cluster to // satisfy a request that may only have needed one node, and the next pass can // always free more. func (r *Reconciler) makeRoomFor(ctx context.Context, m catalog.Model) bool { candidates := r.evictable(m.Name) if len(candidates) == 0 { return false } // The longest idle first. It is the replica least likely to be wanted // again, and the one whose KV cache is worth the least. best := candidates[0] r.log.Info("evicting an idle replica to make room", "stopping", best.ID, "idle_model", best.Model, "idle", best.Idle.Round(time.Second), "gpus", best.GPUs, "for", m.Name) ctx, span := startSpan(ctx, "evict", attribute.String("llmgw.for", m.Name), attribute.String("llmgw.replica", best.ID), attribute.String("llmgw.model", best.Model), attribute.Float64("llmgw.idle_seconds", best.Idle.Seconds()), attribute.Int("llmgw.gpus", best.GPUs), attribute.Int("llmgw.candidates", len(candidates))) defer span.End() r.stopReplica(ctx, best.ID, "evicted, idle, to make room for "+m.Name) return true } // evictable lists replicas that could be stopped for a model that needs room, // longest idle first. func (r *Reconciler) evictable(wantedBy string) []evictionCandidate { cat := r.cat.Load() now := r.now() r.mu.Lock() defer r.mu.Unlock() var out []evictionCandidate for _, rep := range r.replicas { if rep.Model == wantedBy || !rep.State.Counts() { continue } other, ok := cat.Resolve(rep.Model) if !ok || !other.Scaling.OnDemand() { continue } // Unknown is not idle. A replica the router has said nothing about — // after a restart, that is all of them — must not be evicted on the // strength of a fact nobody has reported. if rep.LastUsedAt.IsZero() { continue } idle := now.Sub(rep.LastUsedAt) if idle < r.cfg.EvictionGrace { continue } // A replica that has only just started is left alone however idle it // looks. Two on-demand models on a cluster with room for one would // otherwise take turns evicting each other, paying a model load each // time — and a load is expensive enough that waiting is strictly // better than winning. if !rep.ReadyAt.IsZero() && now.Sub(rep.ReadyAt) < r.cfg.MinLifetime { continue } nodes := map[string]struct{}{} gpus := 0 for _, sh := range rep.Shards { nodes[sh.NodeID] = struct{}{} gpus += len(sh.GPUIndices) } if len(nodes) > 1 { continue } out = append(out, evictionCandidate{ ID: rep.ID, Model: rep.Model, Idle: idle, GPUs: gpus, Nodes: len(nodes), }) } sort.Slice(out, func(i, j int) bool { return out[i].Idle > out[j].Idle }) return out } // EvictionReason explains why room could not be made, for placement failures. func (r *Reconciler) EvictionReason(model string) string { if c := r.evictable(model); len(c) > 0 { return fmt.Sprintf("%d idle replica(s) could be stopped to make room", len(c)) } return "nothing idle enough to stop" }