llmgw/internal/control/reconciler/evict.go

package reconciler

import (
	"context"
	"fmt"
	"sort"
	"time"

	"go.opentelemetry.io/otel/attribute"

	"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
)

// evictionCandidate is a replica that could be stopped to make room.
type evictionCandidate struct {
	ID    string
	Model string
	Idle  time.Duration
	GPUs  int
	// Nodes is how many machines the replica spans. More than one is left
	// alone — see makeRoomFor.
	Nodes int
}

// makeRoomFor stops an idle replica so a wanted model can be placed.
//
// Without this, on-demand loading works right up to the moment the cluster is
// full and then stops meaning anything: the model somebody just asked for
// cannot be placed, and the cards are held by a model nobody has asked for in
// an hour. That is the exact situation the feature exists to resolve, and it is
// the only situation in which it matters.
//
// Deliberately conservative, on three counts.
//
// Only idle replicas of on-demand models. A model at min 1 is a promise that it
// stays up, and quietly breaking that to serve a different request would make
// every guarantee in the catalog conditional on load.
//
// Only single-node replicas. Evicting a multi-node replica frees several
// machines at once and takes tens of minutes to rebuild, which is a decision
// worth making deliberately rather than as a side effect of one request.
//
// One at a time. Freeing everything that qualifies would empty a cluster to
// satisfy a request that may only have needed one node, and the next pass can
// always free more.
func (r *Reconciler) makeRoomFor(ctx context.Context, m catalog.Model) bool {
	candidates := r.evictable(m.Name)
	if len(candidates) == 0 {
		return false
	}

	// The longest idle first. It is the replica least likely to be wanted
	// again, and the one whose KV cache is worth the least.
	best := candidates[0]
	r.log.Info("evicting an idle replica to make room",
		"stopping", best.ID, "idle_model", best.Model,
		"idle", best.Idle.Round(time.Second), "gpus", best.GPUs, "for", m.Name)
	ctx, span := startSpan(ctx, "evict",
		attribute.String("llmgw.for", m.Name),
		attribute.String("llmgw.replica", best.ID),
		attribute.String("llmgw.model", best.Model),
		attribute.Float64("llmgw.idle_seconds", best.Idle.Seconds()),
		attribute.Int("llmgw.gpus", best.GPUs),
		attribute.Int("llmgw.candidates", len(candidates)))
	defer span.End()
	r.stopReplica(ctx, best.ID, "evicted, idle, to make room for "+m.Name)
	return true
}

// evictable lists replicas that could be stopped for a model that needs room,
// longest idle first.
func (r *Reconciler) evictable(wantedBy string) []evictionCandidate {
	cat := r.cat.Load()
	now := r.now()

	r.mu.Lock()
	defer r.mu.Unlock()

	var out []evictionCandidate
	for _, rep := range r.replicas {
		if rep.Model == wantedBy || !rep.State.Counts() {
			continue
		}
		other, ok := cat.Resolve(rep.Model)
		if !ok || !other.Scaling.OnDemand() {
			continue
		}
		// Unknown is not idle. A replica the router has said nothing about —
		// after a restart, that is all of them — must not be evicted on the
		// strength of a fact nobody has reported.
		if rep.LastUsedAt.IsZero() {
			continue
		}
		idle := now.Sub(rep.LastUsedAt)
		if idle < r.cfg.EvictionGrace {
			continue
		}
		// A replica that has only just started is left alone however idle it
		// looks. Two on-demand models on a cluster with room for one would
		// otherwise take turns evicting each other, paying a model load each
		// time — and a load is expensive enough that waiting is strictly
		// better than winning.
		if !rep.ReadyAt.IsZero() && now.Sub(rep.ReadyAt) < r.cfg.MinLifetime {
			continue
		}
		nodes := map[string]struct{}{}
		gpus := 0
		for _, sh := range rep.Shards {
			nodes[sh.NodeID] = struct{}{}
			gpus += len(sh.GPUIndices)
		}
		if len(nodes) > 1 {
			continue
		}
		out = append(out, evictionCandidate{
			ID: rep.ID, Model: rep.Model, Idle: idle, GPUs: gpus, Nodes: len(nodes),
		})
	}

	sort.Slice(out, func(i, j int) bool { return out[i].Idle > out[j].Idle })
	return out
}

// EvictionReason explains why room could not be made, for placement failures.
func (r *Reconciler) EvictionReason(model string) string {
	if c := r.evictable(model); len(c) > 0 {
		return fmt.Sprintf("%d idle replica(s) could be stopped to make room", len(c))
	}
	return "nothing idle enough to stop"
}