package reconciler import ( "slices" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" "gitea.mynodes.xyz/ilker/llmgw/internal/control/placement" "gitea.mynodes.xyz/ilker/llmgw/internal/control/quota" ) // Headroom is how many more replicas of a model the cluster has room for now. type Headroom struct { // Current is the replicas the model has that hold capacity: starting or // ready, not those on their way out. Current int // More is how many more could be placed now, at most the limit asked. More int // Nodes is where each of those would go, in order, a replica's nodes // joined when it spans several. Nodes [][]string // Limited is set when More reached the limit asked, so there may be room // for more than that. A CPU profile that declares no memory is always // limited: nothing it asks for runs out. Limited bool // Blocked is why one more than More does not fit, a line per node that // was considered. Empty when Limited. Blocked []string // TokPerSec is what one more replica was measured to serve, on the // profile placement would give it; zero when it was never measured. TokPerSec float64 // ByPreempting is how many more, beyond More, would fit by stopping // replicas the model's class may take, and Preempts the models those // replicas belong to: what "room" means for a model that outranks others. ByPreempting int Preempts []string } // Headroom tries placing up to limit more replicas of a model on a copy of // the inventory, each one's allocation made on the copy before the next is // tried, exactly as the reconciler would place them one pass at a time. // // The scheduler's own decisions, not an estimate beside them: what the page // offers is then what the next passes will do, and a count that the // arithmetic of a second implementation allowed but placement refuses would // leave a model short with nobody told why. func (r *Reconciler) Headroom(model string, limit int) (Headroom, bool) { m, ok := r.cat.Load().Resolve(model) if !ok { return Headroom{}, false } var h Headroom var avoid []string r.mu.Lock() for _, rep := range r.replicas { if rep.Model == m.Name && rep.State.Counts() && rep.RetiringAt.IsZero() { h.Current++ avoid = append(avoid, rep.NodeIDs()...) } } r.mu.Unlock() inv := r.inv.Clone() sched := placement.New(inv) pol := r.policy() var usage map[string]quota.Amount if pol != nil { usage = pol.Usage(r.Holdings()) } place := func() placement.Result { res := sched.Place(r.orderedProfiles(m), avoid) if res.Placement == nil && len(avoid) > 0 { // As scaleUp does: spreading is a preference, not a rule. res = sched.Place(r.orderedProfiles(m), nil) } return res } // What the model's class may take, in the order preemption would, each // released on the copy only when free room has run out. victims, nextVictim, preempting := r.victims(m), 0, false for h.More+h.ByPreempting < limit { res := place() // Named only once they have made room: candidates tried on the way // to finding there is none would otherwise read as whose replicas a // placement would stop. var tried []string for res.Placement == nil && nextVictim < len(victims) { v := victims[nextVictim] nextVictim++ inv.ReleaseReplica(v.ID) tried = append(tried, v.Model) res = place() } if res.Placement != nil && len(tried) > 0 { preempting = true for _, m := range tried { if !slices.Contains(h.Preempts, m) { h.Preempts = append(h.Preempts, m) } } } if res.Placement == nil { for _, rej := range res.Rejections { h.Blocked = append(h.Blocked, rej.String()) } slices.Sort(h.Blocked) h.Blocked = slices.Compact(h.Blocked) return h, true } // A quota limit is as much an edge as a full node. if pol != nil && m.Owner != "" { need := placementNeed(m, res.Placement) if why := pol.AtLimit(m.Owner, usage, need); why != "" { h.Blocked = []string{why} return h, true } for _, s := range pol.Chain(m.Owner) { usage[s] = usage[s].Plus(need) } } mem, cores := 0, 0 if prof, ok := m.ProfileByName(res.Placement.Profile.Name); ok { mem, cores = prof.MinMemoryGB, cpuCores(prof) } if h.More == 0 { if meas, ok := r.measured(m.Name, res.Placement.Profile.Name); ok { h.TokPerSec = meas.TokPerSec } } var nodes []string for _, a := range res.Placement.Nodes { inv.Allocate(a.NodeID, inventory.Allocation{ ShardID: "headroom", ReplicaID: "headroom", GPUIndices: a.GPUIndices, MemoryGB: mem, CPUCores: cores, PowerW: a.PowerW, }) nodes = append(nodes, a.NodeID) } avoid = append(avoid, nodes...) // Once anything was stopped, what fits after is counted as fitting // by preempting: it may be sitting in the room that was freed. if preempting { h.ByPreempting++ } else { h.Nodes = append(h.Nodes, nodes) h.More++ } } h.Limited = true return h, true }