package reconciler import ( "sort" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/control/placement" ) // Measured is what one replica of a model on a profile was measured to serve // while meeting its SLO, by `llmgw ctl bench --save`. type Measured struct { // MaxConcurrency is the most requests at once that still met the SLO. MaxConcurrency int // TokPerSec is the throughput at that concurrency. TokPerSec float64 } // SetMeasurements supplies benchmark results, keyed model/profile, read // afresh each time it is called. Without them scaling goes by queue alone and // profiles are chosen by priority alone. func (r *Reconciler) SetMeasurements(f func() map[string]Measured) { r.mu.Lock() defer r.mu.Unlock() r.measurements = f } // measured is a model's measurement on one profile. func (r *Reconciler) measured(model, profile string) (Measured, bool) { r.mu.Lock() f := r.measurements r.mu.Unlock() if f == nil { return Measured{}, false } m, ok := f()[model+"/"+profile] return m, ok && m.MaxConcurrency > 0 } // orderedProfiles is a model's profiles as placement should try them: by // priority, and among equal priorities the one measured to serve the most per // GPU — or per core, for a profile with none — first. Unmeasured profiles keep // their catalog order behind measured ones of the same priority. // // Placement sorts by priority itself, so the order found here is written back // as priorities, descending, for it to keep. func (r *Reconciler) orderedProfiles(m catalog.Model) []placement.Profile { ps := m.PlacementProfiles() eff := make([]float64, len(ps)) for i, p := range ps { meas, ok := r.measured(m.Name, p.Name) if !ok { continue } units := float64(max(p.GPUsPerNode, 0) * max(p.Nodes, 1)) if units == 0 { units = float64(max(p.CPUCores, 1)) } eff[i] = meas.TokPerSec / units } idx := make([]int, len(ps)) for i := range idx { idx[i] = i } sort.SliceStable(idx, func(a, b int) bool { pa, pb := ps[idx[a]], ps[idx[b]] if pa.Priority != pb.Priority { return pa.Priority > pb.Priority } return eff[idx[a]] > eff[idx[b]] }) out := make([]placement.Profile, len(ps)) for rank, i := range idx { out[rank] = ps[i] out[rank].Priority = len(ps) - rank } return out } // MeasuredFor is the measurement that sizes a model: that of the profile its // replicas run on, or, with none running, of the first profile placement // would try that was measured. func (r *Reconciler) MeasuredFor(model string) (Measured, string, bool) { r.mu.Lock() profile := "" for _, rep := range r.replicas { if rep.Model == model && rep.State.Counts() { profile = rep.Profile break } } r.mu.Unlock() if profile != "" { if m, ok := r.measured(model, profile); ok { return m, profile, true } } cm, ok := r.cat.Load().Resolve(model) if !ok { return Measured{}, "", false } for _, p := range r.orderedProfiles(cm) { if m, ok := r.measured(cm.Name, p.Name); ok { return m, p.Name, true } } return Measured{}, "", false }