Ilker ManapandClaude Opus 5.5
78e41460f9
Stored ctl bench results reach the reconciler: requests per replica past the measured SLO concurrency scale a model up, a replica is taken back only if the rest stay under 70% of theirs, and among profiles of equal priority the most measured tokens per GPU or core is tried first. ctl model target sizes a model for a throughput against the room there is. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
109 lines
3.0 KiB
Go
109 lines
3.0 KiB
Go
package reconciler
|
|
|
|
import (
|
|
"sort"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
|
|
)
|
|
|
|
// Measured is what one replica of a model on a profile was measured to serve
|
|
// while meeting its SLO, by `llmgw ctl bench --save`.
|
|
type Measured struct {
|
|
// MaxConcurrency is the most requests at once that still met the SLO.
|
|
MaxConcurrency int
|
|
// TokPerSec is the throughput at that concurrency.
|
|
TokPerSec float64
|
|
}
|
|
|
|
// SetMeasurements supplies benchmark results, keyed model/profile, read
|
|
// afresh each time it is called. Without them scaling goes by queue alone and
|
|
// profiles are chosen by priority alone.
|
|
func (r *Reconciler) SetMeasurements(f func() map[string]Measured) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
r.measurements = f
|
|
}
|
|
|
|
// measured is a model's measurement on one profile.
|
|
func (r *Reconciler) measured(model, profile string) (Measured, bool) {
|
|
r.mu.Lock()
|
|
f := r.measurements
|
|
r.mu.Unlock()
|
|
if f == nil {
|
|
return Measured{}, false
|
|
}
|
|
m, ok := f()[model+"/"+profile]
|
|
return m, ok && m.MaxConcurrency > 0
|
|
}
|
|
|
|
// orderedProfiles is a model's profiles as placement should try them: by
|
|
// priority, and among equal priorities the one measured to serve the most per
|
|
// GPU — or per core, for a profile with none — first. Unmeasured profiles keep
|
|
// their catalog order behind measured ones of the same priority.
|
|
//
|
|
// Placement sorts by priority itself, so the order found here is written back
|
|
// as priorities, descending, for it to keep.
|
|
func (r *Reconciler) orderedProfiles(m catalog.Model) []placement.Profile {
|
|
ps := m.PlacementProfiles()
|
|
eff := make([]float64, len(ps))
|
|
for i, p := range ps {
|
|
meas, ok := r.measured(m.Name, p.Name)
|
|
if !ok {
|
|
continue
|
|
}
|
|
units := float64(max(p.GPUsPerNode, 0) * max(p.Nodes, 1))
|
|
if units == 0 {
|
|
units = float64(max(p.CPUCores, 1))
|
|
}
|
|
eff[i] = meas.TokPerSec / units
|
|
}
|
|
idx := make([]int, len(ps))
|
|
for i := range idx {
|
|
idx[i] = i
|
|
}
|
|
sort.SliceStable(idx, func(a, b int) bool {
|
|
pa, pb := ps[idx[a]], ps[idx[b]]
|
|
if pa.Priority != pb.Priority {
|
|
return pa.Priority > pb.Priority
|
|
}
|
|
return eff[idx[a]] > eff[idx[b]]
|
|
})
|
|
out := make([]placement.Profile, len(ps))
|
|
for rank, i := range idx {
|
|
out[rank] = ps[i]
|
|
out[rank].Priority = len(ps) - rank
|
|
}
|
|
return out
|
|
}
|
|
|
|
// MeasuredFor is the measurement that sizes a model: that of the profile its
|
|
// replicas run on, or, with none running, of the first profile placement
|
|
// would try that was measured.
|
|
func (r *Reconciler) MeasuredFor(model string) (Measured, string, bool) {
|
|
r.mu.Lock()
|
|
profile := ""
|
|
for _, rep := range r.replicas {
|
|
if rep.Model == model && rep.State.Counts() {
|
|
profile = rep.Profile
|
|
break
|
|
}
|
|
}
|
|
r.mu.Unlock()
|
|
if profile != "" {
|
|
if m, ok := r.measured(model, profile); ok {
|
|
return m, profile, true
|
|
}
|
|
}
|
|
cm, ok := r.cat.Load().Resolve(model)
|
|
if !ok {
|
|
return Measured{}, "", false
|
|
}
|
|
for _, p := range r.orderedProfiles(cm) {
|
|
if m, ok := r.measured(cm.Name, p.Name); ok {
|
|
return m, p.Name, true
|
|
}
|
|
}
|
|
return Measured{}, "", false
|
|
}
|