Scaling by the clock: schedule windows with their own min and max, prewarm, control.timezone
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
74429f1c00
commit
bce54f0941
14 files changed
+382
-9
No files matched your search
@@ -88,17 +88,43 @@ func (r *Reconciler) desiredFor(m catalog.Model) int {
|
||||
r.mu.Lock()
|
||||
extra := r.extra[m.Name]
|
||||
r.mu.Unlock()
|
||||
return min(base+extra, max(m.Scaling.Max, base))
|
||||
return min(base+extra, max(r.maxFor(m), base))
|
||||
}
|
||||
|
||||
// minFor is a model's minimum now: its schedule's window's, or its own
|
||||
// outside every window.
|
||||
func (r *Reconciler) minFor(m catalog.Model) int {
|
||||
if w, ok := m.Scaling.Window(r.localNow()); ok {
|
||||
return w.Min
|
||||
}
|
||||
return m.Scaling.Min
|
||||
}
|
||||
|
||||
// maxFor is the ceiling load scaling reaches now.
|
||||
func (r *Reconciler) maxFor(m catalog.Model) int {
|
||||
if w, ok := m.Scaling.Window(r.localNow()); ok && w.Max > 0 {
|
||||
return w.Max
|
||||
}
|
||||
return m.Scaling.Max
|
||||
}
|
||||
|
||||
// localNow is now in the time zone schedules are written in.
|
||||
func (r *Reconciler) localNow() time.Time {
|
||||
if r.cfg.Location != nil {
|
||||
return r.now().In(r.cfg.Location)
|
||||
}
|
||||
return r.now()
|
||||
}
|
||||
|
||||
// baseFor is what a model runs without load: its minimum, or for an on-demand
|
||||
// model one while somebody is asking and none once they stop.
|
||||
// model one while somebody is asking and none once they stop. A schedule's
|
||||
// window keeps its minimum running however quiet it is.
|
||||
func (r *Reconciler) baseFor(m catalog.Model) int {
|
||||
if m.IsExternal() {
|
||||
return 0 // served elsewhere: nothing to place, whatever its scaling says
|
||||
}
|
||||
if !m.Scaling.OnDemand() {
|
||||
return m.Scaling.Min
|
||||
if lo := r.minFor(m); !m.Scaling.OnDemand() || lo > 0 {
|
||||
return lo
|
||||
}
|
||||
r.mu.Lock()
|
||||
last, asked := r.wanted[m.Name]
|
||||
@@ -188,7 +214,8 @@ func (r *Reconciler) RecordLoad(loads map[string]Load) {
|
||||
for _, m := range r.cat.Load().Models() {
|
||||
base := r.baseFor(m)
|
||||
r.mu.Lock()
|
||||
if base == 0 || m.Scaling.Max <= base {
|
||||
ceiling := r.maxFor(m)
|
||||
if base == 0 || ceiling <= base {
|
||||
delete(r.extra, m.Name)
|
||||
delete(r.busySince, m.Name)
|
||||
delete(r.quietSince, m.Name)
|
||||
@@ -217,7 +244,7 @@ func (r *Reconciler) RecordLoad(loads map[string]Load) {
|
||||
r.busySince[m.Name] = now
|
||||
break
|
||||
}
|
||||
if now.Sub(since) >= r.cfg.ScaleUpAfter && base+extra < m.Scaling.Max && !starting {
|
||||
if now.Sub(since) >= r.cfg.ScaleUpAfter && base+extra < ceiling && !starting {
|
||||
r.extra[m.Name] = extra + 1
|
||||
r.busySince[m.Name] = now
|
||||
r.log.Info("scaling up for load", "model", m.Name, "replicas", base+extra+1,
|
||||
|
||||
@@ -196,7 +196,7 @@ func (r *Reconciler) victims(m catalog.Model) []victim {
|
||||
cat := r.cat.Load()
|
||||
now := r.now()
|
||||
myRank := catalog.ClassRank(m.EffectiveClass())
|
||||
belowMin := r.liveCount(m.Name) < m.Scaling.Min
|
||||
belowMin := r.liveCount(m.Name) < r.minFor(m)
|
||||
|
||||
pol := r.policy()
|
||||
var borrowed map[string]bool
|
||||
@@ -229,7 +229,7 @@ func (r *Reconciler) victims(m catalog.Model) []victim {
|
||||
class := other.EffectiveClass()
|
||||
rank := catalog.ClassRank(class)
|
||||
sort.Slice(reps, func(i, j int) bool { return reps[i].CreatedAt.After(reps[j].CreatedAt) })
|
||||
above := max(len(reps)-other.Scaling.Min, 0)
|
||||
above := max(len(reps)-r.minFor(other), 0)
|
||||
for i, rep := range reps {
|
||||
aboveMin := i < above
|
||||
tier := -1
|
||||
|
||||
@@ -50,6 +50,9 @@ type Dispatcher interface {
|
||||
|
||||
// Config tunes the reconciliation loop.
|
||||
type Config struct {
|
||||
// Location is the time zone models' scaling schedules are written in;
|
||||
// nil is the control plane's own.
|
||||
Location *time.Location
|
||||
// Interval is how often desired and actual are compared when nothing else
|
||||
// happens. Events wake the loop sooner.
|
||||
Interval time.Duration
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
package reconciler
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
)
|
||||
|
||||
func TestAScheduleRaisesTheMinimumThroughItsHours(t *testing.T) {
|
||||
m := testModel(1)
|
||||
m.Scaling = catalog.Scaling{Min: 1, Max: 4, Prewarm: 30 * time.Minute,
|
||||
Schedule: []catalog.ScaleWindow{{From: "08:00", To: "18:00", Min: 2, Max: 3}}}
|
||||
h := newHarness(t, m, gpuNode("1", 8))
|
||||
at := func(hm string) {
|
||||
target, _ := time.Parse("2006-01-02 15:04", "2026-01-02 "+hm)
|
||||
h.clock.advance(target.Sub(h.clock.now()))
|
||||
}
|
||||
cur := func() catalog.Model { got, _ := h.live.Load().Resolve(m.Name); return got }
|
||||
at("03:00")
|
||||
if n := h.r.desiredFor(cur()); n != 1 {
|
||||
t.Errorf("at night: %d", n)
|
||||
}
|
||||
at("07:45") // prewarmed
|
||||
if n := h.r.desiredFor(cur()); n != 2 {
|
||||
t.Errorf("before the window, prewarming: %d", n)
|
||||
}
|
||||
if mx := h.r.maxFor(cur()); mx != 3 {
|
||||
t.Errorf("the window's ceiling: %d", mx)
|
||||
}
|
||||
at("18:30")
|
||||
if n := h.r.desiredFor(cur()); n != 1 {
|
||||
t.Errorf("after the window: %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnOnDemandModelIsKeptLoadedThroughItsWindow(t *testing.T) {
|
||||
m := testModel(1)
|
||||
m.Scaling = catalog.Scaling{Min: 0, Max: 1,
|
||||
Schedule: []catalog.ScaleWindow{{From: "09:00", To: "17:00", Min: 1}}}
|
||||
h := newHarness(t, m, gpuNode("1", 8))
|
||||
target, _ := time.Parse("2006-01-02 15:04", "2026-01-02 10:00")
|
||||
h.clock.advance(target.Sub(h.clock.now()))
|
||||
got, _ := h.live.Load().Resolve(m.Name)
|
||||
if n := h.r.desiredFor(got); n != 1 {
|
||||
t.Errorf("unasked-for in its window: %d", n)
|
||||
}
|
||||
h.clock.advance(8 * time.Hour)
|
||||
if n := h.r.desiredFor(got); n != 0 {
|
||||
t.Errorf("unasked-for outside its window: %d", n)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user