Scaling by the clock: schedule windows with their own min and max, prewarm, control.timezone

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 23:16:56 +02:00
1 parent 74429f1c00
commit bce54f0941
14 files changed
+382 -9

No files matched your search

+33 -6
View File
@@ -88,17 +88,43 @@ func (r *Reconciler) desiredFor(m catalog.Model) int {
r.mu.Lock()
extra := r.extra[m.Name]
r.mu.Unlock()
return min(base+extra, max(m.Scaling.Max, base))
return min(base+extra, max(r.maxFor(m), base))
}
// minFor is a model's minimum now: its schedule's window's, or its own
// outside every window.
func (r *Reconciler) minFor(m catalog.Model) int {
if w, ok := m.Scaling.Window(r.localNow()); ok {
return w.Min
}
return m.Scaling.Min
}
// maxFor is the ceiling load scaling reaches now.
func (r *Reconciler) maxFor(m catalog.Model) int {
if w, ok := m.Scaling.Window(r.localNow()); ok && w.Max > 0 {
return w.Max
}
return m.Scaling.Max
}
// localNow is now in the time zone schedules are written in.
func (r *Reconciler) localNow() time.Time {
if r.cfg.Location != nil {
return r.now().In(r.cfg.Location)
}
return r.now()
}
// baseFor is what a model runs without load: its minimum, or for an on-demand
// model one while somebody is asking and none once they stop.
// model one while somebody is asking and none once they stop. A schedule's
// window keeps its minimum running however quiet it is.
func (r *Reconciler) baseFor(m catalog.Model) int {
if m.IsExternal() {
return 0 // served elsewhere: nothing to place, whatever its scaling says
}
if !m.Scaling.OnDemand() {
return m.Scaling.Min
if lo := r.minFor(m); !m.Scaling.OnDemand() || lo > 0 {
return lo
}
r.mu.Lock()
last, asked := r.wanted[m.Name]
@@ -188,7 +214,8 @@ func (r *Reconciler) RecordLoad(loads map[string]Load) {
for _, m := range r.cat.Load().Models() {
base := r.baseFor(m)
r.mu.Lock()
if base == 0 || m.Scaling.Max <= base {
ceiling := r.maxFor(m)
if base == 0 || ceiling <= base {
delete(r.extra, m.Name)
delete(r.busySince, m.Name)
delete(r.quietSince, m.Name)
@@ -217,7 +244,7 @@ func (r *Reconciler) RecordLoad(loads map[string]Load) {
r.busySince[m.Name] = now
break
}
if now.Sub(since) >= r.cfg.ScaleUpAfter && base+extra < m.Scaling.Max && !starting {
if now.Sub(since) >= r.cfg.ScaleUpAfter && base+extra < ceiling && !starting {
r.extra[m.Name] = extra + 1
r.busySince[m.Name] = now
r.log.Info("scaling up for load", "model", m.Name, "replicas", base+extra+1,
+2 -2
View File
@@ -196,7 +196,7 @@ func (r *Reconciler) victims(m catalog.Model) []victim {
cat := r.cat.Load()
now := r.now()
myRank := catalog.ClassRank(m.EffectiveClass())
belowMin := r.liveCount(m.Name) < m.Scaling.Min
belowMin := r.liveCount(m.Name) < r.minFor(m)
pol := r.policy()
var borrowed map[string]bool
@@ -229,7 +229,7 @@ func (r *Reconciler) victims(m catalog.Model) []victim {
class := other.EffectiveClass()
rank := catalog.ClassRank(class)
sort.Slice(reps, func(i, j int) bool { return reps[i].CreatedAt.After(reps[j].CreatedAt) })
above := max(len(reps)-other.Scaling.Min, 0)
above := max(len(reps)-r.minFor(other), 0)
for i, rep := range reps {
aboveMin := i < above
tier := -1
@@ -50,6 +50,9 @@ type Dispatcher interface {
// Config tunes the reconciliation loop.
type Config struct {
// Location is the time zone models' scaling schedules are written in;
// nil is the control plane's own.
Location *time.Location
// Interval is how often desired and actual are compared when nothing else
// happens. Events wake the loop sooner.
Interval time.Duration
@@ -0,0 +1,52 @@
package reconciler
import (
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
)
func TestAScheduleRaisesTheMinimumThroughItsHours(t *testing.T) {
m := testModel(1)
m.Scaling = catalog.Scaling{Min: 1, Max: 4, Prewarm: 30 * time.Minute,
Schedule: []catalog.ScaleWindow{{From: "08:00", To: "18:00", Min: 2, Max: 3}}}
h := newHarness(t, m, gpuNode("1", 8))
at := func(hm string) {
target, _ := time.Parse("2006-01-02 15:04", "2026-01-02 "+hm)
h.clock.advance(target.Sub(h.clock.now()))
}
cur := func() catalog.Model { got, _ := h.live.Load().Resolve(m.Name); return got }
at("03:00")
if n := h.r.desiredFor(cur()); n != 1 {
t.Errorf("at night: %d", n)
}
at("07:45") // prewarmed
if n := h.r.desiredFor(cur()); n != 2 {
t.Errorf("before the window, prewarming: %d", n)
}
if mx := h.r.maxFor(cur()); mx != 3 {
t.Errorf("the window's ceiling: %d", mx)
}
at("18:30")
if n := h.r.desiredFor(cur()); n != 1 {
t.Errorf("after the window: %d", n)
}
}
func TestAnOnDemandModelIsKeptLoadedThroughItsWindow(t *testing.T) {
m := testModel(1)
m.Scaling = catalog.Scaling{Min: 0, Max: 1,
Schedule: []catalog.ScaleWindow{{From: "09:00", To: "17:00", Min: 1}}}
h := newHarness(t, m, gpuNode("1", 8))
target, _ := time.Parse("2006-01-02 15:04", "2026-01-02 10:00")
h.clock.advance(target.Sub(h.clock.now()))
got, _ := h.live.Load().Resolve(m.Name)
if n := h.r.desiredFor(got); n != 1 {
t.Errorf("unasked-for in its window: %d", n)
}
h.clock.advance(8 * time.Hour)
if n := h.r.desiredFor(got); n != 0 {
t.Errorf("unasked-for outside its window: %d", n)
}
}