Scaling by the clock: schedule windows with their own min and max, prewarm, control.timezone

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 23:16:56 +02:00
1 parent 74429f1c00
commit bce54f0941
14 files changed
+382 -9

No files matched your search

+119
View File
@@ -28,6 +28,117 @@ type Scaling struct {
// in and out loses that entirely — so the saving has to be worth more than
// the cache, which it only is over a real idle period.
IdleTimeout time.Duration `json:"idle_timeout,omitempty" koanf:"idle_timeout"`
// Schedule changes min and max by the time of day, in the control
// plane's control.timezone: more replicas kept through working hours,
// fewer at night, an on-demand model kept loaded while people are at
// work. The first window that holds now wins; outside them all, Min and
// Max apply.
Schedule []ScaleWindow `json:"schedule,omitempty" koanf:"schedule"`
// Prewarm starts a window this long before it opens, so its replicas
// have loaded by then: loading a model is minutes, and the people who
// arrive at nine are the reason for the window.
Prewarm time.Duration `json:"prewarm,omitempty" koanf:"prewarm"`
}
// ScaleWindow is a part of the week with its own min and max.
type ScaleWindow struct {
// Days are mon … sun, or weekdays, weekends; empty is every day.
Days []string `json:"days,omitempty" koanf:"days"`
// From and To are HH:MM; a window whose To is before its From runs
// past midnight.
From string `json:"from" koanf:"from"`
To string `json:"to" koanf:"to"`
Min int `json:"min" koanf:"min"`
// Max, when set, is the window's ceiling for load scaling; zero keeps
// the model's own.
Max int `json:"max,omitempty" koanf:"max"`
}
// clock parses HH:MM into minutes past midnight.
func clock(s string) (int, bool) {
var h, m int
if n, err := fmt.Sscanf(s, "%d:%d", &h, &m); err != nil || n != 2 || h < 0 || h > 23 || m < 0 || m > 59 || len(s) != 5 {
return 0, false
}
return h*60 + m, true
}
var weekdays = map[string][]time.Weekday{
"mon": {time.Monday}, "tue": {time.Tuesday}, "wed": {time.Wednesday}, "thu": {time.Thursday},
"fri": {time.Friday}, "sat": {time.Saturday}, "sun": {time.Sunday},
"weekdays": {time.Monday, time.Tuesday, time.Wednesday, time.Thursday, time.Friday},
"weekends": {time.Saturday, time.Sunday},
}
func (w ScaleWindow) validate() error {
from, ok1 := clock(w.From)
to, ok2 := clock(w.To)
if !ok1 || !ok2 {
return fmt.Errorf("from and to are HH:MM, like 08:00")
}
if from == to {
return fmt.Errorf("from and to are the same time")
}
for _, d := range w.Days {
if _, ok := weekdays[strings.ToLower(d)]; !ok {
return fmt.Errorf("day %q is not mon … sun, weekdays or weekends", d)
}
}
if w.Min < 0 || w.Max < 0 || w.Max > 0 && w.Max < w.Min {
return fmt.Errorf("min %d and max %d: max is 0 or at least min", w.Min, w.Max)
}
return nil
}
// holds says whether t falls in the window, opened early by lead.
func (w ScaleWindow) holds(t time.Time, lead time.Duration) bool {
from, _ := clock(w.From)
to, _ := clock(w.To)
span := to - from
if span <= 0 {
span += 24 * 60 // past midnight
}
midnight := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location())
// The window as it starts yesterday, today and tomorrow — tomorrow's
// can open today, pulled forward by the lead.
for d := -1; d <= 1; d++ {
day := midnight.AddDate(0, 0, d)
if !w.on(day.Weekday()) {
continue
}
start := day.Add(time.Duration(from)*time.Minute - lead)
end := day.Add(time.Duration(from+span) * time.Minute)
if !t.Before(start) && t.Before(end) {
return true
}
}
return false
}
// on says whether the window starts on a day.
func (w ScaleWindow) on(wd time.Weekday) bool {
if len(w.Days) == 0 {
return true
}
for _, d := range w.Days {
for _, x := range weekdays[strings.ToLower(d)] {
if x == wd {
return true
}
}
}
return false
}
// Window is the window in force at t, in its own time zone, and false
// outside them all.
func (s Scaling) Window(t time.Time) (ScaleWindow, bool) {
for _, w := range s.Schedule {
if w.holds(t, s.Prewarm) {
return w, true
}
}
return ScaleWindow{}, false
}
// OnDemand reports whether this model is loaded when someone asks for it.
@@ -547,6 +658,14 @@ func (m Model) validate() error {
default:
return fmt.Errorf("model %q: role %q is not guard", m.Name, m.Role)
}
for i, w := range m.Scaling.Schedule {
if err := w.validate(); err != nil {
return fmt.Errorf("model %q: scaling.schedule[%d]: %w", m.Name, i, err)
}
}
if m.Scaling.Prewarm < 0 || m.Scaling.Prewarm > 12*time.Hour {
return fmt.Errorf("model %q: scaling.prewarm is between 0 and 12h", m.Name)
}
if m.Scaling.Max < m.Scaling.Min {
return fmt.Errorf("model %q: scaling max %d is below min %d", m.Name, m.Scaling.Max, m.Scaling.Min)
}
+49
View File
@@ -0,0 +1,49 @@
package catalog
import (
"testing"
"time"
)
func TestAScheduleWindowHolds(t *testing.T) {
ist := time.FixedZone("TRT", 3*3600)
at := func(day, hm string) time.Time {
d, _ := time.ParseInLocation("2006-01-02 15:04", day+" "+hm, ist)
return d
}
// 2026-10-05 is a Monday.
work := Scaling{Schedule: []ScaleWindow{{Days: []string{"weekdays"}, From: "08:00", To: "18:00", Min: 2}},
Prewarm: 30 * time.Minute}
night := Scaling{Schedule: []ScaleWindow{{From: "22:00", To: "02:00", Min: 1}}}
for _, c := range []struct {
s Scaling
t time.Time
want bool
}{
{work, at("2026-10-05", "09:00"), true},
{work, at("2026-10-05", "07:40"), true}, // prewarm
{work, at("2026-10-05", "07:20"), false},
{work, at("2026-10-05", "18:00"), false},
{work, at("2026-10-04", "10:00"), false}, // Sunday
{work, at("2026-10-04", "23:50"), false},
{night, at("2026-10-05", "23:00"), true},
{night, at("2026-10-06", "01:30"), true}, // after midnight, from yesterday's start
{night, at("2026-10-06", "02:00"), false},
{night, at("2026-10-06", "12:00"), false},
} {
if _, got := c.s.Window(c.t); got != c.want {
t.Errorf("%v at %s: %v, want %v", c.s.Schedule[0], c.t.Format("Mon 15:04"), got, c.want)
}
}
// Opened the evening before by its lead.
early := Scaling{Schedule: []ScaleWindow{{Days: []string{"tue"}, From: "00:15", To: "06:00", Min: 1}}, Prewarm: time.Hour}
if _, ok := early.Window(at("2026-10-05", "23:30")); !ok {
t.Error("a window after midnight was not prewarmed the evening before")
}
for _, bad := range []ScaleWindow{{From: "8:00", To: "18:00"}, {From: "08:00", To: "08:00"},
{From: "08:00", To: "18:00", Days: []string{"monday"}}, {From: "08:00", To: "18:00", Min: 3, Max: 2}} {
if bad.validate() == nil {
t.Errorf("%+v accepted", bad)
}
}
}
+33 -6
View File
@@ -88,17 +88,43 @@ func (r *Reconciler) desiredFor(m catalog.Model) int {
r.mu.Lock()
extra := r.extra[m.Name]
r.mu.Unlock()
return min(base+extra, max(m.Scaling.Max, base))
return min(base+extra, max(r.maxFor(m), base))
}
// minFor is a model's minimum now: its schedule's window's, or its own
// outside every window.
func (r *Reconciler) minFor(m catalog.Model) int {
if w, ok := m.Scaling.Window(r.localNow()); ok {
return w.Min
}
return m.Scaling.Min
}
// maxFor is the ceiling load scaling reaches now.
func (r *Reconciler) maxFor(m catalog.Model) int {
if w, ok := m.Scaling.Window(r.localNow()); ok && w.Max > 0 {
return w.Max
}
return m.Scaling.Max
}
// localNow is now in the time zone schedules are written in.
func (r *Reconciler) localNow() time.Time {
if r.cfg.Location != nil {
return r.now().In(r.cfg.Location)
}
return r.now()
}
// baseFor is what a model runs without load: its minimum, or for an on-demand
// model one while somebody is asking and none once they stop.
// model one while somebody is asking and none once they stop. A schedule's
// window keeps its minimum running however quiet it is.
func (r *Reconciler) baseFor(m catalog.Model) int {
if m.IsExternal() {
return 0 // served elsewhere: nothing to place, whatever its scaling says
}
if !m.Scaling.OnDemand() {
return m.Scaling.Min
if lo := r.minFor(m); !m.Scaling.OnDemand() || lo > 0 {
return lo
}
r.mu.Lock()
last, asked := r.wanted[m.Name]
@@ -188,7 +214,8 @@ func (r *Reconciler) RecordLoad(loads map[string]Load) {
for _, m := range r.cat.Load().Models() {
base := r.baseFor(m)
r.mu.Lock()
if base == 0 || m.Scaling.Max <= base {
ceiling := r.maxFor(m)
if base == 0 || ceiling <= base {
delete(r.extra, m.Name)
delete(r.busySince, m.Name)
delete(r.quietSince, m.Name)
@@ -217,7 +244,7 @@ func (r *Reconciler) RecordLoad(loads map[string]Load) {
r.busySince[m.Name] = now
break
}
if now.Sub(since) >= r.cfg.ScaleUpAfter && base+extra < m.Scaling.Max && !starting {
if now.Sub(since) >= r.cfg.ScaleUpAfter && base+extra < ceiling && !starting {
r.extra[m.Name] = extra + 1
r.busySince[m.Name] = now
r.log.Info("scaling up for load", "model", m.Name, "replicas", base+extra+1,
+2 -2
View File
@@ -196,7 +196,7 @@ func (r *Reconciler) victims(m catalog.Model) []victim {
cat := r.cat.Load()
now := r.now()
myRank := catalog.ClassRank(m.EffectiveClass())
belowMin := r.liveCount(m.Name) < m.Scaling.Min
belowMin := r.liveCount(m.Name) < r.minFor(m)
pol := r.policy()
var borrowed map[string]bool
@@ -229,7 +229,7 @@ func (r *Reconciler) victims(m catalog.Model) []victim {
class := other.EffectiveClass()
rank := catalog.ClassRank(class)
sort.Slice(reps, func(i, j int) bool { return reps[i].CreatedAt.After(reps[j].CreatedAt) })
above := max(len(reps)-other.Scaling.Min, 0)
above := max(len(reps)-r.minFor(other), 0)
for i, rep := range reps {
aboveMin := i < above
tier := -1
@@ -50,6 +50,9 @@ type Dispatcher interface {
// Config tunes the reconciliation loop.
type Config struct {
// Location is the time zone models' scaling schedules are written in;
// nil is the control plane's own.
Location *time.Location
// Interval is how often desired and actual are compared when nothing else
// happens. Events wake the loop sooner.
Interval time.Duration
@@ -0,0 +1,52 @@
package reconciler
import (
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
)
func TestAScheduleRaisesTheMinimumThroughItsHours(t *testing.T) {
m := testModel(1)
m.Scaling = catalog.Scaling{Min: 1, Max: 4, Prewarm: 30 * time.Minute,
Schedule: []catalog.ScaleWindow{{From: "08:00", To: "18:00", Min: 2, Max: 3}}}
h := newHarness(t, m, gpuNode("1", 8))
at := func(hm string) {
target, _ := time.Parse("2006-01-02 15:04", "2026-01-02 "+hm)
h.clock.advance(target.Sub(h.clock.now()))
}
cur := func() catalog.Model { got, _ := h.live.Load().Resolve(m.Name); return got }
at("03:00")
if n := h.r.desiredFor(cur()); n != 1 {
t.Errorf("at night: %d", n)
}
at("07:45") // prewarmed
if n := h.r.desiredFor(cur()); n != 2 {
t.Errorf("before the window, prewarming: %d", n)
}
if mx := h.r.maxFor(cur()); mx != 3 {
t.Errorf("the window's ceiling: %d", mx)
}
at("18:30")
if n := h.r.desiredFor(cur()); n != 1 {
t.Errorf("after the window: %d", n)
}
}
func TestAnOnDemandModelIsKeptLoadedThroughItsWindow(t *testing.T) {
m := testModel(1)
m.Scaling = catalog.Scaling{Min: 0, Max: 1,
Schedule: []catalog.ScaleWindow{{From: "09:00", To: "17:00", Min: 1}}}
h := newHarness(t, m, gpuNode("1", 8))
target, _ := time.Parse("2006-01-02 15:04", "2026-01-02 10:00")
h.clock.advance(target.Sub(h.clock.now()))
got, _ := h.live.Load().Resolve(m.Name)
if n := h.r.desiredFor(got); n != 1 {
t.Errorf("unasked-for in its window: %d", n)
}
h.clock.advance(8 * time.Hour)
if n := h.r.desiredFor(got); n != 0 {
t.Errorf("unasked-for outside its window: %d", n)
}
}