Scaling by the clock: schedule windows with their own min and max, prewarm, control.timezone
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
74429f1c00
commit
bce54f0941
14 files changed
+382
-9
No files matched your search
@@ -28,6 +28,117 @@ type Scaling struct {
|
||||
// in and out loses that entirely — so the saving has to be worth more than
|
||||
// the cache, which it only is over a real idle period.
|
||||
IdleTimeout time.Duration `json:"idle_timeout,omitempty" koanf:"idle_timeout"`
|
||||
// Schedule changes min and max by the time of day, in the control
|
||||
// plane's control.timezone: more replicas kept through working hours,
|
||||
// fewer at night, an on-demand model kept loaded while people are at
|
||||
// work. The first window that holds now wins; outside them all, Min and
|
||||
// Max apply.
|
||||
Schedule []ScaleWindow `json:"schedule,omitempty" koanf:"schedule"`
|
||||
// Prewarm starts a window this long before it opens, so its replicas
|
||||
// have loaded by then: loading a model is minutes, and the people who
|
||||
// arrive at nine are the reason for the window.
|
||||
Prewarm time.Duration `json:"prewarm,omitempty" koanf:"prewarm"`
|
||||
}
|
||||
|
||||
// ScaleWindow is a part of the week with its own min and max.
|
||||
type ScaleWindow struct {
|
||||
// Days are mon … sun, or weekdays, weekends; empty is every day.
|
||||
Days []string `json:"days,omitempty" koanf:"days"`
|
||||
// From and To are HH:MM; a window whose To is before its From runs
|
||||
// past midnight.
|
||||
From string `json:"from" koanf:"from"`
|
||||
To string `json:"to" koanf:"to"`
|
||||
Min int `json:"min" koanf:"min"`
|
||||
// Max, when set, is the window's ceiling for load scaling; zero keeps
|
||||
// the model's own.
|
||||
Max int `json:"max,omitempty" koanf:"max"`
|
||||
}
|
||||
|
||||
// clock parses HH:MM into minutes past midnight.
|
||||
func clock(s string) (int, bool) {
|
||||
var h, m int
|
||||
if n, err := fmt.Sscanf(s, "%d:%d", &h, &m); err != nil || n != 2 || h < 0 || h > 23 || m < 0 || m > 59 || len(s) != 5 {
|
||||
return 0, false
|
||||
}
|
||||
return h*60 + m, true
|
||||
}
|
||||
|
||||
var weekdays = map[string][]time.Weekday{
|
||||
"mon": {time.Monday}, "tue": {time.Tuesday}, "wed": {time.Wednesday}, "thu": {time.Thursday},
|
||||
"fri": {time.Friday}, "sat": {time.Saturday}, "sun": {time.Sunday},
|
||||
"weekdays": {time.Monday, time.Tuesday, time.Wednesday, time.Thursday, time.Friday},
|
||||
"weekends": {time.Saturday, time.Sunday},
|
||||
}
|
||||
|
||||
func (w ScaleWindow) validate() error {
|
||||
from, ok1 := clock(w.From)
|
||||
to, ok2 := clock(w.To)
|
||||
if !ok1 || !ok2 {
|
||||
return fmt.Errorf("from and to are HH:MM, like 08:00")
|
||||
}
|
||||
if from == to {
|
||||
return fmt.Errorf("from and to are the same time")
|
||||
}
|
||||
for _, d := range w.Days {
|
||||
if _, ok := weekdays[strings.ToLower(d)]; !ok {
|
||||
return fmt.Errorf("day %q is not mon … sun, weekdays or weekends", d)
|
||||
}
|
||||
}
|
||||
if w.Min < 0 || w.Max < 0 || w.Max > 0 && w.Max < w.Min {
|
||||
return fmt.Errorf("min %d and max %d: max is 0 or at least min", w.Min, w.Max)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// holds says whether t falls in the window, opened early by lead.
|
||||
func (w ScaleWindow) holds(t time.Time, lead time.Duration) bool {
|
||||
from, _ := clock(w.From)
|
||||
to, _ := clock(w.To)
|
||||
span := to - from
|
||||
if span <= 0 {
|
||||
span += 24 * 60 // past midnight
|
||||
}
|
||||
midnight := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location())
|
||||
// The window as it starts yesterday, today and tomorrow — tomorrow's
|
||||
// can open today, pulled forward by the lead.
|
||||
for d := -1; d <= 1; d++ {
|
||||
day := midnight.AddDate(0, 0, d)
|
||||
if !w.on(day.Weekday()) {
|
||||
continue
|
||||
}
|
||||
start := day.Add(time.Duration(from)*time.Minute - lead)
|
||||
end := day.Add(time.Duration(from+span) * time.Minute)
|
||||
if !t.Before(start) && t.Before(end) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// on says whether the window starts on a day.
|
||||
func (w ScaleWindow) on(wd time.Weekday) bool {
|
||||
if len(w.Days) == 0 {
|
||||
return true
|
||||
}
|
||||
for _, d := range w.Days {
|
||||
for _, x := range weekdays[strings.ToLower(d)] {
|
||||
if x == wd {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Window is the window in force at t, in its own time zone, and false
|
||||
// outside them all.
|
||||
func (s Scaling) Window(t time.Time) (ScaleWindow, bool) {
|
||||
for _, w := range s.Schedule {
|
||||
if w.holds(t, s.Prewarm) {
|
||||
return w, true
|
||||
}
|
||||
}
|
||||
return ScaleWindow{}, false
|
||||
}
|
||||
|
||||
// OnDemand reports whether this model is loaded when someone asks for it.
|
||||
@@ -547,6 +658,14 @@ func (m Model) validate() error {
|
||||
default:
|
||||
return fmt.Errorf("model %q: role %q is not guard", m.Name, m.Role)
|
||||
}
|
||||
for i, w := range m.Scaling.Schedule {
|
||||
if err := w.validate(); err != nil {
|
||||
return fmt.Errorf("model %q: scaling.schedule[%d]: %w", m.Name, i, err)
|
||||
}
|
||||
}
|
||||
if m.Scaling.Prewarm < 0 || m.Scaling.Prewarm > 12*time.Hour {
|
||||
return fmt.Errorf("model %q: scaling.prewarm is between 0 and 12h", m.Name)
|
||||
}
|
||||
if m.Scaling.Max < m.Scaling.Min {
|
||||
return fmt.Errorf("model %q: scaling max %d is below min %d", m.Name, m.Scaling.Max, m.Scaling.Min)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestAScheduleWindowHolds(t *testing.T) {
|
||||
ist := time.FixedZone("TRT", 3*3600)
|
||||
at := func(day, hm string) time.Time {
|
||||
d, _ := time.ParseInLocation("2006-01-02 15:04", day+" "+hm, ist)
|
||||
return d
|
||||
}
|
||||
// 2026-10-05 is a Monday.
|
||||
work := Scaling{Schedule: []ScaleWindow{{Days: []string{"weekdays"}, From: "08:00", To: "18:00", Min: 2}},
|
||||
Prewarm: 30 * time.Minute}
|
||||
night := Scaling{Schedule: []ScaleWindow{{From: "22:00", To: "02:00", Min: 1}}}
|
||||
for _, c := range []struct {
|
||||
s Scaling
|
||||
t time.Time
|
||||
want bool
|
||||
}{
|
||||
{work, at("2026-10-05", "09:00"), true},
|
||||
{work, at("2026-10-05", "07:40"), true}, // prewarm
|
||||
{work, at("2026-10-05", "07:20"), false},
|
||||
{work, at("2026-10-05", "18:00"), false},
|
||||
{work, at("2026-10-04", "10:00"), false}, // Sunday
|
||||
{work, at("2026-10-04", "23:50"), false},
|
||||
{night, at("2026-10-05", "23:00"), true},
|
||||
{night, at("2026-10-06", "01:30"), true}, // after midnight, from yesterday's start
|
||||
{night, at("2026-10-06", "02:00"), false},
|
||||
{night, at("2026-10-06", "12:00"), false},
|
||||
} {
|
||||
if _, got := c.s.Window(c.t); got != c.want {
|
||||
t.Errorf("%v at %s: %v, want %v", c.s.Schedule[0], c.t.Format("Mon 15:04"), got, c.want)
|
||||
}
|
||||
}
|
||||
// Opened the evening before by its lead.
|
||||
early := Scaling{Schedule: []ScaleWindow{{Days: []string{"tue"}, From: "00:15", To: "06:00", Min: 1}}, Prewarm: time.Hour}
|
||||
if _, ok := early.Window(at("2026-10-05", "23:30")); !ok {
|
||||
t.Error("a window after midnight was not prewarmed the evening before")
|
||||
}
|
||||
for _, bad := range []ScaleWindow{{From: "8:00", To: "18:00"}, {From: "08:00", To: "08:00"},
|
||||
{From: "08:00", To: "18:00", Days: []string{"monday"}}, {From: "08:00", To: "18:00", Min: 3, Max: 2}} {
|
||||
if bad.validate() == nil {
|
||||
t.Errorf("%+v accepted", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -88,17 +88,43 @@ func (r *Reconciler) desiredFor(m catalog.Model) int {
|
||||
r.mu.Lock()
|
||||
extra := r.extra[m.Name]
|
||||
r.mu.Unlock()
|
||||
return min(base+extra, max(m.Scaling.Max, base))
|
||||
return min(base+extra, max(r.maxFor(m), base))
|
||||
}
|
||||
|
||||
// minFor is a model's minimum now: its schedule's window's, or its own
|
||||
// outside every window.
|
||||
func (r *Reconciler) minFor(m catalog.Model) int {
|
||||
if w, ok := m.Scaling.Window(r.localNow()); ok {
|
||||
return w.Min
|
||||
}
|
||||
return m.Scaling.Min
|
||||
}
|
||||
|
||||
// maxFor is the ceiling load scaling reaches now.
|
||||
func (r *Reconciler) maxFor(m catalog.Model) int {
|
||||
if w, ok := m.Scaling.Window(r.localNow()); ok && w.Max > 0 {
|
||||
return w.Max
|
||||
}
|
||||
return m.Scaling.Max
|
||||
}
|
||||
|
||||
// localNow is now in the time zone schedules are written in.
|
||||
func (r *Reconciler) localNow() time.Time {
|
||||
if r.cfg.Location != nil {
|
||||
return r.now().In(r.cfg.Location)
|
||||
}
|
||||
return r.now()
|
||||
}
|
||||
|
||||
// baseFor is what a model runs without load: its minimum, or for an on-demand
|
||||
// model one while somebody is asking and none once they stop.
|
||||
// model one while somebody is asking and none once they stop. A schedule's
|
||||
// window keeps its minimum running however quiet it is.
|
||||
func (r *Reconciler) baseFor(m catalog.Model) int {
|
||||
if m.IsExternal() {
|
||||
return 0 // served elsewhere: nothing to place, whatever its scaling says
|
||||
}
|
||||
if !m.Scaling.OnDemand() {
|
||||
return m.Scaling.Min
|
||||
if lo := r.minFor(m); !m.Scaling.OnDemand() || lo > 0 {
|
||||
return lo
|
||||
}
|
||||
r.mu.Lock()
|
||||
last, asked := r.wanted[m.Name]
|
||||
@@ -188,7 +214,8 @@ func (r *Reconciler) RecordLoad(loads map[string]Load) {
|
||||
for _, m := range r.cat.Load().Models() {
|
||||
base := r.baseFor(m)
|
||||
r.mu.Lock()
|
||||
if base == 0 || m.Scaling.Max <= base {
|
||||
ceiling := r.maxFor(m)
|
||||
if base == 0 || ceiling <= base {
|
||||
delete(r.extra, m.Name)
|
||||
delete(r.busySince, m.Name)
|
||||
delete(r.quietSince, m.Name)
|
||||
@@ -217,7 +244,7 @@ func (r *Reconciler) RecordLoad(loads map[string]Load) {
|
||||
r.busySince[m.Name] = now
|
||||
break
|
||||
}
|
||||
if now.Sub(since) >= r.cfg.ScaleUpAfter && base+extra < m.Scaling.Max && !starting {
|
||||
if now.Sub(since) >= r.cfg.ScaleUpAfter && base+extra < ceiling && !starting {
|
||||
r.extra[m.Name] = extra + 1
|
||||
r.busySince[m.Name] = now
|
||||
r.log.Info("scaling up for load", "model", m.Name, "replicas", base+extra+1,
|
||||
|
||||
@@ -196,7 +196,7 @@ func (r *Reconciler) victims(m catalog.Model) []victim {
|
||||
cat := r.cat.Load()
|
||||
now := r.now()
|
||||
myRank := catalog.ClassRank(m.EffectiveClass())
|
||||
belowMin := r.liveCount(m.Name) < m.Scaling.Min
|
||||
belowMin := r.liveCount(m.Name) < r.minFor(m)
|
||||
|
||||
pol := r.policy()
|
||||
var borrowed map[string]bool
|
||||
@@ -229,7 +229,7 @@ func (r *Reconciler) victims(m catalog.Model) []victim {
|
||||
class := other.EffectiveClass()
|
||||
rank := catalog.ClassRank(class)
|
||||
sort.Slice(reps, func(i, j int) bool { return reps[i].CreatedAt.After(reps[j].CreatedAt) })
|
||||
above := max(len(reps)-other.Scaling.Min, 0)
|
||||
above := max(len(reps)-r.minFor(other), 0)
|
||||
for i, rep := range reps {
|
||||
aboveMin := i < above
|
||||
tier := -1
|
||||
|
||||
@@ -50,6 +50,9 @@ type Dispatcher interface {
|
||||
|
||||
// Config tunes the reconciliation loop.
|
||||
type Config struct {
|
||||
// Location is the time zone models' scaling schedules are written in;
|
||||
// nil is the control plane's own.
|
||||
Location *time.Location
|
||||
// Interval is how often desired and actual are compared when nothing else
|
||||
// happens. Events wake the loop sooner.
|
||||
Interval time.Duration
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
package reconciler
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
)
|
||||
|
||||
func TestAScheduleRaisesTheMinimumThroughItsHours(t *testing.T) {
|
||||
m := testModel(1)
|
||||
m.Scaling = catalog.Scaling{Min: 1, Max: 4, Prewarm: 30 * time.Minute,
|
||||
Schedule: []catalog.ScaleWindow{{From: "08:00", To: "18:00", Min: 2, Max: 3}}}
|
||||
h := newHarness(t, m, gpuNode("1", 8))
|
||||
at := func(hm string) {
|
||||
target, _ := time.Parse("2006-01-02 15:04", "2026-01-02 "+hm)
|
||||
h.clock.advance(target.Sub(h.clock.now()))
|
||||
}
|
||||
cur := func() catalog.Model { got, _ := h.live.Load().Resolve(m.Name); return got }
|
||||
at("03:00")
|
||||
if n := h.r.desiredFor(cur()); n != 1 {
|
||||
t.Errorf("at night: %d", n)
|
||||
}
|
||||
at("07:45") // prewarmed
|
||||
if n := h.r.desiredFor(cur()); n != 2 {
|
||||
t.Errorf("before the window, prewarming: %d", n)
|
||||
}
|
||||
if mx := h.r.maxFor(cur()); mx != 3 {
|
||||
t.Errorf("the window's ceiling: %d", mx)
|
||||
}
|
||||
at("18:30")
|
||||
if n := h.r.desiredFor(cur()); n != 1 {
|
||||
t.Errorf("after the window: %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnOnDemandModelIsKeptLoadedThroughItsWindow(t *testing.T) {
|
||||
m := testModel(1)
|
||||
m.Scaling = catalog.Scaling{Min: 0, Max: 1,
|
||||
Schedule: []catalog.ScaleWindow{{From: "09:00", To: "17:00", Min: 1}}}
|
||||
h := newHarness(t, m, gpuNode("1", 8))
|
||||
target, _ := time.Parse("2006-01-02 15:04", "2026-01-02 10:00")
|
||||
h.clock.advance(target.Sub(h.clock.now()))
|
||||
got, _ := h.live.Load().Resolve(m.Name)
|
||||
if n := h.r.desiredFor(got); n != 1 {
|
||||
t.Errorf("unasked-for in its window: %d", n)
|
||||
}
|
||||
h.clock.advance(8 * time.Hour)
|
||||
if n := h.r.desiredFor(got); n != 0 {
|
||||
t.Errorf("unasked-for outside its window: %d", n)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user