Load a model when someone asks for it, and stop it when nobody does
CI / test (push) Successful in 1m29s
CI / test (push) Successful in 1m29s
The feature the idle tracking was for. A cluster with more models than GPUs can now hold every model in its catalog and only the wanted ones in its cards: min 0 with a max above it means load on demand, and once nobody has asked for idle_timeout the replicas stop and the cards go back. No new field, because min 0 max 1 already reads correctly — keep none running, allow up to one. Which made min 0 max 0 meaningful for the first time: configured and deliberately not running. normalize() used to fill a zero max in, on the reasonable ground that a model with no maximum cannot run; that would now silently turn every switched-off model into an on-demand one, including the ones already written that way in this cluster's own configuration. A model with no scaling stanza stays off for the same reason — defaulting it to on-demand would let any client request start a 70B nobody intended to run. The one real decision is what happens to the request that triggers a load. Refusing makes the feature useless, since the first request for every model fails; waiting without a bound turns a request into a job, and a client that budgeted thirty seconds holds a connection for ten minutes. So it waits briefly and then says what is happening — 503 with Retry-After and "the model is loading", which is a different answer from "no endpoint serves this" and produces different client behaviour. An unknown model still gets the unknown-model error, or on-demand would turn every typo into "loading" and the useful message would disappear. Use keeps a model alive rather than only the request that started it, without which the timer measures time since the load and a busy model is evicted on schedule however hard it is working. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
b619546516
commit
7fe59bd5fb
11 files changed
+521
-5
No files matched your search
@@ -16,8 +16,24 @@ import (
|
||||
type Scaling struct {
|
||||
Min int `json:"min,omitempty" koanf:"min"`
|
||||
Max int `json:"max,omitempty" koanf:"max"`
|
||||
// IdleTimeout is how long an on-demand model may go unasked-for before its
|
||||
// replicas are stopped. Zero takes the cluster default.
|
||||
//
|
||||
// Minutes rather than seconds, always. Stopping a replica throws away the
|
||||
// KV cache prefix routing spent its effort filling, and a model that flaps
|
||||
// in and out loses that entirely — so the saving has to be worth more than
|
||||
// the cache, which it only is over a real idle period.
|
||||
IdleTimeout time.Duration `json:"idle_timeout,omitempty" koanf:"idle_timeout"`
|
||||
}
|
||||
|
||||
// OnDemand reports whether this model is loaded when someone asks for it.
|
||||
//
|
||||
// min 0 with a max above it, which needs no new field and reads correctly: keep
|
||||
// none running, allow up to max. min 0 and max 0 is a model that is configured
|
||||
// and deliberately not running, which is a different and equally useful thing —
|
||||
// the difference between "load it when wanted" and "do not load it".
|
||||
func (s Scaling) OnDemand() bool { return s.Min == 0 && s.Max > 0 }
|
||||
|
||||
// SLO is the latency target a model is expected to meet. `ctl bench` measures
|
||||
// capacity against it (plan §6.2.1).
|
||||
type SLO struct {
|
||||
@@ -193,8 +209,14 @@ func (m Model) normalize() Model {
|
||||
if m.Scaling.Min < 0 {
|
||||
m.Scaling.Min = 0
|
||||
}
|
||||
if m.Scaling.Max == 0 {
|
||||
m.Scaling.Max = max(m.Scaling.Min, 1)
|
||||
// Max 0 used to be an omission to fill in, because a model with no maximum
|
||||
// could not run. It is now also a statement: with min 0 it means configured
|
||||
// and deliberately not running, which is a different and useful thing from
|
||||
// "load it when wanted". Filling it in would silently turn every switched-
|
||||
// off model into an on-demand one — including the ones already written that
|
||||
// way in a running cluster's configuration.
|
||||
if m.Scaling.Max == 0 && m.Scaling.Min > 0 {
|
||||
m.Scaling.Max = m.Scaling.Min
|
||||
}
|
||||
for i := range m.Profiles {
|
||||
p := &m.Profiles[i]
|
||||
|
||||
@@ -124,8 +124,51 @@ func TestDefaults(t *testing.T) {
|
||||
if p.Weight <= 0 {
|
||||
t.Errorf("weight=%v", p.Weight)
|
||||
}
|
||||
if m.Scaling.Max < 1 {
|
||||
t.Errorf("scaling max=%d", m.Scaling.Max)
|
||||
// Max is no longer filled in for a model that names no minimum. With min 0
|
||||
// it now means "configured and deliberately not running", and defaulting it
|
||||
// to 1 would make every model without a scaling stanza loadable on demand —
|
||||
// so any client request could start a 70B nobody intended to run.
|
||||
if m.Scaling.Min == 0 && m.Scaling.Max != 0 {
|
||||
t.Errorf("scaling max=%d; an unscaled model should stay off, not become on-demand", m.Scaling.Max)
|
||||
}
|
||||
if m.Scaling.OnDemand() {
|
||||
t.Error("a model with no scaling stanza was made on-demand")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaxDefaultsToTheMinimumWhenOneIsGiven(t *testing.T) {
|
||||
// A model that says it wants two replicas and nothing about a maximum wants
|
||||
// two, not one and not unbounded.
|
||||
built, err := New([]Model{{
|
||||
Name: "m", Source: "/m", Scaling: Scaling{Min: 2},
|
||||
Profiles: []Profile{{Name: "p", Engine: "llamacpp"}},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, _ := built.Resolve("m")
|
||||
if m.Scaling.Max != 2 {
|
||||
t.Errorf("max=%d, want 2", m.Scaling.Max)
|
||||
}
|
||||
if m.Scaling.OnDemand() {
|
||||
t.Error("a model with min 2 was reported on-demand")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnDemandNeedsBothZeroMinimumAndARealMaximum(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
s Scaling
|
||||
want bool
|
||||
}{
|
||||
{"off", Scaling{Min: 0, Max: 0}, false},
|
||||
{"on demand", Scaling{Min: 0, Max: 1}, true},
|
||||
{"always on", Scaling{Min: 1, Max: 1}, false},
|
||||
{"always on, room to grow", Scaling{Min: 1, Max: 4}, false},
|
||||
} {
|
||||
if got := tc.s.OnDemand(); got != tc.want {
|
||||
t.Errorf("%s: OnDemand() = %v, want %v", tc.name, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -66,6 +66,9 @@ type Config struct {
|
||||
PortRange [2]int
|
||||
// MgmtInterface is the ethernet interface used for NCCL bootstrap.
|
||||
MgmtInterface string
|
||||
// IdleTimeout is how long an on-demand model stays loaded with nobody
|
||||
// asking for it. A model may override it.
|
||||
IdleTimeout time.Duration
|
||||
// SettleFor is how long after start to leave placement alone while agents
|
||||
// reconnect.
|
||||
//
|
||||
@@ -95,6 +98,7 @@ func DefaultConfig() Config {
|
||||
UnreachableAfter: 15 * time.Second,
|
||||
LostAfter: 60 * time.Second,
|
||||
SettleFor: 60 * time.Second,
|
||||
IdleTimeout: 15 * time.Minute,
|
||||
DrainGrace: 5 * time.Minute,
|
||||
PortRange: [2]int{20000, 20999},
|
||||
MgmtInterface: "eth0",
|
||||
@@ -112,6 +116,9 @@ func (c *Config) normalize() {
|
||||
if c.MaxBackoff < c.BaseBackoff {
|
||||
c.MaxBackoff = max(c.BaseBackoff, d.MaxBackoff)
|
||||
}
|
||||
if c.IdleTimeout <= 0 {
|
||||
c.IdleTimeout = d.IdleTimeout
|
||||
}
|
||||
if c.SettleFor < 0 {
|
||||
c.SettleFor = 0
|
||||
} else if c.SettleFor == 0 {
|
||||
@@ -157,6 +164,12 @@ type Reconciler struct {
|
||||
// reservedPorts holds ports the fleet reports as in use but no replica
|
||||
// claims, so the allocator cannot hand out what something is still bound to.
|
||||
reservedPorts map[int]struct{}
|
||||
// wanted records when an on-demand model was last asked for.
|
||||
//
|
||||
// The whole of on-demand loading, kept here rather than in the catalog
|
||||
// because it is state about this moment and not about what the model is.
|
||||
// A model nobody has wanted has no entry and is simply not running.
|
||||
wanted map[string]time.Time
|
||||
|
||||
wake chan struct{}
|
||||
stop chan struct{}
|
||||
@@ -184,6 +197,7 @@ func New(cfg Config, cat *catalog.Live, inv *inventory.Store, disp Dispatcher, l
|
||||
nextAttempt: map[string]time.Time{},
|
||||
attempts: map[string]int{},
|
||||
reservedPorts: map[int]struct{}{},
|
||||
wanted: map[string]time.Time{},
|
||||
lastExplain: map[string]placement.Result{},
|
||||
wake: make(chan struct{}, 1),
|
||||
stop: make(chan struct{}),
|
||||
@@ -357,6 +371,7 @@ func (r *Reconciler) RecordUsage(lastUsed map[string]time.Time) {
|
||||
rep.LastUsedAt = at
|
||||
}
|
||||
}
|
||||
r.noteUsage(lastUsed)
|
||||
}
|
||||
|
||||
// IdleFor reports how long a replica has gone without a request, and whether
|
||||
@@ -375,6 +390,79 @@ func (r *Reconciler) IdleFor(id string) (time.Duration, bool) {
|
||||
return r.now().Sub(rep.LastUsedAt), true
|
||||
}
|
||||
|
||||
// Want marks an on-demand model as asked for, so the next pass places it.
|
||||
//
|
||||
// Called by whatever fields the request — the router, in practice. It returns
|
||||
// whether the model is one this applies to, so a caller can tell "being loaded"
|
||||
// from "there is genuinely nothing serving this", which are different answers
|
||||
// to give a client.
|
||||
func (r *Reconciler) Want(model string) bool {
|
||||
m, ok := r.cat.Load().Resolve(model)
|
||||
if !ok || !m.Scaling.OnDemand() {
|
||||
return false
|
||||
}
|
||||
r.mu.Lock()
|
||||
r.wanted[m.Name] = r.now()
|
||||
r.mu.Unlock()
|
||||
// Immediately, not on the next tick. Someone is waiting on this request,
|
||||
// and five seconds of ticker is five seconds they spend staring at nothing.
|
||||
r.Notify()
|
||||
return true
|
||||
}
|
||||
|
||||
// desiredFor is how many replicas a model should have right now.
|
||||
//
|
||||
// For an ordinary model that is its configured minimum. For an on-demand one it
|
||||
// is one while somebody is asking and none once they stop, which is the whole
|
||||
// feature: a cluster with more models than GPUs can hold every model in its
|
||||
// catalog and only the wanted ones in its cards.
|
||||
func (r *Reconciler) desiredFor(m catalog.Model) int {
|
||||
if !m.Scaling.OnDemand() {
|
||||
return m.Scaling.Min
|
||||
}
|
||||
r.mu.Lock()
|
||||
last, asked := r.wanted[m.Name]
|
||||
r.mu.Unlock()
|
||||
if !asked {
|
||||
return 0
|
||||
}
|
||||
if r.now().Sub(last) > r.idleTimeout(m) {
|
||||
return 0
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
// idleTimeout is how long an on-demand model stays up unasked-for.
|
||||
func (r *Reconciler) idleTimeout(m catalog.Model) time.Duration {
|
||||
if m.Scaling.IdleTimeout > 0 {
|
||||
return m.Scaling.IdleTimeout
|
||||
}
|
||||
return r.cfg.IdleTimeout
|
||||
}
|
||||
|
||||
// noteUsage keeps an on-demand model alive while it is being used.
|
||||
//
|
||||
// Without this a model would be evicted exactly IdleTimeout after the request
|
||||
// that loaded it, however busy it had been since — the timer would measure
|
||||
// time since the first request rather than the last.
|
||||
func (r *Reconciler) noteUsage(lastUsed map[string]time.Time) {
|
||||
byModel := map[string]time.Time{}
|
||||
for id, at := range lastUsed {
|
||||
rep, ok := r.replicas[id]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if at.After(byModel[rep.Model]) {
|
||||
byModel[rep.Model] = at
|
||||
}
|
||||
}
|
||||
for model, at := range byModel {
|
||||
if cur, ok := r.wanted[model]; ok && at.After(cur) {
|
||||
r.wanted[model] = at
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// expireStuckReplicas fails replicas that never finished coming up.
|
||||
//
|
||||
// Every other path out of Starting depends on the agent reporting something. A
|
||||
@@ -581,7 +669,7 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
|
||||
blockedUntil := r.nextAttempt[m.Name]
|
||||
r.mu.Unlock()
|
||||
|
||||
desired := m.Scaling.Min
|
||||
desired := r.desiredFor(m)
|
||||
switch {
|
||||
case len(liveIDs) < desired:
|
||||
if r.now().Before(blockedUntil) {
|
||||
|
||||
@@ -1298,3 +1298,136 @@ func TestAModelRemovedWhileRunningIsStopped(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- on-demand loading and idle eviction ---
|
||||
|
||||
func onDemandModel(idle time.Duration) catalog.Model {
|
||||
m := testModel(1)
|
||||
m.Scaling = catalog.Scaling{Min: 0, Max: 1, IdleTimeout: idle}
|
||||
return m
|
||||
}
|
||||
|
||||
func TestAnOnDemandModelIsNotRunningUntilItIsAskedFor(t *testing.T) {
|
||||
// The point of the whole feature: a cluster with more models than GPUs can
|
||||
// hold every model in its catalog and only the wanted ones in its cards.
|
||||
h := newHarness(t, onDemandModel(time.Hour), gpuNode("1", 8))
|
||||
h.r.Reconcile(t.Context())
|
||||
if got := len(h.r.Replicas()); got != 0 {
|
||||
t.Fatalf("an on-demand model was placed before anyone asked: %d replicas", got)
|
||||
}
|
||||
|
||||
if !h.r.Want("llama") {
|
||||
t.Fatal("Want reported the model is not on-demand")
|
||||
}
|
||||
h.r.Reconcile(t.Context())
|
||||
if got := len(h.r.Replicas()); got != 1 {
|
||||
t.Errorf("asking for the model produced %d replicas, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWantSaysWhenAModelDoesNotLoadOnDemand(t *testing.T) {
|
||||
// The caller needs to tell "being loaded" from "there is genuinely nothing
|
||||
// serving this", because those are different answers to give a client.
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 8)) // min 1, not on-demand
|
||||
if h.r.Want("llama") {
|
||||
t.Error("an ordinary model was reported as loading on demand")
|
||||
}
|
||||
if h.r.Want("a-model-that-does-not-exist") {
|
||||
t.Error("an unknown model was reported as loading on demand")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnIdleOnDemandModelIsStopped(t *testing.T) {
|
||||
// The other half. Without it, on-demand is just a slower way to fill the
|
||||
// cluster permanently.
|
||||
h := newHarness(t, onDemandModel(10*time.Minute), gpuNode("1", 8))
|
||||
h.r.Want("llama")
|
||||
h.r.Reconcile(t.Context())
|
||||
h.markReady()
|
||||
if len(h.r.Replicas()) != 1 {
|
||||
t.Fatalf("the model did not start")
|
||||
}
|
||||
|
||||
h.advance(11 * time.Minute)
|
||||
h.r.Reconcile(t.Context())
|
||||
|
||||
for _, rep := range h.r.Replicas() {
|
||||
if rep.State.Counts() {
|
||||
t.Errorf("a replica idle past its timeout is still counted: %+v", rep)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUseKeepsAnOnDemandModelAlive(t *testing.T) {
|
||||
// Without this the timer measures time since the request that loaded the
|
||||
// model rather than the last one, so a busy model would be evicted exactly
|
||||
// IdleTimeout after it started however hard it was working.
|
||||
h := newHarness(t, onDemandModel(10*time.Minute), gpuNode("1", 8))
|
||||
h.r.Want("llama")
|
||||
h.r.Reconcile(t.Context())
|
||||
h.markReady()
|
||||
reps := h.r.Replicas()
|
||||
if len(reps) != 1 {
|
||||
t.Fatalf("the model did not start")
|
||||
}
|
||||
id := reps[0].ID
|
||||
|
||||
// Busy for half an hour, in six-minute steps.
|
||||
for range 5 {
|
||||
h.advance(6 * time.Minute)
|
||||
h.r.RecordUsage(map[string]time.Time{id: h.clock.now()})
|
||||
h.r.Reconcile(t.Context())
|
||||
}
|
||||
|
||||
var live int
|
||||
for _, rep := range h.r.Replicas() {
|
||||
if rep.State.Counts() {
|
||||
live++
|
||||
}
|
||||
}
|
||||
if live != 1 {
|
||||
t.Errorf("a model in continuous use was evicted: %d live replicas", live)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModelWithMaxZeroStaysOff(t *testing.T) {
|
||||
// min 0 max 0 is configured and deliberately not running, which is a
|
||||
// different and equally useful thing from "load it when wanted".
|
||||
m := testModel(1)
|
||||
m.Scaling = catalog.Scaling{Min: 0, Max: 0}
|
||||
h := newHarness(t, m, gpuNode("1", 8))
|
||||
|
||||
if h.r.Want("llama") {
|
||||
t.Error("a model with max 0 was treated as on-demand")
|
||||
}
|
||||
h.r.Reconcile(t.Context())
|
||||
if got := len(h.r.Replicas()); got != 0 {
|
||||
t.Errorf("a model switched off was placed: %d replicas", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnOrdinaryModelIsUnaffectedByIdleness(t *testing.T) {
|
||||
// Every model that exists today. A model with min 1 stays up whether anyone
|
||||
// uses it or not, because that is what min 1 means.
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
||||
h.r.Reconcile(t.Context())
|
||||
// Ready, or the startup timeout is what removes it twenty-four hours later
|
||||
// and the test proves nothing about idleness.
|
||||
h.markReady()
|
||||
if len(h.r.Replicas()) != 1 {
|
||||
t.Fatal("the model did not start")
|
||||
}
|
||||
|
||||
h.advance(24 * time.Hour)
|
||||
h.r.Reconcile(t.Context())
|
||||
|
||||
var live int
|
||||
for _, rep := range h.r.Replicas() {
|
||||
if rep.State.Counts() {
|
||||
live++
|
||||
}
|
||||
}
|
||||
if live != 1 {
|
||||
t.Errorf("an always-on model was evicted for being idle: %d live", live)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user