Load a model when someone asks for it, and stop it when nobody does
CI / test (push) Successful in 1m29s
CI / test (push) Successful in 1m29s
The feature the idle tracking was for. A cluster with more models than GPUs can now hold every model in its catalog and only the wanted ones in its cards: min 0 with a max above it means load on demand, and once nobody has asked for idle_timeout the replicas stop and the cards go back. No new field, because min 0 max 1 already reads correctly — keep none running, allow up to one. Which made min 0 max 0 meaningful for the first time: configured and deliberately not running. normalize() used to fill a zero max in, on the reasonable ground that a model with no maximum cannot run; that would now silently turn every switched-off model into an on-demand one, including the ones already written that way in this cluster's own configuration. A model with no scaling stanza stays off for the same reason — defaulting it to on-demand would let any client request start a 70B nobody intended to run. The one real decision is what happens to the request that triggers a load. Refusing makes the feature useless, since the first request for every model fails; waiting without a bound turns a request into a job, and a client that budgeted thirty seconds holds a connection for ten minutes. So it waits briefly and then says what is happening — 503 with Retry-After and "the model is loading", which is a different answer from "no endpoint serves this" and produces different client behaviour. An unknown model still gets the unknown-model error, or on-demand would turn every typo into "loading" and the useful message would disappear. Use keeps a model alive rather than only the request that started it, without which the timer measures time since the load and a busy model is evicted on schedule however hard it is working. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
b619546516
commit
7fe59bd5fb
11 files changed
+521
-5
No files matched your search
@@ -1298,3 +1298,136 @@ func TestAModelRemovedWhileRunningIsStopped(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- on-demand loading and idle eviction ---
|
||||
|
||||
func onDemandModel(idle time.Duration) catalog.Model {
|
||||
m := testModel(1)
|
||||
m.Scaling = catalog.Scaling{Min: 0, Max: 1, IdleTimeout: idle}
|
||||
return m
|
||||
}
|
||||
|
||||
func TestAnOnDemandModelIsNotRunningUntilItIsAskedFor(t *testing.T) {
|
||||
// The point of the whole feature: a cluster with more models than GPUs can
|
||||
// hold every model in its catalog and only the wanted ones in its cards.
|
||||
h := newHarness(t, onDemandModel(time.Hour), gpuNode("1", 8))
|
||||
h.r.Reconcile(t.Context())
|
||||
if got := len(h.r.Replicas()); got != 0 {
|
||||
t.Fatalf("an on-demand model was placed before anyone asked: %d replicas", got)
|
||||
}
|
||||
|
||||
if !h.r.Want("llama") {
|
||||
t.Fatal("Want reported the model is not on-demand")
|
||||
}
|
||||
h.r.Reconcile(t.Context())
|
||||
if got := len(h.r.Replicas()); got != 1 {
|
||||
t.Errorf("asking for the model produced %d replicas, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWantSaysWhenAModelDoesNotLoadOnDemand(t *testing.T) {
|
||||
// The caller needs to tell "being loaded" from "there is genuinely nothing
|
||||
// serving this", because those are different answers to give a client.
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 8)) // min 1, not on-demand
|
||||
if h.r.Want("llama") {
|
||||
t.Error("an ordinary model was reported as loading on demand")
|
||||
}
|
||||
if h.r.Want("a-model-that-does-not-exist") {
|
||||
t.Error("an unknown model was reported as loading on demand")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnIdleOnDemandModelIsStopped(t *testing.T) {
|
||||
// The other half. Without it, on-demand is just a slower way to fill the
|
||||
// cluster permanently.
|
||||
h := newHarness(t, onDemandModel(10*time.Minute), gpuNode("1", 8))
|
||||
h.r.Want("llama")
|
||||
h.r.Reconcile(t.Context())
|
||||
h.markReady()
|
||||
if len(h.r.Replicas()) != 1 {
|
||||
t.Fatalf("the model did not start")
|
||||
}
|
||||
|
||||
h.advance(11 * time.Minute)
|
||||
h.r.Reconcile(t.Context())
|
||||
|
||||
for _, rep := range h.r.Replicas() {
|
||||
if rep.State.Counts() {
|
||||
t.Errorf("a replica idle past its timeout is still counted: %+v", rep)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUseKeepsAnOnDemandModelAlive(t *testing.T) {
|
||||
// Without this the timer measures time since the request that loaded the
|
||||
// model rather than the last one, so a busy model would be evicted exactly
|
||||
// IdleTimeout after it started however hard it was working.
|
||||
h := newHarness(t, onDemandModel(10*time.Minute), gpuNode("1", 8))
|
||||
h.r.Want("llama")
|
||||
h.r.Reconcile(t.Context())
|
||||
h.markReady()
|
||||
reps := h.r.Replicas()
|
||||
if len(reps) != 1 {
|
||||
t.Fatalf("the model did not start")
|
||||
}
|
||||
id := reps[0].ID
|
||||
|
||||
// Busy for half an hour, in six-minute steps.
|
||||
for range 5 {
|
||||
h.advance(6 * time.Minute)
|
||||
h.r.RecordUsage(map[string]time.Time{id: h.clock.now()})
|
||||
h.r.Reconcile(t.Context())
|
||||
}
|
||||
|
||||
var live int
|
||||
for _, rep := range h.r.Replicas() {
|
||||
if rep.State.Counts() {
|
||||
live++
|
||||
}
|
||||
}
|
||||
if live != 1 {
|
||||
t.Errorf("a model in continuous use was evicted: %d live replicas", live)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModelWithMaxZeroStaysOff(t *testing.T) {
|
||||
// min 0 max 0 is configured and deliberately not running, which is a
|
||||
// different and equally useful thing from "load it when wanted".
|
||||
m := testModel(1)
|
||||
m.Scaling = catalog.Scaling{Min: 0, Max: 0}
|
||||
h := newHarness(t, m, gpuNode("1", 8))
|
||||
|
||||
if h.r.Want("llama") {
|
||||
t.Error("a model with max 0 was treated as on-demand")
|
||||
}
|
||||
h.r.Reconcile(t.Context())
|
||||
if got := len(h.r.Replicas()); got != 0 {
|
||||
t.Errorf("a model switched off was placed: %d replicas", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnOrdinaryModelIsUnaffectedByIdleness(t *testing.T) {
|
||||
// Every model that exists today. A model with min 1 stays up whether anyone
|
||||
// uses it or not, because that is what min 1 means.
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
||||
h.r.Reconcile(t.Context())
|
||||
// Ready, or the startup timeout is what removes it twenty-four hours later
|
||||
// and the test proves nothing about idleness.
|
||||
h.markReady()
|
||||
if len(h.r.Replicas()) != 1 {
|
||||
t.Fatal("the model did not start")
|
||||
}
|
||||
|
||||
h.advance(24 * time.Hour)
|
||||
h.r.Reconcile(t.Context())
|
||||
|
||||
var live int
|
||||
for _, rep := range h.r.Replicas() {
|
||||
if rep.State.Counts() {
|
||||
live++
|
||||
}
|
||||
}
|
||||
if live != 1 {
|
||||
t.Errorf("an always-on model was evicted for being idle: %d live", live)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user