Load a model when someone asks for it, and stop it when nobody does

The feature the idle tracking was for. A cluster with more models than GPUs
can now hold every model in its catalog and only the wanted ones in its
cards: min 0 with a max above it means load on demand, and once nobody has
asked for idle_timeout the replicas stop and the cards go back.

No new field, because min 0 max 1 already reads correctly — keep none
running, allow up to one. Which made min 0 max 0 meaningful for the first
time: configured and deliberately not running. normalize() used to fill a
zero max in, on the reasonable ground that a model with no maximum cannot
run; that would now silently turn every switched-off model into an
on-demand one, including the ones already written that way in this
cluster's own configuration. A model with no scaling stanza stays off for
the same reason — defaulting it to on-demand would let any client request
start a 70B nobody intended to run.

The one real decision is what happens to the request that triggers a load.
Refusing makes the feature useless, since the first request for every model
fails; waiting without a bound turns a request into a job, and a client that
budgeted thirty seconds holds a connection for ten minutes. So it waits
briefly and then says what is happening — 503 with Retry-After and "the
model is loading", which is a different answer from "no endpoint serves
this" and produces different client behaviour. An unknown model still gets
the unknown-model error, or on-demand would turn every typo into "loading"
and the useful message would disappear.

Use keeps a model alive rather than only the request that started it,
without which the timer measures time since the load and a busy model is
evicted on schedule however hard it is working.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-17 17:44:00 +02:00
1 parent b619546516
commit 7fe59bd5fb
11 files changed
+521 -5

No files matched your search

+69
View File
@@ -0,0 +1,69 @@
package router
import (
"context"
"time"
)
// Activator loads a model that is configured but not running.
//
// The control plane implements it. Behind an interface for the same reason the
// endpoint list is: the router knows a request has arrived for something that
// is not up, the control plane decides what runs, and neither needs to be in
// the other's process to say so.
type Activator interface {
// Want reports whether the model is one that loads on demand. False means
// there is genuinely nothing serving it, which is a different answer.
Want(model string) bool
}
// SetActivator supplies the control plane that can load models on demand.
func (r *Router) SetActivator(a Activator) { r.activator = a }
// activate asks for a model and waits a bounded time for it to serve.
//
// The one real design decision in on-demand loading. A request that triggers a
// load can either wait or be refused, and both are wrong on their own: refusing
// makes the feature useless, because the first request for every model fails;
// waiting without a bound turns a request into a job, and a client that budgeted
// thirty seconds is holding a connection for ten minutes.
//
// So it waits, briefly, and then says what is happening. A forty-second wait is
// a slow request. A fifteen-minute wait is a job pretending to be one, and the
// client should be told to come back rather than left holding a socket.
func (r *Router) activate(ctx context.Context, model string) bool {
if r.activator == nil || !r.activator.Want(model) {
return false
}
wait := r.cfg.ActivationWait
if wait <= 0 {
wait = defaultActivationWait
}
ctx, cancel := context.WithTimeout(ctx, wait)
defer cancel()
// Polled rather than notified. A load takes tens of seconds at best, so the
// latency of a poll is noise against it, and a notification path would be
// another thing to get wrong for no gain a client could measure.
t := time.NewTicker(250 * time.Millisecond)
defer t.Stop()
for {
select {
case <-ctx.Done():
return false
case <-t.C:
if len(r.reg.Snapshots(model)) > 0 {
return true
}
}
}
}
// defaultActivationWait bounds how long a request holds a connection waiting
// for a model to load.
//
// Long enough for a small model on a warm node, short enough that a client's
// own timeout is not what ends the request — being told to retry is a better
// outcome than a socket closing with nothing on it.
const defaultActivationWait = 45 * time.Second
+7
View File
@@ -284,6 +284,13 @@ func (r *Router) finish(w http.ResponseWriter, req *parsedRequest, res dispatchR
if status == 0 {
status = http.StatusOK
}
case errors.Is(res.err, errLoading):
// Retry-After, because this one succeeds on a retry and a client that
// is told so will make it. Without the header a well-behaved client
// backs off exactly as it would from a permanent failure.
status = http.StatusServiceUnavailable
w.Header().Set("Retry-After", "30")
writeError(w, status, "server_error", res.err.Error())
case errors.Is(res.err, errNoCapacity):
status = http.StatusServiceUnavailable
writeError(w, status, "server_error", res.err.Error())
+24
View File
@@ -59,6 +59,9 @@ type Router struct {
// extraMetrics are collectors other components publish on /metrics.
extraMetrics []prometheus.Collector
// activator loads a model that is configured but not currently running.
activator Activator
// landing configures the public page, which the router serves but does not
// otherwise care about.
landing config.Landing
@@ -479,9 +482,30 @@ func (r *Router) dispatch(ctx context.Context, w http.ResponseWriter, req *parse
var errNoCapacity = errors.New("no endpoint available")
// errLoading means the model is being started and the wait ran out.
//
// Distinct from having no capacity because the right client behaviour differs:
// this one succeeds on a retry, and the other does not.
var errLoading = errors.New("model loading")
func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *parsedRequest, model string, entry catalog.Model) dispatchResult {
candidates := r.reg.Snapshots(model)
if len(candidates) == 0 {
// Nothing is serving it. That may be because nothing ever will, or
// because it loads when asked and nobody has asked yet — and the second
// is not an error, it is the feature working.
if r.activate(ctx, model) {
candidates = r.reg.Snapshots(model)
}
}
if len(candidates) == 0 {
if r.activator != nil && r.activator.Want(model) {
// Asked for, still loading. Saying "no endpoint serves this" would
// be a lie, and a client told to retry will succeed where one told
// the model does not exist gives up.
return dispatchResult{err: fmt.Errorf("%w: %s", errLoading,
"the model is loading; retry shortly")}
}
return dispatchResult{err: fmt.Errorf("%w: %s", errNoCapacity, noEndpointFor(model, r.servedModels()))}
}
+91
View File
@@ -501,3 +501,94 @@ func TestAModelWithNoContextPublishesNoContextField(t *testing.T) {
t.Errorf("an unknown context was published as a number: %s", rec.Body.String())
}
}
type fakeActivator struct {
onDemand map[string]bool
asked []string
}
func (f *fakeActivator) Want(model string) bool {
f.asked = append(f.asked, model)
return f.onDemand[model]
}
func TestALoadingModelIsToldToRetryRatherThanRefused(t *testing.T) {
// "no endpoint serves this" would be a lie while a model is starting, and
// the difference matters: a client told to retry succeeds, one told the
// model does not exist gives up.
e := newFakeEngine(t, "m")
cfg := baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"m"}})
// A very short wait, because the point here is what happens when it runs
// out rather than how long it is.
cfg.Router.ActivationWait = 50 * time.Millisecond
rt := newTestRouter(t, cfg)
rt.SetActivator(&fakeActivator{onDemand: map[string]bool{"sleepy": true}})
rec := post(t, rt.Handler(), map[string]any{"model": "sleepy", "messages": []any{}}, nil)
if rec.Code != http.StatusServiceUnavailable {
t.Fatalf("status %d, expected 503", rec.Code)
}
if got := rec.Header().Get("Retry-After"); got == "" {
t.Error("no Retry-After; a well-behaved client backs off as it would from a permanent failure")
}
if !strings.Contains(rec.Body.String(), "loading") {
t.Errorf("the client is not told what is happening: %s", rec.Body.String())
}
}
func TestAModelThatIsSimplyUnknownStillSaysSo(t *testing.T) {
// The other half. On-demand must not turn every typo into "loading",
// because then nothing is ever wrong and the useful error disappears.
e := newFakeEngine(t, "m")
cfg := baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"m"}})
rt := newTestRouter(t, cfg)
rt.SetActivator(&fakeActivator{onDemand: map[string]bool{}})
rec := post(t, rt.Handler(), map[string]any{"model": "typo", "messages": []any{}}, nil)
if rec.Code != http.StatusServiceUnavailable {
t.Fatalf("status %d", rec.Code)
}
if rec.Header().Get("Retry-After") != "" {
t.Error("an unknown model was presented as retryable")
}
if !strings.Contains(rec.Body.String(), "available models") {
t.Errorf("the useful error was lost: %s", rec.Body.String())
}
}
func TestAServedModelNeverWaitsForActivation(t *testing.T) {
// A model that is up must not pay a millisecond for the feature.
e := newFakeEngine(t, "m")
cfg := baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"m"}})
cfg.Router.ActivationWait = 10 * time.Second
rt := newTestRouter(t, cfg)
act := &fakeActivator{onDemand: map[string]bool{"m": true}}
rt.SetActivator(act)
start := time.Now()
rec := post(t, rt.Handler(), map[string]any{"model": "m", "messages": []any{}}, nil)
if rec.Code != http.StatusOK {
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
}
if elapsed := time.Since(start); elapsed > time.Second {
t.Errorf("a served request took %s; activation is on the hot path", elapsed)
}
if len(act.asked) != 0 {
t.Errorf("the activator was consulted for a model that is already serving: %v", act.asked)
}
}
func TestWithoutAnActivatorNothingChanges(t *testing.T) {
// Every deployment before this existed, and a router running without a
// control plane.
e := newFakeEngine(t, "m")
rt := newTestRouter(t, baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"m"}}))
rec := post(t, rt.Handler(), map[string]any{"model": "nope", "messages": []any{}}, nil)
if rec.Code != http.StatusServiceUnavailable {
t.Errorf("status %d", rec.Code)
}
if !strings.Contains(rec.Body.String(), "available models") {
t.Errorf("body = %s", rec.Body.String())
}
}