Load a model when someone asks for it, and stop it when nobody does
CI / test (push) Successful in 1m29s
CI / test (push) Successful in 1m29s
The feature the idle tracking was for. A cluster with more models than GPUs can now hold every model in its catalog and only the wanted ones in its cards: min 0 with a max above it means load on demand, and once nobody has asked for idle_timeout the replicas stop and the cards go back. No new field, because min 0 max 1 already reads correctly — keep none running, allow up to one. Which made min 0 max 0 meaningful for the first time: configured and deliberately not running. normalize() used to fill a zero max in, on the reasonable ground that a model with no maximum cannot run; that would now silently turn every switched-off model into an on-demand one, including the ones already written that way in this cluster's own configuration. A model with no scaling stanza stays off for the same reason — defaulting it to on-demand would let any client request start a 70B nobody intended to run. The one real decision is what happens to the request that triggers a load. Refusing makes the feature useless, since the first request for every model fails; waiting without a bound turns a request into a job, and a client that budgeted thirty seconds holds a connection for ten minutes. So it waits briefly and then says what is happening — 503 with Retry-After and "the model is loading", which is a different answer from "no endpoint serves this" and produces different client behaviour. An unknown model still gets the unknown-model error, or on-demand would turn every typo into "loading" and the useful message would disappear. Use keeps a model alive rather than only the request that started it, without which the timer measures time since the load and a busy model is evicted on schedule however hard it is working. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
b619546516
commit
7fe59bd5fb
11 files changed
+521
-5
No files matched your search
@@ -0,0 +1,69 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Activator loads a model that is configured but not running.
|
||||
//
|
||||
// The control plane implements it. Behind an interface for the same reason the
|
||||
// endpoint list is: the router knows a request has arrived for something that
|
||||
// is not up, the control plane decides what runs, and neither needs to be in
|
||||
// the other's process to say so.
|
||||
type Activator interface {
|
||||
// Want reports whether the model is one that loads on demand. False means
|
||||
// there is genuinely nothing serving it, which is a different answer.
|
||||
Want(model string) bool
|
||||
}
|
||||
|
||||
// SetActivator supplies the control plane that can load models on demand.
|
||||
func (r *Router) SetActivator(a Activator) { r.activator = a }
|
||||
|
||||
// activate asks for a model and waits a bounded time for it to serve.
|
||||
//
|
||||
// The one real design decision in on-demand loading. A request that triggers a
|
||||
// load can either wait or be refused, and both are wrong on their own: refusing
|
||||
// makes the feature useless, because the first request for every model fails;
|
||||
// waiting without a bound turns a request into a job, and a client that budgeted
|
||||
// thirty seconds is holding a connection for ten minutes.
|
||||
//
|
||||
// So it waits, briefly, and then says what is happening. A forty-second wait is
|
||||
// a slow request. A fifteen-minute wait is a job pretending to be one, and the
|
||||
// client should be told to come back rather than left holding a socket.
|
||||
func (r *Router) activate(ctx context.Context, model string) bool {
|
||||
if r.activator == nil || !r.activator.Want(model) {
|
||||
return false
|
||||
}
|
||||
|
||||
wait := r.cfg.ActivationWait
|
||||
if wait <= 0 {
|
||||
wait = defaultActivationWait
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, wait)
|
||||
defer cancel()
|
||||
|
||||
// Polled rather than notified. A load takes tens of seconds at best, so the
|
||||
// latency of a poll is noise against it, and a notification path would be
|
||||
// another thing to get wrong for no gain a client could measure.
|
||||
t := time.NewTicker(250 * time.Millisecond)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-t.C:
|
||||
if len(r.reg.Snapshots(model)) > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// defaultActivationWait bounds how long a request holds a connection waiting
|
||||
// for a model to load.
|
||||
//
|
||||
// Long enough for a small model on a warm node, short enough that a client's
|
||||
// own timeout is not what ends the request — being told to retry is a better
|
||||
// outcome than a socket closing with nothing on it.
|
||||
const defaultActivationWait = 45 * time.Second
|
||||
@@ -284,6 +284,13 @@ func (r *Router) finish(w http.ResponseWriter, req *parsedRequest, res dispatchR
|
||||
if status == 0 {
|
||||
status = http.StatusOK
|
||||
}
|
||||
case errors.Is(res.err, errLoading):
|
||||
// Retry-After, because this one succeeds on a retry and a client that
|
||||
// is told so will make it. Without the header a well-behaved client
|
||||
// backs off exactly as it would from a permanent failure.
|
||||
status = http.StatusServiceUnavailable
|
||||
w.Header().Set("Retry-After", "30")
|
||||
writeError(w, status, "server_error", res.err.Error())
|
||||
case errors.Is(res.err, errNoCapacity):
|
||||
status = http.StatusServiceUnavailable
|
||||
writeError(w, status, "server_error", res.err.Error())
|
||||
|
||||
@@ -59,6 +59,9 @@ type Router struct {
|
||||
// extraMetrics are collectors other components publish on /metrics.
|
||||
extraMetrics []prometheus.Collector
|
||||
|
||||
// activator loads a model that is configured but not currently running.
|
||||
activator Activator
|
||||
|
||||
// landing configures the public page, which the router serves but does not
|
||||
// otherwise care about.
|
||||
landing config.Landing
|
||||
@@ -479,9 +482,30 @@ func (r *Router) dispatch(ctx context.Context, w http.ResponseWriter, req *parse
|
||||
|
||||
var errNoCapacity = errors.New("no endpoint available")
|
||||
|
||||
// errLoading means the model is being started and the wait ran out.
|
||||
//
|
||||
// Distinct from having no capacity because the right client behaviour differs:
|
||||
// this one succeeds on a retry, and the other does not.
|
||||
var errLoading = errors.New("model loading")
|
||||
|
||||
func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *parsedRequest, model string, entry catalog.Model) dispatchResult {
|
||||
candidates := r.reg.Snapshots(model)
|
||||
if len(candidates) == 0 {
|
||||
// Nothing is serving it. That may be because nothing ever will, or
|
||||
// because it loads when asked and nobody has asked yet — and the second
|
||||
// is not an error, it is the feature working.
|
||||
if r.activate(ctx, model) {
|
||||
candidates = r.reg.Snapshots(model)
|
||||
}
|
||||
}
|
||||
if len(candidates) == 0 {
|
||||
if r.activator != nil && r.activator.Want(model) {
|
||||
// Asked for, still loading. Saying "no endpoint serves this" would
|
||||
// be a lie, and a client told to retry will succeed where one told
|
||||
// the model does not exist gives up.
|
||||
return dispatchResult{err: fmt.Errorf("%w: %s", errLoading,
|
||||
"the model is loading; retry shortly")}
|
||||
}
|
||||
return dispatchResult{err: fmt.Errorf("%w: %s", errNoCapacity, noEndpointFor(model, r.servedModels()))}
|
||||
}
|
||||
|
||||
|
||||
@@ -501,3 +501,94 @@ func TestAModelWithNoContextPublishesNoContextField(t *testing.T) {
|
||||
t.Errorf("an unknown context was published as a number: %s", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
type fakeActivator struct {
|
||||
onDemand map[string]bool
|
||||
asked []string
|
||||
}
|
||||
|
||||
func (f *fakeActivator) Want(model string) bool {
|
||||
f.asked = append(f.asked, model)
|
||||
return f.onDemand[model]
|
||||
}
|
||||
|
||||
func TestALoadingModelIsToldToRetryRatherThanRefused(t *testing.T) {
|
||||
// "no endpoint serves this" would be a lie while a model is starting, and
|
||||
// the difference matters: a client told to retry succeeds, one told the
|
||||
// model does not exist gives up.
|
||||
e := newFakeEngine(t, "m")
|
||||
cfg := baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"m"}})
|
||||
// A very short wait, because the point here is what happens when it runs
|
||||
// out rather than how long it is.
|
||||
cfg.Router.ActivationWait = 50 * time.Millisecond
|
||||
rt := newTestRouter(t, cfg)
|
||||
rt.SetActivator(&fakeActivator{onDemand: map[string]bool{"sleepy": true}})
|
||||
|
||||
rec := post(t, rt.Handler(), map[string]any{"model": "sleepy", "messages": []any{}}, nil)
|
||||
if rec.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("status %d, expected 503", rec.Code)
|
||||
}
|
||||
if got := rec.Header().Get("Retry-After"); got == "" {
|
||||
t.Error("no Retry-After; a well-behaved client backs off as it would from a permanent failure")
|
||||
}
|
||||
if !strings.Contains(rec.Body.String(), "loading") {
|
||||
t.Errorf("the client is not told what is happening: %s", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModelThatIsSimplyUnknownStillSaysSo(t *testing.T) {
|
||||
// The other half. On-demand must not turn every typo into "loading",
|
||||
// because then nothing is ever wrong and the useful error disappears.
|
||||
e := newFakeEngine(t, "m")
|
||||
cfg := baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"m"}})
|
||||
rt := newTestRouter(t, cfg)
|
||||
rt.SetActivator(&fakeActivator{onDemand: map[string]bool{}})
|
||||
|
||||
rec := post(t, rt.Handler(), map[string]any{"model": "typo", "messages": []any{}}, nil)
|
||||
if rec.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("status %d", rec.Code)
|
||||
}
|
||||
if rec.Header().Get("Retry-After") != "" {
|
||||
t.Error("an unknown model was presented as retryable")
|
||||
}
|
||||
if !strings.Contains(rec.Body.String(), "available models") {
|
||||
t.Errorf("the useful error was lost: %s", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAServedModelNeverWaitsForActivation(t *testing.T) {
|
||||
// A model that is up must not pay a millisecond for the feature.
|
||||
e := newFakeEngine(t, "m")
|
||||
cfg := baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"m"}})
|
||||
cfg.Router.ActivationWait = 10 * time.Second
|
||||
rt := newTestRouter(t, cfg)
|
||||
act := &fakeActivator{onDemand: map[string]bool{"m": true}}
|
||||
rt.SetActivator(act)
|
||||
|
||||
start := time.Now()
|
||||
rec := post(t, rt.Handler(), map[string]any{"model": "m", "messages": []any{}}, nil)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed > time.Second {
|
||||
t.Errorf("a served request took %s; activation is on the hot path", elapsed)
|
||||
}
|
||||
if len(act.asked) != 0 {
|
||||
t.Errorf("the activator was consulted for a model that is already serving: %v", act.asked)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithoutAnActivatorNothingChanges(t *testing.T) {
|
||||
// Every deployment before this existed, and a router running without a
|
||||
// control plane.
|
||||
e := newFakeEngine(t, "m")
|
||||
rt := newTestRouter(t, baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"m"}}))
|
||||
|
||||
rec := post(t, rt.Handler(), map[string]any{"model": "nope", "messages": []any{}}, nil)
|
||||
if rec.Code != http.StatusServiceUnavailable {
|
||||
t.Errorf("status %d", rec.Code)
|
||||
}
|
||||
if !strings.Contains(rec.Body.String(), "available models") {
|
||||
t.Errorf("body = %s", rec.Body.String())
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user