diff --git a/docs/install.md b/docs/install.md index a4b33ec3..f6dc7e52 100644 --- a/docs/install.md +++ b/docs/install.md @@ -488,6 +488,33 @@ qwen2.5-coder-3b-instruct yes 1/1 ready coder No restart, and it survives one: models added this way are stored, while anything in `server.yaml` is declarative and returns on every start. +### Models that load when someone asks + +A cluster with more models than GPUs can hold every model in its catalog and +only the wanted ones in its cards: + +```yaml +scaling: { min: 0, max: 1, idle_timeout: 15m } +``` + +`min: 0` with a `max` above it means load on demand. The first request for the +model starts it and waits briefly; if it is not serving by then the client gets +a 503 with `Retry-After` and an honest "the model is loading", rather than being +told nothing serves it — a client told to retry succeeds where one told the +model does not exist gives up. Once nobody has asked for `idle_timeout`, the +replicas are stopped and the cards go back to the pool. + +`min: 0, max: 0` is different and equally useful: configured, and deliberately +not running. A model with no `scaling` stanza at all gets that, because making +every unscaled model loadable on demand would let any client request start a +70B nobody intended to run. + +Two things to weigh. A cold start is visible to whoever triggers it, so this +suits the long tail people are experimenting with rather than the two or three +models used daily — keep those at `min: 1`. And stopping a replica throws away +the KV cache prefix routing spent its effort filling, so an idle timeout wants +to be minutes rather than seconds. + For a GPU model, name the class and the width instead: ```yaml diff --git a/internal/cli/server.go b/internal/cli/server.go index 6f6bfcb5..7b82c423 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -122,6 +122,9 @@ func newServerCmd() *cobra.Command { if err := rt.RegisterMetrics(fleetmetrics.New(ctl)); err != nil { return err } + // So a request for a model that is configured but not running + // starts it, rather than being told nothing serves it. + rt.SetActivator(ctl.Reconciler()) log.Info("control plane running", "models", len(cfg.Models), "agent_listen", cfg.AgentListen, diff --git a/internal/config/config.go b/internal/config/config.go index ac269cf6..92671020 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -143,6 +143,15 @@ type Control struct { // Router holds routing behaviour. type Router struct { + // ActivationWait is how long a request waits for an on-demand model to + // load before the client is told to come back. Zero takes the default. + // + // The one real decision in on-demand loading. Refusing outright makes the + // feature useless, since the first request for every model would fail; + // waiting without a bound turns a request into a job, and a client that + // budgeted thirty seconds ends up holding a connection for ten minutes. + ActivationWait time.Duration `koanf:"activation_wait"` + ScrapeInterval time.Duration `koanf:"scrape_interval"` ScrapeTimeout time.Duration `koanf:"scrape_timeout"` // StaleAfter is how long an endpoint may go unscraped before being scored diff --git a/internal/control/catalog/catalog.go b/internal/control/catalog/catalog.go index 92aee863..7a17ce6c 100644 --- a/internal/control/catalog/catalog.go +++ b/internal/control/catalog/catalog.go @@ -16,8 +16,24 @@ import ( type Scaling struct { Min int `json:"min,omitempty" koanf:"min"` Max int `json:"max,omitempty" koanf:"max"` + // IdleTimeout is how long an on-demand model may go unasked-for before its + // replicas are stopped. Zero takes the cluster default. + // + // Minutes rather than seconds, always. Stopping a replica throws away the + // KV cache prefix routing spent its effort filling, and a model that flaps + // in and out loses that entirely — so the saving has to be worth more than + // the cache, which it only is over a real idle period. + IdleTimeout time.Duration `json:"idle_timeout,omitempty" koanf:"idle_timeout"` } +// OnDemand reports whether this model is loaded when someone asks for it. +// +// min 0 with a max above it, which needs no new field and reads correctly: keep +// none running, allow up to max. min 0 and max 0 is a model that is configured +// and deliberately not running, which is a different and equally useful thing — +// the difference between "load it when wanted" and "do not load it". +func (s Scaling) OnDemand() bool { return s.Min == 0 && s.Max > 0 } + // SLO is the latency target a model is expected to meet. `ctl bench` measures // capacity against it (plan §6.2.1). type SLO struct { @@ -193,8 +209,14 @@ func (m Model) normalize() Model { if m.Scaling.Min < 0 { m.Scaling.Min = 0 } - if m.Scaling.Max == 0 { - m.Scaling.Max = max(m.Scaling.Min, 1) + // Max 0 used to be an omission to fill in, because a model with no maximum + // could not run. It is now also a statement: with min 0 it means configured + // and deliberately not running, which is a different and useful thing from + // "load it when wanted". Filling it in would silently turn every switched- + // off model into an on-demand one — including the ones already written that + // way in a running cluster's configuration. + if m.Scaling.Max == 0 && m.Scaling.Min > 0 { + m.Scaling.Max = m.Scaling.Min } for i := range m.Profiles { p := &m.Profiles[i] diff --git a/internal/control/catalog/catalog_test.go b/internal/control/catalog/catalog_test.go index c47e52d1..eafec55d 100644 --- a/internal/control/catalog/catalog_test.go +++ b/internal/control/catalog/catalog_test.go @@ -124,8 +124,51 @@ func TestDefaults(t *testing.T) { if p.Weight <= 0 { t.Errorf("weight=%v", p.Weight) } - if m.Scaling.Max < 1 { - t.Errorf("scaling max=%d", m.Scaling.Max) + // Max is no longer filled in for a model that names no minimum. With min 0 + // it now means "configured and deliberately not running", and defaulting it + // to 1 would make every model without a scaling stanza loadable on demand — + // so any client request could start a 70B nobody intended to run. + if m.Scaling.Min == 0 && m.Scaling.Max != 0 { + t.Errorf("scaling max=%d; an unscaled model should stay off, not become on-demand", m.Scaling.Max) + } + if m.Scaling.OnDemand() { + t.Error("a model with no scaling stanza was made on-demand") + } +} + +func TestMaxDefaultsToTheMinimumWhenOneIsGiven(t *testing.T) { + // A model that says it wants two replicas and nothing about a maximum wants + // two, not one and not unbounded. + built, err := New([]Model{{ + Name: "m", Source: "/m", Scaling: Scaling{Min: 2}, + Profiles: []Profile{{Name: "p", Engine: "llamacpp"}}, + }}) + if err != nil { + t.Fatal(err) + } + m, _ := built.Resolve("m") + if m.Scaling.Max != 2 { + t.Errorf("max=%d, want 2", m.Scaling.Max) + } + if m.Scaling.OnDemand() { + t.Error("a model with min 2 was reported on-demand") + } +} + +func TestOnDemandNeedsBothZeroMinimumAndARealMaximum(t *testing.T) { + for _, tc := range []struct { + name string + s Scaling + want bool + }{ + {"off", Scaling{Min: 0, Max: 0}, false}, + {"on demand", Scaling{Min: 0, Max: 1}, true}, + {"always on", Scaling{Min: 1, Max: 1}, false}, + {"always on, room to grow", Scaling{Min: 1, Max: 4}, false}, + } { + if got := tc.s.OnDemand(); got != tc.want { + t.Errorf("%s: OnDemand() = %v, want %v", tc.name, got, tc.want) + } } } diff --git a/internal/control/reconciler/reconciler.go b/internal/control/reconciler/reconciler.go index aa28106c..aff6cbd4 100644 --- a/internal/control/reconciler/reconciler.go +++ b/internal/control/reconciler/reconciler.go @@ -66,6 +66,9 @@ type Config struct { PortRange [2]int // MgmtInterface is the ethernet interface used for NCCL bootstrap. MgmtInterface string + // IdleTimeout is how long an on-demand model stays loaded with nobody + // asking for it. A model may override it. + IdleTimeout time.Duration // SettleFor is how long after start to leave placement alone while agents // reconnect. // @@ -95,6 +98,7 @@ func DefaultConfig() Config { UnreachableAfter: 15 * time.Second, LostAfter: 60 * time.Second, SettleFor: 60 * time.Second, + IdleTimeout: 15 * time.Minute, DrainGrace: 5 * time.Minute, PortRange: [2]int{20000, 20999}, MgmtInterface: "eth0", @@ -112,6 +116,9 @@ func (c *Config) normalize() { if c.MaxBackoff < c.BaseBackoff { c.MaxBackoff = max(c.BaseBackoff, d.MaxBackoff) } + if c.IdleTimeout <= 0 { + c.IdleTimeout = d.IdleTimeout + } if c.SettleFor < 0 { c.SettleFor = 0 } else if c.SettleFor == 0 { @@ -157,6 +164,12 @@ type Reconciler struct { // reservedPorts holds ports the fleet reports as in use but no replica // claims, so the allocator cannot hand out what something is still bound to. reservedPorts map[int]struct{} + // wanted records when an on-demand model was last asked for. + // + // The whole of on-demand loading, kept here rather than in the catalog + // because it is state about this moment and not about what the model is. + // A model nobody has wanted has no entry and is simply not running. + wanted map[string]time.Time wake chan struct{} stop chan struct{} @@ -184,6 +197,7 @@ func New(cfg Config, cat *catalog.Live, inv *inventory.Store, disp Dispatcher, l nextAttempt: map[string]time.Time{}, attempts: map[string]int{}, reservedPorts: map[int]struct{}{}, + wanted: map[string]time.Time{}, lastExplain: map[string]placement.Result{}, wake: make(chan struct{}, 1), stop: make(chan struct{}), @@ -357,6 +371,7 @@ func (r *Reconciler) RecordUsage(lastUsed map[string]time.Time) { rep.LastUsedAt = at } } + r.noteUsage(lastUsed) } // IdleFor reports how long a replica has gone without a request, and whether @@ -375,6 +390,79 @@ func (r *Reconciler) IdleFor(id string) (time.Duration, bool) { return r.now().Sub(rep.LastUsedAt), true } +// Want marks an on-demand model as asked for, so the next pass places it. +// +// Called by whatever fields the request — the router, in practice. It returns +// whether the model is one this applies to, so a caller can tell "being loaded" +// from "there is genuinely nothing serving this", which are different answers +// to give a client. +func (r *Reconciler) Want(model string) bool { + m, ok := r.cat.Load().Resolve(model) + if !ok || !m.Scaling.OnDemand() { + return false + } + r.mu.Lock() + r.wanted[m.Name] = r.now() + r.mu.Unlock() + // Immediately, not on the next tick. Someone is waiting on this request, + // and five seconds of ticker is five seconds they spend staring at nothing. + r.Notify() + return true +} + +// desiredFor is how many replicas a model should have right now. +// +// For an ordinary model that is its configured minimum. For an on-demand one it +// is one while somebody is asking and none once they stop, which is the whole +// feature: a cluster with more models than GPUs can hold every model in its +// catalog and only the wanted ones in its cards. +func (r *Reconciler) desiredFor(m catalog.Model) int { + if !m.Scaling.OnDemand() { + return m.Scaling.Min + } + r.mu.Lock() + last, asked := r.wanted[m.Name] + r.mu.Unlock() + if !asked { + return 0 + } + if r.now().Sub(last) > r.idleTimeout(m) { + return 0 + } + return 1 +} + +// idleTimeout is how long an on-demand model stays up unasked-for. +func (r *Reconciler) idleTimeout(m catalog.Model) time.Duration { + if m.Scaling.IdleTimeout > 0 { + return m.Scaling.IdleTimeout + } + return r.cfg.IdleTimeout +} + +// noteUsage keeps an on-demand model alive while it is being used. +// +// Without this a model would be evicted exactly IdleTimeout after the request +// that loaded it, however busy it had been since — the timer would measure +// time since the first request rather than the last. +func (r *Reconciler) noteUsage(lastUsed map[string]time.Time) { + byModel := map[string]time.Time{} + for id, at := range lastUsed { + rep, ok := r.replicas[id] + if !ok { + continue + } + if at.After(byModel[rep.Model]) { + byModel[rep.Model] = at + } + } + for model, at := range byModel { + if cur, ok := r.wanted[model]; ok && at.After(cur) { + r.wanted[model] = at + } + } +} + // expireStuckReplicas fails replicas that never finished coming up. // // Every other path out of Starting depends on the agent reporting something. A @@ -581,7 +669,7 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) { blockedUntil := r.nextAttempt[m.Name] r.mu.Unlock() - desired := m.Scaling.Min + desired := r.desiredFor(m) switch { case len(liveIDs) < desired: if r.now().Before(blockedUntil) { diff --git a/internal/control/reconciler/reconciler_test.go b/internal/control/reconciler/reconciler_test.go index 1f5f8d36..ee4cda3d 100644 --- a/internal/control/reconciler/reconciler_test.go +++ b/internal/control/reconciler/reconciler_test.go @@ -1298,3 +1298,136 @@ func TestAModelRemovedWhileRunningIsStopped(t *testing.T) { } } } + +// --- on-demand loading and idle eviction --- + +func onDemandModel(idle time.Duration) catalog.Model { + m := testModel(1) + m.Scaling = catalog.Scaling{Min: 0, Max: 1, IdleTimeout: idle} + return m +} + +func TestAnOnDemandModelIsNotRunningUntilItIsAskedFor(t *testing.T) { + // The point of the whole feature: a cluster with more models than GPUs can + // hold every model in its catalog and only the wanted ones in its cards. + h := newHarness(t, onDemandModel(time.Hour), gpuNode("1", 8)) + h.r.Reconcile(t.Context()) + if got := len(h.r.Replicas()); got != 0 { + t.Fatalf("an on-demand model was placed before anyone asked: %d replicas", got) + } + + if !h.r.Want("llama") { + t.Fatal("Want reported the model is not on-demand") + } + h.r.Reconcile(t.Context()) + if got := len(h.r.Replicas()); got != 1 { + t.Errorf("asking for the model produced %d replicas, want 1", got) + } +} + +func TestWantSaysWhenAModelDoesNotLoadOnDemand(t *testing.T) { + // The caller needs to tell "being loaded" from "there is genuinely nothing + // serving this", because those are different answers to give a client. + h := newHarness(t, testModel(1), gpuNode("1", 8)) // min 1, not on-demand + if h.r.Want("llama") { + t.Error("an ordinary model was reported as loading on demand") + } + if h.r.Want("a-model-that-does-not-exist") { + t.Error("an unknown model was reported as loading on demand") + } +} + +func TestAnIdleOnDemandModelIsStopped(t *testing.T) { + // The other half. Without it, on-demand is just a slower way to fill the + // cluster permanently. + h := newHarness(t, onDemandModel(10*time.Minute), gpuNode("1", 8)) + h.r.Want("llama") + h.r.Reconcile(t.Context()) + h.markReady() + if len(h.r.Replicas()) != 1 { + t.Fatalf("the model did not start") + } + + h.advance(11 * time.Minute) + h.r.Reconcile(t.Context()) + + for _, rep := range h.r.Replicas() { + if rep.State.Counts() { + t.Errorf("a replica idle past its timeout is still counted: %+v", rep) + } + } +} + +func TestUseKeepsAnOnDemandModelAlive(t *testing.T) { + // Without this the timer measures time since the request that loaded the + // model rather than the last one, so a busy model would be evicted exactly + // IdleTimeout after it started however hard it was working. + h := newHarness(t, onDemandModel(10*time.Minute), gpuNode("1", 8)) + h.r.Want("llama") + h.r.Reconcile(t.Context()) + h.markReady() + reps := h.r.Replicas() + if len(reps) != 1 { + t.Fatalf("the model did not start") + } + id := reps[0].ID + + // Busy for half an hour, in six-minute steps. + for range 5 { + h.advance(6 * time.Minute) + h.r.RecordUsage(map[string]time.Time{id: h.clock.now()}) + h.r.Reconcile(t.Context()) + } + + var live int + for _, rep := range h.r.Replicas() { + if rep.State.Counts() { + live++ + } + } + if live != 1 { + t.Errorf("a model in continuous use was evicted: %d live replicas", live) + } +} + +func TestAModelWithMaxZeroStaysOff(t *testing.T) { + // min 0 max 0 is configured and deliberately not running, which is a + // different and equally useful thing from "load it when wanted". + m := testModel(1) + m.Scaling = catalog.Scaling{Min: 0, Max: 0} + h := newHarness(t, m, gpuNode("1", 8)) + + if h.r.Want("llama") { + t.Error("a model with max 0 was treated as on-demand") + } + h.r.Reconcile(t.Context()) + if got := len(h.r.Replicas()); got != 0 { + t.Errorf("a model switched off was placed: %d replicas", got) + } +} + +func TestAnOrdinaryModelIsUnaffectedByIdleness(t *testing.T) { + // Every model that exists today. A model with min 1 stays up whether anyone + // uses it or not, because that is what min 1 means. + h := newHarness(t, testModel(1), gpuNode("1", 8)) + h.r.Reconcile(t.Context()) + // Ready, or the startup timeout is what removes it twenty-four hours later + // and the test proves nothing about idleness. + h.markReady() + if len(h.r.Replicas()) != 1 { + t.Fatal("the model did not start") + } + + h.advance(24 * time.Hour) + h.r.Reconcile(t.Context()) + + var live int + for _, rep := range h.r.Replicas() { + if rep.State.Counts() { + live++ + } + } + if live != 1 { + t.Errorf("an always-on model was evicted for being idle: %d live", live) + } +} diff --git a/internal/router/activate.go b/internal/router/activate.go new file mode 100644 index 00000000..20e5f851 --- /dev/null +++ b/internal/router/activate.go @@ -0,0 +1,69 @@ +package router + +import ( + "context" + "time" +) + +// Activator loads a model that is configured but not running. +// +// The control plane implements it. Behind an interface for the same reason the +// endpoint list is: the router knows a request has arrived for something that +// is not up, the control plane decides what runs, and neither needs to be in +// the other's process to say so. +type Activator interface { + // Want reports whether the model is one that loads on demand. False means + // there is genuinely nothing serving it, which is a different answer. + Want(model string) bool +} + +// SetActivator supplies the control plane that can load models on demand. +func (r *Router) SetActivator(a Activator) { r.activator = a } + +// activate asks for a model and waits a bounded time for it to serve. +// +// The one real design decision in on-demand loading. A request that triggers a +// load can either wait or be refused, and both are wrong on their own: refusing +// makes the feature useless, because the first request for every model fails; +// waiting without a bound turns a request into a job, and a client that budgeted +// thirty seconds is holding a connection for ten minutes. +// +// So it waits, briefly, and then says what is happening. A forty-second wait is +// a slow request. A fifteen-minute wait is a job pretending to be one, and the +// client should be told to come back rather than left holding a socket. +func (r *Router) activate(ctx context.Context, model string) bool { + if r.activator == nil || !r.activator.Want(model) { + return false + } + + wait := r.cfg.ActivationWait + if wait <= 0 { + wait = defaultActivationWait + } + ctx, cancel := context.WithTimeout(ctx, wait) + defer cancel() + + // Polled rather than notified. A load takes tens of seconds at best, so the + // latency of a poll is noise against it, and a notification path would be + // another thing to get wrong for no gain a client could measure. + t := time.NewTicker(250 * time.Millisecond) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return false + case <-t.C: + if len(r.reg.Snapshots(model)) > 0 { + return true + } + } + } +} + +// defaultActivationWait bounds how long a request holds a connection waiting +// for a model to load. +// +// Long enough for a small model on a warm node, short enough that a client's +// own timeout is not what ends the request — being told to retry is a better +// outcome than a socket closing with nothing on it. +const defaultActivationWait = 45 * time.Second diff --git a/internal/router/handlers.go b/internal/router/handlers.go index 357f7021..c83b14f9 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -284,6 +284,13 @@ func (r *Router) finish(w http.ResponseWriter, req *parsedRequest, res dispatchR if status == 0 { status = http.StatusOK } + case errors.Is(res.err, errLoading): + // Retry-After, because this one succeeds on a retry and a client that + // is told so will make it. Without the header a well-behaved client + // backs off exactly as it would from a permanent failure. + status = http.StatusServiceUnavailable + w.Header().Set("Retry-After", "30") + writeError(w, status, "server_error", res.err.Error()) case errors.Is(res.err, errNoCapacity): status = http.StatusServiceUnavailable writeError(w, status, "server_error", res.err.Error()) diff --git a/internal/router/router.go b/internal/router/router.go index 6fecd772..a3c8cdf8 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -59,6 +59,9 @@ type Router struct { // extraMetrics are collectors other components publish on /metrics. extraMetrics []prometheus.Collector + // activator loads a model that is configured but not currently running. + activator Activator + // landing configures the public page, which the router serves but does not // otherwise care about. landing config.Landing @@ -479,9 +482,30 @@ func (r *Router) dispatch(ctx context.Context, w http.ResponseWriter, req *parse var errNoCapacity = errors.New("no endpoint available") +// errLoading means the model is being started and the wait ran out. +// +// Distinct from having no capacity because the right client behaviour differs: +// this one succeeds on a retry, and the other does not. +var errLoading = errors.New("model loading") + func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *parsedRequest, model string, entry catalog.Model) dispatchResult { candidates := r.reg.Snapshots(model) if len(candidates) == 0 { + // Nothing is serving it. That may be because nothing ever will, or + // because it loads when asked and nobody has asked yet — and the second + // is not an error, it is the feature working. + if r.activate(ctx, model) { + candidates = r.reg.Snapshots(model) + } + } + if len(candidates) == 0 { + if r.activator != nil && r.activator.Want(model) { + // Asked for, still loading. Saying "no endpoint serves this" would + // be a lie, and a client told to retry will succeed where one told + // the model does not exist gives up. + return dispatchResult{err: fmt.Errorf("%w: %s", errLoading, + "the model is loading; retry shortly")} + } return dispatchResult{err: fmt.Errorf("%w: %s", errNoCapacity, noEndpointFor(model, r.servedModels()))} } diff --git a/internal/router/router_test.go b/internal/router/router_test.go index 2d510517..85401e0d 100644 --- a/internal/router/router_test.go +++ b/internal/router/router_test.go @@ -501,3 +501,94 @@ func TestAModelWithNoContextPublishesNoContextField(t *testing.T) { t.Errorf("an unknown context was published as a number: %s", rec.Body.String()) } } + +type fakeActivator struct { + onDemand map[string]bool + asked []string +} + +func (f *fakeActivator) Want(model string) bool { + f.asked = append(f.asked, model) + return f.onDemand[model] +} + +func TestALoadingModelIsToldToRetryRatherThanRefused(t *testing.T) { + // "no endpoint serves this" would be a lie while a model is starting, and + // the difference matters: a client told to retry succeeds, one told the + // model does not exist gives up. + e := newFakeEngine(t, "m") + cfg := baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"m"}}) + // A very short wait, because the point here is what happens when it runs + // out rather than how long it is. + cfg.Router.ActivationWait = 50 * time.Millisecond + rt := newTestRouter(t, cfg) + rt.SetActivator(&fakeActivator{onDemand: map[string]bool{"sleepy": true}}) + + rec := post(t, rt.Handler(), map[string]any{"model": "sleepy", "messages": []any{}}, nil) + if rec.Code != http.StatusServiceUnavailable { + t.Fatalf("status %d, expected 503", rec.Code) + } + if got := rec.Header().Get("Retry-After"); got == "" { + t.Error("no Retry-After; a well-behaved client backs off as it would from a permanent failure") + } + if !strings.Contains(rec.Body.String(), "loading") { + t.Errorf("the client is not told what is happening: %s", rec.Body.String()) + } +} + +func TestAModelThatIsSimplyUnknownStillSaysSo(t *testing.T) { + // The other half. On-demand must not turn every typo into "loading", + // because then nothing is ever wrong and the useful error disappears. + e := newFakeEngine(t, "m") + cfg := baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"m"}}) + rt := newTestRouter(t, cfg) + rt.SetActivator(&fakeActivator{onDemand: map[string]bool{}}) + + rec := post(t, rt.Handler(), map[string]any{"model": "typo", "messages": []any{}}, nil) + if rec.Code != http.StatusServiceUnavailable { + t.Fatalf("status %d", rec.Code) + } + if rec.Header().Get("Retry-After") != "" { + t.Error("an unknown model was presented as retryable") + } + if !strings.Contains(rec.Body.String(), "available models") { + t.Errorf("the useful error was lost: %s", rec.Body.String()) + } +} + +func TestAServedModelNeverWaitsForActivation(t *testing.T) { + // A model that is up must not pay a millisecond for the feature. + e := newFakeEngine(t, "m") + cfg := baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"m"}}) + cfg.Router.ActivationWait = 10 * time.Second + rt := newTestRouter(t, cfg) + act := &fakeActivator{onDemand: map[string]bool{"m": true}} + rt.SetActivator(act) + + start := time.Now() + rec := post(t, rt.Handler(), map[string]any{"model": "m", "messages": []any{}}, nil) + if rec.Code != http.StatusOK { + t.Fatalf("status %d: %s", rec.Code, rec.Body.String()) + } + if elapsed := time.Since(start); elapsed > time.Second { + t.Errorf("a served request took %s; activation is on the hot path", elapsed) + } + if len(act.asked) != 0 { + t.Errorf("the activator was consulted for a model that is already serving: %v", act.asked) + } +} + +func TestWithoutAnActivatorNothingChanges(t *testing.T) { + // Every deployment before this existed, and a router running without a + // control plane. + e := newFakeEngine(t, "m") + rt := newTestRouter(t, baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"m"}})) + + rec := post(t, rt.Handler(), map[string]any{"model": "nope", "messages": []any{}}, nil) + if rec.Code != http.StatusServiceUnavailable { + t.Errorf("status %d", rec.Code) + } + if !strings.Contains(rec.Body.String(), "available models") { + t.Errorf("body = %s", rec.Body.String()) + } +}