diff --git a/test/e2e/ondemand_test.go b/test/e2e/ondemand_test.go new file mode 100644 index 0000000..345d94b --- /dev/null +++ b/test/e2e/ondemand_test.go @@ -0,0 +1,196 @@ +package e2e + +import ( + "context" + "fmt" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" + "gitea.mynodes.xyz/ilker/llmgw/internal/config" + "gitea.mynodes.xyz/ilker/llmgw/internal/control" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" + "gitea.mynodes.xyz/ilker/llmgw/internal/router" +) + +// onDemandHarness runs a control plane whose only model loads when asked for. +// +// End to end rather than in the reconciler's own tests, because on-demand +// loading is four components agreeing: the router notices nothing is serving, +// the control plane decides the model may be started, the reconciler places it, +// and the endpoint list carries it back. Every one of those is tested alone. +// The wiring between them is where this session's real bugs have been — a +// collector registered after the registry was built, a readiness check that +// never ran — and none of them failed a unit test. +func onDemandHarness(t *testing.T, idle time.Duration) *controlHarness { + t.Helper() + + cat, err := catalog.New([]catalog.Model{{ + Name: testModel, Source: "/models/test", + Scaling: catalog.Scaling{Min: 0, Max: 1, IdleTimeout: idle}, + Profiles: []catalog.Profile{{Name: "single", GPUClass: "h100-80g", TP: 1, Weight: 1}}, + }}) + if err != nil { + t.Fatal(err) + } + + cfg := config.Default() + cfg.Router.ScrapeInterval = 200 * time.Millisecond + cfg.Router.StaleAfter = 3 * time.Second + cfg.Router.Admission.Enabled = false + cfg.Router.ActivationWait = 10 * time.Second + + rt, err := router.New(cfg, slog.New(slog.DiscardHandler)) + if err != nil { + t.Fatal(err) + } + disp := newSimDispatcher(t) + + ccfg := control.DefaultConfig() + ccfg.Reconciler.Interval = 100 * time.Millisecond + ccfg.Reconciler.BaseBackoff = 200 * time.Millisecond + ccfg.Reconciler.MaxBackoff = 500 * time.Millisecond + ccfg.Reconciler.UnreachableAfter = 300 * time.Millisecond + ccfg.Reconciler.LostAfter = 600 * time.Millisecond + ccfg.Reconciler.SettleFor = 400 * time.Millisecond + ccfg.Reconciler.IdleTimeout = idle + ccfg.PublishInterval = 100 * time.Millisecond + + ctl := control.New(ccfg, catalog.NewLive(cat), disp, rt, slog.New(slog.DiscardHandler)) + disp.onStatus = ctl.Reconciler().OnShardStatus + rt.SetActivator(ctl.Reconciler()) + + ctl.Inventory().Upsert(inventory.Node{ + ID: "n0", MgmtAddr: "127.0.0.1", State: inventory.Ready, + GPUs: []discovery.GPU{{ + Index: 0, Name: "NVIDIA H100 80GB HBM3", + MemoryTotalMB: 81559, ComputeCapability: "9.0", + }}, + }) + + ctx, cancel := context.WithCancel(context.Background()) + rt.Start(ctx) + ctl.Start(ctx) + + h := &controlHarness{ + router: httptest.NewServer(rt.Handler()), + rt: rt, ctl: ctl, disp: disp, inv: ctl.Inventory(), + } + // The node has to keep reporting. UnreachableAfter is 300ms in this + // harness and every wait below is longer than that, so a single heartbeat + // would have the node go quiet and nothing would be placed at all — which + // looks exactly like on-demand loading failing. + beat, stopBeat := context.WithCancel(ctx) + go func() { + t := time.NewTicker(50 * time.Millisecond) + defer t.Stop() + for { + select { + case <-beat.Done(): + return + case <-t.C: + h2 := h + if h2 != nil { + h2.heartbeat("n0") + } + } + } + }() + + t.Cleanup(func() { + stopBeat() + h.router.Close() + ctl.Close() + cancel() + rt.Stop() + }) + return h +} + +func TestARequestLoadsAnOnDemandModelAndIsServed(t *testing.T) { + h := onDemandHarness(t, time.Hour) + // Settled, and still nothing running: that is the feature, not a fault. + time.Sleep(700 * time.Millisecond) + if n := h.disp.liveShards(); n != 0 { + t.Fatalf("%d shards running before anyone asked for the model", n) + } + + // One ordinary request, which should start the model and then be answered + // by it rather than refused. + resp, body := chat(t, h.router.URL, testModel) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status %d: %s", resp.StatusCode, body) + } + if h.disp.liveShards() == 0 { + t.Error("the request was served but nothing is running") + } +} + +func TestAnIdleOnDemandModelIsStoppedEndToEnd(t *testing.T) { + // Short enough to observe, long enough that the model is not evicted while + // the first request is still in flight. + h := onDemandHarness(t, 2*time.Second) + time.Sleep(700 * time.Millisecond) + + if resp, body := chat(t, h.router.URL, testModel); resp.StatusCode != http.StatusOK { + t.Fatalf("status %d: %s", resp.StatusCode, body) + } + if h.disp.liveShards() == 0 { + t.Fatal("the model never started") + } + + // Left alone past its idle timeout, with the node still heartbeating so + // this is eviction rather than node loss. + deadline := time.Now().Add(15 * time.Second) + for time.Now().Before(deadline) { + if h.disp.liveShards() == 0 { + return + } + time.Sleep(200 * time.Millisecond) + } + t.Error("an idle on-demand model was never stopped") +} + +func TestAnUnknownModelIsStillAnErrorWithOnDemandEnabled(t *testing.T) { + // On-demand must not turn every typo into "loading", or nothing is ever + // wrong and the useful error disappears. + h := onDemandHarness(t, time.Hour) + time.Sleep(700 * time.Millisecond) + + resp, body := chat(t, h.router.URL, "a-model-nobody-configured") + if resp.StatusCode != http.StatusServiceUnavailable { + t.Fatalf("status %d: %s", resp.StatusCode, body) + } + if resp.Header.Get("Retry-After") != "" { + t.Error("an unknown model was presented as retryable") + } + // With an on-demand-only cluster and nothing loaded, naming what is served + // means naming nothing — and saying so plainly is the honest answer. + if !strings.Contains(body, "no model is currently being served") { + t.Errorf("the useful error was lost: %s", body) + } +} + +// chat sends one completion and returns the response and its body. +func chat(t *testing.T, base, model string) (*http.Response, string) { + t.Helper() + body := fmt.Sprintf(`{"model":%q,"messages":[{"role":"user","content":"hello"}],"max_tokens":8}`, model) + req, err := http.NewRequest(http.MethodPost, base+"/v1/chat/completions", strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Content-Type", "application/json") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + buf := make([]byte, 4096) + n, _ := resp.Body.Read(buf) + return resp, string(buf[:n]) +}