package e2e import ( "context" "fmt" "slices" "sync" "sync/atomic" "testing" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/config" ) // TestPriorityIsolation is the phase 1.5 exit criterion. // // While a sustained flood of batch requests runs in the bronze band, // interactive requests in the platinum band must keep their latency. This is // the whole reason admission exists: an engine queue is FIFO, so once a batch // job is in it no amount of clever endpoint selection can protect anyone. // // It is meant to stay in CI permanently. func TestPriorityIsolation(t *testing.T) { workers := envInt("LLMGW_E2E_BATCH", 500) probes := envInt("LLMGW_E2E_PROBES", 40) if testing.Short() { workers, probes = 80, 15 } h := start(t, 3, func(c *config.Server) { c.Router.Admission.Enabled = true }, withLoadedEngines()) idle := measureProbes(t, h, probes, "platinum") t.Logf("idle platinum: p50=%v p99=%v", idle.p50.Round(time.Millisecond), idle.p99.Round(time.Millisecond)) load := startBronzeLoad(t, h, workers, "bronze") defer load.stop() loaded := measureProbes(t, h, probes, "platinum") t.Logf("loaded platinum: p50=%v p99=%v (%s)", loaded.p50.Round(time.Millisecond), loaded.p99.Round(time.Millisecond), load.summary()) // The criterion: interactive p99 under load stays within 1.5x of idle. // The floor keeps a few-millisecond baseline from making the bound absurd. budget := max(time.Duration(float64(idle.p99)*1.5), idle.p99+50*time.Millisecond) t.Logf("platinum p99 %v against a %v budget (%.2fx idle)", loaded.p99.Round(time.Millisecond), budget.Round(time.Millisecond), float64(loaded.p99)/float64(max(idle.p99, time.Microsecond))) if loaded.p99 > budget { t.Errorf("interactive p99 rose to %v under bronze load, over the %v budget; "+ "admission control is not protecting the high band", loaded.p99.Round(time.Millisecond), budget.Round(time.Millisecond)) } } // TestAdmissionDisabledShowsTheProblem documents why the layer exists: the same // load does wreck interactive latency once admission is switched off. If this // ever stops showing degradation, the load generator has stopped being a load // and the test above proves less than it appears to. func TestAdmissionDisabledShowsTheProblem(t *testing.T) { if testing.Short() { t.Skip("load-dependent") } workers := envInt("LLMGW_E2E_BATCH", 500) h := start(t, 3, func(c *config.Server) { c.Router.Admission.Enabled = false }, withLoadedEngines()) idle := measureProbes(t, h, 20, "") load := startBronzeLoad(t, h, workers, "") defer load.stop() loaded := measureProbes(t, h, 20, "") ratio := float64(loaded.p99) / float64(max(idle.p99, time.Microsecond)) t.Logf("without admission: idle p99=%v loaded p99=%v (%.1fx)", idle.p99.Round(time.Millisecond), loaded.p99.Round(time.Millisecond), ratio) if ratio < 2 { t.Skipf("the load did not degrade latency (%.1fx); this machine is too fast "+ "for %d workers, so the isolation test proves less than it should", ratio, workers) } } // --- load generation --- // bronzeLoad is a sustained batch workload. // // Workers loop rather than firing once: a one-shot burst drains while the // probes are still running, and the measurement then catches a recovering // cluster instead of a loaded one. type bronzeLoad struct { cancel context.CancelFunc wg sync.WaitGroup done atomic.Int64 failed atomic.Int64 firstEr atomic.Value } func startBronzeLoad(t *testing.T, h *harness, workers int, band string) *bronzeLoad { t.Helper() ctx, cancel := context.WithCancel(context.Background()) l := &bronzeLoad{cancel: cancel} for i := range workers { l.wg.Add(1) go func() { defer l.wg.Done() hdr := map[string]string{"x-session-id": fmt.Sprintf("batch-%d", i%8)} if band != "" { hdr["x-llmgw-band"] = band } for ctx.Err() == nil { _, _, err := streamRequest(ctx, h.router.URL, chatBody(fmt.Sprintf("batch job %d", i), 200, true), hdr) switch { case err == nil: l.done.Add(1) case ctx.Err() != nil: // shutting down, expected default: l.failed.Add(1) l.firstEr.CompareAndSwap(nil, err) } } }() } l.waitUntilBusy(t, h) return l } // waitUntilBusy blocks until the engines are demonstrably working, so probes // are not taken while the load is still ramping up. // // It waits on engines being busy rather than full: when admission is doing its // job the pool is deliberately held below capacity, so a fullness check would // time out exactly when the system is behaving correctly. func (l *bronzeLoad) waitUntilBusy(t *testing.T, h *harness) { t.Helper() const busyFor = 500 * time.Millisecond deadline := time.Now().Add(20 * time.Second) var since time.Time for time.Now().Before(deadline) { if h.running() > 0 { if since.IsZero() { since = time.Now() } else if time.Since(since) >= busyFor { return } } else { since = time.Time{} } time.Sleep(10 * time.Millisecond) } t.Fatalf("engines never became busy; %s", l.summary()) } func (l *bronzeLoad) summary() string { return fmt.Sprintf("bronze: completed=%d failed=%d firstErr=%v", l.done.Load(), l.failed.Load(), l.firstEr.Load()) } func (l *bronzeLoad) stop() { l.cancel() l.wg.Wait() } // --- probes --- type latency struct{ p50, p99 time.Duration } // measureProbes sends sequential short requests and reports their latency. // Sequential on purpose: these stand in for a single interactive user, and // concurrency would measure something else. func measureProbes(t *testing.T, h *harness, n int, band string) latency { t.Helper() hdr := map[string]string{"x-session-id": "probe"} if band != "" { hdr["x-llmgw-band"] = band } samples := make([]time.Duration, 0, n) for range n { ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) start := time.Now() _, _, err := streamRequest(ctx, h.router.URL, chatBody("ping", 4, true), hdr) cancel() if err != nil { t.Fatalf("probe failed: %v", err) } samples = append(samples, time.Since(start)) } slices.Sort(samples) return latency{ p50: samples[len(samples)/2], p99: samples[min(int(float64(len(samples))*0.99), len(samples)-1)], } }