llmgw/test/e2e/phase15_test.go

package e2e

import (
	"context"
	"fmt"
	"slices"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"gitea.mynodes.xyz/ilker/llmgw/internal/config"
)

// TestPriorityIsolation is the phase 1.5 exit criterion.
//
// While a sustained flood of batch requests runs in the bronze band,
// interactive requests in the platinum band must keep their latency. This is
// the whole reason admission exists: an engine queue is FIFO, so once a batch
// job is in it no amount of clever endpoint selection can protect anyone.
//
// It is meant to stay in CI permanently.
func TestPriorityIsolation(t *testing.T) {
	workers := envInt("LLMGW_E2E_BATCH", 500)
	probes := envInt("LLMGW_E2E_PROBES", 40)
	if testing.Short() {
		workers, probes = 80, 15
	}

	h := start(t, 3, func(c *config.Server) {
		c.Router.Admission.Enabled = true
	}, withLoadedEngines())

	idle := measureProbes(t, h, probes, "platinum")
	t.Logf("idle platinum:   p50=%v p99=%v", idle.p50.Round(time.Millisecond), idle.p99.Round(time.Millisecond))

	load := startBronzeLoad(t, h, workers, "bronze")
	defer load.stop()

	loaded := measureProbes(t, h, probes, "platinum")
	t.Logf("loaded platinum: p50=%v p99=%v  (%s)",
		loaded.p50.Round(time.Millisecond), loaded.p99.Round(time.Millisecond), load.summary())

	// The criterion: interactive p99 under load stays within 1.5x of idle.
	// The floor keeps a few-millisecond baseline from making the bound absurd.
	budget := max(time.Duration(float64(idle.p99)*1.5), idle.p99+50*time.Millisecond)
	t.Logf("platinum p99 %v against a %v budget (%.2fx idle)",
		loaded.p99.Round(time.Millisecond), budget.Round(time.Millisecond),
		float64(loaded.p99)/float64(max(idle.p99, time.Microsecond)))

	if loaded.p99 > budget {
		t.Errorf("interactive p99 rose to %v under bronze load, over the %v budget; "+
			"admission control is not protecting the high band",
			loaded.p99.Round(time.Millisecond), budget.Round(time.Millisecond))
	}
}

// TestAdmissionDisabledShowsTheProblem documents why the layer exists: the same
// load does wreck interactive latency once admission is switched off. If this
// ever stops showing degradation, the load generator has stopped being a load
// and the test above proves less than it appears to.
func TestAdmissionDisabledShowsTheProblem(t *testing.T) {
	if testing.Short() {
		t.Skip("load-dependent")
	}
	workers := envInt("LLMGW_E2E_BATCH", 500)

	h := start(t, 3, func(c *config.Server) {
		c.Router.Admission.Enabled = false
	}, withLoadedEngines())

	idle := measureProbes(t, h, 20, "")

	load := startBronzeLoad(t, h, workers, "")
	defer load.stop()

	loaded := measureProbes(t, h, 20, "")
	ratio := float64(loaded.p99) / float64(max(idle.p99, time.Microsecond))
	t.Logf("without admission: idle p99=%v loaded p99=%v (%.1fx)",
		idle.p99.Round(time.Millisecond), loaded.p99.Round(time.Millisecond), ratio)

	if ratio < 2 {
		t.Skipf("the load did not degrade latency (%.1fx); this machine is too fast "+
			"for %d workers, so the isolation test proves less than it should", ratio, workers)
	}
}

// --- load generation ---

// bronzeLoad is a sustained batch workload.
//
// Workers loop rather than firing once: a one-shot burst drains while the
// probes are still running, and the measurement then catches a recovering
// cluster instead of a loaded one.
type bronzeLoad struct {
	cancel  context.CancelFunc
	wg      sync.WaitGroup
	done    atomic.Int64
	failed  atomic.Int64
	firstEr atomic.Value
}

func startBronzeLoad(t *testing.T, h *harness, workers int, band string) *bronzeLoad {
	t.Helper()

	ctx, cancel := context.WithCancel(context.Background())
	l := &bronzeLoad{cancel: cancel}

	for i := range workers {
		l.wg.Add(1)
		go func() {
			defer l.wg.Done()
			hdr := map[string]string{"x-session-id": fmt.Sprintf("batch-%d", i%8)}
			if band != "" {
				hdr["x-llmgw-band"] = band
			}
			for ctx.Err() == nil {
				_, _, err := streamRequest(ctx, h.router.URL,
					chatBody(fmt.Sprintf("batch job %d", i), 200, true), hdr)
				switch {
				case err == nil:
					l.done.Add(1)
				case ctx.Err() != nil: // shutting down, expected
				default:
					l.failed.Add(1)
					l.firstEr.CompareAndSwap(nil, err)
				}
			}
		}()
	}

	l.waitUntilBusy(t, h)
	return l
}

// waitUntilBusy blocks until the engines are demonstrably working, so probes
// are not taken while the load is still ramping up.
//
// It waits on engines being busy rather than full: when admission is doing its
// job the pool is deliberately held below capacity, so a fullness check would
// time out exactly when the system is behaving correctly.
func (l *bronzeLoad) waitUntilBusy(t *testing.T, h *harness) {
	t.Helper()
	const busyFor = 500 * time.Millisecond

	deadline := time.Now().Add(20 * time.Second)
	var since time.Time
	for time.Now().Before(deadline) {
		if h.running() > 0 {
			if since.IsZero() {
				since = time.Now()
			} else if time.Since(since) >= busyFor {
				return
			}
		} else {
			since = time.Time{}
		}
		time.Sleep(10 * time.Millisecond)
	}
	t.Fatalf("engines never became busy; %s", l.summary())
}

func (l *bronzeLoad) summary() string {
	return fmt.Sprintf("bronze: completed=%d failed=%d firstErr=%v",
		l.done.Load(), l.failed.Load(), l.firstEr.Load())
}

func (l *bronzeLoad) stop() {
	l.cancel()
	l.wg.Wait()
}

// --- probes ---

type latency struct{ p50, p99 time.Duration }

// measureProbes sends sequential short requests and reports their latency.
// Sequential on purpose: these stand in for a single interactive user, and
// concurrency would measure something else.
func measureProbes(t *testing.T, h *harness, n int, band string) latency {
	t.Helper()
	hdr := map[string]string{"x-session-id": "probe"}
	if band != "" {
		hdr["x-llmgw-band"] = band
	}

	samples := make([]time.Duration, 0, n)
	for range n {
		ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
		start := time.Now()
		_, _, err := streamRequest(ctx, h.router.URL, chatBody("ping", 4, true), hdr)
		cancel()
		if err != nil {
			t.Fatalf("probe failed: %v", err)
		}
		samples = append(samples, time.Since(start))
	}
	slices.Sort(samples)
	return latency{
		p50: samples[len(samples)/2],
		p99: samples[min(int(float64(len(samples))*0.99), len(samples)-1)],
	}
}