llmgw/internal/modelplan/fit_test.go

package modelplan

import (
	"strings"
	"testing"
)

func h100Node(nodes, gpus int, fabric string) Hardware {
	return Hardware{Class: "h100-80g", GPUMemGB: 80, GPUsPerNode: gpus, Nodes: nodes, Fabric: fabric}
}

func TestASmallModelFitsOneCard(t *testing.T) {
	fits := Fits(coder3B, []Hardware{h100Node(4, 8, "ib-a")}, 32768, []Precision{BF16})
	if len(fits) == 0 {
		t.Fatal("a 3B was reported as not fitting an H100")
	}
	// Best first, and the best answer for a small model is one card.
	if fits[0].TP != 1 {
		t.Errorf("best fit uses %d GPUs; a 3B needs one", fits[0].TP)
	}
}

func TestA70BNeedsTwoCardsAtFullPrecision(t *testing.T) {
	// The configuration people run every day, and the one an earlier overhead
	// model wrongly called impossible by reserving slack twice.
	fits := Fits(llama70B, []Hardware{h100Node(4, 8, "ib-a")}, 8192, []Precision{BF16})
	if len(fits) == 0 {
		t.Fatal("70B bf16 reported as not fitting eight H100s")
	}
	if fits[0].TP != 2 {
		t.Errorf("best fit is tp=%d; 70B at bf16 on two 80 GB cards is routine", fits[0].TP)
	}
	for _, f := range fits {
		if f.TP == 1 {
			t.Errorf("70B bf16 reported as fitting a single card: %s", f.Describe())
		}
	}
}

func TestContextChangesHowManyCardsAModelNeeds(t *testing.T) {
	// The fact that makes this worth computing rather than looking up. The same
	// 70B at the same precision needs two cards for an 8k pool and four for
	// 32k, because the KV cache is what grows — and "70B needs two H100s" is
	// repeated everywhere without the context it was true for.
	hw := []Hardware{h100Node(4, 8, "ib-a")}
	short := Fits(llama70B, hw, 8192, []Precision{BF16})
	long := Fits(llama70B, hw, 32768, []Precision{BF16})
	if len(short) == 0 || len(long) == 0 {
		t.Fatal("expected fits at both context sizes")
	}
	if long[0].TP <= short[0].TP {
		t.Errorf("tp at 32k is %d and at 8k is %d; a larger context should need at least as much",
			long[0].TP, short[0].TP)
	}
}

func TestQuantisationOpensUpSmallerHardware(t *testing.T) {
	// The answer someone with one spare card needs: not "no", but "not at this
	// precision".
	one := []Hardware{h100Node(1, 1, "")}
	if full := Fits(llama70B, one, 8192, []Precision{BF16}); len(full) != 0 {
		t.Errorf("70B bf16 fitted a single card: %s", full[0].Describe())
	}
	if quant := Fits(llama70B, one, 8192, []Precision{Q4GGUF}); len(quant) == 0 {
		t.Error("70B at q4 did not fit a single 80 GB card, which it should")
	}
}

func TestATensorGroupNeverSpansFabrics(t *testing.T) {
	// Two nodes with no fabric are two islands. A multi-node replica placed
	// across them would start, pass health checks and hang on its first
	// all-reduce, which is worse than refusing to plan it.
	noFabric := Hardware{Class: "h100-80g", GPUMemGB: 80, GPUsPerNode: 2, Nodes: 4}
	for _, f := range Fits(llama70B, []Hardware{noFabric}, 8192, []Precision{BF16}) {
		if f.Nodes > 1 {
			t.Errorf("planned %d nodes with no fabric: %s", f.Nodes, f.Describe())
		}
	}
	withFabric := noFabric
	withFabric.Fabric = "ib-a"
	var spanning bool
	for _, f := range Fits(llama70B, []Hardware{withFabric}, 8192, []Precision{BF16}) {
		if f.Nodes > 1 {
			spanning = true
		}
	}
	if !spanning {
		t.Error("a fabric was available and no multi-node fit was offered")
	}
}

func TestTensorWidthFitsTheKVHeads(t *testing.T) {
	// Grouped-query attention leaves far fewer KV heads than attention heads.
	// An engine splits them among the cards, or past their number gives each
	// card a copy of one: so a width that neither divides them nor is a
	// multiple of them fails at launch, and is not offered.
	for _, f := range Fits(llama70B, []Hardware{h100Node(8, 8, "ib-a")}, 8192, []Precision{Q4GGUF}) {
		if llama70B.KVHeads%f.TP != 0 && f.TP%llama70B.KVHeads != 0 {
			t.Errorf("tp=%d does not fit the model's %d KV heads", f.TP, llama70B.KVHeads)
		}
	}
}

func TestOnlyPowersOfTwoAreOffered(t *testing.T) {
	for _, f := range Fits(llama70B, []Hardware{h100Node(4, 6, "ib-a")}, 8192, []Precision{Q4GGUF}) {
		if f.TP&(f.TP-1) != 0 {
			t.Errorf("tp=%d is not a power of two", f.TP)
		}
	}
}

func TestACardIsNeverPlannedToTheBrim(t *testing.T) {
	// A plan that fills a card completely is one that loads on a good day.
	// This matches vLLM's own 0.90 default deliberately: a planner that
	// disagrees with the engine it plans for is worse than none.
	for _, f := range Fits(coder14B, []Hardware{h100Node(1, 8, "")}, 32768, []Precision{BF16}) {
		if f.PerGPUGB > 80*usableFraction+0.01 {
			t.Errorf("planned %.1f GB on an 80 GB card: %s", f.PerGPUGB, f.Describe())
		}
		if f.Headroom < 0 {
			t.Errorf("negative headroom: %s", f.Describe())
		}
	}
}

func TestACPUNodeCanServeASmallModel(t *testing.T) {
	// How this cluster began, and a reasonable way to try llmgw before
	// committing cards to it.
	cpu := Hardware{HostMemGB: 32}
	fits := Fits(coder3B, []Hardware{cpu}, 32768, []Precision{Q4GGUF})
	if len(fits) == 0 {
		t.Fatal("a 3B at q4 did not fit a 32 GB CPU node")
	}
	if !fits[0].CPUOnly {
		t.Error("the fit is not marked CPU-only")
	}
	if !strings.Contains(fits[0].Describe(), "CPU") {
		t.Errorf("the description does not say it is CPU: %q", fits[0].Describe())
	}
	// And a 70B does not, whatever the quantisation.
	if big := Fits(llama70B, []Hardware{cpu}, 8192, []Precision{Q4GGUF}); len(big) != 0 {
		t.Errorf("a 70B was reported as fitting 32 GB of RAM: %s", big[0].Describe())
	}
}

func TestNothingFitsIsAnEmptyAnswerNotAWrongOne(t *testing.T) {
	tiny := Hardware{Class: "t4-16g", GPUMemGB: 16, GPUsPerNode: 1, Nodes: 1}
	if fits := Fits(llama70B, []Hardware{tiny}, 32768, []Precision{BF16}); len(fits) != 0 {
		t.Errorf("70B bf16 fitted a single 16 GB card: %s", fits[0].Describe())
	}
}

func TestIdenticalHardwareOnTwoFabricsIsNotOfferedTwice(t *testing.T) {
	// A single-node replica does not span anything, so which fabric its node is
	// attached to changes nothing about whether the model fits. Listing it once
	// per fabric is noise: on a real fleet a 32B produced twenty-six rows of
	// which thirteen were duplicates.
	hw := []Hardware{h100Node(2, 4, "ib-a"), h100Node(2, 4, "ib-b")}
	fits := Fits(coder14B, hw, 8192, []Precision{BF16})

	seen := map[int]int{}
	for _, f := range fits {
		if f.Nodes == 1 {
			seen[f.TP]++
		}
	}
	for tp, n := range seen {
		if n > 1 {
			t.Errorf("single-node tp=%d offered %d times", tp, n)
		}
	}
}

func TestAMultiNodeFitIsStillOfferedPerFabric(t *testing.T) {
	// The other half: a replica that spans nodes is fabric-specific, and which
	// fabric has room is exactly what the operator needs told.
	hw := []Hardware{h100Node(2, 2, "ib-a"), h100Node(2, 2, "ib-b")}
	fabrics := map[string]bool{}
	for _, f := range Fits(llama70B, hw, 8192, []Precision{Q4GGUF}) {
		if f.Nodes > 1 {
			fabrics[f.Fabric] = true
		}
	}
	if len(fabrics) < 2 {
		t.Errorf("multi-node fits were offered for %d fabrics, want both: %v", len(fabrics), fabrics)
	}
}