package web import ( "html" "strings" "testing" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/hf" ) // nodesFleet is a cluster of the given nodes, all ready. type nodesFleet struct { fakeFleet nodes []adminapi.NodeView } func (f nodesFleet) Nodes() []adminapi.NodeView { return f.nodes } func gpuNode(id, class string, cards int, cc string, ramGB int) adminapi.NodeView { return adminapi.NodeView{ID: id, State: "ready", GPUTotal: cards, GPUFree: cards, MemoryGB: ramGB, Classes: []string{class}, ClassGPUs: []adminapi.ClassGPUs{{Class: class, Total: cards, Free: cards}}, ComputeCapabilities: []string{cc}} } func cpuNode(id string, ramGB int) adminapi.NodeView { return adminapi.NodeView{ID: id, State: "ready", MemoryGB: ramGB} } func stModel(id string, params map[string]int64, tags ...string) hf.Model { var total int64 for dt, n := range params { if dt != "I32" { total += n } } return hf.Model{ID: id, CreatedAt: time.Now(), LastModified: time.Now(), PipelineTag: "text-generation", Tags: append([]string{"safetensors"}, tags...), Safetensors: &hf.SafetensorsInfo{Total: total, Parameters: params}} } // listing is what the explorer shows of these repositories on this cluster. func listing(t *testing.T, nodes []adminapi.NodeView, ms ...hf.Model) (string, *fakeRegistry) { t.Helper() reg := &fakeRegistry{results: ms} mux := mirrorPageLlama(t, MirrorDeps{Mirror: &fakeMirror{}, Registry: reg, Fleet: nodesFleet{nodes: nodes}}) return mirrorGet(t, mux, "/admin/mirror?tab=explore").Body.String(), reg } func TestTheExplorerListsWhatThisClustersHardwareHolds(t *testing.T) { // The PoC: one Turing card of 24 GB, CPU nodes of up to 31 GB. poc := []adminapi.NodeView{gpuNode("cuda01", "6000-24g", 1, "7.5", 251), cpuNode("node05", 31)} body, reg := listing(t, poc, ggufModel("a/Small-GGUF", "qwen3", 8e9, time.Now()), ggufModel("a/Huge-GGUF", "qwen3moe", 671e9, time.Now()), stModel("a/Eight-BF16", map[string]int64{"BF16": 8e9}), stModel("a/ThirtyTwo-BF16", map[string]int64{"BF16": 32e9}), stModel("a/Eight-FP8", map[string]int64{"F8_E4M3": 8e9, "BF16": 1e8}), stModel("a/ThirtyTwo-AWQ", map[string]int64{"BF16": 1e9, "I32": 31e9}, "awq", "4-bit"), stModel("a/Eight-MLX", map[string]int64{"U32": 8e9}, "mlx", "8-bit"), ) if reg.query.GGUF || reg.query.Limit != 100 { t.Errorf("with a card, safetensors are asked for too, more of them: %+v", reg.query) } for id, want := range map[string]bool{ "a/Small-GGUF": true, "a/Huge-GGUF": false, "a/Eight-BF16": true, "a/ThirtyTwo-BF16": false, "a/Eight-FP8": false, "a/ThirtyTwo-AWQ": true, "a/Eight-MLX": false, } { if got := strings.Contains(body, id+"<") || strings.Contains(body, id+"\""); got != want { t.Errorf("%s listed = %v, want %v", id, got, want) } } if !strings.Contains(body, "largest card has 24 GB (compute capability 7.5)") || !strings.Contains(body, "4 more are hidden") { t.Errorf("the page should say what it judged by and how many it left out: %s", body) } } func TestANewerClusterSeesTheModelsItCanRun(t *testing.T) { // A customer's: eight H100s. What the PoC cannot run, this one lists. h100 := []adminapi.NodeView{gpuNode("gpu01", "h100-80g", 8, "9.0", 1024)} body, _ := listing(t, h100, stModel("a/ThirtyTwo-BF16", map[string]int64{"BF16": 32e9}), stModel("a/Eight-FP8", map[string]int64{"F8_E4M3": 8e9, "BF16": 1e8}), ggufModel("a/Huge-GGUF", "qwen3moe", 671e9, time.Now()), ) for _, id := range []string{"a/ThirtyTwo-BF16", "a/Eight-FP8", "a/Huge-GGUF"} { if !strings.Contains(body, id) { t.Errorf("%s should be listed on eight H100s", id) } } } func TestACPUOnlyClusterIsShownGGUFOnly(t *testing.T) { body, reg := listing(t, []adminapi.NodeView{cpuNode("c1", 32)}, ggufModel("a/Small-GGUF", "qwen3", 8e9, time.Now()), ggufModel("a/Big-GGUF", "qwen3", 70e9, time.Now()), stModel("a/Eight-BF16", map[string]int64{"BF16": 8e9}), ) if !reg.query.GGUF { t.Errorf("without cards only GGUF runs, and should be asked for: %+v", reg.query) } if !strings.Contains(body, "a/Small-GGUF") || strings.Contains(body, "a/Big-GGUF") || strings.Contains(body, "a/Eight-BF16") { t.Errorf("a 32 GB CPU node holds the 8B GGUF only: %s", body) } if !strings.Contains(body, "no GPUs, so GGUF only") { t.Error("the page should say it judged by a cluster without cards") } } func TestARepositorysFileTooBigForThisClusterIsNotOffered(t *testing.T) { poc := []adminapi.NodeView{gpuNode("cuda01", "6000-24g", 1, "7.5", 251), cpuNode("node05", 31)} reg := &fakeRegistry{repo: hf.Repo{ID: "a/M", Revision: "abc123def456", Params: 32e9, Files: []hf.File{ {Path: "M-Q4_K_M.gguf", Bytes: 19 << 30}, {Path: "model-00001-of-00002.safetensors", Bytes: 32 << 30}, {Path: "model-00002-of-00002.safetensors", Bytes: 32 << 30}, }}} mux := mirrorPageLlama(t, MirrorDeps{Mirror: &fakeMirror{}, Registry: reg, Fleet: nodesFleet{nodes: poc}}) body := mirrorGet(t, mux, "/admin/mirror?repo=a/M").Body.String() if !strings.Contains(body, "/resolve/abc123def456/M-Q4_K_M.gguf") { t.Errorf("the GGUF fits cuda01's RAM and should be offered: %s", body) } if strings.Contains(body, "/resolve/abc123def456/model-00001-of-00002.safetensors") || !strings.Contains(body, "too big for this cluster") { t.Errorf("64 GB of safetensors on one 24 GB card should be marked, not offered: %s", body) } } func TestPackedWeightsAreSizedAsTheyAreStored(t *testing.T) { // gpt-oss-20b: MXFP4 in U8, counted by the registry at the weights it // unpacks to. At a byte each it read as 22 GB and missed a 24 GB card. oss := stModel("openai/gpt-oss-20b", map[string]int64{"BF16": 1804459584, "U8": 19110297600}, "8-bit", "mxfp4") if gb := float64(oss.WeightBytes()) / (1 << 30); gb < 11 || gb > 13 { t.Errorf("gpt-oss-20b sized %.1f GB, want about 12", gb) } // Tagged mlx, but with a GGUF too: the GGUF runs. both := ggufModel("a/Both", "qwen3", 27e9, time.Now()) both.Tags = []string{"mlx", "safetensors", "gguf"} poc := []adminapi.NodeView{gpuNode("cuda01", "6000-24g", 1, "7.5", 251)} body, _ := listing(t, poc, oss, both) if !strings.Contains(body, "openai/gpt-oss-20b") || !strings.Contains(body, "a/Both") { t.Errorf("both run on cuda01: %s", body) } } func TestTheExplorerSaysHowFastAModelAnswersOnACPU(t *testing.T) { slow, fast := cpuNode("node01", 7), cpuNode("node05", 31) slow.MemBandwidthGBs, fast.MemBandwidthGBs = 24.9, 41.9 body, _ := listing(t, []adminapi.NodeView{slow, fast}, ggufModel("Qwen/Qwen3-8B-GGUF", "qwen3", 8e9, time.Now()), ggufModel("Qwen/Qwen3-30B-A3B-GGUF", "qwen3moe", 30e9, time.Now()), ) // 8B at Q4_K_M is 4.8 GB, on node05's 41.9 GB/s: about 5.8 a second. // The 30B model of experts reads 1.6 times its 3B active part: about 9.4. for _, want := range []string{">~5.8<", ">~9.4<", "on node05's CPU"} { if !strings.Contains(body, want) { t.Errorf("the listing lacks %q: %s", want, body) } } reg := &fakeRegistry{repo: hf.Repo{ID: "Qwen/Qwen3-8B-GGUF", Revision: "abc123def456", Params: 8e9, Files: []hf.File{{Path: "Qwen3-8B-Q4_K_M.gguf", Bytes: 5 << 30}, {Path: "Qwen3-8B-Q8_0.gguf", Bytes: 8 << 30}}}} mux := mirrorPageLlama(t, MirrorDeps{Mirror: &fakeMirror{}, Registry: reg, Fleet: nodesFleet{nodes: []adminapi.NodeView{slow, fast}}}) page := html.UnescapeString(mirrorGet(t, mux, "/admin/mirror?repo=Qwen/Qwen3-8B-GGUF").Body.String()) if !strings.Contains(page, "~5.2 tok/s on node05's CPU") || !strings.Contains(page, "~3.3 tok/s on node05's CPU") { t.Errorf("each file should say its speed on the CPU, by its own size: %s", page) } } // cards is a node of n cards of a class, named as the driver names them, // joined by NVLink or not, on a fabric or none. func cards(id, class, name string, n int, cc string, nvlink bool, fabric string) adminapi.NodeView { v := gpuNode(id, class, n, cc, 1024) for i := 0; i < n; i++ { g := adminapi.GPUView{Index: i, Name: name, Class: class} if nvlink { for j := 0; j < n; j++ { if j != i { g.NVLinkPeers = append(g.NVLinkPeers, j) } } } v.GPUs = append(v.GPUs, g) } if fabric != "" { v.Fabrics = []string{fabric} v.IBPorts = []adminapi.IBPortView{{HCA: "mlx5_0", Port: 1, Fabric: fabric, LinkLayer: "InfiniBand", RateGbps: 200, Active: true}} } return v } func TestTheExplorerSaysHowFastAModelAnswersOnTheCards(t *testing.T) { // The PoC's one Quadro RTX 6000: an 8B GGUF at Q4_K_M, 4.8 GB, on // 672 GB/s. poc := []adminapi.NodeView{cards("cuda01", "6000-24g", "Quadro RTX 6000", 1, "7.5", false, "")} body, _ := listing(t, poc, ggufModel("Qwen/Qwen3-8B-GGUF", "qwen3", 8e9, time.Now())) if !strings.Contains(body, `title="1× 6000-24g, llama.cpp">~103<`) { t.Errorf("an 8B on cuda01's card should be about 103 a second: %s", body) } // A mixed site: eight H100s on NVLink, four L40S on PCIe. 70B in BF16 // fits two H100s or four L40S, and the planner takes the fewest cards: // the H100s are the faster. mixed := []adminapi.NodeView{ cards("h", "h100-80g", "NVIDIA H100 80GB HBM3", 8, "9.0", true, ""), cards("l", "l40s-48g", "NVIDIA L40S", 4, "8.9", false, ""), } body, _ = listing(t, mixed, stModel("meta-llama/Llama-3.3-70B-Instruct", map[string]int64{"BF16": 70e9})) if !strings.Contains(body, "2× h100-80g, NVLink, vLLM") { t.Errorf("70B should run fastest on two NVLinked H100s: %s", body) } // Two nodes of one A100 each on InfiniBand: 70B fits neither, and the // pipeline across them is what the planner offers. ib := []adminapi.NodeView{ cards("a", "a100-80g", "NVIDIA A100-SXM4-80GB", 1, "8.0", false, "ib0"), cards("b", "a100-80g", "NVIDIA A100-SXM4-80GB", 1, "8.0", false, "ib0"), } body, _ = listing(t, ib, stModel("meta-llama/Llama-3.3-70B-Instruct", map[string]int64{"BF16": 70e9})) if !strings.Contains(body, "2 nodes × 1× a100-80g over ib0 (InfiniBand 200 Gb/s), vLLM") { t.Errorf("70B should be laid out across both nodes over ib0: %s", body) } } func TestTwoBridgedPairsAreNotFourCardsOnNVLink(t *testing.T) { pairs := []adminapi.GPUView{{Index: 0, NVLinkPeers: []int{1}}, {Index: 1, NVLinkPeers: []int{0}}, {Index: 2, NVLinkPeers: []int{3}}, {Index: 3, NVLinkPeers: []int{2}}} if got := nvlinkClique(pairs); got != 2 { t.Errorf("bridged pairs: %d cards together, want 2", got) } full := cards("h", "h100-80g", "NVIDIA H100 80GB HBM3", 4, "9.0", true, "").GPUs if got := nvlinkClique(full); got != 4 { t.Errorf("four on one fabric: %d", got) } }