CI / test (push) Failing after 6m37s
Each agent measures its node's memory read rate when it starts: every CPU sums a share of a buffer past any cache, a tenth of free memory at most, the best of four passes, in a fraction of a second; reported in Host (mem_bandwidth_gbs) and NodeView. A token on a CPU reads every weight it uses once, so one reply's speed is about that rate over those bytes: modelplan.CPUTokensPerSecond, 0.68 of the measured rate plus 5 ms a token, calibrated on the PoC against llama.cpp's predicted_per_second (node01 24.9 GB/s, qwen2.5-1.5b: 14.5 measured, 14.7 estimated; node05 41.9 GB/s, qwen2.5-14b: 3.3 and 3.2). A model of experts named …-A3B reads its active part (ActiveShare). The explorer shows it as CPU tok/s, at Q4_K_M on the fastest node that holds the model; a repository's page gives it for each GGUF by its size. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
172 lines
7.5 KiB
Go
172 lines
7.5 KiB
Go
package web
|
|
|
|
import (
|
|
"html"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
|
)
|
|
|
|
// nodesFleet is a cluster of the given nodes, all ready.
|
|
type nodesFleet struct {
|
|
fakeFleet
|
|
nodes []adminapi.NodeView
|
|
}
|
|
|
|
func (f nodesFleet) Nodes() []adminapi.NodeView { return f.nodes }
|
|
|
|
func gpuNode(id, class string, cards int, cc string, ramGB int) adminapi.NodeView {
|
|
return adminapi.NodeView{ID: id, State: "ready", GPUTotal: cards, GPUFree: cards, MemoryGB: ramGB,
|
|
Classes: []string{class}, ClassGPUs: []adminapi.ClassGPUs{{Class: class, Total: cards, Free: cards}},
|
|
ComputeCapabilities: []string{cc}}
|
|
}
|
|
|
|
func cpuNode(id string, ramGB int) adminapi.NodeView {
|
|
return adminapi.NodeView{ID: id, State: "ready", MemoryGB: ramGB}
|
|
}
|
|
|
|
func stModel(id string, params map[string]int64, tags ...string) hf.Model {
|
|
var total int64
|
|
for dt, n := range params {
|
|
if dt != "I32" {
|
|
total += n
|
|
}
|
|
}
|
|
return hf.Model{ID: id, CreatedAt: time.Now(), LastModified: time.Now(), PipelineTag: "text-generation",
|
|
Tags: append([]string{"safetensors"}, tags...), Safetensors: &hf.SafetensorsInfo{Total: total, Parameters: params}}
|
|
}
|
|
|
|
// listing is what the explorer shows of these repositories on this cluster.
|
|
func listing(t *testing.T, nodes []adminapi.NodeView, ms ...hf.Model) (string, *fakeRegistry) {
|
|
t.Helper()
|
|
reg := &fakeRegistry{results: ms}
|
|
mux := mirrorPageLlama(t, MirrorDeps{Mirror: &fakeMirror{}, Registry: reg, Fleet: nodesFleet{nodes: nodes}})
|
|
return mirrorGet(t, mux, "/admin/mirror?tab=explore").Body.String(), reg
|
|
}
|
|
|
|
func TestTheExplorerListsWhatThisClustersHardwareHolds(t *testing.T) {
|
|
// The PoC: one Turing card of 24 GB, CPU nodes of up to 31 GB.
|
|
poc := []adminapi.NodeView{gpuNode("cuda01", "6000-24g", 1, "7.5", 251), cpuNode("node05", 31)}
|
|
body, reg := listing(t, poc,
|
|
ggufModel("a/Small-GGUF", "qwen3", 8e9, time.Now()),
|
|
ggufModel("a/Huge-GGUF", "qwen3moe", 671e9, time.Now()),
|
|
stModel("a/Eight-BF16", map[string]int64{"BF16": 8e9}),
|
|
stModel("a/ThirtyTwo-BF16", map[string]int64{"BF16": 32e9}),
|
|
stModel("a/Eight-FP8", map[string]int64{"F8_E4M3": 8e9, "BF16": 1e8}),
|
|
stModel("a/ThirtyTwo-AWQ", map[string]int64{"BF16": 1e9, "I32": 31e9}, "awq", "4-bit"),
|
|
stModel("a/Eight-MLX", map[string]int64{"U32": 8e9}, "mlx", "8-bit"),
|
|
)
|
|
if reg.query.GGUF || reg.query.Limit != 100 {
|
|
t.Errorf("with a card, safetensors are asked for too, more of them: %+v", reg.query)
|
|
}
|
|
for id, want := range map[string]bool{
|
|
"a/Small-GGUF": true, "a/Huge-GGUF": false,
|
|
"a/Eight-BF16": true, "a/ThirtyTwo-BF16": false,
|
|
"a/Eight-FP8": false, "a/ThirtyTwo-AWQ": true, "a/Eight-MLX": false,
|
|
} {
|
|
if got := strings.Contains(body, id+"<") || strings.Contains(body, id+"\""); got != want {
|
|
t.Errorf("%s listed = %v, want %v", id, got, want)
|
|
}
|
|
}
|
|
if !strings.Contains(body, "largest card has 24 GB (compute capability 7.5)") ||
|
|
!strings.Contains(body, "4 more are hidden") {
|
|
t.Errorf("the page should say what it judged by and how many it left out: %s", body)
|
|
}
|
|
}
|
|
|
|
func TestANewerClusterSeesTheModelsItCanRun(t *testing.T) {
|
|
// A customer's: eight H100s. What the PoC cannot run, this one lists.
|
|
h100 := []adminapi.NodeView{gpuNode("gpu01", "h100-80g", 8, "9.0", 1024)}
|
|
body, _ := listing(t, h100,
|
|
stModel("a/ThirtyTwo-BF16", map[string]int64{"BF16": 32e9}),
|
|
stModel("a/Eight-FP8", map[string]int64{"F8_E4M3": 8e9, "BF16": 1e8}),
|
|
ggufModel("a/Huge-GGUF", "qwen3moe", 671e9, time.Now()),
|
|
)
|
|
for _, id := range []string{"a/ThirtyTwo-BF16", "a/Eight-FP8", "a/Huge-GGUF"} {
|
|
if !strings.Contains(body, id) {
|
|
t.Errorf("%s should be listed on eight H100s", id)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestACPUOnlyClusterIsShownGGUFOnly(t *testing.T) {
|
|
body, reg := listing(t, []adminapi.NodeView{cpuNode("c1", 32)},
|
|
ggufModel("a/Small-GGUF", "qwen3", 8e9, time.Now()),
|
|
ggufModel("a/Big-GGUF", "qwen3", 70e9, time.Now()),
|
|
stModel("a/Eight-BF16", map[string]int64{"BF16": 8e9}),
|
|
)
|
|
if !reg.query.GGUF {
|
|
t.Errorf("without cards only GGUF runs, and should be asked for: %+v", reg.query)
|
|
}
|
|
if !strings.Contains(body, "a/Small-GGUF") || strings.Contains(body, "a/Big-GGUF") || strings.Contains(body, "a/Eight-BF16") {
|
|
t.Errorf("a 32 GB CPU node holds the 8B GGUF only: %s", body)
|
|
}
|
|
if !strings.Contains(body, "no GPUs, so GGUF only") {
|
|
t.Error("the page should say it judged by a cluster without cards")
|
|
}
|
|
}
|
|
|
|
func TestARepositorysFileTooBigForThisClusterIsNotOffered(t *testing.T) {
|
|
poc := []adminapi.NodeView{gpuNode("cuda01", "6000-24g", 1, "7.5", 251), cpuNode("node05", 31)}
|
|
reg := &fakeRegistry{repo: hf.Repo{ID: "a/M", Revision: "abc123def456", Params: 32e9,
|
|
Files: []hf.File{
|
|
{Path: "M-Q4_K_M.gguf", Bytes: 19 << 30},
|
|
{Path: "model-00001-of-00002.safetensors", Bytes: 32 << 30},
|
|
{Path: "model-00002-of-00002.safetensors", Bytes: 32 << 30},
|
|
}}}
|
|
mux := mirrorPageLlama(t, MirrorDeps{Mirror: &fakeMirror{}, Registry: reg, Fleet: nodesFleet{nodes: poc}})
|
|
body := mirrorGet(t, mux, "/admin/mirror?repo=a/M").Body.String()
|
|
if !strings.Contains(body, "/resolve/abc123def456/M-Q4_K_M.gguf") {
|
|
t.Errorf("the GGUF fits cuda01's RAM and should be offered: %s", body)
|
|
}
|
|
if strings.Contains(body, "/resolve/abc123def456/model-00001-of-00002.safetensors") ||
|
|
!strings.Contains(body, "too big for this cluster") {
|
|
t.Errorf("64 GB of safetensors on one 24 GB card should be marked, not offered: %s", body)
|
|
}
|
|
}
|
|
|
|
func TestPackedWeightsAreSizedAsTheyAreStored(t *testing.T) {
|
|
// gpt-oss-20b: MXFP4 in U8, counted by the registry at the weights it
|
|
// unpacks to. At a byte each it read as 22 GB and missed a 24 GB card.
|
|
oss := stModel("openai/gpt-oss-20b", map[string]int64{"BF16": 1804459584, "U8": 19110297600}, "8-bit", "mxfp4")
|
|
if gb := float64(oss.WeightBytes()) / (1 << 30); gb < 11 || gb > 13 {
|
|
t.Errorf("gpt-oss-20b sized %.1f GB, want about 12", gb)
|
|
}
|
|
// Tagged mlx, but with a GGUF too: the GGUF runs.
|
|
both := ggufModel("a/Both", "qwen3", 27e9, time.Now())
|
|
both.Tags = []string{"mlx", "safetensors", "gguf"}
|
|
poc := []adminapi.NodeView{gpuNode("cuda01", "6000-24g", 1, "7.5", 251)}
|
|
body, _ := listing(t, poc, oss, both)
|
|
if !strings.Contains(body, "openai/gpt-oss-20b") || !strings.Contains(body, "a/Both") {
|
|
t.Errorf("both run on cuda01: %s", body)
|
|
}
|
|
}
|
|
|
|
func TestTheExplorerSaysHowFastAModelAnswersOnACPU(t *testing.T) {
|
|
slow, fast := cpuNode("node01", 7), cpuNode("node05", 31)
|
|
slow.MemBandwidthGBs, fast.MemBandwidthGBs = 24.9, 41.9
|
|
body, _ := listing(t, []adminapi.NodeView{slow, fast},
|
|
ggufModel("Qwen/Qwen3-8B-GGUF", "qwen3", 8e9, time.Now()),
|
|
ggufModel("Qwen/Qwen3-30B-A3B-GGUF", "qwen3moe", 30e9, time.Now()),
|
|
)
|
|
// 8B at Q4_K_M is 4.8 GB, on node05's 41.9 GB/s: about 5.8 a second.
|
|
// The 30B model of experts reads its 3B active part: about 15.
|
|
for _, want := range []string{">~5.8<", ">~15<", "on node05's CPU"} {
|
|
if !strings.Contains(body, want) {
|
|
t.Errorf("the listing lacks %q: %s", want, body)
|
|
}
|
|
}
|
|
|
|
reg := &fakeRegistry{repo: hf.Repo{ID: "Qwen/Qwen3-8B-GGUF", Revision: "abc123def456", Params: 8e9,
|
|
Files: []hf.File{{Path: "Qwen3-8B-Q4_K_M.gguf", Bytes: 5 << 30}, {Path: "Qwen3-8B-Q8_0.gguf", Bytes: 8 << 30}}}}
|
|
mux := mirrorPageLlama(t, MirrorDeps{Mirror: &fakeMirror{}, Registry: reg,
|
|
Fleet: nodesFleet{nodes: []adminapi.NodeView{slow, fast}}})
|
|
page := html.UnescapeString(mirrorGet(t, mux, "/admin/mirror?repo=Qwen/Qwen3-8B-GGUF").Body.String())
|
|
if !strings.Contains(page, "~5.2 tok/s on node05's CPU") || !strings.Contains(page, "~3.3 tok/s on node05's CPU") {
|
|
t.Errorf("each file should say its speed on the CPU, by its own size: %s", page)
|
|
}
|
|
}
|