diff --git a/internal/hf/hf.go b/internal/hf/hf.go index 7168570..bed8791 100644 --- a/internal/hf/hf.go +++ b/internal/hf/hf.go @@ -108,7 +108,7 @@ func (m Model) WeightBytes() int64 { if m.HasTag("8-bit") { packed = 1 } - fp4 := m.HasTag("mxfp4") || m.HasTag("nvfp4") + fp4 := m.HasTag("mxfp4") || m.HasTag("nvfp4") || m.HasTag("fp4") var b float64 for dt, n := range m.Safetensors.Parameters { switch per, ok := dtypeBytes[dt]; { diff --git a/internal/modelplan/gpuspeed.go b/internal/modelplan/gpuspeed.go index eb528b3..5bb4507 100644 --- a/internal/modelplan/gpuspeed.go +++ b/internal/modelplan/gpuspeed.go @@ -43,9 +43,9 @@ var gpuBandwidth = []struct { gbs float64 }{ {"gb300", 8000}, {"b300", 8000}, {"gb200", 8000}, {"b200", 8000}, - {"h200 nvl", 4800}, {"h200", 4800}, + {"gh200", 4000}, {"h200 nvl", 4800}, {"h200", 4800}, {"h100 nvl", 3900}, {"h100 pcie", 2000}, {"h100", 3350}, {"h800 pcie", 2000}, {"h800", 3350}, - {"gh200", 4000}, {"h20", 4000}, + {"h20", 4000}, {"a100-sxm4-80gb", 2039}, {"a100 80gb pcie", 1935}, {"a100-pcie-80gb", 1935}, {"a100-sxm4-40gb", 1555}, {"a100-pcie-40gb", 1555}, {"a100 80gb", 2039}, {"a100", 1555}, {"a800", 2039}, {"a40", 696}, {"a30", 933}, {"a10g", 600}, {"a10", 600}, {"a16", 200}, {"a2", 200}, diff --git a/internal/modelplan/gpuspeed_test.go b/internal/modelplan/gpuspeed_test.go index e116c5d..04e145a 100644 --- a/internal/modelplan/gpuspeed_test.go +++ b/internal/modelplan/gpuspeed_test.go @@ -17,6 +17,8 @@ func TestACardsBandwidthIsKnownByItsName(t *testing.T) { "Tesla T4": 320, "Tesla V100-SXM2-32GB": 900, "NVIDIA L4": 300, + "NVIDIA GH200 480GB": 4000, + "NVIDIA H200": 4800, "Some Card 9000": 0, } { if got := GPUBandwidthGBs(name); got != want { diff --git a/internal/web/explore_fit.go b/internal/web/explore_fit.go index 8c440d4..d53f5b6 100644 --- a/internal/web/explore_fit.go +++ b/internal/web/explore_fit.go @@ -47,8 +47,11 @@ type cpuHost struct { bwGBs float64 } -// fp8Capability is the oldest card vLLM runs FP8 weights on. -const fp8Capability = 8.9 +// fp8Capability is the oldest card vLLM runs an FP8 checkpoint on: Ampere, +// by Marlin's kernels, which read FP8 weights on cards without FP8 maths. +// Ada and Hopper compute in it; the agent's 8.9 is for --quantization fp8, +// which makes FP8 of other weights. +const fp8Capability = 8.0 // fitOf is the cluster's hardware now, nil when there is none to judge by: // no fleet to ask, or no node ready. @@ -64,17 +67,26 @@ func (p *AccountPages) fitOf() *clusterFit { c.classBW, c.nvlink, c.fabrics = map[string]float64{}, map[string]int{}, map[string]modelplan.Links{} newest := map[string]float64{} for _, n := range nodes { - joined := map[string]int{} + if n.State != "ready" { + continue + } + byClass := map[string][]adminapi.GPUView{} for _, g := range n.GPUs { - if bw := modelplan.GPUBandwidthGBs(g.Name); bw > c.classBW[g.Class] { + bw := modelplan.GPUBandwidthGBs(g.Name) + // A MIG instance has its share of the card's memory, and of the + // bandwidth to it. + if parent, _, ok := strings.Cut(g.Class, "-mig-"); ok { + if whole := modelplan.MemFromClass(parent); whole > 0 { + bw *= float64(modelplan.MemFromClass(g.Class)) / float64(whole) + } + } + if bw > c.classBW[g.Class] { c.classBW[g.Class] = bw } - if len(g.NVLinkPeers) > 0 { - joined[g.Class]++ - } + byClass[g.Class] = append(byClass[g.Class], g) } - for class, k := range joined { - c.nvlink[class] = max(c.nvlink[class], k) + for class, gs := range byClass { + c.nvlink[class] = max(c.nvlink[class], nvlinkClique(gs)) } for _, p := range n.IBPorts { if p.Fabric != "" && p.Active { @@ -350,3 +362,43 @@ func artifactFormat(name string) string { } return "" } + +// nvlinkClique is the most cards of these that are each joined by NVLink to +// every other: tensor parallel across four needs all four on one fabric, not +// two bridged pairs. +func nvlinkClique(gs []adminapi.GPUView) int { + if len(gs) > 16 { + gs = gs[:16] + } + peer := map[int]map[int]bool{} + for _, g := range gs { + peer[g.Index] = map[int]bool{} + for _, p := range g.NVLinkPeers { + peer[g.Index][p] = true + } + } + best := 0 + for set := 1; set < 1< 1 { + best = len(in) + } + } + return best +} diff --git a/internal/web/explore_fit_test.go b/internal/web/explore_fit_test.go index ffcfbce..6befd67 100644 --- a/internal/web/explore_fit_test.go +++ b/internal/web/explore_fit_test.go @@ -225,3 +225,15 @@ func TestTheExplorerSaysHowFastAModelAnswersOnTheCards(t *testing.T) { t.Errorf("70B should be laid out across both nodes over ib0: %s", body) } } + +func TestTwoBridgedPairsAreNotFourCardsOnNVLink(t *testing.T) { + pairs := []adminapi.GPUView{{Index: 0, NVLinkPeers: []int{1}}, {Index: 1, NVLinkPeers: []int{0}}, + {Index: 2, NVLinkPeers: []int{3}}, {Index: 3, NVLinkPeers: []int{2}}} + if got := nvlinkClique(pairs); got != 2 { + t.Errorf("bridged pairs: %d cards together, want 2", got) + } + full := cards("h", "h100-80g", "NVIDIA H100 80GB HBM3", 4, "9.0", true, "").GPUs + if got := nvlinkClique(full); got != 4 { + t.Errorf("four on one fabric: %d", got) + } +}