Merge C22 MIG
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
commit
330a421118
39 files changed
+2587
-327
No files matched your search
@@ -228,7 +228,12 @@ type NodeView struct {
|
||||
DiskFreeGB int `json:"disk_free_gb,omitempty"`
|
||||
DiskTotalGB int `json:"disk_total_gb,omitempty"`
|
||||
Classes []string `json:"gpu_classes,omitempty"`
|
||||
Fabrics []string `json:"fabrics,omitempty"`
|
||||
// ClassGPUs counts the node's devices by class, in the order of Classes.
|
||||
// A node with a card in MIG mode has several classes — each profile its
|
||||
// instances are cut to, and any whole card beside them — and GPUTotal
|
||||
// alone would add 3g.40gb instances to 80 GB cards.
|
||||
ClassGPUs []ClassGPUs `json:"gpus_by_class,omitempty"`
|
||||
Fabrics []string `json:"fabrics,omitempty"`
|
||||
|
||||
// Unhealthy lists GPUs kept out of placement and why, which is usually the
|
||||
// answer when a node looks idle but nothing will schedule onto it.
|
||||
@@ -255,10 +260,34 @@ type NodeView struct {
|
||||
IMEXChannels int `json:"imex_channels,omitempty"`
|
||||
}
|
||||
|
||||
// ClassGPUs is how many of a node's devices are of one class, and how many
|
||||
// of those are free or kept out of placement.
|
||||
type ClassGPUs struct {
|
||||
Class string `json:"class"`
|
||||
Total int `json:"total"`
|
||||
Free int `json:"free"`
|
||||
Unhealthy int `json:"unhealthy,omitempty"`
|
||||
// MIG marks a class of MIG instances, each of which takes a replica of
|
||||
// its own and never joins a tensor-parallel group.
|
||||
MIG bool `json:"mig,omitempty"`
|
||||
}
|
||||
|
||||
// GPUView is one card as a node's page shows it.
|
||||
type GPUView struct {
|
||||
Index int `json:"index"`
|
||||
Name string `json:"name"`
|
||||
// Label is the device as people are shown it: "3" for a whole card,
|
||||
// "0/mig2" for card 0's MIG instance 2.
|
||||
Label string `json:"label"`
|
||||
Class string `json:"class"`
|
||||
// Card is the physical card's index: the device's own, or for a MIG
|
||||
// instance its parent's, which is what the card's sensors are under.
|
||||
Card int `json:"card"`
|
||||
// MIGProfile is the instance's profile, like 3g.40gb; empty for a
|
||||
// whole card.
|
||||
MIGProfile string `json:"mig_profile,omitempty"`
|
||||
MemoryTotalMB int `json:"memory_total_mb,omitempty"`
|
||||
MemoryUsedMB int `json:"memory_used_mb,omitempty"`
|
||||
// ComputeCapability is what decides which engine builds run on the card:
|
||||
// CUDA 13 builds need 7.5, so a V100 (7.0) runs only older ones.
|
||||
ComputeCapability string `json:"compute_capability,omitempty"`
|
||||
@@ -556,15 +585,25 @@ func (a *API) Nodes() []NodeView {
|
||||
for _, w := range n.Weights {
|
||||
v.Models = append(v.Models, w.Path)
|
||||
}
|
||||
seen := map[string]struct{}{}
|
||||
seen := map[string]int{}
|
||||
free := inv.FreeGPUs(n.ID)
|
||||
for _, g := range n.GPUs {
|
||||
if _, dup := seen[g.Class()]; !dup {
|
||||
seen[g.Class()] = struct{}{}
|
||||
k, dup := seen[g.Class()]
|
||||
if !dup {
|
||||
k = len(v.ClassGPUs)
|
||||
seen[g.Class()] = k
|
||||
v.Classes = append(v.Classes, g.Class())
|
||||
v.ClassGPUs = append(v.ClassGPUs, ClassGPUs{Class: g.Class(), MIG: g.IsMIG()})
|
||||
}
|
||||
if !g.Health.OK() {
|
||||
c := &v.ClassGPUs[k]
|
||||
c.Total++
|
||||
switch {
|
||||
case !g.Health.OK():
|
||||
c.Unhealthy++
|
||||
v.Unhealthy = append(v.Unhealthy,
|
||||
fmt.Sprintf("gpu %d: %s", g.Index, g.Health.Problems[0]))
|
||||
fmt.Sprintf("gpu %s: %s", discovery.DeviceLabel(g.Index), g.Health.Problems[0]))
|
||||
case slices.Contains(free, g.Index):
|
||||
c.Free++
|
||||
}
|
||||
}
|
||||
if !n.LastSeen.IsZero() {
|
||||
@@ -574,7 +613,12 @@ func (a *API) Nodes() []NodeView {
|
||||
v.NVLinkDomain, v.IMEX, v.IMEXChannels = n.NVLinkDomain(), n.IMEX.State, n.IMEX.Channels
|
||||
for _, g := range n.GPUs {
|
||||
gv := GPUView{Index: g.Index, Name: g.Name, ComputeCapability: g.ComputeCapability,
|
||||
NUMA: g.NUMANode, NVLinkPeers: g.NVLinkPeers}
|
||||
NUMA: g.NUMANode, NVLinkPeers: g.NVLinkPeers,
|
||||
Label: discovery.DeviceLabel(g.Index), Class: g.Class(), Card: g.CardIndex(),
|
||||
MemoryTotalMB: g.MemoryTotalMB, MemoryUsedMB: g.MemoryUsedMB}
|
||||
if g.MIG != nil {
|
||||
gv.MIGProfile = g.MIG.Profile
|
||||
}
|
||||
for hca, rel := range g.NICPaths {
|
||||
if gv.NearestNIC == "" || discovery.PathRank(rel) < discovery.PathRank(gv.NICPath) ||
|
||||
(discovery.PathRank(rel) == discovery.PathRank(gv.NICPath) && hca < gv.NearestNIC) {
|
||||
@@ -1619,6 +1663,13 @@ func (a *API) PoolNodes() []modelplan.PoolNode {
|
||||
p.Class = n.Classes[0]
|
||||
p.GPUMemGB = modelplan.MemFromClass(p.Class)
|
||||
}
|
||||
// By class, with each class's own count of devices: a node with a
|
||||
// card in MIG mode is several, and its first class alone would
|
||||
// count every instance and card as that one.
|
||||
for _, c := range n.ClassGPUs {
|
||||
p.Kinds = append(p.Kinds, modelplan.PoolKind{Class: c.Class,
|
||||
GPUMemGB: modelplan.MemFromClass(c.Class), GPUs: c.Total, FreeGPUs: c.Free})
|
||||
}
|
||||
if len(n.Fabrics) > 0 {
|
||||
p.Fabric = n.Fabrics[0]
|
||||
}
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
)
|
||||
|
||||
// migNode has card 0 cut into two 3g.40gb instances and card 1 whole.
|
||||
func migNode(id string) inventory.Node {
|
||||
nd := inventory.Node{ID: id, State: inventory.Ready, AgentVersion: "v1", MemoryTotalMB: 512 * 1024}
|
||||
for dev := range 2 {
|
||||
nd.GPUs = append(nd.GPUs, discovery.GPU{
|
||||
Index: discovery.MIGIndex(0, dev), Name: "NVIDIA A100-SXM4-80GB MIG 3g.40gb", UUID: "MIG-x",
|
||||
MemoryTotalMB: 40192, MemoryUsedMB: 37,
|
||||
MIG: &discovery.MIGDevice{ParentIndex: 0, ParentMemoryTotalMB: 81920, Device: dev, Profile: "3g.40gb"},
|
||||
})
|
||||
}
|
||||
nd.GPUs = append(nd.GPUs, discovery.GPU{Index: 1, Name: "NVIDIA A100-SXM4-80GB", MemoryTotalMB: 81920})
|
||||
return nd
|
||||
}
|
||||
|
||||
func TestANodeWithACardInMIGModeCountsItsDevicesByClass(t *testing.T) {
|
||||
a, ctl := setup(t, testModels(), migNode("a100"))
|
||||
ctl.Inventory().Allocate("a100", inventory.Allocation{ShardID: "s", ReplicaID: "r", GPUIndices: []int{1000}})
|
||||
|
||||
n := a.Nodes()[0]
|
||||
want := []ClassGPUs{
|
||||
{Class: "a100-80g-mig-3g.40gb", Total: 2, Free: 1, MIG: true},
|
||||
{Class: "a100-80g", Total: 1, Free: 1},
|
||||
}
|
||||
if !reflect.DeepEqual(n.ClassGPUs, want) {
|
||||
t.Errorf("by class: %+v\nwant %+v", n.ClassGPUs, want)
|
||||
}
|
||||
// Each device as people are shown it: the instance under its card, with
|
||||
// its own memory, and the card its sensors are read under.
|
||||
g := n.GPUs[1]
|
||||
if g.Label != "0/mig1" || g.Card != 0 || g.MIGProfile != "3g.40gb" || g.MemoryTotalMB != 40192 ||
|
||||
g.Class != "a100-80g-mig-3g.40gb" {
|
||||
t.Errorf("instance view: %+v", g)
|
||||
}
|
||||
if w := n.GPUs[2]; w.Label != "1" || w.Card != 1 || w.MIGProfile != "" {
|
||||
t.Errorf("whole card view: %+v", w)
|
||||
}
|
||||
|
||||
// The fleet summary attributes each device to its own class, where it
|
||||
// once put every card of a node under the node's only class.
|
||||
s := a.Summary()
|
||||
got := map[string][2]int{}
|
||||
for _, c := range s.GPUs {
|
||||
got[c.Class] = [2]int{c.Total, c.Used}
|
||||
}
|
||||
if !reflect.DeepEqual(got, map[string][2]int{"a100-80g-mig-3g.40gb": {2, 1}, "a100-80g": {1, 0}}) {
|
||||
t.Errorf("summary by class: %v", got)
|
||||
}
|
||||
|
||||
// And a plan sees both classes, each with its own count.
|
||||
pool := a.PoolNodes()
|
||||
if len(pool) != 1 || len(pool[0].Kinds) != 2 || pool[0].Kinds[0].GPUMemGB != 40 ||
|
||||
pool[0].Kinds[0].FreeGPUs != 1 || pool[0].Kinds[1].GPUMemGB != 80 {
|
||||
t.Errorf("pool: %+v", pool)
|
||||
}
|
||||
}
|
||||
@@ -148,12 +148,14 @@ func (a *API) Summary() Summary {
|
||||
byClass[c] = &ClassUse{Class: c}
|
||||
}
|
||||
}
|
||||
// A node reports one class in practice; attribute its cards to it.
|
||||
if len(n.Classes) == 1 {
|
||||
u := byClass[n.Classes[0]]
|
||||
u.Total += n.GPUTotal
|
||||
u.Unhealthy += len(n.Unhealthy)
|
||||
u.Used += n.GPUTotal - n.GPUFree - len(n.Unhealthy)
|
||||
// Each class its own devices, which a node with a card in MIG mode
|
||||
// needs: its instances are one class or several, and any whole card
|
||||
// beside them another.
|
||||
for _, c := range n.ClassGPUs {
|
||||
u := byClass[c.Class]
|
||||
u.Total += c.Total
|
||||
u.Unhealthy += c.Unhealthy
|
||||
u.Used += c.Total - c.Free - c.Unhealthy
|
||||
}
|
||||
}
|
||||
for _, u := range byClass {
|
||||
|
||||
Reference in new issue
Block a user