MIG: a card in MIG mode is placed as its instances, each a GPU of class <card>-mig-<profile>
Discovery reports a card in MIG mode as its instances instead of the whole card, on both paths: nvidia-smi (-L names them, -q -x gives their memory, ids and processes) and NVML (GetMigMode, GetMaxMigDeviceCount, GetMigDeviceHandleByIndex, behind an interface so it is tested with a fake). Each instance has its MIG UUID, its own memory, and a class from the card's and the profile: a100-80g-mig-3g.40gb. Placement numbers instances 1000 + 100 x card + instance and every view shows them as card/migN. A replica on MIG runs on exactly one instance: a process opens one MIG device and instances have no NVLink between them, so tp, pp, gpus or nodes above one on a MIG class are refused in catalog validation and again in placement, and the planner offers one instance or none. The agent gives the engine CUDA_VISIBLE_DEVICES=MIG-<uuid> (no Vulkan device variable on MIG). Memory and processes are per instance; heat, power and wear the card's; utilisation is left out rather than reported as zero. A card in MIG mode with nothing carved stays out of placement with the reason, and llmgw doctor reports each card's mode and instances and warns of the states in which nothing can run. ctl nodes, ctl status, the console and the plan count devices by class. docs/install.md has a MIG section. Written against fakes of nvidia-smi and NVML output; not yet run on MIG hardware. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
e81f1cba4c
commit
0fd55a862b
39 files changed
+2587
-327
No files matched your search
@@ -228,7 +228,12 @@ type NodeView struct {
|
||||
DiskFreeGB int `json:"disk_free_gb,omitempty"`
|
||||
DiskTotalGB int `json:"disk_total_gb,omitempty"`
|
||||
Classes []string `json:"gpu_classes,omitempty"`
|
||||
Fabrics []string `json:"fabrics,omitempty"`
|
||||
// ClassGPUs counts the node's devices by class, in the order of Classes.
|
||||
// A node with a card in MIG mode has several classes — each profile its
|
||||
// instances are cut to, and any whole card beside them — and GPUTotal
|
||||
// alone would add 3g.40gb instances to 80 GB cards.
|
||||
ClassGPUs []ClassGPUs `json:"gpus_by_class,omitempty"`
|
||||
Fabrics []string `json:"fabrics,omitempty"`
|
||||
|
||||
// Unhealthy lists GPUs kept out of placement and why, which is usually the
|
||||
// answer when a node looks idle but nothing will schedule onto it.
|
||||
@@ -255,10 +260,34 @@ type NodeView struct {
|
||||
IMEXChannels int `json:"imex_channels,omitempty"`
|
||||
}
|
||||
|
||||
// ClassGPUs is how many of a node's devices are of one class, and how many
|
||||
// of those are free or kept out of placement.
|
||||
type ClassGPUs struct {
|
||||
Class string `json:"class"`
|
||||
Total int `json:"total"`
|
||||
Free int `json:"free"`
|
||||
Unhealthy int `json:"unhealthy,omitempty"`
|
||||
// MIG marks a class of MIG instances, each of which takes a replica of
|
||||
// its own and never joins a tensor-parallel group.
|
||||
MIG bool `json:"mig,omitempty"`
|
||||
}
|
||||
|
||||
// GPUView is one card as a node's page shows it.
|
||||
type GPUView struct {
|
||||
Index int `json:"index"`
|
||||
Name string `json:"name"`
|
||||
// Label is the device as people are shown it: "3" for a whole card,
|
||||
// "0/mig2" for card 0's MIG instance 2.
|
||||
Label string `json:"label"`
|
||||
Class string `json:"class"`
|
||||
// Card is the physical card's index: the device's own, or for a MIG
|
||||
// instance its parent's, which is what the card's sensors are under.
|
||||
Card int `json:"card"`
|
||||
// MIGProfile is the instance's profile, like 3g.40gb; empty for a
|
||||
// whole card.
|
||||
MIGProfile string `json:"mig_profile,omitempty"`
|
||||
MemoryTotalMB int `json:"memory_total_mb,omitempty"`
|
||||
MemoryUsedMB int `json:"memory_used_mb,omitempty"`
|
||||
// ComputeCapability is what decides which engine builds run on the card:
|
||||
// CUDA 13 builds need 7.5, so a V100 (7.0) runs only older ones.
|
||||
ComputeCapability string `json:"compute_capability,omitempty"`
|
||||
@@ -556,15 +585,25 @@ func (a *API) Nodes() []NodeView {
|
||||
for _, w := range n.Weights {
|
||||
v.Models = append(v.Models, w.Path)
|
||||
}
|
||||
seen := map[string]struct{}{}
|
||||
seen := map[string]int{}
|
||||
free := inv.FreeGPUs(n.ID)
|
||||
for _, g := range n.GPUs {
|
||||
if _, dup := seen[g.Class()]; !dup {
|
||||
seen[g.Class()] = struct{}{}
|
||||
k, dup := seen[g.Class()]
|
||||
if !dup {
|
||||
k = len(v.ClassGPUs)
|
||||
seen[g.Class()] = k
|
||||
v.Classes = append(v.Classes, g.Class())
|
||||
v.ClassGPUs = append(v.ClassGPUs, ClassGPUs{Class: g.Class(), MIG: g.IsMIG()})
|
||||
}
|
||||
if !g.Health.OK() {
|
||||
c := &v.ClassGPUs[k]
|
||||
c.Total++
|
||||
switch {
|
||||
case !g.Health.OK():
|
||||
c.Unhealthy++
|
||||
v.Unhealthy = append(v.Unhealthy,
|
||||
fmt.Sprintf("gpu %d: %s", g.Index, g.Health.Problems[0]))
|
||||
fmt.Sprintf("gpu %s: %s", discovery.DeviceLabel(g.Index), g.Health.Problems[0]))
|
||||
case slices.Contains(free, g.Index):
|
||||
c.Free++
|
||||
}
|
||||
}
|
||||
if !n.LastSeen.IsZero() {
|
||||
@@ -574,7 +613,12 @@ func (a *API) Nodes() []NodeView {
|
||||
v.NVLinkDomain, v.IMEX, v.IMEXChannels = n.NVLinkDomain(), n.IMEX.State, n.IMEX.Channels
|
||||
for _, g := range n.GPUs {
|
||||
gv := GPUView{Index: g.Index, Name: g.Name, ComputeCapability: g.ComputeCapability,
|
||||
NUMA: g.NUMANode, NVLinkPeers: g.NVLinkPeers}
|
||||
NUMA: g.NUMANode, NVLinkPeers: g.NVLinkPeers,
|
||||
Label: discovery.DeviceLabel(g.Index), Class: g.Class(), Card: g.CardIndex(),
|
||||
MemoryTotalMB: g.MemoryTotalMB, MemoryUsedMB: g.MemoryUsedMB}
|
||||
if g.MIG != nil {
|
||||
gv.MIGProfile = g.MIG.Profile
|
||||
}
|
||||
for hca, rel := range g.NICPaths {
|
||||
if gv.NearestNIC == "" || discovery.PathRank(rel) < discovery.PathRank(gv.NICPath) ||
|
||||
(discovery.PathRank(rel) == discovery.PathRank(gv.NICPath) && hca < gv.NearestNIC) {
|
||||
@@ -1617,6 +1661,13 @@ func (a *API) PoolNodes() []modelplan.PoolNode {
|
||||
p.Class = n.Classes[0]
|
||||
p.GPUMemGB = modelplan.MemFromClass(p.Class)
|
||||
}
|
||||
// By class, with each class's own count of devices: a node with a
|
||||
// card in MIG mode is several, and its first class alone would
|
||||
// count every instance and card as that one.
|
||||
for _, c := range n.ClassGPUs {
|
||||
p.Kinds = append(p.Kinds, modelplan.PoolKind{Class: c.Class,
|
||||
GPUMemGB: modelplan.MemFromClass(c.Class), GPUs: c.Total, FreeGPUs: c.Free})
|
||||
}
|
||||
if len(n.Fabrics) > 0 {
|
||||
p.Fabric = n.Fabrics[0]
|
||||
}
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
)
|
||||
|
||||
// migNode has card 0 cut into two 3g.40gb instances and card 1 whole.
|
||||
func migNode(id string) inventory.Node {
|
||||
nd := inventory.Node{ID: id, State: inventory.Ready, AgentVersion: "v1", MemoryTotalMB: 512 * 1024}
|
||||
for dev := range 2 {
|
||||
nd.GPUs = append(nd.GPUs, discovery.GPU{
|
||||
Index: discovery.MIGIndex(0, dev), Name: "NVIDIA A100-SXM4-80GB MIG 3g.40gb", UUID: "MIG-x",
|
||||
MemoryTotalMB: 40192, MemoryUsedMB: 37,
|
||||
MIG: &discovery.MIGDevice{ParentIndex: 0, ParentMemoryTotalMB: 81920, Device: dev, Profile: "3g.40gb"},
|
||||
})
|
||||
}
|
||||
nd.GPUs = append(nd.GPUs, discovery.GPU{Index: 1, Name: "NVIDIA A100-SXM4-80GB", MemoryTotalMB: 81920})
|
||||
return nd
|
||||
}
|
||||
|
||||
func TestANodeWithACardInMIGModeCountsItsDevicesByClass(t *testing.T) {
|
||||
a, ctl := setup(t, testModels(), migNode("a100"))
|
||||
ctl.Inventory().Allocate("a100", inventory.Allocation{ShardID: "s", ReplicaID: "r", GPUIndices: []int{1000}})
|
||||
|
||||
n := a.Nodes()[0]
|
||||
want := []ClassGPUs{
|
||||
{Class: "a100-80g-mig-3g.40gb", Total: 2, Free: 1, MIG: true},
|
||||
{Class: "a100-80g", Total: 1, Free: 1},
|
||||
}
|
||||
if !reflect.DeepEqual(n.ClassGPUs, want) {
|
||||
t.Errorf("by class: %+v\nwant %+v", n.ClassGPUs, want)
|
||||
}
|
||||
// Each device as people are shown it: the instance under its card, with
|
||||
// its own memory, and the card its sensors are read under.
|
||||
g := n.GPUs[1]
|
||||
if g.Label != "0/mig1" || g.Card != 0 || g.MIGProfile != "3g.40gb" || g.MemoryTotalMB != 40192 ||
|
||||
g.Class != "a100-80g-mig-3g.40gb" {
|
||||
t.Errorf("instance view: %+v", g)
|
||||
}
|
||||
if w := n.GPUs[2]; w.Label != "1" || w.Card != 1 || w.MIGProfile != "" {
|
||||
t.Errorf("whole card view: %+v", w)
|
||||
}
|
||||
|
||||
// The fleet summary attributes each device to its own class, where it
|
||||
// once put every card of a node under the node's only class.
|
||||
s := a.Summary()
|
||||
got := map[string][2]int{}
|
||||
for _, c := range s.GPUs {
|
||||
got[c.Class] = [2]int{c.Total, c.Used}
|
||||
}
|
||||
if !reflect.DeepEqual(got, map[string][2]int{"a100-80g-mig-3g.40gb": {2, 1}, "a100-80g": {1, 0}}) {
|
||||
t.Errorf("summary by class: %v", got)
|
||||
}
|
||||
|
||||
// And a plan sees both classes, each with its own count.
|
||||
pool := a.PoolNodes()
|
||||
if len(pool) != 1 || len(pool[0].Kinds) != 2 || pool[0].Kinds[0].GPUMemGB != 40 ||
|
||||
pool[0].Kinds[0].FreeGPUs != 1 || pool[0].Kinds[1].GPUMemGB != 80 {
|
||||
t.Errorf("pool: %+v", pool)
|
||||
}
|
||||
}
|
||||
@@ -148,12 +148,14 @@ func (a *API) Summary() Summary {
|
||||
byClass[c] = &ClassUse{Class: c}
|
||||
}
|
||||
}
|
||||
// A node reports one class in practice; attribute its cards to it.
|
||||
if len(n.Classes) == 1 {
|
||||
u := byClass[n.Classes[0]]
|
||||
u.Total += n.GPUTotal
|
||||
u.Unhealthy += len(n.Unhealthy)
|
||||
u.Used += n.GPUTotal - n.GPUFree - len(n.Unhealthy)
|
||||
// Each class its own devices, which a node with a card in MIG mode
|
||||
// needs: its instances are one class or several, and any whole card
|
||||
// beside them another.
|
||||
for _, c := range n.ClassGPUs {
|
||||
u := byClass[c.Class]
|
||||
u.Total += c.Total
|
||||
u.Unhealthy += c.Unhealthy
|
||||
u.Used += c.Total - c.Free - c.Unhealthy
|
||||
}
|
||||
}
|
||||
for _, u := range byClass {
|
||||
|
||||
Reference in new issue
Block a user