Merge C22 MIG

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 00:01:51 +02:00
commit 330a421118
39 files changed
+2587 -327

No files matched your search

+58 -7
View File
@@ -228,7 +228,12 @@ type NodeView struct {
DiskFreeGB int `json:"disk_free_gb,omitempty"`
DiskTotalGB int `json:"disk_total_gb,omitempty"`
Classes []string `json:"gpu_classes,omitempty"`
Fabrics []string `json:"fabrics,omitempty"`
// ClassGPUs counts the node's devices by class, in the order of Classes.
// A node with a card in MIG mode has several classes — each profile its
// instances are cut to, and any whole card beside them — and GPUTotal
// alone would add 3g.40gb instances to 80 GB cards.
ClassGPUs []ClassGPUs `json:"gpus_by_class,omitempty"`
Fabrics []string `json:"fabrics,omitempty"`
// Unhealthy lists GPUs kept out of placement and why, which is usually the
// answer when a node looks idle but nothing will schedule onto it.
@@ -255,10 +260,34 @@ type NodeView struct {
IMEXChannels int `json:"imex_channels,omitempty"`
}
// ClassGPUs is how many of a node's devices are of one class, and how many
// of those are free or kept out of placement.
type ClassGPUs struct {
Class string `json:"class"`
Total int `json:"total"`
Free int `json:"free"`
Unhealthy int `json:"unhealthy,omitempty"`
// MIG marks a class of MIG instances, each of which takes a replica of
// its own and never joins a tensor-parallel group.
MIG bool `json:"mig,omitempty"`
}
// GPUView is one card as a node's page shows it.
type GPUView struct {
Index int `json:"index"`
Name string `json:"name"`
// Label is the device as people are shown it: "3" for a whole card,
// "0/mig2" for card 0's MIG instance 2.
Label string `json:"label"`
Class string `json:"class"`
// Card is the physical card's index: the device's own, or for a MIG
// instance its parent's, which is what the card's sensors are under.
Card int `json:"card"`
// MIGProfile is the instance's profile, like 3g.40gb; empty for a
// whole card.
MIGProfile string `json:"mig_profile,omitempty"`
MemoryTotalMB int `json:"memory_total_mb,omitempty"`
MemoryUsedMB int `json:"memory_used_mb,omitempty"`
// ComputeCapability is what decides which engine builds run on the card:
// CUDA 13 builds need 7.5, so a V100 (7.0) runs only older ones.
ComputeCapability string `json:"compute_capability,omitempty"`
@@ -556,15 +585,25 @@ func (a *API) Nodes() []NodeView {
for _, w := range n.Weights {
v.Models = append(v.Models, w.Path)
}
seen := map[string]struct{}{}
seen := map[string]int{}
free := inv.FreeGPUs(n.ID)
for _, g := range n.GPUs {
if _, dup := seen[g.Class()]; !dup {
seen[g.Class()] = struct{}{}
k, dup := seen[g.Class()]
if !dup {
k = len(v.ClassGPUs)
seen[g.Class()] = k
v.Classes = append(v.Classes, g.Class())
v.ClassGPUs = append(v.ClassGPUs, ClassGPUs{Class: g.Class(), MIG: g.IsMIG()})
}
if !g.Health.OK() {
c := &v.ClassGPUs[k]
c.Total++
switch {
case !g.Health.OK():
c.Unhealthy++
v.Unhealthy = append(v.Unhealthy,
fmt.Sprintf("gpu %d: %s", g.Index, g.Health.Problems[0]))
fmt.Sprintf("gpu %s: %s", discovery.DeviceLabel(g.Index), g.Health.Problems[0]))
case slices.Contains(free, g.Index):
c.Free++
}
}
if !n.LastSeen.IsZero() {
@@ -574,7 +613,12 @@ func (a *API) Nodes() []NodeView {
v.NVLinkDomain, v.IMEX, v.IMEXChannels = n.NVLinkDomain(), n.IMEX.State, n.IMEX.Channels
for _, g := range n.GPUs {
gv := GPUView{Index: g.Index, Name: g.Name, ComputeCapability: g.ComputeCapability,
NUMA: g.NUMANode, NVLinkPeers: g.NVLinkPeers}
NUMA: g.NUMANode, NVLinkPeers: g.NVLinkPeers,
Label: discovery.DeviceLabel(g.Index), Class: g.Class(), Card: g.CardIndex(),
MemoryTotalMB: g.MemoryTotalMB, MemoryUsedMB: g.MemoryUsedMB}
if g.MIG != nil {
gv.MIGProfile = g.MIG.Profile
}
for hca, rel := range g.NICPaths {
if gv.NearestNIC == "" || discovery.PathRank(rel) < discovery.PathRank(gv.NICPath) ||
(discovery.PathRank(rel) == discovery.PathRank(gv.NICPath) && hca < gv.NearestNIC) {
@@ -1619,6 +1663,13 @@ func (a *API) PoolNodes() []modelplan.PoolNode {
p.Class = n.Classes[0]
p.GPUMemGB = modelplan.MemFromClass(p.Class)
}
// By class, with each class's own count of devices: a node with a
// card in MIG mode is several, and its first class alone would
// count every instance and card as that one.
for _, c := range n.ClassGPUs {
p.Kinds = append(p.Kinds, modelplan.PoolKind{Class: c.Class,
GPUMemGB: modelplan.MemFromClass(c.Class), GPUs: c.Total, FreeGPUs: c.Free})
}
if len(n.Fabrics) > 0 {
p.Fabric = n.Fabrics[0]
}
+65
View File
@@ -0,0 +1,65 @@
package adminapi
import (
"reflect"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
// migNode has card 0 cut into two 3g.40gb instances and card 1 whole.
func migNode(id string) inventory.Node {
nd := inventory.Node{ID: id, State: inventory.Ready, AgentVersion: "v1", MemoryTotalMB: 512 * 1024}
for dev := range 2 {
nd.GPUs = append(nd.GPUs, discovery.GPU{
Index: discovery.MIGIndex(0, dev), Name: "NVIDIA A100-SXM4-80GB MIG 3g.40gb", UUID: "MIG-x",
MemoryTotalMB: 40192, MemoryUsedMB: 37,
MIG: &discovery.MIGDevice{ParentIndex: 0, ParentMemoryTotalMB: 81920, Device: dev, Profile: "3g.40gb"},
})
}
nd.GPUs = append(nd.GPUs, discovery.GPU{Index: 1, Name: "NVIDIA A100-SXM4-80GB", MemoryTotalMB: 81920})
return nd
}
func TestANodeWithACardInMIGModeCountsItsDevicesByClass(t *testing.T) {
a, ctl := setup(t, testModels(), migNode("a100"))
ctl.Inventory().Allocate("a100", inventory.Allocation{ShardID: "s", ReplicaID: "r", GPUIndices: []int{1000}})
n := a.Nodes()[0]
want := []ClassGPUs{
{Class: "a100-80g-mig-3g.40gb", Total: 2, Free: 1, MIG: true},
{Class: "a100-80g", Total: 1, Free: 1},
}
if !reflect.DeepEqual(n.ClassGPUs, want) {
t.Errorf("by class: %+v\nwant %+v", n.ClassGPUs, want)
}
// Each device as people are shown it: the instance under its card, with
// its own memory, and the card its sensors are read under.
g := n.GPUs[1]
if g.Label != "0/mig1" || g.Card != 0 || g.MIGProfile != "3g.40gb" || g.MemoryTotalMB != 40192 ||
g.Class != "a100-80g-mig-3g.40gb" {
t.Errorf("instance view: %+v", g)
}
if w := n.GPUs[2]; w.Label != "1" || w.Card != 1 || w.MIGProfile != "" {
t.Errorf("whole card view: %+v", w)
}
// The fleet summary attributes each device to its own class, where it
// once put every card of a node under the node's only class.
s := a.Summary()
got := map[string][2]int{}
for _, c := range s.GPUs {
got[c.Class] = [2]int{c.Total, c.Used}
}
if !reflect.DeepEqual(got, map[string][2]int{"a100-80g-mig-3g.40gb": {2, 1}, "a100-80g": {1, 0}}) {
t.Errorf("summary by class: %v", got)
}
// And a plan sees both classes, each with its own count.
pool := a.PoolNodes()
if len(pool) != 1 || len(pool[0].Kinds) != 2 || pool[0].Kinds[0].GPUMemGB != 40 ||
pool[0].Kinds[0].FreeGPUs != 1 || pool[0].Kinds[1].GPUMemGB != 80 {
t.Errorf("pool: %+v", pool)
}
}
+8 -6
View File
@@ -148,12 +148,14 @@ func (a *API) Summary() Summary {
byClass[c] = &ClassUse{Class: c}
}
}
// A node reports one class in practice; attribute its cards to it.
if len(n.Classes) == 1 {
u := byClass[n.Classes[0]]
u.Total += n.GPUTotal
u.Unhealthy += len(n.Unhealthy)
u.Used += n.GPUTotal - n.GPUFree - len(n.Unhealthy)
// Each class its own devices, which a node with a card in MIG mode
// needs: its instances are one class or several, and any whole card
// beside them another.
for _, c := range n.ClassGPUs {
u := byClass[c.Class]
u.Total += c.Total
u.Unhealthy += c.Unhealthy
u.Used += c.Total - c.Free - c.Unhealthy
}
}
for _, u := range byClass {