MIG: a card in MIG mode is placed as its instances, each a GPU of class <card>-mig-<profile>
Discovery reports a card in MIG mode as its instances instead of the whole card, on both paths: nvidia-smi (-L names them, -q -x gives their memory, ids and processes) and NVML (GetMigMode, GetMaxMigDeviceCount, GetMigDeviceHandleByIndex, behind an interface so it is tested with a fake). Each instance has its MIG UUID, its own memory, and a class from the card's and the profile: a100-80g-mig-3g.40gb. Placement numbers instances 1000 + 100 x card + instance and every view shows them as card/migN. A replica on MIG runs on exactly one instance: a process opens one MIG device and instances have no NVLink between them, so tp, pp, gpus or nodes above one on a MIG class are refused in catalog validation and again in placement, and the planner offers one instance or none. The agent gives the engine CUDA_VISIBLE_DEVICES=MIG-<uuid> (no Vulkan device variable on MIG). Memory and processes are per instance; heat, power and wear the card's; utilisation is left out rather than reported as zero. A card in MIG mode with nothing carved stays out of placement with the reason, and llmgw doctor reports each card's mode and instances and warns of the states in which nothing can run. ctl nodes, ctl status, the console and the plan count devices by class. docs/install.md has a MIG section. Written against fakes of nvidia-smi and NVML output; not yet run on MIG hardware. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
e81f1cba4c
commit
0fd55a862b
39 files changed
+2587
-327
No files matched your search
@@ -159,13 +159,13 @@ func printInventory(inv discovery.Inventory) {
|
||||
if !g.Health.OK() {
|
||||
health = g.Health.Problems[0]
|
||||
}
|
||||
fmt.Fprintf(w, " %d\t%s\t%s\t%d MiB\t%s\t%s\n",
|
||||
g.Index, g.Class(), g.Name, g.MemoryTotalMB, g.ComputeCapability, health)
|
||||
fmt.Fprintf(w, " %s\t%s\t%s\t%d MiB\t%s\t%s\n",
|
||||
discovery.DeviceLabel(g.Index), g.Class(), g.Name, g.MemoryTotalMB, g.ComputeCapability, health)
|
||||
}
|
||||
fmt.Fprintf(w, " classes: %v\n", inv.Classes())
|
||||
for _, g := range inv.GPUs {
|
||||
if len(g.NVLinkPeers) > 0 {
|
||||
fmt.Fprintf(w, " GPU %d NVLink peers: %v\n", g.Index, g.NVLinkPeers)
|
||||
fmt.Fprintf(w, " GPU %s NVLink peers: %v\n", discovery.DeviceLabel(g.Index), g.NVLinkPeers)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+24
-2
@@ -14,6 +14,7 @@ import (
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
)
|
||||
|
||||
@@ -212,7 +213,7 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
// at when a model will not place and the hardware looks
|
||||
// fine.
|
||||
join(n.Engines),
|
||||
join(n.Classes), join(n.Fabrics), n.Replicas, n.LastSeenAgo)
|
||||
classesColumn(n), join(n.Fabrics), n.Replicas, n.LastSeenAgo)
|
||||
}
|
||||
_ = w.Flush()
|
||||
|
||||
@@ -410,7 +411,9 @@ func newReplicasCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
for _, r := range reps {
|
||||
var nodes []string
|
||||
for _, s := range r.Shards {
|
||||
nodes = append(nodes, fmt.Sprintf("%s:%v", s.Node, s.GPUs))
|
||||
// "[0/mig2]" for a MIG instance, not the 1002 it is
|
||||
// numbered by.
|
||||
nodes = append(nodes, fmt.Sprintf("%s:%v", s.Node, discovery.DeviceLabels(s.GPUs)))
|
||||
}
|
||||
// The state says stale rather than a column of its own: most
|
||||
// replicas are not, a column would be empty for all of them,
|
||||
@@ -540,6 +543,25 @@ func emitJSON(v any) error {
|
||||
return enc.Encode(v)
|
||||
}
|
||||
|
||||
// classesColumn is a node's GPU classes, with how many of each when it has
|
||||
// more than one or MIG instances among them: a card cut into two 3g.40gb
|
||||
// instances beside a whole card is "a100-80g-mig-3g.40gb×2,a100-80g×1", where
|
||||
// the GPUS column's 3 alone would read as three of something.
|
||||
func classesColumn(n adminapi.NodeView) string {
|
||||
counted := len(n.ClassGPUs) > 1
|
||||
for _, c := range n.ClassGPUs {
|
||||
counted = counted || c.MIG
|
||||
}
|
||||
if !counted {
|
||||
return join(n.Classes)
|
||||
}
|
||||
parts := make([]string, len(n.ClassGPUs))
|
||||
for i, c := range n.ClassGPUs {
|
||||
parts[i] = fmt.Sprintf("%s×%d", c.Class, c.Total)
|
||||
}
|
||||
return strings.Join(parts, ",")
|
||||
}
|
||||
|
||||
func join(v []string) string {
|
||||
if len(v) == 0 {
|
||||
return "-"
|
||||
|
||||
@@ -110,9 +110,10 @@ func printDoctor(out io.Writer, f doctor.Facts, findings []doctor.Finding, verbo
|
||||
|
||||
w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
|
||||
if len(f.GPUs) > 0 {
|
||||
fmt.Fprintln(w, "\nGPU\tNAME\tNUMA\tNVLINK PEERS\tNEAREST ADAPTER")
|
||||
fmt.Fprintln(w, "\nGPU\tNAME\tCLASS\tNUMA\tNVLINK PEERS\tNEAREST ADAPTER")
|
||||
for _, g := range f.GPUs {
|
||||
fmt.Fprintf(w, "%d\t%s\t%d\t%s\t%s\n", g.Index, g.Name, g.NUMANode,
|
||||
// A MIG instance as 0/mig2, under its card.
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%d\t%s\t%s\n", discovery.DeviceLabel(g.Index), g.Name, g.Class(), g.NUMANode,
|
||||
orDash(ints(g.NVLinkPeers)), orDash(nearest(g.NICPaths)))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
|
||||
)
|
||||
|
||||
func TestAPlanSeesANodesMIGInstancesAsAClassOfTheirOwn(t *testing.T) {
|
||||
// Card 0 cut into two 3g.40gb instances, both free, and card 1 whole and
|
||||
// busy: the instances are a pool of forty-gigabyte devices, and the busy
|
||||
// card no pool at all.
|
||||
hw := hardwareFrom([]adminapi.NodeView{{ID: "a100", State: "ready", GPUTotal: 3, GPUFree: 2,
|
||||
Classes: []string{"a100-80g-mig-3g.40gb", "a100-80g"},
|
||||
ClassGPUs: []adminapi.ClassGPUs{
|
||||
{Class: "a100-80g-mig-3g.40gb", Total: 2, Free: 2, MIG: true},
|
||||
{Class: "a100-80g", Total: 1, Free: 0},
|
||||
}}})
|
||||
if len(hw) != 1 {
|
||||
t.Fatalf("pools: %+v", hw)
|
||||
}
|
||||
want := modelplan.Hardware{Class: "a100-80g-mig-3g.40gb", GPUMemGB: 40, GPUsPerNode: 2, Nodes: 1}
|
||||
if hw[0] != want {
|
||||
t.Errorf("pool %+v, want %+v", hw[0], want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCtlNodesCountsANodesDevicesByClassWhenItHasMIG(t *testing.T) {
|
||||
mig := adminapi.NodeView{Classes: []string{"a100-80g-mig-3g.40gb", "a100-80g"}, ClassGPUs: []adminapi.ClassGPUs{
|
||||
{Class: "a100-80g-mig-3g.40gb", Total: 2, MIG: true}, {Class: "a100-80g", Total: 1}}}
|
||||
if got := classesColumn(mig); got != "a100-80g-mig-3g.40gb×2,a100-80g×1" {
|
||||
t.Errorf("MIG node: %q", got)
|
||||
}
|
||||
// A node of one kind of whole card reads as it always did.
|
||||
plain := adminapi.NodeView{Classes: []string{"h100-80g"}, ClassGPUs: []adminapi.ClassGPUs{{Class: "h100-80g", Total: 8}}}
|
||||
if got := classesColumn(plain); got != "h100-80g" {
|
||||
t.Errorf("plain node: %q", got)
|
||||
}
|
||||
}
|
||||
+35
-32
@@ -155,28 +155,41 @@ func hardwareFrom(nodes []adminapi.NodeView) []modelplan.Hardware {
|
||||
if len(n.Fabrics) > 0 {
|
||||
fabric = n.Fabrics[0]
|
||||
}
|
||||
class := ""
|
||||
if len(n.Classes) > 0 {
|
||||
class = n.Classes[0]
|
||||
// Each class of the node's devices is a pool of its own: a node with
|
||||
// a card in MIG mode has its instances' classes, and any whole card's
|
||||
// beside them. A node that does not count by class is its first
|
||||
// class, as it always was.
|
||||
kinds := n.ClassGPUs
|
||||
if len(kinds) == 0 {
|
||||
class := ""
|
||||
if len(n.Classes) > 0 {
|
||||
class = n.Classes[0]
|
||||
}
|
||||
kinds = []adminapi.ClassGPUs{{Class: class, Total: n.GPUTotal, Free: n.GPUFree}}
|
||||
}
|
||||
k := key{class, fabric}
|
||||
if pools[k] == nil {
|
||||
pools[k] = &modelplan.Hardware{Class: class, Fabric: fabric}
|
||||
}
|
||||
p := pools[k]
|
||||
p.Nodes++
|
||||
// The widest a single node could contribute, not the total: a tensor
|
||||
// group takes the same number of cards from every node it spans.
|
||||
if n.GPUFree > p.GPUsPerNode {
|
||||
p.GPUsPerNode = n.GPUFree
|
||||
}
|
||||
if n.MemoryGB > p.HostMemGB {
|
||||
p.HostMemGB = n.MemoryGB
|
||||
}
|
||||
// Card memory from the class label, which carries it by construction:
|
||||
// "h100-80g" is eighty gigabytes and says so.
|
||||
if class != "" && p.GPUMemGB == 0 {
|
||||
p.GPUMemGB = memFromClass(class)
|
||||
for _, kind := range kinds {
|
||||
class := kind.Class
|
||||
k := key{class, fabric}
|
||||
if pools[k] == nil {
|
||||
pools[k] = &modelplan.Hardware{Class: class, Fabric: fabric}
|
||||
}
|
||||
p := pools[k]
|
||||
p.Nodes++
|
||||
// The widest a single node could contribute, not the total: a
|
||||
// tensor group takes the same number of cards from every node it
|
||||
// spans.
|
||||
if kind.Free > p.GPUsPerNode {
|
||||
p.GPUsPerNode = kind.Free
|
||||
}
|
||||
if n.MemoryGB > p.HostMemGB {
|
||||
p.HostMemGB = n.MemoryGB
|
||||
}
|
||||
// Card memory from the class label, which carries it by
|
||||
// construction: "h100-80g" is eighty gigabytes and says so, and
|
||||
// "a100-80g-mig-3g.40gb" is an instance of forty.
|
||||
if class != "" && p.GPUMemGB == 0 {
|
||||
p.GPUMemGB = memFromClass(class)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -191,17 +204,7 @@ func hardwareFrom(nodes []adminapi.NodeView) []modelplan.Hardware {
|
||||
}
|
||||
|
||||
// memFromClass reads the card size out of a class label such as "h100-80g".
|
||||
func memFromClass(class string) int {
|
||||
i := strings.LastIndex(class, "-")
|
||||
if i < 0 || !strings.HasSuffix(class, "g") {
|
||||
return 0
|
||||
}
|
||||
var gb int
|
||||
if _, err := fmt.Sscanf(class[i+1:], "%dg", &gb); err != nil {
|
||||
return 0
|
||||
}
|
||||
return gb
|
||||
}
|
||||
func memFromClass(class string) int { return modelplan.MemFromClass(class) }
|
||||
|
||||
// catalogEntry renders a fit as the YAML that would serve it.
|
||||
func catalogEntry(m modelplan.Model, f modelplan.Fit, source string, context int) string {
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"github.com/charmbracelet/lipgloss"
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
@@ -1080,7 +1081,7 @@ func (m consoleModel) panel() (title, body, keys string) {
|
||||
for _, r := range m.data.replicas {
|
||||
var nodes []string
|
||||
for _, s := range r.Shards {
|
||||
nodes = append(nodes, fmt.Sprintf("%s%v", s.Node, s.GPUs))
|
||||
nodes = append(nodes, fmt.Sprintf("%s%v", s.Node, discovery.DeviceLabels(s.GPUs)))
|
||||
}
|
||||
rows = append(rows, []string{r.ID, r.Model, r.State, r.Pool, strings.Join(nodes, " "),
|
||||
(time.Duration(r.AgeSec) * time.Second).Round(time.Second).String()})
|
||||
@@ -1224,14 +1225,31 @@ func (m consoleModel) gpuRows() [][]string {
|
||||
for _, n := range m.data.nodes {
|
||||
for _, g := range n.GPUs {
|
||||
busy, mem, temp, power := "—", "—", "—", "—"
|
||||
// A MIG instance under its card: its own memory, the card's heat
|
||||
// and power, and no utilisation, which no driver measures per
|
||||
// instance — said, rather than shown as the card's or as zero.
|
||||
mig := g.MIGProfile != ""
|
||||
if mig {
|
||||
busy = sDim.Render("n/a on MIG")
|
||||
if g.MemoryTotalMB > 0 {
|
||||
mem = fmt.Sprintf("%.1f/%.0f GB", float64(g.MemoryUsedMB)/1024, float64(g.MemoryTotalMB)/1024)
|
||||
}
|
||||
}
|
||||
card := g.Card
|
||||
if g.Label == "" {
|
||||
// A control plane older than MIG, which sends no card.
|
||||
card = g.Index
|
||||
}
|
||||
if n.Load != nil {
|
||||
for _, s := range n.Load.GPUs {
|
||||
if s.Index != g.Index {
|
||||
if s.Index != card {
|
||||
continue
|
||||
}
|
||||
busy = fmt.Sprintf("%.0f%%", s.UtilPercent)
|
||||
if s.MemoryTotalMB > 0 {
|
||||
mem = fmt.Sprintf("%.1f/%.0f GB", s.MemoryUsedMB/1024, s.MemoryTotalMB/1024)
|
||||
if !mig {
|
||||
busy = fmt.Sprintf("%.0f%%", s.UtilPercent)
|
||||
if s.MemoryTotalMB > 0 {
|
||||
mem = fmt.Sprintf("%.1f/%.0f GB", s.MemoryUsedMB/1024, s.MemoryTotalMB/1024)
|
||||
}
|
||||
}
|
||||
if s.TempC > 0 {
|
||||
temp = fmt.Sprintf("%.0f°C", s.TempC)
|
||||
@@ -1245,7 +1263,12 @@ func (m consoleModel) gpuRows() [][]string {
|
||||
if holder == "" {
|
||||
holder = sDim.Render("free")
|
||||
}
|
||||
rows = append(rows, []string{n.ID, strconv.Itoa(g.Index), g.Name, busy, mem, temp, power, holder})
|
||||
label := g.Label
|
||||
if label == "" {
|
||||
// A control plane older than labels.
|
||||
label = strconv.Itoa(g.Index)
|
||||
}
|
||||
rows = append(rows, []string{n.ID, label, g.Name, busy, mem, temp, power, holder})
|
||||
}
|
||||
}
|
||||
return rows
|
||||
@@ -1266,6 +1289,15 @@ func (m consoleModel) nodeDetail(n adminapi.NodeView) string {
|
||||
}
|
||||
kv("disk", fmt.Sprintf("%d GB free of %d", n.DiskFreeGB, n.DiskTotalGB))
|
||||
kv("replicas", strconv.Itoa(n.Replicas))
|
||||
// By class, which on a node with a card in MIG mode is the instances'
|
||||
// profiles beside any whole card: what a profile's gpu_class can name.
|
||||
for _, c := range n.ClassGPUs {
|
||||
what := fmt.Sprintf("%d, %d free", c.Total, c.Free)
|
||||
if c.MIG {
|
||||
what += " · MIG instances, one to a replica"
|
||||
}
|
||||
kv(c.Class, what)
|
||||
}
|
||||
if len(n.Provision) > 0 {
|
||||
b.WriteString("\n " + lipgloss.NewStyle().Bold(true).Render("Engines") + "\n")
|
||||
for _, p := range n.Provision {
|
||||
|
||||
Reference in new issue
Block a user