MIG: a card in MIG mode is placed as its instances, each a GPU of class <card>-mig-<profile>

Discovery reports a card in MIG mode as its instances instead of the whole
card, on both paths: nvidia-smi (-L names them, -q -x gives their memory,
ids and processes) and NVML (GetMigMode, GetMaxMigDeviceCount,
GetMigDeviceHandleByIndex, behind an interface so it is tested with a fake).
Each instance has its MIG UUID, its own memory, and a class from the card's
and the profile: a100-80g-mig-3g.40gb. Placement numbers instances
1000 + 100 x card + instance and every view shows them as card/migN.

A replica on MIG runs on exactly one instance: a process opens one MIG
device and instances have no NVLink between them, so tp, pp, gpus or nodes
above one on a MIG class are refused in catalog validation and again in
placement, and the planner offers one instance or none. The agent gives
the engine CUDA_VISIBLE_DEVICES=MIG-<uuid> (no Vulkan device variable on
MIG). Memory and processes are per instance; heat, power and wear the
card's; utilisation is left out rather than reported as zero. A card in MIG
mode with nothing carved stays out of placement with the reason, and
llmgw doctor reports each card's mode and instances and warns of the states
in which nothing can run. ctl nodes, ctl status, the console and the plan
count devices by class. docs/install.md has a MIG section.

Written against fakes of nvidia-smi and NVML output; not yet run on MIG
hardware.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 00:01:14 +02:00
1 parent e81f1cba4c
commit 0fd55a862b
39 files changed
+2587 -327

No files matched your search

+8 -1
View File
@@ -132,7 +132,14 @@ func fitsOn(m Model, h Hardware, d Demand, p Precision, context int) []Fit {
// Powers of two only. Tensor parallelism divides attention heads across
// ranks, and an engine that cannot divide them evenly refuses the profile —
// so offering tp=3 would produce a plan that fails at launch.
for width := 1; width <= h.GPUsPerNode*h.Nodes; width *= 2 {
//
// One wide on MIG instances, however many there are: a process opens
// only one, and they have no NVLink between them.
widest := h.GPUsPerNode * h.Nodes
if isMIG(h.Class) {
widest = min(widest, 1)
}
for width := 1; width <= widest; width *= 2 {
nodes := 1
perNode := width
if width > h.GPUsPerNode {
+69
View File
@@ -0,0 +1,69 @@
package modelplan
import (
"strings"
"testing"
)
const mig3g = "a100-80g-mig-3g.40gb"
func TestAMIGInstanceIsSizedByItsProfileNotItsCard(t *testing.T) {
for class, want := range map[string]int{
mig3g: 40, "h100-80g-mig-1g.10gb": 10, "h100-80g-mig-1g.10gb-me": 10, "a100-80g": 80, "a100-80g-mig-": 0,
} {
if got := MemFromClass(class); got != want {
t.Errorf("MemFromClass(%q) = %d, want %d", class, got, want)
}
}
}
func TestAModelThatFitsOneMIGInstanceIsPlannedOnOne(t *testing.T) {
// Four instances free on each of two nodes on a fabric: a 3B fits one,
// and a 70B, which no single instance holds, is not spread over several
// — a replica cannot use more than one.
hw := []Hardware{{Class: mig3g, GPUMemGB: 40, GPUsPerNode: 4, Nodes: 2, Fabric: "ib-a"}}
fits := Fits(coder3B, hw, 32768, []Precision{BF16})
if len(fits) == 0 {
t.Fatal("a 3B does not fit a 40 GB MIG instance")
}
for _, f := range fits {
if f.TP != 1 || f.Nodes != 1 {
t.Errorf("planned across MIG instances: %s", f.Describe())
}
}
if fits := Fits(llama70B, hw, 8192, []Precision{BF16}); len(fits) != 0 {
t.Errorf("a 70B was planned on MIG instances: %s", fits[0].Describe())
}
}
func TestANodeWithACardInMIGModeOffersEachOfItsClasses(t *testing.T) {
// Card 0 cut into two 3g.40gb instances, card 1 whole: two classes on
// one node, each with its own count.
nodes := []PoolNode{{ID: "a100", Class: mig3g, GPUs: 3, FreeGPUs: 3, HostMemGB: 512, HostFreeGB: 400,
Kinds: []PoolKind{
{Class: mig3g, GPUMemGB: 40, GPUs: 2, FreeGPUs: 2},
{Class: "a100-80g", GPUMemGB: 80, GPUs: 1, FreeGPUs: 1},
}}}
arts := []Artifact{
{Name: "q4.gguf", Format: "gguf", Bytes: 9 * gib},
{Name: "f16.gguf", Format: "gguf", Bytes: 29 * gib},
{Name: "big.gguf", Format: "gguf", Bytes: 60 * gib},
}
opts := Options(arts, shape14B, nodes, 16384)
got := strings.Join(labels(opts), "; ")
for _, want := range []string{"q4.gguf 1 × " + mig3g, "f16.gguf 1 × " + mig3g, "big.gguf 1 × a100-80g"} {
if !strings.Contains(got, want) {
t.Errorf("missing %q in %s", want, got)
}
}
// Sixty gigabytes is more than an instance holds, and two instances are
// not one bigger device.
if strings.Contains(got, "big.gguf 2 × "+mig3g) {
t.Errorf("planned across two MIG instances: %s", got)
}
for _, o := range opts {
if o.Class == mig3g && (o.Hosts != 1 || !o.FreeNow) {
t.Errorf("the MIG option counts %d hosts, free %v", o.Hosts, o.FreeNow)
}
}
}
+78 -28
View File
@@ -46,8 +46,35 @@ type PoolNode struct {
// HostMemGB is its RAM, and HostFreeGB what is unused now.
HostMemGB int
HostFreeGB int
// Kinds are the node's devices by class, for a node with more than one:
// a card in MIG mode is one or several classes of instance, and a whole
// card beside it another. Empty means Class, GPUMemGB, GPUs and FreeGPUs
// say it all.
Kinds []PoolKind
}
// PoolKind is one class of a node's devices.
type PoolKind struct {
Class string
GPUMemGB int
GPUs int
FreeGPUs int
}
// kinds is the node's devices by class.
func (n PoolNode) kinds() []PoolKind {
if len(n.Kinds) > 0 {
return n.Kinds
}
return []PoolKind{{Class: n.Class, GPUMemGB: n.GPUMemGB, GPUs: n.GPUs, FreeGPUs: n.FreeGPUs}}
}
// isMIG reports whether a class names MIG instances: "a100-80g-mig-3g.40gb".
// A replica runs on exactly one, never a group of them, so a plan offers one
// instance and nothing wider. Read from the class label, as the card's memory
// is, which keeps this package free of the agent's.
func isMIG(class string) bool { return strings.Contains(class, "-mig-") }
// Option is one way to run one artifact.
type Option struct {
Artifact string `json:"artifact"`
@@ -149,11 +176,13 @@ func ggufOptions(a Artifact, kv int64, nodes []PoolNode, bpw float64) []Option {
}
o := Option{Artifact: a.Name, Format: a.Format, Engine: "llamacpp", Class: c.class,
GPUsPerNode: width, Nodes: 1, NeedGB: GB(per), HaveGB: budget, BitsPerWeight: bpw}
for _, n := range nodes {
if n.Class == c.class && n.GPUs >= width {
o.Hosts++
if n.FreeGPUs >= width {
o.FreeNow = true
for _, fab := range c.fabrics {
for _, n := range fab.nodes {
if n.GPUs >= width {
o.Hosts++
if n.FreeGPUs >= width {
o.FreeNow = true
}
}
}
}
@@ -196,7 +225,9 @@ func vllmOptions(a Artifact, kv int64, sh Shape, nodes []PoolNode, bpw float64)
single = true
break
}
if single {
// A MIG instance does not join others, on its node or across a
// fabric: what does not fit one does not fit the class.
if single || isMIG(c.class) {
continue
}
// Several nodes, a node's full width on each, pipeline-parallel
@@ -251,32 +282,42 @@ type fabricPool struct {
}
// classes groups GPU nodes by card, and each by fabric: a replica spans
// nodes only within one.
// nodes only within one. A node with devices of several classes is in each
// class's pool, as the node it is for that class: its devices of that class
// and no others.
func classes(nodes []PoolNode) []classPool {
by := map[string]*classPool{}
var order []string
for _, n := range nodes {
if n.Class == "" || n.GPUs == 0 || n.GPUMemGB == 0 {
continue
}
c := by[n.Class]
if c == nil {
c = &classPool{class: n.Class, mem: n.GPUMemGB}
by[n.Class] = c
order = append(order, n.Class)
}
c.maxGPUs = max(c.maxGPUs, n.GPUs)
i := -1
for j := range c.fabrics {
if c.fabrics[j].name == n.Fabric {
i = j
for _, node := range nodes {
for _, k := range node.kinds() {
if k.Class == "" || k.GPUs == 0 || k.GPUMemGB == 0 {
continue
}
n := node
n.Class, n.GPUMemGB, n.GPUs, n.FreeGPUs, n.Kinds = k.Class, k.GPUMemGB, k.GPUs, k.FreeGPUs, nil
if isMIG(n.Class) {
// One instance a replica, however many the node has.
n.GPUs, n.FreeGPUs = min(n.GPUs, 1), min(n.FreeGPUs, 1)
}
c := by[n.Class]
if c == nil {
c = &classPool{class: n.Class, mem: n.GPUMemGB}
by[n.Class] = c
order = append(order, n.Class)
}
c.maxGPUs = max(c.maxGPUs, n.GPUs)
i := -1
for j := range c.fabrics {
if c.fabrics[j].name == n.Fabric {
i = j
}
}
if i < 0 {
c.fabrics = append(c.fabrics, fabricPool{name: n.Fabric})
i = len(c.fabrics) - 1
}
c.fabrics[i].nodes = append(c.fabrics[i].nodes, n)
}
if i < 0 {
c.fabrics = append(c.fabrics, fabricPool{name: n.Fabric})
i = len(c.fabrics) - 1
}
c.fabrics[i].nodes = append(c.fabrics[i].nodes, n)
}
sort.Strings(order)
out := make([]classPool, 0, len(order))
@@ -363,8 +404,17 @@ func QuantOf(name string) string {
return strings.ToUpper(m[len(m)-1])
}
// MemFromClass reads the card size out of a class label such as "h100-80g".
// MemFromClass reads the card size out of a class label such as "h100-80g",
// and a MIG instance's out of its profile: 40 for "a100-80g-mig-3g.40gb",
// whose card is 80 but whose instance holds 40 of it.
func MemFromClass(class string) int {
if _, profile, ok := strings.Cut(class, "-mig-"); ok {
var compute, gb int
if _, err := fmt.Sscanf(profile, "%dg.%dgb", &compute, &gb); err != nil {
return 0
}
return gb
}
i := strings.LastIndex(class, "-")
if i < 0 || !strings.HasSuffix(class, "g") {
return 0