MIG: a card in MIG mode is placed as its instances, each a GPU of class <card>-mig-<profile>

Discovery reports a card in MIG mode as its instances instead of the whole
card, on both paths: nvidia-smi (-L names them, -q -x gives their memory,
ids and processes) and NVML (GetMigMode, GetMaxMigDeviceCount,
GetMigDeviceHandleByIndex, behind an interface so it is tested with a fake).
Each instance has its MIG UUID, its own memory, and a class from the card's
and the profile: a100-80g-mig-3g.40gb. Placement numbers instances
1000 + 100 x card + instance and every view shows them as card/migN.

A replica on MIG runs on exactly one instance: a process opens one MIG
device and instances have no NVLink between them, so tp, pp, gpus or nodes
above one on a MIG class are refused in catalog validation and again in
placement, and the planner offers one instance or none. The agent gives
the engine CUDA_VISIBLE_DEVICES=MIG-<uuid> (no Vulkan device variable on
MIG). Memory and processes are per instance; heat, power and wear the
card's; utilisation is left out rather than reported as zero. A card in MIG
mode with nothing carved stays out of placement with the reason, and
llmgw doctor reports each card's mode and instances and warns of the states
in which nothing can run. ctl nodes, ctl status, the console and the plan
count devices by class. docs/install.md has a MIG section.

Written against fakes of nvidia-smi and NVML output; not yet run on MIG
hardware.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 00:01:14 +02:00
1 parent e81f1cba4c
commit 0fd55a862b
39 files changed
+2587 -327

No files matched your search

+58 -7
View File
@@ -228,7 +228,12 @@ type NodeView struct {
DiskFreeGB int `json:"disk_free_gb,omitempty"`
DiskTotalGB int `json:"disk_total_gb,omitempty"`
Classes []string `json:"gpu_classes,omitempty"`
Fabrics []string `json:"fabrics,omitempty"`
// ClassGPUs counts the node's devices by class, in the order of Classes.
// A node with a card in MIG mode has several classes — each profile its
// instances are cut to, and any whole card beside them — and GPUTotal
// alone would add 3g.40gb instances to 80 GB cards.
ClassGPUs []ClassGPUs `json:"gpus_by_class,omitempty"`
Fabrics []string `json:"fabrics,omitempty"`
// Unhealthy lists GPUs kept out of placement and why, which is usually the
// answer when a node looks idle but nothing will schedule onto it.
@@ -255,10 +260,34 @@ type NodeView struct {
IMEXChannels int `json:"imex_channels,omitempty"`
}
// ClassGPUs is how many of a node's devices are of one class, and how many
// of those are free or kept out of placement.
type ClassGPUs struct {
Class string `json:"class"`
Total int `json:"total"`
Free int `json:"free"`
Unhealthy int `json:"unhealthy,omitempty"`
// MIG marks a class of MIG instances, each of which takes a replica of
// its own and never joins a tensor-parallel group.
MIG bool `json:"mig,omitempty"`
}
// GPUView is one card as a node's page shows it.
type GPUView struct {
Index int `json:"index"`
Name string `json:"name"`
// Label is the device as people are shown it: "3" for a whole card,
// "0/mig2" for card 0's MIG instance 2.
Label string `json:"label"`
Class string `json:"class"`
// Card is the physical card's index: the device's own, or for a MIG
// instance its parent's, which is what the card's sensors are under.
Card int `json:"card"`
// MIGProfile is the instance's profile, like 3g.40gb; empty for a
// whole card.
MIGProfile string `json:"mig_profile,omitempty"`
MemoryTotalMB int `json:"memory_total_mb,omitempty"`
MemoryUsedMB int `json:"memory_used_mb,omitempty"`
// ComputeCapability is what decides which engine builds run on the card:
// CUDA 13 builds need 7.5, so a V100 (7.0) runs only older ones.
ComputeCapability string `json:"compute_capability,omitempty"`
@@ -556,15 +585,25 @@ func (a *API) Nodes() []NodeView {
for _, w := range n.Weights {
v.Models = append(v.Models, w.Path)
}
seen := map[string]struct{}{}
seen := map[string]int{}
free := inv.FreeGPUs(n.ID)
for _, g := range n.GPUs {
if _, dup := seen[g.Class()]; !dup {
seen[g.Class()] = struct{}{}
k, dup := seen[g.Class()]
if !dup {
k = len(v.ClassGPUs)
seen[g.Class()] = k
v.Classes = append(v.Classes, g.Class())
v.ClassGPUs = append(v.ClassGPUs, ClassGPUs{Class: g.Class(), MIG: g.IsMIG()})
}
if !g.Health.OK() {
c := &v.ClassGPUs[k]
c.Total++
switch {
case !g.Health.OK():
c.Unhealthy++
v.Unhealthy = append(v.Unhealthy,
fmt.Sprintf("gpu %d: %s", g.Index, g.Health.Problems[0]))
fmt.Sprintf("gpu %s: %s", discovery.DeviceLabel(g.Index), g.Health.Problems[0]))
case slices.Contains(free, g.Index):
c.Free++
}
}
if !n.LastSeen.IsZero() {
@@ -574,7 +613,12 @@ func (a *API) Nodes() []NodeView {
v.NVLinkDomain, v.IMEX, v.IMEXChannels = n.NVLinkDomain(), n.IMEX.State, n.IMEX.Channels
for _, g := range n.GPUs {
gv := GPUView{Index: g.Index, Name: g.Name, ComputeCapability: g.ComputeCapability,
NUMA: g.NUMANode, NVLinkPeers: g.NVLinkPeers}
NUMA: g.NUMANode, NVLinkPeers: g.NVLinkPeers,
Label: discovery.DeviceLabel(g.Index), Class: g.Class(), Card: g.CardIndex(),
MemoryTotalMB: g.MemoryTotalMB, MemoryUsedMB: g.MemoryUsedMB}
if g.MIG != nil {
gv.MIGProfile = g.MIG.Profile
}
for hca, rel := range g.NICPaths {
if gv.NearestNIC == "" || discovery.PathRank(rel) < discovery.PathRank(gv.NICPath) ||
(discovery.PathRank(rel) == discovery.PathRank(gv.NICPath) && hca < gv.NearestNIC) {
@@ -1617,6 +1661,13 @@ func (a *API) PoolNodes() []modelplan.PoolNode {
p.Class = n.Classes[0]
p.GPUMemGB = modelplan.MemFromClass(p.Class)
}
// By class, with each class's own count of devices: a node with a
// card in MIG mode is several, and its first class alone would
// count every instance and card as that one.
for _, c := range n.ClassGPUs {
p.Kinds = append(p.Kinds, modelplan.PoolKind{Class: c.Class,
GPUMemGB: modelplan.MemFromClass(c.Class), GPUs: c.Total, FreeGPUs: c.Free})
}
if len(n.Fabrics) > 0 {
p.Fabric = n.Fabrics[0]
}
+65
View File
@@ -0,0 +1,65 @@
package adminapi
import (
"reflect"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
// migNode has card 0 cut into two 3g.40gb instances and card 1 whole.
func migNode(id string) inventory.Node {
nd := inventory.Node{ID: id, State: inventory.Ready, AgentVersion: "v1", MemoryTotalMB: 512 * 1024}
for dev := range 2 {
nd.GPUs = append(nd.GPUs, discovery.GPU{
Index: discovery.MIGIndex(0, dev), Name: "NVIDIA A100-SXM4-80GB MIG 3g.40gb", UUID: "MIG-x",
MemoryTotalMB: 40192, MemoryUsedMB: 37,
MIG: &discovery.MIGDevice{ParentIndex: 0, ParentMemoryTotalMB: 81920, Device: dev, Profile: "3g.40gb"},
})
}
nd.GPUs = append(nd.GPUs, discovery.GPU{Index: 1, Name: "NVIDIA A100-SXM4-80GB", MemoryTotalMB: 81920})
return nd
}
func TestANodeWithACardInMIGModeCountsItsDevicesByClass(t *testing.T) {
a, ctl := setup(t, testModels(), migNode("a100"))
ctl.Inventory().Allocate("a100", inventory.Allocation{ShardID: "s", ReplicaID: "r", GPUIndices: []int{1000}})
n := a.Nodes()[0]
want := []ClassGPUs{
{Class: "a100-80g-mig-3g.40gb", Total: 2, Free: 1, MIG: true},
{Class: "a100-80g", Total: 1, Free: 1},
}
if !reflect.DeepEqual(n.ClassGPUs, want) {
t.Errorf("by class: %+v\nwant %+v", n.ClassGPUs, want)
}
// Each device as people are shown it: the instance under its card, with
// its own memory, and the card its sensors are read under.
g := n.GPUs[1]
if g.Label != "0/mig1" || g.Card != 0 || g.MIGProfile != "3g.40gb" || g.MemoryTotalMB != 40192 ||
g.Class != "a100-80g-mig-3g.40gb" {
t.Errorf("instance view: %+v", g)
}
if w := n.GPUs[2]; w.Label != "1" || w.Card != 1 || w.MIGProfile != "" {
t.Errorf("whole card view: %+v", w)
}
// The fleet summary attributes each device to its own class, where it
// once put every card of a node under the node's only class.
s := a.Summary()
got := map[string][2]int{}
for _, c := range s.GPUs {
got[c.Class] = [2]int{c.Total, c.Used}
}
if !reflect.DeepEqual(got, map[string][2]int{"a100-80g-mig-3g.40gb": {2, 1}, "a100-80g": {1, 0}}) {
t.Errorf("summary by class: %v", got)
}
// And a plan sees both classes, each with its own count.
pool := a.PoolNodes()
if len(pool) != 1 || len(pool[0].Kinds) != 2 || pool[0].Kinds[0].GPUMemGB != 40 ||
pool[0].Kinds[0].FreeGPUs != 1 || pool[0].Kinds[1].GPUMemGB != 80 {
t.Errorf("pool: %+v", pool)
}
}
+8 -6
View File
@@ -148,12 +148,14 @@ func (a *API) Summary() Summary {
byClass[c] = &ClassUse{Class: c}
}
}
// A node reports one class in practice; attribute its cards to it.
if len(n.Classes) == 1 {
u := byClass[n.Classes[0]]
u.Total += n.GPUTotal
u.Unhealthy += len(n.Unhealthy)
u.Used += n.GPUTotal - n.GPUFree - len(n.Unhealthy)
// Each class its own devices, which a node with a card in MIG mode
// needs: its instances are one class or several, and any whole card
// beside them another.
for _, c := range n.ClassGPUs {
u := byClass[c.Class]
u.Total += c.Total
u.Unhealthy += c.Unhealthy
u.Used += c.Total - c.Free - c.Unhealthy
}
}
for _, u := range byClass {
+21
View File
@@ -71,6 +71,7 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node {
TemperatureC: int(g.GetTemperatureC()),
Problems: g.GetProblems(),
},
MIG: migFrom(g.GetMig()),
})
}
n.IMEX = discovery.IMEX{State: r.GetImex().GetState(), Channels: int(r.GetImex().GetChannels())}
@@ -131,6 +132,13 @@ func RegisterFromInventory(nodeID, hostname, mgmtAddr, agentVersion string,
last.FabricState, last.FabricStatus, last.ClusterUuid = f.State, f.Status, f.ClusterUUID
last.CliqueId, last.FabricBandwidth = f.CliqueID, f.Bandwidth
}
if m := g.MIG; m != nil {
r.Gpus[len(r.Gpus)-1].Mig = &llmgwv1.MIGDevice{
ParentIndex: int32(m.ParentIndex), ParentUuid: m.ParentUUID,
ParentMemoryTotalMb: int32(m.ParentMemoryTotalMB), Device: int32(m.Device),
Profile: m.Profile, GpuInstance: int32(m.GPUInstance), ComputeInstance: int32(m.ComputeInstance),
}
}
}
r.Imex = &llmgwv1.IMEX{State: inv.IMEX.State, Channels: int32(inv.IMEX.Channels)}
for _, f := range fabrics {
@@ -286,6 +294,19 @@ func int32Slice(v []int) []int32 {
return out
}
// migFrom is a MIG instance's parentage as the wire carries it, nil for a
// whole card and from every agent older than MIG support.
func migFrom(m *llmgwv1.MIGDevice) *discovery.MIGDevice {
if m == nil {
return nil
}
return &discovery.MIGDevice{
ParentIndex: int(m.GetParentIndex()), ParentUUID: m.GetParentUuid(),
ParentMemoryTotalMB: int(m.GetParentMemoryTotalMb()), Device: int(m.GetDevice()),
Profile: m.GetProfile(), GPUInstance: int(m.GetGpuInstance()), ComputeInstance: int(m.GetComputeInstance()),
}
}
// fabricFrom is a GPU's NVLink fabric as the wire carries it, nil when the
// driver reported none.
func fabricFrom(state, status, cluster, clique, bandwidth string) *discovery.GPUFabric {
+27
View File
@@ -0,0 +1,27 @@
package agentsvc
import (
"reflect"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
)
func TestAMIGInstanceKeepsItsParentageOnTheWire(t *testing.T) {
inst := discovery.GPU{Index: 1001, UUID: "MIG-bbb", Name: "NVIDIA A100-SXM4-80GB MIG 3g.40gb", MemoryTotalMB: 40192,
MIG: &discovery.MIGDevice{ParentIndex: 0, ParentUUID: "GPU-card", ParentMemoryTotalMB: 81920,
Device: 1, Profile: "3g.40gb", GPUInstance: 1, ComputeInstance: 0}}
whole := discovery.GPU{Index: 1, UUID: "GPU-whole", Name: "NVIDIA A100-SXM4-80GB", MemoryTotalMB: 81920}
reg := RegisterFromInventory("n1", "h", "10.0.0.1", "v", nil, discovery.Inventory{GPUs: []discovery.GPU{inst, whole}}, nil)
n := NodeFromRegister(reg)
if len(n.GPUs) != 2 {
t.Fatalf("got %d GPUs", len(n.GPUs))
}
if !reflect.DeepEqual(n.GPUs[0].MIG, inst.MIG) || n.GPUs[0].Class() != "a100-80g-mig-3g.40gb" {
t.Errorf("the instance arrived as %s %+v", n.GPUs[0].Class(), n.GPUs[0].MIG)
}
if n.GPUs[1].MIG != nil || n.GPUs[1].Class() != "a100-80g" {
t.Errorf("the whole card arrived as %s %+v", n.GPUs[1].Class(), n.GPUs[1].MIG)
}
}
+12
View File
@@ -12,6 +12,7 @@ import (
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
)
@@ -701,6 +702,17 @@ func (m Model) validate() error {
}
continue
}
// A MIG instance is a device of its own to CUDA, but a process sees
// only one, and instances have no NVLink or peer-to-peer between
// them: a group of them is neither tensor- nor layer-parallel, it is
// a replica that fails at start. One instance, one node, refused here
// rather than placed and found out.
if discovery.IsMIGClass(p.GPUClass) && (p.GPUsPerNode > 1 || p.Nodes > 1 || p.TP > 1 || p.PP > 1) {
return fmt.Errorf("model %q profile %q: %s is a MIG instance, and a replica runs on exactly one: "+
"a process can open only one MIG device, and instances have no NVLink between them. "+
"Give it tp 1 and one GPU on one node, or use a whole-card class or a larger MIG profile",
m.Name, p.Name, p.GPUClass)
}
// llama.cpp on GPUs splits a model across one node's cards by layer,
// and how many is gpus_per_node. Not tp: llama.cpp has no tensor
// parallelism, and a tp it was given was a width nothing placed — the
+38
View File
@@ -0,0 +1,38 @@
package catalog
import (
"strings"
"testing"
)
func TestAProfileOnAMIGClassRunsOnExactlyOneInstance(t *testing.T) {
// One instance, by default or said outright, is a profile like any other.
for _, p := range []Profile{
{Name: "vllm", Engine: "vllm", GPUClass: "a100-80g-mig-3g.40gb"},
{Name: "llama", Engine: "llamacpp", GPUClass: "a100-80g-mig-3g.40gb", GPUsPerNode: 1, TP: 1},
} {
c, err := New([]Model{model("m", p)})
if err != nil {
t.Fatalf("%s on one MIG instance: %v", p.Name, err)
}
m, _ := c.Resolve("m")
if pl := m.PlacementProfiles()[0]; pl.GPUsPerNode != 1 || pl.Nodes != 1 {
t.Errorf("%s: placement asked for %+v", p.Name, pl)
}
}
// More than one is refused where the model is defined: a process opens
// one MIG device, and instances have no NVLink between them, so the
// replica would be placed and then fail at start.
for _, p := range []Profile{
{Name: "tp2", Engine: "vllm", GPUClass: "a100-80g-mig-3g.40gb", TP: 2},
{Name: "pp2", Engine: "vllm", GPUClass: "a100-80g-mig-3g.40gb", TP: 1, PP: 2, GPUsPerNode: 2},
{Name: "nodes", Engine: "vllm", GPUClass: "a100-80g-mig-3g.40gb", TP: 2, Nodes: 2, GPUsPerNode: 1},
{Name: "layers", Engine: "llamacpp", GPUClass: "a100-80g-mig-3g.40gb", GPUsPerNode: 2},
{Name: "old-tp", Engine: "llamacpp", GPUClass: "a100-80g-mig-3g.40gb", TP: 2},
} {
_, err := New([]Model{model("m", p)})
if err == nil || !strings.Contains(err.Error(), "MIG instance") {
t.Errorf("%s: %v", p.Name, err)
}
}
}
+12 -2
View File
@@ -430,8 +430,18 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) {
byClass[class]++
labels := []string{n.ID, strconv.Itoa(g.Index), class}
s, fresh := sensorFor(n.Load, g.Index, now)
if fresh && s.Busy() {
// The sensors are the card's. A MIG instance shares its card's
// heat and power, but not its memory, and has no utilisation of
// its own that any driver reports: its memory is the instance's
// own reading, and its utilisation is left out rather than
// published as the card's or as zero.
s, fresh := sensorFor(n.Load, g.CardIndex(), now)
if g.IsMIG() {
ch <- prometheus.MustNewConstMetric(c.gpuMemoryTotal, prometheus.GaugeValue,
float64(g.MemoryTotalMB)*1024*1024, labels...)
ch <- prometheus.MustNewConstMetric(c.gpuMemoryUsed, prometheus.GaugeValue,
float64(g.MemoryUsedMB)*1024*1024, labels...)
} else if fresh && s.Busy() {
// What the card is doing now, sampled every heartbeat.
c.gpuBusy(ch, s, labels)
} else {
+36
View File
@@ -0,0 +1,36 @@
package fleetmetrics
import (
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
func TestAMIGInstanceHasItsOwnMemoryItsCardsHeatAndNoUtilisation(t *testing.T) {
// Card 0 in MIG mode: the sensor sample is the card's, 60 GB used across
// both instances and a utilisation the driver does not give per instance.
inst := discovery.GPU{Index: 1001, Name: "NVIDIA A100-SXM4-80GB MIG 3g.40gb", MemoryTotalMB: 40192, MemoryUsedMB: 21000,
MIG: &discovery.MIGDevice{ParentIndex: 0, ParentMemoryTotalMB: 81920, Device: 1, Profile: "3g.40gb"}}
n := inventory.Node{ID: "g1", State: inventory.Ready, GPUs: []discovery.GPU{inst},
Load: &inventory.Load{At: time.Now(), GPUs: []inventory.GPUSensor{{Index: 0, TempC: 61, PowerW: 250,
UtilPercent: 0, MemoryUsedMB: 60000, MemoryTotalMB: 81920}}}}
out := gather(t, fakeSource{nodes: []inventory.Node{n}})
labels := "class=a100-80g-mig-3g.40gb gpu=1001 node=g1"
for _, want := range []string{
"llmgw_gpu_memory_used_bytes " + labels + " -> 22020096000\n",
"llmgw_gpu_memory_total_bytes " + labels + " -> 42144366592\n",
"llmgw_gpu_temperature_celsius " + labels + " -> 61\n",
"llmgw_gpu_power_watts " + labels + " -> 250\n",
} {
if !strings.Contains(out, want) {
t.Errorf("missing %s in:\n%s", want, out)
}
}
// Not the card's zero, not anything: absent.
if strings.Contains(out, "llmgw_gpu_utilization_percent "+labels) {
t.Errorf("a utilisation was published for a MIG instance:\n%s", out)
}
}
+87
View File
@@ -0,0 +1,87 @@
package placement
import (
"slices"
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
// migNode is an A100 80GB node as discovery reports it with card 0 cut into
// two 3g.40gb instances and card 1 whole.
func migNode(id string) inventory.Node {
card := discovery.GPU{Name: "NVIDIA A100-SXM4-80GB", MemoryTotalMB: 81920, ComputeCapability: "8.0"}
nd := inventory.Node{ID: id, State: inventory.Ready}
for dev, uuid := range []string{"MIG-c6d4f1ef", "MIG-0f7c3a2e"} {
g := card
g.Index, g.UUID, g.MemoryTotalMB = discovery.MIGIndex(0, dev), uuid, 40192
g.Name += " MIG 3g.40gb"
g.MIG = &discovery.MIGDevice{ParentIndex: 0, ParentMemoryTotalMB: 81920, Device: dev, Profile: "3g.40gb"}
nd.GPUs = append(nd.GPUs, g)
}
whole := card
whole.Index, whole.UUID = 1, "GPU-whole"
// NVLink to the card in MIG mode, as topo -m still reports it: nothing
// may take that for a link an instance can use.
whole.NVLinkPeers = []int{0}
nd.GPUs = append(nd.GPUs, whole)
return nd
}
func TestAModelIsPlacedOnAMIGInstanceOfItsClass(t *testing.T) {
st := store(migNode("n1"))
s := New(st)
p := Profile{Name: "mig", GPUClass: "a100-80g-mig-3g.40gb", GPUsPerNode: 1}
res := s.Place([]Profile{p}, nil)
if res.Placement == nil {
t.Fatalf("no placement:\n%s", res.Explain())
}
got := res.Placement.Nodes[0].GPUIndices
if !slices.Equal(got, []int{1000}) {
t.Fatalf("placed on %v, want MIG instance 1000", got)
}
if res.Placement.Pool != "none/a100-80g-mig-3g.40gb" {
t.Errorf("pool %q", res.Placement.Pool)
}
// The next replica takes the other instance, and the one after finds
// none: the whole card beside them is another class.
st.Allocate("n1", inventory.Allocation{ShardID: "s1", ReplicaID: "r1", GPUIndices: got})
res = s.Place([]Profile{p}, nil)
if res.Placement == nil || !slices.Equal(res.Placement.Nodes[0].GPUIndices, []int{1001}) {
t.Fatalf("second replica: %+v\n%s", res.Placement, res.Explain())
}
st.Allocate("n1", inventory.Allocation{ShardID: "s2", ReplicaID: "r2", GPUIndices: []int{1001}})
if res = s.Place([]Profile{p}, nil); res.Placement != nil {
t.Errorf("a third replica was placed on %v", res.Placement.Nodes[0].GPUIndices)
}
// A whole-card profile takes the whole card, never an instance.
res = s.Place([]Profile{{Name: "whole", GPUClass: "a100-80g", GPUsPerNode: 1}}, nil)
if res.Placement == nil || !slices.Equal(res.Placement.Nodes[0].GPUIndices, []int{1}) {
t.Errorf("whole-card profile: %+v\n%s", res.Placement, res.Explain())
}
}
func TestMoreThanOneMIGInstanceForAReplicaIsRefused(t *testing.T) {
// Two instances are free on one node, which is what a tp 2 profile would
// otherwise be given — and fail at start, since a process opens only one
// MIG device and instances have no NVLink between them.
s := New(store(migNode("n1"), migNode("n2")))
for _, p := range []Profile{
{Name: "tp2", GPUClass: "a100-80g-mig-3g.40gb", GPUsPerNode: 2},
{Name: "two-nodes", GPUClass: "a100-80g-mig-3g.40gb", GPUsPerNode: 1, Nodes: 2},
} {
res := s.Place([]Profile{p}, nil)
if res.Placement != nil {
t.Errorf("%s was placed: %+v", p.Name, res.Placement.Nodes)
continue
}
if !strings.Contains(res.Explain(), "a replica runs on exactly one") {
t.Errorf("%s refused without saying why:\n%s", p.Name, res.Explain())
}
}
}
+9
View File
@@ -156,6 +156,15 @@ func (s *Scheduler) Place(profiles []Profile, avoid []string) Result {
for _, raw := range ordered {
p := raw.normalize()
// The catalog refuses these; a profile that reaches here some other
// way is refused again rather than placed on instances that cannot
// work together. MIG instances are placed one to a replica.
if discovery.IsMIGClass(p.GPUClass) && (p.GPUsPerNode > 1 || p.Nodes > 1) {
res.Rejections = append(res.Rejections, Rejection{Profile: p.Name, Pool: "none/" + p.GPUClass,
Reason: fmt.Sprintf("asks for %d MIG instances on %d nodes, but a replica runs on exactly one",
p.GPUsPerNode*p.Nodes, p.Nodes)})
continue
}
if p.Nodes == 1 {
if pl, rej := s.placeSingleNode(p, avoid); pl != nil {
res.Placement = pl