MIG: a card in MIG mode is placed as its instances, each a GPU of class <card>-mig-<profile>

Discovery reports a card in MIG mode as its instances instead of the whole
card, on both paths: nvidia-smi (-L names them, -q -x gives their memory,
ids and processes) and NVML (GetMigMode, GetMaxMigDeviceCount,
GetMigDeviceHandleByIndex, behind an interface so it is tested with a fake).
Each instance has its MIG UUID, its own memory, and a class from the card's
and the profile: a100-80g-mig-3g.40gb. Placement numbers instances
1000 + 100 x card + instance and every view shows them as card/migN.

A replica on MIG runs on exactly one instance: a process opens one MIG
device and instances have no NVLink between them, so tp, pp, gpus or nodes
above one on a MIG class are refused in catalog validation and again in
placement, and the planner offers one instance or none. The agent gives
the engine CUDA_VISIBLE_DEVICES=MIG-<uuid> (no Vulkan device variable on
MIG). Memory and processes are per instance; heat, power and wear the
card's; utilisation is left out rather than reported as zero. A card in MIG
mode with nothing carved stays out of placement with the reason, and
llmgw doctor reports each card's mode and instances and warns of the states
in which nothing can run. ctl nodes, ctl status, the console and the plan
count devices by class. docs/install.md has a MIG section.

Written against fakes of nvidia-smi and NVML output; not yet run on MIG
hardware.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 00:01:14 +02:00
1 parent e81f1cba4c
commit 0fd55a862b
39 files changed
+2587 -327

No files matched your search

+28 -5
View File
@@ -1202,18 +1202,20 @@ const (
//
// The interface NCCL bootstraps over is this node's, not the cluster's: the
// control plane sends one name for every node, and a node whose management
// network is on ens3 rather than eth0 would have NCCL bind nothing. And on
// RoCE, which GID entry routes is a property of these adapters.
// network is on ens3 rather than eth0 would have NCCL bind nothing. On RoCE,
// which GID entry routes is a property of these adapters. And a MIG instance
// is opened by its UUID, which only this node knows.
func (a *Agent) localizeSpec(spec *engines.ShardSpec) {
if a.cfg.MgmtInterface != "" {
spec.MgmtInterface = a.cfg.MgmtInterface
}
a.mu.Lock()
hcas, gpus := a.inv.HCAs, a.inv.GPUs
a.mu.Unlock()
spec.GPUUUIDs = migUUIDs(gpus, spec.GPUIndices)
if len(spec.HCAs) == 0 {
return
}
a.mu.Lock()
hcas := a.inv.HCAs
a.mu.Unlock()
want := map[string]bool{}
for _, h := range spec.HCAs {
want[h] = true
@@ -1232,6 +1234,27 @@ func (a *Agent) localizeSpec(spec *engines.ShardSpec) {
}
}
// migUUIDs names a shard's devices by UUID when one of them is a MIG
// instance, and is nil otherwise: whole cards keep being given by index, as
// they always were. A device this node does not have is left to fail where
// it always has, in the engine, rather than guessed at here.
func migUUIDs(gpus []discovery.GPU, indices []int) []string {
var out []string
mig := false
for _, i := range indices {
k := slices.IndexFunc(gpus, func(g discovery.GPU) bool { return g.Index == i })
if k < 0 || gpus[k].UUID == "" {
return nil
}
mig = mig || gpus[k].IsMIG()
out = append(out, gpus[k].UUID)
}
if !mig {
return nil
}
return out
}
// ResolveManagement is the address and interface this node's agent would use,
// found the way normalize finds them, for a tool that reports them.
func ResolveManagement(c Config) (addr, iface string) {
+29 -2
View File
@@ -47,6 +47,11 @@ type GPU struct {
Fabric *GPUFabric `json:"fabric,omitempty"`
// Health carries anything that should keep the GPU out of placement.
Health GPUHealth `json:"health"`
// MIG is set when this is a MIG instance rather than a whole card: its
// Index is then MIGIndex's, its UUID the instance's (MIG-…), and its
// memory the instance's share. Utilisation stays unreported, because no
// driver measures it per instance.
MIG *MIGDevice `json:"mig,omitempty"`
}
// GPUHealth records conditions that make a GPU unsafe to schedule onto.
@@ -94,8 +99,30 @@ func (g GPU) Class() string {
}
// Round to the nearest GiB: vendors report 81559 MiB for an "80GB" card.
gb := int(math.Round(float64(g.MemoryTotalMB) / 1024))
return fmt.Sprintf("%s-%dg", model, gb)
// A MIG instance is named from its card's memory and then its profile,
// so two instances are one class only when both the card and the slice
// are the same.
mem := g.MemoryTotalMB
if g.MIG != nil {
mem = g.MIG.ParentMemoryTotalMB
}
gb := int(math.Round(float64(mem) / 1024))
class := fmt.Sprintf("%s-%dg", model, gb)
if g.MIG != nil {
return migClass(class, g.MIG.Profile)
}
return class
}
// normalizeBusID makes bus ids comparable: NVML reports them uppercase from one
// call and lowercase from another, and sometimes with a shortened domain.
func normalizeBusID(s string) string {
s = strings.ToLower(strings.TrimSpace(s))
// "0000:1b:00.0" and "00000000:1B:00.0" are the same device.
if i := strings.Index(s, ":"); i > 0 {
s = strings.TrimLeft(s[:i], "0") + s[i:]
}
return strings.TrimPrefix(s, ":")
}
// MarshalJSON adds the derived class, so a consumer of the JSON does not have
+77 -11
View File
@@ -14,7 +14,6 @@ package discovery
import (
"context"
"fmt"
"strings"
"github.com/NVIDIA/go-nvml/pkg/nvml"
)
@@ -83,11 +82,18 @@ func DiscoverGPUsNVML(_ context.Context) ([]GPU, error) {
}
assessHealth(&g)
gpus = append(gpus, g)
// A card in MIG mode is its instances, not itself: CUDA will not
// open the whole card while MIG is on.
devs, _ := migFromNVML(g, nvmlCard{dev})
gpus = append(gpus, devs...)
}
// A second pass, because a link's remote end may be a GPU enumerated later.
// A second pass, because a link's remote end may be a GPU enumerated
// later. MIG instances have no NVLink of their own to use.
for i := range gpus {
if gpus[i].IsMIG() {
continue
}
dev, ret := nvml.DeviceGetHandleByIndex(gpus[i].Index)
if ret != nvml.SUCCESS {
continue
@@ -136,13 +142,73 @@ func busIDString(b []int8) string {
return string(buf)
}
// normalizeBusID makes bus ids comparable: NVML reports them uppercase from one
// call and lowercase from another, and sometimes with a shortened domain.
func normalizeBusID(s string) string {
s = strings.ToLower(strings.TrimSpace(s))
// "0000:1b:00.0" and "00000000:1B:00.0" are the same device.
if i := strings.Index(s, ":"); i > 0 {
s = strings.TrimLeft(s[:i], "0") + s[i:]
// nvmlCard adapts an NVML device to what MIG discovery reads of it.
type nvmlCard struct{ dev nvml.Device }
func (c nvmlCard) MIGMode() (current, pending, ok bool) {
cur, pend, ret := c.dev.GetMigMode()
if ret != nvml.SUCCESS {
// NOT_SUPPORTED on a card without MIG, which is most of them.
return false, false, false
}
return strings.TrimPrefix(s, ":")
return cur == nvml.DEVICE_MIG_ENABLE, pend == nvml.DEVICE_MIG_ENABLE, true
}
func (c nvmlCard) MaxMIGDevices() int {
n, ret := c.dev.GetMaxMigDeviceCount()
if ret != nvml.SUCCESS {
return 0
}
return n
}
func (c nvmlCard) MIGDevice(i int) (nvmlMIGInstance, bool) {
h, ret := c.dev.GetMigDeviceHandleByIndex(i)
if ret != nvml.SUCCESS {
// NOT_FOUND for an index with no instance: the count is the most a
// card can hold, not how many it has.
return nil, false
}
return nvmlMIG{h}, true
}
// nvmlMIG adapts a MIG device handle.
type nvmlMIG struct{ dev nvml.Device }
func (m nvmlMIG) UUID() string {
u, _ := m.dev.GetUUID()
return u
}
func (m nvmlMIG) Name() string {
n, _ := m.dev.GetName()
return n
}
func (m nvmlMIG) Memory() (totalMB, usedMB int, ok bool) {
mem, ret := m.dev.GetMemoryInfo()
if ret != nvml.SUCCESS {
return 0, 0, false
}
return int(mem.Total / (1 << 20)), int(mem.Used / (1 << 20)), true
}
func (m nvmlMIG) Instances() (gpuInstance, computeInstance int) {
gi, ret := m.dev.GetGpuInstanceId()
if ret != nvml.SUCCESS {
gi = -1
}
ci, ret := m.dev.GetComputeInstanceId()
if ret != nvml.SUCCESS {
ci = -1
}
return gi, ci
}
func (m nvmlMIG) Attributes() (slices, memoryMB int, ok bool) {
a, ret := m.dev.GetAttributes()
if ret != nvml.SUCCESS {
return 0, 0, false
}
return int(a.GpuInstanceSliceCount), int(a.MemorySizeMB), true
}
+27 -1
View File
@@ -60,7 +60,33 @@ func DiscoverGPUsSMI(ctx context.Context, binary string) ([]GPU, error) {
if err != nil {
return nil, fmt.Errorf("running %s: %w", binary, err)
}
return parseSMI(string(out))
cards, err := parseSMI(string(out))
if err != nil || len(cards) == 0 {
return cards, err
}
// The query lists whole cards, MIG mode or not. A card in MIG mode is
// replaced by its instances, which only -L names and only the full
// report sizes; asked only when some card could be in MIG mode, since
// the report is the slowest thing nvidia-smi does.
if !anyMIGCapable(cards) {
return cards, nil
}
listed, report := readMIG(ctx, binary)
return expandMIG(cards, listed, report), nil
}
// anyMIGCapable reports whether any card could be in MIG mode: MIG began
// with compute capability 8.0 (the A100). The consumer cards of the same
// generations have none, but telling them apart by name is guesswork, and
// asking one costs only two more nvidia-smi runs at registration.
func anyMIGCapable(cards []GPU) bool {
for _, c := range cards {
major, _, _ := strings.Cut(c.ComputeCapability, ".")
if n, err := strconv.Atoi(major); err == nil && n >= 8 {
return true
}
}
return false
}
func parseSMI(out string) ([]GPU, error) {
+47 -25
View File
@@ -5,6 +5,7 @@ import (
"encoding/xml"
"fmt"
"os/exec"
"slices"
"strconv"
"strings"
)
@@ -34,9 +35,16 @@ func GPUProcesses(ctx context.Context, binary string) (map[int][]int, error) {
return parseGPUProcesses(out)
}
// GPUMemory is one card's memory as nvidia-smi reports it at a moment: what
// GPUMemory is one device's memory as nvidia-smi reports it at a moment: what
// is allocated on it, and which processes hold how much of that.
//
// A device is what discovery reports: a whole card, or each MIG instance of a
// card in MIG mode, in the same order, so the readings line up with the
// inventory one for one.
type GPUMemory struct {
// Index is the device's number as placement knows it: the card's
// nvidia-smi index, or MIGIndex's for an instance.
Index int
UsedMB int
Procs []GPUProcess
}
@@ -66,48 +74,62 @@ func GPUMemoryByProcess(ctx context.Context, binary string) ([]GPUMemory, error)
return parseGPUMemory(out)
}
// smiLog is the part of nvidia-smi's XML report this reads: each GPU, in
// index order, its memory and the processes on it.
type smiLog struct {
GPUs []struct {
Used string `xml:"fb_memory_usage>used"`
Processes []struct {
PID string `xml:"pid"`
Type string `xml:"type"`
UsedMB string `xml:"used_memory"`
} `xml:"processes>process_info"`
} `xml:"gpu"`
}
// parseGPUProcesses reads the report. GPUs are listed in index order, which
// is nvidia-smi's own numbering — the one placement assigns by.
// parseGPUProcesses reads the report, by device as placement numbers them:
// nvidia-smi's index for a whole card, MIGIndex's for a MIG instance.
func parseGPUProcesses(report []byte) (map[int][]int, error) {
mem, err := parseGPUMemory(report)
if err != nil {
return nil, err
}
out := map[int][]int{}
for i, g := range mem {
for _, g := range mem {
for _, p := range g.Procs {
out[i] = append(out[i], p.PID)
out[g.Index] = append(out[g.Index], p.PID)
}
}
return out, nil
}
// parseGPUMemory reads the report into each GPU's memory and its holders.
// parseGPUMemory reads the report into each device's memory and its holders.
//
// Cards are listed in index order, which is nvidia-smi's own numbering. A
// card in MIG mode is read as its instances, each with its own memory and
// the processes nvidia-smi attributes to its GPU and compute instance: a
// process on one instance is not on its neighbours, and the card's total
// would have charged every instance with all of them.
func parseGPUMemory(report []byte) ([]GPUMemory, error) {
var log smiLog
var log migLog
if err := xml.Unmarshal(report, &log); err != nil {
return nil, fmt.Errorf("reading nvidia-smi's report: %w", err)
}
out := make([]GPUMemory, len(log.GPUs))
out := make([]GPUMemory, 0, len(log.GPUs))
for i, g := range log.GPUs {
out[i].UsedMB = mib(g.Used)
for _, p := range g.Processes {
if pid, err := strconv.Atoi(strings.TrimSpace(p.PID)); err == nil {
out[i].Procs = append(out[i].Procs, GPUProcess{PID: pid, UsedMB: mib(p.UsedMB)})
procs := func(gi, ci int) []GPUProcess {
var ps []GPUProcess
for _, p := range g.Processes {
pid, err := strconv.Atoi(strings.TrimSpace(p.PID))
if err != nil {
continue
}
if gi >= 0 && (atoiOr(p.GPUInstance, -1) != gi || atoiOr(p.ComputeInstance, -1) != ci) {
continue
}
ps = append(ps, GPUProcess{PID: pid, UsedMB: mib(p.UsedMB)})
}
return ps
}
if !enabled(g.Current) || len(g.Devices) == 0 {
out = append(out, GPUMemory{Index: i, UsedMB: mib(g.Used), Procs: procs(-1, -1)})
continue
}
devs := slices.Clone(g.Devices)
slices.SortFunc(devs, func(a, b migLogDevice) int { return atoiOr(a.Index, -1) - atoiOr(b.Index, -1) })
for _, d := range devs {
out = append(out, GPUMemory{
Index: MIGIndex(i, atoiOr(d.Index, 0)),
UsedMB: mib(d.Used),
Procs: procs(atoiOr(d.GPUInstance, -1), atoiOr(d.ComputeInstance, -1)),
})
}
}
return out, nil
+2 -2
View File
@@ -57,8 +57,8 @@ func TestGPUMemoryIsReadPerProcess(t *testing.T) {
t.Fatal(err)
}
want := []GPUMemory{
{UsedMB: 14514, Procs: []GPUProcess{{PID: 71315, UsedMB: 4932}, {PID: 4100, UsedMB: 9500}}},
{},
{Index: 0, UsedMB: 14514, Procs: []GPUProcess{{PID: 71315, UsedMB: 4932}, {PID: 4100, UsedMB: 9500}}},
{Index: 1},
}
if !reflect.DeepEqual(got, want) {
t.Errorf("got %+v, want %+v", got, want)
+560
View File
@@ -0,0 +1,560 @@
package discovery
// MIG — Multi-Instance GPU — carves one A100, H100 or later card into up to
// seven instances, each with its own memory, its own share of the
// multiprocessors and its own UUID. To CUDA an instance is a device of its
// own, and to placement it is one too: a card in MIG mode is reported as its
// instances instead of as the whole card, each a GPU of a class named from
// the card's and the instance's profile ("a100-80g-mig-3g.40gb").
//
// llmgw does not create instances. Which split a card gets is an operator's
// decision, made with nvidia-smi mig; discovery reads what is there.
//
// Two facts about MIG shape everything below. A process is given one MIG
// device: CUDA enumerates a single instance however many are made visible.
// And instances have no NVLink or peer-to-peer access between them. A replica
// on MIG therefore runs on exactly one instance, never a tensor-parallel group
// of them, and the catalog refuses a profile that asks for more.
import (
"context"
"encoding/xml"
"fmt"
"math"
"os/exec"
"regexp"
"slices"
"strconv"
"strings"
)
// MIGDevice is what a MIG instance carries beyond any GPU's fields: which
// card it is carved from, and how.
type MIGDevice struct {
// ParentIndex is the card's nvidia-smi index, which is what its sensors,
// topology and wear are reported by.
ParentIndex int `json:"parent_index"`
ParentUUID string `json:"parent_uuid,omitempty"`
// ParentMemoryTotalMB is the whole card's memory: the class is named from
// the card, so an instance of an 80 GB card is not mistaken for one of a
// 40 GB card that happens to have the same profile name.
ParentMemoryTotalMB int `json:"parent_memory_total_mb"`
// Device is the instance's number on its card, as nvidia-smi -L lists it.
Device int `json:"device"`
// Profile is NVIDIA's name for the slice, like "3g.40gb": three of the
// card's seven compute slices and forty gigabytes of its memory.
Profile string `json:"profile"`
// GPUInstance and ComputeInstance are the instance's ids in nvidia-smi
// mig -lgi and -lci, which is how nvidia-smi attributes processes to it.
GPUInstance int `json:"gpu_instance"`
ComputeInstance int `json:"compute_instance"`
}
// migIndexBase starts the numbers MIG instances are given.
//
// Placement, the agent protocol and every running shard name a node's devices
// by an integer, which for a whole card is its nvidia-smi index. An instance
// has none of its own, so it is given 1000 + 100 × card + device: card 0's
// instance 2 is 1002, card 3's instance 0 is 1300. Readable, stable while the
// instances are, and never one a whole card could have.
const migIndexBase = 1000
// MIGIndex is the device number placement knows a MIG instance by.
func MIGIndex(card, device int) int { return migIndexBase + 100*card + device }
// IsMIGIndex reports whether a device number names a MIG instance.
func IsMIGIndex(i int) bool { return i >= migIndexBase }
// DeviceLabel is how a device number is shown to people: "3" for a whole
// card, "0/mig2" for card 0's MIG instance 2.
func DeviceLabel(i int) string {
if !IsMIGIndex(i) {
return strconv.Itoa(i)
}
i -= migIndexBase
return fmt.Sprintf("%d/mig%d", i/100, i%100)
}
// DeviceLabels is DeviceLabel for each of a list.
func DeviceLabels(is []int) []string {
out := make([]string, len(is))
for k, i := range is {
out[k] = DeviceLabel(i)
}
return out
}
// IsMIG reports whether this GPU is a MIG instance rather than a whole card.
func (g GPU) IsMIG() bool { return g.MIG != nil }
// CardIndex is the nvidia-smi index of the physical card this GPU is on: its
// own index for a whole card, its parent's for a MIG instance.
func (g GPU) CardIndex() int {
if g.MIG != nil {
return g.MIG.ParentIndex
}
return g.Index
}
// migClassMarker joins a card's class to an instance's profile in a MIG
// class: "a100-80g" + "-mig-" + "3g.40gb".
const migClassMarker = "-mig-"
// IsMIGClass reports whether a GPU class names MIG instances.
func IsMIGClass(class string) bool { return strings.Contains(class, migClassMarker) }
// migProfileClean keeps a profile name to what a class may hold: "1g.10gb+me"
// becomes "1g.10gb-me".
var migProfileClean = regexp.MustCompile(`[^a-z0-9.]+`)
// migClass is the class of a MIG instance: the card's class and the profile.
func migClass(parentClass, profile string) string {
p := migProfileClean.ReplaceAllString(strings.ToLower(strings.TrimSpace(profile)), "-")
return parentClass + migClassMarker + strings.Trim(p, "-")
}
// MIGProfileMemoryGB reads the memory out of a MIG class or profile:
// 40 from "a100-80g-mig-3g.40gb" or "3g.40gb". Zero when there is none.
func MIGProfileMemoryGB(s string) int {
if i := strings.LastIndex(s, migClassMarker); i >= 0 {
s = s[i+len(migClassMarker):]
}
m := migProfileMemory.FindStringSubmatch(strings.ToLower(s))
if m == nil {
return 0
}
gb, _ := strconv.Atoi(m[1])
return gb
}
var migProfileMemory = regexp.MustCompile(`^\d+g\.(\d+)gb`)
// --- nvidia-smi -L ---
// listedMIG is one MIG instance as nvidia-smi -L lists it.
type listedMIG struct {
Device int
Profile string
UUID string
}
var (
// "GPU 0: NVIDIA A100-SXM4-80GB (UUID: GPU-5d5ba0d6-...)"
smiListGPU = regexp.MustCompile(`^GPU (\d+):`)
// " MIG 3g.40gb Device 0: (UUID: MIG-c6d4f1ef-...)". Drivers before
// R470 name the instance MIG-GPU-<card uuid>/<gi>/<ci>, which CUDA of that
// era takes in the same place, so the UUID is read as whatever it is.
smiListMIG = regexp.MustCompile(`^\s+MIG\s+(\S+)\s+Device\s+(\d+):\s*\(UUID:\s*([^)\s]+)\)`)
)
// parseSMIList reads nvidia-smi -L into each card's MIG instances, by the
// card's index, in device order. A card with none is absent.
func parseSMIList(out string) map[int][]listedMIG {
res := map[int][]listedMIG{}
card := -1
for _, line := range strings.Split(out, "\n") {
if m := smiListGPU.FindStringSubmatch(line); m != nil {
card, _ = strconv.Atoi(m[1])
continue
}
m := smiListMIG.FindStringSubmatch(line)
if m == nil || card < 0 {
continue
}
dev, _ := strconv.Atoi(m[2])
res[card] = append(res[card], listedMIG{Device: dev, Profile: m[1], UUID: m[3]})
}
for c := range res {
slices.SortFunc(res[c], func(a, b listedMIG) int { return a.Device - b.Device })
}
return res
}
// --- nvidia-smi -q -x ---
// migLog is the part of nvidia-smi's XML report MIG is read from: each card's
// mode, and each instance's ids and memory.
type migLog struct {
GPUs []migLogGPU `xml:"gpu"`
}
type migLogGPU struct {
ID string `xml:"id,attr"`
Current string `xml:"mig_mode>current_mig"`
Pending string `xml:"mig_mode>pending_mig"`
Devices []migLogDevice `xml:"mig_devices>mig_device"`
// Processes carry the GPU and compute instance they run in on a card
// in MIG mode, and "N/A" for both otherwise.
Processes []struct {
GPUInstance string `xml:"gpu_instance_id"`
ComputeInstance string `xml:"compute_instance_id"`
PID string `xml:"pid"`
UsedMB string `xml:"used_memory"`
} `xml:"processes>process_info"`
Used string `xml:"fb_memory_usage>used"`
}
type migLogDevice struct {
Index string `xml:"index"`
GPUInstance string `xml:"gpu_instance_id"`
ComputeInstance string `xml:"compute_instance_id"`
Total string `xml:"fb_memory_usage>total"`
Used string `xml:"fb_memory_usage>used"`
}
// migState is one card's MIG mode and instances from the report.
type migState struct {
BusID string
Enabled bool
PendingEnabled bool
Devices []migReported
}
// migReported is one instance as the report describes it.
type migReported struct {
Device, GPUInstance, ComputeInstance int
TotalMB, UsedMB int
}
// enabled reads nvidia-smi's mode words: "Enabled", "Disabled", "N/A".
func enabled(s string) bool { return strings.EqualFold(strings.TrimSpace(s), "Enabled") }
func atoiOr(s string, def int) int {
n, err := strconv.Atoi(strings.TrimSpace(s))
if err != nil {
return def
}
return n
}
// parseMIGReport reads each card's MIG state from the XML report, in the
// report's order, which is nvidia-smi's index order.
func parseMIGReport(report []byte) ([]migState, error) {
var log migLog
if err := xml.Unmarshal(report, &log); err != nil {
return nil, fmt.Errorf("reading nvidia-smi's report: %w", err)
}
out := make([]migState, len(log.GPUs))
for i, g := range log.GPUs {
out[i] = migState{BusID: g.ID, Enabled: enabled(g.Current), PendingEnabled: enabled(g.Pending)}
for _, d := range g.Devices {
out[i].Devices = append(out[i].Devices, migReported{
Device: atoiOr(d.Index, -1), GPUInstance: atoiOr(d.GPUInstance, -1),
ComputeInstance: atoiOr(d.ComputeInstance, -1),
TotalMB: mib(d.Total), UsedMB: mib(d.Used),
})
}
slices.SortFunc(out[i].Devices, func(a, b migReported) int { return a.Device - b.Device })
}
return out, nil
}
// --- putting it together ---
// migProblemNoInstances keeps a card in MIG mode with nothing carved from it
// out of placement: CUDA cannot use the whole card while MIG is on, and
// there is no instance to use instead.
const migProblemNoInstances = "MIG is enabled with no instances created: nothing can run on it " +
"until instances are created (nvidia-smi mig -cgi … -C) or MIG is turned off"
// migProblemUnnamed is a card whose instances exist but could not be named:
// an engine is given an instance by its UUID, and without one it cannot be.
const migProblemUnnamed = "MIG is enabled and has instances, but nvidia-smi -L did not name them, " +
"so none can be given to an engine"
// expandMIG replaces each card in MIG mode with its instances.
//
// listed is nvidia-smi -L's instances by card, which names them; report is
// the XML report's state per card, which gives their memory and ids. Either
// may be missing: -L alone still says which instances exist, and the report
// alone still says a card is in MIG mode, so a card it says has instances
// that -L did not name is kept as the whole card with the reason, rather
// than placed as a card CUDA will not open.
func expandMIG(cards []GPU, listed map[int][]listedMIG, report []migState) []GPU {
if len(listed) == 0 && !slices.ContainsFunc(report, func(s migState) bool { return s.Enabled }) {
return cards
}
byBus := map[string]migState{}
for _, s := range report {
byBus[normalizeBusID(s.BusID)] = s
}
out := make([]GPU, 0, len(cards))
for pos, c := range cards {
st, ok := byBus[normalizeBusID(c.PCIBusID)]
if !ok && pos < len(report) {
st = report[pos]
}
devs := listed[c.Index]
if !st.Enabled && len(devs) == 0 {
out = append(out, c)
continue
}
if len(devs) == 0 {
p := migProblemNoInstances
if len(st.Devices) > 0 {
p = migProblemUnnamed
}
c.Health.Problems = append(c.Health.Problems, p)
out = append(out, c)
continue
}
for _, d := range devs {
var rep migReported
found := false
for _, r := range st.Devices {
if r.Device == d.Device {
rep, found = r, true
}
}
mig := migInstance(c, d.Device, d.Profile, d.UUID)
if found {
mig.MIG.GPUInstance, mig.MIG.ComputeInstance = rep.GPUInstance, rep.ComputeInstance
mig.MemoryTotalMB, mig.MemoryUsedMB = rep.TotalMB, rep.UsedMB
}
if mig.MemoryTotalMB == 0 {
// The profile says it to the gigabyte, which is close enough
// to place by when the report could not be read.
mig.MemoryTotalMB = MIGProfileMemoryGB(d.Profile) * 1024
}
assessHealth(&mig)
out = append(out, mig)
}
}
return out
}
// migInstance builds a MIG instance of card: everything about the card that
// is shared — driver, compute capability, NUMA node, bus, temperature and
// ECC errors — and nothing that is not. Not NVLink, which an instance cannot
// use; not utilisation, which no driver reports per instance; and not the
// card's memory or problems, which the instance's own replace.
func migInstance(card GPU, device int, profile, uuid string) GPU {
g := GPU{
Index: MIGIndex(card.Index, device), UUID: uuid,
Name: strings.TrimSpace(card.Name + " MIG " + profile),
ComputeCapability: card.ComputeCapability, NUMANode: card.NUMANode,
PCIBusID: card.PCIBusID, DriverVersion: card.DriverVersion, CUDAVersion: card.CUDAVersion,
NICPaths: card.NICPaths, Fabric: card.Fabric,
Health: GPUHealth{ECCErrors: card.Health.ECCErrors, TemperatureC: card.Health.TemperatureC},
MIG: &MIGDevice{
ParentIndex: card.Index, ParentUUID: card.UUID, ParentMemoryTotalMB: card.MemoryTotalMB,
Device: device, Profile: profile, GPUInstance: -1, ComputeInstance: -1,
},
}
return g
}
// readMIG asks nvidia-smi for what expandMIG needs. A failure of either
// command leaves its half empty, which expandMIG is written to take.
func readMIG(ctx context.Context, binary string) (map[int][]listedMIG, []migState) {
var listed map[int][]listedMIG
if out, err := exec.CommandContext(ctx, binary, "-L").Output(); err == nil {
listed = parseSMIList(string(out))
}
var report []migState
if out, err := exec.CommandContext(ctx, binary, "-q", "-x").Output(); err == nil {
report, _ = parseMIGReport(out)
}
return listed, report
}
// --- NVML ---
// nvmlMIGCard is the part of an NVML device MIG discovery reads, behind an
// interface so the logic is tested without a driver: gpu_nvml.go adapts
// nvml.Device to it.
type nvmlMIGCard interface {
// MIGMode is nvmlDeviceGetMigMode; ok is false on a card without MIG.
MIGMode() (current, pending, ok bool)
// MaxMIGDevices is nvmlDeviceGetMaxMigDeviceCount.
MaxMIGDevices() int
// MIGDevice is nvmlDeviceGetMigDeviceHandleByIndex; ok is false for an
// index with no instance behind it, which is how NVML says so.
MIGDevice(i int) (nvmlMIGInstance, bool)
}
// nvmlMIGInstance is one MIG device handle.
type nvmlMIGInstance interface {
UUID() string
// Name is "NVIDIA A100-SXM4-80GB MIG 3g.40gb" on the drivers that name
// MIG devices, which is where the profile is read from.
Name() string
Memory() (totalMB, usedMB int, ok bool)
Instances() (gpuInstance, computeInstance int)
// Slices and MemoryMB are the instance's attributes, which name the
// profile on a driver whose device name does not.
Attributes() (slices, memoryMB int, ok bool)
}
var nvmlMIGName = regexp.MustCompile(`MIG\s+(\S+)\s*$`)
// migFromNVML is what one card is to placement under NVML: its instances if
// it is in MIG mode, itself if it is not. isMIG says which.
func migFromNVML(card GPU, dev nvmlMIGCard) (out []GPU, isMIG bool) {
current, _, ok := dev.MIGMode()
if !ok || !current {
return []GPU{card}, false
}
n := dev.MaxMIGDevices()
for i := range n {
h, ok := dev.MIGDevice(i)
if !ok {
continue
}
profile := ""
if m := nvmlMIGName.FindStringSubmatch(h.Name()); m != nil {
profile = m[1]
} else if n, mem, ok := h.Attributes(); ok {
// The convention NVIDIA's names follow: compute slices, then the
// memory in whole gigabytes.
profile = fmt.Sprintf("%dg.%dgb", n, int(math.Round(float64(mem)/1024)))
}
g := migInstance(card, i, profile, h.UUID())
g.MIG.GPUInstance, g.MIG.ComputeInstance = h.Instances()
if total, used, ok := h.Memory(); ok {
g.MemoryTotalMB, g.MemoryUsedMB = total, used
} else {
g.MemoryTotalMB = MIGProfileMemoryGB(profile) * 1024
}
assessHealth(&g)
out = append(out, g)
}
if len(out) == 0 {
card.Health.Problems = append(card.Health.Problems, migProblemNoInstances)
return []GPU{card}, true
}
return out, true
}
// --- what the doctor reports ---
// MIGCard is one card's MIG state, as `llmgw doctor` reports it.
type MIGCard struct {
Index int `json:"index"`
Name string `json:"name,omitempty"`
// Enabled is the mode in force; PendingEnabled the mode after the next
// reset of the card, which differs from it until the card is reset.
Enabled bool `json:"enabled"`
PendingEnabled bool `json:"pending_enabled"`
// Instances are the MIG devices an engine can be given, as nvidia-smi -L
// lists them.
Instances []MIGListed `json:"instances,omitempty"`
// GPUInstances and ComputeInstances are nvidia-smi mig -lgi and -lci: a
// GPU instance with no compute instance in it is not a device yet.
GPUInstances []MIGInstance `json:"gpu_instances,omitempty"`
ComputeInstances []MIGInstance `json:"compute_instances,omitempty"`
}
// MIGListed is one instance an engine can be given.
type MIGListed struct {
Device int `json:"device"`
Profile string `json:"profile"`
UUID string `json:"uuid"`
}
// MIGInstance is one row of nvidia-smi mig -lgi or -lci.
type MIGInstance struct {
GPU int `json:"gpu"`
Profile string `json:"profile"`
// ProfileID is NVIDIA's number for the profile.
ProfileID int `json:"profile_id"`
// GPUInstance is the GPU instance's id; ComputeInstance the compute
// instance's within it, -1 in a -lgi row.
GPUInstance int `json:"gpu_instance"`
ComputeInstance int `json:"compute_instance"`
Placement string `json:"placement,omitempty"`
}
var (
// -lgi: "| 0 MIG 3g.40gb 9 2 4:4 |"
migLGIRow = regexp.MustCompile(`^\|\s*(\d+)\s+MIG\s+(\S+)\s+(\d+)\s+(\d+)\s+(\d+:\d+)\s*\|`)
// -lci: "| 0 1 MIG 3g.40gb 2 0 0:3 |"
migLCIRow = regexp.MustCompile(`^\|\s*(\d+)\s+(\d+)\s+MIG\s+(\S+)\s+(\d+)\s+(\d+)\s+(\d+:\d+)\s*\|`)
)
// ParseMIGInstances reads nvidia-smi mig -lgi or -lci, whichever it is
// given: the two tables differ by the GPU instance column -lci adds. "No GPU
// instances found" and the like are no rows, not an error.
func ParseMIGInstances(out string) []MIGInstance {
var res []MIGInstance
for _, line := range strings.Split(out, "\n") {
line = strings.TrimSpace(line)
if m := migLCIRow.FindStringSubmatch(line); m != nil {
res = append(res, MIGInstance{
GPU: atoiOr(m[1], -1), GPUInstance: atoiOr(m[2], -1), Profile: m[3],
ProfileID: atoiOr(m[4], -1), ComputeInstance: atoiOr(m[5], -1), Placement: m[6],
})
continue
}
if m := migLGIRow.FindStringSubmatch(line); m != nil {
res = append(res, MIGInstance{
GPU: atoiOr(m[1], -1), Profile: m[2], ProfileID: atoiOr(m[3], -1),
GPUInstance: atoiOr(m[4], -1), ComputeInstance: -1, Placement: m[5],
})
}
}
return res
}
// DiscoverMIG reads every card's MIG state for the doctor: the mode from the
// XML report, the devices from -L, the instances from nvidia-smi mig. Only
// cards that support MIG are listed, and an error is returned only when the
// report itself cannot be read.
func DiscoverMIG(ctx context.Context, binary string) ([]MIGCard, error) {
out, err := exec.CommandContext(ctx, binary, "-q", "-x").Output()
if err != nil {
return nil, fmt.Errorf("running %s -q -x: %w", binary, err)
}
var list, lgi, lci string
if b, err := exec.CommandContext(ctx, binary, "-L").Output(); err == nil {
list = string(b)
}
// nvidia-smi mig exits non-zero when there is nothing to list, with the
// table or the reason on its output either way.
b, _ := exec.CommandContext(ctx, binary, "mig", "-lgi").CombinedOutput()
lgi = string(b)
b, _ = exec.CommandContext(ctx, binary, "mig", "-lci").CombinedOutput()
lci = string(b)
return buildMIGCards(out, list, lgi, lci)
}
// buildMIGCards is DiscoverMIG from the commands' output.
func buildMIGCards(report []byte, list, lgi, lci string) ([]MIGCard, error) {
var log struct {
GPUs []struct {
Name string `xml:"product_name"`
Current string `xml:"mig_mode>current_mig"`
Pending string `xml:"mig_mode>pending_mig"`
} `xml:"gpu"`
}
if err := xml.Unmarshal(report, &log); err != nil {
return nil, fmt.Errorf("reading nvidia-smi's report: %w", err)
}
listed := parseSMIList(list)
gis, cis := ParseMIGInstances(lgi), ParseMIGInstances(lci)
var out []MIGCard
for i, g := range log.GPUs {
cur, pend := strings.TrimSpace(g.Current), strings.TrimSpace(g.Pending)
// A card without MIG says N/A, or nothing on a driver that predates
// it; neither is worth a line.
if !enabled(cur) && !enabled(pend) && !strings.EqualFold(cur, "Disabled") {
continue
}
c := MIGCard{Index: i, Name: strings.TrimSpace(g.Name), Enabled: enabled(cur), PendingEnabled: enabled(pend)}
for _, l := range listed[i] {
c.Instances = append(c.Instances, MIGListed(l))
}
for _, gi := range gis {
if gi.GPU == i && gi.ComputeInstance < 0 {
c.GPUInstances = append(c.GPUInstances, gi)
}
}
for _, ci := range cis {
if ci.GPU == i && ci.ComputeInstance >= 0 {
c.ComputeInstances = append(c.ComputeInstances, ci)
}
}
out = append(out, c)
}
return out, nil
}
+418
View File
@@ -0,0 +1,418 @@
package discovery
import (
"encoding/json"
"os"
"path/filepath"
"reflect"
"slices"
"strings"
"testing"
)
// Two A100 80GB cards: card 0 in MIG mode, cut into two 3g.40gb instances,
// card 1 whole. What nvidia-smi prints for that machine, by command. None of
// it is from real MIG hardware, which this was written without; the shapes
// follow NVIDIA's documentation and the output of drivers R470 to R550.
const (
migQuery = `0, GPU-5d5ba0d6-d33d-2b2c-524d-9e3d8d2b8a77, NVIDIA A100-SXM4-80GB, 81920, 0, 8.0, 00000000:07:00.0, 550.54.15, 31, 0, [N/A]
1, GPU-1f2e3d4c-5b6a-7980-a1b2-c3d4e5f60718, NVIDIA A100-SXM4-80GB, 81920, 0, 8.0, 00000000:0F:00.0, 550.54.15, 33, 0, 0
`
migList = `GPU 0: NVIDIA A100-SXM4-80GB (UUID: GPU-5d5ba0d6-d33d-2b2c-524d-9e3d8d2b8a77)
MIG 3g.40gb Device 0: (UUID: MIG-c6d4f1ef-42e4-5de3-91c7-45d71c87eb3f)
MIG 3g.40gb Device 1: (UUID: MIG-0f7c3a2e-9d1b-5a8e-b3c4-1e2f3a4b5c6d)
GPU 1: NVIDIA A100-SXM4-80GB (UUID: GPU-1f2e3d4c-5b6a-7980-a1b2-c3d4e5f60718)
`
migReport = `<?xml version="1.0" ?>
<nvidia_smi_log>
<driver_version>550.54.15</driver_version>
<gpu id="00000000:07:00.0">
<product_name>NVIDIA A100-SXM4-80GB</product_name>
<mig_mode><current_mig>Enabled</current_mig><pending_mig>Enabled</pending_mig></mig_mode>
<mig_devices>
<mig_device>
<index>1</index>
<gpu_instance_id>1</gpu_instance_id>
<compute_instance_id>0</compute_instance_id>
<device_attributes><shared><multiprocessor_count>42</multiprocessor_count></shared></device_attributes>
<fb_memory_usage><total>40192 MiB</total><reserved>0 MiB</reserved><used>21000 MiB</used><free>19192 MiB</free></fb_memory_usage>
</mig_device>
<mig_device>
<index>0</index>
<gpu_instance_id>2</gpu_instance_id>
<compute_instance_id>0</compute_instance_id>
<fb_memory_usage><total>40192 MiB</total><reserved>0 MiB</reserved><used>37 MiB</used><free>40155 MiB</free></fb_memory_usage>
</mig_device>
</mig_devices>
<fb_memory_usage><total>81920 MiB</total><used>21037 MiB</used></fb_memory_usage>
<processes>
<process_info>
<gpu_instance_id>1</gpu_instance_id>
<compute_instance_id>0</compute_instance_id>
<pid>4100</pid><type>C</type><process_name>vllm</process_name>
<used_memory>20900 MiB</used_memory>
</process_info>
</processes>
</gpu>
<gpu id="00000000:0F:00.0">
<product_name>NVIDIA A100-SXM4-80GB</product_name>
<mig_mode><current_mig>Disabled</current_mig><pending_mig>Disabled</pending_mig></mig_mode>
<mig_devices>None</mig_devices>
<fb_memory_usage><total>81920 MiB</total><used>300 MiB</used></fb_memory_usage>
<processes>
<process_info>
<gpu_instance_id>N/A</gpu_instance_id>
<compute_instance_id>N/A</compute_instance_id>
<pid>5200</pid><type>C</type><used_memory>250 MiB</used_memory>
</process_info>
</processes>
</gpu>
</nvidia_smi_log>`
migLGI = `+-------------------------------------------------------+
| GPU instances: |
| GPU Name Profile Instance Placement |
| ID ID Start:Size |
|=======================================================|
| 0 MIG 3g.40gb 9 1 4:4 |
+-------------------------------------------------------+
| 0 MIG 3g.40gb 9 2 0:4 |
+-------------------------------------------------------+
`
migLCI = `+--------------------------------------------------------------------+
| Compute instances: |
| GPU GPU Name Profile Instance Placement |
| Instance ID ID Start:Size |
| ID |
|====================================================================|
| 0 1 MIG 3g.40gb 2 0 0:3 |
+--------------------------------------------------------------------+
| 0 2 MIG 3g.40gb 2 0 0:3 |
+--------------------------------------------------------------------+
`
)
func migMachine(t *testing.T) []GPU {
t.Helper()
cards, err := parseSMI(migQuery)
if err != nil {
t.Fatal(err)
}
report, err := parseMIGReport([]byte(migReport))
if err != nil {
t.Fatal(err)
}
return expandMIG(cards, parseSMIList(migList), report)
}
func TestACardInMIGModeIsReportedAsItsInstancesInsteadOfItself(t *testing.T) {
gpus := migMachine(t)
var got []string
for _, g := range gpus {
got = append(got, DeviceLabel(g.Index)+" "+g.Class())
}
want := []string{"0/mig0 a100-80g-mig-3g.40gb", "0/mig1 a100-80g-mig-3g.40gb", "1 a100-80g"}
if !slices.Equal(got, want) {
t.Fatalf("devices = %q, want %q", got, want)
}
mig := gpus[1]
if mig.Index != 1001 || mig.UUID != "MIG-0f7c3a2e-9d1b-5a8e-b3c4-1e2f3a4b5c6d" {
t.Errorf("instance 1 is %d %s; an engine is given it by that UUID", mig.Index, mig.UUID)
}
if mig.MemoryTotalMB != 40192 || mig.MemoryUsedMB != 21000 {
t.Errorf("instance 1 memory %d/%d, want its own 21000/40192 rather than the card's",
mig.MemoryUsedMB, mig.MemoryTotalMB)
}
if m := mig.MIG; m == nil || m.ParentIndex != 0 || m.Device != 1 || m.Profile != "3g.40gb" ||
m.GPUInstance != 1 || m.ComputeInstance != 0 || m.ParentUUID != "GPU-5d5ba0d6-d33d-2b2c-524d-9e3d8d2b8a77" {
t.Errorf("instance 1's parentage: %+v", mig.MIG)
}
if mig.ComputeCapability != "8.0" || mig.DriverVersion != "550.54.15" || mig.Health.TemperatureC != 31 {
t.Errorf("what the card and its instances share was not carried: %+v", mig)
}
// Twenty gigabytes in use on an instance is a foreign tenant there, and
// only there: instance 0 next to it is free.
if mig.Health.OK() || !gpus[0].Health.OK() {
t.Errorf("foreign memory judged per instance: 0 %v, 1 %v", gpus[0].Health.Problems, mig.Health.Problems)
}
// No driver reports utilisation per instance, and none is made up.
if mig.UtilizationPercent != 0 {
t.Errorf("an instance reported utilisation %d", mig.UtilizationPercent)
}
if gpus[2].IsMIG() || gpus[2].Index != 1 || gpus[2].CardIndex() != 1 || gpus[0].CardIndex() != 0 {
t.Errorf("the whole card beside it changed: %+v", gpus[2])
}
}
func TestAMIGInstanceSurvivesTheTripToTheControlPlaneWithItsClass(t *testing.T) {
g := migMachine(t)[0]
b, err := json.Marshal(g)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(b), `"class":"a100-80g-mig-3g.40gb"`) {
t.Errorf("the JSON carries no class for the instance: %s", b)
}
var back GPU
if err := json.Unmarshal(b, &back); err != nil {
t.Fatal(err)
}
if back.Class() != g.Class() || !reflect.DeepEqual(back.MIG, g.MIG) {
t.Errorf("came back as %s %+v", back.Class(), back.MIG)
}
}
func TestMIGClassesAreNamedFromTheCardAndTheProfile(t *testing.T) {
for _, c := range []struct {
name string
cardMB int
profile string
want string
}{
{"NVIDIA A100-SXM4-80GB MIG 3g.40gb", 81920, "3g.40gb", "a100-80g-mig-3g.40gb"},
// The same profile name on a 40 GB card is another class: its
// instance is the same size, but the card is not, and neither is
// the set of profiles that card can be cut to.
{"NVIDIA A100-SXM4-40GB MIG 3g.20gb", 40960, "3g.20gb", "a100-40g-mig-3g.20gb"},
{"NVIDIA H100 80GB HBM3 MIG 1g.10gb+me", 81559, "1g.10gb+me", "h100-80g-mig-1g.10gb-me"},
{"NVIDIA H100 80GB HBM3 MIG 7g.80gb", 81559, "7g.80gb", "h100-80g-mig-7g.80gb"},
} {
g := GPU{Name: c.name, MemoryTotalMB: 10000, MIG: &MIGDevice{ParentMemoryTotalMB: c.cardMB, Profile: c.profile}}
if got := g.Class(); got != c.want {
t.Errorf("%s -> %s, want %s", c.name, got, c.want)
}
if !IsMIGClass(g.Class()) {
t.Errorf("%s is not recognised as a MIG class", g.Class())
}
}
if IsMIGClass("a100-80g") || IsMIGClass("6000-24g") {
t.Error("a whole card's class was taken for a MIG one")
}
if MIGProfileMemoryGB("a100-80g-mig-3g.40gb") != 40 || MIGProfileMemoryGB("1g.10gb+me") != 10 || MIGProfileMemoryGB("a100-80g") != 0 {
t.Error("the instance's memory is misread from its profile")
}
}
func TestDeviceNumbersAreShownAsCardAndInstance(t *testing.T) {
for i, want := range map[int]string{0: "0", 7: "7", MIGIndex(0, 2): "0/mig2", MIGIndex(3, 0): "3/mig0", 1302: "3/mig2"} {
if got := DeviceLabel(i); got != want {
t.Errorf("DeviceLabel(%d) = %s, want %s", i, got, want)
}
}
if IsMIGIndex(15) || !IsMIGIndex(MIGIndex(0, 0)) {
t.Error("whole cards and instances are told apart wrongly")
}
}
func TestACardInMIGModeWithNoInstancesIsKeptOutOfPlacement(t *testing.T) {
cards, _ := parseSMI(migQuery)
report, _ := parseMIGReport([]byte(strings.Replace(migReport,
migReport[strings.Index(migReport, "<mig_devices>"):strings.Index(migReport, "</mig_devices>")+len("</mig_devices>")],
"<mig_devices>None</mig_devices>", 1)))
gpus := expandMIG(cards, parseSMIList("GPU 0: NVIDIA A100-SXM4-80GB (UUID: GPU-x)\nGPU 1: NVIDIA A100-SXM4-80GB (UUID: GPU-y)\n"), report)
if len(gpus) != 2 || gpus[0].IsMIG() {
t.Fatalf("got %+v", gpus)
}
// CUDA will not open the whole card while MIG is on, and there is no
// instance to open instead: placing anything here fails at start.
if gpus[0].Health.OK() || !strings.Contains(gpus[0].Health.Problems[0], "no instances") {
t.Errorf("card 0's problems: %v", gpus[0].Health.Problems)
}
if !gpus[1].Health.OK() {
t.Errorf("card 1, not in MIG mode, was flagged: %v", gpus[1].Health.Problems)
}
// Instances the report sees but -L could not name cannot be given to an
// engine either; the card says why.
report, _ = parseMIGReport([]byte(migReport))
gpus = expandMIG(cards, nil, report)
if len(gpus) != 2 || gpus[0].Health.OK() || !strings.Contains(gpus[0].Health.Problems[0], "did not name") {
t.Errorf("with -L failing: %+v", gpus)
}
}
func TestInstancesAreFoundFromTheListAloneWhenTheReportCannotBeRead(t *testing.T) {
cards, _ := parseSMI(migQuery)
gpus := expandMIG(cards, parseSMIList(migList), nil)
if len(gpus) != 3 || !gpus[0].IsMIG() || gpus[0].MemoryTotalMB != 40*1024 {
t.Errorf("without the report: %+v", gpus)
}
}
func TestAnOlderDriversInstanceUUIDIsTakenAsItIs(t *testing.T) {
// R450 and R460 name an instance by its card and ids, and CUDA of that
// era takes exactly that in CUDA_VISIBLE_DEVICES.
l := parseSMIList("GPU 0: A100-SXM4-40GB (UUID: GPU-5d5ba0d6-d33d-2b2c-524d-9e3d8d2b8a77)\n" +
" MIG 1g.5gb Device 0: (UUID: MIG-GPU-5d5ba0d6-d33d-2b2c-524d-9e3d8d2b8a77/7/0)\n")
if len(l[0]) != 1 || l[0][0].UUID != "MIG-GPU-5d5ba0d6-d33d-2b2c-524d-9e3d8d2b8a77/7/0" || l[0][0].Profile != "1g.5gb" {
t.Errorf("parsed %+v", l)
}
}
func TestProcessesOnAMIGCardAreChargedToTheirOwnInstance(t *testing.T) {
mem, err := parseGPUMemory([]byte(migReport))
if err != nil {
t.Fatal(err)
}
want := []GPUMemory{
{Index: 1000, UsedMB: 37},
{Index: 1001, UsedMB: 21000, Procs: []GPUProcess{{PID: 4100, UsedMB: 20900}}},
{Index: 1, UsedMB: 300, Procs: []GPUProcess{{PID: 5200, UsedMB: 250}}},
}
if !reflect.DeepEqual(mem, want) {
t.Errorf("got %+v\nwant %+v", mem, want)
}
// In the order discovery reports the devices, so a heartbeat's readings
// line up with the inventory one for one.
gpus := migMachine(t)
for i := range gpus {
if mem[i].Index != gpus[i].Index {
t.Errorf("reading %d is for device %d, inventory has %d", i, mem[i].Index, gpus[i].Index)
}
}
procs, _ := parseGPUProcesses([]byte(migReport))
if !reflect.DeepEqual(procs, map[int][]int{1001: {4100}, 1: {5200}}) {
t.Errorf("processes by device: %v", procs)
}
}
func TestAMIGInstanceTakesItsCardsPlaceInTheTopologyButNoNVLink(t *testing.T) {
gpus := migMachine(t)
topo := Topology{
NVLink: map[int]map[int]int{0: {1: 12}, 1: {0: 12}},
NUMA: map[int]int{0: 0, 1: 1},
NICPath: map[int]map[string]string{0: {"mlx5_0": "PIX"}},
}
topo.Apply(gpus)
if len(gpus[0].NVLinkPeers) != 0 || len(gpus[1].NVLinkPeers) != 0 {
t.Errorf("MIG instances were given NVLink peers: %v %v", gpus[0].NVLinkPeers, gpus[1].NVLinkPeers)
}
if gpus[1].NUMANode != 0 || gpus[1].NICPaths["mlx5_0"] != "PIX" {
t.Errorf("instance 1 did not take its card's NUMA node and adapters: %+v", gpus[1])
}
if gpus[2].NUMANode != 1 || !slices.Equal(gpus[2].NVLinkPeers, []int{0}) {
t.Errorf("the whole card's topology: %+v", gpus[2])
}
}
func TestMIGInstancesAreDiscoveredThroughNvidiaSMI(t *testing.T) {
// The real code path against a stub binary that answers each command as
// the machine above would.
dir := t.TempDir()
stub := filepath.Join(dir, "nvidia-smi")
script := "#!/bin/sh\ncase \"$1\" in\n" +
"-L) cat <<'EOF'\n" + migList + "EOF\n;;\n" +
"-q) cat <<'EOF'\n" + migReport + "\nEOF\n;;\n" +
"*) cat <<'EOF'\n" + migQuery + "EOF\n;;\nesac\n"
if err := os.WriteFile(stub, []byte(script), 0o755); err != nil {
t.Fatal(err)
}
gpus, err := DiscoverGPUsSMI(t.Context(), stub)
if err != nil {
t.Fatal(err)
}
inv := Inventory{GPUs: gpus}
if !slices.Equal(inv.Classes(), []string{"a100-80g", "a100-80g-mig-3g.40gb"}) || len(gpus) != 3 {
t.Errorf("classes %v from %d devices", inv.Classes(), len(gpus))
}
}
// fakeMIGCard is NVML's view of one card, as nvmlDeviceGetMigMode,
// nvmlDeviceGetMaxMigDeviceCount and nvmlDeviceGetMigDeviceHandleByIndex
// would give it.
type fakeMIGCard struct {
mode, pending, supported bool
max int
devs map[int]fakeMIGInstance
}
func (c fakeMIGCard) MIGMode() (bool, bool, bool) { return c.mode, c.pending, c.supported }
func (c fakeMIGCard) MaxMIGDevices() int { return c.max }
func (c fakeMIGCard) MIGDevice(i int) (nvmlMIGInstance, bool) {
d, ok := c.devs[i]
return d, ok
}
type fakeMIGInstance struct {
uuid, name string
total, used int
gi, ci int
slices, attrMem int
}
func (m fakeMIGInstance) UUID() string { return m.uuid }
func (m fakeMIGInstance) Name() string { return m.name }
func (m fakeMIGInstance) Memory() (int, int, bool) {
return m.total, m.used, m.total > 0
}
func (m fakeMIGInstance) Instances() (int, int) { return m.gi, m.ci }
func (m fakeMIGInstance) Attributes() (int, int, bool) {
return m.slices, m.attrMem, m.slices > 0
}
func TestNVMLReportsACardInMIGModeAsItsInstances(t *testing.T) {
card := GPU{Index: 2, UUID: "GPU-card", Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559,
ComputeCapability: "9.0", NUMANode: 1, Health: GPUHealth{TemperatureC: 40}}
// Seven slots, three filled, one of them on a driver that does not put
// the profile in the device's name.
dev := fakeMIGCard{mode: true, pending: true, supported: true, max: 7, devs: map[int]fakeMIGInstance{
0: {uuid: "MIG-aaa", name: "NVIDIA H100 80GB HBM3 MIG 3g.40gb", total: 40448, used: 10, gi: 2, ci: 0},
1: {uuid: "MIG-bbb", name: "NVIDIA H100 80GB HBM3 MIG 3g.40gb", total: 40448, used: 10, gi: 1, ci: 0},
4: {uuid: "MIG-ccc", name: "NVIDIA H100 80GB HBM3", total: 9984, gi: 9, ci: 0, slices: 1, attrMem: 9984},
}}
gpus, isMIG := migFromNVML(card, dev)
if !isMIG || len(gpus) != 3 {
t.Fatalf("got %d devices, MIG %v", len(gpus), isMIG)
}
var classes []string
for _, g := range gpus {
classes = append(classes, DeviceLabel(g.Index)+" "+g.UUID+" "+g.Class())
}
want := []string{"2/mig0 MIG-aaa h100-80g-mig-3g.40gb", "2/mig1 MIG-bbb h100-80g-mig-3g.40gb",
"2/mig4 MIG-ccc h100-80g-mig-1g.10gb"}
if !slices.Equal(classes, want) {
t.Errorf("devices %q, want %q", classes, want)
}
if g := gpus[1]; g.MemoryTotalMB != 40448 || g.MIG.GPUInstance != 1 || g.NUMANode != 1 ||
g.Health.TemperatureC != 40 || len(g.NVLinkPeers) != 0 {
t.Errorf("instance 1: %+v", g)
}
// MIG on and nothing carved: the card, kept out of placement.
gpus, isMIG = migFromNVML(card, fakeMIGCard{mode: true, supported: true, max: 7})
if !isMIG || len(gpus) != 1 || gpus[0].IsMIG() || gpus[0].Health.OK() {
t.Errorf("an empty MIG card: %+v", gpus)
}
// MIG off, or a card without it: the card, as it always was.
for _, d := range []fakeMIGCard{{supported: true, max: 7}, {}} {
gpus, isMIG = migFromNVML(card, d)
if isMIG || len(gpus) != 1 || !reflect.DeepEqual(gpus[0], card) {
t.Errorf("a card not in MIG mode came back as %+v", gpus)
}
}
}
func TestMIGInstanceTablesAreRead(t *testing.T) {
gis := ParseMIGInstances(migLGI)
if len(gis) != 2 || gis[0] != (MIGInstance{GPU: 0, Profile: "3g.40gb", ProfileID: 9, GPUInstance: 1,
ComputeInstance: -1, Placement: "4:4"}) {
t.Errorf("-lgi: %+v", gis)
}
cis := ParseMIGInstances(migLCI)
if len(cis) != 2 || cis[1] != (MIGInstance{GPU: 0, GPUInstance: 2, Profile: "3g.40gb", ProfileID: 2,
ComputeInstance: 0, Placement: "0:3"}) {
t.Errorf("-lci: %+v", cis)
}
if got := ParseMIGInstances("No GPU instances found: Not Found\n"); len(got) != 0 {
t.Errorf("nothing to list read as %+v", got)
}
cards, err := buildMIGCards([]byte(migReport), migList, migLGI, migLCI)
if err != nil {
t.Fatal(err)
}
if len(cards) != 2 || !cards[0].Enabled || len(cards[0].Instances) != 2 || len(cards[0].GPUInstances) != 2 ||
len(cards[0].ComputeInstances) != 2 || cards[1].Enabled {
t.Errorf("cards: %+v", cards)
}
}
+10 -4
View File
@@ -166,19 +166,25 @@ func splitTabs(l string) []string {
// Apply puts the topology on the GPUs it describes. What a GPU already knows
// — NVLink peers from NVML, which is the more exact source — is kept.
//
// The matrix is of whole cards. A MIG instance takes its card's NUMA node and
// adapters, and none of its NVLink: instances cannot reach each other or any
// other card over it, and peers listed for one would draw a tensor-parallel
// group onto it.
func (t Topology) Apply(gpus []GPU) {
for i := range gpus {
g := &gpus[i]
if len(g.NVLinkPeers) == 0 {
for peer := range t.NVLink[g.Index] {
card := g.CardIndex()
if len(g.NVLinkPeers) == 0 && !g.IsMIG() {
for peer := range t.NVLink[card] {
g.NVLinkPeers = append(g.NVLinkPeers, peer)
}
sort.Ints(g.NVLinkPeers)
}
if n, ok := t.NUMA[g.Index]; ok {
if n, ok := t.NUMA[card]; ok {
g.NUMANode = n
}
if paths := t.NICPath[g.Index]; len(paths) > 0 {
if paths := t.NICPath[card]; len(paths) > 0 {
g.NICPaths = paths
}
}
+22 -1
View File
@@ -62,6 +62,12 @@ type ShardSpec struct {
// GPUIndices are the local GPUs this shard uses.
GPUIndices []int
// GPUUUIDs name the same devices by UUID, in the same order, and are set
// by the agent only when one of them is a MIG instance. An instance has
// no index CUDA knows: CUDA_VISIBLE_DEVICES=MIG-<uuid> is the only way to
// give a process one, and the indices placement uses for it (1000 and
// up) mean nothing to CUDA.
GPUUUIDs []string `json:"gpu_uuids,omitempty"`
// HCAs are the adapters of the chosen fabric, and only that fabric. A shard
// must never be given HCAs from two fabrics.
HCAs []string
@@ -257,7 +263,7 @@ func NCCLEnv(s ShardSpec) map[string]string { return baseEnv(s) }
// one instead of timing out.
func baseEnv(s ShardSpec) map[string]string {
env := map[string]string{
"CUDA_VISIBLE_DEVICES": joinInts(s.GPUIndices),
"CUDA_VISIBLE_DEVICES": visibleDevices(s),
// The indices above are nvidia-smi's, which counts by PCI bus. CUDA
// counts fastest first unless told otherwise, and on a node with two
// kinds of GPU the shard would get a different card than placement
@@ -329,6 +335,21 @@ func materializeEnv(generated map[string]string) []string {
return out
}
// visibleDevices is CUDA_VISIBLE_DEVICES for a shard: the indices placement
// gave it, or their UUIDs when the agent named them, which it does for a MIG
// instance.
func visibleDevices(s ShardSpec) string {
if len(s.GPUUUIDs) > 0 {
return strings.Join(s.GPUUUIDs, ",")
}
return joinInts(s.GPUIndices)
}
// OnMIG reports whether the shard was given a MIG instance.
func (s ShardSpec) OnMIG() bool {
return slices.ContainsFunc(s.GPUUUIDs, func(u string) bool { return strings.HasPrefix(u, "MIG-") })
}
func joinInts(v []int) string {
parts := make([]string, len(v))
for i, n := range v {
+5 -1
View File
@@ -178,7 +178,11 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) {
// as the driver lists them, which on NVIDIA follows the same bus order
// as nvidia-smi; were it not to, the shard would not hold memory on its
// own GPUs, and starting it checks exactly that.
if len(s.GPUIndices) > 0 {
//
// Not on a MIG instance, which Vulkan does not see at all: only CUDA
// opens one, by the UUID CUDA_VISIBLE_DEVICES carries, and an index here
// would name some other card.
if len(s.GPUIndices) > 0 && !s.OnMIG() {
env["GGML_VK_VISIBLE_DEVICES"] = joinInts(s.GPUIndices)
}
return []Command{{
+49
View File
@@ -0,0 +1,49 @@
package engines
import "testing"
const migUUID = "MIG-c6d4f1ef-42e4-5de3-91c7-45d71c87eb3f"
func TestAShardOnAMIGInstanceIsGivenItByUUID(t *testing.T) {
// Placement numbers card 0's instance 1 as 1001, which CUDA has never
// heard of: CUDA_VISIBLE_DEVICES=1001 is no device at all, and =1 would
// be the wrong card.
s := singleNodeSpec()
s.GPUIndices, s.GPUUUIDs = []int{1001}, []string{migUUID}
s.Profile.TP = 1
env := envMap(buildOne(t, &VLLM{}, s).Env)
if env["CUDA_VISIBLE_DEVICES"] != migUUID {
t.Errorf("vLLM: CUDA_VISIBLE_DEVICES=%q, want the instance's UUID", env["CUDA_VISIBLE_DEVICES"])
}
l := llamaSpec([]int{1001})
l.GPUUUIDs = s.GPUUUIDs
cmds, err := (&LlamaCpp{}).Commands(l)
if err != nil {
t.Fatal(err)
}
env = envMap(cmds[0].Env)
if env["CUDA_VISIBLE_DEVICES"] != migUUID {
t.Errorf("llama.cpp: CUDA_VISIBLE_DEVICES=%q, want the instance's UUID", env["CUDA_VISIBLE_DEVICES"])
}
// Vulkan does not see MIG instances, and an index for it would name
// some other card.
if v, set := env["GGML_VK_VISIBLE_DEVICES"]; set {
t.Errorf("GGML_VK_VISIBLE_DEVICES=%q on a MIG instance", v)
}
if v, _ := flagValue(cmds[0].Args, "--n-gpu-layers"); v != "999" {
t.Errorf("the model was not offloaded to the instance: --n-gpu-layers %q", v)
}
}
func TestAShardOnWholeCardsIsStillGivenThemByIndex(t *testing.T) {
env := envMap(buildOne(t, &VLLM{}, singleNodeSpec()).Env)
if env["CUDA_VISIBLE_DEVICES"] != "0,1,2,3" {
t.Errorf("CUDA_VISIBLE_DEVICES=%q", env["CUDA_VISIBLE_DEVICES"])
}
cmds, _ := (&LlamaCpp{}).Commands(llamaSpec([]int{2}))
if env := envMap(cmds[0].Env); env["GGML_VK_VISIBLE_DEVICES"] != "2" {
t.Errorf("GGML_VK_VISIBLE_DEVICES=%q", env["GGML_VK_VISIBLE_DEVICES"])
}
}
+40
View File
@@ -0,0 +1,40 @@
package agent
import (
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
)
func TestAShardPlacedOnAMIGInstanceIsLaunchedWithItsUUID(t *testing.T) {
// The control plane sends device 1001: card 0's instance 1, by the number
// placement gives it. Only this node knows the instance's UUID, which is
// the one thing CUDA will open it by.
a := &Agent{}
a.inv.GPUs = []discovery.GPU{
{Index: 1000, UUID: "MIG-aaa", MIG: &discovery.MIGDevice{ParentIndex: 0, Device: 0, Profile: "3g.40gb"}},
{Index: 1001, UUID: "MIG-bbb", MIG: &discovery.MIGDevice{ParentIndex: 0, Device: 1, Profile: "3g.40gb"}},
{Index: 1, UUID: "GPU-whole"},
}
spec := engines.ShardSpec{GPUIndices: []int{1001}}
a.localizeSpec(&spec)
if got := engines.NCCLEnv(spec)["CUDA_VISIBLE_DEVICES"]; got != "MIG-bbb" {
t.Errorf("CUDA_VISIBLE_DEVICES=%q, want MIG-bbb", got)
}
// A whole card is given by index, as it always was.
whole := engines.ShardSpec{GPUIndices: []int{1}}
a.localizeSpec(&whole)
if whole.GPUUUIDs != nil || engines.NCCLEnv(whole)["CUDA_VISIBLE_DEVICES"] != "1" {
t.Errorf("a whole card was given as %v / %q", whole.GPUUUIDs, engines.NCCLEnv(whole)["CUDA_VISIBLE_DEVICES"])
}
// A device this node does not have is not guessed at.
gone := engines.ShardSpec{GPUIndices: []int{1002}}
a.localizeSpec(&gone)
if gone.GPUUUIDs != nil {
t.Errorf("an unknown device was named %v", gone.GPUUUIDs)
}
}