diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index d26030c2..bb007382 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -90,7 +90,7 @@ func (x ShardStatus_State) Number() protoreflect.EnumNumber { // Deprecated: Use ShardStatus_State.Descriptor instead. func (ShardStatus_State) EnumDescriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{21, 0} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{22, 0} } // AgentMessage is anything a node tells the control plane. @@ -1149,8 +1149,13 @@ type GPU struct { ClusterUuid string `protobuf:"bytes,19,opt,name=cluster_uuid,json=clusterUuid,proto3" json:"cluster_uuid,omitempty"` CliqueId string `protobuf:"bytes,20,opt,name=clique_id,json=cliqueId,proto3" json:"clique_id,omitempty"` FabricBandwidth string `protobuf:"bytes,21,opt,name=fabric_bandwidth,json=fabricBandwidth,proto3" json:"fabric_bandwidth,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // mig is set when this is a MIG instance rather than a whole card. index + // is then 1000 + 100 × card + device, uuid the instance's (MIG-…), and + // memory the instance's share; utilization_percent stays zero, because no + // driver measures it per instance. + Mig *MIGDevice `protobuf:"bytes,22,opt,name=mig,proto3" json:"mig,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GPU) Reset() { @@ -1330,6 +1335,109 @@ func (x *GPU) GetFabricBandwidth() string { return "" } +func (x *GPU) GetMig() *MIGDevice { + if x != nil { + return x.Mig + } + return nil +} + +// MIGDevice is which card a MIG instance is carved from, and how. +type MIGDevice struct { + state protoimpl.MessageState `protogen:"open.v1"` + ParentIndex int32 `protobuf:"varint,1,opt,name=parent_index,json=parentIndex,proto3" json:"parent_index,omitempty"` + ParentUuid string `protobuf:"bytes,2,opt,name=parent_uuid,json=parentUuid,proto3" json:"parent_uuid,omitempty"` + // parent_memory_total_mb is the whole card's: the class is named from it. + ParentMemoryTotalMb int32 `protobuf:"varint,3,opt,name=parent_memory_total_mb,json=parentMemoryTotalMb,proto3" json:"parent_memory_total_mb,omitempty"` + // device is the instance's number on its card, as nvidia-smi -L lists it. + Device int32 `protobuf:"varint,4,opt,name=device,proto3" json:"device,omitempty"` + // profile is NVIDIA's name for the slice, like 3g.40gb. + Profile string `protobuf:"bytes,5,opt,name=profile,proto3" json:"profile,omitempty"` + GpuInstance int32 `protobuf:"varint,6,opt,name=gpu_instance,json=gpuInstance,proto3" json:"gpu_instance,omitempty"` + ComputeInstance int32 `protobuf:"varint,7,opt,name=compute_instance,json=computeInstance,proto3" json:"compute_instance,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *MIGDevice) Reset() { + *x = MIGDevice{} + mi := &file_llmgw_v1_agent_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *MIGDevice) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MIGDevice) ProtoMessage() {} + +func (x *MIGDevice) ProtoReflect() protoreflect.Message { + mi := &file_llmgw_v1_agent_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MIGDevice.ProtoReflect.Descriptor instead. +func (*MIGDevice) Descriptor() ([]byte, []int) { + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{8} +} + +func (x *MIGDevice) GetParentIndex() int32 { + if x != nil { + return x.ParentIndex + } + return 0 +} + +func (x *MIGDevice) GetParentUuid() string { + if x != nil { + return x.ParentUuid + } + return "" +} + +func (x *MIGDevice) GetParentMemoryTotalMb() int32 { + if x != nil { + return x.ParentMemoryTotalMb + } + return 0 +} + +func (x *MIGDevice) GetDevice() int32 { + if x != nil { + return x.Device + } + return 0 +} + +func (x *MIGDevice) GetProfile() string { + if x != nil { + return x.Profile + } + return "" +} + +func (x *MIGDevice) GetGpuInstance() int32 { + if x != nil { + return x.GpuInstance + } + return 0 +} + +func (x *MIGDevice) GetComputeInstance() int32 { + if x != nil { + return x.ComputeInstance + } + return 0 +} + type FabricAttachment struct { state protoimpl.MessageState `protogen:"open.v1"` // fabric is the operator-assigned name. It is not derived from the GID @@ -1344,7 +1452,7 @@ type FabricAttachment struct { func (x *FabricAttachment) Reset() { *x = FabricAttachment{} - mi := &file_llmgw_v1_agent_proto_msgTypes[8] + mi := &file_llmgw_v1_agent_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1356,7 +1464,7 @@ func (x *FabricAttachment) String() string { func (*FabricAttachment) ProtoMessage() {} func (x *FabricAttachment) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[8] + mi := &file_llmgw_v1_agent_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1369,7 +1477,7 @@ func (x *FabricAttachment) ProtoReflect() protoreflect.Message { // Deprecated: Use FabricAttachment.ProtoReflect.Descriptor instead. func (*FabricAttachment) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{8} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{9} } func (x *FabricAttachment) GetFabric() string { @@ -1411,7 +1519,7 @@ type LocalWeights struct { func (x *LocalWeights) Reset() { *x = LocalWeights{} - mi := &file_llmgw_v1_agent_proto_msgTypes[9] + mi := &file_llmgw_v1_agent_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1423,7 +1531,7 @@ func (x *LocalWeights) String() string { func (*LocalWeights) ProtoMessage() {} func (x *LocalWeights) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[9] + mi := &file_llmgw_v1_agent_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1436,7 +1544,7 @@ func (x *LocalWeights) ProtoReflect() protoreflect.Message { // Deprecated: Use LocalWeights.ProtoReflect.Descriptor instead. func (*LocalWeights) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{9} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{10} } func (x *LocalWeights) GetPath() string { @@ -1492,7 +1600,7 @@ type NodeLoad struct { func (x *NodeLoad) Reset() { *x = NodeLoad{} - mi := &file_llmgw_v1_agent_proto_msgTypes[10] + mi := &file_llmgw_v1_agent_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1504,7 +1612,7 @@ func (x *NodeLoad) String() string { func (*NodeLoad) ProtoMessage() {} func (x *NodeLoad) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[10] + mi := &file_llmgw_v1_agent_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1517,7 +1625,7 @@ func (x *NodeLoad) ProtoReflect() protoreflect.Message { // Deprecated: Use NodeLoad.ProtoReflect.Descriptor instead. func (*NodeLoad) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{10} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11} } func (x *NodeLoad) GetCpuPercent() float64 { @@ -1650,7 +1758,7 @@ type GPUSensor struct { func (x *GPUSensor) Reset() { *x = GPUSensor{} - mi := &file_llmgw_v1_agent_proto_msgTypes[11] + mi := &file_llmgw_v1_agent_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1662,7 +1770,7 @@ func (x *GPUSensor) String() string { func (*GPUSensor) ProtoMessage() {} func (x *GPUSensor) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[11] + mi := &file_llmgw_v1_agent_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1675,7 +1783,7 @@ func (x *GPUSensor) ProtoReflect() protoreflect.Message { // Deprecated: Use GPUSensor.ProtoReflect.Descriptor instead. func (*GPUSensor) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12} } func (x *GPUSensor) GetIndex() int32 { @@ -1889,7 +1997,7 @@ type ShardLoad struct { func (x *ShardLoad) Reset() { *x = ShardLoad{} - mi := &file_llmgw_v1_agent_proto_msgTypes[12] + mi := &file_llmgw_v1_agent_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1901,7 +2009,7 @@ func (x *ShardLoad) String() string { func (*ShardLoad) ProtoMessage() {} func (x *ShardLoad) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[12] + mi := &file_llmgw_v1_agent_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1914,7 +2022,7 @@ func (x *ShardLoad) ProtoReflect() protoreflect.Message { // Deprecated: Use ShardLoad.ProtoReflect.Descriptor instead. func (*ShardLoad) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13} } func (x *ShardLoad) GetShardId() string { @@ -1982,7 +2090,7 @@ type IBPort struct { func (x *IBPort) Reset() { *x = IBPort{} - mi := &file_llmgw_v1_agent_proto_msgTypes[13] + mi := &file_llmgw_v1_agent_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1994,7 +2102,7 @@ func (x *IBPort) String() string { func (*IBPort) ProtoMessage() {} func (x *IBPort) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[13] + mi := &file_llmgw_v1_agent_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2007,7 +2115,7 @@ func (x *IBPort) ProtoReflect() protoreflect.Message { // Deprecated: Use IBPort.ProtoReflect.Descriptor instead. func (*IBPort) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14} } func (x *IBPort) GetHca() string { @@ -2150,7 +2258,7 @@ type SubnetManager struct { func (x *SubnetManager) Reset() { *x = SubnetManager{} - mi := &file_llmgw_v1_agent_proto_msgTypes[14] + mi := &file_llmgw_v1_agent_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2162,7 +2270,7 @@ func (x *SubnetManager) String() string { func (*SubnetManager) ProtoMessage() {} func (x *SubnetManager) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[14] + mi := &file_llmgw_v1_agent_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2175,7 +2283,7 @@ func (x *SubnetManager) ProtoReflect() protoreflect.Message { // Deprecated: Use SubnetManager.ProtoReflect.Descriptor instead. func (*SubnetManager) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15} } func (x *SubnetManager) GetGuid() string { @@ -2218,7 +2326,7 @@ type IMEX struct { func (x *IMEX) Reset() { *x = IMEX{} - mi := &file_llmgw_v1_agent_proto_msgTypes[15] + mi := &file_llmgw_v1_agent_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2230,7 +2338,7 @@ func (x *IMEX) String() string { func (*IMEX) ProtoMessage() {} func (x *IMEX) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[15] + mi := &file_llmgw_v1_agent_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2243,7 +2351,7 @@ func (x *IMEX) ProtoReflect() protoreflect.Message { // Deprecated: Use IMEX.ProtoReflect.Descriptor instead. func (*IMEX) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16} } func (x *IMEX) GetState() string { @@ -2275,7 +2383,7 @@ type GPUFabricState struct { func (x *GPUFabricState) Reset() { *x = GPUFabricState{} - mi := &file_llmgw_v1_agent_proto_msgTypes[16] + mi := &file_llmgw_v1_agent_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2287,7 +2395,7 @@ func (x *GPUFabricState) String() string { func (*GPUFabricState) ProtoMessage() {} func (x *GPUFabricState) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[16] + mi := &file_llmgw_v1_agent_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2300,7 +2408,7 @@ func (x *GPUFabricState) ProtoReflect() protoreflect.Message { // Deprecated: Use GPUFabricState.ProtoReflect.Descriptor instead. func (*GPUFabricState) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{17} } func (x *GPUFabricState) GetIndex() int32 { @@ -2415,7 +2523,7 @@ type Heartbeat struct { func (x *Heartbeat) Reset() { *x = Heartbeat{} - mi := &file_llmgw_v1_agent_proto_msgTypes[17] + mi := &file_llmgw_v1_agent_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2427,7 +2535,7 @@ func (x *Heartbeat) String() string { func (*Heartbeat) ProtoMessage() {} func (x *Heartbeat) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[17] + mi := &file_llmgw_v1_agent_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2440,7 +2548,7 @@ func (x *Heartbeat) ProtoReflect() protoreflect.Message { // Deprecated: Use Heartbeat.ProtoReflect.Descriptor instead. func (*Heartbeat) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{17} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{18} } func (x *Heartbeat) GetNodeId() string { @@ -2595,7 +2703,7 @@ type Provision struct { func (x *Provision) Reset() { *x = Provision{} - mi := &file_llmgw_v1_agent_proto_msgTypes[18] + mi := &file_llmgw_v1_agent_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2607,7 +2715,7 @@ func (x *Provision) String() string { func (*Provision) ProtoMessage() {} func (x *Provision) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[18] + mi := &file_llmgw_v1_agent_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2620,7 +2728,7 @@ func (x *Provision) ProtoReflect() protoreflect.Message { // Deprecated: Use Provision.ProtoReflect.Descriptor instead. func (*Provision) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{18} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{19} } func (x *Provision) GetEngines() []*EngineWant { @@ -2644,7 +2752,7 @@ type EngineWant struct { func (x *EngineWant) Reset() { *x = EngineWant{} - mi := &file_llmgw_v1_agent_proto_msgTypes[19] + mi := &file_llmgw_v1_agent_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2656,7 +2764,7 @@ func (x *EngineWant) String() string { func (*EngineWant) ProtoMessage() {} func (x *EngineWant) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[19] + mi := &file_llmgw_v1_agent_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2669,7 +2777,7 @@ func (x *EngineWant) ProtoReflect() protoreflect.Message { // Deprecated: Use EngineWant.ProtoReflect.Descriptor instead. func (*EngineWant) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{19} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{20} } func (x *EngineWant) GetEngine() string { @@ -2709,7 +2817,7 @@ type ProvisionStatus struct { func (x *ProvisionStatus) Reset() { *x = ProvisionStatus{} - mi := &file_llmgw_v1_agent_proto_msgTypes[20] + mi := &file_llmgw_v1_agent_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2721,7 +2829,7 @@ func (x *ProvisionStatus) String() string { func (*ProvisionStatus) ProtoMessage() {} func (x *ProvisionStatus) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[20] + mi := &file_llmgw_v1_agent_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2734,7 +2842,7 @@ func (x *ProvisionStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use ProvisionStatus.ProtoReflect.Descriptor instead. func (*ProvisionStatus) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{20} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{21} } func (x *ProvisionStatus) GetEngine() string { @@ -2798,7 +2906,7 @@ type ShardStatus struct { func (x *ShardStatus) Reset() { *x = ShardStatus{} - mi := &file_llmgw_v1_agent_proto_msgTypes[21] + mi := &file_llmgw_v1_agent_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2810,7 +2918,7 @@ func (x *ShardStatus) String() string { func (*ShardStatus) ProtoMessage() {} func (x *ShardStatus) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[21] + mi := &file_llmgw_v1_agent_proto_msgTypes[22] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2823,7 +2931,7 @@ func (x *ShardStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use ShardStatus.ProtoReflect.Descriptor instead. func (*ShardStatus) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{21} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{22} } func (x *ShardStatus) GetNodeId() string { @@ -2897,7 +3005,7 @@ type FetchProgress struct { func (x *FetchProgress) Reset() { *x = FetchProgress{} - mi := &file_llmgw_v1_agent_proto_msgTypes[22] + mi := &file_llmgw_v1_agent_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2909,7 +3017,7 @@ func (x *FetchProgress) String() string { func (*FetchProgress) ProtoMessage() {} func (x *FetchProgress) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[22] + mi := &file_llmgw_v1_agent_proto_msgTypes[23] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2922,7 +3030,7 @@ func (x *FetchProgress) ProtoReflect() protoreflect.Message { // Deprecated: Use FetchProgress.ProtoReflect.Descriptor instead. func (*FetchProgress) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{22} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{23} } func (x *FetchProgress) GetSource() string { @@ -2985,7 +3093,7 @@ type StartShard struct { func (x *StartShard) Reset() { *x = StartShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[23] + mi := &file_llmgw_v1_agent_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2997,7 +3105,7 @@ func (x *StartShard) String() string { func (*StartShard) ProtoMessage() {} func (x *StartShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[23] + mi := &file_llmgw_v1_agent_proto_msgTypes[24] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3010,7 +3118,7 @@ func (x *StartShard) ProtoReflect() protoreflect.Message { // Deprecated: Use StartShard.ProtoReflect.Descriptor instead. func (*StartShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{23} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{24} } func (x *StartShard) GetReplicaId() string { @@ -3154,7 +3262,7 @@ type EngineProfile struct { func (x *EngineProfile) Reset() { *x = EngineProfile{} - mi := &file_llmgw_v1_agent_proto_msgTypes[24] + mi := &file_llmgw_v1_agent_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3166,7 +3274,7 @@ func (x *EngineProfile) String() string { func (*EngineProfile) ProtoMessage() {} func (x *EngineProfile) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[24] + mi := &file_llmgw_v1_agent_proto_msgTypes[25] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3179,7 +3287,7 @@ func (x *EngineProfile) ProtoReflect() protoreflect.Message { // Deprecated: Use EngineProfile.ProtoReflect.Descriptor instead. func (*EngineProfile) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{24} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{25} } func (x *EngineProfile) GetName() string { @@ -3282,7 +3390,7 @@ type RemoveWeights struct { func (x *RemoveWeights) Reset() { *x = RemoveWeights{} - mi := &file_llmgw_v1_agent_proto_msgTypes[25] + mi := &file_llmgw_v1_agent_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3294,7 +3402,7 @@ func (x *RemoveWeights) String() string { func (*RemoveWeights) ProtoMessage() {} func (x *RemoveWeights) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[25] + mi := &file_llmgw_v1_agent_proto_msgTypes[26] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3307,7 +3415,7 @@ func (x *RemoveWeights) ProtoReflect() protoreflect.Message { // Deprecated: Use RemoveWeights.ProtoReflect.Descriptor instead. func (*RemoveWeights) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{25} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{26} } func (x *RemoveWeights) GetPaths() []string { @@ -3327,7 +3435,7 @@ type StopShard struct { func (x *StopShard) Reset() { *x = StopShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[26] + mi := &file_llmgw_v1_agent_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3339,7 +3447,7 @@ func (x *StopShard) String() string { func (*StopShard) ProtoMessage() {} func (x *StopShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[26] + mi := &file_llmgw_v1_agent_proto_msgTypes[27] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3352,7 +3460,7 @@ func (x *StopShard) ProtoReflect() protoreflect.Message { // Deprecated: Use StopShard.ProtoReflect.Descriptor instead. func (*StopShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{26} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{27} } func (x *StopShard) GetShardId() string { @@ -3372,7 +3480,7 @@ type DrainShard struct { func (x *DrainShard) Reset() { *x = DrainShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[27] + mi := &file_llmgw_v1_agent_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3384,7 +3492,7 @@ func (x *DrainShard) String() string { func (*DrainShard) ProtoMessage() {} func (x *DrainShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[27] + mi := &file_llmgw_v1_agent_proto_msgTypes[28] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3397,7 +3505,7 @@ func (x *DrainShard) ProtoReflect() protoreflect.Message { // Deprecated: Use DrainShard.ProtoReflect.Descriptor instead. func (*DrainShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{27} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{28} } func (x *DrainShard) GetShardId() string { @@ -3419,7 +3527,7 @@ type Ack struct { func (x *Ack) Reset() { *x = Ack{} - mi := &file_llmgw_v1_agent_proto_msgTypes[28] + mi := &file_llmgw_v1_agent_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3431,7 +3539,7 @@ func (x *Ack) String() string { func (*Ack) ProtoMessage() {} func (x *Ack) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[28] + mi := &file_llmgw_v1_agent_proto_msgTypes[29] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3444,7 +3552,7 @@ func (x *Ack) ProtoReflect() protoreflect.Message { // Deprecated: Use Ack.ProtoReflect.Descriptor instead. func (*Ack) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{28} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{29} } func (x *Ack) GetMessageId() string { @@ -3488,7 +3596,7 @@ type LogsRequest struct { func (x *LogsRequest) Reset() { *x = LogsRequest{} - mi := &file_llmgw_v1_agent_proto_msgTypes[29] + mi := &file_llmgw_v1_agent_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3500,7 +3608,7 @@ func (x *LogsRequest) String() string { func (*LogsRequest) ProtoMessage() {} func (x *LogsRequest) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[29] + mi := &file_llmgw_v1_agent_proto_msgTypes[30] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3513,7 +3621,7 @@ func (x *LogsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use LogsRequest.ProtoReflect.Descriptor instead. func (*LogsRequest) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{29} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{30} } func (x *LogsRequest) GetRequestId() string { @@ -3553,7 +3661,7 @@ type LogsReply struct { func (x *LogsReply) Reset() { *x = LogsReply{} - mi := &file_llmgw_v1_agent_proto_msgTypes[30] + mi := &file_llmgw_v1_agent_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3565,7 +3673,7 @@ func (x *LogsReply) String() string { func (*LogsReply) ProtoMessage() {} func (x *LogsReply) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[30] + mi := &file_llmgw_v1_agent_proto_msgTypes[31] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3578,7 +3686,7 @@ func (x *LogsReply) ProtoReflect() protoreflect.Message { // Deprecated: Use LogsReply.ProtoReflect.Descriptor instead. func (*LogsReply) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{30} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{31} } func (x *LogsReply) GetRequestId() string { @@ -3624,7 +3732,7 @@ type FabricScanRequest struct { func (x *FabricScanRequest) Reset() { *x = FabricScanRequest{} - mi := &file_llmgw_v1_agent_proto_msgTypes[31] + mi := &file_llmgw_v1_agent_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3636,7 +3744,7 @@ func (x *FabricScanRequest) String() string { func (*FabricScanRequest) ProtoMessage() {} func (x *FabricScanRequest) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[31] + mi := &file_llmgw_v1_agent_proto_msgTypes[32] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3649,7 +3757,7 @@ func (x *FabricScanRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use FabricScanRequest.ProtoReflect.Descriptor instead. func (*FabricScanRequest) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{31} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{32} } func (x *FabricScanRequest) GetRequestId() string { @@ -3688,7 +3796,7 @@ type FabricScanReply struct { func (x *FabricScanReply) Reset() { *x = FabricScanReply{} - mi := &file_llmgw_v1_agent_proto_msgTypes[32] + mi := &file_llmgw_v1_agent_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3700,7 +3808,7 @@ func (x *FabricScanReply) String() string { func (*FabricScanReply) ProtoMessage() {} func (x *FabricScanReply) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[32] + mi := &file_llmgw_v1_agent_proto_msgTypes[33] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3713,7 +3821,7 @@ func (x *FabricScanReply) ProtoReflect() protoreflect.Message { // Deprecated: Use FabricScanReply.ProtoReflect.Descriptor instead. func (*FabricScanReply) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{32} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{33} } func (x *FabricScanReply) GetRequestId() string { @@ -3769,7 +3877,7 @@ type FabricTestRequest struct { func (x *FabricTestRequest) Reset() { *x = FabricTestRequest{} - mi := &file_llmgw_v1_agent_proto_msgTypes[33] + mi := &file_llmgw_v1_agent_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3781,7 +3889,7 @@ func (x *FabricTestRequest) String() string { func (*FabricTestRequest) ProtoMessage() {} func (x *FabricTestRequest) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[33] + mi := &file_llmgw_v1_agent_proto_msgTypes[34] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3794,7 +3902,7 @@ func (x *FabricTestRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use FabricTestRequest.ProtoReflect.Descriptor instead. func (*FabricTestRequest) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{33} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{34} } func (x *FabricTestRequest) GetRequestId() string { @@ -3863,7 +3971,7 @@ type FabricTestReply struct { func (x *FabricTestReply) Reset() { *x = FabricTestReply{} - mi := &file_llmgw_v1_agent_proto_msgTypes[34] + mi := &file_llmgw_v1_agent_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3875,7 +3983,7 @@ func (x *FabricTestReply) String() string { func (*FabricTestReply) ProtoMessage() {} func (x *FabricTestReply) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[34] + mi := &file_llmgw_v1_agent_proto_msgTypes[35] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3888,7 +3996,7 @@ func (x *FabricTestReply) ProtoReflect() protoreflect.Message { // Deprecated: Use FabricTestReply.ProtoReflect.Descriptor instead. func (*FabricTestReply) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{34} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{35} } func (x *FabricTestReply) GetRequestId() string { @@ -3948,7 +4056,7 @@ type UpgradeAgent struct { func (x *UpgradeAgent) Reset() { *x = UpgradeAgent{} - mi := &file_llmgw_v1_agent_proto_msgTypes[35] + mi := &file_llmgw_v1_agent_proto_msgTypes[36] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3960,7 +4068,7 @@ func (x *UpgradeAgent) String() string { func (*UpgradeAgent) ProtoMessage() {} func (x *UpgradeAgent) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[35] + mi := &file_llmgw_v1_agent_proto_msgTypes[36] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3973,7 +4081,7 @@ func (x *UpgradeAgent) ProtoReflect() protoreflect.Message { // Deprecated: Use UpgradeAgent.ProtoReflect.Descriptor instead. func (*UpgradeAgent) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{35} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{36} } func (x *UpgradeAgent) GetRequestId() string { @@ -4010,7 +4118,7 @@ type UpgradeReply struct { func (x *UpgradeReply) Reset() { *x = UpgradeReply{} - mi := &file_llmgw_v1_agent_proto_msgTypes[36] + mi := &file_llmgw_v1_agent_proto_msgTypes[37] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4022,7 +4130,7 @@ func (x *UpgradeReply) String() string { func (*UpgradeReply) ProtoMessage() {} func (x *UpgradeReply) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[36] + mi := &file_llmgw_v1_agent_proto_msgTypes[37] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4035,7 +4143,7 @@ func (x *UpgradeReply) ProtoReflect() protoreflect.Message { // Deprecated: Use UpgradeReply.ProtoReflect.Descriptor instead. func (*UpgradeReply) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{36} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{37} } func (x *UpgradeReply) GetRequestId() string { @@ -4071,7 +4179,7 @@ type LoraAdapter struct { func (x *LoraAdapter) Reset() { *x = LoraAdapter{} - mi := &file_llmgw_v1_agent_proto_msgTypes[37] + mi := &file_llmgw_v1_agent_proto_msgTypes[38] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4083,7 +4191,7 @@ func (x *LoraAdapter) String() string { func (*LoraAdapter) ProtoMessage() {} func (x *LoraAdapter) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[37] + mi := &file_llmgw_v1_agent_proto_msgTypes[38] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4096,7 +4204,7 @@ func (x *LoraAdapter) ProtoReflect() protoreflect.Message { // Deprecated: Use LoraAdapter.ProtoReflect.Descriptor instead. func (*LoraAdapter) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{37} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{38} } func (x *LoraAdapter) GetName() string { @@ -4210,7 +4318,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\vleader_port\x18\x0e \x01(\x05R\n" + "leaderPort\x12\x1d\n" + "\n" + - "profile_fp\x18\x0f \x01(\tR\tprofileFp\"\xa3\x06\n" + + "profile_fp\x18\x0f \x01(\tR\tprofileFp\"\xca\x06\n" + "\x03GPU\x12\x14\n" + "\x05index\x18\x01 \x01(\x05R\x05index\x12\x12\n" + "\x04uuid\x18\x02 \x01(\tR\x04uuid\x12\x12\n" + @@ -4235,10 +4343,20 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\rfabric_status\x18\x12 \x01(\tR\ffabricStatus\x12!\n" + "\fcluster_uuid\x18\x13 \x01(\tR\vclusterUuid\x12\x1b\n" + "\tclique_id\x18\x14 \x01(\tR\bcliqueId\x12)\n" + - "\x10fabric_bandwidth\x18\x15 \x01(\tR\x0ffabricBandwidth\x1a;\n" + + "\x10fabric_bandwidth\x18\x15 \x01(\tR\x0ffabricBandwidth\x12%\n" + + "\x03mig\x18\x16 \x01(\v2\x13.llmgw.v1.MIGDeviceR\x03mig\x1a;\n" + "\rNicPathsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"[\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x84\x02\n" + + "\tMIGDevice\x12!\n" + + "\fparent_index\x18\x01 \x01(\x05R\vparentIndex\x12\x1f\n" + + "\vparent_uuid\x18\x02 \x01(\tR\n" + + "parentUuid\x123\n" + + "\x16parent_memory_total_mb\x18\x03 \x01(\x05R\x13parentMemoryTotalMb\x12\x16\n" + + "\x06device\x18\x04 \x01(\x05R\x06device\x12\x18\n" + + "\aprofile\x18\x05 \x01(\tR\aprofile\x12!\n" + + "\fgpu_instance\x18\x06 \x01(\x05R\vgpuInstance\x12)\n" + + "\x10compute_instance\x18\a \x01(\x05R\x0fcomputeInstance\"[\n" + "\x10FabricAttachment\x12\x16\n" + "\x06fabric\x18\x01 \x01(\tR\x06fabric\x12\x12\n" + "\x04hcas\x18\x02 \x03(\tR\x04hcas\x12\x1b\n" + @@ -4532,7 +4650,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte { } var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 44) +var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 45) var file_llmgw_v1_agent_proto_goTypes = []any{ (ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State (*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage @@ -4543,99 +4661,101 @@ var file_llmgw_v1_agent_proto_goTypes = []any{ (*Storage)(nil), // 6: llmgw.v1.Storage (*RunningShard)(nil), // 7: llmgw.v1.RunningShard (*GPU)(nil), // 8: llmgw.v1.GPU - (*FabricAttachment)(nil), // 9: llmgw.v1.FabricAttachment - (*LocalWeights)(nil), // 10: llmgw.v1.LocalWeights - (*NodeLoad)(nil), // 11: llmgw.v1.NodeLoad - (*GPUSensor)(nil), // 12: llmgw.v1.GPUSensor - (*ShardLoad)(nil), // 13: llmgw.v1.ShardLoad - (*IBPort)(nil), // 14: llmgw.v1.IBPort - (*SubnetManager)(nil), // 15: llmgw.v1.SubnetManager - (*IMEX)(nil), // 16: llmgw.v1.IMEX - (*GPUFabricState)(nil), // 17: llmgw.v1.GPUFabricState - (*Heartbeat)(nil), // 18: llmgw.v1.Heartbeat - (*Provision)(nil), // 19: llmgw.v1.Provision - (*EngineWant)(nil), // 20: llmgw.v1.EngineWant - (*ProvisionStatus)(nil), // 21: llmgw.v1.ProvisionStatus - (*ShardStatus)(nil), // 22: llmgw.v1.ShardStatus - (*FetchProgress)(nil), // 23: llmgw.v1.FetchProgress - (*StartShard)(nil), // 24: llmgw.v1.StartShard - (*EngineProfile)(nil), // 25: llmgw.v1.EngineProfile - (*RemoveWeights)(nil), // 26: llmgw.v1.RemoveWeights - (*StopShard)(nil), // 27: llmgw.v1.StopShard - (*DrainShard)(nil), // 28: llmgw.v1.DrainShard - (*Ack)(nil), // 29: llmgw.v1.Ack - (*LogsRequest)(nil), // 30: llmgw.v1.LogsRequest - (*LogsReply)(nil), // 31: llmgw.v1.LogsReply - (*FabricScanRequest)(nil), // 32: llmgw.v1.FabricScanRequest - (*FabricScanReply)(nil), // 33: llmgw.v1.FabricScanReply - (*FabricTestRequest)(nil), // 34: llmgw.v1.FabricTestRequest - (*FabricTestReply)(nil), // 35: llmgw.v1.FabricTestReply - (*UpgradeAgent)(nil), // 36: llmgw.v1.UpgradeAgent - (*UpgradeReply)(nil), // 37: llmgw.v1.UpgradeReply - (*LoraAdapter)(nil), // 38: llmgw.v1.LoraAdapter - nil, // 39: llmgw.v1.Register.LabelsEntry - nil, // 40: llmgw.v1.Register.EngineVersionsEntry - nil, // 41: llmgw.v1.GPU.NicPathsEntry - nil, // 42: llmgw.v1.IBPort.CountersEntry - nil, // 43: llmgw.v1.Heartbeat.EngineVersionsEntry - nil, // 44: llmgw.v1.EngineProfile.EnvEntry + (*MIGDevice)(nil), // 9: llmgw.v1.MIGDevice + (*FabricAttachment)(nil), // 10: llmgw.v1.FabricAttachment + (*LocalWeights)(nil), // 11: llmgw.v1.LocalWeights + (*NodeLoad)(nil), // 12: llmgw.v1.NodeLoad + (*GPUSensor)(nil), // 13: llmgw.v1.GPUSensor + (*ShardLoad)(nil), // 14: llmgw.v1.ShardLoad + (*IBPort)(nil), // 15: llmgw.v1.IBPort + (*SubnetManager)(nil), // 16: llmgw.v1.SubnetManager + (*IMEX)(nil), // 17: llmgw.v1.IMEX + (*GPUFabricState)(nil), // 18: llmgw.v1.GPUFabricState + (*Heartbeat)(nil), // 19: llmgw.v1.Heartbeat + (*Provision)(nil), // 20: llmgw.v1.Provision + (*EngineWant)(nil), // 21: llmgw.v1.EngineWant + (*ProvisionStatus)(nil), // 22: llmgw.v1.ProvisionStatus + (*ShardStatus)(nil), // 23: llmgw.v1.ShardStatus + (*FetchProgress)(nil), // 24: llmgw.v1.FetchProgress + (*StartShard)(nil), // 25: llmgw.v1.StartShard + (*EngineProfile)(nil), // 26: llmgw.v1.EngineProfile + (*RemoveWeights)(nil), // 27: llmgw.v1.RemoveWeights + (*StopShard)(nil), // 28: llmgw.v1.StopShard + (*DrainShard)(nil), // 29: llmgw.v1.DrainShard + (*Ack)(nil), // 30: llmgw.v1.Ack + (*LogsRequest)(nil), // 31: llmgw.v1.LogsRequest + (*LogsReply)(nil), // 32: llmgw.v1.LogsReply + (*FabricScanRequest)(nil), // 33: llmgw.v1.FabricScanRequest + (*FabricScanReply)(nil), // 34: llmgw.v1.FabricScanReply + (*FabricTestRequest)(nil), // 35: llmgw.v1.FabricTestRequest + (*FabricTestReply)(nil), // 36: llmgw.v1.FabricTestReply + (*UpgradeAgent)(nil), // 37: llmgw.v1.UpgradeAgent + (*UpgradeReply)(nil), // 38: llmgw.v1.UpgradeReply + (*LoraAdapter)(nil), // 39: llmgw.v1.LoraAdapter + nil, // 40: llmgw.v1.Register.LabelsEntry + nil, // 41: llmgw.v1.Register.EngineVersionsEntry + nil, // 42: llmgw.v1.GPU.NicPathsEntry + nil, // 43: llmgw.v1.IBPort.CountersEntry + nil, // 44: llmgw.v1.Heartbeat.EngineVersionsEntry + nil, // 45: llmgw.v1.EngineProfile.EnvEntry } var file_llmgw_v1_agent_proto_depIdxs = []int32{ 4, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register - 18, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat - 22, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus - 31, // 3: llmgw.v1.AgentMessage.logs:type_name -> llmgw.v1.LogsReply - 33, // 4: llmgw.v1.AgentMessage.fabric_scan:type_name -> llmgw.v1.FabricScanReply - 35, // 5: llmgw.v1.AgentMessage.fabric_test:type_name -> llmgw.v1.FabricTestReply - 37, // 6: llmgw.v1.AgentMessage.upgrade:type_name -> llmgw.v1.UpgradeReply - 24, // 7: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard - 27, // 8: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard - 28, // 9: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard - 29, // 10: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack - 26, // 11: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights - 19, // 12: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision - 30, // 13: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest - 32, // 14: llmgw.v1.ControlMessage.fabric_scan:type_name -> llmgw.v1.FabricScanRequest - 34, // 15: llmgw.v1.ControlMessage.fabric_test:type_name -> llmgw.v1.FabricTestRequest - 36, // 16: llmgw.v1.ControlMessage.upgrade:type_name -> llmgw.v1.UpgradeAgent + 19, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat + 23, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus + 32, // 3: llmgw.v1.AgentMessage.logs:type_name -> llmgw.v1.LogsReply + 34, // 4: llmgw.v1.AgentMessage.fabric_scan:type_name -> llmgw.v1.FabricScanReply + 36, // 5: llmgw.v1.AgentMessage.fabric_test:type_name -> llmgw.v1.FabricTestReply + 38, // 6: llmgw.v1.AgentMessage.upgrade:type_name -> llmgw.v1.UpgradeReply + 25, // 7: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard + 28, // 8: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard + 29, // 9: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard + 30, // 10: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack + 27, // 11: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights + 20, // 12: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision + 31, // 13: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest + 33, // 14: llmgw.v1.ControlMessage.fabric_scan:type_name -> llmgw.v1.FabricScanRequest + 35, // 15: llmgw.v1.ControlMessage.fabric_test:type_name -> llmgw.v1.FabricTestRequest + 37, // 16: llmgw.v1.ControlMessage.upgrade:type_name -> llmgw.v1.UpgradeAgent 3, // 17: llmgw.v1.ControlMessage.control_planes:type_name -> llmgw.v1.ControlPlanes - 39, // 18: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry + 40, // 18: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry 8, // 19: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU - 9, // 20: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment - 40, // 21: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry + 10, // 20: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment + 41, // 21: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry 7, // 22: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard - 10, // 23: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights + 11, // 23: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights 6, // 24: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage 5, // 25: llmgw.v1.Register.host:type_name -> llmgw.v1.Host - 16, // 26: llmgw.v1.Register.imex:type_name -> llmgw.v1.IMEX + 17, // 26: llmgw.v1.Register.imex:type_name -> llmgw.v1.IMEX 0, // 27: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State - 41, // 28: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry - 13, // 29: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad - 12, // 30: llmgw.v1.NodeLoad.gpus:type_name -> llmgw.v1.GPUSensor - 42, // 31: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry - 15, // 32: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager - 6, // 33: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage - 10, // 34: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights - 43, // 35: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry - 21, // 36: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus - 11, // 37: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad - 14, // 38: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort - 9, // 39: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment - 17, // 40: llmgw.v1.Heartbeat.gpu_fabric:type_name -> llmgw.v1.GPUFabricState - 16, // 41: llmgw.v1.Heartbeat.imex:type_name -> llmgw.v1.IMEX - 20, // 42: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant - 0, // 43: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State - 23, // 44: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress - 25, // 45: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile - 44, // 46: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry - 38, // 47: llmgw.v1.EngineProfile.adapters:type_name -> llmgw.v1.LoraAdapter - 1, // 48: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage - 2, // 49: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage - 49, // [49:50] is the sub-list for method output_type - 48, // [48:49] is the sub-list for method input_type - 48, // [48:48] is the sub-list for extension type_name - 48, // [48:48] is the sub-list for extension extendee - 0, // [0:48] is the sub-list for field type_name + 42, // 28: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry + 9, // 29: llmgw.v1.GPU.mig:type_name -> llmgw.v1.MIGDevice + 14, // 30: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad + 13, // 31: llmgw.v1.NodeLoad.gpus:type_name -> llmgw.v1.GPUSensor + 43, // 32: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry + 16, // 33: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager + 6, // 34: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage + 11, // 35: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights + 44, // 36: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry + 22, // 37: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus + 12, // 38: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad + 15, // 39: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort + 10, // 40: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment + 18, // 41: llmgw.v1.Heartbeat.gpu_fabric:type_name -> llmgw.v1.GPUFabricState + 17, // 42: llmgw.v1.Heartbeat.imex:type_name -> llmgw.v1.IMEX + 21, // 43: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant + 0, // 44: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State + 24, // 45: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress + 26, // 46: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile + 45, // 47: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry + 39, // 48: llmgw.v1.EngineProfile.adapters:type_name -> llmgw.v1.LoraAdapter + 1, // 49: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage + 2, // 50: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage + 50, // [50:51] is the sub-list for method output_type + 49, // [49:50] is the sub-list for method input_type + 49, // [49:49] is the sub-list for extension type_name + 49, // [49:49] is the sub-list for extension extendee + 0, // [0:49] is the sub-list for field type_name } func init() { file_llmgw_v1_agent_proto_init() } @@ -4671,7 +4791,7 @@ func file_llmgw_v1_agent_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)), NumEnums: 1, - NumMessages: 44, + NumMessages: 45, NumExtensions: 0, NumServices: 1, }, diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index 98caf135..e72df176 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -252,6 +252,25 @@ message GPU { string cluster_uuid = 19; string clique_id = 20; string fabric_bandwidth = 21; + // mig is set when this is a MIG instance rather than a whole card. index + // is then 1000 + 100 × card + device, uuid the instance's (MIG-…), and + // memory the instance's share; utilization_percent stays zero, because no + // driver measures it per instance. + MIGDevice mig = 22; +} + +// MIGDevice is which card a MIG instance is carved from, and how. +message MIGDevice { + int32 parent_index = 1; + string parent_uuid = 2; + // parent_memory_total_mb is the whole card's: the class is named from it. + int32 parent_memory_total_mb = 3; + // device is the instance's number on its card, as nvidia-smi -L lists it. + int32 device = 4; + // profile is NVIDIA's name for the slice, like 3g.40gb. + string profile = 5; + int32 gpu_instance = 6; + int32 compute_instance = 7; } message FabricAttachment { diff --git a/docs/install.md b/docs/install.md index c5923ce7..ca3c4025 100644 --- a/docs/install.md +++ b/docs/install.md @@ -428,6 +428,71 @@ sudo llmgw doctor A `-nvml` build is still published beside the portable one; it reads NVLink from NVML directly, and needs a libc no older than the builder's. +### MIG, if you want to cut a card into pieces + +An A100, A30, H100, H200 or later card can be split into MIG (Multi-Instance +GPU) instances, each with its own memory, its own share of the card's compute +and its own UUID. llmgw places a model on an instance as it would on a card, +which makes MIG the way to run several small models on one large card without +them competing for it. + +**llmgw does not create the instances.** How a card is cut is your decision, +made once with `nvidia-smi` on the node; the agent reads what is there when +it starts: + +``` +sudo nvidia-smi -i 0 -mig 1 # MIG mode on card 0; may need a reset +sudo nvidia-smi mig -i 0 -lgip # the profiles this card offers +sudo nvidia-smi mig -i 0 -cgi 3g.40gb,3g.40gb -C # two instances, each with its compute instance +nvidia-smi -L # the instances, with their MIG- UUIDs +sudo systemctl restart llmgw-agent # so it registers the new devices +``` + +`-C` matters: a GPU instance with no compute instance inside it is not a +device yet. Turning MIG on takes effect at the card's next reset +(`nvidia-smi -i 0 -r` with nothing running on it, or a reboot), and instances +do not survive a reboot unless something recreates them — NVIDIA's +`mig-parted` or a systemd unit of your own. + +**Classes.** A card in MIG mode is reported as its instances instead of as +the whole card, and each instance's class is the card's class and the +profile: two `3g.40gb` instances of an A100 80GB are two devices of class +`a100-80g-mig-3g.40gb`, an H100's `1g.10gb+me` is `h100-80g-mig-1g.10gb-me`. +`llmgw ctl nodes` lists the classes, and a profile names one like any other: + +```yaml +profiles: + - name: mig + engine: vllm + gpu_class: a100-80g-mig-3g.40gb +``` + +**One instance a replica.** A process can open only one MIG device, and +instances have no NVLink or peer-to-peer access between them, so a replica on +MIG runs on exactly one instance: `tp: 1`, one GPU, one node. A profile on a +MIG class asking for more is refused when the model is added, with that +reason. A model too large for one instance wants a larger profile or a whole +card. `llmgw ctl model plan` and `ctl model serve` plan the same way: an +instance is offered for a model it holds, and never several together. + +**How they are numbered and shown.** Placement numbers card 0's instance 2 as +device 1002 (1000 + 100 × card + instance), and every view shows it as +`0/mig2`. The engine is given the instance by its UUID — +`CUDA_VISIBLE_DEVICES=MIG-…` — which works for vLLM and for llama.cpp's CUDA +build. llama.cpp's Vulkan build cannot see MIG instances at all. + +**What is measured.** Each instance's memory is its own; temperature, power +and wear are its card's, shown on each of its instances. Utilisation is not +reported for an instance, because no driver measures it per instance: the +console says `n/a on MIG`, and no `llmgw_gpu_utilization_percent` series is +published for it rather than a zero. + +`sudo llmgw doctor` lists each card's MIG mode and instances, and warns of a +card in MIG mode with none created (nothing can run on it: CUDA will not open +the whole card while MIG is on), of GPU instances with no compute instance, +and of a mode change waiting for a reset. Such a card is also kept out of +placement, with the reason, in `llmgw ctl nodes`. + ### InfiniBand, if you have it ``` diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 4d6008bc..f6144632 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -1202,18 +1202,20 @@ const ( // // The interface NCCL bootstraps over is this node's, not the cluster's: the // control plane sends one name for every node, and a node whose management -// network is on ens3 rather than eth0 would have NCCL bind nothing. And on -// RoCE, which GID entry routes is a property of these adapters. +// network is on ens3 rather than eth0 would have NCCL bind nothing. On RoCE, +// which GID entry routes is a property of these adapters. And a MIG instance +// is opened by its UUID, which only this node knows. func (a *Agent) localizeSpec(spec *engines.ShardSpec) { if a.cfg.MgmtInterface != "" { spec.MgmtInterface = a.cfg.MgmtInterface } + a.mu.Lock() + hcas, gpus := a.inv.HCAs, a.inv.GPUs + a.mu.Unlock() + spec.GPUUUIDs = migUUIDs(gpus, spec.GPUIndices) if len(spec.HCAs) == 0 { return } - a.mu.Lock() - hcas := a.inv.HCAs - a.mu.Unlock() want := map[string]bool{} for _, h := range spec.HCAs { want[h] = true @@ -1232,6 +1234,27 @@ func (a *Agent) localizeSpec(spec *engines.ShardSpec) { } } +// migUUIDs names a shard's devices by UUID when one of them is a MIG +// instance, and is nil otherwise: whole cards keep being given by index, as +// they always were. A device this node does not have is left to fail where +// it always has, in the engine, rather than guessed at here. +func migUUIDs(gpus []discovery.GPU, indices []int) []string { + var out []string + mig := false + for _, i := range indices { + k := slices.IndexFunc(gpus, func(g discovery.GPU) bool { return g.Index == i }) + if k < 0 || gpus[k].UUID == "" { + return nil + } + mig = mig || gpus[k].IsMIG() + out = append(out, gpus[k].UUID) + } + if !mig { + return nil + } + return out +} + // ResolveManagement is the address and interface this node's agent would use, // found the way normalize finds them, for a tool that reports them. func ResolveManagement(c Config) (addr, iface string) { diff --git a/internal/agent/discovery/gpu.go b/internal/agent/discovery/gpu.go index 36fa88f6..6495d38c 100644 --- a/internal/agent/discovery/gpu.go +++ b/internal/agent/discovery/gpu.go @@ -47,6 +47,11 @@ type GPU struct { Fabric *GPUFabric `json:"fabric,omitempty"` // Health carries anything that should keep the GPU out of placement. Health GPUHealth `json:"health"` + // MIG is set when this is a MIG instance rather than a whole card: its + // Index is then MIGIndex's, its UUID the instance's (MIG-…), and its + // memory the instance's share. Utilisation stays unreported, because no + // driver measures it per instance. + MIG *MIGDevice `json:"mig,omitempty"` } // GPUHealth records conditions that make a GPU unsafe to schedule onto. @@ -94,8 +99,30 @@ func (g GPU) Class() string { } // Round to the nearest GiB: vendors report 81559 MiB for an "80GB" card. - gb := int(math.Round(float64(g.MemoryTotalMB) / 1024)) - return fmt.Sprintf("%s-%dg", model, gb) + // A MIG instance is named from its card's memory and then its profile, + // so two instances are one class only when both the card and the slice + // are the same. + mem := g.MemoryTotalMB + if g.MIG != nil { + mem = g.MIG.ParentMemoryTotalMB + } + gb := int(math.Round(float64(mem) / 1024)) + class := fmt.Sprintf("%s-%dg", model, gb) + if g.MIG != nil { + return migClass(class, g.MIG.Profile) + } + return class +} + +// normalizeBusID makes bus ids comparable: NVML reports them uppercase from one +// call and lowercase from another, and sometimes with a shortened domain. +func normalizeBusID(s string) string { + s = strings.ToLower(strings.TrimSpace(s)) + // "0000:1b:00.0" and "00000000:1B:00.0" are the same device. + if i := strings.Index(s, ":"); i > 0 { + s = strings.TrimLeft(s[:i], "0") + s[i:] + } + return strings.TrimPrefix(s, ":") } // MarshalJSON adds the derived class, so a consumer of the JSON does not have diff --git a/internal/agent/discovery/gpu_nvml.go b/internal/agent/discovery/gpu_nvml.go index f88c9345..aa52220e 100644 --- a/internal/agent/discovery/gpu_nvml.go +++ b/internal/agent/discovery/gpu_nvml.go @@ -14,7 +14,6 @@ package discovery import ( "context" "fmt" - "strings" "github.com/NVIDIA/go-nvml/pkg/nvml" ) @@ -83,11 +82,18 @@ func DiscoverGPUsNVML(_ context.Context) ([]GPU, error) { } assessHealth(&g) - gpus = append(gpus, g) + // A card in MIG mode is its instances, not itself: CUDA will not + // open the whole card while MIG is on. + devs, _ := migFromNVML(g, nvmlCard{dev}) + gpus = append(gpus, devs...) } - // A second pass, because a link's remote end may be a GPU enumerated later. + // A second pass, because a link's remote end may be a GPU enumerated + // later. MIG instances have no NVLink of their own to use. for i := range gpus { + if gpus[i].IsMIG() { + continue + } dev, ret := nvml.DeviceGetHandleByIndex(gpus[i].Index) if ret != nvml.SUCCESS { continue @@ -136,13 +142,73 @@ func busIDString(b []int8) string { return string(buf) } -// normalizeBusID makes bus ids comparable: NVML reports them uppercase from one -// call and lowercase from another, and sometimes with a shortened domain. -func normalizeBusID(s string) string { - s = strings.ToLower(strings.TrimSpace(s)) - // "0000:1b:00.0" and "00000000:1B:00.0" are the same device. - if i := strings.Index(s, ":"); i > 0 { - s = strings.TrimLeft(s[:i], "0") + s[i:] +// nvmlCard adapts an NVML device to what MIG discovery reads of it. +type nvmlCard struct{ dev nvml.Device } + +func (c nvmlCard) MIGMode() (current, pending, ok bool) { + cur, pend, ret := c.dev.GetMigMode() + if ret != nvml.SUCCESS { + // NOT_SUPPORTED on a card without MIG, which is most of them. + return false, false, false } - return strings.TrimPrefix(s, ":") + return cur == nvml.DEVICE_MIG_ENABLE, pend == nvml.DEVICE_MIG_ENABLE, true +} + +func (c nvmlCard) MaxMIGDevices() int { + n, ret := c.dev.GetMaxMigDeviceCount() + if ret != nvml.SUCCESS { + return 0 + } + return n +} + +func (c nvmlCard) MIGDevice(i int) (nvmlMIGInstance, bool) { + h, ret := c.dev.GetMigDeviceHandleByIndex(i) + if ret != nvml.SUCCESS { + // NOT_FOUND for an index with no instance: the count is the most a + // card can hold, not how many it has. + return nil, false + } + return nvmlMIG{h}, true +} + +// nvmlMIG adapts a MIG device handle. +type nvmlMIG struct{ dev nvml.Device } + +func (m nvmlMIG) UUID() string { + u, _ := m.dev.GetUUID() + return u +} + +func (m nvmlMIG) Name() string { + n, _ := m.dev.GetName() + return n +} + +func (m nvmlMIG) Memory() (totalMB, usedMB int, ok bool) { + mem, ret := m.dev.GetMemoryInfo() + if ret != nvml.SUCCESS { + return 0, 0, false + } + return int(mem.Total / (1 << 20)), int(mem.Used / (1 << 20)), true +} + +func (m nvmlMIG) Instances() (gpuInstance, computeInstance int) { + gi, ret := m.dev.GetGpuInstanceId() + if ret != nvml.SUCCESS { + gi = -1 + } + ci, ret := m.dev.GetComputeInstanceId() + if ret != nvml.SUCCESS { + ci = -1 + } + return gi, ci +} + +func (m nvmlMIG) Attributes() (slices, memoryMB int, ok bool) { + a, ret := m.dev.GetAttributes() + if ret != nvml.SUCCESS { + return 0, 0, false + } + return int(a.GpuInstanceSliceCount), int(a.MemorySizeMB), true } diff --git a/internal/agent/discovery/gpu_smi.go b/internal/agent/discovery/gpu_smi.go index b3d165ba..44c94905 100644 --- a/internal/agent/discovery/gpu_smi.go +++ b/internal/agent/discovery/gpu_smi.go @@ -60,7 +60,33 @@ func DiscoverGPUsSMI(ctx context.Context, binary string) ([]GPU, error) { if err != nil { return nil, fmt.Errorf("running %s: %w", binary, err) } - return parseSMI(string(out)) + cards, err := parseSMI(string(out)) + if err != nil || len(cards) == 0 { + return cards, err + } + // The query lists whole cards, MIG mode or not. A card in MIG mode is + // replaced by its instances, which only -L names and only the full + // report sizes; asked only when some card could be in MIG mode, since + // the report is the slowest thing nvidia-smi does. + if !anyMIGCapable(cards) { + return cards, nil + } + listed, report := readMIG(ctx, binary) + return expandMIG(cards, listed, report), nil +} + +// anyMIGCapable reports whether any card could be in MIG mode: MIG began +// with compute capability 8.0 (the A100). The consumer cards of the same +// generations have none, but telling them apart by name is guesswork, and +// asking one costs only two more nvidia-smi runs at registration. +func anyMIGCapable(cards []GPU) bool { + for _, c := range cards { + major, _, _ := strings.Cut(c.ComputeCapability, ".") + if n, err := strconv.Atoi(major); err == nil && n >= 8 { + return true + } + } + return false } func parseSMI(out string) ([]GPU, error) { diff --git a/internal/agent/discovery/gpuprocs.go b/internal/agent/discovery/gpuprocs.go index 3c455103..4c1c9d11 100644 --- a/internal/agent/discovery/gpuprocs.go +++ b/internal/agent/discovery/gpuprocs.go @@ -5,6 +5,7 @@ import ( "encoding/xml" "fmt" "os/exec" + "slices" "strconv" "strings" ) @@ -34,9 +35,16 @@ func GPUProcesses(ctx context.Context, binary string) (map[int][]int, error) { return parseGPUProcesses(out) } -// GPUMemory is one card's memory as nvidia-smi reports it at a moment: what +// GPUMemory is one device's memory as nvidia-smi reports it at a moment: what // is allocated on it, and which processes hold how much of that. +// +// A device is what discovery reports: a whole card, or each MIG instance of a +// card in MIG mode, in the same order, so the readings line up with the +// inventory one for one. type GPUMemory struct { + // Index is the device's number as placement knows it: the card's + // nvidia-smi index, or MIGIndex's for an instance. + Index int UsedMB int Procs []GPUProcess } @@ -66,48 +74,62 @@ func GPUMemoryByProcess(ctx context.Context, binary string) ([]GPUMemory, error) return parseGPUMemory(out) } -// smiLog is the part of nvidia-smi's XML report this reads: each GPU, in -// index order, its memory and the processes on it. -type smiLog struct { - GPUs []struct { - Used string `xml:"fb_memory_usage>used"` - Processes []struct { - PID string `xml:"pid"` - Type string `xml:"type"` - UsedMB string `xml:"used_memory"` - } `xml:"processes>process_info"` - } `xml:"gpu"` -} - -// parseGPUProcesses reads the report. GPUs are listed in index order, which -// is nvidia-smi's own numbering — the one placement assigns by. +// parseGPUProcesses reads the report, by device as placement numbers them: +// nvidia-smi's index for a whole card, MIGIndex's for a MIG instance. func parseGPUProcesses(report []byte) (map[int][]int, error) { mem, err := parseGPUMemory(report) if err != nil { return nil, err } out := map[int][]int{} - for i, g := range mem { + for _, g := range mem { for _, p := range g.Procs { - out[i] = append(out[i], p.PID) + out[g.Index] = append(out[g.Index], p.PID) } } return out, nil } -// parseGPUMemory reads the report into each GPU's memory and its holders. +// parseGPUMemory reads the report into each device's memory and its holders. +// +// Cards are listed in index order, which is nvidia-smi's own numbering. A +// card in MIG mode is read as its instances, each with its own memory and +// the processes nvidia-smi attributes to its GPU and compute instance: a +// process on one instance is not on its neighbours, and the card's total +// would have charged every instance with all of them. func parseGPUMemory(report []byte) ([]GPUMemory, error) { - var log smiLog + var log migLog if err := xml.Unmarshal(report, &log); err != nil { return nil, fmt.Errorf("reading nvidia-smi's report: %w", err) } - out := make([]GPUMemory, len(log.GPUs)) + out := make([]GPUMemory, 0, len(log.GPUs)) for i, g := range log.GPUs { - out[i].UsedMB = mib(g.Used) - for _, p := range g.Processes { - if pid, err := strconv.Atoi(strings.TrimSpace(p.PID)); err == nil { - out[i].Procs = append(out[i].Procs, GPUProcess{PID: pid, UsedMB: mib(p.UsedMB)}) + procs := func(gi, ci int) []GPUProcess { + var ps []GPUProcess + for _, p := range g.Processes { + pid, err := strconv.Atoi(strings.TrimSpace(p.PID)) + if err != nil { + continue + } + if gi >= 0 && (atoiOr(p.GPUInstance, -1) != gi || atoiOr(p.ComputeInstance, -1) != ci) { + continue + } + ps = append(ps, GPUProcess{PID: pid, UsedMB: mib(p.UsedMB)}) } + return ps + } + if !enabled(g.Current) || len(g.Devices) == 0 { + out = append(out, GPUMemory{Index: i, UsedMB: mib(g.Used), Procs: procs(-1, -1)}) + continue + } + devs := slices.Clone(g.Devices) + slices.SortFunc(devs, func(a, b migLogDevice) int { return atoiOr(a.Index, -1) - atoiOr(b.Index, -1) }) + for _, d := range devs { + out = append(out, GPUMemory{ + Index: MIGIndex(i, atoiOr(d.Index, 0)), + UsedMB: mib(d.Used), + Procs: procs(atoiOr(d.GPUInstance, -1), atoiOr(d.ComputeInstance, -1)), + }) } } return out, nil diff --git a/internal/agent/discovery/gpuprocs_test.go b/internal/agent/discovery/gpuprocs_test.go index 6472fba4..f3fe7029 100644 --- a/internal/agent/discovery/gpuprocs_test.go +++ b/internal/agent/discovery/gpuprocs_test.go @@ -57,8 +57,8 @@ func TestGPUMemoryIsReadPerProcess(t *testing.T) { t.Fatal(err) } want := []GPUMemory{ - {UsedMB: 14514, Procs: []GPUProcess{{PID: 71315, UsedMB: 4932}, {PID: 4100, UsedMB: 9500}}}, - {}, + {Index: 0, UsedMB: 14514, Procs: []GPUProcess{{PID: 71315, UsedMB: 4932}, {PID: 4100, UsedMB: 9500}}}, + {Index: 1}, } if !reflect.DeepEqual(got, want) { t.Errorf("got %+v, want %+v", got, want) diff --git a/internal/agent/discovery/mig.go b/internal/agent/discovery/mig.go new file mode 100644 index 00000000..618b2741 --- /dev/null +++ b/internal/agent/discovery/mig.go @@ -0,0 +1,560 @@ +package discovery + +// MIG — Multi-Instance GPU — carves one A100, H100 or later card into up to +// seven instances, each with its own memory, its own share of the +// multiprocessors and its own UUID. To CUDA an instance is a device of its +// own, and to placement it is one too: a card in MIG mode is reported as its +// instances instead of as the whole card, each a GPU of a class named from +// the card's and the instance's profile ("a100-80g-mig-3g.40gb"). +// +// llmgw does not create instances. Which split a card gets is an operator's +// decision, made with nvidia-smi mig; discovery reads what is there. +// +// Two facts about MIG shape everything below. A process is given one MIG +// device: CUDA enumerates a single instance however many are made visible. +// And instances have no NVLink or peer-to-peer access between them. A replica +// on MIG therefore runs on exactly one instance, never a tensor-parallel group +// of them, and the catalog refuses a profile that asks for more. + +import ( + "context" + "encoding/xml" + "fmt" + "math" + "os/exec" + "regexp" + "slices" + "strconv" + "strings" +) + +// MIGDevice is what a MIG instance carries beyond any GPU's fields: which +// card it is carved from, and how. +type MIGDevice struct { + // ParentIndex is the card's nvidia-smi index, which is what its sensors, + // topology and wear are reported by. + ParentIndex int `json:"parent_index"` + ParentUUID string `json:"parent_uuid,omitempty"` + // ParentMemoryTotalMB is the whole card's memory: the class is named from + // the card, so an instance of an 80 GB card is not mistaken for one of a + // 40 GB card that happens to have the same profile name. + ParentMemoryTotalMB int `json:"parent_memory_total_mb"` + // Device is the instance's number on its card, as nvidia-smi -L lists it. + Device int `json:"device"` + // Profile is NVIDIA's name for the slice, like "3g.40gb": three of the + // card's seven compute slices and forty gigabytes of its memory. + Profile string `json:"profile"` + // GPUInstance and ComputeInstance are the instance's ids in nvidia-smi + // mig -lgi and -lci, which is how nvidia-smi attributes processes to it. + GPUInstance int `json:"gpu_instance"` + ComputeInstance int `json:"compute_instance"` +} + +// migIndexBase starts the numbers MIG instances are given. +// +// Placement, the agent protocol and every running shard name a node's devices +// by an integer, which for a whole card is its nvidia-smi index. An instance +// has none of its own, so it is given 1000 + 100 × card + device: card 0's +// instance 2 is 1002, card 3's instance 0 is 1300. Readable, stable while the +// instances are, and never one a whole card could have. +const migIndexBase = 1000 + +// MIGIndex is the device number placement knows a MIG instance by. +func MIGIndex(card, device int) int { return migIndexBase + 100*card + device } + +// IsMIGIndex reports whether a device number names a MIG instance. +func IsMIGIndex(i int) bool { return i >= migIndexBase } + +// DeviceLabel is how a device number is shown to people: "3" for a whole +// card, "0/mig2" for card 0's MIG instance 2. +func DeviceLabel(i int) string { + if !IsMIGIndex(i) { + return strconv.Itoa(i) + } + i -= migIndexBase + return fmt.Sprintf("%d/mig%d", i/100, i%100) +} + +// DeviceLabels is DeviceLabel for each of a list. +func DeviceLabels(is []int) []string { + out := make([]string, len(is)) + for k, i := range is { + out[k] = DeviceLabel(i) + } + return out +} + +// IsMIG reports whether this GPU is a MIG instance rather than a whole card. +func (g GPU) IsMIG() bool { return g.MIG != nil } + +// CardIndex is the nvidia-smi index of the physical card this GPU is on: its +// own index for a whole card, its parent's for a MIG instance. +func (g GPU) CardIndex() int { + if g.MIG != nil { + return g.MIG.ParentIndex + } + return g.Index +} + +// migClassMarker joins a card's class to an instance's profile in a MIG +// class: "a100-80g" + "-mig-" + "3g.40gb". +const migClassMarker = "-mig-" + +// IsMIGClass reports whether a GPU class names MIG instances. +func IsMIGClass(class string) bool { return strings.Contains(class, migClassMarker) } + +// migProfileClean keeps a profile name to what a class may hold: "1g.10gb+me" +// becomes "1g.10gb-me". +var migProfileClean = regexp.MustCompile(`[^a-z0-9.]+`) + +// migClass is the class of a MIG instance: the card's class and the profile. +func migClass(parentClass, profile string) string { + p := migProfileClean.ReplaceAllString(strings.ToLower(strings.TrimSpace(profile)), "-") + return parentClass + migClassMarker + strings.Trim(p, "-") +} + +// MIGProfileMemoryGB reads the memory out of a MIG class or profile: +// 40 from "a100-80g-mig-3g.40gb" or "3g.40gb". Zero when there is none. +func MIGProfileMemoryGB(s string) int { + if i := strings.LastIndex(s, migClassMarker); i >= 0 { + s = s[i+len(migClassMarker):] + } + m := migProfileMemory.FindStringSubmatch(strings.ToLower(s)) + if m == nil { + return 0 + } + gb, _ := strconv.Atoi(m[1]) + return gb +} + +var migProfileMemory = regexp.MustCompile(`^\d+g\.(\d+)gb`) + +// --- nvidia-smi -L --- + +// listedMIG is one MIG instance as nvidia-smi -L lists it. +type listedMIG struct { + Device int + Profile string + UUID string +} + +var ( + // "GPU 0: NVIDIA A100-SXM4-80GB (UUID: GPU-5d5ba0d6-...)" + smiListGPU = regexp.MustCompile(`^GPU (\d+):`) + // " MIG 3g.40gb Device 0: (UUID: MIG-c6d4f1ef-...)". Drivers before + // R470 name the instance MIG-GPU-//, which CUDA of that + // era takes in the same place, so the UUID is read as whatever it is. + smiListMIG = regexp.MustCompile(`^\s+MIG\s+(\S+)\s+Device\s+(\d+):\s*\(UUID:\s*([^)\s]+)\)`) +) + +// parseSMIList reads nvidia-smi -L into each card's MIG instances, by the +// card's index, in device order. A card with none is absent. +func parseSMIList(out string) map[int][]listedMIG { + res := map[int][]listedMIG{} + card := -1 + for _, line := range strings.Split(out, "\n") { + if m := smiListGPU.FindStringSubmatch(line); m != nil { + card, _ = strconv.Atoi(m[1]) + continue + } + m := smiListMIG.FindStringSubmatch(line) + if m == nil || card < 0 { + continue + } + dev, _ := strconv.Atoi(m[2]) + res[card] = append(res[card], listedMIG{Device: dev, Profile: m[1], UUID: m[3]}) + } + for c := range res { + slices.SortFunc(res[c], func(a, b listedMIG) int { return a.Device - b.Device }) + } + return res +} + +// --- nvidia-smi -q -x --- + +// migLog is the part of nvidia-smi's XML report MIG is read from: each card's +// mode, and each instance's ids and memory. +type migLog struct { + GPUs []migLogGPU `xml:"gpu"` +} + +type migLogGPU struct { + ID string `xml:"id,attr"` + Current string `xml:"mig_mode>current_mig"` + Pending string `xml:"mig_mode>pending_mig"` + Devices []migLogDevice `xml:"mig_devices>mig_device"` + // Processes carry the GPU and compute instance they run in on a card + // in MIG mode, and "N/A" for both otherwise. + Processes []struct { + GPUInstance string `xml:"gpu_instance_id"` + ComputeInstance string `xml:"compute_instance_id"` + PID string `xml:"pid"` + UsedMB string `xml:"used_memory"` + } `xml:"processes>process_info"` + Used string `xml:"fb_memory_usage>used"` +} + +type migLogDevice struct { + Index string `xml:"index"` + GPUInstance string `xml:"gpu_instance_id"` + ComputeInstance string `xml:"compute_instance_id"` + Total string `xml:"fb_memory_usage>total"` + Used string `xml:"fb_memory_usage>used"` +} + +// migState is one card's MIG mode and instances from the report. +type migState struct { + BusID string + Enabled bool + PendingEnabled bool + Devices []migReported +} + +// migReported is one instance as the report describes it. +type migReported struct { + Device, GPUInstance, ComputeInstance int + TotalMB, UsedMB int +} + +// enabled reads nvidia-smi's mode words: "Enabled", "Disabled", "N/A". +func enabled(s string) bool { return strings.EqualFold(strings.TrimSpace(s), "Enabled") } + +func atoiOr(s string, def int) int { + n, err := strconv.Atoi(strings.TrimSpace(s)) + if err != nil { + return def + } + return n +} + +// parseMIGReport reads each card's MIG state from the XML report, in the +// report's order, which is nvidia-smi's index order. +func parseMIGReport(report []byte) ([]migState, error) { + var log migLog + if err := xml.Unmarshal(report, &log); err != nil { + return nil, fmt.Errorf("reading nvidia-smi's report: %w", err) + } + out := make([]migState, len(log.GPUs)) + for i, g := range log.GPUs { + out[i] = migState{BusID: g.ID, Enabled: enabled(g.Current), PendingEnabled: enabled(g.Pending)} + for _, d := range g.Devices { + out[i].Devices = append(out[i].Devices, migReported{ + Device: atoiOr(d.Index, -1), GPUInstance: atoiOr(d.GPUInstance, -1), + ComputeInstance: atoiOr(d.ComputeInstance, -1), + TotalMB: mib(d.Total), UsedMB: mib(d.Used), + }) + } + slices.SortFunc(out[i].Devices, func(a, b migReported) int { return a.Device - b.Device }) + } + return out, nil +} + +// --- putting it together --- + +// migProblemNoInstances keeps a card in MIG mode with nothing carved from it +// out of placement: CUDA cannot use the whole card while MIG is on, and +// there is no instance to use instead. +const migProblemNoInstances = "MIG is enabled with no instances created: nothing can run on it " + + "until instances are created (nvidia-smi mig -cgi … -C) or MIG is turned off" + +// migProblemUnnamed is a card whose instances exist but could not be named: +// an engine is given an instance by its UUID, and without one it cannot be. +const migProblemUnnamed = "MIG is enabled and has instances, but nvidia-smi -L did not name them, " + + "so none can be given to an engine" + +// expandMIG replaces each card in MIG mode with its instances. +// +// listed is nvidia-smi -L's instances by card, which names them; report is +// the XML report's state per card, which gives their memory and ids. Either +// may be missing: -L alone still says which instances exist, and the report +// alone still says a card is in MIG mode, so a card it says has instances +// that -L did not name is kept as the whole card with the reason, rather +// than placed as a card CUDA will not open. +func expandMIG(cards []GPU, listed map[int][]listedMIG, report []migState) []GPU { + if len(listed) == 0 && !slices.ContainsFunc(report, func(s migState) bool { return s.Enabled }) { + return cards + } + byBus := map[string]migState{} + for _, s := range report { + byBus[normalizeBusID(s.BusID)] = s + } + out := make([]GPU, 0, len(cards)) + for pos, c := range cards { + st, ok := byBus[normalizeBusID(c.PCIBusID)] + if !ok && pos < len(report) { + st = report[pos] + } + devs := listed[c.Index] + if !st.Enabled && len(devs) == 0 { + out = append(out, c) + continue + } + if len(devs) == 0 { + p := migProblemNoInstances + if len(st.Devices) > 0 { + p = migProblemUnnamed + } + c.Health.Problems = append(c.Health.Problems, p) + out = append(out, c) + continue + } + for _, d := range devs { + var rep migReported + found := false + for _, r := range st.Devices { + if r.Device == d.Device { + rep, found = r, true + } + } + mig := migInstance(c, d.Device, d.Profile, d.UUID) + if found { + mig.MIG.GPUInstance, mig.MIG.ComputeInstance = rep.GPUInstance, rep.ComputeInstance + mig.MemoryTotalMB, mig.MemoryUsedMB = rep.TotalMB, rep.UsedMB + } + if mig.MemoryTotalMB == 0 { + // The profile says it to the gigabyte, which is close enough + // to place by when the report could not be read. + mig.MemoryTotalMB = MIGProfileMemoryGB(d.Profile) * 1024 + } + assessHealth(&mig) + out = append(out, mig) + } + } + return out +} + +// migInstance builds a MIG instance of card: everything about the card that +// is shared — driver, compute capability, NUMA node, bus, temperature and +// ECC errors — and nothing that is not. Not NVLink, which an instance cannot +// use; not utilisation, which no driver reports per instance; and not the +// card's memory or problems, which the instance's own replace. +func migInstance(card GPU, device int, profile, uuid string) GPU { + g := GPU{ + Index: MIGIndex(card.Index, device), UUID: uuid, + Name: strings.TrimSpace(card.Name + " MIG " + profile), + ComputeCapability: card.ComputeCapability, NUMANode: card.NUMANode, + PCIBusID: card.PCIBusID, DriverVersion: card.DriverVersion, CUDAVersion: card.CUDAVersion, + NICPaths: card.NICPaths, Fabric: card.Fabric, + Health: GPUHealth{ECCErrors: card.Health.ECCErrors, TemperatureC: card.Health.TemperatureC}, + MIG: &MIGDevice{ + ParentIndex: card.Index, ParentUUID: card.UUID, ParentMemoryTotalMB: card.MemoryTotalMB, + Device: device, Profile: profile, GPUInstance: -1, ComputeInstance: -1, + }, + } + return g +} + +// readMIG asks nvidia-smi for what expandMIG needs. A failure of either +// command leaves its half empty, which expandMIG is written to take. +func readMIG(ctx context.Context, binary string) (map[int][]listedMIG, []migState) { + var listed map[int][]listedMIG + if out, err := exec.CommandContext(ctx, binary, "-L").Output(); err == nil { + listed = parseSMIList(string(out)) + } + var report []migState + if out, err := exec.CommandContext(ctx, binary, "-q", "-x").Output(); err == nil { + report, _ = parseMIGReport(out) + } + return listed, report +} + +// --- NVML --- + +// nvmlMIGCard is the part of an NVML device MIG discovery reads, behind an +// interface so the logic is tested without a driver: gpu_nvml.go adapts +// nvml.Device to it. +type nvmlMIGCard interface { + // MIGMode is nvmlDeviceGetMigMode; ok is false on a card without MIG. + MIGMode() (current, pending, ok bool) + // MaxMIGDevices is nvmlDeviceGetMaxMigDeviceCount. + MaxMIGDevices() int + // MIGDevice is nvmlDeviceGetMigDeviceHandleByIndex; ok is false for an + // index with no instance behind it, which is how NVML says so. + MIGDevice(i int) (nvmlMIGInstance, bool) +} + +// nvmlMIGInstance is one MIG device handle. +type nvmlMIGInstance interface { + UUID() string + // Name is "NVIDIA A100-SXM4-80GB MIG 3g.40gb" on the drivers that name + // MIG devices, which is where the profile is read from. + Name() string + Memory() (totalMB, usedMB int, ok bool) + Instances() (gpuInstance, computeInstance int) + // Slices and MemoryMB are the instance's attributes, which name the + // profile on a driver whose device name does not. + Attributes() (slices, memoryMB int, ok bool) +} + +var nvmlMIGName = regexp.MustCompile(`MIG\s+(\S+)\s*$`) + +// migFromNVML is what one card is to placement under NVML: its instances if +// it is in MIG mode, itself if it is not. isMIG says which. +func migFromNVML(card GPU, dev nvmlMIGCard) (out []GPU, isMIG bool) { + current, _, ok := dev.MIGMode() + if !ok || !current { + return []GPU{card}, false + } + n := dev.MaxMIGDevices() + for i := range n { + h, ok := dev.MIGDevice(i) + if !ok { + continue + } + profile := "" + if m := nvmlMIGName.FindStringSubmatch(h.Name()); m != nil { + profile = m[1] + } else if n, mem, ok := h.Attributes(); ok { + // The convention NVIDIA's names follow: compute slices, then the + // memory in whole gigabytes. + profile = fmt.Sprintf("%dg.%dgb", n, int(math.Round(float64(mem)/1024))) + } + g := migInstance(card, i, profile, h.UUID()) + g.MIG.GPUInstance, g.MIG.ComputeInstance = h.Instances() + if total, used, ok := h.Memory(); ok { + g.MemoryTotalMB, g.MemoryUsedMB = total, used + } else { + g.MemoryTotalMB = MIGProfileMemoryGB(profile) * 1024 + } + assessHealth(&g) + out = append(out, g) + } + if len(out) == 0 { + card.Health.Problems = append(card.Health.Problems, migProblemNoInstances) + return []GPU{card}, true + } + return out, true +} + +// --- what the doctor reports --- + +// MIGCard is one card's MIG state, as `llmgw doctor` reports it. +type MIGCard struct { + Index int `json:"index"` + Name string `json:"name,omitempty"` + // Enabled is the mode in force; PendingEnabled the mode after the next + // reset of the card, which differs from it until the card is reset. + Enabled bool `json:"enabled"` + PendingEnabled bool `json:"pending_enabled"` + // Instances are the MIG devices an engine can be given, as nvidia-smi -L + // lists them. + Instances []MIGListed `json:"instances,omitempty"` + // GPUInstances and ComputeInstances are nvidia-smi mig -lgi and -lci: a + // GPU instance with no compute instance in it is not a device yet. + GPUInstances []MIGInstance `json:"gpu_instances,omitempty"` + ComputeInstances []MIGInstance `json:"compute_instances,omitempty"` +} + +// MIGListed is one instance an engine can be given. +type MIGListed struct { + Device int `json:"device"` + Profile string `json:"profile"` + UUID string `json:"uuid"` +} + +// MIGInstance is one row of nvidia-smi mig -lgi or -lci. +type MIGInstance struct { + GPU int `json:"gpu"` + Profile string `json:"profile"` + // ProfileID is NVIDIA's number for the profile. + ProfileID int `json:"profile_id"` + // GPUInstance is the GPU instance's id; ComputeInstance the compute + // instance's within it, -1 in a -lgi row. + GPUInstance int `json:"gpu_instance"` + ComputeInstance int `json:"compute_instance"` + Placement string `json:"placement,omitempty"` +} + +var ( + // -lgi: "| 0 MIG 3g.40gb 9 2 4:4 |" + migLGIRow = regexp.MustCompile(`^\|\s*(\d+)\s+MIG\s+(\S+)\s+(\d+)\s+(\d+)\s+(\d+:\d+)\s*\|`) + // -lci: "| 0 1 MIG 3g.40gb 2 0 0:3 |" + migLCIRow = regexp.MustCompile(`^\|\s*(\d+)\s+(\d+)\s+MIG\s+(\S+)\s+(\d+)\s+(\d+)\s+(\d+:\d+)\s*\|`) +) + +// ParseMIGInstances reads nvidia-smi mig -lgi or -lci, whichever it is +// given: the two tables differ by the GPU instance column -lci adds. "No GPU +// instances found" and the like are no rows, not an error. +func ParseMIGInstances(out string) []MIGInstance { + var res []MIGInstance + for _, line := range strings.Split(out, "\n") { + line = strings.TrimSpace(line) + if m := migLCIRow.FindStringSubmatch(line); m != nil { + res = append(res, MIGInstance{ + GPU: atoiOr(m[1], -1), GPUInstance: atoiOr(m[2], -1), Profile: m[3], + ProfileID: atoiOr(m[4], -1), ComputeInstance: atoiOr(m[5], -1), Placement: m[6], + }) + continue + } + if m := migLGIRow.FindStringSubmatch(line); m != nil { + res = append(res, MIGInstance{ + GPU: atoiOr(m[1], -1), Profile: m[2], ProfileID: atoiOr(m[3], -1), + GPUInstance: atoiOr(m[4], -1), ComputeInstance: -1, Placement: m[5], + }) + } + } + return res +} + +// DiscoverMIG reads every card's MIG state for the doctor: the mode from the +// XML report, the devices from -L, the instances from nvidia-smi mig. Only +// cards that support MIG are listed, and an error is returned only when the +// report itself cannot be read. +func DiscoverMIG(ctx context.Context, binary string) ([]MIGCard, error) { + out, err := exec.CommandContext(ctx, binary, "-q", "-x").Output() + if err != nil { + return nil, fmt.Errorf("running %s -q -x: %w", binary, err) + } + var list, lgi, lci string + if b, err := exec.CommandContext(ctx, binary, "-L").Output(); err == nil { + list = string(b) + } + // nvidia-smi mig exits non-zero when there is nothing to list, with the + // table or the reason on its output either way. + b, _ := exec.CommandContext(ctx, binary, "mig", "-lgi").CombinedOutput() + lgi = string(b) + b, _ = exec.CommandContext(ctx, binary, "mig", "-lci").CombinedOutput() + lci = string(b) + return buildMIGCards(out, list, lgi, lci) +} + +// buildMIGCards is DiscoverMIG from the commands' output. +func buildMIGCards(report []byte, list, lgi, lci string) ([]MIGCard, error) { + var log struct { + GPUs []struct { + Name string `xml:"product_name"` + Current string `xml:"mig_mode>current_mig"` + Pending string `xml:"mig_mode>pending_mig"` + } `xml:"gpu"` + } + if err := xml.Unmarshal(report, &log); err != nil { + return nil, fmt.Errorf("reading nvidia-smi's report: %w", err) + } + listed := parseSMIList(list) + gis, cis := ParseMIGInstances(lgi), ParseMIGInstances(lci) + var out []MIGCard + for i, g := range log.GPUs { + cur, pend := strings.TrimSpace(g.Current), strings.TrimSpace(g.Pending) + // A card without MIG says N/A, or nothing on a driver that predates + // it; neither is worth a line. + if !enabled(cur) && !enabled(pend) && !strings.EqualFold(cur, "Disabled") { + continue + } + c := MIGCard{Index: i, Name: strings.TrimSpace(g.Name), Enabled: enabled(cur), PendingEnabled: enabled(pend)} + for _, l := range listed[i] { + c.Instances = append(c.Instances, MIGListed(l)) + } + for _, gi := range gis { + if gi.GPU == i && gi.ComputeInstance < 0 { + c.GPUInstances = append(c.GPUInstances, gi) + } + } + for _, ci := range cis { + if ci.GPU == i && ci.ComputeInstance >= 0 { + c.ComputeInstances = append(c.ComputeInstances, ci) + } + } + out = append(out, c) + } + return out, nil +} diff --git a/internal/agent/discovery/mig_test.go b/internal/agent/discovery/mig_test.go new file mode 100644 index 00000000..1cef5220 --- /dev/null +++ b/internal/agent/discovery/mig_test.go @@ -0,0 +1,418 @@ +package discovery + +import ( + "encoding/json" + "os" + "path/filepath" + "reflect" + "slices" + "strings" + "testing" +) + +// Two A100 80GB cards: card 0 in MIG mode, cut into two 3g.40gb instances, +// card 1 whole. What nvidia-smi prints for that machine, by command. None of +// it is from real MIG hardware, which this was written without; the shapes +// follow NVIDIA's documentation and the output of drivers R470 to R550. +const ( + migQuery = `0, GPU-5d5ba0d6-d33d-2b2c-524d-9e3d8d2b8a77, NVIDIA A100-SXM4-80GB, 81920, 0, 8.0, 00000000:07:00.0, 550.54.15, 31, 0, [N/A] +1, GPU-1f2e3d4c-5b6a-7980-a1b2-c3d4e5f60718, NVIDIA A100-SXM4-80GB, 81920, 0, 8.0, 00000000:0F:00.0, 550.54.15, 33, 0, 0 +` + migList = `GPU 0: NVIDIA A100-SXM4-80GB (UUID: GPU-5d5ba0d6-d33d-2b2c-524d-9e3d8d2b8a77) + MIG 3g.40gb Device 0: (UUID: MIG-c6d4f1ef-42e4-5de3-91c7-45d71c87eb3f) + MIG 3g.40gb Device 1: (UUID: MIG-0f7c3a2e-9d1b-5a8e-b3c4-1e2f3a4b5c6d) +GPU 1: NVIDIA A100-SXM4-80GB (UUID: GPU-1f2e3d4c-5b6a-7980-a1b2-c3d4e5f60718) +` + migReport = ` + + 550.54.15 + + NVIDIA A100-SXM4-80GB + EnabledEnabled + + + 1 + 1 + 0 + 42 + 40192 MiB0 MiB21000 MiB19192 MiB + + + 0 + 2 + 0 + 40192 MiB0 MiB37 MiB40155 MiB + + + 81920 MiB21037 MiB + + + 1 + 0 + 4100Cvllm + 20900 MiB + + + + + NVIDIA A100-SXM4-80GB + DisabledDisabled + None + 81920 MiB300 MiB + + + N/A + N/A + 5200C250 MiB + + + +` + migLGI = `+-------------------------------------------------------+ +| GPU instances: | +| GPU Name Profile Instance Placement | +| ID ID Start:Size | +|=======================================================| +| 0 MIG 3g.40gb 9 1 4:4 | ++-------------------------------------------------------+ +| 0 MIG 3g.40gb 9 2 0:4 | ++-------------------------------------------------------+ +` + migLCI = `+--------------------------------------------------------------------+ +| Compute instances: | +| GPU GPU Name Profile Instance Placement | +| Instance ID ID Start:Size | +| ID | +|====================================================================| +| 0 1 MIG 3g.40gb 2 0 0:3 | ++--------------------------------------------------------------------+ +| 0 2 MIG 3g.40gb 2 0 0:3 | ++--------------------------------------------------------------------+ +` +) + +func migMachine(t *testing.T) []GPU { + t.Helper() + cards, err := parseSMI(migQuery) + if err != nil { + t.Fatal(err) + } + report, err := parseMIGReport([]byte(migReport)) + if err != nil { + t.Fatal(err) + } + return expandMIG(cards, parseSMIList(migList), report) +} + +func TestACardInMIGModeIsReportedAsItsInstancesInsteadOfItself(t *testing.T) { + gpus := migMachine(t) + var got []string + for _, g := range gpus { + got = append(got, DeviceLabel(g.Index)+" "+g.Class()) + } + want := []string{"0/mig0 a100-80g-mig-3g.40gb", "0/mig1 a100-80g-mig-3g.40gb", "1 a100-80g"} + if !slices.Equal(got, want) { + t.Fatalf("devices = %q, want %q", got, want) + } + + mig := gpus[1] + if mig.Index != 1001 || mig.UUID != "MIG-0f7c3a2e-9d1b-5a8e-b3c4-1e2f3a4b5c6d" { + t.Errorf("instance 1 is %d %s; an engine is given it by that UUID", mig.Index, mig.UUID) + } + if mig.MemoryTotalMB != 40192 || mig.MemoryUsedMB != 21000 { + t.Errorf("instance 1 memory %d/%d, want its own 21000/40192 rather than the card's", + mig.MemoryUsedMB, mig.MemoryTotalMB) + } + if m := mig.MIG; m == nil || m.ParentIndex != 0 || m.Device != 1 || m.Profile != "3g.40gb" || + m.GPUInstance != 1 || m.ComputeInstance != 0 || m.ParentUUID != "GPU-5d5ba0d6-d33d-2b2c-524d-9e3d8d2b8a77" { + t.Errorf("instance 1's parentage: %+v", mig.MIG) + } + if mig.ComputeCapability != "8.0" || mig.DriverVersion != "550.54.15" || mig.Health.TemperatureC != 31 { + t.Errorf("what the card and its instances share was not carried: %+v", mig) + } + // Twenty gigabytes in use on an instance is a foreign tenant there, and + // only there: instance 0 next to it is free. + if mig.Health.OK() || !gpus[0].Health.OK() { + t.Errorf("foreign memory judged per instance: 0 %v, 1 %v", gpus[0].Health.Problems, mig.Health.Problems) + } + // No driver reports utilisation per instance, and none is made up. + if mig.UtilizationPercent != 0 { + t.Errorf("an instance reported utilisation %d", mig.UtilizationPercent) + } + if gpus[2].IsMIG() || gpus[2].Index != 1 || gpus[2].CardIndex() != 1 || gpus[0].CardIndex() != 0 { + t.Errorf("the whole card beside it changed: %+v", gpus[2]) + } +} + +func TestAMIGInstanceSurvivesTheTripToTheControlPlaneWithItsClass(t *testing.T) { + g := migMachine(t)[0] + b, err := json.Marshal(g) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(b), `"class":"a100-80g-mig-3g.40gb"`) { + t.Errorf("the JSON carries no class for the instance: %s", b) + } + var back GPU + if err := json.Unmarshal(b, &back); err != nil { + t.Fatal(err) + } + if back.Class() != g.Class() || !reflect.DeepEqual(back.MIG, g.MIG) { + t.Errorf("came back as %s %+v", back.Class(), back.MIG) + } +} + +func TestMIGClassesAreNamedFromTheCardAndTheProfile(t *testing.T) { + for _, c := range []struct { + name string + cardMB int + profile string + want string + }{ + {"NVIDIA A100-SXM4-80GB MIG 3g.40gb", 81920, "3g.40gb", "a100-80g-mig-3g.40gb"}, + // The same profile name on a 40 GB card is another class: its + // instance is the same size, but the card is not, and neither is + // the set of profiles that card can be cut to. + {"NVIDIA A100-SXM4-40GB MIG 3g.20gb", 40960, "3g.20gb", "a100-40g-mig-3g.20gb"}, + {"NVIDIA H100 80GB HBM3 MIG 1g.10gb+me", 81559, "1g.10gb+me", "h100-80g-mig-1g.10gb-me"}, + {"NVIDIA H100 80GB HBM3 MIG 7g.80gb", 81559, "7g.80gb", "h100-80g-mig-7g.80gb"}, + } { + g := GPU{Name: c.name, MemoryTotalMB: 10000, MIG: &MIGDevice{ParentMemoryTotalMB: c.cardMB, Profile: c.profile}} + if got := g.Class(); got != c.want { + t.Errorf("%s -> %s, want %s", c.name, got, c.want) + } + if !IsMIGClass(g.Class()) { + t.Errorf("%s is not recognised as a MIG class", g.Class()) + } + } + if IsMIGClass("a100-80g") || IsMIGClass("6000-24g") { + t.Error("a whole card's class was taken for a MIG one") + } + if MIGProfileMemoryGB("a100-80g-mig-3g.40gb") != 40 || MIGProfileMemoryGB("1g.10gb+me") != 10 || MIGProfileMemoryGB("a100-80g") != 0 { + t.Error("the instance's memory is misread from its profile") + } +} + +func TestDeviceNumbersAreShownAsCardAndInstance(t *testing.T) { + for i, want := range map[int]string{0: "0", 7: "7", MIGIndex(0, 2): "0/mig2", MIGIndex(3, 0): "3/mig0", 1302: "3/mig2"} { + if got := DeviceLabel(i); got != want { + t.Errorf("DeviceLabel(%d) = %s, want %s", i, got, want) + } + } + if IsMIGIndex(15) || !IsMIGIndex(MIGIndex(0, 0)) { + t.Error("whole cards and instances are told apart wrongly") + } +} + +func TestACardInMIGModeWithNoInstancesIsKeptOutOfPlacement(t *testing.T) { + cards, _ := parseSMI(migQuery) + report, _ := parseMIGReport([]byte(strings.Replace(migReport, + migReport[strings.Index(migReport, ""):strings.Index(migReport, "")+len("")], + "None", 1))) + gpus := expandMIG(cards, parseSMIList("GPU 0: NVIDIA A100-SXM4-80GB (UUID: GPU-x)\nGPU 1: NVIDIA A100-SXM4-80GB (UUID: GPU-y)\n"), report) + if len(gpus) != 2 || gpus[0].IsMIG() { + t.Fatalf("got %+v", gpus) + } + // CUDA will not open the whole card while MIG is on, and there is no + // instance to open instead: placing anything here fails at start. + if gpus[0].Health.OK() || !strings.Contains(gpus[0].Health.Problems[0], "no instances") { + t.Errorf("card 0's problems: %v", gpus[0].Health.Problems) + } + if !gpus[1].Health.OK() { + t.Errorf("card 1, not in MIG mode, was flagged: %v", gpus[1].Health.Problems) + } + + // Instances the report sees but -L could not name cannot be given to an + // engine either; the card says why. + report, _ = parseMIGReport([]byte(migReport)) + gpus = expandMIG(cards, nil, report) + if len(gpus) != 2 || gpus[0].Health.OK() || !strings.Contains(gpus[0].Health.Problems[0], "did not name") { + t.Errorf("with -L failing: %+v", gpus) + } +} + +func TestInstancesAreFoundFromTheListAloneWhenTheReportCannotBeRead(t *testing.T) { + cards, _ := parseSMI(migQuery) + gpus := expandMIG(cards, parseSMIList(migList), nil) + if len(gpus) != 3 || !gpus[0].IsMIG() || gpus[0].MemoryTotalMB != 40*1024 { + t.Errorf("without the report: %+v", gpus) + } +} + +func TestAnOlderDriversInstanceUUIDIsTakenAsItIs(t *testing.T) { + // R450 and R460 name an instance by its card and ids, and CUDA of that + // era takes exactly that in CUDA_VISIBLE_DEVICES. + l := parseSMIList("GPU 0: A100-SXM4-40GB (UUID: GPU-5d5ba0d6-d33d-2b2c-524d-9e3d8d2b8a77)\n" + + " MIG 1g.5gb Device 0: (UUID: MIG-GPU-5d5ba0d6-d33d-2b2c-524d-9e3d8d2b8a77/7/0)\n") + if len(l[0]) != 1 || l[0][0].UUID != "MIG-GPU-5d5ba0d6-d33d-2b2c-524d-9e3d8d2b8a77/7/0" || l[0][0].Profile != "1g.5gb" { + t.Errorf("parsed %+v", l) + } +} + +func TestProcessesOnAMIGCardAreChargedToTheirOwnInstance(t *testing.T) { + mem, err := parseGPUMemory([]byte(migReport)) + if err != nil { + t.Fatal(err) + } + want := []GPUMemory{ + {Index: 1000, UsedMB: 37}, + {Index: 1001, UsedMB: 21000, Procs: []GPUProcess{{PID: 4100, UsedMB: 20900}}}, + {Index: 1, UsedMB: 300, Procs: []GPUProcess{{PID: 5200, UsedMB: 250}}}, + } + if !reflect.DeepEqual(mem, want) { + t.Errorf("got %+v\nwant %+v", mem, want) + } + // In the order discovery reports the devices, so a heartbeat's readings + // line up with the inventory one for one. + gpus := migMachine(t) + for i := range gpus { + if mem[i].Index != gpus[i].Index { + t.Errorf("reading %d is for device %d, inventory has %d", i, mem[i].Index, gpus[i].Index) + } + } + procs, _ := parseGPUProcesses([]byte(migReport)) + if !reflect.DeepEqual(procs, map[int][]int{1001: {4100}, 1: {5200}}) { + t.Errorf("processes by device: %v", procs) + } +} + +func TestAMIGInstanceTakesItsCardsPlaceInTheTopologyButNoNVLink(t *testing.T) { + gpus := migMachine(t) + topo := Topology{ + NVLink: map[int]map[int]int{0: {1: 12}, 1: {0: 12}}, + NUMA: map[int]int{0: 0, 1: 1}, + NICPath: map[int]map[string]string{0: {"mlx5_0": "PIX"}}, + } + topo.Apply(gpus) + if len(gpus[0].NVLinkPeers) != 0 || len(gpus[1].NVLinkPeers) != 0 { + t.Errorf("MIG instances were given NVLink peers: %v %v", gpus[0].NVLinkPeers, gpus[1].NVLinkPeers) + } + if gpus[1].NUMANode != 0 || gpus[1].NICPaths["mlx5_0"] != "PIX" { + t.Errorf("instance 1 did not take its card's NUMA node and adapters: %+v", gpus[1]) + } + if gpus[2].NUMANode != 1 || !slices.Equal(gpus[2].NVLinkPeers, []int{0}) { + t.Errorf("the whole card's topology: %+v", gpus[2]) + } +} + +func TestMIGInstancesAreDiscoveredThroughNvidiaSMI(t *testing.T) { + // The real code path against a stub binary that answers each command as + // the machine above would. + dir := t.TempDir() + stub := filepath.Join(dir, "nvidia-smi") + script := "#!/bin/sh\ncase \"$1\" in\n" + + "-L) cat <<'EOF'\n" + migList + "EOF\n;;\n" + + "-q) cat <<'EOF'\n" + migReport + "\nEOF\n;;\n" + + "*) cat <<'EOF'\n" + migQuery + "EOF\n;;\nesac\n" + if err := os.WriteFile(stub, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + gpus, err := DiscoverGPUsSMI(t.Context(), stub) + if err != nil { + t.Fatal(err) + } + inv := Inventory{GPUs: gpus} + if !slices.Equal(inv.Classes(), []string{"a100-80g", "a100-80g-mig-3g.40gb"}) || len(gpus) != 3 { + t.Errorf("classes %v from %d devices", inv.Classes(), len(gpus)) + } +} + +// fakeMIGCard is NVML's view of one card, as nvmlDeviceGetMigMode, +// nvmlDeviceGetMaxMigDeviceCount and nvmlDeviceGetMigDeviceHandleByIndex +// would give it. +type fakeMIGCard struct { + mode, pending, supported bool + max int + devs map[int]fakeMIGInstance +} + +func (c fakeMIGCard) MIGMode() (bool, bool, bool) { return c.mode, c.pending, c.supported } +func (c fakeMIGCard) MaxMIGDevices() int { return c.max } +func (c fakeMIGCard) MIGDevice(i int) (nvmlMIGInstance, bool) { + d, ok := c.devs[i] + return d, ok +} + +type fakeMIGInstance struct { + uuid, name string + total, used int + gi, ci int + slices, attrMem int +} + +func (m fakeMIGInstance) UUID() string { return m.uuid } +func (m fakeMIGInstance) Name() string { return m.name } +func (m fakeMIGInstance) Memory() (int, int, bool) { + return m.total, m.used, m.total > 0 +} +func (m fakeMIGInstance) Instances() (int, int) { return m.gi, m.ci } +func (m fakeMIGInstance) Attributes() (int, int, bool) { + return m.slices, m.attrMem, m.slices > 0 +} + +func TestNVMLReportsACardInMIGModeAsItsInstances(t *testing.T) { + card := GPU{Index: 2, UUID: "GPU-card", Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559, + ComputeCapability: "9.0", NUMANode: 1, Health: GPUHealth{TemperatureC: 40}} + // Seven slots, three filled, one of them on a driver that does not put + // the profile in the device's name. + dev := fakeMIGCard{mode: true, pending: true, supported: true, max: 7, devs: map[int]fakeMIGInstance{ + 0: {uuid: "MIG-aaa", name: "NVIDIA H100 80GB HBM3 MIG 3g.40gb", total: 40448, used: 10, gi: 2, ci: 0}, + 1: {uuid: "MIG-bbb", name: "NVIDIA H100 80GB HBM3 MIG 3g.40gb", total: 40448, used: 10, gi: 1, ci: 0}, + 4: {uuid: "MIG-ccc", name: "NVIDIA H100 80GB HBM3", total: 9984, gi: 9, ci: 0, slices: 1, attrMem: 9984}, + }} + gpus, isMIG := migFromNVML(card, dev) + if !isMIG || len(gpus) != 3 { + t.Fatalf("got %d devices, MIG %v", len(gpus), isMIG) + } + var classes []string + for _, g := range gpus { + classes = append(classes, DeviceLabel(g.Index)+" "+g.UUID+" "+g.Class()) + } + want := []string{"2/mig0 MIG-aaa h100-80g-mig-3g.40gb", "2/mig1 MIG-bbb h100-80g-mig-3g.40gb", + "2/mig4 MIG-ccc h100-80g-mig-1g.10gb"} + if !slices.Equal(classes, want) { + t.Errorf("devices %q, want %q", classes, want) + } + if g := gpus[1]; g.MemoryTotalMB != 40448 || g.MIG.GPUInstance != 1 || g.NUMANode != 1 || + g.Health.TemperatureC != 40 || len(g.NVLinkPeers) != 0 { + t.Errorf("instance 1: %+v", g) + } + + // MIG on and nothing carved: the card, kept out of placement. + gpus, isMIG = migFromNVML(card, fakeMIGCard{mode: true, supported: true, max: 7}) + if !isMIG || len(gpus) != 1 || gpus[0].IsMIG() || gpus[0].Health.OK() { + t.Errorf("an empty MIG card: %+v", gpus) + } + // MIG off, or a card without it: the card, as it always was. + for _, d := range []fakeMIGCard{{supported: true, max: 7}, {}} { + gpus, isMIG = migFromNVML(card, d) + if isMIG || len(gpus) != 1 || !reflect.DeepEqual(gpus[0], card) { + t.Errorf("a card not in MIG mode came back as %+v", gpus) + } + } +} + +func TestMIGInstanceTablesAreRead(t *testing.T) { + gis := ParseMIGInstances(migLGI) + if len(gis) != 2 || gis[0] != (MIGInstance{GPU: 0, Profile: "3g.40gb", ProfileID: 9, GPUInstance: 1, + ComputeInstance: -1, Placement: "4:4"}) { + t.Errorf("-lgi: %+v", gis) + } + cis := ParseMIGInstances(migLCI) + if len(cis) != 2 || cis[1] != (MIGInstance{GPU: 0, GPUInstance: 2, Profile: "3g.40gb", ProfileID: 2, + ComputeInstance: 0, Placement: "0:3"}) { + t.Errorf("-lci: %+v", cis) + } + if got := ParseMIGInstances("No GPU instances found: Not Found\n"); len(got) != 0 { + t.Errorf("nothing to list read as %+v", got) + } + + cards, err := buildMIGCards([]byte(migReport), migList, migLGI, migLCI) + if err != nil { + t.Fatal(err) + } + if len(cards) != 2 || !cards[0].Enabled || len(cards[0].Instances) != 2 || len(cards[0].GPUInstances) != 2 || + len(cards[0].ComputeInstances) != 2 || cards[1].Enabled { + t.Errorf("cards: %+v", cards) + } +} diff --git a/internal/agent/discovery/topo.go b/internal/agent/discovery/topo.go index 13230a4b..caea337e 100644 --- a/internal/agent/discovery/topo.go +++ b/internal/agent/discovery/topo.go @@ -166,19 +166,25 @@ func splitTabs(l string) []string { // Apply puts the topology on the GPUs it describes. What a GPU already knows // — NVLink peers from NVML, which is the more exact source — is kept. +// +// The matrix is of whole cards. A MIG instance takes its card's NUMA node and +// adapters, and none of its NVLink: instances cannot reach each other or any +// other card over it, and peers listed for one would draw a tensor-parallel +// group onto it. func (t Topology) Apply(gpus []GPU) { for i := range gpus { g := &gpus[i] - if len(g.NVLinkPeers) == 0 { - for peer := range t.NVLink[g.Index] { + card := g.CardIndex() + if len(g.NVLinkPeers) == 0 && !g.IsMIG() { + for peer := range t.NVLink[card] { g.NVLinkPeers = append(g.NVLinkPeers, peer) } sort.Ints(g.NVLinkPeers) } - if n, ok := t.NUMA[g.Index]; ok { + if n, ok := t.NUMA[card]; ok { g.NUMANode = n } - if paths := t.NICPath[g.Index]; len(paths) > 0 { + if paths := t.NICPath[card]; len(paths) > 0 { g.NICPaths = paths } } diff --git a/internal/agent/engines/engine.go b/internal/agent/engines/engine.go index 484ffed8..3a8306c1 100644 --- a/internal/agent/engines/engine.go +++ b/internal/agent/engines/engine.go @@ -62,6 +62,12 @@ type ShardSpec struct { // GPUIndices are the local GPUs this shard uses. GPUIndices []int + // GPUUUIDs name the same devices by UUID, in the same order, and are set + // by the agent only when one of them is a MIG instance. An instance has + // no index CUDA knows: CUDA_VISIBLE_DEVICES=MIG- is the only way to + // give a process one, and the indices placement uses for it (1000 and + // up) mean nothing to CUDA. + GPUUUIDs []string `json:"gpu_uuids,omitempty"` // HCAs are the adapters of the chosen fabric, and only that fabric. A shard // must never be given HCAs from two fabrics. HCAs []string @@ -257,7 +263,7 @@ func NCCLEnv(s ShardSpec) map[string]string { return baseEnv(s) } // one instead of timing out. func baseEnv(s ShardSpec) map[string]string { env := map[string]string{ - "CUDA_VISIBLE_DEVICES": joinInts(s.GPUIndices), + "CUDA_VISIBLE_DEVICES": visibleDevices(s), // The indices above are nvidia-smi's, which counts by PCI bus. CUDA // counts fastest first unless told otherwise, and on a node with two // kinds of GPU the shard would get a different card than placement @@ -329,6 +335,21 @@ func materializeEnv(generated map[string]string) []string { return out } +// visibleDevices is CUDA_VISIBLE_DEVICES for a shard: the indices placement +// gave it, or their UUIDs when the agent named them, which it does for a MIG +// instance. +func visibleDevices(s ShardSpec) string { + if len(s.GPUUUIDs) > 0 { + return strings.Join(s.GPUUUIDs, ",") + } + return joinInts(s.GPUIndices) +} + +// OnMIG reports whether the shard was given a MIG instance. +func (s ShardSpec) OnMIG() bool { + return slices.ContainsFunc(s.GPUUUIDs, func(u string) bool { return strings.HasPrefix(u, "MIG-") }) +} + func joinInts(v []int) string { parts := make([]string, len(v)) for i, n := range v { diff --git a/internal/agent/engines/llamacpp.go b/internal/agent/engines/llamacpp.go index 02a27a60..fb7bf22b 100644 --- a/internal/agent/engines/llamacpp.go +++ b/internal/agent/engines/llamacpp.go @@ -178,7 +178,11 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) { // as the driver lists them, which on NVIDIA follows the same bus order // as nvidia-smi; were it not to, the shard would not hold memory on its // own GPUs, and starting it checks exactly that. - if len(s.GPUIndices) > 0 { + // + // Not on a MIG instance, which Vulkan does not see at all: only CUDA + // opens one, by the UUID CUDA_VISIBLE_DEVICES carries, and an index here + // would name some other card. + if len(s.GPUIndices) > 0 && !s.OnMIG() { env["GGML_VK_VISIBLE_DEVICES"] = joinInts(s.GPUIndices) } return []Command{{ diff --git a/internal/agent/engines/mig_test.go b/internal/agent/engines/mig_test.go new file mode 100644 index 00000000..08cf458a --- /dev/null +++ b/internal/agent/engines/mig_test.go @@ -0,0 +1,49 @@ +package engines + +import "testing" + +const migUUID = "MIG-c6d4f1ef-42e4-5de3-91c7-45d71c87eb3f" + +func TestAShardOnAMIGInstanceIsGivenItByUUID(t *testing.T) { + // Placement numbers card 0's instance 1 as 1001, which CUDA has never + // heard of: CUDA_VISIBLE_DEVICES=1001 is no device at all, and =1 would + // be the wrong card. + s := singleNodeSpec() + s.GPUIndices, s.GPUUUIDs = []int{1001}, []string{migUUID} + s.Profile.TP = 1 + + env := envMap(buildOne(t, &VLLM{}, s).Env) + if env["CUDA_VISIBLE_DEVICES"] != migUUID { + t.Errorf("vLLM: CUDA_VISIBLE_DEVICES=%q, want the instance's UUID", env["CUDA_VISIBLE_DEVICES"]) + } + + l := llamaSpec([]int{1001}) + l.GPUUUIDs = s.GPUUUIDs + cmds, err := (&LlamaCpp{}).Commands(l) + if err != nil { + t.Fatal(err) + } + env = envMap(cmds[0].Env) + if env["CUDA_VISIBLE_DEVICES"] != migUUID { + t.Errorf("llama.cpp: CUDA_VISIBLE_DEVICES=%q, want the instance's UUID", env["CUDA_VISIBLE_DEVICES"]) + } + // Vulkan does not see MIG instances, and an index for it would name + // some other card. + if v, set := env["GGML_VK_VISIBLE_DEVICES"]; set { + t.Errorf("GGML_VK_VISIBLE_DEVICES=%q on a MIG instance", v) + } + if v, _ := flagValue(cmds[0].Args, "--n-gpu-layers"); v != "999" { + t.Errorf("the model was not offloaded to the instance: --n-gpu-layers %q", v) + } +} + +func TestAShardOnWholeCardsIsStillGivenThemByIndex(t *testing.T) { + env := envMap(buildOne(t, &VLLM{}, singleNodeSpec()).Env) + if env["CUDA_VISIBLE_DEVICES"] != "0,1,2,3" { + t.Errorf("CUDA_VISIBLE_DEVICES=%q", env["CUDA_VISIBLE_DEVICES"]) + } + cmds, _ := (&LlamaCpp{}).Commands(llamaSpec([]int{2})) + if env := envMap(cmds[0].Env); env["GGML_VK_VISIBLE_DEVICES"] != "2" { + t.Errorf("GGML_VK_VISIBLE_DEVICES=%q", env["GGML_VK_VISIBLE_DEVICES"]) + } +} diff --git a/internal/agent/mig_test.go b/internal/agent/mig_test.go new file mode 100644 index 00000000..109d3ecf --- /dev/null +++ b/internal/agent/mig_test.go @@ -0,0 +1,40 @@ +package agent + +import ( + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines" +) + +func TestAShardPlacedOnAMIGInstanceIsLaunchedWithItsUUID(t *testing.T) { + // The control plane sends device 1001: card 0's instance 1, by the number + // placement gives it. Only this node knows the instance's UUID, which is + // the one thing CUDA will open it by. + a := &Agent{} + a.inv.GPUs = []discovery.GPU{ + {Index: 1000, UUID: "MIG-aaa", MIG: &discovery.MIGDevice{ParentIndex: 0, Device: 0, Profile: "3g.40gb"}}, + {Index: 1001, UUID: "MIG-bbb", MIG: &discovery.MIGDevice{ParentIndex: 0, Device: 1, Profile: "3g.40gb"}}, + {Index: 1, UUID: "GPU-whole"}, + } + + spec := engines.ShardSpec{GPUIndices: []int{1001}} + a.localizeSpec(&spec) + if got := engines.NCCLEnv(spec)["CUDA_VISIBLE_DEVICES"]; got != "MIG-bbb" { + t.Errorf("CUDA_VISIBLE_DEVICES=%q, want MIG-bbb", got) + } + + // A whole card is given by index, as it always was. + whole := engines.ShardSpec{GPUIndices: []int{1}} + a.localizeSpec(&whole) + if whole.GPUUUIDs != nil || engines.NCCLEnv(whole)["CUDA_VISIBLE_DEVICES"] != "1" { + t.Errorf("a whole card was given as %v / %q", whole.GPUUUIDs, engines.NCCLEnv(whole)["CUDA_VISIBLE_DEVICES"]) + } + + // A device this node does not have is not guessed at. + gone := engines.ShardSpec{GPUIndices: []int{1002}} + a.localizeSpec(&gone) + if gone.GPUUUIDs != nil { + t.Errorf("an unknown device was named %v", gone.GPUUUIDs) + } +} diff --git a/internal/cli/agent.go b/internal/cli/agent.go index 102bf3ff..4f0ae6c5 100644 --- a/internal/cli/agent.go +++ b/internal/cli/agent.go @@ -159,13 +159,13 @@ func printInventory(inv discovery.Inventory) { if !g.Health.OK() { health = g.Health.Problems[0] } - fmt.Fprintf(w, " %d\t%s\t%s\t%d MiB\t%s\t%s\n", - g.Index, g.Class(), g.Name, g.MemoryTotalMB, g.ComputeCapability, health) + fmt.Fprintf(w, " %s\t%s\t%s\t%d MiB\t%s\t%s\n", + discovery.DeviceLabel(g.Index), g.Class(), g.Name, g.MemoryTotalMB, g.ComputeCapability, health) } fmt.Fprintf(w, " classes: %v\n", inv.Classes()) for _, g := range inv.GPUs { if len(g.NVLinkPeers) > 0 { - fmt.Fprintf(w, " GPU %d NVLink peers: %v\n", g.Index, g.NVLinkPeers) + fmt.Fprintf(w, " GPU %s NVLink peers: %v\n", discovery.DeviceLabel(g.Index), g.NVLinkPeers) } } } diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go index 4cf0b822..574df14e 100644 --- a/internal/cli/ctl.go +++ b/internal/cli/ctl.go @@ -14,6 +14,7 @@ import ( "github.com/spf13/cobra" + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" ) @@ -212,7 +213,7 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { // at when a model will not place and the hardware looks // fine. join(n.Engines), - join(n.Classes), join(n.Fabrics), n.Replicas, n.LastSeenAgo) + classesColumn(n), join(n.Fabrics), n.Replicas, n.LastSeenAgo) } _ = w.Flush() @@ -410,7 +411,9 @@ func newReplicasCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { for _, r := range reps { var nodes []string for _, s := range r.Shards { - nodes = append(nodes, fmt.Sprintf("%s:%v", s.Node, s.GPUs)) + // "[0/mig2]" for a MIG instance, not the 1002 it is + // numbered by. + nodes = append(nodes, fmt.Sprintf("%s:%v", s.Node, discovery.DeviceLabels(s.GPUs))) } // The state says stale rather than a column of its own: most // replicas are not, a column would be empty for all of them, @@ -540,6 +543,25 @@ func emitJSON(v any) error { return enc.Encode(v) } +// classesColumn is a node's GPU classes, with how many of each when it has +// more than one or MIG instances among them: a card cut into two 3g.40gb +// instances beside a whole card is "a100-80g-mig-3g.40gb×2,a100-80g×1", where +// the GPUS column's 3 alone would read as three of something. +func classesColumn(n adminapi.NodeView) string { + counted := len(n.ClassGPUs) > 1 + for _, c := range n.ClassGPUs { + counted = counted || c.MIG + } + if !counted { + return join(n.Classes) + } + parts := make([]string, len(n.ClassGPUs)) + for i, c := range n.ClassGPUs { + parts[i] = fmt.Sprintf("%s×%d", c.Class, c.Total) + } + return strings.Join(parts, ",") +} + func join(v []string) string { if len(v) == 0 { return "-" diff --git a/internal/cli/doctor.go b/internal/cli/doctor.go index a46250c0..13d30fb7 100644 --- a/internal/cli/doctor.go +++ b/internal/cli/doctor.go @@ -110,9 +110,10 @@ func printDoctor(out io.Writer, f doctor.Facts, findings []doctor.Finding, verbo w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0) if len(f.GPUs) > 0 { - fmt.Fprintln(w, "\nGPU\tNAME\tNUMA\tNVLINK PEERS\tNEAREST ADAPTER") + fmt.Fprintln(w, "\nGPU\tNAME\tCLASS\tNUMA\tNVLINK PEERS\tNEAREST ADAPTER") for _, g := range f.GPUs { - fmt.Fprintf(w, "%d\t%s\t%d\t%s\t%s\n", g.Index, g.Name, g.NUMANode, + // A MIG instance as 0/mig2, under its card. + fmt.Fprintf(w, "%s\t%s\t%s\t%d\t%s\t%s\n", discovery.DeviceLabel(g.Index), g.Name, g.Class(), g.NUMANode, orDash(ints(g.NVLinkPeers)), orDash(nearest(g.NICPaths))) } } diff --git a/internal/cli/mig_test.go b/internal/cli/mig_test.go new file mode 100644 index 00000000..84084896 --- /dev/null +++ b/internal/cli/mig_test.go @@ -0,0 +1,40 @@ +package cli + +import ( + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/modelplan" +) + +func TestAPlanSeesANodesMIGInstancesAsAClassOfTheirOwn(t *testing.T) { + // Card 0 cut into two 3g.40gb instances, both free, and card 1 whole and + // busy: the instances are a pool of forty-gigabyte devices, and the busy + // card no pool at all. + hw := hardwareFrom([]adminapi.NodeView{{ID: "a100", State: "ready", GPUTotal: 3, GPUFree: 2, + Classes: []string{"a100-80g-mig-3g.40gb", "a100-80g"}, + ClassGPUs: []adminapi.ClassGPUs{ + {Class: "a100-80g-mig-3g.40gb", Total: 2, Free: 2, MIG: true}, + {Class: "a100-80g", Total: 1, Free: 0}, + }}}) + if len(hw) != 1 { + t.Fatalf("pools: %+v", hw) + } + want := modelplan.Hardware{Class: "a100-80g-mig-3g.40gb", GPUMemGB: 40, GPUsPerNode: 2, Nodes: 1} + if hw[0] != want { + t.Errorf("pool %+v, want %+v", hw[0], want) + } +} + +func TestCtlNodesCountsANodesDevicesByClassWhenItHasMIG(t *testing.T) { + mig := adminapi.NodeView{Classes: []string{"a100-80g-mig-3g.40gb", "a100-80g"}, ClassGPUs: []adminapi.ClassGPUs{ + {Class: "a100-80g-mig-3g.40gb", Total: 2, MIG: true}, {Class: "a100-80g", Total: 1}}} + if got := classesColumn(mig); got != "a100-80g-mig-3g.40gb×2,a100-80g×1" { + t.Errorf("MIG node: %q", got) + } + // A node of one kind of whole card reads as it always did. + plain := adminapi.NodeView{Classes: []string{"h100-80g"}, ClassGPUs: []adminapi.ClassGPUs{{Class: "h100-80g", Total: 8}}} + if got := classesColumn(plain); got != "h100-80g" { + t.Errorf("plain node: %q", got) + } +} diff --git a/internal/cli/modelplan.go b/internal/cli/modelplan.go index cb5eb16a..6ee413d0 100644 --- a/internal/cli/modelplan.go +++ b/internal/cli/modelplan.go @@ -155,28 +155,41 @@ func hardwareFrom(nodes []adminapi.NodeView) []modelplan.Hardware { if len(n.Fabrics) > 0 { fabric = n.Fabrics[0] } - class := "" - if len(n.Classes) > 0 { - class = n.Classes[0] + // Each class of the node's devices is a pool of its own: a node with + // a card in MIG mode has its instances' classes, and any whole card's + // beside them. A node that does not count by class is its first + // class, as it always was. + kinds := n.ClassGPUs + if len(kinds) == 0 { + class := "" + if len(n.Classes) > 0 { + class = n.Classes[0] + } + kinds = []adminapi.ClassGPUs{{Class: class, Total: n.GPUTotal, Free: n.GPUFree}} } - k := key{class, fabric} - if pools[k] == nil { - pools[k] = &modelplan.Hardware{Class: class, Fabric: fabric} - } - p := pools[k] - p.Nodes++ - // The widest a single node could contribute, not the total: a tensor - // group takes the same number of cards from every node it spans. - if n.GPUFree > p.GPUsPerNode { - p.GPUsPerNode = n.GPUFree - } - if n.MemoryGB > p.HostMemGB { - p.HostMemGB = n.MemoryGB - } - // Card memory from the class label, which carries it by construction: - // "h100-80g" is eighty gigabytes and says so. - if class != "" && p.GPUMemGB == 0 { - p.GPUMemGB = memFromClass(class) + for _, kind := range kinds { + class := kind.Class + k := key{class, fabric} + if pools[k] == nil { + pools[k] = &modelplan.Hardware{Class: class, Fabric: fabric} + } + p := pools[k] + p.Nodes++ + // The widest a single node could contribute, not the total: a + // tensor group takes the same number of cards from every node it + // spans. + if kind.Free > p.GPUsPerNode { + p.GPUsPerNode = kind.Free + } + if n.MemoryGB > p.HostMemGB { + p.HostMemGB = n.MemoryGB + } + // Card memory from the class label, which carries it by + // construction: "h100-80g" is eighty gigabytes and says so, and + // "a100-80g-mig-3g.40gb" is an instance of forty. + if class != "" && p.GPUMemGB == 0 { + p.GPUMemGB = memFromClass(class) + } } } @@ -191,17 +204,7 @@ func hardwareFrom(nodes []adminapi.NodeView) []modelplan.Hardware { } // memFromClass reads the card size out of a class label such as "h100-80g". -func memFromClass(class string) int { - i := strings.LastIndex(class, "-") - if i < 0 || !strings.HasSuffix(class, "g") { - return 0 - } - var gb int - if _, err := fmt.Sscanf(class[i+1:], "%dg", &gb); err != nil { - return 0 - } - return gb -} +func memFromClass(class string) int { return modelplan.MemFromClass(class) } // catalogEntry renders a fit as the YAML that would serve it. func catalogEntry(m modelplan.Model, f modelplan.Fit, source string, context int) string { diff --git a/internal/cli/tui_console.go b/internal/cli/tui_console.go index e96af5ec..87693bab 100644 --- a/internal/cli/tui_console.go +++ b/internal/cli/tui_console.go @@ -17,6 +17,7 @@ import ( "github.com/charmbracelet/lipgloss" "github.com/spf13/cobra" + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" "gitea.mynodes.xyz/ilker/llmgw/internal/config" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" @@ -1080,7 +1081,7 @@ func (m consoleModel) panel() (title, body, keys string) { for _, r := range m.data.replicas { var nodes []string for _, s := range r.Shards { - nodes = append(nodes, fmt.Sprintf("%s%v", s.Node, s.GPUs)) + nodes = append(nodes, fmt.Sprintf("%s%v", s.Node, discovery.DeviceLabels(s.GPUs))) } rows = append(rows, []string{r.ID, r.Model, r.State, r.Pool, strings.Join(nodes, " "), (time.Duration(r.AgeSec) * time.Second).Round(time.Second).String()}) @@ -1224,14 +1225,31 @@ func (m consoleModel) gpuRows() [][]string { for _, n := range m.data.nodes { for _, g := range n.GPUs { busy, mem, temp, power := "—", "—", "—", "—" + // A MIG instance under its card: its own memory, the card's heat + // and power, and no utilisation, which no driver measures per + // instance — said, rather than shown as the card's or as zero. + mig := g.MIGProfile != "" + if mig { + busy = sDim.Render("n/a on MIG") + if g.MemoryTotalMB > 0 { + mem = fmt.Sprintf("%.1f/%.0f GB", float64(g.MemoryUsedMB)/1024, float64(g.MemoryTotalMB)/1024) + } + } + card := g.Card + if g.Label == "" { + // A control plane older than MIG, which sends no card. + card = g.Index + } if n.Load != nil { for _, s := range n.Load.GPUs { - if s.Index != g.Index { + if s.Index != card { continue } - busy = fmt.Sprintf("%.0f%%", s.UtilPercent) - if s.MemoryTotalMB > 0 { - mem = fmt.Sprintf("%.1f/%.0f GB", s.MemoryUsedMB/1024, s.MemoryTotalMB/1024) + if !mig { + busy = fmt.Sprintf("%.0f%%", s.UtilPercent) + if s.MemoryTotalMB > 0 { + mem = fmt.Sprintf("%.1f/%.0f GB", s.MemoryUsedMB/1024, s.MemoryTotalMB/1024) + } } if s.TempC > 0 { temp = fmt.Sprintf("%.0f°C", s.TempC) @@ -1245,7 +1263,12 @@ func (m consoleModel) gpuRows() [][]string { if holder == "" { holder = sDim.Render("free") } - rows = append(rows, []string{n.ID, strconv.Itoa(g.Index), g.Name, busy, mem, temp, power, holder}) + label := g.Label + if label == "" { + // A control plane older than labels. + label = strconv.Itoa(g.Index) + } + rows = append(rows, []string{n.ID, label, g.Name, busy, mem, temp, power, holder}) } } return rows @@ -1266,6 +1289,15 @@ func (m consoleModel) nodeDetail(n adminapi.NodeView) string { } kv("disk", fmt.Sprintf("%d GB free of %d", n.DiskFreeGB, n.DiskTotalGB)) kv("replicas", strconv.Itoa(n.Replicas)) + // By class, which on a node with a card in MIG mode is the instances' + // profiles beside any whole card: what a profile's gpu_class can name. + for _, c := range n.ClassGPUs { + what := fmt.Sprintf("%d, %d free", c.Total, c.Free) + if c.MIG { + what += " · MIG instances, one to a replica" + } + kv(c.Class, what) + } if len(n.Provision) > 0 { b.WriteString("\n " + lipgloss.NewStyle().Bold(true).Render("Engines") + "\n") for _, p := range n.Provision { diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 920b88b1..7d612faf 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -228,7 +228,12 @@ type NodeView struct { DiskFreeGB int `json:"disk_free_gb,omitempty"` DiskTotalGB int `json:"disk_total_gb,omitempty"` Classes []string `json:"gpu_classes,omitempty"` - Fabrics []string `json:"fabrics,omitempty"` + // ClassGPUs counts the node's devices by class, in the order of Classes. + // A node with a card in MIG mode has several classes — each profile its + // instances are cut to, and any whole card beside them — and GPUTotal + // alone would add 3g.40gb instances to 80 GB cards. + ClassGPUs []ClassGPUs `json:"gpus_by_class,omitempty"` + Fabrics []string `json:"fabrics,omitempty"` // Unhealthy lists GPUs kept out of placement and why, which is usually the // answer when a node looks idle but nothing will schedule onto it. @@ -255,10 +260,34 @@ type NodeView struct { IMEXChannels int `json:"imex_channels,omitempty"` } +// ClassGPUs is how many of a node's devices are of one class, and how many +// of those are free or kept out of placement. +type ClassGPUs struct { + Class string `json:"class"` + Total int `json:"total"` + Free int `json:"free"` + Unhealthy int `json:"unhealthy,omitempty"` + // MIG marks a class of MIG instances, each of which takes a replica of + // its own and never joins a tensor-parallel group. + MIG bool `json:"mig,omitempty"` +} + // GPUView is one card as a node's page shows it. type GPUView struct { Index int `json:"index"` Name string `json:"name"` + // Label is the device as people are shown it: "3" for a whole card, + // "0/mig2" for card 0's MIG instance 2. + Label string `json:"label"` + Class string `json:"class"` + // Card is the physical card's index: the device's own, or for a MIG + // instance its parent's, which is what the card's sensors are under. + Card int `json:"card"` + // MIGProfile is the instance's profile, like 3g.40gb; empty for a + // whole card. + MIGProfile string `json:"mig_profile,omitempty"` + MemoryTotalMB int `json:"memory_total_mb,omitempty"` + MemoryUsedMB int `json:"memory_used_mb,omitempty"` // ComputeCapability is what decides which engine builds run on the card: // CUDA 13 builds need 7.5, so a V100 (7.0) runs only older ones. ComputeCapability string `json:"compute_capability,omitempty"` @@ -556,15 +585,25 @@ func (a *API) Nodes() []NodeView { for _, w := range n.Weights { v.Models = append(v.Models, w.Path) } - seen := map[string]struct{}{} + seen := map[string]int{} + free := inv.FreeGPUs(n.ID) for _, g := range n.GPUs { - if _, dup := seen[g.Class()]; !dup { - seen[g.Class()] = struct{}{} + k, dup := seen[g.Class()] + if !dup { + k = len(v.ClassGPUs) + seen[g.Class()] = k v.Classes = append(v.Classes, g.Class()) + v.ClassGPUs = append(v.ClassGPUs, ClassGPUs{Class: g.Class(), MIG: g.IsMIG()}) } - if !g.Health.OK() { + c := &v.ClassGPUs[k] + c.Total++ + switch { + case !g.Health.OK(): + c.Unhealthy++ v.Unhealthy = append(v.Unhealthy, - fmt.Sprintf("gpu %d: %s", g.Index, g.Health.Problems[0])) + fmt.Sprintf("gpu %s: %s", discovery.DeviceLabel(g.Index), g.Health.Problems[0])) + case slices.Contains(free, g.Index): + c.Free++ } } if !n.LastSeen.IsZero() { @@ -574,7 +613,12 @@ func (a *API) Nodes() []NodeView { v.NVLinkDomain, v.IMEX, v.IMEXChannels = n.NVLinkDomain(), n.IMEX.State, n.IMEX.Channels for _, g := range n.GPUs { gv := GPUView{Index: g.Index, Name: g.Name, ComputeCapability: g.ComputeCapability, - NUMA: g.NUMANode, NVLinkPeers: g.NVLinkPeers} + NUMA: g.NUMANode, NVLinkPeers: g.NVLinkPeers, + Label: discovery.DeviceLabel(g.Index), Class: g.Class(), Card: g.CardIndex(), + MemoryTotalMB: g.MemoryTotalMB, MemoryUsedMB: g.MemoryUsedMB} + if g.MIG != nil { + gv.MIGProfile = g.MIG.Profile + } for hca, rel := range g.NICPaths { if gv.NearestNIC == "" || discovery.PathRank(rel) < discovery.PathRank(gv.NICPath) || (discovery.PathRank(rel) == discovery.PathRank(gv.NICPath) && hca < gv.NearestNIC) { @@ -1617,6 +1661,13 @@ func (a *API) PoolNodes() []modelplan.PoolNode { p.Class = n.Classes[0] p.GPUMemGB = modelplan.MemFromClass(p.Class) } + // By class, with each class's own count of devices: a node with a + // card in MIG mode is several, and its first class alone would + // count every instance and card as that one. + for _, c := range n.ClassGPUs { + p.Kinds = append(p.Kinds, modelplan.PoolKind{Class: c.Class, + GPUMemGB: modelplan.MemFromClass(c.Class), GPUs: c.Total, FreeGPUs: c.Free}) + } if len(n.Fabrics) > 0 { p.Fabric = n.Fabrics[0] } diff --git a/internal/control/adminapi/mig_test.go b/internal/control/adminapi/mig_test.go new file mode 100644 index 00000000..34bbf7f0 --- /dev/null +++ b/internal/control/adminapi/mig_test.go @@ -0,0 +1,65 @@ +package adminapi + +import ( + "reflect" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" +) + +// migNode has card 0 cut into two 3g.40gb instances and card 1 whole. +func migNode(id string) inventory.Node { + nd := inventory.Node{ID: id, State: inventory.Ready, AgentVersion: "v1", MemoryTotalMB: 512 * 1024} + for dev := range 2 { + nd.GPUs = append(nd.GPUs, discovery.GPU{ + Index: discovery.MIGIndex(0, dev), Name: "NVIDIA A100-SXM4-80GB MIG 3g.40gb", UUID: "MIG-x", + MemoryTotalMB: 40192, MemoryUsedMB: 37, + MIG: &discovery.MIGDevice{ParentIndex: 0, ParentMemoryTotalMB: 81920, Device: dev, Profile: "3g.40gb"}, + }) + } + nd.GPUs = append(nd.GPUs, discovery.GPU{Index: 1, Name: "NVIDIA A100-SXM4-80GB", MemoryTotalMB: 81920}) + return nd +} + +func TestANodeWithACardInMIGModeCountsItsDevicesByClass(t *testing.T) { + a, ctl := setup(t, testModels(), migNode("a100")) + ctl.Inventory().Allocate("a100", inventory.Allocation{ShardID: "s", ReplicaID: "r", GPUIndices: []int{1000}}) + + n := a.Nodes()[0] + want := []ClassGPUs{ + {Class: "a100-80g-mig-3g.40gb", Total: 2, Free: 1, MIG: true}, + {Class: "a100-80g", Total: 1, Free: 1}, + } + if !reflect.DeepEqual(n.ClassGPUs, want) { + t.Errorf("by class: %+v\nwant %+v", n.ClassGPUs, want) + } + // Each device as people are shown it: the instance under its card, with + // its own memory, and the card its sensors are read under. + g := n.GPUs[1] + if g.Label != "0/mig1" || g.Card != 0 || g.MIGProfile != "3g.40gb" || g.MemoryTotalMB != 40192 || + g.Class != "a100-80g-mig-3g.40gb" { + t.Errorf("instance view: %+v", g) + } + if w := n.GPUs[2]; w.Label != "1" || w.Card != 1 || w.MIGProfile != "" { + t.Errorf("whole card view: %+v", w) + } + + // The fleet summary attributes each device to its own class, where it + // once put every card of a node under the node's only class. + s := a.Summary() + got := map[string][2]int{} + for _, c := range s.GPUs { + got[c.Class] = [2]int{c.Total, c.Used} + } + if !reflect.DeepEqual(got, map[string][2]int{"a100-80g-mig-3g.40gb": {2, 1}, "a100-80g": {1, 0}}) { + t.Errorf("summary by class: %v", got) + } + + // And a plan sees both classes, each with its own count. + pool := a.PoolNodes() + if len(pool) != 1 || len(pool[0].Kinds) != 2 || pool[0].Kinds[0].GPUMemGB != 40 || + pool[0].Kinds[0].FreeGPUs != 1 || pool[0].Kinds[1].GPUMemGB != 80 { + t.Errorf("pool: %+v", pool) + } +} diff --git a/internal/control/adminapi/summary.go b/internal/control/adminapi/summary.go index 27b88b48..f5dfa2b7 100644 --- a/internal/control/adminapi/summary.go +++ b/internal/control/adminapi/summary.go @@ -148,12 +148,14 @@ func (a *API) Summary() Summary { byClass[c] = &ClassUse{Class: c} } } - // A node reports one class in practice; attribute its cards to it. - if len(n.Classes) == 1 { - u := byClass[n.Classes[0]] - u.Total += n.GPUTotal - u.Unhealthy += len(n.Unhealthy) - u.Used += n.GPUTotal - n.GPUFree - len(n.Unhealthy) + // Each class its own devices, which a node with a card in MIG mode + // needs: its instances are one class or several, and any whole card + // beside them another. + for _, c := range n.ClassGPUs { + u := byClass[c.Class] + u.Total += c.Total + u.Unhealthy += c.Unhealthy + u.Used += c.Total - c.Free - c.Unhealthy } } for _, u := range byClass { diff --git a/internal/control/agentsvc/convert.go b/internal/control/agentsvc/convert.go index 294ee3f6..67a07e03 100644 --- a/internal/control/agentsvc/convert.go +++ b/internal/control/agentsvc/convert.go @@ -71,6 +71,7 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node { TemperatureC: int(g.GetTemperatureC()), Problems: g.GetProblems(), }, + MIG: migFrom(g.GetMig()), }) } n.IMEX = discovery.IMEX{State: r.GetImex().GetState(), Channels: int(r.GetImex().GetChannels())} @@ -131,6 +132,13 @@ func RegisterFromInventory(nodeID, hostname, mgmtAddr, agentVersion string, last.FabricState, last.FabricStatus, last.ClusterUuid = f.State, f.Status, f.ClusterUUID last.CliqueId, last.FabricBandwidth = f.CliqueID, f.Bandwidth } + if m := g.MIG; m != nil { + r.Gpus[len(r.Gpus)-1].Mig = &llmgwv1.MIGDevice{ + ParentIndex: int32(m.ParentIndex), ParentUuid: m.ParentUUID, + ParentMemoryTotalMb: int32(m.ParentMemoryTotalMB), Device: int32(m.Device), + Profile: m.Profile, GpuInstance: int32(m.GPUInstance), ComputeInstance: int32(m.ComputeInstance), + } + } } r.Imex = &llmgwv1.IMEX{State: inv.IMEX.State, Channels: int32(inv.IMEX.Channels)} for _, f := range fabrics { @@ -286,6 +294,19 @@ func int32Slice(v []int) []int32 { return out } +// migFrom is a MIG instance's parentage as the wire carries it, nil for a +// whole card and from every agent older than MIG support. +func migFrom(m *llmgwv1.MIGDevice) *discovery.MIGDevice { + if m == nil { + return nil + } + return &discovery.MIGDevice{ + ParentIndex: int(m.GetParentIndex()), ParentUUID: m.GetParentUuid(), + ParentMemoryTotalMB: int(m.GetParentMemoryTotalMb()), Device: int(m.GetDevice()), + Profile: m.GetProfile(), GPUInstance: int(m.GetGpuInstance()), ComputeInstance: int(m.GetComputeInstance()), + } +} + // fabricFrom is a GPU's NVLink fabric as the wire carries it, nil when the // driver reported none. func fabricFrom(state, status, cluster, clique, bandwidth string) *discovery.GPUFabric { diff --git a/internal/control/agentsvc/mig_test.go b/internal/control/agentsvc/mig_test.go new file mode 100644 index 00000000..f4a895d1 --- /dev/null +++ b/internal/control/agentsvc/mig_test.go @@ -0,0 +1,27 @@ +package agentsvc + +import ( + "reflect" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" +) + +func TestAMIGInstanceKeepsItsParentageOnTheWire(t *testing.T) { + inst := discovery.GPU{Index: 1001, UUID: "MIG-bbb", Name: "NVIDIA A100-SXM4-80GB MIG 3g.40gb", MemoryTotalMB: 40192, + MIG: &discovery.MIGDevice{ParentIndex: 0, ParentUUID: "GPU-card", ParentMemoryTotalMB: 81920, + Device: 1, Profile: "3g.40gb", GPUInstance: 1, ComputeInstance: 0}} + whole := discovery.GPU{Index: 1, UUID: "GPU-whole", Name: "NVIDIA A100-SXM4-80GB", MemoryTotalMB: 81920} + + reg := RegisterFromInventory("n1", "h", "10.0.0.1", "v", nil, discovery.Inventory{GPUs: []discovery.GPU{inst, whole}}, nil) + n := NodeFromRegister(reg) + if len(n.GPUs) != 2 { + t.Fatalf("got %d GPUs", len(n.GPUs)) + } + if !reflect.DeepEqual(n.GPUs[0].MIG, inst.MIG) || n.GPUs[0].Class() != "a100-80g-mig-3g.40gb" { + t.Errorf("the instance arrived as %s %+v", n.GPUs[0].Class(), n.GPUs[0].MIG) + } + if n.GPUs[1].MIG != nil || n.GPUs[1].Class() != "a100-80g" { + t.Errorf("the whole card arrived as %s %+v", n.GPUs[1].Class(), n.GPUs[1].MIG) + } +} diff --git a/internal/control/catalog/catalog.go b/internal/control/catalog/catalog.go index 602082bb..a3640d86 100644 --- a/internal/control/catalog/catalog.go +++ b/internal/control/catalog/catalog.go @@ -12,6 +12,7 @@ import ( "strings" "time" + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines" "gitea.mynodes.xyz/ilker/llmgw/internal/control/placement" ) @@ -701,6 +702,17 @@ func (m Model) validate() error { } continue } + // A MIG instance is a device of its own to CUDA, but a process sees + // only one, and instances have no NVLink or peer-to-peer between + // them: a group of them is neither tensor- nor layer-parallel, it is + // a replica that fails at start. One instance, one node, refused here + // rather than placed and found out. + if discovery.IsMIGClass(p.GPUClass) && (p.GPUsPerNode > 1 || p.Nodes > 1 || p.TP > 1 || p.PP > 1) { + return fmt.Errorf("model %q profile %q: %s is a MIG instance, and a replica runs on exactly one: "+ + "a process can open only one MIG device, and instances have no NVLink between them. "+ + "Give it tp 1 and one GPU on one node, or use a whole-card class or a larger MIG profile", + m.Name, p.Name, p.GPUClass) + } // llama.cpp on GPUs splits a model across one node's cards by layer, // and how many is gpus_per_node. Not tp: llama.cpp has no tensor // parallelism, and a tp it was given was a width nothing placed — the diff --git a/internal/control/catalog/mig_test.go b/internal/control/catalog/mig_test.go new file mode 100644 index 00000000..8fd0e252 --- /dev/null +++ b/internal/control/catalog/mig_test.go @@ -0,0 +1,38 @@ +package catalog + +import ( + "strings" + "testing" +) + +func TestAProfileOnAMIGClassRunsOnExactlyOneInstance(t *testing.T) { + // One instance, by default or said outright, is a profile like any other. + for _, p := range []Profile{ + {Name: "vllm", Engine: "vllm", GPUClass: "a100-80g-mig-3g.40gb"}, + {Name: "llama", Engine: "llamacpp", GPUClass: "a100-80g-mig-3g.40gb", GPUsPerNode: 1, TP: 1}, + } { + c, err := New([]Model{model("m", p)}) + if err != nil { + t.Fatalf("%s on one MIG instance: %v", p.Name, err) + } + m, _ := c.Resolve("m") + if pl := m.PlacementProfiles()[0]; pl.GPUsPerNode != 1 || pl.Nodes != 1 { + t.Errorf("%s: placement asked for %+v", p.Name, pl) + } + } + // More than one is refused where the model is defined: a process opens + // one MIG device, and instances have no NVLink between them, so the + // replica would be placed and then fail at start. + for _, p := range []Profile{ + {Name: "tp2", Engine: "vllm", GPUClass: "a100-80g-mig-3g.40gb", TP: 2}, + {Name: "pp2", Engine: "vllm", GPUClass: "a100-80g-mig-3g.40gb", TP: 1, PP: 2, GPUsPerNode: 2}, + {Name: "nodes", Engine: "vllm", GPUClass: "a100-80g-mig-3g.40gb", TP: 2, Nodes: 2, GPUsPerNode: 1}, + {Name: "layers", Engine: "llamacpp", GPUClass: "a100-80g-mig-3g.40gb", GPUsPerNode: 2}, + {Name: "old-tp", Engine: "llamacpp", GPUClass: "a100-80g-mig-3g.40gb", TP: 2}, + } { + _, err := New([]Model{model("m", p)}) + if err == nil || !strings.Contains(err.Error(), "MIG instance") { + t.Errorf("%s: %v", p.Name, err) + } + } +} diff --git a/internal/control/fleetmetrics/fleetmetrics.go b/internal/control/fleetmetrics/fleetmetrics.go index f8a3c970..16b9b5b7 100644 --- a/internal/control/fleetmetrics/fleetmetrics.go +++ b/internal/control/fleetmetrics/fleetmetrics.go @@ -430,8 +430,18 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) { byClass[class]++ labels := []string{n.ID, strconv.Itoa(g.Index), class} - s, fresh := sensorFor(n.Load, g.Index, now) - if fresh && s.Busy() { + // The sensors are the card's. A MIG instance shares its card's + // heat and power, but not its memory, and has no utilisation of + // its own that any driver reports: its memory is the instance's + // own reading, and its utilisation is left out rather than + // published as the card's or as zero. + s, fresh := sensorFor(n.Load, g.CardIndex(), now) + if g.IsMIG() { + ch <- prometheus.MustNewConstMetric(c.gpuMemoryTotal, prometheus.GaugeValue, + float64(g.MemoryTotalMB)*1024*1024, labels...) + ch <- prometheus.MustNewConstMetric(c.gpuMemoryUsed, prometheus.GaugeValue, + float64(g.MemoryUsedMB)*1024*1024, labels...) + } else if fresh && s.Busy() { // What the card is doing now, sampled every heartbeat. c.gpuBusy(ch, s, labels) } else { diff --git a/internal/control/fleetmetrics/mig_test.go b/internal/control/fleetmetrics/mig_test.go new file mode 100644 index 00000000..bfcabb39 --- /dev/null +++ b/internal/control/fleetmetrics/mig_test.go @@ -0,0 +1,36 @@ +package fleetmetrics + +import ( + "strings" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" +) + +func TestAMIGInstanceHasItsOwnMemoryItsCardsHeatAndNoUtilisation(t *testing.T) { + // Card 0 in MIG mode: the sensor sample is the card's, 60 GB used across + // both instances and a utilisation the driver does not give per instance. + inst := discovery.GPU{Index: 1001, Name: "NVIDIA A100-SXM4-80GB MIG 3g.40gb", MemoryTotalMB: 40192, MemoryUsedMB: 21000, + MIG: &discovery.MIGDevice{ParentIndex: 0, ParentMemoryTotalMB: 81920, Device: 1, Profile: "3g.40gb"}} + n := inventory.Node{ID: "g1", State: inventory.Ready, GPUs: []discovery.GPU{inst}, + Load: &inventory.Load{At: time.Now(), GPUs: []inventory.GPUSensor{{Index: 0, TempC: 61, PowerW: 250, + UtilPercent: 0, MemoryUsedMB: 60000, MemoryTotalMB: 81920}}}} + out := gather(t, fakeSource{nodes: []inventory.Node{n}}) + labels := "class=a100-80g-mig-3g.40gb gpu=1001 node=g1" + for _, want := range []string{ + "llmgw_gpu_memory_used_bytes " + labels + " -> 22020096000\n", + "llmgw_gpu_memory_total_bytes " + labels + " -> 42144366592\n", + "llmgw_gpu_temperature_celsius " + labels + " -> 61\n", + "llmgw_gpu_power_watts " + labels + " -> 250\n", + } { + if !strings.Contains(out, want) { + t.Errorf("missing %s in:\n%s", want, out) + } + } + // Not the card's zero, not anything: absent. + if strings.Contains(out, "llmgw_gpu_utilization_percent "+labels) { + t.Errorf("a utilisation was published for a MIG instance:\n%s", out) + } +} diff --git a/internal/control/placement/mig_test.go b/internal/control/placement/mig_test.go new file mode 100644 index 00000000..4120d5fa --- /dev/null +++ b/internal/control/placement/mig_test.go @@ -0,0 +1,87 @@ +package placement + +import ( + "slices" + "strings" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" +) + +// migNode is an A100 80GB node as discovery reports it with card 0 cut into +// two 3g.40gb instances and card 1 whole. +func migNode(id string) inventory.Node { + card := discovery.GPU{Name: "NVIDIA A100-SXM4-80GB", MemoryTotalMB: 81920, ComputeCapability: "8.0"} + nd := inventory.Node{ID: id, State: inventory.Ready} + for dev, uuid := range []string{"MIG-c6d4f1ef", "MIG-0f7c3a2e"} { + g := card + g.Index, g.UUID, g.MemoryTotalMB = discovery.MIGIndex(0, dev), uuid, 40192 + g.Name += " MIG 3g.40gb" + g.MIG = &discovery.MIGDevice{ParentIndex: 0, ParentMemoryTotalMB: 81920, Device: dev, Profile: "3g.40gb"} + nd.GPUs = append(nd.GPUs, g) + } + whole := card + whole.Index, whole.UUID = 1, "GPU-whole" + // NVLink to the card in MIG mode, as topo -m still reports it: nothing + // may take that for a link an instance can use. + whole.NVLinkPeers = []int{0} + nd.GPUs = append(nd.GPUs, whole) + return nd +} + +func TestAModelIsPlacedOnAMIGInstanceOfItsClass(t *testing.T) { + st := store(migNode("n1")) + s := New(st) + p := Profile{Name: "mig", GPUClass: "a100-80g-mig-3g.40gb", GPUsPerNode: 1} + + res := s.Place([]Profile{p}, nil) + if res.Placement == nil { + t.Fatalf("no placement:\n%s", res.Explain()) + } + got := res.Placement.Nodes[0].GPUIndices + if !slices.Equal(got, []int{1000}) { + t.Fatalf("placed on %v, want MIG instance 1000", got) + } + if res.Placement.Pool != "none/a100-80g-mig-3g.40gb" { + t.Errorf("pool %q", res.Placement.Pool) + } + + // The next replica takes the other instance, and the one after finds + // none: the whole card beside them is another class. + st.Allocate("n1", inventory.Allocation{ShardID: "s1", ReplicaID: "r1", GPUIndices: got}) + res = s.Place([]Profile{p}, nil) + if res.Placement == nil || !slices.Equal(res.Placement.Nodes[0].GPUIndices, []int{1001}) { + t.Fatalf("second replica: %+v\n%s", res.Placement, res.Explain()) + } + st.Allocate("n1", inventory.Allocation{ShardID: "s2", ReplicaID: "r2", GPUIndices: []int{1001}}) + if res = s.Place([]Profile{p}, nil); res.Placement != nil { + t.Errorf("a third replica was placed on %v", res.Placement.Nodes[0].GPUIndices) + } + + // A whole-card profile takes the whole card, never an instance. + res = s.Place([]Profile{{Name: "whole", GPUClass: "a100-80g", GPUsPerNode: 1}}, nil) + if res.Placement == nil || !slices.Equal(res.Placement.Nodes[0].GPUIndices, []int{1}) { + t.Errorf("whole-card profile: %+v\n%s", res.Placement, res.Explain()) + } +} + +func TestMoreThanOneMIGInstanceForAReplicaIsRefused(t *testing.T) { + // Two instances are free on one node, which is what a tp 2 profile would + // otherwise be given — and fail at start, since a process opens only one + // MIG device and instances have no NVLink between them. + s := New(store(migNode("n1"), migNode("n2"))) + for _, p := range []Profile{ + {Name: "tp2", GPUClass: "a100-80g-mig-3g.40gb", GPUsPerNode: 2}, + {Name: "two-nodes", GPUClass: "a100-80g-mig-3g.40gb", GPUsPerNode: 1, Nodes: 2}, + } { + res := s.Place([]Profile{p}, nil) + if res.Placement != nil { + t.Errorf("%s was placed: %+v", p.Name, res.Placement.Nodes) + continue + } + if !strings.Contains(res.Explain(), "a replica runs on exactly one") { + t.Errorf("%s refused without saying why:\n%s", p.Name, res.Explain()) + } + } +} diff --git a/internal/control/placement/placement.go b/internal/control/placement/placement.go index fdb92d7a..6a57db1e 100644 --- a/internal/control/placement/placement.go +++ b/internal/control/placement/placement.go @@ -156,6 +156,15 @@ func (s *Scheduler) Place(profiles []Profile, avoid []string) Result { for _, raw := range ordered { p := raw.normalize() + // The catalog refuses these; a profile that reaches here some other + // way is refused again rather than placed on instances that cannot + // work together. MIG instances are placed one to a replica. + if discovery.IsMIGClass(p.GPUClass) && (p.GPUsPerNode > 1 || p.Nodes > 1) { + res.Rejections = append(res.Rejections, Rejection{Profile: p.Name, Pool: "none/" + p.GPUClass, + Reason: fmt.Sprintf("asks for %d MIG instances on %d nodes, but a replica runs on exactly one", + p.GPUsPerNode*p.Nodes, p.Nodes)}) + continue + } if p.Nodes == 1 { if pl, rej := s.placeSingleNode(p, avoid); pl != nil { res.Placement = pl diff --git a/internal/doctor/doctor.go b/internal/doctor/doctor.go index a23533df..b32b4f4a 100644 --- a/internal/doctor/doctor.go +++ b/internal/doctor/doctor.go @@ -17,6 +17,7 @@ import ( "path/filepath" "slices" "sort" + "strconv" "strings" "time" @@ -57,6 +58,10 @@ type Facts struct { GPUError string `json:"gpu_error,omitempty"` TopoRaw string `json:"topo_raw,omitempty"` TopoErr string `json:"topo_error,omitempty"` + // MIG is each MIG-capable card's mode and instances; MIGErr why it could + // not be read. + MIG []discovery.MIGCard `json:"mig,omitempty"` + MIGErr string `json:"mig_error,omitempty"` // NVSwitch is set when the machine has NVSwitch devices, which need the // fabric manager running before CUDA will start. NVSwitch bool `json:"nvswitch"` @@ -150,6 +155,16 @@ func Collect(ctx context.Context, o Options) Facts { } f.GPUs = gpus f.IMEX = discovery.DiscoverIMEX(ctx) + if slices.ContainsFunc(gpus, func(g discovery.GPU) bool { + cc, _ := strconv.ParseFloat(g.ComputeCapability, 64) + return g.IsMIG() || cc >= 8 + }) { + if mig, err := discovery.DiscoverMIG(ctx, smi); err != nil { + f.MIGErr = err.Error() + } else { + f.MIG = mig + } + } } if m, _ := filepath.Glob("/dev/nvidia-nvswitch*"); len(m) > 0 { f.NVSwitch = true @@ -367,17 +382,22 @@ func Evaluate(f Facts) []Finding { add(OK, "gpu", fmt.Sprintf("%d GPUs: %s", len(f.GPUs), gpuNames(f.GPUs)), "") for _, g := range f.GPUs { if !g.Health.OK() { - add(Warn, "gpu", fmt.Sprintf("GPU %d is kept out of placement: %s", g.Index, strings.Join(g.Health.Problems, "; ")), "") + add(Warn, "gpu", fmt.Sprintf("GPU %s is kept out of placement: %s", + discovery.DeviceLabel(g.Index), strings.Join(g.Health.Problems, "; ")), "") } } } + evaluateMIG(f, add) if f.NVSwitch && f.FabricManager != "active" { add(Fail, "nvlink", "this machine has NVSwitch but nvidia-fabricmanager is "+f.FabricManager+ ": CUDA will not initialise on its GPUs", "systemctl enable --now nvidia-fabricmanager (its version must match the driver's)") } - if len(f.GPUs) > 1 { - pairs, groups := nvlinkGroups(f.GPUs) + // NVLink is between whole cards: a MIG instance has none to report, and + // counting instances would call a node of MIG cards one without NVLink. + cards := slices.DeleteFunc(slices.Clone(f.GPUs), discovery.GPU.IsMIG) + if len(cards) > 1 { + pairs, groups := nvlinkGroups(cards) switch { case f.TopoErr != "": add(Warn, "nvlink", "could not read nvidia-smi topo -m: "+f.TopoErr+ @@ -393,7 +413,7 @@ func Evaluate(f Facts) []Finding { // --- multi-node NVLink --- domain := "" - for _, g := range f.GPUs { + for _, g := range cards { fb := g.Fabric if fb == nil || strings.EqualFold(fb.State, "Not Supported") { continue @@ -662,6 +682,64 @@ func rank(l Level) int { return 3 } +// evaluateMIG reports each MIG-capable card's mode and instances. Quiet for a +// card with MIG off, which is every card on most machines; what it warns of +// are the states in which a card is in MIG mode and nothing can run on it. +func evaluateMIG(f Facts, add func(Level, string, string, string)) { + if f.MIGErr != "" { + add(Warn, "mig", "could not read the cards' MIG state: "+f.MIGErr, "run nvidia-smi -q by hand") + return + } + for _, c := range f.MIG { + card := fmt.Sprintf("GPU %d (%s)", c.Index, c.Name) + if c.Enabled != c.PendingEnabled { + want := "off" + if c.PendingEnabled { + want = "on" + } + add(Warn, "mig", fmt.Sprintf("%s has MIG turning %s at its next reset, and until then runs as before", card, want), + fmt.Sprintf("reset the card when nothing runs on it: nvidia-smi -i %d -r (or reboot)", c.Index)) + } + if !c.Enabled { + continue + } + // A GPU instance with no compute instance is carved but not usable: + // it is not a device until one is made inside it. + var bare []string + for _, gi := range c.GPUInstances { + has := slices.ContainsFunc(c.ComputeInstances, func(ci discovery.MIGInstance) bool { + return ci.GPUInstance == gi.GPUInstance + }) + if !has { + bare = append(bare, fmt.Sprintf("%d (%s)", gi.GPUInstance, gi.Profile)) + } + } + switch { + case len(c.Instances) == 0 && len(c.GPUInstances) == 0: + add(Warn, "mig", card+" is in MIG mode with no instances created: nothing can run on it", + fmt.Sprintf("create instances, for example nvidia-smi mig -i %d -cgi 3g.40gb,3g.40gb -C "+ + "(nvidia-smi mig -i %d -lgip lists the profiles), or turn MIG off: nvidia-smi -i %d -mig 0", + c.Index, c.Index, c.Index)) + case len(c.Instances) == 0: + add(Warn, "mig", fmt.Sprintf("%s is in MIG mode with GPU instances %s but no compute instances: nothing can run on it", + card, strings.Join(bare, ", ")), + fmt.Sprintf("create them: nvidia-smi mig -i %d -cci", c.Index)) + default: + var names []string + for _, in := range c.Instances { + names = append(names, fmt.Sprintf("%d: %s %s", in.Device, in.Profile, in.UUID)) + } + add(OK, "mig", fmt.Sprintf("%s is in MIG mode with %d instance(s), each a device of its own: %s", + card, len(c.Instances), strings.Join(names, "; ")), "") + if len(bare) > 0 { + add(Warn, "mig", fmt.Sprintf("%s has GPU instances %s with no compute instance, which are not devices yet", + card, strings.Join(bare, ", ")), + fmt.Sprintf("create them: nvidia-smi mig -i %d -cci, or destroy the GPU instances", c.Index)) + } + } + } +} + func gpuNames(gpus []discovery.GPU) string { count := map[string]int{} var order []string diff --git a/internal/doctor/mig_test.go b/internal/doctor/mig_test.go new file mode 100644 index 00000000..3ceba6d2 --- /dev/null +++ b/internal/doctor/mig_test.go @@ -0,0 +1,80 @@ +package doctor + +import ( + "strings" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" +) + +// migFacts is the healthy node with card 0 in MIG mode, as given. +func migFacts(card discovery.MIGCard) Facts { + f := healthy() + f.MIG = []discovery.MIGCard{card} + return f +} + +func TestTheDoctorListsACardsMIGInstances(t *testing.T) { + fs := Evaluate(migFacts(discovery.MIGCard{Index: 0, Name: "NVIDIA A100-SXM4-80GB", Enabled: true, PendingEnabled: true, + Instances: []discovery.MIGListed{{Device: 0, Profile: "3g.40gb", UUID: "MIG-aaa"}, {Device: 1, Profile: "3g.40gb", UUID: "MIG-bbb"}}, + GPUInstances: []discovery.MIGInstance{{GPU: 0, Profile: "3g.40gb", GPUInstance: 1, ComputeInstance: -1}, + {GPU: 0, Profile: "3g.40gb", GPUInstance: 2, ComputeInstance: -1}}, + ComputeInstances: []discovery.MIGInstance{{GPU: 0, Profile: "3g.40gb", GPUInstance: 1}, + {GPU: 0, Profile: "3g.40gb", GPUInstance: 2}}, + })) + has(t, fs, OK, "GPU 0 (NVIDIA A100-SXM4-80GB) is in MIG mode with 2 instance(s)") + has(t, fs, OK, "1: 3g.40gb MIG-bbb") + for _, f := range fs { + if f.Area == "mig" && f.Level != OK { + t.Errorf("a card with its instances made was warned of: %+v", f) + } + } +} + +func TestTheDoctorWarnsOfMIGEnabledWithNoInstances(t *testing.T) { + // The state a card is left in by nvidia-smi -mig 1 alone: in MIG mode, + // so CUDA will not open it whole, and nothing carved to open instead. + fs := Evaluate(migFacts(discovery.MIGCard{Index: 0, Name: "NVIDIA A100-SXM4-80GB", Enabled: true, PendingEnabled: true})) + has(t, fs, Warn, "is in MIG mode with no instances created") + for _, f := range fs { + if f.Area == "mig" && f.Level == Warn && !strings.Contains(f.Fix, "nvidia-smi mig -i 0 -cgi") { + t.Errorf("the fix does not say how to create them: %q", f.Fix) + } + } + + // GPU instances with no compute instance in them are not devices either. + fs = Evaluate(migFacts(discovery.MIGCard{Index: 0, Name: "A100", Enabled: true, PendingEnabled: true, + GPUInstances: []discovery.MIGInstance{{GPU: 0, Profile: "7g.80gb", GPUInstance: 0, ComputeInstance: -1}}})) + has(t, fs, Warn, "GPU instances 0 (7g.80gb) but no compute instances") + + // A mode change waiting for a reset. + fs = Evaluate(migFacts(discovery.MIGCard{Index: 0, Name: "A100", Enabled: false, PendingEnabled: true})) + has(t, fs, Warn, "MIG turning on at its next reset") +} + +func TestACardWithMIGOffIsNotMentioned(t *testing.T) { + fs := Evaluate(migFacts(discovery.MIGCard{Index: 0, Name: "A100"})) + for _, f := range fs { + if f.Area == "mig" { + t.Errorf("a card with MIG off was reported: %+v", f) + } + } +} + +func TestNVLinkIsJudgedBetweenWholeCardsOnly(t *testing.T) { + // Every card of the node in MIG mode: the instances have no NVLink, and + // that is not a node "without NVLink" to warn about. + f := healthy() + f.GPUs = nil + for card := range 2 { + for dev := range 2 { + f.GPUs = append(f.GPUs, discovery.GPU{Index: discovery.MIGIndex(card, dev), Name: "A100 MIG 3g.40gb", + MIG: &discovery.MIGDevice{ParentIndex: card, Device: dev, Profile: "3g.40gb"}}) + } + } + for _, x := range Evaluate(f) { + if x.Area == "nvlink" && x.Level != OK { + t.Errorf("MIG instances were judged for NVLink: %+v", x) + } + } +} diff --git a/internal/modelplan/fit.go b/internal/modelplan/fit.go index 8a652b2b..3dacb53f 100644 --- a/internal/modelplan/fit.go +++ b/internal/modelplan/fit.go @@ -132,7 +132,14 @@ func fitsOn(m Model, h Hardware, d Demand, p Precision, context int) []Fit { // Powers of two only. Tensor parallelism divides attention heads across // ranks, and an engine that cannot divide them evenly refuses the profile — // so offering tp=3 would produce a plan that fails at launch. - for width := 1; width <= h.GPUsPerNode*h.Nodes; width *= 2 { + // + // One wide on MIG instances, however many there are: a process opens + // only one, and they have no NVLink between them. + widest := h.GPUsPerNode * h.Nodes + if isMIG(h.Class) { + widest = min(widest, 1) + } + for width := 1; width <= widest; width *= 2 { nodes := 1 perNode := width if width > h.GPUsPerNode { diff --git a/internal/modelplan/mig_test.go b/internal/modelplan/mig_test.go new file mode 100644 index 00000000..289449e5 --- /dev/null +++ b/internal/modelplan/mig_test.go @@ -0,0 +1,69 @@ +package modelplan + +import ( + "strings" + "testing" +) + +const mig3g = "a100-80g-mig-3g.40gb" + +func TestAMIGInstanceIsSizedByItsProfileNotItsCard(t *testing.T) { + for class, want := range map[string]int{ + mig3g: 40, "h100-80g-mig-1g.10gb": 10, "h100-80g-mig-1g.10gb-me": 10, "a100-80g": 80, "a100-80g-mig-": 0, + } { + if got := MemFromClass(class); got != want { + t.Errorf("MemFromClass(%q) = %d, want %d", class, got, want) + } + } +} + +func TestAModelThatFitsOneMIGInstanceIsPlannedOnOne(t *testing.T) { + // Four instances free on each of two nodes on a fabric: a 3B fits one, + // and a 70B, which no single instance holds, is not spread over several + // — a replica cannot use more than one. + hw := []Hardware{{Class: mig3g, GPUMemGB: 40, GPUsPerNode: 4, Nodes: 2, Fabric: "ib-a"}} + fits := Fits(coder3B, hw, 32768, []Precision{BF16}) + if len(fits) == 0 { + t.Fatal("a 3B does not fit a 40 GB MIG instance") + } + for _, f := range fits { + if f.TP != 1 || f.Nodes != 1 { + t.Errorf("planned across MIG instances: %s", f.Describe()) + } + } + if fits := Fits(llama70B, hw, 8192, []Precision{BF16}); len(fits) != 0 { + t.Errorf("a 70B was planned on MIG instances: %s", fits[0].Describe()) + } +} + +func TestANodeWithACardInMIGModeOffersEachOfItsClasses(t *testing.T) { + // Card 0 cut into two 3g.40gb instances, card 1 whole: two classes on + // one node, each with its own count. + nodes := []PoolNode{{ID: "a100", Class: mig3g, GPUs: 3, FreeGPUs: 3, HostMemGB: 512, HostFreeGB: 400, + Kinds: []PoolKind{ + {Class: mig3g, GPUMemGB: 40, GPUs: 2, FreeGPUs: 2}, + {Class: "a100-80g", GPUMemGB: 80, GPUs: 1, FreeGPUs: 1}, + }}} + arts := []Artifact{ + {Name: "q4.gguf", Format: "gguf", Bytes: 9 * gib}, + {Name: "f16.gguf", Format: "gguf", Bytes: 29 * gib}, + {Name: "big.gguf", Format: "gguf", Bytes: 60 * gib}, + } + opts := Options(arts, shape14B, nodes, 16384) + got := strings.Join(labels(opts), "; ") + for _, want := range []string{"q4.gguf 1 × " + mig3g, "f16.gguf 1 × " + mig3g, "big.gguf 1 × a100-80g"} { + if !strings.Contains(got, want) { + t.Errorf("missing %q in %s", want, got) + } + } + // Sixty gigabytes is more than an instance holds, and two instances are + // not one bigger device. + if strings.Contains(got, "big.gguf 2 × "+mig3g) { + t.Errorf("planned across two MIG instances: %s", got) + } + for _, o := range opts { + if o.Class == mig3g && (o.Hosts != 1 || !o.FreeNow) { + t.Errorf("the MIG option counts %d hosts, free %v", o.Hosts, o.FreeNow) + } + } +} diff --git a/internal/modelplan/options.go b/internal/modelplan/options.go index 0265d222..bb1217d9 100644 --- a/internal/modelplan/options.go +++ b/internal/modelplan/options.go @@ -46,8 +46,35 @@ type PoolNode struct { // HostMemGB is its RAM, and HostFreeGB what is unused now. HostMemGB int HostFreeGB int + // Kinds are the node's devices by class, for a node with more than one: + // a card in MIG mode is one or several classes of instance, and a whole + // card beside it another. Empty means Class, GPUMemGB, GPUs and FreeGPUs + // say it all. + Kinds []PoolKind } +// PoolKind is one class of a node's devices. +type PoolKind struct { + Class string + GPUMemGB int + GPUs int + FreeGPUs int +} + +// kinds is the node's devices by class. +func (n PoolNode) kinds() []PoolKind { + if len(n.Kinds) > 0 { + return n.Kinds + } + return []PoolKind{{Class: n.Class, GPUMemGB: n.GPUMemGB, GPUs: n.GPUs, FreeGPUs: n.FreeGPUs}} +} + +// isMIG reports whether a class names MIG instances: "a100-80g-mig-3g.40gb". +// A replica runs on exactly one, never a group of them, so a plan offers one +// instance and nothing wider. Read from the class label, as the card's memory +// is, which keeps this package free of the agent's. +func isMIG(class string) bool { return strings.Contains(class, "-mig-") } + // Option is one way to run one artifact. type Option struct { Artifact string `json:"artifact"` @@ -149,11 +176,13 @@ func ggufOptions(a Artifact, kv int64, nodes []PoolNode, bpw float64) []Option { } o := Option{Artifact: a.Name, Format: a.Format, Engine: "llamacpp", Class: c.class, GPUsPerNode: width, Nodes: 1, NeedGB: GB(per), HaveGB: budget, BitsPerWeight: bpw} - for _, n := range nodes { - if n.Class == c.class && n.GPUs >= width { - o.Hosts++ - if n.FreeGPUs >= width { - o.FreeNow = true + for _, fab := range c.fabrics { + for _, n := range fab.nodes { + if n.GPUs >= width { + o.Hosts++ + if n.FreeGPUs >= width { + o.FreeNow = true + } } } } @@ -196,7 +225,9 @@ func vllmOptions(a Artifact, kv int64, sh Shape, nodes []PoolNode, bpw float64) single = true break } - if single { + // A MIG instance does not join others, on its node or across a + // fabric: what does not fit one does not fit the class. + if single || isMIG(c.class) { continue } // Several nodes, a node's full width on each, pipeline-parallel @@ -251,32 +282,42 @@ type fabricPool struct { } // classes groups GPU nodes by card, and each by fabric: a replica spans -// nodes only within one. +// nodes only within one. A node with devices of several classes is in each +// class's pool, as the node it is for that class: its devices of that class +// and no others. func classes(nodes []PoolNode) []classPool { by := map[string]*classPool{} var order []string - for _, n := range nodes { - if n.Class == "" || n.GPUs == 0 || n.GPUMemGB == 0 { - continue - } - c := by[n.Class] - if c == nil { - c = &classPool{class: n.Class, mem: n.GPUMemGB} - by[n.Class] = c - order = append(order, n.Class) - } - c.maxGPUs = max(c.maxGPUs, n.GPUs) - i := -1 - for j := range c.fabrics { - if c.fabrics[j].name == n.Fabric { - i = j + for _, node := range nodes { + for _, k := range node.kinds() { + if k.Class == "" || k.GPUs == 0 || k.GPUMemGB == 0 { + continue } + n := node + n.Class, n.GPUMemGB, n.GPUs, n.FreeGPUs, n.Kinds = k.Class, k.GPUMemGB, k.GPUs, k.FreeGPUs, nil + if isMIG(n.Class) { + // One instance a replica, however many the node has. + n.GPUs, n.FreeGPUs = min(n.GPUs, 1), min(n.FreeGPUs, 1) + } + c := by[n.Class] + if c == nil { + c = &classPool{class: n.Class, mem: n.GPUMemGB} + by[n.Class] = c + order = append(order, n.Class) + } + c.maxGPUs = max(c.maxGPUs, n.GPUs) + i := -1 + for j := range c.fabrics { + if c.fabrics[j].name == n.Fabric { + i = j + } + } + if i < 0 { + c.fabrics = append(c.fabrics, fabricPool{name: n.Fabric}) + i = len(c.fabrics) - 1 + } + c.fabrics[i].nodes = append(c.fabrics[i].nodes, n) } - if i < 0 { - c.fabrics = append(c.fabrics, fabricPool{name: n.Fabric}) - i = len(c.fabrics) - 1 - } - c.fabrics[i].nodes = append(c.fabrics[i].nodes, n) } sort.Strings(order) out := make([]classPool, 0, len(order)) @@ -363,8 +404,17 @@ func QuantOf(name string) string { return strings.ToUpper(m[len(m)-1]) } -// MemFromClass reads the card size out of a class label such as "h100-80g". +// MemFromClass reads the card size out of a class label such as "h100-80g", +// and a MIG instance's out of its profile: 40 for "a100-80g-mig-3g.40gb", +// whose card is 80 but whose instance holds 40 of it. func MemFromClass(class string) int { + if _, profile, ok := strings.Cut(class, "-mig-"); ok { + var compute, gb int + if _, err := fmt.Sscanf(profile, "%dg.%dgb", &compute, &gb); err != nil { + return 0 + } + return gb + } i := strings.LastIndex(class, "-") if i < 0 || !strings.HasSuffix(class, "g") { return 0 diff --git a/notes/c22.md b/notes/c22.md new file mode 100644 index 00000000..09075209 --- /dev/null +++ b/notes/c22.md @@ -0,0 +1,54 @@ +# C22: MIG + +## CHANGELOG + +### MIG instances as GPUs of their own class + +- **A card in MIG mode is placed as its instances.** Discovery reports a card + in MIG mode as its MIG instances instead of the whole card, each a device of + its own with its UUID (`MIG-…`), its own memory, and a class named from the + card's and the profile: two `3g.40gb` instances of an A100 80GB are two + devices of class `a100-80g-mig-3g.40gb`. A profile names that class like + any other, and placement puts a replica on a free instance of it. Both + discovery paths do it: nvidia-smi (instances from `nvidia-smi -L`, their + memory and ids from `nvidia-smi -q -x`) and NVML (`GetMigMode`, + `GetMaxMigDeviceCount`, `GetMigDeviceHandleByIndex`). llmgw does not create + instances; `docs/install.md` has a MIG section on making them. +- **One instance a replica.** A process opens only one MIG device, and + instances have no NVLink or peer-to-peer between them, so a profile on a + MIG class asking for more than one GPU, tp or pp above 1, or more than one + node is refused when the model is added, with that reason; placement + refuses it again. `ctl model plan` and `ctl model serve` plan an instance + for a model one holds and never several together. +- **Engines are given the instance by UUID.** Placement numbers card 0's + instance 2 as device 1002 (1000 + 100 × card + instance); the agent turns + that into `CUDA_VISIBLE_DEVICES=MIG-` for vLLM and llama.cpp, and + leaves `GGML_VK_VISIBLE_DEVICES` unset (Vulkan cannot see MIG). Every view + shows the device as `0/mig2`. +- **Memory per instance, no made-up utilisation.** Each instance's memory, + and the processes holding it, are its own — read per GPU and compute + instance from nvidia-smi's report, so a foreign process on one instance + keeps that instance out of placement and not its neighbours. Temperature, + power and wear are the card's. No driver reports utilisation per instance, + so none is published for one: the console says `n/a on MIG`, and + `llmgw_gpu_utilization_percent` has no series for it rather than a zero. +- **A card in MIG mode with nothing carved is kept out of placement**, with + the reason in `ctl nodes`, since CUDA will not open the whole card while MIG + is on. +- **`llmgw doctor` reports MIG**: each card's mode and instances, and warns + of a card in MIG mode with no instances, of GPU instances with no compute + instance, and of a mode change waiting for a reset, with the `nvidia-smi` + command for each. +- `ctl nodes` counts a node's devices by class when it has MIG instances or + more than one class (`a100-80g-mig-3g.40gb×2,a100-80g×1`); the console's + GPU list shows instances under their card with their own memory and the + card's heat and power, and a node's page lists its classes with counts. + `ctl status` and the plan count each device under its own class, where a + node's cards were all put under its first class before. +- The agent protocol's `GPU` gains `mig` (parent card, its memory, profile, + GPU and compute instance ids); the node API's GPUs gain `label`, `class`, + `card`, `mig_profile` and memory, and nodes gain `gpus_by_class`. + +## BACKLOG + +**Done (branch `features-c2`):** MIG instances discovered on both paths (nvidia-smi `-L` + `-q -x`; NVML through an interface with a faithful fake) and reported instead of their card, each with its MIG UUID, own memory and class `-mig-`; device numbers 1000 + 100 × card + instance, shown as `card/migN`; placement on MIG classes; tp/pp/gpus/nodes above 1 on a MIG class refused in catalog validation and again in placement; `CUDA_VISIBLE_DEVICES=MIG-` for vLLM and llama.cpp (no Vulkan device variable on MIG); per-instance memory and processes in heartbeats and the GPU check, card sensors under each instance, utilisation absent rather than zero; doctor MIG findings (`mig -lgi`/`-lci` parsed); `ctl nodes`, console, `ctl status`, `ctl model plan`/`serve` by class; docs/install.md MIG section. Written and tested against fakes only — untested on real MIG hardware. Left: run it on an A100/H100 in MIG mode (check `-L`/`-q -x`/`mig -lgi`/`-lci` and NVML MIG device names against the parsers, and that `CUDA_VISIBLE_DEVICES=MIG-…` reaches vLLM's workers through Ray); instances changed while the agent runs are only seen when it restarts (no rescan); llmgw does not create or reshape instances (no mig-parted integration); quotas count an instance as one GPU, the same as a whole card; SGLang on MIG not checked; device numbers assume fewer than 100 instances per card and that instance numbers stay put while shards run on them.