Find multi-node NVLink domains, place across them, and show them as racks
A GB200 NVL72 rack or an NVLink Switch system is a scale-up island: its GPUs reach each other over NVLink whatever node they are in. Each GPU now reports its fabric from nvidia-smi -q - state, status, ClusterUUID, CliqueId, bandwidth - at registration and once a minute after, since a fabric manager restarting drops every card out at once; and each node reports nvidia-imex and its channels, without which the domain's GPUs cannot share memory across nodes. A single machine's NVSwitch, which reports a cluster of zeros, is no domain. A replica spanning nodes is placed inside one domain first, on nodes whose every card is joined and whose IMEX is up, and its shards are told NCCL_MNNVL_ENABLE=1; InfiniBand is the fallback. /admin/nvlink draws each domain as a rack - the NVLink Switch band on top, light running down a backplane, trays of GPU chips that glow when joined, pulse red when not, turn amber on degraded bandwidth - and lists what keeps a tray out. A node's page shows its GPU-by-GPU NVLink matrix, each card's NUMA node and nearest adapter, and its domain and IMEX. doctor checks joining, bandwidth, IMEX and its channel; metrics llmgw_gpu_nvlink_joined and llmgw_imex_up; /api/v1/nvlink. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
5bbc619a31
commit
20479a2d5f
31 files changed
+1535
-131
No files matched your search
+356
-122
@@ -90,7 +90,7 @@ func (x ShardStatus_State) Number() protoreflect.EnumNumber {
|
||||
|
||||
// Deprecated: Use ShardStatus_State.Descriptor instead.
|
||||
func (ShardStatus_State) EnumDescriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{17, 0}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{19, 0}
|
||||
}
|
||||
|
||||
// AgentMessage is anything a node tells the control plane.
|
||||
@@ -470,6 +470,8 @@ type Register struct {
|
||||
// meant "an agent too old to say" and constrained nothing — so a freshly
|
||||
// built node with no engine installed yet was a candidate for every model.
|
||||
EnginesProbed bool `protobuf:"varint,14,opt,name=engines_probed,json=enginesProbed,proto3" json:"engines_probed,omitempty"`
|
||||
// imex is the service that lets a multi-node NVLink domain share memory.
|
||||
Imex *IMEX `protobuf:"bytes,15,opt,name=imex,proto3" json:"imex,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
@@ -602,6 +604,13 @@ func (x *Register) GetEnginesProbed() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (x *Register) GetImex() *IMEX {
|
||||
if x != nil {
|
||||
return x.Imex
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Host describes the operating system and processor of a node.
|
||||
type Host struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
@@ -967,9 +976,17 @@ type GPU struct {
|
||||
UtilizationPercent int32 `protobuf:"varint,15,opt,name=utilization_percent,json=utilizationPercent,proto3" json:"utilization_percent,omitempty"`
|
||||
// nic_paths is how far each network adapter is from this card, by HCA
|
||||
// name: PIX, PXB, PHB, NODE or SYS, from nvidia-smi topo.
|
||||
NicPaths map[string]string `protobuf:"bytes,16,rep,name=nic_paths,json=nicPaths,proto3" json:"nic_paths,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
NicPaths map[string]string `protobuf:"bytes,16,rep,name=nic_paths,json=nicPaths,proto3" json:"nic_paths,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
|
||||
// The card's multi-node NVLink membership, from nvidia-smi -q: state is
|
||||
// Completed once the fabric manager has joined it; cluster_uuid and
|
||||
// clique_id name the domain it joined.
|
||||
FabricState string `protobuf:"bytes,17,opt,name=fabric_state,json=fabricState,proto3" json:"fabric_state,omitempty"`
|
||||
FabricStatus string `protobuf:"bytes,18,opt,name=fabric_status,json=fabricStatus,proto3" json:"fabric_status,omitempty"`
|
||||
ClusterUuid string `protobuf:"bytes,19,opt,name=cluster_uuid,json=clusterUuid,proto3" json:"cluster_uuid,omitempty"`
|
||||
CliqueId string `protobuf:"bytes,20,opt,name=clique_id,json=cliqueId,proto3" json:"clique_id,omitempty"`
|
||||
FabricBandwidth string `protobuf:"bytes,21,opt,name=fabric_bandwidth,json=fabricBandwidth,proto3" json:"fabric_bandwidth,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *GPU) Reset() {
|
||||
@@ -1114,6 +1131,41 @@ func (x *GPU) GetNicPaths() map[string]string {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *GPU) GetFabricState() string {
|
||||
if x != nil {
|
||||
return x.FabricState
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *GPU) GetFabricStatus() string {
|
||||
if x != nil {
|
||||
return x.FabricStatus
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *GPU) GetClusterUuid() string {
|
||||
if x != nil {
|
||||
return x.ClusterUuid
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *GPU) GetCliqueId() string {
|
||||
if x != nil {
|
||||
return x.CliqueId
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *GPU) GetFabricBandwidth() string {
|
||||
if x != nil {
|
||||
return x.FabricBandwidth
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type FabricAttachment struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
// fabric is the operator-assigned name. It is not derived from the GID
|
||||
@@ -1685,6 +1737,145 @@ func (x *SubnetManager) GetState() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// IMEX is the nvidia-imex service and its channels.
|
||||
type IMEX struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
// state is systemd's word for it, empty where it is not installed.
|
||||
State string `protobuf:"bytes,1,opt,name=state,proto3" json:"state,omitempty"`
|
||||
Channels int32 `protobuf:"varint,2,opt,name=channels,proto3" json:"channels,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *IMEX) Reset() {
|
||||
*x = IMEX{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[13]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
||||
func (x *IMEX) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*IMEX) ProtoMessage() {}
|
||||
|
||||
func (x *IMEX) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[13]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use IMEX.ProtoReflect.Descriptor instead.
|
||||
func (*IMEX) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13}
|
||||
}
|
||||
|
||||
func (x *IMEX) GetState() string {
|
||||
if x != nil {
|
||||
return x.State
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *IMEX) GetChannels() int32 {
|
||||
if x != nil {
|
||||
return x.Channels
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// GPUFabricState is one GPU's NVLink domain membership.
|
||||
type GPUFabricState struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
Index int32 `protobuf:"varint,1,opt,name=index,proto3" json:"index,omitempty"`
|
||||
State string `protobuf:"bytes,2,opt,name=state,proto3" json:"state,omitempty"`
|
||||
Status string `protobuf:"bytes,3,opt,name=status,proto3" json:"status,omitempty"`
|
||||
ClusterUuid string `protobuf:"bytes,4,opt,name=cluster_uuid,json=clusterUuid,proto3" json:"cluster_uuid,omitempty"`
|
||||
CliqueId string `protobuf:"bytes,5,opt,name=clique_id,json=cliqueId,proto3" json:"clique_id,omitempty"`
|
||||
Bandwidth string `protobuf:"bytes,6,opt,name=bandwidth,proto3" json:"bandwidth,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *GPUFabricState) Reset() {
|
||||
*x = GPUFabricState{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[14]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
||||
func (x *GPUFabricState) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*GPUFabricState) ProtoMessage() {}
|
||||
|
||||
func (x *GPUFabricState) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[14]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use GPUFabricState.ProtoReflect.Descriptor instead.
|
||||
func (*GPUFabricState) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14}
|
||||
}
|
||||
|
||||
func (x *GPUFabricState) GetIndex() int32 {
|
||||
if x != nil {
|
||||
return x.Index
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *GPUFabricState) GetState() string {
|
||||
if x != nil {
|
||||
return x.State
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *GPUFabricState) GetStatus() string {
|
||||
if x != nil {
|
||||
return x.Status
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *GPUFabricState) GetClusterUuid() string {
|
||||
if x != nil {
|
||||
return x.ClusterUuid
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *GPUFabricState) GetCliqueId() string {
|
||||
if x != nil {
|
||||
return x.CliqueId
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *GPUFabricState) GetBandwidth() string {
|
||||
if x != nil {
|
||||
return x.Bandwidth
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type Heartbeat struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
NodeId string `protobuf:"bytes,1,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"`
|
||||
@@ -1734,13 +1925,18 @@ type Heartbeat struct {
|
||||
// port going down takes its fabric away, and one coming back returns it.
|
||||
FabricsChanged bool `protobuf:"varint,14,opt,name=fabrics_changed,json=fabricsChanged,proto3" json:"fabrics_changed,omitempty"`
|
||||
Fabrics []*FabricAttachment `protobuf:"bytes,15,rep,name=fabrics,proto3" json:"fabrics,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
// gpu_fabric is each GPU's NVLink domain membership as it stands now, sent
|
||||
// once a minute: a fabric manager restarting drops every card out of its
|
||||
// domain, and the node has to stop being offered work that spans it.
|
||||
GpuFabric []*GPUFabricState `protobuf:"bytes,16,rep,name=gpu_fabric,json=gpuFabric,proto3" json:"gpu_fabric,omitempty"`
|
||||
Imex *IMEX `protobuf:"bytes,17,opt,name=imex,proto3" json:"imex,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *Heartbeat) Reset() {
|
||||
*x = Heartbeat{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[13]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[15]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -1752,7 +1948,7 @@ func (x *Heartbeat) String() string {
|
||||
func (*Heartbeat) ProtoMessage() {}
|
||||
|
||||
func (x *Heartbeat) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[13]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[15]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -1765,7 +1961,7 @@ func (x *Heartbeat) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use Heartbeat.ProtoReflect.Descriptor instead.
|
||||
func (*Heartbeat) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15}
|
||||
}
|
||||
|
||||
func (x *Heartbeat) GetNodeId() string {
|
||||
@@ -1873,6 +2069,20 @@ func (x *Heartbeat) GetFabrics() []*FabricAttachment {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *Heartbeat) GetGpuFabric() []*GPUFabricState {
|
||||
if x != nil {
|
||||
return x.GpuFabric
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *Heartbeat) GetImex() *IMEX {
|
||||
if x != nil {
|
||||
return x.Imex
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Provision is the software a node should have: which engines, at which
|
||||
// versions. Nothing more.
|
||||
//
|
||||
@@ -1892,7 +2102,7 @@ type Provision struct {
|
||||
|
||||
func (x *Provision) Reset() {
|
||||
*x = Provision{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[14]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[16]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -1904,7 +2114,7 @@ func (x *Provision) String() string {
|
||||
func (*Provision) ProtoMessage() {}
|
||||
|
||||
func (x *Provision) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[14]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[16]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -1917,7 +2127,7 @@ func (x *Provision) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use Provision.ProtoReflect.Descriptor instead.
|
||||
func (*Provision) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16}
|
||||
}
|
||||
|
||||
func (x *Provision) GetEngines() []*EngineWant {
|
||||
@@ -1938,7 +2148,7 @@ type EngineWant struct {
|
||||
|
||||
func (x *EngineWant) Reset() {
|
||||
*x = EngineWant{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[15]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[17]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -1950,7 +2160,7 @@ func (x *EngineWant) String() string {
|
||||
func (*EngineWant) ProtoMessage() {}
|
||||
|
||||
func (x *EngineWant) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[15]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[17]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -1963,7 +2173,7 @@ func (x *EngineWant) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use EngineWant.ProtoReflect.Descriptor instead.
|
||||
func (*EngineWant) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{17}
|
||||
}
|
||||
|
||||
func (x *EngineWant) GetEngine() string {
|
||||
@@ -1996,7 +2206,7 @@ type ProvisionStatus struct {
|
||||
|
||||
func (x *ProvisionStatus) Reset() {
|
||||
*x = ProvisionStatus{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[16]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[18]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -2008,7 +2218,7 @@ func (x *ProvisionStatus) String() string {
|
||||
func (*ProvisionStatus) ProtoMessage() {}
|
||||
|
||||
func (x *ProvisionStatus) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[16]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[18]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -2021,7 +2231,7 @@ func (x *ProvisionStatus) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use ProvisionStatus.ProtoReflect.Descriptor instead.
|
||||
func (*ProvisionStatus) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{18}
|
||||
}
|
||||
|
||||
func (x *ProvisionStatus) GetEngine() string {
|
||||
@@ -2085,7 +2295,7 @@ type ShardStatus struct {
|
||||
|
||||
func (x *ShardStatus) Reset() {
|
||||
*x = ShardStatus{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[17]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[19]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -2097,7 +2307,7 @@ func (x *ShardStatus) String() string {
|
||||
func (*ShardStatus) ProtoMessage() {}
|
||||
|
||||
func (x *ShardStatus) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[17]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[19]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -2110,7 +2320,7 @@ func (x *ShardStatus) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use ShardStatus.ProtoReflect.Descriptor instead.
|
||||
func (*ShardStatus) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{17}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{19}
|
||||
}
|
||||
|
||||
func (x *ShardStatus) GetNodeId() string {
|
||||
@@ -2184,7 +2394,7 @@ type FetchProgress struct {
|
||||
|
||||
func (x *FetchProgress) Reset() {
|
||||
*x = FetchProgress{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[18]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[20]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -2196,7 +2406,7 @@ func (x *FetchProgress) String() string {
|
||||
func (*FetchProgress) ProtoMessage() {}
|
||||
|
||||
func (x *FetchProgress) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[18]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[20]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -2209,7 +2419,7 @@ func (x *FetchProgress) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use FetchProgress.ProtoReflect.Descriptor instead.
|
||||
func (*FetchProgress) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{18}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{20}
|
||||
}
|
||||
|
||||
func (x *FetchProgress) GetSource() string {
|
||||
@@ -2265,7 +2475,7 @@ type StartShard struct {
|
||||
|
||||
func (x *StartShard) Reset() {
|
||||
*x = StartShard{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[19]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[21]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -2277,7 +2487,7 @@ func (x *StartShard) String() string {
|
||||
func (*StartShard) ProtoMessage() {}
|
||||
|
||||
func (x *StartShard) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[19]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[21]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -2290,7 +2500,7 @@ func (x *StartShard) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use StartShard.ProtoReflect.Descriptor instead.
|
||||
func (*StartShard) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{19}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{21}
|
||||
}
|
||||
|
||||
func (x *StartShard) GetReplicaId() string {
|
||||
@@ -2417,7 +2627,7 @@ type EngineProfile struct {
|
||||
|
||||
func (x *EngineProfile) Reset() {
|
||||
*x = EngineProfile{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[20]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[22]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -2429,7 +2639,7 @@ func (x *EngineProfile) String() string {
|
||||
func (*EngineProfile) ProtoMessage() {}
|
||||
|
||||
func (x *EngineProfile) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[20]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[22]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -2442,7 +2652,7 @@ func (x *EngineProfile) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use EngineProfile.ProtoReflect.Descriptor instead.
|
||||
func (*EngineProfile) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{20}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{22}
|
||||
}
|
||||
|
||||
func (x *EngineProfile) GetName() string {
|
||||
@@ -2538,7 +2748,7 @@ type RemoveWeights struct {
|
||||
|
||||
func (x *RemoveWeights) Reset() {
|
||||
*x = RemoveWeights{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[21]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[23]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -2550,7 +2760,7 @@ func (x *RemoveWeights) String() string {
|
||||
func (*RemoveWeights) ProtoMessage() {}
|
||||
|
||||
func (x *RemoveWeights) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[21]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[23]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -2563,7 +2773,7 @@ func (x *RemoveWeights) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use RemoveWeights.ProtoReflect.Descriptor instead.
|
||||
func (*RemoveWeights) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{21}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{23}
|
||||
}
|
||||
|
||||
func (x *RemoveWeights) GetPaths() []string {
|
||||
@@ -2583,7 +2793,7 @@ type StopShard struct {
|
||||
|
||||
func (x *StopShard) Reset() {
|
||||
*x = StopShard{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[22]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[24]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -2595,7 +2805,7 @@ func (x *StopShard) String() string {
|
||||
func (*StopShard) ProtoMessage() {}
|
||||
|
||||
func (x *StopShard) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[22]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[24]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -2608,7 +2818,7 @@ func (x *StopShard) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use StopShard.ProtoReflect.Descriptor instead.
|
||||
func (*StopShard) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{22}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{24}
|
||||
}
|
||||
|
||||
func (x *StopShard) GetShardId() string {
|
||||
@@ -2628,7 +2838,7 @@ type DrainShard struct {
|
||||
|
||||
func (x *DrainShard) Reset() {
|
||||
*x = DrainShard{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[23]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[25]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -2640,7 +2850,7 @@ func (x *DrainShard) String() string {
|
||||
func (*DrainShard) ProtoMessage() {}
|
||||
|
||||
func (x *DrainShard) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[23]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[25]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -2653,7 +2863,7 @@ func (x *DrainShard) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use DrainShard.ProtoReflect.Descriptor instead.
|
||||
func (*DrainShard) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{23}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{25}
|
||||
}
|
||||
|
||||
func (x *DrainShard) GetShardId() string {
|
||||
@@ -2675,7 +2885,7 @@ type Ack struct {
|
||||
|
||||
func (x *Ack) Reset() {
|
||||
*x = Ack{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[24]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[26]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -2687,7 +2897,7 @@ func (x *Ack) String() string {
|
||||
func (*Ack) ProtoMessage() {}
|
||||
|
||||
func (x *Ack) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[24]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[26]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -2700,7 +2910,7 @@ func (x *Ack) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use Ack.ProtoReflect.Descriptor instead.
|
||||
func (*Ack) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{24}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{26}
|
||||
}
|
||||
|
||||
func (x *Ack) GetMessageId() string {
|
||||
@@ -2744,7 +2954,7 @@ type LogsRequest struct {
|
||||
|
||||
func (x *LogsRequest) Reset() {
|
||||
*x = LogsRequest{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[25]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[27]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -2756,7 +2966,7 @@ func (x *LogsRequest) String() string {
|
||||
func (*LogsRequest) ProtoMessage() {}
|
||||
|
||||
func (x *LogsRequest) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[25]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[27]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -2769,7 +2979,7 @@ func (x *LogsRequest) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use LogsRequest.ProtoReflect.Descriptor instead.
|
||||
func (*LogsRequest) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{25}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{27}
|
||||
}
|
||||
|
||||
func (x *LogsRequest) GetRequestId() string {
|
||||
@@ -2809,7 +3019,7 @@ type LogsReply struct {
|
||||
|
||||
func (x *LogsReply) Reset() {
|
||||
*x = LogsReply{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[26]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[28]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -2821,7 +3031,7 @@ func (x *LogsReply) String() string {
|
||||
func (*LogsReply) ProtoMessage() {}
|
||||
|
||||
func (x *LogsReply) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[26]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[28]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -2834,7 +3044,7 @@ func (x *LogsReply) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use LogsReply.ProtoReflect.Descriptor instead.
|
||||
func (*LogsReply) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{26}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{28}
|
||||
}
|
||||
|
||||
func (x *LogsReply) GetRequestId() string {
|
||||
@@ -2880,7 +3090,7 @@ type FabricScanRequest struct {
|
||||
|
||||
func (x *FabricScanRequest) Reset() {
|
||||
*x = FabricScanRequest{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[27]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[29]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -2892,7 +3102,7 @@ func (x *FabricScanRequest) String() string {
|
||||
func (*FabricScanRequest) ProtoMessage() {}
|
||||
|
||||
func (x *FabricScanRequest) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[27]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[29]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -2905,7 +3115,7 @@ func (x *FabricScanRequest) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use FabricScanRequest.ProtoReflect.Descriptor instead.
|
||||
func (*FabricScanRequest) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{27}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{29}
|
||||
}
|
||||
|
||||
func (x *FabricScanRequest) GetRequestId() string {
|
||||
@@ -2944,7 +3154,7 @@ type FabricScanReply struct {
|
||||
|
||||
func (x *FabricScanReply) Reset() {
|
||||
*x = FabricScanReply{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[28]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[30]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -2956,7 +3166,7 @@ func (x *FabricScanReply) String() string {
|
||||
func (*FabricScanReply) ProtoMessage() {}
|
||||
|
||||
func (x *FabricScanReply) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[28]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[30]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -2969,7 +3179,7 @@ func (x *FabricScanReply) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use FabricScanReply.ProtoReflect.Descriptor instead.
|
||||
func (*FabricScanReply) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{28}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{30}
|
||||
}
|
||||
|
||||
func (x *FabricScanReply) GetRequestId() string {
|
||||
@@ -3023,7 +3233,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\x04logs\x18\a \x01(\v2\x15.llmgw.v1.LogsRequestH\x00R\x04logs\x12>\n" +
|
||||
"\vfabric_scan\x18\b \x01(\v2\x1b.llmgw.v1.FabricScanRequestH\x00R\n" +
|
||||
"fabricScanB\x05\n" +
|
||||
"\x03msg\"\xf2\x05\n" +
|
||||
"\x03msg\"\x96\x06\n" +
|
||||
"\bRegister\x12\x17\n" +
|
||||
"\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1a\n" +
|
||||
"\bhostname\x18\x02 \x01(\tR\bhostname\x12\x1b\n" +
|
||||
@@ -3039,7 +3249,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\aweights\x18\v \x03(\v2\x16.llmgw.v1.LocalWeightsR\aweights\x12+\n" +
|
||||
"\astorage\x18\f \x03(\v2\x11.llmgw.v1.StorageR\astorage\x12\"\n" +
|
||||
"\x04host\x18\r \x01(\v2\x0e.llmgw.v1.HostR\x04host\x12%\n" +
|
||||
"\x0eengines_probed\x18\x0e \x01(\bR\renginesProbed\x1a9\n" +
|
||||
"\x0eengines_probed\x18\x0e \x01(\bR\renginesProbed\x12\"\n" +
|
||||
"\x04imex\x18\x0f \x01(\v2\x0e.llmgw.v1.IMEXR\x04imex\x1a9\n" +
|
||||
"\vLabelsEntry\x12\x10\n" +
|
||||
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
|
||||
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\x1aA\n" +
|
||||
@@ -3083,7 +3294,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\vleader_port\x18\x0e \x01(\x05R\n" +
|
||||
"leaderPort\x12\x1d\n" +
|
||||
"\n" +
|
||||
"profile_fp\x18\x0f \x01(\tR\tprofileFp\"\xf0\x04\n" +
|
||||
"profile_fp\x18\x0f \x01(\tR\tprofileFp\"\xa3\x06\n" +
|
||||
"\x03GPU\x12\x14\n" +
|
||||
"\x05index\x18\x01 \x01(\x05R\x05index\x12\x12\n" +
|
||||
"\x04uuid\x18\x02 \x01(\tR\x04uuid\x12\x12\n" +
|
||||
@@ -3103,7 +3314,12 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\n" +
|
||||
"ecc_errors\x18\x0e \x01(\x05R\teccErrors\x12/\n" +
|
||||
"\x13utilization_percent\x18\x0f \x01(\x05R\x12utilizationPercent\x128\n" +
|
||||
"\tnic_paths\x18\x10 \x03(\v2\x1b.llmgw.v1.GPU.NicPathsEntryR\bnicPaths\x1a;\n" +
|
||||
"\tnic_paths\x18\x10 \x03(\v2\x1b.llmgw.v1.GPU.NicPathsEntryR\bnicPaths\x12!\n" +
|
||||
"\ffabric_state\x18\x11 \x01(\tR\vfabricState\x12#\n" +
|
||||
"\rfabric_status\x18\x12 \x01(\tR\ffabricStatus\x12!\n" +
|
||||
"\fcluster_uuid\x18\x13 \x01(\tR\vclusterUuid\x12\x1b\n" +
|
||||
"\tclique_id\x18\x14 \x01(\tR\bcliqueId\x12)\n" +
|
||||
"\x10fabric_bandwidth\x18\x15 \x01(\tR\x0ffabricBandwidth\x1a;\n" +
|
||||
"\rNicPathsEntry\x12\x10\n" +
|
||||
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
|
||||
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"[\n" +
|
||||
@@ -3161,7 +3377,17 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\x04guid\x18\x01 \x01(\tR\x04guid\x12\x10\n" +
|
||||
"\x03lid\x18\x02 \x01(\x05R\x03lid\x12\x1a\n" +
|
||||
"\bpriority\x18\x03 \x01(\x05R\bpriority\x12\x14\n" +
|
||||
"\x05state\x18\x04 \x01(\tR\x05state\"\xfc\x05\n" +
|
||||
"\x05state\x18\x04 \x01(\tR\x05state\"8\n" +
|
||||
"\x04IMEX\x12\x14\n" +
|
||||
"\x05state\x18\x01 \x01(\tR\x05state\x12\x1a\n" +
|
||||
"\bchannels\x18\x02 \x01(\x05R\bchannels\"\xb2\x01\n" +
|
||||
"\x0eGPUFabricState\x12\x14\n" +
|
||||
"\x05index\x18\x01 \x01(\x05R\x05index\x12\x14\n" +
|
||||
"\x05state\x18\x02 \x01(\tR\x05state\x12\x16\n" +
|
||||
"\x06status\x18\x03 \x01(\tR\x06status\x12!\n" +
|
||||
"\fcluster_uuid\x18\x04 \x01(\tR\vclusterUuid\x12\x1b\n" +
|
||||
"\tclique_id\x18\x05 \x01(\tR\bcliqueId\x12\x1c\n" +
|
||||
"\tbandwidth\x18\x06 \x01(\tR\tbandwidth\"\xd9\x06\n" +
|
||||
"\tHeartbeat\x12\x17\n" +
|
||||
"\anode_id\x18\x01 \x01(\tR\x06nodeId\x12+\n" +
|
||||
"\x12gpu_memory_used_mb\x18\x02 \x03(\x05R\x0fgpuMemoryUsedMb\x12!\n" +
|
||||
@@ -3178,7 +3404,10 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\x04load\x18\f \x01(\v2\x12.llmgw.v1.NodeLoadR\x04load\x12+\n" +
|
||||
"\bib_ports\x18\r \x03(\v2\x10.llmgw.v1.IBPortR\aibPorts\x12'\n" +
|
||||
"\x0ffabrics_changed\x18\x0e \x01(\bR\x0efabricsChanged\x124\n" +
|
||||
"\afabrics\x18\x0f \x03(\v2\x1a.llmgw.v1.FabricAttachmentR\afabrics\x1aA\n" +
|
||||
"\afabrics\x18\x0f \x03(\v2\x1a.llmgw.v1.FabricAttachmentR\afabrics\x127\n" +
|
||||
"\n" +
|
||||
"gpu_fabric\x18\x10 \x03(\v2\x18.llmgw.v1.GPUFabricStateR\tgpuFabric\x12\"\n" +
|
||||
"\x04imex\x18\x11 \x01(\v2\x0e.llmgw.v1.IMEXR\x04imex\x1aA\n" +
|
||||
"\x13EngineVersionsEntry\x12\x10\n" +
|
||||
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
|
||||
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\";\n" +
|
||||
@@ -3312,7 +3541,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte {
|
||||
}
|
||||
|
||||
var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
|
||||
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 35)
|
||||
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 37)
|
||||
var file_llmgw_v1_agent_proto_goTypes = []any{
|
||||
(ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State
|
||||
(*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage
|
||||
@@ -3328,75 +3557,80 @@ var file_llmgw_v1_agent_proto_goTypes = []any{
|
||||
(*ShardLoad)(nil), // 11: llmgw.v1.ShardLoad
|
||||
(*IBPort)(nil), // 12: llmgw.v1.IBPort
|
||||
(*SubnetManager)(nil), // 13: llmgw.v1.SubnetManager
|
||||
(*Heartbeat)(nil), // 14: llmgw.v1.Heartbeat
|
||||
(*Provision)(nil), // 15: llmgw.v1.Provision
|
||||
(*EngineWant)(nil), // 16: llmgw.v1.EngineWant
|
||||
(*ProvisionStatus)(nil), // 17: llmgw.v1.ProvisionStatus
|
||||
(*ShardStatus)(nil), // 18: llmgw.v1.ShardStatus
|
||||
(*FetchProgress)(nil), // 19: llmgw.v1.FetchProgress
|
||||
(*StartShard)(nil), // 20: llmgw.v1.StartShard
|
||||
(*EngineProfile)(nil), // 21: llmgw.v1.EngineProfile
|
||||
(*RemoveWeights)(nil), // 22: llmgw.v1.RemoveWeights
|
||||
(*StopShard)(nil), // 23: llmgw.v1.StopShard
|
||||
(*DrainShard)(nil), // 24: llmgw.v1.DrainShard
|
||||
(*Ack)(nil), // 25: llmgw.v1.Ack
|
||||
(*LogsRequest)(nil), // 26: llmgw.v1.LogsRequest
|
||||
(*LogsReply)(nil), // 27: llmgw.v1.LogsReply
|
||||
(*FabricScanRequest)(nil), // 28: llmgw.v1.FabricScanRequest
|
||||
(*FabricScanReply)(nil), // 29: llmgw.v1.FabricScanReply
|
||||
nil, // 30: llmgw.v1.Register.LabelsEntry
|
||||
nil, // 31: llmgw.v1.Register.EngineVersionsEntry
|
||||
nil, // 32: llmgw.v1.GPU.NicPathsEntry
|
||||
nil, // 33: llmgw.v1.IBPort.CountersEntry
|
||||
nil, // 34: llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
nil, // 35: llmgw.v1.EngineProfile.EnvEntry
|
||||
(*IMEX)(nil), // 14: llmgw.v1.IMEX
|
||||
(*GPUFabricState)(nil), // 15: llmgw.v1.GPUFabricState
|
||||
(*Heartbeat)(nil), // 16: llmgw.v1.Heartbeat
|
||||
(*Provision)(nil), // 17: llmgw.v1.Provision
|
||||
(*EngineWant)(nil), // 18: llmgw.v1.EngineWant
|
||||
(*ProvisionStatus)(nil), // 19: llmgw.v1.ProvisionStatus
|
||||
(*ShardStatus)(nil), // 20: llmgw.v1.ShardStatus
|
||||
(*FetchProgress)(nil), // 21: llmgw.v1.FetchProgress
|
||||
(*StartShard)(nil), // 22: llmgw.v1.StartShard
|
||||
(*EngineProfile)(nil), // 23: llmgw.v1.EngineProfile
|
||||
(*RemoveWeights)(nil), // 24: llmgw.v1.RemoveWeights
|
||||
(*StopShard)(nil), // 25: llmgw.v1.StopShard
|
||||
(*DrainShard)(nil), // 26: llmgw.v1.DrainShard
|
||||
(*Ack)(nil), // 27: llmgw.v1.Ack
|
||||
(*LogsRequest)(nil), // 28: llmgw.v1.LogsRequest
|
||||
(*LogsReply)(nil), // 29: llmgw.v1.LogsReply
|
||||
(*FabricScanRequest)(nil), // 30: llmgw.v1.FabricScanRequest
|
||||
(*FabricScanReply)(nil), // 31: llmgw.v1.FabricScanReply
|
||||
nil, // 32: llmgw.v1.Register.LabelsEntry
|
||||
nil, // 33: llmgw.v1.Register.EngineVersionsEntry
|
||||
nil, // 34: llmgw.v1.GPU.NicPathsEntry
|
||||
nil, // 35: llmgw.v1.IBPort.CountersEntry
|
||||
nil, // 36: llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
nil, // 37: llmgw.v1.EngineProfile.EnvEntry
|
||||
}
|
||||
var file_llmgw_v1_agent_proto_depIdxs = []int32{
|
||||
3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register
|
||||
14, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat
|
||||
18, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus
|
||||
27, // 3: llmgw.v1.AgentMessage.logs:type_name -> llmgw.v1.LogsReply
|
||||
29, // 4: llmgw.v1.AgentMessage.fabric_scan:type_name -> llmgw.v1.FabricScanReply
|
||||
20, // 5: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard
|
||||
23, // 6: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard
|
||||
24, // 7: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard
|
||||
25, // 8: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack
|
||||
22, // 9: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights
|
||||
15, // 10: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision
|
||||
26, // 11: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest
|
||||
28, // 12: llmgw.v1.ControlMessage.fabric_scan:type_name -> llmgw.v1.FabricScanRequest
|
||||
30, // 13: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
|
||||
16, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat
|
||||
20, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus
|
||||
29, // 3: llmgw.v1.AgentMessage.logs:type_name -> llmgw.v1.LogsReply
|
||||
31, // 4: llmgw.v1.AgentMessage.fabric_scan:type_name -> llmgw.v1.FabricScanReply
|
||||
22, // 5: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard
|
||||
25, // 6: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard
|
||||
26, // 7: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard
|
||||
27, // 8: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack
|
||||
24, // 9: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights
|
||||
17, // 10: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision
|
||||
28, // 11: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest
|
||||
30, // 12: llmgw.v1.ControlMessage.fabric_scan:type_name -> llmgw.v1.FabricScanRequest
|
||||
32, // 13: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
|
||||
7, // 14: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU
|
||||
8, // 15: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment
|
||||
31, // 16: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
|
||||
33, // 16: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
|
||||
6, // 17: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard
|
||||
9, // 18: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights
|
||||
5, // 19: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage
|
||||
4, // 20: llmgw.v1.Register.host:type_name -> llmgw.v1.Host
|
||||
0, // 21: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
32, // 22: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry
|
||||
11, // 23: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad
|
||||
33, // 24: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry
|
||||
13, // 25: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager
|
||||
5, // 26: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
|
||||
9, // 27: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
|
||||
34, // 28: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
17, // 29: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus
|
||||
10, // 30: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad
|
||||
12, // 31: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort
|
||||
8, // 32: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment
|
||||
16, // 33: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant
|
||||
0, // 34: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
19, // 35: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
|
||||
21, // 36: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
|
||||
35, // 37: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
|
||||
1, // 38: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
|
||||
2, // 39: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
|
||||
39, // [39:40] is the sub-list for method output_type
|
||||
38, // [38:39] is the sub-list for method input_type
|
||||
38, // [38:38] is the sub-list for extension type_name
|
||||
38, // [38:38] is the sub-list for extension extendee
|
||||
0, // [0:38] is the sub-list for field type_name
|
||||
14, // 21: llmgw.v1.Register.imex:type_name -> llmgw.v1.IMEX
|
||||
0, // 22: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
34, // 23: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry
|
||||
11, // 24: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad
|
||||
35, // 25: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry
|
||||
13, // 26: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager
|
||||
5, // 27: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
|
||||
9, // 28: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
|
||||
36, // 29: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
19, // 30: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus
|
||||
10, // 31: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad
|
||||
12, // 32: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort
|
||||
8, // 33: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment
|
||||
15, // 34: llmgw.v1.Heartbeat.gpu_fabric:type_name -> llmgw.v1.GPUFabricState
|
||||
14, // 35: llmgw.v1.Heartbeat.imex:type_name -> llmgw.v1.IMEX
|
||||
18, // 36: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant
|
||||
0, // 37: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
21, // 38: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
|
||||
23, // 39: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
|
||||
37, // 40: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
|
||||
1, // 41: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
|
||||
2, // 42: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
|
||||
42, // [42:43] is the sub-list for method output_type
|
||||
41, // [41:42] is the sub-list for method input_type
|
||||
41, // [41:41] is the sub-list for extension type_name
|
||||
41, // [41:41] is the sub-list for extension extendee
|
||||
0, // [0:41] is the sub-list for field type_name
|
||||
}
|
||||
|
||||
func init() { file_llmgw_v1_agent_proto_init() }
|
||||
@@ -3427,7 +3661,7 @@ func file_llmgw_v1_agent_proto_init() {
|
||||
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
||||
RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)),
|
||||
NumEnums: 1,
|
||||
NumMessages: 35,
|
||||
NumMessages: 37,
|
||||
NumExtensions: 0,
|
||||
NumServices: 1,
|
||||
},
|
||||
|
||||
@@ -116,6 +116,8 @@ message Register {
|
||||
// meant "an agent too old to say" and constrained nothing — so a freshly
|
||||
// built node with no engine installed yet was a candidate for every model.
|
||||
bool engines_probed = 14;
|
||||
// imex is the service that lets a multi-node NVLink domain share memory.
|
||||
IMEX imex = 15;
|
||||
}
|
||||
|
||||
// Host describes the operating system and processor of a node.
|
||||
@@ -212,6 +214,14 @@ message GPU {
|
||||
// nic_paths is how far each network adapter is from this card, by HCA
|
||||
// name: PIX, PXB, PHB, NODE or SYS, from nvidia-smi topo.
|
||||
map<string, string> nic_paths = 16;
|
||||
// The card's multi-node NVLink membership, from nvidia-smi -q: state is
|
||||
// Completed once the fabric manager has joined it; cluster_uuid and
|
||||
// clique_id name the domain it joined.
|
||||
string fabric_state = 17;
|
||||
string fabric_status = 18;
|
||||
string cluster_uuid = 19;
|
||||
string clique_id = 20;
|
||||
string fabric_bandwidth = 21;
|
||||
}
|
||||
|
||||
message FabricAttachment {
|
||||
@@ -315,6 +325,23 @@ message SubnetManager {
|
||||
string state = 4;
|
||||
}
|
||||
|
||||
// IMEX is the nvidia-imex service and its channels.
|
||||
message IMEX {
|
||||
// state is systemd's word for it, empty where it is not installed.
|
||||
string state = 1;
|
||||
int32 channels = 2;
|
||||
}
|
||||
|
||||
// GPUFabricState is one GPU's NVLink domain membership.
|
||||
message GPUFabricState {
|
||||
int32 index = 1;
|
||||
string state = 2;
|
||||
string status = 3;
|
||||
string cluster_uuid = 4;
|
||||
string clique_id = 5;
|
||||
string bandwidth = 6;
|
||||
}
|
||||
|
||||
message Heartbeat {
|
||||
string node_id = 1;
|
||||
// gpu_memory_used_mb, indexed by GPU, catches a foreign process appearing on
|
||||
@@ -370,6 +397,12 @@ message Heartbeat {
|
||||
// port going down takes its fabric away, and one coming back returns it.
|
||||
bool fabrics_changed = 14;
|
||||
repeated FabricAttachment fabrics = 15;
|
||||
|
||||
// gpu_fabric is each GPU's NVLink domain membership as it stands now, sent
|
||||
// once a minute: a fabric manager restarting drops every card out of its
|
||||
// domain, and the node has to stop being offered work that spans it.
|
||||
repeated GPUFabricState gpu_fabric = 16;
|
||||
IMEX imex = 17;
|
||||
}
|
||||
|
||||
// Provision is the software a node should have: which engines, at which
|
||||
|
||||
@@ -365,6 +365,8 @@ type Agent struct {
|
||||
sm map[string]*discovery.SMInfo
|
||||
smErr map[string]string
|
||||
smAsked time.Time
|
||||
// nvlAsked is when the GPUs' NVLink fabric state was last read.
|
||||
nvlAsked time.Time
|
||||
// sms is every subnet manager on a fabric, asked through one port per
|
||||
// master, keyed by that port; smsErr says why a port could not ask.
|
||||
sms map[string][]discovery.SMRecord
|
||||
@@ -999,6 +1001,7 @@ func (a *Agent) heartbeat() *llmgwv1.AgentMessage {
|
||||
}
|
||||
hb.Load = a.sampleLoad()
|
||||
a.sampleIB(hb)
|
||||
a.sampleNVLink(hb)
|
||||
return &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Heartbeat{Heartbeat: hb}}
|
||||
}
|
||||
|
||||
@@ -1752,3 +1755,45 @@ func (a *Agent) ensureDiags(ctx context.Context) {
|
||||
"err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// discoverGPUFabric and discoverIMEX are replaced in tests.
|
||||
var (
|
||||
discoverGPUFabric = discovery.DiscoverGPUFabric
|
||||
discoverIMEX = discovery.DiscoverIMEX
|
||||
)
|
||||
|
||||
// sampleNVLink reads, once a minute, which NVLink domain each GPU is joined
|
||||
// to and whether IMEX is up. nvidia-smi -q takes a second or two, too long
|
||||
// for every heartbeat, and a fabric that was joined an hour ago rarely
|
||||
// changes in five seconds — but when a fabric manager restarts it does, for
|
||||
// every card at once.
|
||||
func (a *Agent) sampleNVLink(hb *llmgwv1.Heartbeat) {
|
||||
a.mu.Lock()
|
||||
gpus := slices.Clone(a.inv.GPUs)
|
||||
due := time.Since(a.nvlAsked) > time.Minute
|
||||
if due {
|
||||
a.nvlAsked = time.Now()
|
||||
}
|
||||
a.mu.Unlock()
|
||||
if len(gpus) == 0 || !due {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
fab, err := discoverGPUFabric(ctx, a.cfg.Discovery.NvidiaSMI)
|
||||
if err != nil || len(fab) == 0 {
|
||||
return
|
||||
}
|
||||
discovery.ApplyFabric(gpus, fab)
|
||||
for _, g := range gpus {
|
||||
if g.Fabric == nil {
|
||||
continue
|
||||
}
|
||||
hb.GpuFabric = append(hb.GpuFabric, &llmgwv1.GPUFabricState{
|
||||
Index: int32(g.Index), State: g.Fabric.State, Status: g.Fabric.Status,
|
||||
ClusterUuid: g.Fabric.ClusterUUID, CliqueId: g.Fabric.CliqueID, Bandwidth: g.Fabric.Bandwidth,
|
||||
})
|
||||
}
|
||||
m := discoverIMEX(ctx)
|
||||
hb.Imex = &llmgwv1.IMEX{State: m.State, Channels: int32(m.Channels)}
|
||||
}
|
||||
@@ -85,6 +85,13 @@ func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, e
|
||||
} else {
|
||||
topo.Apply(inv.GPUs)
|
||||
}
|
||||
// Multi-node NVLink: which domain each card has joined.
|
||||
if fab, err := DiscoverGPUFabric(ctx, opts.NvidiaSMI); err != nil {
|
||||
log.Warn("could not read the GPUs' NVLink fabric state", "err", err)
|
||||
} else {
|
||||
ApplyFabric(inv.GPUs, fab)
|
||||
}
|
||||
inv.IMEX = DiscoverIMEX(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -42,6 +42,9 @@ type GPU struct {
|
||||
// PIX, PXB, PHB, NODE or SYS, as nvidia-smi topo reports it. What decides
|
||||
// which adapter a GPU's share of a multi-node job should go through.
|
||||
NICPaths map[string]string `json:"nic_paths,omitempty"`
|
||||
// Fabric is the GPU's multi-node NVLink membership, nil where the
|
||||
// driver reports none.
|
||||
Fabric *GPUFabric `json:"fabric,omitempty"`
|
||||
// Health carries anything that should keep the GPU out of placement.
|
||||
Health GPUHealth `json:"health"`
|
||||
}
|
||||
@@ -121,6 +124,9 @@ type Inventory struct {
|
||||
// Storage is the disk behind those weights, one entry per filesystem.
|
||||
// Empty means the same thing it means for Weights: not asked.
|
||||
Storage []Storage `json:"storage,omitempty"`
|
||||
// IMEX is the service a multi-node NVLink domain needs, read where the
|
||||
// node has GPUs.
|
||||
IMEX IMEX `json:"imex"`
|
||||
}
|
||||
|
||||
// Classes returns the distinct GPU classes present, sorted.
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// GPUFabric is a GPU's place in a multi-node NVLink domain, as the driver
|
||||
// reports it: the NVL72 rack, or the NVLink Switch system, it belongs to.
|
||||
type GPUFabric struct {
|
||||
// State is Completed once the fabric manager has joined the GPU to its
|
||||
// domain; In Progress, Not Started or Not Supported otherwise.
|
||||
State string `json:"state"`
|
||||
// Status is Success, or the reason the join failed.
|
||||
Status string `json:"status,omitempty"`
|
||||
// ClusterUUID and CliqueID name the domain: GPUs sharing both can reach
|
||||
// each other over NVLink whatever node they are in.
|
||||
ClusterUUID string `json:"cluster_uuid,omitempty"`
|
||||
CliqueID string `json:"clique_id,omitempty"`
|
||||
// Bandwidth is Full, or how the domain's NVLink is degraded.
|
||||
Bandwidth string `json:"bandwidth,omitempty"`
|
||||
}
|
||||
|
||||
// Domain is the NVLink domain the GPU is joined to, empty when it is in none
|
||||
// that crosses nodes: a card with no fabric, one not joined, or one whose
|
||||
// cluster is all zeros, which is what a single machine's NVSwitch reports.
|
||||
func (f *GPUFabric) Domain() string {
|
||||
if f == nil || !strings.EqualFold(f.State, "Completed") {
|
||||
return ""
|
||||
}
|
||||
u := strings.ToLower(strings.TrimSpace(f.ClusterUUID))
|
||||
if u == "" || strings.Trim(u, "0-") == "" {
|
||||
return ""
|
||||
}
|
||||
return u + "/" + f.CliqueID
|
||||
}
|
||||
|
||||
// ParseGPUFabric reads the Fabric sections of `nvidia-smi -q`, by the PCI
|
||||
// address each GPU's section starts with.
|
||||
func ParseGPUFabric(out string) map[string]GPUFabric {
|
||||
res := map[string]GPUFabric{}
|
||||
var bus string
|
||||
fabricIndent := -1
|
||||
var cur *GPUFabric
|
||||
flush := func() {
|
||||
if bus != "" && cur != nil && cur.State != "" {
|
||||
res[bus] = *cur
|
||||
}
|
||||
cur, fabricIndent = nil, -1
|
||||
}
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
trim := strings.TrimSpace(line)
|
||||
indent := len(line) - len(strings.TrimLeft(line, " \t"))
|
||||
if strings.HasPrefix(line, "GPU ") && indent == 0 {
|
||||
flush()
|
||||
bus = strings.ToLower(strings.TrimSpace(strings.TrimPrefix(line, "GPU ")))
|
||||
continue
|
||||
}
|
||||
if trim == "Fabric" {
|
||||
flush()
|
||||
cur, fabricIndent = &GPUFabric{}, indent
|
||||
continue
|
||||
}
|
||||
if cur == nil {
|
||||
continue
|
||||
}
|
||||
if trim != "" && indent <= fabricIndent {
|
||||
flush()
|
||||
continue
|
||||
}
|
||||
k, v, ok := strings.Cut(trim, ":")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
k, v = strings.TrimSpace(k), strings.TrimSpace(v)
|
||||
if strings.EqualFold(v, "N/A") {
|
||||
v = ""
|
||||
}
|
||||
switch k {
|
||||
case "State":
|
||||
cur.State = v
|
||||
case "Status":
|
||||
cur.Status = v
|
||||
case "ClusterUUID", "Cluster UUID":
|
||||
cur.ClusterUUID = v
|
||||
case "CliqueId", "Clique ID":
|
||||
cur.CliqueID = v
|
||||
case "Bandwidth":
|
||||
cur.Bandwidth = v
|
||||
}
|
||||
}
|
||||
flush()
|
||||
return res
|
||||
}
|
||||
|
||||
// DiscoverGPUFabric runs `nvidia-smi -q` for the Fabric sections. Nothing
|
||||
// is reported on a driver or a card without them.
|
||||
func DiscoverGPUFabric(ctx context.Context, binary string) (map[string]GPUFabric, error) {
|
||||
if binary == "" {
|
||||
if _, err := exec.LookPath("nvidia-smi"); err != nil {
|
||||
return nil, nil
|
||||
}
|
||||
binary = "nvidia-smi"
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
|
||||
defer cancel()
|
||||
out, err := exec.CommandContext(ctx, binary, "-q").Output()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ParseGPUFabric(string(out)), nil
|
||||
}
|
||||
|
||||
// ApplyFabric puts each GPU's fabric on it, matched by PCI address.
|
||||
func ApplyFabric(gpus []GPU, fab map[string]GPUFabric) {
|
||||
for i := range gpus {
|
||||
if f, ok := fab[strings.ToLower(gpus[i].PCIBusID)]; ok {
|
||||
f := f
|
||||
gpus[i].Fabric = &f
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// IMEX is the service that lets GPUs in different nodes of one NVLink
|
||||
// domain share memory: without it, multi-node NVLink carries nothing.
|
||||
type IMEX struct {
|
||||
// State is systemd's word for nvidia-imex: active, inactive, or empty
|
||||
// where it is not installed.
|
||||
State string `json:"state,omitempty"`
|
||||
// Channels is how many IMEX channels exist; a process needs one.
|
||||
Channels int `json:"channels"`
|
||||
}
|
||||
|
||||
// DiscoverIMEX reads the IMEX service and its channels.
|
||||
func DiscoverIMEX(ctx context.Context) IMEX {
|
||||
var m IMEX
|
||||
if entries, err := os.ReadDir("/dev/nvidia-caps-imex-channels"); err == nil {
|
||||
m.Channels = len(entries)
|
||||
}
|
||||
if _, err := exec.LookPath("systemctl"); err == nil {
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
out, _ := exec.CommandContext(ctx, "systemctl", "is-active", "nvidia-imex").Output()
|
||||
m.State = strings.TrimSpace(string(out))
|
||||
if m.State == "inactive" {
|
||||
// "inactive" is also what systemd says of a unit that does not
|
||||
// exist; the binary says which.
|
||||
if _, err := exec.LookPath("nvidia-imex"); err != nil {
|
||||
if _, err := os.Stat(filepath.Join("/usr/bin", "nvidia-imex")); err != nil {
|
||||
m.State = ""
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package discovery
|
||||
|
||||
import "testing"
|
||||
|
||||
// smiQ is an excerpt of `nvidia-smi -q` on a GB200 tray: two GPUs joined to
|
||||
// one NVL72 domain, then a PCIe card that has no fabric at all.
|
||||
const smiQ = `
|
||||
==============NVSMI LOG==============
|
||||
|
||||
Timestamp : Mon Sep 29 10:00:00 2026
|
||||
Driver Version : 570.86.15
|
||||
Attached GPUs : 3
|
||||
GPU 00000008:01:00.0
|
||||
Product Name : NVIDIA GB200
|
||||
Minor Number : 0
|
||||
Fabric
|
||||
State : Completed
|
||||
Status : Success
|
||||
CliqueId : 32766
|
||||
ClusterUUID : 76B1AD76-7c24-4b6b-9f6c-2a1e0d8c11aa
|
||||
Health
|
||||
Bandwidth : Full
|
||||
Processes : None
|
||||
|
||||
GPU 00000009:01:00.0
|
||||
Product Name : NVIDIA GB200
|
||||
Fabric
|
||||
State : In Progress
|
||||
Status : N/A
|
||||
CliqueId : N/A
|
||||
ClusterUUID : N/A
|
||||
Processes : None
|
||||
|
||||
GPU 0000000A:01:00.0
|
||||
Product Name : NVIDIA L40S
|
||||
Fabric
|
||||
State : N/A
|
||||
Status : N/A
|
||||
`
|
||||
|
||||
func TestAGPUsNVLinkDomainIsRead(t *testing.T) {
|
||||
fab := ParseGPUFabric(smiQ)
|
||||
a, ok := fab["00000008:01:00.0"]
|
||||
if !ok {
|
||||
t.Fatalf("fabric = %+v", fab)
|
||||
}
|
||||
if a.State != "Completed" || a.CliqueID != "32766" || a.Bandwidth != "Full" {
|
||||
t.Errorf("GB200 = %+v", a)
|
||||
}
|
||||
if got := a.Domain(); got != "76b1ad76-7c24-4b6b-9f6c-2a1e0d8c11aa/32766" {
|
||||
t.Errorf("domain = %q", got)
|
||||
}
|
||||
// Still joining: no domain yet, and said as such.
|
||||
b := fab["00000009:01:00.0"]
|
||||
if b.State != "In Progress" || b.Domain() != "" {
|
||||
t.Errorf("joining GPU = %+v", b)
|
||||
}
|
||||
// A card with no fabric reports nothing rather than an empty one.
|
||||
if _, ok := fab["0000000a:01:00.0"]; ok {
|
||||
t.Error("a card without a fabric was given one")
|
||||
}
|
||||
|
||||
gpus := []GPU{{Index: 0, PCIBusID: "00000008:01:00.0"}, {Index: 1, PCIBusID: "0000000A:01:00.0"}}
|
||||
ApplyFabric(gpus, fab)
|
||||
if gpus[0].Fabric == nil || gpus[1].Fabric != nil {
|
||||
t.Errorf("applied = %+v %+v", gpus[0].Fabric, gpus[1].Fabric)
|
||||
}
|
||||
}
|
||||
|
||||
func TestASingleMachinesNVSwitchIsNoDomain(t *testing.T) {
|
||||
// An eight-GPU HGX reports its fabric joined, with a cluster of zeros: its
|
||||
// NVLink ends at the chassis.
|
||||
f := &GPUFabric{State: "Completed", ClusterUUID: "00000000-0000-0000-0000-000000000000", CliqueID: "0"}
|
||||
if f.Domain() != "" {
|
||||
t.Errorf("a single machine read as a domain: %q", f.Domain())
|
||||
}
|
||||
}
|
||||
@@ -71,8 +71,9 @@ type ShardSpec struct {
|
||||
Profile Profile
|
||||
|
||||
// Pool and Fabric are control plane concepts the agent cannot derive. They
|
||||
// are carried so the agent can report them back after a restart, and are
|
||||
// not used when building a command.
|
||||
// are carried so the agent can report them back after a restart; Fabric
|
||||
// also says, with an "nvl:" prefix, that the replica is inside a
|
||||
// multi-node NVLink domain, which changes NCCL's environment.
|
||||
Pool string
|
||||
Fabric string
|
||||
// ProfileFP identifies the version of the profile this shard was started
|
||||
@@ -232,6 +233,14 @@ func baseEnv(s ShardSpec) map[string]string {
|
||||
env["NCCL_IB_DISABLE"] = "1"
|
||||
}
|
||||
|
||||
// Placed inside a multi-node NVLink domain: NCCL is to carry the
|
||||
// replica's collectives over NVLink between its nodes, which it only
|
||||
// does when told to, and needs IMEX for.
|
||||
if s.NNodes > 1 && strings.HasPrefix(s.Fabric, "nvl:") {
|
||||
env["NCCL_MNNVL_ENABLE"] = "1"
|
||||
env["NCCL_CUMEM_ENABLE"] = "1"
|
||||
}
|
||||
|
||||
if s.MgmtInterface != "" {
|
||||
env["NCCL_SOCKET_IFNAME"] = s.MgmtInterface
|
||||
env["GLOO_SOCKET_IFNAME"] = s.MgmtInterface
|
||||
|
||||
@@ -451,3 +451,15 @@ func TestQuantizationOf(t *testing.T) {
|
||||
t.Errorf("a dangling flag should yield nothing, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAReplicaInAnNVLinkDomainIsToldToUseIt(t *testing.T) {
|
||||
s := multiNodeSpec(0)
|
||||
s.Fabric = "nvl:76b1ad76-7c24-4b6b-9f6c-2a1e0d8c11aa/7"
|
||||
env := envMap(buildOne(t, &VLLM{cfg: VLLMConfig{LaunchMode: "mp"}}, s).Env)
|
||||
if env["NCCL_MNNVL_ENABLE"] != "1" {
|
||||
t.Errorf("NCCL_MNNVL_ENABLE = %q", env["NCCL_MNNVL_ENABLE"])
|
||||
}
|
||||
if env := envMap(buildOne(t, &VLLM{cfg: VLLMConfig{LaunchMode: "mp"}}, multiNodeSpec(0)).Env); env["NCCL_MNNVL_ENABLE"] != "" {
|
||||
t.Error("a replica on InfiniBand was told to use multi-node NVLink")
|
||||
}
|
||||
}
|
||||
@@ -208,6 +208,7 @@ func newServerCmd() *cobra.Command {
|
||||
adminMux.Handle("GET /_static/", web.Static())
|
||||
adminMux.HandleFunc("GET /nodes/{id}/logs", web.NodeLogsPage(api, true, version.Version))
|
||||
adminMux.HandleFunc("GET /nodes/{id}", web.NodePage(api, history, cfg.Monitoring.Window, true, version.Version, log))
|
||||
adminMux.HandleFunc("GET /nvlink", web.NVLinkPage(api, true, version.Version))
|
||||
adminMux.HandleFunc("GET /fabrics", web.FabricsPage(api, true, version.Version))
|
||||
adminMux.HandleFunc("GET /fabrics/{id}", web.FabricsPage(api, true, version.Version))
|
||||
adminMux.HandleFunc("GET /fabrics/{id}/topology.json", web.FabricTopologyJSON(api))
|
||||
|
||||
@@ -16,11 +16,13 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/bench"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/topology"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
@@ -53,6 +55,9 @@ func (a *API) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("GET /api/v1/fabrics", func(w http.ResponseWriter, _ *http.Request) {
|
||||
writeJSON(w, http.StatusOK, a.Fabrics())
|
||||
})
|
||||
mux.HandleFunc("GET /api/v1/nvlink", func(w http.ResponseWriter, _ *http.Request) {
|
||||
writeJSON(w, http.StatusOK, a.NVLink())
|
||||
})
|
||||
mux.HandleFunc("GET /api/v1/fabrics/{id}/topology", func(w http.ResponseWriter, r *http.Request) {
|
||||
t, ok := a.Topology(r.PathValue("id"))
|
||||
if !ok {
|
||||
@@ -134,6 +139,27 @@ type NodeView struct {
|
||||
Load *LoadView `json:"load,omitempty"`
|
||||
// IBPorts are the node's InfiniBand and RoCE ports at the last heartbeat.
|
||||
IBPorts []IBPortView `json:"ib_ports,omitempty"`
|
||||
// GPUs are the cards, with how they are wired: NVLink peers, the
|
||||
// adapter nearest each, and the NVLink domain each has joined.
|
||||
GPUs []GPUView `json:"gpus,omitempty"`
|
||||
// NVLinkDomain is the multi-node domain every card has joined, if any.
|
||||
NVLinkDomain string `json:"nvlink_domain,omitempty"`
|
||||
IMEX string `json:"imex,omitempty"`
|
||||
IMEXChannels int `json:"imex_channels,omitempty"`
|
||||
}
|
||||
|
||||
// GPUView is one card as a node's page shows it.
|
||||
type GPUView struct {
|
||||
Index int `json:"index"`
|
||||
Name string `json:"name"`
|
||||
NUMA int `json:"numa"`
|
||||
NVLinkPeers []int `json:"nvlink_peers,omitempty"`
|
||||
// NearestNIC is the adapter closest to the card, and how close.
|
||||
NearestNIC string `json:"nearest_nic,omitempty"`
|
||||
NICPath string `json:"nic_path,omitempty"`
|
||||
FabricState string `json:"fabric_state,omitempty"`
|
||||
Domain string `json:"domain,omitempty"`
|
||||
Bandwidth string `json:"bandwidth,omitempty"`
|
||||
}
|
||||
|
||||
// IBPortView is one port as a page shows it.
|
||||
@@ -344,6 +370,20 @@ func (a *API) Nodes() []NodeView {
|
||||
v.LastSeenAgo = time.Since(n.LastSeen).Round(time.Second).String()
|
||||
}
|
||||
v.Load = loadView(n.Load, time.Now())
|
||||
v.NVLinkDomain, v.IMEX, v.IMEXChannels = n.NVLinkDomain(), n.IMEX.State, n.IMEX.Channels
|
||||
for _, g := range n.GPUs {
|
||||
gv := GPUView{Index: g.Index, Name: g.Name, NUMA: g.NUMANode, NVLinkPeers: g.NVLinkPeers}
|
||||
for hca, rel := range g.NICPaths {
|
||||
if gv.NearestNIC == "" || discovery.PathRank(rel) < discovery.PathRank(gv.NICPath) ||
|
||||
(discovery.PathRank(rel) == discovery.PathRank(gv.NICPath) && hca < gv.NearestNIC) {
|
||||
gv.NearestNIC, gv.NICPath = hca, rel
|
||||
}
|
||||
}
|
||||
if f := g.Fabric; f != nil {
|
||||
gv.FabricState, gv.Domain, gv.Bandwidth = f.State, f.Domain(), f.Bandwidth
|
||||
}
|
||||
v.GPUs = append(v.GPUs, gv)
|
||||
}
|
||||
fabricOf := map[string]string{}
|
||||
for _, f := range n.Fabrics {
|
||||
for _, h := range f.HCAs {
|
||||
@@ -959,3 +999,9 @@ func (a *API) Topology(id string) (TopologyView, bool) {
|
||||
}
|
||||
return t, true
|
||||
}
|
||||
|
||||
// NVLink is the cluster's multi-node NVLink domains, and what keeps any of
|
||||
// their nodes out.
|
||||
func (a *API) NVLink() nvlink.Report {
|
||||
return nvlink.Analyze(a.ctl.Inventory().Nodes())
|
||||
}
|
||||
@@ -61,6 +61,8 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node {
|
||||
CUDAVersion: g.GetCudaVersion(),
|
||||
NVLinkPeers: intSlice(g.GetNvlinkPeers()),
|
||||
NICPaths: maps.Clone(g.GetNicPaths()),
|
||||
Fabric: fabricFrom(g.GetFabricState(), g.GetFabricStatus(), g.GetClusterUuid(),
|
||||
g.GetCliqueId(), g.GetFabricBandwidth()),
|
||||
Health: discovery.GPUHealth{
|
||||
ECCErrors: int(g.GetEccErrors()),
|
||||
TemperatureC: int(g.GetTemperatureC()),
|
||||
@@ -68,6 +70,7 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node {
|
||||
},
|
||||
})
|
||||
}
|
||||
n.IMEX = discovery.IMEX{State: r.GetImex().GetState(), Channels: int(r.GetImex().GetChannels())}
|
||||
for _, f := range r.GetFabrics() {
|
||||
n.Fabrics = append(n.Fabrics, inventory.FabricAttachment{
|
||||
Fabric: f.GetFabric(),
|
||||
@@ -119,7 +122,13 @@ func RegisterFromInventory(nodeID, hostname, mgmtAddr, agentVersion string,
|
||||
TemperatureC: int32(g.Health.TemperatureC),
|
||||
EccErrors: int32(g.Health.ECCErrors),
|
||||
})
|
||||
if f := g.Fabric; f != nil {
|
||||
last := r.Gpus[len(r.Gpus)-1]
|
||||
last.FabricState, last.FabricStatus, last.ClusterUuid = f.State, f.Status, f.ClusterUUID
|
||||
last.CliqueId, last.FabricBandwidth = f.CliqueID, f.Bandwidth
|
||||
}
|
||||
}
|
||||
r.Imex = &llmgwv1.IMEX{State: inv.IMEX.State, Channels: int32(inv.IMEX.Channels)}
|
||||
for _, f := range fabrics {
|
||||
r.Fabrics = append(r.Fabrics, &llmgwv1.FabricAttachment{
|
||||
Fabric: f.Fabric, Hcas: f.HCAs, RateGbps: int32(f.RateGbps),
|
||||
@@ -266,3 +275,12 @@ func int32Slice(v []int) []int32 {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// fabricFrom is a GPU's NVLink fabric as the wire carries it, nil when the
|
||||
// driver reported none.
|
||||
func fabricFrom(state, status, cluster, clique, bandwidth string) *discovery.GPUFabric {
|
||||
if state == "" {
|
||||
return nil
|
||||
}
|
||||
return &discovery.GPUFabric{State: state, Status: status, ClusterUUID: cluster, CliqueID: clique, Bandwidth: bandwidth}
|
||||
}
|
||||
@@ -242,3 +242,25 @@ func TestAPortGoingDownTakesItsFabricAway(t *testing.T) {
|
||||
t.Error("a heartbeat without a fabric change emptied the fabrics")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAGPUFallingOutOfItsDomainTakesTheNodeOut(t *testing.T) {
|
||||
// A fabric manager restarting drops cards out of the domain; the node has
|
||||
// to stop being offered work that spans it until they are back.
|
||||
s := newTestService()
|
||||
fab := &discovery.GPUFabric{State: "Completed", ClusterUUID: "76b1ad76-7c24", CliqueID: "1"}
|
||||
nodeWith(s, inventory.Node{ID: "t1", State: inventory.Ready, GPUs: []discovery.GPU{
|
||||
{Index: 0, Fabric: fab}, {Index: 1, Fabric: fab}}})
|
||||
if n, _ := s.inv.Get("t1"); n.NVLinkDomain() == "" {
|
||||
t.Fatal("the node should start in its domain")
|
||||
}
|
||||
s.applyHeartbeat("t1", &llmgwv1.Heartbeat{NodeId: "t1",
|
||||
GpuFabric: []*llmgwv1.GPUFabricState{{Index: 1, State: "In Progress"}},
|
||||
Imex: &llmgwv1.IMEX{State: "active", Channels: 1}})
|
||||
n, _ := s.inv.Get("t1")
|
||||
if n.NVLinkDomain() != "" {
|
||||
t.Errorf("a node with a card out of the domain is still in it")
|
||||
}
|
||||
if n.IMEX.State != "active" || n.IMEX.Channels != 1 {
|
||||
t.Errorf("imex = %+v", n.IMEX)
|
||||
}
|
||||
}
|
||||
@@ -322,7 +322,7 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
|
||||
if len(used) == 0 && len(problems) == 0 && len(storage) == 0 &&
|
||||
!hb.GetWeightsChanged() && memMB == 0 && !hb.GetEnginesChanged() &&
|
||||
!hb.GetProvisionChanged() && hb.GetLoad() == nil && len(hb.GetIbPorts()) == 0 &&
|
||||
!hb.GetFabricsChanged() {
|
||||
!hb.GetFabricsChanged() && len(hb.GetGpuFabric()) == 0 && hb.GetImex() == nil {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -457,6 +457,24 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
|
||||
if len(hb.GetIbPorts()) > 0 {
|
||||
n.IBAt = time.Now()
|
||||
}
|
||||
// NVLink domain membership, which a fabric manager restarting takes
|
||||
// away from every card at once.
|
||||
if fab := hb.GetGpuFabric(); len(fab) > 0 {
|
||||
before := n.NVLinkDomain()
|
||||
for _, f := range fab {
|
||||
for i := range n.GPUs {
|
||||
if n.GPUs[i].Index == int(f.GetIndex()) {
|
||||
n.GPUs[i].Fabric = fabricFrom(f.GetState(), f.GetStatus(), f.GetClusterUuid(), f.GetCliqueId(), f.GetBandwidth())
|
||||
}
|
||||
}
|
||||
}
|
||||
if after := n.NVLinkDomain(); after != before {
|
||||
s.log.Warn("a node's NVLink domain changed", "node", nodeID, "from", before, "to", after)
|
||||
}
|
||||
}
|
||||
if m := hb.GetImex(); m != nil {
|
||||
n.IMEX = discovery.IMEX{State: m.GetState(), Channels: int(m.GetChannels())}
|
||||
}
|
||||
// A port went down, or came back. Placement reads the fabrics, so this is
|
||||
// what stops a node being offered multi-node work it can no longer carry.
|
||||
if hb.GetFabricsChanged() {
|
||||
|
||||
@@ -74,6 +74,9 @@ type Collector struct {
|
||||
ibRate *prometheus.Desc
|
||||
ibCounter *prometheus.Desc
|
||||
|
||||
gpuNVLJoined *prometheus.Desc
|
||||
imexUp *prometheus.Desc
|
||||
|
||||
fabricSMs *prometheus.Desc
|
||||
fabricMaster *prometheus.Desc
|
||||
fabricProblems *prometheus.Desc
|
||||
@@ -166,6 +169,11 @@ func New(src Source) *Collector {
|
||||
ibRate: prometheus.NewDesc("llmgw_ib_port_rate_gbps",
|
||||
"A port's negotiated rate. A link that trained below its usual speed is a "+
|
||||
"cable or transceiver to look at.", []string{"node", "hca", "port"}, nil),
|
||||
gpuNVLJoined: prometheus.NewDesc("llmgw_gpu_nvlink_joined",
|
||||
"1 when a GPU is joined to its multi-node NVLink domain, 0 when it reports a fabric it has not "+
|
||||
"joined. Absent on cards with no multi-node fabric.", []string{"node", "gpu", "domain", "state"}, nil),
|
||||
imexUp: prometheus.NewDesc("llmgw_imex_up",
|
||||
"1 when nvidia-imex is active with a channel, on nodes whose GPUs report a fabric.", []string{"node"}, nil),
|
||||
fabricSMs: prometheus.NewDesc("llmgw_fabric_subnet_managers",
|
||||
"Subnet managers a physical fabric lists, by state. No standby means the fabric "+
|
||||
"cannot recover from a change if its master stops.", []string{"fabric", "physical", "state"}, nil),
|
||||
@@ -189,6 +197,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
|
||||
c.nodeDiskFree, c.nodeDiskSize,
|
||||
c.nodeCPU, c.nodeCPUs, c.nodeLoad, c.nodeMemUsed, c.nodeMemAvail, c.shardCPU, c.shardMemory,
|
||||
c.ibActive, c.ibRate, c.ibCounter, c.fabricSMs, c.fabricMaster, c.fabricProblems,
|
||||
c.gpuNVLJoined, c.imexUp,
|
||||
c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuTemp, c.gpuHealthy,
|
||||
c.gpuNVLink, c.gpuECC,
|
||||
c.replicaState, c.replicaReady, c.replicaIdle,
|
||||
@@ -263,6 +272,28 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) {
|
||||
}
|
||||
}
|
||||
|
||||
fabricGPUs := 0
|
||||
for _, g := range n.GPUs {
|
||||
fb := g.Fabric
|
||||
if fb == nil || strings.EqualFold(fb.State, "Not Supported") {
|
||||
continue
|
||||
}
|
||||
fabricGPUs++
|
||||
joined := 0.0
|
||||
if fb.Domain() != "" {
|
||||
joined = 1
|
||||
}
|
||||
ch <- prometheus.MustNewConstMetric(c.gpuNVLJoined, prometheus.GaugeValue, joined,
|
||||
n.ID, strconv.Itoa(g.Index), fb.Domain(), fb.State)
|
||||
}
|
||||
if fabricGPUs > 0 {
|
||||
up := 0.0
|
||||
if n.IMEX.State == "active" && n.IMEX.Channels > 0 {
|
||||
up = 1
|
||||
}
|
||||
ch <- prometheus.MustNewConstMetric(c.imexUp, prometheus.GaugeValue, up, n.ID)
|
||||
}
|
||||
|
||||
byClass := map[string]int{}
|
||||
for _, g := range n.GPUs {
|
||||
class := g.Class()
|
||||
|
||||
@@ -120,6 +120,24 @@ type Node struct {
|
||||
IBPorts []IBPort `json:"-"`
|
||||
// IBAt is when IBPorts was read, for the rates.
|
||||
IBAt time.Time `json:"-"`
|
||||
// IMEX is the service a multi-node NVLink domain needs on this node.
|
||||
IMEX discovery.IMEX `json:"imex"`
|
||||
}
|
||||
|
||||
// NVLinkDomain is the multi-node NVLink domain this node's GPUs are joined
|
||||
// to, when every one of them is joined to the same. A node half joined — a
|
||||
// card still coming up, or one that fell out — is in none: a replica that
|
||||
// counts on its NVLink would find a card that has none.
|
||||
func (n Node) NVLinkDomain() string {
|
||||
d := ""
|
||||
for _, g := range n.GPUs {
|
||||
gd := g.Fabric.Domain()
|
||||
if gd == "" || (d != "" && gd != d) {
|
||||
return ""
|
||||
}
|
||||
d = gd
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// IBPort is one port's state, identity and cumulative counters.
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
// Package nvlink works out the cluster's multi-node NVLink domains — an NVL72
|
||||
// rack, an NVLink Switch system — from what each GPU reports, and what keeps
|
||||
// one from being usable.
|
||||
//
|
||||
// A domain is a scale-up island: its GPUs reach each other over NVLink
|
||||
// whatever node they are in, so a tensor-parallel group can span its nodes
|
||||
// at NVLink speed. What it needs to be one is every card joined by the
|
||||
// fabric manager and IMEX running on every node; either missing, and a
|
||||
// replica placed across it fails or crawls.
|
||||
package nvlink
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
)
|
||||
|
||||
// Level is how much a problem matters.
|
||||
type Level string
|
||||
|
||||
// The levels.
|
||||
const (
|
||||
Fail Level = "fail"
|
||||
Warn Level = "warn"
|
||||
)
|
||||
|
||||
// Problem is one thing wrong with a domain.
|
||||
type Problem struct {
|
||||
Level Level `json:"level"`
|
||||
Domain string `json:"domain"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
// GPU is one card as its domain sees it.
|
||||
type GPU struct {
|
||||
Index int `json:"index"`
|
||||
Name string `json:"name"`
|
||||
State string `json:"state"`
|
||||
Status string `json:"status,omitempty"`
|
||||
Bandwidth string `json:"bandwidth,omitempty"`
|
||||
Joined bool `json:"joined"`
|
||||
}
|
||||
|
||||
// Member is one node of a domain.
|
||||
type Member struct {
|
||||
Node string `json:"node"`
|
||||
GPUs []GPU `json:"gpus"`
|
||||
// Usable is every card joined and IMEX up: what placement requires.
|
||||
Usable bool `json:"usable"`
|
||||
IMEX string `json:"imex"`
|
||||
IMEXChannels int `json:"imex_channels"`
|
||||
}
|
||||
|
||||
// Domain is one multi-node NVLink domain.
|
||||
type Domain struct {
|
||||
Key string `json:"key"`
|
||||
ClusterUUID string `json:"cluster_uuid"`
|
||||
CliqueID string `json:"clique_id"`
|
||||
Members []Member `json:"members"`
|
||||
GPUs int `json:"gpus"`
|
||||
Joined int `json:"joined"`
|
||||
Usable int `json:"usable_nodes"`
|
||||
}
|
||||
|
||||
// Report is every domain and what is wrong with them.
|
||||
type Report struct {
|
||||
Domains []Domain `json:"domains"`
|
||||
Problems []Problem `json:"problems"`
|
||||
}
|
||||
|
||||
// Analyze reads every node's GPUs.
|
||||
func Analyze(nodes []inventory.Node) Report {
|
||||
var r Report
|
||||
by := map[string]*Domain{}
|
||||
for _, n := range nodes {
|
||||
// The domain a node is in is the one any of its cards joined; a card
|
||||
// still joining has no domain of its own yet, and is counted against
|
||||
// the node's.
|
||||
key := ""
|
||||
for _, g := range n.GPUs {
|
||||
if d := g.Fabric.Domain(); d != "" {
|
||||
key = d
|
||||
break
|
||||
}
|
||||
}
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
d := by[key]
|
||||
if d == nil {
|
||||
uuid, clique, _ := strings.Cut(key, "/")
|
||||
d = &Domain{Key: key, ClusterUUID: uuid, CliqueID: clique}
|
||||
by[key] = d
|
||||
}
|
||||
m := Member{Node: n.ID, IMEX: n.IMEX.State, IMEXChannels: n.IMEX.Channels}
|
||||
joined := 0
|
||||
for _, g := range n.GPUs {
|
||||
v := GPU{Index: g.Index, Name: g.Name}
|
||||
if f := g.Fabric; f != nil {
|
||||
v.State, v.Status, v.Bandwidth = f.State, f.Status, f.Bandwidth
|
||||
v.Joined = f.Domain() == key
|
||||
}
|
||||
if v.Joined {
|
||||
joined++
|
||||
} else {
|
||||
why := v.State
|
||||
if why == "" {
|
||||
why = "no fabric"
|
||||
}
|
||||
if v.Status != "" && !strings.EqualFold(v.Status, "Success") {
|
||||
why += ": " + v.Status
|
||||
}
|
||||
r.Problems = append(r.Problems, Problem{Level: Fail, Domain: key, Message: fmt.Sprintf(
|
||||
"%s GPU %d is not joined to its domain (%s)", n.ID, g.Index, why)})
|
||||
}
|
||||
if v.Bandwidth != "" && !strings.EqualFold(v.Bandwidth, "Full") {
|
||||
r.Problems = append(r.Problems, Problem{Level: Warn, Domain: key, Message: fmt.Sprintf(
|
||||
"%s GPU %d has %s NVLink bandwidth", n.ID, g.Index, strings.ToLower(v.Bandwidth))})
|
||||
}
|
||||
m.GPUs = append(m.GPUs, v)
|
||||
}
|
||||
imexOK := n.IMEX.State == "active" && n.IMEX.Channels > 0
|
||||
switch {
|
||||
case n.IMEX.State != "active":
|
||||
state := n.IMEX.State
|
||||
if state == "" {
|
||||
state = "not installed"
|
||||
}
|
||||
r.Problems = append(r.Problems, Problem{Level: Fail, Domain: key, Message: fmt.Sprintf(
|
||||
"%s: nvidia-imex is %s — without it the domain's GPUs cannot share memory across nodes",
|
||||
n.ID, state)})
|
||||
case n.IMEX.Channels == 0:
|
||||
r.Problems = append(r.Problems, Problem{Level: Fail, Domain: key, Message: fmt.Sprintf(
|
||||
"%s: nvidia-imex runs but there is no IMEX channel for an engine to open (/dev/nvidia-caps-imex-channels)", n.ID)})
|
||||
}
|
||||
m.Usable = joined == len(n.GPUs) && joined > 0 && imexOK
|
||||
if joined > 0 && joined < len(n.GPUs) {
|
||||
r.Problems = append(r.Problems, Problem{Level: Fail, Domain: key, Message: fmt.Sprintf(
|
||||
"%s is half joined (%d of %d GPUs), so no replica spans the domain through it", n.ID, joined, len(n.GPUs))})
|
||||
}
|
||||
d.GPUs += len(n.GPUs)
|
||||
d.Joined += joined
|
||||
if m.Usable {
|
||||
d.Usable++
|
||||
}
|
||||
d.Members = append(d.Members, m)
|
||||
}
|
||||
for _, d := range by {
|
||||
sort.Slice(d.Members, func(i, j int) bool { return d.Members[i].Node < d.Members[j].Node })
|
||||
r.Domains = append(r.Domains, *d)
|
||||
}
|
||||
sort.Slice(r.Domains, func(i, j int) bool { return r.Domains[i].Key < r.Domains[j].Key })
|
||||
sort.SliceStable(r.Problems, func(i, j int) bool {
|
||||
if r.Problems[i].Level != r.Problems[j].Level {
|
||||
return r.Problems[i].Level == Fail
|
||||
}
|
||||
return r.Problems[i].Message < r.Problems[j].Message
|
||||
})
|
||||
return r
|
||||
}
|
||||
|
||||
// Short is a domain's key as a person reads it: the cluster's first group
|
||||
// and the clique.
|
||||
func Short(key string) string {
|
||||
uuid, clique, _ := strings.Cut(key, "/")
|
||||
if i := strings.IndexByte(uuid, '-'); i > 0 {
|
||||
uuid = uuid[:i]
|
||||
}
|
||||
return uuid + "·" + clique
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package nvlink
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
)
|
||||
|
||||
func tray(id string, gpus int, state string) inventory.Node {
|
||||
n := inventory.Node{ID: id, IMEX: discovery.IMEX{State: "active", Channels: 1}}
|
||||
for i := range gpus {
|
||||
n.GPUs = append(n.GPUs, discovery.GPU{Index: i, Name: "NVIDIA GB200", Fabric: &discovery.GPUFabric{
|
||||
State: state, Status: "Success", ClusterUUID: "76b1ad76-7c24-4b6b-9f6c-2a1e0d8c11aa", CliqueID: "1", Bandwidth: "Full"}})
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func has(t *testing.T, r Report, l Level, s string) {
|
||||
t.Helper()
|
||||
for _, p := range r.Problems {
|
||||
if p.Level == l && strings.Contains(p.Message, s) {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Errorf("no %s problem with %q in %+v", l, s, r.Problems)
|
||||
}
|
||||
|
||||
func TestARackOfJoinedTraysIsOneUsableDomain(t *testing.T) {
|
||||
r := Analyze([]inventory.Node{tray("t1", 4, "Completed"), tray("t2", 4, "Completed"), {ID: "cpu"}})
|
||||
if len(r.Domains) != 1 || r.Domains[0].Usable != 2 || r.Domains[0].Joined != 8 || len(r.Problems) != 0 {
|
||||
t.Errorf("report = %+v", r)
|
||||
}
|
||||
if Short(r.Domains[0].Key) != "76b1ad76·1" {
|
||||
t.Errorf("short = %q", Short(r.Domains[0].Key))
|
||||
}
|
||||
}
|
||||
|
||||
func TestWhatKeepsATrayOutIsSaid(t *testing.T) {
|
||||
half := tray("t2", 4, "Completed")
|
||||
half.GPUs[3].Fabric = &discovery.GPUFabric{State: "In Progress"}
|
||||
noIMEX := tray("t3", 4, "Completed")
|
||||
noIMEX.IMEX = discovery.IMEX{State: "inactive"}
|
||||
slow := tray("t4", 4, "Completed")
|
||||
slow.GPUs[0].Fabric.Bandwidth = "Degraded"
|
||||
r := Analyze([]inventory.Node{tray("t1", 4, "Completed"), half, noIMEX, slow})
|
||||
has(t, r, Fail, "t2 GPU 3 is not joined to its domain (In Progress)")
|
||||
has(t, r, Fail, "t2 is half joined")
|
||||
has(t, r, Fail, "t3: nvidia-imex is inactive")
|
||||
has(t, r, Warn, "t4 GPU 0 has degraded NVLink bandwidth")
|
||||
if r.Domains[0].Usable != 2 {
|
||||
t.Errorf("usable = %d, want t1 and t4", r.Domains[0].Usable)
|
||||
}
|
||||
}
|
||||
@@ -239,13 +239,20 @@ func (s *Scheduler) placeMultiNode(p Profile, avoid []string) (*Placement, []Rej
|
||||
var rejections []Rejection
|
||||
|
||||
fabrics := s.inv.Fabrics()
|
||||
if len(fabrics) == 0 {
|
||||
if len(fabrics) == 0 && !s.anyNVLinkDomain() {
|
||||
return nil, []Rejection{{
|
||||
Profile: p.Name,
|
||||
Reason: fmt.Sprintf("needs %d nodes on one fabric, but no node reports a fabric", p.Nodes),
|
||||
}}
|
||||
}
|
||||
|
||||
// A multi-node NVLink domain first: a replica inside one runs tensor
|
||||
// parallelism across its nodes at NVLink speed, which no InfiniBand
|
||||
// fabric matches.
|
||||
if pl := s.placeOnNVLink(p, avoid, &rejections); pl != nil {
|
||||
return pl, rejections
|
||||
}
|
||||
|
||||
canon := inventory.CanonicalSMs(s.inv.Nodes())
|
||||
for _, fabric := range fabrics {
|
||||
all := s.eligibleNodes(p, fabric, avoid, &rejections)
|
||||
@@ -262,6 +269,66 @@ func (s *Scheduler) placeMultiNode(p Profile, avoid []string) (*Placement, []Rej
|
||||
return nil, rejections
|
||||
}
|
||||
|
||||
// NVLinkFabric is how a placement names a multi-node NVLink domain in its
|
||||
// Fabric, so the shards on it are told to use it.
|
||||
const NVLinkFabric = "nvl:"
|
||||
|
||||
func (s *Scheduler) anyNVLinkDomain() bool {
|
||||
for _, n := range s.inv.Nodes() {
|
||||
if n.NVLinkDomain() != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// placeOnNVLink forms a replica inside one multi-node NVLink domain, from
|
||||
// nodes whose every card is joined to it and whose IMEX is up: a node
|
||||
// missing either would leave the replica a card, or its memory, short.
|
||||
func (s *Scheduler) placeOnNVLink(p Profile, avoid []string, rejections *[]Rejection) *Placement {
|
||||
var none []Rejection
|
||||
cands := s.eligibleNodes(p, "", avoid, &none)
|
||||
by := map[string][]candidate{}
|
||||
for _, c := range cands {
|
||||
d := c.node.NVLinkDomain()
|
||||
if d == "" {
|
||||
continue
|
||||
}
|
||||
if c.node.IMEX.State != "active" || c.node.IMEX.Channels == 0 {
|
||||
*rejections = append(*rejections, Rejection{Profile: p.Name, Pool: NVLinkFabric + shortDomain(d), Node: c.node.ID,
|
||||
Reason: "in an NVLink domain, but nvidia-imex is not up"})
|
||||
continue
|
||||
}
|
||||
by[d] = append(by[d], c)
|
||||
}
|
||||
keys := make([]string, 0, len(by))
|
||||
for k := range by {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
if len(by[keys[i]]) != len(by[keys[j]]) {
|
||||
return len(by[keys[i]]) > len(by[keys[j]])
|
||||
}
|
||||
return keys[i] < keys[j]
|
||||
})
|
||||
for _, d := range keys {
|
||||
pool := NVLinkFabric + shortDomain(d) + "/" + p.classLabel()
|
||||
if pl := s.placeOnFabric(p, NVLinkFabric+d, pool, by[d], rejections); pl != nil {
|
||||
return pl
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// shortDomain is an NVLink domain as a pool name carries it.
|
||||
func shortDomain(d string) string {
|
||||
uuid, clique, _ := strings.Cut(d, "/")
|
||||
if i := strings.IndexByte(uuid, '-'); i > 0 {
|
||||
uuid = uuid[:i]
|
||||
}
|
||||
return uuid + "." + clique
|
||||
}
|
||||
|
||||
// fabricGroup is the candidates on one physical fabric.
|
||||
type fabricGroup struct {
|
||||
key string
|
||||
|
||||
@@ -1018,3 +1018,56 @@ func TestAReplicaStaysUnderOneLeafAndOffDegradedCables(t *testing.T) {
|
||||
t.Errorf("placed on %v, want the two good nodes under leaf-b", got)
|
||||
}
|
||||
}
|
||||
|
||||
// inDomain joins every card of a node to one multi-node NVLink domain, with
|
||||
// IMEX up.
|
||||
func inDomain(n inventory.Node, clique string) inventory.Node {
|
||||
for i := range n.GPUs {
|
||||
n.GPUs[i].Fabric = &discovery.GPUFabric{State: "Completed", ClusterUUID: "76b1ad76-7c24-4b6b-9f6c-2a1e0d8c11aa", CliqueID: clique}
|
||||
}
|
||||
n.IMEX = discovery.IMEX{State: "active", Channels: 1}
|
||||
return n
|
||||
}
|
||||
|
||||
func TestAReplicaSpansNodesOverNVLinkBeforeInfiniBand(t *testing.T) {
|
||||
// Four trays on one InfiniBand fabric, two of them in an NVLink domain:
|
||||
// the pair is placed across the domain, where tensor parallelism between
|
||||
// nodes runs at NVLink speed, and given no adapters to use.
|
||||
st := store(
|
||||
onSM(node("ib1", "h100-80g", 4, "ib-a"), "0x01"), onSM(node("ib2", "h100-80g", 4, "ib-a"), "0x01"),
|
||||
inDomain(onSM(node("nv1", "h100-80g", 4, "ib-a"), "0x01"), "7"),
|
||||
inDomain(onSM(node("nv2", "h100-80g", 4, "ib-a"), "0x01"), "7"),
|
||||
)
|
||||
res := New(st).Place([]Profile{{Name: "tp8", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 2}}, nil)
|
||||
if res.Placement == nil {
|
||||
t.Fatalf("no placement:\n%s", res.Explain())
|
||||
}
|
||||
if !strings.HasPrefix(res.Placement.Fabric, NVLinkFabric) || !strings.HasPrefix(res.Placement.Pool, "nvl:76b1ad76.7/") {
|
||||
t.Errorf("fabric %q pool %q", res.Placement.Fabric, res.Placement.Pool)
|
||||
}
|
||||
for _, a := range res.Placement.Nodes {
|
||||
if !strings.HasPrefix(a.NodeID, "nv") || len(a.HCAs) != 0 {
|
||||
t.Errorf("assignment %+v", a)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestANodeWithoutIMEXIsNotCountedInItsDomain(t *testing.T) {
|
||||
a := inDomain(onSM(node("nv1", "h100-80g", 4, "ib-a"), "0x01"), "7")
|
||||
b := inDomain(onSM(node("nv2", "h100-80g", 4, "ib-a"), "0x01"), "7")
|
||||
b.IMEX.State = "inactive"
|
||||
res := New(store(a, b)).Place([]Profile{{Name: "tp8", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 2}}, nil)
|
||||
// Not across the domain; InfiniBand still carries it.
|
||||
if res.Placement == nil || strings.HasPrefix(res.Placement.Fabric, NVLinkFabric) {
|
||||
t.Fatalf("placement %+v\n%s", res.Placement, res.Explain())
|
||||
}
|
||||
|
||||
// With no InfiniBand to fall back on, the refusal says why.
|
||||
a2 := inDomain(node("nv1", "h100-80g", 4), "7")
|
||||
b2 := inDomain(node("nv2", "h100-80g", 4), "7")
|
||||
b2.IMEX.State = "inactive"
|
||||
res = New(store(a2, b2)).Place([]Profile{{Name: "tp8", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 2}}, nil)
|
||||
if res.Placement != nil || !strings.Contains(res.Explain(), "nvidia-imex is not up") {
|
||||
t.Errorf("the explanation should say why the domain was passed over:\n%s", res.Explain())
|
||||
}
|
||||
}
|
||||
@@ -56,8 +56,10 @@ type Facts struct {
|
||||
TopoErr string `json:"topo_error,omitempty"`
|
||||
// NVSwitch is set when the machine has NVSwitch devices, which need the
|
||||
// fabric manager running before CUDA will start.
|
||||
NVSwitch bool `json:"nvswitch"`
|
||||
FabricManager string `json:"fabric_manager,omitempty"`
|
||||
NVSwitch bool `json:"nvswitch"`
|
||||
// IMEX is the service multi-node NVLink needs.
|
||||
IMEX discovery.IMEX `json:"imex"`
|
||||
FabricManager string `json:"fabric_manager,omitempty"`
|
||||
|
||||
HCAs []discovery.HCA `json:"hcas"`
|
||||
// SMErrors say, per port, why the subnet manager could not be asked.
|
||||
@@ -120,7 +122,11 @@ func Collect(ctx context.Context, o Options) Facts {
|
||||
f.TopoErr = err.Error()
|
||||
}
|
||||
}
|
||||
if fab, err := discovery.DiscoverGPUFabric(ctx, smi); err == nil {
|
||||
discovery.ApplyFabric(gpus, fab)
|
||||
}
|
||||
f.GPUs = gpus
|
||||
f.IMEX = discovery.DiscoverIMEX(ctx)
|
||||
}
|
||||
if m, _ := filepath.Glob("/dev/nvidia-nvswitch*"); len(m) > 0 {
|
||||
f.NVSwitch = true
|
||||
@@ -306,6 +312,48 @@ func Evaluate(f Facts) []Finding {
|
||||
}
|
||||
}
|
||||
|
||||
// --- multi-node NVLink ---
|
||||
domain := ""
|
||||
for _, g := range f.GPUs {
|
||||
fb := g.Fabric
|
||||
if fb == nil || strings.EqualFold(fb.State, "Not Supported") {
|
||||
continue
|
||||
}
|
||||
if d := fb.Domain(); d != "" {
|
||||
domain = d
|
||||
if fb.Bandwidth != "" && !strings.EqualFold(fb.Bandwidth, "Full") {
|
||||
add(Warn, "nvlink", fmt.Sprintf("GPU %d's NVLink bandwidth is %s", g.Index, strings.ToLower(fb.Bandwidth)),
|
||||
"check its NVLink cables and trays: nvidia-smi -q -d FABRIC")
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !strings.EqualFold(fb.State, "Completed") {
|
||||
why := fb.State
|
||||
if fb.Status != "" && !strings.EqualFold(fb.Status, "Success") {
|
||||
why += ": " + fb.Status
|
||||
}
|
||||
add(Fail, "nvlink", fmt.Sprintf("GPU %d is not joined to its NVLink domain (%s)", g.Index, why),
|
||||
"the fabric manager joins it: check nvidia-fabricmanager and the NVLink switch trays")
|
||||
}
|
||||
}
|
||||
if domain != "" {
|
||||
add(OK, "nvlink", "the GPUs are in multi-node NVLink domain "+domain, "")
|
||||
switch {
|
||||
case f.IMEX.State != "active":
|
||||
state := f.IMEX.State
|
||||
if state == "" {
|
||||
state = "not installed"
|
||||
}
|
||||
add(Fail, "nvlink", "nvidia-imex is "+state+": without it the domain's GPUs cannot share memory across nodes",
|
||||
"systemctl enable --now nvidia-imex, with every node of the domain in /etc/nvidia-imex/nodes_config.cfg")
|
||||
case f.IMEX.Channels == 0:
|
||||
add(Fail, "nvlink", "nvidia-imex runs but no IMEX channel exists for an engine to open",
|
||||
"load the driver with options nvidia NVreg_CreateImexChannel0=1")
|
||||
default:
|
||||
add(OK, "nvlink", fmt.Sprintf("nvidia-imex is active with %d channel(s)", f.IMEX.Channels), "")
|
||||
}
|
||||
}
|
||||
|
||||
// --- InfiniBand and RoCE ---
|
||||
if len(f.HCAs) == 0 {
|
||||
add(Info, "fabric", "no InfiniBand or RoCE adapters: this node can only run single-node replicas", "")
|
||||
|
||||
@@ -123,3 +123,21 @@ func TestANodeOutsideTheFabricIsAnAdapterLeftAlone(t *testing.T) {
|
||||
t.Errorf("failures: %v", levels(fs, Fail))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnNVLinkDomainNeedsEveryGPUJoinedAndIMEX(t *testing.T) {
|
||||
f := healthy()
|
||||
for i := range f.GPUs {
|
||||
f.GPUs[i].Fabric = &discovery.GPUFabric{State: "Completed", Status: "Success",
|
||||
ClusterUUID: "76b1ad76-7c24-4b6b-9f6c-2a1e0d8c11aa", CliqueID: "1", Bandwidth: "Full"}
|
||||
}
|
||||
f.IMEX = discovery.IMEX{State: "active", Channels: 1}
|
||||
fs := Evaluate(f)
|
||||
has(t, fs, OK, "multi-node NVLink domain")
|
||||
has(t, fs, OK, "nvidia-imex is active")
|
||||
|
||||
f.GPUs[5].Fabric = &discovery.GPUFabric{State: "In Progress"}
|
||||
f.IMEX.Channels = 0
|
||||
fs = Evaluate(f)
|
||||
has(t, fs, Fail, "GPU 5 is not joined to its NVLink domain (In Progress)")
|
||||
has(t, fs, Fail, "no IMEX channel")
|
||||
}
|
||||
@@ -164,6 +164,7 @@ func (r *Router) Handler() http.Handler {
|
||||
}, r.log)))
|
||||
mux.HandleFunc("GET /admin/nodes/{id}/logs", pages.AdminPage(
|
||||
web.NodeLogsPage(r.clusterView, false, version.Version)))
|
||||
mux.HandleFunc("GET /admin/nvlink", pages.AdminPage(web.NVLinkPage(r.clusterView, false, version.Version)))
|
||||
mux.HandleFunc("GET /admin/fabrics", pages.AdminPage(web.FabricsPage(r.clusterView, false, version.Version)))
|
||||
mux.HandleFunc("GET /admin/fabrics/{id}", pages.AdminPage(web.FabricsPage(r.clusterView, false, version.Version)))
|
||||
mux.HandleFunc("GET /admin/fabrics/{id}/topology.json", pages.AdminPage(web.FabricTopologyJSON(r.clusterView)))
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
@@ -28,6 +29,8 @@ type DashboardData struct {
|
||||
// Fabrics is the physical fabrics and what disagrees about them, from a
|
||||
// source that can say.
|
||||
Fabrics *fabrics.Report
|
||||
// NVLink is the multi-node NVLink domains, from a source that can say.
|
||||
NVLink *nvlink.Report
|
||||
|
||||
Users []store.UserUsage
|
||||
Models2 []store.ModelUsage
|
||||
@@ -179,6 +182,11 @@ func Dashboard(src Source, usage UsageSource, version string, refresh time.Durat
|
||||
r := fs.Fabrics()
|
||||
d.Fabrics = &r
|
||||
}
|
||||
if ns, ok := src.(NVLinkSource); ok {
|
||||
if r := ns.NVLink(); len(r.Domains) > 0 {
|
||||
d.NVLink = &r
|
||||
}
|
||||
}
|
||||
}
|
||||
if usage != nil {
|
||||
rows, err := usage.UsageSince(r.Context(), time.Now().Add(-24*time.Hour))
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/topology"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
|
||||
)
|
||||
@@ -145,3 +146,31 @@ func TestAFabricsPageCarriesItsMapSafely(t *testing.T) {
|
||||
t.Errorf("json: %d %s", code, body)
|
||||
}
|
||||
}
|
||||
|
||||
type nvlSource struct{ busySource }
|
||||
|
||||
func (nvlSource) NVLink() nvlink.Report {
|
||||
return nvlink.Report{
|
||||
Domains: []nvlink.Domain{{Key: "76b1ad76-7c24/7", ClusterUUID: "76b1ad76-7c24", CliqueID: "7", GPUs: 8, Joined: 7, Usable: 1,
|
||||
Members: []nvlink.Member{
|
||||
{Node: "t1", Usable: true, IMEX: "active", IMEXChannels: 1, GPUs: []nvlink.GPU{{Index: 0, Joined: true, State: "Completed"}}},
|
||||
{Node: "t2", IMEX: "inactive", GPUs: []nvlink.GPU{{Index: 3, State: "In Progress"}}},
|
||||
}}},
|
||||
Problems: []nvlink.Problem{{Level: nvlink.Fail, Domain: "76b1ad76-7c24/7", Message: "t2: nvidia-imex is inactive"}},
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheNVLinkPageDrawsEachDomainAsARack(t *testing.T) {
|
||||
_, body := get(t, NVLinkPage(nvlSource{}, false, "v1"), "http://x/admin/nvlink", nil)
|
||||
for _, want := range []string{`class="rack troubled"`, "76b1ad76·7", "1/2</b><span>trays usable", `class="chip joined"`,
|
||||
`class="chip lost"`, `class="tray out"`, "t2: nvidia-imex is inactive", `class="imex up"`} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("missing %q", want)
|
||||
}
|
||||
}
|
||||
// And the cluster page points at it.
|
||||
_, body = get(t, Dashboard(nvlSource{}, fakeUsage{}, "v1", time.Second, false, NavExtras{}, nil), "http://x/admin", nil)
|
||||
if !strings.Contains(body, "see the racks") {
|
||||
t.Error("the cluster page does not lead to the racks")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
|
||||
)
|
||||
|
||||
// NVLinkSource is a Source that can report multi-node NVLink domains.
|
||||
type NVLinkSource interface {
|
||||
NVLink() nvlink.Report
|
||||
}
|
||||
|
||||
type nvlinkPageData struct {
|
||||
Nav Nav
|
||||
OnAdminListener bool
|
||||
Nodes string
|
||||
Domains []nvlinkDomainView
|
||||
Problems []nvlink.Problem
|
||||
Version, Now string
|
||||
}
|
||||
|
||||
type nvlinkDomainView struct {
|
||||
nvlink.Domain
|
||||
Short string
|
||||
Problems []nvlink.Problem
|
||||
}
|
||||
|
||||
// NVLinkPage shows every multi-node NVLink domain as a rack of trays.
|
||||
func NVLinkPage(src Source, onAdminListener bool, version string) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
ns, ok := src.(NVLinkSource)
|
||||
if !ok {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
rep := ns.NVLink()
|
||||
d := nvlinkPageData{Nav: navFrom(r, "cluster"), OnAdminListener: onAdminListener, Nodes: "/admin/nodes",
|
||||
Problems: rep.Problems, Version: version, Now: time.Now().UTC().Format(time.RFC3339)}
|
||||
if onAdminListener {
|
||||
d.Nodes = "/nodes"
|
||||
}
|
||||
for _, dom := range rep.Domains {
|
||||
v := nvlinkDomainView{Domain: dom, Short: nvlink.Short(dom.Key)}
|
||||
for _, p := range rep.Problems {
|
||||
if p.Domain == dom.Key {
|
||||
v.Problems = append(v.Problems, p)
|
||||
}
|
||||
}
|
||||
d.Domains = append(d.Domains, v)
|
||||
}
|
||||
noStore(w)
|
||||
render(w, "nvlink", d)
|
||||
}
|
||||
}
|
||||
@@ -677,3 +677,71 @@ ul.fabric-problems li.info { color: var(--muted); }
|
||||
}
|
||||
.mode-rails .fm-host.down circle:not(.fm-halo) { fill: var(--fm-bad); stroke: #fecdd3; }
|
||||
.fm-svg.focus .fm-warn { opacity: .25; }
|
||||
|
||||
/* A node's GPUs: which pairs NVLink joins. */
|
||||
.gpu-wiring { display: flex; flex-wrap: wrap; gap: 1.25rem; align-items: flex-start; }
|
||||
table.nvl-matrix { width: auto; border-collapse: separate; border-spacing: 3px; background: none; font-size: .75rem; }
|
||||
table.nvl-matrix th { font-weight: 600; color: var(--muted); padding: 0 .3rem; text-align: center; border: 0; }
|
||||
table.nvl-matrix td { width: 2rem; height: 2rem; text-align: center; border-radius: 6px; padding: 0; border: 0;
|
||||
font-weight: 700; font-size: .7rem; }
|
||||
table.nvl-matrix td.self { background: var(--line); }
|
||||
table.nvl-matrix td.nv { color: #04211c; background: linear-gradient(135deg, #5eead4, #2dd4bf);
|
||||
box-shadow: 0 0 10px rgba(45, 212, 191, .45); }
|
||||
table.nvl-matrix td.pcie { color: var(--muted); background: var(--card); border: 1px solid var(--line); }
|
||||
table.gpu-list { width: auto; flex: 1 1 22rem; }
|
||||
|
||||
/* ---------------------------------------------------------------------
|
||||
NVLink domains, drawn as racks: the switch band on top, a backplane
|
||||
carrying light down the side, trays of GPU chips that glow when joined.
|
||||
--------------------------------------------------------------------- */
|
||||
.racks { display: grid; grid-template-columns: repeat(auto-fill, minmax(26rem, 1fr)); gap: 1.25rem; margin: 1rem 0 1.5rem; }
|
||||
.rack { border-radius: 16px; overflow: hidden; color: #dbe4ff;
|
||||
background: radial-gradient(700px 260px at 50% -20%, #1a3b3b 0%, #0b1020 60%), #0b1020;
|
||||
border: 1px solid #1c2f3f; box-shadow: 0 20px 50px -28px rgba(45, 212, 191, .45); }
|
||||
.rack.troubled { border-color: #4a1d2c; box-shadow: 0 20px 50px -28px rgba(251, 113, 133, .45); }
|
||||
.rack-head { display: flex; justify-content: space-between; align-items: flex-start; gap: 1rem; padding: .9rem 1rem .6rem; }
|
||||
.rack-head h2 { margin: 0; color: #fff; font-size: 1.15rem; letter-spacing: .02em; }
|
||||
.rack-head .note, .rack-head code { color: #7c89ad; background: none; font-size: .72rem; }
|
||||
.rack-stats { display: flex; gap: 1rem; text-align: right; }
|
||||
.rack-stats b { display: block; font-size: 1.1rem; color: #fff; font-variant-numeric: tabular-nums; }
|
||||
.rack-stats span { font-size: .62rem; letter-spacing: .1em; text-transform: uppercase; color: #7c89ad; }
|
||||
.rack-body { padding: .25rem 1rem 1rem; }
|
||||
.nvswitch { position: relative; height: 1.9rem; border-radius: 8px; overflow: hidden; display: flex; align-items: center;
|
||||
justify-content: center; font-size: .68rem; letter-spacing: .22em; text-transform: uppercase; color: #ccfbf1;
|
||||
background: linear-gradient(90deg, #0f3b3a, #115e59 50%, #0f3b3a); border: 1px solid #2dd4bf55;
|
||||
box-shadow: 0 0 22px rgba(45, 212, 191, .35), inset 0 0 12px rgba(45, 212, 191, .25); }
|
||||
.nvswitch span { position: relative; z-index: 1; text-shadow: 0 0 8px rgba(94, 234, 212, .8); }
|
||||
.nvswitch i { position: absolute; inset: 0; background: linear-gradient(100deg, transparent 30%, rgba(255,255,255,.28) 50%, transparent 70%);
|
||||
transform: translateX(-100%); animation: nv-shimmer 3.2s ease-in-out infinite; }
|
||||
@keyframes nv-shimmer { to { transform: translateX(100%); } }
|
||||
.rack-trays { position: relative; margin-top: .6rem; padding-left: 1.1rem; display: flex; flex-direction: column; gap: .45rem; }
|
||||
.backplane { position: absolute; left: .3rem; top: -.6rem; bottom: .3rem; width: 4px; border-radius: 4px;
|
||||
background: linear-gradient(180deg, #2dd4bf, #0f766e 40%, #2dd4bf 60%, #0f766e); background-size: 100% 300%;
|
||||
box-shadow: 0 0 12px rgba(45, 212, 191, .6); animation: nv-flow 2.4s linear infinite; }
|
||||
@keyframes nv-flow { to { background-position: 0 300%; } }
|
||||
.tray { position: relative; display: flex; align-items: center; gap: .75rem; padding: .45rem .6rem; border-radius: 10px;
|
||||
background: linear-gradient(180deg, rgba(255,255,255,.05), rgba(255,255,255,.015)); border: 1px solid #1f2d4a; }
|
||||
.tray::before { content: ""; position: absolute; left: -.8rem; top: 50%; width: .8rem; height: 2px;
|
||||
background: linear-gradient(90deg, #2dd4bf, transparent); box-shadow: 0 0 6px #2dd4bf; }
|
||||
.tray.out { border-color: #5b2333; background: linear-gradient(180deg, rgba(251,113,133,.08), rgba(251,113,133,.02)); }
|
||||
.tray.out::before { background: linear-gradient(90deg, #fb7185, transparent); box-shadow: 0 0 6px #fb7185; }
|
||||
.tray-name { min-width: 6.5rem; font-size: .8rem; font-weight: 700; color: #e0f2fe; text-decoration: none; }
|
||||
.tray-name:hover { color: #5eead4; }
|
||||
.chips { display: flex; flex-wrap: wrap; gap: .35rem; flex: 1; }
|
||||
.chip { width: 1.7rem; height: 1.7rem; border-radius: 6px; display: inline-flex; align-items: center; justify-content: center;
|
||||
font-size: .62rem; font-weight: 800; color: #04211c; cursor: default; }
|
||||
.chip.joined { background: linear-gradient(135deg, #5eead4, #14b8a6); box-shadow: 0 0 12px rgba(45, 212, 191, .55);
|
||||
animation: nv-breathe 3.6s ease-in-out infinite; }
|
||||
.chip.joined.slow { background: linear-gradient(135deg, #fde68a, #f59e0b); box-shadow: 0 0 12px rgba(245, 158, 11, .6); }
|
||||
.chip.lost { color: #fff; background: linear-gradient(135deg, #fb7185, #be123c); box-shadow: 0 0 14px rgba(251, 113, 133, .7);
|
||||
animation: nv-alarm 1.2s ease-in-out infinite; }
|
||||
@keyframes nv-breathe { 50% { box-shadow: 0 0 4px rgba(45, 212, 191, .25); } }
|
||||
@keyframes nv-alarm { 50% { opacity: .55; } }
|
||||
.imex { font-size: .6rem; font-weight: 800; letter-spacing: .08em; padding: .15rem .4rem; border-radius: 999px;
|
||||
color: #fecdd3; border: 1px solid #fb7185; }
|
||||
.imex.up { color: #99f6e4; border-color: #2dd4bf; box-shadow: 0 0 8px rgba(45, 212, 191, .35); }
|
||||
.rack-problems { padding: 0 1rem .9rem; margin: 0; }
|
||||
.rack-problems li { color: #dbe4ff; }
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.nvswitch i, .backplane, .chip.joined, .chip.lost { animation: none; }
|
||||
}
|
||||
@@ -165,6 +165,13 @@ A replica spanning nodes is only ever formed on one of those. Fully free counts
|
||||
unused nodes, which is what decides whether such a replica can land.</p>
|
||||
{{end}}
|
||||
|
||||
{{with .NVLink}}
|
||||
<h2>NVLink domains</h2>
|
||||
<p>{{len .Domains}} multi-node domain{{if ne (len .Domains) 1}}s{{end}}{{range .Domains}} · <code>{{shortDomain .Key}}</code> {{.Usable}}/{{len .Members}} trays usable{{end}}
|
||||
{{if .Problems}} · <span class="bad">{{len .Problems}} problem{{if ne (len .Problems) 1}}s{{end}}</span>{{end}}
|
||||
· <a href="{{if $.OnAdminListener}}/nvlink{{else}}/admin/nvlink{{end}}">see the racks →</a></p>
|
||||
{{end}}
|
||||
|
||||
{{with .Fleet}}
|
||||
<h2>What the cluster is doing <span class="opt">last {{.Window}}</span></h2>
|
||||
{{if .Err}}
|
||||
|
||||
@@ -52,6 +52,34 @@ the adapter came up: a number that grows is the one to look at.</p>
|
||||
{{end}}
|
||||
{{end}}
|
||||
|
||||
{{if .Node.GPUs}}
|
||||
<h2>GPUs and NVLink</h2>
|
||||
<p class="sub">
|
||||
{{with .Node.NVLinkDomain}}In NVLink domain <code>{{.}}</code>: tensor parallelism can span this node and the rest of its domain.
|
||||
{{else}}NVLink ends at this machine.{{end}}
|
||||
{{if or .Node.NVLinkDomain .Node.IMEX}} · IMEX {{if eq .Node.IMEX "active"}}<span class="state-ready">active</span>, {{.Node.IMEXChannels}} channel{{if ne .Node.IMEXChannels 1}}s{{end}}{{else}}<span class="bad">{{or .Node.IMEX "not installed"}}</span>{{end}}{{end}}
|
||||
</p>
|
||||
<div class="gpu-wiring">
|
||||
<table class="nvl-matrix" aria-label="NVLink between GPUs">
|
||||
<tr><th></th>{{range .Node.GPUs}}<th>{{.Index}}</th>{{end}}</tr>
|
||||
{{range $a := .Node.GPUs}}<tr><th>GPU {{$a.Index}}</th>
|
||||
{{range $b := $.Node.GPUs}}{{if eq $a.Index $b.Index}}<td class="self"></td>{{else if hasInt $a.NVLinkPeers $b.Index}}<td class="nv" title="GPU {{$a.Index}} ↔ GPU {{$b.Index}} over NVLink">NV</td>{{else}}<td class="pcie" title="GPU {{$a.Index}} ↔ GPU {{$b.Index}} over PCIe">·</td>{{end}}{{end}}
|
||||
</tr>{{end}}
|
||||
</table>
|
||||
<table class="compact gpu-list">
|
||||
<tr><th>GPU</th><th>Name</th><th class="num">NUMA</th><th>Nearest adapter</th><th>Multi-node NVLink</th></tr>
|
||||
{{range .Node.GPUs}}<tr>
|
||||
<td>{{.Index}}</td><td>{{.Name}}</td><td class="num">{{.NUMA}}</td>
|
||||
<td>{{if .NearestNIC}}<code>{{.NearestNIC}}</code> <span class="note">{{.NICPath}}</span>{{else}}—{{end}}</td>
|
||||
<td>{{if .Domain}}<span class="state-ready">joined</span>{{if and .Bandwidth (ne .Bandwidth "Full")}} <span class="bad">{{.Bandwidth}} bandwidth</span>{{end}}{{else if .FabricState}}<span class="bad">{{.FabricState}}</span>{{else}}<span class="note">—</span>{{end}}</td>
|
||||
</tr>{{end}}
|
||||
</table>
|
||||
</div>
|
||||
<p class="note">NV is a pair of cards joined by NVLink; a tensor-parallel group kept to NV pairs runs its
|
||||
all-reduces at NVLink speed, and placement picks it that way. The nearest adapter is where a card's share of
|
||||
traffic between nodes should go: PIX and PXB are under the same PCIe switch, SYS is across the processors.</p>
|
||||
{{end}}
|
||||
|
||||
<h2>What runs here</h2>
|
||||
{{if .Engines}}
|
||||
<table><tr><th>Model</th><th>Replica</th><th>State</th><th>CPU</th><th class="num">Memory</th></tr>
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
{{define "nvlink"}}<!doctype html>
|
||||
<html lang="en"><head>
|
||||
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<meta http-equiv="refresh" content="30">
|
||||
<title>NVLink domains · llmgw</title><link rel="stylesheet" href="{{stylesheet}}">
|
||||
</head><body><div class="wrap wide">
|
||||
<header>
|
||||
<h1>NVLink domains</h1>
|
||||
<p class="sub">{{len .Domains}} multi-node domain{{if ne (len .Domains) 1}}s{{end}} — racks whose GPUs reach each other over NVLink across nodes</p>
|
||||
{{if .OnAdminListener}}<nav><a href="/">cluster</a></nav>{{else}}{{template "nav" .Nav}}{{end}}
|
||||
</header>
|
||||
|
||||
{{if not .Domains}}
|
||||
<p class="empty">No GPU reports a multi-node NVLink domain. An eight-GPU machine's NVSwitch ends at its
|
||||
chassis and is shown on its node's page; a domain appears here once the fabric manager has joined the
|
||||
GPUs of an NVL72 rack, or an NVLink Switch system, to one.</p>
|
||||
{{end}}
|
||||
|
||||
<div class="racks">
|
||||
{{range .Domains}}
|
||||
<section class="rack{{if .Problems}} troubled{{end}}">
|
||||
<header class="rack-head">
|
||||
<div><h2>{{.Short}}</h2><p class="note"><code>{{.ClusterUUID}}</code> · clique {{.CliqueID}}</p></div>
|
||||
<div class="rack-stats">
|
||||
<div><b>{{.Usable}}/{{len .Members}}</b><span>trays usable</span></div>
|
||||
<div><b>{{.Joined}}/{{.GPUs}}</b><span>GPUs joined</span></div>
|
||||
</div>
|
||||
</header>
|
||||
<div class="rack-body">
|
||||
<div class="nvswitch"><span>NVLink Switch</span><i></i></div>
|
||||
<div class="rack-trays">
|
||||
<div class="backplane" aria-hidden="true"></div>
|
||||
{{range .Members}}
|
||||
<div class="tray{{if not .Usable}} out{{end}}">
|
||||
<a class="tray-name" href="{{$.Nodes}}/{{.Node}}">{{.Node}}</a>
|
||||
<div class="chips">{{range .GPUs}}<span class="chip{{if .Joined}} joined{{else}} lost{{end}}{{if and .Bandwidth (ne .Bandwidth "Full")}} slow{{end}}"
|
||||
title="GPU {{.Index}} · {{.Name}} · {{or .State "no fabric"}}{{if .Bandwidth}} · {{.Bandwidth}} bandwidth{{end}}">{{.Index}}</span>{{end}}</div>
|
||||
<span class="imex{{if and (eq .IMEX "active") .IMEXChannels}} up{{end}}" title="nvidia-imex {{or .IMEX "not installed"}}, {{.IMEXChannels}} channel(s)">IMEX</span>
|
||||
</div>
|
||||
{{end}}
|
||||
</div>
|
||||
</div>
|
||||
{{if .Problems}}<ul class="fabric-problems rack-problems">{{range .Problems}}
|
||||
<li class="{{.Level}}"><span class="tag{{if eq (printf "%s" .Level) "fail"}} bad{{end}}">{{.Level}}</span> {{.Message}}</li>{{end}}</ul>{{end}}
|
||||
</section>
|
||||
{{end}}
|
||||
</div>
|
||||
|
||||
<p class="note">A replica that spans nodes is placed inside one of these first: tensor parallelism between its
|
||||
nodes then runs over NVLink rather than InfiniBand. Only trays with every GPU joined and IMEX up are used —
|
||||
a tray short of either would leave the replica a card, or its memory, short.</p>
|
||||
|
||||
<footer><span>llmgw {{.Version}} · refreshes every 30s</span><span>{{.Now}}</span></footer>
|
||||
</div></body></html>{{end}}
|
||||
+6
-3
@@ -26,6 +26,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
@@ -128,9 +129,11 @@ var funcs = template.FuncMap{
|
||||
// shows. Binary units because that is what `ls -lh` and every disk-full
|
||||
// message on the host will say, and a page that disagrees with the shell
|
||||
// about the size of the same file starts an argument about neither.
|
||||
"bytes": humanBytes,
|
||||
"u64": func(v uint64) int64 { return int64(min(v, 1<<62)) },
|
||||
"has": func(list []string, v string) bool { return slices.Contains(list, v) },
|
||||
"bytes": humanBytes,
|
||||
"u64": func(v uint64) int64 { return int64(min(v, 1<<62)) },
|
||||
"has": func(list []string, v string) bool { return slices.Contains(list, v) },
|
||||
"hasInt": func(list []int, v int) bool { return slices.Contains(list, v) },
|
||||
"shortDomain": nvlink.Short,
|
||||
// dict lets one chart template be called with named arguments. Go
|
||||
// templates pass a single value, and four charts differing in three
|
||||
// strings each is four copies of the same markup otherwise.
|
||||
|
||||
Reference in new issue
Block a user