diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index 0dd3588..65685a8 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -90,7 +90,7 @@ func (x ShardStatus_State) Number() protoreflect.EnumNumber { // Deprecated: Use ShardStatus_State.Descriptor instead. func (ShardStatus_State) EnumDescriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{17, 0} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{19, 0} } // AgentMessage is anything a node tells the control plane. @@ -470,6 +470,8 @@ type Register struct { // meant "an agent too old to say" and constrained nothing — so a freshly // built node with no engine installed yet was a candidate for every model. EnginesProbed bool `protobuf:"varint,14,opt,name=engines_probed,json=enginesProbed,proto3" json:"engines_probed,omitempty"` + // imex is the service that lets a multi-node NVLink domain share memory. + Imex *IMEX `protobuf:"bytes,15,opt,name=imex,proto3" json:"imex,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -602,6 +604,13 @@ func (x *Register) GetEnginesProbed() bool { return false } +func (x *Register) GetImex() *IMEX { + if x != nil { + return x.Imex + } + return nil +} + // Host describes the operating system and processor of a node. type Host struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -967,9 +976,17 @@ type GPU struct { UtilizationPercent int32 `protobuf:"varint,15,opt,name=utilization_percent,json=utilizationPercent,proto3" json:"utilization_percent,omitempty"` // nic_paths is how far each network adapter is from this card, by HCA // name: PIX, PXB, PHB, NODE or SYS, from nvidia-smi topo. - NicPaths map[string]string `protobuf:"bytes,16,rep,name=nic_paths,json=nicPaths,proto3" json:"nic_paths,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + NicPaths map[string]string `protobuf:"bytes,16,rep,name=nic_paths,json=nicPaths,proto3" json:"nic_paths,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + // The card's multi-node NVLink membership, from nvidia-smi -q: state is + // Completed once the fabric manager has joined it; cluster_uuid and + // clique_id name the domain it joined. + FabricState string `protobuf:"bytes,17,opt,name=fabric_state,json=fabricState,proto3" json:"fabric_state,omitempty"` + FabricStatus string `protobuf:"bytes,18,opt,name=fabric_status,json=fabricStatus,proto3" json:"fabric_status,omitempty"` + ClusterUuid string `protobuf:"bytes,19,opt,name=cluster_uuid,json=clusterUuid,proto3" json:"cluster_uuid,omitempty"` + CliqueId string `protobuf:"bytes,20,opt,name=clique_id,json=cliqueId,proto3" json:"clique_id,omitempty"` + FabricBandwidth string `protobuf:"bytes,21,opt,name=fabric_bandwidth,json=fabricBandwidth,proto3" json:"fabric_bandwidth,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GPU) Reset() { @@ -1114,6 +1131,41 @@ func (x *GPU) GetNicPaths() map[string]string { return nil } +func (x *GPU) GetFabricState() string { + if x != nil { + return x.FabricState + } + return "" +} + +func (x *GPU) GetFabricStatus() string { + if x != nil { + return x.FabricStatus + } + return "" +} + +func (x *GPU) GetClusterUuid() string { + if x != nil { + return x.ClusterUuid + } + return "" +} + +func (x *GPU) GetCliqueId() string { + if x != nil { + return x.CliqueId + } + return "" +} + +func (x *GPU) GetFabricBandwidth() string { + if x != nil { + return x.FabricBandwidth + } + return "" +} + type FabricAttachment struct { state protoimpl.MessageState `protogen:"open.v1"` // fabric is the operator-assigned name. It is not derived from the GID @@ -1685,6 +1737,145 @@ func (x *SubnetManager) GetState() string { return "" } +// IMEX is the nvidia-imex service and its channels. +type IMEX struct { + state protoimpl.MessageState `protogen:"open.v1"` + // state is systemd's word for it, empty where it is not installed. + State string `protobuf:"bytes,1,opt,name=state,proto3" json:"state,omitempty"` + Channels int32 `protobuf:"varint,2,opt,name=channels,proto3" json:"channels,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *IMEX) Reset() { + *x = IMEX{} + mi := &file_llmgw_v1_agent_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *IMEX) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*IMEX) ProtoMessage() {} + +func (x *IMEX) ProtoReflect() protoreflect.Message { + mi := &file_llmgw_v1_agent_proto_msgTypes[13] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use IMEX.ProtoReflect.Descriptor instead. +func (*IMEX) Descriptor() ([]byte, []int) { + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13} +} + +func (x *IMEX) GetState() string { + if x != nil { + return x.State + } + return "" +} + +func (x *IMEX) GetChannels() int32 { + if x != nil { + return x.Channels + } + return 0 +} + +// GPUFabricState is one GPU's NVLink domain membership. +type GPUFabricState struct { + state protoimpl.MessageState `protogen:"open.v1"` + Index int32 `protobuf:"varint,1,opt,name=index,proto3" json:"index,omitempty"` + State string `protobuf:"bytes,2,opt,name=state,proto3" json:"state,omitempty"` + Status string `protobuf:"bytes,3,opt,name=status,proto3" json:"status,omitempty"` + ClusterUuid string `protobuf:"bytes,4,opt,name=cluster_uuid,json=clusterUuid,proto3" json:"cluster_uuid,omitempty"` + CliqueId string `protobuf:"bytes,5,opt,name=clique_id,json=cliqueId,proto3" json:"clique_id,omitempty"` + Bandwidth string `protobuf:"bytes,6,opt,name=bandwidth,proto3" json:"bandwidth,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GPUFabricState) Reset() { + *x = GPUFabricState{} + mi := &file_llmgw_v1_agent_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GPUFabricState) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GPUFabricState) ProtoMessage() {} + +func (x *GPUFabricState) ProtoReflect() protoreflect.Message { + mi := &file_llmgw_v1_agent_proto_msgTypes[14] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GPUFabricState.ProtoReflect.Descriptor instead. +func (*GPUFabricState) Descriptor() ([]byte, []int) { + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14} +} + +func (x *GPUFabricState) GetIndex() int32 { + if x != nil { + return x.Index + } + return 0 +} + +func (x *GPUFabricState) GetState() string { + if x != nil { + return x.State + } + return "" +} + +func (x *GPUFabricState) GetStatus() string { + if x != nil { + return x.Status + } + return "" +} + +func (x *GPUFabricState) GetClusterUuid() string { + if x != nil { + return x.ClusterUuid + } + return "" +} + +func (x *GPUFabricState) GetCliqueId() string { + if x != nil { + return x.CliqueId + } + return "" +} + +func (x *GPUFabricState) GetBandwidth() string { + if x != nil { + return x.Bandwidth + } + return "" +} + type Heartbeat struct { state protoimpl.MessageState `protogen:"open.v1"` NodeId string `protobuf:"bytes,1,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` @@ -1734,13 +1925,18 @@ type Heartbeat struct { // port going down takes its fabric away, and one coming back returns it. FabricsChanged bool `protobuf:"varint,14,opt,name=fabrics_changed,json=fabricsChanged,proto3" json:"fabrics_changed,omitempty"` Fabrics []*FabricAttachment `protobuf:"bytes,15,rep,name=fabrics,proto3" json:"fabrics,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // gpu_fabric is each GPU's NVLink domain membership as it stands now, sent + // once a minute: a fabric manager restarting drops every card out of its + // domain, and the node has to stop being offered work that spans it. + GpuFabric []*GPUFabricState `protobuf:"bytes,16,rep,name=gpu_fabric,json=gpuFabric,proto3" json:"gpu_fabric,omitempty"` + Imex *IMEX `protobuf:"bytes,17,opt,name=imex,proto3" json:"imex,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Heartbeat) Reset() { *x = Heartbeat{} - mi := &file_llmgw_v1_agent_proto_msgTypes[13] + mi := &file_llmgw_v1_agent_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1752,7 +1948,7 @@ func (x *Heartbeat) String() string { func (*Heartbeat) ProtoMessage() {} func (x *Heartbeat) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[13] + mi := &file_llmgw_v1_agent_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1765,7 +1961,7 @@ func (x *Heartbeat) ProtoReflect() protoreflect.Message { // Deprecated: Use Heartbeat.ProtoReflect.Descriptor instead. func (*Heartbeat) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15} } func (x *Heartbeat) GetNodeId() string { @@ -1873,6 +2069,20 @@ func (x *Heartbeat) GetFabrics() []*FabricAttachment { return nil } +func (x *Heartbeat) GetGpuFabric() []*GPUFabricState { + if x != nil { + return x.GpuFabric + } + return nil +} + +func (x *Heartbeat) GetImex() *IMEX { + if x != nil { + return x.Imex + } + return nil +} + // Provision is the software a node should have: which engines, at which // versions. Nothing more. // @@ -1892,7 +2102,7 @@ type Provision struct { func (x *Provision) Reset() { *x = Provision{} - mi := &file_llmgw_v1_agent_proto_msgTypes[14] + mi := &file_llmgw_v1_agent_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1904,7 +2114,7 @@ func (x *Provision) String() string { func (*Provision) ProtoMessage() {} func (x *Provision) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[14] + mi := &file_llmgw_v1_agent_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1917,7 +2127,7 @@ func (x *Provision) ProtoReflect() protoreflect.Message { // Deprecated: Use Provision.ProtoReflect.Descriptor instead. func (*Provision) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16} } func (x *Provision) GetEngines() []*EngineWant { @@ -1938,7 +2148,7 @@ type EngineWant struct { func (x *EngineWant) Reset() { *x = EngineWant{} - mi := &file_llmgw_v1_agent_proto_msgTypes[15] + mi := &file_llmgw_v1_agent_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1950,7 +2160,7 @@ func (x *EngineWant) String() string { func (*EngineWant) ProtoMessage() {} func (x *EngineWant) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[15] + mi := &file_llmgw_v1_agent_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1963,7 +2173,7 @@ func (x *EngineWant) ProtoReflect() protoreflect.Message { // Deprecated: Use EngineWant.ProtoReflect.Descriptor instead. func (*EngineWant) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{17} } func (x *EngineWant) GetEngine() string { @@ -1996,7 +2206,7 @@ type ProvisionStatus struct { func (x *ProvisionStatus) Reset() { *x = ProvisionStatus{} - mi := &file_llmgw_v1_agent_proto_msgTypes[16] + mi := &file_llmgw_v1_agent_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2008,7 +2218,7 @@ func (x *ProvisionStatus) String() string { func (*ProvisionStatus) ProtoMessage() {} func (x *ProvisionStatus) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[16] + mi := &file_llmgw_v1_agent_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2021,7 +2231,7 @@ func (x *ProvisionStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use ProvisionStatus.ProtoReflect.Descriptor instead. func (*ProvisionStatus) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{18} } func (x *ProvisionStatus) GetEngine() string { @@ -2085,7 +2295,7 @@ type ShardStatus struct { func (x *ShardStatus) Reset() { *x = ShardStatus{} - mi := &file_llmgw_v1_agent_proto_msgTypes[17] + mi := &file_llmgw_v1_agent_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2097,7 +2307,7 @@ func (x *ShardStatus) String() string { func (*ShardStatus) ProtoMessage() {} func (x *ShardStatus) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[17] + mi := &file_llmgw_v1_agent_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2110,7 +2320,7 @@ func (x *ShardStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use ShardStatus.ProtoReflect.Descriptor instead. func (*ShardStatus) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{17} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{19} } func (x *ShardStatus) GetNodeId() string { @@ -2184,7 +2394,7 @@ type FetchProgress struct { func (x *FetchProgress) Reset() { *x = FetchProgress{} - mi := &file_llmgw_v1_agent_proto_msgTypes[18] + mi := &file_llmgw_v1_agent_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2196,7 +2406,7 @@ func (x *FetchProgress) String() string { func (*FetchProgress) ProtoMessage() {} func (x *FetchProgress) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[18] + mi := &file_llmgw_v1_agent_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2209,7 +2419,7 @@ func (x *FetchProgress) ProtoReflect() protoreflect.Message { // Deprecated: Use FetchProgress.ProtoReflect.Descriptor instead. func (*FetchProgress) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{18} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{20} } func (x *FetchProgress) GetSource() string { @@ -2265,7 +2475,7 @@ type StartShard struct { func (x *StartShard) Reset() { *x = StartShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[19] + mi := &file_llmgw_v1_agent_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2277,7 +2487,7 @@ func (x *StartShard) String() string { func (*StartShard) ProtoMessage() {} func (x *StartShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[19] + mi := &file_llmgw_v1_agent_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2290,7 +2500,7 @@ func (x *StartShard) ProtoReflect() protoreflect.Message { // Deprecated: Use StartShard.ProtoReflect.Descriptor instead. func (*StartShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{19} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{21} } func (x *StartShard) GetReplicaId() string { @@ -2417,7 +2627,7 @@ type EngineProfile struct { func (x *EngineProfile) Reset() { *x = EngineProfile{} - mi := &file_llmgw_v1_agent_proto_msgTypes[20] + mi := &file_llmgw_v1_agent_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2429,7 +2639,7 @@ func (x *EngineProfile) String() string { func (*EngineProfile) ProtoMessage() {} func (x *EngineProfile) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[20] + mi := &file_llmgw_v1_agent_proto_msgTypes[22] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2442,7 +2652,7 @@ func (x *EngineProfile) ProtoReflect() protoreflect.Message { // Deprecated: Use EngineProfile.ProtoReflect.Descriptor instead. func (*EngineProfile) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{20} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{22} } func (x *EngineProfile) GetName() string { @@ -2538,7 +2748,7 @@ type RemoveWeights struct { func (x *RemoveWeights) Reset() { *x = RemoveWeights{} - mi := &file_llmgw_v1_agent_proto_msgTypes[21] + mi := &file_llmgw_v1_agent_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2550,7 +2760,7 @@ func (x *RemoveWeights) String() string { func (*RemoveWeights) ProtoMessage() {} func (x *RemoveWeights) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[21] + mi := &file_llmgw_v1_agent_proto_msgTypes[23] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2563,7 +2773,7 @@ func (x *RemoveWeights) ProtoReflect() protoreflect.Message { // Deprecated: Use RemoveWeights.ProtoReflect.Descriptor instead. func (*RemoveWeights) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{21} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{23} } func (x *RemoveWeights) GetPaths() []string { @@ -2583,7 +2793,7 @@ type StopShard struct { func (x *StopShard) Reset() { *x = StopShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[22] + mi := &file_llmgw_v1_agent_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2595,7 +2805,7 @@ func (x *StopShard) String() string { func (*StopShard) ProtoMessage() {} func (x *StopShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[22] + mi := &file_llmgw_v1_agent_proto_msgTypes[24] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2608,7 +2818,7 @@ func (x *StopShard) ProtoReflect() protoreflect.Message { // Deprecated: Use StopShard.ProtoReflect.Descriptor instead. func (*StopShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{22} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{24} } func (x *StopShard) GetShardId() string { @@ -2628,7 +2838,7 @@ type DrainShard struct { func (x *DrainShard) Reset() { *x = DrainShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[23] + mi := &file_llmgw_v1_agent_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2640,7 +2850,7 @@ func (x *DrainShard) String() string { func (*DrainShard) ProtoMessage() {} func (x *DrainShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[23] + mi := &file_llmgw_v1_agent_proto_msgTypes[25] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2653,7 +2863,7 @@ func (x *DrainShard) ProtoReflect() protoreflect.Message { // Deprecated: Use DrainShard.ProtoReflect.Descriptor instead. func (*DrainShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{23} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{25} } func (x *DrainShard) GetShardId() string { @@ -2675,7 +2885,7 @@ type Ack struct { func (x *Ack) Reset() { *x = Ack{} - mi := &file_llmgw_v1_agent_proto_msgTypes[24] + mi := &file_llmgw_v1_agent_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2687,7 +2897,7 @@ func (x *Ack) String() string { func (*Ack) ProtoMessage() {} func (x *Ack) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[24] + mi := &file_llmgw_v1_agent_proto_msgTypes[26] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2700,7 +2910,7 @@ func (x *Ack) ProtoReflect() protoreflect.Message { // Deprecated: Use Ack.ProtoReflect.Descriptor instead. func (*Ack) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{24} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{26} } func (x *Ack) GetMessageId() string { @@ -2744,7 +2954,7 @@ type LogsRequest struct { func (x *LogsRequest) Reset() { *x = LogsRequest{} - mi := &file_llmgw_v1_agent_proto_msgTypes[25] + mi := &file_llmgw_v1_agent_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2756,7 +2966,7 @@ func (x *LogsRequest) String() string { func (*LogsRequest) ProtoMessage() {} func (x *LogsRequest) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[25] + mi := &file_llmgw_v1_agent_proto_msgTypes[27] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2769,7 +2979,7 @@ func (x *LogsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use LogsRequest.ProtoReflect.Descriptor instead. func (*LogsRequest) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{25} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{27} } func (x *LogsRequest) GetRequestId() string { @@ -2809,7 +3019,7 @@ type LogsReply struct { func (x *LogsReply) Reset() { *x = LogsReply{} - mi := &file_llmgw_v1_agent_proto_msgTypes[26] + mi := &file_llmgw_v1_agent_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2821,7 +3031,7 @@ func (x *LogsReply) String() string { func (*LogsReply) ProtoMessage() {} func (x *LogsReply) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[26] + mi := &file_llmgw_v1_agent_proto_msgTypes[28] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2834,7 +3044,7 @@ func (x *LogsReply) ProtoReflect() protoreflect.Message { // Deprecated: Use LogsReply.ProtoReflect.Descriptor instead. func (*LogsReply) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{26} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{28} } func (x *LogsReply) GetRequestId() string { @@ -2880,7 +3090,7 @@ type FabricScanRequest struct { func (x *FabricScanRequest) Reset() { *x = FabricScanRequest{} - mi := &file_llmgw_v1_agent_proto_msgTypes[27] + mi := &file_llmgw_v1_agent_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2892,7 +3102,7 @@ func (x *FabricScanRequest) String() string { func (*FabricScanRequest) ProtoMessage() {} func (x *FabricScanRequest) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[27] + mi := &file_llmgw_v1_agent_proto_msgTypes[29] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2905,7 +3115,7 @@ func (x *FabricScanRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use FabricScanRequest.ProtoReflect.Descriptor instead. func (*FabricScanRequest) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{27} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{29} } func (x *FabricScanRequest) GetRequestId() string { @@ -2944,7 +3154,7 @@ type FabricScanReply struct { func (x *FabricScanReply) Reset() { *x = FabricScanReply{} - mi := &file_llmgw_v1_agent_proto_msgTypes[28] + mi := &file_llmgw_v1_agent_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2956,7 +3166,7 @@ func (x *FabricScanReply) String() string { func (*FabricScanReply) ProtoMessage() {} func (x *FabricScanReply) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[28] + mi := &file_llmgw_v1_agent_proto_msgTypes[30] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2969,7 +3179,7 @@ func (x *FabricScanReply) ProtoReflect() protoreflect.Message { // Deprecated: Use FabricScanReply.ProtoReflect.Descriptor instead. func (*FabricScanReply) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{28} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{30} } func (x *FabricScanReply) GetRequestId() string { @@ -3023,7 +3233,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x04logs\x18\a \x01(\v2\x15.llmgw.v1.LogsRequestH\x00R\x04logs\x12>\n" + "\vfabric_scan\x18\b \x01(\v2\x1b.llmgw.v1.FabricScanRequestH\x00R\n" + "fabricScanB\x05\n" + - "\x03msg\"\xf2\x05\n" + + "\x03msg\"\x96\x06\n" + "\bRegister\x12\x17\n" + "\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1a\n" + "\bhostname\x18\x02 \x01(\tR\bhostname\x12\x1b\n" + @@ -3039,7 +3249,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\aweights\x18\v \x03(\v2\x16.llmgw.v1.LocalWeightsR\aweights\x12+\n" + "\astorage\x18\f \x03(\v2\x11.llmgw.v1.StorageR\astorage\x12\"\n" + "\x04host\x18\r \x01(\v2\x0e.llmgw.v1.HostR\x04host\x12%\n" + - "\x0eengines_probed\x18\x0e \x01(\bR\renginesProbed\x1a9\n" + + "\x0eengines_probed\x18\x0e \x01(\bR\renginesProbed\x12\"\n" + + "\x04imex\x18\x0f \x01(\v2\x0e.llmgw.v1.IMEXR\x04imex\x1a9\n" + "\vLabelsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\x1aA\n" + @@ -3083,7 +3294,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\vleader_port\x18\x0e \x01(\x05R\n" + "leaderPort\x12\x1d\n" + "\n" + - "profile_fp\x18\x0f \x01(\tR\tprofileFp\"\xf0\x04\n" + + "profile_fp\x18\x0f \x01(\tR\tprofileFp\"\xa3\x06\n" + "\x03GPU\x12\x14\n" + "\x05index\x18\x01 \x01(\x05R\x05index\x12\x12\n" + "\x04uuid\x18\x02 \x01(\tR\x04uuid\x12\x12\n" + @@ -3103,7 +3314,12 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\n" + "ecc_errors\x18\x0e \x01(\x05R\teccErrors\x12/\n" + "\x13utilization_percent\x18\x0f \x01(\x05R\x12utilizationPercent\x128\n" + - "\tnic_paths\x18\x10 \x03(\v2\x1b.llmgw.v1.GPU.NicPathsEntryR\bnicPaths\x1a;\n" + + "\tnic_paths\x18\x10 \x03(\v2\x1b.llmgw.v1.GPU.NicPathsEntryR\bnicPaths\x12!\n" + + "\ffabric_state\x18\x11 \x01(\tR\vfabricState\x12#\n" + + "\rfabric_status\x18\x12 \x01(\tR\ffabricStatus\x12!\n" + + "\fcluster_uuid\x18\x13 \x01(\tR\vclusterUuid\x12\x1b\n" + + "\tclique_id\x18\x14 \x01(\tR\bcliqueId\x12)\n" + + "\x10fabric_bandwidth\x18\x15 \x01(\tR\x0ffabricBandwidth\x1a;\n" + "\rNicPathsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"[\n" + @@ -3161,7 +3377,17 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x04guid\x18\x01 \x01(\tR\x04guid\x12\x10\n" + "\x03lid\x18\x02 \x01(\x05R\x03lid\x12\x1a\n" + "\bpriority\x18\x03 \x01(\x05R\bpriority\x12\x14\n" + - "\x05state\x18\x04 \x01(\tR\x05state\"\xfc\x05\n" + + "\x05state\x18\x04 \x01(\tR\x05state\"8\n" + + "\x04IMEX\x12\x14\n" + + "\x05state\x18\x01 \x01(\tR\x05state\x12\x1a\n" + + "\bchannels\x18\x02 \x01(\x05R\bchannels\"\xb2\x01\n" + + "\x0eGPUFabricState\x12\x14\n" + + "\x05index\x18\x01 \x01(\x05R\x05index\x12\x14\n" + + "\x05state\x18\x02 \x01(\tR\x05state\x12\x16\n" + + "\x06status\x18\x03 \x01(\tR\x06status\x12!\n" + + "\fcluster_uuid\x18\x04 \x01(\tR\vclusterUuid\x12\x1b\n" + + "\tclique_id\x18\x05 \x01(\tR\bcliqueId\x12\x1c\n" + + "\tbandwidth\x18\x06 \x01(\tR\tbandwidth\"\xd9\x06\n" + "\tHeartbeat\x12\x17\n" + "\anode_id\x18\x01 \x01(\tR\x06nodeId\x12+\n" + "\x12gpu_memory_used_mb\x18\x02 \x03(\x05R\x0fgpuMemoryUsedMb\x12!\n" + @@ -3178,7 +3404,10 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x04load\x18\f \x01(\v2\x12.llmgw.v1.NodeLoadR\x04load\x12+\n" + "\bib_ports\x18\r \x03(\v2\x10.llmgw.v1.IBPortR\aibPorts\x12'\n" + "\x0ffabrics_changed\x18\x0e \x01(\bR\x0efabricsChanged\x124\n" + - "\afabrics\x18\x0f \x03(\v2\x1a.llmgw.v1.FabricAttachmentR\afabrics\x1aA\n" + + "\afabrics\x18\x0f \x03(\v2\x1a.llmgw.v1.FabricAttachmentR\afabrics\x127\n" + + "\n" + + "gpu_fabric\x18\x10 \x03(\v2\x18.llmgw.v1.GPUFabricStateR\tgpuFabric\x12\"\n" + + "\x04imex\x18\x11 \x01(\v2\x0e.llmgw.v1.IMEXR\x04imex\x1aA\n" + "\x13EngineVersionsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\";\n" + @@ -3312,7 +3541,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte { } var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 35) +var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 37) var file_llmgw_v1_agent_proto_goTypes = []any{ (ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State (*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage @@ -3328,75 +3557,80 @@ var file_llmgw_v1_agent_proto_goTypes = []any{ (*ShardLoad)(nil), // 11: llmgw.v1.ShardLoad (*IBPort)(nil), // 12: llmgw.v1.IBPort (*SubnetManager)(nil), // 13: llmgw.v1.SubnetManager - (*Heartbeat)(nil), // 14: llmgw.v1.Heartbeat - (*Provision)(nil), // 15: llmgw.v1.Provision - (*EngineWant)(nil), // 16: llmgw.v1.EngineWant - (*ProvisionStatus)(nil), // 17: llmgw.v1.ProvisionStatus - (*ShardStatus)(nil), // 18: llmgw.v1.ShardStatus - (*FetchProgress)(nil), // 19: llmgw.v1.FetchProgress - (*StartShard)(nil), // 20: llmgw.v1.StartShard - (*EngineProfile)(nil), // 21: llmgw.v1.EngineProfile - (*RemoveWeights)(nil), // 22: llmgw.v1.RemoveWeights - (*StopShard)(nil), // 23: llmgw.v1.StopShard - (*DrainShard)(nil), // 24: llmgw.v1.DrainShard - (*Ack)(nil), // 25: llmgw.v1.Ack - (*LogsRequest)(nil), // 26: llmgw.v1.LogsRequest - (*LogsReply)(nil), // 27: llmgw.v1.LogsReply - (*FabricScanRequest)(nil), // 28: llmgw.v1.FabricScanRequest - (*FabricScanReply)(nil), // 29: llmgw.v1.FabricScanReply - nil, // 30: llmgw.v1.Register.LabelsEntry - nil, // 31: llmgw.v1.Register.EngineVersionsEntry - nil, // 32: llmgw.v1.GPU.NicPathsEntry - nil, // 33: llmgw.v1.IBPort.CountersEntry - nil, // 34: llmgw.v1.Heartbeat.EngineVersionsEntry - nil, // 35: llmgw.v1.EngineProfile.EnvEntry + (*IMEX)(nil), // 14: llmgw.v1.IMEX + (*GPUFabricState)(nil), // 15: llmgw.v1.GPUFabricState + (*Heartbeat)(nil), // 16: llmgw.v1.Heartbeat + (*Provision)(nil), // 17: llmgw.v1.Provision + (*EngineWant)(nil), // 18: llmgw.v1.EngineWant + (*ProvisionStatus)(nil), // 19: llmgw.v1.ProvisionStatus + (*ShardStatus)(nil), // 20: llmgw.v1.ShardStatus + (*FetchProgress)(nil), // 21: llmgw.v1.FetchProgress + (*StartShard)(nil), // 22: llmgw.v1.StartShard + (*EngineProfile)(nil), // 23: llmgw.v1.EngineProfile + (*RemoveWeights)(nil), // 24: llmgw.v1.RemoveWeights + (*StopShard)(nil), // 25: llmgw.v1.StopShard + (*DrainShard)(nil), // 26: llmgw.v1.DrainShard + (*Ack)(nil), // 27: llmgw.v1.Ack + (*LogsRequest)(nil), // 28: llmgw.v1.LogsRequest + (*LogsReply)(nil), // 29: llmgw.v1.LogsReply + (*FabricScanRequest)(nil), // 30: llmgw.v1.FabricScanRequest + (*FabricScanReply)(nil), // 31: llmgw.v1.FabricScanReply + nil, // 32: llmgw.v1.Register.LabelsEntry + nil, // 33: llmgw.v1.Register.EngineVersionsEntry + nil, // 34: llmgw.v1.GPU.NicPathsEntry + nil, // 35: llmgw.v1.IBPort.CountersEntry + nil, // 36: llmgw.v1.Heartbeat.EngineVersionsEntry + nil, // 37: llmgw.v1.EngineProfile.EnvEntry } var file_llmgw_v1_agent_proto_depIdxs = []int32{ 3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register - 14, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat - 18, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus - 27, // 3: llmgw.v1.AgentMessage.logs:type_name -> llmgw.v1.LogsReply - 29, // 4: llmgw.v1.AgentMessage.fabric_scan:type_name -> llmgw.v1.FabricScanReply - 20, // 5: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard - 23, // 6: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard - 24, // 7: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard - 25, // 8: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack - 22, // 9: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights - 15, // 10: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision - 26, // 11: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest - 28, // 12: llmgw.v1.ControlMessage.fabric_scan:type_name -> llmgw.v1.FabricScanRequest - 30, // 13: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry + 16, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat + 20, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus + 29, // 3: llmgw.v1.AgentMessage.logs:type_name -> llmgw.v1.LogsReply + 31, // 4: llmgw.v1.AgentMessage.fabric_scan:type_name -> llmgw.v1.FabricScanReply + 22, // 5: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard + 25, // 6: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard + 26, // 7: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard + 27, // 8: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack + 24, // 9: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights + 17, // 10: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision + 28, // 11: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest + 30, // 12: llmgw.v1.ControlMessage.fabric_scan:type_name -> llmgw.v1.FabricScanRequest + 32, // 13: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry 7, // 14: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU 8, // 15: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment - 31, // 16: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry + 33, // 16: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry 6, // 17: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard 9, // 18: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights 5, // 19: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage 4, // 20: llmgw.v1.Register.host:type_name -> llmgw.v1.Host - 0, // 21: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State - 32, // 22: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry - 11, // 23: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad - 33, // 24: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry - 13, // 25: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager - 5, // 26: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage - 9, // 27: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights - 34, // 28: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry - 17, // 29: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus - 10, // 30: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad - 12, // 31: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort - 8, // 32: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment - 16, // 33: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant - 0, // 34: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State - 19, // 35: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress - 21, // 36: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile - 35, // 37: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry - 1, // 38: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage - 2, // 39: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage - 39, // [39:40] is the sub-list for method output_type - 38, // [38:39] is the sub-list for method input_type - 38, // [38:38] is the sub-list for extension type_name - 38, // [38:38] is the sub-list for extension extendee - 0, // [0:38] is the sub-list for field type_name + 14, // 21: llmgw.v1.Register.imex:type_name -> llmgw.v1.IMEX + 0, // 22: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State + 34, // 23: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry + 11, // 24: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad + 35, // 25: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry + 13, // 26: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager + 5, // 27: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage + 9, // 28: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights + 36, // 29: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry + 19, // 30: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus + 10, // 31: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad + 12, // 32: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort + 8, // 33: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment + 15, // 34: llmgw.v1.Heartbeat.gpu_fabric:type_name -> llmgw.v1.GPUFabricState + 14, // 35: llmgw.v1.Heartbeat.imex:type_name -> llmgw.v1.IMEX + 18, // 36: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant + 0, // 37: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State + 21, // 38: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress + 23, // 39: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile + 37, // 40: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry + 1, // 41: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage + 2, // 42: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage + 42, // [42:43] is the sub-list for method output_type + 41, // [41:42] is the sub-list for method input_type + 41, // [41:41] is the sub-list for extension type_name + 41, // [41:41] is the sub-list for extension extendee + 0, // [0:41] is the sub-list for field type_name } func init() { file_llmgw_v1_agent_proto_init() } @@ -3427,7 +3661,7 @@ func file_llmgw_v1_agent_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)), NumEnums: 1, - NumMessages: 35, + NumMessages: 37, NumExtensions: 0, NumServices: 1, }, diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index 8f934b4..f887658 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -116,6 +116,8 @@ message Register { // meant "an agent too old to say" and constrained nothing — so a freshly // built node with no engine installed yet was a candidate for every model. bool engines_probed = 14; + // imex is the service that lets a multi-node NVLink domain share memory. + IMEX imex = 15; } // Host describes the operating system and processor of a node. @@ -212,6 +214,14 @@ message GPU { // nic_paths is how far each network adapter is from this card, by HCA // name: PIX, PXB, PHB, NODE or SYS, from nvidia-smi topo. map nic_paths = 16; + // The card's multi-node NVLink membership, from nvidia-smi -q: state is + // Completed once the fabric manager has joined it; cluster_uuid and + // clique_id name the domain it joined. + string fabric_state = 17; + string fabric_status = 18; + string cluster_uuid = 19; + string clique_id = 20; + string fabric_bandwidth = 21; } message FabricAttachment { @@ -315,6 +325,23 @@ message SubnetManager { string state = 4; } +// IMEX is the nvidia-imex service and its channels. +message IMEX { + // state is systemd's word for it, empty where it is not installed. + string state = 1; + int32 channels = 2; +} + +// GPUFabricState is one GPU's NVLink domain membership. +message GPUFabricState { + int32 index = 1; + string state = 2; + string status = 3; + string cluster_uuid = 4; + string clique_id = 5; + string bandwidth = 6; +} + message Heartbeat { string node_id = 1; // gpu_memory_used_mb, indexed by GPU, catches a foreign process appearing on @@ -370,6 +397,12 @@ message Heartbeat { // port going down takes its fabric away, and one coming back returns it. bool fabrics_changed = 14; repeated FabricAttachment fabrics = 15; + + // gpu_fabric is each GPU's NVLink domain membership as it stands now, sent + // once a minute: a fabric manager restarting drops every card out of its + // domain, and the node has to stop being offered work that spans it. + repeated GPUFabricState gpu_fabric = 16; + IMEX imex = 17; } // Provision is the software a node should have: which engines, at which diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 54a270e..47e6152 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -365,6 +365,8 @@ type Agent struct { sm map[string]*discovery.SMInfo smErr map[string]string smAsked time.Time + // nvlAsked is when the GPUs' NVLink fabric state was last read. + nvlAsked time.Time // sms is every subnet manager on a fabric, asked through one port per // master, keyed by that port; smsErr says why a port could not ask. sms map[string][]discovery.SMRecord @@ -999,6 +1001,7 @@ func (a *Agent) heartbeat() *llmgwv1.AgentMessage { } hb.Load = a.sampleLoad() a.sampleIB(hb) + a.sampleNVLink(hb) return &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Heartbeat{Heartbeat: hb}} } @@ -1752,3 +1755,45 @@ func (a *Agent) ensureDiags(ctx context.Context) { "err", err) } } + +// discoverGPUFabric and discoverIMEX are replaced in tests. +var ( + discoverGPUFabric = discovery.DiscoverGPUFabric + discoverIMEX = discovery.DiscoverIMEX +) + +// sampleNVLink reads, once a minute, which NVLink domain each GPU is joined +// to and whether IMEX is up. nvidia-smi -q takes a second or two, too long +// for every heartbeat, and a fabric that was joined an hour ago rarely +// changes in five seconds — but when a fabric manager restarts it does, for +// every card at once. +func (a *Agent) sampleNVLink(hb *llmgwv1.Heartbeat) { + a.mu.Lock() + gpus := slices.Clone(a.inv.GPUs) + due := time.Since(a.nvlAsked) > time.Minute + if due { + a.nvlAsked = time.Now() + } + a.mu.Unlock() + if len(gpus) == 0 || !due { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + fab, err := discoverGPUFabric(ctx, a.cfg.Discovery.NvidiaSMI) + if err != nil || len(fab) == 0 { + return + } + discovery.ApplyFabric(gpus, fab) + for _, g := range gpus { + if g.Fabric == nil { + continue + } + hb.GpuFabric = append(hb.GpuFabric, &llmgwv1.GPUFabricState{ + Index: int32(g.Index), State: g.Fabric.State, Status: g.Fabric.Status, + ClusterUuid: g.Fabric.ClusterUUID, CliqueId: g.Fabric.CliqueID, Bandwidth: g.Fabric.Bandwidth, + }) + } + m := discoverIMEX(ctx) + hb.Imex = &llmgwv1.IMEX{State: m.State, Channels: int32(m.Channels)} +} diff --git a/internal/agent/discovery/discover.go b/internal/agent/discovery/discover.go index b459e88..d714b7b 100644 --- a/internal/agent/discovery/discover.go +++ b/internal/agent/discovery/discover.go @@ -85,6 +85,13 @@ func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, e } else { topo.Apply(inv.GPUs) } + // Multi-node NVLink: which domain each card has joined. + if fab, err := DiscoverGPUFabric(ctx, opts.NvidiaSMI); err != nil { + log.Warn("could not read the GPUs' NVLink fabric state", "err", err) + } else { + ApplyFabric(inv.GPUs, fab) + } + inv.IMEX = DiscoverIMEX(ctx) } } diff --git a/internal/agent/discovery/gpu.go b/internal/agent/discovery/gpu.go index cd33515..36fa88f 100644 --- a/internal/agent/discovery/gpu.go +++ b/internal/agent/discovery/gpu.go @@ -42,6 +42,9 @@ type GPU struct { // PIX, PXB, PHB, NODE or SYS, as nvidia-smi topo reports it. What decides // which adapter a GPU's share of a multi-node job should go through. NICPaths map[string]string `json:"nic_paths,omitempty"` + // Fabric is the GPU's multi-node NVLink membership, nil where the + // driver reports none. + Fabric *GPUFabric `json:"fabric,omitempty"` // Health carries anything that should keep the GPU out of placement. Health GPUHealth `json:"health"` } @@ -121,6 +124,9 @@ type Inventory struct { // Storage is the disk behind those weights, one entry per filesystem. // Empty means the same thing it means for Weights: not asked. Storage []Storage `json:"storage,omitempty"` + // IMEX is the service a multi-node NVLink domain needs, read where the + // node has GPUs. + IMEX IMEX `json:"imex"` } // Classes returns the distinct GPU classes present, sorted. diff --git a/internal/agent/discovery/gpu_fabric.go b/internal/agent/discovery/gpu_fabric.go new file mode 100644 index 0000000..1189d23 --- /dev/null +++ b/internal/agent/discovery/gpu_fabric.go @@ -0,0 +1,160 @@ +package discovery + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "time" +) + +// GPUFabric is a GPU's place in a multi-node NVLink domain, as the driver +// reports it: the NVL72 rack, or the NVLink Switch system, it belongs to. +type GPUFabric struct { + // State is Completed once the fabric manager has joined the GPU to its + // domain; In Progress, Not Started or Not Supported otherwise. + State string `json:"state"` + // Status is Success, or the reason the join failed. + Status string `json:"status,omitempty"` + // ClusterUUID and CliqueID name the domain: GPUs sharing both can reach + // each other over NVLink whatever node they are in. + ClusterUUID string `json:"cluster_uuid,omitempty"` + CliqueID string `json:"clique_id,omitempty"` + // Bandwidth is Full, or how the domain's NVLink is degraded. + Bandwidth string `json:"bandwidth,omitempty"` +} + +// Domain is the NVLink domain the GPU is joined to, empty when it is in none +// that crosses nodes: a card with no fabric, one not joined, or one whose +// cluster is all zeros, which is what a single machine's NVSwitch reports. +func (f *GPUFabric) Domain() string { + if f == nil || !strings.EqualFold(f.State, "Completed") { + return "" + } + u := strings.ToLower(strings.TrimSpace(f.ClusterUUID)) + if u == "" || strings.Trim(u, "0-") == "" { + return "" + } + return u + "/" + f.CliqueID +} + +// ParseGPUFabric reads the Fabric sections of `nvidia-smi -q`, by the PCI +// address each GPU's section starts with. +func ParseGPUFabric(out string) map[string]GPUFabric { + res := map[string]GPUFabric{} + var bus string + fabricIndent := -1 + var cur *GPUFabric + flush := func() { + if bus != "" && cur != nil && cur.State != "" { + res[bus] = *cur + } + cur, fabricIndent = nil, -1 + } + for _, line := range strings.Split(out, "\n") { + trim := strings.TrimSpace(line) + indent := len(line) - len(strings.TrimLeft(line, " \t")) + if strings.HasPrefix(line, "GPU ") && indent == 0 { + flush() + bus = strings.ToLower(strings.TrimSpace(strings.TrimPrefix(line, "GPU "))) + continue + } + if trim == "Fabric" { + flush() + cur, fabricIndent = &GPUFabric{}, indent + continue + } + if cur == nil { + continue + } + if trim != "" && indent <= fabricIndent { + flush() + continue + } + k, v, ok := strings.Cut(trim, ":") + if !ok { + continue + } + k, v = strings.TrimSpace(k), strings.TrimSpace(v) + if strings.EqualFold(v, "N/A") { + v = "" + } + switch k { + case "State": + cur.State = v + case "Status": + cur.Status = v + case "ClusterUUID", "Cluster UUID": + cur.ClusterUUID = v + case "CliqueId", "Clique ID": + cur.CliqueID = v + case "Bandwidth": + cur.Bandwidth = v + } + } + flush() + return res +} + +// DiscoverGPUFabric runs `nvidia-smi -q` for the Fabric sections. Nothing +// is reported on a driver or a card without them. +func DiscoverGPUFabric(ctx context.Context, binary string) (map[string]GPUFabric, error) { + if binary == "" { + if _, err := exec.LookPath("nvidia-smi"); err != nil { + return nil, nil + } + binary = "nvidia-smi" + } + ctx, cancel := context.WithTimeout(ctx, 20*time.Second) + defer cancel() + out, err := exec.CommandContext(ctx, binary, "-q").Output() + if err != nil { + return nil, err + } + return ParseGPUFabric(string(out)), nil +} + +// ApplyFabric puts each GPU's fabric on it, matched by PCI address. +func ApplyFabric(gpus []GPU, fab map[string]GPUFabric) { + for i := range gpus { + if f, ok := fab[strings.ToLower(gpus[i].PCIBusID)]; ok { + f := f + gpus[i].Fabric = &f + } + } +} + +// IMEX is the service that lets GPUs in different nodes of one NVLink +// domain share memory: without it, multi-node NVLink carries nothing. +type IMEX struct { + // State is systemd's word for nvidia-imex: active, inactive, or empty + // where it is not installed. + State string `json:"state,omitempty"` + // Channels is how many IMEX channels exist; a process needs one. + Channels int `json:"channels"` +} + +// DiscoverIMEX reads the IMEX service and its channels. +func DiscoverIMEX(ctx context.Context) IMEX { + var m IMEX + if entries, err := os.ReadDir("/dev/nvidia-caps-imex-channels"); err == nil { + m.Channels = len(entries) + } + if _, err := exec.LookPath("systemctl"); err == nil { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + out, _ := exec.CommandContext(ctx, "systemctl", "is-active", "nvidia-imex").Output() + m.State = strings.TrimSpace(string(out)) + if m.State == "inactive" { + // "inactive" is also what systemd says of a unit that does not + // exist; the binary says which. + if _, err := exec.LookPath("nvidia-imex"); err != nil { + if _, err := os.Stat(filepath.Join("/usr/bin", "nvidia-imex")); err != nil { + m.State = "" + } + } + } + } + return m +} diff --git a/internal/agent/discovery/gpu_fabric_test.go b/internal/agent/discovery/gpu_fabric_test.go new file mode 100644 index 0000000..e5021a7 --- /dev/null +++ b/internal/agent/discovery/gpu_fabric_test.go @@ -0,0 +1,77 @@ +package discovery + +import "testing" + +// smiQ is an excerpt of `nvidia-smi -q` on a GB200 tray: two GPUs joined to +// one NVL72 domain, then a PCIe card that has no fabric at all. +const smiQ = ` +==============NVSMI LOG============== + +Timestamp : Mon Sep 29 10:00:00 2026 +Driver Version : 570.86.15 +Attached GPUs : 3 +GPU 00000008:01:00.0 + Product Name : NVIDIA GB200 + Minor Number : 0 + Fabric + State : Completed + Status : Success + CliqueId : 32766 + ClusterUUID : 76B1AD76-7c24-4b6b-9f6c-2a1e0d8c11aa + Health + Bandwidth : Full + Processes : None + +GPU 00000009:01:00.0 + Product Name : NVIDIA GB200 + Fabric + State : In Progress + Status : N/A + CliqueId : N/A + ClusterUUID : N/A + Processes : None + +GPU 0000000A:01:00.0 + Product Name : NVIDIA L40S + Fabric + State : N/A + Status : N/A +` + +func TestAGPUsNVLinkDomainIsRead(t *testing.T) { + fab := ParseGPUFabric(smiQ) + a, ok := fab["00000008:01:00.0"] + if !ok { + t.Fatalf("fabric = %+v", fab) + } + if a.State != "Completed" || a.CliqueID != "32766" || a.Bandwidth != "Full" { + t.Errorf("GB200 = %+v", a) + } + if got := a.Domain(); got != "76b1ad76-7c24-4b6b-9f6c-2a1e0d8c11aa/32766" { + t.Errorf("domain = %q", got) + } + // Still joining: no domain yet, and said as such. + b := fab["00000009:01:00.0"] + if b.State != "In Progress" || b.Domain() != "" { + t.Errorf("joining GPU = %+v", b) + } + // A card with no fabric reports nothing rather than an empty one. + if _, ok := fab["0000000a:01:00.0"]; ok { + t.Error("a card without a fabric was given one") + } + + gpus := []GPU{{Index: 0, PCIBusID: "00000008:01:00.0"}, {Index: 1, PCIBusID: "0000000A:01:00.0"}} + ApplyFabric(gpus, fab) + if gpus[0].Fabric == nil || gpus[1].Fabric != nil { + t.Errorf("applied = %+v %+v", gpus[0].Fabric, gpus[1].Fabric) + } +} + +func TestASingleMachinesNVSwitchIsNoDomain(t *testing.T) { + // An eight-GPU HGX reports its fabric joined, with a cluster of zeros: its + // NVLink ends at the chassis. + f := &GPUFabric{State: "Completed", ClusterUUID: "00000000-0000-0000-0000-000000000000", CliqueID: "0"} + if f.Domain() != "" { + t.Errorf("a single machine read as a domain: %q", f.Domain()) + } +} diff --git a/internal/agent/engines/engine.go b/internal/agent/engines/engine.go index 23baf1f..0e582c4 100644 --- a/internal/agent/engines/engine.go +++ b/internal/agent/engines/engine.go @@ -71,8 +71,9 @@ type ShardSpec struct { Profile Profile // Pool and Fabric are control plane concepts the agent cannot derive. They - // are carried so the agent can report them back after a restart, and are - // not used when building a command. + // are carried so the agent can report them back after a restart; Fabric + // also says, with an "nvl:" prefix, that the replica is inside a + // multi-node NVLink domain, which changes NCCL's environment. Pool string Fabric string // ProfileFP identifies the version of the profile this shard was started @@ -232,6 +233,14 @@ func baseEnv(s ShardSpec) map[string]string { env["NCCL_IB_DISABLE"] = "1" } + // Placed inside a multi-node NVLink domain: NCCL is to carry the + // replica's collectives over NVLink between its nodes, which it only + // does when told to, and needs IMEX for. + if s.NNodes > 1 && strings.HasPrefix(s.Fabric, "nvl:") { + env["NCCL_MNNVL_ENABLE"] = "1" + env["NCCL_CUMEM_ENABLE"] = "1" + } + if s.MgmtInterface != "" { env["NCCL_SOCKET_IFNAME"] = s.MgmtInterface env["GLOO_SOCKET_IFNAME"] = s.MgmtInterface diff --git a/internal/agent/engines/engines_test.go b/internal/agent/engines/engines_test.go index 6189d68..ad145fa 100644 --- a/internal/agent/engines/engines_test.go +++ b/internal/agent/engines/engines_test.go @@ -451,3 +451,15 @@ func TestQuantizationOf(t *testing.T) { t.Errorf("a dangling flag should yield nothing, got %q", got) } } + +func TestAReplicaInAnNVLinkDomainIsToldToUseIt(t *testing.T) { + s := multiNodeSpec(0) + s.Fabric = "nvl:76b1ad76-7c24-4b6b-9f6c-2a1e0d8c11aa/7" + env := envMap(buildOne(t, &VLLM{cfg: VLLMConfig{LaunchMode: "mp"}}, s).Env) + if env["NCCL_MNNVL_ENABLE"] != "1" { + t.Errorf("NCCL_MNNVL_ENABLE = %q", env["NCCL_MNNVL_ENABLE"]) + } + if env := envMap(buildOne(t, &VLLM{cfg: VLLMConfig{LaunchMode: "mp"}}, multiNodeSpec(0)).Env); env["NCCL_MNNVL_ENABLE"] != "" { + t.Error("a replica on InfiniBand was told to use multi-node NVLink") + } +} diff --git a/internal/cli/server.go b/internal/cli/server.go index 3c4e9ec..cd62ad6 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -208,6 +208,7 @@ func newServerCmd() *cobra.Command { adminMux.Handle("GET /_static/", web.Static()) adminMux.HandleFunc("GET /nodes/{id}/logs", web.NodeLogsPage(api, true, version.Version)) adminMux.HandleFunc("GET /nodes/{id}", web.NodePage(api, history, cfg.Monitoring.Window, true, version.Version, log)) + adminMux.HandleFunc("GET /nvlink", web.NVLinkPage(api, true, version.Version)) adminMux.HandleFunc("GET /fabrics", web.FabricsPage(api, true, version.Version)) adminMux.HandleFunc("GET /fabrics/{id}", web.FabricsPage(api, true, version.Version)) adminMux.HandleFunc("GET /fabrics/{id}/topology.json", web.FabricTopologyJSON(api)) diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index b712048..1d17e38 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -16,11 +16,13 @@ import ( "strings" "time" + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" "gitea.mynodes.xyz/ilker/llmgw/internal/bench" "gitea.mynodes.xyz/ilker/llmgw/internal/control" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink" "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" "gitea.mynodes.xyz/ilker/llmgw/internal/control/topology" "gitea.mynodes.xyz/ilker/llmgw/internal/store" @@ -53,6 +55,9 @@ func (a *API) Register(mux *http.ServeMux) { mux.HandleFunc("GET /api/v1/fabrics", func(w http.ResponseWriter, _ *http.Request) { writeJSON(w, http.StatusOK, a.Fabrics()) }) + mux.HandleFunc("GET /api/v1/nvlink", func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, a.NVLink()) + }) mux.HandleFunc("GET /api/v1/fabrics/{id}/topology", func(w http.ResponseWriter, r *http.Request) { t, ok := a.Topology(r.PathValue("id")) if !ok { @@ -134,6 +139,27 @@ type NodeView struct { Load *LoadView `json:"load,omitempty"` // IBPorts are the node's InfiniBand and RoCE ports at the last heartbeat. IBPorts []IBPortView `json:"ib_ports,omitempty"` + // GPUs are the cards, with how they are wired: NVLink peers, the + // adapter nearest each, and the NVLink domain each has joined. + GPUs []GPUView `json:"gpus,omitempty"` + // NVLinkDomain is the multi-node domain every card has joined, if any. + NVLinkDomain string `json:"nvlink_domain,omitempty"` + IMEX string `json:"imex,omitempty"` + IMEXChannels int `json:"imex_channels,omitempty"` +} + +// GPUView is one card as a node's page shows it. +type GPUView struct { + Index int `json:"index"` + Name string `json:"name"` + NUMA int `json:"numa"` + NVLinkPeers []int `json:"nvlink_peers,omitempty"` + // NearestNIC is the adapter closest to the card, and how close. + NearestNIC string `json:"nearest_nic,omitempty"` + NICPath string `json:"nic_path,omitempty"` + FabricState string `json:"fabric_state,omitempty"` + Domain string `json:"domain,omitempty"` + Bandwidth string `json:"bandwidth,omitempty"` } // IBPortView is one port as a page shows it. @@ -344,6 +370,20 @@ func (a *API) Nodes() []NodeView { v.LastSeenAgo = time.Since(n.LastSeen).Round(time.Second).String() } v.Load = loadView(n.Load, time.Now()) + v.NVLinkDomain, v.IMEX, v.IMEXChannels = n.NVLinkDomain(), n.IMEX.State, n.IMEX.Channels + for _, g := range n.GPUs { + gv := GPUView{Index: g.Index, Name: g.Name, NUMA: g.NUMANode, NVLinkPeers: g.NVLinkPeers} + for hca, rel := range g.NICPaths { + if gv.NearestNIC == "" || discovery.PathRank(rel) < discovery.PathRank(gv.NICPath) || + (discovery.PathRank(rel) == discovery.PathRank(gv.NICPath) && hca < gv.NearestNIC) { + gv.NearestNIC, gv.NICPath = hca, rel + } + } + if f := g.Fabric; f != nil { + gv.FabricState, gv.Domain, gv.Bandwidth = f.State, f.Domain(), f.Bandwidth + } + v.GPUs = append(v.GPUs, gv) + } fabricOf := map[string]string{} for _, f := range n.Fabrics { for _, h := range f.HCAs { @@ -959,3 +999,9 @@ func (a *API) Topology(id string) (TopologyView, bool) { } return t, true } + +// NVLink is the cluster's multi-node NVLink domains, and what keeps any of +// their nodes out. +func (a *API) NVLink() nvlink.Report { + return nvlink.Analyze(a.ctl.Inventory().Nodes()) +} diff --git a/internal/control/agentsvc/convert.go b/internal/control/agentsvc/convert.go index 0d1c0f0..a98d22b 100644 --- a/internal/control/agentsvc/convert.go +++ b/internal/control/agentsvc/convert.go @@ -61,6 +61,8 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node { CUDAVersion: g.GetCudaVersion(), NVLinkPeers: intSlice(g.GetNvlinkPeers()), NICPaths: maps.Clone(g.GetNicPaths()), + Fabric: fabricFrom(g.GetFabricState(), g.GetFabricStatus(), g.GetClusterUuid(), + g.GetCliqueId(), g.GetFabricBandwidth()), Health: discovery.GPUHealth{ ECCErrors: int(g.GetEccErrors()), TemperatureC: int(g.GetTemperatureC()), @@ -68,6 +70,7 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node { }, }) } + n.IMEX = discovery.IMEX{State: r.GetImex().GetState(), Channels: int(r.GetImex().GetChannels())} for _, f := range r.GetFabrics() { n.Fabrics = append(n.Fabrics, inventory.FabricAttachment{ Fabric: f.GetFabric(), @@ -119,7 +122,13 @@ func RegisterFromInventory(nodeID, hostname, mgmtAddr, agentVersion string, TemperatureC: int32(g.Health.TemperatureC), EccErrors: int32(g.Health.ECCErrors), }) + if f := g.Fabric; f != nil { + last := r.Gpus[len(r.Gpus)-1] + last.FabricState, last.FabricStatus, last.ClusterUuid = f.State, f.Status, f.ClusterUUID + last.CliqueId, last.FabricBandwidth = f.CliqueID, f.Bandwidth + } } + r.Imex = &llmgwv1.IMEX{State: inv.IMEX.State, Channels: int32(inv.IMEX.Channels)} for _, f := range fabrics { r.Fabrics = append(r.Fabrics, &llmgwv1.FabricAttachment{ Fabric: f.Fabric, Hcas: f.HCAs, RateGbps: int32(f.RateGbps), @@ -266,3 +275,12 @@ func int32Slice(v []int) []int32 { } return out } + +// fabricFrom is a GPU's NVLink fabric as the wire carries it, nil when the +// driver reported none. +func fabricFrom(state, status, cluster, clique, bandwidth string) *discovery.GPUFabric { + if state == "" { + return nil + } + return &discovery.GPUFabric{State: state, Status: status, ClusterUUID: cluster, CliqueID: clique, Bandwidth: bandwidth} +} diff --git a/internal/control/agentsvc/heartbeat_test.go b/internal/control/agentsvc/heartbeat_test.go index bf6b6d8..a5ab1b4 100644 --- a/internal/control/agentsvc/heartbeat_test.go +++ b/internal/control/agentsvc/heartbeat_test.go @@ -242,3 +242,25 @@ func TestAPortGoingDownTakesItsFabricAway(t *testing.T) { t.Error("a heartbeat without a fabric change emptied the fabrics") } } + +func TestAGPUFallingOutOfItsDomainTakesTheNodeOut(t *testing.T) { + // A fabric manager restarting drops cards out of the domain; the node has + // to stop being offered work that spans it until they are back. + s := newTestService() + fab := &discovery.GPUFabric{State: "Completed", ClusterUUID: "76b1ad76-7c24", CliqueID: "1"} + nodeWith(s, inventory.Node{ID: "t1", State: inventory.Ready, GPUs: []discovery.GPU{ + {Index: 0, Fabric: fab}, {Index: 1, Fabric: fab}}}) + if n, _ := s.inv.Get("t1"); n.NVLinkDomain() == "" { + t.Fatal("the node should start in its domain") + } + s.applyHeartbeat("t1", &llmgwv1.Heartbeat{NodeId: "t1", + GpuFabric: []*llmgwv1.GPUFabricState{{Index: 1, State: "In Progress"}}, + Imex: &llmgwv1.IMEX{State: "active", Channels: 1}}) + n, _ := s.inv.Get("t1") + if n.NVLinkDomain() != "" { + t.Errorf("a node with a card out of the domain is still in it") + } + if n.IMEX.State != "active" || n.IMEX.Channels != 1 { + t.Errorf("imex = %+v", n.IMEX) + } +} diff --git a/internal/control/agentsvc/service.go b/internal/control/agentsvc/service.go index 275057b..89828f6 100644 --- a/internal/control/agentsvc/service.go +++ b/internal/control/agentsvc/service.go @@ -322,7 +322,7 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) { if len(used) == 0 && len(problems) == 0 && len(storage) == 0 && !hb.GetWeightsChanged() && memMB == 0 && !hb.GetEnginesChanged() && !hb.GetProvisionChanged() && hb.GetLoad() == nil && len(hb.GetIbPorts()) == 0 && - !hb.GetFabricsChanged() { + !hb.GetFabricsChanged() && len(hb.GetGpuFabric()) == 0 && hb.GetImex() == nil { return } @@ -457,6 +457,24 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) { if len(hb.GetIbPorts()) > 0 { n.IBAt = time.Now() } + // NVLink domain membership, which a fabric manager restarting takes + // away from every card at once. + if fab := hb.GetGpuFabric(); len(fab) > 0 { + before := n.NVLinkDomain() + for _, f := range fab { + for i := range n.GPUs { + if n.GPUs[i].Index == int(f.GetIndex()) { + n.GPUs[i].Fabric = fabricFrom(f.GetState(), f.GetStatus(), f.GetClusterUuid(), f.GetCliqueId(), f.GetBandwidth()) + } + } + } + if after := n.NVLinkDomain(); after != before { + s.log.Warn("a node's NVLink domain changed", "node", nodeID, "from", before, "to", after) + } + } + if m := hb.GetImex(); m != nil { + n.IMEX = discovery.IMEX{State: m.GetState(), Channels: int(m.GetChannels())} + } // A port went down, or came back. Placement reads the fabrics, so this is // what stops a node being offered multi-node work it can no longer carry. if hb.GetFabricsChanged() { diff --git a/internal/control/fleetmetrics/fleetmetrics.go b/internal/control/fleetmetrics/fleetmetrics.go index d9e1c2d..660df99 100644 --- a/internal/control/fleetmetrics/fleetmetrics.go +++ b/internal/control/fleetmetrics/fleetmetrics.go @@ -74,6 +74,9 @@ type Collector struct { ibRate *prometheus.Desc ibCounter *prometheus.Desc + gpuNVLJoined *prometheus.Desc + imexUp *prometheus.Desc + fabricSMs *prometheus.Desc fabricMaster *prometheus.Desc fabricProblems *prometheus.Desc @@ -166,6 +169,11 @@ func New(src Source) *Collector { ibRate: prometheus.NewDesc("llmgw_ib_port_rate_gbps", "A port's negotiated rate. A link that trained below its usual speed is a "+ "cable or transceiver to look at.", []string{"node", "hca", "port"}, nil), + gpuNVLJoined: prometheus.NewDesc("llmgw_gpu_nvlink_joined", + "1 when a GPU is joined to its multi-node NVLink domain, 0 when it reports a fabric it has not "+ + "joined. Absent on cards with no multi-node fabric.", []string{"node", "gpu", "domain", "state"}, nil), + imexUp: prometheus.NewDesc("llmgw_imex_up", + "1 when nvidia-imex is active with a channel, on nodes whose GPUs report a fabric.", []string{"node"}, nil), fabricSMs: prometheus.NewDesc("llmgw_fabric_subnet_managers", "Subnet managers a physical fabric lists, by state. No standby means the fabric "+ "cannot recover from a change if its master stops.", []string{"fabric", "physical", "state"}, nil), @@ -189,6 +197,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) { c.nodeDiskFree, c.nodeDiskSize, c.nodeCPU, c.nodeCPUs, c.nodeLoad, c.nodeMemUsed, c.nodeMemAvail, c.shardCPU, c.shardMemory, c.ibActive, c.ibRate, c.ibCounter, c.fabricSMs, c.fabricMaster, c.fabricProblems, + c.gpuNVLJoined, c.imexUp, c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuTemp, c.gpuHealthy, c.gpuNVLink, c.gpuECC, c.replicaState, c.replicaReady, c.replicaIdle, @@ -263,6 +272,28 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) { } } + fabricGPUs := 0 + for _, g := range n.GPUs { + fb := g.Fabric + if fb == nil || strings.EqualFold(fb.State, "Not Supported") { + continue + } + fabricGPUs++ + joined := 0.0 + if fb.Domain() != "" { + joined = 1 + } + ch <- prometheus.MustNewConstMetric(c.gpuNVLJoined, prometheus.GaugeValue, joined, + n.ID, strconv.Itoa(g.Index), fb.Domain(), fb.State) + } + if fabricGPUs > 0 { + up := 0.0 + if n.IMEX.State == "active" && n.IMEX.Channels > 0 { + up = 1 + } + ch <- prometheus.MustNewConstMetric(c.imexUp, prometheus.GaugeValue, up, n.ID) + } + byClass := map[string]int{} for _, g := range n.GPUs { class := g.Class() diff --git a/internal/control/inventory/inventory.go b/internal/control/inventory/inventory.go index 38d5f4b..52e3e9c 100644 --- a/internal/control/inventory/inventory.go +++ b/internal/control/inventory/inventory.go @@ -120,6 +120,24 @@ type Node struct { IBPorts []IBPort `json:"-"` // IBAt is when IBPorts was read, for the rates. IBAt time.Time `json:"-"` + // IMEX is the service a multi-node NVLink domain needs on this node. + IMEX discovery.IMEX `json:"imex"` +} + +// NVLinkDomain is the multi-node NVLink domain this node's GPUs are joined +// to, when every one of them is joined to the same. A node half joined — a +// card still coming up, or one that fell out — is in none: a replica that +// counts on its NVLink would find a card that has none. +func (n Node) NVLinkDomain() string { + d := "" + for _, g := range n.GPUs { + gd := g.Fabric.Domain() + if gd == "" || (d != "" && gd != d) { + return "" + } + d = gd + } + return d } // IBPort is one port's state, identity and cumulative counters. diff --git a/internal/control/nvlink/nvlink.go b/internal/control/nvlink/nvlink.go new file mode 100644 index 0000000..3efd1a1 --- /dev/null +++ b/internal/control/nvlink/nvlink.go @@ -0,0 +1,172 @@ +// Package nvlink works out the cluster's multi-node NVLink domains — an NVL72 +// rack, an NVLink Switch system — from what each GPU reports, and what keeps +// one from being usable. +// +// A domain is a scale-up island: its GPUs reach each other over NVLink +// whatever node they are in, so a tensor-parallel group can span its nodes +// at NVLink speed. What it needs to be one is every card joined by the +// fabric manager and IMEX running on every node; either missing, and a +// replica placed across it fails or crawls. +package nvlink + +import ( + "fmt" + "sort" + "strings" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" +) + +// Level is how much a problem matters. +type Level string + +// The levels. +const ( + Fail Level = "fail" + Warn Level = "warn" +) + +// Problem is one thing wrong with a domain. +type Problem struct { + Level Level `json:"level"` + Domain string `json:"domain"` + Message string `json:"message"` +} + +// GPU is one card as its domain sees it. +type GPU struct { + Index int `json:"index"` + Name string `json:"name"` + State string `json:"state"` + Status string `json:"status,omitempty"` + Bandwidth string `json:"bandwidth,omitempty"` + Joined bool `json:"joined"` +} + +// Member is one node of a domain. +type Member struct { + Node string `json:"node"` + GPUs []GPU `json:"gpus"` + // Usable is every card joined and IMEX up: what placement requires. + Usable bool `json:"usable"` + IMEX string `json:"imex"` + IMEXChannels int `json:"imex_channels"` +} + +// Domain is one multi-node NVLink domain. +type Domain struct { + Key string `json:"key"` + ClusterUUID string `json:"cluster_uuid"` + CliqueID string `json:"clique_id"` + Members []Member `json:"members"` + GPUs int `json:"gpus"` + Joined int `json:"joined"` + Usable int `json:"usable_nodes"` +} + +// Report is every domain and what is wrong with them. +type Report struct { + Domains []Domain `json:"domains"` + Problems []Problem `json:"problems"` +} + +// Analyze reads every node's GPUs. +func Analyze(nodes []inventory.Node) Report { + var r Report + by := map[string]*Domain{} + for _, n := range nodes { + // The domain a node is in is the one any of its cards joined; a card + // still joining has no domain of its own yet, and is counted against + // the node's. + key := "" + for _, g := range n.GPUs { + if d := g.Fabric.Domain(); d != "" { + key = d + break + } + } + if key == "" { + continue + } + d := by[key] + if d == nil { + uuid, clique, _ := strings.Cut(key, "/") + d = &Domain{Key: key, ClusterUUID: uuid, CliqueID: clique} + by[key] = d + } + m := Member{Node: n.ID, IMEX: n.IMEX.State, IMEXChannels: n.IMEX.Channels} + joined := 0 + for _, g := range n.GPUs { + v := GPU{Index: g.Index, Name: g.Name} + if f := g.Fabric; f != nil { + v.State, v.Status, v.Bandwidth = f.State, f.Status, f.Bandwidth + v.Joined = f.Domain() == key + } + if v.Joined { + joined++ + } else { + why := v.State + if why == "" { + why = "no fabric" + } + if v.Status != "" && !strings.EqualFold(v.Status, "Success") { + why += ": " + v.Status + } + r.Problems = append(r.Problems, Problem{Level: Fail, Domain: key, Message: fmt.Sprintf( + "%s GPU %d is not joined to its domain (%s)", n.ID, g.Index, why)}) + } + if v.Bandwidth != "" && !strings.EqualFold(v.Bandwidth, "Full") { + r.Problems = append(r.Problems, Problem{Level: Warn, Domain: key, Message: fmt.Sprintf( + "%s GPU %d has %s NVLink bandwidth", n.ID, g.Index, strings.ToLower(v.Bandwidth))}) + } + m.GPUs = append(m.GPUs, v) + } + imexOK := n.IMEX.State == "active" && n.IMEX.Channels > 0 + switch { + case n.IMEX.State != "active": + state := n.IMEX.State + if state == "" { + state = "not installed" + } + r.Problems = append(r.Problems, Problem{Level: Fail, Domain: key, Message: fmt.Sprintf( + "%s: nvidia-imex is %s — without it the domain's GPUs cannot share memory across nodes", + n.ID, state)}) + case n.IMEX.Channels == 0: + r.Problems = append(r.Problems, Problem{Level: Fail, Domain: key, Message: fmt.Sprintf( + "%s: nvidia-imex runs but there is no IMEX channel for an engine to open (/dev/nvidia-caps-imex-channels)", n.ID)}) + } + m.Usable = joined == len(n.GPUs) && joined > 0 && imexOK + if joined > 0 && joined < len(n.GPUs) { + r.Problems = append(r.Problems, Problem{Level: Fail, Domain: key, Message: fmt.Sprintf( + "%s is half joined (%d of %d GPUs), so no replica spans the domain through it", n.ID, joined, len(n.GPUs))}) + } + d.GPUs += len(n.GPUs) + d.Joined += joined + if m.Usable { + d.Usable++ + } + d.Members = append(d.Members, m) + } + for _, d := range by { + sort.Slice(d.Members, func(i, j int) bool { return d.Members[i].Node < d.Members[j].Node }) + r.Domains = append(r.Domains, *d) + } + sort.Slice(r.Domains, func(i, j int) bool { return r.Domains[i].Key < r.Domains[j].Key }) + sort.SliceStable(r.Problems, func(i, j int) bool { + if r.Problems[i].Level != r.Problems[j].Level { + return r.Problems[i].Level == Fail + } + return r.Problems[i].Message < r.Problems[j].Message + }) + return r +} + +// Short is a domain's key as a person reads it: the cluster's first group +// and the clique. +func Short(key string) string { + uuid, clique, _ := strings.Cut(key, "/") + if i := strings.IndexByte(uuid, '-'); i > 0 { + uuid = uuid[:i] + } + return uuid + "·" + clique +} diff --git a/internal/control/nvlink/nvlink_test.go b/internal/control/nvlink/nvlink_test.go new file mode 100644 index 0000000..faf352f --- /dev/null +++ b/internal/control/nvlink/nvlink_test.go @@ -0,0 +1,55 @@ +package nvlink + +import ( + "strings" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" +) + +func tray(id string, gpus int, state string) inventory.Node { + n := inventory.Node{ID: id, IMEX: discovery.IMEX{State: "active", Channels: 1}} + for i := range gpus { + n.GPUs = append(n.GPUs, discovery.GPU{Index: i, Name: "NVIDIA GB200", Fabric: &discovery.GPUFabric{ + State: state, Status: "Success", ClusterUUID: "76b1ad76-7c24-4b6b-9f6c-2a1e0d8c11aa", CliqueID: "1", Bandwidth: "Full"}}) + } + return n +} + +func has(t *testing.T, r Report, l Level, s string) { + t.Helper() + for _, p := range r.Problems { + if p.Level == l && strings.Contains(p.Message, s) { + return + } + } + t.Errorf("no %s problem with %q in %+v", l, s, r.Problems) +} + +func TestARackOfJoinedTraysIsOneUsableDomain(t *testing.T) { + r := Analyze([]inventory.Node{tray("t1", 4, "Completed"), tray("t2", 4, "Completed"), {ID: "cpu"}}) + if len(r.Domains) != 1 || r.Domains[0].Usable != 2 || r.Domains[0].Joined != 8 || len(r.Problems) != 0 { + t.Errorf("report = %+v", r) + } + if Short(r.Domains[0].Key) != "76b1ad76·1" { + t.Errorf("short = %q", Short(r.Domains[0].Key)) + } +} + +func TestWhatKeepsATrayOutIsSaid(t *testing.T) { + half := tray("t2", 4, "Completed") + half.GPUs[3].Fabric = &discovery.GPUFabric{State: "In Progress"} + noIMEX := tray("t3", 4, "Completed") + noIMEX.IMEX = discovery.IMEX{State: "inactive"} + slow := tray("t4", 4, "Completed") + slow.GPUs[0].Fabric.Bandwidth = "Degraded" + r := Analyze([]inventory.Node{tray("t1", 4, "Completed"), half, noIMEX, slow}) + has(t, r, Fail, "t2 GPU 3 is not joined to its domain (In Progress)") + has(t, r, Fail, "t2 is half joined") + has(t, r, Fail, "t3: nvidia-imex is inactive") + has(t, r, Warn, "t4 GPU 0 has degraded NVLink bandwidth") + if r.Domains[0].Usable != 2 { + t.Errorf("usable = %d, want t1 and t4", r.Domains[0].Usable) + } +} diff --git a/internal/control/placement/placement.go b/internal/control/placement/placement.go index 69a4674..59ee579 100644 --- a/internal/control/placement/placement.go +++ b/internal/control/placement/placement.go @@ -239,13 +239,20 @@ func (s *Scheduler) placeMultiNode(p Profile, avoid []string) (*Placement, []Rej var rejections []Rejection fabrics := s.inv.Fabrics() - if len(fabrics) == 0 { + if len(fabrics) == 0 && !s.anyNVLinkDomain() { return nil, []Rejection{{ Profile: p.Name, Reason: fmt.Sprintf("needs %d nodes on one fabric, but no node reports a fabric", p.Nodes), }} } + // A multi-node NVLink domain first: a replica inside one runs tensor + // parallelism across its nodes at NVLink speed, which no InfiniBand + // fabric matches. + if pl := s.placeOnNVLink(p, avoid, &rejections); pl != nil { + return pl, rejections + } + canon := inventory.CanonicalSMs(s.inv.Nodes()) for _, fabric := range fabrics { all := s.eligibleNodes(p, fabric, avoid, &rejections) @@ -262,6 +269,66 @@ func (s *Scheduler) placeMultiNode(p Profile, avoid []string) (*Placement, []Rej return nil, rejections } +// NVLinkFabric is how a placement names a multi-node NVLink domain in its +// Fabric, so the shards on it are told to use it. +const NVLinkFabric = "nvl:" + +func (s *Scheduler) anyNVLinkDomain() bool { + for _, n := range s.inv.Nodes() { + if n.NVLinkDomain() != "" { + return true + } + } + return false +} + +// placeOnNVLink forms a replica inside one multi-node NVLink domain, from +// nodes whose every card is joined to it and whose IMEX is up: a node +// missing either would leave the replica a card, or its memory, short. +func (s *Scheduler) placeOnNVLink(p Profile, avoid []string, rejections *[]Rejection) *Placement { + var none []Rejection + cands := s.eligibleNodes(p, "", avoid, &none) + by := map[string][]candidate{} + for _, c := range cands { + d := c.node.NVLinkDomain() + if d == "" { + continue + } + if c.node.IMEX.State != "active" || c.node.IMEX.Channels == 0 { + *rejections = append(*rejections, Rejection{Profile: p.Name, Pool: NVLinkFabric + shortDomain(d), Node: c.node.ID, + Reason: "in an NVLink domain, but nvidia-imex is not up"}) + continue + } + by[d] = append(by[d], c) + } + keys := make([]string, 0, len(by)) + for k := range by { + keys = append(keys, k) + } + sort.Slice(keys, func(i, j int) bool { + if len(by[keys[i]]) != len(by[keys[j]]) { + return len(by[keys[i]]) > len(by[keys[j]]) + } + return keys[i] < keys[j] + }) + for _, d := range keys { + pool := NVLinkFabric + shortDomain(d) + "/" + p.classLabel() + if pl := s.placeOnFabric(p, NVLinkFabric+d, pool, by[d], rejections); pl != nil { + return pl + } + } + return nil +} + +// shortDomain is an NVLink domain as a pool name carries it. +func shortDomain(d string) string { + uuid, clique, _ := strings.Cut(d, "/") + if i := strings.IndexByte(uuid, '-'); i > 0 { + uuid = uuid[:i] + } + return uuid + "." + clique +} + // fabricGroup is the candidates on one physical fabric. type fabricGroup struct { key string diff --git a/internal/control/placement/placement_test.go b/internal/control/placement/placement_test.go index 721232a..87e5274 100644 --- a/internal/control/placement/placement_test.go +++ b/internal/control/placement/placement_test.go @@ -1018,3 +1018,56 @@ func TestAReplicaStaysUnderOneLeafAndOffDegradedCables(t *testing.T) { t.Errorf("placed on %v, want the two good nodes under leaf-b", got) } } + +// inDomain joins every card of a node to one multi-node NVLink domain, with +// IMEX up. +func inDomain(n inventory.Node, clique string) inventory.Node { + for i := range n.GPUs { + n.GPUs[i].Fabric = &discovery.GPUFabric{State: "Completed", ClusterUUID: "76b1ad76-7c24-4b6b-9f6c-2a1e0d8c11aa", CliqueID: clique} + } + n.IMEX = discovery.IMEX{State: "active", Channels: 1} + return n +} + +func TestAReplicaSpansNodesOverNVLinkBeforeInfiniBand(t *testing.T) { + // Four trays on one InfiniBand fabric, two of them in an NVLink domain: + // the pair is placed across the domain, where tensor parallelism between + // nodes runs at NVLink speed, and given no adapters to use. + st := store( + onSM(node("ib1", "h100-80g", 4, "ib-a"), "0x01"), onSM(node("ib2", "h100-80g", 4, "ib-a"), "0x01"), + inDomain(onSM(node("nv1", "h100-80g", 4, "ib-a"), "0x01"), "7"), + inDomain(onSM(node("nv2", "h100-80g", 4, "ib-a"), "0x01"), "7"), + ) + res := New(st).Place([]Profile{{Name: "tp8", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 2}}, nil) + if res.Placement == nil { + t.Fatalf("no placement:\n%s", res.Explain()) + } + if !strings.HasPrefix(res.Placement.Fabric, NVLinkFabric) || !strings.HasPrefix(res.Placement.Pool, "nvl:76b1ad76.7/") { + t.Errorf("fabric %q pool %q", res.Placement.Fabric, res.Placement.Pool) + } + for _, a := range res.Placement.Nodes { + if !strings.HasPrefix(a.NodeID, "nv") || len(a.HCAs) != 0 { + t.Errorf("assignment %+v", a) + } + } +} + +func TestANodeWithoutIMEXIsNotCountedInItsDomain(t *testing.T) { + a := inDomain(onSM(node("nv1", "h100-80g", 4, "ib-a"), "0x01"), "7") + b := inDomain(onSM(node("nv2", "h100-80g", 4, "ib-a"), "0x01"), "7") + b.IMEX.State = "inactive" + res := New(store(a, b)).Place([]Profile{{Name: "tp8", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 2}}, nil) + // Not across the domain; InfiniBand still carries it. + if res.Placement == nil || strings.HasPrefix(res.Placement.Fabric, NVLinkFabric) { + t.Fatalf("placement %+v\n%s", res.Placement, res.Explain()) + } + + // With no InfiniBand to fall back on, the refusal says why. + a2 := inDomain(node("nv1", "h100-80g", 4), "7") + b2 := inDomain(node("nv2", "h100-80g", 4), "7") + b2.IMEX.State = "inactive" + res = New(store(a2, b2)).Place([]Profile{{Name: "tp8", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 2}}, nil) + if res.Placement != nil || !strings.Contains(res.Explain(), "nvidia-imex is not up") { + t.Errorf("the explanation should say why the domain was passed over:\n%s", res.Explain()) + } +} diff --git a/internal/doctor/doctor.go b/internal/doctor/doctor.go index 512d61d..554e53b 100644 --- a/internal/doctor/doctor.go +++ b/internal/doctor/doctor.go @@ -56,8 +56,10 @@ type Facts struct { TopoErr string `json:"topo_error,omitempty"` // NVSwitch is set when the machine has NVSwitch devices, which need the // fabric manager running before CUDA will start. - NVSwitch bool `json:"nvswitch"` - FabricManager string `json:"fabric_manager,omitempty"` + NVSwitch bool `json:"nvswitch"` + // IMEX is the service multi-node NVLink needs. + IMEX discovery.IMEX `json:"imex"` + FabricManager string `json:"fabric_manager,omitempty"` HCAs []discovery.HCA `json:"hcas"` // SMErrors say, per port, why the subnet manager could not be asked. @@ -120,7 +122,11 @@ func Collect(ctx context.Context, o Options) Facts { f.TopoErr = err.Error() } } + if fab, err := discovery.DiscoverGPUFabric(ctx, smi); err == nil { + discovery.ApplyFabric(gpus, fab) + } f.GPUs = gpus + f.IMEX = discovery.DiscoverIMEX(ctx) } if m, _ := filepath.Glob("/dev/nvidia-nvswitch*"); len(m) > 0 { f.NVSwitch = true @@ -306,6 +312,48 @@ func Evaluate(f Facts) []Finding { } } + // --- multi-node NVLink --- + domain := "" + for _, g := range f.GPUs { + fb := g.Fabric + if fb == nil || strings.EqualFold(fb.State, "Not Supported") { + continue + } + if d := fb.Domain(); d != "" { + domain = d + if fb.Bandwidth != "" && !strings.EqualFold(fb.Bandwidth, "Full") { + add(Warn, "nvlink", fmt.Sprintf("GPU %d's NVLink bandwidth is %s", g.Index, strings.ToLower(fb.Bandwidth)), + "check its NVLink cables and trays: nvidia-smi -q -d FABRIC") + } + continue + } + if !strings.EqualFold(fb.State, "Completed") { + why := fb.State + if fb.Status != "" && !strings.EqualFold(fb.Status, "Success") { + why += ": " + fb.Status + } + add(Fail, "nvlink", fmt.Sprintf("GPU %d is not joined to its NVLink domain (%s)", g.Index, why), + "the fabric manager joins it: check nvidia-fabricmanager and the NVLink switch trays") + } + } + if domain != "" { + add(OK, "nvlink", "the GPUs are in multi-node NVLink domain "+domain, "") + switch { + case f.IMEX.State != "active": + state := f.IMEX.State + if state == "" { + state = "not installed" + } + add(Fail, "nvlink", "nvidia-imex is "+state+": without it the domain's GPUs cannot share memory across nodes", + "systemctl enable --now nvidia-imex, with every node of the domain in /etc/nvidia-imex/nodes_config.cfg") + case f.IMEX.Channels == 0: + add(Fail, "nvlink", "nvidia-imex runs but no IMEX channel exists for an engine to open", + "load the driver with options nvidia NVreg_CreateImexChannel0=1") + default: + add(OK, "nvlink", fmt.Sprintf("nvidia-imex is active with %d channel(s)", f.IMEX.Channels), "") + } + } + // --- InfiniBand and RoCE --- if len(f.HCAs) == 0 { add(Info, "fabric", "no InfiniBand or RoCE adapters: this node can only run single-node replicas", "") diff --git a/internal/doctor/doctor_test.go b/internal/doctor/doctor_test.go index 560f5c0..a9ce1f8 100644 --- a/internal/doctor/doctor_test.go +++ b/internal/doctor/doctor_test.go @@ -123,3 +123,21 @@ func TestANodeOutsideTheFabricIsAnAdapterLeftAlone(t *testing.T) { t.Errorf("failures: %v", levels(fs, Fail)) } } + +func TestAnNVLinkDomainNeedsEveryGPUJoinedAndIMEX(t *testing.T) { + f := healthy() + for i := range f.GPUs { + f.GPUs[i].Fabric = &discovery.GPUFabric{State: "Completed", Status: "Success", + ClusterUUID: "76b1ad76-7c24-4b6b-9f6c-2a1e0d8c11aa", CliqueID: "1", Bandwidth: "Full"} + } + f.IMEX = discovery.IMEX{State: "active", Channels: 1} + fs := Evaluate(f) + has(t, fs, OK, "multi-node NVLink domain") + has(t, fs, OK, "nvidia-imex is active") + + f.GPUs[5].Fabric = &discovery.GPUFabric{State: "In Progress"} + f.IMEX.Channels = 0 + fs = Evaluate(f) + has(t, fs, Fail, "GPU 5 is not joined to its NVLink domain (In Progress)") + has(t, fs, Fail, "no IMEX channel") +} diff --git a/internal/router/handlers.go b/internal/router/handlers.go index 545734c..8b4d112 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -164,6 +164,7 @@ func (r *Router) Handler() http.Handler { }, r.log))) mux.HandleFunc("GET /admin/nodes/{id}/logs", pages.AdminPage( web.NodeLogsPage(r.clusterView, false, version.Version))) + mux.HandleFunc("GET /admin/nvlink", pages.AdminPage(web.NVLinkPage(r.clusterView, false, version.Version))) mux.HandleFunc("GET /admin/fabrics", pages.AdminPage(web.FabricsPage(r.clusterView, false, version.Version))) mux.HandleFunc("GET /admin/fabrics/{id}", pages.AdminPage(web.FabricsPage(r.clusterView, false, version.Version))) mux.HandleFunc("GET /admin/fabrics/{id}/topology.json", pages.AdminPage(web.FabricTopologyJSON(r.clusterView))) diff --git a/internal/web/dashboard.go b/internal/web/dashboard.go index 5b8e344..27c3e26 100644 --- a/internal/web/dashboard.go +++ b/internal/web/dashboard.go @@ -9,6 +9,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink" "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) @@ -28,6 +29,8 @@ type DashboardData struct { // Fabrics is the physical fabrics and what disagrees about them, from a // source that can say. Fabrics *fabrics.Report + // NVLink is the multi-node NVLink domains, from a source that can say. + NVLink *nvlink.Report Users []store.UserUsage Models2 []store.ModelUsage @@ -179,6 +182,11 @@ func Dashboard(src Source, usage UsageSource, version string, refresh time.Durat r := fs.Fabrics() d.Fabrics = &r } + if ns, ok := src.(NVLinkSource); ok { + if r := ns.NVLink(); len(r.Domains) > 0 { + d.NVLink = &r + } + } } if usage != nil { rows, err := usage.UsageSince(r.Context(), time.Now().Add(-24*time.Hour)) diff --git a/internal/web/node_page_test.go b/internal/web/node_page_test.go index 636d899..e463946 100644 --- a/internal/web/node_page_test.go +++ b/internal/web/node_page_test.go @@ -7,6 +7,7 @@ import ( "time" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink" "gitea.mynodes.xyz/ilker/llmgw/internal/control/topology" "gitea.mynodes.xyz/ilker/llmgw/internal/promq" ) @@ -145,3 +146,31 @@ func TestAFabricsPageCarriesItsMapSafely(t *testing.T) { t.Errorf("json: %d %s", code, body) } } + +type nvlSource struct{ busySource } + +func (nvlSource) NVLink() nvlink.Report { + return nvlink.Report{ + Domains: []nvlink.Domain{{Key: "76b1ad76-7c24/7", ClusterUUID: "76b1ad76-7c24", CliqueID: "7", GPUs: 8, Joined: 7, Usable: 1, + Members: []nvlink.Member{ + {Node: "t1", Usable: true, IMEX: "active", IMEXChannels: 1, GPUs: []nvlink.GPU{{Index: 0, Joined: true, State: "Completed"}}}, + {Node: "t2", IMEX: "inactive", GPUs: []nvlink.GPU{{Index: 3, State: "In Progress"}}}, + }}}, + Problems: []nvlink.Problem{{Level: nvlink.Fail, Domain: "76b1ad76-7c24/7", Message: "t2: nvidia-imex is inactive"}}, + } +} + +func TestTheNVLinkPageDrawsEachDomainAsARack(t *testing.T) { + _, body := get(t, NVLinkPage(nvlSource{}, false, "v1"), "http://x/admin/nvlink", nil) + for _, want := range []string{`class="rack troubled"`, "76b1ad76·7", "1/2trays usable", `class="chip joined"`, + `class="chip lost"`, `class="tray out"`, "t2: nvidia-imex is inactive", `class="imex up"`} { + if !strings.Contains(body, want) { + t.Errorf("missing %q", want) + } + } + // And the cluster page points at it. + _, body = get(t, Dashboard(nvlSource{}, fakeUsage{}, "v1", time.Second, false, NavExtras{}, nil), "http://x/admin", nil) + if !strings.Contains(body, "see the racks") { + t.Error("the cluster page does not lead to the racks") + } +} diff --git a/internal/web/nvlink_page.go b/internal/web/nvlink_page.go new file mode 100644 index 0000000..fa2e311 --- /dev/null +++ b/internal/web/nvlink_page.go @@ -0,0 +1,56 @@ +package web + +import ( + "net/http" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink" +) + +// NVLinkSource is a Source that can report multi-node NVLink domains. +type NVLinkSource interface { + NVLink() nvlink.Report +} + +type nvlinkPageData struct { + Nav Nav + OnAdminListener bool + Nodes string + Domains []nvlinkDomainView + Problems []nvlink.Problem + Version, Now string +} + +type nvlinkDomainView struct { + nvlink.Domain + Short string + Problems []nvlink.Problem +} + +// NVLinkPage shows every multi-node NVLink domain as a rack of trays. +func NVLinkPage(src Source, onAdminListener bool, version string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + ns, ok := src.(NVLinkSource) + if !ok { + http.NotFound(w, r) + return + } + rep := ns.NVLink() + d := nvlinkPageData{Nav: navFrom(r, "cluster"), OnAdminListener: onAdminListener, Nodes: "/admin/nodes", + Problems: rep.Problems, Version: version, Now: time.Now().UTC().Format(time.RFC3339)} + if onAdminListener { + d.Nodes = "/nodes" + } + for _, dom := range rep.Domains { + v := nvlinkDomainView{Domain: dom, Short: nvlink.Short(dom.Key)} + for _, p := range rep.Problems { + if p.Domain == dom.Key { + v.Problems = append(v.Problems, p) + } + } + d.Domains = append(d.Domains, v) + } + noStore(w) + render(w, "nvlink", d) + } +} diff --git a/internal/web/static/base.css b/internal/web/static/base.css index d83daf8..1782431 100644 --- a/internal/web/static/base.css +++ b/internal/web/static/base.css @@ -677,3 +677,71 @@ ul.fabric-problems li.info { color: var(--muted); } } .mode-rails .fm-host.down circle:not(.fm-halo) { fill: var(--fm-bad); stroke: #fecdd3; } .fm-svg.focus .fm-warn { opacity: .25; } + +/* A node's GPUs: which pairs NVLink joins. */ +.gpu-wiring { display: flex; flex-wrap: wrap; gap: 1.25rem; align-items: flex-start; } +table.nvl-matrix { width: auto; border-collapse: separate; border-spacing: 3px; background: none; font-size: .75rem; } +table.nvl-matrix th { font-weight: 600; color: var(--muted); padding: 0 .3rem; text-align: center; border: 0; } +table.nvl-matrix td { width: 2rem; height: 2rem; text-align: center; border-radius: 6px; padding: 0; border: 0; + font-weight: 700; font-size: .7rem; } +table.nvl-matrix td.self { background: var(--line); } +table.nvl-matrix td.nv { color: #04211c; background: linear-gradient(135deg, #5eead4, #2dd4bf); + box-shadow: 0 0 10px rgba(45, 212, 191, .45); } +table.nvl-matrix td.pcie { color: var(--muted); background: var(--card); border: 1px solid var(--line); } +table.gpu-list { width: auto; flex: 1 1 22rem; } + +/* --------------------------------------------------------------------- + NVLink domains, drawn as racks: the switch band on top, a backplane + carrying light down the side, trays of GPU chips that glow when joined. + --------------------------------------------------------------------- */ +.racks { display: grid; grid-template-columns: repeat(auto-fill, minmax(26rem, 1fr)); gap: 1.25rem; margin: 1rem 0 1.5rem; } +.rack { border-radius: 16px; overflow: hidden; color: #dbe4ff; + background: radial-gradient(700px 260px at 50% -20%, #1a3b3b 0%, #0b1020 60%), #0b1020; + border: 1px solid #1c2f3f; box-shadow: 0 20px 50px -28px rgba(45, 212, 191, .45); } +.rack.troubled { border-color: #4a1d2c; box-shadow: 0 20px 50px -28px rgba(251, 113, 133, .45); } +.rack-head { display: flex; justify-content: space-between; align-items: flex-start; gap: 1rem; padding: .9rem 1rem .6rem; } +.rack-head h2 { margin: 0; color: #fff; font-size: 1.15rem; letter-spacing: .02em; } +.rack-head .note, .rack-head code { color: #7c89ad; background: none; font-size: .72rem; } +.rack-stats { display: flex; gap: 1rem; text-align: right; } +.rack-stats b { display: block; font-size: 1.1rem; color: #fff; font-variant-numeric: tabular-nums; } +.rack-stats span { font-size: .62rem; letter-spacing: .1em; text-transform: uppercase; color: #7c89ad; } +.rack-body { padding: .25rem 1rem 1rem; } +.nvswitch { position: relative; height: 1.9rem; border-radius: 8px; overflow: hidden; display: flex; align-items: center; + justify-content: center; font-size: .68rem; letter-spacing: .22em; text-transform: uppercase; color: #ccfbf1; + background: linear-gradient(90deg, #0f3b3a, #115e59 50%, #0f3b3a); border: 1px solid #2dd4bf55; + box-shadow: 0 0 22px rgba(45, 212, 191, .35), inset 0 0 12px rgba(45, 212, 191, .25); } +.nvswitch span { position: relative; z-index: 1; text-shadow: 0 0 8px rgba(94, 234, 212, .8); } +.nvswitch i { position: absolute; inset: 0; background: linear-gradient(100deg, transparent 30%, rgba(255,255,255,.28) 50%, transparent 70%); + transform: translateX(-100%); animation: nv-shimmer 3.2s ease-in-out infinite; } +@keyframes nv-shimmer { to { transform: translateX(100%); } } +.rack-trays { position: relative; margin-top: .6rem; padding-left: 1.1rem; display: flex; flex-direction: column; gap: .45rem; } +.backplane { position: absolute; left: .3rem; top: -.6rem; bottom: .3rem; width: 4px; border-radius: 4px; + background: linear-gradient(180deg, #2dd4bf, #0f766e 40%, #2dd4bf 60%, #0f766e); background-size: 100% 300%; + box-shadow: 0 0 12px rgba(45, 212, 191, .6); animation: nv-flow 2.4s linear infinite; } +@keyframes nv-flow { to { background-position: 0 300%; } } +.tray { position: relative; display: flex; align-items: center; gap: .75rem; padding: .45rem .6rem; border-radius: 10px; + background: linear-gradient(180deg, rgba(255,255,255,.05), rgba(255,255,255,.015)); border: 1px solid #1f2d4a; } +.tray::before { content: ""; position: absolute; left: -.8rem; top: 50%; width: .8rem; height: 2px; + background: linear-gradient(90deg, #2dd4bf, transparent); box-shadow: 0 0 6px #2dd4bf; } +.tray.out { border-color: #5b2333; background: linear-gradient(180deg, rgba(251,113,133,.08), rgba(251,113,133,.02)); } +.tray.out::before { background: linear-gradient(90deg, #fb7185, transparent); box-shadow: 0 0 6px #fb7185; } +.tray-name { min-width: 6.5rem; font-size: .8rem; font-weight: 700; color: #e0f2fe; text-decoration: none; } +.tray-name:hover { color: #5eead4; } +.chips { display: flex; flex-wrap: wrap; gap: .35rem; flex: 1; } +.chip { width: 1.7rem; height: 1.7rem; border-radius: 6px; display: inline-flex; align-items: center; justify-content: center; + font-size: .62rem; font-weight: 800; color: #04211c; cursor: default; } +.chip.joined { background: linear-gradient(135deg, #5eead4, #14b8a6); box-shadow: 0 0 12px rgba(45, 212, 191, .55); + animation: nv-breathe 3.6s ease-in-out infinite; } +.chip.joined.slow { background: linear-gradient(135deg, #fde68a, #f59e0b); box-shadow: 0 0 12px rgba(245, 158, 11, .6); } +.chip.lost { color: #fff; background: linear-gradient(135deg, #fb7185, #be123c); box-shadow: 0 0 14px rgba(251, 113, 133, .7); + animation: nv-alarm 1.2s ease-in-out infinite; } +@keyframes nv-breathe { 50% { box-shadow: 0 0 4px rgba(45, 212, 191, .25); } } +@keyframes nv-alarm { 50% { opacity: .55; } } +.imex { font-size: .6rem; font-weight: 800; letter-spacing: .08em; padding: .15rem .4rem; border-radius: 999px; + color: #fecdd3; border: 1px solid #fb7185; } +.imex.up { color: #99f6e4; border-color: #2dd4bf; box-shadow: 0 0 8px rgba(45, 212, 191, .35); } +.rack-problems { padding: 0 1rem .9rem; margin: 0; } +.rack-problems li { color: #dbe4ff; } +@media (prefers-reduced-motion: reduce) { + .nvswitch i, .backplane, .chip.joined, .chip.lost { animation: none; } +} diff --git a/internal/web/templates/dashboard.html b/internal/web/templates/dashboard.html index 585d7a2..cec9473 100644 --- a/internal/web/templates/dashboard.html +++ b/internal/web/templates/dashboard.html @@ -165,6 +165,13 @@ A replica spanning nodes is only ever formed on one of those. Fully free counts unused nodes, which is what decides whether such a replica can land.

{{end}} +{{with .NVLink}} +

NVLink domains

+

{{len .Domains}} multi-node domain{{if ne (len .Domains) 1}}s{{end}}{{range .Domains}} · {{shortDomain .Key}} {{.Usable}}/{{len .Members}} trays usable{{end}} +{{if .Problems}} · {{len .Problems}} problem{{if ne (len .Problems) 1}}s{{end}}{{end}} +· see the racks →

+{{end}} + {{with .Fleet}}

What the cluster is doing last {{.Window}}

{{if .Err}} diff --git a/internal/web/templates/node.html b/internal/web/templates/node.html index 4ba4f9b..ae541ba 100644 --- a/internal/web/templates/node.html +++ b/internal/web/templates/node.html @@ -52,6 +52,34 @@ the adapter came up: a number that grows is the one to look at.

{{end}} {{end}} +{{if .Node.GPUs}} +

GPUs and NVLink

+

+ {{with .Node.NVLinkDomain}}In NVLink domain {{.}}: tensor parallelism can span this node and the rest of its domain. + {{else}}NVLink ends at this machine.{{end}} + {{if or .Node.NVLinkDomain .Node.IMEX}} · IMEX {{if eq .Node.IMEX "active"}}active, {{.Node.IMEXChannels}} channel{{if ne .Node.IMEXChannels 1}}s{{end}}{{else}}{{or .Node.IMEX "not installed"}}{{end}}{{end}} +

+
+ + {{range .Node.GPUs}}{{end}} + {{range $a := .Node.GPUs}} + {{range $b := $.Node.GPUs}}{{if eq $a.Index $b.Index}}{{else if hasInt $a.NVLinkPeers $b.Index}}{{else}}{{end}}{{end}} + {{end}} +
{{.Index}}
GPU {{$a.Index}}NV·
+ + + {{range .Node.GPUs}} + + + + {{end}} +
GPUNameNUMANearest adapterMulti-node NVLink
{{.Index}}{{.Name}}{{.NUMA}}{{if .NearestNIC}}{{.NearestNIC}} {{.NICPath}}{{else}}—{{end}}{{if .Domain}}joined{{if and .Bandwidth (ne .Bandwidth "Full")}} {{.Bandwidth}} bandwidth{{end}}{{else if .FabricState}}{{.FabricState}}{{else}}—{{end}}
+
+

NV is a pair of cards joined by NVLink; a tensor-parallel group kept to NV pairs runs its +all-reduces at NVLink speed, and placement picks it that way. The nearest adapter is where a card's share of +traffic between nodes should go: PIX and PXB are under the same PCIe switch, SYS is across the processors.

+{{end}} +

What runs here

{{if .Engines}} diff --git a/internal/web/templates/nvlink.html b/internal/web/templates/nvlink.html new file mode 100644 index 0000000..3b23f8d --- /dev/null +++ b/internal/web/templates/nvlink.html @@ -0,0 +1,54 @@ +{{define "nvlink"}} + + + +NVLink domains · llmgw +
+
+

NVLink domains

+

{{len .Domains}} multi-node domain{{if ne (len .Domains) 1}}s{{end}} — racks whose GPUs reach each other over NVLink across nodes

+ {{if .OnAdminListener}}{{else}}{{template "nav" .Nav}}{{end}} +
+ +{{if not .Domains}} +

No GPU reports a multi-node NVLink domain. An eight-GPU machine's NVSwitch ends at its +chassis and is shown on its node's page; a domain appears here once the fabric manager has joined the +GPUs of an NVL72 rack, or an NVLink Switch system, to one.

+{{end}} + +
+{{range .Domains}} +
+
+

{{.Short}}

{{.ClusterUUID}} · clique {{.CliqueID}}

+
+
{{.Usable}}/{{len .Members}}trays usable
+
{{.Joined}}/{{.GPUs}}GPUs joined
+
+
+
+
NVLink Switch
+
+ + {{range .Members}} +
+ {{.Node}} +
{{range .GPUs}}{{.Index}}{{end}}
+ IMEX +
+ {{end}} +
+
+ {{if .Problems}}
    {{range .Problems}} +
  • {{.Level}} {{.Message}}
  • {{end}}
{{end}} +
+{{end}} +
+ +

A replica that spans nodes is placed inside one of these first: tensor parallelism between its +nodes then runs over NVLink rather than InfiniBand. Only trays with every GPU joined and IMEX up are used — +a tray short of either would leave the replica a card, or its memory, short.

+ +
llmgw {{.Version}} · refreshes every 30s{{.Now}}
+
{{end}} diff --git a/internal/web/web.go b/internal/web/web.go index 106c80b..52ed4d6 100644 --- a/internal/web/web.go +++ b/internal/web/web.go @@ -26,6 +26,7 @@ import ( "sync" "time" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink" "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) @@ -128,9 +129,11 @@ var funcs = template.FuncMap{ // shows. Binary units because that is what `ls -lh` and every disk-full // message on the host will say, and a page that disagrees with the shell // about the size of the same file starts an argument about neither. - "bytes": humanBytes, - "u64": func(v uint64) int64 { return int64(min(v, 1<<62)) }, - "has": func(list []string, v string) bool { return slices.Contains(list, v) }, + "bytes": humanBytes, + "u64": func(v uint64) int64 { return int64(min(v, 1<<62)) }, + "has": func(list []string, v string) bool { return slices.Contains(list, v) }, + "hasInt": func(list []int, v int) bool { return slices.Contains(list, v) }, + "shortDomain": nvlink.Short, // dict lets one chart template be called with named arguments. Go // templates pass a single value, and four charts differing in three // strings each is four copies of the same markup otherwise.
ModelReplicaStateCPUMemory