From 93bd16f00e3d337881d9b3c13f9f260174ce7a63 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Tue, 29 Sep 2026 00:13:54 +0200 Subject: [PATCH] Watch InfiniBand ports while the agent runs, not only when it starts A port's state was read once, at registration: one that went down later left its node on the fabric, offered multi-node work that would hang on its first collective, until somebody restarted the agent. The agent now re-reads every IB and RoCE port on each heartbeat - state, physical state, rate and the error and data counters - and when the fabrics it can carry traffic on change, says so once; the control plane replaces the node's fabrics, which is what placement reads. The ports are exported as llmgw_ib_port_active (with state and phys_state, so an unplugged cable reads differently from a missing subnet manager), llmgw_ib_port_rate_gbps and llmgw_ib_port_counter_total, and a node's page lists them with the fabric each carries and any error counter that is not zero. Co-Authored-By: Claude Opus 5.5 (1M context) --- api/gen/llmgw/v1/agent.pb.go | 346 +++++++++++++----- api/proto/llmgw/v1/agent.proto | 25 ++ internal/agent/agent.go | 95 ++++- internal/agent/agent_test.go | 31 ++ internal/agent/discovery/ib.go | 32 ++ internal/agent/discovery/ib_test.go | 18 + internal/control/adminapi/api.go | 39 ++ internal/control/agentsvc/heartbeat_test.go | 32 ++ internal/control/agentsvc/service.go | 34 +- internal/control/fleetmetrics/fleetmetrics.go | 34 ++ .../control/fleetmetrics/fleetmetrics_test.go | 20 + internal/control/inventory/inventory.go | 17 + internal/web/node_page_test.go | 17 + internal/web/templates/node.html | 20 + internal/web/web.go | 1 + 15 files changed, 654 insertions(+), 107 deletions(-) diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index 67dc12c0..be605db8 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -90,7 +90,7 @@ func (x ShardStatus_State) Number() protoreflect.EnumNumber { // Deprecated: Use ShardStatus_State.Descriptor instead. func (ShardStatus_State) EnumDescriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15, 0} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16, 0} } // AgentMessage is anything a node tells the control plane. @@ -1385,6 +1385,103 @@ func (x *ShardLoad) GetMemoryRssBytes() int64 { return 0 } +// IBPort is one InfiniBand or RoCE port's state and counters. +type IBPort struct { + state protoimpl.MessageState `protogen:"open.v1"` + Hca string `protobuf:"bytes,1,opt,name=hca,proto3" json:"hca,omitempty"` + Port int32 `protobuf:"varint,2,opt,name=port,proto3" json:"port,omitempty"` + // state is ACTIVE, INIT, DOWN...; phys_state tells a cable unplugged + // (Polling, Disabled) from a link up with no subnet manager (LinkUp and INIT). + State string `protobuf:"bytes,3,opt,name=state,proto3" json:"state,omitempty"` + PhysState string `protobuf:"bytes,4,opt,name=phys_state,json=physState,proto3" json:"phys_state,omitempty"` + RateGbps int32 `protobuf:"varint,5,opt,name=rate_gbps,json=rateGbps,proto3" json:"rate_gbps,omitempty"` + // link_layer is InfiniBand or Ethernet (RoCE). + LinkLayer string `protobuf:"bytes,6,opt,name=link_layer,json=linkLayer,proto3" json:"link_layer,omitempty"` + // counters are cumulative, by sysfs name; data counters in bytes. + Counters map[string]uint64 `protobuf:"bytes,7,rep,name=counters,proto3" json:"counters,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"varint,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *IBPort) Reset() { + *x = IBPort{} + mi := &file_llmgw_v1_agent_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *IBPort) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*IBPort) ProtoMessage() {} + +func (x *IBPort) ProtoReflect() protoreflect.Message { + mi := &file_llmgw_v1_agent_proto_msgTypes[11] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use IBPort.ProtoReflect.Descriptor instead. +func (*IBPort) Descriptor() ([]byte, []int) { + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11} +} + +func (x *IBPort) GetHca() string { + if x != nil { + return x.Hca + } + return "" +} + +func (x *IBPort) GetPort() int32 { + if x != nil { + return x.Port + } + return 0 +} + +func (x *IBPort) GetState() string { + if x != nil { + return x.State + } + return "" +} + +func (x *IBPort) GetPhysState() string { + if x != nil { + return x.PhysState + } + return "" +} + +func (x *IBPort) GetRateGbps() int32 { + if x != nil { + return x.RateGbps + } + return 0 +} + +func (x *IBPort) GetLinkLayer() string { + if x != nil { + return x.LinkLayer + } + return "" +} + +func (x *IBPort) GetCounters() map[string]uint64 { + if x != nil { + return x.Counters + } + return nil +} + type Heartbeat struct { state protoimpl.MessageState `protogen:"open.v1"` NodeId string `protobuf:"bytes,1,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` @@ -1424,14 +1521,23 @@ type Heartbeat struct { // load is how busy the machine is, sampled over the interval since the // previous heartbeat. Absent on the first one, which has nothing to take a // difference from, and from an agent on a system without /proc. - Load *NodeLoad `protobuf:"bytes,12,opt,name=load,proto3" json:"load,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + Load *NodeLoad `protobuf:"bytes,12,opt,name=load,proto3" json:"load,omitempty"` + // ib_ports is every InfiniBand or RoCE port as it stands now, re-read on + // each heartbeat: a port that goes down after the agent started used to go + // unnoticed until the agent restarted. + IbPorts []*IBPort `protobuf:"bytes,13,rep,name=ib_ports,json=ibPorts,proto3" json:"ib_ports,omitempty"` + // fabrics is the fabrics this node can now carry traffic on, sent only when + // that changed since registration or the last heartbeat that said so — a + // port going down takes its fabric away, and one coming back returns it. + FabricsChanged bool `protobuf:"varint,14,opt,name=fabrics_changed,json=fabricsChanged,proto3" json:"fabrics_changed,omitempty"` + Fabrics []*FabricAttachment `protobuf:"bytes,15,rep,name=fabrics,proto3" json:"fabrics,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Heartbeat) Reset() { *x = Heartbeat{} - mi := &file_llmgw_v1_agent_proto_msgTypes[11] + mi := &file_llmgw_v1_agent_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1443,7 +1549,7 @@ func (x *Heartbeat) String() string { func (*Heartbeat) ProtoMessage() {} func (x *Heartbeat) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[11] + mi := &file_llmgw_v1_agent_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1456,7 +1562,7 @@ func (x *Heartbeat) ProtoReflect() protoreflect.Message { // Deprecated: Use Heartbeat.ProtoReflect.Descriptor instead. func (*Heartbeat) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12} } func (x *Heartbeat) GetNodeId() string { @@ -1543,6 +1649,27 @@ func (x *Heartbeat) GetLoad() *NodeLoad { return nil } +func (x *Heartbeat) GetIbPorts() []*IBPort { + if x != nil { + return x.IbPorts + } + return nil +} + +func (x *Heartbeat) GetFabricsChanged() bool { + if x != nil { + return x.FabricsChanged + } + return false +} + +func (x *Heartbeat) GetFabrics() []*FabricAttachment { + if x != nil { + return x.Fabrics + } + return nil +} + // Provision is the software a node should have: which engines, at which // versions. Nothing more. // @@ -1562,7 +1689,7 @@ type Provision struct { func (x *Provision) Reset() { *x = Provision{} - mi := &file_llmgw_v1_agent_proto_msgTypes[12] + mi := &file_llmgw_v1_agent_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1574,7 +1701,7 @@ func (x *Provision) String() string { func (*Provision) ProtoMessage() {} func (x *Provision) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[12] + mi := &file_llmgw_v1_agent_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1587,7 +1714,7 @@ func (x *Provision) ProtoReflect() protoreflect.Message { // Deprecated: Use Provision.ProtoReflect.Descriptor instead. func (*Provision) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13} } func (x *Provision) GetEngines() []*EngineWant { @@ -1608,7 +1735,7 @@ type EngineWant struct { func (x *EngineWant) Reset() { *x = EngineWant{} - mi := &file_llmgw_v1_agent_proto_msgTypes[13] + mi := &file_llmgw_v1_agent_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1620,7 +1747,7 @@ func (x *EngineWant) String() string { func (*EngineWant) ProtoMessage() {} func (x *EngineWant) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[13] + mi := &file_llmgw_v1_agent_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1633,7 +1760,7 @@ func (x *EngineWant) ProtoReflect() protoreflect.Message { // Deprecated: Use EngineWant.ProtoReflect.Descriptor instead. func (*EngineWant) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14} } func (x *EngineWant) GetEngine() string { @@ -1666,7 +1793,7 @@ type ProvisionStatus struct { func (x *ProvisionStatus) Reset() { *x = ProvisionStatus{} - mi := &file_llmgw_v1_agent_proto_msgTypes[14] + mi := &file_llmgw_v1_agent_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1678,7 +1805,7 @@ func (x *ProvisionStatus) String() string { func (*ProvisionStatus) ProtoMessage() {} func (x *ProvisionStatus) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[14] + mi := &file_llmgw_v1_agent_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1691,7 +1818,7 @@ func (x *ProvisionStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use ProvisionStatus.ProtoReflect.Descriptor instead. func (*ProvisionStatus) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15} } func (x *ProvisionStatus) GetEngine() string { @@ -1755,7 +1882,7 @@ type ShardStatus struct { func (x *ShardStatus) Reset() { *x = ShardStatus{} - mi := &file_llmgw_v1_agent_proto_msgTypes[15] + mi := &file_llmgw_v1_agent_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1767,7 +1894,7 @@ func (x *ShardStatus) String() string { func (*ShardStatus) ProtoMessage() {} func (x *ShardStatus) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[15] + mi := &file_llmgw_v1_agent_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1780,7 +1907,7 @@ func (x *ShardStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use ShardStatus.ProtoReflect.Descriptor instead. func (*ShardStatus) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16} } func (x *ShardStatus) GetNodeId() string { @@ -1854,7 +1981,7 @@ type FetchProgress struct { func (x *FetchProgress) Reset() { *x = FetchProgress{} - mi := &file_llmgw_v1_agent_proto_msgTypes[16] + mi := &file_llmgw_v1_agent_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1866,7 +1993,7 @@ func (x *FetchProgress) String() string { func (*FetchProgress) ProtoMessage() {} func (x *FetchProgress) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[16] + mi := &file_llmgw_v1_agent_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1879,7 +2006,7 @@ func (x *FetchProgress) ProtoReflect() protoreflect.Message { // Deprecated: Use FetchProgress.ProtoReflect.Descriptor instead. func (*FetchProgress) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{17} } func (x *FetchProgress) GetSource() string { @@ -1935,7 +2062,7 @@ type StartShard struct { func (x *StartShard) Reset() { *x = StartShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[17] + mi := &file_llmgw_v1_agent_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1947,7 +2074,7 @@ func (x *StartShard) String() string { func (*StartShard) ProtoMessage() {} func (x *StartShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[17] + mi := &file_llmgw_v1_agent_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1960,7 +2087,7 @@ func (x *StartShard) ProtoReflect() protoreflect.Message { // Deprecated: Use StartShard.ProtoReflect.Descriptor instead. func (*StartShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{17} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{18} } func (x *StartShard) GetReplicaId() string { @@ -2087,7 +2214,7 @@ type EngineProfile struct { func (x *EngineProfile) Reset() { *x = EngineProfile{} - mi := &file_llmgw_v1_agent_proto_msgTypes[18] + mi := &file_llmgw_v1_agent_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2099,7 +2226,7 @@ func (x *EngineProfile) String() string { func (*EngineProfile) ProtoMessage() {} func (x *EngineProfile) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[18] + mi := &file_llmgw_v1_agent_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2112,7 +2239,7 @@ func (x *EngineProfile) ProtoReflect() protoreflect.Message { // Deprecated: Use EngineProfile.ProtoReflect.Descriptor instead. func (*EngineProfile) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{18} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{19} } func (x *EngineProfile) GetName() string { @@ -2208,7 +2335,7 @@ type RemoveWeights struct { func (x *RemoveWeights) Reset() { *x = RemoveWeights{} - mi := &file_llmgw_v1_agent_proto_msgTypes[19] + mi := &file_llmgw_v1_agent_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2220,7 +2347,7 @@ func (x *RemoveWeights) String() string { func (*RemoveWeights) ProtoMessage() {} func (x *RemoveWeights) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[19] + mi := &file_llmgw_v1_agent_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2233,7 +2360,7 @@ func (x *RemoveWeights) ProtoReflect() protoreflect.Message { // Deprecated: Use RemoveWeights.ProtoReflect.Descriptor instead. func (*RemoveWeights) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{19} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{20} } func (x *RemoveWeights) GetPaths() []string { @@ -2253,7 +2380,7 @@ type StopShard struct { func (x *StopShard) Reset() { *x = StopShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[20] + mi := &file_llmgw_v1_agent_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2265,7 +2392,7 @@ func (x *StopShard) String() string { func (*StopShard) ProtoMessage() {} func (x *StopShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[20] + mi := &file_llmgw_v1_agent_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2278,7 +2405,7 @@ func (x *StopShard) ProtoReflect() protoreflect.Message { // Deprecated: Use StopShard.ProtoReflect.Descriptor instead. func (*StopShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{20} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{21} } func (x *StopShard) GetShardId() string { @@ -2298,7 +2425,7 @@ type DrainShard struct { func (x *DrainShard) Reset() { *x = DrainShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[21] + mi := &file_llmgw_v1_agent_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2310,7 +2437,7 @@ func (x *DrainShard) String() string { func (*DrainShard) ProtoMessage() {} func (x *DrainShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[21] + mi := &file_llmgw_v1_agent_proto_msgTypes[22] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2323,7 +2450,7 @@ func (x *DrainShard) ProtoReflect() protoreflect.Message { // Deprecated: Use DrainShard.ProtoReflect.Descriptor instead. func (*DrainShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{21} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{22} } func (x *DrainShard) GetShardId() string { @@ -2345,7 +2472,7 @@ type Ack struct { func (x *Ack) Reset() { *x = Ack{} - mi := &file_llmgw_v1_agent_proto_msgTypes[22] + mi := &file_llmgw_v1_agent_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2357,7 +2484,7 @@ func (x *Ack) String() string { func (*Ack) ProtoMessage() {} func (x *Ack) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[22] + mi := &file_llmgw_v1_agent_proto_msgTypes[23] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2370,7 +2497,7 @@ func (x *Ack) ProtoReflect() protoreflect.Message { // Deprecated: Use Ack.ProtoReflect.Descriptor instead. func (*Ack) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{22} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{23} } func (x *Ack) GetMessageId() string { @@ -2414,7 +2541,7 @@ type LogsRequest struct { func (x *LogsRequest) Reset() { *x = LogsRequest{} - mi := &file_llmgw_v1_agent_proto_msgTypes[23] + mi := &file_llmgw_v1_agent_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2426,7 +2553,7 @@ func (x *LogsRequest) String() string { func (*LogsRequest) ProtoMessage() {} func (x *LogsRequest) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[23] + mi := &file_llmgw_v1_agent_proto_msgTypes[24] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2439,7 +2566,7 @@ func (x *LogsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use LogsRequest.ProtoReflect.Descriptor instead. func (*LogsRequest) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{23} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{24} } func (x *LogsRequest) GetRequestId() string { @@ -2479,7 +2606,7 @@ type LogsReply struct { func (x *LogsReply) Reset() { *x = LogsReply{} - mi := &file_llmgw_v1_agent_proto_msgTypes[24] + mi := &file_llmgw_v1_agent_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2491,7 +2618,7 @@ func (x *LogsReply) String() string { func (*LogsReply) ProtoMessage() {} func (x *LogsReply) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[24] + mi := &file_llmgw_v1_agent_proto_msgTypes[25] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2504,7 +2631,7 @@ func (x *LogsReply) ProtoReflect() protoreflect.Message { // Deprecated: Use LogsReply.ProtoReflect.Descriptor instead. func (*LogsReply) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{24} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{25} } func (x *LogsReply) GetRequestId() string { @@ -2660,7 +2787,20 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\tShardLoad\x12\x19\n" + "\bshard_id\x18\x01 \x01(\tR\ashardId\x12\x1b\n" + "\tcpu_cores\x18\x02 \x01(\x01R\bcpuCores\x12(\n" + - "\x10memory_rss_bytes\x18\x03 \x01(\x03R\x0ememoryRssBytes\"\xf0\x04\n" + + "\x10memory_rss_bytes\x18\x03 \x01(\x03R\x0ememoryRssBytes\"\x98\x02\n" + + "\x06IBPort\x12\x10\n" + + "\x03hca\x18\x01 \x01(\tR\x03hca\x12\x12\n" + + "\x04port\x18\x02 \x01(\x05R\x04port\x12\x14\n" + + "\x05state\x18\x03 \x01(\tR\x05state\x12\x1d\n" + + "\n" + + "phys_state\x18\x04 \x01(\tR\tphysState\x12\x1b\n" + + "\trate_gbps\x18\x05 \x01(\x05R\brateGbps\x12\x1d\n" + + "\n" + + "link_layer\x18\x06 \x01(\tR\tlinkLayer\x12:\n" + + "\bcounters\x18\a \x03(\v2\x1e.llmgw.v1.IBPort.CountersEntryR\bcounters\x1a;\n" + + "\rCountersEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\x04R\x05value:\x028\x01\"\xfc\x05\n" + "\tHeartbeat\x12\x17\n" + "\anode_id\x18\x01 \x01(\tR\x06nodeId\x12+\n" + "\x12gpu_memory_used_mb\x18\x02 \x03(\x05R\x0fgpuMemoryUsedMb\x12!\n" + @@ -2674,7 +2814,10 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x11provision_changed\x18\n" + " \x01(\bR\x10provisionChanged\x127\n" + "\tprovision\x18\v \x03(\v2\x19.llmgw.v1.ProvisionStatusR\tprovision\x12&\n" + - "\x04load\x18\f \x01(\v2\x12.llmgw.v1.NodeLoadR\x04load\x1aA\n" + + "\x04load\x18\f \x01(\v2\x12.llmgw.v1.NodeLoadR\x04load\x12+\n" + + "\bib_ports\x18\r \x03(\v2\x10.llmgw.v1.IBPortR\aibPorts\x12'\n" + + "\x0ffabrics_changed\x18\x0e \x01(\bR\x0efabricsChanged\x124\n" + + "\afabrics\x18\x0f \x03(\v2\x1a.llmgw.v1.FabricAttachmentR\afabrics\x1aA\n" + "\x13EngineVersionsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\";\n" + @@ -2797,7 +2940,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte { } var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 30) +var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 32) var file_llmgw_v1_agent_proto_goTypes = []any{ (ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State (*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage @@ -2811,66 +2954,71 @@ var file_llmgw_v1_agent_proto_goTypes = []any{ (*LocalWeights)(nil), // 9: llmgw.v1.LocalWeights (*NodeLoad)(nil), // 10: llmgw.v1.NodeLoad (*ShardLoad)(nil), // 11: llmgw.v1.ShardLoad - (*Heartbeat)(nil), // 12: llmgw.v1.Heartbeat - (*Provision)(nil), // 13: llmgw.v1.Provision - (*EngineWant)(nil), // 14: llmgw.v1.EngineWant - (*ProvisionStatus)(nil), // 15: llmgw.v1.ProvisionStatus - (*ShardStatus)(nil), // 16: llmgw.v1.ShardStatus - (*FetchProgress)(nil), // 17: llmgw.v1.FetchProgress - (*StartShard)(nil), // 18: llmgw.v1.StartShard - (*EngineProfile)(nil), // 19: llmgw.v1.EngineProfile - (*RemoveWeights)(nil), // 20: llmgw.v1.RemoveWeights - (*StopShard)(nil), // 21: llmgw.v1.StopShard - (*DrainShard)(nil), // 22: llmgw.v1.DrainShard - (*Ack)(nil), // 23: llmgw.v1.Ack - (*LogsRequest)(nil), // 24: llmgw.v1.LogsRequest - (*LogsReply)(nil), // 25: llmgw.v1.LogsReply - nil, // 26: llmgw.v1.Register.LabelsEntry - nil, // 27: llmgw.v1.Register.EngineVersionsEntry - nil, // 28: llmgw.v1.GPU.NicPathsEntry - nil, // 29: llmgw.v1.Heartbeat.EngineVersionsEntry - nil, // 30: llmgw.v1.EngineProfile.EnvEntry + (*IBPort)(nil), // 12: llmgw.v1.IBPort + (*Heartbeat)(nil), // 13: llmgw.v1.Heartbeat + (*Provision)(nil), // 14: llmgw.v1.Provision + (*EngineWant)(nil), // 15: llmgw.v1.EngineWant + (*ProvisionStatus)(nil), // 16: llmgw.v1.ProvisionStatus + (*ShardStatus)(nil), // 17: llmgw.v1.ShardStatus + (*FetchProgress)(nil), // 18: llmgw.v1.FetchProgress + (*StartShard)(nil), // 19: llmgw.v1.StartShard + (*EngineProfile)(nil), // 20: llmgw.v1.EngineProfile + (*RemoveWeights)(nil), // 21: llmgw.v1.RemoveWeights + (*StopShard)(nil), // 22: llmgw.v1.StopShard + (*DrainShard)(nil), // 23: llmgw.v1.DrainShard + (*Ack)(nil), // 24: llmgw.v1.Ack + (*LogsRequest)(nil), // 25: llmgw.v1.LogsRequest + (*LogsReply)(nil), // 26: llmgw.v1.LogsReply + nil, // 27: llmgw.v1.Register.LabelsEntry + nil, // 28: llmgw.v1.Register.EngineVersionsEntry + nil, // 29: llmgw.v1.GPU.NicPathsEntry + nil, // 30: llmgw.v1.IBPort.CountersEntry + nil, // 31: llmgw.v1.Heartbeat.EngineVersionsEntry + nil, // 32: llmgw.v1.EngineProfile.EnvEntry } var file_llmgw_v1_agent_proto_depIdxs = []int32{ 3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register - 12, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat - 16, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus - 25, // 3: llmgw.v1.AgentMessage.logs:type_name -> llmgw.v1.LogsReply - 18, // 4: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard - 21, // 5: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard - 22, // 6: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard - 23, // 7: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack - 20, // 8: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights - 13, // 9: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision - 24, // 10: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest - 26, // 11: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry + 13, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat + 17, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus + 26, // 3: llmgw.v1.AgentMessage.logs:type_name -> llmgw.v1.LogsReply + 19, // 4: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard + 22, // 5: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard + 23, // 6: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard + 24, // 7: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack + 21, // 8: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights + 14, // 9: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision + 25, // 10: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest + 27, // 11: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry 7, // 12: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU 8, // 13: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment - 27, // 14: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry + 28, // 14: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry 6, // 15: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard 9, // 16: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights 5, // 17: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage 4, // 18: llmgw.v1.Register.host:type_name -> llmgw.v1.Host 0, // 19: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State - 28, // 20: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry + 29, // 20: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry 11, // 21: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad - 5, // 22: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage - 9, // 23: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights - 29, // 24: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry - 15, // 25: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus - 10, // 26: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad - 14, // 27: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant - 0, // 28: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State - 17, // 29: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress - 19, // 30: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile - 30, // 31: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry - 1, // 32: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage - 2, // 33: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage - 33, // [33:34] is the sub-list for method output_type - 32, // [32:33] is the sub-list for method input_type - 32, // [32:32] is the sub-list for extension type_name - 32, // [32:32] is the sub-list for extension extendee - 0, // [0:32] is the sub-list for field type_name + 30, // 22: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry + 5, // 23: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage + 9, // 24: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights + 31, // 25: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry + 16, // 26: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus + 10, // 27: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad + 12, // 28: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort + 8, // 29: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment + 15, // 30: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant + 0, // 31: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State + 18, // 32: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress + 20, // 33: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile + 32, // 34: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry + 1, // 35: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage + 2, // 36: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage + 36, // [36:37] is the sub-list for method output_type + 35, // [35:36] is the sub-list for method input_type + 35, // [35:35] is the sub-list for extension type_name + 35, // [35:35] is the sub-list for extension extendee + 0, // [0:35] is the sub-list for field type_name } func init() { file_llmgw_v1_agent_proto_init() } @@ -2899,7 +3047,7 @@ func file_llmgw_v1_agent_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)), NumEnums: 1, - NumMessages: 30, + NumMessages: 32, NumExtensions: 0, NumServices: 1, }, diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index 00bbaf29..f435bc38 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -264,6 +264,21 @@ message ShardLoad { int64 memory_rss_bytes = 3; } +// IBPort is one InfiniBand or RoCE port's state and counters. +message IBPort { + string hca = 1; + int32 port = 2; + // state is ACTIVE, INIT, DOWN...; phys_state tells a cable unplugged + // (Polling, Disabled) from a link up with no subnet manager (LinkUp and INIT). + string state = 3; + string phys_state = 4; + int32 rate_gbps = 5; + // link_layer is InfiniBand or Ethernet (RoCE). + string link_layer = 6; + // counters are cumulative, by sysfs name; data counters in bytes. + map counters = 7; +} + message Heartbeat { string node_id = 1; // gpu_memory_used_mb, indexed by GPU, catches a foreign process appearing on @@ -309,6 +324,16 @@ message Heartbeat { // previous heartbeat. Absent on the first one, which has nothing to take a // difference from, and from an agent on a system without /proc. NodeLoad load = 12; + + // ib_ports is every InfiniBand or RoCE port as it stands now, re-read on + // each heartbeat: a port that goes down after the agent started used to go + // unnoticed until the agent restarted. + repeated IBPort ib_ports = 13; + // fabrics is the fabrics this node can now carry traffic on, sent only when + // that changed since registration or the last heartbeat that said so — a + // port going down takes its fabric away, and one coming back returns it. + bool fabrics_changed = 14; + repeated FabricAttachment fabrics = 15; } // Provision is the software a node should have: which engines, at which diff --git a/internal/agent/agent.go b/internal/agent/agent.go index f38274ed..35b2a31f 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -352,6 +352,8 @@ type Agent struct { // failover the old stream's heartbeat loop can still be running when the // new one's starts. lastWeights string + // fabricsSent is what the control plane was last told about fabrics. + fabricsSent string // load measures the machine between heartbeats. load *discovery.LoadSampler @@ -574,21 +576,36 @@ func (a *Agent) settleAdopted(sh *supervisor.Shard, o supervisor.Orphan) { // machine where both halves are visible, rather than inferred later from a // replica that will not converge. func (a *Agent) usableFabrics(inv discovery.Inventory) []inventory.FabricAttachment { + out := a.checkFabrics(inv.HCAs, true) + a.mu.Lock() + a.fabricsSent = fabricsFingerprint(out) + a.mu.Unlock() + return out +} + +// checkFabrics is the declared fabrics this node can carry traffic on, from +// the adapters as they are now. Loud logs every problem; the heartbeat checks +// quietly and speaks only when the answer changes. +func (a *Agent) checkFabrics(hcas []discovery.HCA, loud bool) []inventory.FabricAttachment { declared := map[string][]string{} for _, f := range a.cfg.Fabrics { declared[f.Fabric] = f.HCAs } var out []inventory.FabricAttachment - for _, c := range discovery.CheckFabrics(declared, inv.HCAs) { - for _, p := range c.Problems { - a.log.Error("a declared fabric adapter cannot carry traffic", - "fabric", c.Fabric, "problem", p) + for _, c := range discovery.CheckFabrics(declared, hcas) { + if loud { + for _, p := range c.Problems { + a.log.Error("a declared fabric adapter cannot carry traffic", + "fabric", c.Fabric, "problem", p) + } } if !c.OK() { - a.log.Error("fabric not advertised; no declared adapter on it can carry traffic", - "fabric", c.Fabric, - "consequence", "no multi-node replica will be placed here") + if loud { + a.log.Error("fabric not advertised; no declared adapter on it can carry traffic", + "fabric", c.Fabric, + "consequence", "no multi-node replica will be placed here") + } continue } out = append(out, inventory.FabricAttachment{ @@ -598,6 +615,69 @@ func (a *Agent) usableFabrics(inv discovery.Inventory) []inventory.FabricAttachm return out } +// fabricsFingerprint is enough to notice a fabric, an adapter or a rate +// changing. +func fabricsFingerprint(fs []inventory.FabricAttachment) string { + var b strings.Builder + for _, f := range fs { + fmt.Fprintf(&b, "%s:%s:%d;", f.Fabric, strings.Join(f.HCAs, ","), f.RateGbps) + } + return b.String() +} + +// sampleIB re-reads the adapters for the heartbeat: every port's state and +// counters, and the fabrics again when they no longer match what the control +// plane was told. +// +// Every heartbeat, because it is a few small files per port, and because a +// port that went down used to be invisible until the agent restarted — the +// node went on being offered multi-node work that would hang on its first +// collective. +func (a *Agent) sampleIB(hb *llmgwv1.Heartbeat) { + a.mu.Lock() + had := len(a.inv.HCAs) > 0 + a.mu.Unlock() + if !had && len(a.cfg.Fabrics) == 0 { + return + } + hcas, err := discovery.DiscoverIB(a.cfg.Discovery.IBRoot) + if err != nil { + return + } + for _, h := range hcas { + for _, p := range h.Ports { + hb.IbPorts = append(hb.IbPorts, &llmgwv1.IBPort{ + Hca: h.Name, Port: int32(p.Num), State: p.State, PhysState: p.PhysState, + RateGbps: int32(p.RateGbps), LinkLayer: string(p.LinkLayer), Counters: p.Counters, + }) + } + } + a.mu.Lock() + a.inv.HCAs = hcas // a reconnect registers what is true now + a.mu.Unlock() + if len(a.cfg.Fabrics) == 0 { + return + } + now := a.checkFabrics(hcas, false) + fp := fabricsFingerprint(now) + a.mu.Lock() + changed := fp != a.fabricsSent + a.fabricsSent = fp + a.mu.Unlock() + if !changed { + return + } + // Said once, with the reasons, the moment it happens. + a.checkFabrics(hcas, true) + a.log.Warn("the fabrics this node can carry traffic on changed", "now", fp) + hb.FabricsChanged = true + for _, f := range now { + hb.Fabrics = append(hb.Fabrics, &llmgwv1.FabricAttachment{ + Fabric: f.Fabric, Hcas: f.HCAs, RateGbps: int32(f.RateGbps), + }) + } +} + // weights builds the fetcher lazily, so an agent that never starts a shard // never needs a cache directory. func (a *Agent) weights() *weights.Fetcher { @@ -887,6 +967,7 @@ func (a *Agent) heartbeat() *llmgwv1.AgentMessage { } } hb.Load = a.sampleLoad() + a.sampleIB(hb) return &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Heartbeat{Heartbeat: hb}} } diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index 75e910f7..f71d29b3 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -756,3 +756,34 @@ func TestAFabricRateIsTheSlowestActiveLink(t *testing.T) { } } } + +func TestAPortGoingDownIsToldToTheControlPlaneOnce(t *testing.T) { + // Checked only at registration, a port going down went unnoticed until + // the agent restarted, and the node went on being offered multi-node + // work that would hang on its first collective. + root := ibTreeWithActivePorts(t, "mlx5_0") + h := start(t, func(c *Config) { + c.Discovery.IBRoot = root + c.Fabrics = []inventory.FabricAttachment{{Fabric: "ib-a", HCAs: []string{"mlx5_0"}}} + }) + if n, _ := h.inv.Get("n1"); !n.HasFabric("ib-a") { + t.Fatalf("the fabric should be advertised at registration: %+v", n.Fabrics) + } + + hb := h.ag.heartbeat().GetHeartbeat() + if hb.GetFabricsChanged() || len(hb.GetIbPorts()) != 1 || hb.GetIbPorts()[0].GetState() != "ACTIVE" { + t.Fatalf("a quiet heartbeat should carry the port and no change: %+v", hb) + } + + if err := os.WriteFile(filepath.Join(root, "mlx5_0", "ports", "1", "state"), []byte("1: DOWN\n"), 0o644); err != nil { + t.Fatal(err) + } + hb = h.ag.heartbeat().GetHeartbeat() + if !hb.GetFabricsChanged() || len(hb.GetFabrics()) != 0 { + t.Errorf("the port going down should withdraw the fabric: changed=%v fabrics=%v", + hb.GetFabricsChanged(), hb.GetFabrics()) + } + if hb = h.ag.heartbeat().GetHeartbeat(); hb.GetFabricsChanged() { + t.Error("the change should be sent once, not on every heartbeat") + } +} diff --git a/internal/agent/discovery/ib.go b/internal/agent/discovery/ib.go index 52ba016b..b54d1fde 100644 --- a/internal/agent/discovery/ib.go +++ b/internal/agent/discovery/ib.go @@ -42,8 +42,26 @@ type Port struct { GIDPrefix string `json:"gid_prefix"` LID int `json:"lid"` SMLID int `json:"sm_lid"` + // Counters are the port's error and traffic counters, by their sysfs + // names, as far as the driver has them. The data counters are converted + // to bytes; sysfs counts them in four-byte words. + Counters map[string]uint64 `json:"counters,omitempty"` } +// PortCounters are the counters worth watching, from each port's counters/ +// directory. The errors are the ones that say a cable, a transceiver or a +// switch port is going bad before a job fails on it; the data counters say +// whether the fabric is carrying anything at all. +var PortCounters = []string{ + "symbol_error", "link_error_recovery", "link_downed", "port_rcv_errors", + "port_rcv_remote_physical_errors", "port_xmit_discards", + "local_link_integrity_errors", "excessive_buffer_overrun_errors", + "port_xmit_data", "port_rcv_data", +} + +// dataCounter reports whether a counter counts data in four-byte words. +func dataCounter(name string) bool { return name == "port_xmit_data" || name == "port_rcv_data" } + // Active reports whether the port can carry traffic. func (p Port) Active() bool { return strings.EqualFold(p.State, "ACTIVE") } @@ -138,6 +156,20 @@ func readPort(dir string, num int) Port { LID: parseMaybeHex(readTrimmed(filepath.Join(dir, "lid"))), SMLID: parseMaybeHex(readTrimmed(filepath.Join(dir, "sm_lid"))), } + for _, name := range PortCounters { + raw := readTrimmed(filepath.Join(dir, "counters", name)) + v, err := strconv.ParseUint(raw, 10, 64) + if err != nil { + continue // not every driver has every counter + } + if dataCounter(name) { + v *= 4 + } + if p.Counters == nil { + p.Counters = map[string]uint64{} + } + p.Counters[name] = v + } return p } diff --git a/internal/agent/discovery/ib_test.go b/internal/agent/discovery/ib_test.go index 2543026e..fe02209e 100644 --- a/internal/agent/discovery/ib_test.go +++ b/internal/agent/discovery/ib_test.go @@ -31,6 +31,9 @@ func fakeSysfs(t *testing.T) string { write("ib/mlx5_0/ports/1/gids/0", "FEC0:0000:0000:000A:0002:C903:00A0:1234") write("ib/mlx5_0/ports/1/lid", "0x1a") write("ib/mlx5_0/ports/1/sm_lid", "1") + write("ib/mlx5_0/ports/1/counters/symbol_error", "3") + write("ib/mlx5_0/ports/1/counters/link_downed", "1") + write("ib/mlx5_0/ports/1/counters/port_xmit_data", "1000") // mlx5_1: two ports, the second one down. write("ib/mlx5_1/ports/1/state", "4: ACTIVE") @@ -214,3 +217,18 @@ func TestParseHelpers(t *testing.T) { } }) } + +func TestAPortsCountersAreRead(t *testing.T) { + hcas, err := DiscoverIB(fakeSysfs(t)) + if err != nil { + t.Fatal(err) + } + c := hcas[0].Ports[0].Counters + // Data in bytes: sysfs counts four-byte words. + if c["symbol_error"] != 3 || c["link_downed"] != 1 || c["port_xmit_data"] != 4000 { + t.Errorf("counters = %v", c) + } + if _, ok := c["port_rcv_errors"]; ok { + t.Error("a counter the driver does not have should be absent, not zero") + } +} diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index c81d71ff..3bf86328 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -119,6 +119,25 @@ type NodeView struct { // Load is how busy the machine is, absent when its agent has not // measured it lately — which is unknown, not idle. Load *LoadView `json:"load,omitempty"` + // IBPorts are the node's InfiniBand and RoCE ports at the last heartbeat. + IBPorts []IBPortView `json:"ib_ports,omitempty"` +} + +// IBPortView is one port as a page shows it. +type IBPortView struct { + HCA string `json:"hca"` + Port int `json:"port"` + State string `json:"state"` + PhysState string `json:"phys_state,omitempty"` + RateGbps int `json:"rate_gbps"` + LinkLayer string `json:"link_layer,omitempty"` + Active bool `json:"active"` + // Fabric is the declared fabric this port's adapter carries, if any. + Fabric string `json:"fabric,omitempty"` + // Errors are the error counters that are not zero, by name. + Errors map[string]uint64 `json:"errors,omitempty"` + TxBytes uint64 `json:"tx_bytes,omitempty"` + RxBytes uint64 `json:"rx_bytes,omitempty"` } // LoadView is a node's CPU and memory in use, at its last heartbeat. @@ -308,6 +327,26 @@ func (a *API) Nodes() []NodeView { v.LastSeenAgo = time.Since(n.LastSeen).Round(time.Second).String() } v.Load = loadView(n.Load, time.Now()) + fabricOf := map[string]string{} + for _, f := range n.Fabrics { + for _, h := range f.HCAs { + fabricOf[h] = f.Fabric + } + } + for _, p := range n.IBPorts { + pv := IBPortView{HCA: p.HCA, Port: p.Port, State: p.State, PhysState: p.PhysState, + RateGbps: p.RateGbps, LinkLayer: p.LinkLayer, Active: p.Active(), Fabric: fabricOf[p.HCA], + TxBytes: p.Counters["port_xmit_data"], RxBytes: p.Counters["port_rcv_data"]} + for name, c := range p.Counters { + if c > 0 && name != "port_xmit_data" && name != "port_rcv_data" { + if pv.Errors == nil { + pv.Errors = map[string]uint64{} + } + pv.Errors[name] = c + } + } + v.IBPorts = append(v.IBPorts, pv) + } out = append(out, v) } return out diff --git a/internal/control/agentsvc/heartbeat_test.go b/internal/control/agentsvc/heartbeat_test.go index 2ac47f6c..bf6b6d88 100644 --- a/internal/control/agentsvc/heartbeat_test.go +++ b/internal/control/agentsvc/heartbeat_test.go @@ -210,3 +210,35 @@ func TestAHeartbeatCarriesHowBusyTheNodeIs(t *testing.T) { t.Error("a heartbeat without load erased it") } } + +func TestAPortGoingDownTakesItsFabricAway(t *testing.T) { + // Checked only at registration, a port that went down left the node on + // its fabric until the agent restarted, still offered multi-node work. + s := newTestService() + nodeWith(s, inventory.Node{ID: "n1", State: inventory.Ready, + Fabrics: []inventory.FabricAttachment{{Fabric: "ib-a", HCAs: []string{"mlx5_0"}, RateGbps: 400}}}) + + s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1", FabricsChanged: true, + IbPorts: []*llmgwv1.IBPort{{Hca: "mlx5_0", Port: 1, State: "DOWN", PhysState: "Polling", + Counters: map[string]uint64{"link_downed": 4}}}}) + n, _ := s.inv.Get("n1") + if len(n.Fabrics) != 0 { + t.Errorf("the fabric should be gone: %+v", n.Fabrics) + } + if len(n.IBPorts) != 1 || n.IBPorts[0].Active() || n.IBPorts[0].Counters["link_downed"] != 4 { + t.Errorf("ports = %+v", n.IBPorts) + } + + // Back up: the fabric returns. + s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1", FabricsChanged: true, + Fabrics: []*llmgwv1.FabricAttachment{{Fabric: "ib-a", Hcas: []string{"mlx5_0"}, RateGbps: 400}}}) + if n, _ := s.inv.Get("n1"); !n.HasFabric("ib-a") { + t.Error("the fabric should be back") + } + + // A heartbeat that does not say the fabrics changed leaves them alone. + s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1", IbPorts: []*llmgwv1.IBPort{{Hca: "mlx5_0", Port: 1, State: "ACTIVE"}}}) + if n, _ := s.inv.Get("n1"); !n.HasFabric("ib-a") { + t.Error("a heartbeat without a fabric change emptied the fabrics") + } +} diff --git a/internal/control/agentsvc/service.go b/internal/control/agentsvc/service.go index e52b7bc1..edd14e97 100644 --- a/internal/control/agentsvc/service.go +++ b/internal/control/agentsvc/service.go @@ -309,7 +309,8 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) { memMB := int(hb.GetMemoryTotalMb()) if len(used) == 0 && len(problems) == 0 && len(storage) == 0 && !hb.GetWeightsChanged() && memMB == 0 && !hb.GetEnginesChanged() && - !hb.GetProvisionChanged() && hb.GetLoad() == nil { + !hb.GetProvisionChanged() && hb.GetLoad() == nil && len(hb.GetIbPorts()) == 0 && + !hb.GetFabricsChanged() { return } @@ -403,9 +404,40 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) { if l := hb.GetLoad(); l != nil { n.Load = loadFrom(l, time.Now()) } + if ports := hb.GetIbPorts(); len(ports) > 0 { + n.IBPorts = n.IBPorts[:0] + for _, p := range ports { + n.IBPorts = append(n.IBPorts, inventory.IBPort{ + HCA: p.GetHca(), Port: int(p.GetPort()), State: p.GetState(), + PhysState: p.GetPhysState(), RateGbps: int(p.GetRateGbps()), + LinkLayer: p.GetLinkLayer(), Counters: maps.Clone(p.GetCounters()), + }) + } + } + // A port went down, or came back. Placement reads the fabrics, so this is + // what stops a node being offered multi-node work it can no longer carry. + if hb.GetFabricsChanged() { + var fabs []inventory.FabricAttachment + for _, f := range hb.GetFabrics() { + fabs = append(fabs, inventory.FabricAttachment{ + Fabric: f.GetFabric(), HCAs: slices.Clone(f.GetHcas()), RateGbps: int(f.GetRateGbps()), + }) + } + s.log.Warn("a node's fabrics changed", "node", nodeID, + "from", n.FabricNames(), "to", fabricNames(fabs)) + n.Fabrics = fabs + } s.inv.Upsert(n) } +func fabricNames(fs []inventory.FabricAttachment) []string { + out := make([]string, len(fs)) + for i, f := range fs { + out[i] = f.Fabric + } + return out +} + // loadFrom is a heartbeat's load as the inventory keeps it. func loadFrom(l *llmgwv1.NodeLoad, at time.Time) *inventory.Load { out := &inventory.Load{ diff --git a/internal/control/fleetmetrics/fleetmetrics.go b/internal/control/fleetmetrics/fleetmetrics.go index 63273b23..1279f302 100644 --- a/internal/control/fleetmetrics/fleetmetrics.go +++ b/internal/control/fleetmetrics/fleetmetrics.go @@ -67,6 +67,10 @@ type Collector struct { nodeMemAvail *prometheus.Desc shardCPU *prometheus.Desc shardMemory *prometheus.Desc + + ibActive *prometheus.Desc + ibRate *prometheus.Desc + ibCounter *prometheus.Desc } // loadFresh is how old a node's load may be and still be reported. A node @@ -147,6 +151,19 @@ func New(src Source) *Collector { shardMemory: prometheus.NewDesc("llmgw_shard_memory_bytes", "Resident memory of an engine process group.", []string{"node", "model", "replica", "shard"}, nil), + + // Every InfiniBand or RoCE port, re-read on each heartbeat. + ibActive: prometheus.NewDesc("llmgw_ib_port_active", + "1 when an InfiniBand or RoCE port can carry traffic. phys_state tells a cable "+ + "unplugged (Polling, Disabled) from a link with no subnet manager (LinkUp).", + []string{"node", "hca", "port", "link_layer", "state", "phys_state"}, nil), + ibRate: prometheus.NewDesc("llmgw_ib_port_rate_gbps", + "A port's negotiated rate. A link that trained below its usual speed is a "+ + "cable or transceiver to look at.", []string{"node", "hca", "port"}, nil), + ibCounter: prometheus.NewDesc("llmgw_ib_port_counter_total", + "A port's cumulative counters, by sysfs name: symbol_error, link_downed and the "+ + "other errors say a link is going bad before a job fails on it; port_xmit_data "+ + "and port_rcv_data are bytes.", []string{"node", "hca", "port", "counter"}, nil), } } @@ -156,6 +173,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) { c.nodeUp, c.nodeGPUs, c.nodeGPUsFree, c.nodeMemory, c.nodeSeen, c.nodeDiskFree, c.nodeDiskSize, c.nodeCPU, c.nodeCPUs, c.nodeLoad, c.nodeMemUsed, c.nodeMemAvail, c.shardCPU, c.shardMemory, + c.ibActive, c.ibRate, c.ibCounter, c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuTemp, c.gpuHealthy, c.gpuNVLink, c.gpuECC, c.replicaState, c.replicaReady, c.replicaIdle, @@ -214,6 +232,22 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) { loads[n.ID] = l } + for _, p := range n.IBPorts { + port := strconv.Itoa(p.Port) + active := 0.0 + if p.Active() { + active = 1 + } + ch <- prometheus.MustNewConstMetric(c.ibActive, prometheus.GaugeValue, active, + n.ID, p.HCA, port, p.LinkLayer, p.State, p.PhysState) + ch <- prometheus.MustNewConstMetric(c.ibRate, prometheus.GaugeValue, float64(p.RateGbps), + n.ID, p.HCA, port) + for name, v := range p.Counters { + ch <- prometheus.MustNewConstMetric(c.ibCounter, prometheus.CounterValue, float64(v), + n.ID, p.HCA, port, name) + } + } + byClass := map[string]int{} for _, g := range n.GPUs { class := g.Class() diff --git a/internal/control/fleetmetrics/fleetmetrics_test.go b/internal/control/fleetmetrics/fleetmetrics_test.go index cfab934b..a468fd77 100644 --- a/internal/control/fleetmetrics/fleetmetrics_test.go +++ b/internal/control/fleetmetrics/fleetmetrics_test.go @@ -176,3 +176,23 @@ func TestAStaleLoadIsAbsentNotRepeated(t *testing.T) { t.Errorf("a stale load was reported:\n%s", out) } } + +func TestEveryIBPortIsReportedWithItsCounters(t *testing.T) { + n := gpuNode("n1", 0, "ib-a") + n.IBPorts = []inventory.IBPort{ + {HCA: "mlx5_0", Port: 1, State: "ACTIVE", PhysState: "LinkUp", RateGbps: 400, LinkLayer: "InfiniBand", + Counters: map[string]uint64{"symbol_error": 12, "port_xmit_data": 4096}}, + {HCA: "mlx5_1", Port: 1, State: "DOWN", PhysState: "Polling", LinkLayer: "InfiniBand"}, + } + out := gather(t, fakeSource{nodes: []inventory.Node{n}}) + for _, want := range []string{ + "llmgw_ib_port_active hca=mlx5_0 link_layer=InfiniBand node=n1 phys_state=LinkUp port=1 state=ACTIVE -> 1", + "llmgw_ib_port_active hca=mlx5_1 link_layer=InfiniBand node=n1 phys_state=Polling port=1 state=DOWN -> 0", + "llmgw_ib_port_rate_gbps hca=mlx5_0 node=n1 port=1 -> 400", + "llmgw_ib_port_counter_total counter=symbol_error hca=mlx5_0 node=n1 port=1", + } { + if !strings.Contains(out, want) { + t.Errorf("missing %q in:\n%s", want, out) + } + } +} diff --git a/internal/control/inventory/inventory.go b/internal/control/inventory/inventory.go index f9ae0fd7..942bd5d8 100644 --- a/internal/control/inventory/inventory.go +++ b/internal/control/inventory/inventory.go @@ -115,8 +115,25 @@ type Node struct { // it. Nil before the first one, and from an agent too old to say. Not // persisted: it describes a moment, and a restored one would be stale. Load *Load `json:"-"` + // IBPorts are the node's InfiniBand and RoCE ports at the last heartbeat, + // with their counters. Not persisted, for the same reason as Load. + IBPorts []IBPort `json:"-"` } +// IBPort is one port's state and cumulative counters. +type IBPort struct { + HCA string + Port int + State string + PhysState string + RateGbps int + LinkLayer string + Counters map[string]uint64 +} + +// Active reports whether the port can carry traffic. +func (p IBPort) Active() bool { return strings.EqualFold(p.State, "ACTIVE") } + // Load is a node's use of its CPU and memory, and each engine's share. type Load struct { At time.Time diff --git a/internal/web/node_page_test.go b/internal/web/node_page_test.go index 607870a2..c9404e23 100644 --- a/internal/web/node_page_test.go +++ b/internal/web/node_page_test.go @@ -28,6 +28,11 @@ func (busySource) Nodes() []adminapi.NodeView { n := fakeSource{}.Nodes()[0] n.Load = &adminapi.LoadView{CPUPercent: 62, CPUs: 8, Load1: 3.25, Load5: 2, Load15: 1, MemUsed: 6 << 30, MemTotal: 8 << 30, MemPercent: 75} + n.IBPorts = []adminapi.IBPortView{ + {HCA: "mlx5_0", Port: 1, State: "ACTIVE", PhysState: "LinkUp", RateGbps: 400, Active: true, + Fabric: "ib-a", TxBytes: 4 << 30}, + {HCA: "mlx5_1", Port: 1, State: "DOWN", PhysState: "Polling", Errors: map[string]uint64{"link_downed": 7}}, + } return []adminapi.NodeView{n} } @@ -84,3 +89,15 @@ func TestANodeNotMeasuredLatelySaysSoRatherThanZero(t *testing.T) { t.Error("without a metrics store the page should say where history comes from") } } + +func TestANodesPageShowsItsFabricPorts(t *testing.T) { + _, body := get(t, nodeMux(busySource{}, nil), "http://x/admin/nodes/llmgw-node01", nil) + for _, want := range []string{ + "

Fabric

", "On ib-a.", "mlx5_0/1", "400 Gb/s", "4.0 GiB", + `DOWN Polling`, "link_downed 7", + } { + if !strings.Contains(body, want) { + t.Errorf("missing %q", want) + } + } +} diff --git a/internal/web/templates/node.html b/internal/web/templates/node.html index 66f44443..0a083ba5 100644 --- a/internal/web/templates/node.html +++ b/internal/web/templates/node.html @@ -31,6 +31,26 @@ {{if .Unhealthy}}

{{join .Unhealthy}}

{{end}} {{end}} +{{if or .Node.IBPorts .Node.Fabrics}} +

Fabric

+

{{if .Node.Fabrics}}On {{join .Node.Fabrics}}.{{else}}On no fabric: no port a declared fabric names can carry traffic, so no multi-node replica is placed here.{{end}}

+{{if .Node.IBPorts}} + +{{range .Node.IBPorts}} + + + + + + + +{{end}}
AdapterStateRateFabricErrors since bootSentReceived
{{.HCA}}/{{.Port}}{{if eq .LinkLayer "Ethernet"}} RoCE{{end}}{{.State}}{{with .PhysState}} {{.}}{{end}}{{if .RateGbps}}{{.RateGbps}} Gb/s{{else}}—{{end}}{{or .Fabric "—"}}{{if .Errors}}{{range $k, $v := .Errors}}{{$k}} {{$v}} {{end}}{{else}}none{{end}}{{if .TxBytes}}{{bytes (u64 .TxBytes)}}{{else}}—{{end}}{{if .RxBytes}}{{bytes (u64 .RxBytes)}}{{else}}—{{end}}
+

Re-read on every heartbeat. A port that goes down takes its fabric off +this node within seconds; one that comes back returns it. Errors are cumulative since +the adapter came up: a number that grows is the one to look at.

+{{end}} +{{end}} +

What runs here

{{if .Engines}} diff --git a/internal/web/web.go b/internal/web/web.go index 44670908..48a33779 100644 --- a/internal/web/web.go +++ b/internal/web/web.go @@ -102,6 +102,7 @@ var funcs = template.FuncMap{ // message on the host will say, and a page that disagrees with the shell // about the size of the same file starts an argument about neither. "bytes": humanBytes, + "u64": func(v uint64) int64 { return int64(min(v, 1<<62)) }, // dict lets one chart template be called with named arguments. Go // templates pass a single value, and four charts differing in three // strings each is four copies of the same markup otherwise.
ModelReplicaStateCPUMemory