Watch InfiniBand ports while the agent runs, not only when it starts

A port's state was read once, at registration: one that went down later
left its node on the fabric, offered multi-node work that would hang on
its first collective, until somebody restarted the agent. The agent now
re-reads every IB and RoCE port on each heartbeat - state, physical
state, rate and the error and data counters - and when the fabrics it
can carry traffic on change, says so once; the control plane replaces
the node's fabrics, which is what placement reads.

The ports are exported as llmgw_ib_port_active (with state and
phys_state, so an unplugged cable reads differently from a missing
subnet manager), llmgw_ib_port_rate_gbps and llmgw_ib_port_counter_total,
and a node's page lists them with the fabric each carries and any error
counter that is not zero.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-29 00:13:54 +02:00
1 parent bf2d171e89
commit 93bd16f00e
15 files changed
+654 -107

No files matched your search

+247 -99
View File
@@ -90,7 +90,7 @@ func (x ShardStatus_State) Number() protoreflect.EnumNumber {
// Deprecated: Use ShardStatus_State.Descriptor instead.
func (ShardStatus_State) EnumDescriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15, 0}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16, 0}
}
// AgentMessage is anything a node tells the control plane.
@@ -1385,6 +1385,103 @@ func (x *ShardLoad) GetMemoryRssBytes() int64 {
return 0
}
// IBPort is one InfiniBand or RoCE port's state and counters.
type IBPort struct {
state protoimpl.MessageState `protogen:"open.v1"`
Hca string `protobuf:"bytes,1,opt,name=hca,proto3" json:"hca,omitempty"`
Port int32 `protobuf:"varint,2,opt,name=port,proto3" json:"port,omitempty"`
// state is ACTIVE, INIT, DOWN...; phys_state tells a cable unplugged
// (Polling, Disabled) from a link up with no subnet manager (LinkUp and INIT).
State string `protobuf:"bytes,3,opt,name=state,proto3" json:"state,omitempty"`
PhysState string `protobuf:"bytes,4,opt,name=phys_state,json=physState,proto3" json:"phys_state,omitempty"`
RateGbps int32 `protobuf:"varint,5,opt,name=rate_gbps,json=rateGbps,proto3" json:"rate_gbps,omitempty"`
// link_layer is InfiniBand or Ethernet (RoCE).
LinkLayer string `protobuf:"bytes,6,opt,name=link_layer,json=linkLayer,proto3" json:"link_layer,omitempty"`
// counters are cumulative, by sysfs name; data counters in bytes.
Counters map[string]uint64 `protobuf:"bytes,7,rep,name=counters,proto3" json:"counters,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"varint,2,opt,name=value"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *IBPort) Reset() {
*x = IBPort{}
mi := &file_llmgw_v1_agent_proto_msgTypes[11]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *IBPort) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*IBPort) ProtoMessage() {}
func (x *IBPort) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[11]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use IBPort.ProtoReflect.Descriptor instead.
func (*IBPort) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11}
}
func (x *IBPort) GetHca() string {
if x != nil {
return x.Hca
}
return ""
}
func (x *IBPort) GetPort() int32 {
if x != nil {
return x.Port
}
return 0
}
func (x *IBPort) GetState() string {
if x != nil {
return x.State
}
return ""
}
func (x *IBPort) GetPhysState() string {
if x != nil {
return x.PhysState
}
return ""
}
func (x *IBPort) GetRateGbps() int32 {
if x != nil {
return x.RateGbps
}
return 0
}
func (x *IBPort) GetLinkLayer() string {
if x != nil {
return x.LinkLayer
}
return ""
}
func (x *IBPort) GetCounters() map[string]uint64 {
if x != nil {
return x.Counters
}
return nil
}
type Heartbeat struct {
state protoimpl.MessageState `protogen:"open.v1"`
NodeId string `protobuf:"bytes,1,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"`
@@ -1424,14 +1521,23 @@ type Heartbeat struct {
// load is how busy the machine is, sampled over the interval since the
// previous heartbeat. Absent on the first one, which has nothing to take a
// difference from, and from an agent on a system without /proc.
Load *NodeLoad `protobuf:"bytes,12,opt,name=load,proto3" json:"load,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
Load *NodeLoad `protobuf:"bytes,12,opt,name=load,proto3" json:"load,omitempty"`
// ib_ports is every InfiniBand or RoCE port as it stands now, re-read on
// each heartbeat: a port that goes down after the agent started used to go
// unnoticed until the agent restarted.
IbPorts []*IBPort `protobuf:"bytes,13,rep,name=ib_ports,json=ibPorts,proto3" json:"ib_ports,omitempty"`
// fabrics is the fabrics this node can now carry traffic on, sent only when
// that changed since registration or the last heartbeat that said so — a
// port going down takes its fabric away, and one coming back returns it.
FabricsChanged bool `protobuf:"varint,14,opt,name=fabrics_changed,json=fabricsChanged,proto3" json:"fabrics_changed,omitempty"`
Fabrics []*FabricAttachment `protobuf:"bytes,15,rep,name=fabrics,proto3" json:"fabrics,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *Heartbeat) Reset() {
*x = Heartbeat{}
mi := &file_llmgw_v1_agent_proto_msgTypes[11]
mi := &file_llmgw_v1_agent_proto_msgTypes[12]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1443,7 +1549,7 @@ func (x *Heartbeat) String() string {
func (*Heartbeat) ProtoMessage() {}
func (x *Heartbeat) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[11]
mi := &file_llmgw_v1_agent_proto_msgTypes[12]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1456,7 +1562,7 @@ func (x *Heartbeat) ProtoReflect() protoreflect.Message {
// Deprecated: Use Heartbeat.ProtoReflect.Descriptor instead.
func (*Heartbeat) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12}
}
func (x *Heartbeat) GetNodeId() string {
@@ -1543,6 +1649,27 @@ func (x *Heartbeat) GetLoad() *NodeLoad {
return nil
}
func (x *Heartbeat) GetIbPorts() []*IBPort {
if x != nil {
return x.IbPorts
}
return nil
}
func (x *Heartbeat) GetFabricsChanged() bool {
if x != nil {
return x.FabricsChanged
}
return false
}
func (x *Heartbeat) GetFabrics() []*FabricAttachment {
if x != nil {
return x.Fabrics
}
return nil
}
// Provision is the software a node should have: which engines, at which
// versions. Nothing more.
//
@@ -1562,7 +1689,7 @@ type Provision struct {
func (x *Provision) Reset() {
*x = Provision{}
mi := &file_llmgw_v1_agent_proto_msgTypes[12]
mi := &file_llmgw_v1_agent_proto_msgTypes[13]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1574,7 +1701,7 @@ func (x *Provision) String() string {
func (*Provision) ProtoMessage() {}
func (x *Provision) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[12]
mi := &file_llmgw_v1_agent_proto_msgTypes[13]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1587,7 +1714,7 @@ func (x *Provision) ProtoReflect() protoreflect.Message {
// Deprecated: Use Provision.ProtoReflect.Descriptor instead.
func (*Provision) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13}
}
func (x *Provision) GetEngines() []*EngineWant {
@@ -1608,7 +1735,7 @@ type EngineWant struct {
func (x *EngineWant) Reset() {
*x = EngineWant{}
mi := &file_llmgw_v1_agent_proto_msgTypes[13]
mi := &file_llmgw_v1_agent_proto_msgTypes[14]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1620,7 +1747,7 @@ func (x *EngineWant) String() string {
func (*EngineWant) ProtoMessage() {}
func (x *EngineWant) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[13]
mi := &file_llmgw_v1_agent_proto_msgTypes[14]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1633,7 +1760,7 @@ func (x *EngineWant) ProtoReflect() protoreflect.Message {
// Deprecated: Use EngineWant.ProtoReflect.Descriptor instead.
func (*EngineWant) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14}
}
func (x *EngineWant) GetEngine() string {
@@ -1666,7 +1793,7 @@ type ProvisionStatus struct {
func (x *ProvisionStatus) Reset() {
*x = ProvisionStatus{}
mi := &file_llmgw_v1_agent_proto_msgTypes[14]
mi := &file_llmgw_v1_agent_proto_msgTypes[15]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1678,7 +1805,7 @@ func (x *ProvisionStatus) String() string {
func (*ProvisionStatus) ProtoMessage() {}
func (x *ProvisionStatus) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[14]
mi := &file_llmgw_v1_agent_proto_msgTypes[15]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1691,7 +1818,7 @@ func (x *ProvisionStatus) ProtoReflect() protoreflect.Message {
// Deprecated: Use ProvisionStatus.ProtoReflect.Descriptor instead.
func (*ProvisionStatus) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15}
}
func (x *ProvisionStatus) GetEngine() string {
@@ -1755,7 +1882,7 @@ type ShardStatus struct {
func (x *ShardStatus) Reset() {
*x = ShardStatus{}
mi := &file_llmgw_v1_agent_proto_msgTypes[15]
mi := &file_llmgw_v1_agent_proto_msgTypes[16]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1767,7 +1894,7 @@ func (x *ShardStatus) String() string {
func (*ShardStatus) ProtoMessage() {}
func (x *ShardStatus) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[15]
mi := &file_llmgw_v1_agent_proto_msgTypes[16]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1780,7 +1907,7 @@ func (x *ShardStatus) ProtoReflect() protoreflect.Message {
// Deprecated: Use ShardStatus.ProtoReflect.Descriptor instead.
func (*ShardStatus) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16}
}
func (x *ShardStatus) GetNodeId() string {
@@ -1854,7 +1981,7 @@ type FetchProgress struct {
func (x *FetchProgress) Reset() {
*x = FetchProgress{}
mi := &file_llmgw_v1_agent_proto_msgTypes[16]
mi := &file_llmgw_v1_agent_proto_msgTypes[17]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1866,7 +1993,7 @@ func (x *FetchProgress) String() string {
func (*FetchProgress) ProtoMessage() {}
func (x *FetchProgress) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[16]
mi := &file_llmgw_v1_agent_proto_msgTypes[17]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1879,7 +2006,7 @@ func (x *FetchProgress) ProtoReflect() protoreflect.Message {
// Deprecated: Use FetchProgress.ProtoReflect.Descriptor instead.
func (*FetchProgress) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{17}
}
func (x *FetchProgress) GetSource() string {
@@ -1935,7 +2062,7 @@ type StartShard struct {
func (x *StartShard) Reset() {
*x = StartShard{}
mi := &file_llmgw_v1_agent_proto_msgTypes[17]
mi := &file_llmgw_v1_agent_proto_msgTypes[18]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1947,7 +2074,7 @@ func (x *StartShard) String() string {
func (*StartShard) ProtoMessage() {}
func (x *StartShard) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[17]
mi := &file_llmgw_v1_agent_proto_msgTypes[18]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1960,7 +2087,7 @@ func (x *StartShard) ProtoReflect() protoreflect.Message {
// Deprecated: Use StartShard.ProtoReflect.Descriptor instead.
func (*StartShard) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{17}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{18}
}
func (x *StartShard) GetReplicaId() string {
@@ -2087,7 +2214,7 @@ type EngineProfile struct {
func (x *EngineProfile) Reset() {
*x = EngineProfile{}
mi := &file_llmgw_v1_agent_proto_msgTypes[18]
mi := &file_llmgw_v1_agent_proto_msgTypes[19]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -2099,7 +2226,7 @@ func (x *EngineProfile) String() string {
func (*EngineProfile) ProtoMessage() {}
func (x *EngineProfile) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[18]
mi := &file_llmgw_v1_agent_proto_msgTypes[19]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -2112,7 +2239,7 @@ func (x *EngineProfile) ProtoReflect() protoreflect.Message {
// Deprecated: Use EngineProfile.ProtoReflect.Descriptor instead.
func (*EngineProfile) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{18}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{19}
}
func (x *EngineProfile) GetName() string {
@@ -2208,7 +2335,7 @@ type RemoveWeights struct {
func (x *RemoveWeights) Reset() {
*x = RemoveWeights{}
mi := &file_llmgw_v1_agent_proto_msgTypes[19]
mi := &file_llmgw_v1_agent_proto_msgTypes[20]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -2220,7 +2347,7 @@ func (x *RemoveWeights) String() string {
func (*RemoveWeights) ProtoMessage() {}
func (x *RemoveWeights) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[19]
mi := &file_llmgw_v1_agent_proto_msgTypes[20]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -2233,7 +2360,7 @@ func (x *RemoveWeights) ProtoReflect() protoreflect.Message {
// Deprecated: Use RemoveWeights.ProtoReflect.Descriptor instead.
func (*RemoveWeights) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{19}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{20}
}
func (x *RemoveWeights) GetPaths() []string {
@@ -2253,7 +2380,7 @@ type StopShard struct {
func (x *StopShard) Reset() {
*x = StopShard{}
mi := &file_llmgw_v1_agent_proto_msgTypes[20]
mi := &file_llmgw_v1_agent_proto_msgTypes[21]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -2265,7 +2392,7 @@ func (x *StopShard) String() string {
func (*StopShard) ProtoMessage() {}
func (x *StopShard) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[20]
mi := &file_llmgw_v1_agent_proto_msgTypes[21]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -2278,7 +2405,7 @@ func (x *StopShard) ProtoReflect() protoreflect.Message {
// Deprecated: Use StopShard.ProtoReflect.Descriptor instead.
func (*StopShard) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{20}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{21}
}
func (x *StopShard) GetShardId() string {
@@ -2298,7 +2425,7 @@ type DrainShard struct {
func (x *DrainShard) Reset() {
*x = DrainShard{}
mi := &file_llmgw_v1_agent_proto_msgTypes[21]
mi := &file_llmgw_v1_agent_proto_msgTypes[22]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -2310,7 +2437,7 @@ func (x *DrainShard) String() string {
func (*DrainShard) ProtoMessage() {}
func (x *DrainShard) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[21]
mi := &file_llmgw_v1_agent_proto_msgTypes[22]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -2323,7 +2450,7 @@ func (x *DrainShard) ProtoReflect() protoreflect.Message {
// Deprecated: Use DrainShard.ProtoReflect.Descriptor instead.
func (*DrainShard) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{21}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{22}
}
func (x *DrainShard) GetShardId() string {
@@ -2345,7 +2472,7 @@ type Ack struct {
func (x *Ack) Reset() {
*x = Ack{}
mi := &file_llmgw_v1_agent_proto_msgTypes[22]
mi := &file_llmgw_v1_agent_proto_msgTypes[23]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -2357,7 +2484,7 @@ func (x *Ack) String() string {
func (*Ack) ProtoMessage() {}
func (x *Ack) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[22]
mi := &file_llmgw_v1_agent_proto_msgTypes[23]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -2370,7 +2497,7 @@ func (x *Ack) ProtoReflect() protoreflect.Message {
// Deprecated: Use Ack.ProtoReflect.Descriptor instead.
func (*Ack) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{22}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{23}
}
func (x *Ack) GetMessageId() string {
@@ -2414,7 +2541,7 @@ type LogsRequest struct {
func (x *LogsRequest) Reset() {
*x = LogsRequest{}
mi := &file_llmgw_v1_agent_proto_msgTypes[23]
mi := &file_llmgw_v1_agent_proto_msgTypes[24]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -2426,7 +2553,7 @@ func (x *LogsRequest) String() string {
func (*LogsRequest) ProtoMessage() {}
func (x *LogsRequest) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[23]
mi := &file_llmgw_v1_agent_proto_msgTypes[24]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -2439,7 +2566,7 @@ func (x *LogsRequest) ProtoReflect() protoreflect.Message {
// Deprecated: Use LogsRequest.ProtoReflect.Descriptor instead.
func (*LogsRequest) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{23}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{24}
}
func (x *LogsRequest) GetRequestId() string {
@@ -2479,7 +2606,7 @@ type LogsReply struct {
func (x *LogsReply) Reset() {
*x = LogsReply{}
mi := &file_llmgw_v1_agent_proto_msgTypes[24]
mi := &file_llmgw_v1_agent_proto_msgTypes[25]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -2491,7 +2618,7 @@ func (x *LogsReply) String() string {
func (*LogsReply) ProtoMessage() {}
func (x *LogsReply) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[24]
mi := &file_llmgw_v1_agent_proto_msgTypes[25]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -2504,7 +2631,7 @@ func (x *LogsReply) ProtoReflect() protoreflect.Message {
// Deprecated: Use LogsReply.ProtoReflect.Descriptor instead.
func (*LogsReply) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{24}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{25}
}
func (x *LogsReply) GetRequestId() string {
@@ -2660,7 +2787,20 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\tShardLoad\x12\x19\n" +
"\bshard_id\x18\x01 \x01(\tR\ashardId\x12\x1b\n" +
"\tcpu_cores\x18\x02 \x01(\x01R\bcpuCores\x12(\n" +
"\x10memory_rss_bytes\x18\x03 \x01(\x03R\x0ememoryRssBytes\"\xf0\x04\n" +
"\x10memory_rss_bytes\x18\x03 \x01(\x03R\x0ememoryRssBytes\"\x98\x02\n" +
"\x06IBPort\x12\x10\n" +
"\x03hca\x18\x01 \x01(\tR\x03hca\x12\x12\n" +
"\x04port\x18\x02 \x01(\x05R\x04port\x12\x14\n" +
"\x05state\x18\x03 \x01(\tR\x05state\x12\x1d\n" +
"\n" +
"phys_state\x18\x04 \x01(\tR\tphysState\x12\x1b\n" +
"\trate_gbps\x18\x05 \x01(\x05R\brateGbps\x12\x1d\n" +
"\n" +
"link_layer\x18\x06 \x01(\tR\tlinkLayer\x12:\n" +
"\bcounters\x18\a \x03(\v2\x1e.llmgw.v1.IBPort.CountersEntryR\bcounters\x1a;\n" +
"\rCountersEntry\x12\x10\n" +
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
"\x05value\x18\x02 \x01(\x04R\x05value:\x028\x01\"\xfc\x05\n" +
"\tHeartbeat\x12\x17\n" +
"\anode_id\x18\x01 \x01(\tR\x06nodeId\x12+\n" +
"\x12gpu_memory_used_mb\x18\x02 \x03(\x05R\x0fgpuMemoryUsedMb\x12!\n" +
@@ -2674,7 +2814,10 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\x11provision_changed\x18\n" +
" \x01(\bR\x10provisionChanged\x127\n" +
"\tprovision\x18\v \x03(\v2\x19.llmgw.v1.ProvisionStatusR\tprovision\x12&\n" +
"\x04load\x18\f \x01(\v2\x12.llmgw.v1.NodeLoadR\x04load\x1aA\n" +
"\x04load\x18\f \x01(\v2\x12.llmgw.v1.NodeLoadR\x04load\x12+\n" +
"\bib_ports\x18\r \x03(\v2\x10.llmgw.v1.IBPortR\aibPorts\x12'\n" +
"\x0ffabrics_changed\x18\x0e \x01(\bR\x0efabricsChanged\x124\n" +
"\afabrics\x18\x0f \x03(\v2\x1a.llmgw.v1.FabricAttachmentR\afabrics\x1aA\n" +
"\x13EngineVersionsEntry\x12\x10\n" +
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\";\n" +
@@ -2797,7 +2940,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte {
}
var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 30)
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 32)
var file_llmgw_v1_agent_proto_goTypes = []any{
(ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State
(*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage
@@ -2811,66 +2954,71 @@ var file_llmgw_v1_agent_proto_goTypes = []any{
(*LocalWeights)(nil), // 9: llmgw.v1.LocalWeights
(*NodeLoad)(nil), // 10: llmgw.v1.NodeLoad
(*ShardLoad)(nil), // 11: llmgw.v1.ShardLoad
(*Heartbeat)(nil), // 12: llmgw.v1.Heartbeat
(*Provision)(nil), // 13: llmgw.v1.Provision
(*EngineWant)(nil), // 14: llmgw.v1.EngineWant
(*ProvisionStatus)(nil), // 15: llmgw.v1.ProvisionStatus
(*ShardStatus)(nil), // 16: llmgw.v1.ShardStatus
(*FetchProgress)(nil), // 17: llmgw.v1.FetchProgress
(*StartShard)(nil), // 18: llmgw.v1.StartShard
(*EngineProfile)(nil), // 19: llmgw.v1.EngineProfile
(*RemoveWeights)(nil), // 20: llmgw.v1.RemoveWeights
(*StopShard)(nil), // 21: llmgw.v1.StopShard
(*DrainShard)(nil), // 22: llmgw.v1.DrainShard
(*Ack)(nil), // 23: llmgw.v1.Ack
(*LogsRequest)(nil), // 24: llmgw.v1.LogsRequest
(*LogsReply)(nil), // 25: llmgw.v1.LogsReply
nil, // 26: llmgw.v1.Register.LabelsEntry
nil, // 27: llmgw.v1.Register.EngineVersionsEntry
nil, // 28: llmgw.v1.GPU.NicPathsEntry
nil, // 29: llmgw.v1.Heartbeat.EngineVersionsEntry
nil, // 30: llmgw.v1.EngineProfile.EnvEntry
(*IBPort)(nil), // 12: llmgw.v1.IBPort
(*Heartbeat)(nil), // 13: llmgw.v1.Heartbeat
(*Provision)(nil), // 14: llmgw.v1.Provision
(*EngineWant)(nil), // 15: llmgw.v1.EngineWant
(*ProvisionStatus)(nil), // 16: llmgw.v1.ProvisionStatus
(*ShardStatus)(nil), // 17: llmgw.v1.ShardStatus
(*FetchProgress)(nil), // 18: llmgw.v1.FetchProgress
(*StartShard)(nil), // 19: llmgw.v1.StartShard
(*EngineProfile)(nil), // 20: llmgw.v1.EngineProfile
(*RemoveWeights)(nil), // 21: llmgw.v1.RemoveWeights
(*StopShard)(nil), // 22: llmgw.v1.StopShard
(*DrainShard)(nil), // 23: llmgw.v1.DrainShard
(*Ack)(nil), // 24: llmgw.v1.Ack
(*LogsRequest)(nil), // 25: llmgw.v1.LogsRequest
(*LogsReply)(nil), // 26: llmgw.v1.LogsReply
nil, // 27: llmgw.v1.Register.LabelsEntry
nil, // 28: llmgw.v1.Register.EngineVersionsEntry
nil, // 29: llmgw.v1.GPU.NicPathsEntry
nil, // 30: llmgw.v1.IBPort.CountersEntry
nil, // 31: llmgw.v1.Heartbeat.EngineVersionsEntry
nil, // 32: llmgw.v1.EngineProfile.EnvEntry
}
var file_llmgw_v1_agent_proto_depIdxs = []int32{
3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register
12, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat
16, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus
25, // 3: llmgw.v1.AgentMessage.logs:type_name -> llmgw.v1.LogsReply
18, // 4: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard
21, // 5: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard
22, // 6: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard
23, // 7: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack
20, // 8: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights
13, // 9: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision
24, // 10: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest
26, // 11: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
13, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat
17, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus
26, // 3: llmgw.v1.AgentMessage.logs:type_name -> llmgw.v1.LogsReply
19, // 4: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard
22, // 5: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard
23, // 6: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard
24, // 7: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack
21, // 8: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights
14, // 9: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision
25, // 10: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest
27, // 11: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
7, // 12: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU
8, // 13: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment
27, // 14: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
28, // 14: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
6, // 15: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard
9, // 16: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights
5, // 17: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage
4, // 18: llmgw.v1.Register.host:type_name -> llmgw.v1.Host
0, // 19: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
28, // 20: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry
29, // 20: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry
11, // 21: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad
5, // 22: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
9, // 23: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
29, // 24: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
15, // 25: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus
10, // 26: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad
14, // 27: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant
0, // 28: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
17, // 29: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
19, // 30: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
30, // 31: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
1, // 32: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
2, // 33: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
33, // [33:34] is the sub-list for method output_type
32, // [32:33] is the sub-list for method input_type
32, // [32:32] is the sub-list for extension type_name
32, // [32:32] is the sub-list for extension extendee
0, // [0:32] is the sub-list for field type_name
30, // 22: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry
5, // 23: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
9, // 24: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
31, // 25: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
16, // 26: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus
10, // 27: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad
12, // 28: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort
8, // 29: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment
15, // 30: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant
0, // 31: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
18, // 32: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
20, // 33: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
32, // 34: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
1, // 35: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
2, // 36: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
36, // [36:37] is the sub-list for method output_type
35, // [35:36] is the sub-list for method input_type
35, // [35:35] is the sub-list for extension type_name
35, // [35:35] is the sub-list for extension extendee
0, // [0:35] is the sub-list for field type_name
}
func init() { file_llmgw_v1_agent_proto_init() }
@@ -2899,7 +3047,7 @@ func file_llmgw_v1_agent_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)),
NumEnums: 1,
NumMessages: 30,
NumMessages: 32,
NumExtensions: 0,
NumServices: 1,
},
+25
View File
@@ -264,6 +264,21 @@ message ShardLoad {
int64 memory_rss_bytes = 3;
}
// IBPort is one InfiniBand or RoCE port's state and counters.
message IBPort {
string hca = 1;
int32 port = 2;
// state is ACTIVE, INIT, DOWN...; phys_state tells a cable unplugged
// (Polling, Disabled) from a link up with no subnet manager (LinkUp and INIT).
string state = 3;
string phys_state = 4;
int32 rate_gbps = 5;
// link_layer is InfiniBand or Ethernet (RoCE).
string link_layer = 6;
// counters are cumulative, by sysfs name; data counters in bytes.
map<string, uint64> counters = 7;
}
message Heartbeat {
string node_id = 1;
// gpu_memory_used_mb, indexed by GPU, catches a foreign process appearing on
@@ -309,6 +324,16 @@ message Heartbeat {
// previous heartbeat. Absent on the first one, which has nothing to take a
// difference from, and from an agent on a system without /proc.
NodeLoad load = 12;
// ib_ports is every InfiniBand or RoCE port as it stands now, re-read on
// each heartbeat: a port that goes down after the agent started used to go
// unnoticed until the agent restarted.
repeated IBPort ib_ports = 13;
// fabrics is the fabrics this node can now carry traffic on, sent only when
// that changed since registration or the last heartbeat that said so — a
// port going down takes its fabric away, and one coming back returns it.
bool fabrics_changed = 14;
repeated FabricAttachment fabrics = 15;
}
// Provision is the software a node should have: which engines, at which
+88 -7
View File
@@ -352,6 +352,8 @@ type Agent struct {
// failover the old stream's heartbeat loop can still be running when the
// new one's starts.
lastWeights string
// fabricsSent is what the control plane was last told about fabrics.
fabricsSent string
// load measures the machine between heartbeats.
load *discovery.LoadSampler
@@ -574,21 +576,36 @@ func (a *Agent) settleAdopted(sh *supervisor.Shard, o supervisor.Orphan) {
// machine where both halves are visible, rather than inferred later from a
// replica that will not converge.
func (a *Agent) usableFabrics(inv discovery.Inventory) []inventory.FabricAttachment {
out := a.checkFabrics(inv.HCAs, true)
a.mu.Lock()
a.fabricsSent = fabricsFingerprint(out)
a.mu.Unlock()
return out
}
// checkFabrics is the declared fabrics this node can carry traffic on, from
// the adapters as they are now. Loud logs every problem; the heartbeat checks
// quietly and speaks only when the answer changes.
func (a *Agent) checkFabrics(hcas []discovery.HCA, loud bool) []inventory.FabricAttachment {
declared := map[string][]string{}
for _, f := range a.cfg.Fabrics {
declared[f.Fabric] = f.HCAs
}
var out []inventory.FabricAttachment
for _, c := range discovery.CheckFabrics(declared, inv.HCAs) {
for _, p := range c.Problems {
a.log.Error("a declared fabric adapter cannot carry traffic",
"fabric", c.Fabric, "problem", p)
for _, c := range discovery.CheckFabrics(declared, hcas) {
if loud {
for _, p := range c.Problems {
a.log.Error("a declared fabric adapter cannot carry traffic",
"fabric", c.Fabric, "problem", p)
}
}
if !c.OK() {
a.log.Error("fabric not advertised; no declared adapter on it can carry traffic",
"fabric", c.Fabric,
"consequence", "no multi-node replica will be placed here")
if loud {
a.log.Error("fabric not advertised; no declared adapter on it can carry traffic",
"fabric", c.Fabric,
"consequence", "no multi-node replica will be placed here")
}
continue
}
out = append(out, inventory.FabricAttachment{
@@ -598,6 +615,69 @@ func (a *Agent) usableFabrics(inv discovery.Inventory) []inventory.FabricAttachm
return out
}
// fabricsFingerprint is enough to notice a fabric, an adapter or a rate
// changing.
func fabricsFingerprint(fs []inventory.FabricAttachment) string {
var b strings.Builder
for _, f := range fs {
fmt.Fprintf(&b, "%s:%s:%d;", f.Fabric, strings.Join(f.HCAs, ","), f.RateGbps)
}
return b.String()
}
// sampleIB re-reads the adapters for the heartbeat: every port's state and
// counters, and the fabrics again when they no longer match what the control
// plane was told.
//
// Every heartbeat, because it is a few small files per port, and because a
// port that went down used to be invisible until the agent restarted — the
// node went on being offered multi-node work that would hang on its first
// collective.
func (a *Agent) sampleIB(hb *llmgwv1.Heartbeat) {
a.mu.Lock()
had := len(a.inv.HCAs) > 0
a.mu.Unlock()
if !had && len(a.cfg.Fabrics) == 0 {
return
}
hcas, err := discovery.DiscoverIB(a.cfg.Discovery.IBRoot)
if err != nil {
return
}
for _, h := range hcas {
for _, p := range h.Ports {
hb.IbPorts = append(hb.IbPorts, &llmgwv1.IBPort{
Hca: h.Name, Port: int32(p.Num), State: p.State, PhysState: p.PhysState,
RateGbps: int32(p.RateGbps), LinkLayer: string(p.LinkLayer), Counters: p.Counters,
})
}
}
a.mu.Lock()
a.inv.HCAs = hcas // a reconnect registers what is true now
a.mu.Unlock()
if len(a.cfg.Fabrics) == 0 {
return
}
now := a.checkFabrics(hcas, false)
fp := fabricsFingerprint(now)
a.mu.Lock()
changed := fp != a.fabricsSent
a.fabricsSent = fp
a.mu.Unlock()
if !changed {
return
}
// Said once, with the reasons, the moment it happens.
a.checkFabrics(hcas, true)
a.log.Warn("the fabrics this node can carry traffic on changed", "now", fp)
hb.FabricsChanged = true
for _, f := range now {
hb.Fabrics = append(hb.Fabrics, &llmgwv1.FabricAttachment{
Fabric: f.Fabric, Hcas: f.HCAs, RateGbps: int32(f.RateGbps),
})
}
}
// weights builds the fetcher lazily, so an agent that never starts a shard
// never needs a cache directory.
func (a *Agent) weights() *weights.Fetcher {
@@ -887,6 +967,7 @@ func (a *Agent) heartbeat() *llmgwv1.AgentMessage {
}
}
hb.Load = a.sampleLoad()
a.sampleIB(hb)
return &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Heartbeat{Heartbeat: hb}}
}
+31
View File
@@ -756,3 +756,34 @@ func TestAFabricRateIsTheSlowestActiveLink(t *testing.T) {
}
}
}
func TestAPortGoingDownIsToldToTheControlPlaneOnce(t *testing.T) {
// Checked only at registration, a port going down went unnoticed until
// the agent restarted, and the node went on being offered multi-node
// work that would hang on its first collective.
root := ibTreeWithActivePorts(t, "mlx5_0")
h := start(t, func(c *Config) {
c.Discovery.IBRoot = root
c.Fabrics = []inventory.FabricAttachment{{Fabric: "ib-a", HCAs: []string{"mlx5_0"}}}
})
if n, _ := h.inv.Get("n1"); !n.HasFabric("ib-a") {
t.Fatalf("the fabric should be advertised at registration: %+v", n.Fabrics)
}
hb := h.ag.heartbeat().GetHeartbeat()
if hb.GetFabricsChanged() || len(hb.GetIbPorts()) != 1 || hb.GetIbPorts()[0].GetState() != "ACTIVE" {
t.Fatalf("a quiet heartbeat should carry the port and no change: %+v", hb)
}
if err := os.WriteFile(filepath.Join(root, "mlx5_0", "ports", "1", "state"), []byte("1: DOWN\n"), 0o644); err != nil {
t.Fatal(err)
}
hb = h.ag.heartbeat().GetHeartbeat()
if !hb.GetFabricsChanged() || len(hb.GetFabrics()) != 0 {
t.Errorf("the port going down should withdraw the fabric: changed=%v fabrics=%v",
hb.GetFabricsChanged(), hb.GetFabrics())
}
if hb = h.ag.heartbeat().GetHeartbeat(); hb.GetFabricsChanged() {
t.Error("the change should be sent once, not on every heartbeat")
}
}
+32
View File
@@ -42,8 +42,26 @@ type Port struct {
GIDPrefix string `json:"gid_prefix"`
LID int `json:"lid"`
SMLID int `json:"sm_lid"`
// Counters are the port's error and traffic counters, by their sysfs
// names, as far as the driver has them. The data counters are converted
// to bytes; sysfs counts them in four-byte words.
Counters map[string]uint64 `json:"counters,omitempty"`
}
// PortCounters are the counters worth watching, from each port's counters/
// directory. The errors are the ones that say a cable, a transceiver or a
// switch port is going bad before a job fails on it; the data counters say
// whether the fabric is carrying anything at all.
var PortCounters = []string{
"symbol_error", "link_error_recovery", "link_downed", "port_rcv_errors",
"port_rcv_remote_physical_errors", "port_xmit_discards",
"local_link_integrity_errors", "excessive_buffer_overrun_errors",
"port_xmit_data", "port_rcv_data",
}
// dataCounter reports whether a counter counts data in four-byte words.
func dataCounter(name string) bool { return name == "port_xmit_data" || name == "port_rcv_data" }
// Active reports whether the port can carry traffic.
func (p Port) Active() bool { return strings.EqualFold(p.State, "ACTIVE") }
@@ -138,6 +156,20 @@ func readPort(dir string, num int) Port {
LID: parseMaybeHex(readTrimmed(filepath.Join(dir, "lid"))),
SMLID: parseMaybeHex(readTrimmed(filepath.Join(dir, "sm_lid"))),
}
for _, name := range PortCounters {
raw := readTrimmed(filepath.Join(dir, "counters", name))
v, err := strconv.ParseUint(raw, 10, 64)
if err != nil {
continue // not every driver has every counter
}
if dataCounter(name) {
v *= 4
}
if p.Counters == nil {
p.Counters = map[string]uint64{}
}
p.Counters[name] = v
}
return p
}
+18
View File
@@ -31,6 +31,9 @@ func fakeSysfs(t *testing.T) string {
write("ib/mlx5_0/ports/1/gids/0", "FEC0:0000:0000:000A:0002:C903:00A0:1234")
write("ib/mlx5_0/ports/1/lid", "0x1a")
write("ib/mlx5_0/ports/1/sm_lid", "1")
write("ib/mlx5_0/ports/1/counters/symbol_error", "3")
write("ib/mlx5_0/ports/1/counters/link_downed", "1")
write("ib/mlx5_0/ports/1/counters/port_xmit_data", "1000")
// mlx5_1: two ports, the second one down.
write("ib/mlx5_1/ports/1/state", "4: ACTIVE")
@@ -214,3 +217,18 @@ func TestParseHelpers(t *testing.T) {
}
})
}
func TestAPortsCountersAreRead(t *testing.T) {
hcas, err := DiscoverIB(fakeSysfs(t))
if err != nil {
t.Fatal(err)
}
c := hcas[0].Ports[0].Counters
// Data in bytes: sysfs counts four-byte words.
if c["symbol_error"] != 3 || c["link_downed"] != 1 || c["port_xmit_data"] != 4000 {
t.Errorf("counters = %v", c)
}
if _, ok := c["port_rcv_errors"]; ok {
t.Error("a counter the driver does not have should be absent, not zero")
}
}
+39
View File
@@ -119,6 +119,25 @@ type NodeView struct {
// Load is how busy the machine is, absent when its agent has not
// measured it lately — which is unknown, not idle.
Load *LoadView `json:"load,omitempty"`
// IBPorts are the node's InfiniBand and RoCE ports at the last heartbeat.
IBPorts []IBPortView `json:"ib_ports,omitempty"`
}
// IBPortView is one port as a page shows it.
type IBPortView struct {
HCA string `json:"hca"`
Port int `json:"port"`
State string `json:"state"`
PhysState string `json:"phys_state,omitempty"`
RateGbps int `json:"rate_gbps"`
LinkLayer string `json:"link_layer,omitempty"`
Active bool `json:"active"`
// Fabric is the declared fabric this port's adapter carries, if any.
Fabric string `json:"fabric,omitempty"`
// Errors are the error counters that are not zero, by name.
Errors map[string]uint64 `json:"errors,omitempty"`
TxBytes uint64 `json:"tx_bytes,omitempty"`
RxBytes uint64 `json:"rx_bytes,omitempty"`
}
// LoadView is a node's CPU and memory in use, at its last heartbeat.
@@ -308,6 +327,26 @@ func (a *API) Nodes() []NodeView {
v.LastSeenAgo = time.Since(n.LastSeen).Round(time.Second).String()
}
v.Load = loadView(n.Load, time.Now())
fabricOf := map[string]string{}
for _, f := range n.Fabrics {
for _, h := range f.HCAs {
fabricOf[h] = f.Fabric
}
}
for _, p := range n.IBPorts {
pv := IBPortView{HCA: p.HCA, Port: p.Port, State: p.State, PhysState: p.PhysState,
RateGbps: p.RateGbps, LinkLayer: p.LinkLayer, Active: p.Active(), Fabric: fabricOf[p.HCA],
TxBytes: p.Counters["port_xmit_data"], RxBytes: p.Counters["port_rcv_data"]}
for name, c := range p.Counters {
if c > 0 && name != "port_xmit_data" && name != "port_rcv_data" {
if pv.Errors == nil {
pv.Errors = map[string]uint64{}
}
pv.Errors[name] = c
}
}
v.IBPorts = append(v.IBPorts, pv)
}
out = append(out, v)
}
return out
@@ -210,3 +210,35 @@ func TestAHeartbeatCarriesHowBusyTheNodeIs(t *testing.T) {
t.Error("a heartbeat without load erased it")
}
}
func TestAPortGoingDownTakesItsFabricAway(t *testing.T) {
// Checked only at registration, a port that went down left the node on
// its fabric until the agent restarted, still offered multi-node work.
s := newTestService()
nodeWith(s, inventory.Node{ID: "n1", State: inventory.Ready,
Fabrics: []inventory.FabricAttachment{{Fabric: "ib-a", HCAs: []string{"mlx5_0"}, RateGbps: 400}}})
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1", FabricsChanged: true,
IbPorts: []*llmgwv1.IBPort{{Hca: "mlx5_0", Port: 1, State: "DOWN", PhysState: "Polling",
Counters: map[string]uint64{"link_downed": 4}}}})
n, _ := s.inv.Get("n1")
if len(n.Fabrics) != 0 {
t.Errorf("the fabric should be gone: %+v", n.Fabrics)
}
if len(n.IBPorts) != 1 || n.IBPorts[0].Active() || n.IBPorts[0].Counters["link_downed"] != 4 {
t.Errorf("ports = %+v", n.IBPorts)
}
// Back up: the fabric returns.
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1", FabricsChanged: true,
Fabrics: []*llmgwv1.FabricAttachment{{Fabric: "ib-a", Hcas: []string{"mlx5_0"}, RateGbps: 400}}})
if n, _ := s.inv.Get("n1"); !n.HasFabric("ib-a") {
t.Error("the fabric should be back")
}
// A heartbeat that does not say the fabrics changed leaves them alone.
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1", IbPorts: []*llmgwv1.IBPort{{Hca: "mlx5_0", Port: 1, State: "ACTIVE"}}})
if n, _ := s.inv.Get("n1"); !n.HasFabric("ib-a") {
t.Error("a heartbeat without a fabric change emptied the fabrics")
}
}
+33 -1
View File
@@ -309,7 +309,8 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
memMB := int(hb.GetMemoryTotalMb())
if len(used) == 0 && len(problems) == 0 && len(storage) == 0 &&
!hb.GetWeightsChanged() && memMB == 0 && !hb.GetEnginesChanged() &&
!hb.GetProvisionChanged() && hb.GetLoad() == nil {
!hb.GetProvisionChanged() && hb.GetLoad() == nil && len(hb.GetIbPorts()) == 0 &&
!hb.GetFabricsChanged() {
return
}
@@ -403,9 +404,40 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
if l := hb.GetLoad(); l != nil {
n.Load = loadFrom(l, time.Now())
}
if ports := hb.GetIbPorts(); len(ports) > 0 {
n.IBPorts = n.IBPorts[:0]
for _, p := range ports {
n.IBPorts = append(n.IBPorts, inventory.IBPort{
HCA: p.GetHca(), Port: int(p.GetPort()), State: p.GetState(),
PhysState: p.GetPhysState(), RateGbps: int(p.GetRateGbps()),
LinkLayer: p.GetLinkLayer(), Counters: maps.Clone(p.GetCounters()),
})
}
}
// A port went down, or came back. Placement reads the fabrics, so this is
// what stops a node being offered multi-node work it can no longer carry.
if hb.GetFabricsChanged() {
var fabs []inventory.FabricAttachment
for _, f := range hb.GetFabrics() {
fabs = append(fabs, inventory.FabricAttachment{
Fabric: f.GetFabric(), HCAs: slices.Clone(f.GetHcas()), RateGbps: int(f.GetRateGbps()),
})
}
s.log.Warn("a node's fabrics changed", "node", nodeID,
"from", n.FabricNames(), "to", fabricNames(fabs))
n.Fabrics = fabs
}
s.inv.Upsert(n)
}
func fabricNames(fs []inventory.FabricAttachment) []string {
out := make([]string, len(fs))
for i, f := range fs {
out[i] = f.Fabric
}
return out
}
// loadFrom is a heartbeat's load as the inventory keeps it.
func loadFrom(l *llmgwv1.NodeLoad, at time.Time) *inventory.Load {
out := &inventory.Load{
@@ -67,6 +67,10 @@ type Collector struct {
nodeMemAvail *prometheus.Desc
shardCPU *prometheus.Desc
shardMemory *prometheus.Desc
ibActive *prometheus.Desc
ibRate *prometheus.Desc
ibCounter *prometheus.Desc
}
// loadFresh is how old a node's load may be and still be reported. A node
@@ -147,6 +151,19 @@ func New(src Source) *Collector {
shardMemory: prometheus.NewDesc("llmgw_shard_memory_bytes",
"Resident memory of an engine process group.",
[]string{"node", "model", "replica", "shard"}, nil),
// Every InfiniBand or RoCE port, re-read on each heartbeat.
ibActive: prometheus.NewDesc("llmgw_ib_port_active",
"1 when an InfiniBand or RoCE port can carry traffic. phys_state tells a cable "+
"unplugged (Polling, Disabled) from a link with no subnet manager (LinkUp).",
[]string{"node", "hca", "port", "link_layer", "state", "phys_state"}, nil),
ibRate: prometheus.NewDesc("llmgw_ib_port_rate_gbps",
"A port's negotiated rate. A link that trained below its usual speed is a "+
"cable or transceiver to look at.", []string{"node", "hca", "port"}, nil),
ibCounter: prometheus.NewDesc("llmgw_ib_port_counter_total",
"A port's cumulative counters, by sysfs name: symbol_error, link_downed and the "+
"other errors say a link is going bad before a job fails on it; port_xmit_data "+
"and port_rcv_data are bytes.", []string{"node", "hca", "port", "counter"}, nil),
}
}
@@ -156,6 +173,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
c.nodeUp, c.nodeGPUs, c.nodeGPUsFree, c.nodeMemory, c.nodeSeen,
c.nodeDiskFree, c.nodeDiskSize,
c.nodeCPU, c.nodeCPUs, c.nodeLoad, c.nodeMemUsed, c.nodeMemAvail, c.shardCPU, c.shardMemory,
c.ibActive, c.ibRate, c.ibCounter,
c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuTemp, c.gpuHealthy,
c.gpuNVLink, c.gpuECC,
c.replicaState, c.replicaReady, c.replicaIdle,
@@ -214,6 +232,22 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) {
loads[n.ID] = l
}
for _, p := range n.IBPorts {
port := strconv.Itoa(p.Port)
active := 0.0
if p.Active() {
active = 1
}
ch <- prometheus.MustNewConstMetric(c.ibActive, prometheus.GaugeValue, active,
n.ID, p.HCA, port, p.LinkLayer, p.State, p.PhysState)
ch <- prometheus.MustNewConstMetric(c.ibRate, prometheus.GaugeValue, float64(p.RateGbps),
n.ID, p.HCA, port)
for name, v := range p.Counters {
ch <- prometheus.MustNewConstMetric(c.ibCounter, prometheus.CounterValue, float64(v),
n.ID, p.HCA, port, name)
}
}
byClass := map[string]int{}
for _, g := range n.GPUs {
class := g.Class()
@@ -176,3 +176,23 @@ func TestAStaleLoadIsAbsentNotRepeated(t *testing.T) {
t.Errorf("a stale load was reported:\n%s", out)
}
}
func TestEveryIBPortIsReportedWithItsCounters(t *testing.T) {
n := gpuNode("n1", 0, "ib-a")
n.IBPorts = []inventory.IBPort{
{HCA: "mlx5_0", Port: 1, State: "ACTIVE", PhysState: "LinkUp", RateGbps: 400, LinkLayer: "InfiniBand",
Counters: map[string]uint64{"symbol_error": 12, "port_xmit_data": 4096}},
{HCA: "mlx5_1", Port: 1, State: "DOWN", PhysState: "Polling", LinkLayer: "InfiniBand"},
}
out := gather(t, fakeSource{nodes: []inventory.Node{n}})
for _, want := range []string{
"llmgw_ib_port_active hca=mlx5_0 link_layer=InfiniBand node=n1 phys_state=LinkUp port=1 state=ACTIVE -> 1",
"llmgw_ib_port_active hca=mlx5_1 link_layer=InfiniBand node=n1 phys_state=Polling port=1 state=DOWN -> 0",
"llmgw_ib_port_rate_gbps hca=mlx5_0 node=n1 port=1 -> 400",
"llmgw_ib_port_counter_total counter=symbol_error hca=mlx5_0 node=n1 port=1",
} {
if !strings.Contains(out, want) {
t.Errorf("missing %q in:\n%s", want, out)
}
}
}
+17
View File
@@ -115,8 +115,25 @@ type Node struct {
// it. Nil before the first one, and from an agent too old to say. Not
// persisted: it describes a moment, and a restored one would be stale.
Load *Load `json:"-"`
// IBPorts are the node's InfiniBand and RoCE ports at the last heartbeat,
// with their counters. Not persisted, for the same reason as Load.
IBPorts []IBPort `json:"-"`
}
// IBPort is one port's state and cumulative counters.
type IBPort struct {
HCA string
Port int
State string
PhysState string
RateGbps int
LinkLayer string
Counters map[string]uint64
}
// Active reports whether the port can carry traffic.
func (p IBPort) Active() bool { return strings.EqualFold(p.State, "ACTIVE") }
// Load is a node's use of its CPU and memory, and each engine's share.
type Load struct {
At time.Time
+17
View File
@@ -28,6 +28,11 @@ func (busySource) Nodes() []adminapi.NodeView {
n := fakeSource{}.Nodes()[0]
n.Load = &adminapi.LoadView{CPUPercent: 62, CPUs: 8, Load1: 3.25, Load5: 2, Load15: 1,
MemUsed: 6 << 30, MemTotal: 8 << 30, MemPercent: 75}
n.IBPorts = []adminapi.IBPortView{
{HCA: "mlx5_0", Port: 1, State: "ACTIVE", PhysState: "LinkUp", RateGbps: 400, Active: true,
Fabric: "ib-a", TxBytes: 4 << 30},
{HCA: "mlx5_1", Port: 1, State: "DOWN", PhysState: "Polling", Errors: map[string]uint64{"link_downed": 7}},
}
return []adminapi.NodeView{n}
}
@@ -84,3 +89,15 @@ func TestANodeNotMeasuredLatelySaysSoRatherThanZero(t *testing.T) {
t.Error("without a metrics store the page should say where history comes from")
}
}
func TestANodesPageShowsItsFabricPorts(t *testing.T) {
_, body := get(t, nodeMux(busySource{}, nil), "http://x/admin/nodes/llmgw-node01", nil)
for _, want := range []string{
"<h2>Fabric</h2>", "On ib-a.", "<code>mlx5_0/1</code>", "400 Gb/s", "4.0 GiB",
`<td class="bad">DOWN <span class="note">Polling</span></td>`, "link_downed 7",
} {
if !strings.Contains(body, want) {
t.Errorf("missing %q", want)
}
}
}
+20
View File
@@ -31,6 +31,26 @@
{{if .Unhealthy}}<p class="note bad">{{join .Unhealthy}}</p>{{end}}
{{end}}
{{if or .Node.IBPorts .Node.Fabrics}}
<h2>Fabric</h2>
<p class="sub">{{if .Node.Fabrics}}On {{join .Node.Fabrics}}.{{else}}<span class="bad">On no fabric: no port a declared fabric names can carry traffic, so no multi-node replica is placed here.</span>{{end}}</p>
{{if .Node.IBPorts}}
<table><tr><th>Adapter</th><th>State</th><th class="num">Rate</th><th>Fabric</th><th>Errors since boot</th><th class="num">Sent</th><th class="num">Received</th></tr>
{{range .Node.IBPorts}}<tr>
<td><code>{{.HCA}}/{{.Port}}</code>{{if eq .LinkLayer "Ethernet"}} <span class="tag">RoCE</span>{{end}}</td>
<td class="{{if .Active}}state-ready{{else}}bad{{end}}">{{.State}}{{with .PhysState}} <span class="note">{{.}}</span>{{end}}</td>
<td class="num">{{if .RateGbps}}{{.RateGbps}} Gb/s{{else}}—{{end}}</td>
<td>{{or .Fabric "—"}}</td>
<td>{{if .Errors}}{{range $k, $v := .Errors}}<span class="tag bad">{{$k}} {{$v}}</span> {{end}}{{else}}<span class="note">none</span>{{end}}</td>
<td class="num">{{if .TxBytes}}{{bytes (u64 .TxBytes)}}{{else}}—{{end}}</td>
<td class="num">{{if .RxBytes}}{{bytes (u64 .RxBytes)}}{{else}}—{{end}}</td>
</tr>{{end}}</table>
<p class="note">Re-read on every heartbeat. A port that goes down takes its fabric off
this node within seconds; one that comes back returns it. Errors are cumulative since
the adapter came up: a number that grows is the one to look at.</p>
{{end}}
{{end}}
<h2>What runs here</h2>
{{if .Engines}}
<table><tr><th>Model</th><th>Replica</th><th>State</th><th>CPU</th><th class="num">Memory</th></tr>
+1
View File
@@ -102,6 +102,7 @@ var funcs = template.FuncMap{
// message on the host will say, and a page that disagrees with the shell
// about the size of the same file starts an argument about neither.
"bytes": humanBytes,
"u64": func(v uint64) int64 { return int64(min(v, 1<<62)) },
// dict lets one chart template be called with named arguments. Go
// templates pass a single value, and four charts differing in three
// strings each is four copies of the same markup otherwise.