diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index 2d1ce99..0dd3588 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -102,6 +102,7 @@ type AgentMessage struct { // *AgentMessage_Heartbeat // *AgentMessage_Shard // *AgentMessage_Logs + // *AgentMessage_FabricScan Msg isAgentMessage_Msg `protobuf_oneof:"msg"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -180,6 +181,15 @@ func (x *AgentMessage) GetLogs() *LogsReply { return nil } +func (x *AgentMessage) GetFabricScan() *FabricScanReply { + if x != nil { + if x, ok := x.Msg.(*AgentMessage_FabricScan); ok { + return x.FabricScan + } + } + return nil +} + type isAgentMessage_Msg interface { isAgentMessage_Msg() } @@ -200,6 +210,10 @@ type AgentMessage_Logs struct { Logs *LogsReply `protobuf:"bytes,4,opt,name=logs,proto3,oneof"` } +type AgentMessage_FabricScan struct { + FabricScan *FabricScanReply `protobuf:"bytes,5,opt,name=fabric_scan,json=fabricScan,proto3,oneof"` +} + func (*AgentMessage_Register) isAgentMessage_Msg() {} func (*AgentMessage_Heartbeat) isAgentMessage_Msg() {} @@ -208,6 +222,8 @@ func (*AgentMessage_Shard) isAgentMessage_Msg() {} func (*AgentMessage_Logs) isAgentMessage_Msg() {} +func (*AgentMessage_FabricScan) isAgentMessage_Msg() {} + // ControlMessage is anything the control plane tells a node. type ControlMessage struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -220,6 +236,7 @@ type ControlMessage struct { // *ControlMessage_RemoveWeights // *ControlMessage_Provision // *ControlMessage_Logs + // *ControlMessage_FabricScan Msg isControlMessage_Msg `protobuf_oneof:"msg"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -325,6 +342,15 @@ func (x *ControlMessage) GetLogs() *LogsRequest { return nil } +func (x *ControlMessage) GetFabricScan() *FabricScanRequest { + if x != nil { + if x, ok := x.Msg.(*ControlMessage_FabricScan); ok { + return x.FabricScan + } + } + return nil +} + type isControlMessage_Msg interface { isControlMessage_Msg() } @@ -357,6 +383,10 @@ type ControlMessage_Logs struct { Logs *LogsRequest `protobuf:"bytes,7,opt,name=logs,proto3,oneof"` } +type ControlMessage_FabricScan struct { + FabricScan *FabricScanRequest `protobuf:"bytes,8,opt,name=fabric_scan,json=fabricScan,proto3,oneof"` +} + func (*ControlMessage_Start) isControlMessage_Msg() {} func (*ControlMessage_Stop) isControlMessage_Msg() {} @@ -371,6 +401,8 @@ func (*ControlMessage_Provision) isControlMessage_Msg() {} func (*ControlMessage_Logs) isControlMessage_Msg() {} +func (*ControlMessage_FabricScan) isControlMessage_Msg() {} + // Register is the first message on every stream. It carries the node's full // hardware inventory, so the control plane needs no separate discovery call and // a reconnecting agent automatically refreshes what the scheduler knows. @@ -2833,17 +2865,154 @@ func (x *LogsReply) GetError() string { return "" } +// FabricScanRequest asks a node to map the fabric one of its ports is on, +// with ibnetdiscover: every switch, every adapter and every link between +// them. One node per fabric is asked, every few minutes; a fabric looks the +// same from all of its ports. +type FabricScanRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + RequestId string `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` + Hca string `protobuf:"bytes,2,opt,name=hca,proto3" json:"hca,omitempty"` + Port int32 `protobuf:"varint,3,opt,name=port,proto3" json:"port,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FabricScanRequest) Reset() { + *x = FabricScanRequest{} + mi := &file_llmgw_v1_agent_proto_msgTypes[27] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FabricScanRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FabricScanRequest) ProtoMessage() {} + +func (x *FabricScanRequest) ProtoReflect() protoreflect.Message { + mi := &file_llmgw_v1_agent_proto_msgTypes[27] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FabricScanRequest.ProtoReflect.Descriptor instead. +func (*FabricScanRequest) Descriptor() ([]byte, []int) { + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{27} +} + +func (x *FabricScanRequest) GetRequestId() string { + if x != nil { + return x.RequestId + } + return "" +} + +func (x *FabricScanRequest) GetHca() string { + if x != nil { + return x.Hca + } + return "" +} + +func (x *FabricScanRequest) GetPort() int32 { + if x != nil { + return x.Port + } + return 0 +} + +// FabricScanReply is ibnetdiscover's output as it printed it. Parsed by the +// control plane, so a format the parser does not yet know is a control plane +// upgrade rather than one on every node. +type FabricScanReply struct { + state protoimpl.MessageState `protogen:"open.v1"` + RequestId string `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` + Output string `protobuf:"bytes,2,opt,name=output,proto3" json:"output,omitempty"` + Error string `protobuf:"bytes,3,opt,name=error,proto3" json:"error,omitempty"` + TookMs int64 `protobuf:"varint,4,opt,name=took_ms,json=tookMs,proto3" json:"took_ms,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FabricScanReply) Reset() { + *x = FabricScanReply{} + mi := &file_llmgw_v1_agent_proto_msgTypes[28] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FabricScanReply) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FabricScanReply) ProtoMessage() {} + +func (x *FabricScanReply) ProtoReflect() protoreflect.Message { + mi := &file_llmgw_v1_agent_proto_msgTypes[28] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FabricScanReply.ProtoReflect.Descriptor instead. +func (*FabricScanReply) Descriptor() ([]byte, []int) { + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{28} +} + +func (x *FabricScanReply) GetRequestId() string { + if x != nil { + return x.RequestId + } + return "" +} + +func (x *FabricScanReply) GetOutput() string { + if x != nil { + return x.Output + } + return "" +} + +func (x *FabricScanReply) GetError() string { + if x != nil { + return x.Error + } + return "" +} + +func (x *FabricScanReply) GetTookMs() int64 { + if x != nil { + return x.TookMs + } + return 0 +} + var File_llmgw_v1_agent_proto protoreflect.FileDescriptor const file_llmgw_v1_agent_proto_rawDesc = "" + "\n" + - "\x14llmgw/v1/agent.proto\x12\bllmgw.v1\"\xd6\x01\n" + + "\x14llmgw/v1/agent.proto\x12\bllmgw.v1\"\x94\x02\n" + "\fAgentMessage\x120\n" + "\bregister\x18\x01 \x01(\v2\x12.llmgw.v1.RegisterH\x00R\bregister\x123\n" + "\theartbeat\x18\x02 \x01(\v2\x13.llmgw.v1.HeartbeatH\x00R\theartbeat\x12-\n" + "\x05shard\x18\x03 \x01(\v2\x15.llmgw.v1.ShardStatusH\x00R\x05shard\x12)\n" + - "\x04logs\x18\x04 \x01(\v2\x13.llmgw.v1.LogsReplyH\x00R\x04logsB\x05\n" + - "\x03msg\"\xe5\x02\n" + + "\x04logs\x18\x04 \x01(\v2\x13.llmgw.v1.LogsReplyH\x00R\x04logs\x12<\n" + + "\vfabric_scan\x18\x05 \x01(\v2\x19.llmgw.v1.FabricScanReplyH\x00R\n" + + "fabricScanB\x05\n" + + "\x03msg\"\xa5\x03\n" + "\x0eControlMessage\x12,\n" + "\x05start\x18\x01 \x01(\v2\x14.llmgw.v1.StartShardH\x00R\x05start\x12)\n" + "\x04stop\x18\x02 \x01(\v2\x13.llmgw.v1.StopShardH\x00R\x04stop\x12,\n" + @@ -2851,7 +3020,9 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x03ack\x18\x04 \x01(\v2\r.llmgw.v1.AckH\x00R\x03ack\x12@\n" + "\x0eremove_weights\x18\x05 \x01(\v2\x17.llmgw.v1.RemoveWeightsH\x00R\rremoveWeights\x123\n" + "\tprovision\x18\x06 \x01(\v2\x13.llmgw.v1.ProvisionH\x00R\tprovision\x12+\n" + - "\x04logs\x18\a \x01(\v2\x15.llmgw.v1.LogsRequestH\x00R\x04logsB\x05\n" + + "\x04logs\x18\a \x01(\v2\x15.llmgw.v1.LogsRequestH\x00R\x04logs\x12>\n" + + "\vfabric_scan\x18\b \x01(\v2\x1b.llmgw.v1.FabricScanRequestH\x00R\n" + + "fabricScanB\x05\n" + "\x03msg\"\xf2\x05\n" + "\bRegister\x12\x17\n" + "\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1a\n" + @@ -3111,7 +3282,18 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "request_id\x18\x01 \x01(\tR\trequestId\x12\x14\n" + "\x05lines\x18\x02 \x03(\tR\x05lines\x12\x18\n" + "\asources\x18\x03 \x03(\tR\asources\x12\x14\n" + - "\x05error\x18\x04 \x01(\tR\x05error2O\n" + + "\x05error\x18\x04 \x01(\tR\x05error\"X\n" + + "\x11FabricScanRequest\x12\x1d\n" + + "\n" + + "request_id\x18\x01 \x01(\tR\trequestId\x12\x10\n" + + "\x03hca\x18\x02 \x01(\tR\x03hca\x12\x12\n" + + "\x04port\x18\x03 \x01(\x05R\x04port\"w\n" + + "\x0fFabricScanReply\x12\x1d\n" + + "\n" + + "request_id\x18\x01 \x01(\tR\trequestId\x12\x16\n" + + "\x06output\x18\x02 \x01(\tR\x06output\x12\x14\n" + + "\x05error\x18\x03 \x01(\tR\x05error\x12\x17\n" + + "\atook_ms\x18\x04 \x01(\x03R\x06tookMs2O\n" + "\fAgentService\x12?\n" + "\aConnect\x12\x16.llmgw.v1.AgentMessage\x1a\x18.llmgw.v1.ControlMessage(\x010\x01B\x93\x01\n" + "\fcom.llmgw.v1B\n" + @@ -3130,87 +3312,91 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte { } var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 33) +var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 35) var file_llmgw_v1_agent_proto_goTypes = []any{ - (ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State - (*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage - (*ControlMessage)(nil), // 2: llmgw.v1.ControlMessage - (*Register)(nil), // 3: llmgw.v1.Register - (*Host)(nil), // 4: llmgw.v1.Host - (*Storage)(nil), // 5: llmgw.v1.Storage - (*RunningShard)(nil), // 6: llmgw.v1.RunningShard - (*GPU)(nil), // 7: llmgw.v1.GPU - (*FabricAttachment)(nil), // 8: llmgw.v1.FabricAttachment - (*LocalWeights)(nil), // 9: llmgw.v1.LocalWeights - (*NodeLoad)(nil), // 10: llmgw.v1.NodeLoad - (*ShardLoad)(nil), // 11: llmgw.v1.ShardLoad - (*IBPort)(nil), // 12: llmgw.v1.IBPort - (*SubnetManager)(nil), // 13: llmgw.v1.SubnetManager - (*Heartbeat)(nil), // 14: llmgw.v1.Heartbeat - (*Provision)(nil), // 15: llmgw.v1.Provision - (*EngineWant)(nil), // 16: llmgw.v1.EngineWant - (*ProvisionStatus)(nil), // 17: llmgw.v1.ProvisionStatus - (*ShardStatus)(nil), // 18: llmgw.v1.ShardStatus - (*FetchProgress)(nil), // 19: llmgw.v1.FetchProgress - (*StartShard)(nil), // 20: llmgw.v1.StartShard - (*EngineProfile)(nil), // 21: llmgw.v1.EngineProfile - (*RemoveWeights)(nil), // 22: llmgw.v1.RemoveWeights - (*StopShard)(nil), // 23: llmgw.v1.StopShard - (*DrainShard)(nil), // 24: llmgw.v1.DrainShard - (*Ack)(nil), // 25: llmgw.v1.Ack - (*LogsRequest)(nil), // 26: llmgw.v1.LogsRequest - (*LogsReply)(nil), // 27: llmgw.v1.LogsReply - nil, // 28: llmgw.v1.Register.LabelsEntry - nil, // 29: llmgw.v1.Register.EngineVersionsEntry - nil, // 30: llmgw.v1.GPU.NicPathsEntry - nil, // 31: llmgw.v1.IBPort.CountersEntry - nil, // 32: llmgw.v1.Heartbeat.EngineVersionsEntry - nil, // 33: llmgw.v1.EngineProfile.EnvEntry + (ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State + (*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage + (*ControlMessage)(nil), // 2: llmgw.v1.ControlMessage + (*Register)(nil), // 3: llmgw.v1.Register + (*Host)(nil), // 4: llmgw.v1.Host + (*Storage)(nil), // 5: llmgw.v1.Storage + (*RunningShard)(nil), // 6: llmgw.v1.RunningShard + (*GPU)(nil), // 7: llmgw.v1.GPU + (*FabricAttachment)(nil), // 8: llmgw.v1.FabricAttachment + (*LocalWeights)(nil), // 9: llmgw.v1.LocalWeights + (*NodeLoad)(nil), // 10: llmgw.v1.NodeLoad + (*ShardLoad)(nil), // 11: llmgw.v1.ShardLoad + (*IBPort)(nil), // 12: llmgw.v1.IBPort + (*SubnetManager)(nil), // 13: llmgw.v1.SubnetManager + (*Heartbeat)(nil), // 14: llmgw.v1.Heartbeat + (*Provision)(nil), // 15: llmgw.v1.Provision + (*EngineWant)(nil), // 16: llmgw.v1.EngineWant + (*ProvisionStatus)(nil), // 17: llmgw.v1.ProvisionStatus + (*ShardStatus)(nil), // 18: llmgw.v1.ShardStatus + (*FetchProgress)(nil), // 19: llmgw.v1.FetchProgress + (*StartShard)(nil), // 20: llmgw.v1.StartShard + (*EngineProfile)(nil), // 21: llmgw.v1.EngineProfile + (*RemoveWeights)(nil), // 22: llmgw.v1.RemoveWeights + (*StopShard)(nil), // 23: llmgw.v1.StopShard + (*DrainShard)(nil), // 24: llmgw.v1.DrainShard + (*Ack)(nil), // 25: llmgw.v1.Ack + (*LogsRequest)(nil), // 26: llmgw.v1.LogsRequest + (*LogsReply)(nil), // 27: llmgw.v1.LogsReply + (*FabricScanRequest)(nil), // 28: llmgw.v1.FabricScanRequest + (*FabricScanReply)(nil), // 29: llmgw.v1.FabricScanReply + nil, // 30: llmgw.v1.Register.LabelsEntry + nil, // 31: llmgw.v1.Register.EngineVersionsEntry + nil, // 32: llmgw.v1.GPU.NicPathsEntry + nil, // 33: llmgw.v1.IBPort.CountersEntry + nil, // 34: llmgw.v1.Heartbeat.EngineVersionsEntry + nil, // 35: llmgw.v1.EngineProfile.EnvEntry } var file_llmgw_v1_agent_proto_depIdxs = []int32{ 3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register 14, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat 18, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus 27, // 3: llmgw.v1.AgentMessage.logs:type_name -> llmgw.v1.LogsReply - 20, // 4: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard - 23, // 5: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard - 24, // 6: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard - 25, // 7: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack - 22, // 8: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights - 15, // 9: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision - 26, // 10: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest - 28, // 11: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry - 7, // 12: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU - 8, // 13: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment - 29, // 14: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry - 6, // 15: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard - 9, // 16: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights - 5, // 17: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage - 4, // 18: llmgw.v1.Register.host:type_name -> llmgw.v1.Host - 0, // 19: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State - 30, // 20: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry - 11, // 21: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad - 31, // 22: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry - 13, // 23: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager - 5, // 24: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage - 9, // 25: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights - 32, // 26: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry - 17, // 27: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus - 10, // 28: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad - 12, // 29: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort - 8, // 30: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment - 16, // 31: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant - 0, // 32: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State - 19, // 33: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress - 21, // 34: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile - 33, // 35: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry - 1, // 36: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage - 2, // 37: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage - 37, // [37:38] is the sub-list for method output_type - 36, // [36:37] is the sub-list for method input_type - 36, // [36:36] is the sub-list for extension type_name - 36, // [36:36] is the sub-list for extension extendee - 0, // [0:36] is the sub-list for field type_name + 29, // 4: llmgw.v1.AgentMessage.fabric_scan:type_name -> llmgw.v1.FabricScanReply + 20, // 5: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard + 23, // 6: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard + 24, // 7: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard + 25, // 8: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack + 22, // 9: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights + 15, // 10: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision + 26, // 11: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest + 28, // 12: llmgw.v1.ControlMessage.fabric_scan:type_name -> llmgw.v1.FabricScanRequest + 30, // 13: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry + 7, // 14: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU + 8, // 15: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment + 31, // 16: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry + 6, // 17: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard + 9, // 18: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights + 5, // 19: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage + 4, // 20: llmgw.v1.Register.host:type_name -> llmgw.v1.Host + 0, // 21: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State + 32, // 22: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry + 11, // 23: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad + 33, // 24: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry + 13, // 25: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager + 5, // 26: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage + 9, // 27: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights + 34, // 28: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry + 17, // 29: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus + 10, // 30: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad + 12, // 31: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort + 8, // 32: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment + 16, // 33: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant + 0, // 34: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State + 19, // 35: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress + 21, // 36: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile + 35, // 37: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry + 1, // 38: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage + 2, // 39: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage + 39, // [39:40] is the sub-list for method output_type + 38, // [38:39] is the sub-list for method input_type + 38, // [38:38] is the sub-list for extension type_name + 38, // [38:38] is the sub-list for extension extendee + 0, // [0:38] is the sub-list for field type_name } func init() { file_llmgw_v1_agent_proto_init() } @@ -3223,6 +3409,7 @@ func file_llmgw_v1_agent_proto_init() { (*AgentMessage_Heartbeat)(nil), (*AgentMessage_Shard)(nil), (*AgentMessage_Logs)(nil), + (*AgentMessage_FabricScan)(nil), } file_llmgw_v1_agent_proto_msgTypes[1].OneofWrappers = []any{ (*ControlMessage_Start)(nil), @@ -3232,6 +3419,7 @@ func file_llmgw_v1_agent_proto_init() { (*ControlMessage_RemoveWeights)(nil), (*ControlMessage_Provision)(nil), (*ControlMessage_Logs)(nil), + (*ControlMessage_FabricScan)(nil), } type x struct{} out := protoimpl.TypeBuilder{ @@ -3239,7 +3427,7 @@ func file_llmgw_v1_agent_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)), NumEnums: 1, - NumMessages: 33, + NumMessages: 35, NumExtensions: 0, NumServices: 1, }, diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index 0bee445..8f934b4 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -24,6 +24,7 @@ message AgentMessage { Heartbeat heartbeat = 2; ShardStatus shard = 3; LogsReply logs = 4; + FabricScanReply fabric_scan = 5; } } @@ -37,6 +38,7 @@ message ControlMessage { RemoveWeights remove_weights = 5; Provision provision = 6; LogsRequest logs = 7; + FabricScanRequest fabric_scan = 8; } } @@ -543,3 +545,23 @@ message LogsReply { // error says why the lines are absent: an unknown source, say. string error = 4; } + +// FabricScanRequest asks a node to map the fabric one of its ports is on, +// with ibnetdiscover: every switch, every adapter and every link between +// them. One node per fabric is asked, every few minutes; a fabric looks the +// same from all of its ports. +message FabricScanRequest { + string request_id = 1; + string hca = 2; + int32 port = 3; +} + +// FabricScanReply is ibnetdiscover's output as it printed it. Parsed by the +// control plane, so a format the parser does not yet know is a control plane +// upgrade rather than one on every node. +message FabricScanReply { + string request_id = 1; + string output = 2; + string error = 3; + int64 took_ms = 4; +} diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 405b6b9..54a270e 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -1123,6 +1123,23 @@ func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out cha case <-ctx.Done(): } }() + case *llmgwv1.ControlMessage_FabricScan: + // A scan takes seconds to minutes: off the receive loop entirely. + go func(req *llmgwv1.FabricScanRequest) { + start := time.Now() + reply := &llmgwv1.FabricScanReply{RequestId: req.GetRequestId()} + text, err := scanFabric(ctx, req.GetHca(), int(req.GetPort())) + if err != nil { + reply.Error = err.Error() + } else { + reply.Output = text + } + reply.TookMs = time.Since(start).Milliseconds() + select { + case out <- &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_FabricScan{FabricScan: reply}}: + case <-ctx.Done(): + } + }(m.FabricScan) case *llmgwv1.ControlMessage_Ack: if !m.Ack.GetOk() { a.log.Warn("control plane rejected a message", @@ -1639,6 +1656,9 @@ func ResolveManagement(c Config) (addr, iface string) { func smKey(hca string, port int) string { return fmt.Sprintf("%s/%d", hca, port) } +// scanFabric is discovery.ScanFabric, replaced in tests. +var scanFabric = discovery.ScanFabric + // querySMs is discovery.QuerySMs, replaced in tests. var querySMs = func(hca string, port int) ([]discovery.SMRecord, error) { return discovery.QuerySMs(context.Background(), hca, port) diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index 47bf414..07309bc 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -4,6 +4,7 @@ import ( "context" "crypto/tls" "errors" + "fmt" "log/slog" "net" "net/http" @@ -861,3 +862,22 @@ func TestEachFabricsSubnetManagersAreAskedOnce(t *testing.T) { t.Errorf("listed %d times, carried on %d ports; want once, on one", lists, withList) } } + +func TestAFabricScanIsAskedForAndAnswered(t *testing.T) { + // One node per fabric maps it; the control plane asks and parses. + old := scanFabric + t.Cleanup(func() { scanFabric = old }) + var asked string + scanFabric = func(_ context.Context, hca string, port int) (string, error) { + asked = fmt.Sprintf("%s/%d", hca, port) + return "Switch\t36 \"S-00000000000000a1\"\t\t# \"MF0;leaf-a:QM8700/U1\" enhanced port 0 lid 1 lmc 0\n", nil + } + h := start(t) + r, err := h.svc.ScanFabric(t.Context(), "n1", "mlx5_0", 1) + if err != nil { + t.Fatal(err) + } + if asked != "mlx5_0/1" || !strings.Contains(r.GetOutput(), "leaf-a") || r.GetError() != "" { + t.Errorf("asked %q, got %+v", asked, r) + } +} diff --git a/internal/agent/discovery/sminfo.go b/internal/agent/discovery/sminfo.go index ff22b5a..334e910 100644 --- a/internal/agent/discovery/sminfo.go +++ b/internal/agent/discovery/sminfo.go @@ -144,3 +144,29 @@ func QuerySMs(ctx context.Context, hca string, port int) ([]SMRecord, error) { } return ParseSMIR(string(out)) } + +// ScanFabric maps the fabric a port is on with ibnetdiscover: every switch, +// adapter and link. It can take a while on a large fabric, so it is bounded, +// and its output is capped: a fabric of thousands of ports is a few +// megabytes of text. +func ScanFabric(ctx context.Context, hca string, port int) (string, error) { + bin, err := exec.LookPath("ibnetdiscover") + if err != nil { + return "", ErrNoDiags + } + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + out, err := exec.CommandContext(ctx, bin, "-C", hca, "-P", strconv.Itoa(port)).Output() + if err != nil { + var ee *exec.ExitError + if errors.As(err, &ee) { + return "", fmt.Errorf("ibnetdiscover on %s/%d: %w: %s", hca, port, err, strings.TrimSpace(string(ee.Stderr))) + } + return "", fmt.Errorf("ibnetdiscover on %s/%d: %w", hca, port, err) + } + const limit = 16 << 20 + if len(out) > limit { + return "", fmt.Errorf("ibnetdiscover printed %d MB, more than the %d MB a scan may send", len(out)>>20, limit>>20) + } + return string(out), nil +} diff --git a/internal/cli/server.go b/internal/cli/server.go index 2b69c9a..3c4e9ec 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -208,6 +208,9 @@ func newServerCmd() *cobra.Command { adminMux.Handle("GET /_static/", web.Static()) adminMux.HandleFunc("GET /nodes/{id}/logs", web.NodeLogsPage(api, true, version.Version)) adminMux.HandleFunc("GET /nodes/{id}", web.NodePage(api, history, cfg.Monitoring.Window, true, version.Version, log)) + adminMux.HandleFunc("GET /fabrics", web.FabricsPage(api, true, version.Version)) + adminMux.HandleFunc("GET /fabrics/{id}", web.FabricsPage(api, true, version.Version)) + adminMux.HandleFunc("GET /fabrics/{id}/topology.json", web.FabricTopologyJSON(api)) // The same page on the client-facing port at /admin, behind a // session and an admin flag. Handing the router the data rather @@ -417,6 +420,8 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live svc.SetLeadership(lead) } fwd.set(svc) + // Every InfiniBand fabric mapped every five minutes, through one node. + go svc.ScanFabrics(ctx, 5*time.Minute) ctl.Start(ctx) return ctl, svc, nil diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 8d0763f..b712048 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -22,6 +22,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/topology" "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) @@ -52,6 +53,14 @@ func (a *API) Register(mux *http.ServeMux) { mux.HandleFunc("GET /api/v1/fabrics", func(w http.ResponseWriter, _ *http.Request) { writeJSON(w, http.StatusOK, a.Fabrics()) }) + mux.HandleFunc("GET /api/v1/fabrics/{id}/topology", func(w http.ResponseWriter, r *http.Request) { + t, ok := a.Topology(r.PathValue("id")) + if !ok { + http.NotFound(w, r) + return + } + writeJSON(w, http.StatusOK, t) + }) mux.HandleFunc("GET /api/v1/models", a.listModels) mux.HandleFunc("GET /api/v1/replicas", a.listReplicas) // Listing is a read and deleting is a separate verb, so a mistyped @@ -839,3 +848,114 @@ func (a *API) Fabrics() fabrics.Report { inv := a.ctl.Inventory() return fabrics.Analyze(inv.Nodes(), inv.SMChanges(), time.Now()) } + +// TopologyView is one fabric's map, with the cluster's own adapters on it +// live: their state, rate and traffic from the last heartbeat. +type TopologyView struct { + ID string `json:"id"` + Key string `json:"key"` + Label string `json:"label"` + Declared []string `json:"declared"` + ScannedAt time.Time `json:"scanned_at"` + ScannedBy string `json:"scanned_by,omitempty"` + TookMs int64 `json:"took_ms"` + Err string `json:"error,omitempty"` + Map *topology.View `json:"map,omitempty"` + Live map[string]Live `json:"live"` + // Problems are the fabric's own, from the analysis of its identity and + // subnet managers. + Problems []fabrics.Problem `json:"problems,omitempty"` + SM *fabrics.SM `json:"sm,omitempty"` + SMs []fabrics.SM `json:"sms,omitempty"` +} + +// Live is one of the cluster's adapters as its node last reported it. +type Live struct { + Node string `json:"node"` + HCA string `json:"hca"` + State string `json:"state"` + Phys string `json:"phys_state"` + Gbps int `json:"gbps"` + TxBps float64 `json:"tx_bps"` + RxBps float64 `json:"rx_bps"` + Errors map[string]uint64 `json:"errors,omitempty"` +} + +// FabricID is a physical fabric's key as an address can carry it. +func FabricID(key string) string { + return strings.NewReplacer(":", "-", "/", "_").Replace(key) +} + +// Topologies lists every physical fabric with what is known of its map. +func (a *API) Topologies() []TopologyView { + rep := a.Fabrics() + scans := a.ctl.Inventory().Scans() + out := make([]TopologyView, 0, len(rep.Physical)) + for _, ph := range rep.Physical { + t := TopologyView{ID: FabricID(ph.Key), Key: ph.Key, Label: ph.Label, Declared: ph.Declared} + if sc, ok := scans[ph.Key]; ok { + t.ScannedAt, t.ScannedBy, t.TookMs, t.Err = sc.At, sc.Node+" "+sc.Port, sc.Took.Milliseconds(), sc.Err + } + out = append(out, t) + } + return out +} + +// Topology is one fabric's map, by the id Topologies gives it. +func (a *API) Topology(id string) (TopologyView, bool) { + rep := a.Fabrics() + var ph *fabrics.Physical + for i := range rep.Physical { + if FabricID(rep.Physical[i].Key) == id { + ph = &rep.Physical[i] + } + } + if ph == nil { + return TopologyView{}, false + } + t := TopologyView{ID: id, Key: ph.Key, Label: ph.Label, Declared: ph.Declared, Live: map[string]Live{}, + SM: ph.SM, SMs: ph.SMs} + for _, p := range rep.Problems { + for _, d := range ph.Declared { + if p.Fabric == d { + t.Problems = append(t.Problems, p) + break + } + } + } + hosts := map[string]topology.Host{} + for _, n := range a.ctl.Inventory().Nodes() { + for _, p := range n.IBPorts { + if p.NodeGUID == "" { + continue + } + hosts[p.NodeGUID] = topology.Host{Node: n.ID, HCA: p.HCA} + l := Live{Node: n.ID, HCA: p.HCA, State: p.State, Phys: p.PhysState, Gbps: p.RateGbps, + TxBps: p.TxBps, RxBps: p.RxBps} + for name, c := range p.Counters { + if c > 0 && name != "port_xmit_data" && name != "port_rcv_data" { + if l.Errors == nil { + l.Errors = map[string]uint64{} + } + l.Errors[name] = c + } + } + t.Live[p.NodeGUID] = l + } + } + sc, ok := a.ctl.Inventory().Scans()[ph.Key] + if !ok { + return t, true + } + t.ScannedAt, t.ScannedBy, t.TookMs, t.Err = sc.At, sc.Node+" "+sc.Port, sc.Took.Milliseconds(), sc.Err + if sc.Output != "" { + f, err := topology.Parse(sc.Output) + if err != nil { + t.Err = "the map could not be read: " + err.Error() + return t, true + } + v := topology.Build(f, hosts) + t.Map = &v + } + return t, true +} diff --git a/internal/control/agentsvc/service.go b/internal/control/agentsvc/service.go index 17973c4..275057b 100644 --- a/internal/control/agentsvc/service.go +++ b/internal/control/agentsvc/service.go @@ -9,6 +9,7 @@ import ( "maps" "net/http" "slices" + "sort" "sync" "time" @@ -58,6 +59,8 @@ type Service struct { logMu sync.Mutex logWaits map[string]chan *llmgwv1.LogsReply logSeq uint64 + // scanWaits are the fabric scans awaiting a reply, by request id. + scanWaits map[string]chan *llmgwv1.FabricScanReply } // SetEngines sets the engines every node installs, by name and version. @@ -282,6 +285,15 @@ func (s *Service) handle(nodeID string, msg *llmgwv1.AgentMessage) { "last_output", m.Shard.GetLastOutput()) } + case *llmgwv1.AgentMessage_FabricScan: + s.logMu.Lock() + ch, ok := s.scanWaits[m.FabricScan.GetRequestId()] + delete(s.scanWaits, m.FabricScan.GetRequestId()) + s.logMu.Unlock() + if ok { + ch <- m.FabricScan + } + case *llmgwv1.AgentMessage_Logs: s.logMu.Lock() ch, ok := s.logWaits[m.Logs.GetRequestId()] @@ -406,9 +418,13 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) { } if ports := hb.GetIbPorts(); len(ports) > 0 { before := map[string]string{} + prev := map[string]inventory.IBPort{} for _, p := range n.IBPorts { before[fmt.Sprintf("%s/%d", p.HCA, p.Port)] = p.SMGUID + prev[fmt.Sprintf("%s/%d", p.HCA, p.Port)] = p } + now := time.Now() + secs := now.Sub(n.IBAt).Seconds() n.IBPorts = nil for _, p := range ports { // A different master than last time is a failover, or a port @@ -427,7 +443,9 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) { } n.IBPorts = append(n.IBPorts, inventory.IBPort{ SMs: sms, SMsError: p.GetSubnetManagersError(), - HCA: p.GetHca(), Port: int(p.GetPort()), State: p.GetState(), + TxBps: rate(prev[where].Counters["port_xmit_data"], p.GetCounters()["port_xmit_data"], secs), + RxBps: rate(prev[where].Counters["port_rcv_data"], p.GetCounters()["port_rcv_data"], secs), + HCA: p.GetHca(), Port: int(p.GetPort()), State: p.GetState(), PhysState: p.GetPhysState(), RateGbps: int(p.GetRateGbps()), LinkLayer: p.GetLinkLayer(), Counters: maps.Clone(p.GetCounters()), NodeGUID: p.GetNodeGuid(), PortGUID: p.GetPortGuid(), GIDPrefix: p.GetGidPrefix(), @@ -436,6 +454,9 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) { }) } } + if len(hb.GetIbPorts()) > 0 { + n.IBAt = time.Now() + } // A port went down, or came back. Placement reads the fabrics, so this is // what stops a node being offered multi-node work it can no longer carry. if hb.GetFabricsChanged() { @@ -642,3 +663,102 @@ func (s *Service) Logs(ctx context.Context, nodeID, source string, lines int) ([ return nil, nil, fmt.Errorf("node %s did not answer; an agent older than v0.8.4 cannot", nodeID) } } + +// ScanFabric asks a node to map the fabric one of its ports is on. +func (s *Service) ScanFabric(ctx context.Context, nodeID, hca string, port int) (*llmgwv1.FabricScanReply, error) { + s.logMu.Lock() + if s.scanWaits == nil { + s.scanWaits = map[string]chan *llmgwv1.FabricScanReply{} + } + s.logSeq++ + id := fmt.Sprintf("scan-%d", s.logSeq) + ch := make(chan *llmgwv1.FabricScanReply, 1) + s.scanWaits[id] = ch + s.logMu.Unlock() + defer func() { + s.logMu.Lock() + delete(s.scanWaits, id) + s.logMu.Unlock() + }() + if err := s.send(nodeID, &llmgwv1.ControlMessage{Msg: &llmgwv1.ControlMessage_FabricScan{ + FabricScan: &llmgwv1.FabricScanRequest{RequestId: id, Hca: hca, Port: int32(port)}, //nolint:gosec // a port number + }}); err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(ctx, 3*time.Minute) + defer cancel() + select { + case r := <-ch: + return r, nil + case <-ctx.Done(): + return nil, fmt.Errorf("node %s did not answer the fabric scan", nodeID) + } +} + +// ScanFabrics maps every physical InfiniBand fabric every so often, asking +// one node on each: the first, by name, whose port on it is up. A fabric +// looks the same from all its ports, so one is enough, and the rest of its +// nodes are left to their work. +func (s *Service) ScanFabrics(ctx context.Context, every time.Duration) { + if every <= 0 { + every = 5 * time.Minute + } + // Soon after a start, once agents have sent a heartbeat or two. + timer := time.NewTimer(30 * time.Second) + defer timer.Stop() + for { + select { + case <-ctx.Done(): + return + case <-timer.C: + } + s.scanOnce(ctx) + timer.Reset(every) + } +} + +func (s *Service) scanOnce(ctx context.Context) { + nodes := s.inv.Nodes() + sort.Slice(nodes, func(i, j int) bool { return nodes[i].ID < nodes[j].ID }) + canon := inventory.CanonicalSMs(nodes) + chosen := map[string]bool{} + for _, n := range nodes { + if n.State != inventory.Ready { + continue + } + for _, p := range n.IBPorts { + key := p.FabricKeyIn(canon) + if key == "" || p.RoCE() || chosen[key] { + continue + } + chosen[key] = true + where := fmt.Sprintf("%s/%d", p.HCA, p.Port) + r, err := s.ScanFabric(ctx, n.ID, p.HCA, p.Port) + sc := inventory.FabricScan{Key: key, At: time.Now(), Node: n.ID, Port: where} + switch { + case err != nil: + sc.Err = err.Error() + case r.GetError() != "": + sc.Err = r.GetError() + default: + sc.Output = r.GetOutput() + } + if r != nil { + sc.Took = time.Duration(r.GetTookMs()) * time.Millisecond + } + if sc.Err != "" { + s.log.Warn("could not map a fabric", "fabric", key, "node", n.ID, "port", where, "err", sc.Err) + } + s.inv.SetScan(sc) + } + } +} + +// rate is a counter's increase a second, zero when there is no earlier +// reading or the counter went backwards — an adapter reset starts it again. +func rate(before, now uint64, secs float64) float64 { + if before == 0 || now < before || secs <= 0 || secs > 120 { + return 0 + } + return float64(now-before) / secs +} diff --git a/internal/control/inventory/inventory.go b/internal/control/inventory/inventory.go index 6c114bd..38d5f4b 100644 --- a/internal/control/inventory/inventory.go +++ b/internal/control/inventory/inventory.go @@ -118,6 +118,8 @@ type Node struct { // IBPorts are the node's InfiniBand and RoCE ports at the last heartbeat, // with their counters. Not persisted, for the same reason as Load. IBPorts []IBPort `json:"-"` + // IBAt is when IBPorts was read, for the rates. + IBAt time.Time `json:"-"` } // IBPort is one port's state, identity and cumulative counters. @@ -146,6 +148,9 @@ type IBPort struct { // on the one port per fabric that asked; SMsError says why it could not. SMs []SubnetManager SMsError string + // TxBps and RxBps are bytes a second, from this heartbeat's data + // counters against the last one's; zero on the first. + TxBps, RxBps float64 } // SubnetManager is one subnet manager on a fabric. @@ -357,6 +362,8 @@ type Store struct { allocs map[string][]Allocation // smChanges are ports' changes of master subnet manager, newest last. smChanges []SMChange + // scans are the latest ibnetdiscover of each physical fabric, by key. + scans map[string]FabricScan } // NewStore returns an empty inventory. @@ -378,6 +385,43 @@ func (s *Store) RecordSMChange(c SMChange) { } } +// FabricScan is one ibnetdiscover of a physical fabric: its text, and who +// ran it when. +type FabricScan struct { + Key string + At time.Time + Node string + Port string + Output string + Err string + Took time.Duration +} + +// SetScan keeps a fabric's latest scan. A failed one does not replace a +// good one's text: a map from ten minutes ago is worth more than none. +func (s *Store) SetScan(sc FabricScan) { + s.mu.Lock() + defer s.mu.Unlock() + if s.scans == nil { + s.scans = map[string]FabricScan{} + } + if sc.Err != "" { + if old, ok := s.scans[sc.Key]; ok && old.Output != "" { + old.Err = sc.Err + s.scans[sc.Key] = old + return + } + } + s.scans[sc.Key] = sc +} + +// Scans returns the latest scan of every fabric. +func (s *Store) Scans() map[string]FabricScan { + s.mu.Lock() + defer s.mu.Unlock() + return maps.Clone(s.scans) +} + // SMChanges returns the recorded changes, oldest first. func (s *Store) SMChanges() []SMChange { s.mu.Lock() diff --git a/internal/control/placement/placement.go b/internal/control/placement/placement.go index 38fd0d2..69a4674 100644 --- a/internal/control/placement/placement.go +++ b/internal/control/placement/placement.go @@ -333,6 +333,9 @@ func (s *Scheduler) placeOnFabric(p Profile, fabric, pool string, candidates []c sort.SliceStable(candidates, func(i, j int) bool { return len(candidates[i].free) > len(candidates[j].free) }) + // Then by the fabric's map, where there is one: one leaf's nodes + // together, a degraded cable last. + candidates = byLeaf(candidates, hints(s.inv)) assignments := make([]NodeAssignment, 0, p.Nodes) for rank, c := range candidates[:p.Nodes] { diff --git a/internal/control/placement/placement_test.go b/internal/control/placement/placement_test.go index 8c47558..721232a 100644 --- a/internal/control/placement/placement_test.go +++ b/internal/control/placement/placement_test.go @@ -6,6 +6,7 @@ import ( "slices" "strings" "testing" + "time" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" @@ -970,3 +971,50 @@ func TestAFailoverDoesNotSplitAFabricForPlacement(t *testing.T) { t.Fatalf("a failover split the fabric:\n%s", res.Explain()) } } + +// scanOf is ibnetdiscover's output for two leaves: leaf-a holding the first +// two nodes' adapters, leaf-b the other two. degraded names a node whose +// cable trained at half width. +func scanOf(ids []string, degraded string) string { + var b strings.Builder + for li, leaf := range []string{"00000000000000a1", "00000000000000a2"} { + fmt.Fprintf(&b, "Switch\t36 \"S-%s\"\t\t# \"MF0;leaf-%c:QM8700/U1\" enhanced port 0 lid %d lmc 0\n", leaf, 'a'+li, li+1) + for k := 0; k < 2; k++ { + id := ids[li*2+k] + rate := "4xHDR" + if id == degraded { + rate = "2xHDR" + } + fmt.Fprintf(&b, "[%d]\t\"H-%s\"[1](%s) \t\t# \"%s mlx5_0\" lid %d %s\n", k+1, guidOf(id), guidOf(id), id, 10+li*2+k, rate) + } + b.WriteString("\n") + } + return b.String() +} + +func guidOf(id string) string { return fmt.Sprintf("%016x", []byte(id)[len(id)-1]) } + +func TestAReplicaStaysUnderOneLeafAndOffDegradedCables(t *testing.T) { + ids := []string{"n1", "n2", "n3", "n4"} + var nodes []inventory.Node + for _, id := range ids { + n := onSM(node(id, "h100-80g", 8, "ib-a"), "0x01") + n.IBPorts[0].NodeGUID = "0x" + guidOf(id) + nodes = append(nodes, n) + } + st := store(nodes...) + // n1 sits on leaf-a with n2, but its cable trained at half width: leaf-a + // has one good node, leaf-b two. + st.SetScan(inventory.FabricScan{Key: "sm:0x01", At: time.Now(), Output: scanOf(ids, "n1")}) + res := New(st).Place([]Profile{{Name: "pair", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2}}, nil) + if res.Placement == nil { + t.Fatalf("no placement:\n%s", res.Explain()) + } + var got []string + for _, a := range res.Placement.Nodes { + got = append(got, a.NodeID) + } + if !slices.Equal(got, []string{"n3", "n4"}) { + t.Errorf("placed on %v, want the two good nodes under leaf-b", got) + } +} diff --git a/internal/control/placement/topology.go b/internal/control/placement/topology.go new file mode 100644 index 0000000..be42e7d --- /dev/null +++ b/internal/control/placement/topology.go @@ -0,0 +1,135 @@ +package placement + +import ( + "slices" + "sync" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/topology" +) + +// fabricHints is what a fabric's map says about placing on it: the leaf +// switch each node's first adapter hangs off, and which nodes have a cable +// that trained below the rest. +type fabricHints struct { + leaf map[string]string + degraded map[string]bool +} + +// hintCache keeps a parsed map until its fabric is scanned again; parsing +// thousands of lines on every placement would be the scheduler's main cost. +var hintCache struct { + sync.Mutex + at map[string]time.Time + hints map[string]fabricHints +} + +// hints reads every scanned fabric. Nodes on no mapped fabric are absent, +// which leaves them placed as before. +func hints(inv *inventory.Store) fabricHints { + out := fabricHints{leaf: map[string]string{}, degraded: map[string]bool{}} + scans := inv.Scans() + if len(scans) == 0 { + return out + } + var hosts map[string]topology.Host + hintCache.Lock() + defer hintCache.Unlock() + if hintCache.at == nil { + hintCache.at, hintCache.hints = map[string]time.Time{}, map[string]fabricHints{} + } + for key, sc := range scans { + if sc.Output == "" { + continue + } + h, ok := hintCache.hints[key] + if !ok || !hintCache.at[key].Equal(sc.At) { + if hosts == nil { + hosts = map[string]topology.Host{} + for _, n := range inv.Nodes() { + for _, p := range n.IBPorts { + if p.NodeGUID != "" { + hosts[p.NodeGUID] = topology.Host{Node: n.ID, HCA: p.HCA} + } + } + } + } + f, err := topology.Parse(sc.Output) + if err != nil { + continue + } + v := topology.Build(f, hosts) + h = fabricHints{leaf: map[string]string{}, degraded: map[string]bool{}} + guidNode := map[string]string{} + for _, hv := range v.Hosts { + if hv.Node == "" { + continue + } + guidNode[hv.GUID] = hv.Node + if hv.Rail == 0 { + h.leaf[hv.Node] = hv.Leaf + } + } + for _, l := range v.Links { + if l.Host && l.Degraded { + if n := guidNode[l.To]; n != "" { + h.degraded[n] = true + } + } + } + hintCache.hints[key], hintCache.at[key] = h, sc.At + } + for n, l := range h.leaf { + out.leaf[n] = l + } + for n := range h.degraded { + out.degraded[n] = true + } + } + return out +} + +// byLeaf orders candidates for a replica spanning nodes: those whose cables +// all trained at their rate first, and among them the nodes of one leaf +// together — the leaf with the most of them first — so a replica's traffic +// stays under one switch when it can, and crosses the spines only when it +// must. Within a leaf the order given (emptiest first) is kept. +func byLeaf(cands []candidate, h fabricHints) []candidate { + if len(h.leaf) == 0 && len(h.degraded) == 0 { + return cands + } + count := map[string]int{} + for _, c := range cands { + if !h.degraded[c.node.ID] { + count[h.leaf[c.node.ID]]++ + } + } + firstSeen := map[string]int{} + for i, c := range cands { + if _, ok := firstSeen[h.leaf[c.node.ID]]; !ok { + firstSeen[h.leaf[c.node.ID]] = i + } + } + out := make([]candidate, len(cands)) + copy(out, cands) + rank := func(c candidate) (bad int, size int, first int) { + if h.degraded[c.node.ID] { + bad = 1 + } + l := h.leaf[c.node.ID] + return bad, count[l], firstSeen[l] + } + slices.SortStableFunc(out, func(a, b candidate) int { + ba, sa, fa := rank(a) + bb, sb, fb := rank(b) + if ba != bb { + return ba - bb + } + if sa != sb { + return sb - sa + } + return fa - fb + }) + return out +} diff --git a/internal/control/topology/topology.go b/internal/control/topology/topology.go new file mode 100644 index 0000000..15351e7 --- /dev/null +++ b/internal/control/topology/topology.go @@ -0,0 +1,390 @@ +// Package topology reads ibnetdiscover's map of a fabric and works out its +// shape: which switches are leaves and which spines, which adapter belongs to +// which node, which links trained below the rest, and whether the rails line +// up. +package topology + +import ( + "fmt" + "regexp" + "sort" + "strconv" + "strings" +) + +// Device is a switch, an adapter or a router on the fabric. +type Device struct { + GUID string `json:"guid"` + Kind string `json:"kind"` // switch, ca, router + Desc string `json:"desc"` + Ports int `json:"ports"` + LID int `json:"lid,omitempty"` +} + +// End is one side of a link. +type End struct { + GUID string `json:"guid"` + Port int `json:"port"` +} + +// Link is one cable, once. +type Link struct { + A End `json:"a"` + B End `json:"b"` + Width string `json:"width"` // 4x + Speed string `json:"speed"` // HDR + Gbps int `json:"gbps"` +} + +// Fabric is a parsed ibnetdiscover. +type Fabric struct { + Devices map[string]*Device `json:"devices"` + Links []Link `json:"links"` +} + +var ( + header = regexp.MustCompile(`^(Switch|Ca|Rt)\s+(\d+)\s+"([SHR])-([0-9a-fA-F]+)"\s*(?:#\s*"([^"]*)"(.*))?$`) + portLn = regexp.MustCompile(`^\[(\d+)\](?:\([0-9a-fA-F]+\))?\s+"([SHR])-([0-9a-fA-F]+)"\[(\d+)\](?:\([0-9a-fA-F]+\))?\s*(?:#\s*(.*))?$`) + lidRe = regexp.MustCompile(`\blid (\d+)`) + rateRe = regexp.MustCompile(`\b(\d+)x([A-Z]+[0-9]*)\s*$`) +) + +// laneGbps is the signalling rate of one lane at each speed, as the link +// reports it. +var laneGbps = map[string]float64{ + "SDR": 2.5, "DDR": 5, "QDR": 10, "FDR10": 10, "FDR": 14, "EDR": 25, "HDR": 50, "NDR": 100, "XDR": 200, +} + +// Parse reads ibnetdiscover's default output. +func Parse(out string) (*Fabric, error) { + f := &Fabric{Devices: map[string]*Device{}} + seen := map[string]bool{} + var cur *Device + for _, raw := range strings.Split(out, "\n") { + line := strings.TrimSpace(raw) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + if m := header.FindStringSubmatch(line); m != nil { + kind := map[string]string{"Switch": "switch", "Ca": "ca", "Rt": "router"}[m[1]] + n, _ := strconv.Atoi(m[2]) + d := &Device{GUID: guid(m[4]), Kind: kind, Ports: n, Desc: m[5]} + if l := lidRe.FindStringSubmatch(m[6]); l != nil { + d.LID, _ = strconv.Atoi(l[1]) + } + if old, ok := f.Devices[d.GUID]; ok && old.Desc != "" && d.Desc == "" { + d.Desc = old.Desc + } + f.Devices[d.GUID] = d + cur = d + continue + } + m := portLn.FindStringSubmatch(line) + if m == nil || cur == nil { + continue + } + port, _ := strconv.Atoi(m[1]) + peer := guid(m[3]) + peerPort, _ := strconv.Atoi(m[4]) + comment := m[5] + // The peer's own block may come later; its description is in the + // comment, quoted, which is enough to name it until then. + if _, ok := f.Devices[peer]; !ok { + kind := map[string]string{"S": "switch", "H": "ca", "R": "router"}[m[2]] + d := &Device{GUID: peer, Kind: kind} + if q := strings.Index(comment, `"`); q >= 0 { + if e := strings.Index(comment[q+1:], `"`); e >= 0 { + d.Desc = comment[q+1 : q+1+e] + } + } + f.Devices[peer] = d + } + a, b := End{cur.GUID, port}, End{peer, peerPort} + key := linkKey(a, b) + if seen[key] { + continue + } + seen[key] = true + l := Link{A: a, B: b} + if r := rateRe.FindStringSubmatch(comment); r != nil { + l.Width, l.Speed = r[1]+"x", r[2] + lanes, _ := strconv.Atoi(r[1]) + l.Gbps = int(float64(lanes) * laneGbps[r[2]]) + } + f.Links = append(f.Links, l) + } + if len(f.Devices) == 0 { + return nil, fmt.Errorf("no devices in ibnetdiscover output") + } + return f, nil +} + +func guid(hex string) string { + hex = strings.ToLower(hex) + for len(hex) < 16 { + hex = "0" + hex + } + return "0x" + hex +} + +func linkKey(a, b End) string { + x, y := fmt.Sprintf("%s/%d", a.GUID, a.Port), fmt.Sprintf("%s/%d", b.GUID, b.Port) + if x > y { + x, y = y, x + } + return x + "-" + y +} + +// Host is an adapter the cluster knows: which node, which card. +type Host struct { + Node string + HCA string +} + +// View is a fabric's shape, ready to draw. +type View struct { + Switches []SwitchView `json:"switches"` + Hosts []HostView `json:"hosts"` + Links []LinkView `json:"links"` + // Tiers is how many switch levels: 1 for a single switch, 2 for + // leaf and spine, 3 with a core. + Tiers int `json:"tiers"` + // Expected is the rate most links of each kind trained at; a link below + // it is degraded. + ExpectedHost int `json:"expected_host_gbps"` + ExpectedSwitch int `json:"expected_switch_gbps"` + Degraded int `json:"degraded"` + // Rails is, per rail — an adapter's position on its node — the leaves it + // reaches, and Aligned whether each rail has its own. + Rails []Rail `json:"rails"` + Aligned bool `json:"rail_aligned"` + Notes []string `json:"notes,omitempty"` +} + +// SwitchView is a switch and its place in the tiers. +type SwitchView struct { + GUID string `json:"guid"` + Name string `json:"name"` + Tier int `json:"tier"` // 1 leaf, 2 spine, 3 core + Ports int `json:"ports"` + Used int `json:"used"` + LID int `json:"lid,omitempty"` + // Hosts counts adapters hanging off it. + Hosts int `json:"hosts"` +} + +// HostView is an adapter, and the node it is in when the cluster knows it. +type HostView struct { + GUID string `json:"guid"` + Desc string `json:"desc"` + Node string `json:"node,omitempty"` + HCA string `json:"hca,omitempty"` + Rail int `json:"rail"` + Leaf string `json:"leaf,omitempty"` +} + +// LinkView is a cable, with what it trained at and whether that is short of +// its kind. +type LinkView struct { + From string `json:"from"` + FromPort int `json:"from_port"` + To string `json:"to"` + ToPort int `json:"to_port"` + Rate string `json:"rate"` + Gbps int `json:"gbps"` + Host bool `json:"host"` + Degraded bool `json:"degraded"` +} + +// Rail is the leaves one rail's adapters reach. +type Rail struct { + Index int `json:"index"` + Leaves []string `json:"leaves"` +} + +// Build works out the shape. hosts maps adapter GUIDs to the cluster's nodes. +func Build(f *Fabric, hosts map[string]Host) View { + var v View + adj := map[string][]string{} + for _, l := range f.Links { + adj[l.A.GUID] = append(adj[l.A.GUID], l.B.GUID) + adj[l.B.GUID] = append(adj[l.B.GUID], l.A.GUID) + } + // A switch's tier is its distance from the nearest adapter: leaves hold + // adapters, spines hold leaves, cores hold spines. + tier := map[string]int{} + var frontier []string + for g, d := range f.Devices { + if d.Kind == "ca" { + tier[g] = 0 + frontier = append(frontier, g) + } + } + for len(frontier) > 0 { + var next []string + for _, g := range frontier { + for _, p := range adj[g] { + if _, ok := tier[p]; ok || f.Devices[p] == nil || f.Devices[p].Kind == "ca" { + continue + } + tier[p] = tier[g] + 1 + next = append(next, p) + } + } + frontier = next + } + + name := func(g string) string { + d := f.Devices[g] + if d == nil || d.Desc == "" { + return g + } + desc := d.Desc + // Mellanox switches say "MF0;leaf-01:MSB7700/U1": the name is between. + if i := strings.Index(desc, ";"); i >= 0 { + desc = desc[i+1:] + if j := strings.Index(desc, ":"); j >= 0 { + desc = desc[:j] + } + } + return desc + } + + // Expected rate per kind of link: the most common. + hostRates, swRates := map[int]int{}, map[int]int{} + isHost := func(l Link) bool { + return f.Devices[l.A.GUID].Kind == "ca" || f.Devices[l.B.GUID].Kind == "ca" + } + for _, l := range f.Links { + if isHost(l) { + hostRates[l.Gbps]++ + } else { + swRates[l.Gbps]++ + } + } + v.ExpectedHost, v.ExpectedSwitch = mode(hostRates), mode(swRates) + + used := map[string]int{} + hostsOn := map[string]int{} + leafOf := map[string]string{} + for _, l := range f.Links { + used[l.A.GUID]++ + used[l.B.GUID]++ + lv := LinkView{From: l.A.GUID, FromPort: l.A.Port, To: l.B.GUID, ToPort: l.B.Port, + Rate: l.Width + l.Speed, Gbps: l.Gbps, Host: isHost(l)} + // Keep adapters on the To side, so a drawing can go down the tiers. + if f.Devices[lv.From].Kind == "ca" { + lv.From, lv.FromPort, lv.To, lv.ToPort = lv.To, lv.ToPort, lv.From, lv.FromPort + } + if lv.Host { + hostsOn[lv.From]++ + leafOf[lv.To] = lv.From + lv.Degraded = l.Gbps > 0 && l.Gbps < v.ExpectedHost + } else { + lv.Degraded = l.Gbps > 0 && l.Gbps < v.ExpectedSwitch + if tier[lv.From] < tier[lv.To] { + lv.From, lv.FromPort, lv.To, lv.ToPort = lv.To, lv.ToPort, lv.From, lv.FromPort + } + } + if lv.Degraded { + v.Degraded++ + } + v.Links = append(v.Links, lv) + } + + for g, d := range f.Devices { + switch d.Kind { + case "switch": + t := tier[g] + if t == 0 { + t = 1 // a switch with nothing below it is drawn with the leaves + } + v.Switches = append(v.Switches, SwitchView{GUID: g, Name: name(g), Tier: t, Ports: d.Ports, + Used: used[g], LID: d.LID, Hosts: hostsOn[g]}) + v.Tiers = max(v.Tiers, t) + case "ca": + h := HostView{GUID: g, Desc: d.Desc, Leaf: leafOf[g]} + if known, ok := hosts[g]; ok { + h.Node, h.HCA = known.Node, known.HCA + } + v.Hosts = append(v.Hosts, h) + } + } + sort.Slice(v.Switches, func(i, j int) bool { + a, b := v.Switches[i], v.Switches[j] + if a.Tier != b.Tier { + return a.Tier > b.Tier + } + return a.Name < b.Name + }) + + // Rails: an adapter's position among its node's adapters on this fabric. + byNode := map[string][]int{} + for i, h := range v.Hosts { + key := h.Node + if key == "" { + key = strings.Fields(h.Desc + " ?")[0] // "node01 mlx5_0" names its host + } + byNode[key] = append(byNode[key], i) + } + railLeaves := map[int]map[string]bool{} + for _, idx := range byNode { + sort.Slice(idx, func(a, b int) bool { + return hcaName(v.Hosts[idx[a]]) < hcaName(v.Hosts[idx[b]]) + }) + for r, i := range idx { + v.Hosts[i].Rail = r + if railLeaves[r] == nil { + railLeaves[r] = map[string]bool{} + } + if v.Hosts[i].Leaf != "" { + railLeaves[r][v.Hosts[i].Leaf] = true + } + } + } + sort.Slice(v.Hosts, func(i, j int) bool { + a, b := v.Hosts[i], v.Hosts[j] + if a.Leaf != b.Leaf { + return a.Leaf < b.Leaf + } + return a.Desc < b.Desc + }) + owner := map[string]int{} + v.Aligned = len(railLeaves) > 1 + for r := 0; r < len(railLeaves); r++ { + var leaves []string + for l := range railLeaves[r] { + leaves = append(leaves, name(l)) + if prev, ok := owner[l]; ok && prev != r { + v.Aligned = false + } + owner[l] = r + } + sort.Strings(leaves) + v.Rails = append(v.Rails, Rail{Index: r, Leaves: leaves}) + } + if v.Degraded > 0 { + v.Notes = append(v.Notes, fmt.Sprintf("%d link(s) trained below the rest of their kind", v.Degraded)) + } + return v +} + +func hcaName(h HostView) string { + if h.HCA != "" { + return h.HCA + } + return h.Desc +} + +func mode(counts map[int]int) int { + best, n := 0, -1 + for v, c := range counts { + if v == 0 { + continue + } + if c > n || (c == n && v > best) { + best, n = v, c + } + } + return best +} diff --git a/internal/control/topology/topology_test.go b/internal/control/topology/topology_test.go new file mode 100644 index 0000000..ed9ec7c --- /dev/null +++ b/internal/control/topology/topology_test.go @@ -0,0 +1,149 @@ +package topology + +import ( + "fmt" + "strings" + "testing" +) + +// fatTree writes ibnetdiscover's default output for two spines, two leaves +// and four nodes of two adapters each, rail-aligned: every node's mlx5_0 on +// leaf-01 and mlx5_1 on leaf-02. node03's mlx5_1 trained at 2x instead of 4x. +func fatTree() string { + var b strings.Builder + w := func(format string, a ...any) { fmt.Fprintf(&b, format+"\n", a...) } + w("#\n# Topology file: generated on Mon Sep 29 10:00:00 2026\n#\n# Initiated from node 0000000000000a01 port 0000000000000a01\n") + spine := []string{"00000000000000c1", "00000000000000c2"} + leaf := []string{"00000000000000b1", "00000000000000b2"} + hca := func(n, r int) string { return fmt.Sprintf("000000000000%d%02d", r+10, n) } + for si, s := range spine { + w("vendid=0x2c9\ndevid=0xd2f0\nsysimgguid=0x%s\nswitchguid=0x%s(%s)", s, s, s) + w("Switch\t64 \"S-%s\"\t\t# \"MF0;spine-0%d:MQM9700/U1\" enhanced port 0 lid %d lmc 0", s, si+1, 10+si) + for li, l := range leaf { + for k := 0; k < 2; k++ { + w("[%d]\t\"S-%s\"[%d]\t\t# \"MF0;leaf-0%d:MQM9700/U1\" lid %d 4xNDR", li*2+k+1, l, 33+si*2+k, li+1, 20+li) + } + } + w("") + } + for li, l := range leaf { + w("vendid=0x2c9\ndevid=0xd2f0\nsysimgguid=0x%s\nswitchguid=0x%s(%s)", l, l, l) + w("Switch\t64 \"S-%s\"\t\t# \"MF0;leaf-0%d:MQM9700/U1\" enhanced port 0 lid %d lmc 0", l, li+1, 20+li) + for n := 1; n <= 4; n++ { + rate := "4xNDR" + if n == 3 && li == 1 { + rate = "2xNDR" + } + h := hca(n, li) + w("[%d]\t\"H-%s\"[1](%s) \t\t# \"node0%d mlx5_%d\" lid %d %s", n, h, h, n, li, 100+n*2+li, rate) + } + for si, s := range spine { + for k := 0; k < 2; k++ { + w("[%d]\t\"S-%s\"[%d]\t\t# \"MF0;spine-0%d:MQM9700/U1\" lid %d 4xNDR", 33+si*2+k, s, li*2+k+1, si+1, 10+si) + } + } + w("") + } + for n := 1; n <= 4; n++ { + for li, l := range leaf { + h := hca(n, li) + rate := "4xNDR" + if n == 3 && li == 1 { + rate = "2xNDR" + } + w("vendid=0x2c9\ndevid=0x1021\nsysimgguid=0x%s\ncaguid=0x%s", h, h) + w("Ca\t1 \"H-%s\"\t\t# \"node0%d mlx5_%d\"", h, n, li) + w("[1](%s) \t\"S-%s\"[%d]\t\t# lid %d lmc 0 \"MF0;leaf-0%d:MQM9700/U1\" lid %d %s", h, l, n, 100+n*2+li, li+1, 20+li, rate) + w("") + } + } + return b.String() +} + +func TestAFatTreeIsReadIntoTiers(t *testing.T) { + f, err := Parse(fatTree()) + if err != nil { + t.Fatal(err) + } + // 2 spines + 2 leaves + 8 adapters; 8 spine-leaf cables + 8 host cables, + // each once though both ends list it. + if len(f.Devices) != 12 || len(f.Links) != 16 { + t.Fatalf("%d devices, %d links", len(f.Devices), len(f.Links)) + } + hosts := map[string]Host{} + for n := 1; n <= 4; n++ { + for r := 0; r < 2; r++ { + hosts[fmt.Sprintf("0x000000000000%d%02d", r+10, n)] = Host{Node: fmt.Sprintf("n%d", n), HCA: fmt.Sprintf("mlx5_%d", r)} + } + } + v := Build(f, hosts) + if v.Tiers != 2 { + t.Errorf("tiers = %d", v.Tiers) + } + tiers := map[string]int{} + for _, s := range v.Switches { + tiers[s.Name] = s.Tier + } + if tiers["spine-01"] != 2 || tiers["leaf-02"] != 1 { + t.Errorf("tiers = %v", tiers) + } + if v.ExpectedHost != 400 || v.ExpectedSwitch != 400 { + t.Errorf("expected rates %d / %d", v.ExpectedHost, v.ExpectedSwitch) + } + // node03's second adapter trained at half width. + if v.Degraded != 1 { + t.Errorf("degraded = %d", v.Degraded) + } + for _, l := range v.Links { + if l.Degraded && (l.Rate != "2xNDR" || !l.Host) { + t.Errorf("wrong link degraded: %+v", l) + } + } + // Every node's first adapter on leaf-01, its second on leaf-02: aligned. + if !v.Aligned || len(v.Rails) != 2 || v.Rails[0].Leaves[0] != "leaf-01" || v.Rails[1].Leaves[0] != "leaf-02" { + t.Errorf("rails = %+v aligned=%v", v.Rails, v.Aligned) + } + mapped := 0 + for _, h := range v.Hosts { + if h.Node != "" { + mapped++ + } + } + if mapped != 8 { + t.Errorf("%d adapters mapped to nodes", mapped) + } +} + +func TestCrossedRailsAreNotAligned(t *testing.T) { + // node02's two cables swapped at the leaves: its mlx5_0 on leaf-02 and + // its mlx5_1 on leaf-01, so each leaf now carries both rails. + f, err := Parse(fatTree()) + if err != nil { + t.Fatal(err) + } + swap := map[string]string{"0x0000000000001002": "0x00000000000000b2", "0x0000000000001102": "0x00000000000000b1"} + for i, l := range f.Links { + for _, host := range []*End{&f.Links[i].A, &f.Links[i].B} { + if leaf, ok := swap[host.GUID]; ok { + other := &f.Links[i].A + if host == other { + other = &f.Links[i].B + } + other.GUID = leaf + } + } + _ = l + } + v := Build(f, map[string]Host{ + "0x0000000000001002": {Node: "n2", HCA: "mlx5_0"}, "0x0000000000001102": {Node: "n2", HCA: "mlx5_1"}, + }) + if v.Aligned { + t.Errorf("crossed rails read as aligned: %+v", v.Rails) + } +} + +func TestNotAMapIsAnError(t *testing.T) { + if _, err := Parse("ibwarn: [1234] mad_rpc_open_port: can't open UMAD port ((null):0)"); err == nil { + t.Error("expected an error") + } +} diff --git a/internal/router/handlers.go b/internal/router/handlers.go index 95e667c..545734c 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -164,6 +164,9 @@ func (r *Router) Handler() http.Handler { }, r.log))) mux.HandleFunc("GET /admin/nodes/{id}/logs", pages.AdminPage( web.NodeLogsPage(r.clusterView, false, version.Version))) + mux.HandleFunc("GET /admin/fabrics", pages.AdminPage(web.FabricsPage(r.clusterView, false, version.Version))) + mux.HandleFunc("GET /admin/fabrics/{id}", pages.AdminPage(web.FabricsPage(r.clusterView, false, version.Version))) + mux.HandleFunc("GET /admin/fabrics/{id}/topology.json", pages.AdminPage(web.FabricTopologyJSON(r.clusterView))) mux.HandleFunc("GET /admin/nodes/{id}", pages.AdminPage( web.NodePage(r.clusterView, r.history, r.histWindow, false, version.Version, r.log))) pages.RegisterAdminUsers(mux, r.store) diff --git a/internal/web/fabric_page.go b/internal/web/fabric_page.go new file mode 100644 index 0000000..e15449a --- /dev/null +++ b/internal/web/fabric_page.go @@ -0,0 +1,82 @@ +package web + +import ( + "encoding/json" + "html/template" + "net/http" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" +) + +// TopologySource is a Source that can map its fabrics. +type TopologySource interface { + Topologies() []adminapi.TopologyView + Topology(id string) (adminapi.TopologyView, bool) +} + +type fabricPageData struct { + Nav Nav + OnAdminListener bool + Base string + List []adminapi.TopologyView + Current *adminapi.TopologyView + // Data is the current fabric as JSON, for the drawing to start from + // without a round trip. json.Marshal escapes <, > and &, so a switch + // description cannot close the script it sits in. + Data template.JS + Version string + Now string +} + +// FabricsPage shows every physical fabric and, for one, its map. +func FabricsPage(src Source, onAdminListener bool, version string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + ts, ok := src.(TopologySource) + if !ok { + http.NotFound(w, r) + return + } + d := fabricPageData{Nav: navFrom(r, "cluster"), OnAdminListener: onAdminListener, Base: "/admin/fabrics", + List: ts.Topologies(), Version: version, Now: time.Now().UTC().Format(time.RFC3339)} + if onAdminListener { + d.Base = "/fabrics" + } + id := r.PathValue("id") + if id == "" && len(d.List) > 0 { + id = d.List[0].ID + } + if id != "" { + t, ok := ts.Topology(id) + if !ok { + http.NotFound(w, r) + return + } + d.Current = &t + b, _ := json.Marshal(t) + d.Data = template.JS(b) //nolint:gosec // json.Marshal escapes <, > and & + } + noStore(w) + render(w, "fabrics", d) + } +} + +// FabricTopologyJSON is one fabric's map and live state, for the drawing to +// refresh from. +func FabricTopologyJSON(src Source) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + ts, ok := src.(TopologySource) + if !ok { + http.NotFound(w, r) + return + } + t, ok := ts.Topology(r.PathValue("id")) + if !ok { + http.NotFound(w, r) + return + } + noStore(w) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(t) + } +} diff --git a/internal/web/node_page_test.go b/internal/web/node_page_test.go index c9404e2..636d899 100644 --- a/internal/web/node_page_test.go +++ b/internal/web/node_page_test.go @@ -7,6 +7,7 @@ import ( "time" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/topology" "gitea.mynodes.xyz/ilker/llmgw/internal/promq" ) @@ -93,7 +94,7 @@ func TestANodeNotMeasuredLatelySaysSoRatherThanZero(t *testing.T) { func TestANodesPageShowsItsFabricPorts(t *testing.T) { _, body := get(t, nodeMux(busySource{}, nil), "http://x/admin/nodes/llmgw-node01", nil) for _, want := range []string{ - "
mlx5_0/1", "400 Gb/s", "4.0 GiB",
+ "mlx5_0/1", "400 Gb/s", "4.0 GiB",
`