Map every InfiniBand fabric, draw it, and place replicas by it

The control plane asks one node on each physical fabric to run
ibnetdiscover every five minutes and parses the text itself, so a
format the parser has not met is a control plane upgrade, not one on
every node. From the map it works out the switch tiers (leaf, spine,
core, by distance from the adapters), which adapter is which node's,
which cables trained below the rate the rest of their kind run at, and
whether the rails line up: every node's Nth adapter on its own leaf.

A page per fabric (/admin/fabrics) draws it: switches in their tiers,
adapters fanned out under their leaves, cables coloured by rate and
bundled, degraded ones marching red with a marker, and the cluster's
own adapters live - down ports glowing red, and particles running each
cable as fast as its adapter is sending and receiving, from the
counters. It zooms, pans, fades what a hovered switch or adapter does
not touch, opens a panel of detail on a click, colours by rail, finds a
node, and refreshes every fifteen seconds. The data is also at
/api/v1/fabrics/{id}/topology.

Placement reads the map: a replica spanning nodes takes nodes of one
leaf together when it can, and nodes whose cable trained short last.

Static assets are named by their content, as the stylesheet already
was.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-29 08:46:23 +02:00
1 parent 23648fa19a
commit 5bbc619a31
23 files changed
+2171 -83

No files matched your search

+268 -80
View File
@@ -102,6 +102,7 @@ type AgentMessage struct {
// *AgentMessage_Heartbeat
// *AgentMessage_Shard
// *AgentMessage_Logs
// *AgentMessage_FabricScan
Msg isAgentMessage_Msg `protobuf_oneof:"msg"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
@@ -180,6 +181,15 @@ func (x *AgentMessage) GetLogs() *LogsReply {
return nil
}
func (x *AgentMessage) GetFabricScan() *FabricScanReply {
if x != nil {
if x, ok := x.Msg.(*AgentMessage_FabricScan); ok {
return x.FabricScan
}
}
return nil
}
type isAgentMessage_Msg interface {
isAgentMessage_Msg()
}
@@ -200,6 +210,10 @@ type AgentMessage_Logs struct {
Logs *LogsReply `protobuf:"bytes,4,opt,name=logs,proto3,oneof"`
}
type AgentMessage_FabricScan struct {
FabricScan *FabricScanReply `protobuf:"bytes,5,opt,name=fabric_scan,json=fabricScan,proto3,oneof"`
}
func (*AgentMessage_Register) isAgentMessage_Msg() {}
func (*AgentMessage_Heartbeat) isAgentMessage_Msg() {}
@@ -208,6 +222,8 @@ func (*AgentMessage_Shard) isAgentMessage_Msg() {}
func (*AgentMessage_Logs) isAgentMessage_Msg() {}
func (*AgentMessage_FabricScan) isAgentMessage_Msg() {}
// ControlMessage is anything the control plane tells a node.
type ControlMessage struct {
state protoimpl.MessageState `protogen:"open.v1"`
@@ -220,6 +236,7 @@ type ControlMessage struct {
// *ControlMessage_RemoveWeights
// *ControlMessage_Provision
// *ControlMessage_Logs
// *ControlMessage_FabricScan
Msg isControlMessage_Msg `protobuf_oneof:"msg"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
@@ -325,6 +342,15 @@ func (x *ControlMessage) GetLogs() *LogsRequest {
return nil
}
func (x *ControlMessage) GetFabricScan() *FabricScanRequest {
if x != nil {
if x, ok := x.Msg.(*ControlMessage_FabricScan); ok {
return x.FabricScan
}
}
return nil
}
type isControlMessage_Msg interface {
isControlMessage_Msg()
}
@@ -357,6 +383,10 @@ type ControlMessage_Logs struct {
Logs *LogsRequest `protobuf:"bytes,7,opt,name=logs,proto3,oneof"`
}
type ControlMessage_FabricScan struct {
FabricScan *FabricScanRequest `protobuf:"bytes,8,opt,name=fabric_scan,json=fabricScan,proto3,oneof"`
}
func (*ControlMessage_Start) isControlMessage_Msg() {}
func (*ControlMessage_Stop) isControlMessage_Msg() {}
@@ -371,6 +401,8 @@ func (*ControlMessage_Provision) isControlMessage_Msg() {}
func (*ControlMessage_Logs) isControlMessage_Msg() {}
func (*ControlMessage_FabricScan) isControlMessage_Msg() {}
// Register is the first message on every stream. It carries the node's full
// hardware inventory, so the control plane needs no separate discovery call and
// a reconnecting agent automatically refreshes what the scheduler knows.
@@ -2833,17 +2865,154 @@ func (x *LogsReply) GetError() string {
return ""
}
// FabricScanRequest asks a node to map the fabric one of its ports is on,
// with ibnetdiscover: every switch, every adapter and every link between
// them. One node per fabric is asked, every few minutes; a fabric looks the
// same from all of its ports.
type FabricScanRequest struct {
state protoimpl.MessageState `protogen:"open.v1"`
RequestId string `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"`
Hca string `protobuf:"bytes,2,opt,name=hca,proto3" json:"hca,omitempty"`
Port int32 `protobuf:"varint,3,opt,name=port,proto3" json:"port,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *FabricScanRequest) Reset() {
*x = FabricScanRequest{}
mi := &file_llmgw_v1_agent_proto_msgTypes[27]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *FabricScanRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*FabricScanRequest) ProtoMessage() {}
func (x *FabricScanRequest) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[27]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use FabricScanRequest.ProtoReflect.Descriptor instead.
func (*FabricScanRequest) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{27}
}
func (x *FabricScanRequest) GetRequestId() string {
if x != nil {
return x.RequestId
}
return ""
}
func (x *FabricScanRequest) GetHca() string {
if x != nil {
return x.Hca
}
return ""
}
func (x *FabricScanRequest) GetPort() int32 {
if x != nil {
return x.Port
}
return 0
}
// FabricScanReply is ibnetdiscover's output as it printed it. Parsed by the
// control plane, so a format the parser does not yet know is a control plane
// upgrade rather than one on every node.
type FabricScanReply struct {
state protoimpl.MessageState `protogen:"open.v1"`
RequestId string `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"`
Output string `protobuf:"bytes,2,opt,name=output,proto3" json:"output,omitempty"`
Error string `protobuf:"bytes,3,opt,name=error,proto3" json:"error,omitempty"`
TookMs int64 `protobuf:"varint,4,opt,name=took_ms,json=tookMs,proto3" json:"took_ms,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *FabricScanReply) Reset() {
*x = FabricScanReply{}
mi := &file_llmgw_v1_agent_proto_msgTypes[28]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *FabricScanReply) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*FabricScanReply) ProtoMessage() {}
func (x *FabricScanReply) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[28]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use FabricScanReply.ProtoReflect.Descriptor instead.
func (*FabricScanReply) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{28}
}
func (x *FabricScanReply) GetRequestId() string {
if x != nil {
return x.RequestId
}
return ""
}
func (x *FabricScanReply) GetOutput() string {
if x != nil {
return x.Output
}
return ""
}
func (x *FabricScanReply) GetError() string {
if x != nil {
return x.Error
}
return ""
}
func (x *FabricScanReply) GetTookMs() int64 {
if x != nil {
return x.TookMs
}
return 0
}
var File_llmgw_v1_agent_proto protoreflect.FileDescriptor
const file_llmgw_v1_agent_proto_rawDesc = "" +
"\n" +
"\x14llmgw/v1/agent.proto\x12\bllmgw.v1\"\xd6\x01\n" +
"\x14llmgw/v1/agent.proto\x12\bllmgw.v1\"\x94\x02\n" +
"\fAgentMessage\x120\n" +
"\bregister\x18\x01 \x01(\v2\x12.llmgw.v1.RegisterH\x00R\bregister\x123\n" +
"\theartbeat\x18\x02 \x01(\v2\x13.llmgw.v1.HeartbeatH\x00R\theartbeat\x12-\n" +
"\x05shard\x18\x03 \x01(\v2\x15.llmgw.v1.ShardStatusH\x00R\x05shard\x12)\n" +
"\x04logs\x18\x04 \x01(\v2\x13.llmgw.v1.LogsReplyH\x00R\x04logsB\x05\n" +
"\x03msg\"\xe5\x02\n" +
"\x04logs\x18\x04 \x01(\v2\x13.llmgw.v1.LogsReplyH\x00R\x04logs\x12<\n" +
"\vfabric_scan\x18\x05 \x01(\v2\x19.llmgw.v1.FabricScanReplyH\x00R\n" +
"fabricScanB\x05\n" +
"\x03msg\"\xa5\x03\n" +
"\x0eControlMessage\x12,\n" +
"\x05start\x18\x01 \x01(\v2\x14.llmgw.v1.StartShardH\x00R\x05start\x12)\n" +
"\x04stop\x18\x02 \x01(\v2\x13.llmgw.v1.StopShardH\x00R\x04stop\x12,\n" +
@@ -2851,7 +3020,9 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\x03ack\x18\x04 \x01(\v2\r.llmgw.v1.AckH\x00R\x03ack\x12@\n" +
"\x0eremove_weights\x18\x05 \x01(\v2\x17.llmgw.v1.RemoveWeightsH\x00R\rremoveWeights\x123\n" +
"\tprovision\x18\x06 \x01(\v2\x13.llmgw.v1.ProvisionH\x00R\tprovision\x12+\n" +
"\x04logs\x18\a \x01(\v2\x15.llmgw.v1.LogsRequestH\x00R\x04logsB\x05\n" +
"\x04logs\x18\a \x01(\v2\x15.llmgw.v1.LogsRequestH\x00R\x04logs\x12>\n" +
"\vfabric_scan\x18\b \x01(\v2\x1b.llmgw.v1.FabricScanRequestH\x00R\n" +
"fabricScanB\x05\n" +
"\x03msg\"\xf2\x05\n" +
"\bRegister\x12\x17\n" +
"\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1a\n" +
@@ -3111,7 +3282,18 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"request_id\x18\x01 \x01(\tR\trequestId\x12\x14\n" +
"\x05lines\x18\x02 \x03(\tR\x05lines\x12\x18\n" +
"\asources\x18\x03 \x03(\tR\asources\x12\x14\n" +
"\x05error\x18\x04 \x01(\tR\x05error2O\n" +
"\x05error\x18\x04 \x01(\tR\x05error\"X\n" +
"\x11FabricScanRequest\x12\x1d\n" +
"\n" +
"request_id\x18\x01 \x01(\tR\trequestId\x12\x10\n" +
"\x03hca\x18\x02 \x01(\tR\x03hca\x12\x12\n" +
"\x04port\x18\x03 \x01(\x05R\x04port\"w\n" +
"\x0fFabricScanReply\x12\x1d\n" +
"\n" +
"request_id\x18\x01 \x01(\tR\trequestId\x12\x16\n" +
"\x06output\x18\x02 \x01(\tR\x06output\x12\x14\n" +
"\x05error\x18\x03 \x01(\tR\x05error\x12\x17\n" +
"\atook_ms\x18\x04 \x01(\x03R\x06tookMs2O\n" +
"\fAgentService\x12?\n" +
"\aConnect\x12\x16.llmgw.v1.AgentMessage\x1a\x18.llmgw.v1.ControlMessage(\x010\x01B\x93\x01\n" +
"\fcom.llmgw.v1B\n" +
@@ -3130,87 +3312,91 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte {
}
var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 33)
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 35)
var file_llmgw_v1_agent_proto_goTypes = []any{
(ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State
(*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage
(*ControlMessage)(nil), // 2: llmgw.v1.ControlMessage
(*Register)(nil), // 3: llmgw.v1.Register
(*Host)(nil), // 4: llmgw.v1.Host
(*Storage)(nil), // 5: llmgw.v1.Storage
(*RunningShard)(nil), // 6: llmgw.v1.RunningShard
(*GPU)(nil), // 7: llmgw.v1.GPU
(*FabricAttachment)(nil), // 8: llmgw.v1.FabricAttachment
(*LocalWeights)(nil), // 9: llmgw.v1.LocalWeights
(*NodeLoad)(nil), // 10: llmgw.v1.NodeLoad
(*ShardLoad)(nil), // 11: llmgw.v1.ShardLoad
(*IBPort)(nil), // 12: llmgw.v1.IBPort
(*SubnetManager)(nil), // 13: llmgw.v1.SubnetManager
(*Heartbeat)(nil), // 14: llmgw.v1.Heartbeat
(*Provision)(nil), // 15: llmgw.v1.Provision
(*EngineWant)(nil), // 16: llmgw.v1.EngineWant
(*ProvisionStatus)(nil), // 17: llmgw.v1.ProvisionStatus
(*ShardStatus)(nil), // 18: llmgw.v1.ShardStatus
(*FetchProgress)(nil), // 19: llmgw.v1.FetchProgress
(*StartShard)(nil), // 20: llmgw.v1.StartShard
(*EngineProfile)(nil), // 21: llmgw.v1.EngineProfile
(*RemoveWeights)(nil), // 22: llmgw.v1.RemoveWeights
(*StopShard)(nil), // 23: llmgw.v1.StopShard
(*DrainShard)(nil), // 24: llmgw.v1.DrainShard
(*Ack)(nil), // 25: llmgw.v1.Ack
(*LogsRequest)(nil), // 26: llmgw.v1.LogsRequest
(*LogsReply)(nil), // 27: llmgw.v1.LogsReply
nil, // 28: llmgw.v1.Register.LabelsEntry
nil, // 29: llmgw.v1.Register.EngineVersionsEntry
nil, // 30: llmgw.v1.GPU.NicPathsEntry
nil, // 31: llmgw.v1.IBPort.CountersEntry
nil, // 32: llmgw.v1.Heartbeat.EngineVersionsEntry
nil, // 33: llmgw.v1.EngineProfile.EnvEntry
(ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State
(*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage
(*ControlMessage)(nil), // 2: llmgw.v1.ControlMessage
(*Register)(nil), // 3: llmgw.v1.Register
(*Host)(nil), // 4: llmgw.v1.Host
(*Storage)(nil), // 5: llmgw.v1.Storage
(*RunningShard)(nil), // 6: llmgw.v1.RunningShard
(*GPU)(nil), // 7: llmgw.v1.GPU
(*FabricAttachment)(nil), // 8: llmgw.v1.FabricAttachment
(*LocalWeights)(nil), // 9: llmgw.v1.LocalWeights
(*NodeLoad)(nil), // 10: llmgw.v1.NodeLoad
(*ShardLoad)(nil), // 11: llmgw.v1.ShardLoad
(*IBPort)(nil), // 12: llmgw.v1.IBPort
(*SubnetManager)(nil), // 13: llmgw.v1.SubnetManager
(*Heartbeat)(nil), // 14: llmgw.v1.Heartbeat
(*Provision)(nil), // 15: llmgw.v1.Provision
(*EngineWant)(nil), // 16: llmgw.v1.EngineWant
(*ProvisionStatus)(nil), // 17: llmgw.v1.ProvisionStatus
(*ShardStatus)(nil), // 18: llmgw.v1.ShardStatus
(*FetchProgress)(nil), // 19: llmgw.v1.FetchProgress
(*StartShard)(nil), // 20: llmgw.v1.StartShard
(*EngineProfile)(nil), // 21: llmgw.v1.EngineProfile
(*RemoveWeights)(nil), // 22: llmgw.v1.RemoveWeights
(*StopShard)(nil), // 23: llmgw.v1.StopShard
(*DrainShard)(nil), // 24: llmgw.v1.DrainShard
(*Ack)(nil), // 25: llmgw.v1.Ack
(*LogsRequest)(nil), // 26: llmgw.v1.LogsRequest
(*LogsReply)(nil), // 27: llmgw.v1.LogsReply
(*FabricScanRequest)(nil), // 28: llmgw.v1.FabricScanRequest
(*FabricScanReply)(nil), // 29: llmgw.v1.FabricScanReply
nil, // 30: llmgw.v1.Register.LabelsEntry
nil, // 31: llmgw.v1.Register.EngineVersionsEntry
nil, // 32: llmgw.v1.GPU.NicPathsEntry
nil, // 33: llmgw.v1.IBPort.CountersEntry
nil, // 34: llmgw.v1.Heartbeat.EngineVersionsEntry
nil, // 35: llmgw.v1.EngineProfile.EnvEntry
}
var file_llmgw_v1_agent_proto_depIdxs = []int32{
3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register
14, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat
18, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus
27, // 3: llmgw.v1.AgentMessage.logs:type_name -> llmgw.v1.LogsReply
20, // 4: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard
23, // 5: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard
24, // 6: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard
25, // 7: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack
22, // 8: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights
15, // 9: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision
26, // 10: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest
28, // 11: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
7, // 12: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU
8, // 13: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment
29, // 14: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
6, // 15: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard
9, // 16: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights
5, // 17: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage
4, // 18: llmgw.v1.Register.host:type_name -> llmgw.v1.Host
0, // 19: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
30, // 20: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry
11, // 21: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad
31, // 22: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry
13, // 23: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager
5, // 24: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
9, // 25: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
32, // 26: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
17, // 27: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus
10, // 28: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad
12, // 29: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort
8, // 30: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment
16, // 31: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant
0, // 32: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
19, // 33: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
21, // 34: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
33, // 35: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
1, // 36: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
2, // 37: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
37, // [37:38] is the sub-list for method output_type
36, // [36:37] is the sub-list for method input_type
36, // [36:36] is the sub-list for extension type_name
36, // [36:36] is the sub-list for extension extendee
0, // [0:36] is the sub-list for field type_name
29, // 4: llmgw.v1.AgentMessage.fabric_scan:type_name -> llmgw.v1.FabricScanReply
20, // 5: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard
23, // 6: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard
24, // 7: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard
25, // 8: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack
22, // 9: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights
15, // 10: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision
26, // 11: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest
28, // 12: llmgw.v1.ControlMessage.fabric_scan:type_name -> llmgw.v1.FabricScanRequest
30, // 13: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
7, // 14: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU
8, // 15: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment
31, // 16: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
6, // 17: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard
9, // 18: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights
5, // 19: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage
4, // 20: llmgw.v1.Register.host:type_name -> llmgw.v1.Host
0, // 21: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
32, // 22: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry
11, // 23: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad
33, // 24: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry
13, // 25: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager
5, // 26: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
9, // 27: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
34, // 28: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
17, // 29: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus
10, // 30: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad
12, // 31: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort
8, // 32: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment
16, // 33: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant
0, // 34: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
19, // 35: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
21, // 36: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
35, // 37: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
1, // 38: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
2, // 39: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
39, // [39:40] is the sub-list for method output_type
38, // [38:39] is the sub-list for method input_type
38, // [38:38] is the sub-list for extension type_name
38, // [38:38] is the sub-list for extension extendee
0, // [0:38] is the sub-list for field type_name
}
func init() { file_llmgw_v1_agent_proto_init() }
@@ -3223,6 +3409,7 @@ func file_llmgw_v1_agent_proto_init() {
(*AgentMessage_Heartbeat)(nil),
(*AgentMessage_Shard)(nil),
(*AgentMessage_Logs)(nil),
(*AgentMessage_FabricScan)(nil),
}
file_llmgw_v1_agent_proto_msgTypes[1].OneofWrappers = []any{
(*ControlMessage_Start)(nil),
@@ -3232,6 +3419,7 @@ func file_llmgw_v1_agent_proto_init() {
(*ControlMessage_RemoveWeights)(nil),
(*ControlMessage_Provision)(nil),
(*ControlMessage_Logs)(nil),
(*ControlMessage_FabricScan)(nil),
}
type x struct{}
out := protoimpl.TypeBuilder{
@@ -3239,7 +3427,7 @@ func file_llmgw_v1_agent_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)),
NumEnums: 1,
NumMessages: 33,
NumMessages: 35,
NumExtensions: 0,
NumServices: 1,
},
+22
View File
@@ -24,6 +24,7 @@ message AgentMessage {
Heartbeat heartbeat = 2;
ShardStatus shard = 3;
LogsReply logs = 4;
FabricScanReply fabric_scan = 5;
}
}
@@ -37,6 +38,7 @@ message ControlMessage {
RemoveWeights remove_weights = 5;
Provision provision = 6;
LogsRequest logs = 7;
FabricScanRequest fabric_scan = 8;
}
}
@@ -543,3 +545,23 @@ message LogsReply {
// error says why the lines are absent: an unknown source, say.
string error = 4;
}
// FabricScanRequest asks a node to map the fabric one of its ports is on,
// with ibnetdiscover: every switch, every adapter and every link between
// them. One node per fabric is asked, every few minutes; a fabric looks the
// same from all of its ports.
message FabricScanRequest {
string request_id = 1;
string hca = 2;
int32 port = 3;
}
// FabricScanReply is ibnetdiscover's output as it printed it. Parsed by the
// control plane, so a format the parser does not yet know is a control plane
// upgrade rather than one on every node.
message FabricScanReply {
string request_id = 1;
string output = 2;
string error = 3;
int64 took_ms = 4;
}
+20
View File
@@ -1123,6 +1123,23 @@ func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out cha
case <-ctx.Done():
}
}()
case *llmgwv1.ControlMessage_FabricScan:
// A scan takes seconds to minutes: off the receive loop entirely.
go func(req *llmgwv1.FabricScanRequest) {
start := time.Now()
reply := &llmgwv1.FabricScanReply{RequestId: req.GetRequestId()}
text, err := scanFabric(ctx, req.GetHca(), int(req.GetPort()))
if err != nil {
reply.Error = err.Error()
} else {
reply.Output = text
}
reply.TookMs = time.Since(start).Milliseconds()
select {
case out <- &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_FabricScan{FabricScan: reply}}:
case <-ctx.Done():
}
}(m.FabricScan)
case *llmgwv1.ControlMessage_Ack:
if !m.Ack.GetOk() {
a.log.Warn("control plane rejected a message",
@@ -1639,6 +1656,9 @@ func ResolveManagement(c Config) (addr, iface string) {
func smKey(hca string, port int) string { return fmt.Sprintf("%s/%d", hca, port) }
// scanFabric is discovery.ScanFabric, replaced in tests.
var scanFabric = discovery.ScanFabric
// querySMs is discovery.QuerySMs, replaced in tests.
var querySMs = func(hca string, port int) ([]discovery.SMRecord, error) {
return discovery.QuerySMs(context.Background(), hca, port)
+20
View File
@@ -4,6 +4,7 @@ import (
"context"
"crypto/tls"
"errors"
"fmt"
"log/slog"
"net"
"net/http"
@@ -861,3 +862,22 @@ func TestEachFabricsSubnetManagersAreAskedOnce(t *testing.T) {
t.Errorf("listed %d times, carried on %d ports; want once, on one", lists, withList)
}
}
func TestAFabricScanIsAskedForAndAnswered(t *testing.T) {
// One node per fabric maps it; the control plane asks and parses.
old := scanFabric
t.Cleanup(func() { scanFabric = old })
var asked string
scanFabric = func(_ context.Context, hca string, port int) (string, error) {
asked = fmt.Sprintf("%s/%d", hca, port)
return "Switch\t36 \"S-00000000000000a1\"\t\t# \"MF0;leaf-a:QM8700/U1\" enhanced port 0 lid 1 lmc 0\n", nil
}
h := start(t)
r, err := h.svc.ScanFabric(t.Context(), "n1", "mlx5_0", 1)
if err != nil {
t.Fatal(err)
}
if asked != "mlx5_0/1" || !strings.Contains(r.GetOutput(), "leaf-a") || r.GetError() != "" {
t.Errorf("asked %q, got %+v", asked, r)
}
}
+26
View File
@@ -144,3 +144,29 @@ func QuerySMs(ctx context.Context, hca string, port int) ([]SMRecord, error) {
}
return ParseSMIR(string(out))
}
// ScanFabric maps the fabric a port is on with ibnetdiscover: every switch,
// adapter and link. It can take a while on a large fabric, so it is bounded,
// and its output is capped: a fabric of thousands of ports is a few
// megabytes of text.
func ScanFabric(ctx context.Context, hca string, port int) (string, error) {
bin, err := exec.LookPath("ibnetdiscover")
if err != nil {
return "", ErrNoDiags
}
ctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
defer cancel()
out, err := exec.CommandContext(ctx, bin, "-C", hca, "-P", strconv.Itoa(port)).Output()
if err != nil {
var ee *exec.ExitError
if errors.As(err, &ee) {
return "", fmt.Errorf("ibnetdiscover on %s/%d: %w: %s", hca, port, err, strings.TrimSpace(string(ee.Stderr)))
}
return "", fmt.Errorf("ibnetdiscover on %s/%d: %w", hca, port, err)
}
const limit = 16 << 20
if len(out) > limit {
return "", fmt.Errorf("ibnetdiscover printed %d MB, more than the %d MB a scan may send", len(out)>>20, limit>>20)
}
return string(out), nil
}
+5
View File
@@ -208,6 +208,9 @@ func newServerCmd() *cobra.Command {
adminMux.Handle("GET /_static/", web.Static())
adminMux.HandleFunc("GET /nodes/{id}/logs", web.NodeLogsPage(api, true, version.Version))
adminMux.HandleFunc("GET /nodes/{id}", web.NodePage(api, history, cfg.Monitoring.Window, true, version.Version, log))
adminMux.HandleFunc("GET /fabrics", web.FabricsPage(api, true, version.Version))
adminMux.HandleFunc("GET /fabrics/{id}", web.FabricsPage(api, true, version.Version))
adminMux.HandleFunc("GET /fabrics/{id}/topology.json", web.FabricTopologyJSON(api))
// The same page on the client-facing port at /admin, behind a
// session and an admin flag. Handing the router the data rather
@@ -417,6 +420,8 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live
svc.SetLeadership(lead)
}
fwd.set(svc)
// Every InfiniBand fabric mapped every five minutes, through one node.
go svc.ScanFabrics(ctx, 5*time.Minute)
ctl.Start(ctx)
return ctl, svc, nil
+120
View File
@@ -22,6 +22,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/topology"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
@@ -52,6 +53,14 @@ func (a *API) Register(mux *http.ServeMux) {
mux.HandleFunc("GET /api/v1/fabrics", func(w http.ResponseWriter, _ *http.Request) {
writeJSON(w, http.StatusOK, a.Fabrics())
})
mux.HandleFunc("GET /api/v1/fabrics/{id}/topology", func(w http.ResponseWriter, r *http.Request) {
t, ok := a.Topology(r.PathValue("id"))
if !ok {
http.NotFound(w, r)
return
}
writeJSON(w, http.StatusOK, t)
})
mux.HandleFunc("GET /api/v1/models", a.listModels)
mux.HandleFunc("GET /api/v1/replicas", a.listReplicas)
// Listing is a read and deleting is a separate verb, so a mistyped
@@ -839,3 +848,114 @@ func (a *API) Fabrics() fabrics.Report {
inv := a.ctl.Inventory()
return fabrics.Analyze(inv.Nodes(), inv.SMChanges(), time.Now())
}
// TopologyView is one fabric's map, with the cluster's own adapters on it
// live: their state, rate and traffic from the last heartbeat.
type TopologyView struct {
ID string `json:"id"`
Key string `json:"key"`
Label string `json:"label"`
Declared []string `json:"declared"`
ScannedAt time.Time `json:"scanned_at"`
ScannedBy string `json:"scanned_by,omitempty"`
TookMs int64 `json:"took_ms"`
Err string `json:"error,omitempty"`
Map *topology.View `json:"map,omitempty"`
Live map[string]Live `json:"live"`
// Problems are the fabric's own, from the analysis of its identity and
// subnet managers.
Problems []fabrics.Problem `json:"problems,omitempty"`
SM *fabrics.SM `json:"sm,omitempty"`
SMs []fabrics.SM `json:"sms,omitempty"`
}
// Live is one of the cluster's adapters as its node last reported it.
type Live struct {
Node string `json:"node"`
HCA string `json:"hca"`
State string `json:"state"`
Phys string `json:"phys_state"`
Gbps int `json:"gbps"`
TxBps float64 `json:"tx_bps"`
RxBps float64 `json:"rx_bps"`
Errors map[string]uint64 `json:"errors,omitempty"`
}
// FabricID is a physical fabric's key as an address can carry it.
func FabricID(key string) string {
return strings.NewReplacer(":", "-", "/", "_").Replace(key)
}
// Topologies lists every physical fabric with what is known of its map.
func (a *API) Topologies() []TopologyView {
rep := a.Fabrics()
scans := a.ctl.Inventory().Scans()
out := make([]TopologyView, 0, len(rep.Physical))
for _, ph := range rep.Physical {
t := TopologyView{ID: FabricID(ph.Key), Key: ph.Key, Label: ph.Label, Declared: ph.Declared}
if sc, ok := scans[ph.Key]; ok {
t.ScannedAt, t.ScannedBy, t.TookMs, t.Err = sc.At, sc.Node+" "+sc.Port, sc.Took.Milliseconds(), sc.Err
}
out = append(out, t)
}
return out
}
// Topology is one fabric's map, by the id Topologies gives it.
func (a *API) Topology(id string) (TopologyView, bool) {
rep := a.Fabrics()
var ph *fabrics.Physical
for i := range rep.Physical {
if FabricID(rep.Physical[i].Key) == id {
ph = &rep.Physical[i]
}
}
if ph == nil {
return TopologyView{}, false
}
t := TopologyView{ID: id, Key: ph.Key, Label: ph.Label, Declared: ph.Declared, Live: map[string]Live{},
SM: ph.SM, SMs: ph.SMs}
for _, p := range rep.Problems {
for _, d := range ph.Declared {
if p.Fabric == d {
t.Problems = append(t.Problems, p)
break
}
}
}
hosts := map[string]topology.Host{}
for _, n := range a.ctl.Inventory().Nodes() {
for _, p := range n.IBPorts {
if p.NodeGUID == "" {
continue
}
hosts[p.NodeGUID] = topology.Host{Node: n.ID, HCA: p.HCA}
l := Live{Node: n.ID, HCA: p.HCA, State: p.State, Phys: p.PhysState, Gbps: p.RateGbps,
TxBps: p.TxBps, RxBps: p.RxBps}
for name, c := range p.Counters {
if c > 0 && name != "port_xmit_data" && name != "port_rcv_data" {
if l.Errors == nil {
l.Errors = map[string]uint64{}
}
l.Errors[name] = c
}
}
t.Live[p.NodeGUID] = l
}
}
sc, ok := a.ctl.Inventory().Scans()[ph.Key]
if !ok {
return t, true
}
t.ScannedAt, t.ScannedBy, t.TookMs, t.Err = sc.At, sc.Node+" "+sc.Port, sc.Took.Milliseconds(), sc.Err
if sc.Output != "" {
f, err := topology.Parse(sc.Output)
if err != nil {
t.Err = "the map could not be read: " + err.Error()
return t, true
}
v := topology.Build(f, hosts)
t.Map = &v
}
return t, true
}
+121 -1
View File
@@ -9,6 +9,7 @@ import (
"maps"
"net/http"
"slices"
"sort"
"sync"
"time"
@@ -58,6 +59,8 @@ type Service struct {
logMu sync.Mutex
logWaits map[string]chan *llmgwv1.LogsReply
logSeq uint64
// scanWaits are the fabric scans awaiting a reply, by request id.
scanWaits map[string]chan *llmgwv1.FabricScanReply
}
// SetEngines sets the engines every node installs, by name and version.
@@ -282,6 +285,15 @@ func (s *Service) handle(nodeID string, msg *llmgwv1.AgentMessage) {
"last_output", m.Shard.GetLastOutput())
}
case *llmgwv1.AgentMessage_FabricScan:
s.logMu.Lock()
ch, ok := s.scanWaits[m.FabricScan.GetRequestId()]
delete(s.scanWaits, m.FabricScan.GetRequestId())
s.logMu.Unlock()
if ok {
ch <- m.FabricScan
}
case *llmgwv1.AgentMessage_Logs:
s.logMu.Lock()
ch, ok := s.logWaits[m.Logs.GetRequestId()]
@@ -406,9 +418,13 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
}
if ports := hb.GetIbPorts(); len(ports) > 0 {
before := map[string]string{}
prev := map[string]inventory.IBPort{}
for _, p := range n.IBPorts {
before[fmt.Sprintf("%s/%d", p.HCA, p.Port)] = p.SMGUID
prev[fmt.Sprintf("%s/%d", p.HCA, p.Port)] = p
}
now := time.Now()
secs := now.Sub(n.IBAt).Seconds()
n.IBPorts = nil
for _, p := range ports {
// A different master than last time is a failover, or a port
@@ -427,7 +443,9 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
}
n.IBPorts = append(n.IBPorts, inventory.IBPort{
SMs: sms, SMsError: p.GetSubnetManagersError(),
HCA: p.GetHca(), Port: int(p.GetPort()), State: p.GetState(),
TxBps: rate(prev[where].Counters["port_xmit_data"], p.GetCounters()["port_xmit_data"], secs),
RxBps: rate(prev[where].Counters["port_rcv_data"], p.GetCounters()["port_rcv_data"], secs),
HCA: p.GetHca(), Port: int(p.GetPort()), State: p.GetState(),
PhysState: p.GetPhysState(), RateGbps: int(p.GetRateGbps()),
LinkLayer: p.GetLinkLayer(), Counters: maps.Clone(p.GetCounters()),
NodeGUID: p.GetNodeGuid(), PortGUID: p.GetPortGuid(), GIDPrefix: p.GetGidPrefix(),
@@ -436,6 +454,9 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
})
}
}
if len(hb.GetIbPorts()) > 0 {
n.IBAt = time.Now()
}
// A port went down, or came back. Placement reads the fabrics, so this is
// what stops a node being offered multi-node work it can no longer carry.
if hb.GetFabricsChanged() {
@@ -642,3 +663,102 @@ func (s *Service) Logs(ctx context.Context, nodeID, source string, lines int) ([
return nil, nil, fmt.Errorf("node %s did not answer; an agent older than v0.8.4 cannot", nodeID)
}
}
// ScanFabric asks a node to map the fabric one of its ports is on.
func (s *Service) ScanFabric(ctx context.Context, nodeID, hca string, port int) (*llmgwv1.FabricScanReply, error) {
s.logMu.Lock()
if s.scanWaits == nil {
s.scanWaits = map[string]chan *llmgwv1.FabricScanReply{}
}
s.logSeq++
id := fmt.Sprintf("scan-%d", s.logSeq)
ch := make(chan *llmgwv1.FabricScanReply, 1)
s.scanWaits[id] = ch
s.logMu.Unlock()
defer func() {
s.logMu.Lock()
delete(s.scanWaits, id)
s.logMu.Unlock()
}()
if err := s.send(nodeID, &llmgwv1.ControlMessage{Msg: &llmgwv1.ControlMessage_FabricScan{
FabricScan: &llmgwv1.FabricScanRequest{RequestId: id, Hca: hca, Port: int32(port)}, //nolint:gosec // a port number
}}); err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(ctx, 3*time.Minute)
defer cancel()
select {
case r := <-ch:
return r, nil
case <-ctx.Done():
return nil, fmt.Errorf("node %s did not answer the fabric scan", nodeID)
}
}
// ScanFabrics maps every physical InfiniBand fabric every so often, asking
// one node on each: the first, by name, whose port on it is up. A fabric
// looks the same from all its ports, so one is enough, and the rest of its
// nodes are left to their work.
func (s *Service) ScanFabrics(ctx context.Context, every time.Duration) {
if every <= 0 {
every = 5 * time.Minute
}
// Soon after a start, once agents have sent a heartbeat or two.
timer := time.NewTimer(30 * time.Second)
defer timer.Stop()
for {
select {
case <-ctx.Done():
return
case <-timer.C:
}
s.scanOnce(ctx)
timer.Reset(every)
}
}
func (s *Service) scanOnce(ctx context.Context) {
nodes := s.inv.Nodes()
sort.Slice(nodes, func(i, j int) bool { return nodes[i].ID < nodes[j].ID })
canon := inventory.CanonicalSMs(nodes)
chosen := map[string]bool{}
for _, n := range nodes {
if n.State != inventory.Ready {
continue
}
for _, p := range n.IBPorts {
key := p.FabricKeyIn(canon)
if key == "" || p.RoCE() || chosen[key] {
continue
}
chosen[key] = true
where := fmt.Sprintf("%s/%d", p.HCA, p.Port)
r, err := s.ScanFabric(ctx, n.ID, p.HCA, p.Port)
sc := inventory.FabricScan{Key: key, At: time.Now(), Node: n.ID, Port: where}
switch {
case err != nil:
sc.Err = err.Error()
case r.GetError() != "":
sc.Err = r.GetError()
default:
sc.Output = r.GetOutput()
}
if r != nil {
sc.Took = time.Duration(r.GetTookMs()) * time.Millisecond
}
if sc.Err != "" {
s.log.Warn("could not map a fabric", "fabric", key, "node", n.ID, "port", where, "err", sc.Err)
}
s.inv.SetScan(sc)
}
}
}
// rate is a counter's increase a second, zero when there is no earlier
// reading or the counter went backwards — an adapter reset starts it again.
func rate(before, now uint64, secs float64) float64 {
if before == 0 || now < before || secs <= 0 || secs > 120 {
return 0
}
return float64(now-before) / secs
}
+44
View File
@@ -118,6 +118,8 @@ type Node struct {
// IBPorts are the node's InfiniBand and RoCE ports at the last heartbeat,
// with their counters. Not persisted, for the same reason as Load.
IBPorts []IBPort `json:"-"`
// IBAt is when IBPorts was read, for the rates.
IBAt time.Time `json:"-"`
}
// IBPort is one port's state, identity and cumulative counters.
@@ -146,6 +148,9 @@ type IBPort struct {
// on the one port per fabric that asked; SMsError says why it could not.
SMs []SubnetManager
SMsError string
// TxBps and RxBps are bytes a second, from this heartbeat's data
// counters against the last one's; zero on the first.
TxBps, RxBps float64
}
// SubnetManager is one subnet manager on a fabric.
@@ -357,6 +362,8 @@ type Store struct {
allocs map[string][]Allocation
// smChanges are ports' changes of master subnet manager, newest last.
smChanges []SMChange
// scans are the latest ibnetdiscover of each physical fabric, by key.
scans map[string]FabricScan
}
// NewStore returns an empty inventory.
@@ -378,6 +385,43 @@ func (s *Store) RecordSMChange(c SMChange) {
}
}
// FabricScan is one ibnetdiscover of a physical fabric: its text, and who
// ran it when.
type FabricScan struct {
Key string
At time.Time
Node string
Port string
Output string
Err string
Took time.Duration
}
// SetScan keeps a fabric's latest scan. A failed one does not replace a
// good one's text: a map from ten minutes ago is worth more than none.
func (s *Store) SetScan(sc FabricScan) {
s.mu.Lock()
defer s.mu.Unlock()
if s.scans == nil {
s.scans = map[string]FabricScan{}
}
if sc.Err != "" {
if old, ok := s.scans[sc.Key]; ok && old.Output != "" {
old.Err = sc.Err
s.scans[sc.Key] = old
return
}
}
s.scans[sc.Key] = sc
}
// Scans returns the latest scan of every fabric.
func (s *Store) Scans() map[string]FabricScan {
s.mu.Lock()
defer s.mu.Unlock()
return maps.Clone(s.scans)
}
// SMChanges returns the recorded changes, oldest first.
func (s *Store) SMChanges() []SMChange {
s.mu.Lock()
+3
View File
@@ -333,6 +333,9 @@ func (s *Scheduler) placeOnFabric(p Profile, fabric, pool string, candidates []c
sort.SliceStable(candidates, func(i, j int) bool {
return len(candidates[i].free) > len(candidates[j].free)
})
// Then by the fabric's map, where there is one: one leaf's nodes
// together, a degraded cable last.
candidates = byLeaf(candidates, hints(s.inv))
assignments := make([]NodeAssignment, 0, p.Nodes)
for rank, c := range candidates[:p.Nodes] {
@@ -6,6 +6,7 @@ import (
"slices"
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
@@ -970,3 +971,50 @@ func TestAFailoverDoesNotSplitAFabricForPlacement(t *testing.T) {
t.Fatalf("a failover split the fabric:\n%s", res.Explain())
}
}
// scanOf is ibnetdiscover's output for two leaves: leaf-a holding the first
// two nodes' adapters, leaf-b the other two. degraded names a node whose
// cable trained at half width.
func scanOf(ids []string, degraded string) string {
var b strings.Builder
for li, leaf := range []string{"00000000000000a1", "00000000000000a2"} {
fmt.Fprintf(&b, "Switch\t36 \"S-%s\"\t\t# \"MF0;leaf-%c:QM8700/U1\" enhanced port 0 lid %d lmc 0\n", leaf, 'a'+li, li+1)
for k := 0; k < 2; k++ {
id := ids[li*2+k]
rate := "4xHDR"
if id == degraded {
rate = "2xHDR"
}
fmt.Fprintf(&b, "[%d]\t\"H-%s\"[1](%s) \t\t# \"%s mlx5_0\" lid %d %s\n", k+1, guidOf(id), guidOf(id), id, 10+li*2+k, rate)
}
b.WriteString("\n")
}
return b.String()
}
func guidOf(id string) string { return fmt.Sprintf("%016x", []byte(id)[len(id)-1]) }
func TestAReplicaStaysUnderOneLeafAndOffDegradedCables(t *testing.T) {
ids := []string{"n1", "n2", "n3", "n4"}
var nodes []inventory.Node
for _, id := range ids {
n := onSM(node(id, "h100-80g", 8, "ib-a"), "0x01")
n.IBPorts[0].NodeGUID = "0x" + guidOf(id)
nodes = append(nodes, n)
}
st := store(nodes...)
// n1 sits on leaf-a with n2, but its cable trained at half width: leaf-a
// has one good node, leaf-b two.
st.SetScan(inventory.FabricScan{Key: "sm:0x01", At: time.Now(), Output: scanOf(ids, "n1")})
res := New(st).Place([]Profile{{Name: "pair", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2}}, nil)
if res.Placement == nil {
t.Fatalf("no placement:\n%s", res.Explain())
}
var got []string
for _, a := range res.Placement.Nodes {
got = append(got, a.NodeID)
}
if !slices.Equal(got, []string{"n3", "n4"}) {
t.Errorf("placed on %v, want the two good nodes under leaf-b", got)
}
}
+135
View File
@@ -0,0 +1,135 @@
package placement
import (
"slices"
"sync"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/topology"
)
// fabricHints is what a fabric's map says about placing on it: the leaf
// switch each node's first adapter hangs off, and which nodes have a cable
// that trained below the rest.
type fabricHints struct {
leaf map[string]string
degraded map[string]bool
}
// hintCache keeps a parsed map until its fabric is scanned again; parsing
// thousands of lines on every placement would be the scheduler's main cost.
var hintCache struct {
sync.Mutex
at map[string]time.Time
hints map[string]fabricHints
}
// hints reads every scanned fabric. Nodes on no mapped fabric are absent,
// which leaves them placed as before.
func hints(inv *inventory.Store) fabricHints {
out := fabricHints{leaf: map[string]string{}, degraded: map[string]bool{}}
scans := inv.Scans()
if len(scans) == 0 {
return out
}
var hosts map[string]topology.Host
hintCache.Lock()
defer hintCache.Unlock()
if hintCache.at == nil {
hintCache.at, hintCache.hints = map[string]time.Time{}, map[string]fabricHints{}
}
for key, sc := range scans {
if sc.Output == "" {
continue
}
h, ok := hintCache.hints[key]
if !ok || !hintCache.at[key].Equal(sc.At) {
if hosts == nil {
hosts = map[string]topology.Host{}
for _, n := range inv.Nodes() {
for _, p := range n.IBPorts {
if p.NodeGUID != "" {
hosts[p.NodeGUID] = topology.Host{Node: n.ID, HCA: p.HCA}
}
}
}
}
f, err := topology.Parse(sc.Output)
if err != nil {
continue
}
v := topology.Build(f, hosts)
h = fabricHints{leaf: map[string]string{}, degraded: map[string]bool{}}
guidNode := map[string]string{}
for _, hv := range v.Hosts {
if hv.Node == "" {
continue
}
guidNode[hv.GUID] = hv.Node
if hv.Rail == 0 {
h.leaf[hv.Node] = hv.Leaf
}
}
for _, l := range v.Links {
if l.Host && l.Degraded {
if n := guidNode[l.To]; n != "" {
h.degraded[n] = true
}
}
}
hintCache.hints[key], hintCache.at[key] = h, sc.At
}
for n, l := range h.leaf {
out.leaf[n] = l
}
for n := range h.degraded {
out.degraded[n] = true
}
}
return out
}
// byLeaf orders candidates for a replica spanning nodes: those whose cables
// all trained at their rate first, and among them the nodes of one leaf
// together — the leaf with the most of them first — so a replica's traffic
// stays under one switch when it can, and crosses the spines only when it
// must. Within a leaf the order given (emptiest first) is kept.
func byLeaf(cands []candidate, h fabricHints) []candidate {
if len(h.leaf) == 0 && len(h.degraded) == 0 {
return cands
}
count := map[string]int{}
for _, c := range cands {
if !h.degraded[c.node.ID] {
count[h.leaf[c.node.ID]]++
}
}
firstSeen := map[string]int{}
for i, c := range cands {
if _, ok := firstSeen[h.leaf[c.node.ID]]; !ok {
firstSeen[h.leaf[c.node.ID]] = i
}
}
out := make([]candidate, len(cands))
copy(out, cands)
rank := func(c candidate) (bad int, size int, first int) {
if h.degraded[c.node.ID] {
bad = 1
}
l := h.leaf[c.node.ID]
return bad, count[l], firstSeen[l]
}
slices.SortStableFunc(out, func(a, b candidate) int {
ba, sa, fa := rank(a)
bb, sb, fb := rank(b)
if ba != bb {
return ba - bb
}
if sa != sb {
return sb - sa
}
return fa - fb
})
return out
}
+390
View File
@@ -0,0 +1,390 @@
// Package topology reads ibnetdiscover's map of a fabric and works out its
// shape: which switches are leaves and which spines, which adapter belongs to
// which node, which links trained below the rest, and whether the rails line
// up.
package topology
import (
"fmt"
"regexp"
"sort"
"strconv"
"strings"
)
// Device is a switch, an adapter or a router on the fabric.
type Device struct {
GUID string `json:"guid"`
Kind string `json:"kind"` // switch, ca, router
Desc string `json:"desc"`
Ports int `json:"ports"`
LID int `json:"lid,omitempty"`
}
// End is one side of a link.
type End struct {
GUID string `json:"guid"`
Port int `json:"port"`
}
// Link is one cable, once.
type Link struct {
A End `json:"a"`
B End `json:"b"`
Width string `json:"width"` // 4x
Speed string `json:"speed"` // HDR
Gbps int `json:"gbps"`
}
// Fabric is a parsed ibnetdiscover.
type Fabric struct {
Devices map[string]*Device `json:"devices"`
Links []Link `json:"links"`
}
var (
header = regexp.MustCompile(`^(Switch|Ca|Rt)\s+(\d+)\s+"([SHR])-([0-9a-fA-F]+)"\s*(?:#\s*"([^"]*)"(.*))?$`)
portLn = regexp.MustCompile(`^\[(\d+)\](?:\([0-9a-fA-F]+\))?\s+"([SHR])-([0-9a-fA-F]+)"\[(\d+)\](?:\([0-9a-fA-F]+\))?\s*(?:#\s*(.*))?$`)
lidRe = regexp.MustCompile(`\blid (\d+)`)
rateRe = regexp.MustCompile(`\b(\d+)x([A-Z]+[0-9]*)\s*$`)
)
// laneGbps is the signalling rate of one lane at each speed, as the link
// reports it.
var laneGbps = map[string]float64{
"SDR": 2.5, "DDR": 5, "QDR": 10, "FDR10": 10, "FDR": 14, "EDR": 25, "HDR": 50, "NDR": 100, "XDR": 200,
}
// Parse reads ibnetdiscover's default output.
func Parse(out string) (*Fabric, error) {
f := &Fabric{Devices: map[string]*Device{}}
seen := map[string]bool{}
var cur *Device
for _, raw := range strings.Split(out, "\n") {
line := strings.TrimSpace(raw)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
if m := header.FindStringSubmatch(line); m != nil {
kind := map[string]string{"Switch": "switch", "Ca": "ca", "Rt": "router"}[m[1]]
n, _ := strconv.Atoi(m[2])
d := &Device{GUID: guid(m[4]), Kind: kind, Ports: n, Desc: m[5]}
if l := lidRe.FindStringSubmatch(m[6]); l != nil {
d.LID, _ = strconv.Atoi(l[1])
}
if old, ok := f.Devices[d.GUID]; ok && old.Desc != "" && d.Desc == "" {
d.Desc = old.Desc
}
f.Devices[d.GUID] = d
cur = d
continue
}
m := portLn.FindStringSubmatch(line)
if m == nil || cur == nil {
continue
}
port, _ := strconv.Atoi(m[1])
peer := guid(m[3])
peerPort, _ := strconv.Atoi(m[4])
comment := m[5]
// The peer's own block may come later; its description is in the
// comment, quoted, which is enough to name it until then.
if _, ok := f.Devices[peer]; !ok {
kind := map[string]string{"S": "switch", "H": "ca", "R": "router"}[m[2]]
d := &Device{GUID: peer, Kind: kind}
if q := strings.Index(comment, `"`); q >= 0 {
if e := strings.Index(comment[q+1:], `"`); e >= 0 {
d.Desc = comment[q+1 : q+1+e]
}
}
f.Devices[peer] = d
}
a, b := End{cur.GUID, port}, End{peer, peerPort}
key := linkKey(a, b)
if seen[key] {
continue
}
seen[key] = true
l := Link{A: a, B: b}
if r := rateRe.FindStringSubmatch(comment); r != nil {
l.Width, l.Speed = r[1]+"x", r[2]
lanes, _ := strconv.Atoi(r[1])
l.Gbps = int(float64(lanes) * laneGbps[r[2]])
}
f.Links = append(f.Links, l)
}
if len(f.Devices) == 0 {
return nil, fmt.Errorf("no devices in ibnetdiscover output")
}
return f, nil
}
func guid(hex string) string {
hex = strings.ToLower(hex)
for len(hex) < 16 {
hex = "0" + hex
}
return "0x" + hex
}
func linkKey(a, b End) string {
x, y := fmt.Sprintf("%s/%d", a.GUID, a.Port), fmt.Sprintf("%s/%d", b.GUID, b.Port)
if x > y {
x, y = y, x
}
return x + "-" + y
}
// Host is an adapter the cluster knows: which node, which card.
type Host struct {
Node string
HCA string
}
// View is a fabric's shape, ready to draw.
type View struct {
Switches []SwitchView `json:"switches"`
Hosts []HostView `json:"hosts"`
Links []LinkView `json:"links"`
// Tiers is how many switch levels: 1 for a single switch, 2 for
// leaf and spine, 3 with a core.
Tiers int `json:"tiers"`
// Expected is the rate most links of each kind trained at; a link below
// it is degraded.
ExpectedHost int `json:"expected_host_gbps"`
ExpectedSwitch int `json:"expected_switch_gbps"`
Degraded int `json:"degraded"`
// Rails is, per rail — an adapter's position on its node — the leaves it
// reaches, and Aligned whether each rail has its own.
Rails []Rail `json:"rails"`
Aligned bool `json:"rail_aligned"`
Notes []string `json:"notes,omitempty"`
}
// SwitchView is a switch and its place in the tiers.
type SwitchView struct {
GUID string `json:"guid"`
Name string `json:"name"`
Tier int `json:"tier"` // 1 leaf, 2 spine, 3 core
Ports int `json:"ports"`
Used int `json:"used"`
LID int `json:"lid,omitempty"`
// Hosts counts adapters hanging off it.
Hosts int `json:"hosts"`
}
// HostView is an adapter, and the node it is in when the cluster knows it.
type HostView struct {
GUID string `json:"guid"`
Desc string `json:"desc"`
Node string `json:"node,omitempty"`
HCA string `json:"hca,omitempty"`
Rail int `json:"rail"`
Leaf string `json:"leaf,omitempty"`
}
// LinkView is a cable, with what it trained at and whether that is short of
// its kind.
type LinkView struct {
From string `json:"from"`
FromPort int `json:"from_port"`
To string `json:"to"`
ToPort int `json:"to_port"`
Rate string `json:"rate"`
Gbps int `json:"gbps"`
Host bool `json:"host"`
Degraded bool `json:"degraded"`
}
// Rail is the leaves one rail's adapters reach.
type Rail struct {
Index int `json:"index"`
Leaves []string `json:"leaves"`
}
// Build works out the shape. hosts maps adapter GUIDs to the cluster's nodes.
func Build(f *Fabric, hosts map[string]Host) View {
var v View
adj := map[string][]string{}
for _, l := range f.Links {
adj[l.A.GUID] = append(adj[l.A.GUID], l.B.GUID)
adj[l.B.GUID] = append(adj[l.B.GUID], l.A.GUID)
}
// A switch's tier is its distance from the nearest adapter: leaves hold
// adapters, spines hold leaves, cores hold spines.
tier := map[string]int{}
var frontier []string
for g, d := range f.Devices {
if d.Kind == "ca" {
tier[g] = 0
frontier = append(frontier, g)
}
}
for len(frontier) > 0 {
var next []string
for _, g := range frontier {
for _, p := range adj[g] {
if _, ok := tier[p]; ok || f.Devices[p] == nil || f.Devices[p].Kind == "ca" {
continue
}
tier[p] = tier[g] + 1
next = append(next, p)
}
}
frontier = next
}
name := func(g string) string {
d := f.Devices[g]
if d == nil || d.Desc == "" {
return g
}
desc := d.Desc
// Mellanox switches say "MF0;leaf-01:MSB7700/U1": the name is between.
if i := strings.Index(desc, ";"); i >= 0 {
desc = desc[i+1:]
if j := strings.Index(desc, ":"); j >= 0 {
desc = desc[:j]
}
}
return desc
}
// Expected rate per kind of link: the most common.
hostRates, swRates := map[int]int{}, map[int]int{}
isHost := func(l Link) bool {
return f.Devices[l.A.GUID].Kind == "ca" || f.Devices[l.B.GUID].Kind == "ca"
}
for _, l := range f.Links {
if isHost(l) {
hostRates[l.Gbps]++
} else {
swRates[l.Gbps]++
}
}
v.ExpectedHost, v.ExpectedSwitch = mode(hostRates), mode(swRates)
used := map[string]int{}
hostsOn := map[string]int{}
leafOf := map[string]string{}
for _, l := range f.Links {
used[l.A.GUID]++
used[l.B.GUID]++
lv := LinkView{From: l.A.GUID, FromPort: l.A.Port, To: l.B.GUID, ToPort: l.B.Port,
Rate: l.Width + l.Speed, Gbps: l.Gbps, Host: isHost(l)}
// Keep adapters on the To side, so a drawing can go down the tiers.
if f.Devices[lv.From].Kind == "ca" {
lv.From, lv.FromPort, lv.To, lv.ToPort = lv.To, lv.ToPort, lv.From, lv.FromPort
}
if lv.Host {
hostsOn[lv.From]++
leafOf[lv.To] = lv.From
lv.Degraded = l.Gbps > 0 && l.Gbps < v.ExpectedHost
} else {
lv.Degraded = l.Gbps > 0 && l.Gbps < v.ExpectedSwitch
if tier[lv.From] < tier[lv.To] {
lv.From, lv.FromPort, lv.To, lv.ToPort = lv.To, lv.ToPort, lv.From, lv.FromPort
}
}
if lv.Degraded {
v.Degraded++
}
v.Links = append(v.Links, lv)
}
for g, d := range f.Devices {
switch d.Kind {
case "switch":
t := tier[g]
if t == 0 {
t = 1 // a switch with nothing below it is drawn with the leaves
}
v.Switches = append(v.Switches, SwitchView{GUID: g, Name: name(g), Tier: t, Ports: d.Ports,
Used: used[g], LID: d.LID, Hosts: hostsOn[g]})
v.Tiers = max(v.Tiers, t)
case "ca":
h := HostView{GUID: g, Desc: d.Desc, Leaf: leafOf[g]}
if known, ok := hosts[g]; ok {
h.Node, h.HCA = known.Node, known.HCA
}
v.Hosts = append(v.Hosts, h)
}
}
sort.Slice(v.Switches, func(i, j int) bool {
a, b := v.Switches[i], v.Switches[j]
if a.Tier != b.Tier {
return a.Tier > b.Tier
}
return a.Name < b.Name
})
// Rails: an adapter's position among its node's adapters on this fabric.
byNode := map[string][]int{}
for i, h := range v.Hosts {
key := h.Node
if key == "" {
key = strings.Fields(h.Desc + " ?")[0] // "node01 mlx5_0" names its host
}
byNode[key] = append(byNode[key], i)
}
railLeaves := map[int]map[string]bool{}
for _, idx := range byNode {
sort.Slice(idx, func(a, b int) bool {
return hcaName(v.Hosts[idx[a]]) < hcaName(v.Hosts[idx[b]])
})
for r, i := range idx {
v.Hosts[i].Rail = r
if railLeaves[r] == nil {
railLeaves[r] = map[string]bool{}
}
if v.Hosts[i].Leaf != "" {
railLeaves[r][v.Hosts[i].Leaf] = true
}
}
}
sort.Slice(v.Hosts, func(i, j int) bool {
a, b := v.Hosts[i], v.Hosts[j]
if a.Leaf != b.Leaf {
return a.Leaf < b.Leaf
}
return a.Desc < b.Desc
})
owner := map[string]int{}
v.Aligned = len(railLeaves) > 1
for r := 0; r < len(railLeaves); r++ {
var leaves []string
for l := range railLeaves[r] {
leaves = append(leaves, name(l))
if prev, ok := owner[l]; ok && prev != r {
v.Aligned = false
}
owner[l] = r
}
sort.Strings(leaves)
v.Rails = append(v.Rails, Rail{Index: r, Leaves: leaves})
}
if v.Degraded > 0 {
v.Notes = append(v.Notes, fmt.Sprintf("%d link(s) trained below the rest of their kind", v.Degraded))
}
return v
}
func hcaName(h HostView) string {
if h.HCA != "" {
return h.HCA
}
return h.Desc
}
func mode(counts map[int]int) int {
best, n := 0, -1
for v, c := range counts {
if v == 0 {
continue
}
if c > n || (c == n && v > best) {
best, n = v, c
}
}
return best
}
+149
View File
@@ -0,0 +1,149 @@
package topology
import (
"fmt"
"strings"
"testing"
)
// fatTree writes ibnetdiscover's default output for two spines, two leaves
// and four nodes of two adapters each, rail-aligned: every node's mlx5_0 on
// leaf-01 and mlx5_1 on leaf-02. node03's mlx5_1 trained at 2x instead of 4x.
func fatTree() string {
var b strings.Builder
w := func(format string, a ...any) { fmt.Fprintf(&b, format+"\n", a...) }
w("#\n# Topology file: generated on Mon Sep 29 10:00:00 2026\n#\n# Initiated from node 0000000000000a01 port 0000000000000a01\n")
spine := []string{"00000000000000c1", "00000000000000c2"}
leaf := []string{"00000000000000b1", "00000000000000b2"}
hca := func(n, r int) string { return fmt.Sprintf("000000000000%d%02d", r+10, n) }
for si, s := range spine {
w("vendid=0x2c9\ndevid=0xd2f0\nsysimgguid=0x%s\nswitchguid=0x%s(%s)", s, s, s)
w("Switch\t64 \"S-%s\"\t\t# \"MF0;spine-0%d:MQM9700/U1\" enhanced port 0 lid %d lmc 0", s, si+1, 10+si)
for li, l := range leaf {
for k := 0; k < 2; k++ {
w("[%d]\t\"S-%s\"[%d]\t\t# \"MF0;leaf-0%d:MQM9700/U1\" lid %d 4xNDR", li*2+k+1, l, 33+si*2+k, li+1, 20+li)
}
}
w("")
}
for li, l := range leaf {
w("vendid=0x2c9\ndevid=0xd2f0\nsysimgguid=0x%s\nswitchguid=0x%s(%s)", l, l, l)
w("Switch\t64 \"S-%s\"\t\t# \"MF0;leaf-0%d:MQM9700/U1\" enhanced port 0 lid %d lmc 0", l, li+1, 20+li)
for n := 1; n <= 4; n++ {
rate := "4xNDR"
if n == 3 && li == 1 {
rate = "2xNDR"
}
h := hca(n, li)
w("[%d]\t\"H-%s\"[1](%s) \t\t# \"node0%d mlx5_%d\" lid %d %s", n, h, h, n, li, 100+n*2+li, rate)
}
for si, s := range spine {
for k := 0; k < 2; k++ {
w("[%d]\t\"S-%s\"[%d]\t\t# \"MF0;spine-0%d:MQM9700/U1\" lid %d 4xNDR", 33+si*2+k, s, li*2+k+1, si+1, 10+si)
}
}
w("")
}
for n := 1; n <= 4; n++ {
for li, l := range leaf {
h := hca(n, li)
rate := "4xNDR"
if n == 3 && li == 1 {
rate = "2xNDR"
}
w("vendid=0x2c9\ndevid=0x1021\nsysimgguid=0x%s\ncaguid=0x%s", h, h)
w("Ca\t1 \"H-%s\"\t\t# \"node0%d mlx5_%d\"", h, n, li)
w("[1](%s) \t\"S-%s\"[%d]\t\t# lid %d lmc 0 \"MF0;leaf-0%d:MQM9700/U1\" lid %d %s", h, l, n, 100+n*2+li, li+1, 20+li, rate)
w("")
}
}
return b.String()
}
func TestAFatTreeIsReadIntoTiers(t *testing.T) {
f, err := Parse(fatTree())
if err != nil {
t.Fatal(err)
}
// 2 spines + 2 leaves + 8 adapters; 8 spine-leaf cables + 8 host cables,
// each once though both ends list it.
if len(f.Devices) != 12 || len(f.Links) != 16 {
t.Fatalf("%d devices, %d links", len(f.Devices), len(f.Links))
}
hosts := map[string]Host{}
for n := 1; n <= 4; n++ {
for r := 0; r < 2; r++ {
hosts[fmt.Sprintf("0x000000000000%d%02d", r+10, n)] = Host{Node: fmt.Sprintf("n%d", n), HCA: fmt.Sprintf("mlx5_%d", r)}
}
}
v := Build(f, hosts)
if v.Tiers != 2 {
t.Errorf("tiers = %d", v.Tiers)
}
tiers := map[string]int{}
for _, s := range v.Switches {
tiers[s.Name] = s.Tier
}
if tiers["spine-01"] != 2 || tiers["leaf-02"] != 1 {
t.Errorf("tiers = %v", tiers)
}
if v.ExpectedHost != 400 || v.ExpectedSwitch != 400 {
t.Errorf("expected rates %d / %d", v.ExpectedHost, v.ExpectedSwitch)
}
// node03's second adapter trained at half width.
if v.Degraded != 1 {
t.Errorf("degraded = %d", v.Degraded)
}
for _, l := range v.Links {
if l.Degraded && (l.Rate != "2xNDR" || !l.Host) {
t.Errorf("wrong link degraded: %+v", l)
}
}
// Every node's first adapter on leaf-01, its second on leaf-02: aligned.
if !v.Aligned || len(v.Rails) != 2 || v.Rails[0].Leaves[0] != "leaf-01" || v.Rails[1].Leaves[0] != "leaf-02" {
t.Errorf("rails = %+v aligned=%v", v.Rails, v.Aligned)
}
mapped := 0
for _, h := range v.Hosts {
if h.Node != "" {
mapped++
}
}
if mapped != 8 {
t.Errorf("%d adapters mapped to nodes", mapped)
}
}
func TestCrossedRailsAreNotAligned(t *testing.T) {
// node02's two cables swapped at the leaves: its mlx5_0 on leaf-02 and
// its mlx5_1 on leaf-01, so each leaf now carries both rails.
f, err := Parse(fatTree())
if err != nil {
t.Fatal(err)
}
swap := map[string]string{"0x0000000000001002": "0x00000000000000b2", "0x0000000000001102": "0x00000000000000b1"}
for i, l := range f.Links {
for _, host := range []*End{&f.Links[i].A, &f.Links[i].B} {
if leaf, ok := swap[host.GUID]; ok {
other := &f.Links[i].A
if host == other {
other = &f.Links[i].B
}
other.GUID = leaf
}
}
_ = l
}
v := Build(f, map[string]Host{
"0x0000000000001002": {Node: "n2", HCA: "mlx5_0"}, "0x0000000000001102": {Node: "n2", HCA: "mlx5_1"},
})
if v.Aligned {
t.Errorf("crossed rails read as aligned: %+v", v.Rails)
}
}
func TestNotAMapIsAnError(t *testing.T) {
if _, err := Parse("ibwarn: [1234] mad_rpc_open_port: can't open UMAD port ((null):0)"); err == nil {
t.Error("expected an error")
}
}
+3
View File
@@ -164,6 +164,9 @@ func (r *Router) Handler() http.Handler {
}, r.log)))
mux.HandleFunc("GET /admin/nodes/{id}/logs", pages.AdminPage(
web.NodeLogsPage(r.clusterView, false, version.Version)))
mux.HandleFunc("GET /admin/fabrics", pages.AdminPage(web.FabricsPage(r.clusterView, false, version.Version)))
mux.HandleFunc("GET /admin/fabrics/{id}", pages.AdminPage(web.FabricsPage(r.clusterView, false, version.Version)))
mux.HandleFunc("GET /admin/fabrics/{id}/topology.json", pages.AdminPage(web.FabricTopologyJSON(r.clusterView)))
mux.HandleFunc("GET /admin/nodes/{id}", pages.AdminPage(
web.NodePage(r.clusterView, r.history, r.histWindow, false, version.Version, r.log)))
pages.RegisterAdminUsers(mux, r.store)
+82
View File
@@ -0,0 +1,82 @@
package web
import (
"encoding/json"
"html/template"
"net/http"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
)
// TopologySource is a Source that can map its fabrics.
type TopologySource interface {
Topologies() []adminapi.TopologyView
Topology(id string) (adminapi.TopologyView, bool)
}
type fabricPageData struct {
Nav Nav
OnAdminListener bool
Base string
List []adminapi.TopologyView
Current *adminapi.TopologyView
// Data is the current fabric as JSON, for the drawing to start from
// without a round trip. json.Marshal escapes <, > and &, so a switch
// description cannot close the script it sits in.
Data template.JS
Version string
Now string
}
// FabricsPage shows every physical fabric and, for one, its map.
func FabricsPage(src Source, onAdminListener bool, version string) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
ts, ok := src.(TopologySource)
if !ok {
http.NotFound(w, r)
return
}
d := fabricPageData{Nav: navFrom(r, "cluster"), OnAdminListener: onAdminListener, Base: "/admin/fabrics",
List: ts.Topologies(), Version: version, Now: time.Now().UTC().Format(time.RFC3339)}
if onAdminListener {
d.Base = "/fabrics"
}
id := r.PathValue("id")
if id == "" && len(d.List) > 0 {
id = d.List[0].ID
}
if id != "" {
t, ok := ts.Topology(id)
if !ok {
http.NotFound(w, r)
return
}
d.Current = &t
b, _ := json.Marshal(t)
d.Data = template.JS(b) //nolint:gosec // json.Marshal escapes <, > and &
}
noStore(w)
render(w, "fabrics", d)
}
}
// FabricTopologyJSON is one fabric's map and live state, for the drawing to
// refresh from.
func FabricTopologyJSON(src Source) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
ts, ok := src.(TopologySource)
if !ok {
http.NotFound(w, r)
return
}
t, ok := ts.Topology(r.PathValue("id"))
if !ok {
http.NotFound(w, r)
return
}
noStore(w)
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(t)
}
}
+45 -1
View File
@@ -7,6 +7,7 @@ import (
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/topology"
"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
)
@@ -93,7 +94,7 @@ func TestANodeNotMeasuredLatelySaysSoRatherThanZero(t *testing.T) {
func TestANodesPageShowsItsFabricPorts(t *testing.T) {
_, body := get(t, nodeMux(busySource{}, nil), "http://x/admin/nodes/llmgw-node01", nil)
for _, want := range []string{
"<h2>Fabric</h2>", "On ib-a.", "<code>mlx5_0/1</code>", "400 Gb/s", "4.0 GiB",
"<h2>Fabric</h2>", "On ib-a ·", "<code>mlx5_0/1</code>", "400 Gb/s", "4.0 GiB",
`<td class="bad">DOWN <span class="note">Polling</span></td>`, "link_downed 7",
} {
if !strings.Contains(body, want) {
@@ -101,3 +102,46 @@ func TestANodesPageShowsItsFabricPorts(t *testing.T) {
}
}
}
// topoSource is busySource with a mapped fabric.
type topoSource struct{ busySource }
func (topoSource) Topologies() []adminapi.TopologyView {
return []adminapi.TopologyView{{ID: "sm-0x01", Key: "sm:0x01", Label: "fabric of subnet manager 0x01", Declared: []string{"ib-a"}}}
}
func (s topoSource) Topology(id string) (adminapi.TopologyView, bool) {
if id != "sm-0x01" {
return adminapi.TopologyView{}, false
}
t := s.Topologies()[0]
t.Map = &topology.View{Tiers: 2, Switches: []topology.SwitchView{{GUID: "0xs1", Name: "leaf-01</script><script>alert(1)", Tier: 1}}}
return t, true
}
func TestAFabricsPageCarriesItsMapSafely(t *testing.T) {
mux := http.NewServeMux()
mux.HandleFunc("GET /admin/fabrics", FabricsPage(topoSource{}, false, "v1"))
mux.HandleFunc("GET /admin/fabrics/{id}", FabricsPage(topoSource{}, false, "v1"))
mux.HandleFunc("GET /admin/fabrics/{id}/topology.json", FabricTopologyJSON(topoSource{}))
code, body := get(t, mux, "http://x/admin/fabrics", nil)
if code != 200 {
t.Fatalf("status %d", code)
}
for _, want := range []string{`id="fabric-map"`, `data-src="/admin/fabrics/sm-0x01/topology.json"`, "fabric.js?v=", `id="fabric-data"`} {
if !strings.Contains(body, want) {
t.Errorf("missing %q", want)
}
}
// A switch named to break out of the script it is carried in cannot.
if strings.Contains(body, "</script><script>alert(1)") {
t.Error("a switch description closed the data script")
}
if code, _ := get(t, mux, "http://x/admin/fabrics/nope", nil); code != 404 {
t.Errorf("unknown fabric: %d", code)
}
if code, body := get(t, mux, "http://x/admin/fabrics/sm-0x01/topology.json", nil); code != 200 || !strings.Contains(body, `"tiers":2`) {
t.Errorf("json: %d %s", code, body)
}
}
+126
View File
@@ -551,3 +551,129 @@ table.held tr.model-version > td:first-child { padding-left: 1.4rem; }
ul.fabric-problems { list-style: none; padding: 0; margin: .25rem 0 .75rem; font-size: .88rem; }
ul.fabric-problems li { margin: .25rem 0; }
ul.fabric-problems li.info { color: var(--muted); }
/* ---------------------------------------------------------------------
The fabric map. A dark surface in both themes: it is a picture of light
moving through cables, and glows and flows read on dark the way a
network operations wall does.
--------------------------------------------------------------------- */
.wrap.wide { max-width: 1400px; }
.fabric-head h2 { margin: .75rem 0 .1rem; }
.fabric-map {
--fm-bg: #0b1020; --fm-bg2: #111a33; --fm-fg: #dbe4ff; --fm-dim: #7c89ad; --fm-line: #1e2a4a;
--fm-xdr: #f472b6; --fm-ndr: #a78bfa; --fm-hdr: #38bdf8; --fm-edr: #34d399; --fm-slow: #fbbf24; --fm-bad: #fb7185;
position: relative; margin: .75rem 0 1.5rem; border-radius: 14px; overflow: hidden;
background: radial-gradient(1200px 500px at 50% -10%, #1b2a55 0%, var(--fm-bg) 55%) , var(--fm-bg);
border: 1px solid #1c2747; box-shadow: 0 20px 60px -30px rgba(40, 80, 200, .45), inset 0 1px 0 rgba(255,255,255,.04);
color: var(--fm-fg);
}
.fm-bar { display: flex; flex-wrap: wrap; gap: .75rem 1.25rem; justify-content: space-between; align-items: center;
padding: .8rem 1rem; border-bottom: 1px solid var(--fm-line); background: rgba(10, 16, 36, .55); backdrop-filter: blur(6px); }
.fm-stats { display: flex; flex-wrap: wrap; gap: .4rem 1.4rem; }
.fm-stat { display: flex; flex-direction: column; line-height: 1.1; }
.fm-stat b { font-size: 1.15rem; font-variant-numeric: tabular-nums; color: #fff; text-shadow: 0 0 18px rgba(120, 160, 255, .35); }
.fm-stat span { font-size: .68rem; letter-spacing: .08em; text-transform: uppercase; color: var(--fm-dim); }
.fm-stat.bad b { color: var(--fm-bad); text-shadow: 0 0 16px rgba(251, 113, 133, .55); }
.fm-stat.ok b { color: var(--fm-edr); }
.fm-tools { display: flex; flex-wrap: wrap; gap: .5rem; align-items: center; }
.fm-tools .pills { margin: 0; }
.fm-tools button, .fm-tools .pills button {
font: inherit; font-size: .78rem; font-weight: 600; color: var(--fm-dim); background: transparent; cursor: pointer;
border: 1px solid var(--fm-line); border-radius: 999px; padding: .28rem .75rem; }
.fm-tools button:hover { color: var(--fm-fg); border-color: #2f3f6e; }
.fm-tools button.on { color: #fff; background: linear-gradient(180deg, #2a3b73, #1d2a55); border-color: #3b4f8f;
box-shadow: 0 0 14px rgba(100, 140, 255, .35); }
.fabric-map input.fm-search { display: inline-block; width: 11rem; font: inherit; font-size: .8rem; padding: .3rem .7rem; border-radius: 999px; margin: 0;
color: var(--fm-fg); background: rgba(255,255,255,.04); border: 1px solid var(--fm-line); }
.fm-stage { position: relative; }
.fm-canvas { min-height: 22rem; }
.fm-empty { padding: 4rem 2rem; text-align: center; color: var(--fm-dim); }
.fm-svg { display: block; width: 100%; cursor: grab; user-select: none; }
.fm-svg.dragging { cursor: grabbing; }
.fm-gridline { stroke: rgba(120, 150, 255, .06); stroke-width: 1; }
.fm-tier { fill: var(--fm-dim); font-size: 10px; letter-spacing: .18em; text-transform: uppercase; opacity: .75; text-anchor: middle; }
/* Cables, coloured by what they trained at. */
.fm-link { fill: none; stroke-linecap: round; opacity: .72; transition: opacity .2s, stroke-width .2s; }
.fm-link.sp-xdr { stroke: var(--fm-xdr); } .fm-link.sp-ndr { stroke: var(--fm-ndr); }
.fm-link.sp-hdr { stroke: var(--fm-hdr); } .fm-link.sp-edr { stroke: var(--fm-edr); } .fm-link.sp-slow { stroke: var(--fm-slow); }
.fm-link.host { opacity: .55; }
.mode-rails .fm-link.host { stroke: var(--rail); opacity: .8; }
.mode-rails .fm-link:not(.host) { stroke: #3a4670; opacity: .5; }
.fm-link.degraded { stroke: var(--fm-bad) !important; stroke-dasharray: 5 5; opacity: 1; filter: url(#fm-glow);
animation: fm-march 1s linear infinite, fm-pulse 1.6s ease-in-out infinite; }
@keyframes fm-march { to { stroke-dashoffset: -20; } }
@keyframes fm-pulse { 50% { stroke-opacity: .35; } }
.fm-warn circle { fill: #3b0d1a; stroke: var(--fm-bad); stroke-width: 1.5; filter: url(#fm-glow); }
.fm-warn text { fill: var(--fm-bad); font-size: 10px; font-weight: 800; text-anchor: middle; }
.fm-warn { cursor: help; }
/* Traffic: particles running the cables, up for sent, down for received. */
.fm-particle { filter: url(#fm-glow); pointer-events: none; }
.fm-particle.up { fill: #7dd3fc; } .fm-particle.down { fill: #c4b5fd; }
/* Switches. */
.fm-switch { cursor: pointer; }
.fm-switch rect { stroke-width: 1.2; transition: filter .2s; }
.fm-switch.t-leaf rect { stroke: #2dd4bf; } .fm-switch.t-spine rect { stroke: #a78bfa; } .fm-switch.t-core rect { stroke: #fbbf24; }
.fm-switch:hover rect, .fm-switch.hot rect { filter: url(#fm-glow); }
.fm-switch.has-bad rect { stroke: var(--fm-bad); }
.fm-name { fill: #fff; font-size: 11px; font-weight: 700; text-anchor: middle; }
.fm-sub { fill: var(--fm-dim); font-size: 8.5px; text-anchor: middle; }
.fm-use { stroke: rgba(255,255,255,.55); stroke-width: 2; stroke-linecap: round; }
.fm-badge circle { fill: var(--fm-bad); filter: url(#fm-glow); }
.fm-badge text { fill: #fff; font-size: 9px; font-weight: 800; text-anchor: middle; }
/* Adapters. */
.fm-host { cursor: pointer; }
.fm-host circle { fill: #334066; stroke: #4b5a88; stroke-width: 1; }
.fm-host .fm-halo { fill: transparent; stroke: transparent; }
.fm-host.known circle:not(.fm-halo) { fill: #2dd4bf; stroke: #99f6e4; }
.mode-rails .fm-host.known circle:not(.fm-halo) { fill: var(--rail); stroke: #fff; }
.fm-host.known .fm-halo { stroke: rgba(45, 212, 191, .35); animation: fm-breathe 3s ease-in-out infinite; }
.fm-host.errs circle:not(.fm-halo) { stroke: var(--fm-slow); stroke-width: 2; }
.fm-host.down circle:not(.fm-halo) { fill: var(--fm-bad); stroke: #fecdd3; }
.fm-host.down .fm-halo { stroke: var(--fm-bad); animation: fm-breathe 1s ease-in-out infinite; }
.fm-host.found .fm-halo { stroke: #fff; stroke-width: 2; }
@keyframes fm-breathe { 50% { stroke-opacity: .1; r: 11; } }
.fm-hlabel { fill: var(--fm-dim); font-size: 8.5px; }
.fm-host:hover .fm-hlabel, .fm-host.hot .fm-hlabel { fill: #fff; }
/* Focus: hovering or choosing something fades what it does not touch. */
.fm-svg.focus .fm-link:not(.hot) { opacity: .08; }
.fm-svg.focus .fm-nodes > g:not(.hot) { opacity: .28; }
.fm-svg.focus .fm-link.hot { opacity: 1; stroke-width: 2.6; filter: url(#fm-glow); }
.fm-svg.focus .fm-particle { opacity: .35; }
.only-problems .fm-link:not(.degraded) { opacity: .06; }
.only-problems .fm-nodes > g:not(.has-bad):not(.down):not(.errs) { opacity: .2; }
.fm-tip { position: absolute; z-index: 3; max-width: 260px; pointer-events: none; font-size: .78rem; line-height: 1.45;
padding: .5rem .65rem; border-radius: 8px; background: rgba(12, 18, 40, .94); border: 1px solid #2c3b6b;
box-shadow: 0 10px 30px rgba(0,0,0,.5); color: var(--fm-fg); }
.fm-panel { position: absolute; top: .75rem; right: .75rem; z-index: 2; width: 17rem; max-height: calc(100% - 1.5rem); overflow: auto;
padding: .85rem 1rem; border-radius: 12px; background: rgba(12, 18, 40, .92); border: 1px solid #2c3b6b;
box-shadow: 0 16px 40px rgba(0,0,0,.55); backdrop-filter: blur(8px); font-size: .82rem; }
.fm-panel h4 { margin: 0 0 .5rem; font-size: 1rem; color: #fff; }
.fm-panel dl { display: grid; grid-template-columns: auto 1fr; gap: .2rem .75rem; margin: 0 0 .6rem; }
.fm-panel dt { color: var(--fm-dim); } .fm-panel dd { margin: 0; overflow-wrap: anywhere; }
.fm-panel code { background: none; color: var(--fm-fg); font-size: .75rem; }
.fm-panel a { color: #7dd3fc; }
.fm-panel ul { margin: .25rem 0; padding-left: 1rem; }
.fm-close { position: absolute; top: .4rem; right: .5rem; background: none; border: 0; color: var(--fm-dim); font-size: 1.2rem; cursor: pointer; }
.fm-bad { color: var(--fm-bad); } .fm-dim { color: var(--fm-dim); }
.fm-legend { display: flex; flex-wrap: wrap; gap: .4rem 1.1rem; padding: .6rem 1rem .8rem; font-size: .74rem; color: var(--fm-dim);
border-top: 1px solid var(--fm-line); }
.fm-legend span { display: inline-flex; align-items: center; gap: .35rem; }
.fm-legend .sw { width: 18px; height: 3px; border-radius: 2px; display: inline-block; }
.fm-legend .sw.sp-xdr { background: var(--fm-xdr); } .fm-legend .sw.sp-ndr { background: var(--fm-ndr); }
.fm-legend .sw.sp-hdr { background: var(--fm-hdr); } .fm-legend .sw.sp-edr { background: var(--fm-edr); }
.fm-legend .sw.sp-slow { background: var(--fm-slow); }
.fm-legend .sw.degraded { background: repeating-linear-gradient(90deg, var(--fm-bad) 0 4px, transparent 4px 7px); }
.fm-legend .dot { width: 9px; height: 9px; border-radius: 50%; display: inline-block; background: #334066; border: 1px solid #4b5a88; }
.fm-legend .dot.known { background: #2dd4bf; border-color: #99f6e4; } .fm-legend .dot.down { background: var(--fm-bad); }
.fm-legend .note { color: var(--fm-dim); opacity: .8; }
@media (prefers-reduced-motion: reduce) {
.fm-link.degraded, .fm-host .fm-halo { animation: none; }
}
.mode-rails .fm-host.down circle:not(.fm-halo) { fill: var(--fm-bad); stroke: #fecdd3; }
.fm-svg.focus .fm-warn { opacity: .25; }
+447
View File
@@ -0,0 +1,447 @@
// The fabric map: switches in their tiers, adapters under their leaves, every
// cable between them, and the cluster's own adapters live.
//
// Plain SVG and no library, like every other chart here. The layout is the
// shape a fat tree already has — cores over spines over leaves over hosts —
// so it is computed rather than simulated, and it is the same every time the
// page is opened: a map that rearranges itself on each refresh is one nobody
// can learn.
(function () {
"use strict";
const root = document.getElementById("fabric-map");
if (!root) return;
const seed = document.getElementById("fabric-data");
let data = seed ? JSON.parse(seed.textContent) : null;
const url = root.dataset.src;
const svgNS = "http://www.w3.org/2000/svg";
const reduced = window.matchMedia && window.matchMedia("(prefers-reduced-motion: reduce)").matches;
const state = { mode: location.hash === "#rails" ? "rails" : "speed", traffic: !reduced, problems: false, query: "", selected: null,
view: { x: 0, y: 0, k: 1 } };
const canvas = root.querySelector(".fm-canvas");
const tip = root.querySelector(".fm-tip");
const panel = root.querySelector(".fm-panel");
const stats = root.querySelector(".fm-stats");
// --- colours ---------------------------------------------------------
const speedClass = (g) => g >= 800 ? "xdr" : g >= 400 ? "ndr" : g >= 200 ? "hdr" : g >= 100 ? "edr" : "slow";
const railPalette = ["#38bdf8", "#a78bfa", "#34d399", "#fbbf24", "#f472b6", "#fb7185", "#2dd4bf", "#c084fc"];
const fmtBps = (b) => {
const bits = b * 8;
if (bits >= 1e9) return (bits / 1e9).toFixed(1) + " Gb/s";
if (bits >= 1e6) return (bits / 1e6).toFixed(1) + " Mb/s";
if (bits >= 1e3) return (bits / 1e3).toFixed(0) + " kb/s";
return bits.toFixed(0) + " b/s";
};
const el = (name, attrs, parent) => {
const e = document.createElementNS(svgNS, name);
for (const k in attrs) e.setAttribute(k, attrs[k]);
if (parent) parent.appendChild(e);
return e;
};
const esc = (s) => String(s == null ? "" : s).replace(/[&<>"]/g, (c) => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;" }[c]));
// --- layout ----------------------------------------------------------
function layout(map, width) {
const tiers = Math.max(1, map.tiers || 1);
// Rows are placed once the width is known; see below.
let gap = 150;
const rowY = (t) => 70 + (tiers - t) * gap;
const pos = {};
const byGUID = {};
map.switches.forEach((s) => (byGUID[s.guid] = s));
const leaves = map.switches.filter((s) => s.tier === 1).sort((a, b) => a.name.localeCompare(b.name));
// Hosts under their leaves, leaf by leaf, a little gap between groups.
const groups = new Map(leaves.map((l) => [l.guid, []]));
const loose = [];
map.hosts.forEach((h) => (groups.has(h.leaf) ? groups.get(h.leaf) : loose).push(h));
const hostGap = map.hosts.length > 120 ? 16 : map.hosts.length > 48 ? 24 : 34;
// The width decides the height: the drawing is fitted to the page, and a
// fabric four times wider than the page would be a thin strip at the
// height of a small one.
const hostCount = map.hosts.length;
const estWidth = Math.max(width, 60 + hostCount * hostGap + leaves.length * 90);
gap = Math.max(130, (estWidth * 0.42 - 180) / tiers);
const hostY = rowY(1) + gap;
// Room on the left for the tier names, at the size they will be drawn.
let x = 40 + 30 * Math.max(1, estWidth / width);
const leafX = {};
for (const l of leaves) {
const hs = groups.get(l.guid);
if (!hs.length) { leafX[l.guid] = x + 50; x += 130; continue; }
const start = x;
hs.forEach((h) => { pos[h.guid] = { x, y: hostY }; x += hostGap; });
leafX[l.guid] = (start + x - hostGap) / 2;
x += Math.max(40, 130 - hs.length * hostGap / 2);
}
loose.forEach((h) => { pos[h.guid] = { x, y: hostY }; x += hostGap; });
const span = Math.max(x + 20, width);
// Leaves too close for their boxes are spread, keeping their order.
let last = -Infinity;
for (const l of leaves) {
const want = Math.max(leafX[l.guid], last + 124);
pos[l.guid] = { x: want, y: rowY(1) };
last = want;
}
// Each tier above: ordered by where the switches they connect to sit,
// which keeps cables from crossing more than they must, then spread
// evenly across the width.
const adj = {};
map.links.forEach((k) => {
(adj[k.from] = adj[k.from] || []).push(k.to);
(adj[k.to] = adj[k.to] || []).push(k.from);
});
for (let t = 2; t <= tiers; t++) {
const row = map.switches.filter((s) => s.tier === t);
row.forEach((s) => {
const xs = (adj[s.guid] || []).map((g) => pos[g]).filter((p) => p && p.y > rowY(t)).map((p) => p.x);
s._bx = xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : span / 2;
});
row.sort((a, b) => a._bx - b._bx || a.name.localeCompare(b.name));
const lo = Math.min(...leaves.map((l) => pos[l.guid].x), span * 0.15);
const hi = Math.max(...leaves.map((l) => pos[l.guid].x), span * 0.85);
row.forEach((s, i) => {
pos[s.guid] = { x: row.length === 1 ? (lo + hi) / 2 : lo + (hi - lo) * i / (row.length - 1), y: rowY(t) };
});
}
map.switches.forEach((s) => { if (!pos[s.guid]) pos[s.guid] = { x: span / 2, y: rowY(s.tier || 1) }; });
// How much to enlarge switches and adapters so they read at the size
// they would on a page as wide as the fabric — as far as their spacing
// lets them without touching.
const fit = Math.max(1, Math.max(span, width) / width);
let leafGap = Infinity;
for (let i = 1; i < leaves.length; i++) leafGap = Math.min(leafGap, pos[leaves[i].guid].x - pos[leaves[i - 1].guid].x);
const swScale = Math.min(fit, leafGap / 118, 2.4);
const hostScale = Math.min(fit, hostGap / 13, 2.4);
return { pos, width: Math.max(span, width), height: hostY + 110 * Math.max(1, hostScale), rowY, tiers, byGUID,
gap, swScale: Math.max(1, swScale), hostScale: Math.max(1, hostScale), fit };
}
// --- drawing ---------------------------------------------------------
let svg, viewport, flowScale = 1;
function draw() {
canvas.textContent = "";
if (!data || !data.map) {
const msg = data && data.error ? "The map could not be made: " + data.error
: "Not mapped yet: the first scan runs within five minutes of a node on this fabric connecting.";
canvas.innerHTML = '<p class="fm-empty">' + esc(msg) + "</p>";
fillStats();
return;
}
const map = data.map;
const width = canvas.clientWidth || 1000;
const L = layout(map, width);
// Fitted to the width to start with — the whole fabric at once — and
// zoomed from there for detail.
svg = el("svg", { class: "fm-svg mode-" + state.mode + (state.problems ? " only-problems" : ""),
viewBox: `0 0 ${L.width} ${L.height}`, preserveAspectRatio: "xMidYMin meet" });
svg.style.height = Math.max(380, Math.min(780, L.height * width / L.width)) + "px";
const defs = el("defs", {}, svg);
defs.innerHTML = `
<filter id="fm-glow" x="-50%" y="-50%" width="200%" height="200%">
<feGaussianBlur stdDeviation="3.2" result="b"/><feMerge><feMergeNode in="b"/><feMergeNode in="SourceGraphic"/></feMerge>
</filter>
<linearGradient id="fm-g-leaf" x1="0" y1="0" x2="0" y2="1"><stop offset="0" stop-color="#2dd4bf" stop-opacity=".30"/><stop offset="1" stop-color="#0f766e" stop-opacity=".18"/></linearGradient>
<linearGradient id="fm-g-spine" x1="0" y1="0" x2="0" y2="1"><stop offset="0" stop-color="#a78bfa" stop-opacity=".34"/><stop offset="1" stop-color="#5b21b6" stop-opacity=".18"/></linearGradient>
<linearGradient id="fm-g-core" x1="0" y1="0" x2="0" y2="1"><stop offset="0" stop-color="#fbbf24" stop-opacity=".34"/><stop offset="1" stop-color="#92400e" stop-opacity=".18"/></linearGradient>
<pattern id="fm-grid" width="28" height="28" patternUnits="userSpaceOnUse"><path d="M28 0H0V28" fill="none" class="fm-gridline"/></pattern>`;
el("rect", { x: -5000, y: -5000, width: 20000, height: 20000, fill: "url(#fm-grid)" }, svg);
viewport = el("g", { class: "fm-viewport" }, svg);
// Tier labels down the left.
const names = { 1: "leaf", 2: "spine", 3: "core", 4: "super-core" };
// Tier names down the left edge, turned on their side so they take no
// room from the leftmost switch.
const label = (y, text) => el("text", { class: "fm-tier",
transform: `translate(${14 * L.fit},${y}) rotate(-90) scale(${L.fit})` }, viewport).textContent = text;
for (let t = L.tiers; t >= 1; t--) label(L.rowY(t), names[t] || "tier " + t);
label(L.rowY(1) + L.gap, "adapters");
// Cables, parallel ones bundled.
const bundles = new Map();
for (const k of map.links) {
const key = k.from + "|" + k.to;
const b = bundles.get(key) || { from: k.from, to: k.to, links: [] };
b.links.push(k);
bundles.set(key, b);
}
const linkLayer = el("g", { class: "fm-links" }, viewport);
const flowLayer = el("g", { class: "fm-flows" }, viewport);
flowScale = L.hostScale;
const touches = {};
for (const b of bundles.values()) {
const a = L.pos[b.from], z = L.pos[b.to];
if (!a || !z) continue;
const my = (a.y + z.y) / 2;
const d = `M${a.x},${a.y + 17 * L.swScale} C${a.x},${my} ${z.x},${my} ${z.x},${z.y - (b.links[0].host ? 8 * L.hostScale : 17 * L.swScale)}`;
const bad = b.links.some((k) => k.degraded);
const g = Math.min(...b.links.map((k) => k.gbps || 0));
const host = b.links[0].host;
const hostInfo = host ? map.hosts.find((h) => h.guid === b.to) : null;
const path = el("path", { d, class: "fm-link sp-" + speedClass(g) + (bad ? " degraded" : "") + (host ? " host" : ""),
"stroke-width": host ? 1.4 : 1.2 + Math.log2(b.links.length) * 1.4 }, linkLayer);
if (host && hostInfo) path.style.setProperty("--rail", railPalette[hostInfo.rail % railPalette.length]);
path.dataset.ends = b.from + " " + b.to;
path.addEventListener("mouseenter", (e) => showTip(e, linkTip(b, map)));
path.addEventListener("mouseleave", hideTip);
(touches[b.from] = touches[b.from] || []).push(path);
(touches[b.to] = touches[b.to] || []).push(path);
if (bad) {
(touches[b.from].bad = (touches[b.from].bad || 0) + 1);
const mid = pointAt(a, z, my);
const w = el("g", { class: "fm-warn", transform: `translate(${mid.x},${mid.y}) scale(${L.swScale})` }, viewport);
el("circle", { r: 7 }, w);
el("text", { y: 3.5 }, w).textContent = "!";
w.addEventListener("mouseenter", (e) => showTip(e, linkTip(b, map)));
w.addEventListener("mouseleave", hideTip);
}
// Traffic: particles that run as fast as the adapter is busy — up
// the cable for what it sends, down for what it receives.
if (host && state.traffic && data.live && data.live[b.to]) {
const live = data.live[b.to];
flow(flowLayer, d, live.tx_bps, "up", live.gbps);
flow(flowLayer, d, live.rx_bps, "down", live.gbps);
}
}
// Switches.
const nodeLayer = el("g", { class: "fm-nodes" }, viewport);
for (const s of map.switches) {
const p = L.pos[s.guid];
const tierName = s.tier >= 3 ? "core" : s.tier === 2 ? "spine" : "leaf";
const g = el("g", { class: "fm-switch t-" + tierName + (touches[s.guid] && touches[s.guid].bad ? " has-bad" : ""),
transform: `translate(${p.x},${p.y}) scale(${L.swScale})`, tabindex: 0 }, nodeLayer);
el("rect", { x: -56, y: -17, width: 112, height: 34, rx: 10, fill: `url(#fm-g-${tierName})` }, g);
el("text", { class: "fm-name", y: -1 }, g).textContent = s.name.length > 16 ? s.name.slice(0, 15) + "…" : s.name;
el("text", { class: "fm-sub", y: 11 }, g).textContent = `${s.used}/${s.ports} ports` + (s.hosts ? ` · ${s.hosts} hosts` : "");
// Port use as an arc along the bottom edge.
const frac = s.ports ? Math.min(1, s.used / s.ports) : 0;
el("line", { x1: -46, y1: 15.5, x2: -46 + 92 * frac, y2: 15.5, class: "fm-use" }, g);
if (touches[s.guid] && touches[s.guid].bad) {
const bdg = el("g", { class: "fm-badge", transform: "translate(52,-15)" }, g);
el("circle", { r: 8 }, bdg);
el("text", { y: 3.5 }, bdg).textContent = touches[s.guid].bad;
}
wire(g, s.guid, () => switchTip(s, map, touches), () => switchPanel(s, map));
}
// Adapters.
const showLabels = map.hosts.length <= 96;
for (const h of map.hosts) {
const p = L.pos[h.guid];
if (!p) continue;
const live = data.live && data.live[h.guid];
const down = live && live.state && live.state.toUpperCase() !== "ACTIVE";
const errs = live && live.errors && Object.keys(live.errors).length;
const cls = "fm-host" + (h.node ? " known" : "") + (down ? " down" : "") + (errs ? " errs" : "") +
(state.query && matches(h) ? " found" : "");
const g = el("g", { class: cls, transform: `translate(${p.x},${p.y}) scale(${L.hostScale})`, tabindex: 0 }, nodeLayer);
g.style.setProperty("--rail", railPalette[h.rail % railPalette.length]);
el("circle", { r: 7.5, class: "fm-halo" }, g);
el("circle", { r: 5.5 }, g);
if (showLabels) {
const label = (h.node || (h.desc || "").split(" ")[0] || h.guid.slice(-6)) + (h.hca ? " " + h.hca.replace("mlx5_", "#") : "");
el("text", { class: "fm-hlabel", transform: "translate(3,14) rotate(55)" }, g).textContent = label;
}
wire(g, h.guid, () => hostTip(h, live), () => hostPanel(h, live, map));
}
canvas.appendChild(svg);
applyView();
enablePanZoom(svg, L);
if (state.selected) highlight(state.selected, touches);
svg._touches = touches;
fillStats();
}
function pointAt(a, z, my) {
// The middle of the cubic: its two control points share a height.
return { x: (a.x + z.x) / 2, y: my };
}
function flow(layer, d, bps, dir, gbps) {
if (!bps || bps < 1e3) return;
const cap = (gbps || 100) * 1e9 / 8;
const load = Math.min(1, bps / cap);
const count = 1 + Math.round(load * 4);
const dur = Math.max(0.7, 4.5 - Math.log10(bps) * 0.38);
for (let i = 0; i < count; i++) {
const c = el("circle", { r: 2.4 * flowScale, class: "fm-particle " + dir }, layer);
const m = el("animateMotion", { dur: dur + "s", repeatCount: "indefinite", path: d,
begin: (-(dur / count) * i).toFixed(2) + "s" }, c);
if (dir === "up") { m.setAttribute("keyPoints", "1;0"); m.setAttribute("keyTimes", "0;1"); m.setAttribute("calcMode", "linear"); }
}
}
// --- interaction ------------------------------------------------------
function wire(g, guid, tipFn, panelFn) {
g._guid = guid;
g.addEventListener("mouseenter", (e) => { highlight(guid, svg._touches); showTip(e, tipFn()); });
g.addEventListener("mousemove", moveTip);
g.addEventListener("mouseleave", () => { hideTip(); if (!state.selected) unhighlight(); else highlight(state.selected, svg._touches); });
const open = (e) => { e.stopPropagation(); state.selected = guid; highlight(guid, svg._touches); panel.innerHTML = panelFn(); panel.hidden = false; };
g.addEventListener("click", open);
g.addEventListener("keydown", (e) => { if (e.key === "Enter") open(e); });
}
function highlight(guid, touches) {
if (!svg) return;
svg.classList.add("focus");
svg.querySelectorAll(".hot").forEach((n) => n.classList.remove("hot"));
(touches[guid] || []).forEach((p) => {
p.classList.add("hot");
p.dataset.ends.split(" ").forEach((g) => {
svg.querySelectorAll(".fm-nodes > g").forEach((n) => { if (n._guid === g) n.classList.add("hot"); });
});
});
svg.querySelectorAll(".fm-nodes > g").forEach((n) => { if (n._guid === guid) n.classList.add("hot"); });
}
function unhighlight() {
if (!svg) return;
svg.classList.remove("focus");
svg.querySelectorAll(".hot").forEach((n) => n.classList.remove("hot"));
}
function showTip(e, html) { tip.innerHTML = html; tip.hidden = false; moveTip(e); }
function moveTip(e) {
const r = root.getBoundingClientRect();
tip.style.left = Math.min(e.clientX - r.left + 14, r.width - 260) + "px";
tip.style.top = (e.clientY - r.top + 14) + "px";
}
function hideTip() { tip.hidden = true; }
function linkTip(b, map) {
const n = (g) => { const s = map.switches.find((x) => x.guid === g); const h = map.hosts.find((x) => x.guid === g);
return s ? s.name : h ? (h.node ? h.node + " " + (h.hca || "") : h.desc) : g; };
const k = b.links[0];
const exp = k.host ? map.expected_host_gbps : map.expected_switch_gbps;
return `<b>${esc(n(b.from))} ↔ ${esc(n(b.to))}</b><br>${b.links.length > 1 ? b.links.length + " cables, " : ""}${esc(k.rate)} · ${k.gbps} Gb/s` +
(b.links.some((x) => x.degraded) ? `<br><span class="fm-bad">trained below the ${exp} Gb/s the rest of its kind run at</span>` : "");
}
function switchTip(s, map, touches) {
const bad = touches[s.guid] && touches[s.guid].bad;
return `<b>${esc(s.name)}</b><br>${s.tier >= 3 ? "core" : s.tier === 2 ? "spine" : "leaf"} · ${s.used}/${s.ports} ports` +
(s.hosts ? ` · ${s.hosts} adapters` : "") + (s.lid ? ` · LID ${s.lid}` : "") + (bad ? `<br><span class="fm-bad">${bad} degraded cable(s)</span>` : "");
}
function hostTip(h, live) {
let t = `<b>${esc(h.node || h.desc || h.guid)}</b>${h.hca ? " · " + esc(h.hca) : ""}<br>rail ${h.rail}`;
if (live) {
t += ` · ${esc(live.state)}${live.gbps ? " · " + live.gbps + " Gb/s" : ""}`;
if (live.tx_bps || live.rx_bps) t += `<br>↑ ${fmtBps(live.tx_bps)} · ↓ ${fmtBps(live.rx_bps)}`;
if (live.errors) t += `<br><span class="fm-bad">${Object.entries(live.errors).map(([k, v]) => esc(k) + " " + v).join(", ")}</span>`;
} else if (!h.node) {
t += "<br><span class=\"fm-dim\">not one of this cluster's nodes</span>";
}
return t;
}
function switchPanel(s, map) {
const cables = map.links.filter((k) => k.from === s.guid || k.to === s.guid);
const bad = cables.filter((k) => k.degraded);
const rows = bad.map((k) => { const o = k.from === s.guid ? k.to : k.from; const on = map.switches.find((x) => x.guid === o) || map.hosts.find((x) => x.guid === o);
return `<li>port ${k.from === s.guid ? k.from_port : k.to_port} → ${esc(on ? (on.name || on.node || on.desc) : o)}: ${esc(k.rate)}</li>`; }).join("");
return `<button class="fm-close" aria-label="close">×</button><h4>${esc(s.name)}</h4>
<dl><dt>tier</dt><dd>${s.tier >= 3 ? "core" : s.tier === 2 ? "spine" : "leaf"}</dd><dt>GUID</dt><dd><code>${esc(s.guid)}</code></dd>
<dt>LID</dt><dd>${s.lid || "—"}</dd><dt>ports</dt><dd>${s.used} of ${s.ports} cabled</dd><dt>adapters</dt><dd>${s.hosts}</dd></dl>` +
(bad.length ? `<p class="fm-bad">Degraded cables</p><ul>${rows}</ul>` : "<p class=\"fm-dim\">Every cable at its expected rate.</p>");
}
function hostPanel(h, live, map) {
const leaf = map.switches.find((x) => x.guid === h.leaf);
let body = `<dl><dt>adapter</dt><dd>${esc(h.hca || h.desc)}</dd><dt>GUID</dt><dd><code>${esc(h.guid)}</code></dd>
<dt>rail</dt><dd>${h.rail}</dd><dt>leaf</dt><dd>${esc(leaf ? leaf.name : "—")}</dd>`;
if (live) {
body += `<dt>state</dt><dd>${esc(live.state)} (${esc(live.phys_state)})</dd><dt>rate</dt><dd>${live.gbps} Gb/s</dd>
<dt>sending</dt><dd>${fmtBps(live.tx_bps)}</dd><dt>receiving</dt><dd>${fmtBps(live.rx_bps)}</dd>`;
}
body += "</dl>";
if (live && live.errors) body += `<p class="fm-bad">${Object.entries(live.errors).map(([k, v]) => esc(k) + " " + v).join("<br>")}</p>`;
const link = h.node ? `<a href="${esc(root.dataset.nodes + "/" + encodeURIComponent(h.node))}">open ${esc(h.node)} →</a>` : "";
return `<button class="fm-close" aria-label="close">×</button><h4>${esc(h.node || h.desc || "adapter")}</h4>${body}${link}`;
}
panel.addEventListener("click", (e) => {
if (e.target.classList.contains("fm-close")) { panel.hidden = true; state.selected = null; unhighlight(); }
});
canvas.addEventListener("click", () => { panel.hidden = true; state.selected = null; unhighlight(); });
function matches(h) {
const q = state.query.toLowerCase();
return (h.node || "").toLowerCase().includes(q) || (h.desc || "").toLowerCase().includes(q) || (h.hca || "").toLowerCase().includes(q);
}
// --- pan and zoom ------------------------------------------------------
function applyView() {
if (viewport) viewport.setAttribute("transform", `translate(${state.view.x},${state.view.y}) scale(${state.view.k})`);
}
function enablePanZoom(s) {
let drag = null;
s.addEventListener("wheel", (e) => {
e.preventDefault();
const r = s.getBoundingClientRect();
const scale = s.viewBox.baseVal.width / r.width;
const mx = (e.clientX - r.left) * scale, my = (e.clientY - r.top) * scale;
const k = Math.min(4, Math.max(0.25, state.view.k * (e.deltaY < 0 ? 1.12 : 1 / 1.12)));
state.view.x = mx - (mx - state.view.x) * (k / state.view.k);
state.view.y = my - (my - state.view.y) * (k / state.view.k);
state.view.k = k;
applyView();
}, { passive: false });
s.addEventListener("pointerdown", (e) => { if (e.target.closest(".fm-nodes > g")) return; drag = { x: e.clientX, y: e.clientY, vx: state.view.x, vy: state.view.y }; s.setPointerCapture(e.pointerId); s.classList.add("dragging"); });
s.addEventListener("pointermove", (e) => {
if (!drag) return;
const r = s.getBoundingClientRect();
const scale = s.viewBox.baseVal.width / r.width;
state.view.x = drag.vx + (e.clientX - drag.x) * scale;
state.view.y = drag.vy + (e.clientY - drag.y) * scale;
applyView();
});
s.addEventListener("pointerup", () => { drag = null; s.classList.remove("dragging"); });
s.addEventListener("dblclick", () => { state.view = { x: 0, y: 0, k: 1 }; applyView(); });
}
// --- the strip above ----------------------------------------------------
function fillStats() {
if (!stats) return;
const m = data && data.map;
const live = data && data.live ? Object.values(data.live) : [];
const tx = live.reduce((a, l) => a + (l.tx_bps || 0), 0);
const cells = [];
if (m) {
cells.push([m.switches.length, "switches"], [m.hosts.length, "adapters"], [m.links.length, "cables"],
[m.degraded, "degraded", m.degraded ? "bad" : "ok"],
[m.rails.length > 1 ? (m.rail_aligned ? "aligned" : "crossed") : "—", "rails", m.rails.length > 1 && !m.rail_aligned ? "bad" : ""]);
}
cells.push([fmtBps(tx), "sent now"]);
stats.innerHTML = cells.map(([v, k, c]) => `<div class="fm-stat ${c || ""}"><b>${esc(v)}</b><span>${esc(k)}</span></div>`).join("");
}
// --- toolbar -------------------------------------------------------------
root.querySelectorAll("[data-mode]").forEach((b) => b.classList.toggle("on", b.dataset.mode === state.mode));
root.querySelectorAll("[data-mode]").forEach((b) => b.addEventListener("click", () => {
state.mode = b.dataset.mode;
history.replaceState(null, "", state.mode === "rails" ? "#rails" : location.pathname);
root.querySelectorAll("[data-mode]").forEach((x) => x.classList.toggle("on", x === b));
draw();
}));
const tbtn = root.querySelector("[data-toggle=traffic]");
if (tbtn) { tbtn.classList.toggle("on", state.traffic); tbtn.addEventListener("click", () => { state.traffic = !state.traffic; tbtn.classList.toggle("on", state.traffic); draw(); }); }
const pbtn = root.querySelector("[data-toggle=problems]");
if (pbtn) pbtn.addEventListener("click", () => { state.problems = !state.problems; pbtn.classList.toggle("on", state.problems); draw(); });
const search = root.querySelector(".fm-search");
if (search) search.addEventListener("input", () => { state.query = search.value.trim(); draw(); });
const reset = root.querySelector("[data-reset]");
if (reset) reset.addEventListener("click", () => { state.view = { x: 0, y: 0, k: 1 }; applyView(); });
draw();
window.addEventListener("resize", () => { clearTimeout(draw._t); draw._t = setTimeout(draw, 150); });
// Live: adapters' state and traffic every fifteen seconds, the map itself
// when a new scan has made one.
if (url) setInterval(async () => {
if (document.visibilityState !== "visible") return;
try {
const r = await fetch(url, { headers: { Accept: "application/json" } });
if (!r.ok) return;
data = await r.json();
draw();
} catch (_) { /* the next one */ }
}, 15000);
})();
+1
View File
@@ -147,6 +147,7 @@
</tr>{{end}}</table>
{{end}}
{{with .Fabrics}}{{if .Physical}}
<p><a href="{{if $.OnAdminListener}}/fabrics{{else}}/admin/fabrics{{end}}">See each fabric's map →</a></p>
<h3>Subnet managers</h3>
<table><tr><th>Fabric</th><th>Is physically</th><th>Master</th><th>Standby</th><th class="num">Nodes</th><th>Last change of master</th></tr>
{{range .Physical}}<tr>
+69
View File
@@ -0,0 +1,69 @@
{{define "fabrics"}}<!doctype html>
<html lang="en"><head>
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<title>Fabrics · llmgw</title><link rel="stylesheet" href="{{stylesheet}}">
</head><body><div class="wrap wide">
<header>
<h1>Fabrics</h1>
<p class="sub">{{len .List}} physical fabric{{if ne (len .List) 1}}s{{end}}, as their adapters find them</p>
{{if .OnAdminListener}}<nav><a href="/">cluster</a></nav>{{else}}{{template "nav" .Nav}}{{end}}
</header>
{{if not .List}}
<p class="empty">No InfiniBand or RoCE port has reported yet. A node's adapters appear here with its first
heartbeat, and its fabric's map within five minutes of that.</p>
{{else}}
<nav class="pills" aria-label="Fabric">{{range .List}}<a href="{{$.Base}}/{{.ID}}"{{if $.Current}}{{if eq .ID $.Current.ID}} class="on" aria-current="page"{{end}}{{end}}>
{{if .Declared}}{{range $i, $d := .Declared}}{{if $i}}, {{end}}{{$d}}{{end}}{{else}}undeclared{{end}}
<span class="note">{{.Label}}</span></a>{{end}}</nav>
{{end}}
{{with .Current}}
<div class="fabric-head">
<div>
<h2>{{if .Declared}}{{range $i, $d := .Declared}}{{if $i}}, {{end}}<code>{{$d}}</code>{{end}}{{else}}undeclared{{end}}</h2>
<p class="sub">{{.Label}}
{{with .SM}} · master <code>{{.GUID}}</code>{{if .LID}} LID {{.LID}}{{end}}{{end}}
{{if .SMs}} · {{len .SMs}} subnet manager{{if gt (len .SMs) 1}}s{{end}}{{end}}
{{if not .ScannedAt.IsZero}} · mapped {{ago .ScannedAt}} through {{.ScannedBy}} in {{.TookMs}} ms{{end}}</p>
</div>
</div>
{{if .Problems}}<ul class="fabric-problems">{{range .Problems}}
<li class="{{.Level}}"><span class="tag{{if eq (printf "%s" .Level) "fail"}} bad{{end}}">{{.Level}}</span> {{.Message}}</li>{{end}}
</ul>{{end}}
<div id="fabric-map" class="fabric-map" data-src="{{$.Base}}/{{.ID}}/topology.json"
data-nodes="{{if $.OnAdminListener}}/nodes{{else}}/admin/nodes{{end}}">
<div class="fm-bar">
<div class="fm-stats"></div>
<div class="fm-tools">
<span class="pills small">
<button type="button" class="on" data-mode="speed">Speed</button>
<button type="button" data-mode="rails">Rails</button>
</span>
<button type="button" class="fm-toggle" data-toggle="traffic">Traffic</button>
<button type="button" class="fm-toggle" data-toggle="problems">Only problems</button>
<input type="search" class="fm-search" placeholder="Find a node" aria-label="Find a node">
<button type="button" class="fm-toggle" data-reset title="Or double-click the map">Reset view</button>
</div>
</div>
<div class="fm-stage">
<div class="fm-canvas"><noscript><p class="fm-empty">The map is drawn in the browser; the problems above are the same
without it.</p></noscript></div>
<div class="fm-tip" hidden></div>
<aside class="fm-panel" hidden></aside>
</div>
<div class="fm-legend">
<span><i class="sw sp-xdr"></i>XDR</span><span><i class="sw sp-ndr"></i>NDR</span><span><i class="sw sp-hdr"></i>HDR</span>
<span><i class="sw sp-edr"></i>EDR</span><span><i class="sw sp-slow"></i>slower</span>
<span><i class="sw degraded"></i>below its kind</span>
<span><i class="dot known"></i>this cluster's adapter</span><span><i class="dot"></i>another host</span><span><i class="dot down"></i>port down</span>
<span class="note">scroll to zoom · drag to move · click for detail</span>
</div>
</div>
<script type="application/json" id="fabric-data">{{$.Data}}</script>
<script src="{{asset "fabric.js"}}" defer></script>
{{end}}
<footer><span>llmgw {{.Version}}</span><span>{{.Now}}</span></footer>
</div></body></html>{{end}}
+1 -1
View File
@@ -33,7 +33,7 @@
{{if or .Node.IBPorts .Node.Fabrics}}
<h2>Fabric</h2>
<p class="sub">{{if .Node.Fabrics}}On {{join .Node.Fabrics}}.{{else}}<span class="bad">On no fabric: no port a declared fabric names can carry traffic, so no multi-node replica is placed here.</span>{{end}}</p>
<p class="sub">{{if .Node.Fabrics}}On {{join .Node.Fabrics}} · <a href="{{if $.OnAdminListener}}/fabrics{{else}}/admin/fabrics{{end}}">the map →</a>{{else}}<span class="bad">On no fabric: no port a declared fabric names can carry traffic, so no multi-node replica is placed here.</span>{{end}}</p>
{{if .Node.IBPorts}}
<table><tr><th>Adapter</th><th>State</th><th class="num">Rate</th><th>Fabric</th><th>Physically on</th><th>Errors since boot</th><th class="num">Sent</th><th class="num">Received</th></tr>
{{range .Node.IBPorts}}<tr>
+26
View File
@@ -23,6 +23,7 @@ import (
"net/http"
"slices"
"strings"
"sync"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
@@ -51,8 +52,33 @@ var stylesheetURL = func() string {
return "/_static/base.css?v=" + hex.EncodeToString(sum[:4])
}()
// assetURL names any static file by its content, for the same reason as the
// stylesheet: cached for a day, a changed script under its old name reached
// people a day after the page that needed it.
func assetURL(name string) string {
assetMu.Lock()
defer assetMu.Unlock()
if u, ok := assetURLs[name]; ok {
return u
}
b, err := staticFiles.ReadFile("static/" + name)
if err != nil {
panic("web: the asset " + name + " is missing from the binary: " + err.Error())
}
sum := sha256.Sum256(b)
u := "/_static/" + name + "?v=" + hex.EncodeToString(sum[:4])
assetURLs[name] = u
return u
}
var (
assetMu sync.Mutex
assetURLs = map[string]string{}
)
var funcs = template.FuncMap{
"stylesheet": func() string { return stylesheetURL },
"asset": assetURL,
"lower": strings.ToLower,
"sub": func(a, b int) int { return a - b },
"join": func(v []string) string {