Watch InfiniBand ports while the agent runs, not only when it starts

A port's state was read once, at registration: one that went down later
left its node on the fabric, offered multi-node work that would hang on
its first collective, until somebody restarted the agent. The agent now
re-reads every IB and RoCE port on each heartbeat - state, physical
state, rate and the error and data counters - and when the fabrics it
can carry traffic on change, says so once; the control plane replaces
the node's fabrics, which is what placement reads.

The ports are exported as llmgw_ib_port_active (with state and
phys_state, so an unplugged cable reads differently from a missing
subnet manager), llmgw_ib_port_rate_gbps and llmgw_ib_port_counter_total,
and a node's page lists them with the fabric each carries and any error
counter that is not zero.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-29 00:13:54 +02:00
1 parent bf2d171e89
commit 93bd16f00e
15 files changed
+654 -107

No files matched your search

+39
View File
@@ -119,6 +119,25 @@ type NodeView struct {
// Load is how busy the machine is, absent when its agent has not
// measured it lately — which is unknown, not idle.
Load *LoadView `json:"load,omitempty"`
// IBPorts are the node's InfiniBand and RoCE ports at the last heartbeat.
IBPorts []IBPortView `json:"ib_ports,omitempty"`
}
// IBPortView is one port as a page shows it.
type IBPortView struct {
HCA string `json:"hca"`
Port int `json:"port"`
State string `json:"state"`
PhysState string `json:"phys_state,omitempty"`
RateGbps int `json:"rate_gbps"`
LinkLayer string `json:"link_layer,omitempty"`
Active bool `json:"active"`
// Fabric is the declared fabric this port's adapter carries, if any.
Fabric string `json:"fabric,omitempty"`
// Errors are the error counters that are not zero, by name.
Errors map[string]uint64 `json:"errors,omitempty"`
TxBytes uint64 `json:"tx_bytes,omitempty"`
RxBytes uint64 `json:"rx_bytes,omitempty"`
}
// LoadView is a node's CPU and memory in use, at its last heartbeat.
@@ -308,6 +327,26 @@ func (a *API) Nodes() []NodeView {
v.LastSeenAgo = time.Since(n.LastSeen).Round(time.Second).String()
}
v.Load = loadView(n.Load, time.Now())
fabricOf := map[string]string{}
for _, f := range n.Fabrics {
for _, h := range f.HCAs {
fabricOf[h] = f.Fabric
}
}
for _, p := range n.IBPorts {
pv := IBPortView{HCA: p.HCA, Port: p.Port, State: p.State, PhysState: p.PhysState,
RateGbps: p.RateGbps, LinkLayer: p.LinkLayer, Active: p.Active(), Fabric: fabricOf[p.HCA],
TxBytes: p.Counters["port_xmit_data"], RxBytes: p.Counters["port_rcv_data"]}
for name, c := range p.Counters {
if c > 0 && name != "port_xmit_data" && name != "port_rcv_data" {
if pv.Errors == nil {
pv.Errors = map[string]uint64{}
}
pv.Errors[name] = c
}
}
v.IBPorts = append(v.IBPorts, pv)
}
out = append(out, v)
}
return out