Watch InfiniBand ports while the agent runs, not only when it starts
A port's state was read once, at registration: one that went down later left its node on the fabric, offered multi-node work that would hang on its first collective, until somebody restarted the agent. The agent now re-reads every IB and RoCE port on each heartbeat - state, physical state, rate and the error and data counters - and when the fabrics it can carry traffic on change, says so once; the control plane replaces the node's fabrics, which is what placement reads. The ports are exported as llmgw_ib_port_active (with state and phys_state, so an unplugged cable reads differently from a missing subnet manager), llmgw_ib_port_rate_gbps and llmgw_ib_port_counter_total, and a node's page lists them with the fabric each carries and any error counter that is not zero. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
bf2d171e89
commit
93bd16f00e
15 files changed
+654
-107
No files matched your search
@@ -119,6 +119,25 @@ type NodeView struct {
|
||||
// Load is how busy the machine is, absent when its agent has not
|
||||
// measured it lately — which is unknown, not idle.
|
||||
Load *LoadView `json:"load,omitempty"`
|
||||
// IBPorts are the node's InfiniBand and RoCE ports at the last heartbeat.
|
||||
IBPorts []IBPortView `json:"ib_ports,omitempty"`
|
||||
}
|
||||
|
||||
// IBPortView is one port as a page shows it.
|
||||
type IBPortView struct {
|
||||
HCA string `json:"hca"`
|
||||
Port int `json:"port"`
|
||||
State string `json:"state"`
|
||||
PhysState string `json:"phys_state,omitempty"`
|
||||
RateGbps int `json:"rate_gbps"`
|
||||
LinkLayer string `json:"link_layer,omitempty"`
|
||||
Active bool `json:"active"`
|
||||
// Fabric is the declared fabric this port's adapter carries, if any.
|
||||
Fabric string `json:"fabric,omitempty"`
|
||||
// Errors are the error counters that are not zero, by name.
|
||||
Errors map[string]uint64 `json:"errors,omitempty"`
|
||||
TxBytes uint64 `json:"tx_bytes,omitempty"`
|
||||
RxBytes uint64 `json:"rx_bytes,omitempty"`
|
||||
}
|
||||
|
||||
// LoadView is a node's CPU and memory in use, at its last heartbeat.
|
||||
@@ -308,6 +327,26 @@ func (a *API) Nodes() []NodeView {
|
||||
v.LastSeenAgo = time.Since(n.LastSeen).Round(time.Second).String()
|
||||
}
|
||||
v.Load = loadView(n.Load, time.Now())
|
||||
fabricOf := map[string]string{}
|
||||
for _, f := range n.Fabrics {
|
||||
for _, h := range f.HCAs {
|
||||
fabricOf[h] = f.Fabric
|
||||
}
|
||||
}
|
||||
for _, p := range n.IBPorts {
|
||||
pv := IBPortView{HCA: p.HCA, Port: p.Port, State: p.State, PhysState: p.PhysState,
|
||||
RateGbps: p.RateGbps, LinkLayer: p.LinkLayer, Active: p.Active(), Fabric: fabricOf[p.HCA],
|
||||
TxBytes: p.Counters["port_xmit_data"], RxBytes: p.Counters["port_rcv_data"]}
|
||||
for name, c := range p.Counters {
|
||||
if c > 0 && name != "port_xmit_data" && name != "port_rcv_data" {
|
||||
if pv.Errors == nil {
|
||||
pv.Errors = map[string]uint64{}
|
||||
}
|
||||
pv.Errors[name] = c
|
||||
}
|
||||
}
|
||||
v.IBPorts = append(v.IBPorts, pv)
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out
|
||||
|
||||
Reference in new issue
Block a user