Watch InfiniBand ports while the agent runs, not only when it starts
A port's state was read once, at registration: one that went down later left its node on the fabric, offered multi-node work that would hang on its first collective, until somebody restarted the agent. The agent now re-reads every IB and RoCE port on each heartbeat - state, physical state, rate and the error and data counters - and when the fabrics it can carry traffic on change, says so once; the control plane replaces the node's fabrics, which is what placement reads. The ports are exported as llmgw_ib_port_active (with state and phys_state, so an unplugged cable reads differently from a missing subnet manager), llmgw_ib_port_rate_gbps and llmgw_ib_port_counter_total, and a node's page lists them with the fabric each carries and any error counter that is not zero. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
bf2d171e89
commit
93bd16f00e
15 files changed
+654
-107
No files matched your search
@@ -28,6 +28,11 @@ func (busySource) Nodes() []adminapi.NodeView {
|
||||
n := fakeSource{}.Nodes()[0]
|
||||
n.Load = &adminapi.LoadView{CPUPercent: 62, CPUs: 8, Load1: 3.25, Load5: 2, Load15: 1,
|
||||
MemUsed: 6 << 30, MemTotal: 8 << 30, MemPercent: 75}
|
||||
n.IBPorts = []adminapi.IBPortView{
|
||||
{HCA: "mlx5_0", Port: 1, State: "ACTIVE", PhysState: "LinkUp", RateGbps: 400, Active: true,
|
||||
Fabric: "ib-a", TxBytes: 4 << 30},
|
||||
{HCA: "mlx5_1", Port: 1, State: "DOWN", PhysState: "Polling", Errors: map[string]uint64{"link_downed": 7}},
|
||||
}
|
||||
return []adminapi.NodeView{n}
|
||||
}
|
||||
|
||||
@@ -84,3 +89,15 @@ func TestANodeNotMeasuredLatelySaysSoRatherThanZero(t *testing.T) {
|
||||
t.Error("without a metrics store the page should say where history comes from")
|
||||
}
|
||||
}
|
||||
|
||||
func TestANodesPageShowsItsFabricPorts(t *testing.T) {
|
||||
_, body := get(t, nodeMux(busySource{}, nil), "http://x/admin/nodes/llmgw-node01", nil)
|
||||
for _, want := range []string{
|
||||
"<h2>Fabric</h2>", "On ib-a.", "<code>mlx5_0/1</code>", "400 Gb/s", "4.0 GiB",
|
||||
`<td class="bad">DOWN <span class="note">Polling</span></td>`, "link_downed 7",
|
||||
} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("missing %q", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user