Watch each fabric's subnet managers, and keep a fabric one through a failover
A fabric's identity was its master subnet manager, which changes on a failover: for the minute its ports took to agree, one fabric read as two, and a name covering it as two separate fabrics. Each agent now also lists every subnet manager on a fabric (saquery SMIR, through one port per master, once a minute); managers listed together are one fabric, named by the lowest GUID among them, so the identity survives the master changing. The same list says what a fabric can survive. The cluster page's new subnet manager table, /api/v1/fabrics, doctor and metrics (llmgw_fabric_master_sm, llmgw_fabric_subnet_managers, llmgw_fabric_problems) show each fabric's master, priority and standbys, and say when a fabric has no standby, its ports disagree on the master, ports are up with no subnet manager (INIT), or the master changed in the last hour. Every change of master a port reports is kept, and a fabric's last day of them is listed. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
77fe801892
commit
23648fa19a
20 files changed
+857
-136
No files matched your search
@@ -657,14 +657,18 @@ type fabricSource struct{ fakeSource }
|
||||
func (fabricSource) Fabrics() fabrics.Report {
|
||||
return fabrics.Report{
|
||||
Physical: []fabrics.Physical{{Key: "sm:0x01", Kind: "infiniband", Label: "subnet manager 0x01 (LID 1)",
|
||||
Declared: []string{"ib-a"}, RateGbps: 400}},
|
||||
Declared: []string{"ib-a"}, RateGbps: 400,
|
||||
SM: &fabrics.SM{GUID: "0x01", LID: 1, Priority: 15, State: "master"},
|
||||
SMs: []fabrics.SM{{GUID: "0x01", State: "master"}},
|
||||
Changes: []fabrics.Change{{At: time.Now().Add(-3 * time.Hour), From: "0x02", To: "0x01", Ports: 4}}}},
|
||||
Problems: []fabrics.Problem{{Level: fabrics.Fail, Fabric: "ib-a", Message: "ib-a is 2 separate fabrics"}},
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheClusterPageSaysWhatEachFabricPhysicallyIs(t *testing.T) {
|
||||
_, body := get(t, Dashboard(fabricSource{}, fakeUsage{}, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/", nil)
|
||||
for _, want := range []string{"subnet manager 0x01 (LID 1)", "400 Gb/s", "ib-a is 2 separate fabrics", `<span class="tag bad">fail</span>`} {
|
||||
for _, want := range []string{"subnet manager 0x01 (LID 1)", "400 Gb/s", "ib-a is 2 separate fabrics", `<span class="tag bad">fail</span>`,
|
||||
"<h3>Subnet managers</h3>", "LID 1, priority 15", `<span class="bad">none</span>`, "3 hours ago: <code>0x02</code> → <code>0x01</code>"} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("missing %q", want)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user