Watch each fabric's subnet managers, and keep a fabric one through a failover
A fabric's identity was its master subnet manager, which changes on a failover: for the minute its ports took to agree, one fabric read as two, and a name covering it as two separate fabrics. Each agent now also lists every subnet manager on a fabric (saquery SMIR, through one port per master, once a minute); managers listed together are one fabric, named by the lowest GUID among them, so the identity survives the master changing. The same list says what a fabric can survive. The cluster page's new subnet manager table, /api/v1/fabrics, doctor and metrics (llmgw_fabric_master_sm, llmgw_fabric_subnet_managers, llmgw_fabric_problems) show each fabric's master, priority and standbys, and say when a fabric has no standby, its ports disagree on the master, ports are up with no subnet manager (INIT), or the master changed in the last hour. Every change of master a port reports is kept, and a fabric's last day of them is listed. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
77fe801892
commit
23648fa19a
20 files changed
+857
-136
No files matched your search
@@ -4,8 +4,6 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"html/template"
|
||||
"net/http"
|
||||
"sort"
|
||||
@@ -13,6 +11,8 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/sourcecheck"
|
||||
|
||||
@@ -2,13 +2,13 @@ package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
|
||||
"html/template"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
|
||||
@@ -146,6 +146,18 @@
|
||||
<td>{{if .NodesWithWeights}}{{.Models}} model(s) on {{.NodesWithWeights}}/{{.Nodes}} nodes{{else}}<span class="note">not reported</span>{{end}}</td>
|
||||
</tr>{{end}}</table>
|
||||
{{end}}
|
||||
{{with .Fabrics}}{{if .Physical}}
|
||||
<h3>Subnet managers</h3>
|
||||
<table><tr><th>Fabric</th><th>Is physically</th><th>Master</th><th>Standby</th><th class="num">Nodes</th><th>Last change of master</th></tr>
|
||||
{{range .Physical}}<tr>
|
||||
<td>{{if .Declared}}{{range $i, $d := .Declared}}{{if $i}}, {{end}}<code>{{$d}}</code>{{end}}{{else}}<span class="note">undeclared</span>{{end}}</td>
|
||||
<td>{{.Label}}{{if .RateGbps}} <span class="note">· {{.RateGbps}} Gb/s</span>{{end}}</td>
|
||||
<td>{{if eq .Kind "roce"}}<span class="note">none: RoCE</span>{{else}}{{with .SM}}<code>{{.GUID}}</code>{{if .LID}} <span class="note">LID {{.LID}}, priority {{.Priority}}</span>{{end}}{{else}}—{{end}}{{end}}</td>
|
||||
<td>{{if eq .Kind "roce"}}—{{else if .SMs}}{{if .Standbys}}{{.Standbys}}{{else}}<span class="bad">none</span>{{end}}{{else}}<span class="note">not listed</span>{{end}}</td>
|
||||
<td class="num">{{len .Members}}</td>
|
||||
<td>{{with .Changes}}{{with index . 0}}{{ago .At}}: <code>{{.From}}</code> → <code>{{.To}}</code>{{end}}{{if gt (len .) 1}} <span class="note">({{len .}} in a day)</span>{{end}}{{else}}<span class="note">none in a day</span>{{end}}</td>
|
||||
</tr>{{end}}</table>
|
||||
{{end}}{{end}}
|
||||
<p class="note">A fabric's name is what the nodes were told; what it is physically is
|
||||
the subnet manager its InfiniBand ports see, or the network its RoCE ports are on.
|
||||
A replica spanning nodes is only ever formed on one of those. Fully free counts whole
|
||||
|
||||
@@ -657,14 +657,18 @@ type fabricSource struct{ fakeSource }
|
||||
func (fabricSource) Fabrics() fabrics.Report {
|
||||
return fabrics.Report{
|
||||
Physical: []fabrics.Physical{{Key: "sm:0x01", Kind: "infiniband", Label: "subnet manager 0x01 (LID 1)",
|
||||
Declared: []string{"ib-a"}, RateGbps: 400}},
|
||||
Declared: []string{"ib-a"}, RateGbps: 400,
|
||||
SM: &fabrics.SM{GUID: "0x01", LID: 1, Priority: 15, State: "master"},
|
||||
SMs: []fabrics.SM{{GUID: "0x01", State: "master"}},
|
||||
Changes: []fabrics.Change{{At: time.Now().Add(-3 * time.Hour), From: "0x02", To: "0x01", Ports: 4}}}},
|
||||
Problems: []fabrics.Problem{{Level: fabrics.Fail, Fabric: "ib-a", Message: "ib-a is 2 separate fabrics"}},
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheClusterPageSaysWhatEachFabricPhysicallyIs(t *testing.T) {
|
||||
_, body := get(t, Dashboard(fabricSource{}, fakeUsage{}, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/", nil)
|
||||
for _, want := range []string{"subnet manager 0x01 (LID 1)", "400 Gb/s", "ib-a is 2 separate fabrics", `<span class="tag bad">fail</span>`} {
|
||||
for _, want := range []string{"subnet manager 0x01 (LID 1)", "400 Gb/s", "ib-a is 2 separate fabrics", `<span class="tag bad">fail</span>`,
|
||||
"<h3>Subnet managers</h3>", "LID 1, priority 15", `<span class="bad">none</span>`, "3 hours ago: <code>0x02</code> → <code>0x01</code>"} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("missing %q", want)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user